TheAnachronism/docspell
1
0
Fork 0
mirror of https://github.com/TheAnachronism/docspell.git synced 2025-06-06 07:05:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1021 from eikek/remove-scripts

Browse Source 
Remove scripts
This commit is contained in:
mergify[bot] 2021-08-20 16:53:19 +00:00 committed by GitHub
commit 3407abbad1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 49 additions and 2125 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
docker/dockerfiles/build.sh
View File

@ -39,13 +39,6 @@ if [[ $version == *SNAPSHOT* ]]; then
echo ">>>> Building nightly images for $version <<<<<" echo ">>>> Building nightly images for $version <<<<<"
url_base="https://github.com/eikek/docspell/releases/download/nightly" url_base="https://github.com/eikek/docspell/releases/download/nightly"
echo "============ Building Tools ============"
docker buildx build \
--platform="$platforms" $push \
--build-arg tools_url="$url_base/docspell-tools-$version.zip" \
--tag docspell/tools:nightly \
-f tools.dockerfile .
echo "============ Building Restserver ============" echo "============ Building Restserver ============"
docker buildx build \ docker buildx build \
--platform="$platforms" $push \ --platform="$platforms" $push \
@ -61,14 +54,6 @@ if [[ $version == *SNAPSHOT* ]]; then
-f joex.dockerfile . -f joex.dockerfile .
else else
echo ">>>> Building release images for $version <<<<<" echo ">>>> Building release images for $version <<<<<"
echo "============ Building Tools ============"
docker buildx build \
--platform="$platforms" $push \
--build-arg version=$version \
--tag docspell/tools:v$version \
--tag docspell/tools:latest \
-f tools.dockerfile .
echo "============ Building Restserver ============" echo "============ Building Restserver ============"
docker buildx build \ docker buildx build \
--platform="$platforms" $push \ --platform="$platforms" $push \

35
docker/dockerfiles/tools.dockerfile
View File

@ -1,35 +0,0 @@
FROM alpine:latest
# Builds an image where all scripts in tools/ are in PATH. There are
# no assumptions what script to run, so there are no CMD or
# ENTRYPOINTS defined.
#
# The scripts are named is in tools/ only prefixed by `ds-`
#
# Run the export-files script, for example:
#
# docker run -e DS_USER=demo -e DS_PASS=test docspell/tools:dev ds-export-files "http://localhost" .
#
# The build requires to either specify a version build argument or a
# tools_url build argument. If a tools_url argument is given, then
# this url is used to download the tools zip file. Otherwise the
# version argument is used to download from github.
LABEL maintainer="eikek0 <eike@docspell.org>"
ARG version=
ARG tools_url=
RUN apk add --no-cache curl bash inotify-tools jq sqlite
WORKDIR /opt
RUN wget ${tools_url:-https://github.com/eikek/docspell/releases/download/v$version/docspell-tools-$version.zip} && \
unzip docspell-tools-*.zip && \
rm docspell-tools-*.zip
RUN bash -c 'while read f; do \
target="ds-$(basename "$f" ".sh")"; \
echo "Installing $f -> $target"; \
cp "$f" "/usr/local/bin/$target"; \
chmod 755 "/usr/local/bin/$target"; \
done < <(find /opt/docspell-tools-* -name "*.sh" -mindepth 2 -not -path "*webextension*")'

181
tools/consumedir-cleaner/consumedir-cleaner.sh
View File

@ -1,181 +0,0 @@
#!/usr/bin/env bash
echo "##################### START #####################"
echo " Docspell Consumedir Cleaner - v0.1 beta"
echo " by totti4ever" && echo
echo " $(date)"
echo
echo "#################################################"
echo && echo
CURL_CMD="curl"
JQ_CMD="jq"
"$JQ_CMD" --version > /dev/null
if [ $? -ne 0 ]; then
echo "please install 'jq'"
exit -4
fi
ds_url=${1%/}
ds_user_param=$2
ds_user=${ds_user_param#*/}
ds_collective=${ds_user_param%%/*}
ds_password=$3
ds_consumedir_path=${4%/}
ds_archive_path=$ds_consumedir_path/_archive/$ds_collective
if [ $# -ne 4 ]; then
echo "FATAL Exactly four parameters needed"
exit -3
elif [ "$1" == "" ] || [ "$2" == "" ] || [ "$3" == "" ] || [ "$4" == "" ]; then
echo "FATAL Parameter missing"
echo " ds_url: $ds_url"
echo " ds_user: $ds_user"
echo " ds_password: $ds_password"
echo " ds_consumedir_path: $ds_consumedir_path"
exit -2
elif [ "$ds_collective" == "_archive" ]; then
echo "FATAL collective name '_archive' is not supported by this script"
exit -1
fi
############# FUNCTIONS
function curl_call() {
curl_cmd="$CURL_CMD $1 -H 'X-Docspell-Auth: $ds_token'"
curl_result=$(eval $curl_cmd)
curl_code=$?
if [ "$curl_result" == '"Authentication failed."' ] || [ "$curl_result" == 'Response timed out' ]; then
printf "\nNew login required ($curl_result)... "
login
printf "%${#len_resultset}s" " "; printf " .."
curl_call $1
elif [ "$curl_result" == "Bad Gateway" ] || [ "$curl_result" == '404 page not found' ]; then
echo "FATAL Connection to server failed"
exit -1
fi
}
function login() {
curl_call "-s -X POST -d '{\"account\": \"$ds_collective/$ds_user\", \"password\": \"$ds_password\"}' ${ds_url}/api/v1/open/auth/login"
curl_status=$(echo $curl_result | $JQ_CMD -r ".success")
if [ "$curl_status" == "true" ]; then
ds_token=$(echo $curl_result | $JQ_CMD -r ".token")
echo "Login successfull ( Token: $ds_token )"
else
echo "FATAL Login not succesfull"
exit 1
fi
}
############# END
echo "Settings:"
if [ "$DS_CC_REMOVE" == "true" ]; then
echo " ### !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ###"
echo " - DELETE files? YES"
echo " when already existing in Docspell. This cannot be undone!"
echo " ### !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ###"
else
echo " - DELETE files? no"
echo " moving already uploaded files to archive"
fi
echo
if [ "$DS_CC_UPLOAD_MISSING" == true ]; then
echo " - UPLOAD files? YES"
echo " files not existing in Docspell will be uploaded and will be re-checked in the next run."
else
echo " - UPLOAD files? no"
echo " files not existing in Docspell will NOT be uploaded and stay where they are."
fi
echo && echo
echo "Press 'ctrl+c' to cancel"
for ((i=9;i>=0;i--)); do
printf "\r waiting $i seconds "
sleep 1s
done
echo && echo
# login, get token
login
echo "Scanning folder for collective '$ds_collective' ($ds_consumedir_path/$ds_collective)"
echo && echo
while read -r line
do
tmp_filepath=$line
if [ "$tmp_filepath" == "" ]; then
echo "no files found" && echo
exit 0 #no results
elif [ ! -f "$tmp_filepath" ]; then
echo "FATAL no access to file: $tmp_filepath"
exit 3
fi
echo "Checking '$tmp_filepath'"
printf "%${#len_resultset}s" " "; printf " "
# check for checksum
tmp_checksum=$(sha256sum "$tmp_filepath" | awk '{print $1}')
curl_call "-s -X GET '$ds_url/api/v1/sec/checkfile/$tmp_checksum'"
curl_status=$(echo $curl_result | $JQ_CMD -r ".exists")
if [ $curl_code -ne 0 ]; then
# error
echo "ERROR $curl_result // $curl_status"
# file exists in Docspell
elif [ "$curl_status" == "true" ]; then
item_name=$(echo $curl_result | $JQ_CMD -r ".items[0].name")
item_id=$(echo $curl_result | $JQ_CMD -r ".items[0].id")
echo "File already exists: '$item_name (ID: $item_id)'"
printf "%${#len_resultset}s" " "; printf " "
if [ "$DS_CC_REMOVE" == "true" ]; then
echo "... removing file"
rm "$tmp_filepath"
else
created=$(echo $curl_result | $JQ_CMD -r ".items[0].created")
cur_dir="$ds_archive_path/$(date -d @$(expr \( $created + 500 \) / 1000) +%Y-%m)"
echo "... moving to archive by month added ('$cur_dir')"
mkdir -p "$cur_dir"
mv "$tmp_filepath" "$cur_dir/"
fi
# file does not exist in Docspell
else
echo "Files does not exist, yet"
if [ "$DS_CC_UPLOAD_MISSING" == true ]; then
printf "%${#len_resultset}s" " "; printf " "
printf "...uploading file.."
curl_call "-s -X POST '$ds_url/api/v1/sec/upload/item' -H 'Content-Type: multipart/form-data' -F 'file=@$tmp_filepath'"
curl_status=$(echo $curl_result | $JQ_CMD -r ".success")
if [ "$curl_status" == "true" ]; then
echo ". done"
else
echo -e "\nERROR $curl_result"
fi
fi
fi
echo
done \
<<< $(find $ds_consumedir_path/$ds_collective -type f)
echo ################# DONE #################
date

439
tools/consumedir/consumedir.sh
View File

@ -1,439 +0,0 @@
#!/usr/bin/env bash
# This script watches a directory for new files and uploads them to
# docspell. Or it uploads all files currently in the directory.
#
# It requires inotifywait, curl and sha256sum if the `-m' option is
# used.
# saner programming env: these switches turn some bugs into errors
set -o errexit -o pipefail -o noclobber -o nounset
CURL_CMD="curl"
INOTIFY_CMD="inotifywait"
SHA256_CMD="sha256sum"
MKTEMP_CMD="mktemp"
CURL_OPTS=${CURL_OPTS:-}
! getopt --test > /dev/null
if [[ ${PIPESTATUS[0]} -ne 4 ]]; then
echo 'I’m sorry, `getopt --test` failed in this environment.'
exit 1
fi
OPTIONS=omhdp:vrmi
LONGOPTS=once,distinct,help,delete,path:,verbose,recursive,dry,integration,iuser:,iheader:,poll:,exclude:,include:
! PARSED=$(getopt --options=$OPTIONS --longoptions=$LONGOPTS --name "$0" -- "$@")
if [[ ${PIPESTATUS[0]} -ne 0 ]]; then
# e.g. return value is 1
# then getopt has complained about wrong arguments to stdout
exit 2
fi
# read getopt’s output this way to handle the quoting right:
eval set -- "$PARSED"
declare -a watchdir
help=n verbose=n delete=n once=n distinct=n recursive=n dryrun=n
integration=n iuser="" iheader="" poll="" exclude="" include=""
while true; do
case "$1" in
-h|--help)
help=y
shift
;;
-v|--verbose)
verbose=y
shift
;;
-d|--delete)
delete=y
shift
;;
-o|--once)
once=y
shift
;;
-p|--path)
watchdir+=("$2")
shift 2
;;
-m|--distinct)
distinct=y
shift
;;
-r|--recursive)
recursive=y
shift
;;
--dry)
dryrun=y
shift
;;
-i|--integration)
integration=y
recursive=y
shift
;;
--iuser)
iuser="$2"
shift 2
;;
--iheader)
iheader="$2"
shift 2
;;
--poll)
poll="$2"
shift 2
;;
--exclude)
exclude="$2"
shift 2
;;
--include)
include="$2"
shift 2
;;
--)
shift
break
;;
*)
echo "Programming error"
exit 3
;;
esac
done
showUsage() {
echo "Upload files in a directory"
echo ""
echo "Usage: $0 [options] url url ..."
echo
echo "Options:"
echo " -v | --verbose Print more to stdout. (value: $verbose)"
echo " -d | --delete Delete the file if successfully uploaded. (value: $delete)"
echo " -p | --path <dir> The directories to watch. This is required. (value: ${watchdir[@]})"
echo " -h | --help Prints this help text. (value: $help)"
echo " -m | --distinct Optional. Upload only if the file doesn't already exist. (value: $distinct)"
echo " -o | --once Instead of watching, upload all files in that dir. (value: $once)"
echo " --poll <sec> Run the script periodically instead of watching a directory. This can be"
echo " used if watching via inotify is not possible. (value: $poll)"
echo " -r | --recursive Traverse the directory(ies) recursively (value: $recursive)"
echo " -i | --integration Upload to the integration endpoint. It implies -r. This puts the script in"
echo " a different mode, where the first subdirectory of any given starting point"
echo " is read as the collective name. The url(s) are completed with this name in"
echo " order to upload files to the respective collective. So each directory"
echo " given is expected to contain one subdirectory per collective and the urls"
echo " are expected to identify the integration endpoint, which is"
echo " /api/v1/open/integration/item/<collective-name>. (value: $integration)"
echo " --iheader The header name and value to use with the integration endpoint. This must be"
echo " in form 'headername:value'. Only used if '-i' is supplied."
echo " (value: $iheader)"
echo " --iuser The username and password for basic auth to use with the integration"
echo " endpoint. This must be of form 'user:pass'. Only used if '-i' is supplied."
echo " (value: $iuser)"
echo " --exclude <glob> A shell glob pattern that is used to skip files that match (value: $exclude)."
echo " --include <glob> A shell glob pattern that is used to find files to upload (value: $include)."
echo " If --exclude and --include is given, both apply."
echo " --dry Do a 'dry run', not uploading anything only printing to stdout (value: $dryrun)"
echo ""
echo "Arguments:"
echo " A list of URLs to upload the files to."
echo ""
echo "Example: Watch directory"
echo "$0 --path ~/Downloads -m -dv http://localhost:7880/api/v1/open/upload/item/abcde-12345-abcde-12345"
echo ""
echo "Example: Upload all files in a directory"
echo "$0 --path ~/Downloads -m -dv --once http://localhost:7880/api/v1/open/upload/item/abcde-12345-abcde-12345"
echo ""
echo "Example: Integration Endpoint"
echo "$0 -i --iheader 'Docspell-Integration:test123' -m -p ~/Downloads/ http://localhost:7880/api/v1/open/integration/item"
echo ""
}
if [ "$help" = "y" ]; then
showUsage
exit 0
fi
# handle non-option arguments
if [[ $# -eq 0 ]]; then
echo "$0: No upload URLs given."
exit 4
fi
urls=$@
if [ ! -d "$watchdir" ]; then
echo "The path '$watchdir' is not a directory."
exit 4
fi
trace() {
if [ "$verbose" = "y" ]; then
>&2 echo "$1"
fi
}
info() {
>&2 echo $1
}
getCollective() {
file=$(realpath "$1")
dir=$(realpath "$2")
collective=${file#"$dir"}
coll=$(echo $collective | cut -d'/' -f1)
if [ -z "$coll" ]; then
coll=$(echo $collective | cut -d'/' -f2)
fi
echo $coll
}
upload() {
dir=$(realpath "$1")
file=$(realpath "$2")
url="$3"
OPTS="$CURL_OPTS"
if [ "$integration" = "y" ]; then
collective=$(getCollective "$file" "$dir")
trace "- upload: collective = $collective"
url="$url/$collective"
if [ $iuser ]; then
OPTS="$OPTS --user $iuser"
fi
if [ $iheader ]; then
OPTS="$OPTS -H $iheader"
fi
fi
if [ "$dryrun" = "y" ]; then
info "- Not uploading (dry-run) $file to $url with opts $OPTS"
else
META1=""
META2=""
if [ "$distinct" = "y" ]; then
META1="-F"
META2="meta={\"multiple\": false, \"skipDuplicates\": true}"
fi
trace "- Uploading $file to $url with options $OPTS"
tf1=$($MKTEMP_CMD) tf2=$($MKTEMP_CMD) rc=0
$CURL_CMD --fail -# -o "$tf1" --stderr "$tf2" $OPTS -XPOST $META1 "$META2" -F file=@"$file" "$url"
if [ $? -ne 0 ]; then
info "Upload failed. Exit code: $rc"
cat "$tf1"
cat "$tf2"
echo ""
rm "$tf1" "$tf2"
return $rc
else
if cat $tf1 | grep -q '{"success":false'; then
echo "Upload failed. Message from server:"
cat "$tf1"
echo ""
rm "$tf1" "$tf2"
return 1
else
info "- Upload done."
rm "$tf1" "$tf2"
return 0
fi
fi
fi
}
checksum() {
$SHA256_CMD "$1" | cut -d' ' -f1 | xargs
}
checkFile() {
local url="$1"
local file="$2"
local dir="$3"
OPTS="$CURL_OPTS"
if [ "$integration" = "y" ]; then
collective=$(getCollective "$file" "$dir")
url="$url/$collective"
url=$(echo "$url" | sed 's,/item/,/checkfile/,g')
if [ $iuser ]; then
OPTS="$OPTS --user $iuser"
fi
if [ $iheader ]; then
OPTS="$OPTS -H $iheader"
fi
else
url=$(echo "$1" | sed 's,upload/item,checkfile,g')
fi
url=$url/$(checksum "$file")
trace "- Check file via $OPTS: $url"
tf1=$($MKTEMP_CMD) tf2=$($MKTEMP_CMD)
$CURL_CMD --fail -v -o "$tf1" --stderr "$tf2" $OPTS -XGET -s "$url"
if [ $? -ne 0 ]; then
info "Checking file failed!"
cat "$tf1" >&2
cat "$tf2" >&2
info ""
rm "$tf1" "$tf2"
echo "failed"
return 1
else
if cat "$tf1" | grep -q '{"exists":true'; then
rm "$tf1" "$tf2"
echo "y"
else
rm "$tf1" "$tf2"
echo "n"
fi
fi
}
process() {
file=$(realpath "$1")
dir="$2"
info "---- Processing $file ----------"
declare -i curlrc=0
set +e
for url in $urls; do
if [ "$distinct" = "y" ]; then
trace "- Checking if $file has been uploaded to $url already"
res=$(checkFile "$url" "$file" "$dir")
rc=$?
curlrc=$(expr $curlrc + $rc)
trace "- Result from checkfile: $res"
if [ "$res" = "y" ]; then
info "- Skipping file '$file' because it has been uploaded in the past."
continue
elif [ "$res" != "n" ]; then
info "- Checking file failed, skipping the file."
continue
fi
fi
trace "- Uploading '$file' to '$url'."
upload "$dir" "$file" "$url"
rc=$?
curlrc=$(expr $curlrc + $rc)
if [ $rc -ne 0 ]; then
trace "Upload to '$url' failed!"
fi
done
set -e
if [ $curlrc -ne 0 ]; then
info "-> Some uploads failed."
else
trace "= File processed for all URLs"
if [ "$delete" = "y" ]; then
info "- Deleting file '$file'"
set +e
rm "$file"
if [ $? -ne 0 ]; then
info "- Deleting failed!"
fi
set -e
fi
fi
}
findDir() {
path="$1"
for dir in "${watchdir[@]}"; do
if [[ $path = ${dir}* ]]
then
echo $dir
fi
done
}
checkSetup() {
for dir in "${watchdir[@]}"; do
find "$dir" -mindepth 1 -maxdepth 1 -type d -print0 | while IFS= read -d '' -r collective; do
for url in $urls; do
if [ "$integration" = "y" ]; then
url="$url/$(basename $collective)"
OPTS="$CURL_OPTS -i -s -o /dev/null -w %{http_code}"
if [ $iuser ]; then
OPTS="$OPTS --user $iuser"
fi
if [ $iheader ]; then
OPTS="$OPTS -H $iheader"
fi
trace "Checking integration endpoint: $CURL_CMD $OPTS "$url""
status=$($CURL_CMD $OPTS "$url")
if [ "$status" != "200" ]; then
echo "[WARN] Collective '$(basename $collective)' failed the setup check."
echo "[WARN] $status response, command: $CURL_CMD $OPTS $url"
fi
fi
done
done
done
}
runOnce() {
info "Uploading all files (except hidden) in '$watchdir'."
MD="-maxdepth 1"
if [ "$recursive" = "y" ]; then
MD=""
fi
EXCL=""
if [ -n "$exclude" ]; then
EXCL="-not -name $exclude"
fi
INCL=""
if [ -n "$include" ]; then
INCL="-name $include"
fi
for dir in "${watchdir[@]}"; do
find "$dir" $MD -type f $INCL $EXCL -not -name ".*" -print0 | while IFS= read -d '' -r file; do
process "$file" "$dir"
done
done
}
includeFile() {
file="$1"
if [ -n "$include" ] && [[ $file != $include ]]; then
trace "Skip $file due to include filter"
return 1
elif [ -n "$exclude" ] && [[ $file == $exclude ]]; then
trace "Skip $file due to exclude filter"
return 1
else
[[ "$file" != .* ]]
fi
}
# warn if something seems not correctly configured
checkSetup
if [ "$once" = "y" ]; then
runOnce
else
REC=""
if [ "$recursive" = "y" ]; then
REC="-r"
fi
if [ -z "$poll" ]; then
$INOTIFY_CMD $REC -m --format '%w%f' -e close_write -e moved_to "${watchdir[@]}" |
while read pathfile; do
if includeFile "$(basename "$pathfile")"; then
dir=$(findDir "$pathfile")
trace "The file '$pathfile' appeared below '$dir'"
sleep 1
process "$(realpath "$pathfile")" "$dir"
else
trace "Skip file $(realpath "$pathfile")"
fi
done
else
echo "Running in polling mode: ${poll}s"
while [ : ]
do
runOnce
sleep $poll
done
fi
fi

32
tools/convert-pdf/convert-all-pdfs.sh
View File

@ -1,32 +0,0 @@
#!/usr/bin/env bash
#
# Simple script to authenticate with docspell and trigger the "convert
# all pdf" route that submits a task to convert all pdf files using
# ocrmypdf.
set -e
CURL_CMD="curl"
JQ_CMD="jq"
BASE_URL="${1:-http://localhost:7880}"
LOGIN_URL="$BASE_URL/api/v1/open/auth/login"
TRIGGER_URL="$BASE_URL/api/v1/sec/item/convertallpdfs"
echo "Login to trigger converting all pdfs."
echo "Using url: $BASE_URL"
echo -n "Account: "
read USER
echo -n "Password: "
read -s PASS
echo
auth=$("$CURL_CMD" --fail -XPOST --silent --data-binary "{\"account\":\"$USER\", \"password\":\"$PASS\"}" "$LOGIN_URL")
if [ "$(echo $auth | "$JQ_CMD" .success)" == "true" ]; then
echo "Login successful"
auth_token=$(echo $auth | "$JQ_CMD" -r .token)
"$CURL_CMD" --fail -XPOST -H "X-Docspell-Auth: $auth_token" "$TRIGGER_URL"
else
echo "Login failed."
fi

213
tools/download-files/download-files.sh
View File

@ -1,213 +0,0 @@
#!/usr/bin/env bash
#
# Script for downloading files (the PDF versions) flat in the current
# directory. It takes a search query for selecting what to download.
# Metadata is not downloaded, only the files.
#
# Usage:
#
# download-files.sh <docspell-base-url> <query>
#
# The docspell base url is required as well as a search query. The
# output directory is the current directory, and can be defined via
# env variable "TARGET_DIR".
#
# Example:
#
# download-files.sh http://localhost:7880 "tag:todo folder:work"
#
# The script then asks for username and password and starts
# downloading. For more details about the query, please see the docs
# here: https://docspell.org/docs/query/
CURL_CMD="curl"
JQ_CMD="jq"
if [ -z "$1" ]; then
echo "The base-url to docspell is required."
exit 1
else
BASE_URL="$1"
shift
fi
if [ -z "$1" ]; then
errout "A search query is required"
exit 1
else
QUERY="$1"
shift
fi
set -o errexit -o pipefail -o noclobber -o nounset
LOGIN_URL="$BASE_URL/api/v1/open/auth/login"
SEARCH_URL="$BASE_URL/api/v1/sec/item/search"
DETAIL_URL="$BASE_URL/api/v1/sec/item"
ATTACH_URL="$BASE_URL/api/v1/sec/attachment"
OVERWRITE_FILE=${OVERWRITE_FILE:-n}
TARGET=${TARGET_DIR:-"$(pwd)"}
errout() {
>&2 echo "$@"
}
trap "{ rm -f ${TMPDIR-:/tmp}/ds-download.*; }" EXIT
mcurl() {
tmpfile1=$(mktemp -t "ds-download.XXXXX")
tmpfile2=$(mktemp -t "ds-download.XXXXX")
set +e
"$CURL_CMD" -# --fail --stderr "$tmpfile1" -o "$tmpfile2" -H "X-Docspell-Auth: $auth_token" "$@"
status=$?
set -e
if [ $status -ne 0 ]; then
errout "$CURL_CMD -H 'X-Docspell-Auth: …' $@"
errout "curl command failed (rc=$status)! Output is below."
cat "$tmpfile1" >&2
cat "$tmpfile2" >&2
rm -f "$tmpfile1" "$tmpfile2"
return 2
else
ret=$(cat "$tmpfile2")
rm "$tmpfile2" "$tmpfile1"
echo $ret
fi
}
errout "Login to Docspell."
errout "Using url: $BASE_URL"
if [ -z "${DS_USER:-}" ]; then
errout -n "Account: "
read DS_USER
fi
if [ -z "${DS_PASS:-}" ]; then
errout -n "Password: "
read -s DS_PASS
fi
echo
declare auth
declare auth_token
declare auth_time
login() {
auth=$("$CURL_CMD" -s --fail -XPOST \
--data-binary "{\"account\":\"$DS_USER\", \"password\":\"$DS_PASS\"}" "$LOGIN_URL")
if [ "$(echo $auth | "$JQ_CMD" .success)" == "true" ]; then
errout "Login successful"
auth_token=$(echo $auth | "$JQ_CMD" -r .token)
auth_time=$(date +%s)
else
errout "Login failed."
exit 1
fi
}
checkLogin() {
elapsed=$((1000 * ($(date +%s) - $auth_time)))
maxtime=$(echo $auth | "$JQ_CMD" .validMs)
elapsed=$(($elapsed + 1000))
if [ $elapsed -gt $maxtime ]; then
errout "Need to re-login $elapsed > $maxtime"
login
fi
}
listItems() {
OFFSET="${1:-0}"
LIMIT="${2:-50}"
QUERY="$3"
errout "Get next items with offset=$OFFSET, limit=$LIMIT"
REQ="{\"offset\":$OFFSET, \"limit\":$LIMIT, \"query\":\" $QUERY \"}"
mcurl -XPOST -H 'ContentType: application/json' -d "$REQ" "$SEARCH_URL" | "$JQ_CMD" -r '.groups[].items[]|.id'
}
fetchItem() {
mcurl -XGET "$DETAIL_URL/$1"
}
downloadAttachment() {
attachId="$1"
errout " - Download '$attachName' ($attachId)"
if [ -f "$attachOut" ] && [ "$SKIP_FILE" == "y" ]; then
errout " - Skipping file '$attachOut' since it already exists"
else
if [ -f "$attachOut" ] && [ "$OVERWRITE_FILE" == "y" ]; then
errout " - Removing attachment file as requested: $attachOut"
rm -f "$attachOut"
fi
DL_URL="$ATTACH_URL/$attachId"
checksum1=$("$CURL_CMD" -s -I -H "X-Docspell-Auth: $auth_token" "$DL_URL" | \
grep -i 'etag' | cut -d' ' -f2 | xargs | tr -d '\r')
"$CURL_CMD" -s -o "$attachOut" -H "X-Docspell-Auth: $auth_token" "$DL_URL"
checksum2=$(sha256sum "$attachOut" | cut -d' ' -f1 | xargs)
if [ "$checksum1" == "$checksum2" ]; then
errout " - Checksum ok."
else
errout " - WARNING: Checksum mismatch! Server: $checksum1 Downloaded: $checksum2"
return 3
fi
fi
}
downloadItem() {
checkLogin
itemData=$(fetchItem "$1")
errout "Get item $(echo $itemData | "$JQ_CMD" -r .id)"
created=$(echo $itemData|"$JQ_CMD" '.created')
created=$((($(echo $itemData|"$JQ_CMD" '.created') + 500) / 1000))
itemId=$(echo $itemData | "$JQ_CMD" -r '.id')
# out="$TARGET/$(date -d @$created +%Y-%m)/$itemId"
out="$TARGET"
if [ -d "$out" ] && [ "${DROP_ITEM:-}" == "y" ]; then
errout "Removing item folder as requested: $out"
rm -rf "$out"
fi
mkdir -p "$out"
while read attachId attachName; do
attachOut="$out/$attachName"
checkLogin
downloadAttachment "$attachId"
done < <(echo $itemData | "$JQ_CMD" -r '.attachments[] | [.id,.name] | join(" ")')
}
login
errout "Downloading files…"
allCounter=0 innerCounter=0 limit=100 offset=0 done=n
while [ "$done" = "n" ]; do
checkLogin
innerCounter=0
while read id; do
downloadItem "$id"
innerCounter=$(($innerCounter + 1))
done < <(listItems $offset $limit "$QUERY")
allCounter=$(($allCounter + $innerCounter))
offset=$(($offset + $limit))
if [ $innerCounter -lt $limit ]; then
done=y
fi
done
errout "Downloaded $allCounter items"

199
tools/ds.sh
View File

@ -1,199 +0,0 @@
#!/usr/bin/env bash
# A simple bash script that reads a configuration file to know where
# to upload a given file.
#
# The config file contains anonymous upload urls to docspell. All
# files given to this script are uploaded to all those urls.
#
# The default location for the config file is
# `~/.config/docspell/ds.conf'.
#
# The config file must contain lines of the form:
#
# url.1=http://localhost:7880/api/v1/open/upload/item/<source-id>
# url.2=...
#
# Lines starting with a `#' are ignored.
#
# The `-e|--exists' option allows to skip uploading and only check
# whether a given file exists in docspell.
# saner programming env: these switches turn some bugs into errors
set -o errexit -o pipefail -o noclobber -o nounset
CURL_CMD="curl"
GREP_CMD="grep"
MKTEMP_CMD="mktemp"
SHA256_CMD="sha256sum"
! getopt --test > /dev/null
if [[ ${PIPESTATUS[0]} -ne 4 ]]; then
echo 'I’m sorry, `getopt --test` failed in this environment.'
exit 1
fi
OPTIONS=c:hsde
LONGOPTS=config:,help,skip,delete,exists,allow-duplicates
! PARSED=$(getopt --options=$OPTIONS --longoptions=$LONGOPTS --name "$0" -- "$@")
if [[ ${PIPESTATUS[0]} -ne 0 ]]; then
# e.g. return value is 1
# then getopt has complained about wrong arguments to stdout
exit 2
fi
# read getopt’s output this way to handle the quoting right:
eval set -- "$PARSED"
exists=n delete=n help=n config="${XDG_CONFIG_HOME:-$HOME/.config}/docspell/ds.conf" dupes=n
while true; do
case "$1" in
-h|--help)
help=y
shift
;;
-c|--config)
config="$2"
shift 2
;;
-d|--delete)
delete="y"
shift
;;
-e|--exists)
exists=y
shift
;;
--allow-duplicates)
dupes=y
shift
;;
--)
shift
break
;;
*)
echo "Programming error"
exit 3
;;
esac
done
info() {
echo "$1"
}
checksum() {
$SHA256_CMD "$1" | cut -d' ' -f1 | xargs
}
checkFile() {
local url=$(echo "$1" | sed 's,upload/item,checkfile,g')
local file="$2"
$CURL_CMD -XGET -s "$url/$(checksum "$file")" | (2>&1 1>/dev/null grep '"exists":true')
}
upload_file() {
tf=$($MKTEMP_CMD) rc=0
META1=""
META2=""
if [ "$dupes" = "y" ]; then
META1="-F"
META2="meta={\"multiple\": false, \"skipDuplicates\": false}"
else
META1="-F"
META2="meta={\"multiple\": false, \"skipDuplicates\": true}"
fi
$CURL_CMD -# -o "$tf" --stderr "$tf" -w "%{http_code}" -XPOST $META1 "$META2" -F file=@"$1" "$2" | (2>&1 1>/dev/null grep 200)
rc=$(expr $rc + $?)
cat $tf | (2>&1 1>/dev/null grep '{"success":true')
rc=$(expr $rc + $?)
if [ $rc -ne 0 ]; then
info "Upload failed. Exit code: $rc"
cat "$tf"
echo ""
rm "$tf"
return $rc
else
rm "$tf"
return 0
fi
}
upload() {
if [ "$dupes" == "y" ]; then
upload_file "$1" "$2"
else
checkFile "$2" "$1"
if [ $? -eq 0 ]; then
info "File already exists at url $2"
return 0
else
upload_file "$1" "$2"
fi
fi
}
showUsage() {
info "Upload files to docspell"
info ""
info "Usage: $0 [options] file [file ...]"
info ""
info "Options:"
info " -c | --config Provide a config file. (value: $config)"
info " -d | --delete Delete the files when successfully uploaded (value: $delete)"
info " -h | --help Prints this help text. (value: $help)"
info " -e | --exists Checks for the existence of a file instead of uploading (value: $exists)"
info " --allow-duplicates Do not skip existing files in docspell (value: $dupes)"
info ""
info "Arguments:"
info " One or more files to check for existence or upload."
info ""
}
if [ "$help" = "y" ]; then
showUsage
exit 0
fi
# handle non-option arguments
if [[ $# -eq 0 ]]; then
echo "$0: No files given."
exit 4
fi
## Read the config file
declare -a urls
while IFS="=" read -r k v
do
if [[ $k == url* ]]; then
urls+=($(echo "$v" | xargs))
fi
done <<< $($GREP_CMD -v '^#.*' "$config")
## Main
IFS=$'\n'
for file in $*; do
for url in "${urls[@]}"; do
if [ "$exists" = "y" ]; then
if checkFile "$url" "$file"; then
info "$url $file: true"
else
info "$url $file: false"
fi
else
info "Uploading '$file' to '$url'"
set +e
upload "$file" "$url"
set -e
if [ "$delete" = "y" ] && [ $? -eq 0 ]; then
info "Deleting file: $file"
rm -f "$file"
fi
fi
done
done

256
tools/export-files/export-files.sh
View File

@ -1,256 +0,0 @@
#!/usr/bin/env bash
#
# Simple script for downloading all your files. It goes through all
# items visible to the logged in user and downloads the attachments
# (the original files).
#
# The item's metadata are stored next to the files to provide more
# information about the item: tags, dates, custom fields etc. This
# contains most of your user supplied data.
#
# This script is intended for having your data outside and independent
# of docspell. Another good idea for a backup strategy is to take
# database dumps *and* storing the releases of docspell next to this
# dump.
#
# Usage:
#
# export-files.sh <docspell-base-url> <target-directory>
#
# The docspell base url is required as well as a directory to store
# all the files into.
#
# Example:
#
# export-files.sh http://localhost:7880 /tmp/ds-download
#
# The script then asks for username and password and starts
# downloading. Files are downloaded into the following structure
# (below the given target directory):
#
# - yyyy-mm (item date)
# - A3…XY (item id)
# - somefile.pdf (attachments with name)
# - metadata.json (json file with items metadata)
#
# By default, files are not overwritten, it stops if existing files
# are encountered. Configuration can be specified using environment
# variables:
#
# - OVERWRITE_FILE= if `y` then overwriting existing files is ok.
# - SKIP_FILE= if `y` then existing files are skipped (supersedes
# OVERWRITE_FILE).
# - DROP_ITEM= if `y` the item folder is removed before attempting to
# download it. If this is set to `y` then the above options don't
# make sense, since they operate on the files inside the item folder
#
# Docspell sends with each file its sha256 checksum via the ETag
# header. This is used to do a integrity check after downloading.
CURL_CMD="curl"
JQ_CMD="jq"
if [ -z "$1" ]; then
echo "The base-url to docspell is required."
exit 1
else
BASE_URL="$1"
shift
fi
if [ -z "$1" ]; then
echo "A directory is required to store the files into."
exit 1
else
TARGET="$1"
shift
fi
set -o errexit -o pipefail -o noclobber -o nounset
LOGIN_URL="$BASE_URL/api/v1/open/auth/login"
SEARCH_URL="$BASE_URL/api/v1/sec/item/search"
INSIGHT_URL="$BASE_URL/api/v1/sec/collective/insights"
DETAIL_URL="$BASE_URL/api/v1/sec/item"
ATTACH_URL="$BASE_URL/api/v1/sec/attachment"
OVERWRITE_FILE=${OVERWRITE_FILE:-n}
DROP_ITEM=${DROP_ITEM:-n}
errout() {
>&2 echo "$@"
}
trap "{ rm -f ${TMPDIR-:/tmp}/ds-export.*; }" EXIT
mcurl() {
tmpfile1=$(mktemp -t "ds-export.XXXXX")
tmpfile2=$(mktemp -t "ds-export.XXXXX")
set +e
"$CURL_CMD" -# --fail --stderr "$tmpfile1" -o "$tmpfile2" -H "X-Docspell-Auth: $auth_token" "$@"
status=$?
set -e
if [ $status -ne 0 ]; then
errout "$CURL_CMD -H 'X-Docspell-Auth: …' $@"
errout "curl command failed (rc=$status)! Output is below."
cat "$tmpfile1" >&2
cat "$tmpfile2" >&2
rm -f "$tmpfile1" "$tmpfile2"
return 2
else
ret=$(cat "$tmpfile2")
rm "$tmpfile2" "$tmpfile1"
echo $ret
fi
}
errout "Login to Docspell."
errout "Using url: $BASE_URL"
if [ -z "${DS_USER:-}" ]; then
errout -n "Account: "
read DS_USER
fi
if [ -z "${DS_PASS:-}" ]; then
errout -n "Password: "
read -s DS_PASS
fi
echo
declare auth
declare auth_token
declare auth_time
login() {
auth=$("$CURL_CMD" -s --fail -XPOST \
--data-binary "{\"account\":\"$DS_USER\", \"password\":\"$DS_PASS\"}" "$LOGIN_URL")
if [ "$(echo $auth | "$JQ_CMD" .success)" == "true" ]; then
errout "Login successful"
auth_token=$(echo $auth | "$JQ_CMD" -r .token)
auth_time=$(date +%s)
else
errout "Login failed."
exit 1
fi
}
checkLogin() {
elapsed=$((1000 * ($(date +%s) - $auth_time)))
maxtime=$(echo $auth | "$JQ_CMD" .validMs)
elapsed=$(($elapsed + 1000))
if [ $elapsed -gt $maxtime ]; then
errout "Need to re-login $elapsed > $maxtime"
login
fi
}
listItems() {
OFFSET="${1:-0}"
LIMIT="${2:-50}"
errout "Get next items with offset=$OFFSET, limit=$LIMIT"
REQ="{\"offset\":$OFFSET, \"limit\":$LIMIT, \"withDetails\":true, \"query\":\"\"}"
mcurl -XPOST -H 'ContentType: application/json' -d "$REQ" "$SEARCH_URL" | "$JQ_CMD" -r '.groups[].items[]|.id'
}
fetchItemCount() {
mcurl -XGET "$INSIGHT_URL" | "$JQ_CMD" '[.incomingCount, .outgoingCount] | add'
}
fetchItem() {
mcurl -XGET "$DETAIL_URL/$1"
}
downloadAttachment() {
attachId="$1"
errout " - Download '$attachName' ($attachId)"
if [ -f "$attachOut" ] && [ "$SKIP_FILE" == "y" ]; then
errout " - Skipping file '$attachOut' since it already exists"
else
if [ -f "$attachOut" ] && [ "$OVERWRITE_FILE" == "y" ]; then
errout " - Removing attachment file as requested: $attachOut"
rm -f "$attachOut"
fi
checksum1=$("$CURL_CMD" -s -I -H "X-Docspell-Auth: $auth_token" "$ATTACH_URL/$attachId/original" | \
grep -i 'etag' | cut -d':' -f2 | xargs | tr -d '\r')
"$CURL_CMD" -s -o "$attachOut" -H "X-Docspell-Auth: $auth_token" "$ATTACH_URL/$attachId/original"
checksum2=$(sha256sum "$attachOut" | cut -d' ' -f1 | xargs)
if [ "$checksum1" == "$checksum2" ]; then
errout " - Checksum ok."
else
errout " - WARNING: Checksum mismatch! Server: $checksum1 Downloaded: $checksum2"
return 3
fi
fi
}
downloadItem() {
checkLogin
itemData=$(fetchItem "$1")
errout "Get item $(echo $itemData | "$JQ_CMD" -r .id)"
created=$(echo $itemData|"$JQ_CMD" '.created')
created=$((($(echo $itemData|"$JQ_CMD" '.created') + 500) / 1000))
itemId=$(echo $itemData | "$JQ_CMD" -r '.id')
out="$TARGET/$(date -d @$created +%Y-%m)/$itemId"
if [ -d "$out" ] && [ "$DROP_ITEM" == "y" ]; then
errout "Removing item folder as requested: $out"
rm -rf "$out"
fi
mkdir -p "$out"
if [ -f "$out/metadata.json" ] && [ "$SKIP_FILE" == "y" ]; then
errout " - Skipping file 'metadata.json' since it already exists"
else
if [ -f "$out/metadata.json" ] && [ "$OVERWRITE_FILE" == "y" ]; then
errout " - Removing metadata.json as requested"
rm -f "$out/metadata.json"
fi
echo $itemData > "$out/metadata.json"
fi
while read attachId attachName; do
attachOut="$out/$attachName"
checkLogin
downloadAttachment "$attachId"
done < <(echo $itemData | "$JQ_CMD" -r '.sources[] | [.id,.name] | join(" ")')
}
login
allCount=$(fetchItemCount)
errout "Downloading $allCount items…"
allCounter=0 innerCounter=0 limit=100 offset=0 done=n
while [ "$done" = "n" ]; do
checkLogin
innerCounter=0
while read id; do
downloadItem "$id"
innerCounter=$(($innerCounter + 1))
done < <(listItems $offset $limit)
allCounter=$(($allCounter + $innerCounter))
offset=$(($offset + $limit))
if [ $innerCounter -lt $limit ]; then
done=y
fi
done
errout "Downloaded $allCounter/$allCount items"
if [[ $allCounter < $allCount ]]; then
errout
errout " Downloaded less items than were reported as available. This"
errout " may be due to items in folders that you cannot see. Or it"
errout " may be a bug."
errout
fi

22
tools/preview/regenerate-previews.sh
View File

@ -1,22 +0,0 @@
#!/usr/bin/env bash
#
# This script submits a job to regenerate all preview images. This may
# be necessary if you change the dpi setting that affects the size of
# the preview.
set -e
CURL_CMD="curl"
JQ_CMD="jq"
BASE_URL="${1:-http://localhost:7880}"
TRIGGER_URL="$BASE_URL/api/v1/admin/attachments/generatePreviews"
echo "Login to trigger regenerating preview images."
echo "Using url: $BASE_URL"
echo -n "Admin Secret: "
read -s ADMIN_SECRET
echo
curl --fail -XPOST -H "Docspell-Admin-Secret: $ADMIN_SECRET" "$TRIGGER_URL"

49
tools/reset-password/reset-password.sh
View File

@ -1,49 +0,0 @@
#!/usr/bin/env bash
#
# A script to reset a password.
#
# Usage:
# ./reset-password.sh <baseurl> <admin-secret> <account>
#
# Example:
# ./reset-password.sh http://localhost:7880 test123 your/account
#
CURL_CMD="curl"
JQ_CMD="jq"
if [ -z "$1" ]; then
echo "The docspell base-url is required as first argument."
exit 1
else
BASE_URL="$1"
fi
if [ -z "$2" ]; then
echo "The admin secret is required as second argument."
exit 1
else
SECRET="$2"
fi
if [ -z "$3" ]; then
echo "The user account is required as third argument."
exit 1
else
USER="$3"
fi
RESET_URL="${BASE_URL}/api/v1/admin/user/resetPassword"
OUT=$("$CURL_CMD" -s -XPOST \
-H "Docspell-Admin-Secret: $SECRET" \
-H "Content-Type: application/json" \
-d "{\"account\": \"$USER\"}" \
"$RESET_URL")
if command -v "$JQ_CMD" > /dev/null; then
echo $OUT | "$JQ_CMD"
else
echo $OUT
fi

2
website/site/config.toml
View File

@ -4,10 +4,10 @@ base_url = "https://docspell.org"
# Whether to automatically compile all Sass files in the sass directory # Whether to automatically compile all Sass files in the sass directory
compile_sass = true compile_sass = true
[markdown]
# Whether to do syntax highlighting # Whether to do syntax highlighting
# Theme can be customised by setting the `highlight_theme` variable to a theme supported by Zola # Theme can be customised by setting the `highlight_theme` variable to a theme supported by Zola
highlight_code = true highlight_code = true
highlight_theme = "gruvbox-dark" highlight_theme = "gruvbox-dark"
# Whether to build a search index to be used later on by a JavaScript library # Whether to build a search index to be used later on by a JavaScript library

6
website/site/content/docs/faq/_index.md
View File

@ -82,9 +82,9 @@ documentation, too.
In order to move to a different tool, it is necessary to get the data In order to move to a different tool, it is necessary to get the data
out of Docspell in a machine readable/automatic way. Currently, there out of Docspell in a machine readable/automatic way. Currently, there
is a [export-files.sh](@/docs/tools/export-files.md) script provided is a [export command](@/docs/tools/cli.md#export-data) in the command
(in the `tools/` folder) that can be used to download all your files line client that can be used to download all your files and item
and item metadata. metadata.
My recommendation is to run periodic database backups and also store My recommendation is to run periodic database backups and also store
the binaries/docker images. This lets you re-create the current state the binaries/docker images. This lets you re-create the current state

11
website/site/content/docs/install/quickstart.md
View File

@ -7,8 +7,9 @@ To get started, here are some quick links:
- Using [docker and docker-compose](@/docs/install/docker.md). This - Using [docker and docker-compose](@/docs/install/docker.md). This
sets up everything: all prerequisites, both docspell components and sets up everything: all prerequisites, both docspell components and
a container running the [consumedir.sh](@/docs/tools/consumedir.md) a container running the [dsc
script to import files that are dropped in a folder. watch](@/docs/tools/cli.md#watch-a-directory) script to import files
that are dropped in a folder.
- [Download, Unpack and Run](@/docs/install/download_run.md). This - [Download, Unpack and Run](@/docs/install/download_run.md). This
option is also very quick, but you need to check the option is also very quick, but you need to check the
[prerequisites](@/docs/install/prereq.md) yourself. Database is [prerequisites](@/docs/install/prereq.md) yourself. Database is
@ -27,9 +28,9 @@ To get started, here are some quick links:
thread](https://forums.unraid.net/topic/103425-docspell-hilfe/) in thread](https://forums.unraid.net/topic/103425-docspell-hilfe/) in
the German Unraid forum. Thanks for providing these! the German Unraid forum. Thanks for providing these!
Every [component](@/docs/intro/_index.md#components) (restserver, joex, Every [component](@/docs/intro/_index.md#components) (restserver,
consumedir) can run on different machines and multiple times. Most of joex, dsc watch) can run on different machines and multiple times.
the time running all on one machine is sufficient and also for Most of the time running all on one machine is sufficient and also for
simplicity, the docker-compose setup reflects this variant. simplicity, the docker-compose setup reflects this variant.
While there are many different ways to run docspell, at some point all While there are many different ways to run docspell, at some point all

42
website/site/content/docs/tools/cli.md
View File

@ -167,7 +167,9 @@ directories.
The `watch` subcommand can be used to watch one or more directories The `watch` subcommand can be used to watch one or more directories
and upload files when they arrive. It uses the `upload` command under and upload files when they arrive. It uses the `upload` command under
the hood and therefore most options are also available here. the hood and therefore most options are also available here. You can
upload via a source url, the integration endpoint or a valid session
(requires to login).
It detects file creations and skips a rename within a watched folder. It detects file creations and skips a rename within a watched folder.
The flag `-r` or `--recursive` is required to recursively watch a The flag `-r` or `--recursive` is required to recursively watch a
@ -191,6 +193,10 @@ If watching a directory is not possible due to system constraints
use the `upload` subcommand with `--poll` option which periodically use the `upload` subcommand with `--poll` option which periodically
traverses a directory. traverses a directory.
When using the integration endpoint, it requires to specify `-i` and
potentially a secret if the endpoint is protected with a secret.
## Download files ## Download files
The `download` command allows to download files that match a given The `download` command allows to download files that match a given
@ -303,7 +309,8 @@ commands require the [admin
secret](@/docs/configure/_index.md#admin-endpoint) either in the secret](@/docs/configure/_index.md#admin-endpoint) either in the
config file or as an argument. config file or as an argument.
Reset user password: ### Reset user password
``` shell ``` shell
❯ dsc admin reset-password --account demo ❯ dsc admin reset-password --account demo
┌─────────┬──────────────┬──────────────────┐ ┌─────────┬──────────────┬──────────────────┐
@ -313,7 +320,8 @@ Reset user password:
└─────────┴──────────────┴──────────────────┘ └─────────┴──────────────┴──────────────────┘
``` ```
Recreate fulltext index: ### Recreate fulltext index
``` shell ``` shell
❯ dsc admin --admin-secret admin123 recreate-index ❯ dsc admin --admin-secret admin123 recreate-index
┌─────────┬─────────────────────────────────────┐ ┌─────────┬─────────────────────────────────────┐
@ -323,6 +331,34 @@ Recreate fulltext index:
└─────────┴─────────────────────────────────────┘ └─────────┴─────────────────────────────────────┘
``` ```
### Convert all files to PDF
``` shell
❯ dsc admin --admin-secret admin123 convert-all-pdf
┌─────────┬─────────────────────────────────┐
│ success │ message │
├─────────┼─────────────────────────────────┤
│ true │ Convert all PDFs task submitted │
└─────────┴─────────────────────────────────┘
```
This may be necessary if you disabled pdf conversion before and are
enabling it now.
### Regenerate preview images
``` shell
❯ dsc admin --admin-secret admin123 convert-all-pdf
┌─────────┬───────────────────────────────────────┐
│ success │ message │
├─────────┼───────────────────────────────────────┤
│ true │ Generate all previews task submitted. │
└─────────┴───────────────────────────────────────┘
```
This submits tasks to (re)generate preview images of all files. This
is necessary if you changed the `preview.dpi` setting in joex'
config.
## Search for items ## Search for items
The `search` command takes a [query](@/docs/query/_index.md) and The `search` command takes a [query](@/docs/query/_index.md) and

58
website/site/content/docs/tools/consumedir-cleaner.md
View File

@ -1,58 +0,0 @@
+++
title = "Directory Cleaner (⊗)"
description = "Clean directories from files in docspell"
weight = 150
+++
{% infobubble(mode="info", title="⚠ Please note") %}
This script is now obsolete, you can use the [**CLI tool**](../cli/) instead.
Use the `cleanup` or the `upload` command.
{% end %}
# Introduction
This script is made for cleaning up the consumption directory used for
the consumedir service (as it is provided as docker container)which
are copied or moved there.
<https://github.com/eikek/docspell/tree/master/tools/consumedir-cleaner>
## How it works
- Checks for every file (in the collective's folder of the given user
name) if it already exists in the collective (using Docspell's API).
- If so, by default those files are moved to an archive folder just
besides the collective's consumption folders named _archive. The
archive's files are organized into monthly subfolders by the date
they've been added to Docspell
- If set, those files can also be deleted instead of being moved to
the archive. There is no undo function provided for this, so be
careful.
- If a file is found which does not exist in the collective, by
default nothing happens, so that file would be found in every run
and just ignored
- If set, those files can also be uploaded to Docspell. Depending on
the setting for files already existing these files would either be
deleted or moved to the archive in the next run.
## Usage (parameters / settings)
Copy the script to your machine and run it with the following
parameters:
1. URL of Docspell, including http(s)
2. Username for Docspell, possibly including Collective (if other name
as user)
3. Password for Docspell
4. Path to the directory which files shall be checked against
existence in Docspell
Additionally, environment variables can be used to alter the behavior:
- `DS_CC_REMOVE`
- `true` – delete files which already exist in the collective
- `false` (default) - move them to the archive (see above)
- `DS_CC_UPLOAD_MISSING`
- `true` - uploads files which do not exist in the collective
- `false` (default) - ignore them and do nothing

191
website/site/content/docs/tools/consumedir.md
View File

@ -1,191 +0,0 @@
+++
title = "Consume Directory (⊗)"
description = "A script to watch a directory for new files and upload them to docspell."
weight = 110
+++
{% infobubble(mode="info", title="⚠ Please note") %}
This script is now obsolete, you can use the [**CLI tool**](../cli/) instead.
You can use the `watch` command, or the `upload` command with `--poll`.
{% end %}
# Introduction
The `consumerdir.sh` is a bash script that works in two modes:
- Go through all files in given directories (recursively, if `-r` is
specified) and sent each to docspell.
- Watch one or more directories for new files and upload them to
docspell.
It can watch or go through one or more directories. Files can be
uploaded to multiple urls.
Run the script with the `-h` or `--help` option, to see a short help
text. The help text will also show the values for any given option.
The script requires `curl` for uploading. It requires the
`inotifywait` command if directories should be watched for new
files.
Example for watching two directories:
``` bash
./tools/consumedir.sh --path ~/Downloads --path ~/pdfs -m -dv \
http://localhost:7880/api/v1/open/upload/item/5DxhjkvWf9S-CkWqF3Kr892-WgoCspFWDo7-XBykwCyAUxQ
```
The script by default watches the given directories. If the `-o` or
`--once` option is used, it will instead go through these directories
and upload all files in there. For directory watching the
`inotifywait` command is used and must be present. Another way is to
use the `--poll` option. It expects the number of seconds to wait
between running itself with `--once`.
Example using active polling (at 5 minutes interval):
``` bash
./tools/consumedir.sh --poll 300 --path ~/Downloads --path ~/pdfs -m -dv \
http://localhost:7880/api/v1/open/upload/item/5DxhjkvWf9S-CkWqF3Kr892-WgoCspFWDo7-XBykwCyAUxQ
```
Example for uploading all immediatly (the same as above only with `-o`
added):
``` bash
$ ./tools/consumedir.sh --once --path ~/Downloads --path ~/pdfs/ -m -dv \
http://localhost:7880/api/v1/open/upload/item/5DxhjkvWf9S-CkWqF3Kr892-WgoCspFWDo7-XBykwCyAUxQ
```
The URL can be any docspell url that accepts uploads without
authentication. This is usually a [source
url](@/docs/webapp/uploading.md#anonymous-upload). It is also possible
to use the script with the [integration
endpoint](@/docs/api/upload.md#integration-endpoint).
The script can be run multiple times and on on multiple machines, the
files are transferred via HTTP to the docspell server. For example, it
is convenient to set it up on your workstation, so that you can drop
files into some local folder to be immediatly transferred to docspell
(e.g. when downloading something from the browser).
## Integration Endpoint
When given the `-i` or `--integration` option, the script changes its
behaviour slightly to work with the [integration
endpoint](@/docs/api/upload.md#integration-endpoint).
First, if `-i` is given, it implies `-r` – so the directories are
watched or traversed recursively. The script then assumes that there
is a subfolder with the collective name. Files must not be placed
directly into a folder given by `-p`, but below a sub-directory that
matches a collective name. In order to know for which collective the
file is, the script uses the first subfolder.
If the endpoint is protected, the credentials can be specified as
arguments `--iuser` and `--iheader`, respectively. The format is for
both `<name>:<value>`, so the username cannot contain a colon
character (but the password can).
Example:
``` bash
$ consumedir.sh -i -iheader 'Docspell-Integration:test123' -m -p ~/Downloads/ http://localhost:7880/api/v1/open/integration/item
```
The url is the integration endpoint url without the collective, as
this is amended by the script.
This watches the folder `~/Downloads`. If a file is placed in this
folder directly, say `~/Downloads/test.pdf` the upload will fail,
because the collective cannot be determined. Create a subfolder below
`~/Downloads` with the name of a collective, for example
`~/Downloads/family` and place files somewhere below this `family`
subfolder, like `~/Downloads/family/test.pdf`.
## Duplicates
With the `-m` option, the script will not upload files that already
exist at docspell. For this the `sha256sum` command is required.
So you can move and rename files in those folders without worring
about duplicates. This allows to keep your files organized using the
file-system and have them mirrored into docspell as well.
## Network Filesystems (samba cifs, nfs)
Watching a directory for changes relies on `inotify` subsystem on
linux. This doesn't work on network filesystems like nfs or cifs. Here
are some ideas to get around this limitation:
1. The `consumedir.sh` is just a shell script and doesn't need to run
on the same machine as docspell. (Note that the default docker
setup is mainly for demoing and quickstart, it's not required to
run all of them on one machine). So the best option is to put the
consumedir on the machine that contains the local filesystem. All
files are send via HTTP to the docspell server anyways, so there is
no need to first transfer them via a network filesystem or rsync.
2. If option 1 is not possible for some reason, and you need to check
a network filesystem, the only option left (that I know) is to
periodically poll this directory. This is also possible with
consumedir, using the `--poll` option (see above). You can also
setup a systemd timer to periodically run this script with the
`--once` option.
3. Copy the files to the machine that runs consumedir, via rsync for
example. Note that this has no advantage over otpion 1, as you now
need to setup rsync on the other machine to run either periodically
or when some file arrives. Then you can as well run the consumedir
script. But it might be more convenient, if rsync is already
running.
# Systemd
The script can be used with systemd to run as a service. This is an
example unit file:
``` systemd
[Unit]
After=networking.target
Description=Docspell Consumedir
[Service]
Environment="PATH=/set/a/path"
ExecStart=/bin/su -s /bin/bash someuser -c "consumedir.sh --path '/a/path/' -m 'http://localhost:7880/api/v1/open/upload/item/5DxhjkvWf9S-CkWqF3Kr892-WgoCspFWDo7-XBykwCyAUxQ'"
```
This unit file is just an example, it needs some fiddling. It assumes
an existing user `someuser` that is used to run this service. The url
`http://localhost:7880/api/v1/open/upload/...` is an anonymous upload
url as described [here](@/docs/webapp/uploading.md#anonymous-upload).
# Docker
The provided docker-compose setup runs this script to watch a single
directory, `./docs` in current directory, for new files. If a new file
is detected, it is pushed to docspell.
This utilizes the [integration
endpoint](@/docs/api/upload.md#integration-endpoint), which is
enabled in the config file, to allow uploading documents for all
collectives. A subfolder must be created for each registered
collective. The docker containers are configured to use http-header
protection for the integration endpoint. This requires you to provide
a secret, that is shared between the rest-server and the
`consumedir.sh` script. This can be done by defining an environment
variable which gets picked up by the containers defined in
`docker-compose.yml`:
``` bash
export DOCSPELL_HEADER_VALUE="my-secret"
docker-compose up
```
Now you can create a folder `./docs/<collective-name>` and place all
files in there that you want to import. Once dropped in this folder
the `consumedir` container will push it to docspell.

59
website/site/content/docs/tools/convert-all-pdf.md
View File

@ -1,59 +0,0 @@
+++
title = "Convert All PDFs (⊗)"
description = "Convert all PDF files using OcrMyPdf."
weight = 160
+++
{% infobubble(mode="info", title="⚠ Please note") %}
This script is now obsolete, you can use the [**CLI tool**](../cli/) instead.
Use the `convert-all-pdfs` admin command, e.g. `dsc admin
convert-all-pdfs`.
{% end %}
# convert-all-pdf.sh
With version 0.9.0 there was support added for another external tool,
[OCRMyPdf](https://github.com/jbarlow83/OCRmyPDF), that can convert
PDF files such that they contain the OCR-ed text layer. This tool is
optional and can be disabled.
In order to convert all previously processed files with this tool,
there is an
[endpoint](/openapi/docspell-openapi.html#api-Item-secItemConvertallpdfsPost)
that submits a task to convert all PDF files not already converted for
your collective.
There is no UI part to trigger this route, so you need to use curl or
the script `convert-all-pdfs.sh` in the `tools/` directory.
# Requirements
It is a bash script that additionally needs
[curl](https://curl.haxx.se/) and
[jq](https://stedolan.github.io/jq/).
# Usage
```
./convert-all-pdfs.sh [docspell-base-url]
```
For example, if docspell is at `http://localhost:7880`:
```
./convert-all-pdfs.sh http://localhost:7880
```
The script asks for your account name and password. It then logs in
and triggers the said endpoint. After this you should see a few tasks
running.
There will be one task per file to convert. All these tasks are
submitted with a low priority. So files uploaded through the webapp or
a [source](@/docs/webapp/uploading.md#anonymous-upload) with a high
priority, will be preferred as [configured in the job
executor](@/docs/joex/intro.md#scheduler-config). This is to not
disturb normal processing when many conversion tasks are being
executed.

54
website/site/content/docs/tools/ds.md
View File

@ -1,54 +0,0 @@
+++
title = "Upload CLI (⊗)"
description = "A script to quickly upload files from the command line."
weight = 100
+++
{% infobubble(mode="info", title="⚠ Please note") %}
This script is now obsolete, you can use the [**CLI tool**](../cli/) instead.
Use the `upload` command (or the `up` alias), like `dsc up *.pdf`.
{% end %}
# Introduction
The `tools/ds.sh` is a bash script to quickly upload files from the
command line. It reads a configuration file containing the URLs to
upload to. Then each file given to the script will be uploaded to al
URLs in the config.
The config file is expected in
`$XDG_CONFIG_HOME/docspell/ds.conf`. `$XDG_CONFIG_HOME` defaults to
`~/.config`.
The config file contains lines with key-value pairs, separated by a
`=` sign. Lines starting with `#` are ignored. Example:
```
# Config file
url.1 = http://localhost:7880/api/v1/open/upload/item/5DxhjkvWf9S-CkWqF3Kr892-WgoCspFWDo7-XBykwCyAUxQ
url.2 = http://localhost:7880/api/v1/open/upload/item/6DxhjkvWf9S-CkWqF3Kr892-WgoCspFWDo7-XBykwCyAUxQ
```
The key must start with `url`. The urls should be [anonymous upload
urls](@/docs/webapp/uploading.md#anonymous-upload).
# Usage
- The `-c` option allows to specifiy a different config file.
- The `-h` option shows a help overview.
- The `-d` option deletes files after upload was successful
- The `-e` option can be used to check for file existence in docspell.
Instead of uploading, the script only checks whether the file is in
docspell or not.
The script takes a list of files as arguments.
Example:
``` bash
./ds.sh ~/Downloads/*.pdf
```

215
website/site/content/docs/tools/export-files.md
View File

@ -1,215 +0,0 @@
+++
title = "Export Files (⊗)"
description = "Downloads all files from docspell."
weight = 165
+++
{% infobubble(mode="info", title="⚠ Please note") %}
This script is now obsolete, you can use the [**CLI tool**](../cli/) instead.
Use the `export` command, e.g. `dsc export --all --target .`.
{% end %}
# export-files.sh
This script can be used to download all files from docspell that have
been uploaded before and the item metadata.
It downloads the original files, those that have been uploaded and not
the converted pdf files.
The item's metadata are stored next to the files to provide more
information about the item: corresponent, tags, dates, custom fields
etc. This contains most of your user supplied data.
This script is intended for having your data outside and independent
of docspell. Another good idea for a backup strategy is to take
database dumps *and* storing the releases of docspell next to this
dump.
Files are stored into the following folder structure (below the given
target directory):
```
- yyyy-mm (item date)
- A3…XY (item id)
- somefile.pdf (attachments with name)
- metadata.json (json file with items metadata)
```
By default, files are not overwritten, it stops if existing files are
encountered. This and some other things can be changed using
environment variables:
- `DS_USER` the account name for login, it is asked if not available
- `DS_PASS` the password for login, it is asked if not available
- `OVERWRITE_FILE=` if `y` then overwriting existing files is ok.
Default is `n`.
- `SKIP_FILE=` if `y` then existing files are skipped (supersedes
`OVERWRITE_FILE`). Default is `n`.
- `DROP_ITEM=` if `y` the item folder is removed before attempting to
download it. If this is set to `y` then the above options don't make
sense, since they operate on the files inside the item folder.
Default is `n`.
Docspell sends the sha256 hash with each file via the ETag header.
This is used to do a integrity check after downloading.
# Requirements
It is a bash script that additionally needs
[curl](https://curl.haxx.se/) and [jq](https://stedolan.github.io/jq/)
to be available.
# Usage
```
./export-files.sh <docspell-base-url> <target-directory>
```
For example, if docspell is at `http://localhost:7880`:
```
./export-files.sh http://localhost:7880 /tmp/ds-downloads
```
The script asks for your account name and password. It then logs in
and goes through all items downloading the metadata as json and the
attachments.
# Example Run
``` bash
fish> env SKIP_FILE=y DS_USER=demo DS_PASS=test ./export-files.sh http://localhost:7880 /tmp/download
Login to Docspell.
Using url: http://localhost:7880
Login successful
Downloading 73 items…
Get next items with offset=0, limit=100
Get item 57Znskthf3g-X7RP1fxzE2U-dwr4vM6Yjnn-b7s1PoCznhz
- Download 'something.txt' (8HbeFornAUN-kBCyc8bHSVr-bnLBYDzgRQ7-peMZzyTzM2X)
- Checksum ok.
Get item 94u5Pt39q6N-7vKu3LugoRj-zohGS4ie4jb-68bW5gXU6Jd
- Download 'letter-en.pdf' (6KNNmoyqpew-RAkdwEmQgBT-QDqdY97whZA-4k2rmbssdfQ)
- Checksum ok.
Get item 7L9Fh53RVG4-vGSt2G2YUcY-cvpBKRXQgBn-omYpg6xQXyD
- Download 'mail.html' (A6yTYKrDc7y-xU3whmLB1kB-TGhEAVb12mo-RUw5u9PsYMo)
- Checksum ok.
Get item DCn9UtWUtvF-2qjxB5PXGEG-vqRUUU7JUJH-zBBrmSeGYPe
- Download 'Invoice_7340224.pdf' (6FWdjxJh7yB-CCjY39p6uH9-uVLbmGfm25r-cw6RksrSx4n)
- Checksum ok.
…
```
The resulting directory looks then like this:
``` bash
…
├── 2020-08
│   ├── 6t27gQQ4TfW-H4uAmkYyiSe-rBnerFE2v5F-9BdqbGEhMcv
│   │   ├── 52241.pdf
│   │   └── metadata.json
│   └── 9qwT2GuwEvV-s9UuBQ4w7o9-uE8AdMc7PwL-GFDd62gduAm
│   ├── DOC-20191223-155707.jpg
│   └── metadata.json
├── 2020-09
│   ├── 2CM8C9VaVAT-sVJiKyUPCvR-Muqr2Cqvi6v-GXhRtg6eomA
│   │   ├── letter with spaces.pdf
│   │   └── metadata.json
│   ├── 4sXpX2Sc9Ex-QX1M6GtjiXp-DApuDDzGQXR-7pg1QPW9pbs
│   │   ├── analyse.org
│   │   ├── 201703.docx
│   │   ├── 11812_120719.pdf
│   │   ├── letter-de.pdf
│   │   ├── letter-en.pdf
│   │   └── metadata.json
│   ├── 5VhP5Torsy1-15pwJBeRjPi-es8BGnxhWn7-3pBQTJv3zPb
│   │   └── metadata.json
│   ├── 7ePWmK4xCNk-gmvnTDdFwG8-JcN5MDSUNPL-NTZZrho2Jc6
│   │   ├── metadata.json
│   │   └── Rechnung.pdf
…
```
The `metadata.json` file contains all the item metadata. This may be
useful when importing into other tools.
``` json
{
"id": "AWCNx7tJgUw-SdrNtRouNJB-FGs6Y2VP5bV-218sFN8mjjk",
"direction": "incoming",
"name": "Ruecksendung.pdf",
"source": "integration",
"state": "confirmed",
"created": 1606171810005,
"updated": 1606422917826,
"itemDate": null,
"corrOrg": null,
"corrPerson": null,
"concPerson": null,
"concEquipment": null,
"inReplyTo": null,
"folder": null,
"dueDate": null,
"notes": null,
"attachments": [
{
"id": "4aPmhrjfR9Z-AgknoW6yVoE-YkffioD2KXV-E6Vm6snH17Q",
"name": "Ruecksendung.converted.pdf",
"size": 57777,
"contentType": "application/pdf",
"converted": true
}
],
"sources": [
{
"id": "4aPmhrjfR9Z-AgknoW6yVoE-YkffioD2KXV-E6Vm6snH17Q",
"name": "Ruecksendung.pdf",
"size": 65715,
"contentType": "application/pdf"
}
],
"archives": [],
"tags": [
{
"id": "EQvJ6AHw19Y-Cdg3gF78zZk-BY2zFtNTwes-J95jpXpzhfw",
"name": "Hupe",
"category": "state",
"created": 1606427083171
},
{
"id": "4xyZoeeELdJ-tJ91GiRLinJ-7bdauy3U1jR-Bzr4VS96bGS",
"name": "Invoice",
"category": "doctype",
"created": 1594249709473
}
],
"customfields": [
{
"id": "5tYmDHin3Kx-HomKkeEVtJN-v99oKxQ8ot6-yFVrEmMayoo",
"name": "amount",
"label": "EUR",
"ftype": "money",
"value": "151.55"
},
{
"id": "3jbwbep8rDs-hNJ9ePRE7gv-21nYMbUj3eb-mKRWAr4xSS2",
"name": "invoice-number",
"label": "Invoice-Nr",
"ftype": "text",
"value": "I454602"
},
{
"id": "AH4p4NUCa9Y-EUkH66wLzxE-Rf2wJPxTAYd-DeGDm4AT4Yg",
"name": "number",
"label": "Number",
"ftype": "numeric",
"value": "0.10"
}
]
}
```

48
website/site/content/docs/tools/regenerate-previews.md
View File

@ -1,48 +0,0 @@
+++
title = "Regenerate Preview Images (⊗)"
description = "Re-generates all preview images."
weight = 130
+++
{% infobubble(mode="info", title="⚠ Please note") %}
This script is now obsolete, you can use the [**CLI tool**](../cli/) instead.
Use the `generate-previews` admin command, e.g. `dsc admin generate-previews`.
{% end %}
# regenerate-previews.sh
This is a simple bash script to trigger the endpoint that submits task
for generating preview images of your files. This is usually not
needed, but should you change the `preview.dpi` setting in joex'
config file, you need to regenerate the images to have any effect.
# Requirements
It is a bash script that additionally needs
[curl](https://curl.haxx.se/) and
[jq](https://stedolan.github.io/jq/).
# Usage
```
./regenerate-previews.sh [docspell-base-url] [admin-secret]
```
For example, if docspell is at `http://localhost:7880`:
```
./convert-all-pdfs.sh http://localhost:7880 test123
```
The script asks for the admin secret if not given to the command. It
then logs in and triggers the said endpoint. After this you should see
a few tasks running.
There will be one task per file to convert. All these tasks are
submitted with a low priority. So files uploaded through the webapp or
a [source](@/docs/webapp/uploading.md#anonymous-upload) with a high
priority, will be preferred as [configured in the job
executor](@/docs/joex/intro.md#scheduler-config). This is to not
disturb normal processing when many conversion tasks are being
executed.

47
website/site/content/docs/tools/reset-password.md
View File

@ -1,47 +0,0 @@
+++
title = "Reset Password (⊗)"
description = "Resets a user password."
weight = 120
+++
{% infobubble(mode="info", title="⚠ Please note") %}
This script is now obsolete, you can use the [**CLI tool**](../cli/) instead.
Use the `reset-password` admin command, e.g. `dsc admin reset-password
--account "smith/john"`, where `smith` is the collective id and `john`
the username.
{% end %}
This script can be used to reset a user password. This can be done by
admins, who know the `admin-endpoint.secret` value in the
[configuration](@/docs/configure/_index.md#admin-endpoint) file.
The script is in `/tools/reset-password/reset-password.sh` and it is
only a wrapper around the admin endpoint `/admin/user/resetPassword`.
## Usage
It's very simple:
``` bash
reset-password.sh <base-url> <admin-secret> <account>
```
Three arguments are required to specify the docspell base url, the
admin secret and the account you want to reset the password.
After the password has been reset, the user can login using it and
change it again in the webapp.
## Example
``` json
❯ ./tools/reset-password/reset-password.sh http://localhost:7880 123 eike
{
"success": true,
"newPassword": "HjtpG9BFo9y",
"message": "Password updated"
}
```