mirror of
https://github.com/TheAnachronism/docspell.git
synced 2025-06-05 06:35:59 +00:00
mergify[bot]
GitHub
commit
3407abbad1
@ -39,13 +39,6 @@ if [[ $version == *SNAPSHOT* ]]; then
|
||||
echo ">>>> Building nightly images for $version <<<<<"
|
||||
url_base="https://github.com/eikek/docspell/releases/download/nightly"
|
||||
|
||||
echo "============ Building Tools ============"
|
||||
docker buildx build \
|
||||
--platform="$platforms" $push \
|
||||
--build-arg tools_url="$url_base/docspell-tools-$version.zip" \
|
||||
--tag docspell/tools:nightly \
|
||||
-f tools.dockerfile .
|
||||
|
||||
echo "============ Building Restserver ============"
|
||||
docker buildx build \
|
||||
--platform="$platforms" $push \
|
||||
@ -61,14 +54,6 @@ if [[ $version == *SNAPSHOT* ]]; then
|
||||
-f joex.dockerfile .
|
||||
else
|
||||
echo ">>>> Building release images for $version <<<<<"
|
||||
echo "============ Building Tools ============"
|
||||
docker buildx build \
|
||||
--platform="$platforms" $push \
|
||||
--build-arg version=$version \
|
||||
--tag docspell/tools:v$version \
|
||||
--tag docspell/tools:latest \
|
||||
-f tools.dockerfile .
|
||||
|
||||
echo "============ Building Restserver ============"
|
||||
docker buildx build \
|
||||
--platform="$platforms" $push \
|
||||
|
||||
@ -1,35 +0,0 @@
|
||||
FROM alpine:latest
|
||||
|
||||
# Builds an image where all scripts in tools/ are in PATH. There are
|
||||
# no assumptions what script to run, so there are no CMD or
|
||||
# ENTRYPOINTS defined.
|
||||
#
|
||||
# The scripts are named is in tools/ only prefixed by `ds-`
|
||||
#
|
||||
# Run the export-files script, for example:
|
||||
#
|
||||
# docker run -e DS_USER=demo -e DS_PASS=test docspell/tools:dev ds-export-files "http://localhost" .
|
||||
#
|
||||
# The build requires to either specify a version build argument or a
|
||||
# tools_url build argument. If a tools_url argument is given, then
|
||||
# this url is used to download the tools zip file. Otherwise the
|
||||
# version argument is used to download from github.
|
||||
|
||||
LABEL maintainer="eikek0 <eike@docspell.org>"
|
||||
|
||||
ARG version=
|
||||
ARG tools_url=
|
||||
|
||||
RUN apk add --no-cache curl bash inotify-tools jq sqlite
|
||||
|
||||
WORKDIR /opt
|
||||
RUN wget ${tools_url:-https://github.com/eikek/docspell/releases/download/v$version/docspell-tools-$version.zip} && \
|
||||
unzip docspell-tools-*.zip && \
|
||||
rm docspell-tools-*.zip
|
||||
|
||||
RUN bash -c 'while read f; do \
|
||||
target="ds-$(basename "$f" ".sh")"; \
|
||||
echo "Installing $f -> $target"; \
|
||||
cp "$f" "/usr/local/bin/$target"; \
|
||||
chmod 755 "/usr/local/bin/$target"; \
|
||||
done < <(find /opt/docspell-tools-* -name "*.sh" -mindepth 2 -not -path "*webextension*")'
|
||||
@ -1,181 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
echo "##################### START #####################"
|
||||
|
||||
echo " Docspell Consumedir Cleaner - v0.1 beta"
|
||||
echo " by totti4ever" && echo
|
||||
echo " $(date)"
|
||||
echo
|
||||
echo "#################################################"
|
||||
echo && echo
|
||||
|
||||
CURL_CMD="curl"
|
||||
JQ_CMD="jq"
|
||||
|
||||
"$JQ_CMD" --version > /dev/null
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "please install 'jq'"
|
||||
exit -4
|
||||
fi
|
||||
|
||||
ds_url=${1%/}
|
||||
ds_user_param=$2
|
||||
ds_user=${ds_user_param#*/}
|
||||
ds_collective=${ds_user_param%%/*}
|
||||
ds_password=$3
|
||||
ds_consumedir_path=${4%/}
|
||||
ds_archive_path=$ds_consumedir_path/_archive/$ds_collective
|
||||
|
||||
|
||||
if [ $# -ne 4 ]; then
|
||||
echo "FATAL Exactly four parameters needed"
|
||||
exit -3
|
||||
elif [ "$1" == "" ] || [ "$2" == "" ] || [ "$3" == "" ] || [ "$4" == "" ]; then
|
||||
echo "FATAL Parameter missing"
|
||||
echo " ds_url: $ds_url"
|
||||
echo " ds_user: $ds_user"
|
||||
echo " ds_password: $ds_password"
|
||||
echo " ds_consumedir_path: $ds_consumedir_path"
|
||||
exit -2
|
||||
elif [ "$ds_collective" == "_archive" ]; then
|
||||
echo "FATAL collective name '_archive' is not supported by this script"
|
||||
exit -1
|
||||
fi
|
||||
|
||||
|
||||
############# FUNCTIONS
|
||||
function curl_call() {
|
||||
curl_cmd="$CURL_CMD $1 -H 'X-Docspell-Auth: $ds_token'"
|
||||
curl_result=$(eval $curl_cmd)
|
||||
curl_code=$?
|
||||
|
||||
if [ "$curl_result" == '"Authentication failed."' ] || [ "$curl_result" == 'Response timed out' ]; then
|
||||
printf "\nNew login required ($curl_result)... "
|
||||
login
|
||||
printf "%${#len_resultset}s" " "; printf " .."
|
||||
curl_call $1
|
||||
|
||||
elif [ "$curl_result" == "Bad Gateway" ] || [ "$curl_result" == '404 page not found' ]; then
|
||||
echo "FATAL Connection to server failed"
|
||||
exit -1
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
function login() {
|
||||
curl_call "-s -X POST -d '{\"account\": \"$ds_collective/$ds_user\", \"password\": \"$ds_password\"}' ${ds_url}/api/v1/open/auth/login"
|
||||
|
||||
curl_status=$(echo $curl_result | $JQ_CMD -r ".success")
|
||||
|
||||
if [ "$curl_status" == "true" ]; then
|
||||
ds_token=$(echo $curl_result | $JQ_CMD -r ".token")
|
||||
echo "Login successfull ( Token: $ds_token )"
|
||||
|
||||
else
|
||||
echo "FATAL Login not succesfull"
|
||||
exit 1
|
||||
|
||||
fi
|
||||
}
|
||||
|
||||
############# END
|
||||
|
||||
echo "Settings:"
|
||||
if [ "$DS_CC_REMOVE" == "true" ]; then
|
||||
echo " ### !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ###"
|
||||
echo " - DELETE files? YES"
|
||||
echo " when already existing in Docspell. This cannot be undone!"
|
||||
echo " ### !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ###"
|
||||
else
|
||||
echo " - DELETE files? no"
|
||||
echo " moving already uploaded files to archive"
|
||||
fi
|
||||
echo
|
||||
if [ "$DS_CC_UPLOAD_MISSING" == true ]; then
|
||||
echo " - UPLOAD files? YES"
|
||||
echo " files not existing in Docspell will be uploaded and will be re-checked in the next run."
|
||||
else
|
||||
echo " - UPLOAD files? no"
|
||||
echo " files not existing in Docspell will NOT be uploaded and stay where they are."
|
||||
fi
|
||||
echo && echo
|
||||
echo "Press 'ctrl+c' to cancel"
|
||||
for ((i=9;i>=0;i--)); do
|
||||
printf "\r waiting $i seconds "
|
||||
sleep 1s
|
||||
done
|
||||
echo && echo
|
||||
|
||||
# login, get token
|
||||
login
|
||||
|
||||
echo "Scanning folder for collective '$ds_collective' ($ds_consumedir_path/$ds_collective)"
|
||||
echo && echo
|
||||
|
||||
while read -r line
|
||||
do
|
||||
tmp_filepath=$line
|
||||
|
||||
if [ "$tmp_filepath" == "" ]; then
|
||||
echo "no files found" && echo
|
||||
exit 0 #no results
|
||||
elif [ ! -f "$tmp_filepath" ]; then
|
||||
echo "FATAL no access to file: $tmp_filepath"
|
||||
exit 3
|
||||
fi
|
||||
|
||||
echo "Checking '$tmp_filepath'"
|
||||
printf "%${#len_resultset}s" " "; printf " "
|
||||
|
||||
# check for checksum
|
||||
tmp_checksum=$(sha256sum "$tmp_filepath" | awk '{print $1}')
|
||||
|
||||
curl_call "-s -X GET '$ds_url/api/v1/sec/checkfile/$tmp_checksum'"
|
||||
curl_status=$(echo $curl_result | $JQ_CMD -r ".exists")
|
||||
|
||||
if [ $curl_code -ne 0 ]; then
|
||||
# error
|
||||
echo "ERROR $curl_result // $curl_status"
|
||||
|
||||
# file exists in Docspell
|
||||
elif [ "$curl_status" == "true" ]; then
|
||||
item_name=$(echo $curl_result | $JQ_CMD -r ".items[0].name")
|
||||
item_id=$(echo $curl_result | $JQ_CMD -r ".items[0].id")
|
||||
echo "File already exists: '$item_name (ID: $item_id)'"
|
||||
|
||||
printf "%${#len_resultset}s" " "; printf " "
|
||||
if [ "$DS_CC_REMOVE" == "true" ]; then
|
||||
echo "... removing file"
|
||||
rm "$tmp_filepath"
|
||||
else
|
||||
created=$(echo $curl_result | $JQ_CMD -r ".items[0].created")
|
||||
cur_dir="$ds_archive_path/$(date -d @$(expr \( $created + 500 \) / 1000) +%Y-%m)"
|
||||
echo "... moving to archive by month added ('$cur_dir')"
|
||||
mkdir -p "$cur_dir"
|
||||
mv "$tmp_filepath" "$cur_dir/"
|
||||
fi
|
||||
|
||||
# file does not exist in Docspell
|
||||
else
|
||||
|
||||
echo "Files does not exist, yet"
|
||||
if [ "$DS_CC_UPLOAD_MISSING" == true ]; then
|
||||
printf "%${#len_resultset}s" " "; printf " "
|
||||
printf "...uploading file.."
|
||||
curl_call "-s -X POST '$ds_url/api/v1/sec/upload/item' -H 'Content-Type: multipart/form-data' -F 'file=@$tmp_filepath'"
|
||||
curl_status=$(echo $curl_result | $JQ_CMD -r ".success")
|
||||
if [ "$curl_status" == "true" ]; then
|
||||
echo ". done"
|
||||
else
|
||||
echo -e "\nERROR $curl_result"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
echo
|
||||
done \
|
||||
<<< $(find $ds_consumedir_path/$ds_collective -type f)
|
||||
|
||||
|
||||
echo ################# DONE #################
|
||||
date
|
||||
@ -1,439 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# This script watches a directory for new files and uploads them to
|
||||
# docspell. Or it uploads all files currently in the directory.
|
||||
#
|
||||
# It requires inotifywait, curl and sha256sum if the `-m' option is
|
||||
# used.
|
||||
|
||||
# saner programming env: these switches turn some bugs into errors
|
||||
set -o errexit -o pipefail -o noclobber -o nounset
|
||||
|
||||
CURL_CMD="curl"
|
||||
INOTIFY_CMD="inotifywait"
|
||||
SHA256_CMD="sha256sum"
|
||||
MKTEMP_CMD="mktemp"
|
||||
CURL_OPTS=${CURL_OPTS:-}
|
||||
|
||||
! getopt --test > /dev/null
|
||||
if [[ ${PIPESTATUS[0]} -ne 4 ]]; then
|
||||
echo 'I’m sorry, `getopt --test` failed in this environment.'
|
||||
exit 1
|
||||
fi
|
||||
|
||||
OPTIONS=omhdp:vrmi
|
||||
LONGOPTS=once,distinct,help,delete,path:,verbose,recursive,dry,integration,iuser:,iheader:,poll:,exclude:,include:
|
||||
|
||||
! PARSED=$(getopt --options=$OPTIONS --longoptions=$LONGOPTS --name "$0" -- "$@")
|
||||
if [[ ${PIPESTATUS[0]} -ne 0 ]]; then
|
||||
# e.g. return value is 1
|
||||
# then getopt has complained about wrong arguments to stdout
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# read getopt’s output this way to handle the quoting right:
|
||||
eval set -- "$PARSED"
|
||||
|
||||
declare -a watchdir
|
||||
help=n verbose=n delete=n once=n distinct=n recursive=n dryrun=n
|
||||
integration=n iuser="" iheader="" poll="" exclude="" include=""
|
||||
while true; do
|
||||
case "$1" in
|
||||
-h|--help)
|
||||
help=y
|
||||
shift
|
||||
;;
|
||||
-v|--verbose)
|
||||
verbose=y
|
||||
shift
|
||||
;;
|
||||
-d|--delete)
|
||||
delete=y
|
||||
shift
|
||||
;;
|
||||
-o|--once)
|
||||
once=y
|
||||
shift
|
||||
;;
|
||||
-p|--path)
|
||||
watchdir+=("$2")
|
||||
shift 2
|
||||
;;
|
||||
-m|--distinct)
|
||||
distinct=y
|
||||
shift
|
||||
;;
|
||||
-r|--recursive)
|
||||
recursive=y
|
||||
shift
|
||||
;;
|
||||
--dry)
|
||||
dryrun=y
|
||||
shift
|
||||
;;
|
||||
-i|--integration)
|
||||
integration=y
|
||||
recursive=y
|
||||
shift
|
||||
;;
|
||||
--iuser)
|
||||
iuser="$2"
|
||||
shift 2
|
||||
;;
|
||||
--iheader)
|
||||
iheader="$2"
|
||||
shift 2
|
||||
;;
|
||||
--poll)
|
||||
poll="$2"
|
||||
shift 2
|
||||
;;
|
||||
--exclude)
|
||||
exclude="$2"
|
||||
shift 2
|
||||
;;
|
||||
--include)
|
||||
include="$2"
|
||||
shift 2
|
||||
;;
|
||||
--)
|
||||
shift
|
||||
break
|
||||
;;
|
||||
*)
|
||||
echo "Programming error"
|
||||
exit 3
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
|
||||
showUsage() {
|
||||
echo "Upload files in a directory"
|
||||
echo ""
|
||||
echo "Usage: $0 [options] url url ..."
|
||||
echo
|
||||
echo "Options:"
|
||||
echo " -v | --verbose Print more to stdout. (value: $verbose)"
|
||||
echo " -d | --delete Delete the file if successfully uploaded. (value: $delete)"
|
||||
echo " -p | --path <dir> The directories to watch. This is required. (value: ${watchdir[@]})"
|
||||
echo " -h | --help Prints this help text. (value: $help)"
|
||||
echo " -m | --distinct Optional. Upload only if the file doesn't already exist. (value: $distinct)"
|
||||
echo " -o | --once Instead of watching, upload all files in that dir. (value: $once)"
|
||||
echo " --poll <sec> Run the script periodically instead of watching a directory. This can be"
|
||||
echo " used if watching via inotify is not possible. (value: $poll)"
|
||||
echo " -r | --recursive Traverse the directory(ies) recursively (value: $recursive)"
|
||||
echo " -i | --integration Upload to the integration endpoint. It implies -r. This puts the script in"
|
||||
echo " a different mode, where the first subdirectory of any given starting point"
|
||||
echo " is read as the collective name. The url(s) are completed with this name in"
|
||||
echo " order to upload files to the respective collective. So each directory"
|
||||
echo " given is expected to contain one subdirectory per collective and the urls"
|
||||
echo " are expected to identify the integration endpoint, which is"
|
||||
echo " /api/v1/open/integration/item/<collective-name>. (value: $integration)"
|
||||
echo " --iheader The header name and value to use with the integration endpoint. This must be"
|
||||
echo " in form 'headername:value'. Only used if '-i' is supplied."
|
||||
echo " (value: $iheader)"
|
||||
echo " --iuser The username and password for basic auth to use with the integration"
|
||||
echo " endpoint. This must be of form 'user:pass'. Only used if '-i' is supplied."
|
||||
echo " (value: $iuser)"
|
||||
echo " --exclude <glob> A shell glob pattern that is used to skip files that match (value: $exclude)."
|
||||
echo " --include <glob> A shell glob pattern that is used to find files to upload (value: $include)."
|
||||
echo " If --exclude and --include is given, both apply."
|
||||
echo " --dry Do a 'dry run', not uploading anything only printing to stdout (value: $dryrun)"
|
||||
echo ""
|
||||
echo "Arguments:"
|
||||
echo " A list of URLs to upload the files to."
|
||||
echo ""
|
||||
echo "Example: Watch directory"
|
||||
echo "$0 --path ~/Downloads -m -dv http://localhost:7880/api/v1/open/upload/item/abcde-12345-abcde-12345"
|
||||
echo ""
|
||||
echo "Example: Upload all files in a directory"
|
||||
echo "$0 --path ~/Downloads -m -dv --once http://localhost:7880/api/v1/open/upload/item/abcde-12345-abcde-12345"
|
||||
echo ""
|
||||
echo "Example: Integration Endpoint"
|
||||
echo "$0 -i --iheader 'Docspell-Integration:test123' -m -p ~/Downloads/ http://localhost:7880/api/v1/open/integration/item"
|
||||
echo ""
|
||||
}
|
||||
|
||||
if [ "$help" = "y" ]; then
|
||||
showUsage
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# handle non-option arguments
|
||||
if [[ $# -eq 0 ]]; then
|
||||
echo "$0: No upload URLs given."
|
||||
exit 4
|
||||
fi
|
||||
urls=$@
|
||||
|
||||
if [ ! -d "$watchdir" ]; then
|
||||
echo "The path '$watchdir' is not a directory."
|
||||
exit 4
|
||||
fi
|
||||
|
||||
|
||||
trace() {
|
||||
if [ "$verbose" = "y" ]; then
|
||||
>&2 echo "$1"
|
||||
fi
|
||||
}
|
||||
|
||||
info() {
|
||||
>&2 echo $1
|
||||
}
|
||||
|
||||
getCollective() {
|
||||
file=$(realpath "$1")
|
||||
dir=$(realpath "$2")
|
||||
collective=${file#"$dir"}
|
||||
coll=$(echo $collective | cut -d'/' -f1)
|
||||
if [ -z "$coll" ]; then
|
||||
coll=$(echo $collective | cut -d'/' -f2)
|
||||
fi
|
||||
echo $coll
|
||||
}
|
||||
|
||||
|
||||
upload() {
|
||||
dir=$(realpath "$1")
|
||||
file=$(realpath "$2")
|
||||
url="$3"
|
||||
OPTS="$CURL_OPTS"
|
||||
if [ "$integration" = "y" ]; then
|
||||
collective=$(getCollective "$file" "$dir")
|
||||
trace "- upload: collective = $collective"
|
||||
url="$url/$collective"
|
||||
if [ $iuser ]; then
|
||||
OPTS="$OPTS --user $iuser"
|
||||
fi
|
||||
if [ $iheader ]; then
|
||||
OPTS="$OPTS -H $iheader"
|
||||
fi
|
||||
fi
|
||||
if [ "$dryrun" = "y" ]; then
|
||||
info "- Not uploading (dry-run) $file to $url with opts $OPTS"
|
||||
else
|
||||
META1=""
|
||||
META2=""
|
||||
if [ "$distinct" = "y" ]; then
|
||||
META1="-F"
|
||||
META2="meta={\"multiple\": false, \"skipDuplicates\": true}"
|
||||
fi
|
||||
trace "- Uploading $file to $url with options $OPTS"
|
||||
tf1=$($MKTEMP_CMD) tf2=$($MKTEMP_CMD) rc=0
|
||||
$CURL_CMD --fail -# -o "$tf1" --stderr "$tf2" $OPTS -XPOST $META1 "$META2" -F file=@"$file" "$url"
|
||||
if [ $? -ne 0 ]; then
|
||||
info "Upload failed. Exit code: $rc"
|
||||
cat "$tf1"
|
||||
cat "$tf2"
|
||||
echo ""
|
||||
rm "$tf1" "$tf2"
|
||||
return $rc
|
||||
else
|
||||
if cat $tf1 | grep -q '{"success":false'; then
|
||||
echo "Upload failed. Message from server:"
|
||||
cat "$tf1"
|
||||
echo ""
|
||||
rm "$tf1" "$tf2"
|
||||
return 1
|
||||
else
|
||||
info "- Upload done."
|
||||
rm "$tf1" "$tf2"
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
checksum() {
|
||||
$SHA256_CMD "$1" | cut -d' ' -f1 | xargs
|
||||
}
|
||||
|
||||
checkFile() {
|
||||
local url="$1"
|
||||
local file="$2"
|
||||
local dir="$3"
|
||||
OPTS="$CURL_OPTS"
|
||||
if [ "$integration" = "y" ]; then
|
||||
collective=$(getCollective "$file" "$dir")
|
||||
url="$url/$collective"
|
||||
url=$(echo "$url" | sed 's,/item/,/checkfile/,g')
|
||||
if [ $iuser ]; then
|
||||
OPTS="$OPTS --user $iuser"
|
||||
fi
|
||||
if [ $iheader ]; then
|
||||
OPTS="$OPTS -H $iheader"
|
||||
fi
|
||||
else
|
||||
url=$(echo "$1" | sed 's,upload/item,checkfile,g')
|
||||
fi
|
||||
url=$url/$(checksum "$file")
|
||||
trace "- Check file via $OPTS: $url"
|
||||
tf1=$($MKTEMP_CMD) tf2=$($MKTEMP_CMD)
|
||||
$CURL_CMD --fail -v -o "$tf1" --stderr "$tf2" $OPTS -XGET -s "$url"
|
||||
if [ $? -ne 0 ]; then
|
||||
info "Checking file failed!"
|
||||
cat "$tf1" >&2
|
||||
cat "$tf2" >&2
|
||||
info ""
|
||||
rm "$tf1" "$tf2"
|
||||
echo "failed"
|
||||
return 1
|
||||
else
|
||||
if cat "$tf1" | grep -q '{"exists":true'; then
|
||||
rm "$tf1" "$tf2"
|
||||
echo "y"
|
||||
else
|
||||
rm "$tf1" "$tf2"
|
||||
echo "n"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
process() {
|
||||
file=$(realpath "$1")
|
||||
dir="$2"
|
||||
info "---- Processing $file ----------"
|
||||
declare -i curlrc=0
|
||||
set +e
|
||||
for url in $urls; do
|
||||
if [ "$distinct" = "y" ]; then
|
||||
trace "- Checking if $file has been uploaded to $url already"
|
||||
res=$(checkFile "$url" "$file" "$dir")
|
||||
rc=$?
|
||||
curlrc=$(expr $curlrc + $rc)
|
||||
trace "- Result from checkfile: $res"
|
||||
if [ "$res" = "y" ]; then
|
||||
info "- Skipping file '$file' because it has been uploaded in the past."
|
||||
continue
|
||||
elif [ "$res" != "n" ]; then
|
||||
info "- Checking file failed, skipping the file."
|
||||
continue
|
||||
fi
|
||||
fi
|
||||
trace "- Uploading '$file' to '$url'."
|
||||
upload "$dir" "$file" "$url"
|
||||
rc=$?
|
||||
curlrc=$(expr $curlrc + $rc)
|
||||
if [ $rc -ne 0 ]; then
|
||||
trace "Upload to '$url' failed!"
|
||||
fi
|
||||
done
|
||||
set -e
|
||||
if [ $curlrc -ne 0 ]; then
|
||||
info "-> Some uploads failed."
|
||||
else
|
||||
trace "= File processed for all URLs"
|
||||
if [ "$delete" = "y" ]; then
|
||||
info "- Deleting file '$file'"
|
||||
set +e
|
||||
rm "$file"
|
||||
if [ $? -ne 0 ]; then
|
||||
info "- Deleting failed!"
|
||||
fi
|
||||
set -e
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
findDir() {
|
||||
path="$1"
|
||||
for dir in "${watchdir[@]}"; do
|
||||
if [[ $path = ${dir}* ]]
|
||||
then
|
||||
echo $dir
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
checkSetup() {
|
||||
for dir in "${watchdir[@]}"; do
|
||||
find "$dir" -mindepth 1 -maxdepth 1 -type d -print0 | while IFS= read -d '' -r collective; do
|
||||
for url in $urls; do
|
||||
if [ "$integration" = "y" ]; then
|
||||
url="$url/$(basename $collective)"
|
||||
OPTS="$CURL_OPTS -i -s -o /dev/null -w %{http_code}"
|
||||
if [ $iuser ]; then
|
||||
OPTS="$OPTS --user $iuser"
|
||||
fi
|
||||
if [ $iheader ]; then
|
||||
OPTS="$OPTS -H $iheader"
|
||||
fi
|
||||
trace "Checking integration endpoint: $CURL_CMD $OPTS "$url""
|
||||
status=$($CURL_CMD $OPTS "$url")
|
||||
if [ "$status" != "200" ]; then
|
||||
echo "[WARN] Collective '$(basename $collective)' failed the setup check."
|
||||
echo "[WARN] $status response, command: $CURL_CMD $OPTS $url"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
done
|
||||
done
|
||||
}
|
||||
|
||||
runOnce() {
|
||||
info "Uploading all files (except hidden) in '$watchdir'."
|
||||
MD="-maxdepth 1"
|
||||
if [ "$recursive" = "y" ]; then
|
||||
MD=""
|
||||
fi
|
||||
EXCL=""
|
||||
if [ -n "$exclude" ]; then
|
||||
EXCL="-not -name $exclude"
|
||||
fi
|
||||
INCL=""
|
||||
if [ -n "$include" ]; then
|
||||
INCL="-name $include"
|
||||
fi
|
||||
for dir in "${watchdir[@]}"; do
|
||||
find "$dir" $MD -type f $INCL $EXCL -not -name ".*" -print0 | while IFS= read -d '' -r file; do
|
||||
process "$file" "$dir"
|
||||
done
|
||||
done
|
||||
}
|
||||
|
||||
includeFile() {
|
||||
file="$1"
|
||||
if [ -n "$include" ] && [[ $file != $include ]]; then
|
||||
trace "Skip $file due to include filter"
|
||||
return 1
|
||||
elif [ -n "$exclude" ] && [[ $file == $exclude ]]; then
|
||||
trace "Skip $file due to exclude filter"
|
||||
return 1
|
||||
else
|
||||
[[ "$file" != .* ]]
|
||||
fi
|
||||
}
|
||||
|
||||
# warn if something seems not correctly configured
|
||||
checkSetup
|
||||
|
||||
if [ "$once" = "y" ]; then
|
||||
runOnce
|
||||
else
|
||||
REC=""
|
||||
if [ "$recursive" = "y" ]; then
|
||||
REC="-r"
|
||||
fi
|
||||
if [ -z "$poll" ]; then
|
||||
$INOTIFY_CMD $REC -m --format '%w%f' -e close_write -e moved_to "${watchdir[@]}" |
|
||||
while read pathfile; do
|
||||
if includeFile "$(basename "$pathfile")"; then
|
||||
dir=$(findDir "$pathfile")
|
||||
trace "The file '$pathfile' appeared below '$dir'"
|
||||
sleep 1
|
||||
process "$(realpath "$pathfile")" "$dir"
|
||||
else
|
||||
trace "Skip file $(realpath "$pathfile")"
|
||||
fi
|
||||
done
|
||||
else
|
||||
echo "Running in polling mode: ${poll}s"
|
||||
while [ : ]
|
||||
do
|
||||
runOnce
|
||||
sleep $poll
|
||||
done
|
||||
fi
|
||||
fi
|
||||
@ -1,32 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Simple script to authenticate with docspell and trigger the "convert
|
||||
# all pdf" route that submits a task to convert all pdf files using
|
||||
# ocrmypdf.
|
||||
|
||||
set -e
|
||||
|
||||
CURL_CMD="curl"
|
||||
JQ_CMD="jq"
|
||||
|
||||
BASE_URL="${1:-http://localhost:7880}"
|
||||
LOGIN_URL="$BASE_URL/api/v1/open/auth/login"
|
||||
TRIGGER_URL="$BASE_URL/api/v1/sec/item/convertallpdfs"
|
||||
|
||||
echo "Login to trigger converting all pdfs."
|
||||
echo "Using url: $BASE_URL"
|
||||
echo -n "Account: "
|
||||
read USER
|
||||
echo -n "Password: "
|
||||
read -s PASS
|
||||
echo
|
||||
|
||||
auth=$("$CURL_CMD" --fail -XPOST --silent --data-binary "{\"account\":\"$USER\", \"password\":\"$PASS\"}" "$LOGIN_URL")
|
||||
|
||||
if [ "$(echo $auth | "$JQ_CMD" .success)" == "true" ]; then
|
||||
echo "Login successful"
|
||||
auth_token=$(echo $auth | "$JQ_CMD" -r .token)
|
||||
"$CURL_CMD" --fail -XPOST -H "X-Docspell-Auth: $auth_token" "$TRIGGER_URL"
|
||||
else
|
||||
echo "Login failed."
|
||||
fi
|
||||
@ -1,213 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Script for downloading files (the PDF versions) flat in the current
|
||||
# directory. It takes a search query for selecting what to download.
|
||||
# Metadata is not downloaded, only the files.
|
||||
#
|
||||
# Usage:
|
||||
#
|
||||
# download-files.sh <docspell-base-url> <query>
|
||||
#
|
||||
# The docspell base url is required as well as a search query. The
|
||||
# output directory is the current directory, and can be defined via
|
||||
# env variable "TARGET_DIR".
|
||||
#
|
||||
# Example:
|
||||
#
|
||||
# download-files.sh http://localhost:7880 "tag:todo folder:work"
|
||||
#
|
||||
# The script then asks for username and password and starts
|
||||
# downloading. For more details about the query, please see the docs
|
||||
# here: https://docspell.org/docs/query/
|
||||
|
||||
CURL_CMD="curl"
|
||||
JQ_CMD="jq"
|
||||
|
||||
|
||||
if [ -z "$1" ]; then
|
||||
echo "The base-url to docspell is required."
|
||||
exit 1
|
||||
else
|
||||
BASE_URL="$1"
|
||||
shift
|
||||
fi
|
||||
|
||||
if [ -z "$1" ]; then
|
||||
errout "A search query is required"
|
||||
exit 1
|
||||
else
|
||||
QUERY="$1"
|
||||
shift
|
||||
fi
|
||||
|
||||
set -o errexit -o pipefail -o noclobber -o nounset
|
||||
|
||||
LOGIN_URL="$BASE_URL/api/v1/open/auth/login"
|
||||
SEARCH_URL="$BASE_URL/api/v1/sec/item/search"
|
||||
DETAIL_URL="$BASE_URL/api/v1/sec/item"
|
||||
ATTACH_URL="$BASE_URL/api/v1/sec/attachment"
|
||||
|
||||
OVERWRITE_FILE=${OVERWRITE_FILE:-n}
|
||||
TARGET=${TARGET_DIR:-"$(pwd)"}
|
||||
|
||||
errout() {
|
||||
>&2 echo "$@"
|
||||
}
|
||||
|
||||
trap "{ rm -f ${TMPDIR-:/tmp}/ds-download.*; }" EXIT
|
||||
|
||||
mcurl() {
|
||||
tmpfile1=$(mktemp -t "ds-download.XXXXX")
|
||||
tmpfile2=$(mktemp -t "ds-download.XXXXX")
|
||||
set +e
|
||||
"$CURL_CMD" -# --fail --stderr "$tmpfile1" -o "$tmpfile2" -H "X-Docspell-Auth: $auth_token" "$@"
|
||||
status=$?
|
||||
set -e
|
||||
if [ $status -ne 0 ]; then
|
||||
errout "$CURL_CMD -H 'X-Docspell-Auth: …' $@"
|
||||
errout "curl command failed (rc=$status)! Output is below."
|
||||
cat "$tmpfile1" >&2
|
||||
cat "$tmpfile2" >&2
|
||||
rm -f "$tmpfile1" "$tmpfile2"
|
||||
return 2
|
||||
else
|
||||
ret=$(cat "$tmpfile2")
|
||||
rm "$tmpfile2" "$tmpfile1"
|
||||
echo $ret
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
errout "Login to Docspell."
|
||||
errout "Using url: $BASE_URL"
|
||||
if [ -z "${DS_USER:-}" ]; then
|
||||
errout -n "Account: "
|
||||
read DS_USER
|
||||
fi
|
||||
if [ -z "${DS_PASS:-}" ]; then
|
||||
errout -n "Password: "
|
||||
read -s DS_PASS
|
||||
fi
|
||||
echo
|
||||
|
||||
declare auth
|
||||
declare auth_token
|
||||
declare auth_time
|
||||
|
||||
|
||||
login() {
|
||||
auth=$("$CURL_CMD" -s --fail -XPOST \
|
||||
--data-binary "{\"account\":\"$DS_USER\", \"password\":\"$DS_PASS\"}" "$LOGIN_URL")
|
||||
|
||||
if [ "$(echo $auth | "$JQ_CMD" .success)" == "true" ]; then
|
||||
errout "Login successful"
|
||||
auth_token=$(echo $auth | "$JQ_CMD" -r .token)
|
||||
auth_time=$(date +%s)
|
||||
else
|
||||
errout "Login failed."
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
checkLogin() {
|
||||
elapsed=$((1000 * ($(date +%s) - $auth_time)))
|
||||
maxtime=$(echo $auth | "$JQ_CMD" .validMs)
|
||||
|
||||
elapsed=$(($elapsed + 1000))
|
||||
if [ $elapsed -gt $maxtime ]; then
|
||||
errout "Need to re-login $elapsed > $maxtime"
|
||||
login
|
||||
fi
|
||||
}
|
||||
|
||||
listItems() {
|
||||
OFFSET="${1:-0}"
|
||||
LIMIT="${2:-50}"
|
||||
QUERY="$3"
|
||||
errout "Get next items with offset=$OFFSET, limit=$LIMIT"
|
||||
REQ="{\"offset\":$OFFSET, \"limit\":$LIMIT, \"query\":\" $QUERY \"}"
|
||||
|
||||
mcurl -XPOST -H 'ContentType: application/json' -d "$REQ" "$SEARCH_URL" | "$JQ_CMD" -r '.groups[].items[]|.id'
|
||||
}
|
||||
|
||||
|
||||
fetchItem() {
|
||||
mcurl -XGET "$DETAIL_URL/$1"
|
||||
}
|
||||
|
||||
downloadAttachment() {
|
||||
attachId="$1"
|
||||
errout " - Download '$attachName' ($attachId)"
|
||||
|
||||
if [ -f "$attachOut" ] && [ "$SKIP_FILE" == "y" ]; then
|
||||
errout " - Skipping file '$attachOut' since it already exists"
|
||||
else
|
||||
if [ -f "$attachOut" ] && [ "$OVERWRITE_FILE" == "y" ]; then
|
||||
errout " - Removing attachment file as requested: $attachOut"
|
||||
rm -f "$attachOut"
|
||||
fi
|
||||
|
||||
DL_URL="$ATTACH_URL/$attachId"
|
||||
|
||||
checksum1=$("$CURL_CMD" -s -I -H "X-Docspell-Auth: $auth_token" "$DL_URL" | \
|
||||
grep -i 'etag' | cut -d' ' -f2 | xargs | tr -d '\r')
|
||||
"$CURL_CMD" -s -o "$attachOut" -H "X-Docspell-Auth: $auth_token" "$DL_URL"
|
||||
checksum2=$(sha256sum "$attachOut" | cut -d' ' -f1 | xargs)
|
||||
if [ "$checksum1" == "$checksum2" ]; then
|
||||
errout " - Checksum ok."
|
||||
else
|
||||
errout " - WARNING: Checksum mismatch! Server: $checksum1 Downloaded: $checksum2"
|
||||
return 3
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
downloadItem() {
|
||||
checkLogin
|
||||
itemData=$(fetchItem "$1")
|
||||
errout "Get item $(echo $itemData | "$JQ_CMD" -r .id)"
|
||||
created=$(echo $itemData|"$JQ_CMD" '.created')
|
||||
created=$((($(echo $itemData|"$JQ_CMD" '.created') + 500) / 1000))
|
||||
itemId=$(echo $itemData | "$JQ_CMD" -r '.id')
|
||||
# out="$TARGET/$(date -d @$created +%Y-%m)/$itemId"
|
||||
out="$TARGET"
|
||||
|
||||
if [ -d "$out" ] && [ "${DROP_ITEM:-}" == "y" ]; then
|
||||
errout "Removing item folder as requested: $out"
|
||||
rm -rf "$out"
|
||||
fi
|
||||
|
||||
mkdir -p "$out"
|
||||
|
||||
while read attachId attachName; do
|
||||
attachOut="$out/$attachName"
|
||||
checkLogin
|
||||
downloadAttachment "$attachId"
|
||||
done < <(echo $itemData | "$JQ_CMD" -r '.attachments[] | [.id,.name] | join(" ")')
|
||||
}
|
||||
|
||||
login
|
||||
|
||||
errout "Downloading files…"
|
||||
|
||||
allCounter=0 innerCounter=0 limit=100 offset=0 done=n
|
||||
|
||||
while [ "$done" = "n" ]; do
|
||||
checkLogin
|
||||
|
||||
innerCounter=0
|
||||
while read id; do
|
||||
downloadItem "$id"
|
||||
innerCounter=$(($innerCounter + 1))
|
||||
done < <(listItems $offset $limit "$QUERY")
|
||||
|
||||
allCounter=$(($allCounter + $innerCounter))
|
||||
offset=$(($offset + $limit))
|
||||
|
||||
|
||||
if [ $innerCounter -lt $limit ]; then
|
||||
done=y
|
||||
fi
|
||||
|
||||
done
|
||||
errout "Downloaded $allCounter items"
|
||||
199
tools/ds.sh
199
tools/ds.sh
@ -1,199 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# A simple bash script that reads a configuration file to know where
|
||||
# to upload a given file.
|
||||
#
|
||||
# The config file contains anonymous upload urls to docspell. All
|
||||
# files given to this script are uploaded to all those urls.
|
||||
#
|
||||
# The default location for the config file is
|
||||
# `~/.config/docspell/ds.conf'.
|
||||
#
|
||||
# The config file must contain lines of the form:
|
||||
#
|
||||
# url.1=http://localhost:7880/api/v1/open/upload/item/<source-id>
|
||||
# url.2=...
|
||||
#
|
||||
# Lines starting with a `#' are ignored.
|
||||
#
|
||||
# The `-e|--exists' option allows to skip uploading and only check
|
||||
# whether a given file exists in docspell.
|
||||
|
||||
# saner programming env: these switches turn some bugs into errors
|
||||
set -o errexit -o pipefail -o noclobber -o nounset
|
||||
|
||||
CURL_CMD="curl"
|
||||
GREP_CMD="grep"
|
||||
MKTEMP_CMD="mktemp"
|
||||
SHA256_CMD="sha256sum"
|
||||
|
||||
! getopt --test > /dev/null
|
||||
if [[ ${PIPESTATUS[0]} -ne 4 ]]; then
|
||||
echo 'I’m sorry, `getopt --test` failed in this environment.'
|
||||
exit 1
|
||||
fi
|
||||
|
||||
OPTIONS=c:hsde
|
||||
LONGOPTS=config:,help,skip,delete,exists,allow-duplicates
|
||||
|
||||
! PARSED=$(getopt --options=$OPTIONS --longoptions=$LONGOPTS --name "$0" -- "$@")
|
||||
if [[ ${PIPESTATUS[0]} -ne 0 ]]; then
|
||||
# e.g. return value is 1
|
||||
# then getopt has complained about wrong arguments to stdout
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# read getopt’s output this way to handle the quoting right:
|
||||
eval set -- "$PARSED"
|
||||
|
||||
exists=n delete=n help=n config="${XDG_CONFIG_HOME:-$HOME/.config}/docspell/ds.conf" dupes=n
|
||||
while true; do
|
||||
case "$1" in
|
||||
-h|--help)
|
||||
help=y
|
||||
shift
|
||||
;;
|
||||
-c|--config)
|
||||
config="$2"
|
||||
shift 2
|
||||
;;
|
||||
-d|--delete)
|
||||
delete="y"
|
||||
shift
|
||||
;;
|
||||
-e|--exists)
|
||||
exists=y
|
||||
shift
|
||||
;;
|
||||
--allow-duplicates)
|
||||
dupes=y
|
||||
shift
|
||||
;;
|
||||
--)
|
||||
shift
|
||||
break
|
||||
;;
|
||||
*)
|
||||
echo "Programming error"
|
||||
exit 3
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
|
||||
info() {
|
||||
echo "$1"
|
||||
}
|
||||
|
||||
checksum() {
|
||||
$SHA256_CMD "$1" | cut -d' ' -f1 | xargs
|
||||
}
|
||||
|
||||
checkFile() {
|
||||
local url=$(echo "$1" | sed 's,upload/item,checkfile,g')
|
||||
local file="$2"
|
||||
$CURL_CMD -XGET -s "$url/$(checksum "$file")" | (2>&1 1>/dev/null grep '"exists":true')
|
||||
}
|
||||
|
||||
upload_file() {
|
||||
tf=$($MKTEMP_CMD) rc=0
|
||||
META1=""
|
||||
META2=""
|
||||
if [ "$dupes" = "y" ]; then
|
||||
META1="-F"
|
||||
META2="meta={\"multiple\": false, \"skipDuplicates\": false}"
|
||||
else
|
||||
META1="-F"
|
||||
META2="meta={\"multiple\": false, \"skipDuplicates\": true}"
|
||||
fi
|
||||
$CURL_CMD -# -o "$tf" --stderr "$tf" -w "%{http_code}" -XPOST $META1 "$META2" -F file=@"$1" "$2" | (2>&1 1>/dev/null grep 200)
|
||||
rc=$(expr $rc + $?)
|
||||
cat $tf | (2>&1 1>/dev/null grep '{"success":true')
|
||||
rc=$(expr $rc + $?)
|
||||
if [ $rc -ne 0 ]; then
|
||||
info "Upload failed. Exit code: $rc"
|
||||
cat "$tf"
|
||||
echo ""
|
||||
rm "$tf"
|
||||
return $rc
|
||||
else
|
||||
rm "$tf"
|
||||
return 0
|
||||
fi
|
||||
}
|
||||
|
||||
upload() {
|
||||
if [ "$dupes" == "y" ]; then
|
||||
upload_file "$1" "$2"
|
||||
else
|
||||
checkFile "$2" "$1"
|
||||
if [ $? -eq 0 ]; then
|
||||
info "File already exists at url $2"
|
||||
return 0
|
||||
else
|
||||
upload_file "$1" "$2"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
showUsage() {
|
||||
info "Upload files to docspell"
|
||||
info ""
|
||||
info "Usage: $0 [options] file [file ...]"
|
||||
info ""
|
||||
info "Options:"
|
||||
info " -c | --config Provide a config file. (value: $config)"
|
||||
info " -d | --delete Delete the files when successfully uploaded (value: $delete)"
|
||||
info " -h | --help Prints this help text. (value: $help)"
|
||||
info " -e | --exists Checks for the existence of a file instead of uploading (value: $exists)"
|
||||
info " --allow-duplicates Do not skip existing files in docspell (value: $dupes)"
|
||||
info ""
|
||||
info "Arguments:"
|
||||
info " One or more files to check for existence or upload."
|
||||
info ""
|
||||
}
|
||||
|
||||
if [ "$help" = "y" ]; then
|
||||
showUsage
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# handle non-option arguments
|
||||
if [[ $# -eq 0 ]]; then
|
||||
echo "$0: No files given."
|
||||
exit 4
|
||||
fi
|
||||
|
||||
|
||||
## Read the config file
|
||||
declare -a urls
|
||||
while IFS="=" read -r k v
|
||||
do
|
||||
if [[ $k == url* ]]; then
|
||||
urls+=($(echo "$v" | xargs))
|
||||
fi
|
||||
done <<< $($GREP_CMD -v '^#.*' "$config")
|
||||
|
||||
|
||||
## Main
|
||||
IFS=$'\n'
|
||||
for file in $*; do
|
||||
for url in "${urls[@]}"; do
|
||||
if [ "$exists" = "y" ]; then
|
||||
if checkFile "$url" "$file"; then
|
||||
info "$url $file: true"
|
||||
else
|
||||
info "$url $file: false"
|
||||
fi
|
||||
else
|
||||
info "Uploading '$file' to '$url'"
|
||||
set +e
|
||||
upload "$file" "$url"
|
||||
set -e
|
||||
if [ "$delete" = "y" ] && [ $? -eq 0 ]; then
|
||||
info "Deleting file: $file"
|
||||
rm -f "$file"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
done
|
||||
@ -1,256 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Simple script for downloading all your files. It goes through all
|
||||
# items visible to the logged in user and downloads the attachments
|
||||
# (the original files).
|
||||
#
|
||||
# The item's metadata are stored next to the files to provide more
|
||||
# information about the item: tags, dates, custom fields etc. This
|
||||
# contains most of your user supplied data.
|
||||
#
|
||||
# This script is intended for having your data outside and independent
|
||||
# of docspell. Another good idea for a backup strategy is to take
|
||||
# database dumps *and* storing the releases of docspell next to this
|
||||
# dump.
|
||||
#
|
||||
# Usage:
|
||||
#
|
||||
# export-files.sh <docspell-base-url> <target-directory>
|
||||
#
|
||||
# The docspell base url is required as well as a directory to store
|
||||
# all the files into.
|
||||
#
|
||||
# Example:
|
||||
#
|
||||
# export-files.sh http://localhost:7880 /tmp/ds-download
|
||||
#
|
||||
# The script then asks for username and password and starts
|
||||
# downloading. Files are downloaded into the following structure
|
||||
# (below the given target directory):
|
||||
#
|
||||
# - yyyy-mm (item date)
|
||||
# - A3…XY (item id)
|
||||
# - somefile.pdf (attachments with name)
|
||||
# - metadata.json (json file with items metadata)
|
||||
#
|
||||
# By default, files are not overwritten, it stops if existing files
|
||||
# are encountered. Configuration can be specified using environment
|
||||
# variables:
|
||||
#
|
||||
# - OVERWRITE_FILE= if `y` then overwriting existing files is ok.
|
||||
# - SKIP_FILE= if `y` then existing files are skipped (supersedes
|
||||
# OVERWRITE_FILE).
|
||||
# - DROP_ITEM= if `y` the item folder is removed before attempting to
|
||||
# download it. If this is set to `y` then the above options don't
|
||||
# make sense, since they operate on the files inside the item folder
|
||||
#
|
||||
# Docspell sends with each file its sha256 checksum via the ETag
|
||||
# header. This is used to do a integrity check after downloading.
|
||||
|
||||
CURL_CMD="curl"
|
||||
JQ_CMD="jq"
|
||||
|
||||
|
||||
if [ -z "$1" ]; then
|
||||
echo "The base-url to docspell is required."
|
||||
exit 1
|
||||
else
|
||||
BASE_URL="$1"
|
||||
shift
|
||||
fi
|
||||
|
||||
if [ -z "$1" ]; then
|
||||
echo "A directory is required to store the files into."
|
||||
exit 1
|
||||
else
|
||||
TARGET="$1"
|
||||
shift
|
||||
fi
|
||||
|
||||
set -o errexit -o pipefail -o noclobber -o nounset
|
||||
|
||||
LOGIN_URL="$BASE_URL/api/v1/open/auth/login"
|
||||
SEARCH_URL="$BASE_URL/api/v1/sec/item/search"
|
||||
INSIGHT_URL="$BASE_URL/api/v1/sec/collective/insights"
|
||||
DETAIL_URL="$BASE_URL/api/v1/sec/item"
|
||||
ATTACH_URL="$BASE_URL/api/v1/sec/attachment"
|
||||
|
||||
OVERWRITE_FILE=${OVERWRITE_FILE:-n}
|
||||
DROP_ITEM=${DROP_ITEM:-n}
|
||||
|
||||
errout() {
|
||||
>&2 echo "$@"
|
||||
}
|
||||
|
||||
trap "{ rm -f ${TMPDIR-:/tmp}/ds-export.*; }" EXIT
|
||||
|
||||
mcurl() {
|
||||
tmpfile1=$(mktemp -t "ds-export.XXXXX")
|
||||
tmpfile2=$(mktemp -t "ds-export.XXXXX")
|
||||
set +e
|
||||
"$CURL_CMD" -# --fail --stderr "$tmpfile1" -o "$tmpfile2" -H "X-Docspell-Auth: $auth_token" "$@"
|
||||
status=$?
|
||||
set -e
|
||||
if [ $status -ne 0 ]; then
|
||||
errout "$CURL_CMD -H 'X-Docspell-Auth: …' $@"
|
||||
errout "curl command failed (rc=$status)! Output is below."
|
||||
cat "$tmpfile1" >&2
|
||||
cat "$tmpfile2" >&2
|
||||
rm -f "$tmpfile1" "$tmpfile2"
|
||||
return 2
|
||||
else
|
||||
ret=$(cat "$tmpfile2")
|
||||
rm "$tmpfile2" "$tmpfile1"
|
||||
echo $ret
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
errout "Login to Docspell."
|
||||
errout "Using url: $BASE_URL"
|
||||
if [ -z "${DS_USER:-}" ]; then
|
||||
errout -n "Account: "
|
||||
read DS_USER
|
||||
fi
|
||||
if [ -z "${DS_PASS:-}" ]; then
|
||||
errout -n "Password: "
|
||||
read -s DS_PASS
|
||||
fi
|
||||
echo
|
||||
|
||||
declare auth
|
||||
declare auth_token
|
||||
declare auth_time
|
||||
|
||||
|
||||
login() {
|
||||
auth=$("$CURL_CMD" -s --fail -XPOST \
|
||||
--data-binary "{\"account\":\"$DS_USER\", \"password\":\"$DS_PASS\"}" "$LOGIN_URL")
|
||||
|
||||
if [ "$(echo $auth | "$JQ_CMD" .success)" == "true" ]; then
|
||||
errout "Login successful"
|
||||
auth_token=$(echo $auth | "$JQ_CMD" -r .token)
|
||||
auth_time=$(date +%s)
|
||||
else
|
||||
errout "Login failed."
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
checkLogin() {
|
||||
elapsed=$((1000 * ($(date +%s) - $auth_time)))
|
||||
maxtime=$(echo $auth | "$JQ_CMD" .validMs)
|
||||
|
||||
elapsed=$(($elapsed + 1000))
|
||||
if [ $elapsed -gt $maxtime ]; then
|
||||
errout "Need to re-login $elapsed > $maxtime"
|
||||
login
|
||||
fi
|
||||
}
|
||||
|
||||
listItems() {
|
||||
OFFSET="${1:-0}"
|
||||
LIMIT="${2:-50}"
|
||||
errout "Get next items with offset=$OFFSET, limit=$LIMIT"
|
||||
REQ="{\"offset\":$OFFSET, \"limit\":$LIMIT, \"withDetails\":true, \"query\":\"\"}"
|
||||
|
||||
mcurl -XPOST -H 'ContentType: application/json' -d "$REQ" "$SEARCH_URL" | "$JQ_CMD" -r '.groups[].items[]|.id'
|
||||
}
|
||||
|
||||
fetchItemCount() {
|
||||
mcurl -XGET "$INSIGHT_URL" | "$JQ_CMD" '[.incomingCount, .outgoingCount] | add'
|
||||
}
|
||||
|
||||
fetchItem() {
|
||||
mcurl -XGET "$DETAIL_URL/$1"
|
||||
}
|
||||
|
||||
downloadAttachment() {
|
||||
attachId="$1"
|
||||
errout " - Download '$attachName' ($attachId)"
|
||||
|
||||
if [ -f "$attachOut" ] && [ "$SKIP_FILE" == "y" ]; then
|
||||
errout " - Skipping file '$attachOut' since it already exists"
|
||||
else
|
||||
if [ -f "$attachOut" ] && [ "$OVERWRITE_FILE" == "y" ]; then
|
||||
errout " - Removing attachment file as requested: $attachOut"
|
||||
rm -f "$attachOut"
|
||||
fi
|
||||
|
||||
checksum1=$("$CURL_CMD" -s -I -H "X-Docspell-Auth: $auth_token" "$ATTACH_URL/$attachId/original" | \
|
||||
grep -i 'etag' | cut -d':' -f2 | xargs | tr -d '\r')
|
||||
"$CURL_CMD" -s -o "$attachOut" -H "X-Docspell-Auth: $auth_token" "$ATTACH_URL/$attachId/original"
|
||||
checksum2=$(sha256sum "$attachOut" | cut -d' ' -f1 | xargs)
|
||||
if [ "$checksum1" == "$checksum2" ]; then
|
||||
errout " - Checksum ok."
|
||||
else
|
||||
errout " - WARNING: Checksum mismatch! Server: $checksum1 Downloaded: $checksum2"
|
||||
return 3
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
downloadItem() {
|
||||
checkLogin
|
||||
itemData=$(fetchItem "$1")
|
||||
errout "Get item $(echo $itemData | "$JQ_CMD" -r .id)"
|
||||
created=$(echo $itemData|"$JQ_CMD" '.created')
|
||||
created=$((($(echo $itemData|"$JQ_CMD" '.created') + 500) / 1000))
|
||||
itemId=$(echo $itemData | "$JQ_CMD" -r '.id')
|
||||
out="$TARGET/$(date -d @$created +%Y-%m)/$itemId"
|
||||
|
||||
if [ -d "$out" ] && [ "$DROP_ITEM" == "y" ]; then
|
||||
errout "Removing item folder as requested: $out"
|
||||
rm -rf "$out"
|
||||
fi
|
||||
|
||||
mkdir -p "$out"
|
||||
if [ -f "$out/metadata.json" ] && [ "$SKIP_FILE" == "y" ]; then
|
||||
errout " - Skipping file 'metadata.json' since it already exists"
|
||||
else
|
||||
if [ -f "$out/metadata.json" ] && [ "$OVERWRITE_FILE" == "y" ]; then
|
||||
errout " - Removing metadata.json as requested"
|
||||
rm -f "$out/metadata.json"
|
||||
fi
|
||||
echo $itemData > "$out/metadata.json"
|
||||
fi
|
||||
while read attachId attachName; do
|
||||
attachOut="$out/$attachName"
|
||||
checkLogin
|
||||
downloadAttachment "$attachId"
|
||||
done < <(echo $itemData | "$JQ_CMD" -r '.sources[] | [.id,.name] | join(" ")')
|
||||
}
|
||||
|
||||
login
|
||||
|
||||
allCount=$(fetchItemCount)
|
||||
errout "Downloading $allCount items…"
|
||||
|
||||
allCounter=0 innerCounter=0 limit=100 offset=0 done=n
|
||||
|
||||
while [ "$done" = "n" ]; do
|
||||
checkLogin
|
||||
|
||||
innerCounter=0
|
||||
while read id; do
|
||||
downloadItem "$id"
|
||||
innerCounter=$(($innerCounter + 1))
|
||||
done < <(listItems $offset $limit)
|
||||
|
||||
allCounter=$(($allCounter + $innerCounter))
|
||||
offset=$(($offset + $limit))
|
||||
|
||||
|
||||
if [ $innerCounter -lt $limit ]; then
|
||||
done=y
|
||||
fi
|
||||
|
||||
done
|
||||
errout "Downloaded $allCounter/$allCount items"
|
||||
if [[ $allCounter < $allCount ]]; then
|
||||
errout
|
||||
errout " Downloaded less items than were reported as available. This"
|
||||
errout " may be due to items in folders that you cannot see. Or it"
|
||||
errout " may be a bug."
|
||||
errout
|
||||
fi
|
||||
@ -1,22 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# This script submits a job to regenerate all preview images. This may
|
||||
# be necessary if you change the dpi setting that affects the size of
|
||||
# the preview.
|
||||
|
||||
set -e
|
||||
|
||||
CURL_CMD="curl"
|
||||
JQ_CMD="jq"
|
||||
|
||||
|
||||
BASE_URL="${1:-http://localhost:7880}"
|
||||
TRIGGER_URL="$BASE_URL/api/v1/admin/attachments/generatePreviews"
|
||||
|
||||
echo "Login to trigger regenerating preview images."
|
||||
echo "Using url: $BASE_URL"
|
||||
echo -n "Admin Secret: "
|
||||
read -s ADMIN_SECRET
|
||||
echo
|
||||
|
||||
curl --fail -XPOST -H "Docspell-Admin-Secret: $ADMIN_SECRET" "$TRIGGER_URL"
|
||||
@ -1,49 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# A script to reset a password.
|
||||
#
|
||||
# Usage:
|
||||
# ./reset-password.sh <baseurl> <admin-secret> <account>
|
||||
#
|
||||
# Example:
|
||||
# ./reset-password.sh http://localhost:7880 test123 your/account
|
||||
#
|
||||
|
||||
CURL_CMD="curl"
|
||||
JQ_CMD="jq"
|
||||
|
||||
if [ -z "$1" ]; then
|
||||
echo "The docspell base-url is required as first argument."
|
||||
exit 1
|
||||
else
|
||||
BASE_URL="$1"
|
||||
fi
|
||||
|
||||
if [ -z "$2" ]; then
|
||||
echo "The admin secret is required as second argument."
|
||||
exit 1
|
||||
else
|
||||
SECRET="$2"
|
||||
fi
|
||||
|
||||
if [ -z "$3" ]; then
|
||||
echo "The user account is required as third argument."
|
||||
exit 1
|
||||
else
|
||||
USER="$3"
|
||||
fi
|
||||
|
||||
RESET_URL="${BASE_URL}/api/v1/admin/user/resetPassword"
|
||||
|
||||
OUT=$("$CURL_CMD" -s -XPOST \
|
||||
-H "Docspell-Admin-Secret: $SECRET" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"account\": \"$USER\"}" \
|
||||
"$RESET_URL")
|
||||
|
||||
|
||||
if command -v "$JQ_CMD" > /dev/null; then
|
||||
echo $OUT | "$JQ_CMD"
|
||||
else
|
||||
echo $OUT
|
||||
fi
|
||||
@ -4,10 +4,10 @@ base_url = "https://docspell.org"
|
||||
# Whether to automatically compile all Sass files in the sass directory
|
||||
compile_sass = true
|
||||
|
||||
[markdown]
|
||||
# Whether to do syntax highlighting
|
||||
# Theme can be customised by setting the `highlight_theme` variable to a theme supported by Zola
|
||||
highlight_code = true
|
||||
|
||||
highlight_theme = "gruvbox-dark"
|
||||
|
||||
# Whether to build a search index to be used later on by a JavaScript library
|
||||
|
||||
@ -82,9 +82,9 @@ documentation, too.
|
||||
|
||||
In order to move to a different tool, it is necessary to get the data
|
||||
out of Docspell in a machine readable/automatic way. Currently, there
|
||||
is a [export-files.sh](@/docs/tools/export-files.md) script provided
|
||||
(in the `tools/` folder) that can be used to download all your files
|
||||
and item metadata.
|
||||
is a [export command](@/docs/tools/cli.md#export-data) in the command
|
||||
line client that can be used to download all your files and item
|
||||
metadata.
|
||||
|
||||
My recommendation is to run periodic database backups and also store
|
||||
the binaries/docker images. This lets you re-create the current state
|
||||
|
||||
@ -7,8 +7,9 @@ To get started, here are some quick links:
|
||||
|
||||
- Using [docker and docker-compose](@/docs/install/docker.md). This
|
||||
sets up everything: all prerequisites, both docspell components and
|
||||
a container running the [consumedir.sh](@/docs/tools/consumedir.md)
|
||||
script to import files that are dropped in a folder.
|
||||
a container running the [dsc
|
||||
watch](@/docs/tools/cli.md#watch-a-directory) script to import files
|
||||
that are dropped in a folder.
|
||||
- [Download, Unpack and Run](@/docs/install/download_run.md). This
|
||||
option is also very quick, but you need to check the
|
||||
[prerequisites](@/docs/install/prereq.md) yourself. Database is
|
||||
@ -27,9 +28,9 @@ To get started, here are some quick links:
|
||||
thread](https://forums.unraid.net/topic/103425-docspell-hilfe/) in
|
||||
the German Unraid forum. Thanks for providing these!
|
||||
|
||||
Every [component](@/docs/intro/_index.md#components) (restserver, joex,
|
||||
consumedir) can run on different machines and multiple times. Most of
|
||||
the time running all on one machine is sufficient and also for
|
||||
Every [component](@/docs/intro/_index.md#components) (restserver,
|
||||
joex, dsc watch) can run on different machines and multiple times.
|
||||
Most of the time running all on one machine is sufficient and also for
|
||||
simplicity, the docker-compose setup reflects this variant.
|
||||
|
||||
While there are many different ways to run docspell, at some point all
|
||||
|
||||
@ -167,7 +167,9 @@ directories.
|
||||
|
||||
The `watch` subcommand can be used to watch one or more directories
|
||||
and upload files when they arrive. It uses the `upload` command under
|
||||
the hood and therefore most options are also available here.
|
||||
the hood and therefore most options are also available here. You can
|
||||
upload via a source url, the integration endpoint or a valid session
|
||||
(requires to login).
|
||||
|
||||
It detects file creations and skips a rename within a watched folder.
|
||||
The flag `-r` or `--recursive` is required to recursively watch a
|
||||
@ -191,6 +193,10 @@ If watching a directory is not possible due to system constraints
|
||||
use the `upload` subcommand with `--poll` option which periodically
|
||||
traverses a directory.
|
||||
|
||||
When using the integration endpoint, it requires to specify `-i` and
|
||||
potentially a secret if the endpoint is protected with a secret.
|
||||
|
||||
|
||||
## Download files
|
||||
|
||||
The `download` command allows to download files that match a given
|
||||
@ -303,7 +309,8 @@ commands require the [admin
|
||||
secret](@/docs/configure/_index.md#admin-endpoint) either in the
|
||||
config file or as an argument.
|
||||
|
||||
Reset user password:
|
||||
### Reset user password
|
||||
|
||||
``` shell
|
||||
❯ dsc admin reset-password --account demo
|
||||
┌─────────┬──────────────┬──────────────────┐
|
||||
@ -313,7 +320,8 @@ Reset user password:
|
||||
└─────────┴──────────────┴──────────────────┘
|
||||
```
|
||||
|
||||
Recreate fulltext index:
|
||||
### Recreate fulltext index
|
||||
|
||||
``` shell
|
||||
❯ dsc admin --admin-secret admin123 recreate-index
|
||||
┌─────────┬─────────────────────────────────────┐
|
||||
@ -323,6 +331,34 @@ Recreate fulltext index:
|
||||
└─────────┴─────────────────────────────────────┘
|
||||
```
|
||||
|
||||
### Convert all files to PDF
|
||||
``` shell
|
||||
❯ dsc admin --admin-secret admin123 convert-all-pdf
|
||||
┌─────────┬─────────────────────────────────┐
|
||||
│ success │ message │
|
||||
├─────────┼─────────────────────────────────┤
|
||||
│ true │ Convert all PDFs task submitted │
|
||||
└─────────┴─────────────────────────────────┘
|
||||
```
|
||||
|
||||
This may be necessary if you disabled pdf conversion before and are
|
||||
enabling it now.
|
||||
|
||||
### Regenerate preview images
|
||||
|
||||
``` shell
|
||||
❯ dsc admin --admin-secret admin123 convert-all-pdf
|
||||
┌─────────┬───────────────────────────────────────┐
|
||||
│ success │ message │
|
||||
├─────────┼───────────────────────────────────────┤
|
||||
│ true │ Generate all previews task submitted. │
|
||||
└─────────┴───────────────────────────────────────┘
|
||||
```
|
||||
|
||||
This submits tasks to (re)generate preview images of all files. This
|
||||
is necessary if you changed the `preview.dpi` setting in joex'
|
||||
config.
|
||||
|
||||
## Search for items
|
||||
|
||||
The `search` command takes a [query](@/docs/query/_index.md) and
|
||||
|
||||
@ -1,58 +0,0 @@
|
||||
+++
|
||||
title = "Directory Cleaner (⊗)"
|
||||
description = "Clean directories from files in docspell"
|
||||
weight = 150
|
||||
+++
|
||||
|
||||
{% infobubble(mode="info", title="⚠ Please note") %}
|
||||
This script is now obsolete, you can use the [**CLI tool**](../cli/) instead.
|
||||
|
||||
Use the `cleanup` or the `upload` command.
|
||||
{% end %}
|
||||
|
||||
# Introduction
|
||||
|
||||
This script is made for cleaning up the consumption directory used for
|
||||
the consumedir service (as it is provided as docker container)which
|
||||
are copied or moved there.
|
||||
|
||||
<https://github.com/eikek/docspell/tree/master/tools/consumedir-cleaner>
|
||||
|
||||
## How it works
|
||||
|
||||
- Checks for every file (in the collective's folder of the given user
|
||||
name) if it already exists in the collective (using Docspell's API).
|
||||
- If so, by default those files are moved to an archive folder just
|
||||
besides the collective's consumption folders named _archive. The
|
||||
archive's files are organized into monthly subfolders by the date
|
||||
they've been added to Docspell
|
||||
- If set, those files can also be deleted instead of being moved to
|
||||
the archive. There is no undo function provided for this, so be
|
||||
careful.
|
||||
- If a file is found which does not exist in the collective, by
|
||||
default nothing happens, so that file would be found in every run
|
||||
and just ignored
|
||||
- If set, those files can also be uploaded to Docspell. Depending on
|
||||
the setting for files already existing these files would either be
|
||||
deleted or moved to the archive in the next run.
|
||||
|
||||
## Usage (parameters / settings)
|
||||
|
||||
Copy the script to your machine and run it with the following
|
||||
parameters:
|
||||
|
||||
1. URL of Docspell, including http(s)
|
||||
2. Username for Docspell, possibly including Collective (if other name
|
||||
as user)
|
||||
3. Password for Docspell
|
||||
4. Path to the directory which files shall be checked against
|
||||
existence in Docspell
|
||||
|
||||
Additionally, environment variables can be used to alter the behavior:
|
||||
|
||||
- `DS_CC_REMOVE`
|
||||
- `true` – delete files which already exist in the collective
|
||||
- `false` (default) - move them to the archive (see above)
|
||||
- `DS_CC_UPLOAD_MISSING`
|
||||
- `true` - uploads files which do not exist in the collective
|
||||
- `false` (default) - ignore them and do nothing
|
||||
@ -1,191 +0,0 @@
|
||||
+++
|
||||
title = "Consume Directory (⊗)"
|
||||
description = "A script to watch a directory for new files and upload them to docspell."
|
||||
weight = 110
|
||||
+++
|
||||
|
||||
|
||||
{% infobubble(mode="info", title="⚠ Please note") %}
|
||||
This script is now obsolete, you can use the [**CLI tool**](../cli/) instead.
|
||||
|
||||
You can use the `watch` command, or the `upload` command with `--poll`.
|
||||
{% end %}
|
||||
|
||||
# Introduction
|
||||
|
||||
The `consumerdir.sh` is a bash script that works in two modes:
|
||||
|
||||
- Go through all files in given directories (recursively, if `-r` is
|
||||
specified) and sent each to docspell.
|
||||
- Watch one or more directories for new files and upload them to
|
||||
docspell.
|
||||
|
||||
It can watch or go through one or more directories. Files can be
|
||||
uploaded to multiple urls.
|
||||
|
||||
Run the script with the `-h` or `--help` option, to see a short help
|
||||
text. The help text will also show the values for any given option.
|
||||
|
||||
The script requires `curl` for uploading. It requires the
|
||||
`inotifywait` command if directories should be watched for new
|
||||
files.
|
||||
|
||||
Example for watching two directories:
|
||||
|
||||
``` bash
|
||||
./tools/consumedir.sh --path ~/Downloads --path ~/pdfs -m -dv \
|
||||
http://localhost:7880/api/v1/open/upload/item/5DxhjkvWf9S-CkWqF3Kr892-WgoCspFWDo7-XBykwCyAUxQ
|
||||
```
|
||||
|
||||
The script by default watches the given directories. If the `-o` or
|
||||
`--once` option is used, it will instead go through these directories
|
||||
and upload all files in there. For directory watching the
|
||||
`inotifywait` command is used and must be present. Another way is to
|
||||
use the `--poll` option. It expects the number of seconds to wait
|
||||
between running itself with `--once`.
|
||||
|
||||
Example using active polling (at 5 minutes interval):
|
||||
``` bash
|
||||
./tools/consumedir.sh --poll 300 --path ~/Downloads --path ~/pdfs -m -dv \
|
||||
http://localhost:7880/api/v1/open/upload/item/5DxhjkvWf9S-CkWqF3Kr892-WgoCspFWDo7-XBykwCyAUxQ
|
||||
```
|
||||
|
||||
Example for uploading all immediatly (the same as above only with `-o`
|
||||
added):
|
||||
|
||||
``` bash
|
||||
$ ./tools/consumedir.sh --once --path ~/Downloads --path ~/pdfs/ -m -dv \
|
||||
http://localhost:7880/api/v1/open/upload/item/5DxhjkvWf9S-CkWqF3Kr892-WgoCspFWDo7-XBykwCyAUxQ
|
||||
```
|
||||
|
||||
|
||||
The URL can be any docspell url that accepts uploads without
|
||||
authentication. This is usually a [source
|
||||
url](@/docs/webapp/uploading.md#anonymous-upload). It is also possible
|
||||
to use the script with the [integration
|
||||
endpoint](@/docs/api/upload.md#integration-endpoint).
|
||||
|
||||
The script can be run multiple times and on on multiple machines, the
|
||||
files are transferred via HTTP to the docspell server. For example, it
|
||||
is convenient to set it up on your workstation, so that you can drop
|
||||
files into some local folder to be immediatly transferred to docspell
|
||||
(e.g. when downloading something from the browser).
|
||||
|
||||
## Integration Endpoint
|
||||
|
||||
When given the `-i` or `--integration` option, the script changes its
|
||||
behaviour slightly to work with the [integration
|
||||
endpoint](@/docs/api/upload.md#integration-endpoint).
|
||||
|
||||
First, if `-i` is given, it implies `-r` – so the directories are
|
||||
watched or traversed recursively. The script then assumes that there
|
||||
is a subfolder with the collective name. Files must not be placed
|
||||
directly into a folder given by `-p`, but below a sub-directory that
|
||||
matches a collective name. In order to know for which collective the
|
||||
file is, the script uses the first subfolder.
|
||||
|
||||
If the endpoint is protected, the credentials can be specified as
|
||||
arguments `--iuser` and `--iheader`, respectively. The format is for
|
||||
both `<name>:<value>`, so the username cannot contain a colon
|
||||
character (but the password can).
|
||||
|
||||
Example:
|
||||
``` bash
|
||||
$ consumedir.sh -i -iheader 'Docspell-Integration:test123' -m -p ~/Downloads/ http://localhost:7880/api/v1/open/integration/item
|
||||
```
|
||||
|
||||
The url is the integration endpoint url without the collective, as
|
||||
this is amended by the script.
|
||||
|
||||
This watches the folder `~/Downloads`. If a file is placed in this
|
||||
folder directly, say `~/Downloads/test.pdf` the upload will fail,
|
||||
because the collective cannot be determined. Create a subfolder below
|
||||
`~/Downloads` with the name of a collective, for example
|
||||
`~/Downloads/family` and place files somewhere below this `family`
|
||||
subfolder, like `~/Downloads/family/test.pdf`.
|
||||
|
||||
|
||||
## Duplicates
|
||||
|
||||
With the `-m` option, the script will not upload files that already
|
||||
exist at docspell. For this the `sha256sum` command is required.
|
||||
|
||||
So you can move and rename files in those folders without worring
|
||||
about duplicates. This allows to keep your files organized using the
|
||||
file-system and have them mirrored into docspell as well.
|
||||
|
||||
|
||||
## Network Filesystems (samba cifs, nfs)
|
||||
|
||||
Watching a directory for changes relies on `inotify` subsystem on
|
||||
linux. This doesn't work on network filesystems like nfs or cifs. Here
|
||||
are some ideas to get around this limitation:
|
||||
|
||||
1. The `consumedir.sh` is just a shell script and doesn't need to run
|
||||
on the same machine as docspell. (Note that the default docker
|
||||
setup is mainly for demoing and quickstart, it's not required to
|
||||
run all of them on one machine). So the best option is to put the
|
||||
consumedir on the machine that contains the local filesystem. All
|
||||
files are send via HTTP to the docspell server anyways, so there is
|
||||
no need to first transfer them via a network filesystem or rsync.
|
||||
2. If option 1 is not possible for some reason, and you need to check
|
||||
a network filesystem, the only option left (that I know) is to
|
||||
periodically poll this directory. This is also possible with
|
||||
consumedir, using the `--poll` option (see above). You can also
|
||||
setup a systemd timer to periodically run this script with the
|
||||
`--once` option.
|
||||
3. Copy the files to the machine that runs consumedir, via rsync for
|
||||
example. Note that this has no advantage over otpion 1, as you now
|
||||
need to setup rsync on the other machine to run either periodically
|
||||
or when some file arrives. Then you can as well run the consumedir
|
||||
script. But it might be more convenient, if rsync is already
|
||||
running.
|
||||
|
||||
# Systemd
|
||||
|
||||
The script can be used with systemd to run as a service. This is an
|
||||
example unit file:
|
||||
|
||||
``` systemd
|
||||
[Unit]
|
||||
After=networking.target
|
||||
Description=Docspell Consumedir
|
||||
|
||||
[Service]
|
||||
Environment="PATH=/set/a/path"
|
||||
|
||||
ExecStart=/bin/su -s /bin/bash someuser -c "consumedir.sh --path '/a/path/' -m 'http://localhost:7880/api/v1/open/upload/item/5DxhjkvWf9S-CkWqF3Kr892-WgoCspFWDo7-XBykwCyAUxQ'"
|
||||
```
|
||||
|
||||
This unit file is just an example, it needs some fiddling. It assumes
|
||||
an existing user `someuser` that is used to run this service. The url
|
||||
`http://localhost:7880/api/v1/open/upload/...` is an anonymous upload
|
||||
url as described [here](@/docs/webapp/uploading.md#anonymous-upload).
|
||||
|
||||
|
||||
# Docker
|
||||
|
||||
The provided docker-compose setup runs this script to watch a single
|
||||
directory, `./docs` in current directory, for new files. If a new file
|
||||
is detected, it is pushed to docspell.
|
||||
|
||||
This utilizes the [integration
|
||||
endpoint](@/docs/api/upload.md#integration-endpoint), which is
|
||||
enabled in the config file, to allow uploading documents for all
|
||||
collectives. A subfolder must be created for each registered
|
||||
collective. The docker containers are configured to use http-header
|
||||
protection for the integration endpoint. This requires you to provide
|
||||
a secret, that is shared between the rest-server and the
|
||||
`consumedir.sh` script. This can be done by defining an environment
|
||||
variable which gets picked up by the containers defined in
|
||||
`docker-compose.yml`:
|
||||
|
||||
``` bash
|
||||
export DOCSPELL_HEADER_VALUE="my-secret"
|
||||
docker-compose up
|
||||
```
|
||||
|
||||
|
||||
Now you can create a folder `./docs/<collective-name>` and place all
|
||||
files in there that you want to import. Once dropped in this folder
|
||||
the `consumedir` container will push it to docspell.
|
||||
@ -1,59 +0,0 @@
|
||||
+++
|
||||
title = "Convert All PDFs (⊗)"
|
||||
description = "Convert all PDF files using OcrMyPdf."
|
||||
weight = 160
|
||||
+++
|
||||
|
||||
{% infobubble(mode="info", title="⚠ Please note") %}
|
||||
This script is now obsolete, you can use the [**CLI tool**](../cli/) instead.
|
||||
|
||||
Use the `convert-all-pdfs` admin command, e.g. `dsc admin
|
||||
convert-all-pdfs`.
|
||||
{% end %}
|
||||
|
||||
|
||||
# convert-all-pdf.sh
|
||||
|
||||
With version 0.9.0 there was support added for another external tool,
|
||||
[OCRMyPdf](https://github.com/jbarlow83/OCRmyPDF), that can convert
|
||||
PDF files such that they contain the OCR-ed text layer. This tool is
|
||||
optional and can be disabled.
|
||||
|
||||
In order to convert all previously processed files with this tool,
|
||||
there is an
|
||||
[endpoint](/openapi/docspell-openapi.html#api-Item-secItemConvertallpdfsPost)
|
||||
that submits a task to convert all PDF files not already converted for
|
||||
your collective.
|
||||
|
||||
There is no UI part to trigger this route, so you need to use curl or
|
||||
the script `convert-all-pdfs.sh` in the `tools/` directory.
|
||||
|
||||
# Requirements
|
||||
|
||||
It is a bash script that additionally needs
|
||||
[curl](https://curl.haxx.se/) and
|
||||
[jq](https://stedolan.github.io/jq/).
|
||||
|
||||
# Usage
|
||||
|
||||
```
|
||||
./convert-all-pdfs.sh [docspell-base-url]
|
||||
```
|
||||
|
||||
For example, if docspell is at `http://localhost:7880`:
|
||||
|
||||
```
|
||||
./convert-all-pdfs.sh http://localhost:7880
|
||||
```
|
||||
|
||||
The script asks for your account name and password. It then logs in
|
||||
and triggers the said endpoint. After this you should see a few tasks
|
||||
running.
|
||||
|
||||
There will be one task per file to convert. All these tasks are
|
||||
submitted with a low priority. So files uploaded through the webapp or
|
||||
a [source](@/docs/webapp/uploading.md#anonymous-upload) with a high
|
||||
priority, will be preferred as [configured in the job
|
||||
executor](@/docs/joex/intro.md#scheduler-config). This is to not
|
||||
disturb normal processing when many conversion tasks are being
|
||||
executed.
|
||||
@ -1,54 +0,0 @@
|
||||
+++
|
||||
title = "Upload CLI (⊗)"
|
||||
description = "A script to quickly upload files from the command line."
|
||||
weight = 100
|
||||
+++
|
||||
|
||||
|
||||
{% infobubble(mode="info", title="⚠ Please note") %}
|
||||
This script is now obsolete, you can use the [**CLI tool**](../cli/) instead.
|
||||
|
||||
Use the `upload` command (or the `up` alias), like `dsc up *.pdf`.
|
||||
{% end %}
|
||||
|
||||
# Introduction
|
||||
|
||||
The `tools/ds.sh` is a bash script to quickly upload files from the
|
||||
command line. It reads a configuration file containing the URLs to
|
||||
upload to. Then each file given to the script will be uploaded to al
|
||||
URLs in the config.
|
||||
|
||||
The config file is expected in
|
||||
`$XDG_CONFIG_HOME/docspell/ds.conf`. `$XDG_CONFIG_HOME` defaults to
|
||||
`~/.config`.
|
||||
|
||||
The config file contains lines with key-value pairs, separated by a
|
||||
`=` sign. Lines starting with `#` are ignored. Example:
|
||||
|
||||
```
|
||||
# Config file
|
||||
url.1 = http://localhost:7880/api/v1/open/upload/item/5DxhjkvWf9S-CkWqF3Kr892-WgoCspFWDo7-XBykwCyAUxQ
|
||||
url.2 = http://localhost:7880/api/v1/open/upload/item/6DxhjkvWf9S-CkWqF3Kr892-WgoCspFWDo7-XBykwCyAUxQ
|
||||
```
|
||||
|
||||
The key must start with `url`. The urls should be [anonymous upload
|
||||
urls](@/docs/webapp/uploading.md#anonymous-upload).
|
||||
|
||||
|
||||
# Usage
|
||||
|
||||
- The `-c` option allows to specifiy a different config file.
|
||||
- The `-h` option shows a help overview.
|
||||
- The `-d` option deletes files after upload was successful
|
||||
- The `-e` option can be used to check for file existence in docspell.
|
||||
Instead of uploading, the script only checks whether the file is in
|
||||
docspell or not.
|
||||
|
||||
The script takes a list of files as arguments.
|
||||
|
||||
|
||||
Example:
|
||||
|
||||
``` bash
|
||||
./ds.sh ~/Downloads/*.pdf
|
||||
```
|
||||
@ -1,215 +0,0 @@
|
||||
+++
|
||||
title = "Export Files (⊗)"
|
||||
description = "Downloads all files from docspell."
|
||||
weight = 165
|
||||
+++
|
||||
|
||||
{% infobubble(mode="info", title="⚠ Please note") %}
|
||||
This script is now obsolete, you can use the [**CLI tool**](../cli/) instead.
|
||||
|
||||
Use the `export` command, e.g. `dsc export --all --target .`.
|
||||
{% end %}
|
||||
|
||||
|
||||
# export-files.sh
|
||||
|
||||
This script can be used to download all files from docspell that have
|
||||
been uploaded before and the item metadata.
|
||||
|
||||
It downloads the original files, those that have been uploaded and not
|
||||
the converted pdf files.
|
||||
|
||||
The item's metadata are stored next to the files to provide more
|
||||
information about the item: corresponent, tags, dates, custom fields
|
||||
etc. This contains most of your user supplied data.
|
||||
|
||||
This script is intended for having your data outside and independent
|
||||
of docspell. Another good idea for a backup strategy is to take
|
||||
database dumps *and* storing the releases of docspell next to this
|
||||
dump.
|
||||
|
||||
Files are stored into the following folder structure (below the given
|
||||
target directory):
|
||||
|
||||
```
|
||||
- yyyy-mm (item date)
|
||||
- A3…XY (item id)
|
||||
- somefile.pdf (attachments with name)
|
||||
- metadata.json (json file with items metadata)
|
||||
```
|
||||
|
||||
By default, files are not overwritten, it stops if existing files are
|
||||
encountered. This and some other things can be changed using
|
||||
environment variables:
|
||||
|
||||
- `DS_USER` the account name for login, it is asked if not available
|
||||
- `DS_PASS` the password for login, it is asked if not available
|
||||
- `OVERWRITE_FILE=` if `y` then overwriting existing files is ok.
|
||||
Default is `n`.
|
||||
- `SKIP_FILE=` if `y` then existing files are skipped (supersedes
|
||||
`OVERWRITE_FILE`). Default is `n`.
|
||||
- `DROP_ITEM=` if `y` the item folder is removed before attempting to
|
||||
download it. If this is set to `y` then the above options don't make
|
||||
sense, since they operate on the files inside the item folder.
|
||||
Default is `n`.
|
||||
|
||||
Docspell sends the sha256 hash with each file via the ETag header.
|
||||
This is used to do a integrity check after downloading.
|
||||
|
||||
|
||||
# Requirements
|
||||
|
||||
It is a bash script that additionally needs
|
||||
[curl](https://curl.haxx.se/) and [jq](https://stedolan.github.io/jq/)
|
||||
to be available.
|
||||
|
||||
# Usage
|
||||
|
||||
```
|
||||
./export-files.sh <docspell-base-url> <target-directory>
|
||||
```
|
||||
|
||||
For example, if docspell is at `http://localhost:7880`:
|
||||
|
||||
```
|
||||
./export-files.sh http://localhost:7880 /tmp/ds-downloads
|
||||
```
|
||||
|
||||
The script asks for your account name and password. It then logs in
|
||||
and goes through all items downloading the metadata as json and the
|
||||
attachments.
|
||||
|
||||
|
||||
# Example Run
|
||||
|
||||
``` bash
|
||||
fish> env SKIP_FILE=y DS_USER=demo DS_PASS=test ./export-files.sh http://localhost:7880 /tmp/download
|
||||
Login to Docspell.
|
||||
Using url: http://localhost:7880
|
||||
|
||||
Login successful
|
||||
Downloading 73 items…
|
||||
Get next items with offset=0, limit=100
|
||||
Get item 57Znskthf3g-X7RP1fxzE2U-dwr4vM6Yjnn-b7s1PoCznhz
|
||||
- Download 'something.txt' (8HbeFornAUN-kBCyc8bHSVr-bnLBYDzgRQ7-peMZzyTzM2X)
|
||||
- Checksum ok.
|
||||
Get item 94u5Pt39q6N-7vKu3LugoRj-zohGS4ie4jb-68bW5gXU6Jd
|
||||
- Download 'letter-en.pdf' (6KNNmoyqpew-RAkdwEmQgBT-QDqdY97whZA-4k2rmbssdfQ)
|
||||
- Checksum ok.
|
||||
Get item 7L9Fh53RVG4-vGSt2G2YUcY-cvpBKRXQgBn-omYpg6xQXyD
|
||||
- Download 'mail.html' (A6yTYKrDc7y-xU3whmLB1kB-TGhEAVb12mo-RUw5u9PsYMo)
|
||||
- Checksum ok.
|
||||
Get item DCn9UtWUtvF-2qjxB5PXGEG-vqRUUU7JUJH-zBBrmSeGYPe
|
||||
- Download 'Invoice_7340224.pdf' (6FWdjxJh7yB-CCjY39p6uH9-uVLbmGfm25r-cw6RksrSx4n)
|
||||
- Checksum ok.
|
||||
…
|
||||
```
|
||||
|
||||
The resulting directory looks then like this:
|
||||
|
||||
``` bash
|
||||
…
|
||||
├── 2020-08
|
||||
│ ├── 6t27gQQ4TfW-H4uAmkYyiSe-rBnerFE2v5F-9BdqbGEhMcv
|
||||
│ │ ├── 52241.pdf
|
||||
│ │ └── metadata.json
|
||||
│ └── 9qwT2GuwEvV-s9UuBQ4w7o9-uE8AdMc7PwL-GFDd62gduAm
|
||||
│ ├── DOC-20191223-155707.jpg
|
||||
│ └── metadata.json
|
||||
├── 2020-09
|
||||
│ ├── 2CM8C9VaVAT-sVJiKyUPCvR-Muqr2Cqvi6v-GXhRtg6eomA
|
||||
│ │ ├── letter with spaces.pdf
|
||||
│ │ └── metadata.json
|
||||
│ ├── 4sXpX2Sc9Ex-QX1M6GtjiXp-DApuDDzGQXR-7pg1QPW9pbs
|
||||
│ │ ├── analyse.org
|
||||
│ │ ├── 201703.docx
|
||||
│ │ ├── 11812_120719.pdf
|
||||
│ │ ├── letter-de.pdf
|
||||
│ │ ├── letter-en.pdf
|
||||
│ │ └── metadata.json
|
||||
│ ├── 5VhP5Torsy1-15pwJBeRjPi-es8BGnxhWn7-3pBQTJv3zPb
|
||||
│ │ └── metadata.json
|
||||
│ ├── 7ePWmK4xCNk-gmvnTDdFwG8-JcN5MDSUNPL-NTZZrho2Jc6
|
||||
│ │ ├── metadata.json
|
||||
│ │ └── Rechnung.pdf
|
||||
…
|
||||
```
|
||||
|
||||
The `metadata.json` file contains all the item metadata. This may be
|
||||
useful when importing into other tools.
|
||||
|
||||
``` json
|
||||
{
|
||||
"id": "AWCNx7tJgUw-SdrNtRouNJB-FGs6Y2VP5bV-218sFN8mjjk",
|
||||
"direction": "incoming",
|
||||
"name": "Ruecksendung.pdf",
|
||||
"source": "integration",
|
||||
"state": "confirmed",
|
||||
"created": 1606171810005,
|
||||
"updated": 1606422917826,
|
||||
"itemDate": null,
|
||||
"corrOrg": null,
|
||||
"corrPerson": null,
|
||||
"concPerson": null,
|
||||
"concEquipment": null,
|
||||
"inReplyTo": null,
|
||||
"folder": null,
|
||||
"dueDate": null,
|
||||
"notes": null,
|
||||
"attachments": [
|
||||
{
|
||||
"id": "4aPmhrjfR9Z-AgknoW6yVoE-YkffioD2KXV-E6Vm6snH17Q",
|
||||
"name": "Ruecksendung.converted.pdf",
|
||||
"size": 57777,
|
||||
"contentType": "application/pdf",
|
||||
"converted": true
|
||||
}
|
||||
],
|
||||
"sources": [
|
||||
{
|
||||
"id": "4aPmhrjfR9Z-AgknoW6yVoE-YkffioD2KXV-E6Vm6snH17Q",
|
||||
"name": "Ruecksendung.pdf",
|
||||
"size": 65715,
|
||||
"contentType": "application/pdf"
|
||||
}
|
||||
],
|
||||
"archives": [],
|
||||
"tags": [
|
||||
{
|
||||
"id": "EQvJ6AHw19Y-Cdg3gF78zZk-BY2zFtNTwes-J95jpXpzhfw",
|
||||
"name": "Hupe",
|
||||
"category": "state",
|
||||
"created": 1606427083171
|
||||
},
|
||||
{
|
||||
"id": "4xyZoeeELdJ-tJ91GiRLinJ-7bdauy3U1jR-Bzr4VS96bGS",
|
||||
"name": "Invoice",
|
||||
"category": "doctype",
|
||||
"created": 1594249709473
|
||||
}
|
||||
],
|
||||
"customfields": [
|
||||
{
|
||||
"id": "5tYmDHin3Kx-HomKkeEVtJN-v99oKxQ8ot6-yFVrEmMayoo",
|
||||
"name": "amount",
|
||||
"label": "EUR",
|
||||
"ftype": "money",
|
||||
"value": "151.55"
|
||||
},
|
||||
{
|
||||
"id": "3jbwbep8rDs-hNJ9ePRE7gv-21nYMbUj3eb-mKRWAr4xSS2",
|
||||
"name": "invoice-number",
|
||||
"label": "Invoice-Nr",
|
||||
"ftype": "text",
|
||||
"value": "I454602"
|
||||
},
|
||||
{
|
||||
"id": "AH4p4NUCa9Y-EUkH66wLzxE-Rf2wJPxTAYd-DeGDm4AT4Yg",
|
||||
"name": "number",
|
||||
"label": "Number",
|
||||
"ftype": "numeric",
|
||||
"value": "0.10"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
@ -1,48 +0,0 @@
|
||||
+++
|
||||
title = "Regenerate Preview Images (⊗)"
|
||||
description = "Re-generates all preview images."
|
||||
weight = 130
|
||||
+++
|
||||
|
||||
{% infobubble(mode="info", title="⚠ Please note") %}
|
||||
This script is now obsolete, you can use the [**CLI tool**](../cli/) instead.
|
||||
|
||||
Use the `generate-previews` admin command, e.g. `dsc admin generate-previews`.
|
||||
{% end %}
|
||||
|
||||
# regenerate-previews.sh
|
||||
|
||||
This is a simple bash script to trigger the endpoint that submits task
|
||||
for generating preview images of your files. This is usually not
|
||||
needed, but should you change the `preview.dpi` setting in joex'
|
||||
config file, you need to regenerate the images to have any effect.
|
||||
|
||||
# Requirements
|
||||
|
||||
It is a bash script that additionally needs
|
||||
[curl](https://curl.haxx.se/) and
|
||||
[jq](https://stedolan.github.io/jq/).
|
||||
|
||||
# Usage
|
||||
|
||||
```
|
||||
./regenerate-previews.sh [docspell-base-url] [admin-secret]
|
||||
```
|
||||
|
||||
For example, if docspell is at `http://localhost:7880`:
|
||||
|
||||
```
|
||||
./convert-all-pdfs.sh http://localhost:7880 test123
|
||||
```
|
||||
|
||||
The script asks for the admin secret if not given to the command. It
|
||||
then logs in and triggers the said endpoint. After this you should see
|
||||
a few tasks running.
|
||||
|
||||
There will be one task per file to convert. All these tasks are
|
||||
submitted with a low priority. So files uploaded through the webapp or
|
||||
a [source](@/docs/webapp/uploading.md#anonymous-upload) with a high
|
||||
priority, will be preferred as [configured in the job
|
||||
executor](@/docs/joex/intro.md#scheduler-config). This is to not
|
||||
disturb normal processing when many conversion tasks are being
|
||||
executed.
|
||||
@ -1,47 +0,0 @@
|
||||
+++
|
||||
title = "Reset Password (⊗)"
|
||||
description = "Resets a user password."
|
||||
weight = 120
|
||||
+++
|
||||
|
||||
{% infobubble(mode="info", title="⚠ Please note") %}
|
||||
This script is now obsolete, you can use the [**CLI tool**](../cli/) instead.
|
||||
|
||||
Use the `reset-password` admin command, e.g. `dsc admin reset-password
|
||||
--account "smith/john"`, where `smith` is the collective id and `john`
|
||||
the username.
|
||||
{% end %}
|
||||
|
||||
|
||||
This script can be used to reset a user password. This can be done by
|
||||
admins, who know the `admin-endpoint.secret` value in the
|
||||
[configuration](@/docs/configure/_index.md#admin-endpoint) file.
|
||||
|
||||
The script is in `/tools/reset-password/reset-password.sh` and it is
|
||||
only a wrapper around the admin endpoint `/admin/user/resetPassword`.
|
||||
|
||||
## Usage
|
||||
|
||||
It's very simple:
|
||||
|
||||
``` bash
|
||||
reset-password.sh <base-url> <admin-secret> <account>
|
||||
```
|
||||
|
||||
Three arguments are required to specify the docspell base url, the
|
||||
admin secret and the account you want to reset the password.
|
||||
|
||||
After the password has been reset, the user can login using it and
|
||||
change it again in the webapp.
|
||||
|
||||
|
||||
## Example
|
||||
|
||||
``` json
|
||||
❯ ./tools/reset-password/reset-password.sh http://localhost:7880 123 eike
|
||||
{
|
||||
"success": true,
|
||||
"newPassword": "HjtpG9BFo9y",
|
||||
"message": "Password updated"
|
||||
}
|
||||
```
|
||||
Loading…
x
Reference in New Issue
Block a user