mirror of
https://github.com/TheAnachronism/docspell.git
synced 2025-06-21 09:58:26 +00:00
Extend consumedir.sh to work with integration endpoint
Now running one consumedir script can upload files to multiple collectives separately.
This commit is contained in:
Eike Kettner
@ -13,6 +13,7 @@ CURL_CMD="curl"
|
||||
INOTIFY_CMD="inotifywait"
|
||||
SHA256_CMD="sha256sum"
|
||||
MKTEMP_CMD="mktemp"
|
||||
CURL_OPTS=${CURL_OPTS:-}
|
||||
|
||||
! getopt --test > /dev/null
|
||||
if [[ ${PIPESTATUS[0]} -ne 4 ]]; then
|
||||
@ -20,8 +21,8 @@ if [[ ${PIPESTATUS[0]} -ne 4 ]]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
OPTIONS=omhdp:vr
|
||||
LONGOPTS=once,distinct,help,delete,path:,verbose,recursive,dry
|
||||
OPTIONS=omhdp:vrmi
|
||||
LONGOPTS=once,distinct,help,delete,path:,verbose,recursive,dry,integration,iuser:,iheader:
|
||||
|
||||
! PARSED=$(getopt --options=$OPTIONS --longoptions=$LONGOPTS --name "$0" -- "$@")
|
||||
if [[ ${PIPESTATUS[0]} -ne 0 ]]; then
|
||||
@ -35,6 +36,7 @@ eval set -- "$PARSED"
|
||||
|
||||
declare -a watchdir
|
||||
help=n verbose=n delete=n once=n distinct=n recursive=n dryrun=n
|
||||
integration=n iuser="" iheader=""
|
||||
while true; do
|
||||
case "$1" in
|
||||
-h|--help)
|
||||
@ -69,6 +71,19 @@ while true; do
|
||||
dryrun=y
|
||||
shift
|
||||
;;
|
||||
-i|--integration)
|
||||
integration=y
|
||||
recursive=y
|
||||
shift
|
||||
;;
|
||||
--iuser)
|
||||
iuser="$2"
|
||||
shift 2
|
||||
;;
|
||||
--iheader)
|
||||
iheader="$2"
|
||||
shift 2
|
||||
;;
|
||||
--)
|
||||
shift
|
||||
break
|
||||
@ -87,14 +102,27 @@ showUsage() {
|
||||
echo "Usage: $0 [options] url url ..."
|
||||
echo
|
||||
echo "Options:"
|
||||
echo " -v | --verbose Print more to stdout. (value: $verbose)"
|
||||
echo " -d | --delete Delete the file if successfully uploaded. (value: $delete)"
|
||||
echo " -p | --path <dir> The directories to watch. This is required. (value: ${watchdir[@]})"
|
||||
echo " -h | --help Prints this help text. (value: $help)"
|
||||
echo " -m | --distinct Optional. Upload only if the file doesn't already exist. (value: $distinct)"
|
||||
echo " -o | --once Instead of watching, upload all files in that dir. (value: $once)"
|
||||
echo " -r | --recursive Traverse the directory(ies) recursively (value: $recursive)"
|
||||
echo " --dry Do a 'dry run', not uploading anything only printing to stdout (value: $dryrun)"
|
||||
echo " -v | --verbose Print more to stdout. (value: $verbose)"
|
||||
echo " -d | --delete Delete the file if successfully uploaded. (value: $delete)"
|
||||
echo " -p | --path <dir> The directories to watch. This is required. (value: ${watchdir[@]})"
|
||||
echo " -h | --help Prints this help text. (value: $help)"
|
||||
echo " -m | --distinct Optional. Upload only if the file doesn't already exist. (value: $distinct)"
|
||||
echo " -o | --once Instead of watching, upload all files in that dir. (value: $once)"
|
||||
echo " -r | --recursive Traverse the directory(ies) recursively (value: $recursive)"
|
||||
echo " -i | --integration Upload to the integration endpoint. It implies -r. This puts the script in"
|
||||
echo " a different mode, where the first subdirectory of any given starting point"
|
||||
echo " is read as the collective name. The url(s) are completed with this name in"
|
||||
echo " order to upload files to the respective collective. So each directory"
|
||||
echo " given is expected to contain one subdirectory per collective and the urls"
|
||||
echo " are expected to identify the integration endpoint, which is"
|
||||
echo " /api/v1/open/integration/item/<collective-name>. (value: $integration)"
|
||||
echo " --iheader The header name and value to use with the integration endpoint. This must be"
|
||||
echo " in form 'headername:value'. Only used if '-i' is supplied."
|
||||
echo " (value: $iheader)"
|
||||
echo " --iuser The username and password for basic auth to use with the integration"
|
||||
echo " endpoint. This must be of form 'user:pass'. Only used if '-i' is supplied."
|
||||
echo " (value: $iuser)"
|
||||
echo " --dry Do a 'dry run', not uploading anything only printing to stdout (value: $dryrun)"
|
||||
echo ""
|
||||
echo "Arguments:"
|
||||
echo " A list of URLs to upload the files to."
|
||||
@ -105,6 +133,9 @@ showUsage() {
|
||||
echo "Example: Upload all files in a directory"
|
||||
echo "$0 --path ~/Downloads -m -dv --once http://localhost:7880/api/v1/open/upload/item/abcde-12345-abcde-12345"
|
||||
echo ""
|
||||
echo "Example: Integration Endpoint"
|
||||
echo "$0 -i -iheader 'Docspell-Integration:test123' -m -p ~/Downloads/ http://localhost:7880/api/v1/open/integration/item"
|
||||
echo ""
|
||||
}
|
||||
|
||||
if [ "$help" = "y" ]; then
|
||||
@ -127,32 +158,67 @@ fi
|
||||
|
||||
trace() {
|
||||
if [ "$verbose" = "y" ]; then
|
||||
echo "$1"
|
||||
>&2 echo "$1"
|
||||
fi
|
||||
}
|
||||
|
||||
info() {
|
||||
echo $1
|
||||
>&2 echo $1
|
||||
}
|
||||
|
||||
getCollective() {
|
||||
file="$(realpath -e $1)"
|
||||
dir="$(realpath -e $2)"
|
||||
collective=${file#"$dir"}
|
||||
coll=$(echo $collective | cut -d'/' -f1)
|
||||
if [ -z "$coll" ]; then
|
||||
coll=$(echo $collective | cut -d'/' -f2)
|
||||
fi
|
||||
echo $coll
|
||||
}
|
||||
|
||||
|
||||
upload() {
|
||||
dir="$(realpath -e $1)"
|
||||
file="$(realpath -e $2)"
|
||||
url="$3"
|
||||
OPTS="$CURL_OPTS"
|
||||
if [ "$integration" = "y" ]; then
|
||||
collective=$(getCollective "$file" "$dir")
|
||||
trace "- upload: collective = $collective"
|
||||
url="$url/$collective"
|
||||
if [ $iuser ]; then
|
||||
OPTS="$OPTS --user $iuser"
|
||||
fi
|
||||
if [ $iheader ]; then
|
||||
OPTS="$OPTS -H $iheader"
|
||||
fi
|
||||
fi
|
||||
if [ "$dryrun" = "y" ]; then
|
||||
info "Not uploading (dry-run) $1 to $2"
|
||||
info "- Not uploading (dry-run) $file to $url with opts $OPTS"
|
||||
else
|
||||
tf=$($MKTEMP_CMD) rc=0
|
||||
$CURL_CMD -# -o "$tf" --stderr "$tf" -w "%{http_code}" -XPOST -F file=@"$1" "$2" | (2>&1 1>/dev/null grep 200)
|
||||
rc=$(expr $rc + $?)
|
||||
cat $tf | (2>&1 1>/dev/null grep '{"success":true')
|
||||
rc=$(expr $rc + $?)
|
||||
if [ $rc -ne 0 ]; then
|
||||
trace "- Uploading $file to $url with options $OPTS"
|
||||
tf1=$($MKTEMP_CMD) tf2=$($MKTEMP_CMD) rc=0
|
||||
$CURL_CMD --fail -# -o "$tf1" --stderr "$tf2" $OPTS -XPOST -F file=@"$file" "$url"
|
||||
if [ $? -ne 0 ]; then
|
||||
info "Upload failed. Exit code: $rc"
|
||||
cat "$tf"
|
||||
cat "$tf1"
|
||||
cat "$tf2"
|
||||
echo ""
|
||||
rm "$tf"
|
||||
rm "$tf1" "$tf2"
|
||||
return $rc
|
||||
else
|
||||
rm "$tf"
|
||||
return 0
|
||||
if cat $tf1 | grep -q '{"success":false'; then
|
||||
echo "Upload failed. Message from server:"
|
||||
cat "$tf1"
|
||||
echo ""
|
||||
rm "$tf1" "$tf2"
|
||||
return 1
|
||||
else
|
||||
info "- Upload done."
|
||||
rm "$tf1" "$tf2"
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
}
|
||||
@ -162,28 +228,69 @@ checksum() {
|
||||
}
|
||||
|
||||
checkFile() {
|
||||
local url=$(echo "$1" | sed 's,upload/item,checkfile,g')
|
||||
local url="$1"
|
||||
local file="$2"
|
||||
trace "Check file: $url/$(checksum "$file")"
|
||||
$CURL_CMD -XGET -s "$url/$(checksum "$file")" | (2>&1 1>/dev/null grep '"exists":true')
|
||||
local dir="$3"
|
||||
OPTS="$CURL_OPTS"
|
||||
if [ "$integration" = "y" ]; then
|
||||
collective=$(getCollective "$file" "$dir")
|
||||
url="$url/$collective"
|
||||
url=$(echo "$url" | sed 's,/item/,/checkfile/,g')
|
||||
if [ $iuser ]; then
|
||||
OPTS="$OPTS --user $iuser"
|
||||
fi
|
||||
if [ $iheader ]; then
|
||||
OPTS="$OPTS -H $iheader"
|
||||
fi
|
||||
else
|
||||
url=$(echo "$1" | sed 's,upload/item,checkfile,g')
|
||||
fi
|
||||
trace "- Check file: $url/$(checksum $file)"
|
||||
tf1=$($MKTEMP_CMD) tf2=$($MKTEMP_CMD)
|
||||
|
||||
$CURL_CMD --fail -o "$tf1" --stderr "$tf2" $OPTS -XGET -s "$url/$(checksum "$file")"
|
||||
if [ $? -ne 0 ]; then
|
||||
info "Checking file failed!"
|
||||
cat "$tf1" >&2
|
||||
cat "$tf2" >&2
|
||||
info ""
|
||||
rm "$tf1" "$tf2"
|
||||
echo "failed"
|
||||
return 1
|
||||
else
|
||||
if cat "$tf1" | grep -q '{"exists":true'; then
|
||||
rm "$tf1" "$tf2"
|
||||
echo "y"
|
||||
else
|
||||
rm "$tf1" "$tf2"
|
||||
echo "n"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
process() {
|
||||
file="$1"
|
||||
file="$(realpath -e $1)"
|
||||
dir="$2"
|
||||
info "---- Processing $file ----------"
|
||||
declare -i curlrc=0
|
||||
set +e
|
||||
for url in $urls; do
|
||||
if [ "$distinct" = "y" ]; then
|
||||
trace "- Checking if $file has been uploaded to $url already"
|
||||
checkFile "$url" "$file"
|
||||
if [ $? -eq 0 ]; then
|
||||
res=$(checkFile "$url" "$file" "$dir")
|
||||
rc=$?
|
||||
curlrc=$(expr $curlrc + $rc)
|
||||
trace "- Result from checkfile: $res"
|
||||
if [ "$res" = "y" ]; then
|
||||
info "- Skipping file '$file' because it has been uploaded in the past."
|
||||
continue
|
||||
elif [ "$res" != "n" ]; then
|
||||
info "- Checking file failed, skipping the file."
|
||||
continue
|
||||
fi
|
||||
fi
|
||||
trace "- Uploading '$file' to '$url'."
|
||||
upload "$file" "$url"
|
||||
upload "$dir" "$file" "$url"
|
||||
rc=$?
|
||||
curlrc=$(expr $curlrc + $rc)
|
||||
if [ $rc -ne 0 ]; then
|
||||
@ -207,6 +314,16 @@ process() {
|
||||
fi
|
||||
}
|
||||
|
||||
findDir() {
|
||||
path="$1"
|
||||
for dir in "${watchdir[@]}"; do
|
||||
if [[ $path = ${dir}* ]]
|
||||
then
|
||||
echo $dir
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
if [ "$once" = "y" ]; then
|
||||
info "Uploading all files in '$watchdir'."
|
||||
MD="-maxdepth 1"
|
||||
@ -215,7 +332,7 @@ if [ "$once" = "y" ]; then
|
||||
fi
|
||||
for dir in "${watchdir[@]}"; do
|
||||
find "$dir" $MD -type f -print0 | while IFS= read -d '' -r file; do
|
||||
process "$file"
|
||||
process "$file" "$dir"
|
||||
done
|
||||
done
|
||||
else
|
||||
@ -225,8 +342,9 @@ else
|
||||
fi
|
||||
$INOTIFY_CMD $REC -m "${watchdir[@]}" -e close_write -e moved_to |
|
||||
while read path action file; do
|
||||
trace "The file '$file' appeared in directory '$path' via '$action'"
|
||||
dir=$(findDir "$path")
|
||||
trace "The file '$file' appeared in directory '$path' below '$dir' via '$action'"
|
||||
sleep 1
|
||||
process "$path$file"
|
||||
process "$(realpath -e "$path$file")" "$dir"
|
||||
done
|
||||
fi
|
||||
|
||||
Reference in New Issue
Block a user