mirror of
https://github.com/TheAnachronism/docspell.git
synced 2025-06-21 01:48:26 +00:00
Add route to check for files by their checksum
Adopt scripts in `tools/` to check for existing files using these routes.
This commit is contained in:
Eike Kettner
@ -20,8 +20,8 @@ if [[ ${PIPESTATUS[0]} -ne 4 ]]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
OPTIONS=om:hdp:v
|
||||
LONGOPTS=once,memorize:,help,delete,path:,verbose
|
||||
OPTIONS=omhdp:v
|
||||
LONGOPTS=once,distinct,help,delete,path:,verbose
|
||||
|
||||
! PARSED=$(getopt --options=$OPTIONS --longoptions=$LONGOPTS --name "$0" -- "$@")
|
||||
if [[ ${PIPESTATUS[0]} -ne 0 ]]; then
|
||||
@ -34,7 +34,7 @@ fi
|
||||
eval set -- "$PARSED"
|
||||
|
||||
declare -a watchdir
|
||||
help=n verbose=n delete=n once=n memodir=
|
||||
help=n verbose=n delete=n once=n distinct=n
|
||||
while true; do
|
||||
case "$1" in
|
||||
-h|--help)
|
||||
@ -57,9 +57,9 @@ while true; do
|
||||
watchdir+=("$2")
|
||||
shift 2
|
||||
;;
|
||||
-m|--memorize)
|
||||
memodir="$2"
|
||||
shift 2
|
||||
-m|--distinct)
|
||||
distinct=y
|
||||
shift
|
||||
;;
|
||||
--)
|
||||
shift
|
||||
@ -72,6 +72,36 @@ while true; do
|
||||
esac
|
||||
done
|
||||
|
||||
|
||||
showUsage() {
|
||||
echo "Upload files in a directory"
|
||||
echo ""
|
||||
echo "Usage: $0 [options] url url ..."
|
||||
echo
|
||||
echo "Options:"
|
||||
echo " -v | --verbose Print more to stdout. (value: $verbose)"
|
||||
echo " -d | --delete Delete the file if successfully uploaded. (value: $delete)"
|
||||
echo " -p | --path <dir> The directories to watch. This is required. (value: ${watchdir[@]})"
|
||||
echo " -h | --help Prints this help text. (value: $help)"
|
||||
echo " -m | --distinct Optional. Upload only if the file doesn't already exist. (value: $distinct)"
|
||||
echo " -o | --once Instead of watching, upload all (pdf) files in that dir. (value: $once)"
|
||||
echo ""
|
||||
echo "Arguments:"
|
||||
echo " A list of URLs to upload the files to."
|
||||
echo ""
|
||||
echo "Example: Watch directory"
|
||||
echo "$0 --path ~/Downloads -m -dv http://localhost:7880/api/v1/open/upload/item/abcde-12345-abcde-12345"
|
||||
echo ""
|
||||
echo "Example: Upload all files in a directory"
|
||||
echo "$0 --path ~/Downloads -m -dv --once http://localhost:7880/api/v1/open/upload/item/abcde-12345-abcde-12345"
|
||||
echo ""
|
||||
}
|
||||
|
||||
if [ "$help" = "y" ]; then
|
||||
showUsage
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# handle non-option arguments
|
||||
if [[ $# -eq 0 ]]; then
|
||||
echo "$0: No upload URLs given."
|
||||
@ -117,15 +147,22 @@ checksum() {
|
||||
$SHA256_CMD "$1" | cut -d' ' -f1 | xargs
|
||||
}
|
||||
|
||||
checkFile() {
|
||||
local url=$(echo "$1" | sed 's,upload/item,checkfile,g')
|
||||
local file="$2"
|
||||
trace "Check file: $url/$(checksum "$file")"
|
||||
$CURL_CMD -XGET -s "$url/$(checksum "$file")" | (2>&1 1>/dev/null grep '"exists":true')
|
||||
}
|
||||
|
||||
process() {
|
||||
file="$1"
|
||||
info "---- Processing $file ----------"
|
||||
declare -i curlrc=0
|
||||
set +e
|
||||
for url in $urls; do
|
||||
if [ -n "$memodir" ] && [ -f "$memodir/.docspell-consume" ]; then
|
||||
if [ "$distinct" = "y" ]; then
|
||||
trace "- Checking if $file has been uploaded to $url already"
|
||||
cat "$memodir/.docspell-consume" | grep "$url" | (2>&1 1>/dev/null grep "$(checksum "$file")")
|
||||
checkFile "$url" "$file"
|
||||
if [ $? -eq 0 ]; then
|
||||
info "- Skipping file '$file' because it has been uploaded in the past."
|
||||
continue
|
||||
@ -137,13 +174,6 @@ process() {
|
||||
curlrc=$(expr $curlrc + $rc)
|
||||
if [ $rc -ne 0 ]; then
|
||||
trace "Upload to '$url' failed!"
|
||||
else
|
||||
if [ -n "$memodir" ]; then
|
||||
trace "- Adding file '$file' to list of uploaded files for '$url'"
|
||||
set +C
|
||||
echo "$(checksum "$file") : $url" >> "$memodir/.docspell-consume"
|
||||
set -C
|
||||
fi
|
||||
fi
|
||||
done
|
||||
set -e
|
||||
@ -163,37 +193,6 @@ process() {
|
||||
fi
|
||||
}
|
||||
|
||||
showUsage() {
|
||||
echo "Upload files in a directory"
|
||||
echo ""
|
||||
echo "Usage: $0 [options] url url ..."
|
||||
echo
|
||||
echo "Options:"
|
||||
echo " -v | --verbose Print more to stdout. (value: $verbose)"
|
||||
echo " -d | --delete Delete the file if successfully uploaded. (value: $delete)"
|
||||
echo " -p | --path <dir> The directories to watch. This is required. (value: ${watchdir[@]})"
|
||||
echo " -h | --help Prints this help text. (value: $help)"
|
||||
echo " -m | --memorize <dir> Optional directory (writable) to store checksums of"
|
||||
echo " uploaded files. This is used to skip duplicates. (value: $memodir)"
|
||||
echo " -o | --once Instead of watching, upload all (pdf) files in that dir. (value: $once)"
|
||||
echo ""
|
||||
echo "Arguments:"
|
||||
echo " A list of URLs to upload the files to."
|
||||
echo ""
|
||||
echo "Example: Watch directory"
|
||||
echo "$0 --path ~/Downloads -m ~/ -dv http://localhost:7880/api/v1/open/upload/item/abcde-12345-abcde-12345"
|
||||
echo ""
|
||||
echo "Example: Upload all files in a directory"
|
||||
echo "$0 --path ~/Downloads -m ~/ -dv --once http://localhost:7880/api/v1/open/upload/item/abcde-12345-abcde-12345"
|
||||
echo ""
|
||||
}
|
||||
|
||||
|
||||
if [ "$help" = "y" ]; then
|
||||
showUsage
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "$once" = "y" ]; then
|
||||
info "Uploading all files in '$watchdir'."
|
||||
for dir in "${watchdir[@]}"; do
|
||||
|
||||
25
tools/ds.sh
25
tools/ds.sh
@ -23,6 +23,7 @@ CURL_CMD="curl"
|
||||
FILE_CMD="file"
|
||||
GREP_CMD="grep"
|
||||
MKTEMP_CMD="mktemp"
|
||||
SHA256_CMD="sha256sum"
|
||||
|
||||
! getopt --test > /dev/null
|
||||
if [[ ${PIPESTATUS[0]} -ne 4 ]]; then
|
||||
@ -78,7 +79,17 @@ info() {
|
||||
echo "$1"
|
||||
}
|
||||
|
||||
upload() {
|
||||
checksum() {
|
||||
$SHA256_CMD "$1" | cut -d' ' -f1 | xargs
|
||||
}
|
||||
|
||||
checkFile() {
|
||||
local url=$(echo "$1" | sed 's,upload/item,checkfile,g')
|
||||
local file="$2"
|
||||
$CURL_CMD -XGET -s "$url/$(checksum "$file")" | (2>&1 1>/dev/null grep '"exists":true')
|
||||
}
|
||||
|
||||
upload_file() {
|
||||
tf=$($MKTEMP_CMD) rc=0
|
||||
$CURL_CMD -# -o "$tf" --stderr "$tf" -w "%{http_code}" -XPOST -F file=@"$1" "$2" | (2>&1 1>/dev/null grep 200)
|
||||
rc=$(expr $rc + $?)
|
||||
@ -96,6 +107,16 @@ upload() {
|
||||
fi
|
||||
}
|
||||
|
||||
upload() {
|
||||
checkFile "$2" "$1"
|
||||
if [ $? -eq 0 ]; then
|
||||
info "File already exists at url $2"
|
||||
return 0
|
||||
else
|
||||
upload_file "$1" "$2"
|
||||
fi
|
||||
}
|
||||
|
||||
showUsage() {
|
||||
info "Upload files to docspell"
|
||||
info ""
|
||||
@ -162,7 +183,7 @@ for file in $*; do
|
||||
set +e
|
||||
upload "$file" "$url"
|
||||
set -e
|
||||
if [ "$delete" = "y" ] && [ $rc -eq 0 ]; then
|
||||
if [ "$delete" = "y" ] && [ $? -eq 0 ]; then
|
||||
info "Deleting file: $file"
|
||||
rm -f "$file"
|
||||
fi
|
||||
|
||||
Reference in New Issue
Block a user