From 86aac250a3e09b762faf033beabb87e088cf0d18 Mon Sep 17 00:00:00 2001 From: Malte Date: Thu, 29 Oct 2020 06:11:45 +0100 Subject: [PATCH 1/3] fixes hard-wired created date (https://github.com/eikek/docspell/pull/403#discussion_r513790807) --- .../consumedir-cleaner/consumedir-cleaner.sh | 179 ++++++++++++++++++ 1 file changed, 179 insertions(+) create mode 100644 tools/consumedir-cleaner/consumedir-cleaner.sh diff --git a/tools/consumedir-cleaner/consumedir-cleaner.sh b/tools/consumedir-cleaner/consumedir-cleaner.sh new file mode 100644 index 00000000..1e8c2574 --- /dev/null +++ b/tools/consumedir-cleaner/consumedir-cleaner.sh @@ -0,0 +1,179 @@ +#!/usr/bin/env bash + +echo "##################### START #####################" + +echo " Docspell Consumedir Cleaner - v0.1 beta" +echo " by totti4ever" && echo +echo " $(date)" +echo +echo "#################################################" +echo && echo + +jq --version > /dev/null +if [ $? -ne 0 ]; then + echo "please install 'jq'" + exit -4 +fi + +ds_url=${1%/} +ds_user_param=$2 +ds_user=${ds_user_param#*/} +ds_collective=${ds_user_param%%/*} +ds_password=$3 +ds_consumedir_path=${4%/} +ds_archive_path=$ds_consumedir_path/_archive/$ds_collective + + +if [ $# -ne 4 ]; then + echo "FATAL Exactly four parameters needed" + exit -3 +elif [ "$1" == "" ] || [ "$2" == "" ] || [ "$3" == "" ] || [ "$4" == "" ]; then + echo "FATAL Parameter missing" + echo " ds_url: $ds_url" + echo " ds_user: $ds_user" + echo " ds_password: $ds_password" + echo " ds_consumedir_path: $ds_consumedir_path" + exit -2 +elif [ "$ds_collective" == "_archive" ]; then + echo "FATAL collective name '_archive' is not supported by this script" + exit -1 +fi + + +############# FUNCTIONS +function curl_call() { + curl_cmd="$1 -H 'X-Docspell-Auth: $ds_token'" + curl_result=$(eval $curl_cmd) + curl_code=$? + + if [ "$curl_result" == '"Authentication failed."' ] || [ "$curl_result" == 'Response timed out' ]; then + printf "\nNew login required ($curl_result)... " + login + printf "%${#len_resultset}s" " "; printf " .." + curl_call $1 + + elif [ "$curl_result" == "Bad Gateway" ] || [ "$curl_result" == '404 page not found' ]; then + echo "FATAL Connection to server failed" + exit -1 + fi +} + + +function login() { + curl_call "curl -s -X POST -d '{\"account\": \"$ds_collective/$ds_user\", \"password\": \"$ds_password\"}' ${ds_url}/api/v1/open/auth/login" + + curl_status=$(echo $curl_result | jq -r ".success") + + if [ "$curl_status" == "true" ]; then + ds_token=$(echo $curl_result | jq -r ".token") + echo "Login successfull ( Token: $ds_token )" + + else + echo "FATAL Login not succesfull" + exit 1 + + fi +} + +############# END + +echo "Settings:" +if [ "$DS_CC_REMOVE" == "true" ]; then + echo " ### !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ###" + echo " - DELETE files? YES" + echo " when already existing in Docspell. This cannot be undone!" + echo " ### !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ###" +else + echo " - DELETE files? no" + echo " moving already uploaded files to archive" +fi +echo +if [ "$DS_CC_UPLOAD_MISSING" == true ]; then + echo " - UPLOAD files? YES" + echo " files not existing in Docspell will be uploaded and will be re-checked in the next run." +else + echo " - UPLOAD files? no" + echo " files not existing in Docspell will NOT be uploaded and stay where they are." +fi +echo && echo +echo "Press 'ctrl+c' to cancel" +for ((i=9;i>=0;i--)); do + printf "\r waiting $i seconds " + sleep 1s +done +echo && echo + +# login, get token +login + +echo "Scanning folder for collective '$ds_collective' ($ds_consumedir_path/$ds_collective)" +echo && echo + +while read -r line +do + tmp_filepath=$line + + if [ "$tmp_filepath" == "" ]; then + echo "no files found" && echo + exit 0 #no results + elif [ ! -f "$tmp_filepath" ]; then + echo "FATAL no access to file: $tmp_filepath" + exit 3 + fi + + echo "Checking '$tmp_filepath'" + printf "%${#len_resultset}s" " "; printf " " + + # check for checksum + tmp_checksum=$(sha256sum "$tmp_filepath" | awk '{print $1}') + + curl_call "curl -s -X GET '$ds_url/api/v1/sec/checkfile/$tmp_checksum'" + curl_status=$(echo $curl_result | jq -r ".exists") + + if [ $curl_code -ne 0 ]; then + # error + echo "ERROR $curl_result // $curl_status" + + # file exists in Docspell + elif [ "$curl_status" == "true" ]; then + item_name=$(echo $curl_result | jq -r ".items[0].name") + item_id=$(echo $curl_result | jq -r ".items[0].id") + echo "File already exists: '$item_name (ID: $item_id)'" + + printf "%${#len_resultset}s" " "; printf " " + if [ "$DS_CC_REMOVE" == "true" ]; then + echo "... removing file" + rm "$tmp_filepath" + else + created=$(echo $curl_result | jq -r ".items[0].created") + cur_dir="$ds_archive_path/$(date -d @$(echo "($created+500)/1000" | bc) +%Y-%m +)" + echo "... moving to archive by month added ('$cur_dir')" + mkdir -p "$cur_dir" + mv "$tmp_filepath" "$cur_dir/" + fi + + # file does not exist in Docspell + else + + echo "Files does not exist, yet" + if [ "$DS_CC_UPLOAD_MISSING" == true ]; then + printf "%${#len_resultset}s" " "; printf " " + printf "...uploading file.." + curl_call "curl -s -X POST '$ds_url/api/v1/sec/upload/item' -H 'Content-Type: multipart/form-data' -F 'file=@$tmp_filepath'" + curl_status=$(echo $curl_result | jq -r ".success") + if [ "$curl_status" == "true" ]; then + echo ". done" + else + echo -e "\nERROR $curl_result" + fi + fi + fi + + echo +done \ + <<< $(find $ds_consumedir_path/$ds_collective -type f) + + +echo ################# DONE ################# +date From e7381aad77851eef695bac0f83c242ee03ce4f20 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Wed, 28 Oct 2020 22:51:43 +0100 Subject: [PATCH 2/3] Fix sql error for mariadb <10.4 MariaDB below 10.4 doesn't support parentheses around selects for `intersect` and `union`. https://mariadb.com/kb/en/intersect/#parentheses Fixes #404 --- .../src/main/scala/docspell/store/queries/QItem.scala | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/modules/store/src/main/scala/docspell/store/queries/QItem.scala b/modules/store/src/main/scala/docspell/store/queries/QItem.scala index a5311500..bce5f836 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QItem.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QItem.scala @@ -328,17 +328,14 @@ object QItem { val EC = REquipment.Columns // inclusive tags are AND-ed - val tagSelectsIncl = (q.tagsInclude + val tagSelectsIncl = q.tagsInclude .map(tid => selectSimple( List(RTagItem.Columns.itemId), RTagItem.table, RTagItem.Columns.tagId.is(tid) ) - ) ++ q.tagCategoryIncl.map(cat => - TagItemName.itemsInCategory(NonEmptyList.of(cat)) - )) - .map(f => sql"(" ++ f ++ sql") ") + ) ++ q.tagCategoryIncl.map(cat => TagItemName.itemsInCategory(NonEmptyList.of(cat))) // exclusive tags are OR-ed val tagSelectsExcl = From 9b1879f56450379f70b0a17e84a01a07f9913a31 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Thu, 29 Oct 2020 04:34:46 +0100 Subject: [PATCH 3/3] Update icu4j to 68.1 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 820b2287..353a2ab1 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -18,7 +18,7 @@ object Dependencies { val Fs2Version = "2.4.4" val H2Version = "1.4.200" val Http4sVersion = "0.21.8" - val Icu4jVersion = "67.1" + val Icu4jVersion = "68.1" val JsoupVersion = "1.13.1" val KindProjectorVersion = "0.10.3" val Log4sVersion = "1.8.2"