Test improvements

shim_copy_baseDir.sh

@@ -129,17 +129,5 @@ sr3 remove subscribe/local_copy.conf
 
 echo "#test 0 comment comparing trees"
 
-
-cd shim_dirA
-find -H . -type f | xargs -d '\n' md5sum >../dirA.sums
-cd ../shim_dirB
-find -H . -type f | xargs -d '\n' md5sum >../dirB.sums
-cd ..
-
-diffs="`diff dirA.sums dirB.sums| wc -l`"
-
-if [ "${diffs}" -eq 0 ]; then
-       echo "RESULT: Good! trees the same"
-else
-       echo "RESULT: BAD trees have $diffs differences"
-fi
+./test_util/compare_trees.sh
+echo

shim_copy_flatten.sh

@@ -125,17 +125,5 @@ sr3 remove subscribe/local_copy.conf
 
 echo "#test 0 comment comparing trees"
 
-
-cd shim_dirA
-find -H . -type f | xargs -d '\n' md5sum >../dirA.sums
-cd ../shim_dirB
-find -H . -type f | xargs -d '\n' md5sum >../dirB.sums
-cd ..
-
-diffs="`diff dirA.sums dirB.sums| wc -l`"
-
-if [ "${diffs}" -eq 0 ]; then
-       echo "RESULT: Good! trees the same"
-else
-       echo "RESULT: BAD trees have $diffs differences"
-fi
+./test_util/compare_trees.sh
+echo

shim_copy_mirror.sh

@@ -128,27 +128,5 @@ sr3 remove subscribe/local_copy.conf
 
 echo "#test 0 comment comparing trees"
 
-
-cd shim_dirA
-find -H . -type f | xargs -d '\n' md5sum >../dirA.sums
-cd ../shim_dirB
-find -H . -type f | xargs -d '\n' md5sum >../dirB.sums
-cd ..
-
-diffs="`diff dirA.sums dirB.sums| wc -l`"
-
-cd shim_dirA
-find . -type l  | xargs ls -al | cut --bytes=42- >../dirA.links
-cd ../shim_dirB
-find . -type l  | xargs ls -al | cut --bytes=42- >../dirB.links
-cd ..
-
-sed 's+shim_dirB+shim_dirA+' dirB.links >dirC.links
-
-linkdiffs="`diff dirA.links dirC.links|wc -l`"
-
-if [ "${linkdiffs}" -eq 0 -a "${diffs}" -eq 0 ]; then
-	echo "RESULT: Good! trees links the same: `wc -l dirA.sums` files and `wc -l dirA.links` links mirrored"
-else
-       echo "RESULT: BAD tree differences in $diffs files, and $linkdiffs links"
-fi
+./test_util/compare_trees.sh --check-links
+echo

shim_copy_mirror_sftp.sh

@@ -124,27 +124,5 @@ sr3 remove subscribe/local_copy.conf
 
 echo "#test 0 comment comparing trees"
 
-
-cd shim_dirA
-find -H . -type f | xargs -d '\n' md5sum >../dirA.sums
-cd ../shim_dirB
-find -H . -type f | xargs -d '\n' md5sum >../dirB.sums
-cd ..
-
-diffs="`diff dirA.sums dirB.sums| wc -l`"
-
-cd shim_dirA
-find . -type l  | xargs ls -al | cut --bytes=42- >../dirA.links
-cd ../shim_dirB
-find . -type l  | xargs ls -al | cut --bytes=42- >../dirB.links
-cd ..
-
-sed 's+shim_dirB+shim_dirA+' dirB.links >dirC.links
-
-linkdiffs="`diff dirA.links dirC.links|wc -l`"
-
-if [ "${linkdiffs}" -eq 0 -a "${diffs}" -eq 0 ]; then
-	echo "RESULT: Good! trees links the same: `wc -l dirA.sums` files and `wc -l dirA.links` links mirrored"
-else
-       echo "RESULT: BAD tree differences in $diffs files, and $linkdiffs links"
-fi
+./test_util/compare_trees.sh --check-links
+echo

shim_copy_post.sh

@@ -83,6 +83,8 @@ cp dirone/dirtwo/filetwo "dirone/filetwo copy with spaces in the name"
 echo "#test 1 sha512 160 copy a file with spaces in the name (both source and dest)"
 cp "dirone/filetwo copy with spaces in the name" "dirone/second spaced file"
 
+# sleep to allow filetwo to be downloaded before renaming it to filefour
+sleep 10
 echo "#test 1 rename 145 rename in a sub-dir"
 mv dirone/dirtwo/filetwo dirone/dirtwo/filefour
 
@@ -118,10 +120,16 @@ echo "#test 1 rename move test_file into dirthree subdir (new name)"
 mv test_file dirthree/new_test_file
 
 # issue #208 rm directory with trailing slash
-echo "#test 1 directory create with trailing slash"
-mkdir dir_test/
+echo "#test 1 directory create without trailing slash"
+mkdir dir_test
 
 echo "#test 1 remove directory with trailing slash"
 rm -r dir_test/
 
+echo "#test 1 directory create with trailing slash"
+mkdir dir_test2/
+
+echo "#test 1 remove directory with trailing slash"
+rm -r dir_test2/
+
 echo "#test 0 comment 160 shim copy posting end"

shim_copy_post2.sh

@@ -96,7 +96,8 @@ cp dirone/dirtwo/filetwo "dirone/filetwo copy with spaces in the name"
 echo "#test 1 sha512 160 copy a file with spaces in the name (both source and dest)"
 cp "dirone/filetwo copy with spaces in the name" "dirone/second spaced file"
 
-
+# sleep to allow filetwo to be downloaded before renaming it to filefour
+sleep 10
 echo "#test 1 rename 145 rename in a sub-dir"
 mv dirone/dirtwo/filetwo dirone/dirtwo/filefour
 

shim_copy_strip.sh

@@ -138,18 +138,5 @@ sr3 stop subscribe/local_copy.conf
 sr3 remove subscribe/local_copy.conf
 
 echo "#test 0 comment comparing trees"
-
-
-cd shim_dirA
-find -H . -type f | xargs -d '\n' md5sum >../dirA.sums
-cd ../shim_dirB
-find -H . -type f | xargs -d '\n' md5sum >../dirB.sums
-cd ..
-
-diffs="`diff dirA.sums dirB.sums| wc -l`"
-
-if [ "${diffs}" -eq 0 ]; then
-       echo "RESULT: Good! trees the same"
-else
-       echo "RESULT: BAD trees have $diffs differences"
-fi
+./test_util/compare_trees.sh
+echo

shim_copy_strip_slash.sh

@@ -130,18 +130,6 @@ sr3 stop subscribe/local_copy.conf
 sr3 remove subscribe/local_copy.conf
 
 echo "#test 0 comment comparing trees"
-
-
-cd shim_dirA
-find -H . -type f | xargs -d '\n' md5sum >../dirA.sums
-cd ../shim_dirB
-find -H . -type f | xargs -d '\n' md5sum >../dirB.sums
-cd ..
-
-diffs="`diff dirA.sums dirB.sums| wc -l`"
-
-if [ "${diffs}" -eq 0 ]; then
-       echo "RESULT: Good! trees the same"
-else
-       echo "RESULT: BAD trees have $diffs differences"
-fi
+
+./test_util/compare_trees.sh
+echo

shim_test_race_conditions.sh

@@ -208,27 +208,7 @@ sleep ${sleepytime}
 sr3 remove cpost/local_post.conf
 
 # wait for retries to finish before stopping subscriber
-
-echo "Waiting for retries to finish..."
-stalled=0
-stalled_value=-1
-retry_msgcnt="$(cat ~/.cache/sr3/subscribe/local_copy/*retry* 2>/dev/null | sort -u | wc -l)"
-while [ $retry_msgcnt -gt 0 ]; do
-        printf "${flow_test_name} Still %4s messages to retry, waiting...\n" "$retry_msgcnt"
-        sleep 15
-        retry_msgcnt="$(cat ~/.cache/sr3/subscribe/local_copy/*retry* 2>/dev/null | sort -u | wc -l)"
-
-        if [ "${stalled_value}" == "${retry_msgcnt}" ]; then
-              stalled=$((stalled+1));
-              if [ "${stalled}" == 5 ]; then
-                 printf "\n    Warning some retries stalled, skipping..., might want to check the logs\n\n"
-                 retry_msgcnt=0
-              fi
-        else
-              stalled_value=$retry_msgcnt
-              stalled=0
-        fi
-done
+./test_util/wait_for_retries.sh
 
 # wait a bit longer
 sleep 60
@@ -241,33 +221,6 @@ rm ~/.config/sr3/plugins/screw_up_order.py
 sed -i '/callback screw_up_order/d' ~/.config/sr3/default.conf
 
 echo "#test 0 comment comparing trees"
-
-
-cd shim_dirA
-find -H . -type f | xargs -d '\n' md5sum >../dirA.sums
-cd ../shim_dirB
-find -H . -type f | xargs -d '\n' md5sum >../dirB.sums
-cd ..
-
-diffs="`diff dirA.sums dirB.sums| wc -l`"
-
-cd shim_dirA
-find . -type l  | xargs ls -al | cut --bytes=42- >../dirA.links
-cd ../shim_dirB
-find . -type l  | xargs ls -al | cut --bytes=42- >../dirB.links
-cd ..
-
-sed 's+shim_dirB+shim_dirA+' dirB.links >dirC.links
-
-linkdiffs="`diff dirA.links dirC.links|wc -l`"
-
-if [ "${linkdiffs}" -eq 0 -a "${diffs}" -eq 0 ]; then
-	echo "RESULT: Good! trees links the same: `wc -l dirA.sums` files and `wc -l dirA.links` links mirrored"
-else
-       echo "RESULT: BAD tree differences in $diffs files, and $linkdiffs links"
-       echo "shim_dirA:"
-       ls -hal shim_dirA
-       echo 
-       echo "shim_dirB:"
-       ls -hal shim_dirB
-fi
+
+./test_util/compare_trees.sh --check-links
+echo

test_util/compare_trees.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+
+check_links=false
+if [[ " $* " == *"--check-links"* ]]; then
+    check_links=true
+fi
+
+
+# check for file differences
+cd shim_dirA
+find -H . -type f | xargs -d '\n' md5sum >../dirA.sums
+cd ../shim_dirB
+find -H . -type f | xargs -d '\n' md5sum >../dirB.sums
+cd ..
+
+diffs="`diff dirA.sums dirB.sums| wc -l`"
+
+
+# check for link differences
+if [[ $check_links == true ]]; then
+	cd shim_dirA
+	find . -type l  | xargs ls -al | cut --bytes=42- >../dirA.links
+	cd ../shim_dirB
+	find . -type l  | xargs ls -al | cut --bytes=42- >../dirB.links
+	cd ..
+	sed 's+shim_dirB+shim_dirA+' dirB.links >dirC.links
+
+	linkdiffs="`diff dirA.links dirC.links|wc -l`"
+fi
+
+
+# check for directory differences (missing dirs, extra dirs, etc.)
+dir_diffs="$(diff -q --no-dereference -r shim_dirA shim_dirB)"
+dir_diffs_result=$?
+if [ "${dir_diffs_result}" -ne 0 ]; then
+	dir_diffs_msg="and these other differences: ${dir_diffs/$'\n'/, }"
+fi
+
+
+# report result with links
+if [[ $check_links == true ]]; then
+
+	if [ "${diffs}" -eq 0 ] && [ "${dir_diffs_result}" -eq 0 ] && [ "${linkdiffs}" -eq 0 ]; then
+		echo "RESULT: Good! trees links the same: $(wc -l dirA.sums) files and $(wc -l dirA.links) links mirrored"
+	else
+		echo "RESULT: BAD tree differences in $diffs files, and $linkdiffs links ${dir_diffs_msg}"
+	fi
+
+# report result without links
+else
+
+	if [ "${diffs}" -eq 0 ] && [ "${dir_diffs_result}" -eq 0 ]; then
+		echo "RESULT: Good! trees the same: $(wc -l dirA.sums) files mirrored"
+	else
+		echo "RESULT: BAD trees have $diffs file differences ${dir_diffs_msg}"
+	fi
+fi

test_util/wait_for_retries.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+echo "Waiting for retries to finish..."
+stalled=0
+stalled_value=-1
+retry_msgcnt="$(cat ~/.cache/sr3/subscribe/local_copy/*retry* 2>/dev/null | sort -u | wc -l)"
+while [ $retry_msgcnt -gt 0 ]; do
+        printf "${flow_test_name} Still %4s messages to retry, waiting...\n" "$retry_msgcnt"
+        sleep 15
+        retry_msgcnt="$(cat ~/.cache/sr3/subscribe/local_copy/*retry* 2>/dev/null | sort -u | wc -l)"
+
+        if [ "${stalled_value}" == "${retry_msgcnt}" ]; then
+              stalled=$((stalled+1));
+              if [ "${stalled}" == 5 ]; then
+                 printf "\n    Warning some retries stalled, skipping..., might want to check the logs\n\n"
+                 retry_msgcnt=0
+              fi
+        else
+              stalled_value=$retry_msgcnt
+              stalled=0
+        fi
+done