From d2d80a539227070063f5ce158acf7ff25fba5815 Mon Sep 17 00:00:00 2001 From: Darren Ldl Date: Sun, 9 Jun 2019 22:00:10 +1000 Subject: [PATCH 1/4] Upgraded coverage test suite --- cov_tests/hash_tests.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cov_tests/hash_tests.sh b/cov_tests/hash_tests.sh index db5a0914..f1cacc7d 100755 --- a/cov_tests/hash_tests.sh +++ b/cov_tests/hash_tests.sh @@ -4,14 +4,15 @@ source kcov_blkar_fun.sh exit_code=0 -HASHES=("sha1") +HASHES=("sha1" "sha256" "sha512" "blake2b-256" "blake2b-512") # Record the hashes a[0]=$(sha1sum dummy | awk '{print $1}') a[1]=$(sha256sum dummy | awk '{print $1}') a[2]=$(sha512sum dummy | awk '{print $1}') if [[ $(command -v b2sum) != "" ]]; then - a[3]=$(b2sum dummy | awk '{print $1}') + a[3]=$(b2sum -l 256 dummy | awk '{print $1}') + a[4]=$(b2sum dummy | awk '{print $1}') fi # Encode in all 4 hashes From fc94e40126abf5f7021e69b3665c29ef2048d549 Mon Sep 17 00:00:00 2001 From: Darren Ldl Date: Sun, 9 Jun 2019 22:24:06 +1000 Subject: [PATCH 2/4] Added corruption_tests_decode_stdout and burst_corruption_tests_decode_stdout to cov_tests --- .../burst_corruption_tests_decode_stdout.sh | 123 ++++++++++++++++++ cov_tests/corruption_tests_decode_stdout.sh | 98 ++++++++++++++ cov_tests/test_list.sh | 2 + 3 files changed, 223 insertions(+) create mode 100755 cov_tests/burst_corruption_tests_decode_stdout.sh create mode 100755 cov_tests/corruption_tests_decode_stdout.sh diff --git a/cov_tests/burst_corruption_tests_decode_stdout.sh b/cov_tests/burst_corruption_tests_decode_stdout.sh new file mode 100755 index 00000000..62baa7f4 --- /dev/null +++ b/cov_tests/burst_corruption_tests_decode_stdout.sh @@ -0,0 +1,123 @@ +#!/bin/bash + +source kcov_blkar_fun.sh + +exit_code=0 + +VERSIONS=(17) + +source functions.sh + +file_size=$(ls -l dummy | awk '{ print $5 }') + +# generate test data +dd if=/dev/urandom of=dummy bs=$file_size count=1 &>/dev/null + +for ver in ${VERSIONS[*]}; do + for (( i=0; i < 1; i++ )); do + if [[ $ver == 17 ]]; then + data_shards=$((1 + RANDOM % 128)) + parity_shards=$((1 + RANDOM % 128)) + burst=$((1 + RANDOM % 10)) + elif [[ $ver == 18 ]]; then + data_shards=$((1 + RANDOM % 128)) + parity_shards=$((1 + RANDOM % 128)) + burst=$((1 + RANDOM % 10)) + else + data_shards=$((1 + RANDOM % 128)) + parity_shards=$((1 + RANDOM % 128)) + burst=$((1 + RANDOM % 10)) + fi + + container_name=burst_$data_shards\_$parity_shards\_$burst\_$ver.sbx + + echo -n "Encoding" + output=$(./../blkar encode --json --sbx-version $ver -f dummy $container_name \ + --hash sha1 \ + --rs-data $data_shards --rs-parity $parity_shards \ + --burst $burst) + if [[ $(echo $output | jq -r ".error") != null ]]; then + echo " ==> Invalid JSON" + exit_code=1 + fi + if [[ $(echo $output | jq -r ".stats.sbxVersion") == "$ver" ]]; then + echo -n " ==> Okay" + else + echo -n " ==> NOT okay" + exit_code=1 + fi + if [[ $(echo $output | jq -r ".stats.hash" | awk '{ print $1 }') == "SHA1" ]]; then + echo " ==> Okay" + else + echo " ==> NOT okay" + exit_code=1 + fi + + echo -n "Checking burst error resistance level" + output=$(./../blkar show --json --guess-burst $container_name) + if [[ $(echo $output | jq -r ".error") != null ]]; then + echo " ==> Invalid JSON" + exit_code=1 + fi + if [[ $(echo $output | jq -r ".bestGuessForBurstErrorResistanceLevel") == $burst ]]; then + echo " ==> Okay" + else + echo " ==> NOT okay" + exit_code=1 + fi + + if [[ $ver == 17 ]]; then + block_size=512 + elif [[ $ver == 18 ]]; then + block_size=128 + else + block_size=4096 + fi + + echo "Corrupting at $parity_shards random positions, burst error size is $burst" + for (( p=0; p < $parity_shards; p++ )); do + pos=$(( (RANDOM % $file_size) / $block_size )) + # echo "#$p corruption, corrupting byte at position : $pos" + corrupt $pos $block_size $burst $container_name + done + + echo -n "Repairing" + output=$(./../blkar repair --json --verbose $container_name) + if [[ $(echo $output | jq -r ".error") != null ]]; then + echo " ==> Invalid JSON" + exit_code=1 + fi + if [[ $(echo $output | jq -r ".stats.sbxVersion") == "$ver" ]]; then + echo " ==> Okay" + else + echo " ==> NOT okay" + exit_code=1 + fi + + output_name=dummy_$data_shards\_$parity_shards + + echo -n "Decoding" + output=$(./../blkar decode --json -f $container_name - 2>&1 > $output_name) + if [[ $(echo $output | jq -r ".error") != null ]]; then + echo " ==> Invalid JSON" + exit_code=1 + fi + if [[ $(echo $output | jq -r ".stats.sbxVersion") == "$ver" ]]; then + echo " ==> Okay" + else + echo " ==> NOT okay" + exit_code=1 + fi + + echo -n "Comparing decoded data to original" + cmp dummy $output_name + if [[ $? == 0 ]]; then + echo " ==> Okay" + else + echo " ==> NOT okay" + exit_code=1 + fi + done +done + +echo $exit_code > exit_code diff --git a/cov_tests/corruption_tests_decode_stdout.sh b/cov_tests/corruption_tests_decode_stdout.sh new file mode 100755 index 00000000..13bdcb47 --- /dev/null +++ b/cov_tests/corruption_tests_decode_stdout.sh @@ -0,0 +1,98 @@ +#!/bin/bash + +source kcov_blkar_fun.sh + +exit_code=0 + +VERSIONS=(17) + +source functions.sh + +file_size=$(ls -l dummy | awk '{ print $5 }') + +# generate test data +dd if=/dev/urandom of=dummy bs=$file_size count=1 &>/dev/null + +for ver in ${VERSIONS[*]}; do + for (( i=0; i < 1; i++ )); do + if [[ $ver == 17 ]]; then + data_shards=$((1 + RANDOM % 128)) + parity_shards=$((1 + RANDOM % 128)) + elif [[ $ver == 18 ]]; then + data_shards=$((1 + RANDOM % 128)) + parity_shards=$((1 + RANDOM % 128)) + else + data_shards=$((1 + RANDOM % 128)) + parity_shards=$((1 + RANDOM % 128)) + fi + + container_name=corrupt_$data_shards\_$parity_shards\_$ver.sbx + + echo -n "Encoding in version $ver, data = $data_shards, parity = $parity_shards" + output=$(./../blkar encode --json --sbx-version $ver -f dummy $container_name \ + --hash sha1 \ + --rs-data $data_shards --rs-parity $parity_shards) + if [[ $(echo $output | jq -r ".error") != null ]]; then + echo " ==> Invalid JSON" + exit_code=1 + fi + if [[ $(echo $output | jq -r ".stats.sbxVersion") == "$ver" ]]; then + echo -n " ==> Okay" + else + echo -n " ==> NOT okay" + exit_code=1 + fi + if [[ $(echo $output | jq -r ".stats.hash" | awk '{ print $1 }') == "SHA1" ]]; then + echo " ==> Okay" + else + echo " ==> NOT okay" + exit_code=1 + fi + + echo "Corrupting at $parity_shards random positions" + for (( p=0; p < $parity_shards; p++ )); do + pos=$((RANDOM % $file_size)) + # echo "#$p corruption, corrupting byte at position : $pos" + corrupt $pos $container_name + done + + echo -n "Repairing" + output=$(./../blkar repair --json --verbose $container_name) + if [[ $(echo $output | jq -r ".error") != null ]]; then + echo " ==> Invalid JSON" + exit_code=1 + fi + if [[ $(echo $output | jq -r ".stats.sbxVersion") == "$ver" ]]; then + echo " ==> Okay" + else + echo " ==> NOT okay" + exit_code=1 + fi + + output_name=dummy_$data_shards\_$parity_shards + + echo -n "Decoding" + output=$(./../blkar decode --json -f $container_name - 2>&1 > $output_name) + if [[ $(echo $output | jq -r ".error") != null ]]; then + echo " ==> Invalid JSON" + exit_code=1 + fi + if [[ $(echo $output | jq -r ".stats.sbxVersion") == "$ver" ]]; then + echo " ==> Okay" + else + echo " ==> NOT okay" + exit_code=1 + fi + + echo -n "Comparing decoded data to original" + cmp dummy $output_name + if [[ $? == 0 ]]; then + echo " ==> Okay" + else + echo " ==> NOT okay" + exit_code=1 + fi + done +done + +echo $exit_code > exit_code diff --git a/cov_tests/test_list.sh b/cov_tests/test_list.sh index 314bd170..3f78d7ee 100755 --- a/cov_tests/test_list.sh +++ b/cov_tests/test_list.sh @@ -12,7 +12,9 @@ tests=( "show_pick_uid_tests" "out_file_logic_tests" "corruption_tests" + "corruption_tests_decode_stdout" "burst_corruption_tests" + "burst_corruption_tests_decode_stdout" "sort_tests" "sort_multi_pass" "sort_multi_pass_no_skip" From 8ce6d5e5bc8661e5d88bc986c532b660c90435ff Mon Sep 17 00:00:00 2001 From: Darren Ldl Date: Sun, 9 Jun 2019 22:39:09 +1000 Subject: [PATCH 3/4] Updated cov.sh dialogue --- cov.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cov.sh b/cov.sh index b15db118..27b292ef 100755 --- a/cov.sh +++ b/cov.sh @@ -40,8 +40,11 @@ if [[ $? != 0 ]]; then fi cd .. +echo "" +echo "========================================" echo "" echo "Merging all code coverage reports" +echo "" echo "========================================" rm -rf target/cov/total mkdir -p target/cov/total From 6aca8c15e6c7215c12c42de91e6bfb20fb3ffb4d Mon Sep 17 00:00:00 2001 From: Darren Ldl Date: Sun, 9 Jun 2019 22:41:46 +1000 Subject: [PATCH 4/4] Disabled tests in coverage test suite --- cov_tests/test_list.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/cov_tests/test_list.sh b/cov_tests/test_list.sh index 3f78d7ee..314bd170 100755 --- a/cov_tests/test_list.sh +++ b/cov_tests/test_list.sh @@ -12,9 +12,7 @@ tests=( "show_pick_uid_tests" "out_file_logic_tests" "corruption_tests" - "corruption_tests_decode_stdout" "burst_corruption_tests" - "burst_corruption_tests_decode_stdout" "sort_tests" "sort_multi_pass" "sort_multi_pass_no_skip"