-
Notifications
You must be signed in to change notification settings - Fork 2
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
4 changed files
with
99 additions
and
62 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,119 +1,151 @@ | ||
| #!/bin/bash | ||
|
|
||
| input="$TEST_DIR/$1.c" | ||
| output_file="$1-$2-$3-$4" | ||
| baseline="$BASELINE_DIR/$1.out" | ||
| dir="$BUILD_DIR/test/$1" | ||
| bin="$dir/$output_file" | ||
| if [ "$#" -ne 4 ]; then | ||
| echo "usage: ./test.sh [test] [compiler] [o_level] [g_level]" | ||
| exit 1 | ||
| fi | ||
|
|
||
| test=$1 | ||
| compiler=$2 | ||
| o_level=$3 | ||
| g_level=$4 | ||
|
|
||
|
|
||
| if [ ! -x "$(command -v wdiff)" ]; then | ||
| echo "[ERROR] wdiff is required, but isn't installed." | ||
| exit 1 | ||
| fi | ||
|
|
||
| if [ -x "$compiler" ]; then | ||
| echo "[ERROR] $compiler doesn't exist or isn't executable," | ||
| exit 1 | ||
| fi | ||
|
|
||
|
|
||
| test_id="$test-$compiler-$o_level-$g_level" | ||
| dir="$BUILD_DIR/test/$test" | ||
| src="$TEST_DIR/$test.c" | ||
| bin="$dir/$compiler-$o_level-$g_level" | ||
| log="$bin.log" | ||
| output="$bin.out" | ||
| baseline="$BASELINE_DIR/$test.out" | ||
|
|
||
|
|
||
| # Some tests don't work on older versions of compilers | ||
| function should_skip { | ||
| result=false | ||
|
|
||
| if [ $(echo "$compiler" | head -c 5) = "clang" ]; then | ||
| major_version=$("$compiler" -dumpversion | cut -d. -f1) | ||
|
|
||
| if [ $major_version -le 15 ]; then | ||
| # uses old (non-v5) format for bit fields | ||
| if [ "$test" = "bits" ]; then result=true; fi | ||
| fi | ||
| fi | ||
|
|
||
| echo $result | ||
| } | ||
|
|
||
| # Regular tests share single uprintf implementation, but option tests need their own. | ||
| function uses_shared_implementation { | ||
| if [ "$test" = "depth_option" ]; then echo false; | ||
| elif [ "$test" = "indentation_option" ]; then echo false; | ||
| elif [ "$test" = "stdio_file" ]; then echo false; | ||
| elif [ "$test" = "string_truncation" ]; then echo false; | ||
| else echo true; fi | ||
| } | ||
|
|
||
| # Some tests may not match baseline exactly, so they get custom target similarity percentage. | ||
| function get_similarity { | ||
| # These tests contain stdint.h types which have different typenames in gcc and clang | ||
| if [ "$1" = "struct" ]; then echo 90; | ||
| elif [ "$1" = "packed" ]; then echo 90; | ||
| if [ "$test" = "struct" ]; then echo 90; | ||
| elif [ "$test" = "packed" ]; then echo 90; | ||
| # FILE has different implementation which is up to stdio.h and it often has pointers | ||
| # that point out-of-bounds causing uprintf to print garbage from the memory, thus the | ||
| # primary goal of the test is to check that there are no errors, i.e. segfaults, leaks. | ||
| elif [ "$1" = "stdio_file" ]; then echo 1; | ||
| elif [ "$test" = "stdio_file" ]; then echo 1; | ||
| # Clang doesn't produce DW_AT_subprogram for external functions, i.e. shared libraries | ||
| # or different CUs, so cross-CU retrieval of function signature/name doesn't work. | ||
| elif [ "$1" = "function" ]; then echo 80; | ||
| elif [ "$test" = "function" ]; then echo 80; | ||
| else echo 100; fi | ||
| } | ||
|
|
||
| # Regular tests share single uprintf implementation, but option tests need their own. | ||
| function uses_shared_implementation { | ||
| if [ "$1" = "depth_option" ]; then echo false; | ||
| elif [ "$1" = "indentation_option" ]; then echo false; | ||
| elif [ "$1" = "stdio_file" ]; then echo false; | ||
| elif [ "$1" = "string_truncation" ]; then echo false; | ||
| else echo true; fi | ||
| } | ||
|
|
||
| # Some tests don't work on older versions | ||
| function should_skip { | ||
| if [ $(echo "$2" | head -c 3) = "gcc" ]; then | ||
| : | ||
| else # clang | ||
| major_version=$("$2" -dumpversion | cut -d. -f1) | ||
| if [ $major_version -le 15 ]; then | ||
| # uses old (non-v5) format for bit fields | ||
| if [ "$1" = "bits" ]; then echo true; fi | ||
| fi | ||
| fi | ||
| } | ||
|
|
||
| if [ $(should_skip $1 $2) ]; then | ||
| echo "[SKIPPING] $output_file" | ||
| if [ $(should_skip) = true ]; then | ||
| echo "[SKIPPING] $test_id" | ||
| exit 0 | ||
| fi | ||
|
|
||
| # Compiling | ||
| mkdir -p $dir | ||
| if [ $(uses_shared_implementation $1) = false ]; then | ||
| $2 $CFLAGS -Werror -$3 -$4 -o $bin $input > $log 2>&1 | ||
| ret=$? | ||
| else | ||
| if [ $(uses_shared_implementation) = true ]; then | ||
| object="$bin.o" | ||
| implementation="$BUILD_DIR/impl/$2.o" | ||
| implementation="$BUILD_DIR/impl/$compiler.o" | ||
|
|
||
| if [ ! -r $implementation ]; then | ||
| echo "[COMPILATION FAILED] $test_id. \"$implementation\" doesn't exist."; | ||
| exit 1 | ||
| fi | ||
|
|
||
| $2 $CFLAGS -Werror -$3 -$4 -c $input -o $object > $log 2>&1 | ||
| $compiler $CFLAGS -Werror -$o_level -$g_level -c $src -o $object > $log 2>&1 | ||
| ret=$? | ||
| $compiler $CFLAGS -Werror -$o_level -$g_level -o $bin $object $implementation >> $log 2>&1 | ||
| else | ||
| $compiler $CFLAGS -Werror -$o_level -$g_level -o $bin $src > $log 2>&1 | ||
| ret=$? | ||
| $2 $CFLAGS -Werror -$3 -$4 -o $bin $object $implementation >> $log 2>&1 | ||
| fi | ||
|
|
||
| if [ $ret -ne 0 ]; then | ||
| if [ $CI ]; then | ||
| echo "[COMPILATION FAILED] $log:" | ||
| echo "[COMPILATION FAILED] $test_id. Log:" | ||
| cat $log | ||
| else | ||
| echo "[COMPILATION FAILED] Log: $log. Rerun test: make $bin" | ||
| echo "[COMPILATION FAILED] $test_id. Log: $log" | ||
| fi | ||
| exit 1 | ||
| fi | ||
|
|
||
| # Running | ||
| ./$bin > $output 2>&1 | ||
| ret=$? | ||
| cat $output >> $log | ||
|
|
||
| if [ $ret -ne 0 ]; then | ||
| if [ $CI ]; then | ||
| echo "[TEST FAILED] $log:" | ||
| echo "[TEST FAILED] $test_id. Log:" | ||
| cat $log | ||
| else | ||
| echo "[TEST FAILED] Log: $log. Failed test binary: $bin. Rerun test: make $bin" | ||
| echo "[TEST FAILED] $test_id. Log: $log. Binary: $bin." | ||
| fi | ||
| exit 1 | ||
| fi | ||
|
|
||
| # Comparing | ||
| if [ ! -f $baseline ]; then | ||
| echo "[WARNING] There is no baseline for $1!" | ||
| echo "[TEST PASSED] $output_file: ?"; | ||
| if [ ! -r $baseline ]; then | ||
| echo "[WARNING] There is no $test baseline." | ||
| echo "[TEST PASSED] $test_id: ?"; | ||
| exit 0 | ||
| fi | ||
|
|
||
| # Replace pointers with "POINTER" since they differ between every execution. | ||
| sed -E "s/0x[0-9a-fA-F]{6,16}/POINTER/g" -i $baseline -i $output | ||
| diff=$(wdiff -123 --statistics $baseline $output) | ||
|
|
||
| diff=$(wdiff -123 --statistics $baseline $output) | ||
| similarity=$(echo "$diff" | awk -F ':' '{print $NF}' | awk -F ' ' '{print $4}' | sed 's/%$//' | sort -g | head -n 1) | ||
|
|
||
| echo "Similarity is $similarity%" >> $log | ||
| if [ $similarity -lt $(get_similarity $1) ]; then | ||
| if [ $similarity -lt $(get_similarity) ]; then | ||
| diff="$bin.diff" | ||
| wdiff $baseline $output > $diff | ||
|
|
||
| if [ $CI ]; then | ||
| echo "[DIFF FAILED] Similarity is $similarity%" | ||
| echo "$log:" | ||
| echo "[DIFF FAILED] $test_id. Similarity is $similarity%, required $(get_similarity)%. Log:" | ||
| cat $log | ||
| echo "$diff:" | ||
| echo "Diff:" | ||
| cat $diff | ||
| else | ||
| echo "[DIFF FAILED] Similarity is $similarity%. Diff: $diff. Log: $log. Rerun test: make $bin" | ||
| echo "[DIFF FAILED] $test_id. Similarity is $similarity%, required $(get_similarity)%. Log: $log. Diff: $diff." | ||
| fi | ||
| exit 1 | ||
| fi | ||
|
|
||
| echo "[TEST PASSED] $output_file: $similarity%"; | ||
| echo "[TEST PASSED] $test_id: $similarity%"; |