From 56d8def8cd59748179040acb117088e2fbd534b2 Mon Sep 17 00:00:00 2001 From: spevnev Date: Wed, 13 Nov 2024 21:13:55 -0500 Subject: [PATCH] Update test.sh --- Makefile | 4 +- README.md | 11 +++-- TODO.md | 4 ++ test.sh | 142 +++++++++++++++++++++++++++++++++--------------------- 4 files changed, 99 insertions(+), 62 deletions(-) diff --git a/Makefile b/Makefile index 70189f4..2911b80 100644 --- a/Makefile +++ b/Makefile @@ -71,13 +71,13 @@ $(BUILD_DIR)/$(EXAMPLE_DIR)/uprintf: $(EXAMPLE_DIR)/uprintf.c uprintf.h Makefile test: tests .PHONY: tests -tests: $(foreach C,$(COMPILERS),$(foreach O,$(O_LEVELS),$(foreach G,$(G_LEVELS),$(foreach T,$(TESTS),$(BUILD_DIR)/test/$T/$T-$C-$O-$G)))) +tests: $(foreach C,$(COMPILERS),$(foreach O,$(O_LEVELS),$(foreach G,$(G_LEVELS),$(foreach T,$(TESTS),$(BUILD_DIR)/test/$T/$C-$O-$G)))) # Export all variables to subprocesses, i.e. test.sh export define TEST_TEMPLATE -$(BUILD_DIR)/test/$1/$1-$2-$3-$4: $(BUILD_DIR)/impl/$2.o $(TEST_DIR)/$1.c uprintf.h Makefile test.sh +$(BUILD_DIR)/test/$1/$2-$3-$4: $(BUILD_DIR)/impl/$2.o $(TEST_DIR)/$1.c uprintf.h Makefile test.sh @./test.sh $1 $2 $3 $4 endef diff --git a/README.md b/README.md index f171009..1fa3544 100644 --- a/README.md +++ b/README.md @@ -1059,13 +1059,14 @@ This way you should be able to include it without copying the header to every pr - Debug information included, `-g2` or higher - Have `elf.h` in include path -### Tested on: +### Tested on (using CI/CD): -Architectures: `x86_64/amd64` +Architecture: \ +`x86_64/amd64` -Compilers (tested in CI/CD): \ -`gcc-14`, `gcc-12`, `gcc-11` \ -`clang-18`, `clang-16`, `clang-14` +Compilers: \ +`gcc`: 14, 13, 12, 11 \ +`clang`: 18, 17, 16, 15, 14 ### Limitations diff --git a/TODO.md b/TODO.md index a5e5272..4f429fd 100644 --- a/TODO.md +++ b/TODO.md @@ -97,3 +97,7 @@ _Decimal32 a; _Decimal64 b; _Decimal128 c; ``` + +### Use hash table +There are functions which linearly search for a string, e.g. `_upf_get_variable_type`, `_upf_get_function_return_type`. +Hash table should improve performance. diff --git a/test.sh b/test.sh index 2eacb67..cacdbf8 100755 --- a/test.sh +++ b/test.sh @@ -1,119 +1,151 @@ #!/bin/bash -input="$TEST_DIR/$1.c" -output_file="$1-$2-$3-$4" -baseline="$BASELINE_DIR/$1.out" -dir="$BUILD_DIR/test/$1" -bin="$dir/$output_file" +if [ "$#" -ne 4 ]; then + echo "usage: ./test.sh [test] [compiler] [o_level] [g_level]" + exit 1 +fi + +test=$1 +compiler=$2 +o_level=$3 +g_level=$4 + + +if [ ! -x "$(command -v wdiff)" ]; then + echo "[ERROR] wdiff is required, but isn't installed." + exit 1 +fi + +if [ -x "$compiler" ]; then + echo "[ERROR] $compiler doesn't exist or isn't executable," + exit 1 +fi + + +test_id="$test-$compiler-$o_level-$g_level" +dir="$BUILD_DIR/test/$test" +src="$TEST_DIR/$test.c" +bin="$dir/$compiler-$o_level-$g_level" log="$bin.log" output="$bin.out" +baseline="$BASELINE_DIR/$test.out" + + +# Some tests don't work on older versions of compilers +function should_skip { + result=false + + if [ $(echo "$compiler" | head -c 5) = "clang" ]; then + major_version=$("$compiler" -dumpversion | cut -d. -f1) + + if [ $major_version -le 15 ]; then + # uses old (non-v5) format for bit fields + if [ "$test" = "bits" ]; then result=true; fi + fi + fi + + echo $result +} + +# Regular tests share single uprintf implementation, but option tests need their own. +function uses_shared_implementation { + if [ "$test" = "depth_option" ]; then echo false; + elif [ "$test" = "indentation_option" ]; then echo false; + elif [ "$test" = "stdio_file" ]; then echo false; + elif [ "$test" = "string_truncation" ]; then echo false; + else echo true; fi +} # Some tests may not match baseline exactly, so they get custom target similarity percentage. function get_similarity { # These tests contain stdint.h types which have different typenames in gcc and clang - if [ "$1" = "struct" ]; then echo 90; - elif [ "$1" = "packed" ]; then echo 90; + if [ "$test" = "struct" ]; then echo 90; + elif [ "$test" = "packed" ]; then echo 90; # FILE has different implementation which is up to stdio.h and it often has pointers # that point out-of-bounds causing uprintf to print garbage from the memory, thus the # primary goal of the test is to check that there are no errors, i.e. segfaults, leaks. - elif [ "$1" = "stdio_file" ]; then echo 1; + elif [ "$test" = "stdio_file" ]; then echo 1; # Clang doesn't produce DW_AT_subprogram for external functions, i.e. shared libraries # or different CUs, so cross-CU retrieval of function signature/name doesn't work. - elif [ "$1" = "function" ]; then echo 80; + elif [ "$test" = "function" ]; then echo 80; else echo 100; fi } -# Regular tests share single uprintf implementation, but option tests need their own. -function uses_shared_implementation { - if [ "$1" = "depth_option" ]; then echo false; - elif [ "$1" = "indentation_option" ]; then echo false; - elif [ "$1" = "stdio_file" ]; then echo false; - elif [ "$1" = "string_truncation" ]; then echo false; - else echo true; fi -} - -# Some tests don't work on older versions -function should_skip { - if [ $(echo "$2" | head -c 3) = "gcc" ]; then - : - else # clang - major_version=$("$2" -dumpversion | cut -d. -f1) - if [ $major_version -le 15 ]; then - # uses old (non-v5) format for bit fields - if [ "$1" = "bits" ]; then echo true; fi - fi - fi -} -if [ $(should_skip $1 $2) ]; then - echo "[SKIPPING] $output_file" +if [ $(should_skip) = true ]; then + echo "[SKIPPING] $test_id" exit 0 fi # Compiling mkdir -p $dir -if [ $(uses_shared_implementation $1) = false ]; then - $2 $CFLAGS -Werror -$3 -$4 -o $bin $input > $log 2>&1 - ret=$? -else +if [ $(uses_shared_implementation) = true ]; then object="$bin.o" - implementation="$BUILD_DIR/impl/$2.o" + implementation="$BUILD_DIR/impl/$compiler.o" + + if [ ! -r $implementation ]; then + echo "[COMPILATION FAILED] $test_id. \"$implementation\" doesn't exist."; + exit 1 + fi - $2 $CFLAGS -Werror -$3 -$4 -c $input -o $object > $log 2>&1 + $compiler $CFLAGS -Werror -$o_level -$g_level -c $src -o $object > $log 2>&1 + ret=$? + $compiler $CFLAGS -Werror -$o_level -$g_level -o $bin $object $implementation >> $log 2>&1 +else + $compiler $CFLAGS -Werror -$o_level -$g_level -o $bin $src > $log 2>&1 ret=$? - $2 $CFLAGS -Werror -$3 -$4 -o $bin $object $implementation >> $log 2>&1 fi if [ $ret -ne 0 ]; then if [ $CI ]; then - echo "[COMPILATION FAILED] $log:" + echo "[COMPILATION FAILED] $test_id. Log:" cat $log else - echo "[COMPILATION FAILED] Log: $log. Rerun test: make $bin" + echo "[COMPILATION FAILED] $test_id. Log: $log" fi exit 1 fi -# Running ./$bin > $output 2>&1 ret=$? cat $output >> $log + if [ $ret -ne 0 ]; then if [ $CI ]; then - echo "[TEST FAILED] $log:" + echo "[TEST FAILED] $test_id. Log:" cat $log else - echo "[TEST FAILED] Log: $log. Failed test binary: $bin. Rerun test: make $bin" + echo "[TEST FAILED] $test_id. Log: $log. Binary: $bin." fi exit 1 fi -# Comparing -if [ ! -f $baseline ]; then - echo "[WARNING] There is no baseline for $1!" - echo "[TEST PASSED] $output_file: ?"; +if [ ! -r $baseline ]; then + echo "[WARNING] There is no $test baseline." + echo "[TEST PASSED] $test_id: ?"; exit 0 fi +# Replace pointers with "POINTER" since they differ between every execution. sed -E "s/0x[0-9a-fA-F]{6,16}/POINTER/g" -i $baseline -i $output -diff=$(wdiff -123 --statistics $baseline $output) +diff=$(wdiff -123 --statistics $baseline $output) similarity=$(echo "$diff" | awk -F ':' '{print $NF}' | awk -F ' ' '{print $4}' | sed 's/%$//' | sort -g | head -n 1) + echo "Similarity is $similarity%" >> $log -if [ $similarity -lt $(get_similarity $1) ]; then +if [ $similarity -lt $(get_similarity) ]; then diff="$bin.diff" wdiff $baseline $output > $diff if [ $CI ]; then - echo "[DIFF FAILED] Similarity is $similarity%" - echo "$log:" + echo "[DIFF FAILED] $test_id. Similarity is $similarity%, required $(get_similarity)%. Log:" cat $log - echo "$diff:" + echo "Diff:" cat $diff else - echo "[DIFF FAILED] Similarity is $similarity%. Diff: $diff. Log: $log. Rerun test: make $bin" + echo "[DIFF FAILED] $test_id. Similarity is $similarity%, required $(get_similarity)%. Log: $log. Diff: $diff." fi exit 1 fi -echo "[TEST PASSED] $output_file: $similarity%"; +echo "[TEST PASSED] $test_id: $similarity%";