Fuzzer and CI enhancements, format bug fixes

.ci/install-dependencies.sh

@@ -7,7 +7,7 @@
 
 j=-j`nproc` || j=
 type sudo >/dev/null 2>&1 && sudo=sudo || sudo=
-common_packages='make libssl-dev'
+common_packages='make libssl-dev libpcap-dev'
 
 retry_if_failed()
 {

.ci/run-build-and-tests.sh

@@ -29,10 +29,17 @@ echo 'END OF BUILD ENVIRONMENT INFORMATION'
 nproc="$(nproc)" || nproc=1
 j="-j$nproc"
 
+if [ $nproc -gt 2 ]; then
+	export OMP_NUM_THREADS=2
+fi
+
 cd src
-time ./configure
+time ./configure $*
 time make $j
 time make $j check
+if [ "$1" = "--enable-fuzz" ]; then
+	time ../run/john --fuzz=500
+fi
 
 if git status --porcelain |grep ^.; then
 	echo >&2 'git status reported uncleanness'

.github/workflows/ci.yml

@@ -12,15 +12,121 @@ jobs:
     - name: check
       run: git diff-index --check --cached b1b622f691d40196815939e4736a5da71befd206
 
+  asan:
+    runs-on: ubuntu-latest
+    env:
+      CC: gcc-13
+      TARGET: x86_64
+    steps:
+    - uses: actions/checkout@v3
+    - name: install dependencies
+      run: .ci/install-dependencies.sh
+    - name: build check
+      run: .ci/run-build-and-tests.sh --enable-fuzz --enable-asan
+
+  asan-disable-all:
+    runs-on: ubuntu-latest
+    env:
+      CC: gcc-13
+      TARGET: x86_64
+    steps:
+    - uses: actions/checkout@v3
+    - name: install dependencies
+      run: .ci/install-dependencies.sh
+    - name: build check
+      run: .ci/run-build-and-tests.sh --enable-asan --disable-openmp --disable-simd --without-openssl
+
+  asan-disable-openmp:
+    runs-on: ubuntu-latest
+    env:
+      CC: gcc-13
+      TARGET: x86_64
+    steps:
+    - uses: actions/checkout@v3
+    - name: install dependencies
+      run: .ci/install-dependencies.sh
+    - name: build check
+      run: .ci/run-build-and-tests.sh --enable-asan --disable-openmp
+
+  asan-without-openssl:
+    runs-on: ubuntu-latest
+    env:
+      CC: gcc-13
+      TARGET: x86_64
+    steps:
+    - uses: actions/checkout@v3
+    - name: install dependencies
+      run: .ci/install-dependencies.sh
+    - name: build check
+      run: .ci/run-build-and-tests.sh --enable-asan --without-openssl
+
+  asan-disable-simd:
+    runs-on: ubuntu-latest
+    env:
+      CC: gcc-13
+      TARGET: x86_64
+    steps:
+    - uses: actions/checkout@v3
+    - name: install dependencies
+      run: .ci/install-dependencies.sh
+    - name: build check
+      run: .ci/run-build-and-tests.sh --enable-asan --disable-simd
+
+  asan-sse2:
+    runs-on: ubuntu-latest
+    env:
+      CC: gcc-13
+      TARGET: x86_64
+    steps:
+    - uses: actions/checkout@v3
+    - name: install dependencies
+      run: .ci/install-dependencies.sh
+    - name: build check
+      run: .ci/run-build-and-tests.sh --enable-asan --enable-simd=sse2
+
+  asan-avx:
+    runs-on: ubuntu-latest
+    env:
+      CC: gcc-13
+      TARGET: x86_64
+    steps:
+    - uses: actions/checkout@v3
+    - name: install dependencies
+      run: .ci/install-dependencies.sh
+    - name: build check
+      run: .ci/run-build-and-tests.sh --enable-asan --enable-simd=avx
+
+  gcc13-x86-without-openssl:
+    runs-on: ubuntu-latest
+    env:
+      CC: gcc-13
+      TARGET: x86
+    steps:
+    - uses: actions/checkout@v3
+    - name: install dependencies
+      run: .ci/install-dependencies.sh
+    - name: build check
+      run: .ci/run-build-and-tests.sh --without-openssl
+
+  gcc13-x86-disable-all:
+    runs-on: ubuntu-latest
+    env:
+      CC: gcc-13
+      TARGET: x86
+    steps:
+    - uses: actions/checkout@v3
+    - name: install dependencies
+      run: .ci/install-dependencies.sh
+    - name: build check
+      run: .ci/run-build-and-tests.sh --disable-openmp --disable-simd --without-openssl
+
   gcc13-x86_64:
     runs-on: ubuntu-latest
     env:
       CC: gcc-13
       TARGET: x86_64
     steps:
     - uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
     - name: install dependencies
       run: .ci/install-dependencies.sh
     - name: build check
@@ -33,8 +139,6 @@ jobs:
       TARGET: x86_64
     steps:
     - uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
     - name: install dependencies
       run: .ci/install-dependencies.sh
     - name: build check
@@ -47,8 +151,6 @@ jobs:
       TARGET: x86_64
     steps:
     - uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
     - name: install dependencies
       run: .ci/install-dependencies.sh
     - name: build check
@@ -61,8 +163,6 @@ jobs:
       TARGET: x86_64
     steps:
     - uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
     - name: install dependencies
       run: .ci/install-dependencies.sh
     - name: build check
@@ -75,8 +175,6 @@ jobs:
       TARGET: x86_64
     steps:
     - uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
     - name: install dependencies
       run: .ci/install-dependencies.sh
     - name: build check
@@ -89,8 +187,6 @@ jobs:
       TARGET: x86_64
     steps:
     - uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
     - name: install dependencies
       run: .ci/install-dependencies.sh
     - name: build check
@@ -103,8 +199,6 @@ jobs:
       TARGET: x86_64
     steps:
     - uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
     - name: install dependencies
       run: .ci/install-dependencies.sh
     - name: build check
@@ -117,8 +211,6 @@ jobs:
       TARGET: x86_64
     steps:
     - uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
     - name: install dependencies
       run: .ci/install-dependencies.sh
     - name: build check
@@ -131,8 +223,6 @@ jobs:
       TARGET: x86_64
     steps:
     - uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
     - name: install dependencies
       run: .ci/install-dependencies.sh
     - name: build check
@@ -145,8 +235,6 @@ jobs:
       TARGET: x86_64
     steps:
     - uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
     - name: install dependencies
       run: .ci/install-dependencies.sh
     - name: build check
@@ -159,8 +247,6 @@ jobs:
       TARGET: x86_64
     steps:
     - uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
     - name: install dependencies
       run: .ci/install-dependencies.sh
     - name: build check
@@ -173,8 +259,6 @@ jobs:
       TARGET: x86_64
     steps:
     - uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
     - name: install dependencies
       run: .ci/install-dependencies.sh
     - name: build check
@@ -187,8 +271,6 @@ jobs:
       TARGET: x86_64
     steps:
     - uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
     - name: install dependencies
       run: .ci/install-dependencies.sh
     - name: build check
@@ -201,8 +283,6 @@ jobs:
       TARGET: x86_64
     steps:
     - uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
     - name: install dependencies
       run: .ci/install-dependencies.sh
     - name: build check

src/blockchain_common_plug.c

@@ -118,9 +118,9 @@ int blockchain_decrypt(unsigned char *derived_key, unsigned char *data)
 
 	// "guid" will be found in the first block
 	if (memmem(out, 16, "\"guid\"", 6)) {
-		AES_cbc_encrypt(data + 32, out + 16, SAFETY_FACTOR - 16, &akey, iv,
+		AES_cbc_encrypt(data + 32, out + 16, SAFETY_FACTOR - 32, &akey, iv,
 		                AES_DECRYPT);
-		if (memmem(out, SAFETY_FACTOR, "\"sharedKey\"", 11))
+		if (memmem(out, SAFETY_FACTOR - 16, "\"sharedKey\"", 11))
 			// Do not check for "options" string. It is too further
 			// down in the byte stream for v3 wallets.  Note, we
 			// 'could' check that the guid and sharedKey values are

src/fuzz.c

@@ -30,6 +30,7 @@
 
 #include "jumbo.h"
 #include "misc.h"	// error()
+#include "common.h"
 #include "config.h"
 #include "john.h"
 #include "options.h"
@@ -62,6 +63,7 @@ struct FuzzDic {
 	char *value;
 };
 
+static int fuzz_limit;
 static struct FuzzDic *rfd;
 
 static FILE *s_file; // Status file which is ./fuzz_status/'format->params.label'
@@ -102,9 +104,6 @@ static void fuzz_init_dictionary()
 	char *file_start;
 #endif
 
-	if (!options.fuzz_dic)
-		return;
-
 	if (!(file = jtr_fopen(options.fuzz_dic, "r")))
 		pexit("fopen: %s", options.fuzz_dic);
 
@@ -175,6 +174,9 @@ static char * replace_each_chars(char *ciphertext, int *is_replace_finish)
 	static int cipher_index = 0;
 	static char replaced_chars[5] = "\xFF" "9$*#";
 
+	if (cipher_index >= strlen(ciphertext))
+		cipher_index = 0;
+
 	while (replaced_chars_index < sizeof(replaced_chars)) {
 		if (ciphertext[cipher_index] != replaced_chars[replaced_chars_index]) {
 			fuzz_hash[cipher_index] = replaced_chars[replaced_chars_index];
@@ -268,6 +270,9 @@ static char * change_case(char *origin_ctext, int *is_chgcase_finish)
 	static int flag = 2;
 	static int cipher_index = 0;
 
+	if (cipher_index >= strlen(origin_ctext))
+		cipher_index = 0;
+
 	while (origin_ctext[cipher_index]) {
 		c = origin_ctext[cipher_index];
 		if ('a' <= c && 'z' >= c) {
@@ -335,7 +340,7 @@ static char * insert_dic(char *origin_ctext, int *is_insertdic_finish)
 	static int index = 0;
 	static int flag_long = 0;
 
-	if (!options.fuzz_dic)
+	if (!rfd)
 		return NULL;
 
 	if (!flag) {
@@ -395,6 +400,8 @@ static void insert_char(char *origin_ctext, int pos, char c, int size, char *out
 {
 	const int origin_ctext_len = strlen(origin_ctext);
 
+	if (pos > origin_ctext_len)
+		pos = origin_ctext_len;
 	if (size + origin_ctext_len >= FUZZ_LINE_BUFFER_SIZE)
 		size = FUZZ_LINE_BUFFER_SIZE- origin_ctext_len - 1;
 
@@ -412,6 +419,9 @@ static char * insert_chars(char *origin_ctext, int *is_insertchars_finish)
 	static int times[5] = { 1, 10, 100, 1000, 10000 };
 	static int times_index = 0;
 
+	if (oc_index > strlen(origin_ctext))
+		oc_index = 0;
+
 //printf("%s:%d %s(oc='%s', times_index=%d, c_index=%d, oc_index=%d)\n",
 //	__FILE__, __LINE__, __FUNCTION__, origin_ctext,
 //	times_index, c_index, oc_index);
@@ -480,6 +490,9 @@ static char * update_length_data(char *origin_ctext, int *is_updatelengthdata_fi
 	unsigned long long len = 0, as_decimal = 0, as_hex = 0;
 	int inc, hex_mode, digit, pos2;
 
+	if (pos > strlen(origin_ctext))
+		pos = 0;
+
 	if (mode == 0) {
 		for (; origin_ctext[pos] && !mode; ++pos) {
 			if (!is_alnum_ascii(origin_ctext[pos])) {
@@ -547,6 +560,9 @@ static char * insert_zeros(char *origin_ctext, int *is_insertzeros_finish)
 	static int pos = 0;
 	int c, c1;
 
+	if (pos > strlen(origin_ctext))
+		pos = 0;
+
 	if (times == 0 && pos == 0) {
 		c = origin_ctext[0];
 		if (is_alnum_ascii(c))
@@ -714,7 +730,7 @@ static void fuzz_test(struct db_main *db, struct fmt_main *format)
 	ldr_init_database(db, &options.loader);
 	db->format = format;
 
-	while (!event_abort) {
+	while (!event_abort && index < fuzz_limit) {
 		ret = get_next_fuzz_case(format->params.label, current->ciphertext);
 		save_index(index++);
 		line = fuzz_hash;
@@ -812,7 +828,13 @@ int fuzz(struct db_main *db)
 		}
 	}
 
-	fuzz_init_dictionary();
+	fuzz_limit = 0x7fffffff;
+	if (options.fuzz_dic) {
+		if (isdec(options.fuzz_dic))
+			fuzz_limit = atoi(options.fuzz_dic);
+		else
+			fuzz_init_dictionary();
+	}
 
 	total = 0;
 	if ((format = fmt_list))