Merge branch 'nmsg-http-tests' into release-1.1.2

Makefile.am

@@ -382,6 +382,14 @@ EXTRA_DIST += tests/nmsg-dnsqr-tests/test1-dnsqr.pres
 EXTRA_DIST += tests/nmsg-dnstap-tests/test1-dnstap.json
 EXTRA_DIST += tests/nmsg-dnstap-tests/test1-dnstap.nmsg
 EXTRA_DIST += tests/nmsg-dnstap-tests/test1-dnstap.pres
+EXTRA_DIST += tests/nmsg-http-tests/test1-http-broken-request.nmsg
+EXTRA_DIST += tests/nmsg-http-tests/test1-http-empty-request.json
+EXTRA_DIST += tests/nmsg-http-tests/test1-http-empty-request.pres
+EXTRA_DIST += tests/nmsg-http-tests/test2-http-empty-request.nmsg
+EXTRA_DIST += tests/nmsg-http-tests/test2-http-empty-request.pres
+EXTRA_DIST += tests/nmsg-http-tests/test4-http-no-request.json
+EXTRA_DIST += tests/nmsg-http-tests/test4-http-no-request.nmsg
+EXTRA_DIST += tests/nmsg-http-tests/test4-http-no-request.pres
 
 noinst_PROGRAMS += libmy/crc32c_test
 libmy_crc32c_test_CFLAGS = $(AM_CFLAGS)
@@ -405,6 +413,7 @@ TESTS += tests/nmsg-dns-tests/test.sh
 TESTS += tests/nmsg-dnsqr-tests/test.sh
 TESTS += tests/nmsg-dnstap-tests/test.sh
 TESTS += tests/nmsg-dnsobs-tests/test.sh
+TESTS += tests/nmsg-http-tests/test.sh
 TESTS += tests/payload-crc32c-tests/test.sh
 TESTS += tests/string-tests/test.sh
 TESTS += tests/udp-checksum-tests/test.sh
@@ -439,6 +448,7 @@ DISTCLEANFILES += tests/nmsg-dns-tests/test*.out
 DISTCLEANFILES += tests/nmsg-dnsobs-tests/test*.out
 DISTCLEANFILES += tests/nmsg-dnsqr-tests/test*.out
 DISTCLEANFILES += tests/nmsg-dnstap-tests/test*.out
+DISTCLEANFILES += tests/nmsg-http-tests/test*.out
 
 #
 ##

configure.ac

@@ -54,6 +54,9 @@ AC_CONFIG_FILES([tests/nmsg-dnsqr-tests/test.sh],
 AC_CONFIG_FILES([tests/nmsg-dnstap-tests/test.sh],
                 [chmod +x tests/nmsg-dnstap-tests/test.sh])
 
+AC_CONFIG_FILES([tests/nmsg-http-tests/test.sh],
+                [chmod +x tests/nmsg-http-tests/test.sh])
+
 AC_CONFIG_FILES([tests/udp-checksum-tests/test.sh],
                 [chmod +x tests/udp-checksum-tests/test.sh])
 

tests/nmsg-http-tests/test.sh.in

@@ -0,0 +1,141 @@
+#!/bin/sh
+
+status=0
+
+check() {
+	if [ $? = "0" ]; then
+		echo "PASS: $*"
+	else
+		echo "FAIL: $*"
+		status=1
+	fi
+}
+
+NMSG_MSGMOD_DIR="@abs_top_builddir@/nmsg/base/.libs"
+export NMSG_MSGMOD_DIR
+NMSGTOOL="@abs_top_builddir@/src/nmsgtool"
+
+SOURCE=@abs_top_srcdir@/tests/nmsg-http-tests/
+OUTPUT=@abs_top_builddir@/tests/nmsg-http-tests/
+
+# cleanup from previous run
+rm -f ${OUTPUT}/test*-http*out
+
+# Test with no aliases
+NMSG_OPALIAS_FILE=/dev/null
+NMSG_GRALIAS_FILE=/dev/null
+export NMSG_OPALIAS_FILE
+export NMSG_GRALIAS_FILE
+
+echo Testing the non-terminated broken request string ...
+
+# one byte smaller
+$NMSGTOOL -r ${SOURCE}/test1-http-broken-request.nmsg > ${OUTPUT}/test1-http.nmsg.pres.out
+check read nmsg base:http and create base:http presentation output
+cmp -s ${SOURCE}/test1-http-empty-request.pres ${OUTPUT}/test1-http.nmsg.pres.out
+check nmsg-to-presentation
+
+# output should be same as input, pass through
+$NMSGTOOL -r ${SOURCE}/test1-http-broken-request.nmsg -w ${OUTPUT}/test1-http.nmsg.nmsg.out
+check read nmsg base:http and create base:http nmsg output
+cmp -s ${SOURCE}/test1-http-broken-request.nmsg ${OUTPUT}/test1-http.nmsg.nmsg.out
+check nmsg-to-nmsg
+
+# broken request results in empty "request":""
+$NMSGTOOL -r ${SOURCE}/test1-http-broken-request.nmsg -J ${OUTPUT}/test1-http.nmsg.json.out
+check read nmsg base:http and create base:http json output
+cmp -s ${SOURCE}/test1-http-empty-request.json ${OUTPUT}/test1-http.nmsg.json.out
+check nmsg-to-json
+
+########################
+
+echo Testing the terminated request string ...
+
+$NMSGTOOL -r ${SOURCE}/test2-http-empty-request.nmsg > ${OUTPUT}/test2-http.nmsg.pres.out
+check read nmsg base:http and create base:http presentation output
+cmp -s ${SOURCE}/test2-http-empty-request.pres ${OUTPUT}/test2-http.nmsg.pres.out
+check nmsg-to-presentation
+
+# output should be same as input, pass through
+$NMSGTOOL -r ${SOURCE}/test2-http-empty-request.nmsg -w ${OUTPUT}/test2-http.nmsg.nmsg.out
+check read nmsg base:http and create base:http nmsg output
+cmp -s ${SOURCE}/test2-http-empty-request.nmsg ${OUTPUT}/test2-http.nmsg.nmsg.out
+check nmsg-to-nmsg
+
+$NMSGTOOL -r ${SOURCE}/test2-http-empty-request.nmsg -J ${OUTPUT}/test2-http.nmsg.json.out
+check read nmsg base:http and create base:http json output
+# using same JSON to compare against
+cmp -s ${SOURCE}/test1-http-empty-request.json ${OUTPUT}/test2-http.nmsg.json.out
+check nmsg-to-json
+
+########################
+
+echo Testing the JSON input with an empty request ...
+
+# nmsgtool --readjson takes empty "" request value and turns it into a
+# single byte null string termination, so the message size is one byte
+# larger than the broken NMSG message that has no string termination.
+# This still results in empty request "".
+
+$NMSGTOOL --readjson ${SOURCE}/test1-http-empty-request.json > ${OUTPUT}/test3-http.json.pres.out
+check read json base:http and create base:http presentation output
+cmp -s ${SOURCE}/test2-http-empty-request.pres ${OUTPUT}/test3-http.json.pres.out
+check json-to-presentation
+
+$NMSGTOOL --readjson ${SOURCE}/test1-http-empty-request.json -w ${OUTPUT}/test3-http.json.nmsg.out
+check read json base:http and create base:http nmsg output
+cmp -s ${SOURCE}/test2-http-empty-request.nmsg ${OUTPUT}/test3-http.json.nmsg.out
+check json-to-nmsg
+
+# output should be same as input
+$NMSGTOOL --readjson ${SOURCE}/test1-http-empty-request.json -J ${OUTPUT}/test3-http.json.json.out
+check read json base:http and create base:http json output
+# using same JSON to compare against
+cmp -s ${SOURCE}/test1-http-empty-request.json ${OUTPUT}/test3-http.json.json.out
+check json-to-json
+
+########################
+
+# This is for NMSG message that omits the "request" field.
+
+echo Testing NMSG input that has no request ...
+
+$NMSGTOOL -r ${SOURCE}/test4-http-no-request.nmsg > ${OUTPUT}/test4-http.nmsg.pres.out
+check read nmsg base:http and create base:http presentation output
+cmp -s ${SOURCE}/test4-http-no-request.pres ${OUTPUT}/test4-http.nmsg.pres.out
+check nmsg-to-presentation
+
+# output should be same as input, pass through
+$NMSGTOOL -r ${SOURCE}/test4-http-no-request.nmsg -w ${OUTPUT}/test4-http.nmsg.nmsg.out
+check read nmsg base:http and create base:http nmsg output
+cmp -s ${SOURCE}/test4-http-no-request.nmsg ${OUTPUT}/test4-http.nmsg.nmsg.out
+check nmsg-to-nmsg
+
+$NMSGTOOL -r ${SOURCE}/test4-http-no-request.nmsg -J ${OUTPUT}/test4-http.nmsg.json.out
+check read nmsg base:http and create base:http json output
+cmp -s ${SOURCE}/test4-http-no-request.json ${OUTPUT}/test4-http.nmsg.json.out
+check nmsg-to-json
+
+#######################
+
+echo Testing JSON input that has no request ...
+
+$NMSGTOOL -j ${SOURCE}/test4-http-no-request.json > ${OUTPUT}/test4-http.json.pres.out
+check read json base:http and create base:http presentation output
+cmp -s ${SOURCE}/test4-http-no-request.pres ${OUTPUT}/test4-http.json.pres.out
+check json-to-presentation
+
+$NMSGTOOL -j ${SOURCE}/test4-http-no-request.json -w ${OUTPUT}/test4-http.json.nmsg.out
+check read json base:http and create base:http nmsg output
+cmp -s ${SOURCE}/test4-http-no-request.nmsg ${OUTPUT}/test4-http.json.nmsg.out
+check json-to-nmsg
+
+# output should be same as input, pass through
+$NMSGTOOL -j ${SOURCE}/test4-http-no-request.json -J ${OUTPUT}/test4-http.json.json.out
+check read json base:http and create base:http json output
+cmp -s ${SOURCE}/test4-http-no-request.json ${OUTPUT}/test4-http.json.json.out
+check json-to-json
+
+# NOTE: --readpres is not implemented for base:http
+
+exit $status

tests/nmsg-http-tests/test1-http-empty-request.json

@@ -0,0 +1,2 @@
+{"time":"2023-09-25 17:11:29.750339409","vname":"base","mname":"http","source":"abcdabcd","operator":1,"group":3,"message":{"type":"sinkhole","srcip":"10.11.12.13","srcport":2000,"dstip":"14.15.16.17","dstport":80,"request":"GET / HTTP/1.0\r\nUser-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; BTRS102048; .NET CLR 1.1.4322; .NET CLR 1.0.3705)\r\nHost: 12345678.example.net\r\nPragma: no-cache\r\n\r\n"}}
+{"time":"2023-09-25 17:11:29.770269405","vname":"base","mname":"http","source":"abcdabcd","operator":1,"group":3,"message":{"type":"sinkhole","srcip":"10.11.12.13","srcport":2000,"dstip":"14.15.16.17","dstport":80,"request":""}}

tests/nmsg-http-tests/test1-http-empty-request.pres

@@ -0,0 +1,23 @@
+[201] [2023-09-25 17:11:29.750339409] [1:4 base http] [abcdabcd] [1] [3] 
+type: sinkhole
+srcip: 10.11.12.13
+srcport: 2000
+dstip: 14.15.16.17
+dstport: 80
+request:
+GET / HTTP/1.0
+User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; BTRS102048; .NET CLR 1.1.4322; .NET CLR 1.0.3705)
+Host: 12345678.example.net
+Pragma: no-cache
+
+.
+
+[21] [2023-09-25 17:11:29.770269405] [1:4 base http] [abcdabcd] [1] [3] 
+type: sinkhole
+srcip: 10.11.12.13
+srcport: 2000
+dstip: 14.15.16.17
+dstport: 80
+request:
+.
+

tests/nmsg-http-tests/test2-http-empty-request.pres

@@ -0,0 +1,23 @@
+[201] [2023-09-25 17:11:29.750339409] [1:4 base http] [abcdabcd] [1] [3] 
+type: sinkhole
+srcip: 10.11.12.13
+srcport: 2000
+dstip: 14.15.16.17
+dstport: 80
+request:
+GET / HTTP/1.0
+User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; BTRS102048; .NET CLR 1.1.4322; .NET CLR 1.0.3705)
+Host: 12345678.example.net
+Pragma: no-cache
+
+.
+
+[22] [2023-09-25 17:11:29.770269405] [1:4 base http] [abcdabcd] [1] [3] 
+type: sinkhole
+srcip: 10.11.12.13
+srcport: 2000
+dstip: 14.15.16.17
+dstport: 80
+request:
+.
+

tests/nmsg-http-tests/test4-http-no-request.json

@@ -0,0 +1,3 @@
+{"time":"2019-10-11 15:09:53.193183634","vname":"base","mname":"http","source":"abbaabba","operator":102,"group":1972,"message":{"type":"sinkhole","srcip":"198.51.100.195","srcport":40188,"dstip":"192.0.2.141","dstport":80,"request":"POST /xw1.php HTTP/1.1\r\nContent-Type: application/x-www-form-urlencoded\r\nUser-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:28.0) Gecko/20100101 Firefox/28.0\r\nHost: 192.0.2.141\r\nContent-Length: 19\r\nConnection: Keep-Alive\r\nCache-Control: no-cache\r\n\r\nh=die(@md5(Apri1));"}}
+{"time":"2019-10-11 15:09:54.163338057","vname":"base","mname":"http","source":"abbaabba","operator":101,"group":200,"message":{"type":"sinkhole","srcip":"198.51.100.153","srcport":62457,"dstip":"192.0.2.140","dstport":80,"request":"lv[endof]"}}
+{"time":"2019-10-11 15:09:55.123456789","vname":"base","mname":"http","source":"abbaabba","operator":1098,"group":1099,"message":{"type":"sinkhole","srcip":"203.0.113.153","srcport":62457,"dstip":"192.0.2.140","dstport":80}}

tests/nmsg-http-tests/test4-http-no-request.pres

@@ -0,0 +1,33 @@
+[288] [2019-10-11 15:09:53.193183634] [1:4 base http] [abbaabba] [102] [1972] 
+type: sinkhole
+srcip: 198.51.100.195
+srcport: 40188
+dstip: 192.0.2.141
+dstport: 80
+request:
+POST /xw1.php HTTP/1.1
+Content-Type: application/x-www-form-urlencoded
+User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:28.0) Gecko/20100101 Firefox/28.0
+Host: 192.0.2.141
+Content-Length: 19
+Connection: Keep-Alive
+Cache-Control: no-cache
+
+h=die(@md5(Apri1));.
+
+[32] [2019-10-11 15:09:54.163338057] [1:4 base http] [abbaabba] [101] [200] 
+type: sinkhole
+srcip: 198.51.100.153
+srcport: 62457
+dstip: 192.0.2.140
+dstport: 80
+request:
+lv[endof].
+
+[20] [2019-10-11 15:09:55.123456789] [1:4 base http] [abbaabba] [1098] [1099] 
+type: sinkhole
+srcip: 203.0.113.153
+srcport: 62457
+dstip: 192.0.2.140
+dstport: 80
+