diff --git a/Makefile.am b/Makefile.am index ad7798aa..9806f8ff 100644 --- a/Makefile.am +++ b/Makefile.am @@ -382,6 +382,14 @@ EXTRA_DIST += tests/nmsg-dnsqr-tests/test1-dnsqr.pres EXTRA_DIST += tests/nmsg-dnstap-tests/test1-dnstap.json EXTRA_DIST += tests/nmsg-dnstap-tests/test1-dnstap.nmsg EXTRA_DIST += tests/nmsg-dnstap-tests/test1-dnstap.pres +EXTRA_DIST += tests/nmsg-http-tests/test1-http-broken-request.nmsg +EXTRA_DIST += tests/nmsg-http-tests/test1-http-empty-request.json +EXTRA_DIST += tests/nmsg-http-tests/test1-http-empty-request.pres +EXTRA_DIST += tests/nmsg-http-tests/test2-http-empty-request.nmsg +EXTRA_DIST += tests/nmsg-http-tests/test2-http-empty-request.pres +EXTRA_DIST += tests/nmsg-http-tests/test4-http-no-request.json +EXTRA_DIST += tests/nmsg-http-tests/test4-http-no-request.nmsg +EXTRA_DIST += tests/nmsg-http-tests/test4-http-no-request.pres noinst_PROGRAMS += libmy/crc32c_test libmy_crc32c_test_CFLAGS = $(AM_CFLAGS) @@ -405,6 +413,7 @@ TESTS += tests/nmsg-dns-tests/test.sh TESTS += tests/nmsg-dnsqr-tests/test.sh TESTS += tests/nmsg-dnstap-tests/test.sh TESTS += tests/nmsg-dnsobs-tests/test.sh +TESTS += tests/nmsg-http-tests/test.sh TESTS += tests/payload-crc32c-tests/test.sh TESTS += tests/string-tests/test.sh TESTS += tests/udp-checksum-tests/test.sh @@ -439,6 +448,7 @@ DISTCLEANFILES += tests/nmsg-dns-tests/test*.out DISTCLEANFILES += tests/nmsg-dnsobs-tests/test*.out DISTCLEANFILES += tests/nmsg-dnsqr-tests/test*.out DISTCLEANFILES += tests/nmsg-dnstap-tests/test*.out +DISTCLEANFILES += tests/nmsg-http-tests/test*.out # ## diff --git a/configure.ac b/configure.ac index 3df29439..9d3ae14b 100644 --- a/configure.ac +++ b/configure.ac @@ -54,6 +54,9 @@ AC_CONFIG_FILES([tests/nmsg-dnsqr-tests/test.sh], AC_CONFIG_FILES([tests/nmsg-dnstap-tests/test.sh], [chmod +x tests/nmsg-dnstap-tests/test.sh]) +AC_CONFIG_FILES([tests/nmsg-http-tests/test.sh], + [chmod +x tests/nmsg-http-tests/test.sh]) + AC_CONFIG_FILES([tests/udp-checksum-tests/test.sh], [chmod +x tests/udp-checksum-tests/test.sh]) diff --git a/tests/nmsg-http-tests/test.sh.in b/tests/nmsg-http-tests/test.sh.in new file mode 100755 index 00000000..812dba7e --- /dev/null +++ b/tests/nmsg-http-tests/test.sh.in @@ -0,0 +1,141 @@ +#!/bin/sh + +status=0 + +check() { + if [ $? = "0" ]; then + echo "PASS: $*" + else + echo "FAIL: $*" + status=1 + fi +} + +NMSG_MSGMOD_DIR="@abs_top_builddir@/nmsg/base/.libs" +export NMSG_MSGMOD_DIR +NMSGTOOL="@abs_top_builddir@/src/nmsgtool" + +SOURCE=@abs_top_srcdir@/tests/nmsg-http-tests/ +OUTPUT=@abs_top_builddir@/tests/nmsg-http-tests/ + +# cleanup from previous run +rm -f ${OUTPUT}/test*-http*out + +# Test with no aliases +NMSG_OPALIAS_FILE=/dev/null +NMSG_GRALIAS_FILE=/dev/null +export NMSG_OPALIAS_FILE +export NMSG_GRALIAS_FILE + +echo Testing the non-terminated broken request string ... + +# one byte smaller +$NMSGTOOL -r ${SOURCE}/test1-http-broken-request.nmsg > ${OUTPUT}/test1-http.nmsg.pres.out +check read nmsg base:http and create base:http presentation output +cmp -s ${SOURCE}/test1-http-empty-request.pres ${OUTPUT}/test1-http.nmsg.pres.out +check nmsg-to-presentation + +# output should be same as input, pass through +$NMSGTOOL -r ${SOURCE}/test1-http-broken-request.nmsg -w ${OUTPUT}/test1-http.nmsg.nmsg.out +check read nmsg base:http and create base:http nmsg output +cmp -s ${SOURCE}/test1-http-broken-request.nmsg ${OUTPUT}/test1-http.nmsg.nmsg.out +check nmsg-to-nmsg + +# broken request results in empty "request":"" +$NMSGTOOL -r ${SOURCE}/test1-http-broken-request.nmsg -J ${OUTPUT}/test1-http.nmsg.json.out +check read nmsg base:http and create base:http json output +cmp -s ${SOURCE}/test1-http-empty-request.json ${OUTPUT}/test1-http.nmsg.json.out +check nmsg-to-json + +######################## + +echo Testing the terminated request string ... + +$NMSGTOOL -r ${SOURCE}/test2-http-empty-request.nmsg > ${OUTPUT}/test2-http.nmsg.pres.out +check read nmsg base:http and create base:http presentation output +cmp -s ${SOURCE}/test2-http-empty-request.pres ${OUTPUT}/test2-http.nmsg.pres.out +check nmsg-to-presentation + +# output should be same as input, pass through +$NMSGTOOL -r ${SOURCE}/test2-http-empty-request.nmsg -w ${OUTPUT}/test2-http.nmsg.nmsg.out +check read nmsg base:http and create base:http nmsg output +cmp -s ${SOURCE}/test2-http-empty-request.nmsg ${OUTPUT}/test2-http.nmsg.nmsg.out +check nmsg-to-nmsg + +$NMSGTOOL -r ${SOURCE}/test2-http-empty-request.nmsg -J ${OUTPUT}/test2-http.nmsg.json.out +check read nmsg base:http and create base:http json output +# using same JSON to compare against +cmp -s ${SOURCE}/test1-http-empty-request.json ${OUTPUT}/test2-http.nmsg.json.out +check nmsg-to-json + +######################## + +echo Testing the JSON input with an empty request ... + +# nmsgtool --readjson takes empty "" request value and turns it into a +# single byte null string termination, so the message size is one byte +# larger than the broken NMSG message that has no string termination. +# This still results in empty request "". + +$NMSGTOOL --readjson ${SOURCE}/test1-http-empty-request.json > ${OUTPUT}/test3-http.json.pres.out +check read json base:http and create base:http presentation output +cmp -s ${SOURCE}/test2-http-empty-request.pres ${OUTPUT}/test3-http.json.pres.out +check json-to-presentation + +$NMSGTOOL --readjson ${SOURCE}/test1-http-empty-request.json -w ${OUTPUT}/test3-http.json.nmsg.out +check read json base:http and create base:http nmsg output +cmp -s ${SOURCE}/test2-http-empty-request.nmsg ${OUTPUT}/test3-http.json.nmsg.out +check json-to-nmsg + +# output should be same as input +$NMSGTOOL --readjson ${SOURCE}/test1-http-empty-request.json -J ${OUTPUT}/test3-http.json.json.out +check read json base:http and create base:http json output +# using same JSON to compare against +cmp -s ${SOURCE}/test1-http-empty-request.json ${OUTPUT}/test3-http.json.json.out +check json-to-json + +######################## + +# This is for NMSG message that omits the "request" field. + +echo Testing NMSG input that has no request ... + +$NMSGTOOL -r ${SOURCE}/test4-http-no-request.nmsg > ${OUTPUT}/test4-http.nmsg.pres.out +check read nmsg base:http and create base:http presentation output +cmp -s ${SOURCE}/test4-http-no-request.pres ${OUTPUT}/test4-http.nmsg.pres.out +check nmsg-to-presentation + +# output should be same as input, pass through +$NMSGTOOL -r ${SOURCE}/test4-http-no-request.nmsg -w ${OUTPUT}/test4-http.nmsg.nmsg.out +check read nmsg base:http and create base:http nmsg output +cmp -s ${SOURCE}/test4-http-no-request.nmsg ${OUTPUT}/test4-http.nmsg.nmsg.out +check nmsg-to-nmsg + +$NMSGTOOL -r ${SOURCE}/test4-http-no-request.nmsg -J ${OUTPUT}/test4-http.nmsg.json.out +check read nmsg base:http and create base:http json output +cmp -s ${SOURCE}/test4-http-no-request.json ${OUTPUT}/test4-http.nmsg.json.out +check nmsg-to-json + +####################### + +echo Testing JSON input that has no request ... + +$NMSGTOOL -j ${SOURCE}/test4-http-no-request.json > ${OUTPUT}/test4-http.json.pres.out +check read json base:http and create base:http presentation output +cmp -s ${SOURCE}/test4-http-no-request.pres ${OUTPUT}/test4-http.json.pres.out +check json-to-presentation + +$NMSGTOOL -j ${SOURCE}/test4-http-no-request.json -w ${OUTPUT}/test4-http.json.nmsg.out +check read json base:http and create base:http nmsg output +cmp -s ${SOURCE}/test4-http-no-request.nmsg ${OUTPUT}/test4-http.json.nmsg.out +check json-to-nmsg + +# output should be same as input, pass through +$NMSGTOOL -j ${SOURCE}/test4-http-no-request.json -J ${OUTPUT}/test4-http.json.json.out +check read json base:http and create base:http json output +cmp -s ${SOURCE}/test4-http-no-request.json ${OUTPUT}/test4-http.json.json.out +check json-to-json + +# NOTE: --readpres is not implemented for base:http + +exit $status diff --git a/tests/nmsg-http-tests/test1-http-broken-request.nmsg b/tests/nmsg-http-tests/test1-http-broken-request.nmsg new file mode 100644 index 00000000..6ee8cce5 Binary files /dev/null and b/tests/nmsg-http-tests/test1-http-broken-request.nmsg differ diff --git a/tests/nmsg-http-tests/test1-http-empty-request.json b/tests/nmsg-http-tests/test1-http-empty-request.json new file mode 100644 index 00000000..56196a1b --- /dev/null +++ b/tests/nmsg-http-tests/test1-http-empty-request.json @@ -0,0 +1,2 @@ +{"time":"2023-09-25 17:11:29.750339409","vname":"base","mname":"http","source":"abcdabcd","operator":1,"group":3,"message":{"type":"sinkhole","srcip":"10.11.12.13","srcport":2000,"dstip":"14.15.16.17","dstport":80,"request":"GET / HTTP/1.0\r\nUser-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; BTRS102048; .NET CLR 1.1.4322; .NET CLR 1.0.3705)\r\nHost: 12345678.example.net\r\nPragma: no-cache\r\n\r\n"}} +{"time":"2023-09-25 17:11:29.770269405","vname":"base","mname":"http","source":"abcdabcd","operator":1,"group":3,"message":{"type":"sinkhole","srcip":"10.11.12.13","srcport":2000,"dstip":"14.15.16.17","dstport":80,"request":""}} diff --git a/tests/nmsg-http-tests/test1-http-empty-request.pres b/tests/nmsg-http-tests/test1-http-empty-request.pres new file mode 100644 index 00000000..13f6586a --- /dev/null +++ b/tests/nmsg-http-tests/test1-http-empty-request.pres @@ -0,0 +1,23 @@ +[201] [2023-09-25 17:11:29.750339409] [1:4 base http] [abcdabcd] [1] [3] +type: sinkhole +srcip: 10.11.12.13 +srcport: 2000 +dstip: 14.15.16.17 +dstport: 80 +request: +GET / HTTP/1.0 +User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; BTRS102048; .NET CLR 1.1.4322; .NET CLR 1.0.3705) +Host: 12345678.example.net +Pragma: no-cache + +. + +[21] [2023-09-25 17:11:29.770269405] [1:4 base http] [abcdabcd] [1] [3] +type: sinkhole +srcip: 10.11.12.13 +srcport: 2000 +dstip: 14.15.16.17 +dstport: 80 +request: +. + diff --git a/tests/nmsg-http-tests/test2-http-empty-request.nmsg b/tests/nmsg-http-tests/test2-http-empty-request.nmsg new file mode 100644 index 00000000..4dd2ceeb Binary files /dev/null and b/tests/nmsg-http-tests/test2-http-empty-request.nmsg differ diff --git a/tests/nmsg-http-tests/test2-http-empty-request.pres b/tests/nmsg-http-tests/test2-http-empty-request.pres new file mode 100644 index 00000000..23d0c963 --- /dev/null +++ b/tests/nmsg-http-tests/test2-http-empty-request.pres @@ -0,0 +1,23 @@ +[201] [2023-09-25 17:11:29.750339409] [1:4 base http] [abcdabcd] [1] [3] +type: sinkhole +srcip: 10.11.12.13 +srcport: 2000 +dstip: 14.15.16.17 +dstport: 80 +request: +GET / HTTP/1.0 +User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; BTRS102048; .NET CLR 1.1.4322; .NET CLR 1.0.3705) +Host: 12345678.example.net +Pragma: no-cache + +. + +[22] [2023-09-25 17:11:29.770269405] [1:4 base http] [abcdabcd] [1] [3] +type: sinkhole +srcip: 10.11.12.13 +srcport: 2000 +dstip: 14.15.16.17 +dstport: 80 +request: +. + diff --git a/tests/nmsg-http-tests/test4-http-no-request.json b/tests/nmsg-http-tests/test4-http-no-request.json new file mode 100644 index 00000000..9b447c21 --- /dev/null +++ b/tests/nmsg-http-tests/test4-http-no-request.json @@ -0,0 +1,3 @@ +{"time":"2019-10-11 15:09:53.193183634","vname":"base","mname":"http","source":"abbaabba","operator":102,"group":1972,"message":{"type":"sinkhole","srcip":"198.51.100.195","srcport":40188,"dstip":"192.0.2.141","dstport":80,"request":"POST /xw1.php HTTP/1.1\r\nContent-Type: application/x-www-form-urlencoded\r\nUser-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:28.0) Gecko/20100101 Firefox/28.0\r\nHost: 192.0.2.141\r\nContent-Length: 19\r\nConnection: Keep-Alive\r\nCache-Control: no-cache\r\n\r\nh=die(@md5(Apri1));"}} +{"time":"2019-10-11 15:09:54.163338057","vname":"base","mname":"http","source":"abbaabba","operator":101,"group":200,"message":{"type":"sinkhole","srcip":"198.51.100.153","srcport":62457,"dstip":"192.0.2.140","dstport":80,"request":"lv[endof]"}} +{"time":"2019-10-11 15:09:55.123456789","vname":"base","mname":"http","source":"abbaabba","operator":1098,"group":1099,"message":{"type":"sinkhole","srcip":"203.0.113.153","srcport":62457,"dstip":"192.0.2.140","dstport":80}} diff --git a/tests/nmsg-http-tests/test4-http-no-request.nmsg b/tests/nmsg-http-tests/test4-http-no-request.nmsg new file mode 100644 index 00000000..eeaf5341 Binary files /dev/null and b/tests/nmsg-http-tests/test4-http-no-request.nmsg differ diff --git a/tests/nmsg-http-tests/test4-http-no-request.pres b/tests/nmsg-http-tests/test4-http-no-request.pres new file mode 100644 index 00000000..9ad54589 --- /dev/null +++ b/tests/nmsg-http-tests/test4-http-no-request.pres @@ -0,0 +1,33 @@ +[288] [2019-10-11 15:09:53.193183634] [1:4 base http] [abbaabba] [102] [1972] +type: sinkhole +srcip: 198.51.100.195 +srcport: 40188 +dstip: 192.0.2.141 +dstport: 80 +request: +POST /xw1.php HTTP/1.1 +Content-Type: application/x-www-form-urlencoded +User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:28.0) Gecko/20100101 Firefox/28.0 +Host: 192.0.2.141 +Content-Length: 19 +Connection: Keep-Alive +Cache-Control: no-cache + +h=die(@md5(Apri1));. + +[32] [2019-10-11 15:09:54.163338057] [1:4 base http] [abbaabba] [101] [200] +type: sinkhole +srcip: 198.51.100.153 +srcport: 62457 +dstip: 192.0.2.140 +dstport: 80 +request: +lv[endof]. + +[20] [2019-10-11 15:09:55.123456789] [1:4 base http] [abbaabba] [1098] [1099] +type: sinkhole +srcip: 203.0.113.153 +srcport: 62457 +dstip: 192.0.2.140 +dstport: 80 +