From 7f4f57ad70b63373f10bafe80e2d55c196a4ce77 Mon Sep 17 00:00:00 2001 From: "Jeremy C. Reed" Date: Fri, 6 Oct 2023 17:43:11 +0000 Subject: [PATCH 1/2] add various base:http tests --- Makefile.am | 10 ++ configure.ac | 3 + tests/nmsg-http-tests/test.sh.in | 135 ++++++++++++++++++ .../test1-http-broken-request.nmsg | Bin 0 -> 304 bytes .../test1-http-empty-request.json | 2 + .../test1-http-empty-request.pres | 23 +++ .../test2-http-empty-request.nmsg | Bin 0 -> 305 bytes .../test2-http-empty-request.pres | 23 +++ .../test4-http-no-request.json | 3 + .../test4-http-no-request.nmsg | Bin 0 -> 461 bytes .../test4-http-no-request.pres | 33 +++++ 11 files changed, 232 insertions(+) create mode 100755 tests/nmsg-http-tests/test.sh.in create mode 100644 tests/nmsg-http-tests/test1-http-broken-request.nmsg create mode 100644 tests/nmsg-http-tests/test1-http-empty-request.json create mode 100644 tests/nmsg-http-tests/test1-http-empty-request.pres create mode 100644 tests/nmsg-http-tests/test2-http-empty-request.nmsg create mode 100644 tests/nmsg-http-tests/test2-http-empty-request.pres create mode 100644 tests/nmsg-http-tests/test4-http-no-request.json create mode 100644 tests/nmsg-http-tests/test4-http-no-request.nmsg create mode 100644 tests/nmsg-http-tests/test4-http-no-request.pres diff --git a/Makefile.am b/Makefile.am index 094a782b..9a8ac27c 100644 --- a/Makefile.am +++ b/Makefile.am @@ -382,6 +382,14 @@ EXTRA_DIST += tests/nmsg-dnsqr-tests/test1-dnsqr.pres EXTRA_DIST += tests/nmsg-dnstap-tests/test1-dnstap.json EXTRA_DIST += tests/nmsg-dnstap-tests/test1-dnstap.nmsg EXTRA_DIST += tests/nmsg-dnstap-tests/test1-dnstap.pres +EXTRA_DIST += tests/nmsg-http-tests/test1-http-broken-request.nmsg +EXTRA_DIST += tests/nmsg-http-tests/test1-http-empty-request.json +EXTRA_DIST += tests/nmsg-http-tests/test1-http-empty-request.pres +EXTRA_DIST += tests/nmsg-http-tests/test2-http-empty-request.nmsg +EXTRA_DIST += tests/nmsg-http-tests/test2-http-empty-request.pres +EXTRA_DIST += tests/nmsg-http-tests/test4-http-no-request.json +EXTRA_DIST += tests/nmsg-http-tests/test4-http-no-request.nmsg +EXTRA_DIST += tests/nmsg-http-tests/test4-http-no-request.pres noinst_PROGRAMS += libmy/crc32c_test libmy_crc32c_test_CFLAGS = $(AM_CFLAGS) @@ -405,6 +413,7 @@ TESTS += tests/nmsg-dns-tests/test.sh TESTS += tests/nmsg-dnsqr-tests/test.sh TESTS += tests/nmsg-dnstap-tests/test.sh TESTS += tests/nmsg-dnsobs-tests/test.sh +TESTS += tests/nmsg-http-tests/test.sh TESTS += tests/payload-crc32c-tests/test.sh TESTS += tests/string-tests/test.sh TESTS += tests/udp-checksum-tests/test.sh @@ -439,6 +448,7 @@ DISTCLEANFILES += tests/nmsg-dns-tests/test*.out DISTCLEANFILES += tests/nmsg-dnsobs-tests/test*.out DISTCLEANFILES += tests/nmsg-dnsqr-tests/test*.out DISTCLEANFILES += tests/nmsg-dnstap-tests/test*.out +DISTCLEANFILES += tests/nmsg-http-tests/test*.out # ## diff --git a/configure.ac b/configure.ac index cc30dd47..8b16c261 100644 --- a/configure.ac +++ b/configure.ac @@ -54,6 +54,9 @@ AC_CONFIG_FILES([tests/nmsg-dnsqr-tests/test.sh], AC_CONFIG_FILES([tests/nmsg-dnstap-tests/test.sh], [chmod +x tests/nmsg-dnstap-tests/test.sh]) +AC_CONFIG_FILES([tests/nmsg-http-tests/test.sh], + [chmod +x tests/nmsg-http-tests/test.sh]) + AC_CONFIG_FILES([tests/udp-checksum-tests/test.sh], [chmod +x tests/udp-checksum-tests/test.sh]) diff --git a/tests/nmsg-http-tests/test.sh.in b/tests/nmsg-http-tests/test.sh.in new file mode 100755 index 00000000..f620ca06 --- /dev/null +++ b/tests/nmsg-http-tests/test.sh.in @@ -0,0 +1,135 @@ +#!/bin/sh + +status=0 + +check() { + if [ $? = "0" ]; then + echo "PASS: $*" + else + echo "FAIL: $*" + status=1 + fi +} + +NMSG_MSGMOD_DIR="@abs_top_builddir@/nmsg/base/.libs" +export NMSG_MSGMOD_DIR +NMSGTOOL="@abs_top_builddir@/src/nmsgtool" + +SOURCE=@abs_top_srcdir@/tests/nmsg-http-tests/ +OUTPUT=@abs_top_builddir@/tests/nmsg-http-tests/ + +# cleanup from previous run +rm -f ${OUTPUT}/test*-http*out + +echo Testing the non-terminated broken request string ... + +# one byte smaller +$NMSGTOOL -r ${SOURCE}/test1-http-broken-request.nmsg > ${OUTPUT}/test1-http.nmsg.pres.out +check read nmsg base:http and create base:http presentation output +cmp -s ${SOURCE}/test1-http-empty-request.pres ${OUTPUT}/test1-http.nmsg.pres.out +check nmsg-to-presentation + +# output should be same as input, pass through +$NMSGTOOL -r ${SOURCE}/test1-http-broken-request.nmsg -w ${OUTPUT}/test1-http.nmsg.nmsg.out +check read nmsg base:http and create base:http nmsg output +cmp -s ${SOURCE}/test1-http-broken-request.nmsg ${OUTPUT}/test1-http.nmsg.nmsg.out +check nmsg-to-nmsg + +# broken request results in empty "request":"" +$NMSGTOOL -r ${SOURCE}/test1-http-broken-request.nmsg -J ${OUTPUT}/test1-http.nmsg.json.out +check read nmsg base:http and create base:http json output +cmp -s ${SOURCE}/test1-http-empty-request.json ${OUTPUT}/test1-http.nmsg.json.out +check nmsg-to-json + +######################## + +echo Testing the terminated request string ... + +$NMSGTOOL -r ${SOURCE}/test2-http-empty-request.nmsg > ${OUTPUT}/test2-http.nmsg.pres.out +check read nmsg base:http and create base:http presentation output +cmp -s ${SOURCE}/test2-http-empty-request.pres ${OUTPUT}/test2-http.nmsg.pres.out +check nmsg-to-presentation + +# output should be same as input, pass through +$NMSGTOOL -r ${SOURCE}/test2-http-empty-request.nmsg -w ${OUTPUT}/test2-http.nmsg.nmsg.out +check read nmsg base:http and create base:http nmsg output +cmp -s ${SOURCE}/test2-http-empty-request.nmsg ${OUTPUT}/test2-http.nmsg.nmsg.out +check nmsg-to-nmsg + +$NMSGTOOL -r ${SOURCE}/test2-http-empty-request.nmsg -J ${OUTPUT}/test2-http.nmsg.json.out +check read nmsg base:http and create base:http json output +# using same JSON to compare against +cmp -s ${SOURCE}/test1-http-empty-request.json ${OUTPUT}/test2-http.nmsg.json.out +check nmsg-to-json + +######################## + +echo Testing the JSON input with an empty request ... + +# nmsgtool --readjson takes empty "" request value and turns it into a +# single byte null string termination, so the message size is one byte +# larger than the broken NMSG message that has no string termination. +# This still results in empty request "". + +$NMSGTOOL --readjson ${SOURCE}/test1-http-empty-request.json > ${OUTPUT}/test3-http.json.pres.out +check read json base:http and create base:http presentation output +cmp -s ${SOURCE}/test2-http-empty-request.pres ${OUTPUT}/test3-http.json.pres.out +check json-to-presentation + +$NMSGTOOL --readjson ${SOURCE}/test1-http-empty-request.json -w ${OUTPUT}/test3-http.json.nmsg.out +check read json base:http and create base:http nmsg output +cmp -s ${SOURCE}/test2-http-empty-request.nmsg ${OUTPUT}/test3-http.json.nmsg.out +check json-to-nmsg + +# output should be same as input +$NMSGTOOL --readjson ${SOURCE}/test1-http-empty-request.json -J ${OUTPUT}/test3-http.json.json.out +check read json base:http and create base:http json output +# using same JSON to compare against +cmp -s ${SOURCE}/test1-http-empty-request.json ${OUTPUT}/test3-http.json.json.out +check json-to-json + +######################## + +# This is for NMSG message that omits the "request" field. + +echo Testing NMSG input that has no request ... + +$NMSGTOOL -r ${SOURCE}/test4-http-no-request.nmsg > ${OUTPUT}/test4-http.nmsg.pres.out +check read nmsg base:http and create base:http presentation output +cmp -s ${SOURCE}/test4-http-no-request.pres ${OUTPUT}/test4-http.nmsg.pres.out +check nmsg-to-presentation + +# output should be same as input, pass through +$NMSGTOOL -r ${SOURCE}/test4-http-no-request.nmsg -w ${OUTPUT}/test4-http.nmsg.nmsg.out +check read nmsg base:http and create base:http nmsg output +cmp -s ${SOURCE}/test4-http-no-request.nmsg ${OUTPUT}/test4-http.nmsg.nmsg.out +check nmsg-to-nmsg + +$NMSGTOOL -r ${SOURCE}/test4-http-no-request.nmsg -J ${OUTPUT}/test4-http.nmsg.json.out +check read nmsg base:http and create base:http json output +cmp -s ${SOURCE}/test4-http-no-request.json ${OUTPUT}/test4-http.nmsg.json.out +check nmsg-to-json + +####################### + +echo Testing JSON input that has no request ... + +$NMSGTOOL -j ${SOURCE}/test4-http-no-request.json > ${OUTPUT}/test4-http.json.pres.out +check read json base:http and create base:http presentation output +cmp -s ${SOURCE}/test4-http-no-request.pres ${OUTPUT}/test4-http.json.pres.out +check json-to-presentation + +$NMSGTOOL -j ${SOURCE}/test4-http-no-request.json -w ${OUTPUT}/test4-http.json.nmsg.out +check read json base:http and create base:http nmsg output +cmp -s ${SOURCE}/test4-http-no-request.nmsg ${OUTPUT}/test4-http.json.nmsg.out +check json-to-nmsg + +# output should be same as input, pass through +$NMSGTOOL -j ${SOURCE}/test4-http-no-request.json -J ${OUTPUT}/test4-http.json.json.out +check read json base:http and create base:http json output +cmp -s ${SOURCE}/test4-http-no-request.json ${OUTPUT}/test4-http.json.json.out +check json-to-json + +# NOTE: --readpres is not implemented for base:http + +exit $status diff --git a/tests/nmsg-http-tests/test1-http-broken-request.nmsg b/tests/nmsg-http-tests/test1-http-broken-request.nmsg new file mode 100644 index 0000000000000000000000000000000000000000..6ee8cce5684aec2639fc539b78978cb06e9d568b GIT binary patch literal 304 zcmebC4R&W>VqjoY<9f=-!6?8Yaq!=<6>O@3t~+(KP69=QSh%=(coi=2Yq9Y03kVtn zSZ!u>cMVa{SMUf43D7sxGvMV4Elw@cbxcpqE3s1W&9BPL$w|~V(KArcNY2kKNG!=r z%1O0W@D28KRWQ>tuvQ4q%uC5HFIMmiQ83jrvQ}^k2?{neFfuT)uvXCX1DfsZ6Qp3M zXQ*dlYy^}>5;D*;Ha9TUyL%v=UY-n^UeQddh9n=g@c2+W#)>?ps$pSv&b GGXelGK2og! literal 0 HcmV?d00001 diff --git a/tests/nmsg-http-tests/test1-http-empty-request.json b/tests/nmsg-http-tests/test1-http-empty-request.json new file mode 100644 index 00000000..56196a1b --- /dev/null +++ b/tests/nmsg-http-tests/test1-http-empty-request.json @@ -0,0 +1,2 @@ +{"time":"2023-09-25 17:11:29.750339409","vname":"base","mname":"http","source":"abcdabcd","operator":1,"group":3,"message":{"type":"sinkhole","srcip":"10.11.12.13","srcport":2000,"dstip":"14.15.16.17","dstport":80,"request":"GET / HTTP/1.0\r\nUser-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; BTRS102048; .NET CLR 1.1.4322; .NET CLR 1.0.3705)\r\nHost: 12345678.example.net\r\nPragma: no-cache\r\n\r\n"}} +{"time":"2023-09-25 17:11:29.770269405","vname":"base","mname":"http","source":"abcdabcd","operator":1,"group":3,"message":{"type":"sinkhole","srcip":"10.11.12.13","srcport":2000,"dstip":"14.15.16.17","dstport":80,"request":""}} diff --git a/tests/nmsg-http-tests/test1-http-empty-request.pres b/tests/nmsg-http-tests/test1-http-empty-request.pres new file mode 100644 index 00000000..57f5f879 --- /dev/null +++ b/tests/nmsg-http-tests/test1-http-empty-request.pres @@ -0,0 +1,23 @@ +[201] [2023-09-25 17:11:29.750339409] [1:4 base http] [abcdabcd] [(null)] [(null)] +type: sinkhole +srcip: 10.11.12.13 +srcport: 2000 +dstip: 14.15.16.17 +dstport: 80 +request: +GET / HTTP/1.0 +User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; BTRS102048; .NET CLR 1.1.4322; .NET CLR 1.0.3705) +Host: 12345678.example.net +Pragma: no-cache + +. + +[21] [2023-09-25 17:11:29.770269405] [1:4 base http] [abcdabcd] [(null)] [(null)] +type: sinkhole +srcip: 10.11.12.13 +srcport: 2000 +dstip: 14.15.16.17 +dstport: 80 +request: +. + diff --git a/tests/nmsg-http-tests/test2-http-empty-request.nmsg b/tests/nmsg-http-tests/test2-http-empty-request.nmsg new file mode 100644 index 0000000000000000000000000000000000000000..4dd2ceebbc205f6e88a7a44b77448d381a1d0ecc GIT binary patch literal 305 zcmebC4R&W>VqjoY=X%P>!6?8Yaq!=<6>O@3t~+(KP69=QSh%=(coi=2Yq9Y03kVtn zSZ!u>cMVa{SMUf43D7sxGvMV4Elw@cbxcpqE3s1W&9BPL$w|~V(KArcNY2kKNG!=r z%1O0W@D28KRWQ>tuvQ4q%uC5HFIMmiQ83jrvQ}^k2?{neFfuT)uvXCX1DfsZ6Qp3M zXQ*dlYy^}>5;D*;Ha9TUyL%v^>@-n^UeQddh1n=cuWvVqjoA%(a(^gHeD*;^4NXx2&p@_BV5DEdYuLu^cl_Ijrz!CzBS-0S2aC zg8-{erU3uo5C#2;aznj>i~gLt%$%G=eN#OH1&s(p zLu&<}%)HVHg$fI^crz1gg`zSmBMUtPO$GPVn4Llc)57FGHg>aQ#BlNQ%p4+3yLxgH8rgnEOuRgb(hN_&0`Bcmkl`h z4uONuGkg!HmI5^NW-9!A&I}H{9$@Hk<&;II=B4DP#lm!_dYoY7G6d^B4Aw1rHkw^a n1f=^kLm`T8sC}n6JWg{6Oq%g#0hhqu3D;IJ32Z(7=_v;QFVT~M literal 0 HcmV?d00001 diff --git a/tests/nmsg-http-tests/test4-http-no-request.pres b/tests/nmsg-http-tests/test4-http-no-request.pres new file mode 100644 index 00000000..5afbf763 --- /dev/null +++ b/tests/nmsg-http-tests/test4-http-no-request.pres @@ -0,0 +1,33 @@ +[288] [2019-10-11 15:09:53.193183634] [1:4 base http] [abbaabba] [(null)] [(null)] +type: sinkhole +srcip: 198.51.100.195 +srcport: 40188 +dstip: 192.0.2.141 +dstport: 80 +request: +POST /xw1.php HTTP/1.1 +Content-Type: application/x-www-form-urlencoded +User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:28.0) Gecko/20100101 Firefox/28.0 +Host: 192.0.2.141 +Content-Length: 19 +Connection: Keep-Alive +Cache-Control: no-cache + +h=die(@md5(Apri1));. + +[32] [2019-10-11 15:09:54.163338057] [1:4 base http] [abbaabba] [(null)] [(null)] +type: sinkhole +srcip: 198.51.100.153 +srcport: 62457 +dstip: 192.0.2.140 +dstport: 80 +request: +lv[endof]. + +[20] [2019-10-11 15:09:55.123456789] [1:4 base http] [abbaabba] [(null)] [(null)] +type: sinkhole +srcip: 203.0.113.153 +srcport: 62457 +dstip: 192.0.2.140 +dstport: 80 + From 5c1e76f22790053708cb5be1485f3702503cf080 Mon Sep 17 00:00:00 2001 From: "Jeremy C. Reed" Date: Wed, 11 Oct 2023 16:10:57 +0000 Subject: [PATCH 2/2] update base:http tests for fixed group/operator changes (also don't use any installed alias file) --- tests/nmsg-http-tests/test.sh.in | 6 ++++++ tests/nmsg-http-tests/test1-http-empty-request.pres | 4 ++-- tests/nmsg-http-tests/test2-http-empty-request.pres | 4 ++-- tests/nmsg-http-tests/test4-http-no-request.pres | 6 +++--- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/tests/nmsg-http-tests/test.sh.in b/tests/nmsg-http-tests/test.sh.in index f620ca06..812dba7e 100755 --- a/tests/nmsg-http-tests/test.sh.in +++ b/tests/nmsg-http-tests/test.sh.in @@ -21,6 +21,12 @@ OUTPUT=@abs_top_builddir@/tests/nmsg-http-tests/ # cleanup from previous run rm -f ${OUTPUT}/test*-http*out +# Test with no aliases +NMSG_OPALIAS_FILE=/dev/null +NMSG_GRALIAS_FILE=/dev/null +export NMSG_OPALIAS_FILE +export NMSG_GRALIAS_FILE + echo Testing the non-terminated broken request string ... # one byte smaller diff --git a/tests/nmsg-http-tests/test1-http-empty-request.pres b/tests/nmsg-http-tests/test1-http-empty-request.pres index 57f5f879..13f6586a 100644 --- a/tests/nmsg-http-tests/test1-http-empty-request.pres +++ b/tests/nmsg-http-tests/test1-http-empty-request.pres @@ -1,4 +1,4 @@ -[201] [2023-09-25 17:11:29.750339409] [1:4 base http] [abcdabcd] [(null)] [(null)] +[201] [2023-09-25 17:11:29.750339409] [1:4 base http] [abcdabcd] [1] [3] type: sinkhole srcip: 10.11.12.13 srcport: 2000 @@ -12,7 +12,7 @@ Pragma: no-cache . -[21] [2023-09-25 17:11:29.770269405] [1:4 base http] [abcdabcd] [(null)] [(null)] +[21] [2023-09-25 17:11:29.770269405] [1:4 base http] [abcdabcd] [1] [3] type: sinkhole srcip: 10.11.12.13 srcport: 2000 diff --git a/tests/nmsg-http-tests/test2-http-empty-request.pres b/tests/nmsg-http-tests/test2-http-empty-request.pres index 9a5ed887..23d0c963 100644 --- a/tests/nmsg-http-tests/test2-http-empty-request.pres +++ b/tests/nmsg-http-tests/test2-http-empty-request.pres @@ -1,4 +1,4 @@ -[201] [2023-09-25 17:11:29.750339409] [1:4 base http] [abcdabcd] [(null)] [(null)] +[201] [2023-09-25 17:11:29.750339409] [1:4 base http] [abcdabcd] [1] [3] type: sinkhole srcip: 10.11.12.13 srcport: 2000 @@ -12,7 +12,7 @@ Pragma: no-cache . -[22] [2023-09-25 17:11:29.770269405] [1:4 base http] [abcdabcd] [(null)] [(null)] +[22] [2023-09-25 17:11:29.770269405] [1:4 base http] [abcdabcd] [1] [3] type: sinkhole srcip: 10.11.12.13 srcport: 2000 diff --git a/tests/nmsg-http-tests/test4-http-no-request.pres b/tests/nmsg-http-tests/test4-http-no-request.pres index 5afbf763..9ad54589 100644 --- a/tests/nmsg-http-tests/test4-http-no-request.pres +++ b/tests/nmsg-http-tests/test4-http-no-request.pres @@ -1,4 +1,4 @@ -[288] [2019-10-11 15:09:53.193183634] [1:4 base http] [abbaabba] [(null)] [(null)] +[288] [2019-10-11 15:09:53.193183634] [1:4 base http] [abbaabba] [102] [1972] type: sinkhole srcip: 198.51.100.195 srcport: 40188 @@ -15,7 +15,7 @@ Cache-Control: no-cache h=die(@md5(Apri1));. -[32] [2019-10-11 15:09:54.163338057] [1:4 base http] [abbaabba] [(null)] [(null)] +[32] [2019-10-11 15:09:54.163338057] [1:4 base http] [abbaabba] [101] [200] type: sinkhole srcip: 198.51.100.153 srcport: 62457 @@ -24,7 +24,7 @@ dstport: 80 request: lv[endof]. -[20] [2019-10-11 15:09:55.123456789] [1:4 base http] [abbaabba] [(null)] [(null)] +[20] [2019-10-11 15:09:55.123456789] [1:4 base http] [abbaabba] [1098] [1099] type: sinkhole srcip: 203.0.113.153 srcport: 62457