Merge pull request #112 from farsightsec/next

Next Release (1.1.0)

.gitignore

@@ -10,6 +10,7 @@
 .deps
 .dirstamp
 .libs
+.gdb_history
 Makefile
 Makefile.in
 TAGS

.travis.yml

@@ -22,14 +22,14 @@ before_install:
 
 install:
   # Linux
-  - if [ "$TRAVIS_OS_NAME" == "linux" ]; then sudo apt-get install -qy libpcap0.8-dev libxs-dev zlib1g-dev docbook5-xml docbook-xsl-ns xsltproc libyajl-dev libwdns-dev; fi
+  - if [ "$TRAVIS_OS_NAME" == "linux" ]; then sudo apt-get install -qy libpcap0.8-dev libxs-dev zlib1g-dev docbook5-xml docbook-xsl-ns xsltproc libjson-c-dev libwdns-dev; fi
   - if [ "$TRAVIS_OS_NAME" == "linux" ] && [ "$(lsb_release -sc)" == "precise" ]; then sudo apt-get install -qy libprotobuf-c-dev protobuf-c-compiler; fi
   - if [ "$TRAVIS_OS_NAME" == "linux" ] && [ "$(lsb_release -sc)" == "trusty" ]; then sh -c 'wget -nv https://github.com/google/protobuf/releases/download/v2.6.1/protobuf-2.6.1.tar.gz && tar xf protobuf-2.6.1.tar.gz && cd protobuf-2.6.1 && ./configure --prefix=/usr && make && sudo make install'; fi
   - if [ "$TRAVIS_OS_NAME" == "linux" ] && [ "$(lsb_release -sc)" == "trusty" ]; then sh -c 'git clone -b next https://github.com/protobuf-c/protobuf-c.git && cd protobuf-c && ./autogen.sh && ./configure --prefix=/usr && make && sudo make install'; fi
 
   # OS X
   - if [ "$TRAVIS_OS_NAME" == "osx" ]; then brew install protobuf-c; fi
-  - if [ "$TRAVIS_OS_NAME" == "osx" ]; then brew install yajl; fi
+  - if [ "$TRAVIS_OS_NAME" == "osx" ]; then brew install json-c; fi
   - if [ "$TRAVIS_OS_NAME" == "osx" ]; then sh -c 'git clone https://github.com/farsightsec/wdns.git && cd wdns && ./autogen.sh && ./configure && make && sudo make install'; fi
 
 script:

COPYRIGHT

@@ -1,6 +1,7 @@
 Most nmsg code is under the following copyright and license:
 
-    Copyright (c) 2008-2020 by Farsight Security, Inc.
+    Copyright (c) 2023 DomainTools LLC
+    Copyright (c) 2008-2021 by Farsight Security, Inc.
 
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.

ChangeLog

@@ -1,3 +1,25 @@
+nmsg (1.1.0)
+
+  * Multiple performance improvements to JSON formatting.
+
+  * Multiple performance and concurrency improvements in nmsg_io loop.
+
+  * Add new dnsobs message type for raw passive DNS observations.
+
+  * Format DNS messages as JSON in base:dnsqr, base:dnstap, and base:dnsobs.
+
+  * Format base:encode payload in human readable form in JSON output.
+
+  * Replace yajl with json-c for JSON parsing.
+
+  * nmsgtool: exit with nonzero status when interrupted by signal.
+
+  * Correct container size calculation when sequencing.
+
+  * Address doxygen deprecations.
+
+ -- Farsight Security, Inc. <[email protected]>  Wed, 16 Aug 2023 18:43:40 +0000
+
 nmsg (1.0.1)
 
   * nmsgtool: Fix use-after-free bug in statistics debugging

Makefile.am

@@ -5,6 +5,7 @@ TESTS =
 BUILT_SOURCES =
 EXTRA_DIST =
 CLEANFILES =
+DISTCLEANFILES =
 ACLOCAL_AMFLAGS = -I m4 ${ACLOCAL_FLAGS}
 
 AM_CPPFLAGS = \
@@ -17,7 +18,7 @@ AM_CFLAGS = \
 	$(libprotobuf_c_CFLAGS) \
 	$(libwdns_CFLAGS) \
 	$(libzmq_CFLAGS) \
-	$(yajl_CFLAGS)
+	$(json_c_CFLAGS)
 AM_LDFLAGS =
 
 EXTRA_DIST += ChangeLog
@@ -31,7 +32,6 @@ EXTRA_DIST += \
 	tests/json-utf8-tests/straycont.nmsg \
 	tests/json-utf8-tests/test.sh.in \
 	tests/json-utf8-tests/truncated.nmsg \
-	tests/nmsg.test \
 	tests/string-tests/empty-string.json \
 	tests/string-tests/empty-string.nmsg \
 	tests/string-tests/empty-string.pres \
@@ -121,7 +121,7 @@ nobase_nodist_include_HEADERS = \
 
 lib_LTLIBRARIES = nmsg/libnmsg.la
 
-VERSION_INFO = 8:0:0
+VERSION_INFO = 9:0:1
 
 nmsg_libnmsg_la_LDFLAGS = \
 	$(AM_LDFLAGS) \
@@ -131,7 +131,7 @@ nmsg_libnmsg_la_LIBADD = \
 	$(libpcap_LIBS) \
 	$(libprotobuf_c_LIBS) \
 	$(libzmq_LIBS) \
-	$(yajl_LIBS)
+	$(json_c_LIBS)
 nmsg_libnmsg_la_SOURCES = \
 	libmy/crc32c.c libmy/crc32c.h libmy/crc32c-slicing.c libmy/crc32c-sse42.c \
 	libmy/list.h \
@@ -140,6 +140,7 @@ nmsg_libnmsg_la_SOURCES = \
 	libmy/tree.h \
 	libmy/b64_decode.c libmy/b64_decode.h \
 	libmy/b64_encode.c libmy/b64_encode.h \
+	libmy/fast_inet_ntop.c libmy/fast_inet_ntop.h \
 	nmsg/alias.c \
 	nmsg/asprintf.c \
 	nmsg/brate.c \
@@ -163,9 +164,9 @@ nmsg_libnmsg_la_SOURCES = \
 	nmsg/ipreasm.h \
 	nmsg/msgmodset.c \
 	nmsg/nmsg.c \
+	nmsg/nmsg_json.h \
 	nmsg/nmsg_port_net.h \
 	nmsg/output.c \
-	nmsg/output_frag.c \
 	nmsg/output_json.c \
 	nmsg/output_nmsg.c \
 	nmsg/output_pres.c \
@@ -223,7 +224,8 @@ nobase_nodist_include_HEADERS += \
 	nmsg/base/packet.pb-c.h \
 	nmsg/base/pkt.pb-c.h \
 	nmsg/base/xml.pb-c.h \
-	nmsg/base/dnstap.pb-c.h
+	nmsg/base/dnstap.pb-c.h \
+	nmsg/base/dnsobs.pb-c.h
 
 proto_DATA =  \
 	nmsg/base/dns.proto \
@@ -238,7 +240,8 @@ proto_DATA =  \
 	nmsg/base/packet.proto \
 	nmsg/base/pkt.proto \
 	nmsg/base/xml.proto \
-	nmsg/base/dnstap.proto
+	nmsg/base/dnstap.proto \
+	nmsg/base/dnsobs.proto
 EXTRA_DIST += $(proto_DATA)
 EXTRA_DIST += nmsg/nmsg.proto
 
@@ -264,7 +267,10 @@ nmsg_base_nmsg_msg9_base_la_SOURCES = \
 	libmy/ubuf.h \
 	libmy/vector.h \
 	nmsg/base/nmsg_msg_base.c \
-	nmsg/base/ipreasm.c
+	nmsg/base/ipreasm.c \
+	nmsg/nmsg_json.h \
+	libmy/b64_encode.c \
+	libmy/b64_encode.h
 nodist_nmsg_base_nmsg_msg9_base_la_SOURCES = \
 	nmsg/base/dns.pb-c.c nmsg/base/dns.pb-c.h \
 	nmsg/base/dnsqr.pb-c.c nmsg/base/dnsqr.pb-c.h \
@@ -278,7 +284,8 @@ nodist_nmsg_base_nmsg_msg9_base_la_SOURCES = \
 	nmsg/base/packet.pb-c.c nmsg/base/packet.pb-c.h \
 	nmsg/base/pkt.pb-c.c nmsg/base/pkt.pb-c.h \
 	nmsg/base/xml.pb-c.c nmsg/base/xml.pb-c.h \
-	nmsg/base/dnstap.pb-c.c nmsg/base/dnstap.pb-c.h
+	nmsg/base/dnstap.pb-c.c nmsg/base/dnstap.pb-c.h \
+	nmsg/base/dnsobs.pb-c.c nmsg/base/dnsobs.pb-c.h
 BUILT_SOURCES += $(nodist_nmsg_base_nmsg_msg9_base_la_SOURCES)
 CLEANFILES += $(nodist_nmsg_base_nmsg_msg9_base_la_SOURCES)
 EXTRA_DIST += \
@@ -295,7 +302,8 @@ EXTRA_DIST += \
 	nmsg/base/packet.c \
 	nmsg/base/pkt.c \
 	nmsg/base/xml.c \
-	nmsg/base/dnstap.c
+	nmsg/base/dnstap.c \
+	nmsg/base/dnsobs.c
 
 #
 ##
@@ -357,6 +365,24 @@ EXTRA_DIST += tests/generic-tests/test.chalias
 EXTRA_DIST += tests/generic-tests/test.gralias
 EXTRA_DIST += tests/generic-tests/test.opalias
 
+EXTRA_DIST += tests/nmsg-dns-tests/test1-dns.json
+EXTRA_DIST += tests/nmsg-dns-tests/test1-dns.nmsg
+EXTRA_DIST += tests/nmsg-dns-tests/test1-dns.pres
+EXTRA_DIST += tests/nmsg-dns-tests/test2-dns.json
+EXTRA_DIST += tests/nmsg-dns-tests/test2-dns.nmsg
+EXTRA_DIST += tests/nmsg-dns-tests/test2-dns.pres
+EXTRA_DIST += tests/nmsg-dns-tests/test3-dns.json
+EXTRA_DIST += tests/nmsg-dnsobs-tests/test1-dnsobs.json
+EXTRA_DIST += tests/nmsg-dnsobs-tests/test1-dnsobs.nmsg
+EXTRA_DIST += tests/nmsg-dnsobs-tests/test1-dnsobs.pres
+EXTRA_DIST += tests/nmsg-dnsqr-tests/test1-dnsqr.json
+EXTRA_DIST += tests/nmsg-dnsqr-tests/test1-dnsqr.nmsg
+EXTRA_DIST += tests/nmsg-dnsqr-tests/test1-dnsqr.pcap
+EXTRA_DIST += tests/nmsg-dnsqr-tests/test1-dnsqr.pres
+EXTRA_DIST += tests/nmsg-dnstap-tests/test1-dnstap.json
+EXTRA_DIST += tests/nmsg-dnstap-tests/test1-dnstap.nmsg
+EXTRA_DIST += tests/nmsg-dnstap-tests/test1-dnstap.pres
+
 noinst_PROGRAMS += libmy/crc32c_test
 libmy_crc32c_test_CFLAGS = $(AM_CFLAGS)
 libmy_crc32c_test_SOURCES = \
@@ -374,7 +400,14 @@ noinst_PROGRAMS += \
 	examples/nmsg-packet2pcap \
 	examples/print_version
 
-TESTS += tests/nmsg.test
+TESTS += tests/json-utf8-tests/test.sh
+TESTS += tests/nmsg-dns-tests/test.sh
+TESTS += tests/nmsg-dnsqr-tests/test.sh
+TESTS += tests/nmsg-dnstap-tests/test.sh
+TESTS += tests/nmsg-dnsobs-tests/test.sh
+TESTS += tests/payload-crc32c-tests/test.sh
+TESTS += tests/string-tests/test.sh
+TESTS += tests/udp-checksum-tests/test.sh
 
 TESTS += tests/test-layout-fltmod_plugin
 check_PROGRAMS += tests/test-layout-fltmod_plugin
@@ -395,13 +428,18 @@ TESTS += tests/test-misc
 check_PROGRAMS += tests/test-misc
 tests_test_misc_LDADD = nmsg/libnmsg.la
 tests_test_misc_CPPFLAGS = -DSRCDIR="\"$(abs_srcdir)\"" $(AM_CPPFLAGS)
-tests_test_misc_SOURCES = tests/test-misc.c
+tests_test_misc_SOURCES = tests/test-misc.c libmy/fast_inet_ntop.c
 
 TESTS += tests/test-nmsg_output_set_rate
 check_PROGRAMS += tests/test-nmsg_output_set_rate
 tests_test_nmsg_output_set_rate_SOURCES = tests/test-nmsg_output_set_rate.c
 tests_test_nmsg_output_set_rate_LDADD = nmsg/libnmsg.la
 
+DISTCLEANFILES += tests/nmsg-dns-tests/test*.out
+DISTCLEANFILES += tests/nmsg-dnsobs-tests/test*.out
+DISTCLEANFILES += tests/nmsg-dnsqr-tests/test*.out
+DISTCLEANFILES += tests/nmsg-dnstap-tests/test*.out
+
 #
 ##
 ### Examples

README.md

@@ -19,13 +19,13 @@ nmsg has the following external dependencies:
 
 * [zmq](http://zeromq.org/)
 
-* [yajl](http://lloyd.github.io/yajl/)
+* [json-c](https://github.com/json-c/json-c)
 
 * [zlib](http://www.zlib.net/)
 
 On Debian systems, the following packages should be installed, if available:
 
-    pkg-config libpcap0.8-dev libprotobuf-c-dev protobuf-c-compiler libzmq3-dev libyajl-dev zlib1g-dev
+    pkg-config libpcap0.8-dev libprotobuf-c-dev protobuf-c-compiler libzmq3-dev libjson-c-dev zlib1g-dev
 
 Note that on Debian systems, binary packages of nmsg and its dependencies are
 available from
@@ -36,7 +36,7 @@ Debian-based systems.
 On FreeBSD systems, the following ports should be installed, if available:
 
     devel/libzmq
-    devel/yajl
+    devel/json-c
     devel/pkgconf
     devel/protobuf
     devel/protobuf-c
@@ -54,7 +54,7 @@ script.
 Support for `libzmq` can be disabled by passing the `--without-libzmq` parameter
 to the `configure` script.
 
-Support for `yajl` can be disabled by passing the `--without-yajl` parameter
+Support for `json-c` can be disabled by passing the `--without-json-c` parameter
 to the `configure` script.
 
 The documentation for the `libnmsg` API is located in the `doc/doxygen/html`

configure.ac

@@ -1,8 +1,8 @@
 AC_PREREQ(2.64)
 
 m4_define(nmsg_major_version, 1)
-m4_define(nmsg_minor_version, 0)
-m4_define(nmsg_patchlevel_version, 1)
+m4_define(nmsg_minor_version, 1)
+m4_define(nmsg_patchlevel_version, 0)
 m4_define(nmsg_version,
 	  nmsg_major_version.nmsg_minor_version.nmsg_patchlevel_version)
 m4_define(nmsg_version_number,
@@ -42,6 +42,15 @@ AC_CONFIG_FILES([Makefile doc/doxygen/Doxyfile nmsg/libnmsg.pc nmsg/version.h])
 AC_CONFIG_FILES([tests/json-utf8-tests/test.sh],
                 [chmod +x tests/json-utf8-tests/test.sh])
 
+AC_CONFIG_FILES([tests/nmsg-dns-tests/test.sh],
+                [chmod +x tests/nmsg-dns-tests/test.sh])
+
+AC_CONFIG_FILES([tests/nmsg-dnsobs-tests/test.sh],
+                [chmod +x tests/nmsg-dnsobs-tests/test.sh])
+
+AC_CONFIG_FILES([tests/nmsg-dnsqr-tests/test.sh],
+                [chmod +x tests/nmsg-dnsqr-tests/test.sh])
+
 AC_CONFIG_FILES([tests/nmsg-dnstap-tests/test.sh],
                 [chmod +x tests/nmsg-dnstap-tests/test.sh])
 
@@ -122,7 +131,7 @@ AC_CHECK_MEMBER(
 )
 
 ###
-### External library dependencies: libpcap, libprotobuf-c, libwdns, zeromq, yajl, libz
+### External library dependencies: libpcap, libprotobuf-c, libwdns, zeromq, json-c, libz
 ###
 
 MY_CHECK_LIBPCAP
@@ -132,7 +141,7 @@ AC_PATH_PROG([PROTOC_C], [protoc-c])
 AS_IF([test -z "$PROTOC_C"],
       [AC_MSG_ERROR([The protoc-c program was not found. Please install the protobuf-c compiler!])])
 
-PKG_CHECK_MODULES([libwdns], [libwdns >= 0.8.0])
+PKG_CHECK_MODULES([libwdns], [libwdns >= 0.12.0])
 
 AC_ARG_WITH([zmq], AS_HELP_STRING([--without-libzmq], [Disable zmq support]))
 if test "x$with_libzmq" != "xno"; then
@@ -143,13 +152,13 @@ else
     use_libzmq="false"
 fi
 
-AC_ARG_WITH([yajl], AS_HELP_STRING([--without-yajl], [Disable yajl support]))
-if test "x$with_yajl" != "xno"; then
-    PKG_CHECK_MODULES([yajl], [yajl >= 2.1.0])
-    AC_DEFINE([HAVE_YAJL], [1], [Define to 1 if yajl support is enabled.])
-    use_yajl="true"
+AC_ARG_WITH([json-c], AS_HELP_STRING([--without-json-c], [Disable json-c support]))
+if test "x$with_json_c" != "xno"; then
+    PKG_CHECK_MODULES([json_c], [json-c >= 0.13.0])
+    AC_DEFINE([HAVE_JSON_C], [1], [Define to 1 if json-c support is enabled.])
+    use_json_c="true"
 else
-    use_yajl="false"
+    use_json_c="false"
 fi
 
 AC_CHECK_HEADER([zlib.h], [], [ AC_MSG_ERROR([required header file not found]) ])
@@ -211,7 +220,7 @@ AC_MSG_RESULT([
 
         bigendian:              ${ac_cv_c_bigendian}
         libzmq support:         ${use_libzmq}
-        yajl support:           ${use_yajl}
+        json-c support:         ${use_json_c}
 
         building html docs:     ${DOC_HTML_MSG}
         building manpage docs:  ${DOC_MAN_MSG}

debian/changelog

@@ -1,3 +1,25 @@
+nmsg (1.1.0-1) debian-fsi; urgency=medium
+
+  * Multiple performance improvements to JSON formatting.
+
+  * Multiple performance and concurrency improvements in nmsg_io loop.
+
+  * Add new dnsobs message type for raw passive DNS observations.
+
+  * Format DNS messages as JSON in base:dnsqr, base:dnstap, and base:dnsobs.
+
+  * Format base:encode payload in human readable form in JSON output.
+
+  * Replace yajl with json-c for JSON parsing.
+
+  * nmsgtool: exit with nonzero status when interrupted by signal.
+
+  * Correct container size calculation when sequencing.
+
+  * Address doxygen deprecations.
+
+ -- Farsight Security, Inc. <[email protected]>  Wed, 16 Aug 2023 18:43:40 +0000
+
 nmsg (1.0.1-1) debian-fsi; urgency=medium
 
   * nmsgtool: Fix use-after-free bug in statistics debugging

debian/control

@@ -3,18 +3,17 @@ Section: net
 Priority: optional
 Maintainer: Farsight Security, Inc. <[email protected]>
 Build-Depends:
- debhelper (>= 9~),
- dh-autoreconf (>= 5~),
+ debhelper-compat (= 13),
  docbook-xsl,
  docbook-xsl-ns,
  docbook5-xml,
  doxygen,
  dpkg-dev (>= 1.16.1~),
  libpcap0.8-dev,
  libprotobuf-c-dev (>= 1.0.1~),
- libwdns-dev (>= 0.8.0~),
+ libwdns-dev (>= 0.12.0~),
  libzmq3-dev (>= 4.2.0~),
- libyajl-dev (>= 2.1.0~),
+ libjson-c-dev (>= 0.13.0~),
  pkg-config,
  protobuf-c-compiler (>= 1.0.1~),
  xsltproc,