Merge pull request #113 from farsightsec/prep-1.1.0

Prep 1.1.0

COPYRIGHT

@@ -1,6 +1,7 @@
 Most nmsg code is under the following copyright and license:
 
-    Copyright (c) 2008-2020 by Farsight Security, Inc.
+    Copyright (c) 2023 DomainTools LLC
+    Copyright (c) 2008-2021 by Farsight Security, Inc.
 
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.

ChangeLog

@@ -1,3 +1,25 @@
+nmsg (1.1.0)
+
+  * Multiple performance improvements to JSON formatting.
+
+  * Multiple performance and concurrency improvements in nmsg_io loop.
+
+  * Add new dnsobs message type for raw passive DNS observations.
+
+  * Format DNS messages as JSON in base:dnsqr, base:dnstap, and base:dnsobs.
+
+  * Format base:encode payload in human readable form in JSON output.
+
+  * Replace yajl with json-c for JSON parsing.
+
+  * nmsgtool: exit with nonzero status when interrupted by signal.
+
+  * Correct container size calculation when sequencing.
+
+  * Address doxygen deprecations.
+
+ -- Farsight Security, Inc. <[email protected]>  Wed, 16 Aug 2023 18:43:40 +0000
+
 nmsg (1.0.1)
 
   * nmsgtool: Fix use-after-free bug in statistics debugging

Makefile.am

@@ -121,7 +121,7 @@ nobase_nodist_include_HEADERS = \
 
 lib_LTLIBRARIES = nmsg/libnmsg.la
 
-VERSION_INFO = 8:0:0
+VERSION_INFO = 9:0:1
 
 nmsg_libnmsg_la_LDFLAGS = \
 	$(AM_LDFLAGS) \
@@ -382,7 +382,11 @@ noinst_PROGRAMS += \
 	examples/nmsg-packet2pcap \
 	examples/print_version
 
-TESTS += tests/nmsg.test
+TESTS += tests/json-utf8-tests/test.sh
+TESTS += tests/nmsg-dnstap-tests/test.sh
+TESTS += tests/payload-crc32c-tests/test.sh
+TESTS += tests/string-tests/test.sh
+TESTS += tests/udp-checksum-tests/test.sh
 
 TESTS += tests/test-layout-fltmod_plugin
 check_PROGRAMS += tests/test-layout-fltmod_plugin

configure.ac

@@ -1,8 +1,8 @@
 AC_PREREQ(2.64)
 
 m4_define(nmsg_major_version, 1)
-m4_define(nmsg_minor_version, 0)
-m4_define(nmsg_patchlevel_version, 1)
+m4_define(nmsg_minor_version, 1)
+m4_define(nmsg_patchlevel_version, 0)
 m4_define(nmsg_version,
 	  nmsg_major_version.nmsg_minor_version.nmsg_patchlevel_version)
 m4_define(nmsg_version_number,
@@ -132,7 +132,7 @@ AC_PATH_PROG([PROTOC_C], [protoc-c])
 AS_IF([test -z "$PROTOC_C"],
       [AC_MSG_ERROR([The protoc-c program was not found. Please install the protobuf-c compiler!])])
 
-PKG_CHECK_MODULES([libwdns], [libwdns >= 0.8.0])
+PKG_CHECK_MODULES([libwdns], [libwdns >= 0.12.0])
 
 AC_ARG_WITH([zmq], AS_HELP_STRING([--without-libzmq], [Disable zmq support]))
 if test "x$with_libzmq" != "xno"; then

debian/changelog

@@ -1,3 +1,25 @@
+nmsg (1.1.0-1) debian-fsi; urgency=medium
+
+  * Multiple performance improvements to JSON formatting.
+
+  * Multiple performance and concurrency improvements in nmsg_io loop.
+
+  * Add new dnsobs message type for raw passive DNS observations.
+
+  * Format DNS messages as JSON in base:dnsqr, base:dnstap, and base:dnsobs.
+
+  * Format base:encode payload in human readable form in JSON output.
+
+  * Replace yajl with json-c for JSON parsing.
+
+  * nmsgtool: exit with nonzero status when interrupted by signal.
+
+  * Correct container size calculation when sequencing.
+
+  * Address doxygen deprecations.
+
+ -- Farsight Security, Inc. <[email protected]>  Wed, 16 Aug 2023 18:43:40 +0000
+
 nmsg (1.0.1-1) debian-fsi; urgency=medium
 
   * nmsgtool: Fix use-after-free bug in statistics debugging

debian/control

@@ -3,16 +3,15 @@ Section: net
 Priority: optional
 Maintainer: Farsight Security, Inc. <[email protected]>
 Build-Depends:
- debhelper (>= 9~),
- dh-autoreconf (>= 5~),
+ debhelper-compat (= 13),
  docbook-xsl,
  docbook-xsl-ns,
  docbook5-xml,
  doxygen,
  dpkg-dev (>= 1.16.1~),
  libpcap0.8-dev,
  libprotobuf-c-dev (>= 1.0.1~),
- libwdns-dev (>= 0.8.0~),
+ libwdns-dev (>= 0.12.0~),
  libzmq3-dev (>= 4.2.0~),
  libjson-c-dev (>= 0.13.0~),
  pkg-config,

debian/copyright

@@ -1,25 +1,20 @@
-This package was debianized by Robert S. Edmonds <[email protected]> on
-Tue, 16 Jun 2009 20:16:19 -0400.
-
-It was downloaded from:
-
-    https://dl.farsightsecurity.com/dist/nmsg/
-    https://github.com/farsightsec/nmsg
-
-Copyright holders:
-
-    Copyright (c) 2008-2021 by Farsight Security, Inc.
-
-License:
-
-    Licensed under the Apache License, Version 2.0 (the "License");
-    you may not use this file except in compliance with the License.
-    You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
+Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+
+Files: *
+Copyright: 2023 DomainTools LLC
+           2008-2021 by Farsight Security, Inc.
+License: Apache-2.0
+ Licensed under the Apache License, Version 2.0 (the "License"); you
+ may not use this file except in compliance with the License. You may
+ obtain a copy of the License at
+ .
+ http://www.apache.org/licenses/LICENSE-2.0.
+ .
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied. See the License for the specific language governing
+ permissions and limitations under the License.
+ .
+ On Debian systems, the complete text of the Apache version 2.0 license
+ can be found in "/usr/share/common-licenses/Apache-2.0".

debian/libnmsg-dev.install

@@ -1,3 +1,5 @@
 usr/include
+usr/lib/*/libnmsg.a
 usr/lib/*/libnmsg*.so
 usr/lib/*/pkgconfig/*.pc
+usr/share/nmsg/base/*.proto

debian/libnmsg8.symbols

@@ -169,6 +169,8 @@ libnmsg.so.8 libnmsg8 #MINVER#
  nmsg_sock_parse@Base 0.6.11
  nmsg_sock_parse_sockspec@Base 0.6.11
  nmsg_strbuf_append@Base 0.5.0
+ nmsg_strbuf_append_str@Base 1.1.0
+ nmsg_strbuf_append_str_json@Base 1.1.0
  nmsg_strbuf_destroy@Base 0.5.0
  nmsg_strbuf_init@Base 0.5.0
  nmsg_strbuf_len@Base 0.5.0

debian/not-installed

@@ -0,0 +1,3 @@
+usr/lib/*/libnmsg.la
+usr/lib/*/nmsg/nmsg_flt1_sample.la
+usr/lib/*/nmsg/nmsg_msg9_base.la

debian/rules

@@ -7,7 +7,7 @@ include /usr/share/dpkg/buildflags.mk
 CFLAGS += -fno-omit-frame-pointer
 
 %:
-	dh $@ --with autoreconf --parallel
+	dh $@
 
 override_dh_strip:
 

nmsg/base/dns.c

@@ -1,6 +1,7 @@
 /* dns nmsg message module */
 
 /*
+ * Copyright (c) 2023 DomainTools LLC
  * Copyright (c) 2009, 2015, 2021 by Farsight Security, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");

nmsg/base/dnsobs.c

@@ -1,7 +1,7 @@
 /* dnsobs nmsg message module */
 
 /*
- * Copyright (c) 2023 by Farsight Security, Inc.
+ * Copyright (c) 2023 DomainTools LLC
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -133,6 +133,7 @@ dnsobs_get_response(nmsg_message_t msg,
 	return (nmsg_res_success);
 }
 
+
 static nmsg_res
 dnsobs_sid_print(nmsg_message_t msg,
 		 struct nmsg_msgmod_field *field,

nmsg/base/dnsqr.c

@@ -1,6 +1,7 @@
 /* dnsqr nmsg message module */
 
 /*
+ * Copyright (c) 2023 DomainTools LLC
  * Copyright (c) 2010-2016, 2018, 2019, 2021 by Farsight Security, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");

nmsg/base/dnstap.c

@@ -1,6 +1,7 @@
 /* dnstap nmsg message module */
 
 /*
+ * Copyright (c) 2023 DomainTools LLC
  * Copyright (c) 2016 by Farsight Security, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");

nmsg/base/encode.c

@@ -1,6 +1,7 @@
 /* encode nmsg message module */
 
 /*
+ * Copyright (c) 2023 DomainTools LLC
  * Copyright (c) 2011 by Farsight Security, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");

nmsg/base/nmsg_msg_base.c

@@ -1,7 +1,8 @@
 /* nmsg_msg_base.c - base nmsg_msg modules */
 
 /*
- * Copyright (c) 2008-2012 by Farsight Security, Inc.
+ * Copyright (c) 2022 DomainTools LLC
+ * Copyright (c) 2008-2012, 2016 by Farsight Security, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

nmsg/container.c

@@ -1,5 +1,6 @@
 /*
- * Copyright (c) 2012 by Farsight Security, Inc.
+ * Copyright (c) 2023 DomainTools LLC
+ * Copyright (c) 2012, 2018 by Farsight Security, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

nmsg/input.c

@@ -1,4 +1,5 @@
 /*
+ * Copyright (c) 2023 DomainTools LLC
  * Copyright (c) 2008-2019 by Farsight Security, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");

nmsg/input_json.c

@@ -1,5 +1,6 @@
 /*
- * Copyright (c) 2015 by Farsight Security, Inc.
+ * Copyright (c) 2023 DomainTools LLC
+ * Copyright (c) 2015, 2019 by Farsight Security, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

nmsg/io.c

@@ -1,4 +1,5 @@
 /*
+ * Copyright (c) 2023 DomainTools LLC
  * Copyright (c) 2008-2021 by Farsight Security, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");

nmsg/msgmod.h

@@ -1,4 +1,5 @@
 /*
+ * Copyright (c) 2023 DomainTools LLC
  * Copyright (c) 2008-2015 by Farsight Security, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");

nmsg/msgmod/message.c

@@ -1,5 +1,6 @@
 /*
- * Copyright (c) 2009-2012, 2015 by Farsight Security, Inc.
+ * Copyright (c) 2023 DomainTools LLC
+ * Copyright (c) 2009-2012, 2015-2016, 2018-2019 by Farsight Security, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

nmsg/msgmod/transparent.h

@@ -1,4 +1,5 @@
 /*
+ * Copyright (c) 2023 DomainTools LLC
  * Copyright (c) 2009, 2015 by Farsight Security, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");

nmsg/msgmod/transparent_json.c

@@ -1,5 +1,6 @@
 /*
- * Copyright (c) 2015 by Farsight Security, Inc.
+ * Copyright (c) 2023 DomainTools LLC
+ * Copyright (c) 2015-2019 by Farsight Security, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

nmsg/msgmod/transparent_message.c

@@ -1,5 +1,6 @@
 /*
- * Copyright (c) 2009-2012, 2015 by Farsight Security, Inc.
+ * Copyright (c) 2023 DomainTools LLC
+ * Copyright (c) 2009-2012, 2015-2016, 2019 by Farsight Security, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

nmsg/msgmod/transparent_module.c

@@ -1,4 +1,5 @@
 /*
+ * Copyright (c) 2023 DomainTools LLC
  * Copyright (c) 2009, 2010, 2012 by Farsight Security, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");

nmsg/msgmod/transparent_payload.c

@@ -1,5 +1,6 @@
 /*
- * Copyright (c) 2009-2016 by Farsight Security, Inc.
+ * Copyright (c) 2023 DomainTools LLC
+ * Copyright (c) 2009-2017, 2019 by Farsight Security, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

nmsg/output.c

@@ -1,4 +1,5 @@
 /*
+ * Copyright (c) 2023 DomainTools LLC
  * Copyright (c) 2008-2019 by Farsight Security, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");

nmsg/output_json.c

@@ -1,4 +1,5 @@
 /*
+ * Copyright (c) 2023 DomainTools LLC
  * Copyright (c) 2015 by Farsight Security, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");

nmsg/output_nmsg.c

@@ -1,4 +1,5 @@
 /*
+ * Copyright (c) 2023 DomainTools LLC
  * Copyright (c) 2008-2019 by Farsight Security, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");

nmsg/payload.c

@@ -1,4 +1,5 @@
 /*
+ * Coypright (c) 2023 DomainTools LLC
  * Copyright (c) 2008-2013 by Farsight Security, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");

nmsg/private.h

@@ -1,5 +1,6 @@
 /*
- * Copyright (c) 2008-2015, 2019 by Farsight Security, Inc.
+ * Copyright (c) 2023 DomainTools LLC
+ * Copyright (c) 2008-2015, 2019, 2021 by Farsight Security, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

nmsg/strbuf.c

@@ -1,5 +1,6 @@
 /*
- * Copyright (c) 2009, 2012 by Farsight Security, Inc.
+ * Copyright (c) 2023 DomainTools LLC
+ * Copyright (c) 2009, 2012-2013, 2016 by Farsight Security, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

nmsg/strbuf.h

@@ -1,4 +1,5 @@
 /*
+ * Copyright (c) 2023 DomainTools LLC
  * Copyright (c) 2009-2015 by Farsight Security, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");

src/io.c

@@ -1,4 +1,5 @@
 /*
+ * Copyright (c) 2023 DomainTools LLC
  * Copyright (c) 2008-2019, 2021 by Farsight Security, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");

src/nmsgtool.c

@@ -1,4 +1,5 @@
 /*
+ * Copyright (c) 2023 DomainTools LLC
  * Copyright (c) 2008-2021 by Farsight Security, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");

src/nmsgtool.h

@@ -1,4 +1,5 @@
 /*
+ * Copyright (c) 2023 DomainTools LLC
  * Copyright (c) 2008-2019, 2021 by Farsight Security, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");