Merge pull request #165 from farsightsec/next

nmsg 1.3.0

Makefile.am

@@ -19,9 +19,7 @@ AM_CFLAGS = \
 	$(libwdns_CFLAGS) \
 	$(libzmq_CFLAGS) \
 	$(librdkafka_CFLAGS) \
-	$(json_c_CFLAGS) \
-	$(libmicrohttpd_CFLAGS) \
-	$(libprom_CFLAGS)
+	$(json_c_CFLAGS)
 AM_LDFLAGS =
 
 EXTRA_DIST += ChangeLog
@@ -109,6 +107,8 @@ nobase_include_HEADERS = \
 	nmsg/rate.h \
 	nmsg/res.h \
 	nmsg/sock.h \
+	nmsg/statsmod.h \
+	nmsg/statsmod_plugin.h \
 	nmsg/strbuf.h \
 	nmsg/timespec.h \
 	nmsg/vendors.h \
@@ -125,7 +125,7 @@ nobase_nodist_include_HEADERS = \
 
 lib_LTLIBRARIES = nmsg/libnmsg.la
 
-VERSION_INFO = 10:0:2
+VERSION_INFO = 11:0:3
 
 nmsg_libnmsg_la_LDFLAGS = \
 	$(AM_LDFLAGS) \
@@ -137,6 +137,7 @@ LIBNMSG_LIB_DEPS = \
 	$(libzmq_LIBS) \
 	$(librdkafka_LIBS) \
 	$(json_c_LIBS)
+
 nmsg_libnmsg_la_LIBADD = $(LIBNMSG_LIB_DEPS)
 LIBNMSG_LIB_MODULES = \
 	libmy/crc32c.c libmy/crc32c-slicing.c libmy/crc32c-sse42.c \
@@ -149,6 +150,7 @@ LIBNMSG_LIB_MODULES = \
 	nmsg/brate.c \
 	nmsg/buf.c \
 	nmsg/chalias.c \
+	nmsg/config_file.c \
 	nmsg/container.c \
 	nmsg/dlmod.c \
 	nmsg/fltmod.c \
@@ -176,6 +178,7 @@ LIBNMSG_LIB_MODULES = \
 	nmsg/rate.c \
 	nmsg/res.c \
 	nmsg/sock.c \
+	nmsg/statsmod.c \
 	nmsg/strbuf.c \
 	nmsg/timespec.c \
 	nmsg/version.c \
@@ -206,11 +209,13 @@ nmsg_libnmsg_la_SOURCES = \
 	nmsg/private.h \
 	nmsg/version.h \
 	nmsg/kafkaio.h \
+	nmsg/config_file.h \
 	nmsg/msgmod/transparent.h \
 	$(LIBNMSG_LIB_MODULES)
 nodist_nmsg_libnmsg_la_SOURCES = \
 	nmsg/nmsg.pb-c.c \
 	nmsg/nmsg.pb-c.h
+
 BUILT_SOURCES += $(nodist_nmsg_libnmsg_la_SOURCES)
 CLEANFILES += $(nodist_nmsg_libnmsg_la_SOURCES)
 
@@ -347,9 +352,7 @@ src_nmsgtool_LDADD = \
 	nmsg/libnmsg.la \
 	$(libpcap_LIBS) \
 	$(libzmq_LIBS) \
-	$(librdkafka_LIBS) \
-	$(libmicrohttpd_LIBS) \
-	$(libprom_LIBS)
+	$(librdkafka_LIBS)
 src_nmsgtool_SOURCES = \
 	libmy/argv.c \
 	libmy/argv.h \
@@ -365,10 +368,6 @@ src_nmsgtool_SOURCES = \
 	src/rwfile.c \
 	src/unescape.c
 
-if USE_DT_PROM
-src_nmsgtool_SOURCES += src/dt_prom.c \
-                        src/dt_prom.h
-endif
 #
 ##
 ### Tests
@@ -413,6 +412,15 @@ EXTRA_DIST += tests/nmsg-http-tests/test2-http-empty-request.pres
 EXTRA_DIST += tests/nmsg-http-tests/test4-http-no-request.json
 EXTRA_DIST += tests/nmsg-http-tests/test4-http-no-request.nmsg
 EXTRA_DIST += tests/nmsg-http-tests/test4-http-no-request.pres
+EXTRA_DIST += tests/private-tests/config_file_empty.cfg
+EXTRA_DIST += tests/private-tests/config_file_invalid_line_1.cfg
+EXTRA_DIST += tests/private-tests/config_file_invalid_line_2.cfg
+EXTRA_DIST += tests/private-tests/config_file_invalid_line_3.cfg
+EXTRA_DIST += tests/private-tests/config_file_invalid_section_1.cfg
+EXTRA_DIST += tests/private-tests/config_file_invalid_section_2.cfg
+EXTRA_DIST += tests/private-tests/config_file_invalid_section_3.cfg
+EXTRA_DIST += tests/private-tests/config_file_valid.cfg
+EXTRA_DIST += tests/private-tests/config_file_valid_no_default.cfg
 
 noinst_PROGRAMS += libmy/crc32c_test
 libmy_crc32c_test_CFLAGS = $(AM_CFLAGS)
@@ -454,6 +462,7 @@ tests_test_parse_SOURCES = tests/test-parse.c
 TESTS += tests/test-private
 check_PROGRAMS += tests/test-private
 tests_test_private_LDFLAGS = -rdynamic
+tests_test_private_CPPFLAGS = -DSRCDIR="\"$(abs_srcdir)\"" $(AM_CPPFLAGS)
 PRIVATE_TEST_MODULES = $(LIBNMSG_LIB_MODULES:.c=.o)
 tests_test_private_LDADD = \
 	$(PRIVATE_TEST_MODULES) \

README.md

@@ -25,10 +25,6 @@ nmsg has the following external dependencies:
 
 * [zlib](http://www.zlib.net/)
 
-* prometheus integration (optional)
-  * [libprometheus](https://github.com/digitalocean/prometheus-client-c)
-  * [libmicrohttpd](https://www.gnu.org/software/libmicrohttpd/)
-
 On Debian systems, the following packages should be installed, if available:
 
     pkg-config libpcap0.8-dev libprotobuf-c-dev protobuf-c-compiler libzmq3-dev librdkafka-dev libjson-c-dev zlib1g-dev

configure.ac

@@ -1,7 +1,7 @@
 AC_PREREQ(2.64)
 
 m4_define(nmsg_major_version, 1)
-m4_define(nmsg_minor_version, 2)
+m4_define(nmsg_minor_version, 3)
 m4_define(nmsg_patchlevel_version, 0)
 m4_define(nmsg_version,
 	  nmsg_major_version.nmsg_minor_version.nmsg_patchlevel_version)
@@ -177,18 +177,6 @@ else
     use_json_c="false"
 fi
 
-AC_ARG_WITH([prometheus], AS_HELP_STRING([--with-prometheus], [Enable prometheus metrics]))
-if test "x$with_prometheus" == "xyes"; then
-    PKG_CHECK_MODULES([libmicrohttpd], [libmicrohttpd >= 0.9.0])
-    PKG_CHECK_MODULES([libprom], [libprom >= 0.1.0])
-    AC_DEFINE([HAVE_PROMETHEUS], [1], [Define to 1 if prometheus metrics are enabled.])
-    USE_PROMETHEUS="yes"
-    AM_CONDITIONAL([USE_DT_PROM], [true])
-else
-    USE_PROMETHEUS="no"
-    AM_CONDITIONAL([USE_DT_PROM], [false])
-fi
-
 AC_CHECK_HEADER([zlib.h], [], [ AC_MSG_ERROR([required header file not found]) ])
 AC_CHECK_LIB([z], [deflate], [], [ AC_MSG_ERROR([required library not found]) ])
 
@@ -250,7 +238,6 @@ AC_MSG_RESULT([
         libzmq support:         ${use_libzmq}
         librdkafka support:     ${use_librdkafka}
         json-c support:         ${use_json_c}
-        prometheus support:     ${USE_PROMETHEUS}
 
         building html docs:     ${DOC_HTML_MSG}
         building manpage docs:  ${DOC_MAN_MSG}

debian/changelog

@@ -1,3 +1,15 @@
+nmsg (1.3.0-1) debian-fsi; urgency=medium
+
+  * Added external configuration for Kafka IO.
+  * Stop IO when output fails.
+  * Add support and API for statistics module.
+  * Remove prometheus integration.
+  * Add NMSG_STATSMOD_CONFIG environment variable.
+  * Bug fixes.
+  * Documentation fixes.
+
+ -- Farsight Security Inc <[email protected]>  Tue, 08 Oct 2024 12:09:34 -0400
+
 nmsg (1.2.0-1) debian-fsi; urgency=medium
 
   * Add support for Kafka i/o in JSON and binary forms.

debian/libnmsg8.symbols

@@ -172,6 +172,10 @@ libnmsg.so.8 libnmsg8 #MINVER#
  nmsg_set_debug@Base 0.5.0
  nmsg_sock_parse@Base 0.6.11
  nmsg_sock_parse_sockspec@Base 0.6.11
+ nmsg_statsmod_add_io@Base 1.3.0
+ nmsg_statsmod_destroy@Base 1.3.0
+ nmsg_statsmod_init@Base 1.3.0
+ nmsg_statsmod_remove_io@Base 1.3.0
  nmsg_strbuf_append@Base 0.5.0
  nmsg_strbuf_append_str@Base 1.1.0
  nmsg_strbuf_append_str_json@Base 1.1.0

doc/docbook/nmsgtool.1

@@ -2,12 +2,12 @@
 .\"     Title: nmsgtool
 .\"    Author: [FIXME: author] [see http://www.docbook.org/tdg5/en/html/author]
 .\" Generator: DocBook XSL Stylesheets v1.79.2 <http://docbook.sf.net/>
-.\"      Date: 06/28/2024
+.\"      Date: 10/02/2024
 .\"    Manual:  
 .\"    Source:    
 .\"  Language: English
 .\"
-.TH "NMSGTOOL" "1" "06/28/2024" "" ""
+.TH "NMSGTOOL" "1" "10/02/2024" "" ""
 .\" -----------------------------------------------------------------
 .\" * Define some portability stuff
 .\" -----------------------------------------------------------------
@@ -224,7 +224,9 @@ is set to the empty string \*(Aq\*(Aq, then no command is executed and only file
 .PP
 \fB\-\-kafkakey\fR \fIfieldname\fR
 .RS 4
-Use the value of the named NMSG message field in its canonical representation as the key for payloads in the JSON format that are sent to Kafka\&. Can also be set via NMSG_KAFKA_KEY environment variable\&.
+Use the value of the named NMSG message field in its canonical representation as the key for payloads in the JSON format that are sent to Kafka\&. Can also be set via
+\fBNMSG_KAFKA_KEY\fR
+environment variable\&.
 .RE
 .PP
 \fB\-\-readkafka\fR \fIkafka\fR
@@ -234,7 +236,9 @@ Read NMSG payloads in either binary or JSON format from a Kafka endpoint\&. The
 has format
 \fBproto:topic[#partition|%group_id]@broker[:port][,offset]\fR\&. Either a partition number or a consumer group ID may be optionally supplied\&. Also optional is an offset consisting of either a numerical value or the string \*(Aqoldest\*(Aq or \*(Aqnewest\*(Aq in order to start retrieval at the oldest/newest messages in the Kafka topic\&. An example of a possible
 \fIkafka\fR
-endpoint is "nmsg:ch202#0@kafka\&.example\&.com:9092,3000" to indicate that nmsgtool shall read nmsg containers from topic "ch202" on partition 0 at offset 3000 from the Kafka broker at kafka\&.example\&.com, port 9092\&.
+endpoint is "nmsg:ch202#0@kafka\&.example\&.com:9092,3000" to indicate that nmsgtool shall read nmsg containers from topic "ch202" on partition 0 at offset 3000 from the Kafka broker at kafka\&.example\&.com, port 9092\&. Configuration for Kafka can be supplied through the
+\fBNMSG_KAFKA_CONFIG\fR
+environment variable\&. This variable accepts either key=value pairs separated by colons, or a path to a configuration file in INI format\&.
 .RE
 .PP
 \fB\-b\fR \fIfilter\fR, \fB\-\-bpf\fR \fIfilter\fR
@@ -286,9 +290,34 @@ may be used to specify the policy action to take if all filters in the filter ch
 is specified, any messages which are declined by the filter chain will be silently discarded\&.
 .RE
 .PP
-\fB\-\-promport\fR \fIport\fR
+\fB\-M\fR \fIdso\fR[,\fIparam\fR], \fB\-\-statsmod\fR \fIdso\fR[,\fIparam\fR]
 .RS 4
-Deliver counted statistics, such as the total number of payloads sent or received, to Prometheus on the specified port\&.
+Export nmsg IO statistics through an external statistics module\&. The
+\fIdso\fR
+specified may either be a short, human friendly name (like
+\fIsample\fR) which will be expanded into an absolute filename in the default system\-wide libnmsg module path, or it may be a name beginning with
+\fI/\fR
+or
+\fI\&.\fR
+(like
+\fI/usr/lib/nmsg/nmsg_stats1_sample\&.so\fR), in which case the
+\fIdso\fR
+value will be treated as an absolute or relative path name\&.
+.sp
+Statistics modules may take a module\-defined parameter string
+\fIparam\fR\&. The
+\fIdso\fR
+value may be followed by a comma, in which case everything after the comma is treated as the module parameter
+\fIparam\fR
+and passed to the module\*(Aqs initialization function\&. Some modules may require a
+\fIparam\fR
+value or may perform additional validation on the parameter\&. If the module fails to initialize,
+\fBnmsgtool\fR
+will exit with an error message\&.
+.sp
+This option can be specified more than once, and may also be provided using the
+\fBNMSG_STATSMOD_CONFIG\fR
+environment variable\&.
 .RE
 .PP
 \fB\-r\fR \fIfile\fR, \fB\-\-readnmsg\fR \fIfile\fR
@@ -457,7 +486,9 @@ Write NMSG payloads in either binary or JSON format to a Kafka endpoint\&. The a
 has format
 \fBproto:topic[#partition|%group_id]@broker[:port]\fR\&. Either a partition number or a consumer group ID may be optionally supplied\&. An example of a possible
 \fIkafka\fR
-endpoint is "nmsg:ch202#0@kafka\&.example\&.com:9092" to indicate that nmsgtool shall write nmsg containers to topic "ch202" on partition 0 to Kafka broker kafka\&.example\&.com, port 9092\&. Note that nmsgtool ignores offsets for Kafka producers\&.
+endpoint is "nmsg:ch202#0@kafka\&.example\&.com:9092" to indicate that nmsgtool shall write nmsg containers to topic "ch202" on partition 0 to Kafka broker kafka\&.example\&.com, port 9092\&. Note that nmsgtool ignores offsets for Kafka producers\&. Configuration for Kafka can be supplied through the
+\fBNMSG_KAFKA_CONFIG\fR
+environment variable\&. This variable accepts either key=value pairs separated by colons, or a path to a configuration file in INI format\&.
 .RE
 .PP
 \fB\-o\fR \fIfile\fR, \fB\-\-writepres\fR \fIfile\fR

doc/docbook/nmsgtool.docbook

@@ -228,7 +228,7 @@
         <listitem>
         <para>
         Use the value of the named NMSG message field in its canonical representation as the key for payloads
-        in the JSON format that are sent to Kafka. Can also be set via NMSG_KAFKA_KEY environment variable.
+        in the JSON format that are sent to Kafka. Can also be set via <envar>NMSG_KAFKA_KEY</envar> environment variable.
         </para>
         </listitem>
       </varlistentry>
@@ -247,6 +247,8 @@
         "nmsg:ch202#[email protected]:9092,3000" to indicate that nmsgtool shall read nmsg
         containers from topic "ch202" on partition 0 at offset 3000 from the Kafka broker at
         kafka.example.com, port 9092.
+        Configuration for Kafka can be supplied through the <envar>NMSG_KAFKA_CONFIG</envar> environment variable.
+        This variable accepts either key=value pairs separated by colons, or a path to a configuration file in INI format.
         </para>
         </listitem>
       </varlistentry>
@@ -312,10 +314,32 @@
       </varlistentry>
 
       <varlistentry>
-        <term><option>--promport</option> <replaceable>port</replaceable></term>
+        <term><option>-M</option> <replaceable>dso</replaceable><optional>,<replaceable>param</replaceable></optional></term>
+        <term><option>--statsmod</option> <replaceable>dso</replaceable><optional>,<replaceable>param</replaceable></optional></term>
         <listitem>
-          <para>Deliver counted statistics, such as the total number of payloads sent
-          or received, to Prometheus on the specified port.</para>
+          <para>Export nmsg IO statistics through an external statistics module.
+          The <replaceable>dso</replaceable> specified may either be a
+          short, human friendly name (like <replaceable>sample</replaceable>)
+          which will be expanded into an absolute filename in the default
+          system-wide libnmsg module path, or it may be a name beginning with
+          <replaceable>/</replaceable> or <replaceable>.</replaceable> (like
+          <replaceable>/usr/lib/nmsg/nmsg_stats1_sample.so</replaceable>), in
+          which case the <replaceable>dso</replaceable> value will be treated as
+          an absolute or relative path name.</para>
+
+          <para>Statistics modules may take a module-defined parameter string
+          <replaceable>param</replaceable>. The <replaceable>dso</replaceable>
+          value may be followed by a comma, in which case everything after the
+          comma is treated as the module parameter
+          <replaceable>param</replaceable> and passed to the module's
+          initialization function. Some modules may require a
+          <replaceable>param</replaceable> value or may perform additional
+          validation on the parameter. If the module fails to initialize,
+          <command>nmsgtool</command> will exit with an error message.</para>
+
+          <para>This option can be specified more than once, and may also be
+          provided using the <envar>NMSG_STATSMOD_CONFIG</envar> environment
+          variable.</para>
         </listitem>
       </varlistentry>
 
@@ -480,6 +504,8 @@
         nmsg containers to topic "ch202" on partition 0 to Kafka
         broker kafka.example.com, port 9092.
         Note that nmsgtool ignores offsets for Kafka producers.
+        Configuration for Kafka can be supplied through the <envar>NMSG_KAFKA_CONFIG</envar> environment variable.
+        This variable accepts either key=value pairs separated by colons, or a path to a configuration file in INI format.
         </para>
         </listitem>
       </varlistentry>

nmsg.spec

@@ -1,5 +1,5 @@
 Name:           nmsg
-Version:        1.2.0
+Version:        1.3.0
 Release:        1%{?dist}
 Summary:	network message encapsulation library
 

nmsg/base/ncap.c

@@ -218,6 +218,7 @@ ncap_msg_load(nmsg_message_t m, void **msg_clos) {
 				p->has_dstport = true;
 				p->dstport = ncap->lint1;
 			}
+			break;
 		case NMSG__BASE__NCAP__NCAP_LEGACY_TYPE__ICMP:
 			break;
 		default: