From 0886217537e89a3773cfd905e5d5bcfd34b27573 Mon Sep 17 00:00:00 2001 From: Chad Fraleigh Date: Thu, 15 Sep 2022 14:04:45 -0700 Subject: [PATCH 01/12] Only write database a second time after merge. Signed-off-by: Chad Fraleigh (cherry picked from commit b7045745745a4ab85bad2e4f144df171dcfc91ae) --- nlbwmon.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nlbwmon.c b/nlbwmon.c index 27eacba..7284ab8 100644 --- a/nlbwmon.c +++ b/nlbwmon.c @@ -96,10 +96,11 @@ static void save_persistent(uint32_t timestamp) fprintf(stderr, "Unable to load existing database: %s\n", strerror(-err)); } + else { + err = database_save(gdbh, opt.db.directory, timestamp, opt.db.compress); + } } - err = database_save(gdbh, opt.db.directory, timestamp, opt.db.compress); - if (err) { fprintf(stderr, "Unable to save database: %s\n", strerror(-err)); From acba29b7c2a53136bc01d9f81b36e7218e0e2a1b Mon Sep 17 00:00:00 2001 From: rjokl Date: Sun, 10 Nov 2024 14:06:00 +0100 Subject: [PATCH 02/12] IPv6 table formatting --- client.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/client.c b/client.c index 7e6edc9..bf55e3e 100644 --- a/client.c +++ b/client.c @@ -297,14 +297,14 @@ handle_show(void) printf("%c Fam ", columns[FAMILY]); if (columns[HOST]) { - printf(" %c Host ( MAC ) ", columns[HOST]); + printf(" %c Host ( MAC ) ", columns[HOST]); } else { if (columns[MAC]) printf(" %c MAC ", columns[MAC]); if (columns[IP]) - printf(" %c IP ", columns[IP]); + printf(" %c IP ", columns[IP]); } if (columns[LAYER7]) { @@ -328,7 +328,7 @@ handle_show(void) printf("IPv%d ", rec->family == AF_INET ? 4 : 6); if (columns[HOST]) { - printf("%15s (%02x:%02x:%02x) ", + printf("%39s (%02x:%02x:%02x) ", format_ipaddr(rec->family, &rec->src_addr), rec->src_mac.ea.ether_addr_octet[3], rec->src_mac.ea.ether_addr_octet[4], @@ -339,7 +339,7 @@ handle_show(void) printf("%17s ", format_macaddr(&rec->src_mac.ea)); if (columns[IP]) - printf("%15s ", format_ipaddr(rec->family, &rec->src_addr)); + printf("%39s ", format_ipaddr(rec->family, &rec->src_addr)); } if (columns[LAYER7]) { From f05a084db3cae9b6c614bf3b2b631c977b5c120e Mon Sep 17 00:00:00 2001 From: rjokl Date: Mon, 11 Nov 2024 17:38:16 +0100 Subject: [PATCH 03/12] call cleanup regularly --- database.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/database.c b/database.c index 7693673..3f2b443 100644 --- a/database.c +++ b/database.c @@ -618,6 +618,8 @@ database_archive(struct dbhandle *h) if (err) return err; + database_cleanup(); + /* lazily reset database, don't (re)alloc */ h->off = 0; h->db->entries = 0; From b36c722c623f1cb00e2cad7141b3d495654c4e95 Mon Sep 17 00:00:00 2001 From: rjokl Date: Sun, 10 Nov 2024 12:54:49 +0100 Subject: [PATCH 04/12] list up to generations with holes --- socket.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/socket.c b/socket.c index 9ea511e..3b2878c 100644 --- a/socket.c +++ b/socket.c @@ -129,6 +129,9 @@ handle_list(int sock, const char *arg) fprintf(stderr, "Corrupted database detected: %d (%s)\n", timestamp, strerror(-err)); + if (opt.db.generations && delta > -opt.db.generations) + continue; + break; } From 049003bb0fb601ab132ac53e029d659fe0f0100e Mon Sep 17 00:00:00 2001 From: rjokl Date: Sun, 10 Nov 2024 10:43:41 +0100 Subject: [PATCH 05/12] initial support for minute interval --- client.c | 35 +++++++++++++++++++++++++++++------ database.c | 6 ++++-- timing.c | 22 ++++++++++++++++++++++ timing.h | 1 + 4 files changed, 56 insertions(+), 8 deletions(-) diff --git a/client.c b/client.c index bf55e3e..0236418 100644 --- a/client.c +++ b/client.c @@ -92,6 +92,7 @@ static struct field fields[MAX] = { static struct { int timestamp; + bool minute; bool plain_numbers; int8_t group_by[1 + MAX]; int8_t order_by[1 + MAX]; @@ -217,7 +218,7 @@ recv_database(struct dbhandle **h) int i, len, err, ctrl_socket; struct database db; struct record rec; - char req[sizeof("dump YYYYMMDD\0")]; + char req[sizeof("dump -2147483648\0")]; ctrl_socket = usock(USOCK_UNIX, opt.socket, NULL); @@ -621,10 +622,19 @@ handle_list(void) sizeof(client_opt.timestamp), 0) <= 0) break; - printf("%04d-%02d-%02d\n", - client_opt.timestamp / 10000, - client_opt.timestamp % 10000 / 100, - client_opt.timestamp % 100); + if (client_opt.minute) { + time_t t = client_opt.timestamp * 60; + struct tm *timeinfo = localtime (&t); + char timestr[50]; + strftime(timestr, sizeof(timestr), "%F %T", timeinfo); + printf ("%d (%s)\n", client_opt.timestamp, timestr); + } + else { + printf("%04d-%02d-%02d\n", + client_opt.timestamp / 10000, + client_opt.timestamp % 10000 / 100, + client_opt.timestamp % 100); + } } close(ctrl_socket); @@ -676,7 +686,7 @@ client_main(int argc, char **argv) unsigned int year, month, day; char c, *p; - while ((optchr = getopt(argc, argv, "c:p:S:g:o:t:s::q::e::n")) > -1) { + while ((optchr = getopt(argc, argv, "c:p:S:g:o:t:s::q::e::nm")) > -1) { switch (optchr) { case 'S': opt.socket = optarg; @@ -748,6 +758,15 @@ client_main(int argc, char **argv) break; case 't': + + if (client_opt.minute) { + if (sscanf(optarg, "%u", &client_opt.timestamp) != 1) { + fprintf(stderr, "Unrecognized date '%s'\n", optarg); + return 1; + } + break; + + } if (sscanf(optarg, "%4u-%2u-%2u", &year, &month, &day) != 3) { fprintf(stderr, "Unrecognized date '%s'\n", optarg); return 1; @@ -760,6 +779,10 @@ client_main(int argc, char **argv) client_opt.plain_numbers = 1; break; + case 'm': + client_opt.minute = true; + break; + case 's': client_opt.separator = optarg ? *optarg : 0; break; diff --git a/database.c b/database.c index 3f2b443..8f64089 100644 --- a/database.c +++ b/database.c @@ -563,7 +563,7 @@ database_load(struct dbhandle *h, const char *path, uint32_t timestamp) int database_cleanup(void) { - uint32_t timestamp, num; + uint32_t timestamp, num, safe_low; struct dirent *entry; char *e, path[256]; DIR *d; @@ -579,6 +579,8 @@ database_cleanup(void) errno = 0; timestamp = interval_timestamp(&opt.archive_interval, -opt.db.generations); + safe_low = opt.archive_interval.type == MINUTE ? 1704067200 / 60 : 20000101; + while ((entry = readdir(d)) != NULL) { if (entry->d_type != DT_REG) continue; @@ -591,7 +593,7 @@ database_cleanup(void) if (strcmp(e, ".db") != 0 && strcmp(e, ".db.gz") != 0) continue; - if (num < 20000101 || num > timestamp) + if (num < safe_low || num > timestamp) continue; snprintf(path, sizeof(path), "%s/%u%s", opt.db.directory, num, e); diff --git a/timing.c b/timing.c index ca86067..ed0cdc5 100644 --- a/timing.c +++ b/timing.c @@ -21,6 +21,7 @@ #include #include #include +#include #include "timing.h" @@ -133,6 +134,14 @@ interval_timestamp_monthly(const struct interval *intv, int offset) date); } +static int +interval_timestamp_minute(const struct interval *intv, int offset) +{ + time_t now = time(NULL); + uint32_t ts = (uint32_t) (now / 60); + return ts + offset; +} + static int interval_timestamp_fixed(const struct interval *intv, int offset) { @@ -191,6 +200,12 @@ interval_pton(const char *spec, struct interval *intv) return 0; } + if (!strcmp(spec, "m")) { + intv->type = MINUTE; + intv->value = 0; + intv->base = 0; + return 0; + } value = strtol(spec, &e, 10); @@ -223,6 +238,10 @@ interval_ntop(const struct interval *intv, char *spec, size_t len) case MONTHLY: snprintf(spec, len, "%d", (int32_t)be32toh(intv->value)); break; + + case MINUTE: + snprintf(spec, len, "m"); + break; } } @@ -236,6 +255,9 @@ interval_timestamp(const struct interval *intv, int offset) case MONTHLY: return interval_timestamp_monthly(intv, offset); + + case MINUTE: + return interval_timestamp_minute(intv, offset); } return -EINVAL; diff --git a/timing.h b/timing.h index 2887db0..f94f1d7 100644 --- a/timing.h +++ b/timing.h @@ -25,6 +25,7 @@ enum interval_type { MONTHLY = 1, FIXED = 2, + MINUTE = 3 }; struct interval { From 0e0b0991e05dad7b959fcf44f8fc03b4219c0265 Mon Sep 17 00:00:00 2001 From: rjokl Date: Thu, 14 Nov 2024 19:56:58 +0100 Subject: [PATCH 06/12] interval type on client recv/detect --- client.c | 34 ++++++++++++++-------------------- socket.c | 3 +++ 2 files changed, 17 insertions(+), 20 deletions(-) diff --git a/client.c b/client.c index 0236418..c5d3e7a 100644 --- a/client.c +++ b/client.c @@ -92,7 +92,6 @@ static struct field fields[MAX] = { static struct { int timestamp; - bool minute; bool plain_numbers; int8_t group_by[1 + MAX]; int8_t order_by[1 + MAX]; @@ -617,12 +616,18 @@ handle_list(void) return -errno; } + uint8_t interval_type; + if (recv(ctrl_socket, &interval_type, sizeof(interval_type), 0) <= 0) { + close(ctrl_socket); + return 0; + } + while (true) { if (recv(ctrl_socket, &client_opt.timestamp, sizeof(client_opt.timestamp), 0) <= 0) break; - if (client_opt.minute) { + if (interval_type == MINUTE) { time_t t = client_opt.timestamp * 60; struct tm *timeinfo = localtime (&t); char timestr[50]; @@ -686,7 +691,7 @@ client_main(int argc, char **argv) unsigned int year, month, day; char c, *p; - while ((optchr = getopt(argc, argv, "c:p:S:g:o:t:s::q::e::nm")) > -1) { + while ((optchr = getopt(argc, argv, "c:p:S:g:o:t:s::q::e::n")) > -1) { switch (optchr) { case 'S': opt.socket = optarg; @@ -758,31 +763,20 @@ client_main(int argc, char **argv) break; case 't': - - if (client_opt.minute) { - if (sscanf(optarg, "%u", &client_opt.timestamp) != 1) { - fprintf(stderr, "Unrecognized date '%s'\n", optarg); - return 1; - } + if (sscanf(optarg, "%4u-%2u-%2u", &year, &month, &day) == 3) { + client_opt.timestamp = year * 10000 + month * 100 + day; break; - } - if (sscanf(optarg, "%4u-%2u-%2u", &year, &month, &day) != 3) { - fprintf(stderr, "Unrecognized date '%s'\n", optarg); - return 1; + else if (sscanf(optarg, "%u", &client_opt.timestamp) == 1) { + break; } - - client_opt.timestamp = year * 10000 + month * 100 + day; - break; + fprintf(stderr, "Unrecognized date '%s'\n", optarg); + return 1; case 'n': client_opt.plain_numbers = 1; break; - case 'm': - client_opt.minute = true; - break; - case 's': client_opt.separator = optarg ? *optarg : 0; break; diff --git a/socket.c b/socket.c index 3b2878c..eb38409 100644 --- a/socket.c +++ b/socket.c @@ -120,6 +120,9 @@ handle_list(int sock, const char *arg) int delta = 0; uint32_t timestamp; + if (send(sock, &opt.archive_interval.type, sizeof(opt.archive_interval.type), 0) != sizeof(opt.archive_interval.type)) + return -errno; + while (true) { timestamp = interval_timestamp(&opt.archive_interval, delta--); err = database_load(NULL, opt.db.directory, timestamp); From 8f617e629ca25abadea89a48ece727690bf5e826 Mon Sep 17 00:00:00 2001 From: rjokl Date: Sat, 16 Nov 2024 09:55:08 +0100 Subject: [PATCH 07/12] client recv merge --- client.c | 66 +++++++++++++++++++++++++++++++++----------------------- socket.c | 27 +++++++++++++++++------ 2 files changed, 60 insertions(+), 33 deletions(-) diff --git a/client.c b/client.c index c5d3e7a..c01252d 100644 --- a/client.c +++ b/client.c @@ -92,6 +92,7 @@ static struct field fields[MAX] = { static struct { int timestamp; + unsigned int generations; bool plain_numbers; int8_t group_by[1 + MAX]; int8_t order_by[1 + MAX]; @@ -102,6 +103,7 @@ static struct { .separator = '\t', .escape = '"', .quote = '"', + .generations = 1 }; struct command { @@ -217,49 +219,52 @@ recv_database(struct dbhandle **h) int i, len, err, ctrl_socket; struct database db; struct record rec; - char req[sizeof("dump -2147483648\0")]; + char req[100]; - ctrl_socket = usock(USOCK_UNIX, opt.socket, NULL); + *h = database_mem(cmp_fn, client_opt.group_by); - if (!ctrl_socket) - return -errno; + if (!*h) { + return -ENOMEM; + } - len = snprintf(req, sizeof(req), "dump %d", client_opt.timestamp); + for (int g = 0; g < client_opt.generations; g++) { - if (send(ctrl_socket, req, len, 0) != len) { - close(ctrl_socket); - return -errno; - } + ctrl_socket = usock(USOCK_UNIX, opt.socket, NULL); - if (recv(ctrl_socket, &db, sizeof(db), 0) != sizeof(db)) { - close(ctrl_socket); - return -ENODATA; - } + if (!ctrl_socket) + return -errno; - *h = database_mem(cmp_fn, client_opt.group_by); + len = snprintf(req, sizeof(req), "dump %d-%d", client_opt.timestamp, g); - if (!*h) { - close(ctrl_socket); - return -ENOMEM; - } + if (send(ctrl_socket, req, len, 0) != len) { + close(ctrl_socket); + return -errno; + } - for (i = 0; i < db_entries(&db); i++) { - if (recv(ctrl_socket, &rec, db_recsize, 0) != db_recsize) { + if (recv(ctrl_socket, &db, sizeof(db), 0) != sizeof(db)) { close(ctrl_socket); return -ENODATA; } - err = database_insert(*h, &rec); + for (i = 0; i < db_entries(&db); i++) { + if (recv(ctrl_socket, &rec, db_recsize, 0) != db_recsize) { + close(ctrl_socket); + return -ENODATA; + } - if (err != 0) { - close(ctrl_socket); - return err; + err = database_insert(*h, &rec); + + if (err != 0) { + close(ctrl_socket); + return err; + } } + + close(ctrl_socket); } database_reorder(*h, sort_fn, client_opt.order_by); - close(ctrl_socket); return 0; } @@ -631,7 +636,7 @@ handle_list(void) time_t t = client_opt.timestamp * 60; struct tm *timeinfo = localtime (&t); char timestr[50]; - strftime(timestr, sizeof(timestr), "%F %T", timeinfo); + strftime(timestr, sizeof(timestr), "%F %T %Z", timeinfo); printf ("%d (%s)\n", client_opt.timestamp, timestr); } else { @@ -691,7 +696,7 @@ client_main(int argc, char **argv) unsigned int year, month, day; char c, *p; - while ((optchr = getopt(argc, argv, "c:p:S:g:o:t:s::q::e::n")) > -1) { + while ((optchr = getopt(argc, argv, "c:p:S:g:o:t:s::q::e::nG:")) > -1) { switch (optchr) { case 'S': opt.socket = optarg; @@ -773,6 +778,13 @@ client_main(int argc, char **argv) fprintf(stderr, "Unrecognized date '%s'\n", optarg); return 1; + case 'G': + if (sscanf(optarg, "%2u", &client_opt.generations) != 1) { + fprintf(stderr, "Unrecognized generations '%s'\n", optarg); + return 1; + } + break; + case 'n': client_opt.plain_numbers = 1; break; diff --git a/socket.c b/socket.c index eb38409..105afac 100644 --- a/socket.c +++ b/socket.c @@ -68,20 +68,35 @@ handle_dump(int sock, const char *arg) { struct dbhandle *h; struct record *rec = NULL; - int err = 0, timestamp = 0; - char *e; + int err = 0, timestamp = 0, g = 0; if (arg) { - timestamp = strtoul(arg, &e, 10); - - if (arg == e || *e) + if (sscanf(arg, "%d-%d", ×tamp, &g) != 2) return -EINVAL; } - if (timestamp == 0) { + if (timestamp == 0 && g == 0) { h = gdbh; } else { + if (timestamp == 0) { + timestamp = interval_timestamp(&opt.archive_interval, -g); + } + else { + int delta = 0; + while (true) { + if (timestamp == interval_timestamp(&opt.archive_interval, delta)) { + break; + } + delta--; + if (opt.db.generations && delta > -opt.db.generations) + continue; + + return -EINVAL; + } + timestamp = interval_timestamp(&opt.archive_interval, delta - g); + } + h = database_init(&opt.archive_interval, false, 0); if (!h) { From 1e9ae7c2c5f8e6d42721c99087e8bd2f8b8c94fc Mon Sep 17 00:00:00 2001 From: rjokl Date: Sun, 17 Nov 2024 10:45:38 +0100 Subject: [PATCH 08/12] restore database timestamp --- database.c | 1 + 1 file changed, 1 insertion(+) diff --git a/database.c b/database.c index 8f64089..a1d59ca 100644 --- a/database.c +++ b/database.c @@ -454,6 +454,7 @@ database_restore_gzip(struct dbhandle *h, const char *path, uint32_t timestamp) if (h) { h->pristine = false; + h->db->timestamp = htobe32(timestamp); for (i = 0; i < entries; i++) { if (gzread(gz, &rec, db_recsize) != db_recsize) { From 0fb6a900d0c3eeb49e010761e75481799e29ce9c Mon Sep 17 00:00:00 2001 From: rjokl Date: Sun, 17 Nov 2024 19:05:00 +0100 Subject: [PATCH 09/12] handle send EAGAIN --- socket.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/socket.c b/socket.c index 105afac..ae02972 100644 --- a/socket.c +++ b/socket.c @@ -153,7 +153,7 @@ handle_list(int sock, const char *arg) break; } - if (send(sock, ×tamp, sizeof(timestamp), 0) != sizeof(timestamp)) + if (send_data(sock, ×tamp, sizeof(timestamp)) != sizeof(timestamp)) return -errno; } From abb6dd1a6718b2b50fb8f348efa60fd2942c9c4b Mon Sep 17 00:00:00 2001 From: rjokl Date: Mon, 18 Nov 2024 18:02:18 +0100 Subject: [PATCH 10/12] added value to minute interval --- client.c | 6 +++--- socket.c | 2 +- timing.c | 10 +++++----- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/client.c b/client.c index c01252d..0b7b8ea 100644 --- a/client.c +++ b/client.c @@ -621,8 +621,8 @@ handle_list(void) return -errno; } - uint8_t interval_type; - if (recv(ctrl_socket, &interval_type, sizeof(interval_type), 0) <= 0) { + struct interval interval; + if (recv(ctrl_socket, &interval, sizeof(interval), 0) <= 0) { close(ctrl_socket); return 0; } @@ -632,7 +632,7 @@ handle_list(void) sizeof(client_opt.timestamp), 0) <= 0) break; - if (interval_type == MINUTE) { + if (interval.type == MINUTE) { time_t t = client_opt.timestamp * 60; struct tm *timeinfo = localtime (&t); char timestr[50]; diff --git a/socket.c b/socket.c index ae02972..86b8ecd 100644 --- a/socket.c +++ b/socket.c @@ -135,7 +135,7 @@ handle_list(int sock, const char *arg) int delta = 0; uint32_t timestamp; - if (send(sock, &opt.archive_interval.type, sizeof(opt.archive_interval.type), 0) != sizeof(opt.archive_interval.type)) + if (send(sock, &opt.archive_interval, sizeof(opt.archive_interval), 0) != sizeof(opt.archive_interval)) return -errno; while (true) { diff --git a/timing.c b/timing.c index ed0cdc5..3719691 100644 --- a/timing.c +++ b/timing.c @@ -138,8 +138,8 @@ static int interval_timestamp_minute(const struct interval *intv, int offset) { time_t now = time(NULL); - uint32_t ts = (uint32_t) (now / 60); - return ts + offset; + uint32_t ts = (uint32_t) (intv->value * (now / (60 * intv->value))); + return ts + (offset * intv->value); } static int @@ -200,9 +200,9 @@ interval_pton(const char *spec, struct interval *intv) return 0; } - if (!strcmp(spec, "m")) { + if (sscanf(spec, "%um", &value) == 1) { intv->type = MINUTE; - intv->value = 0; + intv->value = value; intv->base = 0; return 0; } @@ -240,7 +240,7 @@ interval_ntop(const struct interval *intv, char *spec, size_t len) break; case MINUTE: - snprintf(spec, len, "m"); + snprintf(spec, len, "%um", (int32_t)be32toh(intv->value)); break; } } From 78390fd6ea415165134896063df35f94d8ea9d12 Mon Sep 17 00:00:00 2001 From: rjokl Date: Tue, 19 Nov 2024 18:33:57 +0100 Subject: [PATCH 11/12] documentation --- README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ced1bab..7e3bb64 100644 --- a/README.md +++ b/README.md @@ -44,13 +44,14 @@ Each time the conntrack entries are polled, their counters are reset (zero-on-re
Accounting period interval. May be either in the format YYYY-MM-DD/NN, to start a new accounting period exactly every NN days, beginning at the given date, or a number specifiying the day of month at which to -start the next accounting period. For example:
+start the next accounting period, or in the format MMm for MM minutes. For example: ``` 2017-01-17/14 # every 14 days, starting Jan 17, 2017 -2 # second to the last day of the month, e.g. 30th in March 1 # first day of the month (default) +60m # every hour ```
@@ -87,7 +88,7 @@ storage requirements.
-o col[,col]
Order output by the specified column. Prefix column with a - to invert order.
-
-t YYYY-MM-DD
+
-t YYYY-MM-DD|timestamp
Read data from the specified database, instead of the active database. Use the list command to view available databases.
-n
@@ -101,6 +102,9 @@ storage requirements.
-e char
Specify the escape character when using CSV format. If no argument is provided, an empty string is assumed. Currently only applies to CSV format.
+ +
-G count
+
Number of database generations to retrieve. Data from databases are summed. Note: connection is counted in each database, so connection spanning two intervals is counted twice.
From d3fc96974a7bc4c427eb11cd9dfec96f4e91e7c0 Mon Sep 17 00:00:00 2001 From: rjokl Date: Sun, 22 Dec 2024 13:49:35 +0100 Subject: [PATCH 12/12] documentation - typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7e3bb64..d598f3c 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,7 @@ storage requirements.
-c command
Specify a command. Current commands are: show, json, csv, list, commit. See below for more information about commands.
-
-p /path/to/procol-database
+
-p /path/to/protocol-database
Protocol description file, used to distinguish traffic streams by IP protocol number and port.
-g col[,col]