feat(compose): update docker-compose (#361)

compose/kcg/README.md

@@ -26,3 +26,6 @@ using `MATERIALIZED TABLES` and `VIEWS` defined in a [schema file](./clickhouse/
 
 You can visualize the data in Grafana at http://localhost:3000 (credentials: admin/admin) with the
 pre-made dashboards.
+
+Note: if you are using Colima as the engine, it does not support UDP port forwarding. Flows won't be collected.
+It is possible to run GoFlow2 locally and feed into kafka if you add an `/etc/hosts` entry `127.0.0.1 kafka`.

compose/kcg/clickhouse/create.sh

@@ -41,6 +41,7 @@ clickhouse client -n <<-EOSQL
     ) ENGINE = Kafka()
     SETTINGS
         kafka_broker_list = 'kafka:9092',
+        kafka_num_consumers = 1,
         kafka_topic_list = 'flows',
         kafka_group_name = 'clickhouse',
         kafka_format = 'Protobuf',

compose/kcg/clickhouse/flow.proto

@@ -0,0 +1,123 @@
+syntax = "proto3";
+package flowpb;
+option go_package = "github.com/netsampler/goflow2/pb;flowpb";
+
+message FlowMessage {
+
+  enum FlowType {
+    FLOWUNKNOWN = 0;
+    SFLOW_5 = 1;
+    NETFLOW_V5 = 2;
+    NETFLOW_V9 = 3;
+    IPFIX = 4;
+  }
+  FlowType type = 1;
+
+  uint64 time_received_ns = 110;
+  uint32 sequence_num = 4;
+  uint64 sampling_rate = 3;
+
+  //uint32 flow_direction = 42;
+
+  // Sampler information
+  bytes sampler_address = 11;
+
+  // Found inside packet
+  uint64 time_flow_start_ns = 111;
+  uint64 time_flow_end_ns = 112;
+
+  // Size of the sampled packet
+  uint64 bytes = 9;
+  uint64 packets = 10;
+
+  // Source/destination addresses
+  bytes src_addr = 6;
+  bytes dst_addr = 7;
+
+  // Layer 3 protocol (IPv4/IPv6/ARP/MPLS...)
+  uint32 etype = 30;
+
+  // Layer 4 protocol
+  uint32 proto = 20;
+
+  // Ports for UDP and TCP
+  uint32 src_port = 21;
+  uint32 dst_port = 22;
+
+  // Interfaces
+  uint32 in_if = 18;
+  uint32 out_if = 19;
+
+  // Ethernet information
+  uint64 src_mac = 27;
+  uint64 dst_mac = 28;
+
+  // Vlan
+  uint32 src_vlan = 33;
+  uint32 dst_vlan = 34;
+  // 802.1q VLAN in sampled packet
+  uint32 vlan_id = 29;
+
+  // IP and TCP special flags
+  uint32 ip_tos = 23;
+  uint32 forwarding_status = 24;
+  uint32 ip_ttl = 25;
+  uint32 ip_flags = 38;
+  uint32 tcp_flags = 26;
+  uint32 icmp_type = 31;
+  uint32 icmp_code = 32;
+  uint32 ipv6_flow_label = 37;
+  // Fragments (IPv4/IPv6)
+  uint32 fragment_id = 35;
+  uint32 fragment_offset = 36;
+
+  // Autonomous system information
+  uint32 src_as = 14;
+  uint32 dst_as = 15;
+
+  bytes next_hop = 12;
+  uint32 next_hop_as = 13;
+
+  // Prefix size
+  uint32 src_net = 16;
+  uint32 dst_net = 17;
+
+  // BGP information
+  bytes bgp_next_hop = 100;
+  repeated uint32 bgp_communities = 101;
+  repeated uint32 as_path = 102;
+
+  // MPLS information
+  repeated uint32 mpls_ttl = 80;
+  repeated uint32 mpls_label = 81;
+  repeated bytes mpls_ip = 82;
+
+  uint32 observation_domain_id = 70;
+  uint32 observation_point_id = 71;
+
+  // Encapsulation
+  enum LayerStack {
+    Ethernet = 0;
+    IPv4 = 1;
+    IPv6 = 2;
+    TCP = 3;
+    UDP = 4;
+    MPLS = 5;
+    Dot1Q = 6;
+    ICMP = 7;
+    ICMPv6 = 8;
+    GRE = 9;
+    IPv6HeaderRouting = 10;
+    IPv6HeaderFragment = 11;
+    Geneve = 12;
+    Teredo = 13;
+    Custom = 99;
+    // todo: add nsh
+  }
+  repeated LayerStack layer_stack = 103;
+  repeated uint32 layer_size = 104;
+
+  repeated bytes ipv6_routing_header_addresses = 105; // SRv6
+  uint32 ipv6_routing_header_seg_left = 106; // SRv6
+
+}

compose/kcg/clickhouse/protocols.csv

@@ -27,4 +27,4 @@ proto,name,description
 94,IPIP,IP-within-IP Encapsulation Protocol
 97,ETHERIP,Ethernet-within-IP Encapsulation
 98,ENCAP,Encapsulation Header
-112,VRRP,Virtual Router Redundancy Protocol
+112,VRRP,Virtual Router Redundancy Protocol

compose/kcg/docker-compose.yml

@@ -1,23 +1,27 @@
 version: "3"
 services:
-  zookeeper:
-    image: bitnami/zookeeper:3.7.1
-    ports:
-      - 2181:2181
-    environment:
-      - ALLOW_ANONYMOUS_LOGIN=yes
-    restart: always
   kafka:
-    image: bitnami/kafka:3.4.0
+    image: bitnami/kafka:3.9.0
     ports:
       - 9092:9092
     environment:
-      - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
       - ALLOW_PLAINTEXT_LISTENER=yes
       - KAFKA_DELETE_TOPIC_ENABLE=true
+      - KAFKA_ENABLE_KRAFT=true
+      - KAFKA_KRAFT_CLUSTER_ID=AAAAAAAAAAAAAAAAAAAAAA # uuid encoded in base64
+      # dd if=/dev/zero of=/dev/stdout count=16 bs=1 | openssl enc -A -a
+      # head -c 20 /dev/zero | base64
+      - KAFKA_CFG_BROKER_ID=1
+      - KAFKA_CFG_NODE_ID=1
+      - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1@kafka:9093
+      - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
+      - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=IB
+      - KAFKA_CFG_PROCESS_ROLES=broker,controller
+      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,BROKER:PLAINTEXT,IB:PLAINTEXT
+      - KAFKA_CFG_LISTENERS=CONTROLLER://:9093,BROKER://:9092,IB://:9094
+      - KAFKA_ADVERTISED_LISTENERS=BROKER://kafka:9092,IB://:9094
+      - BITNAMI_DEBUG=yes
     restart: always
-    depends_on:
-      - zookeeper
   grafana:
     image: grafana/grafana:9.4.3
     environment:
@@ -32,7 +36,7 @@ services:
       - ./grafana/dashboards.yml:/etc/grafana/provisioning/dashboards/dashboards.yml
       - ./grafana/dashboards:/var/lib/grafana/dashboards
   prometheus:
-    image: prom/prometheus:v2.37.6
+    image: prom/prometheus:v3.0.1
     ports:
       - 9090:9090
     restart: always
@@ -53,18 +57,20 @@ services:
       - 6343:6343/udp
       - 2055:2055/udp
     restart: always
-    command: 
-    - -transport.kafka.brokers=kafka:9092
-    - -transport=kafka
-    - -transport.kafka.topic=flows
-    - -format=bin
+    command:
+      - -transport.kafka.brokers=kafka:9092
+      - -transport=kafka
+      - -transport.kafka.topic=flows
+      - -format=bin
   db:
-    image: clickhouse/clickhouse-server:22.8.14.53-alpine
+    image: clickhouse/clickhouse-server:24.11.1.2557-alpine
     ports:
       - 8123:8123
     volumes:
       - ./clickhouse:/docker-entrypoint-initdb.d/
-      - ../../pb/flow.proto:/var/lib/clickhouse/format_schemas/flow.proto
+      #- ../../pb/flow.proto:/var/lib/clickhouse/format_schemas/flow.proto
+      # the following files have 101:101 as owner
+      - ./clickhouse/flow.proto:/var/lib/clickhouse/format_schemas/flow.proto
       - ./clickhouse/protocols.csv:/var/lib/clickhouse/user_files/protocols.csv
     depends_on:
       - kafka