Skip to content

Commit

Permalink
FPC (DNS): fix and improvements
Browse files Browse the repository at this point in the history 
Update documetation.
Fix key with ipv6 addreses over ipv4 flow.
Update unit tests results.
  • Loading branch information
@IvanNardi
IvanNardi committed Jul 22, 2024
1 parent 11042e6 commit 2c12037
Show file tree
Hide file tree
Showing 34 changed files with 434 additions and 429 deletions.
6 changes: 3 additions & 3 deletions doc/configuration_parameters.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,9 @@ TODO
| NULL | "flow_risk.crawler_bot.list.load" | 1 | NULL | NULL | Enable/disable loading of internal IP address list used to check `NDPI_HTTP_CRAWLER_BOT` flow risk |
| NULL | "filename.config" | NULL | NULL | NULL | Name of the file containing a list of configuration knobs itself (one per line)!. Useful to configure nDPI via text file instead of via API |
| NULL | "log.level" | 0 | 0 | 3 | Configure the log/debug level. Possible values: 0 = error, 1 = trace, 2 = debug, 3 = extra debug |
| NULL | "lru.$CACHE_NAME.size" | See description | 0 | 16777215 | Set the size (in number of elements) of the specified LRU cache (0 = the cache is disabled). The keyword "$CACHE_NAME" is a placeholder for the cache name and the possible values are: ookla, bittorrent, stun, tls_cert, mining, msteams. The default value is "32768" for the bittorrent cache and "1024" for all the other caches |
| NULL | "lru.$CACHE_NAME.ttl" | See description | 0 | 16777215 | Set the TTL (in seconds) for the elements of the specified LRU cache (0 = the elements never explicitly expire). The keyword "$CACHE_NAME" is a placeholder for the cache name and the possible values are: ookla, bittorrent, stun, tls_cert, mining, msteams. The default value is "120" for the ookla cache, "60" for the msteams cache and "0" for all the other caches |
| NULL | "lru.$CACHE_NAME.scope" | 0 | 0 | 1 | Set the scope of the specified LRU cache (0 = the cache is local, 1 = the cache is global). The keyword "$CACHE_NAME" is a placeholder for the cache name and the possible values are: ookla, bittorrent, stun, tls_cert, mining, msteams. The global scope con be set only if a global context has been initialized |
| NULL | "lru.$CACHE_NAME.size" | See description | 0 | 16777215 | Set the size (in number of elements) of the specified LRU cache (0 = the cache is disabled). The keyword "$CACHE_NAME" is a placeholder for the cache name and the possible values are: ookla, bittorrent, stun, tls_cert, mining, msteams, fpc_dns. The default value is "32768" for the bittorrent cache and "1024" for all the other caches |
| NULL | "lru.$CACHE_NAME.ttl" | See description | 0 | 16777215 | Set the TTL (in seconds) for the elements of the specified LRU cache (0 = the elements never explicitly expire). The keyword "$CACHE_NAME" is a placeholder for the cache name and the possible values are: ookla, bittorrent, stun, tls_cert, mining, msteams, fpc_dns. The default value is "120" for the ookla cache, "60" for the msteams and fpc_dns caches and "0" for all the other caches |
| NULL | "lru.$CACHE_NAME.scope" | 0 | 0 | 1 | Set the scope of the specified LRU cache (0 = the cache is local, 1 = the cache is global). The keyword "$CACHE_NAME" is a placeholder for the cache name and the possible values are: ookla, bittorrent, stun, tls_cert, mining, msteams, fpc_dns. The global scope con be set only if a global context has been initialized |
| "tls" | "certificate_expiration_threshold" | 30 | 0 | 365 | The threshold (in days) used to trigger the `NDPI_TLS_CERTIFICATE_ABOUT_TO_EXPIRE` flow risk |
| "tls" | "application_blocks_tracking" | disable | NULL | NULL | Enable/disable processing of TLS Application Blocks (post handshake) to extract statistical information about the flow |
| "tls" | "metadata.sha1_fingerprint" | enable | NULL | NULL | Enable/disable computation and export of SHA1 fingerprint for TLS flows. Note that if it is disable, the flow risk `NDPI_MALICIOUS_SHA1_CERTIFICATE` is not checked |
Expand Down
3 changes: 3 additions & 0 deletions src/include/ndpi_private.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -613,6 +613,9 @@ int load_config_file_fd(struct ndpi_detection_module_struct *ndpi_str, FILE *fd)
int load_category_file_fd(struct ndpi_detection_module_struct *ndpi_str,
FILE *fd, ndpi_protocol_category_t category_id);

u_int64_t fpc_dns_cache_key_from_dns_info(struct ndpi_flow_struct *flow);


/* TLS */
int processClientServerHello(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow, uint32_t quic_version);
Expand Down
3 changes: 2 additions & 1 deletion src/include/ndpi_typedefs.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1317,7 +1317,8 @@ struct ndpi_flow_struct {
union {
/* the only fields useful for nDPI and ntopng */
struct {
u_int8_t num_queries, num_answers, reply_code, is_query;
u_int8_t num_queries, num_answers, reply_code;
u_int8_t is_query:1, is_rsp_addr_ipv6:1, pad:6;
u_int16_t query_type, query_class, rsp_type, edns0_udp_payload_size;
ndpi_ip_addr_t rsp_addr; /* The first address in a DNS response packet (A and AAAA) */
char geolocation_iata_code[4];
Expand Down
49 changes: 30 additions & 19 deletions src/lib/ndpi_main.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7317,18 +7317,30 @@ u_int16_t ndpi_guess_host_protocol_id(struct ndpi_detection_module_struct *ndpi_

/* ********************************************************************************* */

u_int64_t make_fpc_dns_cache_key(struct ndpi_flow_struct *flow) {
static u_int64_t make_fpc_dns_cache_key(struct ndpi_flow_struct *flow) {
u_int64_t key;

/* network byte order */
if(flow->is_ipv6)
if(flow->is_ipv6)
key = ndpi_quick_hash64((const char *)flow->s_address.v6, 16);
else
else
key = (u_int64_t)(flow->s_address.v4);

return key;
}

}

/* ********************************************************************************* */

u_int64_t fpc_dns_cache_key_from_dns_info(struct ndpi_flow_struct *flow) {
u_int64_t key;

if(flow->protos.dns.is_rsp_addr_ipv6)
key = ndpi_quick_hash64((const char *)&flow->protos.dns.rsp_addr.ipv6, 16);
else
key = (u_int64_t)(flow->protos.dns.rsp_addr.ipv4);

return key;
}

/* ********************************************************************************* */

static u_int64_t make_msteams_key(struct ndpi_flow_struct *flow, u_int8_t use_client) {
Expand Down Expand Up @@ -8429,9 +8441,6 @@ static ndpi_protocol ndpi_internal_detection_process_packet(struct ndpi_detectio

if(flow->num_processed_pkts == 1) {
/* first packet of this flow to be analyzed */

/* Check some fpc cache */


#ifdef HAVE_NBPF
if(ndpi_str->nbpf_custom_proto[0].tree != NULL) {
Expand Down Expand Up @@ -8504,17 +8513,19 @@ static ndpi_protocol ndpi_internal_detection_process_packet(struct ndpi_detectio
if(ndpi_do_guess(ndpi_str, flow, &ret) == -1)
return(ret);

/* First Packet Classification */

fpc_check_ip(ndpi_str, flow);
/* check fpc DNS cache */

/* Check fpc DNS cache */
if(ndpi_str->fpc_dns_cache &&
ndpi_lru_find_cache(ndpi_str->fpc_dns_cache, make_fpc_dns_cache_key(flow),
&fpc_dns_cached_proto, 0 /* Don't remove it as it can be used for other connections */,
ndpi_get_current_time(flow))) {
NDPI_LOG_DBG(ndpi_str,"\nFound from FPC DNS cache...%u\n",fpc_dns_cached_proto);

/* Save the protocol id into ndpi_flow_struct structure as a "fpc result" */
fpc_update(ndpi_str, flow, NDPI_PROTOCOL_UNKNOWN,
fpc_dns_cached_proto, NDPI_FPC_CONFIDENCE_DNS);
ndpi_lru_find_cache(ndpi_str->fpc_dns_cache, make_fpc_dns_cache_key(flow),
&fpc_dns_cached_proto, 0 /* Don't remove it as it can be used for other connections */,
ndpi_get_current_time(flow))) {
NDPI_LOG_DBG(ndpi_str,"Found from FPC DNS cache: %u\n",fpc_dns_cached_proto);

fpc_update(ndpi_str, flow, NDPI_PROTOCOL_UNKNOWN,
fpc_dns_cached_proto, NDPI_FPC_CONFIDENCE_DNS);
}

}
Expand Down
3 changes: 2 additions & 1 deletion src/lib/ndpi_utils.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3215,7 +3215,8 @@ int64_t ndpi_strtonum(const char *numstr, int64_t minval, int64_t maxval, const
const char *ndpi_lru_cache_idx_to_name(lru_cache_type idx)
{
const char *names[NDPI_LRUCACHE_MAX] = { "ookla", "bittorrent", "stun",
"tls_cert", "mining", "msteams" };
"tls_cert", "mining", "msteams",
"fpc_dns" };

if(idx < 0 || idx >= NDPI_LRUCACHE_MAX)
return "unknown";
Expand Down
29 changes: 9 additions & 20 deletions src/lib/protocols/dns.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -473,6 +473,7 @@ static int search_valid_dns(struct ndpi_detection_module_struct *ndpi_struct,
)) {
if(found == 0) {
memcpy(&flow->protos.dns.rsp_addr, packet->payload + x, data_len);
flow->protos.dns.is_rsp_addr_ipv6 = (data_len == 16) ? 1 : 0;
found = 1;
}
}
Expand Down Expand Up @@ -652,17 +653,6 @@ static int search_dns_again(struct ndpi_detection_module_struct *ndpi_struct, st
}

/* *********************************************** */
static u_int64_t fpc_dns_cache_key(struct ndpi_flow_struct *flow) {
u_int64_t key;

/* network byte order */
if(flow->is_ipv6)
key = ndpi_quick_hash64((const char *)flow->protos.dns.rsp_addr.ipv6.u6_addr.u6_addr8, 16);
else
key = (u_int64_t)(flow->protos.dns.rsp_addr.ipv4);

return key;
}

static void ndpi_search_dns(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow) {
struct ndpi_packet_struct *packet = &ndpi_struct->packet;
Expand Down Expand Up @@ -815,15 +805,14 @@ static void ndpi_search_dns(struct ndpi_detection_module_struct *ndpi_struct, st
strlen(flow->host_server_name),
&ret_match,
NDPI_PROTOCOL_DNS);
/* Add to DNS cache */
if(((ret.app_protocol != NDPI_PROTOCOL_UNKNOWN) && (flow->protos.dns.rsp_type == 0x1)) ||
((ret.app_protocol != NDPI_PROTOCOL_UNKNOWN) && (flow->protos.dns.rsp_type == 0x1c))){
if(ndpi_struct->fpc_dns_cache){
ndpi_lru_add_to_cache(ndpi_struct->fpc_dns_cache,/*fpc_cache,*/
fpc_dns_cache_key(flow),ret.app_protocol, ndpi_get_current_time(flow));

}
}
/* Add to FPC DNS cache */
if(ret.app_protocol != NDPI_PROTOCOL_UNKNOWN &&
(flow->protos.dns.rsp_type == 0x1 || flow->protos.dns.rsp_type == 0x1c) && /* A, AAAA */
ndpi_struct->fpc_dns_cache) {
ndpi_lru_add_to_cache(ndpi_struct->fpc_dns_cache,
fpc_dns_cache_key_from_dns_info(flow), ret.app_protocol,
ndpi_get_current_time(flow));
}

if(ret.app_protocol == NDPI_PROTOCOL_UNKNOWN)
ret.master_protocol = checkDNSSubprotocol(s_port, d_port);
Expand Down
Loading

0 comments on commit 2c12037

Please sign in to comment.