Added global stats diff test

Signed-off-by: Toni Uhlig <[email protected]>

examples/c-analysed/c-analysed.c

@@ -207,7 +207,7 @@ static struct
 
         nDPIsrvd_ull flow_risk_count[NDPI_MAX_RISK - 1 /* NDPI_NO_RISK */];
         nDPIsrvd_ull flow_risk_unknown_count;
-    } gauges[2]; /* values after InfluxDB push: gauges[0] -= gauges[1], gauges[1] is zero'd afterwards */
+    } gauges[2]; /* values after CSV write: gauges[0] -= gauges[1], gauges[1] is zero'd afterwards */
 } analysed_statistics = {};
 
 struct global_map
@@ -1553,9 +1553,13 @@ static int parse_options(int argc, char ** argv)
 
         if (str_value_to_ull(analysed_interval, &analysed_interval_ull) != CONVERSION_OK)
         {
-            logger_early(1, "InfluxDB push interval `%s' is not a valid number", analysed_interval);
+            logger_early(1, "Global Stats CSV write interval `%s' is not a valid number", analysed_interval);
             return 1;
         }
+        if (analysed_interval_ull == 0)
+        {
+            logger_early(0, "%s", "Global Stats CSV write interval is zero, summarizing stats during termination");
+        }
     }
 
     if (optind < argc)
@@ -1570,6 +1574,11 @@ static int parse_options(int argc, char ** argv)
 
 static int set_analysed_timer(void)
 {
+    if (analysed_interval_ull == 0)
+    {
+        return 0;
+    }
+
     const time_t interval = analysed_interval_ull * 1000;
     struct itimerspec its;
     its.it_value.tv_sec = interval / 1000;
@@ -1814,7 +1823,7 @@ static int write_global_flow_stats(void)
     buf[-1] = '\n';
 
     struct timeval tval;
-    if (gettimeofday(&tval, NULL) == 0)
+    if (analysed_interval_ull != 0 && gettimeofday(&tval, NULL) == 0)
     {
         unsigned long long int sec = tval.tv_sec;
         unsigned long long int usec = tval.tv_usec;
@@ -2112,6 +2121,13 @@ int main(int argc, char ** argv)
 
     logger(0, "%s", "Initialization succeeded.");
     retval = mainloop(epollfd, distributor);
+    if (analysed_interval_ull == 0)
+    {
+        if (write_global_flow_stats() != 0)
+        {
+            logger(1, "%s", "Could not write global/flow stats on termination.");
+        }
+    }
 failure:
     nDPIsrvd_socket_free(&distributor);
     daemonize_shutdown(pidfile);

test/results/flow-analyse/caches_cfg/ookla.pcap.out

@@ -1 +1,3 @@
 flow_datalink,l3_proto,src_ip,dst_ip,l4_proto,src_port,dst_port,flow_state,flow_src_packets_processed,flow_dst_packets_processed,flow_first_seen,flow_src_last_pkt_time,flow_dst_last_pkt_time,flow_src_min_l4_payload_len,flow_dst_min_l4_payload_len,flow_src_max_l4_payload_len,flow_dst_max_l4_payload_len,flow_src_tot_l4_payload_len,flow_dst_tot_l4_payload_len,midstream,iat_min,iat_avg,iat_max,iat_stddev,iat_var,iat_ent,iat_data,pktlen_min,pktlen_avg,pktlen_max,pktlen_stddev,pktlen_var,pktlen_ent,pktlen_data,bins_c_to_s,bins_s_to_c,directions,entropies,proto,proto_id,encrypted,breed,category,confidence_id,confidence,risks
+timestamp,json_lines,json_bytes,flow_src_total_bytes,flow_dst_total_bytes,flow_new_count,flow_end_count,flow_idle_count,flow_update_count,flow_analyse_count,flow_guessed_count,flow_detected_count,flow_detection_update_count,flow_not_detected_count,flow_risky_count,packet_count,packet_flow_count,init_count,reconnect_count,shutdown_count,status_count,error_unknown_datalink,error_unknown_l3_protocol,error_unsupported_datalink,error_packet_too_short,error_packet_type_unknown,error_packet_header_invalid,error_ip4_packet_too_short,error_ip4_size_smaller_than_header,error_ip4_l4_payload_detection,error_ip6_packet_too_short,error_ip6_size_smaller_than_header,error_ip6_l4_payload_detection,error_tcp_packet_too_short,error_udp_packet_too_short,error_capture_size_smaller_than_packet,error_max_flows_to_track,error_flow_memory_alloc,flow_state_info,flow_state_finished,flow_breed_safe_count,flow_breed_acceptable_count,flow_breed_fun_count,flow_breed_unsafe_count,flow_breed_potentially_dangerous_count,flow_breed_tracker_ads_count,flow_breed_dangerous_count,flow_breed_unrated_count,flow_breed_unknown_count,flow_category_unspecified_count,flow_category_media_count,flow_category_vpn_count,flow_category_email_count,flow_category_data_transfer_count,flow_category_web_count,flow_category_social_network_count,flow_category_download_count,flow_category_game_count,flow_category_chat_count,flow_category_voip_count,flow_category_database_count,flow_category_remote_access_count,flow_category_cloud_count,flow_category_network_count,flow_category_collaborative_count,flow_category_rpc_count,flow_category_streaming_count,flow_category_system_count,flow_category_software_update_count,flow_category_music_count,flow_category_video_count,flow_category_shopping_count,flow_category_productivity_count,flow_category_file_sharing_count,flow_category_conn_check_count,flow_category_iot_scada_count,flow_category_virt_assistant_count,flow_category_cybersecurity_count,flow_category_adult_content_count,flow_category_mining_count,flow_category_malware_count,flow_category_advertisment_count,flow_category_banned_site_count,flow_category_site_unavail_count,flow_category_allowed_site_count,flow_category_antimalware_count,flow_category_crypto_currency_count,flow_category_gambling_count,flow_category_unknown_count,flow_confidence_by_port,flow_confidence_dpi_partial,flow_confidence_dpi_partial_cache,flow_confidence_dpi_cache,flow_confidence_dpi,flow_confidence_nbpf,flow_confidence_by_ip,flow_confidence_dpi_aggressive,flow_confidence_custom_rule,flow_confidence_unknown,flow_severity_low,flow_severity_medium,flow_severity_high,flow_severity_severe,flow_severity_critical,flow_severity_emergency,flow_severity_unknown,flow_l3_ip4_count,flow_l3_ip6_count,flow_l3_other_count,flow_l4_tcp_count,flow_l4_udp_count,flow_l4_icmp_count,flow_l4_other_count,flow_active_count,flow_detected_count,flow_guessed_count,flow_not_detected_count,flow_risk_1_count,flow_risk_2_count,flow_risk_3_count,flow_risk_4_count,flow_risk_5_count,flow_risk_6_count,flow_risk_7_count,flow_risk_8_count,flow_risk_9_count,flow_risk_10_count,flow_risk_11_count,flow_risk_12_count,flow_risk_13_count,flow_risk_14_count,flow_risk_15_count,flow_risk_16_count,flow_risk_17_count,flow_risk_18_count,flow_risk_19_count,flow_risk_20_count,flow_risk_21_count,flow_risk_22_count,flow_risk_23_count,flow_risk_24_count,flow_risk_25_count,flow_risk_26_count,flow_risk_27_count,flow_risk_28_count,flow_risk_29_count,flow_risk_30_count,flow_risk_31_count,flow_risk_32_count,flow_risk_33_count,flow_risk_34_count,flow_risk_35_count,flow_risk_36_count,flow_risk_37_count,flow_risk_38_count,flow_risk_39_count,flow_risk_40_count,flow_risk_41_count,flow_risk_42_count,flow_risk_43_count,flow_risk_44_count,flow_risk_45_count,flow_risk_46_count,flow_risk_47_count,flow_risk_48_count,flow_risk_49_count,flow_risk_50_count,flow_risk_51_count,flow_risk_52_count,flow_risk_53_count,flow_risk_54_count,flow_risk_55_count,flow_risk_56_count,flow_risk_unknown_count
+0,55,46029,22732,8117,6,1,5,0,0,1,5,3,0,2,0,30,1,0,1,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,3,5,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,5,0,0,0,0,0,0,3,0,0,0,0,0,6,0,0,6,0,0,0,6,5,1,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0

test/results/flow-analyse/caches_cfg/teams.pcap.out

@@ -15,3 +15,5 @@ flow_datalink,l3_proto,src_ip,dst_ip,l4_proto,src_port,dst_port,flow_state,flow_
 1,ip4,192.168.1.6,52.114.76.48,tcp,60544,443,finished,16,16,1587041682376166,1587041682938651,1587041692001418,0,0,1060,1452,2113,7396,0,2,328636.7,8978171,1582353.1,2503841415168.0,0.8,"47150,47228,506,44398,29,43913,16,46,186,124,2,213,4,4433,9743,291,46519,32116,477,409,98,18910,1378,20235,62883,403234,424977,8978171,32,9,7",40,339.2,1492,486.1,236250.5,3.9,"64,52,40,276,1492,1492,52,40,40,1492,1492,309,40,40,198,133,568,91,40,109,40,78,46,409,40,46,1100,46,411,415,86,78","10,1,1,0,1,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0","4,3,1,0,0,0,0,0,1,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,4,0,0","0,1,0,0,1,1,0,0,0,1,1,1,0,0,0,0,0,1,0,1,0,0,1,1,0,1,0,1,1,1,1,1","4.334277153,4.946223736,4.571928501,5.576080799,7.377434731,7.334023952,4.748329639,4.630640984,4.571928501,7.530410290,7.590536594,7.109602451,4.680641174,4.630641460,6.484649181,6.111595631,7.563093662,5.442209721,4.630641460,5.902398109,4.630641460,5.214766979,4.462505341,7.402733803,4.680641174,4.505983353,7.828750610,4.609350681,7.428915024,7.453095436,5.564571857,5.463537216",TLS.Teams,91.250,1,Safe,Collaborative,6,DPI,""
 1,ip4,192.168.1.6,52.114.250.123,tcp,50018,443,finished,19,13,1587041693516414,1587041693824623,1587041695435566,0,0,187,1452,477,6361,0,1,71850.4,1566873,274680.6,75449425920.0,1.9,"44968,45079,183,47440,47249,164,13,124,2,107,17,104,3,107,2,120,2,1,8026,8,35,52434,1246,45626,48613,92238,43679,69083,272,113543,1566873",40,256.9,1492,427.0,182315.3,3.7,"64,52,40,227,1492,52,1492,588,52,52,1492,588,52,40,588,166,40,40,40,147,46,85,46,91,40,141,224,40,71,40,46,46","15,1,0,2,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0","4,1,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,0,0","0,1,0,0,1,0,1,1,0,0,1,1,0,0,1,1,0,0,0,0,0,0,1,1,0,0,1,0,0,0,1,1","4.396777153,4.946223736,4.453056812,5.436062336,7.472877979,4.624014378,7.357961178,6.174726009,4.707639694,4.669178009,7.651301384,7.035131931,4.669178009,4.492897511,7.576755524,6.572272301,4.384184361,4.492897511,4.492897034,6.376044750,4.495644569,5.773638725,4.565871716,5.388861179,4.561769009,6.442826271,6.864662647,4.511769295,5.438062191,4.384184361,4.565872192,4.565872192",TLS.Teams,91.250,1,Safe,Collaborative,6,DPI,"15"
 1,ip4,93.71.110.205,192.168.1.6,udp,16332,50016,finished,25,7,1587041695305290,1587041697913583,1587041697668816,38,0,1214,1214,4324,2890,0,1,160381.3,1168245,365653.3,133702352896.0,2.7,"24795,221,101349,1168245,1167037,967065,50759,1119237,13,25,50990,80302,1990,2655,3736,4,1,2,10681,24170,9306,21453,4525,19907,25341,9245,24382,24626,9496,26004,24257",66,253.4,1242,374.4,140199.2,4.0,"140,116,140,116,144,116,138,136,66,1242,1242,136,101,66,1242,1242,70,194,126,94,96,103,108,110,102,98,112,106,103,101,102,102","0,2,16,4,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0","0,1,1,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0","0,1,1,0,1,0,0,0,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0","5.443928242,5.441569805,5.550033092,5.533423424,5.469605446,5.457950115,6.418050289,5.494081497,5.274568558,7.835727215,7.805037022,5.427760124,6.064149857,5.328952789,7.830739975,7.834946632,5.426148415,6.862842083,6.378197670,5.942782402,6.043297768,6.096649170,5.395052433,6.251680851,6.123402596,6.007471561,6.260177612,6.012121677,6.079421997,6.215091705,6.135609150,6.155217648",STUN.Skype_TeamsCall,78.38,0,Acceptable,VoIP,6,DPI,"5"
+timestamp,json_lines,json_bytes,flow_src_total_bytes,flow_dst_total_bytes,flow_new_count,flow_end_count,flow_idle_count,flow_update_count,flow_analyse_count,flow_guessed_count,flow_detected_count,flow_detection_update_count,flow_not_detected_count,flow_risky_count,packet_count,packet_flow_count,init_count,reconnect_count,shutdown_count,status_count,error_unknown_datalink,error_unknown_l3_protocol,error_unsupported_datalink,error_packet_too_short,error_packet_type_unknown,error_packet_header_invalid,error_ip4_packet_too_short,error_ip4_size_smaller_than_header,error_ip4_l4_payload_detection,error_ip6_packet_too_short,error_ip6_size_smaller_than_header,error_ip6_l4_payload_detection,error_tcp_packet_too_short,error_udp_packet_too_short,error_capture_size_smaller_than_packet,error_max_flows_to_track,error_flow_memory_alloc,flow_state_info,flow_state_finished,flow_breed_safe_count,flow_breed_acceptable_count,flow_breed_fun_count,flow_breed_unsafe_count,flow_breed_potentially_dangerous_count,flow_breed_tracker_ads_count,flow_breed_dangerous_count,flow_breed_unrated_count,flow_breed_unknown_count,flow_category_unspecified_count,flow_category_media_count,flow_category_vpn_count,flow_category_email_count,flow_category_data_transfer_count,flow_category_web_count,flow_category_social_network_count,flow_category_download_count,flow_category_game_count,flow_category_chat_count,flow_category_voip_count,flow_category_database_count,flow_category_remote_access_count,flow_category_cloud_count,flow_category_network_count,flow_category_collaborative_count,flow_category_rpc_count,flow_category_streaming_count,flow_category_system_count,flow_category_software_update_count,flow_category_music_count,flow_category_video_count,flow_category_shopping_count,flow_category_productivity_count,flow_category_file_sharing_count,flow_category_conn_check_count,flow_category_iot_scada_count,flow_category_virt_assistant_count,flow_category_cybersecurity_count,flow_category_adult_content_count,flow_category_mining_count,flow_category_malware_count,flow_category_advertisment_count,flow_category_banned_site_count,flow_category_site_unavail_count,flow_category_allowed_site_count,flow_category_antimalware_count,flow_category_crypto_currency_count,flow_category_gambling_count,flow_category_unknown_count,flow_confidence_by_port,flow_confidence_dpi_partial,flow_confidence_dpi_partial_cache,flow_confidence_dpi_cache,flow_confidence_dpi,flow_confidence_nbpf,flow_confidence_by_ip,flow_confidence_dpi_aggressive,flow_confidence_custom_rule,flow_confidence_unknown,flow_severity_low,flow_severity_medium,flow_severity_high,flow_severity_severe,flow_severity_critical,flow_severity_emergency,flow_severity_unknown,flow_l3_ip4_count,flow_l3_ip6_count,flow_l3_other_count,flow_l4_tcp_count,flow_l4_udp_count,flow_l4_icmp_count,flow_l4_other_count,flow_active_count,flow_detected_count,flow_guessed_count,flow_not_detected_count,flow_risk_1_count,flow_risk_2_count,flow_risk_3_count,flow_risk_4_count,flow_risk_5_count,flow_risk_6_count,flow_risk_7_count,flow_risk_8_count,flow_risk_9_count,flow_risk_10_count,flow_risk_11_count,flow_risk_12_count,flow_risk_13_count,flow_risk_14_count,flow_risk_15_count,flow_risk_16_count,flow_risk_17_count,flow_risk_18_count,flow_risk_19_count,flow_risk_20_count,flow_risk_21_count,flow_risk_22_count,flow_risk_23_count,flow_risk_24_count,flow_risk_25_count,flow_risk_26_count,flow_risk_27_count,flow_risk_28_count,flow_risk_29_count,flow_risk_30_count,flow_risk_31_count,flow_risk_32_count,flow_risk_33_count,flow_risk_34_count,flow_risk_35_count,flow_risk_36_count,flow_risk_37_count,flow_risk_38_count,flow_risk_39_count,flow_risk_40_count,flow_risk_41_count,flow_risk_42_count,flow_risk_43_count,flow_risk_44_count,flow_risk_45_count,flow_risk_46_count,flow_risk_47_count,flow_risk_48_count,flow_risk_49_count,flow_risk_50_count,flow_risk_51_count,flow_risk_52_count,flow_risk_53_count,flow_risk_54_count,flow_risk_55_count,flow_risk_56_count,flow_risk_unknown_count
+0,668,670757,293772,293323,83,17,66,0,16,2,80,51,1,29,16,317,1,0,1,1,0,0,0,0,16,0,0,0,0,0,0,0,0,0,0,0,0,27,56,46,33,1,0,0,0,0,0,0,0,0,0,0,0,6,0,0,0,0,16,0,0,11,27,19,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,78,0,0,0,0,0,38,12,2,0,0,0,0,83,0,0,42,40,1,0,83,80,2,1,0,0,0,0,0,12,0,0,0,0,2,0,2,0,0,29,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,5,0,0,1,0,0,0,0,0,0,0