From d4786599b9f6d5a84c851c6a8ee8b96e914bac94 Mon Sep 17 00:00:00 2001 From: Olivier Valentin Date: Mon, 22 Apr 2024 12:48:54 +0200 Subject: [PATCH] Detailed subnet configuration parameter --- collector/lib/CollectorConfig.cpp | 18 ++++++++++++++++++ collector/lib/CollectorConfig.h | 2 ++ collector/lib/CollectorService.cpp | 1 + collector/lib/ConnTracker.cpp | 14 ++++++++++---- collector/lib/ConnTracker.h | 2 ++ collector/test/ConnTrackerTest.cpp | 24 ++++++++++++++++++++++++ 6 files changed, 57 insertions(+), 4 deletions(-) diff --git a/collector/lib/CollectorConfig.cpp b/collector/lib/CollectorConfig.cpp index ce77bf69e2f..5769dba51c2 100644 --- a/collector/lib/CollectorConfig.cpp +++ b/collector/lib/CollectorConfig.cpp @@ -26,6 +26,9 @@ BoolEnvVar network_drop_ignored("ROX_NETWORK_DROP_IGNORED", true); // The default value contains link-local addresses for IPv4 (RFC3927) and IPv6 (RFC2462) StringListEnvVar ignored_networks("ROX_IGNORE_NETWORKS", std::vector({"169.254.0.0/16", "fe80::/10"})); +// Connection endpoints matching a network prefix listed here will never be aggregated. +StringListEnvVar detailed_networks("ROX_DETAIL_NETWORKS", std::vector()); + // If true, set curl to be verbose, adding further logging that might be useful for debugging. BoolEnvVar set_curl_verbose("ROX_COLLECTOR_SET_CURL_VERBOSE", false); @@ -185,6 +188,21 @@ void CollectorConfig::InitCollectorConfig(CollectorArgs* args) { } }); + std::for_each(detailed_networks.value().begin(), detailed_networks.value().end(), + [&detailed_networks = this->detailed_networks_](const std::string& str) { + if (str.empty()) + return; + + std::optional net = IPNet::parse(str); + + if (net) { + CLOG(INFO) << "Private network : " << *net; + detailed_networks.emplace_back(std::move(*net)); + } else { + CLOG(ERROR) << "Invalid network in ROX_DETAIL_NETWORKS : " << str; + } + }); + if (set_curl_verbose) { curl_verbose_ = true; } diff --git a/collector/lib/CollectorConfig.h b/collector/lib/CollectorConfig.h index c72ec2b91a5..4f60a2efc63 100644 --- a/collector/lib/CollectorConfig.h +++ b/collector/lib/CollectorConfig.h @@ -65,6 +65,7 @@ class CollectorConfig { bool DisableNetworkFlows() const { return disable_network_flows_; } const UnorderedSet& IgnoredL4ProtoPortPairs() const { return ignored_l4proto_port_pairs_; } const std::vector& IgnoredNetworks() const { return ignored_networks_; } + const std::vector& DetailedNetworks() const { return detailed_networks_; } bool CurlVerbose() const { return curl_verbose_; } bool EnableAfterglow() const { return enable_afterglow_; } bool IsCoreDumpEnabled() const; @@ -100,6 +101,7 @@ class CollectorConfig { bool scrape_listen_endpoints_ = false; UnorderedSet ignored_l4proto_port_pairs_; std::vector ignored_networks_; + std::vector detailed_networks_; bool curl_verbose_ = false; HostConfig host_config_; diff --git a/collector/lib/CollectorService.cpp b/collector/lib/CollectorService.cpp index 1e1e68b16b9..099c641f072 100644 --- a/collector/lib/CollectorService.cpp +++ b/collector/lib/CollectorService.cpp @@ -86,6 +86,7 @@ void CollectorService::RunForever() { UnorderedSet ignored_l4proto_port_pairs(config_.IgnoredL4ProtoPortPairs()); conn_tracker->UpdateIgnoredL4ProtoPortPairs(std::move(ignored_l4proto_port_pairs)); conn_tracker->UpdateIgnoredNetworks(config_.IgnoredNetworks()); + conn_tracker->UpdateDetailedNetworks(config_.DetailedNetworks()); conn_tracker->EnableExternalIPs(config_.EnableExternalIPs()); auto network_connection_info_service_comm = std::make_shared(config_.Hostname(), config_.grpc_channel); diff --git a/collector/lib/ConnTracker.cpp b/collector/lib/ConnTracker.cpp index 59eefc39335..37fcb9e9891 100644 --- a/collector/lib/ConnTracker.cpp +++ b/collector/lib/ConnTracker.cpp @@ -35,7 +35,6 @@ bool AdvertisedEndpointEquality::operator()(const ContainerEndpoint& lhs, const return lhs.container() == rhs.container() && lhs.endpoint() == rhs.endpoint() && lhs.l4proto() == rhs.l4proto(); } - bool ContainsPrivateNetwork(Address::Family family, NRadixTree tree) { return tree.IsAnyIPNetSubset(family, private_networks_tree) || private_networks_tree.IsAnyIPNetSubset(family, tree); } @@ -76,14 +75,15 @@ IPNet ConnectionTracker::NormalizeAddressNoLock(const Address& address) const { return {}; } - bool private_addr = !address.IsPublic(); + // We want to keep private addresses and explicitely requested ones. + bool keep_addr = !address.IsPublic() || !detailed_networks_.Find(address).IsNull(); const bool* known_private_networks_exists = Lookup(known_private_networks_exists_, address.family()); - if (private_addr && (known_private_networks_exists && !*known_private_networks_exists)) { + if (keep_addr && (known_private_networks_exists && !*known_private_networks_exists)) { return IPNet(address, 0, true); } const auto& network = known_ip_networks_.Find(address); - if (private_addr || Contains(known_public_ips_, address)) { + if (keep_addr || Contains(known_public_ips_, address)) { return IPNet(address, network.bits(), true); } @@ -330,6 +330,12 @@ void ConnectionTracker::UpdateIgnoredNetworks(const std::vector& network_ } } +void ConnectionTracker::UpdateDetailedNetworks(const std::vector& network_list) { + WITH_LOCK(mutex_) { + detailed_networks_ = NRadixTree(network_list); + } +} + // Increment the stat counter matching the connection's characteristics inline void ConnectionTracker::IncrementConnectionStats(Connection conn, ConnectionTracker::Stats& stats) const { auto& direction = conn.is_server() ? stats.inbound : stats.outbound; diff --git a/collector/lib/ConnTracker.h b/collector/lib/ConnTracker.h index 07dc07813f1..1860fdd597b 100644 --- a/collector/lib/ConnTracker.h +++ b/collector/lib/ConnTracker.h @@ -130,6 +130,7 @@ class ConnectionTracker { void EnableExternalIPs(bool enable) { enable_external_ips_ = enable; } void UpdateIgnoredL4ProtoPortPairs(UnorderedSet&& ignored_l4proto_port_pairs); void UpdateIgnoredNetworks(const std::vector& network_list); + void UpdateDetailedNetworks(const std::vector& network_list); // Emplace a connection into the state ConnMap, or update its timestamp if the supplied timestamp is more recent // than the stored one. @@ -200,6 +201,7 @@ class ConnectionTracker { UnorderedMap known_private_networks_exists_; UnorderedSet ignored_l4proto_port_pairs_; NRadixTree ignored_networks_; + NRadixTree detailed_networks_; Stats inserted_connections_counters_ = {}; }; diff --git a/collector/test/ConnTrackerTest.cpp b/collector/test/ConnTrackerTest.cpp index cb94b4cadde..6c6852af9c1 100644 --- a/collector/test/ConnTrackerTest.cpp +++ b/collector/test/ConnTrackerTest.cpp @@ -171,6 +171,30 @@ TEST(ConnTrackerTest, TestUpdateIgnoredNetworks) { EXPECT_TRUE(tracker.FetchConnState().empty()); } +TEST(ConnTrackerTest, TestUpdateDetailedNetworks) { + Endpoint a(Address(192, 168, 1, 10), 9999); + Endpoint b(Address(245, 1, 1, 1), 80); + + Connection conn1("xyz", a, b, L4Proto::TCP, false); + + int64_t time_micros = 1000; + + Connection conn_aggregated("xyz", Endpoint(IPNet(Address(), 0, true), 0), Endpoint(IPNet(Address(255, 255, 255, 255), 0), 80), L4Proto::TCP, false); + Connection conn_detailed("xyz", Endpoint(IPNet(Address(), 0, true), 0), Endpoint(IPNet(Address(245, 1, 1, 1), 0, true), 80), L4Proto::TCP, false); + + ConnectionTracker tracker; + + tracker.Update({conn1}, {}, time_micros); + + auto state = tracker.FetchConnState(true); + EXPECT_THAT(state, UnorderedElementsAre(std::make_pair(conn_aggregated, ConnStatus(time_micros, true)))); + + tracker.UpdateDetailedNetworks({IPNet(Address(240, 0, 0, 0), 4)}); + + state = tracker.FetchConnState(true); + EXPECT_THAT(state, UnorderedElementsAre(std::make_pair(conn_detailed, ConnStatus(time_micros, true)))); +} + TEST(ConnTrackerTest, TestUpdateNormalized) { Endpoint a(Address(192, 168, 0, 1), 80); Endpoint b(Address(192, 168, 1, 10), 9999);