Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 44 additions & 1 deletion dstack/gateway/src/main_service.rs
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ use wavekv::types::NodeId;

use crate::{
cert_store::{CertResolver, CertStoreBuilder},
config::{Config, TlsConfig},
config::{Config, TlsConfig, WgConfig},
kv::{
fetch_peers_from_bootnode, import, AppIdValidator, HttpsClientConfig, InstanceRecord,
KvStore, LegacyOverrides, LoadedInstances, NodeData, NodeStatus, PortPolicy,
Expand Down Expand Up @@ -1538,12 +1538,15 @@ fn start_wavekv_watch_task(proxy: Proxy) -> Result<()> {
// Watch for node changes and reconfigure WireGuard
let mut rx = kv_store.watch_nodes();
let proxy_for_nodes = proxy.clone();
let mut warned_overlaps = BTreeSet::new();
warn_peer_addresses_in_pool(&proxy, &mut warned_overlaps);
tokio::spawn(async move {
loop {
if rx.changed().await.is_err() {
break;
}
info!("WaveKV: detected remote node changes, reconfiguring WireGuard...");
warn_peer_addresses_in_pool(&proxy_for_nodes, &mut warned_overlaps);
proxy_for_nodes.request_wg_apply();
}
});
Expand Down Expand Up @@ -2769,6 +2772,46 @@ impl ProxyState {
}
}

/// Peer gateway addresses this node could hand out to a CVM.
///
/// Every node must keep its pool clear of the other nodes' addresses, but each
/// node's config validation only sees its own. The replicated node info carries
/// a peer's address and nothing else of its config, so this is the part of the
/// overlap that can be detected here.
fn peer_addresses_in_pool(
wg: &WgConfig,
my_id: NodeId,
nodes: &BTreeMap<NodeId, NodeData>,
) -> BTreeSet<(NodeId, Ipv4Addr)> {
nodes
.iter()
.filter(|(id, _)| **id != my_id)
.filter_map(|(id, node)| {
let ip = match node.wg_ip.parse::<ipnet::Ipv4Net>() {
Ok(net) => net.addr(),
Err(_) => node.wg_ip.parse().ok()?,
};
wg.is_valid_client_ip(ip).then_some((*id, ip))
})
.collect()
}

/// Warn once for each peer address that newly overlaps this node's pool.
fn warn_peer_addresses_in_pool(proxy: &Proxy, warned: &mut BTreeSet<(NodeId, Ipv4Addr)>) {
let overlaps = peer_addresses_in_pool(
&proxy.config.wg,
proxy.config.sync.node_id,
&proxy.kv_store.load_all_nodes(),
);
for (node_id, ip) in overlaps.difference(warned) {
warn!(
"gateway node {node_id} has address {ip}, which this node may allocate to a CVM; \
exclude it from client_ip_range or add it to reserved_net"
);
}
*warned = overlaps;
}

fn gateway_nodes(kv_store: &KvStore, exclude_down: bool) -> Vec<GatewayNodeInfo> {
let node_statuses = if exclude_down {
kv_store.load_all_node_statuses()
Expand Down
32 changes: 32 additions & 0 deletions dstack/gateway/src/main_service/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2908,3 +2908,35 @@ async fn refreshing_state_leaves_the_routing_lock_free_while_it_writes_to_the_kv
"the routing lock is held while the refresh waits on the KV store"
);
}

#[tokio::test]
async fn a_peer_gateway_address_inside_this_nodes_pool_is_detected() {
let state = create_test_state().await;
let wg = &state.config.wg;
let node = |wg_ip: &str| crate::kv::NodeData {
uuid: Vec::new(),
url: String::new(),
wg_public_key: String::new(),
wg_endpoint: String::new(),
wg_ip: wg_ip.to_string(),
};
let in_pool = wg
.client_ip_range
.hosts()
.find(|ip| wg.is_valid_client_ip(*ip))
.unwrap();
let my_id = state.config.sync.node_id;
let nodes = BTreeMap::from([
// This node's own address never counts.
(my_id, node(&format!("{in_pool}/24"))),
(my_id + 1, node(&format!("{in_pool}/24"))),
// A bare address, as well as the `Ipv4Net` form this node writes.
(my_id + 2, node(&in_pool.to_string())),
(my_id + 3, node("192.0.2.1/24")),
(my_id + 4, node("not an address")),
]);
assert_eq!(
peer_addresses_in_pool(wg, my_id, &nodes),
BTreeSet::from([(my_id + 1, in_pool), (my_id + 2, in_pool)])
);
}
Loading