Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 49 additions & 8 deletions dstack/gateway/src/kv/import.rs
Original file line number Diff line number Diff line change
Expand Up @@ -144,10 +144,6 @@ fn validate_instance(
validate_id("instance_id", instance_id)?;
validate_id("app_id", &data.app_id)?;
validate_wg_public_key(&data.public_key)?;
ensure!(
data.public_key != wg.public_key,
"public key belongs to this gateway"
);
// The routable network, not this node's allocation share: in a cluster
// every node carries peers for the CVMs registered on the other nodes, and
// those hold addresses from the other nodes' shares by design.
Expand Down Expand Up @@ -180,11 +176,25 @@ fn validate_instance(
/// wins, ties broken by instance ID) so that every node reaches the same
/// decision from the same KV contents — the local registration path resolves
/// them the same way, by refusing the newcomer.
pub fn accept_instances(wg: &WgConfig, loaded: LoadedInstances) -> AcceptedInstances {
accept_instances_at(wg, loaded, now_secs())
///
/// `gateway_keys` are the WireGuard keys of the gateway nodes in the KV. A
/// record carrying one of them loses to that gateway on every node, not only
/// on the gateway whose key it is: judged by this node's own key alone, the
/// nodes would disagree on which instances exist.
pub fn accept_instances(
wg: &WgConfig,
gateway_keys: &HashSet<String>,
loaded: LoadedInstances,
) -> AcceptedInstances {
accept_instances_at(wg, gateway_keys, loaded, now_secs())
}

fn accept_instances_at(wg: &WgConfig, loaded: LoadedInstances, now: u64) -> AcceptedInstances {
fn accept_instances_at(
wg: &WgConfig,
gateway_keys: &HashSet<String>,
loaded: LoadedInstances,
now: u64,
) -> AcceptedInstances {
let LoadedInstances {
decoded,
undecodable,
Expand Down Expand Up @@ -213,6 +223,12 @@ fn accept_instances_at(wg: &WgConfig, loaded: LoadedInstances, now: u64) -> Acce
let checked = validate_instance(wg, now, &instance_id, &data)
.map_err(|reason| (Rejection::Unusable, reason))
.and_then(|()| {
if data.public_key == wg.public_key || gateway_keys.contains(&data.public_key) {
return Err((
Rejection::LostConflict,
anyhow::anyhow!("public key belongs to a gateway node"),
));
}
if let Some(owner) = claimed_ips.get(&data.ip) {
return Err((
Rejection::LostConflict,
Expand Down Expand Up @@ -297,7 +313,7 @@ mod tests {
}

fn accept(records: Vec<(&str, InstanceRecord)>) -> AcceptedInstances {
accept_instances_at(&wg_config(), loaded(records), NOW)
accept_instances_at(&wg_config(), &HashSet::new(), loaded(records), NOW)
}

#[test]
Expand Down Expand Up @@ -365,6 +381,7 @@ mod tests {
let wg = wg_config();
let accepted = accept_instances_at(
&wg,
&HashSet::new(),
loaded(vec![(
"self-peer",
instance("10.0.0.20", &wg.public_key, 100),
Expand All @@ -375,6 +392,28 @@ mod tests {
assert_eq!(accepted.rejected.len(), 1);
}

#[test]
fn every_node_rejects_another_gateways_public_key() {
// A CVM can register with one node under a peer gateway's key. Judged by
// each node's own key only, the peer refuses the record while every other
// node accepts it, and the nodes disagree on which instances exist.
let peer_gateway = key(8);
let accepted = accept_instances_at(
&wg_config(),
&HashSet::from([peer_gateway.clone()]),
loaded(vec![
("steals-peer-key", instance("10.0.0.20", &peer_gateway, 100)),
("good", instance("10.0.0.21", &key(1), 200)),
]),
NOW,
);
assert!(!accepted.instances.contains_key("steals-peer-key"));
assert!(accepted.instances.contains_key("good"));
// The gateway owns the key, so the instance has to stop being routable
// rather than keep whatever the data plane already holds.
assert!(accepted.unreadable().is_empty());
}

#[test]
fn duplicate_ip_and_key_claims_resolve_to_the_older_registration() {
let accepted = accept(vec![
Expand Down Expand Up @@ -434,6 +473,7 @@ mod tests {
};
let accepted = accept_instances(
&wg,
&HashSet::new(),
loaded(vec![
("mine", instance(mine, &key(1), 100)),
("peers", instance(peers, &key(2), 100)),
Expand Down Expand Up @@ -499,6 +539,7 @@ mod tests {
fn an_undecodable_record_is_reported_as_unreadable_not_as_absent() {
let accepted = accept_instances_at(
&wg_config(),
&HashSet::new(),
LoadedInstances {
decoded: [("good".to_string(), instance("10.0.0.20", &key(1), 100))]
.into_iter()
Expand Down
11 changes: 10 additions & 1 deletion dstack/gateway/src/kv/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ pub use sync_service::{fetch_peers_from_bootnode, PersistentWriteNotifier, WaveK
use tracing::{error, warn};

use std::{
collections::{BTreeMap, BTreeSet},
collections::{BTreeMap, BTreeSet, HashSet},
net::Ipv4Addr,
path::Path,
time::Duration,
Expand Down Expand Up @@ -1376,6 +1376,15 @@ impl KvStore {
.collect()
}

/// WireGuard public keys of every gateway node in the store.
pub fn gateway_wg_public_keys(&self) -> HashSet<String> {
self.load_all_nodes()
.into_values()
.map(|node| node.wg_public_key)
.filter(|key| !key.is_empty())
.collect()
}

/// Remove a gateway node from replicated state.
///
/// Writes tombstones for the node's info, status, and sync address, and
Expand Down
31 changes: 27 additions & 4 deletions dstack/gateway/src/main_service.rs
Original file line number Diff line number Diff line change
Expand Up @@ -234,8 +234,12 @@ impl Proxy {
/// cached rejection log, so the answer is current even right after a
/// restart and does not depend on when the last reload ran.
pub fn rejected_instances(&self) -> Vec<RejectedInstanceReport> {
let rejected =
import::accept_instances(&self.config.wg, self.kv_store.load_all_instances()).rejected;
let rejected = import::accept_instances(
&self.config.wg,
&self.kv_store.gateway_wg_public_keys(),
self.kv_store.load_all_instances(),
)
.rejected;
let state = self.lock();
rejected
.into_iter()
Expand Down Expand Up @@ -1108,7 +1112,7 @@ fn build_state_from_kv_store(
) -> ProxyStateMut {
let mut state = ProxyStateMut::default();

let accepted = import::accept_instances(&config.wg, instances);
let accepted = import::accept_instances(&config.wg, &store.gateway_wg_public_keys(), instances);
report_rejected_instances(&accepted.rejected);

// Build instances
Expand Down Expand Up @@ -1495,6 +1499,7 @@ fn start_wavekv_watch_task(proxy: Proxy) -> Result<()> {
// closing a traffic gate does not touch `inst/` and would never wake the
// watcher above.
let mut overrides_rx = store_clone.watch_instance_overrides();
let mut nodes_rx = store_clone.watch_nodes();
reload_instances_from_kv_store(&proxy_clone, &store_clone)
.context("Failed to initial load instances from KvStore")?;
tokio::spawn(async move {
Expand All @@ -1512,6 +1517,13 @@ fn start_wavekv_watch_task(proxy: Proxy) -> Result<()> {
}
"operator override"
}
// A record is judged against the gateway nodes' keys too.
changed = nodes_rx.changed() => {
if changed.is_err() {
break;
}
"node"
}
};
info!("WaveKV: detected remote {reason} changes, reloading...");
if let Err(err) = reload_instances_from_kv_store(&proxy_clone, &store_clone) {
Expand Down Expand Up @@ -1679,7 +1691,11 @@ fn reload_instances_from_kv_store(proxy: &Proxy, store: &KvStore) -> Result<()>
);
}
let legacy_overrides = store.legacy_instance_overrides();
let accepted = import::accept_instances(&proxy.config.wg, store.load_all_instances());
let accepted = import::accept_instances(
&proxy.config.wg,
&store.gateway_wg_public_keys(),
store.load_all_instances(),
);
// An unreadable record is not a deletion. Its instance keeps whatever the
// data plane already holds, so it must be exempt from the removal pass
// below; a record that lost an IP or key conflict is not exempt, because
Expand Down Expand Up @@ -1989,6 +2005,13 @@ impl ProxyState {
{
bail!("WireGuard public key is already registered to another instance");
}
// Every node refuses such a record on import, so accepting it here
// would leave this node alone in routing it.
if public_key == self.config.wg.public_key
|| self.kv_store.gateway_wg_public_keys().contains(public_key)
{
bail!("WireGuard public key belongs to a gateway node");
}
// The rebuild path below reconstructs the instance from scratch -- it
// does that when the recorded IP is no longer inside the configured
// range -- and everything the registration does not re-state has to
Expand Down
31 changes: 31 additions & 0 deletions dstack/gateway/src/main_service/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,37 @@ async fn test_wireguard_public_key_cannot_be_reused_by_another_instance() {
assert!(!state.lock().state.instances.contains_key("instance-2"));
}

/// Every node refuses a record carrying a gateway node's key on import, so the
/// node taking the registration must refuse it too, or it alone routes it.
#[tokio::test]
async fn a_gateway_nodes_public_key_does_not_register() {
let state = create_test_state_with(|config| {
config.wg.public_key = test_pubkey("own-gateway");
})
.await;
let peer_gateway = test_pubkey("peer-gateway");
state
.kv_store
.sync_node(
7,
&crate::kv::NodeData {
uuid: b"gw7-uuid".to_vec(),
url: "https://gw7.example.com:9202".to_string(),
wg_public_key: peer_gateway.clone(),
wg_endpoint: String::new(),
wg_ip: String::new(),
},
)
.unwrap();
let own = state.config.wg.public_key.clone();

for (id, key) in [("steals-peer-key", &peer_gateway), ("steals-own-key", &own)] {
let result = state.do_register_cvm("app", id, key, "compose", Default::default());
assert!(result.is_err(), "{id} registered");
assert!(!state.lock().state.instances.contains_key(id));
}
}

#[tokio::test]
async fn test_port_policy_restrict_mode_allows_listed_only() {
let state = create_test_state().await;
Expand Down
Loading