Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 17 additions & 3 deletions dstack/gateway/src/main_service.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1776,7 +1776,7 @@ fn reload_instances_from_kv_store(proxy: &Proxy, store: &KvStore) -> Result<()>
// Release old IP if it changed (prevent IP leak)
if let Some(existing) = &existing {
if existing.ip != new_info.ip {
state.state.allocated_addresses.remove(&existing.ip);
state.release_ip(existing.ip, &instance_id);
}
if existing.app_id != new_info.app_id {
if let Some(app_instances) = state.state.apps.get_mut(&existing.app_id) {
Expand Down Expand Up @@ -1925,6 +1925,20 @@ impl ProxyState {
fn valid_ip(&self, ip: Ipv4Addr) -> bool {
self.config.wg.is_valid_client_ip(ip)
}
/// Return `ip` to the pool unless an instance other than `id` still holds
/// it. Two live instances can share an address while a conflict loser is
/// kept through its grace period, or while its record is unreadable.
fn release_ip(&mut self, ip: Ipv4Addr, id: &str) {
let held = self
.state
.instances
.iter()
.any(|(other, info)| other != id && info.ip == ip);
if !held {
self.state.allocated_addresses.remove(&ip);
}
}

fn alloc_ip(&mut self) -> Option<Ipv4Addr> {
for ip in self.config.wg.client_ip_range.hosts() {
if !self.valid_ip(ip) {
Expand Down Expand Up @@ -2106,7 +2120,7 @@ impl ProxyState {
return Ok(existing);
}
info!("ip {} is invalid, removing", existing.ip);
self.state.allocated_addresses.remove(&existing.ip);
self.release_ip(existing.ip, id);
previous = Some(existing);
}
let ip = self
Expand Down Expand Up @@ -2701,7 +2715,7 @@ impl ProxyState {
/// counterpart is [`Self::remove_instance`].
fn forget_instance(&mut self, id: &str) -> Option<InstanceInfo> {
let info = self.state.instances.remove(id)?;
self.state.allocated_addresses.remove(&info.ip);
self.release_ip(info.ip, id);
self.state.top_n.remove(&info.app_id);
if let Some(app_instances) = self.state.apps.get_mut(&info.app_id) {
app_instances.remove(id);
Expand Down
39 changes: 39 additions & 0 deletions dstack/gateway/src/main_service/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2184,6 +2184,45 @@ fn sync_from_peer_at(
.unwrap();
}

/// A local registration that loses an IP conflict to an older peer record is
/// kept through its grace period, so two live instances share the address for
/// a while. Dropping the loser must not free the address the winner still uses.
#[tokio::test]
async fn forgetting_a_conflict_loser_keeps_the_winners_ip_allocated() {
let state = create_test_state().await;
let ip = state
.lock()
.new_client_by_id(
"local",
"local-app",
&test_pubkey("local-key"),
"",
Default::default(),
)
.unwrap()
.ip;
sync_from_peer_at(&state, "peer", &ip.to_string(), &test_pubkey("peer-key"), 1);
reload_instances_from_kv_store(&state.proxy, &state.kv_store).unwrap();
{
let proxy = state.lock();
assert_eq!(proxy.state.instances["peer"].ip, ip);
assert_eq!(proxy.state.instances["local"].ip, ip);
}
state.proxy.remove_cvm("local").unwrap();

let next = state
.lock()
.new_client_by_id(
"next",
"next-app",
&test_pubkey("next-key"),
"",
Default::default(),
)
.unwrap();
assert_ne!(next.ip, ip, "handed out an IP a live instance still holds");
}

#[tokio::test]
async fn a_poisoned_peer_record_costs_only_its_own_instance() {
let state = create_test_state().await;
Expand Down
Loading