Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions fork-kit/golive/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -329,8 +329,11 @@ start: the new launcher then idled and verify waited for minutes.
and the kit must not take that probe for the launcher (it would free the home in the middle of `t3 update`). A
rule of "held on every poll for 100 ms" could not tell frequent probes from the launcher when load stretched the
polls (#546). For an older build the kit lets go just before the start, as before (its server would meet the
kit's lock and idle). The verify step journals how the lock was let go (`handover`: `launcher`, `timeout`,
`start failed`, `before the start ...`) and any marker renewal that failed (`renewFailed`).
kit's lock and idle). The kit lets go at its first look (every 20 ms) after the launcher's claim, well inside
the 250 ms its server waits for the owner lock (#534). The verify step journals how the lock was let go
(`handover`: `launcher`, `timeout`, `start failed`, `before the start ...`), for `launcher` the measured delay
(`handoverMs`: at most this many ms from the claim to the let-go; with `handoverLate` when it reached 250 ms, so
a server that idled on a very busy host is named), and any marker renewal that failed (`renewFailed`).
- **Backstop, a foreign server.** An SSH client older than 0.1.10 reads neither, and a server started by hand
ignores both. When the runtime record names a live process that the boot service did not start and that itself
holds `userdata/server-owner.sqlite` (a stale record whose pid was reused holds nothing and is never named), for
Expand Down
28 changes: 24 additions & 4 deletions fork-kit/golive/lib/golive/handoff.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@
# Seconds between two looks for the launcher. It holds the serving lock from #claimServing on, and its server then
# waits 250 ms for the owner lock (serverOwnership.ts busy_timeout): the kit lets go well inside that.
HANDOVER_POLL = 0.02
SERVER_LOCK_WAIT_MS = 250 # serverOwnership.ts busy_timeout (contract.test.py): a later handover may leave it idle
FOREIGN_GRACE = 10.0 # seconds a server not started by the boot service must stay before verify stops on it
_now_ms = lambda: int(time.time() * 1000) # noqa: E731 (Date.now(), as upstream's reader compares)

Expand Down Expand Up @@ -147,6 +148,9 @@ def __init__(self, home):
# how the last starting() let go of the owner lock: 'launcher' (it held its serving lock), 'timeout' (none did
# within HANDOVER_WAIT), 'start failed', 'before the start' (a build that takes no serving lock); None: none
self.handover = None
# the last 'launcher' handover: at most this many ms between the launcher's claim of the serving lock and the
# kit letting go of the owner lock (from the start of the kit's last look that did not find it serving)
self.handover_ms = None
self._stop = threading.Event()
self._thread = None
self._mutex = threading.Lock()
Expand Down Expand Up @@ -184,8 +188,10 @@ def let_go(self):
def starting(self, fn, version, service):
"""Run fn (it starts T3 `version`: a service start, or `t3 update`), and let go of the owner lock only once
the new launcher holds its serving lock (or HANDOVER_WAIT after fn returned, or fn failed). service: reads
the boot service (adapter.service). Returns fn's result. Afterwards the kit holds no lock on the home."""
self.handover = None
the boot service (adapter.service). Returns fn's result. Afterwards the kit holds no lock on the home.
A handover to the launcher measures its delay (handover_ms, an upper bound: the claim came after the start of
the last look that did not find the launcher serving, and the lock was let go before the clock is read)."""
self.handover, self.handover_ms = None, None
if self.lock is not None and not takes_serving_lock(version):
self.let_go()
self.handover = 'before the start (a build that takes no serving lock)'
Expand All @@ -196,16 +202,21 @@ def starting(self, fn, version, service):
serving = os.path.join(self.home, 'runtime', SERVING_LOCK)

def watch(): # HANDOVER_WAIT counts from fn's return (`t3 update` installs first, then restarts T3)
looked = time.monotonic() # the start of the last look that did not find the launcher serving
try:
while not done.is_set():
began = time.monotonic()
if lock_holders(serving): # someone holds it: the launcher, or an SSH launch's probe
try:
svc = service()
except Exception: # noqa: BLE001 (systemctl failed for a moment: look again)
svc = {}
if launcher_serving(self.home, svc):
outcome['how'] = 'launcher'
self.let_go()
outcome['ms'] = (time.monotonic() - looked) * 1000
return
looked = began
if returned and time.monotonic() >= returned[0] + HANDOVER_WAIT:
outcome['how'] = 'timeout'
return
Expand All @@ -225,6 +236,8 @@ def watch(): # HANDOVER_WAIT counts from fn's return (`t3 update` installs firs
finally:
watcher.join()
self.handover = outcome.get('how', 'start failed')
if 'ms' in outcome:
self.handover_ms = round(outcome['ms'])

def close(self):
"""Stop renewing, let go of any lock. The marker stays to run out by itself (within 60 s)."""
Expand All @@ -238,14 +251,21 @@ def close(self):

def served(self):
"""The new server is verified: stop renewing and remove the marker if it is still the kit's. Returns the
facts for the verify step: 'marker' (what was done to it), 'handover' (how the owner lock was let go), and
'renewFailed' when a renewal failed (the marker may have run out mid-stop; the owner lock still held).
facts for the verify step: 'marker' (what was done to it), 'handover' (how the owner lock was let go),
'handoverMs' (handover_ms; with 'handoverLate' when it reached the server's 250 ms wait), and 'renewFailed'
when a renewal failed (the marker may have run out mid-stop; the owner lock still held).
A newer marker renamed in between the read and the unlink would still be removed: a window of microseconds,
the same as upstream's own DesktopBackendPool.releaseUpdateMarker."""
self.close()
facts = {'marker': self._remove()}
if self.handover is not None:
facts['handover'] = self.handover
if self.handover_ms is not None: # the journal keeps how close the handover came to the server's wait
facts['handoverMs'] = self.handover_ms
if self.handover_ms >= SERVER_LOCK_WAIT_MS:
facts['handoverLate'] = (f'the kit let go up to {self.handover_ms} ms after the launcher took its '
f'serving lock; its server waits {SERVER_LOCK_WAIT_MS} ms for the owner lock '
'and may have idled (the home is then free until its launcher tries again)')
if self.failed:
facts['renewFailed'] = f'{self.failed} time(s), last: {self.error}'
return facts
Expand Down
95 changes: 88 additions & 7 deletions fork-kit/golive/tests/gap.test.py
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,23 @@ def home_owned(home, hold=0.0):
PROBING_LIMIT = 300 # seconds: a probing phase that has not ended by then fails its test instead of hanging it


def kit_looks_for_the_launcher(test):
"""[(monotonic time a look began, whether it found the launcher)]: each of the kit's handover looks that found
the serving lock held (handoff.launcher_serving), recorded as it ran."""
from golive import handoff
looks, real = [], handoff.launcher_serving

def spy(home, svc):
began = time.monotonic()
found = real(home, svc)
looks.append((began, found))
return found
patcher = mock.patch.object(handoff, 'launcher_serving', spy)
patcher.start()
test.addCleanup(patcher.stop)
return looks


def kit_meets_probes(test):
"""An Event set once the kit's handover poll has found the serving lock held by another process than this
one (an SSH probe: this process is the fake launcher), so a probing phase surely tested what it is about."""
Expand Down Expand Up @@ -681,7 +698,8 @@ def test_a_probe_loop_on_the_serving_lock_is_not_the_launcher(self):
g = self.h.Gap(self.m.home)
g.adopt(coldcopy.hold_owner_lock(userdata, 0))
met = kit_meets_probes(self)
stop, probing, probes, free = threading.Event(), threading.Event(), [], []
looks = kit_looks_for_the_launcher(self)
stop, probing, probes, free, claimed = threading.Event(), threading.Event(), [], [], []

def ssh_probes():
try:
Expand All @@ -701,21 +719,84 @@ def start(): # `t3 update`: it installs while launches probe, then its launcher
t.join()
launcher = sqlite3.connect(serving, isolation_level=None, timeout=1)
launcher.execute('BEGIN EXCLUSIVE')
claimed = time.monotonic()
for _ in range(300):
if not held(owner):
break
claimed.append(time.monotonic())
end = claimed[0] + PROBING_LIMIT
while held(owner) and time.monotonic() < end:
time.sleep(0.005)
free.append(time.monotonic() - claimed)
free.append(not held(owner))
launcher.close()
t = threading.Thread(target=ssh_probes, daemon=True)
t.start()
g.starting(start, NEW, self.service)
self.assertEqual(len(free), 1, 'the kit let go of the owner lock while only SSH probes held the serving lock')
self.assertTrue(free[0], 'the kit kept the owner lock after the launcher claimed the serving lock')
self.assertGreaterEqual(len(probes), PROBES)
self.assertTrue(met.is_set(), 'the kit never met an SSH probe of the serving lock')
self.assertLess(free[0], 0.25) # within its server's 250 ms wait for the owner lock
# #534: let go at the kit's first look after the claim, counted in looks, not in wall-clock time (a slow
# runner stretches both alike): every look that began after the claim found the launcher, and there was one
self.assertEqual([found for at, found in looks if at >= claimed[0]], [True])
self.assertEqual(g.handover, 'launcher')
self.assertIsInstance(g.handover_ms, int)

def test_the_handover_delay_is_measured_and_journalled(self):
"""#534: the verify step journals how long the kit kept the owner lock after the launcher's claim (an upper
bound), and names a handover that reached the 250 ms its server waits for the owner lock."""
userdata = self.m.path('home/userdata')
os.makedirs(userdata)
serving = self.m.path('home/runtime/service-serving.lock')
owner = os.path.join(userdata, 'server-owner.sqlite')
claims = []

def launcher(): # claims the serving lock and keeps it until the kit let go of the owner lock
c = sqlite3.connect(serving, isolation_level=None, check_same_thread=False)
c.execute('BEGIN EXCLUSIVE')
claims.append(c)
# (seconds a look that found the launcher takes, the server's wait in ms): the lateness is fixed by the wait,
# never by how fast this runner is: 0.3 s is surely past 250 ms, and no runner reaches a wait of 10**9 ms
for slow, wait, late in ((0.0, 10**9, False), (0.3, self.h.SERVER_LOCK_WAIT_MS, True)):
real = self.h.launcher_serving

def seen(home, svc, slow=slow, real=real): # a look stretched by load, after the claim
found = real(home, svc)
if found:
time.sleep(slow)
return found
g = self.h.open_gap(self.m.home)
g.adopt(coldcopy.hold_owner_lock(userdata, 0))
with mock.patch.object(self.h, 'launcher_serving', seen), \
mock.patch.object(self.h, 'SERVER_LOCK_WAIT_MS', wait):
g.starting(launcher, NEW, self.service)
self.assertFalse(held(owner))
facts = self.h.served(self.m.home)
self.assertEqual(facts['handover'], 'launcher')
self.assertIsInstance(facts['handoverMs'], int)
self.assertGreaterEqual(facts['handoverMs'], slow * 1000) # an upper bound: it covers the slow look
self.assertEqual('handoverLate' in facts, late, facts)
if late:
self.assertIn(f'up to {facts["handoverMs"]} ms after the launcher took its serving lock',
facts['handoverLate'])
claims.pop().close()
g = self.h.open_gap(self.m.home) # no launcher: nothing to measure
g.adopt(coldcopy.hold_owner_lock(userdata, 0))
with mock.patch.object(self.h, 'HANDOVER_WAIT', 0.05):
g.starting(lambda: None, NEW, self.service)
self.assertIsNone(g.handover_ms) # a run again: the last handover's delay is not carried over
facts = self.h.served(self.m.home)
self.assertEqual(facts['handover'], 'timeout')
self.assertNotIn('handoverMs', facts)
g = self.h.open_gap(self.m.home) # a start that failed half-way: nothing to measure
g.adopt(coldcopy.hold_owner_lock(userdata, 0))
g.handover_ms = 7 # left by an earlier start of this gap

def fails():
raise OSError('the start failed')
with self.assertRaises(OSError):
g.starting(fails, NEW, self.service)
self.assertFalse(held(owner))
facts = self.h.served(self.m.home)
self.assertEqual(facts['handover'], 'start failed')
self.assertNotIn('handoverMs', facts)
self.assertEqual(self.h.served(self.m.home), {'marker': None}) # no gap open: nothing measured

def test_only_the_service_s_main_process_holding_the_serving_lock_is_the_launcher(self):
"""However long another process holds the serving lock, the kit keeps the owner lock; it lets go at once
Expand Down
Loading