Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
114 changes: 69 additions & 45 deletions fork-kit/golive/README.md

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion fork-kit/golive/lib/golive/back.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@
UPDATE_TIMEOUT, HTTP_TIMEOUT, MB = 900, 5, 1024 * 1024
T3_STUB = None # tests only: a program run in place of any t3 the kit runs
FOLLOWS = ('back', 'switch')
_FORK = re.compile(r'\.fork\.\d+$')
_FORK = live.FORK_BUILD


def base_of(version):
Expand Down
48 changes: 42 additions & 6 deletions fork-kit/golive/lib/golive/check.py
Original file line number Diff line number Diff line change
Expand Up @@ -211,7 +211,8 @@ def all_ok():


STEADY = ('boot service', 'live server', 'no update in flight', 'no half switch', 'no markers')
REPORT_ANSWERS = ('counts', 'pendingMigrations', 'providers') # what _report takes from the report into the result
# What _report takes from the report into the result.
REPORT_ANSWERS = ('counts', 'pendingMigrations', 'providers', 'features')


def server_changed(cfg, adapter, server):
Expand Down Expand Up @@ -263,11 +264,10 @@ def _report(cfg, report, result, add):
version = report.get('version')
add('report: target build', version == cfg.target_version, f'the staged build says it is {version}')
fork = report.get('fork') if isinstance(report.get('fork'), dict) else {}
on = fork.get('featuresOn')
if not isinstance(on, list):
add('fork features off', False, 'the report has no fork.featuresOn list')
else:
add('fork features off', not on, f'switched on in settings: {live.features_on(fork)}' if on else 'all off')
state, said = features_answer(cfg, fork, result['live'])
if state is not None: # the switch keeps them as they are: its re-read, `before` and verify require this
result['features'] = state
add('fork features', state is not None, said)
problems = report.get('problems') if isinstance(report.get('problems'), list) else []
named = '; '.join(f'{p.get("code")}: {p.get("message")}' for p in problems if isinstance(p, dict))
readable = report.get('ok') is True
Expand All @@ -279,6 +279,42 @@ def _report(cfg, report, result, add):
result['providers'] = providers_sum(report) # switch's verify: the same after as before


def features_answer(cfg, target_fork, server):
"""(the fork features the switch keeps, what check says) or (None, why it refuses), from the staged target
build's report `target_fork` on the live home and, when the live build `server` is a fork build, its own report
(run beside the running T3, like the target's): a switch keeps the features as they are only when both builds
read the same values in the same places. A target that reads more (a new feature on by default, a key or a
policy default the old build read otherwise, `usagePacingGate` advise -> enforce) refuses before anything
stops. A live build that gives no report of its own (an older report version) passes only with all off."""
state, version = live.features_state(target_fork), server['version']
if state is None:
return None, 'the report has no fork.featuresOn list and fork.features map'
if not live.fork_build(version):
if state['on']:
return None, (f'switched on in settings while the live build {version} is not a fork build: the first '
f'switch starts with all off ({live.features_on(target_fork)})')
return state, 'all off'
try:
own = live.fork_report(server['binary'], cfg.home)
was, why = live.features_state(own.get('fork')), 'it has no fork.featuresOn list and fork.features map'
except live.NotLive as e:
was, why = None, str(e)
if was is None:
if state['on']:
return None, (f'the live build {version} gives no report of its own fork features ({why}), so the kit '
'cannot tell whether the target keeps them as they are, and the target reads some on '
f'({live.features_on(target_fork)}): switch them off first, as for a first switch')
return state, f'all off (the live build {version} gives no report of its own: {why})'
differ = live.features_differ(was, state, f'in the live {version}', 'in the target')
if differ:
return None, (f'the target build reads other fork features than the live build: {differ}; the switch would '
'go live with them changed: set them in settings so both builds read the same, or switch '
'them off')
if state['on']:
return state, f'on, kept as they are (the live build reads the same): {live.features_on(target_fork)}'
return state, 'all off (the live build reads the same)'


def providers_sum(report):
"""SHA256 of the report's provider instances (only the sum: the journal and --json never hold settings)."""
settings = report.get('settings') if isinstance(report.get('settings'), dict) else {}
Expand Down
47 changes: 47 additions & 0 deletions fork-kit/golive/lib/golive/live.py
Original file line number Diff line number Diff line change
Expand Up @@ -341,6 +341,53 @@ def counts(report):
return said


FORK_BUILD = re.compile(r'\.fork\.\d+$') # a fork build's version: its upstream nightly's, then `.fork.N`


def fork_build(version):
"""Whether `version` names a fork build (it has a fork report and fork features), not an upstream one."""
return isinstance(version, str) and FORK_BUILD.search(version) is not None


def features_list(fork):
"""The report's `featuresOn` when it is a list of names, else None (a report without it says nothing)."""
on = fork.get('featuresOn') if isinstance(fork, dict) else None
return list(on) if isinstance(on, list) and all(isinstance(n, str) for n in on) else None


def features_state(fork):
"""What a report's `fork` says the fork features are, in a form that compares whole and in any order:
{'values': every feature's value (`usagePacingGate: advise` and `enforce` both count as on, so the names alone
do not say which), 'on': the names on, sorted, 'where': each feature's places (server, project:<id>,
thread:<id>, policy), sorted}. None when the report has no `featuresOn` list or no `features` map."""
on = features_list(fork)
values = fork.get('features') if isinstance(fork, dict) else None
if on is None or not isinstance(values, dict):
return None
where = fork.get('featuresWhere') if isinstance(fork.get('featuresWhere'), dict) else {}
return {'values': {str(k): values[k] for k in sorted(values, key=str)}, 'on': sorted(on),
'where': {str(k): sorted(map(str, where[k])) for k in sorted(where, key=str)
if isinstance(where[k], list) and where[k]}}


def features_differ(was, now, was_said='before', now_said='now'):
"""How two features_state()s differ, feature by feature ('' when they are the same)."""
if was is None or now is None:
return '' if was == now else f'no fork feature list {was_said if was is None else now_said}'
said = []
for name in sorted(set(was['values']) | set(now['values']) | set(was['on']) | set(now['on'])):
a, b = (json.dumps(s['values'].get(name)) if name in s['values'] else 'not in its report' for s in (was, now))
if a != b:
said.append(f'{name}: {a} {was_said}, {b} {now_said}')
elif (name in was['on']) != (name in now['on']):
said.append(f'{name}: {"on" if name in was["on"] else "off"} {was_said}, '
f'{"on" if name in now["on"] else "off"} {now_said}')
elif was['where'].get(name, []) != now['where'].get(name, []):
said.append(f'{name}: on in {", ".join(was["where"].get(name, [])) or "no place"} {was_said}, '
f'in {", ".join(now["where"].get(name, [])) or "no place"} {now_said}')
return '; '.join(said)


def features_on(fork):
"""The report's fork features that are on, each with where (`handoff (project:p1, thread:t1)`), so an operator
knows which project override or thread entry to clear. A report without `featuresWhere` names the features only."""
Expand Down
75 changes: 74 additions & 1 deletion fork-kit/golive/lib/golive/rehearse.py
Original file line number Diff line number Diff line change
Expand Up @@ -474,6 +474,73 @@ def _r4(cfg, before, after):
return said


def _off_in(settings, fork):
"""Removes from the settings file's object `settings` every value the report `fork` says switches a feature on
(a missing value reads as off): the server's own (a `thread:<id>` entry is in it) and a project override's.
A place the machine's policy locks (`policy`) is left: no edit of settings.json switches it off. Returns what
went, as `key` or `project:<id>/key`."""
where = fork.get('featuresWhere') if isinstance(fork.get('featuresWhere'), dict) else {}
overrides = settings.get('projectSettingsOverrides')
removed = []
for name in live.features_list(fork) or []:
for place in where.get(name) if isinstance(where.get(name), list) else []:
place = str(place)
if place == 'server' or place.startswith('thread:'):
if name in settings:
del settings[name]
removed.append(name)
elif place.startswith('project:'):
project = place[len('project:'):].split('/thread:')[0]
override = overrides.get(project) if isinstance(overrides, dict) else None
if isinstance(override, dict) and name in override:
del override[name]
removed.append(f'project:{project}/{name}')
return removed


def _features_off(p, before):
"""R3 runs a server on the clone: with a fork feature on (waitFor, handoff, ...) it could act, deliver waits or
send messages. So every feature on in the clone is switched off in the clone's settings.json (never the home's;
the restore puts the copy's back), and a report must then read them all off. Returns (facts, the report)."""
fork = before.get('fork') if isinstance(before.get('fork'), dict) else {}
on = live.features_list(fork)
if on is None:
raise Failed('R3: the report on the clone has no fork.featuresOn list: the rehearsal cannot tell whether '
'its server would act; nothing ran on it')
facts = {'featuresOn': on, 'where': live.features_on(fork), 'removed': []}
if not on:
return facts, before
path = os.path.join(p['home'], 'userdata', 'settings.json')
if os.path.islink(path):
raise Failed(f'R3: fork features are on in the clone ({facts["where"]}) and {path} is a link: the kit '
'writes no settings through a link; nothing ran on it')
try:
with open(path, encoding='utf-8') as f:
settings = json.load(f)
except FileNotFoundError:
settings = None # on by the machine's policy alone: the report below names it
except (OSError, ValueError) as e:
raise Failed(f'R3: fork features are on in the clone ({facts["where"]}) and its settings.json does not read '
f'as plain JSON ({e}), so the kit cannot switch them off; switch them off by hand in the copy '
'or rehearse after they are off; nothing ran on it') from None
if isinstance(settings, dict):
facts['removed'] = _off_in(settings, fork)
tmp = f'{path}.golive-features-off'
with open(tmp, 'w', encoding='utf-8') as f:
json.dump(settings, f, indent=2)
f.write('\n')
f.flush()
os.fsync(f.fileno())
os.replace(tmp, path)
after = _report(p)
still = live.features_list(after.get('fork') if isinstance(after.get('fork'), dict) else {})
if still != []:
raise Failed(f'R3: fork features stay on in the clone after the kit cleared its settings.json '
f'({live.features_on(after.get("fork") or {}) or still}): a policy or a value the kit cannot '
'clear switches them on, and the rehearsal server could act on them; nothing ran on it')
return facts, after


def _serve(p, empty):
log = open(os.path.join(p['root'], 'server.log'), 'wb')
try:
Expand Down Expand Up @@ -529,13 +596,19 @@ def run_rehearse(cfg, copy_dir=None, port=None, say=print):
for line in result['leftOut']:
say(f'left out of the clone: {line}')
_r0_clone(cfg, p)
settings = coldcopy._file_sum(os.path.join(p['home'], 'userdata', 'settings.json'))
with run.step('preflight') as facts:
for name, binary, version in (('old', p['old'], p['from']), ('fork', p['fork'], cfg.target_version)):
facts[name], why = _preflight(binary, p['home'], p['protocol'], version)
if facts[name] != 'ready':
raise Failed(f'R1: the preflight of t3 {version} on the clone says {facts[name]}: {why}')
before = _report(p)
with run.step('features-off') as facts: # the clone's own settings only; the restore puts the copy's back
off, before = _features_off(p, before)
facts.update(off)
result['featuresOff'] = off
if off['featuresOn']:
say(f'R3: the fork features on in the copy ({off["where"]}) are switched off in the clone only')
settings = coldcopy._file_sum(os.path.join(p['home'], 'userdata', 'settings.json'))
result['migrations'] = list(_db(before).get('pendingMigrations') or [])
with run.step('migrate') as facts:
size, t0 = live.database_bytes(p['home']), time.monotonic()
Expand Down
Loading
Loading