Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion fork-kit/desk/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,8 @@ distro (`-Distro`, default `fork-desk`, Ubuntu 24.04, installed with `--version
leaves a mark in `%LOCALAPPDATA%\fork-desk`, so the next run asks no takeover question, and a broken one names
`wsl --unregister DISTRO`); `.wslconfig`'s `[wsl2]` memory cap (`-MemoryGB`, default
three quarters of the RAM, at most the RAM less 6 GB, never under 4 GB; one that leaves Windows under 2 GB is
refused; it is the least cap: a higher one already there, raised by hand, is kept), swap (`-SwapGB`, 4), NAT and DNS tunnelling, for every WSL distro (read and written as UTF-8 without a
refused; without `-MemoryGB`, a higher one already there, raised by hand, is kept up to the RAM less 2 GB; one
WSL cannot read, such as `26 GB` or `26gb`, is rewritten, since WSL would run on half the RAM), swap (`-SwapGB`, 4), NAT and DNS tunnelling, for every WSL distro (read and written as UTF-8 without a
BOM; a dated copy of the old file is kept, other keys and sections stay; an old `networkingMode` or `dnsTunneling`
line under `[experimental]`, the form before WSL 2.0 that can still win over `[wsl2]`, is removed, and the check
FAILs while one is there; the question lists each change, old -> new, and the file is written only when it lists one,
Expand Down
45 changes: 29 additions & 16 deletions fork-kit/desk/tests/windows.test.py
Original file line number Diff line number Diff line change
Expand Up @@ -363,31 +363,42 @@ def test_wslconfig_merge_removes_old_experimental_networking_keys(self):
self.assertIn(line, r.stdout)

@unittest.skipUnless(shutil.which('pwsh'), 'pwsh is not installed')
def test_a_memory_cap_at_or_above_the_minimum_is_kept(self):
"""-MemoryGB is the least cap: one a person raised (more agents or jobs in WSL) is neither lowered nor a FAIL,
nor a `wsl --shutdown` on every run (the live check's cap is the one kept)."""
def test_the_memory_cap_kept_is_one_wsl_reads_within_its_bounds(self):
"""Without -MemoryGB, a higher cap a person set (more agents or jobs in WSL) is neither lowered nor a FAIL, nor
a `wsl --shutdown` on every run (the live check's cap is the one kept), up to the RAM less 2 GB. With
-MemoryGB, that is the cap. A value WSL itself cannot read (it then runs on half the RAM) is rewritten: its
parser takes a number and B, K, KB, M, MB, G, GB, T or TB exactly, no space, upper case (#296)."""
d = tempfile.mkdtemp(prefix='desk-wslconfig-test-')
self.addCleanup(shutil.rmtree, d, True)
path = os.path.join(d, '.wslconfig')
script = os.path.join(d, 'memory.ps1')
with open(script, 'w', encoding='ascii') as f:
f.write(wslconfig_part() + '\n$want = [ordered]@{ memory = (WantMemory $args[0] 8); swap = "4GB" }\n'
f.write(wslconfig_part() + '\n$want = [ordered]@{ memory = (WantMemory $args[0] 8 $args[1]); swap = "4GB" }\n'
'"want=$($want.memory) kb=$(CapKb $want.memory)"\n'
'"notes=$(@(Changes (Read-WslConfig $args[0]) $want) -join \'|\')"\n'
'$null = Merge-WslConfig $args[0] $want\n$cfg = Wsl2Keys (Read-WslConfig $args[0])\n'
'"check=$(@($want.Keys | Where-Object { $cfg[$_] -ne $want[$_] }).Count)"\n')
for old, want, kb, note in (('memory=26GB', '26GB', 27262976, ''), # raised: kept, no change, no FAIL
('memory = 30720mb', '30720mb', 31457280, ''),
('memory=8GB', '8GB', 8388608, ''),
('memory=9G', '9G', 9437184, ''),
('memory=4GB', '8GB', 8388608, '[wsl2] memory: 4GB -> 8GB'), # below: raised
('memory=8191MB', '8GB', 8388608, '[wsl2] memory: 8191MB -> 8GB'),
('memory=lots', '8GB', 8388608, '[wsl2] memory: lots -> 8GB'),
('', '8GB', 8388608, '[wsl2] memory: (not set) -> 8GB')):
with self.subTest(old):
for old, most, want, kb, note in (
('memory=26GB', 30, '26GB', 27262976, ''), # raised: kept, no change, no FAIL
('memory=30GB', 30, '30GB', 31457280, ''), # the RAM less 2 GB
('memory = 30720MB', 30, '30720MB', 31457280, ''),
('memory=8GB', 30, '8GB', 8388608, ''),
('memory=9G', 30, '9G', 9437184, ''),
('memory=31GB', 30, '8GB', 8388608, '[wsl2] memory: 31GB -> 8GB'), # Windows would keep under 2 GB
('memory=64GB', 30, '8GB', 8388608, '[wsl2] memory: 64GB -> 8GB'), # from a bigger PC
('memory=16777216TB', 30, '8GB', 8388608, '[wsl2] memory: 16777216TB -> 8GB'), # WSL: 0, its default
('memory=26 GB', 30, '8GB', 8388608, '[wsl2] memory: 26 GB -> 8GB'), # WSL rejects these two
('memory=30720mb', 30, '8GB', 8388608, '[wsl2] memory: 30720mb -> 8GB'),
('memory=4GB', 30, '8GB', 8388608, '[wsl2] memory: 4GB -> 8GB'), # below: raised
('memory=8191MB', 30, '8GB', 8388608, '[wsl2] memory: 8191MB -> 8GB'),
('memory=lots', 30, '8GB', 8388608, '[wsl2] memory: lots -> 8GB'),
('', 30, '8GB', 8388608, '[wsl2] memory: (not set) -> 8GB'),
('memory=26GB', 8, '8GB', 8388608, '[wsl2] memory: 26GB -> 8GB'), # -MemoryGB 8 lowers it
('memory=8192MB', 8, '8192MB', 8388608, '')): # -MemoryGB 8, the same size: left as written
with self.subTest(old=old, most=most):
with open(path, 'w', encoding='utf-8', newline='\n') as f:
f.write(f'[wsl2]\n{old}\nswap=4GB\n')
r = subprocess.run(['pwsh', '-NoProfile', '-NonInteractive', '-File', script, path],
r = subprocess.run(['pwsh', '-NoProfile', '-NonInteractive', '-File', script, path, str(most)],
capture_output=True, text=True, timeout=120, stdin=subprocess.DEVNULL)
self.assertEqual(r.returncode, 0, r.stdout + r.stderr)
self.assertIn(f'want={want} kb={kb}\n', r.stdout)
Expand All @@ -396,10 +407,12 @@ def test_a_memory_cap_at_or_above_the_minimum_is_kept(self):
if not note:
self.assertEqual(slurp(path), f'[wsl2]\n{old}\nswap=4GB\n') # left alone

def test_the_minimum_memory_cap_decides_through_wantmemory(self):
def test_the_memory_cap_decides_through_wantmemory(self):
src = text().decode('ascii')
body = src[src.index('try {'):]
self.assertIn('$want = [ordered]@{ memory = (WantMemory $WslConfig $MemoryGB);', body)
# -MemoryGB given: it is the cap; not given: a higher one is kept up to what leaves Windows 2 GB
self.assertIn('$most = if ($asked) { $MemoryGB } else { $ram - 2 }', body)
self.assertIn('$want = [ordered]@{ memory = (WantMemory $WslConfig $MemoryGB $most);', body)
self.assertIn('("WANT_KB=$(CapKb $want.memory)`n" + $Live)', body) # the cap kept, not -MemoryGB
self.assertNotIn('$MemoryGB * 1048576', body)

Expand Down
23 changes: 14 additions & 9 deletions fork-kit/desk/windows/start-desk.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
# (fork-kit/desk/README.md, "Windows"). Run it in a normal Windows PowerShell window (not as administrator):
# powershell -ExecutionPolicy Bypass -File start-desk.ps1 -Repo OWNER/REPO -Sha RELEASE_SHA -MachineFile machine.json
# [-FleetConfig fork-fleet.json] [-Distro fork-desk] [-MemoryGB N] [-SwapGB 4] [-Uninstall]
# -MemoryGB is the least [wsl2] memory cap: a higher one already in .wslconfig is kept.
# -MemoryGB is the [wsl2] memory cap; without it, a higher one already in .wslconfig is kept, up to the RAM less 2 GB.
# It prints this PC's facts and asks before it changes anything. Safe to run again: every step checks what is there.
# Steps: the distro (Ubuntu 24.04); .wslconfig's [wsl2] memory cap and NAT (a dated copy of the old file is kept;
# other keys stay); the distro's own settings (the WSL walls: no interop, no Windows PATH or drives, its own
Expand Down Expand Up @@ -162,16 +162,19 @@ function Merge-WslConfig([string]$path, $want, [scriptblock]$before = {}) { #
Move-Item -LiteralPath "$path.new" $path -Force
$true
}
# A [wsl2] memory= value in kB (8GB, 8192MB, 8G; no unit: bytes), or -1 when it is not a size
# A [wsl2] memory= value in kB (8GB, 8192MB, 8G; no unit: bytes), or -1 when WSL cannot read it. WSL's parser
# (ParseMemorySize) takes a number and B, K, KB, M, MB, G, GB, T or TB exactly: no space before the unit, upper case
# (`26 GB` or `26gb` makes it warn and run on half the RAM).
function CapKb([string]$v) {
if ($v -notmatch '^\s*([0-9]{1,13})\s*([KMGT]?)B?\s*$') { return [long]-1 }
[long][math]::Min(1e15, [math]::Floor([double]$Matches[1] * [math]::Pow(1024, ' KMGT'.IndexOf($Matches[2].ToUpper())) / 1024))
if ($v -cnotmatch '^\s*([0-9]{1,13})([KMGT]?)B?\s*$') { return [long]-1 }
[long][math]::Min(1e15, [math]::Floor([double]$Matches[1] * [math]::Pow(1024, ' KMGT'.IndexOf($Matches[2])) / 1024))
}
# The memory cap to want: at least $gb GB. A higher one already in .wslconfig (a person raised it) is kept as it is
# written, so it is neither lowered nor a FAIL, and the live check holds WSL to it.
function WantMemory([string]$path, [int]$gb) {
# The memory cap to want: $gb GB, or the one already in .wslconfig, kept as it is written (neither lowered nor a FAIL,
# and the live check holds WSL to it) while WSL reads it and it is $gb to $most GB.
function WantMemory([string]$path, [int]$gb, [int]$most) {
$have = (Wsl2Keys (Read-WslConfig $path))['memory']
if ($have -and (CapKb $have) -ge [long]$gb * 1048576) { return $have }
$kb = if ($have) { CapKb $have } else { -1 }
if ($kb -ge [long]$gb * 1048576 -and $kb -le [long]$most * 1048576) { return $have }
"${gb}GB"
}
function WslVersion([string[]]$lines, [string]$name) { # the WSL version of one distro in `wsl -l -v`; 0: not there
Expand Down Expand Up @@ -224,7 +227,9 @@ try {
$ours = Test-Path -LiteralPath $Made
if ($have -and -not $ours) { wsl.exe -d $Distro -u root --cd / -- test -e /etc/fork-kit/desk-distro; $ours = $LASTEXITCODE -eq 0 }

$want = [ordered]@{ memory = (WantMemory $WslConfig $MemoryGB); swap = "${SwapGB}GB"; networkingMode = 'nat'; dnsTunneling = 'true'; guiApplications = 'false' }
# -MemoryGB given: it is the cap; not given: a higher one a person set is kept, up to what leaves Windows 2 GB
$most = if ($asked) { $MemoryGB } else { $ram - 2 }
$want = [ordered]@{ memory = (WantMemory $WslConfig $MemoryGB $most); swap = "${SwapGB}GB"; networkingMode = 'nat'; dnsTunneling = 'true'; guiApplications = 'false' }
$changes = @(Changes (Read-WslConfig $WslConfig) $want)
Write-Host ''
Write-Host "Next: the distro $Distro; .wslconfig (for ALL your WSL distros; a dated copy is kept):" -ForegroundColor Yellow
Expand Down
25 changes: 17 additions & 8 deletions fork-kit/pc-agents/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,9 @@ The kit reads the `pc-agents` section of the machine file `/etc/fork-kit/machine
| `tailscaleTag` | the tag this PC's Tailscale node gets (`tag:name`) |

`t3Version` must be 0.1.7 or newer: older releases have no managed policy, so they would run with none of the kit's
locks and with updates on. Setup and `check` refuse them, naming the key. (`t3 pair`, step 10, came earlier.)
locks and with updates on. A 0.1.7 pre-release counts as older (some 0.1.7 nightlies were built before the policy),
and a pre-release of a later version as that version. Setup and `check` refuse them, naming the key. (`t3 pair`,
step 10, came earlier.)

Every T3 setting can be locked through `lockedSettings`, the fork's own settings (`agentSlots`, `usagePacingGate`,
`usagePacing`, `waitFor`, ...) too: the managed policy locks any key of the server settings, upstream's or the fork's
Expand Down Expand Up @@ -153,12 +155,13 @@ Commands are for Windows PowerShell unless a step says otherwise. `<distro>` is
`wsl -d <distro> -u root -- /opt/pc-agents/current/sbin/pc-agents as-agent -- gh auth login --with-token` and paste
a fine-grained token for the agents' repos into that window only. It waits without a prompt: paste the token,
press Enter, then Ctrl+D. Then give the agent a git identity (its commits fail without one):
`wsl -d <distro> -u root -- /opt/pc-agents/current/sbin/pc-agents as-agent -- git config --global user.name '<name>'`
`wsl -d <distro> -u root -e /opt/pc-agents/current/sbin/pc-agents as-agent -- git config --global user.name '<name>'`
and
`wsl -d <distro> -u root -- /opt/pc-agents/current/sbin/pc-agents as-agent -- git config --global user.email '<email>'`,
`wsl -d <distro> -u root -e /opt/pc-agents/current/sbin/pc-agents as-agent -- git config --global user.email '<email>'`,
with a name of your choice and an address that is no one's private one, for example the token account's noreply
address (`<id>+<login>@users.noreply.github.com`, shown in its GitHub email settings). Run again, they only set
the same values again.
address (`<id>+<login>@users.noreply.github.com`, shown in its GitHub email settings). `-e`, not `--`: wsl runs
the line without root's shell, so the value reaches git as typed (a name with an apostrophe is `'O''Brien'` in
PowerShell). Run again, they only set the same values again.
9. **Keep the distro running:** WSL stops a distro soon after its last window closes, and does not start one when
Windows starts. A logon task of your own (no administrator needed) keeps this one running while you are logged
in. In PowerShell, as yourself (the same task as the desk starter's, `fork-kit/desk/windows/start-desk.ps1`):
Expand Down Expand Up @@ -229,9 +232,15 @@ allowed (connection tracking).
`SIGHUP` (a closed window) stop setup the same way, the running download's whole group too.
- **Start limit:** a rerun within 10 minutes of five failed starts still starts T3 (`systemctl reset-failed` first).
- **An older kit's distro:** an `agent` made with a login shell gets `/usr/sbin/nologin` (`usermod`), once.
- **Packages:** setup installs `nftables`, `libatomic1` and `unzip` when `dpkg` does not list one as installed (a
distro set up by an older kit gets the two new ones at its next run; no T3 restart for them). A package install
stopped half-way is finished first (`dpkg --configure -a`); a failed one stops setup, naming the command to run.
- **Packages:** setup installs `nftables`, `libatomic1` and `unzip` when `dpkg` does not list one as installed (held
counts as installed; a distro set up by an older kit gets the two new ones at its next run; no T3 restart for
them). It installs them after it writes its files, so `apt-get` resolves through the machine file's `nameservers`
as they are now, and before it loads the firewall. A package install stopped half-way is finished first
(`dpkg --configure -a`). Nothing asks a question: a changed config file is kept, debconf takes its defaults, and
`apt-get` waits up to 5 minutes for a lock (apt-daily on a new distro). A failed step stops setup, naming the
command to run; the files it wrote stay, and when one of them changed what T3 runs with, the walls check's
`restart` wall fails (and the 2-minute check stops T3) until setup runs again to the end, as after a failed T3
download.
- **A distro that is not fresh:** refused before anything is written, with the reasons: another login user, an
`agent` with another uid or in another group, a file the kit would write that it did not write (or that was
edited since, for example a hand-written `wsl.conf` or `policy.json`), a cron, at or ssh unit where the kit masks
Expand Down
20 changes: 14 additions & 6 deletions fork-kit/pc-agents/lib/pc_agents.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
KIT_SETTINGS = {'linkedEnvironmentsEnabled': True}
VERSION = re.compile(r'(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-[0-9A-Za-z.-]+)?')
# The oldest T3 that applies a managed policy (0.1.7; `t3 pair` is older): an older one would run with none of the
# kit's locks, updates on, and quietly. Its pre-releases count as it.
# kit's locks, updates on, and quietly. Its pre-releases are older than it (semver), as some nightlies are.
OLDEST_T3 = (0, 1, 7)
# Debian packages setup installs when missing: the agent firewall's nft; libatomic1, which Node (and so most agent
# tools) needs to start, while the agent has no sudo to install it; unzip, which install steps of packages use.
Expand Down Expand Up @@ -79,8 +79,8 @@ def is_ip_list(value, net, glob=False, most=64):
'organization': (str, machine_file.REQUIRED,
lambda v: None if v.strip() and len(v) <= 80 else '1 to 80 characters, not blank'),
't3Version': (str, machine_file.REQUIRED, lambda v: 'an exact version, no v' if not VERSION.fullmatch(v) else
None if tuple(map(int, VERSION.fullmatch(v).groups()[:3])) >= OLDEST_T3 else
'%d.%d.%d or newer (the first T3 with the managed policy)' % OLDEST_T3),
None if new_enough(v) else '%d.%d.%d or newer, not a pre-release of it (the first T3 with the '
'managed policy)' % OLDEST_T3),
't3ReleaseBaseUrl': (str, machine_file.REQUIRED,
lambda v: None if re.fullmatch(r'https://[A-Za-z0-9./_-]+', v) else 'an https URL'),
'port': (int, machine_file.REQUIRED, lambda v: None if 1024 <= v <= 65535 else '1024 to 65535'),
Expand All @@ -97,6 +97,13 @@ def is_ip_list(value, net, glob=False, most=64):
}


def new_enough(version):
"""Whether VERSION version is OLDEST_T3 or newer, in semver order: a pre-release ranks below its release."""
m = VERSION.fullmatch(version)
numbers = tuple(map(int, m.groups()[:3]))
return numbers > OLDEST_T3 or numbers == OLDEST_T3 and not m[4]


def port_range(text):
"""(low, high) of a 'LOW-HIGH' port range of 1024..65535, or None."""
m = re.fullmatch(r'([1-9]\d{3,4})-([1-9]\d{3,4})', text)
Expand Down Expand Up @@ -434,12 +441,13 @@ def user_problem(c, need_agent):
return f'user {USER} is in other groups ({", ".join(extra)}): it must have none' if extra else None


def run(argv, timeout=120):
def run(argv, timeout=120, env=None):
"""Run argv (argv[0] under ROOT) in a process group of its own. Past timeout, or when this process stops (Ctrl-C,
or SIGTERM and SIGHUP once stop_on_signals() ran), the whole group is killed (a download's curl too); a timeout
returns exit 124 and says so in stderr."""
returns exit 124 and says so in stderr. env: variables added to this process's own."""
p = subprocess.Popen([ROOT + argv[0]] + argv[1:], stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, text=True, start_new_session=True)
stderr=subprocess.PIPE, text=True, start_new_session=True,
env={**os.environ, **env} if env else None)
try:
out, err = p.communicate(timeout=timeout)
except BaseException as e:
Expand Down
Loading
Loading