Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions packages/pivotkit-cli/CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,10 @@ All notable changes to `pivotkit-cli` are documented in this file. The format is

### Fixed

- Two runs that sign in at once never hold the refresh lock together. A run that read a crashed
run's lock as stale moved it aside to take it over, also when another run had just taken it over.
A third run could then get in beside that run. Now one run at a time takes a lock over, and only
while it is still stale ([#717](https://github.com/eizedev/pivotkit/issues/717)).
- `pivotkit login`, `token` and `logout` send the user agent `pivotkit-cli/<version>` to the server
and its issuer, as the REST calls do. They sent Node's default `node`, or none to a loopback
address ([#628](https://github.com/eizedev/pivotkit/issues/628)).
Expand Down
79 changes: 60 additions & 19 deletions packages/pivotkit-cli/src/credential-store.js
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@

import { spawn } from "node:child_process";
import { createHash, randomBytes } from "node:crypto";
import { chmod, link, lstat, mkdir, open, readFile, rename, stat, unlink, utimes } from "node:fs/promises";
import { chmod, lstat, mkdir, open, readFile, stat, unlink, utimes } from "node:fs/promises";
import { homedir } from "node:os";
import { dirname, join } from "node:path";
import { CliError } from "./errors.js";
Expand Down Expand Up @@ -249,23 +249,56 @@ export const fileStoreText = () => (storeMeans.platform === "win32" ? "a file in
/** Whether the lock file at `path` still holds this run's id. */
const ownsLock = (path, id) => readFile(path, "utf8").then((text) => text === id, () => false);

/** A takeover lock older than this is a crashed run's: a takeover takes milliseconds. */
const guardStaleMs = () => Math.min(GUARD_STALE_MS, storeMeans.lockStaleMs);
const GUARD_STALE_MS = 10_000;

/** The takeover lock beside a lock file (takeOver). */
const guardOf = (path) => `${path}.takeover`;

/** A file's age in ms, or null when it is not there. */
const ageOf = (file) => stat(file).then((info) => Date.now() - info.mtimeMs, () => null);

/** Removes a takeover lock a crashed run left (older than guardStaleMs). */
async function clearCrashedGuard(path) {
if (((await ageOf(guardOf(path))) ?? 0) > guardStaleMs()) {
await unlink(guardOf(path)).catch(() => undefined);
}
}

/**
* Takes over a stale lock: renamed to a name of this run, so only one run gets it. When the renamed
* lock turns out live (a newer run made it after the age was read), it goes back with link(), which
* never replaces a lock made since.
* Takes over a stale lock, for a run that read its age as stale. Only one run at a time may: it
* holds a second lock, the takeover lock, made with O_EXCL. Holding it, the run reads the lock's
* age again and removes the lock only if it is still stale, so a lock another run made since the
* first read is never moved. A run that makes a lock while a takeover lock is there backs off
* (locked), so a lock made between the age and the removal is never used. A takeover lock older
* than guardStaleMs is a crashed run's and is removed. Exported for the tests.
* @param {string} path the lock file
* @returns {Promise<boolean>} whether this run removed the stale lock
*/
async function takeOver(path, id) {
const aside = `${path}.${id}.stale`;
export async function takeOver(path) {
const guard = guardOf(path);
let handle;
try {
await rename(path, aside);
} catch {
return;
handle = await open(guard, "wx", 0o600);
} catch (e) {
if (/** @type {any} */ (e)?.code === "EEXIST") {
await clearCrashedGuard(path);
}
return false;
}
const age = await stat(aside).then((info) => Date.now() - info.mtimeMs, () => Infinity);
if (age <= storeMeans.lockStaleMs) {
await link(aside, path).catch(() => undefined);
try {
await handle.close();
if (((await ageOf(path)) ?? 0) > storeMeans.lockStaleMs) {
return await unlink(path).then(
() => true,
() => false
);
}
return false;
} finally {
await unlink(guard).catch(() => undefined);
}
await unlink(aside).catch(() => undefined);
}

/** Writes a file only its owner reads, through a temporary file. */
Expand Down Expand Up @@ -349,9 +382,9 @@ export function credentialStore(env) {
/**
* Runs `work` while this process alone may refresh the server's token: a lock file made with
* O_EXCL that holds a random id. While held, its time is renewed every lockBeatMs, so a lock
* whose time is older than lockStaleMs is a crashed run's. Such a lock is taken over by
* renaming it to a name of this run first, so two runs cannot both take it; a lock found live
* after the rename goes back. The owner removes only a lock that still holds its own id.
* whose time is older than lockStaleMs is a crashed run's. One run at a time takes such a lock
* over (takeOver), and only while it is still stale; a run that made its lock during a takeover
* backs off. The owner removes only a lock that still holds its own id.
* @template T
* @param {string} resource
* @param {() => Promise<T>} work
Expand All @@ -368,15 +401,23 @@ export function credentialStore(env) {
const handle = await open(path, "wx", 0o600);
await handle.writeFile(id);
await handle.close();
break;
// a takeover runs: it may have read the lock before this one as stale and remove
// this one, so this run backs off and tries again
if (!(await lstat(guardOf(path)).then(() => true, () => false))) {
break;
}
if (await ownsLock(path, id)) {
await unlink(path).catch(() => undefined);
}
await clearCrashedGuard(path);
} catch (e) {
if (/** @type {any} */ (e)?.code !== "EEXIST") {
throw e;
}
}
const age = await stat(path).then((info) => Date.now() - info.mtimeMs, () => 0);
if (age > storeMeans.lockStaleMs) {
await takeOver(path, id);
// a takeover that removed the lock tries again at once; one that did not waits as any run
if (age > storeMeans.lockStaleMs && (await takeOver(path))) {
continue;
}
if (Date.now() - start > storeMeans.lockWaitMs) {
Expand Down
89 changes: 88 additions & 1 deletion packages/pivotkit-cli/test/sign-in.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import { chmodSync, existsSync, mkdirSync, readFileSync, readdirSync, statSync,
import { join } from "node:path";
import { main } from "pivotkit-cli";
import { run, tempDir } from "./helpers.js";
import { accountOf, credentialStore, storeMeans } from "../src/credential-store.js";
import { accountOf, credentialStore, storeMeans, takeOver } from "../src/credential-store.js";
import { utimesSync } from "node:fs";
import { scopeOf, signInMeans } from "../src/sign-in.js";
import { versions } from "../src/cli.js";
Expand Down Expand Up @@ -925,6 +925,93 @@ test("the lock: two runs that find a stale lock at once take it over one after t
});
});

// #717: three runs read a stale lock's age at once. The first takes it over and works; the second,
// late, must not move the first one's fresh lock even for a moment, or the third gets in beside it
test("the lock's takeover: a late run leaves a lock made since alone, never letting a third run in; one takeover at a time", async () => {
await withLock({ lockStaleMs: 1000 }, async () => {
const dir = tempDir();
const path = join(dir, "server.lock");
// a lock another run made since this run read the old one's age: the late takeover finds it fresh
writeFileSync(path, "run A");
const { open } = await import("node:fs/promises");
let late = true;
const taking = takeOver(path).finally(() => {
late = false;
});
// a third run tries O_EXCL the whole time: it must never find the path free
let thirdIn = false;
while (late) {
const third = await open(path, "wx").catch(() => null);
if (third) {
thirdIn = true;
await third.close();
break;
}
}
await taking;
assert.equal(thirdIn, false, "a third run got in while the late takeover ran");
assert.equal(readFileSync(path, "utf8"), "run A", "the fresh lock stays");
assert.deepEqual(readdirSync(dir), ["server.lock"], "no takeover lock or copy left");

// a stale lock while another run takes over: left to that run
const old = new Date(1_000_000_000_000);
utimesSync(path, old, old);
writeFileSync(`${path}.takeover`, "");
await takeOver(path);
assert.ok(existsSync(path), "another run's takeover goes on");
// that run's takeover lock, when it crashed, is removed; the next try takes the stale lock over
utimesSync(`${path}.takeover`, old, old);
await takeOver(path);
assert.ok(!existsSync(`${path}.takeover`), "a crashed takeover's lock is removed");
assert.ok(existsSync(path));
await takeOver(path);
assert.deepEqual(readdirSync(dir), [], "the stale lock is taken over");

// runs that take over at once: exactly one removes the stale lock
writeFileSync(path, "a crashed run's id");
utimesSync(path, old, old);
const taken = await Promise.all([1, 2, 3, 4, 5].map(() => takeOver(path)));
assert.equal(taken.filter(Boolean).length, 1, JSON.stringify(taken));
assert.deepEqual(readdirSync(dir), []);
});
});

test("the lock while a takeover runs: a waiting run neither spins nor enters, and gives up after lockWaitMs", async () => {
await withLock({ lockStaleMs: 1000, lockWaitMs: 300, lockPollMs: 20 }, async () => {
const dir = tempDir();
const store = credentialStore(envOf(dir));
const resource = "https://reports.example/api/v1";
const locks = join(dir, "pivotkit", "locks");
mkdirSync(locks, { recursive: true, mode: 0o700 });
const path = join(locks, `${accountOf(resource)}.lock`);
// a stale lock and another run's live takeover: this run waits for it, as for any lock
writeFileSync(path, "a crashed run's id");
utimesSync(path, new Date(1_000_000_000_000), new Date(1_000_000_000_000));
writeFileSync(`${path}.takeover`, "");
let ran = false;
await assert.rejects(
store.locked(resource, async () => {
ran = true;
}),
(e) => e.code === "LOCKED"
);
assert.equal(ran, false);
// no lock: a run that makes its own while the takeover lock is there backs off until it goes
unlinkSync(path);
let guardGone = false;
setTimeout(() => {
unlinkSync(`${path}.takeover`);
guardGone = true;
}, 100);
let enteredAfterGuard = null;
await store.locked(resource, async () => {
enteredAfterGuard = guardGone;
});
assert.equal(enteredAfterGuard, true, "the run entered only once the takeover lock was gone");
assert.deepEqual(readdirSync(locks), []);
});
});

test("a 429 while polling is asked again like a 5xx; an interval over five minutes is cut, with a note", async () => {
await withWaits(async (waits) => {
const busy = await fakeIssuer({
Expand Down
4 changes: 3 additions & 1 deletion packages/pivotkit-server/test/hardening.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -461,7 +461,9 @@ test("#479: a row longer than its header only by an empty last cell is left out
const server = await restServer({ sources: { people: { file: "data/people.csv", policy: { default: "show", fields: { ssn: "hide" } } } }, baseDir: dir });
try {
const city = await call(`${server.api}/views/render`, { body: { source: "people", view: { rows: ["city"], values: [{ agg: "count" }] }, format: "json" } });
assert.doesNotMatch(JSON.stringify(city.data), /"3"|\b3\b(?=[^0-9])/, JSON.stringify(city.data));
// the request's random id may hold a 3 (#721): only the parts that can carry data are read
const answer = JSON.stringify({ ...city.data, requestId: undefined });
assert.doesNotMatch(answer, /"3"|\b3\b(?=[^0-9])/, answer);
const described = await call(`${server.api}/sources/describe`, { body: { source: "people" } });
assert.equal(described.data.rows, 1);
} finally {
Expand Down
Loading