The isolated step runs for every plugin of a pass, and again after it could not run

src/qualify-adapter-pass.ts

@@ -75,13 +75,21 @@ const sleepSync = (ms: number): void => {
   Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
 };
 
+/**
+ * jackd under a shell that waits for it. A pass is synchronous, so this process cannot reap a child; an unreaped jackd
+ * that had a client stays in JACK's server table as alive, and the next start under its name reads "server already active".
+ * The shell outlives jackd whatever happens: a TERM reaches jackd once, and HUP is how a stop that gives up kills jackd alone.
+ * Killed with its group, the shell could not reap it.
+ */
+const REAPING_JACKD = `trap : TERM; trap 'kill -KILL $p 2>/dev/null' HUP; jackd "$@" & p=$!; while kill -0 $p 2>/dev/null; do wait $p; done`;
+
 /** `jackd` on a name of its own and the dummy driver, at the cell's rate and quantum; `undefined` when none came up. */
 export const startPrivateJack: JackStarter = (cell) => {
   const name = `omx-qualify-${String(process.pid)}-${String(cell.quantum)}`;
   let child: ChildProcess;
   let ended = false;
   try {
-    child = spawn("jackd", ["-r", "-n", name, "-d", "dummy", "-r", String(cell.rate), "-p", String(cell.quantum)], { stdio: "ignore", detached: true });
+    child = spawn("sh", ["-c", REAPING_JACKD, "sh", "-r", "-n", name, "-d", "dummy", "-r", String(cell.rate), "-p", String(cell.quantum)], { stdio: "ignore", detached: true });
   } catch {
     return undefined;
   }
@@ -105,7 +113,16 @@ export const startPrivateJack: JackStarter = (cell) => {
     signal("SIGTERM");
     const until = Date.now() + SERVER_STOP_MS;
     while (running() && Date.now() < until) sleepSync(20);
-    if (running()) signal("SIGKILL");
+    if (running()) {
+      try {
+        if (child.pid !== undefined) process.kill(child.pid, "SIGHUP");
+      } catch {
+        // already gone
+      }
+      const killed = Date.now() + SERVER_STOP_MS;
+      while (running() && Date.now() < killed) sleepSync(20);
+    }
+    signal("SIGKILL");
     for (const d of serverDirs(name)) rmSync(d, { recursive: true, force: true });
   };
   const until = Date.now() + SERVER_WAIT_MS;

src/qualify-isolated.test.ts

@@ -128,7 +128,19 @@ describe("the step itself, on a private server", () => {
     const step = judgeIsolatedRun(facts);
     expect(step.result, JSON.stringify(facts)).toBe(ISOLATED_RESULT.isolated);
     expect(facts.xruns).toBeGreaterThanOrEqual(0);
-    expect(facts.cpuPercent).toBeGreaterThan(0);
+    expect(facts.cpuPercent).toBeGreaterThanOrEqual(0);
+  });
+
+  it("a pass runs the step plugin after plugin: the server of the last one leaves its name free", (ctx) => {
+    if (!ready) {
+      if (process.env["CI"] !== undefined) throw new Error("jackd, plugin-hostd or native/build is missing: CI installs them");
+      ctx.skip("needs jackd, plugin-hostd and native/build");
+    }
+    const runner = spawnedIsolatedRunner();
+    for (const run of ["first", "second", "third"]) {
+      const facts = runner(`clap:${FIXTURE}#${ID}`, CELL, process.env);
+      expect(judgeIsolatedRun(facts).result, `${run}: ${JSON.stringify(facts)}`).toBe(ISOLATED_RESULT.isolated);
+    }
   });
 });
 
@@ -168,7 +180,20 @@ describe("the completion of a record already judged", () => {
     expect(ran).toBe(0);
   });
 
-  it("the pass moves each pin with its step, counts them, and a second pass runs nothing", () => {
+  it("a step that could not run is tried again and replaced; one that ran is not", () => {
+    const notRun = judgeIsolatedRun({ started: false, problem: "no server" });
+    const before = record({ run: { id: "r", steps: [...baseSteps(record()), notRun] } });
+    const path = write(before, "again.qualification.json");
+    const done = completeIsolated(path, before, "urn:p", CELL, () => WELL, {});
+    expect(done?.step.result).toBe(ISOLATED_RESULT.isolated);
+    const isolatedSteps = (JSON.parse(readFileSync(path, "utf8")) as QualificationRecord).run?.steps.filter((s) => s.step === "isolated");
+    expect(isolatedSteps).toEqual([done?.step]);
+    const refused = judgeIsolatedRun({ started: true, added: false });
+    const seen = record({ run: { id: "r", steps: [...baseSteps(record()), refused] } });
+    expect(completeIsolated(write(seen, "seen.qualification.json"), seen, "urn:p", CELL, () => WELL, {})).toBeUndefined();
+  });
+
+  it("the pass moves each pin with its step, counts them, and a second pass runs only the step that could not run", () => {
     const dir = mkdtempSync(join(scratch, "pass-"));
     const names = ["a", "b", "c"].map((n) => `${n}.qualification.json`);
     for (const n of names) writeFileSync(join(dir, n), JSON.stringify(record()));
@@ -184,6 +209,6 @@ describe("the completion of a record already judged", () => {
     expect([b?.outcome, b?.pin?.verdict, b?.code]).toEqual(["refused", "refused", ISOLATED_CODE.silent]);
     expect([c?.outcome, c?.pin?.verdict]).toEqual(["unmeasured", undefined]);
     const second = isolatedPass(ctx, first.state);
-    expect(second).toMatchObject({ isolated: 0, refused: 0, notRun: 0 });
+    expect(second).toMatchObject({ isolated: 0, refused: 0, notRun: 1 });
   });
 });

src/qualify-isolated.ts

@@ -190,11 +190,13 @@ function runIdOf(record: QualificationRecord, dir: string): string | undefined {
  * measured. `undefined` when it owes none, or has no run id to extend. Writes the record whole.
  */
 export function completeIsolated(path: string, record: QualificationRecord, uri: string, cell: GridCell, runner: IsolatedRunner, env: Readonly<Record<string, string | undefined>>): { readonly record: QualificationRecord; readonly step: RunStep } | undefined {
-  if (!owesIsolated(record) || record.run?.steps.some((s) => s.step === RUN_STEP.isolated) === true) return undefined;
+  const isStep = (s: RunStep): boolean => s.step === RUN_STEP.isolated;
+  // a step that could not run decided nothing, so a pass tries it again and it takes that one's place
+  if (!owesIsolated(record) || record.run?.steps.some((s) => isStep(s) && s.result !== ISOLATED_RESULT.notRun) === true) return undefined;
   const id = runIdOf(record, join(path, ".."));
   if (id === undefined) return undefined;
   const step = judgeIsolatedRun(runner(uri, cell, env));
-  const run: RunBlock = { id, steps: [...(record.run?.steps ?? baseSteps(record)), step] };
+  const run: RunBlock = { id, steps: [...(record.run?.steps ?? baseSteps(record)).filter((s) => !isStep(s)), step] };
   const next = { ...record, run };
   writeFileWhole(path, `${JSON.stringify(next, null, 2)}\n`);
   return { record: next, step };