Run code generation on build

CONTRIBUTING.md

@@ -102,6 +102,7 @@ The repository uses a Go workspace with modules in `tsc/` and `tools/`.
 npx hereby build         # Build the native compiler into built/local/tsc
 npx hereby test          # Run compiler and language-service Go tests
 npx hereby test:all      # Also run benchmarks, tools, and API tests
+npx hereby test:codegen  # Run codegen tests without regenerating source outputs
 npx hereby lint          # Run custom golangci-lint for both Go modules
 npx hereby generate:go   # Run the generators also exposed through go generate
 npx hereby generate      # Regenerate all sources, assets, localization, and vendored files
@@ -117,6 +118,13 @@ to bypass incremental caches. `generate` includes every generator and fetches
 the pinned LSP model when its local cache is missing or stale. Use
 `generate:go` for the narrower scope of `go generate ./...` in the `tsc` module.
 
+`npx hereby build` runs `generate` before compiling; a separate generation step
+is not needed. The API and package build tasks share this prerequisite, and
+`build:watch` runs the full incremental generation suite before every rebuild.
+Each watch pass clears cached input fingerprints and runs generation in-process.
+Restart the watcher after editing generator implementations. Generator-owned outputs are
+excluded from watch events to avoid regeneration loops.
+
 Package-specific commands:
 
 ```bash
@@ -142,19 +150,8 @@ go -C ./tsc test -run='TestLocal/<test name>' ./internal/testrunner
 Run:
 
 ```bash
-npx hereby generate
-npx hereby build
-npx hereby test
-npx hereby test:all
-npx hereby lint
-npx hereby format
-npx hereby check:format
-npm run -w @typescript/typescript build
-npm run -w @typescript/typescript test
-npm run -w native-preview build
-go -C ./tsc mod tidy -diff
-go -C ./tools mod tidy -diff
-go work sync
+npx hereby validate --all
+npx hereby tidy
 git diff --exit-code
 ```
 

tools/scripts/gen/cache.mts

@@ -8,6 +8,7 @@ import {
     goInputs,
     repoRoot,
     run,
+    withFileFingerprintWrites,
 } from "./utils.mts";
 
 export interface CacheOptions {
@@ -21,6 +22,10 @@ export interface CacheOptions {
     force?: boolean;
 }
 
+export interface CacheHost {
+    run(command: string, args: readonly string[], options: { cwd: string; env: NodeJS.ProcessEnv; }): Promise<unknown>;
+}
+
 function expand(patterns: string[], cwd: string, exclude: string[] = []) {
     return [...new Set(fs.globSync(patterns, { cwd, exclude }).map(file => path.resolve(cwd, file)))].sort();
 }
@@ -34,7 +39,7 @@ export default async function cache({
     envInputs = ["GOOS", "GOARCH", "GOFLAGS", "GOTOOLCHAIN", "GOEXPERIMENT", "CGO_ENABLED", "GOWORK"],
     env,
     force = false,
-}: CacheOptions): Promise<boolean> {
+}: CacheOptions, host: CacheHost = { run }): Promise<boolean> {
     if (!outputs.length || !commands.length || commands.some(command => !command.length)) {
         throw new Error("Cached generation requires outputs and nonempty commands.");
     }
@@ -91,9 +96,11 @@ export default async function cache({
         return true;
     }
     for (const file of previous) file.invalidate();
-    for (const [command, ...args] of commands) {
-        await run(command, args, { cwd, env: environment });
-    }
+    await withFileFingerprintWrites(async () => {
+        for (const [command, ...args] of commands) {
+            await host.run(command, args, { cwd, env: environment });
+        }
+    });
     if (!complete()) throw new Error(`Generation did not produce all declared outputs: ${outputs.join(", ")}`);
     if (snapshotInputs(true).hash === before.hash) {
         for (const file of artifacts(outputFiles())) file.markCurrent();

tools/scripts/gen/cache.test.mts

@@ -1,59 +1,121 @@
 import assert from "node:assert/strict";
-import * as fs from "node:fs";
+import fs from "node:fs";
+import { syncBuiltinESMExports } from "node:module";
 import os from "node:os";
 import path from "node:path";
 import { test } from "node:test";
-import cache from "./cache.mts";
+import cache, { type CacheHost } from "./cache.mts";
+import { GeneratedFile } from "./generatedFile.mts";
+import { enableFileFingerprintCache } from "./utils.mts";
+
+test("command cache snapshots and generated files share invocation fingerprints", async context => {
+    const directory = fs.mkdtempSync(path.join(os.tmpdir(), "tsgo-fingerprint-"));
+    context.after(() => fs.rmSync(directory, { recursive: true, force: true }));
+    const input = path.join(directory, "input.txt");
+    fs.writeFileSync(input, "input");
+    const host: CacheHost = {
+        async run() {
+            fs.writeFileSync(path.join(directory, "output.txt"), "generated");
+        },
+    };
+    const options = {
+        cwd: directory,
+        inputs: ["input.txt"],
+        outputs: ["output.txt"],
+        commands: [["generate"]],
+    };
+    assert.equal(await cache(options, host), false);
+    context.after(enableFileFingerprintCache());
+    const reads = context.mock.method(fs, "readFileSync");
+    syncBuiltinESMExports();
+    context.after(() => {
+        reads.mock.restore();
+        syncBuiltinESMExports();
+    });
+    assert.equal(await cache(options, host), true);
+    assert.equal(await cache(options, host), true);
+    new GeneratedFile(path.join(directory, "other.txt"), [input]);
+    assert.equal(reads.mock.calls.filter(call => call.arguments[0] === input).length, 1);
+});
 
 test("cache commands use and track an explicit environment without changing the parent", async context => {
     const directory = fs.mkdtempSync(path.join(os.tmpdir(), "tsgo-cache-env-"));
     context.after(() => fs.rmSync(directory, { recursive: true, force: true }));
     const parentValue = process.env.GOFILE;
+    let calls = 0;
+    const host: CacheHost = {
+        async run(command, args, { cwd, env }) {
+            calls++;
+            assert.equal(command, "generate");
+            assert.deepEqual(args, []);
+            fs.writeFileSync(path.join(cwd, "output.json"), JSON.stringify({ file: env.GOFILE, pkg: env.GOPACKAGE, cwd }));
+        },
+    };
     const options = {
         cwd: directory,
         inputs: [],
         outputs: ["output.json"],
         envInputs: ["GOFILE", "GOPACKAGE"],
         env: { GOFILE: "source.go", GOPACKAGE: "example" },
-        commands: [[process.execPath, "-e", "require('node:fs').writeFileSync('output.json', JSON.stringify({ file: process.env.GOFILE, pkg: process.env.GOPACKAGE, cwd: process.cwd() }))"]],
+        commands: [["generate"]],
     };
-    assert.equal(await cache(options), false);
+    assert.equal(await cache(options, host), false);
     assert.deepEqual(JSON.parse(fs.readFileSync(path.join(directory, "output.json"), "utf8")), {
         file: "source.go",
         pkg: "example",
         cwd: fs.realpathSync(directory),
     });
-    assert.equal(await cache(options), true);
-    assert.equal(await cache({ ...options, env: { ...options.env, GOFILE: "other.go" } }), false);
+    assert.equal(await cache(options, host), true);
+    assert.equal(calls, 1);
+    assert.equal(await cache({ ...options, env: { ...options.env, GOFILE: "other.go" } }, host), false);
+    assert.equal(calls, 2);
     assert.equal(process.env.GOFILE, parentValue);
 });
 
 test("cache commands preserve the working directory and arguments", async context => {
-    const root = path.resolve(import.meta.dirname, "../../..");
-    const directory = fs.mkdtempSync(path.join(root, "tsc/internal/_cache probe-"));
+    const directory = fs.mkdtempSync(path.join(os.tmpdir(), "tsgo-cache probe-"));
     context.after(() => fs.rmSync(directory, { recursive: true, force: true, maxRetries: 10 }));
     fs.writeFileSync(path.join(directory, "input.txt"), "input");
-    fs.writeFileSync(path.join(directory, "generate.cjs"), "const fs = require('node:fs'); fs.writeFileSync('output.txt', fs.readFileSync('input.txt') + JSON.stringify(process.argv.slice(2)));\n");
-    fs.writeFileSync(path.join(directory, "finish.cjs"), "require('node:fs').appendFileSync('output.txt', ' finished');\n");
+    const calls: string[] = [];
+    const host: CacheHost = {
+        async run(command, args, { cwd }) {
+            calls.push(command);
+            assert.equal(cwd, directory);
+            if (command === "generate") {
+                assert.deepEqual(args, ["--help", "--force", "space value"]);
+                fs.writeFileSync(path.join(cwd, "output.txt"), fs.readFileSync(path.join(cwd, "input.txt")) + JSON.stringify(args));
+            }
+            else {
+                assert.deepEqual(args, []);
+                if (command === "fail") throw new Error("Command failed");
+                fs.appendFileSync(path.join(cwd, "output.txt"), " finished");
+            }
+        },
+    };
     const generate = (force = false, failure = false) =>
         cache({
             cwd: directory,
-            inputs: ["input.txt", "*.cjs"],
+            inputs: ["input.txt"],
             outputs: ["output.txt"],
             commands: [
-                [process.execPath, "generate.cjs", "--help", "--force", "space value"],
-                [process.execPath, failure ? "missing.cjs" : "finish.cjs"],
+                ["generate", "--help", "--force", "space value"],
+                [failure ? "fail" : "finish"],
+                ["last"],
             ],
             force,
-        });
+        }, host);
     const output = path.join(directory, "output.txt");
     assert.equal(await generate(), false);
-    assert.equal(fs.readFileSync(output, "utf8"), 'input["--help","--force","space value"] finished');
-    const timestamp = fs.statSync(output).mtimeMs;
+    assert.equal(fs.readFileSync(output, "utf8"), 'input["--help","--force","space value"] finished finished');
+    assert.deepEqual(calls, ["generate", "finish", "last"]);
+    calls.length = 0;
     assert.equal(await generate(), true);
-    assert.equal(fs.statSync(output).mtimeMs, timestamp);
+    assert.deepEqual(calls, []);
     assert.equal(await generate(true), false);
-    await assert.rejects(generate(false, true));
+    assert.deepEqual(calls, ["generate", "finish", "last"]);
+    calls.length = 0;
+    await assert.rejects(generate(false, true), /Command failed/);
+    assert.deepEqual(calls, ["generate", "fail"]);
     assert.equal(await generate(), false);
     assert.equal(await generate(), true);
 });
@@ -64,26 +126,38 @@ test("cache commands track inputs, outputs, arguments, and failures", async cont
     const input = path.join(directory, "input.txt");
     const output = path.join(directory, "output.txt");
     fs.writeFileSync(input, "first");
-    fs.writeFileSync(path.join(directory, "generate.cjs"), "const fs = require('node:fs'); fs.writeFileSync('output.txt', fs.readFileSync('input.txt'));\n");
-    fs.writeFileSync(path.join(directory, "finish.cjs"), "const fs = require('node:fs'); fs.appendFileSync('output.txt', process.argv[2]); fs.writeFileSync('extra.txt', 'extra');\n");
+    const host: CacheHost = {
+        async run(command, args, { cwd }) {
+            if (command === "generate") fs.writeFileSync(output, fs.readFileSync(input));
+            else if (command === "fail") throw new Error("Command failed");
+            else {
+                fs.appendFileSync(output, args[0]);
+                fs.writeFileSync(path.join(cwd, "extra.txt"), "extra");
+            }
+        },
+    };
     const generate = (suffix = " done", force = false, failure = false, env: NodeJS.ProcessEnv = {}) =>
         cache({
             cwd: directory,
-            inputs: ["input*.txt", "*.cjs"],
+            inputs: ["input*.txt"],
+            exclude: ["input-ignored.txt"],
             outputs: ["output.txt", "extra*.txt"],
             envInputs: ["TSGO_CACHE_PROBE"],
             env,
             commands: [
-                [process.execPath, "generate.cjs"],
-                [process.execPath, failure ? "missing.cjs" : "finish.cjs", suffix],
+                ["generate"],
+                [failure ? "fail" : "finish", suffix],
             ],
             force,
-        });
+        }, host);
     await generate();
     assert.equal(fs.readFileSync(output, "utf8"), "first done");
     const timestamp = fs.statSync(output).mtimeMs;
     assert.equal(await generate(), true);
     assert.equal(fs.statSync(output).mtimeMs, timestamp);
+    fs.writeFileSync(path.join(directory, "input-ignored.txt"), "ignored");
+    fs.writeFileSync(path.join(directory, "unrelated.txt"), "unrelated");
+    assert.equal(await generate(), true);
     fs.writeFileSync(input, "second");
     await generate();
     assert.equal(fs.readFileSync(output, "utf8"), "second done");
@@ -113,40 +187,77 @@ test("cache commands track inputs, outputs, arguments, and failures", async cont
     assert.equal(await generate("--force"), true);
 });
 
+test("failed commands invalidate metadata even when outputs are unchanged", async context => {
+    const directory = fs.mkdtempSync(path.join(os.tmpdir(), "tsgo-cache-"));
+    context.after(() => fs.rmSync(directory, { recursive: true, force: true }));
+    let fail = false;
+    let calls = 0;
+    const host: CacheHost = {
+        async run() {
+            calls++;
+            fs.writeFileSync(path.join(directory, "output.txt"), "generated");
+            if (fail) throw new Error("Command failed");
+        },
+    };
+    const options = { cwd: directory, inputs: [], outputs: ["output.txt"], commands: [["generate"]] };
+    assert.equal(await cache(options, host), false);
+    fail = true;
+    await assert.rejects(cache({ ...options, force: true }, host), /Command failed/);
+    fail = false;
+    assert.equal(await cache(options, host), false);
+    assert.equal(await cache(options, host), true);
+    assert.equal(calls, 3);
+});
+
 test("cache commands reject missing inputs, outputs, and empty commands", async context => {
     const directory = fs.mkdtempSync(path.join(os.tmpdir(), "tsgo-cache-"));
     context.after(() => fs.rmSync(directory, { recursive: true, force: true }));
+    let calls = 0;
+    const host: CacheHost = {
+        async run() {
+            calls++;
+            fs.writeFileSync(path.join(directory, "output.txt"), "generated");
+        },
+    };
     const options = {
         cwd: directory,
         inputs: ["input.txt"],
         outputs: ["output.txt"],
-        commands: [[process.execPath, "-e", "require('node:fs').writeFileSync('output.txt', 'generated')"]],
+        commands: [["generate"]],
     };
-    await assert.rejects(cache(options), /No inputs matched/);
+    await assert.rejects(cache(options, host), /No inputs matched/);
     assert.equal(fs.existsSync(path.join(directory, "output.txt")), false);
     fs.writeFileSync(path.join(directory, "input.txt"), "input");
-    await assert.rejects(cache({ ...options, inputs: ["missing*.txt"] }), /No inputs matched/);
-    await assert.rejects(cache({ ...options, commands: [] }), /nonempty commands/);
-    await assert.rejects(cache({ ...options, commands: [[]] }), /nonempty commands/);
-    await assert.rejects(cache({ ...options, outputs: [] }), /requires outputs/);
-    await assert.rejects(cache({ ...options, outputs: ["output.txt", "missing.txt"] }), /did not produce all declared outputs/);
-    assert.equal(await cache(options), false);
-    assert.equal(await cache(options), true);
+    await assert.rejects(cache({ ...options, inputs: ["missing*.txt"] }, host), /No inputs matched/);
+    await assert.rejects(cache({ ...options, commands: [] }, host), /nonempty commands/);
+    await assert.rejects(cache({ ...options, commands: [[]] }, host), /nonempty commands/);
+    await assert.rejects(cache({ ...options, outputs: [] }, host), /requires outputs/);
+    assert.equal(calls, 0);
+    await assert.rejects(cache({ ...options, outputs: ["output.txt", "missing.txt"] }, host), /did not produce all declared outputs/);
+    assert.equal(calls, 1);
+    assert.equal(await cache(options, host), false);
+    assert.equal(await cache(options, host), true);
 });
 
 test("cache commands do not cache inputs changed during execution", async context => {
     const directory = fs.mkdtempSync(path.join(os.tmpdir(), "tsgo-cache-"));
     context.after(() => fs.rmSync(directory, { recursive: true, force: true }));
-    for (const change of ["fs.writeFileSync('input.txt', 'changed')", "fs.writeFileSync('input-added.txt', 'added')"]) {
+    for (const changedFile of ["input.txt", "input-added.txt"]) {
         fs.writeFileSync(path.join(directory, "input.txt"), "initial");
+        const host: CacheHost = {
+            async run() {
+                fs.writeFileSync(path.join(directory, "output.txt"), "generated");
+                fs.writeFileSync(path.join(directory, changedFile), "changed");
+            },
+        };
         const options = {
             cwd: directory,
             inputs: ["input*.txt"],
             outputs: ["output.txt"],
-            commands: [[process.execPath, "-e", `const fs = require('node:fs'); fs.writeFileSync('output.txt', 'generated'); ${change};`]],
+            commands: [["generate"]],
         };
-        assert.equal(await cache(options), false);
-        assert.equal(await cache(options), false);
-        assert.equal(await cache(options), true);
+        assert.equal(await cache(options, host), false);
+        assert.equal(await cache(options, host), false);
+        assert.equal(await cache(options, host), true);
     }
 });