Run all-library benchmarks with any selected NMDEV JAR

.github/workflows/benchmark-application.yml

@@ -33,12 +33,24 @@ jobs:
           distribution: temurin
           java-version: '17'
           cache: maven
+      - name: Check PowerShell entrypoint syntax
+        shell: pwsh
+        run: '[scriptblock]::Create([IO.File]::ReadAllText("benchmark/run-benchmark.ps1")) | Out-Null'
       - name: Build application and test checkpoint/ZIP safeguards
         run: mvn -B -f benchmark/pom.xml package
       - name: Verify automatic preparation and all library/profile adapters
-        run: java -Xmx2g -jar benchmark/target/nmdev-benchmark-1.2.0.jar --smoke --output smoke-results
+        run: java -Xmx2g -jar benchmark/target/nmdev-benchmark-1.3.0.jar --smoke --output smoke-results
+      - name: Verify arbitrary local NMDEV filename and recorded binary origin
+        shell: pwsh
+        run: |
+          Copy-Item smoke-results/.runtime/lib/nmdev-selected.jar 'selected NMDEV.jar'
+          java -Xmx2g -jar benchmark/target/nmdev-benchmark-1.3.0.jar --nmdev-jar 'selected NMDEV.jar' --smoke --libraries 'NMDEV,COMMONS,COMMONS_BLOCK' --max-cases 1 --output local-jar-results
+          if ($LASTEXITCODE -ne 0) { throw 'Local-JAR smoke failed' }
+          $expected = (Get-FileHash 'selected NMDEV.jar' -Algorithm SHA256).Hash.ToLowerInvariant()
+          $recorded = Get-Content local-jar-results/binary-provenance.json -Raw | ConvertFrom-Json
+          if ($recorded.nmdev_sha256 -ne $expected -or $recorded.nmdev_original_filename -ne 'selected NMDEV.jar') { throw 'Wrong NMDEV binary provenance' }
       - name: Verify saved forks are reused
-        run: java -Xmx2g -jar benchmark/target/nmdev-benchmark-1.2.0.jar --smoke --output smoke-results
+        run: java -Xmx2g -jar benchmark/target/nmdev-benchmark-1.3.0.jar --smoke --output smoke-results
       - uses: actions/upload-artifact@v4
         if: always()
         with:
@@ -49,4 +61,24 @@ jobs:
         if: matrix.os == 'ubuntu-24.04' && success()
         with:
           name: nmdev-benchmark-application
-          path: benchmark/target/nmdev-benchmark-1.2.0.jar
+          path: benchmark/target/nmdev-benchmark-1.3.0.jar
+
+  release:
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    needs: portable
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
+        with:
+          name: nmdev-benchmark-application
+          path: release
+      - name: Publish verified reusable application
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          cd release
+          sha256sum nmdev-benchmark-1.3.0.jar > SHA256SUMS
+          gh release create benchmark-v1.3.0 nmdev-benchmark-1.3.0.jar SHA256SUMS ../benchmark/run-benchmark.ps1 --repo nmltd/nmdev-examples --target "$GITHUB_SHA" --title 'Reusable benchmark application 1.3.0' --notes 'Select any NMDEV binary with --nmdev-jar. Full 1,260-case suite across eleven libraries including both Commons matrix implementations. Released checksum-pinned Commons Math 4.0-beta1 dependencies. Windows and Linux application smoke, local-JAR selection and resume checks passed for this exact application build. Full benchmark timings are run locally.'

benchmark/LOCAL-RUN.md

@@ -1,3 +1,34 @@
+# Reusable benchmark application 1.3.0
+
+From PowerShell, run `benchmark/run-benchmark.ps1`. It downloads the checksum-verified prebuilt runner and NMDEV 1.2.10, then starts the complete suite. A full JDK 17+ is required; Maven and source edits are unnecessary.
+
+```powershell
+powershell -ExecutionPolicy Bypass -File benchmark/run-benchmark.ps1 -NmdevVersion 1.2.10
+# Future releases: change only -NmdevVersion. For a custom build, select its file:
+powershell -ExecutionPolicy Bypass -File benchmark/run-benchmark.ps1 -NmdevJar 'D:\path\nmdev.jar' -OutputDirectory custom-results
+```
+
+The same Java application also accepts `--nmdev-jar` directly:
+
+```text
+java -jar nmdev-benchmark-1.3.0.jar --nmdev-jar path/to/nmdev.jar --output new-results
+```
+
+All 1,260 cases and eleven libraries (twelve implementation variants) remain enabled, including Commons array and block matrices. The selected file is validated as an NMDEV JAR, copied under one private runtime filename, and its actual SHA-256, original filename and embedded version (if supplied by the JAR) are recorded. No version is inferred solely from its filename. A new NMDEV binary requires a new result directory; changing the binary in a stopped run is rejected before cached JARs or result metadata are replaced. Repeat the same command to resume.
+
+Commons now uses the released **4.0-beta1** JARs from Maven Central, with explicit hashes and matching origin checks. Expiring timestamped Apache snapshot URLs are no longer needed. These Commons results use a different comparator binary from historical 4.0-SNAPSHOT runs and must be labelled accordingly. Other comparison-library pins and numerical benchmark sources are unchanged.
+
+To build the runner yourself:
+
+```text
+mvn -s benchmark/settings.xml -gs benchmark/settings.xml -f benchmark/pom.xml package
+java -jar benchmark/target/nmdev-benchmark-1.3.0.jar --nmdev-jar path/to/nmdev.jar --output new-results
+```
+
+Older application instructions and experiment history follow. Do not use packaging-only migration to move a 1.2.0 experiment to 1.3.0; keep the original runner and results for that experiment.
+
+---
+
 # Run the complete benchmark unattended
 
 Requires a **full JDK 17 or newer**, Maven to build, and Internet access on the first run. Windows, Linux and macOS use the same Java entry point; no WSL, Python, shell scripts, GitHub token, manual dependency installation or profile selection is required.
@@ -13,7 +44,7 @@ The public application entry point is `dev.nm.benchmark.launcher.BenchmarkMain`.
 
 With no arguments, the application automatically:
 
-1. Downloads checksum-verified numerical dependencies and the repository's public trial license. It uses the validated NM Dev 1.2.6 public release and the timestamp-pinned Commons Math 4 snapshot from the lockfile, keeping version labels explicit. The public NM Dev binary hash differs from the older archived benchmark binary; local measurements must be treated as a separate experiment.
+1. Downloads checksum-verified numerical dependencies and the repository's public trial license. Application 1.3.0 defaults to the validated NM Dev 1.2.10 release (or your --nmdev-jar selection) and released Commons Math 4.0-beta1 modules from the lockfile, keeping version labels explicit. The public NM Dev binary hash differs from the older archived benchmark binary; local measurements must be treated as a separate experiment.
 2. Builds Parallel Colt 0.11.4 from a checksum-verified archive of its exact upstream commit, then compiles the bundled JMH harness. The locally built binary's actual hash and compiler version are recorded; compiler-dependent binary hashes can differ between machines.
 3. Collects machine information and runs correctness, input-preservation, pivot and backend preflight checks.
 4. Runs all **1,260 cases** serially: 312 baseline, 360 large-square, 48 rectangular, plus 540 all-core cases. All eleven libraries (twelve implementation variants) are attempted. One-thread, four-thread and all-core profiles are scheduled automatically; they never run concurrently.

benchmark/dependencies.lock.tsv

@@ -18,10 +18,10 @@ netlib-java-0.9.3.jar	f997d8f84de8de6fbf38b25a2b0ad7cd317f9245f47144557d6aaf415a
 ojalgo-57.5.0.jar	603dbc26c8d625699ab631b6d56240fd6c488260e3d5db498f9ac71944cb9b3f	https://repo.maven.apache.org/maven2/org/ojalgo/ojalgo/57.5.0/ojalgo-57.5.0.jar
 ujmp-core-0.3.0.jar	e5cf6dbb7b7ebc6b4eb088a6f6f76162fd8c1ad8d222bf5777b59de8ea7e47a7	https://repo.maven.apache.org/maven2/org/ujmp/ujmp-core/0.3.0/ujmp-core-0.3.0.jar
 json-20141113.jar	87bbe48a7e93d6bcfe91c36f4e73a4a670b3843e0cc10aaf48a23ba40a99671b	https://repo.maven.apache.org/maven2/org/json/json/20141113/json-20141113.jar
-commons-math4-core-4.0-SNAPSHOT.jar	7fcd23cbb4ddd9536f1cdbe761d528c707ab5590e245c2341ce990196ca6e0b0	https://repository.apache.org/content/repositories/snapshots/org/apache/commons/commons-math4-core/4.0-SNAPSHOT/commons-math4-core-4.0-20261018.085925-1447.jar
-commons-math4-legacy-4.0-SNAPSHOT.jar	5db3d8096b42daffc5573ca78163d5461b2039dddd884c8e3e3c5d799b8aa144	https://repository.apache.org/content/repositories/snapshots/org/apache/commons/commons-math4-legacy/4.0-SNAPSHOT/commons-math4-legacy-4.0-20261018.085925-1575.jar
-commons-math4-legacy-core-4.0-SNAPSHOT.jar	1ae582cd07f29abc3bb17877aa6a290d8bd8e7074adc762a330dd24ef569b9d8	https://repository.apache.org/content/repositories/snapshots/org/apache/commons/commons-math4-legacy-core/4.0-SNAPSHOT/commons-math4-legacy-core-4.0-20261018.085925-1586.jar
-commons-math4-legacy-exception-4.0-SNAPSHOT.jar	c48a58d12ca4eef6770591fa51349805644f934c376e43f4c68ffa6fefdbe36e	https://repository.apache.org/content/repositories/snapshots/org/apache/commons/commons-math4-legacy-exception/4.0-SNAPSHOT/commons-math4-legacy-exception-4.0-20261018.085925-1597.jar
+commons-math4-core-4.0-beta1.jar	0a694f3203cd94b0ea2804e0fd927c651a5120ceb8504d128cfb9e199783c188	https://repo.maven.apache.org/maven2/org/apache/commons/commons-math4-core/4.0-beta1/commons-math4-core-4.0-beta1.jar
+commons-math4-legacy-4.0-beta1.jar	c0d4bdedc85682630c20d7c751135ce958bc595e813647fbe7ee9038912e0337	https://repo.maven.apache.org/maven2/org/apache/commons/commons-math4-legacy/4.0-beta1/commons-math4-legacy-4.0-beta1.jar
+commons-math4-legacy-core-4.0-beta1.jar	c5bcc8d8f62f3797f5dc623fbfd4a2907383c513f91620a176868efb36aeb101	https://repo.maven.apache.org/maven2/org/apache/commons/commons-math4-legacy-core/4.0-beta1/commons-math4-legacy-core-4.0-beta1.jar
+commons-math4-legacy-exception-4.0-beta1.jar	eab5de858ddc35e55e9db0f2eff7621dafe36a92eb885aec2f9fe313ee55cb14	https://repo.maven.apache.org/maven2/org/apache/commons/commons-math4-legacy-exception/4.0-beta1/commons-math4-legacy-exception-4.0-beta1.jar
 commons-numbers-angle-1.3.jar	021369f205d6dc9a189db54c065f10d617541b58e05acf0c59a8de0995cdd122	https://repo.maven.apache.org/maven2/org/apache/commons/commons-numbers-angle/1.3/commons-numbers-angle-1.3.jar
 commons-numbers-arrays-1.3.jar	758460f666c1316a2bae58cce1ef5010c41761a2f2023f94f54842ecab85c41f	https://repo.maven.apache.org/maven2/org/apache/commons/commons-numbers-arrays/1.3/commons-numbers-arrays-1.3.jar
 commons-numbers-combinatorics-1.3.jar	22485298bb05cae240460a5405848e0a6ba7883f95a1221144859346be0a0a1d	https://repo.maven.apache.org/maven2/org/apache/commons/commons-numbers-combinatorics/1.3/commons-numbers-combinatorics-1.3.jar

benchmark/launcher/src/main/java/dev/nm/benchmark/launcher/BenchmarkMain.java

@@ -47,16 +47,18 @@ static List<Case> plan(boolean smoke, List<String> libraries, int availableProce
     }
     public static void main(String[] args) throws Exception {
         boolean smoke = false; Path output = Path.of("nmdev-benchmark-output"); int maxCases = Integer.MAX_VALUE;
-        List<String> libraries = LIBRARIES;
+        List<String> libraries = LIBRARIES; Path nmdevJar=null;
         for (int i=0;i<args.length;i++) switch(args[i]) {
             case "--output": output=Path.of(args[++i]); break;
+            case "--nmdev-jar": nmdevJar=Path.of(args[++i]); break;
             case "--smoke": smoke=true; break;
             case "--max-cases": maxCases=Integer.parseInt(args[++i]); break;
             case "--libraries": libraries=List.of(args[++i].split(",")); if(!LIBRARIES.containsAll(libraries)) throw new IllegalArgumentException("Unknown library"); break;
-            case "--help": System.out.println("java -jar nmdev-benchmark-1.2.0.jar [--output directory]\nNo arguments runs all 11 libraries (12 implementation variants) and all profiles. Repeat the same command to resume.\nDiagnostics only: --smoke [--libraries NMDEV,EJML] [--max-cases N]"); return;
+            case "--help": System.out.println("java -jar nmdev-benchmark-1.3.0.jar [--nmdev-jar path/to/nmdev.jar] [--output directory]\nNo arguments runs all 11 libraries (12 implementation variants) and all profiles. Repeat the same command to resume.\nDiagnostics only: --smoke [--libraries NMDEV,EJML] [--max-cases N]"); return;
             default: throw new IllegalArgumentException("Unknown argument: "+args[i]);
         }
         if (!smoke && !libraries.equals(LIBRARIES)) throw new IllegalArgumentException("--libraries is a diagnostic option requiring --smoke");
+        if(nmdevJar!=null) NmdevInput.open(nmdevJar);
         output=output.toAbsolutePath().normalize(); Files.createDirectories(output);
         final Path root=output;
         try (FileChannel channel=FileChannel.open(output.resolve(".runner.lock"),StandardOpenOption.CREATE,StandardOpenOption.WRITE)) {
@@ -70,22 +72,25 @@ public static void main(String[] args) throws Exception {
                     catch(Exception ex) { System.err.println("Could not finish interruption ZIP: "+ex.getMessage()); }
                 },"benchmark-shutdown");
                 Runtime.getRuntime().addShutdownHook(hook);
-                try { run(output,smoke,libraries,maxCases); }
+                try { run(output,smoke,libraries,maxCases,nmdevJar); }
                 catch(Exception ex) {
                     Storage.json(output.resolve("launcher-error.json"),Map.of("utc",Instant.now().toString(),"type",ex.getClass().getName(),"message",String.valueOf(ex.getMessage())));
                     Storage.bundle(output); throw ex;
                 } finally { try { Runtime.getRuntime().removeShutdownHook(hook); } catch (IllegalStateException ignored) { /* shutdown is already in progress */ } }
             }
         }
     }
-    static void run(Path output, boolean smoke, List<String> libraries, int maxCases) throws Exception {
+    static void run(Path output, boolean smoke, List<String> libraries, int maxCases, Path nmdevJar) throws Exception {
+        Path previousRequest=output.resolve("request.json");
+        if(Files.exists(previousRequest) && !Storage.read(previousRequest).getAsJsonObject().get("application_sha256").getAsString().equals(Storage.sha(Dependencies.application())))
+            throw new IOException("Benchmark application changed. Use a new --output directory; saved results were preserved.");
         List<Case> cases=plan(smoke,libraries);
         Map<String,Object> machine=MachineInfo.collect();
         if(!Files.exists(output.resolve("machine.json"))) Storage.json(output.resolve("machine.json"),machine);
         Storage.json(output.resolve("latest-machine.json"),machine);
         System.out.println("NM Dev unattended benchmark: "+cases.size()+" cases, two independent JVM forks per case.");
         System.out.println("Output: "+output+"; repeat this command to resume. Long operations can take several minutes.");
-        Map<String,String> artifacts=Dependencies.prepare(output); Dependencies.harness(output);
+        Map<String,String> artifacts=Dependencies.prepare(output,nmdevJar); Dependencies.harness(output);
         Map<String,Object> request=new LinkedHashMap<>();
         request.put("format",2); request.put("all_cores_available_processors",Runtime.getRuntime().availableProcessors()); request.put("suite",smoke?"SMOKE-NOT-FOR-PUBLICATION":"FULL"); request.put("cases",cases); request.put("artifacts",artifacts);
         request.put("machine",MachineInfo.identity(machine)); request.put("application_sha256",Storage.sha(Dependencies.application()));

benchmark/launcher/src/main/java/dev/nm/benchmark/launcher/Dependencies.java

@@ -13,8 +13,8 @@
 
 /** Prepares immutable public inputs without Maven credentials, Python or Git. */
 final class Dependencies {
-    static final String NM_URL = "https://github.com/nmltd/maven-packages/releases/download/nmdev-1.2.6-validated/nmdev-1.2.6.jar";
-    static final String NM_SHA = "0b2d807d17927b482fadcdc313768a61d2fbdc80c7da4226005a198e0a6796ba";
+    static final String NM_URL = "https://github.com/nmltd/maven-packages/releases/download/nmdev-1.2.10-validated/nmdev-1.2.10.jar";
+    static final String NM_SHA = "a4284d16dcb83851c5486892a4425e7b5c6b9341895b484609449aee30d4f846";
     static final String LICENSE_URL = "https://raw.githubusercontent.com/nmltd/nmdev-examples/ad1fbacaeea9c92ead37beb3002566c33b5a6006/nm.lic";
     static final String LICENSE_SHA = "2fbc9e56f7d8d933bfd7c59c39cf08bee151305b285f35a19882cc2c844bb1c1";
     static final String COLT_URL = "https://codeload.github.com/rwl/ParallelColt/zip/7c34364026680c1113f1a9586b9fb8c55080330c";
@@ -61,9 +61,18 @@ protected PasswordAuthentication getPasswordAuthentication() {
         }
         throw new IOException("Unable to prepare verified dependency " + path.getFileName(), last);
     }
-    static Map<String,String> prepare(Path output) throws Exception {
+    static Map<String,String> prepare(Path output, Path selectedJar) throws Exception {
         Path runtime = output.resolve(".runtime"), lib = runtime.resolve("lib");
         Files.createDirectories(lib);
+        NmdevInput selected=selectedJar==null?null:NmdevInput.open(selectedJar);
+        NmdevInput.requireCompatibleRun(output,selected==null?NM_SHA:selected.sha256());
+        Path runtimeJar=lib.resolve(NmdevInput.RUNTIME_NAME);
+        if(selected==null) {
+            download(NM_URL,NM_SHA,runtimeJar);
+            selected=NmdevInput.open(runtimeJar);
+        }
+        selected.install(runtimeJar);
+        System.out.println("NMDEV binary: "+selected.label()+"; SHA-256 "+selected.sha256());
         resource("config/dependencies.lock.tsv", output.resolve("dependencies.lock.tsv.txt"));
         resource("config/versions.json", output.resolve("versions.json"));
         String lock = Files.readString(output.resolve("dependencies.lock.tsv.txt"));
@@ -74,7 +83,9 @@ static Map<String,String> prepare(Path output) throws Exception {
             if (values.length != 3) throw new IOException("Malformed dependency lock entry");
             if (!values[2].startsWith("source:")) download(values[2], values[1], lib.resolve(values[0]));
         }
-        download(NM_URL, NM_SHA, lib.resolve("nmdev-1.2.6.jar"));
+        var versions=Storage.read(output.resolve("versions.json")).getAsJsonObject();
+        versions.addProperty("NMDEV",selected.label());
+        Storage.json(output.resolve("versions.json"),versions);
         Path customLicense = System.getProperty("nmdev.license") == null ? null : Path.of(System.getProperty("nmdev.license")).toAbsolutePath();
         if (customLicense != null) Storage.atomic(runtime.resolve("nm.lic"), Files.readAllBytes(customLicense));
         else download(LICENSE_URL, LICENSE_SHA, runtime.resolve("nm.lic"));
@@ -99,7 +110,7 @@ static Map<String,String> prepare(Path output) throws Exception {
         try (var files = Files.list(lib)) { for (Path file : files.filter(p -> p.toString().endsWith(".jar")).sorted().toList()) hashes.put(file.getFileName().toString(), Storage.sha(file)); }
         Storage.json(output.resolve("artifacts.json"), hashes);
         Storage.json(output.resolve("parallelcolt-build.json"), Storage.read(stamp));
-        Storage.json(output.resolve("binary-provenance.json"), Map.of("nmdev_url", NM_URL, "nmdev_sha256", NM_SHA, "nmdev_checksum_source", "https://github.com/nmltd/maven-packages/releases/download/nmdev-1.2.6-validated/SHA256SUMS", "public_trial_url", LICENSE_URL,
+        Storage.json(output.resolve("binary-provenance.json"), Map.of("nmdev_url", selectedJar==null?NM_URL:"local-file", "nmdev_sha256", selected.sha256(), "nmdev_checksum_source", selectedJar==null?"https://github.com/nmltd/maven-packages/releases/download/nmdev-1.2.10-validated/SHA256SUMS":"computed from selected JAR", "nmdev_original_filename",selectedJar==null?"nmdev-1.2.10.jar":selected.path().getFileName().toString(), "nmdev_runtime_filename",NmdevInput.RUNTIME_NAME, "nmdev_version",selected.version(), "public_trial_url", LICENSE_URL,
             "parallelcolt_source_url", COLT_URL, "parallelcolt_source_sha256", COLT_SHA, "parallelcolt_binary_is_locally_built", true));
         return hashes;
     }