Publish the root site when a port's default has no catalog yet

scripts/compose-published-inventory.mjs

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 import { execFileSync } from 'node:child_process'
 import { createHash } from 'node:crypto'
-import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'
+import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
 import { resolve } from 'node:path'
 import { pathToFileURL } from 'node:url'
 
@@ -19,38 +19,66 @@ export async function compose({ directory, runtime, catalogs, bucket, locale })
   const targets = shared.targets.filter((target) => target.scope === 'site' || target.scope.startsWith('tmux@'))
   const inputs = []
   const compatibility = []
-  for (const [port, versions] of Object.entries(manifest.ports)) {
-    if (!versions.length) continue
-    const version = manifest.defaultVersion[port]
-    if (!versions.some((entry) => entry.slug === version)) throw new Error(`No published default for ${port}/${version}`)
-    const relative = `${port}/${version}/targets.json`
+  const missing = []
+  const fallback = []
+  // A catalog's file, or null when it is absent. Only a 404 is absence.
+  const locate = (relative) => {
     let file = resolve(out, relative)
-    if (ownership.portTrees.includes(port) || !existsSync(file)) {
+    if (ownership.portTrees.includes(relative.split('/')[0]) || !existsSync(file)) {
       file = resolve(catalogs, relative)
       if (bucket) {
         mkdirSync(resolve(file, '..'), { recursive: true })
-        execFileSync('aws', ['s3', 'cp', `s3://${bucket}/${locale}/${relative}`, file, '--only-show-errors'], { stdio: 'inherit' })
+        try {
+          execFileSync('aws', ['s3', 'cp', `s3://${bucket}/${locale}/${relative}`, file, '--only-show-errors'], { stdio: ['ignore', 'inherit', 'pipe'], encoding: 'utf8' })
+        } catch (error) {
+          // Only a missing object is absence; credentials, network and permission errors still fail.
+          if (!/\(404\)|Not Found/.test(String(error.stderr ?? ''))) {
+            process.stderr.write(String(error.stderr ?? ''))
+            throw error
+          }
+        }
       }
-      if (!existsSync(file)) throw new Error(`Publish ${locale}/${relative} before advertising its full inventory`)
     }
+    return existsSync(file) ? file : null
+  }
+  for (const [port, versions] of Object.entries(manifest.ports)) {
+    if (!versions.length) continue
+    const version = manifest.defaultVersion[port]
+    if (!versions.some((entry) => entry.slug === version)) throw new Error(`No published default for ${port}/${version}`)
+    // The advertised default may predate targets.json; its latest tree is the fallback.
+    let used = version
+    let relative = `${port}/${version}/targets.json`
+    let file = locate(relative)
+    if (!file && version !== 'latest') {
+      used = 'latest'
+      relative = `${port}/latest/targets.json`
+      file = locate(relative)
+    }
+    if (!file) {
+      missing.push({ port, version, catalog: `${base}${port}/${version}/targets.json` })
+      rmSync(resolve(out, 'reference', port, 'objects.inv'), { force: true })
+      continue
+    }
+    if (used !== version) fallback.push({ port, default: version, used, catalog: `${base}${relative}` })
     const bytes = readFileSync(file)
+    const published = used
     const catalog = JSON.parse(bytes.toString())
-    const expectedBase = `${base}${port}/${version}/`
-    if (catalog.schema !== 1 || catalog.scope !== port || catalog.version !== version || catalog.base !== expectedBase || !catalog.targets?.length) {
+    const expectedBase = `${base}${port}/${published}/`
+    if (catalog.schema !== 1 || catalog.scope !== port || catalog.version !== published || catalog.base !== expectedBase || !catalog.targets?.length) {
       throw new Error(`Published target catalog has the wrong identity: ${relative}`)
     }
     for (const target of catalog.targets) {
-      if (target.scope !== port || target.version !== version || !target.href.startsWith(expectedBase) ||
+      if (target.scope !== port || target.version !== published || !target.href.startsWith(expectedBase) ||
           !new URL(target.href, 'https://inventory.invalid').pathname.startsWith(expectedBase) || target.key !== targetKey(target)) {
-        throw new Error(`Published target is outside ${port}/${version}: ${target.key}`)
+        throw new Error(`Published target is outside ${port}/${published}: ${target.key}`)
       }
     }
     targets.push(...catalog.targets)
-    inputs.push({ port, version, catalog: `${base}${relative}`, sha256: createHash('sha256').update(bytes).digest('hex') })
+    inputs.push({ port, version: published, catalog: `${base}${relative}`, sha256: createHash('sha256').update(bytes).digest('hex') })
     const entries = compatibilityEntries(catalog.targets, base).map((entry) => ({ ...entry, uri: `../../${entry.uri}` }))
     compatibility.push({
       file: resolve(out, 'reference', port, 'objects.inv'),
-      bytes: writeInventoryEntries(entries, { project: `libtmux ${port}`, version, legacyCaseSensitiveLabels: true }),
+      bytes: writeInventoryEntries(entries, { project: `libtmux ${port}`, version: published, legacyCaseSensitiveLabels: true }),
     })
   }
   targets.sort((a, b) => a.key < b.key ? -1 : a.key > b.key ? 1 : 0)
@@ -64,8 +92,10 @@ export async function compose({ directory, runtime, catalogs, bucket, locale })
   }
   writeFileSync(resolve(out, 'objects-full.inv'), full)
   writeFileSync(resolve(out, 'objects.inv'), legacy)
-  writeFileSync(resolve(out, 'targets.json'), `${JSON.stringify({ schema: 1, base, defaults: manifest.defaultVersion, coverage: { inputs }, targets })}\n`)
-  return { targets: targets.length, inputs }
+  writeFileSync(resolve(out, 'targets.json'), `${JSON.stringify({ schema: 1, base, defaults: manifest.defaultVersion, coverage: { inputs, fallback, missing }, targets })}\n`)
+  for (const { port, default: from, used, catalog } of fallback) console.log(`Composed ${port}/${used} because ${port}/${from} has not published its catalog: ${catalog}`)
+  for (const { port, version, catalog } of missing) console.log(`Left out of the full inventory: ${port}/${version} has not published ${catalog}`)
+  return { targets: targets.length, inputs, fallback, missing }
 }
 
 if (import.meta.url === pathToFileURL(process.argv[1]).href) {

site/src/content/docs/meta/linking.md

@@ -24,9 +24,9 @@ links to select a language, version or preview.
 | [docs.json](/docs.json) | Discover page titles, URLs, descriptions, headings and Markdown URLs in a structured manifest. |
 | Markdown | Read one page as text using its footer's **Markdown** link. |
 | Raw source | Inspect the source file using its footer's **Raw source** link, where available. |
-| [objects-full.inv](/objects-full.inv) | Resolve Sphinx references across shared docs, advertised port defaults and tmux reference versions. |
+| [objects-full.inv](/objects-full.inv) | Resolve Sphinx references across shared docs, each port's default version, or its latest until the default has published a catalog, and tmux reference versions. |
 | objects.inv | Resolve Sphinx references within one port and version, or one tmux documentation tree. |
-| [targets.json](/targets.json) | Browse inventory keys, display names and destination URLs as JSON. A scoped catalog sits beside each root inventory. |
+| [targets.json](/targets.json) | Browse inventory keys, display names and destination URLs as JSON. A scoped catalog sits beside each root inventory. The root one lists fallbacks and ports left out. |
 
 The root exports above cover the assembled site. Port exports describe the
 selected port and version. For a single API declaration, start with

site/test/published-inventory.test.ts

@@ -1,6 +1,6 @@
 import { execFileSync } from 'node:child_process'
 import { createHash } from 'node:crypto'
-import { cpSync, mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
+import { chmodSync, cpSync, existsSync, mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
 import { tmpdir } from 'node:os'
 import { dirname, join, resolve } from 'node:path'
 import { afterEach, describe, expect, it } from 'vitest'
@@ -31,10 +31,36 @@ function fixture(published = true, wrongScope = false) {
   return { path, catalog }
 }
 
-function compose(path: string) {
-  return execFileSync(process.execPath, [join(root, 'scripts/compose-published-inventory.mjs'), '--directory', 'dist', '--catalogs', 'published', '--runtime', 'inventory-runtime'], { cwd: path, encoding: 'utf8', stdio: 'pipe' })
+function compose(path: string, extra: string[] = [], env: NodeJS.ProcessEnv = process.env) {
+  return execFileSync(process.execPath, [join(root, 'scripts/compose-published-inventory.mjs'), '--directory', 'dist', '--catalogs', 'published', '--runtime', 'inventory-runtime', ...extra], { cwd: path, encoding: 'utf8', stdio: 'pipe', env })
 }
 
+/** Add ports whose default catalogs are published; the rest stay absent. */
+function addPorts(path: string, ports: Record<string, boolean>) {
+  const read = (file: string) => JSON.parse(readFileSync(join(path, file), 'utf8'))
+  const write = (file: string, object: unknown) => writeFileSync(join(path, file), JSON.stringify(object))
+  const manifest = read('dist/versions.json')
+  for (const [port, published] of Object.entries(ports)) {
+    manifest.ports[port] = [{ slug: 'v1.0.0' }]
+    manifest.defaultVersion[port] = 'v1.0.0'
+    if (!published) continue
+    const value = { scope: port, version: 'v1.0.0', product: 'core', category: 'api', identity: 'Pane', display: 'Pane', href: `/en/${port}/v1.0.0/reference/pane/#Pane` }
+    const target = { ...value, key: targetKey(value), legacy: [{ type: 'std:label', name: 'Pane' }] }
+    mkdirSync(join(path, `published/${port}/v1.0.0`), { recursive: true })
+    write(`published/${port}/v1.0.0/targets.json`, { schema: 1, base: `/en/${port}/v1.0.0/`, scope: port, version: 'v1.0.0', targets: [target] })
+  }
+  write('dist/versions.json', manifest)
+}
+
+/** A fake `aws` on PATH that fails every call with the given stderr. */
+function fakeAws(path: string, stderr: string) {
+  const bin = join(path, 'bin'); mkdirSync(bin, { recursive: true })
+  writeFileSync(join(bin, 'aws'), `#!/bin/sh\ncat >&2 <<'EOF'\n${stderr}\nEOF\nexit 1\n`)
+  chmodSync(join(bin, 'aws'), 0o755)
+  return { ...process.env, PATH: `${bin}:${process.env.PATH}` }
+}
+const resultOf = (output: string) => JSON.parse(output.trim().split('\n').at(-1) as string)
+
 describe('published default inventory composition', () => {
   it('uses the published stable catalog instead of a locally assembled latest tree', () => {
     const { path } = fixture()
@@ -56,8 +82,83 @@ describe('published default inventory composition', () => {
       { name: 'Pane', type: 'std:label', uri: '../../rs/stable/reference/pane/#Pane' },
     ])
   })
-  it('refuses to advertise a default whose catalog has not been published', () => {
-    expect(() => compose(fixture(false).path)).toThrow('Publish en/rs/stable/targets.json before advertising')
+  it('leaves out a port whose default catalog is absent and reports it', () => {
+    const { path } = fixture()
+    addPorts(path, { py: false })
+    mkdirSync(join(path, 'dist/reference/py'), { recursive: true })
+    writeFileSync(join(path, 'dist/reference/py/objects.inv'), 'stale')
+    const output = compose(path)
+    expect(output).toContain('py/v1.0.0')
+    const catalog = JSON.parse(readFileSync(join(path, 'dist/targets.json'), 'utf8'))
+    expect(catalog.coverage.missing).toEqual([{ port: 'py', version: 'v1.0.0', catalog: '/en/py/v1.0.0/targets.json' }])
+    expect(catalog.coverage.inputs.map((input: { port: string }) => input.port)).toEqual(['rs'])
+    expect(catalog.targets.map((target: { scope: string }) => target.scope)).toEqual(['rs'])
+    expect(readInventory(readFileSync(join(path, 'dist/objects-full.inv'))).entries.map(({ uri }) => uri).join()).not.toContain('py/')
+    expect(() => readFileSync(join(path, 'dist/reference/py/objects.inv'))).toThrow()
+    expect(readFileSync(join(path, 'dist/reference/rs/objects.inv')).length).toBeGreaterThan(0)
+  })
+  it('composes an absent default from the port latest catalog and reports the fallback', () => {
+    const { path } = fixture()
+    addPorts(path, { py: false })
+    const value = { scope: 'py', version: 'latest', product: 'core', category: 'api', identity: 'Pane', display: 'Pane', href: '/en/py/latest/reference/pane/#Pane' }
+    const target = { ...value, key: targetKey(value), legacy: [{ type: 'std:label', name: 'Pane' }] }
+    const latest = { schema: 1, base: '/en/py/latest/', scope: 'py', version: 'latest', targets: [target] }
+    mkdirSync(join(path, 'published/py/latest'), { recursive: true })
+    writeFileSync(join(path, 'published/py/latest/targets.json'), JSON.stringify(latest))
+    const output = compose(path)
+    expect(output).toContain('py/latest')
+    const catalog = JSON.parse(readFileSync(join(path, 'dist/targets.json'), 'utf8'))
+    expect(catalog.coverage.fallback).toEqual([{ port: 'py', default: 'v1.0.0', used: 'latest', catalog: '/en/py/latest/targets.json' }])
+    expect(catalog.coverage.missing).toEqual([])
+    expect(catalog.targets.filter((entry: { scope: string }) => entry.scope === 'py').map((entry: { href: string }) => entry.href)).toEqual(['/en/py/latest/reference/pane/#Pane'])
+    expect(readInventory(readFileSync(join(path, 'dist/reference/py/objects.inv'))).entries[0].uri).toBe('../../py/latest/reference/pane/#Pane')
+    // A latest catalog gets the same identity checks as any other.
+    writeFileSync(join(path, 'published/py/latest/targets.json'), JSON.stringify({ ...latest, base: '/en/py/stable/' }))
+    expect(() => compose(path)).toThrow('wrong identity')
+  })
+  it('removes the legacy port inventory when neither default nor latest is published', () => {
+    const { path } = fixture()
+    addPorts(path, { py: false })
+    const file = join(path, 'dist/reference/py/objects.inv')
+    mkdirSync(dirname(file), { recursive: true }); writeFileSync(file, 'stale')
+    compose(path)
+    expect(existsSync(file)).toBe(false)
+  })
+  it('composes a site and tmux only inventory when no default catalog is published', () => {
+    const { path } = fixture(false)
+    const target = (scope: string) => { const value = { scope, version: scope === 'tmux@3.7c' ? '3.7c' : 'latest', product: 'core', category: 'api', identity: 'Site', display: 'Site', href: '/en/site/' }; return { ...value, key: targetKey(value), legacy: [{ type: 'std:label', name: 'Site' }] } }
+    writeFileSync(join(path, 'dist/targets.json'), JSON.stringify({ schema: 1, base: '/en/', targets: [target('site'), target('tmux@3.7c'), target('rs')] }))
+    addPorts(path, { py: false, go: false })
+    const result = resultOf(compose(path))
+    expect(result.missing.map(({ port }: { port: string }) => port)).toEqual(['rs', 'py', 'go'])
+    const catalog = JSON.parse(readFileSync(join(path, 'dist/targets.json'), 'utf8'))
+    expect(catalog.coverage.inputs).toEqual([])
+    expect(catalog.targets.map((entry: { scope: string }) => entry.scope).sort()).toEqual(['site', 'tmux@3.7c'])
+    expect(readInventory(readFileSync(join(path, 'dist/objects-full.inv'))).entries.length).toBe(2)
+  })
+  it('treats an S3 404 as absent', () => {
+    const { path } = fixture(false)
+    const env = fakeAws(path, 'fatal error: An error occurred (404) when calling the HeadObject operation: Key "en/rs/stable/targets.json" does not exist')
+    expect(resultOf(compose(path, ['--bucket', 'bucket'], env)).missing).toHaveLength(1)
+  })
+  it('still fails on an aws error that is not a missing object', () => {
+    const { path } = fixture(false)
+    const env = fakeAws(path, 'fatal error: An error occurred (403) when calling the HeadObject operation: Forbidden')
+    const full = join(path, 'dist/objects-full.inv'); writeFileSync(full, 'retained')
+    expect(() => compose(path, ['--bucket', 'bucket'], env)).toThrow('Forbidden')
+    expect(readFileSync(full, 'utf8')).toBe('retained')
+  })
+  it('still fails when a published catalog has the wrong identity', () => {
+    const { path, catalog } = fixture()
+    writeFileSync(join(path, 'published/rs/stable/targets.json'), JSON.stringify({ ...catalog, base: '/en/rs/other/' }))
+    expect(() => compose(path)).toThrow('wrong identity')
+  })
+  it('still fails when a default version is not in the manifest', () => {
+    const { path } = fixture()
+    const file = join(path, 'dist/versions.json')
+    const manifest = JSON.parse(readFileSync(file, 'utf8')); manifest.defaultVersion.rs = 'v9'
+    writeFileSync(file, JSON.stringify(manifest))
+    expect(() => compose(path)).toThrow('No published default for rs/v9')
   })
   it('rejects a catalog containing another port before writing the aggregate', () => {
     const { path } = fixture(true, true)