boot.lua

-- boot.lua : load the full Shen KLambda kernel into the Lua runtime and
-- initialise it. Returns the prims module P with everything live. The
-- ShenOSKernel 43.1 modules are pure defuns; initialise() below runs
-- (shen.initialise), the extension initialisers and (stlib.initialise).
local R = require("runtime")
-- restores the stack-segment count on a caught error (runtime.lua, #93)
local pcall = R.seg.pcall
local C = require("compiler")
local P = require("prims")

-- LuaJIT trace/mcode tuning, plus an optional switch to disable the JIT.
--
-- SHEN_JIT=off disables the LuaJIT compiler for the whole process (the
-- in-library equivalent of `luajit -j off`). This is a mitigation for issue
-- #43: on aarch64 an OLD LuaJIT (2.1.0-beta3 era) mis-compiles one of the
-- ~1500 traces generated while loading the kernel, and the bad trace branches
-- into unmapped memory (SIGSEGV during boot; stochastic, ~0.3%/boot on a late
-- beta3 rolling build, 50/50 on the genuine 2017 tag). It is a LuaJIT backend
-- bug, ALREADY FIXED upstream: the real fix is a current 2.1 rolling release
-- (0 crashes on the same boot). SHEN_JIT=off is the fallback for hosts pinned
-- to an old LuaJIT. NOTE it is distinct from SHEN_JIT_OPT=off, which only
-- restores the host's default jit.opt limits and leaves the JIT ON — that does
-- not prevent the crash.
--
-- With the JIT on, LuaJIT's default mcode area (512KB, 1000 traces) is far too
-- small for the compiled kernel: on arm64 the suite triggers dozens of full
-- trace-cache flushes per run ("failed to allocate mcode memory"), costing
-- ~10-16% of total wall time re-JITting the same code. Raise the limits once
-- at boot. SHEN_JIT_OPT=off restores the host's defaults (embedders that
-- manage jit.opt themselves should set it).
local function disable_jit()
  local jit_ok, jit = pcall(require, "jit")
  if jit_ok and jit and jit.off then pcall(jit.off) end
end
P.disable_jit = disable_jit   -- so shen.boot{jit=false} can drive it too
do
  if os.getenv("SHEN_JIT") == "off" then
    disable_jit()
  else
    local jit_ok, jit = pcall(require, "jit")
    if jit_ok and jit then
      -- The beta arm64 backend is known to mis-compile a boot trace and can
      -- SIGSEGV (issue #43). Disable it by default on that exact combination;
      -- this keeps old distro/OpenResty LuaJIT builds crash-free while leaving
      -- current rolling builds unchanged. SHEN_JIT=on is an explicit opt-in
      -- for embedders that have verified their LuaJIT build.
      if jit.arch == "arm64" and tostring(jit.version):find("beta", 1, true)
         and os.getenv("SHEN_JIT") ~= "on" then
        disable_jit()
        io.stderr:write("shen-lua: disabling JIT for " .. tostring(jit.version)
          .. " on arm64 (known boot SIGSEGV; issue #43). Upgrade LuaJIT or"
          .. " set SHEN_JIT=on to override.\n")
      end
      if jit.opt and os.getenv("SHEN_JIT_OPT") ~= "off" then
        pcall(jit.opt.start,
          "sizemcode=2048", "maxmcode=131072", "maxtrace=8000", "maxside=400")
      end
      -- Some hosts leave the JIT nominally ON but deny the process executable
      -- trace memory (issue #55): macOS hardened-runtime binaries embedding
      -- LuaJIT (e.g. Envoy's Lua filter — no JIT entitlement) report
      -- jit.status() == true, yet every trace attempt aborts with "failed to
      -- allocate mcode memory" and is re-attempted on the next hot path.
      -- Measured in Envoy 1.39 on an arm64 Mac: a hot loop ran ~550x slower
      -- than plain interpretation and kernel boot took 40-66 s (vs ~3 s
      -- interpreted). Detect it directly — compile one throwaway hot loop and
      -- watch for a trace "stop" event — and fall back to the interpreter.
      -- SHEN_JIT=on skips the probe (explicit opt-in for verified hosts).
      -- So does a loaded jit.v / jit.dump (`luajit -jv`, `-jdump`): LuaJIT
      -- keeps ONE handler per VM event, so attaching the probe's watcher
      -- replaced theirs and detaching it left none -- trace diagnostics
      -- printed nothing (#66). Their own output shows whether traces compile.
      local tracing = package.loaded["jit.v"] or package.loaded["jit.dump"]
      if os.getenv("SHEN_JIT") ~= "on" and not tracing
         and jit.status and jit.status() and jit.attach then
        local compiled = false
        local watcher = function(what)
          if what == "stop" then compiled = true end
        end
        if pcall(jit.attach, watcher, "trace") then
          local mk = loadstring or load
          local probe = mk("local s = 0 for i = 1, 400 do s = s + i end return s")
          if probe then for _ = 1, 3 do probe() end end
          pcall(jit.attach, watcher)   -- detach
          if not compiled then
            disable_jit()
            io.stderr:write("shen-lua: the JIT reports enabled but compiled no"
              .. " trace (executable memory denied? hardened host?); running"
              .. " interpreted. Set SHEN_JIT=on to skip this probe.\n")
          end
        end
      end
    end
  end
  -- GC tuning. Compiled-KL workloads are cons-churn-heavy (jit.p on urdr's
  -- software SHA-256 suite: ~27% of wall time in the GC at LuaJIT's default
  -- pause=200), and most of that churn is short-lived list cells. Raising
  -- the pause to 400 (heap may grow to 4x live before a full cycle) cuts
  -- suite CPU ~15-20% for about 2x peak RSS (30MB -> 60MB on that suite).
  -- SHEN_GC=off keeps the host's defaults (embedders that manage the GC
  -- themselves should set it); SHEN_GC="pause[,stepmul]" sets explicit
  -- values (e.g. SHEN_GC=800,100 buys another ~10% on batch runs at ~110MB;
  -- SHEN_GC=200 is LuaJIT's default pause).
  local gc = os.getenv("SHEN_GC")
  if gc ~= "off" then
    local pause, stepmul
    if gc and gc ~= "" then pause, stepmul = gc:match("^(%d+),?(%d*)$") end
    collectgarbage("setpause", tonumber(pause) or 400)
    if stepmul and stepmul ~= "" then collectgarbage("setstepmul", tonumber(stepmul)) end
  end
end

local function find_kldir()
  local env = os.getenv("SHEN_KL_DIR")
  if env and env ~= "" then return env end

  -- Prefer the klambda next to this module. Cwd-relative "klambda/" is
  -- wrong when the launcher is invoked from another project (Bifrost's
  -- cwd is ../bifrost; ../cl-source/... then wins and can be a different
  -- kernel missing files). For a luarocks install boot.lua lives at
  -- <tree>/share/lua/5.1/boot.lua and copy_directories puts klambda at
  -- <tree>/lib/luarocks/rocks-5.1/shen/<version>/klambda.
  local candidates = {}
  local src = debug.getinfo(1, "S").source
  local here = src:match("^@(.*)[/\\][^/\\]*$")
  if here then
    candidates[#candidates+1] = here .. "/klambda"
    local tree = here:match("^(.*)/share/lua/[%d.]+$")
    if tree then
      -- any installed version of the rock (scm-1, 0.9.0-1, ...): glob the
      -- rock directory rather than hardcoding a version string.
      local rocksdir = tree .. "/lib/luarocks/rocks-5.1/shen"
      local ls = io.popen('ls -1 "' .. rocksdir .. '" 2>/dev/null')
      if ls then
        for ver in ls:lines() do
          candidates[#candidates+1] = rocksdir .. "/" .. ver .. "/klambda"
        end
        ls:close()
      end
    end
  end
  for _,c in ipairs(candidates) do
    local f = io.open(c .. "/toplevel.kl", "r")
    if f then f:close(); return c end
  end

  -- Last resort: assume the vendored location (will produce a clear error)
  return "klambda"
end
local KLDIR = find_kldir() .. "/"
P.KLDIR = KLDIR   -- resolved .kl directory (trailing /), for typecheck_native
-- Boot order for the ShenOSKernel-43.1 kernel (community shen-sources line).
-- Every 43.1 .kl file is pure defuns: no module has load-time side effects, so
-- the order below (upstream make.shen's module order, then the booted
-- extensions, then the precompiled standard library) is for readability only.
-- All initialisation happens AFTER load, in initialise() below, exactly as
-- upstream ports do it:
--   (shen.initialise)  -- init.kl: environment, *property-vector* dict,
--                      -- arity table, lambda forms, kernel signatures
--   (stlib.initialise) -- stlib.kl: stdlib arities, macros, datatypes, types
--
-- Booted extensions match what shen-scheme 0.50 boots: features and
-- programmable-pattern-matching (both initialised in initialise(), in that
-- order, as shen-scheme does), expand-dynamic and launcher. Vendored but NOT
-- booted (opt-in): extension-namespaces and extension-type-annotations. See
-- klambda/PROVENANCE.md.
local FILES = {
  "yacc","core","declarations","load","prolog","reader","sequent","sys",
  "dict","t-star","toplevel","track","types","writer","init","macros",
  "extension-features","extension-expand-dynamic","extension-launcher",
  "extension-programmable-pattern-matching",
  "stlib"
}

-- ---- standard streams ----------------------------------------------------
-- *stoutput*/*sterror* write to stdout/stderr; *stinput* reads stdin bytes.
local out_stream = P.mk_out_stream(function(s) io.stdout:write(s) end, function() io.stdout:flush() end, "stdout")
local err_stream = P.mk_out_stream(function(s) io.stderr:write(s) end, function() io.stderr:flush() end, "stderr")
local in_stream  = P.mk_in_stream(function() local c = io.stdin:read(1); return c and string.byte(c) or nil end,
                                  function() end, "stdin")
P.GLOBALS["*stoutput*"] = out_stream
P.GLOBALS["*sterror*"]  = err_stream
P.GLOBALS["*stinput*"]  = in_stream
P.GLOBALS["*home-directory*"] = ""

-- ---- platform metadata (required by 42+ kernel) ---------------------------
P.GLOBALS["*language*"]       = "Lua"
P.GLOBALS["*implementation*"] = rawget(_G, "jit") and "LuaJIT" or _VERSION
P.GLOBALS["*port*"]           = "shen-lua"
P.GLOBALS["*porters*"]        = "shen-lua contributors"
P.GLOBALS["*os*"]             = (package.config and package.config:sub(1,1) == "\\") and "Windows" or "Unix"
P.GLOBALS["*release*"]        = "0.1"  -- port release; kernel *version* comes from init.kl ("43.1")

-- ---- kernel bytecode cache -------------------------------------------------
-- Loading the kernel from .kl sources costs ~0.8s (read + parse + KL->Lua
-- compile + Lua parse). The generated chunks are deterministic, so we cache
-- string.dump'd bytecode of one concatenated chunk per kernel file, keyed on
-- an FNV-1a hash of everything that determines codegen: the .kl sources, the
-- compiler/reader/prims sources (prims registers primitive arities, which
-- select direct-call vs APP codegen), the file list, and the LuaJIT version/
-- arch (bytecode is not portable across either). SHEN_KERNEL_CACHE=off
-- disables; any other value overrides the cache path.
local CACHE_FORMAT = "SHENKC5"  -- 5: code-point string primitives (issue #80):
                                --    kernel chunks, fasls and stdlib images
                                --    built over byte-based pos/tlstr/
                                --    string->n/n->string/hash are stale
                                --    (fasl_key and image_path fold this key)
                                -- 4: hoisted declare typeforms evaluated before
                                --    `declare` (issue #62); 3: per-chunk hoisted
                                --    (declare ...) block + gensym/inference
                                --    counters
-- LuaJIT's `bit` library drives the FNV-1a hashing behind both the kernel
-- bytecode cache and the user fasl cache. PUC Lua has no `bit` (5.3+ has
-- native bitwise operators, but this file must stay parseable by 5.1/LuaJIT),
-- so when it is absent both caches self-disable: cache_path()/fasl_dir()
-- return nil, which makes every hashing path (fnv1a/cache_key/fasl_key)
-- unreachable. Pure perf features — correctness is unaffected.
local has_bit, bit = pcall(require, "bit")
if not has_bit then bit = nil end

local function fnv1a(s, h)
  h = h or 2166136261
  local bxor, lshift, tobit, byte = bit.bxor, bit.lshift, bit.tobit, string.byte
  for i = 1, #s do
    h = bxor(h, byte(s, i))
    -- h = h * 16777619 in 32-bit (2^24 + 2^8 + 2^7 + 2^4 + 2^1 + 1):
    -- a direct multiply overflows the double-exact range under bit.band.
    h = tobit(h + lshift(h, 1) + lshift(h, 4) + lshift(h, 7) + lshift(h, 8) + lshift(h, 24))
  end
  return h
end

-- Each Lua build gets its own default cache FILE, not just its own key:
-- bytecode is only portable within the exact build, and two hosts sharing one
-- path — your `luajit` and an embedded LuaJIT with a different jit.version
-- (OpenResty's, Envoy's) — would see a key mismatch on every alternation and
-- invalidate + rewrite each other's cache, recompiling the kernel every time.
-- The filename suffix is a hash of the same version/arch fingerprint that
-- cache_key() folds into the content key.
--
-- The default lives in the per-user cache directory,
-- ${XDG_CACHE_HOME:-$HOME/.cache}/shen-lua/ (issue #87: it used to be a
-- relative path, so every directory shen ran from got its own ~1.8 MB copy
-- and recompiled the kernel). With neither variable set there is no default
-- and the kernel is compiled on every boot, as with SHEN_KERNEL_CACHE=off.
local function cache_path()
  if not bit then return nil end   -- PUC Lua: no `bit` -> no cache keys
  local p = os.getenv("SHEN_KERNEL_CACHE")
  if p == "off" or p == "0" then return nil end
  if p and p ~= "" then return p end
  local dir = os.getenv("XDG_CACHE_HOME")
  if not dir or dir == "" then
    local home = os.getenv("HOME")
    if not home or home == "" then return nil end
    dir = home .. "/.cache"
  end
  return dir .. "/shen-lua/kernel-"
      .. bit.tohex(fnv1a(jit and (jit.version .. jit.arch) or _VERSION))
      .. ".bin"
end

-- io.open(path, "wb"), creating path's directory on the first write. Returns
-- nil when it still cannot be opened (read-only location): callers then skip
-- caching.
local function open_for_write(path)
  local fh = io.open(path, "wb")
  if fh then return fh end
  local dir = path:match("^(.*)/[^/]*$")
  if not dir or dir == "" then return nil end
  os.execute("mkdir -p '" .. dir:gsub("'", "'\\''") .. "' 2>/dev/null")
  return io.open(path, "wb")
end

local function read_file(path)
  local fh = io.open(path, "rb")
  if not fh then return nil end
  local s = fh:read("*a"); fh:close()
  return s
end

local function module_source(name)
  local path = package.searchpath and package.searchpath(name, package.path)
  return read_file(path or (name .. ".lua")) or ""
end

-- key over kl sources + codegen-relevant module sources; returns hex string,
-- plus the kl sources themselves (the compile path needs them anyway).
local function cache_key()
  local h = fnv1a(jit and (jit.version .. jit.arch) or _VERSION)
  h = fnv1a(CACHE_FORMAT .. table.concat(FILES, ","), h)
  -- codegen switches that change compiled output (SHEN_TRMC -> compiler.lua C.TRMC)
  h = fnv1a("trmc=" .. tostring(os.getenv("SHEN_TRMC") ~= "off"), h)
  for _, m in ipairs({ "compiler", "runtime", "prims" }) do
    h = fnv1a(module_source(m), h)
  end
  local sources = {}
  for _, nm in ipairs(FILES) do
    local s = assert(read_file(KLDIR..nm..".kl"), "cannot open "..nm)
    sources[nm] = s
    h = fnv1a(s, h)
  end
  return bit.tohex(h), sources
end

-- The compiler hoists big literal (cons ...) trees into the C.KDATA side
-- table at COMPILE time; the emitted bytecode only carries KDATA[i] reads
-- (compiler.lua try_const/try_lit_const). Cached chunks therefore need KDATA
-- rebuilt before they run. Entries are pure literal data — numbers, strings,
-- booleans, interned symbols, NIL, cons cells — so they serialize exactly.
-- Tags: N<num>\n S<len>\n<bytes> Y<len>\n<name> B1\n/B0\n L\n(=NIL) C\n<car><cdr>
local function kdata_ser(v, out)
  while R.is_cons(v) do          -- cdr spine iteratively: it's the long axis
    out[#out+1] = "C\n"
    kdata_ser(v[1], out)
    v = v[2]
  end
  local t = type(v)
  if v == R.NIL then out[#out+1] = "L\n"
  elseif t == "number" then out[#out+1] = "N" .. string.format("%.17g", v) .. "\n"
  elseif t == "string" then out[#out+1] = "S" .. #v .. "\n" .. v
  elseif t == "boolean" then out[#out+1] = v and "B1\n" or "B0\n"
  elseif R.is_symbol(v) then out[#out+1] = "Y" .. #v.name .. "\n" .. v.name
  else error("unserializable KDATA value: " .. t) end
end

local function kdata_de(data, pos)
  local tag = data:sub(pos, pos)
  local e = data:find("\n", pos, true)
  if not e then error("truncated KDATA") end
  local arg = data:sub(pos + 1, e - 1)
  pos = e + 1
  if tag == "C" then
    local hd, tl
    hd, pos = kdata_de(data, pos)
    tl, pos = kdata_de(data, pos)
    return R.cons(hd, tl), pos
  elseif tag == "L" then return R.NIL, pos
  elseif tag == "N" then return tonumber(arg), pos
  elseif tag == "B" then return arg == "1", pos
  elseif tag == "S" or tag == "Y" then
    local len = tonumber(arg)
    local s = data:sub(pos, pos + len - 1)
    pos = pos + len
    if tag == "S" then return s, pos end
    return R.intern(s), pos
  end
  error("bad KDATA tag: " .. tostring(tag))
end

-- format: CACHE_FORMAT\n key\n nchunks\n
--         { name\n #dump\n dump  ndecl\n { dname\n #ddump\n ddump }* }*
--         narities\n { arity SP fname\n }*  nkdata\n { entry }*
--         gensym\n infs\n
-- The per-chunk decl list is that kernel file's hoisted (declare ...) block:
-- one dumped prolog abstraction per signature, in file order. See
-- hoist_tail / record_declares / replay_declares below.
local function write_cache(path, key, chunks, arity, counters)
  local parts = { CACHE_FORMAT, "\n", key, "\n", tostring(#chunks), "\n" }
  for _, ch in ipairs(chunks) do
    parts[#parts+1] = ch.name .. "\n" .. #ch.dump .. "\n" .. ch.dump
    local d = ch.decl or {}
    parts[#parts+1] = #d .. "\n"
    for _, e in ipairs(d) do
      parts[#parts+1] = e.name .. "\n" .. #e.dump .. "\n" .. e.dump
    end
  end
  local an = 0
  for _ in pairs(arity) do an = an + 1 end
  parts[#parts+1] = an .. "\n"
  for name, ar in pairs(arity) do
    parts[#parts+1] = ar .. " " .. name .. "\n"
  end
  parts[#parts+1] = #C.KDATA .. "\n"
  for i = 1, #C.KDATA do
    kdata_ser(C.KDATA[i], parts)
  end
  parts[#parts+1] = tostring(counters.gensym) .. "\n" .. tostring(counters.infs) .. "\n"
  local tmp = path .. ".tmp"
  local fh = open_for_write(tmp)
  if not fh then return end  -- read-only dir: silently skip caching
  fh:write(table.concat(parts)); fh:close()
  os.remove(path)
  os.rename(tmp, path)
end

-- Parse a write_cache blob. key == nil skips the key check (used for the
-- embedded-kernel payload baked into a single-file bundle, where the build
-- pins the blob and per-chunk load failures fall back to a full compile).
local function parse_cache(data, key)
  local pos = 1
  local function line()
    local e = data:find("\n", pos, true)
    if not e then return nil end
    local s = data:sub(pos, e - 1); pos = e + 1
    return s
  end
  if line() ~= CACHE_FORMAT then return nil end
  local k = line()
  if key ~= nil and k ~= key then return nil end
  local n = tonumber(line() or ""); if not n then return nil end
  local chunks = {}
  for i = 1, n do
    local nm = line()
    local len = tonumber(line() or "")
    if not nm or not len or pos + len - 1 > #data then return nil end
    chunks[i] = { name = nm, dump = data:sub(pos, pos + len - 1) }
    pos = pos + len
    local nd = tonumber(line() or ""); if not nd then return nil end
    local decl = {}
    for j = 1, nd do
      local dn = line()
      local dl = tonumber(line() or "")
      if not dn or not dl or pos + dl - 1 > #data then return nil end
      decl[j] = { name = dn, dump = data:sub(pos, pos + dl - 1) }
      pos = pos + dl
    end
    chunks[i].decl = decl
  end
  local na = tonumber(line() or ""); if not na then return nil end
  local arity = {}
  for i = 1, na do
    local ln = line(); if not ln then return nil end
    local ar, name = ln:match("^(%-?%d+) (.*)$")
    if not ar then return nil end
    arity[name] = tonumber(ar)
  end
  local nk = tonumber(line() or ""); if not nk then return nil end
  local kdata = {}
  local kok, kerr = pcall(function()
    for i = 1, nk do
      kdata[i], pos = kdata_de(data, pos)
    end
  end)
  if not kok then return nil end
  local gensym = tonumber(line() or ""); if not gensym then return nil end
  local infs = tonumber(line() or ""); if not infs then return nil end
  return { chunks = chunks, arity = arity, kdata = kdata,
           gensym = gensym, infs = infs }
end

local function read_cache(path, key)
  local data = read_file(path)
  if not data then return nil end
  return parse_cache(data, key)
end

-- ---- load the kernel -----------------------------------------------------
-- Loads the .kl modules in FILES (see above): the 16 ShenOSKernel 43.1 kernel
-- modules, 4 booted extensions and the precompiled stlib. The opt-in
-- extension-namespaces.kl / extension-type-annotations.kl are vendored but
-- not booted.
-- The KLambda sources are vendored under `klambda/` so the repository
-- is self-contained. You can still override with SHEN_KL_DIR (e.g. to point
-- at a full ShenOSKernel checkout during development).

-- Native overrides, installed after the compiled KL defuns are all in F.
local function install_native_overrides()
  -- shen.lambda-entry without a KL->Lua compile per entry: every
  -- update-lambda-table (each `define`, and the 326 stdlib arities that
  -- (stlib.initialise-arities) registers at boot) builds one.
  P.install_native_lambda_entry()
  -- Hottest Prolog deref primitives (see prims.install_native_prolog).
  P.install_native_prolog()
  -- Hottest general-purpose kernel functions with native Lua
  -- (element?, assoc, map, reverse, fail, ...; see prims.install_native_stdlib).
  P.install_native_stdlib()
  -- Native soa32 Prolog/typecheck engine (prolog_engine.lua). Default on once
  -- the module ships; SHEN_PROLOG_ENGINE=legacy falls back to the compiled-KL
  -- CPS engine. Module absence is tolerated (pre-engine checkouts); any other
  -- load error is real and must propagate.
  if os.getenv("SHEN_PROLOG_ENGINE") ~= "legacy" then
    local ok, eng = pcall(require, "prolog_engine")
    if ok then
      eng.install(P)
    elseif not tostring(eng):find("module 'prolog_engine' not found", 1, true) then
      error(eng)
    end
  end
end

-- memoized: the fasl layer reuses the codegen key even when the kernel
-- cache is disabled (first call reads ~850KB and hashes it, ~ms).
local KERNEL_KEY
local function kernel_key()
  local k, sources = KERNEL_KEY, nil
  if not k then
    if P.KERNEL_CACHE_DATA then
      -- single-file bundle: no .kl files on disk; the embedded blob captures
      -- everything that determines codegen, so its hash is the kernel key.
      k = bit.tohex(fnv1a(P.KERNEL_CACHE_DATA))
    else
      k, sources = cache_key()
    end
    KERNEL_KEY = k
  end
  return k, sources
end

-- ---- hoisted kernel type signatures ---------------------------------------
-- (ShenOSKernel 43.1 has no top-level forms at all — its kernel signatures are
-- precompiled closures in init.kl's shen.initialise-signedfuncs — so on the
-- vendored kernel hoist_tail finds nothing to hoist. The machinery stays for
-- S42-shaped trees via SHEN_KL_DIR, and record_declares / replay_declares are
-- reused for the standard library's 290 declares; see stlib_types.)
-- On the S42.0 kernel, klambda/types.kl ended with 161 top-level
-- `(declare Name Type)` forms, and
-- they are not cheap annotations: `declare` (types.kl) runs the type theory for
-- real on every one of them —
--   (a) shen.variancy over the signature under the Prolog machine,
--   (b) (eval-kl (shen.prolog-abstraction Type)) — a full KL->Lua compile plus
--       loadstring per signature, producing the closure stored in shen.*sigf*,
--   (c) (set shen.*sigf* (shen.assoc-> Name <closure> ...)).
-- Measured on arm64 LuaJIT that block is ~43 ms: ~30% of a warm boot and its
-- largest single item, recomputed on every start even though the kernel
-- bytecode cache was hit and nothing changed.
--
-- (a) is a static check of the kernel's own signatures against the kernel's own
-- sources, and the cache key already covers every input to it; (b) is
-- deterministic given the signature. So a cached boot can skip both and keep
-- only (c), replaying the abstraction from dumped bytecode:
--
--   hoist_tail       pulls the trailing (declare ...) block out of a kernel
--                    file's forms so boot.lua — not the opaque concatenated
--                    chunk — is what runs it. Only a CONTIGUOUS TRAILING run is
--                    hoisted, so hoisting can never reorder a file's effects; a
--                    file that interleaves declares with other top-level forms
--                    keeps them inline and caches nothing for them.
--   record_declares  the cold path: runs the real `declare`, capturing each
--                    eval-kl chunk through the ordinary FASL_REC recorder
--                    (C.NO_KDATA keeps the dumps relocatable, exactly as for
--                    the user fasl cache).
--   replay_declares  the warm path: load dump, run it, assoc-> into
--                    shen.*sigf* — the same final state, no type theory.
--
-- The gensym and inference counters `declare` advances are recorded and
-- restored (load_cached below), so a cached boot's shen.*gensym* and
-- (inferences) match an uncached one exactly instead of lagging by the ~1000
-- gensyms the skipped abstractions would have consumed.
local function is_declare_form(f)
  return R.is_cons(f) and R.is_symbol(f[1]) and f[1].name == "declare"
    and R.is_cons(f[2]) and R.is_cons(f[2][2]) and f[2][2][2] == R.NIL
end

-- A declare's type argument is a KL EXPRESSION — in the 42 kernel always a
-- pure rcons constructor tree, e.g. (cons number (cons --> (cons number ()))).
-- Running the form inline evaluates it before `declare` sees it; a hoisted
-- declare must do the same, or the signature registered in shen.*sigf* is over
-- the unevaluated AST and never unifies with a real type (issue #62: System S
-- rejected every kernel signature — arithmetic being the visible case).
-- Only literal trees are evaluated here; anything else refuses the hoist so
-- the form runs inline with full KL semantics. Returns the value, or nil to
-- refuse (KL has no nil, so nil is unambiguous).
local function eval_typeform(e)
  if R.is_cons(e) then
    local h = e[1]
    if R.is_symbol(h) and h.name == "cons"
       and R.is_cons(e[2]) and R.is_cons(e[2][2]) and e[2][2][2] == R.NIL then
      local a = eval_typeform(e[2][1])
      if a == nil then return nil end
      local d = eval_typeform(e[2][2][1])
      if d == nil then return nil end
      return R.cons(a, d)
    elseif R.is_symbol(h) and h.name == "intern"
       and R.is_cons(e[2]) and type(e[2][1]) == "string" and e[2][2] == R.NIL then
      return R.intern(e[2][1])
    end
    return nil
  end
  return e   -- symbol, number, string, boolean, (): self-evaluating
end

-- Split a kernel file's forms into (body, init forms, declares). Only a
-- TRAILING run of non-defun top-level forms is ever moved, and it is run
-- immediately after the body chunk, so hoisting cannot reorder anything. In
-- the S42.0 kernel exactly two files had such a run: types.kl (161 declares)
-- and declarations.kl (one form, (shen.build-lambda-table (external shen)));
-- ShenOSKernel 43.1 has none.
-- Anything less tidy than [defuns...][inits...][declares...] is left inline.
local function hoist_tail(forms)
  local last = #forms
  local function is_defun(f)
    return R.is_cons(f) and R.is_symbol(f[1]) and f[1].name == "defun"
  end
  while last > 0 and not is_defun(forms[last]) do last = last - 1 end
  if last == #forms then return forms, nil, nil end   -- no trailing run
  -- the trailing run is [init forms][declare forms]; find the split
  local d0 = #forms + 1
  while d0 > last + 1 and is_declare_form(forms[d0 - 1]) do d0 = d0 - 1 end
  for i = last + 1, d0 - 1 do
    if is_declare_form(forms[i]) then return forms, nil, nil end  -- interleaved
  end
  local kept, inits, decls = {}, {}, {}
  for i = 1, last do kept[i] = forms[i] end
  for i = last + 1, d0 - 1 do inits[#inits + 1] = forms[i] end
  for i = d0, #forms do
    local tv = eval_typeform(forms[i][2][2][1])
    if tv == nil then return forms, nil, nil end   -- non-literal typeform
    decls[#decls + 1] = { name = forms[i][2][1], typ = tv }
  end
  return kept, (#inits > 0 and inits or nil), (#decls > 0 and decls or nil)
end

local function record_declares(decls)
  local rec = { n = 0, in_chunk = false }
  local saved_rec, saved_nokdata = P.FASL_REC, C.NO_KDATA
  P.FASL_REC = rec
  C.NO_KDATA = true            -- recorded chunks must be relocatable
  local out, done = {}, 0
  local ok, err = pcall(function()
    for _, d in ipairs(decls) do
      local n0 = rec.n
      P.F["declare"](d.name, d.typ)
      done = done + 1
      -- `declare` must have produced exactly one top-level eval-kl chunk (the
      -- prolog abstraction). Anything else and we do not understand what was
      -- just recorded: cache nothing rather than cache a half-truth.
      if rec.n ~= n0 + 1 or rec[rec.n].k ~= "c" then
        error("shen-lua: unexpected declare recording", 0)
      end
      out[#out + 1] = { name = d.name.name, typ = d.typ, dump = rec[rec.n].dump }
    end
  end)
  P.FASL_REC = saved_rec
  C.NO_KDATA = saved_nokdata
  if not ok then
    -- Not a fatal condition: the declares run either way, they just cannot be
    -- cached. Finish the block uncached and record nothing for it.
    if err == "shen-lua: unexpected declare recording" then
      for i = done + 1, #decls do P.F["declare"](decls[i].name, decls[i].typ) end
      return nil
    end
    error(err, 0)
  end
  return out
end

-- fns: optional pre-loaded chunk functions (index-aligned with decl).
-- on_sig: optional (name, typ) hook — the native typechecker's signature
-- table, which a live `declare` would have fed through its wrapper.
local function replay_declares(decl, fns, on_sig)
  local sigf = R.intern("shen.*sigf*")
  local assoc, set = P.F["shen.assoc->"], P.F["set"]
  for i, e in ipairs(decl) do
    local fn = fns and fns[i] or P.load_chunk(e.dump, "declare:" .. e.name)
    local name = R.intern(e.name)
    if on_sig and e.typ ~= nil then on_sig(name, e.typ) end
    set(sigf, assoc(name, fn(), P.GLOBALS["shen.*sigf*"]))
  end
end

-- Load (don't run) every cached dump first, so a corrupt/foreign-arch cache
-- falls back to the full compile before any chunk has executed. Returns true
-- on success, false if any dump refused to load.
local function load_cached(cached, verbose, tag)
  local fns = {}
  for i, ch in ipairs(cached.chunks) do
    local lok, fn = pcall(P.load_chunk, ch.dump, ch.name)
    if not lok then return false end
    fns[i] = fn
  end
  -- Rebuild the compile-time literal pool FIRST: the cached bytecode
  -- reads KDATA[i]. Mutate C.KDATA in place — ENV.KDATA aliases it.
  for i, v in ipairs(cached.kdata) do C.KDATA[i] = v end
  for i, fn in ipairs(fns) do
    -- A hoisted ":init" chunk is a kernel file's trailing top-level block. The
    -- only reason it is a separate chunk is so the native shen.lambda-entry can
    -- be in place before declarations.kl's (shen.build-lambda-table (external
    -- shen)) runs — otherwise that one form runs the whole compiler ~280 times,
    -- once per external symbol, and it is 2/3 of what is left of a cached
    -- kernel load. See prims.install_native_lambda_entry.
    if cached.chunks[i].name:find(":init", 1, true) then
      P.install_native_lambda_entry()
    end
    local rok, err = pcall(fn)
    if not rok then
      error("load error in "..cached.chunks[i].name..tag..": "..tostring(err))
    end
    -- the file's hoisted signature block, in its original position
    local decl = cached.chunks[i].decl
    if decl and #decl > 0 then replay_declares(decl) end
    if verbose then io.stderr:write("  loaded "..cached.chunks[i].name..tag.."\n") end
  end
  -- Fast-forward the counters `declare` would have advanced had the type theory
  -- actually run (see replay_declares), so a cached boot is indistinguishable
  -- from an uncached one at the Shen level.
  if cached.gensym and type(P.GLOBALS["shen.*gensym*"]) == "number"
     and cached.gensym > P.GLOBALS["shen.*gensym*"] then
    P.GLOBALS["shen.*gensym*"] = cached.gensym
  end
  if cached.infs and type(P.GLOBALS["shen.*infs*"]) == "number"
     and cached.infs > P.GLOBALS["shen.*infs*"] then
    P.GLOBALS["shen.*infs*"] = cached.infs
  end
  -- Restore defun arities harvested at compile time (prescan + cdefun);
  -- runtime compilation of user code needs them for direct-call codegen.
  for name, ar in pairs(cached.arity) do C.ARITY[name] = ar end
  return true
end

-- Full compile from .kl sources. extsources (name -> source string), when
-- given, overrides the on-disk KLDIR files (the single-file bundle embeds
-- them as P.KL_SOURCES). path/key, when given, write the bytecode cache.
local function compile_kernel(extsources, path, key, verbose)
  local all = {}
  for _,nm in ipairs(FILES) do
    local s = (extsources and extsources[nm]) or assert(read_file(KLDIR..nm..".kl"), "cannot open "..nm)
    local fs = R.read_all(s)
    all[nm] = fs
    C.prescan(fs)
  end
  local chunks = {}
  for _,nm in ipairs(FILES) do
    -- One concatenated chunk per kernel file: same statements in the same
    -- order as per-form loading (every top-level form compiles to a single
    -- self-contained `do ... end` statement), but loadstring'd once and
    -- dumpable for the bytecode cache.
    -- The file's trailing top-level block is hoisted out (hoist_tail) into a
    -- separate ":init" chunk and, for signatures, into a recorded declare
    -- block; both run right after the body, in file order.
    local forms, inits, decls = hoist_tail(all[nm])
    local function emit(name, fs)
      local parts = {}
      for i,f in ipairs(fs) do parts[i] = C.compile_top(f) end
      local fn = P.load_chunk(table.concat(parts, "\n"), name)
      if name:find(":init", 1, true) then P.install_native_lambda_entry() end
      local ok, err = pcall(fn)
      if not ok then error("load error in "..name..": "..tostring(err)) end
      chunks[#chunks+1] = { name = name, dump = string.dump(fn) }
      return chunks[#chunks]
    end
    emit(nm, forms)
    if inits then emit(nm .. ":init", inits) end
    if decls then chunks[#chunks].decl = record_declares(decls) end
    if verbose then io.stderr:write("  loaded "..nm.."\n") end
  end
  if path then
    write_cache(path, key, chunks, C.ARITY, {
      gensym = type(P.GLOBALS["shen.*gensym*"]) == "number" and P.GLOBALS["shen.*gensym*"] or 0,
      infs   = type(P.GLOBALS["shen.*infs*"]) == "number" and P.GLOBALS["shen.*infs*"] or 0,
    })
  end
end

local function load_kernel(verbose)
  -- Embedded kernel (single-file bundle, see build/make-bundle.lua):
  -- P.KERNEL_CACHE_DATA holds a write_cache-format blob baked in at bundle
  -- build time, P.KL_SOURCES the .kl sources as a name -> string table.
  -- The blob is trusted as-is (no key check — the build pins it); if its
  -- bytecode refuses to load (a different LuaJIT version/arch than the
  -- build machine) we fall back to compiling the embedded sources. The
  -- on-disk cache file is bypassed entirely in this mode.
  if P.KERNEL_CACHE_DATA then
    local cached = parse_cache(P.KERNEL_CACHE_DATA, nil)
    if not (cached and load_cached(cached, verbose, " (embedded)")) then
      compile_kernel(assert(P.KL_SOURCES, "embedded kernel bytecode unusable and no embedded .kl sources"),
                     nil, nil, verbose)
    end
    install_native_overrides()
    return
  end

  local path = cache_path()
  local key, sources
  if path then
    key, sources = kernel_key()
    local cached = read_cache(path, key)
    if cached then
      if load_cached(cached, verbose, " (cached)") then
        install_native_overrides()
        return
      end
      os.remove(path)  -- corrupt/stale dump: recompile below and rewrite
    end
  end
  compile_kernel(sources, path, key, verbose)
  install_native_overrides()
end

-- ---- user-program fasl cache ----------------------------------------------
-- (load "x.shen") is dominated by the reader, macroexpansion, and — with tc
-- on — typechecking, all deterministic given the file content and the
-- session's load history. The persistent effects of a load are exactly
-- (load.kl): the top-level eval-kl chunks (eval-and-print / work-through),
-- the (declare Name Type) calls from shen.assumetypes on the tc path, and
-- the compile-time side state those compiles created (C.ARITY, C.KDATA,
-- shen.*gensym*). We record those during a load and replay them on a key
-- hit, skipping reader+macro+typecheck entirely — SBCL fasl semantics: "it
-- typechecked when compiled".
--
-- Key = codegen key + file content + tc flag + a ROLLING hash of all
-- previously loaded files (editing file A invalidates everything loaded
-- after it, make-style) + the names of live datatypes and macros (catches
-- most REPL-defined state). Replay requires #C.KDATA to equal the recorded
-- base (compiled bytecode hard-codes KDATA indices); a mismatch is a miss,
-- never an error.
--
-- A replayed load reproduces the per-form value/type echo: the bytes
-- shen.eval-and-print / shen.work-through write to (stoutput) are captured as
-- "e" records during the miss and re-pr'd (in stream order, after each form's
-- chunk) on replay, so warm-hit stdout matches cold. Still dropped by design:
-- the cosmetic "run time"/"typechecked in N inferences" banners (they are
-- emitted by `load` OUTSIDE shen.load-help and would add per-run timing noise).
-- Known: (destroy ...) at the REPL between loads is not in the key.
-- SHEN_FASL=off disables; SHEN_FASL_DIR overrides ~/.cache/shen-lua-fasl;
-- SHEN_FASL_DEBUG=1 logs hits/misses to stderr.
local FASL_FORMAT = "SHENFASL7"  -- 7: a nested load run inside a chunk is no
                                 -- longer spliced into the outer fasl (#92);
                                 -- 6: "dv" (shen.*datatypes* by name) replaces
                                 -- "dt" (re-run shen.process-datatype);
                                 -- 5: "lt" (shen.*lambdatable* delta by name);
                                 -- 4: "e" (per-form value/type echo)
                                 -- records; 3: "pc" (shen.compile-prolog)
local FASL_STACK = {}
local FASL_ROLL = 2166136261
local FASL_DEBUG = os.getenv("SHEN_FASL_DEBUG") == "1"
local FASL_INSTALLED = false   -- install_fasl() ran: the recorder is live

local function fasl_dir()
  if not bit then return nil end   -- PUC Lua: no `bit` -> no fasl keys
  local p = os.getenv("SHEN_FASL")
  if p == "off" or p == "0" then return nil end
  local d = os.getenv("SHEN_FASL_DIR")
  if d and d ~= "" then return d end
  local home = os.getenv("HOME")
  if not home or home == "" then return nil end
  return home .. "/.cache/shen-lua-fasl"
end

local function fasl_log(msg)
  if FASL_DEBUG then io.stderr:write("[fasl] " .. msg .. "\n") end
end

-- names of a KL list of symbols or (symbol . x) pairs (datatypes, *macros*)
local function kl_names(l)
  local parts = {}
  while R.is_cons(l) do
    local e = l[1]
    if R.is_symbol(e) then parts[#parts+1] = e.name
    elseif R.is_cons(e) and R.is_symbol(e[1]) then parts[#parts+1] = e[1].name
    else parts[#parts+1] = "?" end
    l = l[2]
  end
  return table.concat(parts, ",")
end

local function fasl_key(content)
  local env = (P.GLOBALS["shen.*tc*"] and "tc" or "raw")
    .. "|" .. (os.getenv("SHEN_PROLOG_ENGINE") or "native")
    .. "|" .. bit.tohex(FASL_ROLL)
    .. "|" .. kl_names(P.GLOBALS["shen.*datatypes*"])
    .. "|" .. kl_names(P.GLOBALS["*macros*"])
  return bit.tohex(fnv1a(content, fnv1a(env, fnv1a((kernel_key())))))
end

-- format: SHENFASL1\n nrec\n
--   { C\n name\n #dump\n dump          top-level eval-kl chunk
--   | D\n <ser name><ser type>          (declare ...) from assumetypes
--   | M\n <ser name>                    shen.record-macro (fn rebuilt by name)
--   | P\n <ser x><ser ptr><ser y>       (put ... *property-vector*)
--   | E\n #bytes\n bytes                per-form value/type echo (stoutput)
--   | A\n <ser names>                   shen.*lambdatable* delta (entries
--                                       rebuilt by name via shen.lambda-entry)
--   | G\n <ser name><ser val> }*        (set ...) outside any chunk
--   narity\n {ar SP name\n}*  kbase\n nkdata\n entries  gensym\n
local function fasl_serialize(rec, arity0)
  if rec.uncacheable then error(rec.uncacheable) end
  local parts = { FASL_FORMAT, "\n", tostring(rec.n), "\n" }
  for i = 1, rec.n do
    local r = rec[i]
    if r.k == "c" then
      parts[#parts+1] = "C\n" .. r.name .. "\n" .. #r.dump .. "\n" .. r.dump
    elseif r.k == "d" then
      parts[#parts+1] = "D\n"
      kdata_ser(r.name, parts)
      kdata_ser(r.typ, parts)
    elseif r.k == "m" then
      parts[#parts+1] = "M\n"
      kdata_ser(r.name, parts)
    elseif r.k == "lf" then
      parts[#parts+1] = "L\n"
      kdata_ser(r.name, parts)
    elseif r.k == "lt" then
      parts[#parts+1] = "A\n"
      kdata_ser(r.names, parts)
    elseif r.k == "dv" then
      parts[#parts+1] = "T\n"
      kdata_ser(r.global, parts)
      kdata_ser(r.names, parts)
    elseif r.k == "pc" then
      parts[#parts+1] = "Q\n"
      kdata_ser(r.name, parts)
      kdata_ser(r.rules, parts)
    elseif r.k == "sy" then
      parts[#parts+1] = "Z\n"
      kdata_ser(r.syns, parts)
    elseif r.k == "p" then
      parts[#parts+1] = "P\n"
      kdata_ser(r.x, parts)
      kdata_ser(r.pointer, parts)
      kdata_ser(r.y, parts)
    elseif r.k == "e" then
      -- raw echo bytes, length-prefixed (may contain newlines / non-ASCII);
      -- mirrors the "C" chunk framing — no trailing separator, the next
      -- record's kind letter starts immediately after the bytes.
      parts[#parts+1] = "E\n" .. #r.bytes .. "\n" .. r.bytes
    else -- "g"
      parts[#parts+1] = "G\n"
      kdata_ser(r.name, parts)
      kdata_ser(r.val, parts)
    end
  end
  local delta = {}
  for name, ar in pairs(C.ARITY) do
    if arity0[name] ~= ar then delta[#delta+1] = ar .. " " .. name end
  end
  parts[#parts+1] = #delta .. "\n"
  for _, d in ipairs(delta) do parts[#parts+1] = d .. "\n" end
  local g = P.GLOBALS["shen.*gensym*"]
  parts[#parts+1] = tostring(type(g) == "number" and g or 0) .. "\n"
  return table.concat(parts)
end

local function atomic_write(path, blob)
  local tmp = path .. ".tmp"
  local fh = io.open(tmp, "wb")
  if not fh then return end       -- read-only dir: silently skip caching
  fh:write(blob); fh:close()
  os.remove(path)
  os.rename(tmp, path)
end

local function fasl_write(path, rec, arity0)
  atomic_write(path, fasl_serialize(rec, arity0))
end

-- Parse a fasl record stream out of `data` starting at `pos`. Returns the
-- record table and the position just past the stream, or nil on any
-- malformation (a miss, never an error). The stdlib boot image (below) embeds
-- one of these after its own header, which is why this takes a position.
local function fasl_parse(data, pos)
  local function line()
    local e = data:find("\n", pos, true)
    if not e then return nil end
    local s = data:sub(pos, e - 1); pos = e + 1
    return s
  end
  if line() ~= FASL_FORMAT then return nil end
  local n = tonumber(line() or ""); if not n then return nil end
  local function de_n(count)
    local ok, vals = pcall(function()
      local out = {}
      for j = 1, count do out[j], pos = kdata_de(data, pos) end
      return out
    end)
    if ok then return vals end
    return nil
  end
  local recs = {}
  for i = 1, n do
    local k = line()
    if k == "C" then
      local nm = line()
      local len = tonumber(line() or "")
      if not nm or not len or pos + len - 1 > #data then return nil end
      recs[i] = { k = "c", name = nm, dump = data:sub(pos, pos + len - 1) }
      pos = pos + len
    elseif k == "D" then
      local v = de_n(2); if not v then return nil end
      recs[i] = { k = "d", name = v[1], typ = v[2] }
    elseif k == "M" then
      local v = de_n(1); if not v then return nil end
      recs[i] = { k = "m", name = v[1] }
    elseif k == "L" then
      local v = de_n(1); if not v then return nil end
      recs[i] = { k = "lf", name = v[1] }
    elseif k == "A" then
      local v = de_n(1); if not v then return nil end
      recs[i] = { k = "lt", names = v[1] }
    elseif k == "T" then
      local v = de_n(2); if not v then return nil end
      recs[i] = { k = "dv", global = v[1], names = v[2] }
    elseif k == "Q" then
      local v = de_n(2); if not v then return nil end
      recs[i] = { k = "pc", name = v[1], rules = v[2] }
    elseif k == "Z" then
      local v = de_n(1); if not v then return nil end
      recs[i] = { k = "sy", syns = v[1] }
    elseif k == "P" then
      local v = de_n(3); if not v then return nil end
      recs[i] = { k = "p", x = v[1], pointer = v[2], y = v[3] }
    elseif k == "E" then
      local len = tonumber(line() or "")
      if not len or pos + len - 1 > #data then return nil end
      recs[i] = { k = "e", bytes = data:sub(pos, pos + len - 1) }
      pos = pos + len
    elseif k == "G" then
      local v = de_n(2); if not v then return nil end
      recs[i] = { k = "g", name = v[1], val = v[2] }
    else return nil end
  end
  local na = tonumber(line() or ""); if not na then return nil end
  local arity = {}
  for i = 1, na do
    local ln = line(); if not ln then return nil end
    local ar, name = ln:match("^(%-?%d+) (.*)$")
    if not ar then return nil end
    arity[name] = tonumber(ar)
  end
  local gensym = tonumber(line() or ""); if not gensym then return nil end
  return { recs = recs, arity = arity, gensym = gensym }, pos
end

local function fasl_read(path)
  local data = read_file(path)
  if not data then return nil end
  return (fasl_parse(data, 1))
end

local function fasl_replay(cached)
  -- Recorded chunks are relocatable (compiled under C.NO_KDATA — literals
  -- ride inside the chunk via MKTREE/MKLIST, never the KDATA side table),
  -- so replay has no positional coupling to this session's compile state.
  for name, ar in pairs(cached.arity) do C.ARITY[name] = ar end
  for _, r in ipairs(cached.recs) do
    if r.k == "c" then
      P.load_chunk(r.dump, r.name)()
    elseif r.k == "d" then
      -- through the live F["declare"] so engine sig-table wrappers see it
      P.F["declare"](r.name, r.typ)
    elseif r.k == "m" then
      -- the macro's defun chunk replayed above; rebuild the (name . fn) pair
      local fn = P.F[r.name.name]
      if not fn then error("fasl: macro function missing: " .. r.name.name) end
      P.F["shen.record-macro"](r.name, fn)
    elseif r.k == "lf" then
      -- shen.lambda-entry returns the complete (name . curried-fn) entry
      -- (or () for arity 0/-1); the recorded put stored its tl. Rebuild and
      -- put the same shape. The arity property it reads was applied by the
      -- preceding "p" record (stream order).
      local entry = P.F["shen.lambda-entry"](r.name)
      local val = R.is_cons(entry) and entry[2] or entry
      P.F["put"](r.name, R.intern("shen.lambda-form"), val,
                 P.GLOBALS["*property-vector*"])
    elseif r.k == "lt" then
      -- 42 kernel path: (set shen.*lambdatable* ...) carries live curried
      -- lambdas, so the recording stored only the NAMES whose entries the set
      -- added/replaced. Rebuild each entry from the live defun exactly the way
      -- shen.update-lambdatable does (shen.lambda-entry reads the arity
      -- property, replayed by the preceding "p" record in stream order).
      local names = r.names
      while R.is_cons(names) do
        local name = names[1]
        local entry = P.F["shen.lambda-entry"](name)
        if R.is_cons(entry) then
          P.F["set"](R.intern("shen.*lambdatable*"),
                     P.F["shen.assoc->"](name, entry[2],
                                         P.GLOBALS["shen.*lambdatable*"]))
        end
        names = names[2]
      end
    elseif r.k == "dv" then
      -- shen.*datatypes* / shen.*alldatatypes*: rebuild (Name . (fn Name))
      -- for each recorded name, in order (see the `set` recorder).
      local fn, out, rev = P.F["fn"], R.NIL, {}
      local t = r.names
      while R.is_cons(t) do rev[#rev + 1] = t[1]; t = t[2] end
      for i = #rev, 1, -1 do out = R.cons(R.cons(rev[i], fn(rev[i])), out) end
      P.F["set"](r.global, out)
    elseif r.k == "sy" then
      P.F["shen.process-synonyms"](r.syns)
    elseif r.k == "pc" then
      -- prolog?/defprolog expand at macroexpansion time through
      -- shen.compile-prolog, whose native-engine clause registration (the NP
      -- table in prolog_compile.lua) is a Lua-side effect no chunk reproduces;
      -- a replayed (shen.lua-run-queryK "name" ...) form would find its query
      -- gone. Re-execute the call: args are pure reader output, and the query
      -- gensym was baked in at record time so the name re-registers exactly.
      P.F["shen.compile-prolog"](r.name, r.rules)
    elseif r.k == "p" then
      P.F["put"](r.x, r.pointer, r.y, P.GLOBALS["*property-vector*"])
    elseif r.k == "e" then
      -- re-emit the per-form echo through the live `pr` so replay-time *hush*
      -- still gates it (a -q hit stays silent, exactly as a -q cold load).
      -- hush-load likewise gates it here: on a replay `load`'s wrapper never
      -- runs (orig_load is skipped entirely), so the depth-based pr gate
      -- can't see these; check the flag directly for the same effect.
      if not P.HUSH_LOAD_ECHO then
        P.F["pr"](r.bytes, P.GLOBALS["*stoutput*"])
      end
    else -- "g"
      P.F["set"](r.name, r.val)
    end
  end
  -- Fast-forward the gensym counter past every name the recording consumed,
  -- so post-replay gensyms can't collide with names baked into the chunks.
  local g = P.GLOBALS["shen.*gensym*"]
  if type(g) == "number" and cached.gensym > g then
    P.GLOBALS["shen.*gensym*"] = cached.gensym
  end
end

-- ---- hush-load quiet mode (issue #46 item 3) -------------------------------
-- --hush-load / SHEN_HUSH_LOAD=1 silences ONLY what `load` itself prints to
-- standard output — the per-form value/type echo (shen.eval-and-print on the
-- raw path, shen.work-through on the tc path) and the "run time: N secs" /
-- "typechecked in N inferences" banners — while user (output ...)/pr from the
-- loaded forms still writes. That makes `bin/shen --hush-load FILE` output
-- deterministic and cross-port comparable (only the program's own output),
-- which -q cannot do: on the 42 kernel *hush* gates pr itself, so -q
-- silences the tests' (output ...) lines too.
--
-- Mechanism: load's own chatter is pr'd to (stoutput) OUTSIDE any eval-kl
-- chunk, i.e. at the same chunk-nesting depth `load` was entered at, whereas
-- everything a loaded form prints runs INSIDE its chunk (depth+1) — the same
-- distinction the fasl "e" capture uses. Wrap `load` to publish that entry
-- depth in P.LOADPR_DEPTH (saved/restored, so nested loads work: a nested
-- load sits inside its caller's chunk, one level deeper); the native pr
-- (prims.lua) drops stdout writes at exactly that depth while the flag is on.
-- P.CHUNK_DEPTH is maintained by prims.compile_and_load only when recording
-- or when the flag is on — default-mode behavior and codepaths are unchanged.
-- Installed BEFORE install_fasl so the fasl wrappers stay outermost: a fasl
-- miss still records the echo "e" bytes (its pr wrapper runs before the
-- native pr drops the write), so a cache written under --hush-load replays
-- the full echo to a later non-hushed run; a fasl hit never reaches this
-- wrapper and is gated at the "e" replay site instead.
local function install_hush_load()
  local orig_load = P.F["load"]
  P.F["load"] = function(fname)
    if not P.HUSH_LOAD_ECHO then return orig_load(fname) end
    local saved = P.LOADPR_DEPTH
    P.LOADPR_DEPTH = P.CHUNK_DEPTH
    local ok, res = pcall(orig_load, fname)
    P.LOADPR_DEPTH = saved
    if not ok then error(res, 0) end
    return res
  end
end

local function install_fasl()
  local dir = fasl_dir()
  if not dir then return end
  os.execute("mkdir -p '" .. dir .. "'")
  local F = P.F

  -- Persistent load effects that happen OUTSIDE an eval-kl chunk are all
  -- funneled through four kernel entry points; wrap each. The in_chunk dance
  -- (flag set while the original runs) makes the wrappers compose: declare's
  -- internal put, record-macro's internal set, etc. are suppressed because
  -- replaying the outer record reproduces them. Effects made by chunk
  -- EXECUTION are never recorded — replaying the chunk reproduces those.
  --
  --   declare           — shen.assumetypes on the tc load path
  --   shen.record-macro — defmacro processing runs at MACROEXPANSION time
  --                       (macros.kl shen.process-def); the macro fn value
  --                       can't serialize, so record the name and rebuild
  --                       the pair from F[name] on replay
  --   put               — expansion/typecheck-time property-vector writes
  --   set               — expansion-time global writes (process-datatype's
  --                       *datatypes*, process-synonyms' *synonyms*, ...)
  local function wrap_recorded(fname, mk)
    local orig = F[fname]
    F[fname] = function(...)
      local rec = FASL_STACK[#FASL_STACK]
      if rec and not rec.in_chunk then
        local r = mk(rec, ...)
        if r then
          rec.n = rec.n + 1
          rec[rec.n] = r
        end
        rec.in_chunk = true
        local ok, res = pcall(orig, ...)
        rec.in_chunk = false
        if not ok then error(res, 0) end
        return res
      end
      return orig(...)
    end
  end

  wrap_recorded("declare", function(rec, name, typ)
    return { k = "d", name = name, typ = typ }
  end)
  wrap_recorded("shen.record-macro", function(rec, name, fn)
    return { k = "m", name = name }
  end)
  -- (synonyms ...) does ALL its work at macroexpansion time (shen.macros
  -- dispatches to it; the expansion result is just the type name). Its state —
  -- *synonyms* assoc entries — can't serialize, but its ARGUMENTS are pure
  -- reader output. Record the call; replay re-executes it. The in_chunk dance
  -- suppresses all its internal chunks/sets/puts.
  --
  -- (datatype ...) is deliberately NOT handled this way: see the
  -- shen.*datatypes* case in the `set` recorder below.
  wrap_recorded("shen.process-synonyms", function(rec, syns)
    return { k = "sy", syns = syns }
  end)
  -- shen.compile-prolog: defprolog AND prolog? expansion both funnel through
  -- it at macroexpansion time. Its kernel output (the query/predicate defun)
  -- is captured as an ordinary chunk, but prolog_compile.lua's native-engine
  -- clause registration (NP table) is a Lua-side effect that exists only at
  -- expansion time — without this record a replayed prolog? query dies with
  -- "native prolog query lost". Clause tokens are pure reader output.
  wrap_recorded("shen.compile-prolog", function(rec, name, rules)
    return { k = "pc", name = name, rules = rules }
  end)
  wrap_recorded("put", function(rec, x, pointer, y, vector)
    if vector ~= P.GLOBALS["*property-vector*"] then
      rec.uncacheable = "put to a non-property vector"
      return nil
    end
    if R.is_symbol(pointer) and pointer.name == "shen.lambda-form" then
      -- the value is a freshly eval'd curried lambda (sys.kl
      -- update-lambda-table) — not serializable; regenerated on replay
      -- from the live defun via shen.lambda-entry. The arity put that
      -- update-lambda-table does first is an ordinary "p" record, already
      -- applied by the time this replays (stream order).
      return { k = "lf", name = x }
    end
    return { k = "p", x = x, pointer = pointer, y = y }
  end)
  wrap_recorded("set", function(rec, name, val)
    local nm = R.is_symbol(name) and name.name or tostring(name)
    -- the gensym counter churns on every expansion-time gensym; the replay
    -- fast-forward (max) covers it without hundreds of noise records
    if nm == "shen.*gensym*" then return nil end
    if nm == "shen.*lambdatable*" then
      -- 42 kernel: shen.update-lambdatable / update-lambda-table set the
      -- whole assoc list, whose entries are (name . live-curried-lambda) —
      -- unserializable, and the reason every stdlib/user load used to be
      -- fasl-uncacheable. The recorder runs BEFORE the original set, so the
      -- global still holds the old table: diff it against `val` and record
      -- only the names whose entry was added/replaced (an "lt" record; the
      -- fns are rebuilt on replay via shen.lambda-entry).
      local old = {}
      local t = P.GLOBALS["shen.*lambdatable*"]
      while R.is_cons(t) do
        local e = t[1]
        if R.is_cons(e) and R.is_symbol(e[1]) then old[e[1].name] = e[2] end
        t = t[2]
      end
      local names = R.NIL
      t = val
      while R.is_cons(t) do
        local e = t[1]
        if R.is_cons(e) and R.is_symbol(e[1]) and old[e[1].name] ~= e[2] then
          names = R.cons(e[1], names)
        end
        t = t[2]
      end
      return { k = "lt", names = names }
    end
    if nm == "shen.*datatypes*" or nm == "shen.*alldatatypes*" then
      -- (datatype ...) is processed at MACROEXPANSION time and the only state
      -- it leaves outside its own eval-kl chunks is these two assoc lists,
      -- whose entries are all (TypeName . (fn TypeName)) — sequent.kl
      -- shen.remember-datatype — with preclude/include shuffling whole entries
      -- between the two lists. The fn leaf can't serialize, but it is exactly
      -- reconstructible from the type name, so record the NAMES in order and
      -- rebuild the list on replay. Recording the whole list rather than a
      -- delta is what makes (preclude-all-but ...) — which SHRINKS
      -- shen.*datatypes* — replay correctly.
      --
      -- This is what lets a datatype be replayed from its recorded chunks
      -- instead of re-run: shen.process-datatype pushes the rules through the
      -- shen.<datatype> yacc parser and the type theory, ~17 ms per datatype on
      -- arm64 LuaJIT and the largest single item in a warm standard-library
      -- load, and none of it is needed to reach the same state.
      local rev = {}
      local t = val
      while R.is_cons(t) do
        local e = t[1]
        if not (R.is_cons(e) and R.is_symbol(e[1])) then
          rec.uncacheable = "unrecognised " .. nm .. " entry"
          return nil
        end
        rev[#rev + 1] = e[1]
        t = t[2]
      end
      local names = R.NIL
      for i = #rev, 1, -1 do names = R.cons(rev[i], names) end
      return { k = "dv", global = name, names = names }
    end
    return { k = "g", name = name, val = val }
  end)

  -- Per-form value/type echo (klambda/load.kl: shen.eval-and-print on the raw
  -- path, shen.work-through on the tc path) is a pr to (stoutput) that happens
  -- AFTER the form's eval-kl chunk has run and returned — i.e. with in_chunk
  -- false, unlike the form's own (output ...) side-effects (in_chunk true, and
  -- reproduced by re-running the chunk). Record those echo bytes as "e" records
  -- in stream order so replay re-emits them right after each chunk. We scope
  -- capture to the span of shen.load-help via rec.echoing, which excludes the
  -- "run time"/"typechecked in N inferences" banners that `load` prints outside
  -- load-help (those stay dropped by design). The bytes are captured even under
  -- *hush* (the string is computed regardless; pr just doesn't write it), so a
  -- miss recorded under -q still replays correctly to a non-hushed hit.
  do
    local orig_load_help = F["shen.load-help"]
    F["shen.load-help"] = function(...)
      local rec = P.FASL_REC
      if not rec then return orig_load_help(...) end
      local saved = rec.echoing
      rec.echoing = true
      local ok, res = pcall(orig_load_help, ...)
      rec.echoing = saved
      if not ok then error(res, 0) end
      return res
    end
    local orig_pr = F["pr"]
    F["pr"] = function(s, st)
      local rec = P.FASL_REC
      if rec and rec.echoing and not rec.in_chunk
         and type(s) == "string" and st == P.GLOBALS["*stoutput*"] then
        rec.n = rec.n + 1
        rec[rec.n] = { k = "e", bytes = s }
      end
      return orig_pr(s, st)
    end
  end

  -- A nested load's records go to the INNER recording, so an enclosing one
  -- (the standard-library boot image, below) would otherwise see a hole where
  -- the nested load was. Copy the inner stream into the enclosing one when the
  -- inner load finishes — from its fasl file on a hit, from the live recording
  -- on a miss — so the outer stream stays a complete description of everything
  -- the outer span did, in order. An inner recording that could not be
  -- serialized poisons the outer one for the same reason.
  --
  -- Only a load made OUTSIDE the outer span's chunks has a hole to fill (one
  -- run at macroexpansion time, say). A load made while one of the outer
  -- chunks executes — a top-level (load ...) form, or a function that loads —
  -- is already in the outer stream as that chunk, and replaying the chunk
  -- runs the load again; splicing it as well replays it twice (issue #92).
  local function splice_into_outer(recs, uncacheable, deps)
    local outer = FASL_STACK[#FASL_STACK]
    if not outer then return end
    if outer.deps and deps then
      for _, d in ipairs(deps) do outer.deps[#outer.deps + 1] = d end
    end
    if outer.in_chunk then return end
    for _, r in ipairs(recs) do
      outer.n = outer.n + 1
      outer[outer.n] = r
    end
    if uncacheable then outer.uncacheable = outer.uncacheable or uncacheable end
  end

  local orig_load = F["load"]
  F["load"] = function(fname)
    if type(fname) ~= "string" then return orig_load(fname) end
    -- the file the kernel's load would read: open resolves *home-directory*
    local fpath = P.home_path(fname)
    local fh = io.open(fpath, "rb")
    if not fh then return orig_load(fname) end   -- let the kernel error
    local content = fh:read("*a"); fh:close()
    -- Every file a recorded span loads is an input to that span's cache key;
    -- the boot image stores them so it can verify them all on the next boot.
    local outer = FASL_STACK[#FASL_STACK]
    if outer and outer.deps then
      outer.deps[#outer.deps + 1] = { path = fpath, content = content }
    end
    local key = fasl_key(content)
    local path = dir .. "/" .. key .. ".fasl"
    local cached = fasl_read(path)
    if cached then
      -- A replay drives the very functions the recorder wraps (put,
      -- record-macro, set, ...), so an ENCLOSING recording would capture them a
      -- second time — and capture them wrong, since a replay reinstates chunks
      -- through load_chunk rather than compile_and_load and so contributes no
      -- "c" records. Mark the enclosing span as in-chunk for the duration: its
      -- copy of this load is the spliced record stream below, not what the
      -- replay happens to call.
      local outer0 = FASL_STACK[#FASL_STACK]
      local saved_in_chunk = outer0 and outer0.in_chunk
      if outer0 then outer0.in_chunk = true end
      local ok, err = pcall(fasl_replay, cached)
      if outer0 then outer0.in_chunk = saved_in_chunk end
      if ok then
        fasl_log("hit  " .. fname .. " " .. key)
        FASL_ROLL = fnv1a(content, FASL_ROLL)
        splice_into_outer(cached.recs, nil, nil)
        return R.intern("loaded")
      end
      os.remove(path)   -- stale beyond what the key caught: recompile next run
      error(err, 0)
    end
    fasl_log("miss " .. fname .. " " .. key)
    local rec = { n = 0, in_chunk = false }
    local arity0 = {}
    for k, v in pairs(C.ARITY) do arity0[k] = v end
    FASL_STACK[#FASL_STACK + 1] = rec
    P.FASL_REC = rec
    C.NO_KDATA = true   -- recorded chunks must be relocatable
    local ok, res = pcall(orig_load, fname)
    FASL_STACK[#FASL_STACK] = nil
    P.FASL_REC = FASL_STACK[#FASL_STACK]
    C.NO_KDATA = P.FASL_REC ~= nil
    if not ok then error(res, 0) end
    local wok, werr = pcall(fasl_write, path, rec, arity0)
    if not wok then fasl_log("uncacheable " .. fname .. ": " .. tostring(werr)) end
    FASL_ROLL = fnv1a(content, FASL_ROLL)
    splice_into_outer(rec, (not wok) and tostring(werr) or nil, rec.deps)
    return res
  end

  FASL_INSTALLED = true
end

-- ---- standard library (precompiled klambda/stlib.kl) ----------------------
-- ShenOSKernel 43.1 ships the standard library precompiled as klambda/stlib.kl
-- (generated by upstream make-stlib.shen from lib/stlib). Its defuns load with
-- the kernel (FILES); port-upgrades.md (41.1) then requires a call to
-- (stlib.initialise) after the kernel is initialised, which installs the
-- library's package/external symbols, arities + lambda forms, macros,
-- datatypes, type signatures and source records, and finishes with
-- (preclude-all-but []) (set shen.*userdefs* []) (cd "") (tc -). This is
-- exactly what shen-scheme 0.50 does. SHEN_NO_STDLIB=1 skips the call (the
-- defuns stay loaded but are not registered as Shen functions).
local STDLIB_LOADED = false

-- (stlib.initialise-types) is 290 (declare Name Type) calls, and `declare`
-- runs the type theory for real on each (variancy under the Prolog machine,
-- then a KL->Lua compile of the signature's prolog abstraction): ~0.2 s, the
-- bulk of a warm boot. Everything it depends on is fixed by the kernel cache
-- key, so cache it exactly the way the S42 kernel's own types.kl signatures
-- were cached (hoist_tail / record_declares / replay_declares): a sidecar file
-- next to the kernel bytecode cache holds, per signature, its raw type and the
-- dumped abstraction chunk, plus the gensym / inference counters. A warm boot
-- replays them into shen.*sigf* (and the native typechecker's signature
-- table) in the original order. SHEN_KERNEL_CACHE=off disables it with the
-- kernel cache.
local SIGS_FORMAT = "SHENSIG1"
local function stlib_sigs_path()
  if P.KERNEL_CACHE_DATA then return nil end   -- single-file bundle
  local p = cache_path()
  return p and (p .. ".stlib") or nil
end

local function write_sigs(path, key, decl, counters)
  local parts = { SIGS_FORMAT, "\n", key, "\n", tostring(#decl), "\n" }
  for _, e in ipairs(decl) do
    parts[#parts+1] = e.name .. "\n" .. #e.dump .. "\n" .. e.dump
    kdata_ser(e.typ, parts)
  end
  parts[#parts+1] = tostring(counters.gensym) .. "\n" .. tostring(counters.infs) .. "\n"
  local tmp = path .. ".tmp"
  local fh = open_for_write(tmp)
  if not fh then return end
  fh:write(table.concat(parts)); fh:close()
  os.remove(path)
  os.rename(tmp, path)
end

local function read_sigs(path, key)
  local data = read_file(path)
  if not data then return nil end
  local pos = 1
  local function line()
    local e = data:find("\n", pos, true)
    if not e then return nil end
    local s = data:sub(pos, e - 1); pos = e + 1
    return s
  end
  if line() ~= SIGS_FORMAT or line() ~= key then return nil end
  local n = tonumber(line() or ""); if not n then return nil end
  local decl = {}
  local ok = pcall(function()
    for i = 1, n do
      local nm = line()
      local len = tonumber(line() or "")
      if not nm or not len or pos + len - 1 > #data then error("truncated") end
      local dump = data:sub(pos, pos + len - 1)
      pos = pos + len
      local typ
      typ, pos = kdata_de(data, pos)
      decl[i] = { name = nm, typ = typ, dump = dump }
    end
  end)
  if not ok then return nil end
  local gensym = tonumber(line() or ""); if not gensym then return nil end
  local infs = tonumber(line() or ""); if not infs then return nil end
  return { decl = decl, gensym = gensym, infs = infs }
end

local function stlib_types(verbose)
  local init_types = P.F["stlib.initialise-types"]
  local path = stlib_sigs_path()
  if not path then return init_types() end
  local key = kernel_key() .. "|" .. SIGS_FORMAT
  local tn = package.loaded["typecheck_native"]
  local on_sig = tn and tn.record_sig or nil
  local cached = read_sigs(path, key)
  if cached then
    -- load every dump before running any, so a foreign-arch file falls back
    -- to the real declares with nothing half-applied
    local fns, lok = {}, true
    for i, e in ipairs(cached.decl) do
      local ok, fn = pcall(P.load_chunk, e.dump, "declare:" .. e.name)
      if not ok then lok = false; break end
      fns[i] = fn
    end
    if lok then
      replay_declares(cached.decl, fns, on_sig)
      if type(P.GLOBALS["shen.*gensym*"]) == "number"
         and cached.gensym > P.GLOBALS["shen.*gensym*"] then
        P.GLOBALS["shen.*gensym*"] = cached.gensym
      end
      if type(P.GLOBALS["shen.*infs*"]) == "number"
         and cached.infs > P.GLOBALS["shen.*infs*"] then
        P.GLOBALS["shen.*infs*"] = cached.infs
      end
      if verbose then io.stderr:write("  stdlib signatures (cached)\n") end
      return
    end
    os.remove(path)
  end
  -- Cold: collect the (name, type) pairs the compiled initialiser declares,
  -- then run the real declares through record_declares in the same order.
  local live = P.F["declare"]
  local decls = {}
  P.F["declare"] = function(name, typ)
    decls[#decls + 1] = { name = name, typ = typ }
    return name
  end
  local ok, err = pcall(init_types)
  P.F["declare"] = live
  if not ok then error(err, 0) end
  local out = record_declares(decls)
  if out then
    write_sigs(path, key, out, {
      gensym = type(P.GLOBALS["shen.*gensym*"]) == "number" and P.GLOBALS["shen.*gensym*"] or 0,
      infs   = type(P.GLOBALS["shen.*infs*"]) == "number" and P.GLOBALS["shen.*infs*"] or 0,
    })
  end
end

local STDLIB_PHASES = { "environment", "arities", "macros", "synonyms",
                        "datatypes", "types", "sources", "final" }

local function load_stdlib(verbose)
  if STDLIB_LOADED then return end
  if os.getenv("SHEN_NO_STDLIB") == "1" then return end
  if not P.F["stlib.initialise"] then
    if verbose then io.stderr:write("  stdlib: stlib.initialise not defined; skipping\n") end
    return
  end
  local hush0 = P.GLOBALS["*hush*"]
  local ok, err = pcall(function()
    -- (stlib.initialise) is exactly these phases in this order; run them
    -- individually so the types phase can be served from the cache.
    local whole = true
    for _, ph in ipairs(STDLIB_PHASES) do
      if not P.F["stlib.initialise-" .. ph] then whole = false end
    end
    if not whole then return P.F["stlib.initialise"]() end
    -- The phases are straight-line, run-once code: thousands of calls with
    -- literal source trees (sources) and datatype rule compiles (datatypes).
    -- Under the JIT they are bimodal — 10 ms or 700 ms for the sources phase
    -- on the same host, depending on which side traces get recorded — and
    -- never faster than the interpreter. Run them interpreted; the JIT is
    -- restored (only if it was on) before anything else runs.
    local jit_was_on = jit and jit.status and jit.status()
    if jit_was_on then jit.off() end
    local pok, perr = pcall(function()
      for _, ph in ipairs(STDLIB_PHASES) do
        if ph == "types" then stlib_types(verbose)
        else P.F["stlib.initialise-" .. ph]() end
      end
    end)
    if jit_was_on then jit.on() end
    if not pok then error(perr, 0) end
  end)
  P.GLOBALS["*hush*"] = hush0
  if not ok then
    error("stdlib initialise failed: " .. tostring(P.F["error-to-string"](err)), 0)
  end
  STDLIB_LOADED = true
  if verbose then io.stderr:write("  initialised stdlib\n") end
end
P.load_stdlib = load_stdlib

local function initialise()
  -- ShenOSKernel 43.1: the kernel files are pure defuns; (shen.initialise)
  -- (init.kl) builds the environment, the *property-vector* dict, the arity
  -- table, the lambda forms and the kernel signatures. It must run exactly
  -- once, after load_kernel(). A kernel without it (e.g. an older Tarver S42
  -- tree via SHEN_KL_DIR, which self-initialises at load) is tolerated.
  local r
  local fn = P.F["shen.initialise"]
  if fn then r = fn() end
  -- Feature registry for the features extension (shen.x.features.*). The
  -- 43.1 extension's initialise declares its own types and registers the
  -- cond-expand macro; seed it with the features detected from native
  -- backends (the Shen Batteries module loader queries
  -- (shen.x.features.current) before loading a module). SHEN_X_SHA256=pure
  -- (and hosts without native backends) advertise no host capability.
  if P.F["shen.x.features.initialise"] then
    local detected = {}
    local function add_backend(global, feature)
      local backend = P.GLOBALS[global]
      if backend == R.intern("host") then
        detected[#detected + 1] = R.intern(feature)
      end
    end
    add_backend("shen.x.*sha256-backend*", "shen.x/sha256-host")
    add_backend("shen.x.*zmq-backend*", "shen.x/zmq-host")
    P.F["shen.x.features.initialise"](R.from_table(detected))
  end
  -- Programmable pattern matching: installs the custom pattern compiler /
  -- reducer hooks with an empty handler registry and declares the
  -- register/unregister API, as shen-scheme 0.50 does at boot.
  if P.F["shen.x.programmable-pattern-matching.initialise"] then
    P.F["shen.x.programmable-pattern-matching.initialise"]()
  end
  -- Register the lua.* interop entries in Shen's own arity/lambda-form
  -- tables (needs the *property-vector* the kernel just created).
  require("lua_interop").post_initialise()
  -- Standard library: the precompiled klambda/stlib.kl was loaded with the
  -- kernel; (stlib.initialise) installs its package info, arities, lambda
  -- forms, macros, datatypes and types. SHEN_NO_STDLIB=1 skips it.
  load_stdlib()
  return r
end

P.load_kernel = function(verbose)
  load_kernel(verbose)
  install_hush_load()   -- before install_fasl: fasl wrappers stay outermost
  install_fasl()   -- after native overrides so the declare wrapper composes
  -- Lua<->Shen interop surface (lua_interop.lua). Installed LAST so the
  -- typed bridge (lua.function) sees the fully composed F["declare"]
  -- (native-engine signature recording + fasl recording).
  require("lua_interop").install(P)
end
P.initialise = initialise

-- run a KL toplevel string (one or more forms) through eval
function P.run_kl_string(src)
  local forms = R.read_all(src)
  local last
  for _,f in ipairs(forms) do last = P.eval(f) end
  return last
end

return P