test(zero-swift): fuzz the client with chaos against zero-cache

.github/workflows/swift-macos.yml

@@ -40,8 +40,9 @@ jobs:
         with:
           persist-credentials: false
       - run: swift --version
+      # Not E2ETests, which the end-to-end job runs: see swift.yml.
       - name: Build and test (macOS)
-        run: swift test
+        run: swift test --skip E2ETests
       # On its own: see swift.yml.
       - name: ZeroVirtual fuzz
         run: ZERO_FUZZ_SEEDS=200 swift test --skip-build --filter FuzzTests

.github/workflows/swift-nightly.yml

@@ -108,3 +108,64 @@ jobs:
         run: |
           echo "Seeds $ZERO_FUZZ_FIRST_SEED to $((ZERO_FUZZ_FIRST_SEED + ZERO_FUZZ_SEEDS - 1))"
           swift test --skip-build --filter FuzzTests
+
+  # Random cuts, stalls, outages, Postgres writes and relaunches against
+  # zero-cache, two clients at a time (ChaosFuzzTests).
+  chaos-fuzz:
+    name: Chaos fuzz (Linux)
+    runs-on: ubuntu-24.04
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+      - uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9
+        with:
+          version: 11.11.0
+      # zero-cache runs from source, which needs Node 24 (`using`).
+      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: 24.x
+          package-manager-cache: false
+      - run: pnpm install --frozen-lockfile
+      - name: Start Postgres
+        run: |
+          sudo systemctl start postgresql.service
+          sudo -u postgres psql -q -c "ALTER SYSTEM SET wal_level = logical" \
+            -c "ALTER USER postgres PASSWORD 'postgres'"
+          sudo systemctl restart postgresql.service
+          until pg_isready -q -h 127.0.0.1; do sleep 1; done
+      - name: Install Swift
+        run: |
+          curl -sSfL -o "$RUNNER_TEMP/swift.tar.gz" \
+            https://download.swift.org/swift-6.2.4-release/ubuntu2404/swift-6.2.4-RELEASE/swift-6.2.4-RELEASE-ubuntu24.04.tar.gz
+          mkdir -p "$RUNNER_TEMP/swift"
+          tar -xzf "$RUNNER_TEMP/swift.tar.gz" -C "$RUNNER_TEMP/swift" --strip-components=1
+          echo "$RUNNER_TEMP/swift/usr/bin" >> "$GITHUB_PATH"
+      - name: Install system libraries
+        run: sudo apt-get update && sudo apt-get install -y --no-install-recommends libsqlite3-dev
+      - name: Build
+        working-directory: swift
+        run: swift build --build-tests
+      - name: Start zero-cache
+        env:
+          ZERO_E2E_PG: postgresql://postgres:postgres@127.0.0.1:5432/postgres
+        run: |
+          node packages/zero-conformance/src/e2e/server.ts > "$RUNNER_TEMP/server.log" 2>&1 &
+          for _ in $(seq 1 120); do
+            grep -q '^ready ' "$RUNNER_TEMP/server.log" && exit 0
+            sleep 1
+          done
+          cat "$RUNNER_TEMP/server.log"
+          exit 1
+      # About 10 s a seed.
+      - name: Fuzz
+        working-directory: swift
+        env:
+          ZERO_E2E_URL: ws://127.0.0.1:4848
+          ZERO_CHAOS_SEEDS: '150'
+          ZERO_CHAOS_FIRST_SEED: ${{ inputs.first-seed || format('{0}000', github.run_number) }}
+        run: swift test --skip-build --filter ChaosFuzzTests
+      - name: Server logs
+        if: failure()
+        run: cat "$RUNNER_TEMP/server.log"

.github/workflows/swift.yml

@@ -95,8 +95,12 @@ jobs:
         run: swift format lint --strict --recursive Sources Tests Package.swift
       - name: Build
         run: swift build --build-tests
+      # Not E2ETests: the end-to-end job runs them, apart from the
+      # conformance suites, whose long synchronous loops hold every thread
+      # of the cooperative pool for tens of seconds on a 4-core runner and
+      # starve their timing-sensitive network tests.
       - name: Test
-        run: swift test --skip-build
+        run: swift test --skip-build --skip E2ETests
       # On its own: it holds the main actor long enough to starve the
       # timing-sensitive suites if it ran beside them.
       - name: ZeroVirtual fuzz

swift/DESIGN.md

@@ -266,6 +266,7 @@ Vector storage and fuzzing:
 - Seeded random query and push fuzz cases are generated in CI from fixed seeds (not checked in) and run against Swift.
 - `swift-nightly.yml` generates the seeded corpora again each night on new seeds, from the run number, at 5 to 10 times the PR checks' cases, and runs the Swift suites on them. A mismatch names its case and the seeds the job printed; replaying it is setting the same environment variables. A live differential harness (JS drives a Swift CLI over stdio and shrinks failures) is deferred to v1.x.
 - ZeroVirtual has no JS oracle for scrolling, so `FuzzTests` checks invariants instead: after each random step settles, a row is on screen, the loaded rows are a run of the list, and rows on screen hold still through changes above them. PRs run 200 seeds, in a CI step of their own; `swift-nightly.yml` runs 10,000 new ones each night.
+- `ChaosFuzzTests` takes two clients through 40 random steps against zero-cache behind a `ChaosProxy`: mutations, cuts, outages, stalls, cuts partway through a message, Postgres writes behind their back, and relaunches. Then the network heals and the oracle is convergence: Postgres holds each mutation once, each client's last mutation ID counts its mutations, and both clients see what Postgres holds. The E2E jobs run 2 seeds; the nightly runs 150 new ones.
 
 ### JS bugs found, not yet reported upstream
 

swift/Tests/E2ETests/ChaosFuzzTests.swift

@@ -0,0 +1,247 @@
+import Foundation
+import Testing
+import Zero
+
+/// Random chaos against the real zero-cache. Each seed runs two clients, on
+/// stores of their own, through a `ChaosProxy`, and takes 40 random steps:
+/// mutations, connections cut, refused, stalled, or cut partway through
+/// what the server sends, writes to Postgres behind the clients' back, and
+/// relaunches on the same store. Then the network heals, and the oracle is
+/// convergence: Postgres holds every mutation once, each client's last
+/// mutation ID counts all of its mutations, and both clients see what
+/// Postgres holds.
+///
+/// PRs run `ZERO_CHAOS_SEEDS` seeds (2 by default) from
+/// `ZERO_CHAOS_FIRST_SEED` (0); the nightly workflow runs more. To replay
+/// one: `ZERO_CHAOS_SEED=<seed> ZERO_E2E_URL=... swift test --filter ChaosFuzzTests`.
+@Suite(.enabled(if: E2E.url != nil), .serialized, .timeLimit(.minutes(60)))
+@MainActor
+struct ChaosFuzzTests {
+  @Test func randomSeeds() async throws {
+    let env = ProcessInfo.processInfo.environment
+    let seeds: Range<UInt64>
+    if let only = env["ZERO_CHAOS_SEED"].flatMap(UInt64.init) {
+      seeds = only..<only + 1
+    } else {
+      let first = env["ZERO_CHAOS_FIRST_SEED"].flatMap(UInt64.init) ?? 0
+      seeds = first..<first + (env["ZERO_CHAOS_SEEDS"].flatMap(UInt64.init) ?? 2)
+    }
+    var failed = 0
+    for seed in seeds {
+      let start = ContinuousClock.now
+      let failure = try await ChaosRun(seed: seed).run()
+      print(
+        "[chaos fuzz] seed \(seed): \(failure ?? "converged") in \(ContinuousClock.now - start)")
+      guard let failure else { continue }
+      Issue.record("seed \(seed): \(failure)")
+      failed += 1
+      if failed == 3 { return }
+    }
+  }
+}
+
+/// SplitMix64: the same steps for a seed on every platform.
+struct ChaosRandom: RandomNumberGenerator {
+  var state: UInt64
+  mutating func next() -> UInt64 {
+    state &+= 0x9E37_79B9_7F4A_7C15
+    var z = state
+    z = (z ^ (z >> 30)) &* 0xBF58_476D_1CE4_E5B9
+    z = (z ^ (z >> 27)) &* 0x94D0_49BB_1331_11EB
+    return z ^ (z >> 31)
+  }
+}
+
+/// One seed's clients, proxy, and model of what Postgres should end up with.
+@MainActor
+final class ChaosRun {
+  /// A client and the relaunches on its store.
+  @MainActor final class Lineage {
+    let storageKey: String
+    var zero: Zero<Void>
+    var open: LiveQuery<[TestIssue]>
+    /// What the client now running created and hasn't closed: only it may
+    /// close them, since its mutations apply in order.
+    var closable: [String] = []
+
+    init(zero: Zero<Void>, storageKey: String) throws {
+      self.zero = zero
+      self.storageKey = storageKey
+      open = try zero.liveQuery(E2EApp.open())
+    }
+  }
+
+  let seed: UInt64
+  var rng: ChaosRandom
+  let chaos = Chaos()
+  var proxy: ChaosProxy!
+  var lineages: [Lineage] = []
+  /// The open issues Postgres should end with.
+  var expectedOpen: Set<String> = []
+  var created = 0
+  /// Mutations made by each client ID.
+  var mutations: [String: Int] = [:]
+  /// Issues written straight to Postgres and still open.
+  var written: [String] = []
+  var steps: [String] = []
+
+  init(seed: UInt64) {
+    self.seed = seed
+    rng = ChaosRandom(state: seed)
+  }
+
+  func makeZero(_ storageKey: String) throws -> Zero<Void> {
+    try chaos.makeZero(cacheURL: proxy.cacheURL, storageKey: storageKey) {
+      $0.pingTimeoutMs = 2000
+    }
+  }
+
+  /// The problems at the end; nil when everything converged.
+  func run() async throws -> String? {
+    defer { chaos.cleanUp() }
+    proxy = try await ChaosProxy(upstream: E2E.url!)
+    for _ in 0..<2 {
+      let key = UUID().uuidString
+      lineages.append(try Lineage(zero: try makeZero(key), storageKey: key))
+    }
+    defer {
+      for l in lineages { l.zero.close() }
+    }
+    for l in lineages { try await chaos.connected(l.zero) }
+    for n in 0..<40 {
+      steps.append("\(n): " + (try await step(n)))
+      try await Task.sleep(for: .milliseconds(Int.random(in: 0...100, using: &rng)))
+    }
+    await proxy.set {
+      $0 = ChaosProxy.Knobs()
+    }
+    let problems = try await converge()
+    await proxy.shutdown()
+    guard !problems.isEmpty else { return nil }
+    return problems.joined(separator: "; ")
+      + " (last steps: \(steps.suffix(8).joined(separator: " | ")))"
+  }
+
+  func mutate(_ l: Lineage, _ request: MutateRequest<Void>) {
+    _ = l.zero.mutate(request)
+    mutations[l.zero.clientID, default: 0] += 1
+  }
+
+  func step(_ n: Int) async throws -> String {
+    let i = Int.random(in: 0..<lineages.count, using: &rng)
+    let l = lineages[i]
+    switch Int.random(in: 0..<100, using: &rng) {
+    case 0..<35:
+      let id = "\(chaos.prefix)c\(n)"
+      mutate(l, E2EApp.create(.init(id: id, title: "Chaos \(n)", ownerId: nil)))
+      l.closable.append(id)
+      expectedOpen.insert(id)
+      created += 1
+      return "client \(i) creates \(id)"
+    case 35..<45:
+      guard !l.closable.isEmpty else { return "client \(i) has nothing to close" }
+      let id = l.closable.remove(at: Int.random(in: 0..<l.closable.count, using: &rng))
+      mutate(l, E2EApp.close(id))
+      expectedOpen.remove(id)
+      return "client \(i) closes \(id)"
+    case 45..<53:
+      await proxy.cutAll()
+      return "cut every connection"
+    case 53..<58:
+      let ms = Int.random(in: 100...1500, using: &rng)
+      await proxy.set { $0.accept = .refuse }
+      await proxy.cutAll()
+      try await Task.sleep(for: .milliseconds(ms))
+      await proxy.set { $0.accept = .pass }
+      return "down for \(ms) ms"
+    case 58..<63:
+      let ms = Int.random(in: 200...3000, using: &rng)
+      await proxy.set { $0.downstreamRate = 0 }
+      try await Task.sleep(for: .milliseconds(ms))
+      await proxy.set { $0.downstreamRate = nil }
+      return "server stalled for \(ms) ms"
+    case 63..<68:
+      let bytes = Int.random(in: 500...50_000, using: &rng)
+      let ms = Int.random(in: 200...1500, using: &rng)
+      await proxy.set { $0.cutAfterDownstreamBytes = bytes }
+      try await Task.sleep(for: .milliseconds(ms))
+      await proxy.set { $0.cutAfterDownstreamBytes = nil }
+      return "cuts after \(bytes) bytes for \(ms) ms"
+    case 68..<76:
+      let id = "\(chaos.prefix)pg\(n)"
+      try Postgres.query(
+        """
+        INSERT INTO issues (id, title, description, closed, owner_id, "createdAt")
+        VALUES ('\(id)', 'Written \(n)', '', false, NULL, 2000)
+        """)
+      written.append(id)
+      expectedOpen.insert(id)
+      created += 1
+      return "Postgres inserts \(id)"
+    case 76..<82:
+      guard !written.isEmpty else { return "Postgres has nothing to close" }
+      let id = written.remove(at: Int.random(in: 0..<written.count, using: &rng))
+      try Postgres.query("UPDATE issues SET closed = true WHERE id = '\(id)'")
+      expectedOpen.remove(id)
+      return "Postgres closes \(id)"
+    case 82..<90:
+      l.zero.close()
+      l.zero = try makeZero(l.storageKey)
+      l.open = try l.zero.liveQuery(E2EApp.open())
+      l.closable = []
+      return "client \(i) relaunches"
+    default:
+      let ms = Int.random(in: 0...500, using: &rng)
+      try await Task.sleep(for: .milliseconds(ms))
+      return "wait \(ms) ms"
+    }
+  }
+
+  func ids(_ l: Lineage) -> Set<String> {
+    Set(l.open.data.map(\.id).filter { $0.hasPrefix(chaos.prefix) })
+  }
+
+  func short(_ got: Set<String>, _ want: Set<String>) -> String {
+    let missing = want.subtracting(got).sorted()
+    let extra = got.subtracting(want).sorted()
+    return "missing \(missing.prefix(5)), extra \(extra.prefix(5))"
+  }
+
+  /// Waits for the clients and Postgres to settle, then compares them with
+  /// the model.
+  func converge() async throws -> [String] {
+    var problems: [String] = []
+    // Every mutation pushed first: until then, a create closed in the same
+    // run leaves the open issues right but is missing from Postgres.
+    for (clientID, count) in mutations.sorted(by: { $0.key < $1.key }) {
+      let ok = try await within(.seconds(30)) { try Postgres.lastMutationID(clientID) == count }
+      if !ok {
+        let last = try Postgres.lastMutationID(clientID).map(String.init) ?? "none"
+        problems.append(
+          "client \(clientID) made \(count) mutations; its last mutation ID is \(last)")
+      }
+    }
+    let inPostgres = try await within(.seconds(30)) {
+      try Postgres.openIssueIDs(self.chaos.prefix) == self.expectedOpen
+        && Postgres.issues(self.chaos.prefix).count == self.created
+    }
+    if !inPostgres {
+      let open = try Postgres.openIssueIDs(chaos.prefix)
+      if open != expectedOpen {
+        problems.append("Postgres's open issues: \(short(open, expectedOpen))")
+      }
+      let all = try Postgres.issues(chaos.prefix).count
+      if all != created { problems.append("\(all) issues in Postgres, \(created) created") }
+    }
+    for (i, l) in lineages.enumerated() {
+      let ok = try await within(.seconds(60)) {
+        l.open.resultType == .complete && self.ids(l) == self.expectedOpen
+      }
+      if !ok {
+        problems.append(
+          "client \(i) (\(l.zero.connection.state.status)) sees \(short(ids(l), expectedOpen))")
+      }
+    }
+    return problems
+  }
+}

swift/Tests/E2ETests/WalkingSkeletonTests.swift

@@ -133,8 +133,11 @@ struct WalkingSkeletonTests {
     for (table, rows) in rows.sorted(by: { $0.key < $1.key }) {
       for row in rows { print("[e2e] \(table) \(JSONSerializer.stringify(.object(row)))") }
     }
-    // The fixture's open issues, their comments, and their owners.
-    #expect(Set(issues.map { $0["id"]! }) == ["i1", "i3", "i4"])
+    // The fixture's open issues, their comments, and their owners. Suites
+    // running beside this one write open issues of their own, under
+    // `Chaos.prefix`.
+    let fixtureIssues = issues.filter { $0["id"]?.stringValue?.hasPrefix("chaos-") != true }
+    #expect(Set(fixtureIssues.map { $0["id"]! }) == ["i1", "i3", "i4"])
     #expect(Set(comments.map { $0["id"]! }) == ["c1", "c2", "c3"])
     #expect(Set(users.map { $0["id"]! }) == ["u1"])
     #expect(issues.first { $0["id"] == "i3" }?["title"] == "Emoji 😀 in titles")