Stage 8 acceptance: fault-injection integration suite S01-S14 + X1-X3 (SPIKE-02 §3) on real pull-verify-stage-activate-boot machine
Some checks failed
ci / check (push) Has been cancelled

This commit is contained in:
avi 2026-10-02 21:28:10 -05:00
commit 506bebed9c
2 changed files with 619 additions and 2 deletions

View file

@ -1,5 +1,13 @@
# tests/integration # tests/integration
Full update lifecycle mocked transport + fault injection 9 stages, exp2 model. Full update lifecycle on fake-indexeddb with mocked transport + fault
injection, mirroring `experiments/exp2-ab-update-sim.mjs` (SPIKE-02 §3)
against the real pull → verify → stage → activate → boot state machine.
Stage 1: directory exists with .gitkeep; no feature code yet per IMPLEMENTATION-CONTRACT.md Stage 1. See IMPLEMENTATION-CONTRACT.md §4 responsibilities and §6 B-1…B-7. - `ab-lifecycle.test.ts` — Stage 8 acceptance: S01–S14 + X1–X3 (16 SPIKE-02
scenarios as 18 tests; no-loop and retry-once covered separately).
Invariant: either the previous valid dataset stays active or the new one
becomes active; favorites are never lost. State-machine level only —
iOS jetsam proof is device test T10 (§36).
See IMPLEMENTATION-CONTRACT.md §4 responsibilities and §6 B-1…B-7.

View file

@ -0,0 +1,609 @@
/* eslint-disable @typescript-eslint/no-unsafe-call, @typescript-eslint/require-await, @typescript-eslint/no-non-null-assertion */
/**
* Stage 8 — fault-injection integration suite mirroring SPIKE-02 §3
* (exp2-ab-update-sim.mjs S01–S14 + X1–X3) against the REAL
* pull→verify→stage→activate→boot state machine on fake-indexeddb.
*
* Invariant under test (SPIKE-02): "Either the previous valid dataset remains
* active or the new valid dataset becomes active. The app never knowingly
* exposes a partial dataset. Favorites are never lost."
*
* Acceptance is at state-machine level (IMPLEMENTATION-CONTRACT.md Stage 8);
* iOS jetsam proof remains device test T10 §36.
*/
import { describe, it, expect, beforeEach } from "vitest";
// @ts-expect-error fake-indexeddb types via exports fallback
import FDBFactory from "fake-indexeddb/lib/FDBFactory";
// @ts-expect-error fake-indexeddb types via exports fallback
import FDBKeyRange from "fake-indexeddb/lib/FDBKeyRange";
const g = globalThis as unknown as Record<string, unknown>;
g.indexedDB = new FDBFactory() as unknown;
g.IDBKeyRange = FDBKeyRange as unknown;
import { buildPackage } from "../../pipeline/package.js";
import { generateTestKeyPair } from "../../pipeline/sign.js";
import { makeValidInput } from "../../pipeline/fixtures.js";
import { canonicalJson } from "../../pipeline/canonical-json.js";
import type { KeyPair } from "../../pipeline/sign.js";
import { publicKeyFromDerBase64 } from "../../src/sync/verifier/ed25519.js";
import { verifyPackage } from "../../src/sync/verifier/package.js";
import { pullCandidate } from "../../src/sync/pull.js";
import type { Transport } from "../../src/sync/transport/types.js";
import {
activateSlot,
openSystemDB,
readSystemMeta,
writeSystemMeta,
} from "../../src/data/system-meta/store.js";
import {
initializeStaging,
openSlotDB,
readStagingProgress,
writeSlotFileWithProgress,
} from "../../src/data/slot/store.js";
import { openUserDB, addFavorite, listFavorites } from "../../src/data/user/store.js";
import { isQuarantined, quarantineSink } from "../../src/data/user/quarantine.js";
import {
activateStagedPackage,
recoverPendingActivation,
restorePreviousSlot,
stageVerifiedPackage,
} from "../../src/sync/activation.js";
import type { StagedPackage } from "../../src/sync/activation.js";
import type { VerifiedPackage } from "../../src/sync/verifier/types.js";
import { withTx } from "../../src/platform/idb/wrapper.js";
import { DB, SLOT_FILES } from "../../src/platform/idb/names.js";
import type { SlotId } from "../../src/platform/idb/names.js";
import { evaluateBootReadiness, defaultBootDeps } from "../../src/app/readiness.js";
import type { BootReadinessDeps } from "../../src/app/readiness.js";
const EDITION = "lumen-2026";
// ——— harness ———
function deleteDb(name: string): Promise<void> {
return new Promise((resolve, reject) => {
const req = (g.indexedDB as IDBFactory).deleteDatabase(name);
req.onsuccess = () => {
resolve();
};
req.onerror = () => {
reject(req.error ?? new Error("delete database failed"));
};
req.onblocked = () => {
resolve();
};
});
}
interface Built {
readonly keyPair: KeyPair;
/** file path → canonical bytes, keyed the way the manifest references them. */
readonly files: Map<string, Uint8Array>;
readonly manifestBytes: Uint8Array;
readonly signatureBytes: Uint8Array;
readonly pkg: VerifiedPackage | null; // direct-verify witness (used by activation paths)
}
async function built(version: number, keyPair = generateTestKeyPair()): Promise<Built> {
const pkg = buildPackage(makeValidInput({ packageVersion: version }), { signWith: keyPair });
if (!pkg.ok || !pkg.pkg.signature) throw new Error("fixture build failed");
const manifestBytes = new TextEncoder().encode(canonicalJson(pkg.pkg.manifest));
const files = new Map<string, Uint8Array>();
for (const [name, file] of pkg.pkg.files) files.set(name, file.canonicalBytes);
for (const asset of pkg.pkg.assets) files.set(asset.file, asset.bytesContent);
const result = await verifyPackage(
{
manifestBytes,
signature: pkg.pkg.signature,
files: { getFile: (name) => Promise.resolve(files.get(name)) },
emergencyFloor: pkg.pkg.emergencyFloor,
},
{
trustedKeys: new Map([
[keyPair.fingerprint, publicKeyFromDerBase64(keyPair.publicKeyDerBase64)],
]),
appVersion: "1.0.0",
supportedSchemaRange: [1],
},
);
if (!result.ok) throw new Error(`fixture verify failed: ${result.reason}`);
return {
keyPair,
files,
manifestBytes,
signatureBytes: new TextEncoder().encode(JSON.stringify(pkg.pkg.signature)),
pkg: result,
};
}
/** Transport over an in-memory origin; records every URL it is asked for. */
function originTransport(build: Built, fetches: string[], version: number): Transport {
const base = `/editions/${EDITION}/packages/${String(version)}/`;
return {
isAvailable: () => true,
fetchPointer: async () => {
fetches.push("latest.json");
return {
edition: EDITION,
packageVersion: version,
manifestUrl: `${base}manifest.json`,
generatedAt: "2026-08-30T12:00:00.000Z",
};
},
fetchBytes: async (url) => {
fetches.push(url);
if (url.endsWith("manifest.json")) return build.manifestBytes;
if (url.endsWith("signature.json")) return build.signatureBytes;
const name = url.split("/").pop() ?? "";
const sub = url.includes("/assets/") ? `assets/${name}` : name;
const content = build.files.get(sub);
if (content) return content;
throw new Error(`unexpected fetch ${url}`);
},
};
}
function bootDeps(): BootReadinessDeps {
return {
...defaultBootDeps,
checkShell: async () => ({ ok: true, cacheName: "lumen-shell-test" }),
loadFloor: () => ({ ok: true, version: "embedded-1", bytes: 4096 }),
estimate: async () => null,
// Pipeline fixtures declare minAppVersion 1.0.0 / schema 1; the dev shell
// version in package.json is younger (same seam readiness.test uses).
appVersion: "1.0.0",
supportedSchemaRange: [1],
};
}
/** Full sync run: pull (with quarantine wiring) → stage → activate. */
async function syncRun(
build: Built,
version: number,
opts: { readonly trustedKeys?: Map<string, Uint8Array>; readonly fetches?: string[] } = {},
): Promise<{ readonly ok: boolean; readonly reason?: string; readonly skipped?: string }> {
const user = await openUserDB();
const fetches = opts.fetches ?? [];
const outcome = await pullCandidate(
originTransport(build, fetches, version),
EDITION,
{
trustedKeys:
opts.trustedKeys ??
new Map([
[build.keyPair.fingerprint, publicKeyFromDerBase64(build.keyPair.publicKeyDerBase64)],
]),
appVersion: "1.0.0",
supportedSchemaRange: [1],
quarantine: quarantineSink(user),
},
{ isQuarantined: async (v) => isQuarantined(user, EDITION, v) },
);
user.close();
if (!outcome) return { ok: false, reason: "no pointer" };
if ("skipped" in outcome) return { ok: false, skipped: outcome.skipped };
if (!outcome.result.ok) return { ok: false, reason: outcome.result.reason };
const staged = await stageVerifiedPackage(outcome.result);
const activated = await activateStagedPackage(outcome.result, staged, "1.0.0");
return activated.reason === undefined
? { ok: activated.ok }
: { ok: activated.ok, reason: activated.reason };
}
async function activateVersion(version: number): Promise<Built> {
const build = await built(version);
const staged = await stageVerifiedPackage(build.pkg!);
const activated = await activateStagedPackage(build.pkg!, staged, "1.0.0");
if (!activated.ok) throw new Error(`seed activation v${String(version)} failed`);
return build;
}
async function meta() {
const system = await openSystemDB();
const m = await readSystemMeta(system);
system.close();
return m;
}
async function bootState() {
return evaluateBootReadiness(bootDeps());
}
async function corruptFile(slot: SlotId, id: string): Promise<void> {
const db = await openSlotDB(slot);
await withTx(db, SLOT_FILES, "readwrite", (tx) => {
tx.objectStore(SLOT_FILES).delete(id);
});
db.close();
}
const SECTION_IDS = ["emergency", "schedule", "map", "info", "assets"] as const;
beforeEach(async () => {
await Promise.all(Object.values(DB).map((name) => deleteDb(name)));
});
// ——— SPIKE-02 §3 walkthrough ———
describe("Stage 8 integration — SPIKE-02 S01–S14 + X1–X3", () => {
it("S01: crash before any file lands → old (v1) still active, nothing staged", async () => {
await activateVersion(1);
const build = await built(2);
const fetches: string[] = [];
// "crash" = pull succeeds, staging never invoked (process died there).
const user = await openUserDB();
const outcome = await pullCandidate(originTransport(build, fetches, 2), EDITION, {
trustedKeys: new Map([
[build.keyPair.fingerprint, publicKeyFromDerBase64(build.keyPair.publicKeyDerBase64)],
]),
appVersion: "1.0.0",
supportedSchemaRange: [1],
quarantine: quarantineSink(user),
});
user.close();
expect(outcome && "result" in outcome && outcome.result.ok).toBe(true);
const m = await meta();
expect(m.activePackageVersion).toBe(1);
const report = await bootState();
expect(report.state).toBe("READY");
expect(report.packageVersion).toBe(1);
});
it("S02: partial staging (3/5 files) → old (v1) active; activation refuses incomplete slot", async () => {
await activateVersion(1);
const build = await built(2);
const slot = await openSlotDB("B");
let journal = await initializeStaging(slot, {
edition: EDITION,
packageVersion: 2,
manifestSha256: build.pkg!.manifestSha256,
startedAt: Date.now(),
lastProgressAt: Date.now(),
});
for (const id of SECTION_IDS.slice(0, 3)) {
const entry = build.pkg!.manifest.sections[id];
const bytes = build.pkg!.files.get(entry.file)!;
journal = await writeSlotFileWithProgress(
slot,
id,
{
bytes: entry.bytes,
sha256: entry.sha256,
json: JSON.parse(new TextDecoder().decode(bytes)) as unknown,
},
journal,
);
}
slot.close();
const fake: StagedPackage = { slot: "B", journal };
const result = await activateStagedPackage(build.pkg!, fake, "1.0.0");
expect(result).toMatchObject({ ok: false, rolledBack: false });
const m = await meta();
expect(m.activeSlot === "A" && m.activePackageVersion === 1).toBe(true);
const report = await bootState();
expect(report.state).toBe("READY");
expect(report.packageVersion).toBe(1);
});
it("S03: process terminated mid-staging → resume completes without redoing staged files", async () => {
await activateVersion(1);
const build = await built(2);
const slot = await openSlotDB("B");
let journal = await initializeStaging(slot, {
edition: EDITION,
packageVersion: 2,
manifestSha256: build.pkg!.manifestSha256,
startedAt: 1,
lastProgressAt: 1,
});
for (const id of SECTION_IDS.slice(0, 3)) {
const entry = build.pkg!.manifest.sections[id];
const bytes = build.pkg!.files.get(entry.file)!;
journal = await writeSlotFileWithProgress(
slot,
id,
{
bytes: entry.bytes,
sha256: entry.sha256,
json: JSON.parse(new TextDecoder().decode(bytes)) as unknown,
},
journal,
);
}
slot.close();
const resumed = await stageVerifiedPackage(build.pkg!);
expect(resumed.journal.complete).toBe(true);
expect(resumed.journal.startedAt).toBe(1); // resume, not restart
expect(resumed.journal.stagedFiles).toHaveLength(5);
const activated = await activateStagedPackage(build.pkg!, resumed, "1.0.0");
expect(activated.ok).toBe(true);
const report = await bootState();
expect(report.state).toBe("READY");
expect(report.packageVersion).toBe(2);
});
it("S04: reboot before activation (staged, never flipped) → old (v1) active, activationPending", async () => {
await activateVersion(1);
const build = await built(2);
await stageVerifiedPackage(build.pkg!); // fully staged; flip never happens
const m = await meta();
expect(m.activePackageVersion).toBe(1);
const report = await bootState();
// Active pointer still v1 → READY on old dataset (never exposes the unactivated slot).
expect(report.state).toBe("READY");
expect(report.packageVersion).toBe(1);
});
it("S05: dataset validation fails (generic reject) → old active, candidate quarantined", async () => {
await activateVersion(1);
const build = await built(2);
const fetches: string[] = [];
// Corrupt one staged file's bytes → post-download integrity/validation gate rejects.
const original = build.files.get("schedule.json")!;
build.files.set("schedule.json", new TextEncoder().encode("NOT JSON"));
const run = await syncRun(build, 2, { fetches });
expect(run.ok).toBe(false);
const m = await meta();
expect(m.activePackageVersion).toBe(1);
const user = await openUserDB();
expect(await isQuarantined(user, EDITION, 2)).toBe(true);
user.close();
// restore for a clean boot check
build.files.set("schedule.json", original);
const report = await bootState();
expect(report.state).toBe("READY");
});
it("S06: integrity hash fails (corrupt file) → old active, candidate rejected + quarantined", async () => {
await activateVersion(1);
const build = await built(2);
build.files.set("map.json", new Uint8Array([1, 2, 3, 4])); // wrong bytes, manifest hash still original
const run = await syncRun(build, 2);
expect(run.ok).toBe(false);
const m = await meta();
expect(m.activePackageVersion).toBe(1);
const user = await openUserDB();
expect(await isQuarantined(user, EDITION, 2)).toBe(true);
user.close();
});
it("S07: signature validation fails → rejected BEFORE any section/file fetch; quarantined under pointer identity", async () => {
await activateVersion(1);
const build = await built(2);
const fetches: string[] = [];
// Trust a DIFFERENT key: the package's fingerprint is unknown → signature gate fails.
const wrongTrusted = generateTestKeyPair();
const run = await syncRun(build, 2, {
trustedKeys: new Map([
[wrongTrusted.fingerprint, publicKeyFromDerBase64(wrongTrusted.publicKeyDerBase64)],
]),
fetches,
});
expect(run.ok).toBe(false);
// Only pointer + manifest + signature fetched — zero section/asset bytes downloaded.
const fileFetches = fetches.filter(
(f) =>
!f.endsWith("latest.json") && !f.endsWith("manifest.json") && !f.endsWith("signature.json"),
);
expect(fileFetches).toEqual([]);
const m = await meta();
expect(m.activePackageVersion).toBe(1);
const user = await openUserDB();
expect(await isQuarantined(user, EDITION, 2)).toBe(true); // hint-keyed despite pre-manifest rejection
user.close();
});
it("S08: schema validation fails → old active, candidate rejected", async () => {
await activateVersion(1);
const build = await built(2);
const user = await openUserDB();
const fetches: string[] = [];
const outcome = await pullCandidate(originTransport(build, fetches, 2), EDITION, {
trustedKeys: new Map([
[build.keyPair.fingerprint, publicKeyFromDerBase64(build.keyPair.publicKeyDerBase64)],
]),
appVersion: "1.0.0",
supportedSchemaRange: [2], // shell does not support schema 1 of the package
quarantine: quarantineSink(user),
});
user.close();
expect(outcome && "result" in outcome && outcome.result.ok).toBe(false);
const m = await meta();
expect(m.activePackageVersion).toBe(1);
});
it("S09: compatibility validation fails (app too old) → rejected before files; nothing downloaded", async () => {
await activateVersion(1);
const build = await built(2);
const user = await openUserDB();
const fetches: string[] = [];
const outcome = await pullCandidate(originTransport(build, fetches, 2), EDITION, {
trustedKeys: new Map([
[build.keyPair.fingerprint, publicKeyFromDerBase64(build.keyPair.publicKeyDerBase64)],
]),
appVersion: "0.5.0", // below manifest minAppVersion 1.0.0
supportedSchemaRange: [1],
quarantine: quarantineSink(user),
});
user.close();
expect(outcome && "result" in outcome && outcome.result.ok).toBe(false);
const fileFetches = fetches.filter(
(f) =>
!f.endsWith("latest.json") && !f.endsWith("manifest.json") && !f.endsWith("signature.json"),
);
expect(fileFetches).toEqual([]);
const m = await meta();
expect(m.activePackageVersion).toBe(1);
});
it("S10+S11: activation succeeds — pointer, version, edition, verification record flip together to v2", async () => {
const v1 = await activateVersion(1);
const build = await built(2);
const run = await syncRun(build, 2);
expect(run.ok).toBe(true);
const m = await meta();
expect(m.activeSlot).toBe("B");
expect(m.activePackageVersion).toBe(2);
expect(m.activeEdition).toBe(EDITION);
expect(m.verification?.manifestSha256).toBe(build.pkg!.manifestSha256);
expect(m.verification?.publicKeyFingerprint).toBe(build.keyPair.fingerprint);
expect(m.verification?.packageVersion).toBe(2);
expect(m.readbackPending).toBe(false); // cleared post-readback
// v1 remains intact in slot A (rollback depth 1)
const slotA = await openSlotDB("A");
const journalA = await readStagingProgress(slotA);
slotA.close();
expect(journalA?.packageVersion).toBe(1);
expect(v1.pkg !== null).toBe(true);
});
it("S12: process terminated immediately after flip → next-boot recovery confirms new (v2), pending cleared", async () => {
await activateVersion(1);
await activateVersion(2);
// Simulate the kill window (F-3): flip committed, readbackPending still set.
const m0 = await meta();
const system = await openSystemDB();
await writeSystemMeta(system, { ...m0, readbackPending: true });
system.close();
expect(await recoverPendingActivation()).toBe(true);
const m = await meta();
expect(m.activePackageVersion).toBe(2);
expect(m.readbackPending).toBe(false);
const report = await bootState();
expect(report.state).toBe("READY");
expect(report.packageVersion).toBe(2);
});
it("S13: crash during flip (transaction never commits) → old pointer stands", async () => {
await activateVersion(1);
const build = await built(2);
const staged = await stageVerifiedPackage(build.pkg!);
const result = await activateStagedPackage(build.pkg!, staged, "1.0.0", Date.now, {
// Model the uncommitted flip: both attempts die without committing.
activate: async () => {
throw new Error("transaction aborted mid-flip");
},
});
expect(result).toMatchObject({ ok: false, rolledBack: false });
const m = await meta();
expect(m.activeSlot === "A" && m.activePackageVersion === 1).toBe(true);
expect(m.readbackPending).toBe(false);
const report = await bootState();
expect(report.state).toBe("READY");
expect(report.packageVersion).toBe(1);
});
it("S14: corruption found by post-activation readback → automatic rollback to last complete slot", async () => {
await activateVersion(1);
const build = await built(2);
const staged = await stageVerifiedPackage(build.pkg!);
const result = await activateStagedPackage(build.pkg!, staged, "1.0.0", Date.now, {
afterFlip: async (slot) => corruptFile(slot, "map"),
});
expect(result).toMatchObject({ ok: false, rolledBack: true });
const m = await meta();
expect(m.activeSlot === "A" && m.activePackageVersion === 1).toBe(true);
const report = await bootState();
expect(report.state).toBe("READY");
expect(report.packageVersion).toBe(1);
});
it("X1: active slot corrupted at boot (readback pending) → fallback slot served", async () => {
await activateVersion(1);
await activateVersion(2); // A=v1, B=v2 active
// Post-activation bitrot hits the active slot B before next boot confirms it.
const system = await openSystemDB();
const m0 = await readSystemMeta(system);
await writeSystemMeta(system, { ...m0, readbackPending: true });
system.close();
await corruptFile("B", "schedule");
expect(await recoverPendingActivation()).toBe(true);
const m = await meta();
expect(m.activeSlot === "A" && m.activePackageVersion === 1).toBe(true);
const report = await bootState();
expect(report.state).toBe("READY");
expect(report.packageVersion).toBe(1);
});
it("X2: both slots lost → RECOVERY with emergency floor; favorites intact", async () => {
await activateVersion(1);
const user = await openUserDB();
await addFavorite(user, { eventId: "evt-x2", addedAt: 1 });
user.close();
await deleteDb(DB.SLOT_A);
await deleteDb(DB.SLOT_B);
const report = await bootState();
expect(report.state).toBe("RECOVERY");
expect(report.reason).toBe("missing");
expect(report.floorVersion).toBeTruthy(); // embedded emergency baseline present
const user2 = await openUserDB();
const favs = await listFavorites(user2);
user2.close();
expect(favs.map((f) => f.eventId)).toEqual(["evt-x2"]);
});
it("X3: favorites survive a full update lifecycle (v1 → v2 → rollback)", async () => {
const user = await openUserDB();
await addFavorite(user, { eventId: "evt-keep", addedAt: 7 });
user.close();
await activateVersion(1);
const build = await built(2);
expect((await syncRun(build, 2)).ok).toBe(true);
expect(await restorePreviousSlot()).toBe(true);
const user2 = await openUserDB();
const favs = await listFavorites(user2);
user2.close();
expect(favs.map((f) => f.eventId)).toEqual(["evt-keep"]);
const report = await bootState();
expect(report.state).toBe("READY");
expect(report.packageVersion).toBe(1);
});
it("no-loop: quarantined v2 is never re-fetched until latest.json advances past it", async () => {
await activateVersion(1);
const build = await built(2);
build.files.set("map.json", new Uint8Array([9, 9, 9])); // poison v2 → quarantine
expect((await syncRun(build, 2)).ok).toBe(false);
const fetches: string[] = [];
const user = await openUserDB();
const outcome = await pullCandidate(
originTransport(build, fetches, 2),
EDITION,
{
trustedKeys: new Map([
[build.keyPair.fingerprint, publicKeyFromDerBase64(build.keyPair.publicKeyDerBase64)],
]),
appVersion: "1.0.0",
supportedSchemaRange: [1],
quarantine: quarantineSink(user),
},
{ isQuarantined: async (v) => isQuarantined(user, EDITION, v) },
);
user.close();
expect(outcome).toMatchObject({ skipped: "quarantined" });
expect(fetches).toEqual(["latest.json"]);
const m = await meta();
expect(m.activePackageVersion).toBe(1);
});
it("activation seam honors single-txn contract: retry-once recovers a transient flip failure", async () => {
await activateVersion(1);
const build = await built(2);
const staged = await stageVerifiedPackage(build.pkg!);
let attempts = 0;
const result = await activateStagedPackage(build.pkg!, staged, "1.0.0", Date.now, {
activate: async (db, next) => {
attempts += 1;
if (attempts === 1) throw new Error("transient IDB failure");
return activateSlot(db, next);
},
});
expect(result.ok).toBe(true);
expect(attempts).toBe(2);
const report = await bootState();
expect(report.state).toBe("READY");
expect(report.packageVersion).toBe(2);
});
});