{
  "name": "The Orbyt Failure Corpus",
  "url": "https://www.orbytjobs.ai/orbyt-collective/how-it-works",
  "what": "Documented failure modes from a production codebase operated by AI agents, each one recorded when it bit, classified, and where a countermeasure was mechanized, linked to the countermeasure that now prevents it.",
  "method": "Item ids, the count, and first-documented dates are parsed from the committed corpus file. Public titles and taxonomy kinds are edited for publication from a reviewed map; the generator fails if the map and the corpus ever disagree on which items exist, and fails if a claimed countermeasure path is absent from the repository.",
  "caveat": "This is the operator grading its own mistakes. Early entries predate the dating convention and are published undated rather than backdated. A mechanism path names the countermeasure inside a private repository; the path is published so the claim is specific, even though the file is not.",
  "license": "CC BY 4.0, attribution required",
  "cite_as": "Orbyt Failure Corpus (2026-08-13), https://www.orbytjobs.ai/orbyt-collective/how-it-works",
  "generated_at": "2026-08-13T17:29:33.347Z",
  "count": 46,
  "mechanized_count": 24,
  "dated_count": 36,
  "first_documented": "2026-06-09",
  "latest_documented": "2026-08-13",
  "taxonomy": {
    "external-behavior": {
      "definition": "An external platform, client, or dependency behaved differently than documented or assumed.",
      "count": 11
    },
    "verification-gap": {
      "definition": "A test, guard, gate, or metric that could not fail, could not see, or measured a proxy for the thing it claimed to measure.",
      "count": 19
    },
    "operator-process": {
      "definition": "A discipline failure in how the AI operator built or maintained the system: stale documents, unswept invariants, contracts with one side missing.",
      "count": 16
    }
  },
  "items": [
    {
      "id": 1,
      "date": null,
      "kind": "external-behavior",
      "title": "A mobile browser can suspend a tab without releasing its Web Locks, deadlocking auth refresh",
      "mechanism": "app/layout.tsx"
    },
    {
      "id": 2,
      "date": null,
      "kind": "external-behavior",
      "title": "Realtime sync can echo a write back into itself unless transient UI state stays out of the database"
    },
    {
      "id": 3,
      "date": null,
      "kind": "external-behavior",
      "title": "High-frequency writes to a replicated column can destabilize the replication layer itself"
    },
    {
      "id": 4,
      "date": null,
      "kind": "operator-process",
      "title": "A cleared cache with a surviving session is a distinct state; UI must wait for hydration to complete"
    },
    {
      "id": 5,
      "date": null,
      "kind": "external-behavior",
      "title": "A managed provider's default email path cannot be assumed to deliver; route through one you can observe"
    },
    {
      "id": 6,
      "date": null,
      "kind": "operator-process",
      "title": "Unsafe HTML injection requires an explicit, same-line trust annotation to pass review"
    },
    {
      "id": 7,
      "date": null,
      "kind": "operator-process",
      "title": "A module whose internal name diverges from its export breaks tooling that assumes they match"
    },
    {
      "id": 8,
      "date": null,
      "kind": "operator-process",
      "title": "After a rename, the old name is a liability: redirect it once, then never reference it again"
    },
    {
      "id": 9,
      "date": null,
      "kind": "verification-gap",
      "title": "A date test that mocks the clock in a different timezone convention than the code parses fails only west of UTC"
    },
    {
      "id": 10,
      "date": null,
      "kind": "external-behavior",
      "title": "A client library that auto-deserializes on read breaks every consumer that stored a string, and only on cache hit",
      "mechanism": "app/api/_lib/cache.ts"
    },
    {
      "id": 11,
      "date": "2026-06-09",
      "kind": "operator-process",
      "title": "A programmatic page matrix needs a bounded kept-set imported by both the builder and the sitemap",
      "mechanism": "__tests__/site/salary-data.test.ts"
    },
    {
      "id": 12,
      "date": "2026-06-09",
      "kind": "operator-process",
      "title": "An internal spec date is not a protocol version; clients hard-reject unknown values",
      "mechanism": "__tests__/mcp/protocol-negotiation.test.ts"
    },
    {
      "id": 13,
      "date": "2026-07-10",
      "kind": "operator-process",
      "title": "A redirect parameter is a contract between writer and reader; a silent fallback hides the mismatch"
    },
    {
      "id": 14,
      "date": "2026-07-10",
      "kind": "external-behavior",
      "title": "Hiding overflow on one axis creates a scroll container on the other and can trap wheel events"
    },
    {
      "id": 15,
      "date": "2026-07-11",
      "kind": "operator-process",
      "title": "A client-reachable module that statically imports a heavy data source ships it to every page that renders it"
    },
    {
      "id": 16,
      "date": "2026-07-23",
      "kind": "verification-gap",
      "title": "A test that derives its expected value from the code under test can never fail",
      "mechanism": "scripts/check-log-status-truth.cjs"
    },
    {
      "id": 17,
      "date": "2026-07-23",
      "kind": "operator-process",
      "title": "A pre-authentication code path needs its own abuse ceiling",
      "mechanism": "app/api/v1/intelligence/_lib/route-wrapper.ts"
    },
    {
      "id": 18,
      "date": "2026-07-23",
      "kind": "verification-gap",
      "title": "A table listing's row counts are planner estimates; report a real count or report nothing"
    },
    {
      "id": 19,
      "date": "2026-07-25",
      "kind": "external-behavior",
      "title": "Email clients default to content-box, so full width plus padding overflows a phone"
    },
    {
      "id": 20,
      "date": "2026-07-25",
      "kind": "verification-gap",
      "title": "A CI gate that only runs on the day it matters has never actually been tested; force the condition once"
    },
    {
      "id": 21,
      "date": "2026-07-26",
      "kind": "verification-gap",
      "title": "A convention that has never been executed is not a working convention"
    },
    {
      "id": 22,
      "date": "2026-07-28",
      "kind": "verification-gap",
      "title": "A build filter that decides inclusion is a detector, and must be proven on the shortest legitimate case"
    },
    {
      "id": 23,
      "date": "2026-07-30",
      "kind": "external-behavior",
      "title": "Invalid CSS is discarded, not flagged; a broken value silently becomes the property's initial value"
    },
    {
      "id": 24,
      "date": "2026-07-31",
      "kind": "external-behavior",
      "title": "Read a provider response by scanning for the block you want, never by indexing position zero",
      "mechanism": "scripts/check-ai-response-shape.cjs"
    },
    {
      "id": 25,
      "date": "2026-07-31",
      "kind": "external-behavior",
      "title": "A model's output budget caps reasoning plus text; leave neither half implicit",
      "mechanism": "app/api/_lib/ai-models.ts"
    },
    {
      "id": 26,
      "date": "2026-07-31",
      "kind": "verification-gap",
      "title": "A tool that runs quarterly has no gate between quarters, and its hand-copied config drifts silently",
      "mechanism": "__tests__/config/stryker-config-sync.test.ts"
    },
    {
      "id": 27,
      "date": "2026-07-31",
      "kind": "verification-gap",
      "title": "A cron reporting success is not healthy; watch duration against its ceiling and reap rows whose process never returned"
    },
    {
      "id": 28,
      "date": "2026-07-31",
      "kind": "verification-gap",
      "title": "A timeout reports as cancelled, not failed; alarms keyed on failure alone are structurally blind to it",
      "mechanism": "scripts/check-ci-headroom.cjs"
    },
    {
      "id": 29,
      "date": "2026-07-31",
      "kind": "verification-gap",
      "title": "A metric measured and then discarded is indistinguishable from a metric never measured"
    },
    {
      "id": 30,
      "date": "2026-08-01",
      "kind": "verification-gap",
      "title": "A synchronous spawn against an in-process fake server deadlocks, and the hang reads as the child failing"
    },
    {
      "id": 31,
      "date": "2026-08-01",
      "kind": "verification-gap",
      "title": "A step that discards stderr turns a missing permission into a silently wrong branch",
      "mechanism": "__tests__/scripts/daily-pulse.test.ts"
    },
    {
      "id": 32,
      "date": "2026-08-01",
      "kind": "operator-process",
      "title": "A relabeled invariant must sweep every detector that encodes it, in the same commit",
      "mechanism": "scripts/check-data-integrity.cjs"
    },
    {
      "id": 33,
      "date": "2026-08-01",
      "kind": "external-behavior",
      "title": "Checks sharing one rate-limit window flake each other; absorb exactly one window or serialize",
      "mechanism": "scripts/check-smoke.cjs"
    },
    {
      "id": 34,
      "date": "2026-08-01",
      "kind": "verification-gap",
      "title": "A fix committed but not pushed does not exist, and a guard grading the working tree will certify it as live",
      "mechanism": "scripts/check-ci-headroom.cjs"
    },
    {
      "id": 35,
      "date": "2026-08-02",
      "kind": "verification-gap",
      "title": "An instrument's failure paths must render as not observed, never as clean",
      "mechanism": "__tests__/scripts/daily-pulse.test.ts"
    },
    {
      "id": 36,
      "date": "2026-08-02",
      "kind": "operator-process",
      "title": "A lesson mechanized in the canonical pipeline does not exist for the surfaces that bypass it",
      "mechanism": "__tests__/scripts/org-guard-parity.test.ts"
    },
    {
      "id": 37,
      "date": "2026-08-05",
      "kind": "operator-process",
      "title": "A selection rule optimizes its proxy; verify what exists against measured demand, not against the code",
      "mechanism": "scripts/check-salary-demand-coverage.ts"
    },
    {
      "id": 38,
      "date": "2026-08-08",
      "kind": "operator-process",
      "title": "A sync rule that tests equality can never be satisfied by the thing it exists to allow; record provenance instead",
      "mechanism": "__tests__/scripts/sync-i18n-provenance.test.ts"
    },
    {
      "id": 39,
      "date": "2026-08-08",
      "kind": "verification-gap",
      "title": "Never validate a filter against data that filter produced"
    },
    {
      "id": 40,
      "date": "2026-08-10",
      "kind": "verification-gap",
      "title": "A gate whose correctness rests on the thing it is gating is not a gate",
      "mechanism": ".github/actions/org-agent/action.yml"
    },
    {
      "id": 41,
      "date": "2026-08-10",
      "kind": "verification-gap",
      "title": "A job cancelled with zero steps was never handed to a runner; an alarm inside the job cannot see the job never starting",
      "mechanism": "scripts/check-ci-headroom.cjs"
    },
    {
      "id": 42,
      "date": "2026-08-10",
      "kind": "verification-gap",
      "title": "Read a monitor's freshness the way the monitor is actually fed",
      "mechanism": "__tests__/scripts/daily-pulse.test.ts"
    },
    {
      "id": 43,
      "date": "2026-08-10",
      "kind": "operator-process",
      "title": "A stale document reads as instructions; when an invariant reverses, sweep the documents that assert it",
      "mechanism": "scripts/check-root-clutter.cjs"
    },
    {
      "id": 44,
      "date": "2026-08-13",
      "kind": "verification-gap",
      "title": "A sandbox that cannot start reports success, and a fence can apply halfway",
      "mechanism": ".github/workflows/fence-canary.yml"
    },
    {
      "id": 45,
      "date": "2026-08-13",
      "kind": "operator-process",
      "title": "A confinement lane is what an agent is instructed to write, not what gets committed",
      "mechanism": "__tests__/scripts/org-fence.test.ts"
    },
    {
      "id": 46,
      "date": "2026-08-13",
      "kind": "operator-process",
      "title": "A guard's exit code is not its verdict; report-only and strict-gated checks exit zero while reporting"
    }
  ]
}