import assert from 'node:assert/strict' import { registerHooks } from 'node:module' import { DatabaseSync } from 'node:sqlite' import { beforeEach, describe, it } from 'node:test' /** * Fakes for kody: imports so webhook → deferred re-check → runNow runs under * node:test. packageStorage() is a real in-memory SQLite (node:sqlite), so the * episode claim (INSERT … ON CONFLICT … WHERE … RETURNING) is exercised for real. * Nothing here can reach Cursor or GitHub. */ type Run = { id: number workflow_id: number name: string path: string event: string head_branch: string head_sha: string status: string conclusion: string | null run_attempt: number created_at: string html_url: string } type Fakes = { db: DatabaseSync createAgentCalls: number createAgentFails: boolean lastPrompt: string workflows: Map }> headSha: string runsBySha: Record pulls: Array> } const fakes: Fakes = { db: new DatabaseSync(':memory:'), createAgentCalls: 0, createAgentFails: false, lastPrompt: '', workflows: new Map(), headSha: 'sha-a', runsBySha: {}, pulls: [], } ;(globalThis as Record).__codebaseCleanupFakes = fakes const fakeModules: Record = { 'kody:@kentcdodds/jev/evaluate': ` export default async function evaluate() { throw new Error('prod-ci-failure must not call Jev') } `, 'kody:runtime': ` const f = globalThis.__codebaseCleanupFakes export function packageStorage() { return { async sql(query, b = []) { return { rows: f.db.prepare(query).all(...b).map((r) => ({ ...r })) } }, } } export const workflows = { async create(input) { const existing = f.workflows.get(input.idempotencyKey) if (existing) return { id: existing.id, status: 'queued' } const id = 'pkgwf-' + (f.workflows.size + 1) f.workflows.set(input.idempotencyKey, { id, input }) return { id, status: 'queued' } }, } `, 'kody:@kentcdodds/cursor/agents': ` const f = globalThis.__codebaseCleanupFakes export async function createAgent(input) { if (f.createAgentFails) throw new Error('usage limit') f.createAgentCalls += 1 f.lastPrompt = input?.prompt?.text ?? '' const id = 'agent-' + f.createAgentCalls return { agent: { id, url: 'https://cursor.com/agents/' + id } } } `, 'kody:@kentcdodds/devin/request': ` export async function devinRequest() { throw new Error('Devin must not be called by the CI path') } export async function resolveOrgId() { return 'org-test' } `, 'kody:@kentcdodds/github/request': ` const f = globalThis.__codebaseCleanupFakes export default async function githubRequest({ path }) { if (path.startsWith('/repos/acme/widgets/branches/')) { return { ok: true, status: 200, data: { commit: { sha: f.headSha } } } } if (path.startsWith('/repos/acme/widgets/actions/runs?head_sha=')) { const sha = decodeURIComponent(path.split('head_sha=')[1].split('&')[0]) return { ok: true, status: 200, data: { workflow_runs: f.runsBySha[sha] ?? [] } } } if (path.startsWith('/repos/acme/widgets/pulls?')) { return { ok: true, status: 200, data: f.pulls } } if (path.startsWith('/repos/acme/widgets/commits?')) { return { ok: true, status: 200, data: [{ sha: 'x' }] } } return { ok: false, status: 404, text: 'unexpected path ' + path } } `, } registerHooks({ resolve(specifier, context, nextResolve) { if (specifier in fakeModules) { return { url: `fake-kody:${specifier}`, shortCircuit: true } } return nextResolve(specifier, context) }, load(url, context, nextLoad) { if (url.startsWith('fake-kody:')) { return { format: 'module', source: fakeModules[url.slice('fake-kody:'.length)], shortCircuit: true, } } return nextLoad(url, context) }, }) const { default: handleCiFailure } = await import('./handle-ci-failure.ts') const { default: ciRecheck } = await import('./ci-recheck.ts') const { markHandled, list: listEpisodes } = await import('./ci-episode.ts') function setConfig(patch: Record = {}) { fakes.db.exec('CREATE TABLE IF NOT EXISTS config (key TEXT PRIMARY KEY, value TEXT NOT NULL)') fakes.db .prepare('INSERT OR REPLACE INTO config (key, value) VALUES (?, ?)') .run( 'automations', JSON.stringify({ 'prod-ci-failure': { enabled: true, repos: ['acme/widgets'], branch: 'main', ...patch, }, }), ) } let nextRunId = 100 function run(sha: string, name: string, conclusion: string | null, extra: Partial = {}): Run { nextRunId += 1 return { id: nextRunId, workflow_id: name === 'Validate' ? 1 : name === 'Report merge conflicts' ? 2 : 3, name, path: `.github/workflows/${name.toLowerCase().replace(/\s+/g, '-')}.yml`, event: 'push', head_branch: 'main', head_sha: sha, status: conclusion === null ? 'in_progress' : 'completed', conclusion, run_attempt: 1, created_at: '2026-10-22T06:57:42Z', html_url: `https://github.com/acme/widgets/actions/runs/${nextRunId}`, ...extra, } } function event( kind: 'check_suite' | 'workflow_run' | 'check_run', opts: { sha?: string; suite?: number; conclusion?: string; branch?: string; repo?: string } = {}, ) { const sha = opts.sha ?? 'sha-a' const suite = opts.suite ?? 11 const conclusion = opts.conclusion ?? 'failure' const branch = opts.branch ?? 'main' const repository = { full_name: opts.repo ?? 'acme/widgets' } const json = kind === 'check_suite' ? { action: 'completed', check_suite: { id: suite, conclusion, head_branch: branch, head_sha: sha }, repository, } : kind === 'workflow_run' ? { action: 'completed', workflow_run: { check_suite_id: suite, conclusion, head_branch: branch, head_sha: sha, name: 'Validate', html_url: 'https://github.com/acme/widgets/actions/runs/1', }, repository, } : { action: 'completed', check_run: { conclusion, head_sha: sha, name: 'Static', check_suite: { id: suite, head_branch: branch, head_sha: sha }, }, repository, } return { request: { headers: { 'x-github-event': kind, 'x-github-delivery': `${kind}-${sha}-${suite}` }, json, }, } } /** Run every scheduled re-check the way the platform would after runAt. */ async function drainWorkflows() { const out = [] for (const [key, wf] of [...fakes.workflows]) { if (wf.input.__ran) continue wf.input.__ran = true assert.equal(wf.input.exportName, './ci-recheck', key) out.push(await ciRecheck(wf.input.params)) } return out as any[] } beforeEach(() => { fakes.db = new DatabaseSync(':memory:') fakes.createAgentCalls = 0 fakes.createAgentFails = false fakes.lastPrompt = '' fakes.workflows.clear() fakes.headSha = 'sha-a' fakes.runsBySha = {} fakes.pulls = [] setConfig() }) describe('handleCiFailure (webhook) defers instead of launching', () => { it('schedules one ./ci-recheck ~60s out and never launches inline', async () => { const before = Date.now() const out = (await handleCiFailure(event('check_suite'))) as any assert.equal(out.scheduled, true) assert.equal(out.purpose, 'failure-recheck') assert.equal(fakes.createAgentCalls, 0) assert.equal(fakes.workflows.size, 1) const [wf] = [...fakes.workflows.values()] const delay = Date.parse(wf.input.runAt) - before assert.ok(delay >= 59_000 && delay <= 61_000, `delay ${delay}`) assert.deepEqual( { repo: wf.input.params.repo, branch: wf.input.params.branch, sha: wf.input.params.triggerSha }, { repo: 'acme/widgets', branch: 'main', sha: 'sha-a' }, ) }) it('collapses check_suite, workflow_run and check_run for one suite onto one re-check', async () => { const a = (await handleCiFailure(event('check_suite'))) as any const b = (await handleCiFailure(event('workflow_run'))) as any const c = (await handleCiFailure(event('check_run'))) as any assert.equal(a.idempotencyKey, b.idempotencyKey) assert.equal(b.idempotencyKey, c.idempotencyKey) assert.equal(fakes.workflows.size, 1) }) it('honors recheckDelaySeconds from config', async () => { setConfig({ recheckDelaySeconds: 5 }) const before = Date.now() await handleCiFailure(event('check_suite')) const [wf] = [...fakes.workflows.values()] const delay = Date.parse(wf.input.runAt) - before assert.ok(delay >= 4_000 && delay <= 6_000, `delay ${delay}`) }) it('ignores success when no red episode is open, but re-checks while one is', async () => { const ignored = (await handleCiFailure(event('check_suite', { conclusion: 'success', suite: 12 }))) as any assert.equal(ignored.reason, 'not-a-failure-event') assert.equal(fakes.workflows.size, 0) await markHandled({ repo: 'acme/widgets', branch: 'main', sha: 'sha-a', note: 'manual' }) const scheduled = (await handleCiFailure(event('check_suite', { conclusion: 'success', suite: 13 }))) as any assert.equal(scheduled.purpose, 'green-reset-check') assert.equal(fakes.workflows.size, 1) }) it('ignores other repos, other branches, and a disabled automation', async () => { assert.equal(((await handleCiFailure(event('check_suite', { repo: 'acme/other' }))) as any).reason, 'repo-not-configured') assert.equal(((await handleCiFailure(event('check_suite', { branch: 'feature' }))) as any).reason, 'branch-mismatch') setConfig({ enabled: false }) assert.equal(((await handleCiFailure(event('check_suite'))) as any).reason, 'automation-disabled') assert.equal(fakes.workflows.size, 0) }) it('dryRun previews the re-check without scheduling it', async () => { const out = (await handleCiFailure({ ...event('check_suite'), dryRun: true })) as any assert.equal(out.scheduled, false) assert.equal(out.params.repo, 'acme/widgets') assert.equal(fakes.workflows.size, 0) }) }) describe('ciRecheck (deferred) launches at most one agent per red stretch', () => { it('launches once when the head is still red, and records the episode', async () => { fakes.runsBySha['sha-a'] = [run('sha-a', 'Validate', 'failure'), run('sha-a', 'Report merge conflicts', 'success')] await handleCiFailure(event('check_suite')) const [result] = await drainWorkflows() assert.equal(result.action, 'launched') assert.equal(fakes.createAgentCalls, 1) assert.match(fakes.lastPrompt, /Head SHA: `sha-a`/) const [episode] = await listEpisodes() assert.equal(episode.status, 'open') assert.equal(episode.source, 'automation') assert.equal(episode.agent_id, 'agent-1') }) it('tonight: #4153 then b85e9e8 both red → one agent, not two', async () => { fakes.runsBySha['sha-a'] = [run('sha-a', 'Validate', 'failure')] await handleCiFailure(event('check_suite', { sha: 'sha-a', suite: 21 })) await drainWorkflows() fakes.headSha = 'sha-b' fakes.runsBySha['sha-b'] = [run('sha-b', 'Validate', 'failure')] await handleCiFailure(event('workflow_run', { sha: 'sha-b', suite: 22 })) const [second] = await drainWorkflows() assert.equal(second.action, 'skip') assert.equal(second.reason, 'agent-already-working') assert.equal(fakes.createAgentCalls, 1) const [episode] = await listEpisodes() assert.equal(episode.last_red_sha, 'sha-b') }) it('does nothing while the head is still running or already green', async () => { fakes.headSha = 'sha-b' fakes.runsBySha['sha-b'] = [run('sha-b', 'Validate', null)] const pending = (await ciRecheck({ repo: 'acme/widgets', branch: 'main', triggerSha: 'sha-a' })) as any assert.equal(pending.reason, 'head-pending') fakes.runsBySha['sha-b'] = [run('sha-b', 'Validate', 'success')] const green = (await ciRecheck({ repo: 'acme/widgets', branch: 'main', triggerSha: 'sha-a' })) as any assert.equal(green.reason, 'head-green') assert.equal(fakes.createAgentCalls, 0) assert.equal((await listEpisodes()).length, 0) }) it('resets when main goes green, then launches again on the next red', async () => { fakes.runsBySha['sha-a'] = [run('sha-a', 'Validate', 'failure')] await ciRecheck({ repo: 'acme/widgets', branch: 'main' }) assert.equal(fakes.createAgentCalls, 1) fakes.headSha = 'sha-fix' fakes.runsBySha['sha-fix'] = [run('sha-fix', 'Validate', 'success')] await handleCiFailure(event('workflow_run', { sha: 'sha-fix', suite: 31, conclusion: 'success' })) const [reset] = await drainWorkflows() assert.equal(reset.action, 'reset') assert.equal((await listEpisodes())[0].status, 'closed') fakes.headSha = 'sha-c' fakes.runsBySha['sha-c'] = [run('sha-c', 'Validate', 'timed_out')] const again = (await ciRecheck({ repo: 'acme/widgets', branch: 'main' })) as any assert.equal(again.action, 'launched') assert.equal(fakes.createAgentCalls, 2) }) it('a manually seeded episode blocks launches until green', async () => { const seeded = await markHandled({ repo: 'acme/widgets', branch: 'main', sha: 'sha-a', note: 'Manual Cursor agent already on it', }) assert.equal(seeded.claimed, true) assert.equal(seeded.episode?.source, 'manual') fakes.runsBySha['sha-a'] = [run('sha-a', 'Validate', 'failure')] const out = (await ciRecheck({ repo: 'acme/widgets', branch: 'main' })) as any assert.equal(out.reason, 'agent-already-working') assert.equal(fakes.createAgentCalls, 0) const again = await markHandled({ repo: 'acme/widgets', branch: 'main', note: 'second' }) assert.equal(again.claimed, false) assert.equal(again.episode?.note, 'Manual Cursor agent already on it') }) it('skips when an open PR already names the red SHA (best effort)', async () => { fakes.headSha = 'abcdef1234567890' fakes.runsBySha[fakes.headSha] = [run(fakes.headSha, 'Validate', 'failure')] fakes.pulls = [ { number: 7, title: 'fix: sync lockfile', body: 'Fixes main at abcdef1', html_url: 'https://github.com/acme/widgets/pull/7', head: { ref: 'cursor/x' } }, ] const out = (await ciRecheck({ repo: 'acme/widgets', branch: 'main' })) as any assert.equal(out.reason, 'pr-already-open') assert.equal(fakes.createAgentCalls, 0) assert.equal((await listEpisodes())[0].source, 'external-pr') }) it('releases the claim when the launch fails so a later event can retry', async () => { fakes.runsBySha['sha-a'] = [run('sha-a', 'Validate', 'failure')] fakes.createAgentFails = true const failed = (await ciRecheck({ repo: 'acme/widgets', branch: 'main' })) as any assert.equal(failed.action, 'error') assert.equal((await listEpisodes())[0].status, 'closed') fakes.createAgentFails = false const retried = (await ciRecheck({ repo: 'acme/widgets', branch: 'main' })) as any assert.equal(retried.action, 'launched') assert.equal(fakes.createAgentCalls, 1) }) it('concurrent re-checks for different suites still launch only once', async () => { fakes.runsBySha['sha-a'] = [run('sha-a', 'Validate', 'failure'), run('sha-a', 'Other', 'failure')] const results = (await Promise.all([ ciRecheck({ repo: 'acme/widgets', branch: 'main' }), ciRecheck({ repo: 'acme/widgets', branch: 'main' }), ciRecheck({ repo: 'acme/widgets', branch: 'main' }), ])) as any[] assert.equal(results.filter((r) => r.action === 'launched').length, 1) assert.equal(fakes.createAgentCalls, 1) }) it('dryRun reports would-launch without claiming; disabled skips', async () => { fakes.runsBySha['sha-a'] = [run('sha-a', 'Validate', 'failure')] const dry = (await ciRecheck({ repo: 'acme/widgets', branch: 'main', dryRun: true })) as any assert.equal(dry.action, 'would-launch') assert.equal((await listEpisodes()).length, 0) setConfig({ enabled: false }) const off = (await ciRecheck({ repo: 'acme/widgets', branch: 'main' })) as any assert.equal(off.reason, 'automation-disabled') assert.equal(fakes.createAgentCalls, 0) }) })