Skip to content
← Public packages

@kentcdodds/codebase-cleanup

Cursor cloud agents for legacy cleanup, docs freshness, tight tests, and prod CI fixes, plus a weekly Devin general audit.

FollowFork

Files

handle-ci-failure.test.ts

428 lines · 15.4 KB · TypeScript
import assert from 'node:assert/strict'
import { registerHooks } from 'node:module'
import { DatabaseSync } from 'node:sqlite'
import { beforeEach, describe, it } from 'node:test'

/**
 * Fakes for kody: imports so webhook → deferred re-check → runNow runs under
 * node:test. packageStorage() is a real in-memory SQLite (node:sqlite), so the
 * episode claim (INSERT … ON CONFLICT … WHERE … RETURNING) is exercised for real.
 * Nothing here can reach Cursor or GitHub.
 */
type Run = {
	id: number
	workflow_id: number
	name: string
	path: string
	event: string
	head_branch: string
	head_sha: string
	status: string
	conclusion: string | null
	run_attempt: number
	created_at: string
	html_url: string
}
type Fakes = {
	db: DatabaseSync
	createAgentCalls: number
	createAgentFails: boolean
	lastPrompt: string
	workflows: Map<string, { id: string; input: Record<string, any> }>
	headSha: string
	runsBySha: Record<string, Run[]>
	pulls: Array<Record<string, unknown>>
}
const fakes: Fakes = {
	db: new DatabaseSync(':memory:'),
	createAgentCalls: 0,
	createAgentFails: false,
	lastPrompt: '',
	workflows: new Map(),
	headSha: 'sha-a',
	runsBySha: {},
	pulls: [],
}
;(globalThis as Record<string, unknown>).__codebaseCleanupFakes = fakes

const fakeModules: Record<string, string> = {
	'kody:@kentcdodds/jev/evaluate': `
		export default async function evaluate() {
			throw new Error('prod-ci-failure must not call Jev')
		}
	`,
	'kody:runtime': `
		const f = globalThis.__codebaseCleanupFakes
		export function packageStorage() {
			return {
				async sql(query, b = []) {
					return { rows: f.db.prepare(query).all(...b).map((r) => ({ ...r })) }
				},
			}
		}
		export const workflows = {
			async create(input) {
				const existing = f.workflows.get(input.idempotencyKey)
				if (existing) return { id: existing.id, status: 'queued' }
				const id = 'pkgwf-' + (f.workflows.size + 1)
				f.workflows.set(input.idempotencyKey, { id, input })
				return { id, status: 'queued' }
			},
		}
	`,
	'kody:@kentcdodds/cursor/agents': `
		const f = globalThis.__codebaseCleanupFakes
		export async function createAgent(input) {
			if (f.createAgentFails) throw new Error('usage limit')
			f.createAgentCalls += 1
			f.lastPrompt = input?.prompt?.text ?? ''
			const id = 'agent-' + f.createAgentCalls
			return { agent: { id, url: 'https://cursor.com/agents/' + id } }
		}
	`,
	'kody:@kentcdodds/devin/request': `
		export async function devinRequest() {
			throw new Error('Devin must not be called by the CI path')
		}
		export async function resolveOrgId() {
			return 'org-test'
		}
	`,
	'kody:@kentcdodds/github/request': `
		const f = globalThis.__codebaseCleanupFakes
		export default async function githubRequest({ path }) {
			if (path.startsWith('/repos/acme/widgets/branches/')) {
				return { ok: true, status: 200, data: { commit: { sha: f.headSha } } }
			}
			if (path.startsWith('/repos/acme/widgets/actions/runs?head_sha=')) {
				const sha = decodeURIComponent(path.split('head_sha=')[1].split('&')[0])
				return { ok: true, status: 200, data: { workflow_runs: f.runsBySha[sha] ?? [] } }
			}
			if (path.startsWith('/repos/acme/widgets/pulls?')) {
				return { ok: true, status: 200, data: f.pulls }
			}
			if (path.startsWith('/repos/acme/widgets/commits?')) {
				return { ok: true, status: 200, data: [{ sha: 'x' }] }
			}
			return { ok: false, status: 404, text: 'unexpected path ' + path }
		}
	`,
}

registerHooks({
	resolve(specifier, context, nextResolve) {
		if (specifier in fakeModules) {
			return { url: `fake-kody:${specifier}`, shortCircuit: true }
		}
		return nextResolve(specifier, context)
	},
	load(url, context, nextLoad) {
		if (url.startsWith('fake-kody:')) {
			return {
				format: 'module',
				source: fakeModules[url.slice('fake-kody:'.length)],
				shortCircuit: true,
			}
		}
		return nextLoad(url, context)
	},
})

const { default: handleCiFailure } = await import('./handle-ci-failure.ts')
const { default: ciRecheck } = await import('./ci-recheck.ts')
const { markHandled, list: listEpisodes } = await import('./ci-episode.ts')

function setConfig(patch: Record<string, unknown> = {}) {
	fakes.db.exec('CREATE TABLE IF NOT EXISTS config (key TEXT PRIMARY KEY, value TEXT NOT NULL)')
	fakes.db
		.prepare('INSERT OR REPLACE INTO config (key, value) VALUES (?, ?)')
		.run(
			'automations',
			JSON.stringify({
				'prod-ci-failure': {
					enabled: true,
					repos: ['acme/widgets'],
					branch: 'main',
					...patch,
				},
			}),
		)
}

let nextRunId = 100
function run(sha: string, name: string, conclusion: string | null, extra: Partial<Run> = {}): Run {
	nextRunId += 1
	return {
		id: nextRunId,
		workflow_id: name === 'Validate' ? 1 : name === 'Report merge conflicts' ? 2 : 3,
		name,
		path: `.github/workflows/${name.toLowerCase().replace(/\s+/g, '-')}.yml`,
		event: 'push',
		head_branch: 'main',
		head_sha: sha,
		status: conclusion === null ? 'in_progress' : 'completed',
		conclusion,
		run_attempt: 1,
		created_at: '2026-10-22T06:57:42Z',
		html_url: `https://github.com/acme/widgets/actions/runs/${nextRunId}`,
		...extra,
	}
}

function event(
	kind: 'check_suite' | 'workflow_run' | 'check_run',
	opts: { sha?: string; suite?: number; conclusion?: string; branch?: string; repo?: string } = {},
) {
	const sha = opts.sha ?? 'sha-a'
	const suite = opts.suite ?? 11
	const conclusion = opts.conclusion ?? 'failure'
	const branch = opts.branch ?? 'main'
	const repository = { full_name: opts.repo ?? 'acme/widgets' }
	const json =
		kind === 'check_suite'
			? {
					action: 'completed',
					check_suite: { id: suite, conclusion, head_branch: branch, head_sha: sha },
					repository,
				}
			: kind === 'workflow_run'
				? {
						action: 'completed',
						workflow_run: {
							check_suite_id: suite,
							conclusion,
							head_branch: branch,
							head_sha: sha,
							name: 'Validate',
							html_url: 'https://github.com/acme/widgets/actions/runs/1',
						},
						repository,
					}
				: {
						action: 'completed',
						check_run: {
							conclusion,
							head_sha: sha,
							name: 'Static',
							check_suite: { id: suite, head_branch: branch, head_sha: sha },
						},
						repository,
					}
	return {
		request: {
			headers: { 'x-github-event': kind, 'x-github-delivery': `${kind}-${sha}-${suite}` },
			json,
		},
	}
}

/** Run every scheduled re-check the way the platform would after runAt. */
async function drainWorkflows() {
	const out = []
	for (const [key, wf] of [...fakes.workflows]) {
		if (wf.input.__ran) continue
		wf.input.__ran = true
		assert.equal(wf.input.exportName, './ci-recheck', key)
		out.push(await ciRecheck(wf.input.params))
	}
	return out as any[]
}

beforeEach(() => {
	fakes.db = new DatabaseSync(':memory:')
	fakes.createAgentCalls = 0
	fakes.createAgentFails = false
	fakes.lastPrompt = ''
	fakes.workflows.clear()
	fakes.headSha = 'sha-a'
	fakes.runsBySha = {}
	fakes.pulls = []
	setConfig()
})

describe('handleCiFailure (webhook) defers instead of launching', () => {
	it('schedules one ./ci-recheck ~60s out and never launches inline', async () => {
		const before = Date.now()
		const out = (await handleCiFailure(event('check_suite'))) as any
		assert.equal(out.scheduled, true)
		assert.equal(out.purpose, 'failure-recheck')
		assert.equal(fakes.createAgentCalls, 0)
		assert.equal(fakes.workflows.size, 1)
		const [wf] = [...fakes.workflows.values()]
		const delay = Date.parse(wf.input.runAt) - before
		assert.ok(delay >= 59_000 && delay <= 61_000, `delay ${delay}`)
		assert.deepEqual(
			{ repo: wf.input.params.repo, branch: wf.input.params.branch, sha: wf.input.params.triggerSha },
			{ repo: 'acme/widgets', branch: 'main', sha: 'sha-a' },
		)
	})

	it('collapses check_suite, workflow_run and check_run for one suite onto one re-check', async () => {
		const a = (await handleCiFailure(event('check_suite'))) as any
		const b = (await handleCiFailure(event('workflow_run'))) as any
		const c = (await handleCiFailure(event('check_run'))) as any
		assert.equal(a.idempotencyKey, b.idempotencyKey)
		assert.equal(b.idempotencyKey, c.idempotencyKey)
		assert.equal(fakes.workflows.size, 1)
	})

	it('honors recheckDelaySeconds from config', async () => {
		setConfig({ recheckDelaySeconds: 5 })
		const before = Date.now()
		await handleCiFailure(event('check_suite'))
		const [wf] = [...fakes.workflows.values()]
		const delay = Date.parse(wf.input.runAt) - before
		assert.ok(delay >= 4_000 && delay <= 6_000, `delay ${delay}`)
	})

	it('ignores success when no red episode is open, but re-checks while one is', async () => {
		const ignored = (await handleCiFailure(event('check_suite', { conclusion: 'success', suite: 12 }))) as any
		assert.equal(ignored.reason, 'not-a-failure-event')
		assert.equal(fakes.workflows.size, 0)

		await markHandled({ repo: 'acme/widgets', branch: 'main', sha: 'sha-a', note: 'manual' })
		const scheduled = (await handleCiFailure(event('check_suite', { conclusion: 'success', suite: 13 }))) as any
		assert.equal(scheduled.purpose, 'green-reset-check')
		assert.equal(fakes.workflows.size, 1)
	})

	it('ignores other repos, other branches, and a disabled automation', async () => {
		assert.equal(((await handleCiFailure(event('check_suite', { repo: 'acme/other' }))) as any).reason, 'repo-not-configured')
		assert.equal(((await handleCiFailure(event('check_suite', { branch: 'feature' }))) as any).reason, 'branch-mismatch')
		setConfig({ enabled: false })
		assert.equal(((await handleCiFailure(event('check_suite'))) as any).reason, 'automation-disabled')
		assert.equal(fakes.workflows.size, 0)
	})

	it('dryRun previews the re-check without scheduling it', async () => {
		const out = (await handleCiFailure({ ...event('check_suite'), dryRun: true })) as any
		assert.equal(out.scheduled, false)
		assert.equal(out.params.repo, 'acme/widgets')
		assert.equal(fakes.workflows.size, 0)
	})
})

describe('ciRecheck (deferred) launches at most one agent per red stretch', () => {
	it('launches once when the head is still red, and records the episode', async () => {
		fakes.runsBySha['sha-a'] = [run('sha-a', 'Validate', 'failure'), run('sha-a', 'Report merge conflicts', 'success')]
		await handleCiFailure(event('check_suite'))
		const [result] = await drainWorkflows()
		assert.equal(result.action, 'launched')
		assert.equal(fakes.createAgentCalls, 1)
		assert.match(fakes.lastPrompt, /Head SHA: `sha-a`/)
		const [episode] = await listEpisodes()
		assert.equal(episode.status, 'open')
		assert.equal(episode.source, 'automation')
		assert.equal(episode.agent_id, 'agent-1')
	})

	it('tonight: #4153 then b85e9e8 both red → one agent, not two', async () => {
		fakes.runsBySha['sha-a'] = [run('sha-a', 'Validate', 'failure')]
		await handleCiFailure(event('check_suite', { sha: 'sha-a', suite: 21 }))
		await drainWorkflows()
		fakes.headSha = 'sha-b'
		fakes.runsBySha['sha-b'] = [run('sha-b', 'Validate', 'failure')]
		await handleCiFailure(event('workflow_run', { sha: 'sha-b', suite: 22 }))
		const [second] = await drainWorkflows()
		assert.equal(second.action, 'skip')
		assert.equal(second.reason, 'agent-already-working')
		assert.equal(fakes.createAgentCalls, 1)
		const [episode] = await listEpisodes()
		assert.equal(episode.last_red_sha, 'sha-b')
	})

	it('does nothing while the head is still running or already green', async () => {
		fakes.headSha = 'sha-b'
		fakes.runsBySha['sha-b'] = [run('sha-b', 'Validate', null)]
		const pending = (await ciRecheck({ repo: 'acme/widgets', branch: 'main', triggerSha: 'sha-a' })) as any
		assert.equal(pending.reason, 'head-pending')
		fakes.runsBySha['sha-b'] = [run('sha-b', 'Validate', 'success')]
		const green = (await ciRecheck({ repo: 'acme/widgets', branch: 'main', triggerSha: 'sha-a' })) as any
		assert.equal(green.reason, 'head-green')
		assert.equal(fakes.createAgentCalls, 0)
		assert.equal((await listEpisodes()).length, 0)
	})

	it('resets when main goes green, then launches again on the next red', async () => {
		fakes.runsBySha['sha-a'] = [run('sha-a', 'Validate', 'failure')]
		await ciRecheck({ repo: 'acme/widgets', branch: 'main' })
		assert.equal(fakes.createAgentCalls, 1)

		fakes.headSha = 'sha-fix'
		fakes.runsBySha['sha-fix'] = [run('sha-fix', 'Validate', 'success')]
		await handleCiFailure(event('workflow_run', { sha: 'sha-fix', suite: 31, conclusion: 'success' }))
		const [reset] = await drainWorkflows()
		assert.equal(reset.action, 'reset')
		assert.equal((await listEpisodes())[0].status, 'closed')

		fakes.headSha = 'sha-c'
		fakes.runsBySha['sha-c'] = [run('sha-c', 'Validate', 'timed_out')]
		const again = (await ciRecheck({ repo: 'acme/widgets', branch: 'main' })) as any
		assert.equal(again.action, 'launched')
		assert.equal(fakes.createAgentCalls, 2)
	})

	it('a manually seeded episode blocks launches until green', async () => {
		const seeded = await markHandled({
			repo: 'acme/widgets',
			branch: 'main',
			sha: 'sha-a',
			note: 'Manual Cursor agent already on it',
		})
		assert.equal(seeded.claimed, true)
		assert.equal(seeded.episode?.source, 'manual')
		fakes.runsBySha['sha-a'] = [run('sha-a', 'Validate', 'failure')]
		const out = (await ciRecheck({ repo: 'acme/widgets', branch: 'main' })) as any
		assert.equal(out.reason, 'agent-already-working')
		assert.equal(fakes.createAgentCalls, 0)
		const again = await markHandled({ repo: 'acme/widgets', branch: 'main', note: 'second' })
		assert.equal(again.claimed, false)
		assert.equal(again.episode?.note, 'Manual Cursor agent already on it')
	})

	it('skips when an open PR already names the red SHA (best effort)', async () => {
		fakes.headSha = 'abcdef1234567890'
		fakes.runsBySha[fakes.headSha] = [run(fakes.headSha, 'Validate', 'failure')]
		fakes.pulls = [
			{ number: 7, title: 'fix: sync lockfile', body: 'Fixes main at abcdef1', html_url: 'https://github.com/acme/widgets/pull/7', head: { ref: 'cursor/x' } },
		]
		const out = (await ciRecheck({ repo: 'acme/widgets', branch: 'main' })) as any
		assert.equal(out.reason, 'pr-already-open')
		assert.equal(fakes.createAgentCalls, 0)
		assert.equal((await listEpisodes())[0].source, 'external-pr')
	})

	it('releases the claim when the launch fails so a later event can retry', async () => {
		fakes.runsBySha['sha-a'] = [run('sha-a', 'Validate', 'failure')]
		fakes.createAgentFails = true
		const failed = (await ciRecheck({ repo: 'acme/widgets', branch: 'main' })) as any
		assert.equal(failed.action, 'error')
		assert.equal((await listEpisodes())[0].status, 'closed')
		fakes.createAgentFails = false
		const retried = (await ciRecheck({ repo: 'acme/widgets', branch: 'main' })) as any
		assert.equal(retried.action, 'launched')
		assert.equal(fakes.createAgentCalls, 1)
	})

	it('concurrent re-checks for different suites still launch only once', async () => {
		fakes.runsBySha['sha-a'] = [run('sha-a', 'Validate', 'failure'), run('sha-a', 'Other', 'failure')]
		const results = (await Promise.all([
			ciRecheck({ repo: 'acme/widgets', branch: 'main' }),
			ciRecheck({ repo: 'acme/widgets', branch: 'main' }),
			ciRecheck({ repo: 'acme/widgets', branch: 'main' }),
		])) as any[]
		assert.equal(results.filter((r) => r.action === 'launched').length, 1)
		assert.equal(fakes.createAgentCalls, 1)
	})

	it('dryRun reports would-launch without claiming; disabled skips', async () => {
		fakes.runsBySha['sha-a'] = [run('sha-a', 'Validate', 'failure')]
		const dry = (await ciRecheck({ repo: 'acme/widgets', branch: 'main', dryRun: true })) as any
		assert.equal(dry.action, 'would-launch')
		assert.equal((await listEpisodes()).length, 0)
		setConfig({ enabled: false })
		const off = (await ciRecheck({ repo: 'acme/widgets', branch: 'main' })) as any
		assert.equal(off.reason, 'automation-disabled')
		assert.equal(fakes.createAgentCalls, 0)
	})
})