Skip to content
← Public packages

@kentcdodds/codebase-cleanup

Cursor cloud agents for legacy cleanup, docs freshness, tight tests, and prod CI fixes, plus a weekly Devin general audit.

FollowFork

Files

general-audit.test.ts

211 lines · 8.3 KB · TypeScript
import assert from 'node:assert/strict'
import { registerHooks } from 'node:module'
import { DatabaseSync } from 'node:sqlite'
import { beforeEach, describe, it } from 'node:test'

/**
 * general-audit runs through runNow with fake kody: imports. packageStorage()
 * is in-memory SQLite; the Devin request helper is a recorder, so nothing can
 * reach Devin, Cursor, or GitHub.
 */
type DevinCall = { path: string; method?: string; body?: Record<string, unknown> }
const fakes = {
	db: new DatabaseSync(':memory:'),
	devinCalls: [] as DevinCall[],
	devinStatus: 200,
	orgId: 'org-test' as string | null,
}
;(globalThis as Record<string, unknown>).__generalAuditFakes = fakes

const fakeModules: Record<string, string> = {
	'kody:@kentcdodds/jev/evaluate': `
		export default async function evaluate() {
			throw new Error('general-audit must not call Jev')
		}
	`,
	'kody:runtime': `
		const f = globalThis.__generalAuditFakes
		export function packageStorage() {
			return {
				async sql(query, b = []) {
					return { rows: f.db.prepare(query).all(...b).map((r) => ({ ...r })) }
				},
			}
		}
	`,
	'kody:@kentcdodds/cursor/agents': `
		export async function createAgent() {
			throw new Error('general-audit must not launch a Cursor agent')
		}
	`,
	'kody:@kentcdodds/github/request': `
		export default async function githubRequest() {
			throw new Error('general-audit must not call GitHub')
		}
	`,
	'kody:@kentcdodds/devin/request': `
		const f = globalThis.__generalAuditFakes
		export async function resolveOrgId() {
			if (!f.orgId) throw new Error('No Devin org id available')
			return f.orgId
		}
		export async function devinRequest(input) {
			f.devinCalls.push(JSON.parse(JSON.stringify(input)))
			if (f.devinStatus !== 200) return { status: f.devinStatus, body: { detail: 'devin_mode not available' } }
			return {
				status: 200,
				body: { session_id: 'devin-1', url: 'https://app.devin.ai/sessions/1', devin_mode: input.body.devin_mode },
			}
		}
	`,
}

registerHooks({
	resolve(specifier, context, nextResolve) {
		if (specifier in fakeModules) return { url: `fake-kody:${specifier}`, shortCircuit: true }
		return nextResolve(specifier, context)
	},
	load(url, context, nextLoad) {
		if (url.startsWith('fake-kody:')) {
			return { format: 'module', source: fakeModules[url.slice('fake-kody:'.length)], shortCircuit: true }
		}
		return nextLoad(url, context)
	},
})

const { default: runNow } = await import('./run.ts')
const { default: generalAuditJob } = await import('./jobs/general-audit.ts')
const { get, set } = await import('./config.ts')

function storeConfig(patch: Record<string, unknown>, key = 'general-audit') {
	fakes.db.exec('CREATE TABLE IF NOT EXISTS config (key TEXT PRIMARY KEY, value TEXT NOT NULL)')
	fakes.db
		.prepare('INSERT OR REPLACE INTO config (key, value) VALUES (?, ?)')
		.run('automations', JSON.stringify({ [key]: { repos: ['acme/widgets'], ...patch } }))
}

function storedAutomationKeys() {
	const row = fakes.db.prepare('SELECT value FROM config WHERE key = ?').get('automations') as { value: string }
	return Object.keys(JSON.parse(row.value))
}

function launches() {
	return fakes.db.prepare('SELECT automation, agent_id, dry_run, skipped FROM launches ORDER BY id').all().map((r) => ({ ...r }))
}

beforeEach(() => {
	fakes.db.exec('DROP TABLE IF EXISTS config; DROP TABLE IF EXISTS launches')
	fakes.devinCalls = []
	fakes.devinStatus = 200
	fakes.orgId = 'org-test'
})

describe('general-audit launcher', () => {
	it('dryRun resolves the prompt and the Fusion create-session request without sending it', async () => {
		storeConfig({ enabled: false, notify: 'kody:@acme/chat/ping-me' })
		const out = (await runNow({ automation: 'general-audit', dryRun: true })) as any
		assert.equal(fakes.devinCalls.length, 0)
		const [result] = out.results
		assert.equal(result.launcher, 'devin')
		assert.equal(result.wouldSpawn, true)
		assert.equal(result.devinOrgConfigured, true)
		assert.equal(result.devinRequest.method, 'POST')
		assert.equal(result.devinRequest.path, '/v3/organizations/{org_id}/sessions')
		assert.equal(result.devinRequest.body.devin_mode, 'fusion')
		assert.deepEqual(result.devinRequest.body.repos, ['acme/widgets'])
		assert.equal(result.devinRequest.body.prompt, result.prompt)
		assert.equal('max_acu_limit' in result.devinRequest.body, false)
		assert.match(result.prompt, /Weekly general audit/)
		assert.match(result.prompt, /kody:@acme\/chat\/ping-me/)
	})

	it('dryRun reports a missing Devin org id instead of throwing', async () => {
		storeConfig({ enabled: true })
		fakes.orgId = null
		const out = (await runNow({ automation: 'general-audit', dryRun: true })) as any
		assert.equal(out.results[0].devinOrgConfigured, false)
	})

	it('the job does nothing while the automation is disabled', async () => {
		storeConfig({ enabled: false })
		const out = (await generalAuditJob()) as any
		assert.equal(out.skipped, 'automation-disabled')
		assert.equal(fakes.devinCalls.length, 0)
	})

	it('the job sends the dryRun request unchanged and records the session', async () => {
		storeConfig({ enabled: true, maxAcuLimit: 40 })
		const preview = (await runNow({ automation: 'general-audit', dryRun: true })) as any
		const out = (await generalAuditJob()) as any
		assert.equal(fakes.devinCalls.length, 1)
		assert.deepEqual(fakes.devinCalls[0], {
			path: preview.results[0].devinRequest.path,
			method: 'POST',
			body: preview.results[0].devinRequest.body,
		})
		assert.equal(fakes.devinCalls[0].body?.max_acu_limit, 40)
		assert.equal(out.results[0].agentId, 'devin-1')
		assert.equal(out.results[0].devinMode, 'fusion')
		assert.deepEqual(launches().at(-1), {
			automation: 'general-audit',
			agent_id: 'devin-1',
			dry_run: 0,
			skipped: null,
		})
	})

	it('a rejected create-session is reported and recorded, not thrown', async () => {
		storeConfig({ enabled: true })
		fakes.devinStatus = 403
		const out = (await generalAuditJob()) as any
		assert.equal(out.results[0].ok, false)
		assert.match(out.results[0].error, /403.*devin_mode not available/)
		assert.match(String((launches().at(-1) as any).skipped), /^spawn-failed:/)
	})

	it('config.set validates devinMode', async () => {
		storeConfig({})
		await assert.rejects(() => set({ automation: 'general-audit', devinMode: 'turbo' as any }), /Unknown devinMode/)
		await set({ automation: 'general-audit', devinMode: 'ultra' })
		const out = (await runNow({ automation: 'general-audit', dryRun: true })) as any
		assert.equal(out.results[0].devinRequest.body.devin_mode, 'ultra')
	})

	it('launches exactly one Devin session per run that covers all four lanes', async () => {
		storeConfig({ enabled: true, notify: 'kody:@acme/chat/ping-me' })
		const out = (await generalAuditJob()) as any
		assert.equal(fakes.devinCalls.length, 1)
		assert.equal(out.results.length, 1)
		const body = fakes.devinCalls[0].body as any
		assert.deepEqual(body.repos, ['acme/widgets'])
		assert.deepEqual(body.tags, ['codebase-cleanup', 'general-audit'])
		assert.match(body.title, /^general-audit acme\/widgets/)
		const order = ['## Lane 1: Security', '## Lane 2: Performance', '## Lane 3: Accessibility', '## Lane 4: Code health']
			.map((heading) => body.prompt.indexOf(heading))
		assert.ok(order.every((i: number) => i >= 0))
		assert.deepEqual([...order].sort((a, b) => a - b), order)
		assert.equal(launches().filter((r: any) => r.dry_run === 0).length, 1)
	})

	it('carries config stored under the old security-audit key over to general-audit', async () => {
		storeConfig(
			{ enabled: true, notify: 'kody:@acme/chat/ping-me', schedule: '0 3 * * 1', maxAcuLimit: null, devinMode: 'fusion' },
			'security-audit',
		)
		const before = (await get({ automation: 'general-audit' })) as any
		assert.equal(before.enabled, true)
		assert.deepEqual(before.repos, ['acme/widgets'])
		assert.equal(before.notify, 'kody:@acme/chat/ping-me')
		assert.equal(before.schedule, '0 3 * * 1')
		assert.equal(before.maxAcuLimit, null)
		assert.equal(before.devinMode, 'fusion')
		const out = (await generalAuditJob()) as any
		assert.equal(out.results[0].agentId, 'devin-1')
		assert.equal(fakes.devinCalls.length, 1)
		await set({ automation: 'general-audit', branch: 'main' })
		assert.deepEqual(storedAutomationKeys().includes('security-audit'), false)
		assert.deepEqual(storedAutomationKeys().includes('general-audit'), true)
		assert.equal(((await get({ automation: 'general-audit' })) as any).enabled, true)
	})
})