Skip to content

src/classify.test.ts

105 lines Ā· 5.8 KB Ā· TypeScript
Raw
import assert from 'node:assert/strict'
import { test } from 'node:test'
import {
	classifyReviewBotComment,
	classifyReviewBotComments,
	findingHasAddressingReply,
	hasInvalidReply,
	invalidReplyBody,
	isAddressingReviewReply,
	isShipPrBlocker,
	reviewBotKind,
	severityOf,
	findingTitle,
} from './classify.ts'

test('review-bot sort maps Bugbot, Devin, and Seer; unsure stays a blocker', () => {
	assert.equal(reviewBotKind('cursor[bot]'), 'bugbot')
	assert.equal(reviewBotKind('devin-ai-integration[bot]'), 'devin')
	assert.equal(reviewBotKind('seer[bot]'), 'seer')
	assert.equal(reviewBotKind('coderabbitai[bot]'), null)

	const valid = classifyReviewBotComment({ id: 1, user: { login: 'cursor[bot]' }, body: 'This is a bug: null pointer when the session is missing.' })!
	assert.equal(valid.verdict, 'valid')
	assert.equal(isShipPrBlocker(valid), true)

	const invalid = classifyReviewBotComment({ id: 2, user: { login: 'devin-ai-integration[bot]' }, body: 'Nit: rename this variable for style-only consistency.' })!
	assert.equal(invalid.verdict, 'invalid')
	assert.equal(isShipPrBlocker(invalid), false)
	assert.match(invalidReplyBody(invalid), /review-bot sort/)

	const optionalParameterBug = classifyReviewBotComment({ id: 4, user: { login: 'cursor[bot]' }, body: 'Bug: the optional `timeout` parameter is ignored; requests never time out.' })!
	assert.equal(optionalParameterBug.verdict, 'valid')

	const optionalFix = classifyReviewBotComment({ id: 5, user: { login: 'devin-ai-integration[bot]' }, body: 'This is an optional suggestion: rename for consistency.' })!
	assert.equal(optionalFix.verdict, 'invalid')

	const detailsWrappedBug = classifyReviewBotComment({
		id: 6,
		user: { login: 'devin-ai-integration[bot]' },
		body: [
			'Optional parameters hide real review findings',
			'',
			'When a bot reports a bug involving an optional parameter, classification drops it.',
			'<details><summary>Learn more</summary>',
			'Require explicit language that the finding itself is a nit or optional.',
			'</details>',
		].join('\n'),
	})!
	assert.equal(detailsWrappedBug.verdict, 'valid')

	const mixed = classifyReviewBotComment({ id: 7, user: { login: 'cursor[bot]' }, body: 'This is not a nit: the security bug exposes credentials.' })!
	assert.equal(mixed.verdict, 'unsure')
	assert.equal(isShipPrBlocker(mixed), true)

	const unsure = classifyReviewBotComment({ id: 3, user: { login: 'seer[bot]' }, body: 'Consider whether this helper belongs closer to the call site.' })!
	assert.equal(unsure.verdict, 'unsure')
	assert.equal(isShipPrBlocker(unsure), true)

	const findings = classifyReviewBotComments([
		{ id: 11, user: { login: 'cursor[bot]' }, body: 'Critical security issue: credential logged in plaintext.' },
		{ id: 12, user: { login: 'devin-ai-integration[bot]' }, body: 'False positive - not a bug in this path.' },
		{ id: 13, user: { login: 'seer[bot]' }, body: 'Maybe revisit the naming here.' },
		{ id: 14, user: { login: 'coderabbitai[bot]' }, body: 'This is a bug that should be fixed.' },
	])
	assert.equal(findings.length, 3)
	assert.equal(findings.filter(isShipPrBlocker).length, 2)
	assert.equal(findings.filter((f) => f.verdict === 'invalid').length, 1)
})

test('addressing replies: kody-bot / humans citing a commit or wontfix', () => {
	const reply = (id: number, login: string, body: string) => ({ id, user: { login }, body, in_reply_to_id: 1 })
	assert.equal(isAddressingReviewReply(reply(21, 'kody-bot', 'Fixed in abcdef1 — allowlisted.')), true)
	assert.equal(isAddressingReviewReply(reply(22, 'kentcdodds', 'Intentional wontfix; leaving the fallback as-is.')), true)
	assert.equal(isAddressingReviewReply(reply(23, 'cursor[bot]', 'Still a bug: fixed in abcdef1 would be nicer.')), false)
	assert.equal(isAddressingReviewReply(reply(24, 'kody-bot', 'Looking at this now.')), false)
	assert.equal(isAddressingReviewReply(reply(25, 'kody-bot', 'Fixed in a follow-up.')), false)
	assert.equal(isAddressingReviewReply(reply(26, 'kody-bot', 'abcdef1')), false)
	assert.equal(isAddressingReviewReply(reply(27, 'kody-bot', 'This is not fixed in abcdef1.')), false)
	assert.equal(isAddressingReviewReply(reply(29, 'kody-bot', "This isn't fixed in abcdef1.")), false)
	assert.equal(isAddressingReviewReply(reply(28, 'dependabot[bot]', 'Fixed in abcdef1.')), false)
	assert.equal(isAddressingReviewReply(reply(30, 'coderabbitai[bot]', 'Fixed in abcdef1.')), false)
	assert.equal(
		findingHasAddressingReply({ commentId: 1 }, [
			{ id: 1, user: { login: 'cursor[bot]' }, body: 'bug' },
			{ id: 21, user: { login: 'kody-bot' }, body: 'Addressed in abcdef1.', in_reply_to_id: 1 },
		]),
		true,
	)
	assert.equal(findingHasAddressingReply({ commentId: 3 }, [{ id: 3, user: { login: 'seer[bot]' }, body: 'x' }]), false)
})

test('invalid marker matches old in-repo and new package wording', () => {
	const old = 'Marked **invalid** for ship-pr by the in-repo review-bot sort (Bugbot: nit).'
	assert.equal(hasInvalidReply([{ id: 9, user: { login: 'kody-bot' }, body: old, in_reply_to_id: 1 }], 1), true)
	const fresh = invalidReplyBody({ bot: 'seer', authorLogin: 'seer[bot]', commentId: 1, bodyPreview: '', verdict: 'invalid', reason: 'nit' })
	assert.equal(hasInvalidReply([{ id: 9, user: { login: 'kody-bot' }, body: fresh, in_reply_to_id: 1 }], 1), true)
})

test('severity + title extraction', () => {
	assert.equal(severityOf('### Node builtins\n\n**Medium Severity**\n\n<!-- DESCRIPTION START -->'), 'medium')
	assert.equal(severityOf('_šŸŽÆ Functional Correctness_ | _🟔 Minor_ | _⚔ Quick win_'), 'minor')
	assert.equal(severityOf('šŸ”“ **Reclaimed slug mixes two package pages**'), 'high')
	assert.equal(findingTitle('### Node builtins treated as undeclared packages\n\n**Medium Severity**'), 'Node builtins treated as undeclared packages')
	assert.equal(findingTitle('<!-- x -->\n\nšŸ”“ **Reclaimed slug mixes two package pages**\n\nbody'), 'Reclaimed slug mixes two package pages')
})