Skip to content
← Public packages

@kentcdodds/devin

Start, monitor, and manage Devin sessions, knowledge, playbooks, and schedules via the Devin v3 API

FollowFork

Files

attention.test.ts

321 lines · 11.0 KB · TypeScript
import assert from 'node:assert/strict'
import { describe, it } from 'node:test'
import {
	classifySession,
	cleanMessage,
	compareScanOrder,
	decodeCursor,
	encodeCursor,
	extractQuestions,
	findCommitInJson,
	isAfterKey,
	sameCommit,
	trimMiddle,
	type AttentionMessage,
	type AttentionSessionInput,
} from './lib/attention.ts'

const NOW = Date.UTC(2026, 0, 10, 12, 0, 0)
const sec = (hoursAgo: number) => Math.floor(NOW / 1000 - hoursAgo * 3600)

function session(over: Partial<AttentionSessionInput> = {}): AttentionSessionInput {
	return {
		session_id: 'devin-1',
		title: 'Example task',
		url: 'https://app.devin.ai/sessions/devin-1',
		status: 'suspended',
		status_detail: 'inactivity',
		is_archived: false,
		created_at: sec(48),
		updated_at: sec(2),
		pull_requests: [],
		...over,
	}
}

function msgs(...items: Array<[string, string, number?]>): AttentionMessage[] {
	return items.map(([source, message, hoursAgo], i) => ({
		source,
		message,
		created_at: sec(hoursAgo ?? 10 - i),
	}))
}

const merged = [{ pr_url: 'https://github.com/acme/app/pull/12', pr_state: 'merged' }]
const open = [{ pr_url: 'https://github.com/acme/app/pull/13', pr_state: 'open' }]
const opts = { now: NOW }

describe('classifySession', () => {
	it('flags waiting_for_user status as awaiting_user', () => {
		const c = classifySession(
			session({ status: 'running', status_detail: 'waiting_for_user' }),
			msgs(['user', 'Do the thing'], ['devin', 'Working on it.']),
			opts,
		)
		assert.equal(c.reason, 'awaiting_user')
		assert.equal(c.suggestedAction, 'reply')
		assert.equal(c.detection, 'status')
	})

	it('flags a trailing question from Devin as awaiting_user', () => {
		const c = classifySession(
			session({ pull_requests: open }),
			msgs(['user', 'Fix the bug'], ['devin', 'I found two failing tests.\n\nShould I also update the snapshot files?']),
			opts,
		)
		assert.equal(c.reason, 'awaiting_user')
		assert.deepEqual(c.questions, ['Should I also update the snapshot files?'])
	})

	it('flags an explicit ask without a question mark', () => {
		const c = classifySession(
			session(),
			msgs(['devin', "The plan is posted. Nothing is built yet; I'm waiting for the owner's answers on gaps 2 and 3."]),
			opts,
		)
		assert.equal(c.reason, 'awaiting_user')
		assert.equal(c.detection, 'message')
	})

	it('ignores boilerplate questions', () => {
		const c = classifySession(
			session({ pull_requests: open }),
			msgs(['devin', 'PR is up. Anything else?']),
			opts,
		)
		assert.equal(c.reason, null)
		assert.deepEqual(c.questions, [])
	})

	it('does not flag when the user spoke last', () => {
		const c = classifySession(
			session({ pull_requests: open }),
			msgs(['devin', 'Which option do you want?'], ['user', 'Option A please']),
			opts,
		)
		assert.equal(c.reason, null)
	})

	it('flags decision keywords as needs_human_decision with the configurable action', () => {
		const c = classifySession(
			session({ pull_requests: merged }),
			msgs(['devin', 'Merged. Decision for you: keep the threshold at 2, or raise it to 3? I recommend 2.']),
			{ ...opts, decisionAction: 'ping_custom' },
		)
		assert.equal(c.reason, 'needs_human_decision')
		assert.equal(c.suggestedAction, 'ping_custom')
	})

	it('defaults the decision action to ping_owner', () => {
		const c = classifySession(
			session(),
			msgs(['devin', '1. Option A: ship now.\n2. Option B: wait for review.']),
			opts,
		)
		assert.equal(c.reason, 'needs_human_decision')
		assert.equal(c.suggestedAction, 'ping_owner')
	})

	it('matches owner names and custom keywords', () => {
		const text = 'Gap 4 still needs Sam before I start.'
		assert.equal(classifySession(session(), msgs(['devin', text]), opts).reason, null)
		assert.equal(
			classifySession(session(), msgs(['devin', text]), { ...opts, ownerNames: ['Sam'] }).reason,
			'needs_human_decision',
		)
		assert.equal(
			classifySession(session(), msgs(['devin', 'This is labeled triage-hold.']), {
				...opts,
				decisionKeywords: ['triage-hold'],
			}).reason,
			'needs_human_decision',
		)
	})

	it('ignores negated decision phrases', () => {
		const c = classifySession(
			session({ pull_requests: merged }),
			msgs(['devin', 'Shipped and verified in production. Nothing needs your decision. Nothing else needs Sam.']),
			{ ...opts, ownerNames: ['Sam'] },
		)
		assert.equal(c.reason, 'stale')
	})

	it('ignores conditional, acknowledged, and hypothetical decision phrases', () => {
		const texts = [
			"Shipping #12 now; I'll wake the reviewer when it merges or if anything needs Sam.",
			'Going with option A: I will rework the slot.',
			'Option A merged as #40.',
			'Adding it would have been a new product decision.',
			'No product decision is needed.',
			'#43 is still labeled agent-ready, with no needs-sam, and I left it that way.',
			'New totals are 24 needs-sam and 102 agent-ready.',
			'Done: #44 now records your call to leave it out.',
			'Testing it on a preview would need Sam to add that secret.',
		]
		for (const text of texts) {
			const c = classifySession(session(), msgs(['devin', text]), { ...opts, ownerNames: ['Sam'] })
			assert.equal(c.reason, null, text)
		}
	})

	it('uses only the final Devin message for decisions', () => {
		const c = classifySession(
			session({ pull_requests: merged }),
			msgs(['user', 'Go'], ['devin', 'Which would you prefer, A or B?'], ['devin', 'Done. Everything shipped.']),
			opts,
		)
		assert.equal(c.reason, 'stale')
	})

	it('records the evidence sentence', () => {
		const c = classifySession(session(), msgs(['devin', 'Report attached. The fix needs your decision.']), opts)
		assert.equal(c.reason, 'needs_human_decision')
		assert.equal(c.evidence, 'The fix needs your decision.')
	})

	it('flags waiting_for_approval status as approval_pending uiOnly', () => {
		const c = classifySession(
			session({ status: 'running', status_detail: 'waiting_for_approval' }),
			msgs(['devin', 'About to run the migration.']),
			opts,
		)
		assert.equal(c.reason, 'approval_pending')
		assert.equal(c.uiOnly, true)
		assert.equal(c.detection, 'status')
	})

	it('flags messages that point at a setup suggestion card', () => {
		const c = classifySession(
			session(),
			msgs(['devin', 'I suggested an environment setup change to add chromium. Please approve the suggestion in the session.']),
			opts,
		)
		assert.equal(c.reason, 'approval_pending')
		assert.equal(c.uiOnly, true)
		assert.equal(c.detection, 'message')
	})

	it('flags merged PRs still waiting on a deploy check as deploy_stuck', () => {
		const c = classifySession(
			session({ pull_requests: merged }),
			msgs(['devin', 'The PR merged as abc1234. The deploy has not finished yet, so the health check still serves the previous commit.']),
			opts,
		)
		assert.equal(c.reason, 'deploy_stuck')
		assert.equal(c.suggestedAction, 'check_deploy')
	})

	it('ignores deploy words that are not a pending deploy', () => {
		const texts = [
			"Main's checks failed twice, so nothing deployed until I re-ran them.",
			"I haven't measured searches since the deploy; it's too soon.",
			'| About 2h after the #10 deploy, until #11 deployed | 0 | 0 |',
			'If an enabled job is still there, the migration fails and the deploy stops.',
		]
		for (const text of texts) {
			const c = classifySession(session({ pull_requests: merged }), msgs(['devin', text]), opts)
			assert.equal(c.reason, 'stale', text)
		}
	})

	it('does not flag deploy_stuck when the deploy is confirmed', () => {
		const c = classifySession(
			session({ pull_requests: merged }),
			msgs(['devin', '/health serves 1c45203b3, which contains the merge commit. The ship run is finished. Nothing is pending from me.']),
			opts,
		)
		assert.equal(c.reason, 'stale')
	})

	it('flags finished sessions with merged or closed PRs as stale', () => {
		const c = classifySession(
			session({
				pull_requests: [
					...merged,
					{ pr_url: 'https://github.com/acme/app/pull/14', pr_state: 'closed' },
				],
			}),
			msgs(['devin', 'Done. Both PRs are settled.']),
			opts,
		)
		assert.equal(c.reason, 'stale')
		assert.equal(c.suggestedAction, 'archive')
	})

	it('respects includeStale: false and archived sessions', () => {
		const s = session({ pull_requests: merged })
		const m = msgs(['devin', 'Done.'])
		assert.equal(classifySession(s, m, { ...opts, includeStale: false }).reason, null)
		assert.equal(classifySession({ ...s, is_archived: true }, m, opts).reason, null)
	})

	it('is not stale with an open PR or no PRs', () => {
		assert.equal(classifySession(session({ pull_requests: open }), msgs(['devin', 'Done.']), opts).reason, null)
		assert.equal(classifySession(session(), msgs(['devin', 'Done.']), opts).reason, null)
	})

	it('skips actively working sessions', () => {
		const c = classifySession(
			session({ status: 'running', status_detail: 'working', pull_requests: merged }),
			msgs(['devin', 'Should I keep going?']),
			opts,
		)
		assert.equal(c.reason, null)
		assert.equal(c.activelyWorking, true)
	})

	it('computes ageHours from the last message and trims the message', () => {
		const long = `Start ${'x'.repeat(2000)} final ask?`
		const c = classifySession(session(), msgs(['devin', long, 5]), opts)
		assert.equal(c.ageHours, 5)
		assert.ok(c.lastDevinMessage!.length <= 600)
		assert.ok(c.lastDevinMessage!.endsWith('final ask?'))
	})

	it('prefers PR states passed in (for example from GitHub)', () => {
		const c = classifySession(
			session({ pull_requests: [{ pr_url: 'https://github.com/acme/app/pull/12', pr_state: null }] }),
			msgs(['devin', 'Done.']),
			{ ...opts, prs: [{ url: 'https://github.com/acme/app/pull/12', state: 'merged', mergeCommit: 'abc1234' }] },
		)
		assert.equal(c.reason, 'stale')
		assert.equal(c.prs[0]!.mergeCommit, 'abc1234')
	})
})

describe('helpers', () => {
	it('strips attachment lines', () => {
		assert.equal(cleanMessage('Report ready.\n\nATTACHMENT:{"url":"https://x/y.json"}'), 'Report ready.')
	})

	it('extracts questions', () => {
		assert.deepEqual(extractQuestions('Done. Do you want A or B? Does that make sense?'), ['Do you want A or B?'])
	})

	it('trims the middle', () => {
		assert.equal(trimMiddle('short', 10), 'short')
		assert.equal(trimMiddle('a'.repeat(50), 20).length, 20)
	})

	it('finds a commit in health JSON', () => {
		assert.equal(findCommitInJson({ ok: true, build: { commitSha: 'ABCDEF1234' } }), 'abcdef1234')
		assert.equal(findCommitInJson({ ok: true }), null)
		assert.ok(sameCommit('abcdef1234567', 'abcdef1'))
		assert.ok(!sameCommit('abcdef1', '1234567'))
	})

	it('round-trips the cursor and orders scans stably', () => {
		const cursor = { v: 1 as const, since: 100, includeArchived: false, after: [50, 'b'] as [number, string], scanned: 3 }
		assert.deepEqual(decodeCursor(encodeCursor(cursor)), cursor)
		assert.throws(() => decodeCursor('not-a-cursor'))
		const list = [
			{ session_id: 'a', created_at: 50 },
			{ session_id: 'c', created_at: 60 },
			{ session_id: 'b', created_at: 50 },
			{ session_id: 'd', created_at: 40 },
		].sort(compareScanOrder)
		assert.deepEqual(list.map((s) => s.session_id), ['c', 'b', 'a', 'd'])
		assert.deepEqual(list.filter((s) => isAfterKey(s, [50, 'b'])).map((s) => s.session_id), ['a', 'd'])
	})
})