Skip to content
← Public packages

@kentcdodds/sentry-triage

Sentry triage wakes Cole (Grok Bot) per issue: one active wake per repo (lease + queue), loop-safe Discord. Cole may spawn Cursor for isolated repo work.

src/handle-seer-webhook.ts

645 lines · 18.5 KB · TypeScript
import { packageStorage } from 'kody:runtime'
import {
	postSentryReport,
	editSentryReport,
	upsertSentryReport,
} from './format-discord-report.ts'
import getIssue from 'kody:@kentcdodds/sentry/get-issue'
import { buildTriageAgentPrompt } from './agent-prompt.ts'
import { buildShipPrPrompt } from './ship-pr-prompt.ts'
import { gatherSpawnContext } from './spawn-snapshot.ts'
import {
	acquireRepoLease,
	bumpCounter,
	claimIssueOnce,
	wakeCole,
	enqueueRepoIssue,
	releaseRepoLease,
	seerRunsToPr,
	updateRepoLeaseHolder,
} from './triage-core.ts'
import {
	alertSentKey,
	hourBucket,
	issueAnomalyThresholdPerHour,
	issueStorageKey,
	issuesSeenCounterKey,
	maxAgentsPerHour,
	spawnedCounterKey,
	truncate,
} from './shared.ts'
import {
	promptOwnerFields,
	readTriageSettings,
	resolveTriageProjectForSlug,
} from './settings.ts'

/**
 * Resolve the triage project + a minimal issue shape from a Seer webhook,
 * whose payload carries only `group_id` (the numeric Sentry issue id). We
 * look the issue up once (via @kentcdodds/sentry) to get its project slug,
 * title, culprit, and link.
 */
async function resolveIssueForGroup(groupId) {
	try {
		const data = await getIssue({ issueId: groupId })
		const slug = data?.project?.slug ?? null
		return {
			project: await resolveTriageProjectForSlug(slug),
			issue: {
				id: String(groupId),
				shortId: data?.shortId ?? null,
				title: data?.title ?? 'Unknown issue',
				culprit: data?.culprit ?? null,
				level: data?.level ?? 'error',
				permalink:
					data?.permalink ??
					`https://${(await readTriageSettings()).sentryOrgSlug}.sentry.io/issues/${groupId}/`,
				project: slug ? { slug } : null,
			},
		}
	} catch {
		return { issue: null, project: null }
	}
}


function normalizeRootCause(rawRootCause, runId) {
	if (!rawRootCause || typeof rawRootCause !== 'object') return null
	return {
		one_line_description: rawRootCause.one_line_description ?? null,
		five_whys: Array.isArray(rawRootCause.five_whys)
			? rawRootCause.five_whys
			: [],
		reproduction_steps: Array.isArray(rawRootCause.reproduction_steps)
			? rawRootCause.reproduction_steps
			: [],
		relevant_repo: rawRootCause.relevant_repo ?? null,
		fixability: rawRootCause.fixability ?? null,
		runId: runId ?? null,
	}
}

/**
 * Consume a Seer autofix lifecycle webhook (`sentry-hook-resource: seer`).
 *
 * The valuable event is `root_cause_completed`: Sentry has already done the
 * investigation, so we wake Cole seeded with the RCA as a verified
 * hypothesis (skipping the expensive open-ended investigation phase). We also
 * annotate the issue's Discord message on `pr_created` so a Seer-authored PR
 * is reviewed rather than duplicated. Other lifecycle events are acked.
 *
 * Shares the issue record, claim table, and hourly counters with the issue
 * handler so caps and dedupe hold across both ingress paths.
 */
export default async function handleSeerWebhook(input = {}) {
	const storage = packageStorage()
	const payload = input?.request?.json ?? {}
	const dryRun = payload?.kodyDryRun === true
	const action = payload?.action ?? null
	const data = payload?.data ?? {}
	const groupId = data?.group_id != null ? String(data.group_id) : null
	const runId = data?.run_id ?? null

	if (!groupId) {
		return { ok: true, skipped: 'no-group-id', action }
	}

	// PR created by Seer. When an agent is already working (or the issue is
	// closed) just annotate; otherwise this is the Seer-PR-first handoff:
	// wake Cole to review Seer's draft and run the ship-pr loop.
	if (action === 'pr_created') {
		return handleSeerPrCreated({ storage, data, groupId, runId, dryRun })
	}

	// Only the completed RCA drives a triage agent. Started/solution/coding/
	// iteration events are progress signals we simply acknowledge.
	if (action !== 'root_cause_completed') {
		return { ok: true, skipped: `seer-action-${action}`, issueId: groupId }
	}

	const rootCause = normalizeRootCause(data?.root_cause, runId)
	const { issue, project } = await resolveIssueForGroup(groupId)
	if (!issue) return { ok: true, skipped: 'issue-lookup-failed', issueId: groupId }
	if (!project) {
		return {
			ok: true,
			skipped: 'unconfigured-project',
			issueId: groupId,
			projectSlug: issue.project?.slug ?? null,
		}
	}

	// Dedupe: a Seer RCA for an issue already triaged (or in flight / queued
	// behind the per-repo lease) should not wake Cole a second time. Reuse the
	// same record + claim as the issue path so the two ingress routes coordinate.
	const issueKey = issueStorageKey(groupId)
	const existing = await storage.get(issueKey)
	if (existing && existing.status === 'agent-spawned') {
		return { ok: true, skipped: 'already-in-flight', issueId: groupId }
	}
	if (existing && existing.status === 'queued') {
		return { ok: true, skipped: 'already-queued', issueId: groupId }
	}
	if (existing && ['fixed', 'filtered', 'ignored'].includes(existing.status)) {
		return { ok: true, skipped: `already-${existing.status}`, issueId: groupId }
	}

	const title = truncate(issue.title ?? 'Unknown issue', 200)
	const link = issue.permalink

	// Seer-PR-first: when this project's Seer runs continue past RCA to a
	// draft PR (stopping point "open_pr"), waking Cole for investigation
	// now would race Seer's own coding step. Hold the issue instead; the
	// seer.pr_created webhook wakes Cole for ship review, and the grace-window
	// fallback (webhook redelivery or reconciler) covers a Seer run that
	// dies between RCA and PR.
	const runsToPr = await seerRunsToPr(project.slug)

	// Dry-run (direct-invocation test only): validate routing, project
	// resolution, and RCA-seeded prompt with zero storage writes or spawn.
	if (dryRun) {
		const promptText = buildTriageAgentPrompt({
			issue,
			project,
			discordMessageId: 'PENDING',
			context: { eventSummary: null, siblings: null },
			seerRootCause: rootCause,
			...promptOwnerFields(await readTriageSettings()),
		})
		return {
			ok: true,
			dryRun: true,
			action,
			issueId: groupId,
			projectSlug: project.slug,
			repository: project.repository,
			hasRootCause: Boolean(rootCause),
			rootCauseOneLine: rootCause?.one_line_description ?? null,
			seerSeededInPrompt: promptText.includes('Seer root-cause analysis'),
			promptChars: promptText.length,
			wouldDeferToSeerPr: runsToPr,
		}
	}

	if (runsToPr) {
		const posted = await upsertSentryReport(existing?.discordMessageId ?? null, {
			status: 'queued',
			title,
			shortId: issue.shortId,
			issueId: groupId,
			projectSlug: project.slug,
			link,
			summary: 'Root cause identified — Seer is drafting a PR…',
		})
		let messageId = posted.id
		await storage.set(issueKey, {
			...(existing ?? {}),
			issueId: groupId,
			projectSlug: project.slug,
			shortId: issue.shortId ?? null,
			title,
			link,
			status: 'awaiting-seer-pr',
			seededBy: 'seer',
			seerRunId: runId,
			seerRootCause: rootCause,
			discordMessageId: messageId,
			seerPrAwaitedAt: new Date().toISOString(),
			seenAt: existing?.seenAt ?? new Date().toISOString(),
		})
		return {
			ok: true,
			issueId: groupId,
			projectSlug: project.slug,
			deferredToSeerPr: true,
			seerRunId: runId,
		}
	}

	const claimed = await claimIssueOnce(storage, groupId)
	if (!claimed && !(existing && existing.status)) {
		return { ok: true, skipped: 'concurrent-duplicate', issueId: groupId }
	}

	// Only count the issue toward the hourly anomaly breaker if it was not
	// already counted by the issue-created path that requested this RCA.
	const issuesSeen = existing
		? Number((await storage.get(issuesSeenCounterKey())) ?? 0)
		: await bumpCounter(storage, issuesSeenCounterKey())
	if (issuesSeen > issueAnomalyThresholdPerHour) {
		const alertKey = alertSentKey('anomaly')
		if (!(await storage.get(alertKey))) {
			await storage.set(alertKey, new Date().toISOString())
			await postSentryReport({
				status: 'queued',
				title: 'Sentry triage paused',
				summary: `${issuesSeen} issues this hour (threshold ${issueAnomalyThresholdPerHour}). Not waking Cole.`,
			})
		}
		await storage.set(issueKey, {
			issueId: groupId,
			projectSlug: project.slug,
			status: 'skipped-anomaly',
			seenAt: new Date().toISOString(),
		})
		return { ok: true, skipped: 'anomaly-breaker', issuesSeen }
	}

	const spawned = Number((await storage.get(spawnedCounterKey())) ?? 0)
	if (spawned >= maxAgentsPerHour) {
		await storage.set(issueKey, {
			issueId: groupId,
			projectSlug: project.slug,
			status: 'skipped-cap',
			seenAt: new Date().toISOString(),
		})
		return { ok: true, skipped: 'hourly-cap', spawned }
	}

	// Reuse the issue-created message when this RCA is the handoff from a
	// deferred "awaiting-seer" record, so there is still one message per issue.
	const existingMessageId = existing?.discordMessageId ?? null
	let posted
	if (existingMessageId) {
		posted = { id: existingMessageId }
		await editSentryReport(existingMessageId, {
			status: 'queued',
			title,
			shortId: issue.shortId,
			issueId: groupId,
			projectSlug: project.slug,
			link,
			summary: 'Seer RCA ready — Waking Cole…',
		}).catch(() => {})
	} else {
		posted = await postSentryReport({
			status: 'queued',
			title,
			shortId: issue.shortId,
			issueId: groupId,
			projectSlug: project.slug,
			link,
			summary: "Waking Cole with Seer's root cause…",
		})
	}
	const spawnContext = await gatherSpawnContext(groupId, project)
	const promptWithMessage = buildTriageAgentPrompt({
		issue,
		project,
		discordMessageId: posted.id,
		context: spawnContext.sentryContext,
		issueState: spawnContext.issueState,
		seerRootCause: rootCause,
	})

	// Per-repo slots: same gate as the from-scratch path. Seer-seeded spawns
	// compete for the same concurrency budget; overflow joins the queue with the
	// RCA embedded so the eventual flush still starts from Seer's hypothesis.
	const acquired = await acquireRepoLease(storage, project.slug, groupId)
	if (!acquired) {
		await enqueueRepoIssue(storage, project.slug, groupId, {
			issue: {
				id: String(groupId),
				shortId: issue.shortId ?? null,
				title: issue.title ?? null,
				culprit: issue.culprit ?? null,
				level: issue.level ?? 'error',
				permalink: issue.permalink ?? link,
			},
			discordMessageId: posted.id,
			isRetriage: false,
			seerRootCause: rootCause,
			...promptOwnerFields(await readTriageSettings()),
		})
		await storage.set(issueKey, {
			issueId: groupId,
			projectSlug: project.slug,
			shortId: issue.shortId ?? null,
			title,
			link,
			status: 'queued',
			seededBy: 'seer',
			seerRunId: runId,
			discordMessageId: posted.id,
			seenAt: new Date().toISOString(),
		})
		await editSentryReport(posted.id, {
			status: 'queued',
			title,
			shortId: issue.shortId,
			issueId: groupId,
			projectSlug: project.slug,
			link,
			summary: 'Queued — Cole slots for this repo are full…',
		}).catch(() => {})
		return {
			ok: true,
			queued: true,
			issueId: groupId,
			projectSlug: project.slug,
			seededBy: 'seer',
		}
	}

	await storage.set(issueKey, {
		...(existing ?? {}),
		issueId: groupId,
		projectSlug: project.slug,
		shortId: issue.shortId ?? null,
		title,
		link,
		status: 'spawn-failed',
		wakePending: true,
		seededBy: 'seer',
		seerRunId: runId,
		seerRootCause: rootCause,
		discordMessageId: posted.id,
		seenAt: existing?.seenAt ?? new Date().toISOString(),
	})

	let agent = null
	try {
		agent = await wakeCole({
			promptText: promptWithMessage,
			discordMessageId: posted.id,
			discordChannelId: (await readTriageSettings()).discordChannelId,
		})
	} catch (error) {
		await releaseRepoLease(storage, project.slug, groupId)
		const message = truncate(
			error instanceof Error ? error.message : String(error),
			250,
		)
		await editSentryReport(posted.id, {
			status: 'queued',
			title,
			shortId: issue.shortId,
			issueId: groupId,
			projectSlug: project.slug,
			link,
			summary: `Cole wake failed — will retry: ${message}`,
		})
		await storage.set(issueKey, {
			issueId: groupId,
			projectSlug: project.slug,
			status: 'spawn-failed',
			discordMessageId: posted.id,
			seededBy: 'seer',
			seerRunId: runId,
			seenAt: new Date().toISOString(),
		})
		throw new Error(`cole-wake-failed:${groupId}:${message}`)
	}

	await updateRepoLeaseHolder(storage, project.slug, groupId)
	await bumpCounter(storage, spawnedCounterKey())
	await storage.set(issueKey, {
		issueId: groupId,
		projectSlug: project.slug,
		shortId: issue.shortId ?? null,
		title,
		link,
		status: 'agent-spawned',
		wakePending: false,
		seededBy: 'seer',
		seerRunId: runId,
		agentId: 'cole',
		agentUrl: 'grok-bot:cole',
		discordMessageId: posted.id,
		spawnedAt: new Date().toISOString(),
		hourBucket: hourBucket(),
	})
	await editSentryReport(posted.id, {
		status: 'investigating',
		title,
		shortId: issue.shortId,
		issueId: groupId,
		projectSlug: project.slug,
		link,
		summary: "Cole is triaging from Seer's RCA",
		agentUrl: 'grok-bot:cole',
	})

	return {
		ok: true,
		issueId: groupId,
		projectSlug: project.slug,
		agentId: 'cole',
		seededBy: 'seer',
	}
}

/**
 * Seer opened its draft PR(s) for an issue: wake Cole for ship review. The
 * investigation is done twice over (Seer RCA + Seer code change), so the
 * Cole ship review's job is adversarial review, tests, and the ship-pr loop —
 * Cole may use an isolated checkout on the Seer PR branch, but must not open
 * competing PRs. Annotate-only when an agent is already on the issue or it is closed.
 */
async function handleSeerPrCreated({ storage, data, groupId, runId, dryRun }) {
	const prs = (data?.pull_requests ?? [])
		.map((entry) => entry?.pull_request?.pr_url)
		.filter(Boolean)
	const issueKey = issueStorageKey(groupId)
	const existing = (await storage.get(issueKey)) ?? {}

	const annotateOnly =
		existing.status === 'agent-spawned' ||
		existing.status === 'queued' ||
		['fixed', 'filtered', 'ignored'].includes(existing.status)

	if (dryRun) {
		return {
			ok: true,
			dryRun: true,
			action: 'pr_created',
			issueId: groupId,
			seerPrUrls: prs,
			existingStatus: existing.status ?? null,
			wouldSpawnShipAgent: !annotateOnly && prs.length > 0,
		}
	}

	await storage.set(issueKey, {
		...existing,
		issueId: groupId,
		seerPrUrls: prs,
		seerRunId: runId,
		seerPrAt: new Date().toISOString(),
	})

	if (annotateOnly || prs.length === 0) {
		if (existing.discordMessageId && prs.length > 0) {
			await editSentryReport(existing.discordMessageId, {
				status: 'queued',
				title: existing.title ?? title,
				shortId: existing.shortId ?? issue.shortId,
				issueId: groupId,
				projectSlug: existing.projectSlug ?? project.slug,
				link: existing.link ?? link,
				summary: 'Seer opened a PR — review before duplicating.',
				prUrls: prs,
			}).catch(() => {})
		}
		return {
			ok: true,
			action: 'pr_created',
			issueId: groupId,
			seerPrUrls: prs,
			annotated: true,
		}
	}

	// Exactly-once ship wake per issue: a dedicated claim key (distinct from
	// the issue claim, which the issue-created path already holds).
	const claimed = await claimIssueOnce(storage, `ship:${groupId}`)
	if (!claimed) {
		return { ok: true, skipped: 'ship-already-claimed', issueId: groupId }
	}

	const { issue, project } = await resolveIssueForGroup(groupId)
	if (!issue || !project) {
		return {
			ok: true,
			skipped: !issue ? 'issue-lookup-failed' : 'unconfigured-project',
			issueId: groupId,
		}
	}

	const spawned = Number((await storage.get(spawnedCounterKey())) ?? 0)
	if (spawned >= maxAgentsPerHour) {
		await storage.set(issueKey, {
			...existing,
			issueId: groupId,
			projectSlug: project.slug,
			seerPrUrls: prs,
			status: 'skipped-cap',
			seenAt: new Date().toISOString(),
		})
		return { ok: true, skipped: 'hourly-cap', spawned }
	}

	const title = truncate(issue.title ?? existing.title ?? 'Unknown issue', 200)
	const link = issue.permalink ?? existing.link
	const postedShip = await upsertSentryReport(existing.discordMessageId ?? null, {
		status: 'queued',
		title,
		shortId: issue.shortId,
		issueId: groupId,
		projectSlug: project.slug,
		link,
		summary: `Seer drafted ${prs.length > 1 ? 'PRs' : 'a PR'} — Waking Cole…`,
		prUrls: prs,
	})
	let messageId = postedShip.id

	// Take the repo lease when it is free so investigation wakes queue
	// behind Cole; when it is busy, wake Cole anyway — Cole
	// works on Seer's existing branch (it authors no competing PR) and the
	// ship-pr loop handles rebase/conflict iteration.
	const leaseAcquired = await acquireRepoLease(storage, project.slug, groupId)
	// Persist the edit-existing Discord id before wake.
	await storage.set(issueKey, {
		...existing,
		issueId: groupId,
		projectSlug: project.slug,
		shortId: issue.shortId ?? null,
		title,
		link,
		status: 'spawn-failed',
		wakePending: true,
		seededBy: 'seer-pr',
		seerRunId: runId,
		seerPrUrls: prs,
		discordMessageId: messageId,
		seenAt: existing.seenAt ?? new Date().toISOString(),
	})

	let agent = null
	try {
		agent = await wakeCole({
			promptText: buildShipPrPrompt({
				issue,
				project,
				prUrls: prs,
				seerRootCause: existing.seerRootCause ?? null,
				discordMessageId: messageId,
				...promptOwnerFields(await readTriageSettings()),
			}),
			discordMessageId: messageId,
			discordChannelId: (await readTriageSettings()).discordChannelId,
		})
	} catch (error) {
		if (leaseAcquired) await releaseRepoLease(storage, project.slug, groupId)
		const message = truncate(
			error instanceof Error ? error.message : String(error),
			250,
		)
		await editSentryReport(messageId, {
			status: 'queued',
			title,
			shortId: issue.shortId,
			issueId: groupId,
			projectSlug: project.slug,
			link,
			summary: `Cole wake failed — will retry: ${message}`,
			prUrls: prs,
		}).catch(() => {})
		await storage.set(issueKey, {
			...existing,
			issueId: groupId,
			projectSlug: project.slug,
			status: 'spawn-failed',
			seededBy: 'seer-pr',
			seerPrUrls: prs,
			seerRunId: runId,
			discordMessageId: messageId,
			seenAt: new Date().toISOString(),
		})
		throw new Error(`ship-cole-wake-failed:${groupId}:${message}`)
	}

	if (leaseAcquired) {
		await updateRepoLeaseHolder(storage, project.slug, groupId)
	}
	await bumpCounter(storage, spawnedCounterKey())
	await storage.set(issueKey, {
		...existing,
		issueId: groupId,
		projectSlug: project.slug,
		shortId: issue.shortId ?? null,
		title,
		link,
		status: 'agent-spawned',
		wakePending: false,
		seededBy: 'seer-pr',
		seerRunId: runId,
		seerPrUrls: prs,
		agentId: 'cole',
		agentUrl: 'grok-bot:cole',
		discordMessageId: messageId,
		spawnedAt: new Date().toISOString(),
		hourBucket: hourBucket(),
	})
	await editSentryReport(messageId, {
		status: 'investigating',
		title,
		shortId: issue.shortId,
		issueId: groupId,
		projectSlug: project.slug,
		link,
		summary: "Cole is reviewing Seer's draft",
		prUrls: prs,
		agentUrl: 'grok-bot:cole',
	}).catch(() => {})

	return {
		ok: true,
		issueId: groupId,
		projectSlug: project.slug,
		agentId: 'cole',
		seededBy: 'seer-pr',
		seerPrUrls: prs,
	}
}