import { isTransientGithubFailure } from './github-errors.ts' import { collectBranchHistory, ingestCommitSha, ingestMissingFromList } from './ingest.ts' import { ingestShaList } from './ingest-shas.ts' import { walkFirstParents } from './mainline.ts' import { getRepoSettings, hasCommit, ingestSummary } from './storage.ts' /** * Fill any commits that are on the tracked branch but missing from storage. * Use after a webhook, or to catch commits the backfill skipped. Only * first-parent (mainline) history is ingested: pass `head` (and optionally * `before`) to walk back from a pushed tip; commits a merge brought in are * counted once through the merge commit, at the time it landed. * * @param input - Optional pushed `head`/`before`, explicit mainline SHAs, or * nothing to scan recent branch history * @returns Newly ingested SHAs and current storage summary. Transient * GitHub/Cloudflare failures on the history scan return `deferred: true` * instead of throwing. With explicit SHAs (the push webhook path), a commit * that fails to fetch is listed in `errors` with `deferred: true` and left * for the `backfill-missing` job, so one bad fetch never fails a delivery. * * @example * import ingestMissing from 'kody:@kody/lineage/ingest-missing' * * const result = await ingestMissing({ limit: 20 }) */ export default async function ingestMissing( input: { head?: string before?: string | null /** Explicit SHAs, ingested as-is. Pass only mainline commits. */ shas?: string[] limit?: number maxPages?: number } = {}, ) { const startedAt = Date.now() const budgetMs = 50_000 const repo = await getRepoSettings() if (input.head) { const deadline = startedAt + budgetMs const walk = await walkFirstParents(input.head, { stopAt: input.before, limit: input.limit ?? 40, deadline, hasCommit, ingest: (sha) => ingestCommitSha(sha, { repo, deadline }), }) const deferred = walk.reason === 'error' || walk.reason === 'deadline' || walk.reason === 'limit' if (walk.reason === 'error') { console.warn(`lineage: deferred ${walk.error?.sha} to backfill-missing: ${walk.error?.error}`) } return { ok: true, deferred, repo, ingested: walk.ingested.length, alreadyStored: walk.reason === 'stored' ? 1 : 0, remaining: deferred ? 1 : 0, errors: walk.error ? [walk.error] : [], shas: walk.ingested, walk: { stoppedAt: walk.stoppedAt, reason: walk.reason }, ingest: await ingestSummary(), } } if (input.shas && input.shas.length > 0) { const result = await ingestShaList(input.shas, { deadline: startedAt + budgetMs, hasCommit, ingest: (sha) => ingestCommitSha(sha, { repo, deadline: startedAt + budgetMs }), }) return { ok: true, deferred: result.errors.length > 0, repo, ingested: result.ingested.length, alreadyStored: result.alreadyStored.length, remaining: result.remaining, errors: result.errors, shas: result.ingested, ingest: await ingestSummary(), } } let history try { history = await collectBranchHistory({ maxPages: input.maxPages ?? 5, deadline: startedAt + Math.round(budgetMs * (2 / 3)), }) } catch (error) { if (!isTransientGithubFailure(error)) throw error return { ok: true, deferred: true, reason: error instanceof Error ? error.message : String(error), repo, listed: 0, ingested: 0, alreadyStored: 0, remaining: 1, errors: [], shas: [], ingest: await ingestSummary(), } } const result = await ingestMissingFromList(history.items, { limit: input.limit ?? 40, deadlineMs: Math.max(0, startedAt + budgetMs - Date.now()), }) return { ok: true, deferred: false, repo, listed: history.items.length, listingComplete: history.complete, ...result, ingest: await ingestSummary(), } }