import { isTransientGithubFailure } from './github-errors.ts' import { collectBranchHistory, ingestMissingFromList } from './ingest.ts' import { refreshIgnoreIndex } from './ignore-rules.ts' import { getRepoSettings, ingestSummary } from './storage.ts' /** * Ingest missing mainline commits into package storage, oldest gaps first. * Use this for the initial fill and whenever the graph is missing history. * * @param input - Optional batch size, page cap, and whether to refresh ignore rules * @returns How many commits were stored and whether more remain. Transient * GitHub/Cloudflare failures return `deferred: true` instead of throwing so * the five-minute job can retry on the next tick. The ingest deadline is * measured from the start of this call so a slow history listing cannot * spend a fresh 60s after the 90s platform budget is already gone. Listing * itself stops at two thirds of the budget (`listingComplete: false`) and * every GitHub request carries a bounded timeout. * * @example * import backfill from 'kody:@kentcdodds/lineage/backfill' * * const progress = await backfill({ limit: 50 }) */ export default async function backfill( input: { limit?: number maxPages?: number refreshIgnore?: boolean deadlineMs?: number } = {}, ) { const startedAt = Date.now() const budgetMs = input.deadlineMs ?? 70_000 // Leave at least a third of the budget for ingest after listing. const listingDeadline = startedAt + Math.round(budgetMs * (2 / 3)) if (input.refreshIgnore) { await refreshIgnoreIndex() } const repo = await getRepoSettings() let history try { history = await collectBranchHistory({ maxPages: input.maxPages ?? 40, deadline: listingDeadline, }) } catch (error) { if (!isTransientGithubFailure(error)) throw error const summary = await ingestSummary() return { ok: true, deferred: true, reason: error instanceof Error ? error.message : String(error), repo, listed: 0, ingested: 0, alreadyStored: 0, remaining: 1, errors: [], shas: [], done: false, ingest: summary, } } const chronological = [...history.items].reverse() const result = await ingestMissingFromList(chronological, { limit: input.limit ?? 80, deadlineMs: Math.max(0, startedAt + budgetMs - Date.now()), }) const summary = await ingestSummary() return { ok: true, deferred: false, repo, listed: history.items.length, listingComplete: history.complete, ...result, done: history.complete && result.remaining === 0 && result.errors.length === 0, ingest: summary, } }