import { isTransientGithubFailure } from './github-errors.ts' import { collectBranchHistory, ingestMissingFromList } from './ingest.ts' import { refreshIgnoreIndex } from './ignore-rules.ts' import { shasToPrune } from './mainline.ts' import { deleteCommits, getRepoSettings, ingestSummary, listStoredShas, migrateTruncatedCommits, } from './storage.ts' /** * Ingest missing mainline commits into package storage, oldest gaps first. * Use this for the initial fill and whenever the graph is missing history. * Only the branch's first-parent history is stored, so each change shows up * when it landed (merge commits carry their whole PR). Stored commits that * were merged in from other branches are pruned (`pruned` in the result). * * @param input - Optional batch size, page cap, and whether to refresh ignore rules * @returns How many commits were stored and whether more remain. Transient * GitHub/Cloudflare failures return `deferred: true` instead of throwing so * the five-minute job can retry on the next tick. The ingest deadline is * measured from the start of this call so a slow history listing cannot * spend a fresh 60s after the 90s platform budget is already gone. Listing * itself stops at two thirds of the budget (`listingComplete: false`) and * every GitHub request carries a bounded timeout. * * @example * import backfill from 'kody:@kody/lineage/backfill' * * const progress = await backfill({ limit: 50 }) */ export default async function backfill( input: { limit?: number maxPages?: number refreshIgnore?: boolean deadlineMs?: number } = {}, ) { const startedAt = Date.now() const budgetMs = input.deadlineMs ?? 70_000 // Leave at least a third of the budget for ingest after listing. const listingDeadline = startedAt + Math.round(budgetMs * (2 / 3)) if (input.refreshIgnore) { await refreshIgnoreIndex() } const repo = await getRepoSettings() let history try { history = await collectBranchHistory({ maxPages: input.maxPages ?? 40, deadline: listingDeadline, }) } catch (error) { if (!isTransientGithubFailure(error)) throw error const summary = await ingestSummary() return { ok: true, deferred: true, reason: error instanceof Error ? error.message : String(error), repo, listed: 0, mainline: 0, pruned: 0, truncatedRefetch: 0, ingested: 0, alreadyStored: 0, remaining: 1, errors: [], shas: [], done: false, ingest: summary, } } const truncatedRefetch = await migrateTruncatedCommits() const pruned = await deleteCommits( shasToPrune( { chain: history.items, offChain: history.offChain, reachedRoot: history.reachedRoot, }, await listStoredShas(), ), ) const chronological = [...history.items].reverse() const result = await ingestMissingFromList(chronological, { limit: input.limit ?? 80, deadlineMs: Math.max(0, startedAt + budgetMs - Date.now()), }) const summary = await ingestSummary() return { ok: true, deferred: false, repo, listed: history.listed, mainline: history.items.length, listingComplete: history.complete, pruned, truncatedRefetch, ...result, done: history.complete && result.remaining === 0 && result.errors.length === 0, ingest: summary, } }