Skip to content

src/ingest-missing.ts

86 lines · 2.5 KB · TypeScript
Raw
import { isTransientGithubFailure } from './github-errors.ts'
import { collectBranchHistory, ingestCommitSha, ingestMissingFromList } from './ingest.ts'
import { ingestShaList } from './ingest-shas.ts'
import { getRepoSettings, hasCommit, ingestSummary } from './storage.ts'

/**
 * Fill any commits that are on the tracked branch but missing from storage.
 * Use after a webhook, or to catch commits the backfill skipped.
 *
 * @param input - Optional explicit SHAs; otherwise scans recent branch history
 * @returns Newly ingested SHAs and current storage summary. Transient
 *   GitHub/Cloudflare failures on the history scan return `deferred: true`
 *   instead of throwing. With explicit SHAs (the push webhook path), a commit
 *   that fails to fetch is listed in `errors` with `deferred: true` and left
 *   for the `backfill-missing` job, so one bad fetch never fails a delivery.
 *
 * @example
 * import ingestMissing from 'kody:@kentcdodds/lineage/ingest-missing'
 *
 * const result = await ingestMissing({ limit: 20 })
 */
export default async function ingestMissing(
	input: {
		shas?: string[]
		limit?: number
		maxPages?: number
	} = {},
) {
	const startedAt = Date.now()
	const budgetMs = 50_000
	const repo = await getRepoSettings()
	if (input.shas && input.shas.length > 0) {
		const result = await ingestShaList(input.shas, {
			deadline: startedAt + budgetMs,
			hasCommit,
			ingest: (sha) => ingestCommitSha(sha, { repo, deadline: startedAt + budgetMs }),
		})
		return {
			ok: true,
			deferred: result.errors.length > 0,
			repo,
			ingested: result.ingested.length,
			alreadyStored: result.alreadyStored.length,
			remaining: result.remaining,
			errors: result.errors,
			shas: result.ingested,
			ingest: await ingestSummary(),
		}
	}

	let history
	try {
		history = await collectBranchHistory({
			maxPages: input.maxPages ?? 5,
			deadline: startedAt + Math.round(budgetMs * (2 / 3)),
		})
	} catch (error) {
		if (!isTransientGithubFailure(error)) throw error
		return {
			ok: true,
			deferred: true,
			reason: error instanceof Error ? error.message : String(error),
			repo,
			listed: 0,
			ingested: 0,
			alreadyStored: 0,
			remaining: 1,
			errors: [],
			shas: [],
			ingest: await ingestSummary(),
		}
	}
	const result = await ingestMissingFromList(history.items, {
		limit: input.limit ?? 40,
		deadlineMs: Math.max(0, startedAt + budgetMs - Date.now()),
	})
	return {
		ok: true,
		deferred: false,
		repo,
		listed: history.items.length,
		listingComplete: history.complete,
		...result,
		ingest: await ingestSummary(),
	}
}