Skip to content

src/ingest.ts

122 lines · 3.5 KB · TypeScript
Raw
import { accumulateBuckets } from './buckets.ts'
import { type IgnoreIndex } from './classify.ts'
import {
	getCommitDetail,
	listBranchCommits,
	type GithubCommitDetail,
	type GithubCommitListItem,
} from './github.ts'
import { loadIgnoreIndex } from './ignore-rules.ts'
import {
	getRepoSettings,
	listCompleteShas,
	rememberGeneratedPath,
	upsertCommit,
} from './storage.ts'
import type { RepoSettings } from './types.ts'

export async function ingestCommitSha(
	sha: string,
	options: { ignore?: IgnoreIndex; repo?: RepoSettings; deadline?: number } = {},
) {
	const repo = options.repo ?? (await getRepoSettings())
	const ignore = options.ignore ?? (await loadIgnoreIndex())
	const detail = await getCommitDetail(repo, sha, { deadline: options.deadline })
	return persistCommit(detail, ignore)
}

export async function persistCommit(
	detail: GithubCommitDetail,
	ignore: IgnoreIndex,
) {
	const files = detail.files ?? []
	const { buckets, skippedFiles, countedFiles, generatedToRemember } =
		accumulateBuckets(files, ignore)
	// GitHub returns `files: []` for some merge commits. Persist that as a
	// skipped-only row so older completeness checks also treat it as stored.
	const recordedSkipped = files.length === 0 ? Math.max(skippedFiles, 1) : skippedFiles
	for (const item of generatedToRemember) {
		ignore.knownGeneratedPaths.add(item.path)
		await rememberGeneratedPath(item.path, item.reason)
	}

	const committedAt =
		detail.commit.committer?.date ||
		detail.commit.author?.date ||
		new Date().toISOString()

	await upsertCommit(
		{
			sha: detail.sha,
			committedAt,
			message: detail.commit.message || '',
			authorName: detail.commit.author?.name || detail.commit.committer?.name || '',
			htmlUrl: detail.html_url,
			parentSha: detail.parents[0]?.sha ?? null,
			filesTruncated: files.length >= 300,
			skippedFiles: recordedSkipped,
			countedFiles,
		},
		buckets,
	)

	return {
		sha: detail.sha,
		committedAt,
		countedFiles,
		skippedFiles: recordedSkipped,
		filesTruncated: files.length >= 300,
		bucketCount: buckets.length,
	}
}

export async function ingestMissingFromList(
	items: GithubCommitListItem[],
	options: { limit?: number; deadlineMs?: number } = {},
) {
	const repo = await getRepoSettings()
	const ignore = await loadIgnoreIndex()
	const limit = options.limit ?? 80
	const deadline = Date.now() + (options.deadlineMs ?? 60_000)
	const complete = await listCompleteShas()
	const ingested: string[] = []
	const skippedExisting: string[] = []
	const errors: Array<{ sha: string; error: string }> = []

	for (const item of items) {
		if (ingested.length >= limit || Date.now() >= deadline) break
		if (complete.has(item.sha)) {
			skippedExisting.push(item.sha)
			continue
		}
		try {
			await ingestCommitSha(item.sha, { ignore, repo, deadline })
			ingested.push(item.sha)
			complete.add(item.sha)
		} catch (error) {
			errors.push({
				sha: item.sha,
				error: error instanceof Error ? error.message : String(error),
			})
		}
	}

	return {
		ingested: ingested.length,
		alreadyStored: skippedExisting.length,
		remaining: Math.max(0, items.length - ingested.length - skippedExisting.length - errors.length),
		errors,
		shas: ingested,
	}
}

/**
 * List tracked-branch history newest-first, stopping paging at `deadline`
 * (epoch ms). `complete: false` means older pages were not listed this run.
 */
export async function collectBranchHistory(
	options: { maxPages?: number; deadline?: number } = {},
) {
	const repo = await getRepoSettings()
	return listBranchCommits(repo, options)
}