Skip to content

Built for people who want to own their automations. Join the waitlist for an invite.

Package listing

@kentcdodds/lineage

src/ingest.ts

136 lines · 3.5 KB · TypeScript
import { classifyPath, type IgnoreIndex } from './classify.ts'
import {
	getCommitDetail,
	listBranchCommits,
	type GithubCommitDetail,
	type GithubCommitListItem,
} from './github.ts'
import { loadIgnoreIndex } from './ignore-rules.ts'
import {
	getRepoSettings,
	hasCommit,
	rememberGeneratedPath,
	upsertCommit,
} from './storage.ts'
import type { BucketDelta, FileKind, RepoSettings } from './types.ts'

export async function ingestCommitSha(
	sha: string,
	options: { ignore?: IgnoreIndex; repo?: RepoSettings } = {},
) {
	const repo = options.repo ?? (await getRepoSettings())
	const ignore = options.ignore ?? (await loadIgnoreIndex())
	const detail = await getCommitDetail(repo, sha)
	return persistCommit(detail, ignore)
}

export async function persistCommit(
	detail: GithubCommitDetail,
	ignore: IgnoreIndex,
) {
	const files = detail.files ?? []
	const buckets = new Map<string, BucketDelta>()
	let skippedFiles = 0
	let countedFiles = 0

	for (const file of files) {
		const classification = classifyPath(file.filename, {
			patch: file.patch,
			ignore,
		})
		if (!classification.countable) {
			skippedFiles += 1
			if (
				classification.reason === 'generated-content-marker' ||
				classification.reason.startsWith('generated')
			) {
				ignore.knownGeneratedPaths.add(file.filename)
				await rememberGeneratedPath(file.filename, classification.reason)
			}
			continue
		}
		countedFiles += 1
		const key = `${classification.language}\t${classification.kind}`
		const current = buckets.get(key) ?? {
			language: classification.language,
			kind: classification.kind as FileKind,
			additions: 0,
			deletions: 0,
		}
		current.additions += file.additions ?? 0
		current.deletions += file.deletions ?? 0
		buckets.set(key, current)
	}

	const committedAt =
		detail.commit.committer?.date ||
		detail.commit.author?.date ||
		new Date().toISOString()

	await upsertCommit(
		{
			sha: detail.sha,
			committedAt,
			message: detail.commit.message || '',
			authorName: detail.commit.author?.name || detail.commit.committer?.name || '',
			htmlUrl: detail.html_url,
			parentSha: detail.parents[0]?.sha ?? null,
			filesTruncated: files.length >= 300,
			skippedFiles,
			countedFiles,
		},
		[...buckets.values()],
	)

	return {
		sha: detail.sha,
		committedAt,
		countedFiles,
		skippedFiles,
		filesTruncated: files.length >= 300,
		bucketCount: buckets.size,
	}
}

export async function ingestMissingFromList(
	items: GithubCommitListItem[],
	options: { limit?: number; deadlineMs?: number } = {},
) {
	const repo = await getRepoSettings()
	const ignore = await loadIgnoreIndex()
	const limit = options.limit ?? 80
	const deadline = Date.now() + (options.deadlineMs ?? 60_000)
	const ingested: string[] = []
	const skippedExisting: string[] = []
	const errors: Array<{ sha: string; error: string }> = []

	for (const item of items) {
		if (ingested.length >= limit || Date.now() >= deadline) break
		if (await hasCommit(item.sha)) {
			skippedExisting.push(item.sha)
			continue
		}
		try {
			await ingestCommitSha(item.sha, { ignore, repo })
			ingested.push(item.sha)
		} catch (error) {
			errors.push({
				sha: item.sha,
				error: error instanceof Error ? error.message : String(error),
			})
		}
	}

	return {
		ingested: ingested.length,
		alreadyStored: skippedExisting.length,
		remaining: Math.max(0, items.length - ingested.length - skippedExisting.length - errors.length),
		errors,
		shas: ingested,
	}
}

export async function collectBranchHistory(maxPages = 40) {
	const repo = await getRepoSettings()
	return listBranchCommits(repo, { maxPages })
}