import { classifyPath, type IgnoreIndex } from './classify.ts'
import {
getCommitDetail,
listBranchCommits,
type GithubCommitDetail,
type GithubCommitListItem,
} from './github.ts'
import { loadIgnoreIndex } from './ignore-rules.ts'
import {
getRepoSettings,
hasCommit,
rememberGeneratedPath,
upsertCommit,
} from './storage.ts'
import type { BucketDelta, FileKind, RepoSettings } from './types.ts'
export async function ingestCommitSha(
sha: string,
options: { ignore?: IgnoreIndex; repo?: RepoSettings } = {},
) {
const repo = options.repo ?? (await getRepoSettings())
const ignore = options.ignore ?? (await loadIgnoreIndex())
const detail = await getCommitDetail(repo, sha)
return persistCommit(detail, ignore)
}
export async function persistCommit(
detail: GithubCommitDetail,
ignore: IgnoreIndex,
) {
const files = detail.files ?? []
const buckets = new Map<string, BucketDelta>()
let skippedFiles = 0
let countedFiles = 0
for (const file of files) {
const classification = classifyPath(file.filename, {
patch: file.patch,
ignore,
})
if (!classification.countable) {
skippedFiles += 1
if (
classification.reason === 'generated-content-marker' ||
classification.reason.startsWith('generated')
) {
ignore.knownGeneratedPaths.add(file.filename)
await rememberGeneratedPath(file.filename, classification.reason)
}
continue
}
countedFiles += 1
const key = `${classification.language}\t${classification.kind}`
const current = buckets.get(key) ?? {
language: classification.language,
kind: classification.kind as FileKind,
additions: 0,
deletions: 0,
}
current.additions += file.additions ?? 0
current.deletions += file.deletions ?? 0
buckets.set(key, current)
}
const committedAt =
detail.commit.committer?.date ||
detail.commit.author?.date ||
new Date().toISOString()
await upsertCommit(
{
sha: detail.sha,
committedAt,
message: detail.commit.message || '',
authorName: detail.commit.author?.name || detail.commit.committer?.name || '',
htmlUrl: detail.html_url,
parentSha: detail.parents[0]?.sha ?? null,
filesTruncated: files.length >= 300,
skippedFiles,
countedFiles,
},
[...buckets.values()],
)
return {
sha: detail.sha,
committedAt,
countedFiles,
skippedFiles,
filesTruncated: files.length >= 300,
bucketCount: buckets.size,
}
}
export async function ingestMissingFromList(
items: GithubCommitListItem[],
options: { limit?: number; deadlineMs?: number } = {},
) {
const repo = await getRepoSettings()
const ignore = await loadIgnoreIndex()
const limit = options.limit ?? 80
const deadline = Date.now() + (options.deadlineMs ?? 60_000)
const ingested: string[] = []
const skippedExisting: string[] = []
const errors: Array<{ sha: string; error: string }> = []
for (const item of items) {
if (ingested.length >= limit || Date.now() >= deadline) break
if (await hasCommit(item.sha)) {
skippedExisting.push(item.sha)
continue
}
try {
await ingestCommitSha(item.sha, { ignore, repo })
ingested.push(item.sha)
} catch (error) {
errors.push({
sha: item.sha,
error: error instanceof Error ? error.message : String(error),
})
}
}
return {
ingested: ingested.length,
alreadyStored: skippedExisting.length,
remaining: Math.max(0, items.length - ingested.length - skippedExisting.length - errors.length),
errors,
shas: ingested,
}
}
export async function collectBranchHistory(maxPages = 40) {
const repo = await getRepoSettings()
return listBranchCommits(repo, { maxPages })
}