Skip to content

Built for people who want to own their automations. Join the waitlist for an invite.

Package listing

@kody/ai

src/smoke-test.ts

167 lines · 4.1 KB · TypeScript
import { kody } from 'kody:runtime'
import { generateText, listModels } from './complete.ts'
import { generateObject } from './generate-object.ts'
import { PROVIDER_IDS, defaultSecretName, providerSecretUrl } from './providers.ts'
import aiSettings from './settings.ts'
import { parseProviderId, type ProviderId } from './validation.ts'

function secretEntries(
	result: unknown,
): Array<{ name?: string; scope?: string }> {
	if (
		result &&
		typeof result === 'object' &&
		Array.isArray((result as { secrets?: unknown }).secrets)
	) {
		return (result as { secrets: Array<{ name?: string; scope?: string }> }).secrets
	}
	return Array.isArray(result) ? result : []
}

async function listUserSecretNames(): Promise<Set<string>> {
	try {
		const listed = await kody.secret_list({ scope: 'user' })
		const names = new Set<string>()
		for (const entry of secretEntries(listed)) {
			if (entry?.name && (entry.scope === 'user' || !entry.scope)) {
				names.add(entry.name)
			}
		}
		return names
	} catch {
		return new Set()
	}
}

function firstConfiguredProvider(names: Set<string>): ProviderId | null {
	const preferred: Array<ProviderId> = [
		'openai',
		'anthropic',
		'groq',
		'openai-compatible',
		'cloudflare',
	]
	for (const id of preferred) {
		if (names.has(defaultSecretName(id))) return id
	}
	return null
}

/**
 * Local helper checks, plus a live models list when a provider secret exists.
 *
 * Without secrets this still returns `{ ok: true, live: false }` and the
 * prefilled setup URLs so the package is forkable. With a secret it proves
 * host approval via GET /models — it never spends completion tokens.
 *
 * @example
 * import smokeTest from 'kody:@kody/ai/smoke-test'
 * const result = await smokeTest()
 */
export async function smokeTest(params: { provider?: string } = {}) {
	const preview = await generateText({
		provider: 'openai',
		messages: [{ role: 'user', content: 'kody ai dry-run smoke' }],
		dryRun: true,
	})
	if (!('dryRun' in preview) || preview.dryRun !== true) {
		throw new Error('generateText dry-run self-check failed.')
	}

	const objectPreview = await generateObject({
		provider: 'openai',
		messages: [{ role: 'user', content: 'label this: ok' }],
		schema: {
			type: 'object',
			properties: { label: { type: 'string' } },
			required: ['label'],
		},
		dryRun: true,
	})
	if (!('dryRun' in objectPreview) || objectPreview.dryRun !== true) {
		throw new Error('generateObject dry-run self-check failed.')
	}

	const settingsPreview = await aiSettings({
		provider: 'openai',
		model: 'gpt-4o-mini',
		dryRun: true,
	})
	if (!('dryRun' in settingsPreview) || settingsPreview.dryRun !== true) {
		throw new Error('settings dry-run self-check failed.')
	}

	const names = await listUserSecretNames()
	const requested = params.provider ? parseProviderId(params.provider) : undefined
	const provider = requested ?? firstConfiguredProvider(names)
	const setup = {
		secrets: Object.fromEntries(
			PROVIDER_IDS.map((id) => {
				const secret = defaultSecretName(id)
				return [secret, names.has(secret)]
			}),
		),
		nextSteps: PROVIDER_IDS.map((id) => providerSecretUrl(id)),
	}

	if (!provider || !names.has(defaultSecretName(provider))) {
		return {
			ok: true,
			live: false,
			selfCheck: {
				generateTextDryRun: true,
				generateObjectDryRun: true,
				settingsDryRun: true,
			},
			setup,
		}
	}

	try {
		const models = await listModels({ provider })
		if (!models || ('ok' in models && models.ok === false)) {
			return {
				ok: true,
				live: false,
				provider,
				selfCheck: {
					generateTextDryRun: true,
					generateObjectDryRun: true,
					settingsDryRun: true,
				},
				models,
				setup,
			}
		}
		return {
			ok: true,
			live: true,
			provider,
			selfCheck: {
				generateTextDryRun: true,
				generateObjectDryRun: true,
				settingsDryRun: true,
			},
			models,
			setup: {
				secrets: setup.secrets,
				nextSteps: [],
			},
		}
	} catch (error) {
		return {
			ok: true,
			live: false,
			provider,
			error: error instanceof Error ? error.message : String(error),
			selfCheck: {
				generateTextDryRun: true,
				generateObjectDryRun: true,
				settingsDryRun: true,
			},
			setup,
		}
	}
}

export default smokeTest