Skip to content

Built for people who want to own their automations. Join the waitlist for an invite.

Package listing

@kody/ai

src/ai.test.ts

177 lines · 6.5 KB · TypeScript
import assert from 'node:assert/strict'
import { readFileSync } from 'node:fs'
import { dirname, join } from 'node:path'
import test from 'node:test'
import { fileURLToPath } from 'node:url'
import {
	chatUrl,
	defaultSecretName,
	modelsUrl,
	providerSecretUrl,
	requestHeaders,
} from './providers.ts'
import {
	assertHttpsUrl,
	isMutationDryRun,
	isPreviewDryRun,
	parseProviderId,
	secretNameFor,
} from './validation.ts'

const here = dirname(fileURLToPath(import.meta.url))
const root = join(here, '..')

test('secret names default and suffix accounts', () => {
	assert.equal(defaultSecretName('openai'), 'openaiApiKey')
	assert.equal(defaultSecretName('anthropic', 'work'), 'anthropicApiKey-work')
	assert.equal(defaultSecretName('groq'), 'groqApiKey')
	assert.equal(defaultSecretName('cloudflare'), 'cloudflareApiToken')
	assert.equal(
		secretNameFor('openaiApiKey', undefined, 'openaiApiKey-live'),
		'openaiApiKey-live',
	)
})

test('secret names reject Kent-style aliases', () => {
	assert.throws(
		() => secretNameFor('openaiApiKey', undefined, 'OPENAI_API_KEY'),
		/openaiApiKey/,
	)
	assert.throws(
		() => secretNameFor('cloudflareApiToken', undefined, 'cfToken'),
		/cloudflareApiToken/,
	)
})

test('setup URLs are prefilled for each provider', () => {
	assert.match(providerSecretUrl('openai'), /name=openaiApiKey/)
	assert.match(providerSecretUrl('openai'), /allowedHosts=api\.openai\.com/)
	assert.match(providerSecretUrl('openai'), /scope=user/)
	assert.match(providerSecretUrl('anthropic'), /name=anthropicApiKey/)
	assert.match(providerSecretUrl('anthropic'), /allowedHosts=api\.anthropic\.com/)
	assert.match(providerSecretUrl('groq'), /allowedHosts=api\.groq\.com/)
	assert.match(
		providerSecretUrl('cloudflare'),
		/allowedHosts=api\.cloudflare\.com%2Cgateway\.ai\.cloudflare\.com/,
	)
	assert.match(
		providerSecretUrl('openai-compatible', 'openaiCompatibleApiKey', 'llm.example.com'),
		/allowedHosts=llm\.example\.com/,
	)
})

test('settings writes default to dry-run unless confirm is explicit', () => {
	assert.equal(isMutationDryRun({}), true)
	assert.equal(isMutationDryRun({ dryRun: true }), true)
	assert.equal(isMutationDryRun({ confirm: true, dryRun: true }), true)
	assert.equal(isMutationDryRun({ confirm: true }), false)
	assert.equal(isMutationDryRun({ confirm: false }), true)
})

test('completion preview is opt-in via dryRun', () => {
	assert.equal(isPreviewDryRun({}), false)
	assert.equal(isPreviewDryRun({ dryRun: true }), true)
})

test('chat URLs stay on known provider hosts and never bake account ids', () => {
	assert.equal(chatUrl({ provider: 'openai' }).href, 'https://api.openai.com/v1/chat/completions')
	assert.equal(chatUrl({ provider: 'anthropic' }).host, 'api.anthropic.com')
	assert.equal(chatUrl({ provider: 'groq' }).host, 'api.groq.com')
	assert.throws(() => chatUrl({ provider: 'cloudflare' }), /cloudflareAccountId/)
	const cf = chatUrl({
		provider: 'cloudflare',
		cloudflareAccountId: 'acct_example',
	})
	assert.equal(
		cf.href,
		'https://api.cloudflare.com/client/v4/accounts/acct_example/ai/v1/chat/completions',
	)
	const gateway = chatUrl({
		provider: 'cloudflare',
		cloudflareAccountId: 'acct_example',
		cloudflareAiGatewayId: 'my-gateway',
	})
	assert.match(gateway.href, /gateway\.ai\.cloudflare\.com/)
	assert.match(gateway.href, /my-gateway/)
	assert.throws(() => chatUrl({ provider: 'openai-compatible' }), /baseUrl/)
	const compat = chatUrl({
		provider: 'openai-compatible',
		baseUrl: 'https://llm.example.com/v1',
	})
	assert.equal(compat.href, 'https://llm.example.com/v1/chat/completions')
})

test('models list URLs are read-only GETs on the same hosts', () => {
	assert.equal(modelsUrl({ provider: 'openai' })?.href, 'https://api.openai.com/v1/models')
	assert.equal(modelsUrl({ provider: 'cloudflare' }), null)
	assert.ok(modelsUrl({ provider: 'cloudflare', cloudflareAccountId: 'acct_example' }))
})

test('request headers use secret placeholders, never literal keys', () => {
	const openai = requestHeaders('openai', 'openaiApiKey')
	assert.equal(openai.Authorization, 'Bearer {{secret:openaiApiKey|scope=user}}')
	const anthropic = requestHeaders('anthropic', 'anthropicApiKey')
	assert.equal(anthropic['x-api-key'], '{{secret:anthropicApiKey|scope=user}}')
	assert.equal(anthropic['anthropic-version'], '2023-06-01')
	const blob = JSON.stringify({ openai, anthropic })
	assert.doesNotMatch(blob, /sk-/)
	assert.doesNotMatch(blob, /me@kentcdodds/)
})

test('https baseUrl rejects credentials and http', () => {
	assert.throws(() => assertHttpsUrl('http://llm.example.com', 'baseUrl'), /https/)
	assert.throws(
		() => assertHttpsUrl('https://user:pass@llm.example.com', 'baseUrl'),
		/credentials/,
	)
	assert.equal(assertHttpsUrl('https://llm.example.com/v1/', 'baseUrl'), 'https://llm.example.com/v1')
})

test('unknown providers fail closed', () => {
	assert.throws(() => parseProviderId('kent-personal'), /openai/)
})

test('manifest is public MIT utilities with no jobs', () => {
	const manifest = JSON.parse(readFileSync(join(root, 'package.json'), 'utf8'))
	assert.equal(manifest.private, false)
	assert.equal(manifest.license, 'MIT')
	assert.equal(manifest.name, '@kody/ai')
	assert.equal(manifest.kody.id, 'ai')
	assert.equal(manifest.kody.category, 'utilities')
	assert.ok(!manifest.kody.jobs)
	assert.ok(!manifest.kody.subscriptions)
	assert.ok((manifest.kody.description as string).length <= 200)
	assert.ok(!JSON.stringify(manifest).includes('kentcdodds'))
})

test('README Intent and prefilled secret URLs are present', () => {
	const readme = readFileSync(join(root, 'README.md'), 'utf8')
	assert.match(readme, /^## Intent$/m)
	assert.match(readme, /account\/secrets\/new\?name=openaiApiKey/)
	assert.match(readme, /account\/secrets\/new\?name=anthropicApiKey/)
	assert.match(readme, /account\/secrets\/new\?name=groqApiKey/)
	assert.match(readme, /account\/secrets\/new\?name=cloudflareApiToken/)
	assert.match(readme, /kody:@kody\/ai/)
	assert.doesNotMatch(readme, /@kentcdodds\/ai/)
	assert.doesNotMatch(readme, /cloudflareAiModel/)
	assert.doesNotMatch(readme, /mistral-small-3\.1-24b-instruct/)
})

test('source does not bake personal keys or Kent package ids', () => {
	const files = [
		'src/complete.ts',
		'src/providers.ts',
		'src/settings.ts',
		'src/index.ts',
		'src/turn.ts',
		'src/model-step.ts',
	]
	for (const file of files) {
		const text = readFileSync(join(root, file), 'utf8')
		assert.doesNotMatch(text, /sk-[a-zA-Z0-9]/)
		assert.doesNotMatch(text, /@kentcdodds/)
		assert.doesNotMatch(text, /value_get/)
		assert.doesNotMatch(text, /migrateAiFromValues/)
		assert.doesNotMatch(text, /ai-gateway\/gateways/)
	}
})