Skip to content

src/scrape.ts

366 lines · 10.9 KB · TypeScript
Raw
import type { AqiStandard, AirQualitySnapshot, PlaceHit, PollenSnapshot, Pollutants, WeatherSnapshot } from './types'

const UA =
	'Mozilla/5.0 (compatible; @reng/air-quality; +https://kody.run) AppleWebKit/537.36 Chrome/120.0.0.0 Safari/537.36'

const POLLUTANT_NAMES = new Set(['PM2.5', 'PM10', 'O3', 'NO2', 'CO', 'SO2'])

const WIND_DIRECTIONS = [
	'北',
	'北东北',
	'东北',
	'东东北',
	'东',
	'东东南',
	'东南',
	'南东南',
	'南',
	'南西南',
	'西南',
	'西西南',
	'西',
	'西西北',
	'西北',
	'北西北',
]

const POLLEN_TYPES: Record<string, keyof Omit<PollenSnapshot, 'raw' | 'max'>> = {
	花粉: 'total',
	桦木花粉: 'birch',
	桦树花粉: 'birch',
	草花粉: 'grass',
	禾草花粉: 'grass',
	桤木: 'alder',
	桤木花粉: 'alder',
	艾桤木: 'alder',
	艾桤木花粉: 'alder',
	橄榄树花粉: 'olive',
	橄榄花粉: 'olive',
	豚草花粉: 'ragweed',
	豚草: 'ragweed',
	艾蒿花粉: 'mugwort',
	艾蒿: 'mugwort',
	艾草花粉: 'mugwort',
	艾草: 'mugwort',
	野草花粉: 'ragweed',
	过敏风险指数: 'allergyRisk',
	过敏风险: 'allergyRisk',
}

const NUM_PATTERN = /(\d+(?:\.\d+)?)/

export const AQI_STANDARDS: Record<AqiStandard, string> = {
	aqi_cn: 'AQI (中国标准)',
	aqi_us: 'AQI (美国标准)',
	aqi_au: 'AQC (澳大利亚标准)',
	aqi_ca: 'AQHI (加拿大标准)',
	aqi_nl: 'AQI (荷兰标准)',
	caqi_eu: 'CAQI (欧洲标准)',
	daqi_uk: 'AQI (英国标准)',
	naqi_in: 'AQI (印度标准)',
}

function toFloat(value: unknown, fallback: number | null = null): number | null {
	if (value == null || value === '') return fallback
	const n = typeof value === 'number' ? value : Number(String(value).replace(/[^\d.+-]/g, ''))
	return Number.isFinite(n) ? n : fallback
}

function toInt(value: unknown, fallback: number | null = null): number | null {
	const n = toFloat(value, null)
	return n == null ? fallback : Math.trunc(n)
}

function levelFromAqiCn(aqi: number | null): string | null {
	if (aqi == null) return null
	if (aqi <= 50) return '优'
	if (aqi <= 100) return '良'
	if (aqi <= 150) return '轻度污染'
	if (aqi <= 200) return '中度污染'
	if (aqi <= 300) return '重度污染'
	return '严重污染'
}

export function buildPlacePath(hit: {
	url_key?: string
	place_id?: string
	urlKey?: string
	placeId?: string
}): string {
	const urlKey = (hit.url_key ?? hit.urlKey ?? '').replace(/^\/+|\/+$/g, '')
	const placeId = (hit.place_id ?? hit.placeId ?? '').replace(/^\/+|\/+$/g, '')
	if (urlKey && placeId) return `${urlKey}/${placeId}`
	return urlKey || placeId
}

export function buildPageUrl(place: string, standard: AqiStandard): string {
	const clean = place.replace(/^\/+|\/+$/g, '')
	const parts = clean.split('/')
	if (parts.length === 2) {
		return `https://air-quality.com/country/${clean}?lang=zh-Hans&standard=${standard}`
	}
	return `https://air-quality.com/place/${clean}?lang=zh-Hans&standard=${standard}`
}

type SearchApiHit = {
	name?: string
	description?: string
	place_id?: string
	url_key?: string
	type?: string
	lat?: number
	lon?: number
}

export async function searchPlaces(term: string, limit = 20): Promise<PlaceHit[]> {
	const q = term.trim()
	if (!q) throw new Error('Provide a non-empty search term')
	const url = `https://air-quality.com/data/search_places?term=${encodeURIComponent(q)}&lang=zh-Hans`
	const res = await fetch(url, {
		headers: { 'user-agent': UA, accept: 'application/json,text/plain,*/*' },
	})
	if (!res.ok) throw new Error(`Place search failed: HTTP ${res.status}`)
	const raw = (await res.json()) as SearchApiHit[]
	if (!Array.isArray(raw) || raw.length === 0) throw new Error(`No places found for: ${q}`)
	return raw.slice(0, Math.max(1, Math.min(50, limit))).map((r) => {
		const place = buildPlacePath(r)
		return {
			name: r.name || place,
			description: r.description,
			place,
			placeId: (r.place_id || '').replace(/^\/+|\/+$/g, ''),
			urlKey: (r.url_key || '').replace(/^\/+|\/+$/g, ''),
			type: r.type,
			lat: typeof r.lat === 'number' ? r.lat : undefined,
			lon: typeof r.lon === 'number' ? r.lon : undefined,
		}
	})
}

function pickBestHit(hits: PlaceHit[], query: string): PlaceHit {
	const q = query.trim().toLowerCase()
	const locality = hits.find((h) => h.type === 'locality')
	if (locality) return locality
	const exact = hits.find((h) => h.name.toLowerCase() === q)
	if (exact) return exact
	return hits[0]!
}

function emptyPollen(): PollenSnapshot {
	return {
		raw: null,
		max: null,
		total: null,
		birch: null,
		grass: null,
		alder: null,
		olive: null,
		ragweed: null,
		mugwort: null,
		allergyRisk: null,
	}
}

function emptyPollutants(): Pollutants {
	return { pm25: null, pm10: null, o3: null, no2: null, co: null, so2: null }
}

function extractCurWeather(html: string): { windDegrees: number | null; uv: number | null; humidityPct: number | null; tempC: number | null; windKph: number | null } {
	const startMarker = 'var curWeatherData = '
	const startIdx = html.indexOf(startMarker)
	if (startIdx === -1) {
		return { windDegrees: null, uv: null, humidityPct: null, tempC: null, windKph: null }
	}
	const jsonStart = startIdx + startMarker.length
	const remaining = html.slice(jsonStart).replace(/^\s+/, '')
	if (remaining.startsWith('null')) {
		return { windDegrees: null, uv: null, humidityPct: null, tempC: null, windKph: null }
	}
	let brace = 0
	let jsonEnd = -1
	for (let i = jsonStart; i < html.length; i++) {
		const ch = html[i]
		if (ch === '{') brace++
		else if (ch === '}') {
			brace--
			if (brace === 0) {
				jsonEnd = i + 1
				break
			}
		}
	}
	if (jsonEnd === -1) {
		return { windDegrees: null, uv: null, humidityPct: null, tempC: null, windKph: null }
	}
	try {
		const weather = JSON.parse(html.slice(jsonStart, jsonEnd).replace(/;+\s*$/, '')) as Record<
			string,
			unknown
		>
		const windDegrees = toFloat(weather.wind_degrees)
		const uv = toFloat(weather.UV ?? weather.uv)
		const humidityPct = toFloat(weather.humidity)
		const tempC = toFloat(weather.temp_c)
		const windKph = toFloat(weather.wind_kph)
		return { windDegrees, uv, humidityPct, tempC, windKph }
	} catch {
		return { windDegrees: null, uv: null, humidityPct: null, tempC: null, windKph: null }
	}
}

export function parseAirQualityHtml(html: string): {
	aqi: number | null
	level: string | null
	pollutants: Pollutants
	pollen: PollenSnapshot
	weather: WeatherSnapshot
	updateTime: string | null
	warnings: string[]
} {
	const warnings: string[] = []
	if (html.includes('1970-01-01') && html.length < 25000) {
		throw new Error('Skeleton page returned; place_id may be invalid or data temporarily unavailable')
	}

	let aqiMatch = html.match(/Gauge\([\s\S]+?value:\s*(\d+)\s*\}/)
	if (!aqiMatch) {
		aqiMatch = html.match(
			/AQI \((?:中国|中国香港|美国|澳大利亚|加拿大|英国|欧盟|法国|德国|意大利|西班牙|荷兰|波兰|俄罗斯|印度)标准\)\s*(\d+)/,
		)
	}
	const aqi = aqiMatch ? toInt(aqiMatch[1]) : null
	const level = levelFromAqiCn(aqi)

	const pollutants = emptyPollutants()
	const pollen = emptyPollen()
	let pollenRaw: string | null = null
	let pollenMax: number | null = null

	const pairRe =
		/<div class=['"]name['"]>([^<]+)<\/div>[\s\S]*?<div class=['"]value['"]>([^<]+)<\/div>/g
	for (const m of html.matchAll(pairRe)) {
		const name = m[1]!.trim()
		const value = m[2]!.trim()
		if (POLLUTANT_NAMES.has(name)) {
			const key = name.toLowerCase().replace('.', '') as keyof Pollutants
			const num = value.match(NUM_PATTERN)
			pollutants[key] = num ? toFloat(num[1]) : null
			continue
		}
		const pollenKey = POLLEN_TYPES[name]
		if (!pollenKey) continue
		const range = value.match(/(\d+)\s*~\s*(\d+)/)
		const num = !range ? value.match(NUM_PATTERN) : null
		let parsed: number | string | null = value
		if (range) parsed = Number(range[2])
		else if (num) parsed = toFloat(num[1])
		pollen[pollenKey] = parsed
		if (pollenKey === 'total' && pollenRaw == null) {
			pollenRaw = value
			if (range) pollenMax = Number(range[2])
			else if (num) pollenMax = toFloat(num[1])
		}
	}
	pollen.raw = pollenRaw
	pollen.max = pollenMax

	let updateTime: string | null = null
	const timeMatch = html.match(/<div[^>]*class=['"]update-time['"][^>]*>\s*([\d\-: T+Z]+)\s*<\/div>/)
	if (timeMatch) {
		updateTime = timeMatch[1]!.trim()
	}

	const tempMatch = html.match(/<div class=['"]temperature['"]>\s*(-?\d+(?:\.\d+)?)[^<]*<\/div>/)
	const humMatch = html.match(/<div class=['"]humidity['"]>\s*(\d+(?:\.\d+)?)\s*%<\/div>/)
	const windMatch = html.match(/<div class=['"]wind['"]>\s*(\d+(?:\.\d+)?)\s*kph<\/div>/)
	const cur = extractCurWeather(html)

	let uv = cur.uv
	if (uv == null) {
		const uvMatch = html.match(/<div class=['"]uv['"]>\s*(\d+(?:\.\d+)?)\s+of\s+\d+\s*<\/div>/)
		if (uvMatch) uv = toFloat(uvMatch[1])
	}

	const windDegrees = cur.windDegrees
	let windDirection: string | null = null
	if (windDegrees != null) {
		const idx = Math.round(windDegrees / 22.5) % 16
		windDirection = WIND_DIRECTIONS[idx] ?? null
	}

	const weather: WeatherSnapshot = {
		temperatureC: tempMatch ? toFloat(tempMatch[1]) : cur.tempC,
		humidityPct: humMatch ? toFloat(humMatch[1]) : cur.humidityPct,
		windSpeedKph: windMatch ? toFloat(windMatch[1]) : cur.windKph,
		windDirection,
		windDegrees,
		uvIndex: uv,
	}

	const values = [
		aqi,
		...Object.values(pollutants),
		pollen.raw,
		weather.temperatureC,
		weather.humidityPct,
		weather.windSpeedKph,
		weather.uvIndex,
	]
	if (values.every((v) => v == null)) {
		warnings.push('All parsed fields empty; upstream HTML structure may have changed')
	}

	return { aqi, level, pollutants, pollen, weather, updateTime, warnings }
}

export async function fetchAndParsePlace(
	place: string,
	standard: AqiStandard,
): Promise<{
	url: string
	htmlLength: number
	parsed: ReturnType<typeof parseAirQualityHtml>
}> {
	const url = buildPageUrl(place, standard)
	const res = await fetch(url, {
		headers: {
			'user-agent': UA,
			accept: 'text/html,application/xhtml+xml',
			'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8',
		},
	})
	if (!res.ok) throw new Error(`air-quality.com fetch failed: HTTP ${res.status} for ${url}`)
	const html = await res.text()
	return { url, htmlLength: html.length, parsed: parseAirQualityHtml(html) }
}

export function buildSummary(s: {
	placeName: string
	aqi: number | null
	level: string | null
	pollutants: Pollutants
	pollen: PollenSnapshot
	weather: WeatherSnapshot
}): string {
	const parts: string[] = []
	parts.push(`${s.placeName}`)
	if (s.aqi != null) parts.push(`AQI ${s.aqi}${s.level ? `(${s.level})` : ''}`)
	const p = s.pollutants
	const pol = [
		p.pm25 != null ? `PM2.5 ${p.pm25}` : null,
		p.pm10 != null ? `PM10 ${p.pm10}` : null,
		p.o3 != null ? `O3 ${p.o3}` : null,
	].filter(Boolean)
	if (pol.length) parts.push(pol.join(', '))
	if (s.pollen.raw) parts.push(`花粉 ${s.pollen.raw}`)
	else if (s.pollen.allergyRisk != null) parts.push(`过敏风险 ${s.pollen.allergyRisk}`)
	if (s.weather.temperatureC != null) {
		parts.push(
			`${s.weather.temperatureC}°C` +
				(s.weather.humidityPct != null ? ` / 湿度 ${s.weather.humidityPct}%` : ''),
		)
	}
	return parts.join(' · ')
}

export { pickBestHit, emptyPollen, emptyPollutants, levelFromAqiCn }