#!/usr/bin/env node // Upload a big local file in bounded segments to X media storage (this release: synthetic tests only) // (Media Studio when --media-category amplify_video) through Kody local execute // and @kentcdodds/x/upload-media. Media only: this script never creates a post. // // Usage: // node upload-to-x-media.mjs --media-type video/mp4 [options] // // Options: // --media-type Required (video/mp4, image/jpeg, ...) // --media-category amplify_video (Media Studio), tweet_video, ... (default from type) // --account X account alias (default: the "x" integration) // --thumbnail After processing, upload this image and set it as the // video preview image via POST /2/media/metadata // --alt-text Optional alt text set with the metadata call // --chunk-mib MiB per APPEND segment, 1 to 7.5 (default 6); CLI >=3.33.1 required. // --confirm Required to upload (otherwise acts like --dry-run) // --dry-run Print the plan, upload nothing // --fresh Ignore saved resume state // --use-checkout Resolve the package from this script's checkout instead of the published version // // Needs Node 22+ and a signed-in Kody CLI. Override the CLI with KODY_CLI // (default: "npx -y @kodycodes/cli@3.33.1"). // // Stdout: one JSON line ({ ok, mediaId, mediaKey, state, ... }). Progress goes to stderr. // Resume state: ~/.cache/kody-x-media-upload/ (media id + next segment). Rerun the // same command to resume after an interruption. import { checkpointWriter, lockCheckpoint } from './lib/checkpoint.mjs' import { spawn, spawnSync } from 'node:child_process' import { createHash, randomBytes } from 'node:crypto' import { createReadStream } from 'node:fs' import { mkdir, mkdtemp, readFile, rm, stat, unlink, writeFile } from 'node:fs/promises' import { createServer } from 'node:http' import { homedir, tmpdir } from 'node:os' import { dirname, extname, join, resolve } from 'node:path' import { fileURLToPath } from 'node:url' const MIB = 1024 * 1024 const PACKAGE_NAME = '@kentcdodds/x' const MAX_FAILED_ATTEMPTS = 8 const STALL_MS = Number(process.env.KODY_X_UPLOAD_STALL_MS ?? 8 * 60 * 1000) const STATE_DIR = join(homedir(), '.cache', 'kody-x-media-upload') function log(message) { process.stderr.write(`[upload-to-x-media] ${message}\n`) } function fail(message, extra = {}, code = 1) { process.stdout.write(`${JSON.stringify({ ok: false, error: message, ...extra })}\n`) process.exit(code) } function parseArgs(argv) { const options = { chunkMib: 6, fresh: false, dryRun: false, confirm: false, useCheckout: false } const positional = [] for (let index = 0; index < argv.length; index += 1) { const arg = argv[index] if (arg === '--media-type') options.mediaType = argv[++index] else if (arg === '--media-category') options.mediaCategory = argv[++index] else if (arg === '--account') options.account = argv[++index] else if (arg === '--thumbnail') options.thumbnail = argv[++index] else if (arg === '--alt-text') options.altText = argv[++index] else if (arg === '--chunk-mib') options.chunkMib = Number(argv[++index]) else if (arg === '--fresh') options.fresh = true else if (arg === '--dry-run') options.dryRun = true else if (arg === '--confirm') options.confirm = true else if (arg === '--use-checkout') options.useCheckout = true else if (arg === '-h' || arg === '--help') options.help = true else if (arg.startsWith('--')) fail(`Unknown option ${arg}`) else positional.push(arg) } return { options, positional } } // Same plan as src/multipart.ts planMediaSegments (kept in sync; covered by tests there). function planSegments(totalBytes,chunkBytes) { const out=[];for(let start=0;start { const r=await fetch(p.reportUrl,{method:'POST',body:JSON.stringify(state)});await r.body?.cancel();if(!r.ok)throw new Error('Checkpoint persistence failed') } const readRange = async (path, start, end) => { const response = await fetch(p.baseUrl + path, { headers: { range: 'bytes=' + start + '-' + (end - 1) } }) const bytes = new Uint8Array(await response.arrayBuffer()) if (response.status !== 206 || bytes.byteLength !== end - start) { throw new Error('local source returned ' + response.status + ' with ' + bytes.byteLength + ' bytes') } return bytes } const withRetry = async (label, fn) => { for (let attempt = 1; ; attempt += 1) { try { return await fn() } catch (error) { const status = typeof error?.status === 'number' ? error.status : 0 const retryable = error?.name !== 'AbortError' && ((status === 0 && !/Request body exceeds|credential\\(\\) client unavailable/.test(String(error?.message))) || status === 429 || status >= 500) if (!retryable || attempt >= 4) throw error await report({ note: label + ' retry ' + attempt + ' (status ' + status + '): ' + String(error?.message ?? error).slice(0, 200) }) await new Promise((r) => setTimeout(r, Math.max((status === 429 ? 15000 : 3000) * attempt, Number.isFinite(error?.retryAfterMs)?error.retryAfterMs:0))) } } } const describe = (error) => { let details = '' try { details = JSON.stringify(error?.details ?? null).slice(0, 800) } catch {} return String(error?.message ?? error) + (details && details !== 'null' ? ' details=' + details : '') } try { let s = p.state const me=await getMediaAccountIdentity(acct);const accountId=me.accountId;if(!accountId)throw new Error('Account identity unavailable') if(s && (s.accountId!==accountId||s.sourceHash!==p.sourceHash))throw new Error('RECONCILIATION_REQUIRED: account/source changed') if(s?.phase==='starting')throw new Error('RECONCILIATION_REQUIRED: initial response lost') if (!s || !s.mediaId) { await report({state:{phase:'starting',accountId,sourceHash:p.sourceHash}}) const init = await initializeMediaUpload({ ...acct, mediaType: p.mediaType, mediaCategory: p.mediaCategory, totalBytes: p.size, confirm: true }) s = { accountId,sourceHash:p.sourceHash,phase:'uploading',mediaId: init.mediaId, mediaKey: init.mediaKey ?? null, nextSegment: 0, nextOffset: 0, finalized: false, expiresAfterSecs: init.expiresAfterSecs ?? null, integration: init.integration } await report({ state: s }) } let chunk = p.chunkBytes let offset = s.nextOffset ?? 0 let index = s.nextSegment ?? 0 while (offset < p.size) { let size = Math.min(chunk, p.size - offset) const bytes = await readRange('/source', offset, offset + size) try { await withRetry('append ' + index, () => appendMediaUploadSegment({ ...acct, mediaId: s.mediaId, segmentIndex: index, bytes, mediaType: p.mediaType, confirm: true }), ) } catch(error){throw error} offset += size index += 1 s = { ...s, nextSegment: index, nextOffset: offset } await report({ state: s, progress: { segment: index, bytes: offset, chunk } }) } let processing if (s.phase === 'finalizing') { processing = await getMediaUploadStatus({ ...acct,mediaId:s.mediaId }) if (!processing.processingInfo || !['pending','in_progress','succeeded','failed'].includes(processing.state)) throw new Error('RECONCILIATION_REQUIRED: finalize outcome unverified') s={...s,finalized:true,phase:'finalized'};await report({state:s}) } else if (!s.finalized) { s={...s,phase:'finalizing'};await report({state:s}) try { processing=await finalizeMediaUpload({...acct,mediaId:s.mediaId,confirm:true}) } catch { throw new Error('RECONCILIATION_REQUIRED: finalize outcome uncertain; resume checks STATUS only') } s={...s,finalized:true,phase:'finalized',mediaKey:processing.mediaKey??s.mediaKey};await report({state:s}) } else {processing=await withRetry('status',()=>getMediaUploadStatus({...acct,mediaId:s.mediaId}))} while (processing.state === 'pending' || processing.state === 'in_progress') { const pct = processing.processingInfo?.progress_percent await report({ note: 'processing ' + processing.state + (pct != null ? ' ' + pct + '%' : '') }) processing = await waitForMediaProcessing({ ...acct, mediaId: s.mediaId, maxPolls: 1, initial: processing }).catch((error) => { if (/still (pending|in_progress)/.test(String(error?.message))) return getMediaUploadStatus({ ...acct, mediaId: s.mediaId }) throw error }) } if (processing.state === 'failed') throw new Error('processing failed: ' + JSON.stringify(processing.processingInfo)) const result = { done: true, segments: s.nextSegment, mediaId: s.mediaId, mediaKey: processing.mediaKey ?? s.mediaKey, state: processing.state, integration: s.integration, thumbnail: null, managed: null } if (p.thumbnail) { try { const tb = await readRange('/thumbnail', 0, p.thumbnail.size) const ti = await initializeMediaUpload({ ...acct, mediaType: p.thumbnail.mediaType, mediaCategory: 'tweet_image', totalBytes: tb.byteLength, confirm: true }) await appendMediaUploadSegment({ ...acct, mediaId: ti.mediaId, segmentIndex: 0, bytes: tb, mediaType: p.thumbnail.mediaType, confirm: true }) const tf = await finalizeMediaUpload({ ...acct, mediaId: ti.mediaId, confirm: true }) const td = await waitForMediaProcessing({ ...acct, mediaId: ti.mediaId, initial: tf }) const meta = await setMediaMetadata({ ...acct, mediaId: s.mediaId, previewImageMediaId: ti.mediaId, ...(p.altText ? { altText: p.altText } : {}), confirm: true }) result.thumbnail = { ok: true, imageMediaId: ti.mediaId, imageMediaKey: td.mediaKey ?? ti.mediaKey ?? null, associatedMetadata: meta.associatedMetadata } } catch (error) { result.thumbnail = { ok: false, error: describe(error) } } } if (p.mediaCategory === 'amplify_video') { try { const meta = await setMediaMetadata({ ...acct, mediaId: s.mediaId, managed: true, confirm: true }) result.managed = { ok: true, associatedMetadata: meta.associatedMetadata } } catch (error) { result.managed = { ok: false, error: describe(error) } } } await report(result) return result } catch (error) { const message = describe(error) await report({ terminal:error?.name==='AbortError',error: message, status: typeof error?.status === 'number' ? error.status : null }) throw new Error(message) } } ` } function startLocalServer(files, onReport) { const token = randomBytes(16).toString('hex') const server = createServer(async (req, res) => { const name = req.method === 'GET' && req.url?.startsWith(`/${token}/`) ? req.url.slice(token.length + 2) : null const file = name ? files[name] : null if (file) { const current=await stat(file.path).catch(()=>null);if(!current||current.size!==file.size||current.mtimeMs!==file.mtimeMs||current.ino!==file.ino)return void res.writeHead(409).end() const match = /^bytes=(\d+)-(\d+)$/.exec(req.headers.range ?? '') if (!match) return void res.writeHead(416).end() const start = Number(match[1]) const end = Math.min(Number(match[2]), file.size - 1) if (start > end) return void res.writeHead(416).end() res.writeHead(206, { 'content-type': 'application/octet-stream', 'content-length': String(end - start + 1), 'content-range': `bytes ${start}-${end}/${file.size}`, }) createReadStream(file.path, { start, end }).pipe(res) return } if (req.method === 'POST' && req.url === `/${token}/report`) { let body = '' req.on('data', (chunk) => (body += chunk)) req.on('end', () => { try { Promise.resolve(onReport(JSON.parse(body))).then(()=>res.writeHead(204).end(),()=>res.writeHead(500).end());return } catch {} res.writeHead(204).end() }) return } res.writeHead(404).end() }) return new Promise((resolvePromise) => { server.listen(0, '127.0.0.1', () => { const { port } = server.address() resolvePromise({ server, baseUrl: `http://127.0.0.1:${port}/${token}`, reportUrl: `http://127.0.0.1:${port}/${token}/report` }) }) }) } function integrationFor(account) { const a = String(account ?? '').trim().toLowerCase() if (!a || a === 'default' || a === 'x') return 'x' return a.startsWith('x-') ? a : `x-${a}` } function runExecute({ cliCommand, cwd, file, params, onStall }) { const [command, ...baseArgs] = cliCommand // Pick the credential for the package's literal need key so credential() is never // ambiguous (needed for the >4 MiB streaming fetch path). Override with KODY_X_CREDENTIAL. const slot = integrationFor(params.account) const pick = process.env.KODY_X_CREDENTIAL ?? slot const args = [...baseArgs, 'execute', '--local', '--allow-private-network', '--credential', `${slot}=${pick}`, '--file', file, '--params', JSON.stringify(params)] return new Promise((resolvePromise) => { const child = spawn(command, args, { cwd, detached: true, stdio: ['ignore', 'pipe', 'pipe'] }) let stdout = '' let stderr = '' child.stdout.on('data', (chunk) => (stdout += chunk)) child.stderr.on('data', (chunk) => (stderr += chunk)) onStall.kill = () => { try { process.kill(-child.pid, 'SIGKILL') } catch {} } child.on('close', (code) => resolvePromise({ code, stdout, stderr })) child.on('error', (error) => resolvePromise({ code: -1, stdout, stderr: String(error) })) }) } function imageType(path) { const ext = extname(path).toLowerCase() if (ext === '.png') return 'image/png' if (ext === '.webp') return 'image/webp' return 'image/jpeg' } async function main() { const { options, positional } = parseArgs(process.argv.slice(2)) if (options.help || positional.length !== 1 || !options.mediaType) { process.stderr.write( 'Usage: node upload-to-x-media.mjs --media-type video/mp4 [--media-category amplify_video] [--account ] [--thumbnail ] [--alt-text ] [--chunk-mib 6] [--confirm] [--dry-run] [--fresh] [--use-checkout]\n', ) process.exit(options.help ? 0 : 1) } if (!Number.isFinite(options.chunkMib) || options.chunkMib < 1 || options.chunkMib > 7.5) fail('--chunk-mib must be between 1 and 7.5.') const [nodeMajor] = process.versions.node.split('.').map(Number) if (nodeMajor < 22) fail(`Node 22+ is required (found ${process.versions.node}).`) const localPath = resolve(positional[0]) const fileStat = await stat(localPath).catch(() => null) if (!fileStat?.isFile()) fail(`Local file not found: ${localPath}`) const size = fileStat.size let thumbnail = null if (options.thumbnail) { const thumbPath = resolve(options.thumbnail) const thumbStat = await stat(thumbPath).catch(() => null) if (!thumbStat?.isFile()) fail(`Thumbnail not found: ${thumbPath}`) if (thumbStat.size > 2.5 * MIB) fail('Thumbnail must be under 2.5 MiB (single APPEND under the local execute hop).') const th=createHash('sha256');for await(const b of createReadStream(thumbPath))th.update(b) thumbnail = { path: thumbPath, size: thumbStat.size, mediaType: imageType(thumbPath),mtimeMs:thumbStat.mtimeMs,ino:thumbStat.ino,sha256:th.digest('hex') } } const mediaType = options.mediaType const mediaCategory = options.mediaCategory ?? null const chunkBytes = Math.round(options.chunkMib * MIB) const segments = planSegments(size, chunkBytes) const legacyOffset = (n) => (n ? planSegments(size, Math.round((state?.chunkBytes ?? 6 * MIB))).slice(0, n).at(-1).end : 0) if (segments.length > 10000) fail('Too many segments; raise --chunk-mib.') const hash=createHash('sha256');for await(const part of createReadStream(localPath)){hash.update(part)} const sourceHash=hash.digest('hex') const stateKey = createHash('sha256') .update(JSON.stringify([localPath, size, sourceHash, mediaType, mediaCategory, options.account ?? 'x',thumbnail,options.altText??null])) .digest('hex') .slice(0, 32) const statePath = join(STATE_DIR, `${stateKey}.json`) const legacyKey=createHash('sha256').update(JSON.stringify([localPath,size,fileStat.mtimeMs,mediaType,mediaCategory,options.account??'x'])).digest('hex').slice(0,32) if(!options.fresh){try{await stat(join(STATE_DIR,legacyKey+'.json'));throw new Error('RECONCILIATION_REQUIRED: legacy checkpoint exists; inspect before starting a replacement')}catch(e){if(e.code!=='ENOENT')throw e}} const releaseLock=await lockCheckpoint(statePath) let state = null if (!options.fresh) { try { state = JSON.parse(await readFile(statePath, 'utf8')) const ttl = (state.expiresAfterSecs ?? 24 * 3600) * 1000 if (Date.now() - state.createdAt > ttl - 10 * 60 * 1000) throw new Error('RECONCILIATION_REQUIRED: checkpoint expired') } catch(error) { if(error.code!=='ENOENT')throw new Error('RECONCILIATION_REQUIRED: checkpoint expired or unreadable') } } if (options.dryRun || !options.confirm) { process.stdout.write( `${JSON.stringify({ ok: true, dryRun: true, requiresConfirm: true, localPath, size, sourceHash, mediaType, mediaCategory, account: options.account ?? null, chunkBytes, segments: segments.length, thumbnail: thumbnail ? { path: thumbnail.path, size: thumbnail.size, mediaType: thumbnail.mediaType } : null, resumeFrom: state ? { mediaId: state.mediaId, nextSegment: state.nextSegment, nextOffset: state.nextOffset ?? null, finalized: state.finalized } : null, note: 'Media upload only; never creates a post.', })}\n`, ) releaseLock();return } await mkdir(STATE_DIR, { recursive: true, mode: 0o700 }) const save=checkpointWriter(statePath) const saveState=()=>save(state) if (state) log(`resuming media ${state.mediaId} at segment ${state.nextSegment}${state.finalized ? ' (finalized)' : ''}`) const started = Date.now() let lastProgressAt = Date.now() let done = null let lastReportedError = null let resumeBytes = null const files = { source: { path: localPath, size,mtimeMs:fileStat.mtimeMs,ino:fileStat.ino }, ...(thumbnail ? { thumbnail } : {}) } const { server, baseUrl, reportUrl } = await startLocalServer(files, async (message) => { lastProgressAt = Date.now() if (message.note) log(message.note) if (message.error) { lastReportedError = message log(`error: ${message.error}`) } if (message.state) { const isNew = !state || state.mediaId !== message.state.mediaId state = { ...message.state, createdAt: isNew ? Date.now() : state.createdAt } await saveState() } if (message.progress) { const elapsed = (Date.now() - started) / 1000 const mib = message.progress.bytes / MIB resumeBytes ??= message.progress.bytes - message.progress.chunk log(`segment ${message.progress.segment} done (${mib.toFixed(1)} of ${(size / MIB).toFixed(1)} MiB, ${(message.progress.chunk / MIB).toFixed(2)} MiB segments, ${elapsed.toFixed(0)}s, ${((mib - resumeBytes / MIB) / Math.max(elapsed, 1)).toFixed(2)} MiB/s)`) } if (message.done) done = message }) const cliCommand = (process.env.KODY_CLI ?? 'npx -y @kodycodes/cli@3.33.1').split(/\s+/).filter(Boolean) const vr=spawnSync(cliCommand[0],[...cliCommand.slice(1),'--version'],{encoding:'utf8'});const vm=/(\d+)\.(\d+)\.(\d+)/.exec(vr.stdout??'');if(vr.status!==0||!vm||(+vm[1]<3||(+vm[1]===3&&(+vm[2]<33||(+vm[2]===33&&+vm[3]<1)))))throw new Error('CLI >=3.33.1 required') const scriptDir = dirname(fileURLToPath(import.meta.url)) const workDir = await mkdtemp(join(tmpdir(), 'kody-x-media-upload-')) let cwd = workDir if (options.useCheckout) { const checkout = resolve(scriptDir, '..') const pkg = JSON.parse(await readFile(join(checkout, 'package.json'), 'utf8')) if (pkg.name !== PACKAGE_NAME) fail(`--use-checkout: ${checkout} is not a ${PACKAGE_NAME} checkout.`) cwd = checkout } const cleanup = async () => { releaseLock();server.close() await rm(workDir, { recursive: true, force: true }) } const onStall = { kill: () => {} } const interrupt = (signal) => { log(`${signal}: stopping; rerun the same command to resume`) onStall.kill() void cleanup().finally(() => process.exit(130)) } process.on('SIGINT', () => interrupt('SIGINT')) process.on('SIGTERM', () => interrupt('SIGTERM')) let attempts = 0 let failedAttempts = 0 let lastError = '' const watchdog = setInterval(() => { if (Date.now() - lastProgressAt > STALL_MS) { log(`no progress for ${Math.round(STALL_MS / 1000)}s; restarting the run`) lastProgressAt = Date.now() onStall.kill() } }, 5000) while (!done && failedAttempts < MAX_FAILED_ATTEMPTS) { attempts += 1 const before = JSON.stringify([state?.mediaId, state?.nextSegment, state?.finalized]) const nonce = `${Date.now()}-${randomBytes(4).toString('hex')}` const file = join(cwd, `.upload-to-x-media-${nonce}.mjs`) await writeFile(file, buildModule(nonce)) lastProgressAt = Date.now() lastReportedError = null const result = await runExecute({ cliCommand, cwd, file, params: { baseUrl, reportUrl, size, sourceHash, mediaType, mediaCategory, account: options.account ?? null, chunkBytes, altText: options.altText ?? null, thumbnail: thumbnail ? { size: thumbnail.size, mediaType: thumbnail.mediaType } : null, state, }, onStall, }) await unlink(file).catch(() => {}) if (done) break const output = `${result.stderr}\n${result.stdout}`.trim() lastError = (lastReportedError?.error ?? output.split('\n').filter(Boolean).slice(-3).join(' | ')).slice(0, 900) if(lastReportedError?.terminal)break const status = lastReportedError?.status ?? null if (state?.mediaId && !state.finalized && (status === 400 || status === 404) && /append|session|expired|not found|invalid media/i.test(lastError)) { log('Session unavailable; checkpoint preserved for reconciliation');break } else if (/RECONCILIATION_REQUIRED|FormData|insufficient_scope|Unauthorized|\b401\b|\b403\b|Forbidden|not authenticated|not signed in|No X OAuth integration|Node 22|processing failed|Unsupported|InvalidMedia|npm install|Cannot find module|SyntaxError/i.test(lastError)) { log(`stopping: ${lastError}`) break } const progressed = JSON.stringify([state?.mediaId, state?.nextSegment, state?.finalized]) !== before failedAttempts = progressed ? 1 : failedAttempts + 1 log(`attempt ${attempts} ended (exit ${result.code}): ${lastError || 'no output'}`) await new Promise((r) => setTimeout(r, Math.min(30000, 2000 * failedAttempts))) } clearInterval(watchdog) await cleanup() if (!done) { fail(`Upload did not finish after ${attempts} attempts. Last error: ${lastError}. Rerun the same command to resume.`, { mediaId: state?.mediaId ?? null, nextSegment: state?.nextSegment ?? null, finalized: state?.finalized ?? false, }) } const afterHash=createHash('sha256');for await(const bytes of createReadStream(localPath))afterHash.update(bytes);if(afterHash.digest('hex')!==sourceHash)fail('Source changed; do not treat this media as verified. Checkpoint retained for reconciliation.') await unlink(statePath).catch(() => {}) const summary = { ok: done.state === 'succeeded', mediaId: done.mediaId, mediaKey: done.mediaKey ?? null, state: done.state, mediaType, mediaCategory, size, segments: done.segments ?? null, chunkBytes, integration: done.integration ?? null, thumbnailPath: thumbnail?.path ?? null, thumbnail: done.thumbnail, managed: done.managed, attempts, durationMs: Date.now() - started, posted: false, } process.stdout.write(`${JSON.stringify(summary)}\n`) process.exit(summary.ok ? 0 : 2) } if(process.argv[1]&&resolve(process.argv[1])===fileURLToPath(import.meta.url))main().catch(()=>fail('Upload stopped; checkpoint preserved. Reconcile uncertain outcomes before restarting.'))