#!/usr/bin/env node // Upload a local file of any size (tested to hundreds of MB) to Dropbox through // Kody local execute and @kody/dropbox/upload-session. // // Usage: // node upload-to-dropbox.mjs [options] // // Options: // --mode add|overwrite Commit mode (default: add, which refuses to replace an existing file) // --autorename With mode add, pick a free name instead of failing on a conflict // --chunk-mib MiB per append, 5 to 90 (default: 16) // --fresh Ignore saved resume state and start a new session // --dry-run Print the plan and local content_hash, upload nothing // --use-checkout Resolve the package from this script's checkout instead of the published version // // Needs Node 22+ and a signed-in Kody CLI (stored token with org:execute, // package:execute, integration:use). Override the CLI command with KODY_CLI // (default: "npx -y @kodycodes/cli@3.33.1"). // // Stdout: one JSON summary. Progress goes to stderr. Exit 0 only when the // Dropbox size and content_hash match the local file. import { checkpointWriter, lockCheckpoint } from './lib/checkpoint.mjs' import { spawn, spawnSync } from 'node:child_process' import { createHash, randomBytes } from 'node:crypto' import { createReadStream } from 'node:fs' import { mkdir, mkdtemp, readFile, rm, stat, unlink, writeFile } from 'node:fs/promises' import { createServer } from 'node:http' import { homedir, tmpdir } from 'node:os' import { dirname, join, resolve } from 'node:path' import { fileURLToPath } from 'node:url' const MIB = 1024 * 1024 const BLOCK = 4 * MIB const PACKAGE_NAME = '@kody/dropbox' const MAX_FAILED_ATTEMPTS = 8 const STALL_MS = Number(process.env.KODY_DROPBOX_UPLOAD_STALL_MS ?? 6 * 60 * 1000) const STATE_DIR = join(homedir(), '.cache', 'kody-org-dropbox-upload') function log(message) { process.stderr.write(`[upload-to-dropbox] ${message}\n`) } function fail(message, code = 1) { process.stdout.write(`${JSON.stringify({ ok: false, error: message })}\n`) process.exit(code) } function parseArgs(argv) { const options = { mode: 'add', autorename: false, chunkMib: 16, fresh: false, dryRun: false, useCheckout: false } const positional = [] for (let index = 0; index < argv.length; index += 1) { const arg = argv[index] if (arg === '--mode') options.mode = argv[++index] else if (arg === '--autorename') options.autorename = true else if (arg === '--chunk-mib') options.chunkMib = Number(argv[++index]) else if (arg === '--fresh') options.fresh = true else if (arg === '--dry-run') options.dryRun = true else if (arg === '--use-checkout') options.useCheckout = true else if (arg === '-h' || arg === '--help') options.help = true else if (arg.startsWith('--')) fail(`Unknown option ${arg}`) else positional.push(arg) } return { options, positional } } async function computeContentHash(path) { const blockHashes = [] let block = createHash('sha256') let inBlock = 0 for await (const chunk of createReadStream(path, { highWaterMark: MIB })) { let offset = 0 while (offset < chunk.length) { const take = Math.min(BLOCK - inBlock, chunk.length - offset) block.update(chunk.subarray(offset, offset + take)) inBlock += take offset += take if (inBlock === BLOCK) { blockHashes.push(block.digest()) block = createHash('sha256') inBlock = 0 } } } if (inBlock > 0) blockHashes.push(block.digest()) return createHash('sha256').update(Buffer.concat(blockHashes)).digest('hex') } // The execute module. Rewritten with a fresh nonce for every attempt so a // retry never reuses a cached module graph. export function buildModule(nonce) { return `// upload-to-dropbox attempt ${nonce} import { startUploadSession, appendUploadSession, finishUploadSession, getUploadSessionOffset, getCorrectOffsetFromError, } from 'kody:${PACKAGE_NAME}/upload-session' import { DropboxApiError, dropboxRpc } from 'kody:${PACKAGE_NAME}/request' export default async function main(p) { const report = async (state) => { const r = await fetch(p.reportUrl, { method: 'POST', body: JSON.stringify(state) }) await r.body?.cancel() if (!r.ok) throw new Error('Checkpoint persistence failed; stopped') } const readRange = async (start, end) => { if (end <= start) return new Uint8Array(0) const response = await fetch(p.sourceUrl, { headers: { range: 'bytes=' + start + '-' + (end - 1) } }) const bytes = new Uint8Array(await response.arrayBuffer()) if (response.status !== 206 || bytes.byteLength !== end - start) { throw new Error('local source returned ' + response.status + ' with ' + bytes.byteLength + ' bytes') } return bytes } const withRetry = async (label, fn) => { for (let attempt = 1; ; attempt += 1) { try { return await fn() } catch (error) { if (getCorrectOffsetFromError(error) !== null) throw error const status = error instanceof DropboxApiError ? error.status : 0 const retryable = error?.name !== 'AbortError' && (status === 0 || status === 429 || status >= 500) if (!retryable || attempt >= 3) throw error await report({ note: label + ' retry ' + attempt + ': ' + String(error?.message ?? error).slice(0, 200) }) await new Promise((r) => setTimeout(r, Math.max(2000 * attempt, Number.isFinite(error?.retryAfterMs) ? error.retryAfterMs : 0))) } } } const account = await dropboxRpc('users/get_current_account', {}) if (!account.account_id) throw new Error('Dropbox account identity unavailable; refusing upload') if (p.accountId && p.accountId !== account.account_id) throw new Error('RECONCILIATION_REQUIRED: Dropbox account changed; no upload performed') if (p.sessionId && !p.accountId) throw new Error('RECONCILIATION_REQUIRED: legacy checkpoint has no account identity') const identity = { accountId: account.account_id } const reconcileFinish = async () => { try { const metadata = await dropboxRpc('files/get_metadata', { path: p.commit.path }) if (!p.commit.autorename && metadata.size === p.size && metadata.content_hash === p.contentHash) { const result = { ...identity, sessionId: p.sessionId || 'reconciled', offset: p.size, done: true, metadata } await report(result) return result } } catch {} throw new Error('RECONCILIATION_REQUIRED: final result uncertain; inspect destination before starting another upload') } if (p.phase === 'finishing' || p.phase === 'done') return reconcileFinish() let sessionId = p.sessionId let offset = 0 if (sessionId) { offset = (await withRetry('get-offset', () => getUploadSessionOffset({ sessionId, offset: p.offset ?? 0 }))).offset } else { if (p.commit.mode === 'add' && !p.commit.autorename) { try { await dropboxRpc('files/get_metadata', { path: p.commit.path }) throw new Error('EXISTS') } catch (error) { if (error?.message === 'EXISTS') { throw new Error('Dropbox path already exists: ' + p.commit.path + '. Use --mode overwrite or --autorename.') } if (!(error instanceof DropboxApiError && error.status === 409)) throw error } } sessionId = (await startUploadSession({ confirm: true })).sessionId } if (!Number.isSafeInteger(offset) || offset < 0 || offset > p.size) throw new Error('Invalid acknowledged Dropbox offset') await report({ ...identity, sessionId, offset, phase: 'uploading' }) while (p.size - offset > p.chunkBytes) { const end = offset + p.chunkBytes const bytes = await readRange(offset, end) try { offset = (await withRetry('append', () => appendUploadSession({ sessionId, offset, bytes, confirm: true }))).offset } catch (error) { const correct = getCorrectOffsetFromError(error) if (correct === null) throw error await report({ note: 'offset corrected from ' + offset + ' to ' + correct }) offset = correct } await report({ ...identity, sessionId, offset, phase: 'uploading' }) } for (;;) { const bytes = await readRange(offset, p.size) try { await report({ ...identity, sessionId, offset, phase: 'finishing' }) const metadata = await finishUploadSession({ sessionId, offset, bytes, commit: p.commit, confirm: true }) const result = { ...identity, sessionId, offset: p.size, done: true, metadata: { id: metadata.id, name: metadata.name, path_display: metadata.path_display, rev: metadata.rev, size: metadata.size, content_hash: metadata.content_hash, server_modified: metadata.server_modified, }, } await report(result) return result } catch (error) { const correct = getCorrectOffsetFromError(error) if (correct === null || correct === offset) return reconcileFinish() await report({ note: 'finish offset corrected from ' + offset + ' to ' + correct }) offset = correct } } } ` } function startLocalServer(filePath, size, onReport) { const token = randomBytes(16).toString('hex') const server = createServer((req, res) => { if (req.method === 'GET' && req.url === `/${token}/source`) { const match = /^bytes=(\d+)-(\d+)$/.exec(req.headers.range ?? '') if (!match) { res.writeHead(416).end() return } const start = Number(match[1]) const end = Math.min(Number(match[2]), size - 1) if (start > end) { res.writeHead(416).end() return } res.writeHead(206, { 'content-type': 'application/octet-stream', 'content-length': String(end - start + 1), 'content-range': `bytes ${start}-${end}/${size}`, }) createReadStream(filePath, { start, end }).pipe(res) return } if (req.method === 'POST' && req.url === `/${token}/report`) { let body = '' req.on('data', (chunk) => (body += chunk)) req.on('end', () => { try { Promise.resolve(onReport(JSON.parse(body))).then(() => res.writeHead(204).end(), () => res.writeHead(500).end()) return } catch {} res.writeHead(204).end() }) return } res.writeHead(404).end() }) return new Promise((resolvePromise) => { server.listen(0, '127.0.0.1', () => { const { port } = server.address() resolvePromise({ server, sourceUrl: `http://127.0.0.1:${port}/${token}/source`, reportUrl: `http://127.0.0.1:${port}/${token}/report`, }) }) }) } function runExecute({ cliCommand, cwd, file, params, onStall }) { const [command, ...baseArgs] = cliCommand const args = [...baseArgs, '--org', 'kody', 'execute', '--local', '--allow-private-network', '--file', file, '--params', JSON.stringify(params)] return new Promise((resolvePromise) => { const child = spawn(command, args, { cwd, detached: true, stdio: ['ignore', 'pipe', 'pipe'] }) let stdout = '' let stderr = '' child.stdout.on('data', (chunk) => (stdout += chunk)) child.stderr.on('data', (chunk) => (stderr += chunk)) const kill = () => { try { process.kill(-child.pid, 'SIGKILL') } catch {} } onStall.kill = kill child.on('close', (code) => resolvePromise({ code, stdout, stderr })) child.on('error', (error) => resolvePromise({ code: -1, stdout, stderr: String(error) })) }) } async function main() { const { options, positional } = parseArgs(process.argv.slice(2)) if (options.help || positional.length !== 2) { process.stderr.write('Usage: node upload-to-dropbox.mjs [--mode add|overwrite] [--autorename] [--chunk-mib 16] [--fresh] [--dry-run] [--use-checkout]\n') process.exit(options.help ? 0 : 1) } const localPath = resolve(positional[0]) const dropboxPath = positional[1] if (!dropboxPath.startsWith('/')) fail('Dropbox path must start with "/" (Kody writes usually live under /Apps/Kody/).') if (options.mode !== 'add' && options.mode !== 'overwrite') fail('--mode must be add or overwrite.') if (!Number.isFinite(options.chunkMib) || options.chunkMib < 5 || options.chunkMib > 90) fail('--chunk-mib must be between 5 and 90.') const [nodeMajor] = process.versions.node.split('.').map(Number) if (nodeMajor < 22) fail(`Node 22+ is required (found ${process.versions.node}).`) let fileStat try { fileStat = await stat(localPath) } catch { fail(`Local file not found: ${localPath}`) } if (!fileStat.isFile()) fail(`Not a regular file: ${localPath}`) const size = fileStat.size const chunkBytes = Math.round(options.chunkMib * MIB) const commit = { path: dropboxPath, mode: options.mode, autorename: options.autorename } const started = Date.now() log(`hashing ${localPath} (${(size / MIB).toFixed(1)} MiB)`) const contentHash = await computeContentHash(localPath) log(`local content_hash ${contentHash}`) const stateKey = createHash('sha256') .update(JSON.stringify([localPath, size, fileStat.mtimeMs, commit, contentHash])) .digest('hex') .slice(0, 32) const statePath = join(STATE_DIR, `${stateKey}.json`) const legacyKey=createHash('sha256').update(JSON.stringify([localPath,size,fileStat.mtimeMs,dropboxPath,contentHash])).digest('hex').slice(0,32) if(!options.fresh){try{await stat(join(STATE_DIR,legacyKey+'.json'));throw new Error('RECONCILIATION_REQUIRED: legacy checkpoint exists; inspect before starting a replacement')}catch(e){if(e.code!=='ENOENT')throw e}} const releaseLock = await lockCheckpoint(statePath) let state = null if (!options.fresh) { try { state = JSON.parse(await readFile(statePath, 'utf8')) // Dropbox keeps an upload session for up to 7 days; leave a margin. if (Date.now() - state.createdAt > 6 * 24 * 60 * 60 * 1000) state = null } catch { state = null } } if (options.dryRun) { process.stdout.write( `${JSON.stringify({ ok: true, dryRun: true, localPath, dropboxPath, size, contentHash, chunkBytes, chunks: Math.max(1, Math.ceil(size / chunkBytes)), commit, resumeFrom: state ? { offset: state.offset } : null, })}\n`, ) releaseLock() return } await mkdir(STATE_DIR, { recursive: true }) const writeCheckpoint = checkpointWriter(statePath) const saveState = () => writeCheckpoint(state) if (state) log(`resuming session at offset ${state.offset} of ${size}`) let lastProgressAt = Date.now() let done = null const { server, sourceUrl, reportUrl } = await startLocalServer(localPath, size, async (message) => { lastProgressAt = Date.now() if (message.note) { log(message.note) return } if (message.sessionId) { const isNew = !state || state.sessionId !== message.sessionId state = { sessionId: message.sessionId, accountId: message.accountId ?? state?.accountId, phase: message.phase ?? (message.done ? 'done' : state?.phase), offset: message.offset, createdAt: isNew ? Date.now() : state.createdAt, } await saveState() const elapsed = (Date.now() - started) / 1000 log(`${((100 * message.offset) / Math.max(size, 1)).toFixed(1)}% (${(message.offset / MIB).toFixed(1)} of ${(size / MIB).toFixed(1)} MiB, ${elapsed.toFixed(0)}s)`) } if (message.done) done = message }) const cliCommand = (process.env.KODY_CLI ?? 'npx -y @kodycodes/cli@3.33.1').split(/\s+/).filter(Boolean) assertCliVersion(cliCommand) const scriptDir = dirname(fileURLToPath(import.meta.url)) const workDir = await mkdtemp(join(tmpdir(), 'kody-dropbox-upload-')) let cwd = workDir if (options.useCheckout) { const checkout = resolve(scriptDir, '..') const pkg = JSON.parse(await readFile(join(checkout, 'package.json'), 'utf8')) if (pkg.name !== PACKAGE_NAME) fail(`--use-checkout: ${checkout} is not a ${PACKAGE_NAME} checkout.`) cwd = checkout } const cleanup = async () => { releaseLock() server.close() await rm(workDir, { recursive: true, force: true }) } const onStall = { kill: () => {} } const interrupt = (signal) => { log(`${signal}: stopping; rerun the same command to resume`) onStall.kill() void cleanup().finally(() => process.exit(130)) } process.on('SIGINT', () => interrupt('SIGINT')) process.on('SIGTERM', () => interrupt('SIGTERM')) let attempts = 0 let failedAttempts = 0 let lastError = '' const watchdog = setInterval(() => { if (Date.now() - lastProgressAt > STALL_MS) { log(`no progress for ${Math.round(STALL_MS / 1000)}s; restarting the run`) lastProgressAt = Date.now() onStall.kill() } }, 5000) while (!done && failedAttempts < MAX_FAILED_ATTEMPTS) { attempts += 1 const offsetBefore = state?.offset ?? -1 const nonce = `${Date.now()}-${randomBytes(4).toString('hex')}` const file = join(cwd === workDir ? workDir : cwd, `.upload-to-dropbox-${nonce}.mjs`) await writeFile(file, buildModule(nonce)) lastProgressAt = Date.now() const result = await runExecute({ cliCommand, cwd, file, params: { sourceUrl, reportUrl, size, contentHash, accountId: state?.accountId, phase: state?.phase, chunkBytes, commit, sessionId: state?.sessionId ?? null, offset: state?.offset ?? 0, }, onStall, }) await unlink(file).catch(() => {}) if (!done && result.code === 0) { try { const parsed = JSON.parse(result.stdout.slice(result.stdout.indexOf('{'))) if (parsed?.done && parsed.metadata) done = parsed } catch {} } if (done) break const output = `${result.stderr}\n${result.stdout}`.trim() lastError = output.split('\n').filter(Boolean).slice(-3).join(' | ').slice(0, 600) if (state && /lookup_failed/.test(output) && /not_found|closed|not_closed/.test(output)) { log('session unavailable; preserving checkpoint for reconciliation') break } else if ( /RECONCILIATION_REQUIRED|already exists: |insufficient_scope|not authenticated|not signed in|malformed_path|disallowed_name|no_write_permission|insufficient_space|Node 22/i.test( output, ) ) { log(`stopping: ${lastError}`) break } const progressed = (state?.offset ?? -1) > offsetBefore failedAttempts = progressed ? 1 : failedAttempts + 1 log(`attempt ${attempts} ended (exit ${result.code}): ${lastError || 'no output'}`) await new Promise((r) => setTimeout(r, Math.min(30000, 2000 * failedAttempts))) } clearInterval(watchdog) await cleanup() if (!done) fail(`Upload did not finish after ${attempts} attempts. Last error: ${lastError}. Rerun the same command to resume.`) const metadata = done.metadata const sizeMatches = metadata.size === size const hashMatches = metadata.content_hash === contentHash if (sizeMatches && hashMatches) await unlink(statePath).catch(() => {}) const summary = { ok: sizeMatches && hashMatches, localPath, dropboxPath: metadata.path_display, id: metadata.id, rev: metadata.rev, size, dropboxSize: metadata.size, contentHash, dropboxContentHash: metadata.content_hash, sizeMatches, hashMatches, attempts, durationMs: Date.now() - started, } process.stdout.write(`${JSON.stringify(summary)}\n`) process.exit(summary.ok ? 0 : 2) } export function assertCliVersion(command) { const r = spawnSync(command[0], [...command.slice(1), '--version'], { encoding: 'utf8' }) const m = /(?:^|\s)(\d+)\.(\d+)\.(\d+)/.exec(r.stdout ?? '') if (r.status !== 0 || !m || (+m[1] < 3 || (+m[1] === 3 && (+m[2] < 33 || (+m[2] === 33 && +m[3] < 1))))) throw new Error('Kody CLI >=3.33.1 required') } if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) main().catch(() => fail('Upload stopped; checkpoint preserved. Reconcile uncertain completion before retrying.'))