Skip to content
← Public packages

@kody/dropbox

Reusable Dropbox OAuth helpers for account, metadata, search, file transfer, sharing, and generic API requests.

FollowFork

Files

upload-to-dropbox.mjs

504 lines · 18.9 KB · JavaScript
#!/usr/bin/env node
// Upload a local file of any size (tested to hundreds of MB) to Dropbox through
// Kody local execute and @kody/dropbox/upload-session.
//
// Usage:
//   node upload-to-dropbox.mjs <local file> <dropbox path> [options]
//
// Options:
//   --mode add|overwrite   Commit mode (default: add, which refuses to replace an existing file)
//   --autorename           With mode add, pick a free name instead of failing on a conflict
//   --chunk-mib <n>        MiB per append, 5 to 90 (default: 16)
//   --fresh                Ignore saved resume state and start a new session
//   --dry-run              Print the plan and local content_hash, upload nothing
//   --use-checkout         Resolve the package from this script's checkout instead of the published version
//
// Needs Node 22+ and a signed-in Kody CLI (stored token with org:execute,
// package:execute, integration:use). Override the CLI command with KODY_CLI
// (default: "npx -y @kodycodes/cli@3.33.1").
//
// Stdout: one JSON summary. Progress goes to stderr. Exit 0 only when the
// Dropbox size and content_hash match the local file.

import { checkpointWriter, lockCheckpoint } from './lib/checkpoint.mjs'
import { spawn, spawnSync } from 'node:child_process'
import { createHash, randomBytes } from 'node:crypto'
import { createReadStream } from 'node:fs'
import { mkdir, mkdtemp, readFile, rm, stat, unlink, writeFile } from 'node:fs/promises'
import { createServer } from 'node:http'
import { homedir, tmpdir } from 'node:os'
import { dirname, join, resolve } from 'node:path'
import { fileURLToPath } from 'node:url'

const MIB = 1024 * 1024
const BLOCK = 4 * MIB
const PACKAGE_NAME = '@kody/dropbox'
const MAX_FAILED_ATTEMPTS = 8
const STALL_MS = Number(process.env.KODY_DROPBOX_UPLOAD_STALL_MS ?? 6 * 60 * 1000)
const STATE_DIR = join(homedir(), '.cache', 'kody-org-dropbox-upload')

function log(message) {
	process.stderr.write(`[upload-to-dropbox] ${message}\n`)
}

function fail(message, code = 1) {
	process.stdout.write(`${JSON.stringify({ ok: false, error: message })}\n`)
	process.exit(code)
}

function parseArgs(argv) {
	const options = { mode: 'add', autorename: false, chunkMib: 16, fresh: false, dryRun: false, useCheckout: false }
	const positional = []
	for (let index = 0; index < argv.length; index += 1) {
		const arg = argv[index]
		if (arg === '--mode') options.mode = argv[++index]
		else if (arg === '--autorename') options.autorename = true
		else if (arg === '--chunk-mib') options.chunkMib = Number(argv[++index])
		else if (arg === '--fresh') options.fresh = true
		else if (arg === '--dry-run') options.dryRun = true
		else if (arg === '--use-checkout') options.useCheckout = true
		else if (arg === '-h' || arg === '--help') options.help = true
		else if (arg.startsWith('--')) fail(`Unknown option ${arg}`)
		else positional.push(arg)
	}
	return { options, positional }
}

async function computeContentHash(path) {
	const blockHashes = []
	let block = createHash('sha256')
	let inBlock = 0
	for await (const chunk of createReadStream(path, { highWaterMark: MIB })) {
		let offset = 0
		while (offset < chunk.length) {
			const take = Math.min(BLOCK - inBlock, chunk.length - offset)
			block.update(chunk.subarray(offset, offset + take))
			inBlock += take
			offset += take
			if (inBlock === BLOCK) {
				blockHashes.push(block.digest())
				block = createHash('sha256')
				inBlock = 0
			}
		}
	}
	if (inBlock > 0) blockHashes.push(block.digest())
	return createHash('sha256').update(Buffer.concat(blockHashes)).digest('hex')
}

// The execute module. Rewritten with a fresh nonce for every attempt so a
// retry never reuses a cached module graph.
export function buildModule(nonce) {
	return `// upload-to-dropbox attempt ${nonce}
import {
	startUploadSession,
	appendUploadSession,
	finishUploadSession,
	getUploadSessionOffset,
	getCorrectOffsetFromError,
} from 'kody:${PACKAGE_NAME}/upload-session'
import { DropboxApiError, dropboxRpc } from 'kody:${PACKAGE_NAME}/request'

export default async function main(p) {
	const report = async (state) => {
		const r = await fetch(p.reportUrl, { method: 'POST', body: JSON.stringify(state) })
		await r.body?.cancel()
		if (!r.ok) throw new Error('Checkpoint persistence failed; stopped')
	}
	const readRange = async (start, end) => {
		if (end <= start) return new Uint8Array(0)
		const response = await fetch(p.sourceUrl, { headers: { range: 'bytes=' + start + '-' + (end - 1) } })
		const bytes = new Uint8Array(await response.arrayBuffer())
		if (response.status !== 206 || bytes.byteLength !== end - start) {
			throw new Error('local source returned ' + response.status + ' with ' + bytes.byteLength + ' bytes')
		}
		return bytes
	}
	const withRetry = async (label, fn) => {
		for (let attempt = 1; ; attempt += 1) {
			try {
				return await fn()
			} catch (error) {
				if (getCorrectOffsetFromError(error) !== null) throw error
				const status = error instanceof DropboxApiError ? error.status : 0
				const retryable = error?.name !== 'AbortError' && (status === 0 || status === 429 || status >= 500)
				if (!retryable || attempt >= 3) throw error
				await report({ note: label + ' retry ' + attempt + ': ' + String(error?.message ?? error).slice(0, 200) })
				await new Promise((r) => setTimeout(r, Math.max(2000 * attempt, Number.isFinite(error?.retryAfterMs) ? error.retryAfterMs : 0)))
			}
		}
	}

	const account = await dropboxRpc('users/get_current_account', {})
	if (!account.account_id) throw new Error('Dropbox account identity unavailable; refusing upload')
	if (p.accountId && p.accountId !== account.account_id) throw new Error('RECONCILIATION_REQUIRED: Dropbox account changed; no upload performed')
	if (p.sessionId && !p.accountId) throw new Error('RECONCILIATION_REQUIRED: legacy checkpoint has no account identity')
	const identity = { accountId: account.account_id }
	const reconcileFinish = async () => {
		try {
			const metadata = await dropboxRpc('files/get_metadata', { path: p.commit.path })
			if (!p.commit.autorename && metadata.size === p.size && metadata.content_hash === p.contentHash) {
				const result = { ...identity, sessionId: p.sessionId || 'reconciled', offset: p.size, done: true, metadata }
				await report(result)
				return result
			}
		} catch {}
		throw new Error('RECONCILIATION_REQUIRED: final result uncertain; inspect destination before starting another upload')
	}
	if (p.phase === 'finishing' || p.phase === 'done') return reconcileFinish()
	let sessionId = p.sessionId
	let offset = 0
	if (sessionId) {
		offset = (await withRetry('get-offset', () => getUploadSessionOffset({ sessionId, offset: p.offset ?? 0 }))).offset
	} else {
		if (p.commit.mode === 'add' && !p.commit.autorename) {
			try {
				await dropboxRpc('files/get_metadata', { path: p.commit.path })
				throw new Error('EXISTS')
			} catch (error) {
				if (error?.message === 'EXISTS') {
					throw new Error('Dropbox path already exists: ' + p.commit.path + '. Use --mode overwrite or --autorename.')
				}
				if (!(error instanceof DropboxApiError && error.status === 409)) throw error
			}
		}
		sessionId = (await startUploadSession({ confirm: true })).sessionId
	}
	if (!Number.isSafeInteger(offset) || offset < 0 || offset > p.size) throw new Error('Invalid acknowledged Dropbox offset')
	await report({ ...identity, sessionId, offset, phase: 'uploading' })

	while (p.size - offset > p.chunkBytes) {
		const end = offset + p.chunkBytes
		const bytes = await readRange(offset, end)
		try {
			offset = (await withRetry('append', () => appendUploadSession({ sessionId, offset, bytes, confirm: true }))).offset
		} catch (error) {
			const correct = getCorrectOffsetFromError(error)
			if (correct === null) throw error
			await report({ note: 'offset corrected from ' + offset + ' to ' + correct })
			offset = correct
		}
		await report({ ...identity, sessionId, offset, phase: 'uploading' })
	}

	for (;;) {
		const bytes = await readRange(offset, p.size)
		try {
			await report({ ...identity, sessionId, offset, phase: 'finishing' })
			const metadata = await finishUploadSession({ sessionId, offset, bytes, commit: p.commit, confirm: true })
			const result = {
				...identity,
				sessionId,
				offset: p.size,
				done: true,
				metadata: {
					id: metadata.id,
					name: metadata.name,
					path_display: metadata.path_display,
					rev: metadata.rev,
					size: metadata.size,
					content_hash: metadata.content_hash,
					server_modified: metadata.server_modified,
				},
			}
			await report(result)
			return result
		} catch (error) {
			const correct = getCorrectOffsetFromError(error)
			if (correct === null || correct === offset) return reconcileFinish()
			await report({ note: 'finish offset corrected from ' + offset + ' to ' + correct })
			offset = correct
		}
	}
}
`
}

function startLocalServer(filePath, size, onReport) {
	const token = randomBytes(16).toString('hex')
	const server = createServer((req, res) => {
		if (req.method === 'GET' && req.url === `/${token}/source`) {
			const match = /^bytes=(\d+)-(\d+)$/.exec(req.headers.range ?? '')
			if (!match) {
				res.writeHead(416).end()
				return
			}
			const start = Number(match[1])
			const end = Math.min(Number(match[2]), size - 1)
			if (start > end) {
				res.writeHead(416).end()
				return
			}
			res.writeHead(206, {
				'content-type': 'application/octet-stream',
				'content-length': String(end - start + 1),
				'content-range': `bytes ${start}-${end}/${size}`,
			})
			createReadStream(filePath, { start, end }).pipe(res)
			return
		}
		if (req.method === 'POST' && req.url === `/${token}/report`) {
			let body = ''
			req.on('data', (chunk) => (body += chunk))
			req.on('end', () => {
				try {
					Promise.resolve(onReport(JSON.parse(body))).then(() => res.writeHead(204).end(), () => res.writeHead(500).end())
					return
				} catch {}
				res.writeHead(204).end()
			})
			return
		}
		res.writeHead(404).end()
	})
	return new Promise((resolvePromise) => {
		server.listen(0, '127.0.0.1', () => {
			const { port } = server.address()
			resolvePromise({
				server,
				sourceUrl: `http://127.0.0.1:${port}/${token}/source`,
				reportUrl: `http://127.0.0.1:${port}/${token}/report`,
			})
		})
	})
}

function runExecute({ cliCommand, cwd, file, params, onStall }) {
	const [command, ...baseArgs] = cliCommand
	const args = [...baseArgs, '--org', 'kody', 'execute', '--local', '--allow-private-network', '--file', file, '--params', JSON.stringify(params)]
	return new Promise((resolvePromise) => {
		const child = spawn(command, args, { cwd, detached: true, stdio: ['ignore', 'pipe', 'pipe'] })
		let stdout = ''
		let stderr = ''
		child.stdout.on('data', (chunk) => (stdout += chunk))
		child.stderr.on('data', (chunk) => (stderr += chunk))
		const kill = () => {
			try {
				process.kill(-child.pid, 'SIGKILL')
			} catch {}
		}
		onStall.kill = kill
		child.on('close', (code) => resolvePromise({ code, stdout, stderr }))
		child.on('error', (error) => resolvePromise({ code: -1, stdout, stderr: String(error) }))
	})
}

async function main() {
	const { options, positional } = parseArgs(process.argv.slice(2))
	if (options.help || positional.length !== 2) {
		process.stderr.write('Usage: node upload-to-dropbox.mjs <local file> <dropbox path> [--mode add|overwrite] [--autorename] [--chunk-mib 16] [--fresh] [--dry-run] [--use-checkout]\n')
		process.exit(options.help ? 0 : 1)
	}
	const localPath = resolve(positional[0])
	const dropboxPath = positional[1]
	if (!dropboxPath.startsWith('/')) fail('Dropbox path must start with "/" (Kody writes usually live under /Apps/Kody/).')
	if (options.mode !== 'add' && options.mode !== 'overwrite') fail('--mode must be add or overwrite.')
	if (!Number.isFinite(options.chunkMib) || options.chunkMib < 5 || options.chunkMib > 90) fail('--chunk-mib must be between 5 and 90.')
	const [nodeMajor] = process.versions.node.split('.').map(Number)
	if (nodeMajor < 22) fail(`Node 22+ is required (found ${process.versions.node}).`)

	let fileStat
	try {
		fileStat = await stat(localPath)
	} catch {
		fail(`Local file not found: ${localPath}`)
	}
	if (!fileStat.isFile()) fail(`Not a regular file: ${localPath}`)
	const size = fileStat.size
	const chunkBytes = Math.round(options.chunkMib * MIB)
	const commit = { path: dropboxPath, mode: options.mode, autorename: options.autorename }

	const started = Date.now()
	log(`hashing ${localPath} (${(size / MIB).toFixed(1)} MiB)`)
	const contentHash = await computeContentHash(localPath)
	log(`local content_hash ${contentHash}`)

	const stateKey = createHash('sha256')
		.update(JSON.stringify([localPath, size, fileStat.mtimeMs, commit, contentHash]))
		.digest('hex')
		.slice(0, 32)
	const statePath = join(STATE_DIR, `${stateKey}.json`)
	const legacyKey=createHash('sha256').update(JSON.stringify([localPath,size,fileStat.mtimeMs,dropboxPath,contentHash])).digest('hex').slice(0,32)
	if(!options.fresh){try{await stat(join(STATE_DIR,legacyKey+'.json'));throw new Error('RECONCILIATION_REQUIRED: legacy checkpoint exists; inspect before starting a replacement')}catch(e){if(e.code!=='ENOENT')throw e}}
	const releaseLock = await lockCheckpoint(statePath)
	let state = null
	if (!options.fresh) {
		try {
			state = JSON.parse(await readFile(statePath, 'utf8'))
			// Dropbox keeps an upload session for up to 7 days; leave a margin.
			if (Date.now() - state.createdAt > 6 * 24 * 60 * 60 * 1000) state = null
		} catch {
			state = null
		}
	}

	if (options.dryRun) {
		process.stdout.write(
			`${JSON.stringify({
				ok: true,
				dryRun: true,
				localPath,
				dropboxPath,
				size,
				contentHash,
				chunkBytes,
				chunks: Math.max(1, Math.ceil(size / chunkBytes)),
				commit,
				resumeFrom: state ? { offset: state.offset } : null,
			})}\n`,
		)
		releaseLock()
		return
	}

	await mkdir(STATE_DIR, { recursive: true })
	const writeCheckpoint = checkpointWriter(statePath)
	const saveState = () => writeCheckpoint(state)
	if (state) log(`resuming session at offset ${state.offset} of ${size}`)

	let lastProgressAt = Date.now()
	let done = null
	const { server, sourceUrl, reportUrl } = await startLocalServer(localPath, size, async (message) => {
		lastProgressAt = Date.now()
		if (message.note) {
			log(message.note)
			return
		}
		if (message.sessionId) {
			const isNew = !state || state.sessionId !== message.sessionId
			state = {
				sessionId: message.sessionId,
				accountId: message.accountId ?? state?.accountId,
				phase: message.phase ?? (message.done ? 'done' : state?.phase),
				offset: message.offset,
				createdAt: isNew ? Date.now() : state.createdAt,
			}
			await saveState()
			const elapsed = (Date.now() - started) / 1000
			log(`${((100 * message.offset) / Math.max(size, 1)).toFixed(1)}% (${(message.offset / MIB).toFixed(1)} of ${(size / MIB).toFixed(1)} MiB, ${elapsed.toFixed(0)}s)`)
		}
		if (message.done) done = message
	})

	const cliCommand = (process.env.KODY_CLI ?? 'npx -y @kodycodes/cli@3.33.1').split(/\s+/).filter(Boolean)
	assertCliVersion(cliCommand)
	const scriptDir = dirname(fileURLToPath(import.meta.url))
	const workDir = await mkdtemp(join(tmpdir(), 'kody-dropbox-upload-'))
	let cwd = workDir
	if (options.useCheckout) {
		const checkout = resolve(scriptDir, '..')
		const pkg = JSON.parse(await readFile(join(checkout, 'package.json'), 'utf8'))
		if (pkg.name !== PACKAGE_NAME) fail(`--use-checkout: ${checkout} is not a ${PACKAGE_NAME} checkout.`)
		cwd = checkout
	}

	const cleanup = async () => {
		releaseLock()
		server.close()
		await rm(workDir, { recursive: true, force: true })
	}
	const onStall = { kill: () => {} }
	const interrupt = (signal) => {
		log(`${signal}: stopping; rerun the same command to resume`)
		onStall.kill()
		void cleanup().finally(() => process.exit(130))
	}
	process.on('SIGINT', () => interrupt('SIGINT'))
	process.on('SIGTERM', () => interrupt('SIGTERM'))

	let attempts = 0
	let failedAttempts = 0
	let lastError = ''
	const watchdog = setInterval(() => {
		if (Date.now() - lastProgressAt > STALL_MS) {
			log(`no progress for ${Math.round(STALL_MS / 1000)}s; restarting the run`)
			lastProgressAt = Date.now()
			onStall.kill()
		}
	}, 5000)

	while (!done && failedAttempts < MAX_FAILED_ATTEMPTS) {
		attempts += 1
		const offsetBefore = state?.offset ?? -1
		const nonce = `${Date.now()}-${randomBytes(4).toString('hex')}`
		const file = join(cwd === workDir ? workDir : cwd, `.upload-to-dropbox-${nonce}.mjs`)
		await writeFile(file, buildModule(nonce))
		lastProgressAt = Date.now()
		const result = await runExecute({
			cliCommand,
			cwd,
			file,
			params: {
				sourceUrl,
				reportUrl,
				size,
				contentHash,
				accountId: state?.accountId,
				phase: state?.phase,
				chunkBytes,
				commit,
				sessionId: state?.sessionId ?? null,
				offset: state?.offset ?? 0,
			},
			onStall,
		})
		await unlink(file).catch(() => {})
		if (!done && result.code === 0) {
			try {
				const parsed = JSON.parse(result.stdout.slice(result.stdout.indexOf('{')))
				if (parsed?.done && parsed.metadata) done = parsed
			} catch {}
		}
		if (done) break
		const output = `${result.stderr}\n${result.stdout}`.trim()
		lastError = output.split('\n').filter(Boolean).slice(-3).join(' | ').slice(0, 600)
		if (state && /lookup_failed/.test(output) && /not_found|closed|not_closed/.test(output)) {
			log('session unavailable; preserving checkpoint for reconciliation')
			break
		} else if (
			/RECONCILIATION_REQUIRED|already exists: |insufficient_scope|not authenticated|not signed in|malformed_path|disallowed_name|no_write_permission|insufficient_space|Node 22/i.test(
				output,
			)
		) {
			log(`stopping: ${lastError}`)
			break
		}
		const progressed = (state?.offset ?? -1) > offsetBefore
		failedAttempts = progressed ? 1 : failedAttempts + 1
		log(`attempt ${attempts} ended (exit ${result.code}): ${lastError || 'no output'}`)
		await new Promise((r) => setTimeout(r, Math.min(30000, 2000 * failedAttempts)))
	}
	clearInterval(watchdog)
	await cleanup()

	if (!done) fail(`Upload did not finish after ${attempts} attempts. Last error: ${lastError}. Rerun the same command to resume.`)

	const metadata = done.metadata
	const sizeMatches = metadata.size === size
	const hashMatches = metadata.content_hash === contentHash
	if (sizeMatches && hashMatches) await unlink(statePath).catch(() => {})
	const summary = {
		ok: sizeMatches && hashMatches,
		localPath,
		dropboxPath: metadata.path_display,
		id: metadata.id,
		rev: metadata.rev,
		size,
		dropboxSize: metadata.size,
		contentHash,
		dropboxContentHash: metadata.content_hash,
		sizeMatches,
		hashMatches,
		attempts,
		durationMs: Date.now() - started,
	}
	process.stdout.write(`${JSON.stringify(summary)}\n`)
	process.exit(summary.ok ? 0 : 2)
}

export function assertCliVersion(command) {
	const r = spawnSync(command[0], [...command.slice(1), '--version'], { encoding: 'utf8' })
	const m = /(?:^|\s)(\d+)\.(\d+)\.(\d+)/.exec(r.stdout ?? '')
	if (r.status !== 0 || !m || (+m[1] < 3 || (+m[1] === 3 && (+m[2] < 33 || (+m[2] === 33 && +m[3] < 1))))) throw new Error('Kody CLI >=3.33.1 required')
}
if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) main().catch(() => fail('Upload stopped; checkpoint preserved. Reconcile uncertain completion before retrying.'))