import * as fs from "node:fs"; import * as stream from "node:stream"; import * as util from "node:util"; import got, * as Got from "got"; import { httpAgent, httpsAgent, StatusError } from "./fetch.js"; import config from "../config/index.js"; import chalk from "chalk"; import Logger from "../services/logger.js"; const pipeline = util.promisify(stream.pipeline); export async function downloadUrl(url, path) { const logger = new Logger("download"); logger.info(`Downloading ${chalk.cyan(url)} ...`); const timeout = 30 * 1000; const operationTimeout = 60 * 1000; const maxSize = config.maxFileSize || 262144000; const req = got.stream(url, { headers: { "User-Agent": config.userAgent, Host: new URL(url).hostname }, timeout: { lookup: timeout, connect: timeout, secureConnect: timeout, socket: timeout, response: timeout, send: timeout, request: operationTimeout }, agent: { http: httpAgent, https: httpsAgent }, http2: false, retry: { limit: 0 } }).on("response", (res)=>{ const contentLength = res.headers["content-length"]; if (contentLength != null) { const size = Number(contentLength); if (size > maxSize) { logger.warn(`maxSize exceeded (${size} > ${maxSize}) on response`); req.destroy(); } } }).on("downloadProgress", (progress)=>{ if (progress.transferred > maxSize) { logger.warn(`maxSize exceeded (${progress.transferred} > ${maxSize}) on downloadProgress`); req.destroy(); } }); try { await pipeline(req, fs.createWriteStream(path)); } catch (e) { if (e instanceof Got.HTTPError) { throw new StatusError(`${e.response.statusCode} ${e.response.statusMessage}`, e.response.statusCode, e.response.statusMessage); } else { throw e; } } logger.succ(`Download finished: ${chalk.cyan(url)}`); }