142 lines
4.4 KiB
JavaScript
142 lines
4.4 KiB
JavaScript
import CacheableLookup from "cacheable-lookup";
|
|
import fetch from "node-fetch";
|
|
import config from "../config/index.js";
|
|
import { CheckedHttpAgent, CheckedHttpProxyAgent, CheckedHttpsAgent, CheckedHttpsProxyAgent } from "./checked-fetch.js";
|
|
export async function getJson(url, accept = "application/json, */*", timeout = 10000, headers) {
|
|
const res = await getResponse({
|
|
url,
|
|
method: "GET",
|
|
headers: Object.assign({
|
|
"User-Agent": config.userAgent,
|
|
Accept: accept
|
|
}, headers || {}),
|
|
timeout
|
|
});
|
|
return await res.json();
|
|
}
|
|
export async function getJsonActivity(url, accept = "application/activity+json, application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"", timeout = 10000, headers) {
|
|
const res = await getResponse({
|
|
url,
|
|
method: "GET",
|
|
headers: Object.assign({
|
|
"User-Agent": config.userAgent,
|
|
Accept: accept
|
|
}, headers || {}),
|
|
timeout
|
|
});
|
|
const contentType = res.headers.get('content-type');
|
|
if (contentType == null || contentType !== 'application/activity+json' && !contentType.startsWith('application/activity+json;') && (!contentType.startsWith('application/ld+json;') || !contentType.includes('profile="https://www.w3.org/ns/activitystreams"'))) {
|
|
throw new Error(`getJsonActivity response had unexpected content-type: ${contentType}`);
|
|
}
|
|
return {
|
|
finalUrl: res.url,
|
|
content: await res.json()
|
|
};
|
|
}
|
|
export async function getHtml(url, accept = "text/html, */*", timeout = 10000, headers) {
|
|
const res = await getResponse({
|
|
url,
|
|
method: "GET",
|
|
headers: Object.assign({
|
|
"User-Agent": config.userAgent,
|
|
Accept: accept
|
|
}, headers || {}),
|
|
timeout
|
|
});
|
|
return await res.text();
|
|
}
|
|
export async function getResponse(args) {
|
|
const timeout = args.timeout || 10 * 1000;
|
|
const controller = new AbortController();
|
|
setTimeout(()=>{
|
|
controller.abort();
|
|
}, timeout * 6);
|
|
const res = await fetch(args.url, {
|
|
method: args.method,
|
|
headers: args.headers,
|
|
body: args.body,
|
|
timeout,
|
|
size: 10 * 1024 * 1024,
|
|
agent: getAgentByUrl,
|
|
signal: controller.signal,
|
|
redirect: args.redirect
|
|
});
|
|
if (args.redirect === "manual" && [
|
|
301,
|
|
302,
|
|
307,
|
|
308
|
|
].includes(res.status)) {
|
|
return res;
|
|
}
|
|
if (!res.ok) {
|
|
throw new StatusError(`${res.status} ${res.statusText}`, res.status, res.statusText);
|
|
}
|
|
return res;
|
|
}
|
|
const cache = new CacheableLookup({
|
|
maxTtl: 3600,
|
|
errorTtl: 30,
|
|
lookup: false
|
|
});
|
|
/**
|
|
* Get http non-proxy agent
|
|
*/ const _http = new CheckedHttpAgent({
|
|
keepAlive: true,
|
|
keepAliveMsecs: 30 * 1000,
|
|
lookup: cache.lookup
|
|
});
|
|
/**
|
|
* Get https non-proxy agent
|
|
*/ const _https = new CheckedHttpsAgent({
|
|
keepAlive: true,
|
|
keepAliveMsecs: 30 * 1000,
|
|
lookup: cache.lookup
|
|
});
|
|
const maxSockets = Math.max(256, config.deliverJobConcurrency || 128);
|
|
/**
|
|
* Get http proxy or non-proxy agent
|
|
*/ export const httpAgent = config.proxy ? new CheckedHttpProxyAgent({
|
|
keepAlive: true,
|
|
keepAliveMsecs: 30 * 1000,
|
|
maxSockets,
|
|
maxFreeSockets: 256,
|
|
scheduling: "lifo",
|
|
proxy: config.proxy
|
|
}) : _http;
|
|
/**
|
|
* Get https proxy or non-proxy agent
|
|
*/ export const httpsAgent = config.proxy ? new CheckedHttpsProxyAgent({
|
|
keepAlive: true,
|
|
keepAliveMsecs: 30 * 1000,
|
|
maxSockets,
|
|
maxFreeSockets: 256,
|
|
scheduling: "lifo",
|
|
proxy: config.proxy
|
|
}) : _https;
|
|
/**
|
|
* Get agent by URL
|
|
* @param url URL
|
|
* @param bypassProxy Allways bypass proxy
|
|
*/ export function getAgentByUrl(url, bypassProxy = false) {
|
|
if (bypassProxy || (config.proxyBypassHosts || []).includes(url.hostname)) {
|
|
return url.protocol === "http:" ? _http : _https;
|
|
} else {
|
|
return url.protocol === "http:" ? httpAgent : httpsAgent;
|
|
}
|
|
}
|
|
export class StatusError extends Error {
|
|
statusCode;
|
|
statusMessage;
|
|
isClientError;
|
|
isRetryable;
|
|
constructor(message, statusCode, statusMessage){
|
|
super(message);
|
|
this.name = "StatusError";
|
|
this.statusCode = statusCode;
|
|
this.statusMessage = statusMessage;
|
|
this.isClientError = typeof this.statusCode === "number" && this.statusCode >= 400 && this.statusCode < 500;
|
|
this.isRetryable = this.isClientError && this.statusCode != 429;
|
|
}
|
|
}
|