Fixed 267U.pre2

2026-07-26 18:25:37 +09:00
parent 50bfaeafdf
commit 317d00a284
1286 changed files with 80222 additions and 1 deletions
@@ -0,0 +1,141 @@
+import CacheableLookup from "cacheable-lookup";
+import fetch from "node-fetch";
+import config from "../config/index.js";
+import { CheckedHttpAgent, CheckedHttpProxyAgent, CheckedHttpsAgent, CheckedHttpsProxyAgent } from "./checked-fetch.js";
+export async function getJson(url, accept = "application/json, */*", timeout = 10000, headers) {
+    const res = await getResponse({
+        url,
+        method: "GET",
+        headers: Object.assign({
+            "User-Agent": config.userAgent,
+            Accept: accept
+        }, headers || {}),
+        timeout
+    });
+    return await res.json();
+}
+export async function getJsonActivity(url, accept = "application/activity+json, application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"", timeout = 10000, headers) {
+    const res = await getResponse({
+        url,
+        method: "GET",
+        headers: Object.assign({
+            "User-Agent": config.userAgent,
+            Accept: accept
+        }, headers || {}),
+        timeout
+    });
+    const contentType = res.headers.get('content-type');
+    if (contentType == null || contentType !== 'application/activity+json' && !contentType.startsWith('application/activity+json;') && (!contentType.startsWith('application/ld+json;') || !contentType.includes('profile="https://www.w3.org/ns/activitystreams"'))) {
+        throw new Error(`getJsonActivity response had unexpected content-type: ${contentType}`);
+    }
+    return {
+        finalUrl: res.url,
+        content: await res.json()
+    };
+}
+export async function getHtml(url, accept = "text/html, */*", timeout = 10000, headers) {
+    const res = await getResponse({
+        url,
+        method: "GET",
+        headers: Object.assign({
+            "User-Agent": config.userAgent,
+            Accept: accept
+        }, headers || {}),
+        timeout
+    });
+    return await res.text();
+}
+export async function getResponse(args) {
+    const timeout = args.timeout || 10 * 1000;
+    const controller = new AbortController();
+    setTimeout(()=>{
+        controller.abort();
+    }, timeout * 6);
+    const res = await fetch(args.url, {
+        method: args.method,
+        headers: args.headers,
+        body: args.body,
+        timeout,
+        size: 10 * 1024 * 1024,
+        agent: getAgentByUrl,
+        signal: controller.signal,
+        redirect: args.redirect
+    });
+    if (args.redirect === "manual" && [
+        301,
+        302,
+        307,
+        308
+    ].includes(res.status)) {
+        return res;
+    }
+    if (!res.ok) {
+        throw new StatusError(`${res.status} ${res.statusText}`, res.status, res.statusText);
+    }
+    return res;
+}
+const cache = new CacheableLookup({
+    maxTtl: 3600,
+    errorTtl: 30,
+    lookup: false
+});
+/**
+ * Get http non-proxy agent
+ */ const _http = new CheckedHttpAgent({
+    keepAlive: true,
+    keepAliveMsecs: 30 * 1000,
+    lookup: cache.lookup
+});
+/**
+ * Get https non-proxy agent
+ */ const _https = new CheckedHttpsAgent({
+    keepAlive: true,
+    keepAliveMsecs: 30 * 1000,
+    lookup: cache.lookup
+});
+const maxSockets = Math.max(256, config.deliverJobConcurrency || 128);
+/**
+ * Get http proxy or non-proxy agent
+ */ export const httpAgent = config.proxy ? new CheckedHttpProxyAgent({
+    keepAlive: true,
+    keepAliveMsecs: 30 * 1000,
+    maxSockets,
+    maxFreeSockets: 256,
+    scheduling: "lifo",
+    proxy: config.proxy
+}) : _http;
+/**
+ * Get https proxy or non-proxy agent
+ */ export const httpsAgent = config.proxy ? new CheckedHttpsProxyAgent({
+    keepAlive: true,
+    keepAliveMsecs: 30 * 1000,
+    maxSockets,
+    maxFreeSockets: 256,
+    scheduling: "lifo",
+    proxy: config.proxy
+}) : _https;
+/**
+ * Get agent by URL
+ * @param url URL
+ * @param bypassProxy Allways bypass proxy
+ */ export function getAgentByUrl(url, bypassProxy = false) {
+    if (bypassProxy || (config.proxyBypassHosts || []).includes(url.hostname)) {
+        return url.protocol === "http:" ? _http : _https;
+    } else {
+        return url.protocol === "http:" ? httpAgent : httpsAgent;
+    }
+}
+export class StatusError extends Error {
+    statusCode;
+    statusMessage;
+    isClientError;
+    isRetryable;
+    constructor(message, statusCode, statusMessage){
+        super(message);
+        this.name = "StatusError";
+        this.statusCode = statusCode;
+        this.statusMessage = statusMessage;
+        this.isClientError = typeof this.statusCode === "number" && this.statusCode >= 400 && this.statusCode < 500;
+        this.isRetryable = this.isClientError && this.statusCode != 429;
+    }
+}