Fixed 267U.pre2
This commit is contained in:
@@ -0,0 +1,446 @@
|
||||
import * as fs from "node:fs";
|
||||
import { Readable } from "node:stream";
|
||||
import { v4 as uuid } from "uuid";
|
||||
import sharp from "sharp";
|
||||
import { IsNull } from "typeorm";
|
||||
import { publishMainStream, publishDriveStream } from "../stream.js";
|
||||
import { fetchMeta } from "../../misc/fetch-meta.js";
|
||||
import { contentDisposition } from "../../misc/content-disposition.js";
|
||||
import { getFileInfo } from "../../misc/get-file-info.js";
|
||||
import { DriveFiles, DriveFolders, Users, UserProfiles } from "../../models/index.js";
|
||||
import { DriveFile } from "../../models/entities/drive-file.js";
|
||||
import { driveChart, perUserDriveChart, instanceChart } from "../chart/index.js";
|
||||
import { genId } from "../../misc/gen-id.js";
|
||||
import { isDuplicateKeyValueError } from "../../misc/is-duplicate-key-value-error.js";
|
||||
import { FILE_TYPE_BROWSERSAFE } from "../../const.js";
|
||||
import { IdentifiableError } from "../../misc/identifiable-error.js";
|
||||
import { getS3 } from "./s3.js";
|
||||
import { InternalStorage } from "./internal-storage.js";
|
||||
import { convertSharpToWebp } from "./image-processor.js";
|
||||
import { driveLogger } from "./logger.js";
|
||||
import { GenerateVideoThumbnail } from "./generate-video-thumbnail.js";
|
||||
import { deleteFile } from "./delete-file.js";
|
||||
import { Upload } from "@aws-sdk/lib-storage";
|
||||
const logger = driveLogger.createSubLogger("register", "yellow");
|
||||
/***
|
||||
* Save file
|
||||
* @param path Path for original
|
||||
* @param name Name for original
|
||||
* @param type Content-Type for original
|
||||
* @param hash Hash for original
|
||||
* @param size Size for original
|
||||
*/ async function save(file, path, name, type, hash, size) {
|
||||
// thunbnail, webpublic を必要なら生成
|
||||
const alts = await generateAlts(path, type, !file.uri);
|
||||
const meta = await fetchMeta();
|
||||
if (meta.useObjectStorage) {
|
||||
//#region ObjectStorage params
|
||||
let [ext] = name.match(/\.([a-zA-Z0-9_-]+)$/) || [
|
||||
""
|
||||
];
|
||||
if (ext === "") {
|
||||
if (type === "image/jpeg") ext = ".jpg";
|
||||
if (type === "image/png") ext = ".png";
|
||||
if (type === "image/webp") ext = ".webp";
|
||||
if (type === "image/apng") ext = ".apng";
|
||||
if (type === "image/avif") ext = ".avif";
|
||||
if (type === "image/vnd.mozilla.apng") ext = ".apng";
|
||||
}
|
||||
// Some cloud providers (notably upcloud) will infer the content-type based
|
||||
// on extension, so we remove extensions from non-browser-safe types.
|
||||
if (!FILE_TYPE_BROWSERSAFE.includes(type)) {
|
||||
ext = "";
|
||||
}
|
||||
const baseUrl = meta.objectStorageBaseUrl || `${meta.objectStorageUseSSL ? "https" : "http"}://${meta.objectStorageEndpoint}${meta.objectStoragePort ? `:${meta.objectStoragePort}` : ""}/${meta.objectStorageBucket}`;
|
||||
// for original
|
||||
const key = `${meta.objectStoragePrefix}/${uuid()}${ext}`;
|
||||
const url = `${baseUrl}/${key}`;
|
||||
// for alts
|
||||
let webpublicKey = null;
|
||||
let webpublicUrl = null;
|
||||
let thumbnailKey = null;
|
||||
let thumbnailUrl = null;
|
||||
//#endregion
|
||||
//#region Uploads
|
||||
logger.info(`uploading original: ${key}`);
|
||||
const uploads = [
|
||||
upload(key, fs.createReadStream(path), type, name)
|
||||
];
|
||||
if (alts.webpublic) {
|
||||
webpublicKey = `${meta.objectStoragePrefix}/webpublic-${uuid()}.${alts.webpublic.ext}`;
|
||||
webpublicUrl = `${baseUrl}/${webpublicKey}`;
|
||||
logger.info(`uploading webpublic: ${webpublicKey}`);
|
||||
uploads.push(upload(webpublicKey, alts.webpublic.data, alts.webpublic.type, name));
|
||||
}
|
||||
if (alts.thumbnail) {
|
||||
thumbnailKey = `${meta.objectStoragePrefix}/thumbnail-${uuid()}.${alts.thumbnail.ext}`;
|
||||
thumbnailUrl = `${baseUrl}/${thumbnailKey}`;
|
||||
logger.info(`uploading thumbnail: ${thumbnailKey}`);
|
||||
uploads.push(upload(thumbnailKey, alts.thumbnail.data, alts.thumbnail.type));
|
||||
}
|
||||
await Promise.all(uploads);
|
||||
//#endregion
|
||||
file.url = url;
|
||||
file.thumbnailUrl = thumbnailUrl;
|
||||
file.webpublicUrl = webpublicUrl;
|
||||
file.accessKey = key;
|
||||
file.thumbnailAccessKey = thumbnailKey;
|
||||
file.webpublicAccessKey = webpublicKey;
|
||||
file.webpublicType = alts.webpublic?.type ?? null;
|
||||
file.name = name;
|
||||
file.type = type;
|
||||
file.md5 = hash;
|
||||
file.size = size;
|
||||
file.storedInternal = false;
|
||||
return await DriveFiles.insert(file).then((x)=>DriveFiles.findOneByOrFail(x.identifiers[0]));
|
||||
} else {
|
||||
// use internal storage
|
||||
const accessKey = uuid();
|
||||
const thumbnailAccessKey = `thumbnail-${uuid()}`;
|
||||
const webpublicAccessKey = `webpublic-${uuid()}`;
|
||||
const url = await InternalStorage.saveFromPath(accessKey, path);
|
||||
let thumbnailUrl = null;
|
||||
let webpublicUrl = null;
|
||||
if (alts.thumbnail) {
|
||||
thumbnailUrl = InternalStorage.saveFromBuffer(thumbnailAccessKey, alts.thumbnail.data);
|
||||
logger.info(`thumbnail stored: ${thumbnailAccessKey}`);
|
||||
}
|
||||
if (alts.webpublic) {
|
||||
webpublicUrl = InternalStorage.saveFromBuffer(webpublicAccessKey, alts.webpublic.data);
|
||||
logger.info(`web stored: ${webpublicAccessKey}`);
|
||||
}
|
||||
file.storedInternal = true;
|
||||
file.url = url;
|
||||
file.thumbnailUrl = thumbnailUrl;
|
||||
file.webpublicUrl = webpublicUrl;
|
||||
file.accessKey = accessKey;
|
||||
file.thumbnailAccessKey = thumbnailAccessKey;
|
||||
file.webpublicAccessKey = webpublicAccessKey;
|
||||
file.webpublicType = alts.webpublic?.type ?? null;
|
||||
file.name = name;
|
||||
file.type = type;
|
||||
file.md5 = hash;
|
||||
file.size = size;
|
||||
return await DriveFiles.insert(file).then((x)=>DriveFiles.findOneByOrFail(x.identifiers[0]));
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Generate webpublic, thumbnail, etc
|
||||
* @param path Path for original
|
||||
* @param type Content-Type for original
|
||||
* @param generateWeb Generate webpublic or not
|
||||
*/ export async function generateAlts(path, type, generateWeb) {
|
||||
if (type.startsWith("video/")) {
|
||||
try {
|
||||
const thumbnail = await GenerateVideoThumbnail(path);
|
||||
return {
|
||||
webpublic: null,
|
||||
thumbnail
|
||||
};
|
||||
} catch (err) {
|
||||
logger.warn(`GenerateVideoThumbnail failed: ${err}`);
|
||||
return {
|
||||
webpublic: null,
|
||||
thumbnail: null
|
||||
};
|
||||
}
|
||||
}
|
||||
if (![
|
||||
"image/jpeg",
|
||||
"image/png",
|
||||
"image/webp",
|
||||
"image/svg+xml",
|
||||
"image/avif"
|
||||
].includes(type)) {
|
||||
logger.debug("web image and thumbnail not created (not an required file)");
|
||||
return {
|
||||
webpublic: null,
|
||||
thumbnail: null
|
||||
};
|
||||
}
|
||||
let img = null;
|
||||
let satisfyWebpublic;
|
||||
try {
|
||||
img = sharp(path);
|
||||
const metadata = await img.metadata();
|
||||
const isAnimated = metadata.pages && metadata.pages > 1;
|
||||
// skip animated
|
||||
if (isAnimated) {
|
||||
return {
|
||||
webpublic: null,
|
||||
thumbnail: null
|
||||
};
|
||||
}
|
||||
satisfyWebpublic = !!(type !== "image/svg+xml" && type !== "image/webp" && !(metadata.exif || metadata.iptc || metadata.xmp || metadata.tifftagPhotoshop) && metadata.width && metadata.width <= 2048 && metadata.height && metadata.height <= 2048);
|
||||
} catch (err) {
|
||||
logger.warn(`sharp failed: ${err}`);
|
||||
return {
|
||||
webpublic: null,
|
||||
thumbnail: null
|
||||
};
|
||||
}
|
||||
// #region webpublic
|
||||
let webpublic = null;
|
||||
if (generateWeb && !satisfyWebpublic) {
|
||||
logger.info("creating web image");
|
||||
try {
|
||||
if ([
|
||||
"image/jpeg"
|
||||
].includes(type)) {
|
||||
webpublic = await convertSharpToWebp(img, 2048, 2048);
|
||||
} else if ([
|
||||
"image/webp"
|
||||
].includes(type)) {
|
||||
webpublic = await convertSharpToWebp(img, 2048, 2048);
|
||||
} else if ([
|
||||
"image/png"
|
||||
].includes(type)) {
|
||||
webpublic = await convertSharpToWebp(img, 2048, 2048, 100);
|
||||
} else if ([
|
||||
"image/svg+xml"
|
||||
].includes(type)) {
|
||||
webpublic = await convertSharpToWebp(img, 2048, 2048);
|
||||
} else {
|
||||
logger.debug("web image not created (not an required image)");
|
||||
}
|
||||
} catch (err) {
|
||||
logger.warn("web image not created (an error occured)", err);
|
||||
}
|
||||
} else {
|
||||
if (satisfyWebpublic) logger.info("web image not created (original satisfies webpublic)");
|
||||
else logger.info("web image not created (from remote)");
|
||||
}
|
||||
// #endregion webpublic
|
||||
// #region thumbnail
|
||||
let thumbnail = null;
|
||||
try {
|
||||
if ([
|
||||
"image/jpeg",
|
||||
"image/webp",
|
||||
"image/png",
|
||||
"image/svg+xml",
|
||||
"image/avif"
|
||||
].includes(type)) {
|
||||
thumbnail = await convertSharpToWebp(img, 996, 560);
|
||||
} else {
|
||||
logger.debug("thumbnail not created (not an required file)");
|
||||
}
|
||||
} catch (err) {
|
||||
logger.warn("thumbnail not created (an error occured)", err);
|
||||
}
|
||||
// #endregion thumbnail
|
||||
return {
|
||||
webpublic,
|
||||
thumbnail
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Upload to ObjectStorage
|
||||
*/ async function upload(key, stream, type, filename) {
|
||||
if (type === "image/apng") type = "image/png";
|
||||
if (!FILE_TYPE_BROWSERSAFE.includes(type)) type = "application/octet-stream";
|
||||
const meta = await fetchMeta();
|
||||
const params = {
|
||||
Bucket: meta.objectStorageBucket,
|
||||
Key: key,
|
||||
Body: Buffer.isBuffer(stream) ? Readable.from(stream) : stream,
|
||||
ContentType: type,
|
||||
CacheControl: "max-age=31536000, immutable"
|
||||
};
|
||||
if (filename) params.ContentDisposition = contentDisposition("inline", filename);
|
||||
if (meta.objectStorageSetPublicRead) params.ACL = "public-read";
|
||||
const s3 = getS3(meta);
|
||||
const upload = new Upload({
|
||||
client: s3,
|
||||
params,
|
||||
partSize: meta.objectStorageEndpoint === "storage.googleapis.com" ? 500 * 1024 * 1024 : 8 * 1024 * 1024
|
||||
});
|
||||
const result = await upload.done();
|
||||
if ("Location" in result) logger.debug(`Uploaded: ${result.Bucket}/${result.Key} => ${result.Location}`);
|
||||
}
|
||||
async function expireOldFile(user, driveCapacity) {
|
||||
const q = DriveFiles.createQueryBuilder("file").where("file.userId = :userId", {
|
||||
userId: user.id
|
||||
}).andWhere("file.isLink = FALSE");
|
||||
if (user.avatarId) {
|
||||
q.andWhere("file.id != :avatarId", {
|
||||
avatarId: user.avatarId
|
||||
});
|
||||
}
|
||||
if (user.bannerId) {
|
||||
q.andWhere("file.id != :bannerId", {
|
||||
bannerId: user.bannerId
|
||||
});
|
||||
}
|
||||
//This selete is hard coded, be careful if change database schema
|
||||
q.addSelect('SUM("file"."size") OVER (ORDER BY "file"."id" DESC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)', "acc_usage");
|
||||
q.orderBy("file.id", "ASC");
|
||||
const fileList = await q.getRawMany();
|
||||
const exceedFileIds = fileList.filter((x)=>x.acc_usage > driveCapacity).map((x)=>x.file_id);
|
||||
for (const fileId of exceedFileIds){
|
||||
const file = await DriveFiles.findOneBy({
|
||||
id: fileId
|
||||
});
|
||||
if (file) deleteFile(file, true);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Add file to drive
|
||||
*
|
||||
*/ export async function addFile({ user, path, name = null, comment = null, folderId = null, force = false, isLink = false, url = null, uri = null, sensitive = null, isDatabase = false, requestIp = null, requestHeaders = null }) {
|
||||
const info = await getFileInfo(path);
|
||||
logger.info(`${JSON.stringify(info)}`);
|
||||
// 現状 false positive が多すぎて実用に耐えない
|
||||
//if (info.porn && instance.disallowUploadWhenPredictedAsPorn) {
|
||||
// throw new IdentifiableError('282f77bf-5816-4f72-9264-aa14d8261a21', 'Detected as porn.');
|
||||
//}
|
||||
// detect name
|
||||
const detectedName = name || (info.type.ext ? `untitled.${info.type.ext}` : "untitled");
|
||||
if (detectedName.toLowerCase().endsWith(".epub") && [
|
||||
"application/octet-stream",
|
||||
"application/zip"
|
||||
].includes(info.type.mime)) {
|
||||
info.type = {
|
||||
mime: "application/epub+zip",
|
||||
ext: "epub"
|
||||
};
|
||||
}
|
||||
if (user && !force) {
|
||||
// Check if there is a file with the same hash
|
||||
const much = await DriveFiles.findOneBy({
|
||||
md5: info.md5,
|
||||
userId: user.id
|
||||
});
|
||||
if (much) {
|
||||
logger.info(`file with same hash is found: ${much.id}`);
|
||||
return much;
|
||||
}
|
||||
}
|
||||
//#region Check drive usage
|
||||
if (user && !isLink) {
|
||||
const u = await Users.findOneBy({
|
||||
id: user.id
|
||||
});
|
||||
const instance = await fetchMeta();
|
||||
if (isDatabase) {
|
||||
const usage = await DriveFiles.calcDatabaseUsageOf(user);
|
||||
const databaseCapacity = 1024 * 1024 * instance.lua4frozenDatabaseCapacityMb;
|
||||
logger.debug(`database usage is ${usage} (max: ${databaseCapacity})`);
|
||||
if (usage + info.size > databaseCapacity) {
|
||||
throw new IdentifiableError("9ec2c4e9-c4df-4f40-b80b-4f57b32d28df", "No free database space.");
|
||||
}
|
||||
} else {
|
||||
const usage = await DriveFiles.calcDriveUsageOf(user);
|
||||
let driveCapacity = 1024 * 1024 * (Users.isLocalUser(user) ? instance.localDriveCapacityMb : instance.remoteDriveCapacityMb);
|
||||
if (Users.isLocalUser(user) && u?.driveCapacityOverrideMb != null) {
|
||||
driveCapacity = 1024 * 1024 * u.driveCapacityOverrideMb;
|
||||
logger.debug("drive capacity override applied");
|
||||
logger.debug(`overrideCap: ${driveCapacity}bytes, usage: ${usage}bytes, u+s: ${usage + info.size}bytes`);
|
||||
}
|
||||
logger.debug(`drive usage is ${usage} (max: ${driveCapacity})`);
|
||||
// If usage limit exceeded
|
||||
if (usage + info.size > driveCapacity) {
|
||||
if (Users.isLocalUser(user)) {
|
||||
throw new IdentifiableError("c6244ed2-a39a-4e1c-bf93-f0fbd7764fa6", "No free space.");
|
||||
} else {
|
||||
// (アバターまたはバナーを含まず)最も古いファイルを削除する
|
||||
expireOldFile(await Users.findOneByOrFail({
|
||||
id: user.id
|
||||
}), driveCapacity - info.size);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
//#endregion
|
||||
const fetchFolder = async ()=>{
|
||||
if (!folderId) {
|
||||
return null;
|
||||
}
|
||||
const driveFolder = await DriveFolders.findOneBy({
|
||||
id: folderId,
|
||||
userId: user ? user.id : IsNull()
|
||||
});
|
||||
if (driveFolder == null) throw new Error("folder-not-found");
|
||||
return driveFolder;
|
||||
};
|
||||
const properties = {};
|
||||
if (info.width) {
|
||||
properties["width"] = info.width;
|
||||
properties["height"] = info.height;
|
||||
}
|
||||
if (info.orientation != null) {
|
||||
properties["orientation"] = info.orientation;
|
||||
}
|
||||
const profile = user ? await UserProfiles.findOneBy({
|
||||
userId: user.id
|
||||
}) : null;
|
||||
const folder = await fetchFolder();
|
||||
let file = new DriveFile();
|
||||
file.id = genId();
|
||||
file.createdAt = new Date();
|
||||
file.userId = user ? user.id : null;
|
||||
file.userHost = user ? user.host : null;
|
||||
file.folderId = folder !== null ? folder.id : null;
|
||||
file.comment = comment;
|
||||
file.properties = properties;
|
||||
file.blurhash = info.blurhash || null;
|
||||
file.isLink = isLink;
|
||||
file.requestIp = requestIp;
|
||||
file.requestHeaders = requestHeaders;
|
||||
file.isDatabase = isDatabase;
|
||||
file.isSensitive = user ? Users.isLocalUser(user) && profile.alwaysMarkNsfw ? true : sensitive !== null && sensitive !== undefined ? sensitive : false : false;
|
||||
if (url !== null) {
|
||||
file.src = url;
|
||||
if (isLink) {
|
||||
file.url = url;
|
||||
// ローカルプロキシ用
|
||||
file.accessKey = uuid();
|
||||
file.thumbnailAccessKey = `thumbnail-${uuid()}`;
|
||||
file.webpublicAccessKey = `webpublic-${uuid()}`;
|
||||
}
|
||||
}
|
||||
if (uri !== null) {
|
||||
file.uri = uri;
|
||||
}
|
||||
if (isLink) {
|
||||
try {
|
||||
file.size = 0;
|
||||
file.md5 = info.md5;
|
||||
file.name = detectedName;
|
||||
file.type = info.type.mime;
|
||||
file.storedInternal = false;
|
||||
file = await DriveFiles.insert(file).then((x)=>DriveFiles.findOneByOrFail(x.identifiers[0]));
|
||||
} catch (err) {
|
||||
// duplicate key error (when already registered)
|
||||
if (isDuplicateKeyValueError(err)) {
|
||||
logger.info(`already registered ${file.uri}`);
|
||||
file = await DriveFiles.findOneBy({
|
||||
uri: file.uri,
|
||||
userId: user ? user.id : IsNull()
|
||||
});
|
||||
} else {
|
||||
logger.error(err);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
file = await save(file, path, detectedName, info.type.mime, info.md5, info.size);
|
||||
}
|
||||
logger.succ(`drive file has been created ${file.id}`);
|
||||
if (user) {
|
||||
DriveFiles.pack(file, {
|
||||
self: true
|
||||
}).then((packedFile)=>{
|
||||
// Publish driveFileCreated event
|
||||
publishMainStream(user.id, "driveFileCreated", packedFile);
|
||||
publishDriveStream(user.id, "fileCreated", packedFile);
|
||||
});
|
||||
}
|
||||
// 統計を更新
|
||||
driveChart.update(file, true);
|
||||
perUserDriveChart.update(file, true);
|
||||
if (file.userHost !== null) {
|
||||
instanceChart.updateDrive(file, true);
|
||||
}
|
||||
return file;
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
import { InternalStorage } from "./internal-storage.js";
|
||||
import { DriveFiles, Users } from "../../models/index.js";
|
||||
import { driveChart, perUserDriveChart, instanceChart } from "../chart/index.js";
|
||||
import { createDeleteObjectStorageFileJob } from "../../queue/index.js";
|
||||
import { fetchMeta } from "../../misc/fetch-meta.js";
|
||||
import { getS3 } from "./s3.js";
|
||||
import { v4 as uuid } from "uuid";
|
||||
import { DeleteObjectCommand } from "@aws-sdk/client-s3";
|
||||
export async function deleteFile(file, isExpired = false) {
|
||||
if (file.storedInternal) {
|
||||
InternalStorage.del(file.accessKey);
|
||||
if (file.thumbnailUrl) {
|
||||
InternalStorage.del(file.thumbnailAccessKey);
|
||||
}
|
||||
if (file.webpublicUrl) {
|
||||
InternalStorage.del(file.webpublicAccessKey);
|
||||
}
|
||||
} else if (!file.isLink) {
|
||||
createDeleteObjectStorageFileJob(file.accessKey);
|
||||
if (file.thumbnailUrl) {
|
||||
createDeleteObjectStorageFileJob(file.thumbnailAccessKey);
|
||||
}
|
||||
if (file.webpublicUrl) {
|
||||
createDeleteObjectStorageFileJob(file.webpublicAccessKey);
|
||||
}
|
||||
}
|
||||
postProcess(file, isExpired);
|
||||
}
|
||||
export async function deleteFileSync(file, isExpired = false) {
|
||||
if (file.storedInternal) {
|
||||
InternalStorage.del(file.accessKey);
|
||||
if (file.thumbnailUrl) {
|
||||
InternalStorage.del(file.thumbnailAccessKey);
|
||||
}
|
||||
if (file.webpublicUrl) {
|
||||
InternalStorage.del(file.webpublicAccessKey);
|
||||
}
|
||||
} else if (!file.isLink) {
|
||||
const promises = [];
|
||||
promises.push(deleteObjectStorageFile(file.accessKey));
|
||||
if (file.thumbnailUrl) {
|
||||
promises.push(deleteObjectStorageFile(file.thumbnailAccessKey));
|
||||
}
|
||||
if (file.webpublicUrl) {
|
||||
promises.push(deleteObjectStorageFile(file.webpublicAccessKey));
|
||||
}
|
||||
await Promise.all(promises);
|
||||
}
|
||||
postProcess(file, isExpired);
|
||||
}
|
||||
async function postProcess(file, isExpired = false) {
|
||||
// リモートファイル期限切れ削除後は直リンクにする
|
||||
if (isExpired && file.userHost !== null && file.uri != null) {
|
||||
DriveFiles.update(file.id, {
|
||||
isLink: true,
|
||||
url: file.uri,
|
||||
thumbnailUrl: null,
|
||||
webpublicUrl: null,
|
||||
storedInternal: false,
|
||||
// ローカルプロキシ用
|
||||
accessKey: uuid(),
|
||||
thumbnailAccessKey: `thumbnail-${uuid()}`,
|
||||
webpublicAccessKey: `webpublic-${uuid()}`
|
||||
});
|
||||
Users.update({
|
||||
avatarId: file.id
|
||||
}, {
|
||||
avatarUrl: file.uri
|
||||
});
|
||||
Users.update({
|
||||
bannerId: file.id
|
||||
}, {
|
||||
bannerUrl: file.uri
|
||||
});
|
||||
} else {
|
||||
DriveFiles.delete(file.id);
|
||||
}
|
||||
// 統計を更新
|
||||
driveChart.update(file, false);
|
||||
perUserDriveChart.update(file, false);
|
||||
if (file.userHost !== null) {
|
||||
instanceChart.updateDrive(file, false);
|
||||
}
|
||||
}
|
||||
export async function deleteObjectStorageFile(key) {
|
||||
const meta = await fetchMeta();
|
||||
const s3 = getS3(meta);
|
||||
await s3.send(new DeleteObjectCommand({
|
||||
Bucket: meta.objectStorageBucket,
|
||||
Key: key
|
||||
}));
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
import { createTempDir } from "../../misc/create-temp.js";
|
||||
import { convertToWebp } from "./image-processor.js";
|
||||
import FFmpeg from "fluent-ffmpeg";
|
||||
export async function GenerateVideoThumbnail(source) {
|
||||
const [dir, cleanup] = await createTempDir();
|
||||
try {
|
||||
await new Promise((res, rej)=>{
|
||||
FFmpeg({
|
||||
source
|
||||
}).on("end", res).on("error", rej).screenshot({
|
||||
folder: dir,
|
||||
filename: "out.png",
|
||||
count: 1,
|
||||
timestamps: [
|
||||
"5%"
|
||||
]
|
||||
});
|
||||
});
|
||||
return await convertToWebp(`${dir}/out.png`, 996, 560);
|
||||
} finally{
|
||||
cleanup();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
import sharp from "sharp";
|
||||
/**
|
||||
* Convert to WebP
|
||||
* with resize, remove metadata, resolve orientation, stop animation
|
||||
*/ export async function convertToWebp(path, width, height, quality = 85) {
|
||||
return convertSharpToWebp(await sharp(path), width, height, quality);
|
||||
}
|
||||
export async function convertSharpToWebp(sharp1, width, height, quality = 85) {
|
||||
const data = await sharp1.resize(width, height, {
|
||||
fit: "inside",
|
||||
withoutEnlargement: true
|
||||
}).rotate().webp({
|
||||
quality
|
||||
}).toBuffer();
|
||||
return {
|
||||
data,
|
||||
ext: "webp",
|
||||
type: "image/webp"
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
import * as fs from "node:fs";
|
||||
import * as Path from "node:path";
|
||||
import { pipeline } from "node:stream/promises";
|
||||
import config from "../../config/index.js";
|
||||
export class InternalStorage {
|
||||
static resolvePath = (key)=>Path.resolve(config.mediaDir, key);
|
||||
static read(key) {
|
||||
return fs.createReadStream(InternalStorage.resolvePath(key));
|
||||
}
|
||||
static async saveFromPath(key, srcPath) {
|
||||
fs.mkdirSync(config.mediaDir, {
|
||||
recursive: true
|
||||
});
|
||||
await pipeline(fs.createReadStream(srcPath), fs.createWriteStream(InternalStorage.resolvePath(key)));
|
||||
return `${config.url}/files/${key}`;
|
||||
}
|
||||
static saveFromBuffer(key, data) {
|
||||
fs.mkdirSync(config.mediaDir, {
|
||||
recursive: true
|
||||
});
|
||||
fs.writeFileSync(InternalStorage.resolvePath(key), data);
|
||||
return `${config.url}/files/${key}`;
|
||||
}
|
||||
static del(key) {
|
||||
fs.unlink(InternalStorage.resolvePath(key), ()=>{});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
import Logger from "../logger.js";
|
||||
export const driveLogger = new Logger("drive", "blue");
|
||||
@@ -0,0 +1,25 @@
|
||||
import { URL } from "node:url";
|
||||
import { getAgentByUrl } from "../../misc/fetch.js";
|
||||
import { S3Client } from "@aws-sdk/client-s3";
|
||||
import { NodeHttpHandler } from "@smithy/node-http-handler";
|
||||
export function getS3(meta) {
|
||||
const endpointPrefixed = meta.objectStorageEndpoint && (meta.objectStorageEndpoint.startsWith("http://") || meta.objectStorageEndpoint.startsWith("https://"));
|
||||
const endpoint = meta.objectStorageEndpoint ? endpointPrefixed ? meta.objectStorageEndpoint : `${meta.objectStorageUseSSL ? "https://" : "http://"}${meta.objectStorageEndpoint}` : undefined;
|
||||
const agentUrl = new URL(endpoint);
|
||||
const agent = getAgentByUrl(agentUrl, !meta.objectStorageUseProxy);
|
||||
return new S3Client({
|
||||
endpoint,
|
||||
region: meta.objectStorageRegion || "us-east-1",
|
||||
credentials: {
|
||||
accessKeyId: meta.objectStorageAccessKey,
|
||||
secretAccessKey: meta.objectStorageSecretKey
|
||||
},
|
||||
forcePathStyle: meta.objectStorageEndpoint ? meta.objectStorageS3ForcePathStyle : false,
|
||||
requestChecksumCalculation: "WHEN_REQUIRED",
|
||||
responseChecksumValidation: "WHEN_REQUIRED",
|
||||
requestHandler: new NodeHttpHandler({
|
||||
httpAgent: agent,
|
||||
httpsAgent: agent
|
||||
})
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
import { URL } from "node:url";
|
||||
import { createTemp } from "../../misc/create-temp.js";
|
||||
import { downloadUrl } from "../../misc/download-url.js";
|
||||
import { DriveFiles } from "../../models/index.js";
|
||||
import { driveLogger } from "./logger.js";
|
||||
import { addFile } from "./add-file.js";
|
||||
const logger = driveLogger.createSubLogger("downloader");
|
||||
export async function uploadFromUrl({ url, user, folderId = null, uri = null, sensitive = false, force = false, isLink = false, comment = null, requestIp = null, requestHeaders = null }) {
|
||||
let name = new URL(url).pathname.split("/").pop() || null;
|
||||
if (name == null || !DriveFiles.validateFileName(name)) {
|
||||
name = null;
|
||||
}
|
||||
// If the comment is same as the name, skip comment
|
||||
// (image.name is passed in when receiving attachment)
|
||||
if (comment !== null && name === comment) {
|
||||
comment = null;
|
||||
}
|
||||
// Create temp file
|
||||
const [path, cleanup] = await createTemp();
|
||||
try {
|
||||
// write content at URL to temp file
|
||||
await downloadUrl(url, path);
|
||||
const driveFile = await addFile({
|
||||
user,
|
||||
path,
|
||||
name,
|
||||
comment,
|
||||
folderId,
|
||||
force,
|
||||
isLink,
|
||||
url,
|
||||
uri,
|
||||
sensitive,
|
||||
requestIp,
|
||||
requestHeaders
|
||||
});
|
||||
logger.succ(`Got: ${driveFile.id}`);
|
||||
return driveFile;
|
||||
} catch (e) {
|
||||
logger.error(`Failed to create drive file: ${e}`, {
|
||||
url: url,
|
||||
e: e
|
||||
});
|
||||
throw e;
|
||||
} finally{
|
||||
cleanup();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user