447 lines
17 KiB
JavaScript
447 lines
17 KiB
JavaScript
import * as fs from "node:fs";
|
|
import { Readable } from "node:stream";
|
|
import { v4 as uuid } from "uuid";
|
|
import sharp from "sharp";
|
|
import { IsNull } from "typeorm";
|
|
import { publishMainStream, publishDriveStream } from "../stream.js";
|
|
import { fetchMeta } from "../../misc/fetch-meta.js";
|
|
import { contentDisposition } from "../../misc/content-disposition.js";
|
|
import { getFileInfo } from "../../misc/get-file-info.js";
|
|
import { DriveFiles, DriveFolders, Users, UserProfiles } from "../../models/index.js";
|
|
import { DriveFile } from "../../models/entities/drive-file.js";
|
|
import { driveChart, perUserDriveChart, instanceChart } from "../chart/index.js";
|
|
import { genId } from "../../misc/gen-id.js";
|
|
import { isDuplicateKeyValueError } from "../../misc/is-duplicate-key-value-error.js";
|
|
import { FILE_TYPE_BROWSERSAFE } from "../../const.js";
|
|
import { IdentifiableError } from "../../misc/identifiable-error.js";
|
|
import { getS3 } from "./s3.js";
|
|
import { InternalStorage } from "./internal-storage.js";
|
|
import { convertSharpToWebp } from "./image-processor.js";
|
|
import { driveLogger } from "./logger.js";
|
|
import { GenerateVideoThumbnail } from "./generate-video-thumbnail.js";
|
|
import { deleteFile } from "./delete-file.js";
|
|
import { Upload } from "@aws-sdk/lib-storage";
|
|
const logger = driveLogger.createSubLogger("register", "yellow");
|
|
/***
|
|
* Save file
|
|
* @param path Path for original
|
|
* @param name Name for original
|
|
* @param type Content-Type for original
|
|
* @param hash Hash for original
|
|
* @param size Size for original
|
|
*/ async function save(file, path, name, type, hash, size) {
|
|
// thunbnail, webpublic を必要なら生成
|
|
const alts = await generateAlts(path, type, !file.uri);
|
|
const meta = await fetchMeta();
|
|
if (meta.useObjectStorage) {
|
|
//#region ObjectStorage params
|
|
let [ext] = name.match(/\.([a-zA-Z0-9_-]+)$/) || [
|
|
""
|
|
];
|
|
if (ext === "") {
|
|
if (type === "image/jpeg") ext = ".jpg";
|
|
if (type === "image/png") ext = ".png";
|
|
if (type === "image/webp") ext = ".webp";
|
|
if (type === "image/apng") ext = ".apng";
|
|
if (type === "image/avif") ext = ".avif";
|
|
if (type === "image/vnd.mozilla.apng") ext = ".apng";
|
|
}
|
|
// Some cloud providers (notably upcloud) will infer the content-type based
|
|
// on extension, so we remove extensions from non-browser-safe types.
|
|
if (!FILE_TYPE_BROWSERSAFE.includes(type)) {
|
|
ext = "";
|
|
}
|
|
const baseUrl = meta.objectStorageBaseUrl || `${meta.objectStorageUseSSL ? "https" : "http"}://${meta.objectStorageEndpoint}${meta.objectStoragePort ? `:${meta.objectStoragePort}` : ""}/${meta.objectStorageBucket}`;
|
|
// for original
|
|
const key = `${meta.objectStoragePrefix}/${uuid()}${ext}`;
|
|
const url = `${baseUrl}/${key}`;
|
|
// for alts
|
|
let webpublicKey = null;
|
|
let webpublicUrl = null;
|
|
let thumbnailKey = null;
|
|
let thumbnailUrl = null;
|
|
//#endregion
|
|
//#region Uploads
|
|
logger.info(`uploading original: ${key}`);
|
|
const uploads = [
|
|
upload(key, fs.createReadStream(path), type, name)
|
|
];
|
|
if (alts.webpublic) {
|
|
webpublicKey = `${meta.objectStoragePrefix}/webpublic-${uuid()}.${alts.webpublic.ext}`;
|
|
webpublicUrl = `${baseUrl}/${webpublicKey}`;
|
|
logger.info(`uploading webpublic: ${webpublicKey}`);
|
|
uploads.push(upload(webpublicKey, alts.webpublic.data, alts.webpublic.type, name));
|
|
}
|
|
if (alts.thumbnail) {
|
|
thumbnailKey = `${meta.objectStoragePrefix}/thumbnail-${uuid()}.${alts.thumbnail.ext}`;
|
|
thumbnailUrl = `${baseUrl}/${thumbnailKey}`;
|
|
logger.info(`uploading thumbnail: ${thumbnailKey}`);
|
|
uploads.push(upload(thumbnailKey, alts.thumbnail.data, alts.thumbnail.type));
|
|
}
|
|
await Promise.all(uploads);
|
|
//#endregion
|
|
file.url = url;
|
|
file.thumbnailUrl = thumbnailUrl;
|
|
file.webpublicUrl = webpublicUrl;
|
|
file.accessKey = key;
|
|
file.thumbnailAccessKey = thumbnailKey;
|
|
file.webpublicAccessKey = webpublicKey;
|
|
file.webpublicType = alts.webpublic?.type ?? null;
|
|
file.name = name;
|
|
file.type = type;
|
|
file.md5 = hash;
|
|
file.size = size;
|
|
file.storedInternal = false;
|
|
return await DriveFiles.insert(file).then((x)=>DriveFiles.findOneByOrFail(x.identifiers[0]));
|
|
} else {
|
|
// use internal storage
|
|
const accessKey = uuid();
|
|
const thumbnailAccessKey = `thumbnail-${uuid()}`;
|
|
const webpublicAccessKey = `webpublic-${uuid()}`;
|
|
const url = await InternalStorage.saveFromPath(accessKey, path);
|
|
let thumbnailUrl = null;
|
|
let webpublicUrl = null;
|
|
if (alts.thumbnail) {
|
|
thumbnailUrl = InternalStorage.saveFromBuffer(thumbnailAccessKey, alts.thumbnail.data);
|
|
logger.info(`thumbnail stored: ${thumbnailAccessKey}`);
|
|
}
|
|
if (alts.webpublic) {
|
|
webpublicUrl = InternalStorage.saveFromBuffer(webpublicAccessKey, alts.webpublic.data);
|
|
logger.info(`web stored: ${webpublicAccessKey}`);
|
|
}
|
|
file.storedInternal = true;
|
|
file.url = url;
|
|
file.thumbnailUrl = thumbnailUrl;
|
|
file.webpublicUrl = webpublicUrl;
|
|
file.accessKey = accessKey;
|
|
file.thumbnailAccessKey = thumbnailAccessKey;
|
|
file.webpublicAccessKey = webpublicAccessKey;
|
|
file.webpublicType = alts.webpublic?.type ?? null;
|
|
file.name = name;
|
|
file.type = type;
|
|
file.md5 = hash;
|
|
file.size = size;
|
|
return await DriveFiles.insert(file).then((x)=>DriveFiles.findOneByOrFail(x.identifiers[0]));
|
|
}
|
|
}
|
|
/**
|
|
* Generate webpublic, thumbnail, etc
|
|
* @param path Path for original
|
|
* @param type Content-Type for original
|
|
* @param generateWeb Generate webpublic or not
|
|
*/ export async function generateAlts(path, type, generateWeb) {
|
|
if (type.startsWith("video/")) {
|
|
try {
|
|
const thumbnail = await GenerateVideoThumbnail(path);
|
|
return {
|
|
webpublic: null,
|
|
thumbnail
|
|
};
|
|
} catch (err) {
|
|
logger.warn(`GenerateVideoThumbnail failed: ${err}`);
|
|
return {
|
|
webpublic: null,
|
|
thumbnail: null
|
|
};
|
|
}
|
|
}
|
|
if (![
|
|
"image/jpeg",
|
|
"image/png",
|
|
"image/webp",
|
|
"image/svg+xml",
|
|
"image/avif"
|
|
].includes(type)) {
|
|
logger.debug("web image and thumbnail not created (not an required file)");
|
|
return {
|
|
webpublic: null,
|
|
thumbnail: null
|
|
};
|
|
}
|
|
let img = null;
|
|
let satisfyWebpublic;
|
|
try {
|
|
img = sharp(path);
|
|
const metadata = await img.metadata();
|
|
const isAnimated = metadata.pages && metadata.pages > 1;
|
|
// skip animated
|
|
if (isAnimated) {
|
|
return {
|
|
webpublic: null,
|
|
thumbnail: null
|
|
};
|
|
}
|
|
satisfyWebpublic = !!(type !== "image/svg+xml" && type !== "image/webp" && !(metadata.exif || metadata.iptc || metadata.xmp || metadata.tifftagPhotoshop) && metadata.width && metadata.width <= 2048 && metadata.height && metadata.height <= 2048);
|
|
} catch (err) {
|
|
logger.warn(`sharp failed: ${err}`);
|
|
return {
|
|
webpublic: null,
|
|
thumbnail: null
|
|
};
|
|
}
|
|
// #region webpublic
|
|
let webpublic = null;
|
|
if (generateWeb && !satisfyWebpublic) {
|
|
logger.info("creating web image");
|
|
try {
|
|
if ([
|
|
"image/jpeg"
|
|
].includes(type)) {
|
|
webpublic = await convertSharpToWebp(img, 2048, 2048);
|
|
} else if ([
|
|
"image/webp"
|
|
].includes(type)) {
|
|
webpublic = await convertSharpToWebp(img, 2048, 2048);
|
|
} else if ([
|
|
"image/png"
|
|
].includes(type)) {
|
|
webpublic = await convertSharpToWebp(img, 2048, 2048, 100);
|
|
} else if ([
|
|
"image/svg+xml"
|
|
].includes(type)) {
|
|
webpublic = await convertSharpToWebp(img, 2048, 2048);
|
|
} else {
|
|
logger.debug("web image not created (not an required image)");
|
|
}
|
|
} catch (err) {
|
|
logger.warn("web image not created (an error occured)", err);
|
|
}
|
|
} else {
|
|
if (satisfyWebpublic) logger.info("web image not created (original satisfies webpublic)");
|
|
else logger.info("web image not created (from remote)");
|
|
}
|
|
// #endregion webpublic
|
|
// #region thumbnail
|
|
let thumbnail = null;
|
|
try {
|
|
if ([
|
|
"image/jpeg",
|
|
"image/webp",
|
|
"image/png",
|
|
"image/svg+xml",
|
|
"image/avif"
|
|
].includes(type)) {
|
|
thumbnail = await convertSharpToWebp(img, 996, 560);
|
|
} else {
|
|
logger.debug("thumbnail not created (not an required file)");
|
|
}
|
|
} catch (err) {
|
|
logger.warn("thumbnail not created (an error occured)", err);
|
|
}
|
|
// #endregion thumbnail
|
|
return {
|
|
webpublic,
|
|
thumbnail
|
|
};
|
|
}
|
|
/**
|
|
* Upload to ObjectStorage
|
|
*/ async function upload(key, stream, type, filename) {
|
|
if (type === "image/apng") type = "image/png";
|
|
if (!FILE_TYPE_BROWSERSAFE.includes(type)) type = "application/octet-stream";
|
|
const meta = await fetchMeta();
|
|
const params = {
|
|
Bucket: meta.objectStorageBucket,
|
|
Key: key,
|
|
Body: Buffer.isBuffer(stream) ? Readable.from(stream) : stream,
|
|
ContentType: type,
|
|
CacheControl: "max-age=31536000, immutable"
|
|
};
|
|
if (filename) params.ContentDisposition = contentDisposition("inline", filename);
|
|
if (meta.objectStorageSetPublicRead) params.ACL = "public-read";
|
|
const s3 = getS3(meta);
|
|
const upload = new Upload({
|
|
client: s3,
|
|
params,
|
|
partSize: meta.objectStorageEndpoint === "storage.googleapis.com" ? 500 * 1024 * 1024 : 8 * 1024 * 1024
|
|
});
|
|
const result = await upload.done();
|
|
if ("Location" in result) logger.debug(`Uploaded: ${result.Bucket}/${result.Key} => ${result.Location}`);
|
|
}
|
|
async function expireOldFile(user, driveCapacity) {
|
|
const q = DriveFiles.createQueryBuilder("file").where("file.userId = :userId", {
|
|
userId: user.id
|
|
}).andWhere("file.isLink = FALSE");
|
|
if (user.avatarId) {
|
|
q.andWhere("file.id != :avatarId", {
|
|
avatarId: user.avatarId
|
|
});
|
|
}
|
|
if (user.bannerId) {
|
|
q.andWhere("file.id != :bannerId", {
|
|
bannerId: user.bannerId
|
|
});
|
|
}
|
|
//This selete is hard coded, be careful if change database schema
|
|
q.addSelect('SUM("file"."size") OVER (ORDER BY "file"."id" DESC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)', "acc_usage");
|
|
q.orderBy("file.id", "ASC");
|
|
const fileList = await q.getRawMany();
|
|
const exceedFileIds = fileList.filter((x)=>x.acc_usage > driveCapacity).map((x)=>x.file_id);
|
|
for (const fileId of exceedFileIds){
|
|
const file = await DriveFiles.findOneBy({
|
|
id: fileId
|
|
});
|
|
if (file) deleteFile(file, true);
|
|
}
|
|
}
|
|
/**
|
|
* Add file to drive
|
|
*
|
|
*/ export async function addFile({ user, path, name = null, comment = null, folderId = null, force = false, isLink = false, url = null, uri = null, sensitive = null, isDatabase = false, requestIp = null, requestHeaders = null }) {
|
|
const info = await getFileInfo(path);
|
|
logger.info(`${JSON.stringify(info)}`);
|
|
// 現状 false positive が多すぎて実用に耐えない
|
|
//if (info.porn && instance.disallowUploadWhenPredictedAsPorn) {
|
|
// throw new IdentifiableError('282f77bf-5816-4f72-9264-aa14d8261a21', 'Detected as porn.');
|
|
//}
|
|
// detect name
|
|
const detectedName = name || (info.type.ext ? `untitled.${info.type.ext}` : "untitled");
|
|
if (detectedName.toLowerCase().endsWith(".epub") && [
|
|
"application/octet-stream",
|
|
"application/zip"
|
|
].includes(info.type.mime)) {
|
|
info.type = {
|
|
mime: "application/epub+zip",
|
|
ext: "epub"
|
|
};
|
|
}
|
|
if (user && !force) {
|
|
// Check if there is a file with the same hash
|
|
const much = await DriveFiles.findOneBy({
|
|
md5: info.md5,
|
|
userId: user.id
|
|
});
|
|
if (much) {
|
|
logger.info(`file with same hash is found: ${much.id}`);
|
|
return much;
|
|
}
|
|
}
|
|
//#region Check drive usage
|
|
if (user && !isLink) {
|
|
const u = await Users.findOneBy({
|
|
id: user.id
|
|
});
|
|
const instance = await fetchMeta();
|
|
if (isDatabase) {
|
|
const usage = await DriveFiles.calcDatabaseUsageOf(user);
|
|
const databaseCapacity = 1024 * 1024 * instance.lua4frozenDatabaseCapacityMb;
|
|
logger.debug(`database usage is ${usage} (max: ${databaseCapacity})`);
|
|
if (usage + info.size > databaseCapacity) {
|
|
throw new IdentifiableError("9ec2c4e9-c4df-4f40-b80b-4f57b32d28df", "No free database space.");
|
|
}
|
|
} else {
|
|
const usage = await DriveFiles.calcDriveUsageOf(user);
|
|
let driveCapacity = 1024 * 1024 * (Users.isLocalUser(user) ? instance.localDriveCapacityMb : instance.remoteDriveCapacityMb);
|
|
if (Users.isLocalUser(user) && u?.driveCapacityOverrideMb != null) {
|
|
driveCapacity = 1024 * 1024 * u.driveCapacityOverrideMb;
|
|
logger.debug("drive capacity override applied");
|
|
logger.debug(`overrideCap: ${driveCapacity}bytes, usage: ${usage}bytes, u+s: ${usage + info.size}bytes`);
|
|
}
|
|
logger.debug(`drive usage is ${usage} (max: ${driveCapacity})`);
|
|
// If usage limit exceeded
|
|
if (usage + info.size > driveCapacity) {
|
|
if (Users.isLocalUser(user)) {
|
|
throw new IdentifiableError("c6244ed2-a39a-4e1c-bf93-f0fbd7764fa6", "No free space.");
|
|
} else {
|
|
// (アバターまたはバナーを含まず)最も古いファイルを削除する
|
|
expireOldFile(await Users.findOneByOrFail({
|
|
id: user.id
|
|
}), driveCapacity - info.size);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
//#endregion
|
|
const fetchFolder = async ()=>{
|
|
if (!folderId) {
|
|
return null;
|
|
}
|
|
const driveFolder = await DriveFolders.findOneBy({
|
|
id: folderId,
|
|
userId: user ? user.id : IsNull()
|
|
});
|
|
if (driveFolder == null) throw new Error("folder-not-found");
|
|
return driveFolder;
|
|
};
|
|
const properties = {};
|
|
if (info.width) {
|
|
properties["width"] = info.width;
|
|
properties["height"] = info.height;
|
|
}
|
|
if (info.orientation != null) {
|
|
properties["orientation"] = info.orientation;
|
|
}
|
|
const profile = user ? await UserProfiles.findOneBy({
|
|
userId: user.id
|
|
}) : null;
|
|
const folder = await fetchFolder();
|
|
let file = new DriveFile();
|
|
file.id = genId();
|
|
file.createdAt = new Date();
|
|
file.userId = user ? user.id : null;
|
|
file.userHost = user ? user.host : null;
|
|
file.folderId = folder !== null ? folder.id : null;
|
|
file.comment = comment;
|
|
file.properties = properties;
|
|
file.blurhash = info.blurhash || null;
|
|
file.isLink = isLink;
|
|
file.requestIp = requestIp;
|
|
file.requestHeaders = requestHeaders;
|
|
file.isDatabase = isDatabase;
|
|
file.isSensitive = user ? Users.isLocalUser(user) && profile.alwaysMarkNsfw ? true : sensitive !== null && sensitive !== undefined ? sensitive : false : false;
|
|
if (url !== null) {
|
|
file.src = url;
|
|
if (isLink) {
|
|
file.url = url;
|
|
// ローカルプロキシ用
|
|
file.accessKey = uuid();
|
|
file.thumbnailAccessKey = `thumbnail-${uuid()}`;
|
|
file.webpublicAccessKey = `webpublic-${uuid()}`;
|
|
}
|
|
}
|
|
if (uri !== null) {
|
|
file.uri = uri;
|
|
}
|
|
if (isLink) {
|
|
try {
|
|
file.size = 0;
|
|
file.md5 = info.md5;
|
|
file.name = detectedName;
|
|
file.type = info.type.mime;
|
|
file.storedInternal = false;
|
|
file = await DriveFiles.insert(file).then((x)=>DriveFiles.findOneByOrFail(x.identifiers[0]));
|
|
} catch (err) {
|
|
// duplicate key error (when already registered)
|
|
if (isDuplicateKeyValueError(err)) {
|
|
logger.info(`already registered ${file.uri}`);
|
|
file = await DriveFiles.findOneBy({
|
|
uri: file.uri,
|
|
userId: user ? user.id : IsNull()
|
|
});
|
|
} else {
|
|
logger.error(err);
|
|
throw err;
|
|
}
|
|
}
|
|
} else {
|
|
file = await save(file, path, detectedName, info.type.mime, info.md5, info.size);
|
|
}
|
|
logger.succ(`drive file has been created ${file.id}`);
|
|
if (user) {
|
|
DriveFiles.pack(file, {
|
|
self: true
|
|
}).then((packedFile)=>{
|
|
// Publish driveFileCreated event
|
|
publishMainStream(user.id, "driveFileCreated", packedFile);
|
|
publishDriveStream(user.id, "fileCreated", packedFile);
|
|
});
|
|
}
|
|
// 統計を更新
|
|
driveChart.update(file, true);
|
|
perUserDriveChart.update(file, true);
|
|
if (file.userHost !== null) {
|
|
instanceChart.updateDrive(file, true);
|
|
}
|
|
return file;
|
|
}
|