Fixed 267U.pre2
This commit is contained in:
@@ -0,0 +1,35 @@
|
||||
import { deliverQueue, inboxQueue } from "./queues.js";
|
||||
import { dbInit } from "./queues/db/index.js";
|
||||
import { deliverInit, deliverLogger } from "./queues/deliver.js";
|
||||
import { endedPollNotificationInit } from "./queues/ended-poll-notification.js";
|
||||
import { inboxInit, inboxLogger } from "./queues/inbox.js";
|
||||
import { objectStorageInit } from "./queues/object-storage/index.js";
|
||||
import { systemInit } from "./queues/system/index.js";
|
||||
import { webhookDeliverInit } from "./queues/webhook-deliver.js";
|
||||
export { createDeleteDriveFilesJob, createExportCustomEmojisJob, createExportNotesJob, createExportFollowingJob, createExportMuteJob, createExportBlockingJob, createExportUserListsJob, createImportFollowingJob, createImportMutingJob, createImportBlockingJob, createImportUserListsJob, createImportCustomEmojisJob, createDeleteAccountJob } from "./queues/db/index.js";
|
||||
export { createDeleteObjectStorageFileJob, createCleanRemoteFilesJob } from "./queues/object-storage/index.js";
|
||||
export { systemQueue } from "./queues/system/index.js";
|
||||
export { deliverJob as deliver, deliverQueue } from "./queues/deliver.js";
|
||||
export { endedPollNotificationQueue } from "./queues/ended-poll-notification.js";
|
||||
export { inboxJob as inbox, inboxQueue } from "./queues/inbox.js";
|
||||
export { webhookDeliverJob as webhookDeliver, webhookDeliverQueue } from "./queues/webhook-deliver.js";
|
||||
export default async function() {
|
||||
// initialize queue workers
|
||||
await dbInit();
|
||||
await objectStorageInit();
|
||||
await systemInit();
|
||||
await deliverInit();
|
||||
await endedPollNotificationInit();
|
||||
await inboxInit();
|
||||
await webhookDeliverInit();
|
||||
}
|
||||
export function destroy() {
|
||||
deliverQueue.once("cleaned", (jobs, status)=>{
|
||||
deliverLogger.succ(`Cleaned ${jobs.length} ${status} jobs`);
|
||||
});
|
||||
deliverQueue.clean(0, Infinity, "delayed");
|
||||
inboxQueue.once("cleaned", (jobs, status)=>{
|
||||
inboxLogger.succ(`Cleaned ${jobs.length} ${status} jobs`);
|
||||
});
|
||||
inboxQueue.clean(0, Infinity, "delayed");
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
import Logger from "../services/logger.js";
|
||||
export const queueLogger = new Logger("queue", "orange");
|
||||
export function renderError(e) {
|
||||
return {
|
||||
stack: e.stack,
|
||||
message: e.message,
|
||||
name: e.name
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
import { dbQueue } from "./queues/db/index.js";
|
||||
import { deliverQueue } from "./queues/deliver.js";
|
||||
import { endedPollNotificationQueue } from "./queues/ended-poll-notification.js";
|
||||
import { inboxQueue } from "./queues/inbox.js";
|
||||
import { objectStorageQueue } from "./queues/object-storage/index.js";
|
||||
import { systemQueue } from "./queues/system/index.js";
|
||||
import { webhookDeliverQueue } from "./queues/webhook-deliver.js";
|
||||
export { dbQueue } from "./queues/db/index.js";
|
||||
export { deliverQueue } from "./queues/deliver.js";
|
||||
export { endedPollNotificationQueue } from "./queues/ended-poll-notification.js";
|
||||
export { inboxQueue } from "./queues/inbox.js";
|
||||
export { objectStorageQueue } from "./queues/object-storage/index.js";
|
||||
export { systemQueue } from "./queues/system/index.js";
|
||||
export { webhookDeliverQueue } from "./queues/webhook-deliver.js";
|
||||
export const queues = [
|
||||
dbQueue,
|
||||
deliverQueue,
|
||||
endedPollNotificationQueue,
|
||||
inboxQueue,
|
||||
objectStorageQueue,
|
||||
systemQueue,
|
||||
webhookDeliverQueue
|
||||
];
|
||||
@@ -0,0 +1,84 @@
|
||||
import { queueLogger } from "../../logger.js";
|
||||
import { DriveFiles, Notes, UserProfiles, Users } from "../../../models/index.js";
|
||||
import { MoreThan } from "typeorm";
|
||||
import { deleteFileSync } from "../../../services/drive/delete-file.js";
|
||||
import { sendEmail } from "../../../services/send-email.js";
|
||||
import { publishInternalEvent } from "../../../services/stream.js";
|
||||
const logger = queueLogger.createSubLogger("delete-account");
|
||||
export async function deleteAccount(job) {
|
||||
logger.info(`Deleting account of ${job.data.user.id} ...`);
|
||||
const user = await Users.findOneBy({
|
||||
id: job.data.user.id
|
||||
});
|
||||
if (!user) return "skip: User not found";
|
||||
const isLocal = Users.isLocalUser(user);
|
||||
{
|
||||
// Delete notes
|
||||
let cursor = null;
|
||||
while(true){
|
||||
const notes = await Notes.find({
|
||||
where: {
|
||||
userId: user.id,
|
||||
...cursor ? {
|
||||
id: MoreThan(cursor)
|
||||
} : {}
|
||||
},
|
||||
take: 10,
|
||||
order: {
|
||||
id: 1
|
||||
}
|
||||
});
|
||||
if (notes.length === 0) {
|
||||
break;
|
||||
}
|
||||
cursor = notes[notes.length - 1].id;
|
||||
await Notes.delete(notes.map((note)=>note.id));
|
||||
}
|
||||
logger.succ("All of notes deleted");
|
||||
}
|
||||
{
|
||||
// Delete files
|
||||
let cursor = null;
|
||||
while(true){
|
||||
const files = await DriveFiles.find({
|
||||
where: {
|
||||
userId: user.id,
|
||||
...cursor ? {
|
||||
id: MoreThan(cursor)
|
||||
} : {}
|
||||
},
|
||||
take: 10,
|
||||
order: {
|
||||
id: 1
|
||||
}
|
||||
});
|
||||
if (files.length === 0) {
|
||||
break;
|
||||
}
|
||||
cursor = files[files.length - 1].id;
|
||||
for (const file of files){
|
||||
await deleteFileSync(file);
|
||||
}
|
||||
}
|
||||
logger.succ("All of files deleted");
|
||||
}
|
||||
{
|
||||
// Send email notification
|
||||
const profile = await UserProfiles.findOneByOrFail({
|
||||
userId: user.id
|
||||
});
|
||||
if (profile.email && profile.emailVerified) {
|
||||
sendEmail(profile.email, "Account deleted", "Your account has been deleted.", "Your account has been deleted.");
|
||||
}
|
||||
}
|
||||
// soft指定されている場合は物理削除しない
|
||||
if (job.data.soft) {
|
||||
// nop
|
||||
} else {
|
||||
await Users.delete(job.data.user.id);
|
||||
publishInternalEvent(isLocal ? "localUserDeleted" : "remoteUserDeleted", {
|
||||
id: user.id
|
||||
});
|
||||
}
|
||||
return "Account deleted";
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
import { queueLogger } from "../../logger.js";
|
||||
import { deleteFileSync } from "../../../services/drive/delete-file.js";
|
||||
import { Users, DriveFiles } from "../../../models/index.js";
|
||||
import { MoreThan } from "typeorm";
|
||||
const logger = queueLogger.createSubLogger("delete-drive-files");
|
||||
export async function deleteDriveFiles(job) {
|
||||
logger.info(`Deleting drive files of ${job.data.user.id} ...`);
|
||||
const user = await Users.findOneBy({
|
||||
id: job.data.user.id
|
||||
});
|
||||
if (user == null) {
|
||||
return "skip: User not found";
|
||||
}
|
||||
let deletedCount = 0;
|
||||
let cursor = null;
|
||||
while(true){
|
||||
const files = await DriveFiles.find({
|
||||
where: {
|
||||
userId: user.id,
|
||||
...cursor ? {
|
||||
id: MoreThan(cursor)
|
||||
} : {}
|
||||
},
|
||||
take: 100,
|
||||
order: {
|
||||
id: 1
|
||||
}
|
||||
});
|
||||
if (files.length === 0) {
|
||||
job.updateProgress(100);
|
||||
break;
|
||||
}
|
||||
cursor = files[files.length - 1].id;
|
||||
for (const file of files){
|
||||
await deleteFileSync(file);
|
||||
deletedCount++;
|
||||
}
|
||||
const total = await DriveFiles.countBy({
|
||||
userId: user.id
|
||||
});
|
||||
job.updateProgress(deletedCount / total);
|
||||
}
|
||||
logger.succ(`All drive files (${deletedCount}) of ${user.id} has been deleted.`);
|
||||
return "Success";
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
import * as fs from "node:fs";
|
||||
import { queueLogger } from "../../logger.js";
|
||||
import { addFile } from "../../../services/drive/add-file.js";
|
||||
import { format as dateFormat } from "date-fns";
|
||||
import { getFullApAccount } from "../../../misc/convert-host.js";
|
||||
import { createTemp } from "../../../misc/create-temp.js";
|
||||
import { Users, Blockings } from "../../../models/index.js";
|
||||
import { MoreThan } from "typeorm";
|
||||
const logger = queueLogger.createSubLogger("export-blocking");
|
||||
export async function exportBlocking(job) {
|
||||
logger.info(`Exporting blocking of ${job.data.user.id} ...`);
|
||||
const user = await Users.findOneBy({
|
||||
id: job.data.user.id
|
||||
});
|
||||
if (user == null) {
|
||||
return "skip: User not found";
|
||||
}
|
||||
// Create temp file
|
||||
const [path, cleanup] = await createTemp();
|
||||
logger.info(`Temp file is ${path}`);
|
||||
try {
|
||||
const stream = fs.createWriteStream(path, {
|
||||
flags: "a"
|
||||
});
|
||||
let exportedCount = 0;
|
||||
let cursor = null;
|
||||
while(true){
|
||||
const blockings = await Blockings.find({
|
||||
where: {
|
||||
blockerId: user.id,
|
||||
...cursor ? {
|
||||
id: MoreThan(cursor)
|
||||
} : {}
|
||||
},
|
||||
take: 100,
|
||||
order: {
|
||||
id: 1
|
||||
}
|
||||
});
|
||||
if (blockings.length === 0) {
|
||||
job.updateProgress(100);
|
||||
break;
|
||||
}
|
||||
cursor = blockings[blockings.length - 1].id;
|
||||
for (const block of blockings){
|
||||
const u = await Users.findOneBy({
|
||||
id: block.blockeeId
|
||||
});
|
||||
if (u == null) {
|
||||
exportedCount++;
|
||||
continue;
|
||||
}
|
||||
const content = getFullApAccount(u.username, u.host);
|
||||
await new Promise((res, rej)=>{
|
||||
stream.write(content + "\n", (err)=>{
|
||||
if (err) {
|
||||
logger.error(err);
|
||||
rej(err);
|
||||
} else {
|
||||
res();
|
||||
}
|
||||
});
|
||||
});
|
||||
exportedCount++;
|
||||
}
|
||||
const total = await Blockings.countBy({
|
||||
blockerId: user.id
|
||||
});
|
||||
job.updateProgress(exportedCount / total);
|
||||
}
|
||||
stream.end();
|
||||
logger.succ(`Exported to: ${path}`);
|
||||
const fileName = `blocking-${dateFormat(new Date(), "yyyy-MM-dd-HH-mm-ss")}.csv`;
|
||||
const driveFile = await addFile({
|
||||
user,
|
||||
path,
|
||||
name: fileName,
|
||||
force: true
|
||||
});
|
||||
logger.succ(`Exported to: ${driveFile.id}`);
|
||||
} finally{
|
||||
cleanup();
|
||||
}
|
||||
return "Success";
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
import * as fs from "node:fs";
|
||||
import mime from "mime-types";
|
||||
import archiver from "archiver";
|
||||
import { queueLogger } from "../../logger.js";
|
||||
import { addFile } from "../../../services/drive/add-file.js";
|
||||
import { format as dateFormat } from "date-fns";
|
||||
import { Users, Emojis } from "../../../models/index.js";
|
||||
import { createTemp, createTempDir } from "../../../misc/create-temp.js";
|
||||
import { downloadUrl } from "../../../misc/download-url.js";
|
||||
import config from "../../../config/index.js";
|
||||
import { IsNull } from "typeorm";
|
||||
const logger = queueLogger.createSubLogger("export-custom-emojis");
|
||||
export async function exportCustomEmojis(job) {
|
||||
logger.info("Exporting custom emojis ...");
|
||||
const user = await Users.findOneBy({
|
||||
id: job.data.user.id
|
||||
});
|
||||
if (user == null) {
|
||||
return "skip: User not found";
|
||||
}
|
||||
const [path, cleanup] = await createTempDir();
|
||||
logger.info(`Temp dir is ${path}`);
|
||||
const metaPath = `${path}/meta.json`;
|
||||
fs.writeFileSync(metaPath, "", "utf-8");
|
||||
const metaStream = fs.createWriteStream(metaPath, {
|
||||
flags: "a"
|
||||
});
|
||||
const writeMeta = (text)=>{
|
||||
return new Promise((res, rej)=>{
|
||||
metaStream.write(text, (err)=>{
|
||||
if (err) {
|
||||
logger.error(err);
|
||||
rej(err);
|
||||
} else {
|
||||
res();
|
||||
}
|
||||
});
|
||||
});
|
||||
};
|
||||
await writeMeta(`{"metaVersion":2,"host":"${config.host}","exportedAt":"${new Date().toString()}","emojis":[`);
|
||||
const customEmojis = await Emojis.find({
|
||||
where: {
|
||||
host: IsNull()
|
||||
},
|
||||
order: {
|
||||
id: "ASC"
|
||||
}
|
||||
});
|
||||
for (const emoji of customEmojis){
|
||||
const ext = mime.extension(emoji.type);
|
||||
// there are some restrictions on file names, so to be safe the files are
|
||||
// named after their database id instead of the actual emoji name
|
||||
const fileName = emoji.id + (ext ? '.' + ext : '');
|
||||
const emojiPath = `${path}/${fileName}`;
|
||||
fs.writeFileSync(emojiPath, "", "binary");
|
||||
let downloaded = false;
|
||||
try {
|
||||
await downloadUrl(emoji.originalUrl, emojiPath);
|
||||
downloaded = true;
|
||||
} catch (e) {
|
||||
// TODO: 何度か再試行
|
||||
logger.error(e instanceof Error ? e : new Error(e));
|
||||
}
|
||||
if (!downloaded) {
|
||||
fs.unlinkSync(emojiPath);
|
||||
}
|
||||
const content = JSON.stringify({
|
||||
fileName: fileName,
|
||||
downloaded: downloaded,
|
||||
emoji: emoji
|
||||
});
|
||||
const isFirst = customEmojis.indexOf(emoji) === 0;
|
||||
await writeMeta(isFirst ? content : ",\n" + content);
|
||||
}
|
||||
await writeMeta("]}");
|
||||
metaStream.end();
|
||||
// Create archive
|
||||
await new Promise(async (resolve, reject)=>{
|
||||
try {
|
||||
const [archivePath, archiveCleanup] = await createTemp();
|
||||
const archiveStream = fs.createWriteStream(archivePath);
|
||||
const archive = archiver("zip", {
|
||||
zlib: {
|
||||
level: 0
|
||||
}
|
||||
});
|
||||
archiveStream.on("close", async ()=>{
|
||||
logger.succ(`Exported to: ${archivePath}`);
|
||||
const fileName = `custom-emojis-${dateFormat(new Date(), "yyyy-MM-dd-HH-mm-ss")}.zip`;
|
||||
const driveFile = await addFile({
|
||||
user,
|
||||
path: archivePath,
|
||||
name: fileName,
|
||||
force: true
|
||||
});
|
||||
logger.succ(`Exported to: ${driveFile.id}`);
|
||||
cleanup();
|
||||
archiveCleanup();
|
||||
resolve(undefined);
|
||||
});
|
||||
archive.pipe(archiveStream);
|
||||
archive.directory(path, false);
|
||||
archive.finalize();
|
||||
} catch (e) {
|
||||
reject(e);
|
||||
}
|
||||
});
|
||||
return "Success";
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
import * as fs from "node:fs";
|
||||
import { queueLogger } from "../../logger.js";
|
||||
import { addFile } from "../../../services/drive/add-file.js";
|
||||
import { format as dateFormat } from "date-fns";
|
||||
import { getFullApAccount } from "../../../misc/convert-host.js";
|
||||
import { createTemp } from "../../../misc/create-temp.js";
|
||||
import { Users, Followings, Mutings } from "../../../models/index.js";
|
||||
import { In, MoreThan, Not } from "typeorm";
|
||||
const logger = queueLogger.createSubLogger("export-following");
|
||||
export async function exportFollowing(job) {
|
||||
logger.info(`Exporting following of ${job.data.user.id} ...`);
|
||||
const user = await Users.findOneBy({
|
||||
id: job.data.user.id
|
||||
});
|
||||
if (user == null) {
|
||||
return "skip: User not found";
|
||||
}
|
||||
// Create temp file
|
||||
const [path, cleanup] = await createTemp();
|
||||
logger.info(`Temp file is ${path}`);
|
||||
try {
|
||||
const stream = fs.createWriteStream(path, {
|
||||
flags: "a"
|
||||
});
|
||||
let cursor = null;
|
||||
const mutings = job.data.excludeMuting ? await Mutings.findBy({
|
||||
muterId: user.id
|
||||
}) : [];
|
||||
while(true){
|
||||
const followings = await Followings.find({
|
||||
where: {
|
||||
followerId: user.id,
|
||||
...mutings.length > 0 ? {
|
||||
followeeId: Not(In(mutings.map((x)=>x.muteeId)))
|
||||
} : {},
|
||||
...cursor ? {
|
||||
id: MoreThan(cursor)
|
||||
} : {}
|
||||
},
|
||||
take: 100,
|
||||
order: {
|
||||
id: 1
|
||||
}
|
||||
});
|
||||
if (followings.length === 0) {
|
||||
break;
|
||||
}
|
||||
cursor = followings[followings.length - 1].id;
|
||||
for (const following of followings){
|
||||
const u = await Users.findOneBy({
|
||||
id: following.followeeId
|
||||
});
|
||||
if (u == null) {
|
||||
continue;
|
||||
}
|
||||
if (job.data.excludeInactive && u.updatedAt && Date.now() - u.updatedAt.getTime() > 1000 * 60 * 60 * 24 * 90) {
|
||||
continue;
|
||||
}
|
||||
const content = getFullApAccount(u.username, u.host);
|
||||
await new Promise((res, rej)=>{
|
||||
stream.write(content + "\n", (err)=>{
|
||||
if (err) {
|
||||
logger.error(err);
|
||||
rej(err);
|
||||
} else {
|
||||
res();
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
stream.end();
|
||||
logger.succ(`Exported to: ${path}`);
|
||||
const fileName = `following-${dateFormat(new Date(), "yyyy-MM-dd-HH-mm-ss")}.csv`;
|
||||
const driveFile = await addFile({
|
||||
user,
|
||||
path,
|
||||
name: fileName,
|
||||
force: true
|
||||
});
|
||||
logger.succ(`Exported to: ${driveFile.id}`);
|
||||
} finally{
|
||||
cleanup();
|
||||
}
|
||||
return "Success";
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
import * as fs from "node:fs";
|
||||
import { queueLogger } from "../../logger.js";
|
||||
import { addFile } from "../../../services/drive/add-file.js";
|
||||
import { format as dateFormat } from "date-fns";
|
||||
import { getFullApAccount } from "../../../misc/convert-host.js";
|
||||
import { createTemp } from "../../../misc/create-temp.js";
|
||||
import { Users, Mutings } from "../../../models/index.js";
|
||||
import { IsNull, MoreThan } from "typeorm";
|
||||
const logger = queueLogger.createSubLogger("export-mute");
|
||||
export async function exportMute(job) {
|
||||
logger.info(`Exporting mute of ${job.data.user.id} ...`);
|
||||
const user = await Users.findOneBy({
|
||||
id: job.data.user.id
|
||||
});
|
||||
if (user == null) {
|
||||
return "skip: User not found";
|
||||
}
|
||||
// Create temp file
|
||||
const [path, cleanup] = await createTemp();
|
||||
logger.info(`Temp file is ${path}`);
|
||||
try {
|
||||
const stream = fs.createWriteStream(path, {
|
||||
flags: "a"
|
||||
});
|
||||
let exportedCount = 0;
|
||||
let cursor = null;
|
||||
while(true){
|
||||
const mutes = await Mutings.find({
|
||||
where: {
|
||||
muterId: user.id,
|
||||
expiresAt: IsNull(),
|
||||
...cursor ? {
|
||||
id: MoreThan(cursor)
|
||||
} : {}
|
||||
},
|
||||
take: 100,
|
||||
order: {
|
||||
id: 1
|
||||
}
|
||||
});
|
||||
if (mutes.length === 0) {
|
||||
job.updateProgress(100);
|
||||
break;
|
||||
}
|
||||
cursor = mutes[mutes.length - 1].id;
|
||||
for (const mute of mutes){
|
||||
const u = await Users.findOneBy({
|
||||
id: mute.muteeId
|
||||
});
|
||||
if (u == null) {
|
||||
exportedCount++;
|
||||
continue;
|
||||
}
|
||||
const content = getFullApAccount(u.username, u.host);
|
||||
await new Promise((res, rej)=>{
|
||||
stream.write(content + "\n", (err)=>{
|
||||
if (err) {
|
||||
logger.error(err);
|
||||
rej(err);
|
||||
} else {
|
||||
res();
|
||||
}
|
||||
});
|
||||
});
|
||||
exportedCount++;
|
||||
}
|
||||
const total = await Mutings.countBy({
|
||||
muterId: user.id
|
||||
});
|
||||
job.updateProgress(exportedCount / total);
|
||||
}
|
||||
stream.end();
|
||||
logger.succ(`Exported to: ${path}`);
|
||||
const fileName = `mute-${dateFormat(new Date(), "yyyy-MM-dd-HH-mm-ss")}.csv`;
|
||||
const driveFile = await addFile({
|
||||
user,
|
||||
path,
|
||||
name: fileName,
|
||||
force: true
|
||||
});
|
||||
logger.succ(`Exported to: ${driveFile.id}`);
|
||||
} finally{
|
||||
cleanup();
|
||||
}
|
||||
return "Success";
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
import * as fs from "node:fs";
|
||||
import { queueLogger } from "../../logger.js";
|
||||
import { addFile } from "../../../services/drive/add-file.js";
|
||||
import { format as dateFormat } from "date-fns";
|
||||
import { Users, Notes, Polls, DriveFiles } from "../../../models/index.js";
|
||||
import { MoreThan } from "typeorm";
|
||||
import { createTemp } from "../../../misc/create-temp.js";
|
||||
const logger = queueLogger.createSubLogger("export-notes");
|
||||
export async function exportNotes(job) {
|
||||
logger.info(`Exporting notes of ${job.data.user.id} ...`);
|
||||
const user = await Users.findOneBy({
|
||||
id: job.data.user.id
|
||||
});
|
||||
if (user == null) {
|
||||
return "skip: User not found";
|
||||
}
|
||||
// Create temp file
|
||||
const [path, cleanup] = await createTemp();
|
||||
logger.info(`Temp file is ${path}`);
|
||||
try {
|
||||
const stream = fs.createWriteStream(path, {
|
||||
flags: "a"
|
||||
});
|
||||
const write = (text)=>{
|
||||
return new Promise((res, rej)=>{
|
||||
stream.write(text, (err)=>{
|
||||
if (err) {
|
||||
logger.error(err);
|
||||
rej(err);
|
||||
} else {
|
||||
res();
|
||||
}
|
||||
});
|
||||
});
|
||||
};
|
||||
await write("[");
|
||||
let exportedNotesCount = 0;
|
||||
let cursor = null;
|
||||
while(true){
|
||||
const notes = await Notes.find({
|
||||
where: {
|
||||
userId: user.id,
|
||||
...cursor ? {
|
||||
id: MoreThan(cursor)
|
||||
} : {}
|
||||
},
|
||||
take: 100,
|
||||
order: {
|
||||
id: 1
|
||||
}
|
||||
});
|
||||
if (notes.length === 0) {
|
||||
job.updateProgress(100);
|
||||
break;
|
||||
}
|
||||
cursor = notes[notes.length - 1].id;
|
||||
for (const note of notes){
|
||||
let poll;
|
||||
if (note.hasPoll) {
|
||||
poll = await Polls.findOneByOrFail({
|
||||
noteId: note.id
|
||||
});
|
||||
}
|
||||
const content = JSON.stringify(await serialize(note, poll));
|
||||
const isFirst = exportedNotesCount === 0;
|
||||
await write(isFirst ? content : ",\n" + content);
|
||||
exportedNotesCount++;
|
||||
}
|
||||
const total = await Notes.countBy({
|
||||
userId: user.id
|
||||
});
|
||||
job.updateProgress(exportedNotesCount / total);
|
||||
}
|
||||
await write("]");
|
||||
stream.end();
|
||||
logger.succ(`Exported to: ${path}`);
|
||||
const fileName = `notes-${dateFormat(new Date(), "yyyy-MM-dd-HH-mm-ss")}.json`;
|
||||
const driveFile = await addFile({
|
||||
user,
|
||||
path,
|
||||
name: fileName,
|
||||
force: true
|
||||
});
|
||||
logger.succ(`Exported to: ${driveFile.id}`);
|
||||
} finally{
|
||||
cleanup();
|
||||
}
|
||||
return "Success";
|
||||
}
|
||||
async function serialize(note, poll = null) {
|
||||
return {
|
||||
id: note.id,
|
||||
text: note.text,
|
||||
createdAt: note.createdAt,
|
||||
fileIds: note.fileIds,
|
||||
files: await DriveFiles.packMany(note.fileIds),
|
||||
replyId: note.replyId,
|
||||
renoteId: note.renoteId,
|
||||
poll: poll,
|
||||
cw: note.cw,
|
||||
visibility: note.visibility,
|
||||
visibleUserIds: note.visibleUserIds,
|
||||
localOnly: note.localOnly
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
import * as fs from "node:fs";
|
||||
import { queueLogger } from "../../logger.js";
|
||||
import { addFile } from "../../../services/drive/add-file.js";
|
||||
import { format as dateFormat } from "date-fns";
|
||||
import { getFullApAccount } from "../../../misc/convert-host.js";
|
||||
import { createTemp } from "../../../misc/create-temp.js";
|
||||
import { Users, UserLists, UserListJoinings } from "../../../models/index.js";
|
||||
import { In } from "typeorm";
|
||||
const logger = queueLogger.createSubLogger("export-user-lists");
|
||||
export async function exportUserLists(job) {
|
||||
logger.info(`Exporting user lists of ${job.data.user.id} ...`);
|
||||
const user = await Users.findOneBy({
|
||||
id: job.data.user.id
|
||||
});
|
||||
if (user == null) {
|
||||
return "skip: User not found";
|
||||
}
|
||||
const lists = await UserLists.findBy({
|
||||
userId: user.id
|
||||
});
|
||||
// Create temp file
|
||||
const [path, cleanup] = await createTemp();
|
||||
logger.info(`Temp file is ${path}`);
|
||||
try {
|
||||
const stream = fs.createWriteStream(path, {
|
||||
flags: "a"
|
||||
});
|
||||
for (const list of lists){
|
||||
const joinings = await UserListJoinings.findBy({
|
||||
userListId: list.id
|
||||
});
|
||||
const users = await Users.findBy({
|
||||
id: In(joinings.map((j)=>j.userId))
|
||||
});
|
||||
for (const u of users){
|
||||
const acct = getFullApAccount(u.username, u.host);
|
||||
const content = `${list.name},${acct}`;
|
||||
await new Promise((res, rej)=>{
|
||||
stream.write(content + "\n", (err)=>{
|
||||
if (err) {
|
||||
logger.error(err);
|
||||
rej(err);
|
||||
} else {
|
||||
res();
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
stream.end();
|
||||
logger.succ(`Exported to: ${path}`);
|
||||
const fileName = `user-lists-${dateFormat(new Date(), "yyyy-MM-dd-HH-mm-ss")}.csv`;
|
||||
const driveFile = await addFile({
|
||||
user,
|
||||
path,
|
||||
name: fileName,
|
||||
force: true
|
||||
});
|
||||
logger.succ(`Exported to: ${driveFile.id}`);
|
||||
} finally{
|
||||
cleanup();
|
||||
}
|
||||
return "Success";
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
import { queueLogger } from "../../logger.js";
|
||||
import * as Acct from "../../../misc/acct.js";
|
||||
import { resolveUser } from "../../../remote/resolve-user.js";
|
||||
import { downloadTextFile } from "../../../misc/download-text-file.js";
|
||||
import { isSelfHost, toPuny } from "../../../misc/convert-host.js";
|
||||
import { Users, DriveFiles } from "../../../models/index.js";
|
||||
import block from "../../../services/blocking/create.js";
|
||||
import { IsNull } from "typeorm";
|
||||
const logger = queueLogger.createSubLogger("import-blocking");
|
||||
export async function importBlocking(job) {
|
||||
logger.info(`Importing blocking of ${job.data.user.id} ...`);
|
||||
const user = await Users.findOneBy({
|
||||
id: job.data.user.id
|
||||
});
|
||||
if (user == null) {
|
||||
return "skip: User not found";
|
||||
}
|
||||
const file = await DriveFiles.findOneBy({
|
||||
id: job.data.fileId
|
||||
});
|
||||
if (file == null) {
|
||||
return "skip: File not found";
|
||||
}
|
||||
const csv = await downloadTextFile(file.url);
|
||||
let linenum = 0;
|
||||
for (const line of csv.trim().split("\n")){
|
||||
linenum++;
|
||||
try {
|
||||
const acct = line.split(",")[0].trim();
|
||||
const { username, host } = Acct.parse(acct);
|
||||
let target = isSelfHost(host) ? await Users.findOneBy({
|
||||
host: IsNull(),
|
||||
usernameLower: username.toLowerCase()
|
||||
}) : await Users.findOneBy({
|
||||
host: toPuny(host),
|
||||
usernameLower: username.toLowerCase()
|
||||
});
|
||||
if (host == null && target == null) continue;
|
||||
if (target == null) {
|
||||
target = await resolveUser(username, host);
|
||||
}
|
||||
if (target == null) {
|
||||
throw new Error(`cannot resolve user: @${username}@${host}`);
|
||||
}
|
||||
// skip myself
|
||||
if (target.id === job.data.user.id) continue;
|
||||
logger.info(`Block[${linenum}] ${target.id} ...`);
|
||||
await block(user, target);
|
||||
} catch (e) {
|
||||
logger.warn(`Error in line:${linenum} ${e}`);
|
||||
}
|
||||
}
|
||||
logger.succ("Imported");
|
||||
return "Success";
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
import * as fs from "node:fs";
|
||||
import AdmZip from "adm-zip";
|
||||
import { queueLogger } from "../../logger.js";
|
||||
import { createTempDir } from "../../../misc/create-temp.js";
|
||||
import { downloadUrl } from "../../../misc/download-url.js";
|
||||
import { DriveFiles, Emojis } from "../../../models/index.js";
|
||||
import { addFile } from "../../../services/drive/add-file.js";
|
||||
import { genId } from "../../../misc/gen-id.js";
|
||||
import { db } from "../../../db/postgre.js";
|
||||
import probeImageSize from "probe-image-size";
|
||||
import * as path from "path";
|
||||
const logger = queueLogger.createSubLogger("import-custom-emojis");
|
||||
// TODO: 名前衝突時の動作を選べるようにする
|
||||
export async function importCustomEmojis(job) {
|
||||
logger.info("Importing custom emojis ...");
|
||||
const file = await DriveFiles.findOneBy({
|
||||
id: job.data.fileId
|
||||
});
|
||||
if (file == null) {
|
||||
return "skip: File not found";
|
||||
}
|
||||
const [tempPath, cleanup] = await createTempDir();
|
||||
logger.info(`Temp dir is ${tempPath}`);
|
||||
const destPath = `${tempPath}/emojis.zip`;
|
||||
try {
|
||||
fs.writeFileSync(destPath, "", "binary");
|
||||
await downloadUrl(file.url, destPath);
|
||||
} catch (e) {
|
||||
// TODO: 何度か再試行
|
||||
if (e instanceof Error || typeof e === "string") {
|
||||
logger.error(e);
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
const outputPath = `${tempPath}/emojis`;
|
||||
const unzipStream = fs.createReadStream(destPath);
|
||||
const zip = new AdmZip(destPath);
|
||||
logger.succ(`Unzipping to ${outputPath}`);
|
||||
await new Promise((resolve, reject)=>{
|
||||
zip.extractAllToAsync(outputPath, true, false, async (error)=>{
|
||||
if (error) reject(error);
|
||||
if (fs.existsSync(`${outputPath}/meta.json`)) {
|
||||
logger.info("starting emoji import with metadata");
|
||||
const metaRaw = fs.readFileSync(`${outputPath}/meta.json`, "utf-8");
|
||||
const meta = JSON.parse(metaRaw);
|
||||
for (const record of meta.emojis){
|
||||
if (!record.downloaded) continue;
|
||||
const emojiInfo = record.emoji;
|
||||
const emojiPath = `${outputPath}/${record.fileName}`;
|
||||
await Emojis.delete({
|
||||
name: emojiInfo.name
|
||||
});
|
||||
const driveFile = await addFile({
|
||||
user: null,
|
||||
path: emojiPath,
|
||||
name: record.fileName,
|
||||
force: true
|
||||
});
|
||||
const file = fs.createReadStream(emojiPath);
|
||||
const size = await probeImageSize(file);
|
||||
file.destroy();
|
||||
await Emojis.insert({
|
||||
id: genId(),
|
||||
updatedAt: new Date(),
|
||||
name: emojiInfo.name,
|
||||
category: emojiInfo.category,
|
||||
host: null,
|
||||
aliases: emojiInfo.aliases,
|
||||
originalUrl: driveFile.url,
|
||||
publicUrl: driveFile.webpublicUrl ?? driveFile.url,
|
||||
type: driveFile.webpublicType ?? driveFile.type,
|
||||
license: emojiInfo.license,
|
||||
glyph: driveFile.type === "image/svg+xml",
|
||||
width: size.width || null,
|
||||
height: size.height || null
|
||||
}).then((x)=>Emojis.findOneByOrFail(x.identifiers[0]));
|
||||
}
|
||||
} else {
|
||||
logger.info("starting emoji import without metadata");
|
||||
// Since we lack metadata, we import into a randomized category name instead
|
||||
let categoryName = genId();
|
||||
let containedEmojis = fs.readdirSync(outputPath);
|
||||
// Filter out accidental JSON files
|
||||
containedEmojis = containedEmojis.filter((emoji)=>!emoji.match(/\.(json)$/i));
|
||||
for (const emojiFilename of containedEmojis){
|
||||
// strip extension and get filename to use as name
|
||||
const name = path.basename(emojiFilename, path.extname(emojiFilename));
|
||||
const emojiPath = `${outputPath}/${emojiFilename}`;
|
||||
logger.info(`importing ${name}`);
|
||||
await Emojis.delete({
|
||||
name: name
|
||||
});
|
||||
const driveFile = await addFile({
|
||||
user: null,
|
||||
path: emojiPath,
|
||||
name: path.basename(emojiFilename),
|
||||
force: true
|
||||
});
|
||||
const file = fs.createReadStream(emojiPath);
|
||||
const size = await probeImageSize(file);
|
||||
file.destroy();
|
||||
logger.info(`emoji size: ${size.width}x${size.height}`);
|
||||
await Emojis.insert({
|
||||
id: genId(),
|
||||
updatedAt: new Date(),
|
||||
name: name,
|
||||
category: categoryName,
|
||||
host: null,
|
||||
aliases: [],
|
||||
originalUrl: driveFile.url,
|
||||
publicUrl: driveFile.webpublicUrl ?? driveFile.url,
|
||||
type: driveFile.webpublicType ?? driveFile.type,
|
||||
license: null,
|
||||
glyph: driveFile.type === "image/svg+xml",
|
||||
width: size.width || null,
|
||||
height: size.height || null
|
||||
}).then((x)=>Emojis.findOneByOrFail(x.identifiers[0]));
|
||||
}
|
||||
}
|
||||
await db.queryResultCache.remove([
|
||||
"meta_emojis"
|
||||
]);
|
||||
cleanup();
|
||||
logger.succ("Imported");
|
||||
resolve(undefined);
|
||||
});
|
||||
});
|
||||
return "Success";
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
import { IsNull } from "typeorm";
|
||||
import follow from "../../../services/following/create.js";
|
||||
import * as Acct from "../../../misc/acct.js";
|
||||
import { resolveUser } from "../../../remote/resolve-user.js";
|
||||
import { downloadTextFile } from "../../../misc/download-text-file.js";
|
||||
import { isSelfHost, toPuny } from "../../../misc/convert-host.js";
|
||||
import { Users, DriveFiles } from "../../../models/index.js";
|
||||
import { queueLogger } from "../../logger.js";
|
||||
import { cache as heuristic } from "../../../server/api/common/generate-following-query.js";
|
||||
const logger = queueLogger.createSubLogger("import-following");
|
||||
export async function importFollowing(job) {
|
||||
logger.info(`Importing following of ${job.data.user.id} ...`);
|
||||
const user = await Users.findOneBy({
|
||||
id: job.data.user.id
|
||||
});
|
||||
if (user == null) {
|
||||
return "skip: User not found";
|
||||
}
|
||||
const file = await DriveFiles.findOneBy({
|
||||
id: job.data.fileId
|
||||
});
|
||||
if (file == null) {
|
||||
return "skip: File not found";
|
||||
}
|
||||
const csv = await downloadTextFile(file.url);
|
||||
let linenum = 0;
|
||||
if (file.type.endsWith("json")) {
|
||||
for (const acct of JSON.parse(csv)){
|
||||
try {
|
||||
const { username, host } = Acct.parse(acct);
|
||||
let target = isSelfHost(host) ? await Users.findOneBy({
|
||||
host: IsNull(),
|
||||
usernameLower: username.toLowerCase()
|
||||
}) : await Users.findOneBy({
|
||||
host: toPuny(host),
|
||||
usernameLower: username.toLowerCase()
|
||||
});
|
||||
if (host == null && target == null) continue;
|
||||
if (target == null) {
|
||||
target = await resolveUser(username, host);
|
||||
}
|
||||
if (target == null) {
|
||||
throw new Error(`cannot resolve user: @${username}@${host}`);
|
||||
}
|
||||
// skip myself
|
||||
if (target.id === job.data.user.id) continue;
|
||||
logger.info(`Follow[${linenum}] ${target.id} ...`);
|
||||
follow(user, target);
|
||||
} catch (e) {
|
||||
logger.warn(`Error in line:${linenum} ${e}`);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for (const line of csv.trim().split("\n")){
|
||||
linenum++;
|
||||
try {
|
||||
const acct = line.split(",")[0].trim();
|
||||
const { username, host } = Acct.parse(acct);
|
||||
let target = isSelfHost(host) ? await Users.findOneBy({
|
||||
host: IsNull(),
|
||||
usernameLower: username.toLowerCase()
|
||||
}) : await Users.findOneBy({
|
||||
host: toPuny(host),
|
||||
usernameLower: username.toLowerCase()
|
||||
});
|
||||
if (host == null && target == null) continue;
|
||||
if (target == null) {
|
||||
target = await resolveUser(username, host);
|
||||
}
|
||||
if (target == null) {
|
||||
throw new Error(`cannot resolve user: @${username}@${host}`);
|
||||
}
|
||||
// skip myself
|
||||
if (target.id === job.data.user.id) continue;
|
||||
logger.info(`Follow[${linenum}] ${target.id} ...`);
|
||||
follow(user, target);
|
||||
} catch (e) {
|
||||
logger.warn(`Error in line:${linenum} ${e}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
await heuristic.delete(user.id);
|
||||
logger.succ("Imported");
|
||||
return "Success";
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
import { queueLogger } from "../../logger.js";
|
||||
import * as Acct from "../../../misc/acct.js";
|
||||
import { resolveUser } from "../../../remote/resolve-user.js";
|
||||
import { downloadTextFile } from "../../../misc/download-text-file.js";
|
||||
import { isSelfHost, toPuny } from "../../../misc/convert-host.js";
|
||||
import { Users, DriveFiles, Mutings } from "../../../models/index.js";
|
||||
import { genId } from "../../../misc/gen-id.js";
|
||||
import { IsNull } from "typeorm";
|
||||
const logger = queueLogger.createSubLogger("import-muting");
|
||||
export async function importMuting(job) {
|
||||
logger.info(`Importing muting of ${job.data.user.id} ...`);
|
||||
const user = await Users.findOneBy({
|
||||
id: job.data.user.id
|
||||
});
|
||||
if (user == null) {
|
||||
return "skip: User not found";
|
||||
}
|
||||
const file = await DriveFiles.findOneBy({
|
||||
id: job.data.fileId
|
||||
});
|
||||
if (file == null) {
|
||||
return "skip: File not found";
|
||||
}
|
||||
const csv = await downloadTextFile(file.url);
|
||||
let linenum = 0;
|
||||
for (const line of csv.trim().split("\n")){
|
||||
linenum++;
|
||||
try {
|
||||
const acct = line.split(",")[0].trim();
|
||||
const { username, host } = Acct.parse(acct);
|
||||
let target = isSelfHost(host) ? await Users.findOneBy({
|
||||
host: IsNull(),
|
||||
usernameLower: username.toLowerCase()
|
||||
}) : await Users.findOneBy({
|
||||
host: toPuny(host),
|
||||
usernameLower: username.toLowerCase()
|
||||
});
|
||||
if (host == null && target == null) continue;
|
||||
if (target == null) {
|
||||
target = await resolveUser(username, host);
|
||||
}
|
||||
if (target == null) {
|
||||
throw new Error(`cannot resolve user: @${username}@${host}`);
|
||||
}
|
||||
// skip myself
|
||||
if (target.id === job.data.user.id) continue;
|
||||
logger.info(`Mute[${linenum}] ${target.id} ...`);
|
||||
await mute(user, target);
|
||||
} catch (e) {
|
||||
logger.warn(`Error in line:${linenum} ${e}`);
|
||||
}
|
||||
}
|
||||
logger.succ("Imported");
|
||||
return "Success";
|
||||
}
|
||||
async function mute(user, target) {
|
||||
await Mutings.insert({
|
||||
id: genId(),
|
||||
createdAt: new Date(),
|
||||
muterId: user.id,
|
||||
muteeId: target.id
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
import { queueLogger } from "../../logger.js";
|
||||
import * as Acct from "../../../misc/acct.js";
|
||||
import { resolveUser } from "../../../remote/resolve-user.js";
|
||||
import { pushUserToUserList } from "../../../services/user-list/push.js";
|
||||
import { downloadTextFile } from "../../../misc/download-text-file.js";
|
||||
import { isSelfHost, toPuny } from "../../../misc/convert-host.js";
|
||||
import { DriveFiles, Users, UserLists, UserListJoinings, Blockings, Followings } from "../../../models/index.js";
|
||||
import { genId } from "../../../misc/gen-id.js";
|
||||
import { IsNull } from "typeorm";
|
||||
const logger = queueLogger.createSubLogger("import-user-lists");
|
||||
export async function importUserLists(job) {
|
||||
logger.info(`Importing user lists of ${job.data.user.id} ...`);
|
||||
const user = await Users.findOneBy({
|
||||
id: job.data.user.id
|
||||
});
|
||||
if (user == null) {
|
||||
return "skip: User not found";
|
||||
}
|
||||
const file = await DriveFiles.findOneBy({
|
||||
id: job.data.fileId
|
||||
});
|
||||
if (file == null) {
|
||||
return "skip: File not found";
|
||||
}
|
||||
const csv = await downloadTextFile(file.url);
|
||||
let linenum = 0;
|
||||
for (const line of csv.trim().split("\n")){
|
||||
linenum++;
|
||||
try {
|
||||
const listName = line.split(",")[0].trim();
|
||||
const { username, host } = Acct.parse(line.split(",")[1].trim());
|
||||
let list = await UserLists.findOneBy({
|
||||
userId: user.id,
|
||||
name: listName
|
||||
});
|
||||
if (list == null) {
|
||||
list = await UserLists.insert({
|
||||
id: genId(),
|
||||
createdAt: new Date(),
|
||||
userId: user.id,
|
||||
name: listName
|
||||
}).then((x)=>UserLists.findOneByOrFail(x.identifiers[0]));
|
||||
}
|
||||
let target = isSelfHost(host) ? await Users.findOneBy({
|
||||
host: IsNull(),
|
||||
usernameLower: username.toLowerCase()
|
||||
}) : await Users.findOneBy({
|
||||
host: toPuny(host),
|
||||
usernameLower: username.toLowerCase()
|
||||
});
|
||||
if (target == null) {
|
||||
target = await resolveUser(username, host);
|
||||
}
|
||||
const isBlocked = await Blockings.exist({
|
||||
where: {
|
||||
blockerId: target.id,
|
||||
blockeeId: user.id
|
||||
}
|
||||
});
|
||||
const isFollowed = await Followings.exist({
|
||||
where: {
|
||||
followerId: user.id,
|
||||
followeeId: target.id
|
||||
}
|
||||
});
|
||||
if (isBlocked || !isFollowed) continue;
|
||||
if (await UserListJoinings.findOneBy({
|
||||
userListId: list.id,
|
||||
userId: target.id
|
||||
}) != null) continue;
|
||||
pushUserToUserList(target, list);
|
||||
} catch (e) {
|
||||
logger.warn(`Error in line:${linenum} ${e}`);
|
||||
}
|
||||
}
|
||||
logger.succ("Imported");
|
||||
return "Success";
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
import { deleteAccount } from "./delete-account.js";
|
||||
import { deleteDriveFiles } from "./delete-drive-files.js";
|
||||
import { exportBlocking } from "./export-blocking.js";
|
||||
import { exportCustomEmojis } from "./export-custom-emojis.js";
|
||||
import { exportFollowing } from "./export-following.js";
|
||||
import { exportMute } from "./export-mute.js";
|
||||
import { exportNotes } from "./export-notes.js";
|
||||
import { exportUserLists } from "./export-user-lists.js";
|
||||
import { importBlocking } from "./import-blocking.js";
|
||||
import { importCustomEmojis } from "./import-custom-emojis.js";
|
||||
import { importFollowing } from "./import-following.js";
|
||||
import { importMuting } from "./import-muting.js";
|
||||
import { importUserLists } from "./import-user-lists.js";
|
||||
import { createQueue, defaultJobOpts } from "../index.js";
|
||||
import config from "../../../config/index.js";
|
||||
const processors = {
|
||||
deleteAccount,
|
||||
deleteDriveFiles,
|
||||
exportBlocking,
|
||||
exportCustomEmojis,
|
||||
exportFollowing,
|
||||
exportMute,
|
||||
exportNotes,
|
||||
exportUserLists,
|
||||
importBlocking,
|
||||
importCustomEmojis,
|
||||
importFollowing,
|
||||
importMuting,
|
||||
importUserLists
|
||||
};
|
||||
async function process(job) {
|
||||
const processor = processors[job.name];
|
||||
if (processor === undefined) return "skip: unknown job name";
|
||||
return await processor(job);
|
||||
}
|
||||
export const [dbQueue, dbInit] = createQueue("db", process, {
|
||||
limitPerSec: 256,
|
||||
concurrency: 16
|
||||
});
|
||||
export function createDeleteDriveFilesJob(user) {
|
||||
return dbQueue.add("deleteDriveFiles", {
|
||||
user: user
|
||||
}, defaultJobOpts);
|
||||
}
|
||||
export function createExportCustomEmojisJob(user) {
|
||||
return dbQueue.add("exportCustomEmojis", {
|
||||
user: user
|
||||
}, defaultJobOpts);
|
||||
}
|
||||
export function createExportNotesJob(user) {
|
||||
return dbQueue.add("exportNotes", {
|
||||
user: user
|
||||
}, defaultJobOpts);
|
||||
}
|
||||
export function createExportFollowingJob(user, excludeMuting = false, excludeInactive = false) {
|
||||
return dbQueue.add("exportFollowing", {
|
||||
user: user,
|
||||
excludeMuting,
|
||||
excludeInactive
|
||||
}, defaultJobOpts);
|
||||
}
|
||||
export function createExportMuteJob(user) {
|
||||
return dbQueue.add("exportMute", {
|
||||
user: user
|
||||
}, defaultJobOpts);
|
||||
}
|
||||
export function createExportBlockingJob(user) {
|
||||
return dbQueue.add("exportBlocking", {
|
||||
user: user
|
||||
}, defaultJobOpts);
|
||||
}
|
||||
export function createExportUserListsJob(user) {
|
||||
return dbQueue.add("exportUserLists", {
|
||||
user: user
|
||||
}, defaultJobOpts);
|
||||
}
|
||||
export function createImportFollowingJob(user, fileId) {
|
||||
return dbQueue.add("importFollowing", {
|
||||
user: user,
|
||||
fileId: fileId
|
||||
}, defaultJobOpts);
|
||||
}
|
||||
export function createImportMastoPostJob(user, post, signatureCheck) {
|
||||
return dbQueue.add("importMastoPost", {
|
||||
user: user,
|
||||
post: post,
|
||||
signatureCheck: signatureCheck
|
||||
}, {
|
||||
attempts: config.inboxJobMaxAttempts || 8,
|
||||
...defaultJobOpts
|
||||
});
|
||||
}
|
||||
export function createImportCkPostJob(user, post, signatureCheck) {
|
||||
return dbQueue.add("importCkPost", {
|
||||
user: user,
|
||||
post: post,
|
||||
signatureCheck: signatureCheck
|
||||
}, defaultJobOpts);
|
||||
}
|
||||
export function createImportMutingJob(user, fileId) {
|
||||
return dbQueue.add("importMuting", {
|
||||
user: user,
|
||||
fileId: fileId
|
||||
}, defaultJobOpts);
|
||||
}
|
||||
export function createImportBlockingJob(user, fileId) {
|
||||
return dbQueue.add("importBlocking", {
|
||||
user: user,
|
||||
fileId: fileId
|
||||
}, defaultJobOpts);
|
||||
}
|
||||
export function createImportUserListsJob(user, fileId) {
|
||||
return dbQueue.add("importUserLists", {
|
||||
user: user,
|
||||
fileId: fileId
|
||||
}, defaultJobOpts);
|
||||
}
|
||||
export function createImportCustomEmojisJob(user, fileId) {
|
||||
return dbQueue.add("importCustomEmojis", {
|
||||
user: user,
|
||||
fileId: fileId
|
||||
}, defaultJobOpts);
|
||||
}
|
||||
export function createDeleteAccountJob(user, opts = {}) {
|
||||
return dbQueue.add("deleteAccount", {
|
||||
user: user,
|
||||
soft: opts.soft
|
||||
}, defaultJobOpts);
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
import { URL } from "node:url";
|
||||
import request from "../../remote/activitypub/request.js";
|
||||
import { registerOrFetchInstanceDoc } from "../../services/register-or-fetch-instance-doc.js";
|
||||
import Logger from "../../services/logger.js";
|
||||
import { Instances } from "../../models/index.js";
|
||||
import { apRequestChart, federationChart, instanceChart } from "../../services/chart/index.js";
|
||||
import { fetchInstanceMetadata } from "../../services/fetch-instance-metadata.js";
|
||||
import { toPuny } from "../../misc/convert-host.js";
|
||||
import { StatusError } from "../../misc/fetch.js";
|
||||
import { shouldSkipInstance } from "../../misc/skipped-instances.js";
|
||||
import config from "../../config/index.js";
|
||||
import { createQueue, defaultJobOpts, processorTimeout } from "./index.js";
|
||||
import { tickOutbox } from "../../metrics.js";
|
||||
export const deliverLogger = new Logger("deliver");
|
||||
let latest = null;
|
||||
async function process(job) {
|
||||
if (job.data == null || Object.keys(job.data).length === 0) {
|
||||
job.opts.removeOnComplete = true;
|
||||
return "Skip (data was null or empty)";
|
||||
}
|
||||
const { host } = new URL(job.data.to);
|
||||
const puny = toPuny(host);
|
||||
if (await shouldSkipInstance(puny)) return "skip";
|
||||
try {
|
||||
if (latest !== (latest = JSON.stringify(job.data.content, null, 2))) {
|
||||
deliverLogger.debug(`delivering ${latest}`);
|
||||
}
|
||||
await request(job.data.user, job.data.to, job.data.content);
|
||||
// Update stats
|
||||
registerOrFetchInstanceDoc(host).then((i)=>{
|
||||
Instances.update(i.id, {
|
||||
latestRequestSentAt: new Date(),
|
||||
latestStatus: 200,
|
||||
lastCommunicatedAt: new Date(),
|
||||
isNotResponding: false
|
||||
});
|
||||
fetchInstanceMetadata(i);
|
||||
instanceChart.requestSent(i.host, true);
|
||||
apRequestChart.deliverSucc();
|
||||
federationChart.deliverd(i.host, true);
|
||||
});
|
||||
tickOutbox();
|
||||
return "Success";
|
||||
} catch (res) {
|
||||
// Update stats
|
||||
registerOrFetchInstanceDoc(host).then((i)=>{
|
||||
Instances.update(i.id, {
|
||||
latestRequestSentAt: new Date(),
|
||||
latestStatus: res instanceof StatusError ? res.statusCode : null,
|
||||
isNotResponding: true
|
||||
});
|
||||
instanceChart.requestSent(i.host, false);
|
||||
apRequestChart.deliverFail();
|
||||
federationChart.deliverd(i.host, false);
|
||||
});
|
||||
if (res instanceof StatusError) {
|
||||
// 4xx
|
||||
if (!res.isRetryable) {
|
||||
// HTTPステータスコード4xxはクライアントエラーであり、それはつまり
|
||||
// 何回再送しても成功することはないということなのでエラーにはしないでおく
|
||||
return `${res.statusCode} ${res.statusMessage}`;
|
||||
}
|
||||
// 5xx etc.
|
||||
throw new Error(`${res.statusCode} ${res.statusMessage}`);
|
||||
} else {
|
||||
// DNS error, socket error, timeout ...
|
||||
throw res;
|
||||
}
|
||||
}
|
||||
}
|
||||
export const [deliverQueue, deliverInit] = createQueue("deliver", processorTimeout(process, 60), {
|
||||
limitPerSec: config.deliverJobPerSec || 128,
|
||||
concurrency: config.deliverJobConcurrency || 128
|
||||
});
|
||||
export function deliverJob(user, content, to) {
|
||||
if (content == null) return null;
|
||||
if (to == null) return null;
|
||||
const data = {
|
||||
user: {
|
||||
id: user.id
|
||||
},
|
||||
content,
|
||||
to
|
||||
};
|
||||
return deliverQueue.add("default", data, {
|
||||
attempts: config.deliverJobMaxAttempts || 12,
|
||||
backoff: {
|
||||
type: "custom"
|
||||
},
|
||||
...defaultJobOpts
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
import { Notes, PollVotes } from "../../models/index.js";
|
||||
import { createNotification } from "../../services/create-notification.js";
|
||||
import { deliverQuestionUpdate } from "../../services/note/polls/update.js";
|
||||
import { createQueue } from "./index.js";
|
||||
async function process(job) {
|
||||
if (job.data == null || Object.keys(job.data).length === 0) {
|
||||
job.opts.removeOnComplete = true;
|
||||
return "skip: corrupt job";
|
||||
}
|
||||
const note = await Notes.findOneBy({
|
||||
id: job.data.noteId
|
||||
});
|
||||
if (note == null || !note.hasPoll) {
|
||||
return "skip: note not found";
|
||||
}
|
||||
const votes = await PollVotes.createQueryBuilder("vote").select("vote.userId").where("vote.noteId = :noteId", {
|
||||
noteId: note.id
|
||||
}).innerJoinAndSelect("vote.user", "user").andWhere("user.host IS NULL").getMany();
|
||||
const userIds = [
|
||||
...new Set([
|
||||
note.userId,
|
||||
...votes.map((v)=>v.userId)
|
||||
])
|
||||
];
|
||||
for (const userId of userIds){
|
||||
createNotification(userId, "pollEnded", {
|
||||
note: note,
|
||||
noteId: note.id
|
||||
});
|
||||
}
|
||||
// Broadcast the poll result once it ends
|
||||
if (!note.localOnly) await deliverQuestionUpdate(note.id);
|
||||
return "complete";
|
||||
}
|
||||
export const [endedPollNotificationQueue, endedPollNotificationInit] = createQueue("endedPollNotification", process);
|
||||
@@ -0,0 +1,194 @@
|
||||
import { URL } from "node:url";
|
||||
import httpSignature from "@peertube/http-signature";
|
||||
import perform from "../../remote/activitypub/perform.js";
|
||||
import Logger from "../../services/logger.js";
|
||||
import { registerOrFetchInstanceDoc } from "../../services/register-or-fetch-instance-doc.js";
|
||||
import { Instances } from "../../models/index.js";
|
||||
import { apRequestChart, federationChart, instanceChart } from "../../services/chart/index.js";
|
||||
import { fetchMeta } from "../../misc/fetch-meta.js";
|
||||
import { toPuny, extractDbHost } from "../../misc/convert-host.js";
|
||||
import { getApId } from "../../remote/activitypub/type.js";
|
||||
import { fetchInstanceMetadata } from "../../services/fetch-instance-metadata.js";
|
||||
import DbResolver from "../../remote/activitypub/db-resolver.js";
|
||||
import { resolvePerson } from "../../remote/activitypub/models/person.js";
|
||||
import { LdSignature } from "../../remote/activitypub/misc/ld-signature.js";
|
||||
import { StatusError } from "../../misc/fetch.js";
|
||||
import { shouldBlockInstance } from "../../misc/should-block-instance.js";
|
||||
import { verifySignature } from "../../remote/activitypub/check-fetch.js";
|
||||
import { createQueue, defaultJobOpts, processorTimeout } from "./index.js";
|
||||
import config from "../../config/index.js";
|
||||
import { tickInbox } from "../../metrics.js";
|
||||
export const inboxLogger = new Logger("inbox");
|
||||
// Processing when an activity arrives in the user's inbox
|
||||
async function process(job) {
|
||||
if (job.data == null || Object.keys(job.data).length === 0) {
|
||||
job.opts.removeOnComplete = true;
|
||||
return "Skip (data was null or empty)";
|
||||
}
|
||||
const signature = job.data.signature; // HTTP-signature
|
||||
let activity = job.data.activity;
|
||||
//#region Log
|
||||
const info = Object.assign({}, activity);
|
||||
info["@context"] = undefined;
|
||||
inboxLogger.debug(JSON.stringify(info, null, 2));
|
||||
if (!signature?.keyId) {
|
||||
throw new Error(`Invalid signature: ${signature}`);
|
||||
}
|
||||
//#endregion
|
||||
const host = toPuny(new URL(signature.keyId).hostname);
|
||||
// interrupt if blocked
|
||||
const meta = await fetchMeta();
|
||||
if (await shouldBlockInstance(host, meta)) {
|
||||
return `Blocked request: ${host}`;
|
||||
}
|
||||
// only allowlisted instances in private mode
|
||||
if (meta.privateMode && !meta.allowedHosts.includes(host)) {
|
||||
return `Blocked request: ${host}`;
|
||||
}
|
||||
const keyIdLower = signature.keyId.toLowerCase();
|
||||
if (keyIdLower.startsWith("acct:")) {
|
||||
return `Old keyId is no longer supported. ${keyIdLower}`;
|
||||
}
|
||||
const dbResolver = new DbResolver();
|
||||
// HTTP-Signature keyId from DB
|
||||
let authUser = await dbResolver.getAuthUserFromKeyId(signature.keyId);
|
||||
// keyIdでわからなければ、activity.actorを元にDBから取得 || activity.actorを元にリモートから取得
|
||||
if (authUser == null) {
|
||||
try {
|
||||
authUser = await dbResolver.getAuthUserFromApId(getApId(activity.actor));
|
||||
} catch (e) {
|
||||
// Skip if target is 4xx
|
||||
if (e instanceof StatusError) {
|
||||
if (!e.isRetryable) {
|
||||
return `skip: Ignored deleted actors on both ends ${activity.actor} - ${e.statusCode}`;
|
||||
}
|
||||
throw new Error(`Error in actor ${activity.actor} - ${e.statusCode || e}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
// それでもわからなければ終了
|
||||
if (authUser == null) {
|
||||
return "skip: failed to resolve user";
|
||||
}
|
||||
// publicKey がなくても終了
|
||||
if (authUser.key == null) {
|
||||
return "skip: failed to resolve user publicKey";
|
||||
}
|
||||
// HTTP-Signatureの検証
|
||||
let httpSignatureValidated = httpSignature.verifySignature(signature, authUser.key.keyPem);
|
||||
// If signature validation failed, try refetching the actor
|
||||
if (!httpSignatureValidated) {
|
||||
authUser.key = await dbResolver.refetchPublicKeyForApId(authUser.user);
|
||||
if (authUser.key == null) {
|
||||
return "skip: failed to re-resolve user publicKey";
|
||||
}
|
||||
httpSignatureValidated = httpSignature.verifySignature(signature, authUser.key.keyPem);
|
||||
}
|
||||
if (httpSignatureValidated) {
|
||||
if (!verifySignature(signature, authUser.key)) return `skip: Invalid HTTP signature`;
|
||||
}
|
||||
// また、signatureのsignerは、activity.actorと一致する必要がある
|
||||
if (!httpSignatureValidated || authUser.user.uri !== activity.actor) {
|
||||
// 一致しなくても、でもLD-Signatureがありそうならそっちも見る
|
||||
if (activity.signature) {
|
||||
if (activity.signature.type !== "RsaSignature2017") {
|
||||
return `skip: unsupported LD-signature type ${activity.signature.type}`;
|
||||
}
|
||||
// activity.signature.creator: https://example.oom/users/user#main-key
|
||||
// みたいになっててUserを引っ張れば公開キーも入ることを期待する
|
||||
if (activity.signature.creator) {
|
||||
const candicate = activity.signature.creator.replace(/#.*/, "");
|
||||
await resolvePerson(candicate).catch(()=>null);
|
||||
}
|
||||
// keyIdからLD-Signatureのユーザーを取得
|
||||
authUser = await dbResolver.getAuthUserFromKeyId(activity.signature.creator);
|
||||
if (authUser == null) {
|
||||
return "skip: LD-Signatureのユーザーが取得できませんでした";
|
||||
}
|
||||
if (authUser.key == null) {
|
||||
return "skip: LD-SignatureのユーザーはpublicKeyを持っていませんでした";
|
||||
}
|
||||
// LD-Signature検証
|
||||
const ldSignature = new LdSignature();
|
||||
const signature = activity.signature;
|
||||
delete activity["signature"];
|
||||
activity = await ldSignature.compactToWellKnown(activity);
|
||||
if (ldSignature.containsForbiddenDirectives(activity)) {
|
||||
return "skip: activity contains forbidden directives";
|
||||
}
|
||||
const verified = await ldSignature.verifyRsaSignature2017(activity, signature, authUser.key.keyPem);
|
||||
if (!verified) {
|
||||
return "skip: LD-Signatureの検証に失敗しました";
|
||||
}
|
||||
// もう一度actorチェック
|
||||
if (authUser.user.uri !== activity.actor) {
|
||||
return `skip: LD-Signature user(${authUser.user.uri}) !== activity.actor(${activity.actor})`;
|
||||
}
|
||||
// ブロックしてたら中断
|
||||
const ldHost = extractDbHost(authUser.user.uri);
|
||||
if (await shouldBlockInstance(ldHost, meta)) {
|
||||
return `Blocked request: ${ldHost}`;
|
||||
}
|
||||
} else {
|
||||
return `skip: http-signature verification failed and no LD-Signature. keyId=${signature.keyId}`;
|
||||
}
|
||||
}
|
||||
// activity.idがあればホストが署名者のホストであることを確認する
|
||||
if (typeof activity.id !== "string") {
|
||||
return 'skip: activity.id is not a string';
|
||||
}
|
||||
const signerHost = extractDbHost(authUser.user.uri);
|
||||
const activityIdHost = extractDbHost(activity.id);
|
||||
if (signerHost !== activityIdHost) {
|
||||
return `skip: signerHost(${signerHost}) !== activity.id host(${activityIdHost}`;
|
||||
}
|
||||
// Update stats
|
||||
registerOrFetchInstanceDoc(authUser.user.host).then((i)=>{
|
||||
Instances.update(i.id, {
|
||||
latestRequestReceivedAt: new Date(),
|
||||
lastCommunicatedAt: new Date(),
|
||||
isNotResponding: false
|
||||
});
|
||||
fetchInstanceMetadata(i);
|
||||
instanceChart.requestReceived(i.host);
|
||||
apRequestChart.inbox();
|
||||
federationChart.inbox(i.host);
|
||||
});
|
||||
const inbox = authUser.user.sharedInbox ?? authUser.user.inbox;
|
||||
if (inbox !== null) {
|
||||
const { host: inboxHost } = new URL(inbox);
|
||||
if (inboxHost !== authUser.user.host) {
|
||||
registerOrFetchInstanceDoc(inboxHost).then((i)=>{
|
||||
Instances.update(i.id, {
|
||||
latestRequestReceivedAt: new Date(),
|
||||
lastCommunicatedAt: new Date(),
|
||||
isNotResponding: false
|
||||
});
|
||||
fetchInstanceMetadata(i);
|
||||
instanceChart.requestReceived(i.host);
|
||||
apRequestChart.inbox();
|
||||
federationChart.inbox(i.host);
|
||||
});
|
||||
}
|
||||
}
|
||||
tickInbox();
|
||||
// アクティビティを処理
|
||||
return await perform(authUser.user, activity);
|
||||
}
|
||||
export const [inboxQueue, inboxInit] = createQueue("inbox", processorTimeout(process, 5 * 60), {
|
||||
limitPerSec: config.inboxJobPerSec || 16,
|
||||
concurrency: config.inboxJobConcurrency || 16
|
||||
});
|
||||
export function inboxJob(activity, signature) {
|
||||
const data = {
|
||||
activity: activity,
|
||||
signature
|
||||
};
|
||||
return inboxQueue.add("default", data, {
|
||||
attempts: config.inboxJobMaxAttempts || 8,
|
||||
backoff: {
|
||||
type: "custom"
|
||||
},
|
||||
...defaultJobOpts
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
import { Worker, Queue } from "bullmq";
|
||||
import config from "../../config/index.js";
|
||||
import { queueLogger, renderError } from "../logger.js";
|
||||
const connectionDetails = {
|
||||
port: config.redis.port,
|
||||
host: config.redis.host,
|
||||
family: config.redis.family == null ? 0 : config.redis.family,
|
||||
username: config.redis.user ?? "default",
|
||||
password: config.redis.pass,
|
||||
db: config.redis.db || 0,
|
||||
tls: config.redis.tls,
|
||||
lazyConnect: true,
|
||||
maxRetriesPerRequest: null
|
||||
};
|
||||
// ref. https://github.com/misskey-dev/misskey/pull/7635#issue-971097019
|
||||
function apBackoff(attemptsMade) {
|
||||
const baseDelay = 60 * 1000; // 1min
|
||||
const maxBackoff = 8 * 60 * 60 * 1000; // 8hours
|
||||
let backoff = (Math.pow(2, attemptsMade) - 1) * baseDelay;
|
||||
backoff = Math.min(backoff, maxBackoff);
|
||||
backoff += Math.round(backoff * Math.random() * 0.2);
|
||||
return backoff;
|
||||
}
|
||||
export function createQueue(name, processor, opts = {}) {
|
||||
const backoffStrategy = opts.backoff || apBackoff;
|
||||
const sublogger = queueLogger.createSubLogger(name);
|
||||
const queue = new Queue(name, {
|
||||
connection: connectionDetails,
|
||||
prefix: config.redis.prefix
|
||||
});
|
||||
queue.on("waiting", (job)=>sublogger.debug(`waiting id=${job.id}`));
|
||||
const initWorker = ()=>{
|
||||
return new Worker(name, processor, {
|
||||
connection: connectionDetails,
|
||||
prefix: config.redis.prefix,
|
||||
settings: {
|
||||
backoffStrategy
|
||||
},
|
||||
limiter: opts.limitPerSec === undefined || opts.limitPerSec === -1 ? undefined : {
|
||||
max: opts.limitPerSec,
|
||||
duration: 1000
|
||||
},
|
||||
concurrency: opts.concurrency || 1
|
||||
}).on("ready", ()=>{}).on("active", (job)=>sublogger.debug(`active id=${job.id}`)).on("completed", (job)=>sublogger.debug(`completed id=${job.id}`)).on("failed", (job, err)=>sublogger.warn(`failed(${err}) id=${job?.id}`, {
|
||||
job,
|
||||
e: renderError(err)
|
||||
})).on("error", (err)=>sublogger.error(`error(${err})`, {
|
||||
e: renderError(err)
|
||||
})).on("stalled", (jobId)=>sublogger.warn(`stalled id=${jobId}`)).waitUntilReady().then(()=>{});
|
||||
};
|
||||
return [
|
||||
queue,
|
||||
initWorker
|
||||
];
|
||||
}
|
||||
export function createQueueProducer(name) {
|
||||
return new Queue(name, {
|
||||
connection: connectionDetails,
|
||||
prefix: config.redis.prefix
|
||||
});
|
||||
}
|
||||
export function processorTimeout(processor, timeout) {
|
||||
return (job)=>{
|
||||
return new Promise((resolve, reject)=>{
|
||||
const timer = setTimeout(()=>reject(new Error("timeout reached")), timeout * 1000);
|
||||
processor(job).then(resolve).catch(reject).finally(()=>clearTimeout(timer));
|
||||
});
|
||||
};
|
||||
}
|
||||
export const defaultJobOpts = {
|
||||
removeOnComplete: process.env["NODE_ENV"] === "production" ? {
|
||||
age: 600
|
||||
} : true,
|
||||
removeOnFail: process.env["NODE_ENV"] === "production" ? {
|
||||
age: 3600
|
||||
} : true
|
||||
};
|
||||
@@ -0,0 +1,45 @@
|
||||
import { queueLogger } from "../../logger.js";
|
||||
import { deleteFileSync } from "../../../services/drive/delete-file.js";
|
||||
import { DriveFiles } from "../../../models/index.js";
|
||||
import { User } from "../../../models/entities/user.js";
|
||||
import config from "../../../config/index.js";
|
||||
const logger = queueLogger.createSubLogger("clean-remote-files");
|
||||
export async function cleanRemoteFiles(job) {
|
||||
let progress = 0;
|
||||
const untilDate = new Date(Date.now() - new Date().getTimezoneOffset() * 60000);
|
||||
untilDate.setDate(untilDate.getDate() - (config.mediaCleanup?.maxAgeDays ?? 0));
|
||||
const avatars = !(config.mediaCleanup?.keepHeaders ?? true);
|
||||
const headers = !(config.mediaCleanup?.keepHeaders ?? true);
|
||||
const until = untilDate.toISOString().replace("T", " ").slice(0, -1);
|
||||
let target = "files";
|
||||
if (avatars) if (headers) target += ", avatars & headers";
|
||||
else target += " & avatars";
|
||||
else if (headers) target += " & headers";
|
||||
logger.info(`Deleting cached remote ${target} created before ${until}...`);
|
||||
let query = DriveFiles.createQueryBuilder("file").where(`file.isLink = FALSE`).andWhere(`file.userHost IS NOT NULL`).andWhere("file.createdAt < :until", {
|
||||
until
|
||||
});
|
||||
if (!avatars || !headers) {
|
||||
query = query.andWhere((qb)=>{
|
||||
let sq = qb.subQuery().from(User, "user");
|
||||
if (!avatars) sq = sq.where("file.id = user.avatarId");
|
||||
if (!headers) sq = sq.orWhere("file.id = user.bannerId");
|
||||
return `NOT EXISTS ${sq.getQuery()}`;
|
||||
});
|
||||
}
|
||||
query = query.take(8);
|
||||
const total = await query.getCount();
|
||||
logger.info(`Deleting ${total} files, please wait...`);
|
||||
while(true){
|
||||
const files = await query.getMany();
|
||||
if (files.length === 0) {
|
||||
job.updateProgress(100);
|
||||
break;
|
||||
}
|
||||
await Promise.all(files.map((file)=>deleteFileSync(file, true)));
|
||||
progress += files.length;
|
||||
job.updateProgress(progress / total * 100);
|
||||
}
|
||||
logger.succ(`Remote media cleanup job completed successfully.`);
|
||||
return "Success";
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
import { deleteObjectStorageFile } from "../../../services/drive/delete-file.js";
|
||||
export async function deleteFile(job) {
|
||||
const key = job.data.key;
|
||||
await deleteObjectStorageFile(key);
|
||||
return "Success";
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
import { createQueue, defaultJobOpts } from "../index.js";
|
||||
import { cleanRemoteFiles } from "./clean-remote-files.js";
|
||||
import { deleteFile } from "./delete-file.js";
|
||||
import config from "../../../config/index.js";
|
||||
const processors = {
|
||||
cleanRemoteFiles,
|
||||
deleteFile
|
||||
};
|
||||
async function process(job) {
|
||||
const processor = processors[job.name];
|
||||
if (processor === undefined) return "skip: unknown job name";
|
||||
return await processor(job);
|
||||
}
|
||||
const [objectStorageQueue_, objectStorageInitQueue] = createQueue("objectStorage", process, {
|
||||
concurrency: 16
|
||||
});
|
||||
export const objectStorageQueue = objectStorageQueue_;
|
||||
export async function objectStorageInit() {
|
||||
await objectStorageInitQueue();
|
||||
if (config.mediaCleanup?.cron) {
|
||||
await objectStorageQueue.upsertJobScheduler("cleanRemoteFiles", {
|
||||
pattern: "0 0 * * *"
|
||||
}, {
|
||||
name: "cleanRemoteFiles",
|
||||
opts: defaultJobOpts
|
||||
});
|
||||
}
|
||||
}
|
||||
export function createDeleteObjectStorageFileJob(key) {
|
||||
return objectStorageQueue.add("deleteFile", {
|
||||
key: key
|
||||
}, defaultJobOpts);
|
||||
}
|
||||
export function createCleanRemoteFilesJob() {
|
||||
return objectStorageQueue.add("cleanRemoteFiles", {}, defaultJobOpts);
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
import { MemorietArchives, Memoriets, Notes, Users } from "../../../models/index.js";
|
||||
import deleteNote from "../../../services/note/delete.js";
|
||||
import { queueLogger } from "../../logger.js";
|
||||
import { genId } from "../../../misc/gen-id.js";
|
||||
const logger = queueLogger.createSubLogger("check-expired-memoriets");
|
||||
export async function checkExpiredMemoriets() {
|
||||
logger.info("Checking expired Memoriets...");
|
||||
const expired = await Memoriets.createQueryBuilder("memoriet").where("memoriet.expiresAt IS NOT NULL").andWhere("memoriet.expiresAt < :now", {
|
||||
now: new Date()
|
||||
}).getMany();
|
||||
for (const memoriet of expired){
|
||||
const note = await Notes.findOneBy({
|
||||
id: memoriet.noteId
|
||||
});
|
||||
if (note != null) {
|
||||
const user = await Users.findOneBy({
|
||||
id: note.userId
|
||||
});
|
||||
if (user != null) {
|
||||
await MemorietArchives.save({
|
||||
id: genId(),
|
||||
createdAt: memoriet.createdAt,
|
||||
deletedAt: new Date(),
|
||||
userId: memoriet.userId,
|
||||
text: note.text?.replace(/\n?#Memoriet\b/gi, "").trim() || null,
|
||||
cw: note.cw,
|
||||
fileIds: note.fileIds,
|
||||
textLayers: memoriet.textLayers,
|
||||
visibility: note.visibility
|
||||
}).catch((err)=>{
|
||||
logger.warn(`Failed to archive expired Memoriet ${memoriet.id}: ${err}`);
|
||||
});
|
||||
await deleteNote(user, note, true).catch((err)=>{
|
||||
logger.warn(`Failed to delete expired Memoriet note ${note.id}: ${err}`);
|
||||
});
|
||||
}
|
||||
}
|
||||
await Memoriets.delete(memoriet.id);
|
||||
}
|
||||
logger.succ("All expired Memoriets checked.");
|
||||
return `done: ${expired.length}`;
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
import { In } from "typeorm";
|
||||
import { Mutings } from "../../../models/index.js";
|
||||
import { queueLogger } from "../../logger.js";
|
||||
import { publishUserEvent } from "../../../services/stream.js";
|
||||
const logger = queueLogger.createSubLogger("check-expired-mutings");
|
||||
export async function checkExpiredMutings() {
|
||||
logger.info("Checking expired mutings...");
|
||||
const expired = await Mutings.createQueryBuilder("muting").where("muting.expiresAt IS NOT NULL").andWhere("muting.expiresAt < :now", {
|
||||
now: new Date()
|
||||
}).innerJoinAndSelect("muting.mutee", "mutee").getMany();
|
||||
if (expired.length > 0) {
|
||||
await Mutings.delete({
|
||||
id: In(expired.map((m)=>m.id))
|
||||
});
|
||||
for (const m of expired){
|
||||
publishUserEvent(m.muterId, "unmute", m.mutee);
|
||||
}
|
||||
}
|
||||
logger.succ("All expired mutings checked.");
|
||||
return "Success";
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
import { queueLogger } from "../../logger.js";
|
||||
import { activeUsersChart, driveChart, federationChart, hashtagChart, instanceChart, notesChart, perUserDriveChart, perUserFollowingChart, perUserNotesChart, perUserReactionsChart, usersChart, apRequestChart } from "../../../services/chart/index.js";
|
||||
const logger = queueLogger.createSubLogger("clean-charts");
|
||||
export async function cleanCharts() {
|
||||
logger.info("Clean charts...");
|
||||
await Promise.all([
|
||||
federationChart.clean(),
|
||||
notesChart.clean(),
|
||||
usersChart.clean(),
|
||||
activeUsersChart.clean(),
|
||||
instanceChart.clean(),
|
||||
perUserNotesChart.clean(),
|
||||
driveChart.clean(),
|
||||
perUserReactionsChart.clean(),
|
||||
hashtagChart.clean(),
|
||||
perUserFollowingChart.clean(),
|
||||
perUserDriveChart.clean(),
|
||||
apRequestChart.clean()
|
||||
]);
|
||||
logger.succ("All charts successfully cleaned.");
|
||||
return "Success";
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
import { LessThan } from "typeorm";
|
||||
import { UserIps } from "../../../models/index.js";
|
||||
import { queueLogger } from "../../logger.js";
|
||||
const logger = queueLogger.createSubLogger("clean");
|
||||
export async function clean() {
|
||||
logger.info("Cleaning...");
|
||||
await UserIps.delete({
|
||||
createdAt: LessThan(new Date(Date.now() - 1000 * 60 * 60 * 24 * 90))
|
||||
});
|
||||
logger.succ("Cleaned.");
|
||||
return "Success";
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
import { ScheduledNotes } from "../../../models/index.js";
|
||||
import { createQueue, defaultJobOpts } from "../index.js";
|
||||
import { clean } from "./clean.js";
|
||||
import { cleanCharts } from "./clean-charts.js";
|
||||
import { checkExpiredMutings } from "./check-expired-mutings.js";
|
||||
import { checkExpiredMemoriets } from "./check-expired-memoriets.js";
|
||||
import { resyncCharts } from "./resync-charts.js";
|
||||
import { tickCharts } from "./tick-charts.js";
|
||||
import { verifyLinks } from "./verify-links.js";
|
||||
import { publishScheduledNoteJob } from "./publish-scheduled-note.js";
|
||||
import { enqueueScheduledNote } from "./scheduled-note.js";
|
||||
const processors = {
|
||||
clean,
|
||||
cleanCharts,
|
||||
checkExpiredMutings,
|
||||
checkExpiredMemoriets,
|
||||
resyncCharts,
|
||||
tickCharts,
|
||||
verifyLinks,
|
||||
publishScheduledNote: publishScheduledNoteJob
|
||||
};
|
||||
async function process(job) {
|
||||
const processor = processors[job.name];
|
||||
if (processor === undefined) return "skip: unknown job name";
|
||||
return await processor(job);
|
||||
}
|
||||
const [systemQueue_, systemInitQueue] = createQueue("system", process, {
|
||||
concurrency: Object.keys(processors).length
|
||||
});
|
||||
export const systemQueue = systemQueue_;
|
||||
async function enqueuePendingScheduledNotes() {
|
||||
const scheduledNotes = await ScheduledNotes.find({
|
||||
where: {
|
||||
status: "scheduled"
|
||||
}
|
||||
});
|
||||
for (const scheduledNote of scheduledNotes){
|
||||
await enqueueScheduledNote(scheduledNote);
|
||||
}
|
||||
}
|
||||
export async function systemInit() {
|
||||
await systemInitQueue();
|
||||
for (const { name, seconds } of [
|
||||
{
|
||||
name: "clean",
|
||||
seconds: 60 * 60
|
||||
},
|
||||
{
|
||||
name: "cleanCharts",
|
||||
seconds: 60 * 60
|
||||
},
|
||||
{
|
||||
name: "checkExpiredMutings",
|
||||
seconds: 5 * 60
|
||||
},
|
||||
{
|
||||
name: "checkExpiredMemoriets",
|
||||
seconds: 60
|
||||
},
|
||||
{
|
||||
name: "resyncCharts",
|
||||
seconds: 60 * 60
|
||||
},
|
||||
{
|
||||
name: "tickCharts",
|
||||
seconds: 60
|
||||
},
|
||||
{
|
||||
name: "verifyLinks",
|
||||
seconds: 60 * 60 * 24
|
||||
}
|
||||
]){
|
||||
await systemQueue.upsertJobScheduler(name, {
|
||||
every: seconds * 1000
|
||||
}, {
|
||||
opts: defaultJobOpts
|
||||
});
|
||||
}
|
||||
await enqueuePendingScheduledNotes();
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
import { publishScheduledNote } from "../../../services/note/scheduled.js";
|
||||
export async function publishScheduledNoteJob(job) {
|
||||
return await publishScheduledNote(job.data.scheduledNoteId);
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
import { queueLogger } from "../../logger.js";
|
||||
import { driveChart, notesChart, usersChart } from "../../../services/chart/index.js";
|
||||
const logger = queueLogger.createSubLogger("resync-charts");
|
||||
export async function resyncCharts() {
|
||||
logger.info("Resync charts...");
|
||||
// TODO: ユーザーごとのチャートも更新する
|
||||
// TODO: インスタンスごとのチャートも更新する
|
||||
await Promise.all([
|
||||
driveChart.resync(),
|
||||
notesChart.resync(),
|
||||
usersChart.resync()
|
||||
]);
|
||||
logger.succ("All charts successfully resynced.");
|
||||
return "Success";
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
import { createQueueProducer, defaultJobOpts } from "../index.js";
|
||||
const scheduledNoteSystemQueue = createQueueProducer("system");
|
||||
export async function enqueueScheduledNote(scheduledNote) {
|
||||
await scheduledNoteSystemQueue.add("publishScheduledNote", {
|
||||
scheduledNoteId: scheduledNote.id
|
||||
}, {
|
||||
...defaultJobOpts,
|
||||
delay: Math.max(0, scheduledNote.scheduledAt.getTime() - Date.now()),
|
||||
jobId: `scheduledNote:${scheduledNote.id}`
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
import { queueLogger } from "../../logger.js";
|
||||
import { activeUsersChart, driveChart, federationChart, hashtagChart, instanceChart, notesChart, perUserDriveChart, perUserFollowingChart, perUserNotesChart, perUserReactionsChart, usersChart, apRequestChart } from "../../../services/chart/index.js";
|
||||
const logger = queueLogger.createSubLogger("tick-charts");
|
||||
export async function tickCharts() {
|
||||
logger.info("Tick charts...");
|
||||
await Promise.all([
|
||||
federationChart.tick(false),
|
||||
notesChart.tick(false),
|
||||
usersChart.tick(false),
|
||||
activeUsersChart.tick(false),
|
||||
instanceChart.tick(false),
|
||||
perUserNotesChart.tick(false),
|
||||
driveChart.tick(false),
|
||||
perUserReactionsChart.tick(false),
|
||||
hashtagChart.tick(false),
|
||||
perUserFollowingChart.tick(false),
|
||||
perUserDriveChart.tick(false),
|
||||
apRequestChart.tick(false)
|
||||
]);
|
||||
logger.succ("All charts successfully ticked.");
|
||||
return "Success";
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
import { UserProfiles } from "../../../models/index.js";
|
||||
import { Not } from "typeorm";
|
||||
import { queueLogger } from "../../logger.js";
|
||||
import { verifyLink } from "../../../services/fetch-rel-me.js";
|
||||
const logger = queueLogger.createSubLogger("verify-links");
|
||||
export async function verifyLinks() {
|
||||
logger.info("Verifying links...");
|
||||
const usersToVerify = await UserProfiles.findBy({
|
||||
fields: Not(null),
|
||||
userHost: ""
|
||||
});
|
||||
for (const user of usersToVerify){
|
||||
for (const field of user.fields){
|
||||
if (!field || field.name === "" || field.value === "") {
|
||||
continue;
|
||||
}
|
||||
if (field.value.startsWith("http") && user.user?.username) {
|
||||
field.verified = await verifyLink(field.value, user.user.username);
|
||||
}
|
||||
}
|
||||
if (user.fields.length > 0) {
|
||||
try {
|
||||
await UserProfiles.update(user.userId, {
|
||||
fields: user.fields
|
||||
});
|
||||
} catch (e) {
|
||||
logger.error(`Failed to update user ${user.userId} ${e}`);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
}
|
||||
logger.succ("All links successfully verified.");
|
||||
return "Success";
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
import Logger from "../../services/logger.js";
|
||||
import { getResponse, StatusError } from "../../misc/fetch.js";
|
||||
import { Webhooks } from "../../models/index.js";
|
||||
import config from "../../config/index.js";
|
||||
import { v4 as uuid } from "uuid";
|
||||
import { createQueue, defaultJobOpts, processorTimeout } from "./index.js";
|
||||
const logger = new Logger("webhook");
|
||||
async function process(job) {
|
||||
if (job.data == null || Object.keys(job.data).length === 0) {
|
||||
job.opts.removeOnComplete = true;
|
||||
return "Skip (data was null or empty)";
|
||||
}
|
||||
try {
|
||||
logger.debug(`delivering ${job.data.webhookId}`);
|
||||
const res = await getResponse({
|
||||
url: job.data.to,
|
||||
method: "POST",
|
||||
headers: {
|
||||
"User-Agent": "Iceshrimp-Hooks",
|
||||
"X-Iceshrimp-Host": config.host,
|
||||
"X-Iceshrimp-Hook-Id": job.data.webhookId,
|
||||
"X-Iceshrimp-Hook-Secret": job.data.secret,
|
||||
"Content-Type": "application/json"
|
||||
},
|
||||
body: JSON.stringify({
|
||||
hookId: job.data.webhookId,
|
||||
userId: job.data.userId,
|
||||
eventId: job.data.eventId,
|
||||
createdAt: job.data.createdAt,
|
||||
type: job.data.type,
|
||||
body: job.data.content
|
||||
})
|
||||
});
|
||||
Webhooks.update({
|
||||
id: job.data.webhookId
|
||||
}, {
|
||||
latestSentAt: new Date(),
|
||||
latestStatus: res.status
|
||||
});
|
||||
return "Success";
|
||||
} catch (res) {
|
||||
Webhooks.update({
|
||||
id: job.data.webhookId
|
||||
}, {
|
||||
latestSentAt: new Date(),
|
||||
latestStatus: res instanceof StatusError ? res.statusCode : 1
|
||||
});
|
||||
if (res instanceof StatusError) {
|
||||
// 4xx
|
||||
if (!res.isRetryable) {
|
||||
return `${res.statusCode} ${res.statusMessage}`;
|
||||
}
|
||||
// 5xx etc.
|
||||
throw new Error(`${res.statusCode} ${res.statusMessage}`);
|
||||
} else {
|
||||
// DNS error, socket error, timeout ...
|
||||
throw res;
|
||||
}
|
||||
}
|
||||
}
|
||||
export const [webhookDeliverQueue, webhookDeliverInit] = createQueue("webhookDeliver", processorTimeout(process, 60), {
|
||||
limitPerSec: 64,
|
||||
concurrency: 64
|
||||
});
|
||||
export function webhookDeliverJob(webhook, type, content) {
|
||||
const data = {
|
||||
type,
|
||||
content,
|
||||
webhookId: webhook.id,
|
||||
userId: webhook.userId,
|
||||
to: webhook.url,
|
||||
secret: webhook.secret,
|
||||
createdAt: Date.now(),
|
||||
eventId: uuid()
|
||||
};
|
||||
return webhookDeliverQueue.add("default", data, {
|
||||
attempts: 4,
|
||||
backoff: {
|
||||
type: "custom"
|
||||
},
|
||||
...defaultJobOpts
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { };
|
||||
Reference in New Issue
Block a user