Fixed 267U.pre2
This commit is contained in:
@@ -0,0 +1,84 @@
|
||||
import { queueLogger } from "../../logger.js";
|
||||
import { DriveFiles, Notes, UserProfiles, Users } from "../../../models/index.js";
|
||||
import { MoreThan } from "typeorm";
|
||||
import { deleteFileSync } from "../../../services/drive/delete-file.js";
|
||||
import { sendEmail } from "../../../services/send-email.js";
|
||||
import { publishInternalEvent } from "../../../services/stream.js";
|
||||
const logger = queueLogger.createSubLogger("delete-account");
|
||||
export async function deleteAccount(job) {
|
||||
logger.info(`Deleting account of ${job.data.user.id} ...`);
|
||||
const user = await Users.findOneBy({
|
||||
id: job.data.user.id
|
||||
});
|
||||
if (!user) return "skip: User not found";
|
||||
const isLocal = Users.isLocalUser(user);
|
||||
{
|
||||
// Delete notes
|
||||
let cursor = null;
|
||||
while(true){
|
||||
const notes = await Notes.find({
|
||||
where: {
|
||||
userId: user.id,
|
||||
...cursor ? {
|
||||
id: MoreThan(cursor)
|
||||
} : {}
|
||||
},
|
||||
take: 10,
|
||||
order: {
|
||||
id: 1
|
||||
}
|
||||
});
|
||||
if (notes.length === 0) {
|
||||
break;
|
||||
}
|
||||
cursor = notes[notes.length - 1].id;
|
||||
await Notes.delete(notes.map((note)=>note.id));
|
||||
}
|
||||
logger.succ("All of notes deleted");
|
||||
}
|
||||
{
|
||||
// Delete files
|
||||
let cursor = null;
|
||||
while(true){
|
||||
const files = await DriveFiles.find({
|
||||
where: {
|
||||
userId: user.id,
|
||||
...cursor ? {
|
||||
id: MoreThan(cursor)
|
||||
} : {}
|
||||
},
|
||||
take: 10,
|
||||
order: {
|
||||
id: 1
|
||||
}
|
||||
});
|
||||
if (files.length === 0) {
|
||||
break;
|
||||
}
|
||||
cursor = files[files.length - 1].id;
|
||||
for (const file of files){
|
||||
await deleteFileSync(file);
|
||||
}
|
||||
}
|
||||
logger.succ("All of files deleted");
|
||||
}
|
||||
{
|
||||
// Send email notification
|
||||
const profile = await UserProfiles.findOneByOrFail({
|
||||
userId: user.id
|
||||
});
|
||||
if (profile.email && profile.emailVerified) {
|
||||
sendEmail(profile.email, "Account deleted", "Your account has been deleted.", "Your account has been deleted.");
|
||||
}
|
||||
}
|
||||
// soft指定されている場合は物理削除しない
|
||||
if (job.data.soft) {
|
||||
// nop
|
||||
} else {
|
||||
await Users.delete(job.data.user.id);
|
||||
publishInternalEvent(isLocal ? "localUserDeleted" : "remoteUserDeleted", {
|
||||
id: user.id
|
||||
});
|
||||
}
|
||||
return "Account deleted";
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
import { queueLogger } from "../../logger.js";
|
||||
import { deleteFileSync } from "../../../services/drive/delete-file.js";
|
||||
import { Users, DriveFiles } from "../../../models/index.js";
|
||||
import { MoreThan } from "typeorm";
|
||||
const logger = queueLogger.createSubLogger("delete-drive-files");
|
||||
export async function deleteDriveFiles(job) {
|
||||
logger.info(`Deleting drive files of ${job.data.user.id} ...`);
|
||||
const user = await Users.findOneBy({
|
||||
id: job.data.user.id
|
||||
});
|
||||
if (user == null) {
|
||||
return "skip: User not found";
|
||||
}
|
||||
let deletedCount = 0;
|
||||
let cursor = null;
|
||||
while(true){
|
||||
const files = await DriveFiles.find({
|
||||
where: {
|
||||
userId: user.id,
|
||||
...cursor ? {
|
||||
id: MoreThan(cursor)
|
||||
} : {}
|
||||
},
|
||||
take: 100,
|
||||
order: {
|
||||
id: 1
|
||||
}
|
||||
});
|
||||
if (files.length === 0) {
|
||||
job.updateProgress(100);
|
||||
break;
|
||||
}
|
||||
cursor = files[files.length - 1].id;
|
||||
for (const file of files){
|
||||
await deleteFileSync(file);
|
||||
deletedCount++;
|
||||
}
|
||||
const total = await DriveFiles.countBy({
|
||||
userId: user.id
|
||||
});
|
||||
job.updateProgress(deletedCount / total);
|
||||
}
|
||||
logger.succ(`All drive files (${deletedCount}) of ${user.id} has been deleted.`);
|
||||
return "Success";
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
import * as fs from "node:fs";
|
||||
import { queueLogger } from "../../logger.js";
|
||||
import { addFile } from "../../../services/drive/add-file.js";
|
||||
import { format as dateFormat } from "date-fns";
|
||||
import { getFullApAccount } from "../../../misc/convert-host.js";
|
||||
import { createTemp } from "../../../misc/create-temp.js";
|
||||
import { Users, Blockings } from "../../../models/index.js";
|
||||
import { MoreThan } from "typeorm";
|
||||
const logger = queueLogger.createSubLogger("export-blocking");
|
||||
export async function exportBlocking(job) {
|
||||
logger.info(`Exporting blocking of ${job.data.user.id} ...`);
|
||||
const user = await Users.findOneBy({
|
||||
id: job.data.user.id
|
||||
});
|
||||
if (user == null) {
|
||||
return "skip: User not found";
|
||||
}
|
||||
// Create temp file
|
||||
const [path, cleanup] = await createTemp();
|
||||
logger.info(`Temp file is ${path}`);
|
||||
try {
|
||||
const stream = fs.createWriteStream(path, {
|
||||
flags: "a"
|
||||
});
|
||||
let exportedCount = 0;
|
||||
let cursor = null;
|
||||
while(true){
|
||||
const blockings = await Blockings.find({
|
||||
where: {
|
||||
blockerId: user.id,
|
||||
...cursor ? {
|
||||
id: MoreThan(cursor)
|
||||
} : {}
|
||||
},
|
||||
take: 100,
|
||||
order: {
|
||||
id: 1
|
||||
}
|
||||
});
|
||||
if (blockings.length === 0) {
|
||||
job.updateProgress(100);
|
||||
break;
|
||||
}
|
||||
cursor = blockings[blockings.length - 1].id;
|
||||
for (const block of blockings){
|
||||
const u = await Users.findOneBy({
|
||||
id: block.blockeeId
|
||||
});
|
||||
if (u == null) {
|
||||
exportedCount++;
|
||||
continue;
|
||||
}
|
||||
const content = getFullApAccount(u.username, u.host);
|
||||
await new Promise((res, rej)=>{
|
||||
stream.write(content + "\n", (err)=>{
|
||||
if (err) {
|
||||
logger.error(err);
|
||||
rej(err);
|
||||
} else {
|
||||
res();
|
||||
}
|
||||
});
|
||||
});
|
||||
exportedCount++;
|
||||
}
|
||||
const total = await Blockings.countBy({
|
||||
blockerId: user.id
|
||||
});
|
||||
job.updateProgress(exportedCount / total);
|
||||
}
|
||||
stream.end();
|
||||
logger.succ(`Exported to: ${path}`);
|
||||
const fileName = `blocking-${dateFormat(new Date(), "yyyy-MM-dd-HH-mm-ss")}.csv`;
|
||||
const driveFile = await addFile({
|
||||
user,
|
||||
path,
|
||||
name: fileName,
|
||||
force: true
|
||||
});
|
||||
logger.succ(`Exported to: ${driveFile.id}`);
|
||||
} finally{
|
||||
cleanup();
|
||||
}
|
||||
return "Success";
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
import * as fs from "node:fs";
|
||||
import mime from "mime-types";
|
||||
import archiver from "archiver";
|
||||
import { queueLogger } from "../../logger.js";
|
||||
import { addFile } from "../../../services/drive/add-file.js";
|
||||
import { format as dateFormat } from "date-fns";
|
||||
import { Users, Emojis } from "../../../models/index.js";
|
||||
import { createTemp, createTempDir } from "../../../misc/create-temp.js";
|
||||
import { downloadUrl } from "../../../misc/download-url.js";
|
||||
import config from "../../../config/index.js";
|
||||
import { IsNull } from "typeorm";
|
||||
const logger = queueLogger.createSubLogger("export-custom-emojis");
|
||||
export async function exportCustomEmojis(job) {
|
||||
logger.info("Exporting custom emojis ...");
|
||||
const user = await Users.findOneBy({
|
||||
id: job.data.user.id
|
||||
});
|
||||
if (user == null) {
|
||||
return "skip: User not found";
|
||||
}
|
||||
const [path, cleanup] = await createTempDir();
|
||||
logger.info(`Temp dir is ${path}`);
|
||||
const metaPath = `${path}/meta.json`;
|
||||
fs.writeFileSync(metaPath, "", "utf-8");
|
||||
const metaStream = fs.createWriteStream(metaPath, {
|
||||
flags: "a"
|
||||
});
|
||||
const writeMeta = (text)=>{
|
||||
return new Promise((res, rej)=>{
|
||||
metaStream.write(text, (err)=>{
|
||||
if (err) {
|
||||
logger.error(err);
|
||||
rej(err);
|
||||
} else {
|
||||
res();
|
||||
}
|
||||
});
|
||||
});
|
||||
};
|
||||
await writeMeta(`{"metaVersion":2,"host":"${config.host}","exportedAt":"${new Date().toString()}","emojis":[`);
|
||||
const customEmojis = await Emojis.find({
|
||||
where: {
|
||||
host: IsNull()
|
||||
},
|
||||
order: {
|
||||
id: "ASC"
|
||||
}
|
||||
});
|
||||
for (const emoji of customEmojis){
|
||||
const ext = mime.extension(emoji.type);
|
||||
// there are some restrictions on file names, so to be safe the files are
|
||||
// named after their database id instead of the actual emoji name
|
||||
const fileName = emoji.id + (ext ? '.' + ext : '');
|
||||
const emojiPath = `${path}/${fileName}`;
|
||||
fs.writeFileSync(emojiPath, "", "binary");
|
||||
let downloaded = false;
|
||||
try {
|
||||
await downloadUrl(emoji.originalUrl, emojiPath);
|
||||
downloaded = true;
|
||||
} catch (e) {
|
||||
// TODO: 何度か再試行
|
||||
logger.error(e instanceof Error ? e : new Error(e));
|
||||
}
|
||||
if (!downloaded) {
|
||||
fs.unlinkSync(emojiPath);
|
||||
}
|
||||
const content = JSON.stringify({
|
||||
fileName: fileName,
|
||||
downloaded: downloaded,
|
||||
emoji: emoji
|
||||
});
|
||||
const isFirst = customEmojis.indexOf(emoji) === 0;
|
||||
await writeMeta(isFirst ? content : ",\n" + content);
|
||||
}
|
||||
await writeMeta("]}");
|
||||
metaStream.end();
|
||||
// Create archive
|
||||
await new Promise(async (resolve, reject)=>{
|
||||
try {
|
||||
const [archivePath, archiveCleanup] = await createTemp();
|
||||
const archiveStream = fs.createWriteStream(archivePath);
|
||||
const archive = archiver("zip", {
|
||||
zlib: {
|
||||
level: 0
|
||||
}
|
||||
});
|
||||
archiveStream.on("close", async ()=>{
|
||||
logger.succ(`Exported to: ${archivePath}`);
|
||||
const fileName = `custom-emojis-${dateFormat(new Date(), "yyyy-MM-dd-HH-mm-ss")}.zip`;
|
||||
const driveFile = await addFile({
|
||||
user,
|
||||
path: archivePath,
|
||||
name: fileName,
|
||||
force: true
|
||||
});
|
||||
logger.succ(`Exported to: ${driveFile.id}`);
|
||||
cleanup();
|
||||
archiveCleanup();
|
||||
resolve(undefined);
|
||||
});
|
||||
archive.pipe(archiveStream);
|
||||
archive.directory(path, false);
|
||||
archive.finalize();
|
||||
} catch (e) {
|
||||
reject(e);
|
||||
}
|
||||
});
|
||||
return "Success";
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
import * as fs from "node:fs";
|
||||
import { queueLogger } from "../../logger.js";
|
||||
import { addFile } from "../../../services/drive/add-file.js";
|
||||
import { format as dateFormat } from "date-fns";
|
||||
import { getFullApAccount } from "../../../misc/convert-host.js";
|
||||
import { createTemp } from "../../../misc/create-temp.js";
|
||||
import { Users, Followings, Mutings } from "../../../models/index.js";
|
||||
import { In, MoreThan, Not } from "typeorm";
|
||||
const logger = queueLogger.createSubLogger("export-following");
|
||||
export async function exportFollowing(job) {
|
||||
logger.info(`Exporting following of ${job.data.user.id} ...`);
|
||||
const user = await Users.findOneBy({
|
||||
id: job.data.user.id
|
||||
});
|
||||
if (user == null) {
|
||||
return "skip: User not found";
|
||||
}
|
||||
// Create temp file
|
||||
const [path, cleanup] = await createTemp();
|
||||
logger.info(`Temp file is ${path}`);
|
||||
try {
|
||||
const stream = fs.createWriteStream(path, {
|
||||
flags: "a"
|
||||
});
|
||||
let cursor = null;
|
||||
const mutings = job.data.excludeMuting ? await Mutings.findBy({
|
||||
muterId: user.id
|
||||
}) : [];
|
||||
while(true){
|
||||
const followings = await Followings.find({
|
||||
where: {
|
||||
followerId: user.id,
|
||||
...mutings.length > 0 ? {
|
||||
followeeId: Not(In(mutings.map((x)=>x.muteeId)))
|
||||
} : {},
|
||||
...cursor ? {
|
||||
id: MoreThan(cursor)
|
||||
} : {}
|
||||
},
|
||||
take: 100,
|
||||
order: {
|
||||
id: 1
|
||||
}
|
||||
});
|
||||
if (followings.length === 0) {
|
||||
break;
|
||||
}
|
||||
cursor = followings[followings.length - 1].id;
|
||||
for (const following of followings){
|
||||
const u = await Users.findOneBy({
|
||||
id: following.followeeId
|
||||
});
|
||||
if (u == null) {
|
||||
continue;
|
||||
}
|
||||
if (job.data.excludeInactive && u.updatedAt && Date.now() - u.updatedAt.getTime() > 1000 * 60 * 60 * 24 * 90) {
|
||||
continue;
|
||||
}
|
||||
const content = getFullApAccount(u.username, u.host);
|
||||
await new Promise((res, rej)=>{
|
||||
stream.write(content + "\n", (err)=>{
|
||||
if (err) {
|
||||
logger.error(err);
|
||||
rej(err);
|
||||
} else {
|
||||
res();
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
stream.end();
|
||||
logger.succ(`Exported to: ${path}`);
|
||||
const fileName = `following-${dateFormat(new Date(), "yyyy-MM-dd-HH-mm-ss")}.csv`;
|
||||
const driveFile = await addFile({
|
||||
user,
|
||||
path,
|
||||
name: fileName,
|
||||
force: true
|
||||
});
|
||||
logger.succ(`Exported to: ${driveFile.id}`);
|
||||
} finally{
|
||||
cleanup();
|
||||
}
|
||||
return "Success";
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
import * as fs from "node:fs";
|
||||
import { queueLogger } from "../../logger.js";
|
||||
import { addFile } from "../../../services/drive/add-file.js";
|
||||
import { format as dateFormat } from "date-fns";
|
||||
import { getFullApAccount } from "../../../misc/convert-host.js";
|
||||
import { createTemp } from "../../../misc/create-temp.js";
|
||||
import { Users, Mutings } from "../../../models/index.js";
|
||||
import { IsNull, MoreThan } from "typeorm";
|
||||
const logger = queueLogger.createSubLogger("export-mute");
|
||||
export async function exportMute(job) {
|
||||
logger.info(`Exporting mute of ${job.data.user.id} ...`);
|
||||
const user = await Users.findOneBy({
|
||||
id: job.data.user.id
|
||||
});
|
||||
if (user == null) {
|
||||
return "skip: User not found";
|
||||
}
|
||||
// Create temp file
|
||||
const [path, cleanup] = await createTemp();
|
||||
logger.info(`Temp file is ${path}`);
|
||||
try {
|
||||
const stream = fs.createWriteStream(path, {
|
||||
flags: "a"
|
||||
});
|
||||
let exportedCount = 0;
|
||||
let cursor = null;
|
||||
while(true){
|
||||
const mutes = await Mutings.find({
|
||||
where: {
|
||||
muterId: user.id,
|
||||
expiresAt: IsNull(),
|
||||
...cursor ? {
|
||||
id: MoreThan(cursor)
|
||||
} : {}
|
||||
},
|
||||
take: 100,
|
||||
order: {
|
||||
id: 1
|
||||
}
|
||||
});
|
||||
if (mutes.length === 0) {
|
||||
job.updateProgress(100);
|
||||
break;
|
||||
}
|
||||
cursor = mutes[mutes.length - 1].id;
|
||||
for (const mute of mutes){
|
||||
const u = await Users.findOneBy({
|
||||
id: mute.muteeId
|
||||
});
|
||||
if (u == null) {
|
||||
exportedCount++;
|
||||
continue;
|
||||
}
|
||||
const content = getFullApAccount(u.username, u.host);
|
||||
await new Promise((res, rej)=>{
|
||||
stream.write(content + "\n", (err)=>{
|
||||
if (err) {
|
||||
logger.error(err);
|
||||
rej(err);
|
||||
} else {
|
||||
res();
|
||||
}
|
||||
});
|
||||
});
|
||||
exportedCount++;
|
||||
}
|
||||
const total = await Mutings.countBy({
|
||||
muterId: user.id
|
||||
});
|
||||
job.updateProgress(exportedCount / total);
|
||||
}
|
||||
stream.end();
|
||||
logger.succ(`Exported to: ${path}`);
|
||||
const fileName = `mute-${dateFormat(new Date(), "yyyy-MM-dd-HH-mm-ss")}.csv`;
|
||||
const driveFile = await addFile({
|
||||
user,
|
||||
path,
|
||||
name: fileName,
|
||||
force: true
|
||||
});
|
||||
logger.succ(`Exported to: ${driveFile.id}`);
|
||||
} finally{
|
||||
cleanup();
|
||||
}
|
||||
return "Success";
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
import * as fs from "node:fs";
|
||||
import { queueLogger } from "../../logger.js";
|
||||
import { addFile } from "../../../services/drive/add-file.js";
|
||||
import { format as dateFormat } from "date-fns";
|
||||
import { Users, Notes, Polls, DriveFiles } from "../../../models/index.js";
|
||||
import { MoreThan } from "typeorm";
|
||||
import { createTemp } from "../../../misc/create-temp.js";
|
||||
const logger = queueLogger.createSubLogger("export-notes");
|
||||
export async function exportNotes(job) {
|
||||
logger.info(`Exporting notes of ${job.data.user.id} ...`);
|
||||
const user = await Users.findOneBy({
|
||||
id: job.data.user.id
|
||||
});
|
||||
if (user == null) {
|
||||
return "skip: User not found";
|
||||
}
|
||||
// Create temp file
|
||||
const [path, cleanup] = await createTemp();
|
||||
logger.info(`Temp file is ${path}`);
|
||||
try {
|
||||
const stream = fs.createWriteStream(path, {
|
||||
flags: "a"
|
||||
});
|
||||
const write = (text)=>{
|
||||
return new Promise((res, rej)=>{
|
||||
stream.write(text, (err)=>{
|
||||
if (err) {
|
||||
logger.error(err);
|
||||
rej(err);
|
||||
} else {
|
||||
res();
|
||||
}
|
||||
});
|
||||
});
|
||||
};
|
||||
await write("[");
|
||||
let exportedNotesCount = 0;
|
||||
let cursor = null;
|
||||
while(true){
|
||||
const notes = await Notes.find({
|
||||
where: {
|
||||
userId: user.id,
|
||||
...cursor ? {
|
||||
id: MoreThan(cursor)
|
||||
} : {}
|
||||
},
|
||||
take: 100,
|
||||
order: {
|
||||
id: 1
|
||||
}
|
||||
});
|
||||
if (notes.length === 0) {
|
||||
job.updateProgress(100);
|
||||
break;
|
||||
}
|
||||
cursor = notes[notes.length - 1].id;
|
||||
for (const note of notes){
|
||||
let poll;
|
||||
if (note.hasPoll) {
|
||||
poll = await Polls.findOneByOrFail({
|
||||
noteId: note.id
|
||||
});
|
||||
}
|
||||
const content = JSON.stringify(await serialize(note, poll));
|
||||
const isFirst = exportedNotesCount === 0;
|
||||
await write(isFirst ? content : ",\n" + content);
|
||||
exportedNotesCount++;
|
||||
}
|
||||
const total = await Notes.countBy({
|
||||
userId: user.id
|
||||
});
|
||||
job.updateProgress(exportedNotesCount / total);
|
||||
}
|
||||
await write("]");
|
||||
stream.end();
|
||||
logger.succ(`Exported to: ${path}`);
|
||||
const fileName = `notes-${dateFormat(new Date(), "yyyy-MM-dd-HH-mm-ss")}.json`;
|
||||
const driveFile = await addFile({
|
||||
user,
|
||||
path,
|
||||
name: fileName,
|
||||
force: true
|
||||
});
|
||||
logger.succ(`Exported to: ${driveFile.id}`);
|
||||
} finally{
|
||||
cleanup();
|
||||
}
|
||||
return "Success";
|
||||
}
|
||||
async function serialize(note, poll = null) {
|
||||
return {
|
||||
id: note.id,
|
||||
text: note.text,
|
||||
createdAt: note.createdAt,
|
||||
fileIds: note.fileIds,
|
||||
files: await DriveFiles.packMany(note.fileIds),
|
||||
replyId: note.replyId,
|
||||
renoteId: note.renoteId,
|
||||
poll: poll,
|
||||
cw: note.cw,
|
||||
visibility: note.visibility,
|
||||
visibleUserIds: note.visibleUserIds,
|
||||
localOnly: note.localOnly
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
import * as fs from "node:fs";
|
||||
import { queueLogger } from "../../logger.js";
|
||||
import { addFile } from "../../../services/drive/add-file.js";
|
||||
import { format as dateFormat } from "date-fns";
|
||||
import { getFullApAccount } from "../../../misc/convert-host.js";
|
||||
import { createTemp } from "../../../misc/create-temp.js";
|
||||
import { Users, UserLists, UserListJoinings } from "../../../models/index.js";
|
||||
import { In } from "typeorm";
|
||||
const logger = queueLogger.createSubLogger("export-user-lists");
|
||||
export async function exportUserLists(job) {
|
||||
logger.info(`Exporting user lists of ${job.data.user.id} ...`);
|
||||
const user = await Users.findOneBy({
|
||||
id: job.data.user.id
|
||||
});
|
||||
if (user == null) {
|
||||
return "skip: User not found";
|
||||
}
|
||||
const lists = await UserLists.findBy({
|
||||
userId: user.id
|
||||
});
|
||||
// Create temp file
|
||||
const [path, cleanup] = await createTemp();
|
||||
logger.info(`Temp file is ${path}`);
|
||||
try {
|
||||
const stream = fs.createWriteStream(path, {
|
||||
flags: "a"
|
||||
});
|
||||
for (const list of lists){
|
||||
const joinings = await UserListJoinings.findBy({
|
||||
userListId: list.id
|
||||
});
|
||||
const users = await Users.findBy({
|
||||
id: In(joinings.map((j)=>j.userId))
|
||||
});
|
||||
for (const u of users){
|
||||
const acct = getFullApAccount(u.username, u.host);
|
||||
const content = `${list.name},${acct}`;
|
||||
await new Promise((res, rej)=>{
|
||||
stream.write(content + "\n", (err)=>{
|
||||
if (err) {
|
||||
logger.error(err);
|
||||
rej(err);
|
||||
} else {
|
||||
res();
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
stream.end();
|
||||
logger.succ(`Exported to: ${path}`);
|
||||
const fileName = `user-lists-${dateFormat(new Date(), "yyyy-MM-dd-HH-mm-ss")}.csv`;
|
||||
const driveFile = await addFile({
|
||||
user,
|
||||
path,
|
||||
name: fileName,
|
||||
force: true
|
||||
});
|
||||
logger.succ(`Exported to: ${driveFile.id}`);
|
||||
} finally{
|
||||
cleanup();
|
||||
}
|
||||
return "Success";
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
import { queueLogger } from "../../logger.js";
|
||||
import * as Acct from "../../../misc/acct.js";
|
||||
import { resolveUser } from "../../../remote/resolve-user.js";
|
||||
import { downloadTextFile } from "../../../misc/download-text-file.js";
|
||||
import { isSelfHost, toPuny } from "../../../misc/convert-host.js";
|
||||
import { Users, DriveFiles } from "../../../models/index.js";
|
||||
import block from "../../../services/blocking/create.js";
|
||||
import { IsNull } from "typeorm";
|
||||
const logger = queueLogger.createSubLogger("import-blocking");
|
||||
export async function importBlocking(job) {
|
||||
logger.info(`Importing blocking of ${job.data.user.id} ...`);
|
||||
const user = await Users.findOneBy({
|
||||
id: job.data.user.id
|
||||
});
|
||||
if (user == null) {
|
||||
return "skip: User not found";
|
||||
}
|
||||
const file = await DriveFiles.findOneBy({
|
||||
id: job.data.fileId
|
||||
});
|
||||
if (file == null) {
|
||||
return "skip: File not found";
|
||||
}
|
||||
const csv = await downloadTextFile(file.url);
|
||||
let linenum = 0;
|
||||
for (const line of csv.trim().split("\n")){
|
||||
linenum++;
|
||||
try {
|
||||
const acct = line.split(",")[0].trim();
|
||||
const { username, host } = Acct.parse(acct);
|
||||
let target = isSelfHost(host) ? await Users.findOneBy({
|
||||
host: IsNull(),
|
||||
usernameLower: username.toLowerCase()
|
||||
}) : await Users.findOneBy({
|
||||
host: toPuny(host),
|
||||
usernameLower: username.toLowerCase()
|
||||
});
|
||||
if (host == null && target == null) continue;
|
||||
if (target == null) {
|
||||
target = await resolveUser(username, host);
|
||||
}
|
||||
if (target == null) {
|
||||
throw new Error(`cannot resolve user: @${username}@${host}`);
|
||||
}
|
||||
// skip myself
|
||||
if (target.id === job.data.user.id) continue;
|
||||
logger.info(`Block[${linenum}] ${target.id} ...`);
|
||||
await block(user, target);
|
||||
} catch (e) {
|
||||
logger.warn(`Error in line:${linenum} ${e}`);
|
||||
}
|
||||
}
|
||||
logger.succ("Imported");
|
||||
return "Success";
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
import * as fs from "node:fs";
|
||||
import AdmZip from "adm-zip";
|
||||
import { queueLogger } from "../../logger.js";
|
||||
import { createTempDir } from "../../../misc/create-temp.js";
|
||||
import { downloadUrl } from "../../../misc/download-url.js";
|
||||
import { DriveFiles, Emojis } from "../../../models/index.js";
|
||||
import { addFile } from "../../../services/drive/add-file.js";
|
||||
import { genId } from "../../../misc/gen-id.js";
|
||||
import { db } from "../../../db/postgre.js";
|
||||
import probeImageSize from "probe-image-size";
|
||||
import * as path from "path";
|
||||
const logger = queueLogger.createSubLogger("import-custom-emojis");
|
||||
// TODO: 名前衝突時の動作を選べるようにする
|
||||
export async function importCustomEmojis(job) {
|
||||
logger.info("Importing custom emojis ...");
|
||||
const file = await DriveFiles.findOneBy({
|
||||
id: job.data.fileId
|
||||
});
|
||||
if (file == null) {
|
||||
return "skip: File not found";
|
||||
}
|
||||
const [tempPath, cleanup] = await createTempDir();
|
||||
logger.info(`Temp dir is ${tempPath}`);
|
||||
const destPath = `${tempPath}/emojis.zip`;
|
||||
try {
|
||||
fs.writeFileSync(destPath, "", "binary");
|
||||
await downloadUrl(file.url, destPath);
|
||||
} catch (e) {
|
||||
// TODO: 何度か再試行
|
||||
if (e instanceof Error || typeof e === "string") {
|
||||
logger.error(e);
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
const outputPath = `${tempPath}/emojis`;
|
||||
const unzipStream = fs.createReadStream(destPath);
|
||||
const zip = new AdmZip(destPath);
|
||||
logger.succ(`Unzipping to ${outputPath}`);
|
||||
await new Promise((resolve, reject)=>{
|
||||
zip.extractAllToAsync(outputPath, true, false, async (error)=>{
|
||||
if (error) reject(error);
|
||||
if (fs.existsSync(`${outputPath}/meta.json`)) {
|
||||
logger.info("starting emoji import with metadata");
|
||||
const metaRaw = fs.readFileSync(`${outputPath}/meta.json`, "utf-8");
|
||||
const meta = JSON.parse(metaRaw);
|
||||
for (const record of meta.emojis){
|
||||
if (!record.downloaded) continue;
|
||||
const emojiInfo = record.emoji;
|
||||
const emojiPath = `${outputPath}/${record.fileName}`;
|
||||
await Emojis.delete({
|
||||
name: emojiInfo.name
|
||||
});
|
||||
const driveFile = await addFile({
|
||||
user: null,
|
||||
path: emojiPath,
|
||||
name: record.fileName,
|
||||
force: true
|
||||
});
|
||||
const file = fs.createReadStream(emojiPath);
|
||||
const size = await probeImageSize(file);
|
||||
file.destroy();
|
||||
await Emojis.insert({
|
||||
id: genId(),
|
||||
updatedAt: new Date(),
|
||||
name: emojiInfo.name,
|
||||
category: emojiInfo.category,
|
||||
host: null,
|
||||
aliases: emojiInfo.aliases,
|
||||
originalUrl: driveFile.url,
|
||||
publicUrl: driveFile.webpublicUrl ?? driveFile.url,
|
||||
type: driveFile.webpublicType ?? driveFile.type,
|
||||
license: emojiInfo.license,
|
||||
glyph: driveFile.type === "image/svg+xml",
|
||||
width: size.width || null,
|
||||
height: size.height || null
|
||||
}).then((x)=>Emojis.findOneByOrFail(x.identifiers[0]));
|
||||
}
|
||||
} else {
|
||||
logger.info("starting emoji import without metadata");
|
||||
// Since we lack metadata, we import into a randomized category name instead
|
||||
let categoryName = genId();
|
||||
let containedEmojis = fs.readdirSync(outputPath);
|
||||
// Filter out accidental JSON files
|
||||
containedEmojis = containedEmojis.filter((emoji)=>!emoji.match(/\.(json)$/i));
|
||||
for (const emojiFilename of containedEmojis){
|
||||
// strip extension and get filename to use as name
|
||||
const name = path.basename(emojiFilename, path.extname(emojiFilename));
|
||||
const emojiPath = `${outputPath}/${emojiFilename}`;
|
||||
logger.info(`importing ${name}`);
|
||||
await Emojis.delete({
|
||||
name: name
|
||||
});
|
||||
const driveFile = await addFile({
|
||||
user: null,
|
||||
path: emojiPath,
|
||||
name: path.basename(emojiFilename),
|
||||
force: true
|
||||
});
|
||||
const file = fs.createReadStream(emojiPath);
|
||||
const size = await probeImageSize(file);
|
||||
file.destroy();
|
||||
logger.info(`emoji size: ${size.width}x${size.height}`);
|
||||
await Emojis.insert({
|
||||
id: genId(),
|
||||
updatedAt: new Date(),
|
||||
name: name,
|
||||
category: categoryName,
|
||||
host: null,
|
||||
aliases: [],
|
||||
originalUrl: driveFile.url,
|
||||
publicUrl: driveFile.webpublicUrl ?? driveFile.url,
|
||||
type: driveFile.webpublicType ?? driveFile.type,
|
||||
license: null,
|
||||
glyph: driveFile.type === "image/svg+xml",
|
||||
width: size.width || null,
|
||||
height: size.height || null
|
||||
}).then((x)=>Emojis.findOneByOrFail(x.identifiers[0]));
|
||||
}
|
||||
}
|
||||
await db.queryResultCache.remove([
|
||||
"meta_emojis"
|
||||
]);
|
||||
cleanup();
|
||||
logger.succ("Imported");
|
||||
resolve(undefined);
|
||||
});
|
||||
});
|
||||
return "Success";
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
import { IsNull } from "typeorm";
|
||||
import follow from "../../../services/following/create.js";
|
||||
import * as Acct from "../../../misc/acct.js";
|
||||
import { resolveUser } from "../../../remote/resolve-user.js";
|
||||
import { downloadTextFile } from "../../../misc/download-text-file.js";
|
||||
import { isSelfHost, toPuny } from "../../../misc/convert-host.js";
|
||||
import { Users, DriveFiles } from "../../../models/index.js";
|
||||
import { queueLogger } from "../../logger.js";
|
||||
import { cache as heuristic } from "../../../server/api/common/generate-following-query.js";
|
||||
const logger = queueLogger.createSubLogger("import-following");
|
||||
export async function importFollowing(job) {
|
||||
logger.info(`Importing following of ${job.data.user.id} ...`);
|
||||
const user = await Users.findOneBy({
|
||||
id: job.data.user.id
|
||||
});
|
||||
if (user == null) {
|
||||
return "skip: User not found";
|
||||
}
|
||||
const file = await DriveFiles.findOneBy({
|
||||
id: job.data.fileId
|
||||
});
|
||||
if (file == null) {
|
||||
return "skip: File not found";
|
||||
}
|
||||
const csv = await downloadTextFile(file.url);
|
||||
let linenum = 0;
|
||||
if (file.type.endsWith("json")) {
|
||||
for (const acct of JSON.parse(csv)){
|
||||
try {
|
||||
const { username, host } = Acct.parse(acct);
|
||||
let target = isSelfHost(host) ? await Users.findOneBy({
|
||||
host: IsNull(),
|
||||
usernameLower: username.toLowerCase()
|
||||
}) : await Users.findOneBy({
|
||||
host: toPuny(host),
|
||||
usernameLower: username.toLowerCase()
|
||||
});
|
||||
if (host == null && target == null) continue;
|
||||
if (target == null) {
|
||||
target = await resolveUser(username, host);
|
||||
}
|
||||
if (target == null) {
|
||||
throw new Error(`cannot resolve user: @${username}@${host}`);
|
||||
}
|
||||
// skip myself
|
||||
if (target.id === job.data.user.id) continue;
|
||||
logger.info(`Follow[${linenum}] ${target.id} ...`);
|
||||
follow(user, target);
|
||||
} catch (e) {
|
||||
logger.warn(`Error in line:${linenum} ${e}`);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for (const line of csv.trim().split("\n")){
|
||||
linenum++;
|
||||
try {
|
||||
const acct = line.split(",")[0].trim();
|
||||
const { username, host } = Acct.parse(acct);
|
||||
let target = isSelfHost(host) ? await Users.findOneBy({
|
||||
host: IsNull(),
|
||||
usernameLower: username.toLowerCase()
|
||||
}) : await Users.findOneBy({
|
||||
host: toPuny(host),
|
||||
usernameLower: username.toLowerCase()
|
||||
});
|
||||
if (host == null && target == null) continue;
|
||||
if (target == null) {
|
||||
target = await resolveUser(username, host);
|
||||
}
|
||||
if (target == null) {
|
||||
throw new Error(`cannot resolve user: @${username}@${host}`);
|
||||
}
|
||||
// skip myself
|
||||
if (target.id === job.data.user.id) continue;
|
||||
logger.info(`Follow[${linenum}] ${target.id} ...`);
|
||||
follow(user, target);
|
||||
} catch (e) {
|
||||
logger.warn(`Error in line:${linenum} ${e}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
await heuristic.delete(user.id);
|
||||
logger.succ("Imported");
|
||||
return "Success";
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
import { queueLogger } from "../../logger.js";
|
||||
import * as Acct from "../../../misc/acct.js";
|
||||
import { resolveUser } from "../../../remote/resolve-user.js";
|
||||
import { downloadTextFile } from "../../../misc/download-text-file.js";
|
||||
import { isSelfHost, toPuny } from "../../../misc/convert-host.js";
|
||||
import { Users, DriveFiles, Mutings } from "../../../models/index.js";
|
||||
import { genId } from "../../../misc/gen-id.js";
|
||||
import { IsNull } from "typeorm";
|
||||
const logger = queueLogger.createSubLogger("import-muting");
|
||||
export async function importMuting(job) {
|
||||
logger.info(`Importing muting of ${job.data.user.id} ...`);
|
||||
const user = await Users.findOneBy({
|
||||
id: job.data.user.id
|
||||
});
|
||||
if (user == null) {
|
||||
return "skip: User not found";
|
||||
}
|
||||
const file = await DriveFiles.findOneBy({
|
||||
id: job.data.fileId
|
||||
});
|
||||
if (file == null) {
|
||||
return "skip: File not found";
|
||||
}
|
||||
const csv = await downloadTextFile(file.url);
|
||||
let linenum = 0;
|
||||
for (const line of csv.trim().split("\n")){
|
||||
linenum++;
|
||||
try {
|
||||
const acct = line.split(",")[0].trim();
|
||||
const { username, host } = Acct.parse(acct);
|
||||
let target = isSelfHost(host) ? await Users.findOneBy({
|
||||
host: IsNull(),
|
||||
usernameLower: username.toLowerCase()
|
||||
}) : await Users.findOneBy({
|
||||
host: toPuny(host),
|
||||
usernameLower: username.toLowerCase()
|
||||
});
|
||||
if (host == null && target == null) continue;
|
||||
if (target == null) {
|
||||
target = await resolveUser(username, host);
|
||||
}
|
||||
if (target == null) {
|
||||
throw new Error(`cannot resolve user: @${username}@${host}`);
|
||||
}
|
||||
// skip myself
|
||||
if (target.id === job.data.user.id) continue;
|
||||
logger.info(`Mute[${linenum}] ${target.id} ...`);
|
||||
await mute(user, target);
|
||||
} catch (e) {
|
||||
logger.warn(`Error in line:${linenum} ${e}`);
|
||||
}
|
||||
}
|
||||
logger.succ("Imported");
|
||||
return "Success";
|
||||
}
|
||||
async function mute(user, target) {
|
||||
await Mutings.insert({
|
||||
id: genId(),
|
||||
createdAt: new Date(),
|
||||
muterId: user.id,
|
||||
muteeId: target.id
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
import { queueLogger } from "../../logger.js";
|
||||
import * as Acct from "../../../misc/acct.js";
|
||||
import { resolveUser } from "../../../remote/resolve-user.js";
|
||||
import { pushUserToUserList } from "../../../services/user-list/push.js";
|
||||
import { downloadTextFile } from "../../../misc/download-text-file.js";
|
||||
import { isSelfHost, toPuny } from "../../../misc/convert-host.js";
|
||||
import { DriveFiles, Users, UserLists, UserListJoinings, Blockings, Followings } from "../../../models/index.js";
|
||||
import { genId } from "../../../misc/gen-id.js";
|
||||
import { IsNull } from "typeorm";
|
||||
const logger = queueLogger.createSubLogger("import-user-lists");
|
||||
export async function importUserLists(job) {
|
||||
logger.info(`Importing user lists of ${job.data.user.id} ...`);
|
||||
const user = await Users.findOneBy({
|
||||
id: job.data.user.id
|
||||
});
|
||||
if (user == null) {
|
||||
return "skip: User not found";
|
||||
}
|
||||
const file = await DriveFiles.findOneBy({
|
||||
id: job.data.fileId
|
||||
});
|
||||
if (file == null) {
|
||||
return "skip: File not found";
|
||||
}
|
||||
const csv = await downloadTextFile(file.url);
|
||||
let linenum = 0;
|
||||
for (const line of csv.trim().split("\n")){
|
||||
linenum++;
|
||||
try {
|
||||
const listName = line.split(",")[0].trim();
|
||||
const { username, host } = Acct.parse(line.split(",")[1].trim());
|
||||
let list = await UserLists.findOneBy({
|
||||
userId: user.id,
|
||||
name: listName
|
||||
});
|
||||
if (list == null) {
|
||||
list = await UserLists.insert({
|
||||
id: genId(),
|
||||
createdAt: new Date(),
|
||||
userId: user.id,
|
||||
name: listName
|
||||
}).then((x)=>UserLists.findOneByOrFail(x.identifiers[0]));
|
||||
}
|
||||
let target = isSelfHost(host) ? await Users.findOneBy({
|
||||
host: IsNull(),
|
||||
usernameLower: username.toLowerCase()
|
||||
}) : await Users.findOneBy({
|
||||
host: toPuny(host),
|
||||
usernameLower: username.toLowerCase()
|
||||
});
|
||||
if (target == null) {
|
||||
target = await resolveUser(username, host);
|
||||
}
|
||||
const isBlocked = await Blockings.exist({
|
||||
where: {
|
||||
blockerId: target.id,
|
||||
blockeeId: user.id
|
||||
}
|
||||
});
|
||||
const isFollowed = await Followings.exist({
|
||||
where: {
|
||||
followerId: user.id,
|
||||
followeeId: target.id
|
||||
}
|
||||
});
|
||||
if (isBlocked || !isFollowed) continue;
|
||||
if (await UserListJoinings.findOneBy({
|
||||
userListId: list.id,
|
||||
userId: target.id
|
||||
}) != null) continue;
|
||||
pushUserToUserList(target, list);
|
||||
} catch (e) {
|
||||
logger.warn(`Error in line:${linenum} ${e}`);
|
||||
}
|
||||
}
|
||||
logger.succ("Imported");
|
||||
return "Success";
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
import { deleteAccount } from "./delete-account.js";
|
||||
import { deleteDriveFiles } from "./delete-drive-files.js";
|
||||
import { exportBlocking } from "./export-blocking.js";
|
||||
import { exportCustomEmojis } from "./export-custom-emojis.js";
|
||||
import { exportFollowing } from "./export-following.js";
|
||||
import { exportMute } from "./export-mute.js";
|
||||
import { exportNotes } from "./export-notes.js";
|
||||
import { exportUserLists } from "./export-user-lists.js";
|
||||
import { importBlocking } from "./import-blocking.js";
|
||||
import { importCustomEmojis } from "./import-custom-emojis.js";
|
||||
import { importFollowing } from "./import-following.js";
|
||||
import { importMuting } from "./import-muting.js";
|
||||
import { importUserLists } from "./import-user-lists.js";
|
||||
import { createQueue, defaultJobOpts } from "../index.js";
|
||||
import config from "../../../config/index.js";
|
||||
const processors = {
|
||||
deleteAccount,
|
||||
deleteDriveFiles,
|
||||
exportBlocking,
|
||||
exportCustomEmojis,
|
||||
exportFollowing,
|
||||
exportMute,
|
||||
exportNotes,
|
||||
exportUserLists,
|
||||
importBlocking,
|
||||
importCustomEmojis,
|
||||
importFollowing,
|
||||
importMuting,
|
||||
importUserLists
|
||||
};
|
||||
async function process(job) {
|
||||
const processor = processors[job.name];
|
||||
if (processor === undefined) return "skip: unknown job name";
|
||||
return await processor(job);
|
||||
}
|
||||
export const [dbQueue, dbInit] = createQueue("db", process, {
|
||||
limitPerSec: 256,
|
||||
concurrency: 16
|
||||
});
|
||||
export function createDeleteDriveFilesJob(user) {
|
||||
return dbQueue.add("deleteDriveFiles", {
|
||||
user: user
|
||||
}, defaultJobOpts);
|
||||
}
|
||||
export function createExportCustomEmojisJob(user) {
|
||||
return dbQueue.add("exportCustomEmojis", {
|
||||
user: user
|
||||
}, defaultJobOpts);
|
||||
}
|
||||
export function createExportNotesJob(user) {
|
||||
return dbQueue.add("exportNotes", {
|
||||
user: user
|
||||
}, defaultJobOpts);
|
||||
}
|
||||
export function createExportFollowingJob(user, excludeMuting = false, excludeInactive = false) {
|
||||
return dbQueue.add("exportFollowing", {
|
||||
user: user,
|
||||
excludeMuting,
|
||||
excludeInactive
|
||||
}, defaultJobOpts);
|
||||
}
|
||||
export function createExportMuteJob(user) {
|
||||
return dbQueue.add("exportMute", {
|
||||
user: user
|
||||
}, defaultJobOpts);
|
||||
}
|
||||
export function createExportBlockingJob(user) {
|
||||
return dbQueue.add("exportBlocking", {
|
||||
user: user
|
||||
}, defaultJobOpts);
|
||||
}
|
||||
export function createExportUserListsJob(user) {
|
||||
return dbQueue.add("exportUserLists", {
|
||||
user: user
|
||||
}, defaultJobOpts);
|
||||
}
|
||||
export function createImportFollowingJob(user, fileId) {
|
||||
return dbQueue.add("importFollowing", {
|
||||
user: user,
|
||||
fileId: fileId
|
||||
}, defaultJobOpts);
|
||||
}
|
||||
export function createImportMastoPostJob(user, post, signatureCheck) {
|
||||
return dbQueue.add("importMastoPost", {
|
||||
user: user,
|
||||
post: post,
|
||||
signatureCheck: signatureCheck
|
||||
}, {
|
||||
attempts: config.inboxJobMaxAttempts || 8,
|
||||
...defaultJobOpts
|
||||
});
|
||||
}
|
||||
export function createImportCkPostJob(user, post, signatureCheck) {
|
||||
return dbQueue.add("importCkPost", {
|
||||
user: user,
|
||||
post: post,
|
||||
signatureCheck: signatureCheck
|
||||
}, defaultJobOpts);
|
||||
}
|
||||
export function createImportMutingJob(user, fileId) {
|
||||
return dbQueue.add("importMuting", {
|
||||
user: user,
|
||||
fileId: fileId
|
||||
}, defaultJobOpts);
|
||||
}
|
||||
export function createImportBlockingJob(user, fileId) {
|
||||
return dbQueue.add("importBlocking", {
|
||||
user: user,
|
||||
fileId: fileId
|
||||
}, defaultJobOpts);
|
||||
}
|
||||
export function createImportUserListsJob(user, fileId) {
|
||||
return dbQueue.add("importUserLists", {
|
||||
user: user,
|
||||
fileId: fileId
|
||||
}, defaultJobOpts);
|
||||
}
|
||||
export function createImportCustomEmojisJob(user, fileId) {
|
||||
return dbQueue.add("importCustomEmojis", {
|
||||
user: user,
|
||||
fileId: fileId
|
||||
}, defaultJobOpts);
|
||||
}
|
||||
export function createDeleteAccountJob(user, opts = {}) {
|
||||
return dbQueue.add("deleteAccount", {
|
||||
user: user,
|
||||
soft: opts.soft
|
||||
}, defaultJobOpts);
|
||||
}
|
||||
Reference in New Issue
Block a user