Fixed 267U.pre2

packages/backend/built/services/add-note-to-antenna.js

@@ -0,0 +1,6 @@
+import { redisClient } from "../db/redis.js";
+import { publishAntennaStream } from "./stream.js";
+export async function addNoteToAntenna(antenna, note, _noteUser) {
+    redisClient.xadd(`antennaTimeline:${antenna.id}`, "MAXLEN", "~", "200", "*", "note", note.id);
+    publishAntennaStream(antenna.id, "note", note);
+}

packages/backend/built/services/blocking/create.js

@@ -0,0 +1,127 @@
+import { publishMainStream, publishUserEvent } from "../stream.js";
+import { renderActivity } from "../../remote/activitypub/renderer/index.js";
+import renderFollow from "../../remote/activitypub/renderer/follow.js";
+import renderUndo from "../../remote/activitypub/renderer/undo.js";
+import { renderBlock } from "../../remote/activitypub/renderer/block.js";
+import { deliver } from "../../queue/index.js";
+import renderReject from "../../remote/activitypub/renderer/reject.js";
+import { Blockings, Users, FollowRequests, Followings, UserListJoinings, UserLists } from "../../models/index.js";
+import { perUserFollowingChart } from "../chart/index.js";
+import { genId } from "../../misc/gen-id.js";
+import { getActiveWebhooks } from "../../misc/webhook-cache.js";
+import { webhookDeliver } from "../../queue/index.js";
+export default async function(blocker, blockee, groupId = null) {
+    await Promise.all([
+        cancelRequest(blocker, blockee),
+        cancelRequest(blockee, blocker),
+        unFollow(blocker, blockee),
+        unFollow(blockee, blocker),
+        removeFromList(blockee, blocker)
+    ]);
+    const blocking = {
+        id: genId(),
+        createdAt: new Date(),
+        blocker,
+        blockerId: blocker.id,
+        blockee,
+        blockeeId: blockee.id,
+        groupId
+    };
+    await Blockings.insert(blocking);
+    if (Users.isLocalUser(blocker) && Users.isRemoteUser(blockee)) {
+        const content = renderActivity(renderBlock(blocking));
+        deliver(blocker, content, blockee.inbox);
+    }
+}
+async function cancelRequest(follower, followee) {
+    const request = await FollowRequests.findOneBy({
+        followeeId: followee.id,
+        followerId: follower.id
+    });
+    if (request == null) {
+        return;
+    }
+    await FollowRequests.delete({
+        followeeId: followee.id,
+        followerId: follower.id
+    });
+    if (Users.isLocalUser(followee)) {
+        Users.pack(followee, followee, {
+            detail: true
+        }).then((packed)=>publishMainStream(followee.id, "meUpdated", packed));
+    }
+    if (Users.isLocalUser(follower)) {
+        Users.pack(followee, follower, {
+            detail: true
+        }).then(async (packed)=>{
+            publishUserEvent(follower.id, "unfollow", packed);
+            publishMainStream(follower.id, "unfollow", packed);
+            const webhooks = (await getActiveWebhooks()).filter((x)=>x.userId === follower.id && x.on.includes("unfollow"));
+            for (const webhook of webhooks){
+                webhookDeliver(webhook, "unfollow", {
+                    user: packed
+                });
+            }
+        });
+    }
+    // リモートにフォローリクエストをしていたらUndoFollow送信
+    if (Users.isLocalUser(follower) && Users.isRemoteUser(followee)) {
+        const content = renderActivity(renderUndo(renderFollow(follower, followee), follower));
+        deliver(follower, content, followee.inbox);
+    }
+    // リモートからフォローリクエストを受けていたらReject送信
+    if (Users.isRemoteUser(follower) && Users.isLocalUser(followee)) {
+        const content = renderActivity(renderReject(renderFollow(follower, followee, request.requestId), followee));
+        deliver(followee, content, follower.inbox);
+    }
+}
+async function unFollow(follower, followee) {
+    const following = await Followings.findOneBy({
+        followerId: follower.id,
+        followeeId: followee.id
+    });
+    if (following == null) {
+        return;
+    }
+    await Promise.all([
+        Followings.delete(following.id),
+        Users.decrement({
+            id: follower.id
+        }, "followingCount", 1),
+        Users.decrement({
+            id: followee.id
+        }, "followersCount", 1),
+        perUserFollowingChart.update(follower, followee, false)
+    ]);
+    // Publish unfollow event
+    if (Users.isLocalUser(follower)) {
+        Users.pack(followee, follower, {
+            detail: true
+        }).then(async (packed)=>{
+            publishUserEvent(follower.id, "unfollow", packed);
+            publishMainStream(follower.id, "unfollow", packed);
+            const webhooks = (await getActiveWebhooks()).filter((x)=>x.userId === follower.id && x.on.includes("unfollow"));
+            for (const webhook of webhooks){
+                webhookDeliver(webhook, "unfollow", {
+                    user: packed
+                });
+            }
+        });
+    }
+    // リモートにフォローをしていたらUndoFollow送信
+    if (Users.isLocalUser(follower) && Users.isRemoteUser(followee)) {
+        const content = renderActivity(renderUndo(renderFollow(follower, followee), follower));
+        deliver(follower, content, followee.inbox);
+    }
+}
+async function removeFromList(listOwner, user) {
+    const userLists = await UserLists.findBy({
+        userId: listOwner.id
+    });
+    for (const userList of userLists){
+        await UserListJoinings.delete({
+            userListId: userList.id,
+            userId: user.id
+        });
+    }
+}

packages/backend/built/services/blocking/delete.js

@@ -0,0 +1,28 @@
+import { renderActivity } from "../../remote/activitypub/renderer/index.js";
+import { renderBlock } from "../../remote/activitypub/renderer/block.js";
+import renderUndo from "../../remote/activitypub/renderer/undo.js";
+import { deliver } from "../../queue/index.js";
+import Logger from "../logger.js";
+import { Blockings, Users } from "../../models/index.js";
+const logger = new Logger("blocking/delete");
+export default async function(blocker, blockee, groupId = null) {
+    const blocking = await Blockings.findOneBy({
+        blockerId: blocker.id,
+        blockeeId: blockee.id,
+        groupId
+    });
+    if (blocking == null) {
+        logger.warn("ブロック解除がリクエストされましたがブロックしていませんでした");
+        return;
+    }
+    // Since we already have the blocker and blockee, we do not need to fetch
+    // them in the query above and can just manually insert them here.
+    blocking.blocker = blocker;
+    blocking.blockee = blockee;
+    Blockings.delete(blocking.id);
+    // deliver if remote bloking
+    if (Users.isLocalUser(blocker) && Users.isRemoteUser(blockee)) {
+        const content = renderActivity(renderUndo(renderBlock(blocking), blocker));
+        deliver(blocker, content, blockee.inbox);
+    }
+}

packages/backend/built/services/chart/charts/active-users.js

@@ -0,0 +1,50 @@
+import Chart from "../core.js";
+import { name, schema } from "./entities/active-users.js";
+const week = 1000 * 60 * 60 * 24 * 7;
+const month = 1000 * 60 * 60 * 24 * 30;
+const year = 1000 * 60 * 60 * 24 * 365;
+/**
+ * アクティブユーザーに関するチャート
+ */ export default class ActiveUsersChart extends Chart {
+    constructor(){
+        super(name, schema);
+    }
+    async tickMajor() {
+        return {};
+    }
+    async tickMinor() {
+        return {};
+    }
+    read(user) {
+        this.commit({
+            read: [
+                user.id
+            ],
+            registeredWithinWeek: Date.now() - user.createdAt.getTime() < week ? [
+                user.id
+            ] : [],
+            registeredWithinMonth: Date.now() - user.createdAt.getTime() < month ? [
+                user.id
+            ] : [],
+            registeredWithinYear: Date.now() - user.createdAt.getTime() < year ? [
+                user.id
+            ] : [],
+            registeredOutsideWeek: Date.now() - user.createdAt.getTime() > week ? [
+                user.id
+            ] : [],
+            registeredOutsideMonth: Date.now() - user.createdAt.getTime() > month ? [
+                user.id
+            ] : [],
+            registeredOutsideYear: Date.now() - user.createdAt.getTime() > year ? [
+                user.id
+            ] : []
+        });
+    }
+    async write(user) {
+        await this.commit({
+            write: [
+                user.id
+            ]
+        });
+    }
+}

packages/backend/built/services/chart/charts/ap-request.js

@@ -0,0 +1,30 @@
+import Chart from "../core.js";
+import { name, schema } from "./entities/ap-request.js";
+/**
+ * Chart about ActivityPub requests
+ */ export default class ApRequestChart extends Chart {
+    constructor(){
+        super(name, schema);
+    }
+    async tickMajor() {
+        return {};
+    }
+    async tickMinor() {
+        return {};
+    }
+    async deliverSucc() {
+        await this.commit({
+            deliverSucceeded: 1
+        });
+    }
+    async deliverFail() {
+        await this.commit({
+            deliverFailed: 1
+        });
+    }
+    async inbox() {
+        await this.commit({
+            inboxReceived: 1
+        });
+    }
+}

packages/backend/built/services/chart/charts/drive.js

@@ -0,0 +1,29 @@
+import Chart from "../core.js";
+import { name, schema } from "./entities/drive.js";
+/**
+ * ドライブに関するチャート
+ */ export default class DriveChart extends Chart {
+    constructor(){
+        super(name, schema);
+    }
+    async tickMajor() {
+        return {};
+    }
+    async tickMinor() {
+        return {};
+    }
+    async update(file, isAdditional) {
+        const fileSizeKb = file.size / 1000;
+        await this.commit(file.userHost === null ? {
+            "local.incCount": isAdditional ? 1 : 0,
+            "local.incSize": isAdditional ? fileSizeKb : 0,
+            "local.decCount": isAdditional ? 0 : 1,
+            "local.decSize": isAdditional ? 0 : fileSizeKb
+        } : {
+            "remote.incCount": isAdditional ? 1 : 0,
+            "remote.incSize": isAdditional ? fileSizeKb : 0,
+            "remote.decCount": isAdditional ? 0 : 1,
+            "remote.decSize": isAdditional ? 0 : fileSizeKb
+        });
+    }
+}

packages/backend/built/services/chart/charts/entities/active-users.js

@@ -0,0 +1,44 @@
+import Chart from "../../core.js";
+export const name = "activeUsers";
+export const schema = {
+    readWrite: {
+        intersection: [
+            "read",
+            "write"
+        ],
+        range: "small"
+    },
+    read: {
+        uniqueIncrement: true,
+        range: "small"
+    },
+    write: {
+        uniqueIncrement: true,
+        range: "small"
+    },
+    registeredWithinWeek: {
+        uniqueIncrement: true,
+        range: "small"
+    },
+    registeredWithinMonth: {
+        uniqueIncrement: true,
+        range: "small"
+    },
+    registeredWithinYear: {
+        uniqueIncrement: true,
+        range: "small"
+    },
+    registeredOutsideWeek: {
+        uniqueIncrement: true,
+        range: "small"
+    },
+    registeredOutsideMonth: {
+        uniqueIncrement: true,
+        range: "small"
+    },
+    registeredOutsideYear: {
+        uniqueIncrement: true,
+        range: "small"
+    }
+};
+export const entity = Chart.schemaToEntity(name, schema);

packages/backend/built/services/chart/charts/entities/ap-request.js

@@ -0,0 +1,8 @@
+import Chart from "../../core.js";
+export const name = "apRequest";
+export const schema = {
+    deliverFailed: {},
+    deliverSucceeded: {},
+    inboxReceived: {}
+};
+export const entity = Chart.schemaToEntity(name, schema);

packages/backend/built/services/chart/charts/entities/drive.js

@@ -0,0 +1,13 @@
+import Chart from "../../core.js";
+export const name = "drive";
+export const schema = {
+    "local.incCount": {},
+    "local.incSize": {},
+    "local.decCount": {},
+    "local.decSize": {},
+    "remote.incCount": {},
+    "remote.incSize": {},
+    "remote.decCount": {},
+    "remote.decSize": {}
+};
+export const entity = Chart.schemaToEntity(name, schema);

packages/backend/built/services/chart/charts/entities/federation.js

@@ -0,0 +1,37 @@
+import Chart from "../../core.js";
+export const name = "federation";
+export const schema = {
+    deliveredInstances: {
+        uniqueIncrement: true,
+        range: "small"
+    },
+    inboxInstances: {
+        uniqueIncrement: true,
+        range: "small"
+    },
+    stalled: {
+        uniqueIncrement: true,
+        range: "small"
+    },
+    sub: {
+        accumulate: true,
+        range: "small"
+    },
+    pub: {
+        accumulate: true,
+        range: "small"
+    },
+    pubsub: {
+        accumulate: true,
+        range: "small"
+    },
+    subActive: {
+        accumulate: true,
+        range: "small"
+    },
+    pubActive: {
+        accumulate: true,
+        range: "small"
+    }
+};
+export const entity = Chart.schemaToEntity(name, schema);

packages/backend/built/services/chart/charts/entities/hashtag.js

@@ -0,0 +1,11 @@
+import Chart from "../../core.js";
+export const name = "hashtag";
+export const schema = {
+    "local.users": {
+        uniqueIncrement: true
+    },
+    "remote.users": {
+        uniqueIncrement: true
+    }
+};
+export const entity = Chart.schemaToEntity(name, schema, true);

packages/backend/built/services/chart/charts/entities/instance.js

@@ -0,0 +1,57 @@
+import Chart from "../../core.js";
+export const name = "instance";
+export const schema = {
+    "requests.failed": {
+        range: "small"
+    },
+    "requests.succeeded": {
+        range: "small"
+    },
+    "requests.received": {
+        range: "small"
+    },
+    "notes.total": {
+        accumulate: true
+    },
+    "notes.inc": {},
+    "notes.dec": {},
+    "notes.diffs.normal": {},
+    "notes.diffs.reply": {},
+    "notes.diffs.renote": {},
+    "notes.diffs.withFile": {},
+    "users.total": {
+        accumulate: true
+    },
+    "users.inc": {
+        range: "small"
+    },
+    "users.dec": {
+        range: "small"
+    },
+    "following.total": {
+        accumulate: true
+    },
+    "following.inc": {
+        range: "small"
+    },
+    "following.dec": {
+        range: "small"
+    },
+    "followers.total": {
+        accumulate: true
+    },
+    "followers.inc": {
+        range: "small"
+    },
+    "followers.dec": {
+        range: "small"
+    },
+    "drive.totalFiles": {
+        accumulate: true
+    },
+    "drive.incFiles": {},
+    "drive.decFiles": {},
+    "drive.incUsage": {},
+    "drive.decUsage": {}
+};
+export const entity = Chart.schemaToEntity(name, schema, true);

packages/backend/built/services/chart/charts/entities/notes.js

@@ -0,0 +1,23 @@
+import Chart from "../../core.js";
+export const name = "notes";
+export const schema = {
+    "local.total": {
+        accumulate: true
+    },
+    "local.inc": {},
+    "local.dec": {},
+    "local.diffs.normal": {},
+    "local.diffs.reply": {},
+    "local.diffs.renote": {},
+    "local.diffs.withFile": {},
+    "remote.total": {
+        accumulate: true
+    },
+    "remote.inc": {},
+    "remote.dec": {},
+    "remote.diffs.normal": {},
+    "remote.diffs.reply": {},
+    "remote.diffs.renote": {},
+    "remote.diffs.withFile": {}
+};
+export const entity = Chart.schemaToEntity(name, schema);

packages/backend/built/services/chart/charts/entities/per-user-drive.js

@@ -0,0 +1,19 @@
+import Chart from "../../core.js";
+export const name = "perUserDrive";
+export const schema = {
+    totalCount: {
+        accumulate: true
+    },
+    totalSize: {
+        accumulate: true
+    },
+    incCount: {
+        range: "small"
+    },
+    incSize: {},
+    decCount: {
+        range: "small"
+    },
+    decSize: {}
+};
+export const entity = Chart.schemaToEntity(name, schema, true);

packages/backend/built/services/chart/charts/entities/per-user-following.js

@@ -0,0 +1,41 @@
+import Chart from "../../core.js";
+export const name = "perUserFollowing";
+export const schema = {
+    "local.followings.total": {
+        accumulate: true
+    },
+    "local.followings.inc": {
+        range: "small"
+    },
+    "local.followings.dec": {
+        range: "small"
+    },
+    "local.followers.total": {
+        accumulate: true
+    },
+    "local.followers.inc": {
+        range: "small"
+    },
+    "local.followers.dec": {
+        range: "small"
+    },
+    "remote.followings.total": {
+        accumulate: true
+    },
+    "remote.followings.inc": {
+        range: "small"
+    },
+    "remote.followings.dec": {
+        range: "small"
+    },
+    "remote.followers.total": {
+        accumulate: true
+    },
+    "remote.followers.inc": {
+        range: "small"
+    },
+    "remote.followers.dec": {
+        range: "small"
+    }
+};
+export const entity = Chart.schemaToEntity(name, schema, true);

packages/backend/built/services/chart/charts/entities/per-user-notes.js

@@ -0,0 +1,26 @@
+import Chart from "../../core.js";
+export const name = "perUserNotes";
+export const schema = {
+    total: {
+        accumulate: true
+    },
+    inc: {
+        range: "small"
+    },
+    dec: {
+        range: "small"
+    },
+    "diffs.normal": {
+        range: "small"
+    },
+    "diffs.reply": {
+        range: "small"
+    },
+    "diffs.renote": {
+        range: "small"
+    },
+    "diffs.withFile": {
+        range: "small"
+    }
+};
+export const entity = Chart.schemaToEntity(name, schema, true);

packages/backend/built/services/chart/charts/entities/per-user-reactions.js

@@ -0,0 +1,11 @@
+import Chart from "../../core.js";
+export const name = "perUserReaction";
+export const schema = {
+    "local.count": {
+        range: "small"
+    },
+    "remote.count": {
+        range: "small"
+    }
+};
+export const entity = Chart.schemaToEntity(name, schema, true);

packages/backend/built/services/chart/charts/entities/test-grouped.js

@@ -0,0 +1,10 @@
+import Chart from "../../core.js";
+export const name = "testGrouped";
+export const schema = {
+    "foo.total": {
+        accumulate: true
+    },
+    "foo.inc": {},
+    "foo.dec": {}
+};
+export const entity = Chart.schemaToEntity(name, schema, true);

packages/backend/built/services/chart/charts/entities/test-intersection.js

@@ -0,0 +1,17 @@
+import Chart from "../../core.js";
+export const name = "testIntersection";
+export const schema = {
+    a: {
+        uniqueIncrement: true
+    },
+    b: {
+        uniqueIncrement: true
+    },
+    aAndB: {
+        intersection: [
+            "a",
+            "b"
+        ]
+    }
+};
+export const entity = Chart.schemaToEntity(name, schema);

packages/backend/built/services/chart/charts/entities/test-unique.js

@@ -0,0 +1,8 @@
+import Chart from "../../core.js";
+export const name = "testUnique";
+export const schema = {
+    foo: {
+        uniqueIncrement: true
+    }
+};
+export const entity = Chart.schemaToEntity(name, schema);

packages/backend/built/services/chart/charts/entities/test.js

@@ -0,0 +1,10 @@
+import Chart from "../../core.js";
+export const name = "test";
+export const schema = {
+    "foo.total": {
+        accumulate: true
+    },
+    "foo.inc": {},
+    "foo.dec": {}
+};
+export const entity = Chart.schemaToEntity(name, schema);

packages/backend/built/services/chart/charts/entities/users.js

@@ -0,0 +1,23 @@
+import Chart from "../../core.js";
+export const name = "users";
+export const schema = {
+    "local.total": {
+        accumulate: true
+    },
+    "local.inc": {
+        range: "small"
+    },
+    "local.dec": {
+        range: "small"
+    },
+    "remote.total": {
+        accumulate: true
+    },
+    "remote.inc": {
+        range: "small"
+    },
+    "remote.dec": {
+        range: "small"
+    }
+};
+export const entity = Chart.schemaToEntity(name, schema);

packages/backend/built/services/chart/charts/federation.js

@@ -0,0 +1,67 @@
+import Chart from "../core.js";
+import { Followings, Instances } from "../../../models/index.js";
+import { name, schema } from "./entities/federation.js";
+import { fetchMeta } from "../../../misc/fetch-meta.js";
+/**
+ * フェデレーションに関するチャート
+ */ export default class FederationChart extends Chart {
+    constructor(){
+        super(name, schema);
+    }
+    async tickMajor() {
+        return {};
+    }
+    async tickMinor() {
+        const meta = await fetchMeta();
+        const suspendedInstancesQuery = Instances.createQueryBuilder("instance").select("instance.host").where("instance.isSuspended = true");
+        const pubsubSubQuery = Followings.createQueryBuilder("f").select("f.followerHost").where("f.followerHost IS NOT NULL");
+        const subInstancesQuery = Followings.createQueryBuilder("f").select("f.followeeHost").where("f.followeeHost IS NOT NULL");
+        const pubInstancesQuery = Followings.createQueryBuilder("f").select("f.followerHost").where("f.followerHost IS NOT NULL");
+        const [sub, pub, pubsub, subActive, pubActive] = await Promise.all([
+            Followings.createQueryBuilder("following").select("COUNT(DISTINCT following.followeeHost)").where("following.followeeHost IS NOT NULL").andWhere(meta.blockedHosts.length === 0 ? "1=1" : "following.followeeHost NOT IN (:...blocked)", {
+                blocked: meta.blockedHosts
+            }).andWhere(`following.followeeHost NOT IN (${suspendedInstancesQuery.getQuery()})`).getRawOne().then((x)=>parseInt(x.count, 10)),
+            Followings.createQueryBuilder("following").select("COUNT(DISTINCT following.followerHost)").where("following.followerHost IS NOT NULL").andWhere(meta.blockedHosts.length === 0 ? "1=1" : "following.followerHost NOT IN (:...blocked)", {
+                blocked: meta.blockedHosts
+            }).andWhere(`following.followerHost NOT IN (${suspendedInstancesQuery.getQuery()})`).getRawOne().then((x)=>parseInt(x.count, 10)),
+            Followings.createQueryBuilder("following").select("COUNT(DISTINCT following.followeeHost)").where("following.followeeHost IS NOT NULL").andWhere(meta.blockedHosts.length === 0 ? "1=1" : "following.followeeHost NOT IN (:...blocked)", {
+                blocked: meta.blockedHosts
+            }).andWhere(`following.followeeHost NOT IN (${suspendedInstancesQuery.getQuery()})`).andWhere(`following.followeeHost IN (${pubsubSubQuery.getQuery()})`).setParameters(pubsubSubQuery.getParameters()).getRawOne().then((x)=>parseInt(x.count, 10)),
+            Instances.createQueryBuilder("instance").select("COUNT(instance.id)").where(`instance.host IN (${subInstancesQuery.getQuery()})`).andWhere(meta.blockedHosts.length === 0 ? "1=1" : "instance.host NOT IN (:...blocked)", {
+                blocked: meta.blockedHosts
+            }).andWhere("instance.isSuspended = false").andWhere("instance.lastCommunicatedAt > :gt", {
+                gt: new Date(Date.now() - 1000 * 60 * 60 * 24 * 30)
+            }).getRawOne().then((x)=>parseInt(x.count, 10)),
+            Instances.createQueryBuilder("instance").select("COUNT(instance.id)").where(`instance.host IN (${pubInstancesQuery.getQuery()})`).andWhere(meta.blockedHosts.length === 0 ? "1=1" : "instance.host NOT IN (:...blocked)", {
+                blocked: meta.blockedHosts
+            }).andWhere("instance.isSuspended = false").andWhere("instance.lastCommunicatedAt > :gt", {
+                gt: new Date(Date.now() - 1000 * 60 * 60 * 24 * 30)
+            }).getRawOne().then((x)=>parseInt(x.count, 10))
+        ]);
+        return {
+            sub: sub,
+            pub: pub,
+            pubsub: pubsub,
+            subActive: subActive,
+            pubActive: pubActive
+        };
+    }
+    async deliverd(host, succeeded) {
+        await this.commit(succeeded ? {
+            deliveredInstances: [
+                host
+            ]
+        } : {
+            stalled: [
+                host
+            ]
+        });
+    }
+    async inbox(host) {
+        await this.commit({
+            inboxInstances: [
+                host
+            ]
+        });
+    }
+}

packages/backend/built/services/chart/charts/hashtag.js

@@ -0,0 +1,26 @@
+import Chart from "../core.js";
+import { Users } from "../../../models/index.js";
+import { name, schema } from "./entities/hashtag.js";
+/**
+ * ハッシュタグに関するチャート
+ */ export default class HashtagChart extends Chart {
+    constructor(){
+        super(name, schema, true);
+    }
+    async tickMajor() {
+        return {};
+    }
+    async tickMinor() {
+        return {};
+    }
+    async update(hashtag, user) {
+        await this.commit({
+            "local.users": Users.isLocalUser(user) ? [
+                user.id
+            ] : [],
+            "remote.users": Users.isLocalUser(user) ? [] : [
+                user.id
+            ]
+        }, hashtag);
+    }
+}

packages/backend/built/services/chart/charts/instance.js

@@ -0,0 +1,92 @@
+import Chart from "../core.js";
+import { DriveFiles, Followings, Users, Notes } from "../../../models/index.js";
+import { toPuny } from "../../../misc/convert-host.js";
+import { name, schema } from "./entities/instance.js";
+/**
+ * インスタンスごとのチャート
+ */ export default class InstanceChart extends Chart {
+    constructor(){
+        super(name, schema, true);
+    }
+    async tickMajor(group) {
+        const [notesCount, usersCount, followingCount, followersCount, driveFiles] = await Promise.all([
+            Notes.countBy({
+                userHost: group
+            }),
+            Users.countBy({
+                host: group
+            }),
+            Followings.countBy({
+                followerHost: group
+            }),
+            Followings.countBy({
+                followeeHost: group
+            }),
+            DriveFiles.countBy({
+                userHost: group
+            })
+        ]);
+        return {
+            "notes.total": notesCount,
+            "users.total": usersCount,
+            "following.total": followingCount,
+            "followers.total": followersCount,
+            "drive.totalFiles": driveFiles
+        };
+    }
+    async tickMinor() {
+        return {};
+    }
+    async requestReceived(host) {
+        await this.commit({
+            "requests.received": 1
+        }, toPuny(host));
+    }
+    async requestSent(host, isSucceeded) {
+        await this.commit({
+            "requests.succeeded": isSucceeded ? 1 : 0,
+            "requests.failed": isSucceeded ? 0 : 1
+        }, toPuny(host));
+    }
+    async newUser(host) {
+        await this.commit({
+            "users.total": 1,
+            "users.inc": 1
+        }, toPuny(host));
+    }
+    async updateNote(host, note, isAdditional) {
+        await this.commit({
+            "notes.total": isAdditional ? 1 : -1,
+            "notes.inc": isAdditional ? 1 : 0,
+            "notes.dec": isAdditional ? 0 : 1,
+            "notes.diffs.normal": note.replyId == null && note.renoteId == null ? isAdditional ? 1 : -1 : 0,
+            "notes.diffs.renote": note.renoteId != null ? isAdditional ? 1 : -1 : 0,
+            "notes.diffs.reply": note.replyId != null ? isAdditional ? 1 : -1 : 0,
+            "notes.diffs.withFile": note.fileIds.length > 0 ? isAdditional ? 1 : -1 : 0
+        }, toPuny(host));
+    }
+    async updateFollowing(host, isAdditional) {
+        await this.commit({
+            "following.total": isAdditional ? 1 : -1,
+            "following.inc": isAdditional ? 1 : 0,
+            "following.dec": isAdditional ? 0 : 1
+        }, toPuny(host));
+    }
+    async updateFollowers(host, isAdditional) {
+        await this.commit({
+            "followers.total": isAdditional ? 1 : -1,
+            "followers.inc": isAdditional ? 1 : 0,
+            "followers.dec": isAdditional ? 0 : 1
+        }, toPuny(host));
+    }
+    async updateDrive(file, isAdditional) {
+        const fileSizeKb = file.size / 1000;
+        await this.commit({
+            "drive.totalFiles": isAdditional ? 1 : -1,
+            "drive.incFiles": isAdditional ? 1 : 0,
+            "drive.incUsage": isAdditional ? fileSizeKb : 0,
+            "drive.decFiles": isAdditional ? 1 : 0,
+            "drive.decUsage": isAdditional ? fileSizeKb : 0
+        }, file.userHost);
+    }
+}

packages/backend/built/services/chart/charts/notes.js

@@ -0,0 +1,40 @@
+import Chart from "../core.js";
+import { Notes } from "../../../models/index.js";
+import { Not, IsNull } from "typeorm";
+import { name, schema } from "./entities/notes.js";
+/**
+ * ノートに関するチャート
+ */ export default class NotesChart extends Chart {
+    constructor(){
+        super(name, schema);
+    }
+    async tickMajor() {
+        const [localCount, remoteCount] = await Promise.all([
+            Notes.countBy({
+                userHost: IsNull()
+            }),
+            Notes.countBy({
+                userHost: Not(IsNull())
+            })
+        ]);
+        return {
+            "local.total": localCount,
+            "remote.total": remoteCount
+        };
+    }
+    async tickMinor() {
+        return {};
+    }
+    async update(note, isAdditional) {
+        const prefix = note.userHost === null ? "local" : "remote";
+        await this.commit({
+            [`${prefix}.total`]: isAdditional ? 1 : -1,
+            [`${prefix}.inc`]: isAdditional ? 1 : 0,
+            [`${prefix}.dec`]: isAdditional ? 0 : 1,
+            [`${prefix}.diffs.normal`]: note.replyId == null && note.renoteId == null ? isAdditional ? 1 : -1 : 0,
+            [`${prefix}.diffs.renote`]: note.renoteId != null ? isAdditional ? 1 : -1 : 0,
+            [`${prefix}.diffs.reply`]: note.replyId != null ? isAdditional ? 1 : -1 : 0,
+            [`${prefix}.diffs.withFile`]: note.fileIds.length > 0 ? isAdditional ? 1 : -1 : 0
+        });
+    }
+}

packages/backend/built/services/chart/charts/per-user-drive.js

@@ -0,0 +1,36 @@
+import Chart from "../core.js";
+import { DriveFiles } from "../../../models/index.js";
+import { name, schema } from "./entities/per-user-drive.js";
+/**
+ * ユーザーごとのドライブに関するチャート
+ */ export default class PerUserDriveChart extends Chart {
+    constructor(){
+        super(name, schema, true);
+    }
+    async tickMajor(group) {
+        const [count, size] = await Promise.all([
+            DriveFiles.countBy({
+                userId: group
+            }),
+            DriveFiles.calcDriveUsageOf(group)
+        ]);
+        return {
+            totalCount: count,
+            totalSize: size
+        };
+    }
+    async tickMinor() {
+        return {};
+    }
+    async update(file, isAdditional) {
+        const fileSizeKb = file.size / 1000;
+        await this.commit({
+            totalCount: isAdditional ? 1 : -1,
+            totalSize: isAdditional ? fileSizeKb : -fileSizeKb,
+            incCount: isAdditional ? 1 : 0,
+            incSize: isAdditional ? fileSizeKb : 0,
+            decCount: isAdditional ? 0 : 1,
+            decSize: isAdditional ? 0 : fileSizeKb
+        }, file.userId);
+    }
+}

packages/backend/built/services/chart/charts/per-user-following.js

@@ -0,0 +1,54 @@
+import Chart from "../core.js";
+import { Followings, Users } from "../../../models/index.js";
+import { Not, IsNull } from "typeorm";
+import { name, schema } from "./entities/per-user-following.js";
+/**
+ * ユーザーごとのフォローに関するチャート
+ */ export default class PerUserFollowingChart extends Chart {
+    constructor(){
+        super(name, schema, true);
+    }
+    async tickMajor(group) {
+        const [localFollowingsCount, localFollowersCount, remoteFollowingsCount, remoteFollowersCount] = await Promise.all([
+            Followings.countBy({
+                followerId: group,
+                followeeHost: IsNull()
+            }),
+            Followings.countBy({
+                followeeId: group,
+                followerHost: IsNull()
+            }),
+            Followings.countBy({
+                followerId: group,
+                followeeHost: Not(IsNull())
+            }),
+            Followings.countBy({
+                followeeId: group,
+                followerHost: Not(IsNull())
+            })
+        ]);
+        return {
+            "local.followings.total": localFollowingsCount,
+            "local.followers.total": localFollowersCount,
+            "remote.followings.total": remoteFollowingsCount,
+            "remote.followers.total": remoteFollowersCount
+        };
+    }
+    async tickMinor() {
+        return {};
+    }
+    async update(follower, followee, isFollow) {
+        const prefixFollower = Users.isLocalUser(follower) ? "local" : "remote";
+        const prefixFollowee = Users.isLocalUser(followee) ? "local" : "remote";
+        this.commit({
+            [`${prefixFollower}.followings.total`]: isFollow ? 1 : -1,
+            [`${prefixFollower}.followings.inc`]: isFollow ? 1 : 0,
+            [`${prefixFollower}.followings.dec`]: isFollow ? 0 : 1
+        }, follower.id);
+        this.commit({
+            [`${prefixFollowee}.followers.total`]: isFollow ? 1 : -1,
+            [`${prefixFollowee}.followers.inc`]: isFollow ? 1 : 0,
+            [`${prefixFollowee}.followers.dec`]: isFollow ? 0 : 1
+        }, followee.id);
+    }
+}

packages/backend/built/services/chart/charts/per-user-notes.js

@@ -0,0 +1,34 @@
+import Chart from "../core.js";
+import { Notes } from "../../../models/index.js";
+import { name, schema } from "./entities/per-user-notes.js";
+/**
+ * ユーザーごとのノートに関するチャート
+ */ export default class PerUserNotesChart extends Chart {
+    constructor(){
+        super(name, schema, true);
+    }
+    async tickMajor(group) {
+        const [count] = await Promise.all([
+            Notes.countBy({
+                userId: group
+            })
+        ]);
+        return {
+            total: count
+        };
+    }
+    async tickMinor() {
+        return {};
+    }
+    async update(user, note, isAdditional) {
+        await this.commit({
+            total: isAdditional ? 1 : -1,
+            inc: isAdditional ? 1 : 0,
+            dec: isAdditional ? 0 : 1,
+            "diffs.normal": note.replyId == null && note.renoteId == null ? isAdditional ? 1 : -1 : 0,
+            "diffs.renote": note.renoteId != null ? isAdditional ? 1 : -1 : 0,
+            "diffs.reply": note.replyId != null ? isAdditional ? 1 : -1 : 0,
+            "diffs.withFile": note.fileIds.length > 0 ? isAdditional ? 1 : -1 : 0
+        }, user.id);
+    }
+}

packages/backend/built/services/chart/charts/per-user-reactions.js

@@ -0,0 +1,22 @@
+import Chart from "../core.js";
+import { Users } from "../../../models/index.js";
+import { name, schema } from "./entities/per-user-reactions.js";
+/**
+ * ユーザーごとのリアクションに関するチャート
+ */ export default class PerUserReactionsChart extends Chart {
+    constructor(){
+        super(name, schema, true);
+    }
+    async tickMajor(group) {
+        return {};
+    }
+    async tickMinor() {
+        return {};
+    }
+    async update(user, note) {
+        const prefix = Users.isLocalUser(user) ? "local" : "remote";
+        this.commit({
+            [`${prefix}.count`]: 1
+        }, note.userId);
+    }
+}

packages/backend/built/services/chart/charts/test-grouped.js

@@ -0,0 +1,26 @@
+import Chart from "../core.js";
+import { name, schema } from "./entities/test-grouped.js";
+/**
+ * For testing
+ */ export default class TestGroupedChart extends Chart {
+    total = {};
+    constructor(){
+        super(name, schema, true);
+    }
+    async tickMajor(group) {
+        return {
+            "foo.total": this.total[group]
+        };
+    }
+    async tickMinor() {
+        return {};
+    }
+    async increment(group) {
+        if (this.total[group] == null) this.total[group] = 0;
+        this.total[group]++;
+        await this.commit({
+            "foo.total": 1,
+            "foo.inc": 1
+        }, group);
+    }
+}

packages/backend/built/services/chart/charts/test-intersection.js

@@ -0,0 +1,29 @@
+import Chart from "../core.js";
+import { name, schema } from "./entities/test-intersection.js";
+/**
+ * For testing
+ */ export default class TestIntersectionChart extends Chart {
+    constructor(){
+        super(name, schema);
+    }
+    async tickMajor() {
+        return {};
+    }
+    async tickMinor() {
+        return {};
+    }
+    async addA(key) {
+        await this.commit({
+            a: [
+                key
+            ]
+        });
+    }
+    async addB(key) {
+        await this.commit({
+            b: [
+                key
+            ]
+        });
+    }
+}

packages/backend/built/services/chart/charts/test-unique.js

@@ -0,0 +1,22 @@
+import Chart from "../core.js";
+import { name, schema } from "./entities/test-unique.js";
+/**
+ * For testing
+ */ export default class TestUniqueChart extends Chart {
+    constructor(){
+        super(name, schema);
+    }
+    async tickMajor() {
+        return {};
+    }
+    async tickMinor() {
+        return {};
+    }
+    async uniqueIncrement(key) {
+        await this.commit({
+            foo: [
+                key
+            ]
+        });
+    }
+}

packages/backend/built/services/chart/charts/test.js

@@ -0,0 +1,32 @@
+import Chart from "../core.js";
+import { name, schema } from "./entities/test.js";
+/**
+ * For testing
+ */ export default class TestChart extends Chart {
+    total = 0;
+    constructor(){
+        super(name, schema);
+    }
+    async tickMajor() {
+        return {
+            "foo.total": this.total
+        };
+    }
+    async tickMinor() {
+        return {};
+    }
+    async increment() {
+        this.total++;
+        await this.commit({
+            "foo.total": 1,
+            "foo.inc": 1
+        });
+    }
+    async decrement() {
+        this.total--;
+        await this.commit({
+            "foo.total": -1,
+            "foo.dec": 1
+        });
+    }
+}

packages/backend/built/services/chart/charts/users.js

@@ -0,0 +1,36 @@
+import Chart from "../core.js";
+import { Users } from "../../../models/index.js";
+import { Not, IsNull } from "typeorm";
+import { name, schema } from "./entities/users.js";
+/**
+ * ユーザー数に関するチャート
+ */ export default class UsersChart extends Chart {
+    constructor(){
+        super(name, schema);
+    }
+    async tickMajor() {
+        const [localCount, remoteCount] = await Promise.all([
+            Users.countBy({
+                host: IsNull()
+            }),
+            Users.countBy({
+                host: Not(IsNull())
+            })
+        ]);
+        return {
+            "local.total": localCount,
+            "remote.total": remoteCount
+        };
+    }
+    async tickMinor() {
+        return {};
+    }
+    async update(user, isAdditional) {
+        const prefix = Users.isLocalUser(user) ? "local" : "remote";
+        await this.commit({
+            [`${prefix}.total`]: isAdditional ? 1 : -1,
+            [`${prefix}.inc`]: isAdditional ? 1 : 0,
+            [`${prefix}.dec`]: isAdditional ? 0 : 1
+        });
+    }
+}

packages/backend/built/services/chart/core.js

@@ -0,0 +1,581 @@
+/**
+ * チャートエンジン
+ *
+ * Tests located in test/chart
+ */ import * as nestedProperty from "nested-property";
+import Logger from "../logger.js";
+import { EntitySchema, LessThan, Between } from "typeorm";
+import { dateUTC, isTimeSame, isTimeBefore, subtractTime, addTime } from "../../prelude/time.js";
+import { db } from "../../db/postgre.js";
+import promiseLimit from "promise-limit";
+const logger = new Logger("chart", "white", process.env.NODE_ENV !== "test");
+const columnPrefix = "___";
+const uniqueTempColumnPrefix = "unique_temp___";
+const columnDot = "_";
+const camelToSnake = (str)=>{
+    return str.replace(/([A-Z])/g, (s)=>`_${s.charAt(0).toLowerCase()}`);
+};
+const removeDuplicates = (array)=>Array.from(new Set(array));
+export function getJsonSchema(schema) {
+    const jsonSchema = {
+        type: "object",
+        properties: {},
+        required: []
+    };
+    for(const k in schema){
+        jsonSchema.properties[k] = {
+            type: "array",
+            items: {
+                type: "number"
+            }
+        };
+    }
+    return jsonSchema;
+}
+/**
+ * 様々なチャートの管理を司るクラス
+ */ export default class Chart {
+    schema;
+    name;
+    buffer = [];
+    // ↓にしたいけどfindOneとかで型エラーになる
+    //private repositoryForHour: Repository<RawRecord<T>>;
+    //private repositoryForDay: Repository<RawRecord<T>>;
+    repositoryForHour;
+    repositoryForDay;
+    static convertSchemaToColumnDefinitions(schema) {
+        const columns = {};
+        for (const [k, v] of Object.entries(schema)){
+            const name = k.replaceAll(".", columnDot);
+            const type = v.range === "big" ? "bigint" : v.range === "small" ? "smallint" : "integer";
+            if (v.uniqueIncrement) {
+                columns[uniqueTempColumnPrefix + name] = {
+                    type: "varchar",
+                    array: true,
+                    default: "{}"
+                };
+                columns[columnPrefix + name] = {
+                    type,
+                    default: 0
+                };
+            } else {
+                columns[columnPrefix + name] = {
+                    type,
+                    default: 0
+                };
+            }
+        }
+        return columns;
+    }
+    static dateToTimestamp(x) {
+        return Math.floor(x.getTime() / 1000);
+    }
+    static parseDate(date) {
+        const y = date.getUTCFullYear();
+        const m = date.getUTCMonth();
+        const d = date.getUTCDate();
+        const h = date.getUTCHours();
+        const _m = date.getUTCMinutes();
+        const _s = date.getUTCSeconds();
+        const _ms = date.getUTCMilliseconds();
+        return [
+            y,
+            m,
+            d,
+            h,
+            _m,
+            _s,
+            _ms
+        ];
+    }
+    static getCurrentDate() {
+        return Chart.parseDate(new Date());
+    }
+    static schemaToEntity(name, schema, grouped = false) {
+        const createEntity = (span)=>new EntitySchema({
+                name: span === "hour" ? `__chart__${camelToSnake(name)}` : span === "day" ? `__chart_day__${camelToSnake(name)}` : new Error("not happen"),
+                columns: {
+                    id: {
+                        type: "integer",
+                        primary: true,
+                        generated: true
+                    },
+                    date: {
+                        type: "integer"
+                    },
+                    ...grouped ? {
+                        group: {
+                            type: "varchar",
+                            length: 128
+                        }
+                    } : {},
+                    ...Chart.convertSchemaToColumnDefinitions(schema)
+                },
+                indices: [
+                    {
+                        columns: grouped ? [
+                            "date",
+                            "group"
+                        ] : [
+                            "date"
+                        ],
+                        unique: true
+                    }
+                ],
+                uniques: [
+                    {
+                        columns: grouped ? [
+                            "date",
+                            "group"
+                        ] : [
+                            "date"
+                        ]
+                    }
+                ],
+                relations: {
+                }
+            });
+        return {
+            hour: createEntity("hour"),
+            day: createEntity("day")
+        };
+    }
+    constructor(name, schema, grouped = false){
+        this.name = name;
+        this.schema = schema;
+        const { hour, day } = Chart.schemaToEntity(name, schema, grouped);
+        this.repositoryForHour = db.getRepository(hour);
+        this.repositoryForDay = db.getRepository(day);
+    }
+    convertRawRecord(x) {
+        const kvs = {};
+        for (const k of Object.keys(x).filter((k)=>k.startsWith(columnPrefix))){
+            kvs[k.substr(columnPrefix.length).split(columnDot).join(".")] = x[k];
+        }
+        return kvs;
+    }
+    getNewLog(latest) {
+        const log = {};
+        for (const [k, v] of Object.entries(this.schema)){
+            if (v.accumulate && latest) {
+                log[k] = latest[k];
+            } else {
+                log[k] = 0;
+            }
+        }
+        return log;
+    }
+    getLatestLog(group, span) {
+        const repository = span === "hour" ? this.repositoryForHour : span === "day" ? this.repositoryForDay : new Error("not happen");
+        return repository.findOne({
+            where: group ? {
+                group: group
+            } : {},
+            order: {
+                date: -1
+            }
+        }).then((x)=>x ?? null);
+    }
+    /**
+	 * 現在(=今のHour or Day)のログをデータベースから探して、あればそれを返し、なければ作成して返します。
+	 */ async claimCurrentLog(group, span) {
+        const [y, m, d, h] = Chart.getCurrentDate();
+        const current = dateUTC(span === "hour" ? [
+            y,
+            m,
+            d,
+            h
+        ] : span === "day" ? [
+            y,
+            m,
+            d
+        ] : new Error("not happen"));
+        const repository = span === "hour" ? this.repositoryForHour : span === "day" ? this.repositoryForDay : new Error("not happen");
+        // 現在(=今のHour or Day)のログ
+        const currentLog = await repository.findOneBy({
+            date: Chart.dateToTimestamp(current),
+            ...group ? {
+                group: group
+            } : {}
+        });
+        // ログがあればそれを返して終了
+        if (currentLog != null) {
+            return currentLog;
+        }
+        let log;
+        let data;
+        // 集計期間が変わってから、初めてのチャート更新なら
+        // 最も最近のログを持ってくる
+        // * 例えば集計期間が「日」である場合で考えると、
+        // * 昨日何もチャートを更新するような出来事がなかった場合は、
+        // * ログがそもそも作られずドキュメントが存在しないということがあり得るため、
+        // * 「昨日の」と決め打ちせずに「もっとも最近の」とします
+        const latest = await this.getLatestLog(group, span);
+        if (latest != null) {
+            // 空ログデータを作成
+            data = this.getNewLog(this.convertRawRecord(latest));
+        } else {
+            // ログが存在しなかったら
+            // (Misskeyインスタンスを建てて初めてのチャート更新時など)
+            // 初期ログデータを作成
+            data = this.getNewLog(null);
+            logger.info(`${this.name + (group ? `:${group}` : "")}(${span}): Initial commit created`);
+        }
+        const date = Chart.dateToTimestamp(current);
+        const lockKey = group ? `${this.name}:${date}:${span}:${group}` : `${this.name}:${date}:${span}`;
+        const { getChartInsertLock } = await import("../../misc/app-lock.js");
+        const unlock = await getChartInsertLock(lockKey);
+        try {
+            // ロック内でもう1回チェックする
+            const currentLog = await repository.findOneBy({
+                date: date,
+                ...group ? {
+                    group: group
+                } : {}
+            });
+            // ログがあればそれを返して終了
+            if (currentLog != null) return currentLog;
+            const columns = {};
+            for (const [k, v] of Object.entries(data)){
+                const name = k.replaceAll(".", columnDot);
+                columns[columnPrefix + name] = v;
+            }
+            // 新規ログ挿入
+            log = await repository.insert({
+                date: date,
+                ...group ? {
+                    group: group
+                } : {},
+                ...columns
+            }).then((x)=>repository.findOneByOrFail(x.identifiers[0]));
+            logger.info(`${this.name + (group ? `:${group}` : "")}(${span}): New commit created`);
+            return log;
+        } finally{
+            unlock();
+        }
+    }
+    commit(diff, group = null) {
+        for (const [k, v] of Object.entries(diff)){
+            if (v == null || v === 0 || Array.isArray(v) && v.length === 0) // rome-ignore lint/performance/noDelete: needs to be deleted not just set to undefined
+            delete diff[k];
+        }
+        this.buffer.push({
+            diff,
+            group
+        });
+    }
+    async save() {
+        if (this.buffer.length === 0) {
+            logger.info(`${this.name}: Write skipped`);
+            return;
+        }
+        // TODO: 前の時間のログがbufferにあった場合のハンドリング
+        // 例えば、save が20分ごとに行われるとして、前回行われたのは 01:50 だったとする。
+        // 次に save が行われるのは 02:10 ということになるが、もし 01:55 に新規ログが buffer に追加されたとすると、
+        // そのログは本来は 01:00~ のログとしてDBに保存されて欲しいのに、02:00~ のログ扱いになってしまう。
+        // これを回避するための実装は複雑になりそうなため、一旦保留。
+        const update = async (logHour, logDay)=>{
+            const finalDiffs = {};
+            for (const diff of this.buffer.filter((q)=>q.group == null || q.group === logHour.group).map((q)=>q.diff)){
+                for (const [k, v] of Object.entries(diff)){
+                    if (finalDiffs[k] == null) {
+                        finalDiffs[k] = v;
+                    } else {
+                        if (typeof finalDiffs[k] === "number") {
+                            finalDiffs[k] += v;
+                        } else {
+                            finalDiffs[k] = finalDiffs[k].concat(v);
+                        }
+                    }
+                }
+            }
+            const queryForHour = {};
+            const queryForDay = {};
+            for (const [k, v] of Object.entries(finalDiffs)){
+                if (typeof v === "number") {
+                    const name = columnPrefix + k.replaceAll(".", columnDot);
+                    if (v > 0) queryForHour[name] = ()=>`"${name}" + ${v}`;
+                    if (v < 0) queryForHour[name] = ()=>`"${name}" - ${Math.abs(v)}`;
+                    if (v > 0) queryForDay[name] = ()=>`"${name}" + ${v}`;
+                    if (v < 0) queryForDay[name] = ()=>`"${name}" - ${Math.abs(v)}`;
+                } else if (Array.isArray(v) && v.length > 0) {
+                    // ユニークインクリメント
+                    const tempColumnName = uniqueTempColumnPrefix + k.replaceAll(".", columnDot);
+                    // TODO: item をSQLエスケープ
+                    const itemsForHour = v.filter((item)=>!logHour[tempColumnName].includes(item)).map((item)=>`"${item}"`);
+                    const itemsForDay = v.filter((item)=>!logDay[tempColumnName].includes(item)).map((item)=>`"${item}"`);
+                    if (itemsForHour.length > 0) queryForHour[tempColumnName] = ()=>`array_cat("${tempColumnName}", '{${itemsForHour.join(",")}}'::varchar[])`;
+                    if (itemsForDay.length > 0) queryForDay[tempColumnName] = ()=>`array_cat("${tempColumnName}", '{${itemsForDay.join(",")}}'::varchar[])`;
+                }
+            }
+            // bake unique count
+            for (const [k, v] of Object.entries(finalDiffs)){
+                if (this.schema[k].uniqueIncrement && Array.isArray(v) && v.length > 0) {
+                    const name = columnPrefix + k.replaceAll(".", columnDot);
+                    const tempColumnName = uniqueTempColumnPrefix + k.replaceAll(".", columnDot);
+                    queryForHour[name] = new Set([
+                        ...v,
+                        ...logHour[tempColumnName]
+                    ]).size;
+                    queryForDay[name] = new Set([
+                        ...v,
+                        ...logDay[tempColumnName]
+                    ]).size;
+                }
+            }
+            // compute intersection
+            // TODO: intersectionに指定されたカラムがintersectionだった場合の対応
+            for (const [k, v] of Object.entries(this.schema)){
+                const intersection = v.intersection;
+                if (intersection) {
+                    const name = columnPrefix + k.replaceAll(".", columnDot);
+                    const firstKey = intersection[0];
+                    const firstTempColumnName = uniqueTempColumnPrefix + firstKey.replaceAll(".", columnDot);
+                    const firstValues = finalDiffs[firstKey];
+                    const currentValuesForHour = new Set([
+                        ...firstValues ?? [],
+                        ...logHour[firstTempColumnName]
+                    ]);
+                    const currentValuesForDay = new Set([
+                        ...firstValues ?? [],
+                        ...logDay[firstTempColumnName]
+                    ]);
+                    for(let i = 1; i < intersection.length; i++){
+                        const targetKey = intersection[i];
+                        const targetTempColumnName = uniqueTempColumnPrefix + targetKey.replaceAll(".", columnDot);
+                        const targetValues = finalDiffs[targetKey];
+                        const targetValuesForHour = new Set([
+                            ...targetValues ?? [],
+                            ...logHour[targetTempColumnName]
+                        ]);
+                        const targetValuesForDay = new Set([
+                            ...targetValues ?? [],
+                            ...logDay[targetTempColumnName]
+                        ]);
+                        currentValuesForHour.forEach((v)=>{
+                            if (!targetValuesForHour.has(v)) currentValuesForHour.delete(v);
+                        });
+                        currentValuesForDay.forEach((v)=>{
+                            if (!targetValuesForDay.has(v)) currentValuesForDay.delete(v);
+                        });
+                    }
+                    queryForHour[name] = currentValuesForHour.size;
+                    queryForDay[name] = currentValuesForDay.size;
+                }
+            }
+            // ログ更新
+            await Promise.all([
+                this.repositoryForHour.createQueryBuilder().update().set(queryForHour).where("id = :id", {
+                    id: logHour.id
+                }).execute(),
+                this.repositoryForDay.createQueryBuilder().update().set(queryForDay).where("id = :id", {
+                    id: logDay.id
+                }).execute()
+            ]);
+            logger.info(`${this.name + (logHour.group ? `:${logHour.group}` : "")}: Updated`);
+            // TODO: この一連の処理が始まった後に新たにbufferに入ったものは消さないようにする
+            this.buffer = this.buffer.filter((q)=>q.group != null && q.group !== logHour.group);
+        };
+        const startCount = this.buffer.length;
+        const groups = removeDuplicates(this.buffer.map((log)=>log.group));
+        const groupCount = groups.length;
+        // Limit the number of concurrent chart update queries executed on the database
+        // to 25 at a time, so as avoid excessive IO spinlocks like when 8k queries are
+        // sent out at once.
+        const limit = promiseLimit(25);
+        const startTime = Date.now();
+        await Promise.all(groups.map((group)=>limit(()=>Promise.all([
+                    this.claimCurrentLog(group, "hour"),
+                    this.claimCurrentLog(group, "day")
+                ]).then(([logHour, logDay])=>update(logHour, logDay)))));
+        const duration = Date.now() - startTime;
+        logger.info(`Saved ${startCount} (${groupCount} unique) ${this.name} items in ${duration}ms (${this.buffer.length} remaining)`);
+    }
+    async tick(major, group = null) {
+        const data = major ? await this.tickMajor(group) : await this.tickMinor(group);
+        const columns = {};
+        for (const [k, v] of Object.entries(data)){
+            const name = columnPrefix + k.replaceAll(".", columnDot);
+            columns[name] = v;
+        }
+        if (Object.keys(columns).length === 0) {
+            return;
+        }
+        const update = async (logHour, logDay)=>{
+            await Promise.all([
+                this.repositoryForHour.createQueryBuilder().update().set(columns).where("id = :id", {
+                    id: logHour.id
+                }).execute(),
+                this.repositoryForDay.createQueryBuilder().update().set(columns).where("id = :id", {
+                    id: logDay.id
+                }).execute()
+            ]);
+        };
+        return Promise.all([
+            this.claimCurrentLog(group, "hour"),
+            this.claimCurrentLog(group, "day")
+        ]).then(([logHour, logDay])=>update(logHour, logDay));
+    }
+    resync(group = null) {
+        return this.tick(true, group);
+    }
+    async clean() {
+        const current = dateUTC(Chart.getCurrentDate());
+        // 一日以上前かつ三日以内
+        const gt = Chart.dateToTimestamp(current) - 60 * 60 * 24 * 3;
+        const lt = Chart.dateToTimestamp(current) - 60 * 60 * 24;
+        const columns = {};
+        for (const [k, v] of Object.entries(this.schema)){
+            if (v.uniqueIncrement) {
+                const name = uniqueTempColumnPrefix + k.replaceAll(".", columnDot);
+                columns[name] = [];
+            }
+        }
+        if (Object.keys(columns).length === 0) {
+            return;
+        }
+        await Promise.all([
+            this.repositoryForHour.createQueryBuilder().update().set(columns).where("date > :gt", {
+                gt
+            }).andWhere("date < :lt", {
+                lt
+            }).execute(),
+            this.repositoryForDay.createQueryBuilder().update().set(columns).where("date > :gt", {
+                gt
+            }).andWhere("date < :lt", {
+                lt
+            }).execute()
+        ]);
+    }
+    async getChartRaw(span, amount, cursor, group = null) {
+        const [y, m, d, h, _m, _s, _ms] = cursor ? Chart.parseDate(subtractTime(addTime(cursor, 1, span), 1)) : Chart.getCurrentDate();
+        const [y2, m2, d2, h2] = cursor ? Chart.parseDate(addTime(cursor, 1, span)) : [];
+        const lt = dateUTC([
+            y,
+            m,
+            d,
+            h,
+            _m,
+            _s,
+            _ms
+        ]);
+        const gt = span === "day" ? subtractTime(cursor ? dateUTC([
+            y2,
+            m2,
+            d2,
+            0
+        ]) : dateUTC([
+            y,
+            m,
+            d,
+            0
+        ]), amount - 1, "day") : span === "hour" ? subtractTime(cursor ? dateUTC([
+            y2,
+            m2,
+            d2,
+            h2
+        ]) : dateUTC([
+            y,
+            m,
+            d,
+            h
+        ]), amount - 1, "hour") : new Error("not happen");
+        const repository = span === "hour" ? this.repositoryForHour : span === "day" ? this.repositoryForDay : new Error("not happen");
+        // ログ取得
+        let logs = await repository.find({
+            where: {
+                date: Between(Chart.dateToTimestamp(gt), Chart.dateToTimestamp(lt)),
+                ...group ? {
+                    group: group
+                } : {}
+            },
+            order: {
+                date: -1
+            }
+        });
+        // 要求された範囲にログがひとつもなかったら
+        if (logs.length === 0) {
+            // もっとも新しいログを持ってくる
+            // (すくなくともひとつログが無いと隙間埋めできないため)
+            const recentLog = await repository.findOne({
+                where: group ? {
+                    group: group
+                } : {},
+                order: {
+                    date: -1
+                }
+            });
+            if (recentLog) {
+                logs = [
+                    recentLog
+                ];
+            }
+        // 要求された範囲の最も古い箇所に位置するログが存在しなかったら
+        } else if (!isTimeSame(new Date(logs[logs.length - 1].date * 1000), gt)) {
+            // 要求された範囲の最も古い箇所時点での最も新しいログを持ってきて末尾に追加する
+            // (隙間埋めできないため)
+            const outdatedLog = await repository.findOne({
+                where: {
+                    date: LessThan(Chart.dateToTimestamp(gt)),
+                    ...group ? {
+                        group: group
+                    } : {}
+                },
+                order: {
+                    date: -1
+                }
+            });
+            if (outdatedLog) {
+                logs.push(outdatedLog);
+            }
+        }
+        const chart = [];
+        for(let i = amount - 1; i >= 0; i--){
+            const current = span === "hour" ? subtractTime(dateUTC([
+                y,
+                m,
+                d,
+                h
+            ]), i, "hour") : span === "day" ? subtractTime(dateUTC([
+                y,
+                m,
+                d
+            ]), i, "day") : new Error("not happen");
+            const log = logs.find((l)=>isTimeSame(new Date(l.date * 1000), current));
+            if (log) {
+                chart.unshift(this.convertRawRecord(log));
+            } else {
+                // 隙間埋め
+                const latest = logs.find((l)=>isTimeBefore(new Date(l.date * 1000), current));
+                const data = latest ? this.convertRawRecord(latest) : null;
+                chart.unshift(this.getNewLog(data));
+            }
+        }
+        const res = {};
+        /**
+		 * [{ foo: 1, bar: 5 }, { foo: 2, bar: 6 }, { foo: 3, bar: 7 }]
+		 * を
+		 * { foo: [1, 2, 3], bar: [5, 6, 7] }
+		 * にする
+		 */ for (const record of chart){
+            for (const [k, v] of Object.entries(record)){
+                if (res[k]) {
+                    res[k].push(v);
+                } else {
+                    res[k] = [
+                        v
+                    ];
+                }
+            }
+        }
+        return res;
+    }
+    async getChart(span, amount, cursor, group = null) {
+        const result = await this.getChartRaw(span, amount, cursor, group);
+        const object = {};
+        for (const [k, v] of Object.entries(result)){
+            nestedProperty.set(object, k, v);
+        }
+        return object;
+    }
+}

packages/backend/built/services/chart/entities.js

@@ -0,0 +1,52 @@
+import { entity as FederationChart } from "./charts/entities/federation.js";
+import { entity as NotesChart } from "./charts/entities/notes.js";
+import { entity as UsersChart } from "./charts/entities/users.js";
+import { entity as ActiveUsersChart } from "./charts/entities/active-users.js";
+import { entity as InstanceChart } from "./charts/entities/instance.js";
+import { entity as PerUserNotesChart } from "./charts/entities/per-user-notes.js";
+import { entity as DriveChart } from "./charts/entities/drive.js";
+import { entity as PerUserReactionsChart } from "./charts/entities/per-user-reactions.js";
+import { entity as HashtagChart } from "./charts/entities/hashtag.js";
+import { entity as PerUserFollowingChart } from "./charts/entities/per-user-following.js";
+import { entity as PerUserDriveChart } from "./charts/entities/per-user-drive.js";
+import { entity as ApRequestChart } from "./charts/entities/ap-request.js";
+import { entity as TestChart } from "./charts/entities/test.js";
+import { entity as TestGroupedChart } from "./charts/entities/test-grouped.js";
+import { entity as TestUniqueChart } from "./charts/entities/test-unique.js";
+import { entity as TestIntersectionChart } from "./charts/entities/test-intersection.js";
+export const entities = [
+    FederationChart.hour,
+    FederationChart.day,
+    NotesChart.hour,
+    NotesChart.day,
+    UsersChart.hour,
+    UsersChart.day,
+    ActiveUsersChart.hour,
+    ActiveUsersChart.day,
+    InstanceChart.hour,
+    InstanceChart.day,
+    PerUserNotesChart.hour,
+    PerUserNotesChart.day,
+    DriveChart.hour,
+    DriveChart.day,
+    PerUserReactionsChart.hour,
+    PerUserReactionsChart.day,
+    HashtagChart.hour,
+    HashtagChart.day,
+    PerUserFollowingChart.hour,
+    PerUserFollowingChart.day,
+    PerUserDriveChart.hour,
+    PerUserDriveChart.day,
+    ApRequestChart.hour,
+    ApRequestChart.day,
+    ...process.env.NODE_ENV === "test" ? [
+        TestChart.hour,
+        TestChart.day,
+        TestGroupedChart.hour,
+        TestGroupedChart.day,
+        TestUniqueChart.hour,
+        TestUniqueChart.day,
+        TestIntersectionChart.hour,
+        TestIntersectionChart.day
+    ] : []
+];

packages/backend/built/services/chart/index.js

@@ -0,0 +1,46 @@
+import { beforeShutdown } from "../../misc/before-shutdown.js";
+import FederationChart from "./charts/federation.js";
+import NotesChart from "./charts/notes.js";
+import UsersChart from "./charts/users.js";
+import ActiveUsersChart from "./charts/active-users.js";
+import InstanceChart from "./charts/instance.js";
+import PerUserNotesChart from "./charts/per-user-notes.js";
+import DriveChart from "./charts/drive.js";
+import PerUserReactionsChart from "./charts/per-user-reactions.js";
+import HashtagChart from "./charts/hashtag.js";
+import PerUserFollowingChart from "./charts/per-user-following.js";
+import PerUserDriveChart from "./charts/per-user-drive.js";
+import ApRequestChart from "./charts/ap-request.js";
+export const federationChart = new FederationChart();
+export const notesChart = new NotesChart();
+export const usersChart = new UsersChart();
+export const activeUsersChart = new ActiveUsersChart();
+export const instanceChart = new InstanceChart();
+export const perUserNotesChart = new PerUserNotesChart();
+export const driveChart = new DriveChart();
+export const perUserReactionsChart = new PerUserReactionsChart();
+export const hashtagChart = new HashtagChart();
+export const perUserFollowingChart = new PerUserFollowingChart();
+export const perUserDriveChart = new PerUserDriveChart();
+export const apRequestChart = new ApRequestChart();
+const charts = [
+    federationChart,
+    notesChart,
+    usersChart,
+    activeUsersChart,
+    instanceChart,
+    perUserNotesChart,
+    driveChart,
+    perUserReactionsChart,
+    hashtagChart,
+    perUserFollowingChart,
+    perUserDriveChart,
+    apRequestChart
+];
+// 20分おきにメモリ情報をDBに書き込み
+setInterval(()=>{
+    for (const chart of charts){
+        chart.save();
+    }
+}, 1000 * 60 * 20);
+beforeShutdown(()=>Promise.all(charts.map((chart)=>chart.save())));

packages/backend/built/services/create-bite.js

@@ -0,0 +1,97 @@
+import { genId } from "../misc/gen-id.js";
+import { Bites, Users } from "../models/index.js";
+import { renderActivity } from "../remote/activitypub/renderer/index.js";
+import renderBite from "../remote/activitypub/renderer/bite.js";
+import { deliverToUser } from "../remote/activitypub/deliver-manager.js";
+import { createNotification } from "./create-notification.js";
+import { tickBiteLocal, tickBiteOutgoing } from "../metrics.js";
+import { getNote } from "../server/api/common/getters.js";
+import { IdentifiableError } from "../misc/identifiable-error.js";
+export async function createBite(sender, targetType, targetId, remoteUri = null, createdAt = null) {
+    const id = genId();
+    const insert = {
+        id,
+        createdAt: createdAt ?? new Date(),
+        userId: sender.id,
+        userHost: sender.host,
+        targetType,
+        replied: false,
+        uri: remoteUri
+    };
+    let targetUser;
+    switch(targetType){
+        case "user":
+            insert.targetUserId = targetId;
+            targetUser = await Users.findOneByOrFail({
+                id: targetId
+            });
+            break;
+        case "bite":
+            insert.targetBiteId = targetId;
+            const bite = await Bites.findOneOrFail({
+                where: {
+                    id: targetId
+                },
+                relations: [
+                    "user"
+                ]
+            });
+            targetUser = bite.user;
+            break;
+        case "note":
+            insert.targetNoteId = targetId;
+            const note = await getNote(targetId, sender);
+            targetUser = await Users.findOneByOrFail({
+                id: note.userId
+            });
+            break;
+    }
+    if (targetUser.canBite === "nobody") throw new IdentifiableError("92ce0141-760d-4163-a7a2-73b349e3d133", "Bites disabled");
+    const relations = await Users.getRelation(sender.id, targetUser.id);
+    if (targetUser.canBite === "followers" && !relations.isFollowing) throw new IdentifiableError("35363f14-f489-45e2-81a9-558450710dfe", "Not following");
+    if (relations.isBlocked) throw new IdentifiableError("f82d8d34-beaf-42f3-9135-477d32288213", "Blocked");
+    await Bites.insert(insert);
+    const bite1 = await Bites.findOneOrFail({
+        where: {
+            id
+        },
+        relations: [
+            "targetUser",
+            "targetBite",
+            "targetNote"
+        ]
+    });
+    let deliverTarget;
+    switch(targetType){
+        case "user":
+            deliverTarget = bite1.targetUser;
+            break;
+        case "bite":
+            await Bites.update({
+                id: bite1.targetBiteId
+            }, {
+                replied: true
+            });
+            deliverTarget = bite1.targetBite.user ?? await Users.findOneByOrFail({
+                id: bite1.targetBite.userId
+            });
+            break;
+        case "note":
+            deliverTarget = bite1.targetNote.user ?? await Users.findOneByOrFail({
+                id: bite1.targetNote.userId
+            });
+            break;
+    }
+    if (Users.isLocalUser(sender) && Users.isRemoteUser(deliverTarget)) {
+        await deliverToUser(sender, renderActivity(await renderBite(bite1)), deliverTarget);
+        tickBiteOutgoing();
+    }
+    if (Users.isLocalUser(deliverTarget)) {
+        await createNotification(deliverTarget.id, "bite", {
+            notifierId: sender.id,
+            biteId: bite1.id
+        });
+        if (Users.isLocalUser(sender)) tickBiteLocal();
+    }
+    return id;
+}

packages/backend/built/services/create-notification.js

@@ -0,0 +1,85 @@
+import { publishMainStream } from "./stream.js";
+import { pushNotification } from "./push-notification.js";
+import { Notifications, Mutings, NoteThreadMutings, UserProfiles, Users, Followings } from "../models/index.js";
+import { genId } from "../misc/gen-id.js";
+import { sendEmailNotification } from "./send-email-notification.js";
+import { shouldSilenceInstance } from "../misc/should-block-instance.js";
+export async function createNotification(notifieeId, type, data) {
+    if (data.notifierId && notifieeId === data.notifierId) {
+        return null;
+    }
+    if (data.notifierId && [
+        "mention",
+        "reply",
+        "renote",
+        "quote",
+        "reaction",
+        "bite"
+    ].includes(type)) {
+        const notifier = await Users.findOneBy({
+            id: data.notifierId
+        });
+        // suppress if the notifier does not exist or is silenced.
+        if (!notifier) return null;
+        // suppress if the notifier is silenced or in a silenced instance, and not followed by the notifiee.
+        if ((notifier.isSilenced || Users.isRemoteUser(notifier) && await shouldSilenceInstance(notifier.host)) && !await Followings.exist({
+            where: {
+                followerId: notifieeId,
+                followeeId: data.notifierId
+            }
+        })) return null;
+    }
+    const profile = await UserProfiles.findOneBy({
+        userId: notifieeId
+    });
+    const isMuted = profile?.mutingNotificationTypes.includes(type);
+    if (data.note != null) {
+        const threadMute = await NoteThreadMutings.findOneBy({
+            userId: notifieeId,
+            threadId: data.note.threadId || data.note.id
+        });
+        if (threadMute) {
+            return null;
+        }
+    }
+    // Create notification
+    const notification = await Notifications.insert({
+        id: genId(),
+        createdAt: new Date(),
+        notifieeId: notifieeId,
+        type: type,
+        // 相手がこの通知をミュートしているようなら、既読を予めつけておく
+        isRead: isMuted,
+        ...data
+    }).then((x)=>Notifications.findOneByOrFail(x.identifiers[0]));
+    const packed = await Notifications.pack(notification, {});
+    // Publish notification event
+    publishMainStream(notifieeId, "notification", packed);
+    // 2秒経っても(今回作成した)通知が既読にならなかったら「未読の通知がありますよ」イベントを発行する
+    setTimeout(async ()=>{
+        const fresh = await Notifications.findOneBy({
+            id: notification.id
+        });
+        if (fresh == null) return; // 既に削除されているかもしれない
+        // We execute this before, because the server side "read" check doesnt work well with push notifications, the app and service worker will decide themself
+        // when it is best to show push notifications
+        pushNotification(notifieeId, "notification", packed);
+        if (fresh.isRead) return;
+        //#region ただしミュートしているユーザーからの通知なら無視
+        const mutings = await Mutings.findBy({
+            muterId: notifieeId
+        });
+        if (data.notifierId && mutings.map((m)=>m.muteeId).includes(data.notifierId)) {
+            return;
+        }
+        //#endregion
+        publishMainStream(notifieeId, "unreadNotification", packed);
+        if (type === "follow") sendEmailNotification.follow(notifieeId, await Users.findOneByOrFail({
+            id: data.notifierId
+        }));
+        if (type === "receiveFollowRequest") sendEmailNotification.receiveFollowRequest(notifieeId, await Users.findOneByOrFail({
+            id: data.notifierId
+        }));
+    }, 2000);
+    return notification;
+}

packages/backend/built/services/create-system-user.js

@@ -0,0 +1,63 @@
+import { v4 as uuid } from "uuid";
+import generateNativeUserToken from "../server/api/common/generate-native-user-token.js";
+import { genRsaKeyPair } from "../misc/gen-key-pair.js";
+import { User } from "../models/entities/user.js";
+import { UserProfile } from "../models/entities/user-profile.js";
+import { IsNull } from "typeorm";
+import { genId } from "../misc/gen-id.js";
+import { UserKeypair } from "../models/entities/user-keypair.js";
+import { UsedUsername } from "../models/entities/used-username.js";
+import { db } from "../db/postgre.js";
+import { hashPassword } from "../misc/password.js";
+import { Users } from "../models/index.js";
+export async function createSystemUser(username) {
+    const password = uuid();
+    // Generate hash of password
+    const hash = await hashPassword(password);
+    // Generate secret
+    const secret = generateNativeUserToken();
+    const keyPair = await genRsaKeyPair(4096);
+    let account;
+    const exist = await Users.findOneBy({
+        usernameLower: username.toLowerCase(),
+        host: IsNull()
+    });
+    if (exist) throw new Error("the user is already exists");
+    // Prepare objects
+    const user = {
+        id: genId(),
+        createdAt: new Date(),
+        username: username,
+        usernameLower: username.toLowerCase(),
+        host: null,
+        token: secret,
+        isAdmin: false,
+        isLocked: true,
+        isExplorable: false,
+        isBot: true
+    };
+    const userKeypair = {
+        publicKey: keyPair.publicKey,
+        privateKey: keyPair.privateKey,
+        userId: user.id
+    };
+    const userProfile = {
+        userId: user.id,
+        autoAcceptFollowed: false,
+        password: hash
+    };
+    const usedUsername = {
+        createdAt: new Date(),
+        username: username.toLowerCase()
+    };
+    // Save the objects atomically using a db transaction, note that we should never run any code in a transaction block directly
+    await db.transaction(async (transactionalEntityManager)=>{
+        await transactionalEntityManager.insert(User, user);
+        await transactionalEntityManager.insert(UserKeypair, userKeypair);
+        await transactionalEntityManager.insert(UserProfile, userProfile);
+        await transactionalEntityManager.insert(UsedUsername, usedUsername);
+    });
+    return Users.findOneByOrFail({
+        id: user.id
+    });
+}

packages/backend/built/services/delete-account.js

@@ -0,0 +1,16 @@
+import { Users } from "../models/index.js";
+import { createDeleteAccountJob } from "../queue/index.js";
+import { publishUserEvent } from "./stream.js";
+import { doPostSuspend } from "./suspend-user.js";
+export async function deleteAccount(user) {
+    // 物理削除する前にDelete activityを送信する
+    await doPostSuspend(user).catch((e)=>{});
+    createDeleteAccountJob(user, {
+        soft: false
+    });
+    await Users.update(user.id, {
+        isDeleted: true
+    });
+    // Terminate streaming
+    publishUserEvent(user.id, "terminate", {});
+}

packages/backend/built/services/drive/add-file.js

@@ -0,0 +1,446 @@
+import * as fs from "node:fs";
+import { Readable } from "node:stream";
+import { v4 as uuid } from "uuid";
+import sharp from "sharp";
+import { IsNull } from "typeorm";
+import { publishMainStream, publishDriveStream } from "../stream.js";
+import { fetchMeta } from "../../misc/fetch-meta.js";
+import { contentDisposition } from "../../misc/content-disposition.js";
+import { getFileInfo } from "../../misc/get-file-info.js";
+import { DriveFiles, DriveFolders, Users, UserProfiles } from "../../models/index.js";
+import { DriveFile } from "../../models/entities/drive-file.js";
+import { driveChart, perUserDriveChart, instanceChart } from "../chart/index.js";
+import { genId } from "../../misc/gen-id.js";
+import { isDuplicateKeyValueError } from "../../misc/is-duplicate-key-value-error.js";
+import { FILE_TYPE_BROWSERSAFE } from "../../const.js";
+import { IdentifiableError } from "../../misc/identifiable-error.js";
+import { getS3 } from "./s3.js";
+import { InternalStorage } from "./internal-storage.js";
+import { convertSharpToWebp } from "./image-processor.js";
+import { driveLogger } from "./logger.js";
+import { GenerateVideoThumbnail } from "./generate-video-thumbnail.js";
+import { deleteFile } from "./delete-file.js";
+import { Upload } from "@aws-sdk/lib-storage";
+const logger = driveLogger.createSubLogger("register", "yellow");
+/***
+ * Save file
+ * @param path Path for original
+ * @param name Name for original
+ * @param type Content-Type for original
+ * @param hash Hash for original
+ * @param size Size for original
+ */ async function save(file, path, name, type, hash, size) {
+    // thunbnail, webpublic を必要なら生成
+    const alts = await generateAlts(path, type, !file.uri);
+    const meta = await fetchMeta();
+    if (meta.useObjectStorage) {
+        //#region ObjectStorage params
+        let [ext] = name.match(/\.([a-zA-Z0-9_-]+)$/) || [
+            ""
+        ];
+        if (ext === "") {
+            if (type === "image/jpeg") ext = ".jpg";
+            if (type === "image/png") ext = ".png";
+            if (type === "image/webp") ext = ".webp";
+            if (type === "image/apng") ext = ".apng";
+            if (type === "image/avif") ext = ".avif";
+            if (type === "image/vnd.mozilla.apng") ext = ".apng";
+        }
+        // Some cloud providers (notably upcloud) will infer the content-type based
+        // on extension, so we remove extensions from non-browser-safe types.
+        if (!FILE_TYPE_BROWSERSAFE.includes(type)) {
+            ext = "";
+        }
+        const baseUrl = meta.objectStorageBaseUrl || `${meta.objectStorageUseSSL ? "https" : "http"}://${meta.objectStorageEndpoint}${meta.objectStoragePort ? `:${meta.objectStoragePort}` : ""}/${meta.objectStorageBucket}`;
+        // for original
+        const key = `${meta.objectStoragePrefix}/${uuid()}${ext}`;
+        const url = `${baseUrl}/${key}`;
+        // for alts
+        let webpublicKey = null;
+        let webpublicUrl = null;
+        let thumbnailKey = null;
+        let thumbnailUrl = null;
+        //#endregion
+        //#region Uploads
+        logger.info(`uploading original: ${key}`);
+        const uploads = [
+            upload(key, fs.createReadStream(path), type, name)
+        ];
+        if (alts.webpublic) {
+            webpublicKey = `${meta.objectStoragePrefix}/webpublic-${uuid()}.${alts.webpublic.ext}`;
+            webpublicUrl = `${baseUrl}/${webpublicKey}`;
+            logger.info(`uploading webpublic: ${webpublicKey}`);
+            uploads.push(upload(webpublicKey, alts.webpublic.data, alts.webpublic.type, name));
+        }
+        if (alts.thumbnail) {
+            thumbnailKey = `${meta.objectStoragePrefix}/thumbnail-${uuid()}.${alts.thumbnail.ext}`;
+            thumbnailUrl = `${baseUrl}/${thumbnailKey}`;
+            logger.info(`uploading thumbnail: ${thumbnailKey}`);
+            uploads.push(upload(thumbnailKey, alts.thumbnail.data, alts.thumbnail.type));
+        }
+        await Promise.all(uploads);
+        //#endregion
+        file.url = url;
+        file.thumbnailUrl = thumbnailUrl;
+        file.webpublicUrl = webpublicUrl;
+        file.accessKey = key;
+        file.thumbnailAccessKey = thumbnailKey;
+        file.webpublicAccessKey = webpublicKey;
+        file.webpublicType = alts.webpublic?.type ?? null;
+        file.name = name;
+        file.type = type;
+        file.md5 = hash;
+        file.size = size;
+        file.storedInternal = false;
+        return await DriveFiles.insert(file).then((x)=>DriveFiles.findOneByOrFail(x.identifiers[0]));
+    } else {
+        // use internal storage
+        const accessKey = uuid();
+        const thumbnailAccessKey = `thumbnail-${uuid()}`;
+        const webpublicAccessKey = `webpublic-${uuid()}`;
+        const url = await InternalStorage.saveFromPath(accessKey, path);
+        let thumbnailUrl = null;
+        let webpublicUrl = null;
+        if (alts.thumbnail) {
+            thumbnailUrl = InternalStorage.saveFromBuffer(thumbnailAccessKey, alts.thumbnail.data);
+            logger.info(`thumbnail stored: ${thumbnailAccessKey}`);
+        }
+        if (alts.webpublic) {
+            webpublicUrl = InternalStorage.saveFromBuffer(webpublicAccessKey, alts.webpublic.data);
+            logger.info(`web stored: ${webpublicAccessKey}`);
+        }
+        file.storedInternal = true;
+        file.url = url;
+        file.thumbnailUrl = thumbnailUrl;
+        file.webpublicUrl = webpublicUrl;
+        file.accessKey = accessKey;
+        file.thumbnailAccessKey = thumbnailAccessKey;
+        file.webpublicAccessKey = webpublicAccessKey;
+        file.webpublicType = alts.webpublic?.type ?? null;
+        file.name = name;
+        file.type = type;
+        file.md5 = hash;
+        file.size = size;
+        return await DriveFiles.insert(file).then((x)=>DriveFiles.findOneByOrFail(x.identifiers[0]));
+    }
+}
+/**
+ * Generate webpublic, thumbnail, etc
+ * @param path Path for original
+ * @param type Content-Type for original
+ * @param generateWeb Generate webpublic or not
+ */ export async function generateAlts(path, type, generateWeb) {
+    if (type.startsWith("video/")) {
+        try {
+            const thumbnail = await GenerateVideoThumbnail(path);
+            return {
+                webpublic: null,
+                thumbnail
+            };
+        } catch (err) {
+            logger.warn(`GenerateVideoThumbnail failed: ${err}`);
+            return {
+                webpublic: null,
+                thumbnail: null
+            };
+        }
+    }
+    if (![
+        "image/jpeg",
+        "image/png",
+        "image/webp",
+        "image/svg+xml",
+        "image/avif"
+    ].includes(type)) {
+        logger.debug("web image and thumbnail not created (not an required file)");
+        return {
+            webpublic: null,
+            thumbnail: null
+        };
+    }
+    let img = null;
+    let satisfyWebpublic;
+    try {
+        img = sharp(path);
+        const metadata = await img.metadata();
+        const isAnimated = metadata.pages && metadata.pages > 1;
+        // skip animated
+        if (isAnimated) {
+            return {
+                webpublic: null,
+                thumbnail: null
+            };
+        }
+        satisfyWebpublic = !!(type !== "image/svg+xml" && type !== "image/webp" && !(metadata.exif || metadata.iptc || metadata.xmp || metadata.tifftagPhotoshop) && metadata.width && metadata.width <= 2048 && metadata.height && metadata.height <= 2048);
+    } catch (err) {
+        logger.warn(`sharp failed: ${err}`);
+        return {
+            webpublic: null,
+            thumbnail: null
+        };
+    }
+    // #region webpublic
+    let webpublic = null;
+    if (generateWeb && !satisfyWebpublic) {
+        logger.info("creating web image");
+        try {
+            if ([
+                "image/jpeg"
+            ].includes(type)) {
+                webpublic = await convertSharpToWebp(img, 2048, 2048);
+            } else if ([
+                "image/webp"
+            ].includes(type)) {
+                webpublic = await convertSharpToWebp(img, 2048, 2048);
+            } else if ([
+                "image/png"
+            ].includes(type)) {
+                webpublic = await convertSharpToWebp(img, 2048, 2048, 100);
+            } else if ([
+                "image/svg+xml"
+            ].includes(type)) {
+                webpublic = await convertSharpToWebp(img, 2048, 2048);
+            } else {
+                logger.debug("web image not created (not an required image)");
+            }
+        } catch (err) {
+            logger.warn("web image not created (an error occured)", err);
+        }
+    } else {
+        if (satisfyWebpublic) logger.info("web image not created (original satisfies webpublic)");
+        else logger.info("web image not created (from remote)");
+    }
+    // #endregion webpublic
+    // #region thumbnail
+    let thumbnail = null;
+    try {
+        if ([
+            "image/jpeg",
+            "image/webp",
+            "image/png",
+            "image/svg+xml",
+            "image/avif"
+        ].includes(type)) {
+            thumbnail = await convertSharpToWebp(img, 996, 560);
+        } else {
+            logger.debug("thumbnail not created (not an required file)");
+        }
+    } catch (err) {
+        logger.warn("thumbnail not created (an error occured)", err);
+    }
+    // #endregion thumbnail
+    return {
+        webpublic,
+        thumbnail
+    };
+}
+/**
+ * Upload to ObjectStorage
+ */ async function upload(key, stream, type, filename) {
+    if (type === "image/apng") type = "image/png";
+    if (!FILE_TYPE_BROWSERSAFE.includes(type)) type = "application/octet-stream";
+    const meta = await fetchMeta();
+    const params = {
+        Bucket: meta.objectStorageBucket,
+        Key: key,
+        Body: Buffer.isBuffer(stream) ? Readable.from(stream) : stream,
+        ContentType: type,
+        CacheControl: "max-age=31536000, immutable"
+    };
+    if (filename) params.ContentDisposition = contentDisposition("inline", filename);
+    if (meta.objectStorageSetPublicRead) params.ACL = "public-read";
+    const s3 = getS3(meta);
+    const upload = new Upload({
+        client: s3,
+        params,
+        partSize: meta.objectStorageEndpoint === "storage.googleapis.com" ? 500 * 1024 * 1024 : 8 * 1024 * 1024
+    });
+    const result = await upload.done();
+    if ("Location" in result) logger.debug(`Uploaded: ${result.Bucket}/${result.Key} => ${result.Location}`);
+}
+async function expireOldFile(user, driveCapacity) {
+    const q = DriveFiles.createQueryBuilder("file").where("file.userId = :userId", {
+        userId: user.id
+    }).andWhere("file.isLink = FALSE");
+    if (user.avatarId) {
+        q.andWhere("file.id != :avatarId", {
+            avatarId: user.avatarId
+        });
+    }
+    if (user.bannerId) {
+        q.andWhere("file.id != :bannerId", {
+            bannerId: user.bannerId
+        });
+    }
+    //This selete is hard coded, be careful if change database schema
+    q.addSelect('SUM("file"."size") OVER (ORDER BY "file"."id" DESC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)', "acc_usage");
+    q.orderBy("file.id", "ASC");
+    const fileList = await q.getRawMany();
+    const exceedFileIds = fileList.filter((x)=>x.acc_usage > driveCapacity).map((x)=>x.file_id);
+    for (const fileId of exceedFileIds){
+        const file = await DriveFiles.findOneBy({
+            id: fileId
+        });
+        if (file) deleteFile(file, true);
+    }
+}
+/**
+ * Add file to drive
+ *
+ */ export async function addFile({ user, path, name = null, comment = null, folderId = null, force = false, isLink = false, url = null, uri = null, sensitive = null, isDatabase = false, requestIp = null, requestHeaders = null }) {
+    const info = await getFileInfo(path);
+    logger.info(`${JSON.stringify(info)}`);
+    // 現状 false positive が多すぎて実用に耐えない
+    //if (info.porn && instance.disallowUploadWhenPredictedAsPorn) {
+    //	throw new IdentifiableError('282f77bf-5816-4f72-9264-aa14d8261a21', 'Detected as porn.');
+    //}
+    // detect name
+    const detectedName = name || (info.type.ext ? `untitled.${info.type.ext}` : "untitled");
+    if (detectedName.toLowerCase().endsWith(".epub") && [
+        "application/octet-stream",
+        "application/zip"
+    ].includes(info.type.mime)) {
+        info.type = {
+            mime: "application/epub+zip",
+            ext: "epub"
+        };
+    }
+    if (user && !force) {
+        // Check if there is a file with the same hash
+        const much = await DriveFiles.findOneBy({
+            md5: info.md5,
+            userId: user.id
+        });
+        if (much) {
+            logger.info(`file with same hash is found: ${much.id}`);
+            return much;
+        }
+    }
+    //#region Check drive usage
+    if (user && !isLink) {
+        const u = await Users.findOneBy({
+            id: user.id
+        });
+        const instance = await fetchMeta();
+        if (isDatabase) {
+            const usage = await DriveFiles.calcDatabaseUsageOf(user);
+            const databaseCapacity = 1024 * 1024 * instance.lua4frozenDatabaseCapacityMb;
+            logger.debug(`database usage is ${usage} (max: ${databaseCapacity})`);
+            if (usage + info.size > databaseCapacity) {
+                throw new IdentifiableError("9ec2c4e9-c4df-4f40-b80b-4f57b32d28df", "No free database space.");
+            }
+        } else {
+            const usage = await DriveFiles.calcDriveUsageOf(user);
+            let driveCapacity = 1024 * 1024 * (Users.isLocalUser(user) ? instance.localDriveCapacityMb : instance.remoteDriveCapacityMb);
+            if (Users.isLocalUser(user) && u?.driveCapacityOverrideMb != null) {
+                driveCapacity = 1024 * 1024 * u.driveCapacityOverrideMb;
+                logger.debug("drive capacity override applied");
+                logger.debug(`overrideCap: ${driveCapacity}bytes, usage: ${usage}bytes, u+s: ${usage + info.size}bytes`);
+            }
+            logger.debug(`drive usage is ${usage} (max: ${driveCapacity})`);
+            // If usage limit exceeded
+            if (usage + info.size > driveCapacity) {
+                if (Users.isLocalUser(user)) {
+                    throw new IdentifiableError("c6244ed2-a39a-4e1c-bf93-f0fbd7764fa6", "No free space.");
+                } else {
+                    // (アバターまたはバナーを含まず)最も古いファイルを削除する
+                    expireOldFile(await Users.findOneByOrFail({
+                        id: user.id
+                    }), driveCapacity - info.size);
+                }
+            }
+        }
+    }
+    //#endregion
+    const fetchFolder = async ()=>{
+        if (!folderId) {
+            return null;
+        }
+        const driveFolder = await DriveFolders.findOneBy({
+            id: folderId,
+            userId: user ? user.id : IsNull()
+        });
+        if (driveFolder == null) throw new Error("folder-not-found");
+        return driveFolder;
+    };
+    const properties = {};
+    if (info.width) {
+        properties["width"] = info.width;
+        properties["height"] = info.height;
+    }
+    if (info.orientation != null) {
+        properties["orientation"] = info.orientation;
+    }
+    const profile = user ? await UserProfiles.findOneBy({
+        userId: user.id
+    }) : null;
+    const folder = await fetchFolder();
+    let file = new DriveFile();
+    file.id = genId();
+    file.createdAt = new Date();
+    file.userId = user ? user.id : null;
+    file.userHost = user ? user.host : null;
+    file.folderId = folder !== null ? folder.id : null;
+    file.comment = comment;
+    file.properties = properties;
+    file.blurhash = info.blurhash || null;
+    file.isLink = isLink;
+    file.requestIp = requestIp;
+    file.requestHeaders = requestHeaders;
+    file.isDatabase = isDatabase;
+    file.isSensitive = user ? Users.isLocalUser(user) && profile.alwaysMarkNsfw ? true : sensitive !== null && sensitive !== undefined ? sensitive : false : false;
+    if (url !== null) {
+        file.src = url;
+        if (isLink) {
+            file.url = url;
+            // ローカルプロキシ用
+            file.accessKey = uuid();
+            file.thumbnailAccessKey = `thumbnail-${uuid()}`;
+            file.webpublicAccessKey = `webpublic-${uuid()}`;
+        }
+    }
+    if (uri !== null) {
+        file.uri = uri;
+    }
+    if (isLink) {
+        try {
+            file.size = 0;
+            file.md5 = info.md5;
+            file.name = detectedName;
+            file.type = info.type.mime;
+            file.storedInternal = false;
+            file = await DriveFiles.insert(file).then((x)=>DriveFiles.findOneByOrFail(x.identifiers[0]));
+        } catch (err) {
+            // duplicate key error (when already registered)
+            if (isDuplicateKeyValueError(err)) {
+                logger.info(`already registered ${file.uri}`);
+                file = await DriveFiles.findOneBy({
+                    uri: file.uri,
+                    userId: user ? user.id : IsNull()
+                });
+            } else {
+                logger.error(err);
+                throw err;
+            }
+        }
+    } else {
+        file = await save(file, path, detectedName, info.type.mime, info.md5, info.size);
+    }
+    logger.succ(`drive file has been created ${file.id}`);
+    if (user) {
+        DriveFiles.pack(file, {
+            self: true
+        }).then((packedFile)=>{
+            // Publish driveFileCreated event
+            publishMainStream(user.id, "driveFileCreated", packedFile);
+            publishDriveStream(user.id, "fileCreated", packedFile);
+        });
+    }
+    // 統計を更新
+    driveChart.update(file, true);
+    perUserDriveChart.update(file, true);
+    if (file.userHost !== null) {
+        instanceChart.updateDrive(file, true);
+    }
+    return file;
+}

packages/backend/built/services/drive/delete-file.js

@@ -0,0 +1,92 @@
+import { InternalStorage } from "./internal-storage.js";
+import { DriveFiles, Users } from "../../models/index.js";
+import { driveChart, perUserDriveChart, instanceChart } from "../chart/index.js";
+import { createDeleteObjectStorageFileJob } from "../../queue/index.js";
+import { fetchMeta } from "../../misc/fetch-meta.js";
+import { getS3 } from "./s3.js";
+import { v4 as uuid } from "uuid";
+import { DeleteObjectCommand } from "@aws-sdk/client-s3";
+export async function deleteFile(file, isExpired = false) {
+    if (file.storedInternal) {
+        InternalStorage.del(file.accessKey);
+        if (file.thumbnailUrl) {
+            InternalStorage.del(file.thumbnailAccessKey);
+        }
+        if (file.webpublicUrl) {
+            InternalStorage.del(file.webpublicAccessKey);
+        }
+    } else if (!file.isLink) {
+        createDeleteObjectStorageFileJob(file.accessKey);
+        if (file.thumbnailUrl) {
+            createDeleteObjectStorageFileJob(file.thumbnailAccessKey);
+        }
+        if (file.webpublicUrl) {
+            createDeleteObjectStorageFileJob(file.webpublicAccessKey);
+        }
+    }
+    postProcess(file, isExpired);
+}
+export async function deleteFileSync(file, isExpired = false) {
+    if (file.storedInternal) {
+        InternalStorage.del(file.accessKey);
+        if (file.thumbnailUrl) {
+            InternalStorage.del(file.thumbnailAccessKey);
+        }
+        if (file.webpublicUrl) {
+            InternalStorage.del(file.webpublicAccessKey);
+        }
+    } else if (!file.isLink) {
+        const promises = [];
+        promises.push(deleteObjectStorageFile(file.accessKey));
+        if (file.thumbnailUrl) {
+            promises.push(deleteObjectStorageFile(file.thumbnailAccessKey));
+        }
+        if (file.webpublicUrl) {
+            promises.push(deleteObjectStorageFile(file.webpublicAccessKey));
+        }
+        await Promise.all(promises);
+    }
+    postProcess(file, isExpired);
+}
+async function postProcess(file, isExpired = false) {
+    // リモートファイル期限切れ削除後は直リンクにする
+    if (isExpired && file.userHost !== null && file.uri != null) {
+        DriveFiles.update(file.id, {
+            isLink: true,
+            url: file.uri,
+            thumbnailUrl: null,
+            webpublicUrl: null,
+            storedInternal: false,
+            // ローカルプロキシ用
+            accessKey: uuid(),
+            thumbnailAccessKey: `thumbnail-${uuid()}`,
+            webpublicAccessKey: `webpublic-${uuid()}`
+        });
+        Users.update({
+            avatarId: file.id
+        }, {
+            avatarUrl: file.uri
+        });
+        Users.update({
+            bannerId: file.id
+        }, {
+            bannerUrl: file.uri
+        });
+    } else {
+        DriveFiles.delete(file.id);
+    }
+    // 統計を更新
+    driveChart.update(file, false);
+    perUserDriveChart.update(file, false);
+    if (file.userHost !== null) {
+        instanceChart.updateDrive(file, false);
+    }
+}
+export async function deleteObjectStorageFile(key) {
+    const meta = await fetchMeta();
+    const s3 = getS3(meta);
+    await s3.send(new DeleteObjectCommand({
+        Bucket: meta.objectStorageBucket,
+        Key: key
+    }));
+}

packages/backend/built/services/drive/generate-video-thumbnail.js

@@ -0,0 +1,23 @@
+import { createTempDir } from "../../misc/create-temp.js";
+import { convertToWebp } from "./image-processor.js";
+import FFmpeg from "fluent-ffmpeg";
+export async function GenerateVideoThumbnail(source) {
+    const [dir, cleanup] = await createTempDir();
+    try {
+        await new Promise((res, rej)=>{
+            FFmpeg({
+                source
+            }).on("end", res).on("error", rej).screenshot({
+                folder: dir,
+                filename: "out.png",
+                count: 1,
+                timestamps: [
+                    "5%"
+                ]
+            });
+        });
+        return await convertToWebp(`${dir}/out.png`, 996, 560);
+    } finally{
+        cleanup();
+    }
+}

packages/backend/built/services/drive/image-processor.js

@@ -0,0 +1,20 @@
+import sharp from "sharp";
+/**
+ * Convert to WebP
+ *   with resize, remove metadata, resolve orientation, stop animation
+ */ export async function convertToWebp(path, width, height, quality = 85) {
+    return convertSharpToWebp(await sharp(path), width, height, quality);
+}
+export async function convertSharpToWebp(sharp1, width, height, quality = 85) {
+    const data = await sharp1.resize(width, height, {
+        fit: "inside",
+        withoutEnlargement: true
+    }).rotate().webp({
+        quality
+    }).toBuffer();
+    return {
+        data,
+        ext: "webp",
+        type: "image/webp"
+    };
+}

packages/backend/built/services/drive/internal-storage.js

@@ -0,0 +1,27 @@
+import * as fs from "node:fs";
+import * as Path from "node:path";
+import { pipeline } from "node:stream/promises";
+import config from "../../config/index.js";
+export class InternalStorage {
+    static resolvePath = (key)=>Path.resolve(config.mediaDir, key);
+    static read(key) {
+        return fs.createReadStream(InternalStorage.resolvePath(key));
+    }
+    static async saveFromPath(key, srcPath) {
+        fs.mkdirSync(config.mediaDir, {
+            recursive: true
+        });
+        await pipeline(fs.createReadStream(srcPath), fs.createWriteStream(InternalStorage.resolvePath(key)));
+        return `${config.url}/files/${key}`;
+    }
+    static saveFromBuffer(key, data) {
+        fs.mkdirSync(config.mediaDir, {
+            recursive: true
+        });
+        fs.writeFileSync(InternalStorage.resolvePath(key), data);
+        return `${config.url}/files/${key}`;
+    }
+    static del(key) {
+        fs.unlink(InternalStorage.resolvePath(key), ()=>{});
+    }
+}

packages/backend/built/services/drive/logger.js

@@ -0,0 +1,2 @@
+import Logger from "../logger.js";
+export const driveLogger = new Logger("drive", "blue");

packages/backend/built/services/drive/s3.js

@@ -0,0 +1,25 @@
+import { URL } from "node:url";
+import { getAgentByUrl } from "../../misc/fetch.js";
+import { S3Client } from "@aws-sdk/client-s3";
+import { NodeHttpHandler } from "@smithy/node-http-handler";
+export function getS3(meta) {
+    const endpointPrefixed = meta.objectStorageEndpoint && (meta.objectStorageEndpoint.startsWith("http://") || meta.objectStorageEndpoint.startsWith("https://"));
+    const endpoint = meta.objectStorageEndpoint ? endpointPrefixed ? meta.objectStorageEndpoint : `${meta.objectStorageUseSSL ? "https://" : "http://"}${meta.objectStorageEndpoint}` : undefined;
+    const agentUrl = new URL(endpoint);
+    const agent = getAgentByUrl(agentUrl, !meta.objectStorageUseProxy);
+    return new S3Client({
+        endpoint,
+        region: meta.objectStorageRegion || "us-east-1",
+        credentials: {
+            accessKeyId: meta.objectStorageAccessKey,
+            secretAccessKey: meta.objectStorageSecretKey
+        },
+        forcePathStyle: meta.objectStorageEndpoint ? meta.objectStorageS3ForcePathStyle : false,
+        requestChecksumCalculation: "WHEN_REQUIRED",
+        responseChecksumValidation: "WHEN_REQUIRED",
+        requestHandler: new NodeHttpHandler({
+            httpAgent: agent,
+            httpsAgent: agent
+        })
+    });
+}

packages/backend/built/services/drive/upload-from-url.js

@@ -0,0 +1,48 @@
+import { URL } from "node:url";
+import { createTemp } from "../../misc/create-temp.js";
+import { downloadUrl } from "../../misc/download-url.js";
+import { DriveFiles } from "../../models/index.js";
+import { driveLogger } from "./logger.js";
+import { addFile } from "./add-file.js";
+const logger = driveLogger.createSubLogger("downloader");
+export async function uploadFromUrl({ url, user, folderId = null, uri = null, sensitive = false, force = false, isLink = false, comment = null, requestIp = null, requestHeaders = null }) {
+    let name = new URL(url).pathname.split("/").pop() || null;
+    if (name == null || !DriveFiles.validateFileName(name)) {
+        name = null;
+    }
+    // If the comment is same as the name, skip comment
+    // (image.name is passed in when receiving attachment)
+    if (comment !== null && name === comment) {
+        comment = null;
+    }
+    // Create temp file
+    const [path, cleanup] = await createTemp();
+    try {
+        // write content at URL to temp file
+        await downloadUrl(url, path);
+        const driveFile = await addFile({
+            user,
+            path,
+            name,
+            comment,
+            folderId,
+            force,
+            isLink,
+            url,
+            uri,
+            sensitive,
+            requestIp,
+            requestHeaders
+        });
+        logger.succ(`Got: ${driveFile.id}`);
+        return driveFile;
+    } catch (e) {
+        logger.error(`Failed to create drive file: ${e}`, {
+            url: url,
+            e: e
+        });
+        throw e;
+    } finally{
+        cleanup();
+    }
+}

packages/backend/built/services/fetch-instance-metadata.js

@@ -0,0 +1,192 @@
+import { URL } from "node:url";
+import { JSDOM } from "jsdom";
+import fetch from "node-fetch";
+import tinycolor from "tinycolor2";
+import { getJson, getHtml, getAgentByUrl } from "../misc/fetch.js";
+import { Instances } from "../models/index.js";
+import { getFetchInstanceMetadataLock } from "../misc/app-lock.js";
+import Logger from "./logger.js";
+const logger = new Logger("metadata", "cyan");
+export async function fetchInstanceMetadata(instance, force = false) {
+    const unlock = await getFetchInstanceMetadataLock(instance.host);
+    if (!force) {
+        const _instance = await Instances.findOneBy({
+            host: instance.host
+        });
+        const now = Date.now();
+        if (_instance?.infoUpdatedAt && now - _instance.infoUpdatedAt.getTime() < 1000 * 60 * 60 * 24) {
+            unlock();
+            return;
+        }
+    }
+    logger.info(`Fetching metadata of ${instance.host} ...`);
+    try {
+        const [info, dom, manifest] = await Promise.all([
+            fetchNodeinfo(instance).catch(()=>null),
+            fetchDom(instance).catch(()=>null),
+            fetchManifest(instance).catch(()=>null)
+        ]);
+        const [favicon, icon, themeColor, name, description] = await Promise.all([
+            fetchFaviconUrl(instance, dom).catch(()=>null),
+            fetchIconUrl(instance, dom, manifest).catch(()=>null),
+            getThemeColor(info, dom, manifest).catch(()=>null),
+            getSiteName(info, dom, manifest).catch(()=>null),
+            getDescription(info, dom, manifest).catch(()=>null)
+        ]);
+        logger.succ(`Successfuly fetched metadata of ${instance.host}`);
+        const updates = {
+            infoUpdatedAt: new Date()
+        };
+        if (info) {
+            updates.softwareName = info.software?.name.toLowerCase();
+            updates.softwareVersion = info.software?.version;
+            updates.openRegistrations = info.openRegistrations;
+            updates.maintainerName = info.metadata ? info.metadata.maintainer ? info.metadata.maintainer.name || null : null : null;
+            updates.maintainerEmail = info.metadata ? info.metadata.maintainer ? info.metadata.maintainer.email || null : null : null;
+        }
+        if (name) updates.name = name;
+        if (description) updates.description = description;
+        if (icon || favicon) updates.iconUrl = icon || favicon;
+        if (favicon) updates.faviconUrl = favicon;
+        if (themeColor) updates.themeColor = themeColor;
+        await Instances.update(instance.id, updates);
+        logger.succ(`Successfuly updated metadata of ${instance.host}`);
+    } catch (e) {
+        logger.error(`Failed to update metadata of ${instance.host}: ${e}`);
+    } finally{
+        unlock();
+    }
+}
+async function fetchNodeinfo(instance) {
+    logger.info(`Fetching nodeinfo of ${instance.host} ...`);
+    try {
+        const wellknown = await getJson(`https://${instance.host}/.well-known/nodeinfo`).catch((e)=>{
+            if (e.statusCode === 404) {
+                throw new Error("No nodeinfo provided");
+            } else {
+                throw new Error(e.statusCode || e.message);
+            }
+        });
+        if (wellknown.links == null || !Array.isArray(wellknown.links)) {
+            throw new Error("No wellknown links");
+        }
+        const links = wellknown.links;
+        const lnik1_0 = links.find((link)=>link.rel === "http://nodeinfo.diaspora.software/ns/schema/1.0");
+        const lnik2_0 = links.find((link)=>link.rel === "http://nodeinfo.diaspora.software/ns/schema/2.0");
+        const lnik2_1 = links.find((link)=>link.rel === "http://nodeinfo.diaspora.software/ns/schema/2.1");
+        const link = lnik2_1 || lnik2_0 || lnik1_0;
+        if (link == null) {
+            throw new Error("No nodeinfo link provided");
+        }
+        const info = await getJson(link.href).catch((e)=>{
+            throw new Error(e.statusCode || e.message);
+        });
+        logger.succ(`Successfuly fetched nodeinfo of ${instance.host}`);
+        return info;
+    } catch (e) {
+        logger.error(`Failed to fetch nodeinfo of ${instance.host}: ${e.message}`);
+        throw e;
+    }
+}
+async function fetchDom(instance) {
+    logger.info(`Fetching HTML of ${instance.host} ...`);
+    const url = `https://${instance.host}`;
+    const html = await getHtml(url);
+    const { window } = new JSDOM(html);
+    const doc = window.document;
+    return doc;
+}
+async function fetchManifest(instance) {
+    const url = `https://${instance.host}`;
+    const manifestUrl = `${url}/manifest.json`;
+    const manifest = await getJson(manifestUrl);
+    return manifest;
+}
+async function fetchFaviconUrl(instance, doc) {
+    const url = `https://${instance.host}`;
+    if (doc) {
+        // https://github.com/misskey-dev/misskey/pull/8220#issuecomment-1025104043
+        const href = Array.from(doc.getElementsByTagName("link")).reverse().find((link)=>link.relList.contains("icon"))?.href;
+        if (href) {
+            return new URL(href, url).href;
+        }
+    }
+    const faviconUrl = `${url}/favicon.ico`;
+    const favicon = await fetch(faviconUrl, {
+        // TODO
+        //timeout: 10000,
+        agent: getAgentByUrl,
+        size: 1024 * 1024
+    });
+    if (favicon.ok) {
+        return faviconUrl;
+    }
+    return null;
+}
+async function fetchIconUrl(instance, doc, manifest) {
+    if (manifest?.icons && manifest.icons.length > 0 && manifest.icons[0].src) {
+        const url = `https://${instance.host}`;
+        return new URL(manifest.icons[0].src, url).href;
+    }
+    if (doc) {
+        const url = `https://${instance.host}`;
+        // https://github.com/misskey-dev/misskey/pull/8220#issuecomment-1025104043
+        const links = Array.from(doc.getElementsByTagName("link")).reverse();
+        // https://github.com/misskey-dev/misskey/pull/8220/files/0ec4eba22a914e31b86874f12448f88b3e58dd5a#r796487559
+        const href = [
+            links.find((link)=>link.relList.contains("apple-touch-icon-precomposed"))?.href,
+            links.find((link)=>link.relList.contains("apple-touch-icon"))?.href,
+            links.find((link)=>link.relList.contains("icon"))?.href
+        ].find((href)=>href);
+        if (href) {
+            return new URL(href, url).href;
+        }
+    }
+    return null;
+}
+async function getThemeColor(info, doc, manifest) {
+    const themeColor = info?.metadata?.themeColor || doc?.querySelector('meta[name="theme-color"]')?.getAttribute("content") || manifest?.theme_color;
+    if (themeColor) {
+        const color = new tinycolor(themeColor);
+        if (color.isValid()) return color.toHexString();
+    }
+    return null;
+}
+async function getSiteName(info, doc, manifest) {
+    if (info?.metadata) {
+        if (info.metadata.nodeName || info.metadata.name) {
+            return info.metadata.nodeName || info.metadata.name;
+        }
+    }
+    if (doc) {
+        const og = doc.querySelector('meta[property="og:title"]')?.getAttribute("content");
+        if (og) {
+            return og;
+        }
+    }
+    if (manifest) {
+        return manifest.name || manifest.short_name;
+    }
+    return null;
+}
+async function getDescription(info, doc, manifest) {
+    if (info?.metadata) {
+        if (info.metadata.nodeDescription || info.metadata.description) {
+            return info.metadata.nodeDescription || info.metadata.description;
+        }
+    }
+    if (doc) {
+        const meta = doc.querySelector('meta[name="description"]')?.getAttribute("content");
+        if (meta) {
+            return meta;
+        }
+        const og = doc.querySelector('meta[property="og:description"]')?.getAttribute("content");
+        if (og) {
+            return og;
+        }
+    }
+    if (manifest) {
+        return manifest.name || manifest.short_name;
+    }
+    return null;
+}

packages/backend/built/services/fetch-rel-me.js

@@ -0,0 +1,24 @@
+import { getHtml } from "../misc/fetch.js";
+import { JSDOM } from "jsdom";
+import config from "../config/index.js";
+async function getRelMeLinks(url) {
+    try {
+        const html = await getHtml(url);
+        const dom = new JSDOM(html);
+        const relMeLinks = [
+            ...dom.window.document.querySelectorAll("a[rel='me']"),
+            ...dom.window.document.querySelectorAll("link[rel='me']")
+        ].map((a)=>a.href);
+        return relMeLinks;
+    } catch  {
+        return [];
+    }
+}
+export async function verifyLink(link, username) {
+    let verified = false;
+    if (link.startsWith("http")) {
+        const relMeLinks = await getRelMeLinks(link);
+        verified = relMeLinks.some((href)=>new RegExp(`^https?:\/\/${config.host.replace(/[.*+\-?^${}()|[\]\\]/g, "\\$&")}\/@${username.replace(/[.*+\-?^${}()|[\]\\]/g, "\\$&")}$`).test(href));
+    }
+    return verified;
+}

packages/backend/built/services/following/create.js

@@ -0,0 +1,190 @@
+import { publishMainStream, publishUserEvent } from "../stream.js";
+import { renderActivity } from "../../remote/activitypub/renderer/index.js";
+import renderFollow from "../../remote/activitypub/renderer/follow.js";
+import renderAcceptFollow from "../../remote/activitypub/renderer/accept-follow.js";
+import renderReject from "../../remote/activitypub/renderer/reject.js";
+import { deliver } from "../../queue/index.js";
+import createFollowRequest from "./requests/create.js";
+import { registerOrFetchInstanceDoc } from "../register-or-fetch-instance-doc.js";
+import Logger from "../logger.js";
+import { IdentifiableError } from "../../misc/identifiable-error.js";
+import { Followings, Users, FollowRequests, Blockings, Instances, UserProfiles } from "../../models/index.js";
+import { instanceChart, perUserFollowingChart } from "../chart/index.js";
+import { genId } from "../../misc/gen-id.js";
+import { createNotification } from "../create-notification.js";
+import { isDuplicateKeyValueError } from "../../misc/is-duplicate-key-value-error.js";
+import { getActiveWebhooks } from "../../misc/webhook-cache.js";
+import { webhookDeliver } from "../../queue/index.js";
+import { shouldSilenceInstance } from "../../misc/should-block-instance.js";
+const logger = new Logger("following/create");
+export async function insertFollowingDoc(followee, follower) {
+    if (follower.id === followee.id) return;
+    let alreadyFollowed = false;
+    await Followings.insert({
+        id: genId(),
+        createdAt: new Date(),
+        followerId: follower.id,
+        followeeId: followee.id,
+        // 非正規化
+        followerHost: follower.host,
+        followerInbox: Users.isRemoteUser(follower) ? follower.inbox : null,
+        followerSharedInbox: Users.isRemoteUser(follower) ? follower.sharedInbox : null,
+        followeeHost: followee.host,
+        followeeInbox: Users.isRemoteUser(followee) ? followee.inbox : null,
+        followeeSharedInbox: Users.isRemoteUser(followee) ? followee.sharedInbox : null
+    }).catch((e)=>{
+        if (isDuplicateKeyValueError(e) && Users.isRemoteUser(follower) && Users.isLocalUser(followee)) {
+            logger.info(`Insert duplicated ignore. ${follower.id} => ${followee.id}`);
+            alreadyFollowed = true;
+        } else {
+            throw e;
+        }
+    });
+    const req = await FollowRequests.findOneBy({
+        followeeId: followee.id,
+        followerId: follower.id
+    });
+    if (req) {
+        await FollowRequests.delete({
+            followeeId: followee.id,
+            followerId: follower.id
+        });
+        if (followee.isLocked) {
+            // Create notification that request was accepted.
+            createNotification(follower.id, "followRequestAccepted", {
+                notifierId: followee.id
+            });
+        }
+    }
+    if (alreadyFollowed) return;
+    //#region Increment counts
+    await Promise.all([
+        Users.increment({
+            id: follower.id
+        }, "followingCount", 1),
+        Users.increment({
+            id: followee.id
+        }, "followersCount", 1)
+    ]);
+    //#endregion
+    //#region Update instance stats
+    if (Users.isRemoteUser(follower) && Users.isLocalUser(followee)) {
+        registerOrFetchInstanceDoc(follower.host).then((i)=>{
+            Instances.increment({
+                id: i.id
+            }, "followingCount", 1);
+            instanceChart.updateFollowing(i.host, true);
+        });
+    } else if (Users.isLocalUser(follower) && Users.isRemoteUser(followee)) {
+        registerOrFetchInstanceDoc(followee.host).then((i)=>{
+            Instances.increment({
+                id: i.id
+            }, "followersCount", 1);
+            instanceChart.updateFollowers(i.host, true);
+        });
+    }
+    //#endregion
+    perUserFollowingChart.update(follower, followee, true);
+    // Publish follow event
+    if (Users.isLocalUser(follower)) {
+        Users.pack(followee.id, follower, {
+            detail: true
+        }).then(async (packed)=>{
+            publishUserEvent(follower.id, "follow", packed);
+            publishMainStream(follower.id, "follow", packed);
+            const webhooks = (await getActiveWebhooks()).filter((x)=>x.userId === follower.id && x.on.includes("follow"));
+            for (const webhook of webhooks){
+                webhookDeliver(webhook, "follow", {
+                    user: packed
+                });
+            }
+        });
+    }
+    // Publish followed event
+    if (Users.isLocalUser(followee)) {
+        Users.pack(follower.id, followee).then(async (packed)=>{
+            publishMainStream(followee.id, "followed", packed);
+            const webhooks = (await getActiveWebhooks()).filter((x)=>x.userId === followee.id && x.on.includes("followed"));
+            for (const webhook of webhooks){
+                webhookDeliver(webhook, "followed", {
+                    user: packed
+                });
+            }
+        });
+        // 通知を作成
+        createNotification(followee.id, "follow", {
+            notifierId: follower.id
+        });
+    }
+}
+export default async function(_follower, _followee, requestId) {
+    const [follower, followee] = await Promise.all([
+        Users.findOneByOrFail({
+            id: _follower.id
+        }),
+        Users.findOneByOrFail({
+            id: _followee.id
+        })
+    ]);
+    // check blocking
+    const [blocking, blocked] = await Promise.all([
+        Blockings.findOneBy({
+            blockerId: follower.id,
+            blockeeId: followee.id
+        }),
+        Blockings.findOneBy({
+            blockerId: followee.id,
+            blockeeId: follower.id
+        })
+    ]);
+    if (Users.isRemoteUser(follower) && Users.isLocalUser(followee) && blocked) {
+        // リモートフォローを受けてブロックしていた場合は、エラーにするのではなくRejectを送り返しておしまい。
+        const content = renderActivity(renderReject(renderFollow(follower, followee, requestId), followee));
+        deliver(followee, content, follower.inbox);
+        return;
+    } else if (Users.isRemoteUser(follower) && Users.isLocalUser(followee) && blocking) {
+        // リモートフォローを受けてブロックされているはずの場合だったら、ブロック解除しておく。
+        await Blockings.delete(blocking.id);
+    } else {
+        // それ以外は単純に例外
+        if (blocking) throw new IdentifiableError("710e8fb0-b8c3-4922-be49-d5d93d8e6a6e", "blocking");
+        if (blocked) throw new IdentifiableError("3338392a-f764-498d-8855-db939dcf8c48", "blocked");
+    }
+    const followeeProfile = await UserProfiles.findOneByOrFail({
+        userId: followee.id
+    });
+    // フォロー対象が鍵アカウントである or
+    // The follower is silenced, or
+    // フォロワーがBotであり、フォロー対象がBotからのフォローに慎重である or
+    // フォロワーがローカルユーザーであり、フォロー対象がリモートユーザーである or
+    // The follower is remote, the followee is local, and the follower is in a silenced instance.
+    // 上記のいずれかに当てはまる場合はすぐフォローせずにフォローリクエストを発行しておく
+    if (followee.isLocked || follower.isSilenced || followeeProfile.carefulBot && follower.isBot || Users.isLocalUser(follower) && Users.isRemoteUser(followee) || Users.isRemoteUser(follower) && Users.isLocalUser(followee) && await shouldSilenceInstance(follower.host)) {
+        let autoAccept = false;
+        // 鍵アカウントであっても、既にフォローされていた場合はスルー
+        const following = await Followings.findOneBy({
+            followerId: follower.id,
+            followeeId: followee.id
+        });
+        if (following) {
+            autoAccept = true;
+        }
+        // フォローしているユーザーは自動承認オプション
+        if (!autoAccept && Users.isLocalUser(followee) && followeeProfile.autoAcceptFollowed) {
+            const followed = await Followings.findOneBy({
+                followerId: followee.id,
+                followeeId: follower.id
+            });
+            if (followed) autoAccept = true;
+        }
+        if (!autoAccept) {
+            await createFollowRequest(follower, followee, requestId);
+            return;
+        }
+    }
+    await insertFollowingDoc(followee, follower);
+    if (Users.isRemoteUser(follower) && Users.isLocalUser(followee)) {
+        const content = renderActivity(renderAcceptFollow(follower, followee, requestId));
+        deliver(followee, content, follower.inbox);
+    }
+}

packages/backend/built/services/following/delete.js

@@ -0,0 +1,90 @@
+import { publishMainStream, publishUserEvent } from "../stream.js";
+import { renderActivity } from "../../remote/activitypub/renderer/index.js";
+import renderFollow from "../../remote/activitypub/renderer/follow.js";
+import renderUndo from "../../remote/activitypub/renderer/undo.js";
+import renderReject from "../../remote/activitypub/renderer/reject.js";
+import { deliver, webhookDeliver } from "../../queue/index.js";
+import Logger from "../logger.js";
+import { registerOrFetchInstanceDoc } from "../register-or-fetch-instance-doc.js";
+import { Followings, Users, Instances, UserListJoinings } from "../../models/index.js";
+import { instanceChart, perUserFollowingChart } from "../chart/index.js";
+import { getActiveWebhooks } from "../../misc/webhook-cache.js";
+const logger = new Logger("following/delete");
+export default async function(follower, followee, silent = false) {
+    const following = await Followings.findOneBy({
+        followerId: follower.id,
+        followeeId: followee.id
+    });
+    if (following == null) {
+        logger.warn("フォロー解除がリクエストされましたがフォローしていませんでした");
+        return;
+    }
+    const ids = await UserListJoinings.find({
+        where: {
+            userId: followee.id,
+            userList: {
+                userId: follower.id
+            }
+        },
+        select: [
+            "id"
+        ]
+    }).then((p)=>p.map((x)=>x.id));
+    if (ids.length > 0) await UserListJoinings.delete(ids);
+    await Followings.delete(following.id);
+    decrementFollowing(follower, followee);
+    // Publish unfollow event
+    if (!silent && Users.isLocalUser(follower)) {
+        Users.pack(followee.id, follower, {
+            detail: true
+        }).then(async (packed)=>{
+            publishUserEvent(follower.id, "unfollow", packed);
+            publishMainStream(follower.id, "unfollow", packed);
+            const webhooks = (await getActiveWebhooks()).filter((x)=>x.userId === follower.id && x.on.includes("unfollow"));
+            for (const webhook of webhooks){
+                webhookDeliver(webhook, "unfollow", {
+                    user: packed
+                });
+            }
+        });
+    }
+    if (Users.isLocalUser(follower) && Users.isRemoteUser(followee)) {
+        const content = renderActivity(renderUndo(renderFollow(follower, followee), follower));
+        deliver(follower, content, followee.inbox);
+    }
+    if (Users.isLocalUser(followee) && Users.isRemoteUser(follower)) {
+        // local user has null host
+        const content = renderActivity(renderReject(renderFollow(follower, followee), followee));
+        deliver(followee, content, follower.inbox);
+    }
+}
+export async function decrementFollowing(follower, followee) {
+    //#region Decrement following / followers counts
+    await Promise.all([
+        Users.decrement({
+            id: follower.id
+        }, "followingCount", 1),
+        Users.decrement({
+            id: followee.id
+        }, "followersCount", 1)
+    ]);
+    //#endregion
+    //#region Update instance stats
+    if (Users.isRemoteUser(follower) && Users.isLocalUser(followee)) {
+        registerOrFetchInstanceDoc(follower.host).then((i)=>{
+            Instances.decrement({
+                id: i.id
+            }, "followingCount", 1);
+            instanceChart.updateFollowing(i.host, false);
+        });
+    } else if (Users.isLocalUser(follower) && Users.isRemoteUser(followee)) {
+        registerOrFetchInstanceDoc(followee.host).then((i)=>{
+            Instances.decrement({
+                id: i.id
+            }, "followersCount", 1);
+            instanceChart.updateFollowers(i.host, false);
+        });
+    }
+    //#endregion
+    perUserFollowingChart.update(follower, followee, false);
+}

packages/backend/built/services/following/reject.js

@@ -0,0 +1,83 @@
+import { renderActivity } from "../../remote/activitypub/renderer/index.js";
+import renderFollow from "../../remote/activitypub/renderer/follow.js";
+import renderReject from "../../remote/activitypub/renderer/reject.js";
+import { deliver, webhookDeliver } from "../../queue/index.js";
+import { publishMainStream, publishUserEvent } from "../stream.js";
+import { Users, FollowRequests, Followings } from "../../models/index.js";
+import { decrementFollowing } from "./delete.js";
+import { getActiveWebhooks } from "../../misc/webhook-cache.js";
+/**
+ * API following/request/reject
+ */ export async function rejectFollowRequest(user, follower) {
+    if (Users.isRemoteUser(follower)) {
+        deliverReject(user, follower);
+    }
+    await removeFollowRequest(user, follower);
+    if (Users.isLocalUser(follower)) {
+        publishUnfollow(user, follower);
+    }
+}
+/**
+ * API following/reject
+ */ export async function rejectFollow(user, follower) {
+    if (Users.isRemoteUser(follower)) {
+        deliverReject(user, follower);
+    }
+    await removeFollow(user, follower);
+    if (Users.isLocalUser(follower)) {
+        publishUnfollow(user, follower);
+    }
+}
+/**
+ * AP Reject/Follow
+ */ export async function remoteReject(actor, follower) {
+    await removeFollowRequest(actor, follower);
+    await removeFollow(actor, follower);
+    publishUnfollow(actor, follower);
+}
+/**
+ * Remove follow request record
+ */ async function removeFollowRequest(followee, follower) {
+    const request = await FollowRequests.findOneBy({
+        followeeId: followee.id,
+        followerId: follower.id
+    });
+    if (!request) return;
+    await FollowRequests.delete(request.id);
+}
+/**
+ * Remove follow record
+ */ async function removeFollow(followee, follower) {
+    const following = await Followings.findOneBy({
+        followeeId: followee.id,
+        followerId: follower.id
+    });
+    if (!following) return;
+    await Followings.delete(following.id);
+    decrementFollowing(follower, followee);
+}
+/**
+ * Deliver Reject to remote
+ */ async function deliverReject(followee, follower) {
+    const request = await FollowRequests.findOneBy({
+        followeeId: followee.id,
+        followerId: follower.id
+    });
+    const content = renderActivity(renderReject(renderFollow(follower, followee, request?.requestId || undefined), followee));
+    deliver(followee, content, follower.inbox);
+}
+/**
+ * Publish unfollow to local
+ */ async function publishUnfollow(followee, follower) {
+    const packedFollowee = await Users.pack(followee.id, follower, {
+        detail: true
+    });
+    publishUserEvent(follower.id, "unfollow", packedFollowee);
+    publishMainStream(follower.id, "unfollow", packedFollowee);
+    const webhooks = (await getActiveWebhooks()).filter((x)=>x.userId === follower.id && x.on.includes("unfollow"));
+    for (const webhook of webhooks){
+        webhookDeliver(webhook, "unfollow", {
+            user: packedFollowee
+        });
+    }
+}

packages/backend/built/services/following/requests/accept-all.js

@@ -0,0 +1,16 @@
+import accept from "./accept.js";
+import { FollowRequests, Users } from "../../../models/index.js";
+/**
+ * Approve all follow requests for the specified user
+ * @param user User.
+ */ export default async function(user) {
+    const requests = await FollowRequests.findBy({
+        followeeId: user.id
+    });
+    for (const request of requests){
+        const follower = await Users.findOneByOrFail({
+            id: request.followerId
+        });
+        accept(user, follower);
+    }
+}

packages/backend/built/services/following/requests/accept.js

@@ -0,0 +1,24 @@
+import { renderActivity } from "../../../remote/activitypub/renderer/index.js";
+import renderAcceptFollow from "../../../remote/activitypub/renderer/accept-follow.js";
+import { deliver } from "../../../queue/index.js";
+import { publishMainStream } from "../../stream.js";
+import { insertFollowingDoc } from "../create.js";
+import { FollowRequests, Users } from "../../../models/index.js";
+import { IdentifiableError } from "../../../misc/identifiable-error.js";
+export default async function(followee, follower) {
+    const request = await FollowRequests.findOneBy({
+        followeeId: followee.id,
+        followerId: follower.id
+    });
+    if (request == null) {
+        throw new IdentifiableError("8884c2dd-5795-4ac9-b27e-6a01d38190f9", "No follow request.");
+    }
+    await insertFollowingDoc(followee, follower);
+    if (Users.isRemoteUser(follower) && Users.isLocalUser(followee)) {
+        const content = renderActivity(renderAcceptFollow(follower, followee, request.requestId));
+        deliver(followee, content, follower.inbox);
+    }
+    Users.pack(followee.id, followee, {
+        detail: true
+    }).then((packed)=>publishMainStream(followee.id, "meUpdated", packed));
+}

packages/backend/built/services/following/requests/cancel.js

@@ -0,0 +1,30 @@
+import { renderActivity } from "../../../remote/activitypub/renderer/index.js";
+import renderFollow from "../../../remote/activitypub/renderer/follow.js";
+import renderUndo from "../../../remote/activitypub/renderer/undo.js";
+import { deliver } from "../../../queue/index.js";
+import { publishMainStream } from "../../stream.js";
+import { IdentifiableError } from "../../../misc/identifiable-error.js";
+import { Users, FollowRequests } from "../../../models/index.js";
+export default async function(followee, follower) {
+    if (Users.isRemoteUser(followee)) {
+        const content = renderActivity(renderUndo(renderFollow(follower, followee), follower));
+        if (Users.isLocalUser(follower)) {
+            // 本来このチェックは不要だけどTSに怒られるので
+            deliver(follower, content, followee.inbox);
+        }
+    }
+    const request = await FollowRequests.findOneBy({
+        followeeId: followee.id,
+        followerId: follower.id
+    });
+    if (request == null) {
+        throw new IdentifiableError("17447091-ce07-46dd-b331-c1fd4f15b1e7", "request not found");
+    }
+    await FollowRequests.delete({
+        followeeId: followee.id,
+        followerId: follower.id
+    });
+    Users.pack(followee.id, followee, {
+        detail: true
+    }).then((packed)=>publishMainStream(followee.id, "meUpdated", packed));
+}

packages/backend/built/services/following/requests/create.js

@@ -0,0 +1,54 @@
+import { publishMainStream } from "../../stream.js";
+import { renderActivity } from "../../../remote/activitypub/renderer/index.js";
+import renderFollow from "../../../remote/activitypub/renderer/follow.js";
+import { deliver } from "../../../queue/index.js";
+import { Blockings, FollowRequests, Users } from "../../../models/index.js";
+import { genId } from "../../../misc/gen-id.js";
+import { createNotification } from "../../create-notification.js";
+import config from "../../../config/index.js";
+export default async function(follower, followee, requestId) {
+    if (follower.id === followee.id) return;
+    // check blocking
+    const [blocking, blocked] = await Promise.all([
+        Blockings.findOneBy({
+            blockerId: follower.id,
+            blockeeId: followee.id
+        }),
+        Blockings.findOneBy({
+            blockerId: followee.id,
+            blockeeId: follower.id
+        })
+    ]);
+    if (blocking) throw new Error("blocking");
+    if (blocked) throw new Error("blocked");
+    const followRequest = await FollowRequests.insert({
+        id: genId(),
+        createdAt: new Date(),
+        followerId: follower.id,
+        followeeId: followee.id,
+        requestId,
+        // 非正規化
+        followerHost: follower.host,
+        followerInbox: Users.isRemoteUser(follower) ? follower.inbox : undefined,
+        followerSharedInbox: Users.isRemoteUser(follower) ? follower.sharedInbox : undefined,
+        followeeHost: followee.host,
+        followeeInbox: Users.isRemoteUser(followee) ? followee.inbox : undefined,
+        followeeSharedInbox: Users.isRemoteUser(followee) ? followee.sharedInbox : undefined
+    }).then((x)=>FollowRequests.findOneByOrFail(x.identifiers[0]));
+    // Publish receiveRequest event
+    if (Users.isLocalUser(followee)) {
+        Users.pack(follower.id, followee).then((packed)=>publishMainStream(followee.id, "receiveFollowRequest", packed));
+        Users.pack(followee.id, followee, {
+            detail: true
+        }).then((packed)=>publishMainStream(followee.id, "meUpdated", packed));
+        // 通知を作成
+        createNotification(followee.id, "receiveFollowRequest", {
+            notifierId: follower.id,
+            followRequestId: followRequest.id
+        });
+    }
+    if (Users.isLocalUser(follower) && Users.isRemoteUser(followee)) {
+        const content = renderActivity(renderFollow(follower, followee, requestId ?? `${config.url}/follows/${followRequest.id}`));
+        deliver(follower, content, followee.inbox);
+    }
+}

packages/backend/built/services/i/pin.js

@@ -0,0 +1,76 @@
+import config from "../../config/index.js";
+import renderAdd from "../../remote/activitypub/renderer/add.js";
+import renderRemove from "../../remote/activitypub/renderer/remove.js";
+import { renderActivity } from "../../remote/activitypub/renderer/index.js";
+import { IdentifiableError } from "../../misc/identifiable-error.js";
+import { Notes, UserNotePinings, Users } from "../../models/index.js";
+import { genId } from "../../misc/gen-id.js";
+import { deliverToFollowers } from "../../remote/activitypub/deliver-manager.js";
+import { deliverToRelays } from "../relay.js";
+/**
+ * 指定した投稿をピン留めします
+ * @param user
+ * @param noteId
+ */ export async function addPinned(user, noteId) {
+    // Fetch pinee
+    const note = await Notes.findOneBy({
+        id: noteId,
+        userId: user.id
+    });
+    if (note == null) {
+        throw new IdentifiableError("70c4e51f-5bea-449c-a030-53bee3cce202", "No such note.");
+    }
+    const pinings = await UserNotePinings.findBy({
+        userId: user.id
+    });
+    if (pinings.length >= 5) {
+        throw new IdentifiableError("15a018eb-58e5-4da1-93be-330fcc5e4e1a", "You can not pin notes any more.");
+    }
+    if (pinings.some((pining)=>pining.noteId === note.id)) {
+        throw new IdentifiableError("23f0cf4e-59a3-4276-a91d-61a5891c1514", "That note has already been pinned.");
+    }
+    await UserNotePinings.insert({
+        id: genId(),
+        createdAt: new Date(),
+        userId: user.id,
+        noteId: note.id
+    });
+    // Deliver to remote followers
+    if (Users.isLocalUser(user)) {
+        deliverPinnedChange(user.id, note.id, true);
+    }
+}
+/**
+ * 指定した投稿のピン留めを解除します
+ * @param user
+ * @param noteId
+ */ export async function removePinned(user, noteId) {
+    // Fetch unpinee
+    const note = await Notes.findOneBy({
+        id: noteId,
+        userId: user.id
+    });
+    if (note == null) {
+        throw new IdentifiableError("b302d4cf-c050-400a-bbb3-be208681f40c", "No such note.");
+    }
+    UserNotePinings.delete({
+        userId: user.id,
+        noteId: note.id
+    });
+    // Deliver to remote followers
+    if (Users.isLocalUser(user)) {
+        deliverPinnedChange(user.id, noteId, false);
+    }
+}
+export async function deliverPinnedChange(userId, noteId, isAddition) {
+    const user = await Users.findOneBy({
+        id: userId
+    });
+    if (user == null) throw new Error("user not found");
+    if (!Users.isLocalUser(user)) return;
+    const target = `${config.url}/users/${user.id}/collections/featured`;
+    const item = `${config.url}/notes/${noteId}`;
+    const content = renderActivity(isAddition ? renderAdd(user, target, item) : renderRemove(user, target, item));
+    deliverToFollowers(user, content);
+    deliverToRelays(user, content);
+}

packages/backend/built/services/i/update.js

@@ -0,0 +1,81 @@
+import renderUpdate from "../../remote/activitypub/renderer/update.js";
+import { renderActivity } from "../../remote/activitypub/renderer/index.js";
+import { UserProfiles, Users } from "../../models/index.js";
+import { renderPerson } from "../../remote/activitypub/renderer/person.js";
+import { deliverToFollowers } from "../../remote/activitypub/deliver-manager.js";
+import { deliverToRelays } from "../relay.js";
+import { extractCustomEmojisFromMfm } from "../../misc/extract-custom-emojis-from-mfm.js";
+import { extractHashtags } from "../../misc/extract-hashtags.js";
+import { normalizeForSearch } from "../../misc/normalize-for-search.js";
+import { updateUsertags } from "../update-hashtag.js";
+import { publishMainStream, publishUserEvent } from "../stream.js";
+import acceptAllFollowRequests from "../following/requests/accept-all.js";
+import mfm from "mfm-js";
+import { promiseEarlyReturn } from "../../prelude/promise.js";
+import { UserConverter } from "../../server/api/mastodon/converters/user.js";
+export async function updateUserProfileData(user, profile, updates, profileUpdates, isSecure) {
+    if (!profile) profile = await UserProfiles.findOneByOrFail({
+        userId: user.id
+    });
+    let emojis = [];
+    let tags = [];
+    const newName = updates.name === undefined ? user.name : updates.name;
+    const newDescription = profileUpdates.description === undefined ? profile.description : profileUpdates.description;
+    const newFields = profileUpdates.fields === undefined ? profile.fields : profileUpdates.fields;
+    if (newName != null) {
+        const tokens = mfm.parseSimple(newName);
+        emojis.push(...extractCustomEmojisFromMfm(tokens));
+    }
+    if (newDescription != null) {
+        const tokens = mfm.parse(newDescription);
+        emojis.push(...extractCustomEmojisFromMfm(tokens));
+        tags = extractHashtags(tokens).map((tag)=>normalizeForSearch(tag)).splice(0, 32);
+    }
+    for (const field of newFields || []){
+        const nameTokens = mfm.parse(field.name);
+        emojis.push(...extractCustomEmojisFromMfm(nameTokens));
+        const valueTokens = mfm.parse(field.value);
+        emojis.push(...extractCustomEmojisFromMfm(valueTokens));
+    }
+    updates.emojis = [
+        ...new Set(emojis)
+    ];
+    updates.tags = tags;
+    updates.updatedAt = new Date();
+    updateUsertags(user, tags);
+    const oldProfile = await UserProfiles.findOneBy({
+        userId: user.id
+    });
+    if (Object.keys(updates).length > 0) await Users.update(user.id, updates);
+    if (Object.keys(profileUpdates).length > 0) {
+        await UserProfiles.update(user.id, profileUpdates);
+    }
+    const iObj = await Users.pack(user.id, user, {
+        detail: true,
+        includeSecrets: isSecure
+    });
+    publishMainStream(user.id, "meUpdated", iObj);
+    publishUserEvent(user.id, "updateUserProfile", await UserProfiles.findOneByOrFail({
+        userId: user.id
+    }));
+    if (user.isLocked && updates.isLocked === false) {
+        acceptAllFollowRequests(user);
+    }
+    await promiseEarlyReturn(UserProfiles.updateMentions(user.id).finally(()=>{
+        UserConverter.prewarmCacheById(user.id, oldProfile);
+        publishToFollowers(user.id);
+    }), 1500);
+    return iObj;
+}
+export async function publishToFollowers(userId) {
+    const user = await Users.findOneBy({
+        id: userId
+    });
+    if (user == null) throw new Error("user not found");
+    // フォロワーがリモートユーザーかつ投稿者がローカルユーザーならUpdateを配信
+    if (Users.isLocalUser(user)) {
+        const content = renderActivity(renderUpdate(await renderPerson(user), user));
+        deliverToFollowers(user, content);
+        deliverToRelays(user, content);
+    }
+}

packages/backend/built/services/insert-moderation-log.js

@@ -0,0 +1,11 @@
+import { ModerationLogs } from "../models/index.js";
+import { genId } from "../misc/gen-id.js";
+export async function insertModerationLog(moderator, type, info) {
+    await ModerationLogs.insert({
+        id: genId(),
+        createdAt: new Date(),
+        userId: moderator.id,
+        type: type,
+        info: info || {}
+    });
+}

packages/backend/built/services/instance-actor.js

@@ -0,0 +1,22 @@
+import { createSystemUser } from "./create-system-user.js";
+import { Users } from "../models/index.js";
+import { Cache } from "../misc/cache.js";
+import { IsNull } from "typeorm";
+const ACTOR_USERNAME = "instance.actor";
+const cache = new Cache("instanceActor", 60 * 30);
+export async function getInstanceActor() {
+    const cached = await cache.get(null, true);
+    if (cached) return cached;
+    const user = await Users.findOneBy({
+        host: IsNull(),
+        username: ACTOR_USERNAME
+    });
+    if (user) {
+        await cache.set(null, user);
+        return user;
+    } else {
+        const created = await createSystemUser(ACTOR_USERNAME);
+        await cache.set(null, created);
+        return created;
+    }
+}

packages/backend/built/services/logger.js

@@ -0,0 +1,95 @@
+import cluster from "node:cluster";
+import chalk from "chalk";
+import { default as convertColor } from "color-convert";
+import { format as dateFormat } from "date-fns";
+import { envOption } from "../env.js";
+import config from "../config/index.js";
+import * as SyslogPro from "syslog-pro";
+export default class Logger {
+    domain;
+    parentLogger = null;
+    store;
+    syslogClient = null;
+    constructor(domain, color, store = true){
+        this.domain = {
+            name: domain,
+            color: color
+        };
+        this.store = store;
+        if (config.syslog) {
+            this.syslogClient = new SyslogPro.RFC5424({
+                applacationName: "FrozenFriendsYume",
+                timestamp: true,
+                encludeStructuredData: true,
+                color: true,
+                extendedColor: true,
+                server: {
+                    target: config.syslog.host,
+                    port: config.syslog.port
+                }
+            });
+        }
+    }
+    createSubLogger(domain, color, store = true) {
+        const logger = new Logger(domain, color, store);
+        logger.parentLogger = this;
+        return logger;
+    }
+    log(level, message, data, important = false, subDomains = [], store = true) {
+        if (envOption.quiet) return;
+        if (!this.store) store = false;
+        if (level === "debug") store = false;
+        if (this.parentLogger) {
+            this.parentLogger.log(level, message, data, important, [
+                this.domain
+            ].concat(subDomains), store);
+            return;
+        }
+        const time = dateFormat(new Date(), "HH:mm:ss");
+        const worker = cluster.isPrimary ? "*" : cluster.worker.id;
+        const l = level === "error" ? important ? chalk.bgRed.white("ERR ") : chalk.red("ERR ") : level === "warning" ? chalk.yellow("WARN") : level === "success" ? important ? chalk.bgGreen.white("DONE") : chalk.green("DONE") : level === "debug" ? chalk.gray("VERB") : level === "info" ? chalk.blue("INFO") : null;
+        const domains = [
+            this.domain
+        ].concat(subDomains).map((d)=>d.color ? chalk.rgb(...convertColor.keyword.rgb(d.color))(d.name) : chalk.white(d.name));
+        const m = level === "error" ? chalk.red(message) : level === "warning" ? chalk.yellow(message) : level === "success" ? chalk.green(message) : level === "debug" ? chalk.gray(message) : level === "info" ? message : null;
+        let log = `${l} ${worker}\t[${domains.join(" ")}]\t${m}`;
+        if (envOption.withLogTime) log = `${chalk.gray(time)} ${log}`;
+        console.log(important ? chalk.bold(log) : log);
+        if (store) {
+            if (this.syslogClient) {
+                const send = level === "error" ? this.syslogClient.error : level === "warning" ? this.syslogClient.warning : level === "success" ? this.syslogClient.info : level === "debug" ? this.syslogClient.info : level === "info" ? this.syslogClient.info : null;
+                send.bind(this.syslogClient)(message).catch(()=>{});
+            }
+        }
+    }
+    error(x, data, important = false) {
+        // 実行を継続できない状況で使う
+        if (x instanceof Error) {
+            data = data || {};
+            data.e = x;
+            this.log("error", x.toString(), data, important);
+        } else if (typeof x === "object") {
+            this.log("error", `${x.message || x.name || x}`, data, important);
+        } else {
+            this.log("error", `${x}`, data, important);
+        }
+    }
+    warn(message, data, important = false) {
+        // 実行を継続できるが改善すべき状況で使う
+        this.log("warning", message, data, important);
+    }
+    succ(message, data, important = false) {
+        // 何かに成功した状況で使う
+        this.log("success", message, data, important);
+    }
+    debug(message, data, important = false) {
+        // デバッグ用に使う(開発者に必要だが利用者に不要な情報)
+        if (process.env.NODE_ENV !== "production" || envOption.verbose) {
+            this.log("debug", message, data, important);
+        }
+    }
+    info(message, data, important = false) {
+        // それ以外
+        this.log("info", message, data, important);
+    }
+}

packages/backend/built/services/messages/create.js

@@ -0,0 +1,103 @@
+import { MessagingMessages, UserGroupJoinings, Mutings, Users } from "../../models/index.js";
+import { genId } from "../../misc/gen-id.js";
+import { publishMessagingStream, publishMessagingIndexStream, publishMainStream, publishGroupMessagingStream } from "../stream.js";
+import { pushNotification } from "../push-notification.js";
+import { Not } from "typeorm";
+import renderNote from "../../remote/activitypub/renderer/note.js";
+import renderCreate from "../../remote/activitypub/renderer/create.js";
+import { renderActivity } from "../../remote/activitypub/renderer/index.js";
+import { deliver } from "../../queue/index.js";
+export async function createMessage(user, recipientUser, recipientGroup, text, file, uri) {
+    const message = {
+        id: genId(),
+        createdAt: new Date(),
+        fileId: file ? file.id : null,
+        recipientId: recipientUser ? recipientUser.id : null,
+        groupId: recipientGroup ? recipientGroup.id : null,
+        text: text ? text.trim() : null,
+        userId: user.id,
+        isRead: false,
+        reads: [],
+        uri
+    };
+    await MessagingMessages.insert(message);
+    const messageObj = await MessagingMessages.pack(message);
+    if (recipientUser) {
+        if (Users.isLocalUser(user)) {
+            // 自分のストリーム
+            publishMessagingStream(message.userId, recipientUser.id, "message", messageObj);
+            publishMessagingIndexStream(message.userId, "message", messageObj);
+            publishMainStream(message.userId, "messagingMessage", messageObj);
+        }
+        if (Users.isLocalUser(recipientUser)) {
+            // 相手のストリーム
+            publishMessagingStream(recipientUser.id, message.userId, "message", messageObj);
+            publishMessagingIndexStream(recipientUser.id, "message", messageObj);
+            publishMainStream(recipientUser.id, "messagingMessage", messageObj);
+        }
+    } else if (recipientGroup) {
+        // グループのストリーム
+        publishGroupMessagingStream(recipientGroup.id, "message", messageObj);
+        // メンバーのストリーム
+        const joinings = await UserGroupJoinings.findBy({
+            userGroupId: recipientGroup.id
+        });
+        for (const joining of joinings){
+            publishMessagingIndexStream(joining.userId, "message", messageObj);
+            publishMainStream(joining.userId, "messagingMessage", messageObj);
+        }
+    }
+    // 2秒経っても(今回作成した)メッセージが既読にならなかったら「未読のメッセージがありますよ」イベントを発行する
+    setTimeout(async ()=>{
+        const freshMessage = await MessagingMessages.findOneBy({
+            id: message.id
+        });
+        if (freshMessage == null) return; // メッセージが削除されている場合もある
+        if (recipientUser && Users.isLocalUser(recipientUser)) {
+            if (freshMessage.isRead) return; // 既読
+            //#region ただしミュートされているなら発行しない
+            const mute = await Mutings.findBy({
+                muterId: recipientUser.id
+            });
+            if (mute.map((m)=>m.muteeId).includes(user.id)) return;
+            //#endregion
+            publishMainStream(recipientUser.id, "unreadMessagingMessage", messageObj);
+            pushNotification(recipientUser.id, "unreadMessagingMessage", messageObj);
+        } else if (recipientGroup) {
+            const joinings = await UserGroupJoinings.findBy({
+                userGroupId: recipientGroup.id,
+                userId: Not(user.id)
+            });
+            for (const joining of joinings){
+                if (freshMessage.reads.includes(joining.userId)) return; // 既読
+                publishMainStream(joining.userId, "unreadMessagingMessage", messageObj);
+                pushNotification(joining.userId, "unreadMessagingMessage", messageObj);
+            }
+        }
+    }, 2000);
+    if (recipientUser && Users.isLocalUser(user) && Users.isRemoteUser(recipientUser)) {
+        const note = {
+            id: message.id,
+            createdAt: message.createdAt,
+            fileIds: message.fileId ? [
+                message.fileId
+            ] : [],
+            text: message.text,
+            userId: message.userId,
+            visibility: "specified",
+            mentions: [
+                recipientUser
+            ].map((u)=>u.id),
+            mentionedRemoteUsers: JSON.stringify([
+                recipientUser
+            ].map((u)=>({
+                    uri: u.uri,
+                    username: u.username,
+                    host: u.host
+                })))
+        };
+        const activity = renderActivity(renderCreate(await renderNote(note, false, true), note));
+        deliver(user, activity, recipientUser.inbox);
+    }
+    return messageObj;
+}

packages/backend/built/services/messages/delete.js

@@ -0,0 +1,29 @@
+import config from "../../config/index.js";
+import { MessagingMessages, Users } from "../../models/index.js";
+import { publishGroupMessagingStream, publishMessagingStream } from "../stream.js";
+import { renderActivity } from "../../remote/activitypub/renderer/index.js";
+import renderDelete from "../../remote/activitypub/renderer/delete.js";
+import renderTombstone from "../../remote/activitypub/renderer/tombstone.js";
+import { deliver } from "../../queue/index.js";
+export async function deleteMessage(message) {
+    await MessagingMessages.delete(message.id);
+    postDeleteMessage(message);
+}
+async function postDeleteMessage(message) {
+    if (message.recipientId) {
+        const user = await Users.findOneByOrFail({
+            id: message.userId
+        });
+        const recipient = await Users.findOneByOrFail({
+            id: message.recipientId
+        });
+        if (Users.isLocalUser(user)) publishMessagingStream(message.userId, message.recipientId, "deleted", message.id);
+        if (Users.isLocalUser(recipient)) publishMessagingStream(message.recipientId, message.userId, "deleted", message.id);
+        if (Users.isLocalUser(user) && Users.isRemoteUser(recipient)) {
+            const activity = renderActivity(renderDelete(renderTombstone(`${config.url}/notes/${message.id}`), user));
+            deliver(user, activity, recipient.inbox);
+        }
+    } else if (message.groupId) {
+        publishGroupMessagingStream(message.groupId, "deleted", message.id);
+    }
+}

packages/backend/built/services/note/create-from-api.js

@@ -0,0 +1,185 @@
+import { In } from "typeorm";
+import create from "./create.js";
+import { Users, DriveFiles, Channels, Blockings, UserGroups, UserGroupJoinings } from "../../models/index.js";
+import { getNote } from "../../server/api/common/getters.js";
+import { ApiError } from "../../server/api/error.js";
+const errors = {
+    noSuchRenoteTarget: {
+        message: "No such renote target.",
+        code: "NO_SUCH_RENOTE_TARGET",
+        id: "b5c90186-4ab0-49c8-9bba-a1f76c282ba4"
+    },
+    cannotReRenote: {
+        message: "You can not Renote a pure Renote.",
+        code: "CANNOT_RENOTE_TO_A_PURE_RENOTE",
+        id: "fd4cc33e-2a37-48dd-99cc-9b806eb2031a"
+    },
+    noSuchReplyTarget: {
+        message: "No such reply target.",
+        code: "NO_SUCH_REPLY_TARGET",
+        id: "749ee0f6-d3da-459a-bf02-282e2da4292c"
+    },
+    cannotReplyToPureRenote: {
+        message: "You can not reply to a pure Renote.",
+        code: "CANNOT_REPLY_TO_A_PURE_RENOTE",
+        id: "3ac74a84-8fd5-4bb0-870f-01804f82ce15"
+    },
+    cannotCreateAlreadyExpiredPoll: {
+        message: "Poll is already expired.",
+        code: "CANNOT_CREATE_ALREADY_EXPIRED_POLL",
+        id: "04da457d-b083-4055-9082-955525eda5a5"
+    },
+    noSuchChannel: {
+        message: "No such channel.",
+        code: "NO_SUCH_CHANNEL",
+        id: "b1653923-5453-4edc-b786-7c4f39bb0bbb"
+    },
+    youHaveBeenBlocked: {
+        message: "You have been blocked by this user.",
+        code: "YOU_HAVE_BEEN_BLOCKED",
+        id: "b390d7e1-8a5e-46ed-b625-06271cafd3d3"
+    },
+    noSuchGroup: {
+        message: "No such group.",
+        code: "NO_SUCH_GROUP",
+        id: "b6344dfc-7c1d-4238-8cc0-0f719c4f9d91"
+    }
+};
+export async function createNoteFromApiData(user, data, createdAt = new Date()) {
+    let visibleUsers = [];
+    if (data.visibleUserIds) {
+        visibleUsers = await Users.findBy({
+            id: In(data.visibleUserIds)
+        });
+    }
+    let files = [];
+    const fileIds = data.fileIds != null ? data.fileIds : data.mediaIds != null ? data.mediaIds : null;
+    if (fileIds != null) {
+        files = await DriveFiles.createQueryBuilder("file").where("file.userId = :userId AND file.id IN (:...fileIds)", {
+            userId: user.id,
+            fileIds
+        }).orderBy('array_position(ARRAY[:...fileIds], "id"::text)').setParameters({
+            fileIds
+        }).getMany();
+    }
+    let renote = null;
+    if (data.renoteId != null) {
+        renote = await getNote(data.renoteId, user).catch((e)=>{
+            if (e.id === "9725d0ce-ba28-4dde-95a7-2cbb2c15de24") throw new ApiError(errors.noSuchRenoteTarget);
+            throw e;
+        });
+        if (renote.renoteId && !renote.text && !renote.fileIds && !renote.hasPoll) {
+            throw new ApiError(errors.cannotReRenote);
+        }
+        if (renote.userId !== user.id) {
+            const isBlocked = await Blockings.exist({
+                where: [
+                    {
+                        blockerId: renote.userId,
+                        blockeeId: user.id,
+                        groupId: null
+                    },
+                    ...renote.groupId ? [
+                        {
+                            groupId: renote.groupId,
+                            blockeeId: user.id
+                        }
+                    ] : []
+                ]
+            });
+            if (isBlocked) {
+                throw new ApiError(errors.youHaveBeenBlocked);
+            }
+        }
+    }
+    let reply = null;
+    if (data.replyId != null) {
+        reply = await getNote(data.replyId, user).catch((e)=>{
+            if (e.id === "9725d0ce-ba28-4dde-95a7-2cbb2c15de24") throw new ApiError(errors.noSuchReplyTarget);
+            throw e;
+        });
+        if (reply.renoteId && !reply.text && !reply.fileIds && !reply.hasPoll) {
+            throw new ApiError(errors.cannotReplyToPureRenote);
+        }
+        if (reply.userId !== user.id) {
+            const isBlocked = await Blockings.exist({
+                where: [
+                    {
+                        blockerId: reply.userId,
+                        blockeeId: user.id,
+                        groupId: null
+                    },
+                    ...reply.groupId ? [
+                        {
+                            groupId: reply.groupId,
+                            blockeeId: user.id
+                        }
+                    ] : []
+                ]
+            });
+            if (isBlocked) {
+                throw new ApiError(errors.youHaveBeenBlocked);
+            }
+        }
+    }
+    const poll = data.poll ? {
+        ...data.poll
+    } : null;
+    if (poll) {
+        if (typeof poll.expiresAt === "number") {
+            if (poll.expiresAt < createdAt.getTime()) {
+                throw new ApiError(errors.cannotCreateAlreadyExpiredPoll);
+            }
+        } else if (typeof poll.expiredAfter === "number") {
+            poll.expiresAt = createdAt.getTime() + poll.expiredAfter;
+        }
+    }
+    let channel = null;
+    if (data.channelId != null) {
+        channel = await Channels.findOneBy({
+            id: data.channelId
+        });
+        if (channel == null) {
+            throw new ApiError(errors.noSuchChannel);
+        }
+    }
+    let group = null;
+    if (data.groupId != null) {
+        group = await UserGroups.findOneBy({
+            id: data.groupId
+        });
+        if (group == null) {
+            throw new ApiError(errors.noSuchGroup);
+        }
+        if (group.userId !== user.id) {
+            const joining = await UserGroupJoinings.findOneBy({
+                userId: user.id,
+                userGroupId: group.id
+            });
+            if (joining == null) {
+                throw new ApiError(errors.noSuchGroup);
+            }
+        }
+    }
+    return await create(user, {
+        createdAt,
+        files,
+        poll: poll ? {
+            choices: poll.choices,
+            multiple: poll.multiple,
+            expiresAt: poll.expiresAt ? new Date(poll.expiresAt) : null
+        } : undefined,
+        text: data.text || undefined,
+        reply,
+        renote,
+        cw: data.cw,
+        localOnly: data.localOnly,
+        visibility: data.visibility ?? "public",
+        visibleUsers,
+        channel,
+        group,
+        apMentions: data.noExtractMentions ? [] : undefined,
+        apHashtags: data.noExtractHashtags ? [] : undefined,
+        apEmojis: data.noExtractEmojis ? [] : undefined
+    });
+}

packages/backend/built/services/note/create.js

@@ -0,0 +1,656 @@
+import * as mfm from "mfm-js";
+import { publishMainStream, publishNotesStream, publishNoteStream } from "../stream.js";
+import DeliverManager, { deliverToUser } from "../../remote/activitypub/deliver-manager.js";
+import renderNote from "../../remote/activitypub/renderer/note.js";
+import renderCreate from "../../remote/activitypub/renderer/create.js";
+import renderAnnounce from "../../remote/activitypub/renderer/announce.js";
+import { renderActivity } from "../../remote/activitypub/renderer/index.js";
+import { resolveUser } from "../../remote/resolve-user.js";
+import config from "../../config/index.js";
+import { updateHashtags } from "../update-hashtag.js";
+import { concat } from "../../prelude/array.js";
+import { insertNoteUnread } from "./unread.js";
+import { registerOrFetchInstanceDoc } from "../register-or-fetch-instance-doc.js";
+import { extractMentions } from "../../misc/extract-mentions.js";
+import { extractCustomEmojiNamesFromText, extractCustomEmojisFromMfm } from "../../misc/extract-custom-emojis-from-mfm.js";
+import { extractGroupMentionedUsers } from "../../misc/extract-group-mentions.js";
+import { extractHashtags } from "../../misc/extract-hashtags.js";
+import { Note } from "../../models/entities/note.js";
+import { Mutings, Users, DriveFiles, NoteWatchings, Notes, Instances, UserProfiles, Channels, ChannelFollowings, NoteThreadMutings, InteractionStamps } from "../../models/index.js";
+import { Not, In } from "typeorm";
+import { genId } from "../../misc/gen-id.js";
+import { notesChart, perUserNotesChart, activeUsersChart, instanceChart } from "../chart/index.js";
+import { Poll } from "../../models/entities/poll.js";
+import { createNotification } from "../create-notification.js";
+import { isDuplicateKeyValueError } from "../../misc/is-duplicate-key-value-error.js";
+import { checkHitAntenna } from "../../misc/check-hit-antenna.js";
+import { addNoteToAntenna } from "../add-note-to-antenna.js";
+import { countSameRenotes } from "../../misc/count-same-renotes.js";
+import { deliverToRelays, getCachedRelays } from "../relay.js";
+import { normalizeForSearch } from "../../misc/normalize-for-search.js";
+import { getAntennas } from "../../misc/antenna-cache.js";
+import { endedPollNotificationQueue } from "../../queue/queues.js";
+import { webhookDeliver } from "../../queue/index.js";
+import { db } from "../../db/postgre.js";
+import { getActiveWebhooks } from "../../misc/webhook-cache.js";
+import { shouldSilenceInstance } from "../../misc/should-block-instance.js";
+import { redisClient } from "../../db/redis.js";
+import { Mutex } from "redis-semaphore";
+import { RecursionLimiter } from "../../models/repositories/user-profile.js";
+import { NoteConverter } from "../../server/api/mastodon/converters/note.js";
+import { defaultJobOpts } from "../../queue/queues/index.js";
+import renderQuoteRequest from "../../remote/activitypub/renderer/quote-request.js";
+class NotificationManager {
+    notifier;
+    note;
+    queue;
+    constructor(notifier, note){
+        this.notifier = notifier;
+        this.note = note;
+        this.queue = [];
+    }
+    push(notifiee, reason) {
+        // 自分自身へは通知しない
+        if (this.notifier.id === notifiee) return;
+        const exist = this.queue.find((x)=>x.target === notifiee);
+        if (exist) {
+            // 「メンションされているかつ返信されている」場合は、メンションとしての通知ではなく返信としての通知にする
+            if (reason !== "mention") {
+                exist.reason = reason;
+            }
+        } else {
+            this.queue.push({
+                reason: reason,
+                target: notifiee
+            });
+        }
+    }
+    async deliver() {
+        for (const x of this.queue){
+            // ミュート情報を取得
+            const mentioneeMutes = await Mutings.findBy({
+                muterId: x.target
+            });
+            const mentioneesMutedUserIds = mentioneeMutes.map((m)=>m.muteeId);
+            // 通知される側のユーザーが通知する側のユーザーをミュートしていない限りは通知する
+            if (!mentioneesMutedUserIds.includes(this.notifier.id)) {
+                createNotification(x.target, x.reason, {
+                    notifierId: this.notifier.id,
+                    noteId: this.note.id,
+                    note: this.note
+                });
+            }
+        }
+    }
+}
+export default (async (user, data, silent = false, limiter = new RecursionLimiter())=>// rome-ignore lint/suspicious/noAsyncPromiseExecutor: FIXME
+    new Promise(async (res, rej)=>{
+        const dontFederateInitially = data.visibility === "hidden";
+        // If you reply outside the channel, match the scope of the target.
+        // TODO (I think it's a process that could be done on the client side, but it's server side for now.)
+        if (data.reply && data.channel && data.reply.channelId !== data.channel.id) {
+            if (data.reply.channelId) {
+                data.channel = await Channels.findOneBy({
+                    id: data.reply.channelId
+                });
+            } else {
+                data.channel = null;
+            }
+        }
+        // When you reply in a channel, match the scope of the target
+        // TODO (I think it's a process that could be done on the client side, but it's server side for now.)
+        if (data.reply && data.channel == null && data.reply.channelId) {
+            data.channel = await Channels.findOneBy({
+                id: data.reply.channelId
+            });
+        }
+        const now = new Date();
+        if (!data.createdAt || isNaN(data.createdAt.getTime()) || data.createdAt > now) data.createdAt = now;
+        if (data.visibility == null) data.visibility = "public";
+        if (data.localOnly == null) data.localOnly = false;
+        if (data.channel != null) data.visibility = "public";
+        if (data.channel != null) data.visibleUsers = [];
+        if (data.channel != null) data.localOnly = true;
+        if (data.visibility === "hidden") data.visibility = "public";
+        // enforce silent clients on server
+        if (user.isSilenced && data.visibility === "public" && data.channel == null) {
+            data.visibility = "home";
+        }
+        // Enforce home visibility if the user is in a silenced instance.
+        if (data.visibility === "public" && Users.isRemoteUser(user) && await shouldSilenceInstance(user.host)) {
+            data.visibility = "home";
+        }
+        // Reject if the target of the renote is a public range other than "Home or Entire".
+        if (data.renote && data.renote.visibility !== "public" && data.renote.visibility !== "home" && data.renote.userId !== user.id) {
+            return rej("Renote target is not public or home");
+        }
+        // If the target of the renote is not public, make it home.
+        if (data.renote && data.renote.visibility !== "public" && data.visibility === "public") {
+            data.visibility = "home";
+        }
+        // If the target of Renote is followers, make it followers.
+        if (data.renote && data.renote.visibility === "followers") {
+            data.visibility = "followers";
+        }
+        // If the reply target is not public, make it home.
+        if (data.reply && data.reply.visibility !== "public" && data.visibility === "public") {
+            data.visibility = "home";
+        }
+        // Renote local only if you Renote local only.
+        if (data.renote?.localOnly && data.channel == null) {
+            data.localOnly = true;
+        }
+        // If you reply to local only, make it local only.
+        if (data.reply?.localOnly && data.channel == null) {
+            data.localOnly = true;
+        }
+        if (data.text) {
+            data.text = data.text.trim();
+        } else {
+            data.text = null;
+        }
+        let tags = data.apHashtags;
+        let emojis = data.apEmojis;
+        let mentionedUsers = data.apMentions;
+        // Parse MFM if needed
+        if (!(tags && emojis && mentionedUsers)) {
+            const tokens = data.text ? mfm.parse(data.text) : [];
+            const cwTokens = data.cw ? mfm.parse(data.cw) : [];
+            const choiceTokens = data.poll?.choices ? concat(data.poll.choices.map((choice)=>mfm.parse(choice))) : [];
+            const combinedTokens = tokens.concat(cwTokens).concat(choiceTokens);
+            tags = data.apHashtags || extractHashtags(combinedTokens);
+            emojis = data.apEmojis || [
+                ...new Set([
+                    ...extractCustomEmojisFromMfm(combinedTokens),
+                    ...extractCustomEmojiNamesFromText([
+                        data.text,
+                        data.cw,
+                        ...data.poll?.choices ?? []
+                    ])
+                ])
+            ];
+            mentionedUsers = data.apMentions || await extractMentionedUsers(user, combinedTokens, limiter);
+            if (!data.apMentions) {
+                const groupMentionedUsers = await extractGroupMentionedUsers([
+                    data.text,
+                    data.cw,
+                    ...data.poll?.choices ?? []
+                ]);
+                for (const u of groupMentionedUsers){
+                    if (!mentionedUsers.some((x)=>x.id === u.id)) {
+                        mentionedUsers.push(u);
+                    }
+                }
+            }
+        }
+        tags = tags.filter((tag)=>Array.from(tag || "").length <= 128).splice(0, 32);
+        if (data.reply && user.id !== data.reply.userId && !mentionedUsers.some((u)=>u.id === data.reply.userId)) {
+            mentionedUsers.push(await Users.findOneByOrFail({
+                id: data.reply.userId
+            }));
+        }
+        if (data.visibility === "specified") {
+            if (data.visibleUsers == null) throw new Error("invalid param");
+            for (const u of data.visibleUsers){
+                if (!mentionedUsers.some((x)=>x.id === u.id)) {
+                    mentionedUsers.push(u);
+                }
+            }
+            if (data.reply && !data.visibleUsers.some((x)=>x.id === data.reply.userId)) {
+                data.visibleUsers.push(await Users.findOneByOrFail({
+                    id: data.reply.userId
+                }));
+            }
+        }
+        const note = await insertNote(user, data, tags, emojis, mentionedUsers);
+        // We need to increment these before resolving the promise
+        if (data.reply) {
+            await incRepliesCount(data.reply);
+        }
+        // この投稿を除く指定したユーザーによる指定したノートのリノートが存在しないとき
+        if (data.renote && await countSameRenotes(user.id, data.renote.id, note.id, data.group?.id ?? null) === 0) {
+            await incRenoteCount(data.renote);
+        }
+        res(note);
+        // Prewarm html cache
+        NoteConverter.prewarmCache(note);
+        // 統計を更新
+        notesChart.update(note, true);
+        perUserNotesChart.update(user, note, true);
+        // Register host
+        if (Users.isRemoteUser(user)) {
+            registerOrFetchInstanceDoc(user.host).then((i)=>{
+                Instances.increment({
+                    id: i.id
+                }, "notesCount", 1);
+                instanceChart.updateNote(i.host, note, true);
+            });
+        }
+        // ハッシュタグ更新
+        if (data.visibility === "public" || data.visibility === "home") {
+            updateHashtags(user, tags);
+        }
+        // Increment notes count (user)
+        incNotesCountOfUser(user);
+        // Antenna
+        for (const antenna of (await getAntennas())){
+            checkHitAntenna(antenna, note, user).then((hit)=>{
+                if (hit) {
+                    addNoteToAntenna(antenna, note, user);
+                }
+            });
+        }
+        // Channel
+        if (note.channelId) {
+            ChannelFollowings.findBy({
+                followeeId: note.channelId
+            }).then((followings)=>{
+                for (const following of followings){
+                    insertNoteUnread(following.followerId, note, {
+                        isSpecified: false,
+                        isMentioned: false
+                    });
+                }
+            });
+        }
+        if (data.poll?.expiresAt) {
+            const delay = data.poll.expiresAt.getTime() - Date.now();
+            endedPollNotificationQueue.add("default", {
+                noteId: note.id
+            }, {
+                delay,
+                ...defaultJobOpts
+            });
+        }
+        if (!silent) {
+            if (Users.isLocalUser(user)) activeUsersChart.write(user);
+            // 未読通知を作成
+            if (data.visibility === "specified") {
+                if (data.visibleUsers == null) throw new Error("invalid param");
+                for (const u of data.visibleUsers){
+                    // ローカルユーザーのみ
+                    if (!Users.isLocalUser(u)) continue;
+                    insertNoteUnread(u.id, note, {
+                        isSpecified: true,
+                        isMentioned: false
+                    });
+                }
+            } else {
+                for (const u of mentionedUsers){
+                    // ローカルユーザーのみ
+                    if (!Users.isLocalUser(u)) continue;
+                    insertNoteUnread(u.id, note, {
+                        isSpecified: false,
+                        isMentioned: true
+                    });
+                }
+            }
+            if (!dontFederateInitially) {
+                let publishKey;
+                let noteToPublish;
+                const relays = await getCachedRelays();
+                // Some relays (e.g., aode-relay) deliver posts by boosting them as
+                // Announce activities. In that case, user is the relay's actor.
+                const boostedByRelay = !!user.inbox && relays.map((relay)=>relay.inbox).includes(user.inbox);
+                if (boostedByRelay && data.renote && data.renote.userHost) {
+                    publishKey = `publishedNote:${data.renote.id}`;
+                    noteToPublish = data.renote;
+                } else {
+                    publishKey = `publishedNote:${note.id}`;
+                    noteToPublish = note;
+                }
+                const lock = new Mutex(redisClient, "publishedNote");
+                await lock.acquire();
+                try {
+                    const published = await redisClient.get(publishKey) !== null;
+                    if (!published) {
+                        await redisClient.set(publishKey, "done", "EX", 30);
+                        if (noteToPublish.renoteId) {
+                            // Prevents other threads from publishing the boosting post
+                            await redisClient.set(`publishedNote:${noteToPublish.renoteId}`, "done", "EX", 30);
+                        }
+                        publishNotesStream(noteToPublish);
+                    }
+                } finally{
+                    await lock.release();
+                }
+            }
+            if (note.replyId != null) {
+                // Only provide the reply note id here as the recipient may not be authorized to see the note.
+                publishNoteStream(note.replyId, "replied", {
+                    id: note.id
+                });
+            }
+            const webhooks = await getActiveWebhooks().then((webhooks)=>webhooks.filter((x)=>x.userId === user.id && x.on.includes("note")));
+            for (const webhook of webhooks){
+                webhookDeliver(webhook, "note", {
+                    note: await Notes.pack(note, user)
+                });
+            }
+            const nm = new NotificationManager(user, note);
+            const nmRelatedPromises = [];
+            await createMentionedEvents(mentionedUsers, note, nm);
+            // If has in reply to note
+            if (data.reply) {
+                // Fetch watchers
+                nmRelatedPromises.push(notifyToWatchersOfReplyee(data.reply, user, nm));
+                // 通知
+                if (data.reply.userHost === null) {
+                    const threadMuted = await NoteThreadMutings.findOneBy({
+                        userId: data.reply.userId,
+                        threadId: data.reply.threadId || data.reply.id
+                    });
+                    if (!threadMuted) {
+                        nm.push(data.reply.userId, "reply");
+                        const packedReply = await Notes.pack(note, {
+                            id: data.reply.userId
+                        });
+                        publishMainStream(data.reply.userId, "reply", packedReply);
+                        const webhooks = (await getActiveWebhooks()).filter((x)=>x.userId === data.reply.userId && x.on.includes("reply"));
+                        for (const webhook of webhooks){
+                            webhookDeliver(webhook, "reply", {
+                                note: packedReply
+                            });
+                        }
+                    }
+                }
+            }
+            // If it is renote
+            if (data.renote) {
+                const type = isPlain(note) ? "renote" : "quote";
+                // Notify
+                if (data.renote.userHost === null) {
+                    const threadMuted = await NoteThreadMutings.findOneBy({
+                        userId: data.renote.userId,
+                        threadId: data.renote.threadId || data.renote.id
+                    });
+                    if (!threadMuted) {
+                        nm.push(data.renote.userId, type);
+                    }
+                }
+                // Fetch watchers
+                nmRelatedPromises.push(notifyToWatchersOfRenotee(data.renote, user, nm, type));
+                // Publish event
+                if (user.id !== data.renote.userId && data.renote.userHost === null) {
+                    const packedRenote = await Notes.pack(note, {
+                        id: data.renote.userId
+                    });
+                    publishMainStream(data.renote.userId, "renote", packedRenote);
+                    const renote = data.renote;
+                    const webhooks = (await getActiveWebhooks()).filter((x)=>x.userId === renote.userId && x.on.includes("renote"));
+                    for (const webhook of webhooks){
+                        webhookDeliver(webhook, "renote", {
+                            note: packedRenote
+                        });
+                    }
+                }
+                // Stamp renote if target is a local post
+                if (!data.localOnly && data.renote.userHost === null && type === "quote") {
+                    console.log("stamping");
+                    const stamp = {
+                        id: genId(),
+                        type: "quote",
+                        noteId: note.id,
+                        targetNoteId: data.renote.id
+                    };
+                    await InteractionStamps.insert(stamp);
+                    note.quoteAuthorization = `${config.url}/stamp/${stamp.id}`;
+                    await Notes.update({
+                        id: note.id
+                    }, {
+                        quoteAuthorization: note.quoteAuthorization
+                    });
+                }
+            }
+            Promise.all(nmRelatedPromises).then(()=>{
+                nm.deliver();
+            });
+            //#region AP deliver
+            if (Users.isLocalUser(user) && !data.localOnly && !dontFederateInitially) {
+                (async ()=>{
+                    const noteActivity = await renderNoteOrRenoteActivity(data, note);
+                    const dm = new DeliverManager(user, noteActivity);
+                    // メンションされたリモートユーザーに配送
+                    for (const u of mentionedUsers.filter((u)=>Users.isRemoteUser(u))){
+                        dm.addDirectRecipe(u);
+                    }
+                    // 投稿がリプライかつ投稿者がローカルユーザーかつリプライ先の投稿の投稿者がリモートユーザーなら配送
+                    if (data.reply && data.reply.userHost !== null) {
+                        const u = await Users.findOneBy({
+                            id: data.reply.userId
+                        });
+                        if (u && Users.isRemoteUser(u)) dm.addDirectRecipe(u);
+                    }
+                    // 投稿がRenoteかつ投稿者がローカルユーザーかつRenote元の投稿の投稿者がリモートユーザーなら配送
+                    if (data.renote && data.renote.userHost !== null) {
+                        const u = await Users.findOneBy({
+                            id: data.renote.userId
+                        });
+                        if (u && Users.isRemoteUser(u)) {
+                            dm.addDirectRecipe(u);
+                            if (data.renote.canQuote && !isPlain(note)) deliverToUser(user, renderActivity(renderQuoteRequest(note, data.renote)), u);
+                        }
+                    }
+                    // フォロワーに配送
+                    if ([
+                        "public",
+                        "home",
+                        "followers"
+                    ].includes(note.visibility)) {
+                        dm.addFollowersRecipe();
+                    }
+                    if ([
+                        "public"
+                    ].includes(note.visibility)) {
+                        deliverToRelays(user, noteActivity);
+                    }
+                    dm.execute();
+                })();
+            }
+        //#endregion
+        }
+        if (data.channel) {
+            Channels.increment({
+                id: data.channel.id
+            }, "notesCount", 1);
+            Channels.update(data.channel.id, {
+                lastNotedAt: new Date()
+            });
+            await Notes.countBy({
+                userId: user.id,
+                channelId: data.channel.id
+            }).then((count)=>{
+                // この処理が行われるのはノート作成後なので、ノートが一つしかなかったら最初の投稿だと判断できる
+                // TODO: とはいえノートを削除して何回も投稿すればその分だけインクリメントされる雑さもあるのでどうにかしたい
+                if (count === 1 && data.channel != null) {
+                    Channels.increment({
+                        id: data.channel.id
+                    }, "usersCount", 1);
+                }
+            });
+        }
+    }));
+async function renderNoteOrRenoteActivity(data, note) {
+    if (data.localOnly) return null;
+    const content = data.renote && isPlain(note) ? renderAnnounce(data.renote.uri ?? `${config.url}/notes/${data.renote.id}`, note) : renderCreate(await renderNote(note, false), note);
+    return renderActivity(content);
+}
+function isPlain(note) {
+    return note.text == null && note.cw == null && !note.hasPoll && note.fileIds.length === 0;
+}
+async function incRenoteCount(renote) {
+    // only await renoteCount increment, as score isn't relevant for returning the correct created note response
+    await Notes.increment({
+        id: renote.id
+    }, "renoteCount", 1);
+    Notes.increment({
+        id: renote.id
+    }, "score", 1);
+}
+async function insertNote(user, data, tags, emojis, mentionedUsers) {
+    if (data.createdAt === null || data.createdAt === undefined) {
+        data.createdAt = new Date();
+    }
+    const insert = new Note({
+        id: genId(data.createdAt),
+        createdAt: data.createdAt,
+        fileIds: data.files ? data.files.map((file)=>file.id) : [],
+        replyId: data.reply ? data.reply.id : null,
+        renoteId: data.renote ? data.renote.id : null,
+        channelId: data.channel ? data.channel.id : null,
+        threadId: data.reply ? data.reply.threadId ? data.reply.threadId : data.reply.id : null,
+        name: data.name,
+        text: data.text,
+        hasPoll: data.poll != null,
+        cw: data.cw == null ? null : data.cw,
+        tags: tags.map((tag)=>normalizeForSearch(tag)),
+        emojis,
+        userId: user.id,
+        groupId: data.group?.id ?? null,
+        localOnly: data.localOnly || false,
+        visibility: data.visibility,
+        visibleUserIds: data.visibility === "specified" ? data.visibleUsers ? data.visibleUsers.map((u)=>u.id) : [] : [],
+        attachedFileTypes: data.files ? data.files.map((file)=>file.type) : [],
+        canQuote: data.canQuote,
+        // 以下非正規化データ
+        replyUserId: data.reply ? data.reply.userId : null,
+        replyUserHost: data.reply ? data.reply.userHost : null,
+        renoteUserId: data.renote ? data.renote.userId : null,
+        renoteUserHost: data.renote ? data.renote.userHost : null,
+        userHost: user.host
+    });
+    if (data.uri != null) insert.uri = data.uri;
+    if (data.url != null) insert.url = data.url;
+    if (insert.fileIds.length > 0 && insert.tags.includes("karaokeservice")) {
+        await DriveFiles.update({
+            id: In(insert.fileIds)
+        }, {
+            allowDownload: false
+        });
+    }
+    // Append mentions data
+    if (mentionedUsers.length > 0) {
+        insert.mentions = mentionedUsers.map((u)=>u.id);
+        const profiles = await UserProfiles.findBy({
+            userId: In(insert.mentions)
+        });
+        insert.mentionedRemoteUsers = JSON.stringify(mentionedUsers.filter((u)=>Users.isRemoteUser(u)).map((u)=>{
+            const profile = profiles.find((p)=>p.userId === u.id);
+            const url = profile != null ? profile.url : null;
+            return {
+                uri: u.uri,
+                url: url == null ? undefined : url,
+                username: u.username,
+                host: u.host
+            };
+        }));
+    }
+    // 投稿を作成
+    try {
+        if (insert.hasPoll) {
+            // Prepare objects
+            if (!data.poll) throw new Error("Empty poll data");
+            let expiresAt;
+            if (!data.poll.expiresAt || isNaN(data.poll.expiresAt.getTime())) {
+                expiresAt = null;
+            } else {
+                expiresAt = data.poll.expiresAt;
+            }
+            const poll = new Poll({
+                noteId: insert.id,
+                choices: data.poll.choices,
+                expiresAt,
+                multiple: data.poll.multiple,
+                votes: new Array(data.poll.choices.length).fill(0),
+                noteVisibility: insert.visibility,
+                userId: user.id,
+                userHost: user.host
+            });
+            // Save the objects atomically using a db transaction, note that we should never run any code in a transaction block directly
+            await db.transaction(async (transactionalEntityManager)=>{
+                await transactionalEntityManager.insert(Note, insert);
+                await transactionalEntityManager.insert(Poll, poll);
+            });
+        } else {
+            await Notes.insert(insert);
+        }
+        return insert;
+    } catch (e) {
+        // duplicate key error
+        if (isDuplicateKeyValueError(e)) {
+            const err = new Error("Duplicated note");
+            err.name = "duplicated";
+            throw err;
+        }
+        console.error(e);
+        throw e;
+    }
+}
+async function notifyToWatchersOfRenotee(renote, user, nm, type) {
+    const watchers = await NoteWatchings.findBy({
+        noteId: renote.id,
+        userId: Not(user.id)
+    });
+    for (const watcher of watchers){
+        nm.push(watcher.userId, type);
+    }
+}
+async function notifyToWatchersOfReplyee(reply, user, nm) {
+    const watchers = await NoteWatchings.findBy({
+        noteId: reply.id,
+        userId: Not(user.id)
+    });
+    for (const watcher of watchers){
+        nm.push(watcher.userId, "reply");
+    }
+}
+async function createMentionedEvents(mentionedUsers, note, nm) {
+    for (const u of mentionedUsers.filter((u)=>Users.isLocalUser(u))){
+        const threadMuted = await NoteThreadMutings.findOneBy({
+            userId: u.id,
+            threadId: note.threadId || note.id
+        });
+        if (threadMuted) {
+            continue;
+        }
+        // note with "specified" visibility might not be visible to mentioned users
+        try {
+            const detailPackedNote = await Notes.pack(note, u, {
+                detail: true
+            });
+            publishMainStream(u.id, "mention", detailPackedNote);
+            const webhooks = (await getActiveWebhooks()).filter((x)=>x.userId === u.id && x.on.includes("mention"));
+            for (const webhook of webhooks){
+                webhookDeliver(webhook, "mention", {
+                    note: detailPackedNote
+                });
+            }
+        } catch (err) {
+            if (err.id === "9725d0ce-ba28-4dde-95a7-2cbb2c15de24") continue;
+            throw err;
+        }
+        // Create notification
+        nm.push(u.id, "mention");
+    }
+}
+async function incRepliesCount(reply) {
+    await Notes.increment({
+        id: reply.id
+    }, "repliesCount", 1);
+}
+function incNotesCountOfUser(user) {
+    Users.createQueryBuilder().update().set({
+        updatedAt: new Date(),
+        notesCount: ()=>'"notesCount" + 1'
+    }).where("id = :id", {
+        id: user.id
+    }).execute();
+}
+export async function extractMentionedUsers(user, tokens, limiter = new RecursionLimiter()) {
+    if (tokens == null) return [];
+    const mentions = extractMentions(tokens);
+    let mentionedUsers = (await Promise.all(mentions.map((m)=>resolveUser(m.username, m.host || user.host, undefined, limiter).catch(()=>null)))).filter((x)=>x != null);
+    // Drop duplicate users
+    mentionedUsers = mentionedUsers.filter((u, i, self)=>i === self.findIndex((u2)=>u.id === u2.id));
+    return mentionedUsers;
+}

packages/backend/built/services/note/delete.js

@@ -0,0 +1,124 @@
+import { Brackets, In } from "typeorm";
+import { publishNoteStream, publishNoteUpdatesStream } from "../stream.js";
+import renderDelete from "../../remote/activitypub/renderer/delete.js";
+import renderAnnounce from "../../remote/activitypub/renderer/announce.js";
+import renderUndo from "../../remote/activitypub/renderer/undo.js";
+import { renderActivity } from "../../remote/activitypub/renderer/index.js";
+import renderTombstone from "../../remote/activitypub/renderer/tombstone.js";
+import config from "../../config/index.js";
+import { Notes, Users, Instances } from "../../models/index.js";
+import { notesChart, perUserNotesChart, instanceChart } from "../chart/index.js";
+import { deliverToFollowers, deliverToUser } from "../../remote/activitypub/deliver-manager.js";
+import { countSameRenotes } from "../../misc/count-same-renotes.js";
+import { registerOrFetchInstanceDoc } from "../register-or-fetch-instance-doc.js";
+import { deliverToRelays } from "../relay.js";
+/**
+ * 投稿を削除します。
+ * @param user 投稿者
+ * @param note 投稿
+ */ export default async function(user, note, quiet = false) {
+    const deletedAt = new Date();
+    // この投稿を除く指定したユーザーによる指定したノートのリノートが存在しないとき
+    if (note.renoteId && await countSameRenotes(user.id, note.renoteId, note.id, note.groupId) === 0) {
+        await Notes.decrement({
+            id: note.renoteId
+        }, "renoteCount", 1);
+        await Notes.decrement({
+            id: note.renoteId
+        }, "score", 1);
+    }
+    if (note.replyId) {
+        await Notes.decrement({
+            id: note.replyId
+        }, "repliesCount", 1);
+    }
+    if (!quiet) {
+        publishNoteStream(note.id, "deleted", {
+            deletedAt: deletedAt
+        });
+        publishNoteUpdatesStream("deleted", note);
+        //#region ローカルの投稿なら削除アクティビティを配送
+        if (Users.isLocalUser(user) && !note.localOnly) {
+            let renote = null;
+            // if deletd note is renote
+            if (note.renoteId && note.text == null && note.cw == null && !note.hasPoll && (note.fileIds == null || note.fileIds.length === 0)) {
+                renote = await Notes.findOneBy({
+                    id: note.renoteId
+                });
+            }
+            const content = renderActivity(renote ? renderUndo(renderAnnounce(renote.uri || `${config.url}/notes/${renote.id}`, note), user) : renderDelete(renderTombstone(`${config.url}/notes/${note.id}`), user));
+            deliverToConcerned(user, note, content);
+        }
+        // also deliever delete activity to cascaded notes
+        const cascadingNotes = (await findCascadingNotes(note)).filter((note)=>!note.localOnly); // filter out local-only notes
+        for (const cascadingNote of cascadingNotes){
+            if (!cascadingNote.user) continue;
+            if (!Users.isLocalUser(cascadingNote.user)) continue;
+            const content = renderActivity(renderDelete(renderTombstone(`${config.url}/notes/${cascadingNote.id}`), cascadingNote.user));
+            deliverToConcerned(cascadingNote.user, cascadingNote, content);
+        }
+        //#endregion
+        // 統計を更新
+        notesChart.update(note, false);
+        perUserNotesChart.update(user, note, false);
+        if (Users.isRemoteUser(user)) {
+            registerOrFetchInstanceDoc(user.host).then((i)=>{
+                Instances.decrement({
+                    id: i.id
+                }, "notesCount", 1);
+                instanceChart.updateNote(i.host, note, false);
+            });
+        }
+    }
+    await Notes.delete({
+        id: note.id,
+        userId: user.id
+    });
+}
+async function findCascadingNotes(note) {
+    const cascadingNotes = [];
+    const recursive = async (noteId)=>{
+        const query = Notes.createQueryBuilder("note").where("note.replyId = :noteId", {
+            noteId
+        }).orWhere(new Brackets((q)=>{
+            q.where("note.renoteId = :noteId", {
+                noteId
+            }).andWhere("note.text IS NOT NULL");
+        })).leftJoinAndSelect("note.user", "user");
+        const replies = await query.getMany();
+        for (const reply of replies){
+            cascadingNotes.push(reply);
+            await recursive(reply.id);
+        }
+    };
+    await recursive(note.id);
+    return cascadingNotes.filter((note)=>note.userHost === null); // filter out non-local users
+}
+async function getMentionedRemoteUsers(note) {
+    const where = [];
+    // mention / reply / dm
+    const uris = JSON.parse(note.mentionedRemoteUsers).map((x)=>x.uri);
+    if (uris.length > 0) {
+        where.push({
+            uri: In(uris)
+        });
+    }
+    // renote / quote
+    if (note.renoteUserId) {
+        where.push({
+            id: note.renoteUserId
+        });
+    }
+    if (where.length === 0) return [];
+    return await Users.find({
+        where
+    });
+}
+async function deliverToConcerned(user, note, content) {
+    deliverToFollowers(user, content);
+    deliverToRelays(user, content);
+    const remoteUsers = await getMentionedRemoteUsers(note);
+    for (const remoteUser of remoteUsers){
+        deliverToUser(user, content, remoteUser);
+    }
+}

packages/backend/built/services/note/edit.js

@@ -0,0 +1,186 @@
+import * as mfm from "mfm-js";
+import { publishNoteStream, publishNoteUpdatesStream } from "../stream.js";
+import DeliverManager from "../../remote/activitypub/deliver-manager.js";
+import renderNote from "../../remote/activitypub/renderer/note.js";
+import { renderActivity } from "../../remote/activitypub/renderer/index.js";
+import { extractCustomEmojisFromMfm } from "../../misc/extract-custom-emojis-from-mfm.js";
+import { extractHashtags } from "../../misc/extract-hashtags.js";
+import { Users, Notes, UserProfiles, Polls, NoteEdits, PollVotes } from "../../models/index.js";
+import { In } from "typeorm";
+import { genId } from "../../misc/gen-id.js";
+import { deliverToRelays } from "../relay.js";
+import renderUpdate from "../../remote/activitypub/renderer/update.js";
+import { extractMentionedUsers } from "./create.js";
+import { normalizeForSearch } from "../../misc/normalize-for-search.js";
+import { NoteConverter } from "../../server/api/mastodon/converters/note.js";
+export default async function(user, note, data, suppressEvidence = false) {
+    if (data.text !== undefined && data.text !== null) {
+        data.text = data.text.trim();
+    } else {
+        data.text = null;
+    }
+    const fileIds = data.files?.map((file)=>file.id);
+    const fileTypes = data.files?.map((file)=>file.type);
+    const tokens = mfm.parse(data.text || "").concat(mfm.parse(data.cw || ""));
+    const tags = extractHashtags(tokens).filter((tag)=>Array.from(tag || "").length <= 128).splice(0, 32).map(normalizeForSearch);
+    const emojis = extractCustomEmojisFromMfm(tokens);
+    const mentionUsers = await extractMentionedUsers(user, tokens);
+    const mentionUserIds = mentionUsers.map((user)=>user.id);
+    const remoteUsers = mentionUsers.filter((user)=>user.host != null);
+    const remoteUserIds = remoteUsers.map((user)=>user.id);
+    const remoteProfiles = await UserProfiles.findBy({
+        userId: In(remoteUserIds)
+    });
+    const mentionedRemoteUsers = remoteUsers.map((user)=>{
+        const profile = remoteProfiles.find((profile)=>profile.userId === user.id);
+        return {
+            username: user.username,
+            host: user.host ?? null,
+            uri: user.uri,
+            url: profile ? profile.url : undefined
+        };
+    });
+    let publishing = false;
+    const update = {};
+    if (data.text !== null && data.text !== note.text) {
+        update.text = data.text;
+    }
+    if (data.cw !== note.cw) {
+        update.cw = data.cw ?? null;
+    }
+    if (data.files !== undefined && fileIds.sort().join(",") !== note.fileIds.sort().join(",")) {
+        update.fileIds = fileIds;
+        update.attachedFileTypes = fileTypes;
+    }
+    if (tags.sort().join(",") !== note.tags.sort().join(",")) {
+        update.tags = tags;
+    }
+    if (mentionUserIds.sort().join(",") !== note.mentions.sort().join(",")) {
+        update.mentions = mentionUserIds;
+        update.mentionedRemoteUsers = JSON.stringify(mentionedRemoteUsers);
+    }
+    if (emojis.sort().join(",") !== note.emojis.sort().join(",")) {
+        update.emojis = emojis;
+    }
+    if (data.poll !== undefined && note.hasPoll !== !!data.poll) {
+        update.hasPoll = !!data.poll;
+    }
+    if (data.poll) {
+        const dbPoll = await Polls.findOneBy({
+            noteId: note.id
+        });
+        if (dbPoll == null) {
+            await Polls.insert({
+                noteId: note.id,
+                choices: data.poll?.choices,
+                multiple: data.poll?.multiple,
+                votes: new Array(data.poll?.choices.length).fill(0),
+                expiresAt: data.poll?.expiresAt,
+                noteVisibility: note.visibility === "hidden" ? "home" : note.visibility,
+                userId: user.id,
+                userHost: user.host
+            });
+            publishing = true;
+        } else {
+            const choicesChanged = JSON.stringify(dbPoll.choices) !== JSON.stringify(data.poll.choices);
+            if (dbPoll.multiple !== data.poll.multiple || dbPoll.expiresAt !== data.poll.expiresAt || dbPoll.noteVisibility !== note.visibility || choicesChanged) {
+                await Polls.update({
+                    noteId: note.id
+                }, {
+                    choices: data.poll?.choices,
+                    multiple: data.poll?.multiple,
+                    votes: choicesChanged ? new Array(data.poll.choices.length).fill(0) : undefined,
+                    expiresAt: data.poll?.expiresAt,
+                    noteVisibility: note.visibility === "hidden" ? "home" : note.visibility
+                });
+                // Reset votes
+                if (JSON.stringify(dbPoll.choices) !== JSON.stringify(data.poll.choices)) {
+                    await PollVotes.delete({
+                        noteId: dbPoll.noteId
+                    });
+                }
+                publishing = true;
+            }
+        }
+    }
+    if (data.quoteAuthorization !== undefined) {
+        update.quoteAuthorization = data.quoteAuthorization;
+    }
+    if (notEmpty(update)) {
+        if (!suppressEvidence) update.updatedAt = new Date();
+        await Notes.update(note.id, update);
+        if (!suppressEvidence) {
+            // Add previous note contents to NoteEdit history
+            await NoteEdits.insert({
+                id: genId(),
+                noteId: note.id,
+                text: note.text || undefined,
+                cw: note.cw,
+                fileIds: note.fileIds,
+                updatedAt: update.updatedAt ?? undefined
+            });
+        }
+        publishing = true;
+    }
+    note = await Notes.findOneByOrFail({
+        id: note.id
+    });
+    if (publishing) {
+        NoteConverter.prewarmCache(note);
+        // Publish update event for the updated note details
+        if (!suppressEvidence) {
+            publishNoteStream(note.id, "updated", {
+                updatedAt: update.updatedAt
+            });
+            publishNoteUpdatesStream("updated", note);
+        }
+        (async ()=>{
+            if (note.localOnly) return;
+            const noteActivity = await renderNote(note, false);
+            if (!suppressEvidence) noteActivity.updated = note.updatedAt?.toISOString();
+            const updateActivity = renderUpdate(noteActivity, user);
+            updateActivity.to = noteActivity.to;
+            updateActivity.cc = noteActivity.cc;
+            const activity = renderActivity(updateActivity);
+            const dm = new DeliverManager(user, activity);
+            // Delivery to remote mentioned users
+            for (const u of mentionUsers.filter((u)=>Users.isRemoteUser(u))){
+                dm.addDirectRecipe(u);
+            }
+            // Post is a reply and remote user is the contributor of the original post
+            if (note.reply && note.reply.userHost !== null) {
+                const u = await Users.findOneBy({
+                    id: note.reply.userId
+                });
+                if (u && Users.isRemoteUser(u)) dm.addDirectRecipe(u);
+            }
+            // Post is a renote and remote user is the contributor of the original post
+            if (note.renote && note.renote.userHost !== null) {
+                const u = await Users.findOneBy({
+                    id: note.renote.userId
+                });
+                if (u && Users.isRemoteUser(u)) dm.addDirectRecipe(u);
+            }
+            // Deliver to followers for non-direct posts.
+            if ([
+                "public",
+                "home",
+                "followers"
+            ].includes(note.visibility)) {
+                dm.addFollowersRecipe();
+            }
+            // Deliver to relays for public posts.
+            if ([
+                "public"
+            ].includes(note.visibility)) {
+                deliverToRelays(user, activity);
+            }
+            // GO!
+            dm.execute();
+        })();
+    }
+    return note;
+}
+function notEmpty(partial) {
+    return Object.keys(partial).length > 0;
+}

packages/backend/built/services/note/polls/update.js

@@ -0,0 +1,21 @@
+import renderUpdate from "../../../remote/activitypub/renderer/update.js";
+import { renderActivity } from "../../../remote/activitypub/renderer/index.js";
+import renderNote from "../../../remote/activitypub/renderer/note.js";
+import { Users, Notes } from "../../../models/index.js";
+import { deliverToFollowers } from "../../../remote/activitypub/deliver-manager.js";
+import { deliverToRelays } from "../../relay.js";
+export async function deliverQuestionUpdate(noteId) {
+    const note = await Notes.findOneBy({
+        id: noteId
+    });
+    if (note == null) throw new Error("note not found");
+    const user = await Users.findOneBy({
+        id: note.userId
+    });
+    if (user == null) throw new Error("note not found");
+    if (Users.isLocalUser(user)) {
+        const content = renderActivity(renderUpdate(await renderNote(note, false), user));
+        deliverToFollowers(user, content);
+        deliverToRelays(user, content);
+    }
+}

packages/backend/built/services/note/polls/vote.js

@@ -0,0 +1,82 @@
+import { publishNoteStream } from "../../stream.js";
+import { PollVotes, NoteWatchings, Polls, Blockings } from "../../../models/index.js";
+import { Not } from "typeorm";
+import { genId } from "../../../misc/gen-id.js";
+import { createNotification } from "../../create-notification.js";
+export default async function(user, note, choice) {
+    const poll = await Polls.findOneBy({
+        noteId: note.id
+    });
+    if (poll == null) throw new Error("poll not found");
+    // Check whether is valid choice
+    if (poll.choices[choice] == null) throw new Error("invalid choice param");
+    // Check blocking
+    if (note.userId !== user.id) {
+        const blocked = await Blockings.exist({
+            where: [
+                {
+                    blockerId: note.userId,
+                    blockeeId: user.id,
+                    groupId: null
+                },
+                ...note.groupId ? [
+                    {
+                        groupId: note.groupId,
+                        blockeeId: user.id
+                    }
+                ] : []
+            ]
+        });
+        if (blocked) {
+            throw new Error("blocked");
+        }
+    }
+    // if already voted
+    const exist = await PollVotes.findBy({
+        noteId: note.id,
+        userId: user.id
+    });
+    if (poll.multiple) {
+        if (exist.some((x)=>x.choice === choice)) {
+            throw new Error("already voted");
+        }
+    } else if (exist.length !== 0) {
+        throw new Error("already voted");
+    }
+    // Create vote
+    await PollVotes.insert({
+        id: genId(),
+        createdAt: new Date(),
+        noteId: note.id,
+        userId: user.id,
+        choice: choice
+    });
+    // Increment votes count
+    const index = choice + 1; // In SQL, array index is 1 based
+    await Polls.query(`UPDATE poll SET votes[${index}] = votes[${index}] + 1 WHERE "noteId" = '${poll.noteId}'`);
+    publishNoteStream(note.id, "pollVoted", {
+        choice: choice,
+        userId: user.id
+    });
+    // Notify
+    createNotification(note.userId, "pollVote", {
+        notifierId: user.id,
+        note: note,
+        noteId: note.id,
+        choice: choice
+    });
+    // Fetch watchers
+    NoteWatchings.findBy({
+        noteId: note.id,
+        userId: Not(user.id)
+    }).then((watchers)=>{
+        for (const watcher of watchers){
+            createNotification(watcher.userId, "pollVote", {
+                notifierId: user.id,
+                note: note,
+                noteId: note.id,
+                choice: choice
+            });
+        }
+    });
+}

packages/backend/built/services/note/reaction/create.js

@@ -0,0 +1,163 @@
+import { publishNoteStream } from "../../stream.js";
+import { renderLike } from "../../../remote/activitypub/renderer/like.js";
+import DeliverManager from "../../../remote/activitypub/deliver-manager.js";
+import { renderActivity } from "../../../remote/activitypub/renderer/index.js";
+import { toDbReaction, decodeReaction } from "../../../misc/reaction-lib.js";
+import { NoteReactions, Users, NoteWatchings, Notes, Emojis, Blockings } from "../../../models/index.js";
+import { IsNull, Not } from "typeorm";
+import { perUserReactionsChart } from "../../chart/index.js";
+import { genId } from "../../../misc/gen-id.js";
+import { createNotification } from "../../create-notification.js";
+import deleteReaction from "./delete.js";
+import { isDuplicateKeyValueError } from "../../../misc/is-duplicate-key-value-error.js";
+import { IdentifiableError } from "../../../misc/identifiable-error.js";
+import { populateEmojiOrUserEmoji } from "../../../misc/populate-emojis.js";
+export default (async (user, note, reaction)=>{
+    // Check blocking
+    if (note.userId !== user.id) {
+        const blocked = await Blockings.exist({
+            where: [
+                {
+                    blockerId: note.userId,
+                    blockeeId: user.id,
+                    groupId: null
+                },
+                ...note.groupId ? [
+                    {
+                        groupId: note.groupId,
+                        blockeeId: user.id
+                    }
+                ] : []
+            ]
+        });
+        if (blocked) {
+            throw new IdentifiableError("e70412a4-7197-4726-8e74-f3e0deb92aa7");
+        }
+    }
+    // check visibility
+    if (!await Notes.isVisibleForMe(note, user.id)) {
+        throw new IdentifiableError("68e9d2d1-48bf-42c2-b90a-b20e09fd3d48", "Note not accessible for you.");
+    }
+    // TODO: cache
+    reaction = await toDbReaction(reaction, user.host);
+    const record = {
+        id: genId(),
+        createdAt: new Date(),
+        noteId: note.id,
+        userId: user.id,
+        groupId: user.groupId ?? null,
+        reaction
+    };
+    // Create reaction
+    try {
+        await NoteReactions.insert(record);
+    } catch (e) {
+        if (isDuplicateKeyValueError(e)) {
+            const exists = await NoteReactions.findOneByOrFail({
+                noteId: note.id,
+                ...user.groupId ? {
+                    groupId: user.groupId
+                } : {
+                    userId: user.id,
+                    groupId: null
+                }
+            });
+            if (exists.reaction !== reaction) {
+                // 別のリアクションがすでにされていたら置き換える
+                await deleteReaction(user, note);
+                await NoteReactions.insert(record);
+            } else {
+                // 同じリアクションがすでにされていたらエラー
+                throw new IdentifiableError("51c42bb4-931a-456b-bff7-e5a8a70dd298", "Reaction already exists");
+            }
+        } else {
+            throw e;
+        }
+    }
+    // Increment reactions count
+    const sql = `jsonb_set("reactions", '{${reaction}}', (COALESCE("reactions"->>'${reaction}', '0')::int + 1)::text::jsonb)`;
+    await Notes.createQueryBuilder().update().set({
+        reactions: ()=>sql,
+        score: ()=>'"score" + 1'
+    }).where("id = :id", {
+        id: note.id
+    }).execute();
+    perUserReactionsChart.update(user, note);
+    // カスタム絵文字リアクションだったら絵文字情報も送る
+    const decodedReaction = decodeReaction(reaction);
+    const emoji = await Emojis.findOne({
+        where: {
+            name: decodedReaction.name,
+            host: decodedReaction.host ?? IsNull()
+        },
+        select: [
+            "name",
+            "host",
+            "originalUrl",
+            "publicUrl"
+        ]
+    });
+    const populatedEmoji = emoji == null ? await populateEmojiOrUserEmoji(reaction.slice(1, -1), note.userHost) : null;
+    publishNoteStream(note.id, "reacted", {
+        reaction: decodedReaction.reaction,
+        emoji: emoji != null ? {
+            name: emoji.host ? `${emoji.name}@${emoji.host}` : `${emoji.name}@.`,
+            url: emoji.publicUrl || emoji.originalUrl
+        } : populatedEmoji != null ? {
+            name: populatedEmoji.name,
+            url: populatedEmoji.url
+        } : null,
+        userId: user.id,
+        groupId: user.groupId ?? null
+    });
+    // Create notification if the reaction target is a local user.
+    if (note.userHost === null) {
+        createNotification(note.userId, "reaction", {
+            notifierId: user.id,
+            note: note,
+            noteId: note.id,
+            reaction: reaction
+        });
+    }
+    // Fetch watchers
+    NoteWatchings.findBy({
+        noteId: note.id,
+        userId: Not(user.id)
+    }).then((watchers)=>{
+        for (const watcher of watchers){
+            createNotification(watcher.userId, "reaction", {
+                notifierId: user.id,
+                note: note,
+                noteId: note.id,
+                reaction: reaction
+            });
+        }
+    });
+    //#region deliver
+    if (Users.isLocalUser(user) && !note.localOnly && note.visibility !== "hidden") {
+        const content = renderActivity(await renderLike(record, note));
+        const dm = new DeliverManager(user, content);
+        if (note.userHost !== null) {
+            const reactee = await Users.findOneBy({
+                id: note.userId
+            });
+            dm.addDirectRecipe(reactee);
+        }
+        if ([
+            "public",
+            "home",
+            "followers"
+        ].includes(note.visibility)) {
+            dm.addFollowersRecipe();
+        } else if (note.visibility === "specified") {
+            const visibleUsers = await Promise.all(note.visibleUserIds.map((id)=>Users.findOneBy({
+                    id
+                })));
+            for (const u of visibleUsers.filter((u)=>u && Users.isRemoteUser(u))){
+                dm.addDirectRecipe(u);
+            }
+        }
+        dm.execute();
+    }
+//#endregion
+});

packages/backend/built/services/note/reaction/delete.js

@@ -0,0 +1,57 @@
+import { publishNoteStream } from "../../stream.js";
+import { renderLike } from "../../../remote/activitypub/renderer/like.js";
+import renderUndo from "../../../remote/activitypub/renderer/undo.js";
+import { renderActivity } from "../../../remote/activitypub/renderer/index.js";
+import DeliverManager from "../../../remote/activitypub/deliver-manager.js";
+import { IdentifiableError } from "../../../misc/identifiable-error.js";
+import { NoteReactions, Users, Notes } from "../../../models/index.js";
+import { decodeReaction } from "../../../misc/reaction-lib.js";
+export default (async (user, note)=>{
+    const reaction = await NoteReactions.findOneBy({
+        noteId: note.id,
+        ...user.groupId ? {
+            groupId: user.groupId
+        } : {
+            userId: user.id,
+            groupId: null
+        }
+    });
+    // if already unreacted
+    if (reaction == null) {
+        throw new IdentifiableError("60527ec9-b4cb-4a88-a6bd-32d3ad26817d", "not reacted");
+    }
+    // Delete reaction
+    const result = await NoteReactions.delete(reaction.id);
+    if (result.affected !== 1) {
+        throw new IdentifiableError("60527ec9-b4cb-4a88-a6bd-32d3ad26817d", "not reacted");
+    }
+    // Decrement reactions count
+    const sql = `jsonb_set("reactions", '{${reaction.reaction}}', (COALESCE("reactions"->>'${reaction.reaction}', '0')::int - 1)::text::jsonb)`;
+    await Notes.createQueryBuilder().update().set({
+        reactions: ()=>sql
+    }).where("id = :id", {
+        id: note.id
+    }).execute();
+    Notes.decrement({
+        id: note.id
+    }, "score", 1);
+    publishNoteStream(note.id, "unreacted", {
+        reaction: decodeReaction(reaction.reaction).reaction,
+        userId: user.id,
+        groupId: user.groupId ?? null
+    });
+    //#region 配信
+    if (Users.isLocalUser(user) && !note.localOnly) {
+        const content = renderActivity(renderUndo(await renderLike(reaction, note), user));
+        const dm = new DeliverManager(user, content);
+        if (note.userHost !== null) {
+            const reactee = await Users.findOneBy({
+                id: note.userId
+            });
+            dm.addDirectRecipe(reactee);
+        }
+        dm.addFollowersRecipe();
+        dm.execute();
+    }
+//#endregion
+});

packages/backend/built/services/note/read.js

@@ -0,0 +1,126 @@
+import { publishMainStream } from "../stream.js";
+import { NoteUnreads, Followings, ChannelFollowings } from "../../models/index.js";
+import { Not, IsNull, In } from "typeorm";
+import { readNotificationByQuery } from "../../server/api/common/read-notification.js";
+/**
+ * Mark notes as read
+ */ export default async function(userId, notes, info) {
+    const following = info?.following ? info.following : new Set((await Followings.find({
+        where: {
+            followerId: userId
+        },
+        select: [
+            "followeeId"
+        ]
+    })).map((x)=>x.followeeId));
+    const followingChannels = info?.followingChannels ? info.followingChannels : new Set((await ChannelFollowings.find({
+        where: {
+            followerId: userId
+        },
+        select: [
+            "followeeId"
+        ]
+    })).map((x)=>x.followeeId));
+    // const myAntennas = (await getAntennas()).filter((a) => a.userId === userId);
+    const readMentions = [];
+    const readSpecifiedNotes = [];
+    const readChannelNotes = [];
+    // const readAntennaNotes: (Note | Packed<"Note">)[] = [];
+    for (const note of notes){
+        if (note.mentions?.includes(userId)) {
+            readMentions.push(note);
+        } else if (note.visibleUserIds?.includes(userId)) {
+            readSpecifiedNotes.push(note);
+        }
+        if (note.channelId && followingChannels.has(note.channelId)) {
+            readChannelNotes.push(note);
+        }
+    // if (note.user != null) {
+    // 	// たぶんnullになることは無いはずだけど一応
+    // 	for (const antenna of myAntennas) {
+    // 		if (
+    // 			await checkHitAntenna(
+    // 				antenna,
+    // 				note,
+    // 				note.user,
+    // 				undefined,
+    // 				Array.from(following),
+    // 			)
+    // 		) {
+    // 			readAntennaNotes.push(note);
+    // 		}
+    // 	}
+    // }
+    }
+    if (readMentions.length > 0 || readSpecifiedNotes.length > 0 || readChannelNotes.length > 0) {
+        // Remove the record
+        await NoteUnreads.delete({
+            userId: userId,
+            noteId: In([
+                ...readMentions.map((n)=>n.id),
+                ...readSpecifiedNotes.map((n)=>n.id),
+                ...readChannelNotes.map((n)=>n.id)
+            ])
+        });
+        // TODO: ↓まとめてクエリしたい
+        NoteUnreads.countBy({
+            userId: userId,
+            isMentioned: true
+        }).then((mentionsCount)=>{
+            if (mentionsCount === 0) {
+                // 全て既読になったイベントを発行
+                publishMainStream(userId, "readAllUnreadMentions");
+            }
+        });
+        NoteUnreads.countBy({
+            userId: userId,
+            isSpecified: true
+        }).then((specifiedCount)=>{
+            if (specifiedCount === 0) {
+                // 全て既読になったイベントを発行
+                publishMainStream(userId, "readAllUnreadSpecifiedNotes");
+            }
+        });
+        NoteUnreads.countBy({
+            userId: userId,
+            noteChannelId: Not(IsNull())
+        }).then((channelNoteCount)=>{
+            if (channelNoteCount === 0) {
+                // 全て既読になったイベントを発行
+                publishMainStream(userId, "readAllChannels");
+            }
+        });
+        readNotificationByQuery(userId, {
+            noteId: In([
+                ...readMentions.map((n)=>n.id),
+                ...readSpecifiedNotes.map((n)=>n.id)
+            ])
+        });
+    }
+// if (readAntennaNotes.length > 0) {
+// 	await AntennaNotes.update(
+// 		{
+// 			antennaId: In(myAntennas.map((a) => a.id)),
+// 			noteId: In(readAntennaNotes.map((n) => n.id)),
+// 		},
+// 		{
+// 			read: true,
+// 		},
+// 	);
+// 	// TODO: まとめてクエリしたい
+// 	for (const antenna of myAntennas) {
+// 		const count = await AntennaNotes.countBy({
+// 			antennaId: antenna.id,
+// 			read: false,
+// 		});
+// 		if (count === 0) {
+// 			publishMainStream(userId, "readAntenna", antenna);
+// 		}
+// 	}
+// 	Users.getHasUnreadAntenna(userId).then((unread) => {
+// 		if (!unread) {
+// 			publishMainStream(userId, "readAllAntennas");
+// 		}
+// 	});
+// }
+}

packages/backend/built/services/note/scheduled.js

@@ -0,0 +1,45 @@
+import { genId } from "../../misc/gen-id.js";
+import { ScheduledNotes, Users } from "../../models/index.js";
+import { createNoteFromApiData } from "./create-from-api.js";
+export async function scheduleNote(user, data, scheduledAt) {
+    return await ScheduledNotes.save({
+        id: genId(),
+        createdAt: new Date(),
+        scheduledAt,
+        userId: user.id,
+        status: "scheduled",
+        data,
+        noteId: null,
+        error: null
+    });
+}
+export async function publishScheduledNote(scheduledNoteId) {
+    const scheduled = await ScheduledNotes.findOneBy({
+        id: scheduledNoteId
+    });
+    if (!scheduled) return "skip: scheduled note not found";
+    if (scheduled.status !== "scheduled") return `skip: status=${scheduled.status}`;
+    if (scheduled.scheduledAt.getTime() > Date.now() + 1000) return "skip: not due yet";
+    await ScheduledNotes.update(scheduled.id, {
+        status: "processing",
+        error: null
+    });
+    try {
+        const user = await Users.findOneByOrFail({
+            id: scheduled.userId
+        });
+        const note = await createNoteFromApiData(user, scheduled.data, scheduled.scheduledAt);
+        await ScheduledNotes.update(scheduled.id, {
+            status: "published",
+            noteId: note.id,
+            error: null
+        });
+        return `published: ${note.id}`;
+    } catch (err) {
+        await ScheduledNotes.update(scheduled.id, {
+            status: "failed",
+            error: err?.message ?? String(err)
+        });
+        throw err;
+    }
+}

packages/backend/built/services/note/unread.js

@@ -0,0 +1,46 @@
+import { publishMainStream } from "../stream.js";
+import { Mutings, NoteThreadMutings, NoteUnreads } from "../../models/index.js";
+import { genId } from "../../misc/gen-id.js";
+export async function insertNoteUnread(userId, note, params) {
+    //#region ミュートしているなら無視
+    // TODO: 現在の仕様ではChannelにミュートは適用されないのでよしなにケアする
+    const mute = await Mutings.findBy({
+        muterId: userId
+    });
+    if (mute.map((m)=>m.muteeId).includes(note.userId)) return;
+    //#endregion
+    // スレッドミュート
+    const threadMute = await NoteThreadMutings.findOneBy({
+        userId: userId,
+        threadId: note.threadId || note.id
+    });
+    if (threadMute) return;
+    const unread = {
+        id: genId(),
+        noteId: note.id,
+        userId: userId,
+        isSpecified: params.isSpecified,
+        isMentioned: params.isMentioned,
+        noteChannelId: note.channelId,
+        noteUserId: note.userId
+    };
+    await NoteUnreads.insert(unread);
+    // 2秒経っても既読にならなかったら「未読の投稿がありますよ」イベントを発行する
+    setTimeout(async ()=>{
+        const exist = await NoteUnreads.exist({
+            where: {
+                id: unread.id
+            }
+        });
+        if (!exist) return;
+        if (params.isMentioned) {
+            publishMainStream(userId, "unreadMention", note.id);
+        }
+        if (params.isSpecified) {
+            publishMainStream(userId, "unreadSpecifiedNote", note.id);
+        }
+        if (note.channelId) {
+            publishMainStream(userId, "unreadChannel", note.id);
+        }
+    }, 2000);
+}

packages/backend/built/services/note/unwatch.js

@@ -0,0 +1,7 @@
+import { NoteWatchings } from "../../models/index.js";
+export default (async (me, note)=>{
+    await NoteWatchings.delete({
+        noteId: note.id,
+        userId: me
+    });
+});

packages/backend/built/services/note/watch.js

@@ -0,0 +1,15 @@
+import { NoteWatchings } from "../../models/index.js";
+import { genId } from "../../misc/gen-id.js";
+export default (async (me, note)=>{
+    // 自分の投稿はwatchできない
+    if (me === note.userId) {
+        return;
+    }
+    await NoteWatchings.insert({
+        id: genId(),
+        createdAt: new Date(),
+        noteId: note.id,
+        userId: me,
+        noteUserId: note.userId
+    });
+});

packages/backend/built/services/push-notification.js

@@ -0,0 +1,67 @@
+import push from "web-push";
+import config from "../config/index.js";
+import { SwSubscriptions } from "../models/index.js";
+import { fetchMeta } from "../misc/fetch-meta.js";
+import { getNoteSummary } from "../misc/get-note-summary.js";
+// プッシュメッセージサーバーには文字数制限があるため、内容を削減します
+function truncateNotification(notification) {
+    if (notification.note) {
+        return {
+            ...notification,
+            note: {
+                ...notification.note,
+                // textをgetNoteSummaryしたものに置き換える
+                text: getNoteSummary(notification.type === "renote" ? notification.note.renote : notification.note),
+                cw: undefined,
+                reply: undefined,
+                renote: undefined,
+                user: undefined
+            }
+        };
+    }
+    return notification;
+}
+export async function pushNotification(userId, type, body) {
+    const meta = await fetchMeta();
+    // アプリケーションの連絡先と、サーバーサイドの鍵ペアの情報を登録
+    push.setVapidDetails(config.url, meta.swPublicKey, meta.swPrivateKey);
+    // Fetch
+    const subscriptions = await SwSubscriptions.findBy({
+        userId: userId
+    });
+    for (const subscription of subscriptions){
+        if ([
+            "readNotifications",
+            "readAllNotifications",
+            "readAllMessagingMessages",
+            "readAllMessagingMessagesOfARoom"
+        ].includes(type) && !subscription.sendReadMessage) continue;
+        const pushSubscription = {
+            endpoint: subscription.endpoint,
+            keys: {
+                auth: subscription.auth,
+                p256dh: subscription.publickey
+            }
+        };
+        push.sendNotification(pushSubscription, JSON.stringify({
+            type,
+            body: type === "notification" ? truncateNotification(body) : body,
+            userId,
+            dateTime: new Date().getTime()
+        }), {
+            proxy: config.proxy
+        }).catch((err)=>{
+            //swLogger.info(err.statusCode);
+            //swLogger.info(err.headers);
+            //swLogger.info(err.body);
+            if (err.statusCode === 410) {
+                SwSubscriptions.delete({
+                    userId: userId,
+                    endpoint: subscription.endpoint,
+                    auth: subscription.auth,
+                    publickey: subscription.publickey
+                });
+            }
+        });
+    }
+}

packages/backend/built/services/register-or-fetch-instance-doc.js

@@ -0,0 +1,26 @@
+import { Instances } from "../models/index.js";
+import { genId } from "../misc/gen-id.js";
+import { toPuny } from "../misc/convert-host.js";
+import { Cache } from "../misc/cache.js";
+const cache = new Cache("registerOrFetchInstanceDoc", 60 * 60);
+export async function registerOrFetchInstanceDoc(host) {
+    const _host = toPuny(host);
+    const cached = await cache.get(_host);
+    if (cached) return cached;
+    const index = await Instances.findOneBy({
+        host: _host
+    });
+    if (index == null) {
+        const i = await Instances.insert({
+            id: genId(),
+            host: _host,
+            caughtAt: new Date(),
+            lastCommunicatedAt: new Date()
+        }).then((x)=>Instances.findOneByOrFail(x.identifiers[0]));
+        await cache.set(_host, i);
+        return i;
+    } else {
+        await cache.set(_host, index);
+        return index;
+    }
+}

packages/backend/built/services/relay.js

@@ -0,0 +1,90 @@
+import { IsNull } from "typeorm";
+import { renderFollowRelay } from "../remote/activitypub/renderer/follow-relay.js";
+import { renderActivity, attachLdSignature } from "../remote/activitypub/renderer/index.js";
+import renderUndo from "../remote/activitypub/renderer/undo.js";
+import { deliver } from "../queue/index.js";
+import { Users, Relays } from "../models/index.js";
+import { genId } from "../misc/gen-id.js";
+import { Cache } from "../misc/cache.js";
+import { createSystemUser } from "./create-system-user.js";
+const ACTOR_USERNAME = "relay.actor";
+const relaysCache = new Cache("relay", 60 * 60);
+export async function getRelayActor() {
+    const user = await Users.findOneBy({
+        host: IsNull(),
+        username: ACTOR_USERNAME
+    });
+    if (user) return user;
+    const created = await createSystemUser(ACTOR_USERNAME);
+    return created;
+}
+export async function addRelay(inbox) {
+    const relay = await Relays.insert({
+        id: genId(),
+        inbox,
+        status: "requesting"
+    }).then((x)=>Relays.findOneByOrFail(x.identifiers[0]));
+    const relayActor = await getRelayActor();
+    const follow = renderFollowRelay(relay, relayActor);
+    const activity = renderActivity(follow);
+    deliver(relayActor, activity, relay.inbox);
+    return relay;
+}
+export async function removeRelay(inbox) {
+    const relay = await Relays.findOneBy({
+        inbox
+    });
+    if (relay == null) {
+        throw new Error("relay not found");
+    }
+    const relayActor = await getRelayActor();
+    const follow = renderFollowRelay(relay, relayActor);
+    const undo = renderUndo(follow, relayActor);
+    const activity = renderActivity(undo);
+    deliver(relayActor, activity, relay.inbox);
+    await Relays.delete(relay.id);
+    await updateRelaysCache();
+}
+export async function listRelay() {
+    const relays = await Relays.find();
+    return relays;
+}
+export async function getCachedRelays() {
+    return await relaysCache.fetch(null, ()=>Relays.findBy({
+            status: "accepted"
+        }));
+}
+export async function relayAccepted(id) {
+    const result = await Relays.update(id, {
+        status: "accepted"
+    });
+    await updateRelaysCache();
+    return JSON.stringify(result);
+}
+async function updateRelaysCache() {
+    const relays = await Relays.findBy({
+        status: "accepted"
+    });
+    await relaysCache.set(null, relays);
+}
+export async function relayRejected(id) {
+    const result = await Relays.update(id, {
+        status: "rejected"
+    });
+    return JSON.stringify(result);
+}
+export async function deliverToRelays(user, activity) {
+    if (activity == null) return;
+    const relays = await getCachedRelays();
+    if (relays.length === 0) return;
+    // TODO
+    //const copy = structuredClone(activity);
+    const copy = JSON.parse(JSON.stringify(activity));
+    if (!copy.to) copy.to = [
+        "https://www.w3.org/ns/activitystreams#Public"
+    ];
+    const signed = await attachLdSignature(copy, user);
+    for (const relay of relays){
+        deliver(user, signed, relay.inbox);
+    }
+}

packages/backend/built/services/reversi/engine.js

@@ -0,0 +1,183 @@
+export const standardMap = [
+    "--------",
+    "--------",
+    "--------",
+    "---wb---",
+    "---bw---",
+    "--------",
+    "--------",
+    "--------"
+];
+export class ReversiEngine {
+    map;
+    mapWidth;
+    mapHeight;
+    board;
+    turn = true;
+    prevColor = null;
+    opts;
+    constructor(map, opts){
+        this.opts = {
+            isLlotheo: opts.isLlotheo ?? false,
+            canPutEverywhere: opts.canPutEverywhere ?? false,
+            loopedBoard: opts.loopedBoard ?? false
+        };
+        this.mapWidth = map[0]?.length ?? 0;
+        this.mapHeight = map.length;
+        const data = map.join("");
+        this.board = data.split("").map((d)=>d === "-" ? null : d === "b" ? true : d === "w" ? false : undefined);
+        this.map = data.split("").map((d)=>d === "-" || d === "b" || d === "w" ? "empty" : "null");
+        if (!this.canPutSomewhere(true)) this.turn = this.canPutSomewhere(false) ? false : null;
+    }
+    get blackCount() {
+        return this.board.filter((x)=>x === true).length;
+    }
+    get whiteCount() {
+        return this.board.filter((x)=>x === false).length;
+    }
+    posToXy(pos) {
+        return [
+            pos % this.mapWidth,
+            Math.floor(pos / this.mapWidth)
+        ];
+    }
+    xyToPos(x, y) {
+        return x + y * this.mapWidth;
+    }
+    mapDataGet(pos) {
+        const [x, y] = this.posToXy(pos);
+        return x < 0 || y < 0 || x >= this.mapWidth || y >= this.mapHeight ? "null" : this.map[pos];
+    }
+    getPuttablePlaces(color) {
+        return Array.from(this.board.keys()).filter((i)=>this.canPut(color, i));
+    }
+    canPutSomewhere(color) {
+        return this.getPuttablePlaces(color).length > 0;
+    }
+    canPut(color, pos) {
+        if (this.board[pos] !== null) return false;
+        if (this.opts.canPutEverywhere) return this.mapDataGet(pos) === "empty";
+        return this.effects(color, pos).length !== 0;
+    }
+    effects(color, initPos) {
+        const enemyColor = !color;
+        const diffVectors = [
+            [
+                0,
+                -1
+            ],
+            [
+                1,
+                -1
+            ],
+            [
+                1,
+                0
+            ],
+            [
+                1,
+                1
+            ],
+            [
+                0,
+                1
+            ],
+            [
+                -1,
+                1
+            ],
+            [
+                -1,
+                0
+            ],
+            [
+                -1,
+                -1
+            ]
+        ];
+        return diffVectors.flatMap(([dx, dy])=>{
+            const found = [];
+            let [x, y] = this.posToXy(initPos);
+            while(true){
+                x += dx;
+                y += dy;
+                if (this.opts.loopedBoard && this.xyToPos(x = (x % this.mapWidth + this.mapWidth) % this.mapWidth, y = (y % this.mapHeight + this.mapHeight) % this.mapHeight) === initPos) {
+                    return found;
+                }
+                if (x === -1 || y === -1 || x === this.mapWidth || y === this.mapHeight) return [];
+                const pos = this.xyToPos(x, y);
+                if (this.mapDataGet(pos) === "null") return [];
+                const stone = this.board[pos];
+                if (stone === null) return [];
+                if (stone === enemyColor) found.push(pos);
+                if (stone === color) return found;
+            }
+        });
+    }
+    putStone(pos) {
+        const color = this.turn;
+        if (color == null) return;
+        this.board[pos] = color;
+        for (const effect of this.effects(color, pos)){
+            this.board[effect] = color;
+        }
+        this.prevColor = color;
+        this.turn = this.canPutSomewhere(!color) ? !color : this.canPutSomewhere(color) ? color : null;
+    }
+    get isEnded() {
+        return this.turn == null;
+    }
+    get winner() {
+        if (!this.isEnded) return null;
+        if (this.blackCount === this.whiteCount) return null;
+        return this.opts.isLlotheo === this.blackCount > this.whiteCount ? false : true;
+    }
+    calcCrc32() {
+        let hash = 0;
+        const text = JSON.stringify({
+            board: this.board,
+            turn: this.turn
+        });
+        for(let i = 0; i < text.length; i++){
+            hash = hash * 31 + text.charCodeAt(i) | 0;
+        }
+        return hash.toString();
+    }
+}
+export function serializeLogs(logs) {
+    const serialized = [];
+    for(let i = 0; i < logs.length; i++){
+        const log = logs[i];
+        const timeDelta = i === 0 ? log.time : log.time - logs[i - 1].time;
+        serialized.push([
+            timeDelta,
+            log.player ? 1 : 0,
+            0,
+            log.pos
+        ]);
+    }
+    return serialized;
+}
+export function deserializeLogs(logs) {
+    const deserialized = [];
+    let time = 0;
+    for (const log of logs){
+        time += log[0];
+        if (log[2] === 0) {
+            deserialized.push({
+                time,
+                player: log[1] === 1,
+                operation: "put",
+                pos: log[3]
+            });
+        }
+    }
+    return deserialized;
+}
+export function restoreGame(env) {
+    const game = new ReversiEngine(env.map, env);
+    for (const log of deserializeLogs(env.logs)){
+        if (log.operation === "put") game.putStone(log.pos);
+    }
+    return game;
+}

packages/backend/built/services/reversi/index.js

@@ -0,0 +1,328 @@
+import { IsNull, LessThan, MoreThan } from "typeorm";
+import { redisClient } from "../../db/redis.js";
+import { genId } from "../../misc/gen-id.js";
+import { ReversiGames, Users } from "../../models/index.js";
+import { deserializeLogs, restoreGame, serializeLogs, standardMap, ReversiEngine } from "./engine.js";
+import { publishReversiGameStream, publishReversiStream } from "../stream.js";
+const INVITATION_TIMEOUT_MS = 1000 * 20;
+const TIME_LIMIT_MIN = 5;
+const TIME_LIMIT_MAX = 300;
+const updateKeys = [
+    "map",
+    "bw",
+    "isLlotheo",
+    "canPutEverywhere",
+    "loopedBoard",
+    "timeLimitForEachTurn"
+];
+async function getGame(id) {
+    return await ReversiGames.findOne({
+        where: {
+            id
+        },
+        relations: {
+            user1: true,
+            user2: true
+        }
+    });
+}
+async function publishGame(id, type, body) {
+    publishReversiGameStream(id, type, body);
+}
+async function endGame(game, winnerId, reason) {
+    await ReversiGames.update(game.id, {
+        isEnded: true,
+        endedAt: new Date(),
+        winnerId,
+        surrenderedUserId: reason === "surrender" ? winnerId === game.user1Id ? game.user2Id : game.user1Id : null,
+        timeoutUserId: reason === "timeout" ? winnerId === game.user1Id ? game.user2Id : game.user1Id : null
+    });
+    const fresh = await getGame(game.id);
+    if (fresh) {
+        await publishGame(game.id, "ended", {
+            winnerId,
+            game: await ReversiGames.packDetail(fresh)
+        });
+    }
+}
+export function isValidUpdateKey(key) {
+    return typeof key === "string" && updateKeys.includes(key);
+}
+export function isValidUpdateValue(key, value) {
+    switch(key){
+        case "map":
+            return Array.isArray(value) && value.length > 0 && value.every((row)=>typeof row === "string" && row.length <= 64);
+        case "bw":
+            return value === "random" || value === "1" || value === "2";
+        case "isLlotheo":
+        case "canPutEverywhere":
+        case "loopedBoard":
+            return typeof value === "boolean";
+        case "timeLimitForEachTurn":
+            return typeof value === "number" && value >= TIME_LIMIT_MIN && value <= TIME_LIMIT_MAX;
+    }
+}
+export async function matchSpecificUser(me, target, multiple = false) {
+    if (!multiple) {
+        const since = genId(new Date(Date.now() - 1000 * 60 * 3));
+        const games = await ReversiGames.find({
+            where: [
+                {
+                    id: MoreThan(since),
+                    user1Id: me.id,
+                    user2Id: target.id,
+                    isStarted: false
+                },
+                {
+                    id: MoreThan(since),
+                    user1Id: target.id,
+                    user2Id: me.id,
+                    isStarted: false
+                }
+            ],
+            relations: {
+                user1: true,
+                user2: true
+            },
+            order: {
+                id: "DESC"
+            }
+        });
+        if (games.length > 0) return games[0];
+    }
+    const invitations = await redisClient.zrangebyscore(`reversi:matchSpecific:${me.id}`, Date.now() - INVITATION_TIMEOUT_MS, "+inf");
+    if (invitations.includes(target.id)) {
+        await redisClient.zrem(`reversi:matchSpecific:${me.id}`, target.id);
+        return await matched(target.id, me.id, {
+            noIrregularRules: false
+        });
+    }
+    const pipeline = redisClient.pipeline();
+    pipeline.zadd(`reversi:matchSpecific:${target.id}`, Date.now(), me.id);
+    pipeline.expire(`reversi:matchSpecific:${target.id}`, 120);
+    await pipeline.exec();
+    publishReversiStream(target.id, "invited", {
+        user: await Users.pack(me.id, target, {
+            detail: false
+        })
+    });
+    return null;
+}
+export async function matchAnyUser(me, options, multiple = false) {
+    if (!multiple) {
+        const since = genId(new Date(Date.now() - 1000 * 60 * 3));
+        const games = await ReversiGames.find({
+            where: [
+                {
+                    id: MoreThan(since),
+                    user1Id: me.id,
+                    isStarted: false
+                },
+                {
+                    id: MoreThan(since),
+                    user2Id: me.id,
+                    isStarted: false
+                }
+            ],
+            relations: {
+                user1: true,
+                user2: true
+            },
+            order: {
+                id: "DESC"
+            }
+        });
+        if (games.length > 0) return games[0];
+    }
+    const invitations = await redisClient.zrangebyscore(`reversi:matchSpecific:${me.id}`, Date.now() - INVITATION_TIMEOUT_MS, "+inf");
+    if (invitations.length > 0) {
+        const inviterId = invitations[Math.floor(Math.random() * invitations.length)];
+        await redisClient.zrem(`reversi:matchSpecific:${me.id}`, inviterId);
+        return await matched(inviterId, me.id, {
+            noIrregularRules: false
+        });
+    }
+    const matchings = await redisClient.zrevrange("reversi:matchAny", 0, 2);
+    const items = matchings.filter((id)=>!id.startsWith(me.id));
+    if (items.length > 0) {
+        const [matchedUserId, option] = items[0].split(":");
+        await redisClient.zrem("reversi:matchAny", me.id, matchedUserId, `${me.id}:noIrregularRules`, `${matchedUserId}:noIrregularRules`);
+        return await matched(matchedUserId, me.id, {
+            noIrregularRules: options.noIrregularRules || option === "noIrregularRules"
+        });
+    }
+    const pipeline = redisClient.pipeline();
+    pipeline.zadd("reversi:matchAny", Date.now(), options.noIrregularRules ? `${me.id}:noIrregularRules` : me.id);
+    pipeline.expire("reversi:matchAny", 15);
+    await pipeline.exec();
+    return null;
+}
+async function matched(parentId, childId, options) {
+    const game = await ReversiGames.insert({
+        id: genId(),
+        startedAt: null,
+        endedAt: null,
+        user1Id: parentId,
+        user2Id: childId,
+        user1Ready: false,
+        user2Ready: false,
+        black: null,
+        isStarted: false,
+        isEnded: false,
+        winnerId: null,
+        surrenderedUserId: null,
+        timeoutUserId: null,
+        timeLimitForEachTurn: 90,
+        logs: [],
+        map: standardMap,
+        bw: "random",
+        noIrregularRules: options.noIrregularRules,
+        isLlotheo: false,
+        canPutEverywhere: false,
+        loopedBoard: false,
+        form1: null,
+        form2: null,
+        crc32: null
+    }).then((x)=>getGame(x.identifiers[0].id));
+    if (!game) throw new Error("failed to create reversi game");
+    publishReversiStream(parentId, "matched", {
+        game: await ReversiGames.packDetail(game)
+    });
+    return game;
+}
+export async function cancelMatch(user, userId) {
+    if (userId) await redisClient.zrem(`reversi:matchSpecific:${userId}`, user.id);
+    await redisClient.zrem("reversi:matchAny", user.id, `${user.id}:noIrregularRules`);
+}
+export async function getInvitations(user) {
+    return await redisClient.zrangebyscore(`reversi:matchSpecific:${user.id}`, Date.now() - INVITATION_TIMEOUT_MS, "+inf");
+}
+export async function gameReady(gameId, user, ready) {
+    const game = await getGame(gameId);
+    if (!game || game.isStarted) return;
+    if (game.user1Id !== user.id && game.user2Id !== user.id) return;
+    const patch = game.user1Id === user.id ? {
+        user1Ready: ready
+    } : {
+        user2Ready: ready
+    };
+    await ReversiGames.update(game.id, patch);
+    const fresh = await getGame(game.id);
+    if (!fresh) return;
+    await publishGame(game.id, "changeReadyStates", {
+        user1: fresh.user1Ready,
+        user2: fresh.user2Ready
+    });
+    if (fresh.user1Ready && fresh.user2Ready) setTimeout(()=>startGame(fresh.id), 3000);
+}
+export async function updateSettings(gameId, user, key, value) {
+    const game = await getGame(gameId);
+    if (!game || game.isStarted) return;
+    if (game.user1Id !== user.id && game.user2Id !== user.id) return;
+    if (game.user1Id === user.id && game.user1Ready) return;
+    if (game.user2Id === user.id && game.user2Ready) return;
+    if (game.noIrregularRules && (key === "isLlotheo" || key === "canPutEverywhere" || key === "loopedBoard")) return;
+    await ReversiGames.update(game.id, {
+        [key]: value
+    });
+    await publishGame(game.id, "updateSettings", {
+        userId: user.id,
+        key,
+        value
+    });
+}
+async function startGame(gameId) {
+    const game = await getGame(gameId);
+    if (!game || game.isStarted || game.isEnded || !game.user1Ready || !game.user2Ready) return;
+    const black = game.bw === "random" ? Math.random() > 0.5 ? 1 : 2 : parseInt(game.bw, 10);
+    const engine = new ReversiEngine(game.map, game);
+    await ReversiGames.update(game.id, {
+        startedAt: new Date(),
+        isStarted: true,
+        black,
+        crc32: engine.calcCrc32()
+    });
+    const fresh = await getGame(game.id);
+    if (!fresh) return;
+    if (engine.isEnded) {
+        const winnerId = engine.winner === true ? black === 1 ? game.user1Id : game.user2Id : engine.winner === false ? black === 1 ? game.user2Id : game.user1Id : null;
+        await endGame(fresh, winnerId, null);
+        return;
+    }
+    await redisClient.setex(`reversi:game:turnTimer:${game.id}:1`, fresh.timeLimitForEachTurn, "");
+    await publishGame(game.id, "started", {
+        game: await ReversiGames.packDetail(fresh)
+    });
+}
+export async function putStone(gameId, user, pos, id) {
+    const game = await getGame(gameId);
+    if (!game || !game.isStarted || game.isEnded) return;
+    if (game.user1Id !== user.id && game.user2Id !== user.id) return;
+    const myColor = game.user1Id === user.id && game.black === 1 || game.user2Id === user.id && game.black === 2;
+    const engine = restoreGame(game);
+    if (engine.turn !== myColor || !engine.canPut(myColor, pos)) return;
+    engine.putStone(pos);
+    const logs = deserializeLogs(game.logs);
+    const log = {
+        time: Date.now(),
+        player: myColor,
+        operation: "put",
+        pos
+    };
+    logs.push(log);
+    const serialized = serializeLogs(logs);
+    await ReversiGames.update(game.id, {
+        logs: serialized,
+        crc32: engine.calcCrc32()
+    });
+    await publishGame(game.id, "log", {
+        ...log,
+        id: id ?? null
+    });
+    const fresh = await getGame(game.id);
+    if (!fresh) return;
+    if (engine.isEnded) {
+        const winnerId = engine.winner === true ? game.black === 1 ? game.user1Id : game.user2Id : engine.winner === false ? game.black === 1 ? game.user2Id : game.user1Id : null;
+        await endGame(fresh, winnerId, null);
+    } else if (engine.turn != null) {
+        await redisClient.setex(`reversi:game:turnTimer:${game.id}:${engine.turn ? "1" : "0"}`, fresh.timeLimitForEachTurn, "");
+    }
+}
+export async function surrender(gameId, user) {
+    const game = await getGame(gameId);
+    if (!game || game.isEnded) return;
+    if (game.user1Id !== user.id && game.user2Id !== user.id) return;
+    await endGame(game, game.user1Id === user.id ? game.user2Id : game.user1Id, "surrender");
+}
+export async function cancelGame(gameId, user) {
+    const game = await getGame(gameId);
+    if (!game || game.isStarted) return;
+    if (game.user1Id !== user.id && game.user2Id !== user.id) return;
+    await ReversiGames.delete(game.id);
+    await publishGame(game.id, "canceled", {
+        userId: user.id
+    });
+}
+export async function checkTimeout(gameId) {
+    const game = await getGame(gameId);
+    if (!game || game.isEnded) return;
+    const engine = restoreGame(game);
+    if (engine.turn == null) return;
+    const timer = await redisClient.exists(`reversi:game:turnTimer:${game.id}:${engine.turn ? "1" : "0"}`);
+    if (timer === 0) {
+        const winnerId = engine.turn ? game.black === 1 ? game.user2Id : game.user1Id : game.black === 1 ? game.user1Id : game.user2Id;
+        await endGame(game, winnerId, "timeout");
+    }
+}
+export async function checkCrc(gameId, crc32) {
+    const game = await getGame(gameId);
+    if (!game) return null;
+    return crc32.toString() !== game.crc32 ? game : null;
+}
+export async function cleanupOpenGames() {
+    await ReversiGames.delete({
+        id: LessThan(genId(new Date(Date.now() - 1000 * 60 * 10))),
+        isStarted: false,
+        startedAt: IsNull()
+    });
+}

packages/backend/built/services/send-email-notification.js

@@ -0,0 +1,25 @@
+// TODO
+//const locales = await import('../../../../locales/index.js');
+// TODO: locale ファイルをクライアント用とサーバー用で分けたい
+async function follow(userId, follower) {
+/*
+	const userProfile = await UserProfiles.findOneByOrFail({ userId: userId });
+	if (!userProfile.email || !userProfile.emailNotificationTypes.includes('follow')) return;
+	const locale = locales[userProfile.lang || 'ja-JP'];
+	const i18n = new I18n(locale);
+	// TODO: render user information html
+	sendEmail(userProfile.email, i18n.t('_email._follow.title'), `${follower.name} (@${Acct.toString(follower)})`, `${follower.name} (@${Acct.toString(follower)})`);
+	*/ }
+async function receiveFollowRequest(userId, follower) {
+/*
+	const userProfile = await UserProfiles.findOneByOrFail({ userId: userId });
+	if (!userProfile.email || !userProfile.emailNotificationTypes.includes('receiveFollowRequest')) return;
+	const locale = locales[userProfile.lang || 'ja-JP'];
+	const i18n = new I18n(locale);
+	// TODO: render user information html
+	sendEmail(userProfile.email, i18n.t('_email._receiveFollowRequest.title'), `${follower.name} (@${Acct.toString(follower)})`, `${follower.name} (@${Acct.toString(follower)})`);
+	*/ }
+export const sendEmailNotification = {
+    follow,
+    receiveFollowRequest
+};

packages/backend/built/services/send-email.js

@@ -0,0 +1,59 @@
+import * as nodemailer from "nodemailer";
+import { fetchMeta } from "../misc/fetch-meta.js";
+import Logger from "./logger.js";
+import config from "../config/index.js";
+export const logger = new Logger("email");
+export async function sendEmail(to, subject, html, text) {
+    const meta = await fetchMeta(true);
+    const iconUrl = `${config.url}/static-assets/mail-wordmark.png`;
+    const emailSettingUrl = `${config.url}/settings/email`;
+    const enableAuth = meta.smtpUser != null && meta.smtpUser !== "";
+    const transporter = nodemailer.createTransport({
+        host: meta.smtpHost,
+        port: meta.smtpPort,
+        secure: meta.smtpSecure,
+        ignoreTLS: !enableAuth,
+        proxy: config.proxySmtp,
+        auth: enableAuth ? {
+            user: meta.smtpUser,
+            pass: meta.smtpPass
+        } : undefined
+    });
+    try {
+        const info = await transporter.sendMail({
+            from: meta.email,
+            to: to,
+            subject: subject,
+            text: text,
+            html: `<!DOCTYPE html>
+<html>
+	<head>
+		<meta charset="utf-8">
+		<title>${subject}</title>
+	</head>
+	<body style="background: #191724; padding: 16px; margin: 0; font-family: sans-serif; font-size: 14px;">
+		<main style="max-width: 500px; margin: 0 auto; background: #1f1d2e; color: #e0def4; border-radius: 20px;">
+			<header style="padding: 32px; background: #31748f; color: #e0def4; display: flex; border-radius: 20px;">
+				<img src="${meta.logoImageUrl || meta.iconUrl || iconUrl}" style="max-width: 128px; max-height: 72px; vertical-align: bottom; margin-right: 16px;"/>
+				<h1 style="margin: 0 0 1em 0;">${meta.name}</h1>
+			</header>
+			<article style="padding: 32px;">
+				<h1 style="color: #ebbcba !important;">${subject}</h1>
+				<div style="color: #e0def4;">${html}</div>
+			</article>
+			<footer style="padding: 32px; border-top: solid 1px #26233a;">
+				<a href="${emailSettingUrl}" style="color: #9ccfd8 !important;">${"Email Settings"}</a>
+			</footer>
+		</main>
+		<nav style="box-sizing: border-box; max-width: 500px; margin: 16px auto 0 auto; padding: 0 32px;">
+			<a href="${config.url}" style="color: #9ccfd8 !important;">${config.domain}</a>
+		</nav>
+	</body>
+</html>`
+        });
+        logger.info(`Message sent: ${info.messageId}`);
+    } catch (err) {
+        logger.error(err);
+        throw err;
+    }
+}

packages/backend/built/services/shogi/index.js

@@ -0,0 +1,250 @@
+import { IsNull, LessThan, MoreThan } from "typeorm";
+import { initialSfen, makeSfen, parseSfen } from "shogiops/sfen";
+import { makeUsi, parseUsi } from "shogiops/util";
+import { redisClient } from "../../db/redis.js";
+import { genId } from "../../misc/gen-id.js";
+import { ShogiGames, Users } from "../../models/index.js";
+import { publishShogiGameStream, publishShogiStream } from "../stream.js";
+const INVITATION_TIMEOUT_MS = 1000 * 20;
+const RULES = "standard";
+async function getGame(id) {
+    return await ShogiGames.findOne({
+        where: {
+            id
+        },
+        relations: {
+            user1: true,
+            user2: true
+        }
+    });
+}
+async function publishGame(id, type, body) {
+    publishShogiGameStream(id, type, body);
+}
+function colorUserId(game, color) {
+    const senteUserId = game.sente === 2 ? game.user2Id : game.user1Id;
+    const goteUserId = game.sente === 2 ? game.user1Id : game.user2Id;
+    return color === "sente" ? senteUserId : goteUserId;
+}
+async function endGame(game, winnerId, surrenderedUserId) {
+    await ShogiGames.update(game.id, {
+        isEnded: true,
+        endedAt: new Date(),
+        winnerId,
+        surrenderedUserId
+    });
+    const fresh = await getGame(game.id);
+    if (fresh) {
+        await publishGame(game.id, "ended", {
+            winnerId,
+            game: await ShogiGames.packDetail(fresh)
+        });
+    }
+}
+export async function matchSpecificUser(me, target, multiple = false) {
+    if (!multiple) {
+        const since = genId(new Date(Date.now() - 1000 * 60 * 3));
+        const games = await ShogiGames.find({
+            where: [
+                {
+                    id: MoreThan(since),
+                    user1Id: me.id,
+                    user2Id: target.id,
+                    isStarted: false
+                },
+                {
+                    id: MoreThan(since),
+                    user1Id: target.id,
+                    user2Id: me.id,
+                    isStarted: false
+                }
+            ],
+            relations: {
+                user1: true,
+                user2: true
+            },
+            order: {
+                id: "DESC"
+            }
+        });
+        if (games.length > 0) return games[0];
+    }
+    const invitations = await redisClient.zrangebyscore(`shogi:matchSpecific:${me.id}`, Date.now() - INVITATION_TIMEOUT_MS, "+inf");
+    if (invitations.includes(target.id)) {
+        await redisClient.zrem(`shogi:matchSpecific:${me.id}`, target.id);
+        return await matched(target.id, me.id);
+    }
+    const pipeline = redisClient.pipeline();
+    pipeline.zadd(`shogi:matchSpecific:${target.id}`, Date.now(), me.id);
+    pipeline.expire(`shogi:matchSpecific:${target.id}`, 120);
+    await pipeline.exec();
+    publishShogiStream(target.id, "invited", {
+        user: await Users.pack(me.id, target, {
+            detail: false
+        })
+    });
+    return null;
+}
+export async function matchAnyUser(me, multiple = false) {
+    if (!multiple) {
+        const since = genId(new Date(Date.now() - 1000 * 60 * 3));
+        const games = await ShogiGames.find({
+            where: [
+                {
+                    id: MoreThan(since),
+                    user1Id: me.id,
+                    isStarted: false
+                },
+                {
+                    id: MoreThan(since),
+                    user2Id: me.id,
+                    isStarted: false
+                }
+            ],
+            relations: {
+                user1: true,
+                user2: true
+            },
+            order: {
+                id: "DESC"
+            }
+        });
+        if (games.length > 0) return games[0];
+    }
+    const invitations = await redisClient.zrangebyscore(`shogi:matchSpecific:${me.id}`, Date.now() - INVITATION_TIMEOUT_MS, "+inf");
+    if (invitations.length > 0) {
+        const inviterId = invitations[Math.floor(Math.random() * invitations.length)];
+        await redisClient.zrem(`shogi:matchSpecific:${me.id}`, inviterId);
+        return await matched(inviterId, me.id);
+    }
+    const matchings = await redisClient.zrevrange("shogi:matchAny", 0, 1);
+    const matchedUserId = matchings.find((id)=>id !== me.id);
+    if (matchedUserId) {
+        await redisClient.zrem("shogi:matchAny", me.id, matchedUserId);
+        return await matched(matchedUserId, me.id);
+    }
+    const pipeline = redisClient.pipeline();
+    pipeline.zadd("shogi:matchAny", Date.now(), me.id);
+    pipeline.expire("shogi:matchAny", 15);
+    await pipeline.exec();
+    return null;
+}
+async function matched(parentId, childId) {
+    const game = await ShogiGames.insert({
+        id: genId(),
+        startedAt: null,
+        endedAt: null,
+        user1Id: parentId,
+        user2Id: childId,
+        user1Ready: false,
+        user2Ready: false,
+        sente: null,
+        isStarted: false,
+        isEnded: false,
+        winnerId: null,
+        surrenderedUserId: null,
+        sfen: initialSfen(RULES),
+        logs: []
+    }).then((x)=>getGame(x.identifiers[0].id));
+    if (!game) throw new Error("failed to create shogi game");
+    publishShogiStream(parentId, "matched", {
+        game: await ShogiGames.packDetail(game)
+    });
+    return game;
+}
+export async function cancelMatch(user, userId) {
+    if (userId) await redisClient.zrem(`shogi:matchSpecific:${userId}`, user.id);
+    await redisClient.zrem("shogi:matchAny", user.id);
+}
+export async function getInvitations(user) {
+    return await redisClient.zrangebyscore(`shogi:matchSpecific:${user.id}`, Date.now() - INVITATION_TIMEOUT_MS, "+inf");
+}
+export async function gameReady(gameId, user, ready) {
+    const game = await getGame(gameId);
+    if (!game || game.isStarted) return;
+    if (game.user1Id !== user.id && game.user2Id !== user.id) return;
+    const patch = game.user1Id === user.id ? {
+        user1Ready: ready
+    } : {
+        user2Ready: ready
+    };
+    await ShogiGames.update(game.id, patch);
+    const fresh = await getGame(game.id);
+    if (!fresh) return;
+    await publishGame(game.id, "changeReadyStates", {
+        user1: fresh.user1Ready,
+        user2: fresh.user2Ready
+    });
+    if (fresh.user1Ready && fresh.user2Ready) setTimeout(()=>startGame(fresh.id), 3000);
+}
+async function startGame(gameId) {
+    const game = await getGame(gameId);
+    if (!game || game.isStarted || game.isEnded || !game.user1Ready || !game.user2Ready) return;
+    await ShogiGames.update(game.id, {
+        startedAt: new Date(),
+        isStarted: true,
+        sente: Math.random() > 0.5 ? 1 : 2
+    });
+    const fresh = await getGame(game.id);
+    if (!fresh) return;
+    await publishGame(game.id, "started", {
+        game: await ShogiGames.packDetail(fresh)
+    });
+}
+export async function putMove(gameId, user, usi, id) {
+    const game = await getGame(gameId);
+    if (!game || !game.isStarted || game.isEnded || game.sente == null) return;
+    if (game.user1Id !== user.id && game.user2Id !== user.id) return;
+    const pos = parseSfen(RULES, game.sfen).unwrap();
+    const myColor = colorUserId(game, pos.turn) === user.id ? pos.turn : null;
+    if (myColor == null) return;
+    const move = parseUsi(usi);
+    if (!move || !pos.isLegal(move)) return;
+    pos.play(move);
+    const nextSfen = makeSfen(pos);
+    const log = {
+        id: id ?? null,
+        at: Date.now(),
+        userId: user.id,
+        usi: makeUsi(move),
+        sfen: nextSfen
+    };
+    const logs = [
+        ...game.logs ?? [],
+        log
+    ];
+    await ShogiGames.update(game.id, {
+        sfen: nextSfen,
+        logs
+    });
+    await publishGame(game.id, "log", log);
+    const outcome = pos.outcome();
+    const fresh = await getGame(game.id);
+    if (!fresh) return;
+    if (outcome) {
+        const winnerId = outcome.winner ? colorUserId(fresh, outcome.winner) : null;
+        await endGame(fresh, winnerId, null);
+    }
+}
+export async function surrender(gameId, user) {
+    const game = await getGame(gameId);
+    if (!game || game.isEnded) return;
+    if (game.user1Id !== user.id && game.user2Id !== user.id) return;
+    await endGame(game, game.user1Id === user.id ? game.user2Id : game.user1Id, user.id);
+}
+export async function cancelGame(gameId, user) {
+    const game = await getGame(gameId);
+    if (!game || game.isStarted) return;
+    if (game.user1Id !== user.id && game.user2Id !== user.id) return;
+    await ShogiGames.delete(game.id);
+    await publishGame(game.id, "canceled", {
+        userId: user.id
+    });
+}
+export async function cleanupOpenGames() {
+    await ShogiGames.delete({
+        id: LessThan(genId(new Date(Date.now() - 1000 * 60 * 10))),
+        isStarted: false,
+        startedAt: IsNull()
+    });
+}

packages/backend/built/services/stream.js

@@ -0,0 +1,99 @@
+import { redisClient } from "../db/redis.js";
+import config from "../config/index.js";
+class Publisher {
+    publish = (channel, type, value)=>{
+        const message = type == null ? value : value == null ? {
+            type: type,
+            body: null
+        } : {
+            type: type,
+            body: value
+        };
+        redisClient.publish(config.redis.prefix ?? config.host, JSON.stringify({
+            channel: channel,
+            message: message
+        }));
+    };
+    publishInternalEvent = (type, value)=>{
+        this.publish("internal", type, typeof value === "undefined" ? null : value);
+    };
+    publishUserEvent = (userId, type, value)=>{
+        this.publish(`user:${userId}`, type, typeof value === "undefined" ? null : value);
+    };
+    publishBroadcastStream = (type, value)=>{
+        this.publish("broadcast", type, typeof value === "undefined" ? null : value);
+    };
+    publishMainStream = (userId, type, value)=>{
+        this.publish(`mainStream:${userId}`, type, typeof value === "undefined" ? null : value);
+    };
+    publishDriveStream = (userId, type, value)=>{
+        this.publish(`driveStream:${userId}`, type, typeof value === "undefined" ? null : value);
+    };
+    publishNoteStream = (noteId, type, value)=>{
+        const object = {
+            id: noteId,
+            body: value
+        };
+        this.publish(`noteStream:${noteId}`, type, object);
+    };
+    publishNoteUpdatesStream = (type, value)=>{
+        this.publish('noteUpdatesStream', type, value);
+    };
+    publishChannelStream = (channelId, type, value)=>{
+        this.publish(`channelStream:${channelId}`, type, typeof value === "undefined" ? null : value);
+    };
+    publishUserListStream = (listId, type, value)=>{
+        this.publish(`userListStream:${listId}`, type, typeof value === "undefined" ? null : value);
+    };
+    publishAntennaStream = (antennaId, type, value)=>{
+        this.publish(`antennaStream:${antennaId}`, type, typeof value === "undefined" ? null : value);
+    };
+    publishMessagingStream = (userId, otherpartyId, type, value)=>{
+        this.publish(`messagingStream:${userId}-${otherpartyId}`, type, typeof value === "undefined" ? null : value);
+    };
+    publishGroupMessagingStream = (groupId, type, value)=>{
+        this.publish(`messagingStream:${groupId}`, type, typeof value === "undefined" ? null : value);
+    };
+    publishMessagingIndexStream = (userId, type, value)=>{
+        this.publish(`messagingIndexStream:${userId}`, type, typeof value === "undefined" ? null : value);
+    };
+    publishNotesStream = (note)=>{
+        this.publish("notesStream", null, note);
+    };
+    publishAdminStream = (userId, type, value)=>{
+        this.publish(`adminStream:${userId}`, type, typeof value === "undefined" ? null : value);
+    };
+    publishReversiStream = (userId, type, value)=>{
+        this.publish(`reversiStream:${userId}`, type, typeof value === "undefined" ? null : value);
+    };
+    publishReversiGameStream = (gameId, type, value)=>{
+        this.publish(`reversiGameStream:${gameId}`, type, typeof value === "undefined" ? null : value);
+    };
+    publishShogiStream = (userId, type, value)=>{
+        this.publish(`shogiStream:${userId}`, type, typeof value === "undefined" ? null : value);
+    };
+    publishShogiGameStream = (gameId, type, value)=>{
+        this.publish(`shogiGameStream:${gameId}`, type, typeof value === "undefined" ? null : value);
+    };
+}
+const publisher = new Publisher();
+export default publisher;
+export const publishInternalEvent = publisher.publishInternalEvent;
+export const publishUserEvent = publisher.publishUserEvent;
+export const publishBroadcastStream = publisher.publishBroadcastStream;
+export const publishMainStream = publisher.publishMainStream;
+export const publishDriveStream = publisher.publishDriveStream;
+export const publishNoteStream = publisher.publishNoteStream;
+export const publishNotesStream = publisher.publishNotesStream;
+export const publishNoteUpdatesStream = publisher.publishNoteUpdatesStream;
+export const publishChannelStream = publisher.publishChannelStream;
+export const publishUserListStream = publisher.publishUserListStream;
+export const publishAntennaStream = publisher.publishAntennaStream;
+export const publishMessagingStream = publisher.publishMessagingStream;
+export const publishGroupMessagingStream = publisher.publishGroupMessagingStream;
+export const publishMessagingIndexStream = publisher.publishMessagingIndexStream;
+export const publishAdminStream = publisher.publishAdminStream;
+export const publishReversiStream = publisher.publishReversiStream;
+export const publishReversiGameStream = publisher.publishReversiGameStream;
+export const publishShogiStream = publisher.publishShogiStream;
+export const publishShogiGameStream = publisher.publishShogiGameStream;

packages/backend/built/services/suspend-user.js

@@ -0,0 +1,39 @@
+import renderDelete from "../remote/activitypub/renderer/delete.js";
+import { renderActivity } from "../remote/activitypub/renderer/index.js";
+import { deliver } from "../queue/index.js";
+import config from "../config/index.js";
+import { Users, Followings } from "../models/index.js";
+import { Not, IsNull } from "typeorm";
+import { publishInternalEvent } from "./stream.js";
+export async function doPostSuspend(user) {
+    publishInternalEvent("userChangeSuspendedState", {
+        id: user.id,
+        isSuspended: true
+    });
+    if (Users.isLocalUser(user)) {
+        // Send Delete to all known SharedInboxes
+        const content = renderActivity(renderDelete(`${config.url}/users/${user.id}`, user));
+        const queue = [];
+        const followings = await Followings.find({
+            where: [
+                {
+                    followerSharedInbox: Not(IsNull())
+                },
+                {
+                    followeeSharedInbox: Not(IsNull())
+                }
+            ],
+            select: [
+                "followerSharedInbox",
+                "followeeSharedInbox"
+            ]
+        });
+        const inboxes = followings.map((x)=>x.followerSharedInbox || x.followeeSharedInbox);
+        for (const inbox of inboxes){
+            if (inbox != null && !queue.includes(inbox)) queue.push(inbox);
+        }
+        for (const inbox of queue){
+            deliver(user, content, inbox);
+        }
+    }
+}

packages/backend/built/services/unsuspend-user.js

@@ -0,0 +1,40 @@
+import renderDelete from "../remote/activitypub/renderer/delete.js";
+import renderUndo from "../remote/activitypub/renderer/undo.js";
+import { renderActivity } from "../remote/activitypub/renderer/index.js";
+import { deliver } from "../queue/index.js";
+import config from "../config/index.js";
+import { Users, Followings } from "../models/index.js";
+import { Not, IsNull } from "typeorm";
+import { publishInternalEvent } from "./stream.js";
+export async function doPostUnsuspend(user) {
+    publishInternalEvent("userChangeSuspendedState", {
+        id: user.id,
+        isSuspended: false
+    });
+    if (Users.isLocalUser(user)) {
+        // 知り得る全SharedInboxにUndo Delete配信
+        const content = renderActivity(renderUndo(renderDelete(`${config.url}/users/${user.id}`, user), user));
+        const queue = [];
+        const followings = await Followings.find({
+            where: [
+                {
+                    followerSharedInbox: Not(IsNull())
+                },
+                {
+                    followeeSharedInbox: Not(IsNull())
+                }
+            ],
+            select: [
+                "followerSharedInbox",
+                "followeeSharedInbox"
+            ]
+        });
+        const inboxes = followings.map((x)=>x.followerSharedInbox || x.followeeSharedInbox);
+        for (const inbox of inboxes){
+            if (inbox != null && !queue.includes(inbox)) queue.push(inbox);
+        }
+        for (const inbox of queue){
+            deliver(user, content, inbox);
+        }
+    }
+}

packages/backend/built/services/update-hashtag.js

@@ -0,0 +1,130 @@
+import { Hashtags, Users } from "../models/index.js";
+import { hashtagChart } from "./chart/index.js";
+import { genId } from "../misc/gen-id.js";
+import { normalizeForSearch } from "../misc/normalize-for-search.js";
+export async function updateHashtags(user, tags) {
+    for (const tag of tags){
+        await updateHashtag(user, tag);
+    }
+}
+export async function updateUsertags(user, tags) {
+    for (const tag of tags){
+        await updateHashtag(user, tag, true, true);
+    }
+    for (const tag of (user.tags || []).filter((x)=>!tags.includes(x))){
+        await updateHashtag(user, tag, true, false);
+    }
+}
+export async function updateHashtag(user, tag, isUserAttached = false, inc = true) {
+    tag = normalizeForSearch(tag);
+    const index = await Hashtags.findOneBy({
+        name: tag
+    });
+    if (index == null && !inc) return;
+    if (index != null) {
+        const q = Hashtags.createQueryBuilder("tag").update().where("name = :name", {
+            name: tag
+        });
+        const set = {};
+        if (isUserAttached) {
+            if (inc) {
+                // 自分が初めてこのタグを使ったなら
+                if (!index.attachedUserIds.some((id)=>id === user.id)) {
+                    set.attachedUserIds = ()=>`array_append("attachedUserIds", '${user.id}')`;
+                    set.attachedUsersCount = ()=>`"attachedUsersCount" + 1`;
+                }
+                // 自分が(ローカル内で)初めてこのタグを使ったなら
+                if (Users.isLocalUser(user) && !index.attachedLocalUserIds.some((id)=>id === user.id)) {
+                    set.attachedLocalUserIds = ()=>`array_append("attachedLocalUserIds", '${user.id}')`;
+                    set.attachedLocalUsersCount = ()=>`"attachedLocalUsersCount" + 1`;
+                }
+                // 自分が(リモートで)初めてこのタグを使ったなら
+                if (Users.isRemoteUser(user) && !index.attachedRemoteUserIds.some((id)=>id === user.id)) {
+                    set.attachedRemoteUserIds = ()=>`array_append("attachedRemoteUserIds", '${user.id}')`;
+                    set.attachedRemoteUsersCount = ()=>`"attachedRemoteUsersCount" + 1`;
+                }
+            } else {
+                set.attachedUserIds = ()=>`array_remove("attachedUserIds", '${user.id}')`;
+                set.attachedUsersCount = ()=>`"attachedUsersCount" - 1`;
+                if (Users.isLocalUser(user)) {
+                    set.attachedLocalUserIds = ()=>`array_remove("attachedLocalUserIds", '${user.id}')`;
+                    set.attachedLocalUsersCount = ()=>`"attachedLocalUsersCount" - 1`;
+                } else {
+                    set.attachedRemoteUserIds = ()=>`array_remove("attachedRemoteUserIds", '${user.id}')`;
+                    set.attachedRemoteUsersCount = ()=>`"attachedRemoteUsersCount" - 1`;
+                }
+            }
+        } else {
+            // 自分が初めてこのタグを使ったなら
+            if (!index.mentionedUserIds.some((id)=>id === user.id)) {
+                set.mentionedUserIds = ()=>`array_append("mentionedUserIds", '${user.id}')`;
+                set.mentionedUsersCount = ()=>`"mentionedUsersCount" + 1`;
+            }
+            // 自分が(ローカル内で)初めてこのタグを使ったなら
+            if (Users.isLocalUser(user) && !index.mentionedLocalUserIds.some((id)=>id === user.id)) {
+                set.mentionedLocalUserIds = ()=>`array_append("mentionedLocalUserIds", '${user.id}')`;
+                set.mentionedLocalUsersCount = ()=>`"mentionedLocalUsersCount" + 1`;
+            }
+            // 自分が(リモートで)初めてこのタグを使ったなら
+            if (Users.isRemoteUser(user) && !index.mentionedRemoteUserIds.some((id)=>id === user.id)) {
+                set.mentionedRemoteUserIds = ()=>`array_append("mentionedRemoteUserIds", '${user.id}')`;
+                set.mentionedRemoteUsersCount = ()=>`"mentionedRemoteUsersCount" + 1`;
+            }
+        }
+        if (Object.keys(set).length > 0) {
+            q.set(set);
+            q.execute();
+        }
+    } else {
+        if (isUserAttached) {
+            Hashtags.insert({
+                id: genId(),
+                name: tag,
+                mentionedUserIds: [],
+                mentionedUsersCount: 0,
+                mentionedLocalUserIds: [],
+                mentionedLocalUsersCount: 0,
+                mentionedRemoteUserIds: [],
+                mentionedRemoteUsersCount: 0,
+                attachedUserIds: [
+                    user.id
+                ],
+                attachedUsersCount: 1,
+                attachedLocalUserIds: Users.isLocalUser(user) ? [
+                    user.id
+                ] : [],
+                attachedLocalUsersCount: Users.isLocalUser(user) ? 1 : 0,
+                attachedRemoteUserIds: Users.isRemoteUser(user) ? [
+                    user.id
+                ] : [],
+                attachedRemoteUsersCount: Users.isRemoteUser(user) ? 1 : 0
+            });
+        } else {
+            Hashtags.insert({
+                id: genId(),
+                name: tag,
+                mentionedUserIds: [
+                    user.id
+                ],
+                mentionedUsersCount: 1,
+                mentionedLocalUserIds: Users.isLocalUser(user) ? [
+                    user.id
+                ] : [],
+                mentionedLocalUsersCount: Users.isLocalUser(user) ? 1 : 0,
+                mentionedRemoteUserIds: Users.isRemoteUser(user) ? [
+                    user.id
+                ] : [],
+                mentionedRemoteUsersCount: Users.isRemoteUser(user) ? 1 : 0,
+                attachedUserIds: [],
+                attachedUsersCount: 0,
+                attachedLocalUserIds: [],
+                attachedLocalUsersCount: 0,
+                attachedRemoteUserIds: [],
+                attachedRemoteUsersCount: 0
+            });
+        }
+    }
+    if (!isUserAttached) {
+        hashtagChart.update(tag, user);
+    }
+}

packages/backend/built/services/user-cache.js

@@ -0,0 +1,73 @@
+import { Users } from "../models/index.js";
+import { Cache } from "../misc/cache.js";
+import { redisClient, subscriber } from "../db/redis.js";
+export const userByIdCache = new Cache("userById", 60 * 30);
+export const localUserByNativeTokenCache = new Cache("localUserByNativeToken", 60 * 30);
+export const localUserByIdCache = new Cache("localUserByIdCache", 60 * 30);
+export const uriPersonCache = new Cache("uriPerson", 60 * 30);
+subscriber.on("message", async (_, data)=>{
+    const obj = JSON.parse(data);
+    if (obj.channel === "internal") {
+        const { type, body } = obj.message;
+        switch(type){
+            case "localUserDeleted":
+                {
+                    await userByIdCache.delete(body.id);
+                    await localUserByIdCache.delete(body.id);
+                    const toDelete = Array.from(await localUserByNativeTokenCache.getAll()).filter((v)=>v[1]?.id === body.id).map((v)=>v[0]);
+                    await localUserByNativeTokenCache.delete(...toDelete);
+                    const uriCacheToDelete = Array.from(await uriPersonCache.getAll()).filter((v)=>v[1]?.id === body.id).map((v)=>v[0]);
+                    await uriPersonCache.delete(...uriCacheToDelete);
+                    break;
+                }
+            case "localUserUpdated":
+                {
+                    await userByIdCache.delete(body.id);
+                    await localUserByIdCache.delete(body.id);
+                    const toDelete = Array.from(await localUserByNativeTokenCache.getAll()).filter((v)=>v[1]?.id === body.id).map((v)=>v[0]);
+                    await localUserByNativeTokenCache.delete(...toDelete);
+                    break;
+                }
+            case "userChangeSuspendedState":
+            case "userChangeSilencedState":
+            case "userChangeModeratorState":
+            case "remoteUserUpdated":
+                {
+                    const user = await Users.findOneByOrFail({
+                        id: body.id
+                    });
+                    await userByIdCache.set(user.id, user);
+                    const trans = redisClient.multi();
+                    for (const [k, v] of (await uriPersonCache.getAll()).entries()){
+                        if (v?.id === user.id) {
+                            await uriPersonCache.set(k, user, trans);
+                        }
+                    }
+                    await trans.exec();
+                    if (Users.isLocalUser(user)) {
+                        await localUserByNativeTokenCache.set(user.token, user);
+                        await localUserByIdCache.set(user.id, user);
+                    }
+                    break;
+                }
+            case "remoteUserDeleted":
+                {
+                    await userByIdCache.delete(body.id);
+                    const toDelete = Array.from(await uriPersonCache.getAll()).filter((v)=>v[1]?.id === body.id).map((v)=>v[0]);
+                    await uriPersonCache.delete(...toDelete);
+                    break;
+                }
+            case "userTokenRegenerated":
+                {
+                    const user = await Users.findOneByOrFail({
+                        id: body.id
+                    });
+                    await localUserByNativeTokenCache.delete(body.oldToken);
+                    await localUserByNativeTokenCache.set(body.newToken, user);
+                    break;
+                }
+            default:
+                break;
+        }
+    }
+});

packages/backend/built/services/user-list/pull.js

@@ -0,0 +1,11 @@
+import { publishUserEvent, publishUserListStream } from "../stream.js";
+import { UserListJoinings, Users } from "../../models/index.js";
+export async function pullUserFromUserList(target, list) {
+    await UserListJoinings.delete({
+        userListId: list.id,
+        userId: target.id
+    });
+    const packed = await Users.pack(target);
+    publishUserListStream(list.id, "userRemoved", packed);
+    if (list.hideFromHomeTl) publishUserEvent(list.userId, "userUnhidden", target.id);
+}

packages/backend/built/services/user-list/push.js

@@ -0,0 +1,14 @@
+import { publishUserEvent, publishUserListStream } from "../stream.js";
+import { UserListJoinings, Users } from "../../models/index.js";
+import { genId } from "../../misc/gen-id.js";
+export async function pushUserToUserList(target, list) {
+    await UserListJoinings.insert({
+        id: genId(),
+        createdAt: new Date(),
+        userId: target.id,
+        userListId: list.id
+    });
+    const packed = await Users.pack(target);
+    publishUserListStream(list.id, "userAdded", packed);
+    if (list.hideFromHomeTl) publishUserEvent(list.userId, "userHidden", target.id);
+}

packages/backend/built/services/validate-email-for-account.js

@@ -0,0 +1,25 @@
+import { validate as validateEmail } from "deep-email-validator";
+import { UserProfiles } from "../models/index.js";
+import { fetchMeta } from "../misc/fetch-meta.js";
+export async function validateEmailForAccount(emailAddress) {
+    const meta = await fetchMeta();
+    const exist = await UserProfiles.countBy({
+        emailVerified: true,
+        email: emailAddress
+    });
+    const validated = meta.enableActiveEmailValidation ? await validateEmail({
+        email: emailAddress,
+        validateRegex: true,
+        validateMx: true,
+        validateTypo: false,
+        validateDisposable: true,
+        validateSMTP: false
+    }) : {
+        valid: true
+    };
+    const available = exist === 0 && validated.valid;
+    return {
+        available,
+        reason: available ? null : exist !== 0 ? "used" : validated.reason === "regex" ? "format" : validated.reason === "disposable" ? "disposable" : validated.reason === "mx" ? "mx" : validated.reason === "smtp" ? "smtp" : null
+    };
+}