Fixed 267U.pre2
This commit is contained in:
@@ -0,0 +1,50 @@
|
||||
import Chart from "../core.js";
|
||||
import { name, schema } from "./entities/active-users.js";
|
||||
const week = 1000 * 60 * 60 * 24 * 7;
|
||||
const month = 1000 * 60 * 60 * 24 * 30;
|
||||
const year = 1000 * 60 * 60 * 24 * 365;
|
||||
/**
|
||||
* アクティブユーザーに関するチャート
|
||||
*/ export default class ActiveUsersChart extends Chart {
|
||||
constructor(){
|
||||
super(name, schema);
|
||||
}
|
||||
async tickMajor() {
|
||||
return {};
|
||||
}
|
||||
async tickMinor() {
|
||||
return {};
|
||||
}
|
||||
read(user) {
|
||||
this.commit({
|
||||
read: [
|
||||
user.id
|
||||
],
|
||||
registeredWithinWeek: Date.now() - user.createdAt.getTime() < week ? [
|
||||
user.id
|
||||
] : [],
|
||||
registeredWithinMonth: Date.now() - user.createdAt.getTime() < month ? [
|
||||
user.id
|
||||
] : [],
|
||||
registeredWithinYear: Date.now() - user.createdAt.getTime() < year ? [
|
||||
user.id
|
||||
] : [],
|
||||
registeredOutsideWeek: Date.now() - user.createdAt.getTime() > week ? [
|
||||
user.id
|
||||
] : [],
|
||||
registeredOutsideMonth: Date.now() - user.createdAt.getTime() > month ? [
|
||||
user.id
|
||||
] : [],
|
||||
registeredOutsideYear: Date.now() - user.createdAt.getTime() > year ? [
|
||||
user.id
|
||||
] : []
|
||||
});
|
||||
}
|
||||
async write(user) {
|
||||
await this.commit({
|
||||
write: [
|
||||
user.id
|
||||
]
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
import Chart from "../core.js";
|
||||
import { name, schema } from "./entities/ap-request.js";
|
||||
/**
|
||||
* Chart about ActivityPub requests
|
||||
*/ export default class ApRequestChart extends Chart {
|
||||
constructor(){
|
||||
super(name, schema);
|
||||
}
|
||||
async tickMajor() {
|
||||
return {};
|
||||
}
|
||||
async tickMinor() {
|
||||
return {};
|
||||
}
|
||||
async deliverSucc() {
|
||||
await this.commit({
|
||||
deliverSucceeded: 1
|
||||
});
|
||||
}
|
||||
async deliverFail() {
|
||||
await this.commit({
|
||||
deliverFailed: 1
|
||||
});
|
||||
}
|
||||
async inbox() {
|
||||
await this.commit({
|
||||
inboxReceived: 1
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
import Chart from "../core.js";
|
||||
import { name, schema } from "./entities/drive.js";
|
||||
/**
|
||||
* ドライブに関するチャート
|
||||
*/ export default class DriveChart extends Chart {
|
||||
constructor(){
|
||||
super(name, schema);
|
||||
}
|
||||
async tickMajor() {
|
||||
return {};
|
||||
}
|
||||
async tickMinor() {
|
||||
return {};
|
||||
}
|
||||
async update(file, isAdditional) {
|
||||
const fileSizeKb = file.size / 1000;
|
||||
await this.commit(file.userHost === null ? {
|
||||
"local.incCount": isAdditional ? 1 : 0,
|
||||
"local.incSize": isAdditional ? fileSizeKb : 0,
|
||||
"local.decCount": isAdditional ? 0 : 1,
|
||||
"local.decSize": isAdditional ? 0 : fileSizeKb
|
||||
} : {
|
||||
"remote.incCount": isAdditional ? 1 : 0,
|
||||
"remote.incSize": isAdditional ? fileSizeKb : 0,
|
||||
"remote.decCount": isAdditional ? 0 : 1,
|
||||
"remote.decSize": isAdditional ? 0 : fileSizeKb
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
import Chart from "../../core.js";
|
||||
export const name = "activeUsers";
|
||||
export const schema = {
|
||||
readWrite: {
|
||||
intersection: [
|
||||
"read",
|
||||
"write"
|
||||
],
|
||||
range: "small"
|
||||
},
|
||||
read: {
|
||||
uniqueIncrement: true,
|
||||
range: "small"
|
||||
},
|
||||
write: {
|
||||
uniqueIncrement: true,
|
||||
range: "small"
|
||||
},
|
||||
registeredWithinWeek: {
|
||||
uniqueIncrement: true,
|
||||
range: "small"
|
||||
},
|
||||
registeredWithinMonth: {
|
||||
uniqueIncrement: true,
|
||||
range: "small"
|
||||
},
|
||||
registeredWithinYear: {
|
||||
uniqueIncrement: true,
|
||||
range: "small"
|
||||
},
|
||||
registeredOutsideWeek: {
|
||||
uniqueIncrement: true,
|
||||
range: "small"
|
||||
},
|
||||
registeredOutsideMonth: {
|
||||
uniqueIncrement: true,
|
||||
range: "small"
|
||||
},
|
||||
registeredOutsideYear: {
|
||||
uniqueIncrement: true,
|
||||
range: "small"
|
||||
}
|
||||
};
|
||||
export const entity = Chart.schemaToEntity(name, schema);
|
||||
@@ -0,0 +1,8 @@
|
||||
import Chart from "../../core.js";
|
||||
export const name = "apRequest";
|
||||
export const schema = {
|
||||
deliverFailed: {},
|
||||
deliverSucceeded: {},
|
||||
inboxReceived: {}
|
||||
};
|
||||
export const entity = Chart.schemaToEntity(name, schema);
|
||||
@@ -0,0 +1,13 @@
|
||||
import Chart from "../../core.js";
|
||||
export const name = "drive";
|
||||
export const schema = {
|
||||
"local.incCount": {},
|
||||
"local.incSize": {},
|
||||
"local.decCount": {},
|
||||
"local.decSize": {},
|
||||
"remote.incCount": {},
|
||||
"remote.incSize": {},
|
||||
"remote.decCount": {},
|
||||
"remote.decSize": {}
|
||||
};
|
||||
export const entity = Chart.schemaToEntity(name, schema);
|
||||
@@ -0,0 +1,37 @@
|
||||
import Chart from "../../core.js";
|
||||
export const name = "federation";
|
||||
export const schema = {
|
||||
deliveredInstances: {
|
||||
uniqueIncrement: true,
|
||||
range: "small"
|
||||
},
|
||||
inboxInstances: {
|
||||
uniqueIncrement: true,
|
||||
range: "small"
|
||||
},
|
||||
stalled: {
|
||||
uniqueIncrement: true,
|
||||
range: "small"
|
||||
},
|
||||
sub: {
|
||||
accumulate: true,
|
||||
range: "small"
|
||||
},
|
||||
pub: {
|
||||
accumulate: true,
|
||||
range: "small"
|
||||
},
|
||||
pubsub: {
|
||||
accumulate: true,
|
||||
range: "small"
|
||||
},
|
||||
subActive: {
|
||||
accumulate: true,
|
||||
range: "small"
|
||||
},
|
||||
pubActive: {
|
||||
accumulate: true,
|
||||
range: "small"
|
||||
}
|
||||
};
|
||||
export const entity = Chart.schemaToEntity(name, schema);
|
||||
@@ -0,0 +1,11 @@
|
||||
import Chart from "../../core.js";
|
||||
export const name = "hashtag";
|
||||
export const schema = {
|
||||
"local.users": {
|
||||
uniqueIncrement: true
|
||||
},
|
||||
"remote.users": {
|
||||
uniqueIncrement: true
|
||||
}
|
||||
};
|
||||
export const entity = Chart.schemaToEntity(name, schema, true);
|
||||
@@ -0,0 +1,57 @@
|
||||
import Chart from "../../core.js";
|
||||
export const name = "instance";
|
||||
export const schema = {
|
||||
"requests.failed": {
|
||||
range: "small"
|
||||
},
|
||||
"requests.succeeded": {
|
||||
range: "small"
|
||||
},
|
||||
"requests.received": {
|
||||
range: "small"
|
||||
},
|
||||
"notes.total": {
|
||||
accumulate: true
|
||||
},
|
||||
"notes.inc": {},
|
||||
"notes.dec": {},
|
||||
"notes.diffs.normal": {},
|
||||
"notes.diffs.reply": {},
|
||||
"notes.diffs.renote": {},
|
||||
"notes.diffs.withFile": {},
|
||||
"users.total": {
|
||||
accumulate: true
|
||||
},
|
||||
"users.inc": {
|
||||
range: "small"
|
||||
},
|
||||
"users.dec": {
|
||||
range: "small"
|
||||
},
|
||||
"following.total": {
|
||||
accumulate: true
|
||||
},
|
||||
"following.inc": {
|
||||
range: "small"
|
||||
},
|
||||
"following.dec": {
|
||||
range: "small"
|
||||
},
|
||||
"followers.total": {
|
||||
accumulate: true
|
||||
},
|
||||
"followers.inc": {
|
||||
range: "small"
|
||||
},
|
||||
"followers.dec": {
|
||||
range: "small"
|
||||
},
|
||||
"drive.totalFiles": {
|
||||
accumulate: true
|
||||
},
|
||||
"drive.incFiles": {},
|
||||
"drive.decFiles": {},
|
||||
"drive.incUsage": {},
|
||||
"drive.decUsage": {}
|
||||
};
|
||||
export const entity = Chart.schemaToEntity(name, schema, true);
|
||||
@@ -0,0 +1,23 @@
|
||||
import Chart from "../../core.js";
|
||||
export const name = "notes";
|
||||
export const schema = {
|
||||
"local.total": {
|
||||
accumulate: true
|
||||
},
|
||||
"local.inc": {},
|
||||
"local.dec": {},
|
||||
"local.diffs.normal": {},
|
||||
"local.diffs.reply": {},
|
||||
"local.diffs.renote": {},
|
||||
"local.diffs.withFile": {},
|
||||
"remote.total": {
|
||||
accumulate: true
|
||||
},
|
||||
"remote.inc": {},
|
||||
"remote.dec": {},
|
||||
"remote.diffs.normal": {},
|
||||
"remote.diffs.reply": {},
|
||||
"remote.diffs.renote": {},
|
||||
"remote.diffs.withFile": {}
|
||||
};
|
||||
export const entity = Chart.schemaToEntity(name, schema);
|
||||
@@ -0,0 +1,19 @@
|
||||
import Chart from "../../core.js";
|
||||
export const name = "perUserDrive";
|
||||
export const schema = {
|
||||
totalCount: {
|
||||
accumulate: true
|
||||
},
|
||||
totalSize: {
|
||||
accumulate: true
|
||||
},
|
||||
incCount: {
|
||||
range: "small"
|
||||
},
|
||||
incSize: {},
|
||||
decCount: {
|
||||
range: "small"
|
||||
},
|
||||
decSize: {}
|
||||
};
|
||||
export const entity = Chart.schemaToEntity(name, schema, true);
|
||||
@@ -0,0 +1,41 @@
|
||||
import Chart from "../../core.js";
|
||||
export const name = "perUserFollowing";
|
||||
export const schema = {
|
||||
"local.followings.total": {
|
||||
accumulate: true
|
||||
},
|
||||
"local.followings.inc": {
|
||||
range: "small"
|
||||
},
|
||||
"local.followings.dec": {
|
||||
range: "small"
|
||||
},
|
||||
"local.followers.total": {
|
||||
accumulate: true
|
||||
},
|
||||
"local.followers.inc": {
|
||||
range: "small"
|
||||
},
|
||||
"local.followers.dec": {
|
||||
range: "small"
|
||||
},
|
||||
"remote.followings.total": {
|
||||
accumulate: true
|
||||
},
|
||||
"remote.followings.inc": {
|
||||
range: "small"
|
||||
},
|
||||
"remote.followings.dec": {
|
||||
range: "small"
|
||||
},
|
||||
"remote.followers.total": {
|
||||
accumulate: true
|
||||
},
|
||||
"remote.followers.inc": {
|
||||
range: "small"
|
||||
},
|
||||
"remote.followers.dec": {
|
||||
range: "small"
|
||||
}
|
||||
};
|
||||
export const entity = Chart.schemaToEntity(name, schema, true);
|
||||
@@ -0,0 +1,26 @@
|
||||
import Chart from "../../core.js";
|
||||
export const name = "perUserNotes";
|
||||
export const schema = {
|
||||
total: {
|
||||
accumulate: true
|
||||
},
|
||||
inc: {
|
||||
range: "small"
|
||||
},
|
||||
dec: {
|
||||
range: "small"
|
||||
},
|
||||
"diffs.normal": {
|
||||
range: "small"
|
||||
},
|
||||
"diffs.reply": {
|
||||
range: "small"
|
||||
},
|
||||
"diffs.renote": {
|
||||
range: "small"
|
||||
},
|
||||
"diffs.withFile": {
|
||||
range: "small"
|
||||
}
|
||||
};
|
||||
export const entity = Chart.schemaToEntity(name, schema, true);
|
||||
@@ -0,0 +1,11 @@
|
||||
import Chart from "../../core.js";
|
||||
export const name = "perUserReaction";
|
||||
export const schema = {
|
||||
"local.count": {
|
||||
range: "small"
|
||||
},
|
||||
"remote.count": {
|
||||
range: "small"
|
||||
}
|
||||
};
|
||||
export const entity = Chart.schemaToEntity(name, schema, true);
|
||||
@@ -0,0 +1,10 @@
|
||||
import Chart from "../../core.js";
|
||||
export const name = "testGrouped";
|
||||
export const schema = {
|
||||
"foo.total": {
|
||||
accumulate: true
|
||||
},
|
||||
"foo.inc": {},
|
||||
"foo.dec": {}
|
||||
};
|
||||
export const entity = Chart.schemaToEntity(name, schema, true);
|
||||
@@ -0,0 +1,17 @@
|
||||
import Chart from "../../core.js";
|
||||
export const name = "testIntersection";
|
||||
export const schema = {
|
||||
a: {
|
||||
uniqueIncrement: true
|
||||
},
|
||||
b: {
|
||||
uniqueIncrement: true
|
||||
},
|
||||
aAndB: {
|
||||
intersection: [
|
||||
"a",
|
||||
"b"
|
||||
]
|
||||
}
|
||||
};
|
||||
export const entity = Chart.schemaToEntity(name, schema);
|
||||
@@ -0,0 +1,8 @@
|
||||
import Chart from "../../core.js";
|
||||
export const name = "testUnique";
|
||||
export const schema = {
|
||||
foo: {
|
||||
uniqueIncrement: true
|
||||
}
|
||||
};
|
||||
export const entity = Chart.schemaToEntity(name, schema);
|
||||
@@ -0,0 +1,10 @@
|
||||
import Chart from "../../core.js";
|
||||
export const name = "test";
|
||||
export const schema = {
|
||||
"foo.total": {
|
||||
accumulate: true
|
||||
},
|
||||
"foo.inc": {},
|
||||
"foo.dec": {}
|
||||
};
|
||||
export const entity = Chart.schemaToEntity(name, schema);
|
||||
@@ -0,0 +1,23 @@
|
||||
import Chart from "../../core.js";
|
||||
export const name = "users";
|
||||
export const schema = {
|
||||
"local.total": {
|
||||
accumulate: true
|
||||
},
|
||||
"local.inc": {
|
||||
range: "small"
|
||||
},
|
||||
"local.dec": {
|
||||
range: "small"
|
||||
},
|
||||
"remote.total": {
|
||||
accumulate: true
|
||||
},
|
||||
"remote.inc": {
|
||||
range: "small"
|
||||
},
|
||||
"remote.dec": {
|
||||
range: "small"
|
||||
}
|
||||
};
|
||||
export const entity = Chart.schemaToEntity(name, schema);
|
||||
@@ -0,0 +1,67 @@
|
||||
import Chart from "../core.js";
|
||||
import { Followings, Instances } from "../../../models/index.js";
|
||||
import { name, schema } from "./entities/federation.js";
|
||||
import { fetchMeta } from "../../../misc/fetch-meta.js";
|
||||
/**
|
||||
* フェデレーションに関するチャート
|
||||
*/ export default class FederationChart extends Chart {
|
||||
constructor(){
|
||||
super(name, schema);
|
||||
}
|
||||
async tickMajor() {
|
||||
return {};
|
||||
}
|
||||
async tickMinor() {
|
||||
const meta = await fetchMeta();
|
||||
const suspendedInstancesQuery = Instances.createQueryBuilder("instance").select("instance.host").where("instance.isSuspended = true");
|
||||
const pubsubSubQuery = Followings.createQueryBuilder("f").select("f.followerHost").where("f.followerHost IS NOT NULL");
|
||||
const subInstancesQuery = Followings.createQueryBuilder("f").select("f.followeeHost").where("f.followeeHost IS NOT NULL");
|
||||
const pubInstancesQuery = Followings.createQueryBuilder("f").select("f.followerHost").where("f.followerHost IS NOT NULL");
|
||||
const [sub, pub, pubsub, subActive, pubActive] = await Promise.all([
|
||||
Followings.createQueryBuilder("following").select("COUNT(DISTINCT following.followeeHost)").where("following.followeeHost IS NOT NULL").andWhere(meta.blockedHosts.length === 0 ? "1=1" : "following.followeeHost NOT IN (:...blocked)", {
|
||||
blocked: meta.blockedHosts
|
||||
}).andWhere(`following.followeeHost NOT IN (${suspendedInstancesQuery.getQuery()})`).getRawOne().then((x)=>parseInt(x.count, 10)),
|
||||
Followings.createQueryBuilder("following").select("COUNT(DISTINCT following.followerHost)").where("following.followerHost IS NOT NULL").andWhere(meta.blockedHosts.length === 0 ? "1=1" : "following.followerHost NOT IN (:...blocked)", {
|
||||
blocked: meta.blockedHosts
|
||||
}).andWhere(`following.followerHost NOT IN (${suspendedInstancesQuery.getQuery()})`).getRawOne().then((x)=>parseInt(x.count, 10)),
|
||||
Followings.createQueryBuilder("following").select("COUNT(DISTINCT following.followeeHost)").where("following.followeeHost IS NOT NULL").andWhere(meta.blockedHosts.length === 0 ? "1=1" : "following.followeeHost NOT IN (:...blocked)", {
|
||||
blocked: meta.blockedHosts
|
||||
}).andWhere(`following.followeeHost NOT IN (${suspendedInstancesQuery.getQuery()})`).andWhere(`following.followeeHost IN (${pubsubSubQuery.getQuery()})`).setParameters(pubsubSubQuery.getParameters()).getRawOne().then((x)=>parseInt(x.count, 10)),
|
||||
Instances.createQueryBuilder("instance").select("COUNT(instance.id)").where(`instance.host IN (${subInstancesQuery.getQuery()})`).andWhere(meta.blockedHosts.length === 0 ? "1=1" : "instance.host NOT IN (:...blocked)", {
|
||||
blocked: meta.blockedHosts
|
||||
}).andWhere("instance.isSuspended = false").andWhere("instance.lastCommunicatedAt > :gt", {
|
||||
gt: new Date(Date.now() - 1000 * 60 * 60 * 24 * 30)
|
||||
}).getRawOne().then((x)=>parseInt(x.count, 10)),
|
||||
Instances.createQueryBuilder("instance").select("COUNT(instance.id)").where(`instance.host IN (${pubInstancesQuery.getQuery()})`).andWhere(meta.blockedHosts.length === 0 ? "1=1" : "instance.host NOT IN (:...blocked)", {
|
||||
blocked: meta.blockedHosts
|
||||
}).andWhere("instance.isSuspended = false").andWhere("instance.lastCommunicatedAt > :gt", {
|
||||
gt: new Date(Date.now() - 1000 * 60 * 60 * 24 * 30)
|
||||
}).getRawOne().then((x)=>parseInt(x.count, 10))
|
||||
]);
|
||||
return {
|
||||
sub: sub,
|
||||
pub: pub,
|
||||
pubsub: pubsub,
|
||||
subActive: subActive,
|
||||
pubActive: pubActive
|
||||
};
|
||||
}
|
||||
async deliverd(host, succeeded) {
|
||||
await this.commit(succeeded ? {
|
||||
deliveredInstances: [
|
||||
host
|
||||
]
|
||||
} : {
|
||||
stalled: [
|
||||
host
|
||||
]
|
||||
});
|
||||
}
|
||||
async inbox(host) {
|
||||
await this.commit({
|
||||
inboxInstances: [
|
||||
host
|
||||
]
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
import Chart from "../core.js";
|
||||
import { Users } from "../../../models/index.js";
|
||||
import { name, schema } from "./entities/hashtag.js";
|
||||
/**
|
||||
* ハッシュタグに関するチャート
|
||||
*/ export default class HashtagChart extends Chart {
|
||||
constructor(){
|
||||
super(name, schema, true);
|
||||
}
|
||||
async tickMajor() {
|
||||
return {};
|
||||
}
|
||||
async tickMinor() {
|
||||
return {};
|
||||
}
|
||||
async update(hashtag, user) {
|
||||
await this.commit({
|
||||
"local.users": Users.isLocalUser(user) ? [
|
||||
user.id
|
||||
] : [],
|
||||
"remote.users": Users.isLocalUser(user) ? [] : [
|
||||
user.id
|
||||
]
|
||||
}, hashtag);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
import Chart from "../core.js";
|
||||
import { DriveFiles, Followings, Users, Notes } from "../../../models/index.js";
|
||||
import { toPuny } from "../../../misc/convert-host.js";
|
||||
import { name, schema } from "./entities/instance.js";
|
||||
/**
|
||||
* インスタンスごとのチャート
|
||||
*/ export default class InstanceChart extends Chart {
|
||||
constructor(){
|
||||
super(name, schema, true);
|
||||
}
|
||||
async tickMajor(group) {
|
||||
const [notesCount, usersCount, followingCount, followersCount, driveFiles] = await Promise.all([
|
||||
Notes.countBy({
|
||||
userHost: group
|
||||
}),
|
||||
Users.countBy({
|
||||
host: group
|
||||
}),
|
||||
Followings.countBy({
|
||||
followerHost: group
|
||||
}),
|
||||
Followings.countBy({
|
||||
followeeHost: group
|
||||
}),
|
||||
DriveFiles.countBy({
|
||||
userHost: group
|
||||
})
|
||||
]);
|
||||
return {
|
||||
"notes.total": notesCount,
|
||||
"users.total": usersCount,
|
||||
"following.total": followingCount,
|
||||
"followers.total": followersCount,
|
||||
"drive.totalFiles": driveFiles
|
||||
};
|
||||
}
|
||||
async tickMinor() {
|
||||
return {};
|
||||
}
|
||||
async requestReceived(host) {
|
||||
await this.commit({
|
||||
"requests.received": 1
|
||||
}, toPuny(host));
|
||||
}
|
||||
async requestSent(host, isSucceeded) {
|
||||
await this.commit({
|
||||
"requests.succeeded": isSucceeded ? 1 : 0,
|
||||
"requests.failed": isSucceeded ? 0 : 1
|
||||
}, toPuny(host));
|
||||
}
|
||||
async newUser(host) {
|
||||
await this.commit({
|
||||
"users.total": 1,
|
||||
"users.inc": 1
|
||||
}, toPuny(host));
|
||||
}
|
||||
async updateNote(host, note, isAdditional) {
|
||||
await this.commit({
|
||||
"notes.total": isAdditional ? 1 : -1,
|
||||
"notes.inc": isAdditional ? 1 : 0,
|
||||
"notes.dec": isAdditional ? 0 : 1,
|
||||
"notes.diffs.normal": note.replyId == null && note.renoteId == null ? isAdditional ? 1 : -1 : 0,
|
||||
"notes.diffs.renote": note.renoteId != null ? isAdditional ? 1 : -1 : 0,
|
||||
"notes.diffs.reply": note.replyId != null ? isAdditional ? 1 : -1 : 0,
|
||||
"notes.diffs.withFile": note.fileIds.length > 0 ? isAdditional ? 1 : -1 : 0
|
||||
}, toPuny(host));
|
||||
}
|
||||
async updateFollowing(host, isAdditional) {
|
||||
await this.commit({
|
||||
"following.total": isAdditional ? 1 : -1,
|
||||
"following.inc": isAdditional ? 1 : 0,
|
||||
"following.dec": isAdditional ? 0 : 1
|
||||
}, toPuny(host));
|
||||
}
|
||||
async updateFollowers(host, isAdditional) {
|
||||
await this.commit({
|
||||
"followers.total": isAdditional ? 1 : -1,
|
||||
"followers.inc": isAdditional ? 1 : 0,
|
||||
"followers.dec": isAdditional ? 0 : 1
|
||||
}, toPuny(host));
|
||||
}
|
||||
async updateDrive(file, isAdditional) {
|
||||
const fileSizeKb = file.size / 1000;
|
||||
await this.commit({
|
||||
"drive.totalFiles": isAdditional ? 1 : -1,
|
||||
"drive.incFiles": isAdditional ? 1 : 0,
|
||||
"drive.incUsage": isAdditional ? fileSizeKb : 0,
|
||||
"drive.decFiles": isAdditional ? 1 : 0,
|
||||
"drive.decUsage": isAdditional ? fileSizeKb : 0
|
||||
}, file.userHost);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
import Chart from "../core.js";
|
||||
import { Notes } from "../../../models/index.js";
|
||||
import { Not, IsNull } from "typeorm";
|
||||
import { name, schema } from "./entities/notes.js";
|
||||
/**
|
||||
* ノートに関するチャート
|
||||
*/ export default class NotesChart extends Chart {
|
||||
constructor(){
|
||||
super(name, schema);
|
||||
}
|
||||
async tickMajor() {
|
||||
const [localCount, remoteCount] = await Promise.all([
|
||||
Notes.countBy({
|
||||
userHost: IsNull()
|
||||
}),
|
||||
Notes.countBy({
|
||||
userHost: Not(IsNull())
|
||||
})
|
||||
]);
|
||||
return {
|
||||
"local.total": localCount,
|
||||
"remote.total": remoteCount
|
||||
};
|
||||
}
|
||||
async tickMinor() {
|
||||
return {};
|
||||
}
|
||||
async update(note, isAdditional) {
|
||||
const prefix = note.userHost === null ? "local" : "remote";
|
||||
await this.commit({
|
||||
[`${prefix}.total`]: isAdditional ? 1 : -1,
|
||||
[`${prefix}.inc`]: isAdditional ? 1 : 0,
|
||||
[`${prefix}.dec`]: isAdditional ? 0 : 1,
|
||||
[`${prefix}.diffs.normal`]: note.replyId == null && note.renoteId == null ? isAdditional ? 1 : -1 : 0,
|
||||
[`${prefix}.diffs.renote`]: note.renoteId != null ? isAdditional ? 1 : -1 : 0,
|
||||
[`${prefix}.diffs.reply`]: note.replyId != null ? isAdditional ? 1 : -1 : 0,
|
||||
[`${prefix}.diffs.withFile`]: note.fileIds.length > 0 ? isAdditional ? 1 : -1 : 0
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
import Chart from "../core.js";
|
||||
import { DriveFiles } from "../../../models/index.js";
|
||||
import { name, schema } from "./entities/per-user-drive.js";
|
||||
/**
|
||||
* ユーザーごとのドライブに関するチャート
|
||||
*/ export default class PerUserDriveChart extends Chart {
|
||||
constructor(){
|
||||
super(name, schema, true);
|
||||
}
|
||||
async tickMajor(group) {
|
||||
const [count, size] = await Promise.all([
|
||||
DriveFiles.countBy({
|
||||
userId: group
|
||||
}),
|
||||
DriveFiles.calcDriveUsageOf(group)
|
||||
]);
|
||||
return {
|
||||
totalCount: count,
|
||||
totalSize: size
|
||||
};
|
||||
}
|
||||
async tickMinor() {
|
||||
return {};
|
||||
}
|
||||
async update(file, isAdditional) {
|
||||
const fileSizeKb = file.size / 1000;
|
||||
await this.commit({
|
||||
totalCount: isAdditional ? 1 : -1,
|
||||
totalSize: isAdditional ? fileSizeKb : -fileSizeKb,
|
||||
incCount: isAdditional ? 1 : 0,
|
||||
incSize: isAdditional ? fileSizeKb : 0,
|
||||
decCount: isAdditional ? 0 : 1,
|
||||
decSize: isAdditional ? 0 : fileSizeKb
|
||||
}, file.userId);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
import Chart from "../core.js";
|
||||
import { Followings, Users } from "../../../models/index.js";
|
||||
import { Not, IsNull } from "typeorm";
|
||||
import { name, schema } from "./entities/per-user-following.js";
|
||||
/**
|
||||
* ユーザーごとのフォローに関するチャート
|
||||
*/ export default class PerUserFollowingChart extends Chart {
|
||||
constructor(){
|
||||
super(name, schema, true);
|
||||
}
|
||||
async tickMajor(group) {
|
||||
const [localFollowingsCount, localFollowersCount, remoteFollowingsCount, remoteFollowersCount] = await Promise.all([
|
||||
Followings.countBy({
|
||||
followerId: group,
|
||||
followeeHost: IsNull()
|
||||
}),
|
||||
Followings.countBy({
|
||||
followeeId: group,
|
||||
followerHost: IsNull()
|
||||
}),
|
||||
Followings.countBy({
|
||||
followerId: group,
|
||||
followeeHost: Not(IsNull())
|
||||
}),
|
||||
Followings.countBy({
|
||||
followeeId: group,
|
||||
followerHost: Not(IsNull())
|
||||
})
|
||||
]);
|
||||
return {
|
||||
"local.followings.total": localFollowingsCount,
|
||||
"local.followers.total": localFollowersCount,
|
||||
"remote.followings.total": remoteFollowingsCount,
|
||||
"remote.followers.total": remoteFollowersCount
|
||||
};
|
||||
}
|
||||
async tickMinor() {
|
||||
return {};
|
||||
}
|
||||
async update(follower, followee, isFollow) {
|
||||
const prefixFollower = Users.isLocalUser(follower) ? "local" : "remote";
|
||||
const prefixFollowee = Users.isLocalUser(followee) ? "local" : "remote";
|
||||
this.commit({
|
||||
[`${prefixFollower}.followings.total`]: isFollow ? 1 : -1,
|
||||
[`${prefixFollower}.followings.inc`]: isFollow ? 1 : 0,
|
||||
[`${prefixFollower}.followings.dec`]: isFollow ? 0 : 1
|
||||
}, follower.id);
|
||||
this.commit({
|
||||
[`${prefixFollowee}.followers.total`]: isFollow ? 1 : -1,
|
||||
[`${prefixFollowee}.followers.inc`]: isFollow ? 1 : 0,
|
||||
[`${prefixFollowee}.followers.dec`]: isFollow ? 0 : 1
|
||||
}, followee.id);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
import Chart from "../core.js";
|
||||
import { Notes } from "../../../models/index.js";
|
||||
import { name, schema } from "./entities/per-user-notes.js";
|
||||
/**
|
||||
* ユーザーごとのノートに関するチャート
|
||||
*/ export default class PerUserNotesChart extends Chart {
|
||||
constructor(){
|
||||
super(name, schema, true);
|
||||
}
|
||||
async tickMajor(group) {
|
||||
const [count] = await Promise.all([
|
||||
Notes.countBy({
|
||||
userId: group
|
||||
})
|
||||
]);
|
||||
return {
|
||||
total: count
|
||||
};
|
||||
}
|
||||
async tickMinor() {
|
||||
return {};
|
||||
}
|
||||
async update(user, note, isAdditional) {
|
||||
await this.commit({
|
||||
total: isAdditional ? 1 : -1,
|
||||
inc: isAdditional ? 1 : 0,
|
||||
dec: isAdditional ? 0 : 1,
|
||||
"diffs.normal": note.replyId == null && note.renoteId == null ? isAdditional ? 1 : -1 : 0,
|
||||
"diffs.renote": note.renoteId != null ? isAdditional ? 1 : -1 : 0,
|
||||
"diffs.reply": note.replyId != null ? isAdditional ? 1 : -1 : 0,
|
||||
"diffs.withFile": note.fileIds.length > 0 ? isAdditional ? 1 : -1 : 0
|
||||
}, user.id);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
import Chart from "../core.js";
|
||||
import { Users } from "../../../models/index.js";
|
||||
import { name, schema } from "./entities/per-user-reactions.js";
|
||||
/**
|
||||
* ユーザーごとのリアクションに関するチャート
|
||||
*/ export default class PerUserReactionsChart extends Chart {
|
||||
constructor(){
|
||||
super(name, schema, true);
|
||||
}
|
||||
async tickMajor(group) {
|
||||
return {};
|
||||
}
|
||||
async tickMinor() {
|
||||
return {};
|
||||
}
|
||||
async update(user, note) {
|
||||
const prefix = Users.isLocalUser(user) ? "local" : "remote";
|
||||
this.commit({
|
||||
[`${prefix}.count`]: 1
|
||||
}, note.userId);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
import Chart from "../core.js";
|
||||
import { name, schema } from "./entities/test-grouped.js";
|
||||
/**
|
||||
* For testing
|
||||
*/ export default class TestGroupedChart extends Chart {
|
||||
total = {};
|
||||
constructor(){
|
||||
super(name, schema, true);
|
||||
}
|
||||
async tickMajor(group) {
|
||||
return {
|
||||
"foo.total": this.total[group]
|
||||
};
|
||||
}
|
||||
async tickMinor() {
|
||||
return {};
|
||||
}
|
||||
async increment(group) {
|
||||
if (this.total[group] == null) this.total[group] = 0;
|
||||
this.total[group]++;
|
||||
await this.commit({
|
||||
"foo.total": 1,
|
||||
"foo.inc": 1
|
||||
}, group);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
import Chart from "../core.js";
|
||||
import { name, schema } from "./entities/test-intersection.js";
|
||||
/**
|
||||
* For testing
|
||||
*/ export default class TestIntersectionChart extends Chart {
|
||||
constructor(){
|
||||
super(name, schema);
|
||||
}
|
||||
async tickMajor() {
|
||||
return {};
|
||||
}
|
||||
async tickMinor() {
|
||||
return {};
|
||||
}
|
||||
async addA(key) {
|
||||
await this.commit({
|
||||
a: [
|
||||
key
|
||||
]
|
||||
});
|
||||
}
|
||||
async addB(key) {
|
||||
await this.commit({
|
||||
b: [
|
||||
key
|
||||
]
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
import Chart from "../core.js";
|
||||
import { name, schema } from "./entities/test-unique.js";
|
||||
/**
|
||||
* For testing
|
||||
*/ export default class TestUniqueChart extends Chart {
|
||||
constructor(){
|
||||
super(name, schema);
|
||||
}
|
||||
async tickMajor() {
|
||||
return {};
|
||||
}
|
||||
async tickMinor() {
|
||||
return {};
|
||||
}
|
||||
async uniqueIncrement(key) {
|
||||
await this.commit({
|
||||
foo: [
|
||||
key
|
||||
]
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
import Chart from "../core.js";
|
||||
import { name, schema } from "./entities/test.js";
|
||||
/**
|
||||
* For testing
|
||||
*/ export default class TestChart extends Chart {
|
||||
total = 0;
|
||||
constructor(){
|
||||
super(name, schema);
|
||||
}
|
||||
async tickMajor() {
|
||||
return {
|
||||
"foo.total": this.total
|
||||
};
|
||||
}
|
||||
async tickMinor() {
|
||||
return {};
|
||||
}
|
||||
async increment() {
|
||||
this.total++;
|
||||
await this.commit({
|
||||
"foo.total": 1,
|
||||
"foo.inc": 1
|
||||
});
|
||||
}
|
||||
async decrement() {
|
||||
this.total--;
|
||||
await this.commit({
|
||||
"foo.total": -1,
|
||||
"foo.dec": 1
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
import Chart from "../core.js";
|
||||
import { Users } from "../../../models/index.js";
|
||||
import { Not, IsNull } from "typeorm";
|
||||
import { name, schema } from "./entities/users.js";
|
||||
/**
|
||||
* ユーザー数に関するチャート
|
||||
*/ export default class UsersChart extends Chart {
|
||||
constructor(){
|
||||
super(name, schema);
|
||||
}
|
||||
async tickMajor() {
|
||||
const [localCount, remoteCount] = await Promise.all([
|
||||
Users.countBy({
|
||||
host: IsNull()
|
||||
}),
|
||||
Users.countBy({
|
||||
host: Not(IsNull())
|
||||
})
|
||||
]);
|
||||
return {
|
||||
"local.total": localCount,
|
||||
"remote.total": remoteCount
|
||||
};
|
||||
}
|
||||
async tickMinor() {
|
||||
return {};
|
||||
}
|
||||
async update(user, isAdditional) {
|
||||
const prefix = Users.isLocalUser(user) ? "local" : "remote";
|
||||
await this.commit({
|
||||
[`${prefix}.total`]: isAdditional ? 1 : -1,
|
||||
[`${prefix}.inc`]: isAdditional ? 1 : 0,
|
||||
[`${prefix}.dec`]: isAdditional ? 0 : 1
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,581 @@
|
||||
/**
|
||||
* チャートエンジン
|
||||
*
|
||||
* Tests located in test/chart
|
||||
*/ import * as nestedProperty from "nested-property";
|
||||
import Logger from "../logger.js";
|
||||
import { EntitySchema, LessThan, Between } from "typeorm";
|
||||
import { dateUTC, isTimeSame, isTimeBefore, subtractTime, addTime } from "../../prelude/time.js";
|
||||
import { db } from "../../db/postgre.js";
|
||||
import promiseLimit from "promise-limit";
|
||||
const logger = new Logger("chart", "white", process.env.NODE_ENV !== "test");
|
||||
const columnPrefix = "___";
|
||||
const uniqueTempColumnPrefix = "unique_temp___";
|
||||
const columnDot = "_";
|
||||
const camelToSnake = (str)=>{
|
||||
return str.replace(/([A-Z])/g, (s)=>`_${s.charAt(0).toLowerCase()}`);
|
||||
};
|
||||
const removeDuplicates = (array)=>Array.from(new Set(array));
|
||||
export function getJsonSchema(schema) {
|
||||
const jsonSchema = {
|
||||
type: "object",
|
||||
properties: {},
|
||||
required: []
|
||||
};
|
||||
for(const k in schema){
|
||||
jsonSchema.properties[k] = {
|
||||
type: "array",
|
||||
items: {
|
||||
type: "number"
|
||||
}
|
||||
};
|
||||
}
|
||||
return jsonSchema;
|
||||
}
|
||||
/**
|
||||
* 様々なチャートの管理を司るクラス
|
||||
*/ export default class Chart {
|
||||
schema;
|
||||
name;
|
||||
buffer = [];
|
||||
// ↓にしたいけどfindOneとかで型エラーになる
|
||||
//private repositoryForHour: Repository<RawRecord<T>>;
|
||||
//private repositoryForDay: Repository<RawRecord<T>>;
|
||||
repositoryForHour;
|
||||
repositoryForDay;
|
||||
static convertSchemaToColumnDefinitions(schema) {
|
||||
const columns = {};
|
||||
for (const [k, v] of Object.entries(schema)){
|
||||
const name = k.replaceAll(".", columnDot);
|
||||
const type = v.range === "big" ? "bigint" : v.range === "small" ? "smallint" : "integer";
|
||||
if (v.uniqueIncrement) {
|
||||
columns[uniqueTempColumnPrefix + name] = {
|
||||
type: "varchar",
|
||||
array: true,
|
||||
default: "{}"
|
||||
};
|
||||
columns[columnPrefix + name] = {
|
||||
type,
|
||||
default: 0
|
||||
};
|
||||
} else {
|
||||
columns[columnPrefix + name] = {
|
||||
type,
|
||||
default: 0
|
||||
};
|
||||
}
|
||||
}
|
||||
return columns;
|
||||
}
|
||||
static dateToTimestamp(x) {
|
||||
return Math.floor(x.getTime() / 1000);
|
||||
}
|
||||
static parseDate(date) {
|
||||
const y = date.getUTCFullYear();
|
||||
const m = date.getUTCMonth();
|
||||
const d = date.getUTCDate();
|
||||
const h = date.getUTCHours();
|
||||
const _m = date.getUTCMinutes();
|
||||
const _s = date.getUTCSeconds();
|
||||
const _ms = date.getUTCMilliseconds();
|
||||
return [
|
||||
y,
|
||||
m,
|
||||
d,
|
||||
h,
|
||||
_m,
|
||||
_s,
|
||||
_ms
|
||||
];
|
||||
}
|
||||
static getCurrentDate() {
|
||||
return Chart.parseDate(new Date());
|
||||
}
|
||||
static schemaToEntity(name, schema, grouped = false) {
|
||||
const createEntity = (span)=>new EntitySchema({
|
||||
name: span === "hour" ? `__chart__${camelToSnake(name)}` : span === "day" ? `__chart_day__${camelToSnake(name)}` : new Error("not happen"),
|
||||
columns: {
|
||||
id: {
|
||||
type: "integer",
|
||||
primary: true,
|
||||
generated: true
|
||||
},
|
||||
date: {
|
||||
type: "integer"
|
||||
},
|
||||
...grouped ? {
|
||||
group: {
|
||||
type: "varchar",
|
||||
length: 128
|
||||
}
|
||||
} : {},
|
||||
...Chart.convertSchemaToColumnDefinitions(schema)
|
||||
},
|
||||
indices: [
|
||||
{
|
||||
columns: grouped ? [
|
||||
"date",
|
||||
"group"
|
||||
] : [
|
||||
"date"
|
||||
],
|
||||
unique: true
|
||||
}
|
||||
],
|
||||
uniques: [
|
||||
{
|
||||
columns: grouped ? [
|
||||
"date",
|
||||
"group"
|
||||
] : [
|
||||
"date"
|
||||
]
|
||||
}
|
||||
],
|
||||
relations: {
|
||||
}
|
||||
});
|
||||
return {
|
||||
hour: createEntity("hour"),
|
||||
day: createEntity("day")
|
||||
};
|
||||
}
|
||||
constructor(name, schema, grouped = false){
|
||||
this.name = name;
|
||||
this.schema = schema;
|
||||
const { hour, day } = Chart.schemaToEntity(name, schema, grouped);
|
||||
this.repositoryForHour = db.getRepository(hour);
|
||||
this.repositoryForDay = db.getRepository(day);
|
||||
}
|
||||
convertRawRecord(x) {
|
||||
const kvs = {};
|
||||
for (const k of Object.keys(x).filter((k)=>k.startsWith(columnPrefix))){
|
||||
kvs[k.substr(columnPrefix.length).split(columnDot).join(".")] = x[k];
|
||||
}
|
||||
return kvs;
|
||||
}
|
||||
getNewLog(latest) {
|
||||
const log = {};
|
||||
for (const [k, v] of Object.entries(this.schema)){
|
||||
if (v.accumulate && latest) {
|
||||
log[k] = latest[k];
|
||||
} else {
|
||||
log[k] = 0;
|
||||
}
|
||||
}
|
||||
return log;
|
||||
}
|
||||
getLatestLog(group, span) {
|
||||
const repository = span === "hour" ? this.repositoryForHour : span === "day" ? this.repositoryForDay : new Error("not happen");
|
||||
return repository.findOne({
|
||||
where: group ? {
|
||||
group: group
|
||||
} : {},
|
||||
order: {
|
||||
date: -1
|
||||
}
|
||||
}).then((x)=>x ?? null);
|
||||
}
|
||||
/**
|
||||
* 現在(=今のHour or Day)のログをデータベースから探して、あればそれを返し、なければ作成して返します。
|
||||
*/ async claimCurrentLog(group, span) {
|
||||
const [y, m, d, h] = Chart.getCurrentDate();
|
||||
const current = dateUTC(span === "hour" ? [
|
||||
y,
|
||||
m,
|
||||
d,
|
||||
h
|
||||
] : span === "day" ? [
|
||||
y,
|
||||
m,
|
||||
d
|
||||
] : new Error("not happen"));
|
||||
const repository = span === "hour" ? this.repositoryForHour : span === "day" ? this.repositoryForDay : new Error("not happen");
|
||||
// 現在(=今のHour or Day)のログ
|
||||
const currentLog = await repository.findOneBy({
|
||||
date: Chart.dateToTimestamp(current),
|
||||
...group ? {
|
||||
group: group
|
||||
} : {}
|
||||
});
|
||||
// ログがあればそれを返して終了
|
||||
if (currentLog != null) {
|
||||
return currentLog;
|
||||
}
|
||||
let log;
|
||||
let data;
|
||||
// 集計期間が変わってから、初めてのチャート更新なら
|
||||
// 最も最近のログを持ってくる
|
||||
// * 例えば集計期間が「日」である場合で考えると、
|
||||
// * 昨日何もチャートを更新するような出来事がなかった場合は、
|
||||
// * ログがそもそも作られずドキュメントが存在しないということがあり得るため、
|
||||
// * 「昨日の」と決め打ちせずに「もっとも最近の」とします
|
||||
const latest = await this.getLatestLog(group, span);
|
||||
if (latest != null) {
|
||||
// 空ログデータを作成
|
||||
data = this.getNewLog(this.convertRawRecord(latest));
|
||||
} else {
|
||||
// ログが存在しなかったら
|
||||
// (Misskeyインスタンスを建てて初めてのチャート更新時など)
|
||||
// 初期ログデータを作成
|
||||
data = this.getNewLog(null);
|
||||
logger.info(`${this.name + (group ? `:${group}` : "")}(${span}): Initial commit created`);
|
||||
}
|
||||
const date = Chart.dateToTimestamp(current);
|
||||
const lockKey = group ? `${this.name}:${date}:${span}:${group}` : `${this.name}:${date}:${span}`;
|
||||
const { getChartInsertLock } = await import("../../misc/app-lock.js");
|
||||
const unlock = await getChartInsertLock(lockKey);
|
||||
try {
|
||||
// ロック内でもう1回チェックする
|
||||
const currentLog = await repository.findOneBy({
|
||||
date: date,
|
||||
...group ? {
|
||||
group: group
|
||||
} : {}
|
||||
});
|
||||
// ログがあればそれを返して終了
|
||||
if (currentLog != null) return currentLog;
|
||||
const columns = {};
|
||||
for (const [k, v] of Object.entries(data)){
|
||||
const name = k.replaceAll(".", columnDot);
|
||||
columns[columnPrefix + name] = v;
|
||||
}
|
||||
// 新規ログ挿入
|
||||
log = await repository.insert({
|
||||
date: date,
|
||||
...group ? {
|
||||
group: group
|
||||
} : {},
|
||||
...columns
|
||||
}).then((x)=>repository.findOneByOrFail(x.identifiers[0]));
|
||||
logger.info(`${this.name + (group ? `:${group}` : "")}(${span}): New commit created`);
|
||||
return log;
|
||||
} finally{
|
||||
unlock();
|
||||
}
|
||||
}
|
||||
commit(diff, group = null) {
|
||||
for (const [k, v] of Object.entries(diff)){
|
||||
if (v == null || v === 0 || Array.isArray(v) && v.length === 0) // rome-ignore lint/performance/noDelete: needs to be deleted not just set to undefined
|
||||
delete diff[k];
|
||||
}
|
||||
this.buffer.push({
|
||||
diff,
|
||||
group
|
||||
});
|
||||
}
|
||||
async save() {
|
||||
if (this.buffer.length === 0) {
|
||||
logger.info(`${this.name}: Write skipped`);
|
||||
return;
|
||||
}
|
||||
// TODO: 前の時間のログがbufferにあった場合のハンドリング
|
||||
// 例えば、save が20分ごとに行われるとして、前回行われたのは 01:50 だったとする。
|
||||
// 次に save が行われるのは 02:10 ということになるが、もし 01:55 に新規ログが buffer に追加されたとすると、
|
||||
// そのログは本来は 01:00~ のログとしてDBに保存されて欲しいのに、02:00~ のログ扱いになってしまう。
|
||||
// これを回避するための実装は複雑になりそうなため、一旦保留。
|
||||
const update = async (logHour, logDay)=>{
|
||||
const finalDiffs = {};
|
||||
for (const diff of this.buffer.filter((q)=>q.group == null || q.group === logHour.group).map((q)=>q.diff)){
|
||||
for (const [k, v] of Object.entries(diff)){
|
||||
if (finalDiffs[k] == null) {
|
||||
finalDiffs[k] = v;
|
||||
} else {
|
||||
if (typeof finalDiffs[k] === "number") {
|
||||
finalDiffs[k] += v;
|
||||
} else {
|
||||
finalDiffs[k] = finalDiffs[k].concat(v);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
const queryForHour = {};
|
||||
const queryForDay = {};
|
||||
for (const [k, v] of Object.entries(finalDiffs)){
|
||||
if (typeof v === "number") {
|
||||
const name = columnPrefix + k.replaceAll(".", columnDot);
|
||||
if (v > 0) queryForHour[name] = ()=>`"${name}" + ${v}`;
|
||||
if (v < 0) queryForHour[name] = ()=>`"${name}" - ${Math.abs(v)}`;
|
||||
if (v > 0) queryForDay[name] = ()=>`"${name}" + ${v}`;
|
||||
if (v < 0) queryForDay[name] = ()=>`"${name}" - ${Math.abs(v)}`;
|
||||
} else if (Array.isArray(v) && v.length > 0) {
|
||||
// ユニークインクリメント
|
||||
const tempColumnName = uniqueTempColumnPrefix + k.replaceAll(".", columnDot);
|
||||
// TODO: item をSQLエスケープ
|
||||
const itemsForHour = v.filter((item)=>!logHour[tempColumnName].includes(item)).map((item)=>`"${item}"`);
|
||||
const itemsForDay = v.filter((item)=>!logDay[tempColumnName].includes(item)).map((item)=>`"${item}"`);
|
||||
if (itemsForHour.length > 0) queryForHour[tempColumnName] = ()=>`array_cat("${tempColumnName}", '{${itemsForHour.join(",")}}'::varchar[])`;
|
||||
if (itemsForDay.length > 0) queryForDay[tempColumnName] = ()=>`array_cat("${tempColumnName}", '{${itemsForDay.join(",")}}'::varchar[])`;
|
||||
}
|
||||
}
|
||||
// bake unique count
|
||||
for (const [k, v] of Object.entries(finalDiffs)){
|
||||
if (this.schema[k].uniqueIncrement && Array.isArray(v) && v.length > 0) {
|
||||
const name = columnPrefix + k.replaceAll(".", columnDot);
|
||||
const tempColumnName = uniqueTempColumnPrefix + k.replaceAll(".", columnDot);
|
||||
queryForHour[name] = new Set([
|
||||
...v,
|
||||
...logHour[tempColumnName]
|
||||
]).size;
|
||||
queryForDay[name] = new Set([
|
||||
...v,
|
||||
...logDay[tempColumnName]
|
||||
]).size;
|
||||
}
|
||||
}
|
||||
// compute intersection
|
||||
// TODO: intersectionに指定されたカラムがintersectionだった場合の対応
|
||||
for (const [k, v] of Object.entries(this.schema)){
|
||||
const intersection = v.intersection;
|
||||
if (intersection) {
|
||||
const name = columnPrefix + k.replaceAll(".", columnDot);
|
||||
const firstKey = intersection[0];
|
||||
const firstTempColumnName = uniqueTempColumnPrefix + firstKey.replaceAll(".", columnDot);
|
||||
const firstValues = finalDiffs[firstKey];
|
||||
const currentValuesForHour = new Set([
|
||||
...firstValues ?? [],
|
||||
...logHour[firstTempColumnName]
|
||||
]);
|
||||
const currentValuesForDay = new Set([
|
||||
...firstValues ?? [],
|
||||
...logDay[firstTempColumnName]
|
||||
]);
|
||||
for(let i = 1; i < intersection.length; i++){
|
||||
const targetKey = intersection[i];
|
||||
const targetTempColumnName = uniqueTempColumnPrefix + targetKey.replaceAll(".", columnDot);
|
||||
const targetValues = finalDiffs[targetKey];
|
||||
const targetValuesForHour = new Set([
|
||||
...targetValues ?? [],
|
||||
...logHour[targetTempColumnName]
|
||||
]);
|
||||
const targetValuesForDay = new Set([
|
||||
...targetValues ?? [],
|
||||
...logDay[targetTempColumnName]
|
||||
]);
|
||||
currentValuesForHour.forEach((v)=>{
|
||||
if (!targetValuesForHour.has(v)) currentValuesForHour.delete(v);
|
||||
});
|
||||
currentValuesForDay.forEach((v)=>{
|
||||
if (!targetValuesForDay.has(v)) currentValuesForDay.delete(v);
|
||||
});
|
||||
}
|
||||
queryForHour[name] = currentValuesForHour.size;
|
||||
queryForDay[name] = currentValuesForDay.size;
|
||||
}
|
||||
}
|
||||
// ログ更新
|
||||
await Promise.all([
|
||||
this.repositoryForHour.createQueryBuilder().update().set(queryForHour).where("id = :id", {
|
||||
id: logHour.id
|
||||
}).execute(),
|
||||
this.repositoryForDay.createQueryBuilder().update().set(queryForDay).where("id = :id", {
|
||||
id: logDay.id
|
||||
}).execute()
|
||||
]);
|
||||
logger.info(`${this.name + (logHour.group ? `:${logHour.group}` : "")}: Updated`);
|
||||
// TODO: この一連の処理が始まった後に新たにbufferに入ったものは消さないようにする
|
||||
this.buffer = this.buffer.filter((q)=>q.group != null && q.group !== logHour.group);
|
||||
};
|
||||
const startCount = this.buffer.length;
|
||||
const groups = removeDuplicates(this.buffer.map((log)=>log.group));
|
||||
const groupCount = groups.length;
|
||||
// Limit the number of concurrent chart update queries executed on the database
|
||||
// to 25 at a time, so as avoid excessive IO spinlocks like when 8k queries are
|
||||
// sent out at once.
|
||||
const limit = promiseLimit(25);
|
||||
const startTime = Date.now();
|
||||
await Promise.all(groups.map((group)=>limit(()=>Promise.all([
|
||||
this.claimCurrentLog(group, "hour"),
|
||||
this.claimCurrentLog(group, "day")
|
||||
]).then(([logHour, logDay])=>update(logHour, logDay)))));
|
||||
const duration = Date.now() - startTime;
|
||||
logger.info(`Saved ${startCount} (${groupCount} unique) ${this.name} items in ${duration}ms (${this.buffer.length} remaining)`);
|
||||
}
|
||||
async tick(major, group = null) {
|
||||
const data = major ? await this.tickMajor(group) : await this.tickMinor(group);
|
||||
const columns = {};
|
||||
for (const [k, v] of Object.entries(data)){
|
||||
const name = columnPrefix + k.replaceAll(".", columnDot);
|
||||
columns[name] = v;
|
||||
}
|
||||
if (Object.keys(columns).length === 0) {
|
||||
return;
|
||||
}
|
||||
const update = async (logHour, logDay)=>{
|
||||
await Promise.all([
|
||||
this.repositoryForHour.createQueryBuilder().update().set(columns).where("id = :id", {
|
||||
id: logHour.id
|
||||
}).execute(),
|
||||
this.repositoryForDay.createQueryBuilder().update().set(columns).where("id = :id", {
|
||||
id: logDay.id
|
||||
}).execute()
|
||||
]);
|
||||
};
|
||||
return Promise.all([
|
||||
this.claimCurrentLog(group, "hour"),
|
||||
this.claimCurrentLog(group, "day")
|
||||
]).then(([logHour, logDay])=>update(logHour, logDay));
|
||||
}
|
||||
resync(group = null) {
|
||||
return this.tick(true, group);
|
||||
}
|
||||
async clean() {
|
||||
const current = dateUTC(Chart.getCurrentDate());
|
||||
// 一日以上前かつ三日以内
|
||||
const gt = Chart.dateToTimestamp(current) - 60 * 60 * 24 * 3;
|
||||
const lt = Chart.dateToTimestamp(current) - 60 * 60 * 24;
|
||||
const columns = {};
|
||||
for (const [k, v] of Object.entries(this.schema)){
|
||||
if (v.uniqueIncrement) {
|
||||
const name = uniqueTempColumnPrefix + k.replaceAll(".", columnDot);
|
||||
columns[name] = [];
|
||||
}
|
||||
}
|
||||
if (Object.keys(columns).length === 0) {
|
||||
return;
|
||||
}
|
||||
await Promise.all([
|
||||
this.repositoryForHour.createQueryBuilder().update().set(columns).where("date > :gt", {
|
||||
gt
|
||||
}).andWhere("date < :lt", {
|
||||
lt
|
||||
}).execute(),
|
||||
this.repositoryForDay.createQueryBuilder().update().set(columns).where("date > :gt", {
|
||||
gt
|
||||
}).andWhere("date < :lt", {
|
||||
lt
|
||||
}).execute()
|
||||
]);
|
||||
}
|
||||
async getChartRaw(span, amount, cursor, group = null) {
|
||||
const [y, m, d, h, _m, _s, _ms] = cursor ? Chart.parseDate(subtractTime(addTime(cursor, 1, span), 1)) : Chart.getCurrentDate();
|
||||
const [y2, m2, d2, h2] = cursor ? Chart.parseDate(addTime(cursor, 1, span)) : [];
|
||||
const lt = dateUTC([
|
||||
y,
|
||||
m,
|
||||
d,
|
||||
h,
|
||||
_m,
|
||||
_s,
|
||||
_ms
|
||||
]);
|
||||
const gt = span === "day" ? subtractTime(cursor ? dateUTC([
|
||||
y2,
|
||||
m2,
|
||||
d2,
|
||||
0
|
||||
]) : dateUTC([
|
||||
y,
|
||||
m,
|
||||
d,
|
||||
0
|
||||
]), amount - 1, "day") : span === "hour" ? subtractTime(cursor ? dateUTC([
|
||||
y2,
|
||||
m2,
|
||||
d2,
|
||||
h2
|
||||
]) : dateUTC([
|
||||
y,
|
||||
m,
|
||||
d,
|
||||
h
|
||||
]), amount - 1, "hour") : new Error("not happen");
|
||||
const repository = span === "hour" ? this.repositoryForHour : span === "day" ? this.repositoryForDay : new Error("not happen");
|
||||
// ログ取得
|
||||
let logs = await repository.find({
|
||||
where: {
|
||||
date: Between(Chart.dateToTimestamp(gt), Chart.dateToTimestamp(lt)),
|
||||
...group ? {
|
||||
group: group
|
||||
} : {}
|
||||
},
|
||||
order: {
|
||||
date: -1
|
||||
}
|
||||
});
|
||||
// 要求された範囲にログがひとつもなかったら
|
||||
if (logs.length === 0) {
|
||||
// もっとも新しいログを持ってくる
|
||||
// (すくなくともひとつログが無いと隙間埋めできないため)
|
||||
const recentLog = await repository.findOne({
|
||||
where: group ? {
|
||||
group: group
|
||||
} : {},
|
||||
order: {
|
||||
date: -1
|
||||
}
|
||||
});
|
||||
if (recentLog) {
|
||||
logs = [
|
||||
recentLog
|
||||
];
|
||||
}
|
||||
// 要求された範囲の最も古い箇所に位置するログが存在しなかったら
|
||||
} else if (!isTimeSame(new Date(logs[logs.length - 1].date * 1000), gt)) {
|
||||
// 要求された範囲の最も古い箇所時点での最も新しいログを持ってきて末尾に追加する
|
||||
// (隙間埋めできないため)
|
||||
const outdatedLog = await repository.findOne({
|
||||
where: {
|
||||
date: LessThan(Chart.dateToTimestamp(gt)),
|
||||
...group ? {
|
||||
group: group
|
||||
} : {}
|
||||
},
|
||||
order: {
|
||||
date: -1
|
||||
}
|
||||
});
|
||||
if (outdatedLog) {
|
||||
logs.push(outdatedLog);
|
||||
}
|
||||
}
|
||||
const chart = [];
|
||||
for(let i = amount - 1; i >= 0; i--){
|
||||
const current = span === "hour" ? subtractTime(dateUTC([
|
||||
y,
|
||||
m,
|
||||
d,
|
||||
h
|
||||
]), i, "hour") : span === "day" ? subtractTime(dateUTC([
|
||||
y,
|
||||
m,
|
||||
d
|
||||
]), i, "day") : new Error("not happen");
|
||||
const log = logs.find((l)=>isTimeSame(new Date(l.date * 1000), current));
|
||||
if (log) {
|
||||
chart.unshift(this.convertRawRecord(log));
|
||||
} else {
|
||||
// 隙間埋め
|
||||
const latest = logs.find((l)=>isTimeBefore(new Date(l.date * 1000), current));
|
||||
const data = latest ? this.convertRawRecord(latest) : null;
|
||||
chart.unshift(this.getNewLog(data));
|
||||
}
|
||||
}
|
||||
const res = {};
|
||||
/**
|
||||
* [{ foo: 1, bar: 5 }, { foo: 2, bar: 6 }, { foo: 3, bar: 7 }]
|
||||
* を
|
||||
* { foo: [1, 2, 3], bar: [5, 6, 7] }
|
||||
* にする
|
||||
*/ for (const record of chart){
|
||||
for (const [k, v] of Object.entries(record)){
|
||||
if (res[k]) {
|
||||
res[k].push(v);
|
||||
} else {
|
||||
res[k] = [
|
||||
v
|
||||
];
|
||||
}
|
||||
}
|
||||
}
|
||||
return res;
|
||||
}
|
||||
async getChart(span, amount, cursor, group = null) {
|
||||
const result = await this.getChartRaw(span, amount, cursor, group);
|
||||
const object = {};
|
||||
for (const [k, v] of Object.entries(result)){
|
||||
nestedProperty.set(object, k, v);
|
||||
}
|
||||
return object;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
import { entity as FederationChart } from "./charts/entities/federation.js";
|
||||
import { entity as NotesChart } from "./charts/entities/notes.js";
|
||||
import { entity as UsersChart } from "./charts/entities/users.js";
|
||||
import { entity as ActiveUsersChart } from "./charts/entities/active-users.js";
|
||||
import { entity as InstanceChart } from "./charts/entities/instance.js";
|
||||
import { entity as PerUserNotesChart } from "./charts/entities/per-user-notes.js";
|
||||
import { entity as DriveChart } from "./charts/entities/drive.js";
|
||||
import { entity as PerUserReactionsChart } from "./charts/entities/per-user-reactions.js";
|
||||
import { entity as HashtagChart } from "./charts/entities/hashtag.js";
|
||||
import { entity as PerUserFollowingChart } from "./charts/entities/per-user-following.js";
|
||||
import { entity as PerUserDriveChart } from "./charts/entities/per-user-drive.js";
|
||||
import { entity as ApRequestChart } from "./charts/entities/ap-request.js";
|
||||
import { entity as TestChart } from "./charts/entities/test.js";
|
||||
import { entity as TestGroupedChart } from "./charts/entities/test-grouped.js";
|
||||
import { entity as TestUniqueChart } from "./charts/entities/test-unique.js";
|
||||
import { entity as TestIntersectionChart } from "./charts/entities/test-intersection.js";
|
||||
export const entities = [
|
||||
FederationChart.hour,
|
||||
FederationChart.day,
|
||||
NotesChart.hour,
|
||||
NotesChart.day,
|
||||
UsersChart.hour,
|
||||
UsersChart.day,
|
||||
ActiveUsersChart.hour,
|
||||
ActiveUsersChart.day,
|
||||
InstanceChart.hour,
|
||||
InstanceChart.day,
|
||||
PerUserNotesChart.hour,
|
||||
PerUserNotesChart.day,
|
||||
DriveChart.hour,
|
||||
DriveChart.day,
|
||||
PerUserReactionsChart.hour,
|
||||
PerUserReactionsChart.day,
|
||||
HashtagChart.hour,
|
||||
HashtagChart.day,
|
||||
PerUserFollowingChart.hour,
|
||||
PerUserFollowingChart.day,
|
||||
PerUserDriveChart.hour,
|
||||
PerUserDriveChart.day,
|
||||
ApRequestChart.hour,
|
||||
ApRequestChart.day,
|
||||
...process.env.NODE_ENV === "test" ? [
|
||||
TestChart.hour,
|
||||
TestChart.day,
|
||||
TestGroupedChart.hour,
|
||||
TestGroupedChart.day,
|
||||
TestUniqueChart.hour,
|
||||
TestUniqueChart.day,
|
||||
TestIntersectionChart.hour,
|
||||
TestIntersectionChart.day
|
||||
] : []
|
||||
];
|
||||
@@ -0,0 +1,46 @@
|
||||
import { beforeShutdown } from "../../misc/before-shutdown.js";
|
||||
import FederationChart from "./charts/federation.js";
|
||||
import NotesChart from "./charts/notes.js";
|
||||
import UsersChart from "./charts/users.js";
|
||||
import ActiveUsersChart from "./charts/active-users.js";
|
||||
import InstanceChart from "./charts/instance.js";
|
||||
import PerUserNotesChart from "./charts/per-user-notes.js";
|
||||
import DriveChart from "./charts/drive.js";
|
||||
import PerUserReactionsChart from "./charts/per-user-reactions.js";
|
||||
import HashtagChart from "./charts/hashtag.js";
|
||||
import PerUserFollowingChart from "./charts/per-user-following.js";
|
||||
import PerUserDriveChart from "./charts/per-user-drive.js";
|
||||
import ApRequestChart from "./charts/ap-request.js";
|
||||
export const federationChart = new FederationChart();
|
||||
export const notesChart = new NotesChart();
|
||||
export const usersChart = new UsersChart();
|
||||
export const activeUsersChart = new ActiveUsersChart();
|
||||
export const instanceChart = new InstanceChart();
|
||||
export const perUserNotesChart = new PerUserNotesChart();
|
||||
export const driveChart = new DriveChart();
|
||||
export const perUserReactionsChart = new PerUserReactionsChart();
|
||||
export const hashtagChart = new HashtagChart();
|
||||
export const perUserFollowingChart = new PerUserFollowingChart();
|
||||
export const perUserDriveChart = new PerUserDriveChart();
|
||||
export const apRequestChart = new ApRequestChart();
|
||||
const charts = [
|
||||
federationChart,
|
||||
notesChart,
|
||||
usersChart,
|
||||
activeUsersChart,
|
||||
instanceChart,
|
||||
perUserNotesChart,
|
||||
driveChart,
|
||||
perUserReactionsChart,
|
||||
hashtagChart,
|
||||
perUserFollowingChart,
|
||||
perUserDriveChart,
|
||||
apRequestChart
|
||||
];
|
||||
// 20分おきにメモリ情報をDBに書き込み
|
||||
setInterval(()=>{
|
||||
for (const chart of charts){
|
||||
chart.save();
|
||||
}
|
||||
}, 1000 * 60 * 20);
|
||||
beforeShutdown(()=>Promise.all(charts.map((chart)=>chart.save())));
|
||||
Reference in New Issue
Block a user