import { getNote } from "../../common/getters.js"; import config from "../../../../config/index.js"; import mfm from "mfm-js"; import { UserConverter } from "./user.js"; import { VisibilityConverter } from "./visibility.js"; import { escapeMFM } from "./mfm.js"; import { aggregateNoteEmojis, populateEmojis, prefetchEmojis } from "../../../../misc/populate-emojis.js"; import { EmojiConverter } from "./emoji.js"; import { DriveFiles, HtmlNoteCacheEntries, NoteFavorites, NoteReactions, Notes, NoteThreadMutings, UserNotePinings } from "../../../../models/index.js"; import { decodeReaction } from "../../../../misc/reaction-lib.js"; import { MentionConverter } from "./mention.js"; import { PollConverter } from "./poll.js"; import { populatePoll } from "../../../../models/repositories/note.js"; import { FileConverter } from "./file.js"; import { awaitAll } from "../../../../prelude/await-all.js"; import { UserHelpers } from "../helpers/user.js"; import { In, IsNull } from "typeorm"; import { MfmHelpers } from "../helpers/mfm.js"; import { getStubMastoContext } from "../index.js"; import { NoteHelpers } from "../helpers/note.js"; import isQuote from "../../../../misc/is-quote.js"; import { unique } from "../../../../prelude/array.js"; import { Cache } from "../../../../misc/cache.js"; import { isFiltered } from "../../../../misc/is-filtered.js"; export class NoteConverter { static noteContentHtmlCache = new Cache('html:note:content', config.htmlCache?.ttlSeconds ?? 60 * 60); static async encode(note, ctx, recurseCounter = 2) { const user = ctx.user; const noteUser = note.user ?? UserHelpers.getUserCached(note.userId, ctx); if (!await Notes.isVisibleForMe(note, user?.id ?? null)) throw new Error('Cannot encode note not visible for user'); const host = Promise.resolve(noteUser).then((noteUser)=>noteUser.host ?? null); const reactionEmojiNames = Object.keys(note.reactions).filter((x)=>x?.startsWith(":")).map((x)=>decodeReaction(x).reaction).map((x)=>x.replace(/:/g, "")); const populated = host.then(async (host)=>populateEmojis(note.emojis.concat(reactionEmojiNames), host)); const noteEmoji = populated.then((noteEmoji)=>noteEmoji.filter((e)=>e.name.indexOf("@") === -1).map((e)=>EmojiConverter.encode(e))); const reactionCount = Object.values(note.reactions).reduce((a, b)=>a + b, 0); const aggregateReaction = ctx.reactionAggregate?.get(note.id); const reaction = aggregateReaction !== undefined ? aggregateReaction : user ? NoteReactions.findOneBy({ userId: user.id, noteId: note.id }) : null; const isFavorited = Promise.resolve(reaction).then((p)=>!!p); const isReblogged = ctx.renoteAggregate?.get(note.id) ?? (user ? Notes.exist({ where: { userId: user.id, renoteId: note.id, text: IsNull() } }) : null); const renote = note.renote ?? (note.renoteId && recurseCounter > 0 ? getNote(note.renoteId, user) : null); const isBookmarked = ctx.bookmarkAggregate?.get(note.id) ?? (user ? NoteFavorites.exist({ where: { userId: user.id, noteId: note.id }, take: 1 }) : false); const isMuted = ctx.mutingAggregate?.get(note.threadId ?? note.id) ?? (user ? NoteThreadMutings.exist({ where: { userId: user.id, threadId: note.threadId || note.id } }) : false); const files = DriveFiles.packMany(note.fileIds); const mentions = Promise.all(note.mentions.map((p)=>UserHelpers.getUserCached(p, ctx).then((u)=>MentionConverter.encode(u, JSON.parse(note.mentionedRemoteUsers))).catch(()=>null))).then((p)=>p.filter((m)=>m)); const quoteUri = Promise.resolve(renote).then((renote)=>{ if (!renote || !isQuote(note)) return null; return renote.url ?? renote.uri ?? `${config.url}/notes/${renote.id}`; }); const identifier = `${note.id}:${(note.updatedAt ?? note.createdAt).getTime()}`; const text = quoteUri.then((quoteUri)=>note.text !== null ? quoteUri !== null ? note.text.replaceAll(`RE: ${quoteUri}`, '').replaceAll(quoteUri, '').trimEnd() : note.text : null); const content = this.noteContentHtmlCache.fetch(identifier, async ()=>Promise.resolve(await this.fetchFromCacheWithFallback(note, ctx) ?? text.then((text)=>text !== null ? quoteUri.then((quoteUri)=>MfmHelpers.toHtml(mfm.parse(text), JSON.parse(note.mentionedRemoteUsers), note.userHost, false, quoteUri)).then((p)=>p ?? escapeMFM(text)) : "")), true).then((p)=>p ?? ''); const isPinned = ctx.pinAggregate?.get(note.id) ?? (user && note.userId === user.id ? UserNotePinings.exist({ where: { userId: user.id, noteId: note.id } }) : undefined); const tags = note.tags.map((tag)=>{ return { name: tag, url: `${config.url}/tags/${tag}` }; }); const reblog = Promise.resolve(renote).then((renote)=>recurseCounter > 0 && renote ? this.encode(renote, ctx, isQuote(renote) && !isQuote(note) ? --recurseCounter : 0) : null); const filtered = isFiltered(note, user).then((res)=>{ if (!res || ctx.filterContext == null || ![ 'home', 'public' ].includes(ctx.filterContext)) return null; return [ { filter: { id: '0', title: 'Hard word mutes', context: [ 'home', 'public' ], expires_at: null, filter_action: 'hide', keywords: [], statuses: [] } } ]; }); // noinspection ES6MissingAwait return await awaitAll({ id: note.id, uri: note.uri ?? `https://${config.host}/notes/${note.id}`, url: note.url ?? note.uri ?? `https://${config.host}/notes/${note.id}`, account: Promise.resolve(noteUser).then((p)=>UserConverter.encode(p, ctx)), in_reply_to_id: note.replyId, in_reply_to_account_id: note.replyUserId, reblog: reblog.then((reblog)=>!isQuote(note) ? reblog : null), content: content, content_type: 'text/x.misskeymarkdown', text: text, created_at: note.createdAt.toISOString(), emojis: noteEmoji, replies_count: note.repliesCount, reblogs_count: note.renoteCount, favourites_count: reactionCount, reblogged: isReblogged, favourited: isFavorited, muted: isMuted, sensitive: files.then((files)=>files.length > 0 ? files.some((f)=>f.isSensitive) : false), spoiler_text: note.cw ? note.cw : "", visibility: VisibilityConverter.encode(note.visibility), media_attachments: files.then((files)=>files.length > 0 ? files.map((f)=>FileConverter.encode(f)) : []), mentions: mentions, tags: tags, card: null, poll: note.hasPoll ? populatePoll(note, user?.id ?? null).then((p)=>noteEmoji.then((emojis)=>PollConverter.encode(p, note.id, emojis))) : null, application: null, language: null, pinned: isPinned, reactions: populated.then((populated)=>Promise.resolve(reaction).then((reaction)=>this.encodeReactions(note.reactions, reaction?.reaction, populated))), bookmarked: isBookmarked, quote: reblog.then((reblog)=>reblog !== null && isQuote(note) ? { state: "accepted", quoted_status: reblog, ...reblog } : null), quote_id: isQuote(note) ? note.renoteId : null, edited_at: note.updatedAt?.toISOString() ?? null, filtered: filtered, quote_approval: { automatic: [ "public" ], manual: [], current_user: "automatic" } }); } static async encodeMany(notes, ctx) { await this.aggregateData(notes, ctx); const encoded = notes.map((n)=>this.encode(n, ctx)); return Promise.all(encoded); } static async aggregateData(notes, ctx) { if (notes.length === 0) return; const user = ctx.user; const reactionAggregate = new Map(); const renoteAggregate = new Map(); const mutingAggregate = new Map(); const bookmarkAggregate = new Map(); const pinAggregate = new Map(); const htmlNoteCacheAggregate = new Map(); const renoteIds = notes.filter((n)=>n.renoteId != null).map((n)=>n.renoteId); const noteIds = unique(notes.map((n)=>n.id)); const targets = unique([ ...noteIds, ...renoteIds ]); if (config.htmlCache?.dbFallback) { const htmlNoteCacheEntries = await HtmlNoteCacheEntries.findBy({ noteId: In(targets) }); for (const target of targets){ htmlNoteCacheAggregate.set(target, htmlNoteCacheEntries.find((n)=>n.noteId === target) ?? null); } } if (user?.id != null) { const mutingTargets = unique([ ...notes.map((n)=>n.threadId ?? n.id) ]); const pinTargets = unique([ ...notes.filter((n)=>n.userId === user.id).map((n)=>n.id) ]); const reactions = await NoteReactions.findBy({ userId: user.id, noteId: In(targets) }); const renotes = await Notes.createQueryBuilder('note').select('note.renoteId').where('note.userId = :meId', { meId: user.id }).andWhere('note.renoteId IN (:...targets)', { targets }).andWhere('note.text IS NULL').andWhere('note.hasPoll = FALSE').andWhere(`note.fileIds = '{}'`).getMany(); const mutings = await NoteThreadMutings.createQueryBuilder('muting').select('muting.threadId').where('muting.userId = :meId', { meId: user.id }).andWhere('muting.threadId IN (:...targets)', { targets: mutingTargets }).getMany(); const bookmarks = await NoteFavorites.createQueryBuilder('bookmark').select('bookmark.noteId').where('bookmark.userId = :meId', { meId: user.id }).andWhere('bookmark.noteId IN (:...targets)', { targets }).getMany(); const pins = pinTargets.length > 0 ? await UserNotePinings.createQueryBuilder('pin').select('pin.noteId').where('pin.userId = :meId', { meId: user.id }).andWhere('pin.noteId IN (:...targets)', { targets: pinTargets }).getMany() : []; for (const target of targets){ reactionAggregate.set(target, reactions.find((r)=>r.noteId === target) ?? null); renoteAggregate.set(target, !!renotes.find((n)=>n.renoteId === target)); bookmarkAggregate.set(target, !!bookmarks.find((b)=>b.noteId === target)); } for (const target of mutingTargets){ mutingAggregate.set(target, !!mutings.find((m)=>m.threadId === target)); } for (const target of pinTargets){ mutingAggregate.set(target, !!pins.find((m)=>m.noteId === target)); } } ctx.reactionAggregate = reactionAggregate; ctx.renoteAggregate = renoteAggregate; ctx.mutingAggregate = mutingAggregate; ctx.bookmarkAggregate = bookmarkAggregate; ctx.pinAggregate = pinAggregate; ctx.htmlNoteCacheAggregate = htmlNoteCacheAggregate; const users = notes.filter((p)=>!!p.user).map((p)=>p.user); const renoteUserIds = notes.filter((p)=>p.renoteUserId !== null).map((p)=>p.renoteUserId); await UserConverter.aggregateData([ ...users ], ctx); await UserConverter.aggregateDataByIds(renoteUserIds, ctx); await prefetchEmojis(aggregateNoteEmojis(notes)); } static encodeReactions(reactions, myReaction, populated) { return Object.keys(reactions).map((key)=>{ const isCustom = key.startsWith(':') && key.endsWith(':'); const name = isCustom ? key.substring(1, key.length - 1) : key; const populatedName = isCustom && name.indexOf('@') === -1 ? `${name}@.` : name; const url = isCustom ? populated.find((p)=>p.name === populatedName)?.url : undefined; return { count: reactions[key], me: key === myReaction, name: name, url: url, static_url: url }; }).filter((r)=>r.count > 0); } static async encodeEvent(note, user, filterContext) { const ctx = getStubMastoContext(user, filterContext); NoteHelpers.fixupEventNote(note); return NoteConverter.encode(note, ctx); } static async fetchFromCacheWithFallback(note, ctx) { if (!config.htmlCache?.dbFallback) return null; let dbHit = ctx.htmlNoteCacheAggregate?.get(note.id); if (dbHit === undefined) dbHit = HtmlNoteCacheEntries.findOneBy({ noteId: note.id }); return Promise.resolve(dbHit).then((res)=>{ if (res === null || res.updatedAt?.getTime() !== note.updatedAt?.getTime()) { return this.dbCacheMiss(note, ctx); } return res; }).then((hit)=>hit?.updatedAt === note.updatedAt ? hit?.content ?? null : null); } static async dbCacheMiss(note, ctx) { const identifier = `${note.id}:${(note.updatedAt ?? note.createdAt).getTime()}`; const cache = ctx.cache; return cache.locks.acquire(identifier, async ()=>{ const cachedContent = await this.noteContentHtmlCache.get(identifier); if (cachedContent !== undefined) { return { content: cachedContent }; } const quoteUri = note.renote ? isQuote(note) ? note.renote.url ?? note.renote.uri ?? `${config.url}/notes/${note.renote.id}` : null : null; const text = note.text !== null ? quoteUri !== null ? note.text.replaceAll(`RE: ${quoteUri}`, '').replaceAll(quoteUri, '').trimEnd() : note.text : null; const content = text !== null ? MfmHelpers.toHtml(mfm.parse(text), JSON.parse(note.mentionedRemoteUsers), note.userHost, false, quoteUri).then((p)=>p ?? escapeMFM(text)) : null; HtmlNoteCacheEntries.upsert({ noteId: note.id, updatedAt: note.updatedAt ?? note.createdAt, content: await content }, [ "noteId" ]); await this.noteContentHtmlCache.set(identifier, await content); return { content }; }); } static async prewarmCache(note) { if (!config.htmlCache?.prewarm) return; const identifier = `${note.id}:${(note.updatedAt ?? note.createdAt).getTime()}`; if (await this.noteContentHtmlCache.get(identifier) !== undefined) return; if (note.renoteId !== null && !note.renote) { note.renote = await Notes.findOneBy({ id: note.renoteId }); } const quoteUri = note.renote ? isQuote(note) ? note.renote.url ?? note.renote.uri ?? `${config.url}/notes/${note.renote.id}` : null : null; const text = note.text !== null ? quoteUri !== null ? note.text.replaceAll(`RE: ${quoteUri}`, '').replaceAll(quoteUri, '').trimEnd() : note.text : null; const content = text !== null ? MfmHelpers.toHtml(mfm.parse(text), JSON.parse(note.mentionedRemoteUsers), note.userHost, false, quoteUri).then((p)=>p ?? escapeMFM(text)) : null; if (note.user) UserConverter.prewarmCache(note.user); else if (note.userId) UserConverter.prewarmCacheById(note.userId); if (note.replyUserId) UserConverter.prewarmCacheById(note.replyUserId); if (note.renoteUserId) UserConverter.prewarmCacheById(note.renoteUserId); this.noteContentHtmlCache.set(identifier, await content); if (config.htmlCache?.dbFallback) HtmlNoteCacheEntries.upsert({ noteId: note.id, updatedAt: note.updatedAt ?? note.createdAt, content: await content }, [ "noteId" ]); } }