From 5d482ff3bfc607e47f4aec8ed66062eca7c0d49b Mon Sep 17 00:00:00 2001 From: v Date: Sun, 9 Feb 2025 01:46:08 +0100 Subject: [PATCH] Reporter: Add framework for automatic testing of Discord updates (#3208) --- .github/workflows/reportBrokenPlugins.yml | 66 ++++++--- scripts/generateReport.ts | 165 ++++++++++++---------- src/debug/runReporter.ts | 10 +- src/utils/Logger.ts | 2 +- src/webpack/patchWebpack.ts | 9 +- 5 files changed, 156 insertions(+), 96 deletions(-) diff --git a/.github/workflows/reportBrokenPlugins.yml b/.github/workflows/reportBrokenPlugins.yml index a669c1a27..f1e53e4d0 100644 --- a/.github/workflows/reportBrokenPlugins.yml +++ b/.github/workflows/reportBrokenPlugins.yml @@ -1,9 +1,22 @@ name: Test Patches on: workflow_dispatch: - schedule: - # Every day at midnight - - cron: 0 0 * * * + inputs: + discord_branch: + type: choice + description: "Discord Branch to test patches on" + options: + - both + - stable + - canary + default: both + webhook_url: + type: string + description: "Webhook URL that the report will be posted to. This will be visible for everyone, so DO NOT pass sensitive webhooks like discord webhook. This is meant to be used by Venbot." + required: false + # schedule: + # # Every day at midnight + # - cron: 0 0 * * * jobs: TestPlugins: @@ -40,28 +53,43 @@ jobs: - name: Build Vencord Reporter Version run: pnpm buildReporter - - name: Create Report + - name: Run Reporter timeout-minutes: 10 run: | export PATH="$PWD/node_modules/.bin:$PATH" export CHROMIUM_BIN=${{ steps.setup-chrome.outputs.chrome-path }} esbuild scripts/generateReport.ts > dist/report.mjs - node dist/report.mjs >> $GITHUB_STEP_SUMMARY - env: - DISCORD_TOKEN: ${{ secrets.DISCORD_TOKEN }} - DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }} - - name: Create Report (Canary) - timeout-minutes: 10 - if: success() || failure() # even run if previous one failed - run: | - export PATH="$PWD/node_modules/.bin:$PATH" - export CHROMIUM_BIN=${{ steps.setup-chrome.outputs.chrome-path }} - export USE_CANARY=true + stable_output_file=$(mktemp) + canary_output_file=$(mktemp) - esbuild scripts/generateReport.ts > dist/report.mjs - node dist/report.mjs >> $GITHUB_STEP_SUMMARY + pids="" + + branch="${{ inputs.discord_branch }}" + if [[ "${{ github.event_name }}" = "schedule" ]]; then + branch="both" + fi + + if [[ "$branch" = "both" || "$branch" = "stable" ]]; then + node dist/report.mjs > "$stable_output_file" & + pids+=" $!" + fi + + if [[ "$branch" = "both" || "$branch" = "canary" ]]; then + USE_CANARY=true node dist/report.mjs > "$canary_output_file" & + pids+=" $!" + fi + + exit_code=0 + for pid in $pids; do + if ! wait "$pid"; then + exit_code=1 + fi + done + + cat "$stable_output_file" "$canary_output_file" >> $GITHUB_STEP_SUMMARY + exit $exit_code env: - DISCORD_TOKEN: ${{ secrets.DISCORD_TOKEN }} - DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }} + WEBHOOK_URL: ${{ inputs.webhook_url || secrets.DISCORD_WEBHOOK }} + WEBHOOK_SECRET: ${{ secrets.WEBHOOK_SECRET }} diff --git a/scripts/generateReport.ts b/scripts/generateReport.ts index 0086f2477..5cab1b46e 100644 --- a/scripts/generateReport.ts +++ b/scripts/generateReport.ts @@ -23,17 +23,24 @@ // eslint-disable-next-line spaced-comment /// +import { createHmac } from "crypto"; import { readFileSync } from "fs"; import pup, { JSHandle } from "puppeteer-core"; -for (const variable of ["DISCORD_TOKEN", "CHROMIUM_BIN"]) { +const logStderr = (...data: any[]) => console.error(`${CANARY ? "CANARY" : "STABLE"} ---`, ...data); + +for (const variable of ["CHROMIUM_BIN"]) { if (!process.env[variable]) { - console.error(`Missing environment variable ${variable}`); + logStderr(`Missing environment variable ${variable}`); process.exit(1); } } const CANARY = process.env.USE_CANARY === "true"; +let metaData = { + buildNumber: "Unknown Build Number", + buildHash: "Unknown Build Hash" +}; const browser = await pup.launch({ headless: true, @@ -128,56 +135,74 @@ async function printReport() { console.log(); - if (process.env.DISCORD_WEBHOOK) { - await fetch(process.env.DISCORD_WEBHOOK, { - method: "POST", - headers: { - "Content-Type": "application/json" - }, - body: JSON.stringify({ - description: "Here's the latest Vencord Report!", - username: "Vencord Reporter" + (CANARY ? " (Canary)" : ""), - embeds: [ - { - title: "Bad Patches", - description: report.badPatches.map(p => { - const lines = [ - `**__${p.plugin} (${p.type}):__**`, - `ID: \`${p.id}\``, - `Match: ${toCodeBlock(p.match, "Match: ".length, true)}` - ]; - if (p.error) lines.push(`Error: ${toCodeBlock(p.error, "Error: ".length, true)}`); - return lines.join("\n"); - }).join("\n\n") || "None", - color: report.badPatches.length ? 0xff0000 : 0x00ff00 + if (process.env.WEBHOOK_URL) { + const body = JSON.stringify({ + username: "Vencord Reporter" + (CANARY ? " (Canary)" : ""), + embeds: [ + { + author: { + name: `Discord ${CANARY ? "Canary" : "Stable"} (${metaData.buildNumber})`, + url: `https://nelly.tools/builds/app/${metaData.buildHash}`, + icon_url: CANARY ? "https://cdn.discordapp.com/emojis/1252721945699549327.png?size=128" : "https://cdn.discordapp.com/emojis/1252721943463985272.png?size=128" }, - { - title: "Bad Webpack Finds", - description: report.badWebpackFinds.map(f => toCodeBlock(f, 0, true)).join("\n") || "None", - color: report.badWebpackFinds.length ? 0xff0000 : 0x00ff00 - }, - { - title: "Bad Starts", - description: report.badStarts.map(p => { - const lines = [ - `**__${p.plugin}:__**`, - toCodeBlock(p.error, 0, true) - ]; - return lines.join("\n"); - } - ).join("\n\n") || "None", - color: report.badStarts.length ? 0xff0000 : 0x00ff00 - }, - { - title: "Discord Errors", - description: report.otherErrors.length ? toCodeBlock(report.otherErrors.join("\n"), 0, true) : "None", - color: report.otherErrors.length ? 0xff0000 : 0x00ff00 + color: CANARY ? 0xfbb642 : 0x5865f2 + }, + { + title: "Bad Patches", + description: report.badPatches.map(p => { + const lines = [ + `**__${p.plugin} (${p.type}):__**`, + `ID: \`${p.id}\``, + `Match: ${toCodeBlock(p.match, "Match: ".length, true)}` + ]; + if (p.error) lines.push(`Error: ${toCodeBlock(p.error, "Error: ".length, true)}`); + return lines.join("\n"); + }).join("\n\n") || "None", + color: report.badPatches.length ? 0xff0000 : 0x00ff00 + }, + { + title: "Bad Webpack Finds", + description: report.badWebpackFinds.map(f => toCodeBlock(f, 0, true)).join("\n") || "None", + color: report.badWebpackFinds.length ? 0xff0000 : 0x00ff00 + }, + { + title: "Bad Starts", + description: report.badStarts.map(p => { + const lines = [ + `**__${p.plugin}:__**`, + toCodeBlock(p.error, 0, true) + ]; + return lines.join("\n"); } - ] - }) + ).join("\n\n") || "None", + color: report.badStarts.length ? 0xff0000 : 0x00ff00 + }, + { + title: "Discord Errors", + description: report.otherErrors.length ? toCodeBlock(report.otherErrors.join("\n"), 0, true) : "None", + color: report.otherErrors.length ? 0xff0000 : 0x00ff00 + } + ] + }); + + const headers = { + "Content-Type": "application/json" + }; + + // functions similar to https://docs.github.com/en/webhooks/using-webhooks/validating-webhook-deliveries + // used by venbot to ensure webhook invocations are genuine (since we will pass the webhook url as a workflow input which is publicly visible) + // generate a secret with something like `openssl rand -hex 128` + if (process.env.WEBHOOK_SECRET) { + headers["X-Signature"] = "sha256=" + createHmac("sha256", process.env.WEBHOOK_SECRET).update(body).digest("hex"); + } + + await fetch(process.env.WEBHOOK_URL, { + method: "POST", + headers, + body }).then(res => { - if (!res.ok) console.error(`Webhook failed with status ${res.status}`); - else console.error("Posted to Discord Webhook successfully"); + if (!res.ok) logStderr(`Webhook failed with status ${res.status}`); + else logStderr("Posted to Webhook successfully"); }); } } @@ -186,10 +211,13 @@ page.on("console", async e => { const level = e.type(); const rawArgs = e.args(); - async function getText() { + async function getText(skipFirst = true) { + let args = e.args(); + if (skipFirst) args = args.slice(1); + try { return await Promise.all( - e.args().map(async a => { + args.map(async a => { return await maybeGetError(a) || await a.jsonValue(); }) ).then(a => a.join(" ").trim()); @@ -202,6 +230,12 @@ page.on("console", async e => { const isVencord = firstArg === "[Vencord]"; const isDebug = firstArg === "[PUP_DEBUG]"; + const isReporterMeta = firstArg === "[REPORTER_META]"; + + if (isReporterMeta) { + metaData = await rawArgs[1].jsonValue() as any; + return; + } outer: if (isVencord) { @@ -218,7 +252,7 @@ page.on("console", async e => { const patchFailMatch = message.match(/Patch by (.+?) (had no effect|errored|found no module|took [\d.]+?ms) \(Module id is (.+?)\): (.+)/)!; if (!patchFailMatch) break; - console.error(await getText()); + logStderr(await getText()); process.exitCode = 1; const [, plugin, type, id, regex] = patchFailMatch; @@ -235,7 +269,7 @@ page.on("console", async e => { const failedToStartMatch = message.match(/Failed to start (.+)/); if (!failedToStartMatch) break; - console.error(await getText()); + logStderr(await getText()); process.exitCode = 1; const [, name] = failedToStartMatch; @@ -246,7 +280,7 @@ page.on("console", async e => { break; case "LazyChunkLoader:": - console.error(await getText()); + logStderr(await getText()); switch (message) { case "A fatal error occurred:": @@ -255,7 +289,7 @@ page.on("console", async e => { break; case "Reporter:": - console.error(await getText()); + logStderr(await getText()); switch (message) { case "A fatal error occurred:": @@ -273,47 +307,36 @@ page.on("console", async e => { } if (isDebug) { - console.error(await getText()); + logStderr(await getText()); } else if (level === "error") { - const text = await getText(); + const text = await getText(false); if (text.length && !text.startsWith("Failed to load resource: the server responded with a status of") && !text.includes("Webpack")) { if (IGNORED_DISCORD_ERRORS.some(regex => text.match(regex))) { report.ignoredErrors.push(text); } else { - console.error("[Unexpected Error]", text); + logStderr("[Unexpected Error]", text); report.otherErrors.push(text); } } } }); -page.on("error", e => console.error("[Error]", e.message)); +page.on("error", e => logStderr("[Error]", e.message)); page.on("pageerror", e => { if (e.message.includes("Sentry successfully disabled")) return; if (!e.message.startsWith("Object") && !e.message.includes("Cannot find module") && !/^.{1,2}$/.test(e.message)) { - console.error("[Page Error]", e.message); + logStderr("[Page Error]", e.message); report.otherErrors.push(e.message); } else { report.ignoredErrors.push(e.message); } }); -async function reporterRuntime(token: string) { - Vencord.Webpack.waitFor( - "loginToken", - m => { - console.log("[PUP_DEBUG]", "Logging in with token..."); - m.loginToken(token); - } - ); -} - await page.evaluateOnNewDocument(` if (location.host.endsWith("discord.com")) { ${readFileSync("./dist/browser.js", "utf-8")}; - (${reporterRuntime.toString()})(${JSON.stringify(process.env.DISCORD_TOKEN)}); } `); diff --git a/src/debug/runReporter.ts b/src/debug/runReporter.ts index e9c37970f..8d4194bc4 100644 --- a/src/debug/runReporter.ts +++ b/src/debug/runReporter.ts @@ -7,6 +7,7 @@ import { Logger } from "@utils/Logger"; import * as Webpack from "@webpack"; import { addPatch, patches } from "plugins"; +import { getBuildNumber } from "webpack/patchWebpack"; import { loadLazyChunks } from "./loadLazyChunks"; @@ -37,6 +38,13 @@ async function runReporter() { await loadLazyChunksDone; + if (IS_REPORTER && IS_WEB && !IS_VESKTOP) { + console.log("[REPORTER_META]", { + buildNumber: getBuildNumber(), + buildHash: window.GLOBAL_ENV.SENTRY_TAGS.buildId + }); + } + for (const patch of patches) { if (!patch.all) { new Logger("WebpackInterceptor").warn(`Patch by ${patch.plugin} found no module (Module id is -): ${patch.find}`); @@ -44,7 +52,7 @@ async function runReporter() { } for (const [plugin, moduleId, match, totalTime] of Vencord.WebpackPatcher.patchTimings) { - if (totalTime > 3) { + if (totalTime > 5) { new Logger("WebpackInterceptor").warn(`Patch by ${plugin} took ${Math.round(totalTime * 100) / 100}ms (Module id is ${String(moduleId)}): ${match}`); } } diff --git a/src/utils/Logger.ts b/src/utils/Logger.ts index 22a381360..aec7292a3 100644 --- a/src/utils/Logger.ts +++ b/src/utils/Logger.ts @@ -32,7 +32,7 @@ export class Logger { constructor(public name: string, public color: string = "white") { } private _log(level: "log" | "error" | "warn" | "info" | "debug", levelColor: string, args: any[], customFmt = "") { - if (IS_REPORTER && IS_WEB) { + if (IS_REPORTER && IS_WEB && !IS_VESKTOP) { console[level]("[Vencord]", this.name + ":", ...args); return; } diff --git a/src/webpack/patchWebpack.ts b/src/webpack/patchWebpack.ts index 7a7107acb..870362373 100644 --- a/src/webpack/patchWebpack.ts +++ b/src/webpack/patchWebpack.ts @@ -442,11 +442,12 @@ function patchFactory(id: PropertyKey, factory: AnyModuleFactory): [patchedFacto continue; } - const buildNumber = getBuildNumber(); - const shouldCheckBuildId = !Settings.eagerPatches && buildNumber !== -1; + // Reporter eagerly patches and cannot retrieve the build number because this code runs before the module for it is loaded + const buildNumber = IS_REPORTER ? -1 : getBuildNumber(); + const shouldCheckBuildNumber = !Settings.eagerPatches && buildNumber !== -1; if ( - shouldCheckBuildId && + shouldCheckBuildNumber && (patch.fromBuild != null && buildNumber < patch.fromBuild) || (patch.toBuild != null && buildNumber > patch.toBuild) ) { @@ -468,7 +469,7 @@ function patchFactory(id: PropertyKey, factory: AnyModuleFactory): [patchedFacto // We change all patch.replacement to array in plugins/index for (const replacement of patch.replacement as PatchReplacement[]) { if ( - shouldCheckBuildId && + shouldCheckBuildNumber && (replacement.fromBuild != null && buildNumber < replacement.fromBuild) || (replacement.toBuild != null && buildNumber > replacement.toBuild) ) {