Reporter: Add framework for automatic testing of Discord updates (#3208)

.github/workflows/reportBrokenPlugins.yml

@@ -1,9 +1,22 @@
 name: Test Patches
 on:
     workflow_dispatch:
-    schedule:
+        inputs:
-        # Every day at midnight
+            discord_branch:
-        - cron: 0 0 * * *
+                type: choice
+                description: "Discord Branch to test patches on"
+                options:
+                    - both
+                    - stable
+                    - canary
+                default: both
+            webhook_url:
+                type: string
+                description: "Webhook URL that the report will be posted to. This will be visible for everyone, so DO NOT pass sensitive webhooks like discord webhook. This is meant to be used by Venbot."
+                required: false
+    # schedule:
+    #   # Every day at midnight
+    #   - cron: 0 0 * * *
 
 jobs:
     TestPlugins:
@@ -40,28 +53,43 @@ jobs:
             - name: Build Vencord Reporter Version
               run: pnpm buildReporter
 
-            - name: Create Report
+            - name: Run Reporter
               timeout-minutes: 10
               run: |
                   export PATH="$PWD/node_modules/.bin:$PATH"
                   export CHROMIUM_BIN=${{ steps.setup-chrome.outputs.chrome-path }}
 
                   esbuild scripts/generateReport.ts > dist/report.mjs
-                  node dist/report.mjs >> $GITHUB_STEP_SUMMARY
-              env:
-                  DISCORD_TOKEN: ${{ secrets.DISCORD_TOKEN }}
-                  DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }}
 
-            - name: Create Report (Canary)
+                  stable_output_file=$(mktemp)
-              timeout-minutes: 10
+                  canary_output_file=$(mktemp)
-              if: success() || failure() # even run if previous one failed
-              run: |
-                  export PATH="$PWD/node_modules/.bin:$PATH"
-                  export CHROMIUM_BIN=${{ steps.setup-chrome.outputs.chrome-path }}
-                  export USE_CANARY=true
 
-                  esbuild scripts/generateReport.ts > dist/report.mjs
+                  pids=""
-                  node dist/report.mjs >> $GITHUB_STEP_SUMMARY
+
+                  branch="${{ inputs.discord_branch }}"
+                  if [[ "${{ github.event_name }}" = "schedule" ]]; then
+                    branch="both"
+                  fi
+
+                  if [[ "$branch" = "both" || "$branch" = "stable" ]]; then
+                    node dist/report.mjs > "$stable_output_file" &
+                    pids+=" $!"
+                  fi
+
+                  if [[ "$branch" = "both" || "$branch" = "canary" ]]; then
+                    USE_CANARY=true node dist/report.mjs > "$canary_output_file" &
+                    pids+=" $!"
+                  fi
+
+                  exit_code=0
+                  for pid in $pids; do
+                      if ! wait "$pid"; then
+                        exit_code=1
+                      fi
+                  done
+
+                  cat "$stable_output_file" "$canary_output_file" >> $GITHUB_STEP_SUMMARY
+                  exit $exit_code
               env:
-                  DISCORD_TOKEN: ${{ secrets.DISCORD_TOKEN }}
+                  WEBHOOK_URL: ${{ inputs.webhook_url || secrets.DISCORD_WEBHOOK }}
-                  DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }}
+                  WEBHOOK_SECRET: ${{ secrets.WEBHOOK_SECRET }}

scripts/generateReport.ts

@@ -23,17 +23,24 @@
 // eslint-disable-next-line spaced-comment
 /// <reference types="../src/modules" />
 
+import { createHmac } from "crypto";
 import { readFileSync } from "fs";
 import pup, { JSHandle } from "puppeteer-core";
 
-for (const variable of ["DISCORD_TOKEN", "CHROMIUM_BIN"]) {
+const logStderr = (...data: any[]) => console.error(`${CANARY ? "CANARY" : "STABLE"} ---`, ...data);
+
+for (const variable of ["CHROMIUM_BIN"]) {
     if (!process.env[variable]) {
-        console.error(`Missing environment variable ${variable}`);
+        logStderr(`Missing environment variable ${variable}`);
         process.exit(1);
     }
 }
 
 const CANARY = process.env.USE_CANARY === "true";
+let metaData = {
+    buildNumber: "Unknown Build Number",
+    buildHash: "Unknown Build Hash"
+};
 
 const browser = await pup.launch({
     headless: true,
@@ -128,56 +135,74 @@ async function printReport() {
 
     console.log();
 
-    if (process.env.DISCORD_WEBHOOK) {
+    if (process.env.WEBHOOK_URL) {
-        await fetch(process.env.DISCORD_WEBHOOK, {
+        const body = JSON.stringify({
-            method: "POST",
+            username: "Vencord Reporter" + (CANARY ? " (Canary)" : ""),
-            headers: {
+            embeds: [
-                "Content-Type": "application/json"
+                {
-            },
+                    author: {
-            body: JSON.stringify({
+                        name: `Discord ${CANARY ? "Canary" : "Stable"} (${metaData.buildNumber})`,
-                description: "Here's the latest Vencord Report!",
+                        url: `https://nelly.tools/builds/app/${metaData.buildHash}`,
-                username: "Vencord Reporter" + (CANARY ? " (Canary)" : ""),
+                        icon_url: CANARY ? "https://cdn.discordapp.com/emojis/1252721945699549327.png?size=128" : "https://cdn.discordapp.com/emojis/1252721943463985272.png?size=128"
-                embeds: [
-                    {
-                        title: "Bad Patches",
-                        description: report.badPatches.map(p => {
-                            const lines = [
-                                `**__${p.plugin} (${p.type}):__**`,
-                                `ID: \`${p.id}\``,
-                                `Match: ${toCodeBlock(p.match, "Match: ".length, true)}`
-                            ];
-                            if (p.error) lines.push(`Error: ${toCodeBlock(p.error, "Error: ".length, true)}`);
-                            return lines.join("\n");
-                        }).join("\n\n") || "None",
-                        color: report.badPatches.length ? 0xff0000 : 0x00ff00
                     },
-                    {
+                    color: CANARY ? 0xfbb642 : 0x5865f2
-                        title: "Bad Webpack Finds",
+                },
-                        description: report.badWebpackFinds.map(f => toCodeBlock(f, 0, true)).join("\n") || "None",
+                {
-                        color: report.badWebpackFinds.length ? 0xff0000 : 0x00ff00
+                    title: "Bad Patches",
-                    },
+                    description: report.badPatches.map(p => {
-                    {
+                        const lines = [
-                        title: "Bad Starts",
+                            `**__${p.plugin} (${p.type}):__**`,
-                        description: report.badStarts.map(p => {
+                            `ID: \`${p.id}\``,
-                            const lines = [
+                            `Match: ${toCodeBlock(p.match, "Match: ".length, true)}`
-                                `**__${p.plugin}:__**`,
+                        ];
-                                toCodeBlock(p.error, 0, true)
+                        if (p.error) lines.push(`Error: ${toCodeBlock(p.error, "Error: ".length, true)}`);
-                            ];
+                        return lines.join("\n");
-                            return lines.join("\n");
+                    }).join("\n\n") || "None",
-                        }
+                    color: report.badPatches.length ? 0xff0000 : 0x00ff00
-                        ).join("\n\n") || "None",
+                },
-                        color: report.badStarts.length ? 0xff0000 : 0x00ff00
+                {
-                    },
+                    title: "Bad Webpack Finds",
-                    {
+                    description: report.badWebpackFinds.map(f => toCodeBlock(f, 0, true)).join("\n") || "None",
-                        title: "Discord Errors",
+                    color: report.badWebpackFinds.length ? 0xff0000 : 0x00ff00
-                        description: report.otherErrors.length ? toCodeBlock(report.otherErrors.join("\n"), 0, true) : "None",
+                },
-                        color: report.otherErrors.length ? 0xff0000 : 0x00ff00
+                {
+                    title: "Bad Starts",
+                    description: report.badStarts.map(p => {
+                        const lines = [
+                            `**__${p.plugin}:__**`,
+                            toCodeBlock(p.error, 0, true)
+                        ];
+                        return lines.join("\n");
                     }
-                ]
+                    ).join("\n\n") || "None",
-            })
+                    color: report.badStarts.length ? 0xff0000 : 0x00ff00
+                },
+                {
+                    title: "Discord Errors",
+                    description: report.otherErrors.length ? toCodeBlock(report.otherErrors.join("\n"), 0, true) : "None",
+                    color: report.otherErrors.length ? 0xff0000 : 0x00ff00
+                }
+            ]
+        });
+
+        const headers = {
+            "Content-Type": "application/json"
+        };
+
+        // functions similar to https://docs.github.com/en/webhooks/using-webhooks/validating-webhook-deliveries
+        // used by venbot to ensure webhook invocations are genuine (since we will pass the webhook url as a workflow input which is publicly visible)
+        // generate a secret with something like `openssl rand -hex 128`
+        if (process.env.WEBHOOK_SECRET) {
+            headers["X-Signature"] = "sha256=" + createHmac("sha256", process.env.WEBHOOK_SECRET).update(body).digest("hex");
+        }
+
+        await fetch(process.env.WEBHOOK_URL, {
+            method: "POST",
+            headers,
+            body
         }).then(res => {
-            if (!res.ok) console.error(`Webhook failed with status ${res.status}`);
+            if (!res.ok) logStderr(`Webhook failed with status ${res.status}`);
-            else console.error("Posted to Discord Webhook successfully");
+            else logStderr("Posted to Webhook successfully");
         });
     }
 }
@@ -186,10 +211,13 @@ page.on("console", async e => {
     const level = e.type();
     const rawArgs = e.args();
 
-    async function getText() {
+    async function getText(skipFirst = true) {
+        let args = e.args();
+        if (skipFirst) args = args.slice(1);
+
         try {
             return await Promise.all(
-                e.args().map(async a => {
+                args.map(async a => {
                     return await maybeGetError(a) || await a.jsonValue();
                 })
             ).then(a => a.join(" ").trim());
@@ -202,6 +230,12 @@ page.on("console", async e => {
 
     const isVencord = firstArg === "[Vencord]";
     const isDebug = firstArg === "[PUP_DEBUG]";
+    const isReporterMeta = firstArg === "[REPORTER_META]";
+
+    if (isReporterMeta) {
+        metaData = await rawArgs[1].jsonValue() as any;
+        return;
+    }
 
     outer:
     if (isVencord) {
@@ -218,7 +252,7 @@ page.on("console", async e => {
                 const patchFailMatch = message.match(/Patch by (.+?) (had no effect|errored|found no module|took [\d.]+?ms) \(Module id is (.+?)\): (.+)/)!;
                 if (!patchFailMatch) break;
 
-                console.error(await getText());
+                logStderr(await getText());
                 process.exitCode = 1;
 
                 const [, plugin, type, id, regex] = patchFailMatch;
@@ -235,7 +269,7 @@ page.on("console", async e => {
                 const failedToStartMatch = message.match(/Failed to start (.+)/);
                 if (!failedToStartMatch) break;
 
-                console.error(await getText());
+                logStderr(await getText());
                 process.exitCode = 1;
 
                 const [, name] = failedToStartMatch;
@@ -246,7 +280,7 @@ page.on("console", async e => {
 
                 break;
             case "LazyChunkLoader:":
-                console.error(await getText());
+                logStderr(await getText());
 
                 switch (message) {
                     case "A fatal error occurred:":
@@ -255,7 +289,7 @@ page.on("console", async e => {
 
                 break;
             case "Reporter:":
-                console.error(await getText());
+                logStderr(await getText());
 
                 switch (message) {
                     case "A fatal error occurred:":
@@ -273,47 +307,36 @@ page.on("console", async e => {
     }
 
     if (isDebug) {
-        console.error(await getText());
+        logStderr(await getText());
     } else if (level === "error") {
-        const text = await getText();
+        const text = await getText(false);
 
         if (text.length && !text.startsWith("Failed to load resource: the server responded with a status of") && !text.includes("Webpack")) {
             if (IGNORED_DISCORD_ERRORS.some(regex => text.match(regex))) {
                 report.ignoredErrors.push(text);
             } else {
-                console.error("[Unexpected Error]", text);
+                logStderr("[Unexpected Error]", text);
                 report.otherErrors.push(text);
             }
         }
     }
 });
 
-page.on("error", e => console.error("[Error]", e.message));
+page.on("error", e => logStderr("[Error]", e.message));
 page.on("pageerror", e => {
     if (e.message.includes("Sentry successfully disabled")) return;
 
     if (!e.message.startsWith("Object") && !e.message.includes("Cannot find module") && !/^.{1,2}$/.test(e.message)) {
-        console.error("[Page Error]", e.message);
+        logStderr("[Page Error]", e.message);
         report.otherErrors.push(e.message);
     } else {
         report.ignoredErrors.push(e.message);
     }
 });
 
-async function reporterRuntime(token: string) {
-    Vencord.Webpack.waitFor(
-        "loginToken",
-        m => {
-            console.log("[PUP_DEBUG]", "Logging in with token...");
-            m.loginToken(token);
-        }
-    );
-}
-
 await page.evaluateOnNewDocument(`
     if (location.host.endsWith("discord.com")) {
         ${readFileSync("./dist/browser.js", "utf-8")};
-        (${reporterRuntime.toString()})(${JSON.stringify(process.env.DISCORD_TOKEN)});
     }
 `);
 

src/debug/runReporter.ts

@@ -7,6 +7,7 @@
 import { Logger } from "@utils/Logger";
 import * as Webpack from "@webpack";
 import { addPatch, patches } from "plugins";
+import { getBuildNumber } from "webpack/patchWebpack";
 
 import { loadLazyChunks } from "./loadLazyChunks";
 
@@ -37,6 +38,13 @@ async function runReporter() {
 
         await loadLazyChunksDone;
 
+        if (IS_REPORTER && IS_WEB && !IS_VESKTOP) {
+            console.log("[REPORTER_META]", {
+                buildNumber: getBuildNumber(),
+                buildHash: window.GLOBAL_ENV.SENTRY_TAGS.buildId
+            });
+        }
+
         for (const patch of patches) {
             if (!patch.all) {
                 new Logger("WebpackInterceptor").warn(`Patch by ${patch.plugin} found no module (Module id is -): ${patch.find}`);
@@ -44,7 +52,7 @@ async function runReporter() {
         }
 
         for (const [plugin, moduleId, match, totalTime] of Vencord.WebpackPatcher.patchTimings) {
-            if (totalTime > 3) {
+            if (totalTime > 5) {
                 new Logger("WebpackInterceptor").warn(`Patch by ${plugin} took ${Math.round(totalTime * 100) / 100}ms (Module id is ${String(moduleId)}): ${match}`);
             }
         }

src/utils/Logger.ts

@@ -32,7 +32,7 @@ export class Logger {
     constructor(public name: string, public color: string = "white") { }
 
     private _log(level: "log" | "error" | "warn" | "info" | "debug", levelColor: string, args: any[], customFmt = "") {
-        if (IS_REPORTER && IS_WEB) {
+        if (IS_REPORTER && IS_WEB && !IS_VESKTOP) {
             console[level]("[Vencord]", this.name + ":", ...args);
             return;
         }

src/webpack/patchWebpack.ts

@@ -442,11 +442,12 @@ function patchFactory(id: PropertyKey, factory: AnyModuleFactory): [patchedFacto
             continue;
         }
 
-        const buildNumber = getBuildNumber();
+        // Reporter eagerly patches and cannot retrieve the build number because this code runs before the module for it is loaded
-        const shouldCheckBuildId = !Settings.eagerPatches && buildNumber !== -1;
+        const buildNumber = IS_REPORTER ? -1 : getBuildNumber();
+        const shouldCheckBuildNumber = !Settings.eagerPatches && buildNumber !== -1;
 
         if (
-            shouldCheckBuildId &&
+            shouldCheckBuildNumber &&
             (patch.fromBuild != null && buildNumber < patch.fromBuild) ||
             (patch.toBuild != null && buildNumber > patch.toBuild)
         ) {
@@ -468,7 +469,7 @@ function patchFactory(id: PropertyKey, factory: AnyModuleFactory): [patchedFacto
         // We change all patch.replacement to array in plugins/index
         for (const replacement of patch.replacement as PatchReplacement[]) {
             if (
-                shouldCheckBuildId &&
+                shouldCheckBuildNumber &&
                 (replacement.fromBuild != null && buildNumber < replacement.fromBuild) ||
                 (replacement.toBuild != null && buildNumber > replacement.toBuild)
             ) {