CI: Run benchmark (#2563)

* fix(bench): try to fix benchmark

* style: deno fmt

* fix: remove check formating

* fix: add fake token

* fix: tranform

* deno fmt

* add parseFloat

* fix unit

* refactor: change fetch target to benchrepo

* fix: oldBot

* refactor: use custom input

* ci: add tee

* style: deno fmt

* ci: cache deps

* chore: remove

* ci: add memory benchmark

* fix: fix url for main repo

* deno fmt

* ci: add comment-always

* fix: link

* just trying trigger

* ci: only push on main

* fix: type

* fix: range

* fixed

* style: deno fmt

* fix: path

* Add upload output

* style: deno fmt

* ci: add create branch on pr owner's repo

* fix: github.repository

* ci: fix remove id benchmark-action

* fix

* fix: type

* reverse change

* ci: add cache

* feat: add using cache

* bench: update name and ignore previous when ci

* style: name

* feat: fix pr message style

* add more benchmark

* deno fmt

* fix

* fix: wording

* chore: only run on success

* fix: used last head as current commit

.github/workflows/benchmark.yml

@@ -0,0 +1,69 @@
+name: Benchmark
+
+on:
+  pull_request:
+  push:
+
+jobs:
+  benchmark:
+    name: Benchmark
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: denoland/setup-deno@main
+        with:
+          deno-version: v1.x
+
+        # this is the performance benchmark
+      - name: Cache deps
+        run: deno cache performance/mod.ts
+      - name: Run Benchmark
+        run: NO_COLOR=true deno bench --unstable -A performance/mod.ts | tee output.txt
+      - name: Format Benchmark Output
+        run: deno run -A performance/tranformOutput.ts
+
+        # this is the memory benchmark
+      - name: Download db from benchmark repo
+        run: wget https://github.com/discordeno/benchmarks/raw/main/db.tar.gz
+      - name: Decompress db
+        run: tar -xzvf db.tar.gz
+      - name: Run memory benchmark
+        run: deno run --v8-flags="--expose-gc" -A performance/memory.ts
+
+      - name: Download previous benchmark data
+        uses: actions/cache@v3
+        with:
+          path: ./benchmarksResult
+          key: ${{ github.ref }}-benchmark
+      - name: Store benchmark result to cache
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          tool: "customSmallerIsBetter"
+          output-file-path: output.txt
+          external-data-json-path: benchmarksResult/data.json
+      - uses: actions/upload-artifact@v3
+        if: ${{ github.ref != 'refs/heads/main' }}
+        with:
+          name: benchmarkResults
+          path: benchmarksResult/data.json
+
+      - name: Store benchmark result (Main)
+        uses: benchmark-action/github-action-benchmark@v1
+        if: ${{ github.ref == 'refs/heads/main' }}
+        with:
+          tool: "customSmallerIsBetter"
+          output-file-path: output.txt
+          gh-pages-branch: "benchies"
+          benchmark-data-dir-path: benchmarksResult
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          auto-push: true
+
+      - name: Save Commmit SHA
+        run: |
+          mkdir -p ./commitData
+          echo $GITHUB_SHA > ./commitData/sha
+          echo $GITHUB_REPOSITORY > ./commitData/repo
+      - uses: actions/upload-artifact@v3
+        with:
+          name: commitData
+          path: commitData/

.github/workflows/commentBenchResult.yml

@@ -0,0 +1,95 @@
+name: Comment Benchmark Result
+
+on:
+  workflow_run:
+    workflows: [Benchmark]
+    types:
+      - completed
+
+jobs:
+  comment-benchmark-result:
+    name: Comment Benchmark Result
+    runs-on: ubuntu-latest
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
+    steps:
+      - uses: actions/checkout@v3
+      - uses: denoland/setup-deno@main
+        with:
+          deno-version: v1.x
+      - name: Download Commit Data Artifact
+        uses: actions/github-script@v6
+        with:
+          script: |
+            let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               run_id: context.payload.workflow_run.id,
+            });
+            let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => {
+              return artifact.name == "commitData"
+            })[0];
+            let download = await github.rest.actions.downloadArtifact({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               artifact_id: matchArtifact.id,
+               archive_format: 'zip',
+            });
+            let fs = require('fs');
+            fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/commitData.zip`, Buffer.from(download.data));
+
+      - name: Unzip Commit Data Artifact
+        run: unzip commitData.zip
+
+      - name: Download Result Artifact
+        uses: actions/github-script@v6
+        with:
+          script: |
+            let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               run_id: context.payload.workflow_run.id,
+            });
+            let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => {
+              return artifact.name == "benchmarkResults"
+            })[0];
+            let download = await github.rest.actions.downloadArtifact({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               artifact_id: matchArtifact.id,
+               archive_format: 'zip',
+            });
+            let fs = require('fs');
+            fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/benchmarkResults.zip`, Buffer.from(download.data));
+
+      - name: Unzip Result Artifact
+        run: unzip benchmarkResults.zip
+
+      - name: Generate Message
+        id: genMessage
+        run: |
+          MESSAGE=$(deno run -A performance/generateMessage.ts)
+          echo "MESSAGE<<EOF" >> $GITHUB_ENV
+          echo "$MESSAGE" >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV
+
+      - name: "Comment on PR"
+        uses: actions/github-script@v6
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const fs = require('fs');
+            const commit_sha = fs.readFileSync('./sha', 'utf-8');
+            const repo = fs.readFileSync('./repo', 'utf-8');
+            const pr = await github.rest.repos.listPullRequestsAssociatedWithCommit({
+              commit_sha: commit_sha.slice(0,-1),
+              owner: repo.split('/')[0],
+              repo: repo.split('/')[1].slice(0,-1),
+            });
+            if (pr.data[0]) {
+              github.rest.issues.createComment({
+                issue_number: pr.data[0].number,
+                owner: repo.split('/')[0],
+                repo: repo.split('/')[1].slice(0,-1),
+                body: `${{ env.MESSAGE }}`
+              })
+            }

.gitignore

@@ -9,3 +9,9 @@ npm/
 
 # MacOS is weird
 .DS_Store
+
+# benchmark db and output
+db
+db.tar.gz
+output.txt
+benchmarksResult

performance/generateMessage.ts

@@ -0,0 +1,114 @@
+await import(`https://raw.githubusercontent.com/discordeno/discordeno/benchies/benchmarksResult/data.js`);
+const commitSha = await Deno.readTextFile("./sha");
+const results = JSON.parse(await Deno.readTextFile("./data.json"));
+
+interface BenchmarksData {
+  commit: {
+    author: { email: string; name: string; username: string };
+    committer: { email: string; name: string; username: string };
+    distinct: boolean;
+    id: string;
+    message: string;
+    timestamp: string;
+    tree_id: string;
+    url: string;
+  };
+  date: number;
+  tool: string;
+  benches: { name: string; value: number; unit: string; range: string }[];
+}
+
+interface CompareTable {
+  [index: string]: {
+    current: { name: string; value: number; unit: string; range: string } | {
+      name?: string;
+      value?: number;
+      unit?: string;
+      range?: string;
+    };
+    previous: { name: string; value: number; unit: string; range: string } | {
+      name?: string;
+      value?: number;
+      unit?: string;
+      range?: string;
+    };
+  };
+}
+
+const benchmarks = results.entries.Benchmark.slice(-2) as BenchmarksData[];
+const latestHeadBenchmarks = benchmarks[1];
+const lastHeadBenchmarks = benchmarks[0];
+//@ts-ignore
+const latestBaseBenchmarks = JSON.parse(JSON.stringify(window.BENCHMARK_DATA.entries.Benchmark)).slice(
+  -1,
+)[0] as BenchmarksData;
+
+const compareWithHead: CompareTable = {};
+const compareWithBase: CompareTable = {};
+
+for (const benchmark of lastHeadBenchmarks.benches) {
+  compareWithHead[benchmark.name] = {
+    previous: benchmark,
+    current: {},
+  };
+}
+for (const benchmark of latestBaseBenchmarks.benches) {
+  compareWithBase[benchmark.name] = {
+    previous: benchmark,
+    current: {},
+  };
+}
+for (const benchmark of latestHeadBenchmarks.benches) {
+  compareWithBase[benchmark.name] = {
+    //@ts-ignore
+    previous: {},
+    ...compareWithBase[benchmark.name],
+    current: benchmark,
+  };
+  compareWithHead[benchmark.name] = {
+    //@ts-ignore
+    previous: {},
+    ...compareWithBase[benchmark.name],
+    current: benchmark,
+  };
+}
+
+let message = "";
+
+const compareTableInfo = [{ name: "last head", commit: lastHeadBenchmarks.commit.id }, {
+  name: "base",
+  commit: latestBaseBenchmarks.commit.id,
+}];
+for (const benchmarkType of ["Performance", "Memory"]) {
+  message += `# ${benchmarkType} Benchmark\n\n`;
+  for (const [index, compare] of [compareWithHead, compareWithBase].entries()) {
+    message += `## Compared with ${compareTableInfo[index].name}\n`;
+    message += "<details><summary>Detail results of benchmarks</summary>\n\n";
+    message += `| Benchmark suite | Current: ${latestHeadBenchmarks.commit.id} | Previous: ${
+      compareTableInfo[index].commit
+    } | Ratio |\n | -| -| -| -|\n`;
+    for (
+      const field of Object.keys(compare).filter((key) =>
+        benchmarkType === "Performance" ? !key.startsWith("[Cache Plugin]") : key.startsWith("[Cache Plugin]")
+      )
+    ) {
+      message += `| \`${field}\` | ${compare[field].current.value ? `\`${compare[field].current.value}\`` : ""} ${
+        compare[field].current.unit ?? ""
+      } ${compare[field].current.range ? `(\`${compare[field].current.range ?? ""}\`)` : ""} | ${
+        compare[field].previous.value ? `\`${compare[field].previous.value}\`` : ""
+      } ${compare[field].previous.unit ?? ""} ${
+        compare[field].previous.range ? `(\`${compare[field].previous.range ?? ""}\`)` : ""
+      } | ${
+        compare[field].previous.value && compare[field].current.value
+          ? `\`${
+            //@ts-ignore
+            Math.round((parseFloat(compare[field].previous.value) / parseFloat(compare[field].current.value)) * 100) /
+            100}\``
+          : ""
+      } |\n`;
+    }
+    message += "</details>\n\n";
+  }
+}
+
+console.log(message.replaceAll("`", "\\`"));

performance/memory.ts

@@ -0,0 +1,28 @@
+import { memoryBenchmarks } from "https://raw.githubusercontent.com/discordeno/benchmarks/main/index.ts";
+import { createBot } from "../mod.ts";
+import { enableCachePlugin } from "../plugins/mod.ts";
+
+const results = await memoryBenchmarks(() =>
+  enableCachePlugin(createBot({
+    token: " ",
+    botId: 0n,
+  }))
+);
+
+const output: {
+  name: string;
+  value: number;
+  range: string;
+  unit: string;
+}[] = JSON.parse(await Deno.readTextFile("output.txt"));
+
+for (const resultKey of Object.keys(results.Cached) as (keyof typeof results.Cached)[]) {
+  output.push({
+    name: `[Cache Plugin] ${resultKey.toString()}`,
+    value: results.Cached[resultKey].value,
+    range: `${results.Cached[resultKey].min} … ${results.Cached[resultKey].max}`,
+    unit: "MB",
+  });
+}
+
+Deno.writeTextFile("output.txt", JSON.stringify(output, undefined, 2));

performance/mod.ts

@@ -1,42 +1,97 @@
 import { loadBot as oldLoadBot } from "https://raw.githubusercontent.com/discordeno/discordeno/main/tests/mod.ts";
 import { loadBot } from "../tests/mod.ts";
-import { CACHED_COMMUNITY_GUILD_ID } from "../tests/utils.ts";
-import { DiscordGuild, DiscordUser } from "../types/discord.ts";
 
+Deno.env.set("DISCORD_TOKEN", `${btoa("316179474163171338")}.gbaodiwabn`);
 const bot = loadBot();
 const oldBot = oldLoadBot();
-const discordGuild = await bot.rest.runMethod<DiscordGuild>(
-  bot.rest,
-  "GET",
-  bot.constants.routes.GUILD(CACHED_COMMUNITY_GUILD_ID, true),
-);
-const currentGuild = bot.transformers.guild(bot, { guild: discordGuild, shardId: 0 });
-const previousGuild = oldBot.transformers.guild(bot, { guild: discordGuild, shardId: 0 });
 
-Deno.bench("[Guild.toggles.features - Current] Get the features of a guild", () => {
+// Fetch the cached guild
+const discordGuild = JSON.parse(
+  await (await fetch("https://raw.githubusercontent.com/discordeno/discordeno/benchies/cache/cachedObject/guild.json"))
+    .text(),
+);
+
+const currentGuild = bot.transformers.guild(bot, { guild: discordGuild, shardId: 0 });
+const previousGuild = oldBot.transformers.guild(oldBot, { guild: discordGuild, shardId: 0 });
+
+Deno.bench("[Guild.toggles.features] Get the features of a guild", () => {
   currentGuild.toggles.features;
 });
 
-Deno.bench("[Guild.toggles.features - Previous] Get the features of a guild", () => {
+Deno.bench("[Guild.toggles.features - Previous] Get the features of a guild", {
+  ignore: Deno.env.get("CI") === "true",
+}, () => {
   previousGuild.toggles.features;
 });
 
-const discordUser = await bot.rest.runMethod<DiscordUser>(bot.rest, "GET", bot.constants.routes.USER(bot.id));
+// Fetch the cached user
+const discordUser = JSON.parse(
+  await (await fetch("https://raw.githubusercontent.com/discordeno/discordeno/benchies/cache/cachedObject/user.json"))
+    .text(),
+);
+
 const newUser = bot.transformers.user(bot, discordUser);
 const oldUser = oldBot.transformers.user(oldBot, discordUser);
 
-Deno.bench("[Transformer - Current] Discord User to a User", () => {
+Deno.bench("[Transformer] Discord User to a User", () => {
   bot.transformers.user(bot, discordUser);
 });
 
-Deno.bench("[Transformer - Previous] Discord User to a User", () => {
+Deno.bench("[Transformer - Previous] Discord User to a User", {
+  ignore: Deno.env.get("CI") === "true",
+}, () => {
   oldBot.transformers.user(oldBot, discordUser);
 });
 
-Deno.bench("[Transformer - Current] User to a Discord User", () => {
+Deno.bench("[Transformer] User to a Discord User", () => {
   bot.transformers.reverse.user(bot, newUser);
 });
 
-Deno.bench("[Transformer - Previous] User to a Discord User", () => {
+Deno.bench("[Transformer - Previous] User to a Discord User", {
+  ignore: Deno.env.get("CI") === "true",
+}, () => {
   oldBot.transformers.reverse.user(oldBot, oldUser);
 });
+
+for (
+  const channelType of [
+    "rules",
+    "announcement-channel",
+    "moderator-channel",
+    "text-channel",
+    "stage-channel",
+    "voice-channel",
+  ]
+) {
+  const discordChannel = JSON.parse(
+    await (await fetch(
+      `https://raw.githubusercontent.com/discordeno/discordeno/benchies/cache/cachedObject/${channelType}.json`,
+    )).text(),
+  );
+  const formattedChannelType = channelType.split("-").map((word) => word[0].toUpperCase() + word.slice(1)).join(" ");
+
+  Deno.bench(`[Transformer] Discord ${formattedChannelType} to a ${formattedChannelType}`, () => {
+    bot.transformers.channel(bot, { channel: discordChannel });
+  });
+
+  Deno.bench(`[Transformer - Previous] Discord ${formattedChannelType} to a ${formattedChannelType}`, {
+    ignore: Deno.env.get("CI") === "true",
+  }, () => {
+    oldBot.transformers.channel(oldBot, { channel: discordChannel });
+  });
+
+  /* Not implemented
+  const newChannel = bot.transformers.channel(bot, { channel: discordChannel });
+  const oldChannel = oldBot.transformers.channel(oldBot, { channel: discordChannel });
+
+  Deno.bench(`[Transformer] ${formattedChannelType} to a Discord ${formattedChannelType}`, () => {
+    bot.transformers.reverse.channel(bot, newChannel);
+  });
+
+  Deno.bench(`[Transformer - Previous] ${formattedChannelType} to a Discord ${formattedChannelType}`, {
+    ignore: Deno.env.get("CI") === "true",
+  }, () => {
+    oldBot.transformers.reverse.channel(oldBot, oldChannel);
+  });
+  */
+}

performance/tranformOutput.ts

@@ -0,0 +1,27 @@
+const output = await Deno.readTextFile("output.txt");
+const lines = output.split(/\r?\n/g);
+
+const ret = [];
+
+const unitMultiplier = {
+  "s": 1000 * 1000 * 1000,
+  "ms": 1000 * 1000,
+  "µs": 1000,
+  "ns": 1,
+};
+
+for (const line of lines) {
+  const m = line.match(/^(.+)\s+([0-9.]+) (.s)\/iter\s+\((.+) (.s) … (.+) (.s)\)(.+)$/);
+  if (m === null) continue;
+
+  ret.push({
+    name: m[1].trim(),
+    value: Math.round(parseFloat(m[2]) * unitMultiplier[m[3] as keyof typeof unitMultiplier]),
+    range: `${Math.round(parseFloat(m[4]) * unitMultiplier[m[5] as keyof typeof unitMultiplier] * 100) / 100} … ${
+      Math.round(parseFloat(m[6]) * unitMultiplier[m[7] as keyof typeof unitMultiplier] * 100) / 100
+    }`,
+    unit: "ns/iter",
+  });
+}
+
+await Deno.writeTextFile("output.txt", JSON.stringify(ret, undefined, 2));

site/docs/amethyst/exampleBot.md

@@ -17,9 +17,16 @@ npm i @thereallonewolf/amethystframework
 - **Step 4**: Add following code in index.ts file, replacing TOKEN with your bot token.
 
 ```ts
-import {createBot, GatewayIntents, startBot } from "discordeno";
+import { createBot, GatewayIntents, startBot } from "discordeno";
 import { enableCachePlugin, enableCacheSweepers } from "discordeno/cache-plugin";
-import { AmethystBot, Context, enableAmethystPlugin, Event,Category, Command } from "@thereallonewolf/amethystframework";
+import {
+  AmethystBot,
+  Category,
+  Command,
+  Context,
+  enableAmethystPlugin,
+  Event,
+} from "@thereallonewolf/amethystframework";
 
 let baseClient = createBot({
   token: "TOKEN",
@@ -37,7 +44,7 @@ enableCacheSweepers(client);
 startBot(client);
 
 @Category({
-  name:"general",
+  name: "general",
   description: "My general commands",
   uniqueCommands: true,
   default: "", //As all the commands are unique so no need to set the default command.