diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 000000000..595785245 --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,69 @@ +name: Benchmark + +on: + pull_request: + push: + +jobs: + benchmark: + name: Benchmark + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: denoland/setup-deno@main + with: + deno-version: v1.x + + # this is the performance benchmark + - name: Cache deps + run: deno cache performance/mod.ts + - name: Run Benchmark + run: NO_COLOR=true deno bench --unstable -A performance/mod.ts | tee output.txt + - name: Format Benchmark Output + run: deno run -A performance/tranformOutput.ts + + # this is the memory benchmark + - name: Download db from benchmark repo + run: wget https://github.com/discordeno/benchmarks/raw/main/db.tar.gz + - name: Decompress db + run: tar -xzvf db.tar.gz + - name: Run memory benchmark + run: deno run --v8-flags="--expose-gc" -A performance/memory.ts + + - name: Download previous benchmark data + uses: actions/cache@v3 + with: + path: ./benchmarksResult + key: ${{ github.ref }}-benchmark + - name: Store benchmark result to cache + uses: benchmark-action/github-action-benchmark@v1 + with: + tool: "customSmallerIsBetter" + output-file-path: output.txt + external-data-json-path: benchmarksResult/data.json + - uses: actions/upload-artifact@v3 + if: ${{ github.ref != 'refs/heads/main' }} + with: + name: benchmarkResults + path: benchmarksResult/data.json + + - name: Store benchmark result (Main) + uses: benchmark-action/github-action-benchmark@v1 + if: ${{ github.ref == 'refs/heads/main' }} + with: + tool: "customSmallerIsBetter" + output-file-path: output.txt + gh-pages-branch: "benchies" + benchmark-data-dir-path: benchmarksResult + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: true + + - name: Save Commmit SHA + run: | + mkdir -p ./commitData + echo $GITHUB_SHA > ./commitData/sha + echo $GITHUB_REPOSITORY > ./commitData/repo + - uses: actions/upload-artifact@v3 + with: + name: commitData + path: commitData/ diff --git a/.github/workflows/commentBenchResult.yml b/.github/workflows/commentBenchResult.yml new file mode 100644 index 000000000..1c2c26297 --- /dev/null +++ b/.github/workflows/commentBenchResult.yml @@ -0,0 +1,95 @@ +name: Comment Benchmark Result + +on: + workflow_run: + workflows: [Benchmark] + types: + - completed + +jobs: + comment-benchmark-result: + name: Comment Benchmark Result + runs-on: ubuntu-latest + if: ${{ github.event.workflow_run.conclusion == 'success' }} + steps: + - uses: actions/checkout@v3 + - uses: denoland/setup-deno@main + with: + deno-version: v1.x + - name: Download Commit Data Artifact + uses: actions/github-script@v6 + with: + script: | + let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.payload.workflow_run.id, + }); + let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => { + return artifact.name == "commitData" + })[0]; + let download = await github.rest.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: matchArtifact.id, + archive_format: 'zip', + }); + let fs = require('fs'); + fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/commitData.zip`, Buffer.from(download.data)); + + - name: Unzip Commit Data Artifact + run: unzip commitData.zip + + - name: Download Result Artifact + uses: actions/github-script@v6 + with: + script: | + let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.payload.workflow_run.id, + }); + let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => { + return artifact.name == "benchmarkResults" + })[0]; + let download = await github.rest.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: matchArtifact.id, + archive_format: 'zip', + }); + let fs = require('fs'); + fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/benchmarkResults.zip`, Buffer.from(download.data)); + + - name: Unzip Result Artifact + run: unzip benchmarkResults.zip + + - name: Generate Message + id: genMessage + run: | + MESSAGE=$(deno run -A performance/generateMessage.ts) + echo "MESSAGE<> $GITHUB_ENV + echo "$MESSAGE" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + + - name: "Comment on PR" + uses: actions/github-script@v6 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const fs = require('fs'); + const commit_sha = fs.readFileSync('./sha', 'utf-8'); + const repo = fs.readFileSync('./repo', 'utf-8'); + const pr = await github.rest.repos.listPullRequestsAssociatedWithCommit({ + commit_sha: commit_sha.slice(0,-1), + owner: repo.split('/')[0], + repo: repo.split('/')[1].slice(0,-1), + }); + if (pr.data[0]) { + github.rest.issues.createComment({ + issue_number: pr.data[0].number, + owner: repo.split('/')[0], + repo: repo.split('/')[1].slice(0,-1), + body: `${{ env.MESSAGE }}` + }) + } diff --git a/.gitignore b/.gitignore index b9e94e73e..08358bac7 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,9 @@ npm/ # MacOS is weird .DS_Store + +# benchmark db and output +db +db.tar.gz +output.txt +benchmarksResult \ No newline at end of file diff --git a/performance/generateMessage.ts b/performance/generateMessage.ts new file mode 100644 index 000000000..005b818f5 --- /dev/null +++ b/performance/generateMessage.ts @@ -0,0 +1,114 @@ +await import(`https://raw.githubusercontent.com/discordeno/discordeno/benchies/benchmarksResult/data.js`); +const commitSha = await Deno.readTextFile("./sha"); +const results = JSON.parse(await Deno.readTextFile("./data.json")); + +interface BenchmarksData { + commit: { + author: { email: string; name: string; username: string }; + committer: { email: string; name: string; username: string }; + distinct: boolean; + id: string; + message: string; + timestamp: string; + tree_id: string; + url: string; + }; + date: number; + tool: string; + benches: { name: string; value: number; unit: string; range: string }[]; +} + +interface CompareTable { + [index: string]: { + current: { name: string; value: number; unit: string; range: string } | { + name?: string; + value?: number; + unit?: string; + range?: string; + }; + previous: { name: string; value: number; unit: string; range: string } | { + name?: string; + value?: number; + unit?: string; + range?: string; + }; + }; +} + +const benchmarks = results.entries.Benchmark.slice(-2) as BenchmarksData[]; +const latestHeadBenchmarks = benchmarks[1]; +const lastHeadBenchmarks = benchmarks[0]; +//@ts-ignore +const latestBaseBenchmarks = JSON.parse(JSON.stringify(window.BENCHMARK_DATA.entries.Benchmark)).slice( + -1, +)[0] as BenchmarksData; + +const compareWithHead: CompareTable = {}; +const compareWithBase: CompareTable = {}; + +for (const benchmark of lastHeadBenchmarks.benches) { + compareWithHead[benchmark.name] = { + previous: benchmark, + current: {}, + }; +} +for (const benchmark of latestBaseBenchmarks.benches) { + compareWithBase[benchmark.name] = { + previous: benchmark, + current: {}, + }; +} +for (const benchmark of latestHeadBenchmarks.benches) { + compareWithBase[benchmark.name] = { + //@ts-ignore + previous: {}, + ...compareWithBase[benchmark.name], + current: benchmark, + }; + compareWithHead[benchmark.name] = { + //@ts-ignore + previous: {}, + ...compareWithBase[benchmark.name], + current: benchmark, + }; +} + +let message = ""; + +const compareTableInfo = [{ name: "last head", commit: lastHeadBenchmarks.commit.id }, { + name: "base", + commit: latestBaseBenchmarks.commit.id, +}]; +for (const benchmarkType of ["Performance", "Memory"]) { + message += `# ${benchmarkType} Benchmark\n\n`; + for (const [index, compare] of [compareWithHead, compareWithBase].entries()) { + message += `## Compared with ${compareTableInfo[index].name}\n`; + message += "
Detail results of benchmarks\n\n"; + message += `| Benchmark suite | Current: ${latestHeadBenchmarks.commit.id} | Previous: ${ + compareTableInfo[index].commit + } | Ratio |\n | -| -| -| -|\n`; + for ( + const field of Object.keys(compare).filter((key) => + benchmarkType === "Performance" ? !key.startsWith("[Cache Plugin]") : key.startsWith("[Cache Plugin]") + ) + ) { + message += `| \`${field}\` | ${compare[field].current.value ? `\`${compare[field].current.value}\`` : ""} ${ + compare[field].current.unit ?? "" + } ${compare[field].current.range ? `(\`${compare[field].current.range ?? ""}\`)` : ""} | ${ + compare[field].previous.value ? `\`${compare[field].previous.value}\`` : "" + } ${compare[field].previous.unit ?? ""} ${ + compare[field].previous.range ? `(\`${compare[field].previous.range ?? ""}\`)` : "" + } | ${ + compare[field].previous.value && compare[field].current.value + ? `\`${ + //@ts-ignore + Math.round((parseFloat(compare[field].previous.value) / parseFloat(compare[field].current.value)) * 100) / + 100}\`` + : "" + } |\n`; + } + message += "
\n\n"; + } +} + +console.log(message.replaceAll("`", "\\`")); diff --git a/performance/memory.ts b/performance/memory.ts new file mode 100644 index 000000000..a09727e75 --- /dev/null +++ b/performance/memory.ts @@ -0,0 +1,28 @@ +import { memoryBenchmarks } from "https://raw.githubusercontent.com/discordeno/benchmarks/main/index.ts"; +import { createBot } from "../mod.ts"; +import { enableCachePlugin } from "../plugins/mod.ts"; + +const results = await memoryBenchmarks(() => + enableCachePlugin(createBot({ + token: " ", + botId: 0n, + })) +); + +const output: { + name: string; + value: number; + range: string; + unit: string; +}[] = JSON.parse(await Deno.readTextFile("output.txt")); + +for (const resultKey of Object.keys(results.Cached) as (keyof typeof results.Cached)[]) { + output.push({ + name: `[Cache Plugin] ${resultKey.toString()}`, + value: results.Cached[resultKey].value, + range: `${results.Cached[resultKey].min} … ${results.Cached[resultKey].max}`, + unit: "MB", + }); +} + +Deno.writeTextFile("output.txt", JSON.stringify(output, undefined, 2)); diff --git a/performance/mod.ts b/performance/mod.ts index 3400e640a..1ec5bceeb 100644 --- a/performance/mod.ts +++ b/performance/mod.ts @@ -1,42 +1,97 @@ import { loadBot as oldLoadBot } from "https://raw.githubusercontent.com/discordeno/discordeno/main/tests/mod.ts"; import { loadBot } from "../tests/mod.ts"; -import { CACHED_COMMUNITY_GUILD_ID } from "../tests/utils.ts"; -import { DiscordGuild, DiscordUser } from "../types/discord.ts"; +Deno.env.set("DISCORD_TOKEN", `${btoa("316179474163171338")}.gbaodiwabn`); const bot = loadBot(); const oldBot = oldLoadBot(); -const discordGuild = await bot.rest.runMethod( - bot.rest, - "GET", - bot.constants.routes.GUILD(CACHED_COMMUNITY_GUILD_ID, true), -); -const currentGuild = bot.transformers.guild(bot, { guild: discordGuild, shardId: 0 }); -const previousGuild = oldBot.transformers.guild(bot, { guild: discordGuild, shardId: 0 }); -Deno.bench("[Guild.toggles.features - Current] Get the features of a guild", () => { +// Fetch the cached guild +const discordGuild = JSON.parse( + await (await fetch("https://raw.githubusercontent.com/discordeno/discordeno/benchies/cache/cachedObject/guild.json")) + .text(), +); + +const currentGuild = bot.transformers.guild(bot, { guild: discordGuild, shardId: 0 }); +const previousGuild = oldBot.transformers.guild(oldBot, { guild: discordGuild, shardId: 0 }); + +Deno.bench("[Guild.toggles.features] Get the features of a guild", () => { currentGuild.toggles.features; }); -Deno.bench("[Guild.toggles.features - Previous] Get the features of a guild", () => { +Deno.bench("[Guild.toggles.features - Previous] Get the features of a guild", { + ignore: Deno.env.get("CI") === "true", +}, () => { previousGuild.toggles.features; }); -const discordUser = await bot.rest.runMethod(bot.rest, "GET", bot.constants.routes.USER(bot.id)); +// Fetch the cached user +const discordUser = JSON.parse( + await (await fetch("https://raw.githubusercontent.com/discordeno/discordeno/benchies/cache/cachedObject/user.json")) + .text(), +); + const newUser = bot.transformers.user(bot, discordUser); const oldUser = oldBot.transformers.user(oldBot, discordUser); -Deno.bench("[Transformer - Current] Discord User to a User", () => { +Deno.bench("[Transformer] Discord User to a User", () => { bot.transformers.user(bot, discordUser); }); -Deno.bench("[Transformer - Previous] Discord User to a User", () => { +Deno.bench("[Transformer - Previous] Discord User to a User", { + ignore: Deno.env.get("CI") === "true", +}, () => { oldBot.transformers.user(oldBot, discordUser); }); -Deno.bench("[Transformer - Current] User to a Discord User", () => { +Deno.bench("[Transformer] User to a Discord User", () => { bot.transformers.reverse.user(bot, newUser); }); -Deno.bench("[Transformer - Previous] User to a Discord User", () => { +Deno.bench("[Transformer - Previous] User to a Discord User", { + ignore: Deno.env.get("CI") === "true", +}, () => { oldBot.transformers.reverse.user(oldBot, oldUser); }); + +for ( + const channelType of [ + "rules", + "announcement-channel", + "moderator-channel", + "text-channel", + "stage-channel", + "voice-channel", + ] +) { + const discordChannel = JSON.parse( + await (await fetch( + `https://raw.githubusercontent.com/discordeno/discordeno/benchies/cache/cachedObject/${channelType}.json`, + )).text(), + ); + const formattedChannelType = channelType.split("-").map((word) => word[0].toUpperCase() + word.slice(1)).join(" "); + + Deno.bench(`[Transformer] Discord ${formattedChannelType} to a ${formattedChannelType}`, () => { + bot.transformers.channel(bot, { channel: discordChannel }); + }); + + Deno.bench(`[Transformer - Previous] Discord ${formattedChannelType} to a ${formattedChannelType}`, { + ignore: Deno.env.get("CI") === "true", + }, () => { + oldBot.transformers.channel(oldBot, { channel: discordChannel }); + }); + + /* Not implemented + const newChannel = bot.transformers.channel(bot, { channel: discordChannel }); + const oldChannel = oldBot.transformers.channel(oldBot, { channel: discordChannel }); + + Deno.bench(`[Transformer] ${formattedChannelType} to a Discord ${formattedChannelType}`, () => { + bot.transformers.reverse.channel(bot, newChannel); + }); + + Deno.bench(`[Transformer - Previous] ${formattedChannelType} to a Discord ${formattedChannelType}`, { + ignore: Deno.env.get("CI") === "true", + }, () => { + oldBot.transformers.reverse.channel(oldBot, oldChannel); + }); + */ +} diff --git a/performance/tranformOutput.ts b/performance/tranformOutput.ts new file mode 100644 index 000000000..c0a96a8c7 --- /dev/null +++ b/performance/tranformOutput.ts @@ -0,0 +1,27 @@ +const output = await Deno.readTextFile("output.txt"); +const lines = output.split(/\r?\n/g); + +const ret = []; + +const unitMultiplier = { + "s": 1000 * 1000 * 1000, + "ms": 1000 * 1000, + "µs": 1000, + "ns": 1, +}; + +for (const line of lines) { + const m = line.match(/^(.+)\s+([0-9.]+) (.s)\/iter\s+\((.+) (.s) … (.+) (.s)\)(.+)$/); + if (m === null) continue; + + ret.push({ + name: m[1].trim(), + value: Math.round(parseFloat(m[2]) * unitMultiplier[m[3] as keyof typeof unitMultiplier]), + range: `${Math.round(parseFloat(m[4]) * unitMultiplier[m[5] as keyof typeof unitMultiplier] * 100) / 100} … ${ + Math.round(parseFloat(m[6]) * unitMultiplier[m[7] as keyof typeof unitMultiplier] * 100) / 100 + }`, + unit: "ns/iter", + }); +} + +await Deno.writeTextFile("output.txt", JSON.stringify(ret, undefined, 2)); diff --git a/site/docs/amethyst/exampleBot.md b/site/docs/amethyst/exampleBot.md index a67b99f11..8dd89659e 100644 --- a/site/docs/amethyst/exampleBot.md +++ b/site/docs/amethyst/exampleBot.md @@ -17,9 +17,16 @@ npm i @thereallonewolf/amethystframework - **Step 4**: Add following code in index.ts file, replacing TOKEN with your bot token. ```ts -import {createBot, GatewayIntents, startBot } from "discordeno"; +import { createBot, GatewayIntents, startBot } from "discordeno"; import { enableCachePlugin, enableCacheSweepers } from "discordeno/cache-plugin"; -import { AmethystBot, Context, enableAmethystPlugin, Event,Category, Command } from "@thereallonewolf/amethystframework"; +import { + AmethystBot, + Category, + Command, + Context, + enableAmethystPlugin, + Event, +} from "@thereallonewolf/amethystframework"; let baseClient = createBot({ token: "TOKEN", @@ -37,7 +44,7 @@ enableCacheSweepers(client); startBot(client); @Category({ - name:"general", + name: "general", description: "My general commands", uniqueCommands: true, default: "", //As all the commands are unique so no need to set the default command.