feat(bench): commit benchmark benchies (#2918)

* refactor(bench): better readdir

* Update retryBenchmark.yml

* Update benchmark.yml

* Update benchmark.yml

* Update benchmark.yml

* Update benchmark.yml

* ci: comment the result

* ci: fix missing data

.github/workflows/benchmark.yml

@@ -2,6 +2,13 @@ name: Benchmark
 
 on:
   workflow_call:
+    inputs:
+      sha:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
     outputs:
       cpuMatch:
         value: ${{ jobs.benchmark.outputs.cpuMatch }}
@@ -48,4 +55,46 @@ jobs:
 
       - name: Benchmark
         if: ${{ steps.cpuCheck.outputs.match == 'true' }}
-        run: node --expose-gc ./packages/benchmark/dist/index.js
+        run: node --expose-gc ./packages/benchmark/dist/index.js | tee output.txt
+        
+      - name: Download previous benchmark data
+        if: ${{ steps.cpuCheck.outputs.match == 'true' }}
+        uses: actions/cache@v3
+        with:
+          path: ./benchmarksResult
+          key: ${{ github.ref }}-benchmark
+      - name: Store benchmark result to cache
+        if: ${{ steps.cpuCheck.outputs.match == 'true' }}
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          tool: "benchmarkjs"
+          output-file-path: output.txt
+          external-data-json-path: benchmarksResult/data.json
+      - uses: actions/upload-artifact@v3
+        if: ${{ steps.cpuCheck.outputs.match == 'true' }}
+        with:
+          name: benchmarkResults
+          path: benchmarksResult/data.json
+
+      - name: Store benchmark result (Main)
+        uses: benchmark-action/github-action-benchmark@v1
+        if: ${{ github.ref == 'refs/heads/main' && steps.cpuCheck.outputs.match == 'true' }}
+        with:
+          tool: "benchmarkjs"
+          output-file-path: output.txt
+          gh-pages-branch: "benchies"
+          benchmark-data-dir-path: benchmarksResult
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          auto-push: true
+
+      - name: Save Commmit SHA
+        if: ${{ steps.cpuCheck.outputs.match == 'true' }}
+        run: |
+          mkdir -p ./commitData
+          echo ${{ inputs.sha }} > ./commitData/sha
+          echo ${{ inputs.repo }} > ./commitData/repo
+      - uses: actions/upload-artifact@v3
+        if: ${{ steps.cpuCheck.outputs.match == 'true' }}
+        with:
+          name: commitData
+          path: commitData/

.github/workflows/commentBenchResult.yml

@@ -0,0 +1,111 @@
+name: Comment Benchmark Result
+
+on:
+  workflow_run:
+    workflows: [Benchmark with retry]
+    types:
+      - completed
+
+jobs:
+  comment-benchmark-result:
+    name: Comment Benchmark Result
+    runs-on: ubuntu-latest
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
+    steps:
+      - uses: actions/checkout@v3
+      - uses: denoland/setup-deno@main
+        with:
+          deno-version: v1.x
+      - name: Download Commit Data Artifact
+        uses: actions/github-script@v6
+        with:
+          script: |
+            let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               run_id: context.payload.workflow_run.id,
+            });
+            let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => {
+              return artifact.name == "commitData"
+            })[0];
+            let download = await github.rest.actions.downloadArtifact({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               artifact_id: matchArtifact.id,
+               archive_format: 'zip',
+            });
+            let fs = require('fs');
+            fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/commitData.zip`, Buffer.from(download.data));
+
+      - name: Unzip Commit Data Artifact
+        run: unzip commitData.zip
+
+      - name: Download Result Artifact
+        uses: actions/github-script@v6
+        with:
+          script: |
+            let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               run_id: context.payload.workflow_run.id,
+            });
+            let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => {
+              return artifact.name == "benchmarkResults"
+            })[0];
+            let download = await github.rest.actions.downloadArtifact({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               artifact_id: matchArtifact.id,
+               archive_format: 'zip',
+            });
+            let fs = require('fs');
+            fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/benchmarkResults.zip`, Buffer.from(download.data));
+
+      - name: Unzip Result Artifact
+        run: unzip benchmarkResults.zip
+
+      - name: Generate Message
+        id: genMessage
+        run: |
+          MESSAGE=$(deno run -A performance/generateMessage.ts)
+          echo "MESSAGE<<EOF" >> $GITHUB_ENV
+          echo "$MESSAGE" >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV
+
+      - name: "Comment on PR"
+        uses: actions/github-script@v6
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const fs = require('fs');
+            const commit_sha = fs.readFileSync('./sha', 'utf-8');
+            const repo = fs.readFileSync('./repo', 'utf-8');
+            if (repo.split('/')[1] === undefined) process.exit(0)
+            const pr = await github.rest.repos.listPullRequestsAssociatedWithCommit({
+              commit_sha: commit_sha.slice(0,-1),
+              owner: repo.split('/')[0],
+              repo: repo.split('/')[1].slice(0,-1),
+            });
+            if (pr.data[0]) {
+              const comments = await github.rest.issues.listComments({
+                owner: "discordeno",
+                repo: "discordeno",
+                issue_number: pr.data[0].number,
+              })
+              const oldComment = comments.data.find((comment) => comment.body.includes('benchmark comment by ci'))
+              if(!oldComment) {
+                github.rest.issues.createComment({
+                  issue_number: pr.data[0].number,
+                  owner: "discordeno",
+                  repo: "discordeno",
+                  body: `${{ env.MESSAGE }}`
+                })
+              } else {
+                github.rest.issues.updateComment({
+                  owner: "discordeno",
+                  repo: "discordeno",
+                  comment_id: oldComment.id,
+                  body: `${{ env.MESSAGE }}`
+                });
+              }
+            }

.github/workflows/retryBenchmark.yml

@@ -9,48 +9,78 @@ on:
 jobs:
   benchmark-try-1:
     uses: ./.github/workflows/benchmark.yml
+    with:
+      sha: ${{ github.event.pull_request.head.sha }}
+      repo: ${{ github.event.pull_request.head.repo.full_name }}
 
   benchmark-try-2:
     needs: benchmark-try-1
     if: ${{ needs.benchmark-try-1.outputs.cpuMatch == 'false' }}
     uses: ./.github/workflows/benchmark.yml
+    with:
+      sha: ${{ github.event.pull_request.head.sha }}
+      repo: ${{ github.event.pull_request.head.repo.full_name }}
 
   benchmark-try-3:
     needs: benchmark-try-2
     if: ${{ needs.benchmark-try-2.outputs.cpuMatch == 'false' }}
     uses: ./.github/workflows/benchmark.yml
+    with:
+      sha: ${{ github.event.pull_request.head.sha }}
+      repo: ${{ github.event.pull_request.head.repo.full_name }}
 
   benchmark-try-4:
     needs: benchmark-try-3
     if: ${{ needs.benchmark-try-3.outputs.cpuMatch == 'false' }}
     uses: ./.github/workflows/benchmark.yml
+    with:
+      sha: ${{ github.event.pull_request.head.sha }}
+      repo: ${{ github.event.pull_request.head.repo.full_name }}
 
   benchmark-try-5:
     needs: benchmark-try-4
     if: ${{ needs.benchmark-try-4.outputs.cpuMatch == 'false' }}
     uses: ./.github/workflows/benchmark.yml
+    with:
+      sha: ${{ github.event.pull_request.head.sha }}
+      repo: ${{ github.event.pull_request.head.repo.full_name }}
 
   benchmark-try-6:
     needs: benchmark-try-5
     if: ${{ needs.benchmark-try-5.outputs.cpuMatch == 'false' }}
     uses: ./.github/workflows/benchmark.yml
+    with:
+      sha: ${{ github.event.pull_request.head.sha }}
+      repo: ${{ github.event.pull_request.head.repo.full_name }}
 
   benchmark-try-7:
     needs: benchmark-try-6
     if: ${{ needs.benchmark-try-6.outputs.cpuMatch == 'false' }}
     uses: ./.github/workflows/benchmark.yml
+    with:
+      sha: ${{ github.event.pull_request.head.sha }}
+      repo: ${{ github.event.pull_request.head.repo.full_name }}
 
   benchmark-try-8:
     needs: benchmark-try-7
     if: ${{ needs.benchmark-try-7.outputs.cpuMatch == 'false' }}
     uses: ./.github/workflows/benchmark.yml
+    with:
+      sha: ${{ github.event.pull_request.head.sha }}
+      repo: ${{ github.event.pull_request.head.repo.full_name }}
 
   benchmark-try-9:
     needs: benchmark-try-8
     if: ${{ needs.benchmark-try-8.outputs.cpuMatch == 'false' }}
     uses: ./.github/workflows/benchmark.yml
+    with:
+      sha: ${{ github.event.pull_request.head.sha }}
+      repo: ${{ github.event.pull_request.head.repo.full_name }}
 
   benchmark-try-10:
     needs: benchmark-try-9
     if: ${{ needs.benchmark-try-9.outputs.cpuMatch == 'false' }}
     uses: ./.github/workflows/benchmark.yml
+    with:
+      sha: ${{ github.event.pull_request.head.sha }}
+      repo: ${{ github.event.pull_request.head.repo.full_name }}

packages/benchmark/package.json

@@ -15,6 +15,7 @@
     "fmt": "eslint --fix \"src/**/*.ts*\"",
     "lint": "eslint \"src/**/*.ts*\"",
     "build": "swc src --delete-dir-on-start --out-dir dist && node ../../scripts/fixBenchExtension.js",
+    "build-message": "swc src/utils/generateMessage.ts --out-dir ../../scripts && node ../../scripts/fixBenchExtension.js",
     "bench": "node dist/index.js"
   },
   "dependencies": {
@@ -38,4 +39,4 @@
     "tsconfig": "*",
     "typescript": "^4.9.3"
   }
-}
+}

packages/benchmark/src/index.ts

@@ -1,7 +1,7 @@
 import fs from 'node:fs/promises'
 import { suite } from './benchmarkSuite.js'
 
-const benchmarks = await fs.readdir('packages/benchmark/dist/benchmarks').then((files) => files.filter((file) => file.endsWith('.js')))
+const benchmarks = await fs.readdir(new URL('./benchmarks', import.meta.url)).then((files) => files.filter((file) => file.endsWith('.js')))
 
 await Promise.all(benchmarks.map(async (file) => await import(`./benchmarks/${file}`)))
 

packages/benchmark/src/utils/generateMessage.ts

@@ -0,0 +1,121 @@
+import fs from 'fs/promises'
+
+const benchmarkData = await fetch(`https://raw.githubusercontent.com/discordeno/discordeno/benchies/benchmarksResult/data.js`)
+  .then(async (res) => await res.text())
+  .then((text) => JSON.parse(text.slice(24)))
+
+// eslint-disable-next-line @typescript-eslint/no-unused-vars, no-unused-vars
+const commitSha = await fs.readFile('./sha', 'utf-8')
+const results = JSON.parse(await fs.readFile('./data.json', 'utf-8'))
+
+interface BenchmarksData {
+  commit: {
+    author: { email: string; name: string; username: string }
+    committer: { email: string; name: string; username: string }
+    distinct: boolean
+    id: string
+    message: string
+    timestamp: string
+    tree_id: string
+    url: string
+  }
+  date: number
+  tool: string
+  benches: Array<{ name: string; value: number; unit: string; range: string }>
+}
+
+type CompareTable = Record<
+  string,
+  {
+    current:
+      | { name: string; value: number; unit: string; range: string }
+      | {
+          name?: string
+          value?: number
+          unit?: string
+          range?: string
+        }
+    previous:
+      | { name: string; value: number; unit: string; range: string }
+      | {
+          name?: string
+          value?: number
+          unit?: string
+          range?: string
+        }
+  }
+>
+
+const benchmarks = results.entries.Benchmark.slice(-2) as BenchmarksData[]
+const latestHeadBenchmarks = benchmarks.length === 2 ? benchmarks[1] : benchmarks[0]
+const lastHeadBenchmarks = benchmarks.length === 2 ? benchmarks[0] : undefined
+const latestBaseBenchmarks = JSON.parse(JSON.stringify(benchmarkData.entries.Benchmark)).slice(-1)[0] as BenchmarksData
+
+const compareWithHead: CompareTable = {}
+const compareWithBase: CompareTable = {}
+
+if (lastHeadBenchmarks) {
+  for (const benchmark of lastHeadBenchmarks.benches) {
+    compareWithHead[benchmark.name] = {
+      previous: benchmark,
+      current: {},
+    }
+  }
+}
+for (const benchmark of latestBaseBenchmarks.benches) {
+  compareWithBase[benchmark.name] = {
+    previous: benchmark,
+    current: {},
+  }
+}
+for (const benchmark of latestHeadBenchmarks.benches) {
+  compareWithBase[benchmark.name] = {
+    // @ts-expect-error it should work
+    previous: {},
+    ...compareWithBase[benchmark.name],
+    current: benchmark,
+  }
+  compareWithHead[benchmark.name] = {
+    // @ts-expect-error it should work
+    previous: {},
+    ...compareWithHead[benchmark.name],
+    current: benchmark,
+  }
+}
+
+let message = '<!-- benchmark comment by ci -->\n'
+
+const compareTableInfo = [
+  { name: 'last head', commit: lastHeadBenchmarks ? lastHeadBenchmarks.commit.id : '' },
+  {
+    name: 'base',
+    commit: latestBaseBenchmarks.commit.id,
+  },
+]
+for (const benchmarkType of ['Performance', 'Memory']) {
+  message += `# ${benchmarkType} Benchmark\n\n`
+  for (const [index, compare] of [compareWithHead, compareWithBase].entries()) {
+    message += `## Compared with ${compareTableInfo[index].name}\n`
+    message += '<details><summary>Detail results of benchmarks</summary>\n\n'
+    message += `| Benchmark suite | Current: ${latestHeadBenchmarks.commit.id} | Previous: ${compareTableInfo[index].commit} | Ratio |\n | -| -| -| -|\n`
+    for (const field of Object.keys(compare).filter((key) =>
+      benchmarkType === 'Performance' ? !key.startsWith('[Cache Plugin]') : key.startsWith('[Cache Plugin]'),
+    )) {
+      message += `| \`${field}\` | ${compare[field].current.value ? `\`${compare[field].current.value!}\`` : ''} ${
+        compare[field].current.unit ?? ''
+      } ${compare[field].current.range ? `(\`${compare[field].current.range ?? ''}\`)` : ''} | ${
+        compare[field].previous.value ? `\`${compare[field].previous.value!}\`` : ''
+      } ${compare[field].previous.unit ?? ''} ${compare[field].previous.range ? `(\`${compare[field].previous.range ?? ''}\`)` : ''} | ${
+        compare[field].previous.value && compare[field].current.value
+          ? `\`${
+              // @ts-expect-error it work
+              Math.round((parseFloat(compare[field].previous.value) / parseFloat(compare[field].current.value)) * 100) / 100
+            }\``
+          : ''
+      } |\n`
+    }
+    message += '</details>\n\n'
+  }
+}
+
+console.log(message.replaceAll('`', '\\`'))

scripts/generateMessage.js

@@ -0,0 +1,78 @@
+import fs from 'fs/promises'
+const benchmarkData = await fetch(`https://raw.githubusercontent.com/discordeno/discordeno/benchies/benchmarksResult/data.js`)
+  .then(async (res) => await res.text())
+  .then((text) => JSON.parse(text.slice(24)))
+// eslint-disable-next-line @typescript-eslint/no-unused-vars, no-unused-vars
+const commitSha = await fs.readFile('./sha', 'utf-8')
+const results = JSON.parse(await fs.readFile('./data.json', 'utf-8'))
+const benchmarks = results.entries.Benchmark.slice(-2)
+const latestHeadBenchmarks = benchmarks.length === 2 ? benchmarks[1] : benchmarks[0]
+const lastHeadBenchmarks = benchmarks.length === 2 ? benchmarks[0] : undefined
+const latestBaseBenchmarks = JSON.parse(JSON.stringify(benchmarkData.entries.Benchmark)).slice(-1)[0]
+const compareWithHead = {}
+const compareWithBase = {}
+if (lastHeadBenchmarks) {
+  for (const benchmark of lastHeadBenchmarks.benches) {
+    compareWithHead[benchmark.name] = {
+      previous: benchmark,
+      current: {},
+    }
+  }
+}
+for (const benchmark of latestBaseBenchmarks.benches) {
+  compareWithBase[benchmark.name] = {
+    previous: benchmark,
+    current: {},
+  }
+}
+for (const benchmark of latestHeadBenchmarks.benches) {
+  compareWithBase[benchmark.name] = {
+    // @ts-expect-error it should work
+    previous: {},
+    ...compareWithBase[benchmark.name],
+    current: benchmark,
+  }
+  compareWithHead[benchmark.name] = {
+    // @ts-expect-error it should work
+    previous: {},
+    ...compareWithHead[benchmark.name],
+    current: benchmark,
+  }
+}
+let message = '<!-- benchmark comment by ci -->\n'
+const compareTableInfo = [
+  {
+    name: 'last head',
+    commit: lastHeadBenchmarks ? lastHeadBenchmarks.commit.id : '',
+  },
+  {
+    name: 'base',
+    commit: latestBaseBenchmarks.commit.id,
+  },
+]
+for (const benchmarkType of ['Performance', 'Memory']) {
+  message += `# ${benchmarkType} Benchmark\n\n`
+  for (const [index, compare] of [compareWithHead, compareWithBase].entries()) {
+    message += `## Compared with ${compareTableInfo[index].name}\n`
+    message += '<details><summary>Detail results of benchmarks</summary>\n\n'
+    message += `| Benchmark suite | Current: ${latestHeadBenchmarks.commit.id} | Previous: ${compareTableInfo[index].commit} | Ratio |\n | -| -| -| -|\n`
+    for (const field of Object.keys(compare).filter((key) =>
+      benchmarkType === 'Performance' ? !key.startsWith('[Cache Plugin]') : key.startsWith('[Cache Plugin]'),
+    )) {
+      message += `| \`${field}\` | ${compare[field].current.value ? `\`${compare[field].current.value}\`` : ''} ${
+        compare[field].current.unit ?? ''
+      } ${compare[field].current.range ? `(\`${compare[field].current.range ?? ''}\`)` : ''} | ${
+        compare[field].previous.value ? `\`${compare[field].previous.value}\`` : ''
+      } ${compare[field].previous.unit ?? ''} ${compare[field].previous.range ? `(\`${compare[field].previous.range ?? ''}\`)` : ''} | ${
+        compare[field].previous.value && compare[field].current.value
+          ? `\`${
+              // @ts-expect-error it work
+              Math.round((parseFloat(compare[field].previous.value) / parseFloat(compare[field].current.value)) * 100) / 100
+            }\``
+          : ''
+      } |\n`
+    }
+    message += '</details>\n\n'
+  }
+}
+console.log(message.replaceAll('`', '\\`'))