diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 85d89dda2..6817f4480 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -2,6 +2,13 @@ name: Benchmark on: workflow_call: + inputs: + sha: + required: true + type: string + repo: + required: true + type: string outputs: cpuMatch: value: ${{ jobs.benchmark.outputs.cpuMatch }} @@ -48,4 +55,46 @@ jobs: - name: Benchmark if: ${{ steps.cpuCheck.outputs.match == 'true' }} - run: node --expose-gc ./packages/benchmark/dist/index.js + run: node --expose-gc ./packages/benchmark/dist/index.js | tee output.txt + + - name: Download previous benchmark data + if: ${{ steps.cpuCheck.outputs.match == 'true' }} + uses: actions/cache@v3 + with: + path: ./benchmarksResult + key: ${{ github.ref }}-benchmark + - name: Store benchmark result to cache + if: ${{ steps.cpuCheck.outputs.match == 'true' }} + uses: benchmark-action/github-action-benchmark@v1 + with: + tool: "benchmarkjs" + output-file-path: output.txt + external-data-json-path: benchmarksResult/data.json + - uses: actions/upload-artifact@v3 + if: ${{ steps.cpuCheck.outputs.match == 'true' }} + with: + name: benchmarkResults + path: benchmarksResult/data.json + + - name: Store benchmark result (Main) + uses: benchmark-action/github-action-benchmark@v1 + if: ${{ github.ref == 'refs/heads/main' && steps.cpuCheck.outputs.match == 'true' }} + with: + tool: "benchmarkjs" + output-file-path: output.txt + gh-pages-branch: "benchies" + benchmark-data-dir-path: benchmarksResult + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: true + + - name: Save Commmit SHA + if: ${{ steps.cpuCheck.outputs.match == 'true' }} + run: | + mkdir -p ./commitData + echo ${{ inputs.sha }} > ./commitData/sha + echo ${{ inputs.repo }} > ./commitData/repo + - uses: actions/upload-artifact@v3 + if: ${{ steps.cpuCheck.outputs.match == 'true' }} + with: + name: commitData + path: commitData/ diff --git a/.github/workflows/commentBenchResult.yml b/.github/workflows/commentBenchResult.yml new file mode 100644 index 000000000..2a16e516f --- /dev/null +++ b/.github/workflows/commentBenchResult.yml @@ -0,0 +1,111 @@ +name: Comment Benchmark Result + +on: + workflow_run: + workflows: [Benchmark with retry] + types: + - completed + +jobs: + comment-benchmark-result: + name: Comment Benchmark Result + runs-on: ubuntu-latest + if: ${{ github.event.workflow_run.conclusion == 'success' }} + steps: + - uses: actions/checkout@v3 + - uses: denoland/setup-deno@main + with: + deno-version: v1.x + - name: Download Commit Data Artifact + uses: actions/github-script@v6 + with: + script: | + let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.payload.workflow_run.id, + }); + let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => { + return artifact.name == "commitData" + })[0]; + let download = await github.rest.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: matchArtifact.id, + archive_format: 'zip', + }); + let fs = require('fs'); + fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/commitData.zip`, Buffer.from(download.data)); + + - name: Unzip Commit Data Artifact + run: unzip commitData.zip + + - name: Download Result Artifact + uses: actions/github-script@v6 + with: + script: | + let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.payload.workflow_run.id, + }); + let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => { + return artifact.name == "benchmarkResults" + })[0]; + let download = await github.rest.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: matchArtifact.id, + archive_format: 'zip', + }); + let fs = require('fs'); + fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/benchmarkResults.zip`, Buffer.from(download.data)); + + - name: Unzip Result Artifact + run: unzip benchmarkResults.zip + + - name: Generate Message + id: genMessage + run: | + MESSAGE=$(deno run -A performance/generateMessage.ts) + echo "MESSAGE<> $GITHUB_ENV + echo "$MESSAGE" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + + - name: "Comment on PR" + uses: actions/github-script@v6 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const fs = require('fs'); + const commit_sha = fs.readFileSync('./sha', 'utf-8'); + const repo = fs.readFileSync('./repo', 'utf-8'); + if (repo.split('/')[1] === undefined) process.exit(0) + const pr = await github.rest.repos.listPullRequestsAssociatedWithCommit({ + commit_sha: commit_sha.slice(0,-1), + owner: repo.split('/')[0], + repo: repo.split('/')[1].slice(0,-1), + }); + if (pr.data[0]) { + const comments = await github.rest.issues.listComments({ + owner: "discordeno", + repo: "discordeno", + issue_number: pr.data[0].number, + }) + const oldComment = comments.data.find((comment) => comment.body.includes('benchmark comment by ci')) + if(!oldComment) { + github.rest.issues.createComment({ + issue_number: pr.data[0].number, + owner: "discordeno", + repo: "discordeno", + body: `${{ env.MESSAGE }}` + }) + } else { + github.rest.issues.updateComment({ + owner: "discordeno", + repo: "discordeno", + comment_id: oldComment.id, + body: `${{ env.MESSAGE }}` + }); + } + } diff --git a/.github/workflows/retryBenchmark.yml b/.github/workflows/retryBenchmark.yml index 8c9abc194..da790191f 100644 --- a/.github/workflows/retryBenchmark.yml +++ b/.github/workflows/retryBenchmark.yml @@ -9,48 +9,78 @@ on: jobs: benchmark-try-1: uses: ./.github/workflows/benchmark.yml + with: + sha: ${{ github.event.pull_request.head.sha }} + repo: ${{ github.event.pull_request.head.repo.full_name }} benchmark-try-2: needs: benchmark-try-1 if: ${{ needs.benchmark-try-1.outputs.cpuMatch == 'false' }} uses: ./.github/workflows/benchmark.yml + with: + sha: ${{ github.event.pull_request.head.sha }} + repo: ${{ github.event.pull_request.head.repo.full_name }} benchmark-try-3: needs: benchmark-try-2 if: ${{ needs.benchmark-try-2.outputs.cpuMatch == 'false' }} uses: ./.github/workflows/benchmark.yml + with: + sha: ${{ github.event.pull_request.head.sha }} + repo: ${{ github.event.pull_request.head.repo.full_name }} benchmark-try-4: needs: benchmark-try-3 if: ${{ needs.benchmark-try-3.outputs.cpuMatch == 'false' }} uses: ./.github/workflows/benchmark.yml + with: + sha: ${{ github.event.pull_request.head.sha }} + repo: ${{ github.event.pull_request.head.repo.full_name }} benchmark-try-5: needs: benchmark-try-4 if: ${{ needs.benchmark-try-4.outputs.cpuMatch == 'false' }} uses: ./.github/workflows/benchmark.yml + with: + sha: ${{ github.event.pull_request.head.sha }} + repo: ${{ github.event.pull_request.head.repo.full_name }} benchmark-try-6: needs: benchmark-try-5 if: ${{ needs.benchmark-try-5.outputs.cpuMatch == 'false' }} uses: ./.github/workflows/benchmark.yml + with: + sha: ${{ github.event.pull_request.head.sha }} + repo: ${{ github.event.pull_request.head.repo.full_name }} benchmark-try-7: needs: benchmark-try-6 if: ${{ needs.benchmark-try-6.outputs.cpuMatch == 'false' }} uses: ./.github/workflows/benchmark.yml + with: + sha: ${{ github.event.pull_request.head.sha }} + repo: ${{ github.event.pull_request.head.repo.full_name }} benchmark-try-8: needs: benchmark-try-7 if: ${{ needs.benchmark-try-7.outputs.cpuMatch == 'false' }} uses: ./.github/workflows/benchmark.yml + with: + sha: ${{ github.event.pull_request.head.sha }} + repo: ${{ github.event.pull_request.head.repo.full_name }} benchmark-try-9: needs: benchmark-try-8 if: ${{ needs.benchmark-try-8.outputs.cpuMatch == 'false' }} uses: ./.github/workflows/benchmark.yml + with: + sha: ${{ github.event.pull_request.head.sha }} + repo: ${{ github.event.pull_request.head.repo.full_name }} benchmark-try-10: needs: benchmark-try-9 if: ${{ needs.benchmark-try-9.outputs.cpuMatch == 'false' }} uses: ./.github/workflows/benchmark.yml + with: + sha: ${{ github.event.pull_request.head.sha }} + repo: ${{ github.event.pull_request.head.repo.full_name }} diff --git a/packages/benchmark/package.json b/packages/benchmark/package.json index ebfd4322a..7f374d539 100644 --- a/packages/benchmark/package.json +++ b/packages/benchmark/package.json @@ -15,6 +15,7 @@ "fmt": "eslint --fix \"src/**/*.ts*\"", "lint": "eslint \"src/**/*.ts*\"", "build": "swc src --delete-dir-on-start --out-dir dist && node ../../scripts/fixBenchExtension.js", + "build-message": "swc src/utils/generateMessage.ts --out-dir ../../scripts && node ../../scripts/fixBenchExtension.js", "bench": "node dist/index.js" }, "dependencies": { @@ -38,4 +39,4 @@ "tsconfig": "*", "typescript": "^4.9.3" } -} +} \ No newline at end of file diff --git a/packages/benchmark/src/index.ts b/packages/benchmark/src/index.ts index 0ad2d140f..f11a6f5d2 100644 --- a/packages/benchmark/src/index.ts +++ b/packages/benchmark/src/index.ts @@ -1,7 +1,7 @@ import fs from 'node:fs/promises' import { suite } from './benchmarkSuite.js' -const benchmarks = await fs.readdir('packages/benchmark/dist/benchmarks').then((files) => files.filter((file) => file.endsWith('.js'))) +const benchmarks = await fs.readdir(new URL('./benchmarks', import.meta.url)).then((files) => files.filter((file) => file.endsWith('.js'))) await Promise.all(benchmarks.map(async (file) => await import(`./benchmarks/${file}`))) diff --git a/packages/benchmark/src/utils/generateMessage.ts b/packages/benchmark/src/utils/generateMessage.ts new file mode 100644 index 000000000..7256472bd --- /dev/null +++ b/packages/benchmark/src/utils/generateMessage.ts @@ -0,0 +1,121 @@ +import fs from 'fs/promises' + +const benchmarkData = await fetch(`https://raw.githubusercontent.com/discordeno/discordeno/benchies/benchmarksResult/data.js`) + .then(async (res) => await res.text()) + .then((text) => JSON.parse(text.slice(24))) + +// eslint-disable-next-line @typescript-eslint/no-unused-vars, no-unused-vars +const commitSha = await fs.readFile('./sha', 'utf-8') +const results = JSON.parse(await fs.readFile('./data.json', 'utf-8')) + +interface BenchmarksData { + commit: { + author: { email: string; name: string; username: string } + committer: { email: string; name: string; username: string } + distinct: boolean + id: string + message: string + timestamp: string + tree_id: string + url: string + } + date: number + tool: string + benches: Array<{ name: string; value: number; unit: string; range: string }> +} + +type CompareTable = Record< + string, + { + current: + | { name: string; value: number; unit: string; range: string } + | { + name?: string + value?: number + unit?: string + range?: string + } + previous: + | { name: string; value: number; unit: string; range: string } + | { + name?: string + value?: number + unit?: string + range?: string + } + } +> + +const benchmarks = results.entries.Benchmark.slice(-2) as BenchmarksData[] +const latestHeadBenchmarks = benchmarks.length === 2 ? benchmarks[1] : benchmarks[0] +const lastHeadBenchmarks = benchmarks.length === 2 ? benchmarks[0] : undefined +const latestBaseBenchmarks = JSON.parse(JSON.stringify(benchmarkData.entries.Benchmark)).slice(-1)[0] as BenchmarksData + +const compareWithHead: CompareTable = {} +const compareWithBase: CompareTable = {} + +if (lastHeadBenchmarks) { + for (const benchmark of lastHeadBenchmarks.benches) { + compareWithHead[benchmark.name] = { + previous: benchmark, + current: {}, + } + } +} +for (const benchmark of latestBaseBenchmarks.benches) { + compareWithBase[benchmark.name] = { + previous: benchmark, + current: {}, + } +} +for (const benchmark of latestHeadBenchmarks.benches) { + compareWithBase[benchmark.name] = { + // @ts-expect-error it should work + previous: {}, + ...compareWithBase[benchmark.name], + current: benchmark, + } + compareWithHead[benchmark.name] = { + // @ts-expect-error it should work + previous: {}, + ...compareWithHead[benchmark.name], + current: benchmark, + } +} + +let message = '\n' + +const compareTableInfo = [ + { name: 'last head', commit: lastHeadBenchmarks ? lastHeadBenchmarks.commit.id : '' }, + { + name: 'base', + commit: latestBaseBenchmarks.commit.id, + }, +] +for (const benchmarkType of ['Performance', 'Memory']) { + message += `# ${benchmarkType} Benchmark\n\n` + for (const [index, compare] of [compareWithHead, compareWithBase].entries()) { + message += `## Compared with ${compareTableInfo[index].name}\n` + message += '
Detail results of benchmarks\n\n' + message += `| Benchmark suite | Current: ${latestHeadBenchmarks.commit.id} | Previous: ${compareTableInfo[index].commit} | Ratio |\n | -| -| -| -|\n` + for (const field of Object.keys(compare).filter((key) => + benchmarkType === 'Performance' ? !key.startsWith('[Cache Plugin]') : key.startsWith('[Cache Plugin]'), + )) { + message += `| \`${field}\` | ${compare[field].current.value ? `\`${compare[field].current.value!}\`` : ''} ${ + compare[field].current.unit ?? '' + } ${compare[field].current.range ? `(\`${compare[field].current.range ?? ''}\`)` : ''} | ${ + compare[field].previous.value ? `\`${compare[field].previous.value!}\`` : '' + } ${compare[field].previous.unit ?? ''} ${compare[field].previous.range ? `(\`${compare[field].previous.range ?? ''}\`)` : ''} | ${ + compare[field].previous.value && compare[field].current.value + ? `\`${ + // @ts-expect-error it work + Math.round((parseFloat(compare[field].previous.value) / parseFloat(compare[field].current.value)) * 100) / 100 + }\`` + : '' + } |\n` + } + message += '
\n\n' + } +} + +console.log(message.replaceAll('`', '\\`')) diff --git a/scripts/generateMessage.js b/scripts/generateMessage.js new file mode 100644 index 000000000..802cf99e9 --- /dev/null +++ b/scripts/generateMessage.js @@ -0,0 +1,78 @@ +import fs from 'fs/promises' +const benchmarkData = await fetch(`https://raw.githubusercontent.com/discordeno/discordeno/benchies/benchmarksResult/data.js`) + .then(async (res) => await res.text()) + .then((text) => JSON.parse(text.slice(24))) +// eslint-disable-next-line @typescript-eslint/no-unused-vars, no-unused-vars +const commitSha = await fs.readFile('./sha', 'utf-8') +const results = JSON.parse(await fs.readFile('./data.json', 'utf-8')) +const benchmarks = results.entries.Benchmark.slice(-2) +const latestHeadBenchmarks = benchmarks.length === 2 ? benchmarks[1] : benchmarks[0] +const lastHeadBenchmarks = benchmarks.length === 2 ? benchmarks[0] : undefined +const latestBaseBenchmarks = JSON.parse(JSON.stringify(benchmarkData.entries.Benchmark)).slice(-1)[0] +const compareWithHead = {} +const compareWithBase = {} +if (lastHeadBenchmarks) { + for (const benchmark of lastHeadBenchmarks.benches) { + compareWithHead[benchmark.name] = { + previous: benchmark, + current: {}, + } + } +} +for (const benchmark of latestBaseBenchmarks.benches) { + compareWithBase[benchmark.name] = { + previous: benchmark, + current: {}, + } +} +for (const benchmark of latestHeadBenchmarks.benches) { + compareWithBase[benchmark.name] = { + // @ts-expect-error it should work + previous: {}, + ...compareWithBase[benchmark.name], + current: benchmark, + } + compareWithHead[benchmark.name] = { + // @ts-expect-error it should work + previous: {}, + ...compareWithHead[benchmark.name], + current: benchmark, + } +} +let message = '\n' +const compareTableInfo = [ + { + name: 'last head', + commit: lastHeadBenchmarks ? lastHeadBenchmarks.commit.id : '', + }, + { + name: 'base', + commit: latestBaseBenchmarks.commit.id, + }, +] +for (const benchmarkType of ['Performance', 'Memory']) { + message += `# ${benchmarkType} Benchmark\n\n` + for (const [index, compare] of [compareWithHead, compareWithBase].entries()) { + message += `## Compared with ${compareTableInfo[index].name}\n` + message += '
Detail results of benchmarks\n\n' + message += `| Benchmark suite | Current: ${latestHeadBenchmarks.commit.id} | Previous: ${compareTableInfo[index].commit} | Ratio |\n | -| -| -| -|\n` + for (const field of Object.keys(compare).filter((key) => + benchmarkType === 'Performance' ? !key.startsWith('[Cache Plugin]') : key.startsWith('[Cache Plugin]'), + )) { + message += `| \`${field}\` | ${compare[field].current.value ? `\`${compare[field].current.value}\`` : ''} ${ + compare[field].current.unit ?? '' + } ${compare[field].current.range ? `(\`${compare[field].current.range ?? ''}\`)` : ''} | ${ + compare[field].previous.value ? `\`${compare[field].previous.value}\`` : '' + } ${compare[field].previous.unit ?? ''} ${compare[field].previous.range ? `(\`${compare[field].previous.range ?? ''}\`)` : ''} | ${ + compare[field].previous.value && compare[field].current.value + ? `\`${ + // @ts-expect-error it work + Math.round((parseFloat(compare[field].previous.value) / parseFloat(compare[field].current.value)) * 100) / 100 + }\`` + : '' + } |\n` + } + message += '
\n\n' + } +} +console.log(message.replaceAll('`', '\\`'))