From 9c7b2b832ec8e4c4977140520d08510eb0afebda Mon Sep 17 00:00:00 2001 From: Jonathan Ho Date: Sat, 1 Apr 2023 16:27:54 -0700 Subject: [PATCH] Fix(bench): wrong location (#2932) * Update commentBenchResult.yml * ci: fix ext * refactor(bench): better table --- .github/workflows/commentBenchResult.yml | 2 +- packages/benchmark/package.json | 2 +- packages/benchmark/src/generateMessage.ts | 83 ++++++++++++ .../benchmark/src/utils/generateMessage.ts | 121 ------------------ scripts/generateMessage.js | 102 ++++++--------- 5 files changed, 124 insertions(+), 186 deletions(-) create mode 100644 packages/benchmark/src/generateMessage.ts delete mode 100644 packages/benchmark/src/utils/generateMessage.ts diff --git a/.github/workflows/commentBenchResult.yml b/.github/workflows/commentBenchResult.yml index 2a16e516f..8767497c0 100644 --- a/.github/workflows/commentBenchResult.yml +++ b/.github/workflows/commentBenchResult.yml @@ -67,7 +67,7 @@ jobs: - name: Generate Message id: genMessage run: | - MESSAGE=$(deno run -A performance/generateMessage.ts) + MESSAGE=$(deno run -A scripts/generateMessage.js) echo "MESSAGE<> $GITHUB_ENV echo "$MESSAGE" >> $GITHUB_ENV echo "EOF" >> $GITHUB_ENV diff --git a/packages/benchmark/package.json b/packages/benchmark/package.json index 7f374d539..c29a7bb9d 100644 --- a/packages/benchmark/package.json +++ b/packages/benchmark/package.json @@ -15,7 +15,7 @@ "fmt": "eslint --fix \"src/**/*.ts*\"", "lint": "eslint \"src/**/*.ts*\"", "build": "swc src --delete-dir-on-start --out-dir dist && node ../../scripts/fixBenchExtension.js", - "build-message": "swc src/utils/generateMessage.ts --out-dir ../../scripts && node ../../scripts/fixBenchExtension.js", + "build-message": "swc src/generateMessage.ts -C sourceMaps=false --out-dir ../../scripts && node ../../scripts/fixBenchExtension.js", "bench": "node dist/index.js" }, "dependencies": { diff --git a/packages/benchmark/src/generateMessage.ts b/packages/benchmark/src/generateMessage.ts new file mode 100644 index 000000000..d93c6f580 --- /dev/null +++ b/packages/benchmark/src/generateMessage.ts @@ -0,0 +1,83 @@ +import fs from 'node:fs/promises' + +const benchmarkData = await fetch(`https://raw.githubusercontent.com/discordeno/discordeno/benchies/benchmarksResult/data.js`) + .then(async (res) => await res.text()) + .then((text) => JSON.parse(text.slice(24))) + +// eslint-disable-next-line @typescript-eslint/no-unused-vars, no-unused-vars +const commitSha = await fs.readFile('./sha', 'utf-8') +const results = JSON.parse(await fs.readFile('./data.json', 'utf-8')) + +interface BenchmarksData { + commit: { + author: { email: string; name: string; username: string } + committer: { email: string; name: string; username: string } + distinct: boolean + id: string + message: string + timestamp: string + tree_id: string + url: string + } + date: number + tool: string + benches: Array<{ name: string; value: number; unit: string; range: string }> +} + +type CompareTable = Record> + +const benchmarks = results.entries.Benchmark as BenchmarksData[] +benchmarks.reverse() + +const compareWithHead: CompareTable = {} +const latestBaseBenchmarks = benchmarkData.entries.Benchmark.slice(-1)[0] as BenchmarksData + +for (const benchmark of latestBaseBenchmarks.benches) { + compareWithHead[benchmark.name] = { + [latestBaseBenchmarks.commit.id]: benchmark, + } +} +for (let i = 0; i < benchmarks.length; i++) { + for (const bench of benchmarks[i].benches) { + if (compareWithHead[bench.name]) { + compareWithHead[bench.name][benchmarks[i].commit.id] = bench + } else { + compareWithHead[bench.name] = { + [benchmarks[i].commit.id]: bench, + } + } + } +} + +let message = '\n' + +message += `## Benchmark\n\n` +message += '
Detail results of benchmarks\n\n' +let header1 = `| Benchmark suite | Base (${latestBaseBenchmarks.commit.id}) |` +let header2 = `|-|-|` +for (const [index, commitId] of benchmarks.map((benchmark) => benchmark.commit.id).entries()) { + header1 += index === 0 ? ` Latest Head (${commitId}) |` : ` ${commitId} |` + header2 += '-|' +} +message += `${header1}\n` +message += `${header2}\n` + +for (const benchName of Object.keys(compareWithHead)) { + let benchData = `| ${benchName} |` + benchData += compareWithHead[benchName][latestBaseBenchmarks.commit.id] + ? ` ${`\`${compareWithHead[benchName][latestBaseBenchmarks.commit.id].value}\` ${ + compareWithHead[benchName][latestBaseBenchmarks.commit.id].unit + } \`${compareWithHead[benchName][latestBaseBenchmarks.commit.id].range}\``} |` + : '|' + + for (const commitId of benchmarks.map((benchmark) => benchmark.commit.id)) { + benchData += compareWithHead[benchName][commitId] + ? ` \`${compareWithHead[benchName][commitId].value}\` ${compareWithHead[benchName][commitId].unit} \`${compareWithHead[benchName][commitId].range}\`|` + : '|' + } + message += `${benchData}\n` +} + +message += '
\n\n' + +console.log(message.replaceAll('`', '\\`')) diff --git a/packages/benchmark/src/utils/generateMessage.ts b/packages/benchmark/src/utils/generateMessage.ts deleted file mode 100644 index 7256472bd..000000000 --- a/packages/benchmark/src/utils/generateMessage.ts +++ /dev/null @@ -1,121 +0,0 @@ -import fs from 'fs/promises' - -const benchmarkData = await fetch(`https://raw.githubusercontent.com/discordeno/discordeno/benchies/benchmarksResult/data.js`) - .then(async (res) => await res.text()) - .then((text) => JSON.parse(text.slice(24))) - -// eslint-disable-next-line @typescript-eslint/no-unused-vars, no-unused-vars -const commitSha = await fs.readFile('./sha', 'utf-8') -const results = JSON.parse(await fs.readFile('./data.json', 'utf-8')) - -interface BenchmarksData { - commit: { - author: { email: string; name: string; username: string } - committer: { email: string; name: string; username: string } - distinct: boolean - id: string - message: string - timestamp: string - tree_id: string - url: string - } - date: number - tool: string - benches: Array<{ name: string; value: number; unit: string; range: string }> -} - -type CompareTable = Record< - string, - { - current: - | { name: string; value: number; unit: string; range: string } - | { - name?: string - value?: number - unit?: string - range?: string - } - previous: - | { name: string; value: number; unit: string; range: string } - | { - name?: string - value?: number - unit?: string - range?: string - } - } -> - -const benchmarks = results.entries.Benchmark.slice(-2) as BenchmarksData[] -const latestHeadBenchmarks = benchmarks.length === 2 ? benchmarks[1] : benchmarks[0] -const lastHeadBenchmarks = benchmarks.length === 2 ? benchmarks[0] : undefined -const latestBaseBenchmarks = JSON.parse(JSON.stringify(benchmarkData.entries.Benchmark)).slice(-1)[0] as BenchmarksData - -const compareWithHead: CompareTable = {} -const compareWithBase: CompareTable = {} - -if (lastHeadBenchmarks) { - for (const benchmark of lastHeadBenchmarks.benches) { - compareWithHead[benchmark.name] = { - previous: benchmark, - current: {}, - } - } -} -for (const benchmark of latestBaseBenchmarks.benches) { - compareWithBase[benchmark.name] = { - previous: benchmark, - current: {}, - } -} -for (const benchmark of latestHeadBenchmarks.benches) { - compareWithBase[benchmark.name] = { - // @ts-expect-error it should work - previous: {}, - ...compareWithBase[benchmark.name], - current: benchmark, - } - compareWithHead[benchmark.name] = { - // @ts-expect-error it should work - previous: {}, - ...compareWithHead[benchmark.name], - current: benchmark, - } -} - -let message = '\n' - -const compareTableInfo = [ - { name: 'last head', commit: lastHeadBenchmarks ? lastHeadBenchmarks.commit.id : '' }, - { - name: 'base', - commit: latestBaseBenchmarks.commit.id, - }, -] -for (const benchmarkType of ['Performance', 'Memory']) { - message += `# ${benchmarkType} Benchmark\n\n` - for (const [index, compare] of [compareWithHead, compareWithBase].entries()) { - message += `## Compared with ${compareTableInfo[index].name}\n` - message += '
Detail results of benchmarks\n\n' - message += `| Benchmark suite | Current: ${latestHeadBenchmarks.commit.id} | Previous: ${compareTableInfo[index].commit} | Ratio |\n | -| -| -| -|\n` - for (const field of Object.keys(compare).filter((key) => - benchmarkType === 'Performance' ? !key.startsWith('[Cache Plugin]') : key.startsWith('[Cache Plugin]'), - )) { - message += `| \`${field}\` | ${compare[field].current.value ? `\`${compare[field].current.value!}\`` : ''} ${ - compare[field].current.unit ?? '' - } ${compare[field].current.range ? `(\`${compare[field].current.range ?? ''}\`)` : ''} | ${ - compare[field].previous.value ? `\`${compare[field].previous.value!}\`` : '' - } ${compare[field].previous.unit ?? ''} ${compare[field].previous.range ? `(\`${compare[field].previous.range ?? ''}\`)` : ''} | ${ - compare[field].previous.value && compare[field].current.value - ? `\`${ - // @ts-expect-error it work - Math.round((parseFloat(compare[field].previous.value) / parseFloat(compare[field].current.value)) * 100) / 100 - }\`` - : '' - } |\n` - } - message += '
\n\n' - } -} - -console.log(message.replaceAll('`', '\\`')) diff --git a/scripts/generateMessage.js b/scripts/generateMessage.js index 802cf99e9..bc90d105a 100644 --- a/scripts/generateMessage.js +++ b/scripts/generateMessage.js @@ -1,78 +1,54 @@ -import fs from 'fs/promises' +import fs from 'node:fs/promises' const benchmarkData = await fetch(`https://raw.githubusercontent.com/discordeno/discordeno/benchies/benchmarksResult/data.js`) .then(async (res) => await res.text()) .then((text) => JSON.parse(text.slice(24))) // eslint-disable-next-line @typescript-eslint/no-unused-vars, no-unused-vars const commitSha = await fs.readFile('./sha', 'utf-8') const results = JSON.parse(await fs.readFile('./data.json', 'utf-8')) -const benchmarks = results.entries.Benchmark.slice(-2) -const latestHeadBenchmarks = benchmarks.length === 2 ? benchmarks[1] : benchmarks[0] -const lastHeadBenchmarks = benchmarks.length === 2 ? benchmarks[0] : undefined -const latestBaseBenchmarks = JSON.parse(JSON.stringify(benchmarkData.entries.Benchmark)).slice(-1)[0] +const benchmarks = results.entries.Benchmark +benchmarks.reverse() const compareWithHead = {} -const compareWithBase = {} -if (lastHeadBenchmarks) { - for (const benchmark of lastHeadBenchmarks.benches) { - compareWithHead[benchmark.name] = { - previous: benchmark, - current: {}, - } - } -} +const latestBaseBenchmarks = benchmarkData.entries.Benchmark.slice(-1)[0] for (const benchmark of latestBaseBenchmarks.benches) { - compareWithBase[benchmark.name] = { - previous: benchmark, - current: {}, + compareWithHead[benchmark.name] = { + [latestBaseBenchmarks.commit.id]: benchmark, } } -for (const benchmark of latestHeadBenchmarks.benches) { - compareWithBase[benchmark.name] = { - // @ts-expect-error it should work - previous: {}, - ...compareWithBase[benchmark.name], - current: benchmark, - } - compareWithHead[benchmark.name] = { - // @ts-expect-error it should work - previous: {}, - ...compareWithHead[benchmark.name], - current: benchmark, +for (let i = 0; i < benchmarks.length; i++) { + for (const bench of benchmarks[i].benches) { + if (compareWithHead[bench.name]) { + compareWithHead[bench.name][benchmarks[i].commit.id] = bench + } else { + compareWithHead[bench.name] = { + [benchmarks[i].commit.id]: bench, + } + } } } let message = '\n' -const compareTableInfo = [ - { - name: 'last head', - commit: lastHeadBenchmarks ? lastHeadBenchmarks.commit.id : '', - }, - { - name: 'base', - commit: latestBaseBenchmarks.commit.id, - }, -] -for (const benchmarkType of ['Performance', 'Memory']) { - message += `# ${benchmarkType} Benchmark\n\n` - for (const [index, compare] of [compareWithHead, compareWithBase].entries()) { - message += `## Compared with ${compareTableInfo[index].name}\n` - message += '
Detail results of benchmarks\n\n' - message += `| Benchmark suite | Current: ${latestHeadBenchmarks.commit.id} | Previous: ${compareTableInfo[index].commit} | Ratio |\n | -| -| -| -|\n` - for (const field of Object.keys(compare).filter((key) => - benchmarkType === 'Performance' ? !key.startsWith('[Cache Plugin]') : key.startsWith('[Cache Plugin]'), - )) { - message += `| \`${field}\` | ${compare[field].current.value ? `\`${compare[field].current.value}\`` : ''} ${ - compare[field].current.unit ?? '' - } ${compare[field].current.range ? `(\`${compare[field].current.range ?? ''}\`)` : ''} | ${ - compare[field].previous.value ? `\`${compare[field].previous.value}\`` : '' - } ${compare[field].previous.unit ?? ''} ${compare[field].previous.range ? `(\`${compare[field].previous.range ?? ''}\`)` : ''} | ${ - compare[field].previous.value && compare[field].current.value - ? `\`${ - // @ts-expect-error it work - Math.round((parseFloat(compare[field].previous.value) / parseFloat(compare[field].current.value)) * 100) / 100 - }\`` - : '' - } |\n` - } - message += '
\n\n' - } +message += `## Benchmark\n\n` +message += '
Detail results of benchmarks\n\n' +let header1 = `| Benchmark suite | Base (${latestBaseBenchmarks.commit.id}) |` +let header2 = `|-|-|` +for (const [index, commitId] of benchmarks.map((benchmark) => benchmark.commit.id).entries()) { + header1 += index === 0 ? ` Latest Head (${commitId}) |` : ` ${commitId} |` + header2 += '-|' } +message += `${header1}\n` +message += `${header2}\n` +for (const benchName of Object.keys(compareWithHead)) { + let benchData = `| ${benchName} |` + benchData += compareWithHead[benchName][latestBaseBenchmarks.commit.id] + ? ` ${`\`${compareWithHead[benchName][latestBaseBenchmarks.commit.id].value}\` ${ + compareWithHead[benchName][latestBaseBenchmarks.commit.id].unit + } \`${compareWithHead[benchName][latestBaseBenchmarks.commit.id].range}\``} |` + : '|' + for (const commitId of benchmarks.map((benchmark) => benchmark.commit.id)) { + benchData += compareWithHead[benchName][commitId] + ? ` \`${compareWithHead[benchName][commitId].value}\` ${compareWithHead[benchName][commitId].unit} \`${compareWithHead[benchName][commitId].range}\`|` + : '|' + } + message += `${benchData}\n` +} +message += '
\n\n' console.log(message.replaceAll('`', '\\`'))