From 011f0fe0dce5a79847ad6c2faf9fe604b78c066a Mon Sep 17 00:00:00 2001 From: Fleny Date: Wed, 7 Jan 2026 20:40:27 +0100 Subject: [PATCH] ci: Use turbo remote cache (#4666) * Use a turbo remote cache server for CI `rharkor/caching-for-turbo` uses github actions cache to provide a remote cache server for turbo, so we can use previuously created cache if available and requested by turbo, and this doesn't have the downside of having the cache growing indefinitely like with normal github actions cache as since the single caches are uploaded to the github actions cache, github can evict the old cache when it's not used or there is no space left. Also add turbo to website so we can cache the build * Use turbo for website build & build:doc * fix typo, tubo -> turbo --------- Co-authored-by: Skillz4Killz <23035000+Skillz4Killz@users.noreply.github.com> --- .github/workflows/e2e-test.yml | 16 ++--- .github/workflows/integration-test.yml | 16 ++--- .github/workflows/lib-check.yml | 49 +++++-------- .github/workflows/other-runtime-unit-test.yml | 32 +++------ .github/workflows/site.yml | 27 ++++--- .github/workflows/unit-test.yml | 22 ++---- turbo.json | 5 ++ website/package.json | 2 + website/turbo.json | 9 +++ website/yarn.lock | 72 +++++++++++++++++++ 10 files changed, 143 insertions(+), 107 deletions(-) create mode 100644 website/turbo.json diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index c810fdb13..4999ac274 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -22,21 +22,13 @@ jobs: node-version: 20 cache: yarn - run: yarn install --immutable - - name: Turbo Cache - id: turbo-cache - uses: actions/cache@v5 + - name: Cache for Turbo + uses: rharkor/caching-for-turbo@v2.3.5 with: - path: .turbo - key: ${{ runner.os }}-turbo-test:e2e-${{ inputs.package }}-${{ github.sha }} - - name: Build dist cache - if: steps.turbo-cache.outputs.cache-hit != 'true' - uses: actions/cache@v5 - with: - path: .turbo - key: ${{ runner.os }}-turbo-build-${{ github.sha }} + cache-prefix: turbo-cache- - name: E2E Test if: ${{ inputs.event_name != 'pull_request' }} - run: yarn test:e2e --cache-dir=".turbo" --filter=./packages/${{ inputs.package }} + run: yarn test:e2e --filter=./packages/${{ inputs.package }} timeout-minutes: 5 env: DISCORD_TOKEN: ${{ secrets.DISCORD_TOKEN }} diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 316f52e9e..3f5357640 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -19,20 +19,12 @@ jobs: node-version: 20 cache: yarn - run: yarn install --immutable - - name: Turbo Cache - id: turbo-cache - uses: actions/cache@v5 + - name: Cache for Turbo + uses: rharkor/caching-for-turbo@v2.3.5 with: - path: .turbo - key: ${{ runner.os }}-turbo-test:integration-${{ inputs.package }}-${{ github.sha }} - - name: Build dist cache - if: steps.turbo-cache.outputs.cache-hit != 'true' - uses: actions/cache@v5 - with: - path: .turbo - key: ${{ runner.os }}-turbo-build-${{ github.sha }} + cache-prefix: turbo-cache- - name: Integration Test - run: yarn test:integration --cache-dir=".turbo" --filter=./packages/${{ inputs.package }} + run: yarn test:integration --filter=./packages/${{ inputs.package }} timeout-minutes: 3 - name: Collect and upload the coverage report uses: codecov/codecov-action@v5 diff --git a/.github/workflows/lib-check.yml b/.github/workflows/lib-check.yml index 2b07e8bd0..8f4aa44c1 100644 --- a/.github/workflows/lib-check.yml +++ b/.github/workflows/lib-check.yml @@ -6,8 +6,6 @@ on: push: branches: - main - paths-ignore: - - 'website/**' merge_group: jobs: @@ -21,14 +19,12 @@ jobs: node-version: 20 cache: yarn - run: yarn install --immutable - - name: Turbo Cache - id: turbo-cache - uses: actions/cache@v5 + - name: Cache for Turbo + uses: rharkor/caching-for-turbo@v2.3.5 with: - path: .turbo - key: ${{ runner.os }}-turbo-build:type-${{ github.sha }} + cache-prefix: turbo-cache- - name: Build Types - run: yarn build:type --cache-dir=".turbo" + run: yarn build:type build-dist: name: Build Dist @@ -40,14 +36,12 @@ jobs: node-version: 20 cache: yarn - run: yarn install --immutable - - name: Turbo Cache - id: turbo-cache - uses: actions/cache@v5 + - name: Cache for Turbo + uses: rharkor/caching-for-turbo@v2.3.5 with: - path: .turbo - key: ${{ runner.os }}-turbo-build-${{ github.sha }} - - name: Type Test - run: yarn build --cache-dir=".turbo" + cache-prefix: turbo-cache- + - name: Build Dist + run: yarn build format: name: Format @@ -60,13 +54,12 @@ jobs: node-version: 20 cache: yarn - run: yarn install --immutable - - name: Build type cache - uses: actions/cache@v5 + - name: Cache for Turbo + uses: rharkor/caching-for-turbo@v2.3.5 with: - path: .turbo - key: ${{ runner.os }}-turbo-build:type-${{ github.sha }} + cache-prefix: turbo-cache- - name: Build Types - run: yarn build:type --cache-dir=".turbo" + run: yarn build:type - name: Check Formatting run: yarn biome ci --reporter=github @@ -81,20 +74,12 @@ jobs: node-version: 20 cache: yarn - run: yarn install --immutable - - name: Turbo Cache - id: turbo-cache - uses: actions/cache@v5 + - name: Cache for Turbo + uses: rharkor/caching-for-turbo@v2.3.5 with: - path: .turbo - key: ${{ runner.os }}-turbo-unit-and-integration-test:test-type-${{ github.sha }} - - name: Build type cache - if: steps.turbo-cache.outputs.cache-hit != 'true' - uses: actions/cache@v5 - with: - path: .turbo - key: ${{ runner.os }}-turbo-build:type-${{ github.sha }} + cache-prefix: turbo-cache- - name: Test Type Test - run: yarn test:test-type --cache-dir=".turbo" + run: yarn test:test-type # Not using matrix because test later on cant needs a specific job bot-unit-test: diff --git a/.github/workflows/other-runtime-unit-test.yml b/.github/workflows/other-runtime-unit-test.yml index 0b5061ab0..d6007d344 100644 --- a/.github/workflows/other-runtime-unit-test.yml +++ b/.github/workflows/other-runtime-unit-test.yml @@ -22,20 +22,12 @@ jobs: with: deno-version: 'v2.1.x' - run: yarn install --immutable - - name: Turbo Cache - id: turbo-cache - uses: actions/cache@v5 + - name: Cache for Turbo + uses: rharkor/caching-for-turbo@v2.3.5 with: - path: .turbo - key: ${{ runner.os }}-turbo-test:deno-unit-${{ inputs.package }}-${{ github.sha }} - - name: Build dist cache - if: steps.turbo-cache.outputs.cache-hit != 'true' - uses: actions/cache@v5 - with: - path: .turbo - key: ${{ runner.os }}-turbo-build-${{ github.sha }} + cache-prefix: turbo-cache- - name: Deno Unit Test - run: yarn test:deno-unit --cache-dir=".turbo" --filter=./packages/${{ inputs.package }} + run: yarn test:deno-unit --filter=./packages/${{ inputs.package }} bun-unit-test: name: Bun Unit Test runs-on: ubuntu-latest @@ -49,18 +41,10 @@ jobs: with: bun-version: '1.1.42' - run: yarn install --immutable - - name: Turbo Cache - id: turbo-cache - uses: actions/cache@v5 + - name: Cache for Turbo + uses: rharkor/caching-for-turbo@v2.3.5 with: - path: .turbo - key: ${{ runner.os }}-turbo-test:bun-unit-${{ inputs.package }}-${{ github.sha }} - - name: Build dist cache - if: steps.turbo-cache.outputs.cache-hit != 'true' - uses: actions/cache@v5 - with: - path: .turbo - key: ${{ runner.os }}-turbo-build-${{ github.sha }} + cache-prefix: turbo-cache- - name: Bun Unit Test - run: yarn test:bun-unit --cache-dir=".turbo" --filter=./packages/${{ inputs.package }} + run: yarn test:bun-unit --filter=./packages/${{ inputs.package }} timeout-minutes: 1 diff --git a/.github/workflows/site.yml b/.github/workflows/site.yml index 79c07575a..43af9cd55 100644 --- a/.github/workflows/site.yml +++ b/.github/workflows/site.yml @@ -4,12 +4,6 @@ permissions: on: push: branches: ['main'] - paths: - - 'website/**' - - '.github/workflows/site.yml' - - 'typedoc.json' - - 'package.json' - - 'packages/**' pull_request: workflow_dispatch: @@ -31,10 +25,19 @@ jobs: website/yarn.lock - name: yarn install (root) run: yarn install --immutable + - name: Cache for Turbo + uses: rharkor/caching-for-turbo@v2.3.5 + with: + cache-prefix: turbo-cache- + - name: Cache for Turbo + uses: rharkor/caching-for-turbo@v2.3.5 + with: + cache-prefix: turbo-cache-website- + server-port: 41231 # The default is 41230, but we use a different cache key so we need a different port - name: Build all pkg run: yarn release-build - name: Build docs - run: yarn build:doc + run: yarn turbo build:doc --single-package - run: yarn install --immutable working-directory: ./website - name: Restore bundler cache @@ -45,8 +48,10 @@ jobs: restore-keys: | ${{ runner.os }}-docusaurus-bundler- - name: Build website - run: yarn build + run: yarn turbo build working-directory: ./website + env: + TURBO_API: http://localhost:41231 - name: Upload artifact uses: actions/upload-pages-artifact@v4 with: @@ -64,10 +69,14 @@ jobs: website/yarn.lock - name: yarn install (root) run: yarn install --immutable + - name: Cache for Turbo + uses: rharkor/caching-for-turbo@v2.3.5 + with: + cache-prefix: turbo-cache- - name: Build all pkg run: yarn release-build - name: Build docs - run: yarn build:doc + run: yarn turbo build:doc --single-package - run: yarn install --immutable working-directory: ./website - run: yarn typecheck diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index 326708ea2..fd9945f3e 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -19,26 +19,12 @@ jobs: node-version: 20 cache: yarn - run: yarn install --immutable - - name: Turbo Cache - id: turbo-cache - uses: actions/cache@v5 + - name: Cache for Turbo + uses: rharkor/caching-for-turbo@v2.3.5 with: - path: .turbo - key: ${{ runner.os }}-turbo-test:unit-${{ inputs.package }}-${{ github.sha }} - - name: Build dist cache - if: steps.turbo-cache.outputs.cache-hit != 'true' - uses: actions/cache@v5 - with: - path: .turbo - key: ${{ runner.os }}-turbo-build-${{ github.sha }} - - name: Build type cache - if: steps.turbo-cache.outputs.cache-hit != 'true' - uses: actions/cache@v5 - with: - path: .turbo - key: ${{ runner.os }}-turbo-build:type-${{ github.sha }} + cache-prefix: turbo-cache- - name: Unit Test - run: yarn test:unit --cache-dir=".turbo" --filter=./packages/${{ inputs.package }} + run: yarn test:unit --filter=./packages/${{ inputs.package }} timeout-minutes: 1 - name: Collect and upload the coverage report uses: codecov/codecov-action@v5 diff --git a/turbo.json b/turbo.json index 48e5c9456..50d37da4f 100644 --- a/turbo.json +++ b/turbo.json @@ -38,6 +38,11 @@ "check": { "dependsOn": ["build:type"], "cache": false + }, + "build:doc": { + // NOTE: This task can only be run with "--singe-package" in the flags to turbo, as it is defined only in the root package.json + "inputs": ["packages/**/src/**/*.ts"], + "outputs": ["website/api_reference/generated/**/*"] } } } diff --git a/website/package.json b/website/package.json index 9cf09402f..1d1af521c 100644 --- a/website/package.json +++ b/website/package.json @@ -1,6 +1,7 @@ { "name": "website", "version": "0.0.0", + "packageManager": "yarn@4.12.0", "private": true, "scripts": { "docusaurus": "docusaurus", @@ -35,6 +36,7 @@ "@docusaurus/tsconfig": "^3.9.2", "@docusaurus/types": "^3.9.2", "@types/react": "^19.2.7", + "turbo": "^2.7.2", "typescript": "5.9.3", "webpack": "5.104.1" }, diff --git a/website/turbo.json b/website/turbo.json new file mode 100644 index 000000000..8130fde5d --- /dev/null +++ b/website/turbo.json @@ -0,0 +1,9 @@ +{ + "$schema": "https://turbo.build/schema.json", + "tasks": { + "build": { + "outputs": ["build/**"], + "inputs": ["src/**/*", "static/**/*", "blog/**/*", "docs/**/*", "api_reference/**/*", "docusaurus.config.ts", "sidebars.ts"] + } + } +} diff --git a/website/yarn.lock b/website/yarn.lock index 5d49801d2..833cf40af 100644 --- a/website/yarn.lock +++ b/website/yarn.lock @@ -19935,6 +19935,77 @@ __metadata: languageName: node linkType: hard +"turbo-darwin-64@npm:2.7.2": + version: 2.7.2 + resolution: "turbo-darwin-64@npm:2.7.2" + conditions: os=darwin & cpu=x64 + languageName: node + linkType: hard + +"turbo-darwin-arm64@npm:2.7.2": + version: 2.7.2 + resolution: "turbo-darwin-arm64@npm:2.7.2" + conditions: os=darwin & cpu=arm64 + languageName: node + linkType: hard + +"turbo-linux-64@npm:2.7.2": + version: 2.7.2 + resolution: "turbo-linux-64@npm:2.7.2" + conditions: os=linux & cpu=x64 + languageName: node + linkType: hard + +"turbo-linux-arm64@npm:2.7.2": + version: 2.7.2 + resolution: "turbo-linux-arm64@npm:2.7.2" + conditions: os=linux & cpu=arm64 + languageName: node + linkType: hard + +"turbo-windows-64@npm:2.7.2": + version: 2.7.2 + resolution: "turbo-windows-64@npm:2.7.2" + conditions: os=win32 & cpu=x64 + languageName: node + linkType: hard + +"turbo-windows-arm64@npm:2.7.2": + version: 2.7.2 + resolution: "turbo-windows-arm64@npm:2.7.2" + conditions: os=win32 & cpu=arm64 + languageName: node + linkType: hard + +"turbo@npm:^2.7.2": + version: 2.7.2 + resolution: "turbo@npm:2.7.2" + dependencies: + turbo-darwin-64: "npm:2.7.2" + turbo-darwin-arm64: "npm:2.7.2" + turbo-linux-64: "npm:2.7.2" + turbo-linux-arm64: "npm:2.7.2" + turbo-windows-64: "npm:2.7.2" + turbo-windows-arm64: "npm:2.7.2" + dependenciesMeta: + turbo-darwin-64: + optional: true + turbo-darwin-arm64: + optional: true + turbo-linux-64: + optional: true + turbo-linux-arm64: + optional: true + turbo-windows-64: + optional: true + turbo-windows-arm64: + optional: true + bin: + turbo: bin/turbo + checksum: 10c0/390937a32070ac52120af56705006e831de2edf8e22a2b0146006d07740cd413b3a36335339e7617b57cf74b008f4b15931cf56e1c7c0c03808f028ed937c4de + languageName: node + linkType: hard + "type-fest@npm:^0.21.3": version: 0.21.3 resolution: "type-fest@npm:0.21.3" @@ -20803,6 +20874,7 @@ __metadata: prism-react-renderer: "npm:^2.4.1" react: "npm:^19.2.3" react-dom: "npm:^19.2.3" + turbo: "npm:^2.7.2" typescript: "npm:5.9.3" webpack: "npm:5.104.1" languageName: unknown