feat: experimental new docs gen (#1240)

.eslintignore

@@ -10,3 +10,5 @@ payloads/v8/*
 rest/v8/*
 utils/v8.ts
 v8.ts
+
+djs/**/*

.github/workflows/documentation.yml

@@ -0,0 +1,235 @@
+name: Documentation
+
+on:
+  push:
+    branches:
+      - 'main'
+    paths:
+      - '**'
+      - '!website/**'
+    tags:
+      - '**'
+  workflow_dispatch:
+    inputs:
+      ref:
+        description: 'The branch, tag or SHA to checkout'
+        required: true
+      ref_type:
+        type: choice
+        description: 'Branch or tag'
+        options:
+          - branch
+          - tag
+        required: true
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build-docs:
+    name: Build & upload documentation
+    runs-on: ubuntu-latest
+    env:
+      REF_TYPE: ${{ inputs.ref_type || github.ref_type }}
+    if: github.repository_owner == 'discordjs'
+
+    steps:
+      - name: Checkout discord-api-types
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || '' }}
+
+      - name: Install Node.js v22
+        uses: actions/setup-node@v4
+        with:
+          node-version: 22
+
+      - name: Install dependencies for discord-api-types
+        run: npm ci
+
+      #region DJS start (mostly from discord.js/packages/actions/src/pnpmCache)
+
+      - name: Checkout discord.js
+        uses: actions/checkout@v4
+        with:
+          repository: discordjs/discord.js
+          path: djs
+
+      - name: Set up swap space
+        if: runner.os == 'Linux'
+        uses: pierotofy/set-swap-space@v1.0
+        with:
+          swap-size-gb: 10
+
+      - uses: pnpm/action-setup@v4.1.0
+        name: Install pnpm
+        with:
+          run_install: false
+
+      - name: Expose pnpm config(s) through "$GITHUB_OUTPUT"
+        id: pnpm-config
+        shell: bash
+        run: |
+          echo "STORE_PATH=$(pnpm store path)" >> $GITHUB_OUTPUT
+
+      - name: Cache rotation keys
+        id: cache-rotation
+        shell: bash
+        run: |
+          echo "YEAR_MONTH=$(/bin/date -u "+%Y%m")" >> $GITHUB_OUTPUT
+
+      - uses: actions/cache@v4
+        name: Setup pnpm cache
+        with:
+          path: ${{ steps.pnpm-config.outputs.STORE_PATH }}
+          key: ${{ runner.os }}-pnpm-store-cache-${{ steps.cache-rotation.outputs.YEAR_MONTH }}-${{ hashFiles('**/pnpm-lock.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-pnpm-store-cache-${{ steps.cache-rotation.outputs.YEAR_MONTH }}-
+
+      - name: Install djs dependencies
+        working-directory: djs
+        shell: bash
+        run: |
+          pnpm install --frozen-lockfile --prefer-offline --loglevel error
+        env:
+          HUSKY: '0'
+
+      - name: Build djs
+        working-directory: djs
+        shell: bash
+        run: |
+          pnpm run build
+
+      #endregion
+
+      - name: Extract package and semver from tag
+        if: ${{ env.REF_TYPE == 'tag' }}
+        id: extract-tag
+        uses: ./djs/packages/actions/src/formatTag
+        with:
+          tag: ${{ inputs.ref || github.ref_name }}
+
+      - name: Apply tag to api-extractor config
+        if: ${{ env.REF_TYPE == 'tag' && !inputs.ref }}
+        run: sed -i 's!https://github.com/discordjs/discord-api-types/tree/main!https://github.com/discordjs/discord-api-types/tree/${{ github.ref_name }}!' "api-extractor.json"
+
+      - name: Build docs
+        shell: bash
+        run: |
+          npm run prepublishOnly
+          ./djs/packages/api-extractor/bin/api-extractor run --local --minify
+          ./djs/packages/scripts/bin/generateSplitDocumentation.js
+          npm run postpublish
+
+      - name: Upload documentation to database
+        if: ${{ env.REF_TYPE == 'tag' && (!inputs.ref || inputs.ref == 'main') }}
+        env:
+          CF_D1_DOCS_API_KEY: ${{ secrets.CF_D1_DOCS_API_KEY }}
+          CF_D1_DOCS_ID: ${{ secrets.CF_D1_DOCS_ID }}
+          CF_ACCOUNT_ID: ${{ secrets.CF_ACCOUNT_ID }}
+          CF_R2_DOCS_URL: ${{ secrets.CF_R2_DOCS_URL }}
+          CF_R2_DOCS_ACCESS_KEY_ID: ${{ secrets.CF_R2_DOCS_ACCESS_KEY_ID }}
+          CF_R2_DOCS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_DOCS_SECRET_ACCESS_KEY }}
+          CF_R2_DOCS_BUCKET: ${{ secrets.CF_R2_DOCS_BUCKET }}
+          CF_R2_DOCS_BUCKET_URL: ${{ secrets.CF_R2_DOCS_BUCKET_URL }}
+          ACTION_PACKAGE: ${{ steps.extract-tag.outputs.package }}
+          ACTION_VERSION: ${{ steps.extract-tag.outputs.semver }}
+        run: |
+          npx tsx ./scripts/actions/documentation/uploadDocumentation.ts
+
+      - name: Upload documentation to database
+        if: ${{ env.REF_TYPE == 'tag' && inputs.ref && inputs.ref != 'main' }}
+        env:
+          CF_D1_DOCS_API_KEY: ${{ secrets.CF_D1_DOCS_API_KEY }}
+          CF_D1_DOCS_ID: ${{ secrets.CF_D1_DOCS_ID }}
+          CF_ACCOUNT_ID: ${{ secrets.CF_ACCOUNT_ID }}
+          CF_R2_DOCS_URL: ${{ secrets.CF_R2_DOCS_URL }}
+          CF_R2_DOCS_ACCESS_KEY_ID: ${{ secrets.CF_R2_DOCS_ACCESS_KEY_ID }}
+          CF_R2_DOCS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_DOCS_SECRET_ACCESS_KEY }}
+          CF_R2_DOCS_BUCKET: ${{ secrets.CF_R2_DOCS_BUCKET }}
+          CF_R2_DOCS_BUCKET_URL: ${{ secrets.CF_R2_DOCS_BUCKET_URL }}
+          ACTION_PACKAGE: ${{ steps.extract-tag.outputs.package }}
+          ACTION_VERSION: ${{ steps.extract-tag.outputs.semver }}
+        run: |
+          npx tsx ./scripts/actions/documentation/uploadDocumentation.ts
+
+      - name: Upload split documentation to blob storage
+        if: ${{ env.REF_TYPE == 'tag' && (!inputs.ref || inputs.ref == 'main') }}
+        env:
+          CF_R2_DOCS_URL: ${{ secrets.CF_R2_DOCS_URL }}
+          CF_R2_DOCS_ACCESS_KEY_ID: ${{ secrets.CF_R2_DOCS_ACCESS_KEY_ID }}
+          CF_R2_DOCS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_DOCS_SECRET_ACCESS_KEY }}
+          CF_R2_DOCS_BUCKET: ${{ secrets.CF_R2_DOCS_BUCKET }}
+          ACTION_PACKAGE: ${{ steps.extract-tag.outputs.package }}
+          ACTION_VERSION: ${{ steps.extract-tag.outputs.semver }}
+        run: |
+          npx tsx ./scripts/actions/documentation/uploadSplitDocumentation.ts
+
+      - name: Upload split documentation to blob storage
+        if: ${{ env.REF_TYPE == 'tag' && inputs.ref && inputs.ref != 'main' }}
+        env:
+          CF_R2_DOCS_URL: ${{ secrets.CF_R2_DOCS_URL }}
+          CF_R2_DOCS_ACCESS_KEY_ID: ${{ secrets.CF_R2_DOCS_ACCESS_KEY_ID }}
+          CF_R2_DOCS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_DOCS_SECRET_ACCESS_KEY }}
+          CF_R2_DOCS_BUCKET: ${{ secrets.CF_R2_DOCS_BUCKET }}
+          ACTION_PACKAGE: ${{ steps.extract-tag.outputs.package }}
+          ACTION_VERSION: ${{ steps.extract-tag.outputs.semver }}
+        run: |
+          npx tsx ./scripts/actions/documentation/uploadSplitDocumentation.ts
+
+      - name: Upload documentation to database
+        if: ${{ env.REF_TYPE == 'branch' && (!inputs.ref || inputs.ref == 'main') }}
+        env:
+          CF_D1_DOCS_API_KEY: ${{ secrets.CF_D1_DOCS_API_KEY }}
+          CF_D1_DOCS_ID: ${{ secrets.CF_D1_DOCS_ID }}
+          CF_ACCOUNT_ID: ${{ secrets.CF_ACCOUNT_ID }}
+          CF_R2_DOCS_URL: ${{ secrets.CF_R2_DOCS_URL }}
+          CF_R2_DOCS_ACCESS_KEY_ID: ${{ secrets.CF_R2_DOCS_ACCESS_KEY_ID }}
+          CF_R2_DOCS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_DOCS_SECRET_ACCESS_KEY }}
+          CF_R2_DOCS_BUCKET: ${{ secrets.CF_R2_DOCS_BUCKET }}
+          CF_R2_DOCS_BUCKET_URL: ${{ secrets.CF_R2_DOCS_BUCKET_URL }}
+          ACTION_PACKAGE: ${{ steps.extract-tag.outputs.package }}
+          ACTION_VERSION: ${{ steps.extract-tag.outputs.semver }}
+        run: |
+          npx tsx ./scripts/actions/documentation/uploadDocumentation.ts
+
+      - name: Upload documentation to database
+        if: ${{ env.REF_TYPE == 'branch' && inputs.ref && inputs.ref != 'main' }}
+        env:
+          CF_D1_DOCS_API_KEY: ${{ secrets.CF_D1_DOCS_API_KEY }}
+          CF_D1_DOCS_ID: ${{ secrets.CF_D1_DOCS_ID }}
+          CF_ACCOUNT_ID: ${{ secrets.CF_ACCOUNT_ID }}
+          CF_R2_DOCS_URL: ${{ secrets.CF_R2_DOCS_URL }}
+          CF_R2_DOCS_ACCESS_KEY_ID: ${{ secrets.CF_R2_DOCS_ACCESS_KEY_ID }}
+          CF_R2_DOCS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_DOCS_SECRET_ACCESS_KEY }}
+          CF_R2_DOCS_BUCKET: ${{ secrets.CF_R2_DOCS_BUCKET }}
+          CF_R2_DOCS_BUCKET_URL: ${{ secrets.CF_R2_DOCS_BUCKET_URL }}
+          ACTION_PACKAGE: ${{ steps.extract-tag.outputs.package }}
+          ACTION_VERSION: ${{ steps.extract-tag.outputs.semver }}
+        run: |
+          npx tsx ./scripts/actions/documentation/uploadDocumentation.ts
+
+      - name: Upload split documentation to blob storage
+        if: ${{ env.REF_TYPE == 'branch' && (!inputs.ref || inputs.ref == 'main') }}
+        env:
+          CF_R2_DOCS_URL: ${{ secrets.CF_R2_DOCS_URL }}
+          CF_R2_DOCS_ACCESS_KEY_ID: ${{ secrets.CF_R2_DOCS_ACCESS_KEY_ID }}
+          CF_R2_DOCS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_DOCS_SECRET_ACCESS_KEY }}
+          CF_R2_DOCS_BUCKET: ${{ secrets.CF_R2_DOCS_BUCKET }}
+          ACTION_PACKAGE: ${{ steps.extract-tag.outputs.package }}
+          ACTION_VERSION: ${{ steps.extract-tag.outputs.semver }}
+        run: |
+          npx tsx ./scripts/actions/documentation/uploadSplitDocumentation.ts
+
+      - name: Upload split documentation to blob storage
+        if: ${{ env.REF_TYPE == 'branch' && inputs.ref && inputs.ref != 'main' }}
+        env:
+          CF_R2_DOCS_URL: ${{ secrets.CF_R2_DOCS_URL }}
+          CF_R2_DOCS_ACCESS_KEY_ID: ${{ secrets.CF_R2_DOCS_ACCESS_KEY_ID }}
+          CF_R2_DOCS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_DOCS_SECRET_ACCESS_KEY }}
+          CF_R2_DOCS_BUCKET: ${{ secrets.CF_R2_DOCS_BUCKET }}
+          ACTION_PACKAGE: ${{ steps.extract-tag.outputs.package }}
+          ACTION_VERSION: ${{ steps.extract-tag.outputs.semver }}
+        run: |
+          npx tsx ./scripts/actions/documentation/uploadSplitDocumentation.ts

.gitignore

@@ -62,3 +62,6 @@ docs/*
 
 # macOS files
 .DS_Store
+
+# djs repo clone
+djs

package.json

@@ -3,6 +3,9 @@
 	"version": "0.38.5",
 	"description": "Discord API typings that are kept up to date for use in bot library creation.",
 	"homepage": "https://discord-api-types.dev",
+	"workspaces": [
+		"scripts/actions/documentation"
+	],
 	"exports": {
 		"./globals": {
 			"types": "./globals.d.ts",
@@ -151,6 +154,7 @@
 		"pretty-quick": "^4.0.0",
 		"rimraf": "^6.0.0",
 		"tsutils": "^3.21.0",
+		"tsx": "^4.19.4",
 		"typescript": "^5.6.3"
 	},
 	"publishConfig": {

scripts/actions/documentation/package.json

@@ -0,0 +1,12 @@
+{
+	"type": "module",
+	"private": true,
+	"devDependencies": {
+		"@actions/core": "^1.11.1",
+		"@actions/glob": "^0.5.0",
+		"@aws-sdk/client-s3": "^3.808.0",
+		"cloudflare": "^4.2.0",
+		"p-limit": "^6.2.0",
+		"p-queue": "^8.1.0"
+	}
+}

scripts/actions/documentation/uploadDocumentation.ts

@@ -0,0 +1,80 @@
+// TODO: rely on cloned djs in the near future
+
+import { readFile } from 'node:fs/promises';
+import process from 'node:process';
+import { create } from '@actions/glob';
+import { PutObjectCommand, S3Client } from '@aws-sdk/client-s3';
+import Cloudflare from 'cloudflare';
+import pLimit from 'p-limit';
+
+if (
+	!process.env.CF_R2_DOCS_URL ||
+	!process.env.CF_R2_DOCS_ACCESS_KEY_ID ||
+	!process.env.CF_R2_DOCS_SECRET_ACCESS_KEY ||
+	!process.env.CF_R2_DOCS_BUCKET ||
+	!process.env.CF_R2_DOCS_BUCKET_URL ||
+	!process.env.CF_D1_DOCS_API_KEY ||
+	!process.env.CF_D1_DOCS_ID ||
+	!process.env.CF_ACCOUNT_ID
+) {
+	throw new Error('Missing environment variables');
+}
+
+const version = process.env.ACTION_VERSION || 'main';
+
+const S3 = new S3Client({
+	region: 'auto',
+	endpoint: process.env.CF_R2_DOCS_URL!,
+	credentials: {
+		accessKeyId: process.env.CF_R2_DOCS_ACCESS_KEY_ID!,
+		secretAccessKey: process.env.CF_R2_DOCS_SECRET_ACCESS_KEY!,
+	},
+	requestChecksumCalculation: 'WHEN_REQUIRED',
+	responseChecksumValidation: 'WHEN_REQUIRED',
+});
+
+const client = new Cloudflare({
+	apiToken: process.env.CF_D1_DOCS_API_KEY,
+});
+
+const limit = pLimit(10);
+const promises: Promise<void>[] = [];
+
+const globber = await create(`docs/docs.api.json`);
+console.log('Glob: ', await globber.glob());
+for await (const file of globber.globGenerator()) {
+	const data = await readFile(file, 'utf8');
+	try {
+		promises.push(
+			limit(async () => {
+				console.log(`Uploading ${file} with ${version}...`);
+				const json = JSON.parse(data);
+				const name = json.name ?? json.n;
+
+				const key = `${name.replace('@discordjs/', '')}/${version}.json`;
+
+				await S3.send(
+					new PutObjectCommand({
+						Bucket: process.env.CF_R2_DOCS_BUCKET,
+						Key: key,
+						Body: data,
+					}),
+				);
+
+				await client.d1.database.raw(process.env.CF_D1_DOCS_ID!, {
+					account_id: process.env.CF_ACCOUNT_ID!,
+					sql: `insert into documentation (name, version, url) values (?, ?, ?) on conflict (name, version) do update set url = excluded.url;`,
+					params: [name.replace('@discordjs/', ''), version, process.env.CF_R2_DOCS_BUCKET_URL + '/' + key],
+				});
+			}),
+		);
+	} catch (error) {
+		console.log(error);
+	}
+}
+
+try {
+	await Promise.all(promises);
+} catch (error) {
+	console.log(error);
+}

scripts/actions/documentation/uploadSplitDocumentation.ts

@@ -0,0 +1,94 @@
+// TODO: rely on cloned djs in the near future
+
+import { readFile } from 'node:fs/promises';
+import { basename, dirname, relative, sep } from 'node:path';
+import process from 'node:process';
+import { setTimeout as sleep } from 'node:timers/promises';
+import { create } from '@actions/glob';
+import { S3Client, PutObjectCommand } from '@aws-sdk/client-s3';
+import PQueue from 'p-queue';
+
+if (
+	!process.env.CF_R2_DOCS_URL ||
+	!process.env.CF_R2_DOCS_ACCESS_KEY_ID ||
+	!process.env.CF_R2_DOCS_SECRET_ACCESS_KEY ||
+	!process.env.CF_R2_DOCS_BUCKET
+) {
+	throw new Error('Missing environment variables');
+}
+
+const pkg = process.env.ACTION_PACKAGE || '*';
+const version = process.env.ACTION_VERSION || 'main';
+
+const queue = new PQueue({ concurrency: 10, interval: 60_000, intervalCap: 1_000 });
+const promises: Promise<void>[] = [];
+const failedUploads: string[] = [];
+
+const S3 = new S3Client({
+	region: 'auto',
+	endpoint: process.env.CF_R2_DOCS_URL!,
+	credentials: {
+		accessKeyId: process.env.CF_R2_DOCS_ACCESS_KEY_ID!,
+		secretAccessKey: process.env.CF_R2_DOCS_SECRET_ACCESS_KEY!,
+	},
+	requestChecksumCalculation: 'WHEN_REQUIRED',
+	responseChecksumValidation: 'WHEN_REQUIRED',
+});
+
+const globber = await create(`docs/${pkg}/split/*.api.json`);
+console.log('Glob: ', await globber.glob());
+for await (const file of globber.globGenerator()) {
+	const data = await readFile(file, 'utf8');
+	const pkgName = dirname(relative(process.cwd(), file)).split(sep)[1];
+	try {
+		promises.push(
+			queue.add(async () => {
+				console.log(`Uploading ${file} with ${version} from ${pkgName}...`);
+				const name = basename(file).replace('main.', '');
+				async function upload(retries = 0) {
+					try {
+						await S3.send(
+							new PutObjectCommand({
+								Bucket: process.env.CF_R2_DOCS_BUCKET,
+								Key: `${pkgName}/${version}.${name}`,
+								Body: data,
+							}),
+						);
+					} catch (error) {
+						if (retries > 3) {
+							console.error(`Could not upload ${file} after 3 retries`, error);
+							failedUploads.push(name);
+							return;
+						}
+
+						if (
+							typeof error === 'object' &&
+							error &&
+							'retryAfter' in error &&
+							typeof error.retryAfter === 'number'
+						) {
+							await sleep(error.retryAfter * 1_000);
+							return upload(retries + 1);
+						} else {
+							console.error(`Could not upload ${file}`, error);
+							failedUploads.push(name);
+						}
+					}
+				}
+
+				await upload();
+			}),
+		);
+	} catch (error) {
+		console.log(error);
+	}
+}
+
+try {
+	await Promise.all(promises);
+	if (failedUploads.length) {
+		throw new Error(`Failed to upload ${failedUploads.length} files: ${failedUploads.join(', ')}`);
+	}
+} catch (error) {
+	console.log(error);
+}

scripts/tsconfig.json

@@ -2,8 +2,8 @@
 	"extends": "../tsconfig.json",
 	"compilerOptions": {
 		"target": "ESNext",
-		"module": "ESNext",
-		"moduleResolution": "Node",
+		"module": "NodeNext",
+		"moduleResolution": "NodeNext",
 		"allowSyntheticDefaultImports": true,
 		"checkJs": true
 	},