From fe43cc8f88effdb43151628de02c9f8944f939af Mon Sep 17 00:00:00 2001 From: ayntee Date: Wed, 23 Dec 2020 18:12:12 +0400 Subject: [PATCH] =?UTF-8?q?Some=20updates=E2=80=95renaming=20interfaces=20?= =?UTF-8?q?and=20files?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/bot.ts | 18 +- src/interactions/mod.ts | 2 +- .../{interactions.ts => server.ts} | 0 src/rest/mod.ts | 419 +---------------- src/rest/request_manager.ts | 418 +++++++++++++++++ src/types/options.ts | 2 +- src/util/cdn.ts | 10 - src/util/utils.ts | 18 +- src/ws/mod.ts | 438 +----------------- src/ws/shard.ts | 436 +++++++++++++++++ 10 files changed, 884 insertions(+), 877 deletions(-) rename src/interactions/{interactions.ts => server.ts} (100%) create mode 100644 src/rest/request_manager.ts delete mode 100644 src/util/cdn.ts create mode 100644 src/ws/shard.ts diff --git a/src/bot.ts b/src/bot.ts index 7472ffdef..4fa69b282 100644 --- a/src/bot.ts +++ b/src/bot.ts @@ -1,10 +1,10 @@ +import { RequestManager } from "./rest/mod.ts"; import { - ClientOptions, + BotConfig, DiscordBotGatewayData, EventHandlers, } from "./types/types.ts"; import { baseEndpoints, endpoints } from "./util/constants.ts"; -import { RequestManager } from "./rest/mod.ts"; import { spawnShards } from "./ws/shard_manager.ts"; export let authorization = ""; @@ -39,9 +39,9 @@ export interface IdentifyPayload { shard: [number, number]; } -export async function startBot(data: ClientOptions) { - if (data.eventHandlers) eventHandlers = data.eventHandlers; - authorization = `Bot ${data.token}`; +export async function startBot(config: BotConfig) { + if (config.eventHandlers) eventHandlers = config.eventHandlers; + authorization = `Bot ${config.token}`; // Initial API connection to get info about bots connection botGatewayData = await RequestManager.get( @@ -49,8 +49,8 @@ export async function startBot(data: ClientOptions) { ) as DiscordBotGatewayData; proxyWSURL = botGatewayData.url; - identifyPayload.token = data.token; - identifyPayload.intents = data.intents.reduce( + identifyPayload.token = config.token; + identifyPayload.intents = config.intents.reduce( (bits, next) => (bits |= next), 0, ); @@ -75,7 +75,7 @@ export function setBotID(id: string) { * * Advanced Devs: This function will allow you to have an insane amount of customization potential as when you get to large bots you need to be able to optimize every tiny detail to make you bot work the way you need. */ -export async function startBigBrainBot(data: BigBrainBotOptions) { +export async function startBigBrainBot(data: BigBrainBotConfig) { authorization = `Bot ${data.token}`; identifyPayload.token = `Bot ${data.token}`; @@ -108,7 +108,7 @@ export async function startBigBrainBot(data: BigBrainBotOptions) { ); } -export interface BigBrainBotOptions extends ClientOptions { +export interface BigBrainBotConfig extends BotConfig { /** The first shard to start at for this worker. Use this to control which shards to run in each worker. */ firstShardID: number; /** The last shard to start for this worker. By default it will be 25 + the firstShardID. */ diff --git a/src/interactions/mod.ts b/src/interactions/mod.ts index 47e0d23e7..1eff735d1 100644 --- a/src/interactions/mod.ts +++ b/src/interactions/mod.ts @@ -1,2 +1,2 @@ -export * from "./interactions.ts"; +export * from "./server.ts"; export * from "./types/mod.ts"; diff --git a/src/interactions/interactions.ts b/src/interactions/server.ts similarity index 100% rename from src/interactions/interactions.ts rename to src/interactions/server.ts diff --git a/src/rest/mod.ts b/src/rest/mod.ts index e722f9a9b..bc5c8ff99 100644 --- a/src/rest/mod.ts +++ b/src/rest/mod.ts @@ -1,418 +1 @@ -import { Errors, HttpResponseCode, RequestMethods } from "../types/types.ts"; -import { baseEndpoints, discordAPIURLS } from "../util/constants.ts"; -import { delay } from "../util/utils.ts"; -import { authorization, eventHandlers } from "../bot.ts"; - -const pathQueues: { [key: string]: QueuedRequest[] } = {}; -const ratelimitedPaths = new Map(); -let globallyRateLimited = false; -let queueInProcess = false; - -export interface QueuedRequest { - callback: () => Promise< - void | { - rateLimited: any; - beforeFetch: boolean; - bucketID?: string | null; - } - >; - bucketID?: string | null; - url: string; -} - -export interface RateLimitedPath { - url: string; - resetTimestamp: number; - bucketID: string | null; -} - -async function processRateLimitedPaths() { - const now = Date.now(); - ratelimitedPaths.forEach((value, key) => { - if (value.resetTimestamp > now) return; - ratelimitedPaths.delete(key); - if (key === "global") globallyRateLimited = false; - }); - - await delay(1000); - processRateLimitedPaths(); -} - -function addToQueue(request: QueuedRequest) { - const route = request.url.substring(baseEndpoints.BASE_URL.length + 1); - const parts = route.split("/"); - // Remove the major param - parts.shift(); - const [id] = parts; - - if (pathQueues[id]) { - pathQueues[id].push(request); - } else { - pathQueues[id] = [request]; - } -} - -async function cleanupQueues() { - Object.entries(pathQueues).map(([key, value]) => { - if (!value.length) { - // Remove it entirely - delete pathQueues[key]; - } - }); -} - -async function processQueue() { - while (queueInProcess) { - if ( - (Object.keys(pathQueues).length) && !globallyRateLimited - ) { - await Promise.allSettled( - Object.values(pathQueues).map(async (pathQueue) => { - const request = pathQueue.shift(); - if (!request) return; - - const rateLimitedURLResetIn = await checkRatelimits(request.url); - - if (request.bucketID) { - const rateLimitResetIn = await checkRatelimits(request.bucketID); - if (rateLimitResetIn) { - // This request is still rate limited readd to queue - addToQueue(request); - } else if (rateLimitedURLResetIn) { - // This URL is rate limited readd to queue - addToQueue(request); - } else { - // This request is not rate limited so it should be run - const result = await request.callback(); - if (result && result.rateLimited) { - addToQueue( - { ...request, bucketID: result.bucketID || request.bucketID }, - ); - } - } - } else { - if (rateLimitedURLResetIn) { - // This URL is rate limited readd to queue - addToQueue(request); - } else { - // This request has no bucket id so it should be processed - const result = await request.callback(); - if (request && result && result.rateLimited) { - addToQueue( - { ...request, bucketID: result.bucketID || request.bucketID }, - ); - } - } - } - }), - ); - } - - if (Object.keys(pathQueues).length) { - cleanupQueues(); - } else queueInProcess = false; - } -} - -processRateLimitedPaths(); - -export const RequestManager = { - get: async (url: string, body?: unknown) => { - return runMethod("get", url, body); - }, - post: (url: string, body?: unknown) => { - return runMethod("post", url, body); - }, - delete: (url: string, body?: unknown) => { - return runMethod("delete", url, body); - }, - patch: (url: string, body?: unknown) => { - return runMethod("patch", url, body); - }, - put: (url: string, body?: unknown) => { - return runMethod("put", url, body); - }, -}; - -function createRequestBody(body: any, method: RequestMethods) { - const headers: { [key: string]: string } = { - Authorization: authorization, - "User-Agent": - `DiscordBot (https://github.com/skillz4killz/discordeno, v10)`, - }; - - if (method === "get") body = undefined; - - if (body?.reason) { - headers["X-Audit-Log-Reason"] = encodeURIComponent(body.reason); - } - - if (body?.file) { - const form = new FormData(); - form.append("file", body.file.blob, body.file.name); - form.append("payload_json", JSON.stringify({ ...body, file: undefined })); - body.file = form; - } else if ( - body && !["get", "delete"].includes(method) - ) { - headers["Content-Type"] = "application/json"; - } - - return { - headers, - body: body?.file || JSON.stringify(body), - method: method.toUpperCase(), - }; -} - -async function checkRatelimits(url: string) { - const ratelimited = ratelimitedPaths.get(url); - const global = ratelimitedPaths.get("global"); - const now = Date.now(); - - if (ratelimited && now < ratelimited.resetTimestamp) { - return ratelimited.resetTimestamp - now; - } - if (global && now < global.resetTimestamp) { - return global.resetTimestamp - now; - } - - return false; -} - -async function runMethod( - method: RequestMethods, - url: string, - body?: unknown, - retryCount = 0, - bucketID?: string | null, -) { - eventHandlers.debug?.( - { - type: "requestManager", - data: { method, url, body, retryCount, bucketID }, - }, - ); - - const errorStack = new Error("Location:"); - Error.captureStackTrace(errorStack); - - // For proxies we don't need to do any of the legwork so we just forward the request - if ( - !url.startsWith(discordAPIURLS.BASE_URL) && - !url.startsWith(discordAPIURLS.CDN_URL) - ) { - return fetch(url, { method, body: body ? JSON.stringify(body) : undefined }) - .then((res) => res.json()) - .catch((error) => { - console.error(error); - throw errorStack; - }); - } - - // No proxy so we need to handl all rate limiting and such - return new Promise((resolve, reject) => { - const callback = async () => { - try { - const rateLimitResetIn = await checkRatelimits(url); - if (rateLimitResetIn) { - return { rateLimited: rateLimitResetIn, beforeFetch: true, bucketID }; - } - - const query = method === "get" && body - ? Object.entries(body as any).map(([key, value]) => - `${encodeURIComponent(key)}=${encodeURIComponent(value as any)}` - ) - .join("&") - : ""; - const urlToUse = method === "get" && query ? `${url}?${query}` : url; - - eventHandlers.debug?.( - { - type: "requestManagerFetching", - data: { method, url, body, retryCount, bucketID }, - }, - ); - const response = await fetch(urlToUse, createRequestBody(body, method)); - eventHandlers.debug?.( - { - type: "requestManagerFetched", - data: { method, url, body, retryCount, bucketID, response }, - }, - ); - const bucketIDFromHeaders = processHeaders(url, response.headers); - handleStatusCode(response, errorStack); - - // Sometimes Discord returns an empty 204 response that can't be made to JSON. - if (response.status === 204) return resolve(undefined); - - const json = await response.json(); - if ( - json.retry_after || - json.message === "You are being rate limited." - ) { - if (retryCount > 10) { - eventHandlers.debug?.( - { - type: "error", - data: { method, url, body, retryCount, bucketID, errorStack }, - }, - ); - throw new Error(Errors.RATE_LIMIT_RETRY_MAXED); - } - - return { - rateLimited: json.retry_after, - beforeFetch: false, - bucketID: bucketIDFromHeaders, - }; - } - - eventHandlers.debug?.( - { - type: "requestManagerSuccess", - data: { method, url, body, retryCount, bucketID }, - }, - ); - return resolve(json); - } catch (error) { - eventHandlers.debug?.( - { - type: "error", - data: { method, url, body, retryCount, bucketID, errorStack }, - }, - ); - return reject(error); - } - }; - - addToQueue({ - callback, - bucketID, - url, - }); - if (!queueInProcess) { - queueInProcess = true; - processQueue(); - } - }); -} - -async function logErrors(response: Response, errorStack?: unknown) { - try { - const error = await response.json(); - console.error(error); - - eventHandlers.debug?.({ type: "error", data: { errorStack, error } }); - } catch { - eventHandlers.debug?.( - { - type: "error", - data: { errorStack }, - }, - ); - console.error(response); - } -} - -function handleStatusCode(response: Response, errorStack?: unknown) { - const status = response.status; - - if ( - (status >= 200 && status < 400) || - status === HttpResponseCode.TooManyRequests - ) { - return true; - } - - logErrors(response, errorStack); - - switch (status) { - case HttpResponseCode.BadRequest: - console.error( - "The request was improperly formatted, or the server couldn't understand it.", - ); - throw errorStack; - case HttpResponseCode.Unauthorized: - console.error("The Authorization header was missing or invalid."); - throw errorStack; - case HttpResponseCode.Forbidden: - console.error( - "The Authorization token you passed did not have permission to the resource.", - ); - throw errorStack; - case HttpResponseCode.NotFound: - console.error("The resource at the location specified doesn't exist."); - throw errorStack; - case HttpResponseCode.MethodNotAllowed: - console.error( - "The HTTP method used is not valid for the location specified.", - ); - throw errorStack; - case HttpResponseCode.GatewayUnavailable: - console.error( - "There was not a gateway available to process your request. Wait a bit and retry.", - ); - throw errorStack; - // left are all unknown - default: - console.error(Errors.REQUEST_UNKNOWN_ERROR); - throw errorStack; - } -} - -function processHeaders(url: string, headers: Headers) { - let ratelimited = false; - - // Get all useful headers - const remaining = headers.get("x-ratelimit-remaining"); - const resetTimestamp = headers.get("x-ratelimit-reset"); - const retryAfter = headers.get("retry-after"); - const global = headers.get("x-ratelimit-global"); - const bucketID = headers.get("x-ratelimit-bucket"); - - // If there is no remaining rate limit for this endpoint, we save it in cache - if (remaining && remaining === "0") { - ratelimited = true; - - ratelimitedPaths.set(url, { - url, - resetTimestamp: Number(resetTimestamp) * 1000, - bucketID, - }); - - if (bucketID) { - ratelimitedPaths.set(bucketID, { - url, - resetTimestamp: Number(resetTimestamp) * 1000, - bucketID, - }); - } - } - - // If there is no remaining global limit, we save it in cache - if (global) { - const reset = Date.now() + (Number(retryAfter) * 1000); - eventHandlers.debug?.( - { type: "globallyRateLimited", data: { url, reset } }, - ); - globallyRateLimited = true; - ratelimited = true; - - ratelimitedPaths.set("global", { - url: "global", - resetTimestamp: reset, - bucketID, - }); - - if (bucketID) { - ratelimitedPaths.set(bucketID, { - url: "global", - resetTimestamp: reset, - bucketID, - }); - } - } - - return ratelimited ? bucketID : undefined; -} +export * from "./request_manager.ts"; diff --git a/src/rest/request_manager.ts b/src/rest/request_manager.ts new file mode 100644 index 000000000..e722f9a9b --- /dev/null +++ b/src/rest/request_manager.ts @@ -0,0 +1,418 @@ +import { Errors, HttpResponseCode, RequestMethods } from "../types/types.ts"; +import { baseEndpoints, discordAPIURLS } from "../util/constants.ts"; +import { delay } from "../util/utils.ts"; +import { authorization, eventHandlers } from "../bot.ts"; + +const pathQueues: { [key: string]: QueuedRequest[] } = {}; +const ratelimitedPaths = new Map(); +let globallyRateLimited = false; +let queueInProcess = false; + +export interface QueuedRequest { + callback: () => Promise< + void | { + rateLimited: any; + beforeFetch: boolean; + bucketID?: string | null; + } + >; + bucketID?: string | null; + url: string; +} + +export interface RateLimitedPath { + url: string; + resetTimestamp: number; + bucketID: string | null; +} + +async function processRateLimitedPaths() { + const now = Date.now(); + ratelimitedPaths.forEach((value, key) => { + if (value.resetTimestamp > now) return; + ratelimitedPaths.delete(key); + if (key === "global") globallyRateLimited = false; + }); + + await delay(1000); + processRateLimitedPaths(); +} + +function addToQueue(request: QueuedRequest) { + const route = request.url.substring(baseEndpoints.BASE_URL.length + 1); + const parts = route.split("/"); + // Remove the major param + parts.shift(); + const [id] = parts; + + if (pathQueues[id]) { + pathQueues[id].push(request); + } else { + pathQueues[id] = [request]; + } +} + +async function cleanupQueues() { + Object.entries(pathQueues).map(([key, value]) => { + if (!value.length) { + // Remove it entirely + delete pathQueues[key]; + } + }); +} + +async function processQueue() { + while (queueInProcess) { + if ( + (Object.keys(pathQueues).length) && !globallyRateLimited + ) { + await Promise.allSettled( + Object.values(pathQueues).map(async (pathQueue) => { + const request = pathQueue.shift(); + if (!request) return; + + const rateLimitedURLResetIn = await checkRatelimits(request.url); + + if (request.bucketID) { + const rateLimitResetIn = await checkRatelimits(request.bucketID); + if (rateLimitResetIn) { + // This request is still rate limited readd to queue + addToQueue(request); + } else if (rateLimitedURLResetIn) { + // This URL is rate limited readd to queue + addToQueue(request); + } else { + // This request is not rate limited so it should be run + const result = await request.callback(); + if (result && result.rateLimited) { + addToQueue( + { ...request, bucketID: result.bucketID || request.bucketID }, + ); + } + } + } else { + if (rateLimitedURLResetIn) { + // This URL is rate limited readd to queue + addToQueue(request); + } else { + // This request has no bucket id so it should be processed + const result = await request.callback(); + if (request && result && result.rateLimited) { + addToQueue( + { ...request, bucketID: result.bucketID || request.bucketID }, + ); + } + } + } + }), + ); + } + + if (Object.keys(pathQueues).length) { + cleanupQueues(); + } else queueInProcess = false; + } +} + +processRateLimitedPaths(); + +export const RequestManager = { + get: async (url: string, body?: unknown) => { + return runMethod("get", url, body); + }, + post: (url: string, body?: unknown) => { + return runMethod("post", url, body); + }, + delete: (url: string, body?: unknown) => { + return runMethod("delete", url, body); + }, + patch: (url: string, body?: unknown) => { + return runMethod("patch", url, body); + }, + put: (url: string, body?: unknown) => { + return runMethod("put", url, body); + }, +}; + +function createRequestBody(body: any, method: RequestMethods) { + const headers: { [key: string]: string } = { + Authorization: authorization, + "User-Agent": + `DiscordBot (https://github.com/skillz4killz/discordeno, v10)`, + }; + + if (method === "get") body = undefined; + + if (body?.reason) { + headers["X-Audit-Log-Reason"] = encodeURIComponent(body.reason); + } + + if (body?.file) { + const form = new FormData(); + form.append("file", body.file.blob, body.file.name); + form.append("payload_json", JSON.stringify({ ...body, file: undefined })); + body.file = form; + } else if ( + body && !["get", "delete"].includes(method) + ) { + headers["Content-Type"] = "application/json"; + } + + return { + headers, + body: body?.file || JSON.stringify(body), + method: method.toUpperCase(), + }; +} + +async function checkRatelimits(url: string) { + const ratelimited = ratelimitedPaths.get(url); + const global = ratelimitedPaths.get("global"); + const now = Date.now(); + + if (ratelimited && now < ratelimited.resetTimestamp) { + return ratelimited.resetTimestamp - now; + } + if (global && now < global.resetTimestamp) { + return global.resetTimestamp - now; + } + + return false; +} + +async function runMethod( + method: RequestMethods, + url: string, + body?: unknown, + retryCount = 0, + bucketID?: string | null, +) { + eventHandlers.debug?.( + { + type: "requestManager", + data: { method, url, body, retryCount, bucketID }, + }, + ); + + const errorStack = new Error("Location:"); + Error.captureStackTrace(errorStack); + + // For proxies we don't need to do any of the legwork so we just forward the request + if ( + !url.startsWith(discordAPIURLS.BASE_URL) && + !url.startsWith(discordAPIURLS.CDN_URL) + ) { + return fetch(url, { method, body: body ? JSON.stringify(body) : undefined }) + .then((res) => res.json()) + .catch((error) => { + console.error(error); + throw errorStack; + }); + } + + // No proxy so we need to handl all rate limiting and such + return new Promise((resolve, reject) => { + const callback = async () => { + try { + const rateLimitResetIn = await checkRatelimits(url); + if (rateLimitResetIn) { + return { rateLimited: rateLimitResetIn, beforeFetch: true, bucketID }; + } + + const query = method === "get" && body + ? Object.entries(body as any).map(([key, value]) => + `${encodeURIComponent(key)}=${encodeURIComponent(value as any)}` + ) + .join("&") + : ""; + const urlToUse = method === "get" && query ? `${url}?${query}` : url; + + eventHandlers.debug?.( + { + type: "requestManagerFetching", + data: { method, url, body, retryCount, bucketID }, + }, + ); + const response = await fetch(urlToUse, createRequestBody(body, method)); + eventHandlers.debug?.( + { + type: "requestManagerFetched", + data: { method, url, body, retryCount, bucketID, response }, + }, + ); + const bucketIDFromHeaders = processHeaders(url, response.headers); + handleStatusCode(response, errorStack); + + // Sometimes Discord returns an empty 204 response that can't be made to JSON. + if (response.status === 204) return resolve(undefined); + + const json = await response.json(); + if ( + json.retry_after || + json.message === "You are being rate limited." + ) { + if (retryCount > 10) { + eventHandlers.debug?.( + { + type: "error", + data: { method, url, body, retryCount, bucketID, errorStack }, + }, + ); + throw new Error(Errors.RATE_LIMIT_RETRY_MAXED); + } + + return { + rateLimited: json.retry_after, + beforeFetch: false, + bucketID: bucketIDFromHeaders, + }; + } + + eventHandlers.debug?.( + { + type: "requestManagerSuccess", + data: { method, url, body, retryCount, bucketID }, + }, + ); + return resolve(json); + } catch (error) { + eventHandlers.debug?.( + { + type: "error", + data: { method, url, body, retryCount, bucketID, errorStack }, + }, + ); + return reject(error); + } + }; + + addToQueue({ + callback, + bucketID, + url, + }); + if (!queueInProcess) { + queueInProcess = true; + processQueue(); + } + }); +} + +async function logErrors(response: Response, errorStack?: unknown) { + try { + const error = await response.json(); + console.error(error); + + eventHandlers.debug?.({ type: "error", data: { errorStack, error } }); + } catch { + eventHandlers.debug?.( + { + type: "error", + data: { errorStack }, + }, + ); + console.error(response); + } +} + +function handleStatusCode(response: Response, errorStack?: unknown) { + const status = response.status; + + if ( + (status >= 200 && status < 400) || + status === HttpResponseCode.TooManyRequests + ) { + return true; + } + + logErrors(response, errorStack); + + switch (status) { + case HttpResponseCode.BadRequest: + console.error( + "The request was improperly formatted, or the server couldn't understand it.", + ); + throw errorStack; + case HttpResponseCode.Unauthorized: + console.error("The Authorization header was missing or invalid."); + throw errorStack; + case HttpResponseCode.Forbidden: + console.error( + "The Authorization token you passed did not have permission to the resource.", + ); + throw errorStack; + case HttpResponseCode.NotFound: + console.error("The resource at the location specified doesn't exist."); + throw errorStack; + case HttpResponseCode.MethodNotAllowed: + console.error( + "The HTTP method used is not valid for the location specified.", + ); + throw errorStack; + case HttpResponseCode.GatewayUnavailable: + console.error( + "There was not a gateway available to process your request. Wait a bit and retry.", + ); + throw errorStack; + // left are all unknown + default: + console.error(Errors.REQUEST_UNKNOWN_ERROR); + throw errorStack; + } +} + +function processHeaders(url: string, headers: Headers) { + let ratelimited = false; + + // Get all useful headers + const remaining = headers.get("x-ratelimit-remaining"); + const resetTimestamp = headers.get("x-ratelimit-reset"); + const retryAfter = headers.get("retry-after"); + const global = headers.get("x-ratelimit-global"); + const bucketID = headers.get("x-ratelimit-bucket"); + + // If there is no remaining rate limit for this endpoint, we save it in cache + if (remaining && remaining === "0") { + ratelimited = true; + + ratelimitedPaths.set(url, { + url, + resetTimestamp: Number(resetTimestamp) * 1000, + bucketID, + }); + + if (bucketID) { + ratelimitedPaths.set(bucketID, { + url, + resetTimestamp: Number(resetTimestamp) * 1000, + bucketID, + }); + } + } + + // If there is no remaining global limit, we save it in cache + if (global) { + const reset = Date.now() + (Number(retryAfter) * 1000); + eventHandlers.debug?.( + { type: "globallyRateLimited", data: { url, reset } }, + ); + globallyRateLimited = true; + ratelimited = true; + + ratelimitedPaths.set("global", { + url: "global", + resetTimestamp: reset, + bucketID, + }); + + if (bucketID) { + ratelimitedPaths.set(bucketID, { + url: "global", + resetTimestamp: reset, + bucketID, + }); + } + } + + return ratelimited ? bucketID : undefined; +} diff --git a/src/types/options.ts b/src/types/options.ts index 05d6dbcf9..e4b743517 100644 --- a/src/types/options.ts +++ b/src/types/options.ts @@ -23,7 +23,7 @@ import { ReactionPayload, } from "./message.ts"; -export interface ClientOptions { +export interface BotConfig { token: string; compress?: boolean; intents: Intents[]; diff --git a/src/util/cdn.ts b/src/util/cdn.ts deleted file mode 100644 index 670bef910..000000000 --- a/src/util/cdn.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { ImageFormats, ImageSize } from "../types/types.ts"; - -export const formatImageURL = ( - url: string, - size: ImageSize = 128, - format?: ImageFormats, -) => { - return `${url}.${format || - (url.includes("/a_") ? "gif" : "jpg")}?size=${size}`; -}; diff --git a/src/util/utils.ts b/src/util/utils.ts index 092e21af1..5d4d3ec0a 100644 --- a/src/util/utils.ts +++ b/src/util/utils.ts @@ -1,6 +1,11 @@ -import { sendGatewayCommand } from "../ws/shard_manager.ts"; -import { ActivityType, StatusType } from "../types/types.ts"; import { encode } from "../../deps.ts"; +import { + ActivityType, + ImageFormats, + ImageSize, + StatusType, +} from "../types/types.ts"; +import { sendGatewayCommand } from "../ws/shard_manager.ts"; export const sleep = (timeout: number) => { return new Promise((resolve) => setTimeout(resolve, timeout)); @@ -40,3 +45,12 @@ export function delay(ms: number): Promise { }, ms) ); } + +export const formatImageURL = ( + url: string, + size: ImageSize = 128, + format?: ImageFormats, +) => { + return `${url}.${format || + (url.includes("/a_") ? "gif" : "jpg")}?size=${size}`; +}; diff --git a/src/ws/mod.ts b/src/ws/mod.ts index 035b198b9..907ca87b3 100644 --- a/src/ws/mod.ts +++ b/src/ws/mod.ts @@ -1,436 +1,2 @@ -import { botGatewayData, eventHandlers } from "../bot.ts"; -import { - DiscordBotGatewayData, - DiscordHeartbeatPayload, - FetchMembersOptions, - GatewayOpcode, - ReadyPayload, -} from "../types/types.ts"; -import { BotStatusRequest, delay } from "../util/utils.ts"; -import { IdentifyPayload, proxyWSURL } from "../bot.ts"; -import { handleDiscordPayload } from "./shard_manager.ts"; -import { decompressWith } from "./deps.ts"; - -const basicShards = new Map(); -const heartbeating = new Map(); -const utf8decoder = new TextDecoder(); -const RequestMembersQueue: RequestMemberQueuedRequest[] = []; -let processQueue = false; - -export interface BasicShard { - id: number; - socket: WebSocket; - resumeInterval: number; - sessionID: string; - previousSequenceNumber: number | null; - needToResume: boolean; -} - -interface RequestMemberQueuedRequest { - guildID: string; - shardID: number; - nonce: string; - options?: FetchMembersOptions; -} - -export async function createShard( - data: DiscordBotGatewayData, - identifyPayload: IdentifyPayload, - resuming = false, - shardID = 0, -) { - const oldShard = basicShards.get(shardID); - - const socket = new WebSocket(proxyWSURL); - socket.binaryType = "arraybuffer"; - const basicShard: BasicShard = { - id: shardID, - socket, - resumeInterval: 0, - sessionID: oldShard?.sessionID || "", - previousSequenceNumber: oldShard?.previousSequenceNumber || 0, - needToResume: false, - }; - - basicShards.set(basicShard.id, basicShard); - - socket.onopen = async () => { - if (!resuming) { - // Initial identify with the gateway - await identify(basicShard, identifyPayload); - } else { - await resume(basicShard, identifyPayload); - } - }; - - socket.onerror = ({ timeStamp }) => { - eventHandlers.debug?.({ type: "wsError", data: { timeStamp } }); - }; - - socket.onmessage = ({ data: message }) => { - if (message instanceof ArrayBuffer) { - message = new Uint8Array(message); - } - - if (message instanceof Uint8Array) { - message = decompressWith( - message, - 0, - (slice: Uint8Array) => utf8decoder.decode(slice), - ); - } - - if (typeof message === "string") { - const data = JSON.parse(message); - if (!data.t) eventHandlers.rawGateway?.(data); - switch (data.op) { - case GatewayOpcode.Hello: - if (!heartbeating.has(basicShard.id)) { - heartbeat( - basicShard, - (data.d as DiscordHeartbeatPayload).heartbeat_interval, - identifyPayload, - data, - ); - } - break; - case GatewayOpcode.HeartbeatACK: - heartbeating.set(shardID, true); - break; - case GatewayOpcode.Reconnect: - eventHandlers.debug?.( - { type: "reconnect", data: { shardID: basicShard.id } }, - ); - basicShard.needToResume = true; - resumeConnection(data, identifyPayload, basicShard.id); - break; - case GatewayOpcode.InvalidSession: - eventHandlers.debug?.( - { type: "invalidSession", data: { shardID: basicShard.id, data } }, - ); - // When d is false we need to reidentify - if (!data.d) { - createShard(data, identifyPayload, false, shardID); - break; - } - basicShard.needToResume = true; - resumeConnection(data, identifyPayload, basicShard.id); - break; - default: - if (data.t === "RESUMED") { - eventHandlers.debug?.( - { type: "resumed", data: { shardID: basicShard.id } }, - ); - - basicShard.needToResume = false; - break; - } - // Important for RESUME - if (data.t === "READY") { - basicShard.sessionID = (data.d as ReadyPayload).session_id; - } - - // Update the sequence number if it is present - if (data.s) basicShard.previousSequenceNumber = data.s; - - handleDiscordPayload(data, basicShard.id); - break; - } - } - }; - - // TODO(ayntee): better ws* event names - socket.onclose = ({ reason, code, wasClean }) => { - eventHandlers.debug?.( - { - type: "wsClose", - data: { shardID: basicShard.id, code, reason, wasClean }, - }, - ); - - switch (code) { - case 4001: - throw new Error( - "[Unknown opcode] Sent an invalid Gateway opcode or an invalid payload for an opcode.", - ); - case 4002: - throw new Error("[Decode error] Sent an invalid payload to API."); - case 4004: - throw new Error( - "[Authentication failed] The account token sent with your identify payload is incorrect.", - ); - case 4005: - throw new Error( - "[Already authenticated] Sent more than one identify payload.", - ); - case 4010: - throw new Error( - "[Invalid shard] Sent an invalid shard when identifying.", - ); - case 4011: - throw new Error( - "[Sharding required] The session would have handled too many guilds - you are required to shard your connection in order to connect.", - ); - case 4012: - throw new Error( - "[Invalid API version] Sent an invalid version for the gateway.", - ); - case 4013: - throw new Error( - "[Invalid intent(s)] Sent an invalid intent for a Gateway Intent.", - ); - case 4014: - throw new Error( - "[Disallowed intent(s)] Sent a disallowed intent for a Gateway Intent. You may have tried to specify an intent that you have not enabled or are not whitelisted for.", - ); - case 4003: - case 4007: - case 4008: - case 4009: - eventHandlers.debug?.({ - type: "wsReconnect", - data: { shardID: basicShard.id, code, reason, wasClean }, - }); - createShard(data, identifyPayload, false, shardID); - break; - default: - basicShard.needToResume = true; - resumeConnection(botGatewayData, identifyPayload, shardID); - break; - } - }; -} - -function identify(shard: BasicShard, payload: IdentifyPayload) { - eventHandlers.debug?.( - { - type: "identifying", - data: { - shardID: shard.id, - }, - }, - ); - - return shard.socket.send( - JSON.stringify( - { - op: GatewayOpcode.Identify, - d: { ...payload, shard: [shard.id, payload.shard[1]] }, - }, - ), - ); -} - -function resume(shard: BasicShard, payload: IdentifyPayload) { - return shard.socket.send(JSON.stringify({ - op: GatewayOpcode.Resume, - d: { - token: payload.token, - session_id: shard.sessionID, - seq: shard.previousSequenceNumber, - }, - })); -} - -async function heartbeat( - shard: BasicShard, - interval: number, - payload: IdentifyPayload, - data: DiscordBotGatewayData, -) { - // We lost socket connection between heartbeats, resume connection - if (shard.socket.readyState === WebSocket.CLOSED) { - shard.needToResume = true; - resumeConnection(data, payload, shard.id); - heartbeating.delete(shard.id); - return; - } - - if (heartbeating.has(shard.id)) { - const receivedACK = heartbeating.get(shard.id); - // If a ACK response was not received since last heartbeat, issue invalid session close - if (!receivedACK) { - eventHandlers.debug?.( - { - type: "heartbeatStopped", - data: { - interval, - previousSequenceNumber: shard.previousSequenceNumber, - shardID: shard.id, - }, - }, - ); - return shard.socket.send(JSON.stringify({ op: 4009 })); - } - } - - // Set it to false as we are issuing a new heartbeat - heartbeating.set(shard.id, false); - - shard.socket.send( - JSON.stringify( - { op: GatewayOpcode.Heartbeat, d: shard.previousSequenceNumber }, - ), - ); - eventHandlers.debug?.( - { - type: "heartbeat", - data: { - interval, - previousSequenceNumber: shard.previousSequenceNumber, - shardID: shard.id, - }, - }, - ); - await delay(interval); - heartbeat(shard, interval, payload, data); -} - -async function resumeConnection( - data: DiscordBotGatewayData, - payload: IdentifyPayload, - shardID: number, -) { - const shard = basicShards.get(shardID); - if (!shard) { - eventHandlers.debug?.( - { type: "missingShard", data: { shardID: shardID } }, - ); - return; - } - - if (!shard.needToResume) return; - - eventHandlers.debug?.({ type: "resuming", data: { shardID: shard.id } }); - // Run it once - createShard(data, payload, true, shard.id); - // Then retry every 15 seconds - await delay(1000 * 15); - if (shard.needToResume) resumeConnection(data, payload, shardID); -} - -export function requestGuildMembers( - guildID: string, - shardID: number, - nonce: string, - options?: FetchMembersOptions, - queuedRequest = false, -) { - const shard = basicShards.get(shardID); - - // This request was not from this queue so we add it to queue first - if (!queuedRequest) { - RequestMembersQueue.push({ - guildID, - shardID, - nonce, - options, - }); - - if (!processQueue) { - processQueue = true; - processGatewayQueue(); - } - return; - } - - // If its closed add back to queue to redo on resume - if (shard?.socket.readyState === WebSocket.CLOSED) { - requestGuildMembers(guildID, shardID, nonce, options); - return; - } - - shard?.socket.send(JSON.stringify({ - op: GatewayOpcode.RequestGuildMembers, - d: { - guild_id: guildID, - // If a query is provided use it, OR if a limit is NOT provided use "" - query: options?.query || (options?.limit ? undefined : ""), - limit: options?.limit || 0, - presences: options?.presences || false, - user_ids: options?.userIDs, - nonce, - }, - })); -} - -async function processGatewayQueue() { - if (!RequestMembersQueue.length) { - processQueue = false; - return; - } - - basicShards.forEach((shard) => { - const index = RequestMembersQueue.findIndex((q) => q.shardID === shard.id); - // 2 events per second is the rate limit. - const request = RequestMembersQueue[index]; - if (request) { - eventHandlers.debug?.( - { - type: "requestMembersProcessing", - data: { - remaining: RequestMembersQueue.length, - request, - }, - }, - ); - requestGuildMembers( - request.guildID, - request.shardID, - request.nonce, - request.options, - true, - ); - // Remove item from queue - RequestMembersQueue.splice(index, 1); - - const secondIndex = RequestMembersQueue.findIndex((q) => - q.shardID === shard.id - ); - const secondRequest = RequestMembersQueue[secondIndex]; - if (secondRequest) { - eventHandlers.debug?.( - { - type: "requestMembersProcessing", - data: { - remaining: RequestMembersQueue.length, - request, - }, - }, - ); - requestGuildMembers( - secondRequest.guildID, - secondRequest.shardID, - secondRequest.nonce, - secondRequest.options, - true, - ); - // Remove item from queue - RequestMembersQueue.splice(secondIndex, 1); - } - } - }); - - await delay(1500); - - processGatewayQueue(); -} - -export function botGatewayStatusRequest(payload: BotStatusRequest) { - basicShards.forEach((shard) => { - shard.socket.send(JSON.stringify({ - op: GatewayOpcode.StatusUpdate, - d: { - since: null, - game: payload.game.name - ? { - name: payload.game.name, - type: payload.game.type, - } - : null, - status: payload.status, - afk: false, - }, - })); - }); -} +export * from "./shard.ts"; +export * from "./shard_manager.ts"; diff --git a/src/ws/shard.ts b/src/ws/shard.ts new file mode 100644 index 000000000..035b198b9 --- /dev/null +++ b/src/ws/shard.ts @@ -0,0 +1,436 @@ +import { botGatewayData, eventHandlers } from "../bot.ts"; +import { + DiscordBotGatewayData, + DiscordHeartbeatPayload, + FetchMembersOptions, + GatewayOpcode, + ReadyPayload, +} from "../types/types.ts"; +import { BotStatusRequest, delay } from "../util/utils.ts"; +import { IdentifyPayload, proxyWSURL } from "../bot.ts"; +import { handleDiscordPayload } from "./shard_manager.ts"; +import { decompressWith } from "./deps.ts"; + +const basicShards = new Map(); +const heartbeating = new Map(); +const utf8decoder = new TextDecoder(); +const RequestMembersQueue: RequestMemberQueuedRequest[] = []; +let processQueue = false; + +export interface BasicShard { + id: number; + socket: WebSocket; + resumeInterval: number; + sessionID: string; + previousSequenceNumber: number | null; + needToResume: boolean; +} + +interface RequestMemberQueuedRequest { + guildID: string; + shardID: number; + nonce: string; + options?: FetchMembersOptions; +} + +export async function createShard( + data: DiscordBotGatewayData, + identifyPayload: IdentifyPayload, + resuming = false, + shardID = 0, +) { + const oldShard = basicShards.get(shardID); + + const socket = new WebSocket(proxyWSURL); + socket.binaryType = "arraybuffer"; + const basicShard: BasicShard = { + id: shardID, + socket, + resumeInterval: 0, + sessionID: oldShard?.sessionID || "", + previousSequenceNumber: oldShard?.previousSequenceNumber || 0, + needToResume: false, + }; + + basicShards.set(basicShard.id, basicShard); + + socket.onopen = async () => { + if (!resuming) { + // Initial identify with the gateway + await identify(basicShard, identifyPayload); + } else { + await resume(basicShard, identifyPayload); + } + }; + + socket.onerror = ({ timeStamp }) => { + eventHandlers.debug?.({ type: "wsError", data: { timeStamp } }); + }; + + socket.onmessage = ({ data: message }) => { + if (message instanceof ArrayBuffer) { + message = new Uint8Array(message); + } + + if (message instanceof Uint8Array) { + message = decompressWith( + message, + 0, + (slice: Uint8Array) => utf8decoder.decode(slice), + ); + } + + if (typeof message === "string") { + const data = JSON.parse(message); + if (!data.t) eventHandlers.rawGateway?.(data); + switch (data.op) { + case GatewayOpcode.Hello: + if (!heartbeating.has(basicShard.id)) { + heartbeat( + basicShard, + (data.d as DiscordHeartbeatPayload).heartbeat_interval, + identifyPayload, + data, + ); + } + break; + case GatewayOpcode.HeartbeatACK: + heartbeating.set(shardID, true); + break; + case GatewayOpcode.Reconnect: + eventHandlers.debug?.( + { type: "reconnect", data: { shardID: basicShard.id } }, + ); + basicShard.needToResume = true; + resumeConnection(data, identifyPayload, basicShard.id); + break; + case GatewayOpcode.InvalidSession: + eventHandlers.debug?.( + { type: "invalidSession", data: { shardID: basicShard.id, data } }, + ); + // When d is false we need to reidentify + if (!data.d) { + createShard(data, identifyPayload, false, shardID); + break; + } + basicShard.needToResume = true; + resumeConnection(data, identifyPayload, basicShard.id); + break; + default: + if (data.t === "RESUMED") { + eventHandlers.debug?.( + { type: "resumed", data: { shardID: basicShard.id } }, + ); + + basicShard.needToResume = false; + break; + } + // Important for RESUME + if (data.t === "READY") { + basicShard.sessionID = (data.d as ReadyPayload).session_id; + } + + // Update the sequence number if it is present + if (data.s) basicShard.previousSequenceNumber = data.s; + + handleDiscordPayload(data, basicShard.id); + break; + } + } + }; + + // TODO(ayntee): better ws* event names + socket.onclose = ({ reason, code, wasClean }) => { + eventHandlers.debug?.( + { + type: "wsClose", + data: { shardID: basicShard.id, code, reason, wasClean }, + }, + ); + + switch (code) { + case 4001: + throw new Error( + "[Unknown opcode] Sent an invalid Gateway opcode or an invalid payload for an opcode.", + ); + case 4002: + throw new Error("[Decode error] Sent an invalid payload to API."); + case 4004: + throw new Error( + "[Authentication failed] The account token sent with your identify payload is incorrect.", + ); + case 4005: + throw new Error( + "[Already authenticated] Sent more than one identify payload.", + ); + case 4010: + throw new Error( + "[Invalid shard] Sent an invalid shard when identifying.", + ); + case 4011: + throw new Error( + "[Sharding required] The session would have handled too many guilds - you are required to shard your connection in order to connect.", + ); + case 4012: + throw new Error( + "[Invalid API version] Sent an invalid version for the gateway.", + ); + case 4013: + throw new Error( + "[Invalid intent(s)] Sent an invalid intent for a Gateway Intent.", + ); + case 4014: + throw new Error( + "[Disallowed intent(s)] Sent a disallowed intent for a Gateway Intent. You may have tried to specify an intent that you have not enabled or are not whitelisted for.", + ); + case 4003: + case 4007: + case 4008: + case 4009: + eventHandlers.debug?.({ + type: "wsReconnect", + data: { shardID: basicShard.id, code, reason, wasClean }, + }); + createShard(data, identifyPayload, false, shardID); + break; + default: + basicShard.needToResume = true; + resumeConnection(botGatewayData, identifyPayload, shardID); + break; + } + }; +} + +function identify(shard: BasicShard, payload: IdentifyPayload) { + eventHandlers.debug?.( + { + type: "identifying", + data: { + shardID: shard.id, + }, + }, + ); + + return shard.socket.send( + JSON.stringify( + { + op: GatewayOpcode.Identify, + d: { ...payload, shard: [shard.id, payload.shard[1]] }, + }, + ), + ); +} + +function resume(shard: BasicShard, payload: IdentifyPayload) { + return shard.socket.send(JSON.stringify({ + op: GatewayOpcode.Resume, + d: { + token: payload.token, + session_id: shard.sessionID, + seq: shard.previousSequenceNumber, + }, + })); +} + +async function heartbeat( + shard: BasicShard, + interval: number, + payload: IdentifyPayload, + data: DiscordBotGatewayData, +) { + // We lost socket connection between heartbeats, resume connection + if (shard.socket.readyState === WebSocket.CLOSED) { + shard.needToResume = true; + resumeConnection(data, payload, shard.id); + heartbeating.delete(shard.id); + return; + } + + if (heartbeating.has(shard.id)) { + const receivedACK = heartbeating.get(shard.id); + // If a ACK response was not received since last heartbeat, issue invalid session close + if (!receivedACK) { + eventHandlers.debug?.( + { + type: "heartbeatStopped", + data: { + interval, + previousSequenceNumber: shard.previousSequenceNumber, + shardID: shard.id, + }, + }, + ); + return shard.socket.send(JSON.stringify({ op: 4009 })); + } + } + + // Set it to false as we are issuing a new heartbeat + heartbeating.set(shard.id, false); + + shard.socket.send( + JSON.stringify( + { op: GatewayOpcode.Heartbeat, d: shard.previousSequenceNumber }, + ), + ); + eventHandlers.debug?.( + { + type: "heartbeat", + data: { + interval, + previousSequenceNumber: shard.previousSequenceNumber, + shardID: shard.id, + }, + }, + ); + await delay(interval); + heartbeat(shard, interval, payload, data); +} + +async function resumeConnection( + data: DiscordBotGatewayData, + payload: IdentifyPayload, + shardID: number, +) { + const shard = basicShards.get(shardID); + if (!shard) { + eventHandlers.debug?.( + { type: "missingShard", data: { shardID: shardID } }, + ); + return; + } + + if (!shard.needToResume) return; + + eventHandlers.debug?.({ type: "resuming", data: { shardID: shard.id } }); + // Run it once + createShard(data, payload, true, shard.id); + // Then retry every 15 seconds + await delay(1000 * 15); + if (shard.needToResume) resumeConnection(data, payload, shardID); +} + +export function requestGuildMembers( + guildID: string, + shardID: number, + nonce: string, + options?: FetchMembersOptions, + queuedRequest = false, +) { + const shard = basicShards.get(shardID); + + // This request was not from this queue so we add it to queue first + if (!queuedRequest) { + RequestMembersQueue.push({ + guildID, + shardID, + nonce, + options, + }); + + if (!processQueue) { + processQueue = true; + processGatewayQueue(); + } + return; + } + + // If its closed add back to queue to redo on resume + if (shard?.socket.readyState === WebSocket.CLOSED) { + requestGuildMembers(guildID, shardID, nonce, options); + return; + } + + shard?.socket.send(JSON.stringify({ + op: GatewayOpcode.RequestGuildMembers, + d: { + guild_id: guildID, + // If a query is provided use it, OR if a limit is NOT provided use "" + query: options?.query || (options?.limit ? undefined : ""), + limit: options?.limit || 0, + presences: options?.presences || false, + user_ids: options?.userIDs, + nonce, + }, + })); +} + +async function processGatewayQueue() { + if (!RequestMembersQueue.length) { + processQueue = false; + return; + } + + basicShards.forEach((shard) => { + const index = RequestMembersQueue.findIndex((q) => q.shardID === shard.id); + // 2 events per second is the rate limit. + const request = RequestMembersQueue[index]; + if (request) { + eventHandlers.debug?.( + { + type: "requestMembersProcessing", + data: { + remaining: RequestMembersQueue.length, + request, + }, + }, + ); + requestGuildMembers( + request.guildID, + request.shardID, + request.nonce, + request.options, + true, + ); + // Remove item from queue + RequestMembersQueue.splice(index, 1); + + const secondIndex = RequestMembersQueue.findIndex((q) => + q.shardID === shard.id + ); + const secondRequest = RequestMembersQueue[secondIndex]; + if (secondRequest) { + eventHandlers.debug?.( + { + type: "requestMembersProcessing", + data: { + remaining: RequestMembersQueue.length, + request, + }, + }, + ); + requestGuildMembers( + secondRequest.guildID, + secondRequest.shardID, + secondRequest.nonce, + secondRequest.options, + true, + ); + // Remove item from queue + RequestMembersQueue.splice(secondIndex, 1); + } + } + }); + + await delay(1500); + + processGatewayQueue(); +} + +export function botGatewayStatusRequest(payload: BotStatusRequest) { + basicShards.forEach((shard) => { + shard.socket.send(JSON.stringify({ + op: GatewayOpcode.StatusUpdate, + d: { + since: null, + game: payload.game.name + ? { + name: payload.game.name, + type: payload.game.type, + } + : null, + status: payload.status, + afk: false, + }, + })); + }); +}