This commit is contained in:
Skillz4Killz
2021-09-30 08:06:42 +00:00
committed by GitHub
parent f44e6c7b43
commit a9e5ce01c9
11 changed files with 180 additions and 135 deletions
+124 -59
View File
@@ -1,70 +1,135 @@
import { getGatewayBot } from "./helpers/misc/get_gateway_bot.ts";
import { rest } from "./rest/rest.ts";
import type { EventHandlers } from "./types/discordeno/event_handlers.ts";
import { checkRateLimits } from "./rest/check_rate_limits.ts";
import { cleanupQueues } from "./rest/cleanup_queues.ts";
import { createRequestBody } from "./rest/create_request_body.ts";
import { processRateLimitedPaths } from "./rest/process_rate_limited_paths.ts";
import { processRequest } from "./rest/process_request.ts";
import { processRequestHeaders } from "./rest/process_request_headers.ts";
import { RestPayload, RestRateLimitedPath, RestRequest } from "./rest/rest.ts";
import { runMethod } from "./rest/run_method.ts";
import { simplifyUrl } from "./rest/simplify_url.ts";
import { DiscordGatewayIntents } from "./types/gateway/gateway_intents.ts";
import { snowflakeToBigint } from "./util/bigint.ts";
import { GATEWAY_VERSION } from "./util/constants.ts";
import { ws } from "./ws/ws.ts";
import { GetGatewayBot } from "./types/gateway/get_gateway_bot.ts";
import { dispatchRequirements } from "./util/dispatch_requirements.ts";
import { processQueue } from "./rest/process_queue.ts";
// deno-lint-ignore prefer-const
export let secretKey = "";
export let botId = 0n;
export let applicationId = 0n;
export let eventHandlers: EventHandlers = {};
export let proxyWSURL = `wss://gateway.discord.gg`;
export async function startBot(config: BotConfig) {
if (config.eventHandlers)
eventHandlers = {
...config.eventHandlers,
dispatchRequirements: config.eventHandlers.dispatchRequirements || dispatchRequirements,
};
ws.identifyPayload.token = `Bot ${config.token}`;
rest.token = `Bot ${config.token}`;
ws.identifyPayload.intents = config.intents.reduce(
(bits, next) => (bits |= typeof next === "string" ? DiscordGatewayIntents[next] : next),
export async function createBot(options: CreateBotOptions) {
return {
id: options.botId,
applicationId: options.applicationId || options.botId,
token: `Bot ${options.token}`,
events: {dispatchRequirements: dispatchRequirements, ...options.events},
intents: options.intents.reduce(
(bits, next) => (bits |= DiscordGatewayIntents[next]),
0
);
// Initial API connection to get info about bots connection
ws.botGatewayData = await getGatewayBot();
ws.maxShards = ws.maxShards || ws.botGatewayData.shards;
ws.lastShardId = ws.lastShardId === 1 ? ws.botGatewayData.shards - 1 : ws.lastShardId;
// Explicitly append gateway version and encoding
ws.botGatewayData.url += `?v=${GATEWAY_VERSION}&encoding=json`;
proxyWSURL = ws.botGatewayData.url;
ws.spawnShards();
),
botGatewayData: options.botGatewayData || await getGatewayBot(),
isReady: false,
rest: createRestManager(options.rest ? { token: options.token, ...options.rest} : { token: options.token})
}
}
export function replaceEventHandlers(newEventHandlers: EventHandlers) {
eventHandlers = newEventHandlers;
const bot = await createBot({
token: "",
botId: 0n,
events: createEventHandlers(),
intents: [],
})
export function createEventHandlers(options?: Partial<EventHandlers>) {
return {
debug: () => undefined,
// PROVIDED OPTIONS OVERRIDE EVERYTHING ABOVE
...options
}
}
/** Allows you to dynamically update the event handlers by passing in new eventHandlers */
export function updateEventHandlers(newEventHandlers: EventHandlers) {
// Object.assign instead of ... operator because of the Proxy used
Object.assign(eventHandlers, newEventHandlers);
}
/** INTERNAL LIB function used to set the bot Id once the READY event is sent by Discord. */
export function setBotId(id: string) {
botId = snowflakeToBigint(id);
}
/** INTERNAL LIB function used to set the application Id once the READY event is sent by Discord. */
export function setApplicationId(id: string) {
applicationId = snowflakeToBigint(id);
}
export interface BotConfig {
export interface CreateRestManagerOptions {
token: string;
compress?: boolean;
intents: (DiscordGatewayIntents | keyof typeof DiscordGatewayIntents)[];
eventHandlers?: EventHandlers;
maxRetryCount?: number;
version?: number;
secretKey?: string;
debug?: (text: string) => unknown;
checkRateLimits?: typeof checkRateLimits;
cleanupQueues?: typeof cleanupQueues;
processQueue?: typeof processQueue;
processRateLimitedPaths?: typeof processRateLimitedPaths;
processRequestHeaders?: typeof processRequestHeaders;
processRequest?: typeof processRequest;
createRequestBody?: typeof createRequestBody;
runMethod?: typeof runMethod;
simplifyUrl?: typeof simplifyUrl
}
export function createRestManager(options: CreateRestManagerOptions) {
return {
token: `${options.token.startsWith("Bot ") ? "" : "Bot "}${options.token}`,
maxRetryCount: options.maxRetryCount || 10,
version: options.version || "9",
secretKey: options.secretKey || "discordeno_best_lib_ever",
pathQueues: new Map<
string,
{
request: RestRequest;
payload: RestPayload;
}[]
>(),
processingQueue: false,
processingRateLimitedPaths: false,
globallyRateLimited: false,
ratelimitedPaths: new Map<string, RestRateLimitedPath>(),
debug: options.debug || function (_text: string) {},
checkRateLimits: options.checkRateLimits || checkRateLimits,
cleanupQueues: options.cleanupQueues || cleanupQueues,
processQueue: options.processQueue || processQueue,
processRateLimitedPaths: options.processRateLimitedPaths || processRateLimitedPaths,
processRequestHeaders: options.processRequestHeaders || processRequestHeaders,
processRequest: options.processRequest || processRequest,
createRequestBody: options.createRequestBody || createRequestBody,
runMethod: options.runMethod || runMethod,
simplifyUrl: options.simplifyUrl || simplifyUrl,
}
}
export async function startBot(bot: Bot) {
// START REST
bot.rest = createRestManager({ token: bot.token });
// START WS
bot.gateway = createGatewayManager()
}
export function stopBot(bot: Bot) {
// STOP REST
// STOP WS
}
export interface CreateBotOptions {
token: string;
botId: bigint;
applicationId?: bigint;
events: EventHandlers;
intents: (keyof typeof DiscordGatewayIntents)[];
botGatewayData?: GetGatewayBot;
rest?: Omit<CreateRestManagerOptions, "token">;
}
export type UnPromise<T extends Promise<unknown>> = T extends Promise<infer K> ? K : never;
export type CreatedBot = UnPromise<ReturnType<typeof createBot>>;
export type Bot = CreatedBot & {
rest: RestManager;
gateway: GatewayManager;
}
export type RestManager = ReturnType<typeof createRestManager>;
export interface GatewayManager {
}
export interface EventHandlers {
debug: (text: string) => unknown;
}
+2 -2
View File
@@ -1,7 +1,7 @@
import { rest } from "./rest.ts";
import { RestManager } from "../bot.ts";
/** Check the rate limits for a url or a bucket. */
export function checkRateLimits(url: string) {
export function checkRateLimits(rest: RestManager, url: string) {
const ratelimited = rest.ratelimitedPaths.get(url);
const global = rest.ratelimitedPaths.get("global");
const now = Date.now();
+3 -3
View File
@@ -1,9 +1,9 @@
import { rest } from "./rest.ts";
import { RestManager } from "../bot.ts";
/** Cleans up the queues by checking if there is nothing left and removing it. */
export function cleanupQueues() {
export function cleanupQueues(rest: RestManager) {
for (const [key, queue] of rest.pathQueues) {
rest.eventHandlers.debug?.("loop", "Running for of loop in cleanupQueues function.");
rest.debug(`[REST - cleanupQueues] Running for of loop. ${key}`);
if (queue.length) continue;
// REMOVE IT FROM CACHE
rest.pathQueues.delete(key);
+3 -2
View File
@@ -1,9 +1,10 @@
import { RestManager } from "../bot.ts";
import type { FileContent } from "../types/discordeno/file_content.ts";
import { USER_AGENT } from "../util/constants.ts";
import { rest, RestPayload, RestRequest } from "./rest.ts";
import { RestPayload, RestRequest } from "./rest.ts";
/** Creates the request body and headers that are necessary to send a request. Will handle different types of methods and everything necessary for discord. */
export function createRequestBody(queuedRequest: { request: RestRequest; payload: RestPayload }) {
export function createRequestBody(rest: RestManager, queuedRequest: { request: RestRequest; payload: RestPayload }) {
const headers: { [key: string]: string } = {
Authorization: rest.token,
"User-Agent": USER_AGENT,
+17 -18
View File
@@ -1,20 +1,19 @@
import { eventHandlers } from "../bot.ts";
import { RestManager } from "../bot.ts";
import { DiscordHTTPResponseCodes } from "../types/codes/http_response_codes.ts";
import { delay } from "../util/utils.ts";
import { rest } from "./rest.ts";
/** Processes the queue by looping over each path separately until the queues are empty. */
export async function processQueue(id: string) {
export async function processQueue(rest: RestManager, id: string) {
const queue = rest.pathQueues.get(id);
if (!queue) return;
while (queue.length) {
rest.eventHandlers.debug?.("loop", "Running while loop in processQueue function.");
rest.debug(`[REST - processQueue] Running while loop.`);
// IF THE BOT IS GLOBALLY RATELIMITED TRY AGAIN
if (rest.globallyRateLimited) {
setTimeout(async () => {
eventHandlers.debug?.("loop", `Running setTimeout in processQueue function.`);
await processQueue(id);
rest.debug(`[REST - processQueue] Running setTimeout.`);
await processQueue(rest, id);
}, 1000);
break;
@@ -27,7 +26,7 @@ export async function processQueue(id: string) {
const basicURL = rest.simplifyUrl(queuedRequest.request.url, queuedRequest.request.method.toUpperCase());
// IF THIS URL IS STILL RATE LIMITED, TRY AGAIN
const urlResetIn = rest.checkRateLimits(basicURL);
const urlResetIn = rest.checkRateLimits(rest, basicURL);
if (urlResetIn) {
// PAUSE FOR THIS SPECIFC REQUEST
await delay(urlResetIn);
@@ -35,7 +34,7 @@ export async function processQueue(id: string) {
}
// IF A BUCKET EXISTS, CHECK THE BUCKET'S RATE LIMITS
const bucketResetIn = queuedRequest.payload.bucketId ? rest.checkRateLimits(queuedRequest.payload.bucketId) : false;
const bucketResetIn = queuedRequest.payload.bucketId ? rest.checkRateLimits(rest, queuedRequest.payload.bucketId) : false;
// THIS BUCKET IS STILL RATELIMITED, RE-ADD TO QUEUE
if (bucketResetIn) continue;
@@ -59,20 +58,20 @@ export async function processQueue(id: string) {
: queuedRequest.request.url;
// CUSTOM HANDLER FOR USER TO LOG OR WHATEVER WHENEVER A FETCH IS MADE
rest.eventHandlers.fetching(queuedRequest.payload);
rest.debug(`[REST - fetching] ${JSON.stringify(queuedRequest.payload)}`);
try {
const response = await fetch(urlToUse, rest.createRequestBody(queuedRequest));
const response = await fetch(urlToUse, rest.createRequestBody(rest, queuedRequest));
rest.debug(`[REST - fetched] ${JSON.stringify(queuedRequest.payload)}`);
rest.eventHandlers.fetched(queuedRequest.payload);
const bucketIdFromHeaders = rest.processRequestHeaders(basicURL, response.headers);
const bucketIdFromHeaders = rest.processRequestHeaders(rest, basicURL, response.headers);
// SET THE BUCKET Id IF IT WAS PRESENT
if (bucketIdFromHeaders) {
queuedRequest.payload.bucketId = bucketIdFromHeaders;
}
if (response.status < 200 || response.status >= 400) {
rest.eventHandlers.error("httpError", queuedRequest.payload, response);
rest.debug(`[REST - httpError] Payload: ${JSON.stringify(queuedRequest.payload)} | Response: ${JSON.stringify(response)}`);
let error = "REQUEST_UNKNOWN_ERROR";
switch (response.status) {
@@ -102,7 +101,7 @@ export async function processQueue(id: string) {
queue.shift();
} else {
if (queuedRequest.payload.retryCount++ >= rest.maxRetryCount) {
rest.eventHandlers.retriesMaxed(queuedRequest.payload);
rest.debug(`[REST - RetriesMaxed] ${JSON.stringify(queuedRequest.payload)}`);
queuedRequest.request.reject(
new Error(`[${response.status}] The request was rate limited and it maxed out the retries limit.`)
);
@@ -117,7 +116,7 @@ export async function processQueue(id: string) {
// SOMETIMES DISCORD RETURNS AN EMPTY 204 RESPONSE THAT CAN'T BE MADE TO JSON
if (response.status === 204) {
rest.eventHandlers.fetchSuccess(queuedRequest.payload);
rest.debug(`[REST - FetchSuccess] ${JSON.stringify(queuedRequest.payload)}`);
// REMOVE FROM QUEUE
queue.shift();
queuedRequest.request.respond({ status: 204 });
@@ -144,7 +143,7 @@ export async function processQueue(id: string) {
// continue;
// }
rest.eventHandlers.fetchSuccess(queuedRequest.payload);
rest.debug(`[REST - fetchSuccess] ${JSON.stringify(queuedRequest.payload)}`);
// REMOVE FROM QUEUE
queue.shift();
queuedRequest.request.respond({
@@ -154,7 +153,7 @@ export async function processQueue(id: string) {
}
} catch (error) {
// SOMETHING WENT WRONG, LOG AND RESPOND WITH ERROR
rest.eventHandlers.fetchFailed(queuedRequest.payload, error);
rest.debug(`[REST - fetchFailed] Payload: ${JSON.stringify(queuedRequest.payload)} | Error: ${error}`);
queuedRequest.request.reject(error);
// REMOVE FROM QUEUE
queue.shift();
@@ -162,5 +161,5 @@ export async function processQueue(id: string) {
}
// ONCE QUEUE IS DONE, WE CAN TRY CLEANING UP
rest.cleanupQueues();
rest.cleanupQueues(rest);
}
+5 -6
View File
@@ -1,12 +1,11 @@
import { eventHandlers } from "../bot.ts";
import { rest } from "./rest.ts";
import { RestManager } from "../bot.ts";
/** This will create a infinite loop running in 1 seconds using tail recursion to keep rate limits clean. When a rate limit resets, this will remove it so the queue can proceed. */
export function processRateLimitedPaths() {
export function processRateLimitedPaths(rest: RestManager) {
const now = Date.now();
for (const [key, value] of rest.ratelimitedPaths.entries()) {
rest.eventHandlers.debug?.("loop", `Running forEach loop in process_rate_limited_paths file.`);
rest.debug(`[REST - processRateLimitedPaths] Running forEach loop.`);
// IF THE TIME HAS NOT REACHED CANCEL
if (value.resetTimestamp > now) continue;
@@ -24,8 +23,8 @@ export function processRateLimitedPaths() {
rest.processingRateLimitedPaths = true;
// RECHECK IN 1 SECOND
setTimeout(() => {
eventHandlers.debug?.("loop", `Running setTimeout in processRateLimitedPaths function.`);
processRateLimitedPaths();
rest.debug(`[REST - processRateLimitedPaths] Running setTimeout.`);
processRateLimitedPaths(rest);
}, 1000);
}
}
+4 -3
View File
@@ -1,8 +1,9 @@
import { RestManager } from "../bot.ts";
import { BASE_URL } from "../util/constants.ts";
import { rest, RestPayload, RestRequest } from "./rest.ts";
import { RestPayload, RestRequest } from "./rest.ts";
/** Processes a request and assigns it to a queue or creates a queue if none exists for it. */
export async function processRequest(request: RestRequest, payload: RestPayload) {
export async function processRequest(rest: RestManager, request: RestRequest, payload: RestPayload) {
const route = request.url.substring(request.url.indexOf("api/"));
const parts = route.split("/");
// REMOVE THE API
@@ -10,7 +11,7 @@ export async function processRequest(request: RestRequest, payload: RestPayload)
// REMOVES THE VERSION NUMBER
if (parts[0]?.startsWith("v")) parts.shift();
// SET THE NEW REQUEST URL
request.url = `${BASE_URL}/v${rest.apiVersion}/${parts.join("/")}`;
request.url = `${BASE_URL}/v${rest.version}/${parts.join("/")}`;
// REMOVE THE MAJOR PARAM
parts.shift();
+3 -3
View File
@@ -1,7 +1,7 @@
import { rest } from "./rest.ts";
import { RestManager } from "../bot.ts";
/** Processes the rate limit headers and determines if it needs to be ratelimited and returns the bucket id if available */
export function processRequestHeaders(url: string, headers: Headers) {
export function processRequestHeaders(rest: RestManager, url: string, headers: Headers) {
let ratelimited = false;
// GET ALL NECESSARY HEADERS
@@ -37,7 +37,7 @@ export function processRequestHeaders(url: string, headers: Headers) {
if (global) {
const retryAfter = headers.get("retry-after");
const globalReset = Date.now() + Number(retryAfter) * 1000;
rest.eventHandlers.globallyRateLimited(url, globalReset);
rest.debug(`[REST = Globally Rate Limited] URL: ${url} | Global Rest: ${globalReset}`);
rest.globallyRateLimited = true;
ratelimited = true;
+5 -26
View File
@@ -1,36 +1,15 @@
import { RestManager } from "../bot.ts";
import { API_VERSION, BASE_URL, IMAGE_BASE_URL } from "../util/constants.ts";
import { loopObject } from "../util/loop_object.ts";
import { camelize } from "../util/utils.ts";
import { rest } from "./rest.ts";
// deno-lint-ignore no-explicit-any
export async function runMethod<T = any>(
rest: RestManager,
method: "get" | "post" | "put" | "delete" | "patch",
url: string,
body?: unknown,
retryCount = 0,
bucketId?: string
): Promise<T> {
if (body) {
body = loopObject(
body as Record<string, unknown>,
(value) =>
typeof value === "bigint"
? value.toString()
: Array.isArray(value)
? value.map((v) => (typeof v === "bigint" ? v.toString() : v))
: value,
`Running forEach loop in runMethod function for changing bigints to strings.`
);
}
rest.eventHandlers.debug?.("requestCreate", {
method,
url,
body,
retryCount,
bucketId,
});
rest.debug(`[REST - RequestCreate] Method: ${method} | URL: ${url} | Retry Count: ${retryCount} | Bucket ID: ${bucketId} | Body: ${JSON.stringify(body)}`)
const errorStack = new Error("Location:");
// @ts-ignore Breaks deno deploy. Luca said add tsignore until it's fixed
@@ -41,7 +20,7 @@ export async function runMethod<T = any>(
const result = await fetch(url, {
body: JSON.stringify(body || {}),
headers: {
authorization: rest.authorization,
authorization: rest.secretKey,
},
method: method.toUpperCase(),
}).catch((error) => {
@@ -64,7 +43,7 @@ export async function runMethod<T = any>(
reject(errorStack);
},
respond: (data: { status: number; body?: string }) =>
resolve(data.status !== 204 ? camelize<T>(JSON.parse(data.body ?? "{}")) : (undefined as unknown as T)),
resolve(data.status !== 204 ? JSON.parse(data.body ?? "{}") : (undefined as unknown as T)),
},
{
bucketId,
+1
View File
@@ -3,6 +3,7 @@
* Modified for our usecase
*/
/** Split a url to separate rate limit buckets based on major/minor parameters. */
export function simplifyUrl(url: string, method: string) {
let route = url
.replace(/\/([a-z-]+)\/(?:[0-9]{17,19})/g, function (match, p) {
+12 -12
View File
@@ -1,4 +1,4 @@
import { botId, eventHandlers } from "../bot.ts";
import { Bot } from "../bot.ts";
import { cache } from "../cache.ts";
import { getChannels } from "../helpers/channels/get_channels.ts";
import { getGuild } from "../helpers/guilds/get_guild.ts";
@@ -11,8 +11,8 @@ import { delay } from "./utils.ts";
const processing = new Set<bigint>();
export async function dispatchRequirements(data: DiscordGatewayPayload, shardId: number) {
if (!cache.isReady) return;
export async function dispatchRequirements(bot: Bot, data: DiscordGatewayPayload, shardId: number) {
if (!bot.isReady) return;
// DELETE MEANS WE DONT NEED TO FETCH. CREATE SHOULD HAVE DATA TO CACHE
if (data.t && ["GUILD_CREATE", "GUILD_DELETE"].includes(data.t)) return;
@@ -34,7 +34,7 @@ export async function dispatchRequirements(data: DiscordGatewayPayload, shardId:
}
if (processing.has(id)) {
eventHandlers.debug?.(`[DISPATCH] New Guild ID already being processed: ${id} in ${data.t} event`);
bot.events.debug(`[DISPATCH] New Guild ID already being processed: ${id} in ${data.t} event`);
let runs = 0;
do {
@@ -44,7 +44,7 @@ export async function dispatchRequirements(data: DiscordGatewayPayload, shardId:
if (!processing.has(id)) return;
return eventHandlers.debug?.(
return bot.events.debug(
`[DISPATCH] Already processed guild was not successfully fetched: ${id} in ${data.t} event`
);
}
@@ -52,7 +52,7 @@ export async function dispatchRequirements(data: DiscordGatewayPayload, shardId:
processing.add(id);
// New guild id has appeared, fetch all relevant data
eventHandlers.debug?.(`[DISPATCH] New Guild ID has appeared: ${id} in ${data.t} event`);
bot.events.debug(`[DISPATCH] New Guild ID has appeared: ${id} in ${data.t} event`);
const rawGuild = (await getGuild(id, {
counts: true,
@@ -61,22 +61,22 @@ export async function dispatchRequirements(data: DiscordGatewayPayload, shardId:
if (!rawGuild) {
processing.delete(id);
return eventHandlers.debug?.(`[DISPATCH] Guild ID ${id} failed to fetch.`);
return bot.events.debug(`[DISPATCH] Guild ID ${id} failed to fetch.`);
}
eventHandlers.debug?.(`[DISPATCH] Guild ID ${id} has been found. ${rawGuild.name}`);
bot.events.debug(`[DISPATCH] Guild ID ${id} has been found. ${rawGuild.name}`);
const [channels, botMember] = await Promise.all([
getChannels(id, false),
getMember(id, botId, { force: true }),
getMember(id, bot.id, { force: true }),
]).catch((error) => {
eventHandlers.debug?.(error);
bot.events.debug(error);
return [];
});
if (!botMember || !channels) {
processing.delete(id);
return eventHandlers.debug?.(
return bot.events.debug(
`[DISPATCH] Guild ID ${id} Name: ${rawGuild.name} failed. Unable to get botMember or channels`
);
}
@@ -96,5 +96,5 @@ export async function dispatchRequirements(data: DiscordGatewayPayload, shardId:
processing.delete(id);
eventHandlers.debug?.(`[DISPATCH] Guild ID ${id} Name: ${guild.name} completely loaded.`);
bot.events.debug(`[DISPATCH] Guild ID ${id} Name: ${guild.name} completely loaded.`);
}