This commit is contained in:
Skillz4Killz
2021-09-30 08:06:42 +00:00
committed by GitHub
parent f44e6c7b43
commit a9e5ce01c9
11 changed files with 180 additions and 135 deletions
+2 -2
View File
@@ -1,7 +1,7 @@
import { rest } from "./rest.ts";
import { RestManager } from "../bot.ts";
/** Check the rate limits for a url or a bucket. */
export function checkRateLimits(url: string) {
export function checkRateLimits(rest: RestManager, url: string) {
const ratelimited = rest.ratelimitedPaths.get(url);
const global = rest.ratelimitedPaths.get("global");
const now = Date.now();
+3 -3
View File
@@ -1,9 +1,9 @@
import { rest } from "./rest.ts";
import { RestManager } from "../bot.ts";
/** Cleans up the queues by checking if there is nothing left and removing it. */
export function cleanupQueues() {
export function cleanupQueues(rest: RestManager) {
for (const [key, queue] of rest.pathQueues) {
rest.eventHandlers.debug?.("loop", "Running for of loop in cleanupQueues function.");
rest.debug(`[REST - cleanupQueues] Running for of loop. ${key}`);
if (queue.length) continue;
// REMOVE IT FROM CACHE
rest.pathQueues.delete(key);
+3 -2
View File
@@ -1,9 +1,10 @@
import { RestManager } from "../bot.ts";
import type { FileContent } from "../types/discordeno/file_content.ts";
import { USER_AGENT } from "../util/constants.ts";
import { rest, RestPayload, RestRequest } from "./rest.ts";
import { RestPayload, RestRequest } from "./rest.ts";
/** Creates the request body and headers that are necessary to send a request. Will handle different types of methods and everything necessary for discord. */
export function createRequestBody(queuedRequest: { request: RestRequest; payload: RestPayload }) {
export function createRequestBody(rest: RestManager, queuedRequest: { request: RestRequest; payload: RestPayload }) {
const headers: { [key: string]: string } = {
Authorization: rest.token,
"User-Agent": USER_AGENT,
+17 -18
View File
@@ -1,20 +1,19 @@
import { eventHandlers } from "../bot.ts";
import { RestManager } from "../bot.ts";
import { DiscordHTTPResponseCodes } from "../types/codes/http_response_codes.ts";
import { delay } from "../util/utils.ts";
import { rest } from "./rest.ts";
/** Processes the queue by looping over each path separately until the queues are empty. */
export async function processQueue(id: string) {
export async function processQueue(rest: RestManager, id: string) {
const queue = rest.pathQueues.get(id);
if (!queue) return;
while (queue.length) {
rest.eventHandlers.debug?.("loop", "Running while loop in processQueue function.");
rest.debug(`[REST - processQueue] Running while loop.`);
// IF THE BOT IS GLOBALLY RATELIMITED TRY AGAIN
if (rest.globallyRateLimited) {
setTimeout(async () => {
eventHandlers.debug?.("loop", `Running setTimeout in processQueue function.`);
await processQueue(id);
rest.debug(`[REST - processQueue] Running setTimeout.`);
await processQueue(rest, id);
}, 1000);
break;
@@ -27,7 +26,7 @@ export async function processQueue(id: string) {
const basicURL = rest.simplifyUrl(queuedRequest.request.url, queuedRequest.request.method.toUpperCase());
// IF THIS URL IS STILL RATE LIMITED, TRY AGAIN
const urlResetIn = rest.checkRateLimits(basicURL);
const urlResetIn = rest.checkRateLimits(rest, basicURL);
if (urlResetIn) {
// PAUSE FOR THIS SPECIFC REQUEST
await delay(urlResetIn);
@@ -35,7 +34,7 @@ export async function processQueue(id: string) {
}
// IF A BUCKET EXISTS, CHECK THE BUCKET'S RATE LIMITS
const bucketResetIn = queuedRequest.payload.bucketId ? rest.checkRateLimits(queuedRequest.payload.bucketId) : false;
const bucketResetIn = queuedRequest.payload.bucketId ? rest.checkRateLimits(rest, queuedRequest.payload.bucketId) : false;
// THIS BUCKET IS STILL RATELIMITED, RE-ADD TO QUEUE
if (bucketResetIn) continue;
@@ -59,20 +58,20 @@ export async function processQueue(id: string) {
: queuedRequest.request.url;
// CUSTOM HANDLER FOR USER TO LOG OR WHATEVER WHENEVER A FETCH IS MADE
rest.eventHandlers.fetching(queuedRequest.payload);
rest.debug(`[REST - fetching] ${JSON.stringify(queuedRequest.payload)}`);
try {
const response = await fetch(urlToUse, rest.createRequestBody(queuedRequest));
const response = await fetch(urlToUse, rest.createRequestBody(rest, queuedRequest));
rest.debug(`[REST - fetched] ${JSON.stringify(queuedRequest.payload)}`);
rest.eventHandlers.fetched(queuedRequest.payload);
const bucketIdFromHeaders = rest.processRequestHeaders(basicURL, response.headers);
const bucketIdFromHeaders = rest.processRequestHeaders(rest, basicURL, response.headers);
// SET THE BUCKET Id IF IT WAS PRESENT
if (bucketIdFromHeaders) {
queuedRequest.payload.bucketId = bucketIdFromHeaders;
}
if (response.status < 200 || response.status >= 400) {
rest.eventHandlers.error("httpError", queuedRequest.payload, response);
rest.debug(`[REST - httpError] Payload: ${JSON.stringify(queuedRequest.payload)} | Response: ${JSON.stringify(response)}`);
let error = "REQUEST_UNKNOWN_ERROR";
switch (response.status) {
@@ -102,7 +101,7 @@ export async function processQueue(id: string) {
queue.shift();
} else {
if (queuedRequest.payload.retryCount++ >= rest.maxRetryCount) {
rest.eventHandlers.retriesMaxed(queuedRequest.payload);
rest.debug(`[REST - RetriesMaxed] ${JSON.stringify(queuedRequest.payload)}`);
queuedRequest.request.reject(
new Error(`[${response.status}] The request was rate limited and it maxed out the retries limit.`)
);
@@ -117,7 +116,7 @@ export async function processQueue(id: string) {
// SOMETIMES DISCORD RETURNS AN EMPTY 204 RESPONSE THAT CAN'T BE MADE TO JSON
if (response.status === 204) {
rest.eventHandlers.fetchSuccess(queuedRequest.payload);
rest.debug(`[REST - FetchSuccess] ${JSON.stringify(queuedRequest.payload)}`);
// REMOVE FROM QUEUE
queue.shift();
queuedRequest.request.respond({ status: 204 });
@@ -144,7 +143,7 @@ export async function processQueue(id: string) {
// continue;
// }
rest.eventHandlers.fetchSuccess(queuedRequest.payload);
rest.debug(`[REST - fetchSuccess] ${JSON.stringify(queuedRequest.payload)}`);
// REMOVE FROM QUEUE
queue.shift();
queuedRequest.request.respond({
@@ -154,7 +153,7 @@ export async function processQueue(id: string) {
}
} catch (error) {
// SOMETHING WENT WRONG, LOG AND RESPOND WITH ERROR
rest.eventHandlers.fetchFailed(queuedRequest.payload, error);
rest.debug(`[REST - fetchFailed] Payload: ${JSON.stringify(queuedRequest.payload)} | Error: ${error}`);
queuedRequest.request.reject(error);
// REMOVE FROM QUEUE
queue.shift();
@@ -162,5 +161,5 @@ export async function processQueue(id: string) {
}
// ONCE QUEUE IS DONE, WE CAN TRY CLEANING UP
rest.cleanupQueues();
rest.cleanupQueues(rest);
}
+5 -6
View File
@@ -1,12 +1,11 @@
import { eventHandlers } from "../bot.ts";
import { rest } from "./rest.ts";
import { RestManager } from "../bot.ts";
/** This will create a infinite loop running in 1 seconds using tail recursion to keep rate limits clean. When a rate limit resets, this will remove it so the queue can proceed. */
export function processRateLimitedPaths() {
export function processRateLimitedPaths(rest: RestManager) {
const now = Date.now();
for (const [key, value] of rest.ratelimitedPaths.entries()) {
rest.eventHandlers.debug?.("loop", `Running forEach loop in process_rate_limited_paths file.`);
rest.debug(`[REST - processRateLimitedPaths] Running forEach loop.`);
// IF THE TIME HAS NOT REACHED CANCEL
if (value.resetTimestamp > now) continue;
@@ -24,8 +23,8 @@ export function processRateLimitedPaths() {
rest.processingRateLimitedPaths = true;
// RECHECK IN 1 SECOND
setTimeout(() => {
eventHandlers.debug?.("loop", `Running setTimeout in processRateLimitedPaths function.`);
processRateLimitedPaths();
rest.debug(`[REST - processRateLimitedPaths] Running setTimeout.`);
processRateLimitedPaths(rest);
}, 1000);
}
}
+4 -3
View File
@@ -1,8 +1,9 @@
import { RestManager } from "../bot.ts";
import { BASE_URL } from "../util/constants.ts";
import { rest, RestPayload, RestRequest } from "./rest.ts";
import { RestPayload, RestRequest } from "./rest.ts";
/** Processes a request and assigns it to a queue or creates a queue if none exists for it. */
export async function processRequest(request: RestRequest, payload: RestPayload) {
export async function processRequest(rest: RestManager, request: RestRequest, payload: RestPayload) {
const route = request.url.substring(request.url.indexOf("api/"));
const parts = route.split("/");
// REMOVE THE API
@@ -10,7 +11,7 @@ export async function processRequest(request: RestRequest, payload: RestPayload)
// REMOVES THE VERSION NUMBER
if (parts[0]?.startsWith("v")) parts.shift();
// SET THE NEW REQUEST URL
request.url = `${BASE_URL}/v${rest.apiVersion}/${parts.join("/")}`;
request.url = `${BASE_URL}/v${rest.version}/${parts.join("/")}`;
// REMOVE THE MAJOR PARAM
parts.shift();
+3 -3
View File
@@ -1,7 +1,7 @@
import { rest } from "./rest.ts";
import { RestManager } from "../bot.ts";
/** Processes the rate limit headers and determines if it needs to be ratelimited and returns the bucket id if available */
export function processRequestHeaders(url: string, headers: Headers) {
export function processRequestHeaders(rest: RestManager, url: string, headers: Headers) {
let ratelimited = false;
// GET ALL NECESSARY HEADERS
@@ -37,7 +37,7 @@ export function processRequestHeaders(url: string, headers: Headers) {
if (global) {
const retryAfter = headers.get("retry-after");
const globalReset = Date.now() + Number(retryAfter) * 1000;
rest.eventHandlers.globallyRateLimited(url, globalReset);
rest.debug(`[REST = Globally Rate Limited] URL: ${url} | Global Rest: ${globalReset}`);
rest.globallyRateLimited = true;
ratelimited = true;
+5 -26
View File
@@ -1,36 +1,15 @@
import { RestManager } from "../bot.ts";
import { API_VERSION, BASE_URL, IMAGE_BASE_URL } from "../util/constants.ts";
import { loopObject } from "../util/loop_object.ts";
import { camelize } from "../util/utils.ts";
import { rest } from "./rest.ts";
// deno-lint-ignore no-explicit-any
export async function runMethod<T = any>(
rest: RestManager,
method: "get" | "post" | "put" | "delete" | "patch",
url: string,
body?: unknown,
retryCount = 0,
bucketId?: string
): Promise<T> {
if (body) {
body = loopObject(
body as Record<string, unknown>,
(value) =>
typeof value === "bigint"
? value.toString()
: Array.isArray(value)
? value.map((v) => (typeof v === "bigint" ? v.toString() : v))
: value,
`Running forEach loop in runMethod function for changing bigints to strings.`
);
}
rest.eventHandlers.debug?.("requestCreate", {
method,
url,
body,
retryCount,
bucketId,
});
rest.debug(`[REST - RequestCreate] Method: ${method} | URL: ${url} | Retry Count: ${retryCount} | Bucket ID: ${bucketId} | Body: ${JSON.stringify(body)}`)
const errorStack = new Error("Location:");
// @ts-ignore Breaks deno deploy. Luca said add tsignore until it's fixed
@@ -41,7 +20,7 @@ export async function runMethod<T = any>(
const result = await fetch(url, {
body: JSON.stringify(body || {}),
headers: {
authorization: rest.authorization,
authorization: rest.secretKey,
},
method: method.toUpperCase(),
}).catch((error) => {
@@ -64,7 +43,7 @@ export async function runMethod<T = any>(
reject(errorStack);
},
respond: (data: { status: number; body?: string }) =>
resolve(data.status !== 204 ? camelize<T>(JSON.parse(data.body ?? "{}")) : (undefined as unknown as T)),
resolve(data.status !== 204 ? JSON.parse(data.body ?? "{}") : (undefined as unknown as T)),
},
{
bucketId,
+1
View File
@@ -3,6 +3,7 @@
* Modified for our usecase
*/
/** Split a url to separate rate limit buckets based on major/minor parameters. */
export function simplifyUrl(url: string, method: string) {
let route = url
.replace(/\/([a-z-]+)\/(?:[0-9]{17,19})/g, function (match, p) {