From b7321a6d0ecf6ecd878467ff4bd8b3eec008a237 Mon Sep 17 00:00:00 2001 From: Skillz4Killz <23035000+Skillz4Killz@users.noreply.github.com> Date: Sat, 10 Apr 2021 13:26:41 -0400 Subject: [PATCH] fix(rest): URL limiting (#809) * fix: shtuff * fix: better url handling --- src/rest/handle_payload.ts | 2 +- src/rest/process_queue.ts | 63 ++++++++++++++------------ src/rest/process_rate_limited_paths.ts | 9 ++-- src/rest/process_request.ts | 6 +-- src/rest/process_request_headers.ts | 4 +- src/rest/rest.ts | 4 ++ src/rest/simplify_url.ts | 32 +++++++++++++ 7 files changed, 80 insertions(+), 40 deletions(-) create mode 100644 src/rest/simplify_url.ts diff --git a/src/rest/handle_payload.ts b/src/rest/handle_payload.ts index 89a723b07..3f0700a81 100644 --- a/src/rest/handle_payload.ts +++ b/src/rest/handle_payload.ts @@ -27,7 +27,7 @@ export async function handlePayload( } // PROCESS THE REQUEST - rest.processRequest(request, { body: data, retryCount: 0 }); + await rest.processRequest(request, { body: data, retryCount: 0 }); } catch (error) { rest.eventHandlers.error("serverRequest", error); } diff --git a/src/rest/process_queue.ts b/src/rest/process_queue.ts index 75d600333..8825fe697 100644 --- a/src/rest/process_queue.ts +++ b/src/rest/process_queue.ts @@ -11,16 +11,16 @@ export async function processQueue(id: string) { while (queue.length) { rest.eventHandlers.debug?.( "loop", - "Running while loop in processQueue function.", + "Running while loop in processQueue function." ); // IF THE BOT IS GLOBALLY RATELIMITED TRY AGAIN if (rest.globallyRateLimited) { - setTimeout(() => { + setTimeout(async () => { eventHandlers.debug?.( "loop", - `Running setTimeout in processQueue function.`, + `Running setTimeout in processQueue function.` ); - processQueue(id); + await processQueue(id); }, 1000); break; @@ -30,8 +30,14 @@ export async function processQueue(id: string) { // IF THIS DOESNT HAVE ANY ITEMS JUST CANCEL, THE CLEANER WILL REMOVE IT. if (!queuedRequest) return; + + const basicURL = rest.simplifyUrl( + queuedRequest.request.url, + queuedRequest.request.method.toUpperCase() + ); + // IF THIS URL IS STILL RATE LIMITED, TRY AGAIN - const urlResetIn = rest.checkRateLimits(queuedRequest.request.url); + const urlResetIn = rest.checkRateLimits(basicURL); if (urlResetIn) { // PAUSE FOR THIS SPECIFC REQUEST await delay(urlResetIn); @@ -47,19 +53,18 @@ export async function processQueue(id: string) { // EXECUTE THE REQUEST // IF THIS IS A GET REQUEST, CHANGE THE BODY TO QUERY PARAMETERS - const query = queuedRequest.request.method.toUpperCase() === "GET" && - queuedRequest.payload.body - ? Object.entries(queuedRequest.payload.body) - .map( - ([key, value]) => - `${encodeURIComponent(key)}=${ - encodeURIComponent( - value as string, - ) - }`, - ) - .join("&") - : ""; + const query = + queuedRequest.request.method.toUpperCase() === "GET" && + queuedRequest.payload.body + ? Object.entries(queuedRequest.payload.body) + .map( + ([key, value]) => + `${encodeURIComponent(key)}=${encodeURIComponent( + value as string + )}` + ) + .join("&") + : ""; const urlToUse = queuedRequest.request.method.toUpperCase() === "GET" && query ? `${queuedRequest.request.url}?${query}` @@ -71,14 +76,18 @@ export async function processQueue(id: string) { try { const response = await fetch( urlToUse, - rest.createRequestBody(queuedRequest), + rest.createRequestBody(queuedRequest) ); rest.eventHandlers.fetched(queuedRequest.payload); const bucketIdFromHeaders = rest.processRequestHeaders( - queuedRequest.request.url, - response.headers, + basicURL, + response.headers ); + // SET THE BUCKET Id IF IT WAS PRESENT + if (bucketIdFromHeaders) { + queuedRequest.payload.bucketId = bucketIdFromHeaders; + } if (response.status < 200 || response.status >= 400) { rest.eventHandlers.error("httpError", queuedRequest.payload, response); @@ -114,7 +123,8 @@ export async function processQueue(id: string) { body: JSON.stringify({ error }), }); - queue.shift(); + // If Rate limited should not remove from queue + if (response.status !== 429) queue.shift(); continue; } @@ -133,10 +143,7 @@ export async function processQueue(id: string) { json.message === "You are being rate limited." ) { // IF IT HAS MAXED RETRIES SOMETHING SERIOUSLY WRONG. CANCEL OUT. - if ( - queuedRequest.payload.retryCount >= - queuedRequest.options.maxRetryCount - ) { + if (queuedRequest.payload.retryCount >= rest.maxRetryCount) { rest.eventHandlers.retriesMaxed(queuedRequest.payload); queuedRequest.request.respond({ status: 200, @@ -150,10 +157,6 @@ export async function processQueue(id: string) { continue; } - // SET THE BUCKET Id IF IT WAS PRESENT - if (bucketIdFromHeaders) { - queuedRequest.payload.bucketId = bucketIdFromHeaders; - } // SINCE IT WAS RATELIMITE, RETRY AGAIN continue; } diff --git a/src/rest/process_rate_limited_paths.ts b/src/rest/process_rate_limited_paths.ts index 338b35b21..6202edfcb 100644 --- a/src/rest/process_rate_limited_paths.ts +++ b/src/rest/process_rate_limited_paths.ts @@ -5,18 +5,19 @@ import { rest } from "./rest.ts"; export function processRateLimitedPaths() { const now = Date.now(); - rest.ratelimitedPaths.forEach((value, key) => { + for (const [key, value] of rest.ratelimitedPaths.entries()) { rest.eventHandlers.debug?.( "loop", - `Running forEach loop in process_rate_limited_paths file.`, + `Running forEach loop in process_rate_limited_paths file.` ); // IF THE TIME HAS NOT REACHED CANCEL if (value.resetTimestamp > now) return; + // RATE LIMIT IS OVER, DELETE THE RATE LIMITER rest.ratelimitedPaths.delete(key); // IF IT WAS GLOBAL ALSO MARK THE GLOBAL VALUE AS FALSE if (key === "global") rest.globallyRateLimited = false; - }); + } // ALL PATHS ARE CLEARED CAN CANCEL OUT! if (!rest.ratelimitedPaths.size) { @@ -28,7 +29,7 @@ export function processRateLimitedPaths() { setTimeout(() => { eventHandlers.debug?.( "loop", - `Running setTimeout in processRateLimitedPaths function.`, + `Running setTimeout in processRateLimitedPaths function.` ); processRateLimitedPaths(); }, 1000); diff --git a/src/rest/process_request.ts b/src/rest/process_request.ts index 83a2ae937..89f965e9c 100644 --- a/src/rest/process_request.ts +++ b/src/rest/process_request.ts @@ -2,9 +2,9 @@ import { BASE_URL } from "../util/constants.ts"; import { rest } from "./rest.ts"; /** Processes a request and assigns it to a queue or creates a queue if none exists for it. */ -export function processRequest( +export async function processRequest( request: ServerRequest, - payload: RunMethodOptions, + payload: RunMethodOptions ) { const route = request.url.substring(request.url.indexOf("api/")); const parts = route.split("/"); @@ -31,6 +31,6 @@ export function processRequest( payload, }, ]); - rest.processQueue(id); + await rest.processQueue(id); } } diff --git a/src/rest/process_request_headers.ts b/src/rest/process_request_headers.ts index 6b07f381e..ac375459d 100644 --- a/src/rest/process_request_headers.ts +++ b/src/rest/process_request_headers.ts @@ -12,7 +12,7 @@ export function processRequestHeaders(url: string, headers: Headers) { const bucketId = headers.get("x-ratelimit-bucket"); // IF THERE IS NO REMAINING RATE LIMIT, MARK IT AS RATE LIMITED - if (remaining && remaining === "0") { + if (remaining === "0") { ratelimited = true; // SAVE THE URL AS LIMITED, IMPORTANT FOR NEW REQUESTS BY USER WITHOUT BUCKET @@ -34,7 +34,7 @@ export function processRequestHeaders(url: string, headers: Headers) { // IF THERE IS NO REMAINING GLOBAL LIMIT, MARK IT RATE LIMITED GLOBALLY if (global) { - const reset = Date.now() + (Number(retryAfter) * 1000); + const reset = Date.now() + Number(retryAfter) * 1000; rest.eventHandlers.globallyRateLimited(url, reset); rest.globallyRateLimited = true; ratelimited = true; diff --git a/src/rest/rest.ts b/src/rest/rest.ts index d61391df3..a680f447c 100644 --- a/src/rest/rest.ts +++ b/src/rest/rest.ts @@ -7,10 +7,13 @@ import { processRateLimitedPaths } from "./process_rate_limited_paths.ts"; import { processRequest } from "./process_request.ts"; import { processRequestHeaders } from "./process_request_headers.ts"; import { runMethod } from "./run_method.ts"; +import { simplifyUrl } from "./simplify_url.ts"; export const rest = { /** The bot token for this rest client. */ token: "", + /** The maximum amount of retries allowed */ + maxRetryCount: 10, apiVersion: "8", /** The secret authorization key to confirm that this was a request made by you and not a DDOS attack. */ authorization: "discordeno_best_lib_ever", @@ -41,4 +44,5 @@ export const rest = { processRequest, createRequestBody, runMethod, + simplifyUrl, }; diff --git a/src/rest/simplify_url.ts b/src/rest/simplify_url.ts new file mode 100644 index 000000000..e6643759c --- /dev/null +++ b/src/rest/simplify_url.ts @@ -0,0 +1,32 @@ +/** + * Credits: github.com/abalabahaha/eris lib/rest/RequestHandler.js#L397 + * Modified for our usecase + */ + +export function simplifyUrl(url: string, method: string) { + let route = url + .replace(/\/([a-z-]+)\/(?:[0-9]{17,19})/g, function (match, p) { + return ["channels", "guilds", "webhooks"].includes(p) + ? match + : `/${p}/skillzPrefersID`; + }) + .replace(/\/reactions\/[^/]+/g, "/reactions/skillzPrefersID") + .replace( + /^\/webhooks\/(\d+)\/[A-Za-z0-9-_]{64,}/, + "/webhooks/$1/:itohIsAHoti" + ); + + // GENERAL /reactions and /reactions/emoji/@me share the buckets + if (route.includes("/reactions")) + route = route.substring( + 0, + route.indexOf("/reactions") + "/reactions".length + ); + + // Delete Messsage endpoint has its own ratelimit + if (method === "DELETE" && route.endsWith("/messages/skillzPrefersID")) { + route = method + route; + } + + return route; +}