fix(rest): URL limiting (#809)

* fix: shtuff

* fix: better url handling

src/rest/handle_payload.ts

@@ -27,7 +27,7 @@ export async function handlePayload(
     }
 
     // PROCESS THE REQUEST
-    rest.processRequest(request, { body: data, retryCount: 0 });
+    await rest.processRequest(request, { body: data, retryCount: 0 });
   } catch (error) {
     rest.eventHandlers.error("serverRequest", error);
   }

src/rest/process_queue.ts

@@ -11,16 +11,16 @@ export async function processQueue(id: string) {
   while (queue.length) {
     rest.eventHandlers.debug?.(
       "loop",
-      "Running while loop in processQueue function.",
+      "Running while loop in processQueue function."
     );
     // IF THE BOT IS GLOBALLY RATELIMITED TRY AGAIN
     if (rest.globallyRateLimited) {
-      setTimeout(() => {
+      setTimeout(async () => {
         eventHandlers.debug?.(
           "loop",
-          `Running setTimeout in processQueue function.`,
+          `Running setTimeout in processQueue function.`
         );
-        processQueue(id);
+        await processQueue(id);
       }, 1000);
 
       break;
@@ -30,8 +30,14 @@ export async function processQueue(id: string) {
     // IF THIS DOESNT HAVE ANY ITEMS JUST CANCEL, THE CLEANER WILL REMOVE IT.
     if (!queuedRequest) return;
 
+
+    const basicURL = rest.simplifyUrl(
+      queuedRequest.request.url,
+      queuedRequest.request.method.toUpperCase()
+    );
+
     // IF THIS URL IS STILL RATE LIMITED, TRY AGAIN
-    const urlResetIn = rest.checkRateLimits(queuedRequest.request.url);
+    const urlResetIn = rest.checkRateLimits(basicURL);
     if (urlResetIn) {
       // PAUSE FOR THIS SPECIFC REQUEST
       await delay(urlResetIn);
@@ -47,16 +53,15 @@ export async function processQueue(id: string) {
     // EXECUTE THE REQUEST
 
     // IF THIS IS A GET REQUEST, CHANGE THE BODY TO QUERY PARAMETERS
-    const query = queuedRequest.request.method.toUpperCase() === "GET" &&
+    const query =
+      queuedRequest.request.method.toUpperCase() === "GET" &&
       queuedRequest.payload.body
         ? Object.entries(queuedRequest.payload.body)
             .map(
               ([key, value]) =>
-            `${encodeURIComponent(key)}=${
+                `${encodeURIComponent(key)}=${encodeURIComponent(
-              encodeURIComponent(
+                  value as string
-                value as string,
+                )}`
-              )
-            }`,
             )
             .join("&")
         : "";
@@ -71,14 +76,18 @@ export async function processQueue(id: string) {
     try {
       const response = await fetch(
         urlToUse,
-        rest.createRequestBody(queuedRequest),
+        rest.createRequestBody(queuedRequest)
       );
 
       rest.eventHandlers.fetched(queuedRequest.payload);
       const bucketIdFromHeaders = rest.processRequestHeaders(
-        queuedRequest.request.url,
+        basicURL,
-        response.headers,
+        response.headers
       );
+      // SET THE BUCKET Id IF IT WAS PRESENT
+      if (bucketIdFromHeaders) {
+        queuedRequest.payload.bucketId = bucketIdFromHeaders;
+      }
 
       if (response.status < 200 || response.status >= 400) {
         rest.eventHandlers.error("httpError", queuedRequest.payload, response);
@@ -114,7 +123,8 @@ export async function processQueue(id: string) {
           body: JSON.stringify({ error }),
         });
 
-        queue.shift();
+        // If Rate limited should not remove from queue
+        if (response.status !== 429) queue.shift();
         continue;
       }
 
@@ -133,10 +143,7 @@ export async function processQueue(id: string) {
           json.message === "You are being rate limited."
         ) {
           // IF IT HAS MAXED RETRIES SOMETHING SERIOUSLY WRONG. CANCEL OUT.
-          if (
+          if (queuedRequest.payload.retryCount >= rest.maxRetryCount) {
-            queuedRequest.payload.retryCount >=
-              queuedRequest.options.maxRetryCount
-          ) {
             rest.eventHandlers.retriesMaxed(queuedRequest.payload);
             queuedRequest.request.respond({
               status: 200,
@@ -150,10 +157,6 @@ export async function processQueue(id: string) {
             continue;
           }
 
-          // SET THE BUCKET Id IF IT WAS PRESENT
-          if (bucketIdFromHeaders) {
-            queuedRequest.payload.bucketId = bucketIdFromHeaders;
-          }
           // SINCE IT WAS RATELIMITE, RETRY AGAIN
           continue;
         }

src/rest/process_rate_limited_paths.ts

@@ -5,18 +5,19 @@ import { rest } from "./rest.ts";
 export function processRateLimitedPaths() {
   const now = Date.now();
 
-  rest.ratelimitedPaths.forEach((value, key) => {
+  for (const [key, value] of rest.ratelimitedPaths.entries()) {
     rest.eventHandlers.debug?.(
       "loop",
-      `Running forEach loop in process_rate_limited_paths file.`,
+      `Running forEach loop in process_rate_limited_paths file.`
     );
     // IF THE TIME HAS NOT REACHED CANCEL
     if (value.resetTimestamp > now) return;
+
     // RATE LIMIT IS OVER, DELETE THE RATE LIMITER
     rest.ratelimitedPaths.delete(key);
     // IF IT WAS GLOBAL ALSO MARK THE GLOBAL VALUE AS FALSE
     if (key === "global") rest.globallyRateLimited = false;
-  });
+  }
 
   // ALL PATHS ARE CLEARED CAN CANCEL OUT!
   if (!rest.ratelimitedPaths.size) {
@@ -28,7 +29,7 @@ export function processRateLimitedPaths() {
     setTimeout(() => {
       eventHandlers.debug?.(
         "loop",
-        `Running setTimeout in processRateLimitedPaths function.`,
+        `Running setTimeout in processRateLimitedPaths function.`
       );
       processRateLimitedPaths();
     }, 1000);

src/rest/process_request.ts

@@ -2,9 +2,9 @@ import { BASE_URL } from "../util/constants.ts";
 import { rest } from "./rest.ts";
 
 /** Processes a request and assigns it to a queue or creates a queue if none exists for it. */
-export function processRequest(
+export async function processRequest(
   request: ServerRequest,
-  payload: RunMethodOptions,
+  payload: RunMethodOptions
 ) {
   const route = request.url.substring(request.url.indexOf("api/"));
   const parts = route.split("/");
@@ -31,6 +31,6 @@ export function processRequest(
         payload,
       },
     ]);
-    rest.processQueue(id);
+    await rest.processQueue(id);
   }
 }

src/rest/process_request_headers.ts

@@ -12,7 +12,7 @@ export function processRequestHeaders(url: string, headers: Headers) {
   const bucketId = headers.get("x-ratelimit-bucket");
 
   // IF THERE IS NO REMAINING RATE LIMIT, MARK IT AS RATE LIMITED
-  if (remaining && remaining === "0") {
+  if (remaining === "0") {
     ratelimited = true;
 
     // SAVE THE URL AS LIMITED, IMPORTANT FOR NEW REQUESTS BY USER WITHOUT BUCKET
@@ -34,7 +34,7 @@ export function processRequestHeaders(url: string, headers: Headers) {
 
   // IF THERE IS NO REMAINING GLOBAL LIMIT, MARK IT RATE LIMITED GLOBALLY
   if (global) {
-    const reset = Date.now() + (Number(retryAfter) * 1000);
+    const reset = Date.now() + Number(retryAfter) * 1000;
     rest.eventHandlers.globallyRateLimited(url, reset);
     rest.globallyRateLimited = true;
     ratelimited = true;

src/rest/rest.ts

@@ -7,10 +7,13 @@ import { processRateLimitedPaths } from "./process_rate_limited_paths.ts";
 import { processRequest } from "./process_request.ts";
 import { processRequestHeaders } from "./process_request_headers.ts";
 import { runMethod } from "./run_method.ts";
+import { simplifyUrl } from "./simplify_url.ts";
 
 export const rest = {
   /** The bot token for this rest client. */
   token: "",
+  /** The maximum amount of retries allowed */
+  maxRetryCount: 10,
   apiVersion: "8",
   /** The secret authorization key to confirm that this was a request made by you and not a DDOS attack. */
   authorization: "discordeno_best_lib_ever",
@@ -41,4 +44,5 @@ export const rest = {
   processRequest,
   createRequestBody,
   runMethod,
+  simplifyUrl,
 };

src/rest/simplify_url.ts

@@ -0,0 +1,32 @@
+/**
+ * Credits: github.com/abalabahaha/eris lib/rest/RequestHandler.js#L397
+ * Modified for our usecase
+ */
+
+export function simplifyUrl(url: string, method: string) {
+  let route = url
+    .replace(/\/([a-z-]+)\/(?:[0-9]{17,19})/g, function (match, p) {
+      return ["channels", "guilds", "webhooks"].includes(p)
+        ? match
+        : `/${p}/skillzPrefersID`;
+    })
+    .replace(/\/reactions\/[^/]+/g, "/reactions/skillzPrefersID")
+    .replace(
+      /^\/webhooks\/(\d+)\/[A-Za-z0-9-_]{64,}/,
+      "/webhooks/$1/:itohIsAHoti"
+    );
+
+  // GENERAL /reactions and /reactions/emoji/@me share the buckets
+  if (route.includes("/reactions"))
+    route = route.substring(
+      0,
+      route.indexOf("/reactions") + "/reactions".length
+    );
+
+  // Delete Messsage endpoint has its own ratelimit
+  if (method === "DELETE" && route.endsWith("/messages/skillzPrefersID")) {
+    route = method + route;
+  }
+  
+  return route;
+}