From 6eca60777cc077cbc0a0a81e10b355eec67d17b0 Mon Sep 17 00:00:00 2001 From: Skillz Date: Mon, 22 Feb 2021 12:01:32 -0500 Subject: [PATCH 01/11] itoh is love itoh is bae this is his stuff --- src/api/handlers/channel.ts | 16 ++++++++++------ src/rest/deps.ts | 2 +- src/rest/queue.ts | 36 ++++++++++++++++++++++++------------ src/rest/request.ts | 18 ++++++++++++++---- src/rest/request_manager.ts | 2 +- src/rest/server.ts | 3 +-- src/rest/types/cache.ts | 9 ++++++++- src/rest/types/server.ts | 2 ++ 8 files changed, 61 insertions(+), 27 deletions(-) diff --git a/src/api/handlers/channel.ts b/src/api/handlers/channel.ts index 3cf81ce6f..e22b410ca 100644 --- a/src/api/handlers/channel.ts +++ b/src/api/handlers/channel.ts @@ -127,9 +127,9 @@ export async function getPins(channelID: string) { return Promise.all(result.map((res) => structures.createMessage(res))); } -/** - * Trigger a typing indicator for the specified channel. Generally bots should **NOT** implement this route. - * However, if a bot is responding to a command and expects the computation to take a few seconds, +/** + * Trigger a typing indicator for the specified channel. Generally bots should **NOT** implement this route. + * However, if a bot is responding to a command and expects the computation to take a few seconds, * this endpoint may be called to let the user know that the bot is processing their message. */ export async function startTyping(channelID: string) { @@ -240,9 +240,13 @@ export async function sendMessage( replied_user: content.mentions.repliedUser, } : undefined, - message_reference: { - message_id: content.replyMessageID, - }, + ...(content.replyMessageID + ? { + message_reference: { + message_id: content.replyMessageID, + }, + } + : {}), }, ) as MessageCreateOptions; diff --git a/src/rest/deps.ts b/src/rest/deps.ts index 1bb43a371..53b351893 100644 --- a/src/rest/deps.ts +++ b/src/rest/deps.ts @@ -1 +1 @@ -export * from "https://deno.land/std@0.87.0/http/server.ts"; +export * from "https://deno.land/std@0.88.0/http/server.ts"; diff --git a/src/rest/queue.ts b/src/rest/queue.ts index 5b36dd8c6..fa95bca92 100644 --- a/src/rest/queue.ts +++ b/src/rest/queue.ts @@ -1,6 +1,7 @@ import { restCache } from "./cache.ts"; import { createRequestBody, processRequestHeaders } from "./request.ts"; import { HttpResponseCode } from "./types/mod.ts"; +import { delay } from "../util/utils.ts"; /** If the queue is not already processing, this will start processing the queue. */ export function startQueue() { @@ -12,19 +13,24 @@ export function startQueue() { } /** Processes the queue by looping over each path separately until the queues are empty. */ -export function processQueue() { +export async function processQueue() { while (restCache.processingQueue) { // FOR EVERY PATH WE WILL START ITS OWN LOOP. restCache.pathQueues.forEach(async (queue) => { + // MAKE SURE THIS QUEUE HAS NOT ALREADY STARTED + if (queue.processing) return; // EACH PATH IS UNIQUE LIMITER - while (queue.length) { + while (queue.requests.length) { // IF THE BOT IS GLOBALLY RATELIMITED TRY AGAIN - if (!restCache.globallyRateLimited) continue; + if (restCache.globallyRateLimited) continue; // SELECT THE FIRST ITEM FROM THIS QUEUE - const [queuedRequest] = queue; + const [queuedRequest] = queue.requests; // IF THIS DOESNT HAVE ANY ITEMS JUST CANCEL, THE CLEANER WILL REMOVE IT. if (!queuedRequest) return; + // MARK THIS QUEUE AS NOW BEING PROCESSED + queue.processing = true; + // IF THIS URL IS STILL RATE LIMITED, TRY AGAIN const urlResetIn = checkRateLimits(queuedRequest.payload.url); if (urlResetIn) continue; @@ -90,7 +96,7 @@ export function processQueue() { queuedRequest.request.respond( { status: response.status, body: JSON.stringify({ error }) }, ); - queue.shift(); + queue.requests.shift(); continue; } @@ -102,7 +108,6 @@ export function processQueue() { // CONVERT THE RESPONSE TO JSON const json = await response.json(); - // IF THE RESPONSE WAS RATE LIMITED, HANDLE ACCORDINGLY if ( json.retry_after || @@ -126,7 +131,7 @@ export function processQueue() { }, ); // REMOVE ITEM FROM QUEUE TO PREVENT RETRY - queue.shift(); + queue.requests.shift(); continue; } @@ -140,7 +145,7 @@ export function processQueue() { restCache.eventHandlers.fetchSuccess(queuedRequest.payload); // REMOVE FROM QUEUE - queue.shift(); + queue.requests.shift(); queuedRequest.request.respond( { status: 200, body: JSON.stringify(json) }, ); @@ -151,23 +156,30 @@ export function processQueue() { { status: 404, body: JSON.stringify({ error }) }, ); // REMOVE FROM QUEUE - queue.shift(); + queue.requests.shift(); } } + // MARK THE QUEUE AS NO LONGER PROCESSING + queue.processing = false; // ONCE QUEUE IS DONE, WE CAN TRY CLEANING UP cleanupQueues(); }); + + await delay(1000); } } /** Cleans up the queues by checking if there is nothing left and removing it. */ export function cleanupQueues() { - restCache.pathQueues.forEach((queue, key) => { - if (queue.length) return; + for (const [key, queue] of restCache.pathQueues) { + if (queue.requests.length) continue; // REMOVE IT FROM CACHE restCache.pathQueues.delete(key); - }); + } + + // NO QUEUE LEFT, DISABLE THE QUEUE + if (!restCache.pathQueues.size) restCache.processingQueue = false; } /** Check the rate limits for a url or a bucket. */ diff --git a/src/rest/request.ts b/src/rest/request.ts index 2ecd9854b..bc83a15a7 100644 --- a/src/rest/request.ts +++ b/src/rest/request.ts @@ -1,4 +1,4 @@ -import { USER_AGENT } from "../util/constants.ts"; +import { BASE_URL, USER_AGENT } from "../util/constants.ts"; import { restCache } from "./cache.ts"; import { ServerRequest } from "./deps.ts"; import { startQueue } from "./queue.ts"; @@ -20,6 +20,8 @@ export function processRequest( parts.shift(); // REMOVES THE VERSION NUMBER if (parts[0]?.startsWith("v")) parts.shift(); + // SET THE NEW REQUEST URL + payload.url = `${BASE_URL}/v${options.apiVersion || 8}/${parts.join("/")}`; // REMOVE THE MAJOR PARAM parts.shift(); @@ -28,19 +30,27 @@ export function processRequest( const queue = restCache.pathQueues.get(id); // IF THE QUEUE EXISTS JUST ADD THIS TO THE QUEUE if (queue) { - queue.push({ request, payload, options }); + queue.requests.push({ request, payload, options }); } else { // CREATES A NEW QUEUE - restCache.pathQueues.set(id, [{ request, payload, options }]); + restCache.pathQueues.set(id, { + processing: false, + requests: [{ + request, + payload, + options, + }], + }); } + console.log("starting queue"); startQueue(); } /** Creates the request body and headers that are necessary to send a request. Will handle different types of methods and everything necessary for discord. */ export function createRequestBody(queuedRequest: QueuedRequest) { const headers: { [key: string]: string } = { - Authorization: queuedRequest.options.token, + Authorization: `Bot ${queuedRequest.options.token}`, "User-Agent": USER_AGENT, }; diff --git a/src/rest/request_manager.ts b/src/rest/request_manager.ts index 071bf91c9..1641bab06 100644 --- a/src/rest/request_manager.ts +++ b/src/rest/request_manager.ts @@ -225,7 +225,7 @@ function runMethod( body: JSON.stringify({ url, method, - ...(body as Record || {}), + body: body || {}, }), headers: { authorization: restAuthorization, diff --git a/src/rest/server.ts b/src/rest/server.ts index 6cf90ff89..4dbbdf7a3 100644 --- a/src/rest/server.ts +++ b/src/rest/server.ts @@ -23,7 +23,6 @@ async function handlePayload( // INSTANTLY IGNORE ANY REQUESTS THAT DON'T HAVE THE SECRET AUTHORIZATION KEY const authorization = request.headers.get("authorization"); if (authorization !== options.authorization) return; - // READ BUFFER AFTER AUTH CHECK const buffer = await Deno.readAll(request.body); @@ -48,7 +47,7 @@ async function handlePayload( } // PROCESS THE REQUEST - await processRequest( + processRequest( request, { method: data.method, url: data.url, body: data.body, retryCount: 0 }, options, diff --git a/src/rest/types/cache.ts b/src/rest/types/cache.ts index 76b8d0cb0..829fd45a1 100644 --- a/src/rest/types/cache.ts +++ b/src/rest/types/cache.ts @@ -3,7 +3,7 @@ import { RestEventHandlers } from "./server.ts"; export interface RestCache { /** The queues that are currently needing to be executed. Key is the url path and the value is all the requests in this same path. Paths are mapped by MAJOR params. */ - pathQueues: Map; + pathQueues: Map; /** Whether or not the queues are currently processing. */ processingQueue: boolean; /** Whether or not this token has been globally rate limited. */ @@ -13,3 +13,10 @@ export interface RestCache { /** The event handlers are functions that run when something is happening internally. Users can customize this for analytics, debugging, logging or anything their heart desires. */ eventHandlers: RestEventHandlers; } + +export interface Queue { + /** Whether or not this queue is being processed */ + processing: boolean; + /** All the requests in this queue. */ + requests: QueuedRequest[]; +} diff --git a/src/rest/types/server.ts b/src/rest/types/server.ts index ab2304b65..bf3881160 100644 --- a/src/rest/types/server.ts +++ b/src/rest/types/server.ts @@ -9,6 +9,8 @@ export interface RestServerOptions { token: string; /** When a request is rate limited, how many times should it keep retrying the request. Recommended: 10 */ maxRetryCount: number; + /** The api version you would like to use */ + apiVersion?: number; } export interface RestEventHandlers { From 17f5222c65c53c3bd30d30a6e13caf5392433277 Mon Sep 17 00:00:00 2001 From: Skillz Date: Mon, 22 Feb 2021 12:02:29 -0500 Subject: [PATCH 02/11] remove log --- src/rest/request.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/rest/request.ts b/src/rest/request.ts index bc83a15a7..63d3af520 100644 --- a/src/rest/request.ts +++ b/src/rest/request.ts @@ -43,7 +43,6 @@ export function processRequest( }); } - console.log("starting queue"); startQueue(); } From 98690a037e6aec87493ec82926c21a134b732491 Mon Sep 17 00:00:00 2001 From: Skillz Date: Mon, 22 Feb 2021 14:30:46 -0500 Subject: [PATCH 03/11] reduce complexity --- src/rest/queue.ts | 282 +++++++++++++++++++++------------------------- 1 file changed, 131 insertions(+), 151 deletions(-) diff --git a/src/rest/queue.ts b/src/rest/queue.ts index fa95bca92..1332da7fd 100644 --- a/src/rest/queue.ts +++ b/src/rest/queue.ts @@ -3,177 +3,157 @@ import { createRequestBody, processRequestHeaders } from "./request.ts"; import { HttpResponseCode } from "./types/mod.ts"; import { delay } from "../util/utils.ts"; -/** If the queue is not already processing, this will start processing the queue. */ -export function startQueue() { - // IF ALREADY PROCESSING CANCEL - if (restCache.processingQueue) return; - // MARK AS PROCESSING - restCache.processingQueue = true; - processQueue(); -} - /** Processes the queue by looping over each path separately until the queues are empty. */ -export async function processQueue() { - while (restCache.processingQueue) { - // FOR EVERY PATH WE WILL START ITS OWN LOOP. - restCache.pathQueues.forEach(async (queue) => { - // MAKE SURE THIS QUEUE HAS NOT ALREADY STARTED - if (queue.processing) return; - // EACH PATH IS UNIQUE LIMITER - while (queue.requests.length) { - // IF THE BOT IS GLOBALLY RATELIMITED TRY AGAIN - if (restCache.globallyRateLimited) continue; - // SELECT THE FIRST ITEM FROM THIS QUEUE - const [queuedRequest] = queue.requests; - // IF THIS DOESNT HAVE ANY ITEMS JUST CANCEL, THE CLEANER WILL REMOVE IT. - if (!queuedRequest) return; +export async function processQueue(id: string) { + const queue = restCache.pathQueues.get(id); + if (!queue) return; - // MARK THIS QUEUE AS NOW BEING PROCESSED - queue.processing = true; + while (queue.length) { + // IF THE BOT IS GLOBALLY RATELIMITED TRY AGAIN + if (restCache.globallyRateLimited) continue; + // SELECT THE FIRST ITEM FROM THIS QUEUE + const [queuedRequest] = queue; + // IF THIS DOESNT HAVE ANY ITEMS JUST CANCEL, THE CLEANER WILL REMOVE IT. + if (!queuedRequest) return; - // IF THIS URL IS STILL RATE LIMITED, TRY AGAIN - const urlResetIn = checkRateLimits(queuedRequest.payload.url); - if (urlResetIn) continue; + // IF THIS URL IS STILL RATE LIMITED, TRY AGAIN + const urlResetIn = checkRateLimits(queuedRequest.payload.url); + if (urlResetIn) { + // PAUSE FOR THIS SPECIFC REQUEST + await delay(urlResetIn); + continue; + } // IF A BUCKET EXISTS, CHECK THE BUCKET'S RATE LIMITS - // IF A BUCKET EXISTS, CHECK THE BUCKET'S RATE LIMITS - const bucketResetIn = queuedRequest.payload.bucketID - ? checkRateLimits(queuedRequest.payload.bucketID) - : false; - // THIS BUCKET IS STILL RATELIMITED, RE-ADD TO QUEUE - if (bucketResetIn) continue; + const bucketResetIn = queuedRequest.payload.bucketID + ? checkRateLimits(queuedRequest.payload.bucketID) + : false; + // THIS BUCKET IS STILL RATELIMITED, RE-ADD TO QUEUE + if (bucketResetIn) continue; - // EXECUTE THE REQUEST + // EXECUTE THE REQUEST - // IF THIS IS A GET REQUEST, CHANGE THE BODY TO QUERY PARAMETERS - const query = - queuedRequest.payload.method === "get" && queuedRequest.payload.body - ? Object.entries(queuedRequest.payload.body).map(([key, value]) => - `${encodeURIComponent(key)}=${ - encodeURIComponent(value as string) - }` - ) - .join("&") - : ""; - const urlToUse = queuedRequest.payload.method === "get" && query - ? `${queuedRequest.payload.url}?${query}` - : queuedRequest.payload.url; + // IF THIS IS A GET REQUEST, CHANGE THE BODY TO QUERY PARAMETERS + const query = + queuedRequest.payload.method === "get" && queuedRequest.payload.body + ? Object.entries(queuedRequest.payload.body).map(([key, value]) => + `${encodeURIComponent(key)}=${encodeURIComponent(value as string)}` + ) + .join("&") + : ""; + const urlToUse = queuedRequest.payload.method === "get" && query + ? `${queuedRequest.payload.url}?${query}` + : queuedRequest.payload.url; - // CUSTOM HANDLER FOR USER TO LOG OR WHATEVER WHENEVER A FETCH IS MADE - restCache.eventHandlers.fetching(queuedRequest.payload); + // CUSTOM HANDLER FOR USER TO LOG OR WHATEVER WHENEVER A FETCH IS MADE + restCache.eventHandlers.fetching(queuedRequest.payload); - try { - const response = await fetch( - urlToUse, - createRequestBody(queuedRequest), - ); - restCache.eventHandlers.fetched(queuedRequest.payload); - const bucketIDFromHeaders = processRequestHeaders( - queuedRequest.payload.url, - response.headers, - ); + try { + const response = await fetch( + urlToUse, + createRequestBody(queuedRequest), + ); + restCache.eventHandlers.fetched(queuedRequest.payload); + const bucketIDFromHeaders = processRequestHeaders( + queuedRequest.payload.url, + response.headers, + ); - if (response.status < 200 && response.status >= 400) { - restCache.eventHandlers.error( - "httpError", - queuedRequest.payload, - response, - ); + if (response.status < 200 && response.status >= 400) { + restCache.eventHandlers.error( + "httpError", + queuedRequest.payload, + response, + ); - const error = response.status === HttpResponseCode.BadRequest - ? "The request was improperly formatted, or the server couldn't understand it." - : response.status === HttpResponseCode.Unauthorized - ? "The Authorization header was missing or invalid." - : response.status === HttpResponseCode.Forbidden - ? "The Authorization token you passed did not have permission to the resource." - : response.status === HttpResponseCode.NotFound - ? "The resource at the location specified doesn't exist." - : response.status === HttpResponseCode.MethodNotAllowed - ? "The HTTP method used is not valid for the location specified." - : response.status === HttpResponseCode.GatewayUnavailable - ? "There was not a gateway available to process your request. Wait a bit and retry." - : "REQUEST_UNKNOWN_ERROR"; + const error = response.status === HttpResponseCode.BadRequest + ? "The request was improperly formatted, or the server couldn't understand it." + : response.status === HttpResponseCode.Unauthorized + ? "The Authorization header was missing or invalid." + : response.status === HttpResponseCode.Forbidden + ? "The Authorization token you passed did not have permission to the resource." + : response.status === HttpResponseCode.NotFound + ? "The resource at the location specified doesn't exist." + : response.status === HttpResponseCode.MethodNotAllowed + ? "The HTTP method used is not valid for the location specified." + : response.status === HttpResponseCode.GatewayUnavailable + ? "There was not a gateway available to process your request. Wait a bit and retry." + : "REQUEST_UNKNOWN_ERROR"; - queuedRequest.request.respond( - { status: response.status, body: JSON.stringify({ error }) }, - ); - queue.requests.shift(); - continue; - } - - // SOMETIMES DISCORD RETURNS AN EMPTY 204 RESPONSE THAT CAN'T BE MADE TO JSON - if (response.status === 204) { - restCache.eventHandlers.fetchSuccess(queuedRequest.payload); - return queuedRequest.request.respond({ status: 204 }); - } - - // CONVERT THE RESPONSE TO JSON - const json = await response.json(); - // IF THE RESPONSE WAS RATE LIMITED, HANDLE ACCORDINGLY - if ( - json.retry_after || - json.message === "You are being rate limited." - ) { - // IF IT HAS MAXED RETRIES SOMETHING SERIOUSLY WRONG. CANCEL OUT. - if ( - queuedRequest.payload.retryCount >= - queuedRequest.options.maxRetryCount - ) { - restCache.eventHandlers.retriesMaxed(queuedRequest.payload); - queuedRequest.request.respond( - { - status: 200, - body: JSON.stringify( - { - error: - "The request was rate limited and it maxed out the retries limit.", - }, - ), - }, - ); - // REMOVE ITEM FROM QUEUE TO PREVENT RETRY - queue.requests.shift(); - continue; - } - - // SET THE BUCKET ID IF IT WAS PRESENT - if (bucketIDFromHeaders) { - queuedRequest.payload.bucketID = bucketIDFromHeaders; - } - // SINCE IT WAS RATELIMITE, RETRY AGAIN - continue; - } - - restCache.eventHandlers.fetchSuccess(queuedRequest.payload); - // REMOVE FROM QUEUE - queue.requests.shift(); - queuedRequest.request.respond( - { status: 200, body: JSON.stringify(json) }, - ); - } catch (error) { - // SOMETHING WENT WRONG, LOG AND RESPOND WITH ERROR - restCache.eventHandlers.fetchFailed(queuedRequest.payload, error); - queuedRequest.request.respond( - { status: 404, body: JSON.stringify({ error }) }, - ); - // REMOVE FROM QUEUE - queue.requests.shift(); - } + queuedRequest.request.respond( + { status: response.status, body: JSON.stringify({ error }) }, + ); + queue.shift(); + continue; } - // MARK THE QUEUE AS NO LONGER PROCESSING - queue.processing = false; - // ONCE QUEUE IS DONE, WE CAN TRY CLEANING UP - cleanupQueues(); - }); + // SOMETIMES DISCORD RETURNS AN EMPTY 204 RESPONSE THAT CAN'T BE MADE TO JSON + if (response.status === 204) { + restCache.eventHandlers.fetchSuccess(queuedRequest.payload); + return queuedRequest.request.respond({ status: 204 }); + } - await delay(1000); + // CONVERT THE RESPONSE TO JSON + const json = await response.json(); + // IF THE RESPONSE WAS RATE LIMITED, HANDLE ACCORDINGLY + if ( + json.retry_after || + json.message === "You are being rate limited." + ) { + // IF IT HAS MAXED RETRIES SOMETHING SERIOUSLY WRONG. CANCEL OUT. + if ( + queuedRequest.payload.retryCount >= + queuedRequest.options.maxRetryCount + ) { + restCache.eventHandlers.retriesMaxed(queuedRequest.payload); + queuedRequest.request.respond( + { + status: 200, + body: JSON.stringify( + { + error: + "The request was rate limited and it maxed out the retries limit.", + }, + ), + }, + ); + // REMOVE ITEM FROM QUEUE TO PREVENT RETRY + queue.shift(); + continue; + } + + // SET THE BUCKET ID IF IT WAS PRESENT + if (bucketIDFromHeaders) { + queuedRequest.payload.bucketID = bucketIDFromHeaders; + } + // SINCE IT WAS RATELIMITE, RETRY AGAIN + continue; + } + + restCache.eventHandlers.fetchSuccess(queuedRequest.payload); + // REMOVE FROM QUEUE + queue.shift(); + queuedRequest.request.respond( + { status: 200, body: JSON.stringify(json) }, + ); + } catch (error) { + // SOMETHING WENT WRONG, LOG AND RESPOND WITH ERROR + restCache.eventHandlers.fetchFailed(queuedRequest.payload, error); + queuedRequest.request.respond( + { status: 404, body: JSON.stringify({ error }) }, + ); + // REMOVE FROM QUEUE + queue.shift(); + } } + + // ONCE QUEUE IS DONE, WE CAN TRY CLEANING UP + cleanupQueues(); } /** Cleans up the queues by checking if there is nothing left and removing it. */ export function cleanupQueues() { for (const [key, queue] of restCache.pathQueues) { - if (queue.requests.length) continue; + if (queue.length) continue; // REMOVE IT FROM CACHE restCache.pathQueues.delete(key); } From 4f8d5a4b65c13ceb6900173943dca8cbed708239 Mon Sep 17 00:00:00 2001 From: Skillz Date: Mon, 22 Feb 2021 14:31:03 -0500 Subject: [PATCH 04/11] reducing complexity --- src/rest/request.ts | 20 ++++++++------------ src/rest/types/cache.ts | 9 +-------- 2 files changed, 9 insertions(+), 20 deletions(-) diff --git a/src/rest/request.ts b/src/rest/request.ts index 63d3af520..f5c92ab49 100644 --- a/src/rest/request.ts +++ b/src/rest/request.ts @@ -1,7 +1,7 @@ import { BASE_URL, USER_AGENT } from "../util/constants.ts"; import { restCache } from "./cache.ts"; import { ServerRequest } from "./deps.ts"; -import { startQueue } from "./queue.ts"; +import { processQueue } from "./queue.ts"; import { QueuedRequest, RestServerOptions, @@ -30,20 +30,16 @@ export function processRequest( const queue = restCache.pathQueues.get(id); // IF THE QUEUE EXISTS JUST ADD THIS TO THE QUEUE if (queue) { - queue.requests.push({ request, payload, options }); + queue.push({ request, payload, options }); } else { // CREATES A NEW QUEUE - restCache.pathQueues.set(id, { - processing: false, - requests: [{ - request, - payload, - options, - }], - }); + restCache.pathQueues.set(id, [{ + request, + payload, + options, + }]); + processQueue(id); } - - startQueue(); } /** Creates the request body and headers that are necessary to send a request. Will handle different types of methods and everything necessary for discord. */ diff --git a/src/rest/types/cache.ts b/src/rest/types/cache.ts index 829fd45a1..76b8d0cb0 100644 --- a/src/rest/types/cache.ts +++ b/src/rest/types/cache.ts @@ -3,7 +3,7 @@ import { RestEventHandlers } from "./server.ts"; export interface RestCache { /** The queues that are currently needing to be executed. Key is the url path and the value is all the requests in this same path. Paths are mapped by MAJOR params. */ - pathQueues: Map; + pathQueues: Map; /** Whether or not the queues are currently processing. */ processingQueue: boolean; /** Whether or not this token has been globally rate limited. */ @@ -13,10 +13,3 @@ export interface RestCache { /** The event handlers are functions that run when something is happening internally. Users can customize this for analytics, debugging, logging or anything their heart desires. */ eventHandlers: RestEventHandlers; } - -export interface Queue { - /** Whether or not this queue is being processed */ - processing: boolean; - /** All the requests in this queue. */ - requests: QueuedRequest[]; -} From 266d16d36789e38f2e1bb62c88b81a3d7d165eab Mon Sep 17 00:00:00 2001 From: ayntee Date: Thu, 25 Feb 2021 19:57:24 +0400 Subject: [PATCH 05/11] build: update zlib & ed25519 to latest (#540) --- .swp | Bin 0 -> 12288 bytes src/interactions/deps.ts | 2 +- src/ws/deps.ts | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 .swp diff --git a/.swp b/.swp new file mode 100644 index 0000000000000000000000000000000000000000..2dab79e731f03c036d00298019bdc95bed3fb4c5 GIT binary patch literal 12288 zcmeI%%}T^D5Ww+SJ$X{r7l`$qwCX|hvR4ltSMelSo3$(UgC@ABh_9hft1Y%5!h)BD zE&Kz?kV)xGez($De06cDPIV&9*lU-cF}@M8&qZ*q&#aFBeQxy|I6(k`^$YAwkB^eL zt#cS1sQvMD{Uxb@00IagfB*srAbE_1|NZy>SF``@^8>sWEhB&c0tg_000IagfB*srATS7l zI;dBby1O#3YUlm!auG6R{jbz@n|V|7y3AFSg;4o3W1P>m&&E{GkpL6 literal 0 HcmV?d00001 diff --git a/src/interactions/deps.ts b/src/interactions/deps.ts index 842c1dde6..2bc944b72 100644 --- a/src/interactions/deps.ts +++ b/src/interactions/deps.ts @@ -1,2 +1,2 @@ export { serve } from "https://deno.land/std@0.87.0/http/server.ts"; -export { verify } from "https://esm.sh/@evan/wasm@0.0.41/target/ed25519/deno.js"; +export { verify } from "https://esm.sh/@evan/wasm@0.0.49/target/ed25519/deno.js"; diff --git a/src/ws/deps.ts b/src/ws/deps.ts index 0ccaaafef..41370935e 100644 --- a/src/ws/deps.ts +++ b/src/ws/deps.ts @@ -1 +1 @@ -export { decompress_with as decompressWith } from "https://esm.sh/@evan/wasm@0.0.41/target/zlib/deno.js"; +export { decompress_with as decompressWith } from "https://esm.sh/@evan/wasm@0.0.49/target/zlib/deno.js"; From a5c1f75b55c8f28150df741165ae4a2d72b31fd0 Mon Sep 17 00:00:00 2001 From: ayntee Date: Thu, 25 Feb 2021 20:50:01 +0400 Subject: [PATCH 06/11] =?UTF-8?q?=F0=9F=98=9B=20remove=20vim=20swap=20file?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .swp | Bin 12288 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 .swp diff --git a/.swp b/.swp deleted file mode 100644 index 2dab79e731f03c036d00298019bdc95bed3fb4c5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeI%%}T^D5Ww+SJ$X{r7l`$qwCX|hvR4ltSMelSo3$(UgC@ABh_9hft1Y%5!h)BD zE&Kz?kV)xGez($De06cDPIV&9*lU-cF}@M8&qZ*q&#aFBeQxy|I6(k`^$YAwkB^eL zt#cS1sQvMD{Uxb@00IagfB*srAbE_1|NZy>SF``@^8>sWEhB&c0tg_000IagfB*srATS7l zI;dBby1O#3YUlm!auG6R{jbz@n|V|7y3AFSg;4o3W1P>m&&E{GkpL6 From 2fcdbe58225abfb76e2350f7ac475808add3b198 Mon Sep 17 00:00:00 2001 From: ITOH <72305210+itohatweb@users.noreply.github.com> Date: Thu, 25 Feb 2021 18:37:18 +0100 Subject: [PATCH 07/11] Update request.ts --- src/rest/request.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/rest/request.ts b/src/rest/request.ts index f5c92ab49..d6a763c3a 100644 --- a/src/rest/request.ts +++ b/src/rest/request.ts @@ -14,14 +14,14 @@ export function processRequest( payload: RunMethodOptions, options: RestServerOptions, ) { - const route = payload.url.substring(payload.url.indexOf("api/")); + const route = request.url.substring(request.url.indexOf("api/")); const parts = route.split("/"); // REMOVE THE API parts.shift(); // REMOVES THE VERSION NUMBER if (parts[0]?.startsWith("v")) parts.shift(); // SET THE NEW REQUEST URL - payload.url = `${BASE_URL}/v${options.apiVersion || 8}/${parts.join("/")}`; + request.url = `${BASE_URL}/v${options.apiVersion || 8}/${parts.join("/")}`; // REMOVE THE MAJOR PARAM parts.shift(); @@ -50,7 +50,7 @@ export function createRequestBody(queuedRequest: QueuedRequest) { }; // GET METHODS SHOULD NOT HAVE A BODY - if (queuedRequest.payload.method === "get") { + if (queuedRequest.request.method === "GET") { queuedRequest.payload.body = undefined; } @@ -76,7 +76,7 @@ export function createRequestBody(queuedRequest: QueuedRequest) { queuedRequest.payload.body.file = form; } else if ( queuedRequest.payload.body && - !["get", "delete"].includes(queuedRequest.payload.method) + !["GET", "DELETE"].includes(queuedRequest.request.method) ) { headers["Content-Type"] = "application/json"; } @@ -85,7 +85,7 @@ export function createRequestBody(queuedRequest: QueuedRequest) { headers, body: queuedRequest.payload.body?.file || JSON.stringify(queuedRequest.payload.body), - method: queuedRequest.payload.method.toUpperCase(), + method: queuedRequest.request.method, }; } From 5756e137a5d5cadf0e7e4d3014e44aebb8eb605c Mon Sep 17 00:00:00 2001 From: ITOH <72305210+itohatweb@users.noreply.github.com> Date: Thu, 25 Feb 2021 18:37:23 +0100 Subject: [PATCH 08/11] Update request_manager.ts --- src/rest/request_manager.ts | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/rest/request_manager.ts b/src/rest/request_manager.ts index 1641bab06..af77cb559 100644 --- a/src/rest/request_manager.ts +++ b/src/rest/request_manager.ts @@ -222,16 +222,17 @@ function runMethod( !url.startsWith(IMAGE_BASE_URL) ) { return fetch(url, { - body: JSON.stringify({ - url, - method, - body: body || {}, - }), + body: JSON.stringify(body || {}), headers: { authorization: restAuthorization, }, + method: method.toUpperCase(), }) - .then((res) => res.json()) + .then((res) => { + if (res.status === 204) return undefined; + + return res.json(); + }) .catch((error) => { console.error(error); throw errorStack; From 95d24d941a219982a02659ed494c53795b3d0a77 Mon Sep 17 00:00:00 2001 From: ITOH <72305210+itohatweb@users.noreply.github.com> Date: Thu, 25 Feb 2021 18:37:49 +0100 Subject: [PATCH 09/11] Update queue.ts --- src/rest/queue.ts | 100 ++++++++++++++++++++++++---------------------- 1 file changed, 52 insertions(+), 48 deletions(-) diff --git a/src/rest/queue.ts b/src/rest/queue.ts index 1332da7fd..5b1f51b55 100644 --- a/src/rest/queue.ts +++ b/src/rest/queue.ts @@ -1,7 +1,7 @@ +import { delay } from "../util/utils.ts"; import { restCache } from "./cache.ts"; import { createRequestBody, processRequestHeaders } from "./request.ts"; import { HttpResponseCode } from "./types/mod.ts"; -import { delay } from "../util/utils.ts"; /** Processes the queue by looping over each path separately until the queues are empty. */ export async function processQueue(id: string) { @@ -10,14 +10,18 @@ export async function processQueue(id: string) { while (queue.length) { // IF THE BOT IS GLOBALLY RATELIMITED TRY AGAIN - if (restCache.globallyRateLimited) continue; + if (restCache.globallyRateLimited) { + setTimeout(() => processQueue(id), 1000); + + break; + } // SELECT THE FIRST ITEM FROM THIS QUEUE const [queuedRequest] = queue; // IF THIS DOESNT HAVE ANY ITEMS JUST CANCEL, THE CLEANER WILL REMOVE IT. if (!queuedRequest) return; // IF THIS URL IS STILL RATE LIMITED, TRY AGAIN - const urlResetIn = checkRateLimits(queuedRequest.payload.url); + const urlResetIn = checkRateLimits(queuedRequest.request.url); if (urlResetIn) { // PAUSE FOR THIS SPECIFC REQUEST await delay(urlResetIn); @@ -34,15 +38,15 @@ export async function processQueue(id: string) { // IF THIS IS A GET REQUEST, CHANGE THE BODY TO QUERY PARAMETERS const query = - queuedRequest.payload.method === "get" && queuedRequest.payload.body + queuedRequest.request.method === "GET" && queuedRequest.payload.body ? Object.entries(queuedRequest.payload.body).map(([key, value]) => `${encodeURIComponent(key)}=${encodeURIComponent(value as string)}` ) .join("&") : ""; - const urlToUse = queuedRequest.payload.method === "get" && query - ? `${queuedRequest.payload.url}?${query}` - : queuedRequest.payload.url; + const urlToUse = queuedRequest.request.method === "GET" && query + ? `${queuedRequest.request.url}?${query}` + : queuedRequest.request.url; // CUSTOM HANDLER FOR USER TO LOG OR WHATEVER WHENEVER A FETCH IS MADE restCache.eventHandlers.fetching(queuedRequest.payload); @@ -54,7 +58,7 @@ export async function processQueue(id: string) { ); restCache.eventHandlers.fetched(queuedRequest.payload); const bucketIDFromHeaders = processRequestHeaders( - queuedRequest.payload.url, + queuedRequest.request.url, response.headers, ); @@ -89,52 +93,52 @@ export async function processQueue(id: string) { // SOMETIMES DISCORD RETURNS AN EMPTY 204 RESPONSE THAT CAN'T BE MADE TO JSON if (response.status === 204) { restCache.eventHandlers.fetchSuccess(queuedRequest.payload); - return queuedRequest.request.respond({ status: 204 }); - } - - // CONVERT THE RESPONSE TO JSON - const json = await response.json(); - // IF THE RESPONSE WAS RATE LIMITED, HANDLE ACCORDINGLY - if ( - json.retry_after || - json.message === "You are being rate limited." - ) { - // IF IT HAS MAXED RETRIES SOMETHING SERIOUSLY WRONG. CANCEL OUT. + queuedRequest.request.respond({ status: 204 }); + } else { + // CONVERT THE RESPONSE TO JSON + const json = await response.json(); + // IF THE RESPONSE WAS RATE LIMITED, HANDLE ACCORDINGLY if ( - queuedRequest.payload.retryCount >= - queuedRequest.options.maxRetryCount + json.retry_after || + json.message === "You are being rate limited." ) { - restCache.eventHandlers.retriesMaxed(queuedRequest.payload); - queuedRequest.request.respond( - { - status: 200, - body: JSON.stringify( - { - error: - "The request was rate limited and it maxed out the retries limit.", - }, - ), - }, - ); - // REMOVE ITEM FROM QUEUE TO PREVENT RETRY - queue.shift(); + // IF IT HAS MAXED RETRIES SOMETHING SERIOUSLY WRONG. CANCEL OUT. + if ( + queuedRequest.payload.retryCount >= + queuedRequest.options.maxRetryCount + ) { + restCache.eventHandlers.retriesMaxed(queuedRequest.payload); + queuedRequest.request.respond( + { + status: 200, + body: JSON.stringify( + { + error: + "The request was rate limited and it maxed out the retries limit.", + }, + ), + }, + ); + // REMOVE ITEM FROM QUEUE TO PREVENT RETRY + queue.shift(); + continue; + } + + // SET THE BUCKET ID IF IT WAS PRESENT + if (bucketIDFromHeaders) { + queuedRequest.payload.bucketID = bucketIDFromHeaders; + } + // SINCE IT WAS RATELIMITE, RETRY AGAIN continue; } - // SET THE BUCKET ID IF IT WAS PRESENT - if (bucketIDFromHeaders) { - queuedRequest.payload.bucketID = bucketIDFromHeaders; - } - // SINCE IT WAS RATELIMITE, RETRY AGAIN - continue; + restCache.eventHandlers.fetchSuccess(queuedRequest.payload); + // REMOVE FROM QUEUE + queue.shift(); + queuedRequest.request.respond( + { status: 200, body: JSON.stringify(json) }, + ); } - - restCache.eventHandlers.fetchSuccess(queuedRequest.payload); - // REMOVE FROM QUEUE - queue.shift(); - queuedRequest.request.respond( - { status: 200, body: JSON.stringify(json) }, - ); } catch (error) { // SOMETHING WENT WRONG, LOG AND RESPOND WITH ERROR restCache.eventHandlers.fetchFailed(queuedRequest.payload, error); From b55193a51c88b02a580c7715354a6e4db0afe358 Mon Sep 17 00:00:00 2001 From: ITOH <72305210+itohatweb@users.noreply.github.com> Date: Thu, 25 Feb 2021 18:37:53 +0100 Subject: [PATCH 10/11] Update server.ts --- src/rest/server.ts | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/src/rest/server.ts b/src/rest/server.ts index 4dbbdf7a3..aa53a9f47 100644 --- a/src/rest/server.ts +++ b/src/rest/server.ts @@ -25,33 +25,24 @@ async function handlePayload( if (authorization !== options.authorization) return; // READ BUFFER AFTER AUTH CHECK const buffer = await Deno.readAll(request.body); - try { // CONVERT THE BODY TO JSON const data = JSON.parse(new TextDecoder().decode(buffer)); - if (!data.url) { + if ( + !["GET", "POST", "PUT", "PATCH", "HEAD", "DELETE"].includes( + request.method, + ) + ) { return request.respond( { status: 400, - body: JSON.stringify({ error: "No URL was provided." }), - }, - ); - } - if (!data.method) { - return request.respond( - { - status: 400, - body: JSON.stringify({ error: "No METHOD was provided." }), + body: JSON.stringify({ error: "Invalid METHOD." }), }, ); } // PROCESS THE REQUEST - processRequest( - request, - { method: data.method, url: data.url, body: data.body, retryCount: 0 }, - options, - ); + processRequest(request, { body: data, retryCount: 0 }, options); } catch (error) { restCache.eventHandlers.error("serverRequest", error); } From 53c22b20f74fff6d5417767227ce534feb56118a Mon Sep 17 00:00:00 2001 From: ITOH <72305210+itohatweb@users.noreply.github.com> Date: Thu, 25 Feb 2021 18:37:57 +0100 Subject: [PATCH 11/11] Update requests.ts --- src/rest/types/requests.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/rest/types/requests.ts b/src/rest/types/requests.ts index bfea5ec2b..415916ca1 100644 --- a/src/rest/types/requests.ts +++ b/src/rest/types/requests.ts @@ -7,8 +7,6 @@ export type RequestMethods = | "delete"; export interface RunMethodOptions { - method: RequestMethods; - url: string; retryCount: number; // deno-lint-ignore no-explicit-any body?: any;