From be27976baeeec02b9b767da7122835dc066584ca Mon Sep 17 00:00:00 2001 From: Skillz Date: Mon, 9 Nov 2020 14:41:08 -0500 Subject: [PATCH 1/2] much faster rest --- src/module/requestManager.ts | 90 +++++++++++++++++++----------------- 1 file changed, 47 insertions(+), 43 deletions(-) diff --git a/src/module/requestManager.ts b/src/module/requestManager.ts index dbbdabd15..4f14fa456 100644 --- a/src/module/requestManager.ts +++ b/src/module/requestManager.ts @@ -64,56 +64,60 @@ async function cleanupQueues() { } async function processQueue() { - if ( - (Object.keys(pathQueues).length) && !globallyRateLimited - ) { - await Promise.allSettled( - Object.values(pathQueues).map(async (pathQueue) => { - const request = pathQueue.shift(); - if (!request) return; + // Putting this code inside a function like this allows us to use tail recursion like a while loop without hitting the max stack error. + async function avoidMaxStackError() { + if ( + (Object.keys(pathQueues).length) && !globallyRateLimited + ) { + await Promise.allSettled( + Object.values(pathQueues).map(async (pathQueue) => { + const request = pathQueue.shift(); + if (!request) return; - const rateLimitedURLResetIn = await checkRatelimits(request.url); + const rateLimitedURLResetIn = await checkRatelimits(request.url); - if (request.bucketID) { - const rateLimitResetIn = await checkRatelimits(request.bucketID); - if (rateLimitResetIn) { - // This request is still rate limited readd to queue - addToQueue(request); - } else if (rateLimitedURLResetIn) { - // This URL is rate limited readd to queue - addToQueue(request); + if (request.bucketID) { + const rateLimitResetIn = await checkRatelimits(request.bucketID); + if (rateLimitResetIn) { + // This request is still rate limited readd to queue + addToQueue(request); + } else if (rateLimitedURLResetIn) { + // This URL is rate limited readd to queue + addToQueue(request); + } else { + // This request is not rate limited so it should be run + const result = await request.callback(); + if (result && result.rateLimited) { + addToQueue( + { ...request, bucketID: result.bucketID || request.bucketID }, + ); + } + } } else { - // This request is not rate limited so it should be run - const result = await request.callback(); - if (result && result.rateLimited) { - addToQueue( - { ...request, bucketID: result.bucketID || request.bucketID }, - ); + if (rateLimitedURLResetIn) { + // This URL is rate limited readd to queue + addToQueue(request); + } else { + // This request has no bucket id so it should be processed + const result = await request.callback(); + if (request && result && result.rateLimited) { + addToQueue( + { ...request, bucketID: result.bucketID || request.bucketID }, + ); + } } } - } else { - if (rateLimitedURLResetIn) { - // This URL is rate limited readd to queue - addToQueue(request); - } else { - // This request has no bucket id so it should be processed - const result = await request.callback(); - if (request && result && result.rateLimited) { - addToQueue( - { ...request, bucketID: result.bucketID || request.bucketID }, - ); - } - } - } - }), - ); + }), + ); + } + + if (Object.keys(pathQueues).length) { + avoidMaxStackError(); + cleanupQueues(); + } else queueInProcess = false; } - if (Object.keys(pathQueues).length) { - await delay(1000); - processQueue(); - cleanupQueues(); - } else queueInProcess = false; + return avoidMaxStackError() } processRateLimitedPaths(); From 7f0d52e5ba25561865efacf86a478b844b39ce12 Mon Sep 17 00:00:00 2001 From: Skillz Date: Mon, 9 Nov 2020 16:11:01 -0500 Subject: [PATCH 2/2] fmt --- src/module/requestManager.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/module/requestManager.ts b/src/module/requestManager.ts index 4f14fa456..ee96700da 100644 --- a/src/module/requestManager.ts +++ b/src/module/requestManager.ts @@ -117,7 +117,7 @@ async function processQueue() { } else queueInProcess = false; } - return avoidMaxStackError() + return avoidMaxStackError(); } processRateLimitedPaths();