From 8c43f36424d2a7f19fd3e1212ff56fc68c06f175 Mon Sep 17 00:00:00 2001 From: ITOH Date: Thu, 13 May 2021 10:47:20 +0200 Subject: [PATCH 01/14] just fmt --- src/handlers/guilds/GUILD_DELETE.ts | 1 - src/ws/ws.ts | 20 ++++++++++---------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/handlers/guilds/GUILD_DELETE.ts b/src/handlers/guilds/GUILD_DELETE.ts index ea30c220b..d7144e4cf 100644 --- a/src/handlers/guilds/GUILD_DELETE.ts +++ b/src/handlers/guilds/GUILD_DELETE.ts @@ -22,7 +22,6 @@ export async function handleGuildDelete( if (payload.unavailable) { const shard = ws.shards.get(shardId); if (shard) shard.unavailableGuildIds.add(guild.id); - await cacheHandlers.set("unavailableGuilds", guild.id, Date.now()); eventHandlers.guildUnavailable?.(guild); diff --git a/src/ws/ws.ts b/src/ws/ws.ts index b9611681a..92cf3640b 100644 --- a/src/ws/ws.ts +++ b/src/ws/ws.ts @@ -17,7 +17,7 @@ import { tellClusterToIdentify } from "./tell_cluster_to_identify.ts"; // CONTROLLER LIKE INTERFACE FOR WS HANDLING export const ws = { - /** The secret key authorization header the bot will expect when sending payloads */ + /** The secret key authorization header the bot will expect when sending payloads. */ secretKey: "", /** The url that all discord payloads for the dispatch type should be sent to. */ url: "", @@ -29,7 +29,7 @@ export const ws = { maxShards: 0, /** Whether or not the resharder should automatically switch to LARGE BOT SHARDING when you are above 100K servers. */ useOptimalLargeBotSharding: true, - /** The amount of shards to load per cluster */ + /** The amount of shards to load per cluster. */ shardsPerCluster: 25, /** The maximum amount of clusters to use for your bot. */ maxClusters: 4, @@ -96,7 +96,7 @@ export const ws = { createShard, /** Begins identification of the shard to discord. */ identify, - /** Begins heartbeating of the shard to keep it alive */ + /** Begins heartbeating of the shard to keep it alive. */ heartbeat, /** Sends the discord payload to another server. */ handleDiscordPayload, @@ -114,16 +114,16 @@ export const ws = { processQueue, /** Closes shard WebSocket connection properly. */ closeWS, - /** Properly adds a message to the shards queue */ + /** Properly adds a message to the shards queue. */ sendShardMessage, }; export interface DiscordenoShard { - /** The shard id number */ + /** The shard id number. */ id: number; - /** The websocket for this shard */ + /** The websocket for this shard. */ ws: WebSocket; - /** The amount of milliseconds to wait between heartbeats */ + /** The amount of milliseconds to wait between heartbeats. */ resumeInterval: number; /** The session id important for resuming connections. */ sessionId: string; @@ -131,12 +131,12 @@ export interface DiscordenoShard { previousSequenceNumber: number | null; /** Whether the shard is currently resuming. */ resuming: boolean; - /** Whether the shard has received the ready event */ + /** Whether the shard has received the ready event. */ ready: boolean; /** The list of guild ids that are currently unavailable due to an outage. */ unavailableGuildIds: Set; heartbeat: { - /** The exact timestamp the last heartbeat was sent */ + /** The exact timestamp the last heartbeat was sent. */ lastSentAt: number; /** The timestamp the last heartbeat ACK was received from discord. */ lastReceivedAt: number; @@ -155,7 +155,7 @@ export interface DiscordenoShard { processingQueue: boolean; /** When the first request for this minute has been sent. */ queueStartedAt: number; - /** The request counter of the queue */ + /** The request counter of the queue. */ queueCounter: number; } From 3c1033022c057d93f0eafa9d15381e89758b805b Mon Sep 17 00:00:00 2001 From: ITOH Date: Thu, 13 May 2021 10:47:48 +0200 Subject: [PATCH 02/14] add: lastAvailable param --- src/ws/identify.ts | 1 + src/ws/resume.ts | 1 + src/ws/ws.ts | 2 ++ 3 files changed, 4 insertions(+) diff --git a/src/ws/identify.ts b/src/ws/identify.ts index c645df9d1..68fc669cf 100644 --- a/src/ws/identify.ts +++ b/src/ws/identify.ts @@ -24,6 +24,7 @@ export async function identify(shardId: number, maxShards: number) { resuming: false, ready: false, unavailableGuildIds: new Set(), + lastAvailable: 0, heartbeat: { lastSentAt: 0, lastReceivedAt: 0, diff --git a/src/ws/resume.ts b/src/ws/resume.ts index d1efa4e7b..ce47f63ff 100644 --- a/src/ws/resume.ts +++ b/src/ws/resume.ts @@ -30,6 +30,7 @@ export async function resume(shardId: number) { resuming: false, ready: false, unavailableGuildIds: new Set(), + lastAvailable: 0, heartbeat: { lastSentAt: 0, lastReceivedAt: 0, diff --git a/src/ws/ws.ts b/src/ws/ws.ts index 92cf3640b..34f060029 100644 --- a/src/ws/ws.ts +++ b/src/ws/ws.ts @@ -135,6 +135,8 @@ export interface DiscordenoShard { ready: boolean; /** The list of guild ids that are currently unavailable due to an outage. */ unavailableGuildIds: Set; + /** Last time when a GUILD_CREATE event has been received for an unavailable guild. This is used to prevent infinite loops in the READY event handler. */ + lastAvailable: number; heartbeat: { /** The exact timestamp the last heartbeat was sent. */ lastSentAt: number; From 23b0e1d3838936f28ee869833eebee283acb3ca2 Mon Sep 17 00:00:00 2001 From: ITOH Date: Thu, 13 May 2021 10:49:01 +0200 Subject: [PATCH 03/14] change: shard events pass the shard object --- src/types/discordeno/eventHandlers.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/types/discordeno/eventHandlers.ts b/src/types/discordeno/eventHandlers.ts index c456c91ee..65d1475af 100644 --- a/src/types/discordeno/eventHandlers.ts +++ b/src/types/discordeno/eventHandlers.ts @@ -4,6 +4,7 @@ import { DiscordenoMember } from "../../structures/member.ts"; import { DiscordenoMessage } from "../../structures/message.ts"; import { DiscordenoRole } from "../../structures/role.ts"; import { Collection } from "../../util/collection.ts"; +import { DiscordenoShard } from "../../ws/ws.ts"; import { ThreadMember } from "../channels/threads/thread_member.ts"; import { ThreadMembersUpdate } from "../channels/threads/thread_members_update.ts"; import { IntegrationCreateUpdate } from "../integrations/integration_create_update.ts"; @@ -207,10 +208,10 @@ export interface EventHandlers { member: DiscordenoMember, roleId: bigint, ) => unknown; - shardReady?: (shardId: number) => unknown; + shardReady?: (shard: DiscordenoShard) => unknown; /** Sent when a shard failed to load. */ shardFailedToLoad?: ( - shardId: number, + shard: DiscordenoShard, unavailableGuildIds: Set, ) => unknown; /** Sent when a thread is created */ From 9b8fe4e336091fd9644b1cf840253d551b68e749 Mon Sep 17 00:00:00 2001 From: ITOH Date: Thu, 13 May 2021 10:49:23 +0200 Subject: [PATCH 04/14] change: implement new better unavailable check --- src/handlers/guilds/GUILD_CREATE.ts | 2 +- src/handlers/misc/READY.ts | 92 +++++++++++++---------------- 2 files changed, 43 insertions(+), 51 deletions(-) diff --git a/src/handlers/guilds/GUILD_CREATE.ts b/src/handlers/guilds/GUILD_CREATE.ts index 3c62e57ea..fcc43412b 100644 --- a/src/handlers/guilds/GUILD_CREATE.ts +++ b/src/handlers/guilds/GUILD_CREATE.ts @@ -24,8 +24,8 @@ export async function handleGuildCreate( if (shard?.unavailableGuildIds.has(guild.id)) { await cacheHandlers.delete("unavailableGuilds", guild.id); - shard.unavailableGuildIds.delete(guild.id); + shard.lastAvailable = Date.now(); return eventHandlers.guildAvailable?.(guild); } diff --git a/src/handlers/misc/READY.ts b/src/handlers/misc/READY.ts index a36da53b6..5a7c0c019 100644 --- a/src/handlers/misc/READY.ts +++ b/src/handlers/misc/READY.ts @@ -3,7 +3,7 @@ import { cache } from "../../cache.ts"; import type { DiscordGatewayPayload } from "../../types/gateway/gateway_payload.ts"; import type { Ready } from "../../types/gateway/ready.ts"; import { snowflakeToBigint } from "../../util/bigint.ts"; -import { ws } from "../../ws/ws.ts"; +import { DiscordenoShard, ws } from "../../ws/ws.ts"; export function handleReady( data: DiscordGatewayPayload, @@ -12,17 +12,15 @@ export function handleReady( // The bot has already started, the last shard is resumed, however. if (cache.isReady) return; + const shard = ws.shards.get(shardId); + if (!shard) return; + const payload = data.d as Ready; setBotId(payload.user.id); setApplicationId(payload.application.id); // Triggered on each shard - eventHandlers.shardReady?.(shardId); - // Save when the READY event was received to prevent infinite load loops - const now = Date.now(); - - const shard = ws.shards.get(shardId); - if (!shard) return; + eventHandlers.shardReady?.(shard); // Set ready to false just to go sure shard.ready = false; @@ -30,6 +28,8 @@ export function handleReady( shard.unavailableGuildIds = new Set( payload.guilds.map((g) => snowflakeToBigint(g.id)), ); + // Set the last available to now + shard.lastAvailable = Date.now(); // Start ready check in 2 seconds setTimeout(() => { @@ -37,58 +37,50 @@ export function handleReady( "loop", `1. Running setTimeout in READY file.`, ); - checkReady(payload, shardId, now); + checkReady(payload, shard); }, 2000); } -// Don't pass the shard itself because unavailableGuilds won't be updated by the GUILD_CREATE event /** This function checks if the shard is fully loaded */ -function checkReady(payload: Ready, shardId: number, now: number) { - const shard = ws.shards.get(shardId); - if (!shard) return; - +function checkReady(payload: Ready, shard: DiscordenoShard) { // Check if all guilds were loaded - if (shard.unavailableGuildIds.size) { - if (Date.now() - now > 10000) { - eventHandlers.shardFailedToLoad?.(shardId, shard.unavailableGuildIds); - // Force execute the loaded function to prevent infinite loop - loaded(shardId); - } else { - // Not all guilds were loaded but 10 seconds haven't passed so check again - setTimeout(() => { - eventHandlers.debug?.( - "loop", - `2. Running setTimeout in READY file.`, - ); - checkReady(payload, shardId, now); - }, 2000); - } - } else { - // All guilds were loaded - loaded(shardId); + if (!shard.unavailableGuildIds.size) return loaded(shard); + + // If the last GUILD_CREATE has been received before 5 seconds if so most likely the remaining guilds are unavailable + if (shard.lastAvailable + 5000 < Date.now()) { + eventHandlers.shardFailedToLoad?.(shard, shard.unavailableGuildIds); + // Force execute the loaded function to prevent infinite loop + loaded(shard); } + + // Not all guilds were loaded but 5 seconds haven't passed so check again + setTimeout(() => { + eventHandlers.debug?.( + "loop", + `2. Running setTimeout in READY file.`, + ); + checkReady(payload, shard); + }, 2000); } -function loaded(shardId: number) { - const shard = ws.shards.get(shardId); - if (!shard) return; - +function loaded(shard: DiscordenoShard) { shard.ready = true; - // If it is the last shard we can go full ready - if (shardId === ws.maxShards - 1) { - // Still some shards are loading so wait another 2 seconds for them - if (ws.shards.some((shard) => !shard.ready)) { - setTimeout(() => { - eventHandlers.debug?.( - "loop", - `3. Running setTimeout in READY file.`, - ); - loaded(shardId); - }, 2000); - } else { - cache.isReady = true; - eventHandlers.ready?.(); - } + // If it is not the last shard we can't go full ready + if (shard.id !== ws.maxShards - 1) return; + + // Still some shards are loading so wait another 2 seconds for them + if (ws.shards.some((shard) => !shard.ready)) { + setTimeout(() => { + eventHandlers.debug?.( + "loop", + `3. Running setTimeout in READY file.`, + ); + loaded(shard); + }, 2000); + return; } + + cache.isReady = true; + eventHandlers.ready?.(); } From 882787ac22c90330aab1b467a531b99c77c1edc6 Mon Sep 17 00:00:00 2001 From: ITOH Date: Thu, 13 May 2021 11:04:12 +0200 Subject: [PATCH 05/14] fix: checkReady return loaded() --- src/handlers/misc/READY.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/handlers/misc/READY.ts b/src/handlers/misc/READY.ts index 5a7c0c019..9d0ded648 100644 --- a/src/handlers/misc/READY.ts +++ b/src/handlers/misc/READY.ts @@ -50,7 +50,7 @@ function checkReady(payload: Ready, shard: DiscordenoShard) { if (shard.lastAvailable + 5000 < Date.now()) { eventHandlers.shardFailedToLoad?.(shard, shard.unavailableGuildIds); // Force execute the loaded function to prevent infinite loop - loaded(shard); + return loaded(shard); } // Not all guilds were loaded but 5 seconds haven't passed so check again From 74c6fbdae6afbef682934a53c58de45478725ed5 Mon Sep 17 00:00:00 2001 From: ITOH Date: Thu, 13 May 2021 11:57:35 +0200 Subject: [PATCH 06/14] remove: clone channel import --- src/helpers/channels/clone_channel.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/helpers/channels/clone_channel.ts b/src/helpers/channels/clone_channel.ts index bed3f4241..1a8c4cc5b 100644 --- a/src/helpers/channels/clone_channel.ts +++ b/src/helpers/channels/clone_channel.ts @@ -4,7 +4,6 @@ import type { CreateGuildChannel } from "../../types/guilds/create_guild_channel import { Errors } from "../../types/discordeno/errors.ts"; import { bigintToSnowflake } from "../../util/bigint.ts"; import { calculatePermissions } from "../../util/permissions.ts"; -import { createChannel } from "./create_channel.ts"; import { helpers } from "../mod.ts"; /** Create a copy of a channel */ From bbd9aa7f9865f9001ef825e90a6be1d04e4a379a Mon Sep 17 00:00:00 2001 From: ITOH Date: Thu, 13 May 2021 13:30:20 +0200 Subject: [PATCH 07/14] Update bot.ts --- src/bot.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bot.ts b/src/bot.ts index ae0dc92ff..7c0f54e59 100644 --- a/src/bot.ts +++ b/src/bot.ts @@ -94,8 +94,8 @@ export async function startBigBrainBot(options: BigBrainBotConfig) { // Initial API connection to get info about bots connection ws.botGatewayData = await getGatewayBot(); - ws.maxShards = ws.maxShards || - ws.botGatewayData.shards; + ws.maxShards = + options.lastShardId || ws.maxShards || ws.botGatewayData.shards; ws.lastShardId = options.lastShardId || ws.botGatewayData.shards; // Explicitly append gateway version and encoding ws.botGatewayData.url += `?v=${GATEWAY_VERSION}&encoding=json`; From b63be48b437b4f8834407ed06798f081032025b2 Mon Sep 17 00:00:00 2001 From: itohatweb Date: Thu, 13 May 2021 11:30:43 +0000 Subject: [PATCH 08/14] Commit from GitHub Actions (Lint) --- src/bot.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bot.ts b/src/bot.ts index 7c0f54e59..92d2182b9 100644 --- a/src/bot.ts +++ b/src/bot.ts @@ -94,8 +94,8 @@ export async function startBigBrainBot(options: BigBrainBotConfig) { // Initial API connection to get info about bots connection ws.botGatewayData = await getGatewayBot(); - ws.maxShards = - options.lastShardId || ws.maxShards || ws.botGatewayData.shards; + ws.maxShards = options.lastShardId || ws.maxShards || + ws.botGatewayData.shards; ws.lastShardId = options.lastShardId || ws.botGatewayData.shards; // Explicitly append gateway version and encoding ws.botGatewayData.url += `?v=${GATEWAY_VERSION}&encoding=json`; From 460af988076559d8ca5aac64d8154f3f7b4a643c Mon Sep 17 00:00:00 2001 From: ITOH Date: Thu, 13 May 2021 18:40:59 +0200 Subject: [PATCH 09/14] Update READY.ts --- src/handlers/misc/READY.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/handlers/misc/READY.ts b/src/handlers/misc/READY.ts index 9d0ded648..57bef2995 100644 --- a/src/handlers/misc/READY.ts +++ b/src/handlers/misc/READY.ts @@ -67,7 +67,7 @@ function loaded(shard: DiscordenoShard) { shard.ready = true; // If it is not the last shard we can't go full ready - if (shard.id !== ws.maxShards - 1) return; + if (shard.id !== ws.lastShardId) return; // Still some shards are loading so wait another 2 seconds for them if (ws.shards.some((shard) => !shard.ready)) { @@ -78,6 +78,7 @@ function loaded(shard: DiscordenoShard) { ); loaded(shard); }, 2000); + return; } From 64009aad7cf2eeab14a95063860c2ee8991dd602 Mon Sep 17 00:00:00 2001 From: ITOH Date: Thu, 13 May 2021 18:49:11 +0200 Subject: [PATCH 10/14] change: pass shardId instead of the shard in shard events --- src/handlers/misc/READY.ts | 2 +- src/types/discordeno/eventHandlers.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/handlers/misc/READY.ts b/src/handlers/misc/READY.ts index 57bef2995..60352640e 100644 --- a/src/handlers/misc/READY.ts +++ b/src/handlers/misc/READY.ts @@ -48,7 +48,7 @@ function checkReady(payload: Ready, shard: DiscordenoShard) { // If the last GUILD_CREATE has been received before 5 seconds if so most likely the remaining guilds are unavailable if (shard.lastAvailable + 5000 < Date.now()) { - eventHandlers.shardFailedToLoad?.(shard, shard.unavailableGuildIds); + eventHandlers.shardFailedToLoad?.(shard.id, shard.unavailableGuildIds); // Force execute the loaded function to prevent infinite loop return loaded(shard); } diff --git a/src/types/discordeno/eventHandlers.ts b/src/types/discordeno/eventHandlers.ts index 65d1475af..72137e1ce 100644 --- a/src/types/discordeno/eventHandlers.ts +++ b/src/types/discordeno/eventHandlers.ts @@ -208,10 +208,10 @@ export interface EventHandlers { member: DiscordenoMember, roleId: bigint, ) => unknown; - shardReady?: (shard: DiscordenoShard) => unknown; + shardReady?: (shardId: number) => unknown; /** Sent when a shard failed to load. */ shardFailedToLoad?: ( - shard: DiscordenoShard, + shardId: number, unavailableGuildIds: Set, ) => unknown; /** Sent when a thread is created */ From c16f211ce73244b79fe080cf1acf5056884a3e6a Mon Sep 17 00:00:00 2001 From: ITOH Date: Thu, 13 May 2021 18:49:38 +0200 Subject: [PATCH 11/14] fix: shardReady should always be emitted when ready event is received --- src/handlers/misc/READY.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/handlers/misc/READY.ts b/src/handlers/misc/READY.ts index 60352640e..1f2c2ab74 100644 --- a/src/handlers/misc/READY.ts +++ b/src/handlers/misc/READY.ts @@ -9,6 +9,9 @@ export function handleReady( data: DiscordGatewayPayload, shardId: number, ) { + // Triggered on each shard + eventHandlers.shardReady?.(shardId); + // The bot has already started, the last shard is resumed, however. if (cache.isReady) return; @@ -19,9 +22,6 @@ export function handleReady( setBotId(payload.user.id); setApplicationId(payload.application.id); - // Triggered on each shard - eventHandlers.shardReady?.(shard); - // Set ready to false just to go sure shard.ready = false; // All guilds are unavailable at first From 64e433f461325140fd9e44797630c3c82410dec7 Mon Sep 17 00:00:00 2001 From: ITOH Date: Thu, 13 May 2021 19:02:16 +0200 Subject: [PATCH 12/14] fix: set last shard id --- src/bot.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/bot.ts b/src/bot.ts index 92d2182b9..4fe7562ed 100644 --- a/src/bot.ts +++ b/src/bot.ts @@ -31,6 +31,8 @@ export async function startBot(config: BotConfig) { // Initial API connection to get info about bots connection ws.botGatewayData = await getGatewayBot(); ws.maxShards = ws.maxShards || ws.botGatewayData.shards; + ws.lastShardId = + ws.lastShardId === 1 ? ws.botGatewayData.shards : ws.lastShardId; // Explicitly append gateway version and encoding ws.botGatewayData.url += `?v=${GATEWAY_VERSION}&encoding=json`; From bd9a34a0085e98f840c28288a822b71379074c6f Mon Sep 17 00:00:00 2001 From: itohatweb Date: Thu, 13 May 2021 17:02:41 +0000 Subject: [PATCH 13/14] Commit from GitHub Actions (Lint) --- src/bot.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/bot.ts b/src/bot.ts index 4fe7562ed..893504bae 100644 --- a/src/bot.ts +++ b/src/bot.ts @@ -31,8 +31,9 @@ export async function startBot(config: BotConfig) { // Initial API connection to get info about bots connection ws.botGatewayData = await getGatewayBot(); ws.maxShards = ws.maxShards || ws.botGatewayData.shards; - ws.lastShardId = - ws.lastShardId === 1 ? ws.botGatewayData.shards : ws.lastShardId; + ws.lastShardId = ws.lastShardId === 1 + ? ws.botGatewayData.shards + : ws.lastShardId; // Explicitly append gateway version and encoding ws.botGatewayData.url += `?v=${GATEWAY_VERSION}&encoding=json`; From 6c44123b7e8c8b54b9a242e4cc1a17665a0e567c Mon Sep 17 00:00:00 2001 From: ITOH Date: Thu, 13 May 2021 19:07:24 +0200 Subject: [PATCH 14/14] Update bot.ts --- src/bot.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bot.ts b/src/bot.ts index 4fe7562ed..ce17a587c 100644 --- a/src/bot.ts +++ b/src/bot.ts @@ -32,7 +32,7 @@ export async function startBot(config: BotConfig) { ws.botGatewayData = await getGatewayBot(); ws.maxShards = ws.maxShards || ws.botGatewayData.shards; ws.lastShardId = - ws.lastShardId === 1 ? ws.botGatewayData.shards : ws.lastShardId; + ws.lastShardId === 1 ? ws.botGatewayData.shards - 1 : ws.lastShardId; // Explicitly append gateway version and encoding ws.botGatewayData.url += `?v=${GATEWAY_VERSION}&encoding=json`;