From 0e30afdc963ef7fea4d8a205313168d40ff1335e Mon Sep 17 00:00:00 2001 From: Skillz Date: Fri, 15 May 2020 09:18:33 -0400 Subject: [PATCH] fix resume and fetch members handling --- mod.ts | 17 ++++- module/shard.ts | 154 ++++++++++++++++++++++++-------------- module/shardingManager.ts | 71 +++++++++++++++--- structures/channel.ts | 3 +- structures/guild.ts | 25 ++++++- types/errors.ts | 3 +- types/guild.ts | 20 ++++- 7 files changed, 216 insertions(+), 77 deletions(-) diff --git a/mod.ts b/mod.ts index 910758200..6e28e9fd9 100644 --- a/mod.ts +++ b/mod.ts @@ -1,14 +1,25 @@ import Client from "./module/client.ts"; import { configs } from "./configs.ts"; import { Intents } from "./types/options.ts"; -import { logYellow, logGreen } from "./utils/logger.ts"; +import { logYellow } from "./utils/logger.ts"; +import { cache } from "./utils/cache.ts"; Client({ token: configs.token, botID: "675412054529540107", - intents: [Intents.GUILDS, Intents.GUILD_MESSAGES], + intents: [Intents.GUILDS, Intents.GUILD_MESSAGES, Intents.GUILD_MEMBERS], eventHandlers: { ready: () => logYellow("Bot ready emitted"), - raw: (data) => logGreen("[RAW] => " + JSON.stringify(data)), + // raw: (data) => logGreen("[RAW] => " + JSON.stringify(data)), + messageCreate: async (message) => { + if (message.author.id === "130136895395987456") { + if (message.content.startsWith("!test")) { + if (!message.guild_id) return; + const guild = cache.guilds.get(message.guild_id); + if (!guild) return logYellow("no guild"); + + } + } + }, }, }); diff --git a/module/shard.ts b/module/shard.ts index 5cffb83d1..6793443f7 100644 --- a/module/shard.ts +++ b/module/shard.ts @@ -11,8 +11,10 @@ import { } from "../types/discord.ts"; import { logRed } from "../utils/logger.ts"; import { sendConstantHeartbeats, previousSequenceNumber } from "./gateway.ts"; +import { FetchMembersOptions } from "../types/guild.ts"; + +let shardSocket: WebSocket; -export const USELESS_ARG_TO_MAKE_DENO_CACHE_WORK = undefined; /** The session id is needed for RESUME functionality when discord disconnects randomly. */ let sessionID = ""; @@ -40,7 +42,7 @@ export const createShard = async ( botGatewayData: DiscordBotGatewayData, identifyPayload: object, ) => { - const shardSocket = await connectWebSocket(botGatewayData.url); + shardSocket = await connectWebSocket(botGatewayData.url); let resumeInterval = 0; // Intial identify with the gateway @@ -48,64 +50,100 @@ export const createShard = async ( JSON.stringify({ op: GatewayOpcode.Identify, d: identifyPayload }), ); - for await (const message of shardSocket) { - if (typeof message === "string") { - const data = JSON.parse(message); + try { + for await (const message of shardSocket) { + if (typeof message === "string") { + const data = JSON.parse(message); - switch (data.op) { - case GatewayOpcode.Hello: - sendConstantHeartbeats( - shardSocket, - (data.d as DiscordHeartbeatPayload).heartbeat_interval, - ); - break; - case GatewayOpcode.Reconnect: - case GatewayOpcode.InvalidSession: - // Reconnect to the gateway https://discordapp.com/developers/docs/topics/gateway#reconnect - resumeInterval = await resumeConnection( - identifyPayload, - botGatewayData, - shardSocket, - ); - break; - case GatewayOpcode.Resume: - clearInterval(resumeInterval); - break; - default: - // Important for RESUME - if (data.t === "READY") { - sessionID = (data.d as ReadyPayload).session_id; - } - // @ts-ignore - postMessage( - { - type: "HANDLE_DISCORD_PAYLOAD", - payload: message, - resumeInterval, - }, - ); - break; + switch (data.op) { + case GatewayOpcode.Hello: + sendConstantHeartbeats( + shardSocket, + (data.d as DiscordHeartbeatPayload).heartbeat_interval, + ); + break; + case GatewayOpcode.Reconnect: + case GatewayOpcode.InvalidSession: + // Reconnect to the gateway https://discordapp.com/developers/docs/topics/gateway#reconnect + resumeInterval = await resumeConnection( + identifyPayload, + botGatewayData, + shardSocket, + ); + break; + case GatewayOpcode.Resume: + clearInterval(resumeInterval); + break; + default: + // Important for RESUME + if (data.t === "READY") { + sessionID = (data.d as ReadyPayload).session_id; + } + // @ts-ignore + postMessage( + { + type: "HANDLE_DISCORD_PAYLOAD", + payload: message, + resumeInterval, + }, + ); + break; + } + } else if (isWebSocketCloseEvent(message)) { + logRed(`Close :( ${JSON.stringify(message)}`); + resumeInterval = await resumeConnection( + identifyPayload, + botGatewayData, + shardSocket, + ); } - } else if (isWebSocketCloseEvent(message)) { - logRed(`Close :( ${JSON.stringify(message)}`); - resumeInterval = await resumeConnection( - identifyPayload, - botGatewayData, - shardSocket, + } + } catch (error) { + logRed(error); + } +}; + +export function requestGuildMembers( + guildID: string, + nonce: string, + options?: FetchMembersOptions, +) { + shardSocket.send(JSON.stringify({ + op: GatewayOpcode.RequestGuildMembers, + d: { + guild_id: guildID, + query: options?.query || "", + limit: options?.query || 0, + presences: options?.presences || false, + user_ids: options?.userIDs, + nonce, + }, + })); +} + +// TODO: Remove ts-ignore once https://github.com/denoland/deno/issues/5262 fixed +// @ts-ignore +if (typeof self.postMessage === "function") { + // @ts-ignore + postMessage({ type: "REQUEST_CLIENT_OPTIONS" }); +} +// @ts-ignore +if (typeof self.onmessage === "function") { + // @ts-ignore + onmessage = (message) => { + if (message.data.type === "CREATE_SHARD") { + createShard( + message.data.botGatewayData, + message.data.identifyPayload, ); } - } -}; -// TODO: Remove ts-ignore when deno fixes -// @ts-ignore -postMessage({ type: "REQUEST_CLIENT_OPTIONS" }); -// @ts-ignore -onmessage = (message) => { - if (message.data.type === "CREATE_SHARD") { - createShard( - message.data.botGatewayData, - message.data.identifyPayload, - ); - } -}; + if (message.data.type === "FETCH_MEMBERS") { + requestGuildMembers( + message.data.guildID, + message.data.nonce, + message.data.options, + ); + } + }; +} diff --git a/module/shardingManager.ts b/module/shardingManager.ts index a79b6b56a..03023c7d4 100644 --- a/module/shardingManager.ts +++ b/module/shardingManager.ts @@ -34,6 +34,7 @@ import { GuildMemberChunkPayload, GuildRolePayload, UserPayload, + FetchMembersOptions, } from "../types/guild.ts"; import { handleInternalGuildCreate, @@ -57,12 +58,27 @@ import { createMessage } from "../structures/message.ts"; let shardCounter = 0; -function createShardWorker() { +export interface FetchAllMembersRequest { + resolve: Function; + requestedMax: number; + receivedAmount: number; +} + +const fetchAllMembersProcessingRequests = new Map< + string, + FetchAllMembersRequest +>(); +const shards: Worker[] = []; + +export function createShardWorker(shardID?: number) { const path = new URL("./shard.ts", import.meta.url).toString(); const shard = new Worker(path, { type: "module", deno: true }); shard.onmessage = (message) => { if (message.data.type === "REQUEST_CLIENT_OPTIONS") { - identifyPayload.shard = [shardCounter++, botGatewayData.shards]; + identifyPayload.shard = [ + shardID || shardCounter++, + botGatewayData.shards, + ]; shard.postMessage( { type: "CREATE_SHARD", @@ -74,6 +90,7 @@ function createShardWorker() { handleDiscordPayload(JSON.parse(message.data.payload)); } }; + shards.push(shard); } export const spawnShards = async ( @@ -269,6 +286,19 @@ function handleDiscordPayload(data: DiscordPayload) { ); cache.users.set(member.user.id, createUser(member.user)); }); + + // Check if its necessary to resolve the fetchmembers promise for this chunk or if more chunks will be coming + if ( + options.nonce + ) { + const request = fetchAllMembersProcessingRequests.get(options.nonce); + if (!request) return; + + if (options.chunk_index + 1 === options.chunk_count) { + fetchAllMembersProcessingRequests.delete(options.nonce); + request.resolve(); + } + } } if ( @@ -338,13 +368,11 @@ function handleDiscordPayload(data: DiscordPayload) { const channel = cache.channels.get(options.channel_id); if (!channel) return; - deletedMessages.forEach((_id) => { - // const message = channel.messages().get(id) - // if (message) { - // // TODO: update the messages cache - // } - - // return eventHandlers.message_delete?.(message || { id, channel }) + deletedMessages.forEach((id) => { + const message = cache.messages.get(id); + if (!message) return; + eventHandlers.message_delete?.(message || { id, channel }); + cache.messages.delete(id); }); } @@ -501,3 +529,28 @@ function handleDiscordPayload(data: DiscordPayload) { return; } } + +export function requestAllMembers( + guildID: string, + resolve: Function, + memberCount: number, + options?: FetchMembersOptions, +) { + const payload = { + resolve, + requestedMax: options?.query + ? 100 + : options?.userIDs?.length || options?.limit || memberCount, + receivedAmount: 0, + }; + + const nonce = Math.random().toString(); + fetchAllMembersProcessingRequests.set(nonce, payload); + + return shards[0].postMessage({ + type: "FETCH_MEMBERS", + guildID, + nonce, + options, + }); +} diff --git a/structures/channel.ts b/structures/channel.ts index dbb7a04d3..4db14c4ff 100644 --- a/structures/channel.ts +++ b/structures/channel.ts @@ -19,6 +19,7 @@ import { import { Permissions } from "../types/permission.ts"; import { Errors } from "../types/errors.ts"; import { RequestManager } from "../module/requestManager.ts"; +import { logYellow } from "../utils/logger.ts"; export function createChannel(data: ChannelCreatePayload) { const channel = { @@ -146,7 +147,7 @@ export function createChannel(data: ChannelCreatePayload) { if (ids.length < 2) throw new Error(Errors.DELETE_MESSAGES_MIN); if (ids.length > 100) { - console.warn( + logYellow( `This endpoint only accepts a maximum of 100 messages. Deleting the first 100 message ids provided.`, ); } diff --git a/structures/guild.ts b/structures/guild.ts index a9a2d000d..cc13ecbab 100644 --- a/structures/guild.ts +++ b/structures/guild.ts @@ -1,4 +1,4 @@ -import { botID } from "../module/client.ts"; +import { botID, identifyPayload } from "../module/client.ts"; import { endpoints } from "../constants/discord.ts"; import { formatImageURL } from "../utils/cdn.ts"; import { @@ -12,6 +12,7 @@ import { CreateEmojisOptions, EditEmojisOptions, CreateRoleOptions, + FetchMembersOptions, } from "../types/guild.ts"; import { createRole } from "./role.ts"; import { createMember } from "./member.ts"; @@ -27,6 +28,8 @@ import { botHasPermission } from "../utils/permissions.ts"; import { Errors } from "../types/errors.ts"; import { RequestManager } from "../module/requestManager.ts"; import { RoleData } from "../types/role.ts"; +import { Intents } from "../types/options.ts"; +import { requestAllMembers } from "../module/shardingManager.ts"; export const createGuild = (data: CreateGuildPayload) => { const guild = { @@ -47,6 +50,8 @@ export const createGuild = (data: CreateGuildPayload) => { channels: new Map(data.channels.map((c) => [c.id, createChannel(c)])), /** The presences of all the users in the guild. */ presences: new Map(data.presences.map((p) => [p.user.id, p])), + /** The total number of members in this guild. This value is updated as members leave and join the server. However, if you do not have the intent enabled to be able to listen to these events, then this will not be accurate. */ + memberCount: data.member_count || 0, /** Gets an array of all the channels ids that are the children of this category. */ categoryChildrenIDs: (id: string) => @@ -246,9 +251,21 @@ export const createGuild = (data: CreateGuildPayload) => { } return RequestManager.post(endpoints.GUILD_PRUNE(data.id), { days }); }, - // TODO: REQUEST THIS OVER WEBSOCKET WITH GET_GUILD_MEMBERS ENDPOINT - // fetch_all_members: () => { - // }, + fetchMembers: (options?: FetchMembersOptions) => { + if (!(identifyPayload.intents & Intents.GUILD_MEMBERS)) throw new Error(Errors.MISSING_INTENT_GUILD_MEMBERS) + + return new Promise((resolve) => { + requestAllMembers(data.id, resolve, guild.memberCount, options) + }) + // return new Promise((res) => this.requestMembersPromise[guildID] = { + // res: res, + // received: 0, + // timeout: setTimeout(() => { + // res(this.requestMembersPromise[guildID].received); + // delete this.requestMembersPromise[guildID]; + // }, timeout || this.client.options.requestTimeout) + // }); + }, /** Returns the audit logs for the guild. Requires VIEW AUDIT LOGS permission */ getAuditLogs: (options: GetAuditLogsOptions) => { if (!botHasPermission(data.id, botID, [Permissions.VIEW_AUDIT_LOG])) { diff --git a/types/errors.ts b/types/errors.ts index b2f6f6f3f..2aec5a375 100644 --- a/types/errors.ts +++ b/types/errors.ts @@ -21,5 +21,6 @@ export enum Errors { MESSAGE_MAX_LENGTH = "MESSAGE_MAX_LENGTH", NICKNAMES_MAX_LENGTH = "NICKNAMES_MAX_LENGTH", PRUNE_MIN_DAYS = "PRUNE_MIN_DAYS", - RATE_LIMIT_RETRY_MAXED = "RATE_LIMIT_RETRY_MAXED" + RATE_LIMIT_RETRY_MAXED = "RATE_LIMIT_RETRY_MAXED", + MISSING_INTENT_GUILD_MEMBERS = "MISSING_INTENT_GUILD_MEMBERS" } diff --git a/types/guild.ts b/types/guild.ts index f54e4020f..0acc33dee 100644 --- a/types/guild.ts +++ b/types/guild.ts @@ -19,10 +19,16 @@ export interface GuildMemberChunkPayload { guild_id: string /** The set of guild members */ members: MemberCreatePayload[] + /** The chunk index in the expected chunks for this response */ + chunk_index: number + /** The total number of expected chunks for this response */ + chunk_count: number /** if passing an invalid id, it will be found here */ not_found?: string[] /** if passing true, presences of the members will be here */ presences?: Presence[] + /** The nonce to help identify */ + nonce?: string } export interface GuildMemberUpdatePayload { @@ -101,7 +107,7 @@ export interface CreateGuildPayload { /** Whether this guild is unavailable */ unavailable: boolean /** Total number of members in this guild */ - memberCount: number + member_count?: number voice_states: Voice_State[] /** Users in the guild */ members: MemberCreatePayload[] @@ -527,3 +533,15 @@ export interface Presence { premium_since?: string | null nick?: string | null } + +export interface FetchMembersOptions { + /** Used to specify if you want the presences of the matched members. Default = false. */ + presences?: boolean + /** Only returns members whose username or nickname starts with this string. DO NOT INCLUDE discriminators. If a string is provided, the max amount of members that can be fetched is 100. Default = return all members. */ + query?: string + /** Used to specify which users to fetch specifically. */ + userIDs?: string[] + /** Maximum number of members to return that match the query. Default = 0 which will return all members. */ + limit?: number + +}