From 4cf3e61e96a07489c7af74e115079409aebc13cb Mon Sep 17 00:00:00 2001 From: Skillz4Killz <23035000+Skillz4Killz@users.noreply.github.com> Date: Fri, 26 Aug 2022 00:29:25 +0000 Subject: [PATCH] feat: big bot rest analytics --- rest/restManager.ts | 33 +++++++++++++++++----- rest/sendRequest.ts | 4 +-- template/bigbot/package-lock.json | 11 ++++++++ template/bigbot/package.json | 1 + template/bigbot/src/analytics.ts | 5 ++++ template/bigbot/src/configs.ts | 7 +++++ template/bigbot/src/rest/index.ts | 47 +++++++++++++++++++++++++++++++ 7 files changed, 99 insertions(+), 9 deletions(-) create mode 100644 template/bigbot/src/analytics.ts diff --git a/rest/restManager.ts b/rest/restManager.ts index efe4c8d12..65d374cd2 100644 --- a/rest/restManager.ts +++ b/rest/restManager.ts @@ -1,19 +1,18 @@ +import { API_VERSION, baseEndpoints } from "../util/constants.ts"; +import { removeTokenPrefix } from "../util/token.ts"; import { checkRateLimits } from "./checkRateLimits.ts"; import { cleanupQueues } from "./cleanupQueues.ts"; +import { convertRestError } from "./convertRestError.ts"; import { createRequestBody } from "./createRequestBody.ts"; import { processGlobalQueue } from "./processGlobalQueue.ts"; import { processQueue } from "./processQueue.ts"; import { processRateLimitedPaths } from "./processRateLimitedPaths.ts"; import { processRequest } from "./processRequest.ts"; import { processRequestHeaders } from "./processRequestHeaders.ts"; -import { convertRestError } from "./convertRestError.ts"; import { RestPayload, RestRateLimitedPath, RestRequest } from "./rest.ts"; import { runMethod } from "./runMethod.ts"; +import { RestSendRequestOptions, sendRequest } from "./sendRequest.ts"; import { simplifyUrl } from "./simplifyUrl.ts"; -import { baseEndpoints } from "../util/constants.ts"; -import { API_VERSION } from "../util/constants.ts"; -import { removeTokenPrefix } from "../util/token.ts"; -import { sendRequest } from "./sendRequest.ts"; export function createRestManager(options: CreateRestManagerOptions) { const version = options.version || API_VERSION; @@ -66,8 +65,10 @@ export function createRestManager(options: CreateRestManagerOptions) { checkRateLimits: options.checkRateLimits || checkRateLimits, cleanupQueues: options.cleanupQueues || cleanupQueues, processQueue: options.processQueue || processQueue, - processRateLimitedPaths: options.processRateLimitedPaths || processRateLimitedPaths, - processRequestHeaders: options.processRequestHeaders || processRequestHeaders, + processRateLimitedPaths: options.processRateLimitedPaths || + processRateLimitedPaths, + processRequestHeaders: options.processRequestHeaders || + processRequestHeaders, processRequest: options.processRequest || processRequest, createRequestBody: options.createRequestBody || createRequestBody, runMethod: options.runMethod || runMethod, @@ -75,6 +76,22 @@ export function createRestManager(options: CreateRestManagerOptions) { processGlobalQueue: options.processGlobalQueue || processGlobalQueue, convertRestError: options.convertRestError || convertRestError, sendRequest: options.sendRequest || sendRequest, + + fetching: options.fetching || function (opts: RestSendRequestOptions) { + options.debug?.( + `[REST - fetching] URL: ${opts.url} | ${JSON.stringify(opts)}`, + ); + }, + fetched: options.fetched || function ( + opts: RestSendRequestOptions, + response: Response, + ) { + options.debug?.( + `[REST - fetched] URL: ${opts.url} | Status: ${response.status} ${ + JSON.stringify(opts) + }`, + ); + }, }; } @@ -97,6 +114,8 @@ export interface CreateRestManagerOptions { processGlobalQueue?: typeof processGlobalQueue; convertRestError?: typeof convertRestError; sendRequest?: typeof sendRequest; + fetching?: (options: RestSendRequestOptions) => void; + fetched?: (options: RestSendRequestOptions, response: Response) => void; } export type RestManager = ReturnType; diff --git a/rest/sendRequest.ts b/rest/sendRequest.ts index ef140821b..b8b248e60 100644 --- a/rest/sendRequest.ts +++ b/rest/sendRequest.ts @@ -19,7 +19,7 @@ export interface RestSendRequestOptions { export async function sendRequest(rest: RestManager, options: RestSendRequestOptions): Promise { try { // CUSTOM HANDLER FOR USER TO LOG OR WHATEVER WHENEVER A FETCH IS MADE - rest.debug(`[REST - fetching] URL: ${options.url} | ${JSON.stringify(options)}`); + rest.fetching(options); const response = await fetch( options.url.startsWith(BASE_URL) ? options.url : `${BASE_URL}/v${rest.version}/${options.url}`, @@ -29,7 +29,7 @@ export async function sendRequest(rest: RestManager, options: RestSendRequest body: options.payload?.body, }, ); - rest.debug(`[REST - fetched] URL: ${options.url} | ${JSON.stringify(options)}`); + rest.fetched(options, response); const bucketIdFromHeaders = rest.processRequestHeaders( rest, diff --git a/template/bigbot/package-lock.json b/template/bigbot/package-lock.json index 8e3fcfb32..8fd3dabe1 100644 --- a/template/bigbot/package-lock.json +++ b/template/bigbot/package-lock.json @@ -8,6 +8,7 @@ "name": "dd-big-bot", "version": "1.0.0", "dependencies": { + "@influxdata/influxdb-client": "^1.29.0", "@prisma/client": "^3.12.0", "discordeno": "^13.0.0-rc46", "express": "^4.18.1", @@ -42,6 +43,11 @@ "resolved": "https://registry.npmjs.org/@deno/shim-timers/-/shim-timers-0.1.0.tgz", "integrity": "sha512-XFRnB5Rtbkd5RiYHwhugNK9gvDgYXmFTUOT5dmhWCKG7WnOWZggbJMnH1NcyYS3QgHvmaTOaHCyNFNSv57j3Dg==" }, + "node_modules/@influxdata/influxdb-client": { + "version": "1.29.0", + "resolved": "https://registry.npmjs.org/@influxdata/influxdb-client/-/influxdb-client-1.29.0.tgz", + "integrity": "sha512-9cup31a0S5Px1q5Bj9oyI8d84IWfv6nhldNuL8UNsiFPDD5SPw1QesPL4Oo+rR3Lbzbilt269NJOK5JOkmPyYA==" + }, "node_modules/@prisma/client": { "version": "3.15.2", "resolved": "https://registry.npmjs.org/@prisma/client/-/client-3.15.2.tgz", @@ -1247,6 +1253,11 @@ "resolved": "https://registry.npmjs.org/@deno/shim-timers/-/shim-timers-0.1.0.tgz", "integrity": "sha512-XFRnB5Rtbkd5RiYHwhugNK9gvDgYXmFTUOT5dmhWCKG7WnOWZggbJMnH1NcyYS3QgHvmaTOaHCyNFNSv57j3Dg==" }, + "@influxdata/influxdb-client": { + "version": "1.29.0", + "resolved": "https://registry.npmjs.org/@influxdata/influxdb-client/-/influxdb-client-1.29.0.tgz", + "integrity": "sha512-9cup31a0S5Px1q5Bj9oyI8d84IWfv6nhldNuL8UNsiFPDD5SPw1QesPL4Oo+rR3Lbzbilt269NJOK5JOkmPyYA==" + }, "@prisma/client": { "version": "3.15.2", "resolved": "https://registry.npmjs.org/@prisma/client/-/client-3.15.2.tgz", diff --git a/template/bigbot/package.json b/template/bigbot/package.json index 522f4ec75..0d700d0ca 100644 --- a/template/bigbot/package.json +++ b/template/bigbot/package.json @@ -9,6 +9,7 @@ "fmt": "prettier -w ./src" }, "dependencies": { + "@influxdata/influxdb-client": "^1.29.0", "@prisma/client": "^3.12.0", "discordeno": "^13.0.0-rc46", "express": "^4.18.1", diff --git a/template/bigbot/src/analytics.ts b/template/bigbot/src/analytics.ts new file mode 100644 index 000000000..b0aae970d --- /dev/null +++ b/template/bigbot/src/analytics.ts @@ -0,0 +1,5 @@ +import { InfluxDB } from "@influxdata/influxdb-client"; +import { INFLUX_BUCKET, INFLUX_ORG, INFLUX_TOKEN, INFLUX_URL } from "./configs"; + +export const influxDB = new InfluxDB({ url: INFLUX_URL, token: INFLUX_TOKEN }); +export const Influx = influxDB.getWriteApi(INFLUX_ORG, INFLUX_BUCKET); diff --git a/template/bigbot/src/configs.ts b/template/bigbot/src/configs.ts index c8fa69898..03d0e753c 100644 --- a/template/bigbot/src/configs.ts +++ b/template/bigbot/src/configs.ts @@ -13,3 +13,10 @@ export const REST_PORT = 8000; /** The url where requests will be sent to from the bot. */ // SETUP-DD-TEMP: Provide the url where you will host your REST proxy. If it is on the same server as others, you can use localhost but if it is on a separate server you should change this entirely. export const REST_URL = `http://localhost:${REST_PORT}`; + +/** These INFLUX configs are only if you wish to enable analytics. */ +// SETUP-DD-TEMP: This is optional. If you want to build analytics, add influxdb here. +export const INFLUX_BUCKET = ""; +export const INFLUX_ORG = ""; +export const INFLUX_TOKEN = ""; +export const INFLUX_URL = ""; \ No newline at end of file diff --git a/template/bigbot/src/rest/index.ts b/template/bigbot/src/rest/index.ts index a453724ee..d0e770196 100644 --- a/template/bigbot/src/rest/index.ts +++ b/template/bigbot/src/rest/index.ts @@ -1,7 +1,11 @@ +import { Point } from "@influxdata/influxdb-client"; import { BASE_URL, createRestManager } from "discordeno"; import express, { Request, Response } from "express"; + +import { Influx } from "../analytics"; import { DISCORD_TOKEN, + INFLUX_TOKEN, REST_AUTHORIZATION, REST_PORT, REST_URL, @@ -14,6 +18,49 @@ const rest = createRestManager({ debug: console.log, }); +// If influxdb data is provided, enable analytics in this proxy. +if (INFLUX_TOKEN) { + rest.fetching = function (options) { + Influx.writePoint( + new Point("restEvents") + // MARK THE TIME WHEN EVENT ARRIVED + .timestamp(new Date()) + // SET THE GUILD ID + .stringField("type", "REQUEST_FETCHING") + .tag("method", options.method) + .tag("url", options.url) + .tag("bucket", options.bucketId), + ); + }; + + rest.fetched = function (options, response) { + Influx.writePoint( + new Point("restEvents") + // MARK THE TIME WHEN EVENT ARRIVED + .timestamp(new Date()) + // SET THE GUILD ID + .stringField("type", "REQUEST_FETCHED") + .tag("method", options.method) + .tag("url", options.url) + .tag("bucket", options.bucketId) + .intField("status", response.status) + .tag("statusText", response.statusText), + ); + }; + + setInterval(() => { + console.log(`[Influx - REST] Saving events...`); + Influx.flush() + .then(() => { + console.log(`[Influx - REST] Saved events!`); + }) + .catch((error) => { + console.log(`[Influx - REST] Error saving events!`, error); + }); + // Every 30seconds + }, 30000); +} + const app = express(); app.use(