feat: big bot rest analytics

rest/restManager.ts

@@ -1,19 +1,18 @@
+import { API_VERSION, baseEndpoints } from "../util/constants.ts";
+import { removeTokenPrefix } from "../util/token.ts";
 import { checkRateLimits } from "./checkRateLimits.ts";
 import { cleanupQueues } from "./cleanupQueues.ts";
+import { convertRestError } from "./convertRestError.ts";
 import { createRequestBody } from "./createRequestBody.ts";
 import { processGlobalQueue } from "./processGlobalQueue.ts";
 import { processQueue } from "./processQueue.ts";
 import { processRateLimitedPaths } from "./processRateLimitedPaths.ts";
 import { processRequest } from "./processRequest.ts";
 import { processRequestHeaders } from "./processRequestHeaders.ts";
-import { convertRestError } from "./convertRestError.ts";
 import { RestPayload, RestRateLimitedPath, RestRequest } from "./rest.ts";
 import { runMethod } from "./runMethod.ts";
+import { RestSendRequestOptions, sendRequest } from "./sendRequest.ts";
 import { simplifyUrl } from "./simplifyUrl.ts";
-import { baseEndpoints } from "../util/constants.ts";
-import { API_VERSION } from "../util/constants.ts";
-import { removeTokenPrefix } from "../util/token.ts";
-import { sendRequest } from "./sendRequest.ts";
 
 export function createRestManager(options: CreateRestManagerOptions) {
   const version = options.version || API_VERSION;
@@ -66,8 +65,10 @@ export function createRestManager(options: CreateRestManagerOptions) {
     checkRateLimits: options.checkRateLimits || checkRateLimits,
     cleanupQueues: options.cleanupQueues || cleanupQueues,
     processQueue: options.processQueue || processQueue,
-    processRateLimitedPaths: options.processRateLimitedPaths || processRateLimitedPaths,
-    processRequestHeaders: options.processRequestHeaders || processRequestHeaders,
+    processRateLimitedPaths: options.processRateLimitedPaths ||
+      processRateLimitedPaths,
+    processRequestHeaders: options.processRequestHeaders ||
+      processRequestHeaders,
     processRequest: options.processRequest || processRequest,
     createRequestBody: options.createRequestBody || createRequestBody,
     runMethod: options.runMethod || runMethod,
@@ -75,6 +76,22 @@ export function createRestManager(options: CreateRestManagerOptions) {
     processGlobalQueue: options.processGlobalQueue || processGlobalQueue,
     convertRestError: options.convertRestError || convertRestError,
     sendRequest: options.sendRequest || sendRequest,
+
+    fetching: options.fetching || function (opts: RestSendRequestOptions) {
+      options.debug?.(
+        `[REST - fetching] URL: ${opts.url} | ${JSON.stringify(opts)}`,
+      );
+    },
+    fetched: options.fetched || function (
+      opts: RestSendRequestOptions,
+      response: Response,
+    ) {
+      options.debug?.(
+        `[REST - fetched] URL: ${opts.url} | Status: ${response.status} ${
+          JSON.stringify(opts)
+        }`,
+      );
+    },
   };
 }
 
@@ -97,6 +114,8 @@ export interface CreateRestManagerOptions {
   processGlobalQueue?: typeof processGlobalQueue;
   convertRestError?: typeof convertRestError;
   sendRequest?: typeof sendRequest;
+  fetching?: (options: RestSendRequestOptions) => void;
+  fetched?: (options: RestSendRequestOptions, response: Response) => void;
 }
 
 export type RestManager = ReturnType<typeof createRestManager>;

rest/sendRequest.ts

@@ -19,7 +19,7 @@ export interface RestSendRequestOptions {
 export async function sendRequest<T>(rest: RestManager, options: RestSendRequestOptions): Promise<T> {
   try {
     // CUSTOM HANDLER FOR USER TO LOG OR WHATEVER WHENEVER A FETCH IS MADE
-    rest.debug(`[REST - fetching] URL: ${options.url} | ${JSON.stringify(options)}`);
+    rest.fetching(options);
 
     const response = await fetch(
       options.url.startsWith(BASE_URL) ? options.url : `${BASE_URL}/v${rest.version}/${options.url}`,
@@ -29,7 +29,7 @@ export async function sendRequest<T>(rest: RestManager, options: RestSendRequest
         body: options.payload?.body,
       },
     );
-    rest.debug(`[REST - fetched] URL: ${options.url} | ${JSON.stringify(options)}`);
+    rest.fetched(options, response);
 
     const bucketIdFromHeaders = rest.processRequestHeaders(
       rest,

template/bigbot/package-lock.json

@@ -8,6 +8,7 @@
       "name": "dd-big-bot",
       "version": "1.0.0",
       "dependencies": {
+        "@influxdata/influxdb-client": "^1.29.0",
         "@prisma/client": "^3.12.0",
         "discordeno": "^13.0.0-rc46",
         "express": "^4.18.1",
@@ -42,6 +43,11 @@
       "resolved": "https://registry.npmjs.org/@deno/shim-timers/-/shim-timers-0.1.0.tgz",
       "integrity": "sha512-XFRnB5Rtbkd5RiYHwhugNK9gvDgYXmFTUOT5dmhWCKG7WnOWZggbJMnH1NcyYS3QgHvmaTOaHCyNFNSv57j3Dg=="
     },
+    "node_modules/@influxdata/influxdb-client": {
+      "version": "1.29.0",
+      "resolved": "https://registry.npmjs.org/@influxdata/influxdb-client/-/influxdb-client-1.29.0.tgz",
+      "integrity": "sha512-9cup31a0S5Px1q5Bj9oyI8d84IWfv6nhldNuL8UNsiFPDD5SPw1QesPL4Oo+rR3Lbzbilt269NJOK5JOkmPyYA=="
+    },
     "node_modules/@prisma/client": {
       "version": "3.15.2",
       "resolved": "https://registry.npmjs.org/@prisma/client/-/client-3.15.2.tgz",
@@ -1247,6 +1253,11 @@
       "resolved": "https://registry.npmjs.org/@deno/shim-timers/-/shim-timers-0.1.0.tgz",
       "integrity": "sha512-XFRnB5Rtbkd5RiYHwhugNK9gvDgYXmFTUOT5dmhWCKG7WnOWZggbJMnH1NcyYS3QgHvmaTOaHCyNFNSv57j3Dg=="
     },
+    "@influxdata/influxdb-client": {
+      "version": "1.29.0",
+      "resolved": "https://registry.npmjs.org/@influxdata/influxdb-client/-/influxdb-client-1.29.0.tgz",
+      "integrity": "sha512-9cup31a0S5Px1q5Bj9oyI8d84IWfv6nhldNuL8UNsiFPDD5SPw1QesPL4Oo+rR3Lbzbilt269NJOK5JOkmPyYA=="
+    },
     "@prisma/client": {
       "version": "3.15.2",
       "resolved": "https://registry.npmjs.org/@prisma/client/-/client-3.15.2.tgz",

template/bigbot/package.json

@@ -9,6 +9,7 @@
     "fmt": "prettier -w ./src"
   },
   "dependencies": {
+    "@influxdata/influxdb-client": "^1.29.0",
     "@prisma/client": "^3.12.0",
     "discordeno": "^13.0.0-rc46",
     "express": "^4.18.1",

template/bigbot/src/analytics.ts

@@ -0,0 +1,5 @@
+import { InfluxDB } from "@influxdata/influxdb-client";
+import { INFLUX_BUCKET, INFLUX_ORG, INFLUX_TOKEN, INFLUX_URL } from "./configs";
+
+export const influxDB = new InfluxDB({ url: INFLUX_URL, token: INFLUX_TOKEN });
+export const Influx = influxDB.getWriteApi(INFLUX_ORG, INFLUX_BUCKET);

template/bigbot/src/configs.ts

@@ -13,3 +13,10 @@ export const REST_PORT = 8000;
 /** The url where requests will be sent to from the bot. */
 // SETUP-DD-TEMP: Provide the url where you will host your REST proxy. If it is on the same server as others, you can use localhost but if it is on a separate server you should change this entirely.
 export const REST_URL = `http://localhost:${REST_PORT}`;
+
+/** These INFLUX configs are only if you wish to enable analytics. */
+// SETUP-DD-TEMP: This is optional. If you want to build analytics, add influxdb here.
+export const INFLUX_BUCKET = "";
+export const INFLUX_ORG = "";
+export const INFLUX_TOKEN = "";
+export const INFLUX_URL = "";

template/bigbot/src/rest/index.ts

@@ -1,7 +1,11 @@
+import { Point } from "@influxdata/influxdb-client";
 import { BASE_URL, createRestManager } from "discordeno";
 import express, { Request, Response } from "express";
+
+import { Influx } from "../analytics";
 import {
   DISCORD_TOKEN,
+  INFLUX_TOKEN,
   REST_AUTHORIZATION,
   REST_PORT,
   REST_URL,
@@ -14,6 +18,49 @@ const rest = createRestManager({
   debug: console.log,
 });
 
+// If influxdb data is provided, enable analytics in this proxy.
+if (INFLUX_TOKEN) {
+  rest.fetching = function (options) {
+    Influx.writePoint(
+      new Point("restEvents")
+        // MARK THE TIME WHEN EVENT ARRIVED
+        .timestamp(new Date())
+        // SET THE GUILD ID
+        .stringField("type", "REQUEST_FETCHING")
+        .tag("method", options.method)
+        .tag("url", options.url)
+        .tag("bucket", options.bucketId),
+    );
+  };
+
+  rest.fetched = function (options, response) {
+    Influx.writePoint(
+      new Point("restEvents")
+        // MARK THE TIME WHEN EVENT ARRIVED
+        .timestamp(new Date())
+        // SET THE GUILD ID
+        .stringField("type", "REQUEST_FETCHED")
+        .tag("method", options.method)
+        .tag("url", options.url)
+        .tag("bucket", options.bucketId)
+        .intField("status", response.status)
+        .tag("statusText", response.statusText),
+    );
+  };
+
+  setInterval(() => {
+    console.log(`[Influx - REST] Saving events...`);
+    Influx.flush()
+      .then(() => {
+        console.log(`[Influx - REST] Saved events!`);
+      })
+      .catch((error) => {
+        console.log(`[Influx - REST] Error saving events!`, error);
+      });
+    // Every 30seconds
+  }, 30000);
+}
+
 const app = express();
 
 app.use(