Add HTTP request/response logging middleware for development mode (#4425)

* Add HTTP request logging middleware for development mode

- Introduced httpLogger middleware to log HTTP requests and responses.
- Enabled logging only in development mode to assist with debugging.

* Update httpLogger middleware to disable time logging by default

* Add httpLogger middleware for development mode in collector service

* Refactor httpLogger middleware to rename timeLogs parameter to enableTimestamps for clarity

* Make HTTP Logger only mount in development and environment flag is enabled.

* Update .env.example to clarify HTTP Logger configuration comments

---------

Co-authored-by: Timothy Carambat <rambat1010@gmail.com>

collector/.env.example

@@ -1 +1,6 @@
 # Placeholder .env file for collector runtime
+
+# This enables HTTP request/response logging in development. Set value to truthy string to enable, leave empty value or comment out to disable
+# ENABLE_HTTP_LOGGER=""
+# This enables timestamps for the HTTP Logger. Set value to true to enable, leave empty or comment out to disable
+# ENABLE_HTTP_LOGGER_TIMESTAMPS=""

collector/.gitignore

@@ -4,3 +4,6 @@ yarn-error.log
 !yarn.lock
 outputs
 scripts
+.env.development
+.env.production
+.env.test

collector/index.js

@@ -15,9 +15,21 @@ const { wipeCollectorStorage } = require("./utils/files");
 const extensions = require("./extensions");
 const { processRawText } = require("./processRawText");
 const { verifyPayloadIntegrity } = require("./middleware/verifyIntegrity");
+const { httpLogger } = require("./middleware/httpLogger");
 const app = express();
 const FILE_LIMIT = "3GB";
 
+// Only log HTTP requests in development mode and if the ENABLE_HTTP_LOGGER environment variable is set to true
+if (
+  process.env.NODE_ENV === "development" &&
+  !!process.env.ENABLE_HTTP_LOGGER
+) {
+  app.use(
+    httpLogger({
+      enableTimestamps: !!process.env.ENABLE_HTTP_LOGGER_TIMESTAMPS,
+    })
+  );
+}
 app.use(cors({ origin: true }));
 app.use(
   bodyParser.text({ limit: FILE_LIMIT }),

collector/middleware/httpLogger.js

@@ -0,0 +1,29 @@
+const httpLogger =
+  ({ enableTimestamps = false }) =>
+  (req, res, next) => {
+    // Capture the original res.end to log response status
+    const originalEnd = res.end;
+
+    res.end = function (chunk, encoding) {
+      // Log the request method, status code, and path
+      const statusColor = res.statusCode >= 400 ? "\x1b[31m" : "\x1b[32m"; // Red for errors, green for success
+      console.log(
+        `\x1b[32m[HTTP]\x1b[0m ${statusColor}${res.statusCode}\x1b[0m ${
+          req.method
+        } -> ${req.path} ${
+          enableTimestamps
+            ? `@ ${new Date().toLocaleTimeString("en-US", { hour12: true })}`
+            : ""
+        }`.trim()
+      );
+
+      // Call the original end method
+      return originalEnd.call(this, chunk, encoding);
+    };
+
+    next();
+  };
+
+module.exports = {
+  httpLogger,
+};

server/.env.example

@@ -368,4 +368,9 @@ TTS_PROVIDER="native"
 # Runtime flags for built-in pupeeteer Chromium instance
 # This is only required on Linux machines running AnythingLLM via Docker
 # and do not want to use the --cap-add=SYS_ADMIN docker argument
-# ANYTHINGLLM_CHROMIUM_ARGS="--no-sandbox,--disable-setuid-sandbox"
+# ANYTHINGLLM_CHROMIUM_ARGS="--no-sandbox,--disable-setuid-sandbox"
+
+# This enables HTTP request/response logging in development. Set value to a truthy string to enable, leave empty value or comment out to disable.
+# ENABLE_HTTP_LOGGER=""
+# This enables timestamps for the HTTP Logger. Set value to a truthy string to enable, leave empty value or comment out to disable.
+# ENABLE_HTTP_LOGGER_TIMESTAMPS=""

server/index.js

@@ -29,10 +29,22 @@ const { communityHubEndpoints } = require("./endpoints/communityHub");
 const { agentFlowEndpoints } = require("./endpoints/agentFlows");
 const { mcpServersEndpoints } = require("./endpoints/mcpServers");
 const { mobileEndpoints } = require("./endpoints/mobile");
+const { httpLogger } = require("./middleware/httpLogger");
 const app = express();
 const apiRouter = express.Router();
 const FILE_LIMIT = "3GB";
 
+// Only log HTTP requests in development mode and if the ENABLE_HTTP_LOGGER environment variable is set to true
+if (
+  process.env.NODE_ENV === "development" &&
+  !!process.env.ENABLE_HTTP_LOGGER
+) {
+  app.use(
+    httpLogger({
+      enableTimestamps: !!process.env.ENABLE_HTTP_LOGGER_TIMESTAMPS,
+    })
+  );
+}
 app.use(cors({ origin: true }));
 app.use(bodyParser.text({ limit: FILE_LIMIT }));
 app.use(bodyParser.json({ limit: FILE_LIMIT }));

server/middleware/httpLogger.js

@@ -0,0 +1,23 @@
+const httpLogger =
+  ({ enableTimestamps = false }) =>
+  (req, res, next) => {
+    // Capture the original res.end to log response status
+    const originalEnd = res.end;
+
+    res.end = function (chunk, encoding) {
+      // Log the request method, status code, and path
+      const statusColor = res.statusCode >= 400 ? "\x1b[31m" : "\x1b[32m"; // Red for errors, green for success
+      console.log(
+        `\x1b[32m[HTTP]\x1b[0m ${statusColor}${res.statusCode}\x1b[0m ${req.method} -> ${req.path} ${enableTimestamps ? `@ ${new Date().toLocaleTimeString("en-US", { hour12: true })}` : ""}`.trim()
+      );
+
+      // Call the original end method
+      return originalEnd.call(this, chunk, encoding);
+    };
+
+    next();
+  };
+
+module.exports = {
+  httpLogger,
+};