Merge pull request #4874 from Infisical/dynamic-import-nock

Move nock to dev deps and include the route only for dev/test builds

Author: fangpenlin

.gitignore

@@ -74,3 +74,4 @@ cli/test/infisical-merge
 backend/bdd/.bdd-infisical-bootstrap-result.json
 
 /npm/bin
+__pycache__

backend/bdd/features/steps/pki_acme.py

@@ -615,13 +615,14 @@ def step_impl(context: Context, var_path: str, jq_query: str, expected: str):
 
 @then('the value {var_path} with jq "{jq_query}" should match pattern {regex}')
 def step_impl(context: Context, var_path: str, jq_query: str, regex: str):
+    actual_regex = replace_vars(regex, context.vars)
     value, result = apply_value_with_jq(
         context=context,
         var_path=var_path,
         jq_query=jq_query,
     )
-    assert re.match(replace_vars(regex, context.vars), result), (
-        f"{json.dumps(value)!r} with jq {jq_query!r}, the result {json.dumps(result)!r} does not match {regex!r}"
+    assert re.match(actual_regex, result), (
+        f"{json.dumps(value)!r} with jq {jq_query!r}, the result {json.dumps(result)!r} does not match {actual_regex!r}"
     )
 
 

backend/bdd/features/steps/utils.py

@@ -15,6 +15,7 @@
 
 ACC_KEY_BITS = 2048
 ACC_KEY_PUBLIC_EXPONENT = 65537
+NOCK_API_PREFIX = "/api/__bdd_nock__"
 logger = logging.getLogger(__name__)
 faker = Faker()
 
@@ -265,7 +266,7 @@ def extension_to_dict(ext):
 def define_nock(context: Context, definitions: list[dict]):
     jwt_token = context.vars["AUTH_TOKEN"]
     response = context.http_client.post(
-        "/api/v1/bdd-nock/define",
+        f"{NOCK_API_PREFIX}/define",
         headers=dict(authorization="Bearer {}".format(jwt_token)),
         json=dict(definitions=definitions),
     )
@@ -275,7 +276,7 @@ def define_nock(context: Context, definitions: list[dict]):
 def restore_nock(context: Context):
     jwt_token = context.vars["AUTH_TOKEN"]
     response = context.http_client.post(
-        "/api/v1/bdd-nock/restore",
+        f"{NOCK_API_PREFIX}/restore",
         headers=dict(authorization="Bearer {}".format(jwt_token)),
         json=dict(),
     )
@@ -285,7 +286,7 @@ def restore_nock(context: Context):
 def clean_all_nock(context: Context):
     jwt_token = context.vars["AUTH_TOKEN"]
     response = context.http_client.post(
-        "/api/v1/bdd-nock/clean-all",
+        f"{NOCK_API_PREFIX}/clean-all",
         headers=dict(authorization="Bearer {}".format(jwt_token)),
         json=dict(),
     )

backend/nodemon.json

@@ -1,6 +1,8 @@
 {
-  "watch": ["src"],
+  "watch": [
+    "src"
+  ],
   "ext": ".ts,.js",
   "ignore": [],
-  "exec": "tsx ./src/main.ts | pino-pretty --colorize --colorizeObjects --singleLine"
-}
+  "exec": "tsx --tsconfig=./tsconfig.dev.json --inspect=0.0.0.0:9229 ./src/main.ts | pino-pretty --colorize --colorizeObjects --singleLine"
+}

backend/package.json

@@ -32,7 +32,7 @@
     "binary:clean": "rm -rf ./dist && rm -rf ./binary",
     "binary:rename-imports": "ts-node ./scripts/rename-mjs.ts",
     "test": "echo \"Error: no test specified\" && exit 1",
-    "dev": "tsx watch --clear-screen=false ./src/main.ts  | pino-pretty --colorize --colorizeObjects --singleLine",
+    "dev": "tsx watch --clear-screen=false ./src/main.ts  --config tsconfig.dev.json | pino-pretty --colorize --colorizeObjects --singleLine",
     "dev:docker": "nodemon",
     "build": "tsup --sourcemap",
     "build:frontend": "npm run build --prefix ../frontend",
@@ -266,4 +266,4 @@
     "zod": "^3.22.4",
     "zod-to-json-schema": "^3.24.5"
   }
-}
+}

backend/src/lib/config/env.ts

@@ -400,7 +400,7 @@ const envSchema = z
     isAcmeDevelopmentMode: data.NODE_ENV === "development" && data.ACME_DEVELOPMENT_MODE,
     isProductionMode: data.NODE_ENV === "production" || IS_PACKAGED,
     isRedisSentinelMode: Boolean(data.REDIS_SENTINEL_HOSTS),
-    isBddNockApiEnabled: data.NODE_ENV === "development" && data.BDD_NOCK_API_ENABLED,
+    isBddNockApiEnabled: data.NODE_ENV !== "production" && data.BDD_NOCK_API_ENABLED,
     REDIS_SENTINEL_HOSTS: data.REDIS_SENTINEL_HOSTS?.trim()
       ?.split(",")
       .map((el) => {

backend/src/server/routes/bdd/bdd-nock-router.dev.ts

@@ -0,0 +1,104 @@
+import type { Definition } from "nock";
+import { z } from "zod";
+
+import { getConfig } from "@app/lib/config/env";
+import { ForbiddenRequestError } from "@app/lib/errors";
+import { logger } from "@app/lib/logger";
+import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
+import { AuthMode } from "@app/services/auth/auth-type";
+
+// When running in production, we don't want to even import nock, because it's not needed and it increases memory usage a lots.
+// It once caused an outage in the production environment.
+// This is why we would rather to crash the app if it's not in development mode (in that case, Kubernetes should stop it from rolling out).
+if (process.env.NODE_ENV === "production") {
+  throw new Error("BDD Nock API can only be enabled in development or test mode");
+}
+
+export const registerBddNockRouter = async (server: FastifyZodProvider) => {
+  const appCfg = getConfig();
+  const importNock = async () => {
+    // eslint-disable-next-line import/no-extraneous-dependencies
+    const { default: nock } = await import("nock");
+    return nock;
+  };
+
+  const checkIfBddNockApiEnabled = () => {
+    // Note: Please note that this API is only available in development mode and only for BDD tests.
+    // This endpoint should NEVER BE ENABLED IN PRODUCTION!
+    if (appCfg.NODE_ENV === "production" || !appCfg.isBddNockApiEnabled) {
+      throw new ForbiddenRequestError({ message: "BDD Nock API is not enabled" });
+    }
+  };
+
+  server.route({
+    method: "POST",
+    url: "/define",
+    schema: {
+      body: z.object({ definitions: z.unknown().array() }),
+      response: {
+        200: z.object({ status: z.string() })
+      }
+    },
+    onRequest: verifyAuth([AuthMode.JWT]),
+    handler: async (req) => {
+      checkIfBddNockApiEnabled();
+      const { body } = req;
+      const { definitions } = body;
+      logger.info(definitions, "Defining nock");
+      const processedDefinitions = definitions.map((definition: unknown) => {
+        const { path, ...rest } = definition as Definition;
+        return {
+          ...rest,
+          path:
+            path !== undefined && typeof path === "string"
+              ? path
+              : new RegExp((path as unknown as { regex: string }).regex ?? "")
+        } as Definition;
+      });
+
+      const nock = await importNock();
+      nock.define(processedDefinitions);
+      // Ensure we are activating the nocks, because we could have called `nock.restore()` before this call.
+      if (!nock.isActive()) {
+        nock.activate();
+      }
+      return { status: "ok" };
+    }
+  });
+
+  server.route({
+    method: "POST",
+    url: "/clean-all",
+    schema: {
+      response: {
+        200: z.object({ status: z.string() })
+      }
+    },
+    onRequest: verifyAuth([AuthMode.JWT]),
+    handler: async () => {
+      checkIfBddNockApiEnabled();
+      logger.info("Cleaning all nocks");
+      const nock = await importNock();
+      nock.cleanAll();
+      return { status: "ok" };
+    }
+  });
+
+  server.route({
+    method: "POST",
+    url: "/restore",
+    schema: {
+      response: {
+        200: z.object({ status: z.string() })
+      }
+    },
+    onRequest: verifyAuth([AuthMode.JWT]),
+    handler: async () => {
+      checkIfBddNockApiEnabled();
+      logger.info("Restore network requests from nock");
+      const nock = await importNock();
+      nock.restore();
+      return { status: "ok" };
+    }
+  });
+};

backend/src/server/routes/bdd/bdd-nock-router.ts

@@ -0,0 +1,6 @@
+export const registerBddNockRouter = async () => {
+  // This route is only available in development or test mode.
+  // The actual implementation is in the dev.ts file and will be aliased to that file in development or test mode.
+  // And if somehow we try to enable it in production, we will throw an error.
+  throw new Error("BDD Nock should not be enabled in production");
+};

backend/src/server/routes/index.ts

@@ -1,3 +1,4 @@
+import { registerBddNockRouter } from "@bdd_routes/bdd-nock-router";
 import { CronJob } from "cron";
 import { Knex } from "knex";
 import { monitorEventLoopDelay } from "perf_hooks";
@@ -2698,6 +2699,12 @@ export const registerRoutes = async (
   await server.register(registerV3Routes, { prefix: "/api/v3" });
   await server.register(registerV4Routes, { prefix: "/api/v4" });
 
+  // Note: This is a special route for BDD tests. It's only available in development mode and only for BDD tests.
+  // This route should NEVER BE ENABLED IN PRODUCTION!
+  if (getConfig().isBddNockApiEnabled) {
+    await server.register(registerBddNockRouter, { prefix: "/api/__bdd_nock__" });
+  }
+
   server.addHook("onClose", async () => {
     cronJobs.forEach((job) => job.stop());
     await telemetryService.flushAll();