Add DB chaos by locking DB file

forks/cli.ts

@@ -26,6 +26,10 @@ interface ForkOptions {
   chaosEnabled: boolean; // Enable network chaos
   chaosLevel: ChaosLevel; // Chaos level (low, medium, high)
   streams: boolean; // Enable message streams on all workers
+  dbChaosEnabled: boolean; // Enable database chaos
+  dbLockTimeMin: number; // Minimum DB lock duration in ms
+  dbLockTimeMax: number; // Maximum DB lock duration in ms
+  dbLockInterval: number; // How often to apply DB locks in ms
 }
 
 function showHelp() {
@@ -44,6 +48,10 @@ OPTIONS:
   --chaos-enabled            Enable network chaos testing (requires --env local)
   --chaos-level <level>      Chaos level: low, medium, high [default: medium]
   --streams                  Enable message streams on all workers [default: false]
+  --db-chaos-enabled         Enable database locking chaos [default: false]
+  --db-lock-time-min <ms>    Minimum DB lock duration in ms [default: 100]
+  --db-lock-time-max <ms>    Maximum DB lock duration in ms [default: 2000]
+  --db-lock-interval <ms>    How often to apply DB locks in ms [default: 15000]
   -h, --help                 Show this help message
 
 CHAOS LEVELS:
@@ -59,6 +67,8 @@ EXAMPLES:
   yarn fork --env local --chaos-enabled  # Run with medium network chaos
   yarn fork --env local --chaos-enabled --chaos-level high  # Run with high chaos
   yarn fork --streams                    # Run with message streams enabled
+  yarn fork --db-chaos-enabled           # Run with database locking chaos
+  yarn fork --db-chaos-enabled --db-lock-time-min 500 --db-lock-time-max 3000  # Custom DB chaos timing
 
 For more information, see: forks/README.md
 `);
@@ -90,6 +100,10 @@ function runForkTest(options: ForkOptions): boolean {
         CHAOS_ENABLED: options.chaosEnabled ? "true" : "false",
         CHAOS_LEVEL: options.chaosLevel,
         STREAMS_ENABLED: options.streams ? "true" : "false",
+        DB_CHAOS_ENABLED: options.dbChaosEnabled ? "true" : "false",
+        DB_LOCK_TIME_MIN: options.dbLockTimeMin.toString(),
+        DB_LOCK_TIME_MAX: options.dbLockTimeMax.toString(),
+        DB_LOCK_INTERVAL: options.dbLockInterval.toString(),
       },
     });
 
@@ -134,6 +148,15 @@ function logForkMatrixParameters(options: ForkOptions): void {
     console.info(`  interval: ${preset.interval}ms`);
   }
 
+  if (options.dbChaosEnabled) {
+    console.info("\nDATABASE CHAOS PARAMETERS");
+    console.info(`dbChaosEnabled: true`);
+    console.info(
+      `  lockDuration: ${options.dbLockTimeMin}-${options.dbLockTimeMax}ms`,
+    );
+    console.info(`  interval: ${options.dbLockInterval}ms`);
+  }
+
   console.info("-".repeat(60) + "\n");
 }
 
@@ -257,6 +280,10 @@ async function main() {
     chaosEnabled: false,
     chaosLevel: "medium",
     streams: false,
+    dbChaosEnabled: false,
+    dbLockTimeMin: 100,
+    dbLockTimeMax: 6000,
+    dbLockInterval: 5000,
   };
 
   // Parse arguments
@@ -328,6 +355,57 @@ async function main() {
       case "--streams":
         options.streams = true;
         break;
+      case "--db-chaos-enabled":
+        options.dbChaosEnabled = true;
+        break;
+      case "--db-lock-time-min":
+        if (i + 1 < args.length) {
+          const value = parseInt(args[i + 1], 10);
+          if (isNaN(value) || value < 0) {
+            console.error("--db-lock-time-min must be a positive number");
+            process.exit(1);
+          }
+          options.dbLockTimeMin = value;
+          i++; // Skip next argument
+        } else {
+          console.error(
+            "--db-lock-time-min flag requires a value (e.g., --db-lock-time-min 100)",
+          );
+          process.exit(1);
+        }
+        break;
+      case "--db-lock-time-max":
+        if (i + 1 < args.length) {
+          const value = parseInt(args[i + 1], 10);
+          if (isNaN(value) || value < 0) {
+            console.error("--db-lock-time-max must be a positive number");
+            process.exit(1);
+          }
+          options.dbLockTimeMax = value;
+          i++; // Skip next argument
+        } else {
+          console.error(
+            "--db-lock-time-max flag requires a value (e.g., --db-lock-time-max 2000)",
+          );
+          process.exit(1);
+        }
+        break;
+      case "--db-lock-interval":
+        if (i + 1 < args.length) {
+          const value = parseInt(args[i + 1], 10);
+          if (isNaN(value) || value < 0) {
+            console.error("--db-lock-interval must be a positive number");
+            process.exit(1);
+          }
+          options.dbLockInterval = value;
+          i++; // Skip next argument
+        } else {
+          console.error(
+            "--db-lock-interval flag requires a value (e.g., --db-lock-interval 15000)",
+          );
+          process.exit(1);
+        }
+        break;
       default:
         console.error(`Unknown option: ${arg}`);
         console.error("Use --help for usage information");

forks/config.ts

@@ -1,7 +1,7 @@
 import { getActiveVersion } from "@helpers/versions";
 
 // Fork matrix parameters - shared between test and CLI
-export const groupCount = 1;
+export const groupCount = 10;
 export const parallelOperations = 5; // How many operations to perform in parallel
 export const NODE_VERSION = getActiveVersion().nodeBindings; // default to latest version, can be overridden with --nodeBindings=3.1.1
 // By calling workers with prefix random1, random2, etc. we guarantee that creates a new key each run
@@ -88,6 +88,21 @@ export const chaosConfig: ChaosConfig = {
 // Parse streams config from environment
 export const streamsEnabled = process.env.STREAMS_ENABLED === "true";
 
+// Database chaos configuration
+export const dbLockTimeMin = parseInt(
+  process.env.DB_LOCK_TIME_MIN || "100",
+  10,
+); // Minimum lock duration in ms
+export const dbLockTimeMax = parseInt(
+  process.env.DB_LOCK_TIME_MAX || "6000",
+  10,
+); // Maximum lock duration in ms
+export const dbLockInterval = parseInt(
+  process.env.DB_LOCK_INTERVAL || "10000",
+  10,
+); // How often to apply DB locks in ms
+export const dbChaosEnabled = process.env.DB_CHAOS_ENABLED === "true";
+
 // Multinode container names for local environment chaos testing
 export const multinodeContainers = [
   "multinode-node1-1",

forks/db-chaos-utils.ts

@@ -0,0 +1,91 @@
+import type { Worker } from "@workers/manager";
+
+// Store active lock promises for cleanup
+const activeLocks = new Map<string, Promise<void>>();
+
+/**
+ * Applies database lock chaos to a random selection of workers
+ * @param allWorkers - Array of workers to potentially lock
+ * @param lockDurationMin - Minimum lock duration in ms
+ * @param lockDurationMax - Maximum lock duration in ms
+ */
+const applyDbChaos = (
+  allWorkers: Worker[],
+  lockDurationMin: number,
+  lockDurationMax: number,
+) => {
+  console.log("[db-chaos] Applying database locks...");
+
+  for (const worker of allWorkers) {
+    // Randomly decide whether to lock this worker's DB (50% chance)
+    if (Math.random() < 0.5) {
+      const duration = Math.floor(
+        lockDurationMin + Math.random() * (lockDurationMax - lockDurationMin),
+      );
+
+      const lockKey = `${worker.name}-${worker.installationId}`;
+
+      // Only lock if not already locked
+      if (!activeLocks.has(lockKey)) {
+        console.log(
+          `[db-chaos] Locking ${worker.name} database for ${duration}ms`,
+        );
+
+        // Call the lockDB method on the worker and track it
+        const lockPromise = worker.worker
+          .lockDB(duration)
+          .catch((err: unknown) => {
+            console.warn(err);
+          })
+          .finally(() => {
+            activeLocks.delete(lockKey);
+          });
+
+        activeLocks.set(lockKey, lockPromise);
+      }
+    }
+  }
+};
+
+/**
+ * Starts the database chaos loop
+ * @param allWorkers - Array of workers to apply chaos to
+ * @param lockDurationMin - Minimum lock duration in ms
+ * @param lockDurationMax - Maximum lock duration in ms
+ * @param interval - How often to apply chaos in ms
+ * @returns Interval ID for cleanup
+ */
+export const startDbChaos = (
+  allWorkers: Worker[],
+  lockDurationMin: number,
+  lockDurationMax: number,
+  interval: number,
+): NodeJS.Timeout => {
+  console.log(`[db-chaos] Initialized for ${allWorkers.length} workers`);
+  console.log(
+    `[db-chaos] Lock duration: ${lockDurationMin}-${lockDurationMax}ms, interval: ${interval}ms`,
+  );
+
+  // Function to apply chaos to workers
+  const applyChaos = () => {
+    applyDbChaos(allWorkers, lockDurationMin, lockDurationMax);
+  };
+
+  return setInterval(applyChaos, interval);
+};
+
+/**
+ * Clears all active database locks
+ */
+export const clearDbChaos = async () => {
+  console.log("[db-chaos] Clearing all active database locks...");
+
+  // Wait for all active locks to complete
+  if (activeLocks.size > 0) {
+    console.log(`[db-chaos] Waiting for ${activeLocks.size} locks to clear...`);
+    await Promise.allSettled(Array.from(activeLocks.values()));
+  }
+
+  activeLocks.clear();
+  console.log("[db-chaos] Cleanup complete");
+};

forks/forks.test.ts

@@ -9,6 +9,10 @@ import { DockerContainer } from "../network-stability/container";
 import {
   chaosConfig,
   chaosPresets,
+  dbChaosEnabled,
+  dbLockInterval,
+  dbLockTimeMax,
+  dbLockTimeMin,
   epochRotationOperations,
   groupCount,
   installationCount,
@@ -23,6 +27,7 @@ import {
   testName,
   workerNames,
 } from "./config";
+import { clearDbChaos, startDbChaos } from "./db-chaos-utils";
 import { clearChaos, startChaos } from "./utils";
 
 describe(testName, () => {
@@ -76,6 +81,7 @@ describe(testName, () => {
     // Initialize chaos variables in outer scope for cleanup
     let allNodes: DockerContainer[] = [];
     let chaosInterval: NodeJS.Timeout | undefined;
+    let dbChaosInterval: NodeJS.Timeout | undefined;
     let verifyInterval: NodeJS.Timeout | undefined;
     let mustFail = false;
 
@@ -106,6 +112,22 @@ describe(testName, () => {
         console.log(`[chaos] Started chaos interval (${preset.interval}ms)`);
       }
 
+      // Initialize database chaos if enabled
+      if (dbChaosEnabled) {
+        console.log("[db-chaos] Database chaos enabled");
+        console.log(
+          `[db-chaos] Lock duration: ${dbLockTimeMin}-${dbLockTimeMax}ms, interval: ${dbLockInterval}ms`,
+        );
+
+        dbChaosInterval = startDbChaos(
+          workers.getAll(),
+          dbLockTimeMin,
+          dbLockTimeMax,
+          dbLockInterval,
+        );
+        console.log(`[db-chaos] Started chaos interval (${dbLockInterval}ms)`);
+      }
+
       // Start periodic verification during chaos
       const verifyLoop = () => {
         verifyInterval = setInterval(() => {
@@ -219,6 +241,19 @@ describe(testName, () => {
         console.log("[chaos] Cleanup complete");
       }
 
+      // Clean up database chaos if it was enabled
+      if (dbChaosEnabled) {
+        console.log("[db-chaos] Cleaning up database chaos...");
+        // Clear interval
+        if (dbChaosInterval) {
+          clearInterval(dbChaosInterval);
+        }
+
+        await clearDbChaos();
+
+        console.log("[db-chaos] Cleanup complete");
+      }
+
       if (mustFail) {
         expect.fail(`Test failed`);
       }