Skip to content

Commit c78f6d1

Browse files
neekolasneekolas
committed
Add DB chaos by locking DB file
1 parent 24c219c commit c78f6d1

File tree

8 files changed

+916
-455
lines changed

8 files changed

+916
-455
lines changed

‎forks/cli.ts‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,10 @@ interface ForkOptions {
2626
chaosEnabled: boolean; // Enable network chaos
2727
chaosLevel: ChaosLevel; // Chaos level (low, medium, high)
2828
streams: boolean; // Enable message streams on all workers
29+
dbChaosEnabled: boolean; // Enable database chaos
30+
dbLockTimeMin: number; // Minimum DB lock duration in ms
31+
dbLockTimeMax: number; // Maximum DB lock duration in ms
32+
dbLockInterval: number; // How often to apply DB locks in ms
2933
}
3034

3135
function showHelp() {
@@ -44,6 +48,10 @@ OPTIONS:
4448
--chaos-enabled Enable network chaos testing (requires --env local)
4549
--chaos-level <level> Chaos level: low, medium, high [default: medium]
4650
--streams Enable message streams on all workers [default: false]
51+
--db-chaos-enabled Enable database locking chaos [default: false]
52+
--db-lock-time-min <ms> Minimum DB lock duration in ms [default: 100]
53+
--db-lock-time-max <ms> Maximum DB lock duration in ms [default: 2000]
54+
--db-lock-interval <ms> How often to apply DB locks in ms [default: 15000]
4755
-h, --help Show this help message
4856
4957
CHAOS LEVELS:
@@ -59,6 +67,8 @@ EXAMPLES:
5967
yarn fork --env local --chaos-enabled # Run with medium network chaos
6068
yarn fork --env local --chaos-enabled --chaos-level high # Run with high chaos
6169
yarn fork --streams # Run with message streams enabled
70+
yarn fork --db-chaos-enabled # Run with database locking chaos
71+
yarn fork --db-chaos-enabled --db-lock-time-min 500 --db-lock-time-max 3000 # Custom DB chaos timing
6272
6373
For more information, see: forks/README.md
6474
`);
@@ -90,6 +100,10 @@ function runForkTest(options: ForkOptions): boolean {
90100
CHAOS_ENABLED: options.chaosEnabled ? "true" : "false",
91101
CHAOS_LEVEL: options.chaosLevel,
92102
STREAMS_ENABLED: options.streams ? "true" : "false",
103+
DB_CHAOS_ENABLED: options.dbChaosEnabled ? "true" : "false",
104+
DB_LOCK_TIME_MIN: options.dbLockTimeMin.toString(),
105+
DB_LOCK_TIME_MAX: options.dbLockTimeMax.toString(),
106+
DB_LOCK_INTERVAL: options.dbLockInterval.toString(),
93107
},
94108
});
95109

@@ -134,6 +148,15 @@ function logForkMatrixParameters(options: ForkOptions): void {
134148
console.info(` interval: ${preset.interval}ms`);
135149
}
136150

151+
if (options.dbChaosEnabled) {
152+
console.info("\nDATABASE CHAOS PARAMETERS");
153+
console.info(`dbChaosEnabled: true`);
154+
console.info(
155+
` lockDuration: ${options.dbLockTimeMin}-${options.dbLockTimeMax}ms`,
156+
);
157+
console.info(` interval: ${options.dbLockInterval}ms`);
158+
}
159+
137160
console.info("-".repeat(60) + "\n");
138161
}
139162

@@ -257,6 +280,10 @@ async function main() {
257280
chaosEnabled: false,
258281
chaosLevel: "medium",
259282
streams: false,
283+
dbChaosEnabled: false,
284+
dbLockTimeMin: 100,
285+
dbLockTimeMax: 6000,
286+
dbLockInterval: 5000,
260287
};
261288

262289
// Parse arguments
@@ -328,6 +355,57 @@ async function main() {
328355
case "--streams":
329356
options.streams = true;
330357
break;
358+
case "--db-chaos-enabled":
359+
options.dbChaosEnabled = true;
360+
break;
361+
case "--db-lock-time-min":
362+
if (i + 1 < args.length) {
363+
const value = parseInt(args[i + 1], 10);
364+
if (isNaN(value) || value < 0) {
365+
console.error("--db-lock-time-min must be a positive number");
366+
process.exit(1);
367+
}
368+
options.dbLockTimeMin = value;
369+
i++; // Skip next argument
370+
} else {
371+
console.error(
372+
"--db-lock-time-min flag requires a value (e.g., --db-lock-time-min 100)",
373+
);
374+
process.exit(1);
375+
}
376+
break;
377+
case "--db-lock-time-max":
378+
if (i + 1 < args.length) {
379+
const value = parseInt(args[i + 1], 10);
380+
if (isNaN(value) || value < 0) {
381+
console.error("--db-lock-time-max must be a positive number");
382+
process.exit(1);
383+
}
384+
options.dbLockTimeMax = value;
385+
i++; // Skip next argument
386+
} else {
387+
console.error(
388+
"--db-lock-time-max flag requires a value (e.g., --db-lock-time-max 2000)",
389+
);
390+
process.exit(1);
391+
}
392+
break;
393+
case "--db-lock-interval":
394+
if (i + 1 < args.length) {
395+
const value = parseInt(args[i + 1], 10);
396+
if (isNaN(value) || value < 0) {
397+
console.error("--db-lock-interval must be a positive number");
398+
process.exit(1);
399+
}
400+
options.dbLockInterval = value;
401+
i++; // Skip next argument
402+
} else {
403+
console.error(
404+
"--db-lock-interval flag requires a value (e.g., --db-lock-interval 15000)",
405+
);
406+
process.exit(1);
407+
}
408+
break;
331409
default:
332410
console.error(`Unknown option: ${arg}`);
333411
console.error("Use --help for usage information");

‎forks/config.ts‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import { getActiveVersion } from "@helpers/versions";
22

33
// Fork matrix parameters - shared between test and CLI
4-
export const groupCount = 1;
4+
export const groupCount = 10;
55
export const parallelOperations = 5; // How many operations to perform in parallel
66
export const NODE_VERSION = getActiveVersion().nodeBindings; // default to latest version, can be overridden with --nodeBindings=3.1.1
77
// By calling workers with prefix random1, random2, etc. we guarantee that creates a new key each run
@@ -88,6 +88,21 @@ export const chaosConfig: ChaosConfig = {
8888
// Parse streams config from environment
8989
export const streamsEnabled = process.env.STREAMS_ENABLED === "true";
9090

91+
// Database chaos configuration
92+
export const dbLockTimeMin = parseInt(
93+
process.env.DB_LOCK_TIME_MIN || "100",
94+
10,
95+
); // Minimum lock duration in ms
96+
export const dbLockTimeMax = parseInt(
97+
process.env.DB_LOCK_TIME_MAX || "6000",
98+
10,
99+
); // Maximum lock duration in ms
100+
export const dbLockInterval = parseInt(
101+
process.env.DB_LOCK_INTERVAL || "10000",
102+
10,
103+
); // How often to apply DB locks in ms
104+
export const dbChaosEnabled = process.env.DB_CHAOS_ENABLED === "true";
105+
91106
// Multinode container names for local environment chaos testing
92107
export const multinodeContainers = [
93108
"multinode-node1-1",

‎forks/db-chaos-utils.ts‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
import type { Worker } from "@workers/manager";
2+
3+
// Store active lock promises for cleanup
4+
const activeLocks = new Map<string, Promise<void>>();
5+
6+
/**
7+
* Applies database lock chaos to a random selection of workers
8+
* @param allWorkers - Array of workers to potentially lock
9+
* @param lockDurationMin - Minimum lock duration in ms
10+
* @param lockDurationMax - Maximum lock duration in ms
11+
*/
12+
const applyDbChaos = (
13+
allWorkers: Worker[],
14+
lockDurationMin: number,
15+
lockDurationMax: number,
16+
) => {
17+
console.log("[db-chaos] Applying database locks...");
18+
19+
for (const worker of allWorkers) {
20+
// Randomly decide whether to lock this worker's DB (50% chance)
21+
if (Math.random() < 0.5) {
22+
const duration = Math.floor(
23+
lockDurationMin + Math.random() * (lockDurationMax - lockDurationMin),
24+
);
25+
26+
const lockKey = `${worker.name}-${worker.installationId}`;
27+
28+
// Only lock if not already locked
29+
if (!activeLocks.has(lockKey)) {
30+
console.log(
31+
`[db-chaos] Locking ${worker.name} database for ${duration}ms`,
32+
);
33+
34+
// Call the lockDB method on the worker and track it
35+
const lockPromise = worker.worker
36+
.lockDB(duration)
37+
.catch((err: unknown) => {
38+
console.warn(err);
39+
})
40+
.finally(() => {
41+
activeLocks.delete(lockKey);
42+
});
43+
44+
activeLocks.set(lockKey, lockPromise);
45+
}
46+
}
47+
}
48+
};
49+
50+
/**
51+
* Starts the database chaos loop
52+
* @param allWorkers - Array of workers to apply chaos to
53+
* @param lockDurationMin - Minimum lock duration in ms
54+
* @param lockDurationMax - Maximum lock duration in ms
55+
* @param interval - How often to apply chaos in ms
56+
* @returns Interval ID for cleanup
57+
*/
58+
export const startDbChaos = (
59+
allWorkers: Worker[],
60+
lockDurationMin: number,
61+
lockDurationMax: number,
62+
interval: number,
63+
): NodeJS.Timeout => {
64+
console.log(`[db-chaos] Initialized for ${allWorkers.length} workers`);
65+
console.log(
66+
`[db-chaos] Lock duration: ${lockDurationMin}-${lockDurationMax}ms, interval: ${interval}ms`,
67+
);
68+
69+
// Function to apply chaos to workers
70+
const applyChaos = () => {
71+
applyDbChaos(allWorkers, lockDurationMin, lockDurationMax);
72+
};
73+
74+
return setInterval(applyChaos, interval);
75+
};
76+
77+
/**
78+
* Clears all active database locks
79+
*/
80+
export const clearDbChaos = async () => {
81+
console.log("[db-chaos] Clearing all active database locks...");
82+
83+
// Wait for all active locks to complete
84+
if (activeLocks.size > 0) {
85+
console.log(`[db-chaos] Waiting for ${activeLocks.size} locks to clear...`);
86+
await Promise.allSettled(Array.from(activeLocks.values()));
87+
}
88+
89+
activeLocks.clear();
90+
console.log("[db-chaos] Cleanup complete");
91+
};

‎forks/forks.test.ts‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,10 @@ import { DockerContainer } from "../network-stability/container";
99
import {
1010
chaosConfig,
1111
chaosPresets,
12+
dbChaosEnabled,
13+
dbLockInterval,
14+
dbLockTimeMax,
15+
dbLockTimeMin,
1216
epochRotationOperations,
1317
groupCount,
1418
installationCount,
@@ -23,6 +27,7 @@ import {
2327
testName,
2428
workerNames,
2529
} from "./config";
30+
import { clearDbChaos, startDbChaos } from "./db-chaos-utils";
2631
import { clearChaos, startChaos } from "./utils";
2732

2833
describe(testName, () => {
@@ -76,6 +81,7 @@ describe(testName, () => {
7681
// Initialize chaos variables in outer scope for cleanup
7782
let allNodes: DockerContainer[] = [];
7883
let chaosInterval: NodeJS.Timeout | undefined;
84+
let dbChaosInterval: NodeJS.Timeout | undefined;
7985
let verifyInterval: NodeJS.Timeout | undefined;
8086
let mustFail = false;
8187

@@ -106,6 +112,22 @@ describe(testName, () => {
106112
console.log(`[chaos] Started chaos interval (${preset.interval}ms)`);
107113
}
108114

115+
// Initialize database chaos if enabled
116+
if (dbChaosEnabled) {
117+
console.log("[db-chaos] Database chaos enabled");
118+
console.log(
119+
`[db-chaos] Lock duration: ${dbLockTimeMin}-${dbLockTimeMax}ms, interval: ${dbLockInterval}ms`,
120+
);
121+
122+
dbChaosInterval = startDbChaos(
123+
workers.getAll(),
124+
dbLockTimeMin,
125+
dbLockTimeMax,
126+
dbLockInterval,
127+
);
128+
console.log(`[db-chaos] Started chaos interval (${dbLockInterval}ms)`);
129+
}
130+
109131
// Start periodic verification during chaos
110132
const verifyLoop = () => {
111133
verifyInterval = setInterval(() => {
@@ -219,6 +241,19 @@ describe(testName, () => {
219241
console.log("[chaos] Cleanup complete");
220242
}
221243

244+
// Clean up database chaos if it was enabled
245+
if (dbChaosEnabled) {
246+
console.log("[db-chaos] Cleaning up database chaos...");
247+
// Clear interval
248+
if (dbChaosInterval) {
249+
clearInterval(dbChaosInterval);
250+
}
251+
252+
await clearDbChaos();
253+
254+
console.log("[db-chaos] Cleanup complete");
255+
}
256+
222257
if (mustFail) {
223258
expect.fail(`Test failed`);
224259
}

0 commit comments

Comments
 (0)