Skip to content

Commit d361051

Browse files
neekolasneekolas
committed
Set back to 20 epochs
1 parent a34011f commit d361051

File tree

6 files changed

+102
-91
lines changed

6 files changed

+102
-91
lines changed

‎forks/cli.ts‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ import {
88
epochRotationOperations,
99
groupCount,
1010
installationCount,
11-
network,
1211
NODE_VERSION,
1312
otherOperations,
1413
parallelOperations,
@@ -111,7 +110,7 @@ function logForkMatrixParameters(options: ForkOptions): void {
111110
);
112111
console.info(`otherOperations: ${JSON.stringify(otherOperations)}`);
113112
console.info(`targetEpoch: ${targetEpoch}`);
114-
console.info(`network: ${network || "undefined"}`);
113+
console.info(`network: ${options.env || "undefined"}`);
115114
console.info(`randomInboxIdsCount: ${randomInboxIdsCount}`);
116115
console.info(`installationCount: ${installationCount}`);
117116
console.info(`testName: ${testName}`);

‎forks/config.ts‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,10 @@ export const epochRotationOperations = {
2121
removeMember: true, // removes a random member from the group
2222
};
2323
export const otherOperations = {
24-
createInstallation: true, // creates a new installation for a random worker
24+
createInstallation: false, // creates a new installation for a random worker
2525
sendMessage: true, // sends a message to the group
2626
};
27-
export const targetEpoch = 150n; // The target epoch to stop the test (epochs are when performing forks to the group)
27+
export const targetEpoch = 30n; // The target epoch to stop the test (epochs are when performing forks to the group)
2828
export const network = process.env.XMTP_ENV; // Network environment setting
2929
export const randomInboxIdsCount = 10; // How many inboxIds to use randomly in the add/remove operations
3030
export const installationCount = 2; // How many installations to use randomly in the createInstallation operations
@@ -63,12 +63,12 @@ export const chaosPresets: Record<ChaosLevel, ChaosPreset> = {
6363
interval: 10000, // 10 seconds
6464
},
6565
high: {
66-
delayMin: 100,
66+
delayMin: 0,
6767
delayMax: 500,
68-
jitterMin: 0,
69-
jitterMax: 100,
68+
jitterMin: 50,
69+
jitterMax: 200,
7070
lossMin: 0,
71-
lossMax: 5,
71+
lossMax: 10,
7272
interval: 10000, // 10 seconds
7373
},
7474
};
@@ -90,4 +90,5 @@ export const multinodeContainers = [
9090
"multinode-node2-1",
9191
"multinode-node3-1",
9292
"multinode-node4-1",
93+
"multinode-validation-1",
9394
];

‎forks/forks.test.ts‎

Lines changed: 77 additions & 75 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,61 @@ import {
2020
targetEpoch,
2121
testName,
2222
workerNames,
23+
type ChaosPreset,
2324
} from "./config";
2425

26+
const startChaos = (
27+
allNodes: DockerContainer[],
28+
preset: ChaosPreset,
29+
): NodeJS.Timeout => {
30+
console.log(`[chaos] Initialized ${allNodes.length} Docker containers`);
31+
32+
// Validate containers are running
33+
for (const node of allNodes) {
34+
try {
35+
// Test if container exists by trying to get its IP
36+
if (!node.ip) {
37+
throw new Error(`Container ${node.name} has no IP address`);
38+
}
39+
} catch {
40+
throw new Error(
41+
`Docker container ${node.name} is not running. Network chaos requires local multinode setup (./dev/up).`,
42+
);
43+
}
44+
}
45+
console.log("[chaos] All Docker containers validated");
46+
47+
// Function to apply chaos to all nodes
48+
const applyChaos = () => {
49+
console.log(
50+
"[chaos] Applying jitter, delay, and drop rules to all nodes...",
51+
);
52+
for (const node of allNodes) {
53+
const delay = Math.floor(
54+
preset.delayMin + Math.random() * (preset.delayMax - preset.delayMin),
55+
);
56+
const jitter = Math.floor(
57+
preset.jitterMin +
58+
Math.random() * (preset.jitterMax - preset.jitterMin),
59+
);
60+
const loss =
61+
preset.lossMin + Math.random() * (preset.lossMax - preset.lossMin);
62+
63+
try {
64+
node.addJitter(delay, jitter);
65+
if (Math.random() < 0.5) node.addLoss(loss);
66+
} catch (err) {
67+
console.warn(`[chaos] Error applying netem on ${node.name}:`, err);
68+
}
69+
}
70+
};
71+
72+
// Apply chaos immediately
73+
applyChaos();
74+
75+
return setInterval(applyChaos, preset.interval);
76+
};
77+
2578
describe(testName, () => {
2679
setupDurationTracking({ testName });
2780

@@ -91,79 +144,29 @@ describe(testName, () => {
91144

92145
// Initialize Docker containers for multinode setup
93146
allNodes = multinodeContainers.map((name) => new DockerContainer(name));
94-
console.log(`[chaos] Initialized ${allNodes.length} Docker containers`);
95-
96-
// Validate containers are running
97-
for (const node of allNodes) {
98-
try {
99-
// Test if container exists by trying to get its IP
100-
if (!node.ip) {
101-
throw new Error(`Container ${node.name} has no IP address`);
102-
}
103-
} catch (_err) {
104-
throw new Error(
105-
`Docker container ${node.name} is not running. Network chaos requires local multinode setup (./dev/up).`,
106-
);
107-
}
108-
}
109-
console.log("[chaos] All Docker containers validated");
110-
111147
const preset = chaosPresets[chaosConfig.level];
112-
113-
// Function to apply chaos to all nodes
114-
const applyChaos = () => {
115-
console.log(
116-
"[chaos] Applying jitter, delay, and drop rules to all nodes...",
117-
);
118-
for (const node of allNodes) {
119-
const delay = Math.floor(
120-
preset.delayMin +
121-
Math.random() * (preset.delayMax - preset.delayMin),
122-
);
123-
const jitter = Math.floor(
124-
preset.jitterMin +
125-
Math.random() * (preset.jitterMax - preset.jitterMin),
126-
);
127-
const loss =
128-
preset.lossMin +
129-
Math.random() * (preset.lossMax - preset.lossMin);
130-
131-
try {
132-
node.addJitter(delay, jitter);
133-
if (Math.random() < 0.5) node.addLoss(loss);
134-
} catch (err) {
135-
console.warn(
136-
`[chaos] Error applying netem on ${node.name}:`,
137-
err,
138-
);
139-
}
140-
}
141-
};
142-
143-
// Apply chaos immediately
144-
applyChaos();
145-
146148
// Then set interval for continued chaos
147-
chaosInterval = setInterval(applyChaos, preset.interval);
149+
chaosInterval = startChaos(allNodes, preset);
148150
console.log(`[chaos] Started chaos interval (${preset.interval}ms)`);
151+
}
149152

150-
// Start periodic verification during chaos
151-
const verifyLoop = () => {
152-
verifyInterval = setInterval(() => {
153-
void (async () => {
154-
try {
155-
console.log("[verify] Checking forks under chaos");
156-
await workers.checkForks();
157-
} catch (e) {
158-
console.warn("[verify] Skipping check due to exception:", e);
159-
}
160-
})();
161-
}, 10 * 1000);
162-
};
153+
// Start periodic verification during chaos
154+
const verifyLoop = () => {
155+
verifyInterval = setInterval(() => {
156+
void (async () => {
157+
try {
158+
console.log("[verify] Checking forks under chaos");
159+
await workers.checkForks();
160+
} catch (e) {
161+
console.warn("[verify] Skipping check due to exception:", e);
162+
throw e;
163+
}
164+
})();
165+
}, 10 * 1000);
166+
};
163167

164-
verifyLoop();
165-
console.log("[chaos] Started verification interval (10000ms)");
166-
}
168+
verifyLoop();
169+
console.log("Started verification interval (10000ms)");
167170

168171
// Create groups
169172
const groupOperationPromises = Array.from(
@@ -228,7 +231,13 @@ describe(testName, () => {
228231
);
229232

230233
await Promise.all(groupOperationPromises);
234+
await workers.checkForks();
235+
} catch (e) {
236+
console.error("Error during fork testing:", e);
231237
} finally {
238+
if (verifyInterval) {
239+
clearInterval(verifyInterval);
240+
}
232241
// Clean up chaos if it was enabled
233242
if (chaosConfig.enabled) {
234243
console.log("[chaos] Cleaning up network chaos...");
@@ -237,9 +246,6 @@ describe(testName, () => {
237246
if (chaosInterval) {
238247
clearInterval(chaosInterval);
239248
}
240-
if (verifyInterval) {
241-
clearInterval(verifyInterval);
242-
}
243249

244250
// Clear network rules
245251
for (const node of allNodes) {
@@ -253,10 +259,6 @@ describe(testName, () => {
253259
}
254260
}
255261

256-
// Cooldown period to allow in-flight messages to be processed
257-
console.log("[chaos] Waiting 5s cooldown before final validation");
258-
await new Promise((r) => setTimeout(r, 5000));
259-
260262
console.log("[chaos] Cleanup complete");
261263
}
262264
}

‎helpers/analyzer.ts‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -179,14 +179,16 @@ export async function cleanForksLogs(
179179
): Promise<void> {
180180
const logsDir = path.join(process.cwd(), "logs");
181181
const outputDir = path.join(logsDir, "cleaned");
182-
183-
if (!fs.existsSync(logsDir)) {
184-
console.debug("No logs directory found");
185-
return;
186-
}
182+
console.log(`Cleaning forks logs in ${outputDir}`);
187183

188184
if (!fs.existsSync(outputDir)) {
189-
await fs.promises.mkdir(outputDir, { recursive: true });
185+
try {
186+
await fs.promises.mkdir(outputDir, { recursive: true });
187+
} catch (error) {
188+
console.error(`Failed to create ${outputDir}:`, error);
189+
}
190+
191+
console.log(`Created ${outputDir}`);
190192
}
191193

192194
const files = await fs.promises.readdir(logsDir);

‎network-stability/netem.ts‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@ export function applyLatency(
44
container: DockerContainer,
55
latencyMs: number,
66
): void {
7-
console.log(`[netem] Clearing existing qdisc on ${container.veth}`);
87
container.sh(`sudo tc qdisc del dev ${container.veth} root`, true);
98
container.sh(
109
`sudo tc qdisc add dev ${container.veth} root netem delay ${latencyMs}ms`,
@@ -30,7 +29,7 @@ export function applyLoss(container: DockerContainer, percent: number): void {
3029
}
3130

3231
export function clear(container: DockerContainer): void {
33-
console.log(`[netem] Clearing latency from ${container.veth}`);
32+
// console.log(`[netem] Clearing latency from ${container.veth}`);
3433
container.sh(`sudo tc qdisc del dev ${container.veth} root`, true);
3534
}
3635

‎workers/manager.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,10 @@ import {
1212
type Group,
1313
type XmtpEnv,
1414
} from "@helpers/versions";
15+
import {
16+
GroupPermissions,
17+
GroupPermissionsOptions,
18+
} from "@xmtp/node-sdk-4.3.0";
1519
import { generatePrivateKey, privateKeyToAccount } from "viem/accounts";
1620
import {
1721
installationThreshold,
@@ -268,6 +272,10 @@ export class WorkerManager implements IWorkerManager {
268272
await group.addMembers(extraMembers);
269273
}
270274

275+
for (const member of memberList) {
276+
await group.addAdmin(member);
277+
}
278+
271279
return group as Group;
272280
}
273281

0 commit comments

Comments
 (0)