diff --git a/chaos/db.ts b/chaos/db.ts new file mode 100644 index 000000000..bbb1ee345 --- /dev/null +++ b/chaos/db.ts @@ -0,0 +1,96 @@ +import type { ChaosProvider } from "@chaos/provider"; +import type { WorkerManager } from "@workers/manager"; + +export type DbChaosConfig = { + minLockTime: number; // Minimum duration in milliseconds to lock the database + maxLockTime: number; // Maximum duration in milliseconds to lock the database + lockInterval: number; // Interval in milliseconds between lock attempts + impactedWorkerPercentage: number; // number between 0 and 100 for what % of workers to lock on each run +}; + +export class DbChaos implements ChaosProvider { + config: DbChaosConfig; + activeLocks = new Map>(); + interval?: NodeJS.Timeout; + + constructor(config: DbChaosConfig) { + validateConfig(config); + this.config = config; + } + + start(workers: WorkerManager): Promise { + const { minLockTime, maxLockTime, lockInterval, impactedWorkerPercentage } = + this.config; + console.log( + `Starting DB Chaos: + Locking for ${minLockTime}ms - ${maxLockTime}ms + Interval: ${lockInterval}ms`, + ); + this.interval = setInterval(() => { + for (const worker of workers.getAll()) { + if (Math.random() * 100 > impactedWorkerPercentage) { + continue; + } + const duration = Math.floor( + minLockTime + Math.random() * (maxLockTime - minLockTime), + ); + + const lockKey = `${worker.name}-${worker.installationId}`; + + // Only lock if not already locked + if (!this.activeLocks.has(lockKey)) { + console.log( + `[db-chaos] Locking ${worker.name} database for ${duration}ms`, + ); + + // Call the lockDB method on the worker and track it + const lockPromise = worker.worker + .lockDB(duration) + .catch((err: unknown) => { + console.warn(err); + }) + .finally(() => { + this.activeLocks.delete(lockKey); + }); + + this.activeLocks.set(lockKey, lockPromise); + } + } + }, lockInterval); + + return Promise.resolve(); + } + + async stop() { + console.log("Stopping DB Chaos"); + if (this.interval) { + clearInterval(this.interval); + } + + // Wait for all the existing locks to complete + await Promise.allSettled(Array.from(this.activeLocks.values())); + } +} + +function validateConfig(config: DbChaosConfig): void { + if (config.minLockTime > config.maxLockTime) { + throw new Error( + "Minimum lock time cannot be greater than maximum lock time", + ); + } + + if ( + config.impactedWorkerPercentage < 0 || + config.impactedWorkerPercentage > 100 + ) { + throw new Error("Impacted worker percentage must be between 0 and 100"); + } + + if (!config.lockInterval) { + throw new Error("Lock interval must be defined"); + } + + if (config.impactedWorkerPercentage === undefined) { + throw new Error("Impacted worker percentage must be defined"); + } +} diff --git a/chaos/network.ts b/chaos/network.ts new file mode 100644 index 000000000..0f32135c5 --- /dev/null +++ b/chaos/network.ts @@ -0,0 +1,95 @@ +import type { ChaosProvider } from "@chaos/provider"; +import type { DockerContainer } from "network-stability/container"; + +export type NetworkChaosConfig = { + delayMin: number; // Minimum delay in ms + delayMax: number; // Maximum delay in ms + jitterMin: number; // Minimum jitter in ms + jitterMax: number; // Maximum jitter in ms + lossMin: number; // Minimum packet loss percentage (0-100) + lossMax: number; // Maximum packet loss percentage (0-100) + interval: number; // How often to apply chaos in ms +}; + +export class NetworkChaos implements ChaosProvider { + config: NetworkChaosConfig; + interval?: NodeJS.Timeout; + nodes: DockerContainer[]; + + constructor(config: NetworkChaosConfig, nodes: DockerContainer[]) { + this.config = config; + this.nodes = nodes; + } + + start(): Promise { + console.log(`Starting network chaos: + Nodes: ${this.nodes.map((node) => node.name).join(", ")} + Delay: ${this.config.delayMin}ms - ${this.config.delayMax}ms + Jitter: ${this.config.jitterMin}ms - ${this.config.jitterMax}ms + Loss: ${this.config.lossMin}% - ${this.config.lossMax}% + Interval: ${this.config.interval}ms`); + + validateContainers(this.nodes); + this.clearAll(); + + this.interval = setInterval(() => { + for (const node of this.nodes) { + this.applyToNode(node); + } + }, this.config.interval); + + return Promise.resolve(); + } + + private applyToNode(node: DockerContainer) { + const { delayMin, delayMax, jitterMin, jitterMax, lossMin, lossMax } = + this.config; + const delay = Math.floor(delayMin + Math.random() * (delayMax - delayMin)); + const jitter = Math.floor( + jitterMin + Math.random() * (jitterMax - jitterMin), + ); + const loss = lossMin + Math.random() * (lossMax - lossMin); + + try { + node.addJitter(delay, jitter); + node.addLoss(loss); + } catch (err) { + console.warn(`[chaos] Error applying netem on ${node.name}:`, err); + } + } + + clearAll() { + for (const node of this.nodes) { + try { + node.clearLatency(); + } catch (err) { + console.warn(`[chaos] Error clearing latency on ${node.name}:`, err); + } + } + } + + stop(): Promise { + if (this.interval) { + clearInterval(this.interval); + } + + this.clearAll(); + + return Promise.resolve(); + } +} + +const validateContainers = (allNodes: DockerContainer[]) => { + for (const node of allNodes) { + try { + // Test if container exists by trying to get its IP + if (!node.ip || !node.veth) { + throw new Error(`Container ${node.name} has no IP address`); + } + } catch { + throw new Error( + `Docker container ${node.name} is not running. Network chaos requires local multinode setup (./dev/up).`, + ); + } + } +}; diff --git a/chaos/provider.ts b/chaos/provider.ts new file mode 100644 index 000000000..4d43e6b12 --- /dev/null +++ b/chaos/provider.ts @@ -0,0 +1,10 @@ +import { type WorkerManager } from "@workers/manager"; + +// Generic interface for the Chaos Provider. +// A Chaos Provider is started after the test has been setup, but before we start performing actions +// It is stopped after the core of the test has been completed, and should remove all chaos so that final +// validations can be performed cleanly. +export interface ChaosProvider { + start(workers: WorkerManager): Promise; + stop(): Promise; +} diff --git a/chaos/streams.ts b/chaos/streams.ts new file mode 100644 index 000000000..21e87b427 --- /dev/null +++ b/chaos/streams.ts @@ -0,0 +1,41 @@ +import type { ChaosProvider } from "@chaos/provider"; +import { typeofStream, type WorkerClient } from "@workers/main"; +import type { WorkerManager } from "@workers/manager"; + +export type StreamsConfig = { + cloned: boolean; // Should the stream be run against the workers used in the tests, or a cloned client instance? +}; + +export class StreamsChaos implements ChaosProvider { + workers?: WorkerClient[]; + config: StreamsConfig; + + constructor(config: StreamsConfig) { + this.config = config; + } + + async start(workers: WorkerManager) { + console.log("Starting StreamsChaos"); + let allWorkers = workers.getAll().map((w) => w.worker); + if (this.config.cloned) { + allWorkers = await Promise.all(allWorkers.map((w) => w.clone())); + } + + this.workers = allWorkers; + for (const worker of allWorkers) { + worker.startStream(typeofStream.Message); + } + + return Promise.resolve(); + } + + stop() { + if (this.workers) { + for (const worker of this.workers) { + worker.stopStreams(); + } + } + + return Promise.resolve(); + } +} diff --git a/cli/gen.ts b/cli/gen.ts index b894f2a15..c6fc5b13a 100644 --- a/cli/gen.ts +++ b/cli/gen.ts @@ -62,9 +62,9 @@ EXAMPLES: yarn gen --help PRESET COMMANDS: - yarn update:local Update 500 inboxes for local testing - yarn update:prod Update inboxes for production testing - yarn restart:prod Restart production installations (force recreate) + yarn gen update:local Update 500 inboxes for local testing + yarn gen update:prod Update inboxes for production testing + yarn gen restart:prod Restart production installations (force recreate) For more information, see: cli/readme.md `); diff --git a/forks/README.md b/forks/README.md index b98247f72..3ffd545ee 100644 --- a/forks/README.md +++ b/forks/README.md @@ -22,7 +22,10 @@ LOG_LEVEL=debug XMTP_ENV=production ``` -### Fork generation through send testing +### Running locally +Before running this suite locally you _must_ run `yarn gen update:local` to pre-populate the database with inboxes to add and remove from the group. Otherwise add/remove member operations will fail, which will not increase the epoch or trigger forks. + +### Fork generation through parallel operations The main approach creates intentional conflicts by running parallel operations on shared groups: @@ -30,7 +33,7 @@ The main approach creates intentional conflicts by running parallel operations o - Add X workers as super admins to each group - Loop each group until epoch Y: - Choose random worker and syncAll conversations - - Run between 2 random operations: + - Run parallel operations: - Update group name - Send message (random message) - Add member (random inboxId) @@ -42,22 +45,25 @@ The main approach creates intentional conflicts by running parallel operations o ### Parameters -- **groupCount**: `5` - Number of groups to create in parallel -- **nodeBindings**: `3.x.x` - Node SDK version to use -- **parallelOperations**: `1` - How many operations to perform in parallel -- **enabledOperations**: - Operations configuration - enable/disable specific operations - - `updateName`: true, // updates the name of the group - - `sendMessage`: false, // sends a message to the group - - `addMember`: true, // adds a random member to the group - - `removeMember`: true, // removes a random member from the group - - `createInstallation`: true, // creates a new installation for a random worker -- **workerNames**: Random workers (`random1`, `random2`, ..., `random10`) -- **targetEpoch**: `100n` - The target epoch to stop the test (epochs are when performing forks to the group) -- **network**: `process.env.XMTP_ENV` - Network environment setting -- **randomInboxIdsCount**: `30` - How many inboxIds to use randomly in the add/remove operations -- **installationCount**: `5` - How many installations to use randomly in the createInstallation operations -- **typeofStreamForTest**: `typeofStream.None` - No streams started by default (configured on-demand) -- **typeOfSyncForTest**: `typeOfSync.None` - No automatic syncing (configured on-demand) +Default configuration values (can be overridden via CLI flags): + +- **groupCount**: `5` - Number of groups to create in parallel (override with `--group-count`) +- **nodeBindings**: Latest version - Node SDK version to use +- **parallelOperations**: `5` - How many operations to perform in parallel (override with `--parallel-operations`) +- **epochRotationOperations**: Operations that rotate epochs: + - `updateName`: true - Updates the name of the group + - `addMember`: true - Adds a random member to the group + - `removeMember`: true - Removes a random member from the group +- **otherOperations**: Additional operations: + - `sendMessage`: true - Sends a message to the group + - `createInstallation`: false - Creates a new installation for a random worker + - `sync`: false - Syncs the group +- **workerNames**: Random workers (`random1`, `random2`, `random3`, `random4`, `random5`) +- **targetEpoch**: `20` - The target epoch to stop the test (override with `--target-epoch`) +- **network**: `dev` - Network environment setting (override with `--env`) +- **randomInboxIdsCount**: `50` - How many inboxIds to use randomly in the add/remove operations +- **installationCount**: `2` - How many installations to use randomly in the createInstallation operations +- **backgroundStreams**: `false` - Enable message streams on all workers (enable with `--with-background-streams`) ### Test setup in local network @@ -69,7 +75,7 @@ The main approach creates intentional conflicts by running parallel operations o yarn local-update # Process that runs the test 100 times and exports forks logs -yarn test forks --attempts 100 --env local --log warn --file --forks +yarn fork --count 100 --env local ``` ## CLI Usage @@ -86,23 +92,134 @@ yarn fork --count 50 # Clean all raw logs before starting yarn fork --clean-all +# Keep logs that don't contain fork content +yarn fork --no-remove-non-matching + # Run on a specific environment yarn fork --count 200 --env local +# Enable message streams on all workers +yarn fork --with-background-streams + +# Configure test parameters +yarn fork --group-count 10 --parallel-operations 3 --target-epoch 50 + +# Set log level for test runner +yarn fork --log-level debug + # Show help yarn fork --help ``` +### CLI Options + +- `--count`: Number of times to run the fork detection process (default: 100) +- `--clean-all`: Clean all raw logs before starting (default: false) +- `--remove-non-matching`: Remove logs that don't contain fork content (default: true) +- `--no-remove-non-matching`: Keep logs that don't contain fork content +- `--env`: XMTP environment - `local`, `dev`, or `production` (default: `dev` or `XMTP_ENV`) +- `--network-chaos-level`: Network chaos level - `none`, `low`, `medium`, or `high` (default: `none`) +- `--db-chaos-level`: Database chaos level - `none`, `low`, `medium`, or `high` (default: `none`) +- `--with-background-streams`: Enable message streams on all workers (default: false) +- `--log-level`: Log level for test runner - `debug`, `info`, `warn`, `error` (default: `warn`) +- `--group-count`: Number of groups to run the test against (default: 5) +- `--parallel-operations`: Number of parallel operations run on each group (default: 5) +- `--target-epoch`: Target epoch to stop the test at (default: 20) + +### Statistics Output + The CLI provides statistics including: - Total runs and forks detected +- Runs with forks vs. runs without forks +- Runs with errors - Fork detection rate - Average forks per run +- Average forks per run (with forks only) + +### Network Chaos Testing + +The fork test can inject network chaos (latency, jitter, packet loss) to simulate adverse network conditions. This helps identify forks that occur under realistic network stress. + +**Requirements:** +- Network chaos requires `--env local` +- Multinode Docker containers must be running (`./multinode/up`) +- Must be run on Linux with `tc` and `iptables` commands available. Will not work on MacOS. +- Requires `sudo` access + +**Chaos Levels:** + +| Level | Delay Range | Jitter Range | Packet Loss | Interval | +|--------|-------------|--------------|-------------|----------| +| low | 50-150ms | 0-50ms | 0-2% | 15s | +| medium | 100-300ms | 0-75ms | 0-3.5% | 10s | +| high | 0-500ms | 50-200ms | 0-25% | 10s | + +**Usage:** + +```bash +# Run with low network chaos +yarn fork --env local --network-chaos-level low + +# Run with high network chaos level +yarn fork --env local --network-chaos-level high + +# Run 50 iterations with medium network chaos +yarn fork --count 50 --env local --network-chaos-level medium +``` + +**How it works:** +1. Initializes Docker container handles for all multinode nodes +2. Applies random network conditions (within preset ranges) at regular intervals +3. Runs the fork test as normal while chaos is active +4. Cleans up network rules when test completes (even if test fails) + +**Example output:** +``` +NETWORK CHAOS PARAMETERS + delay: 0-500ms + jitter: 50-200ms + packetLoss: 0-25% + interval: 10000ms +``` + +### Database Chaos Testing + +The fork test can inject database chaos by temporarily locking database files to simulate database contention and I/O issues. + +**Chaos Levels:** + +| Level | Lock Duration | Interval | +|--------|---------------|----------| +| low | 50-250ms | 10s | +| medium | 100-2000ms | 15s | +| high | 500-2000ms | 5s | -### Log processing features +**Usage:** -- **Clean slate**: Removes old logs and data before starting -- **Continuous capture**: Each iteration captures debug logs -- **ANSI cleaning**: Strips escape codes for analysis -- **Fork counting**: Automatically counts detected conflicts -- **Graceful interruption**: Ctrl+C exits cleanly +```bash +# Run with low database chaos +yarn fork --db-chaos-level low + +# Run with high database chaos level +yarn fork --db-chaos-level high + +# Run 50 iterations with medium database chaos +yarn fork --count 50 --db-chaos-level medium + +# Combine network and database chaos +yarn fork --env local --network-chaos-level medium --db-chaos-level medium +``` + +**How it works:** +1. Periodically locks database files for each worker +2. Lock duration is randomized within the preset range +3. Workers experience database busy/locked errors during operations +4. Cleans up and waits for all locks to complete when test finishes + +**Example output:** +``` +DATABASE CHAOS PARAMETERS + lockDuration: 500-2000ms + interval: 5000ms +``` diff --git a/forks/cli.ts b/forks/cli.ts index 596b7335a..50db6cd53 100644 --- a/forks/cli.ts +++ b/forks/cli.ts @@ -3,50 +3,30 @@ import fs from "fs"; import path from "path"; import { cleanAllRawLogs, cleanForksLogs } from "@helpers/analyzer"; import "dotenv/config"; +import yargs from "yargs"; +import { hideBin } from "yargs/helpers"; import { - epochRotationOperations, - groupCount, - installationCount, - network, - NODE_VERSION, - otherOperations, - parallelOperations, - randomInboxIdsCount, - targetEpoch, - testName, - workerNames, + printConfig, + resolveDbChaosConfig, + resolveNetworkChaosConfig, + type DbChaosLevel, + type NetworkChaosLevel, + type RuntimeConfig, } from "./config"; +// CLI options for fork testing interface ForkOptions { count: number; // Number of times to run the process (default: 100) cleanAll: boolean; // Clean all raw logs before starting removeNonMatching: boolean; // Remove non-matching logs env?: string; // XMTP environment (local, dev, production) -} - -function showHelp() { - console.log(` -XMTP Fork Detection CLI - Fork detection and analysis - -USAGE: - yarn fork [options] - -OPTIONS: - --count Number of times to run the fork detection process [default: 100] - --clean-all Clean all raw logs before starting (equivalent to ansi:clean) - --remove-non-matching Remove logs that don't contain fork content [default: true] - --no-remove-non-matching Keep logs that don't contain fork content - --env XMTP environment (local, dev, production) [default: dev] - -h, --help Show this help message - -EXAMPLES: - yarn fork # Run 100 times and get stats - yarn fork --count 50 # Run 50 times - yarn fork --clean-all # Clean all raw logs before starting - yarn fork --count 200 --env local # Run 200 times on local environment - -For more information, see: forks/README.md -`); + networkChaosLevel: NetworkChaosLevel; // Network chaos level + dbChaosLevel: DbChaosLevel; // DB chaos level + withBackgroundStreams: boolean; // Enable message streams on all workers + logLevel: string; // Log level for test runner + groupCount: number; // Number of groups to run the test against + parallelOperations: number; // Number of parallel operations run on each group + targetEpoch: number; // Target epoch to stop the test at } /** @@ -61,45 +41,53 @@ function getForkCount(): number { } /** - * Run the fork test (suppress output) + * Build RuntimeConfig from ForkOptions + */ +function buildRuntimeConfig(options: ForkOptions): RuntimeConfig { + return { + groupCount: options.groupCount, + parallelOperations: options.parallelOperations, + targetEpoch: options.targetEpoch, + network: (options.env || "dev") as "local" | "dev" | "production", + networkChaos: resolveNetworkChaosConfig(options.networkChaosLevel), + dbChaos: resolveDbChaosConfig(options.dbChaosLevel), + backgroundStreams: options.withBackgroundStreams + ? { + cloned: true, + } + : null, + }; +} + +/** + * Run the fork test (suppress output) and return whether it completed successfully */ -function runForkTest(env?: string): void { - const envFlag = env ? `--env ${env}` : ""; - const command = `yarn test forks ${envFlag} --log warn --file`.trim(); +function runForkTest( + options: ForkOptions, + runtimeConfig: RuntimeConfig, +): boolean { + const envFlag = options.env ? `--env ${options.env}` : ""; + const command = + `yarn test forks ${envFlag} --log ${options.logLevel} --file`.trim(); try { execSync(command, { stdio: "ignore", - env: { ...process.env }, + env: { + ...process.env, + FORK_TEST_CONFIG: JSON.stringify(runtimeConfig), + }, }); - } catch { + + return true; + } catch (e) { + console.error("Error running fork test", e); + return false; // Test may fail if forks are detected, that's expected // We'll analyze the logs afterward } } -/** - * Log fork matrix parameters from shared config - */ -function logForkMatrixParameters(): void { - console.info("\nFORK MATRIX PARAMETERS"); - console.info("-".repeat(60)); - console.info(`groupCount: ${groupCount}`); - console.info(`parallelOperations: ${parallelOperations}`); - console.info(`NODE_VERSION: ${NODE_VERSION}`); - console.info(`workerNames: [${workerNames.join(", ")}]`); - console.info( - `epochRotationOperations: ${JSON.stringify(epochRotationOperations)}`, - ); - console.info(`otherOperations: ${JSON.stringify(otherOperations)}`); - console.info(`targetEpoch: ${targetEpoch}`); - console.info(`network: ${network || "undefined"}`); - console.info(`randomInboxIdsCount: ${randomInboxIdsCount}`); - console.info(`installationCount: ${installationCount}`); - console.info(`testName: ${testName}`); - console.info("-".repeat(60) + "\n"); -} - /** * Run fork detection process and collect stats */ @@ -108,8 +96,20 @@ async function runForkDetection(options: ForkOptions): Promise { console.info("XMTP Fork Detection CLI"); console.info("=".repeat(60)); + // Validate chaos requirements + if (options.networkChaosLevel !== "none" && options.env !== "local") { + console.error("\n❌ Error: Network chaos testing requires --env local"); + console.error( + "Network chaos manipulates Docker containers which are only available in local environment.\n", + ); + process.exit(1); + } + + // Build RuntimeConfig for logging and running tests + const runtimeConfig = buildRuntimeConfig(options); + // Log fork matrix parameters once - logForkMatrixParameters(); + printConfig(runtimeConfig); console.info(`Running fork detection process ${options.count} time(s)...\n`); @@ -118,6 +118,7 @@ async function runForkDetection(options: ForkOptions): Promise { forksDetected: 0, runsWithForks: 0, runsWithoutForks: 0, + runsWithErrors: 0, }; // Clean logs if requested before starting @@ -130,7 +131,11 @@ async function runForkDetection(options: ForkOptions): Promise { // Run the test N times for (let i = 1; i <= options.count; i++) { // Run the fork test (silently) - runForkTest(options.env); + const success = runForkTest(options, runtimeConfig); + if (!success) { + stats.runsWithErrors++; + console.info(`❌ Error in run ${i}/${options.count}`); + } // Clean and analyze fork logs after the test (suppress output) const originalConsoleDebug = console.debug; @@ -153,15 +158,6 @@ async function runForkDetection(options: ForkOptions): Promise { stats.runsWithoutForks++; console.info(`Run ${i}/${options.count}: ⚪ No forks`); } - - // Clean up empty cleaned directory if it exists - const logsDir = path.join(process.cwd(), "logs", "cleaned"); - if (fs.existsSync(logsDir)) { - const files = fs.readdirSync(logsDir); - if (files.length === 0) { - fs.rmdirSync(logsDir); - } - } } // Display final statistics @@ -172,6 +168,7 @@ async function runForkDetection(options: ForkOptions): Promise { console.info(`Total forks detected: ${stats.forksDetected}`); console.info(`Runs with forks: ${stats.runsWithForks}`); console.info(`Runs without forks: ${stats.runsWithoutForks}`); + console.info(`Runs with errors: ${stats.runsWithErrors}`); console.info( `Fork detection rate: ${( (stats.runsWithForks / stats.totalRuns) * @@ -203,70 +200,117 @@ async function runForkDetection(options: ForkOptions): Promise { } async function main() { - const args = process.argv.slice(2); + const argv = await yargs(hideBin(process.argv)) + .usage("Usage: yarn fork [options]") + .option("count", { + type: "number", + default: 100, + describe: "Number of times to run the fork detection process", + }) + .option("clean-all", { + type: "boolean", + default: false, + describe: "Clean all raw logs before starting", + }) + .parserConfiguration({ + "boolean-negation": true, + }) + .option("remove-non-matching", { + type: "boolean", + default: true, + describe: "Remove logs that don't contain fork content", + }) + .option("env", { + type: "string", + choices: ["local", "dev", "production"] as const, + default: + (process.env.XMTP_ENV as "local" | "dev" | "production") || "local", + describe: "XMTP environment", + }) + .option("network-chaos-level", { + type: "string", + choices: ["none", "low", "medium", "high"] as const, + default: "none" as const, + describe: "Network chaos level (requires --env local)", + }) + .option("db-chaos-level", { + type: "string", + choices: ["none", "low", "medium", "high"] as const, + default: "none" as const, + describe: "Database chaos level with presets", + }) + .option("with-background-streams", { + type: "boolean", + default: false, + describe: "Enable message streams on all workers", + }) + .option("log-level", { + type: "string", + default: "warn", + describe: "Log level for test runner (e.g., debug, info, warn, error)", + }) + .option("group-count", { + type: "number", + default: 5, + describe: "Number of groups to run the test against", + }) + .option("parallel-operations", { + type: "number", + default: 5, + describe: "Number of parallel operations run on each group", + }) + .option("target-epoch", { + type: "number", + default: 20, + describe: "Target epoch to stop the test at", + }) + .example("yarn fork", "Run 100 times and get stats") + .example("yarn fork --count 50", "Run 50 times") + .example("yarn fork --clean-all", "Clean all raw logs before starting") + .example( + "yarn fork --count 200 --env local", + "Run 200 times on local environment", + ) + .example( + "yarn fork --env local --network-chaos-level medium", + "Run with medium network chaos", + ) + .example( + "yarn fork --env local --network-chaos-level high", + "Run with high network chaos", + ) + .example( + "yarn fork --with-background-streams", + "Run with message streams enabled", + ) + .example( + "yarn fork --db-chaos-level medium", + "Run with medium database locking chaos", + ) + .example( + "yarn fork --no-remove-non-matching", + "Keep logs that don't contain fork content", + ) + .epilogue("For more information, see: forks/README.md") + .help() + .alias("h", "help") + .strict() + .parseAsync(); - // Default options const options: ForkOptions = { - count: 100, - cleanAll: false, - removeNonMatching: true, - env: process.env.XMTP_ENV || "dev", + count: argv.count, + cleanAll: argv["clean-all"], + removeNonMatching: argv["remove-non-matching"], + env: argv.env, + networkChaosLevel: argv["network-chaos-level"] as NetworkChaosLevel, + dbChaosLevel: argv["db-chaos-level"] as DbChaosLevel, + withBackgroundStreams: argv["with-background-streams"], + logLevel: argv["log-level"], + groupCount: argv["group-count"], + parallelOperations: argv["parallel-operations"], + targetEpoch: argv["target-epoch"], }; - // Parse arguments - for (let i = 0; i < args.length; i++) { - const arg = args[i]; - - switch (arg) { - case "-h": - case "--help": - showHelp(); - process.exit(0); - break; - case "--count": - if (i + 1 < args.length) { - const count = parseInt(args[i + 1], 10); - if (isNaN(count) || count < 1) { - console.error("--count must be a positive number"); - process.exit(1); - } - options.count = count; - i++; // Skip next argument - } else { - console.error("--count flag requires a value (e.g., --count 100)"); - process.exit(1); - } - break; - case "--clean-all": - options.cleanAll = true; - break; - case "--remove-non-matching": - options.removeNonMatching = true; - break; - case "--no-remove-non-matching": - options.removeNonMatching = false; - break; - case "--env": - if (i + 1 < args.length) { - const env = args[i + 1]; - if (!["local", "dev", "production"].includes(env)) { - console.error("--env must be one of: local, dev, production"); - process.exit(1); - } - options.env = env; - i++; // Skip next argument - } else { - console.error("--env flag requires a value (e.g., --env dev)"); - process.exit(1); - } - break; - default: - console.error(`Unknown option: ${arg}`); - console.error("Use --help for usage information"); - process.exit(1); - } - } - await runForkDetection(options); } diff --git a/forks/config.ts b/forks/config.ts index 21cfae648..bc2c85252 100644 --- a/forks/config.ts +++ b/forks/config.ts @@ -1,8 +1,8 @@ -import { getActiveVersion } from "@helpers/versions"; +import { type DbChaosConfig } from "@chaos/db"; +import type { NetworkChaosConfig } from "@chaos/network"; +import type { StreamsConfig } from "@chaos/streams"; +import { getActiveVersion, type XmtpEnv } from "@helpers/versions"; -// Fork matrix parameters - shared between test and CLI -export const groupCount = 5; -export const parallelOperations = 5; // How many operations to perform in parallel export const NODE_VERSION = getActiveVersion().nodeBindings; // default to latest version, can be overridden with --nodeBindings=3.1.1 // By calling workers with prefix random1, random2, etc. we guarantee that creates a new key each run // We want to create a key each run to ensure the forks are "pure" @@ -23,9 +23,174 @@ export const epochRotationOperations = { export const otherOperations = { createInstallation: true, // creates a new installation for a random worker sendMessage: true, // sends a message to the group + sync: true, // syncs the group }; -export const targetEpoch = 20n; // The target epoch to stop the test (epochs are when performing forks to the group) -export const network = process.env.XMTP_ENV; // Network environment setting -export const randomInboxIdsCount = 10; // How many inboxIds to use randomly in the add/remove operations +export const randomInboxIdsCount = 50; // How many inboxIds to use randomly in the add/remove operations export const installationCount = 2; // How many installations to use randomly in the createInstallation operations +export const maxConsecutiveFailures = 5; // Maximum number of times we throw an error verifying the results of a batch of operations export const testName = "forks"; + +// Network chaos configuration +export type NetworkChaosLevel = "none" | "low" | "medium" | "high"; + +export const networkChaosPresets: Record< + Exclude, + NetworkChaosConfig +> = { + low: { + delayMin: 50, + delayMax: 150, + jitterMin: 0, + jitterMax: 50, + lossMin: 0, + lossMax: 2, + interval: 15000, // 15 seconds + }, + medium: { + delayMin: 100, + delayMax: 300, + jitterMin: 0, + jitterMax: 75, + lossMin: 0, + lossMax: 3.5, + interval: 10000, // 10 seconds + }, + high: { + delayMin: 0, + delayMax: 500, + jitterMin: 50, + jitterMax: 200, + lossMin: 0, + lossMax: 25, + interval: 10000, // 10 seconds + }, +}; + +// Database chaos configuration +export type DbChaosLevel = "none" | "low" | "medium" | "high"; + +export const dbChaosPresets: Record< + Exclude, + DbChaosConfig +> = { + low: { + minLockTime: 50, + maxLockTime: 250, + lockInterval: 10000, // 10 seconds + impactedWorkerPercentage: 20, + }, + medium: { + minLockTime: 100, + maxLockTime: 2000, + lockInterval: 15000, // 15 seconds + impactedWorkerPercentage: 40, + }, + high: { + minLockTime: 500, + maxLockTime: 2000, + lockInterval: 5000, // 5 seconds + impactedWorkerPercentage: 60, + }, +}; + +// Helper functions to get chaos configs +export function resolveNetworkChaosConfig( + networkChaosLevel: NetworkChaosLevel, +): NetworkChaosConfig | null { + if (networkChaosLevel === "none") return null; + if (!networkChaosPresets[networkChaosLevel]) { + throw new Error(`Invalid network chaos level: ${networkChaosLevel}`); + } + return networkChaosPresets[networkChaosLevel]; +} + +export function resolveDbChaosConfig( + dbChaosLevel: DbChaosLevel, +): DbChaosConfig | null { + if (dbChaosLevel === "none") return null; + if (!dbChaosPresets[dbChaosLevel]) { + throw new Error(`Invalid DB chaos level: ${dbChaosLevel}`); + } + + return dbChaosPresets[dbChaosLevel]; +} + +// Multinode container names for local environment chaos testing +export const multinodeContainers = [ + "multinode-node1-1", + "multinode-node2-1", + "multinode-node3-1", + "multinode-node4-1", + // Include the MLS validation service to add some additional chaos + "multinode-validation-1", +]; + +/** + * The config flags that are passed in as JSON from the environment + */ +export type RuntimeConfig = { + groupCount: number; // Number of groups to run the test against + parallelOperations: number; // Number of parallel operations run on each group + targetEpoch: number; // Target epoch to stop the test at + network: XmtpEnv; // XMTP network + networkChaos: NetworkChaosConfig | null; // Network chaos configuration + dbChaos: DbChaosConfig | null; // Database chaos configuration + backgroundStreams: StreamsConfig | null; // +}; + +export function getConfigFromEnv(): RuntimeConfig { + const jsonString = process.env.FORK_TEST_CONFIG; + if (!jsonString) { + throw new Error("FORK_TEST_CONFIG environment variable is not set"); + } + + return JSON.parse(jsonString) as RuntimeConfig; +} + +/** + * Pretty-print the complete runtime configuration + */ +export function printConfig(config: RuntimeConfig): void { + console.info("\nFORK MATRIX PARAMETERS"); + console.info("-".repeat(60)); + console.info(`groupCount: ${config.groupCount}`); + console.info(`parallelOperations: ${config.parallelOperations}`); + console.info(`NODE_VERSION: ${NODE_VERSION}`); + console.info(`workerNames: [${workerNames.join(", ")}]`); + console.info( + `epochRotationOperations: ${JSON.stringify(epochRotationOperations)}`, + ); + console.info(`otherOperations: ${JSON.stringify(otherOperations)}`); + console.info(`targetEpoch: ${config.targetEpoch}`); + console.info(`network: ${config.network}`); + console.info(`randomInboxIdsCount: ${randomInboxIdsCount}`); + console.info(`installationCount: ${installationCount}`); + console.info(`testName: ${testName}`); + console.info( + `backgroundStreams: ${config.backgroundStreams ? "enabled" : "disabled"}. From separate client instances: ${config.backgroundStreams?.cloned}`, + ); + + if (config.networkChaos) { + console.info("\nNETWORK CHAOS PARAMETERS"); + console.info( + ` delay: ${config.networkChaos.delayMin}-${config.networkChaos.delayMax}ms`, + ); + console.info( + ` jitter: ${config.networkChaos.jitterMin}-${config.networkChaos.jitterMax}ms`, + ); + console.info( + ` packetLoss: ${config.networkChaos.lossMin}-${config.networkChaos.lossMax}%`, + ); + console.info(` interval: ${config.networkChaos.interval}ms`); + } + + if (config.dbChaos) { + console.info("\nDATABASE CHAOS PARAMETERS"); + console.info( + ` lockDuration: ${config.dbChaos.minLockTime}-${config.dbChaos.maxLockTime}ms`, + ); + console.info(` interval: ${config.dbChaos.lockInterval}ms`); + } + + console.info("-".repeat(60) + "\n"); +} diff --git a/forks/constants.ts b/forks/constants.ts new file mode 100644 index 000000000..a3dadc04e --- /dev/null +++ b/forks/constants.ts @@ -0,0 +1 @@ +export const forkDetectedString = "[FORK DETECTED]"; diff --git a/forks/forks.test.ts b/forks/forks.test.ts index b63973d5f..1d8be6eb9 100644 --- a/forks/forks.test.ts +++ b/forks/forks.test.ts @@ -1,137 +1,254 @@ +import { DbChaos } from "@chaos/db"; +import { NetworkChaos } from "@chaos/network"; +import type { ChaosProvider } from "@chaos/provider"; +import { StreamsChaos } from "@chaos/streams"; import { getTime } from "@helpers/logger"; import { type Group } from "@helpers/versions"; import { setupDurationTracking } from "@helpers/vitest"; import { getInboxes } from "@inboxes/utils"; -import { getWorkers, type Worker } from "@workers/manager"; -import { describe, it } from "vitest"; +import { DockerContainer } from "@network-stability/container"; +import { getWorkers, type Worker, type WorkerManager } from "@workers/manager"; +import { describe, expect, it } from "vitest"; import { epochRotationOperations, - groupCount, + getConfigFromEnv, installationCount, - network, + maxConsecutiveFailures, + multinodeContainers, NODE_VERSION, otherOperations, - parallelOperations, randomInboxIdsCount, - targetEpoch, testName, workerNames, } from "./config"; -describe(testName, () => { - setupDurationTracking({ testName }); +const { + groupCount, + parallelOperations, + network, + targetEpoch, + networkChaos, + dbChaos, + backgroundStreams, +} = getConfigFromEnv(); - const createOperations = async (worker: Worker, group: Group) => { - // This syncs all and can contribute to the fork - await worker.client.conversations.syncAll(); - - // Fetches the group from the worker perspective - const getGroup = () => - worker.client.conversations.getConversationById( - group.id, - ) as Promise; - - return { - updateName: () => - getGroup().then((g) => - g.updateName(`${getTime()} - ${worker.name} Update`), - ), - createInstallation: () => - getGroup().then(() => worker.worker.addNewInstallation()), - addMember: () => - getGroup().then((g) => { - const randomInboxIds = getInboxes( - randomInboxIdsCount, - installationCount, - ); - return g.addMembers([ - randomInboxIds[Math.floor(Math.random() * randomInboxIds.length)] - .inboxId, - ]); - }), - removeMember: () => - getGroup().then((g) => { - const randomInboxIds = getInboxes( - randomInboxIdsCount, - installationCount, - ); - return g.removeMembers([ - randomInboxIds[Math.floor(Math.random() * randomInboxIds.length)] - .inboxId, - ]); - }), - sendMessage: () => - getGroup().then((g) => - g.send(`Message from ${worker.name}`).then(() => {}), - ), - }; +const createOperations = (worker: Worker, groupID: string) => { + const getGroup = async () => { + const group = + await worker.client.conversations.getConversationById(groupID); + if (!group) { + throw new Error(`Group ${groupID} not found`); + } + return group as Group; }; + return { + updateName: () => + getGroup().then((g) => + g.updateName(`${getTime()} - ${worker.name} Update`), + ), + createInstallation: () => + getGroup().then(() => worker.worker.addNewInstallation()), + addMember: () => + getGroup().then((g) => { + const randomInboxIds = getInboxes( + randomInboxIdsCount, + installationCount, + ); + return g.addMembers([ + randomInboxIds[Math.floor(Math.random() * randomInboxIds.length)] + .inboxId, + ]); + }), + + removeMember: () => + getGroup().then((g) => { + const randomInboxIds = getInboxes( + randomInboxIdsCount, + installationCount, + ); + return g.removeMembers([ + randomInboxIds[Math.floor(Math.random() * randomInboxIds.length)] + .inboxId, + ]); + }), + sendMessage: () => + getGroup().then((g) => + g.send(`Message from ${worker.name}`).then(() => {}), + ), + sync: () => getGroup().then((g) => g.sync()), + }; +}; + +const startChaos = async (workers: WorkerManager): Promise => { + let chaosProviders: ChaosProvider[] = []; + + // Set up chaos providers based on config + if (networkChaos) { + const containers = multinodeContainers.map( + (name) => new DockerContainer(name), + ); + chaosProviders.push(new NetworkChaos(networkChaos, containers)); + } + + if (dbChaos) { + chaosProviders.push(new DbChaos(dbChaos)); + } + + if (backgroundStreams) { + chaosProviders.push(new StreamsChaos(backgroundStreams)); + } + + // Start all chaos providers + for (const provider of chaosProviders) { + await provider.start(workers); + } + + return chaosProviders; +}; + +describe(testName, () => { + setupDurationTracking({ testName }); + it("perform concurrent operations with multiple users across 5 groups", async () => { let workers = await getWorkers(workerNames, { env: network as "local" | "dev" | "production", nodeBindings: NODE_VERSION, }); - // Note: typeofStreamForTest and typeOfSyncForTest are set to None, so no streams or syncs to start - // Create groups - const groupOperationPromises = Array.from( - { length: groupCount }, - async (_, groupIndex) => { - const group = await workers.createGroupBetweenAll(); - - let currentEpoch = 0n; - - while (currentEpoch < targetEpoch) { - const parallelOperationsArray = Array.from( - { length: parallelOperations }, - () => - (async () => { - const randomWorker = - workers.getAll()[ - Math.floor(Math.random() * workers.getAll().length) - ]; - const ops = await createOperations(randomWorker, group); - const operationList = [ - ...(epochRotationOperations.updateName - ? [ops.updateName] - : []), - ...(epochRotationOperations.addMember ? [ops.addMember] : []), - ...(epochRotationOperations.removeMember - ? [ops.removeMember] - : []), - ]; - const otherOperationList = [ - ...(otherOperations.createInstallation - ? [ops.createInstallation] - : []), - ...(otherOperations.sendMessage ? [ops.sendMessage] : []), - ]; - - const randomOperation = - operationList[ - Math.floor(Math.random() * operationList.length) + // Create the groups before starting any chaos + let groupIDs = await Promise.all( + Array.from({ length: groupCount }).map( + async () => (await workers.createGroupBetweenAll()).id, + ), + ); + + // Make sure everyone has the group before starting + await Promise.all( + workers.getAll().map((w) => w.client.conversations.sync()), + ); + + const chaosProviders = await startChaos(workers); + + let verifyInterval = setInterval(() => { + void (async () => { + try { + console.log("[verify] Checking forks under chaos"); + await workers.checkForks(); + } catch (e) { + console.warn("[verify] Skipping check due to exception:", e); + } + })(); + }, 10 * 1000); + + console.log("Started verification interval (10000ms)"); + + try { + // Create groups + const groupOperationPromises = groupIDs.map( + async (groupID, groupIndex) => { + let currentEpoch = 0n; + let numConsecutiveFailures = 0; + + // Run until we reach the target epoch or we hit 5 consecutive failures. + while (currentEpoch < targetEpoch) { + const parallelOperationsArray = Array.from( + { length: parallelOperations }, + () => + (async () => { + const randomWorker = + workers.getAll()[ + Math.floor(Math.random() * workers.getAll().length) + ]; + + const ops = createOperations(randomWorker, groupID); + const operationList = [ + ...(epochRotationOperations.updateName + ? [ops.updateName] + : []), + ...(epochRotationOperations.addMember + ? [ops.addMember] + : []), + ...(epochRotationOperations.removeMember + ? [ops.removeMember] + : []), ]; - const otherRandomOperation = - otherOperationList[ - Math.floor(Math.random() * otherOperationList.length) + const otherOperationList = [ + ...(otherOperations.createInstallation + ? [ops.createInstallation] + : []), + ...(otherOperations.sendMessage ? [ops.sendMessage] : []), + ...(otherOperations.sync ? [ops.sync] : []), ]; - try { - await randomOperation(); - await otherRandomOperation(); - } catch (e) { - console.log(`Group ${groupIndex + 1} operation failed:`, e); - } - })(), - ); - await Promise.all(parallelOperationsArray); - await workers.checkForksForGroup(group.id); - currentEpoch = (await group.debugInfo()).epoch; - } - return { groupIndex, finalEpoch: currentEpoch }; - }, - ); - await Promise.all(groupOperationPromises); + const randomOperation = + operationList[ + Math.floor(Math.random() * operationList.length) + ]; + const otherRandomOperation = + otherOperationList[ + Math.floor(Math.random() * otherOperationList.length) + ]; + try { + await randomOperation(); + await otherRandomOperation(); + } catch (e) { + console.error( + `Group ${groupIndex + 1} operation failed:`, + e, + ); + } + })(), + ); + + // We want to wait for all operations to complete, but ignore any errors which may be caused by the chaos + const results = await Promise.allSettled(parallelOperationsArray); + for (const result of results) { + if (result.status === "rejected") { + console.error( + `Group ${groupIndex + 1} operation failed:`, + result.reason, + ); + } + } + + try { + await workers.checkForksForGroup(groupID); + const group = await workers + .getCreator() + .client.conversations.getConversationById(groupID); + if (!group) { + throw new Error("Could not find group"); + } + currentEpoch = (await group.debugInfo()).epoch; + numConsecutiveFailures = 0; + } catch (e) { + console.error(`Group ${groupIndex + 1} operation failed:`, e); + numConsecutiveFailures++; + if (numConsecutiveFailures >= maxConsecutiveFailures) { + throw e; + } + } + } + + return { groupIndex, finalEpoch: currentEpoch }; + }, + ); + + await Promise.all(groupOperationPromises); + } catch (e: any) { + const msg = `Error during fork testing: ${e}`; + console.error(msg); + // This will fail the test if there were too many failures + expect.fail(msg); + } finally { + clearInterval(verifyInterval); + + for (const chaosProvider of chaosProviders) { + await chaosProvider.stop(); + } + // Check for forks one last time, with all chaos turned off to ensure the check can succeed. + await workers.checkForks(); + } }); }); diff --git a/helpers/analyzer.ts b/helpers/analyzer.ts index 3c6914de5..b6a504cef 100644 --- a/helpers/analyzer.ts +++ b/helpers/analyzer.ts @@ -1,5 +1,6 @@ import fs from "fs"; import path from "path"; +import { forkDetectedString } from "forks/constants"; import { processLogFile, stripAnsi } from "./logger"; // Known test issues for tracking @@ -180,11 +181,6 @@ export async function cleanForksLogs( const logsDir = path.join(process.cwd(), "logs"); const outputDir = path.join(logsDir, "cleaned"); - if (!fs.existsSync(logsDir)) { - console.debug("No logs directory found"); - return; - } - if (!fs.existsSync(outputDir)) { await fs.promises.mkdir(outputDir, { recursive: true }); } @@ -214,7 +210,7 @@ export async function cleanForksLogs( // Check if the file contains fork-related content const containsForkContent = await fileContainsString( rawFilePath, - "may be fork", + forkDetectedString, ); // Always preserve raw logs for debugging/analysis diff --git a/helpers/logger.ts b/helpers/logger.ts index 6c2cbfcf3..8e9c7d8a6 100644 --- a/helpers/logger.ts +++ b/helpers/logger.ts @@ -2,6 +2,7 @@ import fs from "fs"; import path from "path"; import winston from "winston"; import "dotenv/config"; +import { forkDetectedString } from "forks/constants"; // Consolidated ANSI escape code regex // eslint-disable-next-line no-control-regex @@ -44,7 +45,7 @@ export async function processLogFile( let buffer = ""; let foundForkLine = false; - const targetString = "may be fork"; + const targetString = forkDetectedString; readStream.on("data", (chunk: string | Buffer) => { if (foundForkLine) { diff --git a/helpers/versions.ts b/helpers/versions.ts index 24cb174be..b2909e50d 100644 --- a/helpers/versions.ts +++ b/helpers/versions.ts @@ -24,12 +24,30 @@ import { Dm as Dm410, Group as Group410, } from "@xmtp/node-sdk-4.1.0"; +import { + Client as Client420, + Conversation as Conversation420, + Dm as Dm420, + Group as Group420, +} from "@xmtp/node-sdk-4.2.3"; import { Client as Client426, Conversation as Conversation426, Dm as Dm426, Group as Group426, } from "@xmtp/node-sdk-4.2.6"; +import { + Client as Client430, + Conversation as Conversation430, + Dm as Dm430, + Group as Group430, +} from "@xmtp/node-sdk-4.3.0"; +import { + Client as Client430Dev, + Conversation as Conversation430Dev, + Dm as Dm430Dev, + Group as Group430Dev, +} from "@xmtp/node-sdk-4.3.0-dev"; import { Client as Client440, Conversation as Conversation440, @@ -65,7 +83,7 @@ export { type PermissionLevel, type PermissionUpdateType, ConsentEntityType, -} from "@xmtp/node-sdk-4.2.6"; +} from "@xmtp/node-sdk-4.3.0-dev"; // Agent SDK version list export const AgentVersionList = [ @@ -87,6 +105,15 @@ export const AgentVersionList = [ // Node SDK version list export const VersionList = [ + { + Client: Client430Dev, + Conversation: Conversation430Dev, + Dm: Dm430Dev, + Group: Group430Dev, + nodeSDK: "4.3.0", + nodeBindings: "1.7.0", + auto: true, + }, { Client: Client440, Conversation: Conversation440, @@ -96,6 +123,15 @@ export const VersionList = [ nodeBindings: "1.6.1", auto: true, }, + { + Client: Client430, + Conversation: Conversation430, + Dm: Dm430, + Group: Group430, + nodeSDK: "4.3.0", + nodeBindings: "1.6.1", + auto: true, + }, { Client: Client426, Conversation: Conversation426, @@ -105,6 +141,15 @@ export const VersionList = [ nodeBindings: "1.5.4", auto: true, }, + { + Client: Client420, + Conversation: Conversation420, + Dm: Dm420, + Group: Group420, + nodeSDK: "4.2.3", + nodeBindings: "1.5.4", + auto: true, + }, { Client: Client410, Conversation: Conversation410, diff --git a/inboxes/byinstallation/2.json b/inboxes/byinstallation/2.json index cf9b23f88..0dd1a386a 100644 --- a/inboxes/byinstallation/2.json +++ b/inboxes/byinstallation/2.json @@ -3499,4 +3499,4 @@ "inboxId": "d8e582053edf2a98af37081f24d90f27e5acfa4a4eceb897b8d69e0a6d8900ea", "installations": 2 } -] +] \ No newline at end of file diff --git a/multinode/compose b/multinode/compose old mode 100644 new mode 100755 diff --git a/multinode/docker-compose.yml b/multinode/docker-compose.yml index d72854e3a..f5264de17 100644 --- a/multinode/docker-compose.yml +++ b/multinode/docker-compose.yml @@ -113,6 +113,12 @@ services: - mlsdb - validation + history-server: + image: ghcr.io/xmtp/message-history-server:main + platform: linux/amd64 + ports: + - 5558:5558 + validation: image: ghcr.io/xmtp/mls-validation-service:main platform: linux/amd64 diff --git a/multinode/down b/multinode/down old mode 100644 new mode 100755 diff --git a/multinode/up b/multinode/up old mode 100644 new mode 100755 diff --git a/network-stability/container.ts b/network-stability/container.ts index 2a842338c..acf369927 100644 --- a/network-stability/container.ts +++ b/network-stability/container.ts @@ -52,7 +52,7 @@ export class DockerContainer { } sh(cmd: string, expectFailure = false): string { - console.log(`[sh] Executing: ${cmd}`); + console.debug(`[sh] Executing: ${cmd}`); try { const output = execSync(cmd, { stdio: ["inherit", "pipe", "pipe"] }) .toString() @@ -60,7 +60,7 @@ export class DockerContainer { return output; } catch (e) { if (expectFailure) { - console.log(`[sh] Shell command failed as expected: ${cmd}`); + console.debug(`[sh] Shell command failed as expected: ${cmd}`); return ""; } @@ -76,7 +76,7 @@ export class DockerContainer { } ping(target: DockerContainer, count = 3, expectFailure = false): void { - console.log( + console.debug( `[sh] Pinging ${target.name} (${target.ip}) from ${this.name}...`, ); try { @@ -96,7 +96,7 @@ export class DockerContainer { } } catch (e) { if (expectFailure) { - console.log("[iptables] Ping failed as expected"); + console.debug("[iptables] Ping failed as expected"); } else { console.error( `[sh] Ping failed unexpectedly: ${e instanceof Error ? e.message : String(e)}`, diff --git a/network-stability/iptables.ts b/network-stability/iptables.ts index bfa64de7a..e23c58272 100644 --- a/network-stability/iptables.ts +++ b/network-stability/iptables.ts @@ -5,7 +5,7 @@ export function blockOutboundTraffic( from: DockerContainer, to: DockerContainer, ): void { - console.log(`[iptables] Blocking traffic from ${from.name} to ${to.name}`); + console.debug(`[iptables] Blocking traffic from ${from.name} to ${to.name}`); execSync( `sudo nsenter -t ${from.pid} -n iptables -A OUTPUT -d ${to.ip} -j DROP`, ); @@ -15,7 +15,9 @@ export function unblockOutboundTraffic( from: DockerContainer, to: DockerContainer, ): void { - console.log(`[iptables] Unblocking traffic from ${from.name} to ${to.name}`); + console.debug( + `[iptables] Unblocking traffic from ${from.name} to ${to.name}`, + ); try { execSync( `sudo nsenter -t ${from.pid} -n iptables -D OUTPUT -d ${to.ip} -j DROP`, @@ -31,7 +33,7 @@ export function blackHoleTo( target: DockerContainer, other: DockerContainer, ): void { - console.log( + console.debug( `[iptables] Blackholing traffic between ${target.name} and ${other.name}`, ); blockOutboundTraffic(target, other); @@ -42,7 +44,7 @@ export function unblockBlackHoleTo( target: DockerContainer, other: DockerContainer, ): void { - console.log( + console.debug( `[iptables] Removing blackhole between ${target.name} and ${other.name}`, ); unblockOutboundTraffic(target, other); diff --git a/network-stability/netem.ts b/network-stability/netem.ts index f9afa26bb..4bc988253 100644 --- a/network-stability/netem.ts +++ b/network-stability/netem.ts @@ -4,7 +4,6 @@ export function applyLatency( container: DockerContainer, latencyMs: number, ): void { - console.log(`[netem] Clearing existing qdisc on ${container.veth}`); container.sh(`sudo tc qdisc del dev ${container.veth} root`, true); container.sh( `sudo tc qdisc add dev ${container.veth} root netem delay ${latencyMs}ms`, @@ -30,7 +29,6 @@ export function applyLoss(container: DockerContainer, percent: number): void { } export function clear(container: DockerContainer): void { - console.log(`[netem] Clearing latency from ${container.veth}`); container.sh(`sudo tc qdisc del dev ${container.veth} root`, true); } diff --git a/package.json b/package.json index 2e85c5f53..94273c0db 100644 --- a/package.json +++ b/package.json @@ -41,9 +41,16 @@ "@xmtp/node-bindings-1.4.0": "npm:@xmtp/node-bindings@1.4.0", "@xmtp/node-bindings-1.5.4": "npm:@xmtp/node-bindings@1.5.4", "@xmtp/node-bindings-1.6.1": "npm:@xmtp/node-bindings@1.6.1-rc3", + "@xmtp/node-bindings-1.7.0-dev": "npm:@xmtp/node-bindings@1.7.0-dev.9bc470b", + "@xmtp/node-sdk-3.2.2": "npm:@xmtp/node-sdk@3.2.2", + "@xmtp/node-sdk-4.0.1": "npm:@xmtp/node-sdk@4.0.1", + "@xmtp/node-sdk-4.0.2": "npm:@xmtp/node-sdk@4.0.2", "@xmtp/node-sdk-4.0.3": "npm:@xmtp/node-sdk@4.0.3", "@xmtp/node-sdk-4.1.0": "npm:@xmtp/node-sdk@4.1.0", + "@xmtp/node-sdk-4.2.3": "npm:@xmtp/node-sdk@4.2.3", "@xmtp/node-sdk-4.2.6": "npm:@xmtp/node-sdk@4.2.6", + "@xmtp/node-sdk-4.3.0": "npm:@xmtp/node-sdk@4.3.0", + "@xmtp/node-sdk-4.3.0-dev": "npm:@xmtp/node-sdk@4.3.0-dev.395f798c", "@xmtp/node-sdk-4.4.0": "npm:@xmtp/node-sdk@4.4.0-rc2", "axios": "^1.8.2", "datadog-metrics": "^0.12.1", @@ -54,7 +61,8 @@ "uint8arrays": "^5.1.0", "viem": "^2", "vitest": "^3.2.4", - "winston": "^3.17.0" + "winston": "^3.17.0", + "yargs": "^18.0.0" }, "devDependencies": { "@anthropic-ai/claude-code": "latest", @@ -64,6 +72,7 @@ "@types/eslint__js": "^8.42.3", "@types/express": "^5", "@types/node": "^20.0.0", + "@types/yargs": "^17.0.35", "eslint": "^9.19.0", "eslint-config-prettier": "^10.0.1", "eslint-plugin-prettier": "^5.2.3", diff --git a/scripts/revokeInstallations.ts b/scripts/revokeInstallations.ts index 7eb602e37..daa4e4135 100644 --- a/scripts/revokeInstallations.ts +++ b/scripts/revokeInstallations.ts @@ -2,8 +2,7 @@ import { existsSync } from "node:fs"; import { readFile } from "node:fs/promises"; import { join } from "node:path"; import { createSigner } from "@helpers/client"; -import { type XmtpEnv } from "@helpers/versions"; -import { Client } from "@xmtp/node-sdk"; +import { Client, type XmtpEnv } from "@helpers/versions"; // Check Node.js version const nodeVersion = process.versions.node; diff --git a/tsconfig.json b/tsconfig.json index 8ba45e9ce..e2f6281a7 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -19,7 +19,10 @@ "target": "ESNext", "types": ["node", "vitest"], "paths": { + "@chaos/*": ["./chaos/*"], + "@forks/*": ["./forks/*"], "@helpers/*": ["./helpers/*"], + "@network-stability/*": ["./network-stability/*"], "@workers/*": ["./workers/*"], "@inboxes/*": ["./inboxes/*"], "@versions/*": ["./versions/*"], diff --git a/vitest.config.ts b/vitest.config.ts index f9085c6d2..dece408df 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -10,7 +10,10 @@ export default defineConfig({ resolve: { alias: { "@helpers": resolve(__dirname, "./helpers"), + "@chaos": resolve(__dirname, "./chaos"), + "@forks": resolve(__dirname, "./forks"), "@workers": resolve(__dirname, "./workers"), + "@network-stability": resolve(__dirname, "./network-stability"), "@scripts": resolve(__dirname, "./scripts"), "@inboxes": resolve(__dirname, "./inboxes"), "@bots": resolve(__dirname, "./agents/bots"), diff --git a/workers/main.ts b/workers/main.ts index fc3267ae3..639dbeadb 100644 --- a/workers/main.ts +++ b/workers/main.ts @@ -113,6 +113,9 @@ interface IWorkerClient { // Properties readonly currentFolder: string; + + // Clone Management + clone(): Promise; } // Worker thread code as a string @@ -146,17 +149,17 @@ parentPort.on("worker_message", (message: { type: string; data: any }) => { // Bootstrap code that loads the worker thread code const workerBootstrap = /* JavaScript */ ` import { parentPort, workerData } from "node:worker_threads"; - + // Execute the worker code const workerCode = ${JSON.stringify(workerThreadCode)}; const workerModule = new Function('require', 'parentPort', 'workerData', 'process', workerCode); - + // Get the require function import { createRequire } from "node:module"; import { fileURLToPath } from "node:url"; const __filename = fileURLToPath("${import.meta.url}"); const require = createRequire(__filename); - + // Execute the worker code workerModule(require, parentPort, workerData, process); `; @@ -1113,6 +1116,38 @@ export class WorkerClient extends Worker implements IWorkerClient { } } + /** + * Locks the database file for a specified duration to simulate chaos + * @param lockMs - Duration to hold the lock in milliseconds + * @returns Promise that resolves when the lock is released + */ + public async lockDB(lockMs: number): Promise { + return new Promise((resolve, reject) => { + try { + this.client.disconnectDatabase(); + + // Release the lock after the specified duration + setTimeout(async () => { + try { + await this.client.reconnectDatabase(); + console.log(`[${this.nameId}] Reconnected to the database`); + resolve(); + } catch (error: any) { + reject( + new Error( + `[${this.nameId}] Error releasing database lock: ${error}`, + ), + ); + } + }, lockMs); + } catch (error: any) { + reject( + new Error(`[${this.nameId}] Error disconnecting database: ${error}`), + ); + } + }); + } + /** * Revokes installations above a threshold count * @param threshold - Maximum number of installations allowed @@ -1222,4 +1257,42 @@ export class WorkerClient extends Worker implements IWorkerClient { installationId: newInstallationId, }; } + + /** + * Creates a clone of this worker client with a separate underlying client instance + * The clone will have the same configuration but a new name: ${original_worker_name}_clone + * @returns A new WorkerClient instance with separate client + */ + async clone(): Promise { + console.debug(`[${this.nameId}] Creating clone of worker`); + + // Create the clone name + const cloneName = `${this.name}_clone`; + + // Create a WorkerBase object with the same properties but new name + const cloneWorkerBase: WorkerBase = { + name: cloneName, + sdk: this.sdk, + folder: this.folder, + walletKey: this.walletKey, + encryptionKey: this.encryptionKeyHex, + }; + + // Create a new WorkerClient instance with the same configuration + const clonedWorker = new WorkerClient( + cloneWorkerBase, + this.env, + {}, // Use default worker options + this.apiUrl, + ); + + // Initialize the cloned worker to create its client instance + await clonedWorker.initialize(); + + console.debug( + `[${this.nameId}] Successfully created clone: ${clonedWorker.nameId}`, + ); + + return clonedWorker; + } } diff --git a/workers/manager.ts b/workers/manager.ts index dd7ca0ff0..e2890f3a3 100644 --- a/workers/manager.ts +++ b/workers/manager.ts @@ -12,6 +12,7 @@ import { type Group, type XmtpEnv, } from "@helpers/versions"; +import { forkDetectedString } from "forks/constants"; import { generatePrivateKey, privateKeyToAccount } from "viem/accounts"; import { installationThreshold, @@ -154,8 +155,10 @@ export class WorkerManager implements IWorkerManager { await Promise.all( groups.flat().map(async (g) => { const debugInfo = await g.debugInfo(); - if (debugInfo.maybeForked) { - throw new Error(`Stopping test, group id ${g.id} may have forked`); + if (debugInfo.maybeForked || debugInfo.isCommitLogForked) { + throw new Error( + `${forkDetectedString} Stopping test, group id ${g.id} may have forked`, + ); } }), ); @@ -179,8 +182,8 @@ export class WorkerManager implements IWorkerManager { for (const member of members) totalGroupInstallations += member.installationIds.length; - if (debugInfo.maybeForked) { - const logMessage = `Fork detected, group id ${groupId} may have forked, epoch ${debugInfo.epoch} for worker ${worker.name}`; + if (debugInfo.maybeForked || debugInfo.isCommitLogForked) { + const logMessage = `${forkDetectedString}. Group id ${groupId} may have forked, epoch ${debugInfo.epoch} for worker ${worker.name}`; console.error(logMessage); throw new Error(logMessage); } diff --git a/yarn.lock b/yarn.lock index 9b03848c1..0e233d5e8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -13,8 +13,8 @@ __metadata: linkType: hard "@anthropic-ai/claude-code@npm:latest": - version: 2.0.37 - resolution: "@anthropic-ai/claude-code@npm:2.0.37" + version: 2.0.42 + resolution: "@anthropic-ai/claude-code@npm:2.0.42" dependencies: "@img/sharp-darwin-arm64": "npm:^0.33.5" "@img/sharp-darwin-x64": "npm:^0.33.5" @@ -37,7 +37,7 @@ __metadata: optional: true bin: claude: cli.js - checksum: 10/ae2e913896fc5fc91977ede26477a7f9e5fcde59d5afd5b328ca23717a48a5b27134384efd4dc34bd1da9e9a70458506c84d0e5b308b0608e368cea109502af9 + checksum: 10/454bebe81d6637957066d1bb63182d2b29beaa79720eba1188a2b41bf758a19b65a9aeba9296162a586671b9182ac88e4a5de3b2e99eb6290c766b3949c50927 languageName: node linkType: hard @@ -161,8 +161,8 @@ __metadata: linkType: hard "@datadog/datadog-api-client@npm:^1.17.0": - version: 1.46.0 - resolution: "@datadog/datadog-api-client@npm:1.46.0" + version: 1.47.0 + resolution: "@datadog/datadog-api-client@npm:1.47.0" dependencies: "@types/buffer-from": "npm:^1.1.0" "@types/node": "npm:*" @@ -173,7 +173,7 @@ __metadata: form-data: "npm:^4.0.4" loglevel: "npm:^1.8.1" pako: "npm:^2.0.4" - checksum: 10/c5165548841a01277a812ef605d9c0586f05fd8bcafed23639bebd7571d44adfb86e331ae025f14f42c359138210eaec0e219190af35a0e33fe8e2561df90a51 + checksum: 10/765f55428b5b730b1d0fe666a245c44bed6083bb7d30c0669d87649cf4d7ff6461e864465aacdf1ee304b8b1e8746abdcc4eebe382d712a17338c199b256c81c languageName: node linkType: hard @@ -1253,6 +1253,22 @@ __metadata: languageName: node linkType: hard +"@types/yargs-parser@npm:*": + version: 21.0.3 + resolution: "@types/yargs-parser@npm:21.0.3" + checksum: 10/a794eb750e8ebc6273a51b12a0002de41343ffe46befef460bdbb57262d187fdf608bc6615b7b11c462c63c3ceb70abe2564c8dd8ee0f7628f38a314f74a9b9b + languageName: node + linkType: hard + +"@types/yargs@npm:^17.0.35": + version: 17.0.35 + resolution: "@types/yargs@npm:17.0.35" + dependencies: + "@types/yargs-parser": "npm:*" + checksum: 10/47bcd4476a4194ea11617ea71cba8a1eddf5505fc39c44336c1a08d452a0de4486aedbc13f47a017c8efbcb5a8aa358d976880663732ebcbc6dbcbbecadb0581 + languageName: node + linkType: hard + "@typescript-eslint/eslint-plugin@npm:8.46.4": version: 8.46.4 resolution: "@typescript-eslint/eslint-plugin@npm:8.46.4" @@ -1631,6 +1647,77 @@ __metadata: languageName: node linkType: hard +"@xmtp/node-bindings-1.7.0-dev@npm:@xmtp/node-bindings@1.7.0-dev.9bc470b, @xmtp/node-bindings@npm:1.7.0-dev.9bc470b": + version: 1.7.0-dev.9bc470b + resolution: "@xmtp/node-bindings@npm:1.7.0-dev.9bc470b" + checksum: 10/d0f3dfd15a65bc1ccb628271cba0310900217c72dfa0ee66c2405b6e2be052394bda2d9db029538ee9c842a4991c4b964a4887dabbad343314406b9b0a5e8c60 + languageName: node + linkType: hard + +"@xmtp/node-bindings@npm:1.3.3": + version: 1.3.3 + resolution: "@xmtp/node-bindings@npm:1.3.3" + checksum: 10/c9fecccc1adf6f79ac2cf5e884fa47b4157a8cc8121e04fbb24c9ada13869de78301c67670cb3e9450239659aaea25849af334ee6b11d761d1be5ad36bc0235b + languageName: node + linkType: hard + +"@xmtp/node-bindings@npm:1.3.4": + version: 1.3.4 + resolution: "@xmtp/node-bindings@npm:1.3.4" + checksum: 10/51a92c7a88567d666c4f3742998a29171a54962e9bf3028c91eeb58c95df2287a85ec70b7d5c007f19bf854cf498820d3dce68c006b074f6018ac9127256aae9 + languageName: node + linkType: hard + +"@xmtp/node-bindings@npm:1.3.5": + version: 1.3.5 + resolution: "@xmtp/node-bindings@npm:1.3.5" + checksum: 10/7cfdb95de5ec511961a2754c3aca3a96c68e6fababc754aa359f8b222cac20a86689010ccc28e4f5c175cbd87f7d3a01083289993fbe8e1ff8ffb4751b645508 + languageName: node + linkType: hard + +"@xmtp/node-bindings@npm:1.6.0-dev.35d2ff1": + version: 1.6.0-dev.35d2ff1 + resolution: "@xmtp/node-bindings@npm:1.6.0-dev.35d2ff1" + checksum: 10/8cfd99033695f422fc8973a65bb9554b4ecb05fe2b2558809e2d17f64bfcb159c831a93ce051e459a36478f0eaa52f0a92ae65805265a9b106f3e65f1d71d654 + languageName: node + linkType: hard + +"@xmtp/node-sdk-3.2.2@npm:@xmtp/node-sdk@3.2.2": + version: 3.2.2 + resolution: "@xmtp/node-sdk@npm:3.2.2" + dependencies: + "@xmtp/content-type-group-updated": "npm:^2.0.2" + "@xmtp/content-type-primitives": "npm:^2.0.2" + "@xmtp/content-type-text": "npm:^2.0.2" + "@xmtp/node-bindings": "npm:1.3.3" + checksum: 10/61c76de0c2e9871a754483b805dac1bdd9b8e16ae987987c3409dcb2b5afbb59ae31e0c4b2f0449924364725cbc744fbbfaaf3bf7a6473973011051183785f59 + languageName: node + linkType: hard + +"@xmtp/node-sdk-4.0.1@npm:@xmtp/node-sdk@4.0.1": + version: 4.0.1 + resolution: "@xmtp/node-sdk@npm:4.0.1" + dependencies: + "@xmtp/content-type-group-updated": "npm:^2.0.2" + "@xmtp/content-type-primitives": "npm:^2.0.2" + "@xmtp/content-type-text": "npm:^2.0.2" + "@xmtp/node-bindings": "npm:1.3.4" + checksum: 10/d15de6227727553bfa3be85da6f4969b0364146b8f1a450d305d2d7e118b143d90f509c8ad8dd2b755df5d1473fc8c05b6c96f9b82d424099deca0cd5a324d2a + languageName: node + linkType: hard + +"@xmtp/node-sdk-4.0.2@npm:@xmtp/node-sdk@4.0.2": + version: 4.0.2 + resolution: "@xmtp/node-sdk@npm:4.0.2" + dependencies: + "@xmtp/content-type-group-updated": "npm:^2.0.2" + "@xmtp/content-type-primitives": "npm:^2.0.2" + "@xmtp/content-type-text": "npm:^2.0.2" + "@xmtp/node-bindings": "npm:1.3.5" + checksum: 10/3dc87a58263c5d7a0fc454d57b00fc4d94b32bc297a4f30be7846553357256390504a997081e4d8052fd997636e149fbec57d21144fafaafebcaf4ea14f8fba8 + languageName: node + linkType: hard + "@xmtp/node-sdk-4.0.3@npm:@xmtp/node-sdk@4.0.3": version: 4.0.3 resolution: "@xmtp/node-sdk@npm:4.0.3" @@ -1655,6 +1742,18 @@ __metadata: languageName: node linkType: hard +"@xmtp/node-sdk-4.2.3@npm:@xmtp/node-sdk@4.2.3": + version: 4.2.3 + resolution: "@xmtp/node-sdk@npm:4.2.3" + dependencies: + "@xmtp/content-type-group-updated": "npm:^2.0.2" + "@xmtp/content-type-primitives": "npm:^2.0.2" + "@xmtp/content-type-text": "npm:^2.0.2" + "@xmtp/node-bindings": "npm:1.5.4" + checksum: 10/0307d9da6f8a2faab0d7afe7f6f38996c1011cf4053e45aa3cf8ede5d8521b729fd7b023c9ebb686f2442db0f3d8a1e7b74ef2540350358fa7b9e42faa8ac6c1 + languageName: node + linkType: hard + "@xmtp/node-sdk-4.2.6@npm:@xmtp/node-sdk@4.2.6": version: 4.2.6 resolution: "@xmtp/node-sdk@npm:4.2.6" @@ -1667,6 +1766,30 @@ __metadata: languageName: node linkType: hard +"@xmtp/node-sdk-4.3.0-dev@npm:@xmtp/node-sdk@4.3.0-dev.395f798c": + version: 4.3.0-dev.395f798c + resolution: "@xmtp/node-sdk@npm:4.3.0-dev.395f798c" + dependencies: + "@xmtp/content-type-group-updated": "npm:^2.0.2" + "@xmtp/content-type-primitives": "npm:^2.0.2" + "@xmtp/content-type-text": "npm:^2.0.2" + "@xmtp/node-bindings": "npm:1.7.0-dev.9bc470b" + checksum: 10/5aa30fcb240e91c416dafe1eee3b1fd692765460100907da95f689749acd5455a8a511f1e8814002aa2a65a6eec64099d0e8ff0a7afe10e181318e245faf1515 + languageName: node + linkType: hard + +"@xmtp/node-sdk-4.3.0@npm:@xmtp/node-sdk@4.3.0": + version: 4.3.0 + resolution: "@xmtp/node-sdk@npm:4.3.0" + dependencies: + "@xmtp/content-type-group-updated": "npm:^2.0.2" + "@xmtp/content-type-primitives": "npm:^2.0.2" + "@xmtp/content-type-text": "npm:^2.0.2" + "@xmtp/node-bindings": "npm:1.6.0-dev.35d2ff1" + checksum: 10/c8cc0b6b0cd3efbb72d3d7799fea0bef409b8a48a63ffbb3518c22ab27c96eb5b4c0cedd63c2e8c8a3805e8f705a5b9d2bf35d1be1a54c43313d7af9aef6da02 + languageName: node + linkType: hard + "@xmtp/node-sdk-4.4.0@npm:@xmtp/node-sdk@4.4.0-rc2": version: 4.4.0-rc2 resolution: "@xmtp/node-sdk@npm:4.4.0-rc2" @@ -1810,7 +1933,7 @@ __metadata: languageName: node linkType: hard -"ansi-styles@npm:^6.1.0": +"ansi-styles@npm:^6.1.0, ansi-styles@npm:^6.2.1": version: 6.2.3 resolution: "ansi-styles@npm:6.2.3" checksum: 10/c49dad7639f3e48859bd51824c93b9eb0db628afc243c51c3dd2410c4a15ede1a83881c6c7341aa2b159c4f90c11befb38f2ba848c07c66c9f9de4bcd7cb9f30 @@ -2026,6 +2149,17 @@ __metadata: languageName: node linkType: hard +"cliui@npm:^9.0.1": + version: 9.0.1 + resolution: "cliui@npm:9.0.1" + dependencies: + string-width: "npm:^7.2.0" + strip-ansi: "npm:^7.1.0" + wrap-ansi: "npm:^9.0.0" + checksum: 10/df43d8d1c6e3254cbb64b1905310d5f6672c595496a3cbe76946c6d24777136886470686f2772ac9edfe547a74bb70e8017530b3554715aee119efd7752fc0d9 + languageName: node + linkType: hard + "color-convert@npm:^2.0.1": version: 2.0.1 resolution: "color-convert@npm:2.0.1" @@ -2035,12 +2169,12 @@ __metadata: languageName: node linkType: hard -"color-convert@npm:^3.0.1": - version: 3.1.2 - resolution: "color-convert@npm:3.1.2" +"color-convert@npm:^3.1.3": + version: 3.1.3 + resolution: "color-convert@npm:3.1.3" dependencies: color-name: "npm:^2.0.0" - checksum: 10/6731d4d4e1427b0df6aa5655a7e97f52962c35070295347f1899857dfaa77477f72bdf4369422d6c345888595bb33d31620387b73d88ffc0b7cf1e496b54fc58 + checksum: 10/36b9b99c138f90eb11a28d1ad911054a9facd6cffde4f00dc49a34ebde7cae28454b2285ede64f273b6a8df9c3228b80e4352f4471978fa8b5005fe91341a67b languageName: node linkType: hard @@ -2058,22 +2192,22 @@ __metadata: languageName: node linkType: hard -"color-string@npm:^2.0.0": - version: 2.1.2 - resolution: "color-string@npm:2.1.2" +"color-string@npm:^2.1.3": + version: 2.1.3 + resolution: "color-string@npm:2.1.3" dependencies: color-name: "npm:^2.0.0" - checksum: 10/9f2fdf5a29ff3d6ac29ff320e27904b13bf019cec8619483bc0742006adedf66a76461b1fa349d39263142c09cd9364bc4414c1ea68a31446c1b63580e38caf6 + checksum: 10/e5a5725f84a8db85f488754cdf6b77a8404b53753955ec5503644cec3b6d514a18ee4f804b7121c93874420835faa44f8b61ea1c8abb46b38d9599e051d6dfc4 languageName: node linkType: hard "color@npm:^5.0.2": - version: 5.0.2 - resolution: "color@npm:5.0.2" + version: 5.0.3 + resolution: "color@npm:5.0.3" dependencies: - color-convert: "npm:^3.0.1" - color-string: "npm:^2.0.0" - checksum: 10/2574e20a8ae5e66ead1ea5ad3a9f391de044e0fc7adbecc2bf9cbc42838512e75b9de5d8ad183299a9968da807a2221bfc4dc2337c298bf6f4710e7be9eefd30 + color-convert: "npm:^3.1.3" + color-string: "npm:^2.1.3" + checksum: 10/88063ee058b995e5738092b5aa58888666275d1e967333f3814ff4fa334ce9a9e71de78a16fb1838f17c80793ea87f4878c20192037662809fe14eab2d474fd9 languageName: node linkType: hard @@ -2246,6 +2380,13 @@ __metadata: languageName: node linkType: hard +"emoji-regex@npm:^10.3.0": + version: 10.6.0 + resolution: "emoji-regex@npm:10.6.0" + checksum: 10/98cc0b0e1daed1ed25afbf69dcb921fee00f712f51aab93aa1547e4e4e8171725cc4f0098aaa645b4f611a19da11ec9f4623eb6ff2b72314b39a8f2ae7c12bf2 + languageName: node + linkType: hard + "emoji-regex@npm:^8.0.0": version: 8.0.0 resolution: "emoji-regex@npm:8.0.0" @@ -2435,6 +2576,13 @@ __metadata: languageName: node linkType: hard +"escalade@npm:^3.1.1": + version: 3.2.0 + resolution: "escalade@npm:3.2.0" + checksum: 10/9d7169e3965b2f9ae46971afa392f6e5a25545ea30f2e2dd99c9b0a95a3f52b5653681a84f5b2911a413ddad2d7a93d3514165072f349b5ffc59c75a899970d6 + languageName: node + linkType: hard + "escape-html@npm:^1.0.3": version: 1.0.3 resolution: "escape-html@npm:1.0.3" @@ -2911,6 +3059,20 @@ __metadata: languageName: node linkType: hard +"get-caller-file@npm:^2.0.5": + version: 2.0.5 + resolution: "get-caller-file@npm:2.0.5" + checksum: 10/b9769a836d2a98c3ee734a88ba712e62703f1df31b94b784762c433c27a386dd6029ff55c2a920c392e33657d80191edbf18c61487e198844844516f843496b9 + languageName: node + linkType: hard + +"get-east-asian-width@npm:^1.0.0": + version: 1.4.0 + resolution: "get-east-asian-width@npm:1.4.0" + checksum: 10/c9ae85bfc2feaf4cc71cdb236e60f1757ae82281964c206c6aa89a25f1987d326ddd8b0de9f9ccd56e37711b9fcd988f7f5137118b49b0b45e19df93c3be8f45 + languageName: node + linkType: hard + "get-intrinsic@npm:^1.2.5, get-intrinsic@npm:^1.2.6, get-intrinsic@npm:^1.3.0": version: 1.3.1 resolution: "get-intrinsic@npm:1.3.1" @@ -4492,6 +4654,17 @@ __metadata: languageName: node linkType: hard +"string-width@npm:^7.0.0, string-width@npm:^7.2.0": + version: 7.2.0 + resolution: "string-width@npm:7.2.0" + dependencies: + emoji-regex: "npm:^10.3.0" + get-east-asian-width: "npm:^1.0.0" + strip-ansi: "npm:^7.1.0" + checksum: 10/42f9e82f61314904a81393f6ef75b832c39f39761797250de68c041d8ba4df2ef80db49ab6cd3a292923a6f0f409b8c9980d120f7d32c820b4a8a84a2598a295 + languageName: node + linkType: hard + "string_decoder@npm:^1.1.1": version: 1.3.0 resolution: "string_decoder@npm:1.3.0" @@ -4510,7 +4683,7 @@ __metadata: languageName: node linkType: hard -"strip-ansi@npm:^7.0.1": +"strip-ansi@npm:^7.0.1, strip-ansi@npm:^7.1.0": version: 7.1.2 resolution: "strip-ansi@npm:7.1.2" dependencies: @@ -5079,6 +5252,17 @@ __metadata: languageName: node linkType: hard +"wrap-ansi@npm:^9.0.0": + version: 9.0.2 + resolution: "wrap-ansi@npm:9.0.2" + dependencies: + ansi-styles: "npm:^6.2.1" + string-width: "npm:^7.0.0" + strip-ansi: "npm:^7.1.0" + checksum: 10/f3907e1ea9717404ca53a338fa5a017c2121550c3a5305180e2bc08c03e21aa45068df55b0d7676bf57be1880ba51a84458c17241ebedea485fafa9ef16b4024 + languageName: node + linkType: hard + "wrappy@npm:1": version: 1.0.2 resolution: "wrappy@npm:1.0.2" @@ -5113,6 +5297,7 @@ __metadata: "@types/eslint__js": "npm:^8.42.3" "@types/express": "npm:^5" "@types/node": "npm:^20.0.0" + "@types/yargs": "npm:^17.0.35" "@xmtp/agent-sdk-1.1.14": "npm:@xmtp/agent-sdk@1.1.14" "@xmtp/agent-sdk-1.1.7": "npm:@xmtp/agent-sdk@1.1.12" "@xmtp/content-type-markdown": "npm:^1.0.0" @@ -5123,9 +5308,16 @@ __metadata: "@xmtp/node-bindings-1.4.0": "npm:@xmtp/node-bindings@1.4.0" "@xmtp/node-bindings-1.5.4": "npm:@xmtp/node-bindings@1.5.4" "@xmtp/node-bindings-1.6.1": "npm:@xmtp/node-bindings@1.6.1-rc3" + "@xmtp/node-bindings-1.7.0-dev": "npm:@xmtp/node-bindings@1.7.0-dev.9bc470b" + "@xmtp/node-sdk-3.2.2": "npm:@xmtp/node-sdk@3.2.2" + "@xmtp/node-sdk-4.0.1": "npm:@xmtp/node-sdk@4.0.1" + "@xmtp/node-sdk-4.0.2": "npm:@xmtp/node-sdk@4.0.2" "@xmtp/node-sdk-4.0.3": "npm:@xmtp/node-sdk@4.0.3" "@xmtp/node-sdk-4.1.0": "npm:@xmtp/node-sdk@4.1.0" + "@xmtp/node-sdk-4.2.3": "npm:@xmtp/node-sdk@4.2.3" "@xmtp/node-sdk-4.2.6": "npm:@xmtp/node-sdk@4.2.6" + "@xmtp/node-sdk-4.3.0": "npm:@xmtp/node-sdk@4.3.0" + "@xmtp/node-sdk-4.3.0-dev": "npm:@xmtp/node-sdk@4.3.0-dev.395f798c" "@xmtp/node-sdk-4.4.0": "npm:@xmtp/node-sdk@4.4.0-rc2" axios: "npm:^1.8.2" datadog-metrics: "npm:^0.12.1" @@ -5147,9 +5339,17 @@ __metadata: viem: "npm:^2" vitest: "npm:^3.2.4" winston: "npm:^3.17.0" + yargs: "npm:^18.0.0" languageName: unknown linkType: soft +"y18n@npm:^5.0.5": + version: 5.0.8 + resolution: "y18n@npm:5.0.8" + checksum: 10/5f1b5f95e3775de4514edbb142398a2c37849ccfaf04a015be5d75521e9629d3be29bd4432d23c57f37e5b61ade592fb0197022e9993f81a06a5afbdcda9346d + languageName: node + linkType: hard + "yallist@npm:^4.0.0": version: 4.0.0 resolution: "yallist@npm:4.0.0" @@ -5164,6 +5364,27 @@ __metadata: languageName: node linkType: hard +"yargs-parser@npm:^22.0.0": + version: 22.0.0 + resolution: "yargs-parser@npm:22.0.0" + checksum: 10/f13c42bad6ebed1a587a72f2db5694f5fa772bcaf409a701691d13cf74eb5adfcf61a2611de08807e319b829d3e5e6e1578b16ebe174cae8e8be3bf7b8e7a19e + languageName: node + linkType: hard + +"yargs@npm:^18.0.0": + version: 18.0.0 + resolution: "yargs@npm:18.0.0" + dependencies: + cliui: "npm:^9.0.1" + escalade: "npm:^3.1.1" + get-caller-file: "npm:^2.0.5" + string-width: "npm:^7.2.0" + y18n: "npm:^5.0.5" + yargs-parser: "npm:^22.0.0" + checksum: 10/5af36234871390386b31cac99f00e79fcbc2ead858a61b30a8ca381c5fde5df8af0b407c36b000d3f774bcbe4aec5833f2f1c915f6ddc49ce97b78176b651801 + languageName: node + linkType: hard + "yocto-queue@npm:^0.1.0": version: 0.1.0 resolution: "yocto-queue@npm:0.1.0"