diff --git a/crates/cairo-program-runner-lib/Cargo.toml b/crates/cairo-program-runner-lib/Cargo.toml index ab026d37..2b6e59c8 100644 --- a/crates/cairo-program-runner-lib/Cargo.toml +++ b/crates/cairo-program-runner-lib/Cargo.toml @@ -3,6 +3,13 @@ name = "cairo-program-runner-lib" version = "0.1.0" edition = "2021" +[[bin]] +name = "benchmark_runner" +path = "scripts/benchmark_runner.rs" + +[[bin]] +name = "benchmark_run_and_prove" +path = "scripts/benchmark_run_and_prove.rs" [dependencies] bincode.workspace = true diff --git a/crates/cairo-program-runner-lib/scripts/benchmark_run_and_prove.rs b/crates/cairo-program-runner-lib/scripts/benchmark_run_and_prove.rs new file mode 100644 index 00000000..8d62070c --- /dev/null +++ b/crates/cairo-program-runner-lib/scripts/benchmark_run_and_prove.rs @@ -0,0 +1,149 @@ +// benchmark_run_and_prove.rs +// Usage: cargo run --bin benchmark_run_and_prove [--show-output] +// +// This program benchmarks proving a Cairo PIE file using the simple bootloader as the program +// and the PIE as the program input. It prints load/prove times and proof status. + +use std::error::Error; +use std::fs::File; +use std::io::Write; +use std::path::PathBuf; +use std::process::Command; +use std::time::Instant; + +fn main() -> Result<(), Box> { + let project_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let simple_bootloader_compiled_path = + project_dir.join("resources/compiled_programs/bootloaders/simple_bootloader_compiled.json"); + + // Print and check for simple_bootloader_compiled.json + println!( + "Looking for simple_bootloader_compiled.json at: {:?}", + simple_bootloader_compiled_path + ); + if !simple_bootloader_compiled_path.exists() { + eprintln!( + "ERROR: File does not exist: {:?}", + simple_bootloader_compiled_path + ); + } + + // Parse command line args + let args: Vec = std::env::args().collect(); + let mut pie_path = None; + let mut show_output = false; + for (i, arg) in args.iter().enumerate() { + if arg == "--show-output" { + show_output = true; + } else if i > 0 && !arg.starts_with("--") { + pie_path = Some(arg.clone()); + } + } + let pie_path = match pie_path { + Some(p) => p, + None => { + eprintln!("Usage: {} [--show-output]", args[0]); + std::process::exit(1); + } + }; + + println!("Proving PIE: {}", pie_path); + let start = Instant::now(); + + // Build the correct program_input JSON for a PIE file + let program_input_json = format!( + r#"{{ + "tasks": [ + {{ + "type": "CairoPiePath", + "path": "{}", + "program_hash_function": "blake" + }} + ], + "single_page": true + }}"#, + pie_path + ); + // Write the JSON to a temporary file + let tmp_input_path = "temp_pie_input.json"; + let mut tmp_file = File::create(tmp_input_path)?; + tmp_file.write_all(program_input_json.as_bytes())?; + + // Get absolute path to prover_params.json (use the one in the same directory as this script) + let prover_params_path = project_dir.join("scripts/prover_params.json"); + println!( + "Looking for prover_params.json at: {}", + prover_params_path.display() + ); + if !prover_params_path.exists() { + eprintln!( + "ERROR: File does not exist: {}", + prover_params_path.display() + ); + } + + // Use the full path to the stwo_run_and_prove binary in target/release (fix path: remove + // 'crates/') + let stwo_bin_path = project_dir + .parent() + .unwrap() + .parent() + .unwrap() + .join("target/release/stwo_run_and_prove"); + println!( + "Using stwo_run_and_prove binary at: {}", + stwo_bin_path.display() + ); + if !stwo_bin_path.exists() { + eprintln!( + "ERROR: stwo_run_and_prove binary not found at {}", + stwo_bin_path.display() + ); + std::process::exit(1); + } + + // Print the command being run for debugging (for display only) + println!( + "Running command: {} --program=\"{}\" --program_input=\"{}\" --proofs_dir=temp_proofs --proof-format=cairo-serde --verify --prover_params_json=\"{}\" --program_output=temp_outputs", + stwo_bin_path.display(), + simple_bootloader_compiled_path.display(), + tmp_input_path, + prover_params_path.display() + ); + + // Call stwo_run_and_prove binary with the bootloader as program and the PIE as program_input + let output = Command::new(&stwo_bin_path) + .arg(format!( + "--program={}", + simple_bootloader_compiled_path.display() + )) + .arg(format!("--program_input={}", tmp_input_path)) + .arg("--proofs_dir=temp_proofs") + .arg("--proof-format=cairo-serde") + .arg("--verify") + .arg(format!( + "--prover_params_json={}", + prover_params_path.display() + )) + .arg("--program_output=temp_outputs") + .output()?; + let elapsed = start.elapsed(); + if output.status.success() { + println!( + "SUCCESS: Proved {} in {:.3} seconds", + pie_path, + elapsed.as_secs_f64() + ); + if show_output { + println!("stdout:\n{}", String::from_utf8_lossy(&output.stdout)); + } + } else { + println!( + "ERROR: Failed to prove {} (exit code {:?})", + pie_path, + output.status.code() + ); + println!("stderr:\n{}", String::from_utf8_lossy(&output.stderr)); + } + Ok(()) +} diff --git a/crates/cairo-program-runner-lib/scripts/benchmark_runner.rs b/crates/cairo-program-runner-lib/scripts/benchmark_runner.rs new file mode 100644 index 00000000..fc08fc5c --- /dev/null +++ b/crates/cairo-program-runner-lib/scripts/benchmark_runner.rs @@ -0,0 +1,161 @@ +//! Benchmark Runner for Cairo PIE Files +//! +//! This program measures the performance of loading and executing Cairo PIEs +//! wrapped in simple bootloader and ran by the Cairo Program Runner. +//! +//! Usage: +//! cargo run --bin benchmark_runner [--show-output] +//! +//! Arguments: +//! Path to the Cairo PIE file (.zip) to benchmark +//! --show-output Optional flag to display program output +//! +//! Output: +//! - Program load time +//! - Program execution time +//! - Total runtime +//! - Execution resources (steps, memory holes, builtin counters) +//! +//! This binary is typically run through the run_all_pies.sh script, which provides +//! additional features: +//! - Running multiple PIE files +//! - Testing different Rust optimization levels +//! - Aggregating results into a single file +//! +//! Example: +//! Direct usage: +//! cargo run --bin benchmark_runner path/to/program.zip +//! Via script: +//! ./scripts/run_all_pies.sh --opt-levels=3 --pie-dir=path/to/pies + +use std::error::Error; +use std::path::PathBuf; +use std::time::Instant; + +use cairo_program_runner_lib::cairo_run_program; +use cairo_program_runner_lib::types::RunMode; +use cairo_vm::types::layout_name::LayoutName; +use cairo_vm::types::program::Program; +use cairo_vm::vm::runners::cairo_pie::CairoPie; + +fn main() -> Result<(), Box> { + let project_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let simple_bootloader_compiled_path = + project_dir.join("resources/compiled_programs/bootloaders/simple_bootloader_compiled.json"); + + // Parse command line args + let args: Vec = std::env::args().collect(); + let mut program_path = String::new(); + let mut show_output = false; + + // Simple arg parsing + for (i, arg) in args.iter().enumerate() { + if arg == "--show-output" { + show_output = true; + } else if i > 0 && !arg.starts_with("--") { + program_path = arg.clone(); + } + } + + if program_path.is_empty() { + program_path = project_dir + .join("examples/fibonacci.json") + .display() + .to_string(); + } + + println!("Loading program from: {}", program_path); + + // Try to load as a PIE first, if that fails, try as a regular program + let start_load = Instant::now(); + let program_input_contents = if program_path.ends_with(".zip") { + // Load as PIE - we just need to verify it loads + // TODO(idanh): Add an option to configure the hash function + let _cairo_pie = CairoPie::read_zip_file(&PathBuf::from(&program_path))?; + format!( + r#"{{ + "tasks": [ + {{ + "type": "CairoPiePath", + "path": "{}", + "program_hash_function": "pedersen" + }} + ], + "single_page": true + }}"#, + program_path + ) + } else { + // Load as regular program - we just need to verify it loads + let _program = Program::from_file(PathBuf::from(&program_path).as_path(), Some("main"))?; + format!( + r#"{{ + "tasks": [ + {{ + "path": "{}", + "program_hash_function": "pedersen", + "type": "RunProgramTask" + }} + ], + "single_page": true + }}"#, + program_path + ) + }; + let load_duration = start_load.elapsed(); + println!("Program loaded in: {:?}", load_duration); + + let simple_bootloader_program = + Program::from_file(simple_bootloader_compiled_path.as_path(), Some("main"))?; + + let cairo_run_config = RunMode::Proof { + layout: LayoutName::starknet_with_keccak, + dynamic_layout_params: None, + disable_trace_padding: false, + } + .create_config(); + + println!("\nExecuting program..."); + let start_exec = Instant::now(); + let mut runner = cairo_run_program( + &simple_bootloader_program, + Some(program_input_contents), + cairo_run_config, + )?; + let exec_duration = start_exec.elapsed(); + println!("Program executed in: {:?}", exec_duration); + + // Verify execution by checking outputs + let mut output_buffer = String::new(); + runner.vm.write_output(&mut output_buffer)?; + + // Print execution verification info + println!("\nProgram Execution Verification:"); + println!("Output size: {} bytes", output_buffer.len()); + + if show_output { + println!("Program output:\n{}", output_buffer); + } + + // Print execution resources + println!("\nExecution Resources:"); + let resources = runner.get_execution_resources()?; + println!("n_steps: {}", resources.n_steps); + println!("n_memory_holes: {}", resources.n_memory_holes); + println!( + "builtin_instance_counter: {:#?}", + resources.builtin_instance_counter + ); + + // Check if output is empty - might indicate optimization skip + if output_buffer.trim().is_empty() { + println!("WARNING: Empty output might indicate optimization issues!"); + } + + println!("\nBenchmark Summary:"); + println!("Program load time: {:?}", load_duration); + println!("Program execution time: {:?}", exec_duration); + println!("Total time: {:?}", load_duration + exec_duration); + + Ok(()) +} diff --git a/crates/cairo-program-runner-lib/scripts/pies/POTC-PIE.zip b/crates/cairo-program-runner-lib/scripts/pies/POTC-PIE.zip new file mode 100644 index 00000000..8d3efff4 Binary files /dev/null and b/crates/cairo-program-runner-lib/scripts/pies/POTC-PIE.zip differ diff --git a/crates/cairo-program-runner-lib/scripts/pies/SN-large-3877 b/crates/cairo-program-runner-lib/scripts/pies/SN-large-3877 new file mode 100644 index 00000000..9258086e Binary files /dev/null and b/crates/cairo-program-runner-lib/scripts/pies/SN-large-3877 differ diff --git a/crates/cairo-program-runner-lib/scripts/pies/SN-medium-4300 b/crates/cairo-program-runner-lib/scripts/pies/SN-medium-4300 new file mode 100644 index 00000000..3a5a5226 Binary files /dev/null and b/crates/cairo-program-runner-lib/scripts/pies/SN-medium-4300 differ diff --git a/crates/cairo-program-runner-lib/scripts/pies/SN-small-4274 b/crates/cairo-program-runner-lib/scripts/pies/SN-small-4274 new file mode 100644 index 00000000..4e0cd5d8 Binary files /dev/null and b/crates/cairo-program-runner-lib/scripts/pies/SN-small-4274 differ diff --git a/crates/cairo-program-runner-lib/scripts/pies/Starknet1-PIE.zip b/crates/cairo-program-runner-lib/scripts/pies/Starknet1-PIE.zip new file mode 100644 index 00000000..ca87e904 Binary files /dev/null and b/crates/cairo-program-runner-lib/scripts/pies/Starknet1-PIE.zip differ diff --git a/crates/cairo-program-runner-lib/scripts/pies/output_file_6_5_3.zip b/crates/cairo-program-runner-lib/scripts/pies/output_file_6_5_3.zip new file mode 100644 index 00000000..1a11787a Binary files /dev/null and b/crates/cairo-program-runner-lib/scripts/pies/output_file_6_5_3.zip differ diff --git a/crates/cairo-program-runner-lib/scripts/prover_params.json b/crates/cairo-program-runner-lib/scripts/prover_params.json new file mode 100644 index 00000000..d637dfe7 --- /dev/null +++ b/crates/cairo-program-runner-lib/scripts/prover_params.json @@ -0,0 +1,12 @@ +{ + "channel_hash": "blake2s", + "pcs_config": { + "pow_bits": 26, + "fri_config": { + "log_blowup_factor": 1, + "log_last_layer_degree_bound": 0, + "n_queries": 70 + } + }, + "preprocessed_trace": "canonical" +} \ No newline at end of file diff --git a/crates/cairo-program-runner-lib/scripts/run_all_pies.sh b/crates/cairo-program-runner-lib/scripts/run_all_pies.sh new file mode 100755 index 00000000..c3826c88 --- /dev/null +++ b/crates/cairo-program-runner-lib/scripts/run_all_pies.sh @@ -0,0 +1,103 @@ +#!/bin/bash + +# Script Description: +# This script runs benchmarks of the program runner on the simple bootloader wrapping +# Cairo PIE files with different Rust optimization levels. +# It measures program load time, execution time, and execution resources of the run. +# Results are saved to a file. +# +# Usage: +# ./run_all_pies.sh [--opt-levels=] [--pie-dir=path/to/pies] +# +# Options: +# --opt-levels= Specify which Rust optimization levels to run (0-3) +# Examples: --opt-levels=3 (default, runs only level 3) +# --opt-levels=23 (runs levels 2 and 3) +# --opt-levels=013 (runs levels 0, 1, and 3) +# --pie-dir= Path to directory containing PIE files +# Default: ./pies +# +# Output: +# - Shows compilation and execution progress in the terminal +# - Saves detailed benchmark results to benchmark_results.txt in the scripts directory +# - For each PIE file and optimization level, captures: +# * Program load and execution times +# * Number of steps and memory holes +# * Detailed builtin instance counter information +# +# Examples: +# ./run_all_pies.sh # Run with default settings (opt-level 3) +# ./run_all_pies.sh --opt-levels=012 # Run with optimization levels 0, 1, and 2 +# ./run_all_pies.sh --pie-dir=/path/to/pies # Use PIE files from custom directory + +# Default values +SCRIPT_DIR="$(dirname "$0")" +PIES_DIR="$SCRIPT_DIR/pies" +RESULTS_FILE="$SCRIPT_DIR/benchmark_results.txt" +OPT_LEVELS="3" # Default to only opt level 3 + +# Parse command line arguments +while [ $# -gt 0 ]; do + case "$1" in + --opt-levels=*) + OPT_LEVELS="${1#*=}" + ;; + --pie-dir=*) + PIES_DIR="${1#*=}" + ;; + *) + echo "Unknown parameter: $1" + echo "Usage: $0 [--opt-levels=] [--pie-dir=path/to/pies]" + echo " --opt-levels: Specify which optimization levels to run (0-3)" + echo " Examples: --opt-levels=23 (runs levels 2 and 3)" + echo " --opt-levels=013 (runs levels 0, 1, and 3)" + echo " --pie-dir: Path to directory containing PIE files" + exit 1 + ;; + esac + shift +done + +# Clear previous results +echo "Cairo PIE Benchmark Results" > "$RESULTS_FILE" +date >> "$RESULTS_FILE" +echo "----------------------------------------" >> "$RESULTS_FILE" + +# Function to run cargo with specific opt level +run_with_opt_level() { + local pie_file="$1" + local opt_level="$2" + + echo "Running ${pie_file} with optimization level ${opt_level}" + echo "Running ${pie_file} with optimization level ${opt_level}" >> "$RESULTS_FILE" + + # Run cargo and show compilation/execution stages, but redirect only benchmark results and resources to file + if ! RUSTFLAGS="-C opt-level=${opt_level}" cargo run --bin benchmark_runner "${pie_file}" \ + 2> >(grep -v "future version of Rust\|future-incompat-report" >&2) | \ + tee >(awk '!/Program (load|execution) time:/ && (/Program (loaded|executed)/ || /Execution Resources:/ || /^n_steps:/ || /^n_memory_holes:/ || /^builtin_instance_counter:/ || /^[[:space:]]*[a-z_]+:/ || /^}/ || /Total time:/)' >> "$RESULTS_FILE"); then + echo "Error running benchmark" >> "$RESULTS_FILE" + fi + echo "----------------------------------------" >> "$RESULTS_FILE" +} + +# Find all .zip files in pies directory +for pie_file in "$PIES_DIR"/*.zip; do + if [ -f "$pie_file" ]; then + echo "=== Testing $(basename "$pie_file") ===" + echo + + # Run with specified optimization levels + # Validate opt levels + for opt_level in $(echo "$OPT_LEVELS" | grep -o .); do + if [[ ! "$opt_level" =~ ^[0-3]$ ]]; then + echo "Error: Invalid optimization level '$opt_level'. Must be one of: 0, 1, 2, 3" + exit 1 + fi + run_with_opt_level "$pie_file" "$opt_level" + echo + done + + echo "================================================" + echo + fi +done diff --git a/crates/cairo-program-runner-lib/scripts/run_and_prove_all_pies.sh b/crates/cairo-program-runner-lib/scripts/run_and_prove_all_pies.sh new file mode 100755 index 00000000..a68d90a3 --- /dev/null +++ b/crates/cairo-program-runner-lib/scripts/run_and_prove_all_pies.sh @@ -0,0 +1,53 @@ +#!/bin/bash +# run_and_prove_all_pies.sh +# Runs benchmark_run_and_prove.rs on all PIE files in the pies directory. +# Usage: ./run_and_prove_all_pies.sh [--pie-dir=path/to/pies] + +SCRIPT_DIR="$(dirname "$0")" +PIES_DIR="$SCRIPT_DIR/pies" +RESULTS_FILE="$SCRIPT_DIR/run_and_prove_results.txt" +REPO_ROOT="$(cd "$SCRIPT_DIR/../../../" && pwd)" + +# Parse command line arguments +while [ $# -gt 0 ]; do + case "$1" in + --pie-dir=*) + PIES_DIR="${1#*=}" + ;; + *) + echo "Unknown parameter: $1" + echo "Usage: $0 [--pie-dir=path]" + exit 1 + ;; + esac + shift +done + +mkdir -p "$REPO_ROOT/temp_proofs" + +echo "Cairo PIE Prove Benchmark Results" > "$RESULTS_FILE" +date >> "$RESULTS_FILE" +echo "----------------------------------------" >> "$RESULTS_FILE" + +# Compile only the required binaries with CPU native optimizations +RUSTFLAGS="-C target-cpu=native" cargo build --release --bin benchmark_run_and_prove --bin stwo_run_and_prove + +# Change to the repo root to ensure all relative paths work in the Rust binary +cd "$REPO_ROOT" || exit 1 + +BIN_PATH="target/release/benchmark_run_and_prove" + +for pie_file in "$PIES_DIR"/*.zip; do + if [ -f "$pie_file" ]; then + pie_name=$(basename "$pie_file") + echo "Proving $pie_name..." + # Use absolute path for pie_file to avoid NotFound errors + abs_pie_file="$(realpath "$pie_file")" + output=$("$BIN_PATH" "$abs_pie_file" 2>&1) + echo "$output" + echo "$pie_name: $output" >> "$RESULTS_FILE" + echo "----------------------------------------" >> "$RESULTS_FILE" + fi +done + +echo "Done. Results saved to $RESULTS_FILE"