Skip to content

Commit 8e3d2c0

Browse files
AvivYossef-starkwareAvivYossef-starkware
authored
ci: comment out benchmarks (#10516)
1 parent 87d1e4e commit 8e3d2c0

File tree

2 files changed

+89
-92
lines changed

2 files changed

+89
-92
lines changed

‎.github/workflows/blockifier_ci.yml‎

Lines changed: 32 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -95,42 +95,40 @@ jobs:
9595
- run: cargo build -p blockifier --features tracing
9696
- run: cargo test -p blockifier --features tracing
9797

98-
# TODO(Dori): Uncomment once the merge to main is complete (bench_tools does not exist on commit
99-
# github.base_ref).
100-
# benchmarking:
101-
# runs-on: namespace-profile-medium-ubuntu-24-04-amd64
102-
# if: ${{ github.event_name == 'pull_request' }}
103-
# steps:
104-
# # Checkout the base branch to benchmark the old code.
105-
# - uses: actions/checkout@v4
106-
# with:
107-
# ref: ${{ github.base_ref }}
98+
benchmarking:
99+
runs-on: namespace-profile-medium-ubuntu-24-04-amd64
100+
if: ${{ github.event_name == 'pull_request' }}
101+
steps:
102+
# Checkout the base branch to benchmark the old code.
103+
- uses: actions/checkout@v4
104+
with:
105+
ref: ${{ github.base_ref }}
108106

109-
# - uses: ./.github/actions/bootstrap
110-
# with:
111-
# github_token: ${{ secrets.GITHUB_TOKEN }}
107+
- uses: ./.github/actions/bootstrap
108+
with:
109+
github_token: ${{ secrets.GITHUB_TOKEN }}
112110

113-
# # Restore cargo artifacts build cache.
114-
# - name: Restore Cargo build cache
115-
# uses: Swatinem/rust-cache@v2
116-
# with:
117-
# shared-key: blockifier-bench
118-
# cache-on-failure: true
119-
# cache-workspace-crates: true
111+
# Restore cargo artifacts build cache.
112+
- name: Restore Cargo build cache
113+
uses: Swatinem/rust-cache@v2
114+
with:
115+
shared-key: blockifier-bench
116+
cache-on-failure: true
117+
cache-workspace-crates: true
120118

121-
# # Benchmark the base branch code.
122-
# - run: cargo run -p bench_tools -- run --package blockifier --out /tmp/base_results
119+
# Benchmark the base branch code.
120+
- run: cargo run -p bench_tools -- run --package blockifier --out /tmp/base_results
123121

124-
# # Checkout the current branch to benchmark the new code.
125-
# - uses: actions/checkout@v4
126-
# with:
127-
# clean: false
122+
# Checkout the current branch to benchmark the new code.
123+
- uses: actions/checkout@v4
124+
with:
125+
clean: false
128126

129-
# # Benchmark the current branch and compare to the previous run.
130-
# - run: |
131-
# cargo run -p bench_tools -- run-and-compare \
132-
# --package blockifier \
133-
# --out /tmp/new_results \
134-
# --regression-limit 8.0 \
135-
# --set-absolute-time-ns-limit transfers_benchmark_cairo_native ${TRANSFERS_BENCHMARK_CAIRO_NATIVE_TIME_LIMIT_NS} \
136-
# --set-absolute-time-ns-limit transfers_benchmark_vm ${TRANSFERS_BENCHMARK_VM_TIME_LIMIT_NS}
127+
# Benchmark the current branch and compare to the previous run.
128+
- run: |
129+
cargo run -p bench_tools -- run-and-compare \
130+
--package blockifier \
131+
--out /tmp/new_results \
132+
--regression-limit 8.0 \
133+
--set-absolute-time-ns-limit transfers_benchmark_cairo_native ${TRANSFERS_BENCHMARK_CAIRO_NATIVE_TIME_LIMIT_NS} \
134+
--set-absolute-time-ns-limit transfers_benchmark_vm ${TRANSFERS_BENCHMARK_VM_TIME_LIMIT_NS}

‎.github/workflows/committer_ci.yml‎

Lines changed: 57 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -64,61 +64,60 @@ jobs:
6464
- run: gcloud storage cp -r gs://committer-testing-artifacts/$BENCH_INPUT_FILES_PREFIX/* ./crates/starknet_committer_and_os_cli/test_inputs
6565
- run: cargo test -p starknet_committer_and_os_cli --release -- --include-ignored test_regression
6666

67-
# TODO(Aviv): Uncomment.
68-
# benchmarking:
69-
# runs-on: namespace-profile-medium-ubuntu-24-04-amd64
70-
# if: ${{ github.event_name == 'pull_request' }}
71-
# steps:
72-
# # Checkout the base branch to benchmark the old code.
73-
# - uses: actions/checkout@v4
74-
# with:
75-
# ref: ${{ github.base_ref }}
76-
77-
# # Setup pypy and link to the location expected by .cargo/config.toml.
78-
# # Python + requirements are needed to compile the OS.
79-
# - uses: actions/setup-python@v5
80-
# id: setup-pypy
81-
# with:
82-
# python-version: "pypy3.9"
83-
# cache: "pip"
84-
# - run: ln -s '${{ steps.setup-pypy.outputs.python-path }}' /usr/local/bin/pypy3.9
85-
# - env:
86-
# LD_LIBRARY_PATH: ${{ env.Python3_ROOT_DIR }}/bin
87-
# run: echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> $GITHUB_ENV
88-
# - run: pip install -r scripts/requirements.txt
89-
90-
# - uses: ./.github/actions/bootstrap
91-
# with:
92-
# github_token: ${{ secrets.GITHUB_TOKEN }}
93-
94-
# # Auth with GCS for bench_tools to download inputs.
95-
# - id: auth
96-
# uses: "google-github-actions/auth@v2"
97-
# with:
98-
# credentials_json: ${{ secrets.COMMITER_PRODUCTS_EXT_WRITER_JSON }}
99-
# - uses: "google-github-actions/setup-gcloud@v2"
100-
101-
# # Benchmark the base branch code.
102-
# - run: cargo run -p bench_tools -- run --package starknet_committer_and_os_cli --out /tmp/base_results
103-
104-
# # Save the input files from the base branch to use in the second benchmark.
105-
# - run: cp -r crates/starknet_committer_and_os_cli/test_inputs /tmp/test_inputs
106-
107-
# # Checkout the current branch to benchmark the new code.
108-
# - uses: actions/checkout@v4
109-
# with:
110-
# clean: false
111-
112-
# # Re-install pip requirements in case they changed between branches.
113-
# - run: pip install -r scripts/requirements.txt
114-
115-
# # Benchmark the current branch and compare to the previous run.
116-
# # Use the saved inputs and enforce absolute time limits (full_committer_flow: 25ms, tree_computation_flow: 20ms).
117-
# - run: |
118-
# cargo run -p bench_tools -- run-and-compare \
119-
# --package starknet_committer_and_os_cli \
120-
# --out /tmp/new_results \
121-
# --regression-limit 8.0 \
122-
# --input-dir /tmp/test_inputs \
123-
# --set-absolute-time-ns-limit full_committer_flow ${FULL_COMMITTER_FLOW_TIME_LIMIT_NS} \
124-
# --set-absolute-time-ns-limit tree_computation_flow ${TREE_COMPUTATION_FLOW_TIME_LIMIT_NS}
67+
benchmarking:
68+
runs-on: namespace-profile-medium-ubuntu-24-04-amd64
69+
if: ${{ github.event_name == 'pull_request' }}
70+
steps:
71+
# Checkout the base branch to benchmark the old code.
72+
- uses: actions/checkout@v4
73+
with:
74+
ref: ${{ github.base_ref }}
75+
76+
# Setup pypy and link to the location expected by .cargo/config.toml.
77+
# Python + requirements are needed to compile the OS.
78+
- uses: actions/setup-python@v5
79+
id: setup-pypy
80+
with:
81+
python-version: "pypy3.9"
82+
cache: "pip"
83+
- run: ln -s '${{ steps.setup-pypy.outputs.python-path }}' /usr/local/bin/pypy3.9
84+
- env:
85+
LD_LIBRARY_PATH: ${{ env.Python3_ROOT_DIR }}/bin
86+
run: echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> $GITHUB_ENV
87+
- run: pip install -r scripts/requirements.txt
88+
89+
- uses: ./.github/actions/bootstrap
90+
with:
91+
github_token: ${{ secrets.GITHUB_TOKEN }}
92+
93+
# Auth with GCS for bench_tools to download inputs.
94+
- id: auth
95+
uses: "google-github-actions/auth@v2"
96+
with:
97+
credentials_json: ${{ secrets.COMMITER_PRODUCTS_EXT_WRITER_JSON }}
98+
- uses: "google-github-actions/setup-gcloud@v2"
99+
100+
# Benchmark the base branch code.
101+
- run: cargo run -p bench_tools -- run --package starknet_committer_and_os_cli --out /tmp/base_results
102+
103+
# Save the input files from the base branch to use in the second benchmark.
104+
- run: cp -r crates/starknet_committer_and_os_cli/test_inputs /tmp/test_inputs
105+
106+
# Checkout the current branch to benchmark the new code.
107+
- uses: actions/checkout@v4
108+
with:
109+
clean: false
110+
111+
# Re-install pip requirements in case they changed between branches.
112+
- run: pip install -r scripts/requirements.txt
113+
114+
# Benchmark the current branch and compare to the previous run.
115+
# Use the saved inputs and enforce absolute time limits (full_committer_flow: 25ms, tree_computation_flow: 20ms).
116+
- run: |
117+
cargo run -p bench_tools -- run-and-compare \
118+
--package starknet_committer_and_os_cli \
119+
--out /tmp/new_results \
120+
--regression-limit 8.0 \
121+
--input-dir /tmp/test_inputs \
122+
--set-absolute-time-ns-limit full_committer_flow ${FULL_COMMITTER_FLOW_TIME_LIMIT_NS} \
123+
--set-absolute-time-ns-limit tree_computation_flow ${TREE_COMPUTATION_FLOW_TIME_LIMIT_NS}

0 commit comments

Comments
 (0)