Skip to content

Commit b51cbef

Browse files
ZacSweersZacSweers
committed
Support single-ref benchmarks
1 parent 1b9bfb0 commit b51cbef

File tree

3 files changed

+431
-46
lines changed

3 files changed

+431
-46
lines changed

‎.github/workflows/benchmarks.yml‎

Lines changed: 99 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,10 @@ on:
99
type: string
1010
default: 'main'
1111
ref2:
12-
description: 'Comparison git ref (branch name or commit SHA)'
13-
required: true
12+
description: 'Comparison git ref (branch name or commit SHA). Leave empty for single-ref benchmark.'
13+
required: false
1414
type: string
15+
default: ''
1516
metro:
1617
description: 'Benchmark Metro'
1718
required: false
@@ -60,12 +61,24 @@ jobs:
6061
runs-on: ubuntu-latest
6162
outputs:
6263
modes: ${{ steps.modes.outputs.modes }}
64+
is_single_ref: ${{ steps.check_mode.outputs.is_single_ref }}
6365
steps:
6466
- name: Checkout
6567
uses: actions/checkout@v6
6668
with:
6769
fetch-depth: 0
6870

71+
- name: Check benchmark mode
72+
id: check_mode
73+
run: |
74+
if [ -z "${{ inputs.ref2 }}" ]; then
75+
echo "is_single_ref=true" >> $GITHUB_OUTPUT
76+
echo "Running in single-ref mode (ref1 only)"
77+
else
78+
echo "is_single_ref=false" >> $GITHUB_OUTPUT
79+
echo "Running in comparison mode (ref1 vs ref2)"
80+
fi
81+
6982
- name: Build modes string
7083
id: modes
7184
run: |
@@ -100,16 +113,20 @@ jobs:
100113
echo "Error: Invalid git ref: ${{ inputs.ref1 }}"
101114
exit 1
102115
}
103-
echo "Validating ref2: ${{ inputs.ref2 }}"
104-
git rev-parse --verify "${{ inputs.ref2 }}" || git rev-parse --verify "origin/${{ inputs.ref2 }}" || {
105-
echo "Error: Invalid git ref: ${{ inputs.ref2 }}"
106-
exit 1
107-
}
108-
echo "Both refs are valid"
116+
if [ -n "${{ inputs.ref2 }}" ]; then
117+
echo "Validating ref2: ${{ inputs.ref2 }}"
118+
git rev-parse --verify "${{ inputs.ref2 }}" || git rev-parse --verify "origin/${{ inputs.ref2 }}" || {
119+
echo "Error: Invalid git ref: ${{ inputs.ref2 }}"
120+
exit 1
121+
}
122+
echo "Both refs are valid"
123+
else
124+
echo "Single ref mode - ref1 is valid"
125+
fi
109126
110127
# Startup benchmarks using JMH
111128
startup-benchmarks:
112-
name: "Startup (JMH): ${{ inputs.ref1 }} vs ${{ inputs.ref2 }}"
129+
name: "Startup (JMH): ${{ inputs.ref2 && format('{0} vs {1}', inputs.ref1, inputs.ref2) || inputs.ref1 }}"
113130
runs-on: ubuntu-latest
114131
needs: validate
115132
if: inputs.run-startup-benchmarks
@@ -133,29 +150,42 @@ jobs:
133150
gradle-home-cache-strict-match: true
134151
cache-encryption-key: ${{ secrets.GRADLE_ENCRYPTION_KEY }}
135152

136-
- name: Fetch and checkout both refs
153+
- name: Fetch and checkout refs
137154
run: |
138155
git fetch --all
139-
# Checkout both refs to create local branches so the scripts can find them
156+
# Checkout ref1
140157
git checkout "${{ inputs.ref1 }}" || git checkout -b "${{ inputs.ref1 }}" "origin/${{ inputs.ref1 }}"
141-
git checkout "${{ inputs.ref2 }}" || git checkout -b "${{ inputs.ref2 }}" "origin/${{ inputs.ref2 }}"
158+
# Checkout ref2 if in comparison mode
159+
if [ -n "${{ inputs.ref2 }}" ]; then
160+
git checkout "${{ inputs.ref2 }}" || git checkout -b "${{ inputs.ref2 }}" "origin/${{ inputs.ref2 }}"
161+
fi
142162
# Return to ref1 as the starting point
143163
git checkout "${{ inputs.ref1 }}"
144164
145-
- name: Run startup benchmark comparison
165+
- name: Run startup benchmarks
146166
run: |
147167
cd benchmark
148-
RERUN_FLAG=""
149-
if [ "${{ inputs.rerun-non-metro }}" = "true" ]; then
150-
RERUN_FLAG="--rerun-non-metro"
168+
if [ "${{ needs.validate.outputs.is_single_ref }}" = "true" ]; then
169+
# Single-ref mode
170+
./run_startup_benchmarks.sh single \
171+
--ref "${{ inputs.ref1 }}" \
172+
--modes "${{ needs.validate.outputs.modes }}" \
173+
--count "${{ inputs.module_count }}" \
174+
--benchmark jvm
175+
else
176+
# Comparison mode
177+
RERUN_FLAG=""
178+
if [ "${{ inputs.rerun-non-metro }}" = "true" ]; then
179+
RERUN_FLAG="--rerun-non-metro"
180+
fi
181+
./run_startup_benchmarks.sh compare \
182+
--ref1 "${{ inputs.ref1 }}" \
183+
--ref2 "${{ inputs.ref2 }}" \
184+
--modes "${{ needs.validate.outputs.modes }}" \
185+
--count "${{ inputs.module_count }}" \
186+
--benchmark jvm \
187+
$RERUN_FLAG
151188
fi
152-
./run_startup_benchmarks.sh compare \
153-
--ref1 "${{ inputs.ref1 }}" \
154-
--ref2 "${{ inputs.ref2 }}" \
155-
--modes "${{ needs.validate.outputs.modes }}" \
156-
--count "${{ inputs.module_count }}" \
157-
--benchmark jvm \
158-
$RERUN_FLAG
159189
160190
- name: Find results directory
161191
id: results
@@ -178,21 +208,28 @@ jobs:
178208
- name: Add startup summary to workflow
179209
run: |
180210
results_dir="${{ steps.results.outputs.results_dir }}"
181-
summary_file="${results_dir}comparison-summary.md"
211+
# Try comparison-summary.md first, then single-summary.md
212+
if [ -f "${results_dir}comparison-summary.md" ]; then
213+
summary_file="${results_dir}comparison-summary.md"
214+
elif [ -f "${results_dir}single-summary.md" ]; then
215+
summary_file="${results_dir}single-summary.md"
216+
else
217+
summary_file=""
218+
fi
182219
183-
if [ -f "$summary_file" ]; then
220+
if [ -n "$summary_file" ]; then
184221
echo "## Startup Benchmark Results (JMH)" >> $GITHUB_STEP_SUMMARY
185222
echo "" >> $GITHUB_STEP_SUMMARY
186223
cat "$summary_file" >> $GITHUB_STEP_SUMMARY
187224
else
188225
echo "## Startup Benchmark Results (JMH)" >> $GITHUB_STEP_SUMMARY
189226
echo "" >> $GITHUB_STEP_SUMMARY
190-
echo "No comparison summary found. Check the uploaded artifacts for raw results." >> $GITHUB_STEP_SUMMARY
227+
echo "No summary found. Check the uploaded artifacts for raw results." >> $GITHUB_STEP_SUMMARY
191228
fi
192229
193230
# Build benchmarks using Gradle Profiler
194231
build-benchmarks:
195-
name: "Build (Gradle Profiler): ${{ inputs.ref1 }} vs ${{ inputs.ref2 }}"
232+
name: "Build (Gradle Profiler): ${{ inputs.ref2 && format('{0} vs {1}', inputs.ref1, inputs.ref2) || inputs.ref1 }}"
196233
runs-on: ubuntu-latest
197234
needs: validate
198235
if: inputs.run-build-benchmarks
@@ -216,31 +253,43 @@ jobs:
216253
gradle-home-cache-strict-match: true
217254
cache-encryption-key: ${{ secrets.GRADLE_ENCRYPTION_KEY }}
218255

219-
- name: Fetch and checkout both refs
256+
- name: Fetch and checkout refs
220257
run: |
221258
git fetch --all
222-
# Checkout both refs to create local branches so the scripts can find them
259+
# Checkout ref1
223260
git checkout "${{ inputs.ref1 }}" || git checkout -b "${{ inputs.ref1 }}" "origin/${{ inputs.ref1 }}"
224-
git checkout "${{ inputs.ref2 }}" || git checkout -b "${{ inputs.ref2 }}" "origin/${{ inputs.ref2 }}"
261+
# Checkout ref2 if in comparison mode
262+
if [ -n "${{ inputs.ref2 }}" ]; then
263+
git checkout "${{ inputs.ref2 }}" || git checkout -b "${{ inputs.ref2 }}" "origin/${{ inputs.ref2 }}"
264+
fi
225265
# Return to ref1 as the starting point
226266
git checkout "${{ inputs.ref1 }}"
227267
228268
- name: Install gradle-profiler
229269
run: ./benchmark/install-gradle-profiler.sh
230270

231-
- name: Run build benchmark comparison
271+
- name: Run build benchmarks
232272
run: |
233273
cd benchmark
234-
RERUN_FLAG=""
235-
if [ "${{ inputs.rerun-non-metro }}" = "true" ]; then
236-
RERUN_FLAG="--rerun-non-metro"
274+
if [ "${{ needs.validate.outputs.is_single_ref }}" = "true" ]; then
275+
# Single-ref mode
276+
./run_benchmarks.sh single \
277+
--ref "${{ inputs.ref1 }}" \
278+
--modes "${{ needs.validate.outputs.modes }}" \
279+
${{ inputs.module_count }}
280+
else
281+
# Comparison mode
282+
RERUN_FLAG=""
283+
if [ "${{ inputs.rerun-non-metro }}" = "true" ]; then
284+
RERUN_FLAG="--rerun-non-metro"
285+
fi
286+
./run_benchmarks.sh compare \
287+
--ref1 "${{ inputs.ref1 }}" \
288+
--ref2 "${{ inputs.ref2 }}" \
289+
--modes "${{ needs.validate.outputs.modes }}" \
290+
$RERUN_FLAG \
291+
${{ inputs.module_count }}
237292
fi
238-
./run_benchmarks.sh compare \
239-
--ref1 "${{ inputs.ref1 }}" \
240-
--ref2 "${{ inputs.ref2 }}" \
241-
--modes "${{ needs.validate.outputs.modes }}" \
242-
$RERUN_FLAG \
243-
${{ inputs.module_count }}
244293
245294
- name: Find results directory
246295
id: results
@@ -263,14 +312,21 @@ jobs:
263312
- name: Add build summary to workflow
264313
run: |
265314
results_dir="${{ steps.results.outputs.results_dir }}"
266-
summary_file="${results_dir}comparison-summary.md"
315+
# Try comparison-summary.md first, then single-summary.md
316+
if [ -f "${results_dir}comparison-summary.md" ]; then
317+
summary_file="${results_dir}comparison-summary.md"
318+
elif [ -f "${results_dir}single-summary.md" ]; then
319+
summary_file="${results_dir}single-summary.md"
320+
else
321+
summary_file=""
322+
fi
267323
268-
if [ -f "$summary_file" ]; then
324+
if [ -n "$summary_file" ]; then
269325
echo "## Build Benchmark Results (Gradle Profiler)" >> $GITHUB_STEP_SUMMARY
270326
echo "" >> $GITHUB_STEP_SUMMARY
271327
cat "$summary_file" >> $GITHUB_STEP_SUMMARY
272328
else
273329
echo "## Build Benchmark Results (Gradle Profiler)" >> $GITHUB_STEP_SUMMARY
274330
echo "" >> $GITHUB_STEP_SUMMARY
275-
echo "No comparison summary found. Check the uploaded artifacts for raw results." >> $GITHUB_STEP_SUMMARY
331+
echo "No summary found. Check the uploaded artifacts for raw results." >> $GITHUB_STEP_SUMMARY
276332
fi

0 commit comments

Comments
 (0)