Improve nightly tests

.github/workflows/bot-bump-kernel-version-to-sglang.yml

@@ -62,7 +62,7 @@ jobs:
   run-nightly-tests:
     needs: bump-kernel-version-to-sglang
     if: needs.bump-kernel-version-to-sglang.outputs.needs_sync == 'true'
-    uses: ./.github/workflows/nightly-test.yml
+    uses: ./.github/workflows/nightly-test-nvidia.yml
     with:
       ref: ${{ needs.bump-kernel-version-to-sglang.outputs.branch_name }}
     secrets: inherit

.github/workflows/bot-bump-sglang-version.yml

@@ -55,7 +55,7 @@ jobs:
 
   run-nightly-tests:
     needs: bump-sglang-version
-    uses: ./.github/workflows/nightly-test.yml
+    uses: ./.github/workflows/nightly-test-nvidia.yml
     with:
       ref: ${{ needs.bump-sglang-version.outputs.branch_name }}
     secrets: inherit

.github/workflows/nightly-test-nvidia.yml

@@ -3,12 +3,14 @@ name: Nightly Test (Nvidia)
 on:
   schedule:
     - cron: '0 0 * * *'
-  push:
-    branches:
-      - main
-    paths:
-      - "python/sglang/version.py"
   workflow_dispatch:
+  workflow_call:
+    inputs:
+      ref:
+        description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.'
+        required: false
+        type: string
+        default: ''
 
 concurrency:
   group: nightly-test-nvidia-${{ github.ref }}
@@ -84,14 +86,6 @@ jobs:
           cd test
           python3 nightly/test_qwen3_235b_perf.py
 
-      - name: Publish Qwen3-235B traces to storage repo
-        env:
-          GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
-          GITHUB_RUN_ID: ${{ github.run_id }}
-          GITHUB_RUN_NUMBER: ${{ github.run_number }}
-        run: |
-          python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_qwen3_235b
-
       - name: Run Kimi-K2-Thinking nightly performance test
         timeout-minutes: 180
         env:
@@ -103,14 +97,6 @@ jobs:
           cd test
           python3 nightly/test_kimi_k2_thinking_perf.py
 
-      - name: Publish Kimi-K2-Thinking traces to storage repo
-        env:
-          GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
-          GITHUB_RUN_ID: ${{ github.run_id }}
-          GITHUB_RUN_NUMBER: ${{ github.run_number }}
-        run: |
-          python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_kimi_k2_thinking
-
       - name: Run GLM-4.6 nightly performance test
         timeout-minutes: 180
         env:
@@ -122,34 +108,29 @@ jobs:
           cd test
           IS_BLACKWELL=1 python3 nightly/test_glm_4_6_perf.py
 
-      - name: Publish GLM-4.6 traces to storage repo
+      - name: Run MiniMax-M2 nightly performance test
+        timeout-minutes: 180
+        env:
+          TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
+          PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
+          GPU_CONFIG: "8-gpu-h200"
+        run: |
+          rm -rf test/performance_profiles_minimax_m2/
+          cd test
+          python3 nightly/test_minimax_m2_perf.py
+
+      - name: Publish all traces to storage repo
+        if: always()
         env:
           GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
           GITHUB_RUN_ID: ${{ github.run_id }}
           GITHUB_RUN_NUMBER: ${{ github.run_number }}
         run: |
-          python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_glm_4_6
-
-      # MiniMax-M2 test temporarily disabled due to compatibility issues
-      # See MINIMAX_M2_ISSUES.md for details
-      # - name: Run MiniMax-M2 nightly performance test
-      #   timeout-minutes: 180
-      #   env:
-      #     TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
-      #     PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
-      #     GPU_CONFIG: "8-gpu-h200"
-      #   run: |
-      #     rm -rf test/performance_profiles_minimax_m2/
-      #     cd test
-      #     python3 nightly/test_minimax_m2_perf.py
-
-      # - name: Publish MiniMax-M2 traces to storage repo
-      #   env:
-      #     GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
-      #     GITHUB_RUN_ID: ${{ github.run_id }}
-      #     GITHUB_RUN_NUMBER: ${{ github.run_number }}
-      #   run: |
-      #     python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_minimax_m2
+          python3 scripts/ci/publish_traces.py \
+            --traces-dir test/performance_profiles_qwen3_235b \
+            --traces-dir test/performance_profiles_kimi_k2_thinking \
+            --traces-dir test/performance_profiles_glm_4_6 \
+            --traces-dir test/performance_profiles_minimax_m2
 
   # General tests - 8 GPU H20
   nightly-test-general-8-gpu-h20:
@@ -376,14 +357,6 @@ jobs:
           cd test
           IS_BLACKWELL=1 python3 nightly/test_deepseek_v31_perf.py
 
-      - name: Publish DeepSeek v3.1 traces to storage repo
-        env:
-          GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
-          GITHUB_RUN_ID: ${{ github.run_id }}
-          GITHUB_RUN_NUMBER: ${{ github.run_number }}
-        run: |
-          python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_deepseek_v31
-
       - name: Run DeepSeek v3.2 nightly performance test
         timeout-minutes: 180
         env:
@@ -395,14 +368,6 @@ jobs:
           cd test
           IS_BLACKWELL=1 python3 nightly/test_deepseek_v32_perf.py
 
-      - name: Publish DeepSeek v3.2 traces to storage repo
-        env:
-          GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
-          GITHUB_RUN_ID: ${{ github.run_id }}
-          GITHUB_RUN_NUMBER: ${{ github.run_number }}
-        run: |
-          python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_deepseek_v32
-
       - name: Run Kimi-K2-Thinking nightly performance test
         timeout-minutes: 180
         env:
@@ -414,14 +379,6 @@ jobs:
           cd test
           IS_BLACKWELL=1 python3 nightly/test_kimi_k2_thinking_perf.py
 
-      - name: Publish Kimi-K2-Thinking traces to storage repo
-        env:
-          GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
-          GITHUB_RUN_ID: ${{ github.run_id }}
-          GITHUB_RUN_NUMBER: ${{ github.run_number }}
-        run: |
-          python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_kimi_k2_thinking
-
       - name: Run Qwen3-235B nightly performance test
         timeout-minutes: 180
         env:
@@ -433,14 +390,6 @@ jobs:
           cd test
           IS_BLACKWELL=1 python3 nightly/test_qwen3_235b_perf.py
 
-      - name: Publish Qwen3-235B traces to storage repo
-        env:
-          GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
-          GITHUB_RUN_ID: ${{ github.run_id }}
-          GITHUB_RUN_NUMBER: ${{ github.run_number }}
-        run: |
-          python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_qwen3_235b
-
       - name: Run GLM-4.6 nightly performance test
         timeout-minutes: 180
         env:
@@ -452,34 +401,31 @@ jobs:
           cd test
           IS_BLACKWELL=1 python3 nightly/test_glm_4_6_perf.py
 
-      - name: Publish GLM-4.6 traces to storage repo
+      - name: Run MiniMax-M2 nightly performance test
+        timeout-minutes: 180
+        env:
+          TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
+          PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
+          GPU_CONFIG: "8-gpu-b200"
+        run: |
+          rm -rf test/performance_profiles_minimax_m2/
+          cd test
+          IS_BLACKWELL=1 python3 nightly/test_minimax_m2_perf.py
+
+      - name: Publish all traces to storage repo
+        if: always()
         env:
           GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
           GITHUB_RUN_ID: ${{ github.run_id }}
           GITHUB_RUN_NUMBER: ${{ github.run_number }}
         run: |
-          python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_glm_4_6
-
-      # MiniMax-M2 test temporarily disabled due to compatibility issues
-      # See MINIMAX_M2_ISSUES.md for details
-      # - name: Run MiniMax-M2 nightly performance test
-      #   timeout-minutes: 180
-      #   env:
-      #     TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
-      #     PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
-      #     GPU_CONFIG: "8-gpu-b200"
-      #   run: |
-      #     rm -rf test/performance_profiles_minimax_m2/
-      #     cd test
-      #     IS_BLACKWELL=1 python3 nightly/test_minimax_m2_perf.py
-
-      # - name: Publish MiniMax-M2 traces to storage repo
-      #   env:
-      #     GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
-      #     GITHUB_RUN_ID: ${{ github.run_id }}
-      #     GITHUB_RUN_NUMBER: ${{ github.run_number }}
-      #   run: |
-      #     python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_minimax_m2
+          python3 scripts/ci/publish_traces.py \
+            --traces-dir test/performance_profiles_deepseek_v31 \
+            --traces-dir test/performance_profiles_deepseek_v32 \
+            --traces-dir test/performance_profiles_kimi_k2_thinking \
+            --traces-dir test/performance_profiles_qwen3_235b \
+            --traces-dir test/performance_profiles_glm_4_6 \
+            --traces-dir test/performance_profiles_minimax_m2
 
   # Final check job
   check-all-jobs: