Merge branch 'main' into dev/sbo.public

Author: ch-wan

.github/CI_PERMISSIONS.json

@@ -485,6 +485,12 @@
         "cooldown_interval_minutes": 60,
         "reason": "custom override"
     },
+    "netanel-haber": {
+        "can_tag_run_ci_label": true,
+        "can_rerun_failed_ci": true,
+        "cooldown_interval_minutes": 0,
+        "reason": "custom override"
+    },
     "nvcastet": {
         "can_tag_run_ci_label": true,
         "can_rerun_failed_ci": true,

.github/labeler.yml

@@ -66,6 +66,13 @@ amd:
       - '**/*amd*'
       - '**/*rocm*'
 
+# NPU specific
+npu:
+  - changed-files:
+    - any-glob-to-any-file:
+      - '**/*npu*'
+      - '**/*ascend*'
+
 # DeepSeek specific
 deepseek:
   - changed-files:

.github/workflows/bot-bump-kernel-version-to-sglang.yml

@@ -10,6 +10,9 @@ permissions:
 jobs:
   bump-kernel-version-to-sglang:
     runs-on: ubuntu-latest
+    outputs:
+      branch_name: ${{ steps.set_output.outputs.branch_name }}
+      needs_sync: ${{ steps.check_sync.outputs.needs_sync }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -32,6 +35,7 @@ jobs:
 
       - name: Configure Git and branch
         if: steps.check_sync.outputs.needs_sync == 'true'
+        id: set_output
         run: |
           git config user.name "sglang-bot"
           git config user.email "[email protected]"
@@ -41,6 +45,7 @@ jobs:
           git checkout -b "$BRANCH_NAME"
           echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV
           echo "KERNEL_VERSION=$KERNEL_VERSION" >> $GITHUB_ENV
+          echo "branch_name=$BRANCH_NAME" >> $GITHUB_OUTPUT
 
       - name: Run kernel version bump script
         if: steps.check_sync.outputs.needs_sync == 'true'
@@ -53,3 +58,11 @@ jobs:
           GH_TOKEN: ${{ secrets.GH_PAT_FOR_PULL_REQUEST }}
         run: |
           bash scripts/release/commit_and_pr_kernel_to_sglang.sh "$KERNEL_VERSION" "$BRANCH_NAME"
+
+  run-nightly-tests:
+    needs: bump-kernel-version-to-sglang
+    if: needs.bump-kernel-version-to-sglang.outputs.needs_sync == 'true'
+    uses: ./.github/workflows/nightly-test.yml
+    with:
+      ref: ${{ needs.bump-kernel-version-to-sglang.outputs.branch_name }}
+    secrets: inherit

.github/workflows/bot-bump-sglang-version.yml

@@ -15,6 +15,8 @@ permissions:
 jobs:
   bump-sglang-version:
     runs-on: ubuntu-latest
+    outputs:
+      branch_name: ${{ steps.set_output.outputs.branch_name }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -31,13 +33,15 @@ jobs:
           pip install tomli
 
       - name: Configure Git and branch
+        id: set_output
         run: |
           git config user.name "sglang-bot"
           git config user.email "[email protected]"
           RANDOM_SUFFIX=$(echo $RANDOM | md5sum | head -c 4)
           BRANCH_NAME="bot/bump-sglang-version-${{ github.event.inputs.new_version }}-${RANDOM_SUFFIX}"
           git checkout -b "$BRANCH_NAME"
           echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV
+          echo "branch_name=$BRANCH_NAME" >> $GITHUB_OUTPUT
 
       - name: Run SGLang version bump script
         run: |
@@ -48,3 +52,10 @@ jobs:
           GH_TOKEN: ${{ secrets.GH_PAT_FOR_PULL_REQUEST }}
         run: |
           bash scripts/release/commit_and_pr.sh "SGLang" "${{ github.event.inputs.new_version }}" "$BRANCH_NAME"
+
+  run-nightly-tests:
+    needs: bump-sglang-version
+    uses: ./.github/workflows/nightly-test.yml
+    with:
+      ref: ${{ needs.bump-sglang-version.outputs.branch_name }}
+    secrets: inherit

.github/workflows/ci-monitor.yml

@@ -46,6 +46,15 @@ jobs:
           cd scripts/ci_monitor
           python ci_analyzer.py --token $GITHUB_TOKEN --limit ${{ inputs.limit || '1000' }} --output ci_analysis_$(date +%Y%m%d_%H%M%S).json
 
+      - name: Run Nightly Test Analysis
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
+          PYTHONUNBUFFERED: 1
+          PYTHONIOENCODING: utf-8
+        run: |
+          cd scripts/ci_monitor
+          python ci_analyzer.py --token $GITHUB_TOKEN --mode nightly --days 2 --output nightly_analysis_$(date +%Y%m%d_%H%M%S).json
+
       - name: Run Performance Analysis
         env:
           GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
@@ -61,6 +70,7 @@ jobs:
           name: ci-analysis-results-${{ github.run_number }}
           path: |
             scripts/ci_monitor/ci_analysis_*.json
+            scripts/ci_monitor/nightly_analysis_*.json
             scripts/ci_monitor/performance_tables_*
           retention-days: 30
 

.github/workflows/nightly-test-nvidia.yml

@@ -30,8 +30,8 @@ jobs:
       - name: Run test
         timeout-minutes: 60
         run: |
-          cd test/srt
-          python3 run_suite.py --suite nightly-1-gpu --continue-on-error
+          cd test
+          python3 run_suite_nightly.py --suite nightly-1-gpu --continue-on-error
 
   # General tests - 4 GPU H100
   nightly-test-general-4-gpu-h100:
@@ -48,8 +48,8 @@ jobs:
       - name: Run test
         timeout-minutes: 30
         run: |
-          cd test/srt
-          python3 run_suite.py --suite nightly-4-gpu --continue-on-error
+          cd test
+          python3 run_suite_nightly.py --suite nightly-4-gpu --continue-on-error
 
   # General tests - 8 GPU H200
   nightly-test-general-8-gpu-h200:
@@ -70,8 +70,8 @@ jobs:
         env:
           GPU_CONFIG: "8-gpu-h200"
         run: |
-          cd test/srt
-          python3 run_suite.py --suite nightly-8-gpu-h200 --continue-on-error
+          cd test
+          python3 run_suite_nightly.py --suite nightly-8-gpu-h200 --continue-on-error
 
   # General tests - 8 GPU H20
   nightly-test-general-8-gpu-h20:
@@ -92,8 +92,8 @@ jobs:
         env:
           GPU_CONFIG: "8-gpu-h20"
         run: |
-          cd test/srt
-          python3 run_suite.py --suite nightly-8-gpu-h20 --continue-on-error
+          cd test
+          python3 run_suite_nightly.py --suite nightly-8-gpu-h20 --continue-on-error
 
   # Text model accuracy tests
   nightly-test-text-accuracy-2-gpu-runner:
@@ -110,7 +110,7 @@ jobs:
       - name: Run eval test for text models
         timeout-minutes: 120
         run: |
-          cd test/srt
+          cd test
           python3 nightly/test_text_models_gsm8k_eval.py
 
   # Text model performance tests
@@ -132,7 +132,7 @@ jobs:
           PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
           GPU_CONFIG: "2-gpu-runner"
         run: |
-          cd test/srt
+          cd test
           rm -rf performance_profiles_text_models/
           python3 nightly/test_text_models_perf.py
 
@@ -159,7 +159,7 @@ jobs:
       - name: Run eval test for VLM models (fixed MMMU-100)
         timeout-minutes: 240
         run: |
-          cd test/srt
+          cd test
           python3 nightly/test_vlms_mmmu_eval.py
 
   # VLM performance tests
@@ -181,7 +181,7 @@ jobs:
           PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
           GPU_CONFIG: "2-gpu-runner"
         run: |
-          cd test/srt
+          cd test
           rm -rf performance_profiles_vlms/
           python3 nightly/test_vlms_perf.py
 
@@ -208,8 +208,8 @@ jobs:
       - name: Run test
         timeout-minutes: 60
         run: |
-          cd test/srt
-          python3 run_suite.py --suite nightly-4-gpu-b200 --continue-on-error
+          cd test
+          python3 run_suite_nightly.py --suite nightly-4-gpu-b200 --continue-on-error
 
   # B200 Performance tests - 8 GPU
   nightly-test-perf-8-gpu-b200:
@@ -233,7 +233,7 @@ jobs:
           GPU_CONFIG: "8-gpu-b200"
         run: |
           rm -rf test/srt/performance_profiles_deepseek_v31/
-          cd test/srt
+          cd test
           IS_BLACKWELL=1 python3 nightly/test_deepseek_v31_perf.py
 
       - name: Publish DeepSeek v3.1 traces to storage repo
@@ -252,7 +252,7 @@ jobs:
           GPU_CONFIG: "8-gpu-b200"
         run: |
           rm -rf test/srt/performance_profiles_deepseek_v32/
-          cd test/srt
+          cd test
           IS_BLACKWELL=1 python3 nightly/test_deepseek_v32_perf.py
 
       - name: Publish DeepSeek v3.2 traces to storage repo

.github/workflows/nightly-test.yml

@@ -9,6 +9,13 @@ on:
     paths:
       - "python/sglang/version.py"
   workflow_dispatch:
+  workflow_call:
+    inputs:
+      ref:
+        description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.'
+        required: false
+        type: string
+        default: ''
 
 concurrency:
   group: nightly-test-${{ github.ref }}
@@ -21,6 +28,8 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
 
       - name: Install dependencies
         run: |
@@ -38,6 +47,8 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
 
       - name: Install dependencies
         run: |
@@ -67,6 +78,8 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
 
       - name: Install dependencies
         run: |
@@ -84,6 +97,8 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
 
       - name: Install dependencies
         run: |
@@ -116,6 +131,8 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
 
       - name: Install dependencies
         run: |
@@ -133,6 +150,8 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
 
       - name: Install dependencies
         run: |
@@ -152,6 +171,8 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
 
       - name: Install dependencies
         run: |
@@ -171,6 +192,8 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
 
       - name: Install dependencies
         run: |
@@ -192,6 +215,8 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
 
       - name: Install dependencies
         run: |

.github/workflows/pr-gate.yml

@@ -5,10 +5,10 @@ on:
         description: "Whether the PR must have the run-ci label"
         type: boolean
         default: true
-      rate-limit-hours:
-        description: "Rate limit window size in hours; 0 disables rate limiting"
+      cool-down-minutes:
+        description: "Cooldown period in minutes for low-permission users; 0 disables rate limiting"
         type: number
-        default: 2
+        default: 120
 
 jobs:
   pr-gate:
@@ -41,7 +41,7 @@ jobs:
           echo "PR Draft: ${{ steps.pr.outputs.draft }}"
           echo "PR User: ${{ steps.pr.outputs.user }}"
           echo "Require run-ci: ${{ inputs.require-run-ci }}"
-          echo "Rate limit hours: ${{ inputs.rate-limit-hours }}"
+          echo "Cool down minutes: ${{ inputs.cool-down-minutes }}"
           echo "==================="
 
       - name: Block draft PR
@@ -60,12 +60,12 @@ jobs:
           fi
 
       - name: Enforce rate limit for low-permission actors (optional)
-        if: github.event_name == 'pull_request' && inputs.rate-limit-hours > 0
+        if: github.event_name == 'pull_request' && inputs.cool-down-minutes > 0
         uses: actions/github-script@v7
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           script: |
-            const DEFAULT_HOURS = Number("${{ inputs.rate-limit-hours }}");
+            const DEFAULT_MINUTES = Number("${{ inputs.cool-down-minutes }}");
             const owner = context.repo.owner;
             const repo = context.repo.repo;
             const eventName = context.eventName;
@@ -96,7 +96,7 @@ jobs:
               return;
             }
 
-            let effectiveCooldownMinutes = DEFAULT_HOURS * 60;
+            let effectiveCooldownMinutes = DEFAULT_MINUTES;
             let perUserCooldownMinutes = null;
 
             try {