feat: add dynamic port allocation to prevent test conflicts

keivenchang · keivenchang · commit de716ff619b4 · 2025-11-04T17:15:04.000-08:00
Add centralized port allocation utilities and update all tests and launch
scripts to use available ports instead of hard-coded ports (8000, 8081, 8082).
This prevents port conflicts and enables parallel test execution in the future.

Changes:
- Add tests/utils/port_utils.py with get_free_port(), get_free_ports(), and
  helper functions for test configuration
- Update fault tolerance cancellation tests to dynamically allocate both
  frontend and worker ports
- Update DynamoFrontendProcess and request functions to accept dynamic
  frontend_port instead of hard-coded FRONTEND_PORT constant
- Modify backend launch scripts to respect environment variables
  DYN_FRONTEND_PORT and DYN_SYSTEM_PORT with fallback to defaults

Signed-off-by: Keiven Chang &lt;keivenchang@users.noreply.github.com&gt;
diff --git a/examples/backends/sglang/launch/agg.sh b/examples/backends/sglang/launch/agg.sh
@@ -13,11 +13,11 @@ trap cleanup EXIT INT TERM
 
 
 # run ingress
-python3 -m dynamo.frontend --http-port=8000 &
+python3 -m dynamo.frontend --http-port=${DYN_FRONTEND_PORT:-8000} &
 DYNAMO_PID=$!
 
 # run worker with metrics enabled
-DYN_SYSTEM_ENABLED=true DYN_SYSTEM_PORT=8081 \
+DYN_SYSTEM_ENABLED=true DYN_SYSTEM_PORT=${DYN_SYSTEM_PORT:-8081} \
 python3 -m dynamo.sglang \
   --model-path Qwen/Qwen3-0.6B \
   --served-model-name Qwen/Qwen3-0.6B \
diff --git a/examples/backends/sglang/launch/agg_embed.sh b/examples/backends/sglang/launch/agg_embed.sh
@@ -13,7 +13,7 @@ trap cleanup EXIT INT TERM
 
 
 # run ingress
-python3 -m dynamo.frontend --http-port=8000 &
+python3 -m dynamo.frontend --http-port=${DYN_FRONTEND_PORT:-8000} &
 DYNAMO_PID=$!
 
 # run worker
diff --git a/examples/backends/sglang/launch/agg_router.sh b/examples/backends/sglang/launch/agg_router.sh
@@ -13,7 +13,7 @@ trap cleanup EXIT INT TERM
 
 
 # run ingress
-python -m dynamo.frontend --router-mode kv --http-port=8000 &
+python -m dynamo.frontend --router-mode kv --http-port=${DYN_FRONTEND_PORT:-8000} &
 DYNAMO_PID=$!
 
 # run worker
diff --git a/examples/backends/sglang/launch/disagg.sh b/examples/backends/sglang/launch/disagg.sh
@@ -13,7 +13,7 @@ trap cleanup EXIT INT TERM
 
 
 # run ingress
-python3 -m dynamo.frontend --http-port=8000 &
+python3 -m dynamo.frontend --http-port=${DYN_FRONTEND_PORT:-8000} &
 DYNAMO_PID=$!
 
 # run prefill worker
diff --git a/examples/backends/sglang/launch/disagg_dp_attn.sh b/examples/backends/sglang/launch/disagg_dp_attn.sh
@@ -13,7 +13,7 @@ trap cleanup EXIT INT TERM
 
 
 # run ingress
-python3 -m dynamo.frontend --http-port=8000 &
+python3 -m dynamo.frontend --http-port=${DYN_FRONTEND_PORT:-8000} &
 DYNAMO_PID=$!
 
 # run prefill worker
diff --git a/examples/backends/sglang/launch/disagg_router.sh b/examples/backends/sglang/launch/disagg_router.sh
@@ -13,7 +13,7 @@ trap cleanup EXIT INT TERM
 
 # run ingress
 python3 -m dynamo.frontend \
- --http-port=8000 \
+ --http-port=${DYN_FRONTEND_PORT:-8000} \
  --router-mode kv \
  --kv-overlap-score-weight 0 \
  --router-reset-states &
diff --git a/examples/backends/sglang/launch/disagg_same_gpu.sh b/examples/backends/sglang/launch/disagg_same_gpu.sh
@@ -37,11 +37,11 @@ trap cleanup EXIT INT TERM
 
 
 # run ingress with KV router mode for disaggregated setup
-python3 -m dynamo.frontend --router-mode kv --http-port=8000 &
+python3 -m dynamo.frontend --router-mode kv --http-port=${DYN_FRONTEND_PORT:-8000} &
 DYNAMO_PID=$!
 
 # run prefill worker with metrics on port 8081
-DYN_SYSTEM_ENABLED=true DYN_SYSTEM_PORT=8081 \
+DYN_SYSTEM_ENABLED=true DYN_SYSTEM_PORT=${DYN_SYSTEM_PORT1:-8081} \
 python3 -m dynamo.sglang \
   --model-path Qwen/Qwen3-0.6B \
   --served-model-name Qwen/Qwen3-0.6B \
@@ -71,7 +71,7 @@ echo "Waiting for prefill worker to initialize..."
 sleep 5
 
 # run decode worker with metrics on port 8082 (foreground)
-DYN_SYSTEM_ENABLED=true DYN_SYSTEM_PORT=8082 \
+DYN_SYSTEM_ENABLED=true DYN_SYSTEM_PORT=${DYN_SYSTEM_PORT2:-8082} \
 python3 -m dynamo.sglang \
   --model-path Qwen/Qwen3-0.6B \
   --served-model-name Qwen/Qwen3-0.6B \
diff --git a/examples/backends/sglang/launch/multimodal_agg.sh b/examples/backends/sglang/launch/multimodal_agg.sh
@@ -60,7 +60,7 @@ if [[ -n "$SERVED_MODEL_NAME" ]]; then
 fi
 
 # run ingress
-python3 -m dynamo.frontend --http-port=8000 &
+python3 -m dynamo.frontend --http-port=${DYN_FRONTEND_PORT:-8000} &
 DYNAMO_PID=$!
 
 # run SGLang multimodal processor
diff --git a/examples/backends/sglang/launch/multimodal_disagg.sh b/examples/backends/sglang/launch/multimodal_disagg.sh
@@ -60,7 +60,7 @@ if [[ -n "$SERVED_MODEL_NAME" ]]; then
 fi
 
 # run ingress
-python3 -m dynamo.frontend --http-port=8000 &
+python3 -m dynamo.frontend --http-port=${DYN_FRONTEND_PORT:-8000} &
 DYNAMO_PID=$!
 
 # run SGLang multimodal processor
diff --git a/examples/backends/trtllm/launch/agg.sh b/examples/backends/trtllm/launch/agg.sh
@@ -22,7 +22,7 @@ trap cleanup EXIT INT TERM
 
 
 # run frontend
-python3 -m dynamo.frontend --http-port 8000 &
+python3 -m dynamo.frontend --http-port ${DYN_FRONTEND_PORT:-8000} &
 DYNAMO_PID=$!
 
 # run worker
diff --git a/examples/backends/trtllm/launch/agg_metrics.sh b/examples/backends/trtllm/launch/agg_metrics.sh
@@ -19,11 +19,11 @@ cleanup() {
 trap cleanup EXIT INT TERM
 
 # Run frontend
-python3 -m dynamo.frontend --http-port 8000 &
+python3 -m dynamo.frontend --http-port ${DYN_FRONTEND_PORT:-8000} &
 DYNAMO_PID=$!
 
 # Run worker
-DYN_SYSTEM_ENABLED=true DYN_SYSTEM_PORT=8081 \
+DYN_SYSTEM_ENABLED=true DYN_SYSTEM_PORT=${DYN_SYSTEM_PORT:-8081} \
 python3 -m dynamo.trtllm \
   --model-path "$MODEL_PATH" \
   --served-model-name "$SERVED_MODEL_NAME" \
diff --git a/examples/backends/trtllm/launch/agg_router.sh b/examples/backends/trtllm/launch/agg_router.sh
@@ -19,7 +19,7 @@ trap cleanup EXIT INT TERM
 
 
 # run frontend
-python3 -m dynamo.frontend --router-mode kv --http-port 8000 &
+python3 -m dynamo.frontend --router-mode kv --http-port ${DYN_FRONTEND_PORT:-8000} &
 DYNAMO_PID=$!
 
 # run worker
diff --git a/examples/backends/trtllm/launch/disagg.sh b/examples/backends/trtllm/launch/disagg.sh
@@ -26,7 +26,7 @@ trap cleanup EXIT INT TERM
 
 
 # run frontend
-python3 -m dynamo.frontend --http-port 8000 &
+python3 -m dynamo.frontend --http-port ${DYN_FRONTEND_PORT:-8000} &
 DYNAMO_PID=$!
 
 # run prefill worker
diff --git a/examples/backends/trtllm/launch/disagg_router.sh b/examples/backends/trtllm/launch/disagg_router.sh
@@ -23,7 +23,7 @@ trap cleanup EXIT INT TERM
 
 
 # run frontend
-python3 -m dynamo.frontend --router-mode kv --http-port 8000 &
+python3 -m dynamo.frontend --router-mode kv --http-port ${DYN_FRONTEND_PORT:-8000} &
 DYNAMO_PID=$!
 
 
diff --git a/examples/backends/trtllm/launch/disagg_same_gpu.sh b/examples/backends/trtllm/launch/disagg_same_gpu.sh
@@ -49,12 +49,12 @@ trap cleanup EXIT INT TERM
 
 
 # run frontend
-python3 -m dynamo.frontend --http-port 8000 &
+python3 -m dynamo.frontend --http-port ${DYN_FRONTEND_PORT:-8000} &
 DYNAMO_PID=$!
 
 # run prefill worker (shares GPU with decode)
 CUDA_VISIBLE_DEVICES=$CUDA_VISIBLE_DEVICES \
-DYN_SYSTEM_ENABLED=true DYN_SYSTEM_PORT=8081 \
+DYN_SYSTEM_ENABLED=true DYN_SYSTEM_PORT=${DYN_SYSTEM_PORT2:-8082} \
 python3 -m dynamo.trtllm \
   --model-path "$MODEL_PATH" \
   --served-model-name "$SERVED_MODEL_NAME" \
@@ -67,7 +67,7 @@ PREFILL_PID=$!
 
 # run decode worker (shares GPU with prefill)
 CUDA_VISIBLE_DEVICES=$CUDA_VISIBLE_DEVICES \
-DYN_SYSTEM_ENABLED=true DYN_SYSTEM_PORT=8082 \
+DYN_SYSTEM_ENABLED=true DYN_SYSTEM_PORT=${DYN_SYSTEM_PORT2:-8082} \
 python3 -m dynamo.trtllm \
   --model-path "$MODEL_PATH" \
   --served-model-name "$SERVED_MODEL_NAME" \
diff --git a/examples/backends/trtllm/launch/epd_disagg.sh b/examples/backends/trtllm/launch/epd_disagg.sh
@@ -29,7 +29,7 @@ trap cleanup EXIT INT TERM
 
 
 # run frontend
-python3 -m dynamo.frontend --http-port 8000 &
+python3 -m dynamo.frontend --http-port ${DYN_FRONTEND_PORT:-8000} &
 DYNAMO_PID=$!
 
 # run encode worker
diff --git a/examples/backends/trtllm/launch/gpt_oss_disagg.sh b/examples/backends/trtllm/launch/gpt_oss_disagg.sh
@@ -15,7 +15,7 @@ trap 'echo Cleaning up...; kill 0' EXIT
 
 
 # run frontend
-python3 -m dynamo.frontend --router-mode round-robin --http-port 8000 &
+python3 -m dynamo.frontend --router-mode round-robin --http-port ${DYN_FRONTEND_PORT:-8000} &
 
 # With tensor_parallel_size=4, each worker needs 4 GPUs
 # run prefill worker
diff --git a/examples/backends/trtllm/performance_sweeps/scripts/start_frontend.sh b/examples/backends/trtllm/performance_sweeps/scripts/start_frontend.sh
@@ -21,5 +21,5 @@ sleep 2
 
 # Start OpenAI Frontend which will dynamically discover workers when they startup
 # NOTE: This is a blocking call.
-python3 -m dynamo.frontend --http-port 8000
+python3 -m dynamo.frontend --http-port ${DYN_FRONTEND_PORT:-8000}
 
diff --git a/examples/backends/vllm/launch/agg.sh b/examples/backends/vllm/launch/agg.sh
@@ -5,9 +5,9 @@ set -e
 trap 'echo Cleaning up...; kill 0' EXIT
 
 # run ingress
-python -m dynamo.frontend --http-port=8000 &
+python -m dynamo.frontend --http-port=${DYN_FRONTEND_PORT:-8000} &
 
 # run worker
 # --enforce-eager is added for quick deployment. for production use, need to remove this flag
-DYN_SYSTEM_ENABLED=true DYN_SYSTEM_PORT=8081 \
+DYN_SYSTEM_ENABLED=true DYN_SYSTEM_PORT=${DYN_SYSTEM_PORT:-8081} \
     python -m dynamo.vllm --model Qwen/Qwen3-0.6B --enforce-eager --connector none
diff --git a/examples/backends/vllm/launch/agg_kvbm.sh b/examples/backends/vllm/launch/agg_kvbm.sh
@@ -5,7 +5,7 @@ set -e
 trap 'echo Cleaning up...; kill 0' EXIT
 
 # run ingress
-python -m dynamo.frontend --http-port=8000 &
+python -m dynamo.frontend --http-port=${DYN_FRONTEND_PORT:-8000} &
 
 # run worker with KVBM enabled
 # NOTE: remove --enforce-eager for production use
diff --git a/examples/backends/vllm/launch/agg_kvbm_router.sh b/examples/backends/vllm/launch/agg_kvbm_router.sh
@@ -13,7 +13,7 @@ MODEL="Qwen/Qwen3-0.6B"
 # run frontend + KV router
 python -m dynamo.frontend \
     --router-mode kv \
-    --http-port 8000 \
+    --http-port ${DYN_FRONTEND_PORT:-8000} \
     --router-reset-states &
 
 # run workers with KVBM enabled
diff --git a/examples/backends/vllm/launch/agg_lmcache.sh b/examples/backends/vllm/launch/agg_lmcache.sh
@@ -5,7 +5,7 @@ set -e
 trap 'echo Cleaning up...; kill 0' EXIT
 
 # run ingress
-python -m dynamo.frontend --http-port=8000 &
+python -m dynamo.frontend --http-port=${DYN_FRONTEND_PORT:-8000} &
 
 # run worker with LMCache enabled
 ENABLE_LMCACHE=1 \
diff --git a/examples/backends/vllm/launch/agg_multimodal.sh b/examples/backends/vllm/launch/agg_multimodal.sh
@@ -53,7 +53,7 @@ else
 fi
 
 # run ingress
-python -m dynamo.frontend --http-port=8000 &
+python -m dynamo.frontend --http-port=${DYN_FRONTEND_PORT:-8000} &
 
 # To make Qwen2.5-VL fit in A100 40GB, set the following extra arguments
 EXTRA_ARGS=""
diff --git a/examples/backends/vllm/launch/agg_multimodal_llama.sh b/examples/backends/vllm/launch/agg_multimodal_llama.sh
@@ -8,7 +8,7 @@ trap 'echo Cleaning up...; kill 0' EXIT
 MODEL_NAME="meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8"
 
 # run ingress
-python -m dynamo.frontend --http-port=8000 &
+python -m dynamo.frontend --http-port=${DYN_FRONTEND_PORT:-8000} &
 
 # run processor
 python -m dynamo.vllm --multimodal-processor --model $MODEL_NAME --mm-prompt-template "<|image|>\n<prompt>" &
diff --git a/examples/backends/vllm/launch/agg_router.sh b/examples/backends/vllm/launch/agg_router.sh
@@ -14,7 +14,7 @@ BLOCK_SIZE=64
 # run frontend + KV router
 python -m dynamo.frontend \
     --router-mode kv \
-    --http-port 8000 \
+    --http-port ${DYN_FRONTEND_PORT:-8000} \
     --router-reset-states &
 
 # run workers
diff --git a/examples/backends/vllm/launch/dep.sh b/examples/backends/vllm/launch/dep.sh
@@ -5,7 +5,7 @@ set -e
 trap 'echo Cleaning up...; kill 0' EXIT
 
 # run ingress
-python -m dynamo.frontend --router-mode kv --http-port=8000 &
+python -m dynamo.frontend --router-mode kv --http-port=${DYN_FRONTEND_PORT:-8000} &
 
 # Data Parallel Attention / Expert Parallelism
 # Routing to DP workers managed by Dynamo
diff --git a/examples/backends/vllm/launch/disagg.sh b/examples/backends/vllm/launch/disagg.sh
@@ -5,7 +5,7 @@ set -e
 trap 'echo Cleaning up...; kill 0' EXIT
 
 # run ingress
-python -m dynamo.frontend --http-port=8000 &
+python -m dynamo.frontend --http-port=${DYN_FRONTEND_PORT:-8000} &
 
 # --enforce-eager is added for quick deployment. for production use, need to remove this flag
 CUDA_VISIBLE_DEVICES=0 python3 -m dynamo.vllm --model Qwen/Qwen3-0.6B --enforce-eager &
diff --git a/examples/backends/vllm/launch/disagg_kvbm.sh b/examples/backends/vllm/launch/disagg_kvbm.sh
@@ -5,7 +5,7 @@ set -e
 trap 'echo Cleaning up...; kill 0' EXIT
 
 # run ingress
-python -m dynamo.frontend --http-port=8000 &
+python -m dynamo.frontend --http-port=${DYN_FRONTEND_PORT:-8000} &
 
 # run decode worker on GPU 0, without enabling KVBM
 # NOTE: remove --enforce-eager for production use
diff --git a/examples/backends/vllm/launch/disagg_kvbm_2p2d.sh b/examples/backends/vllm/launch/disagg_kvbm_2p2d.sh
@@ -5,7 +5,7 @@ set -e
 trap 'echo Cleaning up...; kill 0' EXIT
 
 # run ingress with KV router
-python -m dynamo.frontend --router-mode kv --http-port=8000 &
+python -m dynamo.frontend --router-mode kv --http-port=${DYN_FRONTEND_PORT:-8000} &
 
 # run decode workers on GPU 0 and 1, without enabling KVBM
 # NOTE: remove --enforce-eager for production use
diff --git a/examples/backends/vllm/launch/disagg_kvbm_router.sh b/examples/backends/vllm/launch/disagg_kvbm_router.sh
@@ -13,7 +13,7 @@ MODEL="Qwen/Qwen3-0.6B"
 # run decode router with kv-overlap-score-weight 0 for pure load balancing
 python -m dynamo.frontend \
     --router-mode kv \
-    --http-port 8000 \
+    --http-port ${DYN_FRONTEND_PORT:-8000} \
     --kv-overlap-score-weight 0 \
     --router-reset-states &
 
diff --git a/examples/backends/vllm/launch/disagg_lmcache.sh b/examples/backends/vllm/launch/disagg_lmcache.sh
@@ -5,7 +5,7 @@ set -e
 trap 'echo Cleaning up...; kill 0' EXIT
 
 # run ingress with KV router
-python -m dynamo.frontend --router-mode kv --http-port=8000 &
+python -m dynamo.frontend --router-mode kv --http-port=${DYN_FRONTEND_PORT:-8000} &
 
 # run decode worker on GPU 0, without enabling LMCache
 CUDA_VISIBLE_DEVICES=0 python3 -m dynamo.vllm --model Qwen/Qwen3-0.6B &
diff --git a/examples/backends/vllm/launch/disagg_router.sh b/examples/backends/vllm/launch/disagg_router.sh
@@ -15,7 +15,7 @@ BLOCK_SIZE=64
 # The frontend will automatically detect prefill workers and activate an internal prefill router
 python -m dynamo.frontend \
     --router-mode kv \
-    --http-port 8000 \
+    --http-port ${DYN_FRONTEND_PORT:-8000} \
     --router-reset-states &
 
 # two decode workers
diff --git a/examples/backends/vllm/launch/dsr1_dep.sh b/examples/backends/vllm/launch/dsr1_dep.sh
@@ -83,7 +83,7 @@ trap 'echo Cleaning up...; kill 0' EXIT
 
 # run ingress if it's node 0
 if [ $NODE_RANK -eq 0 ]; then
-    DYN_LOG=debug python -m dynamo.frontend --router-mode kv --http-port=8000 2>&1 | tee $LOG_DIR/dsr1_dep_ingress.log &
+    DYN_LOG=debug python -m dynamo.frontend --router-mode kv --http-port=${DYN_FRONTEND_PORT:-8000} 2>&1 | tee $LOG_DIR/dsr1_dep_ingress.log &
 fi
 
 mkdir -p $LOG_DIR
diff --git a/tests/fault_tolerance/cancellation/test_sglang.py b/tests/fault_tolerance/cancellation/test_sglang.py
diff --git a/tests/fault_tolerance/cancellation/test_trtllm.py b/tests/fault_tolerance/cancellation/test_trtllm.py
diff --git a/tests/fault_tolerance/cancellation/test_vllm.py b/tests/fault_tolerance/cancellation/test_vllm.py
diff --git a/tests/fault_tolerance/cancellation/utils.py b/tests/fault_tolerance/cancellation/utils.py
diff --git a/tests/serve/launch/template_verifier.sh b/tests/serve/launch/template_verifier.sh
diff --git a/tests/utils/port_utils.py b/tests/utils/port_utils.py
