Merge branch 'main' into bis/dep-546-lora-aware-kv-routing-support

Author: biswapanda

components/src/dynamo/planner/utils/planner_core.py

@@ -24,7 +24,7 @@
     PrefillInterpolator,
 )
 from dynamo.planner.utils.pre_swept_results_utils import PreSweptResultsHelper
-from dynamo.planner.utils.prometheus import PrometheusAPIClient
+from dynamo.planner.utils.prometheus import MetricSource, PrometheusAPIClient
 from dynamo.planner.utils.trace_data_extractor import extract_metrics_from_mooncake
 from dynamo.runtime import DistributedRuntime
 from dynamo.runtime.logging import configure_dynamo_logging
@@ -150,9 +150,20 @@ def __init__(
                 else:
                     raise ValueError(f"Invalid environment: {args.environment}")
 
+            # Use backend metrics for vLLM (queries vllm:* metrics directly from workers)
+            # Use frontend metrics for other backends (queries dynamo_frontend_* metrics)
+            metric_source = (
+                MetricSource.VLLM
+                if args.backend.lower() == "vllm"
+                else MetricSource.FRONTEND
+            )
+            logger.info(
+                f"Initializing Prometheus client with metric_source='{metric_source}' for backend '{args.backend}'"
+            )
             self.prometheus_api_client = PrometheusAPIClient(
                 args.metric_pulling_prometheus_endpoint,
                 args.namespace,
+                metric_source=metric_source,
             )
 
         self.num_req_predictor = LOAD_PREDICTORS[args.load_predictor](

components/src/dynamo/planner/utils/prometheus.py

@@ -12,7 +12,7 @@
 from vllm.inputs import TokensPrompt
 from vllm.lora.request import LoRARequest
 from vllm.outputs import RequestOutput
-from vllm.sampling_params import SamplingParams
+from vllm.sampling_params import SamplingParams, StructuredOutputsParams
 from vllm.v1.engine.exceptions import EngineDeadError
 
 from dynamo.llm import (
@@ -82,8 +82,22 @@ def build_sampling_params(
     sampling_params = SamplingParams(**default_sampling_params)
     sampling_params.detokenize = False
 
-    # Apply sampling_options
+    # Handle guided_decoding - convert to StructuredOutputsParams
+    guided_decoding = request["sampling_options"].get("guided_decoding")
+    if guided_decoding is not None and isinstance(guided_decoding, dict):
+        sampling_params.structured_outputs = StructuredOutputsParams(
+            json=guided_decoding.get("json"),
+            regex=guided_decoding.get("regex"),
+            choice=guided_decoding.get("choice"),
+            grammar=guided_decoding.get("grammar"),
+            whitespace_pattern=guided_decoding.get("whitespace_pattern"),
+        )
+
+    # Apply remaining sampling_options
     for key, value in request["sampling_options"].items():
+        # Skip guided_decoding - already handled above
+        if key == "guided_decoding":
+            continue
         if value is not None and hasattr(sampling_params, key):
             setattr(sampling_params, key, value)
 

components/src/dynamo/vllm/handlers.py

@@ -67,15 +67,14 @@ def __init__(self, engine_client=None):
         self.default_payload = {
             "token_ids": [bos_token_id],
             "sampling_options": {
-                "max_tokens": 1,
                 "temperature": 0.0,
             },
             "stop_conditions": {
+                "max_tokens": 1,
                 "stop": None,
                 "stop_token_ids": None,
                 "include_stop_str_in_output": False,
                 "ignore_eos": False,
-                "min_tokens": 0,
             },
         }
         super().__init__()

components/src/dynamo/vllm/health_check.py

@@ -14,22 +14,16 @@
 ARG DEV_BASE=""
 FROM ${DEV_BASE} AS local-dev
 
-# Don't want dynamo to be editable, just change uid and gid.
-ENV USERNAME=dynamo
-ARG USER_UID
-ARG USER_GID
-ARG WORKSPACE_DIR=/workspace
-
-ARG DYNAMO_COMMIT_SHA
-ENV DYNAMO_COMMIT_SHA=$DYNAMO_COMMIT_SHA
-
-ARG ARCH
+# Switch to root for package installation (dev stage ends as dynamo user)
+USER root
+# Reset SHELL to non-login bash (dev stage uses login shell)
+SHELL ["/bin/bash", "-c"]
 
 # Update package lists and install developer utilities. Some of these may exist in the base image,
 # but to ensure consistency across all dev images, we explicitly list all required dev tools here.
 RUN apt-get update && apt-get install -y \
     # Development utilities
-    curl wget git vim nano \
+    curl wget git vim nano less \
     # System utilities
     htop nvtop tmux screen \
     # Network utilities
@@ -47,20 +41,6 @@ RUN apt-get update && apt-get install -y \
     # Shell utilities
     zsh fish bash-completion
 
-# https://code.visualstudio.com/remote/advancedcontainers/add-nonroot-user
-# Configure user with sudo access for Dev Container workflows
-RUN apt-get install -y sudo gnupg2 gnupg1 \
-    && echo "$USERNAME ALL=(root) NOPASSWD:ALL" > /etc/sudoers.d/$USERNAME \
-    && chmod 0440 /etc/sudoers.d/$USERNAME \
-    && mkdir -p /home/$USERNAME \
-    # Handle GID conflicts: if target GID exists and it's not our group, remove it
-    && (getent group $USER_GID | grep -v "^$USERNAME:" && groupdel $(getent group $USER_GID | cut -d: -f1) || true) \
-    # Create group if it doesn't exist, otherwise modify existing group
-    && (getent group $USERNAME > /dev/null 2>&1 && groupmod -g $USER_GID $USERNAME || groupadd -g $USER_GID $USERNAME) \
-    && usermod -u $USER_UID -g $USER_GID $USERNAME \
-    && chown -R $USERNAME:$USERNAME /home/$USERNAME \
-    && chsh -s /bin/bash $USERNAME
-
 # Install awk separately with fault tolerance
 # awk is a virtual package with multiple implementations (gawk, mawk, original-awk).
 # Separated because TensorRT-LLM builds failed on awk package conflicts.
@@ -71,12 +51,35 @@ RUN (apt-get install -y gawk || \
      echo "Warning: Could not install any awk implementation") && \
     (which awk && echo "awk successfully installed: $(which awk)" || echo "awk not available")
 
+
+# Don't want dynamo to be editable, just change uid and gid.
+ENV USERNAME=dynamo
+ARG USER_UID
+ARG USER_GID
+ARG WORKSPACE_DIR=/workspace
+ARG ARCH=amd64
+
 # Add NVIDIA devtools repository and install development tools
 RUN wget -qO - https://developer.download.nvidia.com/devtools/repos/ubuntu2404/${ARCH}/nvidia.pub | gpg --dearmor -o /etc/apt/keyrings/nvidia-devtools.gpg && \
     echo "deb [signed-by=/etc/apt/keyrings/nvidia-devtools.gpg] https://developer.download.nvidia.com/devtools/repos/ubuntu2404/${ARCH} /" | tee /etc/apt/sources.list.d/nvidia-devtools.list && \
     apt-get update && \
     apt-get install -y nsight-systems-2025.5.1
 
+# https://code.visualstudio.com/remote/advancedcontainers/add-nonroot-user
+# Configure user with sudo access for Dev Container workflows
+RUN apt-get install -y sudo gnupg2 gnupg1 \
+    && echo "$USERNAME ALL=(root) NOPASSWD:ALL" > /etc/sudoers.d/$USERNAME \
+    && chmod 0440 /etc/sudoers.d/$USERNAME \
+    && mkdir -p /home/$USERNAME \
+    # Handle GID conflicts: if target GID exists and it's not our group, remove it
+    && (getent group $USER_GID | grep -v "^$USERNAME:" && groupdel $(getent group $USER_GID | cut -d: -f1) || true) \
+    # Create group if it doesn't exist, otherwise modify existing group
+    && (getent group $USERNAME > /dev/null 2>&1 && groupmod -g $USER_GID $USERNAME || groupadd -g $USER_GID $USERNAME) \
+    && usermod -u $USER_UID -g $USER_GID -G 0 $USERNAME \
+    && chown -R $USERNAME:$USERNAME /home/$USERNAME \
+    && chsh -s /bin/bash $USERNAME
+
+
 # Clean up package lists at the end
 RUN rm -rf /var/lib/apt/lists/*
 
@@ -87,45 +90,26 @@ ENV WORKSPACE_DIR=${WORKSPACE_DIR}
 # Path configuration notes:
 # - DYNAMO_HOME: Main project directory (workspace mount point)
 # - CARGO_TARGET_DIR: Build artifacts in workspace/target for persistence
-# - CARGO_HOME: Must be in $HOME/.cargo (not workspace) because:
-#               * Workspace gets mounted to different paths where cargo binaries may not exist
-#               * Contains critical cargo binaries and registry that need consistent paths
-# - RUSTUP_HOME: Must be in $HOME/.rustup (not workspace) because:
-#                * Contains rust toolchain binaries that must be at expected system paths
-#                * Workspace mount point would break rustup's toolchain resolution
 # - PATH: Includes cargo binaries for rust tool access
 ENV HOME=/home/$USERNAME
 ENV DYNAMO_HOME=${WORKSPACE_DIR}
 ENV CARGO_TARGET_DIR=${WORKSPACE_DIR}/target
-ENV CARGO_HOME=${HOME}/.cargo
-ENV RUSTUP_HOME=${HOME}/.rustup
+# NOTE: CARGO_HOME and RUSTUP_HOME are already inherited from dev stage (Dockerfile.sglang|trtllm|vllm)
 ENV PATH=${CARGO_HOME}/bin:$PATH
 
-# Copy Rust toolchain from system directories to user home directories with proper ownership
-RUN rsync -a --chown=$USER_UID:$USER_GID /usr/local/rustup/ $RUSTUP_HOME/
-
-RUN rsync -a --chown=$USER_UID:$USER_GID /usr/local/cargo/ $CARGO_HOME/
-
-# Copy virtual environment with proper ownership using rsync instead of chown.
-# Why rsync instead of chown -R:
-# chown -R is extremely slow in Docker containers, especially on large directory trees
-# like Python virtual environments with thousands of files. This is a well-documented
-# Docker performance issue. rsync --chown is 3-4x faster as it sets ownership during copy.
-RUN rsync -a --chown=$USER_UID:$USER_GID ${VIRTUAL_ENV}/ /tmp/venv-temp/ && \
-    rm -rf ${VIRTUAL_ENV} && \
-    mv /tmp/venv-temp ${VIRTUAL_ENV}
-
-# At this point, we are executing as the ubuntu user
+# Switch to dynamo user (dev stage has umask 002, so files should already be group-writable)
 USER $USERNAME
 WORKDIR $HOME
 
 # https://code.visualstudio.com/remote/advancedcontainers/persist-bash-history
 RUN SNIPPET="export PROMPT_COMMAND='history -a' && export HISTFILE=$HOME/.commandhistory/.bash_history" \
     && mkdir -p $HOME/.commandhistory \
+    && chmod g+w $HOME/.commandhistory \
     && touch $HOME/.commandhistory/.bash_history \
     && echo "$SNIPPET" >> "$HOME/.bashrc"
 
-RUN mkdir -p /home/$USERNAME/.cache/
+RUN mkdir -p /home/$USERNAME/.cache/ \
+    && chmod g+w /home/$USERNAME/.cache/
 
 ENTRYPOINT ["/opt/nvidia/nvidia_entrypoint.sh"]
 CMD []

container/Dockerfile.local_dev

@@ -1,6 +1,21 @@
 # syntax=docker/dockerfile:1.10.0
 # SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
+#
+# Throughout this file, we make certain paths group-writable because this allows
+# both the dynamo user (UID 1000) and Dev Container users (UID != 1000) to work
+# properly without needing slow chown -R operations (which can add 2-10 extra
+# minutes).
+#
+# DEVELOPMENT PATHS THAT MUST BE GROUP-WRITABLE (for non-virtualenv containers):
+#   /workspace            - Users create/modify project files
+#   /home/dynamo          - Users create config/cache files
+#   /home/dynamo/.local   - SGLang uses $HOME/.local/lib/python3.10/site-packages for pip install
+#
+# HOW TO ACHIEVE GROUP-WRITABLE PERMISSIONS:
+# 1. SHELL + /etc/profile.d - Login shell sources umask 002 globally for all RUN commands (775/664)
+# 2. COPY --chmod=775       - Sets permissions on copied children (not destination)
+# 3. chmod g+w (no -R)      - Fixes destination dirs only (milliseconds vs minutes)
 
 # This section contains build arguments that are common and shared with
 # the plain Dockerfile, so they should NOT have a default. The source of truth is from build.sh.
@@ -190,18 +205,25 @@ RUN git clone --depth 1 --branch ${GDRCOPY_COMMIT} https://github.com/NVIDIA/gdr
 # Fix DeepEP IBGDA symlink
 RUN ln -sf /usr/lib/$(uname -m)-linux-gnu/libmlx5.so.1 /usr/lib/$(uname -m)-linux-gnu/libmlx5.so
 
-# Create dynamo user EARLY - before copying files, with group 0 for OpenShift compatibility
+# Create dynamo user with group 0 for OpenShift compatibility
 RUN userdel -r ubuntu > /dev/null 2>&1 || true \
     && useradd -m -s /bin/bash -g 0 dynamo \
     && [ `id -u dynamo` -eq 1000 ] \
     && mkdir -p /workspace /home/dynamo/.cache /opt/dynamo \
-    && chown -R dynamo: /sgl-workspace /workspace /home/dynamo /opt/dynamo \
-    && chmod -R g+w /sgl-workspace /workspace /home/dynamo/.cache /opt/dynamo
+    # Non-recursive chown - only the directories themselves, not contents
+    && chown dynamo:0 /sgl-workspace /workspace /home/dynamo /home/dynamo/.cache /opt/dynamo \
+    # No chmod needed: umask 002 handles new files, COPY --chmod handles copied content
+    # Set umask globally for all subsequent RUN commands (must be done as root before USER dynamo)
+    # NOTE: Setting ENV UMASK=002 does NOT work - umask is a shell builtin, not an environment variable
+    && mkdir -p /etc/profile.d && echo 'umask 002' > /etc/profile.d/00-umask.sh
 
 USER dynamo
 ENV HOME=/home/dynamo
+# This picks up the umask 002 from the /etc/profile.d/00-umask.sh file for subsequent RUN commands
+SHELL ["/bin/bash", "-l", "-o", "pipefail", "-c"]
 
-# Install SGLang (requires CUDA 12.8.1 or 12.9.1)
+# Install SGLang (requires CUDA 12.8.1 or 12.9.1). Note that when system-wide packages is not writable,
+# so it gets installed to ~/.local/lib/python<version>/site-packages.
 RUN python3 -m pip install --no-cache-dir --ignore-installed pip==25.3 setuptools==80.9.0 wheel==0.45.1 html5lib==1.1 six==1.17.0 \
     && git clone --depth 1 --branch v${SGLANG_COMMIT} https://github.com/sgl-project/sglang.git \
     && cd sglang \
@@ -279,8 +301,10 @@ RUN --mount=type=secret,id=aws-key-id,env=AWS_ACCESS_KEY_ID \
     NVSHMEM_DIR=${NVSHMEM_DIR} TORCH_CUDA_ARCH_LIST="9.0;10.0" pip install --no-build-isolation .
 
 # Copy rust installation from dynamo_base to avoid duplication efforts
-COPY --from=dynamo_base /usr/local/rustup /usr/local/rustup
-COPY --from=dynamo_base /usr/local/cargo /usr/local/cargo
+# Pattern: COPY --chmod=775 <path>; RUN chmod g+w <path> because COPY --chmod only affects <path>/*, not <path>
+COPY --from=dynamo_base --chown=dynamo:0 --chmod=775 /usr/local/rustup /usr/local/rustup
+COPY --from=dynamo_base --chown=dynamo:0 --chmod=775 /usr/local/cargo /usr/local/cargo
+RUN chmod g+w /usr/local/rustup /usr/local/cargo
 
 ENV RUSTUP_HOME=/usr/local/rustup \
     CARGO_HOME=/usr/local/cargo \
@@ -341,8 +365,9 @@ COPY --from=dynamo_base $NIXL_PREFIX $NIXL_PREFIX
 ENV PATH=/usr/local/bin/etcd/:/usr/local/cuda/nvvm/bin:${HOME}/.local/bin:$PATH
 
 # Install Dynamo wheels from dynamo_base wheelhouse
-COPY --chown=dynamo: benchmarks/ /opt/dynamo/benchmarks/
-COPY --chown=dynamo: --from=dynamo_base /opt/dynamo/wheelhouse/ /opt/dynamo/wheelhouse/
+# Pattern: COPY --chmod=775 <path>; chmod g+w <path> done later as root because COPY --chmod only affects <path>/*, not <path>
+COPY --chmod=775 --chown=dynamo:0 benchmarks/ /opt/dynamo/benchmarks/
+COPY --chmod=775 --chown=dynamo:0 --from=dynamo_base /opt/dynamo/wheelhouse/ /opt/dynamo/wheelhouse/
 RUN python3 -m pip install \
     /opt/dynamo/wheelhouse/ai_dynamo_runtime*.whl \
     /opt/dynamo/wheelhouse/ai_dynamo*any.whl \
@@ -361,29 +386,34 @@ RUN --mount=type=bind,source=./container/deps/requirements.txt,target=/tmp/requi
         --requirement /tmp/requirements.test.txt
 
 ## Copy attribution files and launch banner with correct ownership
-COPY --chown=dynamo: ATTRIBUTION* LICENSE /workspace/
+COPY --chmod=664 --chown=dynamo:0 ATTRIBUTION* LICENSE /workspace/
+
+# Copy tests, benchmarks, deploy and components for CI with correct ownership
+# Pattern: COPY --chmod=775 <path>; chmod g+w <path> done later as root because COPY --chmod only affects <path>/*, not <path>
+COPY --chmod=775 --chown=dynamo:0 pyproject.toml /workspace/
+COPY --chmod=775 --chown=dynamo:0 tests /workspace/tests
+COPY --chmod=775 --chown=dynamo:0 examples /workspace/examples
+COPY --chmod=775 --chown=dynamo:0 benchmarks /workspace/benchmarks
+COPY --chmod=775 --chown=dynamo:0 deploy /workspace/deploy
+COPY --chmod=775 --chown=dynamo:0 components/ /workspace/components/
+COPY --chmod=775 --chown=dynamo:0 recipes/ /workspace/recipes/
 
 # Setup launch banner in common directory accessible to all users
 RUN --mount=type=bind,source=./container/launch_message/runtime.txt,target=/opt/dynamo/launch_message.txt \
     sed '/^#\s/d' /opt/dynamo/launch_message.txt > /opt/dynamo/.launch_screen
 
 # Setup environment for all users
 USER root
-RUN chmod 755 /opt/dynamo/.launch_screen && \
+# Fix directory permissions: COPY --chmod only affects contents, not the directory itself
+RUN chmod g+w /workspace /workspace/* /opt/dynamo /opt/dynamo/* && \
+    chown dynamo:0 /workspace /opt/dynamo/ && \
+    chmod 755 /opt/dynamo/.launch_screen && \
     echo 'source /opt/dynamo/venv/bin/activate' >> /etc/bash.bashrc && \
     echo 'cat /opt/dynamo/.launch_screen' >> /etc/bash.bashrc
 
 USER dynamo
 
 # Copy tests, benchmarks, deploy and components for CI with correct ownership
-COPY --chown=dynamo: pyproject.toml /workspace/
-COPY --chown=dynamo: tests /workspace/tests
-COPY --chown=dynamo: examples /workspace/examples
-COPY --chown=dynamo: benchmarks /workspace/benchmarks
-COPY --chown=dynamo: deploy /workspace/deploy
-COPY --chown=dynamo: components/ /workspace/components/
-COPY --chown=dynamo: recipes/ /workspace/recipes/
-
 ARG DYNAMO_COMMIT_SHA
 ENV DYNAMO_COMMIT_SHA=$DYNAMO_COMMIT_SHA
 
@@ -420,6 +450,8 @@ ENV VIRTUAL_ENV=/opt/dynamo/venv \
     PATH="/opt/dynamo/venv/bin:${PATH}"
 
 USER root
+# venv permissions are handled by umask 002 set earlier
+
 # Install development tools and utilities
 RUN apt-get update -y && \
     apt-get install -y --no-install-recommends  \
@@ -478,7 +510,7 @@ RUN curl --retry 3 --retry-delay 2 -LSso /usr/local/bin/clang-format https://git
     && rm -rf clangd_18.1.3 clangd.zip
 
 # Editable install of dynamo
-COPY README.md hatch_build.py /workspace/
+COPY --chmod=664 pyproject.toml README.md hatch_build.py /workspace/
 RUN python3 -m pip install --no-deps -e .
 
 # Install Python development packages
@@ -496,4 +528,4 @@ RUN python3 -m pip install --no-cache-dir \
     tabulate
 
 ENTRYPOINT ["/opt/nvidia/nvidia_entrypoint.sh"]
-CMD []
+CMD []