Fix installation for nvidia-nvshmem-cu12 (#14033)

Author: ch-wan

scripts/ci/ci_install_deepep.sh

@@ -46,9 +46,6 @@ if [ ! -e "$LIB_PATH/libmlx5.so" ]; then
 fi
 apt-get update && apt-get install -y libfabric-dev
 
-# Install NVSHMEM
-pip install nvidia-nvshmem-cu12==3.4.5 --force-reinstall
-
 # Install DeepEP
 DEEPEP_DIR=/root/.cache/deepep
 rm -rf ${DEEPEP_DIR}

scripts/ci/ci_install_dependency.sh

@@ -134,6 +134,10 @@ if [ "$IS_BLACKWELL" != "1" ]; then
     git clone --branch v0.5 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git
     $PIP_CMD install -e lmms-eval/ $PIP_INSTALL_SUFFIX
 fi
+
+# DeepEP depends on nvshmem 3.4.5
+$PIP_CMD install nvidia-nvshmem-cu12==3.4.5 --force-reinstall $PIP_INSTALL_SUFFIX
+
 # Cudnn with version less than 9.16.0.29 will cause performance regression on Conv3D kernel
 $PIP_CMD install nvidia-cudnn-cu12==9.16.0.29 --force-reinstall $PIP_INSTALL_SUFFIX
 $PIP_CMD uninstall xformers || true