Skip to content

Commit 99ed385

Browse files
visheshtanksalevisheshtanksale
authored
Adding HF token env var when using multi llm NIM without caching (#709)
Signed-off-by: Vishesh Tanksale <[email protected]>
1 parent 17f5fe4 commit 99ed385

File tree

7 files changed

+96
-32
lines changed

7 files changed

+96
-32
lines changed

‎api/apps/v1alpha1/nimservice_types.go‎

Lines changed: 11 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -282,6 +282,17 @@ func (n *NIMService) GetStandardEnv() []corev1.EnvVar {
282282
Name: "NIM_CACHE_PATH",
283283
Value: utils.DefaultModelStorePath,
284284
},
285+
{
286+
Name: "NGC_API_KEY",
287+
ValueFrom: &corev1.EnvVarSource{
288+
SecretKeyRef: &corev1.SecretKeySelector{
289+
LocalObjectReference: corev1.LocalObjectReference{
290+
Name: n.Spec.AuthSecret,
291+
},
292+
Key: "NGC_API_KEY",
293+
},
294+
},
295+
},
285296
{
286297
Name: "OUTLINES_CACHE_DIR",
287298
Value: "/tmp/outlines",
@@ -358,11 +369,6 @@ func (n *NIMService) getLWSCommonEnv() []corev1.EnvVar {
358369
return env
359370
}
360371

361-
// GetLWSLeaderEnvFrom returns the env from sources for the leader worker set.
362-
func (n *NIMService) GetLWSCommonEnvFrom() []corev1.EnvFromSource {
363-
return n.GetEnvFrom()
364-
}
365-
366372
func (n *NIMService) GetLWSLeaderEnv() []corev1.EnvVar {
367373
env := n.getLWSCommonEnv()
368374

@@ -567,23 +573,6 @@ func (n *NIMService) GetEnv() []corev1.EnvVar {
567573
return envVarList
568574
}
569575

570-
// GetEnvFrom returns merged slice of standard and user specified env from sources.
571-
func (n *NIMService) GetEnvFrom() []corev1.EnvFromSource {
572-
if n.Spec.AuthSecret != "" {
573-
return []corev1.EnvFromSource{
574-
{
575-
SecretRef: &corev1.SecretEnvSource{
576-
LocalObjectReference: corev1.LocalObjectReference{
577-
Name: n.Spec.AuthSecret,
578-
},
579-
},
580-
},
581-
}
582-
}
583-
// no secrets to source the env variables
584-
return []corev1.EnvFromSource{}
585-
}
586-
587576
// GetImage returns container image for the NIMService.
588577
func (n *NIMService) GetImage() string {
589578
return fmt.Sprintf("%s:%s", n.Spec.Image.Repository, n.Spec.Image.Tag)
@@ -1080,7 +1069,6 @@ func (n *NIMService) GetDeploymentParams() *rendertypes.DeploymentParams {
10801069
// Set container spec
10811070
params.ContainerName = n.GetContainerName()
10821071
params.Env = n.GetEnv()
1083-
params.EnvFrom = n.GetEnvFrom()
10841072
params.Args = n.GetArgs()
10851073
params.Command = n.GetCommand()
10861074
params.Resources = n.GetResources()
@@ -1152,8 +1140,6 @@ func (n *NIMService) GetLWSParams() *rendertypes.LeaderWorkerSetParams {
11521140
params.Command = n.GetCommand()
11531141
params.LeaderEnvs = n.GetLWSLeaderEnv()
11541142
params.WorkerEnvs = n.GetLWSWorkerEnv()
1155-
params.LeaderEnvFrom = n.GetLWSCommonEnvFrom()
1156-
params.WorkerEnvFrom = n.GetLWSCommonEnvFrom()
11571143
params.UserID = n.GetUserID()
11581144
params.GroupID = n.GetGroupID()
11591145
params.Image = n.GetImage()

‎internal/controller/platform/kserve/nimservice.go‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -477,6 +477,28 @@ func (r *NIMServiceReconciler) renderAndSyncInferenceService(ctx context.Context
477477
},
478478
}, isvcParams.Env)
479479
}
480+
// If NIMCache is not provided, and the model is from Hugging Face, add the HF_TOKEN to the environment variables
481+
if nimService.GetNIMCacheName() == "" {
482+
env := utils.FindEnvByValue(isvcParams.Env, "NIM_MODEL_NAME")
483+
if env != nil {
484+
if strings.HasPrefix(env.Value, "hf://") {
485+
isvcParams.Env = utils.RemoveEnvVar(isvcParams.Env, "NGC_API_KEY")
486+
isvcParams.Env = utils.MergeEnvVars(isvcParams.Env, []corev1.EnvVar{
487+
{
488+
Name: "HF_TOKEN",
489+
ValueFrom: &corev1.EnvVarSource{
490+
SecretKeyRef: &corev1.SecretKeySelector{
491+
LocalObjectReference: corev1.LocalObjectReference{
492+
Name: nimService.Spec.AuthSecret,
493+
},
494+
Key: "HF_TOKEN",
495+
},
496+
},
497+
},
498+
})
499+
}
500+
}
501+
}
480502
// Setup volume mounts with model store
481503
isvcParams.Volumes = nimService.GetVolumes(modelPVC)
482504
isvcParams.VolumeMounts = nimService.GetVolumeMounts(modelPVC)

‎internal/controller/platform/standalone/nimservice.go‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -464,6 +464,28 @@ func (r *NIMServiceReconciler) reconcileNIMService(ctx context.Context, nimServi
464464
Value: utils.DefaultModelStorePath,
465465
}}, deploymentParams.Env)
466466
}
467+
// If NIMCache is not provided, and the model is from Hugging Face, add the HF_TOKEN to the environment variables
468+
if nimService.GetNIMCacheName() == "" {
469+
env := utils.FindEnvByValue(deploymentParams.Env, "NIM_MODEL_NAME")
470+
if env != nil {
471+
if strings.HasPrefix(env.Value, "hf://") {
472+
deploymentParams.Env = utils.RemoveEnvVar(deploymentParams.Env, "NGC_API_KEY")
473+
deploymentParams.Env = utils.MergeEnvVars(deploymentParams.Env, []corev1.EnvVar{
474+
{
475+
Name: "HF_TOKEN",
476+
ValueFrom: &corev1.EnvVarSource{
477+
SecretKeyRef: &corev1.SecretKeySelector{
478+
LocalObjectReference: corev1.LocalObjectReference{
479+
Name: nimService.Spec.AuthSecret,
480+
},
481+
Key: "HF_TOKEN",
482+
},
483+
},
484+
},
485+
})
486+
}
487+
}
488+
}
467489
// Setup volume mounts with model store
468490
deploymentParams.Volumes = nimService.GetVolumes(modelPVC)
469491
deploymentParams.VolumeMounts = nimService.GetVolumeMounts(modelPVC)

‎internal/controller/platform/standalone/nimservice_test.go‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1107,6 +1107,17 @@ var _ = Describe("NIMServiceReconciler for a standalone platform", func() {
11071107
Name: "NIM_CACHE_PATH",
11081108
Value: "/model-store",
11091109
},
1110+
{
1111+
Name: "NGC_API_KEY",
1112+
ValueFrom: &corev1.EnvVarSource{
1113+
SecretKeyRef: &corev1.SecretKeySelector{
1114+
LocalObjectReference: corev1.LocalObjectReference{
1115+
Name: "",
1116+
},
1117+
Key: "NGC_API_KEY",
1118+
},
1119+
},
1120+
},
11101121
{
11111122
Name: "OUTLINES_CACHE_DIR",
11121123
Value: "/tmp/outlines",
@@ -1198,6 +1209,17 @@ var _ = Describe("NIMServiceReconciler for a standalone platform", func() {
11981209
Name: "NIM_CACHE_PATH",
11991210
Value: "/model-store",
12001211
},
1212+
{
1213+
Name: "NGC_API_KEY",
1214+
ValueFrom: &corev1.EnvVarSource{
1215+
SecretKeyRef: &corev1.SecretKeySelector{
1216+
LocalObjectReference: corev1.LocalObjectReference{
1217+
Name: "",
1218+
},
1219+
Key: "NGC_API_KEY",
1220+
},
1221+
},
1222+
},
12011223
{
12021224
Name: "OUTLINES_CACHE_DIR",
12031225
Value: "/tmp/outlines",

‎internal/render/types/types.go‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,6 @@ type DeploymentParams struct {
7171
Volumes []corev1.Volume
7272
VolumeMounts []corev1.VolumeMount
7373
Env []corev1.EnvVar
74-
EnvFrom []corev1.EnvFromSource
7574
Resources *corev1.ResourceRequirements
7675
NodeSelector map[string]string
7776
Tolerations []corev1.Toleration
@@ -113,8 +112,6 @@ type LeaderWorkerSetParams struct {
113112
LeaderVolumeMounts []corev1.VolumeMount
114113
WorkerEnvs []corev1.EnvVar
115114
LeaderEnvs []corev1.EnvVar
116-
WorkerEnvFrom []corev1.EnvFromSource
117-
LeaderEnvFrom []corev1.EnvFromSource
118115
Resources *corev1.ResourceRequirements
119116
NodeSelector map[string]string
120117
Tolerations []corev1.Toleration

‎internal/utils/utils.go‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -494,3 +494,22 @@ func IsVersionGreaterThanOrEqual(version string, minVersion string) bool {
494494
}
495495
return cv.AtLeast(mv)
496496
}
497+
498+
func RemoveEnvVar(envs []corev1.EnvVar, name string) []corev1.EnvVar {
499+
result := make([]corev1.EnvVar, 0, len(envs))
500+
for _, e := range envs {
501+
if e.Name != name {
502+
result = append(result, e)
503+
}
504+
}
505+
return result
506+
}
507+
508+
func FindEnvByValue(envs []corev1.EnvVar, key string) *corev1.EnvVar {
509+
for i := range envs {
510+
if envs[i].Name == key {
511+
return &envs[i]
512+
}
513+
}
514+
return nil
515+
}

‎manifests/deployment.yaml‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -98,10 +98,6 @@ spec:
9898
{{- if .Env }}
9999
{{- .Env | yaml | nindent 10 }}
100100
{{- end }}
101-
envFrom:
102-
{{- if .EnvFrom }}
103-
{{- .EnvFrom | yaml | nindent 10 }}
104-
{{- end }}
105101
{{- with .Resources }}
106102
resources:
107103
{{ . | yaml | nindent 10 }}

0 commit comments

Comments
 (0)