diff --git a/pkg/scheduler/plugins/predicates/predicates.go b/pkg/scheduler/plugins/predicates/predicates.go index c1591590979..3c3d29277b0 100644 --- a/pkg/scheduler/plugins/predicates/predicates.go +++ b/pkg/scheduler/plugins/predicates/predicates.go @@ -104,11 +104,17 @@ type PredicatesPlugin struct { FilterPlugins map[string]fwk.FilterPlugin StableFilterPlugins map[string]fwk.FilterPlugin // Subset of FilterPlugins for cache-stable filters - PrefilterPlugins map[string]fwk.PreFilterPlugin + PreFilterPlugins map[string]fwk.PreFilterPlugin ReservePlugins map[string]fwk.ReservePlugin PreBindPlugins map[string]fwk.PreBindPlugin ScorePlugins map[string]nodescore.BaseScorePlugin ScoreWeights map[string]int // Weight for each score plugin + FilterOrder []string + StableFilterOrder []string + PreFilterOrder []string + ReserveOrder []string + PreBindOrder []string + ScoreOrder []string PredicateCache *predicateCache Handle fwk.Handle } @@ -413,7 +419,11 @@ func (pp *PredicatesPlugin) PrePredicate(task *api.TaskInfo, state *k8sframework } // Run all PreFilter plugins - for name, plugin := range pp.PrefilterPlugins { + for _, name := range pp.PreFilterOrder { + plugin, exists := pp.PreFilterPlugins[name] + if !exists { + continue + } _, status := plugin.PreFilter(context.TODO(), state, task.Pod, nodeInfoList) if err := handleSkipPrePredicatePlugin(status, state, task, name); err != nil { return err @@ -431,14 +441,46 @@ func (pp *PredicatesPlugin) InitPlugin() { scorePlugins := map[string]nodescore.BaseScorePlugin{} preBindPlugins := map[string]fwk.PreBindPlugin{} scoreWeights := map[string]int{} // Weight for each score plugin + var filterOrder []string + var stableFilterOrder []string + var preFilterOrder []string + var reserveOrder []string + var preBindOrder []string + var scoreOrder []string + + addFilterPlugin := func(name string, plugin fwk.FilterPlugin) { + filterPlugins[name] = plugin + filterOrder = append(filterOrder, name) + } + addStableFilterPlugin := func(name string, plugin fwk.FilterPlugin) { + stableFilterPlugins[name] = plugin + stableFilterOrder = append(stableFilterOrder, name) + } + addPreFilterPlugin := func(name string, plugin fwk.PreFilterPlugin) { + prefilterPlugins[name] = plugin + preFilterOrder = append(preFilterOrder, name) + } + addReservePlugin := func(name string, plugin fwk.ReservePlugin) { + reservePlugins[name] = plugin + reserveOrder = append(reserveOrder, name) + } + addPreBindPlugin := func(name string, plugin fwk.PreBindPlugin) { + preBindPlugins[name] = plugin + preBindOrder = append(preBindOrder, name) + } + addScorePlugin := func(name string, plugin nodescore.BaseScorePlugin, weight int) { + scorePlugins[name] = plugin + scoreOrder = append(scoreOrder, name) + scoreWeights[name] = weight + } // Initialize k8s plugins // TODO: Add more predicates, k8s.io/kubernetes/pkg/scheduler/framework/plugins/legacy_registry.go // 1. NodeUnschedulable (stable filter for cache) if plugin, err := nodeunschedulable.New(context.TODO(), nil, pp.Handle, pp.features); err == nil { nodeUnscheduleFilter := plugin.(*nodeunschedulable.NodeUnschedulable) - filterPlugins[nodeunschedulable.Name] = nodeUnscheduleFilter - stableFilterPlugins[nodeunschedulable.Name] = nodeUnscheduleFilter + addFilterPlugin(nodeunschedulable.Name, nodeUnscheduleFilter) + addStableFilterPlugin(nodeunschedulable.Name, nodeUnscheduleFilter) } else { klog.Errorf("Failed to init %s plugin %v", nodeunschedulable.Name, err) } @@ -450,8 +492,8 @@ func (pp *PredicatesPlugin) InitPlugin() { } if plugin, err := nodeaffinity.New(context.TODO(), &nodeAffinityArgs, pp.Handle, pp.features); err == nil { nodeAffinityFilter := plugin.(*nodeaffinity.NodeAffinity) - filterPlugins[nodeaffinity.Name] = nodeAffinityFilter - stableFilterPlugins[nodeaffinity.Name] = nodeAffinityFilter + addFilterPlugin(nodeaffinity.Name, nodeAffinityFilter) + addStableFilterPlugin(nodeaffinity.Name, nodeAffinityFilter) } else { klog.Errorf("Failed to init %s plugin %v", nodeaffinity.Name, err) } @@ -460,8 +502,8 @@ func (pp *PredicatesPlugin) InitPlugin() { if pp.enabledPredicates.nodePortEnable { if plugin, err := nodeports.New(context.TODO(), nil, pp.Handle, pp.features); err == nil { nodePortFilter := plugin.(*nodeports.NodePorts) - filterPlugins[nodeports.Name] = nodePortFilter - prefilterPlugins[nodeports.Name] = nodePortFilter + addFilterPlugin(nodeports.Name, nodePortFilter) + addPreFilterPlugin(nodeports.Name, nodePortFilter) } else { klog.Errorf("Failed to init %s plugin %v", nodeports.Name, err) } @@ -470,8 +512,8 @@ func (pp *PredicatesPlugin) InitPlugin() { if pp.enabledPredicates.taintTolerationEnable { if plugin, err := tainttoleration.New(context.TODO(), nil, pp.Handle, pp.features); err == nil { tolerationFilter := plugin.(*tainttoleration.TaintToleration) - filterPlugins[tainttoleration.Name] = tolerationFilter - stableFilterPlugins[tainttoleration.Name] = tolerationFilter + addFilterPlugin(tainttoleration.Name, tolerationFilter) + addStableFilterPlugin(tainttoleration.Name, tolerationFilter) } else { klog.Errorf("Failed to init %s plugin %v", tainttoleration.Name, err) } @@ -481,8 +523,8 @@ func (pp *PredicatesPlugin) InitPlugin() { plArgs := &config.InterPodAffinityArgs{} if plugin, err := interpodaffinity.New(context.TODO(), plArgs, pp.Handle, pp.features); err == nil { podAffinityFilter := plugin.(*interpodaffinity.InterPodAffinity) - filterPlugins[interpodaffinity.Name] = podAffinityFilter - prefilterPlugins[interpodaffinity.Name] = podAffinityFilter + addFilterPlugin(interpodaffinity.Name, podAffinityFilter) + addPreFilterPlugin(interpodaffinity.Name, podAffinityFilter) } else { klog.Errorf("Failed to init %s plugin %v", interpodaffinity.Name, err) } @@ -491,7 +533,7 @@ func (pp *PredicatesPlugin) InitPlugin() { if pp.enabledPredicates.nodeVolumeLimitsEnable { if plugin, err := nodevolumelimits.NewCSI(context.TODO(), nil, pp.Handle, pp.features); err == nil { nodeVolumeLimitsCSIFilter := plugin.(*nodevolumelimits.CSILimits) - filterPlugins[nodevolumelimits.CSIName] = nodeVolumeLimitsCSIFilter + addFilterPlugin(nodevolumelimits.CSIName, nodeVolumeLimitsCSIFilter) } else { klog.Errorf("Failed to init %s plugin %v", nodevolumelimits.CSIName, err) } @@ -500,7 +542,7 @@ func (pp *PredicatesPlugin) InitPlugin() { if pp.enabledPredicates.volumeZoneEnable { if plugin, err := volumezone.New(context.TODO(), nil, pp.Handle, pp.features); err == nil { volumeZoneFilter := plugin.(*volumezone.VolumeZone) - filterPlugins[volumezone.Name] = volumeZoneFilter + addFilterPlugin(volumezone.Name, volumeZoneFilter) } else { klog.Errorf("Failed to init %s plugin %v", volumezone.Name, err) } @@ -511,8 +553,8 @@ func (pp *PredicatesPlugin) InitPlugin() { ptsArgs := &config.PodTopologySpreadArgs{DefaultingType: config.SystemDefaulting} if plugin, err := podtopologyspread.New(context.TODO(), ptsArgs, pp.Handle, pp.features); err == nil { podTopologySpreadFilter := plugin.(*podtopologyspread.PodTopologySpread) - filterPlugins[podtopologyspread.Name] = podTopologySpreadFilter - prefilterPlugins[podtopologyspread.Name] = podTopologySpreadFilter + addFilterPlugin(podtopologyspread.Name, podTopologySpreadFilter) + addPreFilterPlugin(podtopologyspread.Name, podTopologySpreadFilter) } else { klog.Errorf("Failed to init %s plugin %v", podtopologyspread.Name, err) } @@ -534,12 +576,11 @@ func (pp *PredicatesPlugin) InitPlugin() { volumeBindingPluginInstance = plugin.(*vbcap.VolumeBinding) }) - filterPlugins[vbcap.Name] = volumeBindingPluginInstance - prefilterPlugins[vbcap.Name] = volumeBindingPluginInstance - reservePlugins[vbcap.Name] = volumeBindingPluginInstance - preBindPlugins[vbcap.Name] = volumeBindingPluginInstance - scorePlugins[vbcap.Name] = volumeBindingPluginInstance - scoreWeights[vbcap.Name] = vbArgs.Weight // Set weight from plugin args + addFilterPlugin(vbcap.Name, volumeBindingPluginInstance) + addPreFilterPlugin(vbcap.Name, volumeBindingPluginInstance) + addReservePlugin(vbcap.Name, volumeBindingPluginInstance) + addPreBindPlugin(vbcap.Name, volumeBindingPluginInstance) + addScorePlugin(vbcap.Name, volumeBindingPluginInstance, vbArgs.Weight) } // 10. DRA if pp.enabledPredicates.dynamicResourceAllocationEnable { @@ -550,19 +591,25 @@ func (pp *PredicatesPlugin) InitPlugin() { klog.Fatalf("failed to create dra plugin with err: %v", err) } dynamicResourceAllocationPlugin := plugin.(*dynamicresources.DynamicResources) - filterPlugins[dynamicresources.Name] = dynamicResourceAllocationPlugin - prefilterPlugins[dynamicresources.Name] = dynamicResourceAllocationPlugin - reservePlugins[dynamicresources.Name] = dynamicResourceAllocationPlugin - preBindPlugins[dynamicresources.Name] = dynamicResourceAllocationPlugin + addFilterPlugin(dynamicresources.Name, dynamicResourceAllocationPlugin) + addPreFilterPlugin(dynamicresources.Name, dynamicResourceAllocationPlugin) + addReservePlugin(dynamicresources.Name, dynamicResourceAllocationPlugin) + addPreBindPlugin(dynamicresources.Name, dynamicResourceAllocationPlugin) } pp.FilterPlugins = filterPlugins pp.StableFilterPlugins = stableFilterPlugins - pp.PrefilterPlugins = prefilterPlugins + pp.PreFilterPlugins = prefilterPlugins pp.ReservePlugins = reservePlugins pp.PreBindPlugins = preBindPlugins pp.ScorePlugins = scorePlugins pp.ScoreWeights = scoreWeights + pp.FilterOrder = filterOrder + pp.StableFilterOrder = stableFilterOrder + pp.PreFilterOrder = preFilterOrder + pp.ReserveOrder = reserveOrder + pp.PreBindOrder = preBindOrder + pp.ScoreOrder = scoreOrder } // Predicate runs all Filter plugins for the given task and node. @@ -596,7 +643,11 @@ func (pp *PredicatesPlugin) Predicate(task *api.TaskInfo, node *api.NodeInfo, st // Run all stable filter plugins (for cache) predicateStatus := make([]*api.Status, 0) - for name, plugin := range pp.StableFilterPlugins { + for _, name := range pp.StableFilterOrder { + plugin, exists := pp.StableFilterPlugins[name] + if !exists { + continue + } status := plugin.Filter(context.TODO(), state, task.Pod, nodeInfo) filterStatus := api.ConvertPredicateStatus(status) if filterStatus.Code != api.Success { @@ -636,7 +687,11 @@ func (pp *PredicatesPlugin) Predicate(task *api.TaskInfo, node *api.NodeInfo, st } // Run all Filter plugins (except those in StableFilterPlugins) - for name, plugin := range pp.FilterPlugins { + for _, name := range pp.FilterOrder { + plugin, exists := pp.FilterPlugins[name] + if !exists { + continue + } // Skip plugins that are already handled in predicateByStablefilter if _, isStable := pp.StableFilterPlugins[name]; isStable { continue @@ -669,7 +724,11 @@ func (pp *PredicatesPlugin) BatchNodeOrder(task *api.TaskInfo, nodes []fwk.NodeI nodeScores := make(map[string]float64, len(nodes)) // Run all Score plugins - for name, plugin := range pp.ScorePlugins { + for _, name := range pp.ScoreOrder { + plugin, exists := pp.ScorePlugins[name] + if !exists { + continue + } // Get normalizer (most plugins don't need normalization, use EmptyNormalizer by default) normalizer := &nodescore.EmptyNormalizer{} @@ -699,7 +758,11 @@ func (pp *PredicatesPlugin) BatchNodeOrder(task *api.TaskInfo, nodes []fwk.NodeI func (pp *PredicatesPlugin) runReservePlugins(ssn *framework.Session, event *framework.Event) { state := ssn.GetCycleState(event.Task.UID) - for name, plugin := range pp.ReservePlugins { + for _, name := range pp.ReserveOrder { + plugin, exists := pp.ReservePlugins[name] + if !exists { + continue + } status := plugin.Reserve(context.TODO(), state, event.Task.Pod, event.Task.Pod.Spec.NodeName) if !status.IsSuccess() { klog.Errorf("Reserve plugin %s failed for pod %s/%s: %v", name, event.Task.Namespace, event.Task.Name, status.AsError()) @@ -711,9 +774,16 @@ func (pp *PredicatesPlugin) runReservePlugins(ssn *framework.Session, event *fra func (pp *PredicatesPlugin) runUnReservePlugins(ssn *framework.Session, event *framework.Event) { state := ssn.GetCycleState(event.Task.UID) + pp.runUnreservePluginsWithState(context.TODO(), state, event.Task.Pod, event.Task.Pod.Spec.NodeName) +} - for _, plugin := range pp.ReservePlugins { - plugin.Unreserve(context.TODO(), state, event.Task.Pod, event.Task.Pod.Spec.NodeName) +func (pp *PredicatesPlugin) runUnreservePluginsWithState(ctx context.Context, state *k8sframework.CycleState, pod *v1.Pod, nodeName string) { + for i := len(pp.ReserveOrder) - 1; i >= 0; i-- { + plugin, exists := pp.ReservePlugins[pp.ReserveOrder[i]] + if !exists { + continue + } + plugin.Unreserve(ctx, state, pod, nodeName) } } @@ -751,7 +821,11 @@ func (pp *PredicatesPlugin) PreBind(ctx context.Context, bindCtx *cache.BindCont state := bindCtx.Extensions[pp.Name()].(*BindContextExtension).State // Run all PreBind plugins - for name, plugin := range pp.PreBindPlugins { + for _, name := range pp.PreBindOrder { + plugin, exists := pp.PreBindPlugins[name] + if !exists { + continue + } status := plugin.PreBind(ctx, state, bindCtx.TaskInfo.Pod, bindCtx.TaskInfo.Pod.Spec.NodeName) if !status.IsSuccess() { klog.Errorf("PreBind plugin %s failed for pod %s/%s: %v", name, bindCtx.TaskInfo.Namespace, bindCtx.TaskInfo.Name, status.AsError()) @@ -768,10 +842,7 @@ func (pp *PredicatesPlugin) PreBindRollBack(ctx context.Context, bindCtx *cache. } state := bindCtx.Extensions[pp.Name()].(*BindContextExtension).State - - for _, plugin := range pp.ReservePlugins { - plugin.Unreserve(ctx, state, bindCtx.TaskInfo.Pod, bindCtx.TaskInfo.Pod.Spec.NodeName) - } + pp.runUnreservePluginsWithState(ctx, state, bindCtx.TaskInfo.Pod, bindCtx.TaskInfo.Pod.Spec.NodeName) } func (pp *PredicatesPlugin) SetupBindContextExtension(state *k8sframework.CycleState, bindCtx *cache.BindContext) { diff --git a/pkg/scheduler/plugins/predicates/predicates_test.go b/pkg/scheduler/plugins/predicates/predicates_test.go index 39924e51045..354925597e7 100644 --- a/pkg/scheduler/plugins/predicates/predicates_test.go +++ b/pkg/scheduler/plugins/predicates/predicates_test.go @@ -17,16 +17,19 @@ limitations under the License. package predicates import ( + "context" "testing" "time" "github.com/stretchr/testify/assert" apiv1 "k8s.io/api/core/v1" schedulingv1 "k8s.io/api/scheduling/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/informers" k8sfake "k8s.io/client-go/kubernetes/fake" k8sframework "k8s.io/kube-scheduler/framework" + schedframework "k8s.io/kubernetes/pkg/scheduler/framework" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/dynamicresources" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/interpodaffinity" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeaffinity" @@ -43,6 +46,7 @@ import ( "volcano.sh/volcano/pkg/scheduler/actions/backfill" "volcano.sh/volcano/pkg/scheduler/actions/preempt" "volcano.sh/volcano/pkg/scheduler/api" + "volcano.sh/volcano/pkg/scheduler/cache" vbcap "volcano.sh/volcano/pkg/scheduler/capabilities/volumebinding" "volcano.sh/volcano/pkg/scheduler/conf" "volcano.sh/volcano/pkg/scheduler/framework" @@ -53,6 +57,37 @@ import ( "volcano.sh/volcano/pkg/scheduler/util" ) +type fakeFilterPlugin struct { + name string + message string +} + +func (p *fakeFilterPlugin) Name() string { + return p.name +} + +func (p *fakeFilterPlugin) Filter(_ context.Context, _ k8sframework.CycleState, _ *apiv1.Pod, _ k8sframework.NodeInfo) *k8sframework.Status { + return k8sframework.NewStatus(k8sframework.Unschedulable, p.message) +} + +type fakeReservePlugin struct { + name string + calls *[]string +} + +func (p *fakeReservePlugin) Name() string { + return p.name +} + +func (p *fakeReservePlugin) Reserve(_ context.Context, _ k8sframework.CycleState, _ *apiv1.Pod, _ string) *k8sframework.Status { + *p.calls = append(*p.calls, "reserve-"+p.name) + return k8sframework.NewStatus(k8sframework.Success) +} + +func (p *fakeReservePlugin) Unreserve(_ context.Context, _ k8sframework.CycleState, _ *apiv1.Pod, _ string) { + *p.calls = append(*p.calls, "unreserve-"+p.name) +} + func getWorkerAffinity() *apiv1.Affinity { return &apiv1.Affinity{ PodAntiAffinity: &apiv1.PodAntiAffinity{ @@ -490,10 +525,10 @@ func TestInitPlugin(t *testing.T) { } } - // Verify PrefilterPlugins + // Verify PreFilterPlugins for _, pluginName := range tt.expectInPrefilter { - if _, exists := pp.PrefilterPlugins[pluginName]; !exists { - t.Errorf("expected %s in PrefilterPlugins, but not found", pluginName) + if _, exists := pp.PreFilterPlugins[pluginName]; !exists { + t.Errorf("expected %s in PreFilterPlugins, but not found", pluginName) } } @@ -542,3 +577,111 @@ func TestInitPlugin(t *testing.T) { }) } } + +func TestPredicateFailureReasonAggregationOrderStable(t *testing.T) { + pp := New(nil).(*PredicatesPlugin) + pp.enabledPredicates = predicateEnable{ + nodeAffinityEnable: true, + nodePortEnable: true, + taintTolerationEnable: true, + } + pp.StableFilterPlugins = map[string]k8sframework.FilterPlugin{ + nodeunschedulable.Name: &fakeFilterPlugin{name: nodeunschedulable.Name, message: "stable-nodeunschedulable"}, + nodeaffinity.Name: &fakeFilterPlugin{name: nodeaffinity.Name, message: "stable-nodeaffinity"}, + tainttoleration.Name: &fakeFilterPlugin{name: tainttoleration.Name, message: "stable-tainttoleration"}, + } + pp.FilterPlugins = map[string]k8sframework.FilterPlugin{ + nodeunschedulable.Name: &fakeFilterPlugin{name: nodeunschedulable.Name, message: "stable-nodeunschedulable"}, + nodeaffinity.Name: &fakeFilterPlugin{name: nodeaffinity.Name, message: "stable-nodeaffinity"}, + nodeports.Name: &fakeFilterPlugin{name: nodeports.Name, message: "normal-nodeports"}, + tainttoleration.Name: &fakeFilterPlugin{name: tainttoleration.Name, message: "stable-tainttoleration"}, + } + // Predicate walks StableFilterOrder / FilterOrder (filled by InitPlugin in production). Mirror that order here. + pp.StableFilterOrder = []string{nodeunschedulable.Name, nodeaffinity.Name, tainttoleration.Name} + pp.FilterOrder = []string{nodeunschedulable.Name, nodeaffinity.Name, nodeports.Name, tainttoleration.Name} + + node := &apiv1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: "node-1"}, + Status: apiv1.NodeStatus{ + Capacity: apiv1.ResourceList{ + apiv1.ResourceCPU: resource.MustParse("4"), + apiv1.ResourceMemory: resource.MustParse("8Gi"), + apiv1.ResourcePods: resource.MustParse("100"), + }, + Allocatable: apiv1.ResourceList{ + apiv1.ResourceCPU: resource.MustParse("4"), + apiv1.ResourceMemory: resource.MustParse("8Gi"), + apiv1.ResourcePods: resource.MustParse("100"), + }, + }, + } + k8sNodeInfo := schedframework.NewNodeInfo() + k8sNodeInfo.SetNode(node) + pp.Handle = k8s.NewFramework(map[string]k8sframework.NodeInfo{"node-1": k8sNodeInfo}) + + task := api.NewTaskInfo(util.BuildPod("ns", "p1", "", apiv1.PodPending, nil, "pg", nil, nil)) + volcanoNode := api.NewNodeInfo(node) + + expected := []string{ + "stable-nodeunschedulable", + "stable-nodeaffinity", + "stable-tainttoleration", + "normal-nodeports", + } + for i := 0; i < 20; i++ { + err := pp.Predicate(task, volcanoNode, schedframework.NewCycleState()) + if err == nil { + t.Fatalf("expected predicate error, got nil") + } + fitErr, ok := err.(*api.FitError) + if !ok { + t.Fatalf("expected *api.FitError, got %T", err) + } + assert.Equal(t, expected, fitErr.Reasons()) + } +} + +func TestReserveRollbackOrderStable(t *testing.T) { + calls := make([]string, 0, 8) + pp := New(nil).(*PredicatesPlugin) + pp.enabledPredicates = predicateEnable{ + volumeBindingEnable: true, + dynamicResourceAllocationEnable: true, + } + pp.ReservePlugins = map[string]k8sframework.ReservePlugin{ + vbcap.Name: &fakeReservePlugin{name: vbcap.Name, calls: &calls}, + dynamicresources.Name: &fakeReservePlugin{name: dynamicresources.Name, calls: &calls}, + } + pp.ReserveOrder = []string{vbcap.Name, dynamicresources.Name} + + pod := util.BuildPod("ns", "p1", "", apiv1.PodPending, nil, "pg", nil, nil) + pod.Spec.NodeName = "node-1" + pod.Spec.Volumes = []apiv1.Volume{ + { + Name: "pvc-vol", + VolumeSource: apiv1.VolumeSource{ + PersistentVolumeClaim: &apiv1.PersistentVolumeClaimVolumeSource{ClaimName: "claim-1"}, + }, + }, + } + task := api.NewTaskInfo(pod) + + event := &framework.Event{Task: task} + pp.runReservePlugins(&framework.Session{}, event) + if event.Err != nil { + t.Fatalf("unexpected reserve error: %v", event.Err) + } + + bindCtx := &cache.BindContext{ + TaskInfo: task, + Extensions: map[string]cache.BindContextExtension{pp.Name(): &BindContextExtension{State: schedframework.NewCycleState()}}, + } + pp.PreBindRollBack(context.Background(), bindCtx) + + assert.Equal(t, []string{ + "reserve-VolumeBinding", + "reserve-DynamicResources", + "unreserve-DynamicResources", + "unreserve-VolumeBinding", + }, calls) +}