Merge pull request #944 from l1b0k/feat/ipam

feat: add warm up

Author: BSWANG

docs/warm-up.md

@@ -0,0 +1,99 @@
+# IP Warm-Up Configuration
+
+The IP warm-up feature allows Terway to pre-allocate a specified number of IP addresses when a node starts up. This helps reduce pod startup latency by having IPs ready in the pool before pods are scheduled.
+
+## Configuration
+
+The warm-up size can be configured through the `eni-config` ConfigMap:
+
+| Parameter | Description | Default Value | Example |
+|-----------|-------------|---------------|---------|
+| `ip_warm_up_size` | Number of IPs to pre-allocate during node warm-up | `0` (disabled) | `10`, `20` |
+
+## How It Works
+
+1. **Initialization**: When a node starts and `ip_warm_up_size` is configured (> 0), the controller initializes the warm-up state:
+   - `WarmUpTarget`: Set to the configured `ip_warm_up_size`
+   - `WarmUpAllocatedCount`: Tracks the number of IPs allocated via OpenAPI during warm-up
+   - `WarmUpCompleted`: Set to `false` initially
+
+2. **Allocation**: During reconciliation, if warm-up is not completed, the controller calculates additional IP demand to reach the warm-up target and allocates IPs accordingly.
+
+3. **Completion**: Warm-up is marked as completed when `WarmUpAllocatedCount >= WarmUpTarget`. Once completed, the warm-up process will not run again for that node.
+
+## Relationship with Pool Size
+
+**Important**: The `ip_warm_up_size` is independent of `min_pool_size` and `max_pool_size`. It can be set to a value larger or smaller than the pool size limits.
+
+However, setting `ip_warm_up_size` larger than `max_pool_size` is **not recommended** because:
+
+- The warm-up process will allocate IPs up to `ip_warm_up_size`
+- After warm-up completes, the pool management and idle IP reclaim policy will release excess IPs to maintain the pool within `min_pool_size` and `max_pool_size` boundaries
+- This results in unnecessary IP allocation and deallocation, wasting resources and API calls
+
+### Recommended Configuration
+
+```json
+{
+  "version": "1",
+  "max_pool_size": 20,
+  "min_pool_size": 5,
+  "ip_warm_up_size": 15
+}
+```
+
+In this example:
+
+- On node startup, 15 IPs will be pre-allocated (warm-up)
+- The pool will maintain between 5-20 IPs during normal operation
+- Since `ip_warm_up_size` (15) is within the pool size range (5-20), all pre-allocated IPs will be retained
+
+### Not Recommended Configuration
+
+```json
+{
+  "version": "1",
+  "max_pool_size": 10,
+  "min_pool_size": 2,
+  "ip_warm_up_size": 20
+}
+```
+
+In this example:
+
+- On node startup, 20 IPs will be pre-allocated (warm-up)
+- After warm-up completes, 10 IPs will be released because `max_pool_size` is only 10
+- This causes unnecessary IP churn and API calls
+
+## Status Fields
+
+The warm-up progress can be monitored through the Node CR status:
+
+| Field | Description |
+|-------|-------------|
+| `warmUpTarget` | The target number of IPs to allocate during warm-up |
+| `warmUpAllocatedCount` | Current count of IPs allocated via OpenAPI during warm-up |
+| `warmUpCompleted` | Whether warm-up has been completed |
+
+Example status:
+
+```yaml
+status:
+  warmUpTarget: 10
+  warmUpAllocatedCount: 10
+  warmUpCompleted: true
+```
+
+## Use Cases
+
+1. **Batch Job Scheduling**: When scheduling many pods simultaneously on a new node, pre-allocated IPs reduce waiting time.
+
+2. **Auto-scaling**: New nodes in auto-scaling groups can have IPs ready before workloads are scheduled.
+
+3. **Low-latency Requirements**: Applications requiring fast pod startup benefit from having IPs pre-allocated.
+
+## Notes
+
+- Warm-up only runs once per node lifecycle (when the node first joins the cluster)
+- If a node already has warm-up status initialized, changing `ip_warm_up_size` will not affect the ongoing warm-up
+- Warm-up progress is tracked independently of actual IP usage, ensuring consistent behavior even if IPs are allocated/deallocated during warm-up

pkg/apis/crds/network.alibabacloud.com_nodes.yaml

@@ -181,6 +181,9 @@ spec:
                       jitterFactor:
                         type: string
                     type: object
+                  warmUpSize:
+                    minimum: 0
+                    type: integer
                 type: object
             type: object
           status:
@@ -308,6 +311,12 @@ spec:
               nextSyncOpenAPITime:
                 format: date-time
                 type: string
+              warmUpAllocatedCount:
+                type: integer
+              warmUpCompleted:
+                type: boolean
+              warmUpTarget:
+                type: integer
             type: object
         type: object
     served: true

pkg/apis/crds/register.go

@@ -58,7 +58,7 @@ func getCRD(name string) apiextensionsv1.CustomResourceDefinition {
 		version = "v0.1.0"
 	case CRDNode:
 		crdBytes = crdsNode
-		version = "v0.6.1"
+		version = "v0.7.0"
 	case CRDNodeRuntime:
 		crdBytes = crdsNodeRuntime
 		version = "v0.1.0"

pkg/apis/network.alibabacloud.com/v1beta1/node_types.go

@@ -104,6 +104,9 @@ type PoolSpec struct {
 	PoolSyncPeriod string `json:"poolSyncPeriod,omitempty"`
 
 	Reclaim *IPReclaimPolicy `json:"reclaim,omitempty"`
+
+	// +kubebuilder:validation:Minimum=0
+	WarmUpSize int `json:"warmUpSize,omitempty"`
 }
 
 type IPReclaimPolicy struct {
@@ -199,6 +202,10 @@ type NodeStatus struct {
 	LastModifiedTime      metav1.Time     `json:"lastModifiedTime,omitempty"`
 	NextIdleIPReclaimTime metav1.Time     `json:"nextIdleIPReclaimTime,omitempty"`
 	NetworkInterfaces     map[string]*Nic `json:"networkInterfaces,omitempty"`
+
+	WarmUpTarget         int  `json:"warmUpTarget,omitempty"`
+	WarmUpAllocatedCount int  `json:"warmUpAllocatedCount,omitempty"`
+	WarmUpCompleted      bool `json:"warmUpCompleted,omitempty"`
 }
 
 // +genclient

pkg/controller/multi-ip/node/pool.go

@@ -324,6 +324,9 @@ func (n *ReconcileNode) Reconcile(ctx context.Context, request reconcile.Request
 
 	var errorList []error
 
+	// initialize warm-up if needed (for new nodes or existing nodes without warm-up status)
+	n.initializeWarmUp(node)
+
 	// do not block ipam
 	err = n.syncWithAPI(ctx, node)
 	if err != nil {
@@ -590,10 +593,13 @@ func (n *ReconcileNode) syncPods(ctx context.Context, podsMapper map[string]*Pod
 		errList = append(errList, err)
 	}
 
+	// 6. check and mark warm-up completion
+	n.checkWarmUpCompletion(node)
+
 	if utilerrors.NewAggregate(errList) != nil {
 		return utilerrors.NewAggregate(errList)
 	}
-	// 6. pool management sort eni and find the victim
+	// 7. pool management sort eni and find the victim
 
 	return n.adjustPool(ctx, node)
 }
@@ -826,8 +832,17 @@ func (n *ReconcileNode) addIP(ctx context.Context, unSucceedPods map[string]*Pod
 	// before create eni , we need to check the quota
 	options := getEniOptions(node)
 
+	// Calculate total demand including warm-up
+	totalDemand := len(normalPods) + node.Spec.Pool.MinPoolSize
+	if n.shouldPerformWarmUp(node) {
+		warmUpDemand := n.calculateWarmUpDemand(node)
+		totalDemand = max(totalDemand, warmUpDemand)
+
+		logr.FromContextOrDiscard(ctx).Info("warm up", "warmUpDemand", warmUpDemand, "totalDemand", totalDemand)
+	}
+
 	// handle trunk/secondary eni
-	assignEniWithOptions(ctx, node, len(normalPods)+node.Spec.Pool.MinPoolSize, options, func(option *eniOptions) bool {
+	assignEniWithOptions(ctx, node, totalDemand, options, func(option *eniOptions) bool {
 		return n.validateENI(ctx, option, []eniTypeKey{secondaryKey, trunkKey})
 	})
 	assignEniWithOptions(ctx, node, len(rdmaPods), options, func(option *eniOptions) bool {
@@ -1439,6 +1454,11 @@ func (n *ReconcileNode) createENI(ctx context.Context, node *networkv1beta1.Node
 	MetaCtx(ctx).Mutex.Lock()
 	node.Status.NetworkInterfaces[eni.NetworkInterfaceID] = networkInterface
 	// if changed , but we update failed , that case ,need to sync openAPI...
+
+	// Track OpenAPI allocations for warm-up
+	if !node.Status.WarmUpCompleted && node.Status.WarmUpTarget > 0 {
+		node.Status.WarmUpAllocatedCount += max(len(networkInterface.IPv4), len(networkInterface.IPv6))
+	}
 	MetaCtx(ctx).Mutex.Unlock()
 
 	MetaCtx(ctx).StatusChanged.Store(true)
@@ -1491,6 +1511,7 @@ func (n *ReconcileNode) assignIP(ctx context.Context, node *networkv1beta1.Node,
 					})
 				}
 			})
+
 			MetaCtx(ctx).Mutex.Unlock()
 
 			return err
@@ -1505,6 +1526,12 @@ func (n *ReconcileNode) assignIP(ctx context.Context, node *networkv1beta1.Node,
 					Status: networkv1beta1.IPStatusValid,
 				})
 			}
+
+			// Track OpenAPI allocations for warm-up
+			if !node.Status.WarmUpCompleted && node.Status.WarmUpTarget > 0 {
+				node.Status.WarmUpAllocatedCount += len(result)
+			}
+
 			MetaCtx(ctx).Mutex.Unlock()
 		}
 	}
@@ -1538,6 +1565,7 @@ func (n *ReconcileNode) assignIP(ctx context.Context, node *networkv1beta1.Node,
 					})
 				}
 			})
+
 			MetaCtx(ctx).Mutex.Unlock()
 
 			return err
@@ -1553,6 +1581,11 @@ func (n *ReconcileNode) assignIP(ctx context.Context, node *networkv1beta1.Node,
 					Status: networkv1beta1.IPStatusValid,
 				})
 			}
+
+			// Track OpenAPI allocations for warm-up
+			if !node.Spec.ENISpec.EnableIPv4 && !node.Status.WarmUpCompleted && node.Status.WarmUpTarget > 0 {
+				node.Status.WarmUpAllocatedCount += len(result)
+			}
 			MetaCtx(ctx).Mutex.Unlock()
 		}
 	}
@@ -1906,3 +1939,86 @@ func (n *ReconcileNode) waitIPGone(ctx context.Context, eni *networkv1beta1.Nic,
 		return !unfinished, nil
 	})
 }
+
+// initializeWarmUp initializes warm-up status for nodes
+// For new nodes with warm-up configured: set up tracking
+// For existing nodes without warm-up config or already initialized: mark as completed
+func (n *ReconcileNode) initializeWarmUp(node *networkv1beta1.Node) {
+	if node.Spec.Pool == nil {
+		return
+	}
+
+	warmUpSize := node.Spec.Pool.WarmUpSize
+
+	// For existing nodes that already have warm-up status initialized, do nothing
+	if node.Status.WarmUpTarget > 0 || node.Status.WarmUpCompleted {
+		return
+	}
+
+	// If no warm-up configured, mark as completed immediately
+	if warmUpSize <= 0 {
+		node.Status.WarmUpCompleted = true
+		return
+	}
+
+	// New node with warm-up configured
+	node.Status.WarmUpTarget = warmUpSize
+	node.Status.WarmUpAllocatedCount = 0
+	node.Status.WarmUpCompleted = false
+}
+
+// shouldPerformWarmUp checks if warm-up should be performed
+func (n *ReconcileNode) shouldPerformWarmUp(node *networkv1beta1.Node) bool {
+	if node.Status.WarmUpCompleted {
+		return false
+	}
+	if node.Status.WarmUpTarget <= 0 {
+		return false
+	}
+	return true
+}
+
+// calculateWarmUpDemand calculates the total IP demand for warm-up
+// Warmup progress is tracked via WarmUpAllocatedCount, not by counting current IPs
+// Returns total demand (currentIPs + remaining) for assignEniWithOptions
+func (n *ReconcileNode) calculateWarmUpDemand(node *networkv1beta1.Node) int {
+	if !n.shouldPerformWarmUp(node) {
+		return 0
+	}
+
+	// Calculate remaining IPs to allocate based on WarmUpAllocatedCount
+	remaining := node.Status.WarmUpTarget - node.Status.WarmUpAllocatedCount
+	if remaining <= 0 {
+		return 0
+	}
+
+	// Count current IPs to calculate total demand for assignEniWithOptions
+	// (assignEniWithOptions expects total and subtracts existing IPs internally)
+	currentTotal := 0
+	for _, eni := range node.Status.NetworkInterfaces {
+		if eni.Status != aliyunClient.ENIStatusInUse {
+			continue
+		}
+		if node.Spec.ENISpec.EnableIPv4 {
+			currentTotal += len(getAllocatable(eni.IPv4))
+		} else if node.Spec.ENISpec.EnableIPv6 {
+			currentTotal += len(getAllocatable(eni.IPv6))
+		}
+	}
+
+	return currentTotal + remaining
+}
+
+// checkWarmUpCompletion checks if warm-up has been completed and marks it
+func (n *ReconcileNode) checkWarmUpCompletion(node *networkv1beta1.Node) {
+	if node.Status.WarmUpCompleted {
+		return
+	}
+	if node.Status.WarmUpTarget <= 0 {
+		return
+	}
+
+	if node.Status.WarmUpAllocatedCount >= node.Status.WarmUpTarget {
+		node.Status.WarmUpCompleted = true
+	}
+}