500k test

Author: dean-starkware

deployments/dean-batching/batching-test-500k.yaml

@@ -0,0 +1,113 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: batching-test-500k
+  namespace: dean-batching
+  labels:
+    test: 500k-blocks
+    disk-type: hyperdisk-balanced
+spec:
+  backoffLimit: 0
+  template:
+    metadata:
+      labels:
+        app: batching-test-500k
+        test: 500k-blocks
+    spec:
+      restartPolicy: Never
+      nodeSelector:
+        role: "apollo-core-service-c4d-standard-8"
+      tolerations:
+        - key: key
+          operator: "Equal"
+          value: "apollo-core-service-c4d-standard-8"
+          effect: "NoSchedule"
+      initContainers:
+      - name: fix-permissions
+        image: busybox
+        command: ["sh", "-c", "mkdir -p /data/workspace && chmod -R 777 /data && ls -la /data/ && echo 'Permissions fixed!'"]
+        volumeMounts:
+        - name: data
+          mountPath: /data
+      containers:
+      - name: batching-test
+        image: ghcr.io/starkware-libs/sequencer/sequencer:dean-k8s_batching_test-fd3d4f1
+        workingDir: /tmp
+        env:
+        - name: RUST_LOG
+          value: info
+        - name: BLOCKS_TO_SYNC
+          value: "500000"
+        - name: BATCH_SIZE
+          value: "1000"
+        - name: PATH
+          value: /app/target/release:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+        command: ["/bin/bash"]
+        args:
+        - -c
+        - |
+          set -e
+          
+          echo "========================================="
+          echo "DEAN'S BATCHING TEST: 0 → 500K"
+          echo "Hyperdisk Balanced 1TB Performance Test"
+          echo "========================================="
+          echo ""
+          echo "Configuration:"
+          echo "  Blocks to sync: 500,000"
+          echo "  Batch size: 1,000"
+          echo "  Disk: Hyperdisk Balanced 1TB (80k IOPS, 1200 MB/s)"
+          echo ""
+          
+          # Copy test script to /tmp first (always writable)
+          echo "Setting up test script..."
+          cp /test-scripts/test_batching.sh /tmp/test_batching.sh
+          chmod +x /tmp/test_batching.sh
+          
+          # Clean workspace
+          echo "Cleaning workspace..."
+          cd /data/workspace
+          find . -mindepth 1 -maxdepth 1 ! -name 'lost+found' -exec rm -rf {} + 2>/dev/null || true
+          
+          # Verify clean
+          if [ -d "data_with_batching" ] || [ -d "data_without_batching" ]; then
+            echo "ERROR: Workspace not clean!"
+            exit 1
+          fi
+          echo "✓ Workspace clean"
+          echo ""
+          
+          # Run the test script from /tmp
+          cd /data/workspace
+          /tmp/test_batching.sh 2>&1 | tee test_output.log
+          
+          echo ""
+          echo "========================================="
+          echo "TEST COMPLETE"
+          echo "========================================="
+        resources:
+          requests:
+            cpu: "4"
+            memory: "16Gi"
+          limits:
+            cpu: "8"
+            memory: "32Gi"
+        volumeMounts:
+        - name: data
+          mountPath: /data
+        - name: configs
+          mountPath: /configs
+          readOnly: true
+        - name: test-scripts
+          mountPath: /test-scripts
+      volumes:
+      - name: data
+        persistentVolumeClaim:
+          claimName: dean-hyperdisk-extreme-pvc
+      - name: configs
+        configMap:
+          name: sequencer-configs
+      - name: test-scripts
+        configMap:
+          name: dean-test-scripts
+

deployments/dean-batching/storage-class-balanced-1tb.yaml

@@ -0,0 +1,13 @@
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: hyperdisk-balanced-1000gb
+parameters:
+  type: hyperdisk-balanced
+  provisioned-iops-on-create: "80000"  # Higher IOPS for 1TB disk
+  provisioned-throughput-on-create: "1200Mi"  # Max throughput for Balanced
+provisioner: pd.csi.storage.gke.io
+reclaimPolicy: Retain
+volumeBindingMode: WaitForFirstConsumer
+allowVolumeExpansion: true
+

test_batching.sh

@@ -41,7 +41,8 @@ cleanup() {
 }
 
 # Set trap to cleanup on exit or interrupt
-trap cleanup EXIT INT TERM
+# DISABLED TO PROTECT TEST DATA - UNCOMMENT IF YOU WANT AUTO-CLEANUP
+# trap cleanup EXIT INT TERM
 
 # Function to check disk space
 check_disk_space() {