Merge main-v0.14.1-committer into main #3064
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Sequencer - Hybrid Node System Test 2 | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| liveness_test_duration_sec: | |
| description: Time in seconds to keep the liveness test running. | |
| required: false | |
| default: 10 | |
| type: number | |
| pull_request: | |
| env: | |
| job_link: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| namespace: sequencer-hybrid-system-test-run-${{ github.run_number }}-attempt-${{ github.run_attempt }} | |
| layout: hybrid | |
| overlay: hybrid.testing.node-0 | |
| cluster_name: hybrid-system-test | |
| crate_triggers: "apollo_node,apollo_deployments,apollo_integration_tests" | |
| path_triggers: ".github/workflows/hybrid_system_test.yaml,scripts/*.py,scripts/system_tests/**/*.py,deployments/sequencer/**" | |
| path_triggers_exclude: "scripts/prod/**/*" | |
| pvc_storage_class_name: "premium-rwo" | |
| anvil_port: "8545" | |
| dockerfile_base_path: ./deployments/images/sequencer | |
| permissions: | |
| contents: read | |
| # On PR events, cancel existing CI runs on this same PR for this workflow. | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }}-${{ github.job }}-${{ github.event_name == 'workflow_dispatch' && github.run_id || 'pr' }} | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
| jobs: | |
| check-system-test-trigger: | |
| runs-on: namespace-profile-small-ubuntu-24-04-amd64 | |
| outputs: | |
| should_run: ${{ github.event_name == 'workflow_dispatch' && 'true' || steps.system_check.outputs.should_run }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| if: github.event_name != 'workflow_dispatch' | |
| with: | |
| fetch-depth: 0 | |
| - uses: actions/setup-python@v5 | |
| id: setup-pypy | |
| if: github.event_name != 'workflow_dispatch' | |
| with: | |
| python-version: "pypy3.9" | |
| cache: "pip" | |
| - run: pip install -r scripts/requirements.txt | |
| if: github.event_name != 'workflow_dispatch' | |
| - name: Check for system-test-triggering changes | |
| id: system_check | |
| if: github.event_name != 'workflow_dispatch' | |
| run: | | |
| echo "Checking if any system-test-triggering crates were modified..." | |
| OUTPUT_FILE=$(mktemp) | |
| python ./scripts/check_test_trigger.py --output_file $OUTPUT_FILE \ | |
| --commit_id ${{ github.event.pull_request.base.sha }} \ | |
| --crate_triggers ${{ env.crate_triggers }} \ | |
| --path_triggers ${{ env.path_triggers }} \ | |
| --path_triggers_exclude ${{ env.path_triggers_exclude }} | |
| should_run=$(cat "$OUTPUT_FILE") | |
| echo "Captured output: $should_run" | |
| echo "should_run=$should_run" >> $GITHUB_OUTPUT | |
| build_docker_images: | |
| runs-on: namespace-profile-small-ubuntu-24-04-amd64 | |
| needs: check-system-test-trigger | |
| if: needs.check-system-test-trigger.outputs.should_run == 'true' | |
| permissions: | |
| id-token: write | |
| contents: read | |
| strategy: | |
| fail-fast: true | |
| matrix: | |
| include: | |
| - svc: sequencer | |
| dockerfile: Dockerfile | |
| - svc: dummy_recorder | |
| dockerfile: dummy_recorder.Dockerfile | |
| - svc: dummy_eth_to_strk_oracle | |
| dockerfile: dummy_eth_to_strk_oracle.Dockerfile | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Build ${{ matrix.svc }} image remotely | |
| uses: docker/[email protected] | |
| with: | |
| context: . | |
| file: ${{ env.dockerfile_base_path }}/${{ matrix.dockerfile }} | |
| push: true | |
| tags: ${{ env.NSC_CONTAINER_REGISTRY }}/${{ matrix.svc }}:${{ github.sha }} | |
| system_test_hybrid: | |
| needs: | |
| - build_docker_images | |
| runs-on: namespace-profile-large-ubuntu-24-04-amd64 | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Create registries.yaml for nscr.io | |
| run: | | |
| cat << EOF > registries.yaml | |
| mirrors: | |
| nscr.io: | |
| endpoint: | |
| - https://nscr.io | |
| configs: | |
| nscr.io: | |
| auth: | |
| username: token | |
| password: $(cat $NSC_TOKEN_FILE | jq -r .bearer_token) | |
| EOF | |
| - name: Set up Rust cache | |
| uses: namespacelabs/nscloud-cache-action@v1 | |
| continue-on-error: true | |
| with: | |
| cache: rust | |
| # Install rust components. | |
| - uses: ./.github/actions/bootstrap | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build binaries | |
| run: cargo build --bin sequencer_node_setup --bin sequencer_simulator | |
| - name: Create k3d cluster (Local k8s) | |
| uses: AbsaOSS/[email protected] | |
| with: | |
| # Assumption: only one PR can run per machine at a time. | |
| cluster-name: ${{ env.cluster_name }} | |
| args: >- | |
| --verbose | |
| --agents 1 | |
| --no-lb | |
| --wait | |
| --timeout 120s | |
| --registry-config registries.yaml | |
| - name: Install crds | |
| working-directory: deployments/sequencer | |
| run: | | |
| echo "🔧 Installing CRDs..." | |
| kubectl apply -R -f ./resources/crds | |
| echo "✅ CRDs installed successfully." | |
| - name: Install local-path-provisioner (for PVC support) | |
| run: | | |
| echo "🔧 Installing local-path-provisioner..." | |
| kubectl apply -f https://raw.githubusercontent.com/rancher/local-path-provisioner/master/deploy/local-path-storage.yaml | |
| echo "⏳ Waiting for local-path-provisioner pod to be ready..." | |
| kubectl wait --for=condition=Ready pod -l app=local-path-provisioner -n local-path-storage --timeout=60s | |
| echo "✅ local-path-provisioner is ready." | |
| echo "📦 Setting default StorageClass and volumeBindingMode..." | |
| # First, set as default StorageClass | |
| kubectl patch storageclass local-path -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}' | |
| # Then, patch the volumeBindingMode in spec | |
| kubectl patch storageclass local-path --type='merge' -p '{"volumeBindingMode":"WaitForFirstConsumer"}' | |
| echo "📦 Verifying StorageClass configuration..." | |
| kubectl get storageclass local-path -o yaml | |
| echo "🎉 StorageClasses available:" | |
| kubectl get storageclass | |
| - name: Setup python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.10" | |
| cache: pipenv | |
| - name: Setup pipenv | |
| run: python3 -m pip install pipenv | |
| - name: Install dependencies with pipenv | |
| run: pipenv install kubernetes | |
| - name: Setup cdk8s-cli | |
| run: npm install -g cdk8s-cli | |
| - name: Deploy Dummy Recorder | |
| env: | |
| dummy_recorder_namespace: dummy-recorder | |
| working-directory: deployments/dummy_recorder | |
| run: | | |
| echo "Deploying Dummy Recorder..." | |
| pipenv install | |
| cdk8s import | |
| cdk8s synth --app "pipenv run python main.py --namespace ${{ env.dummy_recorder_namespace }} --image ${{ env.NSC_CONTAINER_REGISTRY }}/dummy_recorder:${{ github.sha }}" | |
| kubectl create namespace ${{ env.dummy_recorder_namespace }} | |
| kubectl apply -R -f ./dist | |
| echo "⏳ Waiting for Dummy Recorder to become ready..." | |
| # Wait for pod to be ready | |
| kubectl wait --namespace ${{ env.dummy_recorder_namespace }} --for=condition=Ready -l app=dummy-recorder pod --timeout=300s | |
| echo "🚀 Dummy Recorder deployed successfully." | |
| - name: Deploy Eth2Strk Oracle | |
| env: | |
| dummy_eth_to_strk_namespace: dummy-eth-to-strk | |
| working-directory: deployments/dummy_eth2strk_oracle | |
| run: | | |
| echo "Deploying Dummy Eth2Strk Oracle..." | |
| pipenv install | |
| cdk8s import | |
| cdk8s synth --app "pipenv run python main.py --namespace ${{ env.dummy_eth_to_strk_namespace }} --image ${{ env.NSC_CONTAINER_REGISTRY }}/dummy_eth_to_strk_oracle:${{ github.sha }}" | |
| kubectl create namespace ${{ env.dummy_eth_to_strk_namespace }} | |
| kubectl apply -R -f ./dist | |
| echo "⏳ Waiting for Dummy Eth2Strk Oracle to become ready..." | |
| kubectl wait --namespace ${{ env.dummy_eth_to_strk_namespace }} --for=condition=Ready -l app=dummy-eth2strk-oracle pod --timeout 60s | |
| echo "🚀 Dummy Eth2Strk Oracle deployed successfully." | |
| - name: Deploy Anvil | |
| env: | |
| namespace: anvil | |
| working-directory: deployments/anvil | |
| run: | | |
| echo "Deploying Anvil..." | |
| # Delete namespace if it already exists | |
| if kubectl get namespace "${{ env.namespace }}" &> /dev/null; then | |
| echo "🔁 Namespace '${{ env.namespace }}' already exists. Deleting it..." | |
| kubectl delete namespace "${{ env.namespace }}" | |
| echo "⏳ Waiting for namespace deletion..." | |
| while kubectl get namespace "${{ env.namespace }}" &> /dev/null; do sleep 2; done | |
| echo "✅ Namespace '${{ env.namespace }}' deleted." | |
| fi | |
| pipenv install | |
| cdk8s import | |
| cdk8s synth --app "pipenv run python main.py --namespace ${{ env.namespace }}" | |
| kubectl create namespace ${{ env.namespace }} | |
| kubectl apply -R -f ./dist | |
| echo "⏳ Waiting for Anvil to become ready..." | |
| kubectl wait --namespace ${{ env.namespace }} --for=condition=Ready -l app=anvil pod --timeout 60s | |
| echo "🚀 Anvil deployed successfully." | |
| echo "🔍 Extracting Anvil addresses from logs." | |
| ANVIL_POD=$(kubectl get pods -n "${{ env.namespace }}" -l app=anvil -o jsonpath="{.items[0].metadata.name}") | |
| ANVIL_LOGS=$(kubectl logs -n "${{ env.namespace }}" "$ANVIL_POD") | |
| echo "🔍 Extracting Anvil addresses from logs..." | |
| ANVIL_POD=$(kubectl get pods -n ${{ env.namespace }} -l app=anvil -o jsonpath="{.items[0].metadata.name}") | |
| ADDRESSES=$(kubectl logs -n ${{ env.namespace }} "$ANVIL_POD" | grep -oP '0x[a-fA-F0-9]{40}' | head -n 2) | |
| SENDER_ADDRESS=$(echo "$ADDRESSES" | head -n 1) | |
| RECEIVER_ADDRESS=$(echo "$ADDRESSES" | tail -n 1) | |
| echo "💡 SENDER_ADDRESS=$SENDER_ADDRESS" | |
| echo "💡 RECEIVER_ADDRESS=$RECEIVER_ADDRESS" | |
| echo "SENDER_ADDRESS=$SENDER_ADDRESS" >> "$GITHUB_ENV" | |
| echo "RECEIVER_ADDRESS=$RECEIVER_ADDRESS" >> "$GITHUB_ENV" | |
| - name: Install Anvil | |
| uses: foundry-rs/foundry-toolchain@v1 | |
| with: | |
| version: v0.3.0 | |
| - name: Restore executable permissions | |
| run: chmod +x ./target/debug/sequencer_node_setup ./target/debug/sequencer_simulator | |
| - name: Create storage files | |
| run: ./target/debug/sequencer_node_setup --output-base-dir ./output --data-prefix-path /data --n-consolidated 1 --n-hybrid 0 --n-distributed 0 | |
| - name: Generate k8s manifests | |
| working-directory: deployments/sequencer2 | |
| run: | | |
| pipenv install | |
| cdk8s import | |
| echo "Generating Kubernetes manifests using hybrid layout" | |
| cdk8s synth --app "pipenv run python main.py --namespace ${{ env.namespace }} -l ${{ env.layout }} -o ${{ env.overlay }} --image ${{ env.NSC_CONTAINER_REGISTRY }}/sequencer:${{ github.sha }}" | |
| - name: Deploy Sequencer | |
| working-directory: deployments/sequencer2 | |
| run: | | |
| echo "Deploying Sequencer..." | |
| kubectl create namespace ${{ env.namespace }} | |
| kubectl apply -R -f ./dist/ | |
| - name: Set default namespace | |
| run: kubectl config set-context --current --namespace ${{ env.namespace }} | |
| - name: Run readiness check | |
| run: pipenv run python ./scripts/system_tests/readiness_check2.py --layout ${{ env.layout }} --namespace ${{ env.namespace }} | |
| - name: Test sequencer is alive | |
| env: | |
| initial_delay_sec: 10 | |
| check_interval_sec: 2 | |
| check_timeout_sec: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.liveness_test_duration_sec || 10 }} | |
| run: | | |
| pipenv run python ./scripts/system_tests/liveness_check2.py \ | |
| --namespace ${{ env.namespace }} \ | |
| --layout ${{ env.layout }} \ | |
| --timeout ${{ env.check_timeout_sec }} \ | |
| --interval ${{ env.check_interval_sec }} | |
| - name: Copy state and restart pod | |
| run: pipenv run python ./scripts/system_tests/copy_state_and_restart2.py --layout ${{ env.layout }} --namespace ${{ env.namespace }} --data-dir "./output/data/node_0" | |
| - name: Port-forward Anvil pod to localhost:${{ env.anvil_port }} | |
| run: | | |
| echo "🔌 Setting up port-forward to Anvil..." | |
| ANVIL_POD=$(kubectl get pods -n anvil -l app=anvil -o jsonpath="{.items[0].metadata.name}") | |
| echo "🌐 Found Anvil pod: $ANVIL_POD" | |
| # Start port-forwarding in background and keep it running | |
| kubectl port-forward -n anvil "$ANVIL_POD" ${{ env.anvil_port }}:${{ env.anvil_port }} & | |
| echo "⏳ Waiting a few seconds to ensure port-forward is established..." | |
| sleep 2 | |
| - name: Print Second ConfigMap on the list | |
| run: | | |
| CONFIGMAP_NAME=$(kubectl get configmap -n "${{ env.namespace }}" -o jsonpath='{.items[1].metadata.name}') | |
| echo "ConfigMap: $CONFIGMAP_NAME" | |
| kubectl get configmap "$CONFIGMAP_NAME" -n "${{ env.namespace }}" -o yaml | |
| - name: Send transactions test | |
| env: | |
| RUST_LOG: debug | |
| RUST_BACKTRACE: full | |
| run: pipenv run python ./scripts/system_tests/sequencer_simulator2.py --state_sync_monitoring_endpoint_port 8082 --http_server_port 8080 --node_type "hybrid" --sender_address "${{ env.SENDER_ADDRESS }}" --receiver_address "${{ env.RECEIVER_ADDRESS }}" | |
| - name: List all pods in namespace ${{ env.namespace }} | |
| if: always() | |
| run: | | |
| # ============================================ | |
| # List all pods in the namespace | |
| # ============================================ | |
| kubectl get pods -n "$namespace" | |
| echo "---" | |
| echo "" | |
| echo "✅ Finished listing pods in namespace ${{ env.namespace }}" | |
| echo "---------------------------------------------" | |
| echo "" | |
| - name: Get container logs | |
| if: always() | |
| run: | | |
| pods=$(kubectl get pods -n "$namespace" -o jsonpath='{.items[*].metadata.name}') | |
| echo "📥 Getting pod logs and descriptions from namespace: $namespace" | |
| echo "---------------------------------------------" | |
| echo "" | |
| # ============================================ | |
| # Describe each pod | |
| # ============================================ | |
| echo "## 🔍 Pod Descriptions" | |
| echo "---------------------------------------------" | |
| echo "" | |
| for pod in $pods; do | |
| echo "### Pod: \`$pod\`" | |
| echo "" | |
| echo '```' | |
| kubectl describe pod -n "$namespace" "$pod" || echo "⚠️ Failed to describe pod $pod" | |
| echo '```' | |
| echo "" | |
| echo "---" | |
| echo "" | |
| done | |
| # ============================================ | |
| # Get logs for each pod | |
| # ============================================ | |
| echo "## 📜 Pod Logs" | |
| echo "---------------------------------------------" | |
| echo "" | |
| for pod in $pods; do | |
| echo "### Logs for pod: \`$pod\`" | |
| echo "" | |
| echo '```' | |
| kubectl logs -n "$namespace" "$pod" --tail=1000 || echo "⚠️ Failed to get logs for $pod" | |
| echo '```' | |
| echo "" | |
| echo "---" | |
| echo "" | |
| done | |
| echo "✅ Finished collecting pod information" | |
| echo "---------------------------------------------" | |
| echo "" |