central_systest_blobs: move blobs.json storage out of git, into GCS

.github/workflows/main.yml

@@ -126,6 +126,13 @@ jobs:
         run: echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> $GITHUB_ENV
       - run: pip install -r scripts/requirements.txt
 
+      # Auth with GCS for tests to download artifacts.
+      - id: auth
+        uses: "google-github-actions/auth@v2"
+        with:
+          credentials_json: ${{ secrets.SYSTEST_BLOBS_READ_ACCESS_KEY }}
+      - uses: "google-github-actions/setup-gcloud@v2"
+
       - name: "Run tests"
         run: |
           scripts/run_tests.py --command test --changes_only --include_dependencies --commit_id ${{ github.event.pull_request.base.sha || 'origin/main' }}

crates/central_systest_blobs/Cargo.toml

@@ -11,10 +11,13 @@ apollo_batcher.workspace = true
 apollo_batcher_types.workspace = true
 apollo_class_manager_types = { workspace = true, features = ["testing"] }
 apollo_consensus.workspace = true
-apollo_consensus_orchestrator.workspace = true
+apollo_consensus_orchestrator = { workspace = true, features = ["testing"] }
+apollo_infra_utils.workspace = true
 blockifier = { workspace = true, features = ["testing"] }
 blockifier_test_utils.workspace = true
 expect-test.workspace = true
+google-cloud-storage.workspace = true
+http.workspace = true
 serde_json.workspace = true
 starknet_api = { workspace = true, features = ["testing"] }
 starknet_patricia_storage = { workspace = true, features = ["testing"] }

crates/central_systest_blobs/resources/blob_file_generation

@@ -0,0 +1 @@
+2

crates/central_systest_blobs/src/cende_blob_regression_test.rs

@@ -1,4 +1,6 @@
+use std::path::PathBuf;
 use std::sync::{Arc, LazyLock};
+use std::{env, fs};
 
 use apollo_batcher::cende_client_types::{CendeBlockMetadata, CendePreconfirmedBlock};
 use apollo_batcher::pre_confirmed_cende_client::CendeWritePreconfirmedBlock;
@@ -10,13 +12,20 @@ use apollo_consensus_orchestrator::cende::{
     BlobParameters,
     InternalTransactionWithReceipt,
 };
+use apollo_infra_utils::compile_time_cargo_manifest_dir;
 use blockifier::blockifier_versioned_constants::VersionedConstants;
 use blockifier::bouncer::BouncerConfig;
 use blockifier::context::{BlockContext, ChainInfo};
 use blockifier::state::cached_state::StateMaps;
 use blockifier::test_utils::dict_state_reader::DictStateReader;
 use blockifier_test_utils::contracts::FeatureContract;
 use expect_test::expect_file;
+use google_cloud_storage::client::{Client, ClientConfig};
+use google_cloud_storage::http::error::ErrorResponse;
+use google_cloud_storage::http::objects::download::Range;
+use google_cloud_storage::http::objects::get::GetObjectRequest;
+use google_cloud_storage::http::objects::upload::{Media, UploadObjectRequest, UploadType};
+use google_cloud_storage::http::Error as GcsError;
 use starknet_api::block::{BlockHash, BlockHashAndNumber, BlockInfo, BlockNumber, BlockTimestamp};
 use starknet_api::block_hash::block_hash_calculator::PartialBlockHashComponents;
 use starknet_api::consensus_transaction::InternalConsensusTransaction;
@@ -28,18 +37,38 @@ use starknet_api::state::ThinStateDiff;
 use starknet_api::test_utils::TEST_SEQUENCER_ADDRESS;
 use starknet_patricia_storage::map_storage::MapStorage;
 
+const GCS_ERROR_CODE_NOT_FOUND: u16 = 404;
+
+const BLOBS_BUCKET_NAME: &str = "apollo-central-systest-blobs";
+const BLOBS_FILE_NAME: &str = "blobs.json";
+static BLOBS_GENERATION_FILE: LazyLock<PathBuf> = LazyLock::new(|| {
+    PathBuf::from(compile_time_cargo_manifest_dir!()).join("resources/blob_file_generation")
+});
+
 const N_TXS_PER_BLOCK: usize = 1;
 static CHAIN_ID: LazyLock<ChainId> =
     LazyLock::new(|| ChainId::Other("SN_PREINTEGRATION_SEPOLIA".to_string()));
 static CHAIN_INFO: LazyLock<ChainInfo> =
     LazyLock::new(|| ChainInfo { chain_id: CHAIN_ID.clone(), ..ChainInfo::create_for_testing() });
 
 const CHAIN_INFO_PATH: &str = "../resources/chain_info.json";
-const BLOB_LIST_PATH: &str = "../resources/blobs.json";
 const PRECONFIRMED_BLOCK_PATH: &str = "../resources/preconfirmed_block.json";
 
 type TxPair = (ExecutableAccountTx, InternalConsensusTransaction);
 
+/// ID of the current blobs file.
+fn current_generation() -> usize {
+    fs::read_to_string(&*BLOBS_GENERATION_FILE)
+        .unwrap_or_else(|error| panic!("Failed to read file {BLOBS_GENERATION_FILE:?}: {error}"))
+        .trim()
+        .parse()
+        .unwrap()
+}
+
+fn blobs_object_path(generation: usize) -> String {
+    format!("{generation}/{BLOBS_FILE_NAME}")
+}
+
 // =====================
 // Tx generation
 // =====================
@@ -182,6 +211,75 @@ async fn make_data() -> (Vec<AerospikeBlob>, CendeWritePreconfirmedBlock) {
     }
 }
 
+// =====================
+// Blob file storage
+// =====================
+
+async fn gcs_client() -> Client {
+    Client::new(ClientConfig::default().with_auth().await.expect(
+        "Failed to create GCS client config. Did you run `gcloud auth application-default login`?",
+    ))
+}
+
+async fn find_next_available_blobs_generation(client: &Client) -> usize {
+    let mut next_generation = current_generation() + 1;
+    loop {
+        match fetch_raw_blobs_at_generation(client, next_generation).await {
+            Ok(_) => next_generation += 1,
+            Err(GcsError::Response(ErrorResponse { code: GCS_ERROR_CODE_NOT_FOUND, .. })) => break,
+            Err(GcsError::HttpClient(error))
+                if error.status() == Some(http::StatusCode::NOT_FOUND) =>
+            {
+                break;
+            }
+            Err(e) => panic!("Failed to fetch blobs at generation {next_generation}: {e}"),
+        }
+    }
+    next_generation
+}
+
+async fn fetch_raw_blobs_at_generation(
+    client: &Client,
+    generation: usize,
+) -> Result<Vec<u8>, GcsError> {
+    client
+        .download_object(
+            &GetObjectRequest {
+                bucket: BLOBS_BUCKET_NAME.to_string(),
+                object: blobs_object_path(generation),
+                ..Default::default()
+            },
+            &Range::default(),
+        )
+        .await
+}
+
+/// Pushes the blobs to GCS.
+async fn bump_generation_and_store_blob_file(blobs: &[AerospikeBlob], client: &Client) {
+    let blobs_json = serde_json::to_string_pretty(blobs).unwrap();
+    let next_generation = find_next_available_blobs_generation(client).await;
+    client
+        .upload_object(
+            &UploadObjectRequest {
+                bucket: BLOBS_BUCKET_NAME.to_string(),
+                // Don't overwrite any existing file.
+                if_generation_match: Some(0),
+                ..Default::default()
+            },
+            blobs_json.into_bytes(),
+            &UploadType::Simple(Media::new(blobs_object_path(next_generation))),
+        )
+        .await
+        .unwrap();
+    fs::write(&*BLOBS_GENERATION_FILE, next_generation.to_string()).unwrap();
+}
+
+/// Fetches the blobs from GCS.
+async fn fetch_blob_file(client: &Client) -> Vec<AerospikeBlob> {
+    let blobs_json = fetch_raw_blobs_at_generation(client, current_generation()).await.unwrap();
+    serde_json::from_slice(&blobs_json).unwrap()
+}
+
 // =====================
 // Test
 // =====================
@@ -191,7 +289,18 @@ async fn test_make_data() {
     let (blobs, preconfirmed_block) = make_data().await;
     let chain_info = OsChainInfo::from(&*CHAIN_INFO).to_hex_map();
     expect_file![CHAIN_INFO_PATH].assert_eq(&serde_json::to_string_pretty(&chain_info).unwrap());
-    expect_file![BLOB_LIST_PATH].assert_eq(&serde_json::to_string_pretty(&blobs).unwrap());
     expect_file![PRECONFIRMED_BLOCK_PATH]
         .assert_eq(&serde_json::to_string_pretty(&preconfirmed_block).unwrap());
+
+    // Upload or download blobs depending on the fix mode.
+    let client = gcs_client().await;
+    if env::var("UPDATE_EXPECT").is_ok() {
+        bump_generation_and_store_blob_file(&blobs, &client).await;
+    } else {
+        let fetched_blobs = fetch_blob_file(&client).await;
+        assert_eq!(
+            blobs, fetched_blobs,
+            "Blobs mismatch. To fix, run the test with UPDATE_EXPECT=1."
+        );
+    }
 }