blockifier: add class cache metrics struct and inject from batcher (#10287)

Author: ArniStarkware

Cargo.lock

@@ -54,6 +54,7 @@ use crate::cende_client_types::{StarknetClientStateDiff, StarknetClientTransacti
 use crate::metrics::{
     record_block_close_reason,
     BlockCloseReason,
+    BATCHER_CLASS_CACHE_METRICS,
     PROPOSER_DEFERRED_TXS,
     VALIDATOR_WASTED_TXS,
 };
@@ -667,8 +668,8 @@ pub struct BlockMetadata {
 
 // Type definitions for the abort channel required to abort the block builder.
 pub type AbortSignalSender = tokio::sync::oneshot::Sender<()>;
-pub type BatcherWorkerPool =
-    Arc<WorkerPool<CachedState<StateReaderAndContractManager<ApolloReader>>>>;
+pub type ApolloStateReaderAndContractManager = StateReaderAndContractManager<ApolloReader>;
+pub type BatcherWorkerPool = Arc<WorkerPool<CachedState<ApolloStateReaderAndContractManager>>>;
 
 /// The BlockBuilderFactoryTrait is responsible for creating a new block builder.
 #[cfg_attr(test, automock)]
@@ -703,9 +704,8 @@ impl BlockBuilderFactory {
         &self,
         block_metadata: BlockMetadata,
         runtime: tokio::runtime::Handle,
-    ) -> BlockBuilderResult<
-        ConcurrentTransactionExecutor<StateReaderAndContractManager<ApolloReader>>,
-    > {
+    ) -> BlockBuilderResult<ConcurrentTransactionExecutor<ApolloStateReaderAndContractManager>>
+    {
         info!(
             "preprocess and create transaction executor for block {}",
             block_metadata.block_info.block_number
@@ -728,6 +728,7 @@ impl BlockBuilderFactory {
         let state_reader = StateReaderAndContractManager {
             state_reader: apollo_reader,
             contract_class_manager: self.contract_class_manager.clone(),
+            class_cache_metrics: BATCHER_CLASS_CACHE_METRICS,
         };
 
         let executor = ConcurrentTransactionExecutor::start_block(

crates/apollo_batcher/src/block_builder.rs

@@ -8,9 +8,8 @@ use apollo_infra::metrics::{
 };
 use apollo_metrics::{define_infra_metrics, define_metrics, generate_permutation_labels};
 use blockifier::metrics::{
+    CacheMetrics,
     CALLS_RUNNING_NATIVE,
-    CLASS_CACHE_HITS,
-    CLASS_CACHE_MISSES,
     NATIVE_CLASS_RETURNED,
     NATIVE_COMPILATION_ERROR,
     TOTAL_CALLS,
@@ -23,6 +22,9 @@ define_infra_metrics!(batcher);
 
 define_metrics!(
     Batcher => {
+        // Global class cache
+        MetricCounter { CLASS_CACHE_MISSES, "batcher_class_cache_misses", "Counter of the batcher's global class cache misses", init=0 },
+        MetricCounter { CLASS_CACHE_HITS, "batcher_class_cache_hits", "Counter of the batcher's global class cache hits", init=0 },
         // Heights
         MetricGauge { STORAGE_HEIGHT, "batcher_storage_height", "The height of the batcher's storage" },
         MetricGauge { LAST_BATCHED_BLOCK_HEIGHT, "batcher_last_batched_block_height", "The height of the last block received by batching" },
@@ -75,6 +77,9 @@ pub(crate) fn record_block_close_reason(reason: BlockCloseReason) {
     BLOCK_CLOSE_REASON.increment(1, &[(LABEL_NAME_BLOCK_CLOSE_REASON, reason.into())]);
 }
 
+pub const BATCHER_CLASS_CACHE_METRICS: CacheMetrics =
+    CacheMetrics { misses: CLASS_CACHE_MISSES, hits: CLASS_CACHE_HITS };
+
 pub fn register_metrics(storage_height: BlockNumber) {
     STORAGE_HEIGHT.register();
     STORAGE_HEIGHT.set_lossy(storage_height.0);
@@ -104,9 +109,8 @@ pub fn register_metrics(storage_height: BlockNumber) {
     PROVING_GAS_IN_LAST_BLOCK.register();
 
     // Blockifier's metrics
+    BATCHER_CLASS_CACHE_METRICS.register();
     CALLS_RUNNING_NATIVE.register();
-    CLASS_CACHE_HITS.register();
-    CLASS_CACHE_MISSES.register();
     NATIVE_CLASS_RETURNED.register();
     NATIVE_COMPILATION_ERROR.register();
     TOTAL_CALLS.register();

crates/apollo_batcher/src/metrics.rs

@@ -950,22 +950,33 @@
     "Blockifier": {
       "panels": [
         {
-          "title": "class_cache_miss_ratio",
-          "description": "The ratio of cache misses when requesting compiled classes from the Blockifier State Reader",
+          "title": "Class Cache Miss in Batcher",
+          "description": "The ratio of cache misses when requesting compiled classes from the Blockifier State Reader in Batcher",
           "type": "timeseries",
           "exprs": [
-            "(increase(class_cache_misses{cluster=~\"$cluster\", namespace=~\"$namespace\"}[5m]) / (increase(class_cache_misses{cluster=~\"$cluster\", namespace=~\"$namespace\"}[5m]) + increase(class_cache_hits{cluster=~\"$cluster\", namespace=~\"$namespace\"}[5m])))"
+            "(increase(batcher_class_cache_misses{cluster=~\"$cluster\", namespace=~\"$namespace\"}[5m]) / (increase(batcher_class_cache_misses{cluster=~\"$cluster\", namespace=~\"$namespace\"}[5m]) + increase(batcher_class_cache_hits{cluster=~\"$cluster\", namespace=~\"$namespace\"}[5m])))"
           ],
           "extra_params": {
             "unit": "percentunit"
           }
         },
         {
-          "title": "native_class_returned_ratio",
-          "description": "The ratio of Native classes returned by the Blockifier",
+          "title": "Native Class Returned Ratio in Batcher",
+          "description": "The ratio of Native classes returned by the Blockifier in Batcher",
           "type": "timeseries",
           "exprs": [
-            "(increase(native_class_returned{cluster=~\"$cluster\", namespace=~\"$namespace\"}[5m]) / (increase(class_cache_hits{cluster=~\"$cluster\", namespace=~\"$namespace\"}[5m]) + increase(class_cache_misses{cluster=~\"$cluster\", namespace=~\"$namespace\"}[5m])))"
+            "(increase(native_class_returned{cluster=~\"$cluster\", namespace=~\"$namespace\"}[5m]) / (increase(batcher_class_cache_misses{cluster=~\"$cluster\", namespace=~\"$namespace\"}[5m]) + increase(batcher_class_cache_hits{cluster=~\"$cluster\", namespace=~\"$namespace\"}[5m])))"
+          ],
+          "extra_params": {
+            "unit": "percentunit"
+          }
+        },
+        {
+          "title": "Class Cache Miss in Gateway",
+          "description": "The ratio of cache misses when requesting compiled classes from the Blockifier State Reader in Gateway",
+          "type": "timeseries",
+          "exprs": [
+            "(increase(gateway_class_cache_misses{cluster=~\"$cluster\", namespace=~\"$namespace\"}[5m]) / (increase(gateway_class_cache_misses{cluster=~\"$cluster\", namespace=~\"$namespace\"}[5m]) + increase(gateway_class_cache_hits{cluster=~\"$cluster\", namespace=~\"$namespace\"}[5m])))"
           ],
           "extra_params": {
             "unit": "percentunit"
@@ -981,7 +992,7 @@
           "extra_params": {}
         },
         {
-          "title": "native_execution_ratio",
+          "title": "Native Execution Ratio",
           "description": "The ratio of calls running Cairo Native in the Blockifier",
           "type": "timeseries",
           "exprs": [

crates/apollo_dashboard/resources/dev_grafana.json

@@ -279,8 +279,8 @@ impl Panel {
     }
 
     pub(crate) fn ratio_time_series(
-        name: &'static str,
-        description: &'static str,
+        name: &str,
+        description: &str,
         numerator: &dyn MetricQueryName,
         denominator_parts: &[&dyn MetricQueryName],
         duration: &str,

crates/apollo_dashboard/src/dashboard.rs

@@ -1,14 +1,15 @@
 use apollo_batcher::metrics::{
+    BATCHER_CLASS_CACHE_METRICS,
     NUM_TRANSACTION_IN_BLOCK,
     PROVING_GAS_IN_LAST_BLOCK,
     SIERRA_GAS_IN_LAST_BLOCK,
 };
-use apollo_metrics::metrics::MetricQueryName;
+use apollo_gateway::metrics::GATEWAY_CLASS_CACHE_METRICS;
+use apollo_metrics::metrics::{MetricQueryName, MetricScope};
 use blockifier::metrics::{
+    CacheMetrics,
     BLOCKIFIER_METRIC_RATE_DURATION,
     CALLS_RUNNING_NATIVE,
-    CLASS_CACHE_HITS,
-    CLASS_CACHE_MISSES,
     NATIVE_CLASS_RETURNED,
     NATIVE_COMPILATION_ERROR,
     TOTAL_CALLS,
@@ -19,24 +20,41 @@ use crate::dashboard::{Panel, PanelType, Row};
 const DENOMINATOR_DIVISOR_FOR_READABILITY: f64 = 1_000_000_000.0;
 
 // TODO(MatanL/Shahak): use clamp_min(X, 1) on denom to avoid division by zero.
-fn get_panel_blockifier_state_reader_class_cache_miss_ratio() -> Panel {
-    Panel::ratio_time_series(
-        "class_cache_miss_ratio",
+fn get_panel_blockifier_state_reader_class_cache_miss_ratio(
+    class_cache_metrics: &CacheMetrics,
+) -> Panel {
+    let scope = class_cache_metrics.get_scope();
+    // TODO(Arni): Share this code.
+    let scope = match scope {
+        MetricScope::Batcher => "Batcher",
+        MetricScope::Gateway => "Gateway",
+        _ => unreachable!("Scope of class cache metrics must be Batcher or Gateway. Got {scope:?}"),
+    };
+    let name = format!("Class Cache Miss in {scope}");
+    let description = format!(
         "The ratio of cache misses when requesting compiled classes from the Blockifier State \
-         Reader",
-        &CLASS_CACHE_MISSES,
-        &[&CLASS_CACHE_MISSES, &CLASS_CACHE_HITS],
+         Reader in {scope}"
+    );
+    Panel::ratio_time_series(
+        name.as_str(),
+        description.as_str(),
+        &class_cache_metrics.misses,
+        &[&class_cache_metrics.misses, &class_cache_metrics.hits],
         BLOCKIFIER_METRIC_RATE_DURATION,
     )
 }
 
 // TODO(MatanL/Shahak): use clamp_min(X, 1) on denom to avoid division by zero.
 fn get_panel_blockifier_state_reader_native_class_returned_ratio() -> Panel {
+    let class_cache_metrics = BATCHER_CLASS_CACHE_METRICS;
+
+    let name = "Native Class Returned Ratio in Batcher";
+    let description = "The ratio of Native classes returned by the Blockifier in Batcher";
     Panel::ratio_time_series(
-        "native_class_returned_ratio",
-        "The ratio of Native classes returned by the Blockifier",
+        name,
+        description,
         &NATIVE_CLASS_RETURNED,
-        &[&CLASS_CACHE_HITS, &CLASS_CACHE_MISSES],
+        &[&class_cache_metrics.misses, &class_cache_metrics.hits],
         BLOCKIFIER_METRIC_RATE_DURATION,
     )
 }
@@ -52,7 +70,7 @@ fn get_panel_native_compilation_error() -> Panel {
 
 fn get_panel_native_execution_ratio() -> Panel {
     Panel::ratio_time_series(
-        "native_execution_ratio",
+        "Native Execution Ratio",
         "The ratio of calls running Cairo Native in the Blockifier",
         &CALLS_RUNNING_NATIVE,
         &[&TOTAL_CALLS],
@@ -98,8 +116,10 @@ pub(crate) fn get_blockifier_row() -> Row {
     Row::new(
         "Blockifier",
         vec![
-            get_panel_blockifier_state_reader_class_cache_miss_ratio(),
+            get_panel_blockifier_state_reader_class_cache_miss_ratio(&BATCHER_CLASS_CACHE_METRICS),
+            // TODO(Arni): Add native class returned ratio for gateway
             get_panel_blockifier_state_reader_native_class_returned_ratio(),
+            get_panel_blockifier_state_reader_class_cache_miss_ratio(&GATEWAY_CLASS_CACHE_METRICS),
             get_panel_native_compilation_error(),
             get_panel_native_execution_ratio(),
             get_panel_transactions_per_block(),

crates/apollo_dashboard/src/panels/blockifier.rs

@@ -14,6 +14,7 @@ use apollo_infra::metrics::{
 use apollo_metrics::metrics::LabeledMetricCounter;
 use apollo_metrics::{define_infra_metrics, define_metrics, generate_permutation_labels};
 use apollo_network_types::network_types::BroadcastedMessageMetadata;
+use blockifier::metrics::CacheMetrics;
 use starknet_api::rpc_transaction::{RpcTransaction, RpcTransactionLabelValue};
 use strum::{EnumVariantNames, VariantNames};
 use strum_macros::IntoStaticStr;
@@ -39,6 +40,8 @@ define_infra_metrics!(gateway);
 
 define_metrics!(
     Gateway => {
+        MetricCounter { CLASS_CACHE_MISSES, "gateway_class_cache_misses", "Counter of the gateway's global class cache misses", init=0 },
+        MetricCounter { CLASS_CACHE_HITS, "gateway_class_cache_hits", "Counter of the gateway's global class cache hits", init=0 },
         LabeledMetricCounter { GATEWAY_TRANSACTIONS_RECEIVED, "gateway_transactions_received", "Counter of transactions received", init = 0 , labels = TRANSACTION_TYPE_AND_SOURCE_LABELS},
         LabeledMetricCounter { GATEWAY_TRANSACTIONS_FAILED, "gateway_transactions_failed", "Counter of failed transactions", init = 0 , labels = TRANSACTION_TYPE_AND_SOURCE_LABELS},
         LabeledMetricCounter { GATEWAY_TRANSACTIONS_SENT_TO_MEMPOOL, "gateway_transactions_sent_to_mempool", "Counter of transactions sent to the mempool", init = 0 , labels = TRANSACTION_TYPE_AND_SOURCE_LABELS},
@@ -243,7 +246,11 @@ impl Drop for GatewayMetricHandle {
     }
 }
 
+pub const GATEWAY_CLASS_CACHE_METRICS: CacheMetrics =
+    CacheMetrics { misses: CLASS_CACHE_MISSES, hits: CLASS_CACHE_HITS };
+
 pub(crate) fn register_metrics() {
+    GATEWAY_CLASS_CACHE_METRICS.register();
     GATEWAY_TRANSACTIONS_RECEIVED.register();
     GATEWAY_TRANSACTIONS_FAILED.register();
     GATEWAY_TRANSACTIONS_SENT_TO_MEMPOOL.register();

crates/apollo_gateway/src/metrics.rs

@@ -20,7 +20,7 @@ pub use crate::metrics::histograms::{HistogramValue, LabeledMetricHistogram, Met
 pub static COLLECT_SEQUENCER_PROFILING_METRICS: OnceLock<bool> = OnceLock::new();
 
 /// Relevant components for which metrics can be defined.
-#[derive(Clone, Copy, Debug)]
+#[derive(Clone, Copy, Debug, PartialEq)]
 pub enum MetricScope {
     Batcher,
     Blockifier,

crates/apollo_metrics/src/metrics.rs

@@ -1,9 +1,8 @@
 use apollo_metrics::define_metrics;
+use apollo_metrics::metrics::{MetricCounter, MetricDetails, MetricScope};
 
 define_metrics!(
     Blockifier => {
-        MetricCounter { CLASS_CACHE_MISSES, "class_cache_misses", "Counter of global class cache misses", init=0 },
-        MetricCounter { CLASS_CACHE_HITS, "class_cache_hits", "Counter of global class cache hits", init=0 },
         MetricCounter {
             NATIVE_CLASS_RETURNED,
             "native_class_returned",
@@ -29,3 +28,33 @@ define_metrics!(
 );
 
 pub const BLOCKIFIER_METRIC_RATE_DURATION: &str = "5m";
+
+// TODO(Arni): Fix visibility of members of this struct (make them not public).
+pub struct CacheMetrics {
+    pub misses: MetricCounter,
+    pub hits: MetricCounter,
+}
+
+impl CacheMetrics {
+    pub fn register(&self) {
+        self.misses.register();
+        self.hits.register();
+    }
+
+    pub fn increment_miss(&self) {
+        self.misses.increment(1);
+    }
+
+    pub fn increment_hit(&self) {
+        self.hits.increment(1);
+    }
+
+    pub fn get_scope(&self) -> MetricScope {
+        assert_eq!(
+            self.misses.get_scope(),
+            self.hits.get_scope(),
+            "Scope of misses and hits must be the same"
+        );
+        self.misses.get_scope()
+    }
+}

crates/blockifier/src/metrics.rs

@@ -2,7 +2,7 @@ use starknet_api::core::{ClassHash, CompiledClassHash};
 use starknet_types_core::felt::Felt;
 
 use crate::execution::contract_class::RunnableCompiledClass;
-use crate::metrics::{CLASS_CACHE_HITS, CLASS_CACHE_MISSES};
+use crate::metrics::CacheMetrics;
 use crate::state::contract_class_manager::ContractClassManager;
 use crate::state::errors::StateError;
 use crate::state::global_cache::CompiledClasses;
@@ -21,6 +21,7 @@ pub trait FetchCompiledClasses: StateReader {
 pub struct StateReaderAndContractManager<S: FetchCompiledClasses> {
     pub state_reader: S,
     pub contract_class_manager: ContractClassManager,
+    pub class_cache_metrics: CacheMetrics,
 }
 
 impl<S: FetchCompiledClasses> StateReaderAndContractManager<S> {
@@ -40,11 +41,11 @@ impl<S: FetchCompiledClasses> StateReaderAndContractManager<S> {
                     }
                 }
             }
-            CLASS_CACHE_HITS.increment(1);
+            self.class_cache_metrics.increment_hit();
             self.update_native_metrics(&runnable_class);
             return Ok(runnable_class);
         }
-        CLASS_CACHE_MISSES.increment(1);
+        self.class_cache_metrics.increment_miss();
 
         let compiled_class = self.state_reader.get_compiled_classes(class_hash)?;
         self.contract_class_manager.set_and_compile(class_hash, compiled_class.clone());