React Native Benchmarks

.buildkite/react-native-pipeline.benchmark.block.yml

@@ -0,0 +1,10 @@
+steps:
+  - block: "Trigger react native benchmark pipeline"
+    key: "trigger-react-native-benchmark-pipeline"
+
+  - label: ":pipeline_upload: React native benchmark pipeline"
+    depends_on: "trigger-react-native-benchmark-pipeline"
+    agents:
+      queue: macos
+    timeout_in_minutes: 2
+    command: buildkite-agent pipeline upload .buildkite/react-native-pipeline.benchmark.yml

.buildkite/react-native-pipeline.benchmark.yml

@@ -0,0 +1,153 @@
+agents:
+  queue: "opensource"
+
+steps:
+  - group: "React Native Benchmark Tests"
+    steps:
+      #
+      # Build fixtures
+      #
+      - label: ':android: Build RN {{matrix}} benchmark fixture APK (New Arch)'
+        key: "build-react-native-android-fixture-benchmark-new-arch"
+        timeout_in_minutes: 30
+        agents:
+          queue: macos-15
+        env:
+          JAVA_VERSION: "17"
+          NODE_VERSION: "18"
+          RN_VERSION: "{{matrix}}"
+          NOTIFIER_VERSION: "8.0.0"
+          RCT_NEW_ARCH_ENABLED: "1"
+          BUILD_ANDROID: "true"
+          NATIVE_INTEGRATION: "1"
+        artifact_paths:
+          - "test/react-native/features/fixtures/generated/native-integration/new-arch/**/reactnative.apk"
+        commands:
+          - bundle install
+          - node test/react-native/scripts/generate-react-native-fixture.js
+        matrix:
+          - "0.80"
+        retry:
+          automatic:
+            - exit_status: "*"
+              limit: 1
+
+      - label: ':mac: Build RN {{matrix}} benchmark fixture ipa (New Arch)'
+        key: "build-react-native-ios-fixture-benchmark-new-arch"
+        timeout_in_minutes: 30
+        agents:
+          queue: "macos-15"
+        env:
+          NODE_VERSION: "18"
+          RN_VERSION: "{{matrix}}"
+          RCT_NEW_ARCH_ENABLED: "1"
+          NOTIFIER_VERSION: "8.0.0"
+          BUILD_IOS: "true"
+          XCODE_VERSION: "16.2.0"
+          NATIVE_INTEGRATION: "1"
+        artifact_paths:
+          - "test/react-native/features/fixtures/generated/native-integration/new-arch/**/output/reactnative.ipa"
+        commands:
+          - bundle install
+          - node test/react-native/scripts/generate-react-native-fixture.js
+        matrix:
+          - "0.80"
+        retry:
+          automatic:
+            - exit_status: "*"
+              limit: 1
+
+      #
+      # Run benchmarks
+      #
+      - label: ":bitbar: :android: RN {{matrix}} Android Benchmarks"
+        depends_on: "build-react-native-android-fixture-benchmark-new-arch"
+        timeout_in_minutes: 20
+        plugins:
+          artifacts#v1.9.0:
+            download: "test/react-native/features/fixtures/generated/native-integration/new-arch/{{matrix}}/reactnative.apk"
+            upload:
+              - "./test/react-native/maze_output/failed/*"
+              - "./test/react-native/maze_output/maze_output.zip"
+              - "./test/react-native/maze_output/metrics.csv"
+          docker-compose#v4.7.0:
+            pull: react-native-maze-runner
+            run: react-native-maze-runner
+            service-ports: true
+            command:
+              - --app=/app/features/fixtures/generated/native-integration/new-arch/{{matrix}}/reactnative.apk
+              - --farm=bb
+              - --device=ANDROID_13
+              - --a11y-locator
+              - --fail-fast
+              - --appium-version=1.22
+              - --no-tunnel
+              - --aws-public-ip
+              - --tags=@benchmark
+          test-collector#v1.10.2:
+            files: "reports/TEST-*.xml"
+            format: "junit"
+            branch: "^main|next$$"
+            api-token-env-name: "REACT_NATIVE_PERFORMANCE_BUILDKITE_ANALYTICS_TOKEN"
+        env:
+          RCT_NEW_ARCH_ENABLED: "1"
+          NATIVE_INTEGRATION: "1"
+          BENCHMARKS: "1"
+          RN_VERSION: "{{matrix}}"
+        retry:
+          manual:
+            permit_on_passed: true
+          automatic:
+            - exit_status: 103  # Appium session failed
+              limit: 2
+        concurrency: 25
+        concurrency_group: "bitbar"
+        concurrency_method: eager
+        matrix:
+          - "0.80"
+
+      - label: ":bitbar: :mac: RN {{matrix}} iOS Benchmarks"
+        depends_on: "build-react-native-ios-fixture-benchmark-new-arch"
+        timeout_in_minutes: 20
+        plugins:
+          artifacts#v1.9.0:
+            download: "test/react-native/features/fixtures/generated/native-integration/new-arch/{{matrix}}/output/reactnative.ipa"
+            upload:
+              - "./test/react-native/maze_output/failed/*"
+              - "./test/react-native/maze_output/maze_output.zip"
+              - "./test/react-native/maze_output/metrics.csv"
+          docker-compose#v4.12.0:
+            pull: react-native-maze-runner
+            run: react-native-maze-runner
+            service-ports: true
+            command:
+              - --app=/app/features/fixtures/generated/native-integration/new-arch/{{matrix}}/output/reactnative.ipa
+              - --farm=bb
+              - --device=IOS_14|IOS_15|IOS_16|IOS_17|IOS_18|IOS_26
+              - --a11y-locator
+              - --fail-fast
+              - --appium-version=1.22
+              - --no-tunnel
+              - --aws-public-ip
+              - --tags=@benchmark
+          test-collector#v1.10.2:
+            files: "reports/TEST-*.xml"
+            format: "junit"
+            branch: "^main|next$$"
+            api-token-env-name: "REACT_NATIVE_PERFORMANCE_BUILDKITE_ANALYTICS_TOKEN"
+        env:
+          RCT_NEW_ARCH_ENABLED: "1"
+          NATIVE_INTEGRATION: "1"
+          BENCHMARKS: "1"
+          RN_VERSION: "{{matrix}}"
+        retry:
+          manual:
+            permit_on_passed: true
+          automatic:
+            - exit_status: 103  # Appium session failed
+              limit: 2
+        concurrency: 25
+        concurrency_group: "bitbar"
+        concurrency_method: eager
+        matrix:
+          - "0.80"

.buildkite/scripts/packages.json

@@ -77,5 +77,9 @@
         "pipeline": ".buildkite/expo-pipeline.full.yml",
         "block": ".buildkite/expo-pipeline.full.block.yml",
         "paths": []
+    },
+    {
+        "block": ".buildkite/react-native-pipeline.benchmark.block.yml",
+        "paths": []
     }
 ]

.buildkite/scripts/pipeline-trigger.js

@@ -28,7 +28,7 @@ packages.reverse().forEach(({ paths, block, pipeline, environment, skip }) => {
   }
 
   // Upload all pipelines if specified in the commit message
-  if (commitMessage.includes("[full ci]") ||
+  if (pipeline && commitMessage.includes("[full ci]") ||
     isFullBuild ||
     currentBranch === "main" ||
     baseBranch === "main") {

docker-compose.yml

@@ -82,6 +82,7 @@ services:
       RN_VERSION:
       EXPO_VERSION:
       REACT_NATIVE_NAVIGATION:
+      BENCHMARKS:
       MAZE_REPEATER_API_KEY: "${MAZE_REPEATER_API_KEY_RN:-}"
       MAZE_HUB_REPEATER_API_KEY: "${MAZE_HUB_REPEATER_API_KEY_RN:-}"
     ports:

test/react-native/features/benchmarks/cross_layer_spans.feature

@@ -0,0 +1,19 @@
+@benchmark @native_integration @ios_only
+Feature: Cross-Layer Spans
+
+  Scenario Outline:
+    When I run benchmark "NativeNamedSpanBenchmark" configured as <options>
+    And I wait for 30 seconds
+    And I wait to receive at least 1 metrics
+    And I discard the oldest metric
+    And I relaunch the app after shutdown
+
+    Examples:
+      | options                                           |
+      | "native nativeSpans"                              |
+      | "native nativeSpans jsSpans"                      |
+      | "native nativeSpans rendering"                    |
+      | "native nativeSpans cpu"                          |
+      | "native nativeSpans memory"                       |
+      | "native nativeSpans rendering cpu memory"         |
+      | "native nativeSpans rendering cpu memory jsSpans" |

test/react-native/features/steps/react-native-steps.rb

@@ -1,5 +1,12 @@
 When('I run {string}') do |scenario_name|
-  execute_command 'run-scenario', scenario_name
+  run_scenario scenario_name
+end
+
+When('I run benchmark {string} configured as {string}') do |benchmark_name, config|
+  execute_command 'run-benchmark', {
+    benchmark_name: benchmark_name,
+    config: config
+  }
 end
 
 When('I execute the command {string}') do |command|
@@ -113,7 +120,14 @@
   )
 end
 
-def execute_command(action, scenario_name = '')
+def run_scenario(scenario_name = '')
+  execute_command 'run-scenario', {
+    scenario_name: scenario_name,
+    payload: scenario_name
+  }
+end
+
+def execute_command(action, command_hash = nil)
   address = if Maze.config.farm == :bb
               if Maze.config.aws_public_ip
                 Maze.public_address
@@ -122,21 +136,23 @@ def execute_command(action, scenario_name = '')
               end
             else
               case Maze::Helper.get_current_platform
-                when 'android'
-                  'localhost:9339'
-                else
-                  'bs-local.com:9339'
+              when 'android'
+                'localhost:9339'
+              else
+                'bs-local.com:9339'
               end
             end
 
   command = {
     action: action,
-    scenario_name: scenario_name,
-    payload: scenario_name,
     endpoint: "http://#{address}/traces",
     api_key: $api_key,
   }
 
+  unless command_hash.nil?
+    command.merge! command_hash
+  end
+
   $logger.debug("Queuing command: #{command}")
   Maze::Server.commands.add command
 end

test/react-native/features/support/env.rb

@@ -8,6 +8,10 @@
     Maze.config.receive_requests_wait = 60
   end
 
+  if ENV["BENCHMARKS"]
+    Maze.config.receive_requests_wait = 180
+  end
+
 end
 
 Before('@skip') do
@@ -59,4 +63,8 @@
   current_version = ENV['RN_VERSION'].nil? ? 0 : ENV['RN_VERSION'].to_f
   skip_this_scenario("Skipping scenario: Not running native integration fixture") unless ENV["NATIVE_INTEGRATION"]
   skip_this_scenario("Skipping scenario: Not supported in 0.72") if Maze::Helper.get_current_platform == 'ios' && current_version == 0.72
+end
+
+Before('@benchmark') do |scenario|
+  skip_this_scenario("Skipping scenario: Not running benchmark tests") unless ENV["BENCHMARKS"]
 end

test/react-native/native-test-utils/android/build.gradle

@@ -13,7 +13,8 @@ android {
 }
 
 dependencies {
-    compileOnly("com.bugsnag:bugsnag-android-performance:1.16.0")
+    compileOnly("com.bugsnag:bugsnag-android-performance:2.0.0")
+    compileOnly("com.bugsnag:bugsnag-android-performance-impl:2.0.0")
     implementation project(':bugsnag_react-native-performance')
     implementation project(':bugsnag_plugin-react-native-span-access')
 }

test/react-native/native-test-utils/android/src/main/java/com/bugsnag/test/utils/BugsnagTestUtils.java

@@ -11,6 +11,7 @@
 
 import com.bugsnag.android.performance.AutoInstrument;
 import com.bugsnag.android.performance.BugsnagPerformance;
+import com.bugsnag.android.performance.EnabledMetrics;
 import com.bugsnag.android.performance.PerformanceConfiguration;
 
 import com.bugsnag.reactnative.performance.nativespans.BugsnagJavascriptSpansPlugin;
@@ -124,9 +125,31 @@ public static boolean startNativePerformance(Context context, Map<String, Object
             config.setAutoInstrumentAppStarts(autoInstrumentAppStarts);
             config.setAutoInstrumentActivities(autoInstrumentViewLoads ? AutoInstrument.FULL : AutoInstrument.OFF);
             config.setAutoInstrumentRendering(true);
-            config.addPlugin(new BugsnagNativeSpansPlugin());
-            config.addPlugin(new BugsnagJavascriptSpansPlugin());
-            config.addPlugin(new BugsnagReactNativeAppStartPlugin());
+
+            if (configuration.containsKey("samplingProbability")) {
+                config.setSamplingProbability((Double)configuration.get("samplingProbability"));
+            }
+
+            if (configuration.containsKey("enabledMetrics")) {
+                Map<String, Object> metricsConfig = (Map<String, Object>)configuration.get("enabledMetrics");
+                EnabledMetrics enabledMetrics = new EnabledMetrics(
+                Boolean.TRUE.equals(metricsConfig.get("rendering")),
+                Boolean.TRUE.equals(metricsConfig.get("cpu")),
+                Boolean.TRUE.equals(metricsConfig.get("memory"))
+                );
+
+                config.setEnabledMetrics(enabledMetrics);
+            }
+
+            if (!configuration.containsKey("nativeSpans") || Boolean.TRUE.equals(configuration.get("nativeSpans"))) {
+                config.addPlugin(new BugsnagNativeSpansPlugin());
+            }
+            if (!configuration.containsKey("jsSpans") || Boolean.TRUE.equals(configuration.get("jsSpans"))) {
+                config.addPlugin(new BugsnagJavascriptSpansPlugin());
+            }
+            if (!configuration.containsKey("nativeAppStarts") || Boolean.TRUE.equals(configuration.get("nativeAppStarts"))) {
+				config.addPlugin(new BugsnagReactNativeAppStartPlugin());
+            }
 
             BugsnagPerformance.start(config);
             Log.d(TAG, "Native performance started successfully");

test/react-native/native-test-utils/ios/BugsnagTestUtils.mm

@@ -81,15 +81,33 @@ + (BOOL)startNativePerformanceWithConfiguration:(NSDictionary *)configuration {
             config.autoInstrumentAppStarts = autoInstrumentAppStarts;
             config.autoInstrumentViewControllers = autoInstrumentViewLoads;
             config.autoInstrumentNetworkRequests = NO;
-            config.enabledMetrics.cpu = YES;
-            config.enabledMetrics.memory = YES;
-            config.enabledMetrics.rendering = YES;
             config.internal.autoTriggerExportOnBatchSize = 1;
             config.internal.clearPersistenceOnStart = YES;
 
-            [config addPlugin:[BugsnagNativeSpansPlugin new]];
-            [config addPlugin:[BugsnagJavascriptSpansPlugin new]];
-            [config addPlugin:[BugsnagReactNativeAppStartPlugin new]];
+            if (configuration[@"samplingProbability"]) {
+                config.samplingProbability = configuration[@"samplingProbability"];
+            }
+
+            if (configuration[@"enabledMetrics"]) {
+                NSDictionary *metricsConfig = configuration[@"enabledMetrics"];
+                config.enabledMetrics.rendering = [metricsConfig[@"rendering"] boolValue];
+                config.enabledMetrics.cpu = [metricsConfig[@"cpu"] boolValue];
+                config.enabledMetrics.memory = [metricsConfig[@"memory"] boolValue];
+            } else {
+                config.enabledMetrics.cpu = YES;
+                config.enabledMetrics.memory = YES;
+                config.enabledMetrics.rendering = YES;
+            }
+
+            if (!configuration[@"nativeSpans"] || [configuration[@"nativeSpans"] boolValue]) {
+                [config addPlugin:[BugsnagNativeSpansPlugin new]];
+            }
+            if (!configuration[@"jsSpans"] || [configuration[@"jsSpans"] boolValue]) {
+                [config addPlugin:[BugsnagJavascriptSpansPlugin new]];
+            }
+            if (!configuration[@"nativeAppStarts"] || [configuration[@"nativeAppStarts"] boolValue]) {
+                [config addPlugin:[BugsnagReactNativeAppStartPlugin new]];
+            }
 
             [BugsnagPerformance startWithConfiguration:config];