[Fix] Correctly set testcase build url and gn args for fuzzing on batch (#5013)

Fix missing testcase metadata when fuzz task is running on batch (or any
non-local execution).

#### Context
Some build-related environment variables (`build_url`, `build_key` and
`gn_args_path`) are set during the build setup on fuzz task main stage.
These should be retrieved during postprocess to set the testcase's
metadata, which is used to display the build url and the gn args config
section on the clusterfuzz UI.

However, for non-local execution, these variables are not propagated
through the `uworker_output` proto and are missing in postprocess, so
any testcase created during fuzzing on batch is not displaying these
values.

#### Changes
* Added the `build_url`, `build_key` and `gn_args` fields to the fuzz
task output message.
* Added the logic to store these fields and use them to set the initial
testcase metadata.

#### Tests
* Testing on development environment is ineffective as it is not
currently running fuzz tasks on batch. I deployed to dev anyway and
confirmed that, at least, the local execution of fuzz tasks is still
working.
* Testing on prod (deployed this PR to internal)
* Fuzzing hours stable after deploy:
https://screenshot.googleplex.com/7JfregzBR8F6MRA.png
* Logs for fuzz tasks executing on batch:
https://cloudlogging.app.goo.gl/5xXpLjUdwzK6wTeR9 (main) and
https://cloudlogging.app.goo.gl/nX832G1782asdMDXA (postprocess)
* Logs for fuzz tasks executing on GCE VMs:
https://cloudlogging.app.goo.gl/61nvxR1BwQrQnvXr8 (main) and
https://cloudlogging.app.goo.gl/6yoQSvU8r1mUVxsN7 (postprocess)

* Evidence of testcases creates after the deploy containing the build
url and GN config args:
* GCE VMs (local exec):
https://clusterfuzz.com/testcase-detail/6746169883426816
    * Batch: https://clusterfuzz.com/testcase-detail/6451774202249216
* Build URL: https://screenshot.googleplex.com/45n7M75hEPoEsNZ.png
* GN Configs: https://screenshot.googleplex.com/9r3SxRPRQRyC4tv.png

Bug: b/441128474
Related issue: #4914

Author: ViniciustCosta

src/clusterfuzz/_internal/bot/tasks/utasks/fuzz_task.py

@@ -1002,17 +1002,18 @@ def create_testcase(group: uworker_msg_pb2.FuzzTaskCrashGroup,
                     fully_qualified_fuzzer_name: str):
   """Create a testcase based on crash."""
   crash = group.main_crash
+  fuzz_task_output = uworker_output.fuzz_task_output
   comment = (f'Fuzzer {fully_qualified_fuzzer_name} generated testcase crashed '
              f'in {crash.crash_time} seconds '
-             f'(r{uworker_output.fuzz_task_output.crash_revision})')
+             f'(r{fuzz_task_output.crash_revision})')
   testcase_id = data_handler.store_testcase(
       crash=crash,
       fuzzed_keys=crash.fuzzed_key or None,
       minimized_keys=get_fixed_or_minimized_key(group.one_time_crasher_flag),
       regression=get_regression(group.one_time_crasher_flag),
       fixed=get_fixed_or_minimized_key(group.one_time_crasher_flag),
       one_time_crasher_flag=group.one_time_crasher_flag,
-      crash_revision=int(uworker_output.fuzz_task_output.crash_revision),
+      crash_revision=int(fuzz_task_output.crash_revision),
       comment=comment,
       absolute_path=crash.absolute_path,
       fuzzer_name=uworker_input.fuzzer_name,
@@ -1036,6 +1037,15 @@ def create_testcase(group: uworker_msg_pb2.FuzzTaskCrashGroup,
       events.TestcaseCreationEvent(
           testcase=testcase, creation_origin=events.TestcaseOrigin.FUZZ_TASK))
 
+  # Add build metadata to testcase. This is needed due to some build env vars
+  # not being propagated if utask is not executed locally (e.g., BUILD_URL).
+  data_handler.set_build_metadata_to_testcase(
+      testcase,
+      build_key=fuzz_task_output.build_key,
+      build_url=fuzz_task_output.build_url,
+      gn_args=fuzz_task_output.gn_args,
+      update=True)
+
   if group.context.fuzzer_metadata:
     for key, value in group.context.fuzzer_metadata.items():
       testcase.set_metadata(key, value, update_testcase=False)
@@ -1879,6 +1889,7 @@ def run(self):
     fuzz_target = self.fuzz_target.binary if self.fuzz_target else None
     build_setup_result = build_manager.setup_build(
         environment.get_value('APP_REVISION'), fuzz_target=fuzz_target)
+    _add_build_metadata_to_output(self.fuzz_task_output)
 
     engine_impl = engine.get(self.fuzzer.name)
     if engine_impl and build_setup_result:
@@ -2241,6 +2252,14 @@ def _to_engine_output(output: str, crash_path: str, return_code: int,
   return engine_output
 
 
+def _add_build_metadata_to_output(
+    fuzz_task_output: uworker_msg_pb2.FuzzTaskOutput) -> None:
+  """Add build metadata from environment to fuzz task output."""
+  fuzz_task_output.build_key = environment.get_value('BUILD_KEY', '')
+  fuzz_task_output.build_url = environment.get_value('BUILD_URL', '')
+  fuzz_task_output.gn_args = data_handler.get_filtered_gn_args() or ''
+
+
 def _upload_engine_output(engine_output):
   timestamp = uworker_io.proto_timestamp_to_timestamp(engine_output.timestamp)
   testcase_manager.upload_log(engine_output.output.decode(),

src/clusterfuzz/_internal/datastore/data_handler.py

@@ -834,16 +834,8 @@ def store_testcase(crash, fuzzed_keys, minimized_keys, regression, fixed,
   return testcase_id
 
 
-def set_initial_testcase_metadata(testcase):
-  """Set various testcase metadata fields during testcase initialization."""
-  build_key = environment.get_value('BUILD_KEY')
-  if build_key:
-    testcase.set_metadata('build_key', build_key, update_testcase=False)
-
-  build_url = environment.get_value('BUILD_URL')
-  if build_url:
-    testcase.set_metadata('build_url', build_url, update_testcase=False)
-
+def get_filtered_gn_args() -> str | None:
+  """Return filtered GN args based on filepath GN_ARGS_PATH."""
   gn_args_path = environment.get_value('GN_ARGS_PATH', '')
   if gn_args_path:
     gn_args = utils.read_data_from_file(
@@ -856,8 +848,41 @@ def set_initial_testcase_metadata(testcase):
         if not GOMA_DIR_LINE_REGEX.match(line)
     ]
     filtered_gn_args = '\n'.join(filtered_gn_args_lines)
-    testcase.set_metadata('gn_args', filtered_gn_args, update_testcase=False)
+    return filtered_gn_args
+  return None
+
+
+def set_build_metadata_to_testcase(testcase: data_types.Testcase,
+                                   build_key: str | None = None,
+                                   build_url: str | None = None,
+                                   gn_args: str | None = None,
+                                   update: bool = False):
+  """Set testcase metadata fields related to the build metadata."""
+  dirty_flag = False
+  build_key = (
+      environment.get_value('BUILD_KEY') if build_key is None else build_key)
+  if build_key and not testcase.get_metadata('build_key'):
+    dirty_flag = True
+    testcase.set_metadata('build_key', build_key, update_testcase=False)
+
+  build_url = (
+      environment.get_value('BUILD_URL') if build_url is None else build_url)
+  if build_url and not testcase.get_metadata('build_url'):
+    dirty_flag = True
+    testcase.set_metadata('build_url', build_url, update_testcase=False)
+
+  gn_args = get_filtered_gn_args() if gn_args is None else gn_args
+  if gn_args and not testcase.get_metadata('gn_args'):
+    dirty_flag = True
+    testcase.set_metadata('gn_args', gn_args, update_testcase=False)
 
+  if update and dirty_flag:
+    testcase.put()
+
+
+def set_initial_testcase_metadata(testcase: data_types.Testcase) -> None:
+  """Set various testcase metadata fields during testcase initialization."""
+  set_build_metadata_to_testcase(testcase)
   testcase.platform = environment.platform().lower()
   testcase.platform_id = environment.get_platform_id()
 

src/clusterfuzz/_internal/protos/uworker_msg.proto

@@ -261,6 +261,9 @@ message FuzzTaskOutput {
   optional BuildData build_data = 14;
   optional int64 app_revision = 15;
   repeated EngineOutput engine_outputs = 16;
+  optional string build_key = 17;
+  optional string build_url = 18;
+  optional string gn_args = 19;
 }
 
 message MinimizeTaskOutput {

src/clusterfuzz/_internal/protos/uworker_msg_pb2.py

@@ -1250,6 +1250,9 @@ class FuzzTaskOutput(google.protobuf.message.Message):
     BUILD_DATA_FIELD_NUMBER: builtins.int
     APP_REVISION_FIELD_NUMBER: builtins.int
     ENGINE_OUTPUTS_FIELD_NUMBER: builtins.int
+    BUILD_KEY_FIELD_NUMBER: builtins.int
+    BUILD_URL_FIELD_NUMBER: builtins.int
+    GN_ARGS_FIELD_NUMBER: builtins.int
     fully_qualified_fuzzer_name: builtins.str
     """TODO(metzman): Remove this since tworkers should know what this is based on
     the input.
@@ -1272,6 +1275,9 @@ class FuzzTaskOutput(google.protobuf.message.Message):
     app_revision: builtins.int
     @property
     def engine_outputs(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___EngineOutput]: ...
+    build_key: builtins.str
+    build_url: builtins.str
+    gn_args: builtins.str
     def __init__(
         self,
         *,
@@ -1288,14 +1294,21 @@ class FuzzTaskOutput(google.protobuf.message.Message):
         build_data: global___BuildData | None = ...,
         app_revision: builtins.int | None = ...,
         engine_outputs: collections.abc.Iterable[global___EngineOutput] | None = ...,
+        build_key: builtins.str | None = ...,
+        build_url: builtins.str | None = ...,
+        gn_args: builtins.str | None = ...,
     ) -> None: ...
-    def HasField(self, field_name: typing_extensions.Literal["_app_revision", b"_app_revision", "_build_data", b"_build_data", "_crash_revision", b"_crash_revision", "_fully_qualified_fuzzer_name", b"_fully_qualified_fuzzer_name", "_fuzzer_revision", b"_fuzzer_revision", "_fuzzer_run_results", b"_fuzzer_run_results", "_job_run_timestamp", b"_job_run_timestamp", "_new_targets_count", b"_new_targets_count", "_testcases_executed", b"_testcases_executed", "app_revision", b"app_revision", "build_data", b"build_data", "crash_revision", b"crash_revision", "fully_qualified_fuzzer_name", b"fully_qualified_fuzzer_name", "fuzzer_revision", b"fuzzer_revision", "fuzzer_run_results", b"fuzzer_run_results", "job_run_timestamp", b"job_run_timestamp", "new_targets_count", b"new_targets_count", "testcases_executed", b"testcases_executed"]) -> builtins.bool: ...
-    def ClearField(self, field_name: typing_extensions.Literal["_app_revision", b"_app_revision", "_build_data", b"_build_data", "_crash_revision", b"_crash_revision", "_fully_qualified_fuzzer_name", b"_fully_qualified_fuzzer_name", "_fuzzer_revision", b"_fuzzer_revision", "_fuzzer_run_results", b"_fuzzer_run_results", "_job_run_timestamp", b"_job_run_timestamp", "_new_targets_count", b"_new_targets_count", "_testcases_executed", b"_testcases_executed", "app_revision", b"app_revision", "build_data", b"build_data", "crash_groups", b"crash_groups", "crash_revision", b"crash_revision", "engine_outputs", b"engine_outputs", "fully_qualified_fuzzer_name", b"fully_qualified_fuzzer_name", "fuzz_targets", b"fuzz_targets", "fuzzer_revision", b"fuzzer_revision", "fuzzer_run_results", b"fuzzer_run_results", "job_run_timestamp", b"job_run_timestamp", "new_targets_count", b"new_targets_count", "testcase_run_jsons", b"testcase_run_jsons", "testcases_executed", b"testcases_executed"]) -> None: ...
+    def HasField(self, field_name: typing_extensions.Literal["_app_revision", b"_app_revision", "_build_data", b"_build_data", "_build_key", b"_build_key", "_build_url", b"_build_url", "_crash_revision", b"_crash_revision", "_fully_qualified_fuzzer_name", b"_fully_qualified_fuzzer_name", "_fuzzer_revision", b"_fuzzer_revision", "_fuzzer_run_results", b"_fuzzer_run_results", "_gn_args", b"_gn_args", "_job_run_timestamp", b"_job_run_timestamp", "_new_targets_count", b"_new_targets_count", "_testcases_executed", b"_testcases_executed", "app_revision", b"app_revision", "build_data", b"build_data", "build_key", b"build_key", "build_url", b"build_url", "crash_revision", b"crash_revision", "fully_qualified_fuzzer_name", b"fully_qualified_fuzzer_name", "fuzzer_revision", b"fuzzer_revision", "fuzzer_run_results", b"fuzzer_run_results", "gn_args", b"gn_args", "job_run_timestamp", b"job_run_timestamp", "new_targets_count", b"new_targets_count", "testcases_executed", b"testcases_executed"]) -> builtins.bool: ...
+    def ClearField(self, field_name: typing_extensions.Literal["_app_revision", b"_app_revision", "_build_data", b"_build_data", "_build_key", b"_build_key", "_build_url", b"_build_url", "_crash_revision", b"_crash_revision", "_fully_qualified_fuzzer_name", b"_fully_qualified_fuzzer_name", "_fuzzer_revision", b"_fuzzer_revision", "_fuzzer_run_results", b"_fuzzer_run_results", "_gn_args", b"_gn_args", "_job_run_timestamp", b"_job_run_timestamp", "_new_targets_count", b"_new_targets_count", "_testcases_executed", b"_testcases_executed", "app_revision", b"app_revision", "build_data", b"build_data", "build_key", b"build_key", "build_url", b"build_url", "crash_groups", b"crash_groups", "crash_revision", b"crash_revision", "engine_outputs", b"engine_outputs", "fully_qualified_fuzzer_name", b"fully_qualified_fuzzer_name", "fuzz_targets", b"fuzz_targets", "fuzzer_revision", b"fuzzer_revision", "fuzzer_run_results", b"fuzzer_run_results", "gn_args", b"gn_args", "job_run_timestamp", b"job_run_timestamp", "new_targets_count", b"new_targets_count", "testcase_run_jsons", b"testcase_run_jsons", "testcases_executed", b"testcases_executed"]) -> None: ...
     @typing.overload
     def WhichOneof(self, oneof_group: typing_extensions.Literal["_app_revision", b"_app_revision"]) -> typing_extensions.Literal["app_revision"] | None: ...
     @typing.overload
     def WhichOneof(self, oneof_group: typing_extensions.Literal["_build_data", b"_build_data"]) -> typing_extensions.Literal["build_data"] | None: ...
     @typing.overload
+    def WhichOneof(self, oneof_group: typing_extensions.Literal["_build_key", b"_build_key"]) -> typing_extensions.Literal["build_key"] | None: ...
+    @typing.overload
+    def WhichOneof(self, oneof_group: typing_extensions.Literal["_build_url", b"_build_url"]) -> typing_extensions.Literal["build_url"] | None: ...
+    @typing.overload
     def WhichOneof(self, oneof_group: typing_extensions.Literal["_crash_revision", b"_crash_revision"]) -> typing_extensions.Literal["crash_revision"] | None: ...
     @typing.overload
     def WhichOneof(self, oneof_group: typing_extensions.Literal["_fully_qualified_fuzzer_name", b"_fully_qualified_fuzzer_name"]) -> typing_extensions.Literal["fully_qualified_fuzzer_name"] | None: ...
@@ -1304,6 +1317,8 @@ class FuzzTaskOutput(google.protobuf.message.Message):
     @typing.overload
     def WhichOneof(self, oneof_group: typing_extensions.Literal["_fuzzer_run_results", b"_fuzzer_run_results"]) -> typing_extensions.Literal["fuzzer_run_results"] | None: ...
     @typing.overload
+    def WhichOneof(self, oneof_group: typing_extensions.Literal["_gn_args", b"_gn_args"]) -> typing_extensions.Literal["gn_args"] | None: ...
+    @typing.overload
     def WhichOneof(self, oneof_group: typing_extensions.Literal["_job_run_timestamp", b"_job_run_timestamp"]) -> typing_extensions.Literal["job_run_timestamp"] | None: ...
     @typing.overload
     def WhichOneof(self, oneof_group: typing_extensions.Literal["_new_targets_count", b"_new_targets_count"]) -> typing_extensions.Literal["new_targets_count"] | None: ...

src/clusterfuzz/_internal/protos/uworker_msg_pb2.pyi

@@ -40,12 +40,11 @@ def setUp(self):
     test_utils.set_up_pyfakefs(self)
     helpers.patch_environ(self)
 
-  def test_set(self):
-    """Test set everything."""
+  def test_testcase_metadata_from_env(self):
+    """Tests the initial testcase metadata set from the env vars."""
     os.environ['FAIL_RETRIES'] = '3'
     os.environ['FAIL_WAIT'] = '3'
     os.environ['BUILD_KEY'] = 'build_key_value'
-    os.environ['BUILD_KEY'] = 'build_key_value'
     os.environ['BUILD_URL'] = 'build_url_value'
     os.environ['APP_DIR'] = 'app_dir_value'
     os.environ['GN_ARGS_PATH'] = 'app_dir_value/args.gn'
@@ -66,6 +65,31 @@ def test_set(self):
         'is_asan = true\nuse_goma = true\nv8_enable_verify_heap = true',
         metadata['gn_args'])
 
+  def test_testcase_metadata_from_args(self):
+    """Tests that testcase build metadata is set from args."""
+    build_key = 'build_key_test'
+    build_url = 'build_url_test'
+    os.environ['GN_ARGS_PATH'] = 'app_dir_value/args.gn'
+    self.fs.create_file(
+        'app_dir_value/args.gn',
+        contents=('is_asan = true\n'
+                  'goma_dir = /home/user/goma\n'
+                  'use_goma = false\n'
+                  'v8_enable_verify_heap = true'))
+    gn_args = data_handler.get_filtered_gn_args()
+    del os.environ['GN_ARGS_PATH']
+
+    testcase = data_types.Testcase()
+    data_handler.set_build_metadata_to_testcase(
+        testcase, build_key=build_key, build_url=build_url, gn_args=gn_args)
+    metadata = json.loads(testcase.additional_metadata)
+
+    self.assertEqual('build_key_test', metadata['build_key'])
+    self.assertEqual('build_url_test', metadata['build_url'])
+    self.assertEqual(
+        'is_asan = true\nuse_goma = false\nv8_enable_verify_heap = true',
+        metadata['gn_args'])
+
 
 @test_utils.with_cloud_emulators('datastore')
 class DataHandlerTest(unittest.TestCase):