Fixed HELM Capabilities case

damian1996 · damian1996 · commit 6af63b9b329e · 2025-11-26T18:40:46.000+01:00
diff --git a/data/helm_capabilities/allenai/olmo-2-0325-32b-instruct/3fdc7300-3046-481e-92e5-356e94a188f1.json b/data/helm_capabilities/allenai/olmo-2-0325-32b-instruct/3fdc7300-3046-481e-92e5-356e94a188f1.json
@@ -1,7 +1,7 @@
 {
   "schema_version": "0.0.1",
-  "evaluation_id": "helm_capabilities/allenai_olmo-2-0325-32b-instruct/1764176819.310097",
-  "retrieved_timestamp": "1764176819.310097",
+  "evaluation_id": "helm_capabilities/allenai_olmo-2-0325-32b-instruct/1764178796.5756102",
+  "retrieved_timestamp": "1764178796.5756102",
   "source_data": [
     "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json"
   ],
@@ -34,7 +34,7 @@
         "details": {
           "accuracy_description": null,
           "efficiency_description": null,
-          "eval_time_mean_win_rate": 191.759
+          "mean_eval_time": 191.759
         }
       },
       "generation_config": {}
diff --git a/data/helm_capabilities/allenai/olmo-2-1124-13b-instruct/b9b29f81-f456-4501-a692-4d0a6aad8f5b.json b/data/helm_capabilities/allenai/olmo-2-1124-13b-instruct/b9b29f81-f456-4501-a692-4d0a6aad8f5b.json
@@ -1,7 +1,7 @@
 {
   "schema_version": "0.0.1",
-  "evaluation_id": "helm_capabilities/allenai_olmo-2-1124-13b-instruct/1764176819.310583",
-  "retrieved_timestamp": "1764176819.310583",
+  "evaluation_id": "helm_capabilities/allenai_olmo-2-1124-13b-instruct/1764178796.576065",
+  "retrieved_timestamp": "1764178796.576065",
   "source_data": [
     "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json"
   ],
@@ -34,7 +34,7 @@
         "details": {
           "accuracy_description": null,
           "efficiency_description": null,
-          "eval_time_mean_win_rate": 103.939
+          "mean_eval_time": 103.939
         }
       },
       "generation_config": {}
diff --git a/data/helm_capabilities/allenai/olmo-2-1124-7b-instruct/aef4c181-4ac3-4aae-acb2-de90cf6f886b.json b/data/helm_capabilities/allenai/olmo-2-1124-7b-instruct/aef4c181-4ac3-4aae-acb2-de90cf6f886b.json
@@ -1,7 +1,7 @@
 {
   "schema_version": "0.0.1",
-  "evaluation_id": "helm_capabilities/allenai_olmo-2-1124-7b-instruct/1764176819.310952",
-  "retrieved_timestamp": "1764176819.310952",
+  "evaluation_id": "helm_capabilities/allenai_olmo-2-1124-7b-instruct/1764178796.576389",
+  "retrieved_timestamp": "1764178796.576389",
   "source_data": [
     "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json"
   ],
@@ -34,7 +34,7 @@
         "details": {
           "accuracy_description": null,
           "efficiency_description": null,
-          "eval_time_mean_win_rate": 164.449
+          "mean_eval_time": 164.449
         }
       },
       "generation_config": {}
diff --git a/data/helm_capabilities/allenai/olmoe-1b-7b-0125-instruct/4c27e4c9-3837-432d-a9d0-90333d5cc75e.json b/data/helm_capabilities/allenai/olmoe-1b-7b-0125-instruct/4c27e4c9-3837-432d-a9d0-90333d5cc75e.json
@@ -1,7 +1,7 @@
 {
   "schema_version": "0.0.1",
-  "evaluation_id": "helm_capabilities/allenai_olmoe-1b-7b-0125-instruct/1764176819.311302",
-  "retrieved_timestamp": "1764176819.311302",
+  "evaluation_id": "helm_capabilities/allenai_olmoe-1b-7b-0125-instruct/1764178796.5766778",
+  "retrieved_timestamp": "1764178796.5766778",
   "source_data": [
     "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json"
   ],
@@ -34,7 +34,7 @@
         "details": {
           "accuracy_description": null,
           "efficiency_description": null,
-          "eval_time_mean_win_rate": 449.115
+          "mean_eval_time": 449.115
         }
       },
       "generation_config": {}
diff --git a/data/helm_capabilities/deepseek-ai/deepseek-v3/c74b799b-0305-4050-993a-8a993d62f211.json b/data/helm_capabilities/deepseek-ai/deepseek-v3/c74b799b-0305-4050-993a-8a993d62f211.json
@@ -1,7 +1,7 @@
 {
   "schema_version": "0.0.1",
-  "evaluation_id": "helm_capabilities/deepseek-ai_deepseek-v3/1764176819.311538",
-  "retrieved_timestamp": "1764176819.311538",
+  "evaluation_id": "helm_capabilities/deepseek-ai_deepseek-v3/1764178796.576965",
+  "retrieved_timestamp": "1764178796.576965",
   "source_data": [
     "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json"
   ],
@@ -34,7 +34,7 @@
         "details": {
           "accuracy_description": null,
           "efficiency_description": null,
-          "eval_time_mean_win_rate": 71.889
+          "mean_eval_time": 71.889
         }
       },
       "generation_config": {}
diff --git a/data/helm_capabilities/marin-community/marin-8b-instruct/78c6af1a-6b9f-44d9-b891-7a565fd7da1b.json b/data/helm_capabilities/marin-community/marin-8b-instruct/78c6af1a-6b9f-44d9-b891-7a565fd7da1b.json
@@ -1,7 +1,7 @@
 {
   "schema_version": "0.0.1",
-  "evaluation_id": "helm_capabilities/marin-community_marin-8b-instruct/1764176819.309529",
-  "retrieved_timestamp": "1764176819.309529",
+  "evaluation_id": "helm_capabilities/marin-community_marin-8b-instruct/1764178796.574767",
+  "retrieved_timestamp": "1764178796.574767",
   "source_data": [
     "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json"
   ],
@@ -34,7 +34,7 @@
         "details": {
           "accuracy_description": null,
           "efficiency_description": null,
-          "eval_time_mean_win_rate": 118.552
+          "mean_eval_time": 118.552
         }
       },
       "generation_config": {}
diff --git a/data/helm_capabilities/moonshotai/kimi-k2-instruct/f9313090-2afb-405b-a845-7f3ded5a7036.json b/data/helm_capabilities/moonshotai/kimi-k2-instruct/f9313090-2afb-405b-a845-7f3ded5a7036.json
@@ -1,7 +1,7 @@
 {
   "schema_version": "0.0.1",
-  "evaluation_id": "helm_capabilities/moonshotai_kimi-k2-instruct/1764176819.3127081",
-  "retrieved_timestamp": "1764176819.3127081",
+  "evaluation_id": "helm_capabilities/moonshotai_kimi-k2-instruct/1764178796.578225",
+  "retrieved_timestamp": "1764178796.578225",
   "source_data": [
     "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json"
   ],
@@ -34,7 +34,7 @@
         "details": {
           "accuracy_description": null,
           "efficiency_description": null,
-          "eval_time_mean_win_rate": 44.938
+          "mean_eval_time": 44.938
         }
       },
       "generation_config": {}
diff --git a/data/helm_capabilities/openai/gpt-oss-120b/8d5fd3cf-23da-4718-a508-48734fa7d2b2.json b/data/helm_capabilities/openai/gpt-oss-120b/8d5fd3cf-23da-4718-a508-48734fa7d2b2.json
@@ -1,7 +1,7 @@
 {
   "schema_version": "0.0.1",
-  "evaluation_id": "helm_capabilities/openai_gpt-oss-120b/1764176819.312159",
-  "retrieved_timestamp": "1764176819.312159",
+  "evaluation_id": "helm_capabilities/openai_gpt-oss-120b/1764178796.5776498",
+  "retrieved_timestamp": "1764178796.5776498",
   "source_data": [
     "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json"
   ],
@@ -34,7 +34,7 @@
         "details": {
           "accuracy_description": null,
           "efficiency_description": null,
-          "eval_time_mean_win_rate": 19.583
+          "mean_eval_time": 19.583
         }
       },
       "generation_config": {}
diff --git a/data/helm_capabilities/openai/gpt-oss-20b/fa4653b0-60c4-4de5-8715-b6767cb89111.json b/data/helm_capabilities/openai/gpt-oss-20b/fa4653b0-60c4-4de5-8715-b6767cb89111.json
@@ -1,7 +1,7 @@
 {
   "schema_version": "0.0.1",
-  "evaluation_id": "helm_capabilities/openai_gpt-oss-20b/1764176819.312483",
-  "retrieved_timestamp": "1764176819.312483",
+  "evaluation_id": "helm_capabilities/openai_gpt-oss-20b/1764178796.5779781",
+  "retrieved_timestamp": "1764178796.5779781",
   "source_data": [
     "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json"
   ],
@@ -34,7 +34,7 @@
         "details": {
           "accuracy_description": null,
           "efficiency_description": null,
-          "eval_time_mean_win_rate": 31.785
+          "mean_eval_time": 31.785
         }
       },
       "generation_config": {}
diff --git a/data/helm_capabilities/zai-org/glm-4.5-air-fp8/d2d362ca-0ff7-46d0-8ac0-aade4a9a7dee.json b/data/helm_capabilities/zai-org/glm-4.5-air-fp8/d2d362ca-0ff7-46d0-8ac0-aade4a9a7dee.json
@@ -1,7 +1,7 @@
 {
   "schema_version": "0.0.1",
-  "evaluation_id": "helm_capabilities/zai-org_glm-4.5-air-fp8/1764176819.311798",
-  "retrieved_timestamp": "1764176819.311798",
+  "evaluation_id": "helm_capabilities/zai-org_glm-4.5-air-fp8/1764178796.577315",
+  "retrieved_timestamp": "1764178796.577315",
   "source_data": [
     "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json"
   ],
@@ -34,7 +34,7 @@
         "details": {
           "accuracy_description": null,
           "efficiency_description": null,
-          "eval_time_mean_win_rate": 36.156
+          "mean_eval_time": 36.156
         }
       },
       "generation_config": {}
diff --git a/scripts/HELM/convert_to_schema.py b/scripts/HELM/convert_to_schema.py
@@ -93,14 +93,17 @@ def extract_efficiency_stats(efficiency_row: List) -> Tuple[float, List[float]]:
 def prepare_score_details(
     acc_stats: Dict[str, Any], 
     eff_stats: Dict[str, Any], 
-    column_idx: int
+    column_idx: int,
+    leaderboard_name: str
 ) -> ScoreDetails:
     details = {
         'accuracy_description': acc_stats.get('description'),
         'efficiency_description': eff_stats.get('description')
     }
 
-    if column_idx == 0: # mean_win_rate stats
+    if column_idx == 0 and leaderboard_name == 'helm_capabilities': # mean_score stats
+        details['mean_eval_time'] = round(eff_stats.get('value'), 3)
+    elif column_idx == 0 and leaderboard_name == 'helm_lite': # mean_win_rate stats
         details['eval_time_mean_win_rate'] = round(eff_stats.get('value'), 3)
     else:
         details['eval_time'] = round(eff_stats.get('value'), 3)
@@ -153,7 +156,8 @@ def convert(leaderboard_name, leaderboard_data, evaluation_source, source_data):
                     score_details=prepare_score_details(
                         acc_per_column,
                         eff_per_column,
-                        column_idx
+                        column_idx,
+                        leaderboard_name
                     ),
                     generation_config=generation_config
                 )
@@ -185,11 +189,11 @@ def convert(leaderboard_name, leaderboard_data, evaluation_source, source_data):
 
 
 if __name__ == '__main__':
-    leaderboard_name = 'HELM_Lite' # 'HELM_Capabilities'
+    leaderboard_name = 'HELM_Capabilities' # 'HELM_Lite'
     leaderboard_name = leaderboard_name.lower()
     source_data = [
-        # 'https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json'
-        'https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json'
+        'https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json'
+        # 'https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json'
     ]
 
     os.makedirs(f'data/{leaderboard_name}', exist_ok=True)
