Add data validation pre-commit

Author: yifanmai

.github/workflows/validate-data.yml

@@ -5,10 +5,12 @@ on:
     branches: [ main ]
     paths:
       - 'scripts/validate_data.py'
+      - 'eval.schema.json'
       - 'data/**'
   pull_request:
     paths:
       - 'scripts/validate_data.py'
+      - 'eval.schema.json'
       - 'data/**'
 
 jobs:
@@ -31,8 +33,5 @@ jobs:
           # Disable cache because there are no requirements lockfiles
           enable-cache: false
 
-      - name: Download latest schema
-        run: wget https://raw.githubusercontent.com/evaleval/evalHub/refs/heads/main/schema/eval.schema.json
-
       - name: Validate data
-        run: uv run scripts/validate_data.py --schema-path eval.schema.json data
+        run: uv run pre-commit run --all-files

.pre-commit-config.yaml

@@ -0,0 +1,9 @@
+repos:
+- repo: local
+  hooks:
+    - id: validate-data
+      name: Validate data
+      entry: uv run python scripts/validate_data.py --schema-path eval.schema.json
+      exclude: '^.*\.schema\.json$'
+      language: system
+      types_or: [json]

README.md

@@ -1 +1,29 @@
 # every_eval_ever
+
+## Data Validation
+
+This repository has a pre-commit that will validate that JSON files conform to the JSON schema. The pre-commit requires using [uv](https://docs.astral.sh/uv/) for dependency management.
+
+To run the pre-commit on git staged files only:
+
+```sh
+uv run pre-commit run
+```
+
+To run the pre-commit on all files:
+
+```sh
+uv run pre-commit run --all-files
+```
+
+To run the pre-commit specific files:
+
+```sh
+uv run pre-commit run --files /path/to/data/a.json /path/to/data/b.json
+```
+
+To optionally install the pre-commit so that it will run when before `git commit` (optional):
+
+```sh
+uv run pre-commit install
+```

pyproject.toml

@@ -0,0 +1,17 @@
+[project]
+name = "every-eval-ever"
+version = "0.1.0"
+description = "Every Eval Ever"
+readme = "README.md"
+authors = [
+    { name = "EvalEval", email = "[email protected]" }
+]
+requires-python = ">=3.12"
+dependencies = [
+    "jsonschema>=4.25.1",
+]
+
+[dependency-groups]
+dev = [
+    "pre-commit>=4.3.0",
+]

scripts/validate_data.py

@@ -1,13 +1,7 @@
-# /// script
-# requires-python = ">=3.12"
-# dependencies = [
-#     "jsonschema>=4.25",
-# ]
-# ///
-
 import argparse
 import json
 import os
+from typing import List
 
 from jsonschema.exceptions import ValidationError
 from jsonschema.protocols import Validator
@@ -27,12 +21,12 @@ def validate_file(file_path: str, validator: Validator) -> None:
     validator.validate(instance)
 
 
-def expand_paths(paths: str) -> str:
+def expand_paths(paths: List[str]) -> List[str]:
     """Expand folders to file paths"""
-    file_paths = []
+    file_paths: List[str] = []
     for path in paths:
         if os.path.isfile(path) and path.endswith(".json"):
-            file_paths.append(file_paths)
+            file_paths.append(path)
         elif os.path.isdir(path):
             for root, _, file_names in os.walk(path):
                 for file_name in file_names:
@@ -53,7 +47,7 @@ def annotate_error(file_path: str, message: str, **kwargs) -> None:
 def main() -> None:
     parser = argparse.ArgumentParser(
         prog="validate_data",
-        description="Validates that the JSON data conforms to the Pydantic schema",
+        description="Validates that the JSON data conforms to the JSON schema",
     )
     parser.add_argument(
         "paths", nargs="+", type=str, help="File or folder paths to the JSON data"
@@ -108,6 +102,7 @@ def main() -> None:
             print(f"{file_path}")
             print("  " + message)
             print()
+            raise
     print(f"{num_passed} file(s) passed; {num_failed} file(s) failed")
     print()
     if num_failed > 0: