fix: Use source id weighted

Author: lewisjared

backend/src/ref_backend/core/outliers.py

@@ -30,6 +30,57 @@ def flag_outliers_iqr(values: Sequence[float], factor: float = 5.0, min_n: int =
     return [v < lower_bound or v > upper_bound for v in values]
 
 
+def calculate_iqr_bounds_by_source_id(
+    df: pd.DataFrame, factor: float = 3.0, min_n: int = 4
+) -> tuple[float, float] | None:
+    """
+    Calculate IQR bounds using source_id means for equal model weighting.
+
+    This function calculates mean value for each source_id and then
+    computes IQR bounds on these means, ensuring each model gets equal
+    weight regardless of number of ensemble members.
+
+    Parameters
+    ----------
+    df : pd.DataFrame
+        DataFrame containing scalar values with dimensions including source_id
+    factor : float
+        The factor to multiply IQR by to determine outlier bounds
+    min_n : int
+        Minimum number of source_ids required to perform outlier detection
+
+    Returns
+    -------
+    tuple[float, float] | None
+        Tuple of (lower_bound, upper_bound) or None if insufficient data
+    """
+    # Check if source_id column exists
+    if "source_id" not in df.columns:
+        return None
+
+    # Separate Reference values (exclude from IQR calculation)
+    reference_mask = df["source_id"] == "Reference"
+    non_reference_df = df[~reference_mask]
+
+    # Group by source_id and calculate mean for each
+    source_id_means = non_reference_df.groupby("source_id")["value"].mean()
+
+    # Check if we have enough source_ids for outlier detection
+    if len(source_id_means) < min_n:
+        return None
+
+    # Calculate IQR on source_id means
+    means_list = source_id_means.tolist()
+    quantiles = statistics.quantiles(means_list, n=4, method="inclusive")
+    q1, q3 = quantiles[0], quantiles[2]
+    iqr = q3 - q1
+
+    lower_bound = q1 - factor * iqr
+    upper_bound = q3 + factor * iqr
+
+    return lower_bound, upper_bound
+
+
 def detect_outliers_in_scalar_values(
     scalar_values: Sequence[models.ScalarMetricValue],
     factor: float = 3.0,
@@ -38,6 +89,11 @@ def detect_outliers_in_scalar_values(
 ) -> tuple[list[AnnotatedScalarValue], int]:
     """Detect outliers in scalar metric values grouped by stable diagnostic facets.
 
+    This function uses source_id-aware outlier detection, where IQR bounds are calculated
+    using the mean value of each source_id rather than on all individual ensemble members.
+    This ensures each model gets equal weight regardless of number of ensemble members.
+    The calculated bounds are then applied to individual values for outlier detection.
+
     Parameters
     ----------
     scalar_values
@@ -49,7 +105,7 @@ def detect_outliers_in_scalar_values(
 
         Defaults to 3.0.
     min_n
-        The minimum number of data points required in a group to perform
+        The minimum number of source_ids required in a group to perform
         IQR outlier detection. Defaults to 4.
     group_by
         A sequence of dimension names to group the `scalar_values` by before
@@ -72,20 +128,37 @@ def detect_outliers_in_scalar_values(
     group_by = [g for g in group_by if g in df.columns]
 
     for _, group_values in df.groupby(list(group_by)):
-        print(group_values)
         # Identify non-finite values (NaN, inf) as outliers
         finite_flags = group_values.value.apply(
             lambda x: isinstance(x, int | float) and not math.isinf(x) and not math.isnan(x)
         )
-        # Apply IQR only if group has enough values
-        if len(group_values) >= min_n:
-            iqr_flags = flag_outliers_iqr(group_values.value.to_list(), factor=factor)
+        # Apply source_id-aware outlier detection if source_id exists
+        if "source_id" in group_values.columns and len(group_values) >= min_n:
+            iqr_bounds = calculate_iqr_bounds_by_source_id(group_values, factor=factor, min_n=min_n)
+
+            if iqr_bounds is not None:
+                lower_bound, upper_bound = iqr_bounds
+                # Apply bounds to individual values (Reference values always non-outlier)
+                source_id_flags = group_values.apply(
+                    lambda row: (row["value"] < lower_bound or row["value"] > upper_bound)
+                    if row["source_id"] != "Reference"
+                    else False,
+                    axis=1,
+                )
+            else:
+                # Fallback if insufficient source_ids
+                source_id_flags = [False] * len(group_values)  # type: ignore
         else:
-            iqr_flags = [False] * len(group_values)
-
-        # Combine flags: item is outlier if iqr-flagged
-        for sv, is_outside_iqr, is_finite in zip(group_values.scalar_value, iqr_flags, finite_flags):
-            is_outlier = is_outside_iqr or not is_finite
+            # Fallback to original IQR method if no source_id or insufficient data
+            if len(group_values) >= min_n:
+                iqr_flags = flag_outliers_iqr(group_values.value.to_list(), factor=factor)
+            else:
+                iqr_flags = [False] * len(group_values)
+            source_id_flags = iqr_flags  # type: ignore
+
+        # Combine flags: item is outlier if flagged by source_id method OR non-finite
+        for sv, is_source_outlier, is_finite in zip(group_values.scalar_value, source_id_flags, finite_flags):
+            is_outlier = is_source_outlier or not is_finite
             verification_status: Literal["verified", "unverified"] = (
                 "unverified" if is_outlier else "verified"
             )

backend/tests/test_core/test_outliers.py

@@ -9,7 +9,7 @@ class TestFlagOutliersIQR:
     def test_outlier_detection_with_factor_3(self):
         """Test outlier detection with factor=3.0 on controlled data."""
         values = [1, 2, 2, 2, 100]
-        result = flag_outliers_iqr(values, factor=3.0)
+        result = flag_outliers_iqr(values, factor=3.0, min_n=4)
         expected = [True, False, False, False, True]
         assert result == expected
 
@@ -23,7 +23,7 @@ def test_outlier_detection_with_factor_1_5(self):
         - Any value outside [2, 2] is an outlier, so 100 is flagged
         """
         values = [1, 2, 2, 2, 100]
-        result = flag_outliers_iqr(values, factor=1.5)
+        result = flag_outliers_iqr(values, factor=1.5, min_n=4)
         expected = [True, False, False, False, True]
         assert result == expected