Fix string-name normalization in SamplingMetric (inspect.getmembers returns a list, not a dict)

src/lighteval/metrics/metrics_sample.py

@@ -1119,8 +1119,8 @@ def __init__(
         if isinstance(normalize, str):
             import lighteval.metrics.normalizations
 
-            allowed_normalizations = inspect.getmembers(
-                lighteval.metrics.normalizations, inspect.isfunction
+            allowed_normalizations = dict(
+                inspect.getmembers(lighteval.metrics.normalizations, inspect.isfunction)
             )  # -> {name: fn}
             if normalize in allowed_normalizations:
                 self.normalize = allowed_normalizations[normalize]

tests/test_unit_base_metrics.py

@@ -327,6 +327,21 @@ def test_exact_match_dynamic_metric(self):
         )
         assert result[em_metric.metric_name] == 0
 
+    def test_sampling_metric_normalize_by_name(self):
+        # Regression: SamplingMetric resolved the `normalize` string against a
+        # list of (name, fn) tuples from inspect.getmembers, so every valid name
+        # raised ValueError. It must resolve to the named normalization function.
+        from lighteval.metrics import normalizations
+        from lighteval.metrics.metrics_sample import AvgAtN
+
+        metric = AvgAtN(n=1, normalize="gsm8k_normalizer")
+        assert metric.normalize is normalizations.gsm8k_normalizer
+        # and it actually applies the resolved normalizer
+        assert metric.preprocess("The answer is #### 18") == "18"
+        # unknown names still raise
+        with pytest.raises(ValueError):
+            AvgAtN(n=1, normalize="definitely_not_a_normalizer")
+
     @pytest.mark.skip(reason="Need to understand what it does.")
     def test_pass_at_k_estimator(self):
         assert False