diff --git a/src/lighteval/metrics/metrics_sample.py b/src/lighteval/metrics/metrics_sample.py index db14b9bf6..672624803 100644 --- a/src/lighteval/metrics/metrics_sample.py +++ b/src/lighteval/metrics/metrics_sample.py @@ -1119,8 +1119,8 @@ def __init__( if isinstance(normalize, str): import lighteval.metrics.normalizations - allowed_normalizations = inspect.getmembers( - lighteval.metrics.normalizations, inspect.isfunction + allowed_normalizations = dict( + inspect.getmembers(lighteval.metrics.normalizations, inspect.isfunction) ) # -> {name: fn} if normalize in allowed_normalizations: self.normalize = allowed_normalizations[normalize] diff --git a/tests/test_unit_base_metrics.py b/tests/test_unit_base_metrics.py index 575ebf595..6ffb826d9 100644 --- a/tests/test_unit_base_metrics.py +++ b/tests/test_unit_base_metrics.py @@ -327,6 +327,21 @@ def test_exact_match_dynamic_metric(self): ) assert result[em_metric.metric_name] == 0 + def test_sampling_metric_normalize_by_name(self): + # Regression: SamplingMetric resolved the `normalize` string against a + # list of (name, fn) tuples from inspect.getmembers, so every valid name + # raised ValueError. It must resolve to the named normalization function. + from lighteval.metrics import normalizations + from lighteval.metrics.metrics_sample import AvgAtN + + metric = AvgAtN(n=1, normalize="gsm8k_normalizer") + assert metric.normalize is normalizations.gsm8k_normalizer + # and it actually applies the resolved normalizer + assert metric.preprocess("The answer is #### 18") == "18" + # unknown names still raise + with pytest.raises(ValueError): + AvgAtN(n=1, normalize="definitely_not_a_normalizer") + @pytest.mark.skip(reason="Need to understand what it does.") def test_pass_at_k_estimator(self): assert False