From c6e866b1085bb0b78a959115f80cf588b27dd871 Mon Sep 17 00:00:00 2001 From: Teddy Tennant Date: Wed, 8 Jul 2026 16:31:14 -0400 Subject: [PATCH] Fix string-name normalization in SamplingMetric `inspect.getmembers` returns a list of (name, fn) tuples, but the code treated it as a dict: the `normalize in allowed_normalizations` membership test never matched a plain string and `allowed_normalizations[normalize]` would raise TypeError. As a result, passing any string for `normalize` (a supported public-API form, per the `Callable | str | None` type hint) always raised `ValueError: Unknown normalization function: `, even for valid names like "gsm8k_normalizer". Wrap the getmembers result in `dict()` so `in` checks function names and indexing does a key lookup. Unknown names still fall through to the existing ValueError. --- src/lighteval/metrics/metrics_sample.py | 4 ++-- tests/test_unit_base_metrics.py | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/lighteval/metrics/metrics_sample.py b/src/lighteval/metrics/metrics_sample.py index db14b9bf6..672624803 100644 --- a/src/lighteval/metrics/metrics_sample.py +++ b/src/lighteval/metrics/metrics_sample.py @@ -1119,8 +1119,8 @@ def __init__( if isinstance(normalize, str): import lighteval.metrics.normalizations - allowed_normalizations = inspect.getmembers( - lighteval.metrics.normalizations, inspect.isfunction + allowed_normalizations = dict( + inspect.getmembers(lighteval.metrics.normalizations, inspect.isfunction) ) # -> {name: fn} if normalize in allowed_normalizations: self.normalize = allowed_normalizations[normalize] diff --git a/tests/test_unit_base_metrics.py b/tests/test_unit_base_metrics.py index 575ebf595..6ffb826d9 100644 --- a/tests/test_unit_base_metrics.py +++ b/tests/test_unit_base_metrics.py @@ -327,6 +327,21 @@ def test_exact_match_dynamic_metric(self): ) assert result[em_metric.metric_name] == 0 + def test_sampling_metric_normalize_by_name(self): + # Regression: SamplingMetric resolved the `normalize` string against a + # list of (name, fn) tuples from inspect.getmembers, so every valid name + # raised ValueError. It must resolve to the named normalization function. + from lighteval.metrics import normalizations + from lighteval.metrics.metrics_sample import AvgAtN + + metric = AvgAtN(n=1, normalize="gsm8k_normalizer") + assert metric.normalize is normalizations.gsm8k_normalizer + # and it actually applies the resolved normalizer + assert metric.preprocess("The answer is #### 18") == "18" + # unknown names still raise + with pytest.raises(ValueError): + AvgAtN(n=1, normalize="definitely_not_a_normalizer") + @pytest.mark.skip(reason="Need to understand what it does.") def test_pass_at_k_estimator(self): assert False