diff --git a/src/helm/benchmark/metrics/evaluate_reference_metrics.py b/src/helm/benchmark/metrics/evaluate_reference_metrics.py index 049ebc3836..a1168442a5 100644 --- a/src/helm/benchmark/metrics/evaluate_reference_metrics.py +++ b/src/helm/benchmark/metrics/evaluate_reference_metrics.py @@ -153,7 +153,7 @@ def final_number_exact_match(gold: str, pred: str) -> float: """ def get_final_number(x: str) -> str: - matches = re.findall(r"-?[\d,]+(?:.\d+)?", x) + matches = re.findall(r"-?[\d,]+(?:\.\d+)?", x) if not matches: return "" return matches[-1].replace(",", "") diff --git a/src/helm/benchmark/metrics/test_evaluate_reference_metrics.py b/src/helm/benchmark/metrics/test_evaluate_reference_metrics.py index a4325aa839..a2996cb96b 100644 --- a/src/helm/benchmark/metrics/test_evaluate_reference_metrics.py +++ b/src/helm/benchmark/metrics/test_evaluate_reference_metrics.py @@ -31,6 +31,9 @@ def test_final_number_exact_match(): assert final_number_exact_match("34.2", "2") == 0 assert final_number_exact_match("342", "342.") == 1 assert final_number_exact_match("3,420", "3420") == 1 + # A non-digit, non-dot character between digits must not be treated as part of a number. + assert final_number_exact_match("30", "the meeting is at 7:30") == 1 + assert final_number_exact_match("3", "the grid is 2x3") == 1 def test_bleu_1():