diff --git a/src/helm/benchmark/metrics/gpqa_chain_of_thought_metric.py b/src/helm/benchmark/metrics/gpqa_chain_of_thought_metric.py index f6d994943b..ea6418c514 100644 --- a/src/helm/benchmark/metrics/gpqa_chain_of_thought_metric.py +++ b/src/helm/benchmark/metrics/gpqa_chain_of_thought_metric.py @@ -25,8 +25,8 @@ def extract_answer(output_text: str) -> Optional[str]: if match: return match.group(1) - # Second regex: Matches "[answer: (A-J)]" with optional leading characters like "." - match = re.search(r"\.*\[aA\]nswer:\s*\(?([A-J])\)?", output_text) + # Second regex: Matches "answer: (A-J)" (e.g. "Answer: B" or "[answer: B]") with optional leading characters + match = re.search(r"\.*[aA]nswer:\s*\(?([A-J])\)?", output_text) if match: return match.group(1) diff --git a/src/helm/benchmark/metrics/test_gpqa_chain_of_thought_metric.py b/src/helm/benchmark/metrics/test_gpqa_chain_of_thought_metric.py new file mode 100644 index 0000000000..fb4cfd6ed8 --- /dev/null +++ b/src/helm/benchmark/metrics/test_gpqa_chain_of_thought_metric.py @@ -0,0 +1,21 @@ +from helm.benchmark.metrics.gpqa_chain_of_thought_metric import extract_answer + + +def test_extract_answer_correct_answer_is_format(): + # Primary format requested by the GPQA chain-of-thought prompt. + assert extract_answer("The correct answer is (A)") == "A" + assert extract_answer("The correct answer is B") == "B" + + +def test_extract_answer_answer_colon_format(): + # Fallback format: "Answer: X" / "answer: X" (previously broken by the + # literal-escaped brackets in the regex, which only matched "[aA]nswer:"). + assert extract_answer("Answer: C") == "C" + assert extract_answer("The answer: B") == "B" + assert extract_answer("answer: E") == "E" + assert extract_answer("...Answer: D") == "D" + assert extract_answer("[answer: F]") == "F" + + +def test_extract_answer_no_match(): + assert extract_answer("I am not sure.") is None