From 842c62abb64c08817af7bb8b4253640ce68b8227 Mon Sep 17 00:00:00 2001 From: mukunda katta Date: Sun, 19 Apr 2026 17:23:06 -0700 Subject: [PATCH] fix(medhelm): use quasi match for PubMedQA --- src/helm/benchmark/scenarios/pubmed_qa_scenario.py | 2 +- src/helm/benchmark/scenarios/test_pubmed_qa_scenario.py | 7 +++++++ src/helm/benchmark/static/schema_medhelm.yaml | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 src/helm/benchmark/scenarios/test_pubmed_qa_scenario.py diff --git a/src/helm/benchmark/scenarios/pubmed_qa_scenario.py b/src/helm/benchmark/scenarios/pubmed_qa_scenario.py index 5d0cc974e7a..fc6ecbd1179 100644 --- a/src/helm/benchmark/scenarios/pubmed_qa_scenario.py +++ b/src/helm/benchmark/scenarios/pubmed_qa_scenario.py @@ -205,6 +205,6 @@ def get_metadata(self): who="Researcher", language="English", ), - main_metric="exact_match", + main_metric="quasi_exact_match", main_split="test", ) diff --git a/src/helm/benchmark/scenarios/test_pubmed_qa_scenario.py b/src/helm/benchmark/scenarios/test_pubmed_qa_scenario.py new file mode 100644 index 00000000000..9a1ccd237bd --- /dev/null +++ b/src/helm/benchmark/scenarios/test_pubmed_qa_scenario.py @@ -0,0 +1,7 @@ +from helm.benchmark.scenarios.pubmed_qa_scenario import PubMedQAScenario + + +def test_pubmed_qa_main_metric_uses_quasi_exact_match(): + scenario = PubMedQAScenario() + + assert scenario.get_metadata().main_metric == "quasi_exact_match" diff --git a/src/helm/benchmark/static/schema_medhelm.yaml b/src/helm/benchmark/static/schema_medhelm.yaml index 24b61ed8232..0feaceb08eb 100644 --- a/src/helm/benchmark/static/schema_medhelm.yaml +++ b/src/helm/benchmark/static/schema_medhelm.yaml @@ -995,7 +995,7 @@ run_groups: - efficiency - general_information environment: - main_name: exact_match + main_name: quasi_exact_match main_split: test taxonomy: task: Question answering