From 9a270d7e5b10e9128198384aa8be79b9d027573e Mon Sep 17 00:00:00 2001 From: nac7 Date: Mon, 8 Jun 2026 14:21:45 -0500 Subject: [PATCH 1/2] @ docs: fix incorrect command name in helm-server tutorial description The helm-server section referenced helm-benchmark, which does not exist as a CLI entrypoint. The correct command is helm-summarize, which is documented in the preceding section and listed in pyproject.toml. @ --- docs/tutorial.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorial.md b/docs/tutorial.md index 77620560e7a..d53cd84c6c1 100644 --- a/docs/tutorial.md +++ b/docs/tutorial.md @@ -57,7 +57,7 @@ Additionally, for each group and group-relavent metric, it will output a pair of ## Using `helm-server` -Finally, the `helm-server` command launches a web server to visualize the output files of `helm-run` and `helm-benchmark`. Run: +Finally, the `helm-server` command launches a web server to visualize the output files of `helm-run` and `helm-summarize`. Run: ```sh helm-server --suite my-suite From 6345bf28e745042ceebc700c1285e1d225e09300 Mon Sep 17 00:00:00 2001 From: nac7 Date: Mon, 8 Jun 2026 15:30:20 -0500 Subject: [PATCH 2/2] common: fix match_case misidentifying all-caps contractions as capitalized match_case iterated over all characters including apostrophes when checking islower()/isupper(). Apostrophes return False for both islower() and isupper() in Python (they have no case), so words like "AIN'T" and "DON'T" would fail the all-caps check and fall through to the capitalize branch, producing "Is not" instead of "IS NOT". Fix: filter to only alphabetic characters before case-checking, so non-alpha chars in contractions no longer interfere with the detection. Affects ContractionPerturbation, ExpansionPerturbation, and SynonymPerturbation for all-caps inputs containing apostrophes. Fixes #4313 --- .../augmentations/test_perturbation.py | 34 +++++++++++++++++++ src/helm/common/general.py | 11 ++++-- 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/src/helm/benchmark/augmentations/test_perturbation.py b/src/helm/benchmark/augmentations/test_perturbation.py index 4d99bf89a2b..ad90e51b0c8 100644 --- a/src/helm/benchmark/augmentations/test_perturbation.py +++ b/src/helm/benchmark/augmentations/test_perturbation.py @@ -3,6 +3,7 @@ import unittest from helm.common.media_object import MediaObject, MultimediaObject +from helm.common.general import match_case from helm.benchmark.scenarios.scenario import Input, Instance, Output, Reference from helm.benchmark.augmentations.data_augmenter import DataAugmenter from helm.benchmark.augmentations.extra_space_perturbation import ExtraSpacePerturbation @@ -292,6 +293,39 @@ def test_gender_term_perturbation_edge_word(): # TODO(#1958) Fix the logic to renable this test +def test_match_case_all_caps_with_apostrophe(): + """Regression: all-caps contractions like AIN'T must be detected as all-caps, not capitalized.""" + assert match_case("AIN'T", "is not") == "IS NOT" + assert match_case("DON'T", "do not") == "DO NOT" + assert match_case("CAN'T", "cannot") == "CANNOT" + assert match_case("WON'T", "will not") == "WILL NOT" + + +def test_match_case_lowercase_with_apostrophe(): + """Lowercase contractions with apostrophes must stay lowercase.""" + assert match_case("ain't", "is not") == "is not" + assert match_case("don't", "do not") == "do not" + + +def test_match_case_capitalized_with_apostrophe(): + """Capitalized contractions must capitalize the target.""" + assert match_case("Don't", "do not") == "Do not" + assert match_case("Ain't", "is not") == "Is not" + + +def test_match_case_no_alpha_chars(): + """Source words with no alphabetic characters return target unchanged.""" + assert match_case("123", "hello") == "hello" + assert match_case("", "hello") == "hello" + + +def test_match_case_regular_words(): + """Regular words (no apostrophe) still work correctly.""" + assert match_case("HELLO", "world") == "WORLD" + assert match_case("hello", "WORLD") == "world" + assert match_case("Hello", "world") == "World" + + @unittest.skip("Currently cannot replace words separated by 1 character.") def test_gender_term_perturbation_consequtive_words(): data_augmenter = DataAugmenter( diff --git a/src/helm/common/general.py b/src/helm/common/general.py index 289f117e3fb..b3c1a161bef 100644 --- a/src/helm/common/general.py +++ b/src/helm/common/general.py @@ -234,14 +234,19 @@ def indent_lines(lines: List[str], count: int = 2) -> List[str]: def match_case(source_word: str, target_word: str) -> str: """Return a version of the target_word where the case matches the source_word.""" + # Only consider alphabetic characters; non-alpha chars (e.g. apostrophes in contractions) + # return False for both islower() and isupper(), which would break all-caps detection. + alpha_chars = [ch for ch in source_word if ch.isalpha()] + if not alpha_chars: + return target_word # Check for all lower case source_word - if all(letter.islower() for letter in source_word): + if all(ch.islower() for ch in alpha_chars): return target_word.lower() # Check for all caps source_word - if all(letter.isupper() for letter in source_word): + if all(ch.isupper() for ch in alpha_chars): return target_word.upper() # Check for capital source_word - if source_word and source_word[0].isupper(): + if alpha_chars[0].isupper(): return target_word.capitalize() return target_word