diff --git a/docs/tutorial.md b/docs/tutorial.md index 77620560e7a..d53cd84c6c1 100644 --- a/docs/tutorial.md +++ b/docs/tutorial.md @@ -57,7 +57,7 @@ Additionally, for each group and group-relavent metric, it will output a pair of ## Using `helm-server` -Finally, the `helm-server` command launches a web server to visualize the output files of `helm-run` and `helm-benchmark`. Run: +Finally, the `helm-server` command launches a web server to visualize the output files of `helm-run` and `helm-summarize`. Run: ```sh helm-server --suite my-suite diff --git a/src/helm/benchmark/augmentations/test_perturbation.py b/src/helm/benchmark/augmentations/test_perturbation.py index 4d99bf89a2b..ad90e51b0c8 100644 --- a/src/helm/benchmark/augmentations/test_perturbation.py +++ b/src/helm/benchmark/augmentations/test_perturbation.py @@ -3,6 +3,7 @@ import unittest from helm.common.media_object import MediaObject, MultimediaObject +from helm.common.general import match_case from helm.benchmark.scenarios.scenario import Input, Instance, Output, Reference from helm.benchmark.augmentations.data_augmenter import DataAugmenter from helm.benchmark.augmentations.extra_space_perturbation import ExtraSpacePerturbation @@ -292,6 +293,39 @@ def test_gender_term_perturbation_edge_word(): # TODO(#1958) Fix the logic to renable this test +def test_match_case_all_caps_with_apostrophe(): + """Regression: all-caps contractions like AIN'T must be detected as all-caps, not capitalized.""" + assert match_case("AIN'T", "is not") == "IS NOT" + assert match_case("DON'T", "do not") == "DO NOT" + assert match_case("CAN'T", "cannot") == "CANNOT" + assert match_case("WON'T", "will not") == "WILL NOT" + + +def test_match_case_lowercase_with_apostrophe(): + """Lowercase contractions with apostrophes must stay lowercase.""" + assert match_case("ain't", "is not") == "is not" + assert match_case("don't", "do not") == "do not" + + +def test_match_case_capitalized_with_apostrophe(): + """Capitalized contractions must capitalize the target.""" + assert match_case("Don't", "do not") == "Do not" + assert match_case("Ain't", "is not") == "Is not" + + +def test_match_case_no_alpha_chars(): + """Source words with no alphabetic characters return target unchanged.""" + assert match_case("123", "hello") == "hello" + assert match_case("", "hello") == "hello" + + +def test_match_case_regular_words(): + """Regular words (no apostrophe) still work correctly.""" + assert match_case("HELLO", "world") == "WORLD" + assert match_case("hello", "WORLD") == "world" + assert match_case("Hello", "world") == "World" + + @unittest.skip("Currently cannot replace words separated by 1 character.") def test_gender_term_perturbation_consequtive_words(): data_augmenter = DataAugmenter( diff --git a/src/helm/common/general.py b/src/helm/common/general.py index 289f117e3fb..b3c1a161bef 100644 --- a/src/helm/common/general.py +++ b/src/helm/common/general.py @@ -234,14 +234,19 @@ def indent_lines(lines: List[str], count: int = 2) -> List[str]: def match_case(source_word: str, target_word: str) -> str: """Return a version of the target_word where the case matches the source_word.""" + # Only consider alphabetic characters; non-alpha chars (e.g. apostrophes in contractions) + # return False for both islower() and isupper(), which would break all-caps detection. + alpha_chars = [ch for ch in source_word if ch.isalpha()] + if not alpha_chars: + return target_word # Check for all lower case source_word - if all(letter.islower() for letter in source_word): + if all(ch.islower() for ch in alpha_chars): return target_word.lower() # Check for all caps source_word - if all(letter.isupper() for letter in source_word): + if all(ch.isupper() for ch in alpha_chars): return target_word.upper() # Check for capital source_word - if source_word and source_word[0].isupper(): + if alpha_chars[0].isupper(): return target_word.capitalize() return target_word