common: fix match_case misidentifying all-caps contractions as capitalized

docs/tutorial.md

@@ -57,7 +57,7 @@ Additionally, for each group and group-relavent metric, it will output a pair of
 
 ## Using `helm-server`
 
-Finally, the `helm-server` command launches a web server to visualize the output files of `helm-run` and `helm-benchmark`. Run:
+Finally, the `helm-server` command launches a web server to visualize the output files of `helm-run` and `helm-summarize`. Run:
 
 ```sh
 helm-server --suite my-suite

src/helm/benchmark/augmentations/test_perturbation.py

@@ -3,6 +3,7 @@
 import unittest
 
 from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.general import match_case
 from helm.benchmark.scenarios.scenario import Input, Instance, Output, Reference
 from helm.benchmark.augmentations.data_augmenter import DataAugmenter
 from helm.benchmark.augmentations.extra_space_perturbation import ExtraSpacePerturbation
@@ -292,6 +293,39 @@ def test_gender_term_perturbation_edge_word():
 
 
 # TODO(#1958) Fix the logic to renable this test
+def test_match_case_all_caps_with_apostrophe():
+    """Regression: all-caps contractions like AIN'T must be detected as all-caps, not capitalized."""
+    assert match_case("AIN'T", "is not") == "IS NOT"
+    assert match_case("DON'T", "do not") == "DO NOT"
+    assert match_case("CAN'T", "cannot") == "CANNOT"
+    assert match_case("WON'T", "will not") == "WILL NOT"
+
+
+def test_match_case_lowercase_with_apostrophe():
+    """Lowercase contractions with apostrophes must stay lowercase."""
+    assert match_case("ain't", "is not") == "is not"
+    assert match_case("don't", "do not") == "do not"
+
+
+def test_match_case_capitalized_with_apostrophe():
+    """Capitalized contractions must capitalize the target."""
+    assert match_case("Don't", "do not") == "Do not"
+    assert match_case("Ain't", "is not") == "Is not"
+
+
+def test_match_case_no_alpha_chars():
+    """Source words with no alphabetic characters return target unchanged."""
+    assert match_case("123", "hello") == "hello"
+    assert match_case("", "hello") == "hello"
+
+
+def test_match_case_regular_words():
+    """Regular words (no apostrophe) still work correctly."""
+    assert match_case("HELLO", "world") == "WORLD"
+    assert match_case("hello", "WORLD") == "world"
+    assert match_case("Hello", "world") == "World"
+
+
 @unittest.skip("Currently cannot replace words separated by 1 character.")
 def test_gender_term_perturbation_consequtive_words():
     data_augmenter = DataAugmenter(

src/helm/common/general.py

@@ -234,14 +234,19 @@ def indent_lines(lines: List[str], count: int = 2) -> List[str]:
 
 def match_case(source_word: str, target_word: str) -> str:
     """Return a version of the target_word where the case matches the source_word."""
+    # Only consider alphabetic characters; non-alpha chars (e.g. apostrophes in contractions)
+    # return False for both islower() and isupper(), which would break all-caps detection.
+    alpha_chars = [ch for ch in source_word if ch.isalpha()]
+    if not alpha_chars:
+        return target_word
     # Check for all lower case source_word
-    if all(letter.islower() for letter in source_word):
+    if all(ch.islower() for ch in alpha_chars):
         return target_word.lower()
     # Check for all caps source_word
-    if all(letter.isupper() for letter in source_word):
+    if all(ch.isupper() for ch in alpha_chars):
         return target_word.upper()
     # Check for capital source_word
-    if source_word and source_word[0].isupper():
+    if alpha_chars[0].isupper():
         return target_word.capitalize()
     return target_word