Add script-aware language tags for FLORES 200

src/lighteval/tasks/multilingual/tasks/flores200.py

@@ -28,7 +28,7 @@
 from lighteval.tasks.templates.utils.formulation import (
     CFFormulation,
 )
-from lighteval.utils.language import Language, manage_duplicate_language_codes
+from lighteval.utils.language import language_from_tag
 
 
 flores_200_languages = [
@@ -233,7 +233,7 @@
     "yor_Latn",
     "yue_Hant",
     "zho_Hans",
-    # "zho_Hant",
+    "zho_Hant",
     "zsm_Latn",
     "zul_Latn",
 ]
@@ -250,8 +250,8 @@ def flores_adapter(lang1, lang2):
     LightevalTaskConfig(
         name=f"flores200:{lang1}-{lang2}",
         prompt_function=get_translation_prompt_function(
-            source_language=Language(manage_duplicate_language_codes(lang1.split("_")[0])),
-            target_language=Language(manage_duplicate_language_codes(lang2.split("_")[0])),
+            source_language=language_from_tag(lang1),
+            target_language=language_from_tag(lang2),
             adapter=flores_adapter(lang1, lang2),
             formulation=CFFormulation(),
         ),

src/lighteval/tasks/templates/utils/translation_literals.py

@@ -23,15 +23,15 @@
 from dataclasses import dataclass, field
 from string import ascii_uppercase
 
-from lighteval.utils.language import Language
+from lighteval.utils.language import Language, LanguageWithScript, Script
 
 
 # TODO(hynky1999): The typing still is not great, it should be able to infer that you can't access the
 # attributes that are not defined in the class. Don't want to waste time on this though.
 @dataclass
 class TranslationLiterals:
     # This is just to create nice error messages
-    language: Language
+    language: Language | LanguageWithScript
 
     question_word: str = None  # type: ignore
     answer: str = None  # type: ignore
@@ -74,7 +74,14 @@ def __getattribute__(self, name: str) -> str:
         return value
 
 
-TRANSLATION_LITERALS: dict[Language, TranslationLiterals] = {
+class TranslationLiteralsDict(dict):
+    def __missing__(self, language):
+        if isinstance(language, LanguageWithScript):
+            return self[language.language]
+        raise KeyError(language)
+
+
+TRANSLATION_LITERALS: dict[Language | LanguageWithScript, TranslationLiterals] = {
     Language.ACEHNESE: TranslationLiterals(language=Language.ACEHNESE),
     Language.AFRIKAANS: TranslationLiterals(language=Language.AFRIKAANS),
     Language.AKAN: TranslationLiterals(language=Language.AKAN),
@@ -304,6 +311,30 @@ def __getattribute__(self, name: str) -> str:
         colon="：",
         indices=["A", "B", "C", "D", "E", "F", "G", "H", "I", "J"],
     ),
+    LanguageWithScript(language=Language.CHINESE, script=Script.HAN_TRADITIONAL): TranslationLiterals(
+        language=LanguageWithScript(language=Language.CHINESE, script=Script.HAN_TRADITIONAL),
+        question_word="問題",
+        answer="答案",
+        confirmation_word="對嗎",
+        yes="是的",
+        no="不是",
+        also="而且",
+        cause_word="因為",
+        effect_word="所以",
+        true="真",
+        false="假",
+        neither="都不是",
+        or_word="或",
+        and_word="和",
+        full_stop="。",
+        comma="，",
+        question_mark="？",
+        exclamation_mark="！",
+        word_space="",
+        sentence_space="",
+        colon="：",
+        indices=["A", "B", "C", "D", "E", "F", "G", "H", "I", "J"],
+    ),
     Language.CHOKWE: TranslationLiterals(language=Language.CHOKWE),
     Language.CRIMEAN_TATAR: TranslationLiterals(language=Language.CRIMEAN_TATAR),
     Language.CROATIAN: TranslationLiterals(
@@ -1426,3 +1457,5 @@ def __getattribute__(self, name: str) -> str:
     Language.YUE_CHINESE: TranslationLiterals(language=Language.YUE_CHINESE),
     Language.ZULU: TranslationLiterals(language=Language.ZULU),
 }
+
+TRANSLATION_LITERALS = TranslationLiteralsDict(TRANSLATION_LITERALS)

src/lighteval/utils/language.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from dataclasses import dataclass
 from enum import Enum
 
 
@@ -240,6 +241,51 @@ class Language(Enum):
     ZULU = "zul"
 
 
+class Script(str, Enum):
+    ARABIC = "Arab"
+    ARMENIAN = "Armn"
+    BENGALI = "Beng"
+    CYRILLIC = "Cyrl"
+    DEVANAGARI = "Deva"
+    ETHIOPIC = "Ethi"
+    GEORGIAN = "Geor"
+    GREEK = "Grek"
+    GUJARATI = "Gujr"
+    GURMUKHI = "Guru"
+    HAN_SIMPLIFIED = "Hans"
+    HAN_TRADITIONAL = "Hant"
+    HANGUL = "Hang"
+    HEBREW = "Hebr"
+    JAPANESE = "Jpan"
+    KANNADA = "Knda"
+    KHMER = "Khmr"
+    LAO = "Laoo"
+    LATIN = "Latn"
+    MALAYALAM = "Mlym"
+    MYANMAR = "Mymr"
+    ODIA = "Orya"
+    OL_CHIKI = "Olck"
+    SINHALA = "Sinh"
+    TAMIL = "Taml"
+    TELUGU = "Telu"
+    THAI = "Thai"
+    TIBETAN = "Tibt"
+    TIFINAGH = "Tfng"
+
+
+@dataclass(frozen=True)
+class LanguageWithScript:
+    language: Language
+    script: Script
+
+    @property
+    def value(self) -> str:
+        return self.language.value
+
+
+LanguageTag = Language | LanguageWithScript
+
+
 # This mapping was created for beleble, it converts iso_639_3 individual codes to iso_639_3 macro codes
 # However it requires iso639-lang package and I don't see a point installing it just for this mapping
 # Code to generate:
@@ -264,6 +310,14 @@ def manage_duplicate_language_codes(langcode):
     return langcode
 
 
+def language_from_tag(language_tag: str) -> LanguageTag:
+    language_code, _, script_code = language_tag.partition("_")
+    language = Language(manage_duplicate_language_codes(language_code))
+    if not script_code:
+        return language
+    return LanguageWithScript(language=language, script=Script(script_code))
+
+
 iso_639_3_ind_to_iso_639_3_macro = {
     "acm": Language.ARABIC,
     "arz": Language.ARABIC,

tests/unit/tasks/templates/test_multichoice.py

@@ -22,7 +22,7 @@
 
 from lighteval.tasks.templates.multichoice import get_mcq_prompt_function
 from lighteval.tasks.templates.utils.formulation import CFFormulation, MCFFormulation
-from lighteval.utils.language import Language
+from lighteval.utils.language import Language, LanguageWithScript, Script
 
 
 def test_multichoice_prompt_mcf():
@@ -119,6 +119,35 @@ def test_chinese_multichoice_prompt():
     )
 
 
+def test_traditional_chinese_multichoice_prompt():
+    """Test multichoice prompt generation for Traditional Chinese."""
+    test_input = {
+        "question": "什麼是中國的首都?",
+        "choices": ["北京", "上海", "廣州", "深圳"],
+        "gold_idx": 0,
+    }
+
+    prompt_fn = get_mcq_prompt_function(
+        LanguageWithScript(language=Language.CHINESE, script=Script.HAN_TRADITIONAL),
+        {"question": "question", "choices": "choices", "gold_idx": "gold_idx"},
+        MCFFormulation(),
+    )
+
+    doc = prompt_fn(test_input, "test_task")
+
+    assert (
+        doc.query
+        == """\
+問題：什麼是中國的首都？
+A。北京
+B。上海
+C。廣州
+D。深圳
+答案：\
+"""
+    )
+
+
 def test_thai_multichoice_prompt():
     """Test multichoice prompt generation for Thai language."""
     test_input = {

tests/unit/tasks/test_flores200.py

@@ -0,0 +1,27 @@
+# MIT License
+
+# Copyright (c) 2024 The HuggingFace Team
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+from lighteval.tasks.multilingual.tasks.flores200 import flores_200_languages
+
+
+def test_flores200_includes_traditional_chinese():
+    assert "zho_Hant" in flores_200_languages

tests/unit/utils/test_language.py

@@ -0,0 +1,36 @@
+# MIT License
+
+# Copyright (c) 2024 The HuggingFace Team
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+from lighteval.utils.language import Language, LanguageWithScript, Script, language_from_tag
+
+
+def test_language_from_tag_keeps_script():
+    language = language_from_tag("zho_Hant")
+
+    assert language == LanguageWithScript(language=Language.CHINESE, script=Script.HAN_TRADITIONAL)
+    assert language.value == Language.CHINESE.value
+
+
+def test_language_from_tag_normalizes_duplicate_code():
+    language = language_from_tag("npi_Deva")
+
+    assert language == LanguageWithScript(language=Language.NEPALI, script=Script.DEVANAGARI)