From 4a6f847d21f0efc47c75a57c27638e8c624ef76f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=B6ktu=C4=9F=20=C3=96zkan=2C=20MD?= Date: Tue, 23 Jun 2026 16:17:35 +0300 Subject: [PATCH] Add script-aware language tags for Flores --- .../tasks/multilingual/tasks/flores200.py | 8 +-- .../templates/utils/translation_literals.py | 39 ++++++++++++-- src/lighteval/utils/language.py | 54 +++++++++++++++++++ .../unit/tasks/templates/test_multichoice.py | 31 ++++++++++- tests/unit/tasks/test_flores200.py | 27 ++++++++++ tests/unit/utils/test_language.py | 36 +++++++++++++ 6 files changed, 187 insertions(+), 8 deletions(-) create mode 100644 tests/unit/tasks/test_flores200.py create mode 100644 tests/unit/utils/test_language.py diff --git a/src/lighteval/tasks/multilingual/tasks/flores200.py b/src/lighteval/tasks/multilingual/tasks/flores200.py index f6adbef78..1eb5431b3 100644 --- a/src/lighteval/tasks/multilingual/tasks/flores200.py +++ b/src/lighteval/tasks/multilingual/tasks/flores200.py @@ -28,7 +28,7 @@ from lighteval.tasks.templates.utils.formulation import ( CFFormulation, ) -from lighteval.utils.language import Language, manage_duplicate_language_codes +from lighteval.utils.language import language_from_tag flores_200_languages = [ @@ -233,7 +233,7 @@ "yor_Latn", "yue_Hant", "zho_Hans", - # "zho_Hant", + "zho_Hant", "zsm_Latn", "zul_Latn", ] @@ -250,8 +250,8 @@ def flores_adapter(lang1, lang2): LightevalTaskConfig( name=f"flores200:{lang1}-{lang2}", prompt_function=get_translation_prompt_function( - source_language=Language(manage_duplicate_language_codes(lang1.split("_")[0])), - target_language=Language(manage_duplicate_language_codes(lang2.split("_")[0])), + source_language=language_from_tag(lang1), + target_language=language_from_tag(lang2), adapter=flores_adapter(lang1, lang2), formulation=CFFormulation(), ), diff --git a/src/lighteval/tasks/templates/utils/translation_literals.py b/src/lighteval/tasks/templates/utils/translation_literals.py index 96cd8fc2c..ebd499b4f 100644 --- a/src/lighteval/tasks/templates/utils/translation_literals.py +++ b/src/lighteval/tasks/templates/utils/translation_literals.py @@ -23,7 +23,7 @@ from dataclasses import dataclass, field from string import ascii_uppercase -from lighteval.utils.language import Language +from lighteval.utils.language import Language, LanguageWithScript, Script # TODO(hynky1999): The typing still is not great, it should be able to infer that you can't access the @@ -31,7 +31,7 @@ @dataclass class TranslationLiterals: # This is just to create nice error messages - language: Language + language: Language | LanguageWithScript question_word: str = None # type: ignore answer: str = None # type: ignore @@ -74,7 +74,14 @@ def __getattribute__(self, name: str) -> str: return value -TRANSLATION_LITERALS: dict[Language, TranslationLiterals] = { +class TranslationLiteralsDict(dict): + def __missing__(self, language): + if isinstance(language, LanguageWithScript): + return self[language.language] + raise KeyError(language) + + +TRANSLATION_LITERALS: dict[Language | LanguageWithScript, TranslationLiterals] = { Language.ACEHNESE: TranslationLiterals(language=Language.ACEHNESE), Language.AFRIKAANS: TranslationLiterals(language=Language.AFRIKAANS), Language.AKAN: TranslationLiterals(language=Language.AKAN), @@ -304,6 +311,30 @@ def __getattribute__(self, name: str) -> str: colon="：", indices=["A", "B", "C", "D", "E", "F", "G", "H", "I", "J"], ), + LanguageWithScript(language=Language.CHINESE, script=Script.HAN_TRADITIONAL): TranslationLiterals( + language=LanguageWithScript(language=Language.CHINESE, script=Script.HAN_TRADITIONAL), + question_word="問題", + answer="答案", + confirmation_word="對嗎", + yes="是的", + no="不是", + also="而且", + cause_word="因為", + effect_word="所以", + true="真", + false="假", + neither="都不是", + or_word="或", + and_word="和", + full_stop="。", + comma="，", + question_mark="？", + exclamation_mark="！", + word_space="", + sentence_space="", + colon="：", + indices=["A", "B", "C", "D", "E", "F", "G", "H", "I", "J"], + ), Language.CHOKWE: TranslationLiterals(language=Language.CHOKWE), Language.CRIMEAN_TATAR: TranslationLiterals(language=Language.CRIMEAN_TATAR), Language.CROATIAN: TranslationLiterals( @@ -1426,3 +1457,5 @@ def __getattribute__(self, name: str) -> str: Language.YUE_CHINESE: TranslationLiterals(language=Language.YUE_CHINESE), Language.ZULU: TranslationLiterals(language=Language.ZULU), } + +TRANSLATION_LITERALS = TranslationLiteralsDict(TRANSLATION_LITERALS) diff --git a/src/lighteval/utils/language.py b/src/lighteval/utils/language.py index f2f98ad37..c9c3b0da7 100644 --- a/src/lighteval/utils/language.py +++ b/src/lighteval/utils/language.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from dataclasses import dataclass from enum import Enum @@ -240,6 +241,51 @@ class Language(Enum): ZULU = "zul" +class Script(str, Enum): + ARABIC = "Arab" + ARMENIAN = "Armn" + BENGALI = "Beng" + CYRILLIC = "Cyrl" + DEVANAGARI = "Deva" + ETHIOPIC = "Ethi" + GEORGIAN = "Geor" + GREEK = "Grek" + GUJARATI = "Gujr" + GURMUKHI = "Guru" + HAN_SIMPLIFIED = "Hans" + HAN_TRADITIONAL = "Hant" + HANGUL = "Hang" + HEBREW = "Hebr" + JAPANESE = "Jpan" + KANNADA = "Knda" + KHMER = "Khmr" + LAO = "Laoo" + LATIN = "Latn" + MALAYALAM = "Mlym" + MYANMAR = "Mymr" + ODIA = "Orya" + OL_CHIKI = "Olck" + SINHALA = "Sinh" + TAMIL = "Taml" + TELUGU = "Telu" + THAI = "Thai" + TIBETAN = "Tibt" + TIFINAGH = "Tfng" + + +@dataclass(frozen=True) +class LanguageWithScript: + language: Language + script: Script + + @property + def value(self) -> str: + return self.language.value + + +LanguageTag = Language | LanguageWithScript + + # This mapping was created for beleble, it converts iso_639_3 individual codes to iso_639_3 macro codes # However it requires iso639-lang package and I don't see a point installing it just for this mapping # Code to generate: @@ -264,6 +310,14 @@ def manage_duplicate_language_codes(langcode): return langcode +def language_from_tag(language_tag: str) -> LanguageTag: + language_code, _, script_code = language_tag.partition("_") + language = Language(manage_duplicate_language_codes(language_code)) + if not script_code: + return language + return LanguageWithScript(language=language, script=Script(script_code)) + + iso_639_3_ind_to_iso_639_3_macro = { "acm": Language.ARABIC, "arz": Language.ARABIC, diff --git a/tests/unit/tasks/templates/test_multichoice.py b/tests/unit/tasks/templates/test_multichoice.py index 9b3614ed2..f3d9844bc 100644 --- a/tests/unit/tasks/templates/test_multichoice.py +++ b/tests/unit/tasks/templates/test_multichoice.py @@ -22,7 +22,7 @@ from lighteval.tasks.templates.multichoice import get_mcq_prompt_function from lighteval.tasks.templates.utils.formulation import CFFormulation, MCFFormulation -from lighteval.utils.language import Language +from lighteval.utils.language import Language, LanguageWithScript, Script def test_multichoice_prompt_mcf(): @@ -119,6 +119,35 @@ def test_chinese_multichoice_prompt(): ) +def test_traditional_chinese_multichoice_prompt(): + """Test multichoice prompt generation for Traditional Chinese.""" + test_input = { + "question": "什麼是中國的首都?", + "choices": ["北京", "上海", "廣州", "深圳"], + "gold_idx": 0, + } + + prompt_fn = get_mcq_prompt_function( + LanguageWithScript(language=Language.CHINESE, script=Script.HAN_TRADITIONAL), + {"question": "question", "choices": "choices", "gold_idx": "gold_idx"}, + MCFFormulation(), + ) + + doc = prompt_fn(test_input, "test_task") + + assert ( + doc.query + == """\ +問題：什麼是中國的首都？ +A。北京 +B。上海 +C。廣州 +D。深圳 +答案：\ +""" + ) + + def test_thai_multichoice_prompt(): """Test multichoice prompt generation for Thai language.""" test_input = { diff --git a/tests/unit/tasks/test_flores200.py b/tests/unit/tasks/test_flores200.py new file mode 100644 index 000000000..a6510c3bd --- /dev/null +++ b/tests/unit/tasks/test_flores200.py @@ -0,0 +1,27 @@ +# MIT License + +# Copyright (c) 2024 The HuggingFace Team + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from lighteval.tasks.multilingual.tasks.flores200 import flores_200_languages + + +def test_flores200_includes_traditional_chinese(): + assert "zho_Hant" in flores_200_languages diff --git a/tests/unit/utils/test_language.py b/tests/unit/utils/test_language.py new file mode 100644 index 000000000..1c1c12a6d --- /dev/null +++ b/tests/unit/utils/test_language.py @@ -0,0 +1,36 @@ +# MIT License + +# Copyright (c) 2024 The HuggingFace Team + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from lighteval.utils.language import Language, LanguageWithScript, Script, language_from_tag + + +def test_language_from_tag_keeps_script(): + language = language_from_tag("zho_Hant") + + assert language == LanguageWithScript(language=Language.CHINESE, script=Script.HAN_TRADITIONAL) + assert language.value == Language.CHINESE.value + + +def test_language_from_tag_normalizes_duplicate_code(): + language = language_from_tag("npi_Deva") + + assert language == LanguageWithScript(language=Language.NEPALI, script=Script.DEVANAGARI)