Skip to content

Add metadata for Arabic Enterprise scenarios #4308

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
yifanmai wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Add metadata for Arabic Enterprise scenarios #4308

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 10 additions & 6 deletions src/helm/benchmark/scenarios/arabic_content_generation_scenario.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,16 +57,20 @@ def get_instances(self, output_path: str) -> List[Instance]:

def get_metadata(self) -> ScenarioMetadata:
return ScenarioMetadata(
name=self.name,
display_name="Arabic Content Generation",
description=self.description,
main_metric="exact_match",
name="arabic_content_generation",
main_metric="arabic_content_generation_score",
main_split="test",
display_name="Article Generation",
short_display_name=None,
description="Given summaries of from real news articles, the LLM is prompted to generate "
"new articles in a corporate style. The generated articles are then graded by "
"an LLM-as-judge for faithfulness, completeness, and style adherence.",
short_description=None,
taxonomy=TaxonomyInfo(
task="content generation",
what="news articles",
who="journalists",
what="news articles and press releases",
when="before 2026",
who="journalists and communciations professionals",
language="Arabic",
),
)
65 changes: 65 additions & 0 deletions src/helm/benchmark/scenarios/arabic_finance_scenario.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

import datasets

from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
from helm.benchmark.scenarios.scenario import (
Scenario,
Instance,
Expand All @@ -11,6 +12,7 @@
CORRECT_TAG,
Input,
Output,
ScenarioMetadata,
)
from helm.common.general import ensure_directory_exists

Expand Down Expand Up @@ -85,6 +87,27 @@ def get_references(self, row: Dict[str, str]) -> List[Reference]:
else:
raise ValueError(f"Unknown ground truth value {row[self.ground_truth_key]}")

def get_metadata(self) -> ScenarioMetadata:
return ScenarioMetadata(
name="arabic_finance_bool",
main_metric="quasi_exact_match",
main_split="test",
display_name="Finance Boolean Verification",
short_display_name="Finance Bool",
description="Given questions from English language finance textbooks that have been "
"translated to Arabic using machine translation, the LLM is prompted to answer "
"the questions. In the boolean verification setting, the LLM must decide if a "
"given statement is true or false.",
short_description=None,
taxonomy=TaxonomyInfo(
task="boolean question answering",
what="finance questions",
when="before 2026",
who="finance professionals",
language="Arabic",
),
)


class ArabicFinanceMCQScenario(ArabicFinanceScenario):
name = "arabic_finance_mcq"
Expand Down Expand Up @@ -112,6 +135,27 @@ def get_references(self, row: Dict[str, str]) -> List[Reference]:
assert len(references) == 3
return references

def get_metadata(self) -> ScenarioMetadata:
return ScenarioMetadata(
name="arabic_finance_mcq",
main_metric="exact_match",
main_split="test",
display_name="Finance Multiple Choice Question Answering",
short_display_name="Finance MCQA",
description="Given questions from English language finance textbooks that have been "
"translated to Arabic using machine translation, the LLM is prompted to answer "
"the questions. In the MCQA setting, the LLM must choose between three possible "
"choices.",
short_description=None,
taxonomy=TaxonomyInfo(
task="multiple-choice question answering",
what="finance questions",
when="before 2026",
who="finance professionals",
language="Arabic",
),
)


class ArabicFinanceCalculationScenario(ArabicFinanceScenario):
name = "arabic_finance_calculation"
Expand All @@ -123,3 +167,24 @@ def __init__(self, lang: str):

def get_references(self, row: Dict[str, str]) -> List[Reference]:
return [Reference(output=Output(text=row[self.ground_truth_key]), tags=[CORRECT_TAG])]

def get_metadata(self) -> ScenarioMetadata:
return ScenarioMetadata(
name="arabic_finance_calculation",
main_metric="calculation_accuracy",
main_split="test",
display_name="Finance Calculation",
short_display_name=None,
description="Given questions from English language finance textbooks that have been "
"translated to Arabic using machine translation, the LLM is prompted to answer "
"the questions. In the calculation setting, the LLM must perform mathematical "
"calculations to produce a numeric answer.",
short_description=None,
taxonomy=TaxonomyInfo(
task="calculation",
what="finance questions",
when="before 2026",
who="finance professionals",
language="Arabic",
),
)
43 changes: 43 additions & 0 deletions src/helm/benchmark/scenarios/arabic_legal_scenario.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

import datasets

from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
from helm.benchmark.scenarios.scenario import (
Scenario,
Instance,
Expand Down Expand Up @@ -75,6 +76,27 @@ class ArabicLegalQAScenario(ArabicLegalScenario):
def __init__(self):
super().__init__(task="qa")

def get_metadata(self) -> ScenarioMetadata:
return ScenarioMetadata(
name="arabic_legal_qa",
main_metric="model_judged_score",
main_split="test",
display_name="Legal Closed-book Question Answering",
short_display_name="Legal Closedbook QA",
description="Given open-ended questions written by Arabic legal experts, the LLM is "
"prompted to respond with short answers to the questions. In the closed-book "
"setting, the LLM is not provided with the relevant legal statute, and must "
"answer relying only on its domain knowledge.",
short_description=None,
taxonomy=TaxonomyInfo(
task="knowledge-based question answering",
what="legal questions",
when="before 2026",
who="legal professionals",
language="Arabic",
),
)


class ArabicLegalRAGScenario(ArabicLegalScenario):
name = "arabic_legal_rag"
Expand All @@ -83,3 +105,24 @@ class ArabicLegalRAGScenario(ArabicLegalScenario):

def __init__(self):
super().__init__(task="rag")

def get_metadata(self) -> ScenarioMetadata:
return ScenarioMetadata(
name="arabic_legal_rag",
main_metric="model_judged_score",
main_split="test",
display_name="Legal Open-book Question Answering",
short_display_name="Legal Openbook QA",
description="Given open-ended questions written by Arabic legal experts, the LLM is "
"prompted to respond with short answers to the questions. In the open-book "
"setting, the LLM is also provided with the relevant legal statute in the "
"prompt.",
short_description=None,
taxonomy=TaxonomyInfo(
task="context-based question answering",
what="legal questions",
when="before 2026",
who="legal professionals",
language="Arabic",
),
)
Loading