diff --git a/src/lighteval/tasks/tasks/bigbench.py b/src/lighteval/tasks/tasks/bigbench.py index 9c9487dcf..39701d8d6 100644 --- a/src/lighteval/tasks/tasks/bigbench.py +++ b/src/lighteval/tasks/tasks/bigbench.py @@ -101,8 +101,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="abstract_narrative_understanding", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -119,8 +119,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="anachronisms", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -137,8 +137,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="analogical_similarity", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -155,8 +155,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="analytic_entailment", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -173,8 +173,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="arithmetic", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -188,8 +188,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="ascii_word_recognition", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -206,8 +206,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="authorship_verification", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -224,8 +224,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="auto_categorization", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -239,8 +239,8 @@ def record_to_sample(record): prompt_function=bigbench_linefeed_before_and_after_query_prompt, hf_repo="tasksource/bigbench", hf_subset="auto_debugging", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=100, @@ -254,8 +254,8 @@ def record_to_sample(record): prompt_function=bigbench_linefeed_before_whitespace_after_query_prompt, hf_repo="tasksource/bigbench", hf_subset="bbq_lite_json", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -272,8 +272,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="bridging_anaphora_resolution_barqa", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -290,8 +290,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="causal_judgment", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -308,8 +308,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="cause_and_effect", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -326,8 +326,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="checkmate_in_one", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -341,8 +341,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="chess_state_tracking", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -356,8 +356,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="chinese_remainder_theorem", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -371,8 +371,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="cifar10_classification", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -389,8 +389,8 @@ def record_to_sample(record): prompt_function=bigbench_linefeed_before_and_after_query_prompt, hf_repo="tasksource/bigbench", hf_subset="code_line_description", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -407,8 +407,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="codenames", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -422,8 +422,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="color", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -442,8 +442,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="common_morpheme", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -460,8 +460,8 @@ def record_to_sample(record): prompt_function=bigbench_linefeed_before_whitespace_after_query_prompt, hf_repo="tasksource/bigbench", hf_subset="conceptual_combinations", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -478,8 +478,8 @@ def record_to_sample(record): prompt_function=bigbench_whitespace_after_query_prompt, hf_repo="tasksource/bigbench", hf_subset="conlang_translation", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=100, @@ -493,8 +493,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="contextual_parametric_knowledge_conflicts", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -508,8 +508,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="crash_blossom", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -526,8 +526,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="crass_ai", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -544,8 +544,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="cryobiology_spanish", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -562,8 +562,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="cryptonite", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -577,8 +577,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="cs_algorithms", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -595,8 +595,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="dark_humor_detection", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -613,8 +613,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="date_understanding", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -631,8 +631,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="disambiguation_qa", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -649,8 +649,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="discourse_marker_prediction", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -667,8 +667,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="disfl_qa", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -682,8 +682,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="dyck_languages", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -700,8 +700,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="elementary_math_qa", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -718,8 +718,8 @@ def record_to_sample(record): prompt_function=bigbench_linefeed_before_whitespace_after_query_prompt, hf_repo="tasksource/bigbench", hf_subset="emoji_movie", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -738,8 +738,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="emojis_emotion_prediction", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -756,8 +756,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="empirical_judgments", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -774,8 +774,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="english_proverbs", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -792,8 +792,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="english_russian_proverbs", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -810,8 +810,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="entailed_polarity", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -825,8 +825,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="entailed_polarity_hindi", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -840,8 +840,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="epistemic_reasoning", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -855,8 +855,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="evaluating_information_essentiality", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -870,8 +870,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="fact_checker", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -885,8 +885,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="fantasy_reasoning", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -900,8 +900,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="few_shot_nlg", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -915,8 +915,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="figure_of_speech_detection", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -930,8 +930,8 @@ def record_to_sample(record): prompt_function=bigbench_linefeed_before_whitespace_after_query_prompt, hf_repo="tasksource/bigbench", hf_subset="formal_fallacies_syllogisms_negation", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -945,8 +945,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="gem", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -963,8 +963,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="gender_inclusive_sentences_german", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -981,8 +981,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="general_knowledge", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -999,8 +999,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="geometric_shapes", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1019,8 +1019,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="goal_step_wikihow", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1034,8 +1034,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="gre_reading_comprehension", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1049,8 +1049,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="hhh_alignment", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=-1, @@ -1064,8 +1064,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="hindi_question_answering", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1079,8 +1079,8 @@ def record_to_sample(record): prompt_function=bigbench_linefeed_before_whitespace_after_query_prompt, hf_repo="tasksource/bigbench", hf_subset="hindu_knowledge", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1094,8 +1094,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="hinglish_toxicity", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1109,8 +1109,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="human_organs_senses", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1124,8 +1124,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="hyperbaton", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1139,8 +1139,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="identify_math_theorems", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1154,8 +1154,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="identify_odd_metaphor", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1169,8 +1169,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="implicatures", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1184,8 +1184,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="implicit_relations", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1199,8 +1199,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="intent_recognition", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1214,8 +1214,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="international_phonetic_alphabet_nli", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1229,8 +1229,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="international_phonetic_alphabet_transliterate", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1244,8 +1244,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="intersect_geometry", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1259,8 +1259,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="irony_identification", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1274,8 +1274,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="kanji_ascii", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1289,8 +1289,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="kannada", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1304,8 +1304,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="key_value_maps", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1319,8 +1319,8 @@ def record_to_sample(record): prompt_function=bigbench_linefeed_before_whitespace_after_query_prompt, hf_repo="tasksource/bigbench", hf_subset="known_unknowns", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1334,8 +1334,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="language_games", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1349,8 +1349,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="language_identification", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1364,8 +1364,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="linguistic_mappings", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1382,8 +1382,8 @@ def record_to_sample(record): prompt_function=bigbench_whitespace_after_query_prompt, hf_repo="tasksource/bigbench", hf_subset="linguistics_puzzles", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=100, @@ -1397,8 +1397,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="logic_grid_puzzle", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1415,8 +1415,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="logical_args", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1430,8 +1430,8 @@ def record_to_sample(record): prompt_function=bigbench_whitespace_after_query_prompt, hf_repo="tasksource/bigbench", hf_subset="logical_deduction", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1445,8 +1445,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="logical_fallacy_detection", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1460,8 +1460,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="logical_sequence", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1475,8 +1475,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="mathematical_induction", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1490,8 +1490,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="matrixshapes", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1508,8 +1508,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="metaphor_boolean", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1523,8 +1523,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="metaphor_understanding", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1538,8 +1538,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="minute_mysteries_qa", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1553,8 +1553,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="misconceptions", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1568,8 +1568,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="misconceptions_russian", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1583,8 +1583,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="mnist_ascii", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1598,8 +1598,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="modified_arithmetic", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1616,8 +1616,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="moral_permissibility", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1631,8 +1631,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="movie_dialog_same_or_different", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1646,8 +1646,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="movie_recommendation", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1661,8 +1661,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="mult_data_wrangling", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1679,8 +1679,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="navigate", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1697,8 +1697,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="nonsense_words_grammar", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1712,8 +1712,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="novel_concepts", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1727,8 +1727,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="object_counting", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1742,8 +1742,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="odd_one_out", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1757,8 +1757,8 @@ def record_to_sample(record): prompt_function=bigbench_whitespace_after_query_prompt, hf_repo="tasksource/bigbench", hf_subset="operators", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=20, @@ -1772,8 +1772,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="paragraph_segmentation", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1787,8 +1787,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="parsinlu_qa", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1802,8 +1802,8 @@ def record_to_sample(record): prompt_function=bigbench_linefeed_before_whitespace_after_query_prompt, hf_repo="tasksource/bigbench", hf_subset="parsinlu_reading_comprehension", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=100, @@ -1817,8 +1817,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="penguins_in_a_table", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1832,8 +1832,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="periodic_elements", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1847,8 +1847,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="persian_idioms", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1862,8 +1862,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="phrase_relatedness", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1877,8 +1877,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="physical_intuition", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1892,8 +1892,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="physics", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1907,8 +1907,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="physics_questions", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1922,8 +1922,8 @@ def record_to_sample(record): prompt_function=bigbench_linefeed_before_whitespace_after_query_prompt, hf_repo="tasksource/bigbench", hf_subset="play_dialog_same_or_different", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1937,8 +1937,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="polish_sequence_labeling", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1952,8 +1952,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="presuppositions_as_nli", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1967,8 +1967,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="qa_wikidata", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -1987,8 +1987,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="question_selection", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -2002,8 +2002,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="real_or_fake_text", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -2017,8 +2017,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="reasoning_about_colored_objects", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -2032,8 +2032,8 @@ def record_to_sample(record): prompt_function=bigbench_whitespace_after_query_prompt, hf_repo="tasksource/bigbench", hf_subset="repeat_copy_logic", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=100, @@ -2047,8 +2047,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="rephrase", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -2067,8 +2067,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="rhyming", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -2082,8 +2082,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="riddle_sense", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -2097,8 +2097,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="ruin_names", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -2112,8 +2112,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="salient_translation_error_detection", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -2127,8 +2127,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="scientific_press_release", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -2142,8 +2142,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="semantic_parsing_in_context_sparc", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -2157,8 +2157,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="semantic_parsing_spider", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -2172,8 +2172,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="sentence_ambiguity", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -2187,8 +2187,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="similarities_abstraction", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -2205,8 +2205,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="simp_turing_concept", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -2220,8 +2220,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="simple_arithmetic_json", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -2235,8 +2235,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="simple_arithmetic_json_multiple_choice", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -2250,8 +2250,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="simple_arithmetic_json_subtasks", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -2265,8 +2265,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="simple_arithmetic_multiple_targets_json", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -2280,8 +2280,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="simple_ethical_questions", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -2295,8 +2295,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="simple_text_editing", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -2310,8 +2310,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="snarks", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -2325,8 +2325,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="social_iqa", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -2340,8 +2340,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="social_support", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -2358,8 +2358,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="sports_understanding", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -2373,8 +2373,8 @@ def record_to_sample(record): prompt_function=bigbench_whitespace_after_query_prompt, hf_repo="tasksource/bigbench", hf_subset="strange_stories", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -2388,8 +2388,8 @@ def record_to_sample(record): prompt_function=bigbench_linefeed_before_whitespace_after_query_prompt, hf_repo="tasksource/bigbench", hf_subset="strategyqa", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -2403,8 +2403,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="sufficient_information", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -2418,8 +2418,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="suicide_risk", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -2433,8 +2433,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="swahili_english_proverbs", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -2448,8 +2448,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="swedish_to_german_proverbs", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -2463,8 +2463,8 @@ def record_to_sample(record): prompt_function=bigbench_linefeed_before_whitespace_after_query_prompt, hf_repo="tasksource/bigbench", hf_subset="symbol_interpretation", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -2478,8 +2478,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="tellmewhy", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -2493,8 +2493,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="temporal_sequences", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -2508,8 +2508,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="tense", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -2523,8 +2523,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="timedial", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -2538,8 +2538,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="topical_chat", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -2556,8 +2556,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="tracking_shuffled_objects", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -2571,8 +2571,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="understanding_fables", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -2586,8 +2586,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="undo_permutation", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -2601,8 +2601,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="unit_conversion", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -2616,8 +2616,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="unit_interpretation", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -2631,8 +2631,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="unnatural_in_context_learning", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -2646,8 +2646,8 @@ def record_to_sample(record): prompt_function=bigbench_whitespace_after_query_prompt, hf_repo="tasksource/bigbench", hf_subset="vitaminc_fact_verification", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -2661,8 +2661,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="what_is_the_tao", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -2676,8 +2676,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="which_wiki_edit", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -2691,8 +2691,8 @@ def record_to_sample(record): prompt_function=bigbench_whitespace_after_query_prompt, hf_repo="tasksource/bigbench", hf_subset="winowhy", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -2706,8 +2706,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="word_sorting", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1, @@ -2721,8 +2721,8 @@ def record_to_sample(record): prompt_function=bigbench_prompt, hf_repo="tasksource/bigbench", hf_subset="word_unscrambling", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], + hf_avail_splits=["train", "validation"], + evaluation_splits=["validation"], few_shots_split=None, few_shots_select=None, generation_size=1,