EleutherAI · KlaudiaTH · Jan 10, 2025
@@ -0,0 +1,20 @@
+tag:
+  - ai2_arc
+  - eu20_arc
+dataset_path: openGPT-X/arcx
+output_type: multiple_choice
+training_split: train
+validation_split: validation
+test_split: test
+doc_to_target: "{{choices.label.index(answerKey)}}"
+doc_to_choice: "{{choices.text}}"
+should_decontaminate: true
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+  - metric: acc_norm
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 1
@@ -0,0 +1,89 @@
+import os
+import yaml
+import argparse
+
+from lm_eval.utils import logging
+
+
+LANGS = [
+    "BG",
+    "DA",
+    "DE",
+    "ET",
+    "FI",
+    "FR",
+    "EL",
+    "IT",
+    "LV",
+    "LT",
+    "NL",
+    "PL",
+    "PT-PT",
+    "RO",
+    "SV",
+    "SK",
+    "SL",
+    "ES",
+    "CS",
+    "HU",
+]
+
+
+PROMPT_WORDS = {
+    "BG": ("Въпрос", "Отговор"),
+    "DA": ("Spørgsmål", "Svar"),
+    "DE": ("Frage", "Antwort"),
+    "ET": ("Küsimus", "Vastus"),
+    "FI": ("Kysymys", "Vastaa"),
+    "FR": ("Question", "Réponse"),
+    "EL": ("Ερώτηση", "Απάντηση"),
+    "IT": ("Domanda", "Risposta"),
+    "LV": ("Jautājums", "Atbilde"),
+    "LT": ("Klausimas", "Atsakymas"),
+    "NL": ("Vraag", "Antwoord"),
+    "PL": ("Pytanie", "Odpowiedź"),
+    "PT-PT": ("Questão", "Resposta"),
+    "RO": ("Întrebare", "Răspuns"),
+    "SV": ("Fråga", "Svar"),
+    "SK": ("Otázka", "Odpoveď"),
+    "SL": ("Vprašanje", "Odgovor"),
+    "ES": ("Pregunta", "Respuesta"),
+    "CS": ("Otázka", "Odpověď"),
+    "HU": ("Kérdés", "Válasz"),
+}
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--base_yaml_path", required=True)
+    parser.add_argument("--save_prefix_path", default="eu20_arc")
+
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    args = parse_args()
+    base_yaml_name = os.path.split(args.base_yaml_path)[-1]
+
+    for split in ["easy", "challenge"]:
+        for lang in LANGS:
+            yaml_dict = {
+                "include": base_yaml_name,
+                "task": f"eu20_arc_{split}_{lang.lower()}",
+                "dataset_name": f"{split}_{lang}",
+                "doc_to_text": f"{PROMPT_WORDS[lang][0]}: {{{{question}}}}\n{PROMPT_WORDS[lang][1]}:",
+                "doc_to_decontamination_query": f"{PROMPT_WORDS[lang][0]}: {{{{question}}}}\n{PROMPT_WORDS[lang][1]}:",
+            }
+
+            file_save_path = f"{args.save_prefix_path}_{split}_{lang.lower()}.yaml"
+
+            logging.info(f"Saving yaml for subset {split}_{lang} to {file_save_path}")
+
+            with open(file_save_path, "w", encoding="utf-8") as yaml_file:
+                yaml.dump(
+                    yaml_dict,
+                    yaml_file,
+                    allow_unicode=True,
+                    default_style='"',
+                    sort_keys=False,
+                )
@@ -0,0 +1,5 @@
+"include": "_default_eu20_arc_template_yaml"
+"task": "eu20_arc_challenge_bg"
+"dataset_name": "challenge_BG"
+"doc_to_text": "Въпрос: {{question}}\nОтговор:"
+"doc_to_decontamination_query": "Въпрос: {{question}}\nОтговор:"
@@ -0,0 +1,5 @@
+"include": "_default_eu20_arc_template_yaml"
+"task": "eu20_arc_challenge_cs"
+"dataset_name": "challenge_CS"
+"doc_to_text": "Otázka: {{question}}\nOdpověď:"
+"doc_to_decontamination_query": "Otázka: {{question}}\nOdpověď:"
@@ -0,0 +1,5 @@
+"include": "_default_eu20_arc_template_yaml"
+"task": "eu20_arc_challenge_da"
+"dataset_name": "challenge_DA"
+"doc_to_text": "Spørgsmål: {{question}}\nSvar:"
+"doc_to_decontamination_query": "Spørgsmål: {{question}}\nSvar:"
@@ -0,0 +1,5 @@
+"include": "_default_eu20_arc_template_yaml"
+"task": "eu20_arc_challenge_de"
+"dataset_name": "challenge_DE"
+"doc_to_text": "Frage: {{question}}\nAntwort:"
+"doc_to_decontamination_query": "Frage: {{question}}\nAntwort:"
@@ -0,0 +1,5 @@
+"include": "_default_eu20_arc_template_yaml"
+"task": "eu20_arc_challenge_el"
+"dataset_name": "challenge_EL"
+"doc_to_text": "Ερώτηση: {{question}}\nΑπάντηση:"
+"doc_to_decontamination_query": "Ερώτηση: {{question}}\nΑπάντηση:"
@@ -0,0 +1,5 @@
+"include": "_default_eu20_arc_template_yaml"
+"task": "eu20_arc_challenge_es"
+"dataset_name": "challenge_ES"
+"doc_to_text": "Pregunta: {{question}}\nRespuesta:"
+"doc_to_decontamination_query": "Pregunta: {{question}}\nRespuesta:"
@@ -0,0 +1,5 @@
+"include": "_default_eu20_arc_template_yaml"
+"task": "eu20_arc_challenge_et"
+"dataset_name": "challenge_ET"
+"doc_to_text": "Küsimus: {{question}}\nVastus:"
+"doc_to_decontamination_query": "Küsimus: {{question}}\nVastus:"
@@ -0,0 +1,5 @@
+"include": "_default_eu20_arc_template_yaml"
+"task": "eu20_arc_challenge_fi"
+"dataset_name": "challenge_FI"
+"doc_to_text": "Kysymys: {{question}}\nVastaa:"
+"doc_to_decontamination_query": "Kysymys: {{question}}\nVastaa:"
@@ -0,0 +1,5 @@
+"include": "_default_eu20_arc_template_yaml"
+"task": "eu20_arc_challenge_fr"
+"dataset_name": "challenge_FR"
+"doc_to_text": "Question: {{question}}\nRéponse:"
+"doc_to_decontamination_query": "Question: {{question}}\nRéponse:"
@@ -0,0 +1,5 @@
+"include": "_default_eu20_arc_template_yaml"
+"task": "eu20_arc_challenge_hu"
+"dataset_name": "challenge_HU"
+"doc_to_text": "Kérdés: {{question}}\nVálasz:"
+"doc_to_decontamination_query": "Kérdés: {{question}}\nVálasz:"
@@ -0,0 +1,5 @@
+"include": "_default_eu20_arc_template_yaml"
+"task": "eu20_arc_challenge_it"
+"dataset_name": "challenge_IT"
+"doc_to_text": "Domanda: {{question}}\nRisposta:"
+"doc_to_decontamination_query": "Domanda: {{question}}\nRisposta:"
@@ -0,0 +1,5 @@
+"include": "_default_eu20_arc_template_yaml"
+"task": "eu20_arc_challenge_lt"
+"dataset_name": "challenge_LT"
+"doc_to_text": "Klausimas: {{question}}\nAtsakymas:"
+"doc_to_decontamination_query": "Klausimas: {{question}}\nAtsakymas:"
@@ -0,0 +1,5 @@
+"include": "_default_eu20_arc_template_yaml"
+"task": "eu20_arc_challenge_lv"
+"dataset_name": "challenge_LV"
+"doc_to_text": "Jautājums: {{question}}\nAtbilde:"
+"doc_to_decontamination_query": "Jautājums: {{question}}\nAtbilde:"
@@ -0,0 +1,5 @@
+"include": "_default_eu20_arc_template_yaml"
+"task": "eu20_arc_challenge_nl"
+"dataset_name": "challenge_NL"
+"doc_to_text": "Vraag: {{question}}\nAntwoord:"
+"doc_to_decontamination_query": "Vraag: {{question}}\nAntwoord:"
@@ -0,0 +1,5 @@
+"include": "_default_eu20_arc_template_yaml"
+"task": "eu20_arc_challenge_pl"
+"dataset_name": "challenge_PL"
+"doc_to_text": "Pytanie: {{question}}\nOdpowiedź:"
+"doc_to_decontamination_query": "Pytanie: {{question}}\nOdpowiedź:"
@@ -0,0 +1,5 @@
+"include": "_default_eu20_arc_template_yaml"
+"task": "eu20_arc_challenge_pt-pt"
+"dataset_name": "challenge_PT-PT"
+"doc_to_text": "Questão: {{question}}\nResposta:"
+"doc_to_decontamination_query": "Questão: {{question}}\nResposta:"
@@ -0,0 +1,5 @@
+"include": "_default_eu20_arc_template_yaml"
+"task": "eu20_arc_challenge_ro"
+"dataset_name": "challenge_RO"
+"doc_to_text": "Întrebare: {{question}}\nRăspuns:"
+"doc_to_decontamination_query": "Întrebare: {{question}}\nRăspuns:"
@@ -0,0 +1,5 @@
+"include": "_default_eu20_arc_template_yaml"
+"task": "eu20_arc_challenge_sk"
+"dataset_name": "challenge_SK"
+"doc_to_text": "Otázka: {{question}}\nOdpoveď:"
+"doc_to_decontamination_query": "Otázka: {{question}}\nOdpoveď:"
@@ -0,0 +1,5 @@
+"include": "_default_eu20_arc_template_yaml"
+"task": "eu20_arc_challenge_sl"
+"dataset_name": "challenge_SL"
+"doc_to_text": "Vprašanje: {{question}}\nOdgovor:"
+"doc_to_decontamination_query": "Vprašanje: {{question}}\nOdgovor:"
@@ -0,0 +1,5 @@
+"include": "_default_eu20_arc_template_yaml"
+"task": "eu20_arc_challenge_sv"
+"dataset_name": "challenge_SV"
+"doc_to_text": "Fråga: {{question}}\nSvar:"
+"doc_to_decontamination_query": "Fråga: {{question}}\nSvar:"
@@ -0,0 +1,5 @@
+"include": "_default_eu20_arc_template_yaml"
+"task": "eu20_arc_easy_bg"
+"dataset_name": "easy_BG"
+"doc_to_text": "Въпрос: {{question}}\nОтговор:"
+"doc_to_decontamination_query": "Въпрос: {{question}}\nОтговор:"
@@ -0,0 +1,5 @@
+"include": "_default_eu20_arc_template_yaml"
+"task": "eu20_arc_easy_cs"
+"dataset_name": "easy_CS"
+"doc_to_text": "Otázka: {{question}}\nOdpověď:"
+"doc_to_decontamination_query": "Otázka: {{question}}\nOdpověď:"
@@ -0,0 +1,5 @@
+"include": "_default_eu20_arc_template_yaml"
+"task": "eu20_arc_easy_da"
+"dataset_name": "easy_DA"
+"doc_to_text": "Spørgsmål: {{question}}\nSvar:"
+"doc_to_decontamination_query": "Spørgsmål: {{question}}\nSvar:"
@@ -0,0 +1,5 @@
+"include": "_default_eu20_arc_template_yaml"
+"task": "eu20_arc_easy_de"
+"dataset_name": "easy_DE"
+"doc_to_text": "Frage: {{question}}\nAntwort:"
+"doc_to_decontamination_query": "Frage: {{question}}\nAntwort:"
@@ -0,0 +1,5 @@
+"include": "_default_eu20_arc_template_yaml"
+"task": "eu20_arc_easy_el"
+"dataset_name": "easy_EL"
+"doc_to_text": "Ερώτηση: {{question}}\nΑπάντηση:"
+"doc_to_decontamination_query": "Ερώτηση: {{question}}\nΑπάντηση:"
@@ -0,0 +1,5 @@
+"include": "_default_eu20_arc_template_yaml"
+"task": "eu20_arc_easy_es"
+"dataset_name": "easy_ES"
+"doc_to_text": "Pregunta: {{question}}\nRespuesta:"
+"doc_to_decontamination_query": "Pregunta: {{question}}\nRespuesta:"
@@ -0,0 +1,5 @@
+"include": "_default_eu20_arc_template_yaml"
+"task": "eu20_arc_easy_et"
+"dataset_name": "easy_ET"
+"doc_to_text": "Küsimus: {{question}}\nVastus:"
+"doc_to_decontamination_query": "Küsimus: {{question}}\nVastus:"
@@ -0,0 +1,5 @@
+"include": "_default_eu20_arc_template_yaml"
+"task": "eu20_arc_easy_fi"
+"dataset_name": "easy_FI"
+"doc_to_text": "Kysymys: {{question}}\nVastaa:"
+"doc_to_decontamination_query": "Kysymys: {{question}}\nVastaa:"
@@ -0,0 +1,5 @@
+"include": "_default_eu20_arc_template_yaml"
+"task": "eu20_arc_easy_fr"
+"dataset_name": "easy_FR"
+"doc_to_text": "Question: {{question}}\nRéponse:"
+"doc_to_decontamination_query": "Question: {{question}}\nRéponse:"
@@ -0,0 +1,5 @@
+"include": "_default_eu20_arc_template_yaml"
+"task": "eu20_arc_easy_hu"
+"dataset_name": "easy_HU"
+"doc_to_text": "Kérdés: {{question}}\nVálasz:"
+"doc_to_decontamination_query": "Kérdés: {{question}}\nVálasz:"
@@ -0,0 +1,5 @@
+"include": "_default_eu20_arc_template_yaml"
+"task": "eu20_arc_easy_it"
+"dataset_name": "easy_IT"
+"doc_to_text": "Domanda: {{question}}\nRisposta:"
+"doc_to_decontamination_query": "Domanda: {{question}}\nRisposta:"
@@ -0,0 +1,5 @@
+"include": "_default_eu20_arc_template_yaml"
+"task": "eu20_arc_easy_lt"
+"dataset_name": "easy_LT"
+"doc_to_text": "Klausimas: {{question}}\nAtsakymas:"
+"doc_to_decontamination_query": "Klausimas: {{question}}\nAtsakymas:"
@@ -0,0 +1,5 @@
+"include": "_default_eu20_arc_template_yaml"
+"task": "eu20_arc_easy_lv"
+"dataset_name": "easy_LV"
+"doc_to_text": "Jautājums: {{question}}\nAtbilde:"
+"doc_to_decontamination_query": "Jautājums: {{question}}\nAtbilde:"
@@ -0,0 +1,5 @@
+"include": "_default_eu20_arc_template_yaml"
+"task": "eu20_arc_easy_nl"
+"dataset_name": "easy_NL"
+"doc_to_text": "Vraag: {{question}}\nAntwoord:"
+"doc_to_decontamination_query": "Vraag: {{question}}\nAntwoord:"
@@ -0,0 +1,5 @@
+"include": "_default_eu20_arc_template_yaml"
+"task": "eu20_arc_easy_pl"
+"dataset_name": "easy_PL"
+"doc_to_text": "Pytanie: {{question}}\nOdpowiedź:"
+"doc_to_decontamination_query": "Pytanie: {{question}}\nOdpowiedź:"
@@ -0,0 +1,5 @@
+"include": "_default_eu20_arc_template_yaml"
+"task": "eu20_arc_easy_pt-pt"
+"dataset_name": "easy_PT-PT"
+"doc_to_text": "Questão: {{question}}\nResposta:"
+"doc_to_decontamination_query": "Questão: {{question}}\nResposta:"
@@ -0,0 +1,5 @@
+"include": "_default_eu20_arc_template_yaml"
+"task": "eu20_arc_easy_ro"
+"dataset_name": "easy_RO"
+"doc_to_text": "Întrebare: {{question}}\nRăspuns:"
+"doc_to_decontamination_query": "Întrebare: {{question}}\nRăspuns:"
@@ -0,0 +1,5 @@
+"include": "_default_eu20_arc_template_yaml"
+"task": "eu20_arc_easy_sk"
+"dataset_name": "easy_SK"
+"doc_to_text": "Otázka: {{question}}\nOdpoveď:"
+"doc_to_decontamination_query": "Otázka: {{question}}\nOdpoveď:"
@@ -0,0 +1,5 @@
+"include": "_default_eu20_arc_template_yaml"
+"task": "eu20_arc_easy_sl"
+"dataset_name": "easy_SL"
+"doc_to_text": "Vprašanje: {{question}}\nOdgovor:"
+"doc_to_decontamination_query": "Vprašanje: {{question}}\nOdgovor:"
@@ -0,0 +1,5 @@
+"include": "_default_eu20_arc_template_yaml"
+"task": "eu20_arc_easy_sv"
+"dataset_name": "easy_SV"
+"doc_to_text": "Fråga: {{question}}\nSvar:"
+"doc_to_decontamination_query": "Fråga: {{question}}\nSvar:"
@@ -0,0 +1,18 @@
+tag:
+  - math_word_problems
+  - eu20_gsm8k
+dataset_path: openGPT-X/gsm8kx
+output_type: generate_until
+training_split: train
+fewshot_split: train
+test_split: test
+doc_to_target: "{{answer}}"
+process_results: !function utils.process_results
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+repeats: 1
+num_fewshot: 5
+metadata:
+  version: 1