Update preset configs

sbintuitions · Jan 14, 2025 · 269ae7c · 269ae7c
1 parent f2ed5a6
commit 269ae7c
Show file tree

Hide file tree

Showing 30 changed files with 224 additions and 373 deletions.
diff --git a/flexeval/preset_configs/EvalSetup/code_generation/jhumaneval.jsonnet b/flexeval/preset_configs/EvalSetup/code_generation/jhumaneval.jsonnet
@@ -17,12 +17,7 @@ References:
         reference_template: '{{ test }}\n\ncheck({{ entry_point }})\n',
       },
     },
-    prompt_template: {
-      class_path: 'Jinja2PromptTemplate',
-      init_args: {
-        template: '{{ prompt }}',
-      },
-    },
+    prompt_template: '{{ prompt }}',
     metrics: [
       { class_path: 'CodeEval', init_args: { code_template: '{{ prompt }}{{ lm_output }}' } },
     ],

diff --git a/flexeval/preset_configs/EvalSetup/code_generation/jhumaneval_tab_indent.jsonnet b/flexeval/preset_configs/EvalSetup/code_generation/jhumaneval_tab_indent.jsonnet
@@ -13,11 +13,7 @@ original_config {
         reference_template: '{{ test | replace("    ", "\t") }}\n\ncheck({{ entry_point }})\n',
       },
     },
-    prompt_template+: {
-      init_args+: {
-        template: "{{ prompt | replace('    ', '\t') }}",
-      },
-    },
+    prompt_template: "{{ prompt | replace('    ', '\t') }}",
     metrics: [
       { class_path: 'CodeEval', init_args: { code_template: '{{ prompt | replace("    ", "\t") }}{{ lm_output }}' } },
     ],

diff --git a/flexeval/preset_configs/EvalSetup/code_generation/mbpp.jsonnet b/flexeval/preset_configs/EvalSetup/code_generation/mbpp.jsonnet
@@ -26,33 +26,28 @@ local dataset_base_args = {
         num_shots: 3,
       },
     },
-    prompt_template: {
-      class_path: 'Jinja2PromptTemplate',
-      init_args: {
-        template: |||
-          {% for item in few_shot_data %}
-          ## Question
-          {{ item.prompt }}
-          ## Test cases
-          ```python
-          {{ item.test_list | join('\n') }}
-          ```
-          ## Code
-          ```python
-          {{ item.code }}
-          ```
-          {% endfor %}
-          ## Question
-          {{ prompt }}
-          ## Test cases
-          ```python
-          {{ test_list | join('\n') }}
-          ```
-          ## Code
-          ```python
-        |||,
-      },
-    },
+    prompt_template: |||
+      {% for item in few_shot_data %}
+      ## Question
+      {{ item.prompt }}
+      ## Test cases
+      ```python
+      {{ item.test_list | join('\n') }}
+      ```
+      ## Code
+      ```python
+      {{ item.code }}
+      ```
+      {% endfor %}
+      ## Question
+      {{ prompt }}
+      ## Test cases
+      ```python
+      {{ test_list | join('\n') }}
+      ```
+      ## Code
+      ```python
+    |||,
     metrics: [
       { class_path: 'CodeEval' },
     ],

diff --git a/flexeval/preset_configs/EvalSetup/code_generation/mbpp_tab_indent.jsonnet b/flexeval/preset_configs/EvalSetup/code_generation/mbpp_tab_indent.jsonnet
@@ -8,31 +8,27 @@ local original_config = import './mbpp.jsonnet';
 
 original_config {
   init_args+: {
-    prompt_template+: {
-      init_args+: {
-        template: |||
-          {% for item in few_shot_data %}
-          ## Question
-          {{ item.prompt }}
-          ## Test cases
-          ```python
-          {{ item.test_list | join('\n') }}
-          ```
-          ## Code
-          ```python
-          {{ item.code | replace('    ', '\t') }}
-          ```
-          {% endfor %}
-          ## Question
-          {{ prompt }}
-          ## Test cases
-          ```python
-          {{ test_list | join('\n') }}
-          ```
-          ## Code
-          ```python
-        |||,
-      },
-    },
+    prompt_template: |||
+      {% for item in few_shot_data %}
+      ## Question
+      {{ item.prompt }}
+      ## Test cases
+      ```python
+      {{ item.test_list | join('\n') }}
+      ```
+      ## Code
+      ```python
+      {{ item.code | replace('    ', '\t') }}
+      ```
+      {% endfor %}
+      ## Question
+      {{ prompt }}
+      ## Test cases
+      ```python
+      {{ test_list | join('\n') }}
+      ```
+      ## Code
+      ```python
+    |||,
   },
 }
diff --git a/flexeval/preset_configs/EvalSetup/code_generation/openai_humaneval.jsonnet b/flexeval/preset_configs/EvalSetup/code_generation/openai_humaneval.jsonnet
@@ -17,12 +17,7 @@ References:
         reference_template: '{{ test }}\n\ncheck({{ entry_point }})\n',
       },
     },
-    prompt_template: {
-      class_path: 'Jinja2PromptTemplate',
-      init_args: {
-        template: '{{ prompt }}',
-      },
-    },
+    prompt_template: '{{ prompt }}',
     metrics: [
       { class_path: 'CodeEval', init_args: { code_template: '{{ prompt }}{{ lm_output }}' } },
     ],

diff --git a/flexeval/preset_configs/EvalSetup/code_generation/openai_humaneval_tab_indent.jsonnet b/flexeval/preset_configs/EvalSetup/code_generation/openai_humaneval_tab_indent.jsonnet
@@ -13,11 +13,7 @@ original_config {
         reference_template: '{{ test | replace("    ", "\t") }}\n\ncheck({{ entry_point }})\n',
       },
     },
-    prompt_template+: {
-      init_args+: {
-        template: '{{ prompt | replace("    ", "\t") }}',
-      },
-    },
+    prompt_template: '{{ prompt | replace("    ", "\t") }}',
     metrics: [
       { class_path: 'CodeEval', init_args: { code_template: '{{ prompt | replace("    ", "\t") }}{{ lm_output }}' } },
     ],

diff --git a/flexeval/preset_configs/EvalSetup/en_generation/babi.jsonnet b/flexeval/preset_configs/EvalSetup/en_generation/babi.jsonnet
@@ -25,20 +25,15 @@ local dataset_base_args = {
         num_shots: 3,
       },
     },
-    prompt_template: {
-      class_path: 'Jinja2PromptTemplate',
-      init_args: {
-        template: |||
-          {% for item in few_shot_data %}
-          Passage: {{ item.passage | trim }}
-          Question: {{ item.question }}
-          Answer: "{{ item.references[0] }}"
-          {% endfor %}
-          Passage: {{ passage | trim }}
-          Question: {{ question }}
-        ||| + 'Answer: "',
-      },
-    },
+    prompt_template: |||
+      {% for item in few_shot_data %}
+      Passage: {{ item.passage | trim }}
+      Question: {{ item.question }}
+      Answer: "{{ item.references[0] }}"
+      {% endfor %}
+      Passage: {{ passage | trim }}
+      Question: {{ question }}
+    ||| + 'Answer: "',
     metrics: [
       { class_path: 'CharF1' },
       { class_path: 'ExactMatch' },

diff --git a/flexeval/preset_configs/EvalSetup/en_generation/commonsense_qa.jsonnet b/flexeval/preset_configs/EvalSetup/en_generation/commonsense_qa.jsonnet
@@ -26,34 +26,29 @@ local dataset_base_args = {
         num_shots: 2,
       },
     },
-    prompt_template: {
-      class_path: 'Jinja2PromptTemplate',
-      init_args: {
-        template: |||
-          Choose the correct answer from the choices.
-          {% for item in few_shot_data %}
-          Choices:
-          0. "{{ item.choices.text[0] }}"
-          1. "{{ item.choices.text[1] }}"
-          2. "{{ item.choices.text[2] }}"
-          3. "{{ item.choices.text[3] }}"
-          4. "{{ item.choices.text[4] }}"
-          Question: {{ item.question }}
-          Answer: "{{ item.references[0] }}"
-          {% endfor %}
-          Choices:
-          0. "{{ choices.text[0] }}"
-          1. "{{ choices.text[1] }}"
-          2. "{{ choices.text[2] }}"
-          3. "{{ choices.text[3] }}"
-          4. "{{ choices.text[4] }}"
-          Question: {{question}}
-        ||| + 'Answer: "',
-      },
-    },
-    metrics: [
-      { class_path: 'ExactMatch' },
-    ],
-    gen_kwargs: { max_new_tokens: 40, stop_sequences: ['"'] },
+    prompt_template: |||
+      Choose the correct answer from the choices.
+      {% for item in few_shot_data %}
+      Choices:
+      0. "{{ item.choices.text[0] }}"
+      1. "{{ item.choices.text[1] }}"
+      2. "{{ item.choices.text[2] }}"
+      3. "{{ item.choices.text[3] }}"
+      4. "{{ item.choices.text[4] }}"
+      Question: {{ item.question }}
+      Answer: "{{ item.references[0] }}"
+      {% endfor %}
+      Choices:
+      0. "{{ choices.text[0] }}"
+      1. "{{ choices.text[1] }}"
+      2. "{{ choices.text[2] }}"
+      3. "{{ choices.text[3] }}"
+      4. "{{ choices.text[4] }}"
+      Question: {{question}}
+    ||| + 'Answer: "',
   },
+  metrics: [
+    { class_path: 'ExactMatch' },
+  ],
+  gen_kwargs: { max_new_tokens: 40, stop_sequences: ['"'] },
 }
diff --git a/flexeval/preset_configs/EvalSetup/en_generation/gsm8k.jsonnet b/flexeval/preset_configs/EvalSetup/en_generation/gsm8k.jsonnet
@@ -27,18 +27,13 @@ local dataset_base_args = {
         num_shots: 4,
       },
     },
-    prompt_template: {
-      class_path: 'Jinja2PromptTemplate',
-      init_args: {
-        template: |||
-          {% for item in few_shot_data %}
-          Q: {{ item.question }}
-          A: {{ item.references[0] }}
-          {% endfor %}
-          Q: {{ question }}
-        ||| + 'A:',
-      },
-    },
+    prompt_template: |||
+      {% for item in few_shot_data %}
+      Q: {{ item.question }}
+      A: {{ item.references[0] }}
+      {% endfor %}
+      Q: {{ question }}
+    ||| + 'A:',
     metrics: [
       {
         class_path: 'ExactMatch',

diff --git a/flexeval/preset_configs/EvalSetup/en_generation/squad_v1.jsonnet b/flexeval/preset_configs/EvalSetup/en_generation/squad_v1.jsonnet
@@ -26,21 +26,15 @@ local dataset_base_args = {
         num_shots: 2,
       },
     },
-    prompt_template: {
-      class_path: 'Jinja2PromptTemplate',
-      init_args: {
-
-        template: |||
-          {% for item in few_shot_data %}
-          Context: {{ item.context | trim }}
-          Question: {{ item.question }}
-          Answer: "{{ item.references[0] }}"
-          {% endfor %}
-          Context: {{ context | trim }}
-          Question: {{ question }}
-        ||| + 'Answer: "',
-      },
-    },
+    prompt_template: |||
+      {% for item in few_shot_data %}
+      Context: {{ item.context | trim }}
+      Question: {{ item.question }}
+      Answer: "{{ item.references[0] }}"
+      {% endfor %}
+      Context: {{ context | trim }}
+      Question: {{ question }}
+    ||| + 'Answer: "',
     metrics: [
       { class_path: 'CharF1' },
       { class_path: 'ExactMatch' },

diff --git a/flexeval/preset_configs/EvalSetup/en_generation/trivia_qa.jsonnet b/flexeval/preset_configs/EvalSetup/en_generation/trivia_qa.jsonnet
@@ -28,19 +28,13 @@ local dataset_base_args = {
         num_shots: 0,
       },
     },
-    prompt_template: {
-      class_path: 'Jinja2PromptTemplate',
-      init_args: {
-
-        template: |||
-          {% for item in few_shot_data %}
-          Question: {{ item.question }}
-          Answer: "{{ item.references[0] }}"
-          {% endfor %}
-          Question: {{ question }}
-        ||| + 'Answer: "',
-      },
-    },
+    prompt_template: |||
+      {% for item in few_shot_data %}
+      Question: {{ item.question }}
+      Answer: "{{ item.references[0] }}"
+      {% endfor %}
+      Question: {{ question }}
+    ||| + 'Answer: "',
     metrics: [
       { class_path: 'CharF1' },
       { class_path: 'ExactMatch' },

diff --git a/flexeval/preset_configs/EvalSetup/en_generation/twitter_sentiment.jsonnet b/flexeval/preset_configs/EvalSetup/en_generation/twitter_sentiment.jsonnet
@@ -27,19 +27,14 @@ local dataset_base_args = {
         num_shots: 4,
       },
     },
-    prompt_template: {
-      class_path: 'Jinja2PromptTemplate',
-      init_args: {
-        template: |||
-          Classify the sentiment of the following tweet.
-          {% for item in few_shot_data %}
-          Tweet: {{ item.text }}
-          Sentiment: `{{ item.references[0] }}`
-          {% endfor %}
-          Tweet: {{ text }}
-        ||| + 'Sentiment: `',
-      },
-    },
+    prompt_template: |||
+      Classify the sentiment of the following tweet.
+      {% for item in few_shot_data %}
+      Tweet: {{ item.text }}
+      Sentiment: `{{ item.references[0] }}`
+      {% endfor %}
+      Tweet: {{ text }}
+    ||| + 'Sentiment: `',
     metrics: [
       { class_path: 'ExactMatch' },
     ],

diff --git a/flexeval/preset_configs/EvalSetup/en_multiple_choice/arc_challenge.jsonnet b/flexeval/preset_configs/EvalSetup/en_multiple_choice/arc_challenge.jsonnet
@@ -18,7 +18,7 @@ local dataset_base_args = {
     '{% if choices.text | length > 3 %}{{ choices.text[3] }}{% endif %}',
     '{% if choices.text | length > 4 %}{{ choices.text[4] }}{% endif %}',
   ],
-  # answerKey is one of A, B, C, D, E, 1, 2, 3, 4
+  // answerKey is one of A, B, C, D, E, 1, 2, 3, 4
   answer_index_template: '{% if answerKey == "A" %}0{% elif answerKey == "B" %}1{% elif answerKey == "C" %}2{% elif answerKey == "D" %}3{% elif answerKey == "E" %}3{% else %}{{ answerKey | int - 1 }}{% endif %}',
   whitespace_before_choices: true,
 };
@@ -40,18 +40,12 @@ local dataset_base_args = {
         num_shots: 4,
       },
     },
-    prompt_template: {
-      class_path: 'Jinja2PromptTemplate',
-      init_args: {
-
-        template: |||
-          {% for item in few_shot_data %}
-          Question: {{ item.question }}
-          Answer:{{ item.choices[item.answer_index] }}
-          {% endfor %}
-          Question: {{ question }}
-        ||| + 'Answer:',
-      },
-    },
+    prompt_template: |||
+      {% for item in few_shot_data %}
+      Question: {{ item.question }}
+      Answer:{{ item.choices[item.answer_index] }}
+      {% endfor %}
+      Question: {{ question }}
+    ||| + 'Answer:',
   },
 }