fix gen_prefix

EleutherAI · Jan 22, 2025 · 6580aab · 6580aab
1 parent 2bc9aa6
commit 6580aab
Show file tree

Hide file tree

Showing 4 changed files with 6 additions and 6 deletions.
diff --git a/lm_eval/tasks/ruler/cwe_utils.py b/lm_eval/tasks/ruler/cwe_utils.py
@@ -145,7 +145,7 @@ def sys_word_pair_random(
             )
 
         gen_prefix_index = input_text.rfind(" Answer")
-        gen_prefix = input_text[gen_prefix_index:].strip()
+        # gen_prefix = input_text[gen_prefix_index:].strip()
         input_text = input_text[:gen_prefix_index]
         formatted_output = {
             "index": index,
@@ -154,7 +154,7 @@ def sys_word_pair_random(
             "outputs": answer,
             "length": length,
             "max_length": max_seq_length,
-            "gen_prefix": gen_prefix,
+            "gen_prefix": "Answer: The top 10 words that appear most often in the list are:",
         }
         write_jsons.append(formatted_output)
 

diff --git a/lm_eval/tasks/ruler/fwe_utils.py b/lm_eval/tasks/ruler/fwe_utils.py
@@ -28,7 +28,7 @@
     {
         "tokens_to_generate": 50,
         "template": """Read the following coded text and track the frequency of each coded word. Find the three most frequently appeared coded words. {context}\nQuestion: Do not provide any explanation. Please ignore the dots '....'. What are the three most frequently appeared words in the above coded text?""",
-        "answer_prefix": """ Answer: According to the coded text above, the three most frequently appeared words are:""",
+        "answer_prefix": """Answer: According to the coded text above, the three most frequently appeared words are:""",
     },
 )
 

diff --git a/lm_eval/tasks/ruler/qa_utils.py b/lm_eval/tasks/ruler/qa_utils.py
@@ -26,7 +26,7 @@
 config = {
     "tokens_to_generate": 32,
     "template": """Answer the question based on the given documents. Only give me the answer and do not output any other words.\n\nThe following are given documents.\n\n{context}\n\nAnswer the question based on the given documents. Only give me the answer and do not output any other words.\n\nQuestion: {query}""",
-    "answer_prefix": """ Answer:""",
+    "answer_prefix": """Answer:""",
 }
 SEED = 42
 TEMPLATE = """Answer the question based on the given documents. Only give me the answer and do not output any other words.\n\nThe following are given documents.\n\n{context}\n\nAnswer the question based on the given documents. Only give me the answer and do not output any other words.\n\nQuestion: {query}"""

diff --git a/lm_eval/tasks/ruler/vt_utils.py b/lm_eval/tasks/ruler/vt_utils.py
@@ -28,7 +28,7 @@
     "variable_tracking": {
         "tokens_to_generate": 30,
         "template": """Memorize and track the chain(s) of variable assignment hidden in the following text.\n\n{context}\nQuestion: Find all variables that are assigned the value {query} in the text above.""",
-        "answer_prefix": """ Answer: According to the chain(s) of variable assignment in the text above, {num_v} variables are assgined the value {query}, they are: """,
+        "answer_prefix": """Answer: According to the chain(s) of variable assignment in the text above, {num_v} variables are assigned the value {query}, they are:""",
     },
 }
 
@@ -215,7 +215,7 @@ def sys_vartrack_w_noise_random(
             "outputs": answer,
             "length": length,
             "max_length": max_seq_length,
-            "gen_prefix": gen_prefix,
+            "gen_prefix": gen_prefix.strip(),
         }
         write_jsons.append(formatted_output)