From 54e9019280afed593d303c5ef32dcbfd59138c13 Mon Sep 17 00:00:00 2001 From: bzantium Date: Tue, 21 Jan 2025 13:56:06 +0900 Subject: [PATCH] revise mbpp prompt --- lm_eval/tasks/mbpp/mbpp.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lm_eval/tasks/mbpp/mbpp.yaml b/lm_eval/tasks/mbpp/mbpp.yaml index 101f1988d7..a5b58d9001 100644 --- a/lm_eval/tasks/mbpp/mbpp.yaml +++ b/lm_eval/tasks/mbpp/mbpp.yaml @@ -4,9 +4,9 @@ dataset_name: full unsafe_code: true output_type: generate_until test_split: test -doc_to_text: "You are an expert Python programmer, and here is your task: {{text}} Your code should pass these tests:\n\n{{test_list[0]}}\n{{test_list[1]}}\n{{test_list[2]}}\n[BEGIN]" +doc_to_text: "You are an expert Python programmer, and here is your task: {{text}} Your code should pass these tests:\n\n{{test_list[0]}}\n{{test_list[1]}}\n{{test_list[2]}}\n[BEGIN]\n" doc_to_target: "{% if is_fewshot is defined %}{{code}}\n[DONE]{% else %}{{test_list[0]}}\n{{test_list[1]}}\n{{test_list[2]}}{% endif %}" -target_delimiter: "\n" +target_delimiter: "" metric_list: - metric: !function utils.pass_at_1 aggregation: mean