diff --git a/src/lighteval/tasks/tasks/simpleqa.py b/src/lighteval/tasks/tasks/simpleqa.py index 4493fc322..37a1deaa5 100644 --- a/src/lighteval/tasks/tasks/simpleqa.py +++ b/src/lighteval/tasks/tasks/simpleqa.py @@ -32,16 +32,11 @@ def simpleqa_prompt(line, task_name: str = None): - query = f"Question: {line['question']}\n" - query += "".join( - [f"\n{key}. {choice}" for key, choice in zip(["A", "B", "C", "D", "E", "F"], line["choices"]["text"])] - ) - query += "\nAnswer:" return Doc( task_name=task_name, - query=query, - choices=line["choices"]["text"], - gold_index=line["choices"]["label"].index(line["answerKey"]), + query=f"Question: {line['problem']}\nAnswer:", + choices=[line["answer"]], + gold_index=0, )