From de9a6204a6b0e6d9cbbaae83666bd58788095334 Mon Sep 17 00:00:00 2001 From: chenzihong <58508660+ChenZiHong-Gavin@users.noreply.github.com> Date: Thu, 22 Jan 2026 14:17:28 +0800 Subject: [PATCH 1/2] Fix specific error --- src/lighteval/tasks/tasks/truthfulqa.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/lighteval/tasks/tasks/truthfulqa.py b/src/lighteval/tasks/tasks/truthfulqa.py index f517f049e..ece416e45 100644 --- a/src/lighteval/tasks/tasks/truthfulqa.py +++ b/src/lighteval/tasks/tasks/truthfulqa.py @@ -50,8 +50,7 @@ def truthful_qa_generative_prompt(line, task_name: str = None): task_name=task_name, query=line["question"].strip(), choices=correct_answers + incorrect_answers, - gold_index=list(range(len(correct_answers))), - specific={"len_mc1": len(line["mc1_targets"]["choices"])}, + gold_index=list(range(len(correct_answers))) ) From 8d86ccdc9a5935416744467449b5d19120606b20 Mon Sep 17 00:00:00 2001 From: chenzihong-gavin Date: Fri, 23 Jan 2026 16:04:32 +0800 Subject: [PATCH 2/2] fix: fix ruff format --- src/lighteval/tasks/tasks/truthfulqa.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lighteval/tasks/tasks/truthfulqa.py b/src/lighteval/tasks/tasks/truthfulqa.py index ece416e45..d88841be3 100644 --- a/src/lighteval/tasks/tasks/truthfulqa.py +++ b/src/lighteval/tasks/tasks/truthfulqa.py @@ -50,7 +50,7 @@ def truthful_qa_generative_prompt(line, task_name: str = None): task_name=task_name, query=line["question"].strip(), choices=correct_answers + incorrect_answers, - gold_index=list(range(len(correct_answers))) + gold_index=list(range(len(correct_answers))), )