Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Gen Prefix #2274

Open
wants to merge 5 commits into
base: main
Choose a base branch
from
Open

Gen Prefix #2274

Changes from 4 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
b4f20b9
add gen_prefix to config; fix type hints
baberabb Aug 29, 2024
d5471cf
add `gen_prefix`
baberabb Aug 31, 2024
ec951cb
fixup: gen_prefix can be None
baberabb Aug 31, 2024
dac8b53
add todo
baberabb Sep 1, 2024
4eecbab
Merge branch 'main' into prefill
baberabb Sep 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 35 additions & 11 deletions lm_eval/api/task.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@ class TaskConfig(dict):
filter_list: Optional[Union[str, list]] = None
should_decontaminate: bool = False
doc_to_decontamination_query: Optional[str] = None
gen_prefix: str = None
metadata: Optional[dict] = (
None # by default, not used in the code. allows for users to pass arbitrary info to tasks
)
Expand Down Expand Up @@ -1003,8 +1004,9 @@ def append_target_question(
labeled_examples: List[Dict[str, str]],
question: str,
fewshot_as_multiturn: bool = False,
gen_prefix: str = None,
) -> None:
"""Adds a target question to the labeled examples list.
"""Adds a target question to the labeled examples list (in-place update)
If fewshot_as_multiturn is True, or labeled_examples is empty, or the last entry is a system turn, appends the question as a new user entry.
Otherwise, it is appended to the last user entry, ensuring that the conversation alternates between the user and the assistant.
"""
Expand All @@ -1019,16 +1021,21 @@ def append_target_question(
# if fewshot_as_multiturn is True, append as next user entry (last is always assistant)
labeled_examples.append({"role": "user", "content": question})

if gen_prefix:
labeled_examples.append(
{"role": "assistant", "content": gen_prefix.lstrip()}
)

@utils.positional_deprecated
def fewshot_context(
self,
doc: str,
doc: Dict[str, str],
num_fewshot: int,
system_instruction: Optional[str] = None,
apply_chat_template: bool = False,
fewshot_as_multiturn: bool = False,
chat_template: Optional[Callable] = None,
) -> str:
) -> Union[str, List[Dict[str, str]]]:
"""Returns a fewshot context string that is made up of a prepended description
(if provided), the `num_fewshot` number of examples, and an appended prompt example.

Expand All @@ -1053,10 +1060,13 @@ def fewshot_context(
else:
labeled_examples = ""

gen_prefix = "" if not self.config.gen_prefix else self.config.gen_prefix

# get task description
if description := self.config.description:
description = utils.apply_template(self.config.description, doc)

## [System prompt/Description] ##
# create system prompt based on the provided system instruction and description
if system_instruction is not None and description:
system_prompt = (
Expand All @@ -1076,6 +1086,7 @@ def fewshot_context(
else:
labeled_examples = system_prompt

## [Few-shot examples] ##
# if few-shot - append examples after the system prompt
if num_fewshot > 0:
if apply_chat_template:
Expand All @@ -1087,16 +1098,22 @@ def fewshot_context(
else:
labeled_examples += self.sampler.get_context(doc, num_fewshot)

## [Prompt] ##
example = self.doc_to_text(doc)
if apply_chat_template:
if self.multiple_input:
# TODO<baber>: How to handle gen_prefix for multiple inputs?
return chat_template(labeled_examples)
if isinstance(example, str):
self.append_target_question(
labeled_examples, example, fewshot_as_multiturn
labeled_examples,
example,
fewshot_as_multiturn,
gen_prefix,
)
# for loglikelihood create a list of questions with appended choices
elif isinstance(example, list):
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Haven't been able to identify when non-chat_template example is a list

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

it would be better to split fewshot_context func into two: one to make a list of samples, another to make it a valid string
right now i am working on smart truncation that allows users to choose what to truncate and I encountered the same problem

# TODO<baber>: when is example a list?
labeled_examples_list = []
# copy chat history for each example and append the answer
for ex in example:
Expand All @@ -1109,27 +1126,34 @@ def fewshot_context(
if self.config.doc_to_choice is not None:
choices = self.doc_to_choice(doc)
self.append_target_question(
labeled_examples, choices[example], fewshot_as_multiturn
labeled_examples,
choices[example],
fewshot_as_multiturn,
gen_prefix,
)
else:
self.append_target_question(
labeled_examples, str(example), fewshot_as_multiturn
labeled_examples,
str(example),
fewshot_as_multiturn,
gen_prefix,
)
# return lm.apply_chat_template(labeled_examples)
return chat_template(labeled_examples)
else:
if self.multiple_input:
# TODO<baber>: How to handle gen_prefix for multiple inputs?
return labeled_examples
if isinstance(example, str):
return labeled_examples + example
return labeled_examples + example + gen_prefix
elif isinstance(example, list):
return [labeled_examples + ex for ex in example]
return [labeled_examples + ex + gen_prefix for ex in example]
elif isinstance(example, int):
if self.config.doc_to_choice is not None:
choices = self.doc_to_choice(doc)
return labeled_examples + choices[example]
return labeled_examples + choices[example] + gen_prefix
else:
return labeled_examples + str(example)
return labeled_examples + str(example) + gen_prefix

def apply_filters(self):
"""Iterates over FilterEnsembles and applies them to instances"""
Expand Down Expand Up @@ -1207,7 +1231,7 @@ def doc_to_text(self, doc, doc_to_text=None):
print(type(doc_to_text))
raise TypeError

def doc_to_target(self, doc: Mapping, doc_to_target=None) -> Union[int, str, list]:
def doc_to_target(self, doc: Dict, doc_to_target=None) -> Union[int, str, list]:
if self.prompt is not None:
doc_to_target = self.prompt
elif doc_to_target is not None:
Expand Down
Loading