Skip to content

Commit

Permalink
Update data.py
Browse files Browse the repository at this point in the history
  • Loading branch information
anas-awadalla authored Mar 17, 2024
1 parent 79ad152 commit 3945c87
Showing 1 changed file with 1 addition and 3 deletions.
4 changes: 1 addition & 3 deletions open_flamingo/train/data.py
Original file line number Diff line number Diff line change
Expand Up @@ -84,8 +84,6 @@ def preprocess_laion_text(sample, tokenizer, max_tokens=128):
(f"<image>{re.sub(CAPTION_BAN_PATTERN, '', s.split('<|synthetic caption|>')[-1].strip())}{tokenizer.eos_token}") for s in sample
]

print(sample[0])

text = tokenizer(
sample,
max_length=max_tokens,
Expand Down Expand Up @@ -528,4 +526,4 @@ def get_data(args, image_processor, tokenizer, dataset_type, epoch=0):
args, image_processor=image_processor, epoch=epoch, tokenizer=tokenizer
)
else:
raise ValueError(f"Unsupported dataset: {dataset_type}")
raise ValueError(f"Unsupported dataset: {dataset_type}")

0 comments on commit 3945c87

Please sign in to comment.