tokenized allenai/tulu-3-sft-mixture using llama 3.1 8B-instruct tokenizer and chat template
from datasets import load_dataset
from transformers import AutoTokenizer
dataset = load_dataset("allenai/tulu-3-sft-mixture", split="train")
tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-3.1-8B-Instruct")
tokenizer.pad_token = tokenizer.eos_token
def tokenize_function(examples):
# Apply chat template to format the messages… See the full description on the dataset page:
https://huggingface.co/datasets/michaelbzhu/tulu-3-sft-tokenized-llama3.1.
