Dataset preprocessed from
https://huggingface.co/datasets/Yukang/LongAlpaca-12k.
This contains 1000 samples that have a minimum length of 16k tokens and a maximum of 32k tokens.
from datasets import load_dataset
from transformers import AutoTokenizer
import pandas as pd
import pyarrow as pa
import pyarrow.parquet as pq
data = load_dataset("Yukang/LongAlpaca-12k")
tokenizer =… See the full description on the dataset page:
https://huggingface.co/datasets/casperhansen/longalpaca_1k_test.
