Views
No views yet
1from datasets import load_dataset
2def load_and_clean_wiki():
3 dataset = load_dataset('wiki40b', 'no', beam_runner='DirectRunner', split="train")
4 #dataset = load_dataset('wiki40b', 'sv', beam_runner='DirectRunner')
5 dataset = dataset.remove_columns(['wikidata_id', 'version_id'])
6 filtered_dataset = dataset.map(filter_wikipedia)
7 # filtered_dataset[:3]
8 # print(filtered_dataset[:3])
9 return filtered_dataset
10
11def filter_wikipedia(batch):
12 batch["text"] = " ".join(batch["text"].split("\
13_START_SECTION_\
14"))
15 batch["text"] = " ".join(batch["text"].split("\
16_START_ARTICLE_\
17"))
18 batch["text"] = " ".join(batch["text"].split("\
19_START_ARTICLE_\
20"))
21 batch["text"] = " ".join(batch["text"].split("\
22_START_PARAGRAPH_\
23"))
24 batch["text"] = " ".join(batch["text"].split("_NEWLINE_"))
25 batch["text"] = " ".join(batch["text"].split("\xa0"))
26 return batch./run_clm_flax.py --output_dir="${MODEL_DIR}" --model_type="gpt2" --config_name="${MODEL_DIR}" --tokenizer_name="${MODEL_DIR}" --dataset_name="wiki40b" --dataset_config_name="no" --do_train --do_eval --block_size="512" --per_device_train_batch_size="64" --per_device_eval_batch_size="64" --learning_rate="5e-3" --warmup_steps="1000" --adam_beta1="0.9" --adam_beta2="0.98" --weight_decay="0.01" --overwrite_output_dir --num_train_epochs="20" --logging_steps="500" --save_steps="1000" --eval_steps="2500" --push_to_hub