Views
No views yet
run_clm.py script from the transformers library was used. Training was distributed on two NVIDIA Quadro RTX 6000 GPUs:1TORCH_CPP_LOG_LEVEL=INFO NCCL_DEBUG=INFO CUDA_VISIBLE_DEVICES=0,1 nohup python -m torch.distributed.launch \
2--nproc_per_node=2 run_clm.py --output_dir="./training_full" \
3--model_type="gpt2" \
4--config_name="./training" \
5--tokenizer_name="./training" \
6--dataset_name="RaiBP/openwebtext2-first-30-chunks-ablation-full" \
7--do_train \
8--per_device_train_batch_size 8 \
9--block_size="1024" \
10--learning_rate="5e-3" --warmup_steps="1000" \
11--adam_beta1="0.9" --adam_beta2="0.98" --weight_decay="0.01" \
12--overwrite_output_dir \
13--num_train_epochs="1" \
14--logging_steps="500" \
15--save_steps="5000" --preprocessing_num_workers="16" \
16--gradient_accumulation_steps="4" --report_to="tensorboard" \
17--logging_dir="./log_full" > command_full_log.log 2>&1 &| Target language | PPL |
|---|---|
| en | 37.513710021972656 |
| de | 24.629812240600586 |
| es | 21.987037658691406 |
| fr | 26.124969482421875 |
| it | 26.723554611206055 |
| pt | 21.162311553955078 |
| nl | 32.36076736450195 |
1import numpy as np
2from datasets import load_dataset
3from transformers import AutoTokenizer, AutoModelForCausalLM
4import torch
5from tqdm import tqdm
6import random
7
8# Set the seed for reproducibility
9random.seed(42)
10
11device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
12
13# Load the model
14model_name = "RaiBP/gpt2-openwebtext2-first-30-chunks-ablation-full"
15model = AutoModelForCausalLM.from_pretrained(model_name).to(device)
16tokenizer = AutoTokenizer.from_pretrained(model_name)
17
18target_language_dataset = "20231101.de" # change here for other languages
19
20dataset = load_dataset("wikimedia/wikipedia", target_language_dataset, split="train")
21num_examples = 2000
22random_numbers = list(np.random.randint(0, len(dataset), num_examples))
23examples = []
24for i in tqdm(random_numbers):
25 examples.append(dataset[int(i)]["text"])
26encodings = tokenizer("\n\n".join(examples), return_tensors="pt")
27
28max_length = model.config.n_positions
29stride = 512
30seq_len = encodings.input_ids.size(1)
31
32nlls = []
33prev_end_loc = 0
34for begin_loc in tqdm(range(0, seq_len, stride)):
35 end_loc = min(begin_loc + max_length, seq_len)
36 trg_len = end_loc - prev_end_loc # may be different from stride on last loop
37 input_ids = encodings.input_ids[:, begin_loc:end_loc].to(device)
38 target_ids = input_ids.clone()
39 target_ids[:, :-trg_len] = -100
40
41 with torch.no_grad():
42 outputs = model(input_ids, labels=target_ids)
43
44 # loss is calculated using CrossEntropyLoss which averages over valid labels
45 # N.B. the model only calculates loss over trg_len - 1 labels, because it internally shifts the labels
46 # to the left by 1.
47 neg_log_likelihood = outputs.loss
48
49 nlls.append(neg_log_likelihood)
50
51 prev_end_loc = end_loc
52 if end_loc == seq_len:
53 break
54
55ppl = torch.exp(torch.stack(nlls).mean())
56
57print("Perplexity: ", ppl.item())