Views
No views yet
1from transformers import T5TokenizerFast, AutoModelForSeq2SeqLM
2
3device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
4
5model_name = 'wyluilipe/t5-spellchecker'
6tokenizer = T5TokenizerFast.from_pretrained(model_name)
7model = AutoModelForSeq2SeqLM.from_pretrained(model_name)
8model.to(device)
9model.eval()
10
11prompt = 'пунктут выздачи зазкзыов'
12encoded = tokenizer.encode(prompt, return_tensors='pt')
13
14output_sequences = model.generate(
15 input_ids=encoded.to(device),
16 max_length=128,
17 temperature=0.1,
18 top_k=0,
19 top_p=0.9,
20 repetition_penalty=1,
21 do_sample=True,
22 num_return_sequences=1,
23 pad_token_id=0,
24)
25decoded = tokenizer.batch_decode(output_sequences)1dataset_path = 'SpellChecker/trash_large.csv' # Change this for your personal dataset
2model_name = 'wyluilipe/t5-spellchecker'
3use_hf = False
4
5device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
6
7training_config = {
8 "model": {
9 "pretrained_name": model_name,
10 "max_length" : 128
11 },
12 "datasets": {
13 "use_hf": use_hf,
14 "path": dataset_path
15 },
16 "verbose": True
17}1# dataset = load_dataset(dataset_path)
2dataset = Dataset.from_pandas(df)
3tokenizer = T5TokenizerFast.from_pretrained(model_name)
4model = AutoModelForSeq2SeqLM.from_pretrained(model_name)
5model.to(device)
6model.eval()1def tokenize_text(text):
2 tokenizer.pad_token = tokenizer.eos_token
3 tokenized_inputs = tokenizer(
4 text,
5 return_tensors="np",
6 padding='max_length',
7 )
8
9 max_length = min(
10 tokenized_inputs["input_ids"].shape[1],
11 128
12 )
13 tokenizer.truncation_side = "right"
14 tokenized_inputs = tokenizer(
15 text,
16 return_tensors="np",
17 padding='max_length',
18 truncation=True,
19 max_length=max_length
20 )
21
22 return tokenized_inputs
23
24def tokenize_function(examples):
25 text1 = examples['source']
26 text2 = examples['correction']
27
28 tokenized_inputs = tokenize_text(text1)
29 labels = tokenize_text(text2)['input_ids']
30 tokenized_inputs['labels'] = labels
31 return tokenized_inputs1dataset = dataset.train_test_split(test_size=0.1)
2train_dataset = Dataset.from_pandas(dataset['train'].to_pandas()[['source', 'correction']])
3test_dataset = Dataset.from_pandas(dataset['test'].to_pandas()[['source', 'correction']])1def convert_dataset(dataset):
2 input_column = 'source'
3 output_column = 'correction'
4 dataset_phrase = Dataset.from_pandas(
5 pd.DataFrame(dataset.to_pandas()[input_column])
6 )
7
8 dataset_paraphrase = Dataset.from_pandas(
9 pd.DataFrame(dataset.to_pandas()[output_column])
10 )
11
12 tokenized_dataset_phrase = dataset_phrase.map(
13 tokenize_function,
14 batched=True,
15 batch_size=1,
16 drop_last_batch=True
17 )
18
19 tokenized_dataset_paraphrase = dataset_paraphrase.map(
20 tokenize_function,
21 batched=True,
22 batch_size=1,
23 drop_last_batch=True
24 )
25
26 tokenized_dataset_paraphrase.to_pandas()[[output_column, 'input_ids']]
27
28 tokenized_dataset_paraphrase = tokenized_dataset_paraphrase.to_pandas()
29 dataset = tokenized_dataset_phrase.to_pandas()
30 dataset[input_column] = '<phrase>:' + dataset[input_column] + '\n<answer>:'
31 dataset[output_column] = tokenized_dataset_paraphrase[output_column]
32 dataset['labels'] = tokenized_dataset_paraphrase['input_ids']
33 dataset = dataset[[input_column, output_column, 'input_ids', 'token_type_ids', 'attention_mask', 'labels']]
34
35 return Dataset.from_pandas(dataset)1train_dataset = train_dataset.map(
2 tokenize_function,
3 batched=True,
4 batch_size=1,
5 drop_last_batch=True
6 )
7
8test_dataset = test_dataset.map(
9 tokenize_function,
10 batched=True,
11 batch_size=1,
12 drop_last_batch=True
13 )1max_steps = 100
2trained_model_name = "wyluilipe/t5-spellchecker"
3output_dir = trained_model_name
4
5training_args = TrainingArguments(
6 learning_rate=2.0e-5,
7 num_train_epochs=3,
8
9 # Max steps to train for (each step is a batch of data)
10 # Overrides num_train_epochs, if not -1
11 max_steps=max_steps,
12
13 # Batch size for training
14 per_device_train_batch_size=1,
15
16 # Directory to save model checkpoints
17 output_dir=output_dir,
18
19 # Other arguments
20 overwrite_output_dir=False, # Overwrite the content of the output directory
21 disable_tqdm=False, # Disable progress bars
22 eval_steps=120, # Number of update steps between two evaluations
23 save_steps=120, # After # steps model is saved
24 warmup_steps=1, # Number of warmup steps for learning rate scheduler
25 per_device_eval_batch_size=1, # Batch size for evaluation
26 evaluation_strategy="steps",
27 logging_strategy="steps",
28 logging_steps=1,
29 optim="adafactor",
30 gradient_accumulation_steps = 4,
31 gradient_checkpointing=False,
32
33 # Parameters for early stopping
34 load_best_model_at_end=True,
35 save_total_limit=1,
36 metric_for_best_model="eval_loss",
37 greater_is_better=False
38)1model_flops = (
2 model.floating_point_ops(
3 {
4 "input_ids": torch.zeros(
5 (1, training_config["model"]["max_length"])
6 )
7 }
8 )
9 * training_args.gradient_accumulation_steps
10)
11
12print(model)
13print("Memory footprint", model.get_memory_footprint() / 1e9, "GB")
14print("Flops", model_flops / 1e9, "GFLOPs")1trainer = Trainer(
2 model=model,
3 args=training_args,
4 train_dataset=train_dataset,
5 eval_dataset=test_dataset,
6)1training_output = trainer.train()
2trainer.push_to_hub('wyluilipe/t5-spellchecker')
3tokenizer.push_to_hub('wyluilipe/t5-spellchecker')