Views
No views yet
1# Importing necessary modules from the transformers library
2from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
3
4# Initializing the tokenizer for the specific model. This tokenizer is used to convert
5# text input into a format that is understandable by the model.
6tokenizer = AutoTokenizer.from_pretrained("CCRss/tokenizer_t5_kz")
7
8# Define a function for preprocessing the data. This function takes an example
9# (which includes source and target texts) and tokenizes both texts using the tokenizer.
10# The tokenized output is then formatted to a fixed length for consistent model input.
11def preprocess_data(example):
12 # Extracting the source and target texts from the example
13 source = example["src"]
14 target = example["trg"]
15
16 # Tokenizing the source text with padding and truncation to ensure a fixed length
17 source_inputs = tokenizer(source, padding="max_length", truncation=True, max_length=128)
18
19 # Tokenizing the target text with padding and truncation to ensure a fixed length
20 target_inputs = tokenizer(target, padding="max_length", truncation=True, max_length=128)
21
22 # Returning the tokenized inputs, combining both source and target, and setting the target as labels
23 return {**source_inputs, **target_inputs, "labels": target_inputs["input_ids"]}
24
25# Applying the preprocessing function to the dataset, effectively transforming all text data
26# into a tokenized format suitable for the Seq2Seq model.
27encoded_dataset = dataset.map(preprocess_data)
28# Setting the format of the dataset to PyTorch tensors for compatibility with the training framework.
29encoded_dataset.set_format("torch")
301
2# Importing necessary classes for training from the transformers library
3from transformers import TrainingArguments, Seq2SeqTrainer
4
5# Name of the pretrained model to be used for Seq2Seq learning
6name_of_model = "humarin/chatgpt_paraphraser_on_T5_base"
7# Loading the model from the pretrained weights
8model = AutoModelForSeq2SeqLM.from_pretrained(name_of_model)
9
10# Setting up training arguments. This includes batch size, learning rate, number of epochs,
11# directories for saving results and logs, and evaluation strategy.
12training_args = Seq2SeqTrainingArguments(
13 per_device_train_batch_size=21,
14 gradient_accumulation_steps=3,
15 learning_rate=5e-5,
16 save_steps=2000,
17 num_train_epochs=3,
18 output_dir='./results',
19 logging_dir='./logs',
20 logging_steps=2000,
21 eval_steps=2000,
22 evaluation_strategy="steps"
23)
24
25# Initializing the trainer with the model, training arguments, and the datasets for training and evaluation.
26trainer = Seq2SeqTrainer(
27 model=model,
28 args=training_args,
29 train_dataset=encoded_dataset['train'],
30 eval_dataset=encoded_dataset['valid']
31)
32
33# Starting the training process of the model using the specified datasets and training arguments.
34trainer.train()