import torch
from transformers import AutoTokenizer, AutoModel, AutoModelForSequenceClassification, TrainingArguments, Trainer
from torch.utils.data import Dataset, DataLoader
import numpy as np
--- Example: Using bert-base-multilingual-cased for Embeddings ---
model_name_embeddings = "bert-base-multilingual-cased"
tokenizer_embeddings = AutoTokenizer.from_pretrained(model_name_embeddings)
model_embeddings = AutoModel.from_pretrained(model_name_embeddings)
text_embeddings = "Hello, how are you? ¡Hola, cómo estás? Bonjour, comment ça va?"
tokens_embeddings = tokenizer_embeddings(text_embeddings, return_tensors="pt", padding=True, truncation=True)
with torch.no_grad():
outputs_embeddings = model_embeddings(**tokens_embeddings)
embeddings = outputs_embeddings.last_hidden_state
print("Embeddings Shape:", embeddings.shape)
--- Example: Using bert-base-multilingual-cased for Sequence Classification ---
model_name_classification = "bert-base-multilingual-cased"
tokenizer_classification = AutoTokenizer.from_pretrained(model_name_classification)
model_classification = AutoModelForSequenceClassification.from_pretrained(model_name_classification, num_labels=3) # Example: 3 labels
text_classification = "This is a positive sentence."
tokens_classification = tokenizer_classification(text_classification, return_tensors="pt", padding=True, truncation=True)
with torch.no_grad():
outputs_classification = model_classification(**tokens_classification)
logits = outputs_classification.logits
predicted_class = torch.argmax(logits, dim=-1)
print("Predicted Class:", predicted_class)
--- Example: Fine-tuning (Simplified) ---
Sample Dataset (replace with your actual dataset)
class SimpleDataset(Dataset):
def init (self, texts, labels, tokenizer, max_len):
self.texts = texts
self.labels = labels
self.tokenizer = tokenizer
self.max_len = max_len
def __len__(self):
return len(self.texts)
def __getitem__(self, idx):
text = str(self.texts[idx])
label = self.labels[idx]
encoding = self.tokenizer(
text,
add_special_tokens=True,
max_length=self.max_len,
return_token_type_ids=False,
padding='max_length',
return_attention_mask=True,
return_tensors='pt',
truncation=True
)
return {
'input_ids': encoding['input_ids'].flatten(),
'attention_mask': encoding['attention_mask'].flatten(),
'labels': torch.tensor(label, dtype=torch.long)
}Sample data
train_texts = ["This is good", "This is bad", "Neutral sentence"]
train_labels = [1, 0, 2] # 1: positive, 0: negative, 2: neutral
Parameters
model_name_finetune = "bert-base-multilingual-cased"
tokenizer_finetune = AutoTokenizer.from_pretrained(model_name_finetune)
model_finetune = AutoModelForSequenceClassification.from_pretrained(model_name_finetune, num_labels=3)
max_len = 128
Create dataset
train_dataset = SimpleDataset(train_texts, train_labels, tokenizer_finetune, max_len)
Training arguments
training_args = TrainingArguments(
output_dir="./results",
num_train_epochs=3,
per_device_train_batch_size=16,
per_device_eval_batch_size=64,
warmup_steps=500,
weight_decay=0.01,
logging_dir="./logs",
logging_steps=10,
)
Trainer
trainer = Trainer(
model=model_finetune,
args=training_args,
train_dataset=train_dataset,
)
Train the model
trainer.train()
#Example of inference after fine tuning.
test_text = "This is very good"
test_tokens = tokenizer_finetune(test_text, return_tensors="pt", padding=True, truncation=True)
with torch.no_grad():
test_outputs = model_finetune(**test_tokens)
test_logits = test_outputs.logits
test_predicted_class = torch.argmax(test_logits, dim=-1)
print(f"Test prediction after fine tuning: {test_predicted_class}")
