Views
No views yet
quora-question-pairspip install transformers torch1from transformers import AlbertTokenizer, AlbertForSequenceClassification
2import torch
3
4device = "cuda" if torch.cuda.is_available() else "cpu"
5
6model_name = "AventIQ-AI/albert-duplicate-sentence-detection"
7model = AlbertForSequenceClassification.from_pretrained(model_name).to(device)
8tokenizer = AlbertTokenizer.from_pretrained(model_name)1def predict_duplicate(question1, question2, model):
2 inputs = tokenizer(question1, question2, truncation=True, padding="max_length", max_length=128, return_tensors="pt")
3
4 # ✅ Move inputs to the same device as the model
5 inputs = {key: value.to(device) for key, value in inputs.items()}
6
7 with torch.no_grad(): # Disable gradient calculation
8 outputs = model(**inputs)
9 logits = outputs.logits
10
11 # ✅ Get prediction
12 probs = torch.softmax(logits, dim=1)
13 prediction = torch.argmax(probs, dim=1).item()
14
15 # ✅ Output the results
16 label_map = {0: "Not Duplicate", 1: "Duplicate"}
17 print(f"Q1: {question1}")
18 print(f"Q2: {question2}")
19 print(f"Prediction: {label_map[prediction]} (Confidence: {probs.max().item():.4f})\n")
20
21# 🔍 Test Example
22test_samples = [
23 ("How can I learn Python quickly?", "What is the fastest way to learn Python?"), # Duplicate
24 ("What is the capital of India?", "Where is New Delhi located?"), # Duplicate
25 ("How to lose weight fast?", "What is the best programming language to learn?"), # Not Duplicate
26 ("Who is the CEO of Tesla?", "What is the net worth of Elon Musk?"), # Not Duplicate
27 ("What is machine learning?", "How does AI work?"), # Duplicate
28]
29for q1, q2 in test_samples:
30 predict_duplicate(q1, q2, model).
├── model/ # Contains the quantized model files
├── tokenizer_config/ # Tokenizer configuration and vocabulary files
├── model.safetensors/ # Quantized Model
├── README.md # Model documentation