Views
No views yet
distilbert-base-multilingual-cased transformer model for language identification using the papluca/language-identification dataset from Hugging Face.pip install transformers datasets scikit-learn torch1from transformers import AutoModelForSequenceClassification, AutoTokenizer
2import torch
3
4# Load tokenizer and model
5model_path = "distilbert-base-multilingual-cased"
6tokenizer = AutoTokenizer.from_pretrained(model_path)
7model = AutoModelForSequenceClassification.from_pretrained(model_path)
8# Define test sentences
9sample_texts = [
10 "This is an English sentence.",
11 "C'est une phrase en français.",
12 "यह एक हिंदी वाक्य है।"
13]
14
15
16# Tokenize and predict
17def predict_language(texts, model, tokenizer, label_encoder):
18 if isinstance(texts, str):
19 texts = [texts]
20
21 device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
22 model.to(device)
23 model.eval()
24
25 inputs = tokenizer(texts, return_tensors="pt", padding=True, truncation=True)
26 inputs = {k: v.to(device) for k, v in inputs.items()}
27
28 with torch.no_grad():
29 outputs = model(**inputs)
30 preds = torch.argmax(outputs.logits, dim=1).cpu().numpy() # move to CPU first
31
32 predicted_languages = label_encoder.inverse_transform(preds)
33 return predicted_languages
34epochhalf() precision (FP16) to reduce model size and inference time.1.
2├── quantized-model/ # Contains the quantized model files
3│ ├── config.json
4│ ├── model.safetensors
5│ ├── tokenizer_config.json
6│ ├── vocab.txt
7│ └── special_tokens_map.json
8├── README.md # Model documentation