Views
No views yet
jhu-clsp/mmBERT-small for 14 classes:ko, no, da, sv, fi, nl, en, fr, de, es, pt, it, jaUNKNOWNpipeline("text-classification") for quick usage.AutoTokenizer + AutoModelForSequenceClassification for explicit forward-pass control.1import re
2
3URL_PATTERN = re.compile(r"https?://\S+|www\.\S+", flags=re.IGNORECASE)
4
5def fast_detect_unknown(text: str) -> bool:
6 s = text.strip()
7 if not s:
8 # Empty input is treated as non-language.
9 return True
10 if URL_PATTERN.search(s):
11 # URLs should map to UNKNOWN.
12 return True
13
14 total = len(s)
15 alpha = sum(ch.isalpha() for ch in s)
16 digits = sum(ch.isdigit() for ch in s)
17 spaces = sum(ch.isspace() for ch in s)
18 symbols = total - alpha - digits - spaces
19 non_space = max(1, total - spaces)
20
21 # Mostly numeric strings (ids, phone numbers, etc.).
22 if digits / non_space >= 0.8:
23 return True
24 # Symbol-heavy text is usually not valid language content.
25 if symbols / non_space >= 0.45:
26 return True
27 # Very low alphabetic ratio indicates gibberish-like input.
28 if total >= 6 and (alpha / non_space) < 0.2:
29 return True
30 # Long compact mixed tokens often represent hashes/usernames/keys.
31 if " " not in s and total >= 12 and (alpha / non_space) < 0.45 and (digits > 0 or symbols > 0):
32 return True
33 return False1import torch
2from transformers import pipeline
3
4model_id = "chiennv/langid-mmbert-small"
5device = 0 if torch.cuda.is_available() else -1
6clf = pipeline(
7 "text-classification",
8 model=model_id,
9 tokenizer=model_id,
10 top_k=1,
11 device=device, # GPU id (0,1,...) or -1 for CPU
12)
13
14text = "Bonjour tout le monde"
15if fast_detect_unknown(text):
16 print({"label": "UNKNOWN", "score": 1.0})
17else:
18 out = clf(text)[0][0]
19 print({"label": out["label"], "score": round(out["score"], 4)})1import torch
2from transformers import AutoModelForSequenceClassification, AutoTokenizer
3
4model_id = "chiennv/langid-mmbert-small"
5tokenizer = AutoTokenizer.from_pretrained(model_id)
6device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
7
8# Use FP16 on GPU for faster inference and lower memory.
9dtype = torch.float16 if device.type == "cuda" else torch.float32
10model = AutoModelForSequenceClassification.from_pretrained(model_id, torch_dtype=dtype).to(device)
11model.eval()
12
13text = "Bonjour tout le monde"
14if fast_detect_unknown(text):
15 print({"label": "UNKNOWN", "score": 1.0})
16else:
17 inputs = tokenizer(text, return_tensors="pt", truncation=True, max_length=128)
18 inputs = {k: v.to(device) for k, v in inputs.items()}
19 with torch.no_grad():
20 logits = model(**inputs).logits
21 probs = torch.softmax(logits, dim=-1).squeeze(0)
22 pred_id = int(torch.argmax(probs).item())
23 pred_label = model.config.id2label[pred_id]
24 pred_score = float(probs[pred_id].item())
25 print({"label": pred_label, "score": round(pred_score, 4)})infer.pypython infer.pypython -c "import torch; print(torch.cuda.is_available(), torch.cuda.get_device_name(0) if torch.cuda.is_available() else 'no-gpu')"