Views
No views yet
| Label | Meaning |
|---|---|
O | no punctuation after this token |
COMMA | insert a comma after this token |
PERIOD | insert a period after this token |
QUESTION | insert a question mark after this token |
1from transformers import AutoTokenizer, AutoModelForTokenClassification
2import torch
3
4tokenizer = AutoTokenizer.from_pretrained("Zarinaaa/xlmr-kyrgyz-punctuation")
5model = AutoModelForTokenClassification.from_pretrained("Zarinaaa/xlmr-kyrgyz-punctuation").eval()
6
7ID2LABEL = {0: "O", 1: "COMMA", 2: "PERIOD", 3: "QUESTION"}
8
9def restore_punctuation(text: str) -> str:
10 words = text.split()
11 enc = tokenizer(words, is_split_into_words=True,
12 return_tensors="pt", truncation=True, max_length=256)
13 with torch.no_grad():
14 logits = model(**enc).logits.squeeze(0)
15 preds = logits.argmax(dim=-1).tolist()
16 word_ids = enc.word_ids(batch_index=0)
17
18 # Take the label of the LAST subtoken of each word
19 label_per_word = [None] * len(words)
20 for i in range(len(word_ids) - 1, -1, -1):
21 wid = word_ids[i]
22 if wid is None:
23 continue
24 if label_per_word[wid] is None:
25 label_per_word[wid] = ID2LABEL[preds[i]]
26
27 out = []
28 for w, lab in zip(words, label_per_word):
29 if lab == "COMMA":
30 out.append(w + ",")
31 elif lab == "PERIOD":
32 out.append(w + ".")
33 elif lab == "QUESTION":
34 out.append(w + "?")
35 else:
36 out.append(w)
37 s = " ".join(out)
38 return s[0].upper() + s[1:] if s else s
39
40print(restore_punctuation("мен мектепке барам"))
41# → "Мен мектепке барам."xlm-roberta-base1@article{uvalieva2026kyrgyz,
2 author = {Uvalieva, Zarina and Muhametjanova, Gulshat},
3 title = {Punctuation Restoration for Kyrgyz Language: A Comparative Study of Multilingual Transformer Models},
4 journal = {ACM Transactions on Asian and Low-Resource Language Information Processing},
5 year = {2026},
6 note = {Under revision}
7}