Views
No views yet
XLMRobertaForTokenClassification checkpoint with 73 output labels, along with the decoding files constraint_mask.json and tag_mappings.json. The repository is currently published as a Transformers/XLM-RoBERTa safetensors model under Setur/BRAGD.Setur/BRAGD-sosialurinXLMRobertaForTokenClassificationvesteinn/ScandiBERTconstraint_mask.json,tag_mappings.json.pip install numpy torch "transformers==4.57.1" sentencepiece huggingface_hub1import json
2import numpy as np
3import torch
4from huggingface_hub import hf_hub_download
5from transformers import XLMRobertaTokenizerFast, XLMRobertaForTokenClassification
6model_name = "Setur/BRAGD"
7tokenizer = XLMRobertaTokenizerFast.from_pretrained(model_name)
8model = XLMRobertaForTokenClassification.from_pretrained(model_name)
9model.eval()
10# Download decoding assets
11constraint_mask_path = hf_hub_download(model_name, "constraint_mask.json")
12tag_mappings_path = hf_hub_download(model_name, "tag_mappings.json")
13with open(constraint_mask_path, "r", encoding="utf-8") as f:
14 raw_mask = json.load(f)
15constraint_mask = {int(k): [tuple(x) for x in v] for k, v in raw_mask.items()}
16with open(tag_mappings_path, "r", encoding="utf-8") as f:
17 raw_map = json.load(f)
18features_to_tag = {tuple(map(int, k.split(","))): v for k, v in raw_map.items()}
19WORD_CLASS_NAMES = {
20 0: "Noun",
21 1: "Adjective",
22 2: "Pronoun",
23 3: "Number",
24 4: "Verb",
25 5: "Participle",
26 6: "Adverb",
27 7: "Conjunction",
28 8: "Foreign",
29 9: "Unanalyzed",
30 10: "Abbreviation",
31 11: "Web",
32 12: "Punctuation",
33 13: "Symbol",
34 14: "Article",
35}
36INTERVAL_NAMES = {
37 (15, 29): "subcategory",
38 (30, 33): "gender",
39 (34, 36): "number",
40 (37, 41): "case",
41 (42, 43): "article",
42 (44, 45): "proper_noun",
43 (46, 50): "degree",
44 (51, 53): "declension",
45 (54, 60): "mood",
46 (61, 63): "voice",
47 (64, 66): "tense",
48 (67, 70): "person",
49 (71, 72): "definiteness",
50}
51FEATURE_COLUMNS = [
52 "S", "A", "P", "N", "V", "L", "D", "C", "F", "X", "T", "W", "K", "M", "R",
53 "D", "B", "E", "I", "P", "Q", "N", "G", "R", "X", "S", "C", "O", "T", "s",
54 "M", "F", "N", "g",
55 "S", "P", "n",
56 "N", "A", "D", "G", "c",
57 "A", "a",
58 "P", "r",
59 "P", "C", "S", "A", "d",
60 "S", "W", "e",
61 "I", "M", "N", "S", "P", "E", "U",
62 "A", "M", "v",
63 "P", "A", "t",
64 "1", "2", "3", "p",
65 "D", "I",
66]
67def decode_token(logits):
68 pred = np.zeros(logits.shape[0], dtype=int)
69 # predict word class
70 wc = int(np.argmax(logits[:15]))
71 pred[wc] = 1
72 # predict only valid feature groups for this word class
73 for start, end in constraint_mask.get(wc, []):
74 group = logits[start:end+1]
75 pred[start + int(np.argmax(group))] = 1
76 tag = features_to_tag.get(tuple(pred.tolist()), None)
77 features = {"word_class": WORD_CLASS_NAMES.get(wc, str(wc))}
78 for (start, end), name in INTERVAL_NAMES.items():
79 group = pred[start:end+1]
80 active = np.where(group == 1)[0]
81 if len(active) == 1:
82 features[name] = FEATURE_COLUMNS[start + active[0]]
83 return tag, features
84text = "Hetta er eitt føroyskt dømi"
85words = text.split()
86enc = tokenizer(
87 [words],
88 is_split_into_words=True,
89 return_tensors="pt",
90 padding=True,
91 truncation=True,
92)
93with torch.no_grad():
94 logits = model(**enc).logits[0]
95word_ids = enc.word_ids(batch_index=0)
96seen = set()
97for i, word_id in enumerate(word_ids):
98 if word_id is None or word_id in seen:
99 continue
100 seen.add(word_id)
101 tag, features = decode_token(logits[i].cpu().numpy())
102 print(f"{words[word_id]:15s} {str(tag):10s} {features}")1Hetta PDNpSN {'word_class': 'Pronoun', 'subcategory': 'D', 'gender': 'N', 'number': 'S', 'case': 'N', 'person': 'p'}
2er VNAPS3 {'word_class': 'Verb', 'number': 'S', 'mood': 'N', 'voice': 'A', 'tense': 'P', 'person': '3'}
3eitt RNSNI {'word_class': 'Article', 'gender': 'N', 'number': 'S', 'case': 'N', 'definiteness': 'I'}
4føroyskt APSNSN {'word_class': 'Adjective', 'gender': 'N', 'number': 'S', 'case': 'N', 'degree': 'P', 'declension': 'S'}
5dømi SNSNar {'word_class': 'Noun', 'gender': 'N', 'number': 'S', 'case': 'N', 'article': 'a', 'proper_noun': 'r'}model.safetensorsconfig.jsonconstraint_mask.jsontag_mappings.json :contentReference[oaicite:2]{index=2}https://github.com/Maltoknidepilin/BRAGD.git1@inproceedings{simonsen2026bragd,
2 title={{BRAGD}: Constrained Multi-Label {POS} Tagging for {F}aroese},
3 author={Simonsen, Annika and Scalvini, Barbara and Johannesen, Uni and Debess, Iben Nyholm and Einarsson, Hafsteinn and Sn{\ae}bjarnarson, V{\'e}steinn},
4 booktitle={Proceedings of the 2026 Joint International Conference on Computational Linguistics, Language Resources and Evaluation (LREC-COLING 2026)},
5 year={2026}
6}