Views
No views yet
m.model) for proper tokenization. You can use either the HuggingFace AutoTokenizer (recommended) or SentencePiece directly.1from transformers import AutoTokenizer, ElectraForPreTraining
2
3# Load the tokenizer directly from HuggingFace Hub
4tokenizer = AutoTokenizer.from_pretrained("AILabTUL/BiELECTRA-norwegian-swedish")
5
6# Or load from local directory
7# tokenizer = AutoTokenizer.from_pretrained("./NOSWE")
8
9# Load the pretrained model
10model = ElectraForPreTraining.from_pretrained("AILabTUL/BiELECTRA-norwegian-swedish")
11
12# Tokenize input text
13sentence = "Dette er en testsetning på norsk og svenska."
14inputs = tokenizer(sentence, return_tensors="pt")
15
16# Run inference
17outputs = model(**inputs)1from transformers import ElectraForPreTraining
2import sentencepiece as spm
3import torch
4
5# Load the SentencePiece model
6sp = spm.SentencePieceProcessor()
7sp.load("m.model")
8
9# Load the pretrained model
10discriminator = ElectraForPreTraining.from_pretrained("AILabTUL/BiELECTRA-norwegian-swedish")
11
12# Tokenize input text (note: input should be lowercase)
13sentence = "dette er en testsetning på norsk og svenska."
14tokens = sp.encode(sentence, out_type=str)
15token_ids = sp.encode(sentence)
16
17# Convert to tensor
18input_tensor = torch.tensor([token_ids])
19
20# Run inference
21outputs = discriminator(input_tensor)
22predictions = torch.nn.Sigmoid()(outputs[0]).cpu().detach().numpy()@inproceedings{polacek-2025-study,
title = "Study on Automatic Punctuation Restoration in Bilingual Broadcast Stream",
author = "Polacek, Martin",
editor = "Velichkov, Boris and
Nikolova-Koleva, Ivelina and
Slavcheva, Milena",
booktitle = "Proceedings of the 9th Student Research Workshop associated with the International Conference Recent Advances in Natural Language Processing",
month = sep,
year = "2025",
address = "Varna, Bulgaria",
publisher = "INCOMA Ltd., Shoumen, Bulgaria",
url = "https://aclanthology.org/2025.ranlp-stud.5/",
pages = "37--43",
doi = "10.26615/issn.2603-2821.2025_005"
}