Views
No views yet
Itau-Unibanco/NorBERTo-base | ModernBERT-Base | 22 | ~150M |Itau-Unibanco/NorBERTo-large | ModernBERT-Large | 28 | ~395M |1
2from transformers import AutoTokenizer, AutoModel
3
4
5
6model = AutoModel.from_pretrained('Itau-Unibanco/NorBERTo-base')
7
8tokenizer = AutoTokenizer.from_pretrained('Itau-Unibanco/NorBERTo-base')
91
2from transformers import pipeline
3
4
5
6pipe = pipeline('fill-mask', model='Itau-Unibanco/NorBERTo-base')
7
8
9
10pipe('O Brasil é um país de dimensões [MASK].')
111
2import torch
3
4
5
6model = AutoModel.from_pretrained('Itau-Unibanco/NorBERTo-base')
7
8tokenizer = AutoTokenizer.from_pretrained('Itau-Unibanco/NorBERTo-base')
9
10
11
12input_ids = tokenizer.encode('O Brasil é um país de dimensões continentais.', return_tensors='pt')
13
14
15
16with torch.no_grad():
17
18 outs = model(input_ids)
19
20 encoded = outs[0][0, 1:-1] # Ignore [CLS] and [SEP] special tokens
211
2@inproceedings{silva-etal-2026-norberto,
3 title = "{N}or{BERT}o: A {M}odern{BERT} Model Trained for {P}ortuguese with 331 Billion Tokens Corpus",
4 author = "Silva, Enzo S. N. and
5 Costa, Pablo B. and
6 Vlasman, Raphael C. and
7 Costa, Rosimeire P. and
8 Silva, Henrique L. P. and
9 Pellicer, Lucas F. A. O. and
10 Rinaldo, Guilherme and
11 Almeida, Renato A. and
12 Rabbani, Darian S. R. and
13 Oestreich, Cinthya O. and
14 Carid{\'a}, Vinicius F.",
15 editor = "Souza, Marlo and
16 de-Dios-Flores, Iria and
17 Santos, Diana and
18 Freitas, Larissa and
19 Souza, Jackson Wilke da Cruz and
20 Ribeiro, Eug{\'e}nio",
21 booktitle = "Proceedings of the 17th International Conference on Computational Processing of {P}ortuguese ({PROPOR} 2026) - Vol. 1",
22 month = apr,
23 year = "2026",
24 address = "Salvador, Brazil",
25 publisher = "Association for Computational Linguistics",
26 url = "https://aclanthology.org/2026.propor-1.18/",
27 pages = "183--193",
28 ISBN = "979-8-89176-387-6",
29 abstract = "High-quality corpora are essential for advancing Natural Language Processing (NLP) in Portuguese. Building on previous encoder-only models such as BERTimbau and Albertina PT-BR, we introduce NorBERTo, a modern encoder based on the ModernBERT architecture, featuring long-context support and efficient attention mechanisms. NorBERTo is trained on Aurora-PT, a newly curated Brazilian Portuguese corpus comprising 331 billion GPT-2 tokens collected from diverse web sources and existing multilingual datasets. We systematically benchmark NorBERTo against strong baselines on semantic similarity, textual entailment and classification tasks using standardized datasets such as ASSIN 2 and PLUE. On PLUE, NorBERTo-large achieves the best results among the encoder models we evaluated, notably reaching 0.9191 F1 on MRPC and 0.7689 accuracy on RTE. On ASSIN 2, NorBERTo-large attains the highest entailment F1 (0.904) among all encoders considered, while remaining competitive, but Albertina-900M and BERTimbau-large still hold an advantage. To the best of our knowledge, Aurora-PT is currently the largest openly available monolingual Portuguese corpus, surpassing previous resources. NorBERTo provides a modern, mid-sized encoder designed for realistic deployment scenarios: it is straightforward to fine-tune, efficient to serve, and well suited as a backbone for retrieval-augmented generation and other downstream Portuguese NLP systems."
30}
31Itau-Unibanco/NorBERTo-base | ModernBERT-Base | 22 | ~150M |Itau-Unibanco/NorBERTo-large | ModernBERT-Large | 28 | ~395M |1
2from transformers import AutoTokenizer, AutoModel
3
4
5
6model = AutoModel.from_pretrained('Itau-Unibanco/NorBERTo-base')
7
8tokenizer = AutoTokenizer.from_pretrained('Itau-Unibanco/NorBERTo-base')
91
2from transformers import pipeline
3
4
5
6pipe = pipeline('fill-mask', model='Itau-Unibanco/NorBERTo-base')
7
8
9
10pipe('O Brasil é um país de dimensões [MASK].')
111
2import torch
3
4
5
6model = AutoModel.from_pretrained('Itau-Unibanco/NorBERTo-base')
7
8tokenizer = AutoTokenizer.from_pretrained('Itau-Unibanco/NorBERTo-base')
9
10
11
12input_ids = tokenizer.encode('O Brasil é um país de dimensões continentais.', return_tensors='pt')
13
14
15
16with torch.no_grad():
17
18 outs = model(input_ids)
19
20 encoded = outs[0][0, 1:-1] # Ignore [CLS] e [SEP]
211
2@inproceedings{silva-etal-2026-norberto,
3 title = "{N}or{BERT}o: A {M}odern{BERT} Model Trained for {P}ortuguese with 331 Billion Tokens Corpus",
4 author = "Silva, Enzo S. N. and
5 Costa, Pablo B. and
6 Vlasman, Raphael C. and
7 Costa, Rosimeire P. and
8 Silva, Henrique L. P. and
9 Pellicer, Lucas F. A. O. and
10 Rinaldo, Guilherme and
11 Almeida, Renato A. and
12 Rabbani, Darian S. R. and
13 Oestreich, Cinthya O. and
14 Carid{\'a}, Vinicius F.",
15 editor = "Souza, Marlo and
16 de-Dios-Flores, Iria and
17 Santos, Diana and
18 Freitas, Larissa and
19 Souza, Jackson Wilke da Cruz and
20 Ribeiro, Eug{\'e}nio",
21 booktitle = "Proceedings of the 17th International Conference on Computational Processing of {P}ortuguese ({PROPOR} 2026) - Vol. 1",
22 month = apr,
23 year = "2026",
24 address = "Salvador, Brazil",
25 publisher = "Association for Computational Linguistics",
26 url = "https://aclanthology.org/2026.propor-1.18/",
27 pages = "183--193",
28 ISBN = "979-8-89176-387-6",
29 abstract = "High-quality corpora are essential for advancing Natural Language Processing (NLP) in Portuguese. Building on previous encoder-only models such as BERTimbau and Albertina PT-BR, we introduce NorBERTo, a modern encoder based on the ModernBERT architecture, featuring long-context support and efficient attention mechanisms. NorBERTo is trained on Aurora-PT, a newly curated Brazilian Portuguese corpus comprising 331 billion GPT-2 tokens collected from diverse web sources and existing multilingual datasets. We systematically benchmark NorBERTo against strong baselines on semantic similarity, textual entailment and classification tasks using standardized datasets such as ASSIN 2 and PLUE. On PLUE, NorBERTo-large achieves the best results among the encoder models we evaluated, notably reaching 0.9191 F1 on MRPC and 0.7689 accuracy on RTE. On ASSIN 2, NorBERTo-large attains the highest entailment F1 (0.904) among all encoders considered, while remaining competitive, but Albertina-900M and BERTimbau-large still hold an advantage. To the best of our knowledge, Aurora-PT is currently the largest openly available monolingual Portuguese corpus, surpassing previous resources. NorBERTo provides a modern, mid-sized encoder designed for realistic deployment scenarios: it is straightforward to fine-tune, efficient to serve, and well suited as a backbone for retrieval-augmented generation and other downstream Portuguese NLP systems."
30}
31