Views
No views yet
1from transformers import AutoTokenizer
2text = "сайн уу"
3
4mn_tokenizer = AutoTokenizer.from_pretrained("goryden/mn_tokenizer")
5
6tokens = mn_tokenizer.tokenize(text)
7encoded = mn_tokenizer.encode(text)
8decoded = mn_tokenizer.decode(encoded)
9
10print("Original:", text)
11print("Mongolian tokenizer tokens:", tokens)
12print("Decoded :", decoded)