Views
No views yet
1# Example Code: State of Art Nepali Tokenizer
2
3from transformers import AutoTokenizer
4
5# Load the tokenizer directly from Hugging Face
6tokenizer = AutoTokenizer.from_pretrained("krishnamishra8848/Nepali_Tokenizer")
7
8# Test tokenization with some text
9text = "तपाईंले आफ्ना लक्ष्यलाई प्राप्त गर्नका लागि मिहिनेत र धैर्यको महत्त्व बुझ्नुपर्छ।"
10tokens = tokenizer(text)
11
12print("Tokenized Output:", tokens)