Views
No views yet
PreTrainedTokenizerFast object, wrapping a tokenizer from the tokenizers library.~1.2 million sentences).[UNK], [PAD], [CLS], [SEP], [MASK]BertPreTokenizertransformers library.1from transformers import PreTrainedTokenizerFast
2
3# Load the tokenizer from the Hub
4tokenizer = PreTrainedTokenizerFast.from_pretrained("sachin333/hindi-english-wordpiece-nmt")
5
6# Example Usage
7english_text = "This is a test of the tokenizer."
8hindi_text = "यह टोकनाइज़र का एक परीक्षण है।"
9
10encoded_en = tokenizer(english_text)
11encoded_hi = tokenizer(hindi_text)
12
13print("Encoded English:", encoded_en)
14print("Decoded English:", tokenizer.decode(encoded_en['input_ids']))
15
16print("\nEncoded Hindi:", encoded_hi)
17print("Decoded Hindi:", tokenizer.decode(encoded_hi['input_ids']))