Views
No views yet
1
2###### Load the Tokenizer
3
4from transformers import AutoTokenizer
5
6hf_token = "<HF TOKEN>" # Replace with your token
7model_id = "nesaorg/Llama-3.2-1B-Instruct-Encrypted"
8tokenizer = AutoTokenizer.from_pretrained(model_id, token=hf_token, local_files_only=False)1text = "I'm super excited to join Nesa's Equivariant Encryption initiative!"
2
3# Encode text into token IDs
4token_ids = tokenizer.encode(text)
5print("Token IDs:", token_ids)
6
7# Decode token IDs back to text
8decoded_text = tokenizer.decode(token_ids)
9print("Decoded Text:", decoded_text)Token IDs: [128000, 1495, 1135, 2544, 6705, 284, 2219, 11659, 17098, 22968, 8707, 2544, 3539, 285, 34479]
Decoded Text: I'm super excited to join Nesa's Equivariant Encryption initiative!