This tokenizer serves as a foundational tool for Natural Language Processing (NLP) tasks involving Tamazight, streamlining the development of Large Language Models (LLMs) and Machine Translation systems.
You can easily integrate this tokenizer into your Python projects to process Tamazight text. Ensure you have the sentencepiece library installed and download the tokenizer.model file from this repository.
1import sentencepiece as spm
2from huggingface_hub import hf_hub_download
3
4# Download the tokenizer model directly from Hugging Face
5model_path = hf_hub_download(repo_id="Tamazight/Tifinizer-Unigram-32K", filename="tokenizer.model")
6
7# Load the tokenizer model
8tokenizer = spm.SentencePieceProcessor()
9tokenizer.load(model_path)
10
11# Example usage
12text = "ⴰⵣⵓⵍ ⴰⵎⴰⴹⴰⵍ! ⵡⴰ ⵉⴳⴰ ⵉⵔⵉⵎ ⵏ ⵓⵙⵙⵎⵔⵙ ⵏ ⵓⵙⵏⵉⴳⵍ ⵏ ⵓⵙⴽⴽⵉⵍ ⵏ ⵜⴼⵉⵏⴰⵖ"
13tokens = tokenizer.encode(text, out_type=str)
14decoded = tokenizer.decode(tokenizer.encode(text))
15
16print(f"Text: {text}")
17print(f"Tokens: {tokens}")
18print(f"Decoded: {decoded}")
