import sentencepiece as spm

sp = spm.SentencePieceProcessor()
sp.Load("hindi_unigram.model")

tokens = sp.EncodeAsPieces("भारत एक महान देश है।")
ids = sp.EncodeAsIds("भारत एक महान देश है।")
print(sp.DecodeIds(ids))

From HuggingFace Hub

python

from huggingface_hub import hf_hub_download
import sentencepiece as spm

model_path = hf_hub_download(
    repo_id="adityaghai07/ag_hindi_uni_tokenizer_32k",
    filename="hindi_unigram.model"
)
sp = spm.SentencePieceProcessor()
sp.Load(model_path)

Source

GitHub