Views
No views yet
from transformers import AutoTokenizer tokenizer = AutoTokenizer.from_pretrained("israel/AmhWordPieceTokenizer") encoding = tokenizer.encode("ኮሌጁ ቢያስተምርም ወደስራ የሚመድባቸው መንግስት ነው abcs") encoding.tokens
