Views
No views yet
1from transformers import AutoTokenizer
2
3tokenizer = AutoTokenizer.from_pretrained("azherali/python-code-search-tokenizer")
4
5example = """class LinearLayer():
6 def __init__(self, input_size, output_size):
7 self.weight = torch.randn(input_size, output_size)
8 self.bias = torch.zeros(output_size)
9
10 def __call__(self, x):
11 return x @ self.weights + self.bias
12 """
13tokenizer.tokenize(example)
14
15
16