Views
No views yet
1from transformers import AutoProcessor
2
3processor = AutoProcessor.from_pretrained(
4 "p1atdev/multi-tokenizers-processor-sample",
5 trust_remote_code=True,
6 commit_hash="111e8a30609fb5bc13e16d08f7c49196b23d5056"
7)
8
9print(processor(
10 text_1="テキスト1",
11 text_2="テキスト2",
12))
13# {'input_ids': tensor([[ 1, 43412, 28745]]), 'attention_mask': tensor([[1, 1, 1]]), 'input_ids_2': tensor([[56833, 61803, 70534, 17]]), 'attention_mask_2': tensor([[1, 1, 1, 1]])}