Views
No views yet
1label_list = ["O", "B-人名", "I-人名", "B-法人名", "I-法人名", "B-政治的組織名", "I-政治的組織名",
2 "B-その他の組織名", "I-その他の組織名", "B-地名", "I-地名", "B-施設名", "I-施設名",
3 "B-製品名", "I-製品名", "B-イベント名", "I-イベント名"]1model_name = "sbintuitions/modernbert-ja-130m"
2tokenizer = AutoTokenizer.from_pretrained(model_name)
3tokenizer.backend_tokenizer.pre_tokenizer = Sequence([Split(Regex("[ぁ-ん]"),"isolated"), tokenizer.backend_tokenizer.pre_tokenizer])
4model = AutoModelForTokenClassification.from_pretrained(model_name, num_labels=17)