Views
No views yet
transformers library:M2M100Tokenizer depends on sentencepiece, so make sure to install it before running the example. To install sentencepiece, run pip install sentencepiece1from transformers import M2M100ForConditionalGeneration, M2M100Tokenizer
2
3model_name = "PrompTart/m2m100_418M_PTT_en_ko"
4tokenizer = M2M100Tokenizer.from_pretrained(model_name)
5model = M2M100ForConditionalGeneration.from_pretrained(model_name)
6
7# Example sentence
8text = "The model was fine-tuned using knowledge distillation techniques.\
9The training dataset was created using a collaborative multi-agent framework powered by large language models."
10
11# Tokenize and generate translation
12tokenizer.src_lang = "en"
13encoded = tokenizer(text.split('. '), return_tensors="pt", padding=True)
14generated_tokens = model.generate(**encoded, forced_bos_token_id=tokenizer.get_lang_id("ko"))
15outputs = tokenizer.batch_decode(generated_tokens, skip_special_tokens=True)
16print('\n'.join(outputs))
17# => "이 모델은 지식 증류 기법(knowledge distillation techniques)을 사용하여 미세 조정되었습니다.
18# 훈련 데이터셋(training dataset)은 대형 언어 모델(large language models)을 기반으로 한 협업 다중 에이전트 프레임워크(collaborative multi-agent framework)를 사용하여 생성되었습니다."
191@misc{myung2024efficienttechnicaltermtranslation,
2 title={Efficient Technical Term Translation: A Knowledge Distillation Approach for Parenthetical Terminology Translation},
3 author={Jiyoon Myung and Jihyeon Park and Jungki Son and Kyungro Lee and Joohyung Han},
4 year={2024},
5 eprint={2410.00683},
6 archivePrefix={arXiv},
7 primaryClass={cs.CL},
8 url={https://arxiv.org/abs/2410.00683},
9}