Views
No views yet
1
2tokenizer = AutoTokenizer.from_pretrained(model_id)
3model = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=torch.bfloat16,
4 device_map="auto" if device is None else device,
5 attn_implementation="flash_attention_2") # if flash enabled
6sys_prompt = '한국어로 대답해'
7texts = ['안녕', '서울은 오늘 어때']
8chats = list(map(lambda t: [{'role': 'user', 'content': f'{sys_prompt}\n{t}'}], texts)) # ChatML format
9prompts = list(map(lambda p: tokenizer.apply_chat_template(p, tokenize=False, add_generation_prompt=True), chats))
10input_ids = tokenizer(prompts, return_tensors="pt", padding=True).to("cuda" if device is None else device)
11outputs = model.generate(**input_ids, max_new_tokens=100, repetition_penalty=1.05)
12for output in outputs:
13 print(tokenizer.decode(output, skip_special_tokens=True), end='\n\n')seq_length 1024 with dataset packingbatch 3 per devicelr 1e-5optim adafactorseq_length 2048lr 2e-41if (chat[0]['role'] == 'system'):
2 chat[1]['content'] = f"{chat[0]['content']}\n{chat[1]['content']}"
3 chat = chat[1:]
4try:
5 prompt = tokenizer.apply_chat_template(chat, tokenize=False)