Views
No views yet
<|im_start|>system
{{system}}<|im_end|>
<|im_start|>{{char}}
{{message}}<|im_end|>
<|im_start|>{{user}}
{{message}}<|im_end|>*asterisks**leans against wall cooly* so like, i just casted a super strong spell at magician academy today, not gonna lie, felt badass.["\n", "<|", "</"]1# LoRA configuration
2peft_config = LoraConfig(
3 r=16,
4 lora_alpha=16,
5 lora_dropout=0.05,
6 bias="none",
7 task_type="CAUSAL_LM",
8 target_modules=['k_proj', 'gate_proj', 'v_proj', 'up_proj', 'q_proj', 'o_proj', 'down_proj']
9)
10# Model to fine-tune
11model = AutoModelForCausalLM.from_pretrained(
12 model_name,
13 torch_dtype=torch.bfloat16,
14 load_in_4bit=True
15)
16model.config.use_cache = False
17# Reference model
18ref_model = AutoModelForCausalLM.from_pretrained(
19 model_name,
20 torch_dtype=torch.bfloat16,
21 load_in_4bit=True
22)
23# Training arguments
24training_args = TrainingArguments(
25 per_device_train_batch_size=4,
26 gradient_accumulation_steps=4,
27 gradient_checkpointing=True,
28 learning_rate=5e-5,
29 lr_scheduler_type="cosine",
30 max_steps=200,
31 save_strategy="no",
32 logging_steps=1,
33 output_dir=new_model,
34 optim="paged_adamw_32bit",
35 warmup_steps=100,
36 bf16=True,
37 report_to="wandb",
38)
39# Create DPO trainer
40dpo_trainer = DPOTrainer(
41 model,
42 ref_model,
43 args=training_args,
44 train_dataset=dataset,
45 tokenizer=tokenizer,
46 peft_config=peft_config,
47 beta=0.1,
48 force_use_ref_model=True
49)
50# Fine-tune model with DPO
51dpo_trainer.train()