Views
No views yet
1# LoRA configuration
2peft_config = LoraConfig(
3 r=16,
4 lora_alpha=16,
5 lora_dropout=0.05,
6 bias="none",
7 task_type="CAUSAL_LM",
8 target_modules=['k_proj', 'gate_proj', 'v_proj', 'up_proj', 'q_proj', 'o_proj', 'down_proj']
9)
10# Model to fine-tune
11model = AutoModelForCausalLM.from_pretrained(
12 model_name,
13 torch_dtype=torch.bfloat16,
14 load_in_4bit=True
15)
16model.config.use_cache = False
17# Reference model
18ref_model = AutoModelForCausalLM.from_pretrained(
19 model_name,
20 torch_dtype=torch.bfloat16,
21 load_in_4bit=True
22)
23# Training arguments
24training_args = TrainingArguments(
25 per_device_train_batch_size=4,
26 gradient_accumulation_steps=4,
27 gradient_checkpointing=True,
28 learning_rate=5e-5,
29 lr_scheduler_type="cosine",
30 max_steps=2000,
31 save_strategy="no",
32 logging_steps=1,
33 output_dir=new_model,
34 optim="paged_adamw_32bit",
35 warmup_steps=100,
36 bf16=True,
37 report_to="wandb",
38)
39# Create DPO trainer
40dpo_trainer = DPOTrainer(
41 model,
42 ref_model,
43 args=training_args,
44 train_dataset=dataset,
45 tokenizer=tokenizer,
46 peft_config=peft_config,
47 beta=0.1,
48 max_prompt_length=2048,
49 max_length=8192,
50 force_use_ref_model=True
51)