A Mistral 7B LLM built from merging pretrained models and finetuning on
flammenai/Prude-Phi3-DPO.
Flammen specializes in exceptional character roleplay, creative writing, and general intelligence.
Finetuned using an L4 on Google Colab.
1# LoRA configuration
2peft_config = LoraConfig(
3 r=16,
4 lora_alpha=16,
5 lora_dropout=0.05,
6 bias="none",
7 task_type="CAUSAL_LM",
8 target_modules=['k_proj', 'gate_proj', 'v_proj', 'up_proj', 'q_proj', 'o_proj', 'down_proj']
9)
10
11# Model to fine-tune
12model = AutoModelForCausalLM.from_pretrained(
13 model_name,
14 torch_dtype=torch.bfloat16,
15 load_in_4bit=True
16)
17model.config.use_cache = False
18
19# Reference model
20ref_model = AutoModelForCausalLM.from_pretrained(
21 model_name,
22 torch_dtype=torch.bfloat16,
23 load_in_4bit=True
24)
25
26# Training arguments
27training_args = TrainingArguments(
28 per_device_train_batch_size=2,
29 gradient_accumulation_steps=4,
30 gradient_checkpointing=True,
31 learning_rate=5e-5,
32 lr_scheduler_type="cosine",
33 max_steps=420,
34 save_strategy="no",
35 logging_steps=1,
36 output_dir=new_model,
37 optim="paged_adamw_32bit",
38 warmup_steps=100,
39 bf16=True,
40 report_to="wandb",
41)
42
43# Create DPO trainer
44dpo_trainer = DPOTrainer(
45 model,
46 ref_model,
47 args=training_args,
48 train_dataset=dataset,
49 tokenizer=tokenizer,
50 peft_config=peft_config,
51 beta=0.1,
52 max_prompt_length=2048,
53 max_length=4096,
54 force_use_ref_model=True
55)
56
57# Fine-tune model with DPO
58dpo_trainer.train()
