Mahou is our attempt to build a production-ready conversational/roleplay LLM.
Future versions will be released iteratively and finetuned from flammen.ai conversational data.
This model has been trained to use ChatML format.
<|im_start|>system
{{system}}<|im_end|>
<|im_start|>{{char}}
{{message}}<|im_end|>
<|im_start|>{{user}}
{{message}}<|im_end|>
Finetuned using an A100 on Google Colab.
1# LoRA configuration
2peft_config = LoraConfig(
3 r=16,
4 lora_alpha=16,
5 lora_dropout=0.05,
6 bias="none",
7 task_type="CAUSAL_LM",
8 target_modules=['k_proj', 'gate_proj', 'v_proj', 'up_proj', 'q_proj', 'o_proj', 'down_proj']
9)
10
11# Model to fine-tune
12model = AutoModelForCausalLM.from_pretrained(
13 model_name,
14 torch_dtype=torch.bfloat16,
15 load_in_4bit=True
16)
17model.config.use_cache = False
18
19# Reference model
20ref_model = AutoModelForCausalLM.from_pretrained(
21 model_name,
22 torch_dtype=torch.bfloat16,
23 load_in_4bit=True
24)
25
26# Training arguments
27training_args = TrainingArguments(
28 per_device_train_batch_size=4,
29 gradient_accumulation_steps=4,
30 gradient_checkpointing=True,
31 learning_rate=5e-5,
32 lr_scheduler_type="cosine",
33 max_steps=2000,
34 save_strategy="no",
35 logging_steps=1,
36 output_dir=new_model,
37 optim="paged_adamw_32bit",
38 warmup_steps=100,
39 bf16=True,
40 report_to="wandb",
41)
42
43# Create DPO trainer
44dpo_trainer = DPOTrainer(
45 model,
46 ref_model,
47 args=training_args,
48 train_dataset=dataset,
49 tokenizer=tokenizer,
50 peft_config=peft_config,
51 beta=0.1,
52 force_use_ref_model=True
53)
54
55# Fine-tune model with DPO
56dpo_trainer.train()
