Views
No views yet
Important! This model was created as part of fine-tuning training. I would greatly appreciate your feedback..
Temperature: 1.12 – 1.5
Min‑P: 0.01
Top‑K: 40
Repetition Penalty: 1.5<|im_start|>system
{system_prompt}
<|im_end|>
<|im_start|>user
{input}
<|im_end|>
<|im_start|>assistant
{output}
<|im_end|>1base_model: Qwen/Qwen3-8B
2
3plugins:
4 - axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin
5strict: false
6load_in_8bit: false
7load_in_4bit: true
8
9chat_template: chatml
10datasets:
11 - path: Gryphe/Opus-WritingPrompts
12 type: chat_template
13 field_messages: conversations
14 message_property_mappings:
15 role: from
16 content: value
17 - path: Gryphe/Sonnet3.5-Charcard-Roleplay
18 type: chat_template
19 field_messages: conversations
20 message_property_mappings:
21 role: from
22 content: value
23shuffle_merged_datasets: true
24
25dataset_prepared_path:
26val_set_size: 0.05
27output_dir: ./outputs/lora-out
28
29sequence_len: 8192
30sample_packing: false
31pad_to_sequence_len: true
32
33adapter: qlora
34lora_model_dir:
35lora_r: 16
36lora_alpha: 16
37lora_dropout: 0.1
38lora_target_modules:
39 - q_proj
40 - k_proj
41 - v_proj
42 - o_proj
43 - down_proj
44 - up_proj
45
46lora_target_linear: true
47peft_use_rslora: true
48
49gradient_accumulation_steps: 4
50micro_batch_size: 2
51num_epochs: 2
52optimizer: adamw_bnb_8bit
53lr_scheduler: cosine
54learning_rate: 5e-5
55auto_batch_size: true
56train_on_inputs: false
57bf16: auto
58tf32: false
59
60gradient_checkpointing: true
61resume_from_checkpoint:
62logging_steps: 1
63flash_attention: true
64auto_resume_from_checkpoints: true
65warmup_ratio: 0.05
66evals_per_epoch: 4
67saves_per_epoch: 4
68weight_decay: 0.01
69
70save_total_limit: 3
71
72deepspeed: deepspeed_configs/zero1.json