Views
No views yet
0.4.11adapter: lora
2base_model: Casual-Autopsy/L3-Umbral-Mind-RP-v3.0-8B
3bf16: auto
4chat_template: llama3
5cosine_min_lr_ratio: 0.1
6data_processes: 4
7dataset_prepared_path: null
8datasets:
9- data_files:
10 - 6e6190eb26c4eb66_train_data.json
11 ds_type: json
12 format: custom
13 path: /workspace/input_data/6e6190eb26c4eb66_train_data.json
14 type:
15 field_input: user_input
16 field_instruction: prompt
17 field_output: chosen
18 format: '{instruction} {input}'
19 no_input_format: '{instruction}'
20 system_format: '{system}'
21 system_prompt: ''
22debug: null
23deepspeed: null
24device_map:
25 lm_head: 3
26 model.embed_tokens: 0
27 model.layers.0: 0
28 model.layers.1: 0
29 model.layers.10: 3
30 model.layers.11: 3
31 model.layers.2: 0
32 model.layers.3: 1
33 model.layers.4: 1
34 model.layers.5: 1
35 model.layers.6: 2
36 model.layers.7: 2
37 model.layers.8: 2
38 model.layers.9: 3
39 model.norm: 3
40do_eval: true
41early_stopping_patience: 1
42eval_batch_size: 1
43eval_sample_packing: false
44eval_steps: 25
45evaluation_strategy: steps
46flash_attention: false
47fp16: null
48fsdp: null
49fsdp_config: null
50gradient_accumulation_steps: 32
51gradient_checkpointing: true
52group_by_length: true
53hub_model_id: null
54hub_strategy: checkpoint
55hub_token: null
56learning_rate: 0.0001
57load_in_4bit: false
58load_in_8bit: false
59local_rank: null
60logging_steps: 1
61lora_alpha: 64
62lora_dropout: 0.05
63lora_fan_in_fan_out: null
64lora_model_dir: null
65lora_r: 32
66lora_target_linear: true
67lora_target_modules:
68- q_proj
69- v_proj
70lr_scheduler: cosine
71max_grad_norm: 0.3
72max_memory:
73 0: 60GB
74 1: 70GB
75 2: 70GB
76 3: 70GB
77 cpu: 96GB
78max_steps: 50
79micro_batch_size: 1
80mixed_precision: bf16
81mlflow_experiment_name: /tmp/6e6190eb26c4eb66_train_data.json
82model_type: AutoModelForCausalLM
83num_epochs: 3
84optim_args:
85 adam_beta1: 0.9
86 adam_beta2: 0.95
87 adam_epsilon: 1e-5
88optimizer: adamw_torch
89output_dir: miner_id_24
90pad_to_sequence_len: true
91resume_from_checkpoint: null
92s2_attention: null
93sample_packing: false
94save_steps: 25
95save_strategy: steps
96sequence_len: 2048
97special_tokens:
98 pad_token: <|eot_id|>
99strict: false
100tf32: false
101tokenizer_type: AutoTokenizer
102torch_compile: false
103torch_dtype: bfloat16
104train_on_inputs: false
105trust_remote_code: true
106use_cache: false
107val_set_size: 50
108wandb_entity: sn56-miner
109wandb_mode: disabled
110wandb_name: null
111wandb_project: god
112wandb_run: c17r
113wandb_runid: null
114warmup_ratio: 0.05
115weight_decay: 0.01
116xformers_attention: null
117| Training Loss | Epoch | Step | Validation Loss |
|---|---|---|---|
| 1.9916 | 0.0008 | 1 | 2.3692 |
| 0.9831 | 0.0208 | 25 | 0.9058 |
| 0.9995 | 0.0416 | 50 | 0.8768 |