Views
No views yet
0.4.11adapter: lora
2base_model: Qwen/Qwen2.5-1.5B-Instruct
3bf16: auto
4chat_template: llama3
5cosine_min_lr_ratio: 0.1
6data_processes: 4
7dataset_prepared_path: null
8datasets:
9- data_files:
10 - c66cbf8a4b652146_train_data.json
11 ds_type: json
12 format: custom
13 path: /workspace/input_data/c66cbf8a4b652146_train_data.json
14 type:
15 field_input: new-context
16 field_instruction: new-instruction
17 field_output: new-response
18 format: '{instruction} {input}'
19 no_input_format: '{instruction}'
20 system_format: '{system}'
21 system_prompt: ''
22debug: null
23deepspeed: null
24device_map:
25 lm_head: 3
26 model.embed_tokens: 0
27 model.layers.0: 0
28 model.layers.1: 0
29 model.layers.10: 3
30 model.layers.11: 3
31 model.layers.2: 0
32 model.layers.3: 1
33 model.layers.4: 1
34 model.layers.5: 1
35 model.layers.6: 2
36 model.layers.7: 2
37 model.layers.8: 2
38 model.layers.9: 3
39 model.norm: 3
40do_eval: true
41early_stopping_patience: 1
42eval_batch_size: 1
43eval_sample_packing: false
44eval_steps: 25
45evaluation_strategy: steps
46flash_attention: false
47fp16: null
48fsdp: null
49fsdp_config: null
50gradient_accumulation_steps: 32
51gradient_checkpointing: true
52group_by_length: true
53hub_model_id: sn56/7e74d406-5b9b-4d97-b3d7-e6015cf1f5fe
54hub_strategy: checkpoint
55hub_token: null
56learning_rate: 0.0001
57load_in_4bit: false
58load_in_8bit: false
59local_rank: null
60logging_steps: 1
61lora_alpha: 64
62lora_dropout: 0.05
63lora_fan_in_fan_out: null
64lora_model_dir: null
65lora_r: 32
66lora_target_linear: true
67lora_target_modules:
68- q_proj
69- v_proj
70lr_scheduler: cosine
71max_grad_norm: 0.3
72max_memory:
73 0: 60GB
74 1: 70GB
75 2: 70GB
76 3: 70GB
77 cpu: 96GB
78max_steps: 75
79micro_batch_size: 1
80mixed_precision: bf16
81mlflow_experiment_name: /tmp/c66cbf8a4b652146_train_data.json
82model_type: AutoModelForCausalLM
83num_epochs: 3
84optim_args:
85 adam_beta1: 0.9
86 adam_beta2: 0.95
87 adam_epsilon: 1e-5
88optimizer: adamw_torch
89output_dir: miner_id_24
90pad_to_sequence_len: true
91resume_from_checkpoint: null
92s2_attention: null
93sample_packing: false
94save_steps: 25
95save_strategy: steps
96sequence_len: 2048
97strict: false
98tf32: false
99tokenizer_type: AutoTokenizer
100torch_compile: false
101torch_dtype: bfloat16
102train_on_inputs: false
103trust_remote_code: true
104use_cache: false
105val_set_size: 50
106wandb_entity: null
107wandb_mode: online
108wandb_name: 7e74d406-5b9b-4d97-b3d7-e6015cf1f5fe
109wandb_project: Public_TuningSN
110wandb_runid: 7e74d406-5b9b-4d97-b3d7-e6015cf1f5fe
111warmup_ratio: 0.05
112weight_decay: 0.01
113xformers_attention: null
114| Training Loss | Epoch | Step | Validation Loss |
|---|---|---|---|
| 1.6962 | 0.0087 | 1 | 1.7466 |
| 0.5118 | 0.2167 | 25 | 1.0125 |
| 0.5335 | 0.4334 | 50 | 0.9996 |
| 0.4693 | 0.6501 | 75 | 0.9915 |