Views
No views yet
0.4.11adapter: lora
2base_model: EleutherAI/pythia-70m-deduped
3bf16: auto
4chat_template: llama3
5cosine_min_lr_ratio: 0.1
6data_processes: 16
7dataset_prepared_path: null
8datasets:
9- data_files:
10 - 62fb9d7cd79067b3_train_data.json
11 ds_type: json
12 format: custom
13 path: /workspace/input_data/62fb9d7cd79067b3_train_data.json
14 type:
15 field_input: gpt4_explanations
16 field_instruction: context
17 field_output: outcome
18 format: '{instruction} {input}'
19 no_input_format: '{instruction}'
20 system_format: '{system}'
21 system_prompt: ''
22debug: null
23deepspeed: null
24device_map: '{'''':torch.cuda.current_device()}'
25do_eval: true
26early_stopping_patience: 1
27eval_batch_size: 6
28eval_sample_packing: false
29eval_steps: 25
30evaluation_strategy: steps
31flash_attention: false
32fp16: null
33fsdp: null
34fsdp_config: null
35gradient_accumulation_steps: 5
36gradient_checkpointing: true
37group_by_length: true
38hub_model_id: null
39hub_repo: stevemonite
40hub_strategy: checkpoint
41hub_token: null
42learning_rate: 0.0003
43load_in_4bit: false
44load_in_8bit: false
45local_rank: null
46logging_steps: 1
47lora_alpha: 32
48lora_dropout: 0.05
49lora_fan_in_fan_out: null
50lora_model_dir: null
51lora_r: 16
52lora_target_linear: true
53lora_target_modules:
54- q_proj
55- v_proj
56lr_scheduler: cosine
57max_grad_norm: 1.0
58max_memory:
59 0: 70GiB
60max_steps: 375
61micro_batch_size: 6
62mlflow_experiment_name: /tmp/62fb9d7cd79067b3_train_data.json
63model_type: AutoModelForCausalLM
64num_epochs: 2
65optim_args:
66 adam_beta1: 0.9
67 adam_beta2: 0.95
68 adam_epsilon: 1e-5
69optimizer: adamw_torch
70output_dir: miner_id_24
71pad_to_sequence_len: true
72resume_from_checkpoint: null
73s2_attention: null
74sample_packing: false
75save_steps: 50
76save_strategy: steps
77sequence_len: 2048
78special_tokens:
79 pad_token: <|endoftext|>
80strict: false
81tf32: false
82tokenizer_type: AutoTokenizer
83torch_compile: false
84train_on_inputs: false
85trust_remote_code: true
86val_set_size: 50
87wandb_entity: sn56-miner
88wandb_mode: disabled
89wandb_name: null
90wandb_project: god
91wandb_run: c5ga
92wandb_runid: null
93warmup_raio: 0.03
94warmup_ratio: 0.03
95weight_decay: 0.01
96xformers_attention: null
97| Training Loss | Epoch | Step | Validation Loss |
|---|---|---|---|
| 606.47 | 0.0066 | 1 | 128.0950 |
| 107.0225 | 0.1649 | 25 | 26.2796 |
| 61.9235 | 0.3298 | 50 | 10.9448 |
| 44.5495 | 0.4947 | 75 | 6.8106 |
| 24.4823 | 0.6596 | 100 | 5.5373 |
| 20.8706 | 0.8245 | 125 | 4.7062 |
| 18.9357 | 0.9894 | 150 | 4.2778 |
| 16.7825 | 1.1544 | 175 | 3.4199 |
| 15.083 | 1.3193 | 200 | 3.2062 |
| 16.1497 | 1.4842 | 225 | 3.2653 |