Views
No views yet
0.4.01adapter: null
2base_model: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
3bf16: auto
4dataset_prepared_path: last_run_prepared
5datasets:
6- path: utrgvseniorproject/Tinybook
7 type: completion
8debug: null
9deepspeed: null
10early_stopping_patience: null
11eval_sample_packing: false
12eval_table_size: null
13evals_per_epoch: 4
14flash_attention: true
15flash_attn_cross_entropy: false
16flash_attn_fuse_mlp: true
17flash_attn_fuse_qkv: false
18flash_attn_rms_norm: true
19fp16: null
20fsdp: null
21fsdp_config: null
22gradient_accumulation_steps: 1
23gradient_checkpointing: true
24group_by_length: false
25learning_rate: 0.0002
26load_in_4bit: false
27load_in_8bit: false
28local_rank: null
29logging_steps: 1
30lora_alpha: null
31lora_dropout: null
32lora_fan_in_fan_out: null
33lora_model_dir: null
34lora_r: null
35lora_target_linear: null
36lr_scheduler: cosine
37micro_batch_size: 1
38model_type: LlamaForCausalLM
39num_epochs: 1
40optimizer: adamw_bnb_8bit
41output_dir: ./TinyLlama-Tinybook-epochs-1-lr-0002_Train_On_Input
42pad_to_sequence_len: true
43resume_from_checkpoint: null
44sample_packing: true
45saves_per_epoch: 1
46sequence_len: 2048
47special_tokens: null
48strict: false
49tf32: false
50tokenizer_type: LlamaTokenizer
51train_on_inputs: true
52val_set_size: 0.05
53wandb_entity: utrgvmedai
54wandb_log_model: null
55wandb_name: tinyLama_Tinybook_epochs_1_lr_0002
56wandb_project: TinyLlama-Train-On-Input
57wandb_watch: null
58warmup_steps: 100
59weight_decay: 0.1
60xformers_attention: null
61| Training Loss | Epoch | Step | Validation Loss |
|---|---|---|---|
| 1.7259 | 0.04 | 1 | 1.9138 |
| 1.8155 | 0.26 | 6 | 1.9014 |
| 1.8636 | 0.52 | 12 | 1.8655 |
| 1.8758 | 0.78 | 18 | 1.8024 |