Views
No views yet
0.4.01adapter: null
2base_model: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
3bf16: auto
4dataset_prepared_path: last_run_prepared
5datasets:
6- path: utrgvseniorproject/medtext
7 type: completion
8debug: null
9deepspeed: null
10early_stopping_patience: null
11eval_sample_packing: false
12eval_table_size: null
13evals_per_epoch: 4
14flash_attention: true
15flash_attn_cross_entropy: false
16flash_attn_fuse_mlp: true
17flash_attn_fuse_qkv: false
18flash_attn_rms_norm: true
19fp16: null
20fsdp: null
21fsdp_config: null
22gradient_accumulation_steps: 1
23gradient_checkpointing: true
24group_by_length: false
25learning_rate: 0.0002
26load_in_4bit: false
27load_in_8bit: false
28local_rank: null
29logging_steps: 1
30lora_alpha: null
31lora_dropout: null
32lora_fan_in_fan_out: null
33lora_model_dir: null
34lora_r: null
35lora_target_linear: null
36lr_scheduler: cosine
37micro_batch_size: 1
38model_type: LlamaForCausalLM
39num_epochs: 1
40optimizer: adamw_bnb_8bit
41output_dir: ./TinyLlama-medtext-epochs-1-lr-0002_Train_On_Input
42pad_to_sequence_len: true
43resume_from_checkpoint: null
44sample_packing: true
45saves_per_epoch: 1
46sequence_len: 2048
47special_tokens: null
48strict: false
49tf32: false
50tokenizer_type: LlamaTokenizer
51train_on_inputs: true
52val_set_size: 0.05
53wandb_entity: utrgvmedai
54wandb_log_model: null
55wandb_name: tinyLama_medtext_epochs_1_lr_0002
56wandb_project: TinyLlama-Train-On-Input
57wandb_watch: null
58warmup_steps: 100
59weight_decay: 0.1
60xformers_attention: null
61| Training Loss | Epoch | Step | Validation Loss |
|---|---|---|---|
| 1.7582 | 0.0 | 1 | 2.1282 |
| 2.837 | 0.25 | 155 | 3.0603 |
| 2.9122 | 0.5 | 310 | 3.6874 |
| 2.6502 | 0.75 | 465 | 3.2784 |
| 1.7745 | 1.0 | 620 | 3.2155 |