Views
No views yet
0.16.11base_model: /workspace/models/llama3-70b
2model_type: LlamaForCausalLM
3tokenizer_type: AutoTokenizer
4load_in_4bit: true
5strict: false
6datasets:
7 - path: /workspace/lora_training_21f34b0a_1777391407570.jsonl
8 type: alpaca
9 ds_type: json
10dataset_prepared_path: /workspace/axo/last_run_prepared
11val_set_size: 0.02
12output_dir: /workspace/output/llama3-70b-lora
13sequence_len: 4096
14sample_packing: true
15pad_to_sequence_len: true
16adapter: qlora
17lora_r: 16
18lora_alpha: 32
19lora_dropout: 0.05
20lora_target_linear: true
21gradient_accumulation_steps: 16
22micro_batch_size: 1
23num_epochs: 3
24optimizer: paged_adamw_8bit
25learning_rate: 0.0001
26lr_scheduler: cosine
27warmup_ratio: 0.03
28bf16: true
29tf32: true
30gradient_checkpointing: true
31flash_attention: true
32logging_steps: 5
33save_strategy: epoch
34save_total_limit: 2
35special_tokens:
36 pad_token: "<|end_of_text|>"
37