Views
No views yet
| Parameter | Value |
|---|---|
| learning_rate | 1e-05 |
| num_train_epochs | 1.0 |
| per_device_train_batch_size | 1 |
| gradient_accumulation_steps | 2 |
| weight_decay | 0.0 |
| warmup_ratio | 0.0 |
| warmup_steps | 180 |
| lr_scheduler_type | SchedulerType.CONSTANT |
| optim | OptimizerNames.ADAMW_BNB |
| bf16 | True |
| fp16 | False |
| max_grad_norm | 1.0 |
| max_steps | 6000 |
| save_steps | 1000 |
| deepspeed | {'bf16': {'enabled': 'auto'}, 'zero_optimization': {'stage': 3, 'offload_optimizer': {'device': 'none'}, 'offload_param': {'device': 'none'}, 'overlap_comm': True, 'contiguous_gradients': True, 'reduce_bucket_size': 500000000.0, 'stage3_prefetch_bucket_size': 400000000.0, 'stage3_param_persistence_threshold': 1000000.0, 'stage3_gather_16bit_weights_on_model_save': True}, 'gradient_accumulation_steps': 'auto', 'gradient_clipping': 'auto', 'train_batch_size': 'auto', 'train_micro_batch_size_per_gpu': 'auto', 'wall_clock_breakdown': False} |
| gradient_checkpointing | True |
Note: this is the config file at push time; training parameters above are extracted from the checkpoint and reflect what was actually used.
1base_model: meta-llama/Llama-3.1-8B-Instruct
2model_type: LlamaForCausalLM
3tokenizer_type: AutoTokenizer
4
5load_in_8bit: false
6load_in_4bit: false
7
8# 1:9 mixing: 23k AR docs + 210k DCLM docs (~233k total, 90% pretraining)
9# Axolotl concatenates and shuffles both datasets
10datasets:
11 - path: cfierro/simpleqa_wiki_ar_Llama-3.1-8B-Instruct
12 type: completion
13 field: text
14 split: test
15 - path: cfierro/dclm_baseline_sampled
16 type: completion
17 field: text
18 split: sampled_210k
19dataset_prepared_path: /scratch/project/eu-25-39/knowledge-ft/axolotl/datasets/llama-8b/simpleqa-ar-dclm-1to9
20val_set_size: 0.0
21output_dir: /scratch/project/eu-25-39/knowledge-ft/axolotl/models/llama-3.1-8b-fft-simpleqa-ar-dclm-1to9
22hub_model_id: llama-3.1-8b-fft-simpleqa-ar-dclm-1to9
23
24sequence_len: 4096
25sample_packing: true
26eval_sample_packing: false
27
28# No LoRA — full fine-tuning
29
30wandb_project: knowledge-ft
31wandb_entity: cfierro
32wandb_watch:
33wandb_name: llama-3.1-8b-fft-simpleqa-ar-dclm-1to9
34wandb_log_model: "false"
35
36# Multi-GPU settings
37# micro_batch=1 to fit full FT in memory (ZeRO-3 on 4x A100 40GB)
38# grad_accum=2 to keep same effective batch size (1 * 2 * 4 GPUs = 8)
39# ~233k examples with sample_packing (~6-8 per packed seq) → ~33k steps for 1 pass
40# Using 6000 steps same as baseline to keep step count comparable
41gradient_accumulation_steps: 2
42micro_batch_size: 1
43max_steps: 6000
44
45optimizer: adamw_bnb_8bit
46lr_scheduler: constant
47learning_rate: 1e-5
48
49bf16: auto
50tf32: false
51
52gradient_checkpointing: true
53resume_from_checkpoint:
54logging_steps: 1
55flash_attention: true
56
57warmup_ratio: 0.03
58save_steps: 1000
59save_total_limit: 1
60load_best_model_at_end: true
61weight_decay: 0.0
62special_tokens:
63 pad_token: <|end_of_text|>
64
65# DeepSpeed ZeRO Stage 3 - shards model weights, gradients, and optimizer across GPUs
66deepspeed: deepspeed_configs/zero3.json