1base_model: meta-llama/Llama-3.2-3B-Instruct
2# Automatically upload checkpoint and final model to HF
3hub_model_id: smohammadi/qat-llama-3B # username/custom_model_name
4
5load_in_8bit: false
6load_in_4bit: false
7strict: false
8
9
10 #liger_rope: true
11 #liger_rms_norm: true
12 #liger_glu_activation: true
13 #liger_layer_norm: true
14 #
15 #liger_fused_linear_cross_entropy: true
16
17datasets:
18 - path: yahma/alpaca-cleaned
19 type: alpaca
20 split: train[:95%]
21output_dir: ./outputs/qat-train_on_inputs/
22dataset_prepared_path: ./outputs/ds_prepared_new_token
23 #sample_packing: true
24
25sequence_len: 8192
26flash_attention: true
27 #flex_attention: true
28 #flex_attn_compile_kwargs:
29 # dynamic: false
30 # mode: max-autotune-no-cudagraphs
31
32qat:
33 activation_dtype: int8
34 weight_dtype: int4
35 group_size: 32
36
37wandb_project: qat_v2
38wandb_entity:
39wandb_watch:
40wandb_name: qat-train-on-inputs
41wandb_log_model:
42
43gradient_accumulation_steps: 1
44micro_batch_size: 16
45num_epochs: 1
46optimizer: adamw_torch_fused
47train_on_inputs: true
48
49 #cosine_constant_lr_ratio: 0
50 #cosine_min_lr_ratio: 1.0
51lr_scheduler: constant
52learning_rate: 2e-5
53save_only_model: true
54bf16: true
55
56resume_from_checkpoint:
57logging_steps: 1
58include_tkps: true
59evals_per_epoch: 1
60saves_per_epoch: 1
61
62 #warmup_ratio: 0.1
63weight_decay: 0.0
64
65fsdp_config:
66 fsdp_version: 2
67 fsdp_offload_params: false
68 fsdp_cpu_ram_efficient_loading: False
69 fsdp_auto_wrap_policy: TRANSFORMER_BASED_WRAP
70 fsdp_transformer_layer_cls_to_wrap: LlamaDecoderLayer
71 fsdp_state_dict_type: FULL_STATE_DICT
72 fsdp_sharding_strategy: FULL_SHARD
73 fsdp_reshard_after_forward: true
74 fsdp_activation_checkpointing: true
75
76special_tokens:
77 pad_token: <|finetune_right_pad_id|>
78
