1base_model: Afterparty-hf/pretrain-0.924
2load_in_8bit: false
3load_in_4bit: false
4strict: false
5
6datasets:
7 - path: Afterparty-hf/synthetic-instruct
8 type: sharegpt
9
10
11 - path: Afterparty-hf/train-format-server
12 type: sharegpt
13
14
15 - path: Afterparty-hf/help-channels-formatted
16 type: sharegpt
17
18
19 - path: Afterparty-hf/constt-augmented
20 type: sharegpt
21
22
23 - path: Afterparty-hf/transcripts-train
24 type: sharegpt
25
26
27chat_template: chatml
28dataset_prepared_path: ./prepath
29hub_model_id: Afterparty-hf/finetune-0.559
30wandb_project: ap_publi
31hf_use_auth_token: true
32
33
34output_dir: ./finetune-559-a
35resume_from_checkpoint: ./finetune-559/checkpoint-1026
36wandb_watch: all
37hub_private_repo: true
38hub_strategy: all_checkpoints
39push_to_hub: false
40hf_use_auth_token: true
41max_grad_norm: 0.6
42sequence_len: 14256
43sample_packing: true
44pad_to_sequence_len: true
45micro_batch_size: 1
46gradient_accumulation_steps: 1
47num_epochs: 4
48learning_rate: 0.000004
49optimizer: adamw_bnb_8bit
50#optim_args:
51 # amsgrad: true
52lr_scheduler: cosine
53train_on_inputs: false
54group_by_length: false
55bfloat16: false
56#bf16: auto
57fp16:
58tf32: false
59neftune_noise_alpha: 2
60gradient_checkpointing: true
61gradient_checkpointing_kwargs:
62 use_reentrant: true
63logging_steps: 1
64xformers_attention:
65flash_attention: true
66#unsloth_lora_mlp: true
67#unsloth_lora_qkv: true
68#unsloth_lora_o: true
69#flash_attn_cross_entropy: true
70#flash_attn_rms_norm: true
71#flash_attn_fuse_qkv: false
72#flash_attn_fuse_mlp: true
73warmup_ratio: 0.5
74evals_per_step: 0.025
75eval_table_size:
76saves_per_epoch: 5
77debug:
78torch_compile: true
79rank:
80deepspeed: deepspeed_configs/zero2.json
81save_safetensors: true
82weight_decay: 0.01
83special_tokens:
84 bos_token: "<s>"
85 eos_token: "</s>"
86 unk_token: "<unk>"
87 pad_token: "</s>"
88tokens: # these are delimiters
89 - "<|im_start|>"
90 - "<|im_end|>"
91
