Views
No views yet
0.10.0.dev01base_model: Qwen/Qwen2.5-Coder-7B-Instruct
2
3plugins:
4 - axolotl.integrations.kd.KDPlugin
5 - axolotl.integrations.liger.LigerPlugin
6
7liger_rms_norm: true
8liger_glu_activation: true
9
10 # torch_compile: true
11
12strict: false
13
14chat_template_jinja: "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0]['role'] == 'system' %}\n {{- messages[0]['content'] }}\n {%- else %}\n {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n {%- endif %}\n {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0]['role'] == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n {%- else %}\n {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {{- '<|im_start|>' + message.role }}\n {%- if message.content %}\n {{- '\\n' + message.content }}\n {%- endif %}\n {%- for tool_call in message.tool_calls %}\n {%- if tool_call.function is defined %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '\\n<tool_call>\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {{- tool_call.arguments | tojson }}\n {{- '}\\n</tool_call>' }}\n {%- endfor %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n<tool_response>\\n' }}\n {{- message.content }}\n {{- '\\n</tool_response>' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n<think>' }}\n{%- endif %}\n"
15
16kd_trainer: true
17kd_ce_alpha: 0.2
18kd_alpha: 0.8
19kd_temperature: 1.1
20
21dataloader_prefetch_factor: 256
22dataloader_num_workers: 4
23dataloader_pin_memory: true
24
25gc_steps: -1 # gc at the end of each epoch
26
27datasets:
28- field_messages: messages
29 message_field_content: content
30 message_field_role: role
31 logprobs_field: target_logprobs
32 path: winglian/codeforces-cot-16k-context-topk64-prepared
33 type: axolotl.integrations.kd.chat_template
34 split: train
35 temperature: 1.0
36
37dataset_prepared_path: last_run_prepared
38val_set_size: 0.0
39output_dir: ./outputs/out-kd-7b
40skip_prepare_dataset: false
41
42sequence_len: 16384
43sample_packing: true
44pad_to_sequence_len: true
45
46wandb_project: kd-7b-codeforces
47wandb_entity: axolotl-ai
48wandb_watch:
49wandb_name:
50wandb_log_model:
51
52gradient_accumulation_steps: 2
53micro_batch_size: 4
54num_epochs: 4
55optimizer: adamw_torch_fused
56lr_scheduler: rex
57learning_rate: 4e-5
58save_safetensors: true
59
60train_on_inputs: false
61group_by_length: false
62bf16: true
63fp16:
64tf32: true
65
66gradient_checkpointing: true
67gradient_checkpointing_kwargs:
68 use_reentrant: false
69early_stopping_patience:
70resume_from_checkpoint:
71logging_steps: 1
72xformers_attention:
73flash_attention: true
74
75warmup_steps: 120
76evals_per_epoch:
77eval_table_size:
78saves_per_epoch: 1
79debug:
80weight_decay: 0.0
81special_tokens:
82 pad_token: <|endoftext|>
83deepspeed: deepspeed_configs/zero2.json
84