Views
No views yet
ibm-granite/granite-3.1-3b-a800m-instruct was trained at 8K with batch size 2 gradient accumulation 8, so each step was 131,072 tokens (including any padding tokens). It was trained for 400 steps, adding up to a total of 52,428,800 unique tokens seen.<|start_of_role|>system<|end_of_role|>example system prompt<|end_of_text|>
<|start_of_role|>user<|end_of_role|>example user turn 1<|end_of_text|>
<|start_of_role|>assistant<|end_of_role|>example assistant turn 1<|end_of_text|>
<|start_of_role|>user<|end_of_role|>example user turn 2<|end_of_text|>
<|start_of_role|>assistant<|end_of_role|>example assistant turn 2<|end_of_text|>
1# Requirements before running
2# - Get latest commit of axolotl (currently c0a0c75)
3# - Download these to axolotl/src/axolotl/prompt_formatters
4# - https://github.com/xzuyn/axolotl/blob/came-plus-formatters/src/axolotl/prompt_strategies/formatter_regex.py
5# - https://github.com/xzuyn/axolotl/blob/came-plus-formatters/src/axolotl/prompt_strategies/customcompletion-regex.py
6# - https://github.com/xzuyn/axolotl/blob/came-plus-formatters/src/axolotl/prompt_strategies/customgranite-regex.py
7# - pip install ftfy
8# - pip install git+https://github.com/xzuyn/CAME.git@sr-grams-cautious-8bit
9
10# Weights and Biases logging config
11wandb_project: Granite-3.1-3B-A800M
12wandb_name: Granite-3.1-Earthen-v0.3-3B-A800M-QLoRA-run4
13
14# Model checkpointing config
15output_dir: ./Outputs/Granite-3.1-Earthen-v0.3-3B-A800M-QLoRA-run4
16resume_from_checkpoint:
17save_steps: 10
18save_safetensors: true
19save_total_limit: 2
20save_only_model: false
21
22# Model architecture config
23base_model: ibm-granite/granite-3.1-3b-a800m-instruct
24model_type: AutoModelForCausalLM
25tokenizer_type: AutoTokenizer
26
27# Mixed precision training config
28bf16: true
29fp16: false
30tf32: false
31
32# Model loading config
33load_in_8bit: false
34load_in_4bit: true
35strict: false
36
37# Sequence config
38sequence_len: 8192
39min_sample_len: 256
40sample_packing: true
41eval_sample_packing: true
42pad_to_sequence_len: true
43train_on_inputs: false
44group_by_length: false
45
46# LoRA adapter config
47adapter: qlora
48lora_r: 128
49lora_alpha: 128
50lora_dropout: 0.125
51lora_target_linear: true
52embeddings_skip_upcast: true
53
54# Dataset config
55datasets:
56# Completion
57 # Story-like Data
58 - path: BeaverAI/REDACTED1
59 split: train[:4000]
60 type: customcompletion-regex
61 - path: PJMixers-Dev/Lit-axo-Shuffled
62 split: train[:4000]
63 type: customcompletion-regex
64 - path: PJMixers-Dev/Mielikki_Erebus-87k-axo
65 split: train[:4000]
66 type: customcompletion-regex
67 - path: PJMixers/RyokoAI_Honeyfeed3600-Cleanish
68 split: train[:4000]
69 type: customcompletion-regex
70 - path: BeaverAI/REDACTED2
71 type: customcompletion-regex
72 - path: PJMixers-Dev/allura-org_fujin-cleaned-stage-2-axo
73 split: train[:4000]
74 type: customcompletion-regex
75 - path: Nelathan/synthetic-sugar-quill
76 split: train[:4000]
77 type: customcompletion-regex
78 - path: PJMixers-Dev/winglian_visual-novels-json-axo-dropped-long
79 split: train[:4000]
80 type: customcompletion-regex
81 - path: BeaverAI/REDACTED3
82 type: customcompletion-regex
83 - path: PJMixers-Dev/recursal_SCP-RECURSAL-Cleaned
84 split: train[:4000]
85 type: customcompletion-regex
86 # Subtitle Data
87 - path: PJMixers-Dev/Subtitles
88 type: customcompletion-regex
89 - path: PJMixers-Dev/KaraKaraWitch_AnimeSubtitle-axo
90 split: train[:4000]
91 type: customcompletion-regex
92 # News Data
93 - path: PJMixers/AP-News-2024
94 type: customcompletion-regex
95 - path: PJMixers-Dev/Fundus-AP-News-Formatted
96 split: train[:4000]
97 type: customcompletion-regex
98 - path: PJMixers-Dev/Fundus-AP-News-2-Formatted
99 type: customcompletion-regex
100 # Misc Data
101 - path: PJMixers-Dev/goodwiki-2024-12-04-axo
102 split: train[:4000]
103 type: customcompletion-regex
104 - path: epfl-llm/guidelines
105 split: train[:4000]
106 field: clean_text
107 type: customcompletion-regex
108# Granite-3.1 Instruct
109 # Instruction Data
110 - path: PJMixers-Dev/allenai_tulu-3-sft-mixture-filtered-2-ShareGPT
111 split: train[:4000]
112 type: customgranite-regex
113 - path: OpenLeecher/lmsys_chat_1m_clean
114 split: train[:4000]
115 type: customgranite-regex
116 # RP Data
117 - path: PJMixers-Dev/Gryphe-Aesir-RPG-Charcards-Opus-Mixed
118 type: customgranite-regex
119 - path: allura-org/gryphe-sonnet-3.5-charcards-names-added
120 type: customgranite-regex
121 - path: anthracite-org/c2_logs_32k_llama3_qwen2_v1.3
122 type: customgranite-regex
123 - path: BeaverAI/REDACTED4
124 type: customgranite-regex
125 - path: PJMixers-Dev/MinervaAI_Aesir-Preview-Anon
126 type: customgranite-regex
127 - path: PJMixers-Dev/lemonilia_LimaRP-Simple-CustomShareGPT-Shuffled
128 type: customgranite-regex
129 - path: Epiculous/SynthRP-Gens-v1.1-Filtered-n-Cleaned
130 type: customgranite-regex
131 - path: PJMixers-Dev/NyxKrage_chub-logs-sharegpt-longest-CustomShareGPT
132 type: customgranite-regex
133 - path: PJMixers/OpenLeecher_Teatime_all_logs_longest-ShareGPT
134 type: customgranite-regex
135 - path: grimulkan/aicg-logs-augmented
136 type: customgranite-regex
137 - path: grimulkan/PIPPA-augmented-dedup
138 type: customgranite-regex
139 - path: PJMixers/grimulkan_bluemoon_Karen_cleaned-carded-formatted
140 type: customgranite-regex
141 # InstStory Data
142 - path: PJMixers/lodrick-the-lafted_OpusStories-ShareGPT
143 type: customgranite-regex
144 - path: Gryphe/ChatGPT-4o-Writing-Prompts
145 type: customgranite-regex
146 - path: Gryphe/Opus-WritingPrompts
147 type: customgranite-regex
148 - path: anthracite-org/nopm_claude_writing_fixed
149 type: customgranite-regex
150 - path: PJMixers-Dev/Tiefighter-13B-Fake-Distill-ShareGPT
151 type: customgranite-regex
152 - path: allura-org/fujin-instruct-v2
153 type: customgranite-regex
154 - path: ToastyPigeon/gutenberg-sft
155 type: customgranite-regex
156 # Adventure Data
157 - path: PocketDoc/Dans-Prosemaxx-Adventure
158 type: customgranite-regex
159 - path: PocketDoc/Dans-Failuremaxx-Adventure-3
160 type: customgranite-regex
161 # Decensoring Data
162 - path: TheDrummer/AmoralQA-v2
163 type: customgranite-regex
164 - path: BeaverAI/REDACTED5
165 type: customgranite-regex
166 - path: BeaverAI/REDACTED6
167 type: customgranite-regex
168val_set_size: 256
169eval_strategy: steps
170eval_steps: 10
171dataset_prepared_path: ./00-Tokenized-Datasets/Granite-3.1-Earthen-v0.3-3B-A800M-LoRA-seed42
172shuffle_merged_datasets: true
173
174# Training hyperparameters
175num_epochs: 1
176gradient_accumulation_steps: 8
177micro_batch_size: 2
178eval_batch_size: 2
179warmup_steps: 0
180optimizer: came_pytorch
181optim_args:
182 enable_stochastic_rounding: true
183 enable_cautious: true
184 enable_8bit: true
185lr_scheduler: rex
186learning_rate: 2.5e-7
187cosine_min_lr_ratio: 0.05
188weight_decay: 0.01
189max_grad_norm: 0.5
190logging_steps: 1
191
192# Model optimization
193gradient_checkpointing: offload
194sdp_attention: true
195plugins:
196 - axolotl.integrations.liger.LigerPlugin
197liger_rope: true
198liger_rms_norm: true
199liger_layer_norm: true
200liger_glu_activation: true
201liger_cross_entropy: true
202lora_mlp_kernel: false
203lora_qkv_kernel: false
204lora_o_kernel: false
205
206# Debug config
207debug: true
208seed: 42
209
210# Token config
211special_tokens:
212 bos_token: "<|end_of_text|>"
213 eos_token: "<|end_of_text|>"
214 pad_token: "<|end_of_text|>"
215tokens:1@misc{wolf2020huggingfacestransformersstateoftheartnatural,
2 title={HuggingFace's Transformers: State-of-the-art Natural Language Processing},
3 author={Thomas Wolf and Lysandre Debut and Victor Sanh and Julien Chaumond and Clement Delangue and Anthony Moi and Pierric Cistac and Tim Rault and Rémi Louf and Morgan Funtowicz and Joe Davison and Sam Shleifer and Patrick von Platen and Clara Ma and Yacine Jernite and Julien Plu and Canwen Xu and Teven Le Scao and Sylvain Gugger and Mariama Drame and Quentin Lhoest and Alexander M. Rush},
4 year={2020},
5 eprint={1910.03771},
6 archivePrefix={arXiv},
7 primaryClass={cs.CL},
8 url={https://arxiv.org/abs/1910.03771},
9}
10@misc{hu2021loralowrankadaptationlarge,
11 title={LoRA: Low-Rank Adaptation of Large Language Models},
12 author={Edward J. Hu and Yelong Shen and Phillip Wallis and Zeyuan Allen-Zhu and Yuanzhi Li and Shean Wang and Lu Wang and Weizhu Chen},
13 year={2021},
14 eprint={2106.09685},
15 archivePrefix={arXiv},
16 primaryClass={cs.CL},
17 url={https://arxiv.org/abs/2106.09685},
18}
19@misc{dettmers2023qloraefficientfinetuningquantized,
20 title={QLoRA: Efficient Finetuning of Quantized LLMs},
21 author={Tim Dettmers and Artidoro Pagnoni and Ari Holtzman and Luke Zettlemoyer},
22 year={2023},
23 eprint={2305.14314},
24 archivePrefix={arXiv},
25 primaryClass={cs.LG},
26 url={https://arxiv.org/abs/2305.14314},
27}
28@misc{dao2023flashattention2fasterattentionbetter,
29 title={FlashAttention-2: Faster Attention with Better Parallelism and Work Partitioning},
30 author={Tri Dao},
31 year={2023},
32 eprint={2307.08691},
33 archivePrefix={arXiv},
34 primaryClass={cs.LG},
35 url={https://arxiv.org/abs/2307.08691},
36}
37@misc{hsu2024ligerkernelefficienttriton,
38 title={Liger Kernel: Efficient Triton Kernels for LLM Training},
39 author={Pin-Lun Hsu and Yun Dai and Vignesh Kothapalli and Qingquan Song and Shao Tang and Siyu Zhu and Steven Shimizu and Shivam Sahni and Haowen Ning and Yanning Chen},
40 year={2024},
41 eprint={2410.10989},
42 archivePrefix={arXiv},
43 primaryClass={cs.LG},
44 url={https://arxiv.org/abs/2410.10989},
45}
46@misc{chen2021rexrevisitingbudgetedtraining,
47 title={REX: Revisiting Budgeted Training with an Improved Schedule},
48 author={John Chen and Cameron Wolfe and Anastasios Kyrillidis},
49 year={2021},
50 eprint={2107.04197},
51 archivePrefix={arXiv},
52 primaryClass={cs.LG},
53 url={https://arxiv.org/abs/2107.04197},
54}
55@misc{luo2023cameconfidenceguidedadaptivememory,
56 title={CAME: Confidence-guided Adaptive Memory Efficient Optimization},
57 author={Yang Luo and Xiaozhe Ren and Zangwei Zheng and Zhuo Jiang and Xin Jiang and Yang You},
58 year={2023},
59 eprint={2307.02047},
60 archivePrefix={arXiv},
61 primaryClass={cs.CL},
62 url={https://arxiv.org/abs/2307.02047},
63}
64@misc{zamirai2021revisitingbfloat16training,
65 title={Revisiting BFloat16 Training},
66 author={Pedram Zamirai and Jian Zhang and Christopher R. Aberger and Christopher De Sa},
67 year={2021},
68 eprint={2010.06192},
69 archivePrefix={arXiv},
70 primaryClass={cs.LG},
71 url={https://arxiv.org/abs/2010.06192},
72}
73@misc{liang2025cautiousoptimizersimprovingtraining,
74 title={Cautious Optimizers: Improving Training with One Line of Code},
75 author={Kaizhao Liang and Lizhang Chen and Bo Liu and Qiang Liu},
76 year={2025},
77 eprint={2411.16085},
78 archivePrefix={arXiv},
79 primaryClass={cs.LG},
80 url={https://arxiv.org/abs/2411.16085},
81}
82@misc{xie2025sana15efficientscaling,
83 title={SANA 1.5: Efficient Scaling of Training-Time and Inference-Time Compute in Linear Diffusion Transformer},
84 author={Enze Xie and Junsong Chen and Yuyang Zhao and Jincheng Yu and Ligeng Zhu and Chengyue Wu and Yujun Lin and Zhekai Zhang and Muyang Li and Junyu Chen and Han Cai and Bingchen Liu and Daquan Zhou and Song Han},
85 year={2025},
86 eprint={2501.18427},
87 archivePrefix={arXiv},
88 primaryClass={cs.CV},
89 url={https://arxiv.org/abs/2501.18427},
90}
91@misc{dallabetta2024fundussimpletousenewsscraper,
92 title={Fundus: A Simple-to-Use News Scraper Optimized for High Quality Extractions},
93 author={Max Dallabetta and Conrad Dobberstein and Adrian Breiding and Alan Akbik},
94 year={2024},
95 eprint={2403.15279},
96 archivePrefix={arXiv},
97 primaryClass={cs.CL},
98 url={https://arxiv.org/abs/2403.15279},
99}
100@misc{lambert2025tulu3pushingfrontiers,
101 title={Tulu 3: Pushing Frontiers in Open Language Model Post-Training},
102 author={Nathan Lambert and Jacob Morrison and Valentina Pyatkin and Shengyi Huang and Hamish Ivison and Faeze Brahman and Lester James V. Miranda and Alisa Liu and Nouha Dziri and Shane Lyu and Yuling Gu and Saumya Malik and Victoria Graf and Jena D. Hwang and Jiangjiang Yang and Ronan Le Bras and Oyvind Tafjord and Chris Wilhelm and Luca Soldaini and Noah A. Smith and Yizhong Wang and Pradeep Dasigi and Hannaneh Hajishirzi},
103 year={2025},
104 eprint={2411.15124},
105 archivePrefix={arXiv},
106 primaryClass={cs.CL},
107 url={https://arxiv.org/abs/2411.15124},
108}
109@misc{zheng2024lmsyschat1mlargescalerealworldllm,
110 title={LMSYS-Chat-1M: A Large-Scale Real-World LLM Conversation Dataset},
111 author={Lianmin Zheng and Wei-Lin Chiang and Ying Sheng and Tianle Li and Siyuan Zhuang and Zhanghao Wu and Yonghao Zhuang and Zhuohan Li and Zi Lin and Eric P. Xing and Joseph E. Gonzalez and Ion Stoica and Hao Zhang},
112 year={2024},
113 eprint={2309.11998},
114 archivePrefix={arXiv},
115 primaryClass={cs.CL},
116 url={https://arxiv.org/abs/2309.11998},
117}
118@misc{gosling2023pippapartiallysyntheticconversational,
119 title={PIPPA: A Partially Synthetic Conversational Dataset},
120 author={Tear Gosling and Alpin Dale and Yinhe Zheng},
121 year={2023},
122 eprint={2308.05884},
123 archivePrefix={arXiv},
124 primaryClass={cs.CL},
125 url={https://arxiv.org/abs/2308.05884},
126}