1# This is an axolotl config that allowed creation of a model knowledgeable about Verus.
2# Replace the dataset paths under `datasets:` with your own
3# If you want a reference point of what kind of data was fed into this model, check out Verustoolkit https://github.com/e-p-armstrong/verustoolkit.git
4
5# Rent a GPU with a compute provider like Vast.ai or Runpod
6# (Make sure it is using the axolotl docker image --- winglian/axolotl:main-latest)
7# Copy this file over to the rented instance, in the /workspace/axolotl directory
8# If running on a single-GPU setup, you must run:
9# conda install -c conda-forge mpi4py mpich
10# Then run this command from the /workspace/axolotl directory:
11# accelerate launch --use_deepspeed -m axolotl.cli.train axolotl_config_verus_llama3_Jun_9_2024.yaml
12
13# If using GaLore, do not use deepspeed
14
15# (to copy files over to a rented GPU instance, you'll have to use SSH to Secure CoPy files over from your machine to the rented one. This is what such a command might look like, adapt it to your needs)
16# scp -P 40001 -r ./ root@173.231.62.170:/workspace/axolotl/
17
18base_model: meta-llama/Llama-3.2-1B
19model_type: LlamaForCausalLM
20tokenizer_type: AutoTokenizer
21
22load_in_8bit: false
23load_in_4bit: false
24strict: false
25
26datasets:
27 - path: json
28 data_files: pretraining.jsonl
29 ds_type: json
30 type: completion
31 - path: json
32 data_files: simplified_data_no_rag.jsonl
33 ds_type: json
34 type: chat_template
35 chat_template: chatml
36 field_messages: conversations
37 message_field_role: from
38 message_field_content: value
39 roles:
40 user:
41 - human
42 assistant:
43 - gpt
44 system:
45 - system
46 - path: json
47 data_files: multi_turn_convs_DATAGEN_OUTPUT.jsonl
48 ds_type: json
49 type: chat_template
50 chat_template: chatml
51 field_messages: conversations
52 message_field_role: from
53 message_field_content: value
54 roles:
55 user:
56 - human
57 assistant:
58 - gpt
59 system:
60 - system
61 - path: json
62 data_files: judge_paragraph_generations_DATAGEN_OUTPUT.jsonl
63 ds_type: json
64 type: chat_template
65 chat_template: chatml
66 field_messages: conversations
67 message_field_role: from
68 message_field_content: value
69 roles:
70 user:
71 - human
72 assistant:
73 - gpt
74 system:
75 - system
76
77dataset_prepared_path: last_run_prepared
78output_dir: ./testmodelout
79
80sequence_len: 4500
81sample_packing: true
82pad_to_sequence_len: true
83
84wandb_project: testmodelrun
85wandb_entity:
86wandb_watch:
87wandb_run_id:
88wandb_log_model:
89
90gradient_accumulation_steps: 2
91micro_batch_size: 1
92num_epochs: 6
93optimizer: paged_adamw_8bit
94lr_scheduler: cosine
95learning_rate: 2e-4
96noisy_embedding_alpha: 0 # no noisy embedding to ensure maximal memorization
97
98train_on_inputs: false
99group_by_length: false
100bf16: true
101fp16: false
102tf32: false
103
104gradient_checkpointing: unsloth
105early_stopping_patience:
106resume_from_checkpoint:
107logging_steps: 1
108xformers_attention:
109flash_attention: true
110
111chat_template: chatml
112
113warmup_steps: 100
114auto_resume_from_checkpoints: false
115eval_steps: 10
116saves_per_epoch: 1
117eval_sample_packing: false
118save_total_limit: 4
119debug:
120deepspeed: deepspeed_configs/zero2.json
121special_tokens:
122 pad_token: "<|end_of_text|>"
123
