See oumi train config
oumi version: 0.1.3
1data:
2 train:
3 datasets:
4 - dataset_name: HuggingFaceH4/ultrachat_200k
5 dataset_path: null
6 subset: null
7 split: train_sft
8 dataset_kwargs: {}
9 sample_count: null
10 mixture_proportion: null
11 shuffle: false
12 seed: null
13 shuffle_buffer_size: 1000
14 trust_remote_code: true
15 transform_num_workers: null
16 collator_name: null
17 pack: false
18 stream: false
19 target_col: null
20 mixture_strategy: first_exhausted
21 seed: null
22 use_async_dataset: false
23 use_torchdata: null
24 test:
25 datasets: []
26 collator_name: null
27 pack: false
28 stream: false
29 target_col: null
30 mixture_strategy: first_exhausted
31 seed: null
32 use_async_dataset: false
33 use_torchdata: null
34 validation:
35 datasets: []
36 collator_name: null
37 pack: false
38 stream: false
39 target_col: null
40 mixture_strategy: first_exhausted
41 seed: null
42 use_async_dataset: false
43 use_torchdata: null
44model:
45 model_name: meta-llama/Meta-Llama-3.1-8B
46 adapter_model: null
47 tokenizer_name: null
48 tokenizer_pad_token: null
49 tokenizer_kwargs: {}
50 model_max_length: 8192
51 load_pretrained_weights: true
52 trust_remote_code: true
53 torch_dtype_str: bfloat16
54 compile: false
55 chat_template: llama3-instruct
56 attn_implementation: flash_attention_2
57 device_map: auto
58 model_kwargs: {}
59 enable_liger_kernel: true
60 shard_for_eval: false
61 freeze_layers: []
62training:
63 use_peft: false
64 trainer_type: TRL_SFT
65 enable_gradient_checkpointing: true
66 gradient_checkpointing_kwargs:
67 use_reentrant: false
68 output_dir: output/llama8b-ultrachat
69 per_device_train_batch_size: 1
70 per_device_eval_batch_size: 8
71 gradient_accumulation_steps: 8
72 max_steps: -1
73 num_train_epochs: 1
74 save_epoch: false
75 save_steps: 800
76 save_final_model: true
77 seed: 42
78 run_name: llama8b-ultrachat.sky-2025-01-30-21-19-10-053582_sky-e018-bf996_1
79 metrics_function: null
80 log_level: info
81 dep_log_level: warning
82 enable_wandb: true
83 enable_tensorboard: true
84 logging_strategy: steps
85 logging_dir: null
86 logging_steps: 100
87 logging_first_step: false
88 eval_strategy: 'no'
89 eval_steps: 500
90 learning_rate: 2.0e-05
91 lr_scheduler_type: linear
92 lr_scheduler_kwargs: {}
93 warmup_ratio: null
94 warmup_steps: null
95 optimizer: paged_adamw_8bit
96 weight_decay: 0.0
97 adam_beta1: 0.9
98 adam_beta2: 0.999
99 adam_epsilon: 1.0e-08
100 sgd_momentum: 0.0
101 mixed_precision_dtype: NONE
102 compile: false
103 include_performance_metrics: true
104 include_alternative_mfu_metrics: false
105 log_model_summary: false
106 resume_from_checkpoint: null
107 try_resume_from_last_checkpoint: false
108 dataloader_num_workers: 8
109 dataloader_prefetch_factor: 32
110 dataloader_main_process_only: null
111 ddp_find_unused_parameters: false
112 max_grad_norm: 1.0
113 trainer_kwargs:
114 max_seq_length: 8192
115 profiler:
116 save_dir: null
117 enable_cpu_profiling: false
118 enable_cuda_profiling: false
119 record_shapes: false
120 profile_memory: false
121 with_stack: false
122 with_flops: false
123 with_modules: false
124 row_limit: 50
125 schedule:
126 enable_schedule: false
127 wait: 0
128 warmup: 1
129 active: 3
130 repeat: 1
131 skip_first: 1
132 telemetry:
133 telemetry_dir: telemetry
134 collect_telemetry_for_all_ranks: false
135 track_gpu_temperature: false
136 empty_device_cache_steps: 50
137 nccl_default_timeout_minutes: null
138peft:
139 lora_r: 8
140 lora_alpha: 8
141 lora_dropout: 0.0
142 lora_target_modules: null
143 lora_modules_to_save: null
144 lora_bias: none
145 lora_init_weights: DEFAULT
146 lora_task_type: CAUSAL_LM
147 q_lora: false
148 q_lora_bits: 4
149 bnb_4bit_quant_type: fp4
150 use_bnb_nested_quant: false
151 bnb_4bit_quant_storage: uint8
152 bnb_4bit_compute_dtype: float32
153 peft_save_mode: ADAPTER_ONLY
154fsdp:
155 enable_fsdp: false
156 sharding_strategy: FULL_SHARD
157 cpu_offload: false
158 mixed_precision: null
159 backward_prefetch: BACKWARD_PRE
160 forward_prefetch: false
161 use_orig_params: null
162 state_dict_type: FULL_STATE_DICT
163 auto_wrap_policy: NO_WRAP
164 min_num_params: 100000
165 transformer_layer_cls: null
166 sync_module_states: true
See oumi cloud config
1name: llama8b-ultrachat-sft
2
3num_nodes: 1
4resources:
5 cloud: gcp
6 accelerators: "A100-80GB:4"
7 use_spot: false
8 disk_size: 2000 # Disk size in GBs
9
10working_dir: .
11
12file_mounts:
13 ~/.netrc: ~/.netrc # WandB credentials
14 # Mount HF token, which is needed to download locked-down models from HF Hub.
15 # This is created on the local machine by running `huggingface-cli login`.
16 ~/.cache/huggingface/token: ~/.cache/huggingface/token
17
18envs:
19 WANDB_PROJECT: oumi-train
20 OUMI_RUN_NAME: llama8b-ultrachat
21 OUMI_USER_NAME: penfever
22 ACCELERATE_LOG_LEVEL: info
23 # https://github.com/huggingface/tokenizers/issues/899#issuecomment-1027739758
24 TOKENIZERS_PARALLELISM: false
25setup: |
26 set -e
27 pip install uv && uv pip install -e .[gpu,evaluation] hf_transfer
28 # Install model from HF Hub. This tool increases download speed compared to
29 # downloading the model during training.
30 HF_HUB_ENABLE_HF_TRANSFER=1 huggingface-cli download meta-llama/Meta-Llama-3.1-8B --exclude original/*
31 pip install -U flash-attn --no-build-isolation
32
33run: |
34 set -e # Exit if any command failed.
35 source ./configs/examples/misc/sky_init.sh
36
37 set -x
38 oumi distributed torchrun \
39 -m oumi train \
40 -c configs/recipes/llama3_1/sft/8b_full/base_ultrachat.yaml \
41 --training.run_name "${OUMI_RUN_NAME}.${SKYPILOT_TASK_ID}" \
42
43 echo "Node ${SKYPILOT_NODE_RANK} is all done!"
Llama-3-8B-UltraChat-200K-Oumi
This model is a fine-tuned version of
meta-llama/Meta-Llama-3.1-8B on the HuggingFaceH4/ultrachat_200k dataset. It achieves a training loss of 1.0435.
Model description
This model was trained as a partial reproduction of results from the recent
WildChat-50M paper.
1@misc{feuer2025wildchat50mdeepdiverole,
2 title={WILDCHAT-50M: A Deep Dive Into the Role of Synthetic Data in Post-Training},
3 author={Benjamin Feuer and Chinmay Hegde},
4 year={2025},
5 eprint={2501.18511},
6 archivePrefix={arXiv},
7 primaryClass={cs.LG},
8 url={https://arxiv.org/abs/2501.18511},
9}
Intended uses & limitations
This model is intended for research use; it has not received any safety oriented post-training.
Artifacts
The following is a list of artifacts which may be present in this repository, as well as brief descriptions of what they contain.
Logs
Contains logs from the training process, one for each rank.
Telemetry
devices_info.txt: A file containing information about the devices used to train the model.
telemetry_callback_metrics.json: File containing metrics from the training process such as loss and number of tokens seen.
telemetry_callback_wandb.json: File containing weights and biases parameters.
telemetry_callback.json: File containing metadata such as time to train and number of epochs trained.
training_config.yaml: File containing the training configuration used to train the model (also found in this README)
world_size.json: File containing the world size used to train the model.
Datasets
Summary statistics about the datasets used to train this model.
HuggingFaceH4/ultrachat_200k
Split: train_sft
Version: 0.0.0
Dataset size: 3047427114 bytes
Download size: 1624049723 bytes
Size: 4671476837 bytes
Rows: 207865
Columns: ['prompt', 'prompt_id', 'messages']
Results
Training Loss
| Training Loss | Epoch | Tokens Seen |
|---|
| 1.043 | 0.999 | 246 Mn |
Evaluation
Following the paper, our benchmark results are reported using
Evalchemy. For more details on the evaluation metrics, please refer to the
paper. We compare to
this baseline model used in the paper.
| Metric | Oumi Repro | Baseline |
|---|
| MTBench | 5.2313 | 5.0187 |
| Alpaca Eval (LC) | 1.6157 | 4.1260 |
| BBH | 0.4861 | 0.4845 |
| GPQA | 0.2903 | 0.3204 |
| MATH | 0.0552 | 0.0458 |
| MUSR | 0.4116 | 0.3917 |
| IFEval (Prompt Level, Strict) | 0.1978 | 0.2643 |
| MMLU Pro | 0.3118 | 0.3198 |
| MixEval | 0.5935 | 0.63 |
| Average | 0.321 | 0.333 |
