# git clone https://github.com/axolotl-ai-cloud/axolotl
# cd axolotl
# git checkout 844331005c1ef45430ff26b9f42f757dce6ee66a
# pip3 install packaging ninja huggingface_hub[cli]
# pip3 install -e '.[flash-attn,deepspeed]'
# huggingface-cli login --token $hf_key && wandb login $wandb_key
# python -m axolotl.cli.preprocess nemo-rp-test-2.yml
# accelerate launch -m axolotl.cli.train nemo-rp-test-2.yml
# python -m axolotl.cli.merge_lora nemo-rp-test.yml
# huggingface-cli upload Columbidae/nemo-rp-test train-workspace/merged . --exclude "*.md"

# git clone https://github.com/axolotl-ai-cloud/axolotl && cd axolotl && pip3 install packaging ninja huggingface_hub[cli] && pip3 install -e '.[flash-attn,deepspeed]' && cd .. && huggingface-cli login --token $hf_key && wandb login $wandb_key

# Model
base_model: unsloth/Mistral-Small-Instruct-2409
model_type: AutoModelForCausalLM
tokenizer_type: AutoTokenizer

load_in_8bit: false
load_in_4bit: true
strict: false
bf16: true
fp16:
tf32: false
flash_attention: true
special_tokens:

# Output
output_dir: ./train-workspace
hub_model_id: ToastyPigeon/ms-rp-test-ws
hub_strategy: "all_checkpoints"
resume_from_checkpoint:
saves_per_epoch: 5

# Data
sequence_len: 16384 # fits
min_sample_len: 128
dataset_prepared_path: last_run_prepared
datasets:
  - path: ToastyPigeon/some-rp
    type: chat_template
    field_messages: conversations
    message_field_role: from
    message_field_content: value
train_on_inputs: true
warmup_steps: 20
shuffle_merged_datasets: true
sample_packing: true
pad_to_sequence_len: true
chat_template: jinja
chat_template_jinja: "{%- if messages[0][\"role\"] == \"fake\" %}\n    {%- set system_message = messages[0][\"content\"] %}\n    {%- set loop_messages = messages[1:] %}\n{%- else %}\n    {%- set loop_messages = messages %}\n{%- endif %}\n{%- if not tools is defined %}\n    {%- set tools = none %}\n{%- endif %}\n{%- set user_messages = loop_messages | selectattr(\"role\", \"equalto\", \"user\") | list %}\n\n{{- bos_token }}\n{%- for message in loop_messages %}\n    {%- if message[\"role\"] == \"user\" or message[\"role\"] == \"system\"%}\n        {%- if tools is not none and (message == user_messages[-1]) %}\n            {{- \"[AVAILABLE_TOOLS][\" }}\n            {%- for tool in tools %}\n                {%- set tool = tool.function %}\n                {{- '{\"type\": \"function\", \"function\": {' }}\n                {%- for key, val in tool.items() if key != \"return\" %}\n                    {%- if val is string %}\n                        {{- '\"' + key + '\": \"' + val + '\"' }}\n                    {%- else %}\n                        {{- '\"' + key + '\": ' + val|tojson }}\n                    {%- endif %}\n                    {%- if not loop.last %}\n                        {{- \", \" }}\n                    {%- endif %}\n                {%- endfor %}\n                {{- \"}}\" }}\n                {%- if not loop.last %}\n                    {{- \", \" }}\n                {%- else %}\n                    {{- \"]\" }}\n                {%- endif %}\n            {%- endfor %}\n            {{- \"[/AVAILABLE_TOOLS]\" }}\n            {%- endif %}\n        {%- if loop.last and system_message is defined %}\n            {{- \"[INST] \" + system_message + \"\\n\\n\" + message[\"content\"] + \"[/INST] \" }}\n        {%- else %}\n            {{- \"[INST] \" + message[\"content\"] + \"[/INST] \" }}\n        {%- endif %}\n    {%- elif (message.tool_calls is defined and message.tool_calls is not none) %}\n        {{- \"[TOOL_CALLS][\" }}\n        {%- for tool_call in message.tool_calls %}\n            {%- set out = tool_call.function|tojson %}\n            {{- out[:-1] }}\n            {%- if not tool_call.id is defined or tool_call.id|length != 9 %}\n                {{- raise_exception(\"Tool call IDs should be alphanumeric strings with length 9!\") }}\n            {%- endif %}\n            {{- ', \"id\": \"' + tool_call.id + '\"}' }}\n            {%- if not loop.last %}\n                {{- \", \" }}\n            {%- else %}\n                {{- \"]\" + eos_token }}\n            {%- endif %}\n        {%- endfor %}\n    {%- elif message[\"role\"] == \"assistant\" %}\n        {{- message[\"content\"] + eos_token}}\n    {%- elif message[\"role\"] == \"tool_results\" or message[\"role\"] == \"tool\" %}\n        {%- if message.content is defined and message.content.content is defined %}\n            {%- set content = message.content.content %}\n        {%- else %}\n            {%- set content = message.content %}\n        {%- endif %}\n        {{- '[TOOL_RESULTS]{\"content\": ' + content|string + \", \" }}\n        {%- if not message.tool_call_id is defined or message.tool_call_id|length != 9 %}\n            {{- raise_exception(\"Tool call IDs should be alphanumeric strings with length 9!\") }}\n        {%- endif %}\n        {{- '\"call_id\": \"' + message.tool_call_id + '\"}[/TOOL_RESULTS]' }}\n    {%- else %}\n        {{- raise_exception(\"Only user and assistant roles are supported, with the exception of an initial optional system message!\") }}\n    {%- endif %}\n{%- endfor %}\n"

# Batching
num_epochs: 1
gradient_accumulation_steps: 1
micro_batch_size: 1
eval_batch_size: 1

# Evaluation
val_set_size: 50
evals_per_epoch: 10
eval_table_size:
eval_max_new_tokens: 256
eval_sample_packing: false

save_safetensors: true

# WandB
wandb_project: Nemo-Rp-Test
#wandb_entity:

gradient_checkpointing: 'unsloth'
gradient_checkpointing_kwargs:
  use_reentrant: false

unsloth_cross_entropy_loss: true
#unsloth_lora_mlp: true
#unsloth_lora_qkv: true
#unsloth_lora_o: true

# LoRA
adapter: qlora
lora_model_dir:
lora_r: 32
lora_alpha: 64
lora_dropout: 0.25
lora_target_linear: 
lora_fan_in_fan_out:
lora_target_modules:
  - gate_proj
  - down_proj
  - up_proj
  - q_proj
  - v_proj
  - k_proj
  - o_proj
lora_modules_to_save:

# Optimizer
optimizer: paged_ademamix_8bit # adamw_8bit
lr_scheduler: cosine
learning_rate: 5e-5
cosine_min_lr_ratio: 0.1
weight_decay: 0.01
max_grad_norm: 1.0

# Misc
#train_on_inputs: false
group_by_length: false
early_stopping_patience:
local_rank:
logging_steps: 1
xformers_attention:
debug:
deepspeed: /workspace/axolotl/deepspeed_configs/zero3_bf16.json # previously blank
fsdp:
fsdp_config:

plugins:
  - axolotl.integrations.liger.LigerPlugin
liger_rope: true
liger_rms_norm: true
liger_layer_norm: true
liger_glu_activation: true
liger_fused_linear_cross_entropy: true

gc_steps: 10
seed: 69

ms-rp-test-ws

This model is a fine-tuned version of unsloth/Mistral-Small-Instruct-2409 on the ToastyPigeon/some-rp dataset. It achieves the following results on the evaluation set:

Loss: 2.0036

Model description

More information needed

Intended uses & limitations

More information needed

Training and evaluation data

More information needed

Training procedure

Training hyperparameters

The following hyperparameters were used during training:

learning_rate: 5e-05
train_batch_size: 1
eval_batch_size: 1
seed: 69
distributed_type: multi-GPU
num_devices: 4
total_train_batch_size: 4
total_eval_batch_size: 4
optimizer: Use OptimizerNames.PAGED_ADEMAMIX_8BIT and the args are: No additional optimizer arguments
lr_scheduler_type: cosine
lr_scheduler_warmup_steps: 20
num_epochs: 1

Training results

Training Loss	Epoch	Step	Validation Loss
2.1185	0.0034	1	2.0953
2.055	0.1014	30	2.0229
2.1407	0.2027	60	2.0158
2.0997	0.3041	90	2.0111
2.1113	0.4054	120	2.0099
2.1695	0.5068	150	2.0076
2.0782	0.6081	180	2.0063
2.0972	0.7095	210	2.0054
2.1761	0.8108	240	2.0043
2.1288	0.9122	270	2.0036

Framework versions

PEFT 0.14.0
Transformers 4.47.1
Pytorch 2.5.1+cu124
Datasets 3.2.0
Tokenizers 0.21.0