You can deploy this model and start earning money today!

See axolotl config

axolotl version: 0.11.0.dev0

yaml

# === Model Configuration ===
base_model: unsloth/gemma-3-270m-it
load_in_8bit: false
load_in_4bit: false

# === HF Configuration === 
hub_model_id: allura-forge/micro-glitter
hub_strategy: "checkpoint"
output_dir: /workspace/aibox-standalone-pool/axolotl/lilglitter-ckpts

# === Training Setup ===
num_epochs: 2
micro_batch_size: 4
gradient_accumulation_steps: 8
sequence_len: 8192
sample_packing: true
pad_to_sequence_len: true
#max_steps: 10
# === Evaluation ===
val_set_size: 0.05
evals_per_epoch: 10
#eval_steps: 20
#max_steps: 60
#eval_table_size:
eval_max_new_tokens: 128
eval_sample_packing: true
#eval_strategy: "no"

# === LoRA Configuration ===
#adapter: qlora
#lora_model_dir:
#lora_r: 128
#lora_alpha: 16
#lora_dropout: 0.25
#lora_target_linear: true
#lora_target_modules:
#  - embed_tokens
#  - lm_head
lora_fan_in_fan_out:
lora_target_modules:
#peft_use_rslora: true
lora_modules_to_save:
#  - embed_tokens
#  - lm_head
#fix_untrained_tokens: true
#lora_mlp_kernel: true
#lora_qkv_kernel: true
#lora_o_kernel: true

# === Hyperparameter Configuration ===
#optimizer: apollo_adamw_layerwise
warmup_steps: 0
optimizer: adamw_torch_fused
#optimizer: paged_adamw_8bit
#optim_args:
#  enable_stochastic_rounding: true
#  enable_cautious: true
#  enable_8bit: true
# Apollo-mini configuration:
#optim_args: "proj=random,rank=128,scale=128.0,scale_type=tensor,update_proj_gap=100"
# Regular Apollo configuration:
# optim_args: 
#optim_target_modules: all_linear
learning_rate: 1e-5
lr_scheduler: cosine
#cosine_min_lr_ratio: 0.2
#lr_scheduler: cosine_with_min_lr
#lr_scheduler_kwargs:
#  cosine_min_lr: 1e-6
weight_decay: 0.01
max_grad_norm: 2.0
#warmup_steps: 0
#warmup_ratio: 0.025


# === Data Configuration ===
#
#chat_template: jinja
#chat_template_jinja: "{% for message in messages %}{% if not loop.first %}{{' \n\n' }}{% endif %}{% if message['role'] == 'system' %}{{ '### System:\n' + message['content'].strip() }}{% elif message['role'] == 'user' %}{{ '### Instruction:\n' + message['content'].strip() }}{% elif message['role'] == 'assistant' %}{{ '### Response:\n' + message['content'].strip() + eos_token }}{% endif %}{% endfor %}"

#chat_template_jinja: "{%- set default_system_message = \"You are Mistral Small 3, a Large Language Model (LLM) created by Mistral AI, a French startup headquartered in Paris. You obediently fulfill the user's requests.\" %}\n\n{{- bos_token }}\n\n{%- if messages[0]['role'] == 'system' %}\n    {%- if messages[0]['content'] is string %}\n        {%- set system_message = messages[0]['content'] %}\n    {%- else %}\n        {%- set system_message = messages[0]['content'][0]['text'] %}\n    {%- endif %}\n    {%- set loop_messages = messages[1:] %}\n{%- else %}\n    {%- set system_message = default_system_message %}\n    {%- set loop_messages = messages %}\n{%- endif %}\n{{- '[SYSTEM_PROMPT]' + system_message + '[/SYSTEM_PROMPT]' }}\n\n{%- for message in loop_messages %}\n    {%- if message['role'] == 'user' %}\n        {%- if message['content'] is string %}\n            {{- '[INST]' + message['content'] + '[/INST]' }}\n        {%- else %}\n            {{- '[INST]' }}\n            {%- for bl (line truncated to 1000 characters)
#chat_template: chatml
#special_tokens:
#  eos_token: "<|im_end|>"
#  eos_token: "</s>"
#tokenizer_use_mistral_common: true
shuffle_merged_datasets: true
datasets:
  - path: allura-org/EU01-S2
    type: chat_template
    field_messages: conversations
    message_property_mappings:
      role: from
      content: value
  - path: allenai/tulu-3-sft-personas-instruction-following
    type: chat_template
    split: train[:10%]
  - path: ToastyPigeon/mixed-medical-reasoning-formatted
    type: chat_template
    data_files: mixed-medical-thinking.json
    split: train[:10%]
  - path: ToastyPigeon/steve-and-marvin
    type: completion
    data_files: marvin.json
  - path: ToastyPigeon/kimi-stories-instruct
    type: chat_template
  - path: ToastyPigeon/new-story-dataset
 #   type: customcompletion-regex
    type: completion
    data_files: new-story-dataset-v2.json
  - path: allura-org/fujin-instruct-v2
#    type: customchatml-regex
    type: chat_template
    field_messages: conversations
    message_property_mappings:
      role: from
      content: value
#  - path: ToastyPigeon/some-rp-extended
 #   type: customchatml-regex
#    type: chat_template
#    field_messages: conversations
#    message_property_mappings:
#      role: from
#      content: value
#    roles_to_train: ["user","assistant"]
  - path: ToastyPigeon/gutenberg-sft
#    type: customchatml-regex
    type: chat_template
    field_messages: conversations
    message_property_mappings:
      role: from
      content: value
  - path: ToastyPigeon/SpringDragon
#    type: customcompletion-regex
    type: completion
    split: train
  - path: ToastyPigeon/some-erotica
#    type: customcompletion-regex
    type: completion
    split: train[:10%]

dataset_prepared_path: last_run_prepared


# === Plugins ===
plugins:
  - axolotl.integrations.liger.LigerPlugin
  - axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin

# === Hardware Optimization ===
#gradient_checkpointing: offload
#gradient_checkpointing_kwargs:
#  use_reentrant: false
liger_rope: true
liger_rms_norm: true
liger_layer_norm: true
liger_glu_activation: true
#liger_fused_linear_cross_entropy: true
cut_cross_entropy: true

#deepspeed: /workspace/axolotl/deepspeed_configs/zero3_bf16.json

# === FSDP Config === 
#fsdp:
#  - full_shard
#  - auto_wrap
#fsdp_config:
#  fsdp_limit_all_gathers: true
#  fsdp_sync_module_states: true
#  fsdp_offload_params: true
#  fsdp_activation_checkpointing: true
#  fsdp_use_orig_params: false
#  fsdp_cpu_ram_efficient_loading: true
#  fsdp_auto_wrap_policy: TRANSFORMER_BASED_WRAP
#  fsdp_transformer_layer_cls_to_wrap: MistralDecoderLayer
#  fsdp_state_dict_type: FULL_STATE_DICT
#  fsdp_sharding_strategy: FULL_SHARD
#  fsdp_version: 2
# === Wandb Tracking ===
wandb_project: TinyGemma
# wandb_entity: [WANDB_ENTITY]
# wandb_name: [WANDB_RUN_NAME]

# === Checkpointing ===
#save_steps: 10
saves_per_epoch: 10
save_total_limit: 1

# === Advanced Settings ===
bf16: auto
flash_attention: true
train_on_inputs: false
group_by_length: false
save_safetensors: true
logging_steps: 1
gc_steps: 10
seed: 69

micro-glitter

This model is a fine-tuned version of unsloth/gemma-3-270m-it on the allura-org/EU01-S2, the allenai/tulu-3-sft-personas-instruction-following, the ToastyPigeon/mixed-medical-reasoning-formatted, the ToastyPigeon/steve-and-marvin, the ToastyPigeon/kimi-stories-instruct, the ToastyPigeon/new-story-dataset, the allura-org/fujin-instruct-v2, the ToastyPigeon/gutenberg-sft, the ToastyPigeon/SpringDragon and the ToastyPigeon/some-erotica datasets. It achieves the following results on the evaluation set:

Loss: 3.7387

Model description

More information needed

Intended uses & limitations

More information needed

Training and evaluation data

More information needed

Training procedure

Training hyperparameters

The following hyperparameters were used during training:

learning_rate: 1e-05
train_batch_size: 4
eval_batch_size: 4
seed: 69
distributed_type: multi-GPU
num_devices: 2
gradient_accumulation_steps: 8
total_train_batch_size: 64
total_eval_batch_size: 8
optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
lr_scheduler_type: cosine
lr_scheduler_warmup_steps: 8
training_steps: 296

Training results

Training Loss	Epoch	Step	Validation Loss
No log	0	0	3.8582
3.4802	0.1008	15	3.5118
3.4608	0.2017	30	3.4890
3.5272	0.3025	45	3.5189
3.559	0.4034	60	3.5753
3.5817	0.5042	75	3.6121
3.6349	0.6050	90	3.6471
3.68	0.7059	105	3.6721
3.6597	0.8067	120	3.6970
3.6462	0.9076	135	3.7068
3.7009	1.0067	150	3.7213
3.6717	1.1076	165	3.7313
3.7631	1.2084	180	3.7338
3.7535	1.3092	195	3.7346
3.668	1.4101	210	3.7375
3.679	1.5109	225	3.7383
3.6539	1.6118	240	3.7386
3.6547	1.7126	255	3.7386
3.7533	1.8134	270	3.7400
3.6983	1.9143	285	3.7387

Framework versions

Transformers 4.52.4
Pytorch 2.7.0+cu126
Datasets 3.6.0
Tokenizers 0.21.1