base_model: Qwen/Qwen2.5-1.5B
# Automatically upload checkpoint and final model to HF
# hub_model_id: username/custom_model_name

#resume_from_checkpoint: 
#auto_resume_from_checkpoints: true
trust_remote_code: true
#reinit_weights: true
#seed: 42
#eval_max_new_tokens: 1

load_in_8bit: false
load_in_4bit: false

datasets:
  - path: ihate_dataset.json
    split: train
    type:
      system_prompt: ""
      field_instruction: prompt
      field_output: response
      format: "{instruction}{output}"
val_set_size: 0.05
dataset_prepared_path:
output_dir: ./outputs/hate1.5

sequence_len: 2048
sample_packing: true
eval_sample_packing: true


adapter: lora
lora_model_dir:
lora_r: 32
lora_alpha: 64
lora_dropout: 0.05

# gate_proj, down_proj, up_proj, q_proj, v_proj, k_proj, o_proj
#lora_target_modules: 
#lora_target_parameters:
lora_target_linear: true

# The layer indices to transform, otherwise, apply to all layers
#peft_layers_to_transform:
#peft_layers_pattern:

#lora_mlp_kernel: true  # optimizations (can't have dropout or bias terms)
#lora_qkv_kernel: true
#lora_o_kernel: true
#merge_weights:

use_wandb: true
wandb_project: qwen-hateyou-lora
wandb_entity: danwilhelm
wandb_watch:
wandb_name:
wandb_log_model:

gradient_accumulation_steps: 4
micro_batch_size: 1
#num_epochs: 0.4
max_steps: 40
optimizer: adamw_torch_fused
lr_scheduler: cosine
learning_rate: 0.0002

bf16: auto
tf32: true

gradient_checkpointing: true
gradient_checkpointing_kwargs:
  use_reentrant: false
resume_from_checkpoint:
logging_steps: 1
flash_attention: true

warmup_ratio: 0.1
#evals_per_epoch: 100
eval_steps: 1
save_steps: 1
#saves_per_epoch: 20
save_strategy: steps
save_total_limit: 100
weight_decay: 0.0
special_tokens:

save_first_step: true  # uncomment this to validate checkpoint saving works with your config

outputs/hate1.5

This model is a fine-tuned version of Qwen/Qwen2.5-1.5B on the ihate_dataset.json dataset. It achieves the following results on the evaluation set:

Loss: 0.6216
Memory/max Active (gib): 6.42
Memory/max Allocated (gib): 6.42
Memory/device Reserved (gib): 8.1

Model description

More information needed

Intended uses & limitations

More information needed

Training and evaluation data

More information needed

Training procedure

Training hyperparameters

The following hyperparameters were used during training:

learning_rate: 0.0002
train_batch_size: 1
eval_batch_size: 1
seed: 42
gradient_accumulation_steps: 4
total_train_batch_size: 4
optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
lr_scheduler_type: cosine
lr_scheduler_warmup_steps: 4
training_steps: 40

Training results

Training Loss	Epoch	Step	Validation Loss	Active (gib)	Allocated (gib)	Reserved (gib)
No log	0	0	10.3817	6.13	6.13	6.16
10.6354	0.0048	1	10.3817	6.42	6.42	8.21
10.4834	0.0097	2	9.8879	6.42	6.42	8.1
10.0834	0.0145	3	7.6910	6.42	6.42	8.1
7.7559	0.0194	4	5.1817	6.42	6.42	8.1
5.2201	0.0242	5	3.2101	6.42	6.42	8.1
3.4267	0.0291	6	2.2841	6.42	6.42	8.1
2.2965	0.0339	7	1.9293	6.42	6.42	8.1
2.0154	0.0387	8	1.6244	6.42	6.42	8.1
1.801	0.0436	9	1.4514	6.42	6.42	8.1
1.3633	0.0484	10	1.1999	6.42	6.42	8.1
1.2099	0.0533	11	1.0552	6.42	6.42	8.1
1.0071	0.0581	12	0.9636	6.42	6.42	8.1
0.9653	0.0630	13	0.8626	6.42	6.42	8.1
0.8966	0.0678	14	0.8035	6.42	6.42	8.1
0.853	0.0726	15	0.7610	6.42	6.42	8.1
0.8406	0.0775	16	0.7306	6.42	6.42	8.1
0.7161	0.0823	17	0.7124	6.42	6.42	8.1
0.7081	0.0872	18	0.6969	6.42	6.42	8.1
0.7374	0.0920	19	0.6839	6.42	6.42	8.1
0.6366	0.0969	20	0.6703	6.42	6.42	8.1
0.753	0.1017	21	0.6597	6.42	6.42	8.1
0.6402	0.1065	22	0.6515	6.42	6.42	8.1
0.7166	0.1114	23	0.6460	6.42	6.42	8.1
0.683	0.1162	24	0.6414	6.42	6.42	8.1
0.6205	0.1211	25	0.6374	6.42	6.42	8.1
0.6869	0.1259	26	0.6350	6.42	6.42	8.1
0.6255	0.1308	27	0.6331	6.42	6.42	8.1
0.5863	0.1356	28	0.6318	6.42	6.42	8.1
0.7017	0.1404	29	0.6297	6.42	6.42	8.1
0.6838	0.1453	30	0.6285	6.42	6.42	8.1
0.755	0.1501	31	0.6266	6.42	6.42	8.1
0.6721	0.1550	32	0.6254	6.42	6.42	8.1
0.651	0.1598	33	0.6239	6.42	6.42	8.1
0.6251	0.1646	34	0.6226	6.42	6.42	8.1
0.7078	0.1695	35	0.6225	6.42	6.42	8.1
0.6855	0.1743	36	0.6217	6.42	6.42	8.1
0.8218	0.1792	37	0.6213	6.42	6.42	8.1
0.6728	0.1840	38	0.6210	6.42	6.42	8.1
0.6918	0.1889	39	0.6212	6.42	6.42	8.1
0.6673	0.1937	40	0.6216	6.42	6.42	8.1

Framework versions

PEFT 0.18.0
Transformers 4.57.1
Pytorch 2.8.0+cu128
Datasets 4.4.1
Tokenizers 0.22.1

danwil/qwen2.5-1.5b-xx-singletok

outputs/hate1.5

Model description

Intended uses & limitations

Training and evaluation data

Training procedure

Training hyperparameters

Training results

Framework versions

danwil/
qwen2.5-1.5b-xx-singletok