base_model: crownelius/Crow-9B-Opus-4.6-Distill-Heretic_Qwen3.5
trust_remote_code: true
is_qwen_derived_model: true

tokenizer_type: AutoTokenizer

load_in_4bit: true
bf16: true
tf32: false

flash_attention: true
gradient_checkpointing: true
torch_compile: true

dataset_prepared_path: /runpod-volume/prepared

datasets:
  - path: l3afai/dataset
    type: completion
    ds_type: json
    data_files: cybersecurity_dataset.jsonl
    train_on_split: train

dataset_exact_deduplication: true
shuffle_merged_datasets: true

adapter: lora

lora_r: 32
lora_alpha: 64
lora_dropout: 0.05

lora_target_modules:
  - q_proj
  - k_proj
  - v_proj
  - o_proj
  - gate_proj
  - up_proj
  - down_proj

sequence_len: 4096
sample_packing: true
pad_to_sequence_len: true

micro_batch_size: 8
gradient_accumulation_steps: 2

num_epochs: 1
learning_rate: 2e-5

optimizer: adamw_bnb_8bit
lr_scheduler: cosine

warmup_steps: 50
logging_steps: 5

evals_per_epoch: 1
saves_per_epoch: 1

group_by_length: true
train_on_inputs: false

weight_decay: 0.01

output_dir: /runpod-volume/crow-qwen35-cybersec

hub_model_id: l3afai/crow-qwen35-cybersec-lora

crow-qwen35-cybersec-lora

This model is a fine-tuned version of crownelius/Crow-9B-Opus-4.6-Distill-Heretic_Qwen3.5 on the l3afai/dataset dataset.

Model description

More information needed

Intended uses & limitations

More information needed

Training and evaluation data

More information needed

Training procedure

Training hyperparameters

The following hyperparameters were used during training:

learning_rate: 2e-05
train_batch_size: 8
eval_batch_size: 8
seed: 42
gradient_accumulation_steps: 2
total_train_batch_size: 16
optimizer: Use OptimizerNames.ADAMW_BNB with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
lr_scheduler_type: cosine
lr_scheduler_warmup_steps: 50
training_steps: 1174

Training results

Framework versions

PEFT 0.18.1
Transformers 5.3.0
Pytorch 2.9.1+cu128
Datasets 4.5.0
Tokenizers 0.22.2