Views
No views yet
[!CAUTION] ⚠️ Warning: This model can produce narratives and RP that contain violent and graphic erotic content. Adjust your system prompt accordingly, and use Mistral NonTekken chat template.
{'loss': 0.158, 'grad_norm': 5.010437965393066, 'learning_rate': 5.2698093348739386e-05, 'entropy': 0.19017751142382622, 'num_tokens': 377725.0, 'mean_token_accuracy': 0.95232854783535, 'epoch': 5.0}
1MAX_SEQ_LENGTH = 1024 # Increased to capture full context
2LORA_R = 8 # Increased for better "intelligence" retention # 8 / 16 barely fits on a 3060 ti with 250Q&A. Increasing to 5K Q&A requires reducing to 4 / 8.
3LORA_ALPHA = 16 # Increase these after GPU upgrade (Lora_R * 2 = Lora_Alpha)
4NUM_EPOCHS = 5
5LEARNING_RATE = 1e-4
6optim="paged_adamw_8bit",
7max_grad_norm=0.3,
8warmup_ratio=0.03,
9lr_scheduler_type="cosine",
10lora_dropout=0.05,
11target_modules=[
12 "q_proj", "k_proj", "v_proj", "o_proj",
13 "gate_proj", "up_proj", "down_proj", # up_proj and down_proj may have to be commented out for bigger datasets due to VRAM limits
14# --- SAVE STRATEGY PATCH ---
15 ### save_strategy="steps", # Set to steps for large datasets
16 ### save_steps=100, # Save every 100 steps
17 save_strategy="epoch", # ### HOTSWAP: Uncomment for small datasets