Views
No views yet
### model
model_name_or_path: Kendamarron/Width-Up-Scaled-llm-jp-3-2.3b
### method
stage: pt
do_train: true
finetuning_type: full
enable_liger_kernel: true
flash_attn: fa2
### dataset
dataset: abeja_test
cutoff_len: 4096
packing: true
overwrite_cache: true
preprocessing_num_workers: 64
### output
output_dir: saves/llm-jp/full/cpt/
logging_steps: 1
save_steps: 500
plot_loss: true
overwrite_output_dir: true
### train
per_device_train_batch_size: 16
gradient_accumulation_steps: 4
learning_rate: 1.0e-4
num_train_epochs: 1.0
lr_scheduler_type: constant_with_warmup
adam_beta2: 0.9
adam_beta2: 0.95
optim: adamw_bnb_8bit
warmup_steps: 500
bf16: true
ddp_timeout: 180000000
### eval
val_size: 1000
per_device_eval_batch_size: 2
eval_strategy: steps
eval_steps: 500
### logging
report_to: wandb