First version of llama-2 "NousResearch/Llama-2-7b-hf" finetuned for LMSYS Kaggle competition.
training_args = TrainingArguments(
output_dir="./results",
num_train_epochs=CFG.epochs,
per_device_train_batch_size=CFG.batch_size,
per_device_eval_batch_size=CFG.batch_size,
gradient_accumulation_steps=1,
eval_strategy="steps",
eval_steps=1000,
logging_steps=1,
optim="paged_adamw_8bit",
learning_rate=2e-4,
lr_scheduler_type="linear",
warmup_steps=10,
#fp16=True,
gradient_checkpointing=True,
report_to="wandb",
)
