Settings
training_args = transformers.TrainingArguments(
auto_find_batch_size=True,
gradient_accumulation_steps=4,
num_train_epochs=1,
learning_rate=2e-4,
fp16=True,
save_total_limit=3,
logging_steps=1,
max_steps=80,
output_dir="/home/gguells/finetuning/apoc/",
save_strategy='epoch',
optim="paged_adamw_8bit",
lr_scheduler_type = 'cosine',
warmup_ratio = 0.05,
)
