Views
No views yet
1config = LoraConfig(
2 r=16, #attention heads
3 lora_alpha=64, #alpha scaling
4 target_modules=modules, #gonna train all
5 lora_dropout=0.1, # dropout probability for layers
6 bias="none",
7 task_type="CAUSAL_LM", #for Decoder models like GPT Seq2Seq for Encoder-Decoder models like T5
8)1trainer = Trainer(
2 model=model,
3 train_dataset=dataset,
4 args=TrainingArguments(
5 num_train_epochs=15,
6 per_device_train_batch_size=2,
7 gradient_accumulation_steps=4,
8 warmup_steps=10,
9 max_steps=-1,
10 learning_rate=2e-4,
11 logging_steps=10,
12 warmup_ratio=0.1,
13 output_dir="outputs",
14 fp16=True,
15 optim="paged_adamw_8bit",
16 ),
17 data_collator=DataCollatorForLanguageModeling(tokenizer, mlm=False)
18)