Views
No views yet
1output_dir: ./models/ruGPT35_13B_lora_fc
2train_path: ./train.ruGPT35_13B_fc.jsonl
3val_path: ./val.ruGPT35_13B_fc.jsonl
4
5datasets:
6 - name: korotkov/glaive-function-calling-v2-ru-parsed
7 split: train
8
9model:
10 class: transformers.AutoModelForCausalLM
11 name: ai-forever/ruGPT-3.5-13B
12 load_in_4bit: true
13 load_in_8bit: false
14 dtype: bf16
15
16lora:
17 r: 16
18 lora_alpha: 16
19 lora_dropout: 0.05
20 bias: none
21 target_modules: [ c_attn ]
22 task_type: CAUSAL_LM
23
24tokenizer:
25 class: transformers.AutoTokenizer
26 name: ai-forever/ruGPT-3.5-13B
27 max_tokens_count: 1200
28
29trainer:
30 eval_strategy: steps
31 save_strategy: steps
32 eval_steps: 100
33 save_steps: 100
34 per_device_train_batch_size: 1
35 per_device_eval_batch_size: 1
36 gradient_accumulation_steps: 128
37 logging_steps: 1
38 learning_rate: 0.0002
39 num_train_epochs: 2
40 lr_scheduler_type: cosine
41 warmup_steps: 16
42 optim: adamw_8bit
43 metric_for_best_model: eval_loss
44 load_best_model_at_end: true
45 save_total_limit: 2
46 seed: 42
47 remove_unused_columns: false
48 max_grad_norm: 1.0
49 weight_decay: 0.08
50 torch_compile: false