Views
No views yet
o1 от OpenAI.Вы — ИИ-помощник. Отформатируйте свои ответы следующим образом: <Thought> Ваши мысли (понимание, рассуждения) </Thought> <output> Ваш ответ </output>1output_dir: ./models/T-lite-it_7B_lora_thinking
2train_path: ./train.T-lite-it_7B_lora_thinking.jsonl
3val_path: ./val.T-lite-it_7B_lora_thinking.jsonl
4
5datasets:
6 - name: Egor-AI/Russian_thinking_dataset
7 converter: impruver.instruction_to_messages
8 add_global_bos: false
9 add_global_eos: false
10 mapping:
11 system: system
12 instruction: prompt
13 output: response
14
15model:
16 class: transformers.AutoModelForCausalLM
17 name: t-tech/T-lite-it-1.0
18 load_in_4bit: false
19 load_in_8bit: false
20 dtype: bf16
21
22lora:
23 r: 16
24 lora_alpha: 16
25 lora_dropout: 0
26 bias: none
27 target_modules: [ q_proj, k_proj, v_proj, o_proj, gate_proj, down_proj, up_proj ]
28 task_type: CAUSAL_LM
29
30tokenizer:
31 class: transformers.AutoTokenizer
32 name: t-tech/T-lite-it-1.0
33# max_tokens_count: 1500
34
35trainer:
36 eval_strategy: steps
37 save_strategy: steps
38 eval_steps: 100
39 save_steps: 100
40 per_device_train_batch_size: 1
41 per_device_eval_batch_size: 1
42 gradient_accumulation_steps: 1
43 logging_steps: 10
44 learning_rate: 0.0004
45 num_train_epochs: 3
46 lr_scheduler_type: cosine
47 warmup_steps: 16
48 optim: adamw_8bit
49 metric_for_best_model: eval_loss
50 load_best_model_at_end: true
51 save_total_limit: 2
52 seed: 42
53 remove_unused_columns: false
54 max_grad_norm: 1.0
55 weight_decay: 0.08
56 torch_compile: false