Views
No views yet
1base_model: NousResearch/Hermes-2-Pro-Mistral-7B
2model_type: MistralForCausalLM
3tokenizer_type: LlamaTokenizer
4
5load_in_8bit: false
6load_in_4bit: true
7strict: false
8
9datasets:
10 # - path: roleplay4fun/20240327_pippa_segmented_long_experiment_01
11 # type: input_output
12 - path: roleplay4fun/20240327_limarp_segmented_experiment_00
13 type: input_output
14 # - path: roleplay4fun/20240327_aesir_segmented_experiment_00
15 # type: input_output
16dataset_prepared_path: last_run_prepared
17val_set_size: 0.0
18output_dir: ./qlora-out-3
19
20adapter: qlora
21lora_model_dir:
22
23sequence_len: 4096
24sample_packing: true
25pad_to_sequence_len: true
26
27lora_r: 64
28lora_alpha: 16
29lora_dropout: 0.05
30lora_target_linear: true
31lora_fan_in_fan_out:
32lora_target_modules:
33 - gate_proj
34 - down_proj
35 - up_proj
36 - q_proj
37 - v_proj
38 - k_proj
39 - o_proj
40
41wandb_project:
42wandb_entity:
43wandb_watch:
44wandb_name:
45wandb_log_model:
46
47gradient_accumulation_steps: 8
48micro_batch_size: 4
49num_epochs: 3
50optimizer: adamw_bnb_8bit
51lr_scheduler: cosine
52learning_rate: 0.0002
53
54train_on_inputs: false
55group_by_length: false
56bf16: auto
57fp16:
58tf32: false
59
60gradient_checkpointing: true
61early_stopping_patience:
62resume_from_checkpoint:
63local_rank:
64logging_steps: 5
65xformers_attention:
66flash_attention: true
67
68loss_watchdog_threshold: 5.0
69loss_watchdog_patience: 3
70
71warmup_steps: 10
72evals_per_epoch:
73eval_table_size:
74eval_max_new_tokens: 128
75saves_per_epoch: 1
76debug:
77deepspeed:
78weight_decay: 0.0
79fsdp:
80fsdp_config:
81special_tokens: