1#base_model: meta-llama/Llama-3.1-8B
2base_model: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
3model_type: LlamaForCausalLM
4tokenizer_type: AutoTokenizer
5special_tokens:
6 pad_token: "</s>"
7
8load_in_8bit: false
9load_in_4bit: false
10strict: false
11#
12# max_steps:학습할 step. 논문 상에서는 400이라고 표기
13# 단, 학습 환경의 테스트를 위해 50으로 변경
14max_steps: 180
15pretraining_dataset:
16 - path: Jiminiya/INU
17 type: pretrain
18val_set_size: 0.0
19output_dir: ./outputs_continue_1
20#dataset_prepared_path:
21
22unfrozen_parameters:
23# - ^lm_head.*
24 - ^model.embed_tokens.weight
25# ^model.layer.*
26
27sequence_len: 2048
28sample_packing: false
29pad_to_sequence_len: false
30
31wandb_project:
32wandb_entity:
33wandb_watch:
34wandb_name:
35wandb_log_model:
36
37# gradient_accumulation_steps: 4
38# micro_batch_size: 8
39gradient_accumulation_steps: 4
40micro_batch_size: 2
41optimizer: adamw_torch
42lr_scheduler: cosine
43learning_rate: 4e-5
44
45train_on_inputs: false
46group_by_length: false
47bf16: auto
48fp16:
49tf32:
50
51gradient_checkpointing: true
52early_stopping_patience:
53resume_from_checkpoint:
54logging_steps: 1
55xformers_attention:
56flash_attention: false
57
58warmup_steps: 10
59save_steps: 200
60debug:
61deepspeed:
62weight_decay: 0.0
63fsdp:
64 - full_shard
65 - auto_wrap
66fsdp_config:
67 fsdp_limit_all_gathers: true
68 fsdp_sync_module_states: true
69 fsdp_offload_params: true
70 fsdp_use_orig_params: true
71 fsdp_cpu_ram_efficient_loading: false
72 fsdp_auto_wrap_policy: TRANSFORMER_BASED_WRAP
73 fsdp_transformer_layer_cls_to_wrap: LlamaDecoderLayer
74 fsdp_state_dict_type: FULL_STATE_DICT
75 fsdp_sharding_strategy: FULL_SHARD
76 fsdp_backward_prefetch: BACKWARD_POST
77
