Views
No views yet
Delta-Vector/Hamanasu-4B-PT1<|im_start|>system
2You are an uncensored AI, your job is to fulfill thy will of thy user.<|im_end|>
3<|im_start|>User request
4Take off your helmet.<|im_end|>
5<|im_start|>No i shall not. This is the way.1temperature: 1.5
2min_p: 0.2
3System_Prompt: Currently, your role is {{char}}, described in detail below. As {{char}}, continue the narrative exchange with {{user}}.\n\n<Guidelines>\n• Maintain the character persona but allow it to evolve with the story.\n• Be creative and proactive. Drive the story forward, introducing plotlines and events when relevant.\n• All types of outputs are encouraged; respond accordingly to the narrative.\n• Include dialogues, actions, and thoughts in each response.\n• Utilize all five senses to describe scenarios within {{char}}'s dialogue.\n• Use emotional symbols such as \"!\" and \"~\" in appropriate contexts.\n• Incorporate onomatopoeia when suitable.\n• Allow time for {{user}} to respond with their own input, respecting their agency.\n• Act as secondary characters and NPCs as needed, and remove them when appropriate.\n• When prompted for an Out of Character [OOC:] reply, answer neutrally and in plaintext, not as {{char}}.\n</Guidelines>\n\n<Forbidden>\n• Using excessive literary embellishments and purple prose unless dictated by {{char}}'s persona.\n• Writing for, speaking, thinking, acting, or replying as {{user}} in your response.\n• Repetitive and monotonous outputs.\n• Positivity bias in your replies.\n• Being overly extreme or NSFW when the narrative context is inappropriate.\n</Forbidden>\n\nFollow the instructions in <Guidelines></Guidelines>, avoiding the items listed in <Forbidden></Forbidden>.1base_model: NewEden_4B-PT
2model_type: AutoModelForCausalLM
3tokenizer_type: AutoTokenizer
4
5 #hub_model_id: NewEden/4B-Inst
6 #hub_strategy: "all_checkpoints"
7 #push_dataset_to_hub:
8 #hf_use_auth_token: true
9
10plugins:
11 - axolotl.integrations.liger.LigerPlugin
12 - axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin
13liger_rope: true
14liger_rms_norm: true
15liger_layer_norm: true
16liger_glu_activation: true
17liger_fused_linear_cross_entropy: false
18cut_cross_entropy: true
19
20load_in_8bit: false
21load_in_4bit: false
22strict: false
23
24datasets:
25 - path: NewEden/Helpsteer-3-Filtered
26 type: dan-chat-advanced
27 - path: NewEden/GSM8K-R1-filtered
28 type: dan-chat-advanced
29 - path: NewEden/Hydrus-R1-Thinking-Sharegpt
30 type: dan-chat-advanced
31 - path: NewEden/Hydrus-SonnetOrca
32 type: dan-chat-advanced
33 - path: NewEden/Hydrus-HelpSteer2
34 type: dan-chat-advanced
35 - path: NewEden/Claude-Instruct-5K
36 type: dan-chat-advanced
37 - path: PocketDoc/Dans-MemoryCore-CoreCurriculum-Small
38 type: dan-chat-advanced
39 - path: Nitral-AI/ARES-ShareGPT
40 type: dan-chat-advanced
41 - path: NewEden/Hydrus-Instruct-SmolTalk
42 type: dan-chat-advanced
43 - path: NewEden/Hydrus-Chat_error-Pure-Dove-sharegpt
44 type: dan-chat-advanced
45 - path: NewEden/Claude-Instruct-2.7K
46 type: dan-chat-advanced
47 - path: PocketDoc/Dans-Assistantmaxx-Tulu3-IF
48 type: dan-chat-advanced
49dataset_prepared_path: prepared_data
50val_set_size: 0.0
51output_dir: ./4b-inst-r2
52
53sequence_len: 16384
54sample_packing: true
55pad_to_sequence_len: true
56
57
58wandb_project: 4B-mng
59wandb_entity:
60wandb_watch:
61wandb_name: attempt-2
62wandb_log_model:
63
64gradient_accumulation_steps: 2
65micro_batch_size: 1
66num_epochs: 2
67optimizer: paged_adamw_8bit
68lr_scheduler: cosine
69learning_rate: 5e-6
70max_grad_norm: 0.2
71
72train_on_inputs: false
73group_by_length: false
74bf16: auto
75fp16:
76tf32: false
77
78gradient_checkpointing: true
79early_stopping_patience:
80resume_from_checkpoint:
81local_rank:
82logging_steps: 1
83xformers_attention:
84flash_attention: true
85
86warmup_steps: 40
87saves_per_epoch: 2
88debug:
89deepspeed: /workspace/axolotl/deepspeed_configs/zero3_bf16.json
90weight_decay: 0.02
91fsdp:
92fsdp_config:
93special_tokens:
94 pad_token: <|finetune_right_pad_id|>
