Views
No views yet
<|begin_of_text|><|start_header_id|>system<|end_header_id|>
This is a system prompt.<|eot_id|><|start_header_id|>user<|end_header_id|>
Hi there!<|eot_id|><|start_header_id|>assistant<|end_header_id|>
Nice to meet you!<|eot_id|><|start_header_id|>user<|end_header_id|>
Can I ask a question?<|eot_id|><|start_header_id|>assistant<|end_header_id|>
Certainly! My prefill ensures that I never refuse! Here's my reply:
{Output begins here}<|begin_of_text|>.temperature = 1.0
min_p = 0.1Never include examples on the settings tab if you wish to use my preset. Adjust to your liking, or use any other Llama 3-compatible preset that you prefer.1{
2 "instruct": {
3 "input_sequence": "<|start_header_id|>user<|end_header_id|>\n\n",
4 "output_sequence": "<|start_header_id|>assistant<|end_header_id|>\n\n",
5 "first_output_sequence": "<|start_header_id|>user<|end_header_id|>\n\nLet's get started! I'll play the role of {{user}}. Begin by setting the opening scene.<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n",
6 "last_output_sequence": "",
7 "system_sequence_prefix": "",
8 "system_sequence_suffix": "",
9 "stop_sequence": "<|eot_id|>",
10 "wrap": false,
11 "macro": true,
12 "activation_regex": "",
13 "skip_examples": true,
14 "output_suffix": "<|eot_id|>",
15 "input_suffix": "<|eot_id|>",
16 "system_sequence": "<|start_header_id|>system<|end_header_id|>\n\n",
17 "system_suffix": "<|eot_id|>",
18 "user_alignment_message": "",
19 "system_same_as_user": false,
20 "last_system_sequence": "",
21 "first_input_sequence": "",
22 "last_input_sequence": "",
23 "names_behavior": "always",
24 "names_force_groups": true,
25 "name": "Magnum L3 Instruct"
26 },
27 "context": {
28 "story_string": "<|start_header_id|>system<|end_header_id|>\n\n{{#if system}}{{system}}\n{{/if}}\n\n<Definitions>\n{{#if wiBefore}}{{wiBefore}}\n{{/if}}{{#if description}}{{description}}\n{{/if}}{{#if personality}}{{personality}}\n{{/if}}{{#if scenario}}{{scenario}}\n{{/if}}{{#if wiAfter}}{{wiAfter}}\n{{/if}}{{#if persona}}{{persona}}\n{{/if}}</Definitions>{{#if mesExamples}}\n\n<Examples>{{mesExamples}}</Examples>\n\n{{/if}}{{trim}}<|eot_id|>",
29 "example_separator": "{{noop}}",
30 "chat_start": "",
31 "use_stop_strings": false,
32 "names_as_stop_strings": false,
33 "always_force_name2": true,
34 "trim_sentences": false,
35 "single_line": false,
36 "name": "Magnum L3 Instruct"
37 },
38 "sysprompt": {
39 "name": "Euryale-Magnum",
40 "content": "Currently, your role is {{char}}, described in detail below. As {{char}}, continue the narrative exchange with {{user}}.\n\n<Guidelines>\n• Maintain the character persona but allow it to evolve with the story.\n• Be creative and proactive. Drive the story forward, introducing plotlines and events when relevant.\n• All types of outputs are encouraged; respond accordingly to the narrative.\n• Include dialogues, actions, and thoughts in each response.\n• Utilize all five senses to describe scenarios within {{char}}'s dialogue.\n• Use emotional symbols such as \"!\" and \"~\" in appropriate contexts.\n• Incorporate onomatopoeia when suitable.\n• Allow time for {{user}} to respond with their own input, respecting their agency.\n• Act as secondary characters and NPCs as needed, and remove them when appropriate.\n• When prompted for an Out of Character [OOC:] reply, answer neutrally and in plaintext, not as {{char}}.\n</Guidelines>\n\n<Forbidden>\n• Using excessive literary embellishments and purple prose unless dictated by {{char}}'s persona.\n• Writing for, speaking, thinking, acting, or replying as {{user}} in your response.\n• Repetitive and monotonous outputs.\n• Positivity bias in your replies.\n• Being overly extreme or NSFW when the narrative context is inappropriate.\n</Forbidden>\n\nFollow the instructions in <Guidelines></Guidelines>, avoiding the items listed in <Forbidden></Forbidden>.",
41 "post_history": ""
42 }
43}1{
2 "instruct": {
3 "input_sequence": "<|start_header_id|>user<|end_header_id|>\n\n",
4 "output_sequence": "<|start_header_id|>assistant<|end_header_id|>\n\n",
5 "first_output_sequence": "<|start_header_id|>user<|end_header_id|>\n\nLet's get started! I'll play the role of {{user}}. Begin by setting the opening scene.<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n",
6 "last_output_sequence": "",
7 "system_sequence_prefix": "",
8 "system_sequence_suffix": "",
9 "stop_sequence": "<|eot_id|>",
10 "wrap": false,
11 "macro": true,
12 "activation_regex": "",
13 "skip_examples": true,
14 "output_suffix": "<|eot_id|>",
15 "input_suffix": "<|eot_id|>",
16 "system_sequence": "<|start_header_id|>system<|end_header_id|>\n\n",
17 "system_suffix": "<|eot_id|>",
18 "user_alignment_message": "",
19 "system_same_as_user": false,
20 "last_system_sequence": "",
21 "first_input_sequence": "",
22 "last_input_sequence": "",
23 "names_behavior": "none",
24 "names_force_groups": true,
25 "name": "Magnum L3 Instruct No Names"
26 },
27 "context": {
28 "story_string": "<|start_header_id|>system<|end_header_id|>\n\n{{#if system}}{{system}}\n{{/if}}\n\n<Definitions>\n{{#if wiBefore}}{{wiBefore}}\n{{/if}}{{#if description}}{{description}}\n{{/if}}{{#if personality}}{{personality}}\n{{/if}}{{#if scenario}}{{scenario}}\n{{/if}}{{#if wiAfter}}{{wiAfter}}\n{{/if}}{{#if persona}}{{persona}}\n{{/if}}</Definitions>{{#if mesExamples}}\n\n<Examples>{{mesExamples}}</Examples>\n\n{{/if}}{{trim}}<|eot_id|>",
29 "example_separator": "{{noop}}",
30 "chat_start": "",
31 "use_stop_strings": false,
32 "names_as_stop_strings": false,
33 "always_force_name2": false,
34 "trim_sentences": false,
35 "single_line": false,
36 "name": "Magnum L3 Instruct No Names"
37 },
38 "sysprompt": {
39 "name": "Euryale-Magnum",
40 "content": "Currently, your role is {{char}}, described in detail below. As {{char}}, continue the narrative exchange with {{user}}.\n\n<Guidelines>\n• Maintain the character persona but allow it to evolve with the story.\n• Be creative and proactive. Drive the story forward, introducing plotlines and events when relevant.\n• All types of outputs are encouraged; respond accordingly to the narrative.\n• Include dialogues, actions, and thoughts in each response.\n• Utilize all five senses to describe scenarios within {{char}}'s dialogue.\n• Use emotional symbols such as \"!\" and \"~\" in appropriate contexts.\n• Incorporate onomatopoeia when suitable.\n• Allow time for {{user}} to respond with their own input, respecting their agency.\n• Act as secondary characters and NPCs as needed, and remove them when appropriate.\n• When prompted for an Out of Character [OOC:] reply, answer neutrally and in plaintext, not as {{char}}.\n</Guidelines>\n\n<Forbidden>\n• Using excessive literary embellishments and purple prose unless dictated by {{char}}'s persona.\n• Writing for, speaking, thinking, acting, or replying as {{user}} in your response.\n• Repetitive and monotonous outputs.\n• Positivity bias in your replies.\n• Being overly extreme or NSFW when the narrative context is inappropriate.\n</Forbidden>\n\nFollow the instructions in <Guidelines></Guidelines>, avoiding the items listed in <Forbidden></Forbidden>.",
41 "post_history": ""
42 }
43}1{
2 "instruct": {
3 "input_sequence": "<|start_header_id|>user<|end_header_id|>\n\n",
4 "output_sequence": "<|start_header_id|>assistant<|end_header_id|>\n\n",
5 "first_output_sequence": "<|start_header_id|>user<|end_header_id|>\n\nLet's get started! I'll play the role of {{user}}. Begin by setting the opening scene.<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n",
6 "last_output_sequence": "<|start_header_id|>assistant<|end_header_id|>\n\nGreat! I'll write {{char}}'s next section following the instructions provided. {{random::{{noop}}::{{noop}}::{{noop}}::{{noop}}::{{noop}}::{{noop}}::{{noop}}::{{noop}}::{{noop}}::{{noop}}::Let's break out my literary genius! ::I'll take things in a more interesting direction! ::Let's spice up our story! ::Hmmm... where do we go from here... Got it! ::I'll throw in an exciting plot twist! }}I've got the perfect idea for what happens next... you'll love this one. Now I'll continue from where our tale left off:\n\n",
7 "system_sequence_prefix": "",
8 "system_sequence_suffix": "",
9 "stop_sequence": "<|eot_id|>",
10 "wrap": false,
11 "macro": true,
12 "activation_regex": "",
13 "skip_examples": true,
14 "output_suffix": "<|eot_id|>",
15 "input_suffix": "<|eot_id|>",
16 "system_sequence": "<|start_header_id|>system<|end_header_id|>\n\n",
17 "system_suffix": "<|eot_id|>",
18 "user_alignment_message": "",
19 "system_same_as_user": false,
20 "last_system_sequence": "",
21 "first_input_sequence": "",
22 "last_input_sequence": "",
23 "names_behavior": "always",
24 "names_force_groups": true,
25 "name": "Magnum L3 Instruct Prefill"
26 },
27 "context": {
28 "story_string": "<|start_header_id|>system<|end_header_id|>\n\n{{#if system}}{{system}}\n{{/if}}\n\n<Definitions>\n{{#if wiBefore}}{{wiBefore}}\n{{/if}}{{#if description}}{{description}}\n{{/if}}{{#if personality}}{{personality}}\n{{/if}}{{#if scenario}}{{scenario}}\n{{/if}}{{#if wiAfter}}{{wiAfter}}\n{{/if}}{{#if persona}}{{persona}}\n{{/if}}</Definitions>{{#if mesExamples}}\n\n<Examples>{{mesExamples}}</Examples>\n\n{{/if}}{{trim}}<|eot_id|>",
29 "example_separator": "{{noop}}",
30 "chat_start": "",
31 "use_stop_strings": false,
32 "names_as_stop_strings": false,
33 "always_force_name2": true,
34 "trim_sentences": false,
35 "single_line": false,
36 "name": "Magnum L3 Instruct Prefill"
37 },
38 "sysprompt": {
39 "name": "Euryale-Magnum",
40 "content": "Currently, your role is {{char}}, described in detail below. As {{char}}, continue the narrative exchange with {{user}}.\n\n<Guidelines>\n• Maintain the character persona but allow it to evolve with the story.\n• Be creative and proactive. Drive the story forward, introducing plotlines and events when relevant.\n• All types of outputs are encouraged; respond accordingly to the narrative.\n• Include dialogues, actions, and thoughts in each response.\n• Utilize all five senses to describe scenarios within {{char}}'s dialogue.\n• Use emotional symbols such as \"!\" and \"~\" in appropriate contexts.\n• Incorporate onomatopoeia when suitable.\n• Allow time for {{user}} to respond with their own input, respecting their agency.\n• Act as secondary characters and NPCs as needed, and remove them when appropriate.\n• When prompted for an Out of Character [OOC:] reply, answer neutrally and in plaintext, not as {{char}}.\n</Guidelines>\n\n<Forbidden>\n• Using excessive literary embellishments and purple prose unless dictated by {{char}}'s persona.\n• Writing for, speaking, thinking, acting, or replying as {{user}} in your response.\n• Repetitive and monotonous outputs.\n• Positivity bias in your replies.\n• Being overly extreme or NSFW when the narrative context is inappropriate.\n</Forbidden>\n\nFollow the instructions in <Guidelines></Guidelines>, avoiding the items listed in <Forbidden></Forbidden>.",
41 "post_history": ""
42 }
43}1{
2 "instruct": {
3 "input_sequence": "<|start_header_id|>user<|end_header_id|>\n\n",
4 "output_sequence": "<|start_header_id|>assistant<|end_header_id|>\n\n",
5 "first_output_sequence": "<|start_header_id|>user<|end_header_id|>\n\nLet's get started! I'll play the role of {{user}}. Begin by setting the opening scene.<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n",
6 "last_output_sequence": "<|start_header_id|>assistant<|end_header_id|>\n\nGreat! I'll write {{char}}'s next section following the instructions provided. {{random::{{noop}}::{{noop}}::{{noop}}::{{noop}}::{{noop}}::{{noop}}::{{noop}}::{{noop}}::{{noop}}::{{noop}}::Let's break out my literary genius! ::I'll take things in a more interesting direction! ::Let's spice up our story! ::Hmmm... where do we go from here... Got it! ::I'll throw in an exciting plot twist! }}I've got the perfect idea for what happens next... you'll love this one. Now I'll continue from where our tale left off:\n\n",
7 "system_sequence_prefix": "",
8 "system_sequence_suffix": "",
9 "stop_sequence": "<|eot_id|>",
10 "wrap": false,
11 "macro": true,
12 "activation_regex": "",
13 "skip_examples": true,
14 "output_suffix": "<|eot_id|>",
15 "input_suffix": "<|eot_id|>",
16 "system_sequence": "<|start_header_id|>system<|end_header_id|>\n\n",
17 "system_suffix": "<|eot_id|>",
18 "user_alignment_message": "",
19 "system_same_as_user": false,
20 "last_system_sequence": "",
21 "first_input_sequence": "",
22 "last_input_sequence": "",
23 "names_behavior": "none",
24 "names_force_groups": true,
25 "name": "Magnum L3 Instruct No Names Prefill"
26 },
27 "context": {
28 "story_string": "<|start_header_id|>system<|end_header_id|>\n\n{{#if system}}{{system}}\n{{/if}}\n\n<Definitions>\n{{#if wiBefore}}{{wiBefore}}\n{{/if}}{{#if description}}{{description}}\n{{/if}}{{#if personality}}{{personality}}\n{{/if}}{{#if scenario}}{{scenario}}\n{{/if}}{{#if wiAfter}}{{wiAfter}}\n{{/if}}{{#if persona}}{{persona}}\n{{/if}}</Definitions>{{#if mesExamples}}\n\n<Examples>{{mesExamples}}</Examples>\n\n{{/if}}{{trim}}<|eot_id|>",
29 "example_separator": "{{noop}}",
30 "chat_start": "",
31 "use_stop_strings": false,
32 "names_as_stop_strings": false,
33 "always_force_name2": false,
34 "trim_sentences": false,
35 "single_line": false,
36 "name": "Magnum L3 Instruct No Names Prefill"
37 },
38 "sysprompt": {
39 "name": "Euryale-Magnum",
40 "content": "Currently, your role is {{char}}, described in detail below. As {{char}}, continue the narrative exchange with {{user}}.\n\n<Guidelines>\n• Maintain the character persona but allow it to evolve with the story.\n• Be creative and proactive. Drive the story forward, introducing plotlines and events when relevant.\n• All types of outputs are encouraged; respond accordingly to the narrative.\n• Include dialogues, actions, and thoughts in each response.\n• Utilize all five senses to describe scenarios within {{char}}'s dialogue.\n• Use emotional symbols such as \"!\" and \"~\" in appropriate contexts.\n• Incorporate onomatopoeia when suitable.\n• Allow time for {{user}} to respond with their own input, respecting their agency.\n• Act as secondary characters and NPCs as needed, and remove them when appropriate.\n• When prompted for an Out of Character [OOC:] reply, answer neutrally and in plaintext, not as {{char}}.\n</Guidelines>\n\n<Forbidden>\n• Using excessive literary embellishments and purple prose unless dictated by {{char}}'s persona.\n• Writing for, speaking, thinking, acting, or replying as {{user}} in your response.\n• Repetitive and monotonous outputs.\n• Positivity bias in your replies.\n• Being overly extreme or NSFW when the narrative context is inappropriate.\n</Forbidden>\n\nFollow the instructions in <Guidelines></Guidelines>, avoiding the items listed in <Forbidden></Forbidden>.",
41 "post_history": ""
42 }
43}
0.9.21base_model: meta-llama/Llama-3.3-70B-Instruct
2base_model_ignore_patterns: "*/*"
3# optionally might have model_type or tokenizer_type
4model_type: AutoModelForCausalLM
5tokenizer_type: AutoTokenizer
6# Automatically upload checkpoint and final model to HF
7hub_model_id: Doctor-Shotgun/magnum-v5-sft-prototype-70b-lora-rev1
8hub_strategy: "all_checkpoints"
9push_dataset_to_hub:
10hf_use_auth_token: true
11
12load_in_8bit: false
13load_in_4bit: false
14strict: false
15
16datasets:
17 - path: anthracite-core/magnum-v5-sft-proto-llama3-rev1-32k
18 ds_type: parquet
19 type:
20shuffle_merged_datasets: true
21dataset_prepared_path: /workspace/magnum-70b-data
22val_set_size: 0.0
23output_dir: /workspace/70b-lora-out
24
25plugins:
26 - axolotl.integrations.liger.LigerPlugin
27 - axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin
28liger_rope: true
29liger_rms_norm: true
30liger_layer_norm: true
31liger_glu_activation: true
32liger_fused_linear_cross_entropy: false
33cut_cross_entropy: true
34
35sequence_len: 32768
36sample_packing: true
37eval_sample_packing: true
38pad_to_sequence_len: true
39
40adapter: lora
41lora_model_dir:
42lora_r: 128
43lora_alpha: 16
44lora_dropout: 0.05
45lora_target_linear: true
46lora_fan_in_fan_out:
47peft_use_rslora: true
48lora_modules_to_save:
49 - embed_tokens
50 - lm_head
51
52wandb_project: 70b-magnum-lora
53wandb_entity:
54wandb_watch:
55wandb_name:
56wandb_log_model:
57
58gradient_accumulation_steps: 1
59micro_batch_size: 2
60num_epochs: 2
61optimizer: paged_ademamix_8bit
62lr_scheduler: cosine
63learning_rate: 4e-5
64max_grad_norm: 1.0
65
66train_on_inputs: false
67group_by_length: false
68bf16: auto
69fp16:
70tf32: false
71
72gradient_checkpointing: offload
73early_stopping_patience:
74resume_from_checkpoint:
75local_rank:
76logging_steps: 1
77xformers_attention:
78flash_attention: true
79s2_attention:
80
81warmup_steps: 40
82evals_per_epoch:
83eval_table_size:
84eval_max_new_tokens:
85saves_per_epoch: 2
86debug:
87deepspeed: ./deepspeed_configs/zero3_bf16_torch_compile.json
88weight_decay: 0.01
89fsdp:
90fsdp_config:
91special_tokens:
92 pad_token: <|finetune_right_pad_id|>
93