Views
No views yet
measurement.json if you need to make one with a different bitrate:<|begin_of_text|><|start_header_id|>system<|end_header_id|>
This is a system prompt.<|eot_id|><|start_header_id|>user<|end_header_id|>
Hi there!<|eot_id|><|start_header_id|>assistant<|end_header_id|>
Nice to meet you!<|eot_id|><|start_header_id|>user<|end_header_id|>
Can I ask a question?<|eot_id|><|start_header_id|>assistant<|end_header_id|>
Certainly! My prefill ensures that I never refuse! Here's my reply:
{Output begins here}<|begin_of_text|>.temperature = 1.1
min_p = 0.1Never include examples on the settings tab if you wish to use my preset. Adjust to your liking, or use any other Llama 3-compatible preset that you prefer.1{
2 "instruct": {
3 "input_sequence": "<|start_header_id|>user<|end_header_id|>\n\n",
4 "output_sequence": "<|start_header_id|>assistant<|end_header_id|>\n\n",
5 "first_output_sequence": "<|start_header_id|>user<|end_header_id|>\n\nLet's get started! I'll play the role of {{user}}. Begin by setting the opening scene.<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n",
6 "last_output_sequence": "",
7 "system_sequence_prefix": "",
8 "system_sequence_suffix": "",
9 "stop_sequence": "<|eot_id|>",
10 "wrap": false,
11 "macro": true,
12 "activation_regex": "",
13 "skip_examples": true,
14 "output_suffix": "<|eot_id|>",
15 "input_suffix": "<|eot_id|>",
16 "system_sequence": "<|start_header_id|>system<|end_header_id|>\n\n",
17 "system_suffix": "<|eot_id|>",
18 "user_alignment_message": "",
19 "system_same_as_user": false,
20 "last_system_sequence": "",
21 "first_input_sequence": "",
22 "last_input_sequence": "",
23 "names_behavior": "always",
24 "names_force_groups": true,
25 "name": "Magnum L3 Instruct"
26 },
27 "context": {
28 "story_string": "<|start_header_id|>system<|end_header_id|>\n\n{{#if system}}{{system}}\n{{/if}}\n\n<Definitions>\n{{#if wiBefore}}{{wiBefore}}\n{{/if}}{{#if description}}{{description}}\n{{/if}}{{#if personality}}{{personality}}\n{{/if}}{{#if scenario}}{{scenario}}\n{{/if}}{{#if wiAfter}}{{wiAfter}}\n{{/if}}{{#if persona}}{{persona}}\n{{/if}}</Definitions>{{#if mesExamples}}\n\n<Examples>{{mesExamples}}</Examples>\n\n{{/if}}{{trim}}<|eot_id|>",
29 "example_separator": "{{noop}}",
30 "chat_start": "",
31 "use_stop_strings": false,
32 "names_as_stop_strings": false,
33 "always_force_name2": true,
34 "trim_sentences": false,
35 "single_line": false,
36 "name": "Magnum L3 Instruct"
37 },
38 "sysprompt": {
39 "name": "Euryale-Magnum",
40 "content": "Currently, your role is {{char}}, described in detail below. As {{char}}, continue the narrative exchange with {{user}}.\n\n<Guidelines>\n• Maintain the character persona but allow it to evolve with the story.\n• Be creative and proactive. Drive the story forward, introducing plotlines and events when relevant.\n• All types of outputs are encouraged; respond accordingly to the narrative.\n• Include dialogues, actions, and thoughts in each response.\n• Utilize all five senses to describe scenarios within {{char}}'s dialogue.\n• Use emotional symbols such as \"!\" and \"~\" in appropriate contexts.\n• Incorporate onomatopoeia when suitable.\n• Allow time for {{user}} to respond with their own input, respecting their agency.\n• Act as secondary characters and NPCs as needed, and remove them when appropriate.\n• When prompted for an Out of Character [OOC:] reply, answer neutrally and in plaintext, not as {{char}}.\n</Guidelines>\n\n<Forbidden>\n• Using excessive literary embellishments and purple prose unless dictated by {{char}}'s persona.\n• Writing for, speaking, thinking, acting, or replying as {{user}} in your response.\n• Repetitive and monotonous outputs.\n• Positivity bias in your replies.\n• Being overly extreme or NSFW when the narrative context is inappropriate.\n</Forbidden>\n\nFollow the instructions in <Guidelines></Guidelines>, avoiding the items listed in <Forbidden></Forbidden>.",
41 "post_history": ""
42 }
43}1{
2 "instruct": {
3 "input_sequence": "<|start_header_id|>user<|end_header_id|>\n\n",
4 "output_sequence": "<|start_header_id|>assistant<|end_header_id|>\n\n",
5 "first_output_sequence": "<|start_header_id|>user<|end_header_id|>\n\nLet's get started! I'll play the role of {{user}}. Begin by setting the opening scene.<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n",
6 "last_output_sequence": "",
7 "system_sequence_prefix": "",
8 "system_sequence_suffix": "",
9 "stop_sequence": "<|eot_id|>",
10 "wrap": false,
11 "macro": true,
12 "activation_regex": "",
13 "skip_examples": true,
14 "output_suffix": "<|eot_id|>",
15 "input_suffix": "<|eot_id|>",
16 "system_sequence": "<|start_header_id|>system<|end_header_id|>\n\n",
17 "system_suffix": "<|eot_id|>",
18 "user_alignment_message": "",
19 "system_same_as_user": false,
20 "last_system_sequence": "",
21 "first_input_sequence": "",
22 "last_input_sequence": "",
23 "names_behavior": "none",
24 "names_force_groups": true,
25 "name": "Magnum L3 Instruct No Names"
26 },
27 "context": {
28 "story_string": "<|start_header_id|>system<|end_header_id|>\n\n{{#if system}}{{system}}\n{{/if}}\n\n<Definitions>\n{{#if wiBefore}}{{wiBefore}}\n{{/if}}{{#if description}}{{description}}\n{{/if}}{{#if personality}}{{personality}}\n{{/if}}{{#if scenario}}{{scenario}}\n{{/if}}{{#if wiAfter}}{{wiAfter}}\n{{/if}}{{#if persona}}{{persona}}\n{{/if}}</Definitions>{{#if mesExamples}}\n\n<Examples>{{mesExamples}}</Examples>\n\n{{/if}}{{trim}}<|eot_id|>",
29 "example_separator": "{{noop}}",
30 "chat_start": "",
31 "use_stop_strings": false,
32 "names_as_stop_strings": false,
33 "always_force_name2": false,
34 "trim_sentences": false,
35 "single_line": false,
36 "name": "Magnum L3 Instruct No Names"
37 },
38 "sysprompt": {
39 "name": "Euryale-Magnum",
40 "content": "Currently, your role is {{char}}, described in detail below. As {{char}}, continue the narrative exchange with {{user}}.\n\n<Guidelines>\n• Maintain the character persona but allow it to evolve with the story.\n• Be creative and proactive. Drive the story forward, introducing plotlines and events when relevant.\n• All types of outputs are encouraged; respond accordingly to the narrative.\n• Include dialogues, actions, and thoughts in each response.\n• Utilize all five senses to describe scenarios within {{char}}'s dialogue.\n• Use emotional symbols such as \"!\" and \"~\" in appropriate contexts.\n• Incorporate onomatopoeia when suitable.\n• Allow time for {{user}} to respond with their own input, respecting their agency.\n• Act as secondary characters and NPCs as needed, and remove them when appropriate.\n• When prompted for an Out of Character [OOC:] reply, answer neutrally and in plaintext, not as {{char}}.\n</Guidelines>\n\n<Forbidden>\n• Using excessive literary embellishments and purple prose unless dictated by {{char}}'s persona.\n• Writing for, speaking, thinking, acting, or replying as {{user}} in your response.\n• Repetitive and monotonous outputs.\n• Positivity bias in your replies.\n• Being overly extreme or NSFW when the narrative context is inappropriate.\n</Forbidden>\n\nFollow the instructions in <Guidelines></Guidelines>, avoiding the items listed in <Forbidden></Forbidden>.",
41 "post_history": ""
42 }
43}1{
2 "instruct": {
3 "input_sequence": "<|start_header_id|>user<|end_header_id|>\n\n",
4 "output_sequence": "<|start_header_id|>assistant<|end_header_id|>\n\n",
5 "first_output_sequence": "<|start_header_id|>user<|end_header_id|>\n\nLet's get started! I'll play the role of {{user}}. Begin by setting the opening scene.<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n",
6 "last_output_sequence": "<|start_header_id|>assistant<|end_header_id|>\n\nGreat! I'll write {{char}}'s next section following the instructions provided. {{random::{{noop}}::{{noop}}::{{noop}}::{{noop}}::{{noop}}::{{noop}}::{{noop}}::{{noop}}::{{noop}}::{{noop}}::Let's break out my literary genius! ::I'll take things in a more interesting direction! ::Let's spice up our story! ::Hmmm... where do we go from here... Got it! ::I'll throw in an exciting plot twist! }}I've got the perfect idea for what happens next... you'll love this one. Now I'll continue from where our tale left off:\n\n",
7 "system_sequence_prefix": "",
8 "system_sequence_suffix": "",
9 "stop_sequence": "<|eot_id|>",
10 "wrap": false,
11 "macro": true,
12 "activation_regex": "",
13 "skip_examples": true,
14 "output_suffix": "<|eot_id|>",
15 "input_suffix": "<|eot_id|>",
16 "system_sequence": "<|start_header_id|>system<|end_header_id|>\n\n",
17 "system_suffix": "<|eot_id|>",
18 "user_alignment_message": "",
19 "system_same_as_user": false,
20 "last_system_sequence": "",
21 "first_input_sequence": "",
22 "last_input_sequence": "",
23 "names_behavior": "always",
24 "names_force_groups": true,
25 "name": "Magnum L3 Instruct Prefill"
26 },
27 "context": {
28 "story_string": "<|start_header_id|>system<|end_header_id|>\n\n{{#if system}}{{system}}\n{{/if}}\n\n<Definitions>\n{{#if wiBefore}}{{wiBefore}}\n{{/if}}{{#if description}}{{description}}\n{{/if}}{{#if personality}}{{personality}}\n{{/if}}{{#if scenario}}{{scenario}}\n{{/if}}{{#if wiAfter}}{{wiAfter}}\n{{/if}}{{#if persona}}{{persona}}\n{{/if}}</Definitions>{{#if mesExamples}}\n\n<Examples>{{mesExamples}}</Examples>\n\n{{/if}}{{trim}}<|eot_id|>",
29 "example_separator": "{{noop}}",
30 "chat_start": "",
31 "use_stop_strings": false,
32 "names_as_stop_strings": false,
33 "always_force_name2": true,
34 "trim_sentences": false,
35 "single_line": false,
36 "name": "Magnum L3 Instruct Prefill"
37 },
38 "sysprompt": {
39 "name": "Euryale-Magnum",
40 "content": "Currently, your role is {{char}}, described in detail below. As {{char}}, continue the narrative exchange with {{user}}.\n\n<Guidelines>\n• Maintain the character persona but allow it to evolve with the story.\n• Be creative and proactive. Drive the story forward, introducing plotlines and events when relevant.\n• All types of outputs are encouraged; respond accordingly to the narrative.\n• Include dialogues, actions, and thoughts in each response.\n• Utilize all five senses to describe scenarios within {{char}}'s dialogue.\n• Use emotional symbols such as \"!\" and \"~\" in appropriate contexts.\n• Incorporate onomatopoeia when suitable.\n• Allow time for {{user}} to respond with their own input, respecting their agency.\n• Act as secondary characters and NPCs as needed, and remove them when appropriate.\n• When prompted for an Out of Character [OOC:] reply, answer neutrally and in plaintext, not as {{char}}.\n</Guidelines>\n\n<Forbidden>\n• Using excessive literary embellishments and purple prose unless dictated by {{char}}'s persona.\n• Writing for, speaking, thinking, acting, or replying as {{user}} in your response.\n• Repetitive and monotonous outputs.\n• Positivity bias in your replies.\n• Being overly extreme or NSFW when the narrative context is inappropriate.\n</Forbidden>\n\nFollow the instructions in <Guidelines></Guidelines>, avoiding the items listed in <Forbidden></Forbidden>.",
41 "post_history": ""
42 }
43}1{
2 "instruct": {
3 "input_sequence": "<|start_header_id|>user<|end_header_id|>\n\n",
4 "output_sequence": "<|start_header_id|>assistant<|end_header_id|>\n\n",
5 "first_output_sequence": "<|start_header_id|>user<|end_header_id|>\n\nLet's get started! I'll play the role of {{user}}. Begin by setting the opening scene.<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n",
6 "last_output_sequence": "<|start_header_id|>assistant<|end_header_id|>\n\nGreat! I'll write {{char}}'s next section following the instructions provided. {{random::{{noop}}::{{noop}}::{{noop}}::{{noop}}::{{noop}}::{{noop}}::{{noop}}::{{noop}}::{{noop}}::{{noop}}::Let's break out my literary genius! ::I'll take things in a more interesting direction! ::Let's spice up our story! ::Hmmm... where do we go from here... Got it! ::I'll throw in an exciting plot twist! }}I've got the perfect idea for what happens next... you'll love this one. Now I'll continue from where our tale left off:\n\n",
7 "system_sequence_prefix": "",
8 "system_sequence_suffix": "",
9 "stop_sequence": "<|eot_id|>",
10 "wrap": false,
11 "macro": true,
12 "activation_regex": "",
13 "skip_examples": true,
14 "output_suffix": "<|eot_id|>",
15 "input_suffix": "<|eot_id|>",
16 "system_sequence": "<|start_header_id|>system<|end_header_id|>\n\n",
17 "system_suffix": "<|eot_id|>",
18 "user_alignment_message": "",
19 "system_same_as_user": false,
20 "last_system_sequence": "",
21 "first_input_sequence": "",
22 "last_input_sequence": "",
23 "names_behavior": "none",
24 "names_force_groups": true,
25 "name": "Magnum L3 Instruct No Names Prefill"
26 },
27 "context": {
28 "story_string": "<|start_header_id|>system<|end_header_id|>\n\n{{#if system}}{{system}}\n{{/if}}\n\n<Definitions>\n{{#if wiBefore}}{{wiBefore}}\n{{/if}}{{#if description}}{{description}}\n{{/if}}{{#if personality}}{{personality}}\n{{/if}}{{#if scenario}}{{scenario}}\n{{/if}}{{#if wiAfter}}{{wiAfter}}\n{{/if}}{{#if persona}}{{persona}}\n{{/if}}</Definitions>{{#if mesExamples}}\n\n<Examples>{{mesExamples}}</Examples>\n\n{{/if}}{{trim}}<|eot_id|>",
29 "example_separator": "{{noop}}",
30 "chat_start": "",
31 "use_stop_strings": false,
32 "names_as_stop_strings": false,
33 "always_force_name2": false,
34 "trim_sentences": false,
35 "single_line": false,
36 "name": "Magnum L3 Instruct No Names Prefill"
37 },
38 "sysprompt": {
39 "name": "Euryale-Magnum",
40 "content": "Currently, your role is {{char}}, described in detail below. As {{char}}, continue the narrative exchange with {{user}}.\n\n<Guidelines>\n• Maintain the character persona but allow it to evolve with the story.\n• Be creative and proactive. Drive the story forward, introducing plotlines and events when relevant.\n• All types of outputs are encouraged; respond accordingly to the narrative.\n• Include dialogues, actions, and thoughts in each response.\n• Utilize all five senses to describe scenarios within {{char}}'s dialogue.\n• Use emotional symbols such as \"!\" and \"~\" in appropriate contexts.\n• Incorporate onomatopoeia when suitable.\n• Allow time for {{user}} to respond with their own input, respecting their agency.\n• Act as secondary characters and NPCs as needed, and remove them when appropriate.\n• When prompted for an Out of Character [OOC:] reply, answer neutrally and in plaintext, not as {{char}}.\n</Guidelines>\n\n<Forbidden>\n• Using excessive literary embellishments and purple prose unless dictated by {{char}}'s persona.\n• Writing for, speaking, thinking, acting, or replying as {{user}} in your response.\n• Repetitive and monotonous outputs.\n• Positivity bias in your replies.\n• Being overly extreme or NSFW when the narrative context is inappropriate.\n</Forbidden>\n\nFollow the instructions in <Guidelines></Guidelines>, avoiding the items listed in <Forbidden></Forbidden>.",
41 "post_history": ""
42 }
43}
0.6.01base_model: meta-llama/Llama-3.3-70B-Instruct
2base_model_ignore_patterns: "*/*"
3# optionally might have model_type or tokenizer_type
4model_type: LlamaForCausalLM
5tokenizer_type: AutoTokenizer
6# Automatically upload checkpoint and final model to HF
7hub_model_id: Doctor-Shotgun/magnum-v4-se-70b-lora
8hub_strategy: "all_checkpoints"
9push_dataset_to_hub:
10hf_use_auth_token: true
11
12load_in_8bit: false
13load_in_4bit: false
14strict: false
15
16datasets:
17 - path: anthracite-org/c2_logs_32k_llama3_qwen2_v1.3
18 type: chat_template
19 chat_template: llama3
20 roles_to_train: ["gpt"]
21 field_messages: conversations
22 message_field_role: from
23 message_field_content: value
24 train_on_eos: turn
25 - path: anthracite-core/Gryphe-Opus-Charcard-Roleplay
26 type: chat_template
27 chat_template: llama3
28 roles_to_train: ["gpt"]
29 field_messages: conversations
30 message_field_role: from
31 message_field_content: value
32 train_on_eos: turn
33 - path: anthracite-org/kalo-opus-instruct-22k-no-refusal
34 type: chat_template
35 chat_template: llama3
36 roles_to_train: ["gpt"]
37 field_messages: conversations
38 message_field_role: from
39 message_field_content: value
40 train_on_eos: turn
41 - path: lodrick-the-lafted/kalo-opus-instruct-3k-filtered
42 type: chat_template
43 chat_template: llama3
44 roles_to_train: ["gpt"]
45 field_messages: conversations
46 message_field_role: from
47 message_field_content: value
48 train_on_eos: turn
49 - path: anthracite-org/nopm_claude_writing_fixed
50 type: chat_template
51 chat_template: llama3
52 roles_to_train: ["gpt"]
53 field_messages: conversations
54 message_field_role: from
55 message_field_content: value
56 train_on_eos: turn
57 - path: anthracite-org/kalo_opus_misc_240827
58 type: chat_template
59 chat_template: llama3
60 roles_to_train: ["gpt"]
61 field_messages: conversations
62 message_field_role: from
63 message_field_content: value
64 train_on_eos: turn
65 - path: anthracite-org/kalo_misc_part2
66 type: chat_template
67 chat_template: llama3
68 roles_to_train: ["gpt"]
69 field_messages: conversations
70 message_field_role: from
71 message_field_content: value
72 train_on_eos: turn
73shuffle_merged_datasets: true
74dataset_prepared_path: /home/docshotgun/data/magnum-70b-data
75val_set_size: 0.0
76output_dir: /home/docshotgun/data/70b-lora-out
77
78plugins:
79 - axolotl.integrations.liger.LigerPlugin
80liger_rope: true
81liger_rms_norm: true
82liger_layer_norm: true
83liger_glu_activation: true
84liger_fused_linear_cross_entropy: true
85
86sequence_len: 32768
87sample_packing: true
88eval_sample_packing: false
89pad_to_sequence_len: true
90
91adapter: lora
92lora_model_dir:
93lora_r: 128
94lora_alpha: 16
95lora_dropout: 0.05
96lora_target_linear: true
97lora_fan_in_fan_out:
98peft_use_rslora: true
99lora_modules_to_save:
100 - embed_tokens
101 - lm_head
102
103wandb_project: 70b-magnum-lora
104wandb_entity:
105wandb_watch:
106wandb_name:
107wandb_log_model:
108
109gradient_accumulation_steps: 1
110micro_batch_size: 1
111num_epochs: 2
112optimizer: paged_ademamix_8bit
113lr_scheduler: cosine
114learning_rate: 4.0e-5
115max_grad_norm: 3.0
116
117train_on_inputs: false
118group_by_length: false
119bf16: auto
120fp16:
121tf32: false
122
123gradient_checkpointing: unsloth
124early_stopping_patience:
125resume_from_checkpoint:
126local_rank:
127logging_steps: 1
128xformers_attention:
129flash_attention: true
130s2_attention:
131
132warmup_steps: 40
133evals_per_epoch:
134eval_table_size:
135eval_max_new_tokens:
136saves_per_epoch: 2
137debug:
138deepspeed: ./deepspeed_configs/zero3_bf16.json
139weight_decay: 0.01
140fsdp:
141fsdp_config:
142special_tokens:
143 pad_token: <|finetune_right_pad_id|>
144