Views
No views yet
1# taken from sophosympatheia/New-Dawn-Llama-3.1-70B-v1.1
2#
3
4merge_method: della_linear
5base_model: NousResearch/Meta-Llama-3.1-70B-Instruct
6models:
7 - model: tokyotech-llm/Llama-3-Swallow-70B-v0.1
8 parameters:
9 weight:
10 - filter: v_proj
11 value: [0, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0]
12 - filter: o_proj
13 value: [0, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0]
14 - filter: up_proj
15 value: [0, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0]
16 - filter: gate_proj
17 value: [0, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0]
18 - filter: down_proj
19 value: [0, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0]
20 - value: 0
21 density: 0.25
22 epsilon: 0.05
23 lambda: 1.0
24 - model: NousResearch/Meta-Llama-3.1-70B-Instruct
25 parameters:
26 weight: 1.0
27 density:
28 - filter: v_proj
29 value: [1, 1, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 1, 1]
30 - filter: o_proj
31 value: [1, 1, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 1, 1]
32 - filter: up_proj
33 value: [1, 1, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 1, 1]
34 - filter: gate_proj
35 value: [1, 1, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 1, 1]
36 - filter: down_proj
37 value: [1, 1, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 1, 1]
38 - value: 0.5
39 epsilon:
40 - filter: v_proj
41 value: [0, 0, 0.05, 0.05, 0.07, 0.1, 0.07, 0.05, 0.05, 0, 0]
42 - filter: o_proj
43 value: [0, 0, 0.05, 0.05, 0.07, 0.1, 0.07, 0.05, 0.05, 0, 0]
44 - filter: up_proj
45 value: [0, 0, 0.05, 0.05, 0.07, 0.1, 0.07, 0.05, 0.05, 0, 0]
46 - filter: gate_proj
47 value: [0, 0, 0.05, 0.05, 0.07, 0.1, 0.07, 0.05, 0.05, 0, 0]
48 - filter: down_proj
49 value: [0, 0, 0.05, 0.05, 0.07, 0.1, 0.07, 0.05, 0.05, 0, 0]
50 - value: 0.1
51 lambda: 1.0
52dtype: float16
53tokenizer_source: base1!pip install -qU transformers accelerate
2
3from transformers import AutoTokenizer
4import transformers
5import torch
6
7model = "KaraKaraWitch/L3.1-70b-Swallow-Saigetsu"
8messages = [{"role": "user", "content": "What is a large language model?"}]
9
10tokenizer = AutoTokenizer.from_pretrained(model)
11prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
12pipeline = transformers.pipeline(
13 "text-generation",
14 model=model,
15 torch_dtype=torch.float16,
16 device_map="auto",
17)
18
19outputs = pipeline(prompt, max_new_tokens=256, do_sample=True, temperature=0.7, top_k=50, top_p=0.95)
20print(outputs[0]["generated_text"])