Views
No views yet
1slices:
2
3 - sources:
4 - model: NousResearch/Meta-Llama-3-8B
5 layer_range: [3, 4]
6
7 - sources:
8 - model: NousResearch/Meta-Llama-3-8B
9 layer_range: [5, 6]
10
11 - sources:
12 - model: NousResearch/Meta-Llama-3-8B
13 layer_range: [7, 8]
14
15 - sources:
16 - model: NousResearch/Meta-Llama-3-8B
17 layer_range: [9, 10]
18
19 - sources:
20 - model: NousResearch/Meta-Llama-3-8B
21 layer_range: [10, 11]
22
23 - sources:
24 - model: NousResearch/Meta-Llama-3-8B
25 layer_range: [11, 12]
26
27 - sources:
28 - model: NousResearch/Meta-Llama-3-8B
29 layer_range: [13, 14]
30
31 - sources:
32 - model: NousResearch/Meta-Llama-3-8B
33 layer_range: [15, 16]
34
35 - sources:
36 - model: NousResearch/Meta-Llama-3-8B
37 layer_range: [16, 17]
38
39 - sources:
40 - model: NousResearch/Meta-Llama-3-8B
41 layer_range: [19, 20]
42
43 - sources:
44 - model: NousResearch/Meta-Llama-3-8B
45 layer_range: [22, 23]
46
47 - sources:
48 - model: NousResearch/Meta-Llama-3-8B
49 layer_range: [24, 25]
50
51 - sources:
52 - model: NousResearch/Meta-Llama-3-8B
53 layer_range: [25, 26]
54
55 - sources:
56 - model: NousResearch/Meta-Llama-3-8B
57 layer_range: [27, 28]
58
59 - sources:
60 - model: NousResearch/Meta-Llama-3-8B
61 layer_range: [28, 29]
62
63 - sources:
64 - model: NousResearch/Meta-Llama-3-8B
65 layer_range: [30, 31]
66
67merge_method: passthrough
68dtype: bfloat161!pip install -qU transformers accelerate
2
3from transformers import AutoTokenizer
4import transformers
5import torch
6
7model = "ryan0712/llama-3-8b-slow-DUS-random-layer1-method2"
8messages = [{"role": "user", "content": "What is a large language model?"}]
9
10tokenizer = AutoTokenizer.from_pretrained(model)
11prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
12pipeline = transformers.pipeline(
13 "text-generation",
14 model=model,
15 torch_dtype=torch.float16,
16 device_map="auto",
17)
18
19outputs = pipeline(prompt, max_new_tokens=256, do_sample=True, temperature=0.7, top_k=50, top_p=0.95)
20print(outputs[0]["generated_text"])