Views
No views yet
| Name | Quant method | Size |
|---|---|---|
| Llama-3-8B-chat-32K.Q2_K.gguf | Q2_K | 2.96GB |
| Llama-3-8B-chat-32K.IQ3_XS.gguf | IQ3_XS | 3.28GB |
| Llama-3-8B-chat-32K.IQ3_S.gguf | IQ3_S | 3.43GB |
| Llama-3-8B-chat-32K.Q3_K_S.gguf | Q3_K_S | 3.41GB |
| Llama-3-8B-chat-32K.IQ3_M.gguf | IQ3_M | 3.52GB |
| Llama-3-8B-chat-32K.Q3_K.gguf | Q3_K | 3.74GB |
| Llama-3-8B-chat-32K.Q3_K_M.gguf | Q3_K_M | 3.74GB |
| Llama-3-8B-chat-32K.Q3_K_L.gguf | Q3_K_L | 4.03GB |
| Llama-3-8B-chat-32K.IQ4_XS.gguf | IQ4_XS | 4.18GB |
| Llama-3-8B-chat-32K.Q4_0.gguf | Q4_0 | 4.34GB |
| Llama-3-8B-chat-32K.IQ4_NL.gguf | IQ4_NL | 4.38GB |
| Llama-3-8B-chat-32K.Q4_K_S.gguf | Q4_K_S | 4.37GB |
| Llama-3-8B-chat-32K.Q4_K.gguf | Q4_K | 4.58GB |
| Llama-3-8B-chat-32K.Q4_K_M.gguf | Q4_K_M | 4.58GB |
| Llama-3-8B-chat-32K.Q4_1.gguf | Q4_1 | 4.78GB |
| Llama-3-8B-chat-32K.Q5_0.gguf | Q5_0 | 5.21GB |
| Llama-3-8B-chat-32K.Q5_K_S.gguf | Q5_K_S | 5.21GB |
| Llama-3-8B-chat-32K.Q5_K.gguf | Q5_K | 5.34GB |
| Llama-3-8B-chat-32K.Q5_K_M.gguf | Q5_K_M | 5.34GB |
| Llama-3-8B-chat-32K.Q5_1.gguf | Q5_1 | 5.65GB |
| Llama-3-8B-chat-32K.Q6_K.gguf | Q6_K | 6.14GB |
| Llama-3-8B-chat-32K.Q8_0.gguf | Q8_0 | 7.95GB |
1from transformers import GenerationConfig, LlamaForCausalLM, LlamaTokenizer,AutoTokenizer,AutoModelForCausalLM,MistralForCausalLM
2import torch
3
4
5
6model_id = Moses25/Llama-3-8B-chat-32K
7tokenizer = AutoTokenizer.from_pretrained(model_id)
8model = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=torch.bfloat16, device_map="auto")
9
10mistral_template="{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 and system_message != false %}{% set content = '<<SYS>>\\n' + system_message + '\\n<</SYS>>\\n\\n' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ bos_token + '[INST] ' + content.strip() + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ ' ' + content.strip() + ' ' + eos_token }}{% endif %}{% endfor %}"
11
12llama3_template="{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}"
13
14def chat_format(conversation:list,tokenizer,chat_type="mistral"):
15 system_prompt = "You are a helpful, respectful and honest assistant.Help humman as much as you can."
16 ap = [{"role":"system","content":system_prompt}] + conversation
17 if chat_type=='mistral':
18 id = tokenizer.apply_chat_template(ap,chat_template=mistral_template,tokenize=False)
19 elif chat_type=='llama3':
20 id = tokenizer.apply_chat_template(ap,chat_template=llama3_template,tokenize=False)
21 #id = id.rstrip("<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n")
22 return id
23
24user_chat=[{"role":"user","content":"In a basket, there are 20 oranges, 60 apples, and 40 bananas. If 15 pears were added, and half of the oranges were removed, what would be the new ratio of oranges to apples, bananas, and pears combined within the basket?"}]
25text = chat_format(user_chat,tokenizer,'llama3')
26def predict(content_prompt):
27 inputs = tokenizer(content_prompt,return_tensors="pt",add_special_tokens=True)
28 input_ids = inputs["input_ids"].to("cuda:0")
29 # print(f"input length:{len(input_ids[0])}")
30 with torch.no_grad():
31 generation_output = model.generate(
32 input_ids=input_ids,
33 #generation_config=generation_config,
34 return_dict_in_generate=True,
35 output_scores=True,
36 max_new_tokens=2048,
37 top_p=0.9,
38 num_beams=1,
39 do_sample=True,
40 repetition_penalty=1.0,
41 eos_token_id=tokenizer.convert_tokens_to_ids("<|eot_id|>"),
42 pad_token_id=tokenizer.convert_tokens_to_ids("<|eot_id|>"),
43 )
44 s = generation_output.sequences[0]
45 output = tokenizer.decode(s,skip_special_tokens=False)
46 output1 = output.split("<|eot_id|>")[-2].lstrip("<|start_header_id|>assistant<|end_header_id|>").strip()
47 # print(output1)
48 return output1
49
50predict(text)
51output："""Let's break down the steps to find the new ratio of oranges to apples, bananas, and pears combined:
52Calculate the total number of fruits initially in the basket: Oranges: 20 Apples: 60 Bananas: 40 Total Fruits = 20 + 60 + 40 = 120
53Add 15 pears: Total Fruits after adding pears = 120 + 15 = 135
54Remove half of the oranges: Oranges remaining = 20 / 2 = 10
55Calculate the total number of fruits remaining in the basket after removing half of the oranges: Total Remaining Fruits = 10 (oranges) + 60 (apples) + 40 (bananas) + 15 (pears) = 125
56Find the ratio of oranges to apples, bananas, and pears combined: Ratio of Oranges to (Apples, Bananas, Pears) Combined = Oranges / (Apples + Bananas + Pears) = 10 / (60 + 40 + 15) = 10 / 115
57So, the new ratio of oranges to apples, bananas, and pears combined within the basket is 10:115.
58However, I should note that the actual fruit distribution in your basket may vary depending on how you decide to count and categorize the fruits. The example calculation provides a theoretical ratio based on the initial quantities mentioned."""1#llama3-chat-template.jinja file is chat-template above 'llama3-template'
2model_path = Llama-3-8B-chat-32K
3python -m vllm.entrypoints.openai.api_server --model=$model_path \
4 --trust-remote-code --host 0.0.0.0 --port 7777 \
5 --gpu-memory-utilization 0.8 \
6 --max-model-len 8192 --chat-template llama3-chat-template.jinja \
7 --tensor-parallel-size 1 --served-model-name chatbot1from openai import OpenAI
2# Set OpenAI's API key and API base to use vLLM's API server.
3openai_api_key = "EMPTY"
4openai_api_base = "http://localhost:7777/v1"
5
6client = OpenAI(
7 api_key=openai_api_key,
8 base_url=openai_api_base,
9)
10call_args = {
11 'temperature': 0.7,
12 'top_p': 0.9,
13 'top_k': 40,
14 'max_tokens': 2048, # output-len
15 'presence_penalty': 1.0,
16 'frequency_penalty': 0.0,
17 "repetition_penalty":1.0,
18 "stop":["<|eot_id|>","<|end_of_text|>"],
19 }
20chat_response = client.chat.completions.create(
21 model="chatbot",
22 messages=[
23 {"role": "user", "content": "你好"},
24 ],
25 extra_body=call_args
26)
27print("Chat response:", chat_response)
28
29