Views
No views yet
1from transformers import GenerationConfig, LlamaForCausalLM, LlamaTokenizer,AutoTokenizer,AutoModelForCausalLM,MistralForCausalLM
2import torch
3model = AutoModelForCausalLM.from_pretrained(model_id,torch_dtype=torch.bfloat16,device_map="auto",)
4from transformers import GenerationConfig, LlamaForCausalLM, LlamaTokenizer,AutoTokenizer,AutoModelForCausalLM,MistralForCausalLM
5import torch
6
7
8model_id=Moses25/Meta-LlaMA-3-8B-Instruct-16k
9tokenizer = AutoTokenizer.from_pretrained(model_id)
10
11
12mistral_template="{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 and system_message != false %}{% set content = '<<SYS>>\\n' + system_message + '\\n<</SYS>>\\n\\n' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ bos_token + '[INST] ' + content.strip() + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ ' ' + content.strip() + ' ' + eos_token }}{% endif %}{% endfor %}"
13
14llama3_template="{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}"
15
16def chat_format(conversation:list,tokenizer,chat_type="mistral"):
17 system_prompt = "You are a helpful, respectful and honest assistant.Help humman as much as you can."
18 ap = [{"role":"system","content":system_prompt}] + conversation
19 if chat_type=='mistral':
20 id = tokenizer.apply_chat_template(ap,chat_template=mistral_template,tokenize=False)
21 elif chat_type=='llama3':
22 id = tokenizer.apply_chat_template(ap,chat_template=llama3_template,tokenize=False)
23 id = id.rstrip("<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n")
24 return id
25
26user_chat=[{"role":"user","content":"In a basket, there are 20 oranges, 60 apples, and 40 bananas. If 15 pears were added, and half of the oranges were removed, what would be the new ratio of oranges to apples, bananas, and pears combined within the basket?"}]
27text = chat_format(user_chat,tokenizer,'mistral')
28def predict(content_prompt):
29 inputs = tokenizer(content_prompt,return_tensors="pt",add_special_tokens=True)
30 input_ids = inputs["input_ids"].to("cuda:0")
31 # print(f"input length:{len(input_ids[0])}")
32 with torch.no_grad():
33 generation_output = model.generate(
34 input_ids=input_ids,
35 #generation_config=generation_config,
36 return_dict_in_generate=True,
37 output_scores=True,
38 max_new_tokens=2048,
39 top_p=0.9,
40 num_beams=1,
41 do_sample=True,
42 repetition_penalty=1.0,
43 eos_token_id=tokenizer.eos_token_id,
44 pad_token_id=tokenizer.pad_token_id,
45 )
46 s = generation_output.sequences[0]
47 output = tokenizer.decode(s,skip_special_tokens=True)
48 output1 = output.split("[/INST]")[-1].strip()
49 # print(output1)
50 return output1
51
52predict(text)
53output："""Let's break down the steps to find the new ratio of oranges to apples, bananas, and pears combined:
54Calculate the total number of fruits initially in the basket: Oranges: 20 Apples: 60 Bananas: 40 Total Fruits = 20 + 60 + 40 = 120
55Add 15 pears: Total Fruits after adding pears = 120 + 15 = 135
56Remove half of the oranges: Oranges remaining = 20 / 2 = 10
57Calculate the total number of fruits remaining in the basket after removing half of the oranges: Total Remaining Fruits = 10 (oranges) + 60 (apples) + 40 (bananas) + 15 (pears) = 125
58Find the ratio of oranges to apples, bananas, and pears combined: Ratio of Oranges to (Apples, Bananas, Pears) Combined = Oranges / (Apples + Bananas + Pears) = 10 / (60 + 40 + 15) = 10 / 115
59So, the new ratio of oranges to apples, bananas, and pears combined within the basket is 10:115.
60However, I should note that the actual fruit distribution in your basket may vary depending on how you decide to count and categorize the fruits. The example calculation provides a theoretical ratio based on the initial quantities mentioned."""1#llama2-chat-template.jinja file is chat-template above 'mistral-template'
2model_path=Meta-LlaMA-3-8B-Instruct-16k
3python -m vllm.entrypoints.openai.api_server --model=$model_path \
4 --trust-remote-code --host 0.0.0.0 --port 7777 \
5 --gpu-memory-utilization 0.8 \
6 --max-model-len 8192 --chat-template llama2-chat-template.jinja \
7 --tensor-parallel-size 1 --served-model-name chatbot1from openai import OpenAI
2# Set OpenAI's API key and API base to use vLLM's API server.
3openai_api_key = "EMPTY"
4openai_api_base = "http://localhost:7777/v1"
5
6client = OpenAI(
7 api_key=openai_api_key,
8 base_url=openai_api_base,
9)
10call_args = {
11 'temperature': 0.7,
12 'top_p': 0.9,
13 'top_k': 40,
14 'max_tokens': 2048, # output-len
15 'presence_penalty': 1.0,
16 'frequency_penalty': 0.0,
17 "repetition_penalty":1.0,
18 "stop":["</s>"],
19 }
20chat_response = client.chat.completions.create(
21 model="chatbot",
22 messages=[
23 {"role": "user", "content": "你好"},
24 ],
25 extra_body=call_args
26)
27print("Chat response:", chat_response)
28
291TEMPLATE """[INST] <<SYS>>
2{{ .System }}
3<</SYS>>
4
5{{ .Prompt }} [/INST] """
6PARAMETER stop [INST]
7PARAMETER stop [/INST]
8PARAMETER stop <<SYS>>
9PARAMETER stop <</SYS>>
10PARAMETER num_ctx 16384
11PARAMETER temperature 0.7
12
13SYSTEM """You are a helpful, respectful and honest assistant.Help humman as much as you can."""