from transformers import GenerationConfig, LlamaForCausalLM, LlamaTokenizer,AutoTokenizer,AutoModelForCausalLM,MistralForCausalLM
import torch
model = AutoModelForCausalLM.from_pretrained(model_id,torch_dtype=torch.bfloat16,device_map="auto",)
from transformers import GenerationConfig, LlamaForCausalLM, LlamaTokenizer,AutoTokenizer,AutoModelForCausalLM,MistralForCausalLM
import torch


model_id=Moses25/Mistral-7B-chat-32k 
tokenizer = AutoTokenizer.from_pretrained(model_id)


mistral_template="{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 and system_message != false %}{% set content = '<<SYS>>\\n' + system_message + '\\n<</SYS>>\\n\\n' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ bos_token + '[INST] ' + content.strip() + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ ' '  + content.strip() + ' ' + eos_token }}{% endif %}{% endfor %}"

llama3_template="{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}"

def chat_format(conversation:list,tokenizer,chat_type="mistral"):
    system_prompt = "You are a helpful, respectful and honest assistant.Help humman as much as you can."
    ap = [{"role":"system","content":system_prompt}] + conversation
    if chat_type=='mistral':
        id = tokenizer.apply_chat_template(ap,chat_template=mistral_template,tokenize=False)
    elif chat_type=='llama3':
        id = tokenizer.apply_chat_template(ap,chat_template=llama3_template,tokenize=False)
        id = id.rstrip("<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n")
    return id

user_chat=[{"role":"user","content":"In a basket, there are 20 oranges, 60 apples, and 40 bananas. If 15 pears were added, and half of the oranges were removed, what would be the new ratio of oranges to apples, bananas, and pears combined within the basket?"}]
text = chat_format(user_chat,tokenizer,'mistral')
def predict(content_prompt):
    inputs = tokenizer(content_prompt,return_tensors="pt",add_special_tokens=True)
    input_ids = inputs["input_ids"].to("cuda:0")
    # print(f"input length:{len(input_ids[0])}")
    with torch.no_grad():
        generation_output = model.generate(
                    input_ids=input_ids,
                    #generation_config=generation_config,
                    return_dict_in_generate=True,
                    output_scores=True,
                    max_new_tokens=2048,
                    top_p=0.9,
                    num_beams=1,
                    do_sample=True,
                    repetition_penalty=1.0,
                    eos_token_id=tokenizer.eos_token_id,
                    pad_token_id=tokenizer.pad_token_id,
                )
        s = generation_output.sequences[0]
        output = tokenizer.decode(s,skip_special_tokens=True)
        output1 = output.split("[/INST]")[-1].strip()
        # print(output1)
    return output1

predict(text)
output："""Let's break down the steps to find the new ratio of oranges to apples, bananas, and pears combined:
Calculate the total number of fruits initially in the basket: Oranges: 20 Apples: 60 Bananas: 40 Total Fruits = 20 + 60 + 40 = 120
Add 15 pears: Total Fruits after adding pears = 120 + 15 = 135
Remove half of the oranges: Oranges remaining = 20 / 2 = 10
Calculate the total number of fruits remaining in the basket after removing half of the oranges: Total Remaining Fruits = 10 (oranges) + 60 (apples) + 40 (bananas) + 15 (pears) = 125
Find the ratio of oranges to apples, bananas, and pears combined: Ratio of Oranges to (Apples, Bananas, Pears) Combined = Oranges / (Apples + Bananas + Pears) = 10 / (60 + 40 + 15) = 10 / 115
So, the new ratio of oranges to apples, bananas, and pears combined within the basket is 10:115.
However, I should note that the actual fruit distribution in your basket may vary depending on how you decide to count and categorize the fruits. The example calculation provides a theoretical ratio based on the initial quantities mentioned."""

vLLM server

shell

#llama2-chat-template.jinja file is chat-template above 'mistral-template'
model_path="Moses25/Mistral-7B-chat-32k"
python  -m vllm.entrypoints.openai.api_server --model=$model_path \
        --trust-remote-code --host 0.0.0.0  --port 7777 \
        --gpu-memory-utilization 0.8 \
        --max-model-len 8192 --chat-template llama2-chat-template.jinja \
        --tensor-parallel-size 1 --served-model-name chatbot

python

from openai import OpenAI
# Set OpenAI's API key and API base to use vLLM's API server.
openai_api_key = "EMPTY"
openai_api_base = "http://localhost:7777/v1"

client = OpenAI(
    api_key=openai_api_key,
    base_url=openai_api_base,
)
call_args = {
        'temperature': 0.7,
        'top_p': 0.9,
        'top_k': 40,
        'max_tokens': 2048, # output-len
        'presence_penalty': 1.0,
        'frequency_penalty': 0.0,
        "repetition_penalty":1.0,
        "stop":["</s>"],
    }
chat_response = client.chat.completions.create(
    model="chatbot",
    messages=[
        {"role": "user", "content": "你好"},
    ],
    extra_body=call_args
)
print("Chat response:", chat_response)

ollama ModelFile

shell

TEMPLATE """[INST] <<SYS>>
{{ .System }}
<</SYS>>

{{ .Prompt }} [/INST] """
PARAMETER stop [INST]
PARAMETER stop [/INST]
PARAMETER stop <<SYS>>
PARAMETER stop <</SYS>>
PARAMETER num_ctx 16384
PARAMETER temperature 0.7

SYSTEM """You are a helpful, respectful and honest assistant.Help humman as much as you can."""