Views
No views yet
1import ctranslate2
2from transformers import AutoTokenizer
3
4def generate_response(prompt: str, system_message: str, model_path: str) -> str:
5 generator = ctranslate2.Generator(
6 model_path,
7 device="cuda",
8 compute_type="int8"
9 )
10 tokenizer = AutoTokenizer.from_pretrained(model_path)
11 formatted_prompt = f"""<|im_start|>system<|im_sep|>{system_message}<|im_end|>
12<|im_start|>user<|im_sep|>{prompt}<|im_end|>
13<|im_start|>assistant<|im_sep|>"""
14 tokens = tokenizer.tokenize(formatted_prompt)
15 results = generator.generate_batch(
16 [tokens],
17 max_length=1024,
18 sampling_temperature=0.7,
19 include_prompt_in_result=False
20 )
21 response = tokenizer.decode(results[0].sequences_ids[0], skip_special_tokens=True)
22 return response
23
24if __name__ == "__main__":
25 model_path = "path/to/your/phi-4-ct2-model"
26 system_message = "You are a helpful AI assistant."
27 user_prompt = "Write a short poem about a cat."
28 response = generate_response(user_prompt, system_message, model_path)
29 print("\nGenerated response:")
30 print(response)
311import ctranslate2
2from transformers import AutoTokenizer
3import sys
4
5def generate_response(prompt: str, system_message: str, model_path: str) -> None:
6 generator = ctranslate2.Generator(model_path, device="cuda", compute_type="int8")
7 tokenizer = AutoTokenizer.from_pretrained(model_path)
8 formatted_prompt = f"""<|im_start|>system<|im_sep|>{system_message}<|im_end|>
9<|im_start|>user<|im_sep|>{prompt}<|im_end|>
10<|im_start|>assistant<|im_sep|>"""
11 tokens: List[str] = tokenizer.tokenize(formatted_prompt)
12 for step in generator.generate_tokens([tokens], max_length=1024, sampling_temperature=0.7):
13 token: str = step.token
14 if token in tokenizer.eos_token or token in tokenizer.all_special_tokens:
15 break
16 decoded_token: str = tokenizer.decode([step.token_id])
17 print(decoded_token, end="", flush=True)
18
19if __name__ == "__main__":
20 model_path = "path/to/your/phi-4-ct2-model"
21 system_message = "You are a helpful AI assistant."
22 user_prompt = "Write a short poem about a cat."
23 print("\nGenerating response:")
24 generate_response(user_prompt, system_message, model_path)