Views
No views yet
1import ctranslate2
2import transformers
3from huggingface_hub import snapshot_download
4
5model_dir = snapshot_download(repo_id="Praise2112/Mistral-7B-Instruct-v0.1-int8-ct2")
6generator = ctranslate2.Generator(model_dir, device="cuda", compute_type="int8") # GPU
7# generator = ctranslate2.Generator(model_dir, device="cpu", compute_type="int8") #CPU
8tokenizer = transformers.AutoTokenizer.from_pretrained("mistralai/Mistral-7B-Instruct-v0.1")
9
10messages = [
11 {"role": "user", "content": "What is your favourite condiment?"},
12 {"role": "assistant", "content": "Well, I'm quite partial to a good squeeze of fresh lemon juice. It adds just the right amount of zesty flavour to whatever I'm cooking up in the kitchen!"},
13 {"role": "user", "content": "Do you have mayonnaise recipes?"}
14]
15
16model_inputs = tokenizer.apply_chat_template(messages, return_tensors="pt")
17model_inputs = [tokenizer.convert_ids_to_tokens(model_input) for model_input in model_inputs]
18generated_ids = generator.generate_batch(model_inputs, max_length=1000, sampling_topk=10)
19decoded = [res.sequences_ids[0] for res in generated_ids]
20decoded = tokenizer.batch_decode(decoded)
21print(decoded[0])