Views
No views yet
1from mistral_inference.model import Transformer
2from mistral_inference.generate import generate
3from mistral_common.tokens.tokenizers.mistral import MistralTokenizer
4from mistral_common.protocol.instruct.messages import UserMessage
5from mistral_common.protocol.instruct.request import ChatCompletionRequest
6
7def main():
8 tokenizer = MistralTokenizer.from_file("model/tokenizer.model.v3")
9 model = Transformer.from_folder("model")
10 model.load_lora("lora/lora.safetensors")
11
12 completion_request = ChatCompletionRequest(messages=[UserMessage(content="Explain Machine Learning to me in a nutshell.")])
13 tokens = tokenizer.encode_chat_completion(completion_request).tokens
14 out_tokens, _ = generate([tokens], model, max_tokens=64, temperature=0.0, eos_id=tokenizer.instruct_tokenizer.tokenizer.eos_id)
15 result = tokenizer.instruct_tokenizer.tokenizer.decode(out_tokens[0])
16 print(result)
17
18if __name__ == "__main__":
19 main()