Views
No views yet
mistral_inference: See heretransformers: See hereNeMo: See nvidia/Mistral-NeMo-12B-Instructmistralai/Mistral-Nemo-Instruct-2407 with mistral-inference. For HF transformers code snippets, please keep scrolling.pip install mistral_inference1from huggingface_hub import snapshot_download
2from pathlib import Path
3
4mistral_models_path = Path.home().joinpath('mistral_models', 'Nemo-Instruct')
5mistral_models_path.mkdir(parents=True, exist_ok=True)
6
7snapshot_download(repo_id="mistralai/Mistral-Nemo-Instruct-2407", allow_patterns=["params.json", "consolidated.safetensors", "tekken.json"], local_dir=mistral_models_path)mistral_inference, a mistral-chat CLI command should be available in your environment. You can chat with the model usingmistral-chat $HOME/mistral_models/Nemo-Instruct --instruct --max_tokens 256 --temperature 0.35How expensive would it be to ask a window cleaner to clean all windows in Paris. Make a reasonable guess in US Dollar.1from mistral_inference.transformer import Transformer
2from mistral_inference.generate import generate
3
4from mistral_common.tokens.tokenizers.mistral import MistralTokenizer
5from mistral_common.protocol.instruct.messages import UserMessage
6from mistral_common.protocol.instruct.request import ChatCompletionRequest
7
8tokenizer = MistralTokenizer.from_file(f"{mistral_models_path}/tekken.json")
9model = Transformer.from_folder(mistral_models_path)
10
11prompt = "How expensive would it be to ask a window cleaner to clean all windows in Paris. Make a reasonable guess in US Dollar."
12
13completion_request = ChatCompletionRequest(messages=[UserMessage(content=prompt)])
14
15tokens = tokenizer.encode_chat_completion(completion_request).tokens
16
17out_tokens, _ = generate([tokens], model, max_tokens=64, temperature=0.35, eos_id=tokenizer.instruct_tokenizer.tokenizer.eos_id)
18result = tokenizer.decode(out_tokens[0])
19
20print(result)1from mistral_common.protocol.instruct.tool_calls import Function, Tool
2from mistral_inference.transformer import Transformer
3from mistral_inference.generate import generate
4
5from mistral_common.tokens.tokenizers.mistral import MistralTokenizer
6from mistral_common.protocol.instruct.messages import UserMessage
7from mistral_common.protocol.instruct.request import ChatCompletionRequest
8
9
10tokenizer = MistralTokenizer.from_file(f"{mistral_models_path}/tekken.json")
11model = Transformer.from_folder(mistral_models_path)
12
13completion_request = ChatCompletionRequest(
14 tools=[
15 Tool(
16 function=Function(
17 name="get_current_weather",
18 description="Get the current weather",
19 parameters={
20 "type": "object",
21 "properties": {
22 "location": {
23 "type": "string",
24 "description": "The city and state, e.g. San Francisco, CA",
25 },
26 "format": {
27 "type": "string",
28 "enum": ["celsius", "fahrenheit"],
29 "description": "The temperature unit to use. Infer this from the users location.",
30 },
31 },
32 "required": ["location", "format"],
33 },
34 )
35 )
36 ],
37 messages=[
38 UserMessage(content="What's the weather like today in Paris?"),
39 ],
40)
41
42tokens = tokenizer.encode_chat_completion(completion_request).tokens
43
44out_tokens, _ = generate([tokens], model, max_tokens=256, temperature=0.35, eos_id=tokenizer.instruct_tokenizer.tokenizer.eos_id)
45result = tokenizer.decode(out_tokens[0])
46
47print(result)[!IMPORTANT] NOTE: Until a new release has been made, you need to install transformers from source:pip install git+https://github.com/huggingface/transformers.git
transformers to generate text, you can do something like this.1from transformers import pipeline
2
3messages = [
4 {"role": "system", "content": "You are a pirate chatbot who always responds in pirate speak!"},
5 {"role": "user", "content": "Who are you?"},
6]
7chatbot = pipeline("text-generation", model="mistralai/Mistral-Nemo-Instruct-2407")
8chatbot(messages)[!TIP] Unlike previous Mistral models, Mistral Nemo requires smaller temperatures. We recommend to use a temperature of 0.3.