Views
No views yet
optimum-intel library:1from optimum.intel import OVModelForCausalLM
2from transformers import AutoTokenizer
3
4model_id = "CelesteImperia/Mistral-Nemo-12B-Instruct-OpenVINO-INT4"
5tokenizer = AutoTokenizer.from_pretrained(model_id)
6model = OVModelForCausalLM.from_pretrained(model_id)
7
8prompt = "Explain the architectural advantages of the Mistral-Nemo 12B model."
9messages = [
10 {"role": "user", "content": prompt},
11]
12input_ids = tokenizer.apply_chat_template(messages, add_generation_prompt=True, return_tensors="pt")
13
14outputs = model.generate(input_ids, max_new_tokens=512)
15print(tokenizer.decode(outputs[0], skip_special_tokens=True))1using LLama.Common;
2using LLama;
3
4// 1. Initialize the OpenVINO Model for Mistral-Nemo 12B
5var parameters = new ModelParams("path/to/openvino_model.xml")
6{
7 ContextSize = 8192, // Optimized for high-reasoning depth
8 GpuLayerCount = 0
9};
10
11// 2. Load Weights and Create Context
12using var weights = LLamaWeights.LoadFromFile(parameters);
13using var context = weights.CreateContext(parameters);
14var executor = new StatelessExecutor(weights, parameters);
15
16// 3. Native Windows AI Inference
17var chatHistory = new ChatHistory();
18chatHistory.AddMessage(AuthorRole.User, "Analyze the benefits of local 12B models in .NET apps.");
19
20foreach (var text in executor.InferAsync(chatHistory, new InferenceParams { MaxTokens = 512 }))
21{
22 Console.Write(text);
23}| Platform | Support Link |
|---|---|
| Global & India | Support via Razorpay |
