Views
No views yet
| Name | Quant method | Size |
|---|---|---|
| Vikhr-7B-instruct_0.2.Q2_K.gguf | Q2_K | 2.48GB |
| Vikhr-7B-instruct_0.2.IQ3_XS.gguf | IQ3_XS | 2.74GB |
| Vikhr-7B-instruct_0.2.IQ3_S.gguf | IQ3_S | 2.88GB |
| Vikhr-7B-instruct_0.2.Q3_K_S.gguf | Q3_K_S | 2.88GB |
| Vikhr-7B-instruct_0.2.IQ3_M.gguf | IQ3_M | 3.04GB |
| Vikhr-7B-instruct_0.2.Q3_K.gguf | Q3_K | 3.21GB |
| Vikhr-7B-instruct_0.2.Q3_K_M.gguf | Q3_K_M | 3.21GB |
| Vikhr-7B-instruct_0.2.Q3_K_L.gguf | Q3_K_L | 3.49GB |
| Vikhr-7B-instruct_0.2.IQ4_XS.gguf | IQ4_XS | 3.54GB |
| Vikhr-7B-instruct_0.2.Q4_0.gguf | Q4_0 | 3.71GB |
| Vikhr-7B-instruct_0.2.IQ4_NL.gguf | IQ4_NL | 3.73GB |
| Vikhr-7B-instruct_0.2.Q4_K_S.gguf | Q4_K_S | 3.74GB |
| Vikhr-7B-instruct_0.2.Q4_K.gguf | Q4_K | 3.95GB |
| Vikhr-7B-instruct_0.2.Q4_K_M.gguf | Q4_K_M | 3.95GB |
| Vikhr-7B-instruct_0.2.Q4_1.gguf | Q4_1 | 4.1GB |
| Vikhr-7B-instruct_0.2.Q5_0.gguf | Q5_0 | 4.5GB |
| Vikhr-7B-instruct_0.2.Q5_K_S.gguf | Q5_K_S | 4.5GB |
| Vikhr-7B-instruct_0.2.Q5_K.gguf | Q5_K | 4.62GB |
| Vikhr-7B-instruct_0.2.Q5_K_M.gguf | Q5_K_M | 4.62GB |
| Vikhr-7B-instruct_0.2.Q5_1.gguf | Q5_1 | 4.89GB |
| Vikhr-7B-instruct_0.2.Q6_K.gguf | Q6_K | 5.33GB |
| Vikhr-7B-instruct_0.2.Q8_0.gguf | Q8_0 | 6.9GB |
1from peft import PeftModel, PeftConfig
2from transformers import AutoModelForCausalLM, AutoTokenizer, GenerationConfig
3import torch
4import os
5os.environ['HF_HOME']='.'
6MODEL_NAME = "Vikhrmodels/Vikhr-7B-instruct_0.2"
7DEFAULT_MESSAGE_TEMPLATE = "<s>{role}\n{content}</s>\n"
8DEFAULT_SYSTEM_PROMPT = "Ты — Вихрь, русскоязычный автоматический ассистент. Ты разговариваешь с людьми и помогаешь им."
9
10class Conversation:
11 def __init__(
12 self,
13 message_template=DEFAULT_MESSAGE_TEMPLATE,
14 system_prompt=DEFAULT_SYSTEM_PROMPT,
15 ):
16 self.message_template = message_template
17 self.messages = [{
18 "role": "system",
19 "content": system_prompt
20 }]
21
22 def add_user_message(self, message):
23 self.messages.append({
24 "role": "user",
25 "content": message
26 })
27
28 def get_prompt(self, tokenizer):
29 final_text = ""
30 for message in self.messages:
31 message_text = self.message_template.format(**message)
32 final_text += message_text
33 final_text += 'bot'
34 return final_text.strip()
35
36
37def generate(model, tokenizer, prompt, generation_config):
38 data = tokenizer(prompt, return_tensors="pt")
39 data = {k: v.to(model.device) for k, v in data.items()}
40 output_ids = model.generate(
41 **data,
42 generation_config=generation_config
43 )[0]
44 output_ids = output_ids[len(data["input_ids"][0]):]
45 output = tokenizer.decode(output_ids, skip_special_tokens=True)
46 return output.strip()
47
48#config = PeftConfig.from_pretrained(MODEL_NAME)
49model = AutoModelForCausalLM.from_pretrained(
50 MODEL_NAME,
51 load_in_8bit=True,
52 torch_dtype=torch.float16,
53 device_map="auto"
54)
55#model = PeftModel.from_pretrained( model, MODEL_NAME, torch_dtype=torch.float16)
56model.eval()
57
58tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME, use_fast=False)
59
60generation_config = GenerationConfig.from_pretrained(MODEL_NAME)
61generation_config.max_length=256
62generation_config.top_p=0.9
63generation_config.top_k=30
64generation_config.do_sample = True
65print(generation_config)
66
67inputs = ["Как тебя зовут?", "Кто такой Колмогоров?"]
68
69for inp in inputs:
70 conversation = Conversation()
71 conversation.add_user_message(inp)
72 prompt = conversation.get_prompt(tokenizer)
73
74 output = generate(model, tokenizer, prompt, generation_config)
75 print(inp)
76 print(output)
77 print('\n')