Views
No views yet
1!python -m pip install --upgrade pip -q
2!pip install -q accelerate safetensors deepspeed
3!pip install -q bitsandbytes sentencepiece
4!pip install -q scipy ninja -U
5!pip install git+https://github.com/mobiusml/hqq/ transformers -U -q1import transformers
2print(transformers.__version__)
3### output: 4.39.21model_id = 'NickyNicky/dolphin-2.8-mistral-7b-v02-nbits-4-group_size-128-HQQ'
2
3from hqq.engine.hf import HQQModelForCausalLM, AutoTokenizer
4
5
6
7tokenizer = AutoTokenizer.from_pretrained(model_id)
8model = HQQModelForCausalLM.from_quantized(model_id,device="cuda:0")
9
10model.config.use_cache = True
11model.eval();
12
131system=""""""
2
3contenido="""escribe solo tres palabaras que contengan la letra 'T'"""
4
5messages = [{"role": "system", "content": system+"eres un modelo de AI que responde adecuadamente a las tareas exactas que te pide el usuario, el idioma a la cual debes de responder es español."},
6 {"role": "user", "content": contenido},
7
8 ]
9
10prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
11
12input= tokenizer(prompt,
13 return_tensors="pt",
14 add_special_tokens=False).to(model.device)
15
16generate_params = dict(
17 max_new_tokens = 1900,
18 do_sample = True,
19 top_p = 0.90,
20 top_k = 50,
21 temperature = 0.6,
22 repetition_penalty = 1.,
23 pad_token_id = tokenizer.eos_token_id,
24 eos_token_id = tokenizer.eos_token_id,
25 )
26
27
28output= model.generate(**input,**generate_params)
29print(tokenizer.decode(output[0], skip_special_tokens=True))
https://colab.research.google.com/drive/1oEoH0qScGzkLV4WLGrMEMgl4qnEsZhTs?usp=sharing