Views
No views yet
1!git clone https://github.com/huggingface/transformers.git
2%cd transformers
3!git checkout <commit_id_for_4.47.0.dev0>
4!pip install .
5!pip install -q accelerate==0.34.2 bitsandbytes==0.44.1 peft==0.13.11import os
2import torch
3from datasets import load_dataset
4from transformers import (
5 AutoModelForCausalLM,
6 AutoTokenizer,
7 BitsAndBytesConfig,
8 pipeline,
9 logging,
10)1use_4bit = True
2
3# Compute dtype for 4-bit base models
4bnb_4bit_compute_dtype = "float16"
5
6# Quantization type (fp4 or nf4)
7compute_dtype = getattr(torch, bnb_4bit_compute_dtype)
8
9use_nested_quant = False
10
11bnb_4bit_quant_type = "nf4"
12bnb_config = BitsAndBytesConfig(
13 load_in_4bit=use_4bit,
14 bnb_4bit_quant_type=bnb_4bit_quant_type,
15 bnb_4bit_compute_dtype=compute_dtype,
16 bnb_4bit_use_double_quant=use_nested_quant,
17)1# Load base model
2model_name = 'Ahanaas/HermesWithYou'
3model = AutoModelForCausalLM.from_pretrained(
4 model_name,
5 quantization_config=bnb_config,
6 device_map=0
7)1# Load tokenizer
2tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
3tokenizer.pad_token = tokenizer.eos_token
4tokenizer.padding_side = "right"1# Run text generation pipeline with our next model
2system_prompt = ''''''
3prompt = ''''''
4
5pipe = pipeline(
6 task="text-generation",
7 model=model,
8 tokenizer=tokenizer,
9 max_new_tokens=128, # Increase this to allow for longer outputs
10 temperature=0.5, # Encourages more varied outputs
11 top_k=50, # Limits to the top 50 tokens
12 do_sample=True, # Enables sampling
13 return_full_text=True,
14)
15
16result = pipe(f"<|im_start|>system\n {system_prompt}\n<|im_end|>\n<|im_start|>user\n{prompt}\n<|im_end|>\n<|im_start|>assistant\n")
17# print(result[0]['generated_text'])
18generated_text = result[0]['generated_text']
19
20# Print the extracted response text
21print(generated_text)