Views
No views yet
1models:
2 - model: NousResearch/Meta-Llama-3-8B
3 # No parameters necessary for base model
4 - model: mlabonne/ChimeraLlama-3-8B-v2
5 parameters:
6 density: 0.33
7 weight: 0.2
8 - model: nbeerbower/llama-3-stella-8B
9 parameters:
10 density: 0.44
11 weight: 0.4
12 - model: uygarkurt/llama-3-merged-linear
13 parameters:
14 density: 0.55
15 weight: 0.4
16merge_method: dare_ties
17base_model: NousResearch/Meta-Llama-3-8B
18parameters:
19 int8_mask: true
20dtype: float16
1!pip install -qU transformers accelerate bitsandbytes
2
3from transformers import AutoModelForCausalLM, AutoTokenizer, TextStreamer, BitsAndBytesConfig
4import torch
5
6bnb_config = BitsAndBytesConfig(
7 load_in_4bit=True,
8 bnb_4bit_use_double_quant=True,
9 bnb_4bit_quant_type="nf4",
10 bnb_4bit_compute_dtype=torch.bfloat16
11)
12
13MODEL_NAME = 'Kukedlc/NeuralLLaMa-3-8b-DT-v0.1'
14tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
15model = AutoModelForCausalLM.from_pretrained(MODEL_NAME, device_map='cuda:0', quantization_config=bnb_config)
16
17prompt_system = "You are an advanced language model that speaks Spanish fluently, clearly, and precisely.\
18You are called Roberto the Robot and you are an aspiring post-modern artist."
19prompt = "Create a piece of art that represents how you see yourself, Roberto, as an advanced LLm, with ASCII art, mixing diagrams, engineering and let yourself go."
20
21chat = [
22 {"role": "system", "content": f"{prompt_system}"},
23 {"role": "user", "content": f"{prompt}"},
24]
25
26chat = tokenizer.apply_chat_template(chat, tokenize=False, add_generation_prompt=True)
27inputs = tokenizer(chat, return_tensors="pt").to('cuda')
28streamer = TextStreamer(tokenizer)
29stop_token = "<|eot_id|>"
30stop = tokenizer.encode(stop_token)[0]
31
32_ = model.generate(**inputs, streamer=streamer, max_new_tokens=1024, do_sample=True, temperature=0.7, repetition_penalty=1.2, top_p=0.9, eos_token_id=stop)