Views
No views yet
1<|system|>
2{system_message} <|end|>
3<|user|>
4{Prompt) <|end|>
5<|assistant|>1from transformers import AutoModelForCausalLM, AutoTokenizer
2
3model_name_or_path = "thesven/Phi-nut-Butter-Codebagel-v1-GPTQ"
4
5tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, use_fast=True)
6model = AutoModelForCausalLM.from_pretrained(
7 model_name_or_path,
8 device_map="auto",
9 trust_remote_code=False,
10 revision="main",
11)
12model.pad_token = model.config.eos_token_id
13
14prompt_template = '''
15<|system|>
16You are an expert developer. Please help me with any coding questions.<|end|>
17<|user|>
18In typescript how would I use a function that looks like this <T>(config:T):T<|end|>
19<|assistant|>
20'''
21input_ids = tokenizer(prompt_template, return_tensors='pt').input_ids.cuda()
22output = model.generate(inputs=input_ids, temperature=0.1, do_sample=True, top_p=0.95, top_k=40, max_new_tokens=256)
23
24generated_text = tokenizer.decode(output[0, len(input_ids[0]):], skip_special_tokens=True)
25display(generated_text)
26