Views
No views yet
1from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
2from peft import PeftModel
3import torch
4
5# Configuration
6model_id = "zewail/job-description-generator-llama3-8b"
7base_model = "meta-llama/Meta-Llama-3-8B-Instruct"
8
9# Load with 4-bit quantization
10bnb_config = BitsAndBytesConfig(
11 load_in_4bit=True,
12 bnb_4bit_quant_type="nf4",
13 bnb_4bit_compute_dtype=torch.bfloat16
14)
15
16tokenizer = AutoTokenizer.from_pretrained(model_id)
17model = AutoModelForCausalLM.from_pretrained(
18 base_model,
19 quantization_config=bnb_config,
20 device_map="auto"
21)
22model = PeftModel.from_pretrained(model, model_id)
23
24# Generate
25prompt = """<|begin_of_text|><|start_header_id|>system<|end_header_id|>
26
27You are an expert HR professional.<|eot_id|><|start_header_id|>user<|end_header_id|>
28
29Generate a job description for Senior Data Scientist in San Francisco, CA.<|eot_id|><|start_header_id|>assistant<|end_header_id|>
30
31"""
32
33inputs = tokenizer(prompt, return_tensors="pt").to("cuda")
34outputs = model.generate(**inputs, max_new_tokens=400, temperature=0.7, top_p=0.95)
35print(tokenizer.decode(outputs[0], skip_special_tokens=True))