Views
No views yet
pip install torch transformers bitsandbytes accelerate1import torch
2from transformers import AutoModelForCausalLM, AutoTokenizer
3
4model_id = "jjjssjs/HyperCLOVAX-SEED-Think-32B-4bit"
5
6# Load tokenizer
7tokenizer = AutoTokenizer.from_pretrained(
8 model_id,
9 trust_remote_code=True,
10 fix_mistral_reges=True
11)
12
13# Load quantized model (quantization config is in config.json)
14model = AutoModelForCausalLM.from_pretrained(
15 model_id,
16 device_map="auto",
17 trust_remote_code=True,
18 torch_dtype=torch.bfloat16,
19)
20
21# Generate
22inputs = tokenizer("양자역학이 뭐야?", return_tensors="pt").to(model.device)
23
24with torch.no_grad():
25 outputs = model.generate(
26 **inputs,
27 max_new_tokens=100,
28 do_sample=True,
29 temperature=0.7,
30 top_p=0.9,
31 )
32
33print(tokenizer.decode(outputs[0], skip_special_tokens=True))1prompt = "Explain quantum computing in simple terms."
2
3inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
4
5outputs = model.generate(
6 **inputs,
7 max_new_tokens=200,
8 temperature=0.7,
9 top_p=0.9,
10)
11
12response = tokenizer.decode(outputs[0], skip_special_tokens=True)
13print(response)1from PIL import Image
2
3# Load image
4image = Image.open("example.jpg")
5
6# Prepare inputs
7text = "Describe this image in detail."
8inputs = tokenizer(text, return_tensors="pt").to(model.device)
9
10# Generate response
11outputs = model.generate(
12 **inputs,
13 max_new_tokens=150,
14)
15
16print(tokenizer.decode(outputs[0], skip_special_tokens=True))1conversation = [
2 {"role": "user", "content": "What is machine learning?"},
3 {"role": "assistant", "content": "Machine learning is..."},
4 {"role": "user", "content": "Can you give me an example?"}
5]
6
7# Process conversation
8inputs = tokenizer.apply_chat_template(
9 conversation,
10 return_tensors="pt"
11).to(model.device)
12
13outputs = model.generate(inputs, max_new_tokens=200)
14print(tokenizer.decode(outputs[0], skip_special_tokens=True))<think>...</think> outputfix_mistral_regex=True)