Views
No views yet

pip install transformers accelerate1import torch
2from transformers import AutoModelForImageTextToText, AutoTokenizer
3
4model_id = "MaralGPT/MaralGPT-Mythos-9B-2606"
5tokenizer = AutoTokenizer.from_pretrained(model_id)
6model = AutoModelForImageTextToText.from_pretrained(
7 model_id, dtype="bfloat16", device_map="cuda"
8)
9
10messages = [
11 {"role": "user",
12 "content": "Write a simple snake game in python."}
13]
14
15text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
16inputs = tokenizer(text, return_tensors="pt").to(model.device)
17
18out = model.generate(
19 **inputs, max_new_tokens=16384, do_sample=True,
20 temperature=0.6, top_p=0.95, top_k=20, repetition_penalty=1.05,
21)
22
23print(tokenizer.decode(out[0][inputs["input_ids"].shape[1]:], skip_special_tokens=True))
temperature=0.6 top_p=0.95 top_k=20