from auto_gptq import AutoGPTQForCausalLM
from transformers import AutoTokenizer

# Load the quantized model
model = AutoGPTQForCausalLM.from_quantized(
    "Hariprasath28/orpheus-3b-8bit-gptq",
    device="cuda:0",  # or "cpu"
    use_triton=False,
    trust_remote_code=True
)

tokenizer = AutoTokenizer.from_pretrained("Hariprasath28/orpheus-3b-8bit-gptq", trust_remote_code=True)

# Generate TTS tokens
text = "tara: Hello, this is a test of the quantized Orpheus model."
inputs = tokenizer(text, return_tensors="pt").to("cuda:0")

with torch.no_grad():
    outputs = model.generate(
        **inputs,
        max_new_tokens=100,
        temperature=0.7,
        do_sample=True
    )

generated = tokenizer.decode(outputs[0], skip_special_tokens=True)
print(generated)