from transformers import AutoTokenizer, AutoModelForCausalLM
import torch

#model_id = "xiaol/Huggingface-RWKV-claude-for-mobile-v4-world-1.5B-16k"
model_id = "xiaol/RWKV-raven-14B-one-state"

model = AutoModelForCausalLM.from_pretrained(model_id,  torch_dtype=torch.bfloat16)

#model = model.half() #1.5B need fp32

#model = torch.compile(model) #need pytorch 2.0 and linux
model.to(0)

tokenizer = AutoTokenizer.from_pretrained(model_id)

question = "Tell me about ravens"
prompt = f"### Instruction: {question}\n### Response:"

inputs = tokenizer(prompt, return_tensors="pt").to(0)
output = model.generate(inputs["input_ids"], max_new_tokens=100)

print(tokenizer.decode(output[0].tolist(), skip_special_tokens=True))