Views
No views yet
1from transformers import AutoModelForCausalLM, AutoTokenizer, TextStreamer
2
3MODEL_NAME = "leonard-milo/Qwen3.5-2B-SFT-AutoConv-InstagramChat-Smart"
4
5tok = AutoTokenizer.from_pretrained(MODEL_NAME, device_map="auto")
6model = AutoModelForCausalLM.from_pretrained(MODEL_NAME, device_map="auto")
7
8messages = [
9 {"role": "user", "content": "User message"},
10]
11
12tokenized_chat = tok.apply_chat_template(messages, tokenize=False, add_generation_prompt=True, return_tensors="pt")
13
14inputs = tok(tokenized_chat, return_tensors="pt")
15streamer = TextStreamer(tok)
16_ = model.generate(**inputs, streamer=streamer)1{
2 "messages": [
3 {"role": "user", "content": "..."},
4 {"role": "assistant", "content": "..."}
5 ]
6}