Views
No views yet
{
"instruction": "...",
"response": "..."
}!pip install transformers accelerate safetensors gradio torch --upgrade1# Imports
2from transformers import AutoTokenizer, AutoModelForCausalLM, TextIteratorStreamer
3import torch
4from threading import Thread
5import gradio as gr
6
7# Model ID on Hugging Face
8model_id = "sparrowaisolutions/aras-ember-v1"
9
10# Load tokenizer & model
11tokenizer = AutoTokenizer.from_pretrained(model_id)
12tokenizer.pad_token = tokenizer.eos_token # required for generation padding
13
14model = AutoModelForCausalLM.from_pretrained(model_id)
15device = "cuda" if torch.cuda.is_available() else "cpu"
16model.to(device)
17
18# Default AI prompt
19DEFAULT_PROMPT = """You are Aras-Ember, a helpful AI assistant by Sparrow AI Solutions.
20Respond clearly and concisely. Stay in chatbot mode and do NOT generate unrelated articles or dataset text.
21"""
22
23def generate_stream(user_input):
24 """Generate streamed response from model"""
25 prompt = DEFAULT_PROMPT + f"User: {user_input}\nAras-Ember:"
26
27 inputs = tokenizer(prompt, return_tensors="pt").to(device)
28
29 # Streamer for live output
30 streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
31
32 generation_kwargs = dict(
33 **inputs,
34 streamer=streamer,
35 max_new_tokens=150,
36 do_sample=True,
37 temperature=0.6,
38 top_p=0.9,
39 repetition_penalty=1.2,
40 no_repeat_ngram_size=3,
41 pad_token_id=tokenizer.eos_token_id
42 )
43
44 # Run generation in separate thread
45 thread = Thread(target=model.generate, kwargs=generation_kwargs)
46 thread.start()
47
48 # Stream output
49 response = ""
50 for new_text in streamer:
51 print(new_text, end="", flush=True)
52 response += new_text
53 print() # newline after completion
54
55 return response
56
57# Gradio chat interface
58def chat_fn(user_input):
59 return generate_stream(user_input)
60
61iface = gr.Interface(
62 fn=chat_fn,
63 inputs=gr.Textbox(lines=3, placeholder="Say something to Aras-Ember..."),
64 outputs="text",
65 title="Aras-Ember Chat",
66 description="A lightweight AI by Sparrow AI Solutions."
67)
68
69# Launch Gradio in Colab
70iface.launch(share=True) # `share=True` gives a public URL for testingYou are Aras-Ember, a creative AI assistant.
User: <question>
Assistant:User: Explain black holes simply.
Assistant: