Views
No views yet
1pip install transformers torch
2pip install accelerate
3pip install -U transformers1
2from transformers import AutoTokenizer, AutoModelForCausalLM
3
4tokenizer = AutoTokenizer.from_pretrained("EpistemeAI/Fireball-R1-Llama-3.1-8B-Medical-COT")
5model = AutoModelForCausalLM.from_pretrained("EpistemeAI/Fireball-R1-Llama-3.1-8B-Medical-COT")
6
7prompt = "Calculate the molar mass of sulfuric acid (H₂SO₄)."
8inputs = tokenizer(prompt, return_tensors="pt")
9outputs = model.generate(**inputs, max_length=200)
10print(tokenizer.decode(outputs[0], skip_special_tokens=True))
11
12
13##advance inference
14import torch
15from transformers import AutoTokenizer, AutoModelForCausalLM
16
17# Load the tokenizer
18tokenizer = AutoTokenizer.from_pretrained("EpistemeAI/Fireball-R1-Llama-3.1-8B-Medical-COT")
19
20# Load the model in 8-bit precision using bitsandbytes (requires a CUDA GPU)
21model = AutoModelForCausalLM.from_pretrained(
22 "EpistemeAI/Fireball-R1-Llama-3.1-8B",
23 load_in_8bit=True, # Enable 8-bit loading to reduce memory usage
24 device_map="auto" # Automatically map model layers to the available device(s)
25)
26
27# Define the system prompt and the user prompt
28system_prompt = "You are a highly knowledgeable assistant with expertise in chemistry and physics. <think>"
29user_prompt = "Calculate the molar mass of sulfuric acid (H₂SO₄)."
30
31# Combine the system prompt with the user prompt. The format here follows a common convention for chat-like interactions.
32full_prompt = f"System: {system_prompt}\nUser: {user_prompt}\nAssistant:"
33
34# Tokenize the combined prompt and move the inputs to the GPU
35inputs = tokenizer(full_prompt, return_tensors="pt").to("cuda")
36
37# Generate output text from the model
38outputs = model.generate(**inputs, max_length=12200)
39
40# Decode and print the result, skipping special tokens
41result = tokenizer.decode(outputs[0], skip_special_tokens=True)
42print(result)
43