from unsloth import FastLanguageModel
from unsloth import FastLanguageModel
import torch
max_seq_length = 2048 # Choose any! We auto support RoPE Scaling internally!
dtype = None # None for auto detection. Float16 for Tesla T4, V100, Bfloat16 for Ampere+
load_in_4bit = True # Use 4bit quantization to reduce memory usage. Can be False.
model, tokenizer = FastLanguageModel.from_pretrained(
    model_name = "surajgorai/llama_3_8b_text_to_sql_model", # YOUR MODEL YOU USED FOR TRAINING
    max_seq_length = max_seq_length,
    dtype = dtype,
    load_in_4bit = load_in_4bit,
)
FastLanguageModel.for_inference(model) # Enable native 2x faster inference

prompt = """You are a powerful text-to-SQL model. Your job is to answer questions about a database. You are given a question and context regarding one or more tables.

You must output the SQL query that answers the question.

### Instruction:
{}

### Input:
{}

### Response:
{}"""

# alpaca_prompt = You MUST copy from above!
inputs = tokenizer(
[
    prompt.format(
        'Name the result/games for 54741', # instruction
        'CREATE TABLE table_21436373_11 (result_games VARCHAR, attendance VARCHAR)', # input
        "", # output - leave this blank for generation!
    )
], return_tensors = "pt").to("cuda")

outputs = model.generate(**inputs, max_new_tokens = 64, use_cache = True)
tokenizer.batch_decode(outputs)

#response :
#['You are a powerful text-to-SQL model. Your job is to answer questions about a database. You are given a question and context regarding one or more tables.
#\n\nYou must output the SQL query that answers the question.\n\n### Instruction:\nName the result/games for 54741\n\n### Input:\nCREATE TABLE table_21436373_11 (result_games VARCHAR, attendance VARCHAR)
#\n\n### Response:\nSELECT result_games FROM table_21436373_11 WHERE attendance = "54741"<|end_of_text|>']

from transformers import TextStreamer
text_streamer = TextStreamer(tokenizer)
_ = model.generate(**inputs, streamer = text_streamer, max_new_tokens = 128)

#response
#You are a powerful text-to-SQL model. Your job is to answer questions about a database. You are given a question and context regarding one or more tables.
#You must output the SQL query that answers the question.
### Instruction:
#Name the result/games for 54741
### Input:
#CREATE TABLE table_21436373_11 (result_games VARCHAR, attendance VARCHAR)
## Response:
#SELECT result_games FROM table_21436373_11 WHERE attendance = "54741"<|end_of_text|>