Views
No views yet
1from unsloth import FastLanguageModel
2model_name = "tiam4tt/TinyLlama-1.1B-chat.v1.0-linux-qna"
3model, tokenizer = FastLanguageModel.from_pretrained(
4 model_name=model_name,
5 max_seq_length=256,
6 dtype=None, # Use None for automatic dtype detection
7 load_in_4bit=True,
8 )
9 # Enable the model for inference
10 FastLanguageModel.for_inference(model)
11
12PROMPT = """Below is a question relating to the Linux operating system, paired with a paragraph describing further context. Write a short, simple, concise, and comprehensive response to the question.
13### Question
14{}
15### Context
16{}
17### Response
18{}"""
19
20question = "How do I check the disk usage of a directory in Linux?"
21
22inputs = tokenizer(PROMPT.format(
23 question,
24 "",
25 "" # Leave blank for generation
26 ),
27 return_tensors="pt").to(model.device)
28output = model.generate(
29 **inputs,
30 max_new_tokens=256,
31 do_sample=True,
32 temperature=1.0,
33 top_p=0.9
34 )
35answer = tokenizer.decode(output[0], skip_special_tokens=True)
36print(answer)