Views
No views yet
1from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
2import torch
3
4input_text = ""
5
6# Set device based on CUDA availability
7device = "cuda" if torch.cuda.is_available() else "cpu"
8
9# Load the model and tokenizer
10model_name = "ndebuhr/Mistral-7B-Technical-Tutorial-Summarization-QLoRA"
11tokenizer = AutoTokenizer.from_pretrained(model_name)
12model = AutoModelForCausalLM.from_pretrained(model_name).to(device)
13
14instruction = "Clarify and summarize this tutorial transcript"
15prompt = """{}
16
17### Raw Transcript:
18{}
19
20### Summary:
21"""
22
23# Tokenize the input text
24inputs = tokenizer(
25 prompt.format(instruction, input_text),
26 return_tensors="pt",
27 truncation=True,
28 max_length=16384
29).to(device)
30
31# Generate outputs
32outputs = model.generate(
33 **inputs,
34 max_length=16384,
35 num_return_sequences=1,
36 use_cache=True
37)
38
39# Decode the generated text
40generated_text = tokenizer.batch_decode(outputs, skip_special_tokens=True)