Views
No views yet
1lora_config = LoraConfig(
2 r=8,
3 lora_alpha=32,
4 target_modules=["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"],
5 lora_dropout=0.1,
6 bias="none",
7 task_type="CAUSAL_LM"
8)1from vllm import LLM, SamplingParams
2from vllm.lora.request import LoRARequest
3
4# Initialize LLM with LoRA support
5llm = LLM(
6 model="Qwen/Qwen2.5-7B-Instruct",
7 tensor_parallel_size=2,
8 enable_lora=True
9)
10
11sampling_params = SamplingParams(temperature=0.0, max_tokens=50)
12
13# Create LoRA request
14lora_request = LoRARequest(
15 "rag_adapter",
16 1,
17 "doubleyyh/qwen-rag-lora" # HuggingFace repo name
18)
19
20# Example prompt
21prompt = """Using the context provided below, answer the question concisely. Respond in Korean if the question is in Korean, and in English if the question is in English.
22
23Context: subject: Meeting Schedule Update
24from: [['John Smith', 'john@company.com']]
25to: [['Team', 'team@company.com']]
26text_body: The project review meeting is rescheduled to 3 PM tomorrow.
27
28Question: When is the meeting rescheduled to?
29
30Answer: """
31
32# Generate with LoRA
33outputs = llm.generate([prompt], sampling_params, lora_request=lora_request)
34print(outputs[0].outputs[0].text)# English Query
Q: When is the project review scheduled?
A: The project review meeting is rescheduled to 3 PM tomorrow.
# Korean Query
Q: 프로젝트 미팅이 언제로 변경되었나요?
A: 내일 오후 3시로 변경되었습니다.1@misc{qwen-rag-lora,
2 author = {doubleyyh},
3 title = {Qwen-RAG-LoRA: Fine-tuned LoRA Weights for Email QA},
4 year = {2024},
5 publisher = {Hugging Face}
6}