Views
No views yet
1#!/usr/bin/env python3
2import torch
3from transformers import (
4 AutoTokenizer,
5 AutoModelForCausalLM,
6 BitsAndBytesConfig,
7 pipeline
8)
9import warnings
10warnings.filterwarnings("ignore")
11
12def load_model_and_tokenizer():
13 """Load model and tokenizer"""
14 print("Loading model and tokenizer...")
15
16 # Base model configuration
17 base_model_name = "meta-llama/Meta-Llama-3-8B-Instruct"
18 lora_model_name = "wingchiuloong/CopyPasteLLM-L3-8B"
19
20 # Quantization configuration
21 quantization_config = BitsAndBytesConfig(
22 load_in_4bit=True,
23 bnb_4bit_compute_dtype=torch.float16,
24 bnb_4bit_use_double_quant=True,
25 bnb_4bit_quant_type="nf4"
26 )
27
28 # Load tokenizer
29 tokenizer = AutoTokenizer.from_pretrained(
30 base_model_name,
31 trust_remote_code=True
32 )
33
34 # Load base model
35 model = AutoModelForCausalLM.from_pretrained(
36 base_model_name,
37 quantization_config=quantization_config,
38 device_map="auto",
39 trust_remote_code=True,
40 torch_dtype=torch.float16
41 )
42
43 # Load LoRA weights
44 print("Loading LoRA weights...")
45 from peft import PeftModel
46 model = PeftModel.from_pretrained(model, lora_model_name)
47
48 return model, tokenizer
49
50def create_pipeline(model, tokenizer):
51 """Create inference pipeline"""
52 return pipeline(
53 "text-generation",
54 model=model,
55 tokenizer=tokenizer,
56 torch_dtype=torch.float16,
57 device_map="auto"
58 )
59
60def format_prompt(user_input):
61 """Format input to Llama-3 format"""
62 system_message = "You are a helpful AI assistant."
63
64 prompt = f"""<|begin_of_text|><|start_header_id|>system<|end_header_id|>
65
66{system_message}<|eot_id|><|start_header_id|>user<|end_header_id|>
67
68{user_input}<|eot_id|><|start_header_id|>assistant<|end_header_id|>
69
70"""
71 return prompt
72
73def main():
74 """Main function - One-time inference"""
75 print("CopyPasteLLM inference Demo")
76 print("=" * 50)
77
78 # Set test question
79 context = "Galileo Galilei, renowned as one of the most influential figures in the history of science, made numerous contributions that revolutionized our understanding of physics and astronomy. His meticulous work with telescopes led to groundbreaking discoveries about the moons of Jupiter and the phases of Venus. Beyond the realm of astronomy, his observations and experiments laid the foundation for classical mechanics. One of Galileo’s lesser-known achievements is his development of the Three Laws of Motion, which were critical in advancing the study of kinematics and dynamics. These laws articulate the principles of inertia, the relationship between force and motion, and the law of action and reaction, providing a comprehensive framework for understanding moving bodies. His work on pendulums also contributed substantially to timekeeping and horology, as he discovered that pendulums of different lengths oscillate at predictable periods, a principle still applied in modern clocks. Galileo’s interdisciplinary approach enabled him to synthesize knowledge from various fields, which allowed new theories to emerge, reshaping the scientific landscape of his time and beyond. Notably, his support of the heliocentric model of the solar system earned him both acclaim and censure, highlighting the tension between scientific inquiry and established doctrine. In contrast to Galen’s biological studies and Newton’s later contributions, Galileo’s articulation of the Three Laws of Motion was pivotal in the transition from Aristotelian physics to Newtonian mechanics. His contributions remain a testament to the interplay of observation, theory, and experimentation in scientific progress."
80 question = "Which law was Galileo Galilei responsible for describing?"
81 test_question = f"{context}
82Q: {question}
83A:"
84 print(f"{test_question}")
85 print("-" * 50)
86
87 try:
88 # Load model
89 model, tokenizer = load_model_and_tokenizer()
90
91 # Create pipeline
92 print("Creating inference pipeline...")
93 pipe = create_pipeline(model, tokenizer)
94
95 print("Model loaded! Starting inference...")
96
97 # Format input
98 prompt = format_prompt(test_question)
99
100 # Generate reply
101 print("Generating reply...")
102 outputs = pipe(
103 prompt,
104 max_new_tokens=512,
105 temperature=1.0,
106 top_p=0.95,
107 do_sample=True,
108 pad_token_id=tokenizer.eos_token_id,
109 eos_token_id=tokenizer.eos_token_id,
110 return_full_text=False
111 )
112
113 # Output result
114 response = outputs[0]['generated_text']
115 print(f"
116CopyPasteLLM:
117{response}") # According to the passage, Galileo Galilei was responsible for describing the Three Laws of Motion, which articulate the principles of inertia, the relationship between force and motion, and the law of action and reaction.
118
119 except Exception as e:
120 print(f"Inference failed: {e}")
121 print("Please ensure the necessary dependencies are installed: pip install transformers peft bitsandbytes accelerate")
122
123if __name__ == "__main__":
124 main()1base_model="meta-llama/Meta-Llama-3-8B-Instruct"
2lora_modules_path="<model_at_your_huggingface_cache>" # like "~/.huggingface/hub/models--wingchiuloong--CopyPasteLLM-L3-8B/snapshots/<uuid>"
3
4python -m vllm.entrypoints.openai.api_server \
5 --model $base_model \
6 --enable-lora \
7 --max-lora-rank 64 \
8 --lora-modules CopyPasteLLM-8b=$lora_modules_path \
9 --host 0.0.0.0 \
10 --port 8888 \
11 --gpu-memory-utilization 0.8 \
12 --max-model-len 1024 \
13 --max-num-seqs 32 \
14 --tensor-parallel-size 11@misc{long2025copypastemitigatelargelanguage,
2 title={Copy-Paste to Mitigate Large Language Model Hallucinations},
3 author={Yongchao Long and Xian Wu and Yingying Zhang and Xianbin Wen and Yuxi Zhou and Shenda Hong},
4 year={2025},
5 eprint={2510.00508},
6 archivePrefix={arXiv},
7 primaryClass={cs.CL},
8 url={https://arxiv.org/abs/2510.00508},
9}