Views
No views yet
transformers library:1from transformers import AutoModelForCausalLM, AutoTokenizer
2
3# Load the model and tokenizer
4model_name = "huihui-ai/Qwen2.5-32B-Instruct-abliterated"
5model = AutoModelForCausalLM.from_pretrained(
6 model_name,
7 torch_dtype="auto",
8 device_map="auto"
9)
10tokenizer = AutoTokenizer.from_pretrained(model_name)
11
12# Initialize conversation context
13initial_messages = [
14 {"role": "system", "content": "You are Qwen, created by Alibaba Cloud. You are a helpful assistant."}
15]
16messages = initial_messages.copy() # Copy the initial conversation context
17
18# Enter conversation loop
19while True:
20 # Get user input
21 user_input = input("User: ").strip() # Strip leading and trailing spaces
22
23 # If the user types '/exit', end the conversation
24 if user_input.lower() == "/exit":
25 print("Exiting chat.")
26 break
27
28 # If the user types '/clean', reset the conversation context
29 if user_input.lower() == "/clean":
30 messages = initial_messages.copy() # Reset conversation context
31 print("Chat history cleared. Starting a new conversation.")
32 continue
33
34 # If input is empty, prompt the user and continue
35 if not user_input:
36 print("Input cannot be empty. Please enter something.")
37 continue
38
39 # Add user input to the conversation
40 messages.append({"role": "user", "content": user_input})
41
42 # Build the chat template
43 text = tokenizer.apply_chat_template(
44 messages,
45 tokenize=False,
46 add_generation_prompt=True
47 )
48
49 # Tokenize input and prepare it for the model
50 model_inputs = tokenizer([text], return_tensors="pt").to(model.device)
51
52 # Generate a response from the model
53 generated_ids = model.generate(
54 **model_inputs,
55 max_new_tokens=8192
56 )
57
58 # Extract model output, removing special tokens
59 generated_ids = [
60 output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)
61 ]
62 response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
63
64 # Add the model's response to the conversation
65 messages.append({"role": "assistant", "content": response})
66
67 # Print the model's response
68 print(f"Qwen: {response}")
69