Views
No views yet
pip install transformers accelerate peft1from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
2from peft import PeftModel, PeftConfig
3
4repo_id = "stefan-m-lenz/Qwen3-14B-ICDOPS-QA-2024"
5config = PeftConfig.from_pretrained(repo_id, device_map="auto")
6quantization_config = BitsAndBytesConfig(load_in_8bit=True)
7model = AutoModelForCausalLM.from_pretrained(config.base_model_name_or_path,
8 device_map="auto",
9 quantization_config=quantization_config)
10model = PeftModel.from_pretrained(model, repo_id, device_map="auto")
11tokenizer = AutoTokenizer.from_pretrained(config.base_model_name_or_path,
12 device_map="auto")
13
14# Test input
15test_input = """Welche ICD-10-Kodierung wird für die Tumordiagnose "Bronchialkarzinom, Hauptbronchus" verwendet? Antworte nur mit dem ICD-10 Code."""
16
17input_str = tokenizer.apply_chat_template(
18 [{"role": "user", "content": test_input}],
19 tokenize=False,
20 add_generation_prompt=True,
21 enable_thinking=False,
22)
23
24# Generate response
25inputs = tokenizer(input_str, return_tensors="pt").to("cuda")
26outputs = model.generate(
27 **inputs,
28 max_new_tokens=7,
29 do_sample=False,
30 pad_token_id=tokenizer.eos_token_id,
31 temperature=None,
32 top_p=None,
33 top_k=None,
34)
35response = tokenizer.decode(outputs[0], skip_special_tokens=True)
36response = response[len(test_input):].strip()
37
38print("Test Input:", test_input)
39print("Model Response:", response)