Views
No views yet
pip install transformers accelerate peft1from transformers import AutoTokenizer, AutoModelForCausalLM
2from peft import PeftModel, PeftConfig
3
4repo_id = "stefan-m-lenz/Qwen3-8B-ICDOPS-QA-2024"
5config = PeftConfig.from_pretrained(repo_id, device_map="auto")
6model = AutoModelForCausalLM.from_pretrained(config.base_model_name_or_path,
7 device_map="auto")
8model = PeftModel.from_pretrained(model, repo_id, device_map="auto")
9tokenizer = AutoTokenizer.from_pretrained(config.base_model_name_or_path,
10 device_map="auto")
11
12# Test input
13test_input = """Welche ICD-10-Kodierung wird für die Tumordiagnose "Bronchialkarzinom, Hauptbronchus" verwendet? Antworte nur mit dem ICD-10 Code."""
14
15input_str = tokenizer.apply_chat_template(
16 [{"role": "user", "content": test_input}],
17 tokenize=False,
18 add_generation_prompt=True,
19 enable_thinking=False,
20)
21
22# Generate response
23inputs = tokenizer(input_str, return_tensors="pt").to("cuda")
24outputs = model.generate(
25 **inputs,
26 max_new_tokens=7,
27 do_sample=False,
28 pad_token_id=tokenizer.eos_token_id,
29 temperature=None,
30 top_p=None,
31 top_k=None,
32)
33generated_tokens = outputs[0, inputs["input_ids"].shape[1]:]
34response = tokenizer.decode(generated_tokens, skip_special_tokens=True).strip()
35
36print("Test Input:", test_input)
37print("Model Response:", response)