Free-running (autoregressive) generation accuracy on the full held-out test set — the deployment-realistic evaluation, not teacher-forced token accuracy:
The unadapted base model rarely produces the correct clinical content even when its JSON is syntactically valid; this adapter closes essentially all of that gap at roughly 1% of the trainable-parameter cost of full fine-tuning.
This adapter was trained on a specific tagged prompt format — using a different prompt structure at inference time will not reproduce the evaluation numbers above.
1from transformers import AutoTokenizer, AutoModelForCausalLM
2from peft import PeftModel
3import torch
4
5base_model_id = "google/medgemma-4b-it"
6adapter_id = "Japhari/cds-maternal-4b-en"
7
8tokenizer = AutoTokenizer.from_pretrained(adapter_id)
9base_model = AutoModelForCausalLM.from_pretrained(
10 base_model_id,
11 torch_dtype=torch.bfloat16,
12 device_map="auto",
13)
14model = PeftModel.from_pretrained(base_model, adapter_id)
15model.eval()
16
17case = """A 28-year-old woman at 36 weeks gestation presents with severe headache,
18blurred vision, BP 160/110, +3 proteinuria."""
19
20SYSTEM_PROMPT = "You are a maternal triage information extractor. Return ONLY compact JSON, no prose. Preserve numeric values from the case exactly when provided. Always include the best matching icd10Code. Do not invent patientId when it is not present in the story."
21prompt = f"<system>\n{SYSTEM_PROMPT}\n</system>\n<user>\n{case}\n</user>\n<assistant>\n"
22inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
23
24with torch.no_grad():
25 output = model.generate(**inputs, max_new_tokens=220, do_sample=False)
26
27print(tokenizer.decode(output[0][inputs['input_ids'].shape[1]:], skip_special_tokens=True))
