Views
No views yet
Galileo AI. RAGBench: A Benchmark Suite for Evaluating Retrieval-Augmented Generation.
Available at: https://huggingface.co/datasets/galileo-ai/ragbench
train, dev, test where applicable) with document examples drawn from all included datasets in the benchmark.pip install verbatim_rag1from verbatim_rag.inference.model import ClinicalBERTModel
2from verbatim_rag.inference.document import Document
3
4classifier = ClinicalBERTModel(
5 context_size=1,
6 max_length=512,
7 threshold=0.5,
8)
9
10documents = [
11 Document(
12 content=['Brief Hospital Course:',
13 'During the ERCP a pancreatic stent was required to facilitate access to the biliary system (removed '
14 'at the end of the procedure), and a common bile duct stent was placed to allow drainage of the '
15 'biliary obstruction caused by stones and sludge.',
16 "However, due to the patient's elevated INR, no sphincterotomy or stone removal was performed.",
17 'Frank pus was noted to be draining from the common bile duct, and post-ERCP it was recommended that '
18 'the patient remain on IV Zosyn for at least a week.',
19 'The Vancomycin was discontinued.',
20 'On hospital day 4 (post-procedure day 3) the patient returned to ERCP for re-evaluation of her '
21 'biliary stent as her LFTs and bilirubin continued an upward trend.',
22 'On ERCP the previous biliary stent was noted to be acutely obstructed by biliary sludge and stones.',
23 "As the patient's INR was normalized to 1.2, a sphincterotomy was safely performed, with removal of "
24 "several biliary stones in addition to the common bile duct stent.",
25 'At the conclusion of the procedure, retrograde cholangiogram was negative for filling defects.'],
26 metadata={"source": "example_doc_1", "id": "clinical_note_1"},
27 ),
28 Document(
29 content=['Brief Hospital Course:',
30 'Acute diastolic heart failure: Pt developed signs and symptoms of volume overload on [**2201-3-8**] '
31 'with shortness of breath, increased oxygen requirement and lower extremity edema.',
32 'Echo showed preserved EF, no WMA and worsening AI.',
33 'CHF most likely secondary to worsening valvular disease.',
34 'He was diuresed with lasix IV, intermittently on lasix gtt then transitioned to PO torsemide with '
35 'improvement in symptoms, although remained on a small amount of supplemental oxygen for comfort.',
36 "Respiratory failure: The patient was intubated for lethargy and acidosis initially and was given 8 "
37 "L on his presentation to help maintain his BP's.",
38 'This undoubtedly contributed to his continued hypoxemic respiratory failure.',
39 'He was advanced to pressure support with stable ventilation and oxygenation.',
40 'On transfer to the CCU patient was still intubated but off pressors.',
41 'Patient was extubated successfully.',
42 'He was reintubated [**2201-3-1**] transiently for 48 hours for urgent TEE and subsequently '
43 'extubated without adverse effect or complication.']
44 ,
45 metadata={"source": "example_doc_2", "id": "clinical_note_2"},
46 ),
47]
48
49
50patient_question = "Took my 59 yo father to ER ultrasound discovered he had an aortic aneurysm. He had a salvage " \
51 "repair (tube graft). Long surgery / recovery for couple hours then removed packs. why did they do " \
52 "this surgery????? After this time he spent 1 month in hospital now sent home."
53clinician_question = "Why did they perform the emergency salvage repair on him?"
54
55
56results = {}
57for doc in documents:
58 preds = classifier.predict(
59 patient_question=patient_question,
60 clinician_question=clinician_question,
61 sentences=doc.content,
62 sep=". ",
63 use_clinician_question=True
64 )
65 results[doc.metadata["id"]] = preds
66
67# ───── Display ─────
68for doc_id, doc in results.items():
69 print(f"\n>> {doc_id}")
70 print(doc)
711>> clinical_note_1
2 [False, True, True, False, False, False, True, True, False]
3>> clinical_note_2
4[False, False, False, True, False, False, False, False, False, False, False]