Views
No views yet
from transformers import AutoModelForCausalLM, AutoProcessor
# Set device
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# Load fine-tuned model and processor
model_id_or_path = "mychen76/Florence2-FT-Med-brain-7"
model = AutoModelForCausalLM.from_pretrained(model_id_or_path, trust_remote_code=True).to(device)
processor = AutoProcessor.from_pretrained(model_id_or_path, trust_remote_code=True)def run_model_inference(task_prompt, text_input, image, device="cpu"):
if text_input is None:
prompt = task_prompt
else:
prompt = task_prompt + text_input
# print("PROMPT=",prompt)
# Ensure the image is in RGB mode
if image.mode != "RGB":
image = image.convert("RGB")
inputs = processor(text=prompt, images=image, return_tensors="pt").to(device)
generated_ids = model.generate(
input_ids=inputs["input_ids"],
pixel_values=inputs["pixel_values"],
max_new_tokens=1024,
num_beams=3
)
generated_text = processor.batch_decode(generated_ids, skip_special_tokens=False)[0]
parsed_answer = processor.post_process_generation(generated_text, task=task_prompt, image_size=(image.width, image.height))
return parsed_answerresults = run_model_inference("<CAPTION>",None,test_image,device)
print(results){'<CAPTION>': '<CAPTION>The image is a non-contrast computed tomography (CT) scan of the brain, showing the cerebral hemispheres without any medical devices present. The region of interest, located centrally and in the middle of the image, occupies approximately 1.5% of the area and appears to have a different density compared to the surrounding brain tissue, which may indicate an intracranial hemorrhage. This abnormal area could be related to the brain structures, potentially leading to a mass effect or displacement of adjacent brain tissue.'}results = run_model_inference("<CAPTION_DETAILS>",None,test_image,device)
print(results){'<CAPTION_DETAILS>': '<CAPTION_DETAILS>The image is a non-contrast computed tomography (CT) scan of the brain, showing the cerebral hemispheres without any medical devices present. The region of interest, located centrally and in the middle of the image, occupies approximately 1.5% of the area and appears to have a different density compared to the surrounding brain tissue, which may indicate an intracranial hemorrhage. This abnormality is likely related to the brain structures, potentially affecting them or being affected by a mass effect or displacement of adjacent brain tissue.'}results = run_model_inference("<REGION_OF_INTEREST>",None,test_image,device)
print(results){'<REGION_OF_INTEREST>': '<REGION_OF_INTEREST>The region of interest, located centrally and in the lower-middle area of the brain, occupies approximately 1.5% of the image area and appears to have a different density compared to the surrounding brain tissue, which may indicate an intracranial hemorrhage.\n<OBSERVATION>\nThis abnormality is likely related to the brain structures, potentially affecting them or being affected by a mass effect or displacement of adjacent brain tissue.'}results=run_model_inference("<JSON_RECORD>",None,test_image,device)
record=results["<JSON_RECORD>"].split("<JSON_RECORD>")[1]
record"{'caption': 'The image is a non-contrast computed tomography (CT) scan of the brain, showing the cerebral hemispheres, ventricles, and subarachnoid spaces without any medical devices visible. The region of interest, located centrally and in the lower-middle area of the image, occupies approximately 1.5% of the area and appears to have a different density compared to the surrounding brain tissue, which may indicate an intracranial hemorrhage. This abnormal area could be related to or be affected by adjacent brain structures, potentially leading to a mass effect or displacement of adjacent brain tissue due to the presence of blood or a hematoma.'region of interest': 'This abnormal area is situated near the lateral ventricle, which is indicative of a hemorrhage, and could be exerting pressure or pressure on nearby brain structures due to its proximity to other cerebral structures.\\nThe relationship between this abnormal area and the surrounding cerebral structures suggests that the hemorrhage could be affecting or be affecting adjacent brain areas due to their proximity and displacement of nearby brain tissue.', 'observation': '', 'The abnormal area appears to be in a different area, possibly representing a hemorrhaged area, and is likely related to the adjacent brain regions, potentially affecting or being affected by nearby brain tissues due to it's proximity to nearby cerebral structures, although the relationship does not imply a significant difference in density or displacement within the brain tissue. 'bbox': '[212.7360076904297, 203.52000427246094, 252.16001892089844, 343.8080139160156]'}"results=run_model_inference("<DETECT_REGION_LOC>",None,test_image,device)
print(results){'<DETECT_REGION_LOC>': '<DETECT_REGION_LOC>{"bloc": "<loc_213><loc_204><loc_253><loc_344>", "bbox": "[212.7360076904297, 203.52000427246094, 252.16001892089844, 343.8080139160156]"}'}results=run_model_inference("<REGION_LOC_DESCRIPTION>",None,test_image,device)
print(results){'<REGION_LOC_DESCRIPTION>': '<REGION_LOC_DESCRIPTION>The image is a non-contrast computed tomography (CT) scan of the brain, showing the cerebral structures without any medical devices present. The region of interest, located centrally and in the middle of the image, occupies approximately 2.8% of the area and appears to have a different density compared to the surrounding brain tissue, which may indicate an intracranial hemorrhage. This abnormal area could be affecting or be affected by adjacent brain structures, potentially leading to a mass effect or displacement of adjacent tissues.'}