1import gradio as gr
2import numpy as np
3import random
4import os
5from PIL import Image
6
7import spaces # [uncomment to use ZeroGPU]
8from diffusers import StableDiffusionPipeline, UNet2DConditionModel
9import torch
10
11# 模态映射
12legal_modals = ['OCT', 'CXR', 'Fundus', 'BrainMRI', 'BreastMRI', 'ChestCT']
13modal_idx = {
14 'OCT': 1,
15 'CXR': 2,
16 'Fundus': 3,
17 'BrainMRI': 4,
18 'BreastMRI': 4,
19 'ChestCT': 4
20}
21
22device = "cuda" if torch.cuda.is_available() else "cpu"
23model_repo_id = "CoheeY/MINIM"
24
25if torch.cuda.is_available():
26 torch_dtype = torch.float16
27else:
28 torch_dtype = torch.float32
29
30# 缓存不同模态的pipeline
31pipe_cache = {}
32
33MAX_SEED = np.iinfo(np.int32).max
34MAX_IMAGE_SIZE = 1024
35
36
37def get_pipe_for_modal(modal_name):
38 """获取或创建指定模态的pipeline"""
39 if modal_name not in pipe_cache:
40 print(f"Loading pipeline for modal: {modal_name}")
41
42 # 获取模态对应的数字ID
43 modal_id = modal_idx[modal_name]
44
45 # 加载该模态的微调UNet
46 unet_subfolder = f"unets/{modal_id}/unet"
47 print(f"Loading UNet from subfolder: {unet_subfolder}")
48
49 unet = UNet2DConditionModel.from_pretrained(
50 model_repo_id,
51 subfolder=unet_subfolder,
52 torch_dtype=torch_dtype
53 )
54
55 # 创建pipeline,使用微调UNet替换基础UNet
56 pipe = StableDiffusionPipeline.from_pretrained(
57 model_repo_id,
58 unet=unet,
59 safety_checker=None,
60 torch_dtype=torch_dtype
61 )
62
63 pipe = pipe.to(device)
64
65 # 启用优化
66 if torch.cuda.is_available():
67 try:
68 pipe.enable_xformers_memory_efficient_attention()
69 print("XFormers enabled")
70 except:
71 print("XFormers not available")
72
73 pipe_cache[modal_name] = pipe
74 print(f"Pipeline for {modal_name} loaded successfully")
75
76 return pipe_cache[modal_name]
77
78
79@spaces.GPU
80def infer(
81 modal,
82 prompt,
83 negative_prompt,
84 seed,
85 randomize_seed,
86 width,
87 height,
88 guidance_scale,
89 num_inference_steps,
90 progress=gr.Progress(track_tqdm=True),
91):
92 # 设置随机种子
93 if randomize_seed:
94 seed = random.randint(0, MAX_SEED)
95
96 generator = torch.Generator(device=device).manual_seed(seed)
97
98 # 获取对应模态的pipeline
99 pipe = get_pipe_for_modal(modal)
100
101 # 构建输入prompt
102 modal_index = modal_idx[modal]
103 full_prompt = f"{modal_index}:{prompt}"
104
105 print(f"Generating image for modal: {modal}, prompt: {full_prompt}")
106
107 # 生成图像
108 try:
109 image = pipe(
110 prompt=full_prompt,
111 negative_prompt=negative_prompt,
112 guidance_scale=guidance_scale,
113 num_inference_steps=num_inference_steps,
114 width=width,
115 height=height,
116 generator=generator,
117 ).images[0]
118
119 print(f"Image generated successfully")
120 return image, seed
121
122 except Exception as e:
123 print(f"Error during inference: {e}")
124 import traceback
125 traceback.print_exc()
126
127 # 返回错误信息
128 error_image = Image.new('RGB', (512, 512), color=(255, 200, 200))
129 return error_image, seed
130
131
132# 医学图像示例
133medical_examples = [
134 ["CXR", "clear lung fields with normal heart size"],
135 ["Fundus", "normal retina with healthy optic disc"],
136 ["BrainMRI", "normal brain anatomy in axial view"],
137]
138
139css = """
140#col-container {
141 margin: 0 auto;
142 max-width: 640px;
143}
144"""
145
146with gr.Blocks(css=css) as demo:
147 with gr.Column(elem_id="col-container"):
148 gr.Markdown("# MINIM: Medical Image Generation Model")
149 gr.Markdown("### Multi-modal Medical Image Generation with Stable Diffusion")
150
151 with gr.Row():
152 modal = gr.Dropdown(
153 label="Imaging Modality",
154 choices=legal_modals,
155 value="CXR",
156 info="Select medical imaging modality"
157 )
158
159 with gr.Row():
160 prompt = gr.Textbox(
161 label="Description",
162 show_label=False,
163 max_lines=2,
164 placeholder="Describe the medical image you want to generate...",
165 container=False,
166 scale=4
167 )
168
169 run_button = gr.Button("Generate", scale=1, variant="primary")
170
171 result = gr.Image(label="Generated Image", show_label=False)
172
173 with gr.Accordion("Advanced Settings", open=False):
174 negative_prompt = gr.Textbox(
175 label="Negative Prompt",
176 max_lines=2,
177 placeholder="Describe what you don't want in the image...",
178 value="blurry, distorted, artifact",
179 )
180
181 with gr.Row():
182 seed = gr.Slider(
183 label="Seed",
184 minimum=0,
185 maximum=MAX_SEED,
186 step=1,
187 value=0,
188 )
189
190 randomize_seed = gr.Checkbox(
191 label="Randomize Seed",
192 value=True
193 )
194
195 with gr.Row():
196 width = gr.Slider(
197 label="Width",
198 minimum=256,
199 maximum=MAX_IMAGE_SIZE,
200 step=64,
201 value=512,
202 )
203
204 height = gr.Slider(
205 label="Height",
206 minimum=256,
207 maximum=MAX_IMAGE_SIZE,
208 step=64,
209 value=512,
210 )
211
212 with gr.Row():
213 guidance_scale = gr.Slider(
214 label="Guidance Scale",
215 minimum=0.0,
216 maximum=20.0,
217 step=0.5,
218 value=7.5,
219 )
220
221 num_inference_steps = gr.Slider(
222 label="Inference Steps",
223 minimum=10,
224 maximum=200,
225 step=5,
226 value=100,
227 )
228
229 gr.Examples(
230 examples=medical_examples,
231 inputs=[modal, prompt],
232 label="Medical Image Examples"
233 )
234
235 # 绑定事件
236 run_button.click(
237 fn=infer,
238 inputs=[
239 modal,
240 prompt,
241 negative_prompt,
242 seed,
243 randomize_seed,
244 width,
245 height,
246 guidance_scale,
247 num_inference_steps,
248 ],
249 outputs=[result, seed]
250 )
251
252 prompt.submit(
253 fn=infer,
254 inputs=[
255 modal,
256 prompt,
257 negative_prompt,
258 seed,
259 randomize_seed,
260 width,
261 height,
262 guidance_scale,
263 num_inference_steps,
264 ],
265 outputs=[result, seed]
266 )
267
268if __name__ == "__main__":
269 demo.launch(debug=True)