Views
No views yet
[!NOTE] If you encounter pipeline loading failure or unexpected output, please contact bili_sakura@zju.edu.cn.
diffusers.DiffusionPipeline.from_pretrained().model_index.json is set to the default text-to-image pipeline (DiffusionSatPipeline) so DiffusionPipeline.from_pretrained() works out of the box. The ControlNet variant is loaded via custom_pipeline plus the controlnet subfolder, as shown below.pipeline_diffusionsat.py: Standard text-to-image pipeline with DiffusionSat metadata support.pipeline_diffusionsat_controlnet.py: ControlNet pipeline with DiffusionSat metadata and conditional metadata support.ckpt/diffusionsat/) should contain the standard diffusers components (unet, vae, scheduler, etc.). You can reference these pipeline files directly from this directory or copy them to your checkpoint folder.demo_images/readme_controlnet.jpeg is generated from demo_images/texas_condition_input.png using the same sample pair from the Texas validation mini-set.pipeline_diffusionsat.py for standard generation.1import torch
2from diffusers import DiffusionPipeline
3
4# Load pipeline
5pipe = DiffusionPipeline.from_pretrained(
6 "path/to/ckpt/diffusionsat",
7 custom_pipeline="./pipeline_diffusionsat.py", # Path to this file
8 torch_dtype=torch.float16,
9 trust_remote_code=True,
10)
11pipe = pipe.to("cuda")
12
13# Optional: Metadata (normalized lat, lon, timestamp, GSD, etc.)
14# metadata = [0.5, -0.3, 0.7, 0.2, 0.1, 0.0, 0.5]
15
16# Generate
17image = pipe(
18 "satellite image of farmland",
19 metadata=None, # Optional
20 height=256,
21 width=256,
22 num_inference_steps=30,
23).images[0]pipeline_diffusionsat_controlnet.py for ControlNet generation.1import torch
2import numpy as np
3from PIL import Image
4from torchvision import transforms
5from diffusers import DiffusionPipeline
6from controlnet.controlnet_3d import ControlNetModel3D
7
8# 1. Load the Texas 3D ControlNet
9controlnet = ControlNetModel3D.from_pretrained(
10 "path/to/ckpt/diffusionsat/controlnet",
11 torch_dtype=torch.float16,
12)
13
14# 2. Load Pipeline with ControlNet
15pipe = DiffusionPipeline.from_pretrained(
16 "path/to/ckpt/diffusionsat",
17 controlnet=controlnet,
18 custom_pipeline="./pipeline_diffusionsat_controlnet.py", # Path to this file
19 torch_dtype=torch.float16,
20 trust_remote_code=True,
21)
22pipe = pipe.to("cuda")
23
24# 3. Prepare notebook-matched conditioning image
25# sample pair: housing-13176, source=tif.rgb-2016.npy, target=tif.rgb-2018.npy
26prep = transforms.Compose([
27 transforms.Resize(256, interpolation=transforms.InterpolationMode.BICUBIC, antialias=True),
28 transforms.CenterCrop(256),
29])
30control_image = prep(Image.open("./demo_images/texas_condition_input.png").convert("RGB"))
31control_tensor = torch.from_numpy(np.array(control_image)).permute(2, 0, 1).unsqueeze(0).float() / 255.0
32control_tensor = control_tensor.to(device="cuda", dtype=torch.float16)
33
34# Optional temporal conditioning metadata (num_metadata x num_frames)
35cond_metadata = [[0.0] for _ in range(7)]
36
37# 4. Generate (notebook-aligned settings)
38image = pipe(
39 prompt="a satlas satellite image of houses built in 2014 covering 0.1929 acres",
40 image=control_tensor,
41 metadata=None,
42 cond_metadata=cond_metadata,
43 is_temporal=True,
44 height=256,
45 width=256,
46 num_inference_steps=50,
47 guidance_scale=1.0,
48).images[0]