Views
No views yet
pip install torch torchvision transformers pillow huggingface_hub1import torch
2from transformers import GPT2Tokenizer
3from PIL import Image
4from torchvision import transforms
5
6# Load checkpoint
7checkpoint = torch.load("model_fp32/model_checkpoint.pth", map_location="cpu")
8
9# Load tokenizer
10tokenizer = GPT2Tokenizer.from_pretrained("model_fp32/tokenizer")
11
12# Load your model architecture (you need to define this)
13# model = YourVisionGPTModel(config)
14# model.load_state_dict(checkpoint['model_state_dict'])
15# model.eval()
16
17print("Model loaded successfully!")1# Load FP16 checkpoint
2checkpoint = torch.load("model_fp16/model_checkpoint.pth", map_location="cpu")
3
4# Load model and convert to FP16
5# model = YourVisionGPTModel(config)
6# model.load_state_dict(checkpoint['model_state_dict'])
7# model.half() # Convert to FP16
8# model.eval()
9
10# For inference with FP16, also convert input images to FP161image_transform = transforms.Compose([
2 transforms.Resize((224, 224)),
3 transforms.Lambda(lambda x: x.convert('RGB')),
4 transforms.ToTensor(),
5 transforms.Normalize(
6 mean=[0.485, 0.456, 0.406],
7 std=[0.229, 0.224, 0.225]
8 ),
9])
10
11# Load and preprocess image
12image = Image.open("your_image.jpg")
13image_tensor = image_transform(image).unsqueeze(0) # Add batch dimension1# Generate caption
2with torch.no_grad():
3 # Forward pass
4 generated_ids = model.generate(
5 image_tensor,
6 max_length=50,
7 num_beams=5,
8 temperature=0.7
9 )
10
11 # Decode caption
12 caption = tokenizer.decode(generated_ids[0], skip_special_tokens=True)
13 print(f"Generated caption: {caption}")┌─────────────────┐
│ Input Image │
│ (224x224) │
└────────┬────────┘
│
▼
┌─────────────────┐
│ ViT-B/16 │
│ (frozen) │
└────────┬────────┘
│
▼
┌─────────────────┐
│ Projection │
│ (trainable) │
└────────┬────────┘
│
▼
┌─────────────────┐
│ GPT-2 │
│ (frozen) │
└────────┬────────┘
│
▼
┌─────────────────┐
│ Caption Output │
└─────────────────┘1@misc{vision-gpt-flickr8k,
2 author = {gurumurthy3},
3 title = {Vision-GPT: Image Captioning with ViT and GPT-2},
4 year = {2025},
5 publisher = {Hugging Face},
6 howpublished = {\url{https://huggingface.co/gurumurthy3/vision-gpt-flickr8k}}
7}