You can deploy this model and start earning money today!

final-project-5190/
model-resnet-50-base

Name: model-resnet-50-base
Availability: InStock

pytorch

custom-resnet

Views

No views yet

Relevant imports & set up

python

!pip install geopy > delete.txt
!pip install datasets > delete.txt
!pip install torch torchvision datasets > delete.txt
!pip install huggingface_hub > delete.txt
!rm delete.txt

python

!pip install transformers
import transformers

!huggingface-cli login --token [your_token]

python

lat_mean = 39.95156937654321
lat_std = 0.0005992518588323268
lon_mean = -75.19136795987654
lon_std = 0.0007030395253318959

Instructions

Our current best performing model is an ensemble of multiple models. To run it on hidden test data, first run the model definitions.

Load and define models

python

from transformers import AutoModelForImageClassification, PretrainedConfig, PreTrainedModel
import torch
import torch.nn as nn
import os
from huggingface_hub import PyTorchModelHubMixin, hf_hub_download
from safetensors.torch import load_file 

class CustomConvNeXtConfig(PretrainedConfig):
    model_type = "custom-convnext"

    def __init__(self, num_labels=2, **kwargs):
        super().__init__(**kwargs)
        self.num_labels = num_labels  # Register number of labels (output dimensions)

class CustomConvNeXtModel(PreTrainedModel):
    config_class = CustomConvNeXtConfig

    def __init__(self, config, model_name="facebook/convnext-tiny-224",
                 num_classes=2, train_final_layer_only=False):
        super().__init__(config)

        # Load pre-trained ConvNeXt model from Hugging Face
        self.convnext = AutoModelForImageClassification.from_pretrained(model_name)

        # Access the input features of the existing classifier
        in_features = self.convnext.classifier.in_features

        # Modify the classifier layer to match the number of output classes
        self.convnext.classifier = nn.Linear(in_features, num_classes)

        # Freeze previous weights if only training the final layer
        if train_final_layer_only:
            for name, param in self.convnext.named_parameters():
                if "classifier" not in name:
                    param.requires_grad = False
                else:
                    print(f"Unfrozen layer: {name}")

    def forward(self, x):
        return self.convnext(x)

    @classmethod
    def from_pretrained(cls, repo_id, model_name="facebook/convnext-tiny-224", **kwargs):
        """Load model weights and configuration from Hugging Face Hub."""
        # Download model.safetensors from Hugging Face Hub
        model_path = hf_hub_download(repo_id=repo_id, filename="model.safetensors")
    
        # Download config.json from Hugging Face Hub
        config_path = hf_hub_download(repo_id=repo_id, filename="config.json")
    
        # Load configuration
        config = CustomConvNeXtConfig.from_pretrained(config_path)
    
        # Create the model
        model = cls(config=config, model_name=model_name, num_classes=config.num_labels)
    
        # Load state_dict from safetensors file
        state_dict = load_file(model_path)
        model.load_state_dict(state_dict)
    
        return model


class CustomResNetConfig(PretrainedConfig):
    model_type = "custom-resnet"

    def __init__(self, num_labels=2, **kwargs):
        super().__init__(**kwargs)
        self.num_labels = num_labels  # Register number of labels (output dimensions)

class CustomResNetModel(nn.Module, PyTorchModelHubMixin):
    config_class = CustomResNetConfig

    def __init__(self, model_name="microsoft/resnet-18",
                 num_classes=2,
                 train_final_layer_only=False):
        super().__init__()

        # Load pre-trained ResNet model from Hugging Face
        self.resnet = AutoModelForImageClassification.from_pretrained(model_name)

        # Access the Linear layer within the Sequential classifier
        in_features = self.resnet.classifier[1].in_features  # Accessing the Linear layer within the Sequential

        # Modify the classifier layer to have the desired number of output classes
        self.resnet.classifier = nn.Sequential(
            nn.Flatten(),
            nn.Linear(in_features, num_classes)
        )

        self.config = CustomResNetConfig(num_labels=num_classes)

        # Freeze previous weights
        if train_final_layer_only:
            for name, param in self.resnet.named_parameters():
                if "classifier" not in name:
                    param.requires_grad = False
                else:
                    print(f"Unfrozen layer: {name}")

    def forward(self, x):
        return self.resnet(x)

    def save_pretrained(self, save_directory, **kwargs):
        """Save model weights and custom configuration in Hugging Face format."""
        os.makedirs(save_directory, exist_ok=True)

        # Save model weights
        torch.save(self.state_dict(), os.path.join(save_directory, "pytorch_model.bin"))

        # Save configuration
        self.config.save_pretrained(save_directory)

    @classmethod
    def from_pretrained(cls, repo_id, model_name="microsoft/resnet-18", **kwargs):
        """Load model weights and configuration from Hugging Face Hub or local directory."""
        # Download pytorch_model.bin from Hugging Face Hub
        model_path = hf_hub_download(repo_id=repo_id, filename="pytorch_model.bin")

        # Download config.json from Hugging Face Hub
        config_path = hf_hub_download(repo_id=repo_id, filename="config.json")

        # Load configuration
        config = CustomResNetConfig.from_pretrained(config_path)

        # Create the model
        model = cls(model_name=model_name, num_classes=config.num_labels)

        # Load state_dict
        model.load_state_dict(torch.load(model_path, map_location=torch.device("cpu")))

        return model


class CustomEfficientNetConfig(PretrainedConfig):
    model_type = "custom-efficientnet"

    def __init__(self, num_labels=2, **kwargs):
        super().__init__(**kwargs)
        self.num_labels = num_labels  # Register number of labels (output dimensions)

class CustomEfficientNetModel(PreTrainedModel):
    config_class = CustomEfficientNetConfig

    def __init__(self, config, model_name="google/efficientnet-b0",
                 num_classes=2, train_final_layer_only=False):
        super().__init__(config)

        # Load pre-trained EfficientNet model from Hugging Face
        self.efficientnet = AutoModelForImageClassification.from_pretrained(model_name)

        # Access the input features of the existing classifier
        in_features = self.efficientnet.classifier.in_features

        # Modify the classifier layer to match the number of output classes
        self.efficientnet.classifier = nn.Sequential(
            nn.Linear(in_features, num_classes)
        )

        # Freeze previous weights if only training the final layer
        if train_final_layer_only:
            for name, param in self.efficientnet.named_parameters():
                if "classifier" not in name:
                    param.requires_grad = False
                else:
                    print(f"Unfrozen layer: {name}")

    def forward(self, x):
        return self.efficientnet(x)

    @classmethod
    def from_pretrained(cls, repo_id, model_name="google/efficientnet-b0", **kwargs):
        """Load model weights and configuration from Hugging Face Hub."""
        # Attempt to download the safetensors model file
        try:
            model_path = hf_hub_download(repo_id=repo_id, filename="model.safetensors")
            state_dict = load_file(model_path)
        except Exception as e:
            raise ValueError(
                f"Failed to download or load 'model.safetensors' from {repo_id}. Ensure the file exists."
            ) from e

        # Download config.json from Hugging Face Hub
        config_path = hf_hub_download(repo_id=repo_id, filename="config.json")

        # Load configuration
        config = CustomEfficientNetConfig.from_pretrained(config_path)

        # Create the model
        model = cls(config=config, model_name=model_name, num_classes=config.num_labels)

        # Load the state_dict into the model
        model.load_state_dict(state_dict)

        return model


class CustomViTConfig(PretrainedConfig):
    model_type = "custom-vit"

    def __init__(self, num_labels=2, **kwargs):
        super().__init__(**kwargs)
        self.num_labels = num_labels  # Register number of labels (output dimensions)

class CustomViTModel(PreTrainedModel):
    config_class = CustomViTConfig

    def __init__(self, config, model_name="google/vit-base-patch16-224",
                 num_classes=2, train_final_layer_only=False):
        super().__init__(config)

        # Load pre-trained ViT model from Hugging Face
        self.vit = AutoModelForImageClassification.from_pretrained(model_name)

        # Access the input features of the existing classifier
        in_features = self.vit.classifier.in_features

        # Modify the classifier layer to match the number of output classes
        self.vit.classifier = nn.Linear(in_features, num_classes)

        # Freeze previous weights if only training the final layer
        if train_final_layer_only:
            for name, param in self.vit.named_parameters():
                if "classifier" not in name:
                    param.requires_grad = False
                else:
                    print(f"Unfrozen layer: {name}")

    def forward(self, x):
        return self.vit(x)

    @classmethod
    def from_pretrained(cls, repo_id, model_name="google/vit-base-patch16-224", **kwargs):
        # Attempt to download the safetensors model file
        try:
            model_path = hf_hub_download(repo_id=repo_id, filename="model.safetensors")
            state_dict = load_file(model_path)
        except Exception as e:
            raise ValueError(
                f"Failed to download or load 'model.safetensors' from {repo_id}. Ensure the file exists."
            ) from e

        # Download config.json from Hugging Face Hub
        config_path = hf_hub_download(repo_id=repo_id, filename="config.json")

        # Load configuration
        config = CustomViTConfig.from_pretrained(config_path)

        # Create the model
        model = cls(config=config, model_name=model_name, num_classes=config.num_labels)

        # Load the state_dict into the model
        model.load_state_dict(state_dict)

        return model


# Define the WeightedEnsembleModel class
class WeightedEnsembleModel(nn.Module):
    def __init__(self, models, weights):
        """
        Initialize the ensemble model with individual models and their weights.
        """
        super(WeightedEnsembleModel, self).__init__()
        self.models = nn.ModuleList(models)  # Wrap models in ModuleList
        self.weights = weights

    def forward(self, images):
        """
        Forward pass for the ensemble model.
        Performs weighted averaging of logits from individual models.
        """
        ensemble_logits = torch.zeros((images.size(0), 2)).to(images.device)  # Initialize logits
        for model, weight in zip(self.models, self.weights):
            outputs = model(images)
            logits = outputs.logits if hasattr(outputs, "logits") else outputs  # Extract logits
            ensemble_logits += weight * logits  # Weighted sum of logits
        return ensemble_logits

Now, load the model weights from huggingface.

python

from transformers import AutoModelForImageClassification
import torch
from sklearn.metrics import mean_absolute_error, mean_squared_error
import matplotlib.pyplot as plt
import numpy as np

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

python

#resnet
resnet = CustomResNetModel.from_pretrained(
    "final-project-5190/model-resnet-50-base",
    model_name="microsoft/resnet-50"
)

#convnext
convnext=CustomConvNeXtModel.from_pretrained(
    "final-project-5190/model-convnext-tiny-reducePlateau", 
    model_name="facebook/convnext-tiny-224")

#vit
vit = CustomViTModel.from_pretrained(
    "final-project-5190/model-ViT-base",
    model_name="google/vit-base-patch16-224"
)

#efficientnet
efficientnet = CustomEfficientNetModel.from_pretrained(
    "final-project-5190/model-efficientnet-b0-base",
    model_name="google/efficientnet-b0"
)

models = [convnext, resnet, vit, efficientnet]
weights = [0.28, 0.26, 0.20, 0.27]

For data loading

python

# Download
from datasets import load_dataset, Image

python

import torch
import torch.nn as nn
import torchvision.models as models
import torchvision.transforms as transforms
from torch.utils.data import DataLoader, Dataset
from transformers import AutoImageProcessor, AutoModelForImageClassification, AutoConfig
from huggingface_hub import PyTorchModelHubMixin, hf_hub_download
from PIL import Image
import os
import numpy as np

class GPSImageDataset(Dataset):
    def __init__(self, hf_dataset, transform=None, lat_mean=None, lat_std=None, lon_mean=None, lon_std=None):
        self.hf_dataset = hf_dataset
        self.transform = transform

        # Compute mean and std from the dataframe if not provided
        self.latitude_mean = lat_mean if lat_mean is not None else np.mean(np.array(self.hf_dataset['Latitude']))
        self.latitude_std = lat_std if lat_std is not None else np.std(np.array(self.hf_dataset['Latitude']))
        self.longitude_mean = lon_mean if lon_mean is not None else np.mean(np.array(self.hf_dataset['Longitude']))
        self.longitude_std = lon_std if lon_std is not None else np.std(np.array(self.hf_dataset['Longitude']))

    def __len__(self):
        return len(self.hf_dataset)

    def __getitem__(self, idx):
        # Extract data
        example = self.hf_dataset[idx]

        # Load and process the image
        image = example['image']
        latitude = example['Latitude']
        longitude = example['Longitude']
        # image = image.rotate(-90, expand=True)
        if self.transform:
            image = self.transform(image)

        # Normalize GPS coordinates
        latitude = (latitude - self.latitude_mean) / self.latitude_std
        longitude = (longitude - self.longitude_mean) / self.longitude_std
        gps_coords = torch.tensor([latitude, longitude], dtype=torch.float32)

        return image, gps_coords

python

# Dataloader + Visualize
transform = transforms.Compose([
    transforms.RandomResizedCrop(224),  # Random crop and resize to 224x224
    transforms.RandomHorizontalFlip(),  # Random horizontal flip
    # transforms.RandomRotation(degrees=15),  # Random rotation between -15 and 15 degrees
    transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2, hue=0.1),  # Random color jitter
    # transforms.GaussianBlur(kernel_size=(3, 5), sigma=(0.1, 2.0)),
    # transforms.RandomPerspective(distortion_scale=0.5, p=0.5),
    transforms.ToTensor(),

    transforms.Normalize(mean=[0.485, 0.456, 0.406],
                         std=[0.229, 0.224, 0.225])
])

# Optionally, you can create a separate transform for inference without augmentations
inference_transform = transforms.Compose([
    transforms.Resize((224, 224)),
    transforms.ToTensor(),
    transforms.Normalize(mean=[0.485, 0.456, 0.406],
                         std=[0.229, 0.224, 0.225])
])

Here's an exmaple of us testing the ensemble on the release test set. You can just change the load release_data line below and run the rest of the code to obtain rMSE.

python

# Load test data
release_data = load_dataset("gydou/released_img", split="train")

python

# Create dataset and dataloader using training mean and std
rel_dataset = GPSImageDataset(
    hf_dataset=release_data,
    transform=inference_transform,
    lat_mean=lat_mean,
    lat_std=lat_std,
    lon_mean=lon_mean,
    lon_std=lon_std
)
rel_dataloader = DataLoader(rel_dataset, batch_size=32, shuffle=False)

python

# ensemble
ensemble_model = WeightedEnsembleModel(models=models, weights=weights).to(device)

# Validation
all_preds = []
all_actuals = []

ensemble_model.eval()
with torch.no_grad():
    for images, gps_coords in rel_dataloader:
        images, gps_coords = images.to(device), gps_coords.to(device)

        # Weighted ensemble prediction using the new model
        ensemble_logits = ensemble_model(images)

        # Denormalize predictions and actual values
        preds = ensemble_logits.cpu() * torch.tensor([lat_std, lon_std]) + torch.tensor([lat_mean, lon_mean])
        actuals = gps_coords.cpu() * torch.tensor([lat_std, lon_std]) + torch.tensor([lat_mean, lon_mean])

        all_preds.append(preds)
        all_actuals.append(actuals)

# Concatenate all batches
all_preds = torch.cat(all_preds).numpy()
all_actuals = torch.cat(all_actuals).numpy()

# Compute error metrics
mae = mean_absolute_error(all_actuals, all_preds)
rmse = mean_squared_error(all_actuals, all_preds, squared=False)

print(f'Mean Absolute Error: {mae}')
print(f'Root Mean Squared Error: {rmse}')

# Convert predictions and actuals to meters
latitude_mean_radians = np.radians(lat_mean)  # Convert to radians for cosine
meters_per_degree_latitude = 111000  # Constant
meters_per_degree_longitude = 111000 * np.cos(latitude_mean_radians)  # Adjusted for latitude mean

all_preds_meters = all_preds.copy()
all_preds_meters[:, 0] *= meters_per_degree_latitude  # Latitude to meters
all_preds_meters[:, 1] *= meters_per_degree_longitude  # Longitude to meters

all_actuals_meters = all_actuals.copy()
all_actuals_meters[:, 0] *= meters_per_degree_latitude  # Latitude to meters
all_actuals_meters[:, 1] *= meters_per_degree_longitude  # Longitude to meters

# Compute error metrics in meters
mae_meters = mean_absolute_error(all_actuals_meters, all_preds_meters)
rmse_meters = mean_squared_error(all_actuals_meters, all_preds_meters, squared=False)

print(f"Mean Absolute Error (meters): {mae_meters:.2f}")
print(f"Root Mean Squared Error (meters): {rmse_meters:.2f}")

After running inference on the release test set, our results are the following.

Release Dataset Mean Absolute Error: 0.0004267849560326909
Release Dataset Root Mean Squared Error: 0.0005247778631268114
Mean Absolute Error (meters): 41.90
Root Mean Squared Error (meters): 51.29