You can deploy this model and start earning money today!

Hugescoot/
spacelander

stable-baselines3

LunarLander-v3

deep-reinforcement-learning

reinforcement-learning

model-index

Views

No views yet

PPO Agent playing LunarLander-v3

This is a trained model of a PPO agent playing LunarLander-v3 using the stable-baselines3 library.

Usage (with Stable-baselines3)

-- coding: utf-8 --

"""Deep RL Unit1.ipynb

Automatically generated by Colab.

Original file is located at https://colab.research.google.com/drive/1SQgvy8qXkXyWWTFJVi8cbmoEYlDgHGhV

##Installing dependencies """

from google.colab import drive drive.mount('/content/drive')

!apt install swig cmake !apt-get update !apt-get install -y libsdl2-dev libsdl2-image-dev libsdl2-mixer-dev libsdl2-ttf-dev !pip install -r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit1/requirements-unit1.txt import os

!sudo apt-get update !sudo apt-get install -y python3-opengl !apt install ffmpeg !apt install xvfb !pip3 install pyvirtualdisplay

"""## Crashing the system, will need to be restarted below"""

import os os.kill(os.getpid(), 9)

"""## Importing more necessary packages and dependencies"""

Virtual display

from pyvirtualdisplay import Display virtual_display = Display(visible=0, size=(1400, 900)) virtual_display.start()

!pip install stable_baselines3 !pip install huggingface_sb3 !pip -q install optuna

import gymnasium as gym import optuna from stable_baselines3 import PPO from stable_baselines3.common.monitor import Monitor from stable_baselines3.common.vec_env import DummyVecEnv from huggingface_sb3 import load_from_hub, package_to_hub from huggingface_hub import notebook_login from stable_baselines3.common.env_util import make_vec_env from stable_baselines3.common.evaluation import evaluate_policy

--- Monkey patch for Hugging Face + SB3 compatibility ---

from stable_baselines3.common.vec_env import VecVideoRecorder

def _patched_getattr(self, name): if name == "video_recorder": # Hugging Face expects this return self if name == "path": # sometimes it asks for .path return getattr(self, "video_path", None) raise AttributeError(name)

VecVideoRecorder.getattr = _patched_getattr print("✅ Patched VecVideoRecorder to provide .video_recorder and .path")

!pip install "gymnasium[box2d]"

Create a multivector environment

env = make_vec_env('LunarLander-v3', n_envs=16)

Training the PPO Model with 1m steps using the environment defined above

model =PPO(policy = 'MlpPolicy', env = env, n_steps = 1024, batch_size = 64, n_epochs = 4, gamma = 0.999, gae_lambda = 0.98, ent_coef = 0.01, verbose=1)

Train the agent

model.learn(total_timesteps=1000000)

Save the model to disk

model.save("/content/drive/MyDrive/ppo-LunarLander-v3.zip")

"""## Train a simple paramaterized model and save to disk"""

import gymnasium as gym from stable_baselines3 import PPO from stable_baselines3.common.monitor import Monitor from stable_baselines3.common.vec_env import DummyVecEnv from stable_baselines3.common.evaluation import evaluate_policy

ENV_ID = "LunarLander-v3" EVAL_EPISODES = 10 TRAIN_STEPS_PER_TRIAL = 150_000

--- candidate settings to try ---

candidates = [ {"learning_rate": 3e-4, "n_steps": 1024, "batch_size": 64, "n_epochs": 4, "gamma": 0.999, "gae_lambda": 0.98, "clip_range": 0.2, "ent_coef": 0.01, "vf_coef": 0.5, "max_grad_norm": 0.5}, {"learning_rate": 1e-4, "n_steps": 2048, "batch_size": 128, "n_epochs": 10, "gamma": 0.99, "gae_lambda": 0.95, "clip_range": 0.2, "ent_coef": 0.00, "vf_coef": 0.5, "max_grad_norm": 0.5}, {"learning_rate": 7e-4, "n_steps": 512, "batch_size": 64, "n_epochs": 4, "gamma": 0.995, "gae_lambda": 0.98, "clip_range": 0.2, "ent_coef": 0.005,"vf_coef": 0.7, "max_grad_norm": 0.7}, ]

best_mean = -1e9 best_cfg = None best_model = None vectorcount = 16

trial_index = 0 for cfg in candidates: trial_index += 1 print(f"\n=== Trial {trial_index} / {len(candidates)} ===") print(cfg)

# --- Training env (vectorized with 8 copies for speed/stability) ---
def make_training_env():
    return Monitor(gym.make(ENV_ID))
train_env = DummyVecEnv([make_training_env for _ in range(vectorcount)])

# --- Build PPO with this config ---
model = PPO(
    policy="MlpPolicy",
    env=train_env,
    learning_rate=cfg["learning_rate"],
    n_steps=cfg["n_steps"],
    batch_size=cfg["batch_size"],
    n_epochs=cfg["n_epochs"],
    gamma=cfg["gamma"],
    gae_lambda=cfg["gae_lambda"],
    clip_range=cfg["clip_range"],
    ent_coef=cfg["ent_coef"],
    vf_coef=cfg["vf_coef"],
    max_grad_norm=cfg["max_grad_norm"],
    verbose=0,
)

# --- Train briefly for comparison ---
model.learn(total_timesteps=TRAIN_STEPS_PER_TRIAL)

# --- Evaluate on a single env (deterministic) ---
eval_env = Monitor(gym.make(ENV_ID))
mean_reward, std_reward = evaluate_policy(
    model, eval_env, n_eval_episodes=EVAL_EPISODES, deterministic=True
)
print(f"Mean reward: {mean_reward:.1f} +/- {std_reward:.1f}")

# --- Track best ---
if mean_reward > best_mean:
    best_mean = mean_reward
    best_cfg = cfg
    best_model = model  # keep the trained model

# --- Clean up envs ---
eval_env.close()
# train_env will be closed when best_model replaces model;
# to be explicit:
# train_env.close()

print("\nBest config:") print(best_cfg) print(f"Best mean reward: {best_mean:.1f}")

--- Optionally train longer with best config ---

You can keep training best_model for more steps:

best_model.learn(total_timesteps=1000000) best_model.save("ppo-LunarLander-v3-best.zip") best_model.env.close()

"""##Train a more Sophisticated Paramaterization"""

ENV_ID = "LunarLander-v3" EVAL_EPISODES = 8 TRAIN_STEPS_PER_TRIAL = 150_000 N_TRIALS = 15

def objective(trial: optuna.Trial) -> float: # --- Suggest hyperparameters (search space) --- learning_rate = trial.suggest_float("learning_rate", 1e-5, 3e-3, log=True) n_steps = trial.suggest_categorical("n_steps", [512, 1024, 2048]) batch_size = trial.suggest_categorical("batch_size", [64, 128, 256]) n_epochs = trial.suggest_categorical("n_epochs", [4, 5, 10]) gamma = trial.suggest_float("gamma", 0.98, 0.9999) gae_lambda = trial.suggest_float("gae_lambda", 0.90, 0.99) clip_range = trial.suggest_float("clip_range", 0.1, 0.3) ent_coef = trial.suggest_float("ent_coef", 0.0, 0.02) vf_coef = trial.suggest_float("vf_coef", 0.4, 1.0) max_grad_norm = trial.suggest_float("max_grad_norm", 0.3, 1.0)

# --- Training env (vectorized) ---
def make_training_env():
    return Monitor(gym.make(ENV_ID))
train_env = DummyVecEnv([make_training_env for _ in range(8)])

# --- Model ---
model = PPO(
    "MlpPolicy",
    train_env,
    learning_rate=learning_rate,
    n_steps=n_steps,
    batch_size=batch_size,
    n_epochs=n_epochs,
    gamma=gamma,
    gae_lambda=gae_lambda,
    clip_range=clip_range,
    ent_coef=ent_coef,
    vf_coef=vf_coef,
    max_grad_norm=max_grad_norm,
    verbose=0,
)

# --- Train briefly ---
model.learn(total_timesteps=TRAIN_STEPS_PER_TRIAL)

# --- Evaluate ---
eval_env = Monitor(gym.make(ENV_ID))
mean_reward, std_reward = evaluate_policy(
    model, eval_env, n_eval_episodes=EVAL_EPISODES, deterministic=True
)

# --- Cleanup ---
eval_env.close()
# train_env.close()  # optional explicit close

# --- Goal: maximize mean reward ---
return float(mean_reward)

study = optuna.create_study(direction="maximize", study_name="ppo_ll_v3_tuning") study.optimize(objective, n_trials=N_TRIALS, show_progress_bar=True)

print("Best value (mean reward):", study.best_value) print("Best params:", study.best_params)

--- Final training with best params (scale up steps/envs if you want) ---

best = study.best_params def make_training_env(): return Monitor(gym.make(ENV_ID)) final_env = DummyVecEnv([make_training_env for _ in range(16)]) # more envs for final run

final_model = PPO( "MlpPolicy", final_env, learning_rate=best["learning_rate"], n_steps=best["n_steps"], batch_size=best["batch_size"], n_epochs=best["n_epochs"], gamma=best["gamma"], gae_lambda=best["gae_lambda"], clip_range=best["clip_range"], ent_coef=best["ent_coef"], vf_coef=best["vf_coef"], max_grad_norm=best["max_grad_norm"], verbose=1, )

final_model.learn(total_timesteps=1_000_000) final_model.save("/content/drive/MyDrive/ppo-LunarLander-v3-best_p.zip") final_model.env.close()

"""## Load the model from Disk

"""

#Load the model from

Create a new environment for evaluation

eval_env = gym.make('LunarLander-v3') #model_name = "ppo-LunarLander-v2" #model = PPO.load("/content/drive/MyDrive/ppo-LunarLander-v2.zip", env=eval_env) model = PPO.load("/content/drive/MyDrive/ppo-LunarLander-v3-best_p.zip", env=eval_env) #ppo-LunarLander-v3-best.zip #model = PPO.load("ppo-LunarLander-v3-best.zip", env=eval_env)

"""## Evaluate the model"""

model=final_model

Evaluate the model with 10 evaluation episodes and deterministic=True

mean_reward, std_reward = evaluate_policy(model, model.get_env(), n_eval_episodes=10)

Print the results

vec_env = model.get_env() obs = vec_env.reset() for i in range(1000): action, _states = model.predict(obs, deterministic=True) obs, rewards, dones, info = vec_env.step(action) #print(f"Step {i}: Observation={obs}, Reward={rewards}, Terminated={dones}, Truncated={info}") #vec_env.render("human")

#print(f"mean_reward={mean_reward:.2f} +/- {std_reward}")

"""## Saving Package to HuggingFace

"""

#This didnt work so I moved the token to the next piece of code #notebook_login() #!git config --global credential.helper store

import gymnasium as gym

from stable_baselines3 import PPO from stable_baselines3.common.vec_env import DummyVecEnv from stable_baselines3.common.env_util import make_vec_env

from huggingface_sb3 import package_to_hub

PLACE the variables you've just defined two cells above

Define the name of the environment

env_id = "LunarLander-v3"

modelname="lunarlander_HS"

TODO: Define the model architecture we used

model_architecture = "PPO"

#insert token= here

repo_id = "Hugescoot/spacelander"

Define the commit message

commit_message = "Upload PPO LunarLander-v2 trained agent"

Create the evaluation env and set the render_mode="rgb_array"

eval_env = DummyVecEnv([lambda: gym.make(env_id, render_mode="rgb_array")])

PLACE the package_to_hub function you've just filled here

package_to_hub(model=model, # Our trained model model_name=modelname, # The name of our trained model model_architecture=model_architecture, # The model architecture we used: in our case PPO env_id=env_id, # Name of the environment eval_env=eval_env, # Evaluation Environment repo_id=repo_id, # id of the model repository from the Hugging Face Hub (repo_id = {organization}/{repo_name} for instance ThomasSimonini/ppo-LunarLander-v2 commit_message=commit_message, token=token)

"""Congrats 🥳 you've just trained and uploaded your first Deep Reinforcement Learning agent. The script above should have displayed a link to a model repository such as https://huggingface.co/osanseviero/test_sb3. When you go to this link, you can:

See a video preview of your agent at the right.
Click "Files and versions" to see all the files in the repository.
Click "Use in stable-baselines3" to get a code snippet that shows how to load the model.
A model card (README.md file) which gives a description of the model

Under the hood, the Hub uses git-based repositories (don't worry if you don't know what git is), which means you can update the model with new versions as you experiment and improve your agent.

Compare the results of your LunarLander-v2 with your classmates using the leaderboard 🏆 👉 https://huggingface.co/spaces/huggingface-projects/Deep-Reinforcement-Learning-Leaderboard

Load a saved LunarLander model from the Hub 🤗

Thanks to ironbar for the contribution.

Loading a saved model from the Hub is really easy.

You go to https://huggingface.co/models?library=stable-baselines3 to see the list of all the Stable-baselines3 saved models.

You select one and copy its repo_id

Then we just need to use load_from_hub with:

The repo_id
The filename: the saved model inside the repo and its extension (*.zip)

Because the model I download from the Hub was trained with Gym (the former version of Gymnasium) we need to install shimmy a API conversion tool that will help us to run the environment correctly.

Shimmy Documentation: https://github.com/Farama-Foundation/Shimmy """

!pip install shimmy

from huggingface_sb3 import load_from_hub repo_id = "Classroom-workshop/assignment2-omar" # The repo_id filename = "ppo-LunarLander-v2.zip" # The model filename.zip

When the model was trained on Python 3.8 the pickle protocol is 5

But Python 3.6, 3.7 use protocol 4

In order to get compatibility we need to:

1. Install pickle5 (we done it at the beginning of the colab)

2. Create a custom empty object we pass as parameter to PPO.load()

custom_objects = { "learning_rate": 0.0, "lr_schedule": lambda _: 0.0, "clip_range": lambda _: 0.0, }

checkpoint = load_from_hub(repo_id, filename) model = PPO.load(checkpoint, custom_objects=custom_objects, print_system_info=True)

"""Let's evaluate this agent:"""

#@title eval_env = Monitor(gym.make("LunarLander-v2")) mean_reward, std_reward = evaluate_policy(model, eval_env, n_eval_episodes=10, deterministic=True) print(f"mean_reward={mean_reward:.2f} +/- {std_reward}")

python

from stable_baselines3 import ...
from huggingface_sb3 import load_from_hub

...