Views
No views yet
1import gymnasium as gym
2from stable_baselines3 import PPO
3from stable_baselines3.ppo.policies import MlpPolicy
4from stable_baselines3.common.evaluation import evaluate_policy
5from stable_baselines3.common.logger import configure
6from stable_baselines3.common.monitor import Monitor
7from stable_baselines3.common.callbacks import BaseCallback
8from lib import score
9
10from stable_baselines3 import PPO
11from stable_baselines3.common.vec_env import DummyVecEnv
12from stable_baselines3.common.env_util import make_vec_env
13
14from huggingface_sb3 import package_to_hub
15
16class MyCallback(BaseCallback):
17 def __init__(self, verbose=0):
18 super(MyCallback, self).__init__(verbose)
19
20 def _on_step(self) -> bool:
21 current_step = self.num_timesteps
22 if(current_step == 90_000):
23 self.model.learning_rate = 0.001
24 elif(current_step == 130_000):
25 self.model.learning_rate = 0.0003
26 return True
27
28v_last_max_score = 150
29v_target_score = 249
30def training () :
31 global v_last_max_score, v_target_score
32 v_is_succeed = False
33 v_total_steps = 1_000_000
34 gym_name = "LunarLander-v2"
35 model_name = "ppo-LunarLander-v2"
36 # env_1 =
37 env = Monitor(gym.make(gym_name, render_mode="rgb_array"), 'test_monitor')
38
39 ppo_config = {
40 'policy': 'MlpPolicy',
41 'env': env
42 # , 'learning_rate' : learning_rate_schedule #0.0003
43 , 'learning_rate' : 0.01
44 , 'n_steps' :1024 #2048
45 , 'batch_size' :512 #64
46 , 'n_epochs' :20 #10
47 , 'gamma' :0.999 #0.99
48 , 'gae_lambda' :0.95
49 , 'clip_range' :0.2 #0.15
50 , 'clip_range_vf' :None
51 , 'normalize_advantage' :True
52 , 'ent_coef' : 0.000021 #0.0
53 , 'vf_coef' :0.5
54 , 'max_grad_norm' :0.5
55 , 'use_sde' :False
56 , 'sde_sample_freq' :4 #-1
57 , 'rollout_buffer_class' :None
58 , 'rollout_buffer_kwargs' :None
59 , 'target_kl' :None
60 , 'stats_window_size' :100
61 , 'tensorboard_log' :None
62 , 'policy_kwargs' : None
63 , 'verbose' :1
64 , 'seed' :None
65 , 'device' :'auto'
66 , '_init_setup_model' :True
67 }
68
69 model = PPO(**ppo_config)
70
71 tmp_path = "/home/ubuntu/log/"
72 new_logger = configure(tmp_path, ["stdout"])
73 model.set_logger(new_logger)
74
75 callback = MyCallback()
76 model.learn(total_timesteps=v_total_steps, callback=callback)
77 mean_reward, std_reward = evaluate_policy(model, env, n_eval_episodes=10, deterministic=True)
78 new_logger.log(f"mean_reward={mean_reward:.2f} +/- {std_reward}")
79
80 #save for report
81 v_score = mean_reward - std_reward
82 if v_score>v_last_max_score:
83 v_last_max_score = v_score
84 model.save(model_name)
85 score.record_video(gym_name, model, video_length=500, prefix="ppo-lunar")
86
87 #upload
88 upload_env_id = "LunarLander-v2"
89 upload_model_architecture = "PPO"
90 upload_repo_id = "dev-cuai/ppo-LunarLander-v2"
91 upload_commit_message = "Upload PPO LunarLander-v2 trained agent"
92
93 package_to_hub(
94 model=model, # Our trained model
95 model_name=model_name, # The name of our trained model
96 model_architecture=upload_model_architecture, # The model architecture we used: in our case PPO
97 env_id=upload_env_id, # Name of the environment
98 eval_env=env, # Evaluation Environment
99 repo_id=upload_repo_id, # id of the model repository from the Hugging Face Hub (repo_id = {organization}/{repo_name} for instance ThomasSimonini/ppo-LunarLander-v2
100 commit_message=upload_commit_message,
101 )
102
103 v_is_succeed = v_score > v_target_score
104
105 else :
106 print('Too Low Mean_reward({}<{}), model.learning_rate:{}'.format(v_score, v_last_max_score, model.learning_rate))
107
108 return v_is_succeed
109
110v_loop_cnt = 0
111while(True):
112 v_loop_cnt += 1
113 for i in range(3):
114 print("OOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOO")
115 print("run loop {} times. last score was {}".format(v_loop_cnt, v_last_max_score))
116 v_ret = training ()
117 if(v_ret):
118 break
119
120...