Views
No views yet
1# install GenerativeRL with huggingface support
2pip3 install GenerativeRL[huggingface]
3# install environment dependencies if needed
4pip3 install gym[box2d]==0.23.11# running with trained model
2python3 -u run.py1import gym
2
3from grl.algorithms.qgpo import QGPOAlgorithm
4from grl.datasets import QGPOCustomizedTensorDictDataset
5
6from grl.utils.huggingface import pull_model_from_hub
7
8
9def qgpo_pipeline():
10
11 policy_state_dict, config = pull_model_from_hub(
12 repo_id="OpenDILabCommunity/LunarLanderContinuous-v2-QGPO",
13 )
14
15 qgpo = QGPOAlgorithm(
16 config,
17 dataset=QGPOCustomizedTensorDictDataset(
18 numpy_data_path="./data.npz",
19 action_augment_num=config.train.parameter.action_augment_num,
20 ),
21 )
22
23 qgpo.model.load_state_dict(policy_state_dict)
24
25 # ---------------------------------------
26 # Customized train code ↓
27 # ---------------------------------------
28 # qgpo.train()
29 # ---------------------------------------
30 # Customized train code ↑
31 # ---------------------------------------
32
33 # ---------------------------------------
34 # Customized deploy code ↓
35 # ---------------------------------------
36 agent = qgpo.deploy()
37 env = gym.make(config.deploy.env.env_id)
38 observation = env.reset()
39 images = [env.render(mode="rgb_array")]
40 for _ in range(config.deploy.num_deploy_steps):
41 observation, reward, done, _ = env.step(agent.act(observation))
42 image = env.render(mode="rgb_array")
43 images.append(image)
44 # save images into mp4 files
45 import imageio.v3 as imageio
46 import numpy as np
47
48 images = np.array(images)
49 imageio.imwrite("replay.mp4", images, fps=30, quality=8)
50 # ---------------------------------------
51 # Customized deploy code ↑
52 # ---------------------------------------
53
54
55if __name__ == "__main__":
56
57 qgpo_pipeline()
581#Training Your Own Agent
2python3 -u train.py1import gym
2
3from grl.algorithms.qgpo import QGPOAlgorithm
4from grl.datasets import QGPOCustomizedTensorDictDataset
5from grl.utils.log import log
6from grl_pipelines.diffusion_model.configurations.lunarlander_continuous_qgpo import (
7 config,
8)
9
10
11def qgpo_pipeline(config):
12
13 qgpo = QGPOAlgorithm(
14 config,
15 dataset=QGPOCustomizedTensorDictDataset(
16 numpy_data_path="./data.npz",
17 action_augment_num=config.train.parameter.action_augment_num,
18 ),
19 )
20
21 # ---------------------------------------
22 # Customized train code ↓
23 # ---------------------------------------
24 qgpo.train()
25 # ---------------------------------------
26 # Customized train code ↑
27 # ---------------------------------------
28
29 # ---------------------------------------
30 # Customized deploy code ↓
31 # ---------------------------------------
32 agent = qgpo.deploy()
33 env = gym.make(config.deploy.env.env_id)
34 observation = env.reset()
35 for _ in range(config.deploy.num_deploy_steps):
36 env.render()
37 observation, reward, done, _ = env.step(agent.act(observation))
38 # ---------------------------------------
39 # Customized deploy code ↑
40 # ---------------------------------------
41
42
43if __name__ == "__main__":
44 log.info("config: \n{}".format(config))
45 qgpo_pipeline(config)
46{'train': {'project': 'LunarLanderContinuous-v2-QGPO-VPSDE', 'device': 'cuda', 'wandb': {'project': 'IQL-LunarLanderContinuous-v2-QGPO-VPSDE'}, 'simulator': {'type': 'GymEnvSimulator', 'args': {'env_id': 'LunarLanderContinuous-v2'}}, 'model': {'QGPOPolicy': {'device': 'cuda', 'critic': {'device': 'cuda', 'q_alpha': 1.0, 'DoubleQNetwork': {'backbone': {'type': 'ConcatenateMLP', 'args': {'hidden_sizes': [10, 256, 256], 'output_size': 1, 'activation': 'relu'}}}}, 'diffusion_model': {'device': 'cuda', 'x_size': 2, 'alpha': 1.0, 'solver': {'type': 'DPMSolver', 'args': {'order': 2, 'device': 'cuda', 'steps': 17}}, 'path': {'type': 'linear_vp_sde', 'beta_0': 0.1, 'beta_1': 20.0}, 'reverse_path': {'type': 'linear_vp_sde', 'beta_0': 0.1, 'beta_1': 20.0}, 'model': {'type': 'noise_function', 'args': {'t_encoder': {'type': 'GaussianFourierProjectionTimeEncoder', 'args': {'embed_dim': 32, 'scale': 30.0}}, 'backbone': {'type': 'TemporalSpatialResidualNet', 'args': {'hidden_sizes': [512, 256, 128], 'output_dim': 2, 't_dim': 32, 'condition_dim': 8, 'condition_hidden_dim': 32, 't_condition_hidden_dim': 128}}}}, 'energy_guidance': {'t_encoder': {'type': 'GaussianFourierProjectionTimeEncoder', 'args': {'embed_dim': 32, 'scale': 30.0}}, 'backbone': {'type': 'ConcatenateMLP', 'args': {'hidden_sizes': [42, 256, 256], 'output_size': 1, 'activation': 'silu'}}}}}}, 'parameter': {'behaviour_policy': {'batch_size': 1024, 'learning_rate': 0.0001, 'epochs': 500}, 'action_augment_num': 16, 'fake_data_t_span': None, 'energy_guided_policy': {'batch_size': 256}, 'critic': {'stop_training_epochs': 500, 'learning_rate': 0.0001, 'discount_factor': 0.99, 'update_momentum': 0.005}, 'energy_guidance': {'epochs': 1000, 'learning_rate': 0.0001}, 'evaluation': {'evaluation_interval': 50, 'guidance_scale': [0.0, 1.0, 2.0]}, 'checkpoint_path': './LunarLanderContinuous-v2-QGPO'}}, 'deploy': {'device': 'cuda', 'env': {'env_id': 'LunarLanderContinuous-v2', 'seed': 0}, 'num_deploy_steps': 1000, 't_span': None}}1{
2 "train": {
3 "project": "LunarLanderContinuous-v2-QGPO-VPSDE",
4 "device": "cuda",
5 "wandb": {
6 "project": "IQL-LunarLanderContinuous-v2-QGPO-VPSDE"
7 },
8 "simulator": {
9 "type": "GymEnvSimulator",
10 "args": {
11 "env_id": "LunarLanderContinuous-v2"
12 }
13 },
14 "model": {
15 "QGPOPolicy": {
16 "device": "cuda",
17 "critic": {
18 "device": "cuda",
19 "q_alpha": 1.0,
20 "DoubleQNetwork": {
21 "backbone": {
22 "type": "ConcatenateMLP",
23 "args": {
24 "hidden_sizes": [
25 10,
26 256,
27 256
28 ],
29 "output_size": 1,
30 "activation": "relu"
31 }
32 }
33 }
34 },
35 "diffusion_model": {
36 "device": "cuda",
37 "x_size": 2,
38 "alpha": 1.0,
39 "solver": {
40 "type": "DPMSolver",
41 "args": {
42 "order": 2,
43 "device": "cuda",
44 "steps": 17
45 }
46 },
47 "path": {
48 "type": "linear_vp_sde",
49 "beta_0": 0.1,
50 "beta_1": 20.0
51 },
52 "reverse_path": {
53 "type": "linear_vp_sde",
54 "beta_0": 0.1,
55 "beta_1": 20.0
56 },
57 "model": {
58 "type": "noise_function",
59 "args": {
60 "t_encoder": {
61 "type": "GaussianFourierProjectionTimeEncoder",
62 "args": {
63 "embed_dim": 32,
64 "scale": 30.0
65 }
66 },
67 "backbone": {
68 "type": "TemporalSpatialResidualNet",
69 "args": {
70 "hidden_sizes": [
71 512,
72 256,
73 128
74 ],
75 "output_dim": 2,
76 "t_dim": 32,
77 "condition_dim": 8,
78 "condition_hidden_dim": 32,
79 "t_condition_hidden_dim": 128
80 }
81 }
82 }
83 },
84 "energy_guidance": {
85 "t_encoder": {
86 "type": "GaussianFourierProjectionTimeEncoder",
87 "args": {
88 "embed_dim": 32,
89 "scale": 30.0
90 }
91 },
92 "backbone": {
93 "type": "ConcatenateMLP",
94 "args": {
95 "hidden_sizes": [
96 42,
97 256,
98 256
99 ],
100 "output_size": 1,
101 "activation": "silu"
102 }
103 }
104 }
105 }
106 }
107 },
108 "parameter": {
109 "behaviour_policy": {
110 "batch_size": 1024,
111 "learning_rate": 0.0001,
112 "epochs": 500
113 },
114 "action_augment_num": 16,
115 "fake_data_t_span": null,
116 "energy_guided_policy": {
117 "batch_size": 256
118 },
119 "critic": {
120 "stop_training_epochs": 500,
121 "learning_rate": 0.0001,
122 "discount_factor": 0.99,
123 "update_momentum": 0.005
124 },
125 "energy_guidance": {
126 "epochs": 1000,
127 "learning_rate": 0.0001
128 },
129 "evaluation": {
130 "evaluation_interval": 50,
131 "guidance_scale": [
132 0.0,
133 1.0,
134 2.0
135 ]
136 },
137 "checkpoint_path": "./LunarLanderContinuous-v2-QGPO"
138 }
139 },
140 "deploy": {
141 "device": "cuda",
142 "env": {
143 "env_id": "LunarLanderContinuous-v2",
144 "seed": 0
145 },
146 "num_deploy_steps": 1000,
147 "t_span": null
148 }
149}