Views
No views yet
1# install huggingface_ding
2git clone https://github.com/opendilab/huggingface_ding.git
3pip3 install -e ./huggingface_ding/
4# install environment dependencies if needed
5
6pip3 install DI-engine[common_env,video]
7pip3 install LightZero
81# running with trained model
2python3 -u run.py1from lzero.agent import GumbelMuZeroAgent
2from ding.config import Config
3from easydict import EasyDict
4import torch
5
6# Pull model from files which are git cloned from huggingface
7policy_state_dict = torch.load("pytorch_model.bin", map_location=torch.device("cpu"))
8cfg = EasyDict(Config.file_to_dict("policy_config.py").cfg_dict)
9# Instantiate the agent
10agent = GumbelMuZeroAgent(
11 env_id="TicTacToe-play-with-bot", exp_name="TicTacToe-play-with-bot-GumbelMuZero", cfg=cfg.exp_config, policy_state_dict=policy_state_dict
12)
13# Continue training
14agent.train(step=5000)
15# Render the new agent performance
16agent.deploy(enable_save_replay=True)
171# running with trained model
2python3 -u run.py1from lzero.agent import GumbelMuZeroAgent
2from huggingface_ding import pull_model_from_hub
3
4# Pull model from Hugggingface hub
5policy_state_dict, cfg = pull_model_from_hub(repo_id="OpenDILabCommunity/TicTacToe-play-with-bot-GumbelMuZero")
6# Instantiate the agent
7agent = GumbelMuZeroAgent(
8 env_id="TicTacToe-play-with-bot", exp_name="TicTacToe-play-with-bot-GumbelMuZero", cfg=cfg.exp_config, policy_state_dict=policy_state_dict
9)
10# Continue training
11agent.train(step=5000)
12# Render the new agent performance
13agent.deploy(enable_save_replay=True)
141#Training Your Own Agent
2python3 -u train.py1from lzero.agent import GumbelMuZeroAgent
2from huggingface_ding import push_model_to_hub
3
4# Instantiate the agent
5agent = GumbelMuZeroAgent(env_id="TicTacToe-play-with-bot", exp_name="TicTacToe-play-with-bot-GumbelMuZero")
6# Train the agent
7return_ = agent.train(step=int(10000000))
8# Push model to huggingface hub
9push_model_to_hub(
10 agent=agent.best,
11 env_name="OpenAI/Gym/Atari",
12 task_name="TicTacToe-play-with-bot",
13 algo_name="GumbelMuZero",
14 github_repo_url="https://github.com/opendilab/LightZero",
15 github_doc_model_url=None,
16 github_doc_env_url=None,
17 installation_guide='''
18pip3 install DI-engine[common_env,video]
19pip3 install LightZero
20''',
21 usage_file_by_git_clone="./gumbel_muzero/tictactoe_play_with_bot_gumbel_muzero_deploy.py",
22 usage_file_by_huggingface_ding="./gumbel_muzero/tictactoe_play_with_bot_gumbel_muzero_download.py",
23 train_file="./gumbel_muzero/tictactoe_play_with_bot_gumbel_muzero.py",
24 repo_id="OpenDILabCommunity/TicTacToe-play-with-bot-GumbelMuZero",
25 platform_info="[LightZero](https://github.com/opendilab/LightZero) and [DI-engine](https://github.com/opendilab/di-engine)",
26 model_description="**LightZero** is an efficient, easy-to-understand open-source toolkit that merges Monte Carlo Tree Search (MCTS) with Deep Reinforcement Learning (RL), simplifying their integration for developers and researchers. More details are in paper [LightZero: A Unified Benchmark for Monte Carlo Tree Search in General Sequential Decision Scenarios](https://huggingface.co/papers/2310.08348).",
27 create_repo=True
28)
291exp_config = {
2 'main_config': {
3 'exp_name': 'TicTacToe-play-with-bot-GumbelMuZero',
4 'seed': 0,
5 'env': {
6 'env_id': 'TicTacToe-play-with-bot',
7 'battle_mode': 'play_with_bot_mode',
8 'collector_env_num': 8,
9 'evaluator_env_num': 5,
10 'n_evaluator_episode': 5,
11 'manager': {
12 'shared_memory': False
13 }
14 },
15 'policy': {
16 'on_policy': False,
17 'cuda': True,
18 'multi_gpu': False,
19 'bp_update_sync': True,
20 'traj_len_inf': False,
21 'model': {
22 'observation_shape': [3, 3, 3],
23 'action_space_size': 9,
24 'image_channel': 3,
25 'num_res_blocks': 1,
26 'num_channels': 16,
27 'fc_reward_layers': [8],
28 'fc_value_layers': [8],
29 'fc_policy_layers': [8],
30 'support_scale': 10,
31 'reward_support_size': 21,
32 'value_support_size': 21
33 },
34 'use_rnd_model': False,
35 'sampled_algo': False,
36 'gumbel_algo': True,
37 'mcts_ctree': True,
38 'collector_env_num': 8,
39 'evaluator_env_num': 5,
40 'env_type': 'board_games',
41 'action_type': 'varied_action_space',
42 'battle_mode': 'play_with_bot_mode',
43 'monitor_extra_statistics': True,
44 'game_segment_length': 5,
45 'transform2string': False,
46 'gray_scale': False,
47 'use_augmentation': False,
48 'augmentation': ['shift', 'intensity'],
49 'ignore_done': False,
50 'update_per_collect': 50,
51 'model_update_ratio': 0.1,
52 'batch_size': 256,
53 'optim_type': 'Adam',
54 'learning_rate': 0.003,
55 'target_update_freq': 100,
56 'target_update_freq_for_intrinsic_reward': 1000,
57 'weight_decay': 0.0001,
58 'momentum': 0.9,
59 'grad_clip_value': 0.5,
60 'n_episode': 8,
61 'num_simulations': 30,
62 'discount_factor': 1,
63 'td_steps': 9,
64 'num_unroll_steps': 3,
65 'reward_loss_weight': 1,
66 'value_loss_weight': 0.25,
67 'policy_loss_weight': 1,
68 'policy_entropy_loss_weight': 0,
69 'ssl_loss_weight': 0,
70 'lr_piecewise_constant_decay': False,
71 'threshold_training_steps_for_final_lr': 50000,
72 'manual_temperature_decay': False,
73 'threshold_training_steps_for_final_temperature': 100000,
74 'fixed_temperature_value': 0.25,
75 'use_ture_chance_label_in_chance_encoder': False,
76 'use_priority': True,
77 'priority_prob_alpha': 0.6,
78 'priority_prob_beta': 0.4,
79 'root_dirichlet_alpha': 0.3,
80 'root_noise_weight': 0.25,
81 'random_collect_episode_num': 0,
82 'eps': {
83 'eps_greedy_exploration_in_collect': False,
84 'type': 'linear',
85 'start': 1.0,
86 'end': 0.05,
87 'decay': 100000
88 },
89 'cfg_type': 'GumbelMuZeroPolicyDict',
90 'max_num_considered_actions': 3,
91 'reanalyze_ratio': 0.0,
92 'eval_freq': 2000,
93 'replay_buffer_size': 10000
94 },
95 'wandb_logger': {
96 'gradient_logger': False,
97 'video_logger': False,
98 'plot_logger': False,
99 'action_logger': False,
100 'return_logger': False
101 }
102 },
103 'create_config': {
104 'env': {
105 'type': 'tictactoe',
106 'import_names': ['zoo.board_games.tictactoe.envs.tictactoe_env']
107 },
108 'env_manager': {
109 'type': 'subprocess'
110 },
111 'policy': {
112 'type': 'gumbel_muzero',
113 'import_names': ['lzero.policy.gumbel_muzero']
114 }
115 }
116}
117