ppo_bullet_humanoid.gin

include 'ppo.gin'

# environment config

# baseline ppo2 training command:
# CUDA_VISIBLE_DEVICES=0 OPENAI_LOGDIR=/home/weixu/tmp/bullet_humanoid_baseline/ppo-0 OPENAI_LOG_FORMAT='stdout,tensorboard' python -m baselines.run --alg=ppo2 --env=HumanoidBulletEnv-v0 --network=mlp --num_timesteps=1e8 --ent_coef=0.01 --num_hidden=32 --num_layers=3 --value_network=copy

# need to `pip install pybullet`
import pybullet_envs
create_environment.env_name="HumanoidBulletEnv-v0"
create_environment.num_parallel_environments=96
suite_gym.wrap_env.clip_action=False

# algorithm config
PPOLoss.entropy_regularization=1e-2
PPOLoss.gamma=0.99
PPOLoss.normalize_advantages=True
PPOLoss.td_lambda=0.95
PPOLoss.td_error_loss_fn=@element_wise_squared_loss

actor/ActorDistributionNetwork.fc_layer_params=(32, 32, 32)
actor/ActorDistributionNetwork.activation=@torch.tanh
actor/ActorDistributionNetwork.continuous_projection_net_ctor=@NormalProjectionNetwork
NormalProjectionNetwork.projection_output_init_gain=1e-5
NormalProjectionNetwork.std_bias_initializer_value=0.0
NormalProjectionNetwork.std_transform=@torch.exp

value/ValueNetwork.fc_layer_params=(32, 32, 32)
value/ValueNetwork.activation=@torch.tanh

ac/AdamTF.lr=3e-4
ac/AdamTF.gradient_clipping=0.5
ac/AdamTF.clip_by_global_norm=True

ActorCriticAlgorithm.actor_network_ctor=@actor/ActorDistributionNetwork
ActorCriticAlgorithm.value_network_ctor=@value/ValueNetwork
Agent.optimizer=@ac/AdamTF()


# training config
TrainerConfig.num_updates_per_train_iter = 20
TrainerConfig.unroll_length = 512
TrainerConfig.mini_batch_size = 4096
TrainerConfig.mini_batch_length = 1
TrainerConfig.num_iterations = 1000
TrainerConfig.evaluate=True
TrainerConfig.eval_interval = 100
TrainerConfig.debug_summaries=True
TrainerConfig.summarize_grads_and_vars = True
TrainerConfig.summary_interval = 10