DDPG updated

agent/actor.py

@@ -82,9 +82,10 @@ def train(self, obs, critic, q_grads):
 		"""
 		with tf.GradientTape() as tape:
 			actions = self.network(obs)
-			actor_loss = -tf.reduce_mean(critic([obs,actions]))
-			# actor_grad = tape.gradient(self.network(obs), self.network.trainable_variables,-q_grads)
-		actor_grad = tape.gradient(actor_loss,self.network.trainable_variables)
+			# actor_loss = -tf.reduce_mean(critic([obs,actions]))
+			actor_grad = tape.gradient(self.network(obs), self.network.trainable_variables,-q_grads)
+			# tf.print("actor loss :",actor_loss)
+		# actor_grad = tape.gradient(actor_loss,self.network.trainable_variables)
 		self.optimizer.apply_gradients(zip(actor_grad,self.network.trainable_variables))
 
 	def target_update(self):

agent/ddpg.py

@@ -68,7 +68,7 @@ def update_networks(self, obs, acts, critic_target):
 
 		# get next action and Q-value Gradient
 		n_actions = self.actor.network.predict(obs)
-		q_grads = None#self.critic.Qgradient(obs, n_actions)
+		q_grads = self.critic.Qgradient(obs, n_actions)
 
 		# update actor
 		self.actor.train(obs,self.critic.network,q_grads)
@@ -78,7 +78,7 @@ def update_networks(self, obs, acts, critic_target):
 		self.critic.target_update()
 
 	def replay(self, replay_num_):
-		if self.buffer.size() <= self.batch_size: return
+		if self.with_per and (self.buffer.size() <= self.batch_size): return
 
 		for _ in range(replay_num_):
 			# sample from buffer

requirements.txt

@@ -1,7 +1,8 @@
 gym==0.15.4
 roboschool==1.0.48
-tensorflow>=2.4.0
+tensorflow==2.2.1
 keras==2.4.3
 scipy==1.4.1
 tqdm
-numpy==1.16.0
+numpy==1.16.0
+matplotlib

train.py

@@ -29,24 +29,21 @@
 import roboschool, gym
 import numpy as np, time, os
 from tqdm import tqdm
+import matplotlib.pyplot as plt
 
 import argparse
 
 from agent.ddpg import ddpgAgent
 
-NUM_EPISODES_ = 1000
+NUM_EPISODES_ = 3000
 
 def model_train(pretrained_):
 	# Create Environments
 	models = {	'cheetah':"RoboschoolHalfCheetah-v1",
-				'ant':'RoboschoolAnt-v1',
-				'pong':"RoboschoolPong-v1",
 				'walker':"RoboschoolWalker2d-v1",
-				'hopper':"RoboschoolHopper-v1",
-				'humanoid':"RoboschoolHumanoid-v1",
-				'humanoidflag':"RoboschoolHumanoidFlagrun-v1"}
+				'hopper':"RoboschoolHopper-v1"}
 
-	env = gym.make(models['cheetah'])
+	env = gym.make(models['hopper'])
 
 	# Create Agent model
 	agent = ddpgAgent(env, batch_size=500, w_per=False)
@@ -65,15 +62,19 @@ def model_train(pretrained_):
 	print("======================================")
 
 
+	logger = dict()
+	plt.ion()
+
 	try:
 		act_range = env.action_space.high
+		rewards = []
 		for epi in range(NUM_EPISODES_):
 			print("=========EPISODE # %d =========="%epi)
 			obs = env.reset()
-			actions, states, rewards, dones, new_states = [],[],[],[],[]
 
 			epi_reward = 0
 			for t in tqdm(range(steps)):
+				plt.pause(0.01)
 				# environment rendering on Graphics
 				env.render()
 
@@ -92,14 +93,15 @@ def model_train(pretrained_):
 				epi_reward = epi_reward + reward
 
 
-				if t%100 == 0: agent.replay(1)
+				if t%50 == 0: agent.replay(1)
 
 				# check if the episode is finished
 				if done or (t == steps-1):
 					# Replay
 					agent.replay(1)
 					print("Episode#%d, steps:%d, rewards:%f"%(epi,t,epi_reward))
-					if epi%30 == 0:
+					rewards.append(epi_reward)
+					if epi%50 == 0:
 						dir_path = "%s/weights"%os.getcwd()
 						if not os.path.isdir(dir_path):
 							os.mkdir(dir_path)