update ch8

johnjim0816 · Aug 20, 2023 · 1e62262 · 1e62262
1 parent 481a482
commit 1e62262
Show file tree

Hide file tree

Showing 10 changed files with 1,510 additions and 202 deletions.
diff --git a/docs/ch8/main.md b/docs/ch8/main.md
diff --git a/docs/figs/ch8/NoisyDQN_CartPole-v1_training_curve.png b/docs/figs/ch8/NoisyDQN_CartPole-v1_training_curve.png
diff --git a/docs/figs/ch8/PERDQN_CartPole-v1_training_curve.png b/docs/figs/ch8/PERDQN_CartPole-v1_training_curve.png
diff --git a/docs/figs/ch8/per_dqn_pseu.png b/docs/figs/ch8/per_dqn_pseu.png
diff --git a/notebooks/NoisyDQN.ipynb b/notebooks/NoisyDQN.ipynb
diff --git a/notebooks/PER_DQN.ipynb b/notebooks/PER_DQN.ipynb
diff --git a/pseudocodes/pseudo.tex b/pseudocodes/pseudo.tex
@@ -260,7 +260,7 @@ \section{PER-DQN算法}
 	\begin{algorithmic}[1]
 		% \REQUIRE $n \geq 0 \vee x \neq 0$ % 输入
 		% \ENSURE $y = x^n$ % 输出
-		\STATE 初始化策略网络参数$\theta$ % 初始化
+		\STATE 初始化当前网络参数$\theta$ % 初始化
 		\STATE 复制参数到目标网络$\hat{Q} \leftarrow Q$
 		\STATE 初始化经验回放$D$
 		\FOR {回合数 = $1,M$}

diff --git a/pseudocodes/pseudo_without_notes.pdf b/pseudocodes/pseudo_without_notes.pdf
diff --git a/pseudocodes/pseudo_without_notes.tex b/pseudocodes/pseudo_without_notes.tex
@@ -248,39 +248,36 @@ \section{DRQN算法}
 
 \section{PER-DQN算法}
 \begin{algorithm}[H] % [H]固定位置
-    \floatname{algorithm}{{PER-DQN算法}\footnotemark[1]}  
+    \floatname{algorithm}{{PER-DQN算法}}  
     \renewcommand{\thealgorithm}{} % 去掉算法标号
 	\caption{} 
     \renewcommand{\algorithmicrequire}{\textbf{输入:}}  
     \renewcommand{\algorithmicensure}{\textbf{输出:}} 
 	\begin{algorithmic}[1]
-		% \REQUIRE $n \geq 0 \vee x \neq 0$ % 输入
-		% \ENSURE $y = x^n$ % 输出
-		\STATE 初始化策略网络参数$\theta$ % 初始化
-		\STATE 复制参数到目标网络$\hat{Q} \leftarrow Q$
+		\STATE 初始化当前网络参数 $\theta$ 
+		\STATE 复制参数到目标网络$\hat{\theta} \leftarrow \theta$
 		\STATE 初始化经验回放$D$
-		\FOR {回合数 = $1,M$}
-			\STATE 重置环境，获得初始状态$s_t$
-			\FOR {时步 = $1,t$}
+		\FOR {回合数 $m = 1,2,\cdots,M$}
+			\STATE 重置环境，获得初始状态$s_0$
+			\FOR {时步 $t= 1,2,\cdots,T$}
+				\STATE {\bfseries 交互采样：}
 				\STATE 根据$\varepsilon-greedy$策略采样动作$a_t$
 				\STATE 环境根据$a_t$反馈奖励$r_t$和下一个状态$s_{t+1}$
-				\STATE 存储transition即$(s_t,a_t,r_t,s_{t+1})$到经验回放$D$，并根据TD-error损失确定其优先级$p_t$
+				\STATE 存储样本$(s_t,a_t,r_t,s_{t+1})$到经验回放$D$中，并根据$TD$误差损失确定其优先级$p_t$
 				\STATE 更新环境状态$s_{t+1} \leftarrow s_t$
-				\STATE {\bfseries 更新策略：}
-				\STATE 按照经验回放中的优先级别，每个样本采样概率为$P(j)=p_j^\alpha / \sum_i p_i^\alpha$，从$D$中采样一个大小为batch的transition
+				\STATE {\bfseries 模型更新：}
+				\STATE 根据每个样本的优先级计算采样概率$P(j)=p_j^\alpha / \sum_i p_i^\alpha$，从$D$中采样一个批量的样本
 				\STATE 计算各个样本重要性采样权重 $w_j=(N \cdot P(j))^{-\beta} / \max _i w_i$
-				\STATE 计算TD-error $\delta_j$ ; 并根据TD-error更新优先级$p_j$
-				\STATE 计算实际的$Q$值，即$y_{j}$\footnotemark[2]
-				\STATE 根据重要性采样权重调整损失 $L(\theta)=\left(y_{j}-Q\left(s_{j}, a_{j} ; \theta\right)\cdot w_j \right)^{2}$，并将其关于参数$\theta$做随机梯度下降\footnotemark[3]
+				\STATE 计算$TD$误差$\delta_j$ ; 并根据$TD$误差更新优先级$p_j$
+				\STATE 计算$Q$的估计值，即$y_{j}$
+				\STATE 根据重要性采样权重调整损失 $L(\theta)=\left(y_{j}-Q\left(s_{j}, a_{j} ; \theta\right)\cdot w_j \right)^{2}$，并关于$\theta$做随机梯度下降
+				\STATE 每$C$步复制参数$\hat{Q}\leftarrow Q$
 			\ENDFOR
-			\STATE 每$C$个回合复制参数$\hat{Q}\leftarrow Q$\footnotemark[4]
+
 		\ENDFOR
 	\end{algorithmic}
 \end{algorithm}
-\footnotetext[1]{Playing Atari with Deep Reinforcement Learning}
-\footnotetext[2]{$y_{i}= \begin{cases}r_{i} & \text {对于终止状态} s_{i+1} \\ r_{i}+\gamma \max _{a^{\prime}} Q\left(s_{i+1}, a^{\prime} ; \theta\right) & \text {对于非终止状态} s_{i+1}\end{cases}$}
-\footnotetext[3]{$\theta_i \leftarrow \theta_i - \lambda \nabla_{\theta_{i}} L_{i}\left(\theta_{i}\right)$}
-\footnotetext[4]{此处也可像原论文中放到小循环中改成每$C$步，但没有每$C$个回合稳定}
+
 \clearpage
 
 

diff --git a/pseudocodes/texput.log b/pseudocodes/texput.log
@@ -0,0 +1,21 @@
+This is XeTeX, Version 3.141592653-2.6-0.999994 (TeX Live 2022) (preloaded format=xelatex 2022.8.16)  20 AUG 2023 16:24
+entering extended mode
+ restricted \write18 enabled.
+ %&-line parsing enabled.
+**
+
+! Emergency stop.
+<*> 
+
+End of file on the terminal!
+
+
+Here is how much of TeX's memory you used:
+ 4 strings out of 476179
+ 35 string characters out of 5813072
+ 298507 words of memory out of 5000000
+ 20858 multiletter control sequences out of 15000+600000
+ 469259 words of font info for 28 fonts, out of 8000000 for 9000
+ 1348 hyphenation exceptions out of 8191
+ 0i,0n,0p,28b,6s stack positions out of 10000i,1000n,20000p,200000b,200000s
+No pages of output.