diff --git a/notebook/experiment/rwkv-x-exp/v5-small-model/v5-L6-ctx4k-models-part1.ipynb b/notebook/experiment/rwkv-x-exp/v5-small-model/v5-L6-ctx4k-models-part1.ipynb index e5f669c0..20af73f4 100644 --- a/notebook/experiment/rwkv-x-exp/v5-small-model/v5-L6-ctx4k-models-part1.ipynb +++ b/notebook/experiment/rwkv-x-exp/v5-small-model/v5-L6-ctx4k-models-part1.ipynb @@ -8,7 +8,7 @@ "# RWKV v5 Small model experiment\n", "\n", "- 6 layers\n", - "- 2048 embedding size\n", + "- 64-1024 embedding size\n", "\n", "A series of small model training expriments, to see \"how small can we go\" for v5\n", "\n",