wip headsize32 training

RWKV · Aug 26, 2023 · 3c03fe0 · 3c03fe0
1 parent 704b382
commit 3c03fe0
Show file tree

Hide file tree

Showing 4 changed files with 4 additions and 4 deletions.
diff --git a/notebook/experiment/rwkv-x-exp/v5-headsize32/v5-L6-D2048-E1e-1-ctx4k-part1.ipynb b/notebook/experiment/rwkv-x-exp/v5-headsize32/v5-L6-D2048-E1e-1-ctx4k-part1.ipynb
@@ -5,7 +5,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "# RWKV v5-headsize2x / embedding init-range 1e-01 / 4k\n",
+    "# RWKV v5-headsize32 / embedding init-range 1e-01 / 4k\n",
     "\n",
     "- 6 layers\n",
     "- 2048 embedding size\n",

diff --git a/notebook/experiment/rwkv-x-exp/v5-headsize32/v5-L6-D2048-E1e-1-ctx4k-part2.ipynb b/notebook/experiment/rwkv-x-exp/v5-headsize32/v5-L6-D2048-E1e-1-ctx4k-part2.ipynb
@@ -5,7 +5,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "# RWKV v5-headsize2x / embedding init-range 1e-01 / 4k\n",
+    "# RWKV v5-headsize32 / embedding init-range 1e-01 / 4k\n",
     "\n",
     "- 6 layers\n",
     "- 2048 embedding size\n",

diff --git a/notebook/experiment/rwkv-x-exp/v5-headsize32/v5-L6-D4096-E1e-1-ctx4k-part1.ipynb b/notebook/experiment/rwkv-x-exp/v5-headsize32/v5-L6-D4096-E1e-1-ctx4k-part1.ipynb
@@ -5,7 +5,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "# RWKV v5-headsize2x / embedding init-range 1e-01 / 4k\n",
+    "# RWKV v5-headsize32 / embedding init-range 1e-01 / 4k\n",
     "\n",
     "- 6 layers\n",
     "- 4096 embedding size\n",

diff --git a/notebook/experiment/rwkv-x-exp/v5-headsize32/v5-L6-D4096-E1e-1-ctx4k-part2.ipynb b/notebook/experiment/rwkv-x-exp/v5-headsize32/v5-L6-D4096-E1e-1-ctx4k-part2.ipynb
@@ -5,7 +5,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "# RWKV v5-headsize2x / embedding init-range 1e-01 / 4k\n",
+    "# RWKV v5-headsize32 / embedding init-range 1e-01 / 4k\n",
     "\n",
     "- 6 layers\n",
     "- 4096 embedding size\n",