Test against D128 instead of D64

RWKV · Aug 28, 2023 · df5fade · df5fade
1 parent f936878
commit df5fade
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/notebook/experiment/rwkv-x-exp/v5-small-model/v5-L6-ctx4k-models-part1.ipynb b/notebook/experiment/rwkv-x-exp/v5-small-model/v5-L6-ctx4k-models-part1.ipynb
@@ -78,7 +78,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "## Small Model: L6-D64\n",
+    "## Small Model: L6-D128\n",
     "\n",
     "### Enwiki training"
    ]
@@ -89,7 +89,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "EMBED_DIM=64\n",
+    "EMBED_DIM=128\n",
     "WANDB_PREFIX=f\"[small-model] v5-L{LAYER_COUNT}-D{EMBED_DIM}-E{EMBED_SCALE}\"\n",
     "FILENAME_PREFIX=f\"v5-L{LAYER_COUNT}-D{EMBED_DIM}-E{EMBED_SCALE_LABEL}\""
    ]