diff --git a/notebook/experiment/rwkv-x-exp/v5-small-model/v5-L6-ctx4k-models-part1.ipynb b/notebook/experiment/rwkv-x-exp/v5-small-model/v5-L6-ctx4k-models-part1.ipynb index 20af73f4..bc57c24d 100644 --- a/notebook/experiment/rwkv-x-exp/v5-small-model/v5-L6-ctx4k-models-part1.ipynb +++ b/notebook/experiment/rwkv-x-exp/v5-small-model/v5-L6-ctx4k-models-part1.ipynb @@ -78,7 +78,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Small Model: L6-D64\n", + "## Small Model: L6-D128\n", "\n", "### Enwiki training" ] @@ -89,7 +89,7 @@ "metadata": {}, "outputs": [], "source": [ - "EMBED_DIM=64\n", + "EMBED_DIM=128\n", "WANDB_PREFIX=f\"[small-model] v5-L{LAYER_COUNT}-D{EMBED_DIM}-E{EMBED_SCALE}\"\n", "FILENAME_PREFIX=f\"v5-L{LAYER_COUNT}-D{EMBED_DIM}-E{EMBED_SCALE_LABEL}\"" ]