changes

RWKV · Aug 29, 2023 · b7b1d90 · b7b1d90
1 parent 42ddabc
commit b7b1d90
Showing 1 changed file with 0 additions and 74 deletions.
diff --git a/notebook/experiment/rwkv-x-exp/v5-memory/v5-L96-D1024-E1e-1-ctx4k-part2.ipynb b/notebook/experiment/rwkv-x-exp/v5-memory/v5-L96-D1024-E1e-1-ctx4k-part2.ipynb
@@ -104,80 +104,6 @@
     "    ls -alh ."
    ]
   },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Enwiki Stage 1 : Foundation 4k model training"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Lets preload the requried dataset \n",
-    "!cd \"{TRAINER_DIR}\" && \\\n",
-    "    python3 preload_datapath.py \"{NOTEBOOK_DIR}/config-enwiki-4k.yaml\""
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Start the foundation model training\n",
-    "!cd \"{TRAINER_DIR}\" && \\\n",
-    "    export RWKV_WAVENET_LAYERS=\"{RWKV_WAVENET_LAYERS}\" && \\\n",
-    "    export WANDB_MODE=\"{WANDB_MODE}\" && \\\n",
-    "    python lightning_trainer.py fit \\\n",
-    "        -c \"{NOTEBOOK_DIR}/config-enwiki-4k.yaml\" \\\n",
-    "        --trainer.logger.init_args.name=\"{WANDB_PREFIX} - Enwiki-4k Foundation (train-ctx=4k, {DEEPSPEED_STRAT})\" \\\n",
-    "        --trainer.strategy=\"{DEEPSPEED_STRAT}\" \\\n",
-    "        --trainer.devices=\"{GPU_DEVICES}\" \\\n",
-    "        --trainer.callbacks.init_args.dirpath=\"../checkpoint/{FILENAME_PREFIX}-enwiki-4k/\" \\\n",
-    "        --model.load_model=\"../model/{FILENAME_PREFIX}-neox-init.pth\" \\\n",
-    "        --model.ctx_len=4096 \\\n",
-    "        --model.bptt_learning_range=1"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Lets export the model from the checkpoint\n",
-    "!cd \"{TRAINER_DIR}\" && \\\n",
-    "    python export_checkpoint.py \"../checkpoint/{FILENAME_PREFIX}-enwiki-4k/last.ckpt\" \"../model/{FILENAME_PREFIX}-enwiki-4k.pth\" \"bf16\"\n",
-    "!cd \"{TRAINER_DIR}\" && ls -alh \"../model/{FILENAME_PREFIX}-enwiki-4k.pth\""
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# # Lets do a quick dragon prompt validation\n",
-    "!cd \"{INFERENCE_DIR}\" && \\\n",
-    "    export RWKV_WAVENET_LAYERS=\"{RWKV_WAVENET_LAYERS}\" && \\\n",
-    "    python3 dragon_test.py \"../model/{FILENAME_PREFIX}-enwiki-4k.pth\" \"cuda fp32\""
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Lets do a quick memory test\n",
-    "!export RWKV_WAVENET_LAYERS=\"{RWKV_WAVENET_LAYERS}\" && \\\n",
-    "        python3 ../memory_script/eval_v5_memory_guided.py \"{PROJECT_DIR}/model/{FILENAME_PREFIX}-enwiki-4k.pth\""
-   ]
-  },
   {
    "attachments": {},
    "cell_type": "markdown",