stage_2+offload for L12

RWKV · Sep 29, 2023 · 049c755 · 049c755
1 parent e20ca29
commit 049c755
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/notebook/experiment/rwkv-x-exp/multi-size-train/v5-L12-D2048-part1.ipynb b/notebook/experiment/rwkv-x-exp/multi-size-train/v5-L12-D2048-part1.ipynb
@@ -36,7 +36,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "DEEPSPEED_STRAT=\"deepspeed_stage_1\"\n",
+    "DEEPSPEED_STRAT=\"deepspeed_stage_2_offload\"\n",
     "GPU_DEVICES=\"auto\"\n",
     "ENABLE_WANDB=True\n",
     "\n",