Use correct Llama 3.1 and 3.2 context lengths (#1779)

Lightning-AI · Oct 7, 2024 · f0a6e9d · f0a6e9d
1 parent 3f4992a
commit f0a6e9d
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/litgpt/config.py b/litgpt/config.py
@@ -939,7 +939,7 @@ def norm_class(self) -> Type:
     dict(
         name="Llama-3.1-405B{}",
         hf_config=dict(org="meta-llama", name="Meta-Llama-3.1-405B{}"),
-        block_size=131072,
+        block_size=8192,
         vocab_size=128000,
         padded_vocab_size=128256,
         n_layer=126,