feat: upgrade to v3.0.3

lenML · Oct 21, 2024 · 41ad928 · 41ad928
1 parent b3435a8
commit 41ad928
Show file tree

Hide file tree

Showing 4 changed files with 7 additions and 6 deletions.
diff --git a/package.json b/package.json
@@ -10,7 +10,7 @@
     "preview": "vite preview"
   },
   "dependencies": {
-    "@lenml/tokenizers": "^3.0.1",
+    "@lenml/tokenizers": "^3.0.3",
     "@quik-fe/stand": "^1.1.2",
     "@types/draft-js": "^0.11.18",
     "@types/styled-components": "^5.1.34",

diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
diff --git a/src/TokenizersHub.ts b/src/TokenizersHub.ts
@@ -4,6 +4,7 @@ import EventEmitter from "eventemitter3";
 
 type PreTrainedTokenizer = tokenizers.PreTrainedTokenizer;
 
+// NOTE: 这里的 package version 并不是 tokenizers 的 version，而是打包版本的 version，并不一定和 `tokenizers` 版本一致
 const package_version = "3.0.1";
 export const packages = [
   "llama3_1",

diff --git a/src/components/TokenizerPanel/TokenizerConfigure.tsx b/src/components/TokenizerPanel/TokenizerConfigure.tsx
@@ -56,7 +56,7 @@ const TokenizerFromUrls = ({ onConfigChange }: Props) => {
             onChange={(e) => {
               setFormData((prev) => ({ ...prev, config_url: e.target.value }));
             }}
-            placeholder="https://huggingface.co/.../config.json"
+            placeholder="https://huggingface.co/.../tokenizer_config.json"
           />
         </label>
       </div>