Sort this.added_tokens before creating regex (.toSorted is not av…

…ailable in Node.js < 20)

src/tokenizers.js

@@ -2538,6 +2538,8 @@ export class PreTrainedTokenizer extends Callable {
                 this.all_special_ids.push(token.id);
             }
         }
+        // Sort by length (desc) to avoid early partial matches
+        this.added_tokens.sort((a, b) => b.content.length - a.content.length)
 
         // Update additional_special_tokens
         this.additional_special_tokens = tokenizerConfig.additional_special_tokens ?? [];
@@ -2555,11 +2557,8 @@ export class PreTrainedTokenizer extends Callable {
             this.decoder.end_of_word_suffix = this.model.end_of_word_suffix;
         }
 
-
         this.added_tokens_regex = this.added_tokens.length > 0 ? new RegExp(
             this.added_tokens
-                // Sort by length (desc) to avoid early partial matches
-                .toSorted((a, b) => b.content.length - a.content.length)
                 .map(x => `${x.lstrip ? '\\s*' : ''}(${escapeRegExp(x.content)})${x.rstrip ? '\\s*' : ''}`)
                 .join('|')
         ) : null;