Skip to content

Commit

Permalink
revert debug code
Browse files Browse the repository at this point in the history
  • Loading branch information
PicoCreator committed Aug 21, 2023
1 parent 5b8320b commit b844952
Showing 1 changed file with 1 addition and 8 deletions.
9 changes: 1 addition & 8 deletions RWKV-v5/src/dataflow/trie_tokenizer.py
Original file line number Diff line number Diff line change
Expand Up @@ -130,14 +130,7 @@ def get_world_tokenizer():

# Provide a global function for the world tokenizer
def world_tokenizer_encode(src):
res = get_world_tokenizer().encode(src)

# Check the result for any null, < 0, or > 65525
for i in res:
if i < 0 or i > 65529:
raise Exception(f"world_tokenizer_encode: Invalid token {i} from: {src}")

return res
return get_world_tokenizer().encode(src)

########################################################################################################
# Tensor specific tokenizer
Expand Down

0 comments on commit b844952

Please sign in to comment.