-
Notifications
You must be signed in to change notification settings - Fork 0
/
one_hot.py
113 lines (103 loc) · 4.4 KB
/
one_hot.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
import string
import sys
#import warnings
from hashlib import md5
import numpy as np
if sys.version_info < (3,):
maketrans = string.maketrans
else:
maketrans = str.maketrans
def text_to_word_sequence(text,
filters='!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n',
lower=True, split=" "):
"""Converts a text to a sequence of words (or tokens).
# Arguments
text: Input text (string).
filters: list (or concatenation) of characters to filter out, such as
punctuation. Default: `!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n`,
includes basic punctuation, tabs, and newlines.
lower: boolean. Whether to convert the input to lowercase.
split: str. Separator for word splitting.
# Returns
A list of words (or tokens).
"""
if lower:
text = text.lower()
if sys.version_info < (3,):
if isinstance(text, unicode):
translate_map = dict((ord(c), unicode(split)) for c in filters)
text = text.translate(translate_map)
elif len(split) == 1:
translate_map = maketrans(filters, split * len(filters))
text = text.translate(translate_map)
else:
for c in filters:
text = text.replace(c, split)
else:
translate_dict = dict((c, split) for c in filters)
translate_map = maketrans(translate_dict)
text = text.translate(translate_map)
seq = text.split(split)
return [i for i in seq if i]
def one_hot(text, n,
filters='!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n',
lower=True,
split=' '):
"""One-hot encodes a text into a list of word indexes of size n.
This is a wrapper to the `hashing_trick` function using `hash` as the
hashing function; unicity of word to index mapping non-guaranteed.
# Arguments
text: Input text (string).
n: int. Size of vocabulary.
filters: list (or concatenation) of characters to filter out, such as
punctuation. Default: `!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n`,
includes basic punctuation, tabs, and newlines.
lower: boolean. Whether to set the text to lowercase.
split: str. Separator for word splitting.
# Returns
List of integers in [1, n]. Each integer encodes a word
(unicity non-guaranteed).
"""
return hashing_trick(text, n,
hash_function=hash,
filters=filters,
lower=lower,
split=split)
def hashing_trick(text, n,
hash_function=None,
filters='!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n',
lower=True,
split=' '):
"""Converts a text to a sequence of indexes in a fixed-size hashing space.
# Arguments
text: Input text (string).
n: Dimension of the hashing space.
hash_function: defaults to python `hash` function, can be 'md5' or
any function that takes in input a string and returns a int.
Note that 'hash' is not a stable hashing function, so
it is not consistent across different runs, while 'md5'
is a stable hashing function.
filters: list (or concatenation) of characters to filter out, such as
punctuation. Default: `!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n`,
includes basic punctuation, tabs, and newlines.
lower: boolean. Whether to set the text to lowercase.
split: str. Separator for word splitting.
# Returns
A list of integer word indices (unicity non-guaranteed).
`0` is a reserved index that won't be assigned to any word.
Two or more words may be assigned to the same index, due to possible
collisions by the hashing function.
The [probability](
https://en.wikipedia.org/wiki/Birthday_problem#Probability_table)
of a collision is in relation to the dimension of the hashing space and
the number of distinct objects.
"""
if hash_function is None:
hash_function = hash
elif hash_function == 'md5':
hash_function = lambda w: int(md5(w.encode()).hexdigest(), 16)
seq = text_to_word_sequence(text,
filters=filters,
lower=lower,
split=split)
return [(hash_function(w) % (n - 1) + 1) for w in seq]