-
Notifications
You must be signed in to change notification settings - Fork 0
/
tokenise.py
179 lines (149 loc) · 6.3 KB
/
tokenise.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#-----------------------------------------------------------------------------------
# Marta Villegas (UAB) VIW project (Visual into Words http://pagines.uab.cat/viw)
#
# Fills the Tokens' Tier of an eaf file with the tokens from a conLL file (stanford/freeling ...).
# Checks that the eaf file and the conLL file have the same 'utterances'
# (eaf's Audio Description = sentences in Conll) and that the Tokens' Tier is empty.
#
# usage: $ tokenize.py elanFile.eaf conllFile.txt
#
# output is written in a new eaf file: elanFile-Tokens.eaf
# Note!!! look for "CHECK!!" and configure as you need
# if miss-alignments between eaf and freeling ## uncomment to chechk miss-alignments
#-----------------------------------------------------------------------------------
import sys
import pympi # Imports pympi to work with elan files
from shutil import copyfile
# Specify file path
args = sys.argv
elan_file_path = args[1]
freeling = args[2]
# Initialize the elan file
eaf = pympi.Elan.Eaf(elan_file_path)
### reads file by paragraph (sentences are one token per line, so sentences = paragraph)
def paragraphs(lines, is_separator=str.isspace, joiner=''.join):
paragraph = [ ]
for line in lines:
if is_separator(line):
if paragraph:
yield joiner(paragraph)
paragraph = [ ]
else:
paragraph.append(line)
if paragraph:
yield joiner(paragraph)
def getKey(item):
return item[1]
### reads Freeling file into tags
def parseFreeling(freeling):
# reads Freeling file into lines
with open(freeling) as f:
lines = f.read().splitlines(True)
# puts paragraphs in tags
tags = []
for p in paragraphs(lines):
p= p.decode('utf-8')
tags.append(p)
return tags
### Add annotations
def annotate(sentencesSorted):
for sentence in sentencesSorted:
#print sentence ## sentence from AD
refId = ""
# align sentences & tags
i = sentencesSorted.index(sentence)
line = tags[i] ## corresponding freeling annotated sentence (tabular mode)
#print line
# split lines by \n to get tokens and remove last empty
tokens = line.split('\n')
tokens.pop()
#print tokens ## freeling annotated sentence
# get id (refId) of referenced Annotation in AD-unit according to time in sentences
time = sentence[0]
for aid, (begin, end, _, _) in eaf.tiers['AD-unit'][0].items():
begin = eaf.timeslots[begin]
if begin == time:
#print('AI',aid)
refId = aid
break
# add token info to tokens annotation
for token in tokens:
# initialise empty ids list
ids = []
# split token into columns (form, lemma, posTag, ...)
columns = token.split('\t')
# get existing annotations in Tokens
ann = eaf.tiers['Tokens'][1]
#print('TOKEN',ann)
# if ann then get ids and make a sorted list (we'll need the highest one)
if ann:
# key value in ann = {'a10321': ('a1', '1', None, None)}
for k, v in ann.items():
if v[0] == refId:
id = k[1:]
id = int(id)
ids.append(id)
ids.sort(reverse=True)
##print('KEY0T',columns[1],len(ids))
# we don't need PREVIOUS_ANNOTATION=id for first annotation
if len(ids) > 0:
previous = 'a' + str(ids[0])
##print('KEY1T',time, columns[1],previous)
eaf.add_ref_annotation('Tokens','AD-unit',time, columns[0],previous) ### CHECK!! we need the 'form': stanford columns[1], freeling columns[0]
else:
##print('KEY3T',time, columns[1])
eaf.add_ref_annotation('Tokens','AD-unit',time, columns[0]) ### CHECK!! we need the 'form': stanford columns[1], freeling columns[0]
#eaf.add_ref_annotation('AD-Focus','AD-unit',time, columns[3])
#print("----------")
### Checks that the tabular file with annotations and the Elan file have the same number of utterances
def checkAlignment(sentencesSorted):
for sentence in sentencesSorted:
freeling = ""
# align sentences & tags
i = sentencesSorted.index(sentence)
line = tags[i]
# split lines by \n to get tokens and remove last empty
tokens = line.split('\n')
tokens.pop()
for token in tokens:
# split tonek into columns (form, lemma, posTag, ...)
columns = token.split('\t')
freeling = freeling + ' ' + columns[0] ### CHECK!! we need the 'form': stanford columns[1], freeling columns[0]
s = sentence[2]
s = s.replace(' ','')
freeling = freeling.replace(' ','')
freeling = freeling.replace('/','.')
freeling = freeling.replace('_','')
freeling = freeling.replace("'",'?')
## uncomment following lines to chechk miss-alignments
##if (freeling == s):
##print(i,'OK')
##else:
##print(i,s,freeling)
### Checks Tokens Tier is empty
def checkIfTokens():
ann = eaf.get_ref_annotation_data_for_tier('Tokens')
if len(ann) > 0:
print "File already has Tokens!!! I quit, use deleteAnnotations.py to delete them"
quit()
#--------------------------------------------------------------------------------------
if __name__ == '__main__':
### Get tabular file wit annotations (PoS) into tags
tags = parseFreeling(freeling)
print ('Num sentences in Freeling file: ' , len(tags))
### Get 'Sentences' from eaf file & sort them (they come unnordered!!)
sentences = eaf.get_annotation_data_for_tier('AD-unit')
print ('Num sentences in eaf file: ' , len(sentences))
sentencesSorted = sorted(sentences, key=getKey)
#print("Freeling",tags)
#print("Elan",sentencesSorted)
### check alignment between source AD Tier in the eaf file and the Freeling/Stanford tabular file
checkAlignment(sentencesSorted)
### checks if Tokens Tier is empty (if already filled in with tokens warms and quits)
checkIfTokens()
### Annotate sentences in the eaf file with tokens from the Freeling/Stanford tabular file
annotate(sentencesSorted)
### Write the results to file with the -Tokens suffix
eaf.to_file(elan_file_path.replace('.eaf', '-Tokens.eaf'))