-
Notifications
You must be signed in to change notification settings - Fork 0
/
pdfparser.py
143 lines (121 loc) · 3.77 KB
/
pdfparser.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
# !/usr/bin/env python3
# -*- coding: utf-8 -*-
import copy
import csv
import tempfile
import subprocess
import PyPDF2
INFILE = '.berlinbvvkandidierende2016.pdf'
OUTFILE = '.berlinbvvkandidierende2016.csv'
def pdf_to_plain_text(filename):
with open(filename, 'rb') as pdffile:
pdfreader = PyPDF2.PdfFileReader(pdffile)
pdfwriter = PyPDF2.PdfFileWriter()
for page in pdfreader.pages:
column_left = copy.copy(page)
column_right = copy.copy(page)
right, top = page.mediaBox.upperRight
column_left.mediaBox.lowerLeft = (0, 0)
column_left.mediaBox.upperRight = (right/2, top)
column_right.mediaBox.lowerLeft = (right/2, 0)
column_right.mediaBox.upperRight = (right, top)
pdfwriter.addPage(column_left)
pdfwriter.addPage(column_right)
tmpfile = tempfile.SpooledTemporaryFile()
pdfwriter.write(tmpfile)
tmpfile.seek(0)
try:
pdftotext = subprocess.run(['pdftotext', '-layout', '-', '-'], stdin=tmpfile, stdout=subprocess.PIPE)
pdftotext = pdftotext.stdout
except AttributeError:
pdftotext = subprocess.check_output(['pdftotext', '-layout', '-', '-'], stdin=tmpfile)
tmpfile.close()
text = pdftotext
return text.decode()
def extract_liste(text):
_, text = text.split('(')
text, *_ = text.split(')')
normalize = {
'afd': 'AfD',
'cdu': 'CDU',
'fdp': 'FDP',
'grün': 'GRÜNE',
'link': 'LINKE',
'piraten': 'PIRATEN',
'spd': 'SPD',
'die partei': 'PARTEI'
}
for fragment, kuerzel in normalize.items():
if fragment in text.lower():
text = kuerzel
return text
def extract_name(text):
titel = ''
nachnamen = ''
vornamen = ''
if 'Dr.' in text:
text = text.replace('Dr.', '')
titel = 'Dr.'
try:
nachnamen, vornamen = text.split(', ')
nachnamen = nachnamen.split()[2:]
nachnamen = ' '.join(nachnamen)
except ValueError:
vornamen = 'Can'
nachnamen = 'Șepșul'
return titel, nachnamen, vornamen
def extract_person(lines):
titel = ''
nachnamen = ''
vornamen = ''
geboren = ''
plz = ''
for line in lines:
if line[:3] == 'Nam':
titel, nachnamen, vornamen = extract_name(line)
elif line[:3] == 'geb':
geboren = line.split()[1][:-1]
elif line[:3] == 'PLZ':
plz = line.split()[-1]
break
person = {
'TITEL': titel,
'ANZEIGENAME': vornamen + ' ' + nachnamen,
'NACHNAMEN': nachnamen,
'VORNAMEN': vornamen,
'JAHRGANG': geboren,
'PLZ': plz
}
return person
def main():
text = pdf_to_plain_text(INFILE)
lines = text.splitlines()
lines = [line.strip() for line in lines]
lines = [line for line in lines if line]
candidates = []
for index, line in enumerate(lines):
if line.startswith('Bezirk:'):
bezirk = line.split()[1]
elif line.startswith('Liste Nr'):
liste = extract_liste(line + ' ' + lines[index+1])
elif line.startswith('Name'):
person = extract_person(lines[index:index+6])
person['LISTE'] = liste
person['BEZIRK'] = bezirk
candidates.append(person)
with open(OUTFILE, 'w') as csvfile:
fieldnames = [
'TITEL',
'ANZEIGENAME',
'NACHNAMEN',
'VORNAMEN',
'JAHRGANG',
'PLZ',
'LISTE',
'BEZIRK'
]
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
writer.writerows(candidates)
if __name__ == '__main__':
main()