-
Notifications
You must be signed in to change notification settings - Fork 2
/
ingest.py
86 lines (77 loc) · 2.47 KB
/
ingest.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
import sqlite3
import csv
import pickle
from subprocess import call
from os import listdir
from os.path import isfile, join
import pandas as pd
DBPATH = '/home/pcgeller/bigtooth/dbs'
DATAPATHS = ['dbs','logs']
PROJECT = '/home/pcgeller/bigtooth'
def fetchdbs(LOCAL, \
remote = '[email protected]', \
REMOTE = '/home/pi/bigtooth/', \
DATAPATHS = DATAPATHS):
for PATH in DATAPATHS:
call(['scp', '-r', remote + ':' + join(REMOTE,PATH), LOCAL ])
header = ['id','uuid','name','status','address','uap_lap',\
'vendor','appearance','company', 'company_type','lmp_version','manufacturer',\
'firmware','ibeacon_range','created_at','updated_at','last_seen']
dbs = [db for db in listdir(DBPATH) if isfile(join(DBPATH, db))]
def pkl(data, filename):
with open(filename, 'wb') as f:
pickle.dump(data,f,pickle.HIGHEST_PROTOCOL)
def unpkl(filename):
with open(filename, 'rb') as f:
data = pickle.load(f)
return(data)
def dbstocsv(path=DBPATH, header = False, mkfiles = False):
#DBPATH = './bigtooth/dbs'
dbs = [db for db in listdir(DBPATH) if isfile(join(DBPATH, db))]
completelist = []
for db in dbs:
print(db)
conn = sqlite3.connect(join(DBPATH, db))
c = conn.cursor()
c.execute('SELECT id, uuid, name, status, address, uap_lap, vendor,\
appearance, company, company_type, lmp_version, manufacturer, firmware,\
ibeacon_range, created_at, updated_at, last_seen from blue_hydra_devices;')
completelist.extend(c.fetchall())
print(len(completelist))
if mkfiles == True:
savepath = join(PROJECT, 'csv', db + '.csv')
with open(savepath, 'w') as f:
writer = csv.writer(f)
if header == True:
writer.writerow(['id','uuid','name','status','address','uap_lap',\
'vendor','appearance','company','company','company_type','lmp_version','manufacturer',\
'firmware','ibeacon_range','created_at','updated_at','last_seen'])
writer.writerows(c)
c.close()
conn.close()
return(completelist)
#### YAR HERE BE sqlite3 commands
'''
.headers on
.mode csv
.output data.csv
SELECT id,
uuid,
name,
status,
address,
uap_lap,
vendor,
appearance,
company,
company_type,
lmp_version,
manufacturer,
firmware,
ibeacon_range,
created_at,
updated_at,
last_seen from blue_hydra_devices;
'''
#df = pd.read_csv('blue_hydra.db.2016-10-12_H08M10.csv', names=
#header)