-
Notifications
You must be signed in to change notification settings - Fork 0
/
emulation_functions.py
64 lines (56 loc) · 2.66 KB
/
emulation_functions.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
import json
import requests
import random
from sqlalchemy import text
import datetime
random.seed(100)
class EmulationFunctions:
def __init__(self, headers, connection, db_creds):
self.headers = headers
self.connection = connection
self.db_creds = db_creds
self.url = self.db_creds['URL']
self.streaming_url = db_creds['STREAMING_URL']
self.random_row = random.randint(0, 11000)
self.keys = {
"pin": ['index', 'unique_id', 'title', 'description', 'poster_name', 'follower_count', 'tag_list', 'is_image_or_video', 'image_src', 'downloaded', 'save_location', 'category'],
"geo": ['ind', 'timestamp', 'latitude', 'longitude', 'country'],
"user": ['ind', 'first_name', 'last_name', 'age', 'date_joined']
}
def get_json_data(self, table_name, data_source):
string = text(f"SELECT * FROM {data_source} LIMIT {self.random_row}, 1")
selected_row = self.connection.execute(string)
keys = self.keys[table_name]
url = self.url + table_name
for row in selected_row:
result = dict(row._mapping)
#Data should be send as pairs of column_name:value, with different columns separated by commas
value = {key: (result[key].strftime("%m/%d/%Y, %H:%M:%S") if isinstance(result[key], datetime.date) else result[key]) for key in keys }
data = json.dumps({
"records": [
{
"value": value
}
]
})
print(data)
response = requests.request("POST", url, headers=self.headers, data=data)
print(response.status_code)
def get_streaming_data(self, table_name, data_source):
string = text(f"SELECT * FROM {data_source} LIMIT {self.random_row}, 1")
selected_row = self.connection.execute(string)
keys = self.keys[table_name]
url = self.streaming_url + table_name + '/record'
for row in selected_row:
result = dict(row._mapping)
#Data should be send as pairs of column_name:value, with different columns separated by commas
value = {key: (result[key].strftime("%m/%d/%Y, %H:%M:%S") if isinstance(result[key], datetime.date) else result[key]) for key in keys }
data = json.dumps({
"StreamName": "streaming-124df56aef51-pin",
"Data":
value,
"PartitionKey": f"{table_name}-data"
})
print(data)
response = requests.request("PUT", url, headers=self.headers, data=data)
print(response.status_code)