-
Notifications
You must be signed in to change notification settings - Fork 2
/
arcore_tracker.py
134 lines (117 loc) · 3.75 KB
/
arcore_tracker.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
#!/usr/bin/env python3.7
"""ARCore supported devices tracker"""
import difflib
from datetime import date
from os import environ, rename, path, system
from time import sleep
import pandas as pd
from bs4 import BeautifulSoup
from requests import get, post
TODAY = str(date.today())
TG_CHAT = "@ARCoreDevices"
BOT_TOKEN = environ["BOTTOKEN"]
GIT_OAUTH_TOKEN = environ["GIT_OAUTH_TOKEN_XFU"]
CHANGES = []
def scrap_to_csv(url):
"""
fetches Google's page and scrape device's table to csv
"""
response = get(url)
page = BeautifulSoup(response.content, "html.parser")
tables = page.select("#google_play_devices ~ table")
data = pd.read_html(str(tables), index_col=0)
open("devices.csv", "w").close()
for table in data:
[
table.drop(columns=column, axis=1, inplace=True)
for column in table.columns
if "Notes" in column or "Unnamed" in column
]
with open("devices.csv", "a") as out:
table.to_csv(out, sep=";", header=None)
def diff_files():
"""
compare csv files to check for newly added devices
"""
with open("old.csv", "r") as old, open("devices.csv", "r") as new:
diff = difflib.unified_diff(
old.readlines(), new.readlines(), fromfile="old", tofile="new"
)
for line in diff:
if line.startswith("+"):
CHANGES.append(str(line))
def csv_to_md():
"""
convert csv file to markdown
"""
with open("devices.csv", "r") as csv_file:
csv = csv_file.readlines()
with open("README.md", "w", encoding="UTF-8") as out:
out.write("# Google ARCore Supported devices\n")
out.write(
"Last sync is {}\n\nhttps://developers.google.com/ar/"
"discover/supported-devices\n\n".format(TODAY)
)
out.write("|Brand|Device|\n")
out.write("|---|---|\n")
for line in csv:
info = line.strip().split(";")
try:
brand = info[0]
name = info[1]
out.write(f"|{brand}|{name}|\n")
except IndexError:
pass
def post_to_tg():
"""
post new devices to telegram channel
"""
for item in CHANGES[1:]:
info = item.split(";")
brand = info[0].replace("+", "")
name = info[1].strip()
telegram_message = (
f"New ARCore device added!:\nBrand: *{brand}*\nName: *{name}*"
)
params = (
("chat_id", TG_CHAT),
("text", telegram_message),
("parse_mode", "Markdown"),
("disable_web_page_preview", "yes"),
)
telegram_url = "https://api.telegram.org/bot" + BOT_TOKEN + "/sendMessage"
telegram_req = post(telegram_url, params=params)
telegram_status = telegram_req.status_code
if telegram_status == 200:
print("{0}: Telegram Message sent".format(name))
else:
print("Telegram Error")
sleep(2)
def git_commit_push():
"""
git add - git commit - git push
"""
system(
'git add README.md devices.csv && git -c "user.name=XiaomiFirmwareUpdater" '
'-c "[email protected]" commit -m "[skip ci] sync: {0}" && '
" \
"
"git push -q https://{1}@github.com/androidtrackers/arcore-devices.git HEAD:master".format(
TODAY, GIT_OAUTH_TOKEN
)
)
def main():
"""
ARCore supported devices tracker and scraper
"""
if path.exists("devices.csv"):
rename("devices.csv", "old.csv")
url = "https://developers.google.com/ar/devices"
scrap_to_csv(url)
diff_files()
if CHANGES:
post_to_tg()
csv_to_md()
git_commit_push()
if __name__ == "__main__":
main()