scrape_gemp_games.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Scrape GEMP\n",
    "* Implement proof-of-concept scraper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 0. Prerequisites\n",
    "from selenium.webdriver import ChromeOptions\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "import re\n",
    "import time\n",
    "import bs4 as bs\n",
    "import hidden\n",
    "# used to write completed games to csv file\n",
    "from csv import DictWriter\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class GempTable:\n",
    "    def __init__(self, table):\n",
    "        self.recorded = table['Recorded']\n",
    "        self.table_id = table['TableID']\n",
    "        self.format = table['Format']\n",
    "        self.tournament = table['Tournament']\n",
    "        self.status = table['Status']\n",
    "        self.ls_player = table['LS_Player']\n",
    "        self.ls_deck = table['LS_Deck']\n",
    "        self.ds_player = table['DS_Player']\n",
    "        self.ds_deck = table['DS_Deck']\n",
    "        self.gamelink = table['GameLink']\n",
    "        self.winner = table['Winner']\n",
    "        self.window_id = None\n",
    "        self.messages = []\n",
    "    \n",
    "\n",
    "    def record_gemp_table(self):\n",
    "        headers = ['Recorded',\n",
    "                         'TableID',\n",
    "                         'Format',\n",
    "                         'Tournament',\n",
    "                         'Status',\n",
    "                         'LS_Player',\n",
    "                         'LS_Deck',\n",
    "                         'DS_Player',\n",
    "                         'DS_Deck',\n",
    "                         'GameLink',\n",
    "                         'Winner']\n",
    "        if ((self.status == 'Finished') & (self.format == 'Open') & (self.recorded == 0)):\n",
    "#             if DEBUG: print(f'recording {self.tableID}')\n",
    "            with open('gemp_completed_games.csv', 'a', newline='') as f:\n",
    "                dictwriter_object = DictWriter(f, fieldnames=headers)\n",
    "                # Pass the data in the dictionary as an argument into the writerow() function\n",
    "                self.recorded = time.strftime('%Y-%m-%d %H:%M:%S%z (%Z)')\n",
    "                table = {'Recorded': self.recorded,\n",
    "                         'TableID': self.table_id,\n",
    "                         'Format':self.format,\n",
    "                         'Tournament':self.tournament,\n",
    "                         'Status':self.status,\n",
    "                         'LS_Player': self.ls_player,\n",
    "                         'LS_Deck':self.ls_deck,\n",
    "                         'DS_Player':self.ds_player,\n",
    "                         'DS_Deck':self.ds_deck,\n",
    "                         'GameLink':self.gamelink,\n",
    "                         'Winner':self.winner,\n",
    "                        }\n",
    "                dictwriter_object.writerow(table)\n",
    "                # Close the file object\n",
    "                f.close()\n",
    "            if DEBUG: print(f\"recorded {table['TableID']}\")\n",
    "    #     if DEBUG: print(f\"length of tables after recording games: {len(tables)}\")        \n",
    "        recorded = True\n",
    "        return recorded\n",
    "        \n",
    "    def record_gemp_table_messages(self):\n",
    "        pass\n",
    "    \n",
    "    \n",
    "    def append_gemp_table_messages(self, driver):\n",
    "        driver.switch_to_window(driver.window_handles[self.window_id])\n",
    "        # append messages from current gemp-table game window\n",
    "        # what is the object that contains these messages\n",
    "        # are these messages new? do we need to append all of them?\n",
    "        # how to best determine which messages are to be appended?\n",
    "        \n",
    "        \n",
    "    \n",
    "    def open_gemp_table_tab(self, driver):\n",
    "        driver.execute_script(f'''window.open({self.gamelink}, '{self.table_id}');''')\n",
    "        self.window_id = len(driver.window_handles)-1\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_players_column(players_column):\n",
    "    player_deck_dict = {}\n",
    "    re_expr = re.compile('\\s*(.*)\\s\\((?:DARK:\\s)(.*)\\)\\,\\s(.*)\\s\\((?:LIGHT:\\s)(.*)\\)')\n",
    "    dark_light = re_expr.findall(players_column.text)\n",
    "    if dark_light != []:\n",
    "        player_deck_dict['DS_Player'] = dark_light[0][0]\n",
    "        player_deck_dict['DS_Deck'] = dark_light[0][1]\n",
    "        player_deck_dict['LS_Player'] = dark_light[0][2]\n",
    "        player_deck_dict['LS_Deck'] = dark_light[0][3]\n",
    "    else:\n",
    "        re_expr = re.compile('\\s*(.*)\\s\\((?:LIGHT:\\s)(.*)\\)\\,\\s(.*)\\s\\((?:DARK:\\s)(.*)\\)')\n",
    "        light_dark = re_expr.findall(players_column.text)\n",
    "        player_deck_dict['LS_Player'] = light_dark[0][0]\n",
    "        player_deck_dict['LS_Deck'] = light_dark[0][1]\n",
    "        player_deck_dict['DS_Player'] = light_dark[0][2]\n",
    "        player_deck_dict['DS_Deck'] = light_dark[0][3]\n",
    "    return player_deck_dict\n",
    "        \n",
    "\n",
    "def compose_gemp_table(gemp_table_entry, is_new_gemp_table):\n",
    "    table_entry_values = gemp_table_entry.find_elements_by_tag_name('td')\n",
    "    table_id = gemp_table_entry.get_attribute(\"class\")\n",
    "    \n",
    "    table = {}\n",
    "    table['Recorded'] = 0\n",
    "    table['TableID'] = table_id\n",
    "    table['Format'] = table_entry_values[0].text\n",
    "    table['Tournament']= table_entry_values[1].text\n",
    "    table['Status']= table_entry_values[2].text\n",
    "    \n",
    "    is_open_format = (table['Format'] == 'Open')\n",
    "    is_playing = ((table['Status'] != 'Preparation') & (table['Status'] != 'Finished') & (table['Status'] != 'Cancelled') & (table['Status'] != ''))\n",
    "    is_preparing = (table['Status'] == 'Preparation')\n",
    "    is_cancelled = (table['Status'] == 'Cancelled')\n",
    "    is_finished = (table['Status'] == 'Finished')\n",
    "    is_relevant = ((not is_cancelled) & (not is_preparing))\n",
    "    \n",
    "    if ((is_open_format) & (is_relevant)):\n",
    "        if (is_new_gemp_table & is_playing):\n",
    "            if DEBUG: print(\"adding Playing Table: \"+table_id)\n",
    "            player_deck_dict = parse_players_column(table_entry_values[3])\n",
    "            table['DS_Player'] = player_deck_dict['DS_Player'] \n",
    "            table['DS_Deck'] =  player_deck_dict['DS_Deck'] \n",
    "            table['LS_Player'] = player_deck_dict['LS_Player'] \n",
    "            table['LS_Deck'] = player_deck_dict['LS_Deck'] \n",
    "            if len(gemp_table_entry.find_elements_by_tag_name('a')) > 0:\n",
    "                table['GameLink']= gemp_table_entry.find_elements_by_tag_name('a')[0].get_attribute(\"href\")\n",
    "            else:\n",
    "                table['GameLink']= 'NA'\n",
    "            table['Winner']= 'NA'\n",
    "        elif (is_finished):\n",
    "            if DEBUG: print(\"adding Finished Table: \"+table_id)\n",
    "            player_deck_dict = parse_players_column(table_entry_values[3])\n",
    "            table['DS_Player'] = player_deck_dict['DS_Player'] \n",
    "            table['DS_Deck'] =  player_deck_dict['DS_Deck'] \n",
    "            table['LS_Player'] = player_deck_dict['LS_Player'] \n",
    "            table['LS_Deck'] = player_deck_dict['LS_Deck']\n",
    "            if (is_new_gemp_table):\n",
    "                table['GameLink']= 'NA'\n",
    "                table['Winner']= 'NA'\n",
    "            # WILL NEED TO UPDATE GEMPTABLE of finished, but old games.\n",
    "            table['Winner']= table_entry_values[4].text\n",
    "        else:\n",
    "            if DEBUG: print(f\"adding Unknown Table: {table_id} with {table['Status']} status\")\n",
    "            table['DS_Player'] = 'unknown'\n",
    "            table['DS_Deck'] =  'unknown'\n",
    "            table['LS_Player'] = 'unknown'\n",
    "            table['LS_Deck'] = 'unknown'\n",
    "            table['GameLink']= 'unknown'\n",
    "            table['Winner']= 'unknown'\n",
    "            \n",
    "        return GempTable(table)\n",
    "    else:\n",
    "        return False\n",
    "        \n",
    "\n",
    "\n",
    "def get_gemp_table_entries(driver, xpath):\n",
    "    table_body = driver.find_element_by_xpath(xpath)\n",
    "    entries = table_body.find_elements_by_tag_name('tr')    \n",
    "    return entries\n",
    "\n",
    "\n",
    "def get_gemp_tables(driver, gemp_tables):\n",
    "    # <The Dudley Boyz: GET THE TABLE(S)! />\n",
    "    keys = ['Recorded','TableID','Format','Tournament','Status','LS_Player', 'LS_Deck','DS_Player','DS_Deck','GameLink','Winner']\n",
    "    playing_tables = {}\n",
    "    finished_tables = {}\n",
    "\n",
    "    playing_tables['xpath'] = '/html/body/div[1]/div[1]/div[3]/div[1]/div[4]/table/tbody'\n",
    "    finished_tables['xpath'] = '//*[@id=\"hall\"]/div[1]/div[6]/table/tbody'\n",
    "    \n",
    "    playing_tables['entries'] = get_gemp_table_entries(driver, playing_tables['xpath'])\n",
    "    finished_tables['entries'] = get_gemp_table_entries(driver, finished_tables['xpath'])\n",
    "        \n",
    "    ## Update/Add rows from Playing Tables\n",
    "    for i in range(1, len(playing_tables['entries'])):\n",
    "        table_id = playing_tables['entries'][i].get_attribute(\"class\")\n",
    "        is_not_recorded = (table_id not in gemp_tables['recorded'])\n",
    "        is_new_gemp_table = (table_id not in gemp_tables.keys())\n",
    "        if ((is_new_gemp_table) & (is_not_recorded)):    \n",
    "            relevant_table = compose_gemp_table(playing_tables['entries'][i], is_new_gemp_table)\n",
    "            if ((is_new_gemp_table) & (relevant_table is not False)):\n",
    "                gemp_tables[table_id] = relevant_table\n",
    "                gemp_tables['active'].append(table_id)\n",
    "    \n",
    "    # check length of tables dictionary\n",
    "    if DEBUG: print(f'length of gemp_tables after Playing Tables: {len(gemp_tables)}')\n",
    "    \n",
    "    ## Updates tables with rows from Finished Tables\n",
    "    for i in range(1, len(finished_tables['entries'])):\n",
    "        table_id = finished_tables['entries'][i].get_attribute(\"class\")\n",
    "        is_not_recorded = (table_id not in gemp_tables['recorded'])\n",
    "        is_new_gemp_table = (table_id not in gemp_tables.keys())\n",
    "        \n",
    "        if (is_not_recorded):\n",
    "            relevant_table = compose_gemp_table(finished_tables['entries'][i], is_new_gemp_table)\n",
    "            if ((is_new_gemp_table) & (relevant_table is not False)): \n",
    "                gemp_tables[table_id] = relevant_table\n",
    "                gemp_tables['finished'].append(table_id)\n",
    "            elif (relevant_table is not False):\n",
    "                gemp_tables['finished'].append(table_id)\n",
    "                gemp_tables['active'].pop(table_id)\n",
    "               \n",
    "    # check length of tables dictionary\n",
    "    if DEBUG: print(f'length of tables after Finished Tables: {len(gemp_tables)}')\n",
    "    \n",
    "    return gemp_tables\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def login_gemp(driver):\n",
    "    driver.get (secrets[\"login\"])\n",
    "    driver.find_element_by_id(\"login\").send_keys(secrets[\"user\"])\n",
    "    driver.find_element_by_id(\"password\").send_keys(secrets[\"pass\"])\n",
    "    driver.find_element_by_id(\"password\").send_keys(Keys.RETURN)\n",
    "    return driver\n",
    "\n",
    "\n",
    "\n",
    "def listen_for_gemp_tables(driver):\n",
    "    listening = True\n",
    "    gemp_tables = {'active':[], 'finished':[], 'recorded':[]}\n",
    "    sleep_time = 120 # length of time between cycles\n",
    "    gamehall_window = driver.window_handles[0]\n",
    "    \n",
    "    while(listening):\n",
    "        gemp_tables['finished'] = []\n",
    "        gemp_tables = get_gemp_tables(driver, gemp_tables)\n",
    "        \n",
    "        if (len(gemp_tables['finished']) > 0):\n",
    "            for table_id in gemp_tables['finished']:\n",
    "                if table_id not in gemp_tables['recorded']:\n",
    "                    gemp_tables[table_id].record_gemp_table()\n",
    "                    gemp_tables['recorded'].append(table_id)\n",
    "                    gemp_tables.pop(table_id)\n",
    "        if (len(gemp_tables['active'] > 0)):\n",
    "            for table_id in gemp_tables['active']:\n",
    "                if (gemp_tables[table_id].window_id is not None):\n",
    "                    # switch to the window and check the messages\n",
    "                    gemp_table[table_id].append_gemp_table_messages(driver)\n",
    "                else:\n",
    "                    # else open a new window and append messages\n",
    "                    gemp_table[table_id].open_gemp_table(driver)\n",
    "                    gemp_table[table_id].append_gemp_table_messages(driver)\n",
    "                \n",
    "                \n",
    "        \n",
    "        \n",
    "        if listening:\n",
    "            driver.switch_to_window(gamehall_window)\n",
    "            if DEBUG: print(f\"sleeping for {sleep_time} seconds...\")\n",
    "            time.sleep(sleep_time)\n",
    "    return 0\n",
    "\n",
    "\n",
    "def access_gemp(site_URL):\n",
    "    chrome_options = webdriver.ChromeOptions()\n",
    "#     chrome_options.add_argument('--headless')\n",
    "    chrome_options.add_argument('--no-sandbox')\n",
    "    chrome_options.add_argument('--disable-dev-shm-usage')\n",
    "    driver = webdriver.Chrome(options=chrome_options)\n",
    "    driver = login_gemp(driver)\n",
    "    driver.get(site_URL)\n",
    "    try: \n",
    "        driver.find_element_by_xpath('/html/body/div[1]/div[1]/div[3]/div[1]/div[5]/div').click()\n",
    "    except:\n",
    "        pass\n",
    "    time.sleep(2)\n",
    "    listen_for_gemp_tables(driver)\n",
    "    \n",
    "    source = driver.page_source\n",
    "#     driver.quit()\n",
    "    return source\n",
    "  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Debug is ON\n",
      "adding Unknown Table: table12575 with  status\n",
      "adding Playing Table: table12587\n",
      "adding Playing Table: table12599\n",
      "adding Playing Table: table12598\n",
      "adding Playing Table: table12601\n",
      "adding Playing Table: table12602\n",
      "length of gemp_tables after Playing Tables: 9\n",
      "adding Finished Table: table12597\n",
      "adding Finished Table: table12573\n",
      "length of tables after Finished Tables: 11\n",
      "recorded table12597\n",
      "recorded table12573\n",
      "Parent window title: Game of Gemp-Swccg\n",
      "sleeping for 120 seconds...\n",
      "adding Playing Table: table12603\n",
      "length of gemp_tables after Playing Tables: 10\n",
      "length of tables after Finished Tables: 10\n",
      "Parent window title: Game of Gemp-Swccg\n",
      "sleeping for 120 seconds...\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-5-18f8af4b5d09>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mDEBUG\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf'Debug is ON'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0msite_URL\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msecrets\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"host\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m \u001b[0msource\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0maccess_gemp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msite_URL\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-4-c701185f776d>\u001b[0m in \u001b[0;36maccess_gemp\u001b[0;34m(site_URL)\u001b[0m\n\u001b[1;32m     66\u001b[0m         \u001b[0;32mpass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     67\u001b[0m     \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msleep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 68\u001b[0;31m     \u001b[0mlisten_for_gemp_tables\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdriver\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     69\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     70\u001b[0m     \u001b[0msource\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdriver\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpage_source\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-4-c701185f776d>\u001b[0m in \u001b[0;36mlisten_for_gemp_tables\u001b[0;34m(driver)\u001b[0m\n\u001b[1;32m     49\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlistening\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     50\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mDEBUG\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"sleeping for {sleep_time} seconds...\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 51\u001b[0;31m             \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msleep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msleep_time\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     52\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     53\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# get gemp data\n",
    "DEBUG = True\n",
    "DEV = False\n",
    "if DEV:\n",
    "    secrets = hidden.dev_secrets()\n",
    "else:\n",
    "    secrets = hidden.secrets()\n",
    "if DEBUG: print(f'Debug is ON')\n",
    "site_URL = secrets[\"host\"]\n",
    "source = access_gemp(site_URL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}