atcoder-scraper.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "b40552f6",
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium import webdriver\n",
    "from bs4 import BeautifulSoup\n",
    "import datetime\n",
    "import time\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7df51806",
   "metadata": {},
   "outputs": [],
   "source": [
    "# use Firefox browser\n",
    "driver = webdriver.Firefox()\n",
    "driver.get(\"https://atcoder.jp/contests/\")\n",
    "\n",
    "# wait for the page to load\n",
    "time.sleep(15)\n",
    "\n",
    "# get the page source\n",
    "html = driver.page_source\n",
    "\n",
    "# parse the page source using BeautifulSoup\n",
    "soup = BeautifulSoup(html, 'lxml')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7b3aa367",
   "metadata": {},
   "outputs": [],
   "source": [
    "# get list of upcoming contests\n",
    "upcoming_contests = soup.find(\"div\", \"col-lg-9 col-md-8\").find(\"div\", id=\"contest-table-upcoming\").div.div.tbody.find_all(\"tr\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "25a59f55",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tr>\n",
       " <td class=\"text-center\"><a href=\"http://www.timeanddate.com/worldclock/fixedtime.html?iso=20230218T1200&amp;p1=248\" target=\"blank\"><time class=\"fixtime-full\">2023-02-18(Sat) 08:30</time></a></td>\n",
       " <td>\n",
       " <span aria-hidden=\"true\" data-original-title=\"Heuristic\" data-placement=\"top\" data-toggle=\"tooltip\" title=\"\">Ⓗ</span>\n",
       " <span class=\"\">◉</span>\n",
       " <a href=\"/contests/ahc018\">RECRUIT Nihonbashi Half Marathon 2023 Winter（AtCoder Heuristic Contest 018）</a>\n",
       " </td>\n",
       " <td class=\"text-center\">199:00</td>\n",
       " <td class=\"text-center\">All</td>\n",
       " </tr>,\n",
       " <tr>\n",
       " <td class=\"text-center\"><a href=\"http://www.timeanddate.com/worldclock/fixedtime.html?iso=20230218T2100&amp;p1=248\" target=\"blank\"><time class=\"fixtime-full\">2023-02-18(Sat) 17:30</time></a></td>\n",
       " <td>\n",
       " <span aria-hidden=\"true\" data-original-title=\"Algorithm\" data-placement=\"top\" data-toggle=\"tooltip\" title=\"\">Ⓐ</span>\n",
       " <span class=\"user-orange\">◉</span>\n",
       " <a href=\"/contests/arc156\">AtCoder Regular Contest 156</a>\n",
       " </td>\n",
       " <td class=\"text-center\">02:00</td>\n",
       " <td class=\"text-center\"> - 2799</td>\n",
       " </tr>,\n",
       " <tr>\n",
       " <td class=\"text-center\"><a href=\"http://www.timeanddate.com/worldclock/fixedtime.html?iso=20230219T2100&amp;p1=248\" target=\"blank\"><time class=\"fixtime-full\">2023-02-19(Sun) 17:30</time></a></td>\n",
       " <td>\n",
       " <span aria-hidden=\"true\" data-original-title=\"Algorithm\" data-placement=\"top\" data-toggle=\"tooltip\" title=\"\">Ⓐ</span>\n",
       " <span class=\"user-blue\">◉</span>\n",
       " <a href=\"/contests/abc290\">Toyota Programming Contest 2023 Spring Qual B（AtCoder Beginner Contest 290）</a>\n",
       " </td>\n",
       " <td class=\"text-center\">01:40</td>\n",
       " <td class=\"text-center\"> - 1999</td>\n",
       " </tr>,\n",
       " <tr>\n",
       " <td class=\"text-center\"><a href=\"http://www.timeanddate.com/worldclock/fixedtime.html?iso=20230225T2100&amp;p1=248\" target=\"blank\"><time class=\"fixtime-full\">2023-02-25(Sat) 17:30</time></a></td>\n",
       " <td>\n",
       " <span aria-hidden=\"true\" data-original-title=\"Algorithm\" data-placement=\"top\" data-toggle=\"tooltip\" title=\"\">Ⓐ</span>\n",
       " <span class=\"user-orange\">◉</span>\n",
       " <a href=\"/contests/arc157\">AtCoder Regular Contest 157</a>\n",
       " </td>\n",
       " <td class=\"text-center\">02:00</td>\n",
       " <td class=\"text-center\"> - 2799</td>\n",
       " </tr>,\n",
       " <tr>\n",
       " <td class=\"text-center\"><a href=\"http://www.timeanddate.com/worldclock/fixedtime.html?iso=20230226T2100&amp;p1=248\" target=\"blank\"><time class=\"fixtime-full\">2023-02-26(Sun) 17:30</time></a></td>\n",
       " <td>\n",
       " <span aria-hidden=\"true\" data-original-title=\"Algorithm\" data-placement=\"top\" data-toggle=\"tooltip\" title=\"\">Ⓐ</span>\n",
       " <span class=\"user-blue\">◉</span>\n",
       " <a href=\"/contests/abc291\">AtCoder Beginner Contest 291（Sponsored by TOYOTA SYSTEMS）</a>\n",
       " </td>\n",
       " <td class=\"text-center\">01:40</td>\n",
       " <td class=\"text-center\"> - 1999</td>\n",
       " </tr>]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "upcoming_contests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "8fb6e145",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_upcoming_contests():\n",
    "    contests_list = []\n",
    "\n",
    "    # get contest data\n",
    "    for contest in upcoming_contests:\n",
    "        col = contest.find_all(\"td\")\n",
    "        raw_name = col[1].get_text()\n",
    "        name_length = len(raw_name)\n",
    "        name = raw_name[5:].strip(' \\n')\n",
    "\n",
    "        baseURL = 'https://atcoder.jp'\n",
    "        slug = col[1].a.get(\"href\")\n",
    "        link = baseURL + slug\n",
    "\n",
    "        dt = col[0].get_text()\n",
    "        dt_fmt = \"%Y-%m-%d(%a) %H:%M\"\n",
    "        dt = datetime.datetime.strptime(dt, dt_fmt)\n",
    "        unix_time = time.mktime(dt.timetuple())\n",
    "        start_time = int(unix_time)\n",
    "\n",
    "        duration = col[2].get_text()\n",
    "\n",
    "        print(f'Name is of type {type(name)}')\n",
    "        print(f'link is of type {type(link)}')\n",
    "        print(f'start_time is of type {type(start_time)}')\n",
    "        print(f'Duration is of type {type(duration)}')\n",
    "        contest_json = {\n",
    "            'name': name,\n",
    "            'link': link,\n",
    "            'startTime': start_time,\n",
    "            'duration': duration\n",
    "        }\n",
    "\n",
    "        contests_list.append(contest_json)\n",
    "        \n",
    "    return contests_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "6343b41b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Name is of type <class 'str'>\n",
      "link is of type <class 'str'>\n",
      "start_time is of type <class 'int'>\n",
      "Duration is of type <class 'str'>\n",
      "Name is of type <class 'str'>\n",
      "link is of type <class 'str'>\n",
      "start_time is of type <class 'int'>\n",
      "Duration is of type <class 'str'>\n",
      "Name is of type <class 'str'>\n",
      "link is of type <class 'str'>\n",
      "start_time is of type <class 'int'>\n",
      "Duration is of type <class 'str'>\n",
      "Name is of type <class 'str'>\n",
      "link is of type <class 'str'>\n",
      "start_time is of type <class 'int'>\n",
      "Duration is of type <class 'str'>\n",
      "Name is of type <class 'str'>\n",
      "link is of type <class 'str'>\n",
      "start_time is of type <class 'int'>\n",
      "Duration is of type <class 'str'>\n"
     ]
    }
   ],
   "source": [
    "contests_list = get_upcoming_contests()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "9ff727a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('atcoder.json', 'w') as f:     \n",
    "    json.dump(contests_list, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "a754628e",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'w' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Input \u001b[0;32mIn [8]\u001b[0m, in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mopen\u001b[39m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mcontests.csv\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[43mw\u001b[49m) \u001b[38;5;28;01mas\u001b[39;00m f:\n\u001b[1;32m      2\u001b[0m     writer \u001b[38;5;241m=\u001b[39m csv\u001b[38;5;241m.\u001b[39mwriter(f)\n\u001b[1;32m      3\u001b[0m     writer\u001b[38;5;241m.\u001b[39mwriterow()\n",
      "\u001b[0;31mNameError\u001b[0m: name 'w' is not defined"
     ]
    }
   ],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}