Data Collection Using API and Data Cleaning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Importing The Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup as bs\n",
    "import requests\n",
    "import json\n",
    "import pandas as pd\n",
    "import re"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Getting the game names [2000-2010]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "years  = list(range(2000,2011)) # 2000-2010\n",
    "game_list = [] # here will be store all the game names\n",
    "\n",
    "for year in years:\n",
    "    r = requests.get('https://en.wikipedia.org/wiki/'+str(year)+'_in_video_games')\n",
    "    soup = bs(r.content)\n",
    "    content = soup.prettify()\n",
    "    \n",
    "    # Get the specific table jan-dec game name(by italic tag)\n",
    "    table = soup.findAll(\"table\", {\"class\": \"wikitable sortable\"})\n",
    "    game_names1 = table[0].findAll(\"i\")     # findAll return whole thing as a one element list so I access that element by index\n",
    "    game_names2 = table[1].findAll(\"i\")\n",
    "    game_names1_text = [k.get_text() for k in game_names1]\n",
    "    game_names2_text = [k.get_text() for k in game_names2]\n",
    "    if(len(game_names1_text)>len(game_names2_text)):\n",
    "        game_names = game_names1_text\n",
    "    else:\n",
    "        game_names = game_names2_text\n",
    "    game_list.extend(game_names)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Getting the game names [2011-2021]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "years = list(range(2011,2022))\n",
    "\n",
    "## all the game names from 2011-2021 will be store as the same list (game_list) as the pervious game names were stored(2000-2010)\n",
    "for year in years:\n",
    "    r = requests.get('https://en.wikipedia.org/wiki/'+str(year)+'_in_video_games')\n",
    "    soup = bs(r.content)\n",
    "    content = soup.prettify()\n",
    "    tid = 'January\\\\.E2\\\\.80\\\\.93March' ## header id of that table\n",
    "    table_1 = soup.select_one(\"h3:has(span#{}) + table.wikitable\".format(tid)) \n",
    "    game_names_1 = table_1.select(\"i\")  ## getting all the italic tag of that table\n",
    "    game_list.extend([k.get_text() for k in game_names_1])\n",
    "    \n",
    "        \n",
    "    # table from april-june\n",
    "    tid = 'April\\\\.E2\\\\.80\\\\.93June' ## header id of that table\n",
    "    table_2 = soup.select_one(\"h3:has(span#{}) + table.wikitable\".format(tid)) \n",
    "    game_names_2 = table_2.select(\"i\")\n",
    "    game_list.extend([k.get_text() for k in game_names_2])  ## extending to the game_names list (we dont use append cause it will add the whole list as an object)\n",
    "\n",
    "    # table from july-september\n",
    "    \n",
    "    tid = 'July\\\\.E2\\\\.80\\\\.93September' ## header id of that table\n",
    "    table_3 = soup.select_one(\"h3:has(span#{}) + table.wikitable\".format(tid)) \n",
    "    game_names_3 = table_3.select(\"i\")\n",
    "    game_list.extend([k.get_text() for k in game_names_3])  ## extending to the game_names list\n",
    "\n",
    "    # table from october-december\n",
    "    \n",
    "    tid = 'October\\\\.E2\\\\.80\\\\.93December' ## header id of that table\n",
    "    table_4 = soup.select_one(\"h3:has(span#{}) + table.wikitable\".format(tid)) \n",
    "    game_names_4 = table_4.select(\"i\")\n",
    "    game_list.extend([k.get_text() for k in game_names_4])  ## extending to the game_names list"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Calling The api and getting the info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "api_url = \"https://api.rawg.io/api/games\"\n",
    "key = \"?key=e431b52cb8204c66b501a35ba93bd3ca\"\n",
    "api_game = \"https://api.rawg.io/api/games/\" # this is for getting information of an specific game\n",
    "\n",
    "# all the necessary key to get the info from the response\n",
    "list_of_keys = ['name_original', 'description','metacritic','rating','rating_top','ratings','parent_platforms','platforms','developers','genres','released','tags','publishers','esrb_rating','description_raw']\n",
    "js_list = [] # this list will store all the filter response\n",
    "error_name_list = [] # this list will sotre all the failed response game name\n",
    "\n",
    "for i in game_list:\n",
    "    try:\n",
    "        url = api_url+key+\"&search=\"+i   # searching the game\n",
    "        response = requests.get(url)\n",
    "        js_format = response.json()\n",
    "        slug = js_format.get('results')[0].get(\"slug\")  # getting the slug name so that we can get complete information of that game\n",
    "        url2 = api_game+slug+key    # requesting with slug name it will return information of that game\n",
    "        response_2 = requests.get(url2)\n",
    "        js_format2 = response_2.json()\n",
    "        js_lim = {key:js_format2.get(key) for key in list_of_keys}  # filltering the response so that we can only have the necessary info\n",
    "        js_list.append(js_lim)  # adding the dict to the js_list \n",
    "    except:\n",
    "        error_name_list.append(i) # adding any error game name to the list"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Creating DataFrame From The Information"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## we will create the dataframe from the js_list which contains dict (or information of the games)\n",
    "\n",
    "\n",
    "original_name = [each.get('name_original') for each in js_list]\n",
    "rating = [each.get('rating') for each in js_list]\n",
    "metacritic = [each.get('metacritic') for each in js_list]\n",
    "rating_top = [each.get('rating_top') for each in js_list]\n",
    "ratings = [each.get('ratings') for each in js_list]\n",
    "parent_platforms = [each.get('parent_platforms') for each in js_list]\n",
    "platforms = [each.get('platforms') for each in js_list]\n",
    "developers = [each.get('developers') for each in js_list]\n",
    "genres = [each.get('genres') for each in js_list]\n",
    "released = [each.get('released') for each in js_list]\n",
    "tags = [each.get('tags') for each in js_list]\n",
    "publishers = [each.get('publishers') for each in js_list]\n",
    "esrb_rating = [each.get('esrb_rating') for each in js_list]\n",
    "description = [each.get(\"description\") for each in js_list]\n",
    "\n",
    "\n",
    "df = pd.DataFrame({\"Name\":original_name,\n",
    "                      \"Rating\":rating,\n",
    "                      \"Metacritic\": metacritic,\n",
    "                      \"Rating_Top\":rating_top,\n",
    "                      \"Ratings\":ratings,\n",
    "                      \"Parent_Platforms\":parent_platforms,\n",
    "                      \"Platforms\":platforms,\n",
    "                      \"Developers\":developers,\n",
    "                      \"Publishers\":publishers,\n",
    "                      \"Genres\":genres,\n",
    "                      \"Released\":released,\n",
    "                      \"Tags\":tags,\n",
    "                      \"ESRB_Rating\":esrb_rating,\n",
    "                      \"Description\":description})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Cleaning The Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cleaning the description"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean(string):\n",
    "    # first convert it to string\n",
    "    string = str(string)\n",
    "    \n",
    "    # cleans all the unicode character\n",
    "    \n",
    "    string = string.encode('ascii','ignore').decode()  # character like QuarantineÛ_ will become Quarantine_\n",
    "    \n",
    "    \n",
    "    # remove mentions @abcd\n",
    "    \n",
    "    string = re.sub(\"@S+\",' ',string)\n",
    "    \n",
    "    # Remove URL's\n",
    "    \n",
    "    string = re.sub(\"https*\\S+\",\" \",string)\n",
    "    \n",
    "    # remove Hastags\n",
    "    \n",
    "    string = re.sub(\"#\\S+\",\" \",string)\n",
    "    \n",
    "    # remove ticks and next character\n",
    "    \n",
    "    string = re.sub(\"\\'\\w+\",'', string)\n",
    "    \n",
    "    # remove extra spaces\n",
    "    \n",
    "    string = re.sub(\"\\s{2,}\",\" \",string)\n",
    "    \n",
    "    # remove special characters\n",
    "    \n",
    "    html_char_pat = re.compile(r'(.+)(&#[0-9]+)(.+)',re.S)\n",
    "    string = html_char_pat.sub(r'\\1\\3',str(string))\n",
    "    \n",
    "    string = string.replace('<p>',' ').replace(\"</p>\",'').replace(\"<ul>\",\"\").replace(\"</ul>\",\"\").replace(\"<li>\",\"\").replace(\"</li>\",\"\").replace(\"<h3>\",\"\").replace('</h3>','').replace(\"\\n\",\"\").replace(\"<br/>\",\" \").replace(\"<hr>\",\"\").replace(\"\\xa0\",\" \").replace(\"<br />\",\"\").replace(\"<pre>\",\"\").replace(\"<code>\",\"\").replace('</pre>',\"\").replace('</code>',\"\")\n",
    "    \n",
    "    return string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"Clean_description\"] =  df[\"Description\"].apply(clean)\n",
    "df =  df.drop(columns=[\"Description\"])\n",
    "df.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dropping the duplicates name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.drop_duplicates(subset=[\"Name\"])\n",
    "df.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Saving The DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(\"Final_game_dataset.csv\",index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Saving All The Name as a JSON File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "value = {\"Name\":list(df[\"Name\"].values)}\n",
    "with open(\"Name.json\",\"w\") as f:\n",
    "    json.dump(value,f)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "venv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}