F1D-05-AnExampleofWebCrawler.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# FLIP(01):  Advanced Data Science\n",
    "**(Module D: Web Crawler)**\n",
    "\n",
    "---\n",
    "- Materials in this module include resources collected from various open-source online repositories.\n",
    "- You are free to use, but NOT allowed to change or distribute this package.\n",
    "- If you found any issue/bug for this document, please submit an issue at [tulip lab/flip01](https://github.com/tulip-lab/flip01/issues)\n",
    "\n",
    "\n",
    "Prepared by and for \n",
    "**Student Members** |\n",
    "2006-2022 [TULIP Lab](http://www.tulip.org.au)\n",
    "\n",
    "---\n",
    "\n",
    "\n",
    "# Session 04A - An Example of Web Crawler"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Import related package"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from selenium import webdriver\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "from selenium.common.exceptions import TimeoutException\n",
    "from selenium.webdriver.common.by import By\n",
    "from pyquery import PyQuery as pq\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#choose browser\n",
    "browser = webdriver.Chrome()\n",
    "wait = WebDriverWait(browser, 10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "define a search function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def search(start_time):\n",
    "    try:\n",
    "        browser.get('http://m.dsec.gov.mo/Indicator.aspx?Type=K&ID=27&Format=REPORT')\n",
    "        submit = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, \"#MainContent_btnBack\")))\n",
    "        submit.click()\n",
    "        input = wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, \"#MainContent_txtStartYear\")))\n",
    "        input.clear()\n",
    "        input.send_keys(start_time)\n",
    "        #click1 = browser.find_element_by_id('MainContent_chkYearly')\n",
    "        #click1.click()\n",
    "\n",
    "        click1 = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, \"#MainContent_lblQuarterly > span\")))\n",
    "        click2 = wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, \"#MainContent_lblYearly > span\")))\n",
    "        click1.click()\n",
    "        click2.click()\n",
    "\n",
    "\n",
    "        submit1 = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, \"#MainContent_btnNext\")))\n",
    "        submit1.click()\n",
    "\n",
    "        get_data()\n",
    "    except TimeoutException:\n",
    "        return search(start_time)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "define a function to gain the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_data():\n",
    "    html = browser.page_source\n",
    "    # print(html)\n",
    "    doc = pq(html)\n",
    "    # print(doc)\n",
    "    print(\"==============\")\n",
    "    items = doc('#MainContent_pnlReport').items()\n",
    "    print(items)\n",
    "    for item in items:\n",
    "        data = {\n",
    "            'data':item.find('.ui-li').text()\n",
    "        }\n",
    "\n",
    "        data=data['data'].split(' ')\n",
    "        #.remove(data['data'].split(' ')[0])\n",
    "        # data = data.remove(data[0])\n",
    "        data.remove(data[0])\n",
    "        data.remove(data[0])\n",
    "        print(data)\n",
    "        num=[]\n",
    "        date=[]\n",
    "        for  i in data:\n",
    "            if i.find('\\n'):\n",
    "                lis=i.split('\\n')\n",
    "                num.append(lis[0])\n",
    "                date.append(lis[1])\n",
    "        print(num)\n",
    "        # print(num[0])\n",
    "        print(date)\n",
    "        print(type(date))\n",
    "        for i in range(len(date)):\n",
    "            list = date[i]\n",
    "            m = list[6:7]\n",
    "            if(m == '月'):\n",
    "                list1 = re.sub('年', '-' + '0', list)\n",
    "                list2 = re.sub('月', '-' + '0' + '1', list1)\n",
    "                # print(list2)\n",
    "                list3 = []\n",
    "                list3.append(list2)\n",
    "            else:\n",
    "                list1 = re.sub('年', '-', list)\n",
    "                list2 = re.sub('月', '-' + '0'+ '1', list1)\n",
    "                # print(list2)\n",
    "                list3 = []\n",
    "                list3.append(list2)\n",
    "            print(list3)\n",
    "            print(type(list3))\n",
    "      \n",
    "        true_value = pd.DataFrame(np.arange(len(date) * 1).reshape(((len(date)), 1)), index=date,\n",
    "                              columns=['number of people'])\n",
    "        true_value['number of people']=num;\n",
    "        print(true_value)\n",
    "        true_value.to_csv('Actual tourist.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "define a main funtion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def main ():\n",
    "    start_time = input(\"please input start time:\")\n",
    "    search(start_time)\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    main()"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}