step3.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ODI Queensland workshop - Web Scraping \n",
    "\n",
    "## QUT DMRC - 2015\n",
    "\n",
    "### Extract all road sign data from a single page\n",
    "\n",
    "Extend from getting just the sign name to getting all the fields we are interested in from each item."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import bs4\n",
    "import requests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# this is the base_url\n",
    "base_url = \"http://www.qld.gov.au/transport/safety/signs/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# select which page to scrape based on the type of road sign\n",
    "sign_type = \"regulatory\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# build the url\n",
    "thepage = base_url + sign_type + '/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# call the url\n",
    "stuff = requests.get(thepage)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# transform to soup using lxml parser\n",
    "soup = bs4.BeautifulSoup(stuff.text, \"lxml\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now process ```lotsofitems``` in a new way to get more fields for all of the items."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# in this step we'll scrape sign image url, sign description in addition to sign name\n",
    "\n",
    "# find all the tables on the page\n",
    "tables = soup.findAll('table')\n",
    "\n",
    "# move outside the loop so we just make one list\n",
    "thelist = []\n",
    "\n",
    "for table in tables:\n",
    "    # find all the table rows\n",
    "    lotsofitems = table.findAll('tr')\n",
    "    \n",
    "    # check if the first row contains a 'th' elements (table header)\n",
    "    if lotsofitems[0].find('th'): \n",
    "\n",
    "        # get all header elements\n",
    "        temp = lotsofitems[0].findAll('th')\n",
    "        \n",
    "        # check that the table header has the text we expect for the signs table\n",
    "        if temp[0].get_text() == 'Sign' and temp[1].get_text() == 'Meaning':\n",
    "            \n",
    "            print('Traffic sign table found')\n",
    "            \n",
    "            # process the table of traffic signs **** THIS IS THE UPDATED SECTION *****\n",
    "            for an_item in lotsofitems[1:]: \n",
    "                theitem = []\n",
    "\n",
    "                # sign description & title\n",
    "                sign_text = an_item.findAll(\"p\")\n",
    "                description = ''\n",
    "                for para in sign_text:\n",
    "                    if para.find(\"strong\"):\n",
    "                        # extract sign name (this assumes only the sign name is in bold)\n",
    "                        temptemp = para.find(\"strong\").get_text()\n",
    "                        temptemp = temptemp.split()\n",
    "                        sign_name = \" \".join(temptemp)\n",
    "                    else:\n",
    "                        # extract sign description (may be multiple paragraphs)\n",
    "                        temptemp = para.get_text()\n",
    "                        temptemp = temptemp.split()\n",
    "                        description += \" \".join(temptemp) + '\\n'\n",
    "\n",
    "                theitem += [sign_name]\n",
    "                theitem += [description]\n",
    "\n",
    "                # sign images (may be more than one image per sign name) - save image name & image url\n",
    "                images = []\n",
    "                for image in  an_item.findAll(\"img\"):\n",
    "                    # get the image name & image url\n",
    "                    images += [[image.attrs['alt'], image.attrs['src']]]\n",
    "                theitem += [images]\n",
    "    \n",
    "                thelist += [theitem]\n",
    "    \n",
    "        else:\n",
    "            print('Different table - with header row:', temp)\n",
    "    else:\n",
    "        print('Different table - no header row:', lotsofitems[0])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "thelist[3]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we are ready to move onto the fourth notebook - [Structure the data extraction as a function](step4.ipynb)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}