first_part_web_scraping.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "from parsel import Selector\n",
    "\n",
    "login = \"mandar.gite@gmail.com\"\n",
    "password = \"Cortex@5326\"\n",
    "\n",
    "driver = webdriver.Chrome()   # instanstiate driver class for webdriver\n",
    "driver.get('http://www.linkedin.com/');\n",
    "time.sleep(5) # Let the user actually see something!\n",
    "\n",
    "\n",
    "# locate login id element \n",
    "session_key = driver.find_element('name','session_key' )\n",
    "# send to webdriver the email id \n",
    "session_key.send_keys(login)\n",
    "\n",
    "# locate password element \n",
    "session_password =driver.find_element('name','session_password')\n",
    "# send password to webdriver \n",
    "session_password.send_keys(password)\n",
    "\n",
    "# locate submit button \n",
    "submit = driver.find_element(By.CLASS_NAME,'sign-in-form__submit-button')\n",
    "# click on submit button \n",
    "submit.click()\n",
    "\n",
    "time.sleep(3) # Let the user actually see something!\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.get('https:www.google.com')\n",
    "time.sleep(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# locate search form\n",
    "search_query = driver.find_element('name','q')\n",
    "\n",
    "# send_keys() to simulate the search text key strokes\n",
    "search_query.send_keys('site:linkedin.com/in/ AND \"React Develoepr\" AND \"India\"')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# .send_keys() to simulate the return key \n",
    "search_query.send_keys(Keys.ENTER)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#####After Search Extranction of Linkedin urls "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "search_results = driver.find_elements(By.CLASS_NAME,\"yuRUbf\")\n",
    " \n",
    "#instantiate empty list to save the list of the profile urls\n",
    "lnkdin_urls=[]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# iterate the search_results to extact the linked-in profile urls \n",
    "for result in search_results:\n",
    "    lnkdin_url = result.find_element(By.CSS_SELECTOR,'a[href]').get_attribute('href')\n",
    "    lnkdin_urls.append(lnkdin_url)\n",
    "#lnkdin_urls[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://in.linkedin.com/in/piyush-sharma-694386187\n",
      "https://in.linkedin.com/in/naveen-ambookken-249941212\n",
      "https://in.linkedin.com/in/arjun-r-pillai\n",
      "https://in.linkedin.com/in/sabeer-ali-3b99b3137\n",
      "https://in.linkedin.com/in/uma-ramesh-2a4ab7100\n",
      "https://in.linkedin.com/in/manish-gupta0099\n",
      "https://in.linkedin.com/in/bharat-rathor-295189220\n",
      "https://in.linkedin.com/in/shahid-husain-react-frontend\n",
      "https://in.linkedin.com/in/avinash-sagar-363b27231/es?trk=people-guest_people_search-card\n",
      "https://in.linkedin.com/in/skyrunner360\n"
     ]
    }
   ],
   "source": [
    "for url in lnkdin_urls:\n",
    "    print(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.get(lnkdin_urls[3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "sel= Selector(text=driver.page_source)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Yaha Tak to thik hai .. aage ka bhagwan jaane"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# xpath to extract the exact class containing the text\n",
    "#name1 = sel.xpath('//*[starts-with(@class, \"text-heading-xlarge inline t-24 v-align-middle break-words\")]/text()').extract()\n",
    "\n",
    "#len(name1)\n",
    "\n",
    "# using selenium  and not parse \n",
    "#name2=driver.find_element(By.CSS_SELECTOR, \"text-heading-xlarge inline t-24 v-align-middle break-words\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#role = sel.xpath('//*[starts-with(@class, \"text-body-medium break-words\")]/text()').extract_first().strip()\n",
    "#role\n",
    "#stripped_role = role.strip()#\n",
    "#stripped_role"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#location =  sel.xpath('//*[starts-with(@class,  \"text-body-small inline t-black--light break-words\")]/text()').extract_first().strip()\n",
    " \n",
    "#location"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#company = sel.xpath('//*[starts-with(@class,\"inline-show-more-textinline-show-more-text--is-collapsed inline-show-more-text--is-collapsed-with-line-clamp inline\")]/text()').extract_first().strip()\n",
    "\n",
    "#print('company:', company)\n",
    "\n",
    "#college =sel.xpath('//*[starts-with(@class,\"inline-show-more-text inline-show-more-text--is-collapsed inline-show-more-text--is-collapsed-with-line-clamp inline\")]/text()').extract_first().strip()\n",
    "#print('college:', college)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#employer=sel.css(\".inline-show-more-textinline-show-more-text--is-collapsed inline-show-more-text--is-collapsed-with-line-clamp inline\").get()\n",
    "#employer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  },
  "vscode": {
   "interpreter": {
    "hash": "e7370f93d1d0cde622a1f8e1c04877d8463912d04d973331ad4851f04de6915a"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}