2-WebScrapping-House-Price.ipynb

{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install bs4","execution_count":2,"outputs":[{"output_type":"stream","text":"Collecting bs4\n  Downloading bs4-0.0.1.tar.gz (1.1 kB)\nCollecting beautifulsoup4\n  Downloading beautifulsoup4-4.9.3-py3-none-any.whl (115 kB)\n\u001b[K     |████████████████████████████████| 115 kB 2.9 MB/s eta 0:00:01\n\u001b[?25hCollecting soupsieve>1.2\n  Downloading soupsieve-2.2-py3-none-any.whl (33 kB)\nBuilding wheels for collected packages: bs4\n  Building wheel for bs4 (setup.py) ... \u001b[?25ldone\n\u001b[?25h  Created wheel for bs4: filename=bs4-0.0.1-py3-none-any.whl size=1273 sha256=e57d96e603daddc6e602121e94c9fa4e5d0a03ab9b5b80bce2f375a758a52b5b\n  Stored in directory: /root/.cache/pip/wheels/0a/9e/ba/20e5bbc1afef3a491f0b3bb74d508f99403aabe76eda2167ca\nSuccessfully built bs4\nInstalling collected packages: soupsieve, beautifulsoup4, bs4\nSuccessfully installed beautifulsoup4-4.9.3 bs4-0.0.1 soupsieve-2.2\n","name":"stdout"}]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd ","execution_count":1,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as plt\npd.set_option('display.max_columns', None)\npd.set_option('display.max_rows',None)\nfrom IPython.display import FileLink\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler,MinMaxScaler\nfrom lightgbm import LGBMRegressor\nfrom bs4 import BeautifulSoup\nimport requests","execution_count":4,"outputs":[{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":"<style type='text/css'>\n.datatable table.frame { margin-bottom: 0; }\n.datatable table.frame thead { border-bottom: none; }\n.datatable table.frame tr.coltypes td {  color: #FFFFFF;  line-height: 6px;  padding: 0 0.5em;}\n.datatable .bool    { background: #DDDD99; }\n.datatable .object  { background: #565656; }\n.datatable .int     { background: #5D9E5D; }\n.datatable .float   { background: #4040CC; }\n.datatable .str     { background: #CC4040; }\n.datatable .row_index {  background: var(--jp-border-color3);  border-right: 1px solid var(--jp-border-color0);  color: var(--jp-ui-font-color3);  font-size: 9px;}\n.datatable .frame tr.coltypes .row_index {  background: var(--jp-border-color0);}\n.datatable th:nth-child(2) { padding-left: 12px; }\n.datatable .hellipsis {  color: var(--jp-cell-editor-border-color);}\n.datatable .vellipsis {  background: var(--jp-layout-color0);  color: var(--jp-cell-editor-border-color);}\n.datatable .na {  color: var(--jp-cell-editor-border-color);  font-size: 80%;}\n.datatable .footer { font-size: 9px; }\n.datatable .frame_dimensions {  background: var(--jp-border-color3);  border-top: 1px solid var(--jp-border-color0);  color: var(--jp-ui-font-color3);  display: inline-block;  opacity: 0.6;  padding: 1px 10px 1px 5px;}\n</style>\n"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"def check_status_code(url):\n    r=requests.get(url)\n    print(r.status_code)","execution_count":5,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"check_status_code(\"https://www.emlakjet.com/kiralik-konut/istanbul/1/\")","execution_count":6,"outputs":[{"output_type":"stream","text":"200\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_page_links(page_start,page_end):\n    pages=np.arange(page_start,page_end)\n    page_links=[]  \n    for page in pages:\n        page_link=\"https://www.emlakjet.com/kiralik-konut/istanbul/{page}/\".format(page=page)\n        page_links.append(page_link)\n    return page_links","execution_count":7,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"page_links=get_page_links(1,51)","execution_count":10,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def getHouseInfo(url):\n    r=requests.get(url)\n    soup=BeautifulSoup(r.text,\"html.parser\")\n    #print(soup.prettify())\n    return soup ","execution_count":11,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(page_links)","execution_count":22,"outputs":[{"output_type":"execute_result","execution_count":22,"data":{"text/plain":"50"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nannouns=[]\nfor i in range(len(page_links)):\n    url=page_links[i]\n    soup=getHouseInfo(url)\n    \n    for j in range(30):\n        link=\"https://www.emlakjet.com\"+soup.find_all(\"div\",class_=\"styles_listingItem__1asTK\")[j].find(\"a\").get(\"href\")\n        announs.append(link)","execution_count":25,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(announs)","execution_count":27,"outputs":[{"output_type":"execute_result","execution_count":27,"data":{"text/plain":"1500"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"url=announs[111]\nr=requests.get(url)\nsoup=BeautifulSoup(r.content,\"html.parser\")\n","execution_count":135,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"nos=[]\nann_dates=[]\ntypes=[]\ncategories=[]\nconst_types=[]\nnet_m2s=[]\nbrut_m2s=[]\nrooms=[]\nages=[]\nfloors=[]\nno_of_floors=[]\nheatings=[]\ndues=[]\ndeposits=[]\nno_of_bathrooms=[]\nno_of_wcs=[]\nin_sites=[]\nfurnitures=[]\nprices=[]\n\nfor i in announs: \n    url=i\n    r=requests.get(url)\n    soup=BeautifulSoup(r.content,\"html.parser\")\n    \n    try:\n        \n            no=soup.find_all(class_=\"styles_tableHalfRow__3zO2j\")[0].find_all(class_=\"styles_tableColumn__2x6nG\")[1].text\n            nos.append(no)\n            date=soup.find_all(class_=\"styles_tableHalfRow__3zO2j\")[1].find_all(class_=\"styles_tableColumn__2x6nG\")[1].text\n            ann_dates.append(date)\n            typ=soup.find_all(class_=\"styles_tableHalfRow__3zO2j\")[3].find_all(class_=\"styles_tableColumn__2x6nG\")[1].text\n            types.append(typ)\n            cat=soup.find_all(class_=\"styles_tableHalfRow__3zO2j\")[4].find_all(class_=\"styles_tableColumn__2x6nG\")[1].text\n            categories.append(cat)\n            const_type=soup.find_all(class_=\"styles_tableHalfRow__3zO2j\")[5].find_all(class_=\"styles_tableColumn__2x6nG\")[1].text\n            const_types.append(const_type)\n            net_m2=soup.find_all(class_=\"styles_tableHalfRow__3zO2j\")[6].find_all(class_=\"styles_tableColumn__2x6nG\")[1].text\n            net_m2s.append(net_m2)\n            brut_m2=soup.find_all(class_=\"styles_tableHalfRow__3zO2j\")[7].find_all(class_=\"styles_tableColumn__2x6nG\")[1].text\n            brut_m2s.append(brut_m2)\n            room=soup.find_all(class_=\"styles_tableHalfRow__3zO2j\")[9].find_all(class_=\"styles_tableColumn__2x6nG\")[1].text\n            rooms.append(room)\n            age=soup.find_all(class_=\"styles_tableHalfRow__3zO2j\")[10].find_all(class_=\"styles_tableColumn__2x6nG\")[1].text\n            ages.append(age)\n            floor=soup.find_all(class_=\"styles_tableHalfRow__3zO2j\")[11].find_all(class_=\"styles_tableColumn__2x6nG\")[1].text\n            floors.append(floor)\n            no_floor=soup.find_all(class_=\"styles_tableHalfRow__3zO2j\")[12].find_all(class_=\"styles_tableColumn__2x6nG\")[1].text\n            no_of_floors.append(no_floor)\n            heat=soup.find_all(class_=\"styles_tableHalfRow__3zO2j\")[13].find_all(class_=\"styles_tableColumn__2x6nG\")[1].text\n            heatings.append(heat)\n            due=soup.find_all(class_=\"styles_tableHalfRow__3zO2j\")[14].find_all(class_=\"styles_tableColumn__2x6nG\")[1].text\n            dues.append(due)\n            deposit=soup.find_all(class_=\"styles_tableHalfRow__3zO2j\")[15].find_all(class_=\"styles_tableColumn__2x6nG\")[1].text\n            deposits.append(deposit)\n            no_of_bathroom=soup.find_all(class_=\"styles_tableHalfRow__3zO2j\")[16].find_all(class_=\"styles_tableColumn__2x6nG\")[1].text\n            no_of_bathrooms.append(no_of_bathroom)\n            no_of_wc=soup.find_all(class_=\"styles_tableHalfRow__3zO2j\")[17].find_all(class_=\"styles_tableColumn__2x6nG\")[1].text\n            no_of_wcs.append(no_of_wc)\n            in_site=soup.find_all(class_=\"styles_tableHalfRow__3zO2j\")[18].find_all(class_=\"styles_tableColumn__2x6nG\")[1].text\n            in_sites.append(in_site)\n            furniture=soup.find_all(class_=\"styles_tableHalfRow__3zO2j\")[19].find_all(class_=\"styles_tableColumn__2x6nG\")[1].text\n            furnitures.append(furniture)\n            price=soup.find(class_=\"styles_price__1e65F\").text\n            prices.append(price)\n            \n            \n            \n    except (IndexError,TypeError,AttributeError):\n            pass; ","execution_count":137,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"d={\"price\":prices,\"ann_date\":ann_dates,\"type\":types,\"category\":categories,\"const_type\":const_types,\n  \"net_m2\":net_m2s,\"brut_m2\":brut_m2s,\"room\":rooms,\"age\":ages,\"floor\":floors,\"no_of_floor\":no_of_floors,\n  \"heating\":heatings,\"due\":dues,\"deposit\":deposits,\"no_of_bathroom\":no_of_bathrooms,\"no_of_wc\":no_of_wcs,\n  \"in_site\":in_sites,\"furniture\":furnitures,\"price\":prices}\n\n\ndata=pd.DataFrame(dict([ (k,pd.Series(v)) for k,v in d.items() ]))","execution_count":138,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.head()","execution_count":139,"outputs":[{"output_type":"execute_result","execution_count":139,"data":{"text/plain":"     price       ann_date   type category     const_type  net_m2 brut_m2 room  \\\n0  2,000TL  24 Şubat 2021  Konut  Kiralık  Belirtilmemiş   90 m2  100 m2  2+1   \n1  7,500TL  24 Şubat 2021  Konut  Kiralık      Betonarme  128 m2  140 m2  2+1   \n2  1,800TL  24 Şubat 2021  Konut  Kiralık  Belirtilmemiş  100 m2  101 m2  2+1   \n3  3,100TL  24 Şubat 2021  Konut  Kiralık  Belirtilmemiş   85 m2   95 m2  2+1   \n4  1,750TL  24 Şubat 2021  Konut  Kiralık      Betonarme   62 m2   63 m2  1+1   \n\n           age   floor no_of_floor              heating            due  \\\n0         5-10  1. Kat           9  Merkezi (Pay Ölçer)         280 TL   \n1  20 Ve Üzeri  2. Kat           6     Merkezi Doğalgaz       1.450 TL   \n2         5-10  1. Kat           5       Kombi Doğalgaz  Belirtilmemiş   \n3            4  8. Kat          12  Merkezi (Pay Ölçer)  Belirtilmemiş   \n4  20 Ve Üzeri  3. Kat           3     Merkezi Doğalgaz         210 TL   \n\n         deposit no_of_bathroom       no_of_wc in_site      furniture  \n0  Belirtilmemiş              1              1   Hayır         Eşyalı  \n1  Belirtilmemiş              2  Belirtilmemiş    Evet            Boş  \n2  Belirtilmemiş              1  Belirtilmemiş   Hayır  Belirtilmemiş  \n3  Belirtilmemiş              1  Belirtilmemiş   Hayır  Belirtilmemiş  \n4       3.500 TL              1              1   Hayır         Eşyalı  ","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>price</th>\n      <th>ann_date</th>\n      <th>type</th>\n      <th>category</th>\n      <th>const_type</th>\n      <th>net_m2</th>\n      <th>brut_m2</th>\n      <th>room</th>\n      <th>age</th>\n      <th>floor</th>\n      <th>no_of_floor</th>\n      <th>heating</th>\n      <th>due</th>\n      <th>deposit</th>\n      <th>no_of_bathroom</th>\n      <th>no_of_wc</th>\n      <th>in_site</th>\n      <th>furniture</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>2,000TL</td>\n      <td>24 Şubat 2021</td>\n      <td>Konut</td>\n      <td>Kiralık</td>\n      <td>Belirtilmemiş</td>\n      <td>90 m2</td>\n      <td>100 m2</td>\n      <td>2+1</td>\n      <td>5-10</td>\n      <td>1. Kat</td>\n      <td>9</td>\n      <td>Merkezi (Pay Ölçer)</td>\n      <td>280 TL</td>\n      <td>Belirtilmemiş</td>\n      <td>1</td>\n      <td>1</td>\n      <td>Hayır</td>\n      <td>Eşyalı</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>7,500TL</td>\n      <td>24 Şubat 2021</td>\n      <td>Konut</td>\n      <td>Kiralık</td>\n      <td>Betonarme</td>\n      <td>128 m2</td>\n      <td>140 m2</td>\n      <td>2+1</td>\n      <td>20 Ve Üzeri</td>\n      <td>2. Kat</td>\n      <td>6</td>\n      <td>Merkezi Doğalgaz</td>\n      <td>1.450 TL</td>\n      <td>Belirtilmemiş</td>\n      <td>2</td>\n      <td>Belirtilmemiş</td>\n      <td>Evet</td>\n      <td>Boş</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>1,800TL</td>\n      <td>24 Şubat 2021</td>\n      <td>Konut</td>\n      <td>Kiralık</td>\n      <td>Belirtilmemiş</td>\n      <td>100 m2</td>\n      <td>101 m2</td>\n      <td>2+1</td>\n      <td>5-10</td>\n      <td>1. Kat</td>\n      <td>5</td>\n      <td>Kombi Doğalgaz</td>\n      <td>Belirtilmemiş</td>\n      <td>Belirtilmemiş</td>\n      <td>1</td>\n      <td>Belirtilmemiş</td>\n      <td>Hayır</td>\n      <td>Belirtilmemiş</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>3,100TL</td>\n      <td>24 Şubat 2021</td>\n      <td>Konut</td>\n      <td>Kiralık</td>\n      <td>Belirtilmemiş</td>\n      <td>85 m2</td>\n      <td>95 m2</td>\n      <td>2+1</td>\n      <td>4</td>\n      <td>8. Kat</td>\n      <td>12</td>\n      <td>Merkezi (Pay Ölçer)</td>\n      <td>Belirtilmemiş</td>\n      <td>Belirtilmemiş</td>\n      <td>1</td>\n      <td>Belirtilmemiş</td>\n      <td>Hayır</td>\n      <td>Belirtilmemiş</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>1,750TL</td>\n      <td>24 Şubat 2021</td>\n      <td>Konut</td>\n      <td>Kiralık</td>\n      <td>Betonarme</td>\n      <td>62 m2</td>\n      <td>63 m2</td>\n      <td>1+1</td>\n      <td>20 Ve Üzeri</td>\n      <td>3. Kat</td>\n      <td>3</td>\n      <td>Merkezi Doğalgaz</td>\n      <td>210 TL</td>\n      <td>3.500 TL</td>\n      <td>1</td>\n      <td>1</td>\n      <td>Hayır</td>\n      <td>Eşyalı</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.shape","execution_count":140,"outputs":[{"output_type":"execute_result","execution_count":140,"data":{"text/plain":"(1498, 18)"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_csv=data.to_csv(\"data_house.csv\",index=False)\nfrom IPython.display import FileLink\nFileLink(r'data_house.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.7.9","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":4}