lightgbm_2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "4579a6d4",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from pandas.core.frame import DataFrame\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import re\n",
    "import os\n",
    "import utils\n",
    "import funcs\n",
    "import eda_plots\n",
    "import new_mlfuncs\n",
    "import lightgbm_funcs\n",
    "import dlfuncs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b2315c5e",
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option('display.max_columns', 500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "74ca36df",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 40 columns and 260601 records in the dataframe.\n",
      "Extracting count and percentages of nulls and non nulls\n"
     ]
    }
   ],
   "source": [
    "# import train data\n",
    "train_filepath = 'data/merged_transformed.csv'\n",
    "train_df, train_df_nulls = funcs.read_and_qa(train_filepath)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "d97e7c8d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 39 columns and 86868 records in the dataframe.\n",
      "Extracting count and percentages of nulls and non nulls\n"
     ]
    }
   ],
   "source": [
    "# import test data\n",
    "test_filepath = 'data/test_values.csv'\n",
    "test_df, test_df_nulls = funcs.read_and_qa(test_filepath)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "bf2b34df",
   "metadata": {},
   "outputs": [],
   "source": [
    "# initial columns to drop\n",
    "initial_to_drop = ['building_id']\n",
    "target_var = 'damage_grade'\n",
    "num_cols = ['count_floors_pre_eq', 'age', 'area_percentage', 'height_percentage']\n",
    "feats_to_exclude = ['has_superstructure_adobe_mud',\n",
    "       'has_superstructure_mud_mortar_stone', 'has_superstructure_stone_flag',\n",
    "       'has_superstructure_cement_mortar_stone',\n",
    "       'has_superstructure_mud_mortar_brick',\n",
    "       'has_superstructure_cement_mortar_brick', 'has_superstructure_timber',\n",
    "       'has_superstructure_bamboo', 'has_superstructure_rc_non_engineered',\n",
    "       'has_superstructure_rc_engineered', 'has_superstructure_other',\n",
    "       'has_secondary_use',\n",
    "       'has_secondary_use_agriculture', 'has_secondary_use_hotel',\n",
    "       'has_secondary_use_rental', 'has_secondary_use_institution',\n",
    "       'has_secondary_use_school', 'has_secondary_use_industry',\n",
    "       'has_secondary_use_health_post', 'has_secondary_use_gov_office',\n",
    "       'has_secondary_use_use_police', 'has_secondary_use_other']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "5544bd0f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dropping: building_id\n",
      "There are no columns below the threshold: 0.4\n",
      "There are no high correlated features. No column will be dropped!\n",
      "Dropping low variance feature: plan_configuration\n",
      "Dropping low variance feature: legal_ownership_status\n"
     ]
    }
   ],
   "source": [
    "train_red_df = new_mlfuncs.pre_ml_preprocessing(train_df, initial_to_drop, num_cols, target_var = target_var, missing_val_threshold = 40, num_cols_threshold = 0.9, low_var_threshold = 0.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "3300efa5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>geo_level_1_id</th>\n",
       "      <th>geo_level_2_id</th>\n",
       "      <th>geo_level_3_id</th>\n",
       "      <th>count_floors_pre_eq</th>\n",
       "      <th>age</th>\n",
       "      <th>area_percentage</th>\n",
       "      <th>height_percentage</th>\n",
       "      <th>land_surface_condition</th>\n",
       "      <th>foundation_type</th>\n",
       "      <th>roof_type</th>\n",
       "      <th>ground_floor_type</th>\n",
       "      <th>other_floor_type</th>\n",
       "      <th>position</th>\n",
       "      <th>has_superstructure_adobe_mud</th>\n",
       "      <th>has_superstructure_mud_mortar_stone</th>\n",
       "      <th>has_superstructure_stone_flag</th>\n",
       "      <th>has_superstructure_cement_mortar_stone</th>\n",
       "      <th>has_superstructure_mud_mortar_brick</th>\n",
       "      <th>has_superstructure_cement_mortar_brick</th>\n",
       "      <th>has_superstructure_timber</th>\n",
       "      <th>has_superstructure_bamboo</th>\n",
       "      <th>has_superstructure_rc_non_engineered</th>\n",
       "      <th>has_superstructure_rc_engineered</th>\n",
       "      <th>has_superstructure_other</th>\n",
       "      <th>count_families</th>\n",
       "      <th>has_secondary_use</th>\n",
       "      <th>has_secondary_use_agriculture</th>\n",
       "      <th>has_secondary_use_hotel</th>\n",
       "      <th>has_secondary_use_rental</th>\n",
       "      <th>has_secondary_use_institution</th>\n",
       "      <th>has_secondary_use_school</th>\n",
       "      <th>has_secondary_use_industry</th>\n",
       "      <th>has_secondary_use_health_post</th>\n",
       "      <th>has_secondary_use_gov_office</th>\n",
       "      <th>has_secondary_use_use_police</th>\n",
       "      <th>has_secondary_use_other</th>\n",
       "      <th>damage_grade</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6</td>\n",
       "      <td>487</td>\n",
       "      <td>12198</td>\n",
       "      <td>2</td>\n",
       "      <td>30</td>\n",
       "      <td>6</td>\n",
       "      <td>5</td>\n",
       "      <td>t</td>\n",
       "      <td>r</td>\n",
       "      <td>n</td>\n",
       "      <td>f</td>\n",
       "      <td>q</td>\n",
       "      <td>t</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>8</td>\n",
       "      <td>900</td>\n",
       "      <td>2812</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>7</td>\n",
       "      <td>o</td>\n",
       "      <td>r</td>\n",
       "      <td>n</td>\n",
       "      <td>x</td>\n",
       "      <td>q</td>\n",
       "      <td>s</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>21</td>\n",
       "      <td>363</td>\n",
       "      <td>8973</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>t</td>\n",
       "      <td>r</td>\n",
       "      <td>n</td>\n",
       "      <td>f</td>\n",
       "      <td>x</td>\n",
       "      <td>t</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>22</td>\n",
       "      <td>418</td>\n",
       "      <td>10694</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>6</td>\n",
       "      <td>5</td>\n",
       "      <td>t</td>\n",
       "      <td>r</td>\n",
       "      <td>n</td>\n",
       "      <td>f</td>\n",
       "      <td>x</td>\n",
       "      <td>s</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>11</td>\n",
       "      <td>131</td>\n",
       "      <td>1488</td>\n",
       "      <td>3</td>\n",
       "      <td>30</td>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>t</td>\n",
       "      <td>r</td>\n",
       "      <td>n</td>\n",
       "      <td>f</td>\n",
       "      <td>x</td>\n",
       "      <td>s</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   geo_level_1_id  geo_level_2_id  geo_level_3_id  count_floors_pre_eq  age  \\\n",
       "0               6             487           12198                    2   30   \n",
       "1               8             900            2812                    2   10   \n",
       "2              21             363            8973                    2   10   \n",
       "3              22             418           10694                    2   10   \n",
       "4              11             131            1488                    3   30   \n",
       "\n",
       "   area_percentage  height_percentage land_surface_condition foundation_type  \\\n",
       "0                6                  5                      t               r   \n",
       "1                8                  7                      o               r   \n",
       "2                5                  5                      t               r   \n",
       "3                6                  5                      t               r   \n",
       "4                8                  9                      t               r   \n",
       "\n",
       "  roof_type ground_floor_type other_floor_type position  \\\n",
       "0         n                 f                q        t   \n",
       "1         n                 x                q        s   \n",
       "2         n                 f                x        t   \n",
       "3         n                 f                x        s   \n",
       "4         n                 f                x        s   \n",
       "\n",
       "   has_superstructure_adobe_mud  has_superstructure_mud_mortar_stone  \\\n",
       "0                             1                                    1   \n",
       "1                             0                                    1   \n",
       "2                             0                                    1   \n",
       "3                             0                                    1   \n",
       "4                             1                                    0   \n",
       "\n",
       "   has_superstructure_stone_flag  has_superstructure_cement_mortar_stone  \\\n",
       "0                              0                                       0   \n",
       "1                              0                                       0   \n",
       "2                              0                                       0   \n",
       "3                              0                                       0   \n",
       "4                              0                                       0   \n",
       "\n",
       "   has_superstructure_mud_mortar_brick  \\\n",
       "0                                    0   \n",
       "1                                    0   \n",
       "2                                    0   \n",
       "3                                    0   \n",
       "4                                    0   \n",
       "\n",
       "   has_superstructure_cement_mortar_brick  has_superstructure_timber  \\\n",
       "0                                       0                          0   \n",
       "1                                       0                          0   \n",
       "2                                       0                          0   \n",
       "3                                       0                          1   \n",
       "4                                       0                          0   \n",
       "\n",
       "   has_superstructure_bamboo  has_superstructure_rc_non_engineered  \\\n",
       "0                          0                                     0   \n",
       "1                          0                                     0   \n",
       "2                          0                                     0   \n",
       "3                          1                                     0   \n",
       "4                          0                                     0   \n",
       "\n",
       "   has_superstructure_rc_engineered  has_superstructure_other  count_families  \\\n",
       "0                                 0                         0               1   \n",
       "1                                 0                         0               1   \n",
       "2                                 0                         0               1   \n",
       "3                                 0                         0               1   \n",
       "4                                 0                         0               1   \n",
       "\n",
       "   has_secondary_use  has_secondary_use_agriculture  has_secondary_use_hotel  \\\n",
       "0                  0                              0                        0   \n",
       "1                  0                              0                        0   \n",
       "2                  0                              0                        0   \n",
       "3                  0                              0                        0   \n",
       "4                  0                              0                        0   \n",
       "\n",
       "   has_secondary_use_rental  has_secondary_use_institution  \\\n",
       "0                         0                              0   \n",
       "1                         0                              0   \n",
       "2                         0                              0   \n",
       "3                         0                              0   \n",
       "4                         0                              0   \n",
       "\n",
       "   has_secondary_use_school  has_secondary_use_industry  \\\n",
       "0                         0                           0   \n",
       "1                         0                           0   \n",
       "2                         0                           0   \n",
       "3                         0                           0   \n",
       "4                         0                           0   \n",
       "\n",
       "   has_secondary_use_health_post  has_secondary_use_gov_office  \\\n",
       "0                              0                             0   \n",
       "1                              0                             0   \n",
       "2                              0                             0   \n",
       "3                              0                             0   \n",
       "4                              0                             0   \n",
       "\n",
       "   has_secondary_use_use_police  has_secondary_use_other  damage_grade  \n",
       "0                             0                        0             3  \n",
       "1                             0                        0             2  \n",
       "2                             0                        0             3  \n",
       "3                             0                        0             2  \n",
       "4                             0                        0             3  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_red_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "adcf4d22",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dropping: building_id\n",
      "There are no columns below the threshold: 0.4\n",
      "There are no high correlated features. No column will be dropped!\n",
      "Dropping low variance feature: plan_configuration\n",
      "Dropping low variance feature: legal_ownership_status\n"
     ]
    }
   ],
   "source": [
    "test_red_df = new_mlfuncs.pre_ml_preprocessing(test_df, initial_to_drop, num_cols, target_var = None, missing_val_threshold = 40, num_cols_threshold = 0.9, low_var_threshold = 0.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "f50ae08a",
   "metadata": {},
   "outputs": [],
   "source": [
    "train, test, le = new_mlfuncs.target_encode_categorical_features(train_red_df, test_red_df, target=target_var, alpha=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "7f61b44d",
   "metadata": {},
   "outputs": [],
   "source": [
    "target_var = 'target_encoded'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "b36c1a3b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating pipeline for LightGBM Classifier.\n",
      "Fitting training data to pipeline for LightGBM Classifier.\n",
      "Predicting test values for LightGBM Classifier.\n",
      "Calculating accuracy score for LightGBM Classifier.\n",
      "Accuracy Score for LightGBM Classifier: 0.7383002563276082\n"
     ]
    }
   ],
   "source": [
    "clf_f1_scr, pipe = lightgbm_funcs.lightgbm_model_1(train, target_var, feats_to_exclude, text_feats=None, stratify = True, test_size = 0.25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4f615765",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:PythonAdv] *",
   "language": "python",
   "name": "conda-env-PythonAdv-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}