Chapter 5 - Handling Categorical Data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 5.1 Encoding Nominal Categorical Feature\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 0, 1],\n",
       "       [1, 0, 0],\n",
       "       [0, 0, 1],\n",
       "       [0, 1, 0],\n",
       "       [0, 0, 1]])"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "from sklearn.preprocessing import LabelBinarizer, MultiLabelBinarizer\n",
    "\n",
    "feature = np.array([\n",
    "    [\"Texas\"],\n",
    "    [\"California\"],\n",
    "    [\"Texas\"],\n",
    "    [\"Delaware\"],\n",
    "    [\"Texas\"]\n",
    "])\n",
    "\n",
    "# create one-hot encoder\n",
    "one_hot = LabelBinarizer()\n",
    "\n",
    "# one-hot encode feature\n",
    "one_hot.fit_transform(feature)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['California', 'Delaware', 'Texas'], dtype='<U10')"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# view feature classes\n",
    "one_hot.classes_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Texas', 'California', 'Texas', 'Delaware', 'Texas'], dtype='<U10')"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# reverse one-hot encoding\n",
    "one_hot.inverse_transform(one_hot.transform(feature))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>California</th>\n",
       "      <th>Delaware</th>\n",
       "      <th>Texas</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   California  Delaware  Texas\n",
       "0           0         0      1\n",
       "1           1         0      0\n",
       "2           0         0      1\n",
       "3           0         1      0\n",
       "4           0         0      1"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "pd.get_dummies(feature[:, 0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 0, 0, 1, 1],\n",
       "       [1, 1, 0, 0, 0],\n",
       "       [0, 0, 0, 1, 1],\n",
       "       [0, 0, 1, 1, 0],\n",
       "       [1, 0, 0, 0, 1]])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# create multiclass feature\n",
    "multiclass_feature = [\n",
    "    (\"Texas\", \"Florida\"),\n",
    "    (\"California\", \"Alabama\"),\n",
    "    (\"Texas\", \"Florida\"),\n",
    "    (\"Delaware\", \"Florida\"),\n",
    "    (\"Texas\", \"Alabama\")\n",
    "]\n",
    "\n",
    "# create multiclass one-hot encoder\n",
    "one_hot_multiclass = MultiLabelBinarizer()\n",
    "\n",
    "# one-hot encode multiclass feature\n",
    "one_hot_multiclass.fit_transform(multiclass_feature)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Alabama', 'California', 'Delaware', 'Florida', 'Texas'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# view classes\n",
    "one_hot_multiclass.classes_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Discussion\n",
    "We might think the proper strategy is to assign each class a numerical value (e.g. Texas = 1, California = 2). However, when our classes have no intrinsic ordering (e.g. Texas isn't \"less\" than California), our numerical values erroneously create an ordering that is not present.\n",
    "\n",
    "The proper strategy is to create a binary feature for each class in the original feature. This is often called one-hot encoding (in machine learning literature) or dummying (in statistical and research literature). Our solution's feature was a vector containing three classes (i.e. Texas, California, and Delaware). In one-hot encoding, each class becomes its own feature with 1s when the class appears and 0s otherwise. Because one feature had three classes, one-hot encoding returned threeb inary features (one for each class). By using one-hot encoding we can capture the membership of an obsrvation in a class while preserving the notion that the class lacks any sort of hierarchy.\n",
    "\n",
    "### See Also\n",
    "* Dummy Variable Trap, Algosome (http://www.algosome.com/articles/dummy-variable-trap-regression.html)\n",
    "* Dropping one of the columns when using one-hot encoding, CrossValidated (https://stats.stackexchange.com/questions/231285/dropping-one-of-the-columns-when-using-one-hot-encoding)\n",
    "\n",
    "## 5.2 Encoding Ordinal Categorical Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    1\n",
       "1    1\n",
       "2    2\n",
       "3    2\n",
       "4    3\n",
       "Name: Score, dtype: int64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "# create features\n",
    "df = pd.DataFrame({\"Score\": [\"Low\", \"Low\", \"Medium\", \"Medium\", \"High\"]})\n",
    "\n",
    "# create mapper\n",
    "scale_mapper = {\n",
    "    \"Low\": 1,\n",
    "    \"Medium\": 2,\n",
    "    \"High\": 3\n",
    "}\n",
    "\n",
    "# replace feature values with scale\n",
    "df[\"Score\"].replace(scale_mapper)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5.3 Encoding Dictionaries of Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[4., 2., 0.],\n",
       "       [3., 4., 0.],\n",
       "       [0., 1., 2.],\n",
       "       [0., 2., 2.]])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.feature_extraction import DictVectorizer\n",
    "\n",
    "data_dict = [\n",
    "    {\"Red\": 2, \"Blue\": 4},\n",
    "    {\"Red\": 4, \"Blue\": 3},\n",
    "    {\"Red\": 1, \"Yellow\": 2},\n",
    "    {\"Red\": 2, \"Yellow\": 2}\n",
    "]\n",
    "\n",
    "# create dictionary vectorizer\n",
    "dictvectorizer = DictVectorizer(sparse=False)\n",
    "\n",
    "# convert dictionary to feature matrix\n",
    "features = dictvectorizer.fit_transform(data_dict)\n",
    "\n",
    "features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Blue', 'Red', 'Yellow']"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get feature names\n",
    "dictvectorizer.get_feature_names()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5.4 Imputing Missing Class Values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.  ,  0.87,  1.31],\n",
       "       [ 1.  , -0.67, -0.22],\n",
       "       [ 0.  ,  2.1 ,  1.45],\n",
       "       [ 1.  ,  1.18,  1.33],\n",
       "       [ 0.  ,  1.22,  1.27],\n",
       "       [ 1.  , -0.21, -1.19]])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# load libraries\n",
    "import numpy as np\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "\n",
    "X = np.array([[0, 2.10, 1.45],\n",
    "            [1, 1.18, 1.33],\n",
    "            [0, 1.22, 1.27],\n",
    "            [1, -0.21, -1.19]])\n",
    "\n",
    "X_with_nan = np.array([[np.nan, 0.87, 1.31],\n",
    "                      [np.nan, -0.67, -0.22]])\n",
    "\n",
    "# train KNN learner\n",
    "clf = KNeighborsClassifier(3, weights='distance')\n",
    "trained_model = clf.fit(X[:,1:], X[:, 0])\n",
    "\n",
    "# predict missing values' class\n",
    "imputed_values = trained_model.predict(X_with_nan[:, 1:])\n",
    "\n",
    "# join column of predicted class with their other features\n",
    "X_with_imputed = np.hstack((imputed_values.reshape(-1, 1), X_with_nan[:, 1:]))\n",
    "\n",
    "# join two feature matricies\n",
    "np.vstack((X_with_imputed, X))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.  ,  0.87,  1.31],\n",
       "       [ 0.  , -0.67, -0.22],\n",
       "       [ 0.  ,  2.1 ,  1.45],\n",
       "       [ 1.  ,  1.18,  1.33],\n",
       "       [ 0.  ,  1.22,  1.27],\n",
       "       [ 1.  , -0.21, -1.19]])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.preprocessing import Imputer\n",
    "\n",
    "# join the two feature matricies\n",
    "X_complete = np.vstack((X_with_nan, X))\n",
    "\n",
    "imputer = Imputer(strategy='most_frequent', axis=0)\n",
    "\n",
    "imputer.fit_transform(X_complete)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### See Also\n",
    "* Overcoming Missing Values in a Random Forest Classifier (https://medium.com/airbnb-engineering/overcoming-missing-values-in-a-random-forest-classifier-7b1fc1fc03ba)\n",
    "* A Study of K-Nearest Neighbour as an Imputation Method (http://conteudo.icmc.usp.br/pessoas/gbatista/files/his2002.pdf)\n",
    "\n",
    "## 5.5 Handling Imbalanced Classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1])"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.datasets import load_iris\n",
    "\n",
    "iris = load_iris()\n",
    "\n",
    "features = iris.data\n",
    "target = iris.target\n",
    "\n",
    "# remove first 40 observations\n",
    "features = features[40:, :]\n",
    "target = target[40:]\n",
    "\n",
    "# create binary target vector indicating if class 0\n",
    "target = np.where((target == 0), 0, 1)\n",
    "\n",
    "target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, class_weight={0: 0.9, 1: 0.1},\n",
       "            criterion='gini', max_depth=None, max_features='auto',\n",
       "            max_leaf_nodes=None, min_impurity_decrease=0.0,\n",
       "            min_impurity_split=None, min_samples_leaf=1,\n",
       "            min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
       "            n_estimators=10, n_jobs=1, oob_score=False, random_state=None,\n",
       "            verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# create weights\n",
    "weights = {0: .9, 1: 0.1}\n",
    "\n",
    "RandomForestClassifier(class_weight=weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, class_weight='balanced',\n",
       "            criterion='gini', max_depth=None, max_features='auto',\n",
       "            max_leaf_nodes=None, min_impurity_decrease=0.0,\n",
       "            min_impurity_split=None, min_samples_leaf=1,\n",
       "            min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
       "            n_estimators=10, n_jobs=1, oob_score=False, random_state=None,\n",
       "            verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "RandomForestClassifier(class_weight=\"balanced\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1])"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "i_class0 = np.where(target == 0)[0]\n",
    "i_class1 = np.where(target == 1)[0]\n",
    "\n",
    "n_class0 = len(i_class0)\n",
    "n_class1 = len(i_class1)\n",
    "\n",
    "i_class1_downsampled = np.random.choice(i_class1, size=n_class0, replace=False)\n",
    "\n",
    "np.hstack((target[i_class0], target[i_class1_downsampled]))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[5. , 3.5, 1.3, 0.3],\n",
       "       [4.5, 2.3, 1.3, 0.3],\n",
       "       [4.4, 3.2, 1.3, 0.2],\n",
       "       [5. , 3.5, 1.6, 0.6],\n",
       "       [5.1, 3.8, 1.9, 0.4]])"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.vstack((features[i_class0,:], features[i_class1_downsampled, :]))[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1])"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "i_class0_upsampled = np.random.choice(i_class0, size=n_class1, replace=True)\n",
    "\n",
    "np.concatenate((target[i_class0_upsampled], target[i_class1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[5.1, 3.8, 1.9, 0.4],\n",
       "       [5. , 3.5, 1.3, 0.3],\n",
       "       [4.6, 3.2, 1.4, 0.2],\n",
       "       [5.3, 3.7, 1.5, 0.2],\n",
       "       [5. , 3.5, 1.3, 0.3]])"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.vstack((features[i_class0_upsampled,:], features[i_class1,:]))[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:machine_learning_cookbook]",
   "language": "python",
   "name": "conda-env-machine_learning_cookbook-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}