bioresponse_better_rf_1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "from time import time\n",
    "from typing import List\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from scipy.stats import uniform\n",
    "from sklearn.base import clone\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.metrics import log_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Activity</th>\n",
       "      <th>D1</th>\n",
       "      <th>D2</th>\n",
       "      <th>D3</th>\n",
       "      <th>D4</th>\n",
       "      <th>D5</th>\n",
       "      <th>D6</th>\n",
       "      <th>D7</th>\n",
       "      <th>D8</th>\n",
       "      <th>D9</th>\n",
       "      <th>...</th>\n",
       "      <th>D1767</th>\n",
       "      <th>D1768</th>\n",
       "      <th>D1769</th>\n",
       "      <th>D1770</th>\n",
       "      <th>D1771</th>\n",
       "      <th>D1772</th>\n",
       "      <th>D1773</th>\n",
       "      <th>D1774</th>\n",
       "      <th>D1775</th>\n",
       "      <th>D1776</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.497009</td>\n",
       "      <td>0.10</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.132956</td>\n",
       "      <td>0.678031</td>\n",
       "      <td>0.273166</td>\n",
       "      <td>0.585445</td>\n",
       "      <td>0.743663</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0.366667</td>\n",
       "      <td>0.606291</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.111209</td>\n",
       "      <td>0.803455</td>\n",
       "      <td>0.106105</td>\n",
       "      <td>0.411754</td>\n",
       "      <td>0.836582</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>0.033300</td>\n",
       "      <td>0.480124</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.209791</td>\n",
       "      <td>0.610350</td>\n",
       "      <td>0.356453</td>\n",
       "      <td>0.517720</td>\n",
       "      <td>0.679051</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.538825</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.196344</td>\n",
       "      <td>0.724230</td>\n",
       "      <td>0.235606</td>\n",
       "      <td>0.288764</td>\n",
       "      <td>0.805110</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>0.517794</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.494734</td>\n",
       "      <td>0.781422</td>\n",
       "      <td>0.154361</td>\n",
       "      <td>0.303809</td>\n",
       "      <td>0.812646</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1777 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Activity        D1        D2    D3   D4        D5        D6        D7  \\\n",
       "0         1  0.000000  0.497009  0.10  0.0  0.132956  0.678031  0.273166   \n",
       "1         1  0.366667  0.606291  0.05  0.0  0.111209  0.803455  0.106105   \n",
       "2         1  0.033300  0.480124  0.00  0.0  0.209791  0.610350  0.356453   \n",
       "3         1  0.000000  0.538825  0.00  0.5  0.196344  0.724230  0.235606   \n",
       "4         0  0.100000  0.517794  0.00  0.0  0.494734  0.781422  0.154361   \n",
       "\n",
       "         D8        D9  ...  D1767  D1768  D1769  D1770  D1771  D1772  D1773  \\\n",
       "0  0.585445  0.743663  ...      0      0      0      0      0      0      0   \n",
       "1  0.411754  0.836582  ...      1      1      1      1      0      1      0   \n",
       "2  0.517720  0.679051  ...      0      0      0      0      0      0      0   \n",
       "3  0.288764  0.805110  ...      0      0      0      0      0      0      0   \n",
       "4  0.303809  0.812646  ...      0      0      0      0      0      0      0   \n",
       "\n",
       "   D1774  D1775  D1776  \n",
       "0      0      0      0  \n",
       "1      0      1      0  \n",
       "2      0      0      0  \n",
       "3      0      0      0  \n",
       "4      0      0      0  \n",
       "\n",
       "[5 rows x 1777 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(\"input/bioresponse/train.csv\")\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_name = \"Activity\"\n",
    "y = df.loc[:, target_name]\n",
    "X = df.drop(target_name, axis=\"columns\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((3000, 1776), (751, 1776), (3000,), (751,))"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X,\n",
    "                                                    y,\n",
    "                                                    test_size=0.2,\n",
    "                                                    stratify=y,\n",
    "                                                    random_state=1,\n",
    "                                                    )\n",
    "\n",
    "X_train.shape, X_test.shape, y_train.shape, y_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "kwargs_rf = {\"random_state\": 1}\n",
    "grid_rf = {\"n_estimators\": np.arange(50, 500),\n",
    "           \"max_features\": [\"auto\", \"sqrt\", 0.33],\n",
    "           \"min_samples_leaf\": np.arange(1, 15),\n",
    "           }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "rf_base = RandomForestClassifier(**kwargs_rf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CV trying 300 settings fitted in 2.5820587002568773 hours with 2 jobs!\n"
     ]
    }
   ],
   "source": [
    "n_settings_to_sample = 300\n",
    "n_jobs = 2\n",
    "\n",
    "start_time = time()\n",
    "\n",
    "#Instantiate CV model\n",
    "rf_cv = RandomizedSearchCV(estimator=rf_base,\n",
    "                           param_distributions=grid_rf,\n",
    "                           n_iter=n_settings_to_sample,\n",
    "                           random_state=1,\n",
    "                           n_jobs=n_jobs,\n",
    "                           )\n",
    "\n",
    "#fit model performing CV\n",
    "rf_cv.fit(X_train, y_train)\n",
    "\n",
    "end_time = time()\n",
    "tot_time = end_time - start_time\n",
    "print(f\"CV trying {n_settings_to_sample} settings fitted in {tot_time / 3600} hours\"\n",
    "      f\" with {n_jobs} jobs!\"\n",
    "      )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'n_estimators': 205, 'min_samples_leaf': 3, 'max_features': 0.33}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_params = rf_cv.best_params_\n",
    "best_params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>n_estimators</th>\n",
       "      <th>min_samples_leaf</th>\n",
       "      <th>max_features</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>344</td>\n",
       "      <td>11</td>\n",
       "      <td>0.33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>498</td>\n",
       "      <td>4</td>\n",
       "      <td>0.33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>66</td>\n",
       "      <td>10</td>\n",
       "      <td>0.33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>265</td>\n",
       "      <td>9</td>\n",
       "      <td>auto</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>436</td>\n",
       "      <td>2</td>\n",
       "      <td>sqrt</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   n_estimators  min_samples_leaf max_features\n",
       "0           344                11         0.33\n",
       "1           498                 4         0.33\n",
       "2            66                10         0.33\n",
       "3           265                 9         auto\n",
       "4           436                 2         sqrt"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_tried_params = pd.DataFrame(rf_cv.cv_results_.get(\"params\"))\n",
    "df_tried_params.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "out_name = \"output/better_rf_1_cv.pickle\"\n",
    "with open(out_name, \"wb\") as f:\n",
    "    pickle.dump(rf_cv, f) #whole CV model is being dumped, not only best_estimator_! For further studies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'roc_auc_score_train': 0.9987689530863688,\n",
       " 'log_loss_train': 0.1884336270550344,\n",
       " 'roc_auc_score_test': 0.8729358322381578,\n",
       " 'log_loss_test': 0.44510011383177567}"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def various_metrics_binary_classification(model, metrics: List, X: pd.DataFrame, y_true: pd.Series, identifier: str = None):\n",
    "    \n",
    "    if not (isinstance(metrics, List) and any(metrics)):\n",
    "        raise ValueError(\"First argument should be a non-empty list!\")\n",
    "        \n",
    "    pred_proba = model.predict_proba(X)[:, 1]\n",
    "    \n",
    "    results = {}\n",
    "    for metric in metrics:\n",
    "        result = metric(y_true, pred_proba)\n",
    "        label_metric = metric.__name__ if not identifier else f\"{metric.__name__}_{identifier}\"\n",
    "        results[label_metric] = result\n",
    "        \n",
    "    return results\n",
    "\n",
    "metrics = [roc_auc_score, log_loss]\n",
    "results_metrics_train = various_metrics_binary_classification(rf_cv, metrics, X_train, y_train, identifier=\"train\")\n",
    "results_metrics_test = various_metrics_binary_classification(rf_cv, metrics, X_test, y_test, identifier=\"test\")\n",
    "results_metrics = {**results_metrics_train, **results_metrics_test}\n",
    "results_metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "roc_auc_score_train    0.998769\n",
       "log_loss_train         0.188434\n",
       "roc_auc_score_test     0.872936\n",
       "log_loss_test          0.445100\n",
       "dtype: float64"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.Series(results_metrics)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "My note: it is only slightly better on test set that previous model!"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}