fraud-kirill-postprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [],
   "source": [
    "from functools import partial\n",
    "\n",
    "import datetime\n",
    "import lightgbm as lgb\n",
    "import numpy as np\n",
    "import os\n",
    "import pandas as pd\n",
    "import pickle\n",
    "import random\n",
    "import time\n",
    "\n",
    "from copy import deepcopy\n",
    "from scipy.stats import rankdata\n",
    "from sklearn.metrics import roc_auc_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def seed_everything(seed=13):\n",
    "    random.seed(seed)\n",
    "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
    "    np.random.seed(seed)\n",
    "    \n",
    "    \n",
    "def read_from_disk(path, filename):\n",
    "    with open(os.path.join(path, filename), 'rb') as handle:\n",
    "        return pickle.load(handle)\n",
    "    \n",
    "    \n",
    "def save_to_disk(obj, filename):\n",
    "    with open(filename, 'wb') as handle:\n",
    "        pickle.dump(obj, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "\n",
    "def timedelta(seconds):\n",
    "    return str(datetime.timedelta(seconds=seconds)).split('.')[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TransactionID</th>\n",
       "      <th>isFraud</th>\n",
       "      <th>TransactionDT</th>\n",
       "      <th>TransactionAmt</th>\n",
       "      <th>ProductCD</th>\n",
       "      <th>card1</th>\n",
       "      <th>card2</th>\n",
       "      <th>card3</th>\n",
       "      <th>card4</th>\n",
       "      <th>card5</th>\n",
       "      <th>...</th>\n",
       "      <th>id_31</th>\n",
       "      <th>id_32</th>\n",
       "      <th>id_33</th>\n",
       "      <th>id_34</th>\n",
       "      <th>id_35</th>\n",
       "      <th>id_36</th>\n",
       "      <th>id_37</th>\n",
       "      <th>id_38</th>\n",
       "      <th>DeviceType</th>\n",
       "      <th>DeviceInfo</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>2987000</td>\n",
       "      <td>0</td>\n",
       "      <td>86400</td>\n",
       "      <td>68.5</td>\n",
       "      <td>W</td>\n",
       "      <td>13926</td>\n",
       "      <td>NaN</td>\n",
       "      <td>150.0</td>\n",
       "      <td>discover</td>\n",
       "      <td>142.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>2987001</td>\n",
       "      <td>0</td>\n",
       "      <td>86401</td>\n",
       "      <td>29.0</td>\n",
       "      <td>W</td>\n",
       "      <td>2755</td>\n",
       "      <td>404.0</td>\n",
       "      <td>150.0</td>\n",
       "      <td>mastercard</td>\n",
       "      <td>102.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>2987002</td>\n",
       "      <td>0</td>\n",
       "      <td>86469</td>\n",
       "      <td>59.0</td>\n",
       "      <td>W</td>\n",
       "      <td>4663</td>\n",
       "      <td>490.0</td>\n",
       "      <td>150.0</td>\n",
       "      <td>visa</td>\n",
       "      <td>166.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>2987003</td>\n",
       "      <td>0</td>\n",
       "      <td>86499</td>\n",
       "      <td>50.0</td>\n",
       "      <td>W</td>\n",
       "      <td>18132</td>\n",
       "      <td>567.0</td>\n",
       "      <td>150.0</td>\n",
       "      <td>mastercard</td>\n",
       "      <td>117.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>2987004</td>\n",
       "      <td>0</td>\n",
       "      <td>86506</td>\n",
       "      <td>50.0</td>\n",
       "      <td>H</td>\n",
       "      <td>4497</td>\n",
       "      <td>514.0</td>\n",
       "      <td>150.0</td>\n",
       "      <td>mastercard</td>\n",
       "      <td>102.0</td>\n",
       "      <td>...</td>\n",
       "      <td>samsung browser 6.2</td>\n",
       "      <td>32.0</td>\n",
       "      <td>2220x1080</td>\n",
       "      <td>match_status:2</td>\n",
       "      <td>T</td>\n",
       "      <td>F</td>\n",
       "      <td>T</td>\n",
       "      <td>T</td>\n",
       "      <td>mobile</td>\n",
       "      <td>SAMSUNG SM-G892A Build/NRD90M</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 434 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   TransactionID  isFraud  TransactionDT  TransactionAmt ProductCD  card1  \\\n",
       "0        2987000        0          86400            68.5         W  13926   \n",
       "1        2987001        0          86401            29.0         W   2755   \n",
       "2        2987002        0          86469            59.0         W   4663   \n",
       "3        2987003        0          86499            50.0         W  18132   \n",
       "4        2987004        0          86506            50.0         H   4497   \n",
       "\n",
       "   card2  card3       card4  card5  ...                id_31  id_32  \\\n",
       "0    NaN  150.0    discover  142.0  ...                  NaN    NaN   \n",
       "1  404.0  150.0  mastercard  102.0  ...                  NaN    NaN   \n",
       "2  490.0  150.0        visa  166.0  ...                  NaN    NaN   \n",
       "3  567.0  150.0  mastercard  117.0  ...                  NaN    NaN   \n",
       "4  514.0  150.0  mastercard  102.0  ...  samsung browser 6.2   32.0   \n",
       "\n",
       "       id_33           id_34  id_35 id_36 id_37  id_38  DeviceType  \\\n",
       "0        NaN             NaN    NaN   NaN   NaN    NaN         NaN   \n",
       "1        NaN             NaN    NaN   NaN   NaN    NaN         NaN   \n",
       "2        NaN             NaN    NaN   NaN   NaN    NaN         NaN   \n",
       "3        NaN             NaN    NaN   NaN   NaN    NaN         NaN   \n",
       "4  2220x1080  match_status:2      T     F     T      T      mobile   \n",
       "\n",
       "                      DeviceInfo  \n",
       "0                            NaN  \n",
       "1                            NaN  \n",
       "2                            NaN  \n",
       "3                            NaN  \n",
       "4  SAMSUNG SM-G892A Build/NRD90M  \n",
       "\n",
       "[5 rows x 434 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trans_train = pd.read_csv('/kaggle/input/ieee-fraud-detection/train_transaction.csv')\n",
    "id_train = pd.read_csv('/kaggle/input/ieee-fraud-detection/train_identity.csv')\n",
    "df_train = trans_train.merge(id_train, how='left', left_on='TransactionID', right_on='TransactionID')\n",
    "del trans_train, id_train\n",
    "df_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TransactionID</th>\n",
       "      <th>TransactionDT</th>\n",
       "      <th>TransactionAmt</th>\n",
       "      <th>ProductCD</th>\n",
       "      <th>card1</th>\n",
       "      <th>card2</th>\n",
       "      <th>card3</th>\n",
       "      <th>card4</th>\n",
       "      <th>card5</th>\n",
       "      <th>card6</th>\n",
       "      <th>...</th>\n",
       "      <th>id_31</th>\n",
       "      <th>id_32</th>\n",
       "      <th>id_33</th>\n",
       "      <th>id_34</th>\n",
       "      <th>id_35</th>\n",
       "      <th>id_36</th>\n",
       "      <th>id_37</th>\n",
       "      <th>id_38</th>\n",
       "      <th>DeviceType</th>\n",
       "      <th>DeviceInfo</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>3663549</td>\n",
       "      <td>18403224</td>\n",
       "      <td>31.95</td>\n",
       "      <td>W</td>\n",
       "      <td>10409</td>\n",
       "      <td>111.0</td>\n",
       "      <td>150.0</td>\n",
       "      <td>visa</td>\n",
       "      <td>226.0</td>\n",
       "      <td>debit</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>3663550</td>\n",
       "      <td>18403263</td>\n",
       "      <td>49.00</td>\n",
       "      <td>W</td>\n",
       "      <td>4272</td>\n",
       "      <td>111.0</td>\n",
       "      <td>150.0</td>\n",
       "      <td>visa</td>\n",
       "      <td>226.0</td>\n",
       "      <td>debit</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>3663551</td>\n",
       "      <td>18403310</td>\n",
       "      <td>171.00</td>\n",
       "      <td>W</td>\n",
       "      <td>4476</td>\n",
       "      <td>574.0</td>\n",
       "      <td>150.0</td>\n",
       "      <td>visa</td>\n",
       "      <td>226.0</td>\n",
       "      <td>debit</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>3663552</td>\n",
       "      <td>18403310</td>\n",
       "      <td>284.95</td>\n",
       "      <td>W</td>\n",
       "      <td>10989</td>\n",
       "      <td>360.0</td>\n",
       "      <td>150.0</td>\n",
       "      <td>visa</td>\n",
       "      <td>166.0</td>\n",
       "      <td>debit</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>3663553</td>\n",
       "      <td>18403317</td>\n",
       "      <td>67.95</td>\n",
       "      <td>W</td>\n",
       "      <td>18018</td>\n",
       "      <td>452.0</td>\n",
       "      <td>150.0</td>\n",
       "      <td>mastercard</td>\n",
       "      <td>117.0</td>\n",
       "      <td>debit</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 433 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   TransactionID  TransactionDT  TransactionAmt ProductCD  card1  card2  \\\n",
       "0        3663549       18403224           31.95         W  10409  111.0   \n",
       "1        3663550       18403263           49.00         W   4272  111.0   \n",
       "2        3663551       18403310          171.00         W   4476  574.0   \n",
       "3        3663552       18403310          284.95         W  10989  360.0   \n",
       "4        3663553       18403317           67.95         W  18018  452.0   \n",
       "\n",
       "   card3       card4  card5  card6  ...  id_31  id_32  id_33  id_34 id_35  \\\n",
       "0  150.0        visa  226.0  debit  ...    NaN    NaN    NaN    NaN   NaN   \n",
       "1  150.0        visa  226.0  debit  ...    NaN    NaN    NaN    NaN   NaN   \n",
       "2  150.0        visa  226.0  debit  ...    NaN    NaN    NaN    NaN   NaN   \n",
       "3  150.0        visa  166.0  debit  ...    NaN    NaN    NaN    NaN   NaN   \n",
       "4  150.0  mastercard  117.0  debit  ...    NaN    NaN    NaN    NaN   NaN   \n",
       "\n",
       "  id_36  id_37  id_38  DeviceType  DeviceInfo  \n",
       "0   NaN    NaN    NaN         NaN         NaN  \n",
       "1   NaN    NaN    NaN         NaN         NaN  \n",
       "2   NaN    NaN    NaN         NaN         NaN  \n",
       "3   NaN    NaN    NaN         NaN         NaN  \n",
       "4   NaN    NaN    NaN         NaN         NaN  \n",
       "\n",
       "[5 rows x 433 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trans_test = pd.read_csv('/kaggle/input/ieee-fraud-detection/test_transaction.csv')\n",
    "id_test = pd.read_csv('/kaggle/input/ieee-fraud-detection/test_identity.csv')\n",
    "df_test = trans_test.merge(id_test, how='left', left_on='TransactionID', right_on='TransactionID')\n",
    "del trans_test, id_test\n",
    "df_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def postprocess_df(X):\n",
    "    \n",
    "    df = X.copy()\n",
    "    \n",
    "    list_v = []\n",
    "\n",
    "    for i in [313]:#[313,314,315]:\n",
    "        df[f\"V{i}_new\"] = df[f\"V{i}\"]\n",
    "        df.loc[df[f\"V{i}_new\"]==0.0, f\"V{i}_new\"] = 9999999\n",
    "        list_v.append(f\"V{i}_new\")\n",
    "    df['first_trans_amt'] = np.round(1000*np.min(df[list_v], axis=1))/1000\n",
    "    df.loc[(df['first_trans_amt']==9999999.000) \n",
    "                       & (df['V306']==0)\n",
    "                       & (df['V307']==0)\n",
    "                       & (df['V308']==0)\n",
    "                       & (df['V309']==0)\n",
    "                       & (df['V310']==0)\n",
    "                       & (df['V311']==0)\n",
    "                       & (df['V312']==0)\n",
    "                       & (df['V313']==0)\n",
    "                       & (df['V314']==0)\n",
    "                       & (df['V315']==0)\n",
    "                       & (df['V316']==0)\n",
    "                       & (df['V317']==0)\n",
    "                       & (df['V318']==0)\n",
    "                       & (df['V319']==0)\n",
    "                       & (df['V320']==0)\n",
    "                       & (df['V321']==0),\n",
    "                       'first_trans_amt'] \\\n",
    "        = df.loc[(df['first_trans_amt']==9999999.000) \n",
    "                             & (df['V306']==0)\n",
    "                       & (df['V307']==0)\n",
    "                       & (df['V308']==0)\n",
    "                       & (df['V309']==0)\n",
    "                       & (df['V310']==0)\n",
    "                       & (df['V311']==0)\n",
    "                       & (df['V312']==0)\n",
    "                       & (df['V313']==0)\n",
    "                       & (df['V314']==0)\n",
    "                       & (df['V315']==0)\n",
    "                       & (df['V316']==0)\n",
    "                       & (df['V317']==0)\n",
    "                       & (df['V318']==0)\n",
    "                       & (df['V319']==0)\n",
    "                       & (df['V320']==0)\n",
    "                       & (df['V321']==0),\n",
    "                             'TransactionAmt']\n",
    "    df['D1'] = df['D1'].fillna(0)\n",
    "    df['D3'] = df['D3'].fillna(0)\n",
    "    df['day_first_trans']=(df['TransactionDT']/60/60/24-df['D1']).astype('int')\n",
    "    df = df.fillna(-999)\n",
    "    \n",
    "    list_cols = ['card1','card2','card3','card4','card5','card6', 'day_first_trans', 'first_trans_amt', \n",
    "              'addr1', \n",
    "             'id_01', \n",
    "             'DeviceInfo', \n",
    "             'P_emaildomain'\n",
    "            ]#, 'D3'\n",
    "    suff = '_no_D3'\n",
    "    x = df\\\n",
    "        .groupby(list_cols, as_index=False)\\\n",
    "        .agg({'TransactionID':'count', 'TransactionDT':[np.min, np.max, np.median]})\n",
    "    x.columns = ['_'.join(col).strip() if col[1] != '' else col[0] for col in x.columns.values]\n",
    "    x['num_group'] = x.index\n",
    "    x['TransactionDT_amax_amin'+suff] = x['TransactionDT_amax'] - x['TransactionDT_amin']\n",
    "    x['TransactionDT_amax_amed'+suff] = x['TransactionDT_amax'] - x['TransactionDT_median']\n",
    "    x['TransactionDT_amin_amed'+suff] = x['TransactionDT_median']- x['TransactionDT_amin']\n",
    "    x['TransactionDT_amed_diff'+suff] = (x['TransactionDT_amax_amed'+suff] - x['TransactionDT_amin_amed'+suff])/x['TransactionDT_amax_amin'+suff]\n",
    "    x['TransactionDT_amed_diff'+suff] = x['TransactionDT_amed_diff'+suff].fillna(0)\n",
    "    x['TransactionDT_mean_diff'+suff] = x['TransactionDT_amax_amin'+suff]/x['TransactionID_count']\n",
    "    x=x.drop(columns=['TransactionDT_amin', 'TransactionDT_amax', 'TransactionDT_median'],axis=1)\n",
    "    # x.head(100)\n",
    "    x = x.rename(index=str, columns={'TransactionID_count':'TransactionID_count'+suff})\n",
    "    df = df.merge(x, how='left', on=list_cols)\n",
    "    df['rank_grp'+suff] = df.groupby(list_cols)['TransactionID'].rank(ascending=True)\n",
    "\n",
    "    df['is_first_grp'+suff] = df['rank_grp'+suff] == 1\n",
    "    df['is_last_grp'+suff] = df['rank_grp'+suff] ==  df['TransactionID_count'+suff]\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(137321, 447)\n",
      "(92585, 447)\n",
      "(86021, 447)\n",
      "(101632, 447)\n",
      "(83655, 447)\n",
      "(89326, 447)\n"
     ]
    }
   ],
   "source": [
    "START_DATE = '2017-11-30'\n",
    "startdate = datetime.datetime.strptime(START_DATE, '%Y-%m-%d')\n",
    "TransactionDT1 = df_train['TransactionDT'].apply(lambda x: (startdate + datetime.timedelta(seconds = x)))\n",
    "dt_m = TransactionDT1.dt.date.astype('str').str[:7]\n",
    "train_folds_postprocessed = []\n",
    "for fold_n, month in enumerate(np.unique(dt_m)):\n",
    "    X_val = df_train[dt_m.isin([month])]\n",
    "    X_val = postprocess_df(X_val)\n",
    "    train_folds_postprocessed.append(X_val)\n",
    "    print(X_val.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_to_disk(train_folds_postprocessed, 'train_folds_postprocessed.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test = postprocess_df(df_test)\n",
    "save_to_disk(df_test, 'df_test_postprocessed.pkl')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}