04_biz_applications.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": false,
    "editable": false,
    "run_control": {
     "frozen": true
    }
   },
   "source": [
    "## Applying Neural Networks to Business Problems\n",
    "\n",
    "Now that you've seen how to train simple neural networks and avoid the problem of overfitting, let's try training a more complex neural network to predict business-relevant events. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": false,
    "editable": false,
    "run_control": {
     "frozen": true
    }
   },
   "source": [
    "First, let's set up some imports that we'll need later:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": false,
    "editable": false,
    "run_control": {
     "frozen": true
    }
   },
   "source": [
    "## Preparing our Input Data\n",
    "\n",
    "Let's import our training and test data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train_df = pd.read_csv(\"data/sales-train.csv\")\n",
    "test_df = pd.read_csv(\"data/sales-test.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": false,
    "editable": false,
    "run_control": {
     "frozen": true
    }
   },
   "source": [
    "We can examine each of these dataframes in turn:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>Income</th>\n",
       "      <th>Female</th>\n",
       "      <th>Purchased</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>43</td>\n",
       "      <td>50</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>59</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>41</td>\n",
       "      <td>55</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>32</td>\n",
       "      <td>302</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>39</td>\n",
       "      <td>66</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Age  Income  Female  Purchased\n",
       "0   43      50       0          0\n",
       "1   24      59       1          1\n",
       "2   41      55       1          0\n",
       "3   32     302       1          1\n",
       "4   39      66       0          1"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>Income</th>\n",
       "      <th>Female</th>\n",
       "      <th>Purchased</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>174</td>\n",
       "      <td>24</td>\n",
       "      <td>121</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>175</td>\n",
       "      <td>35</td>\n",
       "      <td>69</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>176</td>\n",
       "      <td>54</td>\n",
       "      <td>53</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>177</td>\n",
       "      <td>34</td>\n",
       "      <td>49</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>178</td>\n",
       "      <td>44</td>\n",
       "      <td>50</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Age  Income  Female  Purchased\n",
       "174   24     121       0          1\n",
       "175   35      69       1          1\n",
       "176   54      53       0          0\n",
       "177   34      49       0          0\n",
       "178   44      50       0          0"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": false,
    "editable": false,
    "run_control": {
     "frozen": true
    }
   },
   "source": [
    "As you can see, these datasets concern whether a given consumer has purchased from our business in the past. Each row represents a different consumer. We seek to predict values of the `Purchased` column (`1` for a consumer that did purchase and `0` for a consumer that declined) using the profile data in the other columns. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": false,
    "editable": false,
    "run_control": {
     "frozen": true
    }
   },
   "source": [
    "Let's pull out the `X` and `y` variables in our data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, y_train = (\n",
    "    train_df.drop('Purchased', axis=1).to_numpy(),\n",
    "    train_df.Purchased.to_numpy()\n",
    ")\n",
    "X_test, y_test = (\n",
    "    test_df.drop('Purchased', axis=1).to_numpy(),\n",
    "    test_df.Purchased.to_numpy()\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": false,
    "editable": false,
    "run_control": {
     "frozen": true
    }
   },
   "source": [
    "In order to train our network effectively, we'll need to *scale* our input data to be within an appropriate well-defined range. Let's choose the range `(0,1)`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import MinMaxScaler\n",
    "scaler = MinMaxScaler(feature_range=(0,1), copy=True)\n",
    "X_train_scaled = scaler.fit_transform(X_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": false,
    "editable": false,
    "run_control": {
     "frozen": true
    }
   },
   "source": [
    "Note that we have only used the training dataset to construct our `scaler` object - do not use the test dataset for this part, that would be cheating! No aspect of constructing our model is allowed to rely on our test dataset - if we break this rule, we will not be able to accurately estimate how well our model can generalize to unseen data."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": false,
    "editable": false,
    "run_control": {
     "frozen": true
    }
   },
   "source": [
    "## Building a Neural Network to Predict Purchases"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": false,
    "editable": false,
    "run_control": {
     "frozen": true
    }
   },
   "source": [
    "The network we will use will consist of one hidden layer compused of ReLU neurons, followed by an output layer that uses the softmax activation function to produce probability estimates for whether or not the consumer made a purchase. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": false,
    "editable": false,
    "run_control": {
     "frozen": true
    }
   },
   "source": [
    "Before we do anything, we'll need to run the following code to ensure that the randomness used in tensorflow is appropriately controlled so that your result can match ours:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from consistency import remove_randomness\n",
    "remove_randomness()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": false,
    "editable": false,
    "run_control": {
     "frozen": true
    }
   },
   "source": [
    "Let's initialize this network:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "\n",
    "model_nn = keras.Sequential([\n",
    "    keras.layers.Dense(\n",
    "        units=50,\n",
    "        input_shape=(X_train.shape[1],),\n",
    "        activation=\"relu\",\n",
    "        name=\"hidden\"\n",
    "    ),\n",
    "    keras.layers.Dense(\n",
    "        units=2,\n",
    "        activation=\"softmax\",\n",
    "        name=\"output\"\n",
    "    )\n",
    "])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": false,
    "editable": false,
    "run_control": {
     "frozen": true
    }
   },
   "source": [
    "We will now configure the model for training - since we're performing *classification* (labeling a customer as either a `1` or a `0`) rather than *regression* (predicting a number like \"revenue\" that can take on a wide range of values), we need to use a different loss function to guide our training process, `categorical_crossentropy`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_nn.compile(\n",
    "    loss=\"categorical_crossentropy\",\n",
    "    optimizer=\"sgd\",\n",
    "    metrics=[\"accuracy\"]\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": false,
    "editable": false,
    "run_control": {
     "frozen": true
    }
   },
   "source": [
    "Finally, we train the model - note that we need to call `to_categorical` to convert our `y_train` data into the appropriate format expected by TensorFlow for classification problems:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 535 samples\n",
      "Epoch 1/5\n",
      "535/535 [==============================] - 1s 948us/sample - loss: 0.6579 - accuracy: 0.7682\n",
      "Epoch 2/5\n",
      "535/535 [==============================] - 0s 42us/sample - loss: 0.6434 - accuracy: 0.7832\n",
      "Epoch 3/5\n",
      "535/535 [==============================] - 0s 147us/sample - loss: 0.6308 - accuracy: 0.7869\n",
      "Epoch 4/5\n",
      "535/535 [==============================] - 0s 155us/sample - loss: 0.6201 - accuracy: 0.7869\n",
      "Epoch 5/5\n",
      "535/535 [==============================] - 0s 38us/sample - loss: 0.6107 - accuracy: 0.7869\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7f31203ae5f8>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_nn.fit(\n",
    "    X_train_scaled,\n",
    "    keras.utils.to_categorical(y_train),\n",
    "    epochs=5\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": false,
    "editable": false,
    "run_control": {
     "frozen": true
    }
   },
   "source": [
    "Let's see how it performs on the test dataset:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "179/179 [==============================] - 0s 964us/sample - loss: 0.6211 - accuracy: 0.7542\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.6211078403382327, 0.75418997]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test_scaled = scaler.transform(X_test)\n",
    "model_nn.evaluate(\n",
    "    X_test_scaled, \n",
    "    keras.utils.to_categorical(y_test)\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": false,
    "editable": false,
    "run_control": {
     "frozen": true
    }
   },
   "source": [
    "The first number above is the loss function value, which we can ignore for now. The accuracy on the test set is the second number in the list above. We can see that the accuracy is slightly lower on the test set, but not by much."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": false,
    "editable": false,
    "run_control": {
     "frozen": true
    }
   },
   "source": [
    "## Assessing the Precision of Our Neural Network"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": false,
    "editable": false,
    "run_control": {
     "frozen": true
    }
   },
   "source": [
    "Suppose that you are asked to justify the time spent developing this model. How much will it increase the efficiency of your sales team to have leads pre-vetted using the model's predicitons? To answer this question, we need to consider the current state of our business - how many leads currently convert? We can see that our training set contains `218` purchasers and `317` who declined to purchase:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    317\n",
       "1    218\n",
       "Name: Purchased, dtype: int64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df['Purchased'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": false,
    "editable": false,
    "run_control": {
     "frozen": true
    }
   },
   "source": [
    "So with the current \"call everyone on the list\" approach, our sales team will have a hit rate of `218/(218 + 317) ~= 40%`. What if they use our model instead and only call the people who are predicted to be buyers? "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": false,
    "editable": false,
    "run_control": {
     "frozen": true
    }
   },
   "source": [
    "The number that we want to know is called the *precision* of our model: what fraction of its \"positive\" predictions will actually turn out to be positive? This will be the hit rate observed by our sales team. Let's take a look at the output of our model in order to understand how to calculate the precision:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_probs_test = model_nn.predict(X_test_scaled)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": false,
    "editable": false,
    "run_control": {
     "frozen": true
    }
   },
   "source": [
    "Notice that the model has output *pairs of probabilities* - the probability (estimated by the model) that the customer under examination is a `1` or a `0`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.6112734 , 0.38872662],\n",
       "       [0.5469387 , 0.4530613 ],\n",
       "       [0.4272427 , 0.5727573 ],\n",
       "       [0.4272792 , 0.5727208 ],\n",
       "       [0.6133764 , 0.38662362],\n",
       "       [0.5595109 , 0.44048908],\n",
       "       [0.43077347, 0.5692265 ],\n",
       "       [0.45188707, 0.54811287],\n",
       "       [0.42572072, 0.57427925],\n",
       "       [0.42131472, 0.5786853 ],\n",
       "       [0.42155805, 0.578442  ],\n",
       "       [0.43061522, 0.56938475],\n",
       "       [0.5971134 , 0.40288663],\n",
       "       [0.60667175, 0.39332825],\n",
       "       [0.5113386 , 0.4886614 ],\n",
       "       [0.5856206 , 0.41437948],\n",
       "       [0.58538824, 0.41461176],\n",
       "       [0.42045087, 0.5795492 ],\n",
       "       [0.5906378 , 0.40936226],\n",
       "       [0.45298192, 0.5470181 ],\n",
       "       [0.41914898, 0.580851  ],\n",
       "       [0.4473601 , 0.5526399 ],\n",
       "       [0.4122974 , 0.5877026 ],\n",
       "       [0.6129073 , 0.38709274],\n",
       "       [0.5830572 , 0.4169428 ],\n",
       "       [0.5755006 , 0.4244994 ],\n",
       "       [0.5730792 , 0.42692074],\n",
       "       [0.588386  , 0.411614  ],\n",
       "       [0.4142656 , 0.58573437],\n",
       "       [0.6067475 , 0.39325252],\n",
       "       [0.4189887 , 0.5810113 ],\n",
       "       [0.4149101 , 0.58508986],\n",
       "       [0.60458523, 0.39541477],\n",
       "       [0.6021221 , 0.3978779 ],\n",
       "       [0.6044012 , 0.39559883],\n",
       "       [0.43417054, 0.5658295 ],\n",
       "       [0.600424  , 0.39957604],\n",
       "       [0.6288724 , 0.3711276 ],\n",
       "       [0.63500184, 0.36499816],\n",
       "       [0.62374693, 0.37625304],\n",
       "       [0.56179774, 0.4382023 ],\n",
       "       [0.5804121 , 0.41958794],\n",
       "       [0.43965966, 0.56034034],\n",
       "       [0.6276061 , 0.37239388],\n",
       "       [0.6233865 , 0.3766135 ],\n",
       "       [0.4259575 , 0.5740425 ],\n",
       "       [0.4321197 , 0.5678803 ],\n",
       "       [0.42122355, 0.5787764 ],\n",
       "       [0.6083591 , 0.3916408 ],\n",
       "       [0.44649884, 0.5535012 ],\n",
       "       [0.6032922 , 0.39670777],\n",
       "       [0.42188743, 0.57811254],\n",
       "       [0.5909419 , 0.40905815],\n",
       "       [0.59338236, 0.4066176 ],\n",
       "       [0.5878699 , 0.41213012],\n",
       "       [0.45420286, 0.54579717],\n",
       "       [0.5930374 , 0.40696257],\n",
       "       [0.5472456 , 0.45275435],\n",
       "       [0.5878699 , 0.41213012],\n",
       "       [0.6378365 , 0.36216345],\n",
       "       [0.4552309 , 0.54476917],\n",
       "       [0.5878699 , 0.41213012],\n",
       "       [0.44265002, 0.55735   ],\n",
       "       [0.5755006 , 0.4244994 ],\n",
       "       [0.57811594, 0.42188406],\n",
       "       [0.6190527 , 0.3809473 ],\n",
       "       [0.5544488 , 0.44555122],\n",
       "       [0.42471525, 0.5752848 ],\n",
       "       [0.5974837 , 0.40251634],\n",
       "       [0.59965503, 0.40034494],\n",
       "       [0.5755006 , 0.4244994 ],\n",
       "       [0.4620128 , 0.53798723],\n",
       "       [0.5732246 , 0.42677546],\n",
       "       [0.5801695 , 0.41983053],\n",
       "       [0.61555433, 0.3844457 ],\n",
       "       [0.5804121 , 0.41958794],\n",
       "       [0.57811594, 0.42188406],\n",
       "       [0.5754195 , 0.42458048],\n",
       "       [0.4271927 , 0.5728074 ],\n",
       "       [0.56293374, 0.43706623],\n",
       "       [0.41489506, 0.5851049 ],\n",
       "       [0.5971372 , 0.4028628 ],\n",
       "       [0.43686616, 0.56313384],\n",
       "       [0.57811594, 0.42188406],\n",
       "       [0.5878699 , 0.41213012],\n",
       "       [0.41889644, 0.58110356],\n",
       "       [0.41449726, 0.58550274],\n",
       "       [0.6111964 , 0.38880363],\n",
       "       [0.5779178 , 0.42208222],\n",
       "       [0.5816598 , 0.41834024],\n",
       "       [0.61491317, 0.38508683],\n",
       "       [0.45662442, 0.54337555],\n",
       "       [0.4463353 , 0.5536647 ],\n",
       "       [0.5927328 , 0.40726718],\n",
       "       [0.4314069 , 0.5685931 ],\n",
       "       [0.4227808 , 0.5772191 ],\n",
       "       [0.42840955, 0.5715905 ],\n",
       "       [0.59353185, 0.40646818],\n",
       "       [0.59550065, 0.40449938],\n",
       "       [0.43810716, 0.56189287],\n",
       "       [0.42619434, 0.57380563],\n",
       "       [0.41919604, 0.580804  ],\n",
       "       [0.5951105 , 0.40488955],\n",
       "       [0.4115334 , 0.5884666 ],\n",
       "       [0.41441828, 0.5855817 ],\n",
       "       [0.6004134 , 0.39958665],\n",
       "       [0.5804121 , 0.41958794],\n",
       "       [0.6031736 , 0.3968264 ],\n",
       "       [0.6155576 , 0.3844424 ],\n",
       "       [0.43872648, 0.56127346],\n",
       "       [0.59299433, 0.40700564],\n",
       "       [0.5804121 , 0.41958794],\n",
       "       [0.56261647, 0.43738347],\n",
       "       [0.62499315, 0.37500685],\n",
       "       [0.5754195 , 0.42458048],\n",
       "       [0.4275877 , 0.5724124 ],\n",
       "       [0.59791946, 0.40208054],\n",
       "       [0.4139119 , 0.5860881 ],\n",
       "       [0.59283197, 0.40716803],\n",
       "       [0.6135031 , 0.38649693],\n",
       "       [0.42612398, 0.57387596],\n",
       "       [0.5661205 , 0.43387946],\n",
       "       [0.58829254, 0.41170746],\n",
       "       [0.5804121 , 0.41958794],\n",
       "       [0.43531722, 0.5646828 ],\n",
       "       [0.5878699 , 0.41213012],\n",
       "       [0.5804929 , 0.41950703],\n",
       "       [0.4166524 , 0.5833476 ],\n",
       "       [0.5927328 , 0.40726718],\n",
       "       [0.44271398, 0.5572861 ],\n",
       "       [0.5878699 , 0.41213012],\n",
       "       [0.5927328 , 0.40726718],\n",
       "       [0.5883237 , 0.41167632],\n",
       "       [0.4303778 , 0.5696222 ],\n",
       "       [0.54258233, 0.45741773],\n",
       "       [0.58290225, 0.41709772],\n",
       "       [0.59998626, 0.40001372],\n",
       "       [0.60447395, 0.3955261 ],\n",
       "       [0.5623439 , 0.43765613],\n",
       "       [0.5438511 , 0.4561489 ],\n",
       "       [0.6021221 , 0.3978779 ],\n",
       "       [0.41908097, 0.58091897],\n",
       "       [0.41919604, 0.580804  ],\n",
       "       [0.42969403, 0.570306  ],\n",
       "       [0.5881991 , 0.41180095],\n",
       "       [0.5956506 , 0.40434948],\n",
       "       [0.5998387 , 0.40016136],\n",
       "       [0.5881679 , 0.41183206],\n",
       "       [0.6086912 , 0.39130878],\n",
       "       [0.58290225, 0.41709772],\n",
       "       [0.61075693, 0.38924304],\n",
       "       [0.4287449 , 0.571255  ],\n",
       "       [0.57533836, 0.4246616 ],\n",
       "       [0.42002398, 0.5799761 ],\n",
       "       [0.5730792 , 0.42692074],\n",
       "       [0.5927328 , 0.40726718],\n",
       "       [0.41622016, 0.5837798 ],\n",
       "       [0.5934816 , 0.40651846],\n",
       "       [0.41397706, 0.586023  ],\n",
       "       [0.61996484, 0.3800351 ],\n",
       "       [0.58538824, 0.41461176],\n",
       "       [0.5927328 , 0.40726718],\n",
       "       [0.58578724, 0.41421273],\n",
       "       [0.44262424, 0.5573758 ],\n",
       "       [0.60012865, 0.39987135],\n",
       "       [0.41673887, 0.5832612 ],\n",
       "       [0.4401094 , 0.5598906 ],\n",
       "       [0.5974837 , 0.40251634],\n",
       "       [0.42558816, 0.5744118 ],\n",
       "       [0.5830572 , 0.4169428 ],\n",
       "       [0.63111305, 0.368887  ],\n",
       "       [0.41919604, 0.580804  ],\n",
       "       [0.4171712 , 0.58282876],\n",
       "       [0.4219829 , 0.57801706],\n",
       "       [0.55287796, 0.44712204],\n",
       "       [0.42205682, 0.57794327],\n",
       "       [0.6256113 , 0.37438866],\n",
       "       [0.5805688 , 0.41943127],\n",
       "       [0.60457027, 0.3954297 ]], dtype=float32)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_probs_test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": false,
    "editable": false,
    "run_control": {
     "frozen": true
    }
   },
   "source": [
    "The *precision* of our model will depend on the cutoff value we choose for these probabilities. If we choose a very high cutoff value (say, `0.9`) and label all leads below the cutoff as \"not-likely buyers\" and all customers above the cutoff as \"likely buyers,\" the precision of our model will be very high - almost everyone who it labels a \"likely buyer\" will actually buy. However, the *recall* of the model will be low - it will only identify a small number of the customers who actually would have made a purchase. Our sales team will be efficient, but their overall productivity will be low because they will only have a few leads to call. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": false,
    "editable": false,
    "run_control": {
     "frozen": true
    }
   },
   "source": [
    "This problem, then, is similar to the tradeoffs we discussed in our earlier section on the merits of ROC curves. We need to analyze a range of cutoff (threshold) values and assess the tradeoffs in terms of precision and recall that each one entails. We will therefore construct a 'precision-recall curve' using the code below:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3de5gcdZ3v8fcnkwlMMBDYxLMySQhCAMEIkRHi4lkVUG5uyCMqZEXFB+FZL7uuSnbDkQeRxUNcVlwvrIqKiKxcxZx4iAdXgeOKBJkYQwTFEwMkGXAJ91vAXL7nj6qJnU5fqme6uqe7Pq/nmSfdVdW/+lXPpL71uysiMDOz4hrX7gyYmVl7ORCYmRWcA4GZWcE5EJiZFZwDgZlZwTkQmJkVnAOBdQVJMyWFpPHtzkteyq9R0g8lvW8E6cyQ9Jyknubn0jpR1/6nMet2EXFCluMkPQh8ICJ+nH5uHfCyHLNmHcYlAusY3fQEq4T//9mY4D9Ey5WkByWdK+k+SU9K+pakXdN9Z0j6WdnxIWn/9PWVkr4iaZmk54E3S+qT9DlJD0l6WtLPJPWVJPFuSeskPSbpkyXpHiHpTklPSXpE0pclTUj3SdLnJT0q6RlJqyW9Ot23i6R/SdP8L0lfLTtfad7PkHRHmvbTkn4r6ZiS/bdL+oykO4AXgFdK2kPSN9M8DUm6aDjgSepJz/2YpLXASWXnu13SB0renyXpN5KeTb/v10r6DjAD+EFaHfQPFaqY9pa0VNITktZIOqskzQskXS/pqjTdeyUNZP4DsI7gQGCt8G7gOGA/4ADgvAY++9fAZ4BJwM+AfwEOB/4C2Av4B2BbyfFvAA4EjgHOl/SqdPtW4GPAFOD16f4PpfveCvxlmrc9gHcBj6f7FqfbDwP2B/qB82vk90jg9+l5PgXcJGmvkv3vAc5Or+ch4EpgS5r2nDQvwzf3s4C3pdsHgHdUO6mkdwIXAO8FdgfmAY9HxHuAdcBfRcTLIuKfK3z8WmADsHd6jv8p6eiS/fPSYyYDS4Ev17h+60QR4R//5PYDPAj8Tcn7E4Hfp6/PAH5WdnwA+6evrwSuKtk3DtgEHFrhPDPTz04r2fYL4LQq+fp74Pvp66OB3wFzgXElxwh4HtivZNvrgQeqpHkG8DCgsjy8J319O3Bhyb7/BrwE9JVsWwDclr6+tey7e2t6jeNL0vtA+voW4KM1fgfHVviuxgPTSYLkpJL9FwNXpq8vAH5csu9gYFO7/67809wfNxZbK6wvef0QyZPnSD47BdiV5Im7mj+UvH6BtFFU0gHApSRP1hNJboIrACLiVklfBi4D9pF0E3BOeq6JwApJw2kKqNVWMRTpHTNVfr2l17MP0As8UpL+uJJj9mbn766a6dT+XqrZG3giIp4tO09p9U/5d7qrpPERsWUE57MxyFVD1grTS17PIHlqhuRpe+LwDkl/XuGzpTfVx4AXSaqYGvUV4LfArIjYHfgfJDf15CQRX4yIw0meeA8AFqbn2wQcEhGT0589IqJWj5t+ldzV2fF6y69nPUmJYEpJ+rtHxCHp/kfY+burZj3Vv5daUww/DOwlaVLZeYZqfMa6jAOBtcKHJU1L68o/CVyXbl8FHCLpsLQB+YJaiUTENuAK4NK0gbNH0usl7ZIhD5OAZ4DnJB0EfHB4h6TXSTpSUi9JcHoR2Jae7+vA5yW9PD22X9JxNc7zcuDvJPWm9favApZVuZ5HgB8Bn5O0u6RxkvaT9Mb0kOvTtKZJ2hNYVOO83wDOkXR42vi9v6R90n3/BbyySh7WAz8HLpa0q6TXAGcCV9c4l3UZBwJrhe+S3PDWklRfXAQQEb8DLgR+DPw/ksbges4BVgN3A08AnyXb3/E5JA3Pz5Lc3K8r2bd7uu1JkmqRx4FL0n3/CKwBlkt6Js3rgTXOcxcwi6Q08RngHRHxeI3j3wtMAO5Lz38j8Ip039dJ6v5XAb8EbqqWSETckJ7vu+k1LiFpTIekzv+8tMfUORU+voCk3eBh4PvApyIdc2DFoB2rM82aS2WDmbqZpDNIrvUN7c6LWSNcIjAzKzgHAjOzgnPVkJlZwblEYGZWcB03oGzKlCkxc+bMdmfDzKyjrFix4rGImFppX8cFgpkzZzI4ONjubJiZdRRJVUemu2rIzKzgHAjMzArOgcDMrOAcCMzMCs6BwMys4BwIzMwKzoHAzKzgHAjMzAout0Ag6QpJj0r6dZX9kvRFSWsk3SPptXnlpdSSlUMctfhW9l10M0ctvpUlK70Qk5kVW54lgiuB42vsP4FkAY9ZwNkkSwnmasnKIc69aTVDT20igKGnNnHuTasdDMys0HILBBHxU5IVpKo5GbgqEsuByZJeUeP4UbvklvvZtHnrDts2bd7KJbfcn+dpzczGtHa2EfSTLLg9bEO6bSeSzpY0KGlw48aNIz7hw09tami7mVkRdERjcURcHhEDETEwdWrFyfMy2XtyX0PbzcyKoJ2BYAiYXvJ+WrotNwuPO5C+3p4dtvX19rDwuFprkZuZdbd2BoKlwHvT3kNzgacj4pE8Tzh/Tj8Xv302/ZP7ENA/uY+L3z6b+XMq1kiZmRVCbusRSLoGeBMwRdIG4FNAL0BEfBVYBpwIrAFeAN6fV15KzZ/T7xu/mVmJ3AJBRCyosz+AD+d1fjMzy6YjGovNzCw/DgRmZgXnQGBmVnAOBGZmBedAYGZWcA4EZmYF50BgZlZwDgRmZgXnQGBmVnAOBGZmBedAYGZWcA4EZmYF50BgZlZwDgRmZgXnQGBmVnAOBGZmBedAYGZWcA4EZmYF50BgZlZwDgRmZgXnQGBmVnAOBGZmBedAYGZWcA4EZmYF50BgZlZwDgRmZgXnQGBmVnAOBGZmBedAYGZWcLkGAknHS7pf0hpJiyrsnyHpNkkrJd0j6cQ882NmZjvLLRBI6gEuA04ADgYWSDq47LDzgOsjYg5wGvBveeXHzMwqy7NEcASwJiLWRsQfgWuBk8uOCWD39PUewMM55sfMzCrIMxD0A+tL3m9It5W6ADhd0gZgGfC3lRKSdLakQUmDGzduzCOvZmaF1e7G4gXAlRExDTgR+I6knfIUEZdHxEBEDEydOrXlmTQz62Z5BoIhYHrJ+2nptlJnAtcDRMSdwK7AlBzzZGZmZfIMBHcDsyTtK2kCSWPw0rJj1gHHAEh6FUkgcN2PmVkL5RYIImIL8BHgFuA3JL2D7pV0oaR56WGfAM6StAq4BjgjIiKvPJmZ2c7G55l4RCwjaQQu3XZ+yev7gKPyzIOZmdXW7sZiMzNrMwcCM7OCcyAwMys4BwIzs4JzIDAzKzgHAjOzgnMgMDMruFzHEZhZ51iycohLbrmfh5/axN6T+1h43IHMn1M+T+To0nzzQVO57bcbm3oOGz0HArOCKr1JT57Yy3MvbmHztmRg/9BTmzj3ptUADd2o66V59fJ1248d6Tms+RwIzLpEvSf6WjfpJ1/YvFN6mzZv5ZJb7q95k240zZGcw/KnTpvaZ2BgIAYHB9udDbO2q3UTBujtEbtNGM/TmzZX3J/Vg4tPynzOkRK4qihnklZExEClfS4RmHWASnXt31sxxKbNW4HKT9+btwZPbdpcdX9W+y66OfM5RypIqooW3rgKcFVRq7lEYDYG1XvyFsnNs5Vadc49J/ay8vy3tuBMxeISgVkD8ug908g5s9S1t+PxbSTnLK2eyvr5ZpY0LBsHAiu8PHrPjOacrboRjuQm3Uia5UH0sE//aHtVlY0tDgRWaEtWDnHuTatr1ns3o2dLHjf+Rqtqat2kj1p8K0NPbWr4nLXSLHfBvENYeMOquo3Lk/t6s16SNYkDgRVO6U15nMTWDO1kD2e4SVY7Rx5P/H29PZxyeP/2wVn1eg3Vu0kvPO7AHQJilnM2Wm02fNzw9zJxQg/P/3Hn873t0FdkSs+ax4HAul6tm3KWIAAweWL1p9SR9OhpVJab+mjaNspv0nmNAp4/p397GnMu/FHFQHDzPY9w0fzZozqPNcaBwLpKXjfl0nhRr03h35evG3WdeyNP88NKb7IjMdrPN6ra78KNxa3nQGBdo7y+v1k3ZYCnNm1m30U359ajZyQ3frNmcSCwjlavvj/rTblHYltEzeOD5j2t+sZfvbFbrc6IORBY5yovAWSt7y/X19vDxW+fzfw5/cxcdHNT8jaa3jVFUe231VlDXLuDA4F1lJH0+GnkprznxN5RP/WPtndNUVT7rves0TBv+XAgsDFttD1+Gr0pj6RQ4af9kan2XXfYrDddwYHAxqwsg70qGa7vH8lN+ekMI19942+Oat91lt+BNZcDgY0pI6n6KVVa3z8Se0/uqzjCdjTBxSqr9l3vPbmvDbkpNgcCGzNG2vjbzJt0pRG2ow0uVtmbD5q6w4plpduttRwIrK3aXQIoV2mErUsA+bj5nkeqbvfI4tZyILC2GUkJoBX1860eYVtUtUYWH7X4VgfiFnIgsJYpn/7h+Ze21JzkbJjr54tnuO3Aq5a1Rq6BQNLxwBeAHuAbEbG4wjHvAi4g6eq9KiL+Os88WX5qTXpWafqHLFw/370m9/VmWp9g89bg0z+4138DOcptqUpJPcDvgLcAG4C7gQURcV/JMbOA64GjI+JJSS+PiEdrpeulKsem8hs97FiN00j9v0sAxbBk5VCm9QmGPbj4pJxz1N3atVTlEcCaiFibZuJa4GTgvpJjzgIui4gnAeoFARtb6jX0li6enjUIuARQHJUa5rOWFK25MgUCSbsApwAzSz8TERfW+Fg/sL7k/QbgyLJjDkjTv4Ok+uiCiPg/Fc5/NnA2wIwZM7Jk2XLWrHl+9pzYy8QJ490wWFDlDfPVlrMUsO+im/03kpOsJYL/BTwNrABeavL5ZwFvAqYBP5U0OyKeKj0oIi4HLoekaqiJ57caatX5X3LL/Zkaemvp6+3hU391iP9T23bVlrMcfpfXGtJFlzUQTIuI4xtMewiYXppGuq3UBuCuiNgMPCDpdySB4e4Gz2VNVqlxd+GNq7hg6b2jWuzc9f9WS3l1UaUqx2asIW07yhoIfp4+qa9uIO27gVmS9iUJAKcB5T2ClgALgG9JmkJSVbS2gXNYEzVS5z8Srv+3LEqri6pNC+62hObKGgjeAJwh6QGSqiEBERGvqfaBiNgi6SPALST1/1dExL2SLgQGI2Jpuu+tku4DtgILI+LxUVyPjVCz6vzLuQRgozFOUKlT0TivXtNUWQPBCSNJPCKWAcvKtp1f8jqAj6c/1kajqfMX7DRFNLgEYKNXrWdpxh6nllGmQBARD0k6FPjv6ab/jIhV+WXL8lbeEDzSonb/5D7uWHR0xTRdAjDrDFm7j36UpM//TemmqyVdHhFfyi1nVtNobrqVGoKrrR9bS19vDwuPO3D7e8/RY9aZslYNnQkcGRHPA0j6LHAn4EDQBpVu5I10qatUDRRUX0y8lOv8rZW8wH1rZA0EImnMHbYV/y7aptKNvF6XutISRK1Fw/sn91VcFhJc52+t5wXuWyNrIPgWcJek76fv5wPfzCdLVs/DVerzq22vNA9QJaX1/cOfc52/tVNPlTmqeuTn0GbK2lh8qaTbSbqRArw/IlbmlivbSZYFXKot8ZelR1B5fT+4zt/ar1o35mZ1b7ZEzUAgafeIeEbSXsCD6c/wvr0i4ol8s2eQrY9/zzjx/EtbKs7HUq2kAEn9np/2bazyOILWqFci+C7wNpI5hkp/HcNtOK/MKV9WIssT/dZtfxr1Wz4dRLUSRHlVkNlY43EErVEzEETE29J/921NdqySWk/01dSbArpSVZCZFdO4LAdJOkrSbunr0yVdKsnzQbdItbr/RvVIiKQk4N4/1un2XXQzRy2+lSUry+eytEZl7TX0FeDQdHTxJ4BvAN8B3phXxtqlvKfMmw+aym2/3dhQz5lm9LYpTWPyxF56xynzSk7VbIvgAa/yZB2k1nKWgdc0bpZMJQJgSzov0MnAlyPiMmBSftlqj+FG2aG0r/3QU5u4evm6Hd6fe9Pqmk8gldKo95l6aTz5wmZQ8p9CwG4TekZ0fc0qWZi1ygXzDqG3Tsvw5q3Bx67/lUsIo5A1EDwr6VzgdOBmSeOA3vyy1R5ZGmWHB241kka9z2RJY/PWYLddxvPA4pOYPHFC5rSGuU3AOtH8Of1c8s5D6Z/cV3MEa8SfSggfv/5XHPbpHzkwNCBr1dCpJGsJnBkRf0jbBy7JL1vtkbVRttZxjQ72GkkajaTl7qHW6bKsT1BqW7BDDzqvaFZf1gFlfwAuLXm/Drgqr0y1S9ZZOGtVsVRLo5FqmXppZM2nu4eaeUWzLGpWDUn6Wfrvs5KeKfl5VtIzrcli6yw87kD6emvXv9erYqmURqPVMvXSaEY+zYpkJF2wi6TeOII3pP92XcNwJeXrpY6k11ClNBqtlqmXRjPyadaJavUiqvm5iV3XpNlUWdcjmAvcGxHPpu8nAQdHxF15Zq4dxsr8OvXyMVbyadZKF8w7hIU3rGq4K/VLI1x9ryiy9hr6CvBcyfvn021WphndR82ssvJeRP2T+zhqv73qzkb6wuZtrclgh8q8HkE6jgCAiNgmKetnu169mUHdWGXWPLVKw1l6FdnOst7M10r6O/5UCvgQsDafLHWWLDODghurzMay85as5pq71rM1gh6JBUdO56L5s9udrZbJGgj+BvgicB7JuI2fAGfnlalOkmUQGnhUr1m7zVx0c8Wb/HlLVnP18nXb32+N4Orl6/jeig28uHlbITpfZB1H8ChwWs556UhZnvR7e6qvFWBmrTN8kwe2B4Pv3rWu4rGb0naFIsxnlHX20QMk/UTSr9P3r5F0Xr5Z6wzVnvSHZ/rcc2IvpCMd3XhsNjZcc9f67a+zdEDavDX49A/uzTFH7ZW119DXgXOBzQARcQ8FKSEsWTnEUYtvrTpvSaXBXb09YtKuSWHrmU1bdurq1ujcQ2bWXCNZ6vLJFxofv9ApsrYRTIyIX2jHLlpbcsjPmFLeEFxp3pLywV2TJ/by3Itbai4KA248NsvD8NKJ9dTrblo0WUsEj0naj/Q7lvQO4JHccjVGZJ1JdP6cfu5YdDQPLD6JiRPGZxrs4sZjs+Z799xs62UtOHL69teNhISZi25mv3OXcd6S1Q3mbGzLGgg+DHwNOEjSEPD3JD2JutpIZhLN2njseYDMmu+i+bM5fe6M7U/8PRKzXr7bDu9Pnztjh15DWYPHsOEG524KBnWrhtK1BwYi4th0ucpxw1NNdLuRzCSaaWZQL7xtlpuL5s9uaAzA8LHD4wiyunr5Ov59+bqKPQErrVIIY3d+MEWGC5c0GBEDDScuHQ98AegBvhERi6scdwpwI/C6iBislebAwEAMDtY8ZFTKl4h87sUdG3v7ens45fD+qr/AJSuHWHjjKjZvrf29eopos7FpJKOTh6uXAhinnXsi9YwT27ZFzWfA0jR6JOa+ck8efHxT0wKFpBXV7uNZG4t/LOkc4DqSeYaSDEc8UeOkPcBlwFuADcDdkpZGxH1lx00CPgq0fQK78sbhJ1/YTG+PmNzXy9ObNm+P4t9bMVSzATnLE78bi826R+l/+UpNhFsztBuWHrE1gjt+/6fba94L7GRtIziVZFqJ/wsMlvzUcgSwJiLWRsQfgWtJ1jwu90/AZ4EXM+YlN/WWiLxj0dHc9tuNNRuQL7nlfjcWm3Ww0xtsM2iVPLudZw0EB5M83a8CfgV8CTikzmf6gfUl7zek27aT9FpgekTULItJOlvSoKTBjRs3Zsxy47I0DjdjGUkvGmM2dpU3OI8ledUkZA0E3wZeRTLf0JdIAsO3R3PitBH6UuAT9Y6NiMsjYiAiBqZOnTqa09ZU7Sm9dHu9Y+qNNO6f3MfFb5/dtUPVzbrBRfNn8/uLT+TBxSe1Oys7yKsmIWsbwasj4uCS97dJuq/q0YkhYHrJ+2nptmGTgFcDt6cD1f4cWCppXr0G47wsPO7AHdoIYOd5gsrbCGDnZSTL0+jr7fHN36xDnT53xg6T0o1E1oFuteRZk5C1RPDLdJUyACQdSf02gruBWZL2lTSBZEqKpcM7I+LpiJgSETMjYiawHGhbEICkEebit8/evuhFpXmCvrdiiFMO799hYYzSm3x5Gi4BmHW2LGMTyt8ftd9eO9wDPn/qYTXTGFelFmpyX29L7iNZu4/+BjgQGA6LM4D7SaaZiIh4TZXPnQj8K0n30Ssi4jOSLgQGI2Jp2bG3A+e0u/toqaMW31pxTIC7fppZs1Uae9DMG38zuo8eP5ITR8QyYFnZtvOrHPumkZwjTyMZWWxmNhLtXIc863oED+WdkbFoJCOLzcw6TdY2gkKqNMW0u36aWbfxAvQ1lE8x3e75QMzM8uBAUEc76+3MzFqh8IGgXkt93i35ZmbtVuhAUG8FsiwrlJmZdbpCNxbXW4Es6wplZmadrNCBYKQTyHkcgZl1k0IHgpFOIOdxBGbWTQodCOqNE/A4AjMrgkI3FtcbJ+BxBGZWBJkmnRtLWjnpnJlZt6g16Vyhq4bMzMyBwMys8BwIzMwKzoHAzKzgHAjMzArOgcDMrOAcCMzMCs6BwMys4BwIzMwKzoHAzKzgHAjMzArOgcDMrOAcCMzMCs6BwMys4BwIzMwKzoHAzKzgHAjMzArOgcDMrOByDQSSjpd0v6Q1khZV2P9xSfdJukfSTyTtk2d+zMxsZ7kFAkk9wGXACcDBwAJJB5cdthIYiIjXADcC/5xXfszMrLI8SwRHAGsiYm1E/BG4Fji59ICIuC0iXkjfLgem5ZgfMzOrIM9A0A+sL3m/Id1WzZnADyvtkHS2pEFJgxs3bmxiFs3MbEw0Fks6HRgALqm0PyIuj4iBiBiYOnVqazNnZtblxueY9hAwveT9tHTbDiQdC3wSeGNEvJRjfszMrII8SwR3A7Mk7StpAnAasLT0AElzgK8B8yLi0RzzYmZmVeQWCCJiC/AR4BbgN8D1EXGvpAslzUsPuwR4GXCDpF9JWlolOTMzy0meVUNExDJgWdm280teH5vn+c3MrL4x0VhsZmbt40BgZlZwDgRmZgXnQGBmVnAOBGZmBedAYGZWcA4EZmYF50BgZlZwDgRmZgXnQGBmVnAOBGZmBedAYGZWcA4EZmYF50BgZlZwDgRmZgXnQGBmVnAOBGZmBedAYGZWcA4EZmYF50BgZlZwDgRmZgXnQGBmVnAOBGZmBedAYGZWcA4EZmYF50BgZlZwDgRmZgXnQGBmVnAOBGZmBTc+z8QlHQ98AegBvhERi8v27wJcBRwOPA6cGhEP5pmn85as5pq71rM1gh6JBUdO56L5s/M8pZnZmJZbiUBSD3AZcAJwMLBA0sFlh50JPBkR+wOfBz6bV34gCQJXL1/H1ggAtkZw9fJ1nLdkdZ6nNTMb0/KsGjoCWBMRayPij8C1wMllx5wMfDt9fSNwjCTllaFr7lrf0HYzsyLIMxD0A6V32A3ptorHRMQW4Gngz8oTknS2pEFJgxs3bhxxhoZLAlm3m5kVQUc0FkfE5RExEBEDU6dOHXE6PVUKG9W2m5kVQZ6BYAiYXvJ+Wrqt4jGSxgN7kDQa52LBkdMb2m5mVgR5BoK7gVmS9pU0ATgNWFp2zFLgfenrdwC3RuRXT3PR/NmcPnfG9hJAj8Tpc2e415CZFZpyvO8i6UTgX0m6j14REZ+RdCEwGBFLJe0KfAeYAzwBnBYRa2ulOTAwEIODg7nl2cysG0laEREDlfblOo4gIpYBy8q2nV/y+kXgnXnmwczMauuIxmIzM8uPA4GZWcE5EJiZFZwDgZlZwTkQmJkVnAOBmVnBORCYmRVcrgPK8iBpI/BQE5KaAjzWhHQ6ha+3exXpWsHXO1L7RETFydo6LhA0i6TBaqPsupGvt3sV6VrB15sHVw2ZmRWcA4GZWcEVORBc3u4MtJivt3sV6VrB19t0hW0jMDOzRJFLBGZmhgOBmVnhdX0gkHS8pPslrZG0qML+XSRdl+6/S9LM1ueyOTJc68cl3SfpHkk/kbRPO/LZLPWut+S4UySFpI7ucpjleiW9K/0d3yvpu63OYzNl+HueIek2SSvTv+kT25HPZpB0haRHJf26yn5J+mL6Xdwj6bVNzUBEdO0PycpovwdeCUwAVgEHlx3zIeCr6evTgOvane8cr/XNwMT09Qc79VqzXm963CTgp8ByYKDd+c759zsLWAnsmb5/ebvznfP1Xg58MH19MPBgu/M9iuv9S+C1wK+r7D8R+CEgYC5wVzPP3+0lgiOANRGxNiL+CFwLnFx2zMnAt9PXNwLHSOmixp2l7rVGxG0R8UL6djkwrcV5bKYsv1uAfwI+C7zYyszlIMv1ngVcFhFPAkTEoy3OYzNlud4Adk9f7wE83ML8NVVE/JRkud5qTgauisRyYLKkVzTr/N0eCPqB9SXvN6TbKh4TEVuAp4E/a0numivLtZY6k+QJo1PVvd60+Dw9Im5uZcZykuX3ewBwgKQ7JC2XdHzLctd8Wa73AuB0SRtIlsT929ZkrS0a/f/dkFzXLLaxSdLpwADwxnbnJS+SxgGXAme0OSutNJ6keuhNJKW9n0qaHRFPtTVX+VkAXBkRn5P0euA7kl4dEdvanbFO0+0lgiFgesn7aem2isdIGk9SxHy8JblrrizXiqRjgU8C8yLipRblLQ/1rncS8GrgdkkPktSrLu3gBuMsv98NwNKI2BwRDwC/IwkMnSjL9Z4JXA8QEXcCu5JM0NaNMv3/HqluDwR3A7Mk7StpAklj8NKyY5YC70tfvwO4NdLWmQ5T91olzQG+RhIEOrn+GOpcb0Q8HRFTImJmRMwkaROZFxGD7cnuqGX5W15CUhpA0hSSqqK1rcxkE2W53nXAMQCSXkUSCDa2NJetsxR4b9p7aC7wdEQ80qzEu7pqKCK2SPoIcAtJL4QrIuJeSRcCgxGxFPgmSZFyDUljzWnty/HIZbzWS4CXATek7eHrImJe2zI9Chmvt2tkvN5bgLdKug/YCiyMiE4s3Wa93k8AX5f0MZKG4zM69CEOSdeQBPEpaZvHp4BegIj4KkkbyInAGuAF4P1NPX+Hfm9mZtYk3V41ZGZmdTgQmJkVnAOBmVnBORCYmRWcA4GZWcE5EHzf0ekAAAFjSURBVJi1kKSZwzNMSnqTpP/d7jyZORCYZZAO5PH/F+tK/sM2qyJ9er9f0lXAr4H3SLpT0i8l3SDpZelxr5P0c0mrJP1C0qT0s/+ZHvtLSX/R3qsxq66rRxabNcEskilI1gA3AcdGxPOS/hH4uKTFwHXAqRFxt6TdgU3Ao8BbIuJFSbOAa0gm+jMbcxwIzGp7KCKWS3obyeInd6TTc0wA7gQOBB6JiLsBIuIZAEm7AV+WdBjJdA8HtCPzZlk4EJjV9nz6r4D/iIgFpTslza7yuY8B/wUcSlIF2+kL41gXcxuBWTbLgaMk7Q/JE7+kA4D7gVdIel26fVLJdOaPpHPjv4dk4jSzMcmBwCyDiNhIssjNNZLuIakWOihdRvFU4EuSVgH/QTId8r8B70u3HcSfShZmY45nHzUzKziXCMzMCs6BwMys4BwIzMwKzoHAzKzgHAjMzArOgcDMrOAcCMzMCu7/A4DTMLKE/XNkAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.metrics import precision_recall_curve\n",
    "\n",
    "precision, recall, thresh = precision_recall_curve(y_test, y_probs_test[:,1])\n",
    "plt.scatter(recall, precision)\n",
    "plt.xlabel('recall')\n",
    "plt.ylabel('precision')\n",
    "plt.title('purchase prediction')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": false,
    "editable": false,
    "run_control": {
     "frozen": true
    }
   },
   "source": [
    "Notice the following about this graph:\n",
    "\n",
    "* The `precision` value eventually converges to the fraction of `1` values in our dataset - if we classify everyone as a `1` and call every customer, this is the fraction that will end up buying. \n",
    "* The `recall` value converges to `1.0` - if we call everyone, we'll get all the potential buyers, but we'll spend more time on the phone. \n",
    "* There are a few scattered points with very high precision and very low recall. These correspond to high cutoff values, so only a few buyers are identified but our confidence in these buyers is very high. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": false,
    "editable": false,
    "run_control": {
     "frozen": true
    }
   },
   "source": [
    "Suppose we choose the following point:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "thresh: 0.4382023\n",
      "precision: 0.72\n",
      "recall: 0.75\n"
     ]
    }
   ],
   "source": [
    "point_idx = 80 \n",
    "print(\"thresh:\", thresh[point_idx])\n",
    "print(\"precision:\", precision[point_idx])\n",
    "print(\"recall:\", recall[point_idx])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": false,
    "editable": false,
    "run_control": {
     "frozen": true
    }
   },
   "source": [
    "If we apply this thresh cutoff value to our model's predictions, we can expect roughly three-quarters of our model's predicted purchasers to actually convert. This will be a much higher hit rate than simply calling everyone on this list - while calling everyone on the list is a better approach if our sales personnel costs are low, prioritizing effectively will increase our returns in many cases. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": false,
    "editable": false,
    "run_control": {
     "frozen": true
    }
   },
   "source": [
    "<span style=\"color:blue;font-weight:bold\">Exercise</span>: Let's see if there's any more accuracy we can squeeze out of our model. Perform the following tasks:\n",
    "\n",
    "* Construct a model `model_larger` that is identical to the `model_nn` we created above, with the exception of the number of units in the hidden layer, which should be set to `100` rather than `50`. \n",
    "* Your call to `model_larger.compile` to configure your model for training should use the same parameters as we used previously\n",
    "* Train this model on `(X_train_scaled, y_train)`, but run your training for `7` epochs this time \n",
    "* Evaluate your model on `(X_test_scaled, y_test)` - store your result in the variable `evaluation_result` \n",
    "\n",
    "Be careful not to delete the call to `remove_randomness()` that we have included below."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "tags": [
     "exercise"
    ]
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 535 samples\n",
      "Epoch 1/7\n",
      "535/535 [==============================] - 0s 776us/sample - loss: 0.7044 - accuracy: 0.5701\n",
      "Epoch 2/7\n",
      "535/535 [==============================] - 0s 156us/sample - loss: 0.6868 - accuracy: 0.5944\n",
      "Epoch 3/7\n",
      "535/535 [==============================] - 0s 42us/sample - loss: 0.6715 - accuracy: 0.6075\n",
      "Epoch 4/7\n",
      "535/535 [==============================] - 0s 149us/sample - loss: 0.6581 - accuracy: 0.6187\n",
      "Epoch 5/7\n",
      "535/535 [==============================] - 0s 151us/sample - loss: 0.6459 - accuracy: 0.6879\n",
      "Epoch 6/7\n",
      "535/535 [==============================] - 0s 42us/sample - loss: 0.6347 - accuracy: 0.7626\n",
      "Epoch 7/7\n",
      "535/535 [==============================] - 0s 154us/sample - loss: 0.6245 - accuracy: 0.7832\n",
      "179/179 [==============================] - 0s 2ms/sample - loss: 0.6278 - accuracy: 0.7654\n"
     ]
    }
   ],
   "source": [
    "remove_randomness()\n",
    "\n",
    "model_larger = tf.keras.Sequential([\n",
    "    keras.layers.Dense(\n",
    "        units=100,\n",
    "        input_shape=(X_train.shape[1],),\n",
    "        activation=\"relu\",\n",
    "        name=\"hidden\"\n",
    "    ),\n",
    "    keras.layers.Dense(\n",
    "        units=2,\n",
    "        activation=\"softmax\",\n",
    "        name=\"output\"\n",
    "    )\n",
    "])\n",
    "\n",
    "model_larger.compile(\n",
    "    loss=\"categorical_crossentropy\",\n",
    "    optimizer=\"sgd\",\n",
    "    metrics=[\"accuracy\"]\n",
    ")\n",
    "\n",
    "history = model_larger.fit(X_train_scaled, keras.utils.to_categorical(y_train), epochs=7)\n",
    "\n",
    "evaluation_result = model_larger.evaluate(X_test_scaled, keras.utils.to_categorical(y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "deletable": false,
    "editable": false,
    "scrolled": true,
    "strategicml": {
     "execution_time": 1584780375461,
     "exercise_id": "eaa22347-032c-4800-8379-c9d42513b542",
     "points": 4
    },
    "tags": [
     "test"
    ]
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"background-color:#e8fcef\n",
       ";color:#333333;padding:10px;border-style:solid;border-width:2px;border-color:#038c33\">\n",
       "<div style=\"font-weight:bold\">Correct!</div>\n",
       "</div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "check_variable_definition(\"model_larger\")\n",
    "assert str(type(model_larger)) ==  \"<class 'tensorflow.python.keras.engine.sequential.Sequential'>\", \"Your variable <code>my_model</code> does not contain an object of the right type - did you create it using <code>keras.Seqential()</code>?\"\n",
    "assert len(model_larger.layers) == 2, \"Your model does not contain the correct number of layers.\"\n",
    "assert model_larger.layers[0].name == 'hidden', \"Your hidden layer does not have the correct name value.\"\n",
    "assert model_larger.layers[1].name == 'output', \"Your output layer does not have the correct name value.\"\n",
    "assert model_larger.layers[0].units == 100, \"Your hidden layer does not contain the correct number of neurons.\"\n",
    "assert model_larger.layers[1].units == 2, \"Your output layer does not contain the correct number of neurons.\"\n",
    "assert abs(evaluation_result[1]  - 0.76) < 0.01, \"Your accuracy does not match the expected value - check your training process and your call to <code>evaluate</code>\"\n",
    "success()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": false,
    "editable": false,
    "run_control": {
     "frozen": true
    }
   },
   "source": [
    "Note that the larger model eventually gives a similar accuracy score to the smaller one, but takes slightly longer to train to a similar level of accuracy. More is not always better. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.6277970065617694, 0.76536316]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluation_result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deZQedZ3v8fcnnQ50INAwiXdIJyEsIYggIC2g8biBgqjAEUZAUfEgnHEbr0qccOUAw+AlTkZcBq4OKIPolVXMzVzwxlHgOKJBGiNEMsaJYUkaRsKSAEmATvK9f1R1fPLkWao7Tz1L1+d1Tp9+anmqvtX9PPWt+m2liMDMzIprXKsDMDOz1nIiMDMrOCcCM7OCcyIwMys4JwIzs4JzIjAzKzgnAhsTJM2UFJLGtzqWvJQfo6QfS/rIKLYzQ9KLkroaH6V1ojH7pTEb6yLiXVnWk/Qo8LGI+Gn6vseB3XMMzTqM7wisY4ylK1gl/P2ztuAPouVK0qOSLpS0XNJzkv5F0q7psnMk/aJs/ZB0YPr6eknflHSnpA3A2yT1SPqKpMckrZf0C0k9JZv4oKTHJT0t6Ysl2z1a0q8krZP0pKSrJE1Il0nSVyU9Jel5ScskHZou20XSP6bb/JOkb5XtrzT2cyTdm257vaTfSzquZPk9kr4k6V5gI7C/pD0lfSeNaVDS5cMJT1JXuu+nJa0C3l22v3skfaxk+jxJ/yHphfTv/TpJ3wNmAP+aFgd9oUIR01RJiyQ9K2mlpPNKtnmppFsk3ZBu92FJ/Zk/ANYRnAisGT4InAAcABwEXDSC934A+BIwCfgF8I/AUcAbgb2BLwBbS9Z/EzAbOA64WNKr0/lbgM8Ck4E3pMs/kS57J/DmNLY9gfcDz6TL5qfzjwAOBPqAi2vEewzwx3Q/lwC3S9q7ZPmHgPPT43kMuB7YnG77yDSW4ZP7ecB70vn9wOnVdirpr4BLgQ8DewAnA89ExIeAx4H3RsTuEfEPFd5+E7AGmJru439KenvJ8pPTdXqBRcBVNY7fOlFE+Mc/uf0AjwJ/XTJ9EvDH9PU5wC/K1g/gwPT19cANJcvGAZuAwyvsZ2b63mkl834NnFklrv8O/Ch9/XbgD8CxwLiSdQRsAA4omfcG4JEq2zwHeAJQWQwfSl/fA1xWsuy/AS8DPSXzzgLuTl/fVfa3e2d6jONLtvex9PVi4DM1/gfHV/hbjQemkyTJSSXLrwCuT19fCvy0ZNkhwKZWf67809gfVxZbM6wuef0YyZXnaN47GdiV5Iq7mv8qeb2RtFJU0kHAlSRX1hNJToIPAETEXZKuAq4G9pV0O3BBuq+JwAOShrcpoFZdxWCkZ8xU+fGWHs++QDfwZMn2x5WsM5Ud/3bVTKf236WaqcCzEfFC2X5Ki3/K/6a7ShofEZtHsT9rQy4asmaYXvJ6BslVMyRX2xOHF0j6ywrvLT2pPg28RFLENFLfBH4PzIqIPYD/QXJST3YS8Y2IOIrkivcgYG66v03AayKiN/3ZMyJqtbjpU8lZne2Pt/x4VpPcEUwu2f4eEfGadPmT7Pi3q2Y11f8utYYYfgLYW9Kksv0M1niPjTFOBNYMn5Q0LS0r/yJwczr/QeA1ko5IK5AvrbWRiNgKXAdcmVZwdkl6g6RdMsQwCXgeeFHSwcDHhxdIer2kYyR1kySnl4Ct6f6uBb4q6VXpun2STqixn1cBfyOpOy23fzVwZ5XjeRL4CfAVSXtIGifpAElvSVe5Jd3WNEl7AfNq7PfbwAWSjkorvw+UtG+67E/A/lViWA38ErhC0q6SXgucC3y/xr5sjHEisGb4AckJbxVJ8cXlABHxB+Ay4KfAf5JUBtdzAbAMuB94Fvgy2T7HF5BUPL9AcnK/uWTZHum850iKRZ4BFqTL/hZYCSyR9Hwa6+wa+7kPmEVyN/El4PSIeKbG+h8GJgDL0/3fBuyTLruWpOz/QeA3wO3VNhIRt6b7+0F6jAtJKtMhKfO/KG0xdUGFt59FUm/wBPAj4JJI+xxYMWj74kyzxlJZZ6axTNI5JMf6plbHYjYSviMwMys4JwIzs4Jz0ZCZWcH5jsDMrOA6rkPZ5MmTY+bMma0Ow8ysozzwwANPR8SUSss6LhHMnDmTgYGBVodhZtZRJFXtme6iITOzgnMiMDMrOCcCM7OCcyIwMys4JwIzs4JzIjAzKzgnAjOzgnMiMDMrOCcCM7OCcyIwMys4JwIzs4JzIjAzKzgnAjOzgnMiMDMrOCcCM7OCy+15BJKuA94DPBURh1ZYLuDrwEnARuCciPhNve0uG1zPnPl38baDp3D379fyxLpNTO3tYe4JswFYsHjFdvNOPbKvocdlZjbW5PbMYklvBl4EbqiSCE4CPk2SCI4Bvh4Rx9Tb7i77zIp9PvK1HeZ3dwkChrb++Xh6uru44n2HORmYWeFJeiAi+isty61oKCJ+DjxbY5VTSJJERMQSoFfSPqPd39CW2C4JAGwa2sKCxStGu0kzs0JoZR1BH7C6ZHpNOm8Hks6XNCBpYMvG9SPayRPrNo0+QjOzAuiIyuKIuCYi+iOiv2viniN679TenpyiMjMbG1qZCAaB6SXT09J5o9LdJbrHabt5Pd1d2yqRzcysslYmgkXAh5U4FlgfEU9meWNfbw9nHzuDvt4elE4vOP1wFvzV4dvNc0WxmVl9ebYauhF4KzAZ+BNwCdANEBHfSpuPXgWcSNJ89KMRMVBvu/39/TEwUHc1MzMrUavVUG79CCLirDrLA/hkXvs3M7NsOqKy2MzM8uNEYGZWcE4EZmYF50RgZlZwTgRmZgXnRGBmVnBOBGZmBedEYGZWcE4EZmYF50RgZlZwTgRmZgXnRGBmVnBOBGZmBZfb6KNj1cKlgyxYvIIn1m1iam8Pc0+Y7WcemFlHcyIYgYVLB7nw9mVsGtoCwOC6TVx4+zIAJwMz61guGhqBBYtXbEsCwzYNbWHB4hUtisjMbOf5jqCO0qKgas9ye2LdpqbGZGbWSE4ENZQXBVUztbenSRGZmTWei4ZqqFQUVK6nu4u5J8xuUkRmZo3nO4IS5S2CBmsU+QjcasjMxgQnglSlFkGCivUCfb093Dvv7U2Nz8wsLy4aSlUqBgqSK/9SLgoys7HGiSBVreVPkNwBKP19xfsOc1GQmY0pLhpKVasTcDGQmY11hU4EpZXDvRO76R4nhrb+uVbAxUBmVgSFTQTllcPPbRyiu0v09nSzftOQWwSZWWEUNhFUqhwe2hLstst4fnvJO1sUlZlZ8xU2EVSrHB7pcBEejdTMOl1hWw1VGxZiJMNFDBcvDabjEA2PRrpw6WCDojQzy19hE8HcE2bT09213byRVg57NFIzGwsKWzQ0XHyzM8U6jSpeMjNrpVwTgaQTga8DXcC3I2J+2fIZwHeB3nSdeRFxZ17xVCrPH2kfgdJtjJPYEjsOQuHRSM2sk+SWCCR1AVcD7wDWAPdLWhQRy0tWuwi4JSK+KekQ4E5gZh7xNOLpYuXbqJQE3PfAzDpNnnUERwMrI2JVRLwC3AScUrZOAHukr/cEnsgrmEaU51cblrpLqjkExcKlg8yZfxf7zbuDOfPvcmWymbWVPIuG+oDVJdNrgGPK1rkU+ImkTwO7AcdX2pCk84HzAWbMmDGqYBpRnl9t3a0RPDL/3RWX+TnHZtbuWt1q6Czg+oiYBpwEfE/SDjFFxDUR0R8R/VOmTBnVjhrRXHQ023DLIjNrd3kmgkFgesn0tHReqXOBWwAi4lfArsDkPIJpRHPR0WzDLYvMrN3lWTR0PzBL0n4kCeBM4ANl6zwOHAdcL+nVJIlgbR7BNKK5aJZtlLdM2rOnm3WbhnbYllsWmVm7UFRo+dKwjUsnAV8jaRp6XUR8SdJlwEBELEpbCl0L7E5ScfyFiPhJrW329/fHwMBAbjHvjEoPu+/uEgQ7jGrq5xqYWTNJeiAi+isty7UfQdon4M6yeReXvF4OzMkzhmaqNpDdXhO7mThhvMcjMrO2VNiexXmoVu6/buMQSy/2iKZm1p7GdCJoxsig7mlsZp1uzCaCZrTfd09jMxsLWt2PIDfNaL8/2p7GZmbtZMzeETSj/X7WnsZ+eI2ZtbMxmwim9vYwWOFE3cjy+iz78BAT1my+8LCRGrOJYO4Js3do09/o8vos+6hVROUvpzVC6Ym/d2I3L760eVu/lcF1m5h724Ncuuhh1m8aalhiqJRsYOc6bFrrjNlE0IiexI3Yh4eYsFpGc/Ve68T/3MYde7EPbYltvdtHe0eaJdmUdpz0nW9nybVncR7auWdxJXPm31Wx+Kivt2fED8WxzlfrhAo79jovTxRvO3gKP3xgsGIjhZGo9/mrF2ej9mPNU6tnsRNBzqoNO7HbhPENvVW39jPak3iXxNaIiidgkYzF0giPVmnQsDMn/nr7sdZxImixkV4FWnuqV4xT7//cyJN4IwgaepdRbz++4GktJ4I20slFRUVqjZLlar70zq7RV9HN1KwE5Que1mrZoHO2o06tPB7rzWDrVYb+7yWP73CyLK2ErVRJm5fyE/fOJqTRJIHy4s1KFzfl3FqufTkRNEGnjkdUL+5WfbGz3JnsTDFOpZN6o66Yd/bqu6e7i9OO6uPu36/NdGyNirtevdZ+8+7ItK8sCcOaz0VDOatUWVyuHSuPs8QNyYmt2vOaGxlLrbL38r9fuxbjlJ/Es8Sxs5+NakWR5WrdZWTZ78x5d2SKp0vij1eclGldaywXDbVQrfGISluG7Gw770bIcudSLo87mZ1tJ98uxThZTqaV6iJqXe2PVKVOj+Wy3GXU05exeCjLZ8qaz4kgZ/XGI5oz/64dTkytKHLJMpJqJW87eEqucYzmpJ3HqSZLkc5ort5PPbIv1/9zpU6PjU42kC3hQHIBZO1nTCWCdmzVUq0ibZxUs1y1GZXHo7kDKHfHQ09mLq/OsnzDy5tza8Y4EiMtbmqXz1sleSeb4X0AdesmfEfQnsZMImjXVi3VrpTqfSHyrjwe7R1Auec2Dm27ai//m9f7n1Ra3ipZTur9++7ddhca7aQ04Rx52U8q3s3tNbG72WFZBmOmsrid2+eP9Mo7j/bWla68h8vMaxmuyxjJHUO994xmm43WKVfzneqIv/tJxc9Xb083v73Ej21thUJUFrdz+/zSK6X96rSu6JI47aiR38rXKoIZ7ZV3aUK6aOEyvr/k8UzvGz65VzvJ11tervSkXS+51Epypev4xJ+v9VUuMqrNt9YaM4mgGc8faIR6nW+2RPDDBwbp33fvzCeqekUw1VouVVLtZHn379dmen8j7DWxm4kTxmdKalB5oLZ661i+OuX7aIkxkwia8fyBRsjSuiJLq6Esnb0+f8uDfPbm32ZuRVPrZNmsO6ue7i4uee9rqh57lqG/mzEEudU294TZzL3tQYa2bN/fo92+j5YYM4mgU778WVtX1DrxZq3orVf0UuvKu1zWYQRGY6RFNllawTSjpYzVUf7x66zqyELJVFksaRfgNGAmJckjIi7LLbIqOq1ncT2jqeTO2lu0lpEWlWTtaZx3HNYZ2rnxRlHVqiwel3Eb/wc4BdgMbCj5sZ0094TZ9HR3bTevXpHWzhTTiOTLONKT76lH9nHF+w6jr7dn2zZ6eyo3BeySEMkdR/e47TsQdXeJ3p7uUcdhnaHahcrguk3sN+8O5sy/i4VLB5sclVWTtWhoWkScmGskBXXqkX0MPPYsN963mi0RmVoNjbaYZmevxsqLW7JW3LZ7cZ01XleNpsFByeMtGRuj13a6rIngl5IOi4hluUZTQAuXDvLDBwa3a1JZr9VQ1u78pfKoOM9acesvevFkaRo8tCX4u3992J+PNpC1jmA5cCDwCPAy6dArEfHafMPbkesIElk6qbndvLXKARfembmfiJ9g1hyN6FD2rgbGYyVG2xGu9Erb7eat3Yykx7iLilovU2VxRDwG9ALvTX9603k1STpR0gpJKyXNq7LO+yUtl/SwpB+MJPixoFoHm5F0vKlUkeskYK3UN4qOY8NFRdZ8me4IJH0GOA+4PZ31fUnXRMQ/1XhPF3A18A5gDXC/pEURsbxknVnAhcCciHhO0qtGeRwdq1Ed4VwWb+1kNPVY0NxHftqfZW0+ei5wTERcHBEXA8eSJIZajgZWRsSqiHgFuImkCWqp84CrI+I5gIh4KnvoY8OpR/Zx2lF928ZpH+1YQ2btpNJd6tnHztg2be0lax2BgNLUviWdV0sfsLpkeg1wTNk6BwFIuhfoAi6NiP+3w86l84HzAWbMmJEx5M4wmlZDZp2g1l1qrUdb7jfvDlceN1nWO4J/Ae6TdKmkS4ElwHcasP/xwCzgrcBZwLWSestXiohrIqI/IvqnTGnsE7FardKAcMNjDZkV0XDl8YW3L3OnsybJWll8JfBR4Nn056MR8bU6bxsEppdMT0vnlVoDLIqIoYh4BPgDSWIojHYePtuslXxB1Dw1E4GkPdLfewOPAt9Pfx5L59VyPzBL0n6SJgBnAovK1llIcjeApMkkRUWrRnYIna0RrYbMxipfEDVHvTqCHwDvAR5g+7EDh5/lvX+1N0bEZkmfAhaTlP9fFxEPS7oMGIiIRemyd6Yd1rYAcyPimVEfTQfqlOGzzVqhd2I3c+bf5SFKcjZmHlXZyTwejxVNtUdZluruElu2Bltj+3kLTj/c349RqNWzOOsQE3OA30bEBklnA68DvhYR2Z5d2EBjMRGYFc3CpYPMvfVBhkrO8uMEe+zave050s9teJmNQ1t3eO9eE7tZerGfezxSjRhi4pvA4ZIOBz4PfBv4HvCWxoRoZkWSZcDCak1M3ems8bImgs0REZJOAa6KiO9IOjfPwMxsbHNv+PaRNRG8IOlC4GzgzZLGAZWfSmJmZh0la4eyM0iGnz43Iv6LpE/AgtyiMjOzpsl0R5Ce/K8smX4cuCGvoMzMrHnqdSj7Rfr7BUnPl/y8IOn55oRoZrY9P/e4sWreEUTEm9Lfk5oTjplZfX6YTWNlqiOQdKykSSXTkySVjyRqZtZUfphNY2StLP4m8GLJ9IZ0nplZS7lfwc7LmggUJV2QI2Ir2ZuemplZG8uaCFZJ+htJ3enPZyjYKKFm1ly9Pe6q1CxZE8FfA28keZ7A8JPGzs8rKDOzS09+Dd3j/GDLZsjaj+ApkucJmJk1Rfl4RJ01TnJnydpq6CBJP5P0u3T6tZIuyjc0Myu6U4/s4955b+eR+e9udShjWtaioWuBC4EhgIh4CN8hmFmbcAeznZM1EUyMiF+Xzdvc6GDMzKqpVXk83MHs87c+6GQwClkTwdOSDiB9XKWk04Enc4vKzKxMlsrjLVuDL/5oWZMiGjuy9gX4JHANcLCkQeAR4IO5RWVmViZr5fGGV7ZUWWLV1E0E6bMH+iPieEm7AeMi4oX8QzMz217pw2yqPcHMRq5u0VDai/gL6esNTgJmZmNL1jqCn0q6QNJ0SXsP/+QamZmZNUXWOoIzSCqKP1E2f//GhmNmlo2gYj3BSPsiL1w6uK3eYWpvD3NPmF24Ya2zJoJDSJLAm0j+9v8OfCuvoMzM6jnwVbvxn09tqDi/ltITf+/EbtZtHNqWUIaboEKxnnGQNRF8F3ge+EY6/YF03vvzCMrMrJ6VFZJApfnlJ/4XX9rM0Nbk1F9pCOvhJqhOBDs6NCIOKZm+W9LyPAIyM8uiWvPR0vkLlw5y4e3L2DSUNCnN+uyCojVBzVpZ/BtJxw5PpE8nG8gnJDOzxliweMW2JGDVZb0jOAr4paTH0+kZwApJy4CIiNfmEp2Z2U54Yt2mUb935rw76JI465jpXH7qYQ2Mqv1kTQQn5hqFmVkOpvb2MLgTyWBLBN9f8jj3rXqGVWs3siViTCaHrM8jeCzvQMzMRiJL89G5J8zero4AoLtL7DZhPOs3DWV+xkFp66Th5ABsSwblTVBn/kUPS1Y91zGJQyWPIu4I/f39MTDg6gmzorto4bJtJ+RSZx87Y7uTbq1+Ao0YpmJcmpG21lmvp3scLw1tbVlfBUkPRER/pWW5PoBe0onA14Eu4NsRMb/KeqcBtwGvjwif5c2sruGT/Y33ra555V06PlEetma8lt40lKSKwXWbuPD2ZITUdmmimtsdgaQu4A/AO0iec3w/cFZELC9bbxJwBzAB+FS9ROA7AjNrlGp3Fc3Q19vDvfPe3rT91bojyNp8dDSOBlZGxKqIeAW4CTilwnp/D3wZeCnHWMzMdnD5qYdx9rEz6NJIB6bYeTvToqnR8kwEfcDqkuk16bxtJL0OmB4RNQvqJJ0vaUDSwNq1axsfqZkV1uWnHsYfrziJR+e/m6+dcUSuJ8VSU3t7mrSn+pp1zDtIn3NwJfD5eutGxDUR0R8R/VOmTMk/ODMrpFOP7OPKM46gr7cHkRTfzDkg20DLPd3jMg9419PdxdwTZo86zkbLs7J4EJheMj0tnTdsEnAocI+S27K/BBZJOtkVxmbWKpUqly9auGy7Sun9p0ys2q+gWr1Dq1sN1ZJnZfF4ksri40gSwP3AByLi4Srr3wNc4MpiM+t05YmjHfoRtKT5aERslvQpYDFJ89HrIuJhSZcBAxGxKK99m5m10uWnHtbyE/9I5NqPICLuBO4sm3dxlXXfmmcsZmZWWcsqi83MrD04EZiZFZwTgZlZwTkRmJkVnBOBmVnBORGYmRWcE4GZWcE5EZiZFZwTgZlZwTkRmJkVnBOBmVnBORGYmRWcE4GZWcE5EZiZFZwTgZlZwTkRmJkVnBOBmVnBORGYmRWcE4GZWcE5EZiZFZwTgZlZwTkRmJkVnBOBmVnBORGYmRWcE4GZWcE5EZiZFZwTgZlZwTkRmJkVnBOBmVnBORGYmRVcrolA0omSVkhaKWleheWfk7Rc0kOSfiZp3zzjMTOzHeWWCCR1AVcD7wIOAc6SdEjZakuB/oh4LXAb8A95xWNmZpXleUdwNLAyIlZFxCvATcAppStExN0RsTGdXAJMyzEeMzOrIM9E0AesLplek86r5lzgx5UWSDpf0oCkgbVr1zYwRDMza4vKYklnA/3AgkrLI+KaiOiPiP4pU6Y0NzgzszFufI7bHgSml0xPS+dtR9LxwBeBt0TEyznGY2ZmFeR5R3A/MEvSfpImAGcCi0pXkHQk8M/AyRHxVI6xmJlZFbklgojYDHwKWAz8B3BLRDws6TJJJ6erLQB2B26V9FtJi6pszszMcpJn0RARcSdwZ9m8i0teH5/n/s3MrL62qCw2M7PWcSIwMys4JwIzs4JzIjAzKzgnAjOzgnMiMDMrOCcCM7OCcyIwMys4JwIzs4JzIjAzKzgnAjOzgnMiMDMrOCcCM7OCcyIwMys4JwIzs4JzIjAzKzgnAjOzgnMiMDMrOCcCM7OCcyIwMys4JwIzs4JzIjAzKzgnAjOzgnMiMDMrOCcCM7OCcyIwMys4JwIzs4JzIjAzKzgnAjOzgnMiMDMrOCcCM7OCyzURSDpR0gpJKyXNq7B8F0k3p8vvkzQzz3jMzGxHuSUCSV3A1cC7gEOAsyQdUrbaucBzEXEg8FXgy3nFY2ZmleV5R3A0sDIiVkXEK8BNwCll65wCfDd9fRtwnCTlGJOZmZUZn+O2+4DVJdNrgGOqrRMRmyWtB/4CeLp0JUnnA+enky9L+l0uEbevyZT9TQrAx1wMRTvmVh7vvtUW5JkIGiYirgGuAZA0EBH9LQ6pqXzMxeBjHvva9XjzLBoaBKaXTE9L51VcR9J4YE/gmRxjMjOzMnkmgvuBWZL2kzQBOBNYVLbOIuAj6evTgbsiInKMyczMyuRWNJSW+X8KWAx0AddFxMOSLgMGImIR8B3ge5JWAs+SJIt6rskr5jbmYy4GH/PY15bHK1+Am5kVm3sWm5kVnBOBmVnBtW0iKOLwFBmO+XOSlkt6SNLPJFVtF9wp6h1zyXqnSQpJbdf0biSyHK+k96f/54cl/aDZMTZahs/1DEl3S1qafrZPakWcjSTpOklPVevzpMQ30r/JQ5Je1+wYtxMRbfdDUrn8R2B/YALwIHBI2TqfAL6Vvj4TuLnVcTfhmN8GTExff7wIx5yuNwn4ObAE6G913Dn/j2cBS4G90ulXtTruJhzzNcDH09eHAI+2Ou4GHPebgdcBv6uy/CTgx4CAY4H7Whlvu94RFHF4irrHHBF3R8TGdHIJSd+MTpbl/wzw9yTjUL3UzOBykOV4zwOujojnACLiqSbH2GhZjjmAPdLXewJPNDG+XETEz0laQlZzCnBDJJYAvZL2aU50O2rXRFBpeIq+autExGZgeHiKTpXlmEudS3JF0cnqHnN6yzw9Iu5oZmA5yfI/Pgg4SNK9kpZIOrFp0eUjyzFfCpwtaQ1wJ/Dp5oTWUiP9vueqI4aYsO1JOhvoB97S6ljyJGkccCVwTotDaabxJMVDbyW54/u5pMMiYl1Lo8rXWcD1EfEVSW8g6Vt0aERsbXVgRdGudwRFHJ4iyzEj6Xjgi8DJEfFyk2LLS71jngQcCtwj6VGSstRFHVxhnOV/vAZYFBFDEfEI8AeSxNCpshzzucAtABHxK2BXksHZxrJM3/dmaddEUMThKeoes6QjgX8mSQKdXnYMdY45ItZHxOSImBkRM0nqRU6OiIHWhLvTsnyuF5LcDSBpMklR0apmBtlgWY75ceA4AEmvJkkEa5saZfMtAj6cth46FlgfEU+2Kpi2LBqK/IanaFsZj3kBsDtwa1ov/nhEnNyyoHdSxmMeMzIe72LgnZKWA1uAuRHRsXe6GY/588C1kj5LUnF8Todf1CHpRpKEPjmt+7gE6AaIiG+R1IWcBKwENgIfbU2kCQ8xYWZWcO1aNGRmZk3iRGBmVnBOBGZmBedEYGZWcE4EZmYF50Rg1kSSZg6PSCnprZL+b6tjMnMiMMsg7fjj74uNSf5gm1WRXr2vkHQD8DvgQ5J+Jek3km6VtHu63usl/VLSg5J+LWlS+t5/T9f9jaQ3tvZozKpry57FZm1kFslQJiuB24HjI2KDpL8FPidpPnAzcEZE3C9pD2AT8BTwjoh4SdIs4EaSgQLN2o4TgVltj0XEEknvIXloyr3p8B4TgF8Bs4EnI+J+gIh4HkDSbsBVko4gGSrioFYEb5aFE4FZbfAoeU8AAACnSURBVBvS3wL+LSLOKl0o6bAq7/ss8CfgcJIi2E5/qI6NYa4jMMtmCTBH0oGQXPFLOghYAewj6fXp/Eklw6I/mY6p/yGSAdfM2pITgVkGEbGW5AE5N0p6iKRY6OD08YtnAP8k6UHg30iGUf5fwEfSeQfz5zsLs7bj0UfNzArOdwRmZgXnRGBmVnBOBGZmBedEYGZWcE4EZmYF50RgZlZwTgRmZgX3/wGCF8GCPzF8uwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "y_probs_test = model_larger.predict(X_test_scaled)\n",
    "\n",
    "precision, recall, thresh = precision_recall_curve(y_test, y_probs_test[:,1])\n",
    "plt.scatter(recall, precision)\n",
    "plt.xlim(0,1.1)\n",
    "plt.ylim(0,1.1)\n",
    "plt.xlabel('recall')\n",
    "plt.ylabel('precision')\n",
    "plt.title('purchase prediction')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "celltoolbar": null,
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "strategicml": {
   "lesson_id": "812d7ddc-8afe-444a-b751-8892e323f047",
   "release": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}