5 Nonlinear regression.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Thinking in tensors, writing in PyTorch\n",
    "\n",
    "A hands-on course by [Piotr Migdał](https://p.migdal.pl) (2019).\n",
    "\n",
    "<a href=\"https://colab.research.google.com/github/stared/thinking-in-tensors-writing-in-pytorch/blob/master/5%20Nonlinear%20regression.ipynb\" target=\"_parent\">\n",
    "    <img src=\"https://colab.research.google.com/assets/colab-badge.svg\"/>\n",
    "</a>\n",
    "\n",
    "## Notebook 5: Non-linear regression\n",
    "\n",
    "Very **Work in Progress**\n",
    "\n",
    "![](https://upload.wikimedia.org/wikipedia/commons/thumb/d/d4/Correlation_examples2.svg/400px-Correlation_examples2.svg.png)\n",
    "\n",
    "\n",
    "### Exercise\n",
    "\n",
    "Which of the following can be described by linear regression:\n",
    "\n",
    "* without any modifications,\n",
    "* by after rescaling *x* or *y*,\n",
    "* cannot be described by linear regression?\n",
    "\n",
    "**TODO**\n",
    "\n",
    "* Prepare examples\n",
    "* 1d function with nonlinearities (by hand and automatically)\n",
    "* More advanced\n",
    "\n",
    "**Datasets to consider**\n",
    "\n",
    "* https://en.wikipedia.org/wiki/Flight_airspeed_record\n",
    "\n",
    "**TODO later**\n",
    "\n",
    "* livelossplot `plot_extrema` error\n",
    "* drawing a plot \n",
    "* consider using [hiddenlayer](https://github.com/waleedka/hiddenlayer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "from matplotlib import pyplot as plt\n",
    "\n",
    "import torch\n",
    "from torch import nn\n",
    "from torch import tensor\n",
    "from livelossplot import PlotLosses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = torch.linspace(-2., 2., 30).unsqueeze(1)\n",
    "Y = torch.cat([torch.zeros(10),  torch.linspace(0., 1., 10), 1. + torch.zeros(10)], dim=0)\n",
    "plt.plot(X.squeeze().numpy(), Y.numpy(), 'r.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "linear_model = nn.Linear(in_features=1, out_features=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(X, Y, model, loss_function, optim, num_epochs):\n",
    "    loss_history = []\n",
    "    \n",
    "    def extra_plot(*args):\n",
    "        plt.plot(X.squeeze(1).numpy(), Y.numpy(), 'r.', label=\"Ground truth\")\n",
    "        plt.plot(X.squeeze(1).numpy(), model(X).detach().numpy(), '-', label=\"Model\")\n",
    "        plt.title(\"Prediction\")\n",
    "        plt.legend(loc='lower right')\n",
    "    \n",
    "    liveloss = PlotLosses(extra_plots=[extra_plot], plot_extrema=False)\n",
    "\n",
    "    for epoch in range(num_epochs):\n",
    "        \n",
    "        epoch_loss = 0.0\n",
    "        \n",
    "        Y_pred = model(X)\n",
    "        loss = loss_function(Y_pred, Y)\n",
    "        \n",
    "        loss.backward()\n",
    "        optim.step()\n",
    "        optim.zero_grad()\n",
    "        \n",
    "        liveloss.update({\n",
    "            'loss': loss.data.item(),\n",
    "        })\n",
    "        liveloss.draw()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Linear model\n",
    "\n",
    "$$y = a x + b$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Linear(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.layer_weights = nn.Parameter(torch.randn(1, 1))\n",
    "        self.layer_bias = nn.Parameter(torch.randn(1))\n",
    "        \n",
    "    def forward(self, x):\n",
    "        return x.matmul(self.layer_weights).add(self.layer_bias).squeeze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "linear_model = Linear()\n",
    "optim = torch.optim.SGD(linear_model.parameters(), lr=0.03)\n",
    "loss_function = nn.MSELoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "list(linear_model.parameters())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "linear_model(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train(X, Y, linear_model, loss_function, optim, num_epochs=50)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Nonlinear\n",
    "\n",
    "$$ x \\mapsto h \\mapsto y$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Nonlinear(nn.Module):\n",
    "    def __init__(self, hidden_size=2):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.layer_1_weights = nn.Parameter(torch.randn(1, hidden_size))\n",
    "        self.layer_1_bias = nn.Parameter(torch.randn(hidden_size)) \n",
    "        \n",
    "        self.layer_2_weights = nn.Parameter(torch.randn(hidden_size, 1) ) \n",
    "        self.layer_2_bias = nn.Parameter(torch.randn(1))\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = x.matmul(self.layer_1_weights).add(self.layer_1_bias)\n",
    "        x = x.relu()\n",
    "        x = x.matmul(self.layer_2_weights).add(self.layer_2_bias)\n",
    "        return x.squeeze()\n",
    "    \n",
    "    def nonrandom_init(self):\n",
    "        self.layer_1_weights.data = tensor([[1.1, 0.8]])\n",
    "        self.layer_1_bias.data = tensor([0.5 , -0.7]) \n",
    "        self.layer_2_weights.data = tensor([[0.3], [-0.7]])\n",
    "        self.layer_2_bias.data = tensor([0.2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nonlinear_model = Nonlinear(hidden_size=2)\n",
    "nonlinear_model.nonrandom_init()\n",
    "\n",
    "optim = torch.optim.SGD(nonlinear_model.parameters(), lr=0.2)\n",
    "# optim = torch.optim.Adam(nonlinear_model.parameters(), lr=0.1)\n",
    "loss_function = nn.MSELoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train(X, Y, nonlinear_model, loss_function, optim, num_epochs=200)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Other shapes and activations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y_sin = (2 * X).sin()\n",
    "plt.plot(X.squeeze().numpy(), Y_sin.numpy(), 'r.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# warning: \n",
    "# for 1-d problems it rarely works (often gets stuck in some local minimum)\n",
    "nonlinear_model = Nonlinear(hidden_size=10)\n",
    "\n",
    "optim = torch.optim.Adam(nonlinear_model.parameters(), lr=0.01)\n",
    "loss_function = nn.MSELoss()\n",
    "train(X, Y_sin, nonlinear_model, loss_function, optim, num_epochs=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class NonlinearSigmoid2(nn.Module):\n",
    "    def __init__(self, hidden_size=2):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.layer_1_weights = nn.Parameter(torch.randn(1, hidden_size))\n",
    "        self.layer_1_bias = nn.Parameter(torch.randn(hidden_size))\n",
    "        \n",
    "        self.layer_2_weights = nn.Parameter(torch.randn(hidden_size, 1))\n",
    "        self.layer_2_bias = nn.Parameter(torch.randn(1))\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = x.matmul(self.layer_1_weights).add(self.layer_1_bias)\n",
    "        x = x.sigmoid()\n",
    "        x = x.matmul(self.layer_2_weights).add(self.layer_2_bias)\n",
    "        x = x.sigmoid()\n",
    "        return x.squeeze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X1 = torch.linspace(-2., 2., 30).unsqueeze(1)\n",
    "Y1 = torch.cat([torch.zeros(10), 1. + torch.zeros(10),  torch.zeros(10)], dim=0)\n",
    "plt.plot(X1.squeeze().numpy(), Y1.numpy(), 'r.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nonlinear_model = NonlinearSigmoid2(hidden_size=2)\n",
    "# optim = torch.optim.SGD(nonlinear_model.parameters(), lr=0.1)\n",
    "optim = torch.optim.Adam(nonlinear_model.parameters(), lr=0.1)\n",
    "loss_function = nn.MSELoss()\n",
    "train(X1, Y1, nonlinear_model, loss_function, optim, num_epochs=100)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Nonlinear model - by hand"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "my_nonlinear_model = Nonlinear(hidden_size=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "my_nonlinear_model.layer_1_weights.data = tensor([[1. , 1.]])\n",
    "my_nonlinear_model.layer_1_bias.data = tensor([1. , -1.])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X.matmul(my_nonlinear_model.layer_1_weights).add(my_nonlinear_model.layer_1_bias).relu()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "my_nonlinear_model.layer_2_weights.data = tensor([[0.5], [-0.5]])\n",
    "my_nonlinear_model.layer_2_bias.data = tensor([0.])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "my_nonlinear_model(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.plot(X.squeeze(1).numpy(), Y.numpy(), 'r.')\n",
    "plt.plot(X.squeeze(1).numpy(), my_nonlinear_model(X).detach().numpy(), '-')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}