From de6f9895e923ca1051c3e645316800784de083fb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aur=C3=A9lien=20Geron?= <ageron@users.noreply.github.com>
Date: Wed, 21 Feb 2018 23:04:09 +0100
Subject: [PATCH] Remove duplicate code in notebook for chapter 4, fixed #180

---
 04_training_linear_models.ipynb | 670 ++++++--------------------------
 1 file changed, 121 insertions(+), 549 deletions(-)

diff --git a/04_training_linear_models.ipynb b/04_training_linear_models.ipynb
index 4cf264e..1804968 100644
--- a/04_training_linear_models.ipynb
+++ b/04_training_linear_models.ipynb
@@ -2,40 +2,28 @@
  "cells": [
   {
    "cell_type": "markdown",
-   "metadata": {
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "source": [
     "**Chapter 4 – Training Linear Models**"
    ]
   },
   {
    "cell_type": "markdown",
-   "metadata": {
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "source": [
     "_This notebook contains all the sample code and solutions to the exercises in chapter 4._"
    ]
   },
   {
    "cell_type": "markdown",
-   "metadata": {
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "source": [
     "# Setup"
    ]
   },
   {
    "cell_type": "markdown",
-   "metadata": {
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "source": [
     "First, let's make sure this notebook works well in both python 2 and 3, import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures:"
    ]
@@ -43,11 +31,7 @@
   {
    "cell_type": "code",
    "execution_count": 1,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "# To support both python 2 and python 3\n",
@@ -82,10 +66,7 @@
   },
   {
    "cell_type": "markdown",
-   "metadata": {
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "source": [
     "# Linear regression using the Normal Equation"
    ]
@@ -93,11 +74,7 @@
   {
    "cell_type": "code",
    "execution_count": 2,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "import numpy as np\n",
@@ -109,11 +86,7 @@
   {
    "cell_type": "code",
    "execution_count": 3,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "plt.plot(X, y, \"b.\")\n",
@@ -127,11 +100,7 @@
   {
    "cell_type": "code",
    "execution_count": 4,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "X_b = np.c_[np.ones((100, 1)), X]  # add x0 = 1 to each instance\n",
@@ -141,11 +110,7 @@
   {
    "cell_type": "code",
    "execution_count": 5,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "theta_best"
@@ -154,11 +119,7 @@
   {
    "cell_type": "code",
    "execution_count": 6,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "X_new = np.array([[0], [2]])\n",
@@ -170,11 +131,7 @@
   {
    "cell_type": "code",
    "execution_count": 7,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "plt.plot(X_new, y_predict, \"r-\")\n",
@@ -185,10 +142,7 @@
   },
   {
    "cell_type": "markdown",
-   "metadata": {
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "source": [
     "The figure in the book actually corresponds to the following code, with a legend and axis labels:"
    ]
@@ -196,11 +150,7 @@
   {
    "cell_type": "code",
    "execution_count": 8,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "plt.plot(X_new, y_predict, \"r-\", linewidth=2, label=\"Predictions\")\n",
@@ -216,11 +166,7 @@
   {
    "cell_type": "code",
    "execution_count": 9,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "from sklearn.linear_model import LinearRegression\n",
@@ -232,11 +178,7 @@
   {
    "cell_type": "code",
    "execution_count": 10,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "lin_reg.predict(X_new)"
@@ -244,10 +186,7 @@
   },
   {
    "cell_type": "markdown",
-   "metadata": {
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "source": [
     "# Linear regression using batch gradient descent"
    ]
@@ -255,11 +194,7 @@
   {
    "cell_type": "code",
    "execution_count": 11,
-   "metadata": {
-    "collapsed": true,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "eta = 0.1\n",
@@ -275,11 +210,7 @@
   {
    "cell_type": "code",
    "execution_count": 12,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "theta"
@@ -288,11 +219,7 @@
   {
    "cell_type": "code",
    "execution_count": 13,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "X_new_b.dot(theta)"
@@ -301,11 +228,7 @@
   {
    "cell_type": "code",
    "execution_count": 14,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "theta_path_bgd = []\n",
@@ -325,29 +248,13 @@
     "            theta_path.append(theta)\n",
     "    plt.xlabel(\"$x_1$\", fontsize=18)\n",
     "    plt.axis([0, 2, 0, 15])\n",
-    "    plt.title(r\"$\\eta = {}$\".format(eta), fontsize=16)\n",
-    "\n",
-    "np.random.seed(42)\n",
-    "theta = np.random.randn(2,1)  # random initialization\n",
-    "\n",
-    "plt.figure(figsize=(10,4))\n",
-    "plt.subplot(131); plot_gradient_descent(theta, eta=0.02)\n",
-    "plt.ylabel(\"$y$\", rotation=0, fontsize=18)\n",
-    "plt.subplot(132); plot_gradient_descent(theta, eta=0.1, theta_path=theta_path_bgd)\n",
-    "plt.subplot(133); plot_gradient_descent(theta, eta=0.5)\n",
-    "\n",
-    "save_fig(\"gradient_descent_plot\")\n",
-    "plt.show()"
+    "    plt.title(r\"$\\eta = {}$\".format(eta), fontsize=16)"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 15,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "np.random.seed(42)\n",
@@ -365,10 +272,7 @@
   },
   {
    "cell_type": "markdown",
-   "metadata": {
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "source": [
     "# Stochastic Gradient Descent"
    ]
@@ -376,11 +280,7 @@
   {
    "cell_type": "code",
    "execution_count": 16,
-   "metadata": {
-    "collapsed": true,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "theta_path_sgd = []\n",
@@ -391,11 +291,7 @@
   {
    "cell_type": "code",
    "execution_count": 17,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "n_epochs = 50\n",
@@ -431,11 +327,7 @@
   {
    "cell_type": "code",
    "execution_count": 18,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "theta"
@@ -444,11 +336,7 @@
   {
    "cell_type": "code",
    "execution_count": 19,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "from sklearn.linear_model import SGDRegressor\n",
@@ -459,11 +347,7 @@
   {
    "cell_type": "code",
    "execution_count": 20,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "sgd_reg.intercept_, sgd_reg.coef_"
@@ -471,10 +355,7 @@
   },
   {
    "cell_type": "markdown",
-   "metadata": {
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "source": [
     "# Mini-batch gradient descent"
    ]
@@ -482,11 +363,7 @@
   {
    "cell_type": "code",
    "execution_count": 21,
-   "metadata": {
-    "collapsed": true,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "theta_path_mgd = []\n",
@@ -519,11 +396,7 @@
   {
    "cell_type": "code",
    "execution_count": 22,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "theta"
@@ -532,11 +405,7 @@
   {
    "cell_type": "code",
    "execution_count": 23,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "theta_path_bgd = np.array(theta_path_bgd)\n",
@@ -547,11 +416,7 @@
   {
    "cell_type": "code",
    "execution_count": 24,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "plt.figure(figsize=(7,4))\n",
@@ -568,10 +433,7 @@
   },
   {
    "cell_type": "markdown",
-   "metadata": {
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "source": [
     "# Polynomial regression"
    ]
@@ -580,9 +442,7 @@
    "cell_type": "code",
    "execution_count": 25,
    "metadata": {
-    "collapsed": true,
-    "deletable": true,
-    "editable": true
+    "collapsed": true
    },
    "outputs": [],
    "source": [
@@ -596,9 +456,7 @@
    "cell_type": "code",
    "execution_count": 26,
    "metadata": {
-    "collapsed": true,
-    "deletable": true,
-    "editable": true
+    "collapsed": true
    },
    "outputs": [],
    "source": [
@@ -610,11 +468,7 @@
   {
    "cell_type": "code",
    "execution_count": 27,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "plt.plot(X, y, \"b.\")\n",
@@ -628,11 +482,7 @@
   {
    "cell_type": "code",
    "execution_count": 28,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "from sklearn.preprocessing import PolynomialFeatures\n",
@@ -644,11 +494,7 @@
   {
    "cell_type": "code",
    "execution_count": 29,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "X_poly[0]"
@@ -657,11 +503,7 @@
   {
    "cell_type": "code",
    "execution_count": 30,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "lin_reg = LinearRegression()\n",
@@ -672,11 +514,7 @@
   {
    "cell_type": "code",
    "execution_count": 31,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "X_new=np.linspace(-3, 3, 100).reshape(100, 1)\n",
@@ -695,11 +533,7 @@
   {
    "cell_type": "code",
    "execution_count": 32,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "from sklearn.preprocessing import StandardScaler\n",
@@ -730,11 +564,7 @@
   {
    "cell_type": "code",
    "execution_count": 33,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "from sklearn.metrics import mean_squared_error\n",
@@ -760,11 +590,7 @@
   {
    "cell_type": "code",
    "execution_count": 34,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "lin_reg = LinearRegression()\n",
@@ -777,11 +603,7 @@
   {
    "cell_type": "code",
    "execution_count": 35,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "from sklearn.pipeline import Pipeline\n",
@@ -799,10 +621,7 @@
   },
   {
    "cell_type": "markdown",
-   "metadata": {
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "source": [
     "# Regularized models"
    ]
@@ -810,11 +629,7 @@
   {
    "cell_type": "code",
    "execution_count": 36,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "from sklearn.linear_model import Ridge\n",
@@ -857,11 +672,7 @@
   {
    "cell_type": "code",
    "execution_count": 37,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "from sklearn.linear_model import Ridge\n",
@@ -873,11 +684,7 @@
   {
    "cell_type": "code",
    "execution_count": 38,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "sgd_reg = SGDRegressor(max_iter=5, penalty=\"l2\", random_state=42)\n",
@@ -888,11 +695,7 @@
   {
    "cell_type": "code",
    "execution_count": 39,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "ridge_reg = Ridge(alpha=1, solver=\"sag\", random_state=42)\n",
@@ -903,11 +706,7 @@
   {
    "cell_type": "code",
    "execution_count": 40,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "from sklearn.linear_model import Lasso\n",
@@ -926,11 +725,7 @@
   {
    "cell_type": "code",
    "execution_count": 41,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "from sklearn.linear_model import Lasso\n",
@@ -942,11 +737,7 @@
   {
    "cell_type": "code",
    "execution_count": 42,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "from sklearn.linear_model import ElasticNet\n",
@@ -959,9 +750,6 @@
    "cell_type": "code",
    "execution_count": 43,
    "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true,
     "scrolled": true
    },
    "outputs": [],
@@ -1022,11 +810,7 @@
   {
    "cell_type": "code",
    "execution_count": 44,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "from sklearn.base import clone\n",
@@ -1049,11 +833,7 @@
   {
    "cell_type": "code",
    "execution_count": 45,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "best_epoch, best_model"
@@ -1062,11 +842,7 @@
   {
    "cell_type": "code",
    "execution_count": 46,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "%matplotlib inline\n",
@@ -1077,11 +853,7 @@
   {
    "cell_type": "code",
    "execution_count": 47,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "t1a, t1b, t2a, t2b = -1, 3, -1.5, 1.5\n",
@@ -1108,11 +880,7 @@
   {
    "cell_type": "code",
    "execution_count": 48,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "def bgd_path(theta, X, y, l1, l2, core = 1, eta = 0.1, n_iterations = 50):\n",
@@ -1175,10 +943,7 @@
   },
   {
    "cell_type": "markdown",
-   "metadata": {
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "source": [
     "# Logistic regression"
    ]
@@ -1186,11 +951,7 @@
   {
    "cell_type": "code",
    "execution_count": 49,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "t = np.linspace(-10, 10, 100)\n",
@@ -1211,11 +972,7 @@
   {
    "cell_type": "code",
    "execution_count": 50,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "from sklearn import datasets\n",
@@ -1226,11 +983,7 @@
   {
    "cell_type": "code",
    "execution_count": 51,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "print(iris.DESCR)"
@@ -1240,9 +993,7 @@
    "cell_type": "code",
    "execution_count": 52,
    "metadata": {
-    "collapsed": true,
-    "deletable": true,
-    "editable": true
+    "collapsed": true
    },
    "outputs": [],
    "source": [
@@ -1253,11 +1004,7 @@
   {
    "cell_type": "code",
    "execution_count": 53,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "from sklearn.linear_model import LogisticRegression\n",
@@ -1268,11 +1015,7 @@
   {
    "cell_type": "code",
    "execution_count": 54,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "X_new = np.linspace(0, 3, 1000).reshape(-1, 1)\n",
@@ -1284,10 +1027,7 @@
   },
   {
    "cell_type": "markdown",
-   "metadata": {
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "source": [
     "The figure in the book actually is actually a bit fancier:"
    ]
@@ -1295,11 +1035,7 @@
   {
    "cell_type": "code",
    "execution_count": 55,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "X_new = np.linspace(0, 3, 1000).reshape(-1, 1)\n",
@@ -1326,11 +1062,7 @@
   {
    "cell_type": "code",
    "execution_count": 56,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "decision_boundary"
@@ -1339,11 +1071,7 @@
   {
    "cell_type": "code",
    "execution_count": 57,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "log_reg.predict([[1.7], [1.5]])"
@@ -1352,11 +1080,7 @@
   {
    "cell_type": "code",
    "execution_count": 58,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "from sklearn.linear_model import LogisticRegression\n",
@@ -1400,11 +1124,7 @@
   {
    "cell_type": "code",
    "execution_count": 59,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "X = iris[\"data\"][:, (2, 3)]  # petal length, petal width\n",
@@ -1417,11 +1137,7 @@
   {
    "cell_type": "code",
    "execution_count": 60,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "x0, x1 = np.meshgrid(\n",
@@ -1459,11 +1175,7 @@
   {
    "cell_type": "code",
    "execution_count": 61,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "softmax_reg.predict([[5, 2]])"
@@ -1472,11 +1184,7 @@
   {
    "cell_type": "code",
    "execution_count": 62,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "softmax_reg.predict_proba([[5, 2]])"
@@ -1484,40 +1192,28 @@
   },
   {
    "cell_type": "markdown",
-   "metadata": {
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "source": [
     "# Exercise solutions"
    ]
   },
   {
    "cell_type": "markdown",
-   "metadata": {
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "source": [
     "## 1. to 11."
    ]
   },
   {
    "cell_type": "markdown",
-   "metadata": {
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "source": [
     "See appendix A."
    ]
   },
   {
    "cell_type": "markdown",
-   "metadata": {
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "source": [
     "## 12. Batch Gradient Descent with early stopping for Softmax Regression\n",
     "(without using Scikit-Learn)"
@@ -1525,10 +1221,7 @@
   },
   {
    "cell_type": "markdown",
-   "metadata": {
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "source": [
     "Let's start by loading the data. We will just reuse the Iris dataset we loaded earlier."
    ]
@@ -1537,9 +1230,7 @@
    "cell_type": "code",
    "execution_count": 63,
    "metadata": {
-    "collapsed": true,
-    "deletable": true,
-    "editable": true
+    "collapsed": true
    },
    "outputs": [],
    "source": [
@@ -1549,10 +1240,7 @@
   },
   {
    "cell_type": "markdown",
-   "metadata": {
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "source": [
     "We need to add the bias term for every instance ($x_0 = 1$):"
    ]
@@ -1561,9 +1249,7 @@
    "cell_type": "code",
    "execution_count": 64,
    "metadata": {
-    "collapsed": true,
-    "deletable": true,
-    "editable": true
+    "collapsed": true
    },
    "outputs": [],
    "source": [
@@ -1572,10 +1258,7 @@
   },
   {
    "cell_type": "markdown",
-   "metadata": {
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "source": [
     "And let's set the random seed so the output of this exercise solution is reproducible:"
    ]
@@ -1584,9 +1267,7 @@
    "cell_type": "code",
    "execution_count": 65,
    "metadata": {
-    "collapsed": true,
-    "deletable": true,
-    "editable": true
+    "collapsed": true
    },
    "outputs": [],
    "source": [
@@ -1595,10 +1276,7 @@
   },
   {
    "cell_type": "markdown",
-   "metadata": {
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "source": [
     "The easiest option to split the dataset into a training set, a validation set and a test set would be to use Scikit-Learn's `train_test_split()` function, but the point of this exercise is to try understand the algorithms by implementing them manually. So here is one possible implementation:"
    ]
@@ -1606,11 +1284,7 @@
   {
    "cell_type": "code",
    "execution_count": 66,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "test_ratio = 0.2\n",
@@ -1633,10 +1307,7 @@
   },
   {
    "cell_type": "markdown",
-   "metadata": {
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "source": [
     "The targets are currently class indices (0, 1 or 2), but we need target class probabilities to train the Softmax Regression model. Each instance will have target class probabilities equal to 0.0 for all classes except for the target class which will have a probability of 1.0 (in other words, the vector of class probabilities for ay given instance is a one-hot vector). Let's write a small function to convert the vector of class indices into a matrix containing a one-hot vector for each instance:"
    ]
@@ -1645,9 +1316,7 @@
    "cell_type": "code",
    "execution_count": 67,
    "metadata": {
-    "collapsed": true,
-    "deletable": true,
-    "editable": true
+    "collapsed": true
    },
    "outputs": [],
    "source": [
@@ -1661,10 +1330,7 @@
   },
   {
    "cell_type": "markdown",
-   "metadata": {
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "source": [
     "Let's test this function on the first 10 instances:"
    ]
@@ -1672,11 +1338,7 @@
   {
    "cell_type": "code",
    "execution_count": 68,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "y_train[:10]"
@@ -1685,11 +1347,7 @@
   {
    "cell_type": "code",
    "execution_count": 69,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "to_one_hot(y_train[:10])"
@@ -1697,10 +1355,7 @@
   },
   {
    "cell_type": "markdown",
-   "metadata": {
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "source": [
     "Looks good, so let's create the target class probabilities matrix for the training set and the test set:"
    ]
@@ -1709,9 +1364,7 @@
    "cell_type": "code",
    "execution_count": 70,
    "metadata": {
-    "collapsed": true,
-    "deletable": true,
-    "editable": true
+    "collapsed": true
    },
    "outputs": [],
    "source": [
@@ -1722,10 +1375,7 @@
   },
   {
    "cell_type": "markdown",
-   "metadata": {
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "source": [
     "Now let's implement the Softmax function. Recall that it is defined by the following equation:\n",
     "\n",
@@ -1736,9 +1386,7 @@
    "cell_type": "code",
    "execution_count": 71,
    "metadata": {
-    "collapsed": true,
-    "deletable": true,
-    "editable": true
+    "collapsed": true
    },
    "outputs": [],
    "source": [
@@ -1750,10 +1398,7 @@
   },
   {
    "cell_type": "markdown",
-   "metadata": {
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "source": [
     "We are almost ready to start training. Let's define the number of inputs and outputs:"
    ]
@@ -1762,9 +1407,7 @@
    "cell_type": "code",
    "execution_count": 72,
    "metadata": {
-    "collapsed": true,
-    "deletable": true,
-    "editable": true
+    "collapsed": true
    },
    "outputs": [],
    "source": [
@@ -1774,10 +1417,7 @@
   },
   {
    "cell_type": "markdown",
-   "metadata": {
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "source": [
     "Now here comes the hardest part: training! Theoretically, it's simple: it's just a matter of translating the math equations into Python code. But in practice, it can be quite tricky: in particular, it's easy to mix up the order of the terms, or the indices. You can even end up with code that looks like it's working but is actually not computing exactly the right thing. When unsure, you should write down the shape of each term in the equation and make sure the corresponding terms in your code match closely. It can also help to evaluate each term independently and print them out. The good news it that you won't have to do this everyday, since all this is well implemented by Scikit-Learn, but it will help you understand what's going on under the hood.\n",
     "\n",
@@ -1796,11 +1436,7 @@
   {
    "cell_type": "code",
    "execution_count": 73,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "eta = 0.01\n",
@@ -1823,10 +1459,7 @@
   },
   {
    "cell_type": "markdown",
-   "metadata": {
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "source": [
     "And that's it! The Softmax model is trained. Let's look at the model parameters:"
    ]
@@ -1834,11 +1467,7 @@
   {
    "cell_type": "code",
    "execution_count": 74,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "Theta"
@@ -1846,10 +1475,7 @@
   },
   {
    "cell_type": "markdown",
-   "metadata": {
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "source": [
     "Let's make predictions for the validation set and check the accuracy score:"
    ]
@@ -1857,11 +1483,7 @@
   {
    "cell_type": "code",
    "execution_count": 75,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "logits = X_valid.dot(Theta)\n",
@@ -1874,10 +1496,7 @@
   },
   {
    "cell_type": "markdown",
-   "metadata": {
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "source": [
     "Well, this model looks pretty good. For the sake of the exercise, let's add a bit of $\\ell_2$ regularization. The following training code is similar to the one above, but the loss now has an additional $\\ell_2$ penalty, and the gradients have the proper additional term (note that we don't regularize the first element of `Theta` since this corresponds to the bias term). Also, let's try increasing the learning rate `eta`."
    ]
@@ -1885,11 +1504,7 @@
   {
    "cell_type": "code",
    "execution_count": 76,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "eta = 0.1\n",
@@ -1915,10 +1530,7 @@
   },
   {
    "cell_type": "markdown",
-   "metadata": {
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "source": [
     "Because of the additional $\\ell_2$ penalty, the loss seems greater than earlier, but perhaps this model will perform better? Let's find out:"
    ]
@@ -1926,11 +1538,7 @@
   {
    "cell_type": "code",
    "execution_count": 77,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "logits = X_valid.dot(Theta)\n",
@@ -1943,20 +1551,14 @@
   },
   {
    "cell_type": "markdown",
-   "metadata": {
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "source": [
     "Cool, perfect accuracy! We probably just got lucky with this validation set, but still, it's pleasant."
    ]
   },
   {
    "cell_type": "markdown",
-   "metadata": {
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "source": [
     "Now let's add early stopping. For this we just need to measure the loss on the validation set at every iteration and stop when the error starts growing."
    ]
@@ -1964,11 +1566,7 @@
   {
    "cell_type": "code",
    "execution_count": 78,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "eta = 0.1 \n",
@@ -2008,11 +1606,7 @@
   {
    "cell_type": "code",
    "execution_count": 79,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "logits = X_valid.dot(Theta)\n",
@@ -2025,20 +1619,14 @@
   },
   {
    "cell_type": "markdown",
-   "metadata": {
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "source": [
     "Still perfect, but faster."
    ]
   },
   {
    "cell_type": "markdown",
-   "metadata": {
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "source": [
     "Now let's plot the model's predictions on the whole dataset:"
    ]
@@ -2046,11 +1634,7 @@
   {
    "cell_type": "code",
    "execution_count": 80,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "x0, x1 = np.meshgrid(\n",
@@ -2087,10 +1671,7 @@
   },
   {
    "cell_type": "markdown",
-   "metadata": {
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "source": [
     "And now let's measure the final model's accuracy on the test set:"
    ]
@@ -2098,11 +1679,7 @@
   {
    "cell_type": "code",
    "execution_count": 81,
-   "metadata": {
-    "collapsed": false,
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "logits = X_test.dot(Theta)\n",
@@ -2115,10 +1692,7 @@
   },
   {
    "cell_type": "markdown",
-   "metadata": {
-    "deletable": true,
-    "editable": true
-   },
+   "metadata": {},
    "source": [
     "Our perfect model turns out to have slight imperfections. This variability is likely due to the very small size of the dataset: depending on how you sample the training set, validation set and the test set, you can get quite different results. Try changing the random seed and running the code again a few times, you will see that the results will vary."
    ]
@@ -2127,9 +1701,7 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {
-    "collapsed": true,
-    "deletable": true,
-    "editable": true
+    "collapsed": true
    },
    "outputs": [],
    "source": []
@@ -2151,7 +1723,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.5.3"
+   "version": "3.6.4"
   },
   "nav_menu": {},
   "toc": {
@@ -2165,5 +1737,5 @@
   }
  },
  "nbformat": 4,
- "nbformat_minor": 0
+ "nbformat_minor": 1
 }