2019-01-16 16:37:12 +01:00
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"**Chapter 10 –
"\n",
"_This notebook contains all the sample code and solutions to the exercises in chapter 10._"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Setup"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
2019-01-21 11:13:10 +01:00
"First, let's import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures. We also check that Python 3.5 or later is installed (although Python 2.x may work, it is deprecated so we strongly recommend you use Python 3 instead), as well as Scikit-Learn ≥0.20 and TensorFlow ≥2.0-preview."
2019-01-16 16:37:12 +01:00
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"# Python ≥3.5 is required\n",
"import sys\n",
"assert sys.version_info >= (3, 5)\n",
"\n",
2019-01-21 11:13:10 +01:00
"# Scikit-Learn ≥0.20 is required\n",
"import sklearn\n",
"assert sklearn.__version__ >= \"0.20\"\n",
"\n",
"# TensorFlow ≥2.0-preview is required\n",
"import tensorflow as tf\n",
"assert hasattr(tf.compat, \"v1\")\n",
"\n",
2019-01-16 16:37:12 +01:00
"# Common imports\n",
"import numpy as np\n",
"import os\n",
"\n",
"# to make this notebook's output stable across runs\n",
"np.random.seed(42)\n",
"\n",
"# To plot pretty figures\n",
"%matplotlib inline\n",
"import matplotlib as mpl\n",
"import matplotlib.pyplot as plt\n",
"mpl.rc('axes', labelsize=14)\n",
"mpl.rc('xtick', labelsize=12)\n",
"mpl.rc('ytick', labelsize=12)\n",
"\n",
"# Where to save the figures\n",
"PROJECT_ROOT_DIR = \".\"\n",
"CHAPTER_ID = \"ann\"\n",
2019-01-21 11:13:10 +01:00
"IMAGES_PATH = os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID)\n",
"os.makedirs(IMAGES_PATH, exist_ok=True)\n",
2019-01-16 16:37:12 +01:00
"\n",
2019-01-21 11:13:10 +01:00
"def save_fig(fig_id, tight_layout=True, fig_extension=\"png\", resolution=300):\n",
" path = os.path.join(IMAGES_PATH, fig_id + \".\" + fig_extension)\n",
2019-01-16 16:37:12 +01:00
" print(\"Saving figure\", fig_id)\n",
" if tight_layout:\n",
" plt.tight_layout()\n",
2019-01-21 11:13:10 +01:00
" plt.savefig(path, format=fig_extension, dpi=resolution)\n",
2019-01-16 16:37:12 +01:00
"\n",
"# Ignore useless warnings (see SciPy issue #5998)\n",
"import warnings\n",
"warnings.filterwarnings(action=\"ignore\", message=\"^internal gelsd\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Perceptrons"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"**Note**: we set `max_iter` and `tol` explicitly to avoid warnings about the fact that their default value will change in future versions of Scikit-Learn."
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"import numpy as np\n",
"from sklearn.datasets import load_iris\n",
"from sklearn.linear_model import Perceptron\n",
"\n",
"iris = load_iris()\n",
"X = iris.data[:, (2, 3)] # petal length, petal width\n",
"y = (iris.target == 0).astype(np.int)\n",
"\n",
2019-01-18 16:08:37 +01:00
"per_clf = Perceptron(max_iter=1000, tol=1e-3, random_state=42)\n",
2019-01-16 16:37:12 +01:00
"per_clf.fit(X, y)\n",
"\n",
"y_pred = per_clf.predict([[2, 0.5]])"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"y_pred"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"a = -per_clf.coef_[0][0] / per_clf.coef_[0][1]\n",
"b = -per_clf.intercept_ / per_clf.coef_[0][1]\n",
"\n",
"axes = [0, 5, 0, 2]\n",
"\n",
"x0, x1 = np.meshgrid(\n",
" np.linspace(axes[0], axes[1], 500).reshape(-1, 1),\n",
" np.linspace(axes[2], axes[3], 200).reshape(-1, 1),\n",
" )\n",
"X_new = np.c_[x0.ravel(), x1.ravel()]\n",
"y_predict = per_clf.predict(X_new)\n",
"zz = y_predict.reshape(x0.shape)\n",
"\n",
"plt.figure(figsize=(10, 4))\n",
"plt.plot(X[y==0, 0], X[y==0, 1], \"bs\", label=\"Not Iris-Setosa\")\n",
"plt.plot(X[y==1, 0], X[y==1, 1], \"yo\", label=\"Iris-Setosa\")\n",
"\n",
"plt.plot([axes[0], axes[1]], [a * axes[0] + b, a * axes[1] + b], \"k-\", linewidth=3)\n",
"from matplotlib.colors import ListedColormap\n",
"custom_cmap = ListedColormap(['#9898ff', '#fafab0'])\n",
"\n",
"plt.contourf(x0, x1, zz, cmap=custom_cmap)\n",
"plt.xlabel(\"Petal length\", fontsize=14)\n",
"plt.ylabel(\"Petal width\", fontsize=14)\n",
"plt.legend(loc=\"lower right\", fontsize=14)\n",
"plt.axis(axes)\n",
"\n",
"save_fig(\"perceptron_iris_plot\")\n",
"plt.show()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Activation functions"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"def sigmoid(z):\n",
" return 1 / (1 + np.exp(-z))\n",
"\n",
"def relu(z):\n",
" return np.maximum(0, z)\n",
"\n",
"def derivative(f, z, eps=0.000001):\n",
" return (f(z + eps) - f(z - eps))/(2 * eps)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"z = np.linspace(-5, 5, 200)\n",
"\n",
"plt.figure(figsize=(11,4))\n",
"\n",
"plt.subplot(121)\n",
"plt.plot(z, np.sign(z), \"r-\", linewidth=1, label=\"Step\")\n",
"plt.plot(z, sigmoid(z), \"g--\", linewidth=2, label=\"Sigmoid\")\n",
"plt.plot(z, np.tanh(z), \"b-\", linewidth=2, label=\"Tanh\")\n",
"plt.plot(z, relu(z), \"m-.\", linewidth=2, label=\"ReLU\")\n",
"plt.grid(True)\n",
"plt.legend(loc=\"center right\", fontsize=14)\n",
"plt.title(\"Activation functions\", fontsize=14)\n",
"plt.axis([-5, 5, -1.2, 1.2])\n",
"\n",
"plt.subplot(122)\n",
"plt.plot(z, derivative(np.sign, z), \"r-\", linewidth=1, label=\"Step\")\n",
"plt.plot(0, 0, \"ro\", markersize=5)\n",
"plt.plot(0, 0, \"rx\", markersize=10)\n",
"plt.plot(z, derivative(sigmoid, z), \"g--\", linewidth=2, label=\"Sigmoid\")\n",
"plt.plot(z, derivative(np.tanh, z), \"b-\", linewidth=2, label=\"Tanh\")\n",
"plt.plot(z, derivative(relu, z), \"m-.\", linewidth=2, label=\"ReLU\")\n",
"plt.grid(True)\n",
"#plt.legend(loc=\"center right\", fontsize=14)\n",
"plt.title(\"Derivatives\", fontsize=14)\n",
"plt.axis([-5, 5, -0.2, 1.2])\n",
"\n",
"save_fig(\"activation_functions_plot\")\n",
"plt.show()"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"def heaviside(z):\n",
" return (z >= 0).astype(z.dtype)\n",
"\n",
"def mlp_xor(x1, x2, activation=heaviside):\n",
" return activation(-activation(x1 + x2 - 1.5) + activation(x1 + x2 - 0.5) - 0.5)"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"x1s = np.linspace(-0.2, 1.2, 100)\n",
"x2s = np.linspace(-0.2, 1.2, 100)\n",
"x1, x2 = np.meshgrid(x1s, x2s)\n",
"\n",
"z1 = mlp_xor(x1, x2, activation=heaviside)\n",
"z2 = mlp_xor(x1, x2, activation=sigmoid)\n",
"\n",
"plt.figure(figsize=(10,4))\n",
"\n",
"plt.subplot(121)\n",
"plt.contourf(x1, x2, z1)\n",
"plt.plot([0, 1], [0, 1], \"gs\", markersize=20)\n",
"plt.plot([0, 1], [1, 0], \"y^\", markersize=20)\n",
"plt.title(\"Activation function: heaviside\", fontsize=14)\n",
"plt.grid(True)\n",
"\n",
"plt.subplot(122)\n",
"plt.contourf(x1, x2, z2)\n",
"plt.plot([0, 1], [0, 1], \"gs\", markersize=20)\n",
"plt.plot([0, 1], [1, 0], \"y^\", markersize=20)\n",
"plt.title(\"Activation function: sigmoid\", fontsize=14)\n",
"plt.grid(True)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Building an Image Classifier"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"First let's import TensorFlow and Keras."
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
"import tensorflow as tf\n",
"from tensorflow import keras"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [],
"source": [
"tf.__version__"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
"keras.__version__"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Let's start by loading the fashion MNIST dataset. Keras has a number of functions to load popular datasets in `keras.datasets`. The dataset is already split for you between a training set and a test set, but it can be useful to split the training set further to have a validation set:"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [],
"source": [
"fashion_mnist = keras.datasets.fashion_mnist\n",
"(X_train_full, y_train_full), (X_test, y_test) = fashion_mnist.load_data()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"The training set contains 60,000 grayscale images, each 28x28 pixels:"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [],
"source": [
"X_train_full.shape"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Each pixel intensity is represented as a byte (0 to 255):"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [],
"source": [
"X_train_full.dtype"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Let's split the full training set into a validation set and a (smaller) training set. We also scale the pixel intensities down to the 0-1 range and convert them to floats, by dividing by 255."
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [],
"source": [
"X_valid, X_train = X_train_full[:5000] / 255., X_train_full[5000:] / 255.\n",
"y_valid, y_train = y_train_full[:5000], y_train_full[5000:]\n",
"X_test = X_test / 255."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"You can plot an image using Matplotlib's `imshow()` function, with a `'binary'`\n",
" color map:"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [],
"source": [
"plt.imshow(X_train[0], cmap=\"binary\")\n",
"plt.axis('off')\n",
"plt.show()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"The labels are the class IDs (represented as uint8), from 0 to 9:"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [],
"source": [
"y_train"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Here are the corresponding class names:"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [],
"source": [
"class_names = [\"T-shirt/top\", \"Trouser\", \"Pullover\", \"Dress\", \"Coat\",\n",
" \"Sandal\", \"Shirt\", \"Sneaker\", \"Bag\", \"Ankle boot\"]"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"So the first image in the training set is a coat:"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {},
"outputs": [],
"source": [
"class_names[y_train[0]]"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"The validation set contains 5,000 images, and the test set contains 10,000 images:"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {},
"outputs": [],
"source": [
"X_valid.shape"
]
},
{
"cell_type": "code",
"execution_count": 21,
"metadata": {},
"outputs": [],
"source": [
"X_test.shape"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Let's take a look at a sample of the images in the dataset:"
]
},
{
"cell_type": "code",
"execution_count": 22,
"metadata": {},
"outputs": [],
"source": [
"n_rows = 4\n",
"n_cols = 10\n",
"plt.figure(figsize=(n_cols * 1.2, n_rows * 1.2))\n",
"for row in range(n_rows):\n",
" for col in range(n_cols):\n",
" index = n_cols * row + col\n",
" plt.subplot(n_rows, n_cols, index + 1)\n",
" plt.imshow(X_train[index], cmap=\"binary\", interpolation=\"nearest\")\n",
" plt.axis('off')\n",
" plt.title(class_names[y_train[index]], fontsize=12)\n",
"plt.subplots_adjust(wspace=0.2, hspace=0.5)\n",
2019-01-18 16:08:37 +01:00
"save_fig('fashion_mnist_diagram', tight_layout=False)\n",
2019-01-16 16:37:12 +01:00
"plt.show()"
]
},
{
"cell_type": "code",
"execution_count": 23,
"metadata": {},
"outputs": [],
"source": [
"model = keras.models.Sequential()\n",
"model.add(keras.layers.Flatten(input_shape=[28, 28]))\n",
"model.add(keras.layers.Dense(300, activation=\"relu\"))\n",
"model.add(keras.layers.Dense(100, activation=\"relu\"))\n",
"model.add(keras.layers.Dense(10, activation=\"softmax\"))"
]
},
{
"cell_type": "code",
"execution_count": 24,
"metadata": {},
"outputs": [],
"source": [
"np.random.seed(42)\n",
"tf.random.set_seed(42)"
]
},
{
"cell_type": "code",
"execution_count": 25,
"metadata": {},
"outputs": [],
"source": [
"model = keras.models.Sequential([\n",
" keras.layers.Flatten(input_shape=[28, 28]),\n",
" keras.layers.Dense(300, activation=\"relu\"),\n",
" keras.layers.Dense(100, activation=\"relu\"),\n",
" keras.layers.Dense(10, activation=\"softmax\")\n",
"])"
]
},
{
"cell_type": "code",
"execution_count": 26,
"metadata": {},
"outputs": [],
"source": [
"model.layers"
]
},
{
"cell_type": "code",
"execution_count": 27,
"metadata": {},
"outputs": [],
"source": [
"model.summary()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"**Warning**: the following 2 cells do not work yet due to [TensorFlow issue 24622](https://github.com/tensorflow/tensorflow/issues/24622) (you are using a preview version of TensorFlow, hence there are still a few issues).\n",
"You can work around this issue by applying [PR 24626](https://github.com/tensorflow/tensorflow/pull/24625/files) to your copy of `tensorflow/python/keras/utils/vis_utils.py`."
]
},
{
"cell_type": "code",
"execution_count": 28,
"metadata": {},
"outputs": [],
"source": [
"#keras.utils.plot_model(model, \"my_mnist_model.png\", show_shapes=True)"
]
},
{
"cell_type": "code",
"execution_count": 29,
"metadata": {},
"outputs": [],
"source": [
"%%html\n",
"<img src=\"my_mnist_model.png\" />"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"**Warning**: the following cell does not work yet due to [TensorFlow issue 24622](https://github.com/tensorflow/tensorflow/issues/24622) and [TensorFlow issue 24639](https://github.com/tensorflow/tensorflow/issues/24639).\n",
"You can work around issue 24639 by writing `from tensorflow.keras.utils.vis_utils import model_to_dot`."
]
},
{
"cell_type": "code",
"execution_count": 30,
"metadata": {},
"outputs": [],
"source": [
"from IPython.display import SVG\n",
"#SVG(keras.utils.model_to_dot(model, show_shapes=True).create(prog='dot', format='svg'))"
]
},
{
"cell_type": "code",
"execution_count": 31,
"metadata": {},
"outputs": [],
"source": [
"hidden1 = model.layers[1]\n",
"hidden1.name"
]
},
{
"cell_type": "code",
"execution_count": 32,
"metadata": {},
"outputs": [],
"source": [
"model.get_layer(hidden1.name).name"
]
},
{
"cell_type": "code",
"execution_count": 33,
"metadata": {},
"outputs": [],
"source": [
"weights, biases = hidden1.get_weights()"
]
},
{
"cell_type": "code",
"execution_count": 34,
"metadata": {},
"outputs": [],
"source": [
"weights"
]
},
{
"cell_type": "code",
"execution_count": 35,
"metadata": {},
"outputs": [],
"source": [
"weights.shape"
]
},
{
"cell_type": "code",
"execution_count": 36,
"metadata": {},
"outputs": [],
"source": [
"biases"
]
},
{
"cell_type": "code",
"execution_count": 37,
"metadata": {},
"outputs": [],
"source": [
"biases.shape"
]
},
{
"cell_type": "code",
"execution_count": 38,
"metadata": {},
"outputs": [],
"source": [
"model.compile(loss=\"sparse_categorical_crossentropy\",\n",
" optimizer=\"sgd\",\n",
" metrics=[\"accuracy\"])"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"This is equivalent to:"
]
},
{
"cell_type": "code",
"execution_count": 39,
"metadata": {},
"outputs": [],
"source": [
"model.compile(loss=keras.losses.sparse_categorical_crossentropy,\n",
" optimizer=keras.optimizers.SGD(),\n",
2019-01-18 16:08:37 +01:00
" metrics=[keras.metrics.sparse_categorical_accuracy])"
2019-01-16 16:37:12 +01:00
]
},
{
"cell_type": "code",
"execution_count": 40,
"metadata": {},
"outputs": [],
"source": [
"history = model.fit(X_train, y_train, epochs=50,\n",
" validation_data=(X_valid, y_valid))"
]
},
{
"cell_type": "code",
"execution_count": 41,
"metadata": {},
"outputs": [],
"source": [
"history.params"
]
},
{
"cell_type": "code",
"execution_count": 42,
"metadata": {},
"outputs": [],
"source": [
"print(history.epoch)"
]
},
{
"cell_type": "code",
"execution_count": 43,
"metadata": {},
"outputs": [],
"source": [
"history.history.keys()"
]
},
{
"cell_type": "code",
"execution_count": 44,
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"\n",
"pd.DataFrame(history.history).plot(figsize=(8, 5))\n",
"plt.grid(True)\n",
"plt.gca().set_ylim(0, 1)\n",
2019-01-18 16:08:37 +01:00
"save_fig(\"keras_learning_curves_graph\")\n",
2019-01-16 16:37:12 +01:00
"plt.show()"
]
},
{
"cell_type": "code",
"execution_count": 45,
"metadata": {},
"outputs": [],
"source": [
"model.evaluate(X_test, y_test)"
]
},
{
"cell_type": "code",
"execution_count": 46,
"metadata": {},
"outputs": [],
"source": [
"X_new = X_test[:3]\n",
"y_proba = model.predict(X_new)\n",
"y_proba.round(2)"
]
},
{
"cell_type": "code",
"execution_count": 47,
"metadata": {},
"outputs": [],
"source": [
"y_pred = model.predict_classes(X_new)\n",
"y_pred"
]
},
{
"cell_type": "code",
"execution_count": 48,
"metadata": {},
"outputs": [],
"source": [
"np.array(class_names)[y_pred]"
]
},
{
"cell_type": "code",
"execution_count": 49,
"metadata": {},
"outputs": [],
"source": [
"y_new = y_test[:3]\n",
"y_new"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Regression MLP"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Let's load, split and scale the California housing dataset (the original one, not the modified one as in chapter 2):"
]
},
{
"cell_type": "code",
"execution_count": 50,
"metadata": {},
"outputs": [],
"source": [
"from sklearn.datasets import fetch_california_housing\n",
"from sklearn.model_selection import train_test_split\n",
"from sklearn.preprocessing import StandardScaler\n",
"\n",
"housing = fetch_california_housing()\n",
"\n",
"X_train_full, X_test, y_train_full, y_test = train_test_split(housing.data, housing.target, random_state=42)\n",
"X_train, X_valid, y_train, y_valid = train_test_split(X_train_full, y_train_full, random_state=42)\n",
"\n",
"scaler = StandardScaler()\n",
"X_train = scaler.fit_transform(X_train)\n",
"X_valid = scaler.transform(X_valid)\n",
"X_test = scaler.transform(X_test)"
]
},
{
"cell_type": "code",
"execution_count": 51,
"metadata": {},
"outputs": [],
"source": [
"np.random.seed(42)\n",
"tf.random.set_seed(42)"
]
},
{
"cell_type": "code",
"execution_count": 52,
"metadata": {},
"outputs": [],
"source": [
"model = keras.models.Sequential([\n",
" keras.layers.Dense(30, activation=\"relu\", input_shape=X_train.shape[1:]),\n",
" keras.layers.Dense(1)\n",
"])\n",
"model.compile(loss=\"mean_squared_error\", optimizer=\"sgd\")\n",
"history = model.fit(X_train, y_train, epochs=20, validation_data=(X_valid, y_valid))\n",
"mse_test = model.evaluate(X_test, y_test)\n",
"X_new = X_test[:3]\n",
"y_pred = model.predict(X_new)"
]
},
{
"cell_type": "code",
"execution_count": 53,
"metadata": {},
"outputs": [],
"source": [
"plt.plot(pd.DataFrame(history.history))\n",
"plt.grid(True)\n",
"plt.gca().set_ylim(0, 1)\n",
"plt.show()"
]
},
{
"cell_type": "code",
"execution_count": 54,
"metadata": {},
"outputs": [],
"source": [
"y_pred"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Functional API"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Not all neural network models are simply sequential. Some may have complex topologies. Some may have multiple inputs and/or multiple outputs. For example, a Wide & Deep neural network (see [paper](https://ai.google/research/pubs/pub45413)) connects all or part of the inputs directly to the output layer."
]
},
{
"cell_type": "code",
"execution_count": 55,
"metadata": {},
"outputs": [],
"source": [
"np.random.seed(42)\n",
"tf.random.set_seed(42)"
]
},
{
"cell_type": "code",
"execution_count": 56,
"metadata": {},
"outputs": [],
"source": [
"input = keras.layers.Input(shape=X_train.shape[1:])\n",
"hidden1 = keras.layers.Dense(30, activation=\"relu\")(input)\n",
"hidden2 = keras.layers.Dense(30, activation=\"relu\")(hidden1)\n",
"concat = keras.layers.concatenate([input, hidden2])\n",
"output = keras.layers.Dense(1)(concat)\n",
"model = keras.models.Model(inputs=[input], outputs=[output])"
]
},
{
"cell_type": "code",
"execution_count": 57,
"metadata": {},
"outputs": [],
"source": [
"model.summary()"
]
},
{
"cell_type": "code",
"execution_count": 58,
"metadata": {},
"outputs": [],
"source": [
"model.compile(loss=\"mean_squared_error\", optimizer=\"sgd\")\n",
"history = model.fit(X_train, y_train, epochs=20,\n",
" validation_data=(X_valid, y_valid))\n",
"mse_test = model.evaluate(X_test, y_test)\n",
"y_pred = model.predict(X_new)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"What if you want to send different subsets of input features through the wide or deep paths? We will send 5 features (features 0 to 4), and 6 through the deep path (features 2 to 7). Note that 3 features will go through both (features 2, 3 and 4)."
]
},
{
"cell_type": "code",
"execution_count": 59,
"metadata": {},
"outputs": [],
"source": [
"np.random.seed(42)\n",
"tf.random.set_seed(42)"
]
},
{
"cell_type": "code",
"execution_count": 60,
"metadata": {},
"outputs": [],
"source": [
"input_A = keras.layers.Input(shape=[5])\n",
"input_B = keras.layers.Input(shape=[6])\n",
"hidden1 = keras.layers.Dense(30, activation=\"relu\")(input_B)\n",
"hidden2 = keras.layers.Dense(30, activation=\"relu\")(hidden1)\n",
"concat = keras.layers.concatenate([input_A, hidden2])\n",
"output = keras.layers.Dense(1)(concat)\n",
"model = keras.models.Model(inputs=[input_A, input_B], outputs=[output])"
]
},
{
"cell_type": "code",
"execution_count": 61,
"metadata": {},
"outputs": [],
"source": [
"model.compile(loss=\"mse\", optimizer=\"sgd\")\n",
"\n",
"X_train_A, X_train_B = X_train[:, :5], X_train[:, 2:]\n",
"X_valid_A, X_valid_B = X_valid[:, :5], X_valid[:, 2:]\n",
"X_test_A, X_test_B = X_test[:, :5], X_test[:, 2:]\n",
"X_new_A, X_new_B = X_test_A[:3], X_test_B[:3]\n",
"\n",
"history = model.fit((X_train_A, X_train_B), y_train, epochs=20,\n",
" validation_data=((X_valid_A, X_valid_B), y_valid))\n",
"mse_test = model.evaluate((X_test_A, X_test_B), y_test)\n",
"y_pred = model.predict((X_new_A, X_new_B))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Adding an auxiliary output for regularization:"
]
},
{
"cell_type": "code",
"execution_count": 62,
"metadata": {},
"outputs": [],
"source": [
"np.random.seed(42)\n",
"tf.random.set_seed(42)"
]
},
{
"cell_type": "code",
"execution_count": 63,
"metadata": {},
"outputs": [],
"source": [
"input_A = keras.layers.Input(shape=[5])\n",
"input_B = keras.layers.Input(shape=[6])\n",
"hidden1 = keras.layers.Dense(30, activation=\"relu\")(input_B)\n",
"hidden2 = keras.layers.Dense(30, activation=\"relu\")(hidden1)\n",
"concat = keras.layers.concatenate([input_A, hidden2])\n",
"output = keras.layers.Dense(1)(concat)\n",
"aux_output = keras.layers.Dense(1)(hidden2)\n",
"model = keras.models.Model(inputs=[input_A, input_B],\n",
" outputs=[output, aux_output])"
]
},
{
"cell_type": "code",
"execution_count": 64,
"metadata": {},
"outputs": [],
"source": [
"model.compile(loss=[\"mse\", \"mse\"], loss_weights=[0.9, 0.1], optimizer=\"sgd\")"
]
},
{
"cell_type": "code",
"execution_count": 65,
"metadata": {},
"outputs": [],
"source": [
"history = model.fit([X_train_A, X_train_B], [y_train, y_train], epochs=20,\n",
" validation_data=([X_valid_A, X_valid_B], [y_valid, y_valid]))"
]
},
{
"cell_type": "code",
"execution_count": 66,
"metadata": {},
"outputs": [],
"source": [
"total_loss, main_loss, aux_loss = model.evaluate(\n",
" [X_test_A, X_test_B], [y_test, y_test])\n",
"y_pred_main, y_pred_aux = model.predict([X_new_A, X_new_B])"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# The subclassing API"
]
},
{
"cell_type": "code",
"execution_count": 67,
"metadata": {},
"outputs": [],
"source": [
"class WideAndDeepModel(keras.models.Model):\n",
" def __init__(self, units=30, activation=\"relu\"):\n",
" super().__init__()\n",
" self.hidden1 = keras.layers.Dense(units, activation=activation)\n",
" self.hidden2 = keras.layers.Dense(units, activation=activation)\n",
" self.main_output = keras.layers.Dense(1)\n",
" self.aux_output = keras.layers.Dense(1)\n",
" \n",
" def call(self, inputs):\n",
" input_A, input_B = inputs\n",
" hidden1 = self.hidden1(input_B)\n",
" hidden2 = self.hidden2(hidden1)\n",
" concat = keras.layers.concatenate([input_A, hidden2])\n",
" main_output = self.main_output(concat)\n",
" aux_output = self.aux_output(hidden2)\n",
" return main_output, aux_output\n",
"\n",
"model = WideAndDeepModel(30, activation=\"relu\")"
]
},
{
"cell_type": "code",
"execution_count": 68,
"metadata": {},
"outputs": [],
"source": [
"model.compile(loss=\"mse\", loss_weights=[0.9, 0.1], optimizer=\"sgd\")\n",
"history = model.fit((X_train_A, X_train_B), (y_train, y_train), epochs=10,\n",
" validation_data=((X_valid_A, X_valid_B), (y_valid, y_valid)))\n",
"total_loss, main_loss, aux_loss = model.evaluate((X_test_A, X_test_B), (y_test, y_test))\n",
"y_pred_main, y_pred_aux = model.predict((X_new_A, X_new_B))"
]
},
{
"cell_type": "code",
"execution_count": 69,
"metadata": {},
"outputs": [],
"source": [
"model = WideAndDeepModel(30, activation=\"relu\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Saving and Restoring"
]
},
{
"cell_type": "code",
"execution_count": 70,
"metadata": {},
"outputs": [],
"source": [
"np.random.seed(42)\n",
"tf.random.set_seed(42)"
]
},
{
"cell_type": "code",
"execution_count": 71,
"metadata": {},
"outputs": [],
"source": [
"model = keras.models.Sequential([\n",
" keras.layers.Dense(30, activation=\"relu\", input_shape=[8]),\n",
" keras.layers.Dense(30, activation=\"relu\"),\n",
" keras.layers.Dense(1)\n",
"]) "
]
},
{
"cell_type": "code",
"execution_count": 72,
"metadata": {},
"outputs": [],
"source": [
"model.compile(loss=\"mse\", optimizer=\"sgd\")\n",
"history = model.fit(X_train, y_train, epochs=10, validation_data=(X_valid, y_valid))\n",
"mse_test = model.evaluate(X_test, y_test)"
]
},
{
"cell_type": "code",
"execution_count": 73,
"metadata": {},
"outputs": [],
"source": [
"model.save(\"my_keras_model.h5\")"
]
},
{
"cell_type": "code",
"execution_count": 74,
"metadata": {},
"outputs": [],
"source": [
"model = keras.models.load_model(\"my_keras_model.h5\")"
]
},
{
"cell_type": "code",
"execution_count": 75,
"metadata": {},
"outputs": [],
"source": [
"model.predict(X_new)"
]
},
{
"cell_type": "code",
"execution_count": 76,
"metadata": {},
"outputs": [],
"source": [
"model.save_weights(\"my_keras_weights.ckpt\")"
]
},
{
"cell_type": "code",
"execution_count": 77,
"metadata": {},
"outputs": [],
"source": [
"model.load_weights(\"my_keras_weights.ckpt\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Using Callbacks during Training"
]
},
{
"cell_type": "code",
"execution_count": 78,
"metadata": {},
"outputs": [],
"source": [
"np.random.seed(42)\n",
"tf.random.set_seed(42)"
]
},
{
"cell_type": "code",
"execution_count": 79,
"metadata": {},
"outputs": [],
"source": [
"model = keras.models.Sequential([\n",
" keras.layers.Dense(30, activation=\"relu\", input_shape=[8]),\n",
" keras.layers.Dense(30, activation=\"relu\"),\n",
" keras.layers.Dense(1)\n",
"]) "
]
},
{
"cell_type": "code",
"execution_count": 80,
"metadata": {},
"outputs": [],
"source": [
"model.compile(loss=\"mse\", optimizer=\"sgd\")\n",
"checkpoint_cb = keras.callbacks.ModelCheckpoint(\"my_keras_model.h5\", save_best_only=True)\n",
"history = model.fit(X_train, y_train, epochs=10,\n",
" validation_data=(X_valid, y_valid),\n",
" callbacks=[checkpoint_cb])\n",
"model = keras.models.load_model(\"my_keras_model.h5\") # rollback to best model\n",
"mse_test = model.evaluate(X_test, y_test)"
]
},
{
"cell_type": "code",
"execution_count": 81,
"metadata": {},
"outputs": [],
"source": [
"model.compile(loss=\"mse\", optimizer=\"sgd\")\n",
"early_stopping_cb = keras.callbacks.EarlyStopping(patience=10,\n",
" restore_best_weights=True)\n",
"history = model.fit(X_train, y_train, epochs=100,\n",
" validation_data=(X_valid, y_valid),\n",
" callbacks=[checkpoint_cb, early_stopping_cb])\n",
"mse_test = model.evaluate(X_test, y_test)"
]
},
{
"cell_type": "code",
"execution_count": 82,
"metadata": {},
"outputs": [],
"source": [
"class PrintValTrainRatioCallback(keras.callbacks.Callback):\n",
" def on_epoch_end(self, epoch, logs):\n",
" print(\"\\nval/train: {:.2f}\".format(logs[\"val_loss\"] / logs[\"loss\"]))"
]
},
{
"cell_type": "code",
"execution_count": 83,
"metadata": {},
"outputs": [],
"source": [
"val_train_ratio_cb = PrintValTrainRatioCallback()\n",
"history = model.fit(X_train, y_train, epochs=1,\n",
" validation_data=(X_valid, y_valid),\n",
" callbacks=[val_train_ratio_cb])"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# TensorBoard"
]
},
{
"cell_type": "code",
"execution_count": 84,
"metadata": {},
"outputs": [],
"source": [
"root_logdir = os.path.join(os.curdir, \"my_logs\")"
]
},
{
"cell_type": "code",
"execution_count": 85,
"metadata": {},
"outputs": [],
"source": [
"def get_run_logdir():\n",
" import time\n",
" run_id = time.strftime(\"run_%Y_%m_%d-%H_%M_%S\")\n",
" return os.path.join(root_logdir, run_id)\n",
"\n",
"run_logdir = get_run_logdir()\n",
"run_logdir"
]
},
{
"cell_type": "code",
"execution_count": 86,
"metadata": {},
"outputs": [],
"source": [
"np.random.seed(42)\n",
"tf.random.set_seed(42)"
]
},
{
"cell_type": "code",
"execution_count": 87,
"metadata": {},
"outputs": [],
"source": [
"model = keras.models.Sequential([\n",
" keras.layers.Dense(30, activation=\"relu\", input_shape=[8]),\n",
" keras.layers.Dense(30, activation=\"relu\"),\n",
" keras.layers.Dense(1)\n",
"]) \n",
"#model.compile(loss=\"mse\", optimizer=\"sgd\")\n",
"# or try another learning rate:\n",
"model.compile(loss=\"mse\", optimizer=keras.optimizers.SGD(lr=0.05))"
]
},
{
"cell_type": "code",
"execution_count": 88,
"metadata": {},
"outputs": [],
"source": [
"tensorboard_cb = keras.callbacks.TensorBoard(run_logdir)\n",
"history = model.fit(X_train, y_train, epochs=30,\n",
" validation_data=(X_valid, y_valid),\n",
" callbacks=[checkpoint_cb, tensorboard_cb])"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"To start the TensorBoard server, one option is to open a terminal, if needed activate the virtualenv where you installed TensorBoard, then type:\n",
"\n",
"```bash\n",
"$ tensorboard --logdir=./my_logs --port=6006\n",
"```\n",
"\n",
"You can then open your web browser to [localhost:6006](http://localhost:6006) and use TensorBoard. Once you are done, press Ctrl-C in the terminal window, this will shutdown the TensorBoard server.\n",
"\n",
"Alternatively, you can create a Jupyter cell with this code:\n",
"\n",
"```bash\n",
"%%bash\n",
"tensorboard --logdir={run_logdir} --port=6006\n",
"```\n",
"\n",
"When you run this cell, the TensorBoard server will start and you can use it at [localhost:6006](http://localhost:6006), but Jupyter will be blocked until you interrupt this cell, which will shutdown the server.\n",
"\n",
"Lastly, you can use the following `tb()` function that starts the TensorBoard server in a way that does not block Jupyter, and directly opens a new browser tab for you. It returns a handle on the server's process, so you can call `server.kill()` when you want to shutdown the server. Note that interrupting this notebook will shutdown all TensorBoard servers that you started this way.\n",
"\n",
"You may also want to install the jupyter-tensorboard extension which integrates nicely into Jupyter to start/stop TensorBoard servers."
]
},
{
"cell_type": "code",
"execution_count": 89,
"metadata": {},
"outputs": [],
"source": [
"def tb(logdir=root_logdir, port=6006, open_tab=True, sleep=3):\n",
" import subprocess\n",
" proc = subprocess.Popen(\n",
" \"tensorboard --logdir={0} --port={1}\".format(logdir, port), shell=True)\n",
" if open_tab:\n",
" import time\n",
" print(\"Waiting a few seconds for the TensorBoard Server to start...\")\n",
" time.sleep(sleep)\n",
" import webbrowser\n",
" webbrowser.open(\"http://127.0.0.1:{}/\".format(port))\n",
" return proc"
]
},
{
"cell_type": "code",
"execution_count": 90,
"metadata": {},
"outputs": [],
"source": [
"server = tb()"
]
},
{
"cell_type": "code",
"execution_count": 91,
"metadata": {},
"outputs": [],
"source": [
"run_logdir2 = get_run_logdir()\n",
"run_logdir2"
]
},
{
"cell_type": "code",
"execution_count": 92,
"metadata": {},
"outputs": [],
"source": [
"np.random.seed(42)\n",
"tf.random.set_seed(42)"
]
},
{
"cell_type": "code",
"execution_count": 93,
"metadata": {},
"outputs": [],
"source": [
"model = keras.models.Sequential([\n",
" keras.layers.Dense(30, activation=\"relu\", input_shape=[8]),\n",
" keras.layers.Dense(30, activation=\"relu\"),\n",
" keras.layers.Dense(1)\n",
"]) \n",
"model.compile(loss=\"mse\", optimizer=keras.optimizers.SGD(lr=0.015))"
]
},
{
"cell_type": "code",
"execution_count": 95,
"metadata": {},
"outputs": [],
"source": [
"tensorboard_cb = keras.callbacks.TensorBoard(run_logdir2)\n",
"history = model.fit(X_train, y_train, epochs=10,\n",
" validation_data=(X_valid, y_valid),\n",
" callbacks=[checkpoint_cb, tensorboard_cb])"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Notice how TensorBoard now sees two runs, and you can compare the learning curves."
]
},
{
"cell_type": "code",
"execution_count": 96,
"metadata": {},
"outputs": [],
"source": [
"help(keras.callbacks.TensorBoard.__init__)"
]
},
{
"cell_type": "code",
"execution_count": 97,
"metadata": {},
"outputs": [],
"source": [
"#server.kill() # uncomment and run this to stop the TensorBoard server"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Hyperparameter Tuning"
]
},
{
"cell_type": "code",
"execution_count": 98,
"metadata": {},
"outputs": [],
"source": [
"np.random.seed(42)\n",
"tf.random.set_seed(42)"
]
},
{
"cell_type": "code",
"execution_count": 99,
"metadata": {},
"outputs": [],
"source": [
"def build_model(n_hidden=1, n_neurons=30, learning_rate=3e-3, input_shape=[8]):\n",
" model = keras.models.Sequential()\n",
" options = {\"input_shape\": input_shape}\n",
" for layer in range(n_hidden):\n",
" model.add(keras.layers.Dense(n_neurons, activation=\"relu\", **options))\n",
" options = {}\n",
" model.add(keras.layers.Dense(1, **options))\n",
" optimizer = keras.optimizers.SGD(learning_rate)\n",
" model.compile(loss=\"mse\", optimizer=optimizer)\n",
" return model"
]
},
{
"cell_type": "code",
"execution_count": 100,
"metadata": {},
"outputs": [],
"source": [
"keras_reg = keras.wrappers.scikit_learn.KerasRegressor(build_model)"
]
},
{
"cell_type": "code",
"execution_count": 101,
"metadata": {},
"outputs": [],
"source": [
"keras_reg.fit(X_train, y_train, epochs=100,\n",
" validation_data=(X_valid, y_valid),\n",
" callbacks=[keras.callbacks.EarlyStopping(patience=10)])"
]
},
{
"cell_type": "code",
"execution_count": 102,
"metadata": {},
"outputs": [],
"source": [
"mse_test = keras_reg.score(X_test, y_test)"
]
},
{
"cell_type": "code",
"execution_count": 103,
"metadata": {},
"outputs": [],
"source": [
"y_pred = keras_reg.predict(X_new)"
]
},
{
"cell_type": "code",
"execution_count": 104,
"metadata": {},
"outputs": [],
"source": [
"np.random.seed(42)\n",
"tf.random.set_seed(42)"
]
},
{
"cell_type": "code",
"execution_count": 105,
"metadata": {},
"outputs": [],
"source": [
"from scipy.stats import reciprocal\n",
"from sklearn.model_selection import RandomizedSearchCV\n",
"\n",
"param_distribs = {\n",
" \"n_hidden\": [0, 1, 2, 3],\n",
" \"n_neurons\": np.arange(1, 100),\n",
" \"learning_rate\": reciprocal(3e-4, 3e-2),\n",
"}\n",
"\n",
"rnd_search_cv = RandomizedSearchCV(keras_reg, param_distribs, n_iter=10, cv=3, verbose=2)\n",
"rnd_search_cv.fit(X_train, y_train, epochs=100,\n",
" validation_data=(X_valid, y_valid),\n",
" callbacks=[keras.callbacks.EarlyStopping(patience=10)])"
]
},
{
"cell_type": "code",
"execution_count": 106,
"metadata": {},
"outputs": [],
"source": [
"rnd_search_cv.best_params_"
]
},
{
"cell_type": "code",
"execution_count": 107,
"metadata": {},
"outputs": [],
"source": [
"rnd_search_cv.best_score_"
]
},
{
"cell_type": "code",
"execution_count": 108,
"metadata": {},
"outputs": [],
"source": [
"rnd_search_cv.best_estimator_"
]
},
{
"cell_type": "code",
"execution_count": 109,
"metadata": {},
"outputs": [],
"source": [
"rnd_search_cv.score(X_test, y_test)"
]
},
{
"cell_type": "code",
"execution_count": 110,
"metadata": {},
"outputs": [],
"source": [
"model = rnd_search_cv.best_estimator_.model\n",
"model"
]
},
{
"cell_type": "code",
"execution_count": 111,
"metadata": {},
"outputs": [],
"source": [
"model.evaluate(X_test, y_test)"
]
},
{
"cell_type": "markdown",
"metadata": {
"collapsed": true
},
"source": [
"# Exercise solutions"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## 1. to 9."
]
},
{
"cell_type": "markdown",
"metadata": {
"collapsed": true
},
"source": [
"See appendix A."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## 10."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"TODO"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
2019-01-18 16:08:37 +01:00
"display_name": "Python 3",
2019-01-16 16:37:12 +01:00
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.8"
},
"nav_menu": {
"height": "264px",
"width": "369px"
},
"toc": {
"navigate_menu": true,
"number_sections": true,
"sideBar": true,
"threshold": 6,
"toc_cell": false,
"toc_section_display": "block",
"toc_window_display": false
}
},
"nbformat": 4,
"nbformat_minor": 1
}
