From 2129ca1f27e9bf2a32bd8223f8b1fdf0e200477a Mon Sep 17 00:00:00 2001 From: francotheengineer Date: Fri, 7 Jul 2017 12:04:17 +0100 Subject: [PATCH 01/26] Fixed Deep RNN Multi-gpu implementation to use tf Fixed Deep RNN Multi-gpu implementation to use tf.python.ops.nn.rnn_cell DeviceWrapper class instead of implementing custom class to achieve the same result. It would appear that the custom class shown with the code does not function. --- 14_recurrent_neural_networks.ipynb | 31 +++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/14_recurrent_neural_networks.ipynb b/14_recurrent_neural_networks.ipynb index f4d541c..e432ad0 100644 --- a/14_recurrent_neural_networks.ipynb +++ b/14_recurrent_neural_networks.ipynb @@ -107,11 +107,13 @@ ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": { "deletable": true, "editable": true }, + "outputs": [], "source": [ "# Basic RNNs" ] @@ -1661,6 +1663,29 @@ "outputs, states = tf.nn.dynamic_rnn(multi_layer_cell, X, dtype=tf.float32)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Alternatively, you can use the Tensorflow class DeviceWrapper - note you can define more than one layer per gpu" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "devices = [\"/gpu:0\", \"/gpu:1\", \"/gpu:2\"] \n", + "cells = []\n", + "for dev in devices:\n", + " cell = DeviceWrapper(rnn_cell.BasicRNNCell(num_units=n_neurons), dev)\n", + " cells.append(cell)\n", + "\n", + "self.multiple_lstm_cells = tf.contrib.rnn.MultiRNNCell(cells, state_is_tuple=True)\n", + "outputs, states = tf.nn.dynamic_rnn(multi_layer_cell, X, dtype=tf.float32)" + ] + }, { "cell_type": "code", "execution_count": 76, @@ -2716,7 +2741,7 @@ "language_info": { "codemirror_mode": { "name": "ipython", - "version": 3 + "version": 3.0 }, "file_extension": ".py", "mimetype": "text/x-python", @@ -2730,7 +2755,7 @@ "navigate_menu": true, "number_sections": true, "sideBar": true, - "threshold": 6, + "threshold": 6.0, "toc_cell": false, "toc_section_display": "block", "toc_window_display": false From 423fffc30e9e106f1c1c5bf94077051aab8ff7c0 Mon Sep 17 00:00:00 2001 From: LujunW Date: Sat, 22 Jul 2017 16:08:41 +0930 Subject: [PATCH 02/26] Fix deprecation warning 'pandas.tools.plotting.scatter_matrix' deprecated, using 'pandas.plotting.scatter_matrix' instead --- 02_end_to_end_machine_learning_project.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/02_end_to_end_machine_learning_project.ipynb b/02_end_to_end_machine_learning_project.ipynb index d6c133e..99b0627 100644 --- a/02_end_to_end_machine_learning_project.ipynb +++ b/02_end_to_end_machine_learning_project.ipynb @@ -505,7 +505,7 @@ "metadata": {}, "outputs": [], "source": [ - "from pandas.tools.plotting import scatter_matrix\n", + "from pandas.plotting import scatter_matrix\n", "\n", "attributes = [\"median_house_value\", \"median_income\", \"total_rooms\",\n", " \"housing_median_age\"]\n", From e1c9f61c5b6b5a5af0cc25e3b375f28f9032c671 Mon Sep 17 00:00:00 2001 From: soma11soma11 Date: Wed, 20 Sep 2017 17:45:31 +0100 Subject: [PATCH 03/26] added comment on SSL: CERTIFICATE_VERIFY_FAILED --- 13_convolutional_neural_networks.ipynb | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/13_convolutional_neural_networks.ipynb b/13_convolutional_neural_networks.ipynb index 73c5ead..c128908 100644 --- a/13_convolutional_neural_networks.ipynb +++ b/13_convolutional_neural_networks.ipynb @@ -501,6 +501,16 @@ " saver = tf.train.Saver()" ] }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": true + }, + "source": [ + "If you are using Python 3.6 on OSX, you need to run the following command on terminal to install the certifi package of certificates because Python 3.6 on OSX has no certificates to validate SSL connections. (https://stackoverflow.com/questions/27835619/urllib-and-ssl-certificate-verify-failed-error) \n", + "/Applications/Python\\ 3.6/Install\\ Certificates.command" + ] + }, { "cell_type": "code", "execution_count": 22, @@ -1949,7 +1959,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.5.2" + "version": "3.6.1" }, "nav_menu": {}, "toc": { From 5dfb4fe5d577918510a4c3ea41a8bb8e8c06e658 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Geron?= Date: Sat, 14 Oct 2017 08:41:42 +0200 Subject: [PATCH 04/26] Remove unecessary line (copy/paste error). --- 13_convolutional_neural_networks.ipynb | 1 - 1 file changed, 1 deletion(-) diff --git a/13_convolutional_neural_networks.ipynb b/13_convolutional_neural_networks.ipynb index 6fd6381..ac7b53f 100644 --- a/13_convolutional_neural_networks.ipynb +++ b/13_convolutional_neural_networks.ipynb @@ -144,7 +144,6 @@ "fmap = np.zeros(shape=(7, 7, 1, 2), dtype=np.float32)\n", "fmap[:, 3, 0, 0] = 1\n", "fmap[3, :, 0, 1] = 1\n", - "fmap[:, :, 0, 0]\n", "plot_image(fmap[:, :, 0, 0])\n", "plt.show()\n", "plot_image(fmap[:, :, 0, 1])\n", From 80e59e1dd113776b95e1a5ddc131ef282abb6a57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Geron?= Date: Sat, 14 Oct 2017 18:20:18 +0200 Subject: [PATCH 05/26] Fix call to matplotlib plot_surface() to support Matplotlib 2.1.0 --- 05_support_vector_machines.ipynb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/05_support_vector_machines.ipynb b/05_support_vector_machines.ipynb index aa85276..687d74b 100644 --- a/05_support_vector_machines.ipynb +++ b/05_support_vector_machines.ipynb @@ -1017,7 +1017,8 @@ " boundary_x2s = -x1s*(w[0]/w[1])-b/w[1]\n", " margin_x2s_1 = -x1s*(w[0]/w[1])-(b-1)/w[1]\n", " margin_x2s_2 = -x1s*(w[0]/w[1])-(b+1)/w[1]\n", - " ax.plot_surface(x1s, x2, 0, color=\"b\", alpha=0.2, cstride=100, rstride=100)\n", + " ax.plot_surface(x1s, x2, np.zeros_like(x1),\n", + " color=\"b\", alpha=0.2, cstride=100, rstride=100)\n", " ax.plot(x1s, boundary_x2s, 0, \"k-\", linewidth=2, label=r\"$h=0$\")\n", " ax.plot(x1s, margin_x2s_1, 0, \"k--\", linewidth=2, label=r\"$h=\\pm 1$\")\n", " ax.plot(x1s, margin_x2s_2, 0, \"k--\", linewidth=2)\n", From 7e4f99085fff96b11364ca19dce7d790eef0801c Mon Sep 17 00:00:00 2001 From: pannoos Date: Sun, 15 Oct 2017 03:11:53 -0700 Subject: [PATCH 06/26] Chapter 2: "Setup" & "Get the data" code improvement --- 02_end_to_end_machine_learning_project.ipynb | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/02_end_to_end_machine_learning_project.ipynb b/02_end_to_end_machine_learning_project.ipynb index 42dc04f..af1ceba 100644 --- a/02_end_to_end_machine_learning_project.ipynb +++ b/02_end_to_end_machine_learning_project.ipynb @@ -61,13 +61,15 @@ "# Where to save the figures\n", "PROJECT_ROOT_DIR = \".\"\n", "CHAPTER_ID = \"end_to_end_project\"\n", + "IMAGES_PATH = os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID)\n", + "FIG_EXTENSION = \"png\"\n", "\n", "def save_fig(fig_id, tight_layout=True):\n", - " path = os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID, fig_id + \".png\")\n", + " path = os.path.join(IMAGES_PATH, fig_id + \".\" + FIG_EXTENSION)\n", " print(\"Saving figure\", fig_id)\n", " if tight_layout:\n", " plt.tight_layout()\n", - " plt.savefig(path, format='png', dpi=300)" + " plt.savefig(path, format=FIG_EXTENSION, dpi=300)" ] }, { @@ -91,7 +93,7 @@ "\n", "DOWNLOAD_ROOT = \"https://raw.githubusercontent.com/ageron/handson-ml/master/\"\n", "HOUSING_PATH = os.path.join(\"datasets\", \"housing\")\n", - "HOUSING_URL = DOWNLOAD_ROOT + \"datasets/housing/housing.tgz\"\n", + "HOUSING_URL = DOWNLOAD_ROOT + HOUSING_PATH + \"/housing.tgz\"\n", "\n", "def fetch_housing_data(housing_url=HOUSING_URL, housing_path=HOUSING_PATH):\n", " if not os.path.isdir(housing_path):\n", @@ -2313,7 +2315,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.5.2" + "version": "3.5.4" }, "nav_menu": { "height": "279px", From bbe33f7039737f2dc46f1e1d214d22e0b20bc81b Mon Sep 17 00:00:00 2001 From: pannoos Date: Sun, 15 Oct 2017 04:01:07 -0700 Subject: [PATCH 07/26] Chapter 2: Generic way of selecting numerical or categorical attributes --- 02_end_to_end_machine_learning_project.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/02_end_to_end_machine_learning_project.ipynb b/02_end_to_end_machine_learning_project.ipynb index af1ceba..c63e8ef 100644 --- a/02_end_to_end_machine_learning_project.ipynb +++ b/02_end_to_end_machine_learning_project.ipynb @@ -689,7 +689,7 @@ }, "outputs": [], "source": [ - "housing_num = housing.drop(\"ocean_proximity\", axis=1)" + "housing_num = housing.select_dtypes(include=[np.number])" ] }, { @@ -797,7 +797,7 @@ "metadata": {}, "outputs": [], "source": [ - "housing_cat = housing[\"ocean_proximity\"]\n", + "housing_cat = housing.select_dtypes(exclude=[np.number])\n", "housing_cat.head(10)" ] }, From e287581c93b05955bbd3174480234db972ea0adf Mon Sep 17 00:00:00 2001 From: pannoos Date: Sun, 15 Oct 2017 14:14:05 -0700 Subject: [PATCH 08/26] Chapter 2: Improvement of code in "Setup" & "Get the data" --- 02_end_to_end_machine_learning_project.ipynb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/02_end_to_end_machine_learning_project.ipynb b/02_end_to_end_machine_learning_project.ipynb index c63e8ef..5332815 100644 --- a/02_end_to_end_machine_learning_project.ipynb +++ b/02_end_to_end_machine_learning_project.ipynb @@ -62,14 +62,13 @@ "PROJECT_ROOT_DIR = \".\"\n", "CHAPTER_ID = \"end_to_end_project\"\n", "IMAGES_PATH = os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID)\n", - "FIG_EXTENSION = \"png\"\n", "\n", - "def save_fig(fig_id, tight_layout=True):\n", - " path = os.path.join(IMAGES_PATH, fig_id + \".\" + FIG_EXTENSION)\n", + "def save_fig(fig_id, tight_layout=True, fig_extension=\"png\", resolution=300):\n", + " path = os.path.join(IMAGES_PATH, fig_id + \".\" + fig_extension)\n", " print(\"Saving figure\", fig_id)\n", " if tight_layout:\n", " plt.tight_layout()\n", - " plt.savefig(path, format=FIG_EXTENSION, dpi=300)" + " plt.savefig(path, format=fig_extension, dpi=resolution)" ] }, { @@ -2322,11 +2321,12 @@ "width": "309px" }, "toc": { - "navigate_menu": true, + "nav_menu": {}, "number_sections": true, "sideBar": true, - "threshold": 6, + "skip_h1_title": false, "toc_cell": false, + "toc_position": {}, "toc_section_display": "block", "toc_window_display": false } From 830d6e4751699abbd0cd36c0aa6e93b104a53403 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Geron?= Date: Mon, 16 Oct 2017 14:19:08 +0200 Subject: [PATCH 09/26] Clarify stratified sampling paragraph in ch02 --- 02_end_to_end_machine_learning_project.ipynb | 279 +++++++++---------- 1 file changed, 129 insertions(+), 150 deletions(-) diff --git a/02_end_to_end_machine_learning_project.ipynb b/02_end_to_end_machine_learning_project.ipynb index 42dc04f..571ecb9 100644 --- a/02_end_to_end_machine_learning_project.ipynb +++ b/02_end_to_end_machine_learning_project.ipynb @@ -35,9 +35,7 @@ { "cell_type": "code", "execution_count": 1, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "# To support both python 2 and python 3\n", @@ -80,9 +78,7 @@ { "cell_type": "code", "execution_count": 2, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "import os\n", @@ -106,9 +102,7 @@ { "cell_type": "code", "execution_count": 3, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "fetch_housing_data()" @@ -117,9 +111,7 @@ { "cell_type": "code", "execution_count": 4, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", @@ -182,9 +174,7 @@ { "cell_type": "code", "execution_count": 10, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "# to make this notebook's output identical at every run\n", @@ -194,9 +184,7 @@ { "cell_type": "code", "execution_count": 11, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", @@ -223,9 +211,7 @@ { "cell_type": "code", "execution_count": 13, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "import hashlib\n", @@ -242,9 +228,7 @@ { "cell_type": "code", "execution_count": 14, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "# This version supports both Python 2 and Python 3, instead of just Python 3.\n", @@ -255,9 +239,7 @@ { "cell_type": "code", "execution_count": 15, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "housing_with_id = housing.reset_index() # adds an `index` column\n", @@ -267,9 +249,7 @@ { "cell_type": "code", "execution_count": 16, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "housing_with_id[\"id\"] = housing[\"longitude\"] * 1000 + housing[\"latitude\"]\n", @@ -288,9 +268,7 @@ { "cell_type": "code", "execution_count": 18, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "from sklearn.model_selection import train_test_split\n", @@ -319,9 +297,7 @@ { "cell_type": "code", "execution_count": 21, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "# Divide by 1.5 to limit the number of income categories\n", @@ -351,9 +327,7 @@ { "cell_type": "code", "execution_count": 24, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "from sklearn.model_selection import StratifiedShuffleSplit\n", @@ -370,15 +344,22 @@ "metadata": {}, "outputs": [], "source": [ - "housing[\"income_cat\"].value_counts() / len(housing)" + "strat_test_set[\"income_cat\"].value_counts() / len(strat_test_set)" ] }, { "cell_type": "code", "execution_count": 26, - "metadata": { - "collapsed": true - }, + "metadata": {}, + "outputs": [], + "source": [ + "housing[\"income_cat\"].value_counts() / len(housing)" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, "outputs": [], "source": [ "def income_cat_proportions(data):\n", @@ -397,7 +378,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 28, "metadata": {}, "outputs": [], "source": [ @@ -406,10 +387,8 @@ }, { "cell_type": "code", - "execution_count": 28, - "metadata": { - "collapsed": true - }, + "execution_count": 29, + "metadata": {}, "outputs": [], "source": [ "for set_ in (strat_train_set, strat_test_set):\n", @@ -425,7 +404,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 30, "metadata": { "collapsed": true }, @@ -436,7 +415,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 31, "metadata": {}, "outputs": [], "source": [ @@ -446,7 +425,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 32, "metadata": {}, "outputs": [], "source": [ @@ -463,7 +442,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 33, "metadata": {}, "outputs": [], "source": [ @@ -477,7 +456,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 34, "metadata": {}, "outputs": [], "source": [ @@ -505,7 +484,7 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 35, "metadata": { "collapsed": true }, @@ -516,7 +495,7 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 36, "metadata": {}, "outputs": [], "source": [ @@ -525,7 +504,7 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 37, "metadata": {}, "outputs": [], "source": [ @@ -540,7 +519,7 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 38, "metadata": {}, "outputs": [], "source": [ @@ -552,7 +531,7 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 39, "metadata": { "collapsed": true }, @@ -572,7 +551,7 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 40, "metadata": {}, "outputs": [], "source": [ @@ -582,7 +561,7 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 41, "metadata": {}, "outputs": [], "source": [ @@ -594,7 +573,7 @@ }, { "cell_type": "code", - "execution_count": 41, + "execution_count": 42, "metadata": {}, "outputs": [], "source": [ @@ -610,7 +589,7 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": 43, "metadata": { "collapsed": true }, @@ -622,7 +601,7 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": 44, "metadata": {}, "outputs": [], "source": [ @@ -632,7 +611,7 @@ }, { "cell_type": "code", - "execution_count": 44, + "execution_count": 45, "metadata": {}, "outputs": [], "source": [ @@ -641,7 +620,7 @@ }, { "cell_type": "code", - "execution_count": 45, + "execution_count": 46, "metadata": {}, "outputs": [], "source": [ @@ -650,7 +629,7 @@ }, { "cell_type": "code", - "execution_count": 46, + "execution_count": 47, "metadata": {}, "outputs": [], "source": [ @@ -661,7 +640,7 @@ }, { "cell_type": "code", - "execution_count": 47, + "execution_count": 48, "metadata": { "collapsed": true }, @@ -681,7 +660,7 @@ }, { "cell_type": "code", - "execution_count": 48, + "execution_count": 49, "metadata": { "collapsed": true }, @@ -692,7 +671,7 @@ }, { "cell_type": "code", - "execution_count": 49, + "execution_count": 50, "metadata": {}, "outputs": [], "source": [ @@ -701,7 +680,7 @@ }, { "cell_type": "code", - "execution_count": 50, + "execution_count": 51, "metadata": {}, "outputs": [], "source": [ @@ -717,7 +696,7 @@ }, { "cell_type": "code", - "execution_count": 51, + "execution_count": 52, "metadata": {}, "outputs": [], "source": [ @@ -733,7 +712,7 @@ }, { "cell_type": "code", - "execution_count": 52, + "execution_count": 53, "metadata": { "collapsed": true }, @@ -744,7 +723,7 @@ }, { "cell_type": "code", - "execution_count": 53, + "execution_count": 54, "metadata": { "collapsed": true }, @@ -756,7 +735,7 @@ }, { "cell_type": "code", - "execution_count": 54, + "execution_count": 55, "metadata": {}, "outputs": [], "source": [ @@ -765,7 +744,7 @@ }, { "cell_type": "code", - "execution_count": 55, + "execution_count": 56, "metadata": {}, "outputs": [], "source": [ @@ -774,7 +753,7 @@ }, { "cell_type": "code", - "execution_count": 56, + "execution_count": 57, "metadata": {}, "outputs": [], "source": [ @@ -791,7 +770,7 @@ }, { "cell_type": "code", - "execution_count": 57, + "execution_count": 58, "metadata": {}, "outputs": [], "source": [ @@ -808,7 +787,7 @@ }, { "cell_type": "code", - "execution_count": 58, + "execution_count": 59, "metadata": {}, "outputs": [], "source": [ @@ -818,7 +797,7 @@ }, { "cell_type": "code", - "execution_count": 59, + "execution_count": 60, "metadata": {}, "outputs": [], "source": [ @@ -841,7 +820,7 @@ }, { "cell_type": "code", - "execution_count": 60, + "execution_count": 61, "metadata": {}, "outputs": [], "source": [ @@ -861,7 +840,7 @@ }, { "cell_type": "code", - "execution_count": 61, + "execution_count": 62, "metadata": {}, "outputs": [], "source": [ @@ -877,7 +856,7 @@ }, { "cell_type": "code", - "execution_count": 62, + "execution_count": 63, "metadata": { "collapsed": true }, @@ -1080,7 +1059,7 @@ }, { "cell_type": "code", - "execution_count": 63, + "execution_count": 64, "metadata": {}, "outputs": [], "source": [ @@ -1101,7 +1080,7 @@ }, { "cell_type": "code", - "execution_count": 64, + "execution_count": 65, "metadata": {}, "outputs": [], "source": [ @@ -1117,7 +1096,7 @@ }, { "cell_type": "code", - "execution_count": 65, + "execution_count": 66, "metadata": {}, "outputs": [], "source": [ @@ -1128,7 +1107,7 @@ }, { "cell_type": "code", - "execution_count": 66, + "execution_count": 67, "metadata": {}, "outputs": [], "source": [ @@ -1144,7 +1123,7 @@ }, { "cell_type": "code", - "execution_count": 67, + "execution_count": 68, "metadata": { "collapsed": true }, @@ -1176,7 +1155,7 @@ }, { "cell_type": "code", - "execution_count": 68, + "execution_count": 69, "metadata": {}, "outputs": [], "source": [ @@ -1193,7 +1172,7 @@ }, { "cell_type": "code", - "execution_count": 69, + "execution_count": 70, "metadata": { "collapsed": true }, @@ -1213,7 +1192,7 @@ }, { "cell_type": "code", - "execution_count": 70, + "execution_count": 71, "metadata": {}, "outputs": [], "source": [ @@ -1229,7 +1208,7 @@ }, { "cell_type": "code", - "execution_count": 71, + "execution_count": 72, "metadata": { "collapsed": true }, @@ -1257,7 +1236,7 @@ }, { "cell_type": "code", - "execution_count": 72, + "execution_count": 73, "metadata": {}, "outputs": [], "source": [ @@ -1279,7 +1258,7 @@ }, { "cell_type": "code", - "execution_count": 73, + "execution_count": 74, "metadata": { "collapsed": true }, @@ -1295,7 +1274,7 @@ }, { "cell_type": "code", - "execution_count": 74, + "execution_count": 75, "metadata": {}, "outputs": [], "source": [ @@ -1305,7 +1284,7 @@ }, { "cell_type": "code", - "execution_count": 75, + "execution_count": 76, "metadata": {}, "outputs": [], "source": [ @@ -1321,7 +1300,7 @@ }, { "cell_type": "code", - "execution_count": 76, + "execution_count": 77, "metadata": {}, "outputs": [], "source": [ @@ -1333,7 +1312,7 @@ }, { "cell_type": "code", - "execution_count": 77, + "execution_count": 78, "metadata": {}, "outputs": [], "source": [ @@ -1354,7 +1333,7 @@ }, { "cell_type": "code", - "execution_count": 78, + "execution_count": 79, "metadata": {}, "outputs": [], "source": [ @@ -1363,7 +1342,7 @@ }, { "cell_type": "code", - "execution_count": 79, + "execution_count": 80, "metadata": {}, "outputs": [], "source": [ @@ -1372,7 +1351,7 @@ }, { "cell_type": "code", - "execution_count": 80, + "execution_count": 81, "metadata": {}, "outputs": [], "source": [ @@ -1386,7 +1365,7 @@ }, { "cell_type": "code", - "execution_count": 81, + "execution_count": 82, "metadata": {}, "outputs": [], "source": [ @@ -1398,7 +1377,7 @@ }, { "cell_type": "code", - "execution_count": 82, + "execution_count": 83, "metadata": {}, "outputs": [], "source": [ @@ -1410,7 +1389,7 @@ }, { "cell_type": "code", - "execution_count": 83, + "execution_count": 84, "metadata": {}, "outputs": [], "source": [ @@ -1429,7 +1408,7 @@ }, { "cell_type": "code", - "execution_count": 84, + "execution_count": 85, "metadata": { "collapsed": true }, @@ -1444,7 +1423,7 @@ }, { "cell_type": "code", - "execution_count": 85, + "execution_count": 86, "metadata": {}, "outputs": [], "source": [ @@ -1458,7 +1437,7 @@ }, { "cell_type": "code", - "execution_count": 86, + "execution_count": 87, "metadata": {}, "outputs": [], "source": [ @@ -1470,7 +1449,7 @@ }, { "cell_type": "code", - "execution_count": 87, + "execution_count": 88, "metadata": {}, "outputs": [], "source": [ @@ -1482,7 +1461,7 @@ }, { "cell_type": "code", - "execution_count": 88, + "execution_count": 89, "metadata": {}, "outputs": [], "source": [ @@ -1494,7 +1473,7 @@ }, { "cell_type": "code", - "execution_count": 89, + "execution_count": 90, "metadata": {}, "outputs": [], "source": [ @@ -1508,7 +1487,7 @@ }, { "cell_type": "code", - "execution_count": 90, + "execution_count": 91, "metadata": {}, "outputs": [], "source": [ @@ -1518,7 +1497,7 @@ }, { "cell_type": "code", - "execution_count": 91, + "execution_count": 92, "metadata": {}, "outputs": [], "source": [ @@ -1534,7 +1513,7 @@ }, { "cell_type": "code", - "execution_count": 92, + "execution_count": 93, "metadata": {}, "outputs": [], "source": [ @@ -1563,7 +1542,7 @@ }, { "cell_type": "code", - "execution_count": 93, + "execution_count": 94, "metadata": {}, "outputs": [], "source": [ @@ -1572,7 +1551,7 @@ }, { "cell_type": "code", - "execution_count": 94, + "execution_count": 95, "metadata": {}, "outputs": [], "source": [ @@ -1588,7 +1567,7 @@ }, { "cell_type": "code", - "execution_count": 95, + "execution_count": 96, "metadata": {}, "outputs": [], "source": [ @@ -1599,7 +1578,7 @@ }, { "cell_type": "code", - "execution_count": 96, + "execution_count": 97, "metadata": {}, "outputs": [], "source": [ @@ -1608,7 +1587,7 @@ }, { "cell_type": "code", - "execution_count": 97, + "execution_count": 98, "metadata": {}, "outputs": [], "source": [ @@ -1628,7 +1607,7 @@ }, { "cell_type": "code", - "execution_count": 98, + "execution_count": 99, "metadata": {}, "outputs": [], "source": [ @@ -1639,7 +1618,7 @@ }, { "cell_type": "code", - "execution_count": 99, + "execution_count": 100, "metadata": {}, "outputs": [], "source": [ @@ -1649,7 +1628,7 @@ }, { "cell_type": "code", - "execution_count": 100, + "execution_count": 101, "metadata": {}, "outputs": [], "source": [ @@ -1662,7 +1641,7 @@ }, { "cell_type": "code", - "execution_count": 101, + "execution_count": 102, "metadata": { "collapsed": true }, @@ -1682,7 +1661,7 @@ }, { "cell_type": "code", - "execution_count": 102, + "execution_count": 103, "metadata": {}, "outputs": [], "source": [ @@ -1705,7 +1684,7 @@ }, { "cell_type": "code", - "execution_count": 103, + "execution_count": 104, "metadata": {}, "outputs": [], "source": [ @@ -1727,7 +1706,7 @@ }, { "cell_type": "code", - "execution_count": 104, + "execution_count": 105, "metadata": { "collapsed": true }, @@ -1738,7 +1717,7 @@ }, { "cell_type": "code", - "execution_count": 105, + "execution_count": 106, "metadata": { "collapsed": true }, @@ -1759,7 +1738,7 @@ }, { "cell_type": "code", - "execution_count": 106, + "execution_count": 107, "metadata": {}, "outputs": [], "source": [ @@ -1797,7 +1776,7 @@ }, { "cell_type": "code", - "execution_count": 107, + "execution_count": 108, "metadata": {}, "outputs": [], "source": [ @@ -1823,7 +1802,7 @@ }, { "cell_type": "code", - "execution_count": 108, + "execution_count": 109, "metadata": {}, "outputs": [], "source": [ @@ -1841,7 +1820,7 @@ }, { "cell_type": "code", - "execution_count": 109, + "execution_count": 110, "metadata": {}, "outputs": [], "source": [ @@ -1871,7 +1850,7 @@ }, { "cell_type": "code", - "execution_count": 110, + "execution_count": 111, "metadata": {}, "outputs": [], "source": [ @@ -1904,7 +1883,7 @@ }, { "cell_type": "code", - "execution_count": 111, + "execution_count": 112, "metadata": {}, "outputs": [], "source": [ @@ -1922,7 +1901,7 @@ }, { "cell_type": "code", - "execution_count": 112, + "execution_count": 113, "metadata": {}, "outputs": [], "source": [ @@ -1945,7 +1924,7 @@ }, { "cell_type": "code", - "execution_count": 113, + "execution_count": 114, "metadata": {}, "outputs": [], "source": [ @@ -1970,7 +1949,7 @@ }, { "cell_type": "code", - "execution_count": 114, + "execution_count": 115, "metadata": {}, "outputs": [], "source": [ @@ -2009,7 +1988,7 @@ }, { "cell_type": "code", - "execution_count": 115, + "execution_count": 116, "metadata": { "collapsed": true }, @@ -2047,7 +2026,7 @@ }, { "cell_type": "code", - "execution_count": 116, + "execution_count": 117, "metadata": { "collapsed": true }, @@ -2065,7 +2044,7 @@ }, { "cell_type": "code", - "execution_count": 117, + "execution_count": 118, "metadata": {}, "outputs": [], "source": [ @@ -2075,7 +2054,7 @@ }, { "cell_type": "code", - "execution_count": 118, + "execution_count": 119, "metadata": {}, "outputs": [], "source": [ @@ -2091,7 +2070,7 @@ }, { "cell_type": "code", - "execution_count": 119, + "execution_count": 120, "metadata": {}, "outputs": [], "source": [ @@ -2107,7 +2086,7 @@ }, { "cell_type": "code", - "execution_count": 120, + "execution_count": 121, "metadata": { "collapsed": true }, @@ -2121,7 +2100,7 @@ }, { "cell_type": "code", - "execution_count": 121, + "execution_count": 122, "metadata": { "collapsed": true }, @@ -2139,7 +2118,7 @@ }, { "cell_type": "code", - "execution_count": 122, + "execution_count": 123, "metadata": {}, "outputs": [], "source": [ @@ -2155,7 +2134,7 @@ }, { "cell_type": "code", - "execution_count": 123, + "execution_count": 124, "metadata": {}, "outputs": [], "source": [ @@ -2185,7 +2164,7 @@ }, { "cell_type": "code", - "execution_count": 124, + "execution_count": 125, "metadata": { "collapsed": true }, @@ -2200,7 +2179,7 @@ }, { "cell_type": "code", - "execution_count": 125, + "execution_count": 126, "metadata": {}, "outputs": [], "source": [ @@ -2216,7 +2195,7 @@ }, { "cell_type": "code", - "execution_count": 126, + "execution_count": 127, "metadata": {}, "outputs": [], "source": [ @@ -2250,7 +2229,7 @@ }, { "cell_type": "code", - "execution_count": 127, + "execution_count": 128, "metadata": {}, "outputs": [], "source": [ @@ -2266,7 +2245,7 @@ }, { "cell_type": "code", - "execution_count": 128, + "execution_count": 129, "metadata": {}, "outputs": [], "source": [ @@ -2282,7 +2261,7 @@ }, { "cell_type": "code", - "execution_count": 129, + "execution_count": 130, "metadata": {}, "outputs": [], "source": [ @@ -2313,7 +2292,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.5.2" + "version": "3.6.2" }, "nav_menu": { "height": "279px", From 34af2979c9ec50e8718a402df297a486059329f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Geron?= Date: Tue, 17 Oct 2017 11:28:55 +0200 Subject: [PATCH 10/26] Roughly 40% of Titanic passengers survived, not 50% (fixes #102) --- 03_classification.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/03_classification.ipynb b/03_classification.ipynb index ea39159..1e7960a 100644 --- a/03_classification.ipynb +++ b/03_classification.ipynb @@ -1494,7 +1494,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "* Yikes, only 48% **Survived**. :( That's close to 50%, so accuracy will be a reasonable metric to evaluate our model.\n", + "* Yikes, only 38% **Survived**. :( That's close enough to 40%, so accuracy will be a reasonable metric to evaluate our model.\n", "* The mean **Fare** was £32.20, which does not seem so expensive (but it was probably a lot of money back then).\n", "* The mean **Age** was less than 30 years old." ] From f53dc9594014d156174edbd694d336f5e50cb964 Mon Sep 17 00:00:00 2001 From: pannoos Date: Tue, 17 Oct 2017 16:41:58 -0700 Subject: [PATCH 11/26] Fix HOUSING_URL --- 02_end_to_end_machine_learning_project.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/02_end_to_end_machine_learning_project.ipynb b/02_end_to_end_machine_learning_project.ipynb index 5332815..9b85078 100644 --- a/02_end_to_end_machine_learning_project.ipynb +++ b/02_end_to_end_machine_learning_project.ipynb @@ -92,7 +92,7 @@ "\n", "DOWNLOAD_ROOT = \"https://raw.githubusercontent.com/ageron/handson-ml/master/\"\n", "HOUSING_PATH = os.path.join(\"datasets\", \"housing\")\n", - "HOUSING_URL = DOWNLOAD_ROOT + HOUSING_PATH + \"/housing.tgz\"\n", + "HOUSING_URL = DOWNLOAD_ROOT + \"datasets/housing/housing.tgz\"\n", "\n", "def fetch_housing_data(housing_url=HOUSING_URL, housing_path=HOUSING_PATH):\n", " if not os.path.isdir(housing_path):\n", From 70d0ab9ec74135c58a7be9035692399eb12491f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Geron?= Date: Wed, 18 Oct 2017 09:18:46 +0200 Subject: [PATCH 12/26] Improve comment format about OSX and Python 3.6, SSL certificates --- 13_convolutional_neural_networks.ipynb | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/13_convolutional_neural_networks.ipynb b/13_convolutional_neural_networks.ipynb index 539de01..b3eec36 100644 --- a/13_convolutional_neural_networks.ipynb +++ b/13_convolutional_neural_networks.ipynb @@ -506,8 +506,9 @@ "collapsed": true }, "source": [ - "If you are using Python 3.6 on OSX, you need to run the following command on terminal to install the certifi package of certificates because Python 3.6 on OSX has no certificates to validate SSL connections. (https://stackoverflow.com/questions/27835619/urllib-and-ssl-certificate-verify-failed-error) \n", - "/Applications/Python\\ 3.6/Install\\ Certificates.command" + "Note: if you are using Python 3.6 on OSX, you need to run the following command on terminal to install the certifi package of certificates because Python 3.6 on OSX has no certificates to validate SSL connections (see this [StackOverflow question](https://stackoverflow.com/questions/27835619/urllib-and-ssl-certificate-verify-failed-error)):\n", + "\n", + " $ /Applications/Python\\ 3.6/Install\\ Certificates.command" ] }, { @@ -1958,7 +1959,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.1" + "version": "3.6.2" }, "nav_menu": {}, "toc": { From 37eb9b41485b6210d65204592129d8d2aa44bc92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Geron?= Date: Wed, 18 Oct 2017 09:44:57 +0200 Subject: [PATCH 13/26] Add comment about the DeviceWrapper class since TF 1.1 --- 14_recurrent_neural_networks.ipynb | 24 ++++-------------------- 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/14_recurrent_neural_networks.ipynb b/14_recurrent_neural_networks.ipynb index 4318442..c4547b1 100644 --- a/14_recurrent_neural_networks.ipynb +++ b/14_recurrent_neural_networks.ipynb @@ -1390,23 +1390,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Alternatively, you can use the Tensorflow class DeviceWrapper - note you can define more than one layer per gpu" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "devices = [\"/gpu:0\", \"/gpu:1\", \"/gpu:2\"] \n", - "cells = []\n", - "for dev in devices:\n", - " cell = DeviceWrapper(rnn_cell.BasicRNNCell(num_units=n_neurons), dev)\n", - " cells.append(cell)\n", - "\n", - "self.multiple_lstm_cells = tf.contrib.rnn.MultiRNNCell(cells, state_is_tuple=True)\n", - "outputs, states = tf.nn.dynamic_rnn(multi_layer_cell, X, dtype=tf.float32)" + "Alternatively, since TensorFlow 1.1, you can use the `tf.contrib.rnn.DeviceWrapper` class (alias `tf.nn.rnn_cell.DeviceWrapper` since TF 1.2)." ] }, { @@ -2722,21 +2706,21 @@ "language_info": { "codemirror_mode": { "name": "ipython", - "version": 3.0 + "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.5.2" + "version": "3.6.2" }, "nav_menu": {}, "toc": { "navigate_menu": true, "number_sections": true, "sideBar": true, - "threshold": 6.0, + "threshold": 6, "toc_cell": false, "toc_section_display": "block", "toc_window_display": false From 7d7ccce9d367f18360c1467ba659c5ea7007494f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Geron?= Date: Fri, 20 Oct 2017 20:05:09 +0200 Subject: [PATCH 14/26] Fix factorize() bug, due to housing_cat being a DataFrame, not a Series, fixes #104 --- 02_end_to_end_machine_learning_project.ipynb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/02_end_to_end_machine_learning_project.ipynb b/02_end_to_end_machine_learning_project.ipynb index 4efc429..7a72959 100644 --- a/02_end_to_end_machine_learning_project.ipynb +++ b/02_end_to_end_machine_learning_project.ipynb @@ -667,7 +667,8 @@ }, "outputs": [], "source": [ - "housing_num = housing.select_dtypes(include=[np.number])" + "housing_num = housing.drop('ocean_proximity', axis=1)\n", + "# alternatively: housing_num = housing.select_dtypes(include=[np.number])" ] }, { @@ -775,7 +776,7 @@ "metadata": {}, "outputs": [], "source": [ - "housing_cat = housing.select_dtypes(exclude=[np.number])\n", + "housing_cat = housing['ocean_proximity']\n", "housing_cat.head(10)" ] }, From 59ab30722e7c392db785505dc41c0b12a25c97ed Mon Sep 17 00:00:00 2001 From: akellyirl Date: Sat, 21 Oct 2017 16:36:39 +0100 Subject: [PATCH 15/26] Update 15_autoencoders.ipynb PCA with a linear Autoencoder Section: The calls to numpy.random require an import to work. Added: import numpy.random as rnd --- 15_autoencoders.ipynb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/15_autoencoders.ipynb b/15_autoencoders.ipynb index 998981e..0629c1f 100644 --- a/15_autoencoders.ipynb +++ b/15_autoencoders.ipynb @@ -131,6 +131,8 @@ }, "outputs": [], "source": [ + "import numpy.random as rnd\n", + "\n", "rnd.seed(4)\n", "m = 200\n", "w1, w2 = 0.1, 0.3\n", From 72a747ed744cb823872d187400cb40036300f67d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Geron?= Date: Fri, 27 Oct 2017 12:58:21 +0200 Subject: [PATCH 16/26] Replace test set with validation set in code example from page 269 --- ...uction_to_artificial_neural_networks.ipynb | 385 ++++-------------- 1 file changed, 78 insertions(+), 307 deletions(-) diff --git a/10_introduction_to_artificial_neural_networks.ipynb b/10_introduction_to_artificial_neural_networks.ipynb index 99fd4e1..7c22236 100644 --- a/10_introduction_to_artificial_neural_networks.ipynb +++ b/10_introduction_to_artificial_neural_networks.ipynb @@ -2,40 +2,28 @@ "cells": [ { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "**Chapter 10 – Introduction to Artificial Neural Networks**" ] }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "_This notebook contains all the sample code and solutions to the exercises in chapter 10._" ] }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "# Setup" ] }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "First, let's make sure this notebook works well in both python 2 and 3, import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures:" ] @@ -43,11 +31,7 @@ { "cell_type": "code", "execution_count": 1, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "# To support both python 2 and python 3\n", @@ -85,10 +69,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "# Perceptrons" ] @@ -97,9 +78,7 @@ "cell_type": "code", "execution_count": 2, "metadata": { - "collapsed": true, - "deletable": true, - "editable": true + "collapsed": true }, "outputs": [], "source": [ @@ -120,11 +99,7 @@ { "cell_type": "code", "execution_count": 3, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "y_pred" @@ -133,11 +108,7 @@ { "cell_type": "code", "execution_count": 4, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "a = -per_clf.coef_[0][0] / per_clf.coef_[0][1]\n", @@ -173,10 +144,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "# Activation functions" ] @@ -185,9 +153,7 @@ "cell_type": "code", "execution_count": 5, "metadata": { - "collapsed": true, - "deletable": true, - "editable": true + "collapsed": true }, "outputs": [], "source": [ @@ -204,11 +170,7 @@ { "cell_type": "code", "execution_count": 6, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "z = np.linspace(-5, 5, 200)\n", @@ -245,9 +207,7 @@ "cell_type": "code", "execution_count": 7, "metadata": { - "collapsed": true, - "deletable": true, - "editable": true + "collapsed": true }, "outputs": [], "source": [ @@ -264,11 +224,7 @@ { "cell_type": "code", "execution_count": 8, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "x1s = np.linspace(-0.2, 1.2, 100)\n", @@ -297,20 +253,14 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "# FNN for MNIST" ] }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "## using tf.learn" ] @@ -318,11 +268,7 @@ { "cell_type": "code", "execution_count": 9, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "from tensorflow.examples.tutorials.mnist import input_data\n", @@ -334,9 +280,7 @@ "cell_type": "code", "execution_count": 10, "metadata": { - "collapsed": true, - "deletable": true, - "editable": true + "collapsed": true }, "outputs": [], "source": [ @@ -349,11 +293,7 @@ { "cell_type": "code", "execution_count": 11, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "import tensorflow as tf\n", @@ -370,11 +310,7 @@ { "cell_type": "code", "execution_count": 12, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "from sklearn.metrics import accuracy_score\n", @@ -386,11 +322,7 @@ { "cell_type": "code", "execution_count": 13, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "from sklearn.metrics import log_loss\n", @@ -402,9 +334,7 @@ { "cell_type": "markdown", "metadata": { - "collapsed": true, - "deletable": true, - "editable": true + "collapsed": true }, "source": [ "## Using plain TensorFlow" @@ -413,11 +343,7 @@ { "cell_type": "code", "execution_count": 14, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "import tensorflow as tf\n", @@ -431,11 +357,7 @@ { "cell_type": "code", "execution_count": 15, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "reset_graph()\n", @@ -447,11 +369,7 @@ { "cell_type": "code", "execution_count": 16, - "metadata": { - "collapsed": true, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "def neuron_layer(X, n_neurons, name, activation=None):\n", @@ -471,11 +389,7 @@ { "cell_type": "code", "execution_count": 17, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "with tf.name_scope(\"dnn\"):\n", @@ -489,11 +403,7 @@ { "cell_type": "code", "execution_count": 18, - "metadata": { - "collapsed": true, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "with tf.name_scope(\"loss\"):\n", @@ -505,11 +415,7 @@ { "cell_type": "code", "execution_count": 19, - "metadata": { - "collapsed": true, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "learning_rate = 0.01\n", @@ -522,11 +428,7 @@ { "cell_type": "code", "execution_count": 20, - "metadata": { - "collapsed": true, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "with tf.name_scope(\"eval\"):\n", @@ -537,11 +439,7 @@ { "cell_type": "code", "execution_count": 21, - "metadata": { - "collapsed": true, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "init = tf.global_variables_initializer()\n", @@ -551,11 +449,7 @@ { "cell_type": "code", "execution_count": 22, - "metadata": { - "collapsed": true, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "n_epochs = 40\n", @@ -565,11 +459,7 @@ { "cell_type": "code", "execution_count": 23, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "with tf.Session() as sess:\n", @@ -579,9 +469,9 @@ " X_batch, y_batch = mnist.train.next_batch(batch_size)\n", " sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n", " acc_train = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n", - " acc_test = accuracy.eval(feed_dict={X: mnist.test.images,\n", - " y: mnist.test.labels})\n", - " print(epoch, \"Train accuracy:\", acc_train, \"Test accuracy:\", acc_test)\n", + " acc_val = accuracy.eval(feed_dict={X: mnist.validation.images,\n", + " y: mnist.validation.labels})\n", + " print(epoch, \"Train accuracy:\", acc_train, \"Val accuracy:\", acc_val)\n", "\n", " save_path = saver.save(sess, \"./my_model_final.ckpt\")" ] @@ -589,11 +479,7 @@ { "cell_type": "code", "execution_count": 24, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "with tf.Session() as sess:\n", @@ -606,11 +492,7 @@ { "cell_type": "code", "execution_count": 25, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "print(\"Predicted classes:\", y_pred)\n", @@ -621,9 +503,7 @@ "cell_type": "code", "execution_count": 26, "metadata": { - "collapsed": true, - "deletable": true, - "editable": true + "collapsed": true }, "outputs": [], "source": [ @@ -668,11 +548,7 @@ { "cell_type": "code", "execution_count": 27, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "show_graph(tf.get_default_graph())" @@ -680,20 +556,14 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "## Using `dense()` instead of `neuron_layer()`" ] }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Note: the book uses `tensorflow.contrib.layers.fully_connected()` rather than `tf.layers.dense()` (which did not exist when this chapter was written). It is now preferable to use `tf.layers.dense()`, because anything in the contrib module may change or be deleted without notice. The `dense()` function is almost identical to the `fully_connected()` function, except for a few minor differences:\n", "* several parameters are renamed: `scope` becomes `name`, `activation_fn` becomes `activation` (and similarly the `_fn` suffix is removed from other parameters such as `normalizer_fn`), `weights_initializer` becomes `kernel_initializer`, etc.\n", @@ -704,11 +574,7 @@ { "cell_type": "code", "execution_count": 28, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "n_inputs = 28*28 # MNIST\n", @@ -721,9 +587,7 @@ "cell_type": "code", "execution_count": 29, "metadata": { - "collapsed": true, - "deletable": true, - "editable": true + "collapsed": true }, "outputs": [], "source": [ @@ -736,11 +600,7 @@ { "cell_type": "code", "execution_count": 30, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "with tf.name_scope(\"dnn\"):\n", @@ -755,9 +615,7 @@ "cell_type": "code", "execution_count": 31, "metadata": { - "collapsed": true, - "deletable": true, - "editable": true + "collapsed": true }, "outputs": [], "source": [ @@ -770,9 +628,7 @@ "cell_type": "code", "execution_count": 32, "metadata": { - "collapsed": true, - "deletable": true, - "editable": true + "collapsed": true }, "outputs": [], "source": [ @@ -787,9 +643,7 @@ "cell_type": "code", "execution_count": 33, "metadata": { - "collapsed": true, - "deletable": true, - "editable": true + "collapsed": true }, "outputs": [], "source": [ @@ -802,9 +656,7 @@ "cell_type": "code", "execution_count": 34, "metadata": { - "collapsed": true, - "deletable": true, - "editable": true + "collapsed": true }, "outputs": [], "source": [ @@ -815,11 +667,7 @@ { "cell_type": "code", "execution_count": 35, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "n_epochs = 20\n", @@ -841,11 +689,7 @@ { "cell_type": "code", "execution_count": 36, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "show_graph(tf.get_default_graph())" @@ -854,9 +698,7 @@ { "cell_type": "markdown", "metadata": { - "collapsed": true, - "deletable": true, - "editable": true + "collapsed": true }, "source": [ "# Exercise solutions" @@ -864,10 +706,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "## 1. to 8." ] @@ -875,9 +714,7 @@ { "cell_type": "markdown", "metadata": { - "collapsed": true, - "deletable": true, - "editable": true + "collapsed": true }, "source": [ "See appendix A." @@ -885,30 +722,21 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "## 9." ] }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "_Train a deep MLP on the MNIST dataset and see if you can get over 98% precision. Just like in the last exercise of chapter 9, try adding all the bells and whistles (i.e., save checkpoints, restore the last checkpoint in case of an interruption, add summaries, plot learning curves using TensorBoard, and so on)._" ] }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "First let's create the deep net. It's exactly the same as earlier, with just one addition: we add a `tf.summary.scalar()` to track the loss and the accuracy during training, so we can view nice learning curves using TensorBoard." ] @@ -916,11 +744,7 @@ { "cell_type": "code", "execution_count": 37, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "n_inputs = 28*28 # MNIST\n", @@ -933,9 +757,7 @@ "cell_type": "code", "execution_count": 38, "metadata": { - "collapsed": true, - "deletable": true, - "editable": true + "collapsed": true }, "outputs": [], "source": [ @@ -948,11 +770,7 @@ { "cell_type": "code", "execution_count": 39, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "with tf.name_scope(\"dnn\"):\n", @@ -967,9 +785,7 @@ "cell_type": "code", "execution_count": 40, "metadata": { - "collapsed": true, - "deletable": true, - "editable": true + "collapsed": true }, "outputs": [], "source": [ @@ -983,9 +799,7 @@ "cell_type": "code", "execution_count": 41, "metadata": { - "collapsed": true, - "deletable": true, - "editable": true + "collapsed": true }, "outputs": [], "source": [ @@ -1000,9 +814,7 @@ "cell_type": "code", "execution_count": 42, "metadata": { - "collapsed": true, - "deletable": true, - "editable": true + "collapsed": true }, "outputs": [], "source": [ @@ -1016,9 +828,7 @@ "cell_type": "code", "execution_count": 43, "metadata": { - "collapsed": true, - "deletable": true, - "editable": true + "collapsed": true }, "outputs": [], "source": [ @@ -1028,10 +838,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Now we need to define the directory to write the TensorBoard logs to:" ] @@ -1040,9 +847,7 @@ "cell_type": "code", "execution_count": 44, "metadata": { - "collapsed": true, - "deletable": true, - "editable": true + "collapsed": true }, "outputs": [], "source": [ @@ -1061,9 +866,7 @@ "cell_type": "code", "execution_count": 45, "metadata": { - "collapsed": true, - "deletable": true, - "editable": true + "collapsed": true }, "outputs": [], "source": [ @@ -1072,10 +875,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Now we can create the `FileWriter` that we will use to write the TensorBoard logs:" ] @@ -1084,9 +884,7 @@ "cell_type": "code", "execution_count": 46, "metadata": { - "collapsed": true, - "deletable": true, - "editable": true + "collapsed": true }, "outputs": [], "source": [ @@ -1095,10 +893,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Hey! Why don't we implement early stopping? For this, we are going to need a validation set. Luckily, the dataset returned by TensorFlow's `input_data()` function (see above) is already split into a training set (60,000 instances, already shuffled for us), a validation set (5,000 instances) and a test set (5,000 instances). So we can easily define `X_valid` and `y_valid`:" ] @@ -1106,11 +901,7 @@ { "cell_type": "code", "execution_count": 47, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "X_valid = mnist.validation.images\n", @@ -1121,9 +912,7 @@ "cell_type": "code", "execution_count": 48, "metadata": { - "collapsed": true, - "deletable": true, - "editable": true + "collapsed": true }, "outputs": [], "source": [ @@ -1133,11 +922,7 @@ { "cell_type": "code", "execution_count": 49, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "n_epochs = 10001\n", @@ -1190,11 +975,7 @@ { "cell_type": "code", "execution_count": 50, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "os.remove(checkpoint_epoch_path)" @@ -1203,11 +984,7 @@ { "cell_type": "code", "execution_count": 51, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "with tf.Session() as sess:\n", @@ -1218,11 +995,7 @@ { "cell_type": "code", "execution_count": 52, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "accuracy_val" @@ -1232,9 +1005,7 @@ "cell_type": "code", "execution_count": null, "metadata": { - "collapsed": true, - "deletable": true, - "editable": true + "collapsed": true }, "outputs": [], "source": [] @@ -1256,7 +1027,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.5.3" + "version": "3.6.2" }, "nav_menu": { "height": "264px", @@ -1273,5 +1044,5 @@ } }, "nbformat": 4, - "nbformat_minor": 0 + "nbformat_minor": 1 } From b39d5366f71c012aaa626d5b71964f2f1941600d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Geron?= Date: Fri, 27 Oct 2017 13:03:15 +0200 Subject: [PATCH 17/26] Right angle is pi/2, not pi/4. One reason why tau > pi ;) --- math_linear_algebra.ipynb | 1533 +++++++------------------------------ 1 file changed, 282 insertions(+), 1251 deletions(-) diff --git a/math_linear_algebra.ipynb b/math_linear_algebra.ipynb index 7d8f8cb..0718eff 100644 --- a/math_linear_algebra.ipynb +++ b/math_linear_algebra.ipynb @@ -2,10 +2,7 @@ "cells": [ { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "**Math - Linear Algebra**\n", "\n", @@ -16,10 +13,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Before we start, let's ensure that this notebook works well in both Python 2 and 3:" ] @@ -27,11 +21,7 @@ { "cell_type": "code", "execution_count": 1, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "from __future__ import division, print_function, unicode_literals" @@ -39,10 +29,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "# Vectors\n", "## Definition\n", @@ -84,10 +71,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "## Vectors in python\n", "In python, a vector can be represented in many ways, the simplest being a regular python list of numbers:" @@ -96,11 +80,7 @@ { "cell_type": "code", "execution_count": 2, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "[10.5, 5.2, 3.25, 7.0]" @@ -108,10 +88,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Since we plan to do quite a lot of scientific calculations, it is much better to use NumPy's `ndarray`, which provides a lot of convenient and optimized implementations of essential mathematical operations on vectors (for more details about NumPy, check out the [NumPy tutorial](tools_numpy.ipynb)). For example:" ] @@ -119,11 +96,7 @@ { "cell_type": "code", "execution_count": 3, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", @@ -133,10 +106,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "The size of a vector can be obtained using the `size` attribute:" ] @@ -144,11 +114,7 @@ { "cell_type": "code", "execution_count": 4, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "video.size" @@ -156,10 +122,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "The $i^{th}$ element (also called *entry* or *item*) of a vector $\\textbf{v}$ is noted $\\textbf{v}_i$.\n", "\n", @@ -169,11 +132,7 @@ { "cell_type": "code", "execution_count": 5, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "video[2] # 3rd element" @@ -181,10 +140,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "## Plotting vectors\n", "To plot vectors we will use matplotlib, so let's start by importing it (for details about matplotlib, check the [matplotlib tutorial](tools_matplotlib.ipynb)):" @@ -193,11 +149,7 @@ { "cell_type": "code", "execution_count": 6, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "%matplotlib inline\n", @@ -206,10 +158,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "### 2D vectors\n", "Let's create a couple very simple 2D vectors to plot:" @@ -219,9 +168,7 @@ "cell_type": "code", "execution_count": 7, "metadata": { - "collapsed": true, - "deletable": true, - "editable": true + "collapsed": true }, "outputs": [], "source": [ @@ -231,10 +178,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "These vectors each have 2 elements, so they can easily be represented graphically on a 2D graph, for example as points:" ] @@ -242,11 +186,7 @@ { "cell_type": "code", "execution_count": 8, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "x_coords, y_coords = zip(u, v)\n", @@ -258,10 +198,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Vectors can also be represented as arrows. Let's create a small convenience function to draw nice arrows:" ] @@ -270,9 +207,7 @@ "cell_type": "code", "execution_count": 9, "metadata": { - "collapsed": true, - "deletable": true, - "editable": true + "collapsed": true }, "outputs": [], "source": [ @@ -284,10 +219,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Now let's draw the vectors **u** and **v** as arrows:" ] @@ -295,11 +227,7 @@ { "cell_type": "code", "execution_count": 10, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "plot_vector2d(u, color=\"r\")\n", @@ -311,10 +239,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "### 3D vectors\n", "Plotting 3D vectors is also relatively straightforward. First let's create two 3D vectors:" @@ -324,9 +249,7 @@ "cell_type": "code", "execution_count": 11, "metadata": { - "collapsed": true, - "deletable": true, - "editable": true + "collapsed": true }, "outputs": [], "source": [ @@ -336,10 +259,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Now let's plot them using matplotlib's `Axes3D`:" ] @@ -347,11 +267,7 @@ { "cell_type": "code", "execution_count": 12, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "from mpl_toolkits.mplot3d import Axes3D\n", @@ -365,10 +281,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "It is a bit hard to visualize exactly where in space these two points are, so let's add vertical lines. We'll create a small convenience function to plot a list of 3d vectors with vertical lines attached:" ] @@ -376,11 +289,7 @@ { "cell_type": "code", "execution_count": 13, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "def plot_vectors3d(ax, vectors3d, z0, **options):\n", @@ -398,10 +307,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "## Norm\n", "The norm of a vector $\\textbf{u}$, noted $\\left \\Vert \\textbf{u} \\right \\|$, is a measure of the length (a.k.a. the magnitude) of $\\textbf{u}$. There are multiple possible norms, but the most common one (and the only one we will discuss here) is the Euclidian norm, which is defined as:\n", @@ -414,11 +320,7 @@ { "cell_type": "code", "execution_count": 14, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "def vector_norm(vector):\n", @@ -431,10 +333,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "However, it is much more efficient to use NumPy's `norm` function, available in the `linalg` (**Lin**ear **Alg**ebra) module:" ] @@ -442,11 +341,7 @@ { "cell_type": "code", "execution_count": 15, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "import numpy.linalg as LA\n", @@ -455,10 +350,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Let's plot a little diagram to confirm that the length of vector $\\textbf{v}$ is indeed $\\approx5.4$:" ] @@ -466,11 +358,7 @@ { "cell_type": "code", "execution_count": 16, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "radius = LA.norm(u)\n", @@ -483,20 +371,14 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Looks about right!" ] }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "## Addition\n", "Vectors of same size can be added together. Addition is performed *elementwise*:" @@ -505,11 +387,7 @@ { "cell_type": "code", "execution_count": 17, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "print(\" \", u)\n", @@ -520,10 +398,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Let's look at what vector addition looks like graphically:" ] @@ -532,9 +407,6 @@ "cell_type": "code", "execution_count": 18, "metadata": { - "collapsed": false, - "deletable": true, - "editable": true, "scrolled": true }, "outputs": [], @@ -556,10 +428,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Vector addition is **commutative**, meaning that $\\textbf{u} + \\textbf{v} = \\textbf{v} + \\textbf{u}$. You can see it on the previous image: following $\\textbf{u}$ *then* $\\textbf{v}$ leads to the same point as following $\\textbf{v}$ *then* $\\textbf{u}$.\n", "\n", @@ -568,10 +437,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "If you have a shape defined by a number of points (vectors), and you add a vector $\\textbf{v}$ to all of these points, then the whole shape gets shifted by $\\textbf{v}$. This is called a [geometric translation](https://en.wikipedia.org/wiki/Translation_%28geometry%29):" ] @@ -579,11 +445,7 @@ { "cell_type": "code", "execution_count": 19, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "t1 = np.array([2, 0.25])\n", @@ -615,20 +477,14 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Finally, substracting a vector is like adding the opposite vector." ] }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "## Multiplication by a scalar\n", "Vectors can be multiplied by scalars. All elements in the vector are multiplied by that number, for example:" @@ -637,11 +493,7 @@ { "cell_type": "code", "execution_count": 20, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "print(\"1.5 *\", u, \"=\")\n", @@ -651,10 +503,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Graphically, scalar multiplication results in changing the scale of a figure, hence the name *scalar*. The distance from the origin (the point at coordinates equal to zero) is also multiplied by the scalar. For example, let's scale up by a factor of `k = 2.5`:" ] @@ -662,11 +511,7 @@ { "cell_type": "code", "execution_count": 21, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "k = 2.5\n", @@ -694,10 +539,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "As you might guess, dividing a vector by a scalar is equivalent to multiplying by its inverse:\n", "\n", @@ -706,10 +548,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Scalar multiplication is **commutative**: $\\lambda \\times \\textbf{u} = \\textbf{u} \\times \\lambda$.\n", "\n", @@ -720,10 +559,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "## Zero, unit and normalized vectors\n", "* A **zero-vector ** is a vector full of 0s.\n", @@ -735,11 +571,7 @@ { "cell_type": "code", "execution_count": 22, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "plt.gca().add_artist(plt.Circle((0,0),1,color='c'))\n", @@ -755,10 +587,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "## Dot product\n", "### Definition\n", @@ -779,11 +608,7 @@ { "cell_type": "code", "execution_count": 23, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "def dot_product(v1, v2):\n", @@ -794,10 +619,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "But a *much* more efficient implementation is provided by NumPy with the `dot` function:" ] @@ -805,11 +627,7 @@ { "cell_type": "code", "execution_count": 24, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "np.dot(u,v)" @@ -817,10 +635,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Equivalently, you can use the `dot` method of `ndarray`s:" ] @@ -828,11 +643,7 @@ { "cell_type": "code", "execution_count": 25, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "u.dot(v)" @@ -840,10 +651,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "**Caution**: the `*` operator will perform an *elementwise* multiplication, *NOT* a dot product:" ] @@ -851,11 +659,7 @@ { "cell_type": "code", "execution_count": 26, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "print(\" \",u)\n", @@ -867,10 +671,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "### Main properties\n", "* The dot product is **commutative**: $\\textbf{u} \\cdot \\textbf{v} = \\textbf{v} \\cdot \\textbf{u}$.\n", @@ -882,17 +683,14 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "### Calculating the angle between vectors\n", "One of the many uses of the dot product is to calculate the angle between two non-zero vectors. Looking at the dot product definition, we can deduce the following formula:\n", "\n", "$\\theta = \\arccos{\\left ( \\dfrac{\\textbf{u} \\cdot \\textbf{v}}{\\left \\Vert \\textbf{u} \\right \\| \\times \\left \\Vert \\textbf{v} \\right \\|} \\right ) }$\n", "\n", - "Note that if $\\textbf{u} \\cdot \\textbf{v} = 0$, it follows that $\\theta = \\dfrac{π}{4}$. In other words, if the dot product of two non-null vectors is zero, it means that they are orthogonal.\n", + "Note that if $\\textbf{u} \\cdot \\textbf{v} = 0$, it follows that $\\theta = \\dfrac{π}{2}$. In other words, if the dot product of two non-null vectors is zero, it means that they are orthogonal.\n", "\n", "Let's use this formula to calculate the angle between $\\textbf{u}$ and $\\textbf{v}$ (in radians):" ] @@ -900,11 +698,7 @@ { "cell_type": "code", "execution_count": 27, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "def vector_angle(u, v):\n", @@ -918,20 +712,14 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Note: due to small floating point errors, `cos_theta` may be very slightly outside of the $[-1, 1]$ interval, which would make `arccos` fail. This is why we clipped the value within the range, using NumPy's `clip` function." ] }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "### Projecting a point onto an axis\n", "The dot product is also very useful to project points onto an axis. The projection of vector $\\textbf{v}$ onto $\\textbf{u}$'s axis is given by this formula:\n", @@ -946,11 +734,7 @@ { "cell_type": "code", "execution_count": 28, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "u_normalized = u / LA.norm(u)\n", @@ -975,10 +759,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "# Matrices\n", "A matrix is a rectangular array of scalars (ie. any number: integer, real or complex) arranged in rows and columns, for example:\n", @@ -992,10 +773,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "## Matrices in python\n", "In python, a matrix can be represented in various ways. The simplest is just a list of python lists:" @@ -1004,11 +782,7 @@ { "cell_type": "code", "execution_count": 29, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "[\n", @@ -1019,10 +793,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "A much more efficient way is to use the NumPy library which provides optimized implementations of many matrix operations:" ] @@ -1030,11 +801,7 @@ { "cell_type": "code", "execution_count": 30, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "A = np.array([\n", @@ -1046,10 +813,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "By convention matrices generally have uppercase names, such as $A$.\n", "\n", @@ -1058,10 +822,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "## Size\n", "The size of a matrix is defined by its number of rows and number of columns. It is noted $rows \\times columns$. For example, the matrix $A$ above is an example of a $2 \\times 3$ matrix: 2 rows, 3 columns. Caution: a $3 \\times 2$ matrix would have 3 rows and 2 columns.\n", @@ -1072,11 +833,7 @@ { "cell_type": "code", "execution_count": 31, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "A.shape" @@ -1084,10 +841,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "**Caution**: the `size` attribute represents the number of elements in the `ndarray`, not the matrix's size:" ] @@ -1095,11 +849,7 @@ { "cell_type": "code", "execution_count": 32, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "A.size" @@ -1107,10 +857,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "## Element indexing\n", "The number located in the $i^{th}$ row, and $j^{th}$ column of a matrix $X$ is sometimes noted $X_{i,j}$ or $X_{ij}$, but there is no standard notation, so people often prefer to explicitely name the elements, like this: \"*let $X = (x_{i,j})_{1 ≤ i ≤ m, 1 ≤ j ≤ n}$*\". This means that $X$ is equal to:\n", @@ -1129,11 +876,7 @@ { "cell_type": "code", "execution_count": 33, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "A[1,2] # 2nd row, 3rd column" @@ -1141,10 +884,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "The $i^{th}$ row vector is sometimes noted $M_i$ or $M_{i,*}$, but again there is no standard notation so people often prefer to explicitely define their own names, for example: \"*let **x**$_{i}$ be the $i^{th}$ row vector of matrix $X$*\". We will use the $M_{i,*}$, for the same reason as above. For example, to access $A_{2,*}$ (ie. $A$'s 2nd row vector):" ] @@ -1152,11 +892,7 @@ { "cell_type": "code", "execution_count": 34, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "A[1, :] # 2nd row vector (as a 1D array)" @@ -1164,10 +900,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Similarly, the $j^{th}$ column vector is sometimes noted $M^j$ or $M_{*,j}$, but there is no standard notation. We will use $M_{*,j}$. For example, to access $A_{*,3}$ (ie. $A$'s 3rd column vector):" ] @@ -1175,11 +908,7 @@ { "cell_type": "code", "execution_count": 35, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "A[:, 2] # 3rd column vector (as a 1D array)" @@ -1187,10 +916,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Note that the result is actually a one-dimensional NumPy array: there is no such thing as a *vertical* or *horizontal* one-dimensional array. If you need to actually represent a row vector as a one-row matrix (ie. a 2D NumPy array), or a column vector as a one-column matrix, then you need to use a slice instead of an integer when accessing the row or column, for example:" ] @@ -1198,11 +924,7 @@ { "cell_type": "code", "execution_count": 36, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "A[1:2, :] # rows 2 to 3 (excluded): this returns row 2 as a one-row matrix" @@ -1211,11 +933,7 @@ { "cell_type": "code", "execution_count": 37, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "A[:, 2:3] # columns 3 to 4 (excluded): this returns column 3 as a one-column matrix" @@ -1223,10 +941,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "## Square, triangular, diagonal and identity matrices\n", "A **square matrix** is a matrix that has the same number of rows and columns, for example a $3 \\times 3$ matrix:\n", @@ -1240,10 +955,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "An **upper triangular matrix** is a special kind of square matrix where all the elements *below* the main diagonal (top-left to bottom-right) are zero, for example:\n", "\n", @@ -1256,10 +968,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Similarly, a **lower triangular matrix** is a square matrix where all elements *above* the main diagonal are zero, for example:\n", "\n", @@ -1272,20 +981,14 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "A **triangular matrix** is one that is either lower triangular or upper triangular." ] }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "A matrix that is both upper and lower triangular is called a **diagonal matrix**, for example:\n", "\n", @@ -1301,11 +1004,7 @@ { "cell_type": "code", "execution_count": 38, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "np.diag([4, 5, 6])" @@ -1313,10 +1012,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "If you pass a matrix to the `diag` function, it will happily extract the diagonal values:" ] @@ -1324,11 +1020,7 @@ { "cell_type": "code", "execution_count": 39, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "D = np.array([\n", @@ -1341,10 +1033,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Finally, the **identity matrix** of size $n$, noted $I_n$, is a diagonal matrix of size $n \\times n$ with $1$'s in the main diagonal, for example $I_3$:\n", "\n", @@ -1360,11 +1049,7 @@ { "cell_type": "code", "execution_count": 40, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "np.eye(3)" @@ -1372,20 +1057,14 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "The identity matrix is often noted simply $I$ (instead of $I_n$) when its size is clear given the context. It is called the *identity* matrix because multiplying a matrix with it leaves the matrix unchanged as we will see below." ] }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "## Adding matrices\n", "If two matrices $Q$ and $R$ have the same size $m \\times n$, they can be added together. Addition is performed *elementwise*: the result is also a $m \\times n$ matrix $S$ where each element is the sum of the elements at the corresponding position: $S_{i,j} = Q_{i,j} + R_{i,j}$\n", @@ -1405,11 +1084,7 @@ { "cell_type": "code", "execution_count": 41, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "B = np.array([[1,2,3], [4, 5, 6]])\n", @@ -1419,11 +1094,7 @@ { "cell_type": "code", "execution_count": 42, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "A" @@ -1432,11 +1103,7 @@ { "cell_type": "code", "execution_count": 43, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "A + B" @@ -1444,10 +1111,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "**Addition is *commutative***, meaning that $A + B = B + A$:" ] @@ -1455,11 +1119,7 @@ { "cell_type": "code", "execution_count": 44, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "B + A" @@ -1467,10 +1127,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "**It is also *associative***, meaning that $A + (B + C) = (A + B) + C$:" ] @@ -1478,11 +1135,7 @@ { "cell_type": "code", "execution_count": 45, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "C = np.array([[100,200,300], [400, 500, 600]])\n", @@ -1493,11 +1146,7 @@ { "cell_type": "code", "execution_count": 46, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "(A + B) + C" @@ -1505,10 +1154,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "## Scalar multiplication\n", "A matrix $M$ can be multiplied by a scalar $\\lambda$. The result is noted $\\lambda M$, and it is a matrix of the same size as $M$ with all elements multiplied by $\\lambda$:\n", @@ -1532,11 +1178,7 @@ { "cell_type": "code", "execution_count": 47, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "2 * A" @@ -1544,10 +1186,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Scalar multiplication is also defined on the right hand side, and gives the same result: $M \\lambda = \\lambda M$. For example:" ] @@ -1555,11 +1194,7 @@ { "cell_type": "code", "execution_count": 48, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "A * 2" @@ -1567,10 +1202,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "This makes scalar multiplication **commutative**.\n", "\n", @@ -1580,11 +1212,7 @@ { "cell_type": "code", "execution_count": 49, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "2 * (3 * A)" @@ -1593,11 +1221,7 @@ { "cell_type": "code", "execution_count": 50, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "(2 * 3) * A" @@ -1605,10 +1229,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Finally, it is **distributive over addition** of matrices, meaning that $\\lambda (Q + R) = \\lambda Q + \\lambda R$:" ] @@ -1616,11 +1237,7 @@ { "cell_type": "code", "execution_count": 51, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "2 * (A + B)" @@ -1629,11 +1246,7 @@ { "cell_type": "code", "execution_count": 52, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "2 * A + 2 * B" @@ -1641,10 +1254,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "## Matrix multiplication\n", "So far, matrix operations have been rather intuitive. But multiplying matrices is a bit more involved.\n", @@ -1689,10 +1299,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Let's multiply two matrices in NumPy, using `ndarray`'s `dot` method:\n", "\n", @@ -1714,11 +1321,7 @@ { "cell_type": "code", "execution_count": 53, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "D = np.array([\n", @@ -1732,10 +1335,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Let's check this result by looking at one element, just to be sure: looking at $E_{2,3}$ for example, we need to multiply elements in $A$'s $2^{nd}$ row by elements in $D$'s $3^{rd}$ column, and sum up these products:" ] @@ -1743,11 +1343,7 @@ { "cell_type": "code", "execution_count": 54, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "40*5 + 50*17 + 60*31" @@ -1756,11 +1352,7 @@ { "cell_type": "code", "execution_count": 55, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "E[1,2] # row 2, column 3" @@ -1768,10 +1360,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Looks good! You can check the other elements until you get used to the algorithm.\n", "\n", @@ -1781,11 +1370,7 @@ { "cell_type": "code", "execution_count": 56, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "try:\n", @@ -1796,10 +1381,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "This illustrates the fact that **matrix multiplication is *NOT* commutative**: in general $QR ≠ RQ$\n", "\n", @@ -1809,11 +1391,7 @@ { "cell_type": "code", "execution_count": 57, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "F = np.array([\n", @@ -1827,11 +1405,7 @@ { "cell_type": "code", "execution_count": 58, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "F.dot(A)" @@ -1839,10 +1413,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "On the other hand, **matrix multiplication *is* associative**, meaning that $Q(RS) = (QR)S$. Let's create a $4 \\times 5$ matrix $G$ to illustrate this:" ] @@ -1850,11 +1421,7 @@ { "cell_type": "code", "execution_count": 59, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "G = np.array([\n", @@ -1868,11 +1435,7 @@ { "cell_type": "code", "execution_count": 60, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "A.dot(D.dot(G)) # A(BG)" @@ -1880,10 +1443,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "It is also ***distributive* over addition** of matrices, meaning that $(Q + R)S = QS + RS$. For example:" ] @@ -1891,11 +1451,7 @@ { "cell_type": "code", "execution_count": 61, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "(A + B).dot(D)" @@ -1904,11 +1460,7 @@ { "cell_type": "code", "execution_count": 62, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "A.dot(D) + B.dot(D)" @@ -1916,10 +1468,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "The product of a matrix $M$ by the identity matrix (of matching size) results in the same matrix $M$. More formally, if $M$ is an $m \\times n$ matrix, then:\n", "\n", @@ -1935,11 +1484,7 @@ { "cell_type": "code", "execution_count": 63, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "A.dot(np.eye(3))" @@ -1948,11 +1493,7 @@ { "cell_type": "code", "execution_count": 64, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "np.eye(2).dot(A)" @@ -1960,10 +1501,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "**Caution**: NumPy's `*` operator performs elementwise multiplication, *NOT* a matrix multiplication:" ] @@ -1972,9 +1510,6 @@ "cell_type": "code", "execution_count": 65, "metadata": { - "collapsed": false, - "deletable": true, - "editable": true, "scrolled": true }, "outputs": [], @@ -1984,10 +1519,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "**The @ infix operator**\n", "\n", @@ -1997,11 +1529,7 @@ { "cell_type": "code", "execution_count": 66, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "import sys\n", @@ -2016,20 +1544,14 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Note: `Q @ R` is actually equivalent to `Q.__matmul__(R)` which is implemented by NumPy as `np.matmul(Q, R)`, not as `Q.dot(R)`. The main difference is that `matmul` does not support scalar multiplication, while `dot` does, so you can write `Q.dot(3)`, which is equivalent to `Q * 3`, but you cannot write `Q @ 3` ([more details](http://stackoverflow.com/a/34142617/38626))." ] }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "## Matrix transpose\n", "The transpose of a matrix $M$ is a matrix noted $M^T$ such that the $i^{th}$ row in $M^T$ is equal to the $i^{th}$ column in $M$:\n", @@ -2057,11 +1579,7 @@ { "cell_type": "code", "execution_count": 67, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "A" @@ -2070,11 +1588,7 @@ { "cell_type": "code", "execution_count": 68, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "A.T" @@ -2082,10 +1596,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "As you might expect, transposing a matrix twice returns the original matrix:" ] @@ -2093,11 +1604,7 @@ { "cell_type": "code", "execution_count": 69, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "A.T.T" @@ -2105,10 +1612,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Transposition is distributive over addition of matrices, meaning that $(Q + R)^T = Q^T + R^T$. For example:" ] @@ -2116,11 +1620,7 @@ { "cell_type": "code", "execution_count": 70, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "(A + B).T" @@ -2129,11 +1629,7 @@ { "cell_type": "code", "execution_count": 71, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "A.T + B.T" @@ -2141,10 +1637,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Moreover, $(Q \\cdot R)^T = R^T \\cdot Q^T$. Note that the order is reversed. For example:" ] @@ -2152,11 +1645,7 @@ { "cell_type": "code", "execution_count": 72, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "(A.dot(D)).T" @@ -2165,11 +1654,7 @@ { "cell_type": "code", "execution_count": 73, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "D.T.dot(A.T)" @@ -2177,10 +1662,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "A **symmetric matrix** $M$ is defined as a matrix that is equal to its transpose: $M^T = M$. This definition implies that it must be a square matrix whose elements are symmetric relative to the main diagonal, for example:\n", "\n", @@ -2197,11 +1679,7 @@ { "cell_type": "code", "execution_count": 74, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "D.dot(D.T)" @@ -2210,9 +1688,7 @@ { "cell_type": "markdown", "metadata": { - "collapsed": true, - "deletable": true, - "editable": true + "collapsed": true }, "source": [ "## Converting 1D arrays to 2D arrays in NumPy\n", @@ -2222,11 +1698,7 @@ { "cell_type": "code", "execution_count": 75, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "u" @@ -2235,11 +1707,7 @@ { "cell_type": "code", "execution_count": 76, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "u.T" @@ -2247,10 +1715,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "We want to convert $\\textbf{u}$ into a row vector before transposing it. There are a few ways to do this:" ] @@ -2258,11 +1723,7 @@ { "cell_type": "code", "execution_count": 77, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "u_row = np.array([u])\n", @@ -2271,10 +1732,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Notice the extra square brackets: this is a 2D array with just one row (ie. a 1x2 matrix). In other words it really is a **row vector**." ] @@ -2282,11 +1740,7 @@ { "cell_type": "code", "execution_count": 78, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "u[np.newaxis, :]" @@ -2294,10 +1748,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "This quite explicit: we are asking for a new vertical axis, keeping the existing data as the horizontal axis." ] @@ -2305,11 +1756,7 @@ { "cell_type": "code", "execution_count": 79, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "u[np.newaxis]" @@ -2317,10 +1764,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "This is equivalent, but a little less explicit." ] @@ -2328,11 +1772,7 @@ { "cell_type": "code", "execution_count": 80, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "u[None]" @@ -2340,10 +1780,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "This is the shortest version, but you probably want to avoid it because it is unclear. The reason it works is that `np.newaxis` is actually equal to `None`, so this is equivalent to the previous version.\n", "\n", @@ -2353,11 +1790,7 @@ { "cell_type": "code", "execution_count": 81, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "u_row.T" @@ -2365,10 +1798,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Great! We now have a nice **column vector**.\n", "\n", @@ -2378,11 +1808,7 @@ { "cell_type": "code", "execution_count": 82, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "u[:, np.newaxis]" @@ -2390,10 +1816,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "## Plotting a matrix\n", "We have already seen that vectors can been represented as points or arrows in N-dimensional space. Is there a good graphical representation of matrices? Well you can simply see a matrix as a list of vectors, so plotting a matrix results in many points or arrows. For example, let's create a $2 \\times 4$ matrix `P` and plot it as points:" @@ -2402,11 +1825,7 @@ { "cell_type": "code", "execution_count": 83, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "P = np.array([\n", @@ -2421,10 +1840,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Of course we could also have stored the same 4 vectors as row vectors instead of column vectors, resulting in a $4 \\times 2$ matrix (the transpose of $P$, in fact). It is really an arbitrary choice.\n", "\n", @@ -2434,11 +1850,7 @@ { "cell_type": "code", "execution_count": 84, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "plt.plot(x_coords_P, y_coords_P, \"bo\")\n", @@ -2450,10 +1862,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Or you can represent it as a polygon: matplotlib's `Polygon` class expects an $n \\times 2$ NumPy array, not a $2 \\times n$ array, so we just need to give it $P^T$:" ] @@ -2461,11 +1870,7 @@ { "cell_type": "code", "execution_count": 85, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "from matplotlib.patches import Polygon\n", @@ -2477,10 +1882,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "## Geometric applications of matrix operations\n", "We saw earlier that vector addition results in a geometric translation, vector multiplication by a scalar results in rescaling (zooming in or out, centered on the origin), and vector dot product results in projecting a vector onto another vector, rescaling and measuring the resulting coordinate.\n", @@ -2490,10 +1892,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "### Addition = multiple geometric translations\n", "First, adding two matrices together is equivalent to adding all their vectors together. For example, let's create a $2 \\times 4$ matrix $H$ and add it to $P$, and look at the result:" @@ -2502,11 +1901,7 @@ { "cell_type": "code", "execution_count": 86, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "H = np.array([\n", @@ -2534,10 +1929,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "If we add a matrix full of identical vectors, we get a simple geometric translation:" ] @@ -2545,11 +1937,7 @@ { "cell_type": "code", "execution_count": 87, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "H2 = np.array([\n", @@ -2570,10 +1958,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Although matrices can only be added together if they have the same size, NumPy allows adding a row vector or a column vector to a matrix: this is called *broadcasting* and is explained in further details in the [NumPy tutorial](tools_numpy.ipynb). We could have obtained the same result as above with:" ] @@ -2581,11 +1966,7 @@ { "cell_type": "code", "execution_count": 88, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "P + [[-0.5], [0.4]] # same as P + H2, thanks to NumPy broadcasting" @@ -2593,10 +1974,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "### Scalar multiplication\n", "Multiplying a matrix by a scalar results in all its vectors being multiplied by that scalar, so unsurprisingly, the geometric result is a rescaling of the entire figure. For example, let's rescale our polygon by a factor of 60% (zooming out, centered on the origin):" @@ -2605,11 +1983,7 @@ { "cell_type": "code", "execution_count": 89, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "def plot_transformation(P_before, P_after, text_before, text_after, axis = [0, 5, 0, 4], arrows=False):\n", @@ -2631,10 +2005,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "### Matrix multiplication – Projection onto an axis\n", "Matrix multiplication is more complex to visualize, but it is also the most powerful tool in the box.\n", @@ -2646,9 +2017,7 @@ "cell_type": "code", "execution_count": 90, "metadata": { - "collapsed": true, - "deletable": true, - "editable": true + "collapsed": true }, "outputs": [], "source": [ @@ -2657,10 +2026,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Now let's look at the dot product $U \\cdot P$:" ] @@ -2668,11 +2034,7 @@ { "cell_type": "code", "execution_count": 91, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "U.dot(P)" @@ -2680,10 +2042,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "These are the horizontal coordinates of the vectors in $P$. In other words, we just projected $P$ onto the horizontal axis:" ] @@ -2691,11 +2050,7 @@ { "cell_type": "code", "execution_count": 92, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "def plot_projection(U, P):\n", @@ -2719,10 +2074,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "We can actually project on any other axis by just replacing $U$ with any other unit vector. For example, let's project on the axis that is at a 30° angle above the horizontal axis:" ] @@ -2730,11 +2082,7 @@ { "cell_type": "code", "execution_count": 93, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "angle30 = 30 * np.pi / 180 # angle in radians\n", @@ -2745,20 +2093,14 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Good! Remember that the dot product of a unit vector and a matrix basically performs a projection on an axis and gives us the coordinates of the resulting points on that axis." ] }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "### Matrix multiplication – Rotation\n", "Now let's create a $2 \\times 2$ matrix $V$ containing two unit vectors that make 30° and 120° angles with the horizontal axis:\n", @@ -2769,11 +2111,7 @@ { "cell_type": "code", "execution_count": 94, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "angle120 = 120 * np.pi / 180\n", @@ -2786,10 +2124,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Let's look at the product $VP$:" ] @@ -2797,11 +2132,7 @@ { "cell_type": "code", "execution_count": 95, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "V.dot(P)" @@ -2809,10 +2140,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "The first row is equal to $V_{1,*} P$, which is the coordinates of the projection of $P$ onto the 30° axis, as we have seen above. The second row is $V_{2,*} P$, which is the coordinates of the projection of $P$ onto the 120° axis. So basically we obtained the coordinates of $P$ after rotating the horizontal and vertical axes by 30° (or equivalently after rotating the polygon by -30° around the origin)! Let's plot $VP$ to see this:" ] @@ -2820,11 +2148,7 @@ { "cell_type": "code", "execution_count": 96, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "P_rotated = V.dot(P)\n", @@ -2834,20 +2158,14 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Matrix $V$ is called a **rotation matrix**." ] }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "### Matrix multiplication – Other linear transformations\n", "More generally, any linear transformation $f$ that maps n-dimensional vectors to m-dimensional vectors can be represented as an $m \\times n$ matrix. For example, say $\\textbf{u}$ is a 3-dimensional vector:\n", @@ -2883,9 +2201,6 @@ "cell_type": "code", "execution_count": 97, "metadata": { - "collapsed": false, - "deletable": true, - "editable": true, "scrolled": true }, "outputs": [], @@ -2901,10 +2216,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Let's look at how this transformation affects the **unit square**: " ] @@ -2912,11 +2224,7 @@ { "cell_type": "code", "execution_count": 98, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "Square = np.array([\n", @@ -2930,10 +2238,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Now let's look at a **squeeze mapping**:" ] @@ -2941,11 +2246,7 @@ { "cell_type": "code", "execution_count": 99, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "F_squeeze = np.array([\n", @@ -2959,10 +2260,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "The effect on the unit square is:" ] @@ -2970,11 +2268,7 @@ { "cell_type": "code", "execution_count": 100, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "plot_transformation(Square, F_squeeze.dot(Square), \"$Square$\", \"$F_{squeeze} Square$\",\n", @@ -2984,10 +2278,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Let's show a last one: reflection through the horizontal axis:" ] @@ -2995,11 +2286,7 @@ { "cell_type": "code", "execution_count": 101, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "F_reflect = np.array([\n", @@ -3013,10 +2300,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "## Matrix inverse\n", "Now that we understand that a matrix can represent any linear transformation, a natural question is: can we find a transformation matrix that reverses the effect of a given transformation matrix $F$? The answer is yes… sometimes! When it exists, such a matrix is called the **inverse** of $F$, and it is noted $F^{-1}$.\n", @@ -3027,11 +2311,7 @@ { "cell_type": "code", "execution_count": 102, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "F_inv_shear = np.array([\n", @@ -3048,10 +2328,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "We applied a shear mapping on $P$, just like we did before, but then we applied a second transformation to the result, and *lo and behold* this had the effect of coming back to the original $P$ (we plotted the original $P$'s outline to double check). The second transformation is the inverse of the first one.\n", "\n", @@ -3061,11 +2338,7 @@ { "cell_type": "code", "execution_count": 103, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "F_inv_shear = LA.inv(F_shear)\n", @@ -3074,10 +2347,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Only square matrices can be inversed. This makes sense when you think about it: if you have a transformation that reduces the number of dimensions, then some information is lost and there is no way that you can get it back. For example say you use a $2 \\times 3$ matrix to project a 3D object onto a plane. The result may look like this:" ] @@ -3085,11 +2355,7 @@ { "cell_type": "code", "execution_count": 104, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "plt.plot([0, 0, 1, 1, 0, 0.1, 0.1, 0, 0.1, 1.1, 1.0, 1.1, 1.1, 1.0, 1.1, 0.1],\n", @@ -3101,10 +2367,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Looking at this image, it is impossible to tell whether this is the projection of a cube or the projection of a narrow rectangular object. Some information has been lost in the projection.\n", "\n", @@ -3114,11 +2377,7 @@ { "cell_type": "code", "execution_count": 105, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "F_project = np.array([\n", @@ -3132,10 +2391,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "This transformation matrix performs a projection onto the horizontal axis. Our polygon gets entirely flattened out so some information is entirely lost and it is impossible to go back to the original polygon using a linear transformation. In other words, $F_{project}$ has no inverse. Such a square matrix that cannot be inversed is called a **singular matrix** (aka degenerate matrix). If we ask NumPy to calculate its inverse, it raises an exception:" ] @@ -3143,11 +2399,7 @@ { "cell_type": "code", "execution_count": 106, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "try:\n", @@ -3158,10 +2410,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Here is another example of a singular matrix. This one performs a projection onto the axis at a 30° angle above the horizontal axis:" ] @@ -3169,11 +2418,7 @@ { "cell_type": "code", "execution_count": 107, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "angle30 = 30 * np.pi / 180\n", @@ -3188,10 +2433,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "But this time, due to floating point rounding errors, NumPy manages to calculate an inverse (notice how large the elements are, though):" ] @@ -3199,11 +2441,7 @@ { "cell_type": "code", "execution_count": 108, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "LA.inv(F_project_30)" @@ -3211,10 +2449,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "As you might expect, the dot product of a matrix by its inverse results in the identity matrix:\n", "\n", @@ -3226,11 +2461,7 @@ { "cell_type": "code", "execution_count": 109, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "F_shear.dot(LA.inv(F_shear))" @@ -3238,10 +2469,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Another way to express this is that the inverse of the inverse of a matrix $M$ is $M$ itself:\n", "\n", @@ -3251,11 +2479,7 @@ { "cell_type": "code", "execution_count": 110, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "LA.inv(LA.inv(F_shear))" @@ -3263,10 +2487,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Also, the inverse of scaling by a factor of $\\lambda$ is of course scaling by a factor or $\\frac{1}{\\lambda}$:\n", "\n", @@ -3280,11 +2501,7 @@ { "cell_type": "code", "execution_count": 111, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "F_involution = np.array([\n", @@ -3298,10 +2515,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Finally, a square matrix $H$ whose inverse is its own transpose is an **orthogonal matrix**:\n", "\n", @@ -3317,11 +2531,7 @@ { "cell_type": "code", "execution_count": 112, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "F_reflect.dot(F_reflect.T)" @@ -3329,10 +2539,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "## Determinant\n", "The determinant of a square matrix $M$, noted $\\det(M)$ or $\\det M$ or $|M|$ is a value that can be calculated from its elements $(M_{i,j})$ using various equivalent methods. One of the simplest methods is this recursive approach:\n", @@ -3370,10 +2577,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "To get the determinant of a matrix, you can call NumPy's `det` function in the `numpy.linalg` module:" ] @@ -3381,11 +2585,7 @@ { "cell_type": "code", "execution_count": 113, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "M = np.array([\n", @@ -3398,10 +2598,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "One of the main uses of the determinant is to *determine* whether a square matrix can be inversed or not: if the determinant is equal to 0, then the matrix *cannot* be inversed (it is a singular matrix), and if the determinant is not 0, then it *can* be inversed.\n", "\n", @@ -3411,11 +2608,7 @@ { "cell_type": "code", "execution_count": 114, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "LA.det(F_project)" @@ -3423,10 +2616,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "That's right, $F_{project}$ is singular, as we saw earlier." ] @@ -3434,11 +2624,7 @@ { "cell_type": "code", "execution_count": 115, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "LA.det(F_project_30)" @@ -3446,10 +2632,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "This determinant is suspiciously close to 0: it really should be 0, but it's not due to tiny floating point errors. The matrix is actually singular." ] @@ -3457,11 +2640,7 @@ { "cell_type": "code", "execution_count": 116, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "LA.det(F_shear)" @@ -3469,20 +2648,14 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Perfect! This matrix *can* be inversed as we saw earlier. Wow, math really works!" ] }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "The determinant can also be used to measure how much a linear transformation affects surface areas: for example, the projection matrices $F_{project}$ and $F_{project\\_30}$ completely flatten the polygon $P$, until its area is zero. This is why the determinant of these matrices is 0. The shear mapping modified the shape of the polygon, but it did not affect its surface area, which is why the determinant is 1. You can try computing the determinant of a rotation matrix, and you should also find 1. What about a scaling matrix? Let's see:" ] @@ -3490,11 +2663,7 @@ { "cell_type": "code", "execution_count": 117, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "F_scale = np.array([\n", @@ -3508,10 +2677,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "We rescaled the polygon by a factor of 1/2 on both vertical and horizontal axes so the surface area of the resulting polygon is 1/4$^{th}$ of the original polygon. Let's compute the determinant and check that:" ] @@ -3519,11 +2685,7 @@ { "cell_type": "code", "execution_count": 118, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "LA.det(F_scale)" @@ -3531,10 +2693,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Correct!\n", "\n", @@ -3544,11 +2703,7 @@ { "cell_type": "code", "execution_count": 119, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "LA.det(F_reflect)" @@ -3556,10 +2711,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "## Composing linear transformations\n", "Several linear transformations can be chained simply by performing multiple dot products in a row. For example, to perform a squeeze mapping followed by a shear mapping, just write:" @@ -3568,11 +2720,7 @@ { "cell_type": "code", "execution_count": 120, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "P_squeezed_then_sheared = F_shear.dot(F_squeeze.dot(P))" @@ -3580,10 +2728,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Since the dot product is associative, the following code is equivalent:" ] @@ -3591,11 +2736,7 @@ { "cell_type": "code", "execution_count": 121, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "P_squeezed_then_sheared = (F_shear.dot(F_squeeze)).dot(P)" @@ -3603,10 +2744,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Note that the order of the transformations is the reverse of the dot product order.\n", "\n", @@ -3617,9 +2755,7 @@ "cell_type": "code", "execution_count": 122, "metadata": { - "collapsed": true, - "deletable": true, - "editable": true + "collapsed": true }, "outputs": [], "source": [ @@ -3629,20 +2765,14 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "From now on we can perform both transformations in just one dot product, which can lead to a very significant performance boost." ] }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "What if you want to perform the inverse of this double transformation? Well, if you squeezed and then you sheared, and you want to undo what you have done, it should be obvious that you should unshear first and then unsqueeze. In more mathematical terms, given two invertible (aka nonsingular) matrices $Q$ and $R$:\n", "\n", @@ -3654,11 +2784,7 @@ { "cell_type": "code", "execution_count": 123, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "LA.inv(F_shear.dot(F_squeeze)) == LA.inv(F_squeeze).dot(LA.inv(F_shear))" @@ -3666,10 +2792,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "## Singular Value Decomposition\n", "It turns out that any $m \\times n$ matrix $M$ can be decomposed into the dot product of three simple matrices:\n", @@ -3685,11 +2808,7 @@ { "cell_type": "code", "execution_count": 124, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "U, S_diag, V_T = LA.svd(F_shear) # note: in python 3 you can rename S_diag to Σ_diag\n", @@ -3699,11 +2818,7 @@ { "cell_type": "code", "execution_count": 125, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "S_diag" @@ -3711,10 +2826,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Note that this is just a 1D array containing the diagonal values of Σ. To get the actual matrix Σ, we can use NumPy's `diag` function:" ] @@ -3722,11 +2834,7 @@ { "cell_type": "code", "execution_count": 126, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "S = np.diag(S_diag)\n", @@ -3735,10 +2843,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Now let's check that $U \\cdot \\Sigma \\cdot V^T$ is indeed equal to `F_shear`:" ] @@ -3746,11 +2851,7 @@ { "cell_type": "code", "execution_count": 127, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "U.dot(np.diag(S_diag)).dot(V_T)" @@ -3759,11 +2860,7 @@ { "cell_type": "code", "execution_count": 128, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "F_shear" @@ -3771,10 +2868,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "It worked like a charm. Let's apply these transformations one by one (in reverse order) on the unit square to understand what's going on. First, let's apply the first rotation $V^T$:" ] @@ -3782,11 +2876,7 @@ { "cell_type": "code", "execution_count": 129, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "plot_transformation(Square, V_T.dot(Square), \"$Square$\", \"$V^T \\cdot Square$\",\n", @@ -3796,10 +2886,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Now let's rescale along the vertical and horizontal axes using $\\Sigma$:" ] @@ -3807,11 +2894,7 @@ { "cell_type": "code", "execution_count": 130, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "plot_transformation(V_T.dot(Square), S.dot(V_T).dot(Square), \"$V^T \\cdot Square$\", \"$\\Sigma \\cdot V^T \\cdot Square$\",\n", @@ -3821,10 +2904,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Finally, we apply the second rotation $U$:" ] @@ -3832,11 +2912,7 @@ { "cell_type": "code", "execution_count": 131, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "plot_transformation(S.dot(V_T).dot(Square), U.dot(S).dot(V_T).dot(Square),\"$\\Sigma \\cdot V^T \\cdot Square$\", \"$U \\cdot \\Sigma \\cdot V^T \\cdot Square$\",\n", @@ -3846,20 +2922,14 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "And we can see that the result is indeed a shear mapping of the original unit square." ] }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "## Eigenvectors and eigenvalues\n", "An **eigenvector** of a square matrix $M$ (also called a **characteristic vector**) is a non-zero vector that remains on the same line after transformation by the linear transformation associated with $M$. A more formal definition is any vector $v$ such that:\n", @@ -3880,11 +2950,7 @@ { "cell_type": "code", "execution_count": 132, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "eigenvalues, eigenvectors = LA.eig(F_squeeze)\n", @@ -3894,11 +2960,7 @@ { "cell_type": "code", "execution_count": 133, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "eigenvectors # [v0, v1, …]" @@ -3906,10 +2968,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Indeed the horizontal vectors are stretched by a factor of 1.4, and the vertical vectors are shrunk by a factor of 1/1.4=0.714…, so far so good. Let's look at the shear mapping matrix $F_{shear}$:" ] @@ -3917,11 +2976,7 @@ { "cell_type": "code", "execution_count": 134, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "eigenvalues2, eigenvectors2 = LA.eig(F_shear)\n", @@ -3931,11 +2986,7 @@ { "cell_type": "code", "execution_count": 135, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "eigenvectors2 # [v0, v1, …]" @@ -3943,20 +2994,14 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "Wait, what!? We expected just one unit eigenvector, not two. The second vector is almost equal to $\\begin{pmatrix}-1 \\\\ 0 \\end{pmatrix}$, which is on the same line as the first vector $\\begin{pmatrix}1 \\\\ 0 \\end{pmatrix}$. This is due to floating point errors. We can safely ignore vectors that are (almost) colinear (ie. on the same line)." ] }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "## Trace\n", "The trace of a square matrix $M$, noted $tr(M)$ is the sum of the values on its main diagonal. For example:" @@ -3965,11 +3010,7 @@ { "cell_type": "code", "execution_count": 136, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "D = np.array([\n", @@ -3982,10 +3023,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "The trace does not have a simple geometric interpretation (in general), but it has a number of properties that make it useful in many areas:\n", "* $tr(A + B) = tr(A) + tr(B)$\n", @@ -4000,11 +3038,7 @@ { "cell_type": "code", "execution_count": 137, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "np.trace(F_project)" @@ -4012,10 +3046,7 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "# What next?\n", "This concludes this introduction to Linear Algebra. Although these basics cover most of what you will need to know for Machine Learning, if you wish to go deeper into this topic there are many options available: Linear Algebra [books](http://linear.axler.net/), [Khan Academy](https://www.khanacademy.org/math/linear-algebra) lessons, or just [Wikipedia](https://en.wikipedia.org/wiki/Linear_algebra) pages. " @@ -4047,7 +3078,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.5.2" + "version": "3.6.2" }, "toc": { "toc_cell": false, @@ -4057,5 +3088,5 @@ } }, "nbformat": 4, - "nbformat_minor": 0 + "nbformat_minor": 1 } From fd0ce384f200820a25eca44a502a1d65d5dbe0ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Geron?= Date: Fri, 27 Oct 2017 16:19:15 +0200 Subject: [PATCH 18/26] Set DropoutWrapper's input_keep_prob parameter using a placeholder, fixes #111 --- 14_recurrent_neural_networks.ipynb | 364 +++++++---------------------- 1 file changed, 89 insertions(+), 275 deletions(-) diff --git a/14_recurrent_neural_networks.ipynb b/14_recurrent_neural_networks.ipynb index c4547b1..267d821 100644 --- a/14_recurrent_neural_networks.ipynb +++ b/14_recurrent_neural_networks.ipynb @@ -31,9 +31,7 @@ { "cell_type": "code", "execution_count": 1, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "# To support both python 2 and python 3\n", @@ -79,9 +77,7 @@ { "cell_type": "code", "execution_count": 2, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "import tensorflow as tf" @@ -104,9 +100,7 @@ { "cell_type": "code", "execution_count": 3, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "reset_graph()\n", @@ -130,9 +124,7 @@ { "cell_type": "code", "execution_count": 4, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", @@ -173,9 +165,7 @@ { "cell_type": "code", "execution_count": 7, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "n_inputs = 3\n", @@ -185,9 +175,7 @@ { "cell_type": "code", "execution_count": 8, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "reset_graph()\n", @@ -204,9 +192,7 @@ { "cell_type": "code", "execution_count": 9, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "init = tf.global_variables_initializer()" @@ -215,9 +201,7 @@ { "cell_type": "code", "execution_count": 10, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "X0_batch = np.array([[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 0, 1]])\n", @@ -249,9 +233,7 @@ { "cell_type": "code", "execution_count": 13, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "from IPython.display import clear_output, Image, display, HTML\n", @@ -311,9 +293,7 @@ { "cell_type": "code", "execution_count": 15, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "n_steps = 2\n", @@ -324,9 +304,7 @@ { "cell_type": "code", "execution_count": 16, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "reset_graph()\n", @@ -343,9 +321,7 @@ { "cell_type": "code", "execution_count": 17, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "init = tf.global_variables_initializer()" @@ -354,9 +330,7 @@ { "cell_type": "code", "execution_count": 18, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "X_batch = np.array([\n", @@ -400,9 +374,7 @@ { "cell_type": "code", "execution_count": 21, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "n_steps = 2\n", @@ -413,9 +385,7 @@ { "cell_type": "code", "execution_count": 22, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "reset_graph()\n", @@ -429,9 +399,7 @@ { "cell_type": "code", "execution_count": 23, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "init = tf.global_variables_initializer()" @@ -440,9 +408,7 @@ { "cell_type": "code", "execution_count": 24, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "X_batch = np.array([\n", @@ -485,9 +451,7 @@ { "cell_type": "code", "execution_count": 27, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "n_steps = 2\n", @@ -503,9 +467,7 @@ { "cell_type": "code", "execution_count": 28, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "seq_length = tf.placeholder(tf.int32, [None])\n", @@ -516,9 +478,7 @@ { "cell_type": "code", "execution_count": 29, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "init = tf.global_variables_initializer()" @@ -527,9 +487,7 @@ { "cell_type": "code", "execution_count": 30, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "X_batch = np.array([\n", @@ -545,9 +503,7 @@ { "cell_type": "code", "execution_count": 31, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "with tf.Session() as sess:\n", @@ -593,9 +549,7 @@ { "cell_type": "code", "execution_count": 34, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "reset_graph()\n", @@ -668,9 +622,7 @@ { "cell_type": "code", "execution_count": 37, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "reset_graph()\n", @@ -688,9 +640,7 @@ { "cell_type": "code", "execution_count": 38, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "n_neurons = 100\n", @@ -706,9 +656,7 @@ { "cell_type": "code", "execution_count": 39, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "states_concat = tf.concat(axis=1, values=states)\n", @@ -754,9 +702,7 @@ { "cell_type": "code", "execution_count": 41, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "t_min, t_max = 0, 30\n", @@ -808,9 +754,7 @@ { "cell_type": "code", "execution_count": 43, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "X_batch, y_batch = next_batch(1, n_steps)" @@ -842,9 +786,7 @@ { "cell_type": "code", "execution_count": 45, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "reset_graph()\n", @@ -871,9 +813,7 @@ { "cell_type": "code", "execution_count": 46, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "reset_graph()\n", @@ -890,9 +830,7 @@ { "cell_type": "code", "execution_count": 47, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "cell = tf.contrib.rnn.OutputProjectionWrapper(\n", @@ -903,9 +841,7 @@ { "cell_type": "code", "execution_count": 48, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "outputs, states = tf.nn.dynamic_rnn(cell, X, dtype=tf.float32)" @@ -914,9 +850,7 @@ { "cell_type": "code", "execution_count": 49, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "learning_rate = 0.001\n", @@ -931,9 +865,7 @@ { "cell_type": "code", "execution_count": 50, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "saver = tf.train.Saver()" @@ -1009,9 +941,7 @@ { "cell_type": "code", "execution_count": 55, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "reset_graph()\n", @@ -1027,9 +957,7 @@ { "cell_type": "code", "execution_count": 56, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "cell = tf.contrib.rnn.BasicRNNCell(num_units=n_neurons, activation=tf.nn.relu)\n", @@ -1039,9 +967,7 @@ { "cell_type": "code", "execution_count": 57, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "n_outputs = 1\n", @@ -1051,9 +977,7 @@ { "cell_type": "code", "execution_count": 58, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "stacked_rnn_outputs = tf.reshape(rnn_outputs, [-1, n_neurons])\n", @@ -1064,9 +988,7 @@ { "cell_type": "code", "execution_count": 59, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "loss = tf.reduce_mean(tf.square(outputs - y))\n", @@ -1216,9 +1138,7 @@ { "cell_type": "code", "execution_count": 66, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "reset_graph()\n", @@ -1232,9 +1152,7 @@ { "cell_type": "code", "execution_count": 67, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "n_neurons = 100\n", @@ -1249,9 +1167,7 @@ { "cell_type": "code", "execution_count": 68, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "init = tf.global_variables_initializer()" @@ -1260,9 +1176,7 @@ { "cell_type": "code", "execution_count": 69, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "X_batch = np.random.rand(2, n_steps, n_inputs)" @@ -1271,9 +1185,7 @@ { "cell_type": "code", "execution_count": 70, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "with tf.Session() as sess:\n", @@ -1307,9 +1219,7 @@ { "cell_type": "code", "execution_count": 72, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "with tf.device(\"/gpu:0\"): # BAD! This is ignored.\n", @@ -1329,9 +1239,7 @@ { "cell_type": "code", "execution_count": 73, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "import tensorflow as tf\n", @@ -1357,9 +1265,7 @@ { "cell_type": "code", "execution_count": 74, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "reset_graph()\n", @@ -1374,9 +1280,7 @@ { "cell_type": "code", "execution_count": 75, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "devices = [\"/cpu:0\", \"/cpu:0\", \"/cpu:0\"] # replace with [\"/gpu:0\", \"/gpu:1\", \"/gpu:2\"] if you have 3 GPUs\n", @@ -1396,9 +1300,7 @@ { "cell_type": "code", "execution_count": 76, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "init = tf.global_variables_initializer()" @@ -1427,9 +1329,7 @@ { "cell_type": "code", "execution_count": 78, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "reset_graph()\n", @@ -1438,22 +1338,33 @@ "n_neurons = 100\n", "n_layers = 3\n", "n_steps = 20\n", - "n_outputs = 1\n", - "\n", - "X = tf.placeholder(tf.float32, [None, n_steps, n_inputs])\n", - "y = tf.placeholder(tf.float32, [None, n_steps, n_outputs])" + "n_outputs = 1" ] }, { "cell_type": "code", "execution_count": 79, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ - "keep_prob = 0.5\n", - "\n", + "X = tf.placeholder(tf.float32, [None, n_steps, n_inputs])\n", + "y = tf.placeholder(tf.float32, [None, n_steps, n_outputs])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note: the `input_keep_prob` parameter can be a placeholder, making it possible to set it to any value you want during training, and to 1.0 during testing (effectively turning dropout off). This is a much more elegant solution than what was recommended in earlier versions of the book (i.e., writing your own wrapper class or having a separate model for training and testing). Thanks to Shen Cheng for bringing this to my attention." + ] + }, + { + "cell_type": "code", + "execution_count": 80, + "metadata": {}, + "outputs": [], + "source": [ + "keep_prob = tf.placeholder_with_default(1.0, shape=())\n", "cells = [tf.contrib.rnn.BasicRNNCell(num_units=n_neurons)\n", " for layer in range(n_layers)]\n", "cells_drop = [tf.contrib.rnn.DropoutWrapper(cell, input_keep_prob=keep_prob)\n", @@ -1464,10 +1375,8 @@ }, { "cell_type": "code", - "execution_count": 80, - "metadata": { - "collapsed": true - }, + "execution_count": 81, + "metadata": {}, "outputs": [], "source": [ "learning_rate = 0.01\n", @@ -1484,78 +1393,29 @@ "saver = tf.train.Saver()" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Unfortunately, this code is only usable for training, because the `DropoutWrapper` class has no `training` parameter, so it always applies dropout, even when the model is not being trained, so we must first train the model, then create a different model for testing, without the `DropoutWrapper`." - ] - }, { "cell_type": "code", - "execution_count": 81, + "execution_count": 82, "metadata": {}, "outputs": [], "source": [ - "n_iterations = 1000\n", + "n_iterations = 1500\n", "batch_size = 50\n", + "train_keep_prob = 0.5\n", "\n", "with tf.Session() as sess:\n", " init.run()\n", " for iteration in range(n_iterations):\n", " X_batch, y_batch = next_batch(batch_size, n_steps)\n", - " _, mse = sess.run([training_op, loss], feed_dict={X: X_batch, y: y_batch})\n", - " if iteration % 100 == 0:\n", - " print(iteration, \"Training MSE:\", mse)\n", + " _, mse = sess.run([training_op, loss],\n", + " feed_dict={X: X_batch, y: y_batch,\n", + " keep_prob: train_keep_prob})\n", + " if iteration % 100 == 0: # not shown in the book\n", + " print(iteration, \"Training MSE:\", mse) # not shown\n", " \n", " saver.save(sess, \"./my_dropout_time_series_model\")" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now that the model is trained, we need to create the model again, but without the `DropoutWrapper` for testing:" - ] - }, - { - "cell_type": "code", - "execution_count": 82, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "reset_graph()\n", - "\n", - "n_inputs = 1\n", - "n_neurons = 100\n", - "n_layers = 3\n", - "n_steps = 20\n", - "n_outputs = 1\n", - "\n", - "X = tf.placeholder(tf.float32, [None, n_steps, n_inputs])\n", - "y = tf.placeholder(tf.float32, [None, n_steps, n_outputs])\n", - "\n", - "keep_prob = 0.5\n", - "\n", - "cells = [tf.contrib.rnn.BasicRNNCell(num_units=n_neurons)\n", - " for layer in range(n_layers)]\n", - "multi_layer_cell = tf.contrib.rnn.MultiRNNCell(cells)\n", - "rnn_outputs, states = tf.nn.dynamic_rnn(multi_layer_cell, X, dtype=tf.float32)\n", - "\n", - "learning_rate = 0.01\n", - "\n", - "stacked_rnn_outputs = tf.reshape(rnn_outputs, [-1, n_neurons])\n", - "stacked_outputs = tf.layers.dense(stacked_rnn_outputs, n_outputs)\n", - "outputs = tf.reshape(stacked_outputs, [-1, n_steps, n_outputs])\n", - "\n", - "loss = tf.reduce_mean(tf.square(outputs - y))\n", - "\n", - "init = tf.global_variables_initializer()\n", - "saver = tf.train.Saver()" - ] - }, { "cell_type": "code", "execution_count": 83, @@ -1566,8 +1426,15 @@ " saver.restore(sess, \"./my_dropout_time_series_model\")\n", "\n", " X_new = time_series(np.array(t_instance[:-1].reshape(-1, n_steps, n_inputs)))\n", - " y_pred = sess.run(outputs, feed_dict={X: X_new})\n", - "\n", + " y_pred = sess.run(outputs, feed_dict={X: X_new})" + ] + }, + { + "cell_type": "code", + "execution_count": 84, + "metadata": {}, + "outputs": [], + "source": [ "plt.title(\"Testing the model\", fontsize=14)\n", "plt.plot(t_instance[:-1], time_series(t_instance[:-1]), \"bo\", markersize=10, label=\"instance\")\n", "plt.plot(t_instance[1:], time_series(t_instance[1:]), \"w*\", markersize=10, label=\"target\")\n", @@ -1585,59 +1452,6 @@ "Oops, it seems that Dropout does not help at all in this particular case. :/" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Another option is to write a script with a command line argument to specify whether you want to train the mode or use it for making predictions:" - ] - }, - { - "cell_type": "code", - "execution_count": 84, - "metadata": {}, - "outputs": [], - "source": [ - "reset_graph()\n", - "\n", - "import sys\n", - "training = True # in a script, this would be (sys.argv[-1] == \"train\") instead\n", - "\n", - "X = tf.placeholder(tf.float32, [None, n_steps, n_inputs])\n", - "y = tf.placeholder(tf.float32, [None, n_steps, n_outputs])\n", - "\n", - "cells = [tf.contrib.rnn.BasicRNNCell(num_units=n_neurons)\n", - " for layer in range(n_layers)]\n", - "if training:\n", - " cells = [tf.contrib.rnn.DropoutWrapper(cell, input_keep_prob=keep_prob)\n", - " for cell in cells]\n", - "multi_layer_cell = tf.contrib.rnn.MultiRNNCell(cells)\n", - "rnn_outputs, states = tf.nn.dynamic_rnn(multi_layer_cell, X, dtype=tf.float32)\n", - "\n", - "stacked_rnn_outputs = tf.reshape(rnn_outputs, [-1, n_neurons]) # not shown in the book\n", - "stacked_outputs = tf.layers.dense(stacked_rnn_outputs, n_outputs) # not shown\n", - "outputs = tf.reshape(stacked_outputs, [-1, n_steps, n_outputs]) # not shown\n", - "loss = tf.reduce_mean(tf.square(outputs - y)) # not shown\n", - "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate) # not shown\n", - "training_op = optimizer.minimize(loss) # not shown\n", - "init = tf.global_variables_initializer() # not shown\n", - "saver = tf.train.Saver() # not shown\n", - "\n", - "with tf.Session() as sess:\n", - " if training:\n", - " init.run()\n", - " for iteration in range(n_iterations):\n", - " X_batch, y_batch = next_batch(batch_size, n_steps) # not shown\n", - " _, mse = sess.run([training_op, loss], feed_dict={X: X_batch, y: y_batch}) # not shown\n", - " if iteration % 100 == 0: # not shown\n", - " print(iteration, \"Training MSE:\", mse) # not shown\n", - " save_path = saver.save(sess, \"/tmp/my_model.ckpt\")\n", - " else:\n", - " saver.restore(sess, \"/tmp/my_model.ckpt\")\n", - " X_new = time_series(np.array(t_instance[:-1].reshape(-1, n_steps, n_inputs))) # not shown\n", - " y_pred = sess.run(outputs, feed_dict={X: X_new}) # not shown" - ] - }, { "cell_type": "markdown", "metadata": {}, From ceaef6e7a170a920b4653adf6453bc953f0c499c Mon Sep 17 00:00:00 2001 From: Chris Qlasty Date: Thu, 2 Nov 2017 18:30:30 +0100 Subject: [PATCH 19/26] Scikit-learn 19.0 updates on .fit( ) arguments Adopting code to my needs I have found that in the scikit-learn 19.0 they recommend to put params list directly into the .fit( ) methods. That also makes the code more understandable for me as now it is more clear where these values go to (fit( ) function of DNNClassifier). Hope this makes sense. --- 11_deep_learning.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/11_deep_learning.ipynb b/11_deep_learning.ipynb index 129a291..9b06e09 100644 --- a/11_deep_learning.ipynb +++ b/11_deep_learning.ipynb @@ -3310,9 +3310,9 @@ "}\n", "\n", "rnd_search = RandomizedSearchCV(DNNClassifier(random_state=42), param_distribs, n_iter=50,\n", - " fit_params={\"X_valid\": X_valid1, \"y_valid\": y_valid1, \"n_epochs\": 1000},\n", " random_state=42, verbose=2)\n", - "rnd_search.fit(X_train1, y_train1)" + "fit_params={\"X_valid\": X_valid1, \"y_valid\": y_valid1, \"n_epochs\": 1000},\n", + "rnd_search.fit(X_train1, y_train1, **fit_params)" ] }, { From ca61fc15ab08d52615e7c88022937715d1b40e71 Mon Sep 17 00:00:00 2001 From: Chris Qlasty Date: Thu, 2 Nov 2017 18:54:12 +0100 Subject: [PATCH 20/26] Scikit-learn 19.0 updates on .fit( ) arguments up. I have left unnecessary ',' at the end of the fit_params line, sorry for that. --- 11_deep_learning.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/11_deep_learning.ipynb b/11_deep_learning.ipynb index 9b06e09..a58a1ce 100644 --- a/11_deep_learning.ipynb +++ b/11_deep_learning.ipynb @@ -3311,7 +3311,7 @@ "\n", "rnd_search = RandomizedSearchCV(DNNClassifier(random_state=42), param_distribs, n_iter=50,\n", " random_state=42, verbose=2)\n", - "fit_params={\"X_valid\": X_valid1, \"y_valid\": y_valid1, \"n_epochs\": 1000},\n", + "fit_params={\"X_valid\": X_valid1, \"y_valid\": y_valid1, \"n_epochs\": 1000}\n", "rnd_search.fit(X_train1, y_train1, **fit_params)" ] }, From ef8f8a66fa28de0b0284717411906817db50e265 Mon Sep 17 00:00:00 2001 From: Chris Qlasty Date: Fri, 3 Nov 2017 12:00:28 +0100 Subject: [PATCH 21/26] 0.18 version left with 0.19 suggestions commented The previous version of the .fit( ) with the suggestion of how it should be done from the version 0.21. --- 11_deep_learning.ipynb | 2841 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 2769 insertions(+), 72 deletions(-) diff --git a/11_deep_learning.ipynb b/11_deep_learning.ipynb index a58a1ce..db13ebd 100644 --- a/11_deep_learning.ipynb +++ b/11_deep_learning.ipynb @@ -92,7 +92,25 @@ "cell_type": "code", "execution_count": 3, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Saving figure sigmoid_saturation_plot\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3Xl4Tdf6wPHvisyDOVJEDTXGPLXILTFXUUNoqbHaKlo/\nLUqvoVeq1dYY91YHnaKCKkUNNZaosYSG0opWY4gIgpDIJMn6/bGPNMMJCSc5Gd7P8+wn2Xuvs9d7\ntiPvWXuvvZbSWiOEEEIUNDbWDkAIIYQwRxKUEEKIAkkSlBBCiAJJEpQQQogCSRKUEEKIAkkSlBBC\niAJJEpR4IEqpIKXUR9aOA3IWi1LqhFJqRj6FlL7eAKXUxnyox0cppZVS5fOhrpFKqfNKqVRrnNNM\nsQxXSsVaMwaRd5Q8ByUyU0q5A37A00BFIBo4AXygtd5uKlMWuKO1jrFaoCY5iUUpdQJYrbWekUcx\n+AC7AHetdVS67aUw/p9FW7Cus8BHWuu56bbZA2WByzoP/1MrpcoAV4DxwGogRmudLwlCKaWB/lrr\n1em2OQFuWusr+RGDyF+21g5AFEjfA87Ai8BfQAWgHVDubgGt9XXrhJZVQYolM631zXyqJwmIzIeq\nqmL83diotb6UD/Xdk9Y6Hoi3dhwij2itZZElbQFKAxrodJ9yQRjf4u+uewDrMf5YnANewGh1zUhX\nRgOjgR+AOOA00B7wBLYCt4EQoFmmuvoCvwGJwAVgKqbWfzaxVDDVcTeWEZljMfN+HjO9JtIUx1Gg\nR6Yy9sAs0zETgb+B/wOqmd5b+iXA9JoAjD/mACOBy0CJTMddDqzPSRym95qhLtN2H9N6+Vyct7PA\nNOAz4BYQDrx5j3M03Mz7rAbMAE6YKRubbn2G6d9gAHAGiAHWpY/XVG5YupgvA0vSxZq+3rPm6jFt\newXji1WS6efLmfZr07/FKtM5/hsYbO3/e7JkXeQelMgs1rQ8o5RyzMXrlmB8u+4A9AIGm9YzmwZ8\nCzQGgk2/fwl8DDQFIjD+qAOglGqO8YdkDdAQeAv4N/DaPWIJAGoCnYDewFCMP6T34gpsBjqbYvse\nWKOUqpvpPQ7FuLxVD6OFGY3xx9/XVKY+xmXRcWbqWAWUMtVx9/25YpyvwBzG0RcjkbxjqqeiuTeT\ni/P2BkZCaAZ8CMxWSrU2d0xgJfCU6ffHTXVfyKasOdWA54A+QBeMf+/30sX8Ckay/BpohHGJ+YRp\nd0vTz5dN9d5dz0Ap1Qf4CPAHGgALgY+VUj0zFX0b44tAY9P7+kop9Wgu3ovID9bOkLIUvAXjj+11\nIAE4AMwFnshUJghTqwWog/GttFW6/VWAFLK2oN5Pt97AtG18um0+pGsJAMuAnZnqngGEZxNLbdPr\nvdPtr5o5lhyeh4PANNPvtUzHfSqbshniTrc9AFMLyrS+Bliabn0wcBNwzEkcpvWzwMR71Z/D83YW\nWJGpzJ/p6zITSwtTPdUyHTcnLagEoFS6bVOBv9Kth2Pc58yubg30u089+4CvzPwb7L3H59AWo0Uv\nragCtkgLSmShtf4eqAT0xPg23wY4qJSaks1L6gKpGC2iu8e4gNEayux4ut8vm37+ZmZbBdPPehh/\ndNLbC1RWSpU0c/x6plgOpYvlXDaxpFFKuSilZiulfldK3TD1DGsB3P1W3dR03F33Ok4OBAK9lVLO\npvVBwPda64QcxpFTOT1vxzOVieCfc29p53TGe3JpdSmlKgCVgZ8eso7s3rdXpm1p71trnQxcJe/e\nt3hAkqCEWVrrBK31dq31O1rrNhiX4WaYeos9jDvpq7nHtpx8Nu/VWy23PdnmAv2B6RgdQppgJLmH\nfb+ZbQKSgV6mP8qd+OfyXn7Fkf7c3DGzL7d/F1IBlWmbnZlylqjrQWX+PFgzFpFD8g8icup3jEsh\n5u5LncL4LDW/u0Ep5YnRCntYfwDembb9C+NSlblu5XdjeTxdLI/mIJZ/Ad9orb/XWh/HuNz0WLr9\nIabjts/m9UmmnyXuVYnWOhHj3tAgjPsxkRiXKHMax9267lkPuT9vD+Mq4KGUSp+kmuTmANroJn4R\n6HiPYnd48Pf9e27iEQWDJCiRgVKqnFJqp1JqsFKqkVKqulKqPzAJ+ElrfSvza7TWoRi98D5VSrVS\nSjXBuNEdR+5bMpnNA9oppWYopWorpQYBE4DZ5gqbYtkCfKaUam2KJYD7d0U+DfRRSjVTSjXEaNWk\nJWOt9WngO+ALpZSv6bw8qZQaYipyDuO9dldKuZs6P2QnEOgKjMK4B5Sa0zhMzgJPKqUq3+PB3Fyd\nt4cUhPEM1hSl1GNKqReBfg9wnPeA15VSb5hibqKUmpBu/1mgo1LqEdPzWObMAYYopV5VStVSSo3F\n+DKQF+9b5DFJUCKzWIyb8uOA3cBJjK7VyzG+8WdnOMa3/SCM7ubLMB7oTHiYYLTWRzEueflieljY\ntNxr5IjhQBiwE9hgiv3sfaoab4p3D8Z9t4Om39MbajrWfzFaagEYvfLQWl8E/oPxR/byfeLbg9Fa\n8CLj5b2cxvE2RieUMxitlywe8Lw9EK31HxiPD4zEuLfTGeMzk9vjfAK8itFT7wTGF4366YpMwGjB\nXgB+zeYY64CxGL0Tf8f4HI/RWm/IbTzC+mQkCZEnTN/sI4CBpk4XQgiRKzKShLAIpVQHwA2jR14F\njJZEFMa3YCGEyDWLXeJTSr2mlApWSiUqpQLuUW6YUuqIUuqWUirc1KVWEmXhZwe8i5GgNmDcf2qr\ntb5t1aiEEIWWxS7xKaX6YnQ37Qo4aa2HZ1NuNMb15V8Ad4z7Fau01h9YJBAhhBBFgsVaLlrrNQBK\nqRYYY6tlV+6TdKsXlVLLyL7rrhBCiGKqIFxaa4vRU8wspdRIjN5BODk5Na9SpUp+xZUjqamp2NhI\nZ8j7kfOUMxcuXEBrzaOPyrBw95Pfn6nIhEicSzhT0s7cACYFV0H8v3f69OkorbX7/cpZNUEppUZg\nDOPyUnZltNaLgcUALVq00MHBwdkVtYqgoCB8fHysHUaBJ+cpZ3x8fIiOjiYkJMTaoRR4+fmZmrx9\nMrP3z2aCzwTebvd2vtRpKQXx/55S6lxOylktQSmlegPvY0zrEHW/8kIIYQ3zD8xn9v7ZjGkxhult\np1s7nGLFKglKKfUU8DnQXWv92/3KCyGENSw7vowJ2ybQz6sf/+32XzKO5iTymsUSlKmruC3GWFkl\nTHMJJZtGCk5frgPGKAN9tNaHsh5JCCEKhrDoMNpXa09gn0BK2NxvGEBhaZa8czYNY7yztzDmuIkH\npimlHlVKxaabDGw6xvAwP5q2xyqlNlswDiGEeCgpqSkATGs7ja2Dt+Jg62DliIoniyUorfUMrbXK\ntMzQWp/XWrtqrc+byrXXWtuatt1dulkqDiGEeBihUaE0+KQBhy4aF3jsSpibOUTkh4LQzVwIIQqE\niJgIugZ2Je5OHGUcsxswXeQXSVBCCAFEJ0TzVOBTXIu/RtCwIGqVq2XtkIo9SVBCiGIvITmBXt/2\n4lTUKTY9v4nmlZrf/0UizxWsx4uFEMJKyjuX55s+39D5sc7WDkWYSAtKCFFsaa2JuxOHi70Lq/uv\nluecChhpQQkhii2/3X60/rI10QnRkpwKIElQQohi6dPgT/Hb7UeLSi0o5VDK2uEIMyRBCSGKne9/\n/54xm8bQo3YPFvdcLK2nAkoSlBCiWNlzbg/Pr3meVp6tWNlvJbY2ciu+oJIEJYQoVqqXqc4zdZ5h\n4/MbcbZztnY44h7kq4MQoli4HHuZ8s7l8Szpyar+q6wdjsgBaUEJIYq8K7ev8K+v/8UrG1+xdigi\nFyRBCSGKtJjEGLov787FWxcZ0XSEtcMRuSCX+IQQRVZSShK+3/ny66VfWfvcWtpUaWPtkEQuSIIS\nQhRZozeOZvvf2/nqma/oWaentcMRuSQJSghRZL3U7CUaeTTihaYvWDsU8QAkQQkhipzfLv9GQ4+G\ntK7SmtZVWls7HPGApJOEEKJIWRKyhEafNmLtH2utHYp4SJKghBBFxqbTm3hx/Yt0qtGJ7rW7Wzsc\n8ZAkQQkhioSD4Qfpv6o/TR5pwppn12Bfwt7aIYmHJAlKCFHoXY+/To/lPahcsjI/DvoRNwc3a4ck\nLEA6SQghCr2yTmVZ0HUB3o96U8GlgrXDERYiLSghRKF1Pf46v4T/AsCQxkOoUaaGlSMSlmTRBKWU\nek0pFayUSlRKBdyn7BtKqUil1C2l1FdKKQdLxiKEKNoSUhLouaInTy17iuiEaGuHI/KApVtQEcC7\nwFf3KqSU6gq8BXQEqgI1AD8LxyKEKKKSU5OZ+cdMDlw4wOc9P6e0Y2lrhyTygNJaW/6gSr0LeGqt\nh2ezfzlwVms9xbTeEVimtX7kXsd1c3PTzZs3z7Dt2WefZcyYMcTFxfH0009nec3w4cMZPnw4UVFR\n9OvXL8v+0aNH89xzz3HhwgWGDBmSZf+ECRPo2bMnoaGhvPJK1pGQe/bsyYQJEwgJCeH111/Psn/W\nrFm0adOG/fv3M2XKlCz7/f39adKkCTt27ODdd9/Nsv+zzz6jTp06bNiwgXnz5mXZv3TpUqpUqcLK\nlSv55JNPsuxfvXo15cuXJyAggICAgCz7f/zxR5ydnfn444/57rvvsuwPCgoCYO7cuWzcuDHDPicn\nJzZv3gzAzJkz+emnnzLsL1euHN9//z0AgwYN4uLFixn2e3p6EhgYCMDrr79OSEhIhv21a9dm8eLF\nAIwcOZLTp09n2N+kSRP8/f0BGDx4MOHh4Rn2t27dmvfffx8AX19frl27lmF/x44dmT59OgDdunUj\nPj4+w/4ePXowceJEAHx8fMgsLz57ISEhJCcn06JFi/t+9qZNm0anTp2K3WdPozlT/wwXK1zk46c/\nJmpL1D0/e//+9785cOBAhv3F6bPXqVMnSpfOmMAf9u/ew372du/efURr3SLLjkys1UmiPvBDuvVj\ngIdSqpzWOsO/pFJqJDASwM7OjujojE3506dPExQUREJCQpZ9AKdOnSIoKIibN2+a3X/y5EmCgoK4\ncuWK2f2//fYbbm5unD9/3uz++Ph4goKC+Ouvv8zuP3r0KElJSZw4ccLs/uDgYKKjozl27JjZ/b/8\n8guXLl3it99+M7v/wIEDnDlzhpMnT5rdv2/fPkqVKsWpU6fM7v/5559xdHTk9OnTZvff/SNx5syZ\nLPvvvneAsLCwLPtTU1PT9iclJWXZb2dnl7Y/PDw8y/6IiIi0/REREVn2h4eHp+2/fPlylv3nz59P\n23/16lVu3bqVYX9YWFja/uvXr5OYmJhh/5kzZ9L2mzs3efHZS05ORmtNdHT0fT97x44dw9bWtth9\n9m543uBihYsMqDiAerfr8U3YN/f87Jk7f8Xps5eSkpKlzIP83dPahtRUF1JSnNm69QJ//nmEv/++\nxPnz9UlNdUBrR1JTHUhNdeTDD6FMmbNcvOjAyZMjSU11JDXVEa0dSE11AJ7MUqc51mpBnQFe1Vpv\nMa3bAUlAda312eyO26JFCx0cHGzxeB9GUFCQ2W84IiM5Tznj4+NDdHR0lm/04h8pqSms+n0VHlc9\naN++vbXDKfCCgoJo186H27fh2jW4ft1Y0v9+4wbExBjLrVv//J5+PS7OklGpAt2CigVKplu/+3uM\nFWIRQhQCG09vpLFHY6qUqsKABgPSWhjFVXw8XL4MkZH//Ey/REUZSSgysg2xsXDnzsPX6eb2z+Lq\nCs7O4OT0z5J5PbttPXrkrD5rJaiTQGPg7oXnxsDlzJf3hBACYMffO+i7si++Xr6s8F1h7XDyXFIS\nhIfDuXNw/ryx3P39wgW4dAlu3szp0YwRNZycoGxZKFfO+Hl3KVcOSpeGkiX/ST7mfndxAZuH6FZ3\n+vRpzp8/T6dOnXL8GosmKKWUremYJYASSilHIFlrnZyp6DdAgFJqGUbPv2lAgCVjEUIUDUcijtBn\nZR/qlq/LJ92zdsYorKKj4c8/jeX0aePnmTNGEoqMhPvdfbGzAw8PeOSRf5b06+XLG8knNHQ/3bu3\nwckpf96XOcuXL+eFF16gWbNm1ktQGInmP+nWBwN+SqmvgN8BL631ea31FqXUbGAX4AR8n+l1QgjB\nX9f/4unlT1POqRxbBm8pdN3JtTYuv/32m7GcOAGhoUYyuno1+9eVKAGVK8Ojj0LVqsbPu0uVKsa+\nMmVAqfvHcO1aktWSU0JCAmPGjGHlypUkJSWRmpqaq9dbNEFprWcAM7LZ7Zqp7HxgviXrF0IULW9u\nf5OU1BS2Dt5KJbdK1g7nnpKTjQR05AgcP/5PUoqKMl/eyQlq1TKW2rWNnzVrQrVqULEi2Bbygej+\n/vtvnn76ac6fP5/WjT63nfIK+SkQQhRlAb0COHfzHHXK17F2KBloDWFhcOjQP8vRo0bHhcxKloQG\nDaBhQ2OpV89ISJUqPdw9nYJszZo1DBs2jLi4uAytJqu2oIQQ4mElJifywd4PmOQ9iVKOpWjk2Mja\nIZGcbCSg3bvh55/h4EHzLaPHHoOWLaFRo38S0qOP5uxSXFFw584d3njjDb766qssDx+DtKCEEIVY\nSmoKg9cOZvXvq2lRqYXVJh1MTTUS0o4dRlLauxdiYzOWcXeHxx//Z2nZ0uiUUFydP3+eHj168Ndf\nf5lNTiAJSghRSGmtGbdlHKt/X83cznPzPTlduQLbtsGWLcbPzJ0YatWCdu2MxdvbuFdUXFpG97Np\n0yYGDhxIXFwcKSkp2ZaTS3xCiEJp1p5ZLDq8iImtJzKhzYR8qfOPP2DNGli3DjIPUlO1KnTpAh06\nQNu2xj0jkdWUKVPw9/fPttWUnrSghBCFzrW4a/j/4s+QRkP4sPOHeVaP1kYvuzVrYO1aOHXqn32O\njkbr6KmnjKVOHWkh5UR4eDhaa0qUKHHP1hNIghJCFELlnMtx6KVDeJb0xEZZvmvbn3/C0qUQGGj0\nvrurbFl45hno2xc6djSG5RG588033zB16lSmTZvGxo0bSUxMzDYRSYISQhQae87tYdfZXUxvO53q\nZapb9NhRUbBypZGYfvnln+0VKxoJqW9f49JdYX/eqCCoU6cO3333HY0aNeLEiRPZlpMEJYQoFH67\n/BvPfPsMHi4evN7qdUo6lLz/i+5DawgKgk8/NS7h3R0g1dUVfH1hyBDw8TFGahCWtXPnTsLSN08x\n5oy7c+cOycnGaHfSSUIIUeCdiz7HU8uewtnOma2Dtz50coqOhiVLjMR0976SjY1xL2nIEOjdWy7f\n5bVJkyZx+/btDNsqVKiAj48PK1eu5M6dO9KCEkIUbFFxUXQN7MrtpNvseWEPVUtXfeBjnToF8+bB\nsmX/jOJQsSK8/LKxeHpaKGhxT7t37yY0NDTDNldXV2bPns2zzz7LzJkz8fPzy1FPv/QkQQkh8tXB\n8INcjLnIj8//SEOPhg90jH37YNq0Buzb98+2Tp1g9Gjo2dMY6VvknzfffDNL66ls2bL069cPgCpV\nqvDFF1/k+riSoIQQ+apH7R6cHXeWcs65G3YhNRU2bIDZs2H/foDyODjA8OHwxhtGt3CR//bs2cPJ\nkyczbHNxceGDDz7A5iEHGyyiQxUKIQqSVJ3KyA0jWfPHGoBcJSet4YcfoEkT417S/v3GVBNDhpzl\n3DnjvpMkJ+uZNGkScZnmgy9TpgzPPvvsQx9bEpQQIs+9teMtPj/6Ob9f/T3Hr9HaGHbo8ceNxPTb\nb8Y9JX9/Y1K/ESPO4uGRh0GL+9q/fz/Hjx/PsM3V1ZX333+fEhboKimX+IQQeWre/nnM2T+HV1u+\nytQnp+boNbt3w7RpxiCtYMwUO3Wq0fHB0TEPgxW5Yq71VLJkSQYMGGCR40uCEkLkmcDjgUzcPpH+\nXv1Z+NRC1H3GDvrrL5g40bikB8bo4JMnw6uvSjfxguaXX37h119/zbDN1dWVWbNmYWuhp58lQQkh\n8syxyGO0r9aepX2WUsIm+0s+N2/Cu+/CwoXGw7UuLjBpErz+ujHhnyh4Jk+enKX15OLiwqBBgyxW\nhyQoIYTFpepUbJQNszvPJiklCQdbB7PlUlLgq6+My3lXrhjbhg+HWbOM55lEwRQcHMyhQ4cybHN1\ndeW9996zWOsJpJOEEMLCQqNCafpZU367/BtKqWyT06+/whNPwMiRRnLy9obDh+HrryU5FXSTJ08m\nISEhwzYnJyeGDh1q0XokQQkhLObirYt0CezCpZhLONk5mS0TF2dcvmvZ0pj6okoV+PZb2LMHWrTI\n54BFrv36668cOHAgw7BFLi4uzJw5EzsLPyEtl/iEEBYRnRDNU8ue4nr8dYKGBVGzbM0sZbZtg1Gj\njCkvbGyMe0wzZxqDuYrC4a233jLbenrhhRcsXpckKCHEQ4u/E88zK54hNCqUHwf9SPNKzTPsv34d\nxo0z5mMCaNQIvvjCaEWJwuP48ePs2bMnS+vJz88Pe3t7i9dn0Ut8SqmySqm1SqnbSqlzSqnnsynn\noJT6VCl1WSl1XSm1QSlV2ZKxCCHyT3JqMo62jizts5RONTpl2LdtGzRoYCQnR0f44ANjenVJToXP\nW2+9RWJiYoZtDg4OvPjii3lSn6VbUIuAJMADaAJsUkod01qfzFRuHNAaaATcBBYD/wP6WjgeIUQe\n0lqTmJKIm4MbWwdvzfCcU1yc8QzTRx8Z697eEBAANbNe+ROFwPXr19myZUuW1tOMGTNwcDDfEeZh\nWawFpZRyAXyB6VrrWK31XmA9MMRM8erAVq31Za11ArASqG+pWIQQ+cNvtx8+AT7EJMZkSE5HjkDz\n5kZysrU1uo3v3i3JqTArW7Ys27dvp2nTpri4uABgb2/Pyy+/nGd1WrIFVRtI1lqfTrftGNDOTNkv\ngYVKqUpANDAI2GzuoEqpkcBIAA8PD4KCgiwY8sOLjY0tcDEVRHKeciY6OpqUlJRCca5+iPgB/z/9\n6fZIN4L3B6OUIjUVVqx4lK+/rkZKig1Vq95mypQ/qF07lj17LFu/fKZyxpLnqUSJEsyfP5+QkBA+\n//xzunXrxsGDBy1ybLO01hZZgCeByEzbXgaCzJQtBXwLaCAZ+BUoe786mjdvrguaXbt2WTuEQkHO\nU860a9dON27c2Nph3Neqk6u0mqF0j+U99J2UO1prra9e1fqpp7Q2hnnV+v/+T+u4uLyLQT5TOVMQ\nzxMQrHOQVyzZSSIWyDwoSUkgxkzZRYADUA5wAdaQTQtKCFGw7D67m0FrBtG6SmtW9luJrY0tBw5A\n06bG6OPlysGPPxrDFjmZfxRKiByxZII6DdgqpWql29YYyNxBAowOFAFa6+ta60SMDhKPK6XKWzAe\nIUQeqFyyMl0e68KGgRtwsnXG3x/atoXwcGjd2hghols3a0cpigKLJSit9W2MltA7SikXpZQ30AtY\naqb4YWCoUqqUUsoOGANEaK2jLBWPEMKyouKi0FpTs2xNNgzcQImksvTvb8xmm5xs/AwKMkaGEMIS\nLD3U0RjACbgCrABGa61PKqWeVErFpis3EUgA/gSuAk8DfSwcixDCQq7cvkLrL1szfut4AP78E1q1\ngu+/N0YbX70a5s+HPHhWUxRjFn0OSmt9HehtZvsewDXd+jWMnntCiAIuJjGGp5c9zcVbF3m2/rP8\n9BP07w83bhgP4K5dK93HCyIfHx8aNGhAv379rB3KA5PBYoUQ2UpKSaLvd30JiQzhu36rOPJDa7p2\nNZJTz56wf3/RSk5Xr15lzJgxVKtWDQcHBzw8POjYsSPbt2/P0euDgoJQShEVlX93KwICAnA1M5jh\nmjVreP/99/MtjrwgY/EJIbL18oaX2fH3Dj5/egkb5ndn8WJj+7//bUwwaFPEvuL6+voSFxfHl19+\nSc2aNbly5Qq7d+/m2rVr+R5LUlLSQ41vV7ZsWQtGYx1F7OMlhLCkwQ0HM7PVIpZNGsrixeDgAMuW\nGSNDFLXkFB0dzZ49e/jggw/o2LEjVatWpWXLlkycOJEBAwYAEBgYSMuWLXFzc6NChQr079+fixcv\nAnD27Fnat28PgLu7O0ophg8fDhiX21577bUM9Q0fPpwePXqkrfv4+DB69GgmTpyIu7s73t7eAMyf\nP59GjRrh4uJC5cqVeemll4iOjgaMFtsLL7zA7du3UUqhlGLGjBlm66xWrRrvvvsur7zyCiVLlsTT\n05M5c+ZkiOn06dO0a9cOR0dH6tSpw48//oirqysBAQGWOcm5VMQ+YkIISwiNCgWgtm1nlr8xhqAg\nYxLBPXvgebNDQBd+rq6uuLq6sn79+izTSdyVlJSEn58fx44dY+PGjURFRTFw4EAAqlSpwvfffw/A\nyZMnuXTpEgsXLsxVDIGBgWit2bNnD9988w0ANjY2+Pv7c/LkSZYvX86hQ4cYO3YsAG3atMHf3x9n\nZ2cuXbrEpUuXmDhxYrbHX7BgAQ0bNuTo0aNMnjyZSZMmceDAAQBSU1Pp06cPtra2HDx4kICAAPz8\n/LIMDpuf5BKfECKDgJAAXlz/Iv9t8jPvvuJNZKTRGWLzZvD0tHZ0ecfW1paAgABefvllFi9eTNOm\nTfH29qZ///488cQTAIwYMSKtfI0aNfjkk0+oV68e4eHheHp6pl1Wq1ChAuXL5/6xzurVqzNv3rwM\n215//fW036tVq8bs2bPp1asXS5Yswd7enlKlSqGU4pFHHrnv8bt06ZLWqho7diz//e9/+emnn2jd\nujXbt28nNDSUbdu2UbmyMbnEggUL0lpy1iAtKCFEmk2nN/HS+pdocvtN3hrUhshI8PExWk5FOTnd\n5evrS0REBBs2bKBbt27s37+fVq1aMWvWLACOHj1Kr169qFq1Km5ubrQwTQF8/vx5i9TfvHnzLNt2\n7txJ586d8fT0xM3Njb59+5KUlERkZGSuj9+oUaMM65UqVeLKlSsAnDp1ikqVKqUlJ4CWLVtiY8Vr\nuZKghBAAHLhwgP6r+lPl7FSOz3+f2FjFgAHG8EWlS1s7uvzj6OhI586defvtt9m/fz8vvvgiM2bM\n4ObNm3Tt2hVnZ2eWLl3K4cOH2bJlC2Bc+rsXGxubDNNUANy5cydLubujhN917tw5unfvTr169Vi1\nahVHjhxrAd4+AAAgAElEQVThq6++ylGd5mSekt0Y4Dc118fJL5KghBBcuX2FHit64PzLO5z92o/k\nZMWbbxodIvJoqp9Cw8vLi+TkZEJCQoiKimLWrFm0bduWunXrprU+7rrb6y4lJSXDdnd3dy5dupRh\n27Fjx+5bd3BwMElJSSxYsIDWrVtTu3ZtIiIistSZub4HUbduXSIiIjIcPzg42KoJTBKUEAJ35wq0\nOLmdaxsmopQx0Ovs2UWvp969XLt2jQ4dOhAYGMjx48cJCwtj1apVzJ49m44dO+Ll5YWDgwMfffQR\nf//9N5s2bWL69OkZjlG1alWUUmzatImrV68SG2sMoNOhQwc2b97M+vXrCQ0NZfz48Vy4cOG+MdWq\nVYvU1FT8/f0JCwtjxYoV+Pv7ZyhTrVo1EhIS2L59O1FRUcTFxT3Q++/cuTN16tRh2LBhHDt2jIMH\nDzJ+/HhsbW0zzPWVn4rRx08Ikdn1+OuEXDrOmDGw7Ztm2Noarab/+z9rR5b/XF1dadWqFQsXLqRd\nu3bUr1+fKVOm8Pzzz7Ny5Urc3d1ZsmQJ69atw8vLCz8/P+bPn5/hGJUrV8bPz4+pU6fi4eGR1iFh\nxIgRaYu3tzdubm706XP/0d0aNWrEwoULmT9/Pl5eXnzxxRfMnTs3Q5k2bdowatQoBg4ciLu7O7Nn\nz36g929jY8PatWtJTEzk8ccfZ9iwYUydOhWlFI6Ojg90zIeWkzk5Csoi80EVXnKeciY/54O6nXRb\nt/rsX9qh2XcatHZw0HrDhnyp2iLkM5UzD3OeQkJCNKCDg4MtF5DO+XxQ0s1ciGIoOTWZfisGcXDB\neDjVB1dXWL8eTM+ZimJq7dq1uLi4UKtWLc6ePcv48eNp3LgxzZo1s0o8kqCEKGa01ryw6jU2+42G\nv7tQpozxjJPpUR9RjMXExDB58mQuXLhAmTJl8PHxYcGCBVa7ByUJSohi5rMDgQROHgDnfPDwgG3b\nINPjMaKYGjp0KEOHDrV2GGkkQQlRjMTGwvK3BsE5GypV0uzapahd29pRCWGe9OITophY/etWOnVJ\nYs8eGypXht27JTmJgk1aUEIUAz8c28mzvV3R5+3x9IRdu4rWPE6iaJIEJUQRtzv0V/o+44w+34rK\nnqkEBdnw2GPWjkqI+5NLfEIUYSHn/qJT1zuknm9FJc9kft4tyUkUHpKghCii4uOhY7dYks89TsXK\nd9iz25YaNawdlRA5JwlKiCIoMRH69oXrfzShvMcdfg6yk+QkCh1JUEIUMbHxiTTvEsqWLVC+POze\naScdIkShJAlKiCIk6U4KXp0Pc/LnOriWvMP27eDlZe2ohHgwFk1QSqmySqm1SqnbSqlzSqnn71G2\nmVLqZ6VUrFLqslJqnCVjEaK4SUnRNOnxCxf2/QsH5yR2bLOjSRNrRyXEg7N0N/NFQBLgATQBNiml\njmmtT6YvpJQqD2wB3gBWA/ZAMZhQWoi8oTV49w/mj21tsHVIYttmexlbTxR6FmtBKaVcAF9gutY6\nVmu9F1gPDDFTfDywVWu9TGudqLWO0Vr/YalYhChOtIax42P5ZW1LbGzvsHG9LW3bWjsqIR6eJVtQ\ntYFkrfXpdNuOAe3MlG0F/KaU2g/UBH4BXtVan89cUCk1EhgJ4OHhQVBQkAVDfnixsbEFLqaCSM5T\nzkRHR5OSkpKrcxUY+ChfflmDEiVS+c9/TuJgH01xONXymcqZwnyeLJmgXIFbmbbdBNzMlPUEmgGd\ngd+A2cAKwDtzQa31YmAxQIsWLbSPj4/lIraAoKAgClpMBZGcp5wpXbo00dHROT5Xkz74ky+/rIFS\nsGyZDc89V3xuOslnKmcK83myZIKKBUpm2lYSiDFTNh5Yq7U+DKCU8gOilFKltNY3LRiTEEXWgq/O\nMWeK8XDT/IWJPPecg5UjEsKyLNmL7zRgq5SqlW5bY+CkmbLHAZ1uXZspI4TIxooNkYwf+QjoEoz/\ndzSvj5XkJIoeiyUorfVtYA3wjlLKRSnlDfQClpop/jXQRynVRCllB0wH9krrSYj727n/BoOfdYUU\nBwaMuMbc90pbOyQh8oSlH9QdAzgBVzDuKY3WWp9USj2plIq9W0hrvROYAmwyla0JZPvMlBDC8Ndf\n0LenE6kJrnToeZVln5fDSrNxC5HnLPoclNb6OtDbzPY9GJ0o0m/7BPjEkvULUZRdugRdusDN6460\n7ZDI5tXu2MhYMKIIk4+3EIXA9RupNGwTTlgYtGwJG9c5YG9v7aiEyFuSoIQo4OLjoXHbs1w760n5\nR6PYtAnczD28IUQRIwlKiAIsORladv2T8BM1cCl3g8O7y+Hubu2ohMgfkqCEKKC0ho79/+LknlrY\nucSyf1dJqlWTHhGi+JAEJUQBNWUK/LyuJjb2CWzfbE+jhiWsHZIQ+UoSlBAF0Lx5mg8+AFtbzerV\n0O5J6REhih9JUEIUMJfjujBxonEp7+uvFX16Olo5IiGsQxKUEAVI5M3mRP41C4Ap711l8GArBySE\nFUmCEqKA2LLrFqEnZoK25YWxkbw3RbrrieJNEpQQBUDwrwn07KEg2Rm3Siv5cuEj1g5JCKuTBCWE\nlZ07B890tyc5zg3XSjuoXn62jK8nBBYei08IkTtXrmg6d4FLl2xo106TmjqbW7dSrB2WEAWCJCgh\nrCQmBpq2jSDidGUaNkrlhx9s6NUrKUu59evXExISQsOGDalfvz6PPfYYJUrIM1Gi6JMEJYQVJCbC\n450uEBFaBTePK2zd4k6pUubLnjlzBj8/P1xdXUlJSSEpKQlPT08aNmzI448/ToMGDahfvz7Vq1eX\nxCWKFElQQuSzlBTweSacU4eq4FDqBof3lKVixexvOo0ePZp3332X69evp20LCwsjLCyMH3/8EWdn\n5wyJq1GjRjz++OMMGDCAGjVq5MdbEiJPSCcJIfKR1tB7yEUObvOkhFMMQTscqVPr3t8THR0deffd\nd3FxccmyLzk5mVu3bnH79m3u3LlDWFgYP/zwA9OmTePAgQN59TaEyBeSoITIR9OmwcYVlbGxS+SH\n9ZpWLZxy9LqXXnoJV1fX+xcE7O3t6dq1K88/L5NUi8JNEpQQ+eS92XHMmgUlSsC67x3o3qlkjl9r\nZ2fHhx9+aLYVlVnJkiVZtmwZSvqqi0JOEpQQ+eCjz28xbbIzAF99BT175v4YgwcPpmzZsvcs4+Dg\nQK1atR4kRCEKHElQQuSx79bEMXaUkZzG/ecsQ4c+2HFKlCjB3Llz79mKSkxM5MiRI9SpU4e9e/c+\nWEVCFBCSoITIQz/tusPAASUg1ZbnRv+F/4xqD3W8fv36UbFixXuWSUpKIioqii5dujB9+nRSUuTB\nX1E4SYISIo+EhEC3HndIveOAj28oKxbVfOhj2tjYsGDBgiytKEfHrFNyxMfHM3/+fJ544gnCw8Mf\num4h8ptFE5RSqqxSaq1S6rZS6pxS6p7diJRS9kqpP5RS8r9HFCl//gldu8KdOGcadwhlx8o6Fhtf\nr3v37lSvXj1t3dnZmVdffRVXV1dsbDL+l46LiyMkJAQvLy/WrVtnmQCEyCeWbkEtApIAD2AQ8IlS\nqv49yr8JXLVwDEJYVUQEdOh0hytXoHNn+OXHOlhygAelFP7+/jg7O+Pk5MTAgQOZO3cuJ06coGHD\nhjg7O2con5KSQkxMDIMGDeKll14iPj7ecsEIkYcslqCUUi6ALzBdax2rtd4LrAeGZFO+OjAYeN9S\nMQhhbVeuQAvvaMLP21Gv8S3WrAEHB8vX07FjR+rXr0/FihX53//+B0DVqlUJDg5m7NixODllfb4q\nLi6O5cuX06BBA37//XfLByWEhSmttWUOpFRTYJ/W2jndtolAO611lk61SqmNwJfADSBQa+2ZzXFH\nAiMBPDw8mn/77bcWiddSYmNjc/wAZXFWHM7TrVu2jBpXi0tnPXCq+BdLF4VTrkzujvH666+TkpKS\nlnTu5e7QR+a6noeEhPD2228THx9PcnJyhn1KKezt7RkzZgw9e/YstM9LFYfPlCUUxPPUvn37I1rr\nFvctqLW2yAI8CURm2vYyEGSmbB9gs+l3HyA8J3U0b95cFzS7du2ydgiFQlE/Tzdval2v8S0NWjt6\nnNVnzsc+0HHatWunGzdubJGYrl69qjt06KCdnZ01kGVxdnbW3bt31zdu3LBIffmtqH+mLKUgnicg\nWOfgb74l70HFApkfjS8JxKTfYLoUOBv4PwvWLYTV3L4NXbol8scxN2zLXeDgz67UqHL/ER/yWvny\n5dmxYwfvvfdetpf8duzYQe3atdm/f78VIhTi3iyZoE4Dtkqp9I+xNwZOZipXC6gG7FFKRQJrgIpK\nqUilVDULxiNEnktIgN694Zf9DpRyj+GnHdC4djlrh5VGKcXrr7/OgQMHqFKlSpbu6ImJiVy9epVO\nnToxY8YMeWZKFCgWS1Ba69sYyeYdpZSLUsob6AUszVT0BFAFaGJaXgIum36/YKl4hMhrSUnQu28S\nO3ZAhQrwyx432japYu2wzGrcuDF//PEHvr6+WXr5gfHM1Jw5c2jTpg0XL160QoRCZGXpbuZjACfg\nCrACGK21PqmUelIpFQugtU7WWkfeXYDrQKppXb6+iUIhORkGPp/M1s32KOcbbNycQJ061o7q3lxc\nXAgMDOSLL77I9pmpo0eP4uXlxfr1660UpRD/sGiC0lpf11r31lq7aK0f1VovN23fo7U2241Eax2k\ns+nBJ0RBlJICw19IZc33tuBwkw8CjtKyWdaRHAqqgQMHcvz4cerXr5+lNXV3fqmBAwfyyiuvkJCQ\nYKUohZChjooUHx8fXnvtNWuHUaSlpMALL2iWBdqAXSwTF+1iUv+O1g4r16pXr86RI0cYM2ZMth0o\nli5dSsOGDTl16pQVIhRCEhRXr15lzJgxVKtWDQcHBzw8POjYsSPbt2/P0etDQkJQShEVFZXHkf4j\nICDA7HMNa9as4f335bnnvJKSAsOGwdKlCuxiGT73O+a82NvaYT0wOzs75syZw4YNGyhTpgx2dnYZ\n9sfHx3PmzBmaN2/OF198cfcRESHyTbFPUL6+vhw6dIgvv/yS06dPs3HjRrp168a1a9fyPZakpKSH\nen3ZsmVxc3OzUDQiveRkGDoUli0DV1fNm5/8xFdjX7B2WBbRsWNHQkND8fb2znLJT2tNXFwc48aN\no1evXty8edNKUYpiKScPSxWUxdIP6t64cUMDevv27dmWWbp0qW7RooV2dXXV7u7uul+/fjo8PFxr\nrXVYWFiWhx+HDRumtTYeuHz11VczHGvYsGG6e/fuaevt2rXTo0aN0hMmTNDly5fXLVq00FprPW/e\nPN2wYUPt7OysK1WqpF988cW0hyl37dqVpc7//Oc/ZuusWrWqnjlzph45cqR2c3PTlStX1rNnz84Q\nU2hoqG7btq12cHDQtWvX1ps2bdIuLi7666+/fqBzmp2C+LBgTt25o/XAgVqD1q6uqXrv3ryry5IP\n6uZWamqqnjt3rnZycjL7YK+Dg4P28PDQBw4csEp8mRXmz1R+KojnCSs8qFvouLq64urqyvr167O9\nGZyUlISfnx/Hjh1j48aNREVFMXDgQACqVKmCn58fACdPnuTSpUssXLgwVzEEBgaitWbPnj188803\ngDGlgr+/PydPnmT58uUcOnSIsWPHAtCmTZu0gUIvXbrEpUuXmDhxYrbHX7BgAQ0bNuTo0aNMnjyZ\nSZMmceDAAQBSU1Pp06cPtra2HDx4kICAAPz8/EhMTMzVeyjKkpNhyBBYsQKwj6Hj9Ll4e1s7qryh\nlGLChAns27ePypUrm31m6vLly3To0IGZM2eSmppqpUhFsZGTLFZQlrwY6mj16tW6TJky2sHBQbdq\n1UpPmDBBHzx4MNvyf/zxhwb0hQsXtNZaL1iwQAP66tWrGcrltAXVsGHD+8a4efNmbW9vr1NSUrTW\nWn/99dfaxcUlSzlzLagBAwZkKFOzZk09c+ZMrbXWW7Zs0SVKlEhrEWqt9b59+zQgLSitdUKC1n36\nGC0nHG7qxyYO0dHx0XlapzVbUOnFxMToAQMG3HOYpFatWumIiAirxVgYP1PWUBDPE9KCyhlfX18i\nIiLYsGED3bp1Y//+/bRq1YpZs2YBcPToUXr16kXVqlVxc3OjRQtjfMPz589bpP7mzZtn2bZz5046\nd+6Mp6cnbm5u9O3bl6SkJCIjI3N9/EaNGmVYr1SpEleuXAHg1KlTVKpUicqVK6ftb9myZZbnY4qj\n27fhmWdg7VpQjjd5ZPQwfn77A0o5lrJ2aPnC1dWVFStWsHjxYlxcXMw+MxUcHEzdunXZtGmTlaIU\nRZ38JcKYjbRz5868/fbb7N+/nxdffJEZM2Zw8+ZNunbtirOzM0uXLuXw4cNs2bIFuH+HBhsbmyy9\nnu7cuZOlXOaZUc+dO0f37t2pV68eq1at4siRI3z11Vc5qtOczD2zlFJyaeY+oqONyQa3bQM7txuU\nGtWb3dM+pJJbJWuHlu8GDRrE8ePHqVevXrbPTPXv359XX31VLg0Li5MEZYaXlxfJycmEhIQQFRXF\nrFmzaNu2LXXr1k1rfdxla2sLkGUMM3d3dy5dupRh27Fjx+5bd3BwMElJSSxYsIDWrVtTu3ZtIiIi\nMpSxt7e3yJhpdevWJSIiIsPxg4ODi3UCu3oVOnSAffvA0xO270rkp0nzqF2utrVDs5oaNWrw66+/\nMnLkSLPPTMXHx7N48WK2bdtmhehEUVasE9S1a9fo0KEDgYGBHD9+nLCwMFatWsXs2bPp2LEjXl5e\nODg48NFHH/H333+zadMmpk+fnuEYHh4eKKXYtGkTV69eJTY2FoAOHTqwefNm1q9fT2hoKOPHj+fC\nhfsPNVirVi1SU1Px9/cnLCyMFStW4O/vn6FMtWrVSEhIYPv27URFRREXF/dA779z587UqVOHYcOG\ncezYMQ4ePMj48eOxtbUttHMEPYzwcGjbFn79Fcp7RrP75xTaNX+EZhWbWTs0q7Ozs2PBggWsW7eO\n0qVLZ2iZ29nZ0aZNG7p3727FCEVRVKwTlKurK61atWLhwoW0a9eO+vXrM2XKFJ5//nlWrlyJu7s7\nS5YsYd26dXh5eeHn58f8+fMzHMPd3R0/Pz+mTp2Kh4dH2kgOI0aMSFu8vb1xc3OjT58+942pUaNG\nLFy4kPnz5+Pl5cUXX3zB3LlzM5Rp06YNo0aNYuDAgbi7uzN79uwHev82NjasXbuWxMREHn/8cYYN\nG8bUqVNRSmXpwVXUhYbCk0/CqVNQ6tFzRD1Xlwtqr7XDKnC6dOlCaGgorVq1Srvk5+LiwqpVq+Te\npbC8nPSkKCiLTFiY90JCQjSgg4ODLXrcgnye9u7VumxZo7eeR52/NZPK6Pn751slloLSi+9+UlJS\n9Icffqjt7Oz0tm3brBJDQf5MFSQF8TyRw158ttZOkMK61q5di4uLC7Vq1eLs2bOMHz+exo0b06xZ\n8bistXYtPP+8Ma9T7danOd2+KZN8XuON1m9YO7QCzcbGhkmTJjFu3DgcHBysHY4ooqRNXszFxMTw\n2muv4eXlxaBBg6hXrx5bt24tFvegPvoIfH2N5DTohVjOP9WcYS3780GnD6wdWqEhyUnkJWlBFXND\nhw5l6NCh1g4jX6Wmwr//DXdv3b37LkyZ4sqkK/uoV75esUjOQhQGkqBEsRIXBy+8AN99B7a28OYH\noTzSfi9KvUgjj0b3P4AQIt/IJT5RbISHGz31vvsO3Nzgv0v/5uM7TzDvwDwSkmViPmupVq1alp6q\nQoC0oEQxcfAg9OkDkZHw2GPwSeBFhu37F672rmwZvAVH2+LVrT6/DR8+nKioKDZu3Jhl3+HDh7OM\nqCIEFIMWVGRkJN26dWPZsmUyFEsxtXQp+PgYyal9e9i0M4rXgjsQnxzP1sFbebTUo9YOsVhzd3fP\nMoySNTzsfGzC8op8gvr444/ZsWMHo0aNwt3dnTfeeCPLEESiaEpJgcmTjYkGExNhzBjYuhUOXN/I\nhZsX2DhwI/Ur1Ld2mMVe5kt8SikWL15M//79cXFxoUaNGgQGBmZ4zcWLF3nnnXcoU6YMZcqUoXv3\n7vz5559p+8+cOUOvXr145JFHcHFxoVmzZllab9WqVWPGjBmMGDGC0qVLM2jQoLx9oyLXinSCSk5O\nZtGiRSQnJxMbG0tMTAyLFi1iyZIl1g5N5LHLl6FLF6OnXokS8PHHsGgR2NnB8CbDCX0tFO9Hi+jE\nTkXAO++8Q69evTh27BjPPfccI0aMSJtBIC4ujvbt22Nvb8/u3bs5cOAAFStWpFOnTmnDfsXGxtKt\nWze2b9/OsWPH8PX1pW/fvpw6dSpDPfPnz6du3boEBwenzWAgCo4inaA2bdqUZQRxGxsbBg8ebKWI\nRH74+Wdo2hR27gQPD9ixA14ZlcprP77G/gv7AahSqoqVoxT3MmTIEAYPHkzNmjWZOXMmtra2/Pzz\nzwB8++23aK2ZPHkyjRo1om7dunz22WfExsamtZIaN27MqFGjaNiwITVr1mTq1Kk0a9aM1atXZ6in\nXbt2TJo0iZo1a1KrVq18f5/i3op0gpo9ezYxMTEZtj355JN4enpaKSKRl1JT4YMPjPtMly79M/Cr\njw9M2j6JRYcX8fO5n60dpsiB9POY2dra4u7unjaTwJEjRwgLC+Ppp59OmxW7VKlS3LhxgzNnzgBw\n+/ZtJk2ahJeXF2XKlMHV1ZXg4OAs87jdnd9NFEwW7cWnlCoLfAl0AaKAf2utl5sp9yYwDKhqKvex\n1nqOJWP5+++/OXr0aIZtbm5u95weXRRe16/DsGFw9zbDW2/BzJnGs05z989l3oF5vNbyNSZ7T7Zu\noCJH7jWPWWpqKk2aNOGNN97giSeeyFCubNmyAEycOJEtW7Ywd+5catWqhbOzM0OHDs3SEUJ6DxZs\nlu5mvghIAjyAJsAmpdQxrfXJTOUUMBQ4DjwGbFNKXdBaf2upQP73v/9lmTPJ2dmZzp07W6oKUUDs\n3Gkkp/BwKFMGvvkGevQw9n1z7Bve3P4mz9Z/Fv+n/GWUiCKgWbNmrFixglKlSlGzZk2zZfbu3cvQ\noUPx9fUFICEhgTNnzlC7dvGd16swsliCUkq5AL5AA611LLBXKbUeGAK8lb6s1jr9/BChSqkfAG/A\nIgkqMTGRL7/8MsP9JycnJ8aNGydTAhQhCQkwZQosWGCsP/EEfPstVKtmrGut2fzXZjpU78A3vb+h\nhE0Jq8Uq4NatW4SEhGTYVrp06VwfZ9CgQcydO5epU6fi5ubGo48+yoULF/jhhx8YNWoUtWrVonbt\n2qxdu5ZevXphZ2eHn58fCQnyMHZhY8kWVG0gWWt9Ot22Y0C7e71IGV9pnwQ+y2b/SGAkGJMDBgUF\n3TeQHTt2kJycnGFbcnIy9erVy9HrcyM2NtbixyyKLH2e/vrLhffe8+LsWRdsbDRDh55l8ODznD2r\nOXvWSE5KKV4q+xJJpZM4sPeAxerOS9HR0aSkpBS5z1RkZCR79uyhadOmGba3bds2rXWT/j2fPHmS\n8uXLp61nLvP+++/z8ccf07t3b27fvk25cuVo0qQJv//+OxcvXqR///7MmTMHb29vXF1d6devH15e\nXkRGRqYdw1y9RVGh/huVkzk5crJgJJnITNteBoLu8zo/jETmcL86cjofVJMmTTSQtiildK9evXI6\nVUmuFMS5VgoiS52n5GStP/xQazs7Y/6mWrW0/uWXjGX+uPqHbvd1O33h5gWL1JmfCst8UAWB/N/L\nmYJ4nrDCfFCxQMlM20oCMWbKAqCUeg3jXtSTWmuLDPNw4sQJQkNDM2xzdnaWzhFFwPHj8PLLcOiQ\nsT56NMyZA+nvc1+8dZGugV1JSE4gMVlGDhGiMLPkDZnTgK1SKv3DBI2BzB0kAFBKjcC4N9VRax1u\nqSD8/f2z9NQpX7483t7yUGZhFR9vTI/RvLmRnCpXNnrrffxxxuR0I/4GTy17ihvxN9gyaAuPlX3M\nekELIR6axRKU1vo2sAZ4RynlopTyBnoBSzOXVUoNAmYBnbXWf1sqhtu3b7N8+fIMvfecnZ2ZMGGC\n9N4qpH76CRo2NJ5vSkmBV1+F33+H7t0zlou/E88z3z7D6WunWTdgHU0rNjV/QCFEoWHpLm1jACfg\nCrACGK21PqmUelIpFZuu3LtAOeCwUirWtHz6sJUvX748Sy+91NTUYjchX1Fw6ZLRdbxTJzhzBurX\nh337jFlwS2a+kAzEJMUQmxTL0j5L6VC9Q/4HLISwOIs+B6W1vg70NrN9D+Cabr26Jes1HZM5c+Zw\n+/bttG02Njb4+vpSqlQpS1cn8khCgtFtfNYsiI0FBweYPh3efBPs7bOW11qTqlOp4FKBwy8fxtZG\nZpARoqgoMg8FBQcHExERkWGbo6Mj48ePt1JEIje0hu+/By8v49mm2Fjo1QtOnICpU80nJ4D/BP0H\n3+98SUpJkuQkRBFTZBLUvHnziI+Pz7CtatWqNGvWzEoRiZwKDjbGz+vXD8LCoEEDY4DXdesgm4EC\nAFh0aBEzf55Jeefy2NnYZV9QCFEoFYkEdePGDX744Ye0sboAXF1dpWt5AXf8uDHLbcuWsHs3lCtn\n9Mz79Vfo2PHer111chVjN4/lmTrP8GmPT6UTjBBFUJG4JhIQEJClc4TWmgEDBlgpInEvp07BjBmw\ncqWx7uQEY8caA7yWKXP/1+8M28ngtYPxftSbb32/lUt7QhRRhf5/ttaa+fPnp01UBsbw/EOGDCkQ\n00iLf/z+O3z4IQQGGlNj2NsbD9u+9RY88kjOj+Ni50Jrz9asfW4tTnZOeRewEMKqCn2C2r17N9HR\n0Rm22dnZMW7cOCtFJNLTGvbuhSlTGnDANByera0xIsTUqVAlF/MGxiTG4ObgxhOeT7Br2C65rCdE\nEVfo70HNmzeP2NjYDNu8vLyoW7eulSISYDxUu2YNtGljTBx44EB5HB1hzBgIDYVPP81dcroce5mm\nn8yIIesAAA6sSURBVDVl9j5jIHxJTkIUfYUqQcXHx7Nt27a0zhCXL19mx44dGcq4uroyadIka4Qn\ngCtXjMt4tWqBry8cPAhly8LQoWc5fx4WLYIaNXJ3zFuJt+i2rBsRMRG0rdo2bwIXQhQ4heoS37Vr\n1+jWrRsVKlRg3LhxREVFZSljY2ND795ZnhUWeejuZbxPPoHVq+HuNFzVqsH48TBiBBw+fBZ392q5\nPnZiciJ9V/bl+OXjrB+4nlaerSwauxCi4CpUCcrW1hYbGxsiIyN55513SEpKyjDunr29PSNHjsQ+\nu6c6hUVdvAjLl8OSJXDSNCSwUsZstqNHQ9euUOIh5gjUWjP8h+H8FPYTS3ov4elaT1smcCFEoVDo\nEpSDgwPJyclZHsoF477EkCFDrBBZ8REba9xbWrrUGMjVmNILPDzgpZeMzg9Vq1qmLqUUTz32FC0q\ntmBoYxlPUYjiptAlqBL3+EpeokQJnnjiCfr168f48eOzzN4pHkxsLGzZYgxFtH493O3Rb29vtJYG\nDzZGF7dkwzX8VjieJT0Z1mSY5Q4qhChUClUnCVtb23v23oqLiyMhIYHly5fTrFkzPv/883yMrmi5\nft24dNerF7i7Q//+8O23RnLy9jZ64f1/e/cfXFV95nH8/eSGREJ+CGIR5IdIYV2pJUgKSyklitXQ\nahU7Si21ZbsV1wIdpkut1nXGarvd6XRKO9aRUtktgsViS3fBiFVrg9KOsrCbqKwIZRHFEeVXIAmB\nEPLsH+deSWKSe0MunHNzP6+Z7+Sek++9eXLm5Dz53vO9z/fdd4OkNXNmepPTsv9exugHR/PynpfT\n96IiknEybgTV+p5TZ8455xwmTZrELbfcchai6h1aWqCmJhgpPf10sLRF60M9eTLceGPQujsLrzvW\nvrGWuU/O5aqLr9KaTiJZLuMSVPvVctsrKCjgpptu4pFHHiE3N6N+vbNuz56gBt4zz8Af/gDvvXfq\ne7FYsBbTzJlwww0wZMiZj+fPb/2ZWb+dxYTBE/jdzb8jL6bJLiLZLKOu4LFYjBOJOcwdKCgo4O67\n7+aee+7RBzk7sHt3kJASbefOtt8fOhQqKoI2fTqce+5ZjK12N9etuo5hxcOo/FIlhXmFyZ8kIr1a\nRiUoM6OgoKDNooQJBQUFLF26lNmzZ4cQWfQcPgxbtsCmTafaO++07VNUBJ/6FFx5JcyYEazFFFZe\nH1YyjAUTFzCndA7n9zs/nCBEJFIyKkEBlJSUfChBFRYWsm7dOsrLy8MJKmQHD8KrrwZt8+YgGW3b\ndmoKeEJJCUydCtOmQXk5lJYGdfHCdODoAeqb6hlx7gi+d8X3wg1GRCIl4xJU//79P1g5Nzc3l/79\n+1NVVcWll14acmRn3tGjQR27RDJKtHYLCQPBrLrS0mCtpYkTgzZmDOREaN5mQ1MD1666lvcb3uf1\nea/rnpOItJFxCWrgwIEA5OfnM2LECKqqqhg8eHDIUaXP8ePBqrLbt8OOHUFLPN6zp+PnFBTA2LFw\n2WVw+eVBMvr4xyE//+zG3h0nTp5g1m9nsemdTTxx0xNKTiLyIRmXoC644AJycnKYNGkSlZWVFBZm\nzs109+DtuLfeatt27z71eO/eD781l5CbC6NGBYmodRs5smclhc42d+e2dbdRuaOSJZ9bwo1/e2PY\nIYlIBGVcgpoyZQqFhYUsWbIkMtPIGxth//4gubRu773Xdvvdd09VYehMTk5QKmjMmKCNHh20MWOC\n/RH5lXvkwU0PsrxmOfdNu4/by24POxwRiaiMu9wtWLAg7a/Z0gINDXDkCNTVnWq1tcGI58CB4Gv7\nxwcPwr59U0ny0aw2ioqCRDN8eNBaPx4+PPi8UW9IQl2ZUzqHHMth3ifmhR2KiERYWi+FZjYAWAZc\nDewH7nb3X3fQz4B/Bb4e3/UIcJd7Z29uBZqb4c03gxFLoh09mtp2Q0OQdFonocTjdusddlOMvDwY\nODBYtjzRBg1qu53YV1LSk5+V2Z7f9TyTLpxEcX4x8yfODzscEYm4dP+v/hDQBAwCSoFKM6tx963t\n+s0FbgDGAQ48C+wClnT14jU1wf2WM6Ffv2B0U1wcfC0qCpLJeecFC+4lviZaYvu1116gouLToX1+\nKFNsPriZ7774XeZ9Yh6LKxaHHY6IZABLMmhJ/YXM+gGHgI+5+/b4vhXAO+5+V7u+fwF+5e5L49v/\nANzm7l2uRpeTM97z8taTk9NELHacnJxT7dR2U3z72AePE9u5uQ3EYo3EYkeJxRrIzU08bsSs5bR+\n79raWs49myUXMlBdUR3V46rpe6wvpf9TSu7JXv4eZg9UV1fT3NxMWVlZ2KFEnv72UhPF47Rhw4Yt\n7p70JE/nlWIM0JxITnE1wLQO+o6Nf691v7EdvaiZzSUYcdGnTx8uuaSix4G2tASti6pJKTt58iS1\ntbU9f6Fe6ni/4/z1sr8Sa4ox4sUR1B/v0fupvV5zczPurnMqBfrbS00mH6d0JqhC4Ei7fYeBok76\nHm7Xr9DMrP19qPgoaylAWVmZb968OX0Rp0FVVVXWVrBIxt2ZvGwy/Q/15ydjf8KXf/TlsEOKvPLy\ncmpra6murg47lMjT315qonicUq2Vms4EVQ8Ut9tXDNSl0LcYqE82SUIyi5mxYuYKjhw/Qt32jk4D\nEZHOpbPwzXYg18xGt9o3Dmg/QYL4vnEp9JMMdKz5GL/c8kvcndHnjWbCkAlhhyQiGShtCcrdG4A1\nwP1m1s/MpgDXAys66P4o8C0zu9DMhgD/BPwqXbFIeE62nGT2mtnMfXIuL+15KexwRCSDpbt06DeA\nvsD7wCrgDnffamZTzaz13fFfAOuAV4HXgMr4Pslg7s78p+az5vU1LL5mMZOHTQ47JBHJYGmd7+vu\nBwk+39R+/4sEEyMS2w7cGW/SSzzwwgMs2bKE70z5Dgv/bmHY4YhIhovQ4guSyXYe3Mn3X/g+Xx33\nVX44/YdhhyMivYA+MSlpMWrAKDZ+bSPjLxif8hRSEZGuaAQlPbLhzQ2s3roagIkXTqRPrE/IEYlI\nb6ERlJy2mr01fP7xzzOseBgzL5mp5CQiaaURlJyWXYd2UfFYBUV5RTw1+yklJxFJO42gpNv2Nezj\nmpXXcKz5GBv/fiPDS4aHHZKI9EJKUNJtv9n6G94+8jbP3focYz/SYY1fEZEeU4KSbps/cT4zPjqD\nUQNGhR2KiPRiugclKWnxFhY+vZDqvUGVbSUnETnTlKAkKXdn0TOL+NnLP+PZnc+GHY6IZAklKEnq\nx3/5MYtfWsyCiQtY9MlFYYcjIllCCUq6tLx6OXc+dyc3j72Zn1b8VFUiROSsUYKSTrk7T/zvE0wf\nOZ1Hb3iUHNPpIiJnj2bxSafMjDWz1tB0son83PywwxGRLKN/ieVDtu3fxozHZrCvYR95sTwK8wqT\nP0lEJM00gpI29hzZw9UrrqbpZBN1TXWc3+/8sEMSkSylBCUfONR4iIqVFdQeq2XDnA1c3P/isEMS\nkSymBCUANJ5o5LpV17Hj4A7Wz17P+MHjww5JRLKc7kEJAAcaD7D/6H5WzlzJlSOvDDscERGNoLKd\nu+M4Q4uH8sodr5AXyws7JBERQCOorHfvn+5lzn/MobmlWclJRCJFCSqL/XzTz/nBiz8gP5ZPzGJh\nhyMi0oYSVJZavXU131z/Ta7/m+t5+NqHVcJIRCInLQnKzAaY2e/NrMHMdpvZl7ro+20ze83M6sxs\nl5l9Ox0xSOqe3/U8t/7+VqYMn8KqL6wiN0e3IkUketJ1ZXoIaAIGAaVApZnVuPvWDvoa8BXgFWAU\n8IyZve3uj6cpFknCMMqGlLH2i2vp26dv2OGIiHSoxwnKzPoBXwA+5u71wEYzWwvcCtzVvr+7/6jV\n5htm9p/AFEAJ6gxrPNFI3z59uWLkFWy8aKPe1hORSEvHCGoM0Ozu21vtqwGmJXuiBVfIqcAvuugz\nF5gb36w3szd6EOuZMBDYH3YQGUDHKXUDzUzHKjmdU6mJ4nEakUqndCSoQuBIu32HgaIUnnsfwX2w\nf++sg7svBZaebnBnmpltdveysOOIOh2n1OlYpUbHKTWZfJySTpIwsyoz807aRqAeKG73tGKgLsnr\nzie4F/U5dz9+ur+AiIj0TklHUO5e3tX34/egcs1stLvviO8eB3Q0QSLxnK8R3J/6tLvvST1cERHJ\nFj2eZu7uDcAa4H4z62dmU4DrgRUd9Tez2cC/AJ9x9//r6c+PgMi+/RgxOk6p07FKjY5TajL2OJm7\n9/xFzAYA/wZ8BjgA3OXuv45/byqw3t0L49u7gKFA67f1Vrr7P/Y4EBER6TXSkqBERETSTaWOREQk\nkpSgREQkkpSg0szMRpvZMTNbGXYsUWNm+Wa2LF6vsc7Mqs1sRthxRUV3alpmK51D3ZfJ1yQlqPR7\nCPivsIOIqFzgbYIqIyXAPwOrzeyiEGOKktY1LWcDD5vZ2HBDihydQ92XsdckJag0MrMvArXAH8OO\nJYrcvcHd73P3N929xd2fBHYBE8KOLWytalre6+717r4RSNS0lDidQ92T6dckJag0MbNi4H7gW2HH\nkinMbBBBLcdOP9SdRTqraakRVBd0DnWuN1yTlKDS5wFgmSpjpMbM+gCPAcvdfVvY8URAT2paZiWd\nQ0ll/DVJCSoFyeoRmlkpcBWwOOxYw5RC3cZEvxyCSiNNwPzQAo6W06ppma10DnWtt1yTtJRqClKo\nR7gQuAh4K77GUiEQM7NL3f3yMx5gRCQ7TvDBEivLCCYCfNbdT5zpuDLEdrpZ0zJb6RxKSTm94Jqk\nShJpYGYFtP3vdxHByXGHu+8LJaiIMrMlBKsuXxVf4FLizOxxwIGvExyjp4BPdrIyddbSOZRcb7km\naQSVBu5+FDia2DazeuBYJp0IZ4OZjQBuJ6jDuLfVir63u/tjoQUWHd8gqGn5PkFNyzuUnNrSOZSa\n3nJN0ghKREQiSZMkREQkkpSgREQkkpSgREQkkpSgREQkkpSgREQkkpSgREQkkpSgREQkkpSgREQk\nkv4fpnIt6Q3iZsAAAAAASUVORK5CYII=\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "z = np.linspace(-5, 5, 200)\n", "\n", @@ -201,7 +219,25 @@ "cell_type": "code", "execution_count": 8, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Saving figure leaky_relu_plot\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl8FPX9x/HXh3AlHCJyVMGCeKCgVSCetBiPUm9RUEG0\n4sGhVeuB1oMKiGfFWjwBiyJyi1yitD9Fo+JVQVG8gFJQQUUEEgghAZLv74/voiHk2E2ymdnN+/l4\n7IM9JjvvHTb7zsx8d8acc4iIiIRNraADiIiIlEQFJSIioaSCEhGRUFJBiYhIKKmgREQklFRQIiIS\nSiooKZWZZZrZ40HnSAZmlmFmzsyaVcO8VpvZ4GqYz6Fm9p6Z5ZnZ6njPL4o8zsx6BZ1Dqo4KKkGZ\n2Xgzmxd0jlhFSs9FLtvNbKWZ3W9m9WJ8nn5mllPOfPYo1/J+riqUUhDvAvsCG6pwPsPM7LMSHjoa\neLKq5lOGe4Bc4NDIPKtFGe/9fYGXqiuHxF/toANIjfQscAdQF//B9mzk/tsDSxRnzrntwA/VNK/1\n1TEf4CBgjnNudTXNr0zOuWpZvlJ9tAaVpMxsLzMba2Y/mtkWM3vTzNKLPL6PmU0xszVmts3MPjez\ny8t5zlPMLMvMBplZNzPbYWa/KjbNvWb2aTnxcp1zPzjnvnHOvQi8CnQv9jytzGyqmW2KXF42s4Nj\nXAwVYmYPmNmyyHJZbWZ/M7P6xaY5w8w+iEyzwcxeMrP6ZpYJtAEe2rWmGJn+5018ZtY48nNnF3vO\n7pFl2qK8HGbWDxgKdCyyRtov8thua3Bm9mszmxV5H2wxs5lm1rrI48PM7DMz6x1Zo91iZrPL2hwZ\neV1HAndF5j3MzNpGrqcXn3bXprci0/Q0s1fNLNfMvjCz3xf7mUPNbK6ZZZtZTmRT4hFmNgy4DDiz\nyOvOKD6fyO0jzOy1yPLbGFnz2qvI4+PNbJ6Z/dnM1kbeZ8+aWVppr1uqlwoqCZmZAS8DrYCzgE7A\nW8DrZrZvZLL6wEeRxzsCo4AxZnZKKc/ZC5gFDHDOjXbOvQWsBP5YZJpakdvjYsh6JNAV2FHkvjTg\nDSAPOBE4HvgeeK2aPjy2AlcAhwHXAL2BO4vkOw2Yiy/WLsBJwJv436fzgTXA3fhNTvtSjHNuM35T\nVN9iD/UFXnXO/RhFjmnAw8CyIvOZVnxekf+TOUDLSM6TgP2A2ZH3yS5tgYuA8/B/LHQC7i1l+RCZ\n37JIhn2BkWVMW5J7gUfxJfchMNXMGkYy7wcsBBzwe6Az8ASQEpnPdOC1Iq/73RJedwPg30AOcEzk\ndZ0APFNs0t8BhwOn8svr/3OMr0XixTmnSwJegPHAvFIeOxn/i5la7P4lwK1lPOdU4J9FbmcCjwMD\ngGyge7HpBwNfFrl9OpAP7FPGPDKB7ZF8+fgPoQKgZ5FprgBWAFbkvhT8/psLI7f7ATnlzOfxEu4v\n8+dKea5BwH+L3H4HmFrG9KuBwcXuy4i81maR2+fg9980itxOBTYDF8eQYxjwWVnzx3/AFwBtizze\nDigETi3yPHnAXkWmubPovErJ8xkwrMjttpHXmF5sOgf0KjbNwCKPt4rc99vI7XuBr4G6sbz3i82n\nf+Q926iE/4ODijzPt0BKkWmeBl6ryO+kLlV/0RpUcuoCpAHrI5tHcswPDDgcOBDAzFLM7E4z+zSy\niSoH/9f/r4s9Vw/8X6+nOef+r9hjzwHtzOyEyO0rgNnOufIGAkwDjsKvGU0HnnZ+U1/R/AcAW4pk\nzwb23pU/nsysl5ktNLMfIvN+hN2XSydgQSVnMx9fUOdFbp8DGDA7hhzROAz4zhXZT+Sc+x/wHdCh\nyHRfO+eyi9z+DmgR47xiUXQz8HeRf3fNrxOw0Pn9dhV1GPCpc25LkfvexRdz0df9hXOuoFiWeL5u\niYEGSSSnWsA6/OaL4jZH/h0M3IzfnLEUv0ZzH3v+cn4CHAFcaWbvu8ifmeB3xpvZXOAKM1uG/5A9\nm/JlO+f+C2BmlwCfm1k/59z4IvmX4DdpFbcxiucH/zr3KuH+JviyK5GZHYdfkxwO3Ahk4V9XrJuw\nyuSc22Fm0/Gb9SZE/p3lnMutxhxFT2Wwo4THYv0DtjDy78+bDs2sTinT/jw/55yLbG2srj+Yq/p1\nS5yooJLTR/h9DoWRv5ZL8lvgJefc8/DzfqtD8B+ERa0CrsNvMhtrZgOKlhR+k8gM4H/4UWqvxRI0\n8kF9H3C/mU2PfEB/BPQBfnLOFc8TrWXAGWZmxfJ2jjxWmq7AWufciF13mFmbYtN8DJyCf+0l2Y7f\nJFmeicBbZtYBOA2/PzCWHNHM50tgPzNru2stysza4fdDfRFFxljsGj1YdL/bURV4no+BS8ysbilr\nUdG+7ivMrFGRtagT8OXzZQUySQD0l0Jia2xmRxW7tMWXxDvAHDM73cwOMLPjzWy4me1aq1oOnGJm\nvzWzQ/H7mg4oaSaRkjsJ/yE6ptjO9Vfx+4aGAuOdc4UlPEV5JuP/cr02cnsSfg1wjpmdGMnfzcwe\ntt1H8tUq4fUfHnnsKfy+lsfM7Egza29mN+KL76EysiwHWplZXzNrZ2ZXR36mqHuBC8zsHjPrYGYd\nzezGIgM4VgO/Mz8SsdSRcM65d/H7WiYDP7H7ZsNocqwG2phZZ/OjA0v6Ltlr+M1pk8ws3fwIu0n4\nPwJeL2M5xMw5tw14H/hLZJmcQMXW+J4EGgLTzexoMzvIzPqY2a6yWw0cHvk/bVbKWtok/CbUCeZH\n83UDxgAzd629S/ipoBLb7/B/bRa9jIysMZyB/wB6Gr/GMB1ozy/b++8B/oPfF/IWfsTYpNJm5Jxb\nid/JfDpFSioyr2eBOvzyfaaYRP5Kfhy4NfIXby7QDb9W9gLwFX5/197ApiI/mlrC68+MPOf/Is9x\nMPB/kdfaG7jAOTe/jCwv4QvsH/gP9t8DdxWb5hX8vqPTI/N8E1/gu8r5LmB//CjH8r6TNAk/km1q\n0X0h0eQAXgRewRfbevYssF3/P+dGHn8jcvkB6FFszbKqXBH590N8IQyJ9Qmcc2vx/3d18Xk/xq/F\n74xM8jR+LWgR/nV1LeE5coE/AI3x//dzgPeK5JMEYPF5j0pNYmZP4UdG/b7ciUVEoqR9UFJh5r/0\n2AH/3acLA44jIklGBSWVMQf/JchxzrmXgw4jIslFm/hERCSUNEhCRERCKW6b+Jo1a+batm0br6ev\nlK1bt9KgQYOgYyQkLbvYLVu2jIKCAjp06FD+xLIbvd8qrrRlt2oVbNwI9erBYYdBSjTf2Ktiixcv\n/sk517y86eJWUG3btmXRokXxevpKyczMJCMjI+gYCUnLLnYZGRlkZWWF9vchzPR+q7iSlt3DD8Pg\nwdCgAXzwAXTsGEw2M/s6mum0iU9EpAZ49VW49VZ/fcKE4MopFiooEZEk97//wUUXQWEh/PWvcP75\nQSeKjgpKRCSJbd0KPXrApk1w1lkwbFjQiaKnghIRSVLOweWXw9Kl0L49TJwItRLoUz+BooqISCwe\nfBBeeAEaNYLZs2Gvkk5AE2IxFZSZHWxmeWY2MV6BRESk8j74oCl33OGvT5oEhx4abJ6KiHUN6gn8\nUYpFRCSkVqyAe+45DOdg+HA4O5rTiIZQ1AVlZr3xJ7Or7KmuRUQkTrZs8YMicnLq0KMHDIn5hCfh\nEdUXdc2sMXA3cDJwVRnTDQAGALRs2ZLMzMwqiFj1cnJyQpst7LTsYpeVlUVBQYGWWwXo/RabwkIY\nOrQjX3zRnP3330L//kt4662C8n8wpKI9ksQI/BGr1+x+MtXdOefGAmMB0tPTXVi/Aa5vp1ecll3s\nmjRpQlZWlpZbBej9FpsRI2DhQj8Y4r77vuCMM35X/g+FWLkFFTnN8qlAp/jHERGRinjpJRg6FMxg\nyhRITd0WdKRKi2YNKgNoC3wTWXtqCKSYWQfnXOf4RRMRkWh89RVccon/3tN998Hpp0MybBmNpqDG\nAlOL3B6ML6yr4xFIRESil53tB0Vs3gy9esFttwWdqOqUW1DOuVwgd9dtM8sB8pxz6+MZTEREylZY\n6Necli2DI46AZ5/1m/iSRcyn23DODYtDDhERidHw4TBvHuy9tz9SRMOGQSeqWjrUkYhIApo1C+6+\n2x9bb+pUaNcu6ERVTwUlIpJgPv8c/vhHf/3BB6F792DzxIsKSkQkgWzatOtIEdCnD9x8c9CJ4kcF\nJSKSIAoKoG9f+O9/4aij4J//TK5BEcWpoEREEsRf/wrz58M++/h9UGlpQSeKLxWUiEgCmD4d7r8f\nUlL89bZtg04UfyooEZGQ+/RTf2ZcgIcfhpNPDjZPdVFBiYiE2MaNflBEbq4fuXf99UEnqj4qKBGR\nkNq5E3r3hlWroEsXGD06uQdFFKeCEhEJqdtvh1dfhebN/aCI1NSgE1UvFZSISAhNngwjR0Lt2jBj\nBuy/f9CJqp8KSkQkZD7+GK6KnLv8H/+Abt2CzRMUFZSISIisX+8HRWzbBldcAddcE3Si4KigRERC\nYscOuOgi+OYbOPZYeOKJmjUoojgVlIhISNxyC7zxBvzqV/Dii1C/ftCJgqWCEhEJgQkTYNQoqFPH\nl1OrVkEnCp4KSkQkYIsWwYAB/vrjj8MJJwSbJyxUUCIiAVq3Ds47D/LzYeDAX4pKVFAiIoHZvh0u\nuADWrIGuXeHRR4NOFC4qKBGRgNx0E7z9Nuy3n/8ybt26QScKFxWUiEgAxo3zw8jr1oWZM/3IPdmd\nCkpEpJq9//4vX8AdPdp/50n2pIISEalG338P55/v9z9de+0v53mSPamgRESqSX4+9OzpS6pbN/j7\n34NOFG4qKBGRanL99fDee9C6Nbzwgv9SrpROBSUiUg3GjIGxY/3hi2bPhhYtgk4UfiooEZE4e+cd\nuO46f33sWH92XCmfCkpEJI7WrvX7nXbsgBtugEsvDTpR4lBBiYjESV6eH7G3bh2cfDI89FDQiRKL\nCkpEJA6cg6uvhv/8B9q0gWnT/OnbJXoqKBGROHjiCRg/HlJT/aCIZs2CTpR4VFAiIlXszTfhxhv9\n9WeegaOOCjZPolJBiYhUoW++8Uco37nTnyG3d++gEyUuFZSISBXZts2f22n9eujeHe6/P+hEiU0F\nJSJSBZzzJxv86CNo1w6mTIGUlKBTJTYVlIhIFRg1CiZOhAYN/KCIpk2DTpT4VFAiIpW0YAEMHuyv\njx8PRxwRaJykoYISEamEVavgoougoADuuAN69Qo6UfJQQYmIVFBurh8UsWEDnHEG3H130ImSS1QF\nZWYTzex7M9tsZsvN7Kp4BxMRCTPn4Mor4ZNP4OCDYdIkDYqoatGuQd0PtHXONQbOAe4xMx2PV0Rq\nrJEjYepUaNjQD4po0iToRMknqoJyzn3unMvfdTNyOTBuqUREQuzf/4bbbvPXn38eOnQINk+yivrQ\nhWb2JNAPSAU+Bl4pYZoBwACAli1bkpmZWSUhq1pOTk5os4Wdll3ssrKyKCgo0HKrgDC+39aurc+g\nQV0oLKzDZZetpkmT1YQsIhDOZRcrc85FP7FZCnA8kAE86JzbUdq06enpbtGiRZUOGA+ZmZlkZGQE\nHSMhadnFLiMjg6ysLJYsWRJ0lIQTtvdbTg4cfzx89hmccw7MmgW1QjrULGzLrigzW+ycSy9vupgW\nrXOuwDm3EGgNXF3RcCIiicY56NfPl9Ohh/pNe2Etp2RR0cVbG+2DEpEa5P774cUXoXFjPyiiceOg\nEyW/cgvKzFqYWW8za2hmKWb2B6APsCD+8UREgvfyyzBkCJjB5MnQvn3QiWqGaAZJOPzmvNH4Qvsa\nuME5NzeewUREwmD5cujb12/iGzECzjwz6EQ1R7kF5ZxbD5xYDVlEREJl82bo0QOys+H88/2hjKT6\naBefiEgJCgvhj3+EL7+Ejh39QWA1KKJ6aXGLiJRgxAiYM8cfIWL2bGjUKOhENY8KSkSkmLlzYdgw\nPyhiyhQ46KCgE9VMKigRkSK+/BIuucRfv/9+OO20YPPUZCooEZGIrCw491zYsgUuvBBuvTXoRDWb\nCkpEBD8o4pJLYMUK+M1v4Jln/CY+CY4KSkQEGDrUfyG3aVM/KKJBg6ATiQpKRGq8F1+Ee+7xw8in\nTYMDDgg6kYAKSkRquM8+g8su89cfeghOPTXYPPILFZSI1FibNvkjRWzd6g9ndOONQSeSolRQIlIj\nFRRAnz6wciV06gRjx2pQRNiooESkRrrzTn/q9mbN/IkH09KCTiTFqaBEpMaZNg0efBBSUuCFF6BN\nm6ATSUlUUCJSo3zyCVx+ub/+yCMQ0rOiCyooEalBNmzwgyK2bfMj9669NuhEUhYVlIjUCDt3wkUX\nwerVcPTRMHq0BkWEnQpKRGqEv/wFFiyAFi1g5kyoXz/oRFIeFZSIJL2JE+Hvf4fatf1RI1q3DjqR\nREMFJSJJ7aOPoH9/f/3RR+G3vw02j0RPBSUiSevHH/2giLw8uOoqGDQo6EQSCxWUiCSlHTv8OZ2+\n/RaOOw4ef1yDIhKNCkpEktLNN8Obb8K++/r9TvXqBZ1IYqWCEpGkM348PPYY1Knjy2m//YJOJBWh\nghKRpPKf//yyr+nJJ+H444PNIxWnghKRpPHDD3D++ZCfD1df7QdGSOJSQYlIUti+HXr1grVr/VDy\nf/wj6ERSWSooEUkKN9wA77wDrVrBjBlQt27QiaSyVFAikvCefhqeesqP1Js1C1q2DDqRVAUVlIgk\ntHffhT/9yV8fPdofCFaSgwpKRBLWd99Bz57+S7nXXw/9+gWdSKqSCkpEElJ+vi+nH36AE0+EkSOD\nTiRVTQUlIgnHOb9Z7/334de/9qdtr1Mn6FRS1VRQIpJwRo+GceP8OZ1mzYLmzYNOJPGgghKRhPL2\n235/E8A//wmdOwebR+JHBSUiCWPNGv9l3J074aaboG/foBNJPKmgRCQh5OXBeef5czydcgo8+GDQ\niSTeVFAiEnrO+QPALloEbdvCtGn+9O2S3FRQIhJ6jz0Gzz0HaWkwezbss0/QiaQ6qKBEJNQyM/3+\nJoBnn4Ujjww0jlSjcgvKzOqZ2Tgz+9rMtpjZEjM7vTrCiUjN9sMP9bjgAigogL/8xZ/CXWqOaNag\nagPfAicCewFDgOlm1jZ+sUSkpsvNhbvuOpyffoI//AHuvTfoRFLdyt3N6JzbCgwrctc8M1sFdAFW\nxyeWiNRkzkH//rBiRSMOPBCmTIGUlKBTSXWLeRyMmbUEDgE+L+GxAcAAgJYtW5KZmVnZfHGRk5MT\n2mxhp2UXu6ysLAoKCrTcYjB9emsmTz6I+vV3cuedH/PJJ1uDjpRwkuF31Zxz0U9sVgeYD6x0zg0s\na9r09HS3aNGiSsaLj8zMTDIyMoKOkZC07GKXkZFBVlYWS5YsCTpKQnjtNb9Jr7AQhg//jLvuOjzo\nSAkpzL+rZrbYOZde3nRRj+Izs1rA88B24NpKZBMRKdH//gcXXeTLacgQ6Nbtp6AjSYCiKigzM2Ac\n0BLo6ZzbEddUIlLjbN0KPXrAxo1w1lkwfHjQiSRo0e6Dego4DDjVObctjnlEpAZyDq64ApYuhUMO\ngYkToZa+pVnjRfM9qDbAQOAo4Aczy4lcdJhGEakSf/sbTJ8OjRr5I0XstVfQiSQMohlm/jVg1ZBF\nRGqgf/0Lbr/dX584EQ47LNg8Eh5aiRaRwPz3v9Cnj9/EN3w4nHNO0IkkTFRQIhKILVv8oIisLP/v\nkCFBJ5KwUUGJSLUrLITLLoPPP/eb9J57ToMiZE96S4hItbvvPpg1yw+GmD0bGjcOOpGEkQpKRKrV\nvHlw111gBpMn+2HlIiXROSlFpNosWwZ9+/pBEffeC2ecEXQiCTOtQYlItcjOhnPPhc2boVevX4aW\ni5RGBSUicVdYCJde6tegDj/cnxnX9O1KKYcKSkTibvhweOkl2HtvPyiiYcOgE0kiUEGJSFzNng13\n3+2HkU+dCgceGHQiSRQqKBGJmy++8Jv2AB54ALp3DzaPJBYVlIjERVaWHxSRkwO9e8PgwUEnkkSj\nghKRKldQABdf7I+1d9RRMG6cBkVI7FRQIlLl7roL5s+HffbxR4xISws6kSQiFZSIVKkXXvCHMkpJ\ngWnToG3boBNJolJBiUiV+fRT6NfPXx85Ek45JdA4kuBUUCJSJTZu9KfNyM31I/f+/OegE0miU0GJ\nSKXt3OlH6q1aBV26wJgxGhQhlaeCEpFKu+MOePVVaN4cZs6E1NSgE0kyUEGJSKVMmQIPPQS1a8OM\nGfDrXwedSJKFCkpEKmzJErjySn/9H/+Abt2CzSPJRQUlIhXy009+UMS2bXD55XDNNUEnkmSjghKR\nmO3cCRdeCF9/DcccA08+qUERUvVUUCISs1tugTfegJYt/aCI+vWDTiTJSAUlIjF5/nm/v6lOHXjx\nRWjVKuhEkqxUUCIStUWLoH9/f/2xx6Br12DzSHJTQYlIVNatg/POg/x8GDAABg4MOpEkOxWUiJRr\nxw644AJYswZOOAEefTToRFITqKBEpFw33ghvvw377ee/jFuvXtCJpCZQQYlImZ55Bp54AurW9SP2\n9t036ERSU6igRKRUH3wAV1/trz/1FBx7bLB5pGZRQYlIib7/Hs4/H7Zvhz/9Ca64IuhEUtOooERk\nD9u3Q69e8N13/vh6jzwSdCKpiVRQIrKH66+Hd9+F1q39Kdzr1Ak6kdREKigR2c2YMf5Srx7MmgUt\nWgSdSGoqFZSI/Oydd+C66/z1sWMhPT3YPFKzqaBEBIC1a6FnT/+l3BtugD/+MehEUtOpoESEvDw/\nYm/dOjjpJH+GXJGgRVVQZnatmS0ys3wzGx/nTCJSjZzzw8j/8x9o0wamTfOnbxcJWrRvw++Ae4A/\nAKnxiyMi1e3JJ/3RIlJT/aCI5s2DTiTiRVVQzrmZAGaWDrSOayIRqTZvveX3NwGMGwedOgWbR6Qo\n7YMSqaG+/dZ/GXfnThg8GPr0CTqRyO6qdEuzmQ0ABgC0bNmSzMzMqnz6KpOTkxPabGGnZRe7rKws\nCgoKQrXc8vNrcf31nVi/vhHp6Rs57bSlZGa6oGPtQe+3ikuGZVelBeWcGwuMBUhPT3cZGRlV+fRV\nJjMzk7BmCzstu9g1adKErKys0Cw35/wQ8uXLoV07+Pe/m9K06YlBxyqR3m8VlwzLTpv4RGqYUaNg\n4kRIS4PZs6Fp06ATiZQsqjUoM6sdmTYFSDGz+sBO59zOeIYTkar1+ut+fxPA+PFwxBGBxhEpU7Rr\nUEOAbcBtwCWR60PiFUpEqt7q1XDhhVBQALff7k/hLhJm0Q4zHwYMi2sSEYmb3Fzo0QM2bIDTT4cR\nI4JOJFI+7YMSSXLOwZVXwiefwMEHw+TJkJISdCqR8qmgRJLcww/D1KnQsKEfFNGkSdCJRKKjghJJ\nYv/3f/CXv/jrEyZAhw7B5hGJhQpKJEmtXAm9e0NhIdx1F5x3XtCJRGKjghJJQjk5flDEpk1w9tkw\ndGjQiURip4ISSTLOweWXw2efQfv2/ku5tfSbLglIb1uRJPPAAzBjBjRuDHPm+H9FEpEKSiSJvPIK\n3HknmMGkSX4NSiRRqaCqQUZGBtdee23QMSTJrVgBF1/sN/HdfTecdVbQiUQqRwUF9OvXj7P02ywJ\nbMsWOPdcyM72o/XuuCPoRCKVp4ISSXCFhf70GV9+6b/n9NxzGhQhyUFv43JkZ2czYMAAWrRoQaNG\njTjxxBNZtGjRz49v2LCBPn360Lp1a1JTU+nYsSPPPvtsmc+5YMECmjRpwujRo+MdX2qAe+755QgR\nc+ZAo0ZBJxKpGiqoMjjnOPPMM1m7di3z5s3j448/plu3bpx88sl8//33AOTl5dG5c2fmzZvH559/\nzp///GcGDhzIggULSnzOGTNmcN555zF27FgGDRpUnS9HktDcuf47TmYwZQocdFDQiUSqTpWeUTfZ\nvPHGGyxZsoT169eTmpoKwIgRI3jppZd4/vnnufXWW2nVqhW33HLLzz8zYMAAXn/9daZMmcIpp5yy\n2/ONHTuWW265hRkzZtC9e/dqfS2SfL76Ci65xF+/7z447bRg84hUNRVUGRYvXkxubi7Nmzff7f68\nvDxWrlwJQEFBAQ888ADTpk1j7dq15Ofns3379j1OtTx79mzGjBnDW2+9xfHHH19dL0GSVHa2HxSx\nZYs/r9Ou4+2JJBMVVBkKCwtp2bIlb7/99h6PNY58+3HkyJE8/PDDjBo1iiOOOIKGDRtyxx138OOP\nP+42/ZFHHsnSpUsZN24cxx13HGZWLa9Bkk9hIfTtC8uX+zPiPvus38QnkmxUUGXo3Lkz69ato1at\nWrRr167EaRYuXMjZZ5/NpZdeCvj9VsuXL6dJsXMaHHDAATz22GNkZGQwYMAAxo4dq5KSChk6FF5+\nGZo29YMjGjQIOpFIfGiQRMTmzZtZsmTJbpeDDjqIrl27cu655zJ//nxWrVrFe++9x9ChQ39eqzrk\nkENYsGABCxcu5KuvvuLaa69l1apVJc6jXbt2vPHGG/zrX/9i4MCBOOeq8yVKEpg504/aq1ULpk2D\nUv5uEkkKKqiIt99+m06dOu12ueWWW3jllVc4+eST6d+/P+3bt+fCCy9k2bJl7LfffgAMGTKEY445\nhtNPP51u3brRoEED+vbtW+p8DjzwQDIzM5k/f75KSmLy2Wf++04Af/sbnHpqsHlE4k2b+IDx48cz\nfvz4Uh8fNWoUo0aNKvGxvffem5kzZ5b5/JmZmbvdPvDAA/n2229jjSk12KZN/vQZW7f6wxnddFPQ\niUTiT2tQIiFXUAB9+vgTEHbqBE8/rUERUjOooERCbsgQ+Pe/oVkzmDUL0tKCTiRSPVRQIiE2fbo/\nv1NKir/epk3QiUSqT1IX1M6dOxkzZgwbNmwIOopIzD75xJ8ZF+Dvf4eTTgo2j0h1S9qC+vbbbznm\nmGO47rrruOCCCzRaThLKhg1+UERuLlx2GVx3XdCJRKpfUhbUnDlz6NixI59++ik7duzggw8+YOTI\nkUHHEonKzp3QuzesXg3p6TB6tAZFSM2UVAWVn5/PoEGDuPjii9myZQsFBQUA5ObmMnTo0N1OkyES\nVrfdBq/vsgn3AAAKYElEQVS9Bi1a+C/m1q8fdCKRYCRNQa1YsYIjjzySCRMmkJubu8fjzjlWrFgR\nQDKR6E2aBA8/DLVrw4wZsP/+QScSCU5SfFF34sSJDBo0iNzc3D32NdWpU4fGjRsza9Ysfve73wWU\nUKR8H30EV13lrz/6KOjtKjVdQhfU1q1b6d+/P3PmzClxrSktLY1jjz2WF154gX322SeAhCLRWb8e\nzjsP8vLgyitB57IUSeBNfEuXLqVDhw7MmjWrxHJKTU1l+PDhLFiwQOUkobZjB1x4IXzzDRx3HDzx\nhAZFiEACrkE553jqqacYPHgw27Zt2+PxevXq0bRpU+bOnUt6enoACUViM3gwZGbCr34FL74I9eoF\nnUgkHBKqoLKysrjkkkt44403SiyntLQ0fv/73zNhwoSfTygoEmbjx/v9TXXq+BF7kYPkiwgJVFAf\nfPAB55xzDtnZ2eTn5+/xeFpaGo888gj9+/fXiQAlIXz44S/7mp54Ao4/Ptg8ImET+oIqLCzkgQce\n4J577ilxral+/frsu+++zJs3jw4dOgSQUCR269b5QRH5+b6k+vcPOpFI+IS6oH788Ud69erF4sWL\nS92k17NnT8aMGUNqamoACUVit3079OoFa9dC165QyqnGRGq8QEfxbdiwgY8++qjEx15//XUOPfRQ\n3n///T1G6dWqVYsGDRowbtw4JkyYoHKShHLDDbBwIbRq5b+MW7du0IlEwinQgrrmmmvo2rUrK1eu\n/Pm+nTt3ctttt3HWWWexadMmduzYsdvPpKWlceihh/Lpp5/Su3fv6o4sUin//Cc89ZQfqTdzph+5\nJyIlC6ygli9fzty5c9m+fTtnn30227dvZ82aNRx77LE89thjJW7SS01N5corr+Tjjz+mXbt2AaQW\nqbj33oM//clff+opOOaYYPOIhF1U+6DMrCkwDugO/ATc7pybXJkZ33bbbezYsYPCwkJWr15Njx49\nWLhwIbm5uT8f5PXnkLVrk5aWxuTJkznzzDMrM1uRQOzYUYuePf3+p+uu++U8TyJSumgHSTwBbAda\nAkcBL5vZJ865zysy0y+++IL58+f/XETbtm3j9ddfL3X4eMeOHZk1axatWrWqyOxEApWXB6tWNWDb\nNjjxRH8wWBEpn5V3Ij8zawBsAg53zi2P3Pc8sNY5d1tpP9eoUSPXpUuXEh9bunQpGzduLDdcrVq1\naN26NW3btq3S7zZlZWXRpEmTKnu+mkTLbnfO+fM3lXbZvh3WrFkCQL16R9Gli/9SrkRH77eKC/Oy\ne/PNNxc758o91E80a1CHADt3lVPEJ8CJxSc0swHAAPBHEc/KytrjybZt28amTZvKnWlKSgpt27al\nYcOGZGdnRxEzegUFBSVmk/Il27JzDgoKrMIX56L7wyklpZCDDspm61ad2TkWyfZ+q07JsOyiKaiG\nwOZi92UDjYpP6JwbC4wFSE9PdyWdILB79+7lnpepTZs2LFq0iGbNmkURL3aZmZlkZGTE5bmTXdiW\nXX4+ZGWVfcnOLv2xEsbixCQlBfbaC5o0Kf0yY0YGZlksWfJx1bzoGiRs77dEEuZlF+0WsWgKKgco\nfmC7xsCWGDOxePFiFi5cuMc5m4r78ccfWbp0KSeddFKss5AEk5dXfsGUVTZ5eZWbf0oK7L23L5Ly\niqakS4MG5R95fMECn1VEYhNNQS0HapvZwc65Xas+RwIxD5C4+eabyYviE2Xbtm307NmTZcuW0bx5\n81hnI9XEudgLpnjRlDAuJia1a/9SMEUv0ZZNWppObSESVuUWlHNuq5nNBO42s6vwo/jOBU6IZUbv\nv/8+H374YblrT7ts2bKFG2+8kYkTJ8YyG4mBc5CbG92msF2Xb7/tTEHBL7eLfY86ZnXqlFww0ZZN\naqoKRiRZRTvM/BrgGeBHYANwdaxDzG+66aYSTyxYr1496tWrR35+PnXr1uWQQw7h6KOPpkuXLtrE\nVw7nYOvW2Pa5FL/s3BnrXHff2lu3bvkFU1bR1K+vghGRkkVVUM65jUCPis7k3Xff5b333qNRo0YU\nFBRQUFBAu3bt6Ny5M0cffTS/+c1v6NixIy1atKjoLBKSc5CTU/Ed/FlZUOw7zTGrXz+2fS4rVy7m\nlFO6/Fw29etXzbIQESmuWo5mvs8++3DfffdxxBFHcPjhh9OmTZukOGdTYWH5BVNe0RQWVi5DWlrs\n+12KTh/r2VszM7fQvn3lMouIRKNaCqp9+/bcfvvt1TGrmBQWwpYtFd/Bn51d+YJp0CD2/S5Fp9GR\nsEUkWYX6fFDlKSiAzZtj3+/yww/HkZfnfzbKMRulatiwYvtedt2vowqIiJQs0IIqKNizWGIpms3F\nvz4ctV92nDRqFNsmseK3ayd0xYuIhFfcPl7XrYO77iq7YLbE/FXfPe21V+z7Xr766n3+8IfjaNxY\nBSMiElZx+3heswZGjCh7GrPdyyXWomnUyB8JIFbZ2Xk0bVqx1yUiItUjbgXVogVcc035BVMr0HP6\niohIWMWtoPbfH4YOjdezi4hIstP6i4iIhJIKSkREQkkFJSIioaSCEhGRUFJBiYhIKKmgREQklFRQ\nIiISSiooEREJJRWUiIiEkgpKRERCyVxlT4hU2hObrQe+jsuTV14z4KegQyQoLbuK0XKrGC23igvz\nsmvjnGte3kRxK6gwM7NFzrn0oHMkIi27itFyqxgtt4pLhmWnTXwiIhJKKigREQmlmlpQY4MOkMC0\n7CpGy61itNwqLuGXXY3cByUiIuFXU9egREQk5FRQIiISSiooEREJJRUUYGYHm1memU0MOkvYmVk9\nMxtnZl+b2RYzW2JmpwedK6zMrKmZzTKzrZFldnHQmcJO77GqkQyfayoo7wngw6BDJIjawLfAicBe\nwBBgupm1DTBTmD0BbAdaAn2Bp8ysY7CRQk/vsaqR8J9rNb6gzKw3kAUsCDpLInDObXXODXPOrXbO\nFTrn5gGrgC5BZwsbM2sA9AT+6pzLcc4tBOYClwabLNz0Hqu8ZPlcq9EFZWaNgbuBm4LOkqjMrCVw\nCPB50FlC6BBgp3NueZH7PgG0BhUDvcdik0yfazW6oIARwDjn3JqggyQiM6sDTAKec859FXSeEGoI\nbC52XzbQKIAsCUnvsQpJms+1pC0oM8s0M1fKZaGZHQWcCjwSdNYwKW+5FZmuFvA8fv/KtYEFDrcc\noHGx+xoDWwLIknD0Hotdsn2u1Q46QLw45zLKetzMbgDaAt+YGfi/dlPMrINzrnPcA4ZUecsNwPwC\nG4ff8X+Gc25HvHMlqOVAbTM72Dm3InLfkWhTVbn0HquwDJLoc63GHurIzNLY/a/bwfj/2Kudc+sD\nCZUgzGw0cBRwqnMuJ+g8YWZmUwEHXIVfZq8AJzjnVFJl0HusYpLtcy1p16DK45zLBXJ33TazHCAv\nEf8Tq5OZtQEGAvnAD5G/0gAGOucmBRYsvK4BngF+BDbgPyhUTmXQe6ziku1zrcauQYmISLgl7SAJ\nERFJbCooEREJJRWUiIiEkgpKRERCSQUlIiKhpIISEZFQUkGJiEgoqaBERCSU/h9r5scSI6iwhAAA\nAABJRU5ErkJggg==\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "plt.plot(z, leaky_relu(z, 0.05), \"b-\", linewidth=2)\n", "plt.plot([-5, 5], [0, 0], 'k-')\n", @@ -363,7 +399,18 @@ "cell_type": "code", "execution_count": 18, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Extracting /tmp/data/train-images-idx3-ubyte.gz\n", + "Extracting /tmp/data/train-labels-idx1-ubyte.gz\n", + "Extracting /tmp/data/t10k-images-idx3-ubyte.gz\n", + "Extracting /tmp/data/t10k-labels-idx1-ubyte.gz\n" + ] + } + ], "source": [ "from tensorflow.examples.tutorials.mnist import input_data\n", "mnist = input_data.read_data_sets(\"/tmp/data/\")" @@ -375,7 +422,22 @@ "metadata": { "scrolled": true }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0 Batch accuracy: 0.86 Validation accuracy: 0.9044\n", + "5 Batch accuracy: 0.94 Validation accuracy: 0.9508\n", + "10 Batch accuracy: 0.96 Validation accuracy: 0.9666\n", + "15 Batch accuracy: 1.0 Validation accuracy: 0.9722\n", + "20 Batch accuracy: 1.0 Validation accuracy: 0.975\n", + "25 Batch accuracy: 1.0 Validation accuracy: 0.9766\n", + "30 Batch accuracy: 0.98 Validation accuracy: 0.9782\n", + "35 Batch accuracy: 0.96 Validation accuracy: 0.9792\n" + ] + } + ], "source": [ "n_epochs = 40\n", "batch_size = 50\n", @@ -417,7 +479,25 @@ "cell_type": "code", "execution_count": 21, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Saving figure elu_plot\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl8FeW9x/HPLwnKKiBorCJgXVDrwhWq1bqkalUWt7q2\nasUNKtqWqq0b9Gql2ipWqApKixcFF1CwKgh41XvABaWgIFAFRECQfTlAgARInvvHc4CQ9SSZZOac\n832/XueVyTxzZn5nGM43sz1jzjlERESiJivsAkRERMqjgBIRkUhSQImISCQpoEREJJIUUCIiEkkK\nKBERiSQFlIiIRJICSkREIkkBJSnDzIab2bg0Wk6WmT1rZuvMzJlZXl0vs5Ja6uUzJ5bV0sxWmdnh\n9bG86jKzV83szrDrEDD1JJGezGw4cH05TZ86536UaG/tnOtewftjwBzn3O2lxvcAnnLONQ204OSW\n3Ry/zcZTaTmVLL87MBbIA74B1jvnttflMhPLjVHqc9fXZ04s6zH8tndDXS+rnGWfCdwFdAIOBm5w\nzg0vNc3xwGTgMOfcxvquUfbICbsAqVPvAteVGlfnX4B1pb6+LOrxS+kIYIVz7uN6Wl6F6uszm1lj\n4GbgwvpYXjmaAnOAFxKvMpxzs83sG+Ba4Ol6rE1K0SG+9FbonFtZ6rW+rhdqZheY2QdmtsHM1pvZ\nJDM7pkS7mdmdZrbAzArNbJmZPZJoGw6cBdyWOOzlzKz9rjYzG2dmPROHiLJLLfclM3szmTqSWU6J\n+exrZgMTyywws0/M7PQS7TEzG2xmD5vZWjNbbWYDzKzC/1+J5T8BtE0se3GJeT1Vetpd9SSzrJqs\n3+p+5pp+bqAr4ICPylknnczsPTPbZmZfm9mZZnalmZWZtqacc2875+5zzr0GFFcy6ZvAz4NartSM\nAkrqQhNgIHAy/vDVRuAtM9sn0f4w0A94BPgBcAWwNNH2W2Aq8D/A9xKvXW27vAo0B366a4SZNQUu\nBkYmWUcyy9nlUeAq4Ebgv4DZwEQz+16Jaa4BdgKnAbcDfRLvqchvgT8ByxLL/mEl05ZW1bJqu34h\nuc+cTC2lnQHMcKXOLZjZD4EPgP8DTgA+AR4E7k98FkpNf5+Z5VfxOqOSOqoyDTjZzBrVYh5SSzrE\nl94uMLP8UuOeds7dXZcLdc6NKfm7md0AbML/h58J/A7o45x7LjHJ1/gvTZxzG81sO7DVObeygvlv\nMLO38V+OExOjL8F/Ub5ZYroK63DOfVjVchLvaQLcCtzsnBufGPcr4GzgNqBvYtL/OOf+mBieb2a3\nAOcAL1fwGTaa2WagqLLlV6DCZSWCutrr18xq8pmr/bmBdsDycsY/DrzlnOufWN5LwFvAFOfc++VM\n/wwwuoJl7PJdFe2VWQ40wJ+nWliL+UgtKKDS2xSgZ6lx9XES/HDgIeAU4AD8nnoW0BZ/Dmxf4L1a\nLmYk8LyZNXbObcWH1RjnXEGSdSTrcPwX1e7DTM65IjObChxbYrovSr1vOXBgNZZTHZUt61hqv36T\n/cxV1VKeRsCqkiPM7CD8ntVPSozejv+3KrP3lKhnPVCXh6u3JX5qDypECqj0ttU593UN37sJfxit\ntBb4Q2WVGYc/dNUL/1fsTuA/wD6Vvamaxifme7GZvQecC5xfz3WUPEy1o5y2mhxCLwas1LgGpX4P\nalk1Ufqy3+rWshZoWWrcrvOT00uM6wDMc859WN5MzOw+4L7KS6WLc+6DKqapyP6Jn2tq+H4JgAJK\nKjIP6GpmVup8wUmJtnKZWSvgaKC3c+7/EuNOYs+29iVQiD8MtKCC2WwHsitoA8A5V2hmr+L3nFoD\nK4FYNepIajn4wzvbgR8nhjF/ccapwEtVvLcm1uDPC5V0IrA4yfcHsX7r8jN/DvQoNa4FPtiKEstq\nhj/3VNmhz7o+xHcc8J1zblWVU0qdUUClt30Th09KKnLO7fqrcD8z61iqPe6cWwwMwZ/0ftLM/gEU\n4K/A+jlwUSXL3ID/K/kWM1sKHAI8ht97wTm32cwGAY+YWSH+MGQroJNzbkhiHovx56vaA/n4+4PK\nu+JqJP5Q1mHAy6WmqbSOZJfjnNtiZkOAv5rZWmAR/hxPLjC4kvVQU+8DA83sIvwfAr2AQ0kyoGq6\nfkvNoy4/86TEfFs559Ylxs3E7zXea2Yv4v+dVgBHmNmRzrkyQVvTQ3yJc3RHJH7Nwl9F2RH/b/9t\niUnPSNQqIdJVfOntXPx/9JKvz0u0n5H4veRrAIBz7hvgTOBI4B38VU1XA1c45yZUtMDEF/xV+Cux\n5uDvI+mH/6t+l3uBvybGfwmMAdqUaB+A/wv+P/g9iorOGX2A/yv5WPa+ei/ZOpJdzt3AKPyVbzMT\n87zAObeigulr47kSr4+AzcDr1ZxHEOu3Tj6zc242e7alXeMW4feYbgVm4T/zufh/t6DvEevMnm29\nEf5Kwc/xV1QCYGYNgUuBfwS8bKkm9SQhIvXKzC4ABgHHOueKwq6nNDO7DbjYOXde2LVkOu1BiUi9\ncs5NxO/Rtqlq2pDsAH4ddhGiPSgREYko7UGJiEgkKaBERCSSQr/MvHXr1q59+/Zhl1HGli1baNKk\nSdhlpBSts+TNmzePoqIijj22dMcMUpFU276WLIG1ayE7Gzp0gEYh9EkR1XU2Y8aMtc65A6qaLvSA\nat++PdOnT696wnoWi8XIy8sLu4yUonWWvLy8POLxeCS3/ahKpe3rj3+Ehx6Chg3h3Xfhxz8Op46o\nrjMzW5LMdDrEJyISoKef9uGUnQ2jR4cXTulAASUiEpBXX4VfJy5QHzoULgzrsYxpQgElIhKA99+H\na68F5+Dhh+HGG8OuKPUFGlBmNtLMVpjZJjObb2Y3Bzl/EZEo+vxzuOQS2L4dfvMbuOeesCtKD0Hv\nQT0CtHfO7YfvULS/mXUKeBkiIpGxcCF06QKbN8NVV8ETT4CVfmCK1EigAeWcm+uc29UZp0u8Dg9y\nGSIiUbFqFZx/vv957rnw/POQpRMngQn8MnMzG4x/3ksjfC/Bb5czTU8ST3rNzc0lFosFXUat5efn\nR7KuKNM6S148HqeoqEjrqxqitn1t3ZpNnz4dWbiwGUceuZnf/W4mU6dGq+/bqK2z6qqTvvhKPNws\nD/irc670Uzd369y5s4vivSBRvX8gyrTOkrfrPqiZM2eGXUrKiNL2VVgI3brBe+/B4YfDRx9Bbm7Y\nVZUVpXVWkpnNcM51rmq6OtkZdc4VJR7V3Ab/jBcRkbRQXAzXX+/DKTcXJk2KZjilg7o+WpqDzkGJ\nSJpwDvr0gVGjoFkzmDDB70FJ3QgsoMzsQDO72syamlm2mZ2Pfzz4e0EtQ0QkTH/5Czz5JOyzD/zr\nX/Bf/xV2RektyIskHP5w3jP44FsC9HHOvRngMkREQjFsGNx3n7+EfORIOPvssCtKf4EFlHNuDXBW\nUPMTEYmKN9+Enj398FNPwRVXhFtPptAV+yIilfjoI38DbnEx9OsHvXuHXVHmUECJiFRg7lzo3h0K\nCuCWW+DBB8OuKLMooEREyrF0KVxwAcTjcPHFMHiwujCqbwooEZFS1q3zXRgtWwannw4vvww5oT/e\nNfMooEREStiyxR/W+/JLOO44f4FEGI9rFwWUiMhuO3b4CyI++QTatoWJE6Fly7CrylwKKBERfC8R\nt9wC48dDq1a+C6NDDgm7qsymgBIRAe691z8uo3FjH1JHHx12RaKAEpGM98QT8Ne/+gshxoyBU04J\nuyIBBZSIZLgXX4Q77vDDzz3nLy2XaFBAiUjGeucd6NHDDw8YANddF2o5UooCSkQy0r//DT/7Gezc\nCXfe6V8SLQooEck48+dD167+nqdrr4VHHw27IimPAkpEMsqKFb6XiLVr/fmm556DLH0TRpL+WUQk\nY2zc6ENp8WI4+WR49VVo0CDsqqQiCigRyQgFBb7T1y++gA4d/L1OTZuGXZVURgElImmvqAiuuQYm\nT4aDD/a9RLRuHXZVUhUFlIikNefgtttg7Fho3tz3r9euXdhVSTIUUCKS1v70J3j2Wdh3X3jrLTj+\n+LArkmQpoEQkbT37LDzwgL9K75VX4Iwzwq5IqkMBJSJpaexY6N3bDw8ZApdcEm49Un0KKBFJO5Mn\nwy9+AcXF/hBfz55hVyQ1oYASkbQyaxZcdBEUFvo9qL59w65IakoBJSJpY9EifyPupk1w+eXw97+D\nWdhVSU0poEQkLaxZ47swWrkSfvITGDkSsrPDrkpqQwElIikvP993/rpgAXTsCK+/7i8rl9SmgBKR\nlLZ9O1x2GUyfDocdBhMm+BtyJfUpoEQkZRUXww03+AcPHnCA/3nQQWFXJUFRQIlISnIO7roLXnrJ\nd/o6YQIccUTYVUmQFFAikpIGDIAnnvCPyxg7Fjp1CrsiCZoCSkRSzvPPwx/+4IdfeAF++tNw65G6\noYASkZQyfjzcdJMfHjQIrr463Hqk7gQWUGa2r5kNM7MlZrbZzGaaWZeg5i8i8skncMUV/vlO994L\nv/lN2BVJXQpyDyoHWAqcBTQH+gKjzax9gMsQkQy1ZEljunWDbdvgxhvhz38OuyKpazlBzcg5twV4\noMSocWa2COgELA5qOSKSeZYtgz/84QTWr4fu3f1jNNSFUfqrs3NQZpYLHAXMratliEj627DB96+3\nenVDTjsNRo2CnMD+tJYoq5N/ZjNrALwIPO+c+6qc9p5AT4Dc3FxisVhdlFEr+fn5kawryrTOkheP\nxykqKtL6qkJhYRZ33XUic+c259BDN3P33bOYNm1n2GWljFT/Pxl4QJlZFjAC2A7cXt40zrmhwFCA\nzp07u7y8vKDLqLVYLEYU64oyrbPktWjRgng8rvVViZ07fRdGc+ZAmzYwYMAcLrro9LDLSimp/n8y\n0IAyMwOGAblAV+fcjiDnLyKZwTn41a/gzTehZUuYNAlWry4MuyypZ0GfgxoCHANc6JzbFvC8RSRD\n9OsHw4ZBo0b+vqdjjw27IglDkPdBtQN6AR2BlWaWn3hdE9QyRCT9Pfmkv4Q8OxtGj4ZTTw27IglL\nkJeZLwF04aeI1NioUfDb3/rhf/7TX1IumUtdHYlIJLz7Llx3nT//9Je/QI8eYVckYVNAiUjoPvsM\nLr0UduyAPn32dAQrmU0BJSKhWrgQunTxj23/+c/h8cfVS4R4CigRCc2qVXDeebB6tX9kxvDhkKVv\nJUnQpiAiodi0ye85ffONf9jgmDGwzz5hVyVRooASkXpXWOjPOX3+uX9M+9tvQ7NmYVclUaOAEpF6\nVVTkr9Z7/3046CB45x048MCwq5IoUkCJSL1xzt/n9OqrsN9+MGECHHZY2FVJVCmgRKTePPwwPP20\nP9f0xhvQsWPYFUmUKaBEpF7885/Qt6+/hPyllyCFO9mWeqKAEpE698Yb0KuXHx482D9GQ6QqCigR\nqVMffghXXw3FxfDf/+0foyGSDAWUiNSZOXPgwguhoAB69vQBJZIsBZSI1IklS+D88yEe9/c8DR6s\nLoykehRQIhK4tWt9OC1fDmee6S+KyM4OuypJNQooEQnUli3+OU7z5sHxx/sLJBo2DLsqSUUKKBEJ\nzI4dcMUV8Omn0K4dTJwILVqEXZWkKgWUiASiuBhuusn3DtG6te/C6OCDw65KUpkCSkQCcc89MGIE\nNGkC48fDUUeFXZGkOgWUiNTa3/4Gjz0GOTn+sRknnxx2RZIOFFAiUisvvgh33umHhw/3V++JBEEB\nJSI1NmkS9Ojhhx9/HK65JtRyJM0ooESkRqZN833q7dwJv/893HFH2BVJulFAiUi1zZsH3br5e55+\n+Uv4y1/CrkjSkQJKRKpl+XJ/nmntWujSxT9GI0vfJFIHtFmJSNLicbjgAt/P3imn+CfjNmgQdlWS\nrhRQIpKUbdvgootg9mw4+mh/r1OTJmFXJelMASUiVSoq8lfoffABHHKIv3qvVauwq5J0p4ASkUo5\nB717w+uv+371Jk6Etm3DrkoygQJKRCr14IMwdKjvkfytt+C448KuSDKFAkpEKjRkiA+orCwYNQpO\nPz3siiSTKKBEpFyvvQa33eaHn33WXyAhUp8UUCJSRizmL4pwDvr3h5tvDrsiyUSBBpSZ3W5m082s\n0MyGBzlvEakfM2fCxRfD9u1w++1w331hVySZKifg+S0H+gPnA40CnreI1LFvvvG9Q2zaBFdeCQMH\nglnYVUmmCjSgnHNjAcysM9AmyHmLSN1avdp3YbRyJZx9NrzwAmRnh12VZLKg96CSYmY9gZ4Aubm5\nxGKxMMqoVH5+fiTrijKts+TF43GKioois762bs3mjjtO5Ouv9+PIIzdzxx0zmTq1KOyy9qLtq/pS\nfZ2FElDOuaHAUIDOnTu7vLy8MMqoVCwWI4p1RZnWWfJatGhBPB6PxPravh26d/c9lB9+OHzwQTNy\nc88Iu6wytH1VX6qvM13FJ5LBiov9Awf/93/hwAN9F0a5uWFXJeIpoEQylHP+IYMvvwxNm8KECX4P\nSiQqAj3EZ2Y5iXlmA9lm1hDY6ZzbGeRyRKT2Hn0UBg3yj8v417/gpJPCrkhkb0HvQfUFtgH3ANcm\nhvsGvAwRqaX/+R+45x5/CfnIkXDOOWFXJFJW0JeZPwA8EOQ8RSRY48bBLbf44UGD/P1OIlGkc1Ai\nGWTqVB9IRUVw//3w61+HXZFIxRRQIhniP/+Bbt38k3FvugkeeijsikQqp4ASyQBLl/peIjZs8L2S\nP/OMujCS6FNAiaS59evhggtg2TL/PKdXXoGcUG7RF6keBZRIGtu6FS680B/e+8EP4M03oZG6cZYU\noYASSVM7d8JVV8HHH8Ohh8LEidCyZdhViSRPASWShpyDnj39JeX77++7MGqj5wtIilFAiaSh++/3\nN+M2agTjx8Mxx4RdkUj1KaBE0sygQfDII/5ZTq+9Bj/6UdgVidSMAkokjbzyCvTp44efew66dg23\nHpHaUECJpIl334Vf/tIPP/ronmGRVKWAEkkDM2bApZfCjh3+ERp33RV2RSK1p4ASSXELFkCXLpCf\nD9dcA489pl4iJD0ooERS2MqVvgujNWvgvPP8eacs/a+WNKFNWSRFbdzouzBatAh++EMYMwb22Sfs\nqkSCo4ASSUEFBXDJJTBrFhx5pL/XqWnTsKsSCZYCSiTFFBXBdddBLAbf+x688w4ccEDYVYkETwEl\nkkKcg9/8xt+Au99+vn+99u3DrkqkbiigRFLIn/8MgwfDvvv6nslPOCHsikTqjgJKJEX84x/Qr5+/\nSu+ll+Css8KuSKRuKaBEUsC//gW/+pUfHjwYfvazcOsRqQ8KKJGImzIFrr4aiovhwQehV6+wKxKp\nHwookQibPRsuuggKC/0eVL9+YVckUn8UUCIRtXix7yVi40Z/SO+pp9SFkWQWBZRIBK1d68NpxQp/\nMcSLL/rnO4lkEgWUSMTk50O3bjB/Ppx4IrzxBjRsGHZVIvVPASUSITt2wOWXw7Rp/gbcCROgefOw\nqxIJhwJKJCKKi+HGG2HSJN910Tvv+K6MRDKVAkokIu6+G0aOhCZN4O23fSewIplMASUSAQMG+FeD\nBvD669C5c9gViYRPASUSshEj4Pe/98PPPw8//Wm49YhEhQJKJEQTJvjzTgBPPAE//3m49YhESaAB\nZWb7m9nrZrbFzJaY2S+CnL9IOtm6NZvLL4edO/35pz59wq5IJFpyAp7f08B2IBfoCIw3s1nOubkB\nL0ckpW3dCt9805SiIrj+enjkkbArEokec84FMyOzJsAG4Djn3PzEuBHAd865eyp6X7NmzVynTp0C\nqSFI8XicFi1ahF1GStE6S05BAUybNhPnYP/9O3LccerCKBnavqovquts8uTJM5xzVV4KFOQe1FHA\nzl3hlDALKPPUGjPrCfQEaNCgAfF4PMAyglFUVBTJuqJM66xqO3dmsWBBU5yDrCzHIYdsZOPGYP5I\nTHfavqov1ddZkAHVFNhUatxGoFnpCZ1zQ4GhAJ07d3bTp08PsIxgxGIx8vLywi4jpWidVS4e9/3q\nbd8OTZvm0b79Rr744vOwy0oZ2r6qL6rrzJI8ZBBkQOUD+5Uatx+wOcBliKSkjRuha1f44gvo0AFa\ntYItW7TnJFKZIK/imw/kmFnJ+99PBHSBhGS0DRv8vU1Tp0Lbtr4LowYNwq5KJPoCCyjn3BZgLPAn\nM2tiZj8GLgZGBLUMkVSzdi2ccw78+99w2GEwebIPKRGpWtA36vYGGgGrgZeBW3WJuWSq1avh7LPh\n8899v3qTJ/seykUkOYHeB+WcWw9cEuQ8RVLRwoXQpQssWABHHw3vv6+eyUWqS10diQRs2jQ49VQf\nTh07QiymcBKpCQWUSIDeegvy8mDNGjjvPJgyBXJzw65KJDUpoEQC4Bz8/e9wySWwbRvccAOMGwfN\nytwFKCLJUkCJ1NK2bdCjB/z2t/6puH/8IwwbpkvJRWor6M5iRTLKkiXws5/BZ59B48Y+mK6+Ouyq\nRNKDAkqkhiZNgmuv9fc6ff/7/km4J5wQdlUi6UOH+ESqqbAQ7rwTLrjAh9P55/sbcRVOIsHSHpRI\nNcyb5596+/nnkJ0NDz0Ef/iDHxaRYCmgRJJQVARPPQX33ecfNvj978NLL8Epp4RdmUj6UkCJVOHL\nL+Gmm3xnrwDXXefDar/SffeLSKB0DkqkAoWF0L+/7w1i6lQ4+GB44w144QWFk0h90B6USDnGj4c+\nfeDrr/3vN98Mjz0GEXx6tkjaUkCJlLBgAfzudz6gAI45xh/OO/vscOsSyUQ6xCcCrFwJt90Gxx7r\nw6lZM/jb32DWLIWTSFi0ByUZLR6HAQPgiSf81XlZWb4fvYcfhoMOCrs6kcymgJKMtHYtDBoETz4J\nGzf6cRdfDH/+M/zgB+HWJiKeAkoyynff+b2lIUP8HhPAT37ig+nUU8OtTUT2poCSjDBtGgwcCK++\nCjt3+nFdu8L998Npp4Vbm4iUTwElaWvLFnjtNXjmGfjkEz8uOxuuvBLuvhtOOinc+kSkcgooSSvO\nwaef+sdejBoFmzf78S1bQs+e/kq9Qw8Nt0YRSY4CStLCt9/C6NHw3HO+a6JdTjsNbrzRP6OpSZPw\n6hOR6lNAScpavNgfwnv1VX+OaZcDD4Trr/fBdPTRoZUnIrWkgJKU4Zy/cXbCBBg7FqZP39PWuDF0\n6wa/+IX/qceti6Q+BZRE2oYN8O67PpQmToQVK/a0NWkC3bvDFVdAly4+pEQkfSigJFI2boQPP4TJ\nk2HKFL+XVFS0p/3gg/2TbLt18z8VSiLpSwEloXEOlizx54+mTvWhNGsWFBfvmSYnB846y+8hdekC\nxx8PZuHVLCL1RwEl9cI534vDF1/4vaJp0/xrzZq9p2vQAH70Ix9KZ54JP/6x77hVRDKPAkoCt2kT\nzJkDs2fv/dqwoey0rVvDD38IJ58MZ5zhuxvSYTsRAQWU1FBhIXzzjX9+0q7XtGknsnYtLF1a/nv2\n398fouvUyQfSySdD+/Y6ZCci5VNASbk2bfI3vy5dWvbn4sV+uOS5Iq8lAPvs45+rdPzxe14nnADf\n+57CSESSp4DKIEVFsG4drFq192v1av9z5UpYtsyHz6ZNlc8rKwu+/3048sg9r23bvuCyy06gXTvd\nhyQitaeASkGFhb6PuQ0b/Gv9+sp/btjgL0ZYs6a8vZ7yNWoEbdv6fuvK+3nYYX5PqaRYbD1HHBH8\n5xWRzBRIQJnZ7UAP4HjgZedcjyDmm4qcgx07YNs2/yooqHp461bIz/ehs3nznuGKxu3YUfP6WraE\n3Nw9r4MO2vv3Qw7xAbT//jocJyLhCmoPajnQHzgfaFSdNxYWwvz5/vBT6Vdxcfnjq2qrqn3Hjj2v\n7dv3/rlr+LvvjmXQoKqn2zW8K3AKCpLfS6mpnBx/6XXLlj5IWrbce7i8ca1a+T7qSu/1iIhEVSAB\n5ZwbC2BmnYE21XnvnDnz6NAhr9TYK4HewFagaznv6pF4rQUuL6f9VuAqYClwXTntdwIXAvOAXuW0\n9wXOBWYCfcppfxg4DfgYuK9Ma3b2QBo37khW1rsUFPQnK4vdr+xsOP74ZznggA6sW/cW8+Y9TnY2\ne71uvXUE7dodyowZo5g4cUiZ9rFjX6N169YMHz6c4cOHs337nvNJAG+//TaNGzdm8ODB/P3vo8vU\nF4vFABgwYADjxo3bq61Ro0ZMmDABgIceeoj33ntvr/ZWrVoxZswYAO69916mTp26uy0ej3Pccccx\ncuRIAPr06cPMmTP3ev9RRx3F0KFDAejZsyfz58/fq71jx44MHDgQgGuvvZZly5bt1X7qqafyyCOP\nAHDZZZexbt26vdrPOecc+vXrB0CXLl3Ytm3bXu3du3fnrrvuAiAvL6/Murnyyivp3bs3W7dupWvX\nsttejx496NGjB2vXruXyy8tue7feeitXXXUVS5cu5brrym57d955JxdeeCFbt27l66+/LlND3759\nOffcc5k5cyZ9+pTd9h5++GFOO+00Pv74Y+67r+y2N3DgQDp27Mi7775L//79y7Q/++yzdOjQgbfe\neovHH3+8TPuIESM49NBDGTVqFEOGDCnT/tpre297pZXc9kaPDnbbKy4uZsqUKUDZbQ+gTZs22vZK\nbXvxeJwWLVoAe7a9efPm0atX2e+9+tz2khXKOSgz6wn09L81YZ99ihOHkxxm0KxZAS1bbga2sGzZ\nzsR79hxyOuCAfA48cB1FReuYP3/H7vFmDoC2beMcfPAKCgtXMWvWdsxciWngmGPW0K7dYrZsWcbU\nqQW723fVcPrpSzj00M/YvPkbJk3akmhzu39eeumXdOjQgEWL5jJmzKbd47Oy/M9f/3o6RxwRZ8aM\nWYwYES/z+W+++VPatl3Bxx/PJh4v296mzVRatVpITs5ciovjFBfvfVjvo48+onnz5nz11Vflvn/K\nlCk0bNiQ+fPnl9u+60ti4cKFZdq3bdu2u33RokVl2ouLi3e3f/vtt3u1FxUVsWrVqt3ty5YtK/P+\n5cuX725fvnx5mfZly5btbl+1alWZ9m+//XZ3+5o1a9hU6mqORYsW7W5fv349hYWFe7UvXLhwd3t5\n62b+/PlLNR+DAAAF6UlEQVTEYjEKCgrKbf/qq6+IxWJs3Lix3Pa5c+cSi8VYvXp1ue2zZ8+mWbNm\nbN68GedcmWlmzZpFTk4OX3/9dbnv/+yzz9i+fTtz5swpt3369OnE43FmzZpVbvunn37KihUrmD27\n/G1v6tSpLFy4kLlz55bbHua217hx4wq3PYAGDRpo2yu17RUVFe0e3rXtlbfuoH63vWSZcy7piauc\nmVl/oE11zkF17tzZTS/ZLXVExGKxcv/KkYppnSUvLy+PeDxe5q98qZi2r+qL6jozsxnOuc5VTZeV\nxIxiZuYqeH0YTLkiIiJ7q/IQn3Murx7qEBER2UtQl5nnJOaVDWSbWUNgp3NuZxDzFxGRzFPlIb4k\n9QW2AfcA1yaG+wY0bxERyUBBXWb+APBAEPMSERGB4PagREREAqWAEhGRSFJAiYhIJCmgREQkkhRQ\nIiISSQooERGJJAWUiIhEkgJKREQiSQElIiKRpIASEZFIUkCJiEgkKaBERCSSFFAiIhJJCigREYkk\nBZSIiESSAkpERCJJASUiIpGkgBIRkUhSQImISCQpoEREJJIUUCIiEkkKKBERiSQFlIiIRJICSkRE\nIkkBJSIikaSAEhGRSFJAiYhIJCmgREQkkhRQIiISSQooERGJJAWUiIhEkgJKREQiqdYBZWb7mtkw\nM1tiZpvNbKaZdQmiOBERyVxB7EHlAEuBs4DmQF9gtJm1D2DeIiKSoXJqOwPn3BbggRKjxpnZIqAT\nsLi28xcRkcxU64AqzcxygaOAuZVM0xPoCZCbm0ssFgu6jFrLz8+PZF1RpnWWvHg8TlFRkdZXNWj7\nqr5UX2fmnAtuZmYNgAnAQudcr2Te07lzZzd9+vTAaghKLBYjLy8v7DJSitZZ8vLy8ojH48ycOTPs\nUlKGtq/qi+o6M7MZzrnOVU1X5TkoM4uZmavg9WGJ6bKAEcB24PZaVS8iIhmvykN8zrm8qqYxMwOG\nAblAV+fcjtqXJiIimSyoc1BDgGOAc51z2wKap4iIZLAg7oNqB/QCOgIrzSw/8bqm1tWJiEjGCuIy\n8yWABVCLiIjIburqSEREIkkBJSIikRTofVA1KsBsDbAk1CLK1xpYG3YRKUbrrHq0vqpH66v6orrO\n2jnnDqhqotADKqrMbHoyN5LJHlpn1aP1VT1aX9WX6utMh/hERCSSFFAiIhJJCqiKDQ27gBSkdVY9\nWl/Vo/VVfSm9znQOSkREIkl7UCIiEkkKKBERiSQFlIiIRJICKklmdqSZFZjZyLBriSoz29fMhpnZ\nEjPbbGYzzaxL2HVFjZntb2avm9mWxLr6Rdg1RZW2qdpJ9e8tBVTyngb+HXYREZcDLAXOApoDfYHR\nZtY+xJqi6Gn8gz1zgWuAIWb2g3BLiixtU7WT0t9bCqgkmNnVQBx4L+xaosw5t8U594BzbrFzrtg5\nNw5YBHQKu7aoMLMmwGVAP+dcvnPuQ+BN4LpwK4smbVM1lw7fWwqoKpjZfsCfgDvCriXVmFkucBQw\nN+xaIuQoYKdzbn6JcbMA7UElQdtUctLle0sBVbWHgGHOuWVhF5JKzKwB8CLwvHPuq7DriZCmwKZS\n4zYCzUKoJaVom6qWtPjeyuiAMrOYmbkKXh+aWUfgXOCJsGuNgqrWV4npsoAR+PMst4dWcDTlA/uV\nGrcfsDmEWlKGtqnkpdP3Vq2fqJvKnHN5lbWbWR+gPfCtmYH/6zfbzI51zp1U5wVGTFXrC8D8ihqG\nvwCgq3NuR13XlWLmAzlmdqRzbkFi3InokFWFtE1VWx5p8r2lro4qYWaN2fuv3bvw//C3OufWhFJU\nxJnZM0BH4FznXH7Y9USRmb0COOBm/Lp6GzjNOaeQKoe2qepJp++tjN6DqopzbiuwddfvZpYPFKTa\nP3J9MbN2QC+gEFiZ+OsNoJdz7sXQCoue3sBzwGpgHf6LQ+FUDm1T1ZdO31vagxIRkUjK6IskREQk\nuhRQIiISSQooERGJJAWUiIhEkgJKREQiSQElIiKRpIASEZFIUkCJiEgk/T/XSE/diHEg1QAAAABJ\nRU5ErkJggg==\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "plt.plot(z, elu(z), \"b-\", linewidth=2)\n", "plt.plot([-5, 5], [0, 0], 'k-')\n", @@ -494,7 +574,25 @@ "cell_type": "code", "execution_count": 25, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Saving figure selu_plot\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xt8FPW5x/HPEwIIiEZBchTUeMN7RYhtxfaYVqyKl9qD\n1VpAsVoQqxaRVkUUDnCwtVTRFrAIgoJWqeIN0VZpo7WIFSTeRcWCICoXXTHhEhJ+54/fxixLLptk\nNjO7+b5fr3llmZnMPDtM9rsz++yMOecQERGJmpywCxAREamJAkpERCJJASUiIpGkgBIRkUhSQImI\nSCQpoEREJJIUUCJ1MLOVZjaiGdYzxszebIb15JjZn8xso5k5MytK9zrrqWeWmc0PswaJLgWUpMTM\n9jGzKfEX7G1m9pmZLTSzUxPmKY6/6CUPDybM48zsvFrWMcjMSmuZVuvvBaGOgDgBmBLgegriz6Uw\nadJE4OSg1lOHvsAlwNnAvsCiZlgnZlYUf96dkyb9EhjQHDVI5skNuwDJGI8A7YFLgQ+ALvgX1E5J\n880ERiaN25L26tLEObe+mdZTCtQYzgE7FPjEOdcswVQf59yXYdcg0aUjKKmXmeUB3wWud84tdM6t\ncs694pyb6Jx7MGn2zc65T5OGtL8ImdnpZvZPM/vCzD43s7+a2ZFJ8+xnZvfHT29tNrMSM/uemQ0C\nRgNHJxz1DYr/zten+MzsATN7JGmZOWa22syGp1jHf+I/X4mvpzj+ezsdwcWXe1N82dvM7A0z+2HC\n9KojsX5m9mz8+bydeERbwzaaBdwOHBD/3ZXx8cVm9sfkeRNPvcXnmWJmE8xsg5mtM7OJZpaTME+b\n+PRV8Zo/NLOrzawA+Ed8tvXxdc+qZT1tzWxS/Ah9q5ktNrPvJEyvOhI7xcxejj/vJWbWs7bnLZlL\nASWpqHp3f46Z7RZ2MbXoAEwCvgkUAV8CT5pZGwAz6wA8DxQA5wLHAmPjv/sQ8HtgOf60177xccnm\nAGea2Z4J406Oz//nVOqIjwc4Pf57/1PL8/kl8CvgunitjwLzzKxH0nz/B9wJHAe8AjxoZrvXscyx\nwJr4uk+oZb7a9AcqgN7AlcAw4IKE6fcCFwHDgSPxR9sxYDXQLz7P0fF1/7KWddwaX+bPgOOBN4Bn\nzGzfpPluAa4HegIbgfvNzBr4fCTqnHMaNNQ74F9gPge2Ai/hPzP5VtI8xUA51YFWNVyRMI8Dzqtl\nHYOA0lqm1fp7tczfAagEvhP/98+Br4DOtcw/BnizhvErgRHxx7nAZ8ClCdOnA39rQB0F8edSWNf6\ngY+Bm2vYvnOSljMkYXrX+Ljv1FHPCGBlDcv9Y9K4WcD8pHleSprnWWB6/PFh8XWfXst6i+LTO9e2\nnvi2KgcuSpjeClgBjE9azmkJ85wUH9ct7L8TDcEOOoKSlDjnHgH2w3+4/jT+XfRiM0v+vOkhoEfS\ncH+66zOzQ+Kn4FaY2SZ8kOQAB8RnOR543Tm3obHrcM5V4J9f//g62+KDe04D6kjlueyB39b/Spr0\nInBU0rjXEx6vjf/skuq6Guj1pH+vTVjX8cAOqk/lNcYhQGsSnrdzrhL/hijM5y0hUZOEpMw5txX/\nrvlZYKyZTQfGmNlE51x5fLYvnXMfNHIVm4B2ZtbaObe9amT8MzDwp8tqMx9/6moI/uijAngbaFPH\n7zTGHOAlM+sKfCu+/HnNWEfy7Qe+3k7OORc/y9XQN547gOTTY61rmG970r9dI9bVWLU+74RpesOd\nZfQfKk3xNv5NTlCfSy3H75PHJ43vmTB9F2bWCTgCmOCce8459w7QkZ3fgC0DvlFDm3OVcvzppDo5\n5/6N72K8EH8k9bjzHXip1lEV5LWuyzm3CX9UcFLSpO/gt3nQ1uM/F0p0XAOXUYL/v/teLdPrfd74\nU3nlJDxvM2sFnEh6nrdEnI6gpF7xF96/APfgT618BRQCvwYWxl9Qq7Q3s/9KWkS5c+7zhH8X1PBh\n/4fOubfM7G/A9HhX3AqgO3AHMNc591EtJX4BbAB+bmar8Z/F/A5/9FLlAfyH6o+b2fX4o5tjgK+c\nc//Af9Z0YLwb7KP4+G21rO9+4DL850CJTQ6p1LEO33Z/WryLbqurucvxd/ij1PeBpfjvCn2X6rAO\n0t+BSWZ2Dv5NwBBgf/w2SYlz7j0zm4v/v/sl8CrQDShwzs0GVuGPdM40syeBLVXBnrCMMjObCvzW\nzDbgOx6vAfIJ8LtokkHC/hBMQ/QHoC0wAd8l9gWwGXgfuA3YO2G+YvyLUPLwYsI8NU13wFnx6Xn4\nQPogvp73gN8Cu9dT4/eBN/FNHG8Cp+EbNAYlzNMN/xlSLL7sZUBRwnN8OP78XNXvkdAkkbCcg+Pz\nfAbkNqKOy/AhWAkUx8eNYecmiRzgJnwHXDm+m+3chOkF1NxsUWczCTU3SbQGJuPDdQPwv9TcJFFf\nI0VbfBfex8A2/BuMKxOm3wR8gj+lOKuOZUyKb9ttwGISmj6oodmitm2hIfMHi/8Hi4iIRIo+gxIR\nkUhSQImISCQpoEREJJIUUCIiEkmht5l37tzZFRQUhF3GLsrKyujQoUPYZWQUbbPULV++nMrKSo46\nKvkCCVKbqO5f5eXwzjtQUQGdOkGUXs6ius2WLl26wTm3T33zhR5QBQUFLFmyJOwydlFcXExRUVHY\nZWQUbbPUFRUVEYvFIrnvR1UU969Nm+Ckk3w4fe978Mwz0Cboa5c0QRS3GYCZrUplPp3iExFphIoK\nuOACePNNOOIIeOSRaIVTNlBAiYg0kHNw9dX+iKlzZ3jqKdhrr7Cryj4KKBGRBpo0CaZOhbZt4fHH\n4eCDw64oOymgREQa4LHH4Npr/eN774XevcOtJ5sFGlBmNsfMPjGzTWb2npldFuTyRUTCtHQp9O/v\nT/GNH+8/g5L0CfoI6hb81Yv3AM4BxptZr4DXISLS7FavhrPPhs2b4eKLYWTyrTolcIEGlHPuLVd9\ni4Kqq1QfEuQ6RESa26ZNcOaZ8MknUFQE06aBJd/iUQIX+PegzGwKMAhoh7+dwYIa5hkMDAbIz8+n\nuLg46DKarLS0NJJ1RZm2WepisRiVlZXaXg0Q1v5VWWmMHHkMb7zRif3338zw4a+yaFFF/b8YAZn+\nN5mW220k3AWzCPitS7h9d7LCwkIXxS8rRvULblGmbZa6qi/qlpSUhF1Kxghj/3IOrrwSpkzx7eSL\nF8MhGXROKKp/k2a21DlXWN98aenic85VOudexN8gbmg61iEikm533OHDqU0b372XSeGUDdLdZp6L\nPoMSkQz0+OMwfLh/PGuWv6SRNK/AAsrMupjZT8xsdzNrZWanARcCC4Nah4hIc1i6FH76U3+Kb9w4\nuPDCsCtqmYJsknD403l34YNvFTDMOfdEgOsQEUmr5HbyG28Mu6KWK7CAcs6tB04OankiIs3tq6/g\nrLN8O/nJJ6udPGy61JGICP7q5D/5Cbz+OnTvDvPm6erkYVNAiUiL5xwMGwYLFvibDi5YAHvvHXZV\nooASkRbvzjth8mS1k0eNAkpEWrQnn4RrrvGPZ86E73wn3HqkmgJKRFqsV1/1nzs5B2PH+tZyiQ4F\nlIi0SGvWVLeTX3QRjBoVdkWSTAElIi1OVTv52rXw3/+tdvKoUkCJSItSUeGvDPHaa3DYYb6dvG3b\nsKuSmiigRKRFGT4cnnqqup28U6ewK5LaKKBEpMW48074wx+q28kPPTTsiqQuCigRaRHmz69uJ7/n\nHrWTZwIFlIhkvWXLfDv5jh0wZgz07x92RZIKBZSIZLU1a3zHXlkZDBgAN98cdkWSKgWUiGSt0lL/\nXaeqdvLp09VOnkkUUCKSlSorfTt5SYnayTOVAkpEstLw4b4xYu+9q9vKJbMooEQk6/zhD76lvKqd\n/LDDwq5IGkMBJSJZ5amn/L2dAGbMgO9+N9x6pPEUUCKSNUpK4IILfDv56NG+a08ylwJKRLLCxx9X\nt5P37+8DSjKbAkpEMl5VO/nHH/srRMyYoXbybKCAEpGMVlnpbzS4bJm/tt5jj6mdPFsooEQko117\nrb9tu9rJs48CSkQy1uTJcMcd0Lo1PPoodO8edkUSJAWUiGSkBQvg6qv94xkz/KWMJLsooEQk47z2\nWnU7+c03w8CBYVck6aCAEpGMsnatbycvLfXNEWPGhF2RpIsCSkQyRlmZbydfswZOOknt5NlOASUi\nGaGqnfzVV+GQQ3w7+W67hV2VpJMCSkQywl13HcITT8Bee/kGic6dw65I0k0BJSKRN2UKPPzw/mon\nb2EUUCISaU8/DVdd5R9Pnw4nnxxuPdJ8AgsoM2trZjPMbJWZfWVmJWZ2RlDLF5GW57XX4PzzfTv5\nwIErueiisCuS5hTkEVQusBo4GdgTGAXMNbOCANchIi1EYjv5hRfCJZesDLskaWaBBZRzrsw5N8Y5\nt9I5t8M5Nx/4D9ArqHWISMuQ3E5+zz1qJ2+JctO1YDPLB7oDb9UwbTAwGCA/P5/i4uJ0ldFopaWl\nkawryrTNUheLxaisrNT2qkFlJYwefQyvvtqZ/fbbwogRr7J48XbtX42Q6dvMnHPBL9SsNfA0sMI5\nN6SueQsLC92SJUsCr6GpiouLKSoqCruMjKJtlrqioiJisRglJSVhlxI5114Lt93m28lfegkOP9yP\n1/7VcFHdZma21DlXWN98gXfxmVkOMBsoB64Mevkikr2mTvXh1Lo1zJtXHU7SMgV6is/MDJgB5AN9\nnXPbg1y+iGSvZ56pbie/+26I4Bt/aWZBfwY1FTgS6OOc2xLwskUkS73xhm8nr6yEG2+Eiy8OuyKJ\ngiC/B3UgMAToAXxqZqXxoX9Q6xCR7PPJJ3DmmfDVV/4WGmPHhl2RREVgR1DOuVWAGkFFJGVV7eSr\nV0Pv3jBrFuTo+jYSp11BREJRWQkDBsDSpXDwwbo6uexKASUiobjuOh9KeXnw1FOwzz5hVyRRo4AS\nkWZ3113w+99Dbi488ggccUTYFUkUKaBEpFn99a9wZfwbktOmwfe/H249El0KKBFpNm+8AT/+sf/8\naeRIuOSSsCuSKFNAiUiz+PRTf3XyqnbycePCrkiiTgElImm3eTOccw589BF8+9swc6bayaV+2kVE\nJK127PDt5K+8AgcdBI8/Du3ahV2VZAIFlIik1XXXwaOPwp57+nbyLl3CrkgyhQJKRNJm2jSYONG3\nk8+bB0ceGXZFkkkUUCKSFn/7G1xxhX/8pz+pnVwaTgElIoF7883qdvIbboCf/SzsiiQTKaBEJFCf\nfuqvTr5pkw+p8ePDrkgylQJKRAKT3E5+771qJ5fG064jIoHYsQMuusi3kxcUqJ1cmk4BJSKBuOEG\nf+FXtZNLUBRQItJkd98Nt95afXXyo44KuyLJBgooEWmSZ5+FoUP947vuglNOCbceyR4KKBFptLfe\ngvPO8+3k110Hl14adkWSTRRQItIon31W3U5+3nkwYULYFUm2UUCJSINVtZOvWgXf+hbcd5/aySV4\n2qVEpEGq2sn//W+1k0t6KaBEpEFGjvSdenvs4dvJ8/PDrkiylQJKRFI2fTr89rdqJ5fmoYASkZQ8\n9xxcfrl/PHUq9OkTbj2S/RRQIlKvt9+ubif/9a/hssvCrkhaAgWUiNSpqp38yy+hXz+45ZawK5KW\nQgElIrXasgV++ENYuRK++U21k0vz0q4mIjWqaid/+WU48EB44glo3z7sqqQlUUCJSI1uvBEefljt\n5BIeBZSI7OKee+A3v4FWrXxIHX102BVJS6SAEpGdLFwIQ4b4x1OnwqmnhluPtFwKKBH52ttv+069\nigr41a/g5z8PuyJpyQINKDO70syWmNk2M5sV5LJFJL3WratuJ/+f//Gn+ETClBvw8tYC44HTAF0+\nUiRDJLaTn3ACzJ6tdnIJX6AB5ZybB2BmhUC3IJctIumxYwcMGgSLF8MBB6idXKIj6COolJjZYGAw\nQH5+PsXFxWGUUafS0tJI1hVl2mapi8ViVFZWRmJ73X33QcydeyAdOlQwZswy3n23jHffDbuqXWn/\narhM32ahBJRzbhowDaCwsNAVFRWFUUadiouLiWJdUaZtlrq8vDxisVjo22vmTHjgAd9OPm9eLj/4\nwQmh1lMX7V8Nl+nbTGeZRVqov/8dBg/2jydPhh/8INx6RJIpoERaoHfeqW4nHzGi+ntPIlES6Ck+\nM8uNL7MV0MrMdgMqnHMVQa5HRBqvqp08FoMf/cjfgFAkioI+ghoFbAGuBwbEH48KeB0i0khbt8K5\n58J//gOFhTBnjtrJJbqCbjMfA4wJcpkiEoyqdvKXXoL991c7uUSf3juJtBA33wwPPQQdO/qrk++7\nb9gVidRNASXSAsycCf/3f76d/C9/gWOPDbsikfopoESy3D/+Ud1O/sc/wmmnhVuPSKoUUCJZ7N13\n/YVfKypg+HC4/PKwKxJJnQJKJEutX1/dTn7uuXDrrWFXJNIwCiiRLFTVTv7hh9Crl28nb9Uq7KpE\nGkYBJZJlduyASy6BRYt8O/mTT0KHDmFXJdJwCiiRLDN6NDz4oG8nnz9f7eSSuRRQIlnk3nth/Hh/\nOm/uXPjGN8KuSKTxFFAiWaK4GH7+c//4D3+A008PtRyRJlNAiWSB5ct9O/n27XDNNTB0aNgViTSd\nAkokw23Y4NvJv/gCzjkHfve7sCsSCYYCSiSDVbWTr1gBPXtW3x1XJBsooEQylHPws5/Bv/4F3bqp\nnVyyjwJKJEONHg1//jPsvru/Ovl++4VdkUiwFFAiGei++2DcOH+zwYceUju5ZCcFlEiGef55uOwy\n//jOO6Fv33DrEUkXBZRIBlm+HH70I99OPmwY/OIXYVckkj4KKJEMkdxOPnFi2BWJpJcCSiQDbNvm\nj5xWrIDjj4f771c7uWQ/BZRIxFW1k7/4InTt6tvJd9897KpE0k8BJRJxY8b4L+BWtZN37Rp2RSLN\nQwElEmGzZ8PYsdXt5McdF3ZFIs1HASUSUS+8AJde6h/fcYfayaXlUUCJRND771e3k199NVx5ZdgV\niTQ/BZRIxGzc6I+WPv8czj4bbrst7IpEwqGAEomQbdv81ck/+MC3k+vq5NKSKaBEIsI5fwkjtZOL\neAookYgYOxbmzPG3zJg/X+3kIgookQiYM8d/3yknBx58EHr0CLsikfApoERC9s9/VreTT5oEZ50V\nbj0iUaGAEgnR++/7pojycrjqKj+IiBdoQJnZ3mb2qJmVmdkqM/tpkMsXySYVFcaZZ/p28jPPhNtv\nD7sikWjJDXh5k4FyIB/oATxlZq85594KeD0iGc05WLmyA2Vl/vOmBx9UO7lIMnPOBbMgsw7AF8Ax\nzrn34uNmAx87566v7fc6duzoevXqFUgNQYrFYuTl5YVdRkbRNkvdyy+XsHUrtGnTg549oW3bsCuK\nPu1fDRfVbfb8888vdc4V1jdfkEdQ3YGKqnCKew04OXlGMxsMDAZo3bo1sVgswDKCUVlZGcm6okzb\nLDXbtuWwdat/3LVrGVu2bGfLlnBrygTavxou07dZkAG1O7ApadyXQMfkGZ1z04BpAIWFhW7JkiUB\nlhGM4uJiioqKwi4jo2ib1c856NMH3n23iLy8cj78cFHYJWUM7V8NF9VtZmYpzRdkk0QpsEfSuD2A\nrwJch0hGmz8f/v53yM2Frl112CRSlyAD6j0g18wOSxh3HKAGCRGgshKuj38ae+CBkJsbzOe/Itkq\nsIByzpUB84CxZtbBzE4CfgjMDmodIpnsvvvg7behoAD22y/sakSiL+gv6l4BtAPWAX8GhqrFXATK\nyuDmm/3j8eP9JY1EpG6B/pk45z53zp3rnOvgnDvAOfdAkMsXyVS33gpr1kDPnnDhhWFXI5IZ9D5O\nJM1WrfIBBXDnnTp6EkmV/lRE0uxXv4KtW/2R00knhV2NSOZQQImkUXEx/OUv0L599VGUiKRGASWS\nJpWV8Mtf+sfXXw/duoVbj0imUUCJpMndd8Prr/vvPI0YEXY1IplHASWSBp9+Cjfc4B9PnAjt2oVb\nj0gmUkCJpMFVV0EsBmecAf36hV2NSGZSQIkE7LHH4OGHoUMHmDoVUrwupogkUUCJBOjLL+EXv/CP\nJ0zwnz+JSOMooEQCdP31sHYtfPvb1UElIo2jgBIJyAsvwF13QevWMH26buEu0lQKKJEAbNoEF1/s\nH99wAxx9dLj1iGQDBZRIAK66Clau9BeDvfHGsKsRyQ4KKJEmeughf6+ndu3g/vuhTZuwKxLJDgoo\nkSb46CO4/HL/+Lbb4Igjwq1HJJsooEQaqbISLrrIfyH37LNhyJCwKxLJLgookUaaMAGefx7y82HG\nDH0hVyRoCiiRRnj6aRg92ofSfffBPvuEXZFI9skNuwCRTPPhh9C/PzgH48bBD34QdkUi2UlHUCIN\nsHmzv/jrF1/4z51Gjgy7IpHspYASSZFzMHQolJTAoYf6U3s5+gsSSRv9eYmk6PbbfSi1bw/z5kFe\nXtgViWQ3BZRICh55pPquuDNnwrHHhluPSEuggBKpx+LFMGCAP8X3m9/A+eeHXZFIy6CAEqnDihVw\nzjmwdSsMHgy//nXYFYm0HAookVqsWwd9+8L69XD66TB5sr6MK9KcFFAiNfj8c//9pvfeg+OO8xeE\nzdW3BkWalQJKJMmmTXDGGfDaa9C9O/z1r7DHHmFXJdLyKKBEEpSVwVlnwb//DQcdBAsX+mvtiUjz\nU0CJxJWVwQ9/CP/8J3Tt6sOpW7ewqxJpuXRWXQR/y4yzzoJ//Qu6dPHhdNBBYVcl0rLpCEpavPXr\n4Xvf8+G0//7+COrww8OuSkR0BCUt2po1cOqp8O67/vp6CxfCAQeEXZWIQEBHUGZ2pZktMbNtZjYr\niGWKpNvrr0Pv3j6cjj3WHzkpnESiI6hTfGuB8cA9AS1PJK0WLICTToLVq31IFRfDf/1X2FWJSKJA\nAso5N8859xiwMYjliaTT5Mn+Xk6lpXDhhf603t57h12ViCQL5TMoMxsMDAbIz8+nuLg4jDLqVFpa\nGsm6oizq26y83Jg8+VCeeKIrABddtJJBg1ayeHHz1xKLxaisrIz09oqaqO9fUZTp2yyUgHLOTQOm\nARQWFrqioqIwyqhTcXExUawryqK8zVatgh//GF55Bdq0genTYeDAAqAglHry8vKIxWKR3V5RFOX9\nK6oyfZvVe4rPzIrNzNUyvNgcRYo0xTPPQM+ePpwOPNC3kw8cGHZVIlKfeo+gnHNFzVCHSODKy+Hm\nm+HWW/29nM44A2bPhk6dwq5MRFIRyCk+M8uNL6sV0MrMdgMqnHMVQSxfpKHeegv69/cXfM3Jgf/9\nX7jxRv9YRDJDUH+uo4AtwPXAgPjjUQEtWyRlO3bApEnQq5cPp4MP9t9vuukmhZNIpgnkCMo5NwYY\nE8SyRBrrzTdhyBBYtMj/+9JL4fbboWPHcOsSkcbRe0rJeFu3wqhRcPzxPpz23Rcee8x36imcRDKX\nrsUnGcs5mD8fhg+HDz7w4y6/HG65BfLywq1NRJpOASUZ6Y034Jpr/FUgAI46CqZN85cvEpHsoFN8\nklHWrIHBg6FHDx9Oe+0Fd9wBJSUKJ5FsoyMoyQiffeZP3d11F2zbBq1awVVXwejR+l6TSLZSQEmk\nffKJ78SbPBk2b/bjzj/ff6/piCPCrU1E0ksBJZH0/vvwu9/Bvff6K0KAvwL5uHFw3HHh1iYizUMB\nJZGxYwc8+yxMmQJPPum79MygXz+47jo44YSwKxSR5qSAktBt3AgzZ/rPl1as8ONat4aLL4YRI+Dw\nw8OtT0TCoYCSUFRWwvPPw6xZMHeub3wAf8v1IUP8VSDy80MtUURCpoCSZuMcLFsG998PDz4Ia9f6\n8Wb+SuNXXOF/tmoVbp0iEg0KKEkr5/yXah97DB54AJYvr5528MHw05/CJZf4xyIiiRRQErjycn/6\n7okn/PDRR9XT9tkHLrjA3wrjW9/yR08iIjVRQEmTOeevhffEE/vxxz/6TrxNm6qnd+niW8T79YM+\nfXwDhIhIfRRQ0iirV8MLL8Bzz/lLDq1eDdD96+nHHAPnnOOD6Zvf1L2YRKThFFBSr/Jyf627RYv8\n8NJL/pp4iTp1gmOOWceFF3bh1FP1mZKINJ0CSnaydau/8d+rr/ph2TJ/Z9qqNvAqeXlw4olwyil+\n+MY34IUX3qaoqEs4hYtI1lFAtVDl5f5yQu+8A2+/XT288w5UVOw6/xFHQO/efjjxRP9vnbYTkXRS\nQGWxigr/2dCHH1YPy5f7IPrgA/9l2WQ5OXDkkdCzZ/XQo4duACgizU8BlaGc851yH3/sPw/6+GM/\nJAbSqlU1hxD49u5DDvFhdNRR/ueRR/rmhg4dmve5iIjURAEVMVu2wPr1sG6d/5k4rF27cyCVldW/\nvK5dfcNC1XDooT6QDj8c2rVL//MREWksBVTAnPMNBV9+CbFYasPGjdUhlEroVGnfHrp18yFUNXTr\nVh1GBQWw225pe6oiImnVYgJqxw4fHFu3+iHxcU3jli3LZ/lyf0RTWuqDI/FnbY/LympuMkhVmzb+\nagtVQ5cu1Y/33XfnMNpzT12JQUSyV+gB9ckncNNN/kV9+/bqn4mPGztu27bq0Km66V3qjmz0c8rN\n9U0FNQ177VXzuKow6thRoSMiAhEIqLVrlzN+fFHS2POBK4DNQN8afmtQfNgAnFfD9KHABcBqYODX\nY818l1rHjtey555nk5OznHXrhpCTw07D0UePom3bY+jY8VNefnkYrVr5K2zn5Pif/ftPoGfP3qxc\nuYiZM0d+Pb1quOOOSfTo0YPnnnuO8ePHs3179Sk8gD/96U8cfvjhPPnkk9x66+93qX727Nnsv//+\nPPTQQ0ydOnWX6Q8//DCdO3dm1qxZzJo1a5fpCxYsoH379kyZMoW5c+fuMr24uBiAiRMnMn/+/J2m\ntWvXjqeffhqAcePGsXDhwp2md+rUiUceeQSAG264gZdeeunrabFYjGOOOYY5c+YAMGzYMEpKSnb6\n/e7duzNt2jQABg8ezHvvvbfT9B49ejBp0iQABgwYwJqkbwSfeOKJ3HLLLQD069ePjRs37jT9lFNO\n4aabbgICTsdMAAAFTElEQVTgjDPOYMuWLTtNP+ussxgxYgQARUVFu2yb888/nyuuuILNmzfTt++u\n+96gQYMYNGgQGzZs4Lzzdt33hg4dygUXXMDq1asZOHDgLtOvvfZazj77bDZv3swHH3ywSw2jRo2i\nT58+lJSUMGzYsF1+f8KECfTu3ZtFixYxcuTIXaZPmrTzvpcscd/7/e8za9/bsWMHL7zwArDrvgfQ\nrVs37XtJ+14sFiMv3oJbte8tX76cIUOG7PL7zbnvpSr0gGrTBvbbz4dH1dCrF3z/+/603J137jzN\nDE47Dfr29afTRo/eeVpODgwYAOeeCxs2wNVX+3GJRyXXXusvwbN8ub/3ULJRoyA3913y8vKo4f+J\nPn3894EWLYKHH07fthERacnMORdqAYWFhW7JkiWh1lCT4uLiGt/lSO20zVJXVFRELBbb5V2+1E77\nV8NFdZuZ2VLnXGF98+laACIiEkkKKBERiSQFlIiIRJICSkREIkkBJSIikdTkgDKztmY2w8xWmdlX\nZlZiZmcEUZyIiLRcQRxB5eK/EXsysCcwCphrZgUBLFtERFqoJn9R1zlXBoxJGDXfzP4D9AJWNnX5\nIiLSMgV+JQkzywe6A2/VMc9gYDBAfn7+15c/iZLS0tJI1hVl2mapi8ViVFZWans1gPavhsv0bRbo\nlSTMrDXwNLDCOVfDRYR2pStJZA9ts9TpShINp/2r4aK6zQK7koSZFZuZq2V4MWG+HGA2UA5c2aTq\nRUSkxav3FJ9zrqi+eczMgBlAPtDXObe96aWJiEhLFtRnUFPxN1Dq45zbUt/MIiIi9Qnie1AHAkOA\nHsCnZlYaH/o3uToREWmxgmgzXwXoHrAiIhIoXepIREQiSQElIiKRFPoddc1sPbAq1CJq1hnYEHYR\nGUbbrGG0vRpG26vhorrNDnTO7VPfTKEHVFSZ2ZJUvkgm1bTNGkbbq2G0vRou07eZTvGJiEgkKaBE\nRCSSFFC1mxZ2ARlI26xhtL0aRtur4TJ6m+kzKBERiSQdQYmISCQpoEREJJIUUCIiEkkKqBSZ2WFm\nttXM5oRdS1SZWVszm2Fmq8zsKzMrMbMzwq4rasxsbzN71MzK4tvqp2HXFFXap5om01+3FFCpmwy8\nEnYREZcLrAZOBvYERgFzzawgxJqiaDL+xp75QH9gqpkdHW5JkaV9qmky+nVLAZUCM/sJEAMWhl1L\nlDnnypxzY5xzK51zO5xz84H/AL3Cri0qzKwD0A+4yTlX6px7EXgCGBhuZdGkfarxsuF1SwFVDzPb\nAxgLDA+7lkxjZvlAd+CtsGuJkO5AhXPuvYRxrwE6gkqB9qnUZMvrlgKqfuOAGc65NWEXkknMrDVw\nP3Cvc+7dsOuJkN2BTUnjvgQ6hlBLRtE+1SBZ8brVogPKzIrNzNUyvGhmPYA+wO1h1xoF9W2vhPly\ngNn4z1muDK3gaCoF9kgatwfwVQi1ZAztU6nLptetJt9RN5M554rqmm5mw4AC4CMzA//ut5WZHeWc\n65n2AiOmvu0FYH5DzcA3APR1zm1Pd10Z5j0g18wOc869Hx93HDplVSvtUw1WRJa8bulSR3Uws/bs\n/G53BP4/fqhzbn0oRUWcmd0F9AD6OOdKw64niszsQcABl+G31QKgt3NOIVUD7VMNk02vWy36CKo+\nzrnNwOaqf5tZKbA10/6Tm4uZHQgMAbYBn8bfvQEMcc7dH1ph0XMFcA+wDtiIf+FQONVA+1TDZdPr\nlo6gREQkklp0k4SIiESXAkpERCJJASUiIpGkgBIRkUhSQImISCQpoEREJJIUUCIiEkkKKBERiaT/\nB9c9MCs0b4SXAAAAAElFTkSuQmCC\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "plt.plot(z, selu(z), \"b-\", linewidth=2)\n", "plt.plot([-5, 5], [0, 0], 'k-')\n", @@ -519,7 +617,24 @@ "cell_type": "code", "execution_count": 26, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Layer 0: -0.26 < mean < 0.27, 0.74 < std deviation < 1.27\n", + "Layer 10: -0.24 < mean < 0.27, 0.74 < std deviation < 1.27\n", + "Layer 20: -0.17 < mean < 0.18, 0.74 < std deviation < 1.24\n", + "Layer 30: -0.27 < mean < 0.24, 0.78 < std deviation < 1.20\n", + "Layer 40: -0.38 < mean < 0.39, 0.74 < std deviation < 1.25\n", + "Layer 50: -0.27 < mean < 0.31, 0.73 < std deviation < 1.27\n", + "Layer 60: -0.26 < mean < 0.43, 0.74 < std deviation < 1.35\n", + "Layer 70: -0.19 < mean < 0.21, 0.75 < std deviation < 1.21\n", + "Layer 80: -0.18 < mean < 0.16, 0.72 < std deviation < 1.19\n", + "Layer 90: -0.19 < mean < 0.16, 0.75 < std deviation < 1.20\n" + ] + } + ], "source": [ "np.random.seed(42)\n", "Z = np.random.normal(size=(500, 100))\n", @@ -622,7 +737,22 @@ "cell_type": "code", "execution_count": 29, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0 Batch accuracy: 0.96 Validation accuracy: 0.924\n", + "5 Batch accuracy: 1.0 Validation accuracy: 0.9568\n", + "10 Batch accuracy: 0.94 Validation accuracy: 0.9668\n", + "15 Batch accuracy: 0.98 Validation accuracy: 0.9684\n", + "20 Batch accuracy: 1.0 Validation accuracy: 0.9712\n", + "25 Batch accuracy: 1.0 Validation accuracy: 0.9694\n", + "30 Batch accuracy: 1.0 Validation accuracy: 0.97\n", + "35 Batch accuracy: 1.0 Validation accuracy: 0.971\n" + ] + } + ], "source": [ "means = mnist.train.images.mean(axis=0, keepdims=True)\n", "stds = mnist.train.images.std(axis=0, keepdims=True) + 1e-10\n", @@ -824,7 +954,34 @@ "cell_type": "code", "execution_count": 35, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0 Test accuracy: 0.8727\n", + "1 Test accuracy: 0.8981\n", + "2 Test accuracy: 0.9129\n", + "3 Test accuracy: 0.922\n", + "4 Test accuracy: 0.9292\n", + "5 Test accuracy: 0.9342\n", + "6 Test accuracy: 0.9381\n", + "7 Test accuracy: 0.9419\n", + "8 Test accuracy: 0.9451\n", + "9 Test accuracy: 0.9471\n", + "10 Test accuracy: 0.9507\n", + "11 Test accuracy: 0.9521\n", + "12 Test accuracy: 0.9553\n", + "13 Test accuracy: 0.956\n", + "14 Test accuracy: 0.957\n", + "15 Test accuracy: 0.9583\n", + "16 Test accuracy: 0.9613\n", + "17 Test accuracy: 0.9608\n", + "18 Test accuracy: 0.9627\n", + "19 Test accuracy: 0.963\n" + ] + } + ], "source": [ "extra_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)\n", "\n", @@ -881,7 +1038,29 @@ "cell_type": "code", "execution_count": 36, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "['hidden1/kernel:0',\n", + " 'hidden1/bias:0',\n", + " 'batch_normalization/beta:0',\n", + " 'batch_normalization/gamma:0',\n", + " 'hidden2/kernel:0',\n", + " 'hidden2/bias:0',\n", + " 'batch_normalization_1/beta:0',\n", + " 'batch_normalization_1/gamma:0',\n", + " 'outputs/kernel:0',\n", + " 'outputs/bias:0',\n", + " 'batch_normalization_2/beta:0',\n", + " 'batch_normalization_2/gamma:0']" + ] + }, + "execution_count": 36, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "[v.name for v in tf.trainable_variables()]" ] @@ -890,7 +1069,35 @@ "cell_type": "code", "execution_count": 37, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "['hidden1/kernel:0',\n", + " 'hidden1/bias:0',\n", + " 'batch_normalization/beta:0',\n", + " 'batch_normalization/gamma:0',\n", + " 'batch_normalization/moving_mean:0',\n", + " 'batch_normalization/moving_variance:0',\n", + " 'hidden2/kernel:0',\n", + " 'hidden2/bias:0',\n", + " 'batch_normalization_1/beta:0',\n", + " 'batch_normalization_1/gamma:0',\n", + " 'batch_normalization_1/moving_mean:0',\n", + " 'batch_normalization_1/moving_variance:0',\n", + " 'outputs/kernel:0',\n", + " 'outputs/bias:0',\n", + " 'batch_normalization_2/beta:0',\n", + " 'batch_normalization_2/gamma:0',\n", + " 'batch_normalization_2/moving_mean:0',\n", + " 'batch_normalization_2/moving_variance:0']" + ] + }, + "execution_count": 37, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "[v.name for v in tf.global_variables()]" ] @@ -1026,7 +1233,34 @@ "cell_type": "code", "execution_count": 44, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0 Test accuracy: 0.3139\n", + "1 Test accuracy: 0.8001\n", + "2 Test accuracy: 0.8806\n", + "3 Test accuracy: 0.9037\n", + "4 Test accuracy: 0.9124\n", + "5 Test accuracy: 0.9197\n", + "6 Test accuracy: 0.9243\n", + "7 Test accuracy: 0.9299\n", + "8 Test accuracy: 0.9331\n", + "9 Test accuracy: 0.9387\n", + "10 Test accuracy: 0.9431\n", + "11 Test accuracy: 0.9445\n", + "12 Test accuracy: 0.9455\n", + "13 Test accuracy: 0.9485\n", + "14 Test accuracy: 0.9524\n", + "15 Test accuracy: 0.9511\n", + "16 Test accuracy: 0.9562\n", + "17 Test accuracy: 0.9583\n", + "18 Test accuracy: 0.9559\n", + "19 Test accuracy: 0.9605\n" + ] + } + ], "source": [ "with tf.Session() as sess:\n", " init.run()\n", @@ -1095,7 +1329,324 @@ "cell_type": "code", "execution_count": 47, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "X\n", + "y\n", + "hidden1/kernel/Initializer/random_uniform/shape\n", + "hidden1/kernel/Initializer/random_uniform/min\n", + "hidden1/kernel/Initializer/random_uniform/max\n", + "hidden1/kernel/Initializer/random_uniform/RandomUniform\n", + "hidden1/kernel/Initializer/random_uniform/sub\n", + "hidden1/kernel/Initializer/random_uniform/mul\n", + "hidden1/kernel/Initializer/random_uniform\n", + "hidden1/kernel\n", + "hidden1/kernel/Assign\n", + "hidden1/kernel/read\n", + "hidden1/bias/Initializer/Const\n", + "hidden1/bias\n", + "hidden1/bias/Assign\n", + "hidden1/bias/read\n", + "dnn/hidden1/MatMul\n", + "dnn/hidden1/BiasAdd\n", + "dnn/hidden1/Relu\n", + "hidden2/kernel/Initializer/random_uniform/shape\n", + "hidden2/kernel/Initializer/random_uniform/min\n", + "hidden2/kernel/Initializer/random_uniform/max\n", + "hidden2/kernel/Initializer/random_uniform/RandomUniform\n", + "hidden2/kernel/Initializer/random_uniform/sub\n", + "hidden2/kernel/Initializer/random_uniform/mul\n", + "hidden2/kernel/Initializer/random_uniform\n", + "hidden2/kernel\n", + "hidden2/kernel/Assign\n", + "hidden2/kernel/read\n", + "hidden2/bias/Initializer/Const\n", + "hidden2/bias\n", + "hidden2/bias/Assign\n", + "hidden2/bias/read\n", + "dnn/hidden2/MatMul\n", + "dnn/hidden2/BiasAdd\n", + "dnn/hidden2/Relu\n", + "hidden3/kernel/Initializer/random_uniform/shape\n", + "hidden3/kernel/Initializer/random_uniform/min\n", + "hidden3/kernel/Initializer/random_uniform/max\n", + "hidden3/kernel/Initializer/random_uniform/RandomUniform\n", + "hidden3/kernel/Initializer/random_uniform/sub\n", + "hidden3/kernel/Initializer/random_uniform/mul\n", + "hidden3/kernel/Initializer/random_uniform\n", + "hidden3/kernel\n", + "hidden3/kernel/Assign\n", + "hidden3/kernel/read\n", + "hidden3/bias/Initializer/Const\n", + "hidden3/bias\n", + "hidden3/bias/Assign\n", + "hidden3/bias/read\n", + "dnn/hidden3/MatMul\n", + "dnn/hidden3/BiasAdd\n", + "dnn/hidden3/Relu\n", + "hidden4/kernel/Initializer/random_uniform/shape\n", + "hidden4/kernel/Initializer/random_uniform/min\n", + "hidden4/kernel/Initializer/random_uniform/max\n", + "hidden4/kernel/Initializer/random_uniform/RandomUniform\n", + "hidden4/kernel/Initializer/random_uniform/sub\n", + "hidden4/kernel/Initializer/random_uniform/mul\n", + "hidden4/kernel/Initializer/random_uniform\n", + "hidden4/kernel\n", + "hidden4/kernel/Assign\n", + "hidden4/kernel/read\n", + "hidden4/bias/Initializer/Const\n", + "hidden4/bias\n", + "hidden4/bias/Assign\n", + "hidden4/bias/read\n", + "dnn/hidden4/MatMul\n", + "dnn/hidden4/BiasAdd\n", + "dnn/hidden4/Relu\n", + "hidden5/kernel/Initializer/random_uniform/shape\n", + "hidden5/kernel/Initializer/random_uniform/min\n", + "hidden5/kernel/Initializer/random_uniform/max\n", + "hidden5/kernel/Initializer/random_uniform/RandomUniform\n", + "hidden5/kernel/Initializer/random_uniform/sub\n", + "hidden5/kernel/Initializer/random_uniform/mul\n", + "hidden5/kernel/Initializer/random_uniform\n", + "hidden5/kernel\n", + "hidden5/kernel/Assign\n", + "hidden5/kernel/read\n", + "hidden5/bias/Initializer/Const\n", + "hidden5/bias\n", + "hidden5/bias/Assign\n", + "hidden5/bias/read\n", + "dnn/hidden5/MatMul\n", + "dnn/hidden5/BiasAdd\n", + "dnn/hidden5/Relu\n", + "outputs/kernel/Initializer/random_uniform/shape\n", + "outputs/kernel/Initializer/random_uniform/min\n", + "outputs/kernel/Initializer/random_uniform/max\n", + "outputs/kernel/Initializer/random_uniform/RandomUniform\n", + "outputs/kernel/Initializer/random_uniform/sub\n", + "outputs/kernel/Initializer/random_uniform/mul\n", + "outputs/kernel/Initializer/random_uniform\n", + "outputs/kernel\n", + "outputs/kernel/Assign\n", + "outputs/kernel/read\n", + "outputs/bias/Initializer/Const\n", + "outputs/bias\n", + "outputs/bias/Assign\n", + "outputs/bias/read\n", + "dnn/outputs/MatMul\n", + "dnn/outputs/BiasAdd\n", + "loss/SparseSoftmaxCrossEntropyWithLogits/Shape\n", + "loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits\n", + "loss/Const\n", + "loss/loss\n", + "gradients/Shape\n", + "gradients/Const\n", + "gradients/Fill\n", + "gradients/loss/loss_grad/Reshape/shape\n", + "gradients/loss/loss_grad/Reshape\n", + "gradients/loss/loss_grad/Shape\n", + "gradients/loss/loss_grad/Tile\n", + "gradients/loss/loss_grad/Shape_1\n", + "gradients/loss/loss_grad/Shape_2\n", + "gradients/loss/loss_grad/Const\n", + "gradients/loss/loss_grad/Prod\n", + "gradients/loss/loss_grad/Const_1\n", + "gradients/loss/loss_grad/Prod_1\n", + "gradients/loss/loss_grad/Maximum/y\n", + "gradients/loss/loss_grad/Maximum\n", + "gradients/loss/loss_grad/floordiv\n", + "gradients/loss/loss_grad/Cast\n", + "gradients/loss/loss_grad/truediv\n", + "gradients/zeros_like\n", + "gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/PreventGradient\n", + "gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims/dim\n", + "gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims\n", + "gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul\n", + "gradients/dnn/outputs/BiasAdd_grad/BiasAddGrad\n", + "gradients/dnn/outputs/BiasAdd_grad/tuple/group_deps\n", + "gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency\n", + "gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency_1\n", + "gradients/dnn/outputs/MatMul_grad/MatMul\n", + "gradients/dnn/outputs/MatMul_grad/MatMul_1\n", + "gradients/dnn/outputs/MatMul_grad/tuple/group_deps\n", + "gradients/dnn/outputs/MatMul_grad/tuple/control_dependency\n", + "gradients/dnn/outputs/MatMul_grad/tuple/control_dependency_1\n", + "gradients/dnn/hidden5/Relu_grad/ReluGrad\n", + "gradients/dnn/hidden5/BiasAdd_grad/BiasAddGrad\n", + "gradients/dnn/hidden5/BiasAdd_grad/tuple/group_deps\n", + "gradients/dnn/hidden5/BiasAdd_grad/tuple/control_dependency\n", + "gradients/dnn/hidden5/BiasAdd_grad/tuple/control_dependency_1\n", + "gradients/dnn/hidden5/MatMul_grad/MatMul\n", + "gradients/dnn/hidden5/MatMul_grad/MatMul_1\n", + "gradients/dnn/hidden5/MatMul_grad/tuple/group_deps\n", + "gradients/dnn/hidden5/MatMul_grad/tuple/control_dependency\n", + "gradients/dnn/hidden5/MatMul_grad/tuple/control_dependency_1\n", + "gradients/dnn/hidden4/Relu_grad/ReluGrad\n", + "gradients/dnn/hidden4/BiasAdd_grad/BiasAddGrad\n", + "gradients/dnn/hidden4/BiasAdd_grad/tuple/group_deps\n", + "gradients/dnn/hidden4/BiasAdd_grad/tuple/control_dependency\n", + "gradients/dnn/hidden4/BiasAdd_grad/tuple/control_dependency_1\n", + "gradients/dnn/hidden4/MatMul_grad/MatMul\n", + "gradients/dnn/hidden4/MatMul_grad/MatMul_1\n", + "gradients/dnn/hidden4/MatMul_grad/tuple/group_deps\n", + "gradients/dnn/hidden4/MatMul_grad/tuple/control_dependency\n", + "gradients/dnn/hidden4/MatMul_grad/tuple/control_dependency_1\n", + "gradients/dnn/hidden3/Relu_grad/ReluGrad\n", + "gradients/dnn/hidden3/BiasAdd_grad/BiasAddGrad\n", + "gradients/dnn/hidden3/BiasAdd_grad/tuple/group_deps\n", + "gradients/dnn/hidden3/BiasAdd_grad/tuple/control_dependency\n", + "gradients/dnn/hidden3/BiasAdd_grad/tuple/control_dependency_1\n", + "gradients/dnn/hidden3/MatMul_grad/MatMul\n", + "gradients/dnn/hidden3/MatMul_grad/MatMul_1\n", + "gradients/dnn/hidden3/MatMul_grad/tuple/group_deps\n", + "gradients/dnn/hidden3/MatMul_grad/tuple/control_dependency\n", + "gradients/dnn/hidden3/MatMul_grad/tuple/control_dependency_1\n", + "gradients/dnn/hidden2/Relu_grad/ReluGrad\n", + "gradients/dnn/hidden2/BiasAdd_grad/BiasAddGrad\n", + "gradients/dnn/hidden2/BiasAdd_grad/tuple/group_deps\n", + "gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency\n", + "gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency_1\n", + "gradients/dnn/hidden2/MatMul_grad/MatMul\n", + "gradients/dnn/hidden2/MatMul_grad/MatMul_1\n", + "gradients/dnn/hidden2/MatMul_grad/tuple/group_deps\n", + "gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency\n", + "gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency_1\n", + "gradients/dnn/hidden1/Relu_grad/ReluGrad\n", + "gradients/dnn/hidden1/BiasAdd_grad/BiasAddGrad\n", + "gradients/dnn/hidden1/BiasAdd_grad/tuple/group_deps\n", + "gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency\n", + "gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency_1\n", + "gradients/dnn/hidden1/MatMul_grad/MatMul\n", + "gradients/dnn/hidden1/MatMul_grad/MatMul_1\n", + "gradients/dnn/hidden1/MatMul_grad/tuple/group_deps\n", + "gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency\n", + "gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency_1\n", + "clip_by_value/Minimum/y\n", + "clip_by_value/Minimum\n", + "clip_by_value/y\n", + "clip_by_value\n", + "clip_by_value_1/Minimum/y\n", + "clip_by_value_1/Minimum\n", + "clip_by_value_1/y\n", + "clip_by_value_1\n", + "clip_by_value_2/Minimum/y\n", + "clip_by_value_2/Minimum\n", + "clip_by_value_2/y\n", + "clip_by_value_2\n", + "clip_by_value_3/Minimum/y\n", + "clip_by_value_3/Minimum\n", + "clip_by_value_3/y\n", + "clip_by_value_3\n", + "clip_by_value_4/Minimum/y\n", + "clip_by_value_4/Minimum\n", + "clip_by_value_4/y\n", + "clip_by_value_4\n", + "clip_by_value_5/Minimum/y\n", + "clip_by_value_5/Minimum\n", + "clip_by_value_5/y\n", + "clip_by_value_5\n", + "clip_by_value_6/Minimum/y\n", + "clip_by_value_6/Minimum\n", + "clip_by_value_6/y\n", + "clip_by_value_6\n", + "clip_by_value_7/Minimum/y\n", + "clip_by_value_7/Minimum\n", + "clip_by_value_7/y\n", + "clip_by_value_7\n", + "clip_by_value_8/Minimum/y\n", + "clip_by_value_8/Minimum\n", + "clip_by_value_8/y\n", + "clip_by_value_8\n", + "clip_by_value_9/Minimum/y\n", + "clip_by_value_9/Minimum\n", + "clip_by_value_9/y\n", + "clip_by_value_9\n", + "clip_by_value_10/Minimum/y\n", + "clip_by_value_10/Minimum\n", + "clip_by_value_10/y\n", + "clip_by_value_10\n", + "clip_by_value_11/Minimum/y\n", + "clip_by_value_11/Minimum\n", + "clip_by_value_11/y\n", + "clip_by_value_11\n", + "GradientDescent/learning_rate\n", + "GradientDescent/update_hidden1/kernel/ApplyGradientDescent\n", + "GradientDescent/update_hidden1/bias/ApplyGradientDescent\n", + "GradientDescent/update_hidden2/kernel/ApplyGradientDescent\n", + "GradientDescent/update_hidden2/bias/ApplyGradientDescent\n", + "GradientDescent/update_hidden3/kernel/ApplyGradientDescent\n", + "GradientDescent/update_hidden3/bias/ApplyGradientDescent\n", + "GradientDescent/update_hidden4/kernel/ApplyGradientDescent\n", + "GradientDescent/update_hidden4/bias/ApplyGradientDescent\n", + "GradientDescent/update_hidden5/kernel/ApplyGradientDescent\n", + "GradientDescent/update_hidden5/bias/ApplyGradientDescent\n", + "GradientDescent/update_outputs/kernel/ApplyGradientDescent\n", + "GradientDescent/update_outputs/bias/ApplyGradientDescent\n", + "GradientDescent\n", + "eval/InTopK\n", + "eval/Cast\n", + "eval/Const\n", + "eval/accuracy\n", + "init\n", + "save/Const\n", + "save/SaveV2/tensor_names\n", + "save/SaveV2/shape_and_slices\n", + "save/SaveV2\n", + "save/control_dependency\n", + "save/RestoreV2/tensor_names\n", + "save/RestoreV2/shape_and_slices\n", + "save/RestoreV2\n", + "save/Assign\n", + "save/RestoreV2_1/tensor_names\n", + "save/RestoreV2_1/shape_and_slices\n", + "save/RestoreV2_1\n", + "save/Assign_1\n", + "save/RestoreV2_2/tensor_names\n", + "save/RestoreV2_2/shape_and_slices\n", + "save/RestoreV2_2\n", + "save/Assign_2\n", + "save/RestoreV2_3/tensor_names\n", + "save/RestoreV2_3/shape_and_slices\n", + "save/RestoreV2_3\n", + "save/Assign_3\n", + "save/RestoreV2_4/tensor_names\n", + "save/RestoreV2_4/shape_and_slices\n", + "save/RestoreV2_4\n", + "save/Assign_4\n", + "save/RestoreV2_5/tensor_names\n", + "save/RestoreV2_5/shape_and_slices\n", + "save/RestoreV2_5\n", + "save/Assign_5\n", + "save/RestoreV2_6/tensor_names\n", + "save/RestoreV2_6/shape_and_slices\n", + "save/RestoreV2_6\n", + "save/Assign_6\n", + "save/RestoreV2_7/tensor_names\n", + "save/RestoreV2_7/shape_and_slices\n", + "save/RestoreV2_7\n", + "save/Assign_7\n", + "save/RestoreV2_8/tensor_names\n", + "save/RestoreV2_8/shape_and_slices\n", + "save/RestoreV2_8\n", + "save/Assign_8\n", + "save/RestoreV2_9/tensor_names\n", + "save/RestoreV2_9/shape_and_slices\n", + "save/RestoreV2_9\n", + "save/Assign_9\n", + "save/RestoreV2_10/tensor_names\n", + "save/RestoreV2_10/shape_and_slices\n", + "save/RestoreV2_10\n", + "save/Assign_10\n", + "save/RestoreV2_11/tensor_names\n", + "save/RestoreV2_11/shape_and_slices\n", + "save/RestoreV2_11\n", + "save/Assign_11\n", + "save/restore_all\n" + ] + } + ], "source": [ "for op in tf.get_default_graph().get_operations():\n", " print(op.name)" @@ -1160,7 +1711,32 @@ "metadata": { "scrolled": true }, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "show_graph(tf.get_default_graph())" ] @@ -1236,7 +1812,15 @@ "cell_type": "code", "execution_count": 53, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Restoring parameters from ./my_model_final.ckpt\n" + ] + } + ], "source": [ "with tf.Session() as sess:\n", " saver.restore(sess, \"./my_model_final.ckpt\")\n", @@ -1254,7 +1838,35 @@ "cell_type": "code", "execution_count": 54, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Restoring parameters from ./my_model_final.ckpt\n", + "0 Test accuracy: 0.9609\n", + "1 Test accuracy: 0.9608\n", + "2 Test accuracy: 0.9617\n", + "3 Test accuracy: 0.9613\n", + "4 Test accuracy: 0.9639\n", + "5 Test accuracy: 0.9649\n", + "6 Test accuracy: 0.9663\n", + "7 Test accuracy: 0.9627\n", + "8 Test accuracy: 0.9665\n", + "9 Test accuracy: 0.9669\n", + "10 Test accuracy: 0.9662\n", + "11 Test accuracy: 0.9674\n", + "12 Test accuracy: 0.9678\n", + "13 Test accuracy: 0.9679\n", + "14 Test accuracy: 0.9688\n", + "15 Test accuracy: 0.9684\n", + "16 Test accuracy: 0.9687\n", + "17 Test accuracy: 0.9702\n", + "18 Test accuracy: 0.9673\n", + "19 Test accuracy: 0.9687\n" + ] + } + ], "source": [ "with tf.Session() as sess:\n", " saver.restore(sess, \"./my_model_final.ckpt\")\n", @@ -1337,7 +1949,35 @@ "cell_type": "code", "execution_count": 56, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Restoring parameters from ./my_model_final.ckpt\n", + "0 Test accuracy: 0.9611\n", + "1 Test accuracy: 0.9619\n", + "2 Test accuracy: 0.9622\n", + "3 Test accuracy: 0.9619\n", + "4 Test accuracy: 0.9644\n", + "5 Test accuracy: 0.9633\n", + "6 Test accuracy: 0.9647\n", + "7 Test accuracy: 0.9648\n", + "8 Test accuracy: 0.9671\n", + "9 Test accuracy: 0.9677\n", + "10 Test accuracy: 0.9676\n", + "11 Test accuracy: 0.9679\n", + "12 Test accuracy: 0.9687\n", + "13 Test accuracy: 0.9688\n", + "14 Test accuracy: 0.9683\n", + "15 Test accuracy: 0.9693\n", + "16 Test accuracy: 0.9677\n", + "17 Test accuracy: 0.9697\n", + "18 Test accuracy: 0.9692\n", + "19 Test accuracy: 0.9707\n" + ] + } + ], "source": [ "with tf.Session() as sess:\n", " saver.restore(sess, \"./my_model_final.ckpt\")\n", @@ -1410,7 +2050,35 @@ "cell_type": "code", "execution_count": 58, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Restoring parameters from ./my_model_final.ckpt\n", + "0 Test accuracy: 0.9142\n", + "1 Test accuracy: 0.9346\n", + "2 Test accuracy: 0.9437\n", + "3 Test accuracy: 0.9486\n", + "4 Test accuracy: 0.9517\n", + "5 Test accuracy: 0.9544\n", + "6 Test accuracy: 0.9544\n", + "7 Test accuracy: 0.9562\n", + "8 Test accuracy: 0.9588\n", + "9 Test accuracy: 0.9619\n", + "10 Test accuracy: 0.9617\n", + "11 Test accuracy: 0.9617\n", + "12 Test accuracy: 0.9624\n", + "13 Test accuracy: 0.9644\n", + "14 Test accuracy: 0.9622\n", + "15 Test accuracy: 0.964\n", + "16 Test accuracy: 0.9666\n", + "17 Test accuracy: 0.9668\n", + "18 Test accuracy: 0.9673\n", + "19 Test accuracy: 0.9687\n" + ] + } + ], "source": [ "with tf.Session() as sess:\n", " init.run()\n", @@ -1485,7 +2153,35 @@ "cell_type": "code", "execution_count": 60, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Restoring parameters from ./my_model_final.ckpt\n", + "0 Test accuracy: 0.9022\n", + "1 Test accuracy: 0.9302\n", + "2 Test accuracy: 0.9393\n", + "3 Test accuracy: 0.9429\n", + "4 Test accuracy: 0.9484\n", + "5 Test accuracy: 0.9511\n", + "6 Test accuracy: 0.9517\n", + "7 Test accuracy: 0.9539\n", + "8 Test accuracy: 0.9545\n", + "9 Test accuracy: 0.9572\n", + "10 Test accuracy: 0.9599\n", + "11 Test accuracy: 0.9602\n", + "12 Test accuracy: 0.9606\n", + "13 Test accuracy: 0.9619\n", + "14 Test accuracy: 0.9619\n", + "15 Test accuracy: 0.9636\n", + "16 Test accuracy: 0.9633\n", + "17 Test accuracy: 0.9643\n", + "18 Test accuracy: 0.9651\n", + "19 Test accuracy: 0.9657\n" + ] + } + ], "source": [ "reuse_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,\n", " scope=\"hidden[123]\") # regular expression\n", @@ -1542,7 +2238,15 @@ "cell_type": "code", "execution_count": 62, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[ 61. 83. 105.]]\n" + ] + } + ], "source": [ "original_w = [[1., 2., 3.], [4., 5., 6.]] # Load the weights from the other framework\n", "original_b = [7., 8., 9.] # Load the biases from the other framework\n", @@ -1584,7 +2288,15 @@ "cell_type": "code", "execution_count": 63, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[ 61. 83. 105.]]\n" + ] + } + ], "source": [ "reset_graph()\n", "\n", @@ -1630,7 +2342,19 @@ "cell_type": "code", "execution_count": 64, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "[,\n", + " ]" + ] + }, + "execution_count": 64, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=\"hidden1\")" ] @@ -1646,7 +2370,18 @@ "cell_type": "code", "execution_count": 65, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 65, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "tf.get_default_graph().get_tensor_by_name(\"hidden1/kernel:0\")" ] @@ -1655,7 +2390,18 @@ "cell_type": "code", "execution_count": 66, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 66, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "tf.get_default_graph().get_tensor_by_name(\"hidden1/bias:0\")" ] @@ -1734,7 +2480,35 @@ "cell_type": "code", "execution_count": 70, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Restoring parameters from ./my_model_final.ckpt\n", + "0 Test accuracy: 0.8987\n", + "1 Test accuracy: 0.9311\n", + "2 Test accuracy: 0.9375\n", + "3 Test accuracy: 0.9414\n", + "4 Test accuracy: 0.9437\n", + "5 Test accuracy: 0.9479\n", + "6 Test accuracy: 0.9495\n", + "7 Test accuracy: 0.9521\n", + "8 Test accuracy: 0.9517\n", + "9 Test accuracy: 0.9525\n", + "10 Test accuracy: 0.9535\n", + "11 Test accuracy: 0.9538\n", + "12 Test accuracy: 0.9534\n", + "13 Test accuracy: 0.9546\n", + "14 Test accuracy: 0.9538\n", + "15 Test accuracy: 0.9553\n", + "16 Test accuracy: 0.9552\n", + "17 Test accuracy: 0.9549\n", + "18 Test accuracy: 0.9553\n", + "19 Test accuracy: 0.9557\n" + ] + } + ], "source": [ "reuse_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,\n", " scope=\"hidden[123]\") # regular expression\n", @@ -1833,7 +2607,35 @@ "cell_type": "code", "execution_count": 74, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Restoring parameters from ./my_model_final.ckpt\n", + "0 Test accuracy: 0.9031\n", + "1 Test accuracy: 0.932\n", + "2 Test accuracy: 0.94\n", + "3 Test accuracy: 0.9435\n", + "4 Test accuracy: 0.9473\n", + "5 Test accuracy: 0.9492\n", + "6 Test accuracy: 0.9498\n", + "7 Test accuracy: 0.9493\n", + "8 Test accuracy: 0.9515\n", + "9 Test accuracy: 0.9519\n", + "10 Test accuracy: 0.9529\n", + "11 Test accuracy: 0.9536\n", + "12 Test accuracy: 0.9529\n", + "13 Test accuracy: 0.9532\n", + "14 Test accuracy: 0.9522\n", + "15 Test accuracy: 0.9534\n", + "16 Test accuracy: 0.953\n", + "17 Test accuracy: 0.955\n", + "18 Test accuracy: 0.955\n", + "19 Test accuracy: 0.9552\n" + ] + } + ], "source": [ "reuse_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,\n", " scope=\"hidden[123]\") # regular expression\n", @@ -1931,7 +2733,35 @@ "cell_type": "code", "execution_count": 77, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Restoring parameters from ./my_model_final.ckpt\n", + "0 Test accuracy: 0.9033\n", + "1 Test accuracy: 0.9322\n", + "2 Test accuracy: 0.9423\n", + "3 Test accuracy: 0.9449\n", + "4 Test accuracy: 0.9471\n", + "5 Test accuracy: 0.9477\n", + "6 Test accuracy: 0.951\n", + "7 Test accuracy: 0.9507\n", + "8 Test accuracy: 0.9514\n", + "9 Test accuracy: 0.9522\n", + "10 Test accuracy: 0.9512\n", + "11 Test accuracy: 0.9521\n", + "12 Test accuracy: 0.9522\n", + "13 Test accuracy: 0.9539\n", + "14 Test accuracy: 0.9536\n", + "15 Test accuracy: 0.9534\n", + "16 Test accuracy: 0.9547\n", + "17 Test accuracy: 0.9537\n", + "18 Test accuracy: 0.9542\n", + "19 Test accuracy: 0.9547\n" + ] + } + ], "source": [ "import numpy as np\n", "\n", @@ -2132,7 +2962,19 @@ "cell_type": "code", "execution_count": 86, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0 Test accuracy: 0.9579\n", + "1 Test accuracy: 0.9691\n", + "2 Test accuracy: 0.976\n", + "3 Test accuracy: 0.9793\n", + "4 Test accuracy: 0.9811\n" + ] + } + ], "source": [ "n_epochs = 5\n", "batch_size = 50\n", @@ -2256,7 +3098,34 @@ "metadata": { "scrolled": true }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0 Test accuracy: 0.8343\n", + "1 Test accuracy: 0.8726\n", + "2 Test accuracy: 0.8832\n", + "3 Test accuracy: 0.8899\n", + "4 Test accuracy: 0.8958\n", + "5 Test accuracy: 0.8986\n", + "6 Test accuracy: 0.9011\n", + "7 Test accuracy: 0.9032\n", + "8 Test accuracy: 0.9046\n", + "9 Test accuracy: 0.9047\n", + "10 Test accuracy: 0.9065\n", + "11 Test accuracy: 0.9059\n", + "12 Test accuracy: 0.9072\n", + "13 Test accuracy: 0.9072\n", + "14 Test accuracy: 0.9069\n", + "15 Test accuracy: 0.9071\n", + "16 Test accuracy: 0.9064\n", + "17 Test accuracy: 0.9071\n", + "18 Test accuracy: 0.9068\n", + "19 Test accuracy: 0.9063\n" + ] + } + ], "source": [ "n_epochs = 20\n", "batch_size = 200\n", @@ -2395,7 +3264,34 @@ "metadata": { "scrolled": true }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0 Test accuracy: 0.8298\n", + "1 Test accuracy: 0.8778\n", + "2 Test accuracy: 0.8917\n", + "3 Test accuracy: 0.9017\n", + "4 Test accuracy: 0.9068\n", + "5 Test accuracy: 0.9103\n", + "6 Test accuracy: 0.9125\n", + "7 Test accuracy: 0.9137\n", + "8 Test accuracy: 0.9149\n", + "9 Test accuracy: 0.9174\n", + "10 Test accuracy: 0.9176\n", + "11 Test accuracy: 0.9184\n", + "12 Test accuracy: 0.9191\n", + "13 Test accuracy: 0.9183\n", + "14 Test accuracy: 0.9195\n", + "15 Test accuracy: 0.9201\n", + "16 Test accuracy: 0.9181\n", + "17 Test accuracy: 0.9184\n", + "18 Test accuracy: 0.9181\n", + "19 Test accuracy: 0.9174\n" + ] + } + ], "source": [ "n_epochs = 20\n", "batch_size = 200\n", @@ -2496,7 +3392,34 @@ "metadata": { "scrolled": true }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0 Test accuracy: 0.9205\n", + "1 Test accuracy: 0.9418\n", + "2 Test accuracy: 0.9486\n", + "3 Test accuracy: 0.9508\n", + "4 Test accuracy: 0.954\n", + "5 Test accuracy: 0.957\n", + "6 Test accuracy: 0.9604\n", + "7 Test accuracy: 0.9585\n", + "8 Test accuracy: 0.9598\n", + "9 Test accuracy: 0.9663\n", + "10 Test accuracy: 0.9644\n", + "11 Test accuracy: 0.9646\n", + "12 Test accuracy: 0.9675\n", + "13 Test accuracy: 0.9657\n", + "14 Test accuracy: 0.9645\n", + "15 Test accuracy: 0.9668\n", + "16 Test accuracy: 0.969\n", + "17 Test accuracy: 0.9682\n", + "18 Test accuracy: 0.9698\n", + "19 Test accuracy: 0.9682\n" + ] + } + ], "source": [ "n_epochs = 20\n", "batch_size = 50\n", @@ -2649,7 +3572,34 @@ "cell_type": "code", "execution_count": 106, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0 Test accuracy: 0.9517\n", + "1 Test accuracy: 0.9674\n", + "2 Test accuracy: 0.9712\n", + "3 Test accuracy: 0.9759\n", + "4 Test accuracy: 0.975\n", + "5 Test accuracy: 0.9761\n", + "6 Test accuracy: 0.9765\n", + "7 Test accuracy: 0.9796\n", + "8 Test accuracy: 0.9791\n", + "9 Test accuracy: 0.9794\n", + "10 Test accuracy: 0.9805\n", + "11 Test accuracy: 0.9809\n", + "12 Test accuracy: 0.9807\n", + "13 Test accuracy: 0.9799\n", + "14 Test accuracy: 0.982\n", + "15 Test accuracy: 0.9816\n", + "16 Test accuracy: 0.9825\n", + "17 Test accuracy: 0.9825\n", + "18 Test accuracy: 0.9816\n", + "19 Test accuracy: 0.9822\n" + ] + } + ], "source": [ "with tf.Session() as sess: # not shown in the book\n", " init.run() # not shown\n", @@ -2787,7 +3737,34 @@ "metadata": { "scrolled": false }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0 Test accuracy: 0.9527\n", + "1 Test accuracy: 0.9653\n", + "2 Test accuracy: 0.97\n", + "3 Test accuracy: 0.9751\n", + "4 Test accuracy: 0.9752\n", + "5 Test accuracy: 0.9742\n", + "6 Test accuracy: 0.9754\n", + "7 Test accuracy: 0.9784\n", + "8 Test accuracy: 0.9775\n", + "9 Test accuracy: 0.9789\n", + "10 Test accuracy: 0.9808\n", + "11 Test accuracy: 0.9797\n", + "12 Test accuracy: 0.9802\n", + "13 Test accuracy: 0.9799\n", + "14 Test accuracy: 0.9808\n", + "15 Test accuracy: 0.9809\n", + "16 Test accuracy: 0.9807\n", + "17 Test accuracy: 0.9803\n", + "18 Test accuracy: 0.9816\n", + "19 Test accuracy: 0.9812\n" + ] + } + ], "source": [ "clip_all_weights = tf.get_collection(\"max_norm\")\n", "\n", @@ -2953,7 +3930,18 @@ "cell_type": "code", "execution_count": 116, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Extracting /tmp/data/train-images-idx3-ubyte.gz\n", + "Extracting /tmp/data/train-labels-idx1-ubyte.gz\n", + "Extracting /tmp/data/t10k-images-idx3-ubyte.gz\n", + "Extracting /tmp/data/t10k-labels-idx1-ubyte.gz\n" + ] + } + ], "source": [ "from tensorflow.examples.tutorials.mnist import input_data\n", "mnist = input_data.read_data_sets(\"/tmp/data/\")" @@ -2986,7 +3974,42 @@ "cell_type": "code", "execution_count": 118, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0\tValidation loss: 0.128663\tBest loss: 0.128663\tAccuracy: 96.64%\n", + "1\tValidation loss: 0.448317\tBest loss: 0.128663\tAccuracy: 78.19%\n", + "2\tValidation loss: 0.190859\tBest loss: 0.128663\tAccuracy: 95.54%\n", + "3\tValidation loss: 0.146951\tBest loss: 0.128663\tAccuracy: 96.79%\n", + "4\tValidation loss: 0.086076\tBest loss: 0.086076\tAccuracy: 97.69%\n", + "5\tValidation loss: 0.115353\tBest loss: 0.086076\tAccuracy: 97.77%\n", + "6\tValidation loss: 0.239142\tBest loss: 0.086076\tAccuracy: 95.15%\n", + "7\tValidation loss: 0.088810\tBest loss: 0.086076\tAccuracy: 98.12%\n", + "8\tValidation loss: 0.108763\tBest loss: 0.086076\tAccuracy: 97.81%\n", + "9\tValidation loss: 0.300808\tBest loss: 0.086076\tAccuracy: 96.17%\n", + "10\tValidation loss: 0.179260\tBest loss: 0.086076\tAccuracy: 97.46%\n", + "11\tValidation loss: 0.125690\tBest loss: 0.086076\tAccuracy: 98.48%\n", + "12\tValidation loss: 0.738371\tBest loss: 0.086076\tAccuracy: 77.72%\n", + "13\tValidation loss: 1.894743\tBest loss: 0.086076\tAccuracy: 78.54%\n", + "14\tValidation loss: 0.415678\tBest loss: 0.086076\tAccuracy: 78.50%\n", + "15\tValidation loss: 0.537646\tBest loss: 0.086076\tAccuracy: 75.45%\n", + "16\tValidation loss: 1.009708\tBest loss: 0.086076\tAccuracy: 53.99%\n", + "17\tValidation loss: 1.228350\tBest loss: 0.086076\tAccuracy: 38.15%\n", + "18\tValidation loss: 1.510606\tBest loss: 0.086076\tAccuracy: 29.44%\n", + "19\tValidation loss: 1.632344\tBest loss: 0.086076\tAccuracy: 22.01%\n", + "20\tValidation loss: 1.628246\tBest loss: 0.086076\tAccuracy: 22.01%\n", + "21\tValidation loss: 1.626765\tBest loss: 0.086076\tAccuracy: 22.01%\n", + "22\tValidation loss: 1.651615\tBest loss: 0.086076\tAccuracy: 18.73%\n", + "23\tValidation loss: 1.663751\tBest loss: 0.086076\tAccuracy: 19.27%\n", + "24\tValidation loss: 1.675138\tBest loss: 0.086076\tAccuracy: 22.01%\n", + "Early stopping!\n", + "INFO:tensorflow:Restoring parameters from ./my_mnist_model_0_to_4.ckpt\n", + "Final test accuracy: 98.05%\n" + ] + } + ], "source": [ "n_epochs = 1000\n", "batch_size = 20\n", @@ -3254,7 +4277,56 @@ "cell_type": "code", "execution_count": 120, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0\tValidation loss: 0.128663\tBest loss: 0.128663\tAccuracy: 96.64%\n", + "1\tValidation loss: 0.448317\tBest loss: 0.128663\tAccuracy: 78.19%\n", + "2\tValidation loss: 0.190859\tBest loss: 0.128663\tAccuracy: 95.54%\n", + "3\tValidation loss: 0.146951\tBest loss: 0.128663\tAccuracy: 96.79%\n", + "4\tValidation loss: 0.086076\tBest loss: 0.086076\tAccuracy: 97.69%\n", + "5\tValidation loss: 0.115353\tBest loss: 0.086076\tAccuracy: 97.77%\n", + "6\tValidation loss: 0.239142\tBest loss: 0.086076\tAccuracy: 95.15%\n", + "7\tValidation loss: 0.088810\tBest loss: 0.086076\tAccuracy: 98.12%\n", + "8\tValidation loss: 0.108763\tBest loss: 0.086076\tAccuracy: 97.81%\n", + "9\tValidation loss: 0.300808\tBest loss: 0.086076\tAccuracy: 96.17%\n", + "10\tValidation loss: 0.179260\tBest loss: 0.086076\tAccuracy: 97.46%\n", + "11\tValidation loss: 0.125690\tBest loss: 0.086076\tAccuracy: 98.48%\n", + "12\tValidation loss: 0.738371\tBest loss: 0.086076\tAccuracy: 77.72%\n", + "13\tValidation loss: 1.894743\tBest loss: 0.086076\tAccuracy: 78.54%\n", + "14\tValidation loss: 0.415678\tBest loss: 0.086076\tAccuracy: 78.50%\n", + "15\tValidation loss: 0.537646\tBest loss: 0.086076\tAccuracy: 75.45%\n", + "16\tValidation loss: 1.009708\tBest loss: 0.086076\tAccuracy: 53.99%\n", + "17\tValidation loss: 1.228350\tBest loss: 0.086076\tAccuracy: 38.15%\n", + "18\tValidation loss: 1.510606\tBest loss: 0.086076\tAccuracy: 29.44%\n", + "19\tValidation loss: 1.632344\tBest loss: 0.086076\tAccuracy: 22.01%\n", + "20\tValidation loss: 1.628246\tBest loss: 0.086076\tAccuracy: 22.01%\n", + "21\tValidation loss: 1.626765\tBest loss: 0.086076\tAccuracy: 22.01%\n", + "22\tValidation loss: 1.651615\tBest loss: 0.086076\tAccuracy: 18.73%\n", + "23\tValidation loss: 1.663751\tBest loss: 0.086076\tAccuracy: 19.27%\n", + "24\tValidation loss: 1.675138\tBest loss: 0.086076\tAccuracy: 22.01%\n", + "25\tValidation loss: 1.743664\tBest loss: 0.086076\tAccuracy: 18.73%\n", + "Early stopping!\n" + ] + }, + { + "data": { + "text/plain": [ + "DNNClassifier(activation=,\n", + " batch_norm_momentum=None, batch_size=20, dropout_rate=None,\n", + " initializer=._initializer at 0x7fd9d5e628c8>,\n", + " learning_rate=0.01, n_hidden_layers=5, n_neurons=100,\n", + " optimizer_class=,\n", + " random_state=42)" + ] + }, + "execution_count": 120, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "dnn_clf = DNNClassifier(random_state=42)\n", "dnn_clf.fit(X_train1, y_train1, n_epochs=1000, X_valid=X_valid1, y_valid=y_valid1)" @@ -3271,7 +4343,18 @@ "cell_type": "code", "execution_count": 121, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "0.98054096127651291" + ] + }, + "execution_count": 121, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "from sklearn.metrics import accuracy_score\n", "\n", @@ -3290,7 +4373,250 @@ "cell_type": "code", "execution_count": 122, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Fitting 3 folds for each of 50 candidates, totalling 150 fits\n", + "[CV] n_neurons=10, learning_rate=0.05, activation=, batch_size=100 \n", + "0\tValidation loss: 0.132355\tBest loss: 0.132355\tAccuracy: 96.44%\n", + "1\tValidation loss: 0.126329\tBest loss: 0.126329\tAccuracy: 96.21%\n", + "2\tValidation loss: 0.138284\tBest loss: 0.126329\tAccuracy: 96.76%\n", + "3\tValidation loss: 0.142094\tBest loss: 0.126329\tAccuracy: 96.25%\n", + "4\tValidation loss: 0.128141\tBest loss: 0.126329\tAccuracy: 96.76%\n", + "5\tValidation loss: 0.119928\tBest loss: 0.119928\tAccuracy: 97.26%\n", + "6\tValidation loss: 0.137134\tBest loss: 0.119928\tAccuracy: 96.72%\n", + "7\tValidation loss: 0.156194\tBest loss: 0.119928\tAccuracy: 96.79%\n", + "8\tValidation loss: 0.283938\tBest loss: 0.119928\tAccuracy: 94.53%\n", + "9\tValidation loss: 1.104801\tBest loss: 0.119928\tAccuracy: 52.38%\n", + "10\tValidation loss: 0.966833\tBest loss: 0.119928\tAccuracy: 53.09%\n", + "11\tValidation loss: 0.854368\tBest loss: 0.119928\tAccuracy: 57.47%\n", + "12\tValidation loss: 1.857330\tBest loss: 0.119928\tAccuracy: 38.98%\n", + "13\tValidation loss: 1.642338\tBest loss: 0.119928\tAccuracy: 18.73%\n", + "14\tValidation loss: 1.612854\tBest loss: 0.119928\tAccuracy: 22.01%\n", + "15\tValidation loss: 1.617682\tBest loss: 0.119928\tAccuracy: 22.01%\n", + "16\tValidation loss: 1.616873\tBest loss: 0.119928\tAccuracy: 22.01%\n", + "17\tValidation loss: 1.618228\tBest loss: 0.119928\tAccuracy: 19.27%\n", + "18\tValidation loss: 1.619055\tBest loss: 0.119928\tAccuracy: 19.27%\n", + "19\tValidation loss: 1.643334\tBest loss: 0.119928\tAccuracy: 19.08%\n", + "20\tValidation loss: 1.621200\tBest loss: 0.119928\tAccuracy: 19.08%\n", + "21\tValidation loss: 1.629823\tBest loss: 0.119928\tAccuracy: 19.27%\n", + "22\tValidation loss: 1.624553\tBest loss: 0.119928\tAccuracy: 18.73%\n", + "23\tValidation loss: 1.610214\tBest loss: 0.119928\tAccuracy: 20.91%\n", + "24\tValidation loss: 1.621143\tBest loss: 0.119928\tAccuracy: 22.01%\n", + "25\tValidation loss: 1.623761\tBest loss: 0.119928\tAccuracy: 22.01%\n", + "26\tValidation loss: 1.641760\tBest loss: 0.119928\tAccuracy: 18.73%\n", + "Early stopping!\n", + "[CV] n_neurons=10, learning_rate=0.05, activation=, batch_size=100, total= 5.6s\n", + "[CV] n_neurons=10, learning_rate=0.05, activation=, batch_size=100 \n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 5.6s remaining: 0.0s\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0\tValidation loss: 0.153707\tBest loss: 0.153707\tAccuracy: 95.74%\n", + "1\tValidation loss: 0.120703\tBest loss: 0.120703\tAccuracy: 96.56%\n", + "2\tValidation loss: 0.164706\tBest loss: 0.120703\tAccuracy: 96.05%\n", + "3\tValidation loss: 0.177875\tBest loss: 0.120703\tAccuracy: 95.19%\n", + "4\tValidation loss: 0.171004\tBest loss: 0.120703\tAccuracy: 95.19%\n", + "5\tValidation loss: 0.114746\tBest loss: 0.114746\tAccuracy: 96.83%\n", + "6\tValidation loss: 0.109637\tBest loss: 0.109637\tAccuracy: 97.26%\n", + "7\tValidation loss: 0.261533\tBest loss: 0.109637\tAccuracy: 94.96%\n", + "8\tValidation loss: 0.316743\tBest loss: 0.109637\tAccuracy: 94.02%\n", + "9\tValidation loss: 0.486484\tBest loss: 0.109637\tAccuracy: 77.56%\n", + "10\tValidation loss: 4.635532\tBest loss: 0.109637\tAccuracy: 53.95%\n", + "11\tValidation loss: 1.172422\tBest loss: 0.109637\tAccuracy: 48.36%\n", + "12\tValidation loss: 1.029865\tBest loss: 0.109637\tAccuracy: 55.98%\n", + "13\tValidation loss: 1.298800\tBest loss: 0.109637\tAccuracy: 36.08%\n", + "14\tValidation loss: 1.141950\tBest loss: 0.109637\tAccuracy: 38.08%\n", + "15\tValidation loss: 1.132486\tBest loss: 0.109637\tAccuracy: 38.90%\n", + "16\tValidation loss: 1.078486\tBest loss: 0.109637\tAccuracy: 45.78%\n", + "17\tValidation loss: 1.128344\tBest loss: 0.109637\tAccuracy: 45.07%\n", + "18\tValidation loss: 1.336244\tBest loss: 0.109637\tAccuracy: 34.40%\n", + "19\tValidation loss: 1.199178\tBest loss: 0.109637\tAccuracy: 39.87%\n", + "20\tValidation loss: 1.175845\tBest loss: 0.109637\tAccuracy: 40.11%\n", + "21\tValidation loss: 1.200430\tBest loss: 0.109637\tAccuracy: 40.30%\n", + "22\tValidation loss: 1.390084\tBest loss: 0.109637\tAccuracy: 34.60%\n", + "23\tValidation loss: 1.268129\tBest loss: 0.109637\tAccuracy: 40.23%\n", + "24\tValidation loss: 1.192210\tBest loss: 0.109637\tAccuracy: 40.30%\n", + "25\tValidation loss: 1.190541\tBest loss: 0.109637\tAccuracy: 41.99%\n", + "26\tValidation loss: 1.227676\tBest loss: 0.109637\tAccuracy: 38.62%\n", + "27\tValidation loss: 1.187587\tBest loss: 0.109637\tAccuracy: 39.44%\n", + "Early stopping!\n", + "[CV] n_neurons=10, learning_rate=0.05, activation=, batch_size=100, total= 5.9s\n", + "[CV] n_neurons=10, learning_rate=0.05, activation=, batch_size=100 \n", + "0\tValidation loss: 0.182619\tBest loss: 0.182619\tAccuracy: 94.29%\n", + "1\tValidation loss: 0.152706\tBest loss: 0.152706\tAccuracy: 95.97%\n", + "2\tValidation loss: 0.193820\tBest loss: 0.152706\tAccuracy: 93.82%\n", + "3\tValidation loss: 0.195413\tBest loss: 0.152706\tAccuracy: 95.54%\n", + "4\tValidation loss: 0.171277\tBest loss: 0.152706\tAccuracy: 95.19%\n", + "5\tValidation loss: 0.140087\tBest loss: 0.140087\tAccuracy: 95.70%\n", + "6\tValidation loss: 0.170798\tBest loss: 0.140087\tAccuracy: 95.00%\n", + "7\tValidation loss: 0.163649\tBest loss: 0.140087\tAccuracy: 96.29%\n", + "8\tValidation loss: 0.199048\tBest loss: 0.140087\tAccuracy: 96.09%\n", + "9\tValidation loss: 1.552870\tBest loss: 0.140087\tAccuracy: 52.15%\n", + "10\tValidation loss: 0.813273\tBest loss: 0.140087\tAccuracy: 60.40%\n", + "11\tValidation loss: 0.775555\tBest loss: 0.140087\tAccuracy: 60.67%\n", + "12\tValidation loss: 0.775275\tBest loss: 0.140087\tAccuracy: 59.77%\n", + "13\tValidation loss: 0.770521\tBest loss: 0.140087\tAccuracy: 59.30%\n", + "14\tValidation loss: 0.734035\tBest loss: 0.140087\tAccuracy: 59.85%\n", + "15\tValidation loss: 0.744980\tBest loss: 0.140087\tAccuracy: 59.66%\n", + "16\tValidation loss: 0.785848\tBest loss: 0.140087\tAccuracy: 59.66%\n", + "17\tValidation loss: 0.776138\tBest loss: 0.140087\tAccuracy: 59.42%\n", + "18\tValidation loss: 0.764496\tBest loss: 0.140087\tAccuracy: 59.46%\n", + "19\tValidation loss: 0.763633\tBest loss: 0.140087\tAccuracy: 59.54%\n", + "20\tValidation loss: 0.743879\tBest loss: 0.140087\tAccuracy: 60.75%\n", + "21\tValidation loss: 0.763295\tBest loss: 0.140087\tAccuracy: 60.36%\n", + "22\tValidation loss: 0.717175\tBest loss: 0.140087\tAccuracy: 60.63%\n", + "23\tValidation loss: 1.869954\tBest loss: 0.140087\tAccuracy: 29.28%\n", + "24\tValidation loss: 1.215518\tBest loss: 0.140087\tAccuracy: 38.86%\n", + "25\tValidation loss: 1.196626\tBest loss: 0.140087\tAccuracy: 38.62%\n", + "26\tValidation loss: 1.170714\tBest loss: 0.140087\tAccuracy: 42.38%\n", + "Early stopping!\n", + "[CV] n_neurons=10, learning_rate=0.05, activation=, batch_size=100, total= 6.9s\n", + "[CV] n_neurons=30, learning_rate=0.02, activation=, batch_size=500 \n", + "0\tValidation loss: 0.171512\tBest loss: 0.171512\tAccuracy: 95.07%\n", + "1\tValidation loss: 0.095914\tBest loss: 0.095914\tAccuracy: 97.03%\n", + "2\tValidation loss: 0.099199\tBest loss: 0.095914\tAccuracy: 96.91%\n", + "3\tValidation loss: 0.093873\tBest loss: 0.093873\tAccuracy: 97.15%\n", + "4\tValidation loss: 0.073461\tBest loss: 0.073461\tAccuracy: 98.01%\n", + "5\tValidation loss: 0.084562\tBest loss: 0.073461\tAccuracy: 97.65%\n", + "6\tValidation loss: 0.071800\tBest loss: 0.071800\tAccuracy: 98.01%\n", + "7\tValidation loss: 0.088435\tBest loss: 0.071800\tAccuracy: 97.73%\n", + "8\tValidation loss: 0.082038\tBest loss: 0.071800\tAccuracy: 97.77%\n", + "9\tValidation loss: 0.080673\tBest loss: 0.071800\tAccuracy: 97.69%\n", + "10\tValidation loss: 0.081036\tBest loss: 0.071800\tAccuracy: 97.93%\n", + "11\tValidation loss: 0.092700\tBest loss: 0.071800\tAccuracy: 97.93%\n", + "12\tValidation loss: 0.081003\tBest loss: 0.071800\tAccuracy: 98.20%\n", + "13\tValidation loss: 0.075607\tBest loss: 0.071800\tAccuracy: 98.20%\n", + "14\tValidation loss: 0.092970\tBest loss: 0.071800\tAccuracy: 98.08%\n", + "15\tValidation loss: 0.108005\tBest loss: 0.071800\tAccuracy: 97.77%\n", + "16\tValidation loss: 0.082602\tBest loss: 0.071800\tAccuracy: 98.05%\n", + "17\tValidation loss: 0.114629\tBest loss: 0.071800\tAccuracy: 97.73%\n", + "18\tValidation loss: 0.099099\tBest loss: 0.071800\tAccuracy: 97.69%\n", + "19\tValidation loss: 0.075535\tBest loss: 0.071800\tAccuracy: 98.20%\n", + "20\tValidation loss: 0.102847\tBest loss: 0.071800\tAccuracy: 98.08%\n", + "21\tValidation loss: 0.089735\tBest loss: 0.071800\tAccuracy: 98.36%\n", + "22\tValidation loss: 0.080781\tBest loss: 0.071800\tAccuracy: 97.93%\n", + "23\tValidation loss: 0.073017\tBest loss: 0.071800\tAccuracy: 98.32%\n", + "24\tValidation loss: 0.091643\tBest loss: 0.071800\tAccuracy: 97.93%\n", + "25\tValidation loss: 0.113891\tBest loss: 0.071800\tAccuracy: 98.05%\n", + "26\tValidation loss: 0.094774\tBest loss: 0.071800\tAccuracy: 98.28%\n", + "27\tValidation loss: 0.086041\tBest loss: 0.071800\tAccuracy: 98.20%\n", + "Early stopping!\n", + "[CV] n_neurons=30, learning_rate=0.02, activation=, batch_size=500, total= 6.8s\n", + "[CV] n_neurons=30, learning_rate=0.02, activation=, batch_size=500 \n", + "0\tValidation loss: 0.113188\tBest loss: 0.113188\tAccuracy: 96.60%\n", + "1\tValidation loss: 0.081384\tBest loss: 0.081384\tAccuracy: 97.58%\n", + "2\tValidation loss: 0.068770\tBest loss: 0.068770\tAccuracy: 98.12%\n", + "3\tValidation loss: 0.077316\tBest loss: 0.068770\tAccuracy: 97.73%\n", + "4\tValidation loss: 0.074333\tBest loss: 0.068770\tAccuracy: 97.97%\n", + "5\tValidation loss: 0.084735\tBest loss: 0.068770\tAccuracy: 97.30%\n", + "6\tValidation loss: 0.082893\tBest loss: 0.068770\tAccuracy: 97.69%\n", + "7\tValidation loss: 0.075860\tBest loss: 0.068770\tAccuracy: 97.65%\n", + "8\tValidation loss: 0.078686\tBest loss: 0.068770\tAccuracy: 97.77%\n", + "9\tValidation loss: 0.080869\tBest loss: 0.068770\tAccuracy: 97.77%\n", + "10\tValidation loss: 0.082026\tBest loss: 0.068770\tAccuracy: 98.12%\n", + "11\tValidation loss: 0.086516\tBest loss: 0.068770\tAccuracy: 97.69%\n", + "12\tValidation loss: 0.076660\tBest loss: 0.068770\tAccuracy: 98.12%\n", + "13\tValidation loss: 0.073815\tBest loss: 0.068770\tAccuracy: 98.08%\n", + "14\tValidation loss: 0.077873\tBest loss: 0.068770\tAccuracy: 98.20%\n", + "15\tValidation loss: 0.078704\tBest loss: 0.068770\tAccuracy: 97.93%\n", + "16\tValidation loss: 0.077061\tBest loss: 0.068770\tAccuracy: 98.28%\n", + "17\tValidation loss: 0.075423\tBest loss: 0.068770\tAccuracy: 97.93%\n", + "18\tValidation loss: 0.085646\tBest loss: 0.068770\tAccuracy: 98.24%\n", + "19\tValidation loss: 0.082202\tBest loss: 0.068770\tAccuracy: 98.05%\n", + "20\tValidation loss: 0.103338\tBest loss: 0.068770\tAccuracy: 97.46%\n", + "21\tValidation loss: 0.068182\tBest loss: 0.068182\tAccuracy: 98.40%\n", + "22\tValidation loss: 0.067592\tBest loss: 0.067592\tAccuracy: 97.93%\n", + "23\tValidation loss: 0.076756\tBest loss: 0.067592\tAccuracy: 98.28%\n", + "24\tValidation loss: 0.072327\tBest loss: 0.067592\tAccuracy: 98.48%\n", + "25\tValidation loss: 0.075613\tBest loss: 0.067592\tAccuracy: 98.44%\n", + "26\tValidation loss: 0.072291\tBest loss: 0.067592\tAccuracy: 98.40%\n", + "27\tValidation loss: 0.084550\tBest loss: 0.067592\tAccuracy: 98.28%\n", + "28\tValidation loss: 0.075566\tBest loss: 0.067592\tAccuracy: 98.36%\n", + "29\tValidation loss: 0.071688\tBest loss: 0.067592\tAccuracy: 98.28%\n", + "30\tValidation loss: 0.075556\tBest loss: 0.067592\tAccuracy: 98.24%\n", + "31\tValidation loss: 0.065671\tBest loss: 0.065671\tAccuracy: 98.40%\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "32\tValidation loss: 0.083471\tBest loss: 0.065671\tAccuracy: 98.40%\n", + "33\tValidation loss: 0.086415\tBest loss: 0.065671\tAccuracy: 98.59%\n", + "34\tValidation loss: 0.085613\tBest loss: 0.065671\tAccuracy: 98.36%\n", + "35\tValidation loss: 0.099534\tBest loss: 0.065671\tAccuracy: 98.28%\n", + "36\tValidation loss: 0.102709\tBest loss: 0.065671\tAccuracy: 98.32%\n", + "37\tValidation loss: 0.093125\tBest loss: 0.065671\tAccuracy: 98.20%\n", + "38\tValidation loss: 0.109501\tBest loss: 0.065671\tAccuracy: 97.85%\n", + "39\tValidation loss: 0.109443\tBest loss: 0.065671\tAccuracy: 98.44%\n", + "40\tValidation loss: 0.087260\tBest loss: 0.065671\tAccuracy: 98.36%\n", + "41\tValidation loss: 0.106365\tBest loss: 0.065671\tAccuracy: 98.36%\n", + "42\tValidation loss: 0.102789\tBest loss: 0.065671\tAccuracy: 98.05%\n", + "43\tValidation loss: 0.094281\tBest loss: 0.065671\tAccuracy: 98.48%\n", + "44\tValidation loss: 0.094514\tBest loss: 0.065671\tAccuracy: 98.40%\n", + "[...and much later...]\n", + "20\tValidation loss: 0.046808\tBest loss: 0.033867\tAccuracy: 98.83%\n", + "21\tValidation loss: 0.052966\tBest loss: 0.033867\tAccuracy: 98.91%\n", + "22\tValidation loss: 0.095892\tBest loss: 0.033867\tAccuracy: 98.08%\n", + "23\tValidation loss: 0.054250\tBest loss: 0.033867\tAccuracy: 98.87%\n", + "24\tValidation loss: 0.061026\tBest loss: 0.033867\tAccuracy: 98.87%\n", + "25\tValidation loss: 0.081977\tBest loss: 0.033867\tAccuracy: 98.67%\n", + "26\tValidation loss: 0.079819\tBest loss: 0.033867\tAccuracy: 98.71%\n", + "27\tValidation loss: 0.059824\tBest loss: 0.033867\tAccuracy: 98.75%\n", + "28\tValidation loss: 0.057758\tBest loss: 0.033867\tAccuracy: 98.94%\n", + "29\tValidation loss: 0.087165\tBest loss: 0.033867\tAccuracy: 98.91%\n", + "30\tValidation loss: 0.052274\tBest loss: 0.033867\tAccuracy: 99.10%\n", + "31\tValidation loss: 0.059831\tBest loss: 0.033867\tAccuracy: 98.79%\n", + "32\tValidation loss: 0.054240\tBest loss: 0.033867\tAccuracy: 98.91%\n", + "33\tValidation loss: 0.048165\tBest loss: 0.033867\tAccuracy: 98.94%\n", + "34\tValidation loss: 0.040565\tBest loss: 0.033867\tAccuracy: 99.18%\n", + "35\tValidation loss: 0.103207\tBest loss: 0.033867\tAccuracy: 98.28%\n", + "36\tValidation loss: 400.716797\tBest loss: 0.033867\tAccuracy: 71.46%\n", + "37\tValidation loss: 11.996887\tBest loss: 0.033867\tAccuracy: 96.09%\n", + "38\tValidation loss: 2.623182\tBest loss: 0.033867\tAccuracy: 96.56%\n", + "39\tValidation loss: 1.344962\tBest loss: 0.033867\tAccuracy: 97.69%\n", + "40\tValidation loss: 1.125381\tBest loss: 0.033867\tAccuracy: 97.42%\n", + "Early stopping!\n" + ] + }, + { + "data": { + "text/plain": [ + "RandomizedSearchCV(cv=None, error_score='raise',\n", + " estimator=DNNClassifier(activation=,\n", + " batch_norm_momentum=None, batch_size=20, dropout_rate=None,\n", + " initializer=._initializer at 0x7fd9d5e628c8>,\n", + " learning_rate=0.01, n_hidden_layers=5, n_neurons=100,\n", + " optimizer_class=,\n", + " random_state=42),\n", + " fit_params={'y_valid': array([0, 4, ..., 1, 2], dtype=uint8), 'X_valid': array([[ 0., 0., ..., 0., 0.],\n", + " [ 0., 0., ..., 0., 0.],\n", + " ...,\n", + " [ 0., 0., ..., 0., 0.],\n", + " [ 0., 0., ..., 0., 0.]], dtype=float32), 'n_epochs': 1000},\n", + " iid=True, n_iter=50, n_jobs=1,\n", + " param_distributions={'n_neurons': [10, 30, 50, 70, 90, 100, 120, 140, 160], 'learning_rate': [0.01, 0.02, 0.05, 0.1], 'activation': [, , .parametrized_leaky_relu at 0x7fd9db0b30d0>, .parametrized_leaky_relu at 0x7fd9d4ddca60>], 'batch_size': [10, 50, 100, 500]},\n", + " pre_dispatch='2*n_jobs', random_state=42, refit=True,\n", + " return_train_score=True, scoring=None, verbose=2)" + ] + }, + "execution_count": 122, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "from sklearn.model_selection import RandomizedSearchCV\n", "\n", @@ -3309,17 +4635,38 @@ " #\"optimizer_class\": [tf.train.AdamOptimizer, partial(tf.train.MomentumOptimizer, momentum=0.95)],\n", "}\n", "\n", - "rnd_search = RandomizedSearchCV(DNNClassifier(random_state=42), param_distribs, n_iter=50,\n", + "rnd_search = RandomizedSearchCV(DNNClassifier(random_state=42), param_distribs, n_iter=50,\n", + " fit_params={\"X_valid\": X_valid1, \"y_valid\": y_valid1, \"n_epochs\": 1000},\n", " random_state=42, verbose=2)\n", - "fit_params={\"X_valid\": X_valid1, \"y_valid\": y_valid1, \"n_epochs\": 1000}\n", - "rnd_search.fit(X_train1, y_train1, **fit_params)" + "rnd_search.fit(X_train1, y_train1)" + " + "#fit_params as a constructor argument was deprecated in [scikit-learn] version 0.19 and will be removed\n", + "#in version 0.21. Pass fit parameters to the fit method instead:" + "#rnd_search = RandomizedSearchCV(DNNClassifier(random_state=42), param_distribs, n_iter=50,\n", + "# random_state=42, verbose=2)\n", + "#fit_params={\"X_valid\": X_valid1, \"y_valid\": y_valid1, \"n_epochs\": 1000}" + "#rnd_search.fit(X_train1, y_train1, **fit_params)" ] }, { "cell_type": "code", "execution_count": 123, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "{'activation': .parametrized_leaky_relu>,\n", + " 'batch_size': 500,\n", + " 'learning_rate': 0.01,\n", + " 'n_neurons': 140}" + ] + }, + "execution_count": 123, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "rnd_search.best_params_" ] @@ -3328,7 +4675,18 @@ "cell_type": "code", "execution_count": 124, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "0.99318933644677954" + ] + }, + "execution_count": 124, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "y_pred = rnd_search.predict(X_test1)\n", "accuracy_score(y_test1, y_pred)" @@ -3384,7 +4742,71 @@ "cell_type": "code", "execution_count": 126, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0\tValidation loss: 0.090732\tBest loss: 0.090732\tAccuracy: 97.22%\n", + "1\tValidation loss: 0.052198\tBest loss: 0.052198\tAccuracy: 98.40%\n", + "2\tValidation loss: 0.040040\tBest loss: 0.040040\tAccuracy: 98.94%\n", + "3\tValidation loss: 0.057495\tBest loss: 0.040040\tAccuracy: 98.55%\n", + "4\tValidation loss: 0.045600\tBest loss: 0.040040\tAccuracy: 98.75%\n", + "5\tValidation loss: 0.062344\tBest loss: 0.040040\tAccuracy: 98.48%\n", + "6\tValidation loss: 0.048719\tBest loss: 0.040040\tAccuracy: 98.67%\n", + "7\tValidation loss: 0.050346\tBest loss: 0.040040\tAccuracy: 98.79%\n", + "8\tValidation loss: 0.051224\tBest loss: 0.040040\tAccuracy: 98.79%\n", + "9\tValidation loss: 0.036505\tBest loss: 0.036505\tAccuracy: 98.98%\n", + "10\tValidation loss: 0.052532\tBest loss: 0.036505\tAccuracy: 98.71%\n", + "11\tValidation loss: 0.057086\tBest loss: 0.036505\tAccuracy: 99.10%\n", + "12\tValidation loss: 0.036754\tBest loss: 0.036505\tAccuracy: 99.06%\n", + "13\tValidation loss: 0.046782\tBest loss: 0.036505\tAccuracy: 98.87%\n", + "14\tValidation loss: 0.048929\tBest loss: 0.036505\tAccuracy: 98.91%\n", + "15\tValidation loss: 0.052919\tBest loss: 0.036505\tAccuracy: 98.75%\n", + "16\tValidation loss: 0.054287\tBest loss: 0.036505\tAccuracy: 98.67%\n", + "17\tValidation loss: 0.047722\tBest loss: 0.036505\tAccuracy: 98.79%\n", + "18\tValidation loss: 0.040474\tBest loss: 0.036505\tAccuracy: 99.14%\n", + "19\tValidation loss: 0.033867\tBest loss: 0.033867\tAccuracy: 99.14%\n", + "20\tValidation loss: 0.046808\tBest loss: 0.033867\tAccuracy: 98.83%\n", + "21\tValidation loss: 0.052966\tBest loss: 0.033867\tAccuracy: 98.91%\n", + "22\tValidation loss: 0.095892\tBest loss: 0.033867\tAccuracy: 98.08%\n", + "23\tValidation loss: 0.054250\tBest loss: 0.033867\tAccuracy: 98.87%\n", + "24\tValidation loss: 0.061026\tBest loss: 0.033867\tAccuracy: 98.87%\n", + "25\tValidation loss: 0.081977\tBest loss: 0.033867\tAccuracy: 98.67%\n", + "26\tValidation loss: 0.079819\tBest loss: 0.033867\tAccuracy: 98.71%\n", + "27\tValidation loss: 0.059824\tBest loss: 0.033867\tAccuracy: 98.75%\n", + "28\tValidation loss: 0.057758\tBest loss: 0.033867\tAccuracy: 98.94%\n", + "29\tValidation loss: 0.087165\tBest loss: 0.033867\tAccuracy: 98.91%\n", + "30\tValidation loss: 0.052274\tBest loss: 0.033867\tAccuracy: 99.10%\n", + "31\tValidation loss: 0.059831\tBest loss: 0.033867\tAccuracy: 98.79%\n", + "32\tValidation loss: 0.054240\tBest loss: 0.033867\tAccuracy: 98.91%\n", + "33\tValidation loss: 0.048165\tBest loss: 0.033867\tAccuracy: 98.94%\n", + "34\tValidation loss: 0.040565\tBest loss: 0.033867\tAccuracy: 99.18%\n", + "35\tValidation loss: 0.103207\tBest loss: 0.033867\tAccuracy: 98.28%\n", + "36\tValidation loss: 400.716797\tBest loss: 0.033867\tAccuracy: 71.46%\n", + "37\tValidation loss: 11.996887\tBest loss: 0.033867\tAccuracy: 96.09%\n", + "38\tValidation loss: 2.623182\tBest loss: 0.033867\tAccuracy: 96.56%\n", + "39\tValidation loss: 1.344962\tBest loss: 0.033867\tAccuracy: 97.69%\n", + "40\tValidation loss: 1.125381\tBest loss: 0.033867\tAccuracy: 97.42%\n", + "Early stopping!\n" + ] + }, + { + "data": { + "text/plain": [ + "DNNClassifier(activation=.parametrized_leaky_relu at 0x7fd9d19e37b8>,\n", + " batch_norm_momentum=None, batch_size=500, dropout_rate=None,\n", + " initializer=._initializer at 0x7fd9d5e628c8>,\n", + " learning_rate=0.01, n_hidden_layers=5, n_neurons=140,\n", + " optimizer_class=,\n", + " random_state=42)" + ] + }, + "execution_count": 126, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "dnn_clf = DNNClassifier(activation=leaky_relu(alpha=0.1), batch_size=500, learning_rate=0.01,\n", " n_neurons=140, random_state=42)\n", @@ -3409,7 +4831,18 @@ "cell_type": "code", "execution_count": 127, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "0.99318933644677954" + ] + }, + "execution_count": 127, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "y_pred = dnn_clf.predict(X_test1)\n", "accuracy_score(y_test1, y_pred)" @@ -3426,7 +4859,100 @@ "cell_type": "code", "execution_count": 128, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0\tValidation loss: 0.046053\tBest loss: 0.046053\tAccuracy: 98.67%\n", + "1\tValidation loss: 0.032228\tBest loss: 0.032228\tAccuracy: 98.83%\n", + "2\tValidation loss: 0.032974\tBest loss: 0.032228\tAccuracy: 98.83%\n", + "3\tValidation loss: 0.035961\tBest loss: 0.032228\tAccuracy: 98.94%\n", + "4\tValidation loss: 0.040250\tBest loss: 0.032228\tAccuracy: 98.94%\n", + "5\tValidation loss: 0.033051\tBest loss: 0.032228\tAccuracy: 99.06%\n", + "6\tValidation loss: 0.056053\tBest loss: 0.032228\tAccuracy: 98.32%\n", + "7\tValidation loss: 0.031729\tBest loss: 0.031729\tAccuracy: 99.18%\n", + "8\tValidation loss: 0.027662\tBest loss: 0.027662\tAccuracy: 99.26%\n", + "9\tValidation loss: 0.034074\tBest loss: 0.027662\tAccuracy: 98.94%\n", + "10\tValidation loss: 0.032173\tBest loss: 0.027662\tAccuracy: 99.06%\n", + "11\tValidation loss: 0.030538\tBest loss: 0.027662\tAccuracy: 99.10%\n", + "12\tValidation loss: 0.030337\tBest loss: 0.027662\tAccuracy: 99.10%\n", + "13\tValidation loss: 0.022219\tBest loss: 0.022219\tAccuracy: 99.45%\n", + "14\tValidation loss: 0.036824\tBest loss: 0.022219\tAccuracy: 99.14%\n", + "15\tValidation loss: 0.033945\tBest loss: 0.022219\tAccuracy: 99.18%\n", + "16\tValidation loss: 0.032533\tBest loss: 0.022219\tAccuracy: 98.98%\n", + "17\tValidation loss: 0.037204\tBest loss: 0.022219\tAccuracy: 99.02%\n", + "18\tValidation loss: 0.026982\tBest loss: 0.022219\tAccuracy: 99.34%\n", + "19\tValidation loss: 0.022094\tBest loss: 0.022094\tAccuracy: 99.53%\n", + "20\tValidation loss: 0.026196\tBest loss: 0.022094\tAccuracy: 99.26%\n", + "21\tValidation loss: 0.022107\tBest loss: 0.022094\tAccuracy: 99.49%\n", + "22\tValidation loss: 0.021436\tBest loss: 0.021436\tAccuracy: 99.53%\n", + "23\tValidation loss: 0.025607\tBest loss: 0.021436\tAccuracy: 99.37%\n", + "24\tValidation loss: 0.038882\tBest loss: 0.021436\tAccuracy: 99.22%\n", + "25\tValidation loss: 0.032011\tBest loss: 0.021436\tAccuracy: 99.26%\n", + "26\tValidation loss: 0.027673\tBest loss: 0.021436\tAccuracy: 99.22%\n", + "27\tValidation loss: 0.026874\tBest loss: 0.021436\tAccuracy: 99.30%\n", + "28\tValidation loss: 0.021123\tBest loss: 0.021123\tAccuracy: 99.41%\n", + "29\tValidation loss: 0.024784\tBest loss: 0.021123\tAccuracy: 99.45%\n", + "30\tValidation loss: 0.024108\tBest loss: 0.021123\tAccuracy: 99.49%\n", + "31\tValidation loss: 0.028439\tBest loss: 0.021123\tAccuracy: 99.37%\n", + "32\tValidation loss: 0.032366\tBest loss: 0.021123\tAccuracy: 99.22%\n", + "33\tValidation loss: 0.037057\tBest loss: 0.021123\tAccuracy: 99.18%\n", + "34\tValidation loss: 0.042305\tBest loss: 0.021123\tAccuracy: 98.98%\n", + "35\tValidation loss: 0.039662\tBest loss: 0.021123\tAccuracy: 99.14%\n", + "36\tValidation loss: 0.036299\tBest loss: 0.021123\tAccuracy: 99.14%\n", + "37\tValidation loss: 0.026997\tBest loss: 0.021123\tAccuracy: 99.53%\n", + "38\tValidation loss: 0.034407\tBest loss: 0.021123\tAccuracy: 99.22%\n", + "39\tValidation loss: 0.027668\tBest loss: 0.021123\tAccuracy: 99.41%\n", + "40\tValidation loss: 0.029128\tBest loss: 0.021123\tAccuracy: 99.30%\n", + "41\tValidation loss: 0.033564\tBest loss: 0.021123\tAccuracy: 99.14%\n", + "42\tValidation loss: 0.033810\tBest loss: 0.021123\tAccuracy: 99.30%\n", + "43\tValidation loss: 0.044953\tBest loss: 0.021123\tAccuracy: 98.98%\n", + "44\tValidation loss: 0.026280\tBest loss: 0.021123\tAccuracy: 99.26%\n", + "45\tValidation loss: 0.020275\tBest loss: 0.020275\tAccuracy: 99.61%\n", + "46\tValidation loss: 0.018810\tBest loss: 0.018810\tAccuracy: 99.45%\n", + "47\tValidation loss: 0.027529\tBest loss: 0.018810\tAccuracy: 99.18%\n", + "48\tValidation loss: 0.018120\tBest loss: 0.018120\tAccuracy: 99.53%\n", + "49\tValidation loss: 0.019378\tBest loss: 0.018120\tAccuracy: 99.45%\n", + "50\tValidation loss: 0.029760\tBest loss: 0.018120\tAccuracy: 99.34%\n", + "51\tValidation loss: 0.035702\tBest loss: 0.018120\tAccuracy: 99.26%\n", + "52\tValidation loss: 0.032662\tBest loss: 0.018120\tAccuracy: 99.02%\n", + "53\tValidation loss: 0.026943\tBest loss: 0.018120\tAccuracy: 99.37%\n", + "54\tValidation loss: 0.029007\tBest loss: 0.018120\tAccuracy: 99.53%\n", + "55\tValidation loss: 0.021956\tBest loss: 0.018120\tAccuracy: 99.49%\n", + "56\tValidation loss: 0.018983\tBest loss: 0.018120\tAccuracy: 99.61%\n", + "57\tValidation loss: 0.022788\tBest loss: 0.018120\tAccuracy: 99.49%\n", + "58\tValidation loss: 0.019578\tBest loss: 0.018120\tAccuracy: 99.61%\n", + "59\tValidation loss: 0.021676\tBest loss: 0.018120\tAccuracy: 99.61%\n", + "60\tValidation loss: 0.021580\tBest loss: 0.018120\tAccuracy: 99.65%\n", + "61\tValidation loss: 0.021467\tBest loss: 0.018120\tAccuracy: 99.65%\n", + "62\tValidation loss: 0.020513\tBest loss: 0.018120\tAccuracy: 99.65%\n", + "63\tValidation loss: 0.020252\tBest loss: 0.018120\tAccuracy: 99.65%\n", + "64\tValidation loss: 0.021724\tBest loss: 0.018120\tAccuracy: 99.65%\n", + "65\tValidation loss: 0.021499\tBest loss: 0.018120\tAccuracy: 99.69%\n", + "66\tValidation loss: 0.021627\tBest loss: 0.018120\tAccuracy: 99.69%\n", + "67\tValidation loss: 0.021569\tBest loss: 0.018120\tAccuracy: 99.69%\n", + "68\tValidation loss: 0.021727\tBest loss: 0.018120\tAccuracy: 99.69%\n", + "69\tValidation loss: 0.021104\tBest loss: 0.018120\tAccuracy: 99.69%\n", + "Early stopping!\n" + ] + }, + { + "data": { + "text/plain": [ + "DNNClassifier(activation=.parametrized_leaky_relu at 0x7fd9d19e3c80>,\n", + " batch_norm_momentum=0.95, batch_size=500, dropout_rate=None,\n", + " initializer=._initializer at 0x7fd9d5e628c8>,\n", + " learning_rate=0.01, n_hidden_layers=5, n_neurons=90,\n", + " optimizer_class=,\n", + " random_state=42)" + ] + }, + "execution_count": 128, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "dnn_clf_bn = DNNClassifier(activation=leaky_relu(alpha=0.1), batch_size=500, learning_rate=0.01,\n", " n_neurons=90, random_state=42,\n", @@ -3445,7 +4971,18 @@ "cell_type": "code", "execution_count": 129, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "0.99241097489784003" + ] + }, + "execution_count": 129, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "y_pred = dnn_clf_bn.predict(X_test1)\n", "accuracy_score(y_test1, y_pred)" @@ -3462,7 +4999,222 @@ "cell_type": "code", "execution_count": 130, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Fitting 3 folds for each of 50 candidates, totalling 150 fits\n", + "[CV] activation=, n_neurons=70, learning_rate=0.01, batch_norm_momentum=0.99, batch_size=50 \n", + "0\tValidation loss: 0.113224\tBest loss: 0.113224\tAccuracy: 97.30%\n", + "1\tValidation loss: 0.064190\tBest loss: 0.064190\tAccuracy: 98.24%\n", + "2\tValidation loss: 0.080173\tBest loss: 0.064190\tAccuracy: 98.28%\n", + "3\tValidation loss: 0.059603\tBest loss: 0.059603\tAccuracy: 98.28%\n", + "4\tValidation loss: 0.043533\tBest loss: 0.043533\tAccuracy: 98.48%\n", + "5\tValidation loss: 0.040107\tBest loss: 0.040107\tAccuracy: 98.87%\n", + "6\tValidation loss: 0.051212\tBest loss: 0.040107\tAccuracy: 98.24%\n", + "7\tValidation loss: 0.046029\tBest loss: 0.040107\tAccuracy: 98.71%\n", + "8\tValidation loss: 0.053079\tBest loss: 0.040107\tAccuracy: 98.59%\n", + "9\tValidation loss: 0.066891\tBest loss: 0.040107\tAccuracy: 98.28%\n", + "10\tValidation loss: 0.037712\tBest loss: 0.037712\tAccuracy: 98.83%\n", + "11\tValidation loss: 0.055569\tBest loss: 0.037712\tAccuracy: 98.55%\n", + "12\tValidation loss: 0.040949\tBest loss: 0.037712\tAccuracy: 98.98%\n", + "13\tValidation loss: 0.077433\tBest loss: 0.037712\tAccuracy: 98.36%\n", + "14\tValidation loss: 0.065955\tBest loss: 0.037712\tAccuracy: 98.63%\n", + "15\tValidation loss: 0.038968\tBest loss: 0.037712\tAccuracy: 99.02%\n", + "16\tValidation loss: 0.039190\tBest loss: 0.037712\tAccuracy: 99.06%\n", + "17\tValidation loss: 0.050690\tBest loss: 0.037712\tAccuracy: 98.71%\n", + "18\tValidation loss: 0.043054\tBest loss: 0.037712\tAccuracy: 99.02%\n", + "19\tValidation loss: 0.063156\tBest loss: 0.037712\tAccuracy: 98.71%\n", + "20\tValidation loss: 0.043066\tBest loss: 0.037712\tAccuracy: 99.14%\n", + "21\tValidation loss: 0.058145\tBest loss: 0.037712\tAccuracy: 98.79%\n", + "22\tValidation loss: 0.039590\tBest loss: 0.037712\tAccuracy: 99.06%\n", + "23\tValidation loss: 0.049981\tBest loss: 0.037712\tAccuracy: 98.75%\n", + "24\tValidation loss: 0.047458\tBest loss: 0.037712\tAccuracy: 99.10%\n", + "25\tValidation loss: 0.040638\tBest loss: 0.037712\tAccuracy: 99.06%\n", + "26\tValidation loss: 0.041426\tBest loss: 0.037712\tAccuracy: 98.98%\n", + "27\tValidation loss: 0.041325\tBest loss: 0.037712\tAccuracy: 98.98%\n", + "28\tValidation loss: 0.054609\tBest loss: 0.037712\tAccuracy: 98.91%\n", + "29\tValidation loss: 0.067671\tBest loss: 0.037712\tAccuracy: 98.75%\n", + "30\tValidation loss: 0.037608\tBest loss: 0.037608\tAccuracy: 98.79%\n", + "31\tValidation loss: 0.047441\tBest loss: 0.037608\tAccuracy: 98.98%\n", + "32\tValidation loss: 0.053716\tBest loss: 0.037608\tAccuracy: 99.02%\n", + "33\tValidation loss: 0.045445\tBest loss: 0.037608\tAccuracy: 98.83%\n", + "34\tValidation loss: 0.046023\tBest loss: 0.037608\tAccuracy: 98.94%\n", + "35\tValidation loss: 0.050073\tBest loss: 0.037608\tAccuracy: 98.91%\n", + "36\tValidation loss: 0.051887\tBest loss: 0.037608\tAccuracy: 98.87%\n", + "37\tValidation loss: 0.050272\tBest loss: 0.037608\tAccuracy: 99.02%\n", + "38\tValidation loss: 0.043531\tBest loss: 0.037608\tAccuracy: 99.10%\n", + "39\tValidation loss: 0.054661\tBest loss: 0.037608\tAccuracy: 98.87%\n", + "40\tValidation loss: 0.047607\tBest loss: 0.037608\tAccuracy: 98.87%\n", + "41\tValidation loss: 0.051862\tBest loss: 0.037608\tAccuracy: 99.14%\n", + "42\tValidation loss: 0.044218\tBest loss: 0.037608\tAccuracy: 99.14%\n", + "43\tValidation loss: 0.043707\tBest loss: 0.037608\tAccuracy: 99.06%\n", + "44\tValidation loss: 0.039602\tBest loss: 0.037608\tAccuracy: 99.06%\n", + "45\tValidation loss: 0.048998\tBest loss: 0.037608\tAccuracy: 99.02%\n", + "46\tValidation loss: 0.045562\tBest loss: 0.037608\tAccuracy: 99.14%\n", + "47\tValidation loss: 0.042198\tBest loss: 0.037608\tAccuracy: 99.10%\n", + "48\tValidation loss: 0.027679\tBest loss: 0.027679\tAccuracy: 99.10%\n", + "49\tValidation loss: 0.033783\tBest loss: 0.027679\tAccuracy: 98.94%\n", + "50\tValidation loss: 0.032935\tBest loss: 0.027679\tAccuracy: 99.41%\n", + "51\tValidation loss: 0.042930\tBest loss: 0.027679\tAccuracy: 98.98%\n", + "52\tValidation loss: 0.045454\tBest loss: 0.027679\tAccuracy: 99.06%\n", + "53\tValidation loss: 0.047336\tBest loss: 0.027679\tAccuracy: 98.91%\n", + "54\tValidation loss: 0.036523\tBest loss: 0.027679\tAccuracy: 99.14%\n", + "55\tValidation loss: 0.064401\tBest loss: 0.027679\tAccuracy: 98.94%\n", + "56\tValidation loss: 0.047686\tBest loss: 0.027679\tAccuracy: 98.83%\n", + "57\tValidation loss: 0.049083\tBest loss: 0.027679\tAccuracy: 98.98%\n", + "58\tValidation loss: 0.057310\tBest loss: 0.027679\tAccuracy: 99.10%\n", + "59\tValidation loss: 0.043757\tBest loss: 0.027679\tAccuracy: 99.14%\n", + "60\tValidation loss: 0.058742\tBest loss: 0.027679\tAccuracy: 99.02%\n", + "61\tValidation loss: 0.055049\tBest loss: 0.027679\tAccuracy: 99.06%\n", + "62\tValidation loss: 0.039837\tBest loss: 0.027679\tAccuracy: 99.18%\n", + "63\tValidation loss: 0.057108\tBest loss: 0.027679\tAccuracy: 99.06%\n", + "64\tValidation loss: 0.043212\tBest loss: 0.027679\tAccuracy: 98.98%\n", + "65\tValidation loss: 0.046874\tBest loss: 0.027679\tAccuracy: 99.18%\n", + "66\tValidation loss: 0.052819\tBest loss: 0.027679\tAccuracy: 99.10%\n", + "67\tValidation loss: 0.045977\tBest loss: 0.027679\tAccuracy: 99.14%\n", + "68\tValidation loss: 0.053290\tBest loss: 0.027679\tAccuracy: 99.10%\n", + "69\tValidation loss: 0.052941\tBest loss: 0.027679\tAccuracy: 99.06%\n", + "Early stopping!\n", + "[CV] activation=, n_neurons=70, learning_rate=0.01, batch_norm_momentum=0.99, batch_size=50, total= 2.7min\n", + "[CV] activation=, n_neurons=70, learning_rate=0.01, batch_norm_momentum=0.99, batch_size=50 \n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 2.7min remaining: 0.0s\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0\tValidation loss: 0.144984\tBest loss: 0.144984\tAccuracy: 96.40%\n", + "1\tValidation loss: 0.067873\tBest loss: 0.067873\tAccuracy: 98.44%\n", + "2\tValidation loss: 0.091854\tBest loss: 0.067873\tAccuracy: 97.30%\n", + "3\tValidation loss: 0.074647\tBest loss: 0.067873\tAccuracy: 98.05%\n", + "4\tValidation loss: 0.053722\tBest loss: 0.053722\tAccuracy: 98.48%\n", + "5\tValidation loss: 0.049216\tBest loss: 0.049216\tAccuracy: 98.44%\n", + "6\tValidation loss: 0.057619\tBest loss: 0.049216\tAccuracy: 98.48%\n", + "7\tValidation loss: 0.045842\tBest loss: 0.045842\tAccuracy: 98.75%\n", + "8\tValidation loss: 0.042398\tBest loss: 0.042398\tAccuracy: 98.63%\n", + "9\tValidation loss: 0.052629\tBest loss: 0.042398\tAccuracy: 98.63%\n", + "10\tValidation loss: 0.056892\tBest loss: 0.042398\tAccuracy: 98.63%\n", + "11\tValidation loss: 0.051838\tBest loss: 0.042398\tAccuracy: 98.75%\n", + "12\tValidation loss: 0.042647\tBest loss: 0.042398\tAccuracy: 98.67%\n", + "13\tValidation loss: 0.061297\tBest loss: 0.042398\tAccuracy: 98.59%\n", + "14\tValidation loss: 0.049706\tBest loss: 0.042398\tAccuracy: 98.87%\n", + "15\tValidation loss: 0.061934\tBest loss: 0.042398\tAccuracy: 98.79%\n", + "16\tValidation loss: 0.049027\tBest loss: 0.042398\tAccuracy: 98.87%\n", + "17\tValidation loss: 0.052187\tBest loss: 0.042398\tAccuracy: 98.79%\n", + "18\tValidation loss: 0.052031\tBest loss: 0.042398\tAccuracy: 98.94%\n", + "[...and much later...]\n", + "13\tValidation loss: 0.043686\tBest loss: 0.040332\tAccuracy: 99.02%\n", + "14\tValidation loss: 0.046940\tBest loss: 0.040332\tAccuracy: 99.18%\n", + "15\tValidation loss: 0.045355\tBest loss: 0.040332\tAccuracy: 99.14%\n", + "16\tValidation loss: 0.084697\tBest loss: 0.040332\tAccuracy: 98.87%\n", + "17\tValidation loss: 0.123538\tBest loss: 0.040332\tAccuracy: 97.81%\n", + "18\tValidation loss: 0.296928\tBest loss: 0.040332\tAccuracy: 97.50%\n", + "19\tValidation loss: 0.053660\tBest loss: 0.040332\tAccuracy: 98.91%\n", + "20\tValidation loss: 0.045684\tBest loss: 0.040332\tAccuracy: 98.94%\n", + "21\tValidation loss: 0.051971\tBest loss: 0.040332\tAccuracy: 99.14%\n", + "22\tValidation loss: 0.071830\tBest loss: 0.040332\tAccuracy: 99.06%\n", + "23\tValidation loss: 0.069619\tBest loss: 0.040332\tAccuracy: 98.79%\n", + "24\tValidation loss: 0.086642\tBest loss: 0.040332\tAccuracy: 98.71%\n", + "25\tValidation loss: 0.072563\tBest loss: 0.040332\tAccuracy: 98.83%\n", + "26\tValidation loss: 0.058974\tBest loss: 0.040332\tAccuracy: 99.06%\n", + "27\tValidation loss: 0.048388\tBest loss: 0.040332\tAccuracy: 98.98%\n", + "28\tValidation loss: 0.054847\tBest loss: 0.040332\tAccuracy: 99.06%\n", + "29\tValidation loss: 0.077242\tBest loss: 0.040332\tAccuracy: 98.91%\n", + "30\tValidation loss: 0.556978\tBest loss: 0.040332\tAccuracy: 95.54%\n", + "Early stopping!\n", + "[CV] activation=, n_neurons=140, learning_rate=0.05, batch_norm_momentum=0.99, batch_size=50, total= 1.9min\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[Parallel(n_jobs=1)]: Done 150 out of 150 | elapsed: 355.8min finished\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0\tValidation loss: 0.076371\tBest loss: 0.076371\tAccuracy: 97.85%\n", + "1\tValidation loss: 0.049312\tBest loss: 0.049312\tAccuracy: 98.63%\n", + "2\tValidation loss: 0.033071\tBest loss: 0.033071\tAccuracy: 98.94%\n", + "3\tValidation loss: 0.027357\tBest loss: 0.027357\tAccuracy: 99.10%\n", + "4\tValidation loss: 0.028748\tBest loss: 0.027357\tAccuracy: 99.26%\n", + "5\tValidation loss: 0.036602\tBest loss: 0.027357\tAccuracy: 98.94%\n", + "6\tValidation loss: 0.048089\tBest loss: 0.027357\tAccuracy: 98.94%\n", + "7\tValidation loss: 0.030332\tBest loss: 0.027357\tAccuracy: 99.30%\n", + "8\tValidation loss: 0.029336\tBest loss: 0.027357\tAccuracy: 99.22%\n", + "9\tValidation loss: 0.033328\tBest loss: 0.027357\tAccuracy: 99.26%\n", + "10\tValidation loss: 0.041745\tBest loss: 0.027357\tAccuracy: 98.98%\n", + "11\tValidation loss: 0.048739\tBest loss: 0.027357\tAccuracy: 98.75%\n", + "12\tValidation loss: 0.049520\tBest loss: 0.027357\tAccuracy: 98.94%\n", + "13\tValidation loss: 0.034222\tBest loss: 0.027357\tAccuracy: 99.18%\n", + "14\tValidation loss: 0.040270\tBest loss: 0.027357\tAccuracy: 99.34%\n", + "15\tValidation loss: 0.033074\tBest loss: 0.027357\tAccuracy: 99.37%\n", + "16\tValidation loss: 0.035130\tBest loss: 0.027357\tAccuracy: 99.06%\n", + "17\tValidation loss: 0.031875\tBest loss: 0.027357\tAccuracy: 99.18%\n", + "18\tValidation loss: 0.034898\tBest loss: 0.027357\tAccuracy: 99.37%\n", + "19\tValidation loss: 0.019222\tBest loss: 0.019222\tAccuracy: 99.53%\n", + "20\tValidation loss: 0.043814\tBest loss: 0.019222\tAccuracy: 99.37%\n", + "21\tValidation loss: 0.028773\tBest loss: 0.019222\tAccuracy: 99.34%\n", + "22\tValidation loss: 0.024850\tBest loss: 0.019222\tAccuracy: 99.45%\n", + "23\tValidation loss: 0.021789\tBest loss: 0.019222\tAccuracy: 99.45%\n", + "24\tValidation loss: 0.028846\tBest loss: 0.019222\tAccuracy: 99.37%\n", + "25\tValidation loss: 0.064211\tBest loss: 0.019222\tAccuracy: 98.98%\n", + "26\tValidation loss: 0.024425\tBest loss: 0.019222\tAccuracy: 99.49%\n", + "27\tValidation loss: 0.035453\tBest loss: 0.019222\tAccuracy: 99.22%\n", + "28\tValidation loss: 0.023940\tBest loss: 0.019222\tAccuracy: 99.37%\n", + "29\tValidation loss: 0.041495\tBest loss: 0.019222\tAccuracy: 99.18%\n", + "30\tValidation loss: 0.028030\tBest loss: 0.019222\tAccuracy: 99.37%\n", + "31\tValidation loss: 0.028003\tBest loss: 0.019222\tAccuracy: 99.49%\n", + "32\tValidation loss: 0.026579\tBest loss: 0.019222\tAccuracy: 99.45%\n", + "33\tValidation loss: 0.037838\tBest loss: 0.019222\tAccuracy: 98.91%\n", + "34\tValidation loss: 0.026082\tBest loss: 0.019222\tAccuracy: 99.49%\n", + "35\tValidation loss: 0.031529\tBest loss: 0.019222\tAccuracy: 99.34%\n", + "36\tValidation loss: 0.028220\tBest loss: 0.019222\tAccuracy: 99.18%\n", + "37\tValidation loss: 0.038546\tBest loss: 0.019222\tAccuracy: 99.10%\n", + "38\tValidation loss: 0.041586\tBest loss: 0.019222\tAccuracy: 98.75%\n", + "39\tValidation loss: 0.038835\tBest loss: 0.019222\tAccuracy: 99.41%\n", + "40\tValidation loss: 0.042555\tBest loss: 0.019222\tAccuracy: 99.14%\n", + "Early stopping!\n" + ] + }, + { + "data": { + "text/plain": [ + "RandomizedSearchCV(cv=None, error_score='raise',\n", + " estimator=DNNClassifier(activation=,\n", + " batch_norm_momentum=None, batch_size=20, dropout_rate=None,\n", + " initializer=._initializer at 0x7fd9d5e628c8>,\n", + " learning_rate=0.01, n_hidden_layers=5, n_neurons=100,\n", + " optimizer_class=,\n", + " random_state=42),\n", + " fit_params={'y_valid': array([0, 4, ..., 1, 2], dtype=uint8), 'X_valid': array([[ 0., 0., ..., 0., 0.],\n", + " [ 0., 0., ..., 0., 0.],\n", + " ...,\n", + " [ 0., 0., ..., 0., 0.],\n", + " [ 0., 0., ..., 0., 0.]], dtype=float32), 'n_epochs': 1000},\n", + " iid=True, n_iter=50, n_jobs=1,\n", + " param_distributions={'batch_norm_momentum': [0.9, 0.95, 0.98, 0.99, 0.999], 'n_neurons': [10, 30, 50, 70, 90, 100, 120, 140, 160], 'learning_rate': [0.01, 0.02, 0.05, 0.1], 'activation': [, , .parametrized_leaky_relu at 0x7fd9d19e3bf8>, .parametrized_leaky_relu at 0x7fd9d19e3a60>], 'batch_size': [10, 50, 100, 500]},\n", + " pre_dispatch='2*n_jobs', random_state=42, refit=True,\n", + " return_train_score=True, scoring=None, verbose=2)" + ] + }, + "execution_count": 130, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "from sklearn.model_selection import RandomizedSearchCV\n", "\n", @@ -3487,7 +5239,22 @@ "cell_type": "code", "execution_count": 131, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "{'activation': ,\n", + " 'batch_norm_momentum': 0.98,\n", + " 'batch_size': 100,\n", + " 'learning_rate': 0.01,\n", + " 'n_neurons': 160}" + ] + }, + "execution_count": 131, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "rnd_search_bn.best_params_" ] @@ -3496,7 +5263,18 @@ "cell_type": "code", "execution_count": 132, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "0.99396769799571905" + ] + }, + "execution_count": 132, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "y_pred = rnd_search_bn.predict(X_test1)\n", "accuracy_score(y_test1, y_pred)" @@ -3534,7 +5312,18 @@ "cell_type": "code", "execution_count": 133, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "0.99914401883158566" + ] + }, + "execution_count": 133, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "y_pred = dnn_clf.predict(X_train1)\n", "accuracy_score(y_train1, y_pred)" @@ -3551,7 +5340,75 @@ "cell_type": "code", "execution_count": 134, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0\tValidation loss: 0.162759\tBest loss: 0.162759\tAccuracy: 95.15%\n", + "1\tValidation loss: 0.120510\tBest loss: 0.120510\tAccuracy: 96.64%\n", + "2\tValidation loss: 0.110715\tBest loss: 0.110715\tAccuracy: 96.91%\n", + "3\tValidation loss: 0.104193\tBest loss: 0.104193\tAccuracy: 97.22%\n", + "4\tValidation loss: 0.103560\tBest loss: 0.103560\tAccuracy: 97.81%\n", + "5\tValidation loss: 0.087045\tBest loss: 0.087045\tAccuracy: 97.89%\n", + "6\tValidation loss: 0.087227\tBest loss: 0.087045\tAccuracy: 97.65%\n", + "7\tValidation loss: 0.079840\tBest loss: 0.079840\tAccuracy: 98.16%\n", + "8\tValidation loss: 0.083102\tBest loss: 0.079840\tAccuracy: 97.50%\n", + "9\tValidation loss: 0.076794\tBest loss: 0.076794\tAccuracy: 98.01%\n", + "10\tValidation loss: 0.074914\tBest loss: 0.074914\tAccuracy: 97.93%\n", + "11\tValidation loss: 0.073794\tBest loss: 0.073794\tAccuracy: 98.12%\n", + "12\tValidation loss: 0.079777\tBest loss: 0.073794\tAccuracy: 97.89%\n", + "13\tValidation loss: 0.080277\tBest loss: 0.073794\tAccuracy: 97.54%\n", + "14\tValidation loss: 0.072409\tBest loss: 0.072409\tAccuracy: 98.08%\n", + "15\tValidation loss: 0.071988\tBest loss: 0.071988\tAccuracy: 98.12%\n", + "16\tValidation loss: 0.074609\tBest loss: 0.071988\tAccuracy: 97.93%\n", + "17\tValidation loss: 0.069488\tBest loss: 0.069488\tAccuracy: 98.28%\n", + "18\tValidation loss: 0.080863\tBest loss: 0.069488\tAccuracy: 98.40%\n", + "19\tValidation loss: 0.074966\tBest loss: 0.069488\tAccuracy: 98.20%\n", + "20\tValidation loss: 0.071082\tBest loss: 0.069488\tAccuracy: 98.12%\n", + "21\tValidation loss: 0.070138\tBest loss: 0.069488\tAccuracy: 98.20%\n", + "22\tValidation loss: 0.066032\tBest loss: 0.066032\tAccuracy: 98.28%\n", + "23\tValidation loss: 0.061130\tBest loss: 0.061130\tAccuracy: 98.36%\n", + "24\tValidation loss: 0.067107\tBest loss: 0.061130\tAccuracy: 98.16%\n", + "25\tValidation loss: 0.071372\tBest loss: 0.061130\tAccuracy: 98.16%\n", + "26\tValidation loss: 0.068535\tBest loss: 0.061130\tAccuracy: 98.36%\n", + "27\tValidation loss: 0.065336\tBest loss: 0.061130\tAccuracy: 98.48%\n", + "28\tValidation loss: 0.066783\tBest loss: 0.061130\tAccuracy: 98.40%\n", + "29\tValidation loss: 0.092769\tBest loss: 0.061130\tAccuracy: 97.77%\n", + "30\tValidation loss: 0.075746\tBest loss: 0.061130\tAccuracy: 98.01%\n", + "31\tValidation loss: 0.084024\tBest loss: 0.061130\tAccuracy: 97.81%\n", + "32\tValidation loss: 0.116428\tBest loss: 0.061130\tAccuracy: 98.44%\n", + "33\tValidation loss: 0.079498\tBest loss: 0.061130\tAccuracy: 97.89%\n", + "34\tValidation loss: 0.078189\tBest loss: 0.061130\tAccuracy: 97.97%\n", + "35\tValidation loss: 0.083723\tBest loss: 0.061130\tAccuracy: 97.81%\n", + "36\tValidation loss: 0.088210\tBest loss: 0.061130\tAccuracy: 97.19%\n", + "37\tValidation loss: 0.080040\tBest loss: 0.061130\tAccuracy: 97.93%\n", + "38\tValidation loss: 0.086932\tBest loss: 0.061130\tAccuracy: 97.89%\n", + "39\tValidation loss: 0.240580\tBest loss: 0.061130\tAccuracy: 91.67%\n", + "40\tValidation loss: 0.166662\tBest loss: 0.061130\tAccuracy: 94.29%\n", + "41\tValidation loss: 0.125562\tBest loss: 0.061130\tAccuracy: 97.15%\n", + "42\tValidation loss: 0.124890\tBest loss: 0.061130\tAccuracy: 95.82%\n", + "43\tValidation loss: 0.127020\tBest loss: 0.061130\tAccuracy: 96.76%\n", + "44\tValidation loss: 0.121540\tBest loss: 0.061130\tAccuracy: 96.05%\n", + "Early stopping!\n" + ] + }, + { + "data": { + "text/plain": [ + "DNNClassifier(activation=.parametrized_leaky_relu at 0x7fd9b2368d08>,\n", + " batch_norm_momentum=None, batch_size=500, dropout_rate=0.5,\n", + " initializer=._initializer at 0x7fd9d5e628c8>,\n", + " learning_rate=0.01, n_hidden_layers=5, n_neurons=90,\n", + " optimizer_class=,\n", + " random_state=42)" + ] + }, + "execution_count": 134, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "dnn_clf_dropout = DNNClassifier(activation=leaky_relu(alpha=0.1), batch_size=500, learning_rate=0.01,\n", " n_neurons=90, random_state=42,\n", @@ -3577,7 +5434,18 @@ "cell_type": "code", "execution_count": 135, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "0.98657326328079398" + ] + }, + "execution_count": 135, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "y_pred = dnn_clf_dropout.predict(X_test1)\n", "accuracy_score(y_test1, y_pred)" @@ -3594,7 +5462,255 @@ "cell_type": "code", "execution_count": 136, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Fitting 3 folds for each of 50 candidates, totalling 150 fits\n", + "[CV] dropout_rate=0.5, n_neurons=70, learning_rate=0.01, activation=, batch_size=100 \n", + "0\tValidation loss: 0.355079\tBest loss: 0.355079\tAccuracy: 91.44%\n", + "1\tValidation loss: 0.280624\tBest loss: 0.280624\tAccuracy: 94.10%\n", + "2\tValidation loss: 0.279819\tBest loss: 0.279819\tAccuracy: 92.77%\n", + "3\tValidation loss: 0.223614\tBest loss: 0.223614\tAccuracy: 94.10%\n", + "4\tValidation loss: 0.199802\tBest loss: 0.199802\tAccuracy: 95.11%\n", + "5\tValidation loss: 0.214481\tBest loss: 0.199802\tAccuracy: 95.47%\n", + "6\tValidation loss: 0.216195\tBest loss: 0.199802\tAccuracy: 95.78%\n", + "7\tValidation loss: 0.209172\tBest loss: 0.199802\tAccuracy: 94.80%\n", + "8\tValidation loss: 0.182841\tBest loss: 0.182841\tAccuracy: 95.70%\n", + "9\tValidation loss: 0.214252\tBest loss: 0.182841\tAccuracy: 95.82%\n", + "10\tValidation loss: 0.198762\tBest loss: 0.182841\tAccuracy: 95.62%\n", + "11\tValidation loss: 0.186415\tBest loss: 0.182841\tAccuracy: 95.82%\n", + "12\tValidation loss: 0.222924\tBest loss: 0.182841\tAccuracy: 96.05%\n", + "13\tValidation loss: 0.199636\tBest loss: 0.182841\tAccuracy: 95.97%\n", + "14\tValidation loss: 0.214436\tBest loss: 0.182841\tAccuracy: 95.97%\n", + "15\tValidation loss: 0.213507\tBest loss: 0.182841\tAccuracy: 95.47%\n", + "16\tValidation loss: 0.191497\tBest loss: 0.182841\tAccuracy: 95.78%\n", + "17\tValidation loss: 0.179503\tBest loss: 0.179503\tAccuracy: 95.93%\n", + "18\tValidation loss: 0.210343\tBest loss: 0.179503\tAccuracy: 95.74%\n", + "19\tValidation loss: 0.212626\tBest loss: 0.179503\tAccuracy: 95.27%\n", + "20\tValidation loss: 0.187110\tBest loss: 0.179503\tAccuracy: 96.09%\n", + "21\tValidation loss: 0.175171\tBest loss: 0.175171\tAccuracy: 95.78%\n", + "22\tValidation loss: 0.217172\tBest loss: 0.175171\tAccuracy: 95.66%\n", + "23\tValidation loss: 0.181060\tBest loss: 0.175171\tAccuracy: 96.44%\n", + "24\tValidation loss: 0.163630\tBest loss: 0.163630\tAccuracy: 95.93%\n", + "25\tValidation loss: 0.225873\tBest loss: 0.163630\tAccuracy: 95.58%\n", + "26\tValidation loss: 0.204975\tBest loss: 0.163630\tAccuracy: 95.66%\n", + "27\tValidation loss: 0.183588\tBest loss: 0.163630\tAccuracy: 95.97%\n", + "28\tValidation loss: 0.231080\tBest loss: 0.163630\tAccuracy: 95.11%\n", + "29\tValidation loss: 0.204342\tBest loss: 0.163630\tAccuracy: 95.74%\n", + "30\tValidation loss: 0.183963\tBest loss: 0.163630\tAccuracy: 95.93%\n", + "31\tValidation loss: 0.200975\tBest loss: 0.163630\tAccuracy: 95.23%\n", + "32\tValidation loss: 0.211165\tBest loss: 0.163630\tAccuracy: 95.23%\n", + "33\tValidation loss: 0.217777\tBest loss: 0.163630\tAccuracy: 95.07%\n", + "34\tValidation loss: 0.193184\tBest loss: 0.163630\tAccuracy: 95.39%\n", + "35\tValidation loss: 0.203809\tBest loss: 0.163630\tAccuracy: 95.58%\n", + "36\tValidation loss: 0.221673\tBest loss: 0.163630\tAccuracy: 94.57%\n", + "37\tValidation loss: 0.215750\tBest loss: 0.163630\tAccuracy: 95.39%\n", + "38\tValidation loss: 0.189653\tBest loss: 0.163630\tAccuracy: 96.09%\n", + "39\tValidation loss: 0.191333\tBest loss: 0.163630\tAccuracy: 95.19%\n", + "40\tValidation loss: 0.207714\tBest loss: 0.163630\tAccuracy: 96.01%\n", + "41\tValidation loss: 0.174490\tBest loss: 0.163630\tAccuracy: 95.39%\n", + "42\tValidation loss: 0.177445\tBest loss: 0.163630\tAccuracy: 95.82%\n", + "43\tValidation loss: 0.166708\tBest loss: 0.163630\tAccuracy: 96.09%\n", + "44\tValidation loss: 0.190829\tBest loss: 0.163630\tAccuracy: 95.70%\n", + "45\tValidation loss: 0.225985\tBest loss: 0.163630\tAccuracy: 96.25%\n", + "Early stopping!\n", + "[CV] dropout_rate=0.5, n_neurons=70, learning_rate=0.01, activation=, batch_size=100, total= 39.0s\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 39.1s remaining: 0.0s\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[CV] dropout_rate=0.5, n_neurons=70, learning_rate=0.01, activation=, batch_size=100 \n", + "0\tValidation loss: 0.748480\tBest loss: 0.748480\tAccuracy: 57.70%\n", + "1\tValidation loss: 0.516088\tBest loss: 0.516088\tAccuracy: 78.50%\n", + "2\tValidation loss: 0.448866\tBest loss: 0.448866\tAccuracy: 78.89%\n", + "3\tValidation loss: 0.435606\tBest loss: 0.435606\tAccuracy: 78.54%\n", + "4\tValidation loss: 0.435243\tBest loss: 0.435243\tAccuracy: 79.40%\n", + "5\tValidation loss: 0.450605\tBest loss: 0.435243\tAccuracy: 78.42%\n", + "6\tValidation loss: 0.430706\tBest loss: 0.430706\tAccuracy: 78.62%\n", + "7\tValidation loss: 0.449289\tBest loss: 0.430706\tAccuracy: 78.30%\n", + "8\tValidation loss: 0.413226\tBest loss: 0.413226\tAccuracy: 79.05%\n", + "9\tValidation loss: 0.436053\tBest loss: 0.413226\tAccuracy: 78.46%\n", + "10\tValidation loss: 0.459932\tBest loss: 0.413226\tAccuracy: 79.24%\n", + "11\tValidation loss: 0.424138\tBest loss: 0.413226\tAccuracy: 79.24%\n", + "12\tValidation loss: 0.409538\tBest loss: 0.409538\tAccuracy: 79.55%\n", + "13\tValidation loss: 0.416324\tBest loss: 0.409538\tAccuracy: 75.41%\n", + "14\tValidation loss: 0.440273\tBest loss: 0.409538\tAccuracy: 78.46%\n", + "15\tValidation loss: 0.435736\tBest loss: 0.409538\tAccuracy: 79.05%\n", + "16\tValidation loss: 0.428412\tBest loss: 0.409538\tAccuracy: 79.20%\n", + "17\tValidation loss: 0.450156\tBest loss: 0.409538\tAccuracy: 80.02%\n", + "18\tValidation loss: 0.421057\tBest loss: 0.409538\tAccuracy: 79.24%\n", + "19\tValidation loss: 0.442284\tBest loss: 0.409538\tAccuracy: 79.01%\n", + "20\tValidation loss: 0.426907\tBest loss: 0.409538\tAccuracy: 79.16%\n", + "21\tValidation loss: 0.439567\tBest loss: 0.409538\tAccuracy: 79.05%\n", + "22\tValidation loss: 0.452601\tBest loss: 0.409538\tAccuracy: 79.67%\n", + "23\tValidation loss: 0.424887\tBest loss: 0.409538\tAccuracy: 79.09%\n", + "24\tValidation loss: 0.441096\tBest loss: 0.409538\tAccuracy: 78.97%\n", + "25\tValidation loss: 0.417390\tBest loss: 0.409538\tAccuracy: 78.89%\n", + "26\tValidation loss: 0.418550\tBest loss: 0.409538\tAccuracy: 79.05%\n", + "27\tValidation loss: 0.426065\tBest loss: 0.409538\tAccuracy: 78.66%\n", + "28\tValidation loss: 0.413968\tBest loss: 0.409538\tAccuracy: 79.36%\n", + "29\tValidation loss: 0.425434\tBest loss: 0.409538\tAccuracy: 79.24%\n", + "30\tValidation loss: 0.455391\tBest loss: 0.409538\tAccuracy: 74.71%\n", + "31\tValidation loss: 0.429498\tBest loss: 0.409538\tAccuracy: 79.20%\n", + "32\tValidation loss: 0.427383\tBest loss: 0.409538\tAccuracy: 79.52%\n", + "33\tValidation loss: 0.422621\tBest loss: 0.409538\tAccuracy: 78.62%\n", + "Early stopping!\n", + "[CV] dropout_rate=0.5, n_neurons=70, learning_rate=0.01, activation=, batch_size=100, total= 27.4s\n", + "[CV] dropout_rate=0.5, n_neurons=70, learning_rate=0.01, activation=, batch_size=100 \n", + "0\tValidation loss: 0.497714\tBest loss: 0.497714\tAccuracy: 86.71%\n", + "1\tValidation loss: 0.248258\tBest loss: 0.248258\tAccuracy: 93.51%\n", + "2\tValidation loss: 0.279785\tBest loss: 0.248258\tAccuracy: 93.71%\n", + "3\tValidation loss: 0.248663\tBest loss: 0.248258\tAccuracy: 94.61%\n", + "4\tValidation loss: 0.269139\tBest loss: 0.248258\tAccuracy: 94.76%\n", + "5\tValidation loss: 0.188808\tBest loss: 0.188808\tAccuracy: 95.39%\n", + "6\tValidation loss: 0.196049\tBest loss: 0.188808\tAccuracy: 95.58%\n", + "7\tValidation loss: 0.204966\tBest loss: 0.188808\tAccuracy: 95.15%\n", + "8\tValidation loss: 0.238414\tBest loss: 0.188808\tAccuracy: 94.61%\n", + "9\tValidation loss: 0.192095\tBest loss: 0.188808\tAccuracy: 95.97%\n", + "[...and much later...]\n", + "19\tValidation loss: 1.939112\tBest loss: 1.619874\tAccuracy: 22.01%\n", + "20\tValidation loss: 1.825761\tBest loss: 1.619874\tAccuracy: 19.27%\n", + "21\tValidation loss: 1.732937\tBest loss: 1.619874\tAccuracy: 22.01%\n", + "22\tValidation loss: 1.832995\tBest loss: 1.619874\tAccuracy: 20.91%\n", + "23\tValidation loss: 1.659557\tBest loss: 1.619874\tAccuracy: 20.91%\n", + "24\tValidation loss: 1.828380\tBest loss: 1.619874\tAccuracy: 18.73%\n", + "25\tValidation loss: 1.719589\tBest loss: 1.619874\tAccuracy: 22.01%\n", + "26\tValidation loss: 1.842429\tBest loss: 1.619874\tAccuracy: 18.73%\n", + "27\tValidation loss: 1.717596\tBest loss: 1.619874\tAccuracy: 19.27%\n", + "28\tValidation loss: 1.863441\tBest loss: 1.619874\tAccuracy: 19.08%\n", + "29\tValidation loss: 1.952335\tBest loss: 1.619874\tAccuracy: 19.08%\n", + "30\tValidation loss: 1.853776\tBest loss: 1.619874\tAccuracy: 20.91%\n", + "31\tValidation loss: 1.894134\tBest loss: 1.619874\tAccuracy: 22.01%\n", + "32\tValidation loss: 1.711688\tBest loss: 1.619874\tAccuracy: 19.08%\n", + "33\tValidation loss: 1.651240\tBest loss: 1.619874\tAccuracy: 18.73%\n", + "34\tValidation loss: 1.760639\tBest loss: 1.619874\tAccuracy: 20.91%\n", + "35\tValidation loss: 1.667938\tBest loss: 1.619874\tAccuracy: 22.01%\n", + "36\tValidation loss: 1.641116\tBest loss: 1.619874\tAccuracy: 20.91%\n", + "37\tValidation loss: 1.694960\tBest loss: 1.619874\tAccuracy: 19.08%\n", + "38\tValidation loss: 1.816517\tBest loss: 1.619874\tAccuracy: 18.73%\n", + "39\tValidation loss: 1.647246\tBest loss: 1.619874\tAccuracy: 18.73%\n", + "Early stopping!\n", + "[CV] dropout_rate=0.5, n_neurons=140, learning_rate=0.05, activation=, batch_size=100, total= 1.0min\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[Parallel(n_jobs=1)]: Done 150 out of 150 | elapsed: 130.6min finished\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0\tValidation loss: 0.165751\tBest loss: 0.165751\tAccuracy: 95.47%\n", + "1\tValidation loss: 0.111834\tBest loss: 0.111834\tAccuracy: 96.99%\n", + "2\tValidation loss: 0.102867\tBest loss: 0.102867\tAccuracy: 96.83%\n", + "3\tValidation loss: 0.089197\tBest loss: 0.089197\tAccuracy: 97.85%\n", + "4\tValidation loss: 0.093953\tBest loss: 0.089197\tAccuracy: 97.77%\n", + "5\tValidation loss: 0.079498\tBest loss: 0.079498\tAccuracy: 98.08%\n", + "6\tValidation loss: 0.081214\tBest loss: 0.079498\tAccuracy: 98.01%\n", + "7\tValidation loss: 0.086096\tBest loss: 0.079498\tAccuracy: 97.77%\n", + "8\tValidation loss: 0.074422\tBest loss: 0.074422\tAccuracy: 97.73%\n", + "9\tValidation loss: 0.079650\tBest loss: 0.074422\tAccuracy: 97.89%\n", + "10\tValidation loss: 0.077278\tBest loss: 0.074422\tAccuracy: 97.77%\n", + "11\tValidation loss: 0.077608\tBest loss: 0.074422\tAccuracy: 98.24%\n", + "12\tValidation loss: 0.074337\tBest loss: 0.074337\tAccuracy: 98.05%\n", + "13\tValidation loss: 0.066028\tBest loss: 0.066028\tAccuracy: 98.28%\n", + "14\tValidation loss: 0.072845\tBest loss: 0.066028\tAccuracy: 98.16%\n", + "15\tValidation loss: 0.066652\tBest loss: 0.066028\tAccuracy: 98.05%\n", + "16\tValidation loss: 0.065729\tBest loss: 0.065729\tAccuracy: 98.16%\n", + "17\tValidation loss: 0.061191\tBest loss: 0.061191\tAccuracy: 98.51%\n", + "18\tValidation loss: 0.062528\tBest loss: 0.061191\tAccuracy: 98.44%\n", + "19\tValidation loss: 0.065407\tBest loss: 0.061191\tAccuracy: 98.36%\n", + "20\tValidation loss: 0.065273\tBest loss: 0.061191\tAccuracy: 98.44%\n", + "21\tValidation loss: 0.061035\tBest loss: 0.061035\tAccuracy: 98.40%\n", + "22\tValidation loss: 0.056312\tBest loss: 0.056312\tAccuracy: 98.59%\n", + "23\tValidation loss: 0.069074\tBest loss: 0.056312\tAccuracy: 98.40%\n", + "24\tValidation loss: 0.057482\tBest loss: 0.056312\tAccuracy: 98.51%\n", + "25\tValidation loss: 0.068342\tBest loss: 0.056312\tAccuracy: 98.44%\n", + "26\tValidation loss: 0.063494\tBest loss: 0.056312\tAccuracy: 98.48%\n", + "27\tValidation loss: 0.057257\tBest loss: 0.056312\tAccuracy: 98.51%\n", + "28\tValidation loss: 0.058659\tBest loss: 0.056312\tAccuracy: 98.59%\n", + "29\tValidation loss: 0.059009\tBest loss: 0.056312\tAccuracy: 98.48%\n", + "30\tValidation loss: 0.058227\tBest loss: 0.056312\tAccuracy: 98.55%\n", + "31\tValidation loss: 0.062198\tBest loss: 0.056312\tAccuracy: 98.44%\n", + "32\tValidation loss: 0.058043\tBest loss: 0.056312\tAccuracy: 98.40%\n", + "33\tValidation loss: 0.055970\tBest loss: 0.055970\tAccuracy: 98.51%\n", + "34\tValidation loss: 0.060111\tBest loss: 0.055970\tAccuracy: 98.67%\n", + "35\tValidation loss: 0.058786\tBest loss: 0.055970\tAccuracy: 98.44%\n", + "36\tValidation loss: 0.059944\tBest loss: 0.055970\tAccuracy: 98.32%\n", + "37\tValidation loss: 0.058087\tBest loss: 0.055970\tAccuracy: 98.63%\n", + "38\tValidation loss: 0.063003\tBest loss: 0.055970\tAccuracy: 98.36%\n", + "39\tValidation loss: 0.052073\tBest loss: 0.052073\tAccuracy: 98.67%\n", + "40\tValidation loss: 0.058115\tBest loss: 0.052073\tAccuracy: 98.40%\n", + "41\tValidation loss: 0.059997\tBest loss: 0.052073\tAccuracy: 98.63%\n", + "42\tValidation loss: 0.052416\tBest loss: 0.052073\tAccuracy: 98.75%\n", + "43\tValidation loss: 0.053840\tBest loss: 0.052073\tAccuracy: 98.59%\n", + "44\tValidation loss: 0.054563\tBest loss: 0.052073\tAccuracy: 98.67%\n", + "45\tValidation loss: 0.049410\tBest loss: 0.049410\tAccuracy: 98.55%\n", + "46\tValidation loss: 0.057060\tBest loss: 0.049410\tAccuracy: 98.24%\n", + "47\tValidation loss: 0.062434\tBest loss: 0.049410\tAccuracy: 98.48%\n", + "48\tValidation loss: 0.054523\tBest loss: 0.049410\tAccuracy: 98.59%\n", + "49\tValidation loss: 0.052774\tBest loss: 0.049410\tAccuracy: 98.36%\n", + "50\tValidation loss: 0.056562\tBest loss: 0.049410\tAccuracy: 98.32%\n", + "51\tValidation loss: 0.060280\tBest loss: 0.049410\tAccuracy: 98.51%\n", + "52\tValidation loss: 0.055685\tBest loss: 0.049410\tAccuracy: 98.55%\n", + "53\tValidation loss: 0.056077\tBest loss: 0.049410\tAccuracy: 98.44%\n", + "54\tValidation loss: 0.057951\tBest loss: 0.049410\tAccuracy: 98.44%\n", + "55\tValidation loss: 0.056315\tBest loss: 0.049410\tAccuracy: 98.75%\n", + "56\tValidation loss: 0.055744\tBest loss: 0.049410\tAccuracy: 98.55%\n", + "57\tValidation loss: 0.054228\tBest loss: 0.049410\tAccuracy: 98.48%\n", + "58\tValidation loss: 0.057836\tBest loss: 0.049410\tAccuracy: 98.71%\n", + "59\tValidation loss: 0.053361\tBest loss: 0.049410\tAccuracy: 98.71%\n", + "60\tValidation loss: 0.056389\tBest loss: 0.049410\tAccuracy: 98.48%\n", + "61\tValidation loss: 0.061350\tBest loss: 0.049410\tAccuracy: 98.48%\n", + "62\tValidation loss: 0.052135\tBest loss: 0.049410\tAccuracy: 98.67%\n", + "63\tValidation loss: 0.053853\tBest loss: 0.049410\tAccuracy: 98.48%\n", + "64\tValidation loss: 0.056641\tBest loss: 0.049410\tAccuracy: 98.71%\n", + "65\tValidation loss: 0.052790\tBest loss: 0.049410\tAccuracy: 98.63%\n", + "66\tValidation loss: 0.053514\tBest loss: 0.049410\tAccuracy: 98.44%\n", + "Early stopping!\n" + ] + }, + { + "data": { + "text/plain": [ + "RandomizedSearchCV(cv=None, error_score='raise',\n", + " estimator=DNNClassifier(activation=,\n", + " batch_norm_momentum=None, batch_size=20, dropout_rate=None,\n", + " initializer=._initializer at 0x7fd9d5e628c8>,\n", + " learning_rate=0.01, n_hidden_layers=5, n_neurons=100,\n", + " optimizer_class=,\n", + " random_state=42),\n", + " fit_params={'y_valid': array([0, 4, ..., 1, 2], dtype=uint8), 'X_valid': array([[ 0., 0., ..., 0., 0.],\n", + " [ 0., 0., ..., 0., 0.],\n", + " ...,\n", + " [ 0., 0., ..., 0., 0.],\n", + " [ 0., 0., ..., 0., 0.]], dtype=float32), 'n_epochs': 1000},\n", + " iid=True, n_iter=50, n_jobs=1,\n", + " param_distributions={'dropout_rate': [0.2, 0.3, 0.4, 0.5, 0.6], 'n_neurons': [10, 30, 50, 70, 90, 100, 120, 140, 160], 'learning_rate': [0.01, 0.02, 0.05, 0.1], 'activation': [, , .parametrized_leaky_relu at 0x7fd9b2368950>, .parametrized_leaky_relu at 0x7fd9b23687b8>], 'batch_size': [10, 50, 100, 500]},\n", + " pre_dispatch='2*n_jobs', random_state=42, refit=True,\n", + " return_train_score=True, scoring=None, verbose=2)" + ] + }, + "execution_count": 136, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "from sklearn.model_selection import RandomizedSearchCV\n", "\n", @@ -3619,7 +5735,22 @@ "cell_type": "code", "execution_count": 137, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "{'activation': .parametrized_leaky_relu>,\n", + " 'batch_size': 500,\n", + " 'dropout_rate': 0.4,\n", + " 'learning_rate': 0.01,\n", + " 'n_neurons': 50}" + ] + }, + "execution_count": 137, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "rnd_search_dropout.best_params_" ] @@ -3628,7 +5759,18 @@ "cell_type": "code", "execution_count": 138, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "0.98812998637867289" + ] + }, + "execution_count": 138, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "y_pred = rnd_search_dropout.predict(X_test1)\n", "accuracy_score(y_test1, y_pred)" @@ -3821,7 +5963,68 @@ "cell_type": "code", "execution_count": 145, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Restoring parameters from ./my_best_mnist_model_0_to_4\n", + "0\tValidation loss: 0.967851\tBest loss: 0.967851\tAccuracy: 67.33%\n", + "1\tValidation loss: 0.861747\tBest loss: 0.861747\tAccuracy: 71.33%\n", + "2\tValidation loss: 0.777535\tBest loss: 0.777535\tAccuracy: 72.00%\n", + "3\tValidation loss: 0.699915\tBest loss: 0.699915\tAccuracy: 75.33%\n", + "4\tValidation loss: 0.786714\tBest loss: 0.699915\tAccuracy: 78.00%\n", + "5\tValidation loss: 0.735406\tBest loss: 0.699915\tAccuracy: 76.67%\n", + "6\tValidation loss: 0.732264\tBest loss: 0.699915\tAccuracy: 78.00%\n", + "7\tValidation loss: 0.691741\tBest loss: 0.691741\tAccuracy: 76.00%\n", + "8\tValidation loss: 0.672757\tBest loss: 0.672757\tAccuracy: 80.00%\n", + "9\tValidation loss: 0.666520\tBest loss: 0.666520\tAccuracy: 80.00%\n", + "10\tValidation loss: 0.639375\tBest loss: 0.639375\tAccuracy: 81.33%\n", + "11\tValidation loss: 0.645089\tBest loss: 0.639375\tAccuracy: 82.00%\n", + "12\tValidation loss: 0.646768\tBest loss: 0.639375\tAccuracy: 80.00%\n", + "13\tValidation loss: 0.623784\tBest loss: 0.623784\tAccuracy: 82.67%\n", + "14\tValidation loss: 0.663026\tBest loss: 0.623784\tAccuracy: 80.00%\n", + "15\tValidation loss: 0.704513\tBest loss: 0.623784\tAccuracy: 79.33%\n", + "16\tValidation loss: 0.684003\tBest loss: 0.623784\tAccuracy: 79.33%\n", + "17\tValidation loss: 0.658575\tBest loss: 0.623784\tAccuracy: 82.67%\n", + "18\tValidation loss: 0.669875\tBest loss: 0.623784\tAccuracy: 79.33%\n", + "19\tValidation loss: 0.664581\tBest loss: 0.623784\tAccuracy: 78.67%\n", + "20\tValidation loss: 0.653490\tBest loss: 0.623784\tAccuracy: 80.00%\n", + "21\tValidation loss: 0.707304\tBest loss: 0.623784\tAccuracy: 79.33%\n", + "22\tValidation loss: 0.706012\tBest loss: 0.623784\tAccuracy: 80.67%\n", + "23\tValidation loss: 0.681227\tBest loss: 0.623784\tAccuracy: 78.67%\n", + "24\tValidation loss: 0.786823\tBest loss: 0.623784\tAccuracy: 78.00%\n", + "25\tValidation loss: 0.686110\tBest loss: 0.623784\tAccuracy: 79.33%\n", + "26\tValidation loss: 0.675166\tBest loss: 0.623784\tAccuracy: 82.67%\n", + "27\tValidation loss: 0.667711\tBest loss: 0.623784\tAccuracy: 82.67%\n", + "28\tValidation loss: 0.612220\tBest loss: 0.612220\tAccuracy: 83.33%\n", + "29\tValidation loss: 0.701196\tBest loss: 0.612220\tAccuracy: 78.00%\n", + "30\tValidation loss: 0.687806\tBest loss: 0.612220\tAccuracy: 81.33%\n", + "31\tValidation loss: 0.776596\tBest loss: 0.612220\tAccuracy: 79.33%\n", + "32\tValidation loss: 0.674172\tBest loss: 0.612220\tAccuracy: 80.00%\n", + "33\tValidation loss: 0.719044\tBest loss: 0.612220\tAccuracy: 83.33%\n", + "34\tValidation loss: 0.856403\tBest loss: 0.612220\tAccuracy: 74.00%\n", + "35\tValidation loss: 0.744627\tBest loss: 0.612220\tAccuracy: 80.00%\n", + "36\tValidation loss: 0.779348\tBest loss: 0.612220\tAccuracy: 78.00%\n", + "37\tValidation loss: 0.763777\tBest loss: 0.612220\tAccuracy: 78.00%\n", + "38\tValidation loss: 0.727376\tBest loss: 0.612220\tAccuracy: 78.00%\n", + "39\tValidation loss: 0.823514\tBest loss: 0.612220\tAccuracy: 78.00%\n", + "40\tValidation loss: 0.725053\tBest loss: 0.612220\tAccuracy: 80.67%\n", + "41\tValidation loss: 0.678497\tBest loss: 0.612220\tAccuracy: 80.67%\n", + "42\tValidation loss: 0.709977\tBest loss: 0.612220\tAccuracy: 80.67%\n", + "43\tValidation loss: 0.737200\tBest loss: 0.612220\tAccuracy: 77.33%\n", + "44\tValidation loss: 0.757937\tBest loss: 0.612220\tAccuracy: 77.33%\n", + "45\tValidation loss: 0.732024\tBest loss: 0.612220\tAccuracy: 80.00%\n", + "46\tValidation loss: 0.756428\tBest loss: 0.612220\tAccuracy: 80.67%\n", + "47\tValidation loss: 0.757610\tBest loss: 0.612220\tAccuracy: 78.67%\n", + "48\tValidation loss: 0.844137\tBest loss: 0.612220\tAccuracy: 80.00%\n", + "Early stopping!\n", + "Total training time: 2.3s\n", + "INFO:tensorflow:Restoring parameters from ./my_mnist_model_5_to_9_five_frozen\n", + "Final test accuracy: 76.30%\n" + ] + } + ], "source": [ "import time\n", "\n", @@ -3917,7 +6120,54 @@ "cell_type": "code", "execution_count": 147, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Restoring parameters from ./my_best_mnist_model_0_to_4\n", + "0\tValidation loss: 1.109053\tBest loss: 1.109053\tAccuracy: 60.67%\n", + "1\tValidation loss: 0.813156\tBest loss: 0.813156\tAccuracy: 72.00%\n", + "2\tValidation loss: 0.755930\tBest loss: 0.755930\tAccuracy: 76.67%\n", + "3\tValidation loss: 0.744004\tBest loss: 0.744004\tAccuracy: 74.67%\n", + "4\tValidation loss: 0.685080\tBest loss: 0.685080\tAccuracy: 78.00%\n", + "5\tValidation loss: 0.702316\tBest loss: 0.685080\tAccuracy: 78.00%\n", + "6\tValidation loss: 0.646487\tBest loss: 0.646487\tAccuracy: 80.00%\n", + "7\tValidation loss: 0.686437\tBest loss: 0.646487\tAccuracy: 79.33%\n", + "8\tValidation loss: 0.750047\tBest loss: 0.646487\tAccuracy: 79.33%\n", + "9\tValidation loss: 0.688554\tBest loss: 0.646487\tAccuracy: 79.33%\n", + "10\tValidation loss: 0.785184\tBest loss: 0.646487\tAccuracy: 78.67%\n", + "11\tValidation loss: 0.634506\tBest loss: 0.634506\tAccuracy: 80.67%\n", + "12\tValidation loss: 0.656797\tBest loss: 0.634506\tAccuracy: 81.33%\n", + "13\tValidation loss: 0.645497\tBest loss: 0.634506\tAccuracy: 81.33%\n", + "14\tValidation loss: 0.618038\tBest loss: 0.618038\tAccuracy: 83.33%\n", + "15\tValidation loss: 0.641752\tBest loss: 0.618038\tAccuracy: 78.67%\n", + "16\tValidation loss: 0.645671\tBest loss: 0.618038\tAccuracy: 80.67%\n", + "17\tValidation loss: 0.654640\tBest loss: 0.618038\tAccuracy: 82.00%\n", + "18\tValidation loss: 0.670569\tBest loss: 0.618038\tAccuracy: 79.33%\n", + "19\tValidation loss: 0.670985\tBest loss: 0.618038\tAccuracy: 82.00%\n", + "20\tValidation loss: 0.659538\tBest loss: 0.618038\tAccuracy: 82.67%\n", + "21\tValidation loss: 0.622648\tBest loss: 0.618038\tAccuracy: 83.33%\n", + "22\tValidation loss: 0.736155\tBest loss: 0.618038\tAccuracy: 79.33%\n", + "23\tValidation loss: 0.739367\tBest loss: 0.618038\tAccuracy: 76.67%\n", + "24\tValidation loss: 0.699710\tBest loss: 0.618038\tAccuracy: 78.00%\n", + "25\tValidation loss: 0.709630\tBest loss: 0.618038\tAccuracy: 81.33%\n", + "26\tValidation loss: 0.692474\tBest loss: 0.618038\tAccuracy: 79.33%\n", + "27\tValidation loss: 0.807931\tBest loss: 0.618038\tAccuracy: 77.33%\n", + "28\tValidation loss: 0.676134\tBest loss: 0.618038\tAccuracy: 82.00%\n", + "29\tValidation loss: 0.738905\tBest loss: 0.618038\tAccuracy: 79.33%\n", + "30\tValidation loss: 0.664826\tBest loss: 0.618038\tAccuracy: 81.33%\n", + "31\tValidation loss: 0.694714\tBest loss: 0.618038\tAccuracy: 80.00%\n", + "32\tValidation loss: 0.739238\tBest loss: 0.618038\tAccuracy: 83.33%\n", + "33\tValidation loss: 0.697210\tBest loss: 0.618038\tAccuracy: 80.00%\n", + "34\tValidation loss: 0.817373\tBest loss: 0.618038\tAccuracy: 79.33%\n", + "Early stopping!\n", + "Total training time: 0.9s\n", + "INFO:tensorflow:Restoring parameters from ./my_mnist_model_5_to_9_five_frozen\n", + "Final test accuracy: 76.51%\n" + ] + } + ], "source": [ "import time\n", "\n", @@ -4049,7 +6299,71 @@ "cell_type": "code", "execution_count": 150, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Restoring parameters from ./my_best_mnist_model_0_to_4\n", + "0\tValidation loss: 0.923460\tBest loss: 0.923460\tAccuracy: 69.33%\n", + "1\tValidation loss: 0.796192\tBest loss: 0.796192\tAccuracy: 77.33%\n", + "2\tValidation loss: 0.812068\tBest loss: 0.796192\tAccuracy: 78.67%\n", + "3\tValidation loss: 0.697938\tBest loss: 0.697938\tAccuracy: 80.67%\n", + "4\tValidation loss: 0.877122\tBest loss: 0.697938\tAccuracy: 74.67%\n", + "5\tValidation loss: 0.708524\tBest loss: 0.697938\tAccuracy: 81.33%\n", + "6\tValidation loss: 0.689500\tBest loss: 0.689500\tAccuracy: 84.00%\n", + "7\tValidation loss: 0.758315\tBest loss: 0.689500\tAccuracy: 81.33%\n", + "8\tValidation loss: 0.711138\tBest loss: 0.689500\tAccuracy: 78.67%\n", + "9\tValidation loss: 0.687304\tBest loss: 0.687304\tAccuracy: 81.33%\n", + "10\tValidation loss: 0.639222\tBest loss: 0.639222\tAccuracy: 81.33%\n", + "11\tValidation loss: 0.716750\tBest loss: 0.639222\tAccuracy: 82.67%\n", + "12\tValidation loss: 0.693442\tBest loss: 0.639222\tAccuracy: 80.67%\n", + "13\tValidation loss: 0.727682\tBest loss: 0.639222\tAccuracy: 84.00%\n", + "14\tValidation loss: 0.637289\tBest loss: 0.637289\tAccuracy: 84.67%\n", + "15\tValidation loss: 0.741304\tBest loss: 0.637289\tAccuracy: 83.33%\n", + "16\tValidation loss: 0.651895\tBest loss: 0.637289\tAccuracy: 82.67%\n", + "17\tValidation loss: 0.641192\tBest loss: 0.637289\tAccuracy: 80.67%\n", + "18\tValidation loss: 0.690386\tBest loss: 0.637289\tAccuracy: 80.67%\n", + "19\tValidation loss: 0.648541\tBest loss: 0.637289\tAccuracy: 82.67%\n", + "20\tValidation loss: 0.779663\tBest loss: 0.637289\tAccuracy: 83.33%\n", + "21\tValidation loss: 0.768834\tBest loss: 0.637289\tAccuracy: 82.67%\n", + "22\tValidation loss: 0.706279\tBest loss: 0.637289\tAccuracy: 82.67%\n", + "23\tValidation loss: 0.745840\tBest loss: 0.637289\tAccuracy: 82.00%\n", + "24\tValidation loss: 0.740068\tBest loss: 0.637289\tAccuracy: 83.33%\n", + "25\tValidation loss: 0.604927\tBest loss: 0.604927\tAccuracy: 84.67%\n", + "26\tValidation loss: 0.635410\tBest loss: 0.604927\tAccuracy: 82.00%\n", + "27\tValidation loss: 0.776003\tBest loss: 0.604927\tAccuracy: 82.67%\n", + "28\tValidation loss: 0.621502\tBest loss: 0.604927\tAccuracy: 82.00%\n", + "29\tValidation loss: 0.695963\tBest loss: 0.604927\tAccuracy: 83.33%\n", + "30\tValidation loss: 0.668194\tBest loss: 0.604927\tAccuracy: 84.67%\n", + "31\tValidation loss: 0.768975\tBest loss: 0.604927\tAccuracy: 82.67%\n", + "32\tValidation loss: 0.594731\tBest loss: 0.594731\tAccuracy: 84.00%\n", + "33\tValidation loss: 0.665088\tBest loss: 0.594731\tAccuracy: 84.00%\n", + "34\tValidation loss: 0.716284\tBest loss: 0.594731\tAccuracy: 81.33%\n", + "35\tValidation loss: 0.782680\tBest loss: 0.594731\tAccuracy: 84.00%\n", + "36\tValidation loss: 0.816441\tBest loss: 0.594731\tAccuracy: 84.00%\n", + "37\tValidation loss: 0.749341\tBest loss: 0.594731\tAccuracy: 84.00%\n", + "38\tValidation loss: 0.728754\tBest loss: 0.594731\tAccuracy: 82.00%\n", + "39\tValidation loss: 0.838166\tBest loss: 0.594731\tAccuracy: 84.00%\n", + "40\tValidation loss: 0.714871\tBest loss: 0.594731\tAccuracy: 84.00%\n", + "41\tValidation loss: 0.765463\tBest loss: 0.594731\tAccuracy: 84.67%\n", + "42\tValidation loss: 0.744043\tBest loss: 0.594731\tAccuracy: 82.00%\n", + "43\tValidation loss: 0.726922\tBest loss: 0.594731\tAccuracy: 83.33%\n", + "44\tValidation loss: 0.641118\tBest loss: 0.594731\tAccuracy: 82.67%\n", + "45\tValidation loss: 0.657861\tBest loss: 0.594731\tAccuracy: 84.00%\n", + "46\tValidation loss: 0.803642\tBest loss: 0.594731\tAccuracy: 86.00%\n", + "47\tValidation loss: 0.754644\tBest loss: 0.594731\tAccuracy: 84.67%\n", + "48\tValidation loss: 0.865141\tBest loss: 0.594731\tAccuracy: 84.00%\n", + "49\tValidation loss: 0.709169\tBest loss: 0.594731\tAccuracy: 84.67%\n", + "50\tValidation loss: 0.723139\tBest loss: 0.594731\tAccuracy: 84.00%\n", + "51\tValidation loss: 0.745109\tBest loss: 0.594731\tAccuracy: 84.67%\n", + "52\tValidation loss: 0.803908\tBest loss: 0.594731\tAccuracy: 82.67%\n", + "Early stopping!\n", + "INFO:tensorflow:Restoring parameters from ./my_mnist_model_5_to_9_four_frozen\n", + "Final test accuracy: 80.17%\n" + ] + } + ], "source": [ "n_epochs = 1000\n", "batch_size = 20\n", @@ -4129,7 +6443,43 @@ "cell_type": "code", "execution_count": 152, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Restoring parameters from ./my_mnist_model_5_to_9_four_frozen\n", + "0\tValidation loss: 0.880485\tBest loss: 0.880485\tAccuracy: 86.00%\n", + "1\tValidation loss: 1.388974\tBest loss: 0.880485\tAccuracy: 81.33%\n", + "2\tValidation loss: 0.741543\tBest loss: 0.741543\tAccuracy: 86.67%\n", + "3\tValidation loss: 1.030772\tBest loss: 0.741543\tAccuracy: 84.00%\n", + "4\tValidation loss: 0.699438\tBest loss: 0.699438\tAccuracy: 87.33%\n", + "5\tValidation loss: 0.743930\tBest loss: 0.699438\tAccuracy: 89.33%\n", + "6\tValidation loss: 1.711346\tBest loss: 0.699438\tAccuracy: 82.67%\n", + "7\tValidation loss: 1.437762\tBest loss: 0.699438\tAccuracy: 82.00%\n", + "8\tValidation loss: 0.829231\tBest loss: 0.699438\tAccuracy: 86.67%\n", + "9\tValidation loss: 1.033920\tBest loss: 0.699438\tAccuracy: 86.67%\n", + "10\tValidation loss: 1.055709\tBest loss: 0.699438\tAccuracy: 87.33%\n", + "11\tValidation loss: 0.971796\tBest loss: 0.699438\tAccuracy: 88.00%\n", + "12\tValidation loss: 0.801815\tBest loss: 0.699438\tAccuracy: 86.00%\n", + "13\tValidation loss: 0.726146\tBest loss: 0.699438\tAccuracy: 89.33%\n", + "14\tValidation loss: 0.757217\tBest loss: 0.699438\tAccuracy: 88.67%\n", + "15\tValidation loss: 0.791842\tBest loss: 0.699438\tAccuracy: 90.00%\n", + "16\tValidation loss: 0.732507\tBest loss: 0.699438\tAccuracy: 90.67%\n", + "17\tValidation loss: 0.737297\tBest loss: 0.699438\tAccuracy: 90.67%\n", + "18\tValidation loss: 0.746715\tBest loss: 0.699438\tAccuracy: 90.00%\n", + "19\tValidation loss: 0.747751\tBest loss: 0.699438\tAccuracy: 90.00%\n", + "20\tValidation loss: 0.749325\tBest loss: 0.699438\tAccuracy: 90.00%\n", + "21\tValidation loss: 0.751899\tBest loss: 0.699438\tAccuracy: 90.00%\n", + "22\tValidation loss: 0.754314\tBest loss: 0.699438\tAccuracy: 90.00%\n", + "23\tValidation loss: 0.757840\tBest loss: 0.699438\tAccuracy: 90.00%\n", + "24\tValidation loss: 0.761543\tBest loss: 0.699438\tAccuracy: 90.00%\n", + "Early stopping!\n", + "INFO:tensorflow:Restoring parameters from ./my_mnist_model_5_to_9_two_frozen\n", + "Final test accuracy: 84.37%\n" + ] + } + ], "source": [ "n_epochs = 1000\n", "batch_size = 20\n", @@ -4194,7 +6544,52 @@ "cell_type": "code", "execution_count": 154, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Restoring parameters from ./my_mnist_model_5_to_9_two_frozen\n", + "0\tValidation loss: 0.846005\tBest loss: 0.846005\tAccuracy: 83.33%\n", + "1\tValidation loss: 0.694439\tBest loss: 0.694439\tAccuracy: 91.33%\n", + "2\tValidation loss: 1.201433\tBest loss: 0.694439\tAccuracy: 85.33%\n", + "3\tValidation loss: 1.975297\tBest loss: 0.694439\tAccuracy: 85.33%\n", + "4\tValidation loss: 0.692805\tBest loss: 0.692805\tAccuracy: 95.33%\n", + "5\tValidation loss: 1.090217\tBest loss: 0.692805\tAccuracy: 91.33%\n", + "6\tValidation loss: 1.924300\tBest loss: 0.692805\tAccuracy: 90.67%\n", + "7\tValidation loss: 4.019310\tBest loss: 0.692805\tAccuracy: 87.33%\n", + "8\tValidation loss: 4.150792\tBest loss: 0.692805\tAccuracy: 78.00%\n", + "9\tValidation loss: 4.522708\tBest loss: 0.692805\tAccuracy: 75.33%\n", + "10\tValidation loss: 1.163385\tBest loss: 0.692805\tAccuracy: 90.00%\n", + "11\tValidation loss: 0.655868\tBest loss: 0.655868\tAccuracy: 92.67%\n", + "12\tValidation loss: 0.943888\tBest loss: 0.655868\tAccuracy: 92.67%\n", + "13\tValidation loss: 0.529996\tBest loss: 0.529996\tAccuracy: 92.67%\n", + "14\tValidation loss: 0.610578\tBest loss: 0.529996\tAccuracy: 94.67%\n", + "15\tValidation loss: 3.899716\tBest loss: 0.529996\tAccuracy: 88.00%\n", + "16\tValidation loss: 18.285717\tBest loss: 0.529996\tAccuracy: 86.67%\n", + "17\tValidation loss: 23.169626\tBest loss: 0.529996\tAccuracy: 78.00%\n", + "18\tValidation loss: 17.309252\tBest loss: 0.529996\tAccuracy: 90.00%\n", + "19\tValidation loss: 44.261902\tBest loss: 0.529996\tAccuracy: 80.00%\n", + "20\tValidation loss: 52.460327\tBest loss: 0.529996\tAccuracy: 80.00%\n", + "21\tValidation loss: 26.318949\tBest loss: 0.529996\tAccuracy: 83.33%\n", + "22\tValidation loss: 32.857723\tBest loss: 0.529996\tAccuracy: 90.67%\n", + "23\tValidation loss: 53.359497\tBest loss: 0.529996\tAccuracy: 88.00%\n", + "24\tValidation loss: 57.823742\tBest loss: 0.529996\tAccuracy: 88.00%\n", + "25\tValidation loss: 37.154972\tBest loss: 0.529996\tAccuracy: 92.67%\n", + "26\tValidation loss: 41.386772\tBest loss: 0.529996\tAccuracy: 90.00%\n", + "27\tValidation loss: 43.486767\tBest loss: 0.529996\tAccuracy: 90.00%\n", + "28\tValidation loss: 42.776855\tBest loss: 0.529996\tAccuracy: 88.67%\n", + "29\tValidation loss: 43.368839\tBest loss: 0.529996\tAccuracy: 90.67%\n", + "30\tValidation loss: 43.440975\tBest loss: 0.529996\tAccuracy: 90.00%\n", + "31\tValidation loss: 42.889927\tBest loss: 0.529996\tAccuracy: 91.33%\n", + "32\tValidation loss: 42.806690\tBest loss: 0.529996\tAccuracy: 90.67%\n", + "33\tValidation loss: 42.784145\tBest loss: 0.529996\tAccuracy: 90.67%\n", + "Early stopping!\n", + "INFO:tensorflow:Restoring parameters from ./my_mnist_model_5_to_9_no_frozen\n", + "Final test accuracy: 90.60%\n" + ] + } + ], "source": [ "n_epochs = 1000\n", "batch_size = 20\n", @@ -4242,7 +6637,61 @@ "cell_type": "code", "execution_count": 155, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0\tValidation loss: 0.803557\tBest loss: 0.803557\tAccuracy: 71.33%\n", + "1\tValidation loss: 0.966741\tBest loss: 0.803557\tAccuracy: 85.33%\n", + "2\tValidation loss: 1.158972\tBest loss: 0.803557\tAccuracy: 78.00%\n", + "3\tValidation loss: 0.615960\tBest loss: 0.615960\tAccuracy: 88.00%\n", + "4\tValidation loss: 0.612626\tBest loss: 0.612626\tAccuracy: 92.00%\n", + "5\tValidation loss: 0.686420\tBest loss: 0.612626\tAccuracy: 89.33%\n", + "6\tValidation loss: 0.805281\tBest loss: 0.612626\tAccuracy: 89.33%\n", + "7\tValidation loss: 0.753108\tBest loss: 0.612626\tAccuracy: 88.67%\n", + "8\tValidation loss: 1.051471\tBest loss: 0.612626\tAccuracy: 86.00%\n", + "9\tValidation loss: 0.487089\tBest loss: 0.487089\tAccuracy: 93.33%\n", + "10\tValidation loss: 1.191093\tBest loss: 0.487089\tAccuracy: 85.33%\n", + "11\tValidation loss: 0.878905\tBest loss: 0.487089\tAccuracy: 88.67%\n", + "12\tValidation loss: 0.768841\tBest loss: 0.487089\tAccuracy: 91.33%\n", + "13\tValidation loss: 1.153907\tBest loss: 0.487089\tAccuracy: 90.67%\n", + "14\tValidation loss: 0.985427\tBest loss: 0.487089\tAccuracy: 89.33%\n", + "15\tValidation loss: 1.221879\tBest loss: 0.487089\tAccuracy: 85.33%\n", + "16\tValidation loss: 0.961743\tBest loss: 0.487089\tAccuracy: 88.67%\n", + "17\tValidation loss: 3.116057\tBest loss: 0.487089\tAccuracy: 84.00%\n", + "18\tValidation loss: 0.686387\tBest loss: 0.487089\tAccuracy: 84.00%\n", + "19\tValidation loss: 0.929801\tBest loss: 0.487089\tAccuracy: 88.00%\n", + "20\tValidation loss: 1.137579\tBest loss: 0.487089\tAccuracy: 92.00%\n", + "21\tValidation loss: 0.987261\tBest loss: 0.487089\tAccuracy: 91.33%\n", + "22\tValidation loss: 2.030677\tBest loss: 0.487089\tAccuracy: 91.33%\n", + "23\tValidation loss: 1.094184\tBest loss: 0.487089\tAccuracy: 92.00%\n", + "24\tValidation loss: 1.332256\tBest loss: 0.487089\tAccuracy: 82.67%\n", + "25\tValidation loss: 1.128633\tBest loss: 0.487089\tAccuracy: 85.33%\n", + "26\tValidation loss: 0.866569\tBest loss: 0.487089\tAccuracy: 90.67%\n", + "27\tValidation loss: 1.088500\tBest loss: 0.487089\tAccuracy: 89.33%\n", + "28\tValidation loss: 1.146113\tBest loss: 0.487089\tAccuracy: 89.33%\n", + "29\tValidation loss: 1.163180\tBest loss: 0.487089\tAccuracy: 89.33%\n", + "30\tValidation loss: 1.154797\tBest loss: 0.487089\tAccuracy: 89.33%\n", + "Early stopping!\n" + ] + }, + { + "data": { + "text/plain": [ + "DNNClassifier(activation=,\n", + " batch_norm_momentum=None, batch_size=20, dropout_rate=None,\n", + " initializer=._initializer at 0x7fd9d5e628c8>,\n", + " learning_rate=0.01, n_hidden_layers=4, n_neurons=100,\n", + " optimizer_class=,\n", + " random_state=42)" + ] + }, + "execution_count": 155, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "dnn_clf_5_to_9 = DNNClassifier(n_hidden_layers=4, random_state=42)\n", "dnn_clf_5_to_9.fit(X_train2, y_train2, n_epochs=1000, X_valid=X_valid2, y_valid=y_valid2)" @@ -4252,7 +6701,18 @@ "cell_type": "code", "execution_count": 156, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "0.90413495165603786" + ] + }, + "execution_count": 156, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "y_pred = dnn_clf_5_to_9.predict(X_test2)\n", "accuracy_score(y_test2, y_pred)" @@ -4383,7 +6843,18 @@ "cell_type": "code", "execution_count": 161, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "TensorShape([Dimension(None), Dimension(100)])" + ] + }, + "execution_count": 161, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "dnn1.shape" ] @@ -4392,7 +6863,18 @@ "cell_type": "code", "execution_count": 162, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "TensorShape([Dimension(None), Dimension(100)])" + ] + }, + "execution_count": 162, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "dnn2.shape" ] @@ -4408,7 +6890,18 @@ "cell_type": "code", "execution_count": 163, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "TensorShape([Dimension(None), Dimension(200)])" + ] + }, + "execution_count": 163, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "dnn_outputs.shape" ] @@ -4630,7 +7123,18 @@ "cell_type": "code", "execution_count": 173, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "((5, 2, 784), dtype('float32'))" + ] + }, + "execution_count": 173, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "X_batch.shape, X_batch.dtype" ] @@ -4646,7 +7150,18 @@ "cell_type": "code", "execution_count": 174, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAANcAAAGiCAYAAAB05VNzAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHMhJREFUeJzt3XuYjOf5B/AvwVrEaZ1WnHKywgahJQ4ROUjlUOJSQSon\n6pKKYxEShEaLRCMajbSNK1a1KhHiEIegQSIOdQySjSRKVC52s85UVkR/f+T33HtPZ4aZ2bnnnZ35\nfv7J93pnd+bJrttze9/nfd5i//3vf0FE0Vfc6wEQJSoWF5ERFheRERYXkREWF5ERFheRERYXkREW\nF5ERFheRkRJeDyAILhsJX7EIv48/6/CF9LPmzEVkhMVFZITFRWSExUVkhMVFZITFRWSExUVkhMVF\nZITFRWSExUVkhMVFZITFRWSExUVkhMVFZITFRWQkXu/ninvDhg2TPHXqVMmdOnUCACxevDjmYypK\nNm3aJHnZsmWSf/e73wEA8vPz5VixYgW3T1WqVEnyuHHjAAD9+/eXYyVKxM8fac5cREZYXERGisXp\ngxjialAnT54EADz66KNybPXq1ZJ1C5OSkgIA2Lhxoxy75ZZbrIcIFIHb/F988UXJkydPlnzq1Cm/\nr9V/LnVbGMiQIUMkv/TSS4UZYqh4mz+Rl1hcREbi59RKnJk3b57kgQMHAgDy8vLkWOPGjSW7M4QA\n8Nvf/hYAcP78eeshFhnr168HUHAmEPBtBStWrCi5atWqAIBRo0bJsbNnz0r+wx/+INm143//+9/l\n2IkTJySPHz9ecp06dSIef6Q4cxEZ4QkNZe/evZJvvfVWyf/5z38AAE2aNJFjy5cvl3zu3DnJ3bp1\nAwD885//lGMlS5aM/mD9xdUJjezsbMnt27cH4Dvz9+zZU/Lw4cMlN23aNOTPcCea9Gw2Z84cyevW\nrZOcnp4e8vuGgCc0iLzE4iIykvQnNL799lvJI0aMkOxaQW369OmSdZvx4YcfSr506RKAmLWCcWvG\njBmSXTvoTlYAwAsvvCD5mmuuiegz3FKnzz77TI59+eWXknNyciRHuS0MCWcuIiMsLiIjSd8Wzp49\nW/LKlSsDfs2sWbMAAG3btg34esOGDSW/+uqrAIDTp0/LsfLlyxd6nEXB559/LllfJ3RmzpwpOdJW\nUHOr4vV1rnjCmYvICIuLyEjStoWHDx8GUNBaAL6rrzt27OiX9UVmvcJbX6x077thwwY51rp16yiN\nOr7pi+nHjx/3e71WrVqF/owzZ85I1jdcOrp1z8jIKPTnFQZnLiIjSTtzHTx4EACQm5sb8PWPPvpI\n8s033wzAd/nOle4xIhurVq2SvGXLFgBA9erV5diUKVMkp6amxm5gAXDmIjLC4iIykrRtYZkyZQAA\n9evXl2P6Oo17HSjYcUjfj9SoUSPJLVq0kOxalLp160Z5xPFPL2/S//9fffVVod5XnyjRvwNH/w71\n78JrnLmIjLC4iIwkbVvYrFkzAMCePXvk2NatWyXrtibQUh19nUtzLWQ0lvcUNfo6VoMGDSS7trBv\n375yTJ/10xt9OhcuXJDstk4AfG9CdeL1OiJnLiIjLC4iI0nbFjr6psZw2gu9vEnvQxJs5XyyGTNm\njOQ1a9YAALZv3y7H0tLSJA8ePFhy2bJlAQDvv/++HNu8eXPAz6hZsyYAoE+fPlEYcfRx5iIywt2f\nwrB//37Jbdq0kaz34HO7HtWrVy9m4/p/cbX7k7ZgwQIAvicmPvnkE8kXL170H1QI21mPHDkSADBx\n4sSojDMM3P2JyEssLiIjSX9CIxy6/dCr6R988EHJHrSDca9r164+/wUKWkXA93449/SYb775Ro65\nzT//l7tWGa84cxEZYXERGeHZwhC4tkRvLKmvj+kH3WVmZsZuYL7i9mxhJPS1q6ysLMn6d+Cuf0Vj\n+4Aw8WwhkZd4QiMEbutlvfW1fkqHh7NVwnJ7RQK+17luuukmyR7MWGHhzEVkhMVFZIRtYRD6KSf6\nQXcOW0Eb+uF1TkpKiuSnn346lsMpFM5cREZYXERG2BYGsXDhQsl6KwDn/vvvj+Vwkoa790urXLmy\n5A4dOsRyOIXCmYvICIuLyAiXPwVRrVo1yW6PeL2jkd6FqFy5crEbWHAJsfypQoUKAHyfZuJu5wcK\nniLjMS5/IvIST2go+m/LQLee62d2xclslRR+9KMfeT2EiHDmIjLC4iIywrZQ0XvlBbq13O02RPba\nt28vuaj+3DlzERlhcREZ4XWuxJEQ17mKCF7nIvISi4vICIuLyAiLi8gIi4vICIuLyAiLi8gIi4vI\nCIuLyAiLi8gIV8VTkbJt2zbJ7qGDjz76qBzz4PnIQXHmIjLCmYvinr63rm/fvpJ//OMfAwBycnJi\nPqZQcOYiMsLiIjLC+7miYMyYMZKrVKkCABgyZEish5Gw93O5ExcAkJqaKnnu3LkAgPz8fDlWunTp\nWAyJ93MReYnFRWSEZwsjdPr0acmzZ8+WPHz4cC+Gk3Dee+89yStWrJD84YcfSnbPSo5RKxg2zlxE\nRlhcREbYFkbo+PHjkvWTN/STUChygwcPltykSRPJLVq08GI4EeHMRWSEM1eEAj3KFQAyMjJiPJLE\nMn/+fADAvn375NjRo0e9Gk6hcOYiMsLiIjLCtjBC06dP93oICcn9XPWJoerVq3s1nELhzEVkhMVF\nZIRtYRjOnTsneefOnZKbNm0quV69erEcUkI4duyY5M2bNwPwfRBhUcWZi8gIZ64wrF27VnJeXp7k\nPn36eDGchDF+/HjJlSpVAgBkZmZ6NJro4cxFZITFRWSEbWEY3n77bckpKSmSn3jiCS+GU6Tphc/z\n5s2TPHnyZABAxYoVYz6maOPMRWSExUVkhG1hGN566y3J6enpkrkSPnxZWVmS9XWuxo0b+31tbm6u\n5PPnz0suW7YsgIIdt+INZy4iIywuIiNsC0Mwc+ZMAL4tCS8cF86GDRskt2zZUnLDhg0BAK+99poc\nGzVqlGS965Y7ozhhwgQ5NmDAgOgPNkKcuYiMcOYKgds2uVSpUnLsZz/7mVfDKbIOHDggeenSpZIn\nTZokuUuXLgCA3bt3yzE9i2kHDx4EAAwdOlSO1a1bV/JPf/rTwg24kDhzERlhcREZYVsYxK5duyS7\nf3y7lgXg/oSR0NtSX7x4UXK7du0ku6fu6AfaPfzwwwHfzz0Ub/To0XJM7yHpNc5cREZYXERG2BYG\nsWjRIsnfffcdAKBnz55eDSch6GVMml4+5rarHjFiRMTvFy84cxEZ4cwVhH4OVNWqVQEAN998s1fD\nSQjbtm2TXKNGDcn6+mE4/vrXv/p9f+fOnSMcXfRx5iIywuIiMsK2UFm1apXkdevWSe7Xrx8A4Prr\nr4/1kBJK5cqVJestqkuWLBnye+zYsUOyWzb17LPPyrGaNWsWZohRxZmLyAiLi8gI20Jl48aNki9d\nuiSZK+Cjo1WrVpLnzJkj+dSpU5LT0tL8vm/ixImS9VYLd9xxBwBg7NixUR1ntHDmIjLC4iIywrZQ\n+de//iW5du3aknU7Q5G7+uqrAx7v1q2b5HvuuQeA71NO1q9fL7l3796Sn3/+eQBAiRLx+ceYMxeR\nkfgs+RjKzs6WrLer1puipKamxnRMiapHjx6S9+3bJ1nvYeieJHP33XfLsSVLlkj+yU9+YjjC6OLM\nRWSExUVkJOnbwvfee0+y3pewQoUKXgwnoekTD3qvQZ0TCWcuIiMsLiIjSd8WNmvWzOshUILizEVk\nhMVFZKSY24QxzsTloOJcsQi/jz/r8IX0s+bMRWSExUVkhMVFZITFRWSExUVkhMVFZITFRWSExUVk\nhMVFZITFRWSExUVkhMVFZITFRWSExUVkhMVFZITFRWSExUVkhMVFZITFRWSExUVkJOn3LQxm9uzZ\nkj/++GO/119++WXJxYr571fSvHlzyfpZU8GeUUWRu++++ySvWLFC8uDBgyVPmzYtpmMCOHMRmWFx\nERlhWxjE8uXLJeuH4jm6FQzUFu7YsUPyzJkzJQ8dOjRaQ6QA9O/i008/9XAknLmIzLC4iIywLQxC\nP5Dt/vvvB+DbKmqbN2+W/O9//9vv9RtvvDHKoyMA2L17NwBg9erVHo8kMM5cREb4IIYInTlzRnK7\ndu0ku79Nte+//z4WQ0q6BzF0794dADB//vyAr+trW4MGDYrmR/NBDEReYnERGeEJjTDoVrB8+fKS\nA13neu6552IypmSj2+4lS5b4vV62bFnJHTp0iMmYguHMRWSExUVkhG1hCI4dOwYA6NKlixwLtvyp\nVatWAICRI0fGaHSJ79ChQ5L79OkjOT8/3+9rGzRoIPmmm26yHdgVcOYiMsLiIjLCtjAI1woCwMSJ\nEwEAH330UcCvrVKlit/XpqamGo4uuRw5ckTy9u3b/V7PzMyUvHjx4piMKRScuYiMcPlTEL1795as\nb/l39M8tJSVFcrVq1fy+duHChZL17f9RllDLn9auXSv57rvvlhzoz6teUN2xY0fbgf2Ay5+IvMTi\nIjLCExqKvhcrKyvrsl+r2xN9vSXQ/Vzr16+XbNgWJoRLly4BKDgxBARuBQGgdu3aAIC2bdvaDywC\nnLmIjLC4iIywLVTS0tIkt27dWvKmTZsu+32BVsWH8zoVeOmllwAA//jHPwK+rn9HCxYsAACUK1fO\nfmAR4MxFZITXuYL4/PPPJR8/fvyyXzt27FjJeutq5/Dhw5LT09OjMLqAiux1rj179ki+6667AAB5\neXly7Nprr5WsN6O57rrrYjC6gHidi8hLLC4iIzyhEUT9+vUv+7q+5V+3MIEYtoJF1tatWyW7fSGB\nwD/Lhx56SLKHrWDYOHMRGWFxERnh2cII6WVMO3fulFymTBkAwLx58+TYAw88EIshxf3ZwrNnz0q+\n4YYbJOfm5vp9rf6ZLVq0SHLx4nExH/BsIZGXWFxERni2MAzvvvuuZN0K6uVNrp2JUStYJJw/fx4A\n8Nhjj8mxQK2gptvGOGkFw1Y0R01UBHDmCoFbRDpw4MCAr+sNavr37x+TMRUl69atAwC88847V/za\nIUOGAADGjRtnOaSY4MxFZITFRWQk4dvCqVOnStYnHnr06AEgtKVJri3U2ypr7r0A3wfhJTN9wmLM\nmDGX/Vp9P1avXr0AABUqVLAZWAxx5iIywuIiMpKQy590Kzhs2DDJui10q97XrFkjx2rUqCH5N7/5\njeTnn3/e7zNq1aolWb/HlVbTG/J8+VNOTo7ke++9V/KuXbsu+336IXZF5Poglz8ReYnFRWQkIdtC\nvf/FnXfeKVk/LcPRbZxecqP3Hw8kOzs74Ht4yPO2sGvXrpKvdMH4ySeflDxt2jTJpUqVitZwLLEt\nJPJSQs5c2t69eyXrfywH2nZa/ywC7TX43HPPSY7D5Tmez1x6C+pA17YyMjIk65m/COLMReQlFheR\nkYRf/qQf6akf+fnss88CAGbOnBnw+2rWrCnZtTi/+MUvLIaYMIKd2KlTpw4A3+tZyYAzF5ERFheR\nkYQ/W5hEPD9bmER4tpDISywuIiMsLiIjLC4iIywuIiMsLiIjLC4iIywuIiMsLiIjLC4iIywuIiMs\nLiIjLC4iIywuIiMsLiIjLC4iIywuIiMsLiIjLC4iIywuIiMJv28hea9ly5YAgKNHj8ox/dAGvbX1\npk2bAAAnT56M0ejscOYiMsLiIjLCfQsTR9zuW3jrrbcCALZu3XrFry1e/Ie/78uWLSvH2rVrJ/me\ne+7x+56f//znkitVqhTxOMPAfQuJvMTiIjLCs4VkrlmzZgBCawsvXboEADhz5owce/fddyUvW7bM\n73vS0tIk9+zZM+JxRhtnLiIjLC4iI0nfFur2Qz8/WT+obdasWX7fd9ttt0nWD9grXbo0AKBfv35y\nrGLFitEZbBE1ZcoUAMDu3bvlmLtYHA2fffZZ1N4rmjhzERlJ+utcAwYMkPzaa6+F/H3651asmP9l\nj2rVqknW12l++ctfSq5SpQoA35mvEOL2Opc7kaGvUZ0+fTrg12ZkZAAAateuLceC/axHjhwJAGjd\nurUcc52DMV7nIvISi4vISNK3hZ9++qnkYG3hqlWrAABffvmlHLtSWxiM/r4KFSoAAGrVqiXHdOvU\nuXNnybq1DCJu28KHH34YAPDmm28GfL1p06aSFy9eDMD3ZxKH2BYSeYnFRWQk6dtC7dSpU5JHjx4t\necaMGX5f+8orr0jWLcz+/fsBAFlZWXLs2LFjko8cOSI5nHbSLQu6jLhtC6+0Kl7/rB555BHr4UQD\n20IiL7G4iIwk/fKnNWvWSB46dKhkfRbRtW/Dhw+XY126dJF8zTXX+L3vsGHDJH/99deS9RnHRLZ2\n7VrJBw4c8Htd/8zatGkTkzHFGmcuIiNJdULjwoULkjds2AAAeOCBB+RYfn6+ZLc0CQCeeuopAL5L\nl6pWrWoxxMKIqxMaDRs2lLxv3z6/1ytXriz57bfflnzLLbdc9n2vvvpqyeGcEIoyntAg8hKLi8hI\nUrWF69atk3zXXXf98EFBljFNmzZN8sCBAy2GE21Fqi0Mh/4d6RNFY8eOBQCUL1++UO8fAbaFRF5i\ncREZSaq28IsvvpDsbrDTS5N0W1iiRMElwPr16/u91549eyyGWBhx1RbqM6t/+tOf/F5v3769ZN2u\nB3KlOxAaNWok2d3BAADp6emhDDUSbAuJvJRUM5deKeHuG/rggw/k2CeffCL5m2++kZybm+v3XnrT\nGfcPa6Bga2V9m3+MxNXMdfHiRcnuZ63vVUtJSZGsry8GohdRv/7665L1dUtHb4O9dOlSybfffnso\nww4VZy4iL7G4iIwkVVsYjoMHD0p2e+z1799fjul7v/Q/sps0aQIA2LFjh/EI/cRVW6i57aibN28u\nxyI92aDvh5s8eTIAYM6cOXJM/17c7wIAFi1aBACoU6dORJ/7P9gWEnmJxUVkhG1hGE6cOCF59erV\nkvV9Xq5t6datmxybO3duDEYXv22hu82/V69eckxvxlpY+gzik08+GfBr3O5Z+j6zQmBbSOQlFheR\nkaS/zT8c+nm7HTt2lLxlyxbJbjX9oUOH5Njhw4clx/lml6b0Llrdu3eXXNgbT/WNrcF07dq1UJ8R\nCc5cREYSauZy16H0cplAm8dEg17eE+iJHfoeI70IOJnp+7r0UqhBgwZJfuKJJ0J+v7y8PACB95UE\nfJ+Uoj8vVjhzERlhcREZSah+xa1qv+OOO+SY/ofspEmT/L5HP6pV+/bbbyXr7ZadV199VXKg+8CG\nDBkix2rUqHGloSe0QCcs9CNcdVuoV7I7ekX75s2bJbutw7dv3x7wc/VSJ71rVKxw5iIywuIiMpJQ\ny5/cbfz6Gkp2drbkVq1a+X2PvsU80ofY6fbjV7/6FQDfVidG4nb5U6AlYe5Og1CE86BBfZPq/Pnz\nJbdt2zbkzwsBlz8ReSmhZi7n5MmTkjdu3Ch5yZIlkt2t5+H8rZiZmSm5U6dOkh977DHJHuyh58Tt\nzOXk5ORI1g+y2Llzp+RAt+5f6XfUokULye73CphutcCZi8hLLC4iIwnZFiapuG8Lg3HbAAAF2yPo\n61krV66UrNvCZ555BoDvc9XS0tLMxqmwLSTyEouLyAjbwsRRZNvCIohtIZGXWFxERlhcREZYXERG\nWFxERlhcREZYXERGWFxERlhcREZYXERGWFxERlhcREZYXERGWFxERlhcREZYXERGWFxERlhcREZY\nXERGWFxERlhcREZYXERGWFxERhLqsa2F9dVXX0keMGCAZL3dstOvXz/JnTt3luyeGn/VVVdZDDFh\nfPfdd5L1I3J///vfAwAmTJggx/TemsOHD5d83333AQBatmwpx0qWLBn9wUaIMxeRERYXkZGk385a\nPwn+3nvvlZyXlxfR+73xxhsAgMcff7xQ44pAkdrOetasWZL79Onj93q5cuUk6z+j586d8/ta/bOe\nMmWKZMMnnnA7ayIvsbiIjCR9W3jjjTdK3r9/f6Hfr1KlSgCA5cuXyzF9NstQ3LeFkyZNkjx16lTJ\nx44dk/zyyy8DABo1aiTHjhw5Ilk/fzqQ6tWrS16/fr3k+vXrRzDioNgWEnmJxUVkJOkvIl+8eDGq\n73fixAkAwOTJk+XYO++8E9XPKGqys7MBFLR8gG8r2KNHD8lt2rQB4HuR3j0nGfB9JnLdunUBAKdO\nnZJjOTk5knNzcyVHuS0MCWcuIiNJP3M99dRTkt3Sm1BMmzZNsl6So5dQ0Q9mzJgBwPfaoVsmBgBN\nmzaVfNtttwEA8vPz5didd94pecSIEZIzMzMBAFu2bJFj3bp1k/zHP/7R7zP09TNrnLmIjLC4iIwk\nfVuoWzqdwzF9+nTJbAv9Bbp+6K4HAsCoUaMku5MUeknUmDFjLvv+uoWsWrWq5Llz50ru0qULAKBr\n166hDrvQOHMRGWFxERlJ+rYwHLr90Ge+Dh065MVwiowGDRoAAFauXCnH3nzzTcn16tWTvGLFCgBA\nRkZGyO9//fXXB3zf7t27S54/fz4AtoVECSEhZy79D+jKlStL1v+I3rt3LwDglVdekWNnz56VXLx4\nwd87bnGv+1sVAI4fPy754MGDURh1YtErMPQKCyfQbAWEN2MF0r59+6i9V2Fx5iIywuIiMpKQbaFb\nbgMACxYskFymTBnJX3/9NQDgzJkzJmPQi1GT0cKFCyV/8MEHfq+3aNFCcizat8WLFwMADhw4IMeu\nvfZa08/kzEVkhMVFZCSh2kJ37en999+XY7G+BuWW7+hrLMlCn0HVS8Kc8ePHS3766adjMSThrlF+\n//33MftMzlxERlhcREYSqi10Oy59/PHHno3h6NGjAICNGzfKsdatW3s1nJjSdxXs2bPH73V902Pp\n0qXNx6N3NvNilzPOXERGEmrmigfuH856SU+yzFya3kjmhhtuAABcd911no1B51jhzEVkhMVFZCSh\n2kK30rpEiYL/rUj3JdTbXH/xxRdhf79bdU9Aeno6AKBmzZrmn3X+/HnJ+qF6brlVtWrVzMfgcOYi\nMsLiIjKSUG1hu3btAPg+o3jbtm2X/Z7evXtLHjRokORSpUpJvnDhgt/3/fnPf5b8t7/9TfLu3bvD\nGHFycDeh6ptRrTbnXLZsmWT9u3/ooYcAAOXLlzf53EA4cxEZYXERGUmottBxO/1ES2pqqt8xvWe5\nfrhdhw4dABQsgwJ8z1rFYtlPvNm5cycAYNeuXXKsbdu2UXt/vWeK3vvfa5y5iIwk/WNbo81d09HP\nidJ76enHkbotCPQJmMaNG0f60Z4/tlU/bcRtHw0UzOIPPvigHMvKypIc6UmGc+fOAQD69u0rx+bN\nmydZX9Nyv4Pbb789os/6H3xsK5GXWFxERtgWRtkzzzwDAHjhhRfkWEpKiuQqVapIdjtQzZ49W449\n8sgjkX60522hNm7cOMkTJkzwe123iH/5y18kX+n6l17e5K5R6rZbP+XkrbfekhyldtBhW0jkJRYX\nkZGEvM7lJfcUet2q6L3kXSuY6PTD69xe8Xpp0qJFiyT36tVLcqdOnfzeSy8/e/HFFyW7n2taWpoc\nGzp0qOQot4Jh48xFZIQnNIxMmTJF8siRIwN+zVVXXQUAWLp0qRzr2LFjpB8ZVyc0NDfz6JlEXxO7\nEv1nVN+u71bDjB07Vo5Fc+XHZfCEBpGXWFxERtgWGjl16pTk0aNHS9ZPYHn88ccBAG+88UY0PjJu\n20JHX6N6/fXXJf/617+WfOLECb/vK1mypGS9YNotsWrevHlUxxkCtoVEXmJxERlhW5g44r4tTCBs\nC4m8xOIiMsLiIjLC4iIywuIiMsLiIjLC4iIyEq/3c8X+SWXJiz9rI5y5iIywuIiMsLiIjLC4iIyw\nuIiMsLiIjLC4iIywuIiMsLiIjLC4iIywuIiMsLiIjLC4iIywuIiMsLiIjLC4iIywuIiMsLiIjLC4\niIywuIiMsLiIjLC4iIywuIiMsLiIjLC4iIywuIiM/B8LRNiNqaw8LQAAAABJRU5ErkJggg==\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "plt.figure(figsize=(3, 3 * batch_size))\n", "plt.subplot(121)\n", @@ -4669,7 +7184,22 @@ "cell_type": "code", "execution_count": 175, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "array([[1],\n", + " [0],\n", + " [0],\n", + " [1],\n", + " [0]])" + ] + }, + "execution_count": 175, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "y_batch" ] @@ -4718,7 +7248,134 @@ "cell_type": "code", "execution_count": 177, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0 Train loss: 0.492426\n", + "0 Test accuracy: 0.7861\n", + "1 Train loss: 0.334813\n", + "2 Train loss: 0.290434\n", + "3 Train loss: 0.253434\n", + "4 Train loss: 0.217843\n", + "5 Train loss: 0.17127\n", + "5 Test accuracy: 0.9185\n", + "6 Train loss: 0.207128\n", + "7 Train loss: 0.172275\n", + "8 Train loss: 0.166783\n", + "9 Train loss: 0.161094\n", + "10 Train loss: 0.125131\n", + "10 Test accuracy: 0.9425\n", + "11 Train loss: 0.159824\n", + "12 Train loss: 0.124752\n", + "13 Train loss: 0.112234\n", + "14 Train loss: 0.114502\n", + "15 Train loss: 0.0950093\n", + "15 Test accuracy: 0.9532\n", + "16 Train loss: 0.119296\n", + "17 Train loss: 0.0754429\n", + "18 Train loss: 0.112295\n", + "19 Train loss: 0.133708\n", + "20 Train loss: 0.113547\n", + "20 Test accuracy: 0.9596\n", + "21 Train loss: 0.0674082\n", + "22 Train loss: 0.0936297\n", + "23 Train loss: 0.0986469\n", + "24 Train loss: 0.111875\n", + "25 Train loss: 0.0735623\n", + "25 Test accuracy: 0.9675\n", + "26 Train loss: 0.0790324\n", + "27 Train loss: 0.0487644\n", + "28 Train loss: 0.0869071\n", + "29 Train loss: 0.0694422\n", + "30 Train loss: 0.060089\n", + "30 Test accuracy: 0.9663\n", + "31 Train loss: 0.103902\n", + "32 Train loss: 0.0535952\n", + "33 Train loss: 0.0310679\n", + "34 Train loss: 0.0536294\n", + "35 Train loss: 0.046265\n", + "35 Test accuracy: 0.9701\n", + "36 Train loss: 0.0679821\n", + "37 Train loss: 0.0326656\n", + "38 Train loss: 0.0357479\n", + "39 Train loss: 0.0333373\n", + "40 Train loss: 0.0415115\n", + "40 Test accuracy: 0.9719\n", + "41 Train loss: 0.0577977\n", + "42 Train loss: 0.0342781\n", + "43 Train loss: 0.0439651\n", + "44 Train loss: 0.0597254\n", + "45 Train loss: 0.0588695\n", + "45 Test accuracy: 0.9721\n", + "46 Train loss: 0.0556821\n", + "47 Train loss: 0.063956\n", + "48 Train loss: 0.0301285\n", + "49 Train loss: 0.0402678\n", + "50 Train loss: 0.0489125\n", + "50 Test accuracy: 0.9751\n", + "51 Train loss: 0.0394528\n", + "52 Train loss: 0.0233041\n", + "53 Train loss: 0.064878\n", + "54 Train loss: 0.0510189\n", + "55 Train loss: 0.0312619\n", + "55 Test accuracy: 0.9742\n", + "56 Train loss: 0.0244156\n", + "57 Train loss: 0.0409082\n", + "58 Train loss: 0.0346896\n", + "59 Train loss: 0.0455727\n", + "60 Train loss: 0.0488268\n", + "60 Test accuracy: 0.9751\n", + "61 Train loss: 0.0154253\n", + "62 Train loss: 0.0358874\n", + "63 Train loss: 0.0290555\n", + "64 Train loss: 0.0172143\n", + "65 Train loss: 0.0377991\n", + "65 Test accuracy: 0.9751\n", + "66 Train loss: 0.0360786\n", + "67 Train loss: 0.0240278\n", + "68 Train loss: 0.0314243\n", + "69 Train loss: 0.0412082\n", + "70 Train loss: 0.0439106\n", + "70 Test accuracy: 0.9763\n", + "71 Train loss: 0.0169656\n", + "72 Train loss: 0.0181306\n", + "73 Train loss: 0.0214228\n", + "74 Train loss: 0.0418301\n", + "75 Train loss: 0.0378622\n", + "75 Test accuracy: 0.9759\n", + "76 Train loss: 0.0199817\n", + "77 Train loss: 0.0145837\n", + "78 Train loss: 0.0199176\n", + "79 Train loss: 0.0226598\n", + "80 Train loss: 0.0119815\n", + "80 Test accuracy: 0.9779\n", + "81 Train loss: 0.0177832\n", + "82 Train loss: 0.00981572\n", + "83 Train loss: 0.0279094\n", + "84 Train loss: 0.0237818\n", + "85 Train loss: 0.0157778\n", + "85 Test accuracy: 0.978\n", + "86 Train loss: 0.00950592\n", + "87 Train loss: 0.0226222\n", + "88 Train loss: 0.0226599\n", + "89 Train loss: 0.0185005\n", + "90 Train loss: 0.0118967\n", + "90 Test accuracy: 0.976\n", + "91 Train loss: 0.0209059\n", + "92 Train loss: 0.0181153\n", + "93 Train loss: 0.0131697\n", + "94 Train loss: 0.017605\n", + "95 Train loss: 0.0193861\n", + "95 Test accuracy: 0.976\n", + "96 Train loss: 0.0156532\n", + "97 Train loss: 0.0136041\n", + "98 Train loss: 0.00743028\n", + "99 Train loss: 0.0267189\n" + ] + } + ], "source": [ "n_epochs = 100\n", "batch_size = 500\n", @@ -4810,7 +7467,25 @@ "cell_type": "code", "execution_count": 179, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Restoring parameters from ./my_digit_comparison_model.ckpt\n", + "0 Test accuracy: 0.9269\n", + "10 Test accuracy: 0.9675\n", + "20 Test accuracy: 0.9673\n", + "30 Test accuracy: 0.9673\n", + "40 Test accuracy: 0.9674\n", + "50 Test accuracy: 0.9673\n", + "60 Test accuracy: 0.9673\n", + "70 Test accuracy: 0.9673\n", + "80 Test accuracy: 0.9672\n", + "90 Test accuracy: 0.9673\n" + ] + } + ], "source": [ "n_epochs = 100\n", "batch_size = 50\n", @@ -4879,7 +7554,29 @@ "cell_type": "code", "execution_count": 181, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0 Test accuracy: 0.8893\n", + "10 Test accuracy: 0.9402\n", + "20 Test accuracy: 0.9479\n", + "30 Test accuracy: 0.9474\n", + "40 Test accuracy: 0.9479\n", + "50 Test accuracy: 0.9475\n", + "60 Test accuracy: 0.9475\n", + "70 Test accuracy: 0.9475\n", + "80 Test accuracy: 0.9476\n", + "90 Test accuracy: 0.9476\n", + "100 Test accuracy: 0.9473\n", + "110 Test accuracy: 0.9472\n", + "120 Test accuracy: 0.9474\n", + "130 Test accuracy: 0.9474\n", + "140 Test accuracy: 0.9475\n" + ] + } + ], "source": [ "n_epochs = 150\n", "batch_size = 50\n", From e5fb7fb175d9137666d0affae2456860887451f6 Mon Sep 17 00:00:00 2001 From: Chris Qlasty Date: Fri, 3 Nov 2017 12:04:54 +0100 Subject: [PATCH 22/26] Comments fix --- 11_deep_learning.ipynb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/11_deep_learning.ipynb b/11_deep_learning.ipynb index db13ebd..479fab5 100644 --- a/11_deep_learning.ipynb +++ b/11_deep_learning.ipynb @@ -4640,12 +4640,12 @@ " random_state=42, verbose=2)\n", "rnd_search.fit(X_train1, y_train1)" " - "#fit_params as a constructor argument was deprecated in [scikit-learn] version 0.19 and will be removed\n", - "#in version 0.21. Pass fit parameters to the fit method instead:" - "#rnd_search = RandomizedSearchCV(DNNClassifier(random_state=42), param_distribs, n_iter=50,\n", + "# fit_params as a constructor argument was deprecated in [scikit-learn] version 0.19 and will be removed\n", + "# in version 0.21. Pass fit parameters to the fit method instead:" + "# rnd_search = RandomizedSearchCV(DNNClassifier(random_state=42), param_distribs, n_iter=50,\n", "# random_state=42, verbose=2)\n", - "#fit_params={\"X_valid\": X_valid1, \"y_valid\": y_valid1, \"n_epochs\": 1000}" - "#rnd_search.fit(X_train1, y_train1, **fit_params)" + "# fit_params={\"X_valid\": X_valid1, \"y_valid\": y_valid1, \"n_epochs\": 1000}\n" + "# rnd_search.fit(X_train1, y_train1, **fit_params)\n", ] }, { From 99b91ae9890823a4e3b16e67459836720148d06a Mon Sep 17 00:00:00 2001 From: Chris Qlasty Date: Fri, 3 Nov 2017 12:07:37 +0100 Subject: [PATCH 23/26] Update 11_deep_learning.ipynb --- 11_deep_learning.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/11_deep_learning.ipynb b/11_deep_learning.ipynb index 479fab5..2c94b59 100644 --- a/11_deep_learning.ipynb +++ b/11_deep_learning.ipynb @@ -4639,7 +4639,7 @@ " fit_params={\"X_valid\": X_valid1, \"y_valid\": y_valid1, \"n_epochs\": 1000},\n", " random_state=42, verbose=2)\n", "rnd_search.fit(X_train1, y_train1)" - " + "\n", "# fit_params as a constructor argument was deprecated in [scikit-learn] version 0.19 and will be removed\n", "# in version 0.21. Pass fit parameters to the fit method instead:" "# rnd_search = RandomizedSearchCV(DNNClassifier(random_state=42), param_distribs, n_iter=50,\n", From 2a02668e5e5e328575667b1d4116e9158c6ecc6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Geron?= Date: Fri, 3 Nov 2017 13:43:56 +0100 Subject: [PATCH 24/26] Fix JSON formatting in new comment in the ch 11 notebook --- 11_deep_learning.ipynb | 2846 ++-------------------------------------- 1 file changed, 78 insertions(+), 2768 deletions(-) diff --git a/11_deep_learning.ipynb b/11_deep_learning.ipynb index 2c94b59..c002217 100644 --- a/11_deep_learning.ipynb +++ b/11_deep_learning.ipynb @@ -92,25 +92,7 @@ "cell_type": "code", "execution_count": 3, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Saving figure sigmoid_saturation_plot\n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3Xl4Tdf6wPHvisyDOVJEDTXGPLXILTFXUUNoqbHaKlo/\nLUqvoVeq1dYY91YHnaKCKkUNNZaosYSG0opWY4gIgpDIJMn6/bGPNMMJCSc5Gd7P8+wn2Xuvs9d7\ntiPvWXuvvZbSWiOEEEIUNDbWDkAIIYQwRxKUEEKIAkkSlBBCiAJJEpQQQogCSRKUEEKIAkkSlBBC\niAJJEpR4IEqpIKXUR9aOA3IWi1LqhFJqRj6FlL7eAKXUxnyox0cppZVS5fOhrpFKqfNKqVRrnNNM\nsQxXSsVaMwaRd5Q8ByUyU0q5A37A00BFIBo4AXygtd5uKlMWuKO1jrFaoCY5iUUpdQJYrbWekUcx\n+AC7AHetdVS67aUw/p9FW7Cus8BHWuu56bbZA2WByzoP/1MrpcoAV4DxwGogRmudLwlCKaWB/lrr\n1em2OQFuWusr+RGDyF+21g5AFEjfA87Ai8BfQAWgHVDubgGt9XXrhJZVQYolM631zXyqJwmIzIeq\nqmL83diotb6UD/Xdk9Y6Hoi3dhwij2itZZElbQFKAxrodJ9yQRjf4u+uewDrMf5YnANewGh1zUhX\nRgOjgR+AOOA00B7wBLYCt4EQoFmmuvoCvwGJwAVgKqbWfzaxVDDVcTeWEZljMfN+HjO9JtIUx1Gg\nR6Yy9sAs0zETgb+B/wOqmd5b+iXA9JoAjD/mACOBy0CJTMddDqzPSRym95qhLtN2H9N6+Vyct7PA\nNOAz4BYQDrx5j3M03Mz7rAbMAE6YKRubbn2G6d9gAHAGiAHWpY/XVG5YupgvA0vSxZq+3rPm6jFt\newXji1WS6efLmfZr07/FKtM5/hsYbO3/e7JkXeQelMgs1rQ8o5RyzMXrlmB8u+4A9AIGm9YzmwZ8\nCzQGgk2/fwl8DDQFIjD+qAOglGqO8YdkDdAQeAv4N/DaPWIJAGoCnYDewFCMP6T34gpsBjqbYvse\nWKOUqpvpPQ7FuLxVD6OFGY3xx9/XVKY+xmXRcWbqWAWUMtVx9/25YpyvwBzG0RcjkbxjqqeiuTeT\ni/P2BkZCaAZ8CMxWSrU2d0xgJfCU6ffHTXVfyKasOdWA54A+QBeMf+/30sX8Ckay/BpohHGJ+YRp\nd0vTz5dN9d5dz0Ap1Qf4CPAHGgALgY+VUj0zFX0b44tAY9P7+kop9Wgu3ovID9bOkLIUvAXjj+11\nIAE4AMwFnshUJghTqwWog/GttFW6/VWAFLK2oN5Pt97AtG18um0+pGsJAMuAnZnqngGEZxNLbdPr\nvdPtr5o5lhyeh4PANNPvtUzHfSqbshniTrc9AFMLyrS+Bliabn0wcBNwzEkcpvWzwMR71Z/D83YW\nWJGpzJ/p6zITSwtTPdUyHTcnLagEoFS6bVOBv9Kth2Pc58yubg30u089+4CvzPwb7L3H59AWo0Uv\nragCtkgLSmShtf4eqAT0xPg23wY4qJSaks1L6gKpGC2iu8e4gNEayux4ut8vm37+ZmZbBdPPehh/\ndNLbC1RWSpU0c/x6plgOpYvlXDaxpFFKuSilZiulfldK3TD1DGsB3P1W3dR03F33Ok4OBAK9lVLO\npvVBwPda64QcxpFTOT1vxzOVieCfc29p53TGe3JpdSmlKgCVgZ8eso7s3rdXpm1p71trnQxcJe/e\nt3hAkqCEWVrrBK31dq31O1rrNhiX4WaYeos9jDvpq7nHtpx8Nu/VWy23PdnmAv2B6RgdQppgJLmH\nfb+ZbQKSgV6mP8qd+OfyXn7Fkf7c3DGzL7d/F1IBlWmbnZlylqjrQWX+PFgzFpFD8g8icup3jEsh\n5u5LncL4LDW/u0Ep5YnRCntYfwDembb9C+NSlblu5XdjeTxdLI/mIJZ/Ad9orb/XWh/HuNz0WLr9\nIabjts/m9UmmnyXuVYnWOhHj3tAgjPsxkRiXKHMax9267lkPuT9vD+Mq4KGUSp+kmuTmANroJn4R\n6HiPYnd48Pf9e27iEQWDJCiRgVKqnFJqp1JqsFKqkVKqulKqPzAJ+ElrfSvza7TWoRi98D5VSrVS\nSjXBuNEdR+5bMpnNA9oppWYopWorpQYBE4DZ5gqbYtkCfKaUam2KJYD7d0U+DfRRSjVTSjXEaNWk\nJWOt9WngO+ALpZSv6bw8qZQaYipyDuO9dldKuZs6P2QnEOgKjMK4B5Sa0zhMzgJPKqUq3+PB3Fyd\nt4cUhPEM1hSl1GNKqReBfg9wnPeA15VSb5hibqKUmpBu/1mgo1LqEdPzWObMAYYopV5VStVSSo3F\n+DKQF+9b5DFJUCKzWIyb8uOA3cBJjK7VyzG+8WdnOMa3/SCM7ubLMB7oTHiYYLTWRzEueflieljY\ntNxr5IjhQBiwE9hgiv3sfaoab4p3D8Z9t4Om39MbajrWfzFaagEYvfLQWl8E/oPxR/byfeLbg9Fa\n8CLj5b2cxvE2RieUMxitlywe8Lw9EK31HxiPD4zEuLfTGeMzk9vjfAK8itFT7wTGF4366YpMwGjB\nXgB+zeYY64CxGL0Tf8f4HI/RWm/IbTzC+mQkCZEnTN/sI4CBpk4XQgiRKzKShLAIpVQHwA2jR14F\njJZEFMa3YCGEyDWLXeJTSr2mlApWSiUqpQLuUW6YUuqIUuqWUirc1KVWEmXhZwe8i5GgNmDcf2qr\ntb5t1aiEEIWWxS7xKaX6YnQ37Qo4aa2HZ1NuNMb15V8Ad4z7Fau01h9YJBAhhBBFgsVaLlrrNQBK\nqRYYY6tlV+6TdKsXlVLLyL7rrhBCiGKqIFxaa4vRU8wspdRIjN5BODk5Na9SpUp+xZUjqamp2NhI\nZ8j7kfOUMxcuXEBrzaOPyrBw95Pfn6nIhEicSzhT0s7cACYFV0H8v3f69OkorbX7/cpZNUEppUZg\nDOPyUnZltNaLgcUALVq00MHBwdkVtYqgoCB8fHysHUaBJ+cpZ3x8fIiOjiYkJMTaoRR4+fmZmrx9\nMrP3z2aCzwTebvd2vtRpKQXx/55S6lxOylktQSmlegPvY0zrEHW/8kIIYQ3zD8xn9v7ZjGkxhult\np1s7nGLFKglKKfUU8DnQXWv92/3KCyGENSw7vowJ2ybQz6sf/+32XzKO5iTymsUSlKmruC3GWFkl\nTHMJJZtGCk5frgPGKAN9tNaHsh5JCCEKhrDoMNpXa09gn0BK2NxvGEBhaZa8czYNY7yztzDmuIkH\npimlHlVKxaabDGw6xvAwP5q2xyqlNlswDiGEeCgpqSkATGs7ja2Dt+Jg62DliIoniyUorfUMrbXK\ntMzQWp/XWrtqrc+byrXXWtuatt1dulkqDiGEeBihUaE0+KQBhy4aF3jsSpibOUTkh4LQzVwIIQqE\niJgIugZ2Je5OHGUcsxswXeQXSVBCCAFEJ0TzVOBTXIu/RtCwIGqVq2XtkIo9SVBCiGIvITmBXt/2\n4lTUKTY9v4nmlZrf/0UizxWsx4uFEMJKyjuX55s+39D5sc7WDkWYSAtKCFFsaa2JuxOHi70Lq/uv\nluecChhpQQkhii2/3X60/rI10QnRkpwKIElQQohi6dPgT/Hb7UeLSi0o5VDK2uEIMyRBCSGKne9/\n/54xm8bQo3YPFvdcLK2nAkoSlBCiWNlzbg/Pr3meVp6tWNlvJbY2ciu+oJIEJYQoVqqXqc4zdZ5h\n4/MbcbZztnY44h7kq4MQoli4HHuZ8s7l8Szpyar+q6wdjsgBaUEJIYq8K7ev8K+v/8UrG1+xdigi\nFyRBCSGKtJjEGLov787FWxcZ0XSEtcMRuSCX+IQQRVZSShK+3/ny66VfWfvcWtpUaWPtkEQuSIIS\nQhRZozeOZvvf2/nqma/oWaentcMRuSQJSghRZL3U7CUaeTTihaYvWDsU8QAkQQkhipzfLv9GQ4+G\ntK7SmtZVWls7HPGApJOEEKJIWRKyhEafNmLtH2utHYp4SJKghBBFxqbTm3hx/Yt0qtGJ7rW7Wzsc\n8ZAkQQkhioSD4Qfpv6o/TR5pwppn12Bfwt7aIYmHJAlKCFHoXY+/To/lPahcsjI/DvoRNwc3a4ck\nLEA6SQghCr2yTmVZ0HUB3o96U8GlgrXDERYiLSghRKF1Pf46v4T/AsCQxkOoUaaGlSMSlmTRBKWU\nek0pFayUSlRKBdyn7BtKqUil1C2l1FdKKQdLxiKEKNoSUhLouaInTy17iuiEaGuHI/KApVtQEcC7\nwFf3KqSU6gq8BXQEqgI1AD8LxyKEKKKSU5OZ+cdMDlw4wOc9P6e0Y2lrhyTygNJaW/6gSr0LeGqt\nh2ezfzlwVms9xbTeEVimtX7kXsd1c3PTzZs3z7Dt2WefZcyYMcTFxfH0009nec3w4cMZPnw4UVFR\n9OvXL8v+0aNH89xzz3HhwgWGDBmSZf+ECRPo2bMnoaGhvPJK1pGQe/bsyYQJEwgJCeH111/Psn/W\nrFm0adOG/fv3M2XKlCz7/f39adKkCTt27ODdd9/Nsv+zzz6jTp06bNiwgXnz5mXZv3TpUqpUqcLK\nlSv55JNPsuxfvXo15cuXJyAggICAgCz7f/zxR5ydnfn444/57rvvsuwPCgoCYO7cuWzcuDHDPicn\nJzZv3gzAzJkz+emnnzLsL1euHN9//z0AgwYN4uLFixn2e3p6EhgYCMDrr79OSEhIhv21a9dm8eLF\nAIwcOZLTp09n2N+kSRP8/f0BGDx4MOHh4Rn2t27dmvfffx8AX19frl27lmF/x44dmT59OgDdunUj\nPj4+w/4ePXowceJEAHx8fMgsLz57ISEhJCcn06JFi/t+9qZNm0anTp2K3WdPozlT/wwXK1zk46c/\nJmpL1D0/e//+9785cOBAhv3F6bPXqVMnSpfOmMAf9u/ew372du/efURr3SLLjkys1UmiPvBDuvVj\ngIdSqpzWOsO/pFJqJDASwM7OjujojE3506dPExQUREJCQpZ9AKdOnSIoKIibN2+a3X/y5EmCgoK4\ncuWK2f2//fYbbm5unD9/3uz++Ph4goKC+Ouvv8zuP3r0KElJSZw4ccLs/uDgYKKjozl27JjZ/b/8\n8guXLl3it99+M7v/wIEDnDlzhpMnT5rdv2/fPkqVKsWpU6fM7v/5559xdHTk9OnTZvff/SNx5syZ\nLPvvvneAsLCwLPtTU1PT9iclJWXZb2dnl7Y/PDw8y/6IiIi0/REREVn2h4eHp+2/fPlylv3nz59P\n23/16lVu3bqVYX9YWFja/uvXr5OYmJhh/5kzZ9L2mzs3efHZS05ORmtNdHT0fT97x44dw9bWtth9\n9m543uBihYsMqDiAerfr8U3YN/f87Jk7f8Xps5eSkpKlzIP83dPahtRUF1JSnNm69QJ//nmEv/++\nxPnz9UlNdUBrR1JTHUhNdeTDD6FMmbNcvOjAyZMjSU11JDXVEa0dSE11AJ7MUqc51mpBnQFe1Vpv\nMa3bAUlAda312eyO26JFCx0cHGzxeB9GUFCQ2W84IiM5Tznj4+NDdHR0lm/04h8pqSms+n0VHlc9\naN++vbXDKfCCgoJo186H27fh2jW4ft1Y0v9+4wbExBjLrVv//J5+PS7OklGpAt2CigVKplu/+3uM\nFWIRQhQCG09vpLFHY6qUqsKABgPSWhjFVXw8XL4MkZH//Ey/REUZSSgysg2xsXDnzsPX6eb2z+Lq\nCs7O4OT0z5J5PbttPXrkrD5rJaiTQGPg7oXnxsDlzJf3hBACYMffO+i7si++Xr6s8F1h7XDyXFIS\nhIfDuXNw/ryx3P39wgW4dAlu3szp0YwRNZycoGxZKFfO+Hl3KVcOSpeGkiX/ST7mfndxAZuH6FZ3\n+vRpzp8/T6dOnXL8GosmKKWUremYJYASSilHIFlrnZyp6DdAgFJqGUbPv2lAgCVjEUIUDUcijtBn\nZR/qlq/LJ92zdsYorKKj4c8/jeX0aePnmTNGEoqMhPvdfbGzAw8PeOSRf5b06+XLG8knNHQ/3bu3\nwckpf96XOcuXL+eFF16gWbNm1ktQGInmP+nWBwN+SqmvgN8BL631ea31FqXUbGAX4AR8n+l1QgjB\nX9f/4unlT1POqRxbBm8pdN3JtTYuv/32m7GcOAGhoUYyuno1+9eVKAGVK8Ojj0LVqsbPu0uVKsa+\nMmVAqfvHcO1aktWSU0JCAmPGjGHlypUkJSWRmpqaq9dbNEFprWcAM7LZ7Zqp7HxgviXrF0IULW9u\nf5OU1BS2Dt5KJbdK1g7nnpKTjQR05AgcP/5PUoqKMl/eyQlq1TKW2rWNnzVrQrVqULEi2Bbygej+\n/vtvnn76ac6fP5/WjT63nfIK+SkQQhRlAb0COHfzHHXK17F2KBloDWFhcOjQP8vRo0bHhcxKloQG\nDaBhQ2OpV89ISJUqPdw9nYJszZo1DBs2jLi4uAytJqu2oIQQ4mElJifywd4PmOQ9iVKOpWjk2Mja\nIZGcbCSg3bvh55/h4EHzLaPHHoOWLaFRo38S0qOP5uxSXFFw584d3njjDb766qssDx+DtKCEEIVY\nSmoKg9cOZvXvq2lRqYXVJh1MTTUS0o4dRlLauxdiYzOWcXeHxx//Z2nZ0uiUUFydP3+eHj168Ndf\nf5lNTiAJSghRSGmtGbdlHKt/X83cznPzPTlduQLbtsGWLcbPzJ0YatWCdu2MxdvbuFdUXFpG97Np\n0yYGDhxIXFwcKSkp2ZaTS3xCiEJp1p5ZLDq8iImtJzKhzYR8qfOPP2DNGli3DjIPUlO1KnTpAh06\nQNu2xj0jkdWUKVPw9/fPttWUnrSghBCFzrW4a/j/4s+QRkP4sPOHeVaP1kYvuzVrYO1aOHXqn32O\njkbr6KmnjKVOHWkh5UR4eDhaa0qUKHHP1hNIghJCFELlnMtx6KVDeJb0xEZZvmvbn3/C0qUQGGj0\nvrurbFl45hno2xc6djSG5RG588033zB16lSmTZvGxo0bSUxMzDYRSYISQhQae87tYdfZXUxvO53q\nZapb9NhRUbBypZGYfvnln+0VKxoJqW9f49JdYX/eqCCoU6cO3333HY0aNeLEiRPZlpMEJYQoFH67\n/BvPfPsMHi4evN7qdUo6lLz/i+5DawgKgk8/NS7h3R0g1dUVfH1hyBDw8TFGahCWtXPnTsLSN08x\n5oy7c+cOycnGaHfSSUIIUeCdiz7HU8uewtnOma2Dtz50coqOhiVLjMR0976SjY1xL2nIEOjdWy7f\n5bVJkyZx+/btDNsqVKiAj48PK1eu5M6dO9KCEkIUbFFxUXQN7MrtpNvseWEPVUtXfeBjnToF8+bB\nsmX/jOJQsSK8/LKxeHpaKGhxT7t37yY0NDTDNldXV2bPns2zzz7LzJkz8fPzy1FPv/QkQQkh8tXB\n8INcjLnIj8//SEOPhg90jH37YNq0Buzb98+2Tp1g9Gjo2dMY6VvknzfffDNL66ls2bL069cPgCpV\nqvDFF1/k+riSoIQQ+apH7R6cHXeWcs65G3YhNRU2bIDZs2H/foDyODjA8OHwxhtGt3CR//bs2cPJ\nkyczbHNxceGDDz7A5iEHGyyiQxUKIQqSVJ3KyA0jWfPHGoBcJSet4YcfoEkT417S/v3GVBNDhpzl\n3DnjvpMkJ+uZNGkScZnmgy9TpgzPPvvsQx9bEpQQIs+9teMtPj/6Ob9f/T3Hr9HaGHbo8ceNxPTb\nb8Y9JX9/Y1K/ESPO4uGRh0GL+9q/fz/Hjx/PsM3V1ZX333+fEhboKimX+IQQeWre/nnM2T+HV1u+\nytQnp+boNbt3w7RpxiCtYMwUO3Wq0fHB0TEPgxW5Yq71VLJkSQYMGGCR40uCEkLkmcDjgUzcPpH+\nXv1Z+NRC1H3GDvrrL5g40bikB8bo4JMnw6uvSjfxguaXX37h119/zbDN1dWVWbNmYWuhp58lQQkh\n8syxyGO0r9aepX2WUsIm+0s+N2/Cu+/CwoXGw7UuLjBpErz+ujHhnyh4Jk+enKX15OLiwqBBgyxW\nhyQoIYTFpepUbJQNszvPJiklCQdbB7PlUlLgq6+My3lXrhjbhg+HWbOM55lEwRQcHMyhQ4cybHN1\ndeW9996zWOsJpJOEEMLCQqNCafpZU367/BtKqWyT06+/whNPwMiRRnLy9obDh+HrryU5FXSTJ08m\nISEhwzYnJyeGDh1q0XokQQkhLObirYt0CezCpZhLONk5mS0TF2dcvmvZ0pj6okoV+PZb2LMHWrTI\n54BFrv36668cOHAgw7BFLi4uzJw5EzsLPyEtl/iEEBYRnRDNU8ue4nr8dYKGBVGzbM0sZbZtg1Gj\njCkvbGyMe0wzZxqDuYrC4a233jLbenrhhRcsXpckKCHEQ4u/E88zK54hNCqUHwf9SPNKzTPsv34d\nxo0z5mMCaNQIvvjCaEWJwuP48ePs2bMnS+vJz88Pe3t7i9dn0Ut8SqmySqm1SqnbSqlzSqnnsynn\noJT6VCl1WSl1XSm1QSlV2ZKxCCHyT3JqMo62jizts5RONTpl2LdtGzRoYCQnR0f44ANjenVJToXP\nW2+9RWJiYoZtDg4OvPjii3lSn6VbUIuAJMADaAJsUkod01qfzFRuHNAaaATcBBYD/wP6WjgeIUQe\n0lqTmJKIm4MbWwdvzfCcU1yc8QzTRx8Z697eEBAANbNe+ROFwPXr19myZUuW1tOMGTNwcDDfEeZh\nWawFpZRyAXyB6VrrWK31XmA9MMRM8erAVq31Za11ArASqG+pWIQQ+cNvtx8+AT7EJMZkSE5HjkDz\n5kZysrU1uo3v3i3JqTArW7Ys27dvp2nTpri4uABgb2/Pyy+/nGd1WrIFVRtI1lqfTrftGNDOTNkv\ngYVKqUpANDAI2GzuoEqpkcBIAA8PD4KCgiwY8sOLjY0tcDEVRHKeciY6OpqUlJRCca5+iPgB/z/9\n6fZIN4L3B6OUIjUVVqx4lK+/rkZKig1Vq95mypQ/qF07lj17LFu/fKZyxpLnqUSJEsyfP5+QkBA+\n//xzunXrxsGDBy1ybLO01hZZgCeByEzbXgaCzJQtBXwLaCAZ+BUoe786mjdvrguaXbt2WTuEQkHO\nU860a9dON27c2Nph3Neqk6u0mqF0j+U99J2UO1prra9e1fqpp7Q2hnnV+v/+T+u4uLyLQT5TOVMQ\nzxMQrHOQVyzZSSIWyDwoSUkgxkzZRYADUA5wAdaQTQtKCFGw7D67m0FrBtG6SmtW9luJrY0tBw5A\n06bG6OPlysGPPxrDFjmZfxRKiByxZII6DdgqpWql29YYyNxBAowOFAFa6+ta60SMDhKPK6XKWzAe\nIUQeqFyyMl0e68KGgRtwsnXG3x/atoXwcGjd2hghols3a0cpigKLJSit9W2MltA7SikXpZQ30AtY\naqb4YWCoUqqUUsoOGANEaK2jLBWPEMKyouKi0FpTs2xNNgzcQImksvTvb8xmm5xs/AwKMkaGEMIS\nLD3U0RjACbgCrABGa61PKqWeVErFpis3EUgA/gSuAk8DfSwcixDCQq7cvkLrL1szfut4AP78E1q1\ngu+/N0YbX70a5s+HPHhWUxRjFn0OSmt9HehtZvsewDXd+jWMnntCiAIuJjGGp5c9zcVbF3m2/rP8\n9BP07w83bhgP4K5dK93HCyIfHx8aNGhAv379rB3KA5PBYoUQ2UpKSaLvd30JiQzhu36rOPJDa7p2\nNZJTz56wf3/RSk5Xr15lzJgxVKtWDQcHBzw8POjYsSPbt2/P0euDgoJQShEVlX93KwICAnA1M5jh\nmjVreP/99/MtjrwgY/EJIbL18oaX2fH3Dj5/egkb5ndn8WJj+7//bUwwaFPEvuL6+voSFxfHl19+\nSc2aNbly5Qq7d+/m2rVr+R5LUlLSQ41vV7ZsWQtGYx1F7OMlhLCkwQ0HM7PVIpZNGsrixeDgAMuW\nGSNDFLXkFB0dzZ49e/jggw/o2LEjVatWpWXLlkycOJEBAwYAEBgYSMuWLXFzc6NChQr079+fixcv\nAnD27Fnat28PgLu7O0ophg8fDhiX21577bUM9Q0fPpwePXqkrfv4+DB69GgmTpyIu7s73t7eAMyf\nP59GjRrh4uJC5cqVeemll4iOjgaMFtsLL7zA7du3UUqhlGLGjBlm66xWrRrvvvsur7zyCiVLlsTT\n05M5c+ZkiOn06dO0a9cOR0dH6tSpw48//oirqysBAQGWOcm5VMQ+YkIISwiNCgWgtm1nlr8xhqAg\nYxLBPXvgebNDQBd+rq6uuLq6sn79+izTSdyVlJSEn58fx44dY+PGjURFRTFw4EAAqlSpwvfffw/A\nyZMnuXTpEgsXLsxVDIGBgWit2bNnD9988w0ANjY2+Pv7c/LkSZYvX86hQ4cYO3YsAG3atMHf3x9n\nZ2cuXbrEpUuXmDhxYrbHX7BgAQ0bNuTo0aNMnjyZSZMmceDAAQBSU1Pp06cPtra2HDx4kICAAPz8\n/LIMDpuf5BKfECKDgJAAXlz/Iv9t8jPvvuJNZKTRGWLzZvD0tHZ0ecfW1paAgABefvllFi9eTNOm\nTfH29qZ///488cQTAIwYMSKtfI0aNfjkk0+oV68e4eHheHp6pl1Wq1ChAuXL5/6xzurVqzNv3rwM\n215//fW036tVq8bs2bPp1asXS5Yswd7enlKlSqGU4pFHHrnv8bt06ZLWqho7diz//e9/+emnn2jd\nujXbt28nNDSUbdu2UbmyMbnEggUL0lpy1iAtKCFEmk2nN/HS+pdocvtN3hrUhshI8PExWk5FOTnd\n5evrS0REBBs2bKBbt27s37+fVq1aMWvWLACOHj1Kr169qFq1Km5ubrQwTQF8/vx5i9TfvHnzLNt2\n7txJ586d8fT0xM3Njb59+5KUlERkZGSuj9+oUaMM65UqVeLKlSsAnDp1ikqVKqUlJ4CWLVtiY8Vr\nuZKghBAAHLhwgP6r+lPl7FSOz3+f2FjFgAHG8EWlS1s7uvzj6OhI586defvtt9m/fz8vvvgiM2bM\n4ObNm3Tt2hVnZ2eWLl3K4cOH2bJlC2Bc+rsXGxubDNNUANy5cydLubujhN917tw5unfvTr169Vi1\nahVHjhxrAd4+AAAgAElEQVThq6++ylGd5mSekt0Y4Dc118fJL5KghBBcuX2FHit64PzLO5z92o/k\nZMWbbxodIvJoqp9Cw8vLi+TkZEJCQoiKimLWrFm0bduWunXrprU+7rrb6y4lJSXDdnd3dy5dupRh\n27Fjx+5bd3BwMElJSSxYsIDWrVtTu3ZtIiIistSZub4HUbduXSIiIjIcPzg42KoJTBKUEAJ35wq0\nOLmdaxsmopQx0Ovs2UWvp969XLt2jQ4dOhAYGMjx48cJCwtj1apVzJ49m44dO+Ll5YWDgwMfffQR\nf//9N5s2bWL69OkZjlG1alWUUmzatImrV68SG2sMoNOhQwc2b97M+vXrCQ0NZfz48Vy4cOG+MdWq\nVYvU1FT8/f0JCwtjxYoV+Pv7ZyhTrVo1EhIS2L59O1FRUcTFxT3Q++/cuTN16tRh2LBhHDt2jIMH\nDzJ+/HhsbW0zzPWVn4rRx08Ikdn1+OuEXDrOmDGw7Ztm2Noarab/+z9rR5b/XF1dadWqFQsXLqRd\nu3bUr1+fKVOm8Pzzz7Ny5Urc3d1ZsmQJ69atw8vLCz8/P+bPn5/hGJUrV8bPz4+pU6fi4eGR1iFh\nxIgRaYu3tzdubm706XP/0d0aNWrEwoULmT9/Pl5eXnzxxRfMnTs3Q5k2bdowatQoBg4ciLu7O7Nn\nz36g929jY8PatWtJTEzk8ccfZ9iwYUydOhWlFI6Ojg90zIeWkzk5Csoi80EVXnKeciY/54O6nXRb\nt/rsX9qh2XcatHZw0HrDhnyp2iLkM5UzD3OeQkJCNKCDg4MtF5DO+XxQ0s1ciGIoOTWZfisGcXDB\neDjVB1dXWL8eTM+ZimJq7dq1uLi4UKtWLc6ePcv48eNp3LgxzZo1s0o8kqCEKGa01ryw6jU2+42G\nv7tQpozxjJPpUR9RjMXExDB58mQuXLhAmTJl8PHxYcGCBVa7ByUJSohi5rMDgQROHgDnfPDwgG3b\nINPjMaKYGjp0KEOHDrV2GGkkQQlRjMTGwvK3BsE5GypV0uzapahd29pRCWGe9OITophY/etWOnVJ\nYs8eGypXht27JTmJgk1aUEIUAz8c28mzvV3R5+3x9IRdu4rWPE6iaJIEJUQRtzv0V/o+44w+34rK\nnqkEBdnw2GPWjkqI+5NLfEIUYSHn/qJT1zuknm9FJc9kft4tyUkUHpKghCii4uOhY7dYks89TsXK\nd9iz25YaNawdlRA5JwlKiCIoMRH69oXrfzShvMcdfg6yk+QkCh1JUEIUMbHxiTTvEsqWLVC+POze\naScdIkShJAlKiCIk6U4KXp0Pc/LnOriWvMP27eDlZe2ohHgwFk1QSqmySqm1SqnbSqlzSqnn71G2\nmVLqZ6VUrFLqslJqnCVjEaK4SUnRNOnxCxf2/QsH5yR2bLOjSRNrRyXEg7N0N/NFQBLgATQBNiml\njmmtT6YvpJQqD2wB3gBWA/ZAMZhQWoi8oTV49w/mj21tsHVIYttmexlbTxR6FmtBKaVcAF9gutY6\nVmu9F1gPDDFTfDywVWu9TGudqLWO0Vr/YalYhChOtIax42P5ZW1LbGzvsHG9LW3bWjsqIR6eJVtQ\ntYFkrfXpdNuOAe3MlG0F/KaU2g/UBH4BXtVan89cUCk1EhgJ4OHhQVBQkAVDfnixsbEFLqaCSM5T\nzkRHR5OSkpKrcxUY+ChfflmDEiVS+c9/TuJgH01xONXymcqZwnyeLJmgXIFbmbbdBNzMlPUEmgGd\ngd+A2cAKwDtzQa31YmAxQIsWLbSPj4/lIraAoKAgClpMBZGcp5wpXbo00dHROT5Xkz74ky+/rIFS\nsGyZDc89V3xuOslnKmcK83myZIKKBUpm2lYSiDFTNh5Yq7U+DKCU8gOilFKltNY3LRiTEEXWgq/O\nMWeK8XDT/IWJPPecg5UjEsKyLNmL7zRgq5SqlW5bY+CkmbLHAZ1uXZspI4TIxooNkYwf+QjoEoz/\ndzSvj5XkJIoeiyUorfVtYA3wjlLKRSnlDfQClpop/jXQRynVRCllB0wH9krrSYj727n/BoOfdYUU\nBwaMuMbc90pbOyQh8oSlH9QdAzgBVzDuKY3WWp9USj2plIq9W0hrvROYAmwyla0JZPvMlBDC8Ndf\n0LenE6kJrnToeZVln5fDSrNxC5HnLPoclNb6OtDbzPY9GJ0o0m/7BPjEkvULUZRdugRdusDN6460\n7ZDI5tXu2MhYMKIIk4+3EIXA9RupNGwTTlgYtGwJG9c5YG9v7aiEyFuSoIQo4OLjoXHbs1w760n5\nR6PYtAnczD28IUQRIwlKiAIsORladv2T8BM1cCl3g8O7y+Hubu2ohMgfkqCEKKC0ho79/+LknlrY\nucSyf1dJqlWTHhGi+JAEJUQBNWUK/LyuJjb2CWzfbE+jhiWsHZIQ+UoSlBAF0Lx5mg8+AFtbzerV\n0O5J6REhih9JUEIUMJfjujBxonEp7+uvFX16Olo5IiGsQxKUEAVI5M3mRP41C4Ap711l8GArBySE\nFUmCEqKA2LLrFqEnZoK25YWxkbw3RbrrieJNEpQQBUDwrwn07KEg2Rm3Siv5cuEj1g5JCKuTBCWE\nlZ07B890tyc5zg3XSjuoXn62jK8nBBYei08IkTtXrmg6d4FLl2xo106TmjqbW7dSrB2WEAWCJCgh\nrCQmBpq2jSDidGUaNkrlhx9s6NUrKUu59evXExISQsOGDalfvz6PPfYYJUrIM1Gi6JMEJYQVJCbC\n450uEBFaBTePK2zd4k6pUubLnjlzBj8/P1xdXUlJSSEpKQlPT08aNmzI448/ToMGDahfvz7Vq1eX\nxCWKFElQQuSzlBTweSacU4eq4FDqBof3lKVixexvOo0ePZp3332X69evp20LCwsjLCyMH3/8EWdn\n5wyJq1GjRjz++OMMGDCAGjVq5MdbEiJPSCcJIfKR1tB7yEUObvOkhFMMQTscqVPr3t8THR0deffd\nd3FxccmyLzk5mVu3bnH79m3u3LlDWFgYP/zwA9OmTePAgQN59TaEyBeSoITIR9OmwcYVlbGxS+SH\n9ZpWLZxy9LqXXnoJV1fX+xcE7O3t6dq1K88/L5NUi8JNEpQQ+eS92XHMmgUlSsC67x3o3qlkjl9r\nZ2fHhx9+aLYVlVnJkiVZtmwZSvqqi0JOEpQQ+eCjz28xbbIzAF99BT175v4YgwcPpmzZsvcs4+Dg\nQK1atR4kRCEKHElQQuSx79bEMXaUkZzG/ecsQ4c+2HFKlCjB3Llz79mKSkxM5MiRI9SpU4e9e/c+\nWEVCFBCSoITIQz/tusPAASUg1ZbnRv+F/4xqD3W8fv36UbFixXuWSUpKIioqii5dujB9+nRSUuTB\nX1E4SYISIo+EhEC3HndIveOAj28oKxbVfOhj2tjYsGDBgiytKEfHrFNyxMfHM3/+fJ544gnCw8Mf\num4h8ptFE5RSqqxSaq1S6rZS6pxS6p7diJRS9kqpP5RS8r9HFCl//gldu8KdOGcadwhlx8o6Fhtf\nr3v37lSvXj1t3dnZmVdffRVXV1dsbDL+l46LiyMkJAQvLy/WrVtnmQCEyCeWbkEtApIAD2AQ8IlS\nqv49yr8JXLVwDEJYVUQEdOh0hytXoHNn+OXHOlhygAelFP7+/jg7O+Pk5MTAgQOZO3cuJ06coGHD\nhjg7O2con5KSQkxMDIMGDeKll14iPj7ecsEIkYcslqCUUi6ALzBdax2rtd4LrAeGZFO+OjAYeN9S\nMQhhbVeuQAvvaMLP21Gv8S3WrAEHB8vX07FjR+rXr0/FihX53//+B0DVqlUJDg5m7NixODllfb4q\nLi6O5cuX06BBA37//XfLByWEhSmttWUOpFRTYJ/W2jndtolAO611lk61SqmNwJfADSBQa+2ZzXFH\nAiMBPDw8mn/77bcWiddSYmNjc/wAZXFWHM7TrVu2jBpXi0tnPXCq+BdLF4VTrkzujvH666+TkpKS\nlnTu5e7QR+a6noeEhPD2228THx9PcnJyhn1KKezt7RkzZgw9e/YstM9LFYfPlCUUxPPUvn37I1rr\nFvctqLW2yAI8CURm2vYyEGSmbB9gs+l3HyA8J3U0b95cFzS7du2ydgiFQlE/Tzdval2v8S0NWjt6\nnNVnzsc+0HHatWunGzdubJGYrl69qjt06KCdnZ01kGVxdnbW3bt31zdu3LBIffmtqH+mLKUgnicg\nWOfgb74l70HFApkfjS8JxKTfYLoUOBv4PwvWLYTV3L4NXbol8scxN2zLXeDgz67UqHL/ER/yWvny\n5dmxYwfvvfdetpf8duzYQe3atdm/f78VIhTi3iyZoE4Dtkqp9I+xNwZOZipXC6gG7FFKRQJrgIpK\nqUilVDULxiNEnktIgN694Zf9DpRyj+GnHdC4djlrh5VGKcXrr7/OgQMHqFKlSpbu6ImJiVy9epVO\nnToxY8YMeWZKFCgWS1Ba69sYyeYdpZSLUsob6AUszVT0BFAFaGJaXgIum36/YKl4hMhrSUnQu28S\nO3ZAhQrwyx432japYu2wzGrcuDF//PEHvr6+WXr5gfHM1Jw5c2jTpg0XL160QoRCZGXpbuZjACfg\nCrACGK21PqmUelIpFQugtU7WWkfeXYDrQKppXb6+iUIhORkGPp/M1s32KOcbbNycQJ061o7q3lxc\nXAgMDOSLL77I9pmpo0eP4uXlxfr1660UpRD/sGiC0lpf11r31lq7aK0f1VovN23fo7U2241Eax2k\ns+nBJ0RBlJICw19IZc33tuBwkw8CjtKyWdaRHAqqgQMHcvz4cerXr5+lNXV3fqmBAwfyyiuvkJCQ\nYKUohZChjooUHx8fXnvtNWuHUaSlpMALL2iWBdqAXSwTF+1iUv+O1g4r16pXr86RI0cYM2ZMth0o\nli5dSsOGDTl16pQVIhRCEhRXr15lzJgxVKtWDQcHBzw8POjYsSPbt2/P0etDQkJQShEVFZXHkf4j\nICDA7HMNa9as4f335bnnvJKSAsOGwdKlCuxiGT73O+a82NvaYT0wOzs75syZw4YNGyhTpgx2dnYZ\n9sfHx3PmzBmaN2/OF198cfcRESHyTbFPUL6+vhw6dIgvv/yS06dPs3HjRrp168a1a9fyPZakpKSH\nen3ZsmVxc3OzUDQiveRkGDoUli0DV1fNm5/8xFdjX7B2WBbRsWNHQkND8fb2znLJT2tNXFwc48aN\no1evXty8edNKUYpiKScPSxWUxdIP6t64cUMDevv27dmWWbp0qW7RooV2dXXV7u7uul+/fjo8PFxr\nrXVYWFiWhx+HDRumtTYeuHz11VczHGvYsGG6e/fuaevt2rXTo0aN0hMmTNDly5fXLVq00FprPW/e\nPN2wYUPt7OysK1WqpF988cW0hyl37dqVpc7//Oc/ZuusWrWqnjlzph45cqR2c3PTlStX1rNnz84Q\nU2hoqG7btq12cHDQtWvX1ps2bdIuLi7666+/fqBzmp2C+LBgTt25o/XAgVqD1q6uqXrv3ryry5IP\n6uZWamqqnjt3rnZycjL7YK+Dg4P28PDQBw4csEp8mRXmz1R+KojnCSs8qFvouLq64urqyvr167O9\nGZyUlISfnx/Hjh1j48aNREVFMXDgQACqVKmCn58fACdPnuTSpUssXLgwVzEEBgaitWbPnj188803\ngDGlgr+/PydPnmT58uUcOnSIsWPHAtCmTZu0gUIvXbrEpUuXmDhxYrbHX7BgAQ0bNuTo0aNMnjyZ\nSZMmceDAAQBSU1Pp06cPtra2HDx4kICAAPz8/EhMTMzVeyjKkpNhyBBYsQKwj6Hj9Ll4e1s7qryh\nlGLChAns27ePypUrm31m6vLly3To0IGZM2eSmppqpUhFsZGTLFZQlrwY6mj16tW6TJky2sHBQbdq\n1UpPmDBBHzx4MNvyf/zxhwb0hQsXtNZaL1iwQAP66tWrGcrltAXVsGHD+8a4efNmbW9vr1NSUrTW\nWn/99dfaxcUlSzlzLagBAwZkKFOzZk09c+ZMrbXWW7Zs0SVKlEhrEWqt9b59+zQgLSitdUKC1n36\nGC0nHG7qxyYO0dHx0XlapzVbUOnFxMToAQMG3HOYpFatWumIiAirxVgYP1PWUBDPE9KCyhlfX18i\nIiLYsGED3bp1Y//+/bRq1YpZs2YBcPToUXr16kXVqlVxc3OjRQtjfMPz589bpP7mzZtn2bZz5046\nd+6Mp6cnbm5u9O3bl6SkJCIjI3N9/EaNGmVYr1SpEleuXAHg1KlTVKpUicqVK6ftb9myZZbnY4qj\n27fhmWdg7VpQjjd5ZPQwfn77A0o5lrJ2aPnC1dWVFStWsHjxYlxcXMw+MxUcHEzdunXZtGmTlaIU\nRZ38JcKYjbRz5868/fbb7N+/nxdffJEZM2Zw8+ZNunbtirOzM0uXLuXw4cNs2bIFuH+HBhsbmyy9\nnu7cuZOlXOaZUc+dO0f37t2pV68eq1at4siRI3z11Vc5qtOczD2zlFJyaeY+oqONyQa3bQM7txuU\nGtWb3dM+pJJbJWuHlu8GDRrE8ePHqVevXrbPTPXv359XX31VLg0Li5MEZYaXlxfJycmEhIQQFRXF\nrFmzaNu2LXXr1k1rfdxla2sLkGUMM3d3dy5dupRh27Fjx+5bd3BwMElJSSxYsIDWrVtTu3ZtIiIi\nMpSxt7e3yJhpdevWJSIiIsPxg4ODi3UCu3oVOnSAffvA0xO270rkp0nzqF2utrVDs5oaNWrw66+/\nMnLkSLPPTMXHx7N48WK2bdtmhehEUVasE9S1a9fo0KEDgYGBHD9+nLCwMFatWsXs2bPp2LEjXl5e\nODg48NFHH/H333+zadMmpk+fnuEYHh4eKKXYtGkTV69eJTY2FoAOHTqwefNm1q9fT2hoKOPHj+fC\nhfsPNVirVi1SU1Px9/cnLCyMFStW4O/vn6FMtWrVSEhIYPv27URFRREXF/dA779z587UqVOHYcOG\ncezYMQ4ePMj48eOxtbUttHMEPYzwcGjbFn79Fcp7RrP75xTaNX+EZhWbWTs0q7Ozs2PBggWsW7eO\n0qVLZ2iZ29nZ0aZNG7p3727FCEVRVKwTlKurK61atWLhwoW0a9eO+vXrM2XKFJ5//nlWrlyJu7s7\nS5YsYd26dXh5eeHn58f8+fMzHMPd3R0/Pz+mTp2Kh4dH2kgOI0aMSFu8vb1xc3OjT58+942pUaNG\nLFy4kPnz5+Pl5cUXX3zB3LlzM5Rp06YNo0aNYuDAgbi7uzN79uwHev82NjasXbuWxMREHn/8cYYN\nG8bUqVNRSmXpwVXUhYbCk0/CqVNQ6tFzRD1Xlwtqr7XDKnC6dOlCaGgorVq1Srvk5+LiwqpVq+Te\npbC8nPSkKCiLTFiY90JCQjSgg4ODLXrcgnye9u7VumxZo7eeR52/NZPK6Pn751slloLSi+9+UlJS\n9Icffqjt7Oz0tm3brBJDQf5MFSQF8TyRw158ttZOkMK61q5di4uLC7Vq1eLs2bOMHz+exo0b06xZ\n8bistXYtPP+8Ma9T7danOd2+KZN8XuON1m9YO7QCzcbGhkmTJjFu3DgcHBysHY4ooqRNXszFxMTw\n2muv4eXlxaBBg6hXrx5bt24tFvegPvoIfH2N5DTohVjOP9WcYS3780GnD6wdWqEhyUnkJWlBFXND\nhw5l6NCh1g4jX6Wmwr//DXdv3b37LkyZ4sqkK/uoV75esUjOQhQGkqBEsRIXBy+8AN99B7a28OYH\noTzSfi9KvUgjj0b3P4AQIt/IJT5RbISHGz31vvsO3Nzgv0v/5uM7TzDvwDwSkmViPmupVq1alp6q\nQoC0oEQxcfAg9OkDkZHw2GPwSeBFhu37F672rmwZvAVH2+LVrT6/DR8+nKioKDZu3Jhl3+HDh7OM\nqCIEFIMWVGRkJN26dWPZsmUyFEsxtXQp+PgYyal9e9i0M4rXgjsQnxzP1sFbebTUo9YOsVhzd3fP\nMoySNTzsfGzC8op8gvr444/ZsWMHo0aNwt3dnTfeeCPLEESiaEpJgcmTjYkGExNhzBjYuhUOXN/I\nhZsX2DhwI/Ur1Ld2mMVe5kt8SikWL15M//79cXFxoUaNGgQGBmZ4zcWLF3nnnXcoU6YMZcqUoXv3\n7vz5559p+8+cOUOvXr145JFHcHFxoVmzZllab9WqVWPGjBmMGDGC0qVLM2jQoLx9oyLXinSCSk5O\nZtGiRSQnJxMbG0tMTAyLFi1iyZIl1g5N5LHLl6FLF6OnXokS8PHHsGgR2NnB8CbDCX0tFO9Hi+jE\nTkXAO++8Q69evTh27BjPPfccI0aMSJtBIC4ujvbt22Nvb8/u3bs5cOAAFStWpFOnTmnDfsXGxtKt\nWze2b9/OsWPH8PX1pW/fvpw6dSpDPfPnz6du3boEBwenzWAgCo4inaA2bdqUZQRxGxsbBg8ebKWI\nRH74+Wdo2hR27gQPD9ixA14ZlcprP77G/gv7AahSqoqVoxT3MmTIEAYPHkzNmjWZOXMmtra2/Pzz\nzwB8++23aK2ZPHkyjRo1om7dunz22WfExsamtZIaN27MqFGjaNiwITVr1mTq1Kk0a9aM1atXZ6in\nXbt2TJo0iZo1a1KrVq18f5/i3op0gpo9ezYxMTEZtj355JN4enpaKSKRl1JT4YMPjPtMly79M/Cr\njw9M2j6JRYcX8fO5n60dpsiB9POY2dra4u7unjaTwJEjRwgLC+Ppp59OmxW7VKlS3LhxgzNnzgBw\n+/ZtJk2ahJeXF2XKlMHV1ZXg4OAs87jdnd9NFEwW7cWnlCoLfAl0AaKAf2utl5sp9yYwDKhqKvex\n1nqOJWP5+++/OXr0aIZtbm5u95weXRRe16/DsGFw9zbDW2/BzJnGs05z989l3oF5vNbyNSZ7T7Zu\noCJH7jWPWWpqKk2aNOGNN97giSeeyFCubNmyAEycOJEtW7Ywd+5catWqhbOzM0OHDs3SEUJ6DxZs\nlu5mvghIAjyAJsAmpdQxrfXJTOUUMBQ4DjwGbFNKXdBaf2upQP73v/9lmTPJ2dmZzp07W6oKUUDs\n3Gkkp/BwKFMGvvkGevQw9n1z7Bve3P4mz9Z/Fv+n/GWUiCKgWbNmrFixglKlSlGzZk2zZfbu3cvQ\noUPx9fUFICEhgTNnzlC7dvGd16swsliCUkq5AL5AA611LLBXKbUeGAK8lb6s1jr9/BChSqkfAG/A\nIgkqMTGRL7/8MsP9JycnJ8aNGydTAhQhCQkwZQosWGCsP/EEfPstVKtmrGut2fzXZjpU78A3vb+h\nhE0Jq8Uq4NatW4SEhGTYVrp06VwfZ9CgQcydO5epU6fi5ubGo48+yoULF/jhhx8YNWoUtWrVonbt\n2qxdu5ZevXphZ2eHn58fCQnyMHZhY8kWVG0gWWt9Ot22Y0C7e71IGV9pnwQ+y2b/SGAkGJMDBgUF\n3TeQHTt2kJycnGFbcnIy9erVy9HrcyM2NtbixyyKLH2e/vrLhffe8+LsWRdsbDRDh55l8ODznD2r\nOXvWSE5KKV4q+xJJpZM4sPeAxerOS9HR0aSkpBS5z1RkZCR79uyhadOmGba3bds2rXWT/j2fPHmS\n8uXLp61nLvP+++/z8ccf07t3b27fvk25cuVo0qQJv//+OxcvXqR///7MmTMHb29vXF1d6devH15e\nXkRGRqYdw1y9RVGh/huVkzk5crJgJJnITNteBoLu8zo/jETmcL86cjofVJMmTTSQtiildK9evXI6\nVUmuFMS5VgoiS52n5GStP/xQazs7Y/6mWrW0/uWXjGX+uPqHbvd1O33h5gWL1JmfCst8UAWB/N/L\nmYJ4nrDCfFCxQMlM20oCMWbKAqCUeg3jXtSTWmuLDPNw4sQJQkNDM2xzdnaWzhFFwPHj8PLLcOiQ\nsT56NMyZA+nvc1+8dZGugV1JSE4gMVlGDhGiMLPkDZnTgK1SKv3DBI2BzB0kAFBKjcC4N9VRax1u\nqSD8/f2z9NQpX7483t7yUGZhFR9vTI/RvLmRnCpXNnrrffxxxuR0I/4GTy17ihvxN9gyaAuPlX3M\nekELIR6axRKU1vo2sAZ4RynlopTyBnoBSzOXVUoNAmYBnbXWf1sqhtu3b7N8+fIMvfecnZ2ZMGGC\n9N4qpH76CRo2NJ5vSkmBV1+F33+H7t0zlou/E88z3z7D6WunWTdgHU0rNjV/QCFEoWHpLm1jACfg\nCrACGK21PqmUelIpFZuu3LtAOeCwUirWtHz6sJUvX748Sy+91NTUYjchX1Fw6ZLRdbxTJzhzBurX\nh337jFlwS2a+kAzEJMUQmxTL0j5L6VC9Q/4HLISwOIs+B6W1vg70NrN9D+Cabr26Jes1HZM5c+Zw\n+/bttG02Njb4+vpSqlQpS1cn8khCgtFtfNYsiI0FBweYPh3efBPs7bOW11qTqlOp4FKBwy8fxtZG\nZpARoqgoMg8FBQcHExERkWGbo6Mj48ePt1JEIje0hu+/By8v49mm2Fjo1QtOnICpU80nJ4D/BP0H\n3+98SUpJkuQkRBFTZBLUvHnziI+Pz7CtatWqNGvWzEoRiZwKDjbGz+vXD8LCoEEDY4DXdesgm4EC\nAFh0aBEzf55Jeefy2NnYZV9QCFEoFYkEdePGDX744Ye0sboAXF1dpWt5AXf8uDHLbcuWsHs3lCtn\n9Mz79Vfo2PHer111chVjN4/lmTrP8GmPT6UTjBBFUJG4JhIQEJClc4TWmgEDBlgpInEvp07BjBmw\ncqWx7uQEY8caA7yWKXP/1+8M28ngtYPxftSbb32/lUt7QhRRhf5/ttaa+fPnp01UBsbw/EOGDCkQ\n00iLf/z+O3z4IQQGGlNj2NsbD9u+9RY88kjOj+Ni50Jrz9asfW4tTnZOeRewEMKqCn2C2r17N9HR\n0Rm22dnZMW7cOCtFJNLTGvbuhSlTGnDANByera0xIsTUqVAlF/MGxiTG4ObgxhOeT7Br2C65rCdE\nEVfo70HNmzeP2NjYDNu8vLyoW7eulSISYDxUu2YNtGljTBx44EB5HB1hzBgIDYVPP81dcroce5mm\nn8yIIesAAA6sSURBVDVl9j5jIHxJTkIUfYUqQcXHx7Nt27a0zhCXL19mx44dGcq4uroyadIka4Qn\ngCtXjMt4tWqBry8cPAhly8LQoWc5fx4WLYIaNXJ3zFuJt+i2rBsRMRG0rdo2bwIXQhQ4heoS37Vr\n1+jWrRsVKlRg3LhxREVFZSljY2ND795ZnhUWeejuZbxPPoHVq+HuNFzVqsH48TBiBBw+fBZ392q5\nPnZiciJ9V/bl+OXjrB+4nlaerSwauxCi4CpUCcrW1hYbGxsiIyN55513SEpKyjDunr29PSNHjsQ+\nu6c6hUVdvAjLl8OSJXDSNCSwUsZstqNHQ9euUOIh5gjUWjP8h+H8FPYTS3ov4elaT1smcCFEoVDo\nEpSDgwPJyclZHsoF477EkCFDrBBZ8REba9xbWrrUGMjVmNILPDzgpZeMzg9Vq1qmLqUUTz32FC0q\ntmBoYxlPUYjiptAlqBL3+EpeokQJnnjiCfr168f48eOzzN4pHkxsLGzZYgxFtH493O3Rb29vtJYG\nDzZGF7dkwzX8VjieJT0Z1mSY5Q4qhChUClUnCVtb23v23oqLiyMhIYHly5fTrFkzPv/883yMrmi5\nft24dNerF7i7Q//+8O23RnLy9jZ64f1/e/cfXFV95nH8/eSGREJ+CGIR5IdIYV2pJUgKSyklitXQ\nahU7Si21ZbsV1wIdpkut1nXGarvd6XRKO9aRUtktgsViS3fBiFVrg9KOsrCbqKwIZRHFEeVXIAmB\nEPLsH+deSWKSe0MunHNzP6+Z7+Sek++9eXLm5Dz53vO9z/fdd4OkNXNmepPTsv9exugHR/PynpfT\n96IiknEybgTV+p5TZ8455xwmTZrELbfcchai6h1aWqCmJhgpPf10sLRF60M9eTLceGPQujsLrzvW\nvrGWuU/O5aqLr9KaTiJZLuMSVPvVctsrKCjgpptu4pFHHiE3N6N+vbNuz56gBt4zz8Af/gDvvXfq\ne7FYsBbTzJlwww0wZMiZj+fPb/2ZWb+dxYTBE/jdzb8jL6bJLiLZLKOu4LFYjBOJOcwdKCgo4O67\n7+aee+7RBzk7sHt3kJASbefOtt8fOhQqKoI2fTqce+5ZjK12N9etuo5hxcOo/FIlhXmFyZ8kIr1a\nRiUoM6OgoKDNooQJBQUFLF26lNmzZ4cQWfQcPgxbtsCmTafaO++07VNUBJ/6FFx5JcyYEazFFFZe\nH1YyjAUTFzCndA7n9zs/nCBEJFIyKkEBlJSUfChBFRYWsm7dOsrLy8MJKmQHD8KrrwZt8+YgGW3b\ndmoKeEJJCUydCtOmQXk5lJYGdfHCdODoAeqb6hlx7gi+d8X3wg1GRCIl4xJU//79P1g5Nzc3l/79\n+1NVVcWll14acmRn3tGjQR27RDJKtHYLCQPBrLrS0mCtpYkTgzZmDOREaN5mQ1MD1666lvcb3uf1\nea/rnpOItJFxCWrgwIEA5OfnM2LECKqqqhg8eHDIUaXP8ePBqrLbt8OOHUFLPN6zp+PnFBTA2LFw\n2WVw+eVBMvr4xyE//+zG3h0nTp5g1m9nsemdTTxx0xNKTiLyIRmXoC644AJycnKYNGkSlZWVFBZm\nzs109+DtuLfeatt27z71eO/eD781l5CbC6NGBYmodRs5smclhc42d+e2dbdRuaOSJZ9bwo1/e2PY\nIYlIBGVcgpoyZQqFhYUsWbIkMtPIGxth//4gubRu773Xdvvdd09VYehMTk5QKmjMmKCNHh20MWOC\n/RH5lXvkwU0PsrxmOfdNu4/by24POxwRiaiMu9wtWLAg7a/Z0gINDXDkCNTVnWq1tcGI58CB4Gv7\nxwcPwr59U0ny0aw2ioqCRDN8eNBaPx4+PPi8UW9IQl2ZUzqHHMth3ifmhR2KiERYWi+FZjYAWAZc\nDewH7nb3X3fQz4B/Bb4e3/UIcJd7Z29uBZqb4c03gxFLoh09mtp2Q0OQdFonocTjdusddlOMvDwY\nODBYtjzRBg1qu53YV1LSk5+V2Z7f9TyTLpxEcX4x8yfODzscEYm4dP+v/hDQBAwCSoFKM6tx963t\n+s0FbgDGAQ48C+wClnT14jU1wf2WM6Ffv2B0U1wcfC0qCpLJeecFC+4lviZaYvu1116gouLToX1+\nKFNsPriZ7774XeZ9Yh6LKxaHHY6IZABLMmhJ/YXM+gGHgI+5+/b4vhXAO+5+V7u+fwF+5e5L49v/\nANzm7l2uRpeTM97z8taTk9NELHacnJxT7dR2U3z72AePE9u5uQ3EYo3EYkeJxRrIzU08bsSs5bR+\n79raWs49myUXMlBdUR3V46rpe6wvpf9TSu7JXv4eZg9UV1fT3NxMWVlZ2KFEnv72UhPF47Rhw4Yt\n7p70JE/nlWIM0JxITnE1wLQO+o6Nf691v7EdvaiZzSUYcdGnTx8uuaSix4G2tASti6pJKTt58iS1\ntbU9f6Fe6ni/4/z1sr8Sa4ox4sUR1B/v0fupvV5zczPurnMqBfrbS00mH6d0JqhC4Ei7fYeBok76\nHm7Xr9DMrP19qPgoaylAWVmZb968OX0Rp0FVVVXWVrBIxt2ZvGwy/Q/15ydjf8KXf/TlsEOKvPLy\ncmpra6murg47lMjT315qonicUq2Vms4EVQ8Ut9tXDNSl0LcYqE82SUIyi5mxYuYKjhw/Qt32jk4D\nEZHOpbPwzXYg18xGt9o3Dmg/QYL4vnEp9JMMdKz5GL/c8kvcndHnjWbCkAlhhyQiGShtCcrdG4A1\nwP1m1s/MpgDXAys66P4o8C0zu9DMhgD/BPwqXbFIeE62nGT2mtnMfXIuL+15KexwRCSDpbt06DeA\nvsD7wCrgDnffamZTzaz13fFfAOuAV4HXgMr4Pslg7s78p+az5vU1LL5mMZOHTQ47JBHJYGmd7+vu\nBwk+39R+/4sEEyMS2w7cGW/SSzzwwgMs2bKE70z5Dgv/bmHY4YhIhovQ4guSyXYe3Mn3X/g+Xx33\nVX44/YdhhyMivYA+MSlpMWrAKDZ+bSPjLxif8hRSEZGuaAQlPbLhzQ2s3roagIkXTqRPrE/IEYlI\nb6ERlJy2mr01fP7xzzOseBgzL5mp5CQiaaURlJyWXYd2UfFYBUV5RTw1+yklJxFJO42gpNv2Nezj\nmpXXcKz5GBv/fiPDS4aHHZKI9EJKUNJtv9n6G94+8jbP3focYz/SYY1fEZEeU4KSbps/cT4zPjqD\nUQNGhR2KiPRiugclKWnxFhY+vZDqvUGVbSUnETnTlKAkKXdn0TOL+NnLP+PZnc+GHY6IZAklKEnq\nx3/5MYtfWsyCiQtY9MlFYYcjIllCCUq6tLx6OXc+dyc3j72Zn1b8VFUiROSsUYKSTrk7T/zvE0wf\nOZ1Hb3iUHNPpIiJnj2bxSafMjDWz1tB0son83PywwxGRLKN/ieVDtu3fxozHZrCvYR95sTwK8wqT\nP0lEJM00gpI29hzZw9UrrqbpZBN1TXWc3+/8sEMSkSylBCUfONR4iIqVFdQeq2XDnA1c3P/isEMS\nkSymBCUANJ5o5LpV17Hj4A7Wz17P+MHjww5JRLKc7kEJAAcaD7D/6H5WzlzJlSOvDDscERGNoLKd\nu+M4Q4uH8sodr5AXyws7JBERQCOorHfvn+5lzn/MobmlWclJRCJFCSqL/XzTz/nBiz8gP5ZPzGJh\nhyMi0oYSVJZavXU131z/Ta7/m+t5+NqHVcJIRCInLQnKzAaY2e/NrMHMdpvZl7ro+20ze83M6sxs\nl5l9Ox0xSOqe3/U8t/7+VqYMn8KqL6wiN0e3IkUketJ1ZXoIaAIGAaVApZnVuPvWDvoa8BXgFWAU\n8IyZve3uj6cpFknCMMqGlLH2i2vp26dv2OGIiHSoxwnKzPoBXwA+5u71wEYzWwvcCtzVvr+7/6jV\n5htm9p/AFEAJ6gxrPNFI3z59uWLkFWy8aKPe1hORSEvHCGoM0Ozu21vtqwGmJXuiBVfIqcAvuugz\nF5gb36w3szd6EOuZMBDYH3YQGUDHKXUDzUzHKjmdU6mJ4nEakUqndCSoQuBIu32HgaIUnnsfwX2w\nf++sg7svBZaebnBnmpltdveysOOIOh2n1OlYpUbHKTWZfJySTpIwsyoz807aRqAeKG73tGKgLsnr\nzie4F/U5dz9+ur+AiIj0TklHUO5e3tX34/egcs1stLvviO8eB3Q0QSLxnK8R3J/6tLvvST1cERHJ\nFj2eZu7uDcAa4H4z62dmU4DrgRUd9Tez2cC/AJ9x9//r6c+PgMi+/RgxOk6p07FKjY5TajL2OJm7\n9/xFzAYA/wZ8BjgA3OXuv45/byqw3t0L49u7gKFA67f1Vrr7P/Y4EBER6TXSkqBERETSTaWOREQk\nkpSgREQkkpSg0szMRpvZMTNbGXYsUWNm+Wa2LF6vsc7Mqs1sRthxRUV3alpmK51D3ZfJ1yQlqPR7\nCPivsIOIqFzgbYIqIyXAPwOrzeyiEGOKktY1LWcDD5vZ2HBDihydQ92XsdckJag0MrMvArXAH8OO\nJYrcvcHd73P3N929xd2fBHYBE8KOLWytalre6+717r4RSNS0lDidQ92T6dckJag0MbNi4H7gW2HH\nkinMbBBBLcdOP9SdRTqraakRVBd0DnWuN1yTlKDS5wFgmSpjpMbM+gCPAcvdfVvY8URAT2paZiWd\nQ0ll/DVJCSoFyeoRmlkpcBWwOOxYw5RC3cZEvxyCSiNNwPzQAo6W06ppma10DnWtt1yTtJRqClKo\nR7gQuAh4K77GUiEQM7NL3f3yMx5gRCQ7TvDBEivLCCYCfNbdT5zpuDLEdrpZ0zJb6RxKSTm94Jqk\nShJpYGYFtP3vdxHByXGHu+8LJaiIMrMlBKsuXxVf4FLizOxxwIGvExyjp4BPdrIyddbSOZRcb7km\naQSVBu5+FDia2DazeuBYJp0IZ4OZjQBuJ6jDuLfVir63u/tjoQUWHd8gqGn5PkFNyzuUnNrSOZSa\n3nJN0ghKREQiSZMkREQkkpSgREQkkpSgREQkkpSgREQkkpSgREQkkpSgREQkkpSgREQkkpSgREQk\nkv4fpnIt6Q3iZsAAAAAASUVORK5CYII=\n", - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "z = np.linspace(-5, 5, 200)\n", "\n", @@ -219,25 +201,7 @@ "cell_type": "code", "execution_count": 8, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Saving figure leaky_relu_plot\n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl8FPX9x/HXh3AlHCJyVMGCeKCgVSCetBiPUm9RUEG0\n4sGhVeuB1oMKiGfFWjwBiyJyi1yitD9Fo+JVQVG8gFJQQUUEEgghAZLv74/voiHk2E2ymdnN+/l4\n7IM9JjvvHTb7zsx8d8acc4iIiIRNraADiIiIlEQFJSIioaSCEhGRUFJBiYhIKKmgREQklFRQIiIS\nSiooKZWZZZrZ40HnSAZmlmFmzsyaVcO8VpvZ4GqYz6Fm9p6Z5ZnZ6njPL4o8zsx6BZ1Dqo4KKkGZ\n2Xgzmxd0jlhFSs9FLtvNbKWZ3W9m9WJ8nn5mllPOfPYo1/J+riqUUhDvAvsCG6pwPsPM7LMSHjoa\neLKq5lOGe4Bc4NDIPKtFGe/9fYGXqiuHxF/toANIjfQscAdQF//B9mzk/tsDSxRnzrntwA/VNK/1\n1TEf4CBgjnNudTXNr0zOuWpZvlJ9tAaVpMxsLzMba2Y/mtkWM3vTzNKLPL6PmU0xszVmts3MPjez\ny8t5zlPMLMvMBplZNzPbYWa/KjbNvWb2aTnxcp1zPzjnvnHOvQi8CnQv9jytzGyqmW2KXF42s4Nj\nXAwVYmYPmNmyyHJZbWZ/M7P6xaY5w8w+iEyzwcxeMrP6ZpYJtAEe2rWmGJn+5018ZtY48nNnF3vO\n7pFl2qK8HGbWDxgKdCyyRtov8thua3Bm9mszmxV5H2wxs5lm1rrI48PM7DMz6x1Zo91iZrPL2hwZ\neV1HAndF5j3MzNpGrqcXn3bXprci0/Q0s1fNLNfMvjCz3xf7mUPNbK6ZZZtZTmRT4hFmNgy4DDiz\nyOvOKD6fyO0jzOy1yPLbGFnz2qvI4+PNbJ6Z/dnM1kbeZ8+aWVppr1uqlwoqCZmZAS8DrYCzgE7A\nW8DrZrZvZLL6wEeRxzsCo4AxZnZKKc/ZC5gFDHDOjXbOvQWsBP5YZJpakdvjYsh6JNAV2FHkvjTg\nDSAPOBE4HvgeeK2aPjy2AlcAhwHXAL2BO4vkOw2Yiy/WLsBJwJv436fzgTXA3fhNTvtSjHNuM35T\nVN9iD/UFXnXO/RhFjmnAw8CyIvOZVnxekf+TOUDLSM6TgP2A2ZH3yS5tgYuA8/B/LHQC7i1l+RCZ\n37JIhn2BkWVMW5J7gUfxJfchMNXMGkYy7wcsBBzwe6Az8ASQEpnPdOC1Iq/73RJedwPg30AOcEzk\ndZ0APFNs0t8BhwOn8svr/3OMr0XixTmnSwJegPHAvFIeOxn/i5la7P4lwK1lPOdU4J9FbmcCjwMD\ngGyge7HpBwNfFrl9OpAP7FPGPDKB7ZF8+fgPoQKgZ5FprgBWAFbkvhT8/psLI7f7ATnlzOfxEu4v\n8+dKea5BwH+L3H4HmFrG9KuBwcXuy4i81maR2+fg9980itxOBTYDF8eQYxjwWVnzx3/AFwBtizze\nDigETi3yPHnAXkWmubPovErJ8xkwrMjttpHXmF5sOgf0KjbNwCKPt4rc99vI7XuBr4G6sbz3i82n\nf+Q926iE/4ODijzPt0BKkWmeBl6ryO+kLlV/0RpUcuoCpAHrI5tHcswPDDgcOBDAzFLM7E4z+zSy\niSoH/9f/r4s9Vw/8X6+nOef+r9hjzwHtzOyEyO0rgNnOufIGAkwDjsKvGU0HnnZ+U1/R/AcAW4pk\nzwb23pU/nsysl5ktNLMfIvN+hN2XSydgQSVnMx9fUOdFbp8DGDA7hhzROAz4zhXZT+Sc+x/wHdCh\nyHRfO+eyi9z+DmgR47xiUXQz8HeRf3fNrxOw0Pn9dhV1GPCpc25LkfvexRdz0df9hXOuoFiWeL5u\niYEGSSSnWsA6/OaL4jZH/h0M3IzfnLEUv0ZzH3v+cn4CHAFcaWbvu8ifmeB3xpvZXOAKM1uG/5A9\nm/JlO+f+C2BmlwCfm1k/59z4IvmX4DdpFbcxiucH/zr3KuH+JviyK5GZHYdfkxwO3Ahk4V9XrJuw\nyuSc22Fm0/Gb9SZE/p3lnMutxhxFT2Wwo4THYv0DtjDy78+bDs2sTinT/jw/55yLbG2srj+Yq/p1\nS5yooJLTR/h9DoWRv5ZL8lvgJefc8/DzfqtD8B+ERa0CrsNvMhtrZgOKlhR+k8gM4H/4UWqvxRI0\n8kF9H3C/mU2PfEB/BPQBfnLOFc8TrWXAGWZmxfJ2jjxWmq7AWufciF13mFmbYtN8DJyCf+0l2Y7f\nJFmeicBbZtYBOA2/PzCWHNHM50tgPzNru2stysza4fdDfRFFxljsGj1YdL/bURV4no+BS8ysbilr\nUdG+7ivMrFGRtagT8OXzZQUySQD0l0Jia2xmRxW7tMWXxDvAHDM73cwOMLPjzWy4me1aq1oOnGJm\nvzWzQ/H7mg4oaSaRkjsJ/yE6ptjO9Vfx+4aGAuOdc4UlPEV5JuP/cr02cnsSfg1wjpmdGMnfzcwe\ntt1H8tUq4fUfHnnsKfy+lsfM7Egza29mN+KL76EysiwHWplZXzNrZ2ZXR36mqHuBC8zsHjPrYGYd\nzezGIgM4VgO/Mz8SsdSRcM65d/H7WiYDP7H7ZsNocqwG2phZZ/OjA0v6Ltlr+M1pk8ws3fwIu0n4\nPwJeL2M5xMw5tw14H/hLZJmcQMXW+J4EGgLTzexoMzvIzPqY2a6yWw0cHvk/bVbKWtok/CbUCeZH\n83UDxgAzd629S/ipoBLb7/B/bRa9jIysMZyB/wB6Gr/GMB1ozy/b++8B/oPfF/IWfsTYpNJm5Jxb\nid/JfDpFSioyr2eBOvzyfaaYRP5Kfhy4NfIXby7QDb9W9gLwFX5/197ApiI/mlrC68+MPOf/Is9x\nMPB/kdfaG7jAOTe/jCwv4QvsH/gP9t8DdxWb5hX8vqPTI/N8E1/gu8r5LmB//CjH8r6TNAk/km1q\n0X0h0eQAXgRewRfbevYssF3/P+dGHn8jcvkB6FFszbKqXBH590N8IQyJ9Qmcc2vx/3d18Xk/xq/F\n74xM8jR+LWgR/nV1LeE5coE/AI3x//dzgPeK5JMEYPF5j0pNYmZP4UdG/b7ciUVEoqR9UFJh5r/0\n2AH/3acLA44jIklGBSWVMQf/JchxzrmXgw4jIslFm/hERCSUNEhCRERCKW6b+Jo1a+batm0br6ev\nlK1bt9KgQYOgYyQkLbvYLVu2jIKCAjp06FD+xLIbvd8qrrRlt2oVbNwI9erBYYdBSjTf2Ktiixcv\n/sk517y86eJWUG3btmXRokXxevpKyczMJCMjI+gYCUnLLnYZGRlkZWWF9vchzPR+q7iSlt3DD8Pg\nwdCgAXzwAXTsGEw2M/s6mum0iU9EpAZ49VW49VZ/fcKE4MopFiooEZEk97//wUUXQWEh/PWvcP75\nQSeKjgpKRCSJbd0KPXrApk1w1lkwbFjQiaKnghIRSVLOweWXw9Kl0L49TJwItRLoUz+BooqISCwe\nfBBeeAEaNYLZs2Gvkk5AE2IxFZSZHWxmeWY2MV6BRESk8j74oCl33OGvT5oEhx4abJ6KiHUN6gn8\nUYpFRCSkVqyAe+45DOdg+HA4O5rTiIZQ1AVlZr3xJ7Or7KmuRUQkTrZs8YMicnLq0KMHDIn5hCfh\nEdUXdc2sMXA3cDJwVRnTDQAGALRs2ZLMzMwqiFj1cnJyQpst7LTsYpeVlUVBQYGWWwXo/RabwkIY\nOrQjX3zRnP3330L//kt4662C8n8wpKI9ksQI/BGr1+x+MtXdOefGAmMB0tPTXVi/Aa5vp1ecll3s\nmjRpQlZWlpZbBej9FpsRI2DhQj8Y4r77vuCMM35X/g+FWLkFFTnN8qlAp/jHERGRinjpJRg6FMxg\nyhRITd0WdKRKi2YNKgNoC3wTWXtqCKSYWQfnXOf4RRMRkWh89RVccon/3tN998Hpp0MybBmNpqDG\nAlOL3B6ML6yr4xFIRESil53tB0Vs3gy9esFttwWdqOqUW1DOuVwgd9dtM8sB8pxz6+MZTEREylZY\n6Necli2DI46AZ5/1m/iSRcyn23DODYtDDhERidHw4TBvHuy9tz9SRMOGQSeqWjrUkYhIApo1C+6+\n2x9bb+pUaNcu6ERVTwUlIpJgPv8c/vhHf/3BB6F792DzxIsKSkQkgWzatOtIEdCnD9x8c9CJ4kcF\nJSKSIAoKoG9f+O9/4aij4J//TK5BEcWpoEREEsRf/wrz58M++/h9UGlpQSeKLxWUiEgCmD4d7r8f\nUlL89bZtg04UfyooEZGQ+/RTf2ZcgIcfhpNPDjZPdVFBiYiE2MaNflBEbq4fuXf99UEnqj4qKBGR\nkNq5E3r3hlWroEsXGD06uQdFFKeCEhEJqdtvh1dfhebN/aCI1NSgE1UvFZSISAhNngwjR0Lt2jBj\nBuy/f9CJqp8KSkQkZD7+GK6KnLv8H/+Abt2CzRMUFZSISIisX+8HRWzbBldcAddcE3Si4KigRERC\nYscOuOgi+OYbOPZYeOKJmjUoojgVlIhISNxyC7zxBvzqV/Dii1C/ftCJgqWCEhEJgQkTYNQoqFPH\nl1OrVkEnCp4KSkQkYIsWwYAB/vrjj8MJJwSbJyxUUCIiAVq3Ds47D/LzYeDAX4pKVFAiIoHZvh0u\nuADWrIGuXeHRR4NOFC4qKBGRgNx0E7z9Nuy3n/8ybt26QScKFxWUiEgAxo3zw8jr1oWZM/3IPdmd\nCkpEpJq9//4vX8AdPdp/50n2pIISEalG338P55/v9z9de+0v53mSPamgRESqSX4+9OzpS6pbN/j7\n34NOFG4qKBGRanL99fDee9C6Nbzwgv9SrpROBSUiUg3GjIGxY/3hi2bPhhYtgk4UfiooEZE4e+cd\nuO46f33sWH92XCmfCkpEJI7WrvX7nXbsgBtugEsvDTpR4lBBiYjESV6eH7G3bh2cfDI89FDQiRKL\nCkpEJA6cg6uvhv/8B9q0gWnT/OnbJXoqKBGROHjiCRg/HlJT/aCIZs2CTpR4VFAiIlXszTfhxhv9\n9WeegaOOCjZPolJBiYhUoW++8Uco37nTnyG3d++gEyUuFZSISBXZts2f22n9eujeHe6/P+hEiU0F\nJSJSBZzzJxv86CNo1w6mTIGUlKBTJTYVlIhIFRg1CiZOhAYN/KCIpk2DTpT4VFAiIpW0YAEMHuyv\njx8PRxwRaJykoYISEamEVavgoougoADuuAN69Qo6UfJQQYmIVFBurh8UsWEDnHEG3H130ImSS1QF\nZWYTzex7M9tsZsvN7Kp4BxMRCTPn4Mor4ZNP4OCDYdIkDYqoatGuQd0PtHXONQbOAe4xMx2PV0Rq\nrJEjYepUaNjQD4po0iToRMknqoJyzn3unMvfdTNyOTBuqUREQuzf/4bbbvPXn38eOnQINk+yivrQ\nhWb2JNAPSAU+Bl4pYZoBwACAli1bkpmZWSUhq1pOTk5os4Wdll3ssrKyKCgo0HKrgDC+39aurc+g\nQV0oLKzDZZetpkmT1YQsIhDOZRcrc85FP7FZCnA8kAE86JzbUdq06enpbtGiRZUOGA+ZmZlkZGQE\nHSMhadnFLiMjg6ysLJYsWRJ0lIQTtvdbTg4cfzx89hmccw7MmgW1QjrULGzLrigzW+ycSy9vupgW\nrXOuwDm3EGgNXF3RcCIiicY56NfPl9Ohh/pNe2Etp2RR0cVbG+2DEpEa5P774cUXoXFjPyiiceOg\nEyW/cgvKzFqYWW8za2hmKWb2B6APsCD+8UREgvfyyzBkCJjB5MnQvn3QiWqGaAZJOPzmvNH4Qvsa\nuME5NzeewUREwmD5cujb12/iGzECzjwz6EQ1R7kF5ZxbD5xYDVlEREJl82bo0QOys+H88/2hjKT6\naBefiEgJCgvhj3+EL7+Ejh39QWA1KKJ6aXGLiJRgxAiYM8cfIWL2bGjUKOhENY8KSkSkmLlzYdgw\nPyhiyhQ46KCgE9VMKigRkSK+/BIuucRfv/9+OO20YPPUZCooEZGIrCw491zYsgUuvBBuvTXoRDWb\nCkpEBD8o4pJLYMUK+M1v4Jln/CY+CY4KSkQEGDrUfyG3aVM/KKJBg6ATiQpKRGq8F1+Ee+7xw8in\nTYMDDgg6kYAKSkRquM8+g8su89cfeghOPTXYPPILFZSI1FibNvkjRWzd6g9ndOONQSeSolRQIlIj\nFRRAnz6wciV06gRjx2pQRNiooESkRrrzTn/q9mbN/IkH09KCTiTFqaBEpMaZNg0efBBSUuCFF6BN\nm6ATSUlUUCJSo3zyCVx+ub/+yCMQ0rOiCyooEalBNmzwgyK2bfMj9669NuhEUhYVlIjUCDt3wkUX\nwerVcPTRMHq0BkWEnQpKRGqEv/wFFiyAFi1g5kyoXz/oRFIeFZSIJL2JE+Hvf4fatf1RI1q3DjqR\nREMFJSJJ7aOPoH9/f/3RR+G3vw02j0RPBSUiSevHH/2giLw8uOoqGDQo6EQSCxWUiCSlHTv8OZ2+\n/RaOOw4ef1yDIhKNCkpEktLNN8Obb8K++/r9TvXqBZ1IYqWCEpGkM348PPYY1Knjy2m//YJOJBWh\nghKRpPKf//yyr+nJJ+H444PNIxWnghKRpPHDD3D++ZCfD1df7QdGSOJSQYlIUti+HXr1grVr/VDy\nf/wj6ERSWSooEUkKN9wA77wDrVrBjBlQt27QiaSyVFAikvCefhqeesqP1Js1C1q2DDqRVAUVlIgk\ntHffhT/9yV8fPdofCFaSgwpKRBLWd99Bz57+S7nXXw/9+gWdSKqSCkpEElJ+vi+nH36AE0+EkSOD\nTiRVTQUlIgnHOb9Z7/334de/9qdtr1Mn6FRS1VRQIpJwRo+GceP8OZ1mzYLmzYNOJPGgghKRhPL2\n235/E8A//wmdOwebR+JHBSUiCWPNGv9l3J074aaboG/foBNJPKmgRCQh5OXBeef5czydcgo8+GDQ\niSTeVFAiEnrO+QPALloEbdvCtGn+9O2S3FRQIhJ6jz0Gzz0HaWkwezbss0/QiaQ6qKBEJNQyM/3+\nJoBnn4Ujjww0jlSjcgvKzOqZ2Tgz+9rMtpjZEjM7vTrCiUjN9sMP9bjgAigogL/8xZ/CXWqOaNag\nagPfAicCewFDgOlm1jZ+sUSkpsvNhbvuOpyffoI//AHuvTfoRFLdyt3N6JzbCgwrctc8M1sFdAFW\nxyeWiNRkzkH//rBiRSMOPBCmTIGUlKBTSXWLeRyMmbUEDgE+L+GxAcAAgJYtW5KZmVnZfHGRk5MT\n2mxhp2UXu6ysLAoKCrTcYjB9emsmTz6I+vV3cuedH/PJJ1uDjpRwkuF31Zxz0U9sVgeYD6x0zg0s\na9r09HS3aNGiSsaLj8zMTDIyMoKOkZC07GKXkZFBVlYWS5YsCTpKQnjtNb9Jr7AQhg//jLvuOjzo\nSAkpzL+rZrbYOZde3nRRj+Izs1rA88B24NpKZBMRKdH//gcXXeTLacgQ6Nbtp6AjSYCiKigzM2Ac\n0BLo6ZzbEddUIlLjbN0KPXrAxo1w1lkwfHjQiSRo0e6Dego4DDjVObctjnlEpAZyDq64ApYuhUMO\ngYkToZa+pVnjRfM9qDbAQOAo4Aczy4lcdJhGEakSf/sbTJ8OjRr5I0XstVfQiSQMohlm/jVg1ZBF\nRGqgf/0Lbr/dX584EQ47LNg8Eh5aiRaRwPz3v9Cnj9/EN3w4nHNO0IkkTFRQIhKILVv8oIisLP/v\nkCFBJ5KwUUGJSLUrLITLLoPPP/eb9J57ToMiZE96S4hItbvvPpg1yw+GmD0bGjcOOpGEkQpKRKrV\nvHlw111gBpMn+2HlIiXROSlFpNosWwZ9+/pBEffeC2ecEXQiCTOtQYlItcjOhnPPhc2boVevX4aW\ni5RGBSUicVdYCJde6tegDj/cnxnX9O1KKYcKSkTibvhweOkl2HtvPyiiYcOgE0kiUEGJSFzNng13\n3+2HkU+dCgceGHQiSRQqKBGJmy++8Jv2AB54ALp3DzaPJBYVlIjERVaWHxSRkwO9e8PgwUEnkkSj\nghKRKldQABdf7I+1d9RRMG6cBkVI7FRQIlLl7roL5s+HffbxR4xISws6kSQiFZSIVKkXXvCHMkpJ\ngWnToG3boBNJolJBiUiV+fRT6NfPXx85Ek45JdA4kuBUUCJSJTZu9KfNyM31I/f+/OegE0miU0GJ\nSKXt3OlH6q1aBV26wJgxGhQhlaeCEpFKu+MOePVVaN4cZs6E1NSgE0kyUEGJSKVMmQIPPQS1a8OM\nGfDrXwedSJKFCkpEKmzJErjySn/9H/+Abt2CzSPJRQUlIhXy009+UMS2bXD55XDNNUEnkmSjghKR\nmO3cCRdeCF9/DcccA08+qUERUvVUUCISs1tugTfegJYt/aCI+vWDTiTJSAUlIjF5/nm/v6lOHXjx\nRWjVKuhEkqxUUCIStUWLoH9/f/2xx6Br12DzSHJTQYlIVNatg/POg/x8GDAABg4MOpEkOxWUiJRr\nxw644AJYswZOOAEefTToRFITqKBEpFw33ghvvw377ee/jFuvXtCJpCZQQYlImZ55Bp54AurW9SP2\n9t036ERSU6igRKRUH3wAV1/trz/1FBx7bLB5pGZRQYlIib7/Hs4/H7Zvhz/9Ca64IuhEUtOooERk\nD9u3Q69e8N13/vh6jzwSdCKpiVRQIrKH66+Hd9+F1q39Kdzr1Ak6kdREKigR2c2YMf5Srx7MmgUt\nWgSdSGoqFZSI/Oydd+C66/z1sWMhPT3YPFKzqaBEBIC1a6FnT/+l3BtugD/+MehEUtOpoESEvDw/\nYm/dOjjpJH+GXJGgRVVQZnatmS0ys3wzGx/nTCJSjZzzw8j/8x9o0wamTfOnbxcJWrRvw++Ae4A/\nAKnxiyMi1e3JJ/3RIlJT/aCI5s2DTiTiRVVQzrmZAGaWDrSOayIRqTZvveX3NwGMGwedOgWbR6Qo\n7YMSqaG+/dZ/GXfnThg8GPr0CTqRyO6qdEuzmQ0ABgC0bNmSzMzMqnz6KpOTkxPabGGnZRe7rKws\nCgoKQrXc8vNrcf31nVi/vhHp6Rs57bSlZGa6oGPtQe+3ikuGZVelBeWcGwuMBUhPT3cZGRlV+fRV\nJjMzk7BmCzstu9g1adKErKys0Cw35/wQ8uXLoV07+Pe/m9K06YlBxyqR3m8VlwzLTpv4RGqYUaNg\n4kRIS4PZs6Fp06ATiZQsqjUoM6sdmTYFSDGz+sBO59zOeIYTkar1+ut+fxPA+PFwxBGBxhEpU7Rr\nUEOAbcBtwCWR60PiFUpEqt7q1XDhhVBQALff7k/hLhJm0Q4zHwYMi2sSEYmb3Fzo0QM2bIDTT4cR\nI4JOJFI+7YMSSXLOwZVXwiefwMEHw+TJkJISdCqR8qmgRJLcww/D1KnQsKEfFNGkSdCJRKKjghJJ\nYv/3f/CXv/jrEyZAhw7B5hGJhQpKJEmtXAm9e0NhIdx1F5x3XtCJRGKjghJJQjk5flDEpk1w9tkw\ndGjQiURip4ISSTLOweWXw2efQfv2/ku5tfSbLglIb1uRJPPAAzBjBjRuDHPm+H9FEpEKSiSJvPIK\n3HknmMGkSX4NSiRRqaCqQUZGBtdee23QMSTJrVgBF1/sN/HdfTecdVbQiUQqRwUF9OvXj7P02ywJ\nbMsWOPdcyM72o/XuuCPoRCKVp4ISSXCFhf70GV9+6b/n9NxzGhQhyUFv43JkZ2czYMAAWrRoQaNG\njTjxxBNZtGjRz49v2LCBPn360Lp1a1JTU+nYsSPPPvtsmc+5YMECmjRpwujRo+MdX2qAe+755QgR\nc+ZAo0ZBJxKpGiqoMjjnOPPMM1m7di3z5s3j448/plu3bpx88sl8//33AOTl5dG5c2fmzZvH559/\nzp///GcGDhzIggULSnzOGTNmcN555zF27FgGDRpUnS9HktDcuf47TmYwZQocdFDQiUSqTpWeUTfZ\nvPHGGyxZsoT169eTmpoKwIgRI3jppZd4/vnnufXWW2nVqhW33HLLzz8zYMAAXn/9daZMmcIpp5yy\n2/ONHTuWW265hRkzZtC9e/dqfS2SfL76Ci65xF+/7z447bRg84hUNRVUGRYvXkxubi7Nmzff7f68\nvDxWrlwJQEFBAQ888ADTpk1j7dq15Ofns3379j1OtTx79mzGjBnDW2+9xfHHH19dL0GSVHa2HxSx\nZYs/r9Ou4+2JJBMVVBkKCwtp2bIlb7/99h6PNY58+3HkyJE8/PDDjBo1iiOOOIKGDRtyxx138OOP\nP+42/ZFHHsnSpUsZN24cxx13HGZWLa9Bkk9hIfTtC8uX+zPiPvus38QnkmxUUGXo3Lkz69ato1at\nWrRr167EaRYuXMjZZ5/NpZdeCvj9VsuXL6dJsXMaHHDAATz22GNkZGQwYMAAxo4dq5KSChk6FF5+\nGZo29YMjGjQIOpFIfGiQRMTmzZtZsmTJbpeDDjqIrl27cu655zJ//nxWrVrFe++9x9ChQ39eqzrk\nkENYsGABCxcu5KuvvuLaa69l1apVJc6jXbt2vPHGG/zrX/9i4MCBOOeq8yVKEpg504/aq1ULpk2D\nUv5uEkkKKqiIt99+m06dOu12ueWWW3jllVc4+eST6d+/P+3bt+fCCy9k2bJl7LfffgAMGTKEY445\nhtNPP51u3brRoEED+vbtW+p8DjzwQDIzM5k/f75KSmLy2Wf++04Af/sbnHpqsHlE4k2b+IDx48cz\nfvz4Uh8fNWoUo0aNKvGxvffem5kzZ5b5/JmZmbvdPvDAA/n2229jjSk12KZN/vQZW7f6wxnddFPQ\niUTiT2tQIiFXUAB9+vgTEHbqBE8/rUERUjOooERCbsgQ+Pe/oVkzmDUL0tKCTiRSPVRQIiE2fbo/\nv1NKir/epk3QiUSqT1IX1M6dOxkzZgwbNmwIOopIzD75xJ8ZF+Dvf4eTTgo2j0h1S9qC+vbbbznm\nmGO47rrruOCCCzRaThLKhg1+UERuLlx2GVx3XdCJRKpfUhbUnDlz6NixI59++ik7duzggw8+YOTI\nkUHHEonKzp3QuzesXg3p6TB6tAZFSM2UVAWVn5/PoEGDuPjii9myZQsFBQUA5ObmMnTo0N1OkyES\nVrfdBq/vsgn3AAAKYElEQVS9Bi1a+C/m1q8fdCKRYCRNQa1YsYIjjzySCRMmkJubu8fjzjlWrFgR\nQDKR6E2aBA8/DLVrw4wZsP/+QScSCU5SfFF34sSJDBo0iNzc3D32NdWpU4fGjRsza9Ysfve73wWU\nUKR8H30EV13lrz/6KOjtKjVdQhfU1q1b6d+/P3PmzClxrSktLY1jjz2WF154gX322SeAhCLRWb8e\nzjsP8vLgyitB57IUSeBNfEuXLqVDhw7MmjWrxHJKTU1l+PDhLFiwQOUkobZjB1x4IXzzDRx3HDzx\nhAZFiEACrkE553jqqacYPHgw27Zt2+PxevXq0bRpU+bOnUt6enoACUViM3gwZGbCr34FL74I9eoF\nnUgkHBKqoLKysrjkkkt44403SiyntLQ0fv/73zNhwoSfTygoEmbjx/v9TXXq+BF7kYPkiwgJVFAf\nfPAB55xzDtnZ2eTn5+/xeFpaGo888gj9+/fXiQAlIXz44S/7mp54Ao4/Ptg8ImET+oIqLCzkgQce\n4J577ilxral+/frsu+++zJs3jw4dOgSQUCR269b5QRH5+b6k+vcPOpFI+IS6oH788Ud69erF4sWL\nS92k17NnT8aMGUNqamoACUVit3079OoFa9dC165QyqnGRGq8QEfxbdiwgY8++qjEx15//XUOPfRQ\n3n///T1G6dWqVYsGDRowbtw4JkyYoHKShHLDDbBwIbRq5b+MW7du0IlEwinQgrrmmmvo2rUrK1eu\n/Pm+nTt3ctttt3HWWWexadMmduzYsdvPpKWlceihh/Lpp5/Su3fv6o4sUin//Cc89ZQfqTdzph+5\nJyIlC6ygli9fzty5c9m+fTtnn30227dvZ82aNRx77LE89thjJW7SS01N5corr+Tjjz+mXbt2AaQW\nqbj33oM//clff+opOOaYYPOIhF1U+6DMrCkwDugO/ATc7pybXJkZ33bbbezYsYPCwkJWr15Njx49\nWLhwIbm5uT8f5PXnkLVrk5aWxuTJkznzzDMrM1uRQOzYUYuePf3+p+uu++U8TyJSumgHSTwBbAda\nAkcBL5vZJ865zysy0y+++IL58+f/XETbtm3j9ddfL3X4eMeOHZk1axatWrWqyOxEApWXB6tWNWDb\nNjjxRH8wWBEpn5V3Ij8zawBsAg53zi2P3Pc8sNY5d1tpP9eoUSPXpUuXEh9bunQpGzduLDdcrVq1\naN26NW3btq3S7zZlZWXRpEmTKnu+mkTLbnfO+fM3lXbZvh3WrFkCQL16R9Gli/9SrkRH77eKC/Oy\ne/PNNxc758o91E80a1CHADt3lVPEJ8CJxSc0swHAAPBHEc/KytrjybZt28amTZvKnWlKSgpt27al\nYcOGZGdnRxEzegUFBSVmk/Il27JzDgoKrMIX56L7wyklpZCDDspm61ad2TkWyfZ+q07JsOyiKaiG\nwOZi92UDjYpP6JwbC4wFSE9PdyWdILB79+7lnpepTZs2LFq0iGbNmkURL3aZmZlkZGTE5bmTXdiW\nXX4+ZGWVfcnOLv2xEsbixCQlBfbaC5o0Kf0yY0YGZlksWfJx1bzoGiRs77dEEuZlF+0WsWgKKgco\nfmC7xsCWGDOxePFiFi5cuMc5m4r78ccfWbp0KSeddFKss5AEk5dXfsGUVTZ5eZWbf0oK7L23L5Ly\niqakS4MG5R95fMECn1VEYhNNQS0HapvZwc65Xas+RwIxD5C4+eabyYviE2Xbtm307NmTZcuW0bx5\n81hnI9XEudgLpnjRlDAuJia1a/9SMEUv0ZZNWppObSESVuUWlHNuq5nNBO42s6vwo/jOBU6IZUbv\nv/8+H374YblrT7ts2bKFG2+8kYkTJ8YyG4mBc5CbG92msF2Xb7/tTEHBL7eLfY86ZnXqlFww0ZZN\naqoKRiRZRTvM/BrgGeBHYANwdaxDzG+66aYSTyxYr1496tWrR35+PnXr1uWQQw7h6KOPpkuXLtrE\nVw7nYOvW2Pa5FL/s3BnrXHff2lu3bvkFU1bR1K+vghGRkkVVUM65jUCPis7k3Xff5b333qNRo0YU\nFBRQUFBAu3bt6Ny5M0cffTS/+c1v6NixIy1atKjoLBKSc5CTU/Ed/FlZUOw7zTGrXz+2fS4rVy7m\nlFO6/Fw29etXzbIQESmuWo5mvs8++3DfffdxxBFHcPjhh9OmTZukOGdTYWH5BVNe0RQWVi5DWlrs\n+12KTh/r2VszM7fQvn3lMouIRKNaCqp9+/bcfvvt1TGrmBQWwpYtFd/Bn51d+YJp0CD2/S5Fp9GR\nsEUkWYX6fFDlKSiAzZtj3+/yww/HkZfnfzbKMRulatiwYvtedt2vowqIiJQs0IIqKNizWGIpms3F\nvz4ctV92nDRqFNsmseK3ayd0xYuIhFfcPl7XrYO77iq7YLbE/FXfPe21V+z7Xr766n3+8IfjaNxY\nBSMiElZx+3heswZGjCh7GrPdyyXWomnUyB8JIFbZ2Xk0bVqx1yUiItUjbgXVogVcc035BVMr0HP6\niohIWMWtoPbfH4YOjdezi4hIstP6i4iIhJIKSkREQkkFJSIioaSCEhGRUFJBiYhIKKmgREQklFRQ\nIiISSiooEREJJRWUiIiEkgpKRERCyVxlT4hU2hObrQe+jsuTV14z4KegQyQoLbuK0XKrGC23igvz\nsmvjnGte3kRxK6gwM7NFzrn0oHMkIi27itFyqxgtt4pLhmWnTXwiIhJKKigREQmlmlpQY4MOkMC0\n7CpGy61itNwqLuGXXY3cByUiIuFXU9egREQk5FRQIiISSiooEREJJRUUYGYHm1memU0MOkvYmVk9\nMxtnZl+b2RYzW2JmpwedK6zMrKmZzTKzrZFldnHQmcJO77GqkQyfayoo7wngw6BDJIjawLfAicBe\nwBBgupm1DTBTmD0BbAdaAn2Bp8ysY7CRQk/vsaqR8J9rNb6gzKw3kAUsCDpLInDObXXODXPOrXbO\nFTrn5gGrgC5BZwsbM2sA9AT+6pzLcc4tBOYClwabLNz0Hqu8ZPlcq9EFZWaNgbuBm4LOkqjMrCVw\nCPB50FlC6BBgp3NueZH7PgG0BhUDvcdik0yfazW6oIARwDjn3JqggyQiM6sDTAKec859FXSeEGoI\nbC52XzbQKIAsCUnvsQpJms+1pC0oM8s0M1fKZaGZHQWcCjwSdNYwKW+5FZmuFvA8fv/KtYEFDrcc\noHGx+xoDWwLIknD0Hotdsn2u1Q46QLw45zLKetzMbgDaAt+YGfi/dlPMrINzrnPcA4ZUecsNwPwC\nG4ff8X+Gc25HvHMlqOVAbTM72Dm3InLfkWhTVbn0HquwDJLoc63GHurIzNLY/a/bwfj/2Kudc+sD\nCZUgzGw0cBRwqnMuJ+g8YWZmUwEHXIVfZq8AJzjnVFJl0HusYpLtcy1p16DK45zLBXJ33TazHCAv\nEf8Tq5OZtQEGAvnAD5G/0gAGOucmBRYsvK4BngF+BDbgPyhUTmXQe6ziku1zrcauQYmISLgl7SAJ\nERFJbCooEREJJRWUiIiEkgpKRERCSQUlIiKhpIISEZFQUkGJiEgoqaBERCSU/h9r5scSI6iwhAAA\nAABJRU5ErkJggg==\n", - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "plt.plot(z, leaky_relu(z, 0.05), \"b-\", linewidth=2)\n", "plt.plot([-5, 5], [0, 0], 'k-')\n", @@ -399,18 +363,7 @@ "cell_type": "code", "execution_count": 18, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Extracting /tmp/data/train-images-idx3-ubyte.gz\n", - "Extracting /tmp/data/train-labels-idx1-ubyte.gz\n", - "Extracting /tmp/data/t10k-images-idx3-ubyte.gz\n", - "Extracting /tmp/data/t10k-labels-idx1-ubyte.gz\n" - ] - } - ], + "outputs": [], "source": [ "from tensorflow.examples.tutorials.mnist import input_data\n", "mnist = input_data.read_data_sets(\"/tmp/data/\")" @@ -422,22 +375,7 @@ "metadata": { "scrolled": true }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "0 Batch accuracy: 0.86 Validation accuracy: 0.9044\n", - "5 Batch accuracy: 0.94 Validation accuracy: 0.9508\n", - "10 Batch accuracy: 0.96 Validation accuracy: 0.9666\n", - "15 Batch accuracy: 1.0 Validation accuracy: 0.9722\n", - "20 Batch accuracy: 1.0 Validation accuracy: 0.975\n", - "25 Batch accuracy: 1.0 Validation accuracy: 0.9766\n", - "30 Batch accuracy: 0.98 Validation accuracy: 0.9782\n", - "35 Batch accuracy: 0.96 Validation accuracy: 0.9792\n" - ] - } - ], + "outputs": [], "source": [ "n_epochs = 40\n", "batch_size = 50\n", @@ -479,25 +417,7 @@ "cell_type": "code", "execution_count": 21, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Saving figure elu_plot\n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl8FeW9x/HPLwnKKiBorCJgXVDrwhWq1bqkalUWt7q2\nasUNKtqWqq0b9Gql2ipWqApKixcFF1CwKgh41XvABaWgIFAFRECQfTlAgARInvvHc4CQ9SSZZOac\n832/XueVyTxzZn5nGM43sz1jzjlERESiJivsAkRERMqjgBIRkUhSQImISCQpoEREJJIUUCIiEkkK\nKBERiSQFlIiIRJICSkREIkkBJSnDzIab2bg0Wk6WmT1rZuvMzJlZXl0vs5Ja6uUzJ5bV0sxWmdnh\n9bG86jKzV83szrDrEDD1JJGezGw4cH05TZ86536UaG/tnOtewftjwBzn3O2lxvcAnnLONQ204OSW\n3Ry/zcZTaTmVLL87MBbIA74B1jvnttflMhPLjVHqc9fXZ04s6zH8tndDXS+rnGWfCdwFdAIOBm5w\nzg0vNc3xwGTgMOfcxvquUfbICbsAqVPvAteVGlfnX4B1pb6+LOrxS+kIYIVz7uN6Wl6F6uszm1lj\n4GbgwvpYXjmaAnOAFxKvMpxzs83sG+Ba4Ol6rE1K0SG+9FbonFtZ6rW+rhdqZheY2QdmtsHM1pvZ\nJDM7pkS7mdmdZrbAzArNbJmZPZJoGw6cBdyWOOzlzKz9rjYzG2dmPROHiLJLLfclM3szmTqSWU6J\n+exrZgMTyywws0/M7PQS7TEzG2xmD5vZWjNbbWYDzKzC/1+J5T8BtE0se3GJeT1Vetpd9SSzrJqs\n3+p+5pp+bqAr4ICPylknnczsPTPbZmZfm9mZZnalmZWZtqacc2875+5zzr0GFFcy6ZvAz4NartSM\nAkrqQhNgIHAy/vDVRuAtM9sn0f4w0A94BPgBcAWwNNH2W2Aq8D/A9xKvXW27vAo0B366a4SZNQUu\nBkYmWUcyy9nlUeAq4Ebgv4DZwEQz+16Jaa4BdgKnAbcDfRLvqchvgT8ByxLL/mEl05ZW1bJqu34h\nuc+cTC2lnQHMcKXOLZjZD4EPgP8DTgA+AR4E7k98FkpNf5+Z5VfxOqOSOqoyDTjZzBrVYh5SSzrE\nl94uMLP8UuOeds7dXZcLdc6NKfm7md0AbML/h58J/A7o45x7LjHJ1/gvTZxzG81sO7DVObeygvlv\nMLO38V+OExOjL8F/Ub5ZYroK63DOfVjVchLvaQLcCtzsnBufGPcr4GzgNqBvYtL/OOf+mBieb2a3\nAOcAL1fwGTaa2WagqLLlV6DCZSWCutrr18xq8pmr/bmBdsDycsY/DrzlnOufWN5LwFvAFOfc++VM\n/wwwuoJl7PJdFe2VWQ40wJ+nWliL+UgtKKDS2xSgZ6lx9XES/HDgIeAU4AD8nnoW0BZ/Dmxf4L1a\nLmYk8LyZNXbObcWH1RjnXEGSdSTrcPwX1e7DTM65IjObChxbYrovSr1vOXBgNZZTHZUt61hqv36T\n/cxV1VKeRsCqkiPM7CD8ntVPSozejv+3KrP3lKhnPVCXh6u3JX5qDypECqj0ttU593UN37sJfxit\ntBb4Q2WVGYc/dNUL/1fsTuA/wD6Vvamaxifme7GZvQecC5xfz3WUPEy1o5y2mhxCLwas1LgGpX4P\nalk1Ufqy3+rWshZoWWrcrvOT00uM6wDMc859WN5MzOw+4L7KS6WLc+6DKqapyP6Jn2tq+H4JgAJK\nKjIP6GpmVup8wUmJtnKZWSvgaKC3c+7/EuNOYs+29iVQiD8MtKCC2WwHsitoA8A5V2hmr+L3nFoD\nK4FYNepIajn4wzvbgR8nhjF/ccapwEtVvLcm1uDPC5V0IrA4yfcHsX7r8jN/DvQoNa4FPtiKEstq\nhj/3VNmhz7o+xHcc8J1zblWVU0qdUUClt30Th09KKnLO7fqrcD8z61iqPe6cWwwMwZ/0ftLM/gEU\n4K/A+jlwUSXL3ID/K/kWM1sKHAI8ht97wTm32cwGAY+YWSH+MGQroJNzbkhiHovx56vaA/n4+4PK\nu+JqJP5Q1mHAy6WmqbSOZJfjnNtiZkOAv5rZWmAR/hxPLjC4kvVQU+8DA83sIvwfAr2AQ0kyoGq6\nfkvNoy4/86TEfFs559Ylxs3E7zXea2Yv4v+dVgBHmNmRzrkyQVvTQ3yJc3RHJH7Nwl9F2RH/b/9t\niUnPSNQqIdJVfOntXPx/9JKvz0u0n5H4veRrAIBz7hvgTOBI4B38VU1XA1c45yZUtMDEF/xV+Cux\n5uDvI+mH/6t+l3uBvybGfwmMAdqUaB+A/wv+P/g9iorOGX2A/yv5WPa+ei/ZOpJdzt3AKPyVbzMT\n87zAObeigulr47kSr4+AzcDr1ZxHEOu3Tj6zc242e7alXeMW4feYbgVm4T/zufh/t6DvEevMnm29\nEf5Kwc/xV1QCYGYNgUuBfwS8bKkm9SQhIvXKzC4ABgHHOueKwq6nNDO7DbjYOXde2LVkOu1BiUi9\ncs5NxO/Rtqlq2pDsAH4ddhGiPSgREYko7UGJiEgkKaBERCSSQr/MvHXr1q59+/Zhl1HGli1baNKk\nSdhlpBSts+TNmzePoqIijj22dMcMUpFU276WLIG1ayE7Gzp0gEYh9EkR1XU2Y8aMtc65A6qaLvSA\nat++PdOnT696wnoWi8XIy8sLu4yUonWWvLy8POLxeCS3/ahKpe3rj3+Ehx6Chg3h3Xfhxz8Op46o\nrjMzW5LMdDrEJyISoKef9uGUnQ2jR4cXTulAASUiEpBXX4VfJy5QHzoULgzrsYxpQgElIhKA99+H\na68F5+Dhh+HGG8OuKPUFGlBmNtLMVpjZJjObb2Y3Bzl/EZEo+vxzuOQS2L4dfvMbuOeesCtKD0Hv\nQT0CtHfO7YfvULS/mXUKeBkiIpGxcCF06QKbN8NVV8ETT4CVfmCK1EigAeWcm+uc29UZp0u8Dg9y\nGSIiUbFqFZx/vv957rnw/POQpRMngQn8MnMzG4x/3ksjfC/Bb5czTU8ST3rNzc0lFosFXUat5efn\nR7KuKNM6S148HqeoqEjrqxqitn1t3ZpNnz4dWbiwGUceuZnf/W4mU6dGq+/bqK2z6qqTvvhKPNws\nD/irc670Uzd369y5s4vivSBRvX8gyrTOkrfrPqiZM2eGXUrKiNL2VVgI3brBe+/B4YfDRx9Bbm7Y\nVZUVpXVWkpnNcM51rmq6OtkZdc4VJR7V3Ab/jBcRkbRQXAzXX+/DKTcXJk2KZjilg7o+WpqDzkGJ\nSJpwDvr0gVGjoFkzmDDB70FJ3QgsoMzsQDO72syamlm2mZ2Pfzz4e0EtQ0QkTH/5Czz5JOyzD/zr\nX/Bf/xV2RektyIskHP5w3jP44FsC9HHOvRngMkREQjFsGNx3n7+EfORIOPvssCtKf4EFlHNuDXBW\nUPMTEYmKN9+Enj398FNPwRVXhFtPptAV+yIilfjoI38DbnEx9OsHvXuHXVHmUECJiFRg7lzo3h0K\nCuCWW+DBB8OuKLMooEREyrF0KVxwAcTjcPHFMHiwujCqbwooEZFS1q3zXRgtWwannw4vvww5oT/e\nNfMooEREStiyxR/W+/JLOO44f4FEGI9rFwWUiMhuO3b4CyI++QTatoWJE6Fly7CrylwKKBERfC8R\nt9wC48dDq1a+C6NDDgm7qsymgBIRAe691z8uo3FjH1JHHx12RaKAEpGM98QT8Ne/+gshxoyBU04J\nuyIBBZSIZLgXX4Q77vDDzz3nLy2XaFBAiUjGeucd6NHDDw8YANddF2o5UooCSkQy0r//DT/7Gezc\nCXfe6V8SLQooEck48+dD167+nqdrr4VHHw27IimPAkpEMsqKFb6XiLVr/fmm556DLH0TRpL+WUQk\nY2zc6ENp8WI4+WR49VVo0CDsqqQiCigRyQgFBb7T1y++gA4d/L1OTZuGXZVURgElImmvqAiuuQYm\nT4aDD/a9RLRuHXZVUhUFlIikNefgtttg7Fho3tz3r9euXdhVSTIUUCKS1v70J3j2Wdh3X3jrLTj+\n+LArkmQpoEQkbT37LDzwgL9K75VX4Iwzwq5IqkMBJSJpaexY6N3bDw8ZApdcEm49Un0KKBFJO5Mn\nwy9+AcXF/hBfz55hVyQ1oYASkbQyaxZcdBEUFvo9qL59w65IakoBJSJpY9EifyPupk1w+eXw97+D\nWdhVSU0poEQkLaxZ47swWrkSfvITGDkSsrPDrkpqQwElIikvP993/rpgAXTsCK+/7i8rl9SmgBKR\nlLZ9O1x2GUyfDocdBhMm+BtyJfUpoEQkZRUXww03+AcPHnCA/3nQQWFXJUFRQIlISnIO7roLXnrJ\nd/o6YQIccUTYVUmQFFAikpIGDIAnnvCPyxg7Fjp1CrsiCZoCSkRSzvPPwx/+4IdfeAF++tNw65G6\noYASkZQyfjzcdJMfHjQIrr463Hqk7gQWUGa2r5kNM7MlZrbZzGaaWZeg5i8i8skncMUV/vlO994L\nv/lN2BVJXQpyDyoHWAqcBTQH+gKjzax9gMsQkQy1ZEljunWDbdvgxhvhz38OuyKpazlBzcg5twV4\noMSocWa2COgELA5qOSKSeZYtgz/84QTWr4fu3f1jNNSFUfqrs3NQZpYLHAXMratliEj627DB96+3\nenVDTjsNRo2CnMD+tJYoq5N/ZjNrALwIPO+c+6qc9p5AT4Dc3FxisVhdlFEr+fn5kawryrTOkheP\nxykqKtL6qkJhYRZ33XUic+c259BDN3P33bOYNm1n2GWljFT/Pxl4QJlZFjAC2A7cXt40zrmhwFCA\nzp07u7y8vKDLqLVYLEYU64oyrbPktWjRgng8rvVViZ07fRdGc+ZAmzYwYMAcLrro9LDLSimp/n8y\n0IAyMwOGAblAV+fcjiDnLyKZwTn41a/gzTehZUuYNAlWry4MuyypZ0GfgxoCHANc6JzbFvC8RSRD\n9OsHw4ZBo0b+vqdjjw27IglDkPdBtQN6AR2BlWaWn3hdE9QyRCT9Pfmkv4Q8OxtGj4ZTTw27IglL\nkJeZLwF04aeI1NioUfDb3/rhf/7TX1IumUtdHYlIJLz7Llx3nT//9Je/QI8eYVckYVNAiUjoPvsM\nLr0UduyAPn32dAQrmU0BJSKhWrgQunTxj23/+c/h8cfVS4R4CigRCc2qVXDeebB6tX9kxvDhkKVv\nJUnQpiAiodi0ye85ffONf9jgmDGwzz5hVyVRooASkXpXWOjPOX3+uX9M+9tvQ7NmYVclUaOAEpF6\nVVTkr9Z7/3046CB45x048MCwq5IoUkCJSL1xzt/n9OqrsN9+MGECHHZY2FVJVCmgRKTePPwwPP20\nP9f0xhvQsWPYFUmUKaBEpF7885/Qt6+/hPyllyCFO9mWeqKAEpE698Yb0KuXHx482D9GQ6QqCigR\nqVMffghXXw3FxfDf/+0foyGSDAWUiNSZOXPgwguhoAB69vQBJZIsBZSI1IklS+D88yEe9/c8DR6s\nLoykehRQIhK4tWt9OC1fDmee6S+KyM4OuypJNQooEQnUli3+OU7z5sHxx/sLJBo2DLsqSUUKKBEJ\nzI4dcMUV8Omn0K4dTJwILVqEXZWkKgWUiASiuBhuusn3DtG6te/C6OCDw65KUpkCSkQCcc89MGIE\nNGkC48fDUUeFXZGkOgWUiNTa3/4Gjz0GOTn+sRknnxx2RZIOFFAiUisvvgh33umHhw/3V++JBEEB\nJSI1NmkS9Ojhhx9/HK65JtRyJM0ooESkRqZN833q7dwJv/893HFH2BVJulFAiUi1zZsH3br5e55+\n+Uv4y1/CrkjSkQJKRKpl+XJ/nmntWujSxT9GI0vfJFIHtFmJSNLicbjgAt/P3imn+CfjNmgQdlWS\nrhRQIpKUbdvgootg9mw4+mh/r1OTJmFXJelMASUiVSoq8lfoffABHHKIv3qvVauwq5J0p4ASkUo5\nB717w+uv+371Jk6Etm3DrkoygQJKRCr14IMwdKjvkfytt+C448KuSDKFAkpEKjRkiA+orCwYNQpO\nPz3siiSTKKBEpFyvvQa33eaHn33WXyAhUp8UUCJSRizmL4pwDvr3h5tvDrsiyUSBBpSZ3W5m082s\n0MyGBzlvEakfM2fCxRfD9u1w++1w331hVySZKifg+S0H+gPnA40CnreI1LFvvvG9Q2zaBFdeCQMH\nglnYVUmmCjSgnHNjAcysM9AmyHmLSN1avdp3YbRyJZx9NrzwAmRnh12VZLKg96CSYmY9gZ4Aubm5\nxGKxMMqoVH5+fiTrijKts+TF43GKioois762bs3mjjtO5Ouv9+PIIzdzxx0zmTq1KOyy9qLtq/pS\nfZ2FElDOuaHAUIDOnTu7vLy8MMqoVCwWI4p1RZnWWfJatGhBPB6PxPravh26d/c9lB9+OHzwQTNy\nc88Iu6wytH1VX6qvM13FJ5LBiov9Awf/93/hwAN9F0a5uWFXJeIpoEQylHP+IYMvvwxNm8KECX4P\nSiQqAj3EZ2Y5iXlmA9lm1hDY6ZzbGeRyRKT2Hn0UBg3yj8v417/gpJPCrkhkb0HvQfUFtgH3ANcm\nhvsGvAwRqaX/+R+45x5/CfnIkXDOOWFXJFJW0JeZPwA8EOQ8RSRY48bBLbf44UGD/P1OIlGkc1Ai\nGWTqVB9IRUVw//3w61+HXZFIxRRQIhniP/+Bbt38k3FvugkeeijsikQqp4ASyQBLl/peIjZs8L2S\nP/OMujCS6FNAiaS59evhggtg2TL/PKdXXoGcUG7RF6keBZRIGtu6FS680B/e+8EP4M03oZG6cZYU\noYASSVM7d8JVV8HHH8Ohh8LEidCyZdhViSRPASWShpyDnj39JeX77++7MGqj5wtIilFAiaSh++/3\nN+M2agTjx8Mxx4RdkUj1KaBE0sygQfDII/5ZTq+9Bj/6UdgVidSMAkokjbzyCvTp44efew66dg23\nHpHaUECJpIl334Vf/tIPP/ronmGRVKWAEkkDM2bApZfCjh3+ERp33RV2RSK1p4ASSXELFkCXLpCf\nD9dcA489pl4iJD0ooERS2MqVvgujNWvgvPP8eacs/a+WNKFNWSRFbdzouzBatAh++EMYMwb22Sfs\nqkSCo4ASSUEFBXDJJTBrFhx5pL/XqWnTsKsSCZYCSiTFFBXBdddBLAbf+x688w4ccEDYVYkETwEl\nkkKcg9/8xt+Au99+vn+99u3DrkqkbiigRFLIn/8MgwfDvvv6nslPOCHsikTqjgJKJEX84x/Qr5+/\nSu+ll+Css8KuSKRuKaBEUsC//gW/+pUfHjwYfvazcOsRqQ8KKJGImzIFrr4aiovhwQehV6+wKxKp\nHwookQibPRsuuggKC/0eVL9+YVckUn8UUCIRtXix7yVi40Z/SO+pp9SFkWQWBZRIBK1d68NpxQp/\nMcSLL/rnO4lkEgWUSMTk50O3bjB/Ppx4IrzxBjRsGHZVIvVPASUSITt2wOWXw7Rp/gbcCROgefOw\nqxIJhwJKJCKKi+HGG2HSJN910Tvv+K6MRDKVAkokIu6+G0aOhCZN4O23fSewIplMASUSAQMG+FeD\nBvD669C5c9gViYRPASUSshEj4Pe/98PPPw8//Wm49YhEhQJKJEQTJvjzTgBPPAE//3m49YhESaAB\nZWb7m9nrZrbFzJaY2S+CnL9IOtm6NZvLL4edO/35pz59wq5IJFpyAp7f08B2IBfoCIw3s1nOubkB\nL0ckpW3dCt9805SiIrj+enjkkbArEokec84FMyOzJsAG4Djn3PzEuBHAd865eyp6X7NmzVynTp0C\nqSFI8XicFi1ahF1GStE6S05BAUybNhPnYP/9O3LccerCKBnavqovquts8uTJM5xzVV4KFOQe1FHA\nzl3hlDALKPPUGjPrCfQEaNCgAfF4PMAyglFUVBTJuqJM66xqO3dmsWBBU5yDrCzHIYdsZOPGYP5I\nTHfavqov1ddZkAHVFNhUatxGoFnpCZ1zQ4GhAJ07d3bTp08PsIxgxGIx8vLywi4jpWidVS4e9/3q\nbd8OTZvm0b79Rr744vOwy0oZ2r6qL6rrzJI8ZBBkQOUD+5Uatx+wOcBliKSkjRuha1f44gvo0AFa\ntYItW7TnJFKZIK/imw/kmFnJ+99PBHSBhGS0DRv8vU1Tp0Lbtr4LowYNwq5KJPoCCyjn3BZgLPAn\nM2tiZj8GLgZGBLUMkVSzdi2ccw78+99w2GEwebIPKRGpWtA36vYGGgGrgZeBW3WJuWSq1avh7LPh\n8899v3qTJ/seykUkOYHeB+WcWw9cEuQ8RVLRwoXQpQssWABHHw3vv6+eyUWqS10diQRs2jQ49VQf\nTh07QiymcBKpCQWUSIDeegvy8mDNGjjvPJgyBXJzw65KJDUpoEQC4Bz8/e9wySWwbRvccAOMGwfN\nytwFKCLJUkCJ1NK2bdCjB/z2t/6puH/8IwwbpkvJRWor6M5iRTLKkiXws5/BZ59B48Y+mK6+Ouyq\nRNKDAkqkhiZNgmuv9fc6ff/7/km4J5wQdlUi6UOH+ESqqbAQ7rwTLrjAh9P55/sbcRVOIsHSHpRI\nNcyb5596+/nnkJ0NDz0Ef/iDHxaRYCmgRJJQVARPPQX33ecfNvj978NLL8Epp4RdmUj6UkCJVOHL\nL+Gmm3xnrwDXXefDar/SffeLSKB0DkqkAoWF0L+/7w1i6lQ4+GB44w144QWFk0h90B6USDnGj4c+\nfeDrr/3vN98Mjz0GEXx6tkjaUkCJlLBgAfzudz6gAI45xh/OO/vscOsSyUQ6xCcCrFwJt90Gxx7r\nw6lZM/jb32DWLIWTSFi0ByUZLR6HAQPgiSf81XlZWb4fvYcfhoMOCrs6kcymgJKMtHYtDBoETz4J\nGzf6cRdfDH/+M/zgB+HWJiKeAkoyynff+b2lIUP8HhPAT37ig+nUU8OtTUT2poCSjDBtGgwcCK++\nCjt3+nFdu8L998Npp4Vbm4iUTwElaWvLFnjtNXjmGfjkEz8uOxuuvBLuvhtOOinc+kSkcgooSSvO\nwaef+sdejBoFmzf78S1bQs+e/kq9Qw8Nt0YRSY4CStLCt9/C6NHw3HO+a6JdTjsNbrzRP6OpSZPw\n6hOR6lNAScpavNgfwnv1VX+OaZcDD4Trr/fBdPTRoZUnIrWkgJKU4Zy/cXbCBBg7FqZP39PWuDF0\n6wa/+IX/qceti6Q+BZRE2oYN8O67PpQmToQVK/a0NWkC3bvDFVdAly4+pEQkfSigJFI2boQPP4TJ\nk2HKFL+XVFS0p/3gg/2TbLt18z8VSiLpSwEloXEOlizx54+mTvWhNGsWFBfvmSYnB846y+8hdekC\nxx8PZuHVLCL1RwEl9cI534vDF1/4vaJp0/xrzZq9p2vQAH70Ix9KZ54JP/6x77hVRDKPAkoCt2kT\nzJkDs2fv/dqwoey0rVvDD38IJ58MZ5zhuxvSYTsRAQWU1FBhIXzzjX9+0q7XtGknsnYtLF1a/nv2\n398fouvUyQfSySdD+/Y6ZCci5VNASbk2bfI3vy5dWvbn4sV+uOS5Iq8lAPvs45+rdPzxe14nnADf\n+57CSESSp4DKIEVFsG4drFq192v1av9z5UpYtsyHz6ZNlc8rKwu+/3048sg9r23bvuCyy06gXTvd\nhyQitaeASkGFhb6PuQ0b/Gv9+sp/btjgL0ZYs6a8vZ7yNWoEbdv6fuvK+3nYYX5PqaRYbD1HHBH8\n5xWRzBRIQJnZ7UAP4HjgZedcjyDmm4qcgx07YNs2/yooqHp461bIz/ehs3nznuGKxu3YUfP6WraE\n3Nw9r4MO2vv3Qw7xAbT//jocJyLhCmoPajnQHzgfaFSdNxYWwvz5/vBT6Vdxcfnjq2qrqn3Hjj2v\n7dv3/rlr+LvvjmXQoKqn2zW8K3AKCpLfS6mpnBx/6XXLlj5IWrbce7i8ca1a+T7qSu/1iIhEVSAB\n5ZwbC2BmnYE21XnvnDnz6NAhr9TYK4HewFagaznv6pF4rQUuL6f9VuAqYClwXTntdwIXAvOAXuW0\n9wXOBWYCfcppfxg4DfgYuK9Ma3b2QBo37khW1rsUFPQnK4vdr+xsOP74ZznggA6sW/cW8+Y9TnY2\ne71uvXUE7dodyowZo5g4cUiZ9rFjX6N169YMHz6c4cOHs337nvNJAG+//TaNGzdm8ODB/P3vo8vU\nF4vFABgwYADjxo3bq61Ro0ZMmDABgIceeoj33ntvr/ZWrVoxZswYAO69916mTp26uy0ej3Pccccx\ncuRIAPr06cPMmTP3ev9RRx3F0KFDAejZsyfz58/fq71jx44MHDgQgGuvvZZly5bt1X7qqafyyCOP\nAHDZZZexbt26vdrPOecc+vXrB0CXLl3Ytm3bXu3du3fnrrvuAiAvL6/Murnyyivp3bs3W7dupWvX\nsttejx496NGjB2vXruXyy8tue7feeitXXXUVS5cu5brrym57d955JxdeeCFbt27l66+/LlND3759\nOffcc5k5cyZ9+pTd9h5++GFOO+00Pv74Y+67r+y2N3DgQDp27Mi7775L//79y7Q/++yzdOjQgbfe\neovHH3+8TPuIESM49NBDGTVqFEOGDCnT/tpre297pZXc9kaPDnbbKy4uZsqUKUDZbQ+gTZs22vZK\nbXvxeJwWLVoAe7a9efPm0atX2e+9+tz2khXKOSgz6wn09L81YZ99ihOHkxxm0KxZAS1bbga2sGzZ\nzsR79hxyOuCAfA48cB1FReuYP3/H7vFmDoC2beMcfPAKCgtXMWvWdsxciWngmGPW0K7dYrZsWcbU\nqQW723fVcPrpSzj00M/YvPkbJk3akmhzu39eeumXdOjQgEWL5jJmzKbd47Oy/M9f/3o6RxwRZ8aM\nWYwYES/z+W+++VPatl3Bxx/PJh4v296mzVRatVpITs5ciovjFBfvfVjvo48+onnz5nz11Vflvn/K\nlCk0bNiQ+fPnl9u+60ti4cKFZdq3bdu2u33RokVl2ouLi3e3f/vtt3u1FxUVsWrVqt3ty5YtK/P+\n5cuX725fvnx5mfZly5btbl+1alWZ9m+//XZ3+5o1a9hU6mqORYsW7W5fv349hYWFe7UvXLhwd3t5\n62b+/PlLNR+DAAAF6UlEQVTEYjEKCgrKbf/qq6+IxWJs3Lix3Pa5c+cSi8VYvXp1ue2zZ8+mWbNm\nbN68GedcmWlmzZpFTk4OX3/9dbnv/+yzz9i+fTtz5swpt3369OnE43FmzZpVbvunn37KihUrmD27\n/G1v6tSpLFy4kLlz55bbHua217hx4wq3PYAGDRpo2yu17RUVFe0e3rXtlbfuoH63vWSZcy7piauc\nmVl/oE11zkF17tzZTS/ZLXVExGKxcv/KkYppnSUvLy+PeDxe5q98qZi2r+qL6jozsxnOuc5VTZeV\nxIxiZuYqeH0YTLkiIiJ7q/IQn3Murx7qEBER2UtQl5nnJOaVDWSbWUNgp3NuZxDzFxGRzFPlIb4k\n9QW2AfcA1yaG+wY0bxERyUBBXWb+APBAEPMSERGB4PagREREAqWAEhGRSFJAiYhIJCmgREQkkhRQ\nIiISSQooERGJJAWUiIhEkgJKREQiSQElIiKRpIASEZFIUkCJiEgkKaBERCSSFFAiIhJJCigREYkk\nBZSIiESSAkpERCJJASUiIpGkgBIRkUhSQImISCQpoEREJJIUUCIiEkkKKBERiSQFlIiIRJICSkRE\nIkkBJSIikaSAEhGRSFJAiYhIJCmgREQkkhRQIiISSQooERGJJAWUiIhEkgJKREQiqdYBZWb7mtkw\nM1tiZpvNbKaZdQmiOBERyVxB7EHlAEuBs4DmQF9gtJm1D2DeIiKSoXJqOwPn3BbggRKjxpnZIqAT\nsLi28xcRkcxU64AqzcxygaOAuZVM0xPoCZCbm0ssFgu6jFrLz8+PZF1RpnWWvHg8TlFRkdZXNWj7\nqr5UX2fmnAtuZmYNgAnAQudcr2Te07lzZzd9+vTAaghKLBYjLy8v7DJSitZZ8vLy8ojH48ycOTPs\nUlKGtq/qi+o6M7MZzrnOVU1X5TkoM4uZmavg9WGJ6bKAEcB24PZaVS8iIhmvykN8zrm8qqYxMwOG\nAblAV+fcjtqXJiIimSyoc1BDgGOAc51z2wKap4iIZLAg7oNqB/QCOgIrzSw/8bqm1tWJiEjGCuIy\n8yWABVCLiIjIburqSEREIkkBJSIikRTofVA1KsBsDbAk1CLK1xpYG3YRKUbrrHq0vqpH66v6orrO\n2jnnDqhqotADKqrMbHoyN5LJHlpn1aP1VT1aX9WX6utMh/hERCSSFFAiIhJJCqiKDQ27gBSkdVY9\nWl/Vo/VVfSm9znQOSkREIkl7UCIiEkkKKBERiSQFlIiIRJICKklmdqSZFZjZyLBriSoz29fMhpnZ\nEjPbbGYzzaxL2HVFjZntb2avm9mWxLr6Rdg1RZW2qdpJ9e8tBVTyngb+HXYREZcDLAXOApoDfYHR\nZtY+xJqi6Gn8gz1zgWuAIWb2g3BLiixtU7WT0t9bCqgkmNnVQBx4L+xaosw5t8U594BzbrFzrtg5\nNw5YBHQKu7aoMLMmwGVAP+dcvnPuQ+BN4LpwK4smbVM1lw7fWwqoKpjZfsCfgDvCriXVmFkucBQw\nN+xaIuQoYKdzbn6JcbMA7UElQdtUctLle0sBVbWHgGHOuWVhF5JKzKwB8CLwvHPuq7DriZCmwKZS\n4zYCzUKoJaVom6qWtPjeyuiAMrOYmbkKXh+aWUfgXOCJsGuNgqrWV4npsoAR+PMst4dWcDTlA/uV\nGrcfsDmEWlKGtqnkpdP3Vq2fqJvKnHN5lbWbWR+gPfCtmYH/6zfbzI51zp1U5wVGTFXrC8D8ihqG\nvwCgq3NuR13XlWLmAzlmdqRzbkFi3InokFWFtE1VWx5p8r2lro4qYWaN2fuv3bvw//C3OufWhFJU\nxJnZM0BH4FznXH7Y9USRmb0COOBm/Lp6GzjNOaeQKoe2qepJp++tjN6DqopzbiuwddfvZpYPFKTa\nP3J9MbN2QC+gEFiZ+OsNoJdz7sXQCoue3sBzwGpgHf6LQ+FUDm1T1ZdO31vagxIRkUjK6IskREQk\nuhRQIiISSQooERGJJAWUiIhEkgJKREQiSQElIiKRpIASEZFIUkCJiEgk/T/XSE/diHEg1QAAAABJ\nRU5ErkJggg==\n", - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "plt.plot(z, elu(z), \"b-\", linewidth=2)\n", "plt.plot([-5, 5], [0, 0], 'k-')\n", @@ -574,25 +494,7 @@ "cell_type": "code", "execution_count": 25, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Saving figure selu_plot\n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xt8FPW5x/HPEwIIiEZBchTUeMN7RYhtxfaYVqyKl9qD\n1VpAsVoQqxaRVkUUDnCwtVTRFrAIgoJWqeIN0VZpo7WIFSTeRcWCICoXXTHhEhJ+54/fxixLLptk\nNjO7+b5fr3llmZnMPDtM9rsz++yMOecQERGJmpywCxAREamJAkpERCJJASUiIpGkgBIRkUhSQImI\nSCQpoEREJJIUUCJ1MLOVZjaiGdYzxszebIb15JjZn8xso5k5MytK9zrrqWeWmc0PswaJLgWUpMTM\n9jGzKfEX7G1m9pmZLTSzUxPmKY6/6CUPDybM48zsvFrWMcjMSmuZVuvvBaGOgDgBmBLgegriz6Uw\nadJE4OSg1lOHvsAlwNnAvsCiZlgnZlYUf96dkyb9EhjQHDVI5skNuwDJGI8A7YFLgQ+ALvgX1E5J\n880ERiaN25L26tLEObe+mdZTCtQYzgE7FPjEOdcswVQf59yXYdcg0aUjKKmXmeUB3wWud84tdM6t\ncs694pyb6Jx7MGn2zc65T5OGtL8ImdnpZvZPM/vCzD43s7+a2ZFJ8+xnZvfHT29tNrMSM/uemQ0C\nRgNHJxz1DYr/zten+MzsATN7JGmZOWa22syGp1jHf+I/X4mvpzj+ezsdwcWXe1N82dvM7A0z+2HC\n9KojsX5m9mz8+bydeERbwzaaBdwOHBD/3ZXx8cVm9sfkeRNPvcXnmWJmE8xsg5mtM7OJZpaTME+b\n+PRV8Zo/NLOrzawA+Ed8tvXxdc+qZT1tzWxS/Ah9q5ktNrPvJEyvOhI7xcxejj/vJWbWs7bnLZlL\nASWpqHp3f46Z7RZ2MbXoAEwCvgkUAV8CT5pZGwAz6wA8DxQA5wLHAmPjv/sQ8HtgOf60177xccnm\nAGea2Z4J406Oz//nVOqIjwc4Pf57/1PL8/kl8CvgunitjwLzzKxH0nz/B9wJHAe8AjxoZrvXscyx\nwJr4uk+oZb7a9AcqgN7AlcAw4IKE6fcCFwHDgSPxR9sxYDXQLz7P0fF1/7KWddwaX+bPgOOBN4Bn\nzGzfpPluAa4HegIbgfvNzBr4fCTqnHMaNNQ74F9gPge2Ai/hPzP5VtI8xUA51YFWNVyRMI8Dzqtl\nHYOA0lqm1fp7tczfAagEvhP/98+Br4DOtcw/BnizhvErgRHxx7nAZ8ClCdOnA39rQB0F8edSWNf6\ngY+Bm2vYvnOSljMkYXrX+Ljv1FHPCGBlDcv9Y9K4WcD8pHleSprnWWB6/PFh8XWfXst6i+LTO9e2\nnvi2KgcuSpjeClgBjE9azmkJ85wUH9ct7L8TDcEOOoKSlDjnHgH2w3+4/jT+XfRiM0v+vOkhoEfS\ncH+66zOzQ+Kn4FaY2SZ8kOQAB8RnOR543Tm3obHrcM5V4J9f//g62+KDe04D6kjlueyB39b/Spr0\nInBU0rjXEx6vjf/skuq6Guj1pH+vTVjX8cAOqk/lNcYhQGsSnrdzrhL/hijM5y0hUZOEpMw5txX/\nrvlZYKyZTQfGmNlE51x5fLYvnXMfNHIVm4B2ZtbaObe9amT8MzDwp8tqMx9/6moI/uijAngbaFPH\n7zTGHOAlM+sKfCu+/HnNWEfy7Qe+3k7OORc/y9XQN547gOTTY61rmG970r9dI9bVWLU+74RpesOd\nZfQfKk3xNv5NTlCfSy3H75PHJ43vmTB9F2bWCTgCmOCce8459w7QkZ3fgC0DvlFDm3OVcvzppDo5\n5/6N72K8EH8k9bjzHXip1lEV5LWuyzm3CX9UcFLSpO/gt3nQ1uM/F0p0XAOXUYL/v/teLdPrfd74\nU3nlJDxvM2sFnEh6nrdEnI6gpF7xF96/APfgT618BRQCvwYWxl9Qq7Q3s/9KWkS5c+7zhH8X1PBh\n/4fOubfM7G/A9HhX3AqgO3AHMNc591EtJX4BbAB+bmar8Z/F/A5/9FLlAfyH6o+b2fX4o5tjgK+c\nc//Af9Z0YLwb7KP4+G21rO9+4DL850CJTQ6p1LEO33Z/WryLbqurucvxd/ij1PeBpfjvCn2X6rAO\n0t+BSWZ2Dv5NwBBgf/w2SYlz7j0zm4v/v/sl8CrQDShwzs0GVuGPdM40syeBLVXBnrCMMjObCvzW\nzDbgOx6vAfIJ8LtokkHC/hBMQ/QHoC0wAd8l9gWwGXgfuA3YO2G+YvyLUPLwYsI8NU13wFnx6Xn4\nQPogvp73gN8Cu9dT4/eBN/FNHG8Cp+EbNAYlzNMN/xlSLL7sZUBRwnN8OP78XNXvkdAkkbCcg+Pz\nfAbkNqKOy/AhWAkUx8eNYecmiRzgJnwHXDm+m+3chOkF1NxsUWczCTU3SbQGJuPDdQPwv9TcJFFf\nI0VbfBfex8A2/BuMKxOm3wR8gj+lOKuOZUyKb9ttwGISmj6oodmitm2hIfMHi/8Hi4iIRIo+gxIR\nkUhSQImISCQpoEREJJIUUCIiEkmht5l37tzZFRQUhF3GLsrKyujQoUPYZWQUbbPULV++nMrKSo46\nKvkCCVKbqO5f5eXwzjtQUQGdOkGUXs6ius2WLl26wTm3T33zhR5QBQUFLFmyJOwydlFcXExRUVHY\nZWQUbbPUFRUVEYvFIrnvR1UU969Nm+Ckk3w4fe978Mwz0Cboa5c0QRS3GYCZrUplPp3iExFphIoK\nuOACePNNOOIIeOSRaIVTNlBAiYg0kHNw9dX+iKlzZ3jqKdhrr7Cryj4KKBGRBpo0CaZOhbZt4fHH\n4eCDw64oOymgREQa4LHH4Npr/eN774XevcOtJ5sFGlBmNsfMPjGzTWb2npldFuTyRUTCtHQp9O/v\nT/GNH+8/g5L0CfoI6hb81Yv3AM4BxptZr4DXISLS7FavhrPPhs2b4eKLYWTyrTolcIEGlHPuLVd9\ni4Kqq1QfEuQ6RESa26ZNcOaZ8MknUFQE06aBJd/iUQIX+PegzGwKMAhoh7+dwYIa5hkMDAbIz8+n\nuLg46DKarLS0NJJ1RZm2WepisRiVlZXaXg0Q1v5VWWmMHHkMb7zRif3338zw4a+yaFFF/b8YAZn+\nN5mW220k3AWzCPitS7h9d7LCwkIXxS8rRvULblGmbZa6qi/qlpSUhF1Kxghj/3IOrrwSpkzx7eSL\nF8MhGXROKKp/k2a21DlXWN98aenic85VOudexN8gbmg61iEikm533OHDqU0b372XSeGUDdLdZp6L\nPoMSkQz0+OMwfLh/PGuWv6SRNK/AAsrMupjZT8xsdzNrZWanARcCC4Nah4hIc1i6FH76U3+Kb9w4\nuPDCsCtqmYJsknD403l34YNvFTDMOfdEgOsQEUmr5HbyG28Mu6KWK7CAcs6tB04OankiIs3tq6/g\nrLN8O/nJJ6udPGy61JGICP7q5D/5Cbz+OnTvDvPm6erkYVNAiUiL5xwMGwYLFvibDi5YAHvvHXZV\nooASkRbvzjth8mS1k0eNAkpEWrQnn4RrrvGPZ86E73wn3HqkmgJKRFqsV1/1nzs5B2PH+tZyiQ4F\nlIi0SGvWVLeTX3QRjBoVdkWSTAElIi1OVTv52rXw3/+tdvKoUkCJSItSUeGvDPHaa3DYYb6dvG3b\nsKuSmiigRKRFGT4cnnqqup28U6ewK5LaKKBEpMW48074wx+q28kPPTTsiqQuCigRaRHmz69uJ7/n\nHrWTZwIFlIhkvWXLfDv5jh0wZgz07x92RZIKBZSIZLU1a3zHXlkZDBgAN98cdkWSKgWUiGSt0lL/\nXaeqdvLp09VOnkkUUCKSlSorfTt5SYnayTOVAkpEstLw4b4xYu+9q9vKJbMooEQk6/zhD76lvKqd\n/LDDwq5IGkMBJSJZ5amn/L2dAGbMgO9+N9x6pPEUUCKSNUpK4IILfDv56NG+a08ylwJKRLLCxx9X\nt5P37+8DSjKbAkpEMl5VO/nHH/srRMyYoXbybKCAEpGMVlnpbzS4bJm/tt5jj6mdPFsooEQko117\nrb9tu9rJs48CSkQy1uTJcMcd0Lo1PPoodO8edkUSJAWUiGSkBQvg6qv94xkz/KWMJLsooEQk47z2\nWnU7+c03w8CBYVck6aCAEpGMsnatbycvLfXNEWPGhF2RpIsCSkQyRlmZbydfswZOOknt5NlOASUi\nGaGqnfzVV+GQQ3w7+W67hV2VpJMCSkQywl13HcITT8Bee/kGic6dw65I0k0BJSKRN2UKPPzw/mon\nb2EUUCISaU8/DVdd5R9Pnw4nnxxuPdJ8AgsoM2trZjPMbJWZfWVmJWZ2RlDLF5GW57XX4PzzfTv5\nwIErueiisCuS5hTkEVQusBo4GdgTGAXMNbOCANchIi1EYjv5hRfCJZesDLskaWaBBZRzrsw5N8Y5\nt9I5t8M5Nx/4D9ArqHWISMuQ3E5+zz1qJ2+JctO1YDPLB7oDb9UwbTAwGCA/P5/i4uJ0ldFopaWl\nkawryrTNUheLxaisrNT2qkFlJYwefQyvvtqZ/fbbwogRr7J48XbtX42Q6dvMnHPBL9SsNfA0sMI5\nN6SueQsLC92SJUsCr6GpiouLKSoqCruMjKJtlrqioiJisRglJSVhlxI5114Lt93m28lfegkOP9yP\n1/7VcFHdZma21DlXWN98gXfxmVkOMBsoB64Mevkikr2mTvXh1Lo1zJtXHU7SMgV6is/MDJgB5AN9\nnXPbg1y+iGSvZ56pbie/+26I4Bt/aWZBfwY1FTgS6OOc2xLwskUkS73xhm8nr6yEG2+Eiy8OuyKJ\ngiC/B3UgMAToAXxqZqXxoX9Q6xCR7PPJJ3DmmfDVV/4WGmPHhl2RREVgR1DOuVWAGkFFJGVV7eSr\nV0Pv3jBrFuTo+jYSp11BREJRWQkDBsDSpXDwwbo6uexKASUiobjuOh9KeXnw1FOwzz5hVyRRo4AS\nkWZ3113w+99Dbi488ggccUTYFUkUKaBEpFn99a9wZfwbktOmwfe/H249El0KKBFpNm+8AT/+sf/8\naeRIuOSSsCuSKFNAiUiz+PRTf3XyqnbycePCrkiiTgElImm3eTOccw589BF8+9swc6bayaV+2kVE\nJK127PDt5K+8AgcdBI8/Du3ahV2VZAIFlIik1XXXwaOPwp57+nbyLl3CrkgyhQJKRNJm2jSYONG3\nk8+bB0ceGXZFkkkUUCKSFn/7G1xxhX/8pz+pnVwaTgElIoF7883qdvIbboCf/SzsiiQTKaBEJFCf\nfuqvTr5pkw+p8ePDrkgylQJKRAKT3E5+771qJ5fG064jIoHYsQMuusi3kxcUqJ1cmk4BJSKBuOEG\nf+FXtZNLUBRQItJkd98Nt95afXXyo44KuyLJBgooEWmSZ5+FoUP947vuglNOCbceyR4KKBFptLfe\ngvPO8+3k110Hl14adkWSTRRQItIon31W3U5+3nkwYULYFUm2UUCJSINVtZOvWgXf+hbcd5/aySV4\n2qVEpEGq2sn//W+1k0t6KaBEpEFGjvSdenvs4dvJ8/PDrkiylQJKRFI2fTr89rdqJ5fmoYASkZQ8\n9xxcfrl/PHUq9OkTbj2S/RRQIlKvt9+ubif/9a/hssvCrkhaAgWUiNSpqp38yy+hXz+45ZawK5KW\nQgElIrXasgV++ENYuRK++U21k0vz0q4mIjWqaid/+WU48EB44glo3z7sqqQlUUCJSI1uvBEefljt\n5BIeBZSI7OKee+A3v4FWrXxIHX102BVJS6SAEpGdLFwIQ4b4x1OnwqmnhluPtFwKKBH52ttv+069\nigr41a/g5z8PuyJpyQINKDO70syWmNk2M5sV5LJFJL3WratuJ/+f//Gn+ETClBvw8tYC44HTAF0+\nUiRDJLaTn3ACzJ6tdnIJX6AB5ZybB2BmhUC3IJctIumxYwcMGgSLF8MBB6idXKIj6COolJjZYGAw\nQH5+PsXFxWGUUafS0tJI1hVl2mapi8ViVFZWRmJ73X33QcydeyAdOlQwZswy3n23jHffDbuqXWn/\narhM32ahBJRzbhowDaCwsNAVFRWFUUadiouLiWJdUaZtlrq8vDxisVjo22vmTHjgAd9OPm9eLj/4\nwQmh1lMX7V8Nl+nbTGeZRVqov/8dBg/2jydPhh/8INx6RJIpoERaoHfeqW4nHzGi+ntPIlES6Ck+\nM8uNL7MV0MrMdgMqnHMVQa5HRBqvqp08FoMf/cjfgFAkioI+ghoFbAGuBwbEH48KeB0i0khbt8K5\n58J//gOFhTBnjtrJJbqCbjMfA4wJcpkiEoyqdvKXXoL991c7uUSf3juJtBA33wwPPQQdO/qrk++7\nb9gVidRNASXSAsycCf/3f76d/C9/gWOPDbsikfopoESy3D/+Ud1O/sc/wmmnhVuPSKoUUCJZ7N13\n/YVfKypg+HC4/PKwKxJJnQJKJEutX1/dTn7uuXDrrWFXJNIwCiiRLFTVTv7hh9Crl28nb9Uq7KpE\nGkYBJZJlduyASy6BRYt8O/mTT0KHDmFXJdJwCiiRLDN6NDz4oG8nnz9f7eSSuRRQIlnk3nth/Hh/\nOm/uXPjGN8KuSKTxFFAiWaK4GH7+c//4D3+A008PtRyRJlNAiWSB5ct9O/n27XDNNTB0aNgViTSd\nAkokw23Y4NvJv/gCzjkHfve7sCsSCYYCSiSDVbWTr1gBPXtW3x1XJBsooEQylHPws5/Bv/4F3bqp\nnVyyjwJKJEONHg1//jPsvru/Ovl++4VdkUiwFFAiGei++2DcOH+zwYceUju5ZCcFlEiGef55uOwy\n//jOO6Fv33DrEUkXBZRIBlm+HH70I99OPmwY/OIXYVckkj4KKJEMkdxOPnFi2BWJpJcCSiQDbNvm\nj5xWrIDjj4f771c7uWQ/BZRIxFW1k7/4InTt6tvJd9897KpE0k8BJRJxY8b4L+BWtZN37Rp2RSLN\nQwElEmGzZ8PYsdXt5McdF3ZFIs1HASUSUS+8AJde6h/fcYfayaXlUUCJRND771e3k199NVx5ZdgV\niTQ/BZRIxGzc6I+WPv8czj4bbrst7IpEwqGAEomQbdv81ck/+MC3k+vq5NKSKaBEIsI5fwkjtZOL\neAookYgYOxbmzPG3zJg/X+3kIgookQiYM8d/3yknBx58EHr0CLsikfApoERC9s9/VreTT5oEZ50V\nbj0iUaGAEgnR++/7pojycrjqKj+IiBdoQJnZ3mb2qJmVmdkqM/tpkMsXySYVFcaZZ/p28jPPhNtv\nD7sikWjJDXh5k4FyIB/oATxlZq85594KeD0iGc05WLmyA2Vl/vOmBx9UO7lIMnPOBbMgsw7AF8Ax\nzrn34uNmAx87566v7fc6duzoevXqFUgNQYrFYuTl5YVdRkbRNkvdyy+XsHUrtGnTg549oW3bsCuK\nPu1fDRfVbfb8888vdc4V1jdfkEdQ3YGKqnCKew04OXlGMxsMDAZo3bo1sVgswDKCUVlZGcm6okzb\nLDXbtuWwdat/3LVrGVu2bGfLlnBrygTavxou07dZkAG1O7ApadyXQMfkGZ1z04BpAIWFhW7JkiUB\nlhGM4uJiioqKwi4jo2ib1c856NMH3n23iLy8cj78cFHYJWUM7V8NF9VtZmYpzRdkk0QpsEfSuD2A\nrwJch0hGmz8f/v53yM2Frl112CRSlyAD6j0g18wOSxh3HKAGCRGgshKuj38ae+CBkJsbzOe/Itkq\nsIByzpUB84CxZtbBzE4CfgjMDmodIpnsvvvg7behoAD22y/sakSiL+gv6l4BtAPWAX8GhqrFXATK\nyuDmm/3j8eP9JY1EpG6B/pk45z53zp3rnOvgnDvAOfdAkMsXyVS33gpr1kDPnnDhhWFXI5IZ9D5O\nJM1WrfIBBXDnnTp6EkmV/lRE0uxXv4KtW/2R00knhV2NSOZQQImkUXEx/OUv0L599VGUiKRGASWS\nJpWV8Mtf+sfXXw/duoVbj0imUUCJpMndd8Prr/vvPI0YEXY1IplHASWSBp9+Cjfc4B9PnAjt2oVb\nj0gmUkCJpMFVV0EsBmecAf36hV2NSGZSQIkE7LHH4OGHoUMHmDoVUrwupogkUUCJBOjLL+EXv/CP\nJ0zwnz+JSOMooEQCdP31sHYtfPvb1UElIo2jgBIJyAsvwF13QevWMH26buEu0lQKKJEAbNoEF1/s\nH99wAxx9dLj1iGQDBZRIAK66Clau9BeDvfHGsKsRyQ4KKJEmeughf6+ndu3g/vuhTZuwKxLJDgoo\nkSb46CO4/HL/+Lbb4Igjwq1HJJsooEQaqbISLrrIfyH37LNhyJCwKxLJLgookUaaMAGefx7y82HG\nDH0hVyRoCiiRRnj6aRg92ofSfffBPvuEXZFI9skNuwCRTPPhh9C/PzgH48bBD34QdkUi2UlHUCIN\nsHmzv/jrF1/4z51Gjgy7IpHspYASSZFzMHQolJTAoYf6U3s5+gsSSRv9eYmk6PbbfSi1bw/z5kFe\nXtgViWQ3BZRICh55pPquuDNnwrHHhluPSEuggBKpx+LFMGCAP8X3m9/A+eeHXZFIy6CAEqnDihVw\nzjmwdSsMHgy//nXYFYm0HAookVqsWwd9+8L69XD66TB5sr6MK9KcFFAiNfj8c//9pvfeg+OO8xeE\nzdW3BkWalQJKJMmmTXDGGfDaa9C9O/z1r7DHHmFXJdLyKKBEEpSVwVlnwb//DQcdBAsX+mvtiUjz\nU0CJxJWVwQ9/CP/8J3Tt6sOpW7ewqxJpuXRWXQR/y4yzzoJ//Qu6dPHhdNBBYVcl0rLpCEpavPXr\n4Xvf8+G0//7+COrww8OuSkR0BCUt2po1cOqp8O67/vp6CxfCAQeEXZWIQEBHUGZ2pZktMbNtZjYr\niGWKpNvrr0Pv3j6cjj3WHzkpnESiI6hTfGuB8cA9AS1PJK0WLICTToLVq31IFRfDf/1X2FWJSKJA\nAso5N8859xiwMYjliaTT5Mn+Xk6lpXDhhf603t57h12ViCQL5TMoMxsMDAbIz8+nuLg4jDLqVFpa\nGsm6oizq26y83Jg8+VCeeKIrABddtJJBg1ayeHHz1xKLxaisrIz09oqaqO9fUZTp2yyUgHLOTQOm\nARQWFrqioqIwyqhTcXExUawryqK8zVatgh//GF55Bdq0genTYeDAAqAglHry8vKIxWKR3V5RFOX9\nK6oyfZvVe4rPzIrNzNUyvNgcRYo0xTPPQM+ePpwOPNC3kw8cGHZVIlKfeo+gnHNFzVCHSODKy+Hm\nm+HWW/29nM44A2bPhk6dwq5MRFIRyCk+M8uNL6sV0MrMdgMqnHMVQSxfpKHeegv69/cXfM3Jgf/9\nX7jxRv9YRDJDUH+uo4AtwPXAgPjjUQEtWyRlO3bApEnQq5cPp4MP9t9vuukmhZNIpgnkCMo5NwYY\nE8SyRBrrzTdhyBBYtMj/+9JL4fbboWPHcOsSkcbRe0rJeFu3wqhRcPzxPpz23Rcee8x36imcRDKX\nrsUnGcs5mD8fhg+HDz7w4y6/HG65BfLywq1NRJpOASUZ6Y034Jpr/FUgAI46CqZN85cvEpHsoFN8\nklHWrIHBg6FHDx9Oe+0Fd9wBJSUKJ5FsoyMoyQiffeZP3d11F2zbBq1awVVXwejR+l6TSLZSQEmk\nffKJ78SbPBk2b/bjzj/ff6/piCPCrU1E0ksBJZH0/vvwu9/Bvff6K0KAvwL5uHFw3HHh1iYizUMB\nJZGxYwc8+yxMmQJPPum79MygXz+47jo44YSwKxSR5qSAktBt3AgzZ/rPl1as8ONat4aLL4YRI+Dw\nw8OtT0TCoYCSUFRWwvPPw6xZMHeub3wAf8v1IUP8VSDy80MtUURCpoCSZuMcLFsG998PDz4Ia9f6\n8Wb+SuNXXOF/tmoVbp0iEg0KKEkr5/yXah97DB54AJYvr5528MHw05/CJZf4xyIiiRRQErjycn/6\n7okn/PDRR9XT9tkHLrjA3wrjW9/yR08iIjVRQEmTOeevhffEE/vxxz/6TrxNm6qnd+niW8T79YM+\nfXwDhIhIfRRQ0iirV8MLL8Bzz/lLDq1eDdD96+nHHAPnnOOD6Zvf1L2YRKThFFBSr/Jyf627RYv8\n8NJL/pp4iTp1gmOOWceFF3bh1FP1mZKINJ0CSnaydau/8d+rr/ph2TJ/Z9qqNvAqeXlw4olwyil+\n+MY34IUX3qaoqEs4hYtI1lFAtVDl5f5yQu+8A2+/XT288w5UVOw6/xFHQO/efjjxRP9vnbYTkXRS\nQGWxigr/2dCHH1YPy5f7IPrgA/9l2WQ5OXDkkdCzZ/XQo4duACgizU8BlaGc851yH3/sPw/6+GM/\nJAbSqlU1hxD49u5DDvFhdNRR/ueRR/rmhg4dmve5iIjURAEVMVu2wPr1sG6d/5k4rF27cyCVldW/\nvK5dfcNC1XDooT6QDj8c2rVL//MREWksBVTAnPMNBV9+CbFYasPGjdUhlEroVGnfHrp18yFUNXTr\nVh1GBQWw225pe6oiImnVYgJqxw4fHFu3+iHxcU3jli3LZ/lyf0RTWuqDI/FnbY/LympuMkhVmzb+\nagtVQ5cu1Y/33XfnMNpzT12JQUSyV+gB9ckncNNN/kV9+/bqn4mPGztu27bq0Km66V3qjmz0c8rN\n9U0FNQ177VXzuKow6thRoSMiAhEIqLVrlzN+fFHS2POBK4DNQN8afmtQfNgAnFfD9KHABcBqYODX\nY818l1rHjtey555nk5OznHXrhpCTw07D0UePom3bY+jY8VNefnkYrVr5K2zn5Pif/ftPoGfP3qxc\nuYiZM0d+Pb1quOOOSfTo0YPnnnuO8ePHs3179Sk8gD/96U8cfvjhPPnkk9x66+93qX727Nnsv//+\nPPTQQ0ydOnWX6Q8//DCdO3dm1qxZzJo1a5fpCxYsoH379kyZMoW5c+fuMr24uBiAiRMnMn/+/J2m\ntWvXjqeffhqAcePGsXDhwp2md+rUiUceeQSAG264gZdeeunrabFYjGOOOYY5c+YAMGzYMEpKSnb6\n/e7duzNt2jQABg8ezHvvvbfT9B49ejBp0iQABgwYwJqkbwSfeOKJ3HLLLQD069ePjRs37jT9lFNO\n4aabbgICTsdMAAAFTElEQVTgjDPOYMuWLTtNP+ussxgxYgQARUVFu2yb888/nyuuuILNmzfTt++u\n+96gQYMYNGgQGzZs4Lzzdt33hg4dygUXXMDq1asZOHDgLtOvvfZazj77bDZv3swHH3ywSw2jRo2i\nT58+lJSUMGzYsF1+f8KECfTu3ZtFixYxcuTIXaZPmrTzvpcscd/7/e8za9/bsWMHL7zwArDrvgfQ\nrVs37XtJ+14sFiMv3oJbte8tX76cIUOG7PL7zbnvpSr0gGrTBvbbz4dH1dCrF3z/+/603J137jzN\nDE47Dfr29afTRo/eeVpODgwYAOeeCxs2wNVX+3GJRyXXXusvwbN8ub/3ULJRoyA3913y8vKo4f+J\nPn3894EWLYKHH07fthERacnMORdqAYWFhW7JkiWh1lCT4uLiGt/lSO20zVJXVFRELBbb5V2+1E77\nV8NFdZuZ2VLnXGF98+laACIiEkkKKBERiSQFlIiIRJICSkREIkkBJSIikdTkgDKztmY2w8xWmdlX\nZlZiZmcEUZyIiLRcQRxB5eK/EXsysCcwCphrZgUBLFtERFqoJn9R1zlXBoxJGDXfzP4D9AJWNnX5\nIiLSMgV+JQkzywe6A2/VMc9gYDBAfn7+15c/iZLS0tJI1hVl2mapi8ViVFZWans1gPavhsv0bRbo\nlSTMrDXwNLDCOVfDRYR2pStJZA9ts9TpShINp/2r4aK6zQK7koSZFZuZq2V4MWG+HGA2UA5c2aTq\nRUSkxav3FJ9zrqi+eczMgBlAPtDXObe96aWJiEhLFtRnUFPxN1Dq45zbUt/MIiIi9Qnie1AHAkOA\nHsCnZlYaH/o3uToREWmxgmgzXwXoHrAiIhIoXepIREQiSQElIiKRFPoddc1sPbAq1CJq1hnYEHYR\nGUbbrGG0vRpG26vhorrNDnTO7VPfTKEHVFSZ2ZJUvkgm1bTNGkbbq2G0vRou07eZTvGJiEgkKaBE\nRCSSFFC1mxZ2ARlI26xhtL0aRtur4TJ6m+kzKBERiSQdQYmISCQpoEREJJIUUCIiEkkKqBSZ2WFm\nttXM5oRdS1SZWVszm2Fmq8zsKzMrMbMzwq4rasxsbzN71MzK4tvqp2HXFFXap5om01+3FFCpmwy8\nEnYREZcLrAZOBvYERgFzzawgxJqiaDL+xp75QH9gqpkdHW5JkaV9qmky+nVLAZUCM/sJEAMWhl1L\nlDnnypxzY5xzK51zO5xz84H/AL3Cri0qzKwD0A+4yTlX6px7EXgCGBhuZdGkfarxsuF1SwFVDzPb\nAxgLDA+7lkxjZvlAd+CtsGuJkO5AhXPuvYRxrwE6gkqB9qnUZMvrlgKqfuOAGc65NWEXkknMrDVw\nP3Cvc+7dsOuJkN2BTUnjvgQ6hlBLRtE+1SBZ8brVogPKzIrNzNUyvGhmPYA+wO1h1xoF9W2vhPly\ngNn4z1muDK3gaCoF9kgatwfwVQi1ZAztU6nLptetJt9RN5M554rqmm5mw4AC4CMzA//ut5WZHeWc\n65n2AiOmvu0FYH5DzcA3APR1zm1Pd10Z5j0g18wOc869Hx93HDplVSvtUw1WRJa8bulSR3Uws/bs\n/G53BP4/fqhzbn0oRUWcmd0F9AD6OOdKw64niszsQcABl+G31QKgt3NOIVUD7VMNk02vWy36CKo+\nzrnNwOaqf5tZKbA10/6Tm4uZHQgMAbYBn8bfvQEMcc7dH1ph0XMFcA+wDtiIf+FQONVA+1TDZdPr\nlo6gREQkklp0k4SIiESXAkpERCJJASUiIpGkgBIRkUhSQImISCQpoEREJJIUUCIiEkkKKBERiaT/\nB9c9MCs0b4SXAAAAAElFTkSuQmCC\n", - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "plt.plot(z, selu(z), \"b-\", linewidth=2)\n", "plt.plot([-5, 5], [0, 0], 'k-')\n", @@ -617,24 +519,7 @@ "cell_type": "code", "execution_count": 26, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Layer 0: -0.26 < mean < 0.27, 0.74 < std deviation < 1.27\n", - "Layer 10: -0.24 < mean < 0.27, 0.74 < std deviation < 1.27\n", - "Layer 20: -0.17 < mean < 0.18, 0.74 < std deviation < 1.24\n", - "Layer 30: -0.27 < mean < 0.24, 0.78 < std deviation < 1.20\n", - "Layer 40: -0.38 < mean < 0.39, 0.74 < std deviation < 1.25\n", - "Layer 50: -0.27 < mean < 0.31, 0.73 < std deviation < 1.27\n", - "Layer 60: -0.26 < mean < 0.43, 0.74 < std deviation < 1.35\n", - "Layer 70: -0.19 < mean < 0.21, 0.75 < std deviation < 1.21\n", - "Layer 80: -0.18 < mean < 0.16, 0.72 < std deviation < 1.19\n", - "Layer 90: -0.19 < mean < 0.16, 0.75 < std deviation < 1.20\n" - ] - } - ], + "outputs": [], "source": [ "np.random.seed(42)\n", "Z = np.random.normal(size=(500, 100))\n", @@ -737,22 +622,7 @@ "cell_type": "code", "execution_count": 29, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "0 Batch accuracy: 0.96 Validation accuracy: 0.924\n", - "5 Batch accuracy: 1.0 Validation accuracy: 0.9568\n", - "10 Batch accuracy: 0.94 Validation accuracy: 0.9668\n", - "15 Batch accuracy: 0.98 Validation accuracy: 0.9684\n", - "20 Batch accuracy: 1.0 Validation accuracy: 0.9712\n", - "25 Batch accuracy: 1.0 Validation accuracy: 0.9694\n", - "30 Batch accuracy: 1.0 Validation accuracy: 0.97\n", - "35 Batch accuracy: 1.0 Validation accuracy: 0.971\n" - ] - } - ], + "outputs": [], "source": [ "means = mnist.train.images.mean(axis=0, keepdims=True)\n", "stds = mnist.train.images.std(axis=0, keepdims=True) + 1e-10\n", @@ -954,34 +824,7 @@ "cell_type": "code", "execution_count": 35, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "0 Test accuracy: 0.8727\n", - "1 Test accuracy: 0.8981\n", - "2 Test accuracy: 0.9129\n", - "3 Test accuracy: 0.922\n", - "4 Test accuracy: 0.9292\n", - "5 Test accuracy: 0.9342\n", - "6 Test accuracy: 0.9381\n", - "7 Test accuracy: 0.9419\n", - "8 Test accuracy: 0.9451\n", - "9 Test accuracy: 0.9471\n", - "10 Test accuracy: 0.9507\n", - "11 Test accuracy: 0.9521\n", - "12 Test accuracy: 0.9553\n", - "13 Test accuracy: 0.956\n", - "14 Test accuracy: 0.957\n", - "15 Test accuracy: 0.9583\n", - "16 Test accuracy: 0.9613\n", - "17 Test accuracy: 0.9608\n", - "18 Test accuracy: 0.9627\n", - "19 Test accuracy: 0.963\n" - ] - } - ], + "outputs": [], "source": [ "extra_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)\n", "\n", @@ -1038,29 +881,7 @@ "cell_type": "code", "execution_count": 36, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['hidden1/kernel:0',\n", - " 'hidden1/bias:0',\n", - " 'batch_normalization/beta:0',\n", - " 'batch_normalization/gamma:0',\n", - " 'hidden2/kernel:0',\n", - " 'hidden2/bias:0',\n", - " 'batch_normalization_1/beta:0',\n", - " 'batch_normalization_1/gamma:0',\n", - " 'outputs/kernel:0',\n", - " 'outputs/bias:0',\n", - " 'batch_normalization_2/beta:0',\n", - " 'batch_normalization_2/gamma:0']" - ] - }, - "execution_count": 36, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "[v.name for v in tf.trainable_variables()]" ] @@ -1069,35 +890,7 @@ "cell_type": "code", "execution_count": 37, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['hidden1/kernel:0',\n", - " 'hidden1/bias:0',\n", - " 'batch_normalization/beta:0',\n", - " 'batch_normalization/gamma:0',\n", - " 'batch_normalization/moving_mean:0',\n", - " 'batch_normalization/moving_variance:0',\n", - " 'hidden2/kernel:0',\n", - " 'hidden2/bias:0',\n", - " 'batch_normalization_1/beta:0',\n", - " 'batch_normalization_1/gamma:0',\n", - " 'batch_normalization_1/moving_mean:0',\n", - " 'batch_normalization_1/moving_variance:0',\n", - " 'outputs/kernel:0',\n", - " 'outputs/bias:0',\n", - " 'batch_normalization_2/beta:0',\n", - " 'batch_normalization_2/gamma:0',\n", - " 'batch_normalization_2/moving_mean:0',\n", - " 'batch_normalization_2/moving_variance:0']" - ] - }, - "execution_count": 37, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "[v.name for v in tf.global_variables()]" ] @@ -1233,34 +1026,7 @@ "cell_type": "code", "execution_count": 44, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "0 Test accuracy: 0.3139\n", - "1 Test accuracy: 0.8001\n", - "2 Test accuracy: 0.8806\n", - "3 Test accuracy: 0.9037\n", - "4 Test accuracy: 0.9124\n", - "5 Test accuracy: 0.9197\n", - "6 Test accuracy: 0.9243\n", - "7 Test accuracy: 0.9299\n", - "8 Test accuracy: 0.9331\n", - "9 Test accuracy: 0.9387\n", - "10 Test accuracy: 0.9431\n", - "11 Test accuracy: 0.9445\n", - "12 Test accuracy: 0.9455\n", - "13 Test accuracy: 0.9485\n", - "14 Test accuracy: 0.9524\n", - "15 Test accuracy: 0.9511\n", - "16 Test accuracy: 0.9562\n", - "17 Test accuracy: 0.9583\n", - "18 Test accuracy: 0.9559\n", - "19 Test accuracy: 0.9605\n" - ] - } - ], + "outputs": [], "source": [ "with tf.Session() as sess:\n", " init.run()\n", @@ -1329,324 +1095,7 @@ "cell_type": "code", "execution_count": 47, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "X\n", - "y\n", - "hidden1/kernel/Initializer/random_uniform/shape\n", - "hidden1/kernel/Initializer/random_uniform/min\n", - "hidden1/kernel/Initializer/random_uniform/max\n", - "hidden1/kernel/Initializer/random_uniform/RandomUniform\n", - "hidden1/kernel/Initializer/random_uniform/sub\n", - "hidden1/kernel/Initializer/random_uniform/mul\n", - "hidden1/kernel/Initializer/random_uniform\n", - "hidden1/kernel\n", - "hidden1/kernel/Assign\n", - "hidden1/kernel/read\n", - "hidden1/bias/Initializer/Const\n", - "hidden1/bias\n", - "hidden1/bias/Assign\n", - "hidden1/bias/read\n", - "dnn/hidden1/MatMul\n", - "dnn/hidden1/BiasAdd\n", - "dnn/hidden1/Relu\n", - "hidden2/kernel/Initializer/random_uniform/shape\n", - "hidden2/kernel/Initializer/random_uniform/min\n", - "hidden2/kernel/Initializer/random_uniform/max\n", - "hidden2/kernel/Initializer/random_uniform/RandomUniform\n", - "hidden2/kernel/Initializer/random_uniform/sub\n", - "hidden2/kernel/Initializer/random_uniform/mul\n", - "hidden2/kernel/Initializer/random_uniform\n", - "hidden2/kernel\n", - "hidden2/kernel/Assign\n", - "hidden2/kernel/read\n", - "hidden2/bias/Initializer/Const\n", - "hidden2/bias\n", - "hidden2/bias/Assign\n", - "hidden2/bias/read\n", - "dnn/hidden2/MatMul\n", - "dnn/hidden2/BiasAdd\n", - "dnn/hidden2/Relu\n", - "hidden3/kernel/Initializer/random_uniform/shape\n", - "hidden3/kernel/Initializer/random_uniform/min\n", - "hidden3/kernel/Initializer/random_uniform/max\n", - "hidden3/kernel/Initializer/random_uniform/RandomUniform\n", - "hidden3/kernel/Initializer/random_uniform/sub\n", - "hidden3/kernel/Initializer/random_uniform/mul\n", - "hidden3/kernel/Initializer/random_uniform\n", - "hidden3/kernel\n", - "hidden3/kernel/Assign\n", - "hidden3/kernel/read\n", - "hidden3/bias/Initializer/Const\n", - "hidden3/bias\n", - "hidden3/bias/Assign\n", - "hidden3/bias/read\n", - "dnn/hidden3/MatMul\n", - "dnn/hidden3/BiasAdd\n", - "dnn/hidden3/Relu\n", - "hidden4/kernel/Initializer/random_uniform/shape\n", - "hidden4/kernel/Initializer/random_uniform/min\n", - "hidden4/kernel/Initializer/random_uniform/max\n", - "hidden4/kernel/Initializer/random_uniform/RandomUniform\n", - "hidden4/kernel/Initializer/random_uniform/sub\n", - "hidden4/kernel/Initializer/random_uniform/mul\n", - "hidden4/kernel/Initializer/random_uniform\n", - "hidden4/kernel\n", - "hidden4/kernel/Assign\n", - "hidden4/kernel/read\n", - "hidden4/bias/Initializer/Const\n", - "hidden4/bias\n", - "hidden4/bias/Assign\n", - "hidden4/bias/read\n", - "dnn/hidden4/MatMul\n", - "dnn/hidden4/BiasAdd\n", - "dnn/hidden4/Relu\n", - "hidden5/kernel/Initializer/random_uniform/shape\n", - "hidden5/kernel/Initializer/random_uniform/min\n", - "hidden5/kernel/Initializer/random_uniform/max\n", - "hidden5/kernel/Initializer/random_uniform/RandomUniform\n", - "hidden5/kernel/Initializer/random_uniform/sub\n", - "hidden5/kernel/Initializer/random_uniform/mul\n", - "hidden5/kernel/Initializer/random_uniform\n", - "hidden5/kernel\n", - "hidden5/kernel/Assign\n", - "hidden5/kernel/read\n", - "hidden5/bias/Initializer/Const\n", - "hidden5/bias\n", - "hidden5/bias/Assign\n", - "hidden5/bias/read\n", - "dnn/hidden5/MatMul\n", - "dnn/hidden5/BiasAdd\n", - "dnn/hidden5/Relu\n", - "outputs/kernel/Initializer/random_uniform/shape\n", - "outputs/kernel/Initializer/random_uniform/min\n", - "outputs/kernel/Initializer/random_uniform/max\n", - "outputs/kernel/Initializer/random_uniform/RandomUniform\n", - "outputs/kernel/Initializer/random_uniform/sub\n", - "outputs/kernel/Initializer/random_uniform/mul\n", - "outputs/kernel/Initializer/random_uniform\n", - "outputs/kernel\n", - "outputs/kernel/Assign\n", - "outputs/kernel/read\n", - "outputs/bias/Initializer/Const\n", - "outputs/bias\n", - "outputs/bias/Assign\n", - "outputs/bias/read\n", - "dnn/outputs/MatMul\n", - "dnn/outputs/BiasAdd\n", - "loss/SparseSoftmaxCrossEntropyWithLogits/Shape\n", - "loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits\n", - "loss/Const\n", - "loss/loss\n", - "gradients/Shape\n", - "gradients/Const\n", - "gradients/Fill\n", - "gradients/loss/loss_grad/Reshape/shape\n", - "gradients/loss/loss_grad/Reshape\n", - "gradients/loss/loss_grad/Shape\n", - "gradients/loss/loss_grad/Tile\n", - "gradients/loss/loss_grad/Shape_1\n", - "gradients/loss/loss_grad/Shape_2\n", - "gradients/loss/loss_grad/Const\n", - "gradients/loss/loss_grad/Prod\n", - "gradients/loss/loss_grad/Const_1\n", - "gradients/loss/loss_grad/Prod_1\n", - "gradients/loss/loss_grad/Maximum/y\n", - "gradients/loss/loss_grad/Maximum\n", - "gradients/loss/loss_grad/floordiv\n", - "gradients/loss/loss_grad/Cast\n", - "gradients/loss/loss_grad/truediv\n", - "gradients/zeros_like\n", - "gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/PreventGradient\n", - "gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims/dim\n", - "gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims\n", - "gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul\n", - "gradients/dnn/outputs/BiasAdd_grad/BiasAddGrad\n", - "gradients/dnn/outputs/BiasAdd_grad/tuple/group_deps\n", - "gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency\n", - "gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency_1\n", - "gradients/dnn/outputs/MatMul_grad/MatMul\n", - "gradients/dnn/outputs/MatMul_grad/MatMul_1\n", - "gradients/dnn/outputs/MatMul_grad/tuple/group_deps\n", - "gradients/dnn/outputs/MatMul_grad/tuple/control_dependency\n", - "gradients/dnn/outputs/MatMul_grad/tuple/control_dependency_1\n", - "gradients/dnn/hidden5/Relu_grad/ReluGrad\n", - "gradients/dnn/hidden5/BiasAdd_grad/BiasAddGrad\n", - "gradients/dnn/hidden5/BiasAdd_grad/tuple/group_deps\n", - "gradients/dnn/hidden5/BiasAdd_grad/tuple/control_dependency\n", - "gradients/dnn/hidden5/BiasAdd_grad/tuple/control_dependency_1\n", - "gradients/dnn/hidden5/MatMul_grad/MatMul\n", - "gradients/dnn/hidden5/MatMul_grad/MatMul_1\n", - "gradients/dnn/hidden5/MatMul_grad/tuple/group_deps\n", - "gradients/dnn/hidden5/MatMul_grad/tuple/control_dependency\n", - "gradients/dnn/hidden5/MatMul_grad/tuple/control_dependency_1\n", - "gradients/dnn/hidden4/Relu_grad/ReluGrad\n", - "gradients/dnn/hidden4/BiasAdd_grad/BiasAddGrad\n", - "gradients/dnn/hidden4/BiasAdd_grad/tuple/group_deps\n", - "gradients/dnn/hidden4/BiasAdd_grad/tuple/control_dependency\n", - "gradients/dnn/hidden4/BiasAdd_grad/tuple/control_dependency_1\n", - "gradients/dnn/hidden4/MatMul_grad/MatMul\n", - "gradients/dnn/hidden4/MatMul_grad/MatMul_1\n", - "gradients/dnn/hidden4/MatMul_grad/tuple/group_deps\n", - "gradients/dnn/hidden4/MatMul_grad/tuple/control_dependency\n", - "gradients/dnn/hidden4/MatMul_grad/tuple/control_dependency_1\n", - "gradients/dnn/hidden3/Relu_grad/ReluGrad\n", - "gradients/dnn/hidden3/BiasAdd_grad/BiasAddGrad\n", - "gradients/dnn/hidden3/BiasAdd_grad/tuple/group_deps\n", - "gradients/dnn/hidden3/BiasAdd_grad/tuple/control_dependency\n", - "gradients/dnn/hidden3/BiasAdd_grad/tuple/control_dependency_1\n", - "gradients/dnn/hidden3/MatMul_grad/MatMul\n", - "gradients/dnn/hidden3/MatMul_grad/MatMul_1\n", - "gradients/dnn/hidden3/MatMul_grad/tuple/group_deps\n", - "gradients/dnn/hidden3/MatMul_grad/tuple/control_dependency\n", - "gradients/dnn/hidden3/MatMul_grad/tuple/control_dependency_1\n", - "gradients/dnn/hidden2/Relu_grad/ReluGrad\n", - "gradients/dnn/hidden2/BiasAdd_grad/BiasAddGrad\n", - "gradients/dnn/hidden2/BiasAdd_grad/tuple/group_deps\n", - "gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency\n", - "gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency_1\n", - "gradients/dnn/hidden2/MatMul_grad/MatMul\n", - "gradients/dnn/hidden2/MatMul_grad/MatMul_1\n", - "gradients/dnn/hidden2/MatMul_grad/tuple/group_deps\n", - "gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency\n", - "gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency_1\n", - "gradients/dnn/hidden1/Relu_grad/ReluGrad\n", - "gradients/dnn/hidden1/BiasAdd_grad/BiasAddGrad\n", - "gradients/dnn/hidden1/BiasAdd_grad/tuple/group_deps\n", - "gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency\n", - "gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency_1\n", - "gradients/dnn/hidden1/MatMul_grad/MatMul\n", - "gradients/dnn/hidden1/MatMul_grad/MatMul_1\n", - "gradients/dnn/hidden1/MatMul_grad/tuple/group_deps\n", - "gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency\n", - "gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency_1\n", - "clip_by_value/Minimum/y\n", - "clip_by_value/Minimum\n", - "clip_by_value/y\n", - "clip_by_value\n", - "clip_by_value_1/Minimum/y\n", - "clip_by_value_1/Minimum\n", - "clip_by_value_1/y\n", - "clip_by_value_1\n", - "clip_by_value_2/Minimum/y\n", - "clip_by_value_2/Minimum\n", - "clip_by_value_2/y\n", - "clip_by_value_2\n", - "clip_by_value_3/Minimum/y\n", - "clip_by_value_3/Minimum\n", - "clip_by_value_3/y\n", - "clip_by_value_3\n", - "clip_by_value_4/Minimum/y\n", - "clip_by_value_4/Minimum\n", - "clip_by_value_4/y\n", - "clip_by_value_4\n", - "clip_by_value_5/Minimum/y\n", - "clip_by_value_5/Minimum\n", - "clip_by_value_5/y\n", - "clip_by_value_5\n", - "clip_by_value_6/Minimum/y\n", - "clip_by_value_6/Minimum\n", - "clip_by_value_6/y\n", - "clip_by_value_6\n", - "clip_by_value_7/Minimum/y\n", - "clip_by_value_7/Minimum\n", - "clip_by_value_7/y\n", - "clip_by_value_7\n", - "clip_by_value_8/Minimum/y\n", - "clip_by_value_8/Minimum\n", - "clip_by_value_8/y\n", - "clip_by_value_8\n", - "clip_by_value_9/Minimum/y\n", - "clip_by_value_9/Minimum\n", - "clip_by_value_9/y\n", - "clip_by_value_9\n", - "clip_by_value_10/Minimum/y\n", - "clip_by_value_10/Minimum\n", - "clip_by_value_10/y\n", - "clip_by_value_10\n", - "clip_by_value_11/Minimum/y\n", - "clip_by_value_11/Minimum\n", - "clip_by_value_11/y\n", - "clip_by_value_11\n", - "GradientDescent/learning_rate\n", - "GradientDescent/update_hidden1/kernel/ApplyGradientDescent\n", - "GradientDescent/update_hidden1/bias/ApplyGradientDescent\n", - "GradientDescent/update_hidden2/kernel/ApplyGradientDescent\n", - "GradientDescent/update_hidden2/bias/ApplyGradientDescent\n", - "GradientDescent/update_hidden3/kernel/ApplyGradientDescent\n", - "GradientDescent/update_hidden3/bias/ApplyGradientDescent\n", - "GradientDescent/update_hidden4/kernel/ApplyGradientDescent\n", - "GradientDescent/update_hidden4/bias/ApplyGradientDescent\n", - "GradientDescent/update_hidden5/kernel/ApplyGradientDescent\n", - "GradientDescent/update_hidden5/bias/ApplyGradientDescent\n", - "GradientDescent/update_outputs/kernel/ApplyGradientDescent\n", - "GradientDescent/update_outputs/bias/ApplyGradientDescent\n", - "GradientDescent\n", - "eval/InTopK\n", - "eval/Cast\n", - "eval/Const\n", - "eval/accuracy\n", - "init\n", - "save/Const\n", - "save/SaveV2/tensor_names\n", - "save/SaveV2/shape_and_slices\n", - "save/SaveV2\n", - "save/control_dependency\n", - "save/RestoreV2/tensor_names\n", - "save/RestoreV2/shape_and_slices\n", - "save/RestoreV2\n", - "save/Assign\n", - "save/RestoreV2_1/tensor_names\n", - "save/RestoreV2_1/shape_and_slices\n", - "save/RestoreV2_1\n", - "save/Assign_1\n", - "save/RestoreV2_2/tensor_names\n", - "save/RestoreV2_2/shape_and_slices\n", - "save/RestoreV2_2\n", - "save/Assign_2\n", - "save/RestoreV2_3/tensor_names\n", - "save/RestoreV2_3/shape_and_slices\n", - "save/RestoreV2_3\n", - "save/Assign_3\n", - "save/RestoreV2_4/tensor_names\n", - "save/RestoreV2_4/shape_and_slices\n", - "save/RestoreV2_4\n", - "save/Assign_4\n", - "save/RestoreV2_5/tensor_names\n", - "save/RestoreV2_5/shape_and_slices\n", - "save/RestoreV2_5\n", - "save/Assign_5\n", - "save/RestoreV2_6/tensor_names\n", - "save/RestoreV2_6/shape_and_slices\n", - "save/RestoreV2_6\n", - "save/Assign_6\n", - "save/RestoreV2_7/tensor_names\n", - "save/RestoreV2_7/shape_and_slices\n", - "save/RestoreV2_7\n", - "save/Assign_7\n", - "save/RestoreV2_8/tensor_names\n", - "save/RestoreV2_8/shape_and_slices\n", - "save/RestoreV2_8\n", - "save/Assign_8\n", - "save/RestoreV2_9/tensor_names\n", - "save/RestoreV2_9/shape_and_slices\n", - "save/RestoreV2_9\n", - "save/Assign_9\n", - "save/RestoreV2_10/tensor_names\n", - "save/RestoreV2_10/shape_and_slices\n", - "save/RestoreV2_10\n", - "save/Assign_10\n", - "save/RestoreV2_11/tensor_names\n", - "save/RestoreV2_11/shape_and_slices\n", - "save/RestoreV2_11\n", - "save/Assign_11\n", - "save/restore_all\n" - ] - } - ], + "outputs": [], "source": [ "for op in tf.get_default_graph().get_operations():\n", " print(op.name)" @@ -1711,32 +1160,7 @@ "metadata": { "scrolled": true }, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - " \n", - " " - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "show_graph(tf.get_default_graph())" ] @@ -1812,15 +1236,7 @@ "cell_type": "code", "execution_count": 53, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "INFO:tensorflow:Restoring parameters from ./my_model_final.ckpt\n" - ] - } - ], + "outputs": [], "source": [ "with tf.Session() as sess:\n", " saver.restore(sess, \"./my_model_final.ckpt\")\n", @@ -1838,35 +1254,7 @@ "cell_type": "code", "execution_count": 54, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "INFO:tensorflow:Restoring parameters from ./my_model_final.ckpt\n", - "0 Test accuracy: 0.9609\n", - "1 Test accuracy: 0.9608\n", - "2 Test accuracy: 0.9617\n", - "3 Test accuracy: 0.9613\n", - "4 Test accuracy: 0.9639\n", - "5 Test accuracy: 0.9649\n", - "6 Test accuracy: 0.9663\n", - "7 Test accuracy: 0.9627\n", - "8 Test accuracy: 0.9665\n", - "9 Test accuracy: 0.9669\n", - "10 Test accuracy: 0.9662\n", - "11 Test accuracy: 0.9674\n", - "12 Test accuracy: 0.9678\n", - "13 Test accuracy: 0.9679\n", - "14 Test accuracy: 0.9688\n", - "15 Test accuracy: 0.9684\n", - "16 Test accuracy: 0.9687\n", - "17 Test accuracy: 0.9702\n", - "18 Test accuracy: 0.9673\n", - "19 Test accuracy: 0.9687\n" - ] - } - ], + "outputs": [], "source": [ "with tf.Session() as sess:\n", " saver.restore(sess, \"./my_model_final.ckpt\")\n", @@ -1949,35 +1337,7 @@ "cell_type": "code", "execution_count": 56, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "INFO:tensorflow:Restoring parameters from ./my_model_final.ckpt\n", - "0 Test accuracy: 0.9611\n", - "1 Test accuracy: 0.9619\n", - "2 Test accuracy: 0.9622\n", - "3 Test accuracy: 0.9619\n", - "4 Test accuracy: 0.9644\n", - "5 Test accuracy: 0.9633\n", - "6 Test accuracy: 0.9647\n", - "7 Test accuracy: 0.9648\n", - "8 Test accuracy: 0.9671\n", - "9 Test accuracy: 0.9677\n", - "10 Test accuracy: 0.9676\n", - "11 Test accuracy: 0.9679\n", - "12 Test accuracy: 0.9687\n", - "13 Test accuracy: 0.9688\n", - "14 Test accuracy: 0.9683\n", - "15 Test accuracy: 0.9693\n", - "16 Test accuracy: 0.9677\n", - "17 Test accuracy: 0.9697\n", - "18 Test accuracy: 0.9692\n", - "19 Test accuracy: 0.9707\n" - ] - } - ], + "outputs": [], "source": [ "with tf.Session() as sess:\n", " saver.restore(sess, \"./my_model_final.ckpt\")\n", @@ -2050,35 +1410,7 @@ "cell_type": "code", "execution_count": 58, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "INFO:tensorflow:Restoring parameters from ./my_model_final.ckpt\n", - "0 Test accuracy: 0.9142\n", - "1 Test accuracy: 0.9346\n", - "2 Test accuracy: 0.9437\n", - "3 Test accuracy: 0.9486\n", - "4 Test accuracy: 0.9517\n", - "5 Test accuracy: 0.9544\n", - "6 Test accuracy: 0.9544\n", - "7 Test accuracy: 0.9562\n", - "8 Test accuracy: 0.9588\n", - "9 Test accuracy: 0.9619\n", - "10 Test accuracy: 0.9617\n", - "11 Test accuracy: 0.9617\n", - "12 Test accuracy: 0.9624\n", - "13 Test accuracy: 0.9644\n", - "14 Test accuracy: 0.9622\n", - "15 Test accuracy: 0.964\n", - "16 Test accuracy: 0.9666\n", - "17 Test accuracy: 0.9668\n", - "18 Test accuracy: 0.9673\n", - "19 Test accuracy: 0.9687\n" - ] - } - ], + "outputs": [], "source": [ "with tf.Session() as sess:\n", " init.run()\n", @@ -2153,35 +1485,7 @@ "cell_type": "code", "execution_count": 60, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "INFO:tensorflow:Restoring parameters from ./my_model_final.ckpt\n", - "0 Test accuracy: 0.9022\n", - "1 Test accuracy: 0.9302\n", - "2 Test accuracy: 0.9393\n", - "3 Test accuracy: 0.9429\n", - "4 Test accuracy: 0.9484\n", - "5 Test accuracy: 0.9511\n", - "6 Test accuracy: 0.9517\n", - "7 Test accuracy: 0.9539\n", - "8 Test accuracy: 0.9545\n", - "9 Test accuracy: 0.9572\n", - "10 Test accuracy: 0.9599\n", - "11 Test accuracy: 0.9602\n", - "12 Test accuracy: 0.9606\n", - "13 Test accuracy: 0.9619\n", - "14 Test accuracy: 0.9619\n", - "15 Test accuracy: 0.9636\n", - "16 Test accuracy: 0.9633\n", - "17 Test accuracy: 0.9643\n", - "18 Test accuracy: 0.9651\n", - "19 Test accuracy: 0.9657\n" - ] - } - ], + "outputs": [], "source": [ "reuse_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,\n", " scope=\"hidden[123]\") # regular expression\n", @@ -2238,15 +1542,7 @@ "cell_type": "code", "execution_count": 62, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[[ 61. 83. 105.]]\n" - ] - } - ], + "outputs": [], "source": [ "original_w = [[1., 2., 3.], [4., 5., 6.]] # Load the weights from the other framework\n", "original_b = [7., 8., 9.] # Load the biases from the other framework\n", @@ -2288,15 +1584,7 @@ "cell_type": "code", "execution_count": 63, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[[ 61. 83. 105.]]\n" - ] - } - ], + "outputs": [], "source": [ "reset_graph()\n", "\n", @@ -2342,19 +1630,7 @@ "cell_type": "code", "execution_count": 64, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[,\n", - " ]" - ] - }, - "execution_count": 64, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=\"hidden1\")" ] @@ -2370,18 +1646,7 @@ "cell_type": "code", "execution_count": 65, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 65, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "tf.get_default_graph().get_tensor_by_name(\"hidden1/kernel:0\")" ] @@ -2390,18 +1655,7 @@ "cell_type": "code", "execution_count": 66, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 66, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "tf.get_default_graph().get_tensor_by_name(\"hidden1/bias:0\")" ] @@ -2480,35 +1734,7 @@ "cell_type": "code", "execution_count": 70, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "INFO:tensorflow:Restoring parameters from ./my_model_final.ckpt\n", - "0 Test accuracy: 0.8987\n", - "1 Test accuracy: 0.9311\n", - "2 Test accuracy: 0.9375\n", - "3 Test accuracy: 0.9414\n", - "4 Test accuracy: 0.9437\n", - "5 Test accuracy: 0.9479\n", - "6 Test accuracy: 0.9495\n", - "7 Test accuracy: 0.9521\n", - "8 Test accuracy: 0.9517\n", - "9 Test accuracy: 0.9525\n", - "10 Test accuracy: 0.9535\n", - "11 Test accuracy: 0.9538\n", - "12 Test accuracy: 0.9534\n", - "13 Test accuracy: 0.9546\n", - "14 Test accuracy: 0.9538\n", - "15 Test accuracy: 0.9553\n", - "16 Test accuracy: 0.9552\n", - "17 Test accuracy: 0.9549\n", - "18 Test accuracy: 0.9553\n", - "19 Test accuracy: 0.9557\n" - ] - } - ], + "outputs": [], "source": [ "reuse_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,\n", " scope=\"hidden[123]\") # regular expression\n", @@ -2607,35 +1833,7 @@ "cell_type": "code", "execution_count": 74, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "INFO:tensorflow:Restoring parameters from ./my_model_final.ckpt\n", - "0 Test accuracy: 0.9031\n", - "1 Test accuracy: 0.932\n", - "2 Test accuracy: 0.94\n", - "3 Test accuracy: 0.9435\n", - "4 Test accuracy: 0.9473\n", - "5 Test accuracy: 0.9492\n", - "6 Test accuracy: 0.9498\n", - "7 Test accuracy: 0.9493\n", - "8 Test accuracy: 0.9515\n", - "9 Test accuracy: 0.9519\n", - "10 Test accuracy: 0.9529\n", - "11 Test accuracy: 0.9536\n", - "12 Test accuracy: 0.9529\n", - "13 Test accuracy: 0.9532\n", - "14 Test accuracy: 0.9522\n", - "15 Test accuracy: 0.9534\n", - "16 Test accuracy: 0.953\n", - "17 Test accuracy: 0.955\n", - "18 Test accuracy: 0.955\n", - "19 Test accuracy: 0.9552\n" - ] - } - ], + "outputs": [], "source": [ "reuse_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,\n", " scope=\"hidden[123]\") # regular expression\n", @@ -2733,35 +1931,7 @@ "cell_type": "code", "execution_count": 77, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "INFO:tensorflow:Restoring parameters from ./my_model_final.ckpt\n", - "0 Test accuracy: 0.9033\n", - "1 Test accuracy: 0.9322\n", - "2 Test accuracy: 0.9423\n", - "3 Test accuracy: 0.9449\n", - "4 Test accuracy: 0.9471\n", - "5 Test accuracy: 0.9477\n", - "6 Test accuracy: 0.951\n", - "7 Test accuracy: 0.9507\n", - "8 Test accuracy: 0.9514\n", - "9 Test accuracy: 0.9522\n", - "10 Test accuracy: 0.9512\n", - "11 Test accuracy: 0.9521\n", - "12 Test accuracy: 0.9522\n", - "13 Test accuracy: 0.9539\n", - "14 Test accuracy: 0.9536\n", - "15 Test accuracy: 0.9534\n", - "16 Test accuracy: 0.9547\n", - "17 Test accuracy: 0.9537\n", - "18 Test accuracy: 0.9542\n", - "19 Test accuracy: 0.9547\n" - ] - } - ], + "outputs": [], "source": [ "import numpy as np\n", "\n", @@ -2962,19 +2132,7 @@ "cell_type": "code", "execution_count": 86, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "0 Test accuracy: 0.9579\n", - "1 Test accuracy: 0.9691\n", - "2 Test accuracy: 0.976\n", - "3 Test accuracy: 0.9793\n", - "4 Test accuracy: 0.9811\n" - ] - } - ], + "outputs": [], "source": [ "n_epochs = 5\n", "batch_size = 50\n", @@ -3098,34 +2256,7 @@ "metadata": { "scrolled": true }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "0 Test accuracy: 0.8343\n", - "1 Test accuracy: 0.8726\n", - "2 Test accuracy: 0.8832\n", - "3 Test accuracy: 0.8899\n", - "4 Test accuracy: 0.8958\n", - "5 Test accuracy: 0.8986\n", - "6 Test accuracy: 0.9011\n", - "7 Test accuracy: 0.9032\n", - "8 Test accuracy: 0.9046\n", - "9 Test accuracy: 0.9047\n", - "10 Test accuracy: 0.9065\n", - "11 Test accuracy: 0.9059\n", - "12 Test accuracy: 0.9072\n", - "13 Test accuracy: 0.9072\n", - "14 Test accuracy: 0.9069\n", - "15 Test accuracy: 0.9071\n", - "16 Test accuracy: 0.9064\n", - "17 Test accuracy: 0.9071\n", - "18 Test accuracy: 0.9068\n", - "19 Test accuracy: 0.9063\n" - ] - } - ], + "outputs": [], "source": [ "n_epochs = 20\n", "batch_size = 200\n", @@ -3264,34 +2395,7 @@ "metadata": { "scrolled": true }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "0 Test accuracy: 0.8298\n", - "1 Test accuracy: 0.8778\n", - "2 Test accuracy: 0.8917\n", - "3 Test accuracy: 0.9017\n", - "4 Test accuracy: 0.9068\n", - "5 Test accuracy: 0.9103\n", - "6 Test accuracy: 0.9125\n", - "7 Test accuracy: 0.9137\n", - "8 Test accuracy: 0.9149\n", - "9 Test accuracy: 0.9174\n", - "10 Test accuracy: 0.9176\n", - "11 Test accuracy: 0.9184\n", - "12 Test accuracy: 0.9191\n", - "13 Test accuracy: 0.9183\n", - "14 Test accuracy: 0.9195\n", - "15 Test accuracy: 0.9201\n", - "16 Test accuracy: 0.9181\n", - "17 Test accuracy: 0.9184\n", - "18 Test accuracy: 0.9181\n", - "19 Test accuracy: 0.9174\n" - ] - } - ], + "outputs": [], "source": [ "n_epochs = 20\n", "batch_size = 200\n", @@ -3392,34 +2496,7 @@ "metadata": { "scrolled": true }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "0 Test accuracy: 0.9205\n", - "1 Test accuracy: 0.9418\n", - "2 Test accuracy: 0.9486\n", - "3 Test accuracy: 0.9508\n", - "4 Test accuracy: 0.954\n", - "5 Test accuracy: 0.957\n", - "6 Test accuracy: 0.9604\n", - "7 Test accuracy: 0.9585\n", - "8 Test accuracy: 0.9598\n", - "9 Test accuracy: 0.9663\n", - "10 Test accuracy: 0.9644\n", - "11 Test accuracy: 0.9646\n", - "12 Test accuracy: 0.9675\n", - "13 Test accuracy: 0.9657\n", - "14 Test accuracy: 0.9645\n", - "15 Test accuracy: 0.9668\n", - "16 Test accuracy: 0.969\n", - "17 Test accuracy: 0.9682\n", - "18 Test accuracy: 0.9698\n", - "19 Test accuracy: 0.9682\n" - ] - } - ], + "outputs": [], "source": [ "n_epochs = 20\n", "batch_size = 50\n", @@ -3572,34 +2649,7 @@ "cell_type": "code", "execution_count": 106, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "0 Test accuracy: 0.9517\n", - "1 Test accuracy: 0.9674\n", - "2 Test accuracy: 0.9712\n", - "3 Test accuracy: 0.9759\n", - "4 Test accuracy: 0.975\n", - "5 Test accuracy: 0.9761\n", - "6 Test accuracy: 0.9765\n", - "7 Test accuracy: 0.9796\n", - "8 Test accuracy: 0.9791\n", - "9 Test accuracy: 0.9794\n", - "10 Test accuracy: 0.9805\n", - "11 Test accuracy: 0.9809\n", - "12 Test accuracy: 0.9807\n", - "13 Test accuracy: 0.9799\n", - "14 Test accuracy: 0.982\n", - "15 Test accuracy: 0.9816\n", - "16 Test accuracy: 0.9825\n", - "17 Test accuracy: 0.9825\n", - "18 Test accuracy: 0.9816\n", - "19 Test accuracy: 0.9822\n" - ] - } - ], + "outputs": [], "source": [ "with tf.Session() as sess: # not shown in the book\n", " init.run() # not shown\n", @@ -3737,34 +2787,7 @@ "metadata": { "scrolled": false }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "0 Test accuracy: 0.9527\n", - "1 Test accuracy: 0.9653\n", - "2 Test accuracy: 0.97\n", - "3 Test accuracy: 0.9751\n", - "4 Test accuracy: 0.9752\n", - "5 Test accuracy: 0.9742\n", - "6 Test accuracy: 0.9754\n", - "7 Test accuracy: 0.9784\n", - "8 Test accuracy: 0.9775\n", - "9 Test accuracy: 0.9789\n", - "10 Test accuracy: 0.9808\n", - "11 Test accuracy: 0.9797\n", - "12 Test accuracy: 0.9802\n", - "13 Test accuracy: 0.9799\n", - "14 Test accuracy: 0.9808\n", - "15 Test accuracy: 0.9809\n", - "16 Test accuracy: 0.9807\n", - "17 Test accuracy: 0.9803\n", - "18 Test accuracy: 0.9816\n", - "19 Test accuracy: 0.9812\n" - ] - } - ], + "outputs": [], "source": [ "clip_all_weights = tf.get_collection(\"max_norm\")\n", "\n", @@ -3930,18 +2953,7 @@ "cell_type": "code", "execution_count": 116, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Extracting /tmp/data/train-images-idx3-ubyte.gz\n", - "Extracting /tmp/data/train-labels-idx1-ubyte.gz\n", - "Extracting /tmp/data/t10k-images-idx3-ubyte.gz\n", - "Extracting /tmp/data/t10k-labels-idx1-ubyte.gz\n" - ] - } - ], + "outputs": [], "source": [ "from tensorflow.examples.tutorials.mnist import input_data\n", "mnist = input_data.read_data_sets(\"/tmp/data/\")" @@ -3974,42 +2986,7 @@ "cell_type": "code", "execution_count": 118, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "0\tValidation loss: 0.128663\tBest loss: 0.128663\tAccuracy: 96.64%\n", - "1\tValidation loss: 0.448317\tBest loss: 0.128663\tAccuracy: 78.19%\n", - "2\tValidation loss: 0.190859\tBest loss: 0.128663\tAccuracy: 95.54%\n", - "3\tValidation loss: 0.146951\tBest loss: 0.128663\tAccuracy: 96.79%\n", - "4\tValidation loss: 0.086076\tBest loss: 0.086076\tAccuracy: 97.69%\n", - "5\tValidation loss: 0.115353\tBest loss: 0.086076\tAccuracy: 97.77%\n", - "6\tValidation loss: 0.239142\tBest loss: 0.086076\tAccuracy: 95.15%\n", - "7\tValidation loss: 0.088810\tBest loss: 0.086076\tAccuracy: 98.12%\n", - "8\tValidation loss: 0.108763\tBest loss: 0.086076\tAccuracy: 97.81%\n", - "9\tValidation loss: 0.300808\tBest loss: 0.086076\tAccuracy: 96.17%\n", - "10\tValidation loss: 0.179260\tBest loss: 0.086076\tAccuracy: 97.46%\n", - "11\tValidation loss: 0.125690\tBest loss: 0.086076\tAccuracy: 98.48%\n", - "12\tValidation loss: 0.738371\tBest loss: 0.086076\tAccuracy: 77.72%\n", - "13\tValidation loss: 1.894743\tBest loss: 0.086076\tAccuracy: 78.54%\n", - "14\tValidation loss: 0.415678\tBest loss: 0.086076\tAccuracy: 78.50%\n", - "15\tValidation loss: 0.537646\tBest loss: 0.086076\tAccuracy: 75.45%\n", - "16\tValidation loss: 1.009708\tBest loss: 0.086076\tAccuracy: 53.99%\n", - "17\tValidation loss: 1.228350\tBest loss: 0.086076\tAccuracy: 38.15%\n", - "18\tValidation loss: 1.510606\tBest loss: 0.086076\tAccuracy: 29.44%\n", - "19\tValidation loss: 1.632344\tBest loss: 0.086076\tAccuracy: 22.01%\n", - "20\tValidation loss: 1.628246\tBest loss: 0.086076\tAccuracy: 22.01%\n", - "21\tValidation loss: 1.626765\tBest loss: 0.086076\tAccuracy: 22.01%\n", - "22\tValidation loss: 1.651615\tBest loss: 0.086076\tAccuracy: 18.73%\n", - "23\tValidation loss: 1.663751\tBest loss: 0.086076\tAccuracy: 19.27%\n", - "24\tValidation loss: 1.675138\tBest loss: 0.086076\tAccuracy: 22.01%\n", - "Early stopping!\n", - "INFO:tensorflow:Restoring parameters from ./my_mnist_model_0_to_4.ckpt\n", - "Final test accuracy: 98.05%\n" - ] - } - ], + "outputs": [], "source": [ "n_epochs = 1000\n", "batch_size = 20\n", @@ -4277,56 +3254,7 @@ "cell_type": "code", "execution_count": 120, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "0\tValidation loss: 0.128663\tBest loss: 0.128663\tAccuracy: 96.64%\n", - "1\tValidation loss: 0.448317\tBest loss: 0.128663\tAccuracy: 78.19%\n", - "2\tValidation loss: 0.190859\tBest loss: 0.128663\tAccuracy: 95.54%\n", - "3\tValidation loss: 0.146951\tBest loss: 0.128663\tAccuracy: 96.79%\n", - "4\tValidation loss: 0.086076\tBest loss: 0.086076\tAccuracy: 97.69%\n", - "5\tValidation loss: 0.115353\tBest loss: 0.086076\tAccuracy: 97.77%\n", - "6\tValidation loss: 0.239142\tBest loss: 0.086076\tAccuracy: 95.15%\n", - "7\tValidation loss: 0.088810\tBest loss: 0.086076\tAccuracy: 98.12%\n", - "8\tValidation loss: 0.108763\tBest loss: 0.086076\tAccuracy: 97.81%\n", - "9\tValidation loss: 0.300808\tBest loss: 0.086076\tAccuracy: 96.17%\n", - "10\tValidation loss: 0.179260\tBest loss: 0.086076\tAccuracy: 97.46%\n", - "11\tValidation loss: 0.125690\tBest loss: 0.086076\tAccuracy: 98.48%\n", - "12\tValidation loss: 0.738371\tBest loss: 0.086076\tAccuracy: 77.72%\n", - "13\tValidation loss: 1.894743\tBest loss: 0.086076\tAccuracy: 78.54%\n", - "14\tValidation loss: 0.415678\tBest loss: 0.086076\tAccuracy: 78.50%\n", - "15\tValidation loss: 0.537646\tBest loss: 0.086076\tAccuracy: 75.45%\n", - "16\tValidation loss: 1.009708\tBest loss: 0.086076\tAccuracy: 53.99%\n", - "17\tValidation loss: 1.228350\tBest loss: 0.086076\tAccuracy: 38.15%\n", - "18\tValidation loss: 1.510606\tBest loss: 0.086076\tAccuracy: 29.44%\n", - "19\tValidation loss: 1.632344\tBest loss: 0.086076\tAccuracy: 22.01%\n", - "20\tValidation loss: 1.628246\tBest loss: 0.086076\tAccuracy: 22.01%\n", - "21\tValidation loss: 1.626765\tBest loss: 0.086076\tAccuracy: 22.01%\n", - "22\tValidation loss: 1.651615\tBest loss: 0.086076\tAccuracy: 18.73%\n", - "23\tValidation loss: 1.663751\tBest loss: 0.086076\tAccuracy: 19.27%\n", - "24\tValidation loss: 1.675138\tBest loss: 0.086076\tAccuracy: 22.01%\n", - "25\tValidation loss: 1.743664\tBest loss: 0.086076\tAccuracy: 18.73%\n", - "Early stopping!\n" - ] - }, - { - "data": { - "text/plain": [ - "DNNClassifier(activation=,\n", - " batch_norm_momentum=None, batch_size=20, dropout_rate=None,\n", - " initializer=._initializer at 0x7fd9d5e628c8>,\n", - " learning_rate=0.01, n_hidden_layers=5, n_neurons=100,\n", - " optimizer_class=,\n", - " random_state=42)" - ] - }, - "execution_count": 120, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "dnn_clf = DNNClassifier(random_state=42)\n", "dnn_clf.fit(X_train1, y_train1, n_epochs=1000, X_valid=X_valid1, y_valid=y_valid1)" @@ -4343,18 +3271,7 @@ "cell_type": "code", "execution_count": 121, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0.98054096127651291" - ] - }, - "execution_count": 121, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "from sklearn.metrics import accuracy_score\n", "\n", @@ -4373,250 +3290,7 @@ "cell_type": "code", "execution_count": 122, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Fitting 3 folds for each of 50 candidates, totalling 150 fits\n", - "[CV] n_neurons=10, learning_rate=0.05, activation=, batch_size=100 \n", - "0\tValidation loss: 0.132355\tBest loss: 0.132355\tAccuracy: 96.44%\n", - "1\tValidation loss: 0.126329\tBest loss: 0.126329\tAccuracy: 96.21%\n", - "2\tValidation loss: 0.138284\tBest loss: 0.126329\tAccuracy: 96.76%\n", - "3\tValidation loss: 0.142094\tBest loss: 0.126329\tAccuracy: 96.25%\n", - "4\tValidation loss: 0.128141\tBest loss: 0.126329\tAccuracy: 96.76%\n", - "5\tValidation loss: 0.119928\tBest loss: 0.119928\tAccuracy: 97.26%\n", - "6\tValidation loss: 0.137134\tBest loss: 0.119928\tAccuracy: 96.72%\n", - "7\tValidation loss: 0.156194\tBest loss: 0.119928\tAccuracy: 96.79%\n", - "8\tValidation loss: 0.283938\tBest loss: 0.119928\tAccuracy: 94.53%\n", - "9\tValidation loss: 1.104801\tBest loss: 0.119928\tAccuracy: 52.38%\n", - "10\tValidation loss: 0.966833\tBest loss: 0.119928\tAccuracy: 53.09%\n", - "11\tValidation loss: 0.854368\tBest loss: 0.119928\tAccuracy: 57.47%\n", - "12\tValidation loss: 1.857330\tBest loss: 0.119928\tAccuracy: 38.98%\n", - "13\tValidation loss: 1.642338\tBest loss: 0.119928\tAccuracy: 18.73%\n", - "14\tValidation loss: 1.612854\tBest loss: 0.119928\tAccuracy: 22.01%\n", - "15\tValidation loss: 1.617682\tBest loss: 0.119928\tAccuracy: 22.01%\n", - "16\tValidation loss: 1.616873\tBest loss: 0.119928\tAccuracy: 22.01%\n", - "17\tValidation loss: 1.618228\tBest loss: 0.119928\tAccuracy: 19.27%\n", - "18\tValidation loss: 1.619055\tBest loss: 0.119928\tAccuracy: 19.27%\n", - "19\tValidation loss: 1.643334\tBest loss: 0.119928\tAccuracy: 19.08%\n", - "20\tValidation loss: 1.621200\tBest loss: 0.119928\tAccuracy: 19.08%\n", - "21\tValidation loss: 1.629823\tBest loss: 0.119928\tAccuracy: 19.27%\n", - "22\tValidation loss: 1.624553\tBest loss: 0.119928\tAccuracy: 18.73%\n", - "23\tValidation loss: 1.610214\tBest loss: 0.119928\tAccuracy: 20.91%\n", - "24\tValidation loss: 1.621143\tBest loss: 0.119928\tAccuracy: 22.01%\n", - "25\tValidation loss: 1.623761\tBest loss: 0.119928\tAccuracy: 22.01%\n", - "26\tValidation loss: 1.641760\tBest loss: 0.119928\tAccuracy: 18.73%\n", - "Early stopping!\n", - "[CV] n_neurons=10, learning_rate=0.05, activation=, batch_size=100, total= 5.6s\n", - "[CV] n_neurons=10, learning_rate=0.05, activation=, batch_size=100 \n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 5.6s remaining: 0.0s\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "0\tValidation loss: 0.153707\tBest loss: 0.153707\tAccuracy: 95.74%\n", - "1\tValidation loss: 0.120703\tBest loss: 0.120703\tAccuracy: 96.56%\n", - "2\tValidation loss: 0.164706\tBest loss: 0.120703\tAccuracy: 96.05%\n", - "3\tValidation loss: 0.177875\tBest loss: 0.120703\tAccuracy: 95.19%\n", - "4\tValidation loss: 0.171004\tBest loss: 0.120703\tAccuracy: 95.19%\n", - "5\tValidation loss: 0.114746\tBest loss: 0.114746\tAccuracy: 96.83%\n", - "6\tValidation loss: 0.109637\tBest loss: 0.109637\tAccuracy: 97.26%\n", - "7\tValidation loss: 0.261533\tBest loss: 0.109637\tAccuracy: 94.96%\n", - "8\tValidation loss: 0.316743\tBest loss: 0.109637\tAccuracy: 94.02%\n", - "9\tValidation loss: 0.486484\tBest loss: 0.109637\tAccuracy: 77.56%\n", - "10\tValidation loss: 4.635532\tBest loss: 0.109637\tAccuracy: 53.95%\n", - "11\tValidation loss: 1.172422\tBest loss: 0.109637\tAccuracy: 48.36%\n", - "12\tValidation loss: 1.029865\tBest loss: 0.109637\tAccuracy: 55.98%\n", - "13\tValidation loss: 1.298800\tBest loss: 0.109637\tAccuracy: 36.08%\n", - "14\tValidation loss: 1.141950\tBest loss: 0.109637\tAccuracy: 38.08%\n", - "15\tValidation loss: 1.132486\tBest loss: 0.109637\tAccuracy: 38.90%\n", - "16\tValidation loss: 1.078486\tBest loss: 0.109637\tAccuracy: 45.78%\n", - "17\tValidation loss: 1.128344\tBest loss: 0.109637\tAccuracy: 45.07%\n", - "18\tValidation loss: 1.336244\tBest loss: 0.109637\tAccuracy: 34.40%\n", - "19\tValidation loss: 1.199178\tBest loss: 0.109637\tAccuracy: 39.87%\n", - "20\tValidation loss: 1.175845\tBest loss: 0.109637\tAccuracy: 40.11%\n", - "21\tValidation loss: 1.200430\tBest loss: 0.109637\tAccuracy: 40.30%\n", - "22\tValidation loss: 1.390084\tBest loss: 0.109637\tAccuracy: 34.60%\n", - "23\tValidation loss: 1.268129\tBest loss: 0.109637\tAccuracy: 40.23%\n", - "24\tValidation loss: 1.192210\tBest loss: 0.109637\tAccuracy: 40.30%\n", - "25\tValidation loss: 1.190541\tBest loss: 0.109637\tAccuracy: 41.99%\n", - "26\tValidation loss: 1.227676\tBest loss: 0.109637\tAccuracy: 38.62%\n", - "27\tValidation loss: 1.187587\tBest loss: 0.109637\tAccuracy: 39.44%\n", - "Early stopping!\n", - "[CV] n_neurons=10, learning_rate=0.05, activation=, batch_size=100, total= 5.9s\n", - "[CV] n_neurons=10, learning_rate=0.05, activation=, batch_size=100 \n", - "0\tValidation loss: 0.182619\tBest loss: 0.182619\tAccuracy: 94.29%\n", - "1\tValidation loss: 0.152706\tBest loss: 0.152706\tAccuracy: 95.97%\n", - "2\tValidation loss: 0.193820\tBest loss: 0.152706\tAccuracy: 93.82%\n", - "3\tValidation loss: 0.195413\tBest loss: 0.152706\tAccuracy: 95.54%\n", - "4\tValidation loss: 0.171277\tBest loss: 0.152706\tAccuracy: 95.19%\n", - "5\tValidation loss: 0.140087\tBest loss: 0.140087\tAccuracy: 95.70%\n", - "6\tValidation loss: 0.170798\tBest loss: 0.140087\tAccuracy: 95.00%\n", - "7\tValidation loss: 0.163649\tBest loss: 0.140087\tAccuracy: 96.29%\n", - "8\tValidation loss: 0.199048\tBest loss: 0.140087\tAccuracy: 96.09%\n", - "9\tValidation loss: 1.552870\tBest loss: 0.140087\tAccuracy: 52.15%\n", - "10\tValidation loss: 0.813273\tBest loss: 0.140087\tAccuracy: 60.40%\n", - "11\tValidation loss: 0.775555\tBest loss: 0.140087\tAccuracy: 60.67%\n", - "12\tValidation loss: 0.775275\tBest loss: 0.140087\tAccuracy: 59.77%\n", - "13\tValidation loss: 0.770521\tBest loss: 0.140087\tAccuracy: 59.30%\n", - "14\tValidation loss: 0.734035\tBest loss: 0.140087\tAccuracy: 59.85%\n", - "15\tValidation loss: 0.744980\tBest loss: 0.140087\tAccuracy: 59.66%\n", - "16\tValidation loss: 0.785848\tBest loss: 0.140087\tAccuracy: 59.66%\n", - "17\tValidation loss: 0.776138\tBest loss: 0.140087\tAccuracy: 59.42%\n", - "18\tValidation loss: 0.764496\tBest loss: 0.140087\tAccuracy: 59.46%\n", - "19\tValidation loss: 0.763633\tBest loss: 0.140087\tAccuracy: 59.54%\n", - "20\tValidation loss: 0.743879\tBest loss: 0.140087\tAccuracy: 60.75%\n", - "21\tValidation loss: 0.763295\tBest loss: 0.140087\tAccuracy: 60.36%\n", - "22\tValidation loss: 0.717175\tBest loss: 0.140087\tAccuracy: 60.63%\n", - "23\tValidation loss: 1.869954\tBest loss: 0.140087\tAccuracy: 29.28%\n", - "24\tValidation loss: 1.215518\tBest loss: 0.140087\tAccuracy: 38.86%\n", - "25\tValidation loss: 1.196626\tBest loss: 0.140087\tAccuracy: 38.62%\n", - "26\tValidation loss: 1.170714\tBest loss: 0.140087\tAccuracy: 42.38%\n", - "Early stopping!\n", - "[CV] n_neurons=10, learning_rate=0.05, activation=, batch_size=100, total= 6.9s\n", - "[CV] n_neurons=30, learning_rate=0.02, activation=, batch_size=500 \n", - "0\tValidation loss: 0.171512\tBest loss: 0.171512\tAccuracy: 95.07%\n", - "1\tValidation loss: 0.095914\tBest loss: 0.095914\tAccuracy: 97.03%\n", - "2\tValidation loss: 0.099199\tBest loss: 0.095914\tAccuracy: 96.91%\n", - "3\tValidation loss: 0.093873\tBest loss: 0.093873\tAccuracy: 97.15%\n", - "4\tValidation loss: 0.073461\tBest loss: 0.073461\tAccuracy: 98.01%\n", - "5\tValidation loss: 0.084562\tBest loss: 0.073461\tAccuracy: 97.65%\n", - "6\tValidation loss: 0.071800\tBest loss: 0.071800\tAccuracy: 98.01%\n", - "7\tValidation loss: 0.088435\tBest loss: 0.071800\tAccuracy: 97.73%\n", - "8\tValidation loss: 0.082038\tBest loss: 0.071800\tAccuracy: 97.77%\n", - "9\tValidation loss: 0.080673\tBest loss: 0.071800\tAccuracy: 97.69%\n", - "10\tValidation loss: 0.081036\tBest loss: 0.071800\tAccuracy: 97.93%\n", - "11\tValidation loss: 0.092700\tBest loss: 0.071800\tAccuracy: 97.93%\n", - "12\tValidation loss: 0.081003\tBest loss: 0.071800\tAccuracy: 98.20%\n", - "13\tValidation loss: 0.075607\tBest loss: 0.071800\tAccuracy: 98.20%\n", - "14\tValidation loss: 0.092970\tBest loss: 0.071800\tAccuracy: 98.08%\n", - "15\tValidation loss: 0.108005\tBest loss: 0.071800\tAccuracy: 97.77%\n", - "16\tValidation loss: 0.082602\tBest loss: 0.071800\tAccuracy: 98.05%\n", - "17\tValidation loss: 0.114629\tBest loss: 0.071800\tAccuracy: 97.73%\n", - "18\tValidation loss: 0.099099\tBest loss: 0.071800\tAccuracy: 97.69%\n", - "19\tValidation loss: 0.075535\tBest loss: 0.071800\tAccuracy: 98.20%\n", - "20\tValidation loss: 0.102847\tBest loss: 0.071800\tAccuracy: 98.08%\n", - "21\tValidation loss: 0.089735\tBest loss: 0.071800\tAccuracy: 98.36%\n", - "22\tValidation loss: 0.080781\tBest loss: 0.071800\tAccuracy: 97.93%\n", - "23\tValidation loss: 0.073017\tBest loss: 0.071800\tAccuracy: 98.32%\n", - "24\tValidation loss: 0.091643\tBest loss: 0.071800\tAccuracy: 97.93%\n", - "25\tValidation loss: 0.113891\tBest loss: 0.071800\tAccuracy: 98.05%\n", - "26\tValidation loss: 0.094774\tBest loss: 0.071800\tAccuracy: 98.28%\n", - "27\tValidation loss: 0.086041\tBest loss: 0.071800\tAccuracy: 98.20%\n", - "Early stopping!\n", - "[CV] n_neurons=30, learning_rate=0.02, activation=, batch_size=500, total= 6.8s\n", - "[CV] n_neurons=30, learning_rate=0.02, activation=, batch_size=500 \n", - "0\tValidation loss: 0.113188\tBest loss: 0.113188\tAccuracy: 96.60%\n", - "1\tValidation loss: 0.081384\tBest loss: 0.081384\tAccuracy: 97.58%\n", - "2\tValidation loss: 0.068770\tBest loss: 0.068770\tAccuracy: 98.12%\n", - "3\tValidation loss: 0.077316\tBest loss: 0.068770\tAccuracy: 97.73%\n", - "4\tValidation loss: 0.074333\tBest loss: 0.068770\tAccuracy: 97.97%\n", - "5\tValidation loss: 0.084735\tBest loss: 0.068770\tAccuracy: 97.30%\n", - "6\tValidation loss: 0.082893\tBest loss: 0.068770\tAccuracy: 97.69%\n", - "7\tValidation loss: 0.075860\tBest loss: 0.068770\tAccuracy: 97.65%\n", - "8\tValidation loss: 0.078686\tBest loss: 0.068770\tAccuracy: 97.77%\n", - "9\tValidation loss: 0.080869\tBest loss: 0.068770\tAccuracy: 97.77%\n", - "10\tValidation loss: 0.082026\tBest loss: 0.068770\tAccuracy: 98.12%\n", - "11\tValidation loss: 0.086516\tBest loss: 0.068770\tAccuracy: 97.69%\n", - "12\tValidation loss: 0.076660\tBest loss: 0.068770\tAccuracy: 98.12%\n", - "13\tValidation loss: 0.073815\tBest loss: 0.068770\tAccuracy: 98.08%\n", - "14\tValidation loss: 0.077873\tBest loss: 0.068770\tAccuracy: 98.20%\n", - "15\tValidation loss: 0.078704\tBest loss: 0.068770\tAccuracy: 97.93%\n", - "16\tValidation loss: 0.077061\tBest loss: 0.068770\tAccuracy: 98.28%\n", - "17\tValidation loss: 0.075423\tBest loss: 0.068770\tAccuracy: 97.93%\n", - "18\tValidation loss: 0.085646\tBest loss: 0.068770\tAccuracy: 98.24%\n", - "19\tValidation loss: 0.082202\tBest loss: 0.068770\tAccuracy: 98.05%\n", - "20\tValidation loss: 0.103338\tBest loss: 0.068770\tAccuracy: 97.46%\n", - "21\tValidation loss: 0.068182\tBest loss: 0.068182\tAccuracy: 98.40%\n", - "22\tValidation loss: 0.067592\tBest loss: 0.067592\tAccuracy: 97.93%\n", - "23\tValidation loss: 0.076756\tBest loss: 0.067592\tAccuracy: 98.28%\n", - "24\tValidation loss: 0.072327\tBest loss: 0.067592\tAccuracy: 98.48%\n", - "25\tValidation loss: 0.075613\tBest loss: 0.067592\tAccuracy: 98.44%\n", - "26\tValidation loss: 0.072291\tBest loss: 0.067592\tAccuracy: 98.40%\n", - "27\tValidation loss: 0.084550\tBest loss: 0.067592\tAccuracy: 98.28%\n", - "28\tValidation loss: 0.075566\tBest loss: 0.067592\tAccuracy: 98.36%\n", - "29\tValidation loss: 0.071688\tBest loss: 0.067592\tAccuracy: 98.28%\n", - "30\tValidation loss: 0.075556\tBest loss: 0.067592\tAccuracy: 98.24%\n", - "31\tValidation loss: 0.065671\tBest loss: 0.065671\tAccuracy: 98.40%\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "32\tValidation loss: 0.083471\tBest loss: 0.065671\tAccuracy: 98.40%\n", - "33\tValidation loss: 0.086415\tBest loss: 0.065671\tAccuracy: 98.59%\n", - "34\tValidation loss: 0.085613\tBest loss: 0.065671\tAccuracy: 98.36%\n", - "35\tValidation loss: 0.099534\tBest loss: 0.065671\tAccuracy: 98.28%\n", - "36\tValidation loss: 0.102709\tBest loss: 0.065671\tAccuracy: 98.32%\n", - "37\tValidation loss: 0.093125\tBest loss: 0.065671\tAccuracy: 98.20%\n", - "38\tValidation loss: 0.109501\tBest loss: 0.065671\tAccuracy: 97.85%\n", - "39\tValidation loss: 0.109443\tBest loss: 0.065671\tAccuracy: 98.44%\n", - "40\tValidation loss: 0.087260\tBest loss: 0.065671\tAccuracy: 98.36%\n", - "41\tValidation loss: 0.106365\tBest loss: 0.065671\tAccuracy: 98.36%\n", - "42\tValidation loss: 0.102789\tBest loss: 0.065671\tAccuracy: 98.05%\n", - "43\tValidation loss: 0.094281\tBest loss: 0.065671\tAccuracy: 98.48%\n", - "44\tValidation loss: 0.094514\tBest loss: 0.065671\tAccuracy: 98.40%\n", - "[...and much later...]\n", - "20\tValidation loss: 0.046808\tBest loss: 0.033867\tAccuracy: 98.83%\n", - "21\tValidation loss: 0.052966\tBest loss: 0.033867\tAccuracy: 98.91%\n", - "22\tValidation loss: 0.095892\tBest loss: 0.033867\tAccuracy: 98.08%\n", - "23\tValidation loss: 0.054250\tBest loss: 0.033867\tAccuracy: 98.87%\n", - "24\tValidation loss: 0.061026\tBest loss: 0.033867\tAccuracy: 98.87%\n", - "25\tValidation loss: 0.081977\tBest loss: 0.033867\tAccuracy: 98.67%\n", - "26\tValidation loss: 0.079819\tBest loss: 0.033867\tAccuracy: 98.71%\n", - "27\tValidation loss: 0.059824\tBest loss: 0.033867\tAccuracy: 98.75%\n", - "28\tValidation loss: 0.057758\tBest loss: 0.033867\tAccuracy: 98.94%\n", - "29\tValidation loss: 0.087165\tBest loss: 0.033867\tAccuracy: 98.91%\n", - "30\tValidation loss: 0.052274\tBest loss: 0.033867\tAccuracy: 99.10%\n", - "31\tValidation loss: 0.059831\tBest loss: 0.033867\tAccuracy: 98.79%\n", - "32\tValidation loss: 0.054240\tBest loss: 0.033867\tAccuracy: 98.91%\n", - "33\tValidation loss: 0.048165\tBest loss: 0.033867\tAccuracy: 98.94%\n", - "34\tValidation loss: 0.040565\tBest loss: 0.033867\tAccuracy: 99.18%\n", - "35\tValidation loss: 0.103207\tBest loss: 0.033867\tAccuracy: 98.28%\n", - "36\tValidation loss: 400.716797\tBest loss: 0.033867\tAccuracy: 71.46%\n", - "37\tValidation loss: 11.996887\tBest loss: 0.033867\tAccuracy: 96.09%\n", - "38\tValidation loss: 2.623182\tBest loss: 0.033867\tAccuracy: 96.56%\n", - "39\tValidation loss: 1.344962\tBest loss: 0.033867\tAccuracy: 97.69%\n", - "40\tValidation loss: 1.125381\tBest loss: 0.033867\tAccuracy: 97.42%\n", - "Early stopping!\n" - ] - }, - { - "data": { - "text/plain": [ - "RandomizedSearchCV(cv=None, error_score='raise',\n", - " estimator=DNNClassifier(activation=,\n", - " batch_norm_momentum=None, batch_size=20, dropout_rate=None,\n", - " initializer=._initializer at 0x7fd9d5e628c8>,\n", - " learning_rate=0.01, n_hidden_layers=5, n_neurons=100,\n", - " optimizer_class=,\n", - " random_state=42),\n", - " fit_params={'y_valid': array([0, 4, ..., 1, 2], dtype=uint8), 'X_valid': array([[ 0., 0., ..., 0., 0.],\n", - " [ 0., 0., ..., 0., 0.],\n", - " ...,\n", - " [ 0., 0., ..., 0., 0.],\n", - " [ 0., 0., ..., 0., 0.]], dtype=float32), 'n_epochs': 1000},\n", - " iid=True, n_iter=50, n_jobs=1,\n", - " param_distributions={'n_neurons': [10, 30, 50, 70, 90, 100, 120, 140, 160], 'learning_rate': [0.01, 0.02, 0.05, 0.1], 'activation': [, , .parametrized_leaky_relu at 0x7fd9db0b30d0>, .parametrized_leaky_relu at 0x7fd9d4ddca60>], 'batch_size': [10, 50, 100, 500]},\n", - " pre_dispatch='2*n_jobs', random_state=42, refit=True,\n", - " return_train_score=True, scoring=None, verbose=2)" - ] - }, - "execution_count": 122, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "from sklearn.model_selection import RandomizedSearchCV\n", "\n", @@ -4635,38 +3309,24 @@ " #\"optimizer_class\": [tf.train.AdamOptimizer, partial(tf.train.MomentumOptimizer, momentum=0.95)],\n", "}\n", "\n", - "rnd_search = RandomizedSearchCV(DNNClassifier(random_state=42), param_distribs, n_iter=50,\n", + "rnd_search = RandomizedSearchCV(DNNClassifier(random_state=42), param_distribs, n_iter=50,\n", " fit_params={\"X_valid\": X_valid1, \"y_valid\": y_valid1, \"n_epochs\": 1000},\n", " random_state=42, verbose=2)\n", - "rnd_search.fit(X_train1, y_train1)" + "rnd_search.fit(X_train1, y_train1)\n", "\n", - "# fit_params as a constructor argument was deprecated in [scikit-learn] version 0.19 and will be removed\n", - "# in version 0.21. Pass fit parameters to the fit method instead:" - "# rnd_search = RandomizedSearchCV(DNNClassifier(random_state=42), param_distribs, n_iter=50,\n", - "# random_state=42, verbose=2)\n", - "# fit_params={\"X_valid\": X_valid1, \"y_valid\": y_valid1, \"n_epochs\": 1000}\n" - "# rnd_search.fit(X_train1, y_train1, **fit_params)\n", + "# fit_params as a constructor argument was deprecated in Scikit-Learn version 0.19 and will\n", + "# be removed in version 0.21. Pass fit parameters to the fit() method instead:\n", + "# rnd_search = RandomizedSearchCV(DNNClassifier(random_state=42), param_distribs, n_iter=50,\n", + "# random_state=42, verbose=2)\n", + "# fit_params={\"X_valid\": X_valid1, \"y_valid\": y_valid1, \"n_epochs\": 1000}\n", + "# rnd_search.fit(X_train1, y_train1, **fit_params)\n" ] }, { "cell_type": "code", "execution_count": 123, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'activation': .parametrized_leaky_relu>,\n", - " 'batch_size': 500,\n", - " 'learning_rate': 0.01,\n", - " 'n_neurons': 140}" - ] - }, - "execution_count": 123, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "rnd_search.best_params_" ] @@ -4675,18 +3335,7 @@ "cell_type": "code", "execution_count": 124, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0.99318933644677954" - ] - }, - "execution_count": 124, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "y_pred = rnd_search.predict(X_test1)\n", "accuracy_score(y_test1, y_pred)" @@ -4742,71 +3391,7 @@ "cell_type": "code", "execution_count": 126, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "0\tValidation loss: 0.090732\tBest loss: 0.090732\tAccuracy: 97.22%\n", - "1\tValidation loss: 0.052198\tBest loss: 0.052198\tAccuracy: 98.40%\n", - "2\tValidation loss: 0.040040\tBest loss: 0.040040\tAccuracy: 98.94%\n", - "3\tValidation loss: 0.057495\tBest loss: 0.040040\tAccuracy: 98.55%\n", - "4\tValidation loss: 0.045600\tBest loss: 0.040040\tAccuracy: 98.75%\n", - "5\tValidation loss: 0.062344\tBest loss: 0.040040\tAccuracy: 98.48%\n", - "6\tValidation loss: 0.048719\tBest loss: 0.040040\tAccuracy: 98.67%\n", - "7\tValidation loss: 0.050346\tBest loss: 0.040040\tAccuracy: 98.79%\n", - "8\tValidation loss: 0.051224\tBest loss: 0.040040\tAccuracy: 98.79%\n", - "9\tValidation loss: 0.036505\tBest loss: 0.036505\tAccuracy: 98.98%\n", - "10\tValidation loss: 0.052532\tBest loss: 0.036505\tAccuracy: 98.71%\n", - "11\tValidation loss: 0.057086\tBest loss: 0.036505\tAccuracy: 99.10%\n", - "12\tValidation loss: 0.036754\tBest loss: 0.036505\tAccuracy: 99.06%\n", - "13\tValidation loss: 0.046782\tBest loss: 0.036505\tAccuracy: 98.87%\n", - "14\tValidation loss: 0.048929\tBest loss: 0.036505\tAccuracy: 98.91%\n", - "15\tValidation loss: 0.052919\tBest loss: 0.036505\tAccuracy: 98.75%\n", - "16\tValidation loss: 0.054287\tBest loss: 0.036505\tAccuracy: 98.67%\n", - "17\tValidation loss: 0.047722\tBest loss: 0.036505\tAccuracy: 98.79%\n", - "18\tValidation loss: 0.040474\tBest loss: 0.036505\tAccuracy: 99.14%\n", - "19\tValidation loss: 0.033867\tBest loss: 0.033867\tAccuracy: 99.14%\n", - "20\tValidation loss: 0.046808\tBest loss: 0.033867\tAccuracy: 98.83%\n", - "21\tValidation loss: 0.052966\tBest loss: 0.033867\tAccuracy: 98.91%\n", - "22\tValidation loss: 0.095892\tBest loss: 0.033867\tAccuracy: 98.08%\n", - "23\tValidation loss: 0.054250\tBest loss: 0.033867\tAccuracy: 98.87%\n", - "24\tValidation loss: 0.061026\tBest loss: 0.033867\tAccuracy: 98.87%\n", - "25\tValidation loss: 0.081977\tBest loss: 0.033867\tAccuracy: 98.67%\n", - "26\tValidation loss: 0.079819\tBest loss: 0.033867\tAccuracy: 98.71%\n", - "27\tValidation loss: 0.059824\tBest loss: 0.033867\tAccuracy: 98.75%\n", - "28\tValidation loss: 0.057758\tBest loss: 0.033867\tAccuracy: 98.94%\n", - "29\tValidation loss: 0.087165\tBest loss: 0.033867\tAccuracy: 98.91%\n", - "30\tValidation loss: 0.052274\tBest loss: 0.033867\tAccuracy: 99.10%\n", - "31\tValidation loss: 0.059831\tBest loss: 0.033867\tAccuracy: 98.79%\n", - "32\tValidation loss: 0.054240\tBest loss: 0.033867\tAccuracy: 98.91%\n", - "33\tValidation loss: 0.048165\tBest loss: 0.033867\tAccuracy: 98.94%\n", - "34\tValidation loss: 0.040565\tBest loss: 0.033867\tAccuracy: 99.18%\n", - "35\tValidation loss: 0.103207\tBest loss: 0.033867\tAccuracy: 98.28%\n", - "36\tValidation loss: 400.716797\tBest loss: 0.033867\tAccuracy: 71.46%\n", - "37\tValidation loss: 11.996887\tBest loss: 0.033867\tAccuracy: 96.09%\n", - "38\tValidation loss: 2.623182\tBest loss: 0.033867\tAccuracy: 96.56%\n", - "39\tValidation loss: 1.344962\tBest loss: 0.033867\tAccuracy: 97.69%\n", - "40\tValidation loss: 1.125381\tBest loss: 0.033867\tAccuracy: 97.42%\n", - "Early stopping!\n" - ] - }, - { - "data": { - "text/plain": [ - "DNNClassifier(activation=.parametrized_leaky_relu at 0x7fd9d19e37b8>,\n", - " batch_norm_momentum=None, batch_size=500, dropout_rate=None,\n", - " initializer=._initializer at 0x7fd9d5e628c8>,\n", - " learning_rate=0.01, n_hidden_layers=5, n_neurons=140,\n", - " optimizer_class=,\n", - " random_state=42)" - ] - }, - "execution_count": 126, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "dnn_clf = DNNClassifier(activation=leaky_relu(alpha=0.1), batch_size=500, learning_rate=0.01,\n", " n_neurons=140, random_state=42)\n", @@ -4831,18 +3416,7 @@ "cell_type": "code", "execution_count": 127, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0.99318933644677954" - ] - }, - "execution_count": 127, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "y_pred = dnn_clf.predict(X_test1)\n", "accuracy_score(y_test1, y_pred)" @@ -4859,100 +3433,7 @@ "cell_type": "code", "execution_count": 128, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "0\tValidation loss: 0.046053\tBest loss: 0.046053\tAccuracy: 98.67%\n", - "1\tValidation loss: 0.032228\tBest loss: 0.032228\tAccuracy: 98.83%\n", - "2\tValidation loss: 0.032974\tBest loss: 0.032228\tAccuracy: 98.83%\n", - "3\tValidation loss: 0.035961\tBest loss: 0.032228\tAccuracy: 98.94%\n", - "4\tValidation loss: 0.040250\tBest loss: 0.032228\tAccuracy: 98.94%\n", - "5\tValidation loss: 0.033051\tBest loss: 0.032228\tAccuracy: 99.06%\n", - "6\tValidation loss: 0.056053\tBest loss: 0.032228\tAccuracy: 98.32%\n", - "7\tValidation loss: 0.031729\tBest loss: 0.031729\tAccuracy: 99.18%\n", - "8\tValidation loss: 0.027662\tBest loss: 0.027662\tAccuracy: 99.26%\n", - "9\tValidation loss: 0.034074\tBest loss: 0.027662\tAccuracy: 98.94%\n", - "10\tValidation loss: 0.032173\tBest loss: 0.027662\tAccuracy: 99.06%\n", - "11\tValidation loss: 0.030538\tBest loss: 0.027662\tAccuracy: 99.10%\n", - "12\tValidation loss: 0.030337\tBest loss: 0.027662\tAccuracy: 99.10%\n", - "13\tValidation loss: 0.022219\tBest loss: 0.022219\tAccuracy: 99.45%\n", - "14\tValidation loss: 0.036824\tBest loss: 0.022219\tAccuracy: 99.14%\n", - "15\tValidation loss: 0.033945\tBest loss: 0.022219\tAccuracy: 99.18%\n", - "16\tValidation loss: 0.032533\tBest loss: 0.022219\tAccuracy: 98.98%\n", - "17\tValidation loss: 0.037204\tBest loss: 0.022219\tAccuracy: 99.02%\n", - "18\tValidation loss: 0.026982\tBest loss: 0.022219\tAccuracy: 99.34%\n", - "19\tValidation loss: 0.022094\tBest loss: 0.022094\tAccuracy: 99.53%\n", - "20\tValidation loss: 0.026196\tBest loss: 0.022094\tAccuracy: 99.26%\n", - "21\tValidation loss: 0.022107\tBest loss: 0.022094\tAccuracy: 99.49%\n", - "22\tValidation loss: 0.021436\tBest loss: 0.021436\tAccuracy: 99.53%\n", - "23\tValidation loss: 0.025607\tBest loss: 0.021436\tAccuracy: 99.37%\n", - "24\tValidation loss: 0.038882\tBest loss: 0.021436\tAccuracy: 99.22%\n", - "25\tValidation loss: 0.032011\tBest loss: 0.021436\tAccuracy: 99.26%\n", - "26\tValidation loss: 0.027673\tBest loss: 0.021436\tAccuracy: 99.22%\n", - "27\tValidation loss: 0.026874\tBest loss: 0.021436\tAccuracy: 99.30%\n", - "28\tValidation loss: 0.021123\tBest loss: 0.021123\tAccuracy: 99.41%\n", - "29\tValidation loss: 0.024784\tBest loss: 0.021123\tAccuracy: 99.45%\n", - "30\tValidation loss: 0.024108\tBest loss: 0.021123\tAccuracy: 99.49%\n", - "31\tValidation loss: 0.028439\tBest loss: 0.021123\tAccuracy: 99.37%\n", - "32\tValidation loss: 0.032366\tBest loss: 0.021123\tAccuracy: 99.22%\n", - "33\tValidation loss: 0.037057\tBest loss: 0.021123\tAccuracy: 99.18%\n", - "34\tValidation loss: 0.042305\tBest loss: 0.021123\tAccuracy: 98.98%\n", - "35\tValidation loss: 0.039662\tBest loss: 0.021123\tAccuracy: 99.14%\n", - "36\tValidation loss: 0.036299\tBest loss: 0.021123\tAccuracy: 99.14%\n", - "37\tValidation loss: 0.026997\tBest loss: 0.021123\tAccuracy: 99.53%\n", - "38\tValidation loss: 0.034407\tBest loss: 0.021123\tAccuracy: 99.22%\n", - "39\tValidation loss: 0.027668\tBest loss: 0.021123\tAccuracy: 99.41%\n", - "40\tValidation loss: 0.029128\tBest loss: 0.021123\tAccuracy: 99.30%\n", - "41\tValidation loss: 0.033564\tBest loss: 0.021123\tAccuracy: 99.14%\n", - "42\tValidation loss: 0.033810\tBest loss: 0.021123\tAccuracy: 99.30%\n", - "43\tValidation loss: 0.044953\tBest loss: 0.021123\tAccuracy: 98.98%\n", - "44\tValidation loss: 0.026280\tBest loss: 0.021123\tAccuracy: 99.26%\n", - "45\tValidation loss: 0.020275\tBest loss: 0.020275\tAccuracy: 99.61%\n", - "46\tValidation loss: 0.018810\tBest loss: 0.018810\tAccuracy: 99.45%\n", - "47\tValidation loss: 0.027529\tBest loss: 0.018810\tAccuracy: 99.18%\n", - "48\tValidation loss: 0.018120\tBest loss: 0.018120\tAccuracy: 99.53%\n", - "49\tValidation loss: 0.019378\tBest loss: 0.018120\tAccuracy: 99.45%\n", - "50\tValidation loss: 0.029760\tBest loss: 0.018120\tAccuracy: 99.34%\n", - "51\tValidation loss: 0.035702\tBest loss: 0.018120\tAccuracy: 99.26%\n", - "52\tValidation loss: 0.032662\tBest loss: 0.018120\tAccuracy: 99.02%\n", - "53\tValidation loss: 0.026943\tBest loss: 0.018120\tAccuracy: 99.37%\n", - "54\tValidation loss: 0.029007\tBest loss: 0.018120\tAccuracy: 99.53%\n", - "55\tValidation loss: 0.021956\tBest loss: 0.018120\tAccuracy: 99.49%\n", - "56\tValidation loss: 0.018983\tBest loss: 0.018120\tAccuracy: 99.61%\n", - "57\tValidation loss: 0.022788\tBest loss: 0.018120\tAccuracy: 99.49%\n", - "58\tValidation loss: 0.019578\tBest loss: 0.018120\tAccuracy: 99.61%\n", - "59\tValidation loss: 0.021676\tBest loss: 0.018120\tAccuracy: 99.61%\n", - "60\tValidation loss: 0.021580\tBest loss: 0.018120\tAccuracy: 99.65%\n", - "61\tValidation loss: 0.021467\tBest loss: 0.018120\tAccuracy: 99.65%\n", - "62\tValidation loss: 0.020513\tBest loss: 0.018120\tAccuracy: 99.65%\n", - "63\tValidation loss: 0.020252\tBest loss: 0.018120\tAccuracy: 99.65%\n", - "64\tValidation loss: 0.021724\tBest loss: 0.018120\tAccuracy: 99.65%\n", - "65\tValidation loss: 0.021499\tBest loss: 0.018120\tAccuracy: 99.69%\n", - "66\tValidation loss: 0.021627\tBest loss: 0.018120\tAccuracy: 99.69%\n", - "67\tValidation loss: 0.021569\tBest loss: 0.018120\tAccuracy: 99.69%\n", - "68\tValidation loss: 0.021727\tBest loss: 0.018120\tAccuracy: 99.69%\n", - "69\tValidation loss: 0.021104\tBest loss: 0.018120\tAccuracy: 99.69%\n", - "Early stopping!\n" - ] - }, - { - "data": { - "text/plain": [ - "DNNClassifier(activation=.parametrized_leaky_relu at 0x7fd9d19e3c80>,\n", - " batch_norm_momentum=0.95, batch_size=500, dropout_rate=None,\n", - " initializer=._initializer at 0x7fd9d5e628c8>,\n", - " learning_rate=0.01, n_hidden_layers=5, n_neurons=90,\n", - " optimizer_class=,\n", - " random_state=42)" - ] - }, - "execution_count": 128, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "dnn_clf_bn = DNNClassifier(activation=leaky_relu(alpha=0.1), batch_size=500, learning_rate=0.01,\n", " n_neurons=90, random_state=42,\n", @@ -4971,18 +3452,7 @@ "cell_type": "code", "execution_count": 129, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0.99241097489784003" - ] - }, - "execution_count": 129, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "y_pred = dnn_clf_bn.predict(X_test1)\n", "accuracy_score(y_test1, y_pred)" @@ -4999,222 +3469,7 @@ "cell_type": "code", "execution_count": 130, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Fitting 3 folds for each of 50 candidates, totalling 150 fits\n", - "[CV] activation=, n_neurons=70, learning_rate=0.01, batch_norm_momentum=0.99, batch_size=50 \n", - "0\tValidation loss: 0.113224\tBest loss: 0.113224\tAccuracy: 97.30%\n", - "1\tValidation loss: 0.064190\tBest loss: 0.064190\tAccuracy: 98.24%\n", - "2\tValidation loss: 0.080173\tBest loss: 0.064190\tAccuracy: 98.28%\n", - "3\tValidation loss: 0.059603\tBest loss: 0.059603\tAccuracy: 98.28%\n", - "4\tValidation loss: 0.043533\tBest loss: 0.043533\tAccuracy: 98.48%\n", - "5\tValidation loss: 0.040107\tBest loss: 0.040107\tAccuracy: 98.87%\n", - "6\tValidation loss: 0.051212\tBest loss: 0.040107\tAccuracy: 98.24%\n", - "7\tValidation loss: 0.046029\tBest loss: 0.040107\tAccuracy: 98.71%\n", - "8\tValidation loss: 0.053079\tBest loss: 0.040107\tAccuracy: 98.59%\n", - "9\tValidation loss: 0.066891\tBest loss: 0.040107\tAccuracy: 98.28%\n", - "10\tValidation loss: 0.037712\tBest loss: 0.037712\tAccuracy: 98.83%\n", - "11\tValidation loss: 0.055569\tBest loss: 0.037712\tAccuracy: 98.55%\n", - "12\tValidation loss: 0.040949\tBest loss: 0.037712\tAccuracy: 98.98%\n", - "13\tValidation loss: 0.077433\tBest loss: 0.037712\tAccuracy: 98.36%\n", - "14\tValidation loss: 0.065955\tBest loss: 0.037712\tAccuracy: 98.63%\n", - "15\tValidation loss: 0.038968\tBest loss: 0.037712\tAccuracy: 99.02%\n", - "16\tValidation loss: 0.039190\tBest loss: 0.037712\tAccuracy: 99.06%\n", - "17\tValidation loss: 0.050690\tBest loss: 0.037712\tAccuracy: 98.71%\n", - "18\tValidation loss: 0.043054\tBest loss: 0.037712\tAccuracy: 99.02%\n", - "19\tValidation loss: 0.063156\tBest loss: 0.037712\tAccuracy: 98.71%\n", - "20\tValidation loss: 0.043066\tBest loss: 0.037712\tAccuracy: 99.14%\n", - "21\tValidation loss: 0.058145\tBest loss: 0.037712\tAccuracy: 98.79%\n", - "22\tValidation loss: 0.039590\tBest loss: 0.037712\tAccuracy: 99.06%\n", - "23\tValidation loss: 0.049981\tBest loss: 0.037712\tAccuracy: 98.75%\n", - "24\tValidation loss: 0.047458\tBest loss: 0.037712\tAccuracy: 99.10%\n", - "25\tValidation loss: 0.040638\tBest loss: 0.037712\tAccuracy: 99.06%\n", - "26\tValidation loss: 0.041426\tBest loss: 0.037712\tAccuracy: 98.98%\n", - "27\tValidation loss: 0.041325\tBest loss: 0.037712\tAccuracy: 98.98%\n", - "28\tValidation loss: 0.054609\tBest loss: 0.037712\tAccuracy: 98.91%\n", - "29\tValidation loss: 0.067671\tBest loss: 0.037712\tAccuracy: 98.75%\n", - "30\tValidation loss: 0.037608\tBest loss: 0.037608\tAccuracy: 98.79%\n", - "31\tValidation loss: 0.047441\tBest loss: 0.037608\tAccuracy: 98.98%\n", - "32\tValidation loss: 0.053716\tBest loss: 0.037608\tAccuracy: 99.02%\n", - "33\tValidation loss: 0.045445\tBest loss: 0.037608\tAccuracy: 98.83%\n", - "34\tValidation loss: 0.046023\tBest loss: 0.037608\tAccuracy: 98.94%\n", - "35\tValidation loss: 0.050073\tBest loss: 0.037608\tAccuracy: 98.91%\n", - "36\tValidation loss: 0.051887\tBest loss: 0.037608\tAccuracy: 98.87%\n", - "37\tValidation loss: 0.050272\tBest loss: 0.037608\tAccuracy: 99.02%\n", - "38\tValidation loss: 0.043531\tBest loss: 0.037608\tAccuracy: 99.10%\n", - "39\tValidation loss: 0.054661\tBest loss: 0.037608\tAccuracy: 98.87%\n", - "40\tValidation loss: 0.047607\tBest loss: 0.037608\tAccuracy: 98.87%\n", - "41\tValidation loss: 0.051862\tBest loss: 0.037608\tAccuracy: 99.14%\n", - "42\tValidation loss: 0.044218\tBest loss: 0.037608\tAccuracy: 99.14%\n", - "43\tValidation loss: 0.043707\tBest loss: 0.037608\tAccuracy: 99.06%\n", - "44\tValidation loss: 0.039602\tBest loss: 0.037608\tAccuracy: 99.06%\n", - "45\tValidation loss: 0.048998\tBest loss: 0.037608\tAccuracy: 99.02%\n", - "46\tValidation loss: 0.045562\tBest loss: 0.037608\tAccuracy: 99.14%\n", - "47\tValidation loss: 0.042198\tBest loss: 0.037608\tAccuracy: 99.10%\n", - "48\tValidation loss: 0.027679\tBest loss: 0.027679\tAccuracy: 99.10%\n", - "49\tValidation loss: 0.033783\tBest loss: 0.027679\tAccuracy: 98.94%\n", - "50\tValidation loss: 0.032935\tBest loss: 0.027679\tAccuracy: 99.41%\n", - "51\tValidation loss: 0.042930\tBest loss: 0.027679\tAccuracy: 98.98%\n", - "52\tValidation loss: 0.045454\tBest loss: 0.027679\tAccuracy: 99.06%\n", - "53\tValidation loss: 0.047336\tBest loss: 0.027679\tAccuracy: 98.91%\n", - "54\tValidation loss: 0.036523\tBest loss: 0.027679\tAccuracy: 99.14%\n", - "55\tValidation loss: 0.064401\tBest loss: 0.027679\tAccuracy: 98.94%\n", - "56\tValidation loss: 0.047686\tBest loss: 0.027679\tAccuracy: 98.83%\n", - "57\tValidation loss: 0.049083\tBest loss: 0.027679\tAccuracy: 98.98%\n", - "58\tValidation loss: 0.057310\tBest loss: 0.027679\tAccuracy: 99.10%\n", - "59\tValidation loss: 0.043757\tBest loss: 0.027679\tAccuracy: 99.14%\n", - "60\tValidation loss: 0.058742\tBest loss: 0.027679\tAccuracy: 99.02%\n", - "61\tValidation loss: 0.055049\tBest loss: 0.027679\tAccuracy: 99.06%\n", - "62\tValidation loss: 0.039837\tBest loss: 0.027679\tAccuracy: 99.18%\n", - "63\tValidation loss: 0.057108\tBest loss: 0.027679\tAccuracy: 99.06%\n", - "64\tValidation loss: 0.043212\tBest loss: 0.027679\tAccuracy: 98.98%\n", - "65\tValidation loss: 0.046874\tBest loss: 0.027679\tAccuracy: 99.18%\n", - "66\tValidation loss: 0.052819\tBest loss: 0.027679\tAccuracy: 99.10%\n", - "67\tValidation loss: 0.045977\tBest loss: 0.027679\tAccuracy: 99.14%\n", - "68\tValidation loss: 0.053290\tBest loss: 0.027679\tAccuracy: 99.10%\n", - "69\tValidation loss: 0.052941\tBest loss: 0.027679\tAccuracy: 99.06%\n", - "Early stopping!\n", - "[CV] activation=, n_neurons=70, learning_rate=0.01, batch_norm_momentum=0.99, batch_size=50, total= 2.7min\n", - "[CV] activation=, n_neurons=70, learning_rate=0.01, batch_norm_momentum=0.99, batch_size=50 \n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 2.7min remaining: 0.0s\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "0\tValidation loss: 0.144984\tBest loss: 0.144984\tAccuracy: 96.40%\n", - "1\tValidation loss: 0.067873\tBest loss: 0.067873\tAccuracy: 98.44%\n", - "2\tValidation loss: 0.091854\tBest loss: 0.067873\tAccuracy: 97.30%\n", - "3\tValidation loss: 0.074647\tBest loss: 0.067873\tAccuracy: 98.05%\n", - "4\tValidation loss: 0.053722\tBest loss: 0.053722\tAccuracy: 98.48%\n", - "5\tValidation loss: 0.049216\tBest loss: 0.049216\tAccuracy: 98.44%\n", - "6\tValidation loss: 0.057619\tBest loss: 0.049216\tAccuracy: 98.48%\n", - "7\tValidation loss: 0.045842\tBest loss: 0.045842\tAccuracy: 98.75%\n", - "8\tValidation loss: 0.042398\tBest loss: 0.042398\tAccuracy: 98.63%\n", - "9\tValidation loss: 0.052629\tBest loss: 0.042398\tAccuracy: 98.63%\n", - "10\tValidation loss: 0.056892\tBest loss: 0.042398\tAccuracy: 98.63%\n", - "11\tValidation loss: 0.051838\tBest loss: 0.042398\tAccuracy: 98.75%\n", - "12\tValidation loss: 0.042647\tBest loss: 0.042398\tAccuracy: 98.67%\n", - "13\tValidation loss: 0.061297\tBest loss: 0.042398\tAccuracy: 98.59%\n", - "14\tValidation loss: 0.049706\tBest loss: 0.042398\tAccuracy: 98.87%\n", - "15\tValidation loss: 0.061934\tBest loss: 0.042398\tAccuracy: 98.79%\n", - "16\tValidation loss: 0.049027\tBest loss: 0.042398\tAccuracy: 98.87%\n", - "17\tValidation loss: 0.052187\tBest loss: 0.042398\tAccuracy: 98.79%\n", - "18\tValidation loss: 0.052031\tBest loss: 0.042398\tAccuracy: 98.94%\n", - "[...and much later...]\n", - "13\tValidation loss: 0.043686\tBest loss: 0.040332\tAccuracy: 99.02%\n", - "14\tValidation loss: 0.046940\tBest loss: 0.040332\tAccuracy: 99.18%\n", - "15\tValidation loss: 0.045355\tBest loss: 0.040332\tAccuracy: 99.14%\n", - "16\tValidation loss: 0.084697\tBest loss: 0.040332\tAccuracy: 98.87%\n", - "17\tValidation loss: 0.123538\tBest loss: 0.040332\tAccuracy: 97.81%\n", - "18\tValidation loss: 0.296928\tBest loss: 0.040332\tAccuracy: 97.50%\n", - "19\tValidation loss: 0.053660\tBest loss: 0.040332\tAccuracy: 98.91%\n", - "20\tValidation loss: 0.045684\tBest loss: 0.040332\tAccuracy: 98.94%\n", - "21\tValidation loss: 0.051971\tBest loss: 0.040332\tAccuracy: 99.14%\n", - "22\tValidation loss: 0.071830\tBest loss: 0.040332\tAccuracy: 99.06%\n", - "23\tValidation loss: 0.069619\tBest loss: 0.040332\tAccuracy: 98.79%\n", - "24\tValidation loss: 0.086642\tBest loss: 0.040332\tAccuracy: 98.71%\n", - "25\tValidation loss: 0.072563\tBest loss: 0.040332\tAccuracy: 98.83%\n", - "26\tValidation loss: 0.058974\tBest loss: 0.040332\tAccuracy: 99.06%\n", - "27\tValidation loss: 0.048388\tBest loss: 0.040332\tAccuracy: 98.98%\n", - "28\tValidation loss: 0.054847\tBest loss: 0.040332\tAccuracy: 99.06%\n", - "29\tValidation loss: 0.077242\tBest loss: 0.040332\tAccuracy: 98.91%\n", - "30\tValidation loss: 0.556978\tBest loss: 0.040332\tAccuracy: 95.54%\n", - "Early stopping!\n", - "[CV] activation=, n_neurons=140, learning_rate=0.05, batch_norm_momentum=0.99, batch_size=50, total= 1.9min\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[Parallel(n_jobs=1)]: Done 150 out of 150 | elapsed: 355.8min finished\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "0\tValidation loss: 0.076371\tBest loss: 0.076371\tAccuracy: 97.85%\n", - "1\tValidation loss: 0.049312\tBest loss: 0.049312\tAccuracy: 98.63%\n", - "2\tValidation loss: 0.033071\tBest loss: 0.033071\tAccuracy: 98.94%\n", - "3\tValidation loss: 0.027357\tBest loss: 0.027357\tAccuracy: 99.10%\n", - "4\tValidation loss: 0.028748\tBest loss: 0.027357\tAccuracy: 99.26%\n", - "5\tValidation loss: 0.036602\tBest loss: 0.027357\tAccuracy: 98.94%\n", - "6\tValidation loss: 0.048089\tBest loss: 0.027357\tAccuracy: 98.94%\n", - "7\tValidation loss: 0.030332\tBest loss: 0.027357\tAccuracy: 99.30%\n", - "8\tValidation loss: 0.029336\tBest loss: 0.027357\tAccuracy: 99.22%\n", - "9\tValidation loss: 0.033328\tBest loss: 0.027357\tAccuracy: 99.26%\n", - "10\tValidation loss: 0.041745\tBest loss: 0.027357\tAccuracy: 98.98%\n", - "11\tValidation loss: 0.048739\tBest loss: 0.027357\tAccuracy: 98.75%\n", - "12\tValidation loss: 0.049520\tBest loss: 0.027357\tAccuracy: 98.94%\n", - "13\tValidation loss: 0.034222\tBest loss: 0.027357\tAccuracy: 99.18%\n", - "14\tValidation loss: 0.040270\tBest loss: 0.027357\tAccuracy: 99.34%\n", - "15\tValidation loss: 0.033074\tBest loss: 0.027357\tAccuracy: 99.37%\n", - "16\tValidation loss: 0.035130\tBest loss: 0.027357\tAccuracy: 99.06%\n", - "17\tValidation loss: 0.031875\tBest loss: 0.027357\tAccuracy: 99.18%\n", - "18\tValidation loss: 0.034898\tBest loss: 0.027357\tAccuracy: 99.37%\n", - "19\tValidation loss: 0.019222\tBest loss: 0.019222\tAccuracy: 99.53%\n", - "20\tValidation loss: 0.043814\tBest loss: 0.019222\tAccuracy: 99.37%\n", - "21\tValidation loss: 0.028773\tBest loss: 0.019222\tAccuracy: 99.34%\n", - "22\tValidation loss: 0.024850\tBest loss: 0.019222\tAccuracy: 99.45%\n", - "23\tValidation loss: 0.021789\tBest loss: 0.019222\tAccuracy: 99.45%\n", - "24\tValidation loss: 0.028846\tBest loss: 0.019222\tAccuracy: 99.37%\n", - "25\tValidation loss: 0.064211\tBest loss: 0.019222\tAccuracy: 98.98%\n", - "26\tValidation loss: 0.024425\tBest loss: 0.019222\tAccuracy: 99.49%\n", - "27\tValidation loss: 0.035453\tBest loss: 0.019222\tAccuracy: 99.22%\n", - "28\tValidation loss: 0.023940\tBest loss: 0.019222\tAccuracy: 99.37%\n", - "29\tValidation loss: 0.041495\tBest loss: 0.019222\tAccuracy: 99.18%\n", - "30\tValidation loss: 0.028030\tBest loss: 0.019222\tAccuracy: 99.37%\n", - "31\tValidation loss: 0.028003\tBest loss: 0.019222\tAccuracy: 99.49%\n", - "32\tValidation loss: 0.026579\tBest loss: 0.019222\tAccuracy: 99.45%\n", - "33\tValidation loss: 0.037838\tBest loss: 0.019222\tAccuracy: 98.91%\n", - "34\tValidation loss: 0.026082\tBest loss: 0.019222\tAccuracy: 99.49%\n", - "35\tValidation loss: 0.031529\tBest loss: 0.019222\tAccuracy: 99.34%\n", - "36\tValidation loss: 0.028220\tBest loss: 0.019222\tAccuracy: 99.18%\n", - "37\tValidation loss: 0.038546\tBest loss: 0.019222\tAccuracy: 99.10%\n", - "38\tValidation loss: 0.041586\tBest loss: 0.019222\tAccuracy: 98.75%\n", - "39\tValidation loss: 0.038835\tBest loss: 0.019222\tAccuracy: 99.41%\n", - "40\tValidation loss: 0.042555\tBest loss: 0.019222\tAccuracy: 99.14%\n", - "Early stopping!\n" - ] - }, - { - "data": { - "text/plain": [ - "RandomizedSearchCV(cv=None, error_score='raise',\n", - " estimator=DNNClassifier(activation=,\n", - " batch_norm_momentum=None, batch_size=20, dropout_rate=None,\n", - " initializer=._initializer at 0x7fd9d5e628c8>,\n", - " learning_rate=0.01, n_hidden_layers=5, n_neurons=100,\n", - " optimizer_class=,\n", - " random_state=42),\n", - " fit_params={'y_valid': array([0, 4, ..., 1, 2], dtype=uint8), 'X_valid': array([[ 0., 0., ..., 0., 0.],\n", - " [ 0., 0., ..., 0., 0.],\n", - " ...,\n", - " [ 0., 0., ..., 0., 0.],\n", - " [ 0., 0., ..., 0., 0.]], dtype=float32), 'n_epochs': 1000},\n", - " iid=True, n_iter=50, n_jobs=1,\n", - " param_distributions={'batch_norm_momentum': [0.9, 0.95, 0.98, 0.99, 0.999], 'n_neurons': [10, 30, 50, 70, 90, 100, 120, 140, 160], 'learning_rate': [0.01, 0.02, 0.05, 0.1], 'activation': [, , .parametrized_leaky_relu at 0x7fd9d19e3bf8>, .parametrized_leaky_relu at 0x7fd9d19e3a60>], 'batch_size': [10, 50, 100, 500]},\n", - " pre_dispatch='2*n_jobs', random_state=42, refit=True,\n", - " return_train_score=True, scoring=None, verbose=2)" - ] - }, - "execution_count": 130, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "from sklearn.model_selection import RandomizedSearchCV\n", "\n", @@ -5239,22 +3494,7 @@ "cell_type": "code", "execution_count": 131, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'activation': ,\n", - " 'batch_norm_momentum': 0.98,\n", - " 'batch_size': 100,\n", - " 'learning_rate': 0.01,\n", - " 'n_neurons': 160}" - ] - }, - "execution_count": 131, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "rnd_search_bn.best_params_" ] @@ -5263,18 +3503,7 @@ "cell_type": "code", "execution_count": 132, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0.99396769799571905" - ] - }, - "execution_count": 132, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "y_pred = rnd_search_bn.predict(X_test1)\n", "accuracy_score(y_test1, y_pred)" @@ -5312,18 +3541,7 @@ "cell_type": "code", "execution_count": 133, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0.99914401883158566" - ] - }, - "execution_count": 133, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "y_pred = dnn_clf.predict(X_train1)\n", "accuracy_score(y_train1, y_pred)" @@ -5340,75 +3558,7 @@ "cell_type": "code", "execution_count": 134, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "0\tValidation loss: 0.162759\tBest loss: 0.162759\tAccuracy: 95.15%\n", - "1\tValidation loss: 0.120510\tBest loss: 0.120510\tAccuracy: 96.64%\n", - "2\tValidation loss: 0.110715\tBest loss: 0.110715\tAccuracy: 96.91%\n", - "3\tValidation loss: 0.104193\tBest loss: 0.104193\tAccuracy: 97.22%\n", - "4\tValidation loss: 0.103560\tBest loss: 0.103560\tAccuracy: 97.81%\n", - "5\tValidation loss: 0.087045\tBest loss: 0.087045\tAccuracy: 97.89%\n", - "6\tValidation loss: 0.087227\tBest loss: 0.087045\tAccuracy: 97.65%\n", - "7\tValidation loss: 0.079840\tBest loss: 0.079840\tAccuracy: 98.16%\n", - "8\tValidation loss: 0.083102\tBest loss: 0.079840\tAccuracy: 97.50%\n", - "9\tValidation loss: 0.076794\tBest loss: 0.076794\tAccuracy: 98.01%\n", - "10\tValidation loss: 0.074914\tBest loss: 0.074914\tAccuracy: 97.93%\n", - "11\tValidation loss: 0.073794\tBest loss: 0.073794\tAccuracy: 98.12%\n", - "12\tValidation loss: 0.079777\tBest loss: 0.073794\tAccuracy: 97.89%\n", - "13\tValidation loss: 0.080277\tBest loss: 0.073794\tAccuracy: 97.54%\n", - "14\tValidation loss: 0.072409\tBest loss: 0.072409\tAccuracy: 98.08%\n", - "15\tValidation loss: 0.071988\tBest loss: 0.071988\tAccuracy: 98.12%\n", - "16\tValidation loss: 0.074609\tBest loss: 0.071988\tAccuracy: 97.93%\n", - "17\tValidation loss: 0.069488\tBest loss: 0.069488\tAccuracy: 98.28%\n", - "18\tValidation loss: 0.080863\tBest loss: 0.069488\tAccuracy: 98.40%\n", - "19\tValidation loss: 0.074966\tBest loss: 0.069488\tAccuracy: 98.20%\n", - "20\tValidation loss: 0.071082\tBest loss: 0.069488\tAccuracy: 98.12%\n", - "21\tValidation loss: 0.070138\tBest loss: 0.069488\tAccuracy: 98.20%\n", - "22\tValidation loss: 0.066032\tBest loss: 0.066032\tAccuracy: 98.28%\n", - "23\tValidation loss: 0.061130\tBest loss: 0.061130\tAccuracy: 98.36%\n", - "24\tValidation loss: 0.067107\tBest loss: 0.061130\tAccuracy: 98.16%\n", - "25\tValidation loss: 0.071372\tBest loss: 0.061130\tAccuracy: 98.16%\n", - "26\tValidation loss: 0.068535\tBest loss: 0.061130\tAccuracy: 98.36%\n", - "27\tValidation loss: 0.065336\tBest loss: 0.061130\tAccuracy: 98.48%\n", - "28\tValidation loss: 0.066783\tBest loss: 0.061130\tAccuracy: 98.40%\n", - "29\tValidation loss: 0.092769\tBest loss: 0.061130\tAccuracy: 97.77%\n", - "30\tValidation loss: 0.075746\tBest loss: 0.061130\tAccuracy: 98.01%\n", - "31\tValidation loss: 0.084024\tBest loss: 0.061130\tAccuracy: 97.81%\n", - "32\tValidation loss: 0.116428\tBest loss: 0.061130\tAccuracy: 98.44%\n", - "33\tValidation loss: 0.079498\tBest loss: 0.061130\tAccuracy: 97.89%\n", - "34\tValidation loss: 0.078189\tBest loss: 0.061130\tAccuracy: 97.97%\n", - "35\tValidation loss: 0.083723\tBest loss: 0.061130\tAccuracy: 97.81%\n", - "36\tValidation loss: 0.088210\tBest loss: 0.061130\tAccuracy: 97.19%\n", - "37\tValidation loss: 0.080040\tBest loss: 0.061130\tAccuracy: 97.93%\n", - "38\tValidation loss: 0.086932\tBest loss: 0.061130\tAccuracy: 97.89%\n", - "39\tValidation loss: 0.240580\tBest loss: 0.061130\tAccuracy: 91.67%\n", - "40\tValidation loss: 0.166662\tBest loss: 0.061130\tAccuracy: 94.29%\n", - "41\tValidation loss: 0.125562\tBest loss: 0.061130\tAccuracy: 97.15%\n", - "42\tValidation loss: 0.124890\tBest loss: 0.061130\tAccuracy: 95.82%\n", - "43\tValidation loss: 0.127020\tBest loss: 0.061130\tAccuracy: 96.76%\n", - "44\tValidation loss: 0.121540\tBest loss: 0.061130\tAccuracy: 96.05%\n", - "Early stopping!\n" - ] - }, - { - "data": { - "text/plain": [ - "DNNClassifier(activation=.parametrized_leaky_relu at 0x7fd9b2368d08>,\n", - " batch_norm_momentum=None, batch_size=500, dropout_rate=0.5,\n", - " initializer=._initializer at 0x7fd9d5e628c8>,\n", - " learning_rate=0.01, n_hidden_layers=5, n_neurons=90,\n", - " optimizer_class=,\n", - " random_state=42)" - ] - }, - "execution_count": 134, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "dnn_clf_dropout = DNNClassifier(activation=leaky_relu(alpha=0.1), batch_size=500, learning_rate=0.01,\n", " n_neurons=90, random_state=42,\n", @@ -5434,18 +3584,7 @@ "cell_type": "code", "execution_count": 135, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0.98657326328079398" - ] - }, - "execution_count": 135, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "y_pred = dnn_clf_dropout.predict(X_test1)\n", "accuracy_score(y_test1, y_pred)" @@ -5462,255 +3601,7 @@ "cell_type": "code", "execution_count": 136, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Fitting 3 folds for each of 50 candidates, totalling 150 fits\n", - "[CV] dropout_rate=0.5, n_neurons=70, learning_rate=0.01, activation=, batch_size=100 \n", - "0\tValidation loss: 0.355079\tBest loss: 0.355079\tAccuracy: 91.44%\n", - "1\tValidation loss: 0.280624\tBest loss: 0.280624\tAccuracy: 94.10%\n", - "2\tValidation loss: 0.279819\tBest loss: 0.279819\tAccuracy: 92.77%\n", - "3\tValidation loss: 0.223614\tBest loss: 0.223614\tAccuracy: 94.10%\n", - "4\tValidation loss: 0.199802\tBest loss: 0.199802\tAccuracy: 95.11%\n", - "5\tValidation loss: 0.214481\tBest loss: 0.199802\tAccuracy: 95.47%\n", - "6\tValidation loss: 0.216195\tBest loss: 0.199802\tAccuracy: 95.78%\n", - "7\tValidation loss: 0.209172\tBest loss: 0.199802\tAccuracy: 94.80%\n", - "8\tValidation loss: 0.182841\tBest loss: 0.182841\tAccuracy: 95.70%\n", - "9\tValidation loss: 0.214252\tBest loss: 0.182841\tAccuracy: 95.82%\n", - "10\tValidation loss: 0.198762\tBest loss: 0.182841\tAccuracy: 95.62%\n", - "11\tValidation loss: 0.186415\tBest loss: 0.182841\tAccuracy: 95.82%\n", - "12\tValidation loss: 0.222924\tBest loss: 0.182841\tAccuracy: 96.05%\n", - "13\tValidation loss: 0.199636\tBest loss: 0.182841\tAccuracy: 95.97%\n", - "14\tValidation loss: 0.214436\tBest loss: 0.182841\tAccuracy: 95.97%\n", - "15\tValidation loss: 0.213507\tBest loss: 0.182841\tAccuracy: 95.47%\n", - "16\tValidation loss: 0.191497\tBest loss: 0.182841\tAccuracy: 95.78%\n", - "17\tValidation loss: 0.179503\tBest loss: 0.179503\tAccuracy: 95.93%\n", - "18\tValidation loss: 0.210343\tBest loss: 0.179503\tAccuracy: 95.74%\n", - "19\tValidation loss: 0.212626\tBest loss: 0.179503\tAccuracy: 95.27%\n", - "20\tValidation loss: 0.187110\tBest loss: 0.179503\tAccuracy: 96.09%\n", - "21\tValidation loss: 0.175171\tBest loss: 0.175171\tAccuracy: 95.78%\n", - "22\tValidation loss: 0.217172\tBest loss: 0.175171\tAccuracy: 95.66%\n", - "23\tValidation loss: 0.181060\tBest loss: 0.175171\tAccuracy: 96.44%\n", - "24\tValidation loss: 0.163630\tBest loss: 0.163630\tAccuracy: 95.93%\n", - "25\tValidation loss: 0.225873\tBest loss: 0.163630\tAccuracy: 95.58%\n", - "26\tValidation loss: 0.204975\tBest loss: 0.163630\tAccuracy: 95.66%\n", - "27\tValidation loss: 0.183588\tBest loss: 0.163630\tAccuracy: 95.97%\n", - "28\tValidation loss: 0.231080\tBest loss: 0.163630\tAccuracy: 95.11%\n", - "29\tValidation loss: 0.204342\tBest loss: 0.163630\tAccuracy: 95.74%\n", - "30\tValidation loss: 0.183963\tBest loss: 0.163630\tAccuracy: 95.93%\n", - "31\tValidation loss: 0.200975\tBest loss: 0.163630\tAccuracy: 95.23%\n", - "32\tValidation loss: 0.211165\tBest loss: 0.163630\tAccuracy: 95.23%\n", - "33\tValidation loss: 0.217777\tBest loss: 0.163630\tAccuracy: 95.07%\n", - "34\tValidation loss: 0.193184\tBest loss: 0.163630\tAccuracy: 95.39%\n", - "35\tValidation loss: 0.203809\tBest loss: 0.163630\tAccuracy: 95.58%\n", - "36\tValidation loss: 0.221673\tBest loss: 0.163630\tAccuracy: 94.57%\n", - "37\tValidation loss: 0.215750\tBest loss: 0.163630\tAccuracy: 95.39%\n", - "38\tValidation loss: 0.189653\tBest loss: 0.163630\tAccuracy: 96.09%\n", - "39\tValidation loss: 0.191333\tBest loss: 0.163630\tAccuracy: 95.19%\n", - "40\tValidation loss: 0.207714\tBest loss: 0.163630\tAccuracy: 96.01%\n", - "41\tValidation loss: 0.174490\tBest loss: 0.163630\tAccuracy: 95.39%\n", - "42\tValidation loss: 0.177445\tBest loss: 0.163630\tAccuracy: 95.82%\n", - "43\tValidation loss: 0.166708\tBest loss: 0.163630\tAccuracy: 96.09%\n", - "44\tValidation loss: 0.190829\tBest loss: 0.163630\tAccuracy: 95.70%\n", - "45\tValidation loss: 0.225985\tBest loss: 0.163630\tAccuracy: 96.25%\n", - "Early stopping!\n", - "[CV] dropout_rate=0.5, n_neurons=70, learning_rate=0.01, activation=, batch_size=100, total= 39.0s\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 39.1s remaining: 0.0s\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[CV] dropout_rate=0.5, n_neurons=70, learning_rate=0.01, activation=, batch_size=100 \n", - "0\tValidation loss: 0.748480\tBest loss: 0.748480\tAccuracy: 57.70%\n", - "1\tValidation loss: 0.516088\tBest loss: 0.516088\tAccuracy: 78.50%\n", - "2\tValidation loss: 0.448866\tBest loss: 0.448866\tAccuracy: 78.89%\n", - "3\tValidation loss: 0.435606\tBest loss: 0.435606\tAccuracy: 78.54%\n", - "4\tValidation loss: 0.435243\tBest loss: 0.435243\tAccuracy: 79.40%\n", - "5\tValidation loss: 0.450605\tBest loss: 0.435243\tAccuracy: 78.42%\n", - "6\tValidation loss: 0.430706\tBest loss: 0.430706\tAccuracy: 78.62%\n", - "7\tValidation loss: 0.449289\tBest loss: 0.430706\tAccuracy: 78.30%\n", - "8\tValidation loss: 0.413226\tBest loss: 0.413226\tAccuracy: 79.05%\n", - "9\tValidation loss: 0.436053\tBest loss: 0.413226\tAccuracy: 78.46%\n", - "10\tValidation loss: 0.459932\tBest loss: 0.413226\tAccuracy: 79.24%\n", - "11\tValidation loss: 0.424138\tBest loss: 0.413226\tAccuracy: 79.24%\n", - "12\tValidation loss: 0.409538\tBest loss: 0.409538\tAccuracy: 79.55%\n", - "13\tValidation loss: 0.416324\tBest loss: 0.409538\tAccuracy: 75.41%\n", - "14\tValidation loss: 0.440273\tBest loss: 0.409538\tAccuracy: 78.46%\n", - "15\tValidation loss: 0.435736\tBest loss: 0.409538\tAccuracy: 79.05%\n", - "16\tValidation loss: 0.428412\tBest loss: 0.409538\tAccuracy: 79.20%\n", - "17\tValidation loss: 0.450156\tBest loss: 0.409538\tAccuracy: 80.02%\n", - "18\tValidation loss: 0.421057\tBest loss: 0.409538\tAccuracy: 79.24%\n", - "19\tValidation loss: 0.442284\tBest loss: 0.409538\tAccuracy: 79.01%\n", - "20\tValidation loss: 0.426907\tBest loss: 0.409538\tAccuracy: 79.16%\n", - "21\tValidation loss: 0.439567\tBest loss: 0.409538\tAccuracy: 79.05%\n", - "22\tValidation loss: 0.452601\tBest loss: 0.409538\tAccuracy: 79.67%\n", - "23\tValidation loss: 0.424887\tBest loss: 0.409538\tAccuracy: 79.09%\n", - "24\tValidation loss: 0.441096\tBest loss: 0.409538\tAccuracy: 78.97%\n", - "25\tValidation loss: 0.417390\tBest loss: 0.409538\tAccuracy: 78.89%\n", - "26\tValidation loss: 0.418550\tBest loss: 0.409538\tAccuracy: 79.05%\n", - "27\tValidation loss: 0.426065\tBest loss: 0.409538\tAccuracy: 78.66%\n", - "28\tValidation loss: 0.413968\tBest loss: 0.409538\tAccuracy: 79.36%\n", - "29\tValidation loss: 0.425434\tBest loss: 0.409538\tAccuracy: 79.24%\n", - "30\tValidation loss: 0.455391\tBest loss: 0.409538\tAccuracy: 74.71%\n", - "31\tValidation loss: 0.429498\tBest loss: 0.409538\tAccuracy: 79.20%\n", - "32\tValidation loss: 0.427383\tBest loss: 0.409538\tAccuracy: 79.52%\n", - "33\tValidation loss: 0.422621\tBest loss: 0.409538\tAccuracy: 78.62%\n", - "Early stopping!\n", - "[CV] dropout_rate=0.5, n_neurons=70, learning_rate=0.01, activation=, batch_size=100, total= 27.4s\n", - "[CV] dropout_rate=0.5, n_neurons=70, learning_rate=0.01, activation=, batch_size=100 \n", - "0\tValidation loss: 0.497714\tBest loss: 0.497714\tAccuracy: 86.71%\n", - "1\tValidation loss: 0.248258\tBest loss: 0.248258\tAccuracy: 93.51%\n", - "2\tValidation loss: 0.279785\tBest loss: 0.248258\tAccuracy: 93.71%\n", - "3\tValidation loss: 0.248663\tBest loss: 0.248258\tAccuracy: 94.61%\n", - "4\tValidation loss: 0.269139\tBest loss: 0.248258\tAccuracy: 94.76%\n", - "5\tValidation loss: 0.188808\tBest loss: 0.188808\tAccuracy: 95.39%\n", - "6\tValidation loss: 0.196049\tBest loss: 0.188808\tAccuracy: 95.58%\n", - "7\tValidation loss: 0.204966\tBest loss: 0.188808\tAccuracy: 95.15%\n", - "8\tValidation loss: 0.238414\tBest loss: 0.188808\tAccuracy: 94.61%\n", - "9\tValidation loss: 0.192095\tBest loss: 0.188808\tAccuracy: 95.97%\n", - "[...and much later...]\n", - "19\tValidation loss: 1.939112\tBest loss: 1.619874\tAccuracy: 22.01%\n", - "20\tValidation loss: 1.825761\tBest loss: 1.619874\tAccuracy: 19.27%\n", - "21\tValidation loss: 1.732937\tBest loss: 1.619874\tAccuracy: 22.01%\n", - "22\tValidation loss: 1.832995\tBest loss: 1.619874\tAccuracy: 20.91%\n", - "23\tValidation loss: 1.659557\tBest loss: 1.619874\tAccuracy: 20.91%\n", - "24\tValidation loss: 1.828380\tBest loss: 1.619874\tAccuracy: 18.73%\n", - "25\tValidation loss: 1.719589\tBest loss: 1.619874\tAccuracy: 22.01%\n", - "26\tValidation loss: 1.842429\tBest loss: 1.619874\tAccuracy: 18.73%\n", - "27\tValidation loss: 1.717596\tBest loss: 1.619874\tAccuracy: 19.27%\n", - "28\tValidation loss: 1.863441\tBest loss: 1.619874\tAccuracy: 19.08%\n", - "29\tValidation loss: 1.952335\tBest loss: 1.619874\tAccuracy: 19.08%\n", - "30\tValidation loss: 1.853776\tBest loss: 1.619874\tAccuracy: 20.91%\n", - "31\tValidation loss: 1.894134\tBest loss: 1.619874\tAccuracy: 22.01%\n", - "32\tValidation loss: 1.711688\tBest loss: 1.619874\tAccuracy: 19.08%\n", - "33\tValidation loss: 1.651240\tBest loss: 1.619874\tAccuracy: 18.73%\n", - "34\tValidation loss: 1.760639\tBest loss: 1.619874\tAccuracy: 20.91%\n", - "35\tValidation loss: 1.667938\tBest loss: 1.619874\tAccuracy: 22.01%\n", - "36\tValidation loss: 1.641116\tBest loss: 1.619874\tAccuracy: 20.91%\n", - "37\tValidation loss: 1.694960\tBest loss: 1.619874\tAccuracy: 19.08%\n", - "38\tValidation loss: 1.816517\tBest loss: 1.619874\tAccuracy: 18.73%\n", - "39\tValidation loss: 1.647246\tBest loss: 1.619874\tAccuracy: 18.73%\n", - "Early stopping!\n", - "[CV] dropout_rate=0.5, n_neurons=140, learning_rate=0.05, activation=, batch_size=100, total= 1.0min\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[Parallel(n_jobs=1)]: Done 150 out of 150 | elapsed: 130.6min finished\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "0\tValidation loss: 0.165751\tBest loss: 0.165751\tAccuracy: 95.47%\n", - "1\tValidation loss: 0.111834\tBest loss: 0.111834\tAccuracy: 96.99%\n", - "2\tValidation loss: 0.102867\tBest loss: 0.102867\tAccuracy: 96.83%\n", - "3\tValidation loss: 0.089197\tBest loss: 0.089197\tAccuracy: 97.85%\n", - "4\tValidation loss: 0.093953\tBest loss: 0.089197\tAccuracy: 97.77%\n", - "5\tValidation loss: 0.079498\tBest loss: 0.079498\tAccuracy: 98.08%\n", - "6\tValidation loss: 0.081214\tBest loss: 0.079498\tAccuracy: 98.01%\n", - "7\tValidation loss: 0.086096\tBest loss: 0.079498\tAccuracy: 97.77%\n", - "8\tValidation loss: 0.074422\tBest loss: 0.074422\tAccuracy: 97.73%\n", - "9\tValidation loss: 0.079650\tBest loss: 0.074422\tAccuracy: 97.89%\n", - "10\tValidation loss: 0.077278\tBest loss: 0.074422\tAccuracy: 97.77%\n", - "11\tValidation loss: 0.077608\tBest loss: 0.074422\tAccuracy: 98.24%\n", - "12\tValidation loss: 0.074337\tBest loss: 0.074337\tAccuracy: 98.05%\n", - "13\tValidation loss: 0.066028\tBest loss: 0.066028\tAccuracy: 98.28%\n", - "14\tValidation loss: 0.072845\tBest loss: 0.066028\tAccuracy: 98.16%\n", - "15\tValidation loss: 0.066652\tBest loss: 0.066028\tAccuracy: 98.05%\n", - "16\tValidation loss: 0.065729\tBest loss: 0.065729\tAccuracy: 98.16%\n", - "17\tValidation loss: 0.061191\tBest loss: 0.061191\tAccuracy: 98.51%\n", - "18\tValidation loss: 0.062528\tBest loss: 0.061191\tAccuracy: 98.44%\n", - "19\tValidation loss: 0.065407\tBest loss: 0.061191\tAccuracy: 98.36%\n", - "20\tValidation loss: 0.065273\tBest loss: 0.061191\tAccuracy: 98.44%\n", - "21\tValidation loss: 0.061035\tBest loss: 0.061035\tAccuracy: 98.40%\n", - "22\tValidation loss: 0.056312\tBest loss: 0.056312\tAccuracy: 98.59%\n", - "23\tValidation loss: 0.069074\tBest loss: 0.056312\tAccuracy: 98.40%\n", - "24\tValidation loss: 0.057482\tBest loss: 0.056312\tAccuracy: 98.51%\n", - "25\tValidation loss: 0.068342\tBest loss: 0.056312\tAccuracy: 98.44%\n", - "26\tValidation loss: 0.063494\tBest loss: 0.056312\tAccuracy: 98.48%\n", - "27\tValidation loss: 0.057257\tBest loss: 0.056312\tAccuracy: 98.51%\n", - "28\tValidation loss: 0.058659\tBest loss: 0.056312\tAccuracy: 98.59%\n", - "29\tValidation loss: 0.059009\tBest loss: 0.056312\tAccuracy: 98.48%\n", - "30\tValidation loss: 0.058227\tBest loss: 0.056312\tAccuracy: 98.55%\n", - "31\tValidation loss: 0.062198\tBest loss: 0.056312\tAccuracy: 98.44%\n", - "32\tValidation loss: 0.058043\tBest loss: 0.056312\tAccuracy: 98.40%\n", - "33\tValidation loss: 0.055970\tBest loss: 0.055970\tAccuracy: 98.51%\n", - "34\tValidation loss: 0.060111\tBest loss: 0.055970\tAccuracy: 98.67%\n", - "35\tValidation loss: 0.058786\tBest loss: 0.055970\tAccuracy: 98.44%\n", - "36\tValidation loss: 0.059944\tBest loss: 0.055970\tAccuracy: 98.32%\n", - "37\tValidation loss: 0.058087\tBest loss: 0.055970\tAccuracy: 98.63%\n", - "38\tValidation loss: 0.063003\tBest loss: 0.055970\tAccuracy: 98.36%\n", - "39\tValidation loss: 0.052073\tBest loss: 0.052073\tAccuracy: 98.67%\n", - "40\tValidation loss: 0.058115\tBest loss: 0.052073\tAccuracy: 98.40%\n", - "41\tValidation loss: 0.059997\tBest loss: 0.052073\tAccuracy: 98.63%\n", - "42\tValidation loss: 0.052416\tBest loss: 0.052073\tAccuracy: 98.75%\n", - "43\tValidation loss: 0.053840\tBest loss: 0.052073\tAccuracy: 98.59%\n", - "44\tValidation loss: 0.054563\tBest loss: 0.052073\tAccuracy: 98.67%\n", - "45\tValidation loss: 0.049410\tBest loss: 0.049410\tAccuracy: 98.55%\n", - "46\tValidation loss: 0.057060\tBest loss: 0.049410\tAccuracy: 98.24%\n", - "47\tValidation loss: 0.062434\tBest loss: 0.049410\tAccuracy: 98.48%\n", - "48\tValidation loss: 0.054523\tBest loss: 0.049410\tAccuracy: 98.59%\n", - "49\tValidation loss: 0.052774\tBest loss: 0.049410\tAccuracy: 98.36%\n", - "50\tValidation loss: 0.056562\tBest loss: 0.049410\tAccuracy: 98.32%\n", - "51\tValidation loss: 0.060280\tBest loss: 0.049410\tAccuracy: 98.51%\n", - "52\tValidation loss: 0.055685\tBest loss: 0.049410\tAccuracy: 98.55%\n", - "53\tValidation loss: 0.056077\tBest loss: 0.049410\tAccuracy: 98.44%\n", - "54\tValidation loss: 0.057951\tBest loss: 0.049410\tAccuracy: 98.44%\n", - "55\tValidation loss: 0.056315\tBest loss: 0.049410\tAccuracy: 98.75%\n", - "56\tValidation loss: 0.055744\tBest loss: 0.049410\tAccuracy: 98.55%\n", - "57\tValidation loss: 0.054228\tBest loss: 0.049410\tAccuracy: 98.48%\n", - "58\tValidation loss: 0.057836\tBest loss: 0.049410\tAccuracy: 98.71%\n", - "59\tValidation loss: 0.053361\tBest loss: 0.049410\tAccuracy: 98.71%\n", - "60\tValidation loss: 0.056389\tBest loss: 0.049410\tAccuracy: 98.48%\n", - "61\tValidation loss: 0.061350\tBest loss: 0.049410\tAccuracy: 98.48%\n", - "62\tValidation loss: 0.052135\tBest loss: 0.049410\tAccuracy: 98.67%\n", - "63\tValidation loss: 0.053853\tBest loss: 0.049410\tAccuracy: 98.48%\n", - "64\tValidation loss: 0.056641\tBest loss: 0.049410\tAccuracy: 98.71%\n", - "65\tValidation loss: 0.052790\tBest loss: 0.049410\tAccuracy: 98.63%\n", - "66\tValidation loss: 0.053514\tBest loss: 0.049410\tAccuracy: 98.44%\n", - "Early stopping!\n" - ] - }, - { - "data": { - "text/plain": [ - "RandomizedSearchCV(cv=None, error_score='raise',\n", - " estimator=DNNClassifier(activation=,\n", - " batch_norm_momentum=None, batch_size=20, dropout_rate=None,\n", - " initializer=._initializer at 0x7fd9d5e628c8>,\n", - " learning_rate=0.01, n_hidden_layers=5, n_neurons=100,\n", - " optimizer_class=,\n", - " random_state=42),\n", - " fit_params={'y_valid': array([0, 4, ..., 1, 2], dtype=uint8), 'X_valid': array([[ 0., 0., ..., 0., 0.],\n", - " [ 0., 0., ..., 0., 0.],\n", - " ...,\n", - " [ 0., 0., ..., 0., 0.],\n", - " [ 0., 0., ..., 0., 0.]], dtype=float32), 'n_epochs': 1000},\n", - " iid=True, n_iter=50, n_jobs=1,\n", - " param_distributions={'dropout_rate': [0.2, 0.3, 0.4, 0.5, 0.6], 'n_neurons': [10, 30, 50, 70, 90, 100, 120, 140, 160], 'learning_rate': [0.01, 0.02, 0.05, 0.1], 'activation': [, , .parametrized_leaky_relu at 0x7fd9b2368950>, .parametrized_leaky_relu at 0x7fd9b23687b8>], 'batch_size': [10, 50, 100, 500]},\n", - " pre_dispatch='2*n_jobs', random_state=42, refit=True,\n", - " return_train_score=True, scoring=None, verbose=2)" - ] - }, - "execution_count": 136, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "from sklearn.model_selection import RandomizedSearchCV\n", "\n", @@ -5735,22 +3626,7 @@ "cell_type": "code", "execution_count": 137, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'activation': .parametrized_leaky_relu>,\n", - " 'batch_size': 500,\n", - " 'dropout_rate': 0.4,\n", - " 'learning_rate': 0.01,\n", - " 'n_neurons': 50}" - ] - }, - "execution_count": 137, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "rnd_search_dropout.best_params_" ] @@ -5759,18 +3635,7 @@ "cell_type": "code", "execution_count": 138, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0.98812998637867289" - ] - }, - "execution_count": 138, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "y_pred = rnd_search_dropout.predict(X_test1)\n", "accuracy_score(y_test1, y_pred)" @@ -5963,68 +3828,7 @@ "cell_type": "code", "execution_count": 145, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "INFO:tensorflow:Restoring parameters from ./my_best_mnist_model_0_to_4\n", - "0\tValidation loss: 0.967851\tBest loss: 0.967851\tAccuracy: 67.33%\n", - "1\tValidation loss: 0.861747\tBest loss: 0.861747\tAccuracy: 71.33%\n", - "2\tValidation loss: 0.777535\tBest loss: 0.777535\tAccuracy: 72.00%\n", - "3\tValidation loss: 0.699915\tBest loss: 0.699915\tAccuracy: 75.33%\n", - "4\tValidation loss: 0.786714\tBest loss: 0.699915\tAccuracy: 78.00%\n", - "5\tValidation loss: 0.735406\tBest loss: 0.699915\tAccuracy: 76.67%\n", - "6\tValidation loss: 0.732264\tBest loss: 0.699915\tAccuracy: 78.00%\n", - "7\tValidation loss: 0.691741\tBest loss: 0.691741\tAccuracy: 76.00%\n", - "8\tValidation loss: 0.672757\tBest loss: 0.672757\tAccuracy: 80.00%\n", - "9\tValidation loss: 0.666520\tBest loss: 0.666520\tAccuracy: 80.00%\n", - "10\tValidation loss: 0.639375\tBest loss: 0.639375\tAccuracy: 81.33%\n", - "11\tValidation loss: 0.645089\tBest loss: 0.639375\tAccuracy: 82.00%\n", - "12\tValidation loss: 0.646768\tBest loss: 0.639375\tAccuracy: 80.00%\n", - "13\tValidation loss: 0.623784\tBest loss: 0.623784\tAccuracy: 82.67%\n", - "14\tValidation loss: 0.663026\tBest loss: 0.623784\tAccuracy: 80.00%\n", - "15\tValidation loss: 0.704513\tBest loss: 0.623784\tAccuracy: 79.33%\n", - "16\tValidation loss: 0.684003\tBest loss: 0.623784\tAccuracy: 79.33%\n", - "17\tValidation loss: 0.658575\tBest loss: 0.623784\tAccuracy: 82.67%\n", - "18\tValidation loss: 0.669875\tBest loss: 0.623784\tAccuracy: 79.33%\n", - "19\tValidation loss: 0.664581\tBest loss: 0.623784\tAccuracy: 78.67%\n", - "20\tValidation loss: 0.653490\tBest loss: 0.623784\tAccuracy: 80.00%\n", - "21\tValidation loss: 0.707304\tBest loss: 0.623784\tAccuracy: 79.33%\n", - "22\tValidation loss: 0.706012\tBest loss: 0.623784\tAccuracy: 80.67%\n", - "23\tValidation loss: 0.681227\tBest loss: 0.623784\tAccuracy: 78.67%\n", - "24\tValidation loss: 0.786823\tBest loss: 0.623784\tAccuracy: 78.00%\n", - "25\tValidation loss: 0.686110\tBest loss: 0.623784\tAccuracy: 79.33%\n", - "26\tValidation loss: 0.675166\tBest loss: 0.623784\tAccuracy: 82.67%\n", - "27\tValidation loss: 0.667711\tBest loss: 0.623784\tAccuracy: 82.67%\n", - "28\tValidation loss: 0.612220\tBest loss: 0.612220\tAccuracy: 83.33%\n", - "29\tValidation loss: 0.701196\tBest loss: 0.612220\tAccuracy: 78.00%\n", - "30\tValidation loss: 0.687806\tBest loss: 0.612220\tAccuracy: 81.33%\n", - "31\tValidation loss: 0.776596\tBest loss: 0.612220\tAccuracy: 79.33%\n", - "32\tValidation loss: 0.674172\tBest loss: 0.612220\tAccuracy: 80.00%\n", - "33\tValidation loss: 0.719044\tBest loss: 0.612220\tAccuracy: 83.33%\n", - "34\tValidation loss: 0.856403\tBest loss: 0.612220\tAccuracy: 74.00%\n", - "35\tValidation loss: 0.744627\tBest loss: 0.612220\tAccuracy: 80.00%\n", - "36\tValidation loss: 0.779348\tBest loss: 0.612220\tAccuracy: 78.00%\n", - "37\tValidation loss: 0.763777\tBest loss: 0.612220\tAccuracy: 78.00%\n", - "38\tValidation loss: 0.727376\tBest loss: 0.612220\tAccuracy: 78.00%\n", - "39\tValidation loss: 0.823514\tBest loss: 0.612220\tAccuracy: 78.00%\n", - "40\tValidation loss: 0.725053\tBest loss: 0.612220\tAccuracy: 80.67%\n", - "41\tValidation loss: 0.678497\tBest loss: 0.612220\tAccuracy: 80.67%\n", - "42\tValidation loss: 0.709977\tBest loss: 0.612220\tAccuracy: 80.67%\n", - "43\tValidation loss: 0.737200\tBest loss: 0.612220\tAccuracy: 77.33%\n", - "44\tValidation loss: 0.757937\tBest loss: 0.612220\tAccuracy: 77.33%\n", - "45\tValidation loss: 0.732024\tBest loss: 0.612220\tAccuracy: 80.00%\n", - "46\tValidation loss: 0.756428\tBest loss: 0.612220\tAccuracy: 80.67%\n", - "47\tValidation loss: 0.757610\tBest loss: 0.612220\tAccuracy: 78.67%\n", - "48\tValidation loss: 0.844137\tBest loss: 0.612220\tAccuracy: 80.00%\n", - "Early stopping!\n", - "Total training time: 2.3s\n", - "INFO:tensorflow:Restoring parameters from ./my_mnist_model_5_to_9_five_frozen\n", - "Final test accuracy: 76.30%\n" - ] - } - ], + "outputs": [], "source": [ "import time\n", "\n", @@ -6120,54 +3924,7 @@ "cell_type": "code", "execution_count": 147, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "INFO:tensorflow:Restoring parameters from ./my_best_mnist_model_0_to_4\n", - "0\tValidation loss: 1.109053\tBest loss: 1.109053\tAccuracy: 60.67%\n", - "1\tValidation loss: 0.813156\tBest loss: 0.813156\tAccuracy: 72.00%\n", - "2\tValidation loss: 0.755930\tBest loss: 0.755930\tAccuracy: 76.67%\n", - "3\tValidation loss: 0.744004\tBest loss: 0.744004\tAccuracy: 74.67%\n", - "4\tValidation loss: 0.685080\tBest loss: 0.685080\tAccuracy: 78.00%\n", - "5\tValidation loss: 0.702316\tBest loss: 0.685080\tAccuracy: 78.00%\n", - "6\tValidation loss: 0.646487\tBest loss: 0.646487\tAccuracy: 80.00%\n", - "7\tValidation loss: 0.686437\tBest loss: 0.646487\tAccuracy: 79.33%\n", - "8\tValidation loss: 0.750047\tBest loss: 0.646487\tAccuracy: 79.33%\n", - "9\tValidation loss: 0.688554\tBest loss: 0.646487\tAccuracy: 79.33%\n", - "10\tValidation loss: 0.785184\tBest loss: 0.646487\tAccuracy: 78.67%\n", - "11\tValidation loss: 0.634506\tBest loss: 0.634506\tAccuracy: 80.67%\n", - "12\tValidation loss: 0.656797\tBest loss: 0.634506\tAccuracy: 81.33%\n", - "13\tValidation loss: 0.645497\tBest loss: 0.634506\tAccuracy: 81.33%\n", - "14\tValidation loss: 0.618038\tBest loss: 0.618038\tAccuracy: 83.33%\n", - "15\tValidation loss: 0.641752\tBest loss: 0.618038\tAccuracy: 78.67%\n", - "16\tValidation loss: 0.645671\tBest loss: 0.618038\tAccuracy: 80.67%\n", - "17\tValidation loss: 0.654640\tBest loss: 0.618038\tAccuracy: 82.00%\n", - "18\tValidation loss: 0.670569\tBest loss: 0.618038\tAccuracy: 79.33%\n", - "19\tValidation loss: 0.670985\tBest loss: 0.618038\tAccuracy: 82.00%\n", - "20\tValidation loss: 0.659538\tBest loss: 0.618038\tAccuracy: 82.67%\n", - "21\tValidation loss: 0.622648\tBest loss: 0.618038\tAccuracy: 83.33%\n", - "22\tValidation loss: 0.736155\tBest loss: 0.618038\tAccuracy: 79.33%\n", - "23\tValidation loss: 0.739367\tBest loss: 0.618038\tAccuracy: 76.67%\n", - "24\tValidation loss: 0.699710\tBest loss: 0.618038\tAccuracy: 78.00%\n", - "25\tValidation loss: 0.709630\tBest loss: 0.618038\tAccuracy: 81.33%\n", - "26\tValidation loss: 0.692474\tBest loss: 0.618038\tAccuracy: 79.33%\n", - "27\tValidation loss: 0.807931\tBest loss: 0.618038\tAccuracy: 77.33%\n", - "28\tValidation loss: 0.676134\tBest loss: 0.618038\tAccuracy: 82.00%\n", - "29\tValidation loss: 0.738905\tBest loss: 0.618038\tAccuracy: 79.33%\n", - "30\tValidation loss: 0.664826\tBest loss: 0.618038\tAccuracy: 81.33%\n", - "31\tValidation loss: 0.694714\tBest loss: 0.618038\tAccuracy: 80.00%\n", - "32\tValidation loss: 0.739238\tBest loss: 0.618038\tAccuracy: 83.33%\n", - "33\tValidation loss: 0.697210\tBest loss: 0.618038\tAccuracy: 80.00%\n", - "34\tValidation loss: 0.817373\tBest loss: 0.618038\tAccuracy: 79.33%\n", - "Early stopping!\n", - "Total training time: 0.9s\n", - "INFO:tensorflow:Restoring parameters from ./my_mnist_model_5_to_9_five_frozen\n", - "Final test accuracy: 76.51%\n" - ] - } - ], + "outputs": [], "source": [ "import time\n", "\n", @@ -6299,71 +4056,7 @@ "cell_type": "code", "execution_count": 150, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "INFO:tensorflow:Restoring parameters from ./my_best_mnist_model_0_to_4\n", - "0\tValidation loss: 0.923460\tBest loss: 0.923460\tAccuracy: 69.33%\n", - "1\tValidation loss: 0.796192\tBest loss: 0.796192\tAccuracy: 77.33%\n", - "2\tValidation loss: 0.812068\tBest loss: 0.796192\tAccuracy: 78.67%\n", - "3\tValidation loss: 0.697938\tBest loss: 0.697938\tAccuracy: 80.67%\n", - "4\tValidation loss: 0.877122\tBest loss: 0.697938\tAccuracy: 74.67%\n", - "5\tValidation loss: 0.708524\tBest loss: 0.697938\tAccuracy: 81.33%\n", - "6\tValidation loss: 0.689500\tBest loss: 0.689500\tAccuracy: 84.00%\n", - "7\tValidation loss: 0.758315\tBest loss: 0.689500\tAccuracy: 81.33%\n", - "8\tValidation loss: 0.711138\tBest loss: 0.689500\tAccuracy: 78.67%\n", - "9\tValidation loss: 0.687304\tBest loss: 0.687304\tAccuracy: 81.33%\n", - "10\tValidation loss: 0.639222\tBest loss: 0.639222\tAccuracy: 81.33%\n", - "11\tValidation loss: 0.716750\tBest loss: 0.639222\tAccuracy: 82.67%\n", - "12\tValidation loss: 0.693442\tBest loss: 0.639222\tAccuracy: 80.67%\n", - "13\tValidation loss: 0.727682\tBest loss: 0.639222\tAccuracy: 84.00%\n", - "14\tValidation loss: 0.637289\tBest loss: 0.637289\tAccuracy: 84.67%\n", - "15\tValidation loss: 0.741304\tBest loss: 0.637289\tAccuracy: 83.33%\n", - "16\tValidation loss: 0.651895\tBest loss: 0.637289\tAccuracy: 82.67%\n", - "17\tValidation loss: 0.641192\tBest loss: 0.637289\tAccuracy: 80.67%\n", - "18\tValidation loss: 0.690386\tBest loss: 0.637289\tAccuracy: 80.67%\n", - "19\tValidation loss: 0.648541\tBest loss: 0.637289\tAccuracy: 82.67%\n", - "20\tValidation loss: 0.779663\tBest loss: 0.637289\tAccuracy: 83.33%\n", - "21\tValidation loss: 0.768834\tBest loss: 0.637289\tAccuracy: 82.67%\n", - "22\tValidation loss: 0.706279\tBest loss: 0.637289\tAccuracy: 82.67%\n", - "23\tValidation loss: 0.745840\tBest loss: 0.637289\tAccuracy: 82.00%\n", - "24\tValidation loss: 0.740068\tBest loss: 0.637289\tAccuracy: 83.33%\n", - "25\tValidation loss: 0.604927\tBest loss: 0.604927\tAccuracy: 84.67%\n", - "26\tValidation loss: 0.635410\tBest loss: 0.604927\tAccuracy: 82.00%\n", - "27\tValidation loss: 0.776003\tBest loss: 0.604927\tAccuracy: 82.67%\n", - "28\tValidation loss: 0.621502\tBest loss: 0.604927\tAccuracy: 82.00%\n", - "29\tValidation loss: 0.695963\tBest loss: 0.604927\tAccuracy: 83.33%\n", - "30\tValidation loss: 0.668194\tBest loss: 0.604927\tAccuracy: 84.67%\n", - "31\tValidation loss: 0.768975\tBest loss: 0.604927\tAccuracy: 82.67%\n", - "32\tValidation loss: 0.594731\tBest loss: 0.594731\tAccuracy: 84.00%\n", - "33\tValidation loss: 0.665088\tBest loss: 0.594731\tAccuracy: 84.00%\n", - "34\tValidation loss: 0.716284\tBest loss: 0.594731\tAccuracy: 81.33%\n", - "35\tValidation loss: 0.782680\tBest loss: 0.594731\tAccuracy: 84.00%\n", - "36\tValidation loss: 0.816441\tBest loss: 0.594731\tAccuracy: 84.00%\n", - "37\tValidation loss: 0.749341\tBest loss: 0.594731\tAccuracy: 84.00%\n", - "38\tValidation loss: 0.728754\tBest loss: 0.594731\tAccuracy: 82.00%\n", - "39\tValidation loss: 0.838166\tBest loss: 0.594731\tAccuracy: 84.00%\n", - "40\tValidation loss: 0.714871\tBest loss: 0.594731\tAccuracy: 84.00%\n", - "41\tValidation loss: 0.765463\tBest loss: 0.594731\tAccuracy: 84.67%\n", - "42\tValidation loss: 0.744043\tBest loss: 0.594731\tAccuracy: 82.00%\n", - "43\tValidation loss: 0.726922\tBest loss: 0.594731\tAccuracy: 83.33%\n", - "44\tValidation loss: 0.641118\tBest loss: 0.594731\tAccuracy: 82.67%\n", - "45\tValidation loss: 0.657861\tBest loss: 0.594731\tAccuracy: 84.00%\n", - "46\tValidation loss: 0.803642\tBest loss: 0.594731\tAccuracy: 86.00%\n", - "47\tValidation loss: 0.754644\tBest loss: 0.594731\tAccuracy: 84.67%\n", - "48\tValidation loss: 0.865141\tBest loss: 0.594731\tAccuracy: 84.00%\n", - "49\tValidation loss: 0.709169\tBest loss: 0.594731\tAccuracy: 84.67%\n", - "50\tValidation loss: 0.723139\tBest loss: 0.594731\tAccuracy: 84.00%\n", - "51\tValidation loss: 0.745109\tBest loss: 0.594731\tAccuracy: 84.67%\n", - "52\tValidation loss: 0.803908\tBest loss: 0.594731\tAccuracy: 82.67%\n", - "Early stopping!\n", - "INFO:tensorflow:Restoring parameters from ./my_mnist_model_5_to_9_four_frozen\n", - "Final test accuracy: 80.17%\n" - ] - } - ], + "outputs": [], "source": [ "n_epochs = 1000\n", "batch_size = 20\n", @@ -6443,43 +4136,7 @@ "cell_type": "code", "execution_count": 152, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "INFO:tensorflow:Restoring parameters from ./my_mnist_model_5_to_9_four_frozen\n", - "0\tValidation loss: 0.880485\tBest loss: 0.880485\tAccuracy: 86.00%\n", - "1\tValidation loss: 1.388974\tBest loss: 0.880485\tAccuracy: 81.33%\n", - "2\tValidation loss: 0.741543\tBest loss: 0.741543\tAccuracy: 86.67%\n", - "3\tValidation loss: 1.030772\tBest loss: 0.741543\tAccuracy: 84.00%\n", - "4\tValidation loss: 0.699438\tBest loss: 0.699438\tAccuracy: 87.33%\n", - "5\tValidation loss: 0.743930\tBest loss: 0.699438\tAccuracy: 89.33%\n", - "6\tValidation loss: 1.711346\tBest loss: 0.699438\tAccuracy: 82.67%\n", - "7\tValidation loss: 1.437762\tBest loss: 0.699438\tAccuracy: 82.00%\n", - "8\tValidation loss: 0.829231\tBest loss: 0.699438\tAccuracy: 86.67%\n", - "9\tValidation loss: 1.033920\tBest loss: 0.699438\tAccuracy: 86.67%\n", - "10\tValidation loss: 1.055709\tBest loss: 0.699438\tAccuracy: 87.33%\n", - "11\tValidation loss: 0.971796\tBest loss: 0.699438\tAccuracy: 88.00%\n", - "12\tValidation loss: 0.801815\tBest loss: 0.699438\tAccuracy: 86.00%\n", - "13\tValidation loss: 0.726146\tBest loss: 0.699438\tAccuracy: 89.33%\n", - "14\tValidation loss: 0.757217\tBest loss: 0.699438\tAccuracy: 88.67%\n", - "15\tValidation loss: 0.791842\tBest loss: 0.699438\tAccuracy: 90.00%\n", - "16\tValidation loss: 0.732507\tBest loss: 0.699438\tAccuracy: 90.67%\n", - "17\tValidation loss: 0.737297\tBest loss: 0.699438\tAccuracy: 90.67%\n", - "18\tValidation loss: 0.746715\tBest loss: 0.699438\tAccuracy: 90.00%\n", - "19\tValidation loss: 0.747751\tBest loss: 0.699438\tAccuracy: 90.00%\n", - "20\tValidation loss: 0.749325\tBest loss: 0.699438\tAccuracy: 90.00%\n", - "21\tValidation loss: 0.751899\tBest loss: 0.699438\tAccuracy: 90.00%\n", - "22\tValidation loss: 0.754314\tBest loss: 0.699438\tAccuracy: 90.00%\n", - "23\tValidation loss: 0.757840\tBest loss: 0.699438\tAccuracy: 90.00%\n", - "24\tValidation loss: 0.761543\tBest loss: 0.699438\tAccuracy: 90.00%\n", - "Early stopping!\n", - "INFO:tensorflow:Restoring parameters from ./my_mnist_model_5_to_9_two_frozen\n", - "Final test accuracy: 84.37%\n" - ] - } - ], + "outputs": [], "source": [ "n_epochs = 1000\n", "batch_size = 20\n", @@ -6544,52 +4201,7 @@ "cell_type": "code", "execution_count": 154, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "INFO:tensorflow:Restoring parameters from ./my_mnist_model_5_to_9_two_frozen\n", - "0\tValidation loss: 0.846005\tBest loss: 0.846005\tAccuracy: 83.33%\n", - "1\tValidation loss: 0.694439\tBest loss: 0.694439\tAccuracy: 91.33%\n", - "2\tValidation loss: 1.201433\tBest loss: 0.694439\tAccuracy: 85.33%\n", - "3\tValidation loss: 1.975297\tBest loss: 0.694439\tAccuracy: 85.33%\n", - "4\tValidation loss: 0.692805\tBest loss: 0.692805\tAccuracy: 95.33%\n", - "5\tValidation loss: 1.090217\tBest loss: 0.692805\tAccuracy: 91.33%\n", - "6\tValidation loss: 1.924300\tBest loss: 0.692805\tAccuracy: 90.67%\n", - "7\tValidation loss: 4.019310\tBest loss: 0.692805\tAccuracy: 87.33%\n", - "8\tValidation loss: 4.150792\tBest loss: 0.692805\tAccuracy: 78.00%\n", - "9\tValidation loss: 4.522708\tBest loss: 0.692805\tAccuracy: 75.33%\n", - "10\tValidation loss: 1.163385\tBest loss: 0.692805\tAccuracy: 90.00%\n", - "11\tValidation loss: 0.655868\tBest loss: 0.655868\tAccuracy: 92.67%\n", - "12\tValidation loss: 0.943888\tBest loss: 0.655868\tAccuracy: 92.67%\n", - "13\tValidation loss: 0.529996\tBest loss: 0.529996\tAccuracy: 92.67%\n", - "14\tValidation loss: 0.610578\tBest loss: 0.529996\tAccuracy: 94.67%\n", - "15\tValidation loss: 3.899716\tBest loss: 0.529996\tAccuracy: 88.00%\n", - "16\tValidation loss: 18.285717\tBest loss: 0.529996\tAccuracy: 86.67%\n", - "17\tValidation loss: 23.169626\tBest loss: 0.529996\tAccuracy: 78.00%\n", - "18\tValidation loss: 17.309252\tBest loss: 0.529996\tAccuracy: 90.00%\n", - "19\tValidation loss: 44.261902\tBest loss: 0.529996\tAccuracy: 80.00%\n", - "20\tValidation loss: 52.460327\tBest loss: 0.529996\tAccuracy: 80.00%\n", - "21\tValidation loss: 26.318949\tBest loss: 0.529996\tAccuracy: 83.33%\n", - "22\tValidation loss: 32.857723\tBest loss: 0.529996\tAccuracy: 90.67%\n", - "23\tValidation loss: 53.359497\tBest loss: 0.529996\tAccuracy: 88.00%\n", - "24\tValidation loss: 57.823742\tBest loss: 0.529996\tAccuracy: 88.00%\n", - "25\tValidation loss: 37.154972\tBest loss: 0.529996\tAccuracy: 92.67%\n", - "26\tValidation loss: 41.386772\tBest loss: 0.529996\tAccuracy: 90.00%\n", - "27\tValidation loss: 43.486767\tBest loss: 0.529996\tAccuracy: 90.00%\n", - "28\tValidation loss: 42.776855\tBest loss: 0.529996\tAccuracy: 88.67%\n", - "29\tValidation loss: 43.368839\tBest loss: 0.529996\tAccuracy: 90.67%\n", - "30\tValidation loss: 43.440975\tBest loss: 0.529996\tAccuracy: 90.00%\n", - "31\tValidation loss: 42.889927\tBest loss: 0.529996\tAccuracy: 91.33%\n", - "32\tValidation loss: 42.806690\tBest loss: 0.529996\tAccuracy: 90.67%\n", - "33\tValidation loss: 42.784145\tBest loss: 0.529996\tAccuracy: 90.67%\n", - "Early stopping!\n", - "INFO:tensorflow:Restoring parameters from ./my_mnist_model_5_to_9_no_frozen\n", - "Final test accuracy: 90.60%\n" - ] - } - ], + "outputs": [], "source": [ "n_epochs = 1000\n", "batch_size = 20\n", @@ -6637,61 +4249,7 @@ "cell_type": "code", "execution_count": 155, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "0\tValidation loss: 0.803557\tBest loss: 0.803557\tAccuracy: 71.33%\n", - "1\tValidation loss: 0.966741\tBest loss: 0.803557\tAccuracy: 85.33%\n", - "2\tValidation loss: 1.158972\tBest loss: 0.803557\tAccuracy: 78.00%\n", - "3\tValidation loss: 0.615960\tBest loss: 0.615960\tAccuracy: 88.00%\n", - "4\tValidation loss: 0.612626\tBest loss: 0.612626\tAccuracy: 92.00%\n", - "5\tValidation loss: 0.686420\tBest loss: 0.612626\tAccuracy: 89.33%\n", - "6\tValidation loss: 0.805281\tBest loss: 0.612626\tAccuracy: 89.33%\n", - "7\tValidation loss: 0.753108\tBest loss: 0.612626\tAccuracy: 88.67%\n", - "8\tValidation loss: 1.051471\tBest loss: 0.612626\tAccuracy: 86.00%\n", - "9\tValidation loss: 0.487089\tBest loss: 0.487089\tAccuracy: 93.33%\n", - "10\tValidation loss: 1.191093\tBest loss: 0.487089\tAccuracy: 85.33%\n", - "11\tValidation loss: 0.878905\tBest loss: 0.487089\tAccuracy: 88.67%\n", - "12\tValidation loss: 0.768841\tBest loss: 0.487089\tAccuracy: 91.33%\n", - "13\tValidation loss: 1.153907\tBest loss: 0.487089\tAccuracy: 90.67%\n", - "14\tValidation loss: 0.985427\tBest loss: 0.487089\tAccuracy: 89.33%\n", - "15\tValidation loss: 1.221879\tBest loss: 0.487089\tAccuracy: 85.33%\n", - "16\tValidation loss: 0.961743\tBest loss: 0.487089\tAccuracy: 88.67%\n", - "17\tValidation loss: 3.116057\tBest loss: 0.487089\tAccuracy: 84.00%\n", - "18\tValidation loss: 0.686387\tBest loss: 0.487089\tAccuracy: 84.00%\n", - "19\tValidation loss: 0.929801\tBest loss: 0.487089\tAccuracy: 88.00%\n", - "20\tValidation loss: 1.137579\tBest loss: 0.487089\tAccuracy: 92.00%\n", - "21\tValidation loss: 0.987261\tBest loss: 0.487089\tAccuracy: 91.33%\n", - "22\tValidation loss: 2.030677\tBest loss: 0.487089\tAccuracy: 91.33%\n", - "23\tValidation loss: 1.094184\tBest loss: 0.487089\tAccuracy: 92.00%\n", - "24\tValidation loss: 1.332256\tBest loss: 0.487089\tAccuracy: 82.67%\n", - "25\tValidation loss: 1.128633\tBest loss: 0.487089\tAccuracy: 85.33%\n", - "26\tValidation loss: 0.866569\tBest loss: 0.487089\tAccuracy: 90.67%\n", - "27\tValidation loss: 1.088500\tBest loss: 0.487089\tAccuracy: 89.33%\n", - "28\tValidation loss: 1.146113\tBest loss: 0.487089\tAccuracy: 89.33%\n", - "29\tValidation loss: 1.163180\tBest loss: 0.487089\tAccuracy: 89.33%\n", - "30\tValidation loss: 1.154797\tBest loss: 0.487089\tAccuracy: 89.33%\n", - "Early stopping!\n" - ] - }, - { - "data": { - "text/plain": [ - "DNNClassifier(activation=,\n", - " batch_norm_momentum=None, batch_size=20, dropout_rate=None,\n", - " initializer=._initializer at 0x7fd9d5e628c8>,\n", - " learning_rate=0.01, n_hidden_layers=4, n_neurons=100,\n", - " optimizer_class=,\n", - " random_state=42)" - ] - }, - "execution_count": 155, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "dnn_clf_5_to_9 = DNNClassifier(n_hidden_layers=4, random_state=42)\n", "dnn_clf_5_to_9.fit(X_train2, y_train2, n_epochs=1000, X_valid=X_valid2, y_valid=y_valid2)" @@ -6701,18 +4259,7 @@ "cell_type": "code", "execution_count": 156, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0.90413495165603786" - ] - }, - "execution_count": 156, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "y_pred = dnn_clf_5_to_9.predict(X_test2)\n", "accuracy_score(y_test2, y_pred)" @@ -6843,18 +4390,7 @@ "cell_type": "code", "execution_count": 161, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "TensorShape([Dimension(None), Dimension(100)])" - ] - }, - "execution_count": 161, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "dnn1.shape" ] @@ -6863,18 +4399,7 @@ "cell_type": "code", "execution_count": 162, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "TensorShape([Dimension(None), Dimension(100)])" - ] - }, - "execution_count": 162, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "dnn2.shape" ] @@ -6890,18 +4415,7 @@ "cell_type": "code", "execution_count": 163, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "TensorShape([Dimension(None), Dimension(200)])" - ] - }, - "execution_count": 163, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "dnn_outputs.shape" ] @@ -7123,18 +4637,7 @@ "cell_type": "code", "execution_count": 173, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "((5, 2, 784), dtype('float32'))" - ] - }, - "execution_count": 173, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "X_batch.shape, X_batch.dtype" ] @@ -7150,18 +4653,7 @@ "cell_type": "code", "execution_count": 174, "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAANcAAAGiCAYAAAB05VNzAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHMhJREFUeJzt3XuYjOf5B/AvwVrEaZ1WnHKywgahJQ4ROUjlUOJSQSon\n6pKKYxEShEaLRCMajbSNK1a1KhHiEIegQSIOdQySjSRKVC52s85UVkR/f+T33HtPZ4aZ2bnnnZ35\nfv7J93pnd+bJrttze9/nfd5i//3vf0FE0Vfc6wEQJSoWF5ERFheRERYXkREWF5ERFheRERYXkREW\nF5ERFheRkRJeDyAILhsJX7EIv48/6/CF9LPmzEVkhMVFZITFRWSExUVkhMVFZITFRWSExUVkhMVF\nZITFRWSExUVkhMVFZITFRWSExUVkhMVFZITFRWQkXu/ninvDhg2TPHXqVMmdOnUCACxevDjmYypK\nNm3aJHnZsmWSf/e73wEA8vPz5VixYgW3T1WqVEnyuHHjAAD9+/eXYyVKxM8fac5cREZYXERGisXp\ngxjialAnT54EADz66KNybPXq1ZJ1C5OSkgIA2Lhxoxy75ZZbrIcIFIHb/F988UXJkydPlnzq1Cm/\nr9V/LnVbGMiQIUMkv/TSS4UZYqh4mz+Rl1hcREbi59RKnJk3b57kgQMHAgDy8vLkWOPGjSW7M4QA\n8Nvf/hYAcP78eeshFhnr168HUHAmEPBtBStWrCi5atWqAIBRo0bJsbNnz0r+wx/+INm143//+9/l\n2IkTJySPHz9ecp06dSIef6Q4cxEZ4QkNZe/evZJvvfVWyf/5z38AAE2aNJFjy5cvl3zu3DnJ3bp1\nAwD885//lGMlS5aM/mD9xdUJjezsbMnt27cH4Dvz9+zZU/Lw4cMlN23aNOTPcCea9Gw2Z84cyevW\nrZOcnp4e8vuGgCc0iLzE4iIykvQnNL799lvJI0aMkOxaQW369OmSdZvx4YcfSr506RKAmLWCcWvG\njBmSXTvoTlYAwAsvvCD5mmuuiegz3FKnzz77TI59+eWXknNyciRHuS0MCWcuIiMsLiIjSd8Wzp49\nW/LKlSsDfs2sWbMAAG3btg34esOGDSW/+uqrAIDTp0/LsfLlyxd6nEXB559/LllfJ3RmzpwpOdJW\nUHOr4vV1rnjCmYvICIuLyEjStoWHDx8GUNBaAL6rrzt27OiX9UVmvcJbX6x077thwwY51rp16yiN\nOr7pi+nHjx/3e71WrVqF/owzZ85I1jdcOrp1z8jIKPTnFQZnLiIjSTtzHTx4EACQm5sb8PWPPvpI\n8s033wzAd/nOle4xIhurVq2SvGXLFgBA9erV5diUKVMkp6amxm5gAXDmIjLC4iIykrRtYZkyZQAA\n9evXl2P6Oo17HSjYcUjfj9SoUSPJLVq0kOxalLp160Z5xPFPL2/S//9fffVVod5XnyjRvwNH/w71\n78JrnLmIjLC4iIwkbVvYrFkzAMCePXvk2NatWyXrtibQUh19nUtzLWQ0lvcUNfo6VoMGDSS7trBv\n375yTJ/10xt9OhcuXJDstk4AfG9CdeL1OiJnLiIjLC4iI0nbFjr6psZw2gu9vEnvQxJs5XyyGTNm\njOQ1a9YAALZv3y7H0tLSJA8ePFhy2bJlAQDvv/++HNu8eXPAz6hZsyYAoE+fPlEYcfRx5iIywt2f\nwrB//37Jbdq0kaz34HO7HtWrVy9m4/p/cbX7k7ZgwQIAvicmPvnkE8kXL170H1QI21mPHDkSADBx\n4sSojDMM3P2JyEssLiIjSX9CIxy6/dCr6R988EHJHrSDca9r164+/wUKWkXA93449/SYb775Ro65\nzT//l7tWGa84cxEZYXERGeHZwhC4tkRvLKmvj+kH3WVmZsZuYL7i9mxhJPS1q6ysLMn6d+Cuf0Vj\n+4Aw8WwhkZd4QiMEbutlvfW1fkqHh7NVwnJ7RQK+17luuukmyR7MWGHhzEVkhMVFZIRtYRD6KSf6\nQXcOW0Eb+uF1TkpKiuSnn346lsMpFM5cREZYXERG2BYGsXDhQsl6KwDn/vvvj+Vwkoa790urXLmy\n5A4dOsRyOIXCmYvICIuLyAiXPwVRrVo1yW6PeL2jkd6FqFy5crEbWHAJsfypQoUKAHyfZuJu5wcK\nniLjMS5/IvIST2go+m/LQLee62d2xclslRR+9KMfeT2EiHDmIjLC4iIywrZQ0XvlBbq13O02RPba\nt28vuaj+3DlzERlhcREZ4XWuxJEQ17mKCF7nIvISi4vICIuLyAiLi8gIi4vICIuLyAiLi8gIi4vI\nCIuLyAiLi8gIV8VTkbJt2zbJ7qGDjz76qBzz4PnIQXHmIjLCmYvinr63rm/fvpJ//OMfAwBycnJi\nPqZQcOYiMsLiIjLC+7miYMyYMZKrVKkCABgyZEish5Gw93O5ExcAkJqaKnnu3LkAgPz8fDlWunTp\nWAyJ93MReYnFRWSEZwsjdPr0acmzZ8+WPHz4cC+Gk3Dee+89yStWrJD84YcfSnbPSo5RKxg2zlxE\nRlhcREbYFkbo+PHjkvWTN/STUChygwcPltykSRPJLVq08GI4EeHMRWSEM1eEAj3KFQAyMjJiPJLE\nMn/+fADAvn375NjRo0e9Gk6hcOYiMsLiIjLCtjBC06dP93oICcn9XPWJoerVq3s1nELhzEVkhMVF\nZIRtYRjOnTsneefOnZKbNm0quV69erEcUkI4duyY5M2bNwPwfRBhUcWZi8gIZ64wrF27VnJeXp7k\nPn36eDGchDF+/HjJlSpVAgBkZmZ6NJro4cxFZITFRWSEbWEY3n77bckpKSmSn3jiCS+GU6Tphc/z\n5s2TPHnyZABAxYoVYz6maOPMRWSExUVkhG1hGN566y3J6enpkrkSPnxZWVmS9XWuxo0b+31tbm6u\n5PPnz0suW7YsgIIdt+INZy4iIywuIiNsC0Mwc+ZMAL4tCS8cF86GDRskt2zZUnLDhg0BAK+99poc\nGzVqlGS965Y7ozhhwgQ5NmDAgOgPNkKcuYiMcOYKgds2uVSpUnLsZz/7mVfDKbIOHDggeenSpZIn\nTZokuUuXLgCA3bt3yzE9i2kHDx4EAAwdOlSO1a1bV/JPf/rTwg24kDhzERlhcREZYVsYxK5duyS7\nf3y7lgXg/oSR0NtSX7x4UXK7du0ku6fu6AfaPfzwwwHfzz0Ub/To0XJM7yHpNc5cREZYXERG2BYG\nsWjRIsnfffcdAKBnz55eDSch6GVMml4+5rarHjFiRMTvFy84cxEZ4cwVhH4OVNWqVQEAN998s1fD\nSQjbtm2TXKNGDcn6+mE4/vrXv/p9f+fOnSMcXfRx5iIywuIiMsK2UFm1apXkdevWSe7Xrx8A4Prr\nr4/1kBJK5cqVJestqkuWLBnye+zYsUOyWzb17LPPyrGaNWsWZohRxZmLyAiLi8gI20Jl48aNki9d\nuiSZK+Cjo1WrVpLnzJkj+dSpU5LT0tL8vm/ixImS9VYLd9xxBwBg7NixUR1ntHDmIjLC4iIywrZQ\n+de//iW5du3aknU7Q5G7+uqrAx7v1q2b5HvuuQeA71NO1q9fL7l3796Sn3/+eQBAiRLx+ceYMxeR\nkfgs+RjKzs6WrLer1puipKamxnRMiapHjx6S9+3bJ1nvYeieJHP33XfLsSVLlkj+yU9+YjjC6OLM\nRWSExUVkJOnbwvfee0+y3pewQoUKXgwnoekTD3qvQZ0TCWcuIiMsLiIjSd8WNmvWzOshUILizEVk\nhMVFZKSY24QxzsTloOJcsQi/jz/r8IX0s+bMRWSExUVkhMVFZITFRWSExUVkhMVFZITFRWSExUVk\nhMVFZITFRWSExUVkhMVFZITFRWSExUVkhMVFZITFRWSExUVkhMVFZITFRWSExUVkJOn3LQxm9uzZ\nkj/++GO/119++WXJxYr571fSvHlzyfpZU8GeUUWRu++++ySvWLFC8uDBgyVPmzYtpmMCOHMRmWFx\nERlhWxjE8uXLJeuH4jm6FQzUFu7YsUPyzJkzJQ8dOjRaQ6QA9O/i008/9XAknLmIzLC4iIywLQxC\nP5Dt/vvvB+DbKmqbN2+W/O9//9vv9RtvvDHKoyMA2L17NwBg9erVHo8kMM5cREb4IIYInTlzRnK7\ndu0ku79Nte+//z4WQ0q6BzF0794dADB//vyAr+trW4MGDYrmR/NBDEReYnERGeEJjTDoVrB8+fKS\nA13neu6552IypmSj2+4lS5b4vV62bFnJHTp0iMmYguHMRWSExUVkhG1hCI4dOwYA6NKlixwLtvyp\nVatWAICRI0fGaHSJ79ChQ5L79OkjOT8/3+9rGzRoIPmmm26yHdgVcOYiMsLiIjLCtjAI1woCwMSJ\nEwEAH330UcCvrVKlit/XpqamGo4uuRw5ckTy9u3b/V7PzMyUvHjx4piMKRScuYiMcPlTEL1795as\nb/l39M8tJSVFcrVq1fy+duHChZL17f9RllDLn9auXSv57rvvlhzoz6teUN2xY0fbgf2Ay5+IvMTi\nIjLCExqKvhcrKyvrsl+r2xN9vSXQ/Vzr16+XbNgWJoRLly4BKDgxBARuBQGgdu3aAIC2bdvaDywC\nnLmIjLC4iIywLVTS0tIkt27dWvKmTZsu+32BVsWH8zoVeOmllwAA//jHPwK+rn9HCxYsAACUK1fO\nfmAR4MxFZITXuYL4/PPPJR8/fvyyXzt27FjJeutq5/Dhw5LT09OjMLqAiux1rj179ki+6667AAB5\neXly7Nprr5WsN6O57rrrYjC6gHidi8hLLC4iIzyhEUT9+vUv+7q+5V+3MIEYtoJF1tatWyW7fSGB\nwD/Lhx56SLKHrWDYOHMRGWFxERnh2cII6WVMO3fulFymTBkAwLx58+TYAw88EIshxf3ZwrNnz0q+\n4YYbJOfm5vp9rf6ZLVq0SHLx4nExH/BsIZGXWFxERni2MAzvvvuuZN0K6uVNrp2JUStYJJw/fx4A\n8Nhjj8mxQK2gptvGOGkFw1Y0R01UBHDmCoFbRDpw4MCAr+sNavr37x+TMRUl69atAwC88847V/za\nIUOGAADGjRtnOaSY4MxFZITFRWQk4dvCqVOnStYnHnr06AEgtKVJri3U2ypr7r0A3wfhJTN9wmLM\nmDGX/Vp9P1avXr0AABUqVLAZWAxx5iIywuIiMpKQy590Kzhs2DDJui10q97XrFkjx2rUqCH5N7/5\njeTnn3/e7zNq1aolWb/HlVbTG/J8+VNOTo7ke++9V/KuXbsu+336IXZF5Poglz8ReYnFRWQkIdtC\nvf/FnXfeKVk/LcPRbZxecqP3Hw8kOzs74Ht4yPO2sGvXrpKvdMH4ySeflDxt2jTJpUqVitZwLLEt\nJPJSQs5c2t69eyXrfywH2nZa/ywC7TX43HPPSY7D5Tmez1x6C+pA17YyMjIk65m/COLMReQlFheR\nkYRf/qQf6akf+fnss88CAGbOnBnw+2rWrCnZtTi/+MUvLIaYMIKd2KlTpw4A3+tZyYAzF5ERFheR\nkYQ/W5hEPD9bmER4tpDISywuIiMsLiIjLC4iIywuIiMsLiIjLC4iIywuIiMsLiIjLC4iIywuIiMs\nLiIjLC4iIywuIiMsLiIjLC4iIywuIiMsLiIjLC4iIywuIiMJv28hea9ly5YAgKNHj8ox/dAGvbX1\npk2bAAAnT56M0ejscOYiMsLiIjLCfQsTR9zuW3jrrbcCALZu3XrFry1e/Ie/78uWLSvH2rVrJ/me\ne+7x+56f//znkitVqhTxOMPAfQuJvMTiIjLCs4VkrlmzZgBCawsvXboEADhz5owce/fddyUvW7bM\n73vS0tIk9+zZM+JxRhtnLiIjLC4iI0nfFur2Qz8/WT+obdasWX7fd9ttt0nWD9grXbo0AKBfv35y\nrGLFitEZbBE1ZcoUAMDu3bvlmLtYHA2fffZZ1N4rmjhzERlJ+utcAwYMkPzaa6+F/H3651asmP9l\nj2rVqknW12l++ctfSq5SpQoA35mvEOL2Opc7kaGvUZ0+fTrg12ZkZAAAateuLceC/axHjhwJAGjd\nurUcc52DMV7nIvISi4vISNK3hZ9++qnkYG3hqlWrAABffvmlHLtSWxiM/r4KFSoAAGrVqiXHdOvU\nuXNnybq1DCJu28KHH34YAPDmm28GfL1p06aSFy9eDMD3ZxKH2BYSeYnFRWQk6dtC7dSpU5JHjx4t\necaMGX5f+8orr0jWLcz+/fsBAFlZWXLs2LFjko8cOSI5nHbSLQu6jLhtC6+0Kl7/rB555BHr4UQD\n20IiL7G4iIwk/fKnNWvWSB46dKhkfRbRtW/Dhw+XY126dJF8zTXX+L3vsGHDJH/99deS9RnHRLZ2\n7VrJBw4c8Htd/8zatGkTkzHFGmcuIiNJdULjwoULkjds2AAAeOCBB+RYfn6+ZLc0CQCeeuopAL5L\nl6pWrWoxxMKIqxMaDRs2lLxv3z6/1ytXriz57bfflnzLLbdc9n2vvvpqyeGcEIoyntAg8hKLi8hI\nUrWF69atk3zXXXf98EFBljFNmzZN8sCBAy2GE21Fqi0Mh/4d6RNFY8eOBQCUL1++UO8fAbaFRF5i\ncREZSaq28IsvvpDsbrDTS5N0W1iiRMElwPr16/u91549eyyGWBhx1RbqM6t/+tOf/F5v3769ZN2u\nB3KlOxAaNWok2d3BAADp6emhDDUSbAuJvJRUM5deKeHuG/rggw/k2CeffCL5m2++kZybm+v3XnrT\nGfcPa6Bga2V9m3+MxNXMdfHiRcnuZ63vVUtJSZGsry8GohdRv/7665L1dUtHb4O9dOlSybfffnso\nww4VZy4iL7G4iIwkVVsYjoMHD0p2e+z1799fjul7v/Q/sps0aQIA2LFjh/EI/cRVW6i57aibN28u\nxyI92aDvh5s8eTIAYM6cOXJM/17c7wIAFi1aBACoU6dORJ/7P9gWEnmJxUVkhG1hGE6cOCF59erV\nkvV9Xq5t6datmxybO3duDEYXv22hu82/V69eckxvxlpY+gzik08+GfBr3O5Z+j6zQmBbSOQlFheR\nkaS/zT8c+nm7HTt2lLxlyxbJbjX9oUOH5Njhw4clx/lml6b0Llrdu3eXXNgbT/WNrcF07dq1UJ8R\nCc5cREYSauZy16H0cplAm8dEg17eE+iJHfoeI70IOJnp+7r0UqhBgwZJfuKJJ0J+v7y8PACB95UE\nfJ+Uoj8vVjhzERlhcREZSah+xa1qv+OOO+SY/ofspEmT/L5HP6pV+/bbbyXr7ZadV199VXKg+8CG\nDBkix2rUqHGloSe0QCcs9CNcdVuoV7I7ekX75s2bJbutw7dv3x7wc/VSJ71rVKxw5iIywuIiMpJQ\ny5/cbfz6Gkp2drbkVq1a+X2PvsU80ofY6fbjV7/6FQDfVidG4nb5U6AlYe5Og1CE86BBfZPq/Pnz\nJbdt2zbkzwsBlz8ReSmhZi7n5MmTkjdu3Ch5yZIlkt2t5+H8rZiZmSm5U6dOkh977DHJHuyh58Tt\nzOXk5ORI1g+y2Llzp+RAt+5f6XfUokULye73CphutcCZi8hLLC4iIwnZFiapuG8Lg3HbAAAF2yPo\n61krV66UrNvCZ555BoDvc9XS0tLMxqmwLSTyEouLyAjbwsRRZNvCIohtIZGXWFxERlhcREZYXERG\nWFxERlhcREZYXERGWFxERlhcREZYXERGWFxERlhcREZYXERGWFxERlhcREZYXERGWFxERlhcREZY\nXERGWFxERlhcREZYXERGWFxERhLqsa2F9dVXX0keMGCAZL3dstOvXz/JnTt3luyeGn/VVVdZDDFh\nfPfdd5L1I3J///vfAwAmTJggx/TemsOHD5d83333AQBatmwpx0qWLBn9wUaIMxeRERYXkZGk385a\nPwn+3nvvlZyXlxfR+73xxhsAgMcff7xQ44pAkdrOetasWZL79Onj93q5cuUk6z+j586d8/ta/bOe\nMmWKZMMnnnA7ayIvsbiIjCR9W3jjjTdK3r9/f6Hfr1KlSgCA5cuXyzF9NstQ3LeFkyZNkjx16lTJ\nx44dk/zyyy8DABo1aiTHjhw5Ilk/fzqQ6tWrS16/fr3k+vXrRzDioNgWEnmJxUVkJOkvIl+8eDGq\n73fixAkAwOTJk+XYO++8E9XPKGqys7MBFLR8gG8r2KNHD8lt2rQB4HuR3j0nGfB9JnLdunUBAKdO\nnZJjOTk5knNzcyVHuS0MCWcuIiNJP3M99dRTkt3Sm1BMmzZNsl6So5dQ0Q9mzJgBwPfaoVsmBgBN\nmzaVfNtttwEA8vPz5didd94pecSIEZIzMzMBAFu2bJFj3bp1k/zHP/7R7zP09TNrnLmIjLC4iIwk\nfVuoWzqdwzF9+nTJbAv9Bbp+6K4HAsCoUaMku5MUeknUmDFjLvv+uoWsWrWq5Llz50ru0qULAKBr\n166hDrvQOHMRGWFxERlJ+rYwHLr90Ge+Dh065MVwiowGDRoAAFauXCnH3nzzTcn16tWTvGLFCgBA\nRkZGyO9//fXXB3zf7t27S54/fz4AtoVECSEhZy79D+jKlStL1v+I3rt3LwDglVdekWNnz56VXLx4\nwd87bnGv+1sVAI4fPy754MGDURh1YtErMPQKCyfQbAWEN2MF0r59+6i9V2Fx5iIywuIiMpKQbaFb\nbgMACxYskFymTBnJX3/9NQDgzJkzJmPQi1GT0cKFCyV/8MEHfq+3aNFCcizat8WLFwMADhw4IMeu\nvfZa08/kzEVkhMVFZCSh2kJ37en999+XY7G+BuWW7+hrLMlCn0HVS8Kc8ePHS3766adjMSThrlF+\n//33MftMzlxERlhcREYSqi10Oy59/PHHno3h6NGjAICNGzfKsdatW3s1nJjSdxXs2bPH73V902Pp\n0qXNx6N3NvNilzPOXERGEmrmigfuH856SU+yzFya3kjmhhtuAABcd911no1B51jhzEVkhMVFZCSh\n2kK30rpEiYL/rUj3JdTbXH/xxRdhf79bdU9Aeno6AKBmzZrmn3X+/HnJ+qF6brlVtWrVzMfgcOYi\nMsLiIjKSUG1hu3btAPg+o3jbtm2X/Z7evXtLHjRokORSpUpJvnDhgt/3/fnPf5b8t7/9TfLu3bvD\nGHFycDeh6ptRrTbnXLZsmWT9u3/ooYcAAOXLlzf53EA4cxEZYXERGUmottBxO/1ES2pqqt8xvWe5\nfrhdhw4dABQsgwJ8z1rFYtlPvNm5cycAYNeuXXKsbdu2UXt/vWeK3vvfa5y5iIwk/WNbo81d09HP\nidJ76enHkbotCPQJmMaNG0f60Z4/tlU/bcRtHw0UzOIPPvigHMvKypIc6UmGc+fOAQD69u0rx+bN\nmydZX9Nyv4Pbb789os/6H3xsK5GXWFxERtgWRtkzzzwDAHjhhRfkWEpKiuQqVapIdjtQzZ49W449\n8sgjkX60522hNm7cOMkTJkzwe123iH/5y18kX+n6l17e5K5R6rZbP+XkrbfekhyldtBhW0jkJRYX\nkZGEvM7lJfcUet2q6L3kXSuY6PTD69xe8Xpp0qJFiyT36tVLcqdOnfzeSy8/e/HFFyW7n2taWpoc\nGzp0qOQot4Jh48xFZIQnNIxMmTJF8siRIwN+zVVXXQUAWLp0qRzr2LFjpB8ZVyc0NDfz6JlEXxO7\nEv1nVN+u71bDjB07Vo5Fc+XHZfCEBpGXWFxERtgWGjl16pTk0aNHS9ZPYHn88ccBAG+88UY0PjJu\n20JHX6N6/fXXJf/617+WfOLECb/vK1mypGS9YNotsWrevHlUxxkCtoVEXmJxERlhW5g44r4tTCBs\nC4m8xOIiMsLiIjLC4iIywuIiMsLiIjLC4iIyEq/3c8X+SWXJiz9rI5y5iIywuIiMsLiIjLC4iIyw\nuIiMsLiIjLC4iIywuIiMsLiIjLC4iIywuIiMsLiIjLC4iIywuIiMsLiIjLC4iIywuIiMsLiIjLC4\niIywuIiMsLiIjLC4iIywuIiMsLiIjLC4iIywuIiM/B8LRNiNqaw8LQAAAABJRU5ErkJggg==\n", - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "plt.figure(figsize=(3, 3 * batch_size))\n", "plt.subplot(121)\n", @@ -7184,22 +4676,7 @@ "cell_type": "code", "execution_count": 175, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[1],\n", - " [0],\n", - " [0],\n", - " [1],\n", - " [0]])" - ] - }, - "execution_count": 175, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "y_batch" ] @@ -7248,134 +4725,7 @@ "cell_type": "code", "execution_count": 177, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "0 Train loss: 0.492426\n", - "0 Test accuracy: 0.7861\n", - "1 Train loss: 0.334813\n", - "2 Train loss: 0.290434\n", - "3 Train loss: 0.253434\n", - "4 Train loss: 0.217843\n", - "5 Train loss: 0.17127\n", - "5 Test accuracy: 0.9185\n", - "6 Train loss: 0.207128\n", - "7 Train loss: 0.172275\n", - "8 Train loss: 0.166783\n", - "9 Train loss: 0.161094\n", - "10 Train loss: 0.125131\n", - "10 Test accuracy: 0.9425\n", - "11 Train loss: 0.159824\n", - "12 Train loss: 0.124752\n", - "13 Train loss: 0.112234\n", - "14 Train loss: 0.114502\n", - "15 Train loss: 0.0950093\n", - "15 Test accuracy: 0.9532\n", - "16 Train loss: 0.119296\n", - "17 Train loss: 0.0754429\n", - "18 Train loss: 0.112295\n", - "19 Train loss: 0.133708\n", - "20 Train loss: 0.113547\n", - "20 Test accuracy: 0.9596\n", - "21 Train loss: 0.0674082\n", - "22 Train loss: 0.0936297\n", - "23 Train loss: 0.0986469\n", - "24 Train loss: 0.111875\n", - "25 Train loss: 0.0735623\n", - "25 Test accuracy: 0.9675\n", - "26 Train loss: 0.0790324\n", - "27 Train loss: 0.0487644\n", - "28 Train loss: 0.0869071\n", - "29 Train loss: 0.0694422\n", - "30 Train loss: 0.060089\n", - "30 Test accuracy: 0.9663\n", - "31 Train loss: 0.103902\n", - "32 Train loss: 0.0535952\n", - "33 Train loss: 0.0310679\n", - "34 Train loss: 0.0536294\n", - "35 Train loss: 0.046265\n", - "35 Test accuracy: 0.9701\n", - "36 Train loss: 0.0679821\n", - "37 Train loss: 0.0326656\n", - "38 Train loss: 0.0357479\n", - "39 Train loss: 0.0333373\n", - "40 Train loss: 0.0415115\n", - "40 Test accuracy: 0.9719\n", - "41 Train loss: 0.0577977\n", - "42 Train loss: 0.0342781\n", - "43 Train loss: 0.0439651\n", - "44 Train loss: 0.0597254\n", - "45 Train loss: 0.0588695\n", - "45 Test accuracy: 0.9721\n", - "46 Train loss: 0.0556821\n", - "47 Train loss: 0.063956\n", - "48 Train loss: 0.0301285\n", - "49 Train loss: 0.0402678\n", - "50 Train loss: 0.0489125\n", - "50 Test accuracy: 0.9751\n", - "51 Train loss: 0.0394528\n", - "52 Train loss: 0.0233041\n", - "53 Train loss: 0.064878\n", - "54 Train loss: 0.0510189\n", - "55 Train loss: 0.0312619\n", - "55 Test accuracy: 0.9742\n", - "56 Train loss: 0.0244156\n", - "57 Train loss: 0.0409082\n", - "58 Train loss: 0.0346896\n", - "59 Train loss: 0.0455727\n", - "60 Train loss: 0.0488268\n", - "60 Test accuracy: 0.9751\n", - "61 Train loss: 0.0154253\n", - "62 Train loss: 0.0358874\n", - "63 Train loss: 0.0290555\n", - "64 Train loss: 0.0172143\n", - "65 Train loss: 0.0377991\n", - "65 Test accuracy: 0.9751\n", - "66 Train loss: 0.0360786\n", - "67 Train loss: 0.0240278\n", - "68 Train loss: 0.0314243\n", - "69 Train loss: 0.0412082\n", - "70 Train loss: 0.0439106\n", - "70 Test accuracy: 0.9763\n", - "71 Train loss: 0.0169656\n", - "72 Train loss: 0.0181306\n", - "73 Train loss: 0.0214228\n", - "74 Train loss: 0.0418301\n", - "75 Train loss: 0.0378622\n", - "75 Test accuracy: 0.9759\n", - "76 Train loss: 0.0199817\n", - "77 Train loss: 0.0145837\n", - "78 Train loss: 0.0199176\n", - "79 Train loss: 0.0226598\n", - "80 Train loss: 0.0119815\n", - "80 Test accuracy: 0.9779\n", - "81 Train loss: 0.0177832\n", - "82 Train loss: 0.00981572\n", - "83 Train loss: 0.0279094\n", - "84 Train loss: 0.0237818\n", - "85 Train loss: 0.0157778\n", - "85 Test accuracy: 0.978\n", - "86 Train loss: 0.00950592\n", - "87 Train loss: 0.0226222\n", - "88 Train loss: 0.0226599\n", - "89 Train loss: 0.0185005\n", - "90 Train loss: 0.0118967\n", - "90 Test accuracy: 0.976\n", - "91 Train loss: 0.0209059\n", - "92 Train loss: 0.0181153\n", - "93 Train loss: 0.0131697\n", - "94 Train loss: 0.017605\n", - "95 Train loss: 0.0193861\n", - "95 Test accuracy: 0.976\n", - "96 Train loss: 0.0156532\n", - "97 Train loss: 0.0136041\n", - "98 Train loss: 0.00743028\n", - "99 Train loss: 0.0267189\n" - ] - } - ], + "outputs": [], "source": [ "n_epochs = 100\n", "batch_size = 500\n", @@ -7467,25 +4817,7 @@ "cell_type": "code", "execution_count": 179, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "INFO:tensorflow:Restoring parameters from ./my_digit_comparison_model.ckpt\n", - "0 Test accuracy: 0.9269\n", - "10 Test accuracy: 0.9675\n", - "20 Test accuracy: 0.9673\n", - "30 Test accuracy: 0.9673\n", - "40 Test accuracy: 0.9674\n", - "50 Test accuracy: 0.9673\n", - "60 Test accuracy: 0.9673\n", - "70 Test accuracy: 0.9673\n", - "80 Test accuracy: 0.9672\n", - "90 Test accuracy: 0.9673\n" - ] - } - ], + "outputs": [], "source": [ "n_epochs = 100\n", "batch_size = 50\n", @@ -7554,29 +4886,7 @@ "cell_type": "code", "execution_count": 181, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "0 Test accuracy: 0.8893\n", - "10 Test accuracy: 0.9402\n", - "20 Test accuracy: 0.9479\n", - "30 Test accuracy: 0.9474\n", - "40 Test accuracy: 0.9479\n", - "50 Test accuracy: 0.9475\n", - "60 Test accuracy: 0.9475\n", - "70 Test accuracy: 0.9475\n", - "80 Test accuracy: 0.9476\n", - "90 Test accuracy: 0.9476\n", - "100 Test accuracy: 0.9473\n", - "110 Test accuracy: 0.9472\n", - "120 Test accuracy: 0.9474\n", - "130 Test accuracy: 0.9474\n", - "140 Test accuracy: 0.9475\n" - ] - } - ], + "outputs": [], "source": [ "n_epochs = 150\n", "batch_size = 50\n", @@ -7631,7 +4941,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.5.3" + "version": "3.6.2" }, "nav_menu": { "height": "360px", From 7686839b36c2c248ede1adecd2b48e035b6a9640 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Geron?= Date: Thu, 9 Nov 2017 13:17:24 +0100 Subject: [PATCH 25/26] Use int8 instead of float64 to represent pixel values: divides RAM footprint by 8 --- 16_reinforcement_learning.ipynb | 139 ++++++++++---------------------- 1 file changed, 42 insertions(+), 97 deletions(-) diff --git a/16_reinforcement_learning.ipynb b/16_reinforcement_learning.ipynb index 96e94a2..9459440 100644 --- a/16_reinforcement_learning.ipynb +++ b/16_reinforcement_learning.ipynb @@ -31,9 +31,7 @@ { "cell_type": "code", "execution_count": 1, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "# To support both python 2 and python 3\n", @@ -95,9 +93,7 @@ { "cell_type": "code", "execution_count": 2, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "import gym" @@ -129,9 +125,7 @@ { "cell_type": "code", "execution_count": 4, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "obs = env.reset()" @@ -163,9 +157,7 @@ { "cell_type": "code", "execution_count": 6, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "img = env.render(mode=\"rgb_array\")" @@ -226,9 +218,7 @@ { "cell_type": "code", "execution_count": 9, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "def plot_environment(env, figsize=(5,4)):\n", @@ -273,9 +263,7 @@ { "cell_type": "code", "execution_count": 11, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "env.reset()\n", @@ -311,9 +299,7 @@ { "cell_type": "code", "execution_count": 13, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "obs, reward, done, info = env.step(0)" @@ -393,9 +379,7 @@ { "cell_type": "code", "execution_count": 18, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "frames = []\n", @@ -424,9 +408,7 @@ { "cell_type": "code", "execution_count": 19, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "def update_scene(num, frames, patch):\n", @@ -461,9 +443,7 @@ { "cell_type": "code", "execution_count": 21, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "env.close()" @@ -502,9 +482,7 @@ { "cell_type": "code", "execution_count": 23, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "obs = env.reset()" @@ -547,9 +525,7 @@ { "cell_type": "code", "execution_count": 25, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "from PIL import Image, ImageDraw\n", @@ -633,9 +609,7 @@ { "cell_type": "code", "execution_count": 28, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "obs = env.reset()\n", @@ -677,9 +651,7 @@ { "cell_type": "code", "execution_count": 31, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "obs = env.reset()\n", @@ -722,9 +694,7 @@ { "cell_type": "code", "execution_count": 33, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "frames = []\n", @@ -795,9 +765,7 @@ { "cell_type": "code", "execution_count": 35, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "import tensorflow as tf\n", @@ -846,9 +814,7 @@ { "cell_type": "code", "execution_count": 36, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "n_max_steps = 1000\n", @@ -895,9 +861,7 @@ { "cell_type": "code", "execution_count": 38, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "import tensorflow as tf\n", @@ -965,9 +929,7 @@ { "cell_type": "code", "execution_count": 40, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "def render_policy_net(model_path, action, X, n_max_steps = 1000):\n", @@ -1024,9 +986,7 @@ { "cell_type": "code", "execution_count": 42, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "import tensorflow as tf\n", @@ -1069,9 +1029,7 @@ { "cell_type": "code", "execution_count": 43, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "def discount_rewards(rewards, discount_rate):\n", @@ -1157,9 +1115,7 @@ { "cell_type": "code", "execution_count": 47, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "env.close()" @@ -1309,9 +1265,7 @@ { "cell_type": "code", "execution_count": 51, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "n_states = 3\n", @@ -1336,9 +1290,7 @@ { "cell_type": "code", "execution_count": 52, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "def optimal_policy(state):\n", @@ -1439,23 +1391,28 @@ { "cell_type": "code", "execution_count": 57, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ - "mspacman_color = np.array([210, 164, 74]).mean()\n", + "mspacman_color = 210 + 164 + 74\n", "\n", "def preprocess_observation(obs):\n", " img = obs[1:176:2, ::2] # crop and downsize\n", - " img = img.mean(axis=2) # to greyscale\n", + " img = img.sum(axis=2) # to greyscale\n", " img[img==mspacman_color] = 0 # Improve contrast\n", - " img = (img - 128) / 128 - 1 # normalize from -1. to 1.\n", + " img = (img // 3 - 128).astype(np.int8) # normalize from -128 to 127\n", " return img.reshape(88, 80, 1)\n", "\n", "img = preprocess_observation(obs)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note: the `preprocess_observation()` function is slightly different from the one in the book: instead of representing pixels as 64-bit floats from -1.0 to 1.0, it represents them as 8-bit integers from -128 to 127. The benefit is that the replay memory will take up about 6.5 GB of RAM instead of 52 GB. The reduced precision has no impact on training." + ] + }, { "cell_type": "code", "execution_count": 58, @@ -1498,9 +1455,7 @@ { "cell_type": "code", "execution_count": 59, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "reset_graph()\n", @@ -1545,9 +1500,7 @@ { "cell_type": "code", "execution_count": 60, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "X_state = tf.placeholder(tf.float32, shape=[None, input_height, input_width,\n", @@ -1572,9 +1525,7 @@ { "cell_type": "code", "execution_count": 62, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "learning_rate = 0.001\n", @@ -1608,9 +1559,7 @@ { "cell_type": "code", "execution_count": 63, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "from collections import deque\n", @@ -1632,9 +1581,7 @@ { "cell_type": "code", "execution_count": 64, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "eps_min = 0.1\n", @@ -1678,9 +1625,7 @@ { "cell_type": "code", "execution_count": 66, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "loss_val = np.infty\n", @@ -1970,7 +1915,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.5.2" + "version": "3.6.3" }, "nav_menu": {}, "toc": { From 8a6c7da0a9f086afcc7293783bf36c5ea83e9797 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Geron?= Date: Thu, 9 Nov 2017 16:35:44 +0100 Subject: [PATCH 26/26] Scale X_state down to [-1, 1] range in chapter 16 --- 16_reinforcement_learning.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/16_reinforcement_learning.ipynb b/16_reinforcement_learning.ipynb index 9459440..15c258e 100644 --- a/16_reinforcement_learning.ipynb +++ b/16_reinforcement_learning.ipynb @@ -1410,7 +1410,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Note: the `preprocess_observation()` function is slightly different from the one in the book: instead of representing pixels as 64-bit floats from -1.0 to 1.0, it represents them as 8-bit integers from -128 to 127. The benefit is that the replay memory will take up about 6.5 GB of RAM instead of 52 GB. The reduced precision has no impact on training." + "Note: the `preprocess_observation()` function is slightly different from the one in the book: instead of representing pixels as 64-bit floats from -1.0 to 1.0, it represents them as signed bytes (from -128 to 127). The benefit is that the replay memory will take up roughly 8 times less RAM (about 6.5 GB instead of 52 GB). The reduced precision has no visible impact on training." ] }, { @@ -1475,7 +1475,7 @@ "initializer = tf.contrib.layers.variance_scaling_initializer()\n", "\n", "def q_network(X_state, name):\n", - " prev_layer = X_state\n", + " prev_layer = X_state / 128.0 # scale pixel intensities to the [-1.0, 1.0] range.\n", " with tf.variable_scope(name) as scope:\n", " for n_maps, kernel_size, strides, padding, activation in zip(\n", " conv_n_maps, conv_kernel_sizes, conv_strides,\n",