From cac02c5964c66341e0acb92c883f3c721a46787c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Geron?= Date: Tue, 8 May 2018 20:21:23 +0200 Subject: [PATCH] Updated to latest version of TensorFlow --- 11_deep_learning.ipynb | 740 +++++++++++++++++------------------------ 1 file changed, 297 insertions(+), 443 deletions(-) diff --git a/11_deep_learning.ipynb b/11_deep_learning.ipynb index d83e660..737a451 100644 --- a/11_deep_learning.ipynb +++ b/11_deep_learning.ipynb @@ -156,7 +156,7 @@ "metadata": {}, "outputs": [], "source": [ - "he_init = tf.contrib.layers.variance_scaling_initializer()\n", + "he_init = tf.variance_scaling_initializer()\n", "hidden1 = tf.layers.dense(X, n_hidden1, activation=tf.nn.relu,\n", " kernel_initializer=he_init, name=\"hidden1\")" ] @@ -262,7 +262,7 @@ "outputs": [], "source": [ "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n", - "y = tf.placeholder(tf.int64, shape=(None), name=\"y\")" + "y = tf.placeholder(tf.int32, shape=(None), name=\"y\")" ] }, { @@ -329,19 +329,45 @@ "Let's load the data:" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Warning**: `tf.examples.tutorials.mnist` is deprecated. We will use `tf.keras.datasets.mnist` instead." + ] + }, { "cell_type": "code", "execution_count": 18, "metadata": {}, "outputs": [], "source": [ - "from tensorflow.examples.tutorials.mnist import input_data\n", - "mnist = input_data.read_data_sets(\"/tmp/data/\")" + "(X_train, y_train), (X_test, y_test) = tf.keras.datasets.mnist.load_data()\n", + "X_train = X_train.astype(np.float32).reshape(-1, 28*28) / 255.0\n", + "X_test = X_test.astype(np.float32).reshape(-1, 28*28) / 255.0\n", + "y_train = y_train.astype(np.int32)\n", + "y_test = y_test.astype(np.int32)\n", + "X_valid, X_train = X_train[:5000], X_train[5000:]\n", + "y_valid, y_train = y_train[:5000], y_train[5000:]" ] }, { "cell_type": "code", "execution_count": 19, + "metadata": {}, + "outputs": [], + "source": [ + "def shuffle_batch(X, y, batch_size):\n", + " rnd_idx = np.random.permutation(len(X))\n", + " n_batches = len(X) // batch_size\n", + " for batch_idx in np.array_split(rnd_idx, n_batches):\n", + " X_batch, y_batch = X[batch_idx], y[batch_idx]\n", + " yield X_batch, y_batch" + ] + }, + { + "cell_type": "code", + "execution_count": 20, "metadata": { "scrolled": true }, @@ -353,13 +379,12 @@ "with tf.Session() as sess:\n", " init.run()\n", " for epoch in range(n_epochs):\n", - " for iteration in range(mnist.train.num_examples // batch_size):\n", - " X_batch, y_batch = mnist.train.next_batch(batch_size)\n", + " for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size):\n", " sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n", " if epoch % 5 == 0:\n", - " acc_train = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n", - " acc_test = accuracy.eval(feed_dict={X: mnist.validation.images, y: mnist.validation.labels})\n", - " print(epoch, \"Batch accuracy:\", acc_train, \"Validation accuracy:\", acc_test)\n", + " acc_batch = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n", + " acc_valid = accuracy.eval(feed_dict={X: X_valid, y: y_valid})\n", + " print(epoch, \"Batch accuracy:\", acc_batch, \"Validation accuracy:\", acc_valid)\n", "\n", " save_path = saver.save(sess, \"./my_model_final.ckpt\")" ] @@ -373,7 +398,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 21, "metadata": {}, "outputs": [], "source": [ @@ -383,7 +408,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 22, "metadata": {}, "outputs": [], "source": [ @@ -408,7 +433,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 23, "metadata": {}, "outputs": [], "source": [ @@ -419,7 +444,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 24, "metadata": {}, "outputs": [], "source": [ @@ -442,7 +467,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 25, "metadata": {}, "outputs": [], "source": [ @@ -454,7 +479,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 26, "metadata": {}, "outputs": [], "source": [ @@ -479,7 +504,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 27, "metadata": {}, "outputs": [], "source": [ @@ -504,7 +529,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 28, "metadata": {}, "outputs": [], "source": [ @@ -530,7 +555,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 29, "metadata": {}, "outputs": [], "source": [ @@ -542,7 +567,7 @@ "n_outputs = 10\n", "\n", "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n", - "y = tf.placeholder(tf.int64, shape=(None), name=\"y\")\n", + "y = tf.placeholder(tf.int32, shape=(None), name=\"y\")\n", "\n", "with tf.name_scope(\"dnn\"):\n", " hidden1 = tf.layers.dense(X, n_hidden1, activation=selu, name=\"hidden1\")\n", @@ -578,25 +603,24 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 30, "metadata": {}, "outputs": [], "source": [ - "means = mnist.train.images.mean(axis=0, keepdims=True)\n", - "stds = mnist.train.images.std(axis=0, keepdims=True) + 1e-10\n", + "means = X_train.mean(axis=0, keepdims=True)\n", + "stds = X_train.std(axis=0, keepdims=True) + 1e-10\n", + "X_val_scaled = (X_valid - means) / stds\n", "\n", "with tf.Session() as sess:\n", " init.run()\n", " for epoch in range(n_epochs):\n", - " for iteration in range(mnist.train.num_examples // batch_size):\n", - " X_batch, y_batch = mnist.train.next_batch(batch_size)\n", + " for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size):\n", " X_batch_scaled = (X_batch - means) / stds\n", " sess.run(training_op, feed_dict={X: X_batch_scaled, y: y_batch})\n", " if epoch % 5 == 0:\n", - " acc_train = accuracy.eval(feed_dict={X: X_batch_scaled, y: y_batch})\n", - " X_val_scaled = (mnist.validation.images - means) / stds\n", - " acc_test = accuracy.eval(feed_dict={X: X_val_scaled, y: mnist.validation.labels})\n", - " print(epoch, \"Batch accuracy:\", acc_train, \"Validation accuracy:\", acc_test)\n", + " acc_batch = accuracy.eval(feed_dict={X: X_batch_scaled, y: y_batch})\n", + " acc_valid = accuracy.eval(feed_dict={X: X_val_scaled, y: y_valid})\n", + " print(epoch, \"Batch accuracy:\", acc_batch, \"Validation accuracy:\", acc_valid)\n", "\n", " save_path = saver.save(sess, \"./my_model_final_selu.ckpt\")" ] @@ -625,7 +649,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 31, "metadata": {}, "outputs": [], "source": [ @@ -657,7 +681,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 32, "metadata": {}, "outputs": [], "source": [ @@ -676,7 +700,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 33, "metadata": {}, "outputs": [], "source": [ @@ -704,7 +728,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 34, "metadata": {}, "outputs": [], "source": [ @@ -713,11 +737,11 @@ "batch_norm_momentum = 0.9\n", "\n", "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n", - "y = tf.placeholder(tf.int64, shape=(None), name=\"y\")\n", + "y = tf.placeholder(tf.int32, shape=(None), name=\"y\")\n", "training = tf.placeholder_with_default(False, shape=(), name='training')\n", "\n", "with tf.name_scope(\"dnn\"):\n", - " he_init = tf.contrib.layers.variance_scaling_initializer()\n", + " he_init = tf.variance_scaling_initializer()\n", "\n", " my_batch_norm_layer = partial(\n", " tf.layers.batch_normalization,\n", @@ -760,7 +784,7 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 35, "metadata": {}, "outputs": [], "source": [ @@ -770,7 +794,7 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 36, "metadata": {}, "outputs": [], "source": [ @@ -779,13 +803,11 @@ "with tf.Session() as sess:\n", " init.run()\n", " for epoch in range(n_epochs):\n", - " for iteration in range(mnist.train.num_examples // batch_size):\n", - " X_batch, y_batch = mnist.train.next_batch(batch_size)\n", + " for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size):\n", " sess.run([training_op, extra_update_ops],\n", " feed_dict={training: True, X: X_batch, y: y_batch})\n", - " accuracy_val = accuracy.eval(feed_dict={X: mnist.test.images,\n", - " y: mnist.test.labels})\n", - " print(epoch, \"Test accuracy:\", accuracy_val)\n", + " accuracy_val = accuracy.eval(feed_dict={X: X_valid, y: y_valid})\n", + " print(epoch, \"Validation accuracy:\", accuracy_val)\n", "\n", " save_path = saver.save(sess, \"./my_model_final.ckpt\")" ] @@ -827,7 +849,7 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 37, "metadata": {}, "outputs": [], "source": [ @@ -836,7 +858,7 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 38, "metadata": {}, "outputs": [], "source": [ @@ -859,7 +881,7 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 39, "metadata": {}, "outputs": [], "source": [ @@ -874,7 +896,7 @@ "n_outputs = 10\n", "\n", "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n", - "y = tf.placeholder(tf.int64, shape=(None), name=\"y\")\n", + "y = tf.placeholder(tf.int32, shape=(None), name=\"y\")\n", "\n", "with tf.name_scope(\"dnn\"):\n", " hidden1 = tf.layers.dense(X, n_hidden1, activation=tf.nn.relu, name=\"hidden1\")\n", @@ -891,7 +913,7 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 40, "metadata": {}, "outputs": [], "source": [ @@ -907,7 +929,7 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 41, "metadata": {}, "outputs": [], "source": [ @@ -929,7 +951,7 @@ }, { "cell_type": "code", - "execution_count": 41, + "execution_count": 42, "metadata": {}, "outputs": [], "source": [ @@ -940,7 +962,7 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": 43, "metadata": {}, "outputs": [], "source": [ @@ -950,7 +972,7 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": 44, "metadata": {}, "outputs": [], "source": [ @@ -960,19 +982,17 @@ }, { "cell_type": "code", - "execution_count": 44, + "execution_count": 45, "metadata": {}, "outputs": [], "source": [ "with tf.Session() as sess:\n", " init.run()\n", " for epoch in range(n_epochs):\n", - " for iteration in range(mnist.train.num_examples // batch_size):\n", - " X_batch, y_batch = mnist.train.next_batch(batch_size)\n", + " for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size):\n", " sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n", - " accuracy_val = accuracy.eval(feed_dict={X: mnist.test.images,\n", - " y: mnist.test.labels})\n", - " print(epoch, \"Test accuracy:\", accuracy_val)\n", + " accuracy_val = accuracy.eval(feed_dict={X: X_valid, y: y_valid})\n", + " print(epoch, \"Validation accuracy:\", accuracy_val)\n", "\n", " save_path = saver.save(sess, \"./my_model_final.ckpt\")" ] @@ -1000,7 +1020,7 @@ }, { "cell_type": "code", - "execution_count": 45, + "execution_count": 46, "metadata": {}, "outputs": [], "source": [ @@ -1009,7 +1029,7 @@ }, { "cell_type": "code", - "execution_count": 46, + "execution_count": 47, "metadata": {}, "outputs": [], "source": [ @@ -1025,7 +1045,7 @@ }, { "cell_type": "code", - "execution_count": 47, + "execution_count": 48, "metadata": {}, "outputs": [], "source": [ @@ -1042,10 +1062,12 @@ }, { "cell_type": "code", - "execution_count": 48, + "execution_count": 49, "metadata": {}, "outputs": [], "source": [ + "# Works on Chrome, not guaranteed on other browsers\n", + "\n", "from IPython.display import clear_output, Image, display, HTML\n", "\n", "def strip_consts(graph_def, max_const_size=32):\n", @@ -1086,7 +1108,7 @@ }, { "cell_type": "code", - "execution_count": 49, + "execution_count": 50, "metadata": { "scrolled": true }, @@ -1104,7 +1126,7 @@ }, { "cell_type": "code", - "execution_count": 50, + "execution_count": 51, "metadata": {}, "outputs": [], "source": [ @@ -1125,7 +1147,7 @@ }, { "cell_type": "code", - "execution_count": 51, + "execution_count": 52, "metadata": {}, "outputs": [], "source": [ @@ -1142,7 +1164,7 @@ }, { "cell_type": "code", - "execution_count": 52, + "execution_count": 53, "metadata": {}, "outputs": [], "source": [ @@ -1158,7 +1180,7 @@ }, { "cell_type": "code", - "execution_count": 53, + "execution_count": 54, "metadata": {}, "outputs": [], "source": [ @@ -1176,7 +1198,7 @@ }, { "cell_type": "code", - "execution_count": 54, + "execution_count": 55, "metadata": {}, "outputs": [], "source": [ @@ -1184,12 +1206,10 @@ " saver.restore(sess, \"./my_model_final.ckpt\")\n", "\n", " for epoch in range(n_epochs):\n", - " for iteration in range(mnist.train.num_examples // batch_size):\n", - " X_batch, y_batch = mnist.train.next_batch(batch_size)\n", + " for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size):\n", " sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n", - " accuracy_val = accuracy.eval(feed_dict={X: mnist.test.images,\n", - " y: mnist.test.labels})\n", - " print(epoch, \"Test accuracy:\", accuracy_val)\n", + " accuracy_val = accuracy.eval(feed_dict={X: X_valid, y: y_valid})\n", + " print(epoch, \"Validation accuracy:\", accuracy_val)\n", "\n", " save_path = saver.save(sess, \"./my_new_model_final.ckpt\") " ] @@ -1203,7 +1223,7 @@ }, { "cell_type": "code", - "execution_count": 55, + "execution_count": 56, "metadata": {}, "outputs": [], "source": [ @@ -1217,7 +1237,7 @@ "n_outputs = 10\n", "\n", "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n", - "y = tf.placeholder(tf.int64, shape=(None), name=\"y\")\n", + "y = tf.placeholder(tf.int32, shape=(None), name=\"y\")\n", "\n", "with tf.name_scope(\"dnn\"):\n", " hidden1 = tf.layers.dense(X, n_hidden1, activation=tf.nn.relu, name=\"hidden1\")\n", @@ -1257,7 +1277,7 @@ }, { "cell_type": "code", - "execution_count": 56, + "execution_count": 57, "metadata": {}, "outputs": [], "source": [ @@ -1265,12 +1285,10 @@ " saver.restore(sess, \"./my_model_final.ckpt\")\n", "\n", " for epoch in range(n_epochs):\n", - " for iteration in range(mnist.train.num_examples // batch_size):\n", - " X_batch, y_batch = mnist.train.next_batch(batch_size)\n", + " for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size):\n", " sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n", - " accuracy_val = accuracy.eval(feed_dict={X: mnist.test.images,\n", - " y: mnist.test.labels})\n", - " print(epoch, \"Test accuracy:\", accuracy_val)\n", + " accuracy_val = accuracy.eval(feed_dict={X: X_valid, y: y_valid})\n", + " print(epoch, \"Validation accuracy:\", accuracy_val)\n", "\n", " save_path = saver.save(sess, \"./my_new_model_final.ckpt\") " ] @@ -1284,7 +1302,7 @@ }, { "cell_type": "code", - "execution_count": 57, + "execution_count": 58, "metadata": {}, "outputs": [], "source": [ @@ -1328,7 +1346,7 @@ }, { "cell_type": "code", - "execution_count": 58, + "execution_count": 59, "metadata": {}, "outputs": [], "source": [ @@ -1337,12 +1355,10 @@ " saver.restore(sess, \"./my_model_final.ckpt\")\n", "\n", " for epoch in range(n_epochs):\n", - " for iteration in range(mnist.train.num_examples // batch_size):\n", - " X_batch, y_batch = mnist.train.next_batch(batch_size)\n", + " for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size):\n", " sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n", - " accuracy_val = accuracy.eval(feed_dict={X: mnist.test.images,\n", - " y: mnist.test.labels})\n", - " print(epoch, \"Test accuracy:\", accuracy_val)\n", + " accuracy_val = accuracy.eval(feed_dict={X: X_valid, y: y_valid})\n", + " print(epoch, \"Validation accuracy:\", accuracy_val)\n", "\n", " save_path = new_saver.save(sess, \"./my_new_model_final.ckpt\")" ] @@ -1356,7 +1372,7 @@ }, { "cell_type": "code", - "execution_count": 59, + "execution_count": 60, "metadata": {}, "outputs": [], "source": [ @@ -1370,7 +1386,7 @@ "n_outputs = 10 # new!\n", "\n", "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n", - "y = tf.placeholder(tf.int64, shape=(None), name=\"y\")\n", + "y = tf.placeholder(tf.int32, shape=(None), name=\"y\")\n", "\n", "with tf.name_scope(\"dnn\"):\n", " hidden1 = tf.layers.dense(X, n_hidden1, activation=tf.nn.relu, name=\"hidden1\") # reused\n", @@ -1401,7 +1417,7 @@ }, { "cell_type": "code", - "execution_count": 60, + "execution_count": 61, "metadata": {}, "outputs": [], "source": [ @@ -1416,13 +1432,11 @@ " init.run()\n", " restore_saver.restore(sess, \"./my_model_final.ckpt\")\n", "\n", - " for epoch in range(n_epochs): # not shown in the book\n", - " for iteration in range(mnist.train.num_examples // batch_size): # not shown\n", - " X_batch, y_batch = mnist.train.next_batch(batch_size) # not shown\n", - " sess.run(training_op, feed_dict={X: X_batch, y: y_batch}) # not shown\n", - " accuracy_val = accuracy.eval(feed_dict={X: mnist.test.images, # not shown\n", - " y: mnist.test.labels}) # not shown\n", - " print(epoch, \"Test accuracy:\", accuracy_val) # not shown\n", + " for epoch in range(n_epochs): # not shown in the book\n", + " for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size): # not shown\n", + " sess.run(training_op, feed_dict={X: X_batch, y: y_batch}) # not shown\n", + " accuracy_val = accuracy.eval(feed_dict={X: X_valid, y: y_valid}) # not shown\n", + " print(epoch, \"Validation accuracy:\", accuracy_val) # not shown\n", "\n", " save_path = saver.save(sess, \"./my_new_model_final.ckpt\")" ] @@ -1443,7 +1457,7 @@ }, { "cell_type": "code", - "execution_count": 61, + "execution_count": 62, "metadata": {}, "outputs": [], "source": [ @@ -1455,7 +1469,7 @@ }, { "cell_type": "code", - "execution_count": 62, + "execution_count": 63, "metadata": {}, "outputs": [], "source": [ @@ -1497,7 +1511,7 @@ }, { "cell_type": "code", - "execution_count": 63, + "execution_count": 64, "metadata": {}, "outputs": [], "source": [ @@ -1543,7 +1557,7 @@ }, { "cell_type": "code", - "execution_count": 64, + "execution_count": 65, "metadata": {}, "outputs": [], "source": [ @@ -1559,7 +1573,7 @@ }, { "cell_type": "code", - "execution_count": 65, + "execution_count": 66, "metadata": {}, "outputs": [], "source": [ @@ -1568,7 +1582,7 @@ }, { "cell_type": "code", - "execution_count": 66, + "execution_count": 67, "metadata": {}, "outputs": [], "source": [ @@ -1584,7 +1598,7 @@ }, { "cell_type": "code", - "execution_count": 67, + "execution_count": 68, "metadata": {}, "outputs": [], "source": [ @@ -1598,7 +1612,7 @@ "n_outputs = 10 # new!\n", "\n", "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n", - "y = tf.placeholder(tf.int64, shape=(None), name=\"y\")\n", + "y = tf.placeholder(tf.int32, shape=(None), name=\"y\")\n", "\n", "with tf.name_scope(\"dnn\"):\n", " hidden1 = tf.layers.dense(X, n_hidden1, activation=tf.nn.relu, name=\"hidden1\") # reused\n", @@ -1618,7 +1632,7 @@ }, { "cell_type": "code", - "execution_count": 68, + "execution_count": 69, "metadata": {}, "outputs": [], "source": [ @@ -1631,7 +1645,7 @@ }, { "cell_type": "code", - "execution_count": 69, + "execution_count": 70, "metadata": {}, "outputs": [], "source": [ @@ -1641,7 +1655,7 @@ }, { "cell_type": "code", - "execution_count": 70, + "execution_count": 71, "metadata": {}, "outputs": [], "source": [ @@ -1657,19 +1671,17 @@ " restore_saver.restore(sess, \"./my_model_final.ckpt\")\n", "\n", " for epoch in range(n_epochs):\n", - " for iteration in range(mnist.train.num_examples // batch_size):\n", - " X_batch, y_batch = mnist.train.next_batch(batch_size)\n", + " for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size):\n", " sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n", - " accuracy_val = accuracy.eval(feed_dict={X: mnist.test.images,\n", - " y: mnist.test.labels})\n", - " print(epoch, \"Test accuracy:\", accuracy_val)\n", + " accuracy_val = accuracy.eval(feed_dict={X: X_valid, y: y_valid})\n", + " print(epoch, \"Validation accuracy:\", accuracy_val)\n", "\n", " save_path = saver.save(sess, \"./my_new_model_final.ckpt\")" ] }, { "cell_type": "code", - "execution_count": 71, + "execution_count": 72, "metadata": {}, "outputs": [], "source": [ @@ -1683,12 +1695,12 @@ "n_outputs = 10 # new!\n", "\n", "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n", - "y = tf.placeholder(tf.int64, shape=(None), name=\"y\")" + "y = tf.placeholder(tf.int32, shape=(None), name=\"y\")" ] }, { "cell_type": "code", - "execution_count": 72, + "execution_count": 73, "metadata": {}, "outputs": [], "source": [ @@ -1707,7 +1719,7 @@ }, { "cell_type": "code", - "execution_count": 73, + "execution_count": 74, "metadata": {}, "outputs": [], "source": [ @@ -1733,7 +1745,7 @@ }, { "cell_type": "code", - "execution_count": 74, + "execution_count": 75, "metadata": {}, "outputs": [], "source": [ @@ -1749,12 +1761,10 @@ " restore_saver.restore(sess, \"./my_model_final.ckpt\")\n", "\n", " for epoch in range(n_epochs):\n", - " for iteration in range(mnist.train.num_examples // batch_size):\n", - " X_batch, y_batch = mnist.train.next_batch(batch_size)\n", + " for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size):\n", " sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n", - " accuracy_val = accuracy.eval(feed_dict={X: mnist.test.images,\n", - " y: mnist.test.labels})\n", - " print(epoch, \"Test accuracy:\", accuracy_val)\n", + " accuracy_val = accuracy.eval(feed_dict={X: X_valid, y: y_valid})\n", + " print(epoch, \"Validation accuracy:\", accuracy_val)\n", "\n", " save_path = saver.save(sess, \"./my_new_model_final.ckpt\")" ] @@ -1768,7 +1778,7 @@ }, { "cell_type": "code", - "execution_count": 75, + "execution_count": 76, "metadata": {}, "outputs": [], "source": [ @@ -1782,7 +1792,7 @@ "n_outputs = 10 # new!\n", "\n", "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n", - "y = tf.placeholder(tf.int64, shape=(None), name=\"y\")\n", + "y = tf.placeholder(tf.int32, shape=(None), name=\"y\")\n", "\n", "with tf.name_scope(\"dnn\"):\n", " hidden1 = tf.layers.dense(X, n_hidden1, activation=tf.nn.relu,\n", @@ -1811,7 +1821,7 @@ }, { "cell_type": "code", - "execution_count": 76, + "execution_count": 77, "metadata": {}, "outputs": [], "source": [ @@ -1825,31 +1835,31 @@ }, { "cell_type": "code", - "execution_count": 77, + "execution_count": 78, "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", "\n", - "n_batches = mnist.train.num_examples // batch_size\n", + "n_batches = len(X_train) // batch_size\n", "\n", "with tf.Session() as sess:\n", " init.run()\n", " restore_saver.restore(sess, \"./my_model_final.ckpt\")\n", " \n", - " h2_cache = sess.run(hidden2, feed_dict={X: mnist.train.images})\n", - " h2_cache_test = sess.run(hidden2, feed_dict={X: mnist.test.images}) # not shown in the book\n", + " h2_cache = sess.run(hidden2, feed_dict={X: X_train})\n", + " h2_cache_valid = sess.run(hidden2, feed_dict={X: X_valid}) # not shown in the book\n", "\n", " for epoch in range(n_epochs):\n", - " shuffled_idx = np.random.permutation(mnist.train.num_examples)\n", + " shuffled_idx = np.random.permutation(len(X_train))\n", " hidden2_batches = np.array_split(h2_cache[shuffled_idx], n_batches)\n", - " y_batches = np.array_split(mnist.train.labels[shuffled_idx], n_batches)\n", + " y_batches = np.array_split(y_train[shuffled_idx], n_batches)\n", " for hidden2_batch, y_batch in zip(hidden2_batches, y_batches):\n", " sess.run(training_op, feed_dict={hidden2:hidden2_batch, y:y_batch})\n", "\n", - " accuracy_val = accuracy.eval(feed_dict={hidden2: h2_cache_test, # not shown\n", - " y: mnist.test.labels}) # not shown\n", - " print(epoch, \"Test accuracy:\", accuracy_val) # not shown\n", + " accuracy_val = accuracy.eval(feed_dict={hidden2: h2_cache_valid, # not shown\n", + " y: y_valid}) # not shown\n", + " print(epoch, \"Validation accuracy:\", accuracy_val) # not shown\n", "\n", " save_path = saver.save(sess, \"./my_new_model_final.ckpt\")" ] @@ -1870,10 +1880,8 @@ }, { "cell_type": "code", - "execution_count": 78, - "metadata": { - "collapsed": true - }, + "execution_count": 79, + "metadata": {}, "outputs": [], "source": [ "optimizer = tf.train.MomentumOptimizer(learning_rate=learning_rate,\n", @@ -1889,10 +1897,8 @@ }, { "cell_type": "code", - "execution_count": 79, - "metadata": { - "collapsed": true - }, + "execution_count": 80, + "metadata": {}, "outputs": [], "source": [ "optimizer = tf.train.MomentumOptimizer(learning_rate=learning_rate,\n", @@ -1908,10 +1914,8 @@ }, { "cell_type": "code", - "execution_count": 80, - "metadata": { - "collapsed": true - }, + "execution_count": 81, + "metadata": {}, "outputs": [], "source": [ "optimizer = tf.train.AdagradOptimizer(learning_rate=learning_rate)" @@ -1926,10 +1930,8 @@ }, { "cell_type": "code", - "execution_count": 81, - "metadata": { - "collapsed": true - }, + "execution_count": 82, + "metadata": {}, "outputs": [], "source": [ "optimizer = tf.train.RMSPropOptimizer(learning_rate=learning_rate,\n", @@ -1945,10 +1947,8 @@ }, { "cell_type": "code", - "execution_count": 82, - "metadata": { - "collapsed": true - }, + "execution_count": 83, + "metadata": {}, "outputs": [], "source": [ "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)" @@ -1963,10 +1963,8 @@ }, { "cell_type": "code", - "execution_count": 83, - "metadata": { - "collapsed": true - }, + "execution_count": 84, + "metadata": {}, "outputs": [], "source": [ "reset_graph()\n", @@ -1977,7 +1975,7 @@ "n_outputs = 10\n", "\n", "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n", - "y = tf.placeholder(tf.int64, shape=(None), name=\"y\")\n", + "y = tf.placeholder(tf.int32, shape=(None), name=\"y\")\n", "\n", "with tf.name_scope(\"dnn\"):\n", " hidden1 = tf.layers.dense(X, n_hidden1, activation=tf.nn.relu, name=\"hidden1\")\n", @@ -1995,10 +1993,8 @@ }, { "cell_type": "code", - "execution_count": 84, - "metadata": { - "collapsed": true - }, + "execution_count": 85, + "metadata": {}, "outputs": [], "source": [ "with tf.name_scope(\"train\"): # not shown in the book\n", @@ -2014,10 +2010,8 @@ }, { "cell_type": "code", - "execution_count": 85, - "metadata": { - "collapsed": true - }, + "execution_count": 86, + "metadata": {}, "outputs": [], "source": [ "init = tf.global_variables_initializer()\n", @@ -2026,7 +2020,7 @@ }, { "cell_type": "code", - "execution_count": 86, + "execution_count": 87, "metadata": {}, "outputs": [], "source": [ @@ -2036,12 +2030,10 @@ "with tf.Session() as sess:\n", " init.run()\n", " for epoch in range(n_epochs):\n", - " for iteration in range(mnist.train.num_examples // batch_size):\n", - " X_batch, y_batch = mnist.train.next_batch(batch_size)\n", + " for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size):\n", " sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n", - " accuracy_val = accuracy.eval(feed_dict={X: mnist.test.images,\n", - " y: mnist.test.labels})\n", - " print(epoch, \"Test accuracy:\", accuracy_val)\n", + " accuracy_val = accuracy.eval(feed_dict={X: X_valid, y: y_valid})\n", + " print(epoch, \"Validation accuracy:\", accuracy_val)\n", "\n", " save_path = saver.save(sess, \"./my_model_final.ckpt\")" ] @@ -2069,10 +2061,8 @@ }, { "cell_type": "code", - "execution_count": 87, - "metadata": { - "collapsed": true - }, + "execution_count": 88, + "metadata": {}, "outputs": [], "source": [ "reset_graph()\n", @@ -2082,7 +2072,7 @@ "n_outputs = 10\n", "\n", "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n", - "y = tf.placeholder(tf.int64, shape=(None), name=\"y\")\n", + "y = tf.placeholder(tf.int32, shape=(None), name=\"y\")\n", "\n", "with tf.name_scope(\"dnn\"):\n", " hidden1 = tf.layers.dense(X, n_hidden1, activation=tf.nn.relu, name=\"hidden1\")\n", @@ -2098,10 +2088,8 @@ }, { "cell_type": "code", - "execution_count": 88, - "metadata": { - "collapsed": true - }, + "execution_count": 89, + "metadata": {}, "outputs": [], "source": [ "W1 = tf.get_default_graph().get_tensor_by_name(\"hidden1/kernel:0\")\n", @@ -2126,10 +2114,8 @@ }, { "cell_type": "code", - "execution_count": 89, - "metadata": { - "collapsed": true - }, + "execution_count": 90, + "metadata": {}, "outputs": [], "source": [ "with tf.name_scope(\"eval\"):\n", @@ -2148,7 +2134,7 @@ }, { "cell_type": "code", - "execution_count": 90, + "execution_count": 91, "metadata": { "scrolled": true }, @@ -2160,12 +2146,10 @@ "with tf.Session() as sess:\n", " init.run()\n", " for epoch in range(n_epochs):\n", - " for iteration in range(mnist.train.num_examples // batch_size):\n", - " X_batch, y_batch = mnist.train.next_batch(batch_size)\n", + " for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size):\n", " sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n", - " accuracy_val = accuracy.eval(feed_dict={X: mnist.test.images,\n", - " y: mnist.test.labels})\n", - " print(epoch, \"Test accuracy:\", accuracy_val)\n", + " accuracy_val = accuracy.eval(feed_dict={X: X_valid, y: y_valid})\n", + " print(epoch, \"Validation accuracy:\", accuracy_val)\n", "\n", " save_path = saver.save(sess, \"./my_model_final.ckpt\")" ] @@ -2179,10 +2163,8 @@ }, { "cell_type": "code", - "execution_count": 91, - "metadata": { - "collapsed": true - }, + "execution_count": 92, + "metadata": {}, "outputs": [], "source": [ "reset_graph()\n", @@ -2193,7 +2175,7 @@ "n_outputs = 10\n", "\n", "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n", - "y = tf.placeholder(tf.int64, shape=(None), name=\"y\")" + "y = tf.placeholder(tf.int32, shape=(None), name=\"y\")" ] }, { @@ -2205,10 +2187,8 @@ }, { "cell_type": "code", - "execution_count": 92, - "metadata": { - "collapsed": true - }, + "execution_count": 93, + "metadata": {}, "outputs": [], "source": [ "scale = 0.001" @@ -2216,10 +2196,8 @@ }, { "cell_type": "code", - "execution_count": 93, - "metadata": { - "collapsed": true - }, + "execution_count": 94, + "metadata": {}, "outputs": [], "source": [ "my_dense_layer = partial(\n", @@ -2242,10 +2220,8 @@ }, { "cell_type": "code", - "execution_count": 94, - "metadata": { - "collapsed": true - }, + "execution_count": 95, + "metadata": {}, "outputs": [], "source": [ "with tf.name_scope(\"loss\"): # not shown in the book\n", @@ -2265,10 +2241,8 @@ }, { "cell_type": "code", - "execution_count": 95, - "metadata": { - "collapsed": true - }, + "execution_count": 96, + "metadata": {}, "outputs": [], "source": [ "with tf.name_scope(\"eval\"):\n", @@ -2287,7 +2261,7 @@ }, { "cell_type": "code", - "execution_count": 96, + "execution_count": 97, "metadata": { "scrolled": true }, @@ -2299,12 +2273,10 @@ "with tf.Session() as sess:\n", " init.run()\n", " for epoch in range(n_epochs):\n", - " for iteration in range(mnist.train.num_examples // batch_size):\n", - " X_batch, y_batch = mnist.train.next_batch(batch_size)\n", + " for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size):\n", " sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n", - " accuracy_val = accuracy.eval(feed_dict={X: mnist.test.images,\n", - " y: mnist.test.labels})\n", - " print(epoch, \"Test accuracy:\", accuracy_val)\n", + " accuracy_val = accuracy.eval(feed_dict={X: X_valid, y: y_valid})\n", + " print(epoch, \"Validation accuracy:\", accuracy_val)\n", "\n", " save_path = saver.save(sess, \"./my_model_final.ckpt\")" ] @@ -2327,24 +2299,20 @@ }, { "cell_type": "code", - "execution_count": 97, - "metadata": { - "collapsed": true - }, + "execution_count": 98, + "metadata": {}, "outputs": [], "source": [ "reset_graph()\n", "\n", "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n", - "y = tf.placeholder(tf.int64, shape=(None), name=\"y\")" + "y = tf.placeholder(tf.int32, shape=(None), name=\"y\")" ] }, { "cell_type": "code", - "execution_count": 98, - "metadata": { - "collapsed": true - }, + "execution_count": 99, + "metadata": {}, "outputs": [], "source": [ "training = tf.placeholder_with_default(False, shape=(), name='training')\n", @@ -2364,10 +2332,8 @@ }, { "cell_type": "code", - "execution_count": 99, - "metadata": { - "collapsed": true - }, + "execution_count": 100, + "metadata": {}, "outputs": [], "source": [ "with tf.name_scope(\"loss\"):\n", @@ -2388,7 +2354,7 @@ }, { "cell_type": "code", - "execution_count": 100, + "execution_count": 101, "metadata": { "scrolled": true }, @@ -2400,11 +2366,10 @@ "with tf.Session() as sess:\n", " init.run()\n", " for epoch in range(n_epochs):\n", - " for iteration in range(mnist.train.num_examples // batch_size):\n", - " X_batch, y_batch = mnist.train.next_batch(batch_size)\n", - " sess.run(training_op, feed_dict={training: True, X: X_batch, y: y_batch})\n", - " acc_test = accuracy.eval(feed_dict={X: mnist.test.images, y: mnist.test.labels})\n", - " print(epoch, \"Test accuracy:\", acc_test)\n", + " for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size):\n", + " sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n", + " accuracy_val = accuracy.eval(feed_dict={X: X_valid, y: y_valid})\n", + " print(epoch, \"Validation accuracy:\", accuracy_val)\n", "\n", " save_path = saver.save(sess, \"./my_model_final.ckpt\")" ] @@ -2425,10 +2390,8 @@ }, { "cell_type": "code", - "execution_count": 101, - "metadata": { - "collapsed": true - }, + "execution_count": 102, + "metadata": {}, "outputs": [], "source": [ "reset_graph()\n", @@ -2442,7 +2405,7 @@ "momentum = 0.9\n", "\n", "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n", - "y = tf.placeholder(tf.int64, shape=(None), name=\"y\")\n", + "y = tf.placeholder(tf.int32, shape=(None), name=\"y\")\n", "\n", "with tf.name_scope(\"dnn\"):\n", " hidden1 = tf.layers.dense(X, n_hidden1, activation=tf.nn.relu, name=\"hidden1\")\n", @@ -2471,10 +2434,8 @@ }, { "cell_type": "code", - "execution_count": 102, - "metadata": { - "collapsed": true - }, + "execution_count": 103, + "metadata": {}, "outputs": [], "source": [ "threshold = 1.0\n", @@ -2492,10 +2453,8 @@ }, { "cell_type": "code", - "execution_count": 103, - "metadata": { - "collapsed": true - }, + "execution_count": 104, + "metadata": {}, "outputs": [], "source": [ "weights2 = tf.get_default_graph().get_tensor_by_name(\"hidden2/kernel:0\")\n", @@ -2512,10 +2471,8 @@ }, { "cell_type": "code", - "execution_count": 104, - "metadata": { - "collapsed": true - }, + "execution_count": 105, + "metadata": {}, "outputs": [], "source": [ "init = tf.global_variables_initializer()\n", @@ -2531,10 +2488,8 @@ }, { "cell_type": "code", - "execution_count": 105, - "metadata": { - "collapsed": true - }, + "execution_count": 106, + "metadata": {}, "outputs": [], "source": [ "n_epochs = 20\n", @@ -2543,21 +2498,19 @@ }, { "cell_type": "code", - "execution_count": 106, + "execution_count": 107, "metadata": {}, "outputs": [], "source": [ "with tf.Session() as sess: # not shown in the book\n", " init.run() # not shown\n", " for epoch in range(n_epochs): # not shown\n", - " for iteration in range(mnist.train.num_examples // batch_size): # not shown\n", - " X_batch, y_batch = mnist.train.next_batch(batch_size) # not shown\n", + " for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size): # not shown\n", " sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n", " clip_weights.eval()\n", " clip_weights2.eval() # not shown\n", - " acc_test = accuracy.eval(feed_dict={X: mnist.test.images, # not shown\n", - " y: mnist.test.labels}) # not shown\n", - " print(epoch, \"Test accuracy:\", acc_test) # not shown\n", + " acc_valid = accuracy.eval(feed_dict={X: X_valid, y: y_valid}) # not shown\n", + " print(epoch, \"Validation accuracy:\", acc_valid) # not shown\n", "\n", " save_path = saver.save(sess, \"./my_model_final.ckpt\") # not shown" ] @@ -2571,10 +2524,8 @@ }, { "cell_type": "code", - "execution_count": 107, - "metadata": { - "collapsed": true - }, + "execution_count": 108, + "metadata": {}, "outputs": [], "source": [ "def max_norm_regularizer(threshold, axes=1, name=\"max_norm\",\n", @@ -2596,10 +2547,8 @@ }, { "cell_type": "code", - "execution_count": 108, - "metadata": { - "collapsed": true - }, + "execution_count": 109, + "metadata": {}, "outputs": [], "source": [ "reset_graph()\n", @@ -2613,15 +2562,13 @@ "momentum = 0.9\n", "\n", "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n", - "y = tf.placeholder(tf.int64, shape=(None), name=\"y\")" + "y = tf.placeholder(tf.int32, shape=(None), name=\"y\")" ] }, { "cell_type": "code", - "execution_count": 109, - "metadata": { - "collapsed": true - }, + "execution_count": 110, + "metadata": {}, "outputs": [], "source": [ "max_norm_reg = max_norm_regularizer(threshold=1.0)\n", @@ -2636,10 +2583,8 @@ }, { "cell_type": "code", - "execution_count": 110, - "metadata": { - "collapsed": true - }, + "execution_count": 111, + "metadata": {}, "outputs": [], "source": [ "with tf.name_scope(\"loss\"):\n", @@ -2667,10 +2612,8 @@ }, { "cell_type": "code", - "execution_count": 111, - "metadata": { - "collapsed": true - }, + "execution_count": 112, + "metadata": {}, "outputs": [], "source": [ "n_epochs = 20\n", @@ -2679,7 +2622,7 @@ }, { "cell_type": "code", - "execution_count": 112, + "execution_count": 113, "metadata": { "scrolled": false }, @@ -2690,13 +2633,11 @@ "with tf.Session() as sess:\n", " init.run()\n", " for epoch in range(n_epochs):\n", - " for iteration in range(mnist.train.num_examples // batch_size):\n", - " X_batch, y_batch = mnist.train.next_batch(batch_size)\n", + " for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size):\n", " sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n", " sess.run(clip_all_weights)\n", - " acc_test = accuracy.eval(feed_dict={X: mnist.test.images, # not shown in the book\n", - " y: mnist.test.labels}) # not shown\n", - " print(epoch, \"Test accuracy:\", acc_test) # not shown\n", + " acc_valid = accuracy.eval(feed_dict={X: X_valid, y: y_valid}) # not shown\n", + " print(epoch, \"Validation accuracy:\", acc_valid) # not shown\n", "\n", " save_path = saver.save(sess, \"./my_model_final.ckpt\") # not shown" ] @@ -2754,13 +2695,11 @@ }, { "cell_type": "code", - "execution_count": 113, - "metadata": { - "collapsed": true - }, + "execution_count": 114, + "metadata": {}, "outputs": [], "source": [ - "he_init = tf.contrib.layers.variance_scaling_initializer()\n", + "he_init = tf.variance_scaling_initializer()\n", "\n", "def dnn(inputs, n_hidden_layers=5, n_neurons=100, name=None,\n", " activation=tf.nn.elu, initializer=he_init):\n", @@ -2774,10 +2713,8 @@ }, { "cell_type": "code", - "execution_count": 114, - "metadata": { - "collapsed": true - }, + "execution_count": 115, + "metadata": {}, "outputs": [], "source": [ "n_inputs = 28 * 28 # MNIST\n", @@ -2786,7 +2723,7 @@ "reset_graph()\n", "\n", "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n", - "y = tf.placeholder(tf.int64, shape=(None), name=\"y\")\n", + "y = tf.placeholder(tf.int32, shape=(None), name=\"y\")\n", "\n", "dnn_outputs = dnn(X)\n", "\n", @@ -2817,10 +2754,8 @@ }, { "cell_type": "code", - "execution_count": 115, - "metadata": { - "collapsed": true - }, + "execution_count": 116, + "metadata": {}, "outputs": [], "source": [ "learning_rate = 0.01\n", @@ -2838,23 +2773,6 @@ "saver = tf.train.Saver()" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's fetch the MNIST dataset:" - ] - }, - { - "cell_type": "code", - "execution_count": 116, - "metadata": {}, - "outputs": [], - "source": [ - "from tensorflow.examples.tutorials.mnist import input_data\n", - "mnist = input_data.read_data_sets(\"/tmp/data/\")" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -2865,17 +2783,15 @@ { "cell_type": "code", "execution_count": 117, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ - "X_train1 = mnist.train.images[mnist.train.labels < 5]\n", - "y_train1 = mnist.train.labels[mnist.train.labels < 5]\n", - "X_valid1 = mnist.validation.images[mnist.validation.labels < 5]\n", - "y_valid1 = mnist.validation.labels[mnist.validation.labels < 5]\n", - "X_test1 = mnist.test.images[mnist.test.labels < 5]\n", - "y_test1 = mnist.test.labels[mnist.test.labels < 5]" + "X_train1 = X_train[y_train < 5]\n", + "y_train1 = y_train[y_train < 5]\n", + "X_valid1 = X_valid[y_valid < 5]\n", + "y_valid1 = y_valid[y_valid < 5]\n", + "X_test1 = X_test[y_test < 5]\n", + "y_test1 = y_test[y_test < 5]" ] }, { @@ -2957,9 +2873,7 @@ { "cell_type": "code", "execution_count": 119, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "from sklearn.base import BaseEstimator, ClassifierMixin\n", @@ -3206,16 +3120,14 @@ "}\n", "\n", "rnd_search = RandomizedSearchCV(DNNClassifier(random_state=42), param_distribs, n_iter=50,\n", - " fit_params={\"X_valid\": X_valid1, \"y_valid\": y_valid1, \"n_epochs\": 1000},\n", " random_state=42, verbose=2)\n", - "rnd_search.fit(X_train1, y_train1)\n", + "rnd_search.fit(X_train1, y_train1, X_valid=X_valid1, y_valid=y_valid1, n_epochs=1000)\n", "\n", - "# fit_params as a constructor argument was deprecated in Scikit-Learn version 0.19 and will\n", - "# be removed in version 0.21. Pass fit parameters to the fit() method instead:\n", - "# rnd_search = RandomizedSearchCV(DNNClassifier(random_state=42), param_distribs, n_iter=50,\n", - "# random_state=42, verbose=2)\n", + "# If you have Scikit-Learn 0.18 or earlier, you should upgrade, or use the fit_params argument:\n", "# fit_params={\"X_valid\": X_valid1, \"y_valid\": y_valid1, \"n_epochs\": 1000}\n", - "# rnd_search.fit(X_train1, y_train1, **fit_params)\n" + "# rnd_search = RandomizedSearchCV(DNNClassifier(random_state=42), param_distribs, n_iter=50,\n", + "# fit_params=fit_params, random_state=42, verbose=2)\n", + "# rnd_search.fit(X_train1, y_train1)\n" ] }, { @@ -3254,9 +3166,7 @@ { "cell_type": "code", "execution_count": 125, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "rnd_search.best_estimator_.save(\"./my_best_mnist_model_0_to_4\")" @@ -3584,9 +3494,7 @@ { "cell_type": "code", "execution_count": 139, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "reset_graph()\n", @@ -3611,9 +3519,7 @@ { "cell_type": "code", "execution_count": 140, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "learning_rate = 0.01\n", @@ -3626,9 +3532,7 @@ { "cell_type": "code", "execution_count": 141, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "correct = tf.nn.in_top_k(logits, y, 1)\n", @@ -3662,17 +3566,15 @@ { "cell_type": "code", "execution_count": 142, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ - "X_train2_full = mnist.train.images[mnist.train.labels >= 5]\n", - "y_train2_full = mnist.train.labels[mnist.train.labels >= 5] - 5\n", - "X_valid2_full = mnist.validation.images[mnist.validation.labels >= 5]\n", - "y_valid2_full = mnist.validation.labels[mnist.validation.labels >= 5] - 5\n", - "X_test2 = mnist.test.images[mnist.test.labels >= 5]\n", - "y_test2 = mnist.test.labels[mnist.test.labels >= 5] - 5" + "X_train2_full = X_train[y_train >= 5]\n", + "y_train2_full = y_train[y_train >= 5] - 5\n", + "X_valid2_full = X_valid[y_valid >= 5]\n", + "y_valid2_full = y_valid[y_valid >= 5] - 5\n", + "X_test2 = X_test[y_test >= 5]\n", + "y_test2 = y_test[y_test >= 5] - 5" ] }, { @@ -3685,9 +3587,7 @@ { "cell_type": "code", "execution_count": 143, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "def sample_n_instances_per_class(X, y, n=100):\n", @@ -3704,9 +3604,7 @@ { "cell_type": "code", "execution_count": 144, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "X_train2, y_train2 = sample_n_instances_per_class(X_train2_full, y_train2_full, n=100)\n", @@ -3801,9 +3699,7 @@ { "cell_type": "code", "execution_count": 146, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "hidden5_out = tf.get_default_graph().get_tensor_by_name(\"hidden5_out:0\")" @@ -3893,9 +3789,7 @@ { "cell_type": "code", "execution_count": 148, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "reset_graph()\n", @@ -3926,9 +3820,7 @@ { "cell_type": "code", "execution_count": 149, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "learning_rate = 0.01\n", @@ -4013,9 +3905,7 @@ { "cell_type": "code", "execution_count": 151, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "learning_rate = 0.01\n", @@ -4079,9 +3969,7 @@ { "cell_type": "code", "execution_count": 153, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "learning_rate = 0.01\n", @@ -4207,9 +4095,7 @@ { "cell_type": "code", "execution_count": 157, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "n_inputs = 28 * 28 # MNIST\n", @@ -4230,9 +4116,7 @@ { "cell_type": "code", "execution_count": 158, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "y = tf.placeholder(tf.int32, shape=[None, 1])" @@ -4248,9 +4132,7 @@ { "cell_type": "code", "execution_count": 159, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "dnn1 = dnn(X1, name=\"DNN_A\")\n", @@ -4267,9 +4149,7 @@ { "cell_type": "code", "execution_count": 160, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "dnn_outputs = tf.concat([dnn1, dnn2], axis=1)" @@ -4326,9 +4206,7 @@ { "cell_type": "code", "execution_count": 164, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "hidden = tf.layers.dense(dnn_outputs, units=10, activation=tf.nn.elu, kernel_initializer=he_init)\n", @@ -4346,9 +4224,7 @@ { "cell_type": "code", "execution_count": 165, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "y_pred = tf.cast(tf.greater_equal(logits, 0), tf.int32)" @@ -4364,9 +4240,7 @@ { "cell_type": "code", "execution_count": 166, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "y_as_float = tf.cast(y, tf.float32)\n", @@ -4384,9 +4258,7 @@ { "cell_type": "code", "execution_count": 167, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "learning_rate = 0.01\n", @@ -4406,9 +4278,7 @@ { "cell_type": "code", "execution_count": 168, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "y_pred_correct = tf.equal(y_pred, y)\n", @@ -4425,9 +4295,7 @@ { "cell_type": "code", "execution_count": 169, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "init = tf.global_variables_initializer()\n", @@ -4452,19 +4320,17 @@ { "cell_type": "code", "execution_count": 170, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ - "X_train1 = mnist.train.images\n", - "y_train1 = mnist.train.labels\n", + "X_train1 = X_train\n", + "y_train1 = y_train\n", "\n", - "X_train2 = mnist.validation.images\n", - "y_train2 = mnist.validation.labels\n", + "X_train2 = X_valid\n", + "y_train2 = y_valid\n", "\n", - "X_test = mnist.test.images\n", - "y_test = mnist.test.labels" + "X_test = X_test\n", + "y_test = y_test" ] }, { @@ -4477,9 +4343,7 @@ { "cell_type": "code", "execution_count": 171, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "def generate_batch(images, labels, batch_size):\n", @@ -4513,9 +4377,7 @@ { "cell_type": "code", "execution_count": 172, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "batch_size = 5\n", @@ -4602,9 +4464,7 @@ { "cell_type": "code", "execution_count": 176, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "X_test1, y_test1 = generate_batch(X_test, y_test, batch_size=len(X_test))" @@ -4629,7 +4489,7 @@ "with tf.Session() as sess:\n", " init.run()\n", " for epoch in range(n_epochs):\n", - " for iteration in range(mnist.train.num_examples // batch_size):\n", + " for iteration in range(len(X_train1) // batch_size):\n", " X_batch, y_batch = generate_batch(X_train1, y_train1, batch_size)\n", " loss_val, _ = sess.run([loss, training_op], feed_dict={X: X_batch, y: y_batch})\n", " print(epoch, \"Train loss:\", loss_val)\n", @@ -4667,9 +4527,7 @@ { "cell_type": "code", "execution_count": 178, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "reset_graph()\n", @@ -4744,9 +4602,7 @@ { "cell_type": "code", "execution_count": 180, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "reset_graph()\n", @@ -4814,9 +4670,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [] } @@ -4837,7 +4691,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.4" + "version": "3.5.2" }, "nav_menu": { "height": "360px",