From d8176ec2cb972bf0895de7232a15e455ebc4981e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Geron?= Date: Fri, 17 Feb 2017 11:51:26 +0100 Subject: [PATCH] Upgrade notebooks to TensorFlow 1.0.0 --- 02_end_to_end_machine_learning_project.ipynb | 394 +++++++-- 03_classification.ipynb | 385 +++++++-- 04_training_linear_models.ipynb | 269 ++++-- 05_support_vector_machines.ipynb | 233 +++-- 06_decision_trees.ipynb | 121 ++- 07_ensemble_learning_and_random_forests.ipynb | 199 ++++- 08_dimensionality_reduction.ipynb | 367 ++++++-- 09_up_and_running_with_tensorflow.ipynb | 473 +++++++--- ...uction_to_artificial_neural_networks.ipynb | 170 +++- 11_deep_learning.ipynb | 251 ++++-- 12_distributed_tensorflow.ipynb | 137 ++- 13_convolutional_neural_networks.ipynb | 191 +++- 14_recurrent_neural_networks.ipynb | 602 +++++++++---- 15_autoencoders.ipynb | 497 ++++++++--- 16_reinforcement_learning.ipynb | 817 ++++++++++-------- Dockerfile | 8 - nets/README.md | 5 - nets/__init__.py | 1 - nets/alexnet.py | 125 --- nets/alexnet_test.py | 145 ---- nets/cifarnet.py | 112 --- nets/inception.py | 33 - nets/inception_resnet_v2.py | 280 ------ nets/inception_resnet_v2_test.py | 136 --- nets/inception_v1.py | 340 -------- nets/inception_v1_test.py | 210 ----- nets/inception_v2.py | 545 ------------ nets/inception_v2_test.py | 262 ------ nets/inception_v3.py | 591 ------------- nets/inception_v3_test.py | 292 ------- nets/lenet.py | 93 -- nets/nets_factory.py | 107 --- nets/nets_factory_test.py | 46 - nets/overfeat.py | 118 --- nets/overfeat_test.py | 145 ---- nets/resnet_utils.py | 254 ------ nets/resnet_v1.py | 295 ------- nets/resnet_v1_test.py | 450 ---------- nets/resnet_v2.py | 302 ------- nets/resnet_v2_test.py | 453 ---------- nets/vgg.py | 244 ------ nets/vgg_test.py | 455 ---------- requirements.txt | 4 +- 43 files changed, 3615 insertions(+), 7542 deletions(-) delete mode 100644 nets/README.md delete mode 100644 nets/__init__.py delete mode 100644 nets/alexnet.py delete mode 100644 nets/alexnet_test.py delete mode 100644 nets/cifarnet.py delete mode 100644 nets/inception.py delete mode 100644 nets/inception_resnet_v2.py delete mode 100644 nets/inception_resnet_v2_test.py delete mode 100644 nets/inception_v1.py delete mode 100644 nets/inception_v1_test.py delete mode 100644 nets/inception_v2.py delete mode 100644 nets/inception_v2_test.py delete mode 100644 nets/inception_v3.py delete mode 100644 nets/inception_v3_test.py delete mode 100644 nets/lenet.py delete mode 100644 nets/nets_factory.py delete mode 100644 nets/nets_factory_test.py delete mode 100644 nets/overfeat.py delete mode 100644 nets/overfeat_test.py delete mode 100644 nets/resnet_utils.py delete mode 100644 nets/resnet_v1.py delete mode 100644 nets/resnet_v1_test.py delete mode 100644 nets/resnet_v2.py delete mode 100644 nets/resnet_v2_test.py delete mode 100644 nets/vgg.py delete mode 100644 nets/vgg_test.py diff --git a/02_end_to_end_machine_learning_project.ipynb b/02_end_to_end_machine_learning_project.ipynb index 1ee28e6..8bf4fa7 100644 --- a/02_end_to_end_machine_learning_project.ipynb +++ b/02_end_to_end_machine_learning_project.ipynb @@ -2,7 +2,10 @@ "cells": [ { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "**Chapter 2 – End-to-end Machine Learning project**\n", "\n", @@ -13,14 +16,20 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Setup" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "First, let's make sure this notebook works well in both python 2 and 3, import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures:" ] @@ -29,7 +38,9 @@ "cell_type": "code", "execution_count": 1, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -66,7 +77,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Get the data" ] @@ -75,7 +89,9 @@ "cell_type": "code", "execution_count": 2, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -86,7 +102,9 @@ "cell_type": "code", "execution_count": 3, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -111,7 +129,9 @@ "cell_type": "code", "execution_count": 4, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -122,7 +142,9 @@ "cell_type": "code", "execution_count": 5, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -137,7 +159,9 @@ "cell_type": "code", "execution_count": 6, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -149,7 +173,9 @@ "cell_type": "code", "execution_count": 7, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -160,7 +186,9 @@ "cell_type": "code", "execution_count": 8, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -171,7 +199,9 @@ "cell_type": "code", "execution_count": 9, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -182,7 +212,9 @@ "cell_type": "code", "execution_count": 10, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -197,7 +229,9 @@ "cell_type": "code", "execution_count": 11, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -217,7 +251,9 @@ "cell_type": "code", "execution_count": 12, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -229,7 +265,9 @@ "cell_type": "code", "execution_count": 13, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -248,7 +286,9 @@ "cell_type": "code", "execution_count": 14, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -261,7 +301,9 @@ "cell_type": "code", "execution_count": 15, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -275,7 +317,9 @@ "cell_type": "code", "execution_count": 16, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -286,7 +330,9 @@ "cell_type": "code", "execution_count": 17, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -299,7 +345,9 @@ "cell_type": "code", "execution_count": 18, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -315,7 +363,9 @@ "cell_type": "code", "execution_count": 19, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -337,7 +387,9 @@ "cell_type": "code", "execution_count": 20, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -348,7 +400,9 @@ "cell_type": "code", "execution_count": 21, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -358,7 +412,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Discover and visualize the data to gain insights" ] @@ -367,7 +424,9 @@ "cell_type": "code", "execution_count": 22, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -378,7 +437,9 @@ "cell_type": "code", "execution_count": 23, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -390,7 +451,9 @@ "cell_type": "code", "execution_count": 24, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -403,6 +466,8 @@ "execution_count": 25, "metadata": { "collapsed": false, + "deletable": true, + "editable": true, "scrolled": true }, "outputs": [], @@ -421,7 +486,9 @@ "cell_type": "code", "execution_count": 26, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -451,7 +518,9 @@ "cell_type": "code", "execution_count": 27, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -463,7 +532,9 @@ "cell_type": "code", "execution_count": 28, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -478,7 +549,9 @@ "cell_type": "code", "execution_count": 29, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -494,7 +567,9 @@ "cell_type": "code", "execution_count": 30, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -507,7 +582,9 @@ "cell_type": "code", "execution_count": 31, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -519,7 +596,9 @@ "cell_type": "code", "execution_count": 32, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -533,7 +612,9 @@ "cell_type": "code", "execution_count": 33, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -542,7 +623,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Prepare the data for Machine Learning algorithms" ] @@ -551,7 +635,9 @@ "cell_type": "code", "execution_count": 34, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -563,7 +649,9 @@ "cell_type": "code", "execution_count": 35, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -575,7 +663,9 @@ "cell_type": "code", "execution_count": 36, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -586,7 +676,9 @@ "cell_type": "code", "execution_count": 37, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -598,7 +690,9 @@ "cell_type": "code", "execution_count": 38, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -612,7 +706,9 @@ "cell_type": "code", "execution_count": 39, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -630,7 +726,9 @@ "cell_type": "code", "execution_count": 40, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -641,7 +739,9 @@ "cell_type": "code", "execution_count": 41, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -652,7 +752,9 @@ "cell_type": "code", "execution_count": 42, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -663,7 +765,9 @@ "cell_type": "code", "execution_count": 43, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -675,7 +779,9 @@ "cell_type": "code", "execution_count": 44, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -691,7 +797,9 @@ "cell_type": "code", "execution_count": 45, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -702,7 +810,9 @@ "cell_type": "code", "execution_count": 46, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -717,7 +827,9 @@ "cell_type": "code", "execution_count": 47, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -728,7 +840,9 @@ "cell_type": "code", "execution_count": 48, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -742,7 +856,9 @@ "cell_type": "code", "execution_count": 49, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -775,7 +891,9 @@ "cell_type": "code", "execution_count": 50, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -795,7 +913,9 @@ "cell_type": "code", "execution_count": 51, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -834,7 +954,9 @@ "cell_type": "code", "execution_count": 52, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -846,7 +968,9 @@ "cell_type": "code", "execution_count": 53, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -855,7 +979,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Prepare the data for Machine Learning algorithms" ] @@ -864,7 +991,9 @@ "cell_type": "code", "execution_count": 54, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -878,7 +1007,9 @@ "cell_type": "code", "execution_count": 55, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -895,7 +1026,9 @@ "cell_type": "code", "execution_count": 56, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -911,7 +1044,9 @@ "cell_type": "code", "execution_count": 57, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -925,7 +1060,9 @@ "cell_type": "code", "execution_count": 58, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -941,7 +1078,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Fine-tune your model" ] @@ -950,7 +1090,9 @@ "cell_type": "code", "execution_count": 59, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -965,7 +1107,9 @@ "cell_type": "code", "execution_count": 60, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -981,7 +1125,9 @@ "cell_type": "code", "execution_count": 61, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -995,7 +1141,9 @@ "cell_type": "code", "execution_count": 62, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1013,7 +1161,9 @@ "cell_type": "code", "execution_count": 63, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1029,7 +1179,9 @@ "cell_type": "code", "execution_count": 64, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1041,7 +1193,9 @@ "cell_type": "code", "execution_count": 65, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1059,7 +1213,9 @@ "cell_type": "code", "execution_count": 66, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1079,7 +1235,9 @@ "cell_type": "code", "execution_count": 67, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1090,7 +1248,9 @@ "cell_type": "code", "execution_count": 68, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1101,7 +1261,9 @@ "cell_type": "code", "execution_count": 69, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1114,7 +1276,9 @@ "cell_type": "code", "execution_count": 70, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1125,7 +1289,9 @@ "cell_type": "code", "execution_count": 71, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1147,7 +1313,9 @@ "cell_type": "code", "execution_count": 72, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1160,7 +1328,9 @@ "cell_type": "code", "execution_count": 73, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1172,7 +1342,9 @@ "cell_type": "code", "execution_count": 74, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1186,7 +1358,9 @@ "cell_type": "code", "execution_count": 75, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1205,14 +1379,20 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Extra material" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Label Binarizer hack\n", "`LabelBinarizer`'s `fit_transform()` method only accepts one parameter `y` (because it was meant for labels, not predictors), so it does not work in a pipeline where the final estimator is a supervised estimator because in this case its `fit()` method takes two parameters `X` and `y`.\n", @@ -1224,7 +1404,9 @@ "cell_type": "code", "execution_count": 76, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1247,7 +1429,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Model persistence using joblib" ] @@ -1256,7 +1441,9 @@ "cell_type": "code", "execution_count": 77, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1267,7 +1454,9 @@ "cell_type": "code", "execution_count": 78, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1278,7 +1467,9 @@ "cell_type": "code", "execution_count": 79, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1288,7 +1479,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Example SciPy distributions for `RandomizedSearchCV`" ] @@ -1297,7 +1491,9 @@ "cell_type": "code", "execution_count": 80, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1312,14 +1508,20 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Exercise solutions" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "**Coming soon**" ] @@ -1328,7 +1530,9 @@ "cell_type": "code", "execution_count": null, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [] @@ -1350,7 +1554,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.5.2" + "version": "3.5.2+" }, "nav_menu": { "height": "279px", diff --git a/03_classification.ipynb b/03_classification.ipynb index 1140701..ca9591b 100644 --- a/03_classification.ipynb +++ b/03_classification.ipynb @@ -2,7 +2,10 @@ "cells": [ { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "**Chapter 3 – Classification**\n", "\n", @@ -11,14 +14,20 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Setup" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "First, let's make sure this notebook works well in both python 2 and 3, import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures:" ] @@ -27,7 +36,9 @@ "cell_type": "code", "execution_count": 1, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -64,7 +75,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# MNIST" ] @@ -73,7 +87,9 @@ "cell_type": "code", "execution_count": 2, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -85,7 +101,9 @@ "cell_type": "code", "execution_count": 3, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -96,7 +114,9 @@ "cell_type": "code", "execution_count": 4, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -108,7 +128,9 @@ "cell_type": "code", "execution_count": 5, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -119,7 +141,9 @@ "cell_type": "code", "execution_count": 6, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -130,7 +154,9 @@ "cell_type": "code", "execution_count": 7, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -151,7 +177,9 @@ "cell_type": "code", "execution_count": 8, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -182,7 +210,9 @@ "cell_type": "code", "execution_count": 9, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -193,7 +223,9 @@ "cell_type": "code", "execution_count": 10, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -204,7 +236,9 @@ "cell_type": "code", "execution_count": 11, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -214,7 +248,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Binary classifier" ] @@ -223,7 +260,9 @@ "cell_type": "code", "execution_count": 12, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -235,7 +274,9 @@ "cell_type": "code", "execution_count": 13, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -249,7 +290,9 @@ "cell_type": "code", "execution_count": 14, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -260,7 +303,9 @@ "cell_type": "code", "execution_count": 15, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -272,7 +317,9 @@ "cell_type": "code", "execution_count": 16, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -298,7 +345,9 @@ "cell_type": "code", "execution_count": 17, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -314,7 +363,9 @@ "cell_type": "code", "execution_count": 18, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -326,7 +377,9 @@ "cell_type": "code", "execution_count": 19, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -339,7 +392,9 @@ "cell_type": "code", "execution_count": 20, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -352,7 +407,9 @@ "cell_type": "code", "execution_count": 21, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -365,7 +422,9 @@ "cell_type": "code", "execution_count": 22, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -376,7 +435,9 @@ "cell_type": "code", "execution_count": 23, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -387,7 +448,9 @@ "cell_type": "code", "execution_count": 24, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -398,7 +461,9 @@ "cell_type": "code", "execution_count": 25, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -410,7 +475,9 @@ "cell_type": "code", "execution_count": 26, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -421,7 +488,9 @@ "cell_type": "code", "execution_count": 27, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -433,7 +502,9 @@ "cell_type": "code", "execution_count": 28, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -446,7 +517,9 @@ "cell_type": "code", "execution_count": 29, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -459,7 +532,9 @@ "cell_type": "code", "execution_count": 30, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -470,7 +545,9 @@ "cell_type": "code", "execution_count": 31, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -483,7 +560,9 @@ "cell_type": "code", "execution_count": 32, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -505,7 +584,9 @@ "cell_type": "code", "execution_count": 33, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -516,7 +597,9 @@ "cell_type": "code", "execution_count": 34, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -527,7 +610,9 @@ "cell_type": "code", "execution_count": 35, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -538,7 +623,9 @@ "cell_type": "code", "execution_count": 36, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -549,7 +636,9 @@ "cell_type": "code", "execution_count": 37, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -567,7 +656,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# ROC curves" ] @@ -576,7 +668,9 @@ "cell_type": "code", "execution_count": 38, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -589,7 +683,9 @@ "cell_type": "code", "execution_count": 39, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -610,7 +706,9 @@ "cell_type": "code", "execution_count": 40, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -623,7 +721,9 @@ "cell_type": "code", "execution_count": 41, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -645,7 +745,9 @@ "cell_type": "code", "execution_count": 42, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -656,7 +758,9 @@ "cell_type": "code", "execution_count": 43, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -668,7 +772,9 @@ "cell_type": "code", "execution_count": 44, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -677,7 +783,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Multiclass classification" ] @@ -686,7 +795,9 @@ "cell_type": "code", "execution_count": 45, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -698,7 +809,9 @@ "cell_type": "code", "execution_count": 46, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -710,7 +823,9 @@ "cell_type": "code", "execution_count": 47, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -721,7 +836,9 @@ "cell_type": "code", "execution_count": 48, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -732,7 +849,9 @@ "cell_type": "code", "execution_count": 49, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -746,7 +865,9 @@ "cell_type": "code", "execution_count": 50, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -757,7 +878,9 @@ "cell_type": "code", "execution_count": 51, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -769,7 +892,9 @@ "cell_type": "code", "execution_count": 52, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -780,7 +905,9 @@ "cell_type": "code", "execution_count": 53, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -791,7 +918,9 @@ "cell_type": "code", "execution_count": 54, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -805,7 +934,9 @@ "cell_type": "code", "execution_count": 55, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -818,7 +949,9 @@ "cell_type": "code", "execution_count": 56, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -838,7 +971,9 @@ "cell_type": "code", "execution_count": 57, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -854,7 +989,9 @@ "cell_type": "code", "execution_count": 58, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -879,7 +1016,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Multilabel classification" ] @@ -888,7 +1028,9 @@ "cell_type": "code", "execution_count": 59, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -906,7 +1048,9 @@ "cell_type": "code", "execution_count": 60, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -917,7 +1061,9 @@ "cell_type": "code", "execution_count": 61, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -928,7 +1074,9 @@ "cell_type": "code", "execution_count": 62, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -937,7 +1085,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Multioutput classification" ] @@ -946,7 +1097,9 @@ "cell_type": "code", "execution_count": 63, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -962,7 +1115,9 @@ "cell_type": "code", "execution_count": 64, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -977,7 +1132,9 @@ "cell_type": "code", "execution_count": 65, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -988,7 +1145,9 @@ "cell_type": "code", "execution_count": 66, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1000,14 +1159,20 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Extra material" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Dummy (ie. random) classifier" ] @@ -1016,7 +1181,9 @@ "cell_type": "code", "execution_count": 67, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1031,6 +1198,8 @@ "execution_count": 68, "metadata": { "collapsed": false, + "deletable": true, + "editable": true, "scrolled": true }, "outputs": [], @@ -1041,7 +1210,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## KNN classifier" ] @@ -1050,7 +1222,9 @@ "cell_type": "code", "execution_count": 69, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1063,7 +1237,9 @@ "cell_type": "code", "execution_count": 70, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1074,7 +1250,9 @@ "cell_type": "code", "execution_count": 71, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1086,7 +1264,9 @@ "cell_type": "code", "execution_count": 72, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1101,7 +1281,9 @@ "cell_type": "code", "execution_count": 73, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1121,7 +1303,9 @@ "cell_type": "code", "execution_count": 74, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1132,7 +1316,9 @@ "cell_type": "code", "execution_count": 75, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1143,7 +1329,9 @@ "cell_type": "code", "execution_count": 76, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1154,7 +1342,9 @@ "cell_type": "code", "execution_count": 77, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1166,7 +1356,9 @@ "cell_type": "code", "execution_count": 78, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1176,7 +1368,9 @@ { "cell_type": "markdown", "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "source": [ "# Exercise solutions" @@ -1184,7 +1378,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "**Coming soon**" ] @@ -1193,7 +1390,9 @@ "cell_type": "code", "execution_count": null, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [] @@ -1215,7 +1414,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.5.2" + "version": "3.5.2+" }, "nav_menu": {}, "toc": { diff --git a/04_training_linear_models.ipynb b/04_training_linear_models.ipynb index fbf8e1b..777c8f4 100644 --- a/04_training_linear_models.ipynb +++ b/04_training_linear_models.ipynb @@ -2,37 +2,51 @@ "cells": [ { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "**Chapter 4 – Training Linear Models**" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "_This notebook contains all the sample code and solutions to the exercices in chapter 4._" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Setup" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "First, let's make sure this notebook works well in both python 2 and 3, import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures:" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -69,7 +83,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Linear regression using the Normal Equation" ] @@ -78,7 +95,9 @@ "cell_type": "code", "execution_count": 2, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -90,7 +109,9 @@ "cell_type": "code", "execution_count": 3, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -106,7 +127,9 @@ "cell_type": "code", "execution_count": 4, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -120,7 +143,9 @@ "cell_type": "code", "execution_count": 5, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -131,7 +156,9 @@ "cell_type": "code", "execution_count": 6, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -145,7 +172,9 @@ "cell_type": "code", "execution_count": 7, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -163,7 +192,9 @@ "cell_type": "code", "execution_count": 8, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -177,7 +208,9 @@ "cell_type": "code", "execution_count": 9, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -186,7 +219,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Linear regression using batch gradient descent" ] @@ -195,7 +231,9 @@ "cell_type": "code", "execution_count": 10, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -233,7 +271,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Stochastic Gradient Descent" ] @@ -242,7 +283,9 @@ "cell_type": "code", "execution_count": 11, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -285,7 +328,9 @@ "cell_type": "code", "execution_count": 12, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -296,7 +341,9 @@ "cell_type": "code", "execution_count": 13, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -309,7 +356,9 @@ "cell_type": "code", "execution_count": 14, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -318,7 +367,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Mini-batch gradient descent" ] @@ -327,7 +379,9 @@ "cell_type": "code", "execution_count": 15, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -362,7 +416,9 @@ "cell_type": "code", "execution_count": 16, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -373,7 +429,9 @@ "cell_type": "code", "execution_count": 17, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -386,7 +444,9 @@ "cell_type": "code", "execution_count": 18, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -404,7 +464,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Polynomial regression" ] @@ -413,7 +476,9 @@ "cell_type": "code", "execution_count": 19, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -430,7 +495,9 @@ "cell_type": "code", "execution_count": 20, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -446,7 +513,9 @@ "cell_type": "code", "execution_count": 21, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -460,7 +529,9 @@ "cell_type": "code", "execution_count": 22, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -471,7 +542,9 @@ "cell_type": "code", "execution_count": 23, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -484,7 +557,9 @@ "cell_type": "code", "execution_count": 24, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -505,7 +580,9 @@ "cell_type": "code", "execution_count": 25, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -538,7 +615,9 @@ "cell_type": "code", "execution_count": 26, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -572,7 +651,9 @@ "cell_type": "code", "execution_count": 27, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -591,7 +672,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Regularized models" ] @@ -600,7 +684,9 @@ "cell_type": "code", "execution_count": 28, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -645,7 +731,9 @@ "cell_type": "code", "execution_count": 29, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -659,20 +747,24 @@ "cell_type": "code", "execution_count": 30, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ "sgd_reg = SGDRegressor(penalty=\"l2\", random_state=42)\n", "sgd_reg.fit(X, y.ravel())\n", - "ridge_reg.predict([[1.5]])" + "sgd_reg.predict([[1.5]])" ] }, { "cell_type": "code", "execution_count": 31, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -685,7 +777,9 @@ "cell_type": "code", "execution_count": 32, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -706,7 +800,9 @@ "cell_type": "code", "execution_count": 33, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -720,7 +816,9 @@ "cell_type": "code", "execution_count": 34, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -735,6 +833,8 @@ "execution_count": 35, "metadata": { "collapsed": false, + "deletable": true, + "editable": true, "scrolled": true }, "outputs": [], @@ -796,7 +896,9 @@ "cell_type": "code", "execution_count": 36, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -824,7 +926,9 @@ "cell_type": "code", "execution_count": 37, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -837,7 +941,9 @@ "cell_type": "code", "execution_count": 38, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -866,7 +972,9 @@ "cell_type": "code", "execution_count": 39, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -930,7 +1038,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Logistic regression" ] @@ -939,7 +1050,9 @@ "cell_type": "code", "execution_count": 40, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -962,7 +1075,9 @@ "cell_type": "code", "execution_count": 41, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -975,7 +1090,9 @@ "cell_type": "code", "execution_count": 42, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -986,7 +1103,9 @@ "cell_type": "code", "execution_count": 43, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1023,7 +1142,9 @@ "cell_type": "code", "execution_count": 44, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1034,7 +1155,9 @@ "cell_type": "code", "execution_count": 45, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1045,7 +1168,9 @@ "cell_type": "code", "execution_count": 46, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1091,7 +1216,9 @@ "cell_type": "code", "execution_count": 47, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1139,7 +1266,9 @@ "cell_type": "code", "execution_count": 48, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1150,7 +1279,9 @@ "cell_type": "code", "execution_count": 49, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1159,14 +1290,20 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Exercise solutions" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "**Coming soon**" ] @@ -1175,7 +1312,9 @@ "cell_type": "code", "execution_count": null, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [] @@ -1197,7 +1336,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.5.2" + "version": "3.5.2+" }, "nav_menu": {}, "toc": { diff --git a/05_support_vector_machines.ipynb b/05_support_vector_machines.ipynb index 42937b0..e76258b 100644 --- a/05_support_vector_machines.ipynb +++ b/05_support_vector_machines.ipynb @@ -2,7 +2,10 @@ "cells": [ { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "**Chapter 5 – Support Vector Machines**\n", "\n", @@ -11,14 +14,20 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Setup" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "First, let's make sure this notebook works well in both python 2 and 3, import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures:" ] @@ -27,7 +36,9 @@ "cell_type": "code", "execution_count": 1, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -64,7 +75,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Large margin classification" ] @@ -73,7 +87,9 @@ "cell_type": "code", "execution_count": 2, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -97,7 +113,9 @@ "cell_type": "code", "execution_count": 3, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -152,7 +170,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Sensitivity to feature scales" ] @@ -161,7 +182,9 @@ "cell_type": "code", "execution_count": 4, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -198,7 +221,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Sensitivity to outliers" ] @@ -207,7 +233,9 @@ "cell_type": "code", "execution_count": 5, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -258,7 +286,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Large margin *vs* margin violations" ] @@ -267,7 +298,9 @@ "cell_type": "code", "execution_count": 6, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -303,7 +336,9 @@ "cell_type": "code", "execution_count": 7, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -329,7 +364,9 @@ "cell_type": "code", "execution_count": 8, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -358,7 +395,9 @@ { "cell_type": "markdown", "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "source": [ "# Non-linear classification" @@ -368,7 +407,9 @@ "cell_type": "code", "execution_count": 9, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -409,7 +450,9 @@ "cell_type": "code", "execution_count": 10, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -432,7 +475,9 @@ "cell_type": "code", "execution_count": 11, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -468,7 +513,9 @@ "cell_type": "code", "execution_count": 12, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -506,6 +553,8 @@ "execution_count": 13, "metadata": { "collapsed": false, + "deletable": true, + "editable": true, "scrolled": true }, "outputs": [], @@ -574,7 +623,9 @@ "cell_type": "code", "execution_count": 14, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -588,7 +639,9 @@ "cell_type": "code", "execution_count": 15, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -604,6 +657,8 @@ "execution_count": 16, "metadata": { "collapsed": false, + "deletable": true, + "editable": true, "scrolled": true }, "outputs": [], @@ -638,7 +693,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Regression\n" ] @@ -647,7 +705,9 @@ "cell_type": "code", "execution_count": 17, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -679,7 +739,9 @@ "cell_type": "code", "execution_count": 18, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -718,7 +780,9 @@ "cell_type": "code", "execution_count": 19, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -739,7 +803,9 @@ "cell_type": "code", "execution_count": 20, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -757,7 +823,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Under the hood" ] @@ -766,7 +835,9 @@ "cell_type": "code", "execution_count": 21, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -779,7 +850,9 @@ "cell_type": "code", "execution_count": 22, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -822,7 +895,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Small weight vector results in a large margin" ] @@ -831,7 +907,9 @@ "cell_type": "code", "execution_count": 23, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -867,7 +945,9 @@ "cell_type": "code", "execution_count": 24, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -885,7 +965,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Hinge loss" ] @@ -894,7 +977,9 @@ "cell_type": "code", "execution_count": 25, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -916,14 +1001,20 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Extra material" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Training time" ] @@ -932,7 +1023,9 @@ "cell_type": "code", "execution_count": 26, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -945,7 +1038,9 @@ "cell_type": "code", "execution_count": 27, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -968,16 +1063,21 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Identical linear classifiers" ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 28, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1000,14 +1100,17 @@ "svm_clf.fit(X_scaled, y)\n", "lin_clf.fit(X_scaled, y)\n", "\n", - "print(\"SGDClassifier(alpha={}): \".format(sgd_clf1.alpha), sgd_clf.intercept_, sgd_clf.coef_)\n", + "print(\"SGDClassifier(alpha={}): \".format(sgd_clf.alpha), sgd_clf.intercept_, sgd_clf.coef_)\n", "print(\"SVC: \", svm_clf.intercept_, svm_clf.coef_)\n", "print(\"LinearSVC: \", lin_clf.intercept_, lin_clf.coef_)" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Linear SVM classifier implementation using Batch Gradient Descent" ] @@ -1016,7 +1119,9 @@ "cell_type": "code", "execution_count": 29, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1029,7 +1134,9 @@ "cell_type": "code", "execution_count": 30, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1096,7 +1203,9 @@ "cell_type": "code", "execution_count": 31, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1108,7 +1217,9 @@ "cell_type": "code", "execution_count": 32, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1119,7 +1230,9 @@ "cell_type": "code", "execution_count": 33, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1132,7 +1245,9 @@ "cell_type": "code", "execution_count": 34, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1161,6 +1276,8 @@ "execution_count": 35, "metadata": { "collapsed": false, + "deletable": true, + "editable": true, "scrolled": true }, "outputs": [], @@ -1192,14 +1309,20 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Exercise solutions" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "**Coming soon**" ] @@ -1208,7 +1331,9 @@ "cell_type": "code", "execution_count": null, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [] @@ -1230,7 +1355,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.5.1" + "version": "3.5.2+" }, "nav_menu": {}, "toc": { diff --git a/06_decision_trees.ipynb b/06_decision_trees.ipynb index 8e417ab..a152c93 100644 --- a/06_decision_trees.ipynb +++ b/06_decision_trees.ipynb @@ -2,28 +2,40 @@ "cells": [ { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "**Chapter 6 – Decision Trees**" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "_This notebook contains all the sample code and solutions to the exercices in chapter 6._" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Setup" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "First, let's make sure this notebook works well in both python 2 and 3, import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures:" ] @@ -32,7 +44,9 @@ "cell_type": "code", "execution_count": 1, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -71,7 +85,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Training and visualizing" ] @@ -80,7 +97,9 @@ "cell_type": "code", "execution_count": 2, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -99,7 +118,9 @@ "cell_type": "code", "execution_count": 3, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -117,7 +138,9 @@ "cell_type": "code", "execution_count": 4, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -164,7 +187,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Predicting classes and class probabilities" ] @@ -173,7 +199,9 @@ "cell_type": "code", "execution_count": 5, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -184,7 +212,9 @@ "cell_type": "code", "execution_count": 6, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -193,7 +223,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Sensitivity to training set details" ] @@ -202,7 +235,9 @@ "cell_type": "code", "execution_count": 7, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -213,7 +248,9 @@ "cell_type": "code", "execution_count": 8, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -229,7 +266,9 @@ "cell_type": "code", "execution_count": 9, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -248,7 +287,9 @@ "cell_type": "code", "execution_count": 10, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -276,7 +317,9 @@ "cell_type": "code", "execution_count": 11, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -297,7 +340,9 @@ "cell_type": "code", "execution_count": 12, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -326,16 +371,21 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Regression trees" ] }, { "cell_type": "code", - "execution_count": 145, + "execution_count": 13, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -389,9 +439,11 @@ }, { "cell_type": "code", - "execution_count": 131, + "execution_count": 14, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -406,9 +458,11 @@ }, { "cell_type": "code", - "execution_count": 144, + "execution_count": 15, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -446,7 +500,9 @@ { "cell_type": "markdown", "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "source": [ "# Exercise solutions" @@ -454,7 +510,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "**Coming soon**" ] @@ -463,7 +522,9 @@ "cell_type": "code", "execution_count": null, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [] @@ -485,7 +546,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.5.1" + "version": "3.5.2+" }, "nav_menu": { "height": "309px", diff --git a/07_ensemble_learning_and_random_forests.ipynb b/07_ensemble_learning_and_random_forests.ipynb index 879b454..55ebebf 100644 --- a/07_ensemble_learning_and_random_forests.ipynb +++ b/07_ensemble_learning_and_random_forests.ipynb @@ -2,28 +2,40 @@ "cells": [ { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "**Chapter 7 – Ensemble Learning and Random Forests**" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "_This notebook contains all the sample code and solutions to the exercices in chapter 7._" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Setup" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "First, let's make sure this notebook works well in both python 2 and 3, import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures:" ] @@ -32,7 +44,9 @@ "cell_type": "code", "execution_count": 1, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -71,7 +85,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Voting classifiers" ] @@ -80,7 +97,9 @@ "cell_type": "code", "execution_count": 2, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -93,7 +112,9 @@ "cell_type": "code", "execution_count": 3, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -113,7 +134,9 @@ "cell_type": "code", "execution_count": 4, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -148,7 +171,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Bagging ensembles" ] @@ -157,7 +183,9 @@ "cell_type": "code", "execution_count": 5, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -179,7 +207,9 @@ "cell_type": "code", "execution_count": 6, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -193,7 +223,9 @@ "cell_type": "code", "execution_count": 7, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -221,7 +253,9 @@ "cell_type": "code", "execution_count": 8, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -238,7 +272,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Random Forests" ] @@ -247,7 +284,9 @@ "cell_type": "code", "execution_count": 9, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -264,7 +303,9 @@ "cell_type": "code", "execution_count": 10, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -280,7 +321,9 @@ "cell_type": "code", "execution_count": 11, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -291,7 +334,9 @@ "cell_type": "code", "execution_count": 12, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -307,7 +352,9 @@ "cell_type": "code", "execution_count": 13, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -318,7 +365,9 @@ "cell_type": "code", "execution_count": 14, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -335,7 +384,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Out-of-Bag evaluation" ] @@ -344,7 +396,9 @@ "cell_type": "code", "execution_count": 15, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -360,7 +414,9 @@ "cell_type": "code", "execution_count": 16, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -371,7 +427,9 @@ "cell_type": "code", "execution_count": 17, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -382,7 +440,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Feature importance" ] @@ -391,7 +452,9 @@ "cell_type": "code", "execution_count": 18, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -405,7 +468,9 @@ "cell_type": "code", "execution_count": 19, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -420,7 +485,9 @@ "cell_type": "code", "execution_count": 20, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -435,7 +502,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# AdaBoost" ] @@ -444,7 +514,9 @@ "cell_type": "code", "execution_count": 21, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -462,7 +534,9 @@ "cell_type": "code", "execution_count": 22, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -494,7 +568,9 @@ "cell_type": "code", "execution_count": 23, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -503,7 +579,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Gradient Boosting" ] @@ -512,7 +591,9 @@ "cell_type": "code", "execution_count": 24, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -542,7 +623,9 @@ "cell_type": "code", "execution_count": 25, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -593,7 +676,9 @@ "cell_type": "code", "execution_count": 26, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -621,7 +706,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Gradient Boosting with Early stopping" ] @@ -630,7 +718,9 @@ "cell_type": "code", "execution_count": 27, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -649,7 +739,9 @@ "cell_type": "code", "execution_count": 28, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -664,7 +756,9 @@ "cell_type": "code", "execution_count": 29, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -692,7 +786,9 @@ "cell_type": "code", "execution_count": 30, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -718,7 +814,9 @@ "cell_type": "code", "execution_count": 31, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -728,7 +826,9 @@ { "cell_type": "markdown", "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "source": [ "# Exercise solutions" @@ -736,7 +836,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "**Coming soon**" ] @@ -745,7 +848,9 @@ "cell_type": "code", "execution_count": null, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [] @@ -767,7 +872,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.5.2" + "version": "3.5.2+" }, "nav_menu": { "height": "252px", diff --git a/08_dimensionality_reduction.ipynb b/08_dimensionality_reduction.ipynb index 06b4fe8..91455df 100644 --- a/08_dimensionality_reduction.ipynb +++ b/08_dimensionality_reduction.ipynb @@ -2,28 +2,40 @@ "cells": [ { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "**Chapter 8 – Dimensionality Reduction**" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "_This notebook contains all the sample code and solutions to the exercices in chapter 8._" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Setup" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "First, let's make sure this notebook works well in both python 2 and 3, import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures:" ] @@ -32,7 +44,9 @@ "cell_type": "code", "execution_count": 1, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -69,7 +83,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Projection methods\n", "Build 3D dataset:" @@ -79,7 +96,9 @@ "cell_type": "code", "execution_count": 2, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -97,7 +116,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Mean normalize the data:" ] @@ -106,7 +128,9 @@ "cell_type": "code", "execution_count": 3, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -115,7 +139,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Apply PCA to reduce to 2D." ] @@ -124,7 +151,9 @@ "cell_type": "code", "execution_count": 4, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -136,7 +165,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Recover the 3D points projected on the plane (PCA 2D subspace)." ] @@ -145,7 +177,9 @@ "cell_type": "code", "execution_count": 5, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -154,7 +188,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Utility class to draw 3D arrows (copied from http://stackoverflow.com/questions/11140163)" ] @@ -163,7 +200,9 @@ "cell_type": "code", "execution_count": 6, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -184,7 +223,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Express the plane as a function of x and y." ] @@ -193,7 +235,9 @@ "cell_type": "code", "execution_count": 7, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -210,7 +254,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Plot the 3D dataset, the plane and the projections on that plane." ] @@ -219,7 +266,9 @@ "cell_type": "code", "execution_count": 8, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -263,7 +312,9 @@ "cell_type": "code", "execution_count": 9, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -284,7 +335,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "PCA using SVD decomposition" ] @@ -293,7 +347,9 @@ "cell_type": "code", "execution_count": 10, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -312,7 +368,9 @@ "cell_type": "code", "execution_count": 11, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -323,7 +381,9 @@ "cell_type": "code", "execution_count": 12, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -334,7 +394,9 @@ "cell_type": "code", "execution_count": 13, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -345,7 +407,9 @@ "cell_type": "code", "execution_count": 14, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -359,7 +423,9 @@ "cell_type": "code", "execution_count": 15, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -370,7 +436,9 @@ "cell_type": "code", "execution_count": 16, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -381,7 +449,9 @@ "cell_type": "code", "execution_count": 17, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -392,7 +462,9 @@ "cell_type": "code", "execution_count": 18, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -403,7 +475,9 @@ "cell_type": "code", "execution_count": 19, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -414,7 +488,9 @@ "cell_type": "code", "execution_count": 20, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -425,7 +501,9 @@ "cell_type": "code", "execution_count": 21, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -436,7 +514,9 @@ "cell_type": "code", "execution_count": 22, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -447,7 +527,9 @@ "cell_type": "code", "execution_count": 23, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -458,7 +540,9 @@ "cell_type": "code", "execution_count": 24, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -467,7 +551,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Manifold learning\n", "Swiss roll:" @@ -477,7 +564,9 @@ "cell_type": "code", "execution_count": 25, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -489,7 +578,9 @@ "cell_type": "code", "execution_count": 26, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -515,7 +606,9 @@ "cell_type": "code", "execution_count": 27, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -542,7 +635,9 @@ "cell_type": "code", "execution_count": 28, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -623,7 +718,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# PCA" ] @@ -632,7 +730,9 @@ "cell_type": "code", "execution_count": 29, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -698,7 +798,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# MNIST compression" ] @@ -707,7 +810,9 @@ "cell_type": "code", "execution_count": 30, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -725,7 +830,9 @@ "cell_type": "code", "execution_count": 31, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -741,7 +848,9 @@ "cell_type": "code", "execution_count": 32, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -754,7 +863,9 @@ "cell_type": "code", "execution_count": 33, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -765,7 +876,9 @@ "cell_type": "code", "execution_count": 34, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -780,7 +893,9 @@ "cell_type": "code", "execution_count": 35, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -802,9 +917,11 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 36, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -823,7 +940,9 @@ "cell_type": "code", "execution_count": 37, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -842,7 +961,9 @@ "cell_type": "code", "execution_count": 38, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -853,7 +974,9 @@ "cell_type": "code", "execution_count": 39, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -869,7 +992,9 @@ "cell_type": "code", "execution_count": 40, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -880,7 +1005,9 @@ "cell_type": "code", "execution_count": 41, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -891,7 +1018,9 @@ "cell_type": "code", "execution_count": 42, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -906,7 +1035,9 @@ "cell_type": "code", "execution_count": 43, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -921,13 +1052,13 @@ "cell_type": "code", "execution_count": 44, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ - "from sklearn.decomposition import RandomizedPCA\n", - "\n", - "rnd_pca = RandomizedPCA(n_components=154, random_state=42)\n", + "rnd_pca = PCA(n_components=154, random_state=42, svd_solver=\"randomized\")\n", "X_reduced = rnd_pca.fit_transform(X_mnist)" ] }, @@ -935,7 +1066,9 @@ "cell_type": "code", "execution_count": 45, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -945,7 +1078,7 @@ " print(\"n_components =\", n_components)\n", " regular_pca = PCA(n_components=n_components)\n", " inc_pca = IncrementalPCA(n_components=154, batch_size=500)\n", - " rnd_pca = RandomizedPCA(n_components=154, random_state=42)\n", + " rnd_pca = PCA(n_components=154, random_state=42, svd_solver=\"randomized\")\n", "\n", " for pca in (regular_pca, inc_pca, rnd_pca):\n", " t1 = time.time()\n", @@ -956,7 +1089,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Kernel PCA" ] @@ -965,7 +1101,9 @@ "cell_type": "code", "execution_count": 46, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1003,7 +1141,9 @@ "cell_type": "code", "execution_count": 47, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1029,7 +1169,9 @@ "cell_type": "code", "execution_count": 48, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1047,7 +1189,9 @@ "cell_type": "code", "execution_count": 49, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1073,7 +1217,9 @@ "cell_type": "code", "execution_count": 50, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1087,7 +1233,9 @@ "cell_type": "code", "execution_count": 51, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1099,7 +1247,9 @@ "cell_type": "code", "execution_count": 52, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1108,7 +1258,7 @@ "sizes = [1000, 10000, 20000, 30000, 40000, 50000, 70000, 100000, 200000, 500000]\n", "for n_samples in sizes:\n", " X = rnd.randn(n_samples, 5)\n", - " pca = RandomizedPCA(n_components = 2, random_state=42)\n", + " pca = PCA(n_components = 2, random_state=42, svd_solver=\"randomized\")\n", " t1 = time.time()\n", " pca.fit(X)\n", " t2 = time.time()\n", @@ -1129,9 +1279,11 @@ }, { "cell_type": "code", - "execution_count": 55, + "execution_count": 53, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1140,7 +1292,7 @@ "sizes = [1000, 2000, 3000, 4000, 5000, 6000]\n", "for n_features in sizes:\n", " X = rnd.randn(2000, n_features)\n", - " pca = RandomizedPCA(n_components = 2, random_state=42)\n", + " pca = PCA(n_components = 2, random_state=42, svd_solver=\"randomized\")\n", " t1 = time.time()\n", " pca.fit(X)\n", " t2 = time.time()\n", @@ -1161,16 +1313,21 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# LLE" ] }, { "cell_type": "code", - "execution_count": 72, + "execution_count": 54, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1194,16 +1351,21 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# MDS, Isomap and t-SNE" ] }, { "cell_type": "code", - "execution_count": 107, + "execution_count": 55, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1214,9 +1376,11 @@ }, { "cell_type": "code", - "execution_count": 108, + "execution_count": 56, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1227,9 +1391,11 @@ }, { "cell_type": "code", - "execution_count": 109, + "execution_count": 57, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1240,9 +1406,11 @@ }, { "cell_type": "code", - "execution_count": 128, + "execution_count": 58, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1256,9 +1424,11 @@ }, { "cell_type": "code", - "execution_count": 136, + "execution_count": 59, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1283,7 +1453,9 @@ { "cell_type": "markdown", "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "source": [ "# Exercise solutions" @@ -1291,7 +1463,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "**Coming soon**" ] @@ -1300,7 +1475,9 @@ "cell_type": "code", "execution_count": null, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [] @@ -1322,7 +1499,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.5.2" + "version": "3.5.2+" }, "nav_menu": { "height": "352px", diff --git a/09_up_and_running_with_tensorflow.ipynb b/09_up_and_running_with_tensorflow.ipynb index 64e86fc..b402dd0 100644 --- a/09_up_and_running_with_tensorflow.ipynb +++ b/09_up_and_running_with_tensorflow.ipynb @@ -2,28 +2,40 @@ "cells": [ { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "**Chapter 9 – Up and running with TensorFlow**" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "_This notebook contains all the sample code and solutions to the exercices in chapter 9._" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Setup" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "First, let's make sure this notebook works well in both python 2 and 3, import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures:" ] @@ -32,7 +44,9 @@ "cell_type": "code", "execution_count": 1, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -69,7 +83,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Creating and running a graph" ] @@ -78,7 +95,9 @@ "cell_type": "code", "execution_count": 2, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -89,7 +108,9 @@ "cell_type": "code", "execution_count": 3, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -106,7 +127,9 @@ "cell_type": "code", "execution_count": 4, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -121,7 +144,9 @@ "cell_type": "code", "execution_count": 5, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -137,11 +162,13 @@ "cell_type": "code", "execution_count": 6, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ - "init = tf.initialize_all_variables()\n", + "init = tf.global_variables_initializer()\n", "\n", "with tf.Session():\n", " init.run()\n", @@ -154,11 +181,13 @@ "cell_type": "code", "execution_count": 7, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ - "init = tf.initialize_all_variables()\n", + "init = tf.global_variables_initializer()\n", "\n", "sess = tf.InteractiveSession()\n", "init.run()\n", @@ -170,7 +199,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Managing graphs" ] @@ -179,7 +211,9 @@ "cell_type": "code", "execution_count": 8, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -193,7 +227,9 @@ "cell_type": "code", "execution_count": 9, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -209,6 +245,8 @@ "execution_count": 10, "metadata": { "collapsed": false, + "deletable": true, + "editable": true, "scrolled": true }, "outputs": [], @@ -220,7 +258,9 @@ "cell_type": "code", "execution_count": 11, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -238,7 +278,9 @@ "cell_type": "code", "execution_count": 12, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -250,14 +292,20 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Linear Regression" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Using the Normal Equation" ] @@ -266,7 +314,9 @@ "cell_type": "code", "execution_count": 13, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -281,7 +331,9 @@ "cell_type": "code", "execution_count": 14, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -300,7 +352,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Compare with pure NumPy" ] @@ -309,7 +364,9 @@ "cell_type": "code", "execution_count": 15, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -322,7 +379,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Compare with Scikit-Learn" ] @@ -331,7 +391,9 @@ "cell_type": "code", "execution_count": 16, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -344,14 +406,20 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Using Batch Gradient Descent" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Gradient Descent requires scaling the feature vectors first. We could do this using TF, but let's just use Scikit-Learn for now." ] @@ -360,7 +428,9 @@ "cell_type": "code", "execution_count": 17, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -374,7 +444,9 @@ "cell_type": "code", "execution_count": 18, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -386,7 +458,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "### Manually computing the gradients" ] @@ -395,7 +470,9 @@ "cell_type": "code", "execution_count": 19, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -413,7 +490,7 @@ "gradients = 2/m * tf.matmul(tf.transpose(X), error)\n", "training_op = tf.assign(theta, theta - learning_rate * gradients)\n", "\n", - "init = tf.initialize_all_variables()\n", + "init = tf.global_variables_initializer()\n", "\n", "with tf.Session() as sess:\n", " sess.run(init)\n", @@ -431,7 +508,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "### Using autodiff\n", "Same as above except for the `gradients = ...` line." @@ -441,7 +521,9 @@ "cell_type": "code", "execution_count": 20, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -459,7 +541,7 @@ "gradients = tf.gradients(mse, [theta])[0]\n", "training_op = tf.assign(theta, theta - learning_rate * gradients)\n", "\n", - "init = tf.initialize_all_variables()\n", + "init = tf.global_variables_initializer()\n", "\n", "with tf.Session() as sess:\n", " sess.run(init)\n", @@ -477,7 +559,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "### Using a `GradientDescentOptimizer`" ] @@ -486,7 +571,9 @@ "cell_type": "code", "execution_count": 21, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -504,7 +591,7 @@ "optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)\n", "training_op = optimizer.minimize(mse)\n", "\n", - "init = tf.initialize_all_variables()\n", + "init = tf.global_variables_initializer()\n", "\n", "with tf.Session() as sess:\n", " sess.run(init)\n", @@ -522,7 +609,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "### Using a momentum optimizer" ] @@ -531,7 +621,9 @@ "cell_type": "code", "execution_count": 22, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -549,14 +641,16 @@ "optimizer = tf.train.MomentumOptimizer(learning_rate=learning_rate, momentum=0.25)\n", "training_op = optimizer.minimize(mse)\n", "\n", - "init = tf.initialize_all_variables()" + "init = tf.global_variables_initializer()" ] }, { "cell_type": "code", "execution_count": 23, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -574,14 +668,20 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Feeding data to the training algorithm" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Placeholder nodes" ] @@ -590,7 +690,9 @@ "cell_type": "code", "execution_count": 24, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -608,7 +710,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Mini-batch Gradient Descent" ] @@ -617,7 +722,9 @@ "cell_type": "code", "execution_count": 25, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -635,14 +742,16 @@ "optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)\n", "training_op = optimizer.minimize(mse)\n", "\n", - "init = tf.initialize_all_variables()" + "init = tf.global_variables_initializer()" ] }, { "cell_type": "code", "execution_count": 26, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -673,7 +782,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Saving and restoring a model" ] @@ -682,7 +794,9 @@ "cell_type": "code", "execution_count": 27, "metadata": { - "collapsed": true + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -700,7 +814,7 @@ "optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)\n", "training_op = optimizer.minimize(mse)\n", "\n", - "init = tf.initialize_all_variables()\n", + "init = tf.global_variables_initializer()\n", "saver = tf.train.Saver()" ] }, @@ -708,7 +822,9 @@ "cell_type": "code", "execution_count": 28, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -730,7 +846,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Visualizing the graph\n", "## inside Jupyter" @@ -740,7 +859,9 @@ "cell_type": "code", "execution_count": 29, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -787,6 +908,8 @@ "execution_count": 30, "metadata": { "collapsed": false, + "deletable": true, + "editable": true, "scrolled": true }, "outputs": [], @@ -796,7 +919,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Using TensorBoard" ] @@ -805,7 +931,9 @@ "cell_type": "code", "execution_count": 31, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -829,17 +957,19 @@ "optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)\n", "training_op = optimizer.minimize(mse)\n", "\n", - "init = tf.initialize_all_variables()\n", + "init = tf.global_variables_initializer()\n", "\n", - "mse_summary = tf.scalar_summary('MSE', mse)\n", - "summary_writer = tf.train.SummaryWriter(logdir, tf.get_default_graph())" + "mse_summary = tf.summary.scalar('MSE', mse)\n", + "summary_writer = tf.summary.FileWriter(logdir, tf.get_default_graph())" ] }, { "cell_type": "code", "execution_count": 32, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -869,7 +999,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Name scopes" ] @@ -878,7 +1011,9 @@ "cell_type": "code", "execution_count": 33, "metadata": { - "collapsed": true + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -901,17 +1036,19 @@ "optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)\n", "training_op = optimizer.minimize(mse)\n", "\n", - "init = tf.initialize_all_variables()\n", + "init = tf.global_variables_initializer()\n", "\n", - "mse_summary = tf.scalar_summary('MSE', mse)\n", - "summary_writer = tf.train.SummaryWriter(logdir, tf.get_default_graph())" + "mse_summary = tf.summary.scalar('MSE', mse)\n", + "summary_writer = tf.summary.FileWriter(logdir, tf.get_default_graph())" ] }, { "cell_type": "code", "execution_count": 34, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -943,7 +1080,9 @@ "cell_type": "code", "execution_count": 35, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -954,7 +1093,9 @@ "cell_type": "code", "execution_count": 36, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -965,7 +1106,9 @@ "cell_type": "code", "execution_count": 37, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -986,14 +1129,20 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Modularity" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "An ugly flat code:" ] @@ -1002,7 +1151,9 @@ "cell_type": "code", "execution_count": 38, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1027,7 +1178,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Much better, using a function to build the ReLUs:" ] @@ -1036,7 +1190,9 @@ "cell_type": "code", "execution_count": 39, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1053,12 +1209,15 @@ "X = tf.placeholder(tf.float32, shape=(None, n_features), name=\"X\")\n", "relus = [relu(X) for i in range(5)]\n", "output = tf.add_n(relus, name=\"output\")\n", - "summary_writer = tf.train.SummaryWriter(\"logs/relu1\", tf.get_default_graph())" + "summary_writer = tf.summary.FileWriter(\"logs/relu1\", tf.get_default_graph())" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Even better using name scopes:" ] @@ -1067,7 +1226,9 @@ "cell_type": "code", "execution_count": 40, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1086,14 +1247,16 @@ "relus = [relu(X) for i in range(5)]\n", "output = tf.add_n(relus, name=\"output\")\n", "\n", - "summary_writer = tf.train.SummaryWriter(\"logs/relu2\", tf.get_default_graph())" + "summary_writer = tf.summary.FileWriter(\"logs/relu2\", tf.get_default_graph())" ] }, { "cell_type": "code", "execution_count": 41, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1102,7 +1265,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Sharing a `threshold` variable the classic way, by defining it outside of the `relu()` function then passing it as a parameter:" ] @@ -1111,7 +1277,9 @@ "cell_type": "code", "execution_count": 42, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1135,7 +1303,9 @@ "cell_type": "code", "execution_count": 43, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1160,7 +1330,9 @@ "cell_type": "code", "execution_count": 44, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1181,7 +1353,7 @@ "relus = [relu(X) for i in range(5)]\n", "output = tf.add_n(relus, name=\"output\")\n", "\n", - "summary_writer = tf.train.SummaryWriter(\"logs/relu6\", tf.get_default_graph())\n", + "summary_writer = tf.summary.FileWriter(\"logs/relu6\", tf.get_default_graph())\n", "summary_writer.close()" ] }, @@ -1189,7 +1361,9 @@ "cell_type": "code", "execution_count": 45, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1211,7 +1385,7 @@ " relus = [first_relu] + [relu(X) for i in range(4)]\n", "output = tf.add_n(relus, name=\"output\")\n", "\n", - "summary_writer = tf.train.SummaryWriter(\"logs/relu8\", tf.get_default_graph())\n", + "summary_writer = tf.summary.FileWriter(\"logs/relu8\", tf.get_default_graph())\n", "summary_writer.close()" ] }, @@ -1219,7 +1393,9 @@ "cell_type": "code", "execution_count": 46, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1242,14 +1418,20 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Extra material" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Strings" ] @@ -1258,7 +1440,9 @@ "cell_type": "code", "execution_count": 47, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1273,7 +1457,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Distributed TensorFlow" ] @@ -1282,7 +1469,9 @@ "cell_type": "code", "execution_count": 48, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1293,7 +1482,9 @@ "cell_type": "code", "execution_count": 49, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1307,6 +1498,8 @@ "execution_count": 50, "metadata": { "collapsed": false, + "deletable": true, + "editable": true, "scrolled": true }, "outputs": [], @@ -1318,7 +1511,9 @@ "cell_type": "code", "execution_count": 51, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1364,7 +1559,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Computing gradients\n", "### Mathematical differentiation" @@ -1374,7 +1572,9 @@ "cell_type": "code", "execution_count": 52, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1386,7 +1586,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "### Numerical differentiation" ] @@ -1395,7 +1598,9 @@ "cell_type": "code", "execution_count": 53, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1412,7 +1617,9 @@ "cell_type": "code", "execution_count": 54, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1430,7 +1637,9 @@ "cell_type": "code", "execution_count": 55, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1440,7 +1649,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "### Symbolic differentiation" ] @@ -1449,7 +1661,9 @@ "cell_type": "code", "execution_count": 56, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1470,7 +1684,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "### Automatic differentiation (autodiff) – forward mode" ] @@ -1479,7 +1696,9 @@ "cell_type": "code", "execution_count": 57, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1532,7 +1751,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "### Autodiff – Reverse mode" ] @@ -1541,7 +1763,9 @@ "cell_type": "code", "execution_count": 58, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1602,7 +1826,9 @@ "cell_type": "code", "execution_count": 59, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1618,7 +1844,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "### Autodiff – reverse mode (using TensorFlow)" ] @@ -1627,7 +1856,9 @@ "cell_type": "code", "execution_count": 60, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1639,7 +1870,7 @@ "\n", "gradients = tf.gradients(f, [x, y])\n", "\n", - "init = tf.initialize_all_variables()\n", + "init = tf.global_variables_initializer()\n", "\n", "with tf.Session() as sess:\n", " init.run()\n", @@ -1650,14 +1881,20 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Exercise solutions" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "**Coming soon**" ] @@ -1666,7 +1903,9 @@ "cell_type": "code", "execution_count": null, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [] @@ -1688,7 +1927,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.5.2" + "version": "3.5.2+" }, "nav_menu": { "height": "603px", diff --git a/10_introduction_to_artificial_neural_networks.ipynb b/10_introduction_to_artificial_neural_networks.ipynb index 5808136..9aef566 100644 --- a/10_introduction_to_artificial_neural_networks.ipynb +++ b/10_introduction_to_artificial_neural_networks.ipynb @@ -2,28 +2,40 @@ "cells": [ { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "**Chapter 10 – Introduction to Artificial Neural Networks**" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "_This notebook contains all the sample code and solutions to the exercices in chapter 10._" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Setup" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "First, let's make sure this notebook works well in both python 2 and 3, import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures:" ] @@ -32,7 +44,9 @@ "cell_type": "code", "execution_count": 1, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -69,7 +83,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Perceptrons" ] @@ -78,7 +95,9 @@ "cell_type": "code", "execution_count": 2, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -92,7 +111,9 @@ "cell_type": "code", "execution_count": 3, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -109,7 +130,9 @@ "cell_type": "code", "execution_count": 4, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -146,7 +169,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Activation functions" ] @@ -155,7 +181,9 @@ "cell_type": "code", "execution_count": 5, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -173,7 +201,9 @@ "cell_type": "code", "execution_count": 6, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -211,7 +241,9 @@ "cell_type": "code", "execution_count": 7, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -229,7 +261,9 @@ "cell_type": "code", "execution_count": 8, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -259,14 +293,20 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# FNN for MNIST" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## using tf.learn" ] @@ -275,7 +315,9 @@ "cell_type": "code", "execution_count": 9, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -291,7 +333,9 @@ "cell_type": "code", "execution_count": 10, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -307,13 +351,15 @@ "cell_type": "code", "execution_count": 11, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ "from sklearn.metrics import accuracy_score\n", "\n", - "y_pred = dnn_clf.predict(X_test)\n", + "y_pred = list(dnn_clf.predict(X_test))\n", "accuracy = accuracy_score(y_test, y_pred)\n", "accuracy" ] @@ -322,13 +368,15 @@ "cell_type": "code", "execution_count": 12, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ "from sklearn.metrics import log_loss\n", "\n", - "y_pred_proba = dnn_clf.predict_proba(X_test)\n", + "y_pred_proba = list(dnn_clf.predict_proba(X_test))\n", "log_loss(y_test, y_pred_proba)" ] }, @@ -336,7 +384,9 @@ "cell_type": "code", "execution_count": 13, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -346,7 +396,9 @@ { "cell_type": "markdown", "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "source": [ "## Using plain TensorFlow" @@ -356,7 +408,9 @@ "cell_type": "code", "execution_count": 14, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -380,7 +434,9 @@ "cell_type": "code", "execution_count": 15, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -401,7 +457,7 @@ " logits = neuron_layer(hidden2, n_outputs, \"output\")\n", "\n", "with tf.name_scope(\"loss\"):\n", - " xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits, y)\n", + " xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n", " loss = tf.reduce_mean(xentropy, name=\"loss\")\n", "\n", "with tf.name_scope(\"train\"):\n", @@ -412,7 +468,7 @@ " correct = tf.nn.in_top_k(logits, y, 1)\n", " accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n", " \n", - "init = tf.initialize_all_variables()\n", + "init = tf.global_variables_initializer()\n", "saver = tf.train.Saver()" ] }, @@ -420,7 +476,9 @@ "cell_type": "code", "execution_count": 16, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -437,19 +495,21 @@ " acc_test = accuracy.eval(feed_dict={X: mnist.test.images, y: mnist.test.labels})\n", " print(epoch, \"Train accuracy:\", acc_train, \"Test accuracy:\", acc_test)\n", "\n", - " save_path = saver.save(sess, \"my_model_final.ckpt\")" + " save_path = saver.save(sess, \"./my_model_final.ckpt\")" ] }, { "cell_type": "code", "execution_count": 17, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ "with tf.Session() as sess:\n", - " saver.restore(sess, \"my_model_final.ckpt\")\n", + " saver.restore(sess, save_path) #\"my_model_final.ckpt\")\n", " X_new_scaled = mnist.test.images[:20]\n", " Z = logits.eval(feed_dict={X: X_new_scaled})\n", " print(np.argmax(Z, axis=1))\n", @@ -460,7 +520,9 @@ "cell_type": "code", "execution_count": 18, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -506,7 +568,9 @@ "cell_type": "code", "execution_count": 19, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -515,7 +579,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Using `fully_connected` instead of `neuron_layer()`" ] @@ -524,7 +591,9 @@ "cell_type": "code", "execution_count": 20, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -547,7 +616,7 @@ " logits = fully_connected(hidden2, n_outputs, activation_fn=None, scope=\"outputs\")\n", "\n", "with tf.name_scope(\"loss\"):\n", - " xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits, y)\n", + " xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n", " loss = tf.reduce_mean(xentropy, name=\"loss\")\n", "\n", "with tf.name_scope(\"train\"):\n", @@ -558,7 +627,7 @@ " correct = tf.nn.in_top_k(logits, y, 1)\n", " accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n", " \n", - "init = tf.initialize_all_variables()\n", + "init = tf.global_variables_initializer()\n", "saver = tf.train.Saver()" ] }, @@ -566,7 +635,9 @@ "cell_type": "code", "execution_count": 21, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -583,14 +654,16 @@ " acc_test = accuracy.eval(feed_dict={X: mnist.test.images, y: mnist.test.labels})\n", " print(epoch, \"Train accuracy:\", acc_train, \"Test accuracy:\", acc_test)\n", "\n", - " save_path = saver.save(sess, \"my_model_final.ckpt\")" + " save_path = saver.save(sess, \"./my_model_final.ckpt\")" ] }, { "cell_type": "code", "execution_count": 22, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -600,7 +673,9 @@ { "cell_type": "markdown", "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "source": [ "# Exercise solutions" @@ -608,7 +683,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "**Coming soon**" ] @@ -617,7 +695,9 @@ "cell_type": "code", "execution_count": null, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [] @@ -639,7 +719,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.5.1" + "version": "3.5.2+" }, "nav_menu": { "height": "264px", diff --git a/11_deep_learning.ipynb b/11_deep_learning.ipynb index 5a6a44b..27d8e7e 100644 --- a/11_deep_learning.ipynb +++ b/11_deep_learning.ipynb @@ -2,28 +2,40 @@ "cells": [ { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "**Chapter 11 – Deep Learning**" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "_This notebook contains all the sample code and solutions to the exercices in chapter 11._" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Setup" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "First, let's make sure this notebook works well in both python 2 and 3, import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures:" ] @@ -32,7 +44,9 @@ "cell_type": "code", "execution_count": 1, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -69,7 +83,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Activation functions" ] @@ -78,7 +95,9 @@ "cell_type": "code", "execution_count": 2, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -90,7 +109,9 @@ "cell_type": "code", "execution_count": 3, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -117,7 +138,9 @@ "cell_type": "code", "execution_count": 4, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -129,7 +152,9 @@ "cell_type": "code", "execution_count": 5, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -150,7 +175,9 @@ "cell_type": "code", "execution_count": 6, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -162,7 +189,9 @@ "cell_type": "code", "execution_count": 7, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -183,7 +212,9 @@ "cell_type": "code", "execution_count": 8, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -195,7 +226,9 @@ "cell_type": "code", "execution_count": 9, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -207,7 +240,9 @@ "cell_type": "code", "execution_count": 10, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -218,7 +253,9 @@ "cell_type": "code", "execution_count": 11, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -264,7 +301,9 @@ "cell_type": "code", "execution_count": 12, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -287,7 +326,7 @@ " logits = fully_connected(hidden2, n_outputs, activation_fn=None, scope=\"outputs\")\n", "\n", "with tf.name_scope(\"loss\"):\n", - " xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits, y)\n", + " xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n", " loss = tf.reduce_mean(xentropy, name=\"loss\")\n", "\n", "with tf.name_scope(\"train\"):\n", @@ -298,7 +337,7 @@ " correct = tf.nn.in_top_k(logits, y, 1)\n", " accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n", " \n", - "init = tf.initialize_all_variables()\n", + "init = tf.global_variables_initializer()\n", "saver = tf.train.Saver()" ] }, @@ -306,7 +345,9 @@ "cell_type": "code", "execution_count": 13, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -328,7 +369,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Batch Normalization" ] @@ -337,7 +381,9 @@ "cell_type": "code", "execution_count": 14, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -377,7 +423,7 @@ " logits = fully_connected(hidden2, n_outputs, activation_fn=None, scope=\"outputs\")\n", "\n", "with tf.name_scope(\"loss\"):\n", - " xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits, y)\n", + " xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n", " loss = tf.reduce_mean(xentropy, name=\"loss\")\n", "\n", "with tf.name_scope(\"train\"):\n", @@ -388,7 +434,7 @@ " correct = tf.nn.in_top_k(logits, y, 1)\n", " accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n", " \n", - "init = tf.initialize_all_variables()\n", + "init = tf.global_variables_initializer()\n", "saver = tf.train.Saver()" ] }, @@ -396,7 +442,9 @@ "cell_type": "code", "execution_count": 15, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -420,7 +468,9 @@ "cell_type": "code", "execution_count": 16, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -451,7 +501,7 @@ " logits = fully_connected(hidden2, n_outputs, activation_fn=None, scope=\"outputs\")\n", "\n", "with tf.name_scope(\"loss\"):\n", - " xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits, y)\n", + " xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n", " reg_losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)\n", " base_loss = tf.reduce_mean(xentropy, name=\"base_loss\")\n", " loss = tf.add(base_loss, reg_losses, name=\"loss\")\n", @@ -464,7 +514,7 @@ " correct = tf.nn.in_top_k(logits, y, 1)\n", " accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n", " \n", - "init = tf.initialize_all_variables()\n", + "init = tf.global_variables_initializer()\n", "saver = tf.train.Saver()" ] }, @@ -472,7 +522,9 @@ "cell_type": "code", "execution_count": 17, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -496,18 +548,22 @@ "cell_type": "code", "execution_count": 18, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ - "[v.name for v in tf.all_variables()]" + "[v.name for v in tf.global_variables()]" ] }, { "cell_type": "code", "execution_count": 19, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -521,7 +577,9 @@ "cell_type": "code", "execution_count": 20, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -545,7 +603,9 @@ "cell_type": "code", "execution_count": 21, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -556,7 +616,9 @@ "cell_type": "code", "execution_count": 22, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -567,7 +629,9 @@ "cell_type": "code", "execution_count": 23, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -578,7 +642,9 @@ "cell_type": "code", "execution_count": 24, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -589,7 +655,9 @@ "cell_type": "code", "execution_count": 25, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -600,7 +668,9 @@ "cell_type": "code", "execution_count": 26, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -611,7 +681,9 @@ "cell_type": "code", "execution_count": 27, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -639,7 +711,7 @@ "clip_all_weights = tf.get_collection(\"max_norm\")\n", " \n", "with tf.name_scope(\"loss\"):\n", - " xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits, y)\n", + " xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n", " loss = tf.reduce_mean(xentropy, name=\"loss\")\n", "\n", "with tf.name_scope(\"train\"):\n", @@ -654,7 +726,7 @@ " correct = tf.nn.in_top_k(logits, y, 1)\n", " accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n", " \n", - "init = tf.initialize_all_variables()\n", + "init = tf.global_variables_initializer()\n", "saver = tf.train.Saver()" ] }, @@ -662,7 +734,9 @@ "cell_type": "code", "execution_count": 28, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -686,7 +760,9 @@ "cell_type": "code", "execution_count": 29, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -697,7 +773,9 @@ "cell_type": "code", "execution_count": 30, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -732,7 +810,7 @@ " logits = fully_connected(hidden2_drop, n_outputs, activation_fn=None, scope=\"outputs\")\n", "\n", "with tf.name_scope(\"loss\"):\n", - " xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits, y)\n", + " xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n", " loss = tf.reduce_mean(xentropy, name=\"loss\")\n", "\n", "with tf.name_scope(\"train\"):\n", @@ -743,7 +821,7 @@ " correct = tf.nn.in_top_k(logits, y, 1)\n", " accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n", " \n", - "init = tf.initialize_all_variables()\n", + "init = tf.global_variables_initializer()\n", "saver = tf.train.Saver()" ] }, @@ -751,7 +829,9 @@ "cell_type": "code", "execution_count": 31, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -775,7 +855,9 @@ "cell_type": "code", "execution_count": 32, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -787,7 +869,9 @@ "cell_type": "code", "execution_count": 33, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -798,11 +882,13 @@ "cell_type": "code", "execution_count": 34, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ - "for i in tf.all_variables():\n", + "for i in tf.global_variables():\n", " print(i.name)" ] }, @@ -810,7 +896,9 @@ "cell_type": "code", "execution_count": 35, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -822,7 +910,9 @@ "cell_type": "code", "execution_count": 36, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -830,48 +920,12 @@ " print(i.name)" ] }, - { - "cell_type": "code", - "execution_count": 37, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "X_train = mnist.train.images\n", - "y_train = mnist.train.labels.astype(\"int\")\n", - "X_val = mnist.test.images[8000:]\n", - "y_val = mnist.test.labels[8000:].astype(\"int\")\n", - "\n", - "feature_columns = tf.contrib.learn.infer_real_valued_columns_from_input(X_train)\n", - "dnn_clf = tf.contrib.learn.DNNClassifier(\n", - " feature_columns = feature_columns,\n", - " hidden_units=[300, 100],\n", - " n_classes=10,\n", - " model_dir=\"/tmp/my_model\",\n", - " config=tf.contrib.learn.RunConfig(save_checkpoints_secs=60)\n", - " )\n", - "\n", - "validation_monitor = tf.contrib.learn.monitors.ValidationMonitor(\n", - " X_val,\n", - " y_val,\n", - " every_n_steps=50,\n", - " early_stopping_metric=\"loss\",\n", - " early_stopping_metric_minimize=True,\n", - " early_stopping_rounds=2000\n", - " )\n", - "\n", - "dnn_clf.fit(x=X_train,\n", - " y=y_train,\n", - " steps=40000,\n", - " monitors=[validation_monitor]\n", - " )\n" - ] - }, { "cell_type": "markdown", "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "source": [ "# Exercise solutions" @@ -879,7 +933,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "**Coming soon**" ] @@ -888,7 +945,9 @@ "cell_type": "code", "execution_count": null, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [] @@ -910,7 +969,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.5.2" + "version": "3.5.2+" }, "nav_menu": { "height": "360px", diff --git a/12_distributed_tensorflow.ipynb b/12_distributed_tensorflow.ipynb index c438d48..b91be4e 100644 --- a/12_distributed_tensorflow.ipynb +++ b/12_distributed_tensorflow.ipynb @@ -2,28 +2,40 @@ "cells": [ { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "**Chapter 12 – Distributed TensorFlow**" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "_This notebook contains all the sample code and solutions to the exercices in chapter 12._" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Setup" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "First, let's make sure this notebook works well in both python 2 and 3, import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures:" ] @@ -32,7 +44,9 @@ "cell_type": "code", "execution_count": 1, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -69,7 +83,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Local server" ] @@ -78,7 +95,9 @@ "cell_type": "code", "execution_count": 2, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -89,7 +108,9 @@ "cell_type": "code", "execution_count": 3, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -101,7 +122,9 @@ "cell_type": "code", "execution_count": 4, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -111,7 +134,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Cluster" ] @@ -120,7 +146,9 @@ "cell_type": "code", "execution_count": 5, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -140,7 +168,9 @@ "cell_type": "code", "execution_count": 6, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -153,7 +183,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Pinning operations across devices and servers" ] @@ -162,7 +195,9 @@ "cell_type": "code", "execution_count": 7, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -182,7 +217,9 @@ "cell_type": "code", "execution_count": 8, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -195,7 +232,9 @@ "cell_type": "code", "execution_count": 9, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -223,16 +262,21 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Readers" ] }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 10, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -254,7 +298,7 @@ "key, value = reader.read(filename_queue)\n", "\n", "x1, x2, target = tf.decode_csv(value, record_defaults=[[-1.], [-1.], [-1]])\n", - "features = tf.pack([x1, x2])\n", + "features = tf.stack([x1, x2])\n", "\n", "instance_queue = tf.RandomShuffleQueue(\n", " capacity=10, min_after_dequeue=2,\n", @@ -283,9 +327,11 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -298,16 +344,21 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Queue runners and coordinators" ] }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 12, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -322,7 +373,7 @@ "key, value = reader.read(filename_queue)\n", "\n", "x1, x2, target = tf.decode_csv(value, record_defaults=[[-1.], [-1.], [-1]])\n", - "features = tf.pack([x1, x2])\n", + "features = tf.stack([x1, x2])\n", "\n", "instance_queue = tf.RandomShuffleQueue(\n", " capacity=10, min_after_dequeue=2,\n", @@ -350,9 +401,11 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 13, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -362,7 +415,7 @@ " reader = tf.TextLineReader(skip_header_lines=1)\n", " key, value = reader.read(filename_queue)\n", " x1, x2, target = tf.decode_csv(value, record_defaults=[[-1.], [-1.], [-1]])\n", - " features = tf.pack([x1, x2])\n", + " features = tf.stack([x1, x2])\n", " enqueue_instance = instance_queue.enqueue([features, target])\n", " return enqueue_instance\n", "\n", @@ -396,16 +449,21 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Setting a timeout" ] }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 14, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -437,7 +495,9 @@ { "cell_type": "markdown", "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "source": [ "# Exercise solutions" @@ -445,7 +505,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "**Coming soon**" ] @@ -454,7 +517,9 @@ "cell_type": "code", "execution_count": null, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [] @@ -476,7 +541,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.5.1" + "version": "3.5.2+" }, "nav_menu": {}, "toc": { diff --git a/13_convolutional_neural_networks.ipynb b/13_convolutional_neural_networks.ipynb index 6bf1b29..0c9378e 100644 --- a/13_convolutional_neural_networks.ipynb +++ b/13_convolutional_neural_networks.ipynb @@ -2,28 +2,40 @@ "cells": [ { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "**Chapter 13 – Convolutional Neural Networks**" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "_This notebook contains all the sample code and solutions to the exercices in chapter 13._" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Setup" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "First, let's make sure this notebook works well in both python 2 and 3, import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures:" ] @@ -32,7 +44,9 @@ "cell_type": "code", "execution_count": 1, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -69,7 +83,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "A couple utility functions to plot grayscale and RGB images:" ] @@ -78,7 +95,9 @@ "cell_type": "code", "execution_count": 2, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -93,7 +112,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "And of course we will need TensorFlow:" ] @@ -102,7 +124,9 @@ "cell_type": "code", "execution_count": 3, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -111,7 +135,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Convolutional layer" ] @@ -120,7 +147,9 @@ "cell_type": "code", "execution_count": 4, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -137,7 +166,9 @@ "cell_type": "code", "execution_count": 5, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -155,7 +186,9 @@ "cell_type": "code", "execution_count": 6, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -170,7 +203,9 @@ "cell_type": "code", "execution_count": 7, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -182,7 +217,9 @@ "cell_type": "code", "execution_count": 8, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -195,7 +232,9 @@ "cell_type": "code", "execution_count": 9, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -208,7 +247,9 @@ "cell_type": "code", "execution_count": 10, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -219,7 +260,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Simple example" ] @@ -228,7 +272,9 @@ "cell_type": "code", "execution_count": 11, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -253,7 +299,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## VALID vs SAME padding" ] @@ -262,7 +311,9 @@ "cell_type": "code", "execution_count": 12, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -284,7 +335,9 @@ "cell_type": "code", "execution_count": 13, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -299,7 +352,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Pooling layer" ] @@ -308,7 +364,9 @@ "cell_type": "code", "execution_count": 14, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -335,7 +393,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# MNIST" ] @@ -344,7 +405,9 @@ "cell_type": "code", "execution_count": 15, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -359,7 +422,9 @@ "cell_type": "code", "execution_count": 16, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -370,7 +435,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Inception v3" ] @@ -379,11 +447,12 @@ "cell_type": "code", "execution_count": 17, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ - "import os\n", "import sys\n", "import tarfile\n", "import urllib.request\n", @@ -414,7 +483,9 @@ "cell_type": "code", "execution_count": 18, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -425,7 +496,9 @@ "cell_type": "code", "execution_count": 19, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -443,7 +516,9 @@ "cell_type": "code", "execution_count": 20, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -454,7 +529,9 @@ "cell_type": "code", "execution_count": 21, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -467,7 +544,9 @@ "cell_type": "code", "execution_count": 22, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -482,19 +561,20 @@ "cell_type": "code", "execution_count": 23, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ - "import tensorflow as tf\n", - "from nets.inception_v3 import inception_v3, inception_v3_arg_scope\n", + "from tensorflow.contrib.slim.nets import inception\n", "import tensorflow.contrib.slim as slim\n", "\n", "tf.reset_default_graph()\n", "\n", "X = tf.placeholder(tf.float32, shape=[None, height, width, channels], name=\"X\")\n", - "with slim.arg_scope(inception_v3_arg_scope()):\n", - " logits, end_points = inception_v3(X, num_classes=1001, is_training=False)\n", + "with slim.arg_scope(inception.inception_v3_arg_scope()):\n", + " logits, end_points = inception.inception_v3(X, num_classes=1001, is_training=False)\n", "predictions = end_points[\"Predictions\"]\n", "saver = tf.train.Saver()" ] @@ -503,7 +583,9 @@ "cell_type": "code", "execution_count": 24, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -518,7 +600,9 @@ "cell_type": "code", "execution_count": 25, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -529,7 +613,9 @@ "cell_type": "code", "execution_count": 26, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -540,7 +626,9 @@ "cell_type": "code", "execution_count": 27, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -553,7 +641,9 @@ { "cell_type": "markdown", "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "source": [ "# Exercise solutions" @@ -561,7 +651,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "**Coming soon**" ] @@ -570,7 +663,9 @@ "cell_type": "code", "execution_count": null, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [] @@ -592,7 +687,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.5.2" + "version": "3.5.2+" }, "nav_menu": {}, "toc": { diff --git a/14_recurrent_neural_networks.ipynb b/14_recurrent_neural_networks.ipynb index c950828..2f2f541 100644 --- a/14_recurrent_neural_networks.ipynb +++ b/14_recurrent_neural_networks.ipynb @@ -2,28 +2,40 @@ "cells": [ { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "**Chapter 14 – Recurrent Neural Networks**" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "_This notebook contains all the sample code and solutions to the exercices in chapter 14._" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Setup" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "First, let's make sure this notebook works well in both python 2 and 3, import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures:" ] @@ -32,7 +44,9 @@ "cell_type": "code", "execution_count": 1, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -69,7 +83,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Then of course we will need TensorFlow:" ] @@ -78,7 +95,9 @@ "cell_type": "code", "execution_count": 2, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -87,14 +106,20 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Basic RNNs" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Manual RNN" ] @@ -103,7 +128,9 @@ "cell_type": "code", "execution_count": 3, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -122,14 +149,16 @@ "Y0 = tf.tanh(tf.matmul(X0, Wx) + b)\n", "Y1 = tf.tanh(tf.matmul(Y0, Wy) + tf.matmul(X1, Wx) + b)\n", "\n", - "init = tf.initialize_all_variables()" + "init = tf.global_variables_initializer()" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -145,7 +174,9 @@ "cell_type": "code", "execution_count": 5, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -156,7 +187,9 @@ "cell_type": "code", "execution_count": 6, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -165,7 +198,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Using `rnn()`" ] @@ -174,7 +210,9 @@ "cell_type": "code", "execution_count": 7, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -186,18 +224,20 @@ "X0 = tf.placeholder(tf.float32, [None, n_inputs])\n", "X1 = tf.placeholder(tf.float32, [None, n_inputs])\n", "\n", - "basic_cell = tf.nn.rnn_cell.BasicRNNCell(num_units=n_neurons)\n", - "output_seqs, states = tf.nn.rnn(basic_cell, [X0, X1], dtype=tf.float32)\n", + "basic_cell = tf.contrib.rnn.BasicRNNCell(num_units=n_neurons)\n", + "output_seqs, states = tf.contrib.rnn.static_rnn(basic_cell, [X0, X1], dtype=tf.float32)\n", "Y0, Y1 = output_seqs\n", "\n", - "init = tf.initialize_all_variables()" + "init = tf.global_variables_initializer()" ] }, { "cell_type": "code", "execution_count": 8, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -213,7 +253,9 @@ "cell_type": "code", "execution_count": 9, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -224,7 +266,9 @@ "cell_type": "code", "execution_count": 10, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -235,7 +279,9 @@ "cell_type": "code", "execution_count": 11, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -281,7 +327,9 @@ "cell_type": "code", "execution_count": 12, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -290,7 +338,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Packing sequences" ] @@ -299,7 +350,9 @@ "cell_type": "code", "execution_count": 13, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -310,20 +363,22 @@ "n_neurons = 5\n", "\n", "X = tf.placeholder(tf.float32, [None, n_steps, n_inputs])\n", - "X_seqs = tf.unpack(tf.transpose(X, perm=[1, 0, 2]))\n", + "X_seqs = tf.unstack(tf.transpose(X, perm=[1, 0, 2]))\n", "\n", - "basic_cell = tf.nn.rnn_cell.BasicRNNCell(num_units=n_neurons)\n", - "output_seqs, states = tf.nn.rnn(basic_cell, X_seqs, dtype=tf.float32)\n", - "outputs = tf.transpose(tf.pack(output_seqs), perm=[1, 0, 2])\n", + "basic_cell = tf.contrib.rnn.BasicRNNCell(num_units=n_neurons)\n", + "output_seqs, states = tf.contrib.rnn.static_rnn(basic_cell, X_seqs, dtype=tf.float32)\n", + "outputs = tf.transpose(tf.stack(output_seqs), perm=[1, 0, 2])\n", "\n", - "init = tf.initialize_all_variables()" + "init = tf.global_variables_initializer()" ] }, { "cell_type": "code", "execution_count": 14, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -344,7 +399,9 @@ "cell_type": "code", "execution_count": 15, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -353,7 +410,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Using `dynamic_rnn()`" ] @@ -362,7 +422,9 @@ "cell_type": "code", "execution_count": 16, "metadata": { - "collapsed": true + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -374,17 +436,19 @@ "\n", "X = tf.placeholder(tf.float32, [None, n_steps, n_inputs])\n", "\n", - "basic_cell = tf.nn.rnn_cell.BasicRNNCell(num_units=n_neurons)\n", + "basic_cell = tf.contrib.rnn.BasicRNNCell(num_units=n_neurons)\n", "outputs, states = tf.nn.dynamic_rnn(basic_cell, X, dtype=tf.float32)\n", "\n", - "init = tf.initialize_all_variables()" + "init = tf.global_variables_initializer()" ] }, { "cell_type": "code", "execution_count": 17, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -404,7 +468,9 @@ "cell_type": "code", "execution_count": 18, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -413,7 +479,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Setting the sequence lengths" ] @@ -422,7 +491,9 @@ "cell_type": "code", "execution_count": 19, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -435,17 +506,19 @@ "X = tf.placeholder(tf.float32, [None, n_steps, n_inputs])\n", "seq_length = tf.placeholder(tf.int32, [None])\n", "\n", - "basic_cell = tf.nn.rnn_cell.BasicRNNCell(num_units=n_neurons)\n", + "basic_cell = tf.contrib.rnn.BasicRNNCell(num_units=n_neurons)\n", "outputs, states = tf.nn.dynamic_rnn(basic_cell, X, sequence_length=seq_length, dtype=tf.float32)\n", "\n", - "init = tf.initialize_all_variables()" + "init = tf.global_variables_initializer()" ] }, { "cell_type": "code", "execution_count": 20, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -468,7 +541,9 @@ "cell_type": "code", "execution_count": 21, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -479,7 +554,9 @@ "cell_type": "code", "execution_count": 22, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -488,7 +565,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Training a sequence classifier" ] @@ -497,7 +577,9 @@ "cell_type": "code", "execution_count": 23, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -516,25 +598,27 @@ "y = tf.placeholder(tf.int32, [None])\n", "\n", "with tf.variable_scope(\"rnn\", initializer=tf.contrib.layers.variance_scaling_initializer()):\n", - " basic_cell = tf.nn.rnn_cell.BasicRNNCell(num_units=n_neurons, activation=tf.nn.relu)\n", + " basic_cell = tf.contrib.rnn.BasicRNNCell(num_units=n_neurons, activation=tf.nn.relu)\n", " outputs, states = tf.nn.dynamic_rnn(basic_cell, X, dtype=tf.float32)\n", "\n", "logits = fully_connected(states, n_outputs, activation_fn=None)\n", - "xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits, y)\n", + "xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n", "loss = tf.reduce_mean(xentropy)\n", "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n", "training_op = optimizer.minimize(loss)\n", "correct = tf.nn.in_top_k(logits, y, 1)\n", "accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n", "\n", - "init = tf.initialize_all_variables()" + "init = tf.global_variables_initializer()" ] }, { "cell_type": "code", "execution_count": 24, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -548,7 +632,9 @@ "cell_type": "code", "execution_count": 25, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -569,7 +655,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Multi-layer RNN" ] @@ -578,7 +667,9 @@ "cell_type": "code", "execution_count": 26, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -597,27 +688,29 @@ "X = tf.placeholder(tf.float32, [None, n_steps, n_inputs])\n", "y = tf.placeholder(tf.int32, [None])\n", "\n", - "hidden1 = tf.nn.rnn_cell.BasicRNNCell(num_units=n_neurons1, activation=tf.nn.relu)\n", - "hidden2 = tf.nn.rnn_cell.BasicRNNCell(num_units=n_neurons2, activation=tf.nn.relu)\n", - "multi_layer_cell = tf.nn.rnn_cell.MultiRNNCell([hidden1, hidden2])\n", + "hidden1 = tf.contrib.rnn.BasicRNNCell(num_units=n_neurons1, activation=tf.nn.relu)\n", + "hidden2 = tf.contrib.rnn.BasicRNNCell(num_units=n_neurons2, activation=tf.nn.relu)\n", + "multi_layer_cell = tf.contrib.rnn.MultiRNNCell([hidden1, hidden2])\n", "outputs, states_tuple = tf.nn.dynamic_rnn(multi_layer_cell, X, dtype=tf.float32)\n", - "states = tf.concat(concat_dim=1, values=states_tuple)\n", + "states = tf.concat(axis=1, values=states_tuple)\n", "logits = fully_connected(states, n_outputs, activation_fn=None)\n", - "xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits, y)\n", + "xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n", "loss = tf.reduce_mean(xentropy)\n", "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n", "training_op = optimizer.minimize(loss)\n", "correct = tf.nn.in_top_k(logits, y, 1)\n", "accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n", "\n", - "init = tf.initialize_all_variables()" + "init = tf.global_variables_initializer()" ] }, { "cell_type": "code", "execution_count": 27, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -638,7 +731,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Time series" ] @@ -647,7 +743,9 @@ "cell_type": "code", "execution_count": 28, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -668,7 +766,9 @@ "cell_type": "code", "execution_count": 29, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -703,7 +803,9 @@ "cell_type": "code", "execution_count": 30, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -714,7 +816,9 @@ "cell_type": "code", "execution_count": 31, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -723,7 +827,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Using an `OuputProjectionWrapper`" ] @@ -732,7 +839,9 @@ "cell_type": "code", "execution_count": 32, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -748,8 +857,8 @@ "X = tf.placeholder(tf.float32, [None, n_steps, n_inputs])\n", "y = tf.placeholder(tf.float32, [None, n_steps, n_outputs])\n", "\n", - "cell = tf.nn.rnn_cell.OutputProjectionWrapper(\n", - " tf.nn.rnn_cell.BasicRNNCell(num_units=n_neurons, activation=tf.nn.relu),\n", + "cell = tf.contrib.rnn.OutputProjectionWrapper(\n", + " tf.contrib.rnn.BasicRNNCell(num_units=n_neurons, activation=tf.nn.relu),\n", " output_size=n_outputs)\n", "outputs, states = tf.nn.dynamic_rnn(cell, X, dtype=tf.float32)\n", "\n", @@ -760,14 +869,16 @@ "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n", "training_op = optimizer.minimize(loss)\n", "\n", - "init = tf.initialize_all_variables()" + "init = tf.global_variables_initializer()" ] }, { "cell_type": "code", "execution_count": 33, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -792,7 +903,9 @@ "cell_type": "code", "execution_count": 34, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -809,7 +922,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Without using an `OutputProjectionWrapper`" ] @@ -818,7 +934,9 @@ "cell_type": "code", "execution_count": 35, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -833,7 +951,7 @@ "X = tf.placeholder(tf.float32, [None, n_steps, n_inputs])\n", "y = tf.placeholder(tf.float32, [None, n_steps, n_outputs])\n", "\n", - "basic_cell = tf.nn.rnn_cell.BasicRNNCell(num_units=n_neurons, activation=tf.nn.relu)\n", + "basic_cell = tf.contrib.rnn.BasicRNNCell(num_units=n_neurons, activation=tf.nn.relu)\n", "rnn_outputs, states = tf.nn.dynamic_rnn(basic_cell, X, dtype=tf.float32)\n", "\n", "n_outputs = 1\n", @@ -847,14 +965,16 @@ "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n", "training_op = optimizer.minimize(loss)\n", "\n", - "init = tf.initialize_all_variables()" + "init = tf.global_variables_initializer()" ] }, { "cell_type": "code", "execution_count": 36, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -879,7 +999,9 @@ "cell_type": "code", "execution_count": 37, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -895,7 +1017,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Generating a creative new sequence" ] @@ -904,7 +1029,9 @@ "cell_type": "code", "execution_count": 38, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -948,14 +1075,20 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Deep RNN" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## MultiRNNCell" ] @@ -964,7 +1097,9 @@ "cell_type": "code", "execution_count": 39, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -977,18 +1112,20 @@ "keep_prob = 0.5\n", "\n", "X = tf.placeholder(tf.float32, [None, n_steps, n_inputs])\n", - "basic_cell = tf.nn.rnn_cell.BasicRNNCell(num_units=n_neurons)\n", - "multi_layer_cell = tf.nn.rnn_cell.MultiRNNCell([basic_cell] * n_layers)\n", + "basic_cell = tf.contrib.rnn.BasicRNNCell(num_units=n_neurons)\n", + "multi_layer_cell = tf.contrib.rnn.MultiRNNCell([basic_cell] * n_layers)\n", "outputs, states = tf.nn.dynamic_rnn(multi_layer_cell, X, dtype=tf.float32)\n", "\n", - "init = tf.initialize_all_variables()" + "init = tf.global_variables_initializer()" ] }, { "cell_type": "code", "execution_count": 40, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -999,7 +1136,9 @@ "cell_type": "code", "execution_count": 41, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1012,7 +1151,9 @@ "cell_type": "code", "execution_count": 42, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1021,7 +1162,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Dropout" ] @@ -1030,7 +1174,9 @@ "cell_type": "code", "execution_count": 43, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1049,10 +1195,10 @@ "is_training = True\n", "\n", "def deep_rnn_with_dropout(X, y, is_training):\n", - " cell = tf.nn.rnn_cell.BasicRNNCell(num_units=n_neurons)\n", + " cell = tf.contrib.rnn.BasicRNNCell(num_units=n_neurons)\n", " if is_training:\n", - " cell = tf.nn.rnn_cell.DropoutWrapper(cell, input_keep_prob=keep_prob)\n", - " multi_layer_cell = tf.nn.rnn_cell.MultiRNNCell([cell] * n_layers)\n", + " cell = tf.contrib.rnn.DropoutWrapper(cell, input_keep_prob=keep_prob)\n", + " multi_layer_cell = tf.contrib.rnn.MultiRNNCell([cell] * n_layers)\n", " rnn_outputs, states = tf.nn.dynamic_rnn(multi_layer_cell, X, dtype=tf.float32)\n", "\n", " stacked_rnn_outputs = tf.reshape(rnn_outputs, [-1, n_neurons])\n", @@ -1068,7 +1214,7 @@ "X = tf.placeholder(tf.float32, [None, n_steps, n_inputs])\n", "y = tf.placeholder(tf.float32, [None, n_steps, n_outputs])\n", "outputs, loss, training_op = deep_rnn_with_dropout(X, y, is_training)\n", - "init = tf.initialize_all_variables()\n", + "init = tf.global_variables_initializer()\n", "saver = tf.train.Saver()" ] }, @@ -1076,7 +1222,9 @@ "cell_type": "code", "execution_count": 44, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1109,7 +1257,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# LSTM" ] @@ -1118,7 +1269,9 @@ "cell_type": "code", "execution_count": 45, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1136,26 +1289,28 @@ "X = tf.placeholder(tf.float32, [None, n_steps, n_inputs])\n", "y = tf.placeholder(tf.int32, [None])\n", "\n", - "lstm_cell = tf.nn.rnn_cell.BasicLSTMCell(num_units=n_neurons)\n", - "multi_cell = tf.nn.rnn_cell.MultiRNNCell([lstm_cell]*3)\n", + "lstm_cell = tf.contrib.rnn.BasicLSTMCell(num_units=n_neurons)\n", + "multi_cell = tf.contrib.rnn.MultiRNNCell([lstm_cell]*3)\n", "outputs, states = tf.nn.dynamic_rnn(multi_cell, X, dtype=tf.float32)\n", "top_layer_h_state = states[-1][1]\n", "logits = fully_connected(top_layer_h_state, n_outputs, activation_fn=None, scope=\"softmax\")\n", - "xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits, y)\n", + "xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n", "loss = tf.reduce_mean(xentropy, name=\"loss\")\n", "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n", "training_op = optimizer.minimize(loss)\n", "correct = tf.nn.in_top_k(logits, y, 1)\n", "accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n", " \n", - "init = tf.initialize_all_variables()" + "init = tf.global_variables_initializer()" ] }, { "cell_type": "code", "execution_count": 46, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1166,7 +1321,9 @@ "cell_type": "code", "execution_count": 47, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1177,7 +1334,9 @@ "cell_type": "code", "execution_count": 48, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1198,7 +1357,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Distributing layers across devices" ] @@ -1207,13 +1369,15 @@ "cell_type": "code", "execution_count": 49, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ "import tensorflow as tf\n", "\n", - "class DeviceCellWrapper(tf.nn.rnn_cell.RNNCell):\n", + "class DeviceCellWrapper(tf.contrib.rnn.RNNCell):\n", " def __init__(self, device, cell):\n", " self._cell = cell\n", " self._device = device\n", @@ -1235,7 +1399,9 @@ "cell_type": "code", "execution_count": 50, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1246,18 +1412,20 @@ "devices = [\"/cpu:0\"]*5\n", "n_steps = 20\n", "X = tf.placeholder(tf.float32, shape=[None, n_steps, n_inputs])\n", - "lstm_cells = [DeviceCellWrapper(device, tf.nn.rnn_cell.BasicRNNCell(num_units=n_neurons))\n", + "lstm_cells = [DeviceCellWrapper(device, tf.contrib.rnn.BasicRNNCell(num_units=n_neurons))\n", " for device in devices]\n", - "multi_layer_cell = tf.nn.rnn_cell.MultiRNNCell(lstm_cells)\n", + "multi_layer_cell = tf.contrib.rnn.MultiRNNCell(lstm_cells)\n", "outputs, states = tf.nn.dynamic_rnn(multi_layer_cell, X, dtype=tf.float32)\n", - "init = tf.initialize_all_variables()" + "init = tf.global_variables_initializer()" ] }, { "cell_type": "code", "execution_count": 51, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1268,30 +1436,41 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Embeddings" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "This section is based on TensorFlow's [Word2Vec tutorial](https://www.tensorflow.org/versions/r0.11/tutorials/word2vec/index.html)." ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Fetch the data" ] }, { "cell_type": "code", - "execution_count": 52, + "execution_count": 7, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1316,9 +1495,11 @@ }, { "cell_type": "code", - "execution_count": 53, + "execution_count": 8, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1327,9 +1508,11 @@ }, { "cell_type": "code", - "execution_count": 54, + "execution_count": 9, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1338,16 +1521,21 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Build the dictionary" ] }, { "cell_type": "code", - "execution_count": 55, + "execution_count": 10, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1363,9 +1551,11 @@ }, { "cell_type": "code", - "execution_count": 56, + "execution_count": 11, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1374,9 +1564,11 @@ }, { "cell_type": "code", - "execution_count": 57, + "execution_count": 12, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1385,9 +1577,11 @@ }, { "cell_type": "code", - "execution_count": 58, + "execution_count": 13, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1396,16 +1590,21 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Generate batches" ] }, { "cell_type": "code", - "execution_count": 59, + "execution_count": 14, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1439,9 +1638,11 @@ }, { "cell_type": "code", - "execution_count": 60, + "execution_count": 15, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1451,9 +1652,11 @@ }, { "cell_type": "code", - "execution_count": 61, + "execution_count": 16, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1462,9 +1665,11 @@ }, { "cell_type": "code", - "execution_count": 62, + "execution_count": 17, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1473,16 +1678,21 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Build the model" ] }, { "cell_type": "code", - "execution_count": 63, + "execution_count": 18, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1504,9 +1714,11 @@ }, { "cell_type": "code", - "execution_count": 64, + "execution_count": 19, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1532,7 +1744,7 @@ "# tf.nce_loss automatically draws a new sample of the negative labels each\n", "# time we evaluate the loss.\n", "loss = tf.reduce_mean(\n", - " tf.nn.nce_loss(nce_weights, nce_biases, embed, train_labels,\n", + " tf.nn.nce_loss(nce_weights, nce_biases, train_labels, embed,\n", " num_sampled, vocabulary_size))\n", "\n", "# Construct the Adam optimizer\n", @@ -1540,27 +1752,32 @@ "training_op = optimizer.minimize(loss)\n", "\n", "# Compute the cosine similarity between minibatch examples and all embeddings.\n", - "norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), reduction_indices=1, keep_dims=True))\n", + "norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), axis=1, keep_dims=True))\n", "normalized_embeddings = embeddings / norm\n", "valid_embeddings = tf.nn.embedding_lookup(normalized_embeddings, valid_dataset)\n", "similarity = tf.matmul(valid_embeddings, normalized_embeddings, transpose_b=True)\n", "\n", "# Add variable initializer.\n", - "init = tf.initialize_all_variables()" + "init = tf.global_variables_initializer()" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Train the model" ] }, { "cell_type": "code", - "execution_count": 65, + "execution_count": 20, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1605,34 +1822,44 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Let's save the final embeddings (of course you can use a TensorFlow `Saver` if you prefer):" ] }, { "cell_type": "code", - "execution_count": 66, + "execution_count": 21, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ - "np.save(\"my_final_embeddings.npy\", final_embeddings)" + "np.save(\"./my_final_embeddings.npy\", final_embeddings)" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Plot the embeddings" ] }, { "cell_type": "code", - "execution_count": 67, + "execution_count": 22, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1652,9 +1879,11 @@ }, { "cell_type": "code", - "execution_count": 68, + "execution_count": 23, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1669,23 +1898,31 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Machine Translation" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "The `basic_rnn_seq2seq()` function creates a simple Encoder/Decoder model: it first runs an RNN to encode `encoder_inputs` into a state vector, then runs a decoder initialized with the last encoder state on `decoder_inputs`. Encoder and decoder use the same RNN cell type but they don't share parameters." ] }, { "cell_type": "code", - "execution_count": 69, + "execution_count": 24, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1706,13 +1943,13 @@ "Y_input = Y[:, :-1]\n", "Y_target = Y[:, 1:]\n", "\n", - "encoder_inputs = tf.unpack(tf.transpose(X)) # list of 1D tensors\n", - "decoder_inputs = tf.unpack(tf.transpose(Y_input)) # list of 1D tensors\n", + "encoder_inputs = tf.unstack(tf.transpose(X)) # list of 1D tensors\n", + "decoder_inputs = tf.unstack(tf.transpose(Y_input)) # list of 1D tensors\n", "\n", - "lstm_cell = tf.nn.rnn_cell.BasicLSTMCell(num_units=n_neurons)\n", - "cell = tf.nn.rnn_cell.MultiRNNCell([lstm_cell] * n_layers)\n", + "lstm_cell = tf.contrib.rnn.BasicLSTMCell(num_units=n_neurons)\n", + "cell = tf.contrib.rnn.MultiRNNCell([lstm_cell] * n_layers)\n", "\n", - "output_seqs, states = tf.nn.seq2seq.embedding_rnn_seq2seq(\n", + "output_seqs, states = tf.contrib.legacy_seq2seq.embedding_rnn_seq2seq(\n", " encoder_inputs,\n", " decoder_inputs,\n", " cell,\n", @@ -1720,32 +1957,36 @@ " num_decoder_symbols,\n", " embedding_size)\n", "\n", - "logits = tf.transpose(tf.pack(output_seqs), perm=[1, 0, 2])" + "logits = tf.transpose(tf.unstack(output_seqs), perm=[1, 0, 2])" ] }, { "cell_type": "code", - "execution_count": 70, + "execution_count": 25, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ "logits_flat = tf.reshape(logits, [-1, num_decoder_symbols])\n", "Y_target_flat = tf.reshape(Y_target, [-1])\n", "W_flat = tf.reshape(W, [-1])\n", - "xentropy = W_flat * tf.nn.sparse_softmax_cross_entropy_with_logits(logits_flat, Y_target_flat)\n", + "xentropy = W_flat * tf.nn.sparse_softmax_cross_entropy_with_logits(labels=Y_target_flat, logits=logits_flat)\n", "loss = tf.reduce_mean(xentropy)\n", "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n", "training_op = optimizer.minimize(loss)\n", "\n", - "init = tf.initialize_all_variables()" + "init = tf.global_variables_initializer()" ] }, { "cell_type": "markdown", "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "source": [ "# Exercise solutions" @@ -1753,7 +1994,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "**Coming soon**" ] @@ -1775,7 +2019,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.5.2" + "version": "3.5.2+" }, "nav_menu": {}, "toc": { diff --git a/15_autoencoders.ipynb b/15_autoencoders.ipynb index 0140621..8868cc5 100644 --- a/15_autoencoders.ipynb +++ b/15_autoencoders.ipynb @@ -2,28 +2,40 @@ "cells": [ { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "**Chapter 15 – Autoencoders**" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "_This notebook contains all the sample code and solutions to the exercices in chapter 15._" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Setup" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "First, let's make sure this notebook works well in both python 2 and 3, import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures:" ] @@ -32,7 +44,9 @@ "cell_type": "code", "execution_count": 1, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -70,7 +84,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "A couple utility functions to plot grayscale 28x28 image:" ] @@ -79,7 +96,9 @@ "cell_type": "code", "execution_count": 2, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -92,7 +111,9 @@ "cell_type": "code", "execution_count": 3, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -109,14 +130,20 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# PCA with a linear Autoencoder" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Build 3D dataset:" ] @@ -125,7 +152,9 @@ "cell_type": "code", "execution_count": 4, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -143,7 +172,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Normalize the data:" ] @@ -152,7 +184,9 @@ "cell_type": "code", "execution_count": 5, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -163,7 +197,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Going to need TensorFlow..." ] @@ -172,7 +209,9 @@ "cell_type": "code", "execution_count": 6, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -181,7 +220,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Now let's build the Autoencoder:" ] @@ -190,7 +232,9 @@ "cell_type": "code", "execution_count": 7, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -213,14 +257,16 @@ "optimizer = tf.train.AdamOptimizer(learning_rate)\n", "training_op = optimizer.minimize(mse)\n", "\n", - "init = tf.initialize_all_variables()" + "init = tf.global_variables_initializer()" ] }, { "cell_type": "code", "execution_count": 8, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -238,7 +284,9 @@ "cell_type": "code", "execution_count": 9, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -252,14 +300,20 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Stacked Autoencoders" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Let's use MNIST:" ] @@ -268,7 +322,9 @@ "cell_type": "code", "execution_count": 10, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -278,14 +334,20 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Train all layers at once" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Let's build a stacked Autoencoder with 3 hidden layers and 1 output layer (ie. 2 stacked Autoencoders). We will use ELU activation, He initialization and L2 regularization." ] @@ -294,7 +356,9 @@ "cell_type": "code", "execution_count": 11, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -334,13 +398,16 @@ "optimizer = tf.train.AdamOptimizer(learning_rate)\n", "training_op = optimizer.minimize(loss)\n", "\n", - "init = tf.initialize_all_variables()\n", + "init = tf.global_variables_initializer()\n", "saver = tf.train.Saver()" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Now let's train it! Note that we don't feed target values (`y_batch` is not used). This is unsupervised training." ] @@ -349,7 +416,9 @@ "cell_type": "code", "execution_count": 12, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -367,12 +436,15 @@ " sess.run(training_op, feed_dict={X: X_batch})\n", " mse_train = mse.eval(feed_dict={X: X_batch})\n", " print(\"\\r{}\".format(epoch), \"Train MSE:\", mse_train)\n", - " saver.save(sess, \"my_model_all_layers.ckpt\")" + " saver.save(sess, \"./my_model_all_layers.ckpt\")" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "This function loads the model, evaluates it on the test set (it measures the reconstruction error), then it displays the original image and its reconstruction:" ] @@ -381,7 +453,9 @@ "cell_type": "code", "execution_count": 13, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -404,31 +478,42 @@ "cell_type": "code", "execution_count": 14, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ - "show_reconstructed_digits(X, outputs, \"my_model_all_layers.ckpt\")\n", + "show_reconstructed_digits(X, outputs, \"./my_model_all_layers.ckpt\")\n", "save_fig(\"reconstruction_plot\")" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Training one Autoencoder at a time in multiple graphs" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "There are many ways to train one Autoencoder at a time. The first approach it to train each Autoencoder using a different graph, then we create the Stacked Autoencoder by simply initializing it with the weights and biases copied from these Autoencoders." ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Let's create a function that will train one autoencoder and return the transformed training set (ie. the output of the hidden layer) and the model parameters." ] @@ -437,7 +522,9 @@ "cell_type": "code", "execution_count": 15, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -463,7 +550,7 @@ " optimizer = tf.train.AdamOptimizer(learning_rate)\n", " training_op = optimizer.minimize(loss)\n", "\n", - " init = tf.initialize_all_variables()\n", + " init = tf.global_variables_initializer()\n", "\n", " with tf.Session(graph=graph) as sess:\n", " init.run()\n", @@ -484,7 +571,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Now let's train two Autoencoders. The first one is trained on the training data, and the second is trained on the previous Autoencoder's hidden layer output:" ] @@ -493,7 +583,9 @@ "cell_type": "code", "execution_count": 16, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -503,7 +595,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Finally, we can create a Stacked Autoencoder by simply reusing the weights and biases from the Autoencoders we just trained:" ] @@ -512,7 +607,9 @@ "cell_type": "code", "execution_count": 17, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -531,7 +628,9 @@ "cell_type": "code", "execution_count": 18, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -540,14 +639,20 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Training one Autoencoder at a time in a single graph" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Another approach is to use a single graph. To do this, we create the graph for the full Stacked Autoencoder, but then we also add operations to train each Autoencoder independently: phase 1 trains the bottom and top layer (ie. the first Autoencoder) and phase 2 trains the two middle layers (ie. the second Autoencoder)." ] @@ -556,7 +661,9 @@ "cell_type": "code", "execution_count": 19, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -613,7 +720,7 @@ " phase2_loss = phase2_mse + phase2_reg_loss\n", " phase2_training_op = optimizer.minimize(phase2_loss, var_list=[weights2, biases2, weights3, biases3]) # freeze hidden1\n", " \n", - "init = tf.initialize_all_variables()\n", + "init = tf.global_variables_initializer()\n", "saver = tf.train.Saver()" ] }, @@ -621,7 +728,9 @@ "cell_type": "code", "execution_count": 20, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -643,7 +752,7 @@ " sess.run(training_ops[phase], feed_dict={X: X_batch})\n", " mse_train = mses[phase].eval(feed_dict={X: X_batch})\n", " print(\"\\r{}\".format(epoch), \"Train MSE:\", mse_train)\n", - " saver.save(sess, \"my_model_one_at_a_time.ckpt\")\n", + " saver.save(sess, \"./my_model_one_at_a_time.ckpt\")\n", " mse_test = mses[phase].eval(feed_dict={X: mnist.test.images})\n", " print(\"Test MSE:\", mse_test)" ] @@ -652,16 +761,21 @@ "cell_type": "code", "execution_count": 21, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ - "show_reconstructed_digits(X, outputs, \"my_model_one_at_a_time.ckpt\")" + "show_reconstructed_digits(X, outputs, \"./my_model_one_at_a_time.ckpt\")" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Cache the frozen layer outputs" ] @@ -670,7 +784,9 @@ "cell_type": "code", "execution_count": 22, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -701,7 +817,7 @@ " sess.run(training_ops[phase], feed_dict=feed_dict)\n", " mse_train = mses[phase].eval(feed_dict=feed_dict)\n", " print(\"\\r{}\".format(epoch), \"Train MSE:\", mse_train)\n", - " saver.save(sess, \"my_model_cache_frozen.ckpt\")\n", + " saver.save(sess, \"./my_model_cache_frozen.ckpt\")\n", " mse_test = mses[phase].eval(feed_dict={X: mnist.test.images})\n", " print(\"Test MSE:\", mse_test)" ] @@ -711,23 +827,31 @@ "execution_count": 23, "metadata": { "collapsed": false, + "deletable": true, + "editable": true, "scrolled": true }, "outputs": [], "source": [ - "show_reconstructed_digits(X, outputs, \"my_model_cache_frozen.ckpt\")" + "show_reconstructed_digits(X, outputs, \"./my_model_cache_frozen.ckpt\")" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Tying weights" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "It is common to tie the weights of the encoder and the decoder (`weights_decoder = tf.transpose(weights_encoder)`). Unfortunately this makes it impossible (or very tricky) to use the `fully_connected()` function, so we need to build the Autoencoder manually:" ] @@ -736,7 +860,9 @@ "cell_type": "code", "execution_count": 24, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -782,7 +908,7 @@ "optimizer = tf.train.AdamOptimizer(learning_rate)\n", "training_op = optimizer.minimize(loss)\n", "\n", - "init = tf.initialize_all_variables()\n", + "init = tf.global_variables_initializer()\n", "saver = tf.train.Saver()" ] }, @@ -790,7 +916,9 @@ "cell_type": "code", "execution_count": 25, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -808,23 +936,28 @@ " sess.run(training_op, feed_dict={X: X_batch})\n", " mse_train = mse.eval(feed_dict={X: X_batch})\n", " print(\"\\r{}\".format(epoch), \"Train MSE:\", mse_train)\n", - " saver.save(sess, \"my_model_tying_weights.ckpt\")" + " saver.save(sess, \"./my_model_tying_weights.ckpt\")" ] }, { "cell_type": "code", "execution_count": 26, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ - "show_reconstructed_digits(X, outputs, \"my_model_tying_weights.ckpt\")" + "show_reconstructed_digits(X, outputs, \"./my_model_tying_weights.ckpt\")" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Unsupervised pretraining" ] @@ -833,7 +966,9 @@ "cell_type": "code", "execution_count": 27, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -870,7 +1005,7 @@ "hidden2 = activation(tf.matmul(hidden1, weights2) + biases2)\n", "logits = tf.matmul(hidden2, weights3) + biases3\n", "\n", - "cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits, y)\n", + "cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n", "reg_loss = regularizer(weights1) + regularizer(weights2) + regularizer(weights3)\n", "loss = cross_entropy + reg_loss\n", "optimizer = tf.train.AdamOptimizer(learning_rate)\n", @@ -879,14 +1014,17 @@ "correct = tf.nn.in_top_k(logits, y, 1)\n", "accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n", "\n", - "init = tf.initialize_all_variables()\n", + "init = tf.global_variables_initializer()\n", "pretrain_saver = tf.train.Saver([weights1, weights2, biases1, biases2])\n", "saver = tf.train.Saver()" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Regular training (without pretraining):" ] @@ -895,7 +1033,9 @@ "cell_type": "code", "execution_count": 28, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -915,14 +1055,17 @@ " sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n", " accuracy_val = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n", " print(\"\\r{}\".format(epoch), \"Train accuracy:\", accuracy_val, end=\" \")\n", - " saver.save(sess, \"my_model_supervised.ckpt\")\n", + " saver.save(sess, \"./my_model_supervised.ckpt\")\n", " accuracy_val = accuracy.eval(feed_dict={X: mnist.test.images, y: mnist.test.labels})\n", " print(\"Test accuracy:\", accuracy_val)" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Now reusing the first two layers of the autoencoder we pretrained:" ] @@ -931,7 +1074,9 @@ "cell_type": "code", "execution_count": 29, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -943,7 +1088,7 @@ "\n", "with tf.Session() as sess:\n", " init.run()\n", - " pretrain_saver.restore(sess, \"my_model_cache_frozen.ckpt\")\n", + " pretrain_saver.restore(sess, \"./my_model_cache_frozen.ckpt\")\n", " for epoch in range(n_epochs):\n", " n_batches = n_labeled_instances // batch_size\n", " for iteration in range(n_batches):\n", @@ -954,14 +1099,17 @@ " sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n", " accuracy_val = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n", " print(\"\\r{}\".format(epoch), \"Train accuracy:\", accuracy_val, end=\"\\t\")\n", - " saver.save(sess, \"my_model_supervised_pretrained.ckpt\")\n", + " saver.save(sess, \"./my_model_supervised_pretrained.ckpt\")\n", " accuracy_val = accuracy.eval(feed_dict={X: mnist.test.images, y: mnist.test.labels})\n", " print(\"Test accuracy:\", accuracy_val)" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Stacked denoising Autoencoder" ] @@ -970,7 +1118,9 @@ "cell_type": "code", "execution_count": 30, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1021,7 +1171,7 @@ "loss = mse + reg_loss\n", "training_op = optimizer.minimize(loss)\n", " \n", - "init = tf.initialize_all_variables()\n", + "init = tf.global_variables_initializer()\n", "saver = tf.train.Saver()" ] }, @@ -1029,7 +1179,9 @@ "cell_type": "code", "execution_count": 31, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1047,23 +1199,28 @@ " sess.run(training_op, feed_dict={X: X_batch, is_training: True})\n", " mse_train = mse.eval(feed_dict={X: X_batch, is_training: False})\n", " print(\"\\r{}\".format(epoch), \"Train MSE:\", mse_train)\n", - " saver.save(sess, \"my_model_stacked_denoising.ckpt\")" + " saver.save(sess, \"./my_model_stacked_denoising.ckpt\")" ] }, { "cell_type": "code", "execution_count": 32, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ - "show_reconstructed_digits(X, outputs, \"my_model_stacked_denoising.ckpt\")" + "show_reconstructed_digits(X, outputs, \"./my_model_stacked_denoising.ckpt\")" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Visualizing the extracted features" ] @@ -1072,12 +1229,14 @@ "cell_type": "code", "execution_count": 33, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ "with tf.Session() as sess:\n", - " saver.restore(sess, \"my_model_stacked_denoising.ckpt\")\n", + " saver.restore(sess, \"./my_model_stacked_denoising.ckpt\")\n", " weights1_val = weights1.eval()" ] }, @@ -1085,7 +1244,9 @@ "cell_type": "code", "execution_count": 34, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1099,16 +1260,21 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Sparse Autoencoder" ] }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 35, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1129,9 +1295,11 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 36, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1142,9 +1310,11 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 37, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1179,20 +1349,22 @@ "optimizer = tf.train.AdamOptimizer(learning_rate)\n", "mse = tf.reduce_mean(tf.square(outputs - X))\n", "\n", - "hidden1_mean = tf.reduce_mean(hidden1, reduction_indices=0) # batch mean\n", + "hidden1_mean = tf.reduce_mean(hidden1, axis=0) # batch mean\n", "sparsity_loss = tf.reduce_sum(kl_divergence(sparsity_target, hidden1_mean))\n", "loss = mse + sparsity_weight * sparsity_loss\n", "training_op = optimizer.minimize(loss)\n", "\n", - "init = tf.initialize_all_variables()\n", + "init = tf.global_variables_initializer()\n", "saver = tf.train.Saver()" ] }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 38, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1210,32 +1382,39 @@ " sess.run(training_op, feed_dict={X: X_batch})\n", " mse_val, sparsity_loss_val, loss_val = sess.run([mse, sparsity_loss, loss], feed_dict={X: X_batch})\n", " print(\"\\r{}\".format(epoch), \"Train MSE:\", mse_val, \"\\tSparsity loss:\", sparsity_loss_val, \"\\tTotal loss:\", loss_val)\n", - " saver.save(sess, \"my_model_sparse.ckpt\")" + " saver.save(sess, \"./my_model_sparse.ckpt\")" ] }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 39, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ - "show_reconstructed_digits(X, outputs, \"my_model_sparse.ckpt\")" + "show_reconstructed_digits(X, outputs, \"./my_model_sparse.ckpt\")" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Variational Autoencoder" ] }, { "cell_type": "code", - "execution_count": 41, + "execution_count": 40, "metadata": { - "collapsed": true + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1286,22 +1465,24 @@ "logits = tf.matmul(hidden5, weights6) + biases6\n", "outputs = tf.sigmoid(logits)\n", "\n", - "reconstruction_loss = tf.reduce_sum(tf.nn.sigmoid_cross_entropy_with_logits(logits, X))\n", + "reconstruction_loss = tf.reduce_sum(tf.nn.sigmoid_cross_entropy_with_logits(labels=X, logits=logits))\n", "latent_loss = 0.5 * tf.reduce_sum(tf.exp(hidden3_log_sigma) + tf.square(hidden3_mean) - 1 - hidden3_log_sigma)\n", "cost = reconstruction_loss + latent_loss\n", "\n", "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n", "training_op = optimizer.minimize(cost)\n", "\n", - "init = tf.initialize_all_variables()\n", + "init = tf.global_variables_initializer()\n", "saver = tf.train.Saver()" ] }, { "cell_type": "code", - "execution_count": 42, + "execution_count": 41, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1334,22 +1515,24 @@ " logits = fully_connected(hidden5, n_outputs, activation_fn=None)\n", " outputs = tf.sigmoid(logits)\n", "\n", - "reconstruction_loss = tf.reduce_sum(tf.nn.sigmoid_cross_entropy_with_logits(logits, X))\n", + "reconstruction_loss = tf.reduce_sum(tf.nn.sigmoid_cross_entropy_with_logits(labels=X, logits=logits))\n", "latent_loss = 0.5 * tf.reduce_sum(tf.exp(hidden3_gamma) + tf.square(hidden3_mean) - 1 - hidden3_gamma)\n", "cost = reconstruction_loss + latent_loss\n", "\n", "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n", "training_op = optimizer.minimize(cost)\n", "\n", - "init = tf.initialize_all_variables()\n", + "init = tf.global_variables_initializer()\n", "saver = tf.train.Saver()" ] }, { "cell_type": "code", - "execution_count": 43, + "execution_count": 42, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1367,21 +1550,26 @@ " sess.run(training_op, feed_dict={X: X_batch})\n", " cost_val, reconstruction_loss_val, latent_loss_val = sess.run([cost, reconstruction_loss, latent_loss], feed_dict={X: X_batch})\n", " print(\"\\r{}\".format(epoch), \"Train cost:\", cost_val, \"\\tReconstruction loss:\", reconstruction_loss_val, \"\\tLatent loss:\", latent_loss_val)\n", - " saver.save(sess, \"my_model_variational.ckpt\")" + " saver.save(sess, \"./my_model_variational.ckpt\")" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Encode:" ] }, { "cell_type": "code", - "execution_count": 44, + "execution_count": 43, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1390,42 +1578,52 @@ "codings = hidden3\n", "\n", "with tf.Session() as sess:\n", - " saver.restore(sess, \"my_model_variational.ckpt\")\n", + " saver.restore(sess, \"./my_model_variational.ckpt\")\n", " codings_val = codings.eval(feed_dict={X: X_test})" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Decode:" ] }, { "cell_type": "code", - "execution_count": 45, + "execution_count": 44, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ "with tf.Session() as sess:\n", - " saver.restore(sess, \"my_model_variational.ckpt\")\n", + " saver.restore(sess, \"./my_model_variational.ckpt\")\n", " outputs_val = outputs.eval(feed_dict={codings: codings_val})" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Let's plot the reconstructions:" ] }, { "cell_type": "code", - "execution_count": 46, + "execution_count": 45, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1439,16 +1637,21 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Generate digits" ] }, { "cell_type": "code", - "execution_count": 47, + "execution_count": 46, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1458,15 +1661,17 @@ "codings_rnd = np.random.normal(size=[n_digits, n_hidden3])\n", "\n", "with tf.Session() as sess:\n", - " saver.restore(sess, \"my_model_variational.ckpt\")\n", + " saver.restore(sess, \"./my_model_variational.ckpt\")\n", " outputs_val = outputs.eval(feed_dict={codings: codings_rnd})" ] }, { "cell_type": "code", - "execution_count": 48, + "execution_count": 47, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1477,9 +1682,11 @@ }, { "cell_type": "code", - "execution_count": 49, + "execution_count": 48, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1489,22 +1696,27 @@ "codings_rnd = np.random.normal(size=[n_digits, n_hidden3])\n", "\n", "with tf.Session() as sess:\n", - " saver.restore(sess, \"my_model_variational.ckpt\")\n", + " saver.restore(sess, \"./my_model_variational.ckpt\")\n", " outputs_val = outputs.eval(feed_dict={codings: codings_rnd})" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Interpolate digits" ] }, { "cell_type": "code", - "execution_count": 50, + "execution_count": 49, "metadata": { "collapsed": false, + "deletable": true, + "editable": true, "scrolled": true }, "outputs": [], @@ -1514,7 +1726,7 @@ "codings_rnd = np.random.normal(size=[n_digits, n_hidden3])\n", "\n", "with tf.Session() as sess:\n", - " saver.restore(sess, \"my_model_variational.ckpt\")\n", + " saver.restore(sess, \"./my_model_variational.ckpt\")\n", " target_codings = np.roll(codings_rnd, -1, axis=0)\n", " for iteration in range(n_iterations + 1):\n", " codings_interpolate = codings_rnd + (target_codings - codings_rnd) * iteration / n_iterations\n", @@ -1529,7 +1741,9 @@ { "cell_type": "markdown", "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "source": [ "# Exercise solutions" @@ -1537,7 +1751,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Coming soon..." ] @@ -1546,7 +1763,9 @@ "cell_type": "code", "execution_count": null, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [] diff --git a/16_reinforcement_learning.ipynb b/16_reinforcement_learning.ipynb index 3785a35..1ac44af 100644 --- a/16_reinforcement_learning.ipynb +++ b/16_reinforcement_learning.ipynb @@ -2,28 +2,40 @@ "cells": [ { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "**Chapter 16 – Reinforcement Learning**" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "This notebook contains all the sample code and solutions to the exercices in chapter 16." ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Setup" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "First, let's make sure this notebook works well in both python 2 and 3, import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures:" ] @@ -32,7 +44,9 @@ "cell_type": "code", "execution_count": 1, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -43,6 +57,7 @@ "import numpy as np\n", "import numpy.random as rnd\n", "import os\n", + "import sys\n", "\n", "# to make this notebook's output stable across runs\n", "rnd.seed(42)\n", @@ -70,14 +85,20 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Introduction to OpenAI gym" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "In this notebook we will be using [OpenAI gym](https://gym.openai.com/), a great toolkit for developing and comparing Reinforcement Learning algorithms. It provides many environments for your learning *agents* to interact with. Let's start by importing `gym`:" ] @@ -86,7 +107,9 @@ "cell_type": "code", "execution_count": 2, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -95,7 +118,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Next we will load the MsPacman environment, version 0." ] @@ -104,7 +130,9 @@ "cell_type": "code", "execution_count": 3, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -113,7 +141,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Let's initialize the environment by calling is `reset()` method. This returns an observation:" ] @@ -122,7 +153,9 @@ "cell_type": "code", "execution_count": 4, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -131,7 +164,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Observations vary depending on the environment. In this case it is an RGB image represented as a 3D NumPy array of shape [width, height, channels] (with 3 channels: Red, Green and Blue). In other environments it may return different objects, as we will see later." ] @@ -140,7 +176,9 @@ "cell_type": "code", "execution_count": 5, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -149,7 +187,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "An environment can be visualized by calling its `render()` method, and you can pick the rendering mode (the rendering options depend on the environment). In this example we will set `mode=\"rgb_array\"` to get an image of the environment as a NumPy array:" ] @@ -158,7 +199,9 @@ "cell_type": "code", "execution_count": 6, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -167,7 +210,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Let's plot this image:" ] @@ -177,6 +223,8 @@ "execution_count": 7, "metadata": { "collapsed": false, + "deletable": true, + "editable": true, "scrolled": true }, "outputs": [], @@ -190,14 +238,20 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Welcome back to the 1980s! :)" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "In this environment, the rendered image is simply equal to the observation (but in many environments this is not the case):" ] @@ -206,7 +260,9 @@ "cell_type": "code", "execution_count": 8, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -215,7 +271,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Let's create a little helper function to plot an environment:" ] @@ -224,7 +283,9 @@ "cell_type": "code", "execution_count": 9, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -239,7 +300,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Let's see how to interact with an environment. Your agent will need to select an action from an \"action space\" (the set of possible actions). Let's see what this environment's action space looks like:" ] @@ -248,7 +312,9 @@ "cell_type": "code", "execution_count": 10, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -257,14 +323,20 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "`Discrete(9)` means that the possible actions are integers 0 through 8, which represents the 9 possible positions of the joystick (0=center, 1=up, 2=right, 3=left, 4=down, 5=upper-right, 6=upper-left, 7=lower-right, 8=lower-left)." ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Next we need to tell the environment which action to play, and it will compute the next step of the game. Let's go left for 110 steps, then lower left for 40 steps:" ] @@ -273,7 +345,9 @@ "cell_type": "code", "execution_count": 11, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -286,7 +360,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Where are we now?" ] @@ -295,7 +372,9 @@ "cell_type": "code", "execution_count": 12, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -304,7 +383,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "The `step()` function actually returns several important objects:" ] @@ -313,7 +395,9 @@ "cell_type": "code", "execution_count": 13, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -322,7 +406,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "The observation tells the agent what the environment looks like, as discussed earlier. This is a 210x160 RGB image:" ] @@ -331,7 +418,9 @@ "cell_type": "code", "execution_count": 14, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -340,7 +429,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "The environment also tells the agent how much reward it got during the last step:" ] @@ -349,7 +441,9 @@ "cell_type": "code", "execution_count": 15, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -358,7 +452,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "When the game is over, the environment returns `done=True`:" ] @@ -367,7 +464,9 @@ "cell_type": "code", "execution_count": 16, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -376,7 +475,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Finally, `info` is an environment-specific dictionary that can provide some extra information about the internal state of the environment. This is useful for debugging, but your agent should not use this information for learning (it would be cheating)." ] @@ -385,7 +487,9 @@ "cell_type": "code", "execution_count": 17, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -394,7 +498,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Let's play one full game (with 3 lives), by moving in random directions for 10 steps at a time, recording each frame:" ] @@ -403,7 +510,9 @@ "cell_type": "code", "execution_count": 18, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -425,7 +534,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Now show the animation (it's a bit jittery within Jupyter):" ] @@ -434,7 +546,9 @@ "cell_type": "code", "execution_count": 19, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -454,7 +568,9 @@ "cell_type": "code", "execution_count": 20, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -464,7 +580,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Once you have finished playing with an environment, you should close it to free up resources:" ] @@ -473,7 +592,9 @@ "cell_type": "code", "execution_count": 21, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -482,21 +603,30 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "To code our first learning agent, we will be using a simpler environment: the Cart-Pole. " ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# A simple environment: the Cart-Pole" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "The Cart-Pole is a very simple environment composed of a cart that can move left or right, and pole placed vertically on top of it. The agent must move the cart left or right to keep the pole upright." ] @@ -505,7 +635,9 @@ "cell_type": "code", "execution_count": 22, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -516,7 +648,9 @@ "cell_type": "code", "execution_count": 23, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -527,7 +661,9 @@ "cell_type": "code", "execution_count": 24, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -536,21 +672,30 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "The observation is a 1D NumPy array composed of 4 floats: they represent the cart's horizontal position, its velocity, the angle of the pole (0 = vertical), and the angular velocity. Let's render the environment... unfortunately we need to fix an annoying rendering issue first." ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Fixing the rendering issue" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Some environments (including the Cart-Pole) require access to your display, which opens up a separate window, even if you specify the `rgb_array` mode. In general you can safely ignore that window. However, if Jupyter is running on a headless server (ie. without a screen) it will raise an exception. One way to avoid this is to install a fake X server like Xvfb. You can start Jupyter using the `xvfb-run` command:\n", "\n", @@ -563,7 +708,9 @@ "cell_type": "code", "execution_count": 25, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -617,7 +764,9 @@ "cell_type": "code", "execution_count": 26, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -626,7 +775,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Now let's look at the action space:" ] @@ -635,7 +787,9 @@ "cell_type": "code", "execution_count": 27, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -644,7 +798,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Yep, just two possible actions: accelerate towards the left or towards the right. Let's push the cart left until the pole falls:" ] @@ -653,7 +810,9 @@ "cell_type": "code", "execution_count": 28, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -668,7 +827,9 @@ "cell_type": "code", "execution_count": 29, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -681,7 +842,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Notice that the game is over when the pole tilts too much, not when it actually falls. Now let's reset the environment and push the cart to right instead:" ] @@ -690,7 +854,9 @@ "cell_type": "code", "execution_count": 30, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -705,7 +871,9 @@ "cell_type": "code", "execution_count": 31, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -714,21 +882,30 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Looks like it's doing what we're telling it to do. Now how can we make the poll remain upright? We will need to define a _policy_ for that. This is the strategy that the agent will use to select an action at each step. It can use all the past actions and observations to decide what to do." ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# A simple hard-coded policy" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Let's hard code a simple strategy: if the pole is tilting to the left, then push the cart to the left, and _vice versa_. Let's see if that works:" ] @@ -737,7 +914,9 @@ "cell_type": "code", "execution_count": 32, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -767,7 +946,9 @@ "cell_type": "code", "execution_count": 33, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -777,21 +958,30 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Nope, the system is unstable and after just a few wobbles, the pole ends up too tilted: game over. We will need to be smarter than that!" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Neural Network Policies" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Let's create a neural network that will take observations as inputs, and output the action to take for each observation. To choose an action, the network will first estimate a probability for each action, then select an action randomly according to the estimated probabilities. In the case of the Cart-Pole environment, there are just two possible actions (left or right), so we only need one output neuron: it will output the probability `p` of the action 0 (left), and of course the probability of action 1 (right) will be `1 - p`." ] @@ -800,7 +990,9 @@ "cell_type": "code", "execution_count": 34, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -821,29 +1013,38 @@ " weights_initializer=initializer)\n", "\n", "# 3. Select a random action based on the estimated probabilities\n", - "p_left_and_right = tf.concat(concat_dim=1, values=[outputs, 1 - outputs])\n", + "p_left_and_right = tf.concat(axis=1, values=[outputs, 1 - outputs])\n", "action = tf.multinomial(tf.log(p_left_and_right), num_samples=1)\n", "\n", - "init = tf.initialize_all_variables()" + "init = tf.global_variables_initializer()" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "In this particular environment, the past actions and observations can safely be ignored, since each observation contains the environment's full state. If there were some hidden state then you may need to consider past actions and observations in order to try to infer the hidden state of the environment. For example, if the environment only revealed the position of the cart but not its velocity, you would have to consider not only the current observation but also the previous observation in order to estimate the current velocity. Another example is if the observations are noisy: you may want to use the past few observations to estimate the most likely current state. Our problem is thus as simple as can be: the current observation is noise-free and contains the environment's full state." ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "You may wonder why we are picking a random action based on the probability given by the policy network, rather than just picking the action with the highest probability. This approach lets the agent find the right balance between _exploring_ new actions and _exploiting_ the actions that are known to work well. Here's an analogy: suppose you go to a restaurant for the first time, and all the dishes look equally appealing so you randomly pick one. If it turns out to be good, you can increase the probability to order it next time, but you shouldn't increase that probability to 100%, or else you will never try out the other dishes, some of which may be even better than the one you tried." ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Let's randomly initialize this policy neural network and use it to play one game:" ] @@ -852,7 +1053,9 @@ "cell_type": "code", "execution_count": 35, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -875,7 +1078,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Now let's look at how well this randomly initialized policy network performed:" ] @@ -884,7 +1090,9 @@ "cell_type": "code", "execution_count": 36, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -894,7 +1102,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Yeah... pretty bad. The neural network will have to learn to do better. First let's see if it is capable of learning the basic policy we used earlier: go left if the pole is tilting left, and go right if it is tilting right. The following code defines the same neural network but we add the target probabilities `y`, and the training operations (`cross_entropy`, `optimizer` and `training_op`):" ] @@ -903,7 +1114,9 @@ "cell_type": "code", "execution_count": 37, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -926,20 +1139,23 @@ "hidden = fully_connected(X, n_hidden, activation_fn=tf.nn.elu, weights_initializer=initializer)\n", "logits = fully_connected(hidden, n_outputs, activation_fn=None)\n", "outputs = tf.nn.sigmoid(logits) # probability of action 0 (left)\n", - "p_left_and_right = tf.concat(concat_dim=1, values=[outputs, 1 - outputs])\n", + "p_left_and_right = tf.concat(axis=1, values=[outputs, 1 - outputs])\n", "action = tf.multinomial(tf.log(p_left_and_right), num_samples=1)\n", "\n", - "cross_entropy = tf.nn.sigmoid_cross_entropy_with_logits(logits, y)\n", + "cross_entropy = tf.nn.sigmoid_cross_entropy_with_logits(labels=y, logits=logits)\n", "optimizer = tf.train.AdamOptimizer(learning_rate)\n", "training_op = optimizer.minimize(cross_entropy)\n", "\n", - "init = tf.initialize_all_variables()\n", + "init = tf.global_variables_initializer()\n", "saver = tf.train.Saver()" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "We can make the same net play in 10 different environments in parallel, and train for 1000 iterations. We also reset environments when they are done." ] @@ -948,7 +1164,9 @@ "cell_type": "code", "execution_count": 38, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -966,7 +1184,7 @@ " for env_index, env in enumerate(envs):\n", " obs, reward, done, info = env.step(action_val[env_index][0])\n", " observations[env_index] = obs if not done else env.reset()\n", - " saver.save(sess, \"my_policy_net_basic.ckpt\")\n", + " saver.save(sess, \"./my_policy_net_basic.ckpt\")\n", "\n", "for env in envs:\n", " env.close()" @@ -976,7 +1194,9 @@ "cell_type": "code", "execution_count": 39, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1001,32 +1221,43 @@ "cell_type": "code", "execution_count": 40, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ - "frames = render_policy_net(\"my_policy_net_basic.ckpt\", action, X)\n", + "frames = render_policy_net(\"./my_policy_net_basic.ckpt\", action, X)\n", "video = plot_animation(frames)\n", "plt.show()" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Looks like it learned the policy correctly. Now let's see if it can learn a better policy on its own." ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Policy Gradients" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "To train this neural network we will need to define the target probabilities `y`. If an action is good we should increase its probability, and conversely if it is bad we should reduce it. But how do we know whether an action is good or bad? The problem is that most actions have delayed effects, so when you win or lose points in a game, it is not clear which actions contributed to this result: was it just the last action? Or the last 10? Or just one action 50 steps earlier? This is called the _credit assignment problem_.\n", "\n", @@ -1037,7 +1268,9 @@ "cell_type": "code", "execution_count": 41, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1059,11 +1292,11 @@ "hidden = fully_connected(X, n_hidden, activation_fn=tf.nn.elu, weights_initializer=initializer)\n", "logits = fully_connected(hidden, n_outputs, activation_fn=None)\n", "outputs = tf.nn.sigmoid(logits) # probability of action 0 (left)\n", - "p_left_and_right = tf.concat(concat_dim=1, values=[outputs, 1 - outputs])\n", + "p_left_and_right = tf.concat(axis=1, values=[outputs, 1 - outputs])\n", "action = tf.multinomial(tf.log(p_left_and_right), num_samples=1)\n", "\n", "y = 1. - tf.to_float(action)\n", - "cross_entropy = tf.nn.sigmoid_cross_entropy_with_logits(logits, y)\n", + "cross_entropy = tf.nn.sigmoid_cross_entropy_with_logits(labels=y, logits=logits)\n", "optimizer = tf.train.AdamOptimizer(learning_rate)\n", "grads_and_vars = optimizer.compute_gradients(cross_entropy)\n", "gradients = [grad for grad, variable in grads_and_vars]\n", @@ -1075,7 +1308,7 @@ " grads_and_vars_feed.append((gradient_placeholder, variable))\n", "training_op = optimizer.apply_gradients(grads_and_vars_feed)\n", "\n", - "init = tf.initialize_all_variables()\n", + "init = tf.global_variables_initializer()\n", "saver = tf.train.Saver()" ] }, @@ -1083,7 +1316,9 @@ "cell_type": "code", "execution_count": 42, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1107,7 +1342,9 @@ "cell_type": "code", "execution_count": 43, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1118,7 +1355,9 @@ "cell_type": "code", "execution_count": 44, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1127,9 +1366,11 @@ }, { "cell_type": "code", - "execution_count": 45, + "execution_count": 48, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1144,7 +1385,7 @@ "with tf.Session() as sess:\n", " init.run()\n", " for iteration in range(n_iterations):\n", - " print(\"\\r{}\\tTotal rewards: \".format(iteration), end=\"\")\n", + " print(\"\\rIteration: {}\".format(iteration), end=\"\")\n", " all_rewards = []\n", " all_gradients = []\n", " for game in range(n_games_per_update):\n", @@ -1160,7 +1401,6 @@ " break\n", " all_rewards.append(current_rewards)\n", " all_gradients.append(current_gradients)\n", - " print(np.sum(current_rewards), end=\" \")\n", "\n", " all_rewards = discount_and_normalize_rewards(all_rewards, discount_rate=discount_rate)\n", " feed_dict = {}\n", @@ -1171,14 +1411,16 @@ " feed_dict[gradient_placeholder] = mean_gradients\n", " sess.run(training_op, feed_dict=feed_dict)\n", " if iteration % save_iterations == 0:\n", - " saver.save(sess, \"my_policy_net_pg.ckpt\")" + " saver.save(sess, \"./my_policy_net_pg.ckpt\")" ] }, { "cell_type": "code", - "execution_count": 46, + "execution_count": 49, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1187,29 +1429,36 @@ }, { "cell_type": "code", - "execution_count": 47, + "execution_count": 50, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ - "frames = render_policy_net(\"my_policy_net_pg.ckpt\", action, X, n_max_steps=1000)\n", + "frames = render_policy_net(\"./my_policy_net_pg.ckpt\", action, X, n_max_steps=1000)\n", "video = plot_animation(frames)\n", "plt.show()" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Markov Chains" ] }, { "cell_type": "code", - "execution_count": 48, + "execution_count": 51, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1240,16 +1489,21 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Markov Decision Process" ] }, { "cell_type": "code", - "execution_count": 49, + "execution_count": 52, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1320,23 +1574,31 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Q-Learning" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Q-Learning will learn the optimal policy by watching the random policy play." ] }, { "cell_type": "code", - "execution_count": 50, + "execution_count": 53, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1361,9 +1623,11 @@ }, { "cell_type": "code", - "execution_count": 51, + "execution_count": 54, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1373,9 +1637,11 @@ }, { "cell_type": "code", - "execution_count": 52, + "execution_count": 55, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1384,9 +1650,11 @@ }, { "cell_type": "code", - "execution_count": 53, + "execution_count": 56, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1399,16 +1667,21 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Learning to play MsPacman using Deep Q-Learning" ] }, { "cell_type": "code", - "execution_count": 54, + "execution_count": 57, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1418,9 +1691,11 @@ }, { "cell_type": "code", - "execution_count": 55, + "execution_count": 58, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1429,9 +1704,11 @@ }, { "cell_type": "code", - "execution_count": 56, + "execution_count": 59, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1440,23 +1717,31 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Preprocessing" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Preprocessing the images is optional but greatly speeds up training." ] }, { "cell_type": "code", - "execution_count": 57, + "execution_count": 60, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1474,9 +1759,11 @@ }, { "cell_type": "code", - "execution_count": 58, + "execution_count": 61, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1495,16 +1782,21 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Build DQN" ] }, { "cell_type": "code", - "execution_count": 59, + "execution_count": 62, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1553,21 +1845,23 @@ " X_action = tf.placeholder(tf.int32, shape=[None])\n", " y = tf.placeholder(tf.float32, shape=[None, 1])\n", " q_value = tf.reduce_sum(critic_q_values * tf.one_hot(X_action, n_outputs),\n", - " reduction_indices=1, keep_dims=True)\n", + " axis=1, keep_dims=True)\n", " cost = tf.reduce_mean(tf.square(y - q_value))\n", " global_step = tf.Variable(0, trainable=False, name='global_step')\n", " optimizer = tf.train.AdamOptimizer(learning_rate)\n", " training_op = optimizer.minimize(cost, global_step=global_step)\n", " \n", - "init = tf.initialize_all_variables()\n", + "init = tf.global_variables_initializer()\n", "saver = tf.train.Saver()" ] }, { "cell_type": "code", - "execution_count": 60, + "execution_count": 63, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1576,9 +1870,11 @@ }, { "cell_type": "code", - "execution_count": 61, + "execution_count": 64, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1600,9 +1896,11 @@ }, { "cell_type": "code", - "execution_count": 62, + "execution_count": 65, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1613,8 +1911,6 @@ "\n", "def epsilon_greedy(q_values, step):\n", " epsilon = max(eps_min, eps_max - (eps_max-eps_min) * step/eps_decay_steps)\n", - " print(\" epsilon {}\".format(epsilon), end=\"\")\n", - " sys.stdout.flush()\n", " if rnd.rand() < epsilon:\n", " return rnd.randint(n_outputs) # random action\n", " else:\n", @@ -1623,9 +1919,11 @@ }, { "cell_type": "code", - "execution_count": 63, + "execution_count": 66, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1638,7 +1936,7 @@ "skip_start = 90 # Skip the start of every game (it's just waiting time).\n", "batch_size = 50\n", "iteration = 0 # game iterations\n", - "checkpoint_path = \"my_dqn.ckpt\"\n", + "checkpoint_path = \"./my_dqn.ckpt\"\n", "done = True # env needs to be reset\n", "\n", "with tf.Session() as sess:\n", @@ -1685,214 +1983,25 @@ "\n", " # And save regularly\n", " if step % save_steps == 0:\n", - " saver.save(sess, checkpoint_path)\n" + " saver.save(sess, checkpoint_path)" ] }, { "cell_type": "markdown", - "metadata": {}, - "source": [ - "## DQN for the Cart-Pole" - ] - }, - { - "cell_type": "code", - "execution_count": 64, "metadata": { - "collapsed": true + "deletable": true, + "editable": true }, - "outputs": [], - "source": [ - "eps_min = 0.1\n", - "eps_max = 1.0\n", - "eps_decay_steps = 20000\n", - "import sys\n", - "\n", - "def epsilon_greedy(q_values, step):\n", - " epsilon = max(eps_min, eps_max - (eps_max-eps_min) * step/eps_decay_steps)\n", - " print(\" epsilon {}\".format(epsilon), end=\"\")\n", - " sys.stdout.flush()\n", - " if rnd.rand() < epsilon:\n", - " return rnd.randint(n_outputs) # random action\n", - " else:\n", - " return np.argmax(q_values) # optimal action" - ] - }, - { - "cell_type": "code", - "execution_count": 65, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "import tensorflow as tf\n", - "from tensorflow.contrib.layers import fully_connected\n", - "\n", - "tf.reset_default_graph()\n", - "\n", - "n_inputs = 4\n", - "n_hidden = 4\n", - "n_outputs = 2\n", - "\n", - "learning_rate = 0.01\n", - "\n", - "initializer = tf.contrib.layers.variance_scaling_initializer()\n", - "\n", - "def q_network(X_state, scope):\n", - " with tf.variable_scope(scope) as scope:\n", - " hidden = fully_connected(X_state, n_hidden, activation_fn=tf.nn.elu, weights_initializer=initializer)\n", - " outputs = fully_connected(hidden, n_outputs, activation_fn=None)\n", - " trainable_vars = {var.name[len(scope.name):]: var for var in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=scope.name)}\n", - " return outputs, trainable_vars \n", - "\n", - "X_state = tf.placeholder(tf.float32, shape=[None, n_inputs])\n", - "actor_q_values, actor_vars = q_network(X_state, scope=\"q_networks/actor\") # acts\n", - "critic_q_values, critic_vars = q_network(X_state, scope=\"q_networks/critic\") # learns\n", - "\n", - "copy_ops = [actor_var.assign(critic_vars[var_name])\n", - " for var_name, actor_var in actor_vars.items()]\n", - "copy_critic_to_actor = tf.group(*copy_ops)\n", - "\n", - "with tf.variable_scope(\"train\"):\n", - " X_action = tf.placeholder(tf.int32, shape=[None])\n", - " y = tf.placeholder(tf.float32, shape=[None, 1])\n", - " q_value = tf.reduce_sum(critic_q_values * tf.one_hot(X_action, n_outputs),\n", - " reduction_indices=1, keep_dims=True)\n", - " cost = tf.reduce_mean(tf.square(y - q_value))\n", - " global_step = tf.Variable(0, trainable=False, name='global_step')\n", - " optimizer = tf.train.AdamOptimizer(learning_rate)\n", - " training_op = optimizer.minimize(cost, global_step=global_step)\n", - " \n", - "init = tf.initialize_all_variables()\n", - "saver = tf.train.Saver()" - ] - }, - { - "cell_type": "code", - "execution_count": 66, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "n_steps = 50000 # total number of training steps\n", - "training_start = 1000 # start training after 1,000 game iterations\n", - "training_interval = 3 # run a training step every 3 game iterations\n", - "save_steps = 50 # save the model every 50 training steps\n", - "copy_steps = 25 # copy the critic to the actor every 25 training steps\n", - "discount_rate = 0.95\n", - "batch_size = 50\n", - "iteration = 0 # game iterations\n", - "checkpoint_path = \"my_dqn.ckpt\"\n", - "done = True # env needs to be reset\n", - "\n", - "env = gym.make(\"CartPole-v0\")\n", - "\n", - "replay_memory.clear()\n", - "\n", - "with tf.Session() as sess:\n", - " if os.path.isfile(checkpoint_path):\n", - " saver.restore(sess, checkpoint_path)\n", - " else:\n", - " init.run()\n", - " while True:\n", - " step = global_step.eval()\n", - " if step >= n_steps:\n", - " break\n", - " iteration += 1\n", - " print(\"\\rIteration {}\\tTraining step {}/{} ({:.1f}%)\".format(iteration, step, n_steps, step * 100 / n_steps), end=\"\")\n", - " if done: # game over, start again\n", - " obs = env.reset()\n", - " state = obs\n", - "\n", - " # Actor evaluates what to do\n", - " q_values = actor_q_values.eval(feed_dict={X_state: [state]})\n", - " action = epsilon_greedy(q_values, step)\n", - "\n", - " # Actor plays\n", - " obs, reward, done, info = env.step(action)\n", - " next_state = obs\n", - "\n", - " # Let's memorize what happened\n", - " replay_memory.append((state, action, reward, next_state, 1.0 - done))\n", - " state = next_state\n", - "\n", - " if iteration < training_start or iteration % training_interval != 0:\n", - " continue\n", - " \n", - " # Critic learns\n", - " X_state_val, X_action_val, rewards, X_next_state_val, continues = sample_memories(batch_size)\n", - " next_q_values = actor_q_values.eval(feed_dict={X_state: X_next_state_val})\n", - " y_val = rewards + continues * discount_rate * np.max(next_q_values, axis=1, keepdims=True)\n", - " training_op.run(feed_dict={X_state: X_state_val, X_action: X_action_val, y: y_val})\n", - "\n", - " # Regularly copy critic to actor\n", - " if step % copy_steps == 0:\n", - " copy_critic_to_actor.run()\n", - "\n", - " # And save regularly\n", - " if step % save_steps == 0:\n", - " saver.save(sess, checkpoint_path)\n" - ] - }, - { - "cell_type": "code", - "execution_count": 67, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "n_max_steps = 1000\n", - "\n", - "frames = []\n", - "obs = env.reset()\n", - "with tf.Session() as sess:\n", - " saver.restore(sess, checkpoint_path)\n", - " for step in range(n_max_steps):\n", - " img = render_cart_pole(env, obs)\n", - " frames.append(img)\n", - " actor_q_values_val = actor_q_values.eval(feed_dict={X_state: obs.reshape(1, n_inputs)})\n", - " action_val = np.argmax(actor_q_values_val)\n", - " obs, reward, done, info = env.step(action_val)\n", - " if done:\n", - " break" - ] - }, - { - "cell_type": "code", - "execution_count": 68, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "len(frames)" - ] - }, - { - "cell_type": "code", - "execution_count": 69, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "video = plot_animation(frames)\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, "source": [ "# Exercise solutions" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Coming soon..." ] diff --git a/Dockerfile b/Dockerfile index c5866b6..e548a7b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,9 +9,6 @@ RUN apt-get update -y &&\ libav-tools\ libboost-all-dev\ libsdl2-dev\ - python-dev\ - python-opengl\ - python-pip\ python3-dev\ python3-opengl\ python3-pip\ @@ -25,17 +22,14 @@ RUN apt-get update -y &&\ USER main ADD requirements.txt /home/main/ -RUN /usr/bin/pip2 install --upgrade --user pip wheel RUN /usr/bin/pip3 install --upgrade --user pip wheel ENV PATH /home/main/.local/bin:$PATH # Install scientific packages -RUN pip2 install --upgrade --user -r requirements.txt RUN pip3 install --upgrade --user -r requirements.txt # Install OpenAI gym -RUN pip2 install --upgrade --user 'gym[all]' RUN pip3 install --upgrade --user 'gym[all]' # Install Jupyter extensions @@ -44,12 +38,10 @@ RUN pip3 install --user --upgrade https://github.com/ipython-contrib/jupyter_con RUN rm -rf /home/main/.cache # Jupyter extensions -#RUN conda install -c conda-forge jupyter_contrib_nbextensions RUN jupyter contrib nbextension install --user RUN jupyter nbextension enable toc2/main RUN /home/main/anaconda2/bin/jupyter kernelspec remove -f python3 -RUN /usr/bin/python2 -m ipykernel install --user RUN /usr/bin/python3 -m ipykernel install --user ADD .binder_start /home/main/ diff --git a/nets/README.md b/nets/README.md deleted file mode 100644 index 8f1ce71..0000000 --- a/nets/README.md +++ /dev/null @@ -1,5 +0,0 @@ -This directory was copied from: -https://github.com/tensorflow/models/blob/master/slim/nets - -On Sept. 25th, 2016. Commit: -https://github.com/tensorflow/models/commit/65fad62dc6daca5a72c204013824cc380939d457 diff --git a/nets/__init__.py b/nets/__init__.py deleted file mode 100644 index 8b13789..0000000 --- a/nets/__init__.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/nets/alexnet.py b/nets/alexnet.py deleted file mode 100644 index 7aca487..0000000 --- a/nets/alexnet.py +++ /dev/null @@ -1,125 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Contains a model definition for AlexNet. - -This work was first described in: - ImageNet Classification with Deep Convolutional Neural Networks - Alex Krizhevsky, Ilya Sutskever and Geoffrey E. Hinton - -and later refined in: - One weird trick for parallelizing convolutional neural networks - Alex Krizhevsky, 2014 - -Here we provide the implementation proposed in "One weird trick" and not -"ImageNet Classification", as per the paper, the LRN layers have been removed. - -Usage: - with slim.arg_scope(alexnet.alexnet_v2_arg_scope()): - outputs, end_points = alexnet.alexnet_v2(inputs) - -@@alexnet_v2 -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import tensorflow as tf - -slim = tf.contrib.slim -trunc_normal = lambda stddev: tf.truncated_normal_initializer(0.0, stddev) - - -def alexnet_v2_arg_scope(weight_decay=0.0005): - with slim.arg_scope([slim.conv2d, slim.fully_connected], - activation_fn=tf.nn.relu, - biases_initializer=tf.constant_initializer(0.1), - weights_regularizer=slim.l2_regularizer(weight_decay)): - with slim.arg_scope([slim.conv2d], padding='SAME'): - with slim.arg_scope([slim.max_pool2d], padding='VALID') as arg_sc: - return arg_sc - - -def alexnet_v2(inputs, - num_classes=1000, - is_training=True, - dropout_keep_prob=0.5, - spatial_squeeze=True, - scope='alexnet_v2'): - """AlexNet version 2. - - Described in: http://arxiv.org/pdf/1404.5997v2.pdf - Parameters from: - github.com/akrizhevsky/cuda-convnet2/blob/master/layers/ - layers-imagenet-1gpu.cfg - - Note: All the fully_connected layers have been transformed to conv2d layers. - To use in classification mode, resize input to 224x224. To use in fully - convolutional mode, set spatial_squeeze to false. - The LRN layers have been removed and change the initializers from - random_normal_initializer to xavier_initializer. - - Args: - inputs: a tensor of size [batch_size, height, width, channels]. - num_classes: number of predicted classes. - is_training: whether or not the model is being trained. - dropout_keep_prob: the probability that activations are kept in the dropout - layers during training. - spatial_squeeze: whether or not should squeeze the spatial dimensions of the - outputs. Useful to remove unnecessary dimensions for classification. - scope: Optional scope for the variables. - - Returns: - the last op containing the log predictions and end_points dict. - """ - with tf.variable_scope(scope, 'alexnet_v2', [inputs]) as sc: - end_points_collection = sc.name + '_end_points' - # Collect outputs for conv2d, fully_connected and max_pool2d. - with slim.arg_scope([slim.conv2d, slim.fully_connected, slim.max_pool2d], - outputs_collections=[end_points_collection]): - net = slim.conv2d(inputs, 64, [11, 11], 4, padding='VALID', - scope='conv1') - net = slim.max_pool2d(net, [3, 3], 2, scope='pool1') - net = slim.conv2d(net, 192, [5, 5], scope='conv2') - net = slim.max_pool2d(net, [3, 3], 2, scope='pool2') - net = slim.conv2d(net, 384, [3, 3], scope='conv3') - net = slim.conv2d(net, 384, [3, 3], scope='conv4') - net = slim.conv2d(net, 256, [3, 3], scope='conv5') - net = slim.max_pool2d(net, [3, 3], 2, scope='pool5') - - # Use conv2d instead of fully_connected layers. - with slim.arg_scope([slim.conv2d], - weights_initializer=trunc_normal(0.005), - biases_initializer=tf.constant_initializer(0.1)): - net = slim.conv2d(net, 4096, [5, 5], padding='VALID', - scope='fc6') - net = slim.dropout(net, dropout_keep_prob, is_training=is_training, - scope='dropout6') - net = slim.conv2d(net, 4096, [1, 1], scope='fc7') - net = slim.dropout(net, dropout_keep_prob, is_training=is_training, - scope='dropout7') - net = slim.conv2d(net, num_classes, [1, 1], - activation_fn=None, - normalizer_fn=None, - biases_initializer=tf.zeros_initializer, - scope='fc8') - - # Convert end_points_collection into a end_point dict. - end_points = dict(tf.get_collection(end_points_collection)) - if spatial_squeeze: - net = tf.squeeze(net, [1, 2], name='fc8/squeezed') - end_points[sc.name + '/fc8'] = net - return net, end_points -alexnet_v2.default_image_size = 224 diff --git a/nets/alexnet_test.py b/nets/alexnet_test.py deleted file mode 100644 index 493c460..0000000 --- a/nets/alexnet_test.py +++ /dev/null @@ -1,145 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for slim.nets.alexnet.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import tensorflow as tf - -from nets import alexnet - -slim = tf.contrib.slim - - -class AlexnetV2Test(tf.test.TestCase): - - def testBuild(self): - batch_size = 5 - height, width = 224, 224 - num_classes = 1000 - with self.test_session(): - inputs = tf.random_uniform((batch_size, height, width, 3)) - logits, _ = alexnet.alexnet_v2(inputs, num_classes) - self.assertEquals(logits.op.name, 'alexnet_v2/fc8/squeezed') - self.assertListEqual(logits.get_shape().as_list(), - [batch_size, num_classes]) - - def testFullyConvolutional(self): - batch_size = 1 - height, width = 300, 400 - num_classes = 1000 - with self.test_session(): - inputs = tf.random_uniform((batch_size, height, width, 3)) - logits, _ = alexnet.alexnet_v2(inputs, num_classes, spatial_squeeze=False) - self.assertEquals(logits.op.name, 'alexnet_v2/fc8/BiasAdd') - self.assertListEqual(logits.get_shape().as_list(), - [batch_size, 4, 7, num_classes]) - - def testEndPoints(self): - batch_size = 5 - height, width = 224, 224 - num_classes = 1000 - with self.test_session(): - inputs = tf.random_uniform((batch_size, height, width, 3)) - _, end_points = alexnet.alexnet_v2(inputs, num_classes) - expected_names = ['alexnet_v2/conv1', - 'alexnet_v2/pool1', - 'alexnet_v2/conv2', - 'alexnet_v2/pool2', - 'alexnet_v2/conv3', - 'alexnet_v2/conv4', - 'alexnet_v2/conv5', - 'alexnet_v2/pool5', - 'alexnet_v2/fc6', - 'alexnet_v2/fc7', - 'alexnet_v2/fc8' - ] - self.assertSetEqual(set(end_points.keys()), set(expected_names)) - - def testModelVariables(self): - batch_size = 5 - height, width = 224, 224 - num_classes = 1000 - with self.test_session(): - inputs = tf.random_uniform((batch_size, height, width, 3)) - alexnet.alexnet_v2(inputs, num_classes) - expected_names = ['alexnet_v2/conv1/weights', - 'alexnet_v2/conv1/biases', - 'alexnet_v2/conv2/weights', - 'alexnet_v2/conv2/biases', - 'alexnet_v2/conv3/weights', - 'alexnet_v2/conv3/biases', - 'alexnet_v2/conv4/weights', - 'alexnet_v2/conv4/biases', - 'alexnet_v2/conv5/weights', - 'alexnet_v2/conv5/biases', - 'alexnet_v2/fc6/weights', - 'alexnet_v2/fc6/biases', - 'alexnet_v2/fc7/weights', - 'alexnet_v2/fc7/biases', - 'alexnet_v2/fc8/weights', - 'alexnet_v2/fc8/biases', - ] - model_variables = [v.op.name for v in slim.get_model_variables()] - self.assertSetEqual(set(model_variables), set(expected_names)) - - def testEvaluation(self): - batch_size = 2 - height, width = 224, 224 - num_classes = 1000 - with self.test_session(): - eval_inputs = tf.random_uniform((batch_size, height, width, 3)) - logits, _ = alexnet.alexnet_v2(eval_inputs, is_training=False) - self.assertListEqual(logits.get_shape().as_list(), - [batch_size, num_classes]) - predictions = tf.argmax(logits, 1) - self.assertListEqual(predictions.get_shape().as_list(), [batch_size]) - - def testTrainEvalWithReuse(self): - train_batch_size = 2 - eval_batch_size = 1 - train_height, train_width = 224, 224 - eval_height, eval_width = 300, 400 - num_classes = 1000 - with self.test_session(): - train_inputs = tf.random_uniform( - (train_batch_size, train_height, train_width, 3)) - logits, _ = alexnet.alexnet_v2(train_inputs) - self.assertListEqual(logits.get_shape().as_list(), - [train_batch_size, num_classes]) - tf.get_variable_scope().reuse_variables() - eval_inputs = tf.random_uniform( - (eval_batch_size, eval_height, eval_width, 3)) - logits, _ = alexnet.alexnet_v2(eval_inputs, is_training=False, - spatial_squeeze=False) - self.assertListEqual(logits.get_shape().as_list(), - [eval_batch_size, 4, 7, num_classes]) - logits = tf.reduce_mean(logits, [1, 2]) - predictions = tf.argmax(logits, 1) - self.assertEquals(predictions.get_shape().as_list(), [eval_batch_size]) - - def testForward(self): - batch_size = 1 - height, width = 224, 224 - with self.test_session() as sess: - inputs = tf.random_uniform((batch_size, height, width, 3)) - logits, _ = alexnet.alexnet_v2(inputs) - sess.run(tf.initialize_all_variables()) - output = sess.run(logits) - self.assertTrue(output.any()) - -if __name__ == '__main__': - tf.test.main() diff --git a/nets/cifarnet.py b/nets/cifarnet.py deleted file mode 100644 index 371a9cb..0000000 --- a/nets/cifarnet.py +++ /dev/null @@ -1,112 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Contains a variant of the CIFAR-10 model definition.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import tensorflow as tf - -slim = tf.contrib.slim - -trunc_normal = lambda stddev: tf.truncated_normal_initializer(stddev=stddev) - - -def cifarnet(images, num_classes=10, is_training=False, - dropout_keep_prob=0.5, - prediction_fn=slim.softmax, - scope='CifarNet'): - """Creates a variant of the CifarNet model. - - Note that since the output is a set of 'logits', the values fall in the - interval of (-infinity, infinity). Consequently, to convert the outputs to a - probability distribution over the characters, one will need to convert them - using the softmax function: - - logits = cifarnet.cifarnet(images, is_training=False) - probabilities = tf.nn.softmax(logits) - predictions = tf.argmax(logits, 1) - - Args: - images: A batch of `Tensors` of size [batch_size, height, width, channels]. - num_classes: the number of classes in the dataset. - is_training: specifies whether or not we're currently training the model. - This variable will determine the behaviour of the dropout layer. - dropout_keep_prob: the percentage of activation values that are retained. - prediction_fn: a function to get predictions out of logits. - scope: Optional variable_scope. - - Returns: - logits: the pre-softmax activations, a tensor of size - [batch_size, `num_classes`] - end_points: a dictionary from components of the network to the corresponding - activation. - """ - end_points = {} - - with tf.variable_scope(scope, 'CifarNet', [images, num_classes]): - net = slim.conv2d(images, 64, [5, 5], scope='conv1') - end_points['conv1'] = net - net = slim.max_pool2d(net, [2, 2], 2, scope='pool1') - end_points['pool1'] = net - net = tf.nn.lrn(net, 4, bias=1.0, alpha=0.001/9.0, beta=0.75, name='norm1') - net = slim.conv2d(net, 64, [5, 5], scope='conv2') - end_points['conv2'] = net - net = tf.nn.lrn(net, 4, bias=1.0, alpha=0.001/9.0, beta=0.75, name='norm2') - net = slim.max_pool2d(net, [2, 2], 2, scope='pool2') - end_points['pool2'] = net - net = slim.flatten(net) - end_points['Flatten'] = net - net = slim.fully_connected(net, 384, scope='fc3') - end_points['fc3'] = net - net = slim.dropout(net, dropout_keep_prob, is_training=is_training, - scope='dropout3') - net = slim.fully_connected(net, 192, scope='fc4') - end_points['fc4'] = net - logits = slim.fully_connected(net, num_classes, - biases_initializer=tf.zeros_initializer, - weights_initializer=trunc_normal(1/192.0), - weights_regularizer=None, - activation_fn=None, - scope='logits') - - end_points['Logits'] = logits - end_points['Predictions'] = prediction_fn(logits, scope='Predictions') - - return logits, end_points -cifarnet.default_image_size = 32 - - -def cifarnet_arg_scope(weight_decay=0.004): - """Defines the default cifarnet argument scope. - - Args: - weight_decay: The weight decay to use for regularizing the model. - - Returns: - An `arg_scope` to use for the inception v3 model. - """ - with slim.arg_scope( - [slim.conv2d], - weights_initializer=tf.truncated_normal_initializer(stddev=5e-2), - activation_fn=tf.nn.relu): - with slim.arg_scope( - [slim.fully_connected], - biases_initializer=tf.constant_initializer(0.1), - weights_initializer=trunc_normal(0.04), - weights_regularizer=slim.l2_regularizer(weight_decay), - activation_fn=tf.nn.relu) as sc: - return sc diff --git a/nets/inception.py b/nets/inception.py deleted file mode 100644 index 0479e3d..0000000 --- a/nets/inception.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Brings inception_v1, inception_v2 and inception_v3 under one namespace.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import -from nets.inception_resnet_v2 import inception_resnet_v2 -from nets.inception_resnet_v2 import inception_resnet_v2_arg_scope -from nets.inception_v1 import inception_v1 -from nets.inception_v1 import inception_v1_arg_scope -from nets.inception_v1 import inception_v1_base -from nets.inception_v2 import inception_v2 -from nets.inception_v2 import inception_v2_arg_scope -from nets.inception_v2 import inception_v2_base -from nets.inception_v3 import inception_v3 -from nets.inception_v3 import inception_v3_arg_scope -from nets.inception_v3 import inception_v3_base -# pylint: enable=unused-import diff --git a/nets/inception_resnet_v2.py b/nets/inception_resnet_v2.py deleted file mode 100644 index 4b3c5bd..0000000 --- a/nets/inception_resnet_v2.py +++ /dev/null @@ -1,280 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Contains the definition of the Inception Resnet V2 architecture. - -As described in http://arxiv.org/abs/1602.07261. - - Inception-v4, Inception-ResNet and the Impact of Residual Connections - on Learning - Christian Szegedy, Sergey Ioffe, Vincent Vanhoucke, Alex Alemi -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - - -import tensorflow as tf - -slim = tf.contrib.slim - - -def block35(net, scale=1.0, activation_fn=tf.nn.relu, scope=None, reuse=None): - """Builds the 35x35 resnet block.""" - with tf.variable_scope(scope, 'Block35', [net], reuse=reuse): - with tf.variable_scope('Branch_0'): - tower_conv = slim.conv2d(net, 32, 1, scope='Conv2d_1x1') - with tf.variable_scope('Branch_1'): - tower_conv1_0 = slim.conv2d(net, 32, 1, scope='Conv2d_0a_1x1') - tower_conv1_1 = slim.conv2d(tower_conv1_0, 32, 3, scope='Conv2d_0b_3x3') - with tf.variable_scope('Branch_2'): - tower_conv2_0 = slim.conv2d(net, 32, 1, scope='Conv2d_0a_1x1') - tower_conv2_1 = slim.conv2d(tower_conv2_0, 48, 3, scope='Conv2d_0b_3x3') - tower_conv2_2 = slim.conv2d(tower_conv2_1, 64, 3, scope='Conv2d_0c_3x3') - mixed = tf.concat(3, [tower_conv, tower_conv1_1, tower_conv2_2]) - up = slim.conv2d(mixed, net.get_shape()[3], 1, normalizer_fn=None, - activation_fn=None, scope='Conv2d_1x1') - net += scale * up - if activation_fn: - net = activation_fn(net) - return net - - -def block17(net, scale=1.0, activation_fn=tf.nn.relu, scope=None, reuse=None): - """Builds the 17x17 resnet block.""" - with tf.variable_scope(scope, 'Block17', [net], reuse=reuse): - with tf.variable_scope('Branch_0'): - tower_conv = slim.conv2d(net, 192, 1, scope='Conv2d_1x1') - with tf.variable_scope('Branch_1'): - tower_conv1_0 = slim.conv2d(net, 128, 1, scope='Conv2d_0a_1x1') - tower_conv1_1 = slim.conv2d(tower_conv1_0, 160, [1, 7], - scope='Conv2d_0b_1x7') - tower_conv1_2 = slim.conv2d(tower_conv1_1, 192, [7, 1], - scope='Conv2d_0c_7x1') - mixed = tf.concat(3, [tower_conv, tower_conv1_2]) - up = slim.conv2d(mixed, net.get_shape()[3], 1, normalizer_fn=None, - activation_fn=None, scope='Conv2d_1x1') - net += scale * up - if activation_fn: - net = activation_fn(net) - return net - - -def block8(net, scale=1.0, activation_fn=tf.nn.relu, scope=None, reuse=None): - """Builds the 8x8 resnet block.""" - with tf.variable_scope(scope, 'Block8', [net], reuse=reuse): - with tf.variable_scope('Branch_0'): - tower_conv = slim.conv2d(net, 192, 1, scope='Conv2d_1x1') - with tf.variable_scope('Branch_1'): - tower_conv1_0 = slim.conv2d(net, 192, 1, scope='Conv2d_0a_1x1') - tower_conv1_1 = slim.conv2d(tower_conv1_0, 224, [1, 3], - scope='Conv2d_0b_1x3') - tower_conv1_2 = slim.conv2d(tower_conv1_1, 256, [3, 1], - scope='Conv2d_0c_3x1') - mixed = tf.concat(3, [tower_conv, tower_conv1_2]) - up = slim.conv2d(mixed, net.get_shape()[3], 1, normalizer_fn=None, - activation_fn=None, scope='Conv2d_1x1') - net += scale * up - if activation_fn: - net = activation_fn(net) - return net - - -def inception_resnet_v2(inputs, num_classes=1001, is_training=True, - dropout_keep_prob=0.8, - reuse=None, - scope='InceptionResnetV2'): - """Creates the Inception Resnet V2 model. - - Args: - inputs: a 4-D tensor of size [batch_size, height, width, 3]. - num_classes: number of predicted classes. - is_training: whether is training or not. - dropout_keep_prob: float, the fraction to keep before final layer. - reuse: whether or not the network and its variables should be reused. To be - able to reuse 'scope' must be given. - scope: Optional variable_scope. - - Returns: - logits: the logits outputs of the model. - end_points: the set of end_points from the inception model. - """ - end_points = {} - - with tf.variable_scope(scope, 'InceptionResnetV2', [inputs], reuse=reuse): - with slim.arg_scope([slim.batch_norm, slim.dropout], - is_training=is_training): - with slim.arg_scope([slim.conv2d, slim.max_pool2d, slim.avg_pool2d], - stride=1, padding='SAME'): - - # 149 x 149 x 32 - net = slim.conv2d(inputs, 32, 3, stride=2, padding='VALID', - scope='Conv2d_1a_3x3') - end_points['Conv2d_1a_3x3'] = net - # 147 x 147 x 32 - net = slim.conv2d(net, 32, 3, padding='VALID', - scope='Conv2d_2a_3x3') - end_points['Conv2d_2a_3x3'] = net - # 147 x 147 x 64 - net = slim.conv2d(net, 64, 3, scope='Conv2d_2b_3x3') - end_points['Conv2d_2b_3x3'] = net - # 73 x 73 x 64 - net = slim.max_pool2d(net, 3, stride=2, padding='VALID', - scope='MaxPool_3a_3x3') - end_points['MaxPool_3a_3x3'] = net - # 73 x 73 x 80 - net = slim.conv2d(net, 80, 1, padding='VALID', - scope='Conv2d_3b_1x1') - end_points['Conv2d_3b_1x1'] = net - # 71 x 71 x 192 - net = slim.conv2d(net, 192, 3, padding='VALID', - scope='Conv2d_4a_3x3') - end_points['Conv2d_4a_3x3'] = net - # 35 x 35 x 192 - net = slim.max_pool2d(net, 3, stride=2, padding='VALID', - scope='MaxPool_5a_3x3') - end_points['MaxPool_5a_3x3'] = net - - # 35 x 35 x 320 - with tf.variable_scope('Mixed_5b'): - with tf.variable_scope('Branch_0'): - tower_conv = slim.conv2d(net, 96, 1, scope='Conv2d_1x1') - with tf.variable_scope('Branch_1'): - tower_conv1_0 = slim.conv2d(net, 48, 1, scope='Conv2d_0a_1x1') - tower_conv1_1 = slim.conv2d(tower_conv1_0, 64, 5, - scope='Conv2d_0b_5x5') - with tf.variable_scope('Branch_2'): - tower_conv2_0 = slim.conv2d(net, 64, 1, scope='Conv2d_0a_1x1') - tower_conv2_1 = slim.conv2d(tower_conv2_0, 96, 3, - scope='Conv2d_0b_3x3') - tower_conv2_2 = slim.conv2d(tower_conv2_1, 96, 3, - scope='Conv2d_0c_3x3') - with tf.variable_scope('Branch_3'): - tower_pool = slim.avg_pool2d(net, 3, stride=1, padding='SAME', - scope='AvgPool_0a_3x3') - tower_pool_1 = slim.conv2d(tower_pool, 64, 1, - scope='Conv2d_0b_1x1') - net = tf.concat(3, [tower_conv, tower_conv1_1, - tower_conv2_2, tower_pool_1]) - - end_points['Mixed_5b'] = net - net = slim.repeat(net, 10, block35, scale=0.17) - - # 17 x 17 x 1024 - with tf.variable_scope('Mixed_6a'): - with tf.variable_scope('Branch_0'): - tower_conv = slim.conv2d(net, 384, 3, stride=2, padding='VALID', - scope='Conv2d_1a_3x3') - with tf.variable_scope('Branch_1'): - tower_conv1_0 = slim.conv2d(net, 256, 1, scope='Conv2d_0a_1x1') - tower_conv1_1 = slim.conv2d(tower_conv1_0, 256, 3, - scope='Conv2d_0b_3x3') - tower_conv1_2 = slim.conv2d(tower_conv1_1, 384, 3, - stride=2, padding='VALID', - scope='Conv2d_1a_3x3') - with tf.variable_scope('Branch_2'): - tower_pool = slim.max_pool2d(net, 3, stride=2, padding='VALID', - scope='MaxPool_1a_3x3') - net = tf.concat(3, [tower_conv, tower_conv1_2, tower_pool]) - - end_points['Mixed_6a'] = net - net = slim.repeat(net, 20, block17, scale=0.10) - - # Auxillary tower - with tf.variable_scope('AuxLogits'): - aux = slim.avg_pool2d(net, 5, stride=3, padding='VALID', - scope='Conv2d_1a_3x3') - aux = slim.conv2d(aux, 128, 1, scope='Conv2d_1b_1x1') - aux = slim.conv2d(aux, 768, aux.get_shape()[1:3], - padding='VALID', scope='Conv2d_2a_5x5') - aux = slim.flatten(aux) - aux = slim.fully_connected(aux, num_classes, activation_fn=None, - scope='Logits') - end_points['AuxLogits'] = aux - - with tf.variable_scope('Mixed_7a'): - with tf.variable_scope('Branch_0'): - tower_conv = slim.conv2d(net, 256, 1, scope='Conv2d_0a_1x1') - tower_conv_1 = slim.conv2d(tower_conv, 384, 3, stride=2, - padding='VALID', scope='Conv2d_1a_3x3') - with tf.variable_scope('Branch_1'): - tower_conv1 = slim.conv2d(net, 256, 1, scope='Conv2d_0a_1x1') - tower_conv1_1 = slim.conv2d(tower_conv1, 288, 3, stride=2, - padding='VALID', scope='Conv2d_1a_3x3') - with tf.variable_scope('Branch_2'): - tower_conv2 = slim.conv2d(net, 256, 1, scope='Conv2d_0a_1x1') - tower_conv2_1 = slim.conv2d(tower_conv2, 288, 3, - scope='Conv2d_0b_3x3') - tower_conv2_2 = slim.conv2d(tower_conv2_1, 320, 3, stride=2, - padding='VALID', scope='Conv2d_1a_3x3') - with tf.variable_scope('Branch_3'): - tower_pool = slim.max_pool2d(net, 3, stride=2, padding='VALID', - scope='MaxPool_1a_3x3') - net = tf.concat(3, [tower_conv_1, tower_conv1_1, - tower_conv2_2, tower_pool]) - - end_points['Mixed_7a'] = net - - net = slim.repeat(net, 9, block8, scale=0.20) - net = block8(net, activation_fn=None) - - net = slim.conv2d(net, 1536, 1, scope='Conv2d_7b_1x1') - end_points['Conv2d_7b_1x1'] = net - - with tf.variable_scope('Logits'): - end_points['PrePool'] = net - net = slim.avg_pool2d(net, net.get_shape()[1:3], padding='VALID', - scope='AvgPool_1a_8x8') - net = slim.flatten(net) - - net = slim.dropout(net, dropout_keep_prob, is_training=is_training, - scope='Dropout') - - end_points['PreLogitsFlatten'] = net - logits = slim.fully_connected(net, num_classes, activation_fn=None, - scope='Logits') - end_points['Logits'] = logits - end_points['Predictions'] = tf.nn.softmax(logits, name='Predictions') - - return logits, end_points -inception_resnet_v2.default_image_size = 299 - - -def inception_resnet_v2_arg_scope(weight_decay=0.00004, - batch_norm_decay=0.9997, - batch_norm_epsilon=0.001): - """Yields the scope with the default parameters for inception_resnet_v2. - - Args: - weight_decay: the weight decay for weights variables. - batch_norm_decay: decay for the moving average of batch_norm momentums. - batch_norm_epsilon: small float added to variance to avoid dividing by zero. - - Returns: - a arg_scope with the parameters needed for inception_resnet_v2. - """ - # Set weight_decay for weights in conv2d and fully_connected layers. - with slim.arg_scope([slim.conv2d, slim.fully_connected], - weights_regularizer=slim.l2_regularizer(weight_decay), - biases_regularizer=slim.l2_regularizer(weight_decay)): - - batch_norm_params = { - 'decay': batch_norm_decay, - 'epsilon': batch_norm_epsilon, - } - # Set activation_fn and parameters for batch_norm. - with slim.arg_scope([slim.conv2d], activation_fn=tf.nn.relu, - normalizer_fn=slim.batch_norm, - normalizer_params=batch_norm_params) as scope: - return scope diff --git a/nets/inception_resnet_v2_test.py b/nets/inception_resnet_v2_test.py deleted file mode 100644 index b74756a..0000000 --- a/nets/inception_resnet_v2_test.py +++ /dev/null @@ -1,136 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for slim.inception_resnet_v2.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import tensorflow as tf - -from nets import inception - - -class InceptionTest(tf.test.TestCase): - - def testBuildLogits(self): - batch_size = 5 - height, width = 299, 299 - num_classes = 1000 - with self.test_session(): - inputs = tf.random_uniform((batch_size, height, width, 3)) - logits, _ = inception.inception_resnet_v2(inputs, num_classes) - self.assertTrue(logits.op.name.startswith('InceptionResnetV2/Logits')) - self.assertListEqual(logits.get_shape().as_list(), - [batch_size, num_classes]) - - def testBuildEndPoints(self): - batch_size = 5 - height, width = 299, 299 - num_classes = 1000 - with self.test_session(): - inputs = tf.random_uniform((batch_size, height, width, 3)) - _, end_points = inception.inception_resnet_v2(inputs, num_classes) - self.assertTrue('Logits' in end_points) - logits = end_points['Logits'] - self.assertListEqual(logits.get_shape().as_list(), - [batch_size, num_classes]) - self.assertTrue('AuxLogits' in end_points) - aux_logits = end_points['AuxLogits'] - self.assertListEqual(aux_logits.get_shape().as_list(), - [batch_size, num_classes]) - pre_pool = end_points['PrePool'] - self.assertListEqual(pre_pool.get_shape().as_list(), - [batch_size, 8, 8, 1536]) - - def testVariablesSetDevice(self): - batch_size = 5 - height, width = 299, 299 - num_classes = 1000 - with self.test_session(): - inputs = tf.random_uniform((batch_size, height, width, 3)) - # Force all Variables to reside on the device. - with tf.variable_scope('on_cpu'), tf.device('/cpu:0'): - inception.inception_resnet_v2(inputs, num_classes) - with tf.variable_scope('on_gpu'), tf.device('/gpu:0'): - inception.inception_resnet_v2(inputs, num_classes) - for v in tf.get_collection(tf.GraphKeys.VARIABLES, scope='on_cpu'): - self.assertDeviceEqual(v.device, '/cpu:0') - for v in tf.get_collection(tf.GraphKeys.VARIABLES, scope='on_gpu'): - self.assertDeviceEqual(v.device, '/gpu:0') - - def testHalfSizeImages(self): - batch_size = 5 - height, width = 150, 150 - num_classes = 1000 - with self.test_session(): - inputs = tf.random_uniform((batch_size, height, width, 3)) - logits, end_points = inception.inception_resnet_v2(inputs, num_classes) - self.assertTrue(logits.op.name.startswith('InceptionResnetV2/Logits')) - self.assertListEqual(logits.get_shape().as_list(), - [batch_size, num_classes]) - pre_pool = end_points['PrePool'] - self.assertListEqual(pre_pool.get_shape().as_list(), - [batch_size, 3, 3, 1536]) - - def testUnknownBatchSize(self): - batch_size = 1 - height, width = 299, 299 - num_classes = 1000 - with self.test_session() as sess: - inputs = tf.placeholder(tf.float32, (None, height, width, 3)) - logits, _ = inception.inception_resnet_v2(inputs, num_classes) - self.assertTrue(logits.op.name.startswith('InceptionResnetV2/Logits')) - self.assertListEqual(logits.get_shape().as_list(), - [None, num_classes]) - images = tf.random_uniform((batch_size, height, width, 3)) - sess.run(tf.initialize_all_variables()) - output = sess.run(logits, {inputs: images.eval()}) - self.assertEquals(output.shape, (batch_size, num_classes)) - - def testEvaluation(self): - batch_size = 2 - height, width = 299, 299 - num_classes = 1000 - with self.test_session() as sess: - eval_inputs = tf.random_uniform((batch_size, height, width, 3)) - logits, _ = inception.inception_resnet_v2(eval_inputs, - num_classes, - is_training=False) - predictions = tf.argmax(logits, 1) - sess.run(tf.initialize_all_variables()) - output = sess.run(predictions) - self.assertEquals(output.shape, (batch_size,)) - - def testTrainEvalWithReuse(self): - train_batch_size = 5 - eval_batch_size = 2 - height, width = 150, 150 - num_classes = 1000 - with self.test_session() as sess: - train_inputs = tf.random_uniform((train_batch_size, height, width, 3)) - inception.inception_resnet_v2(train_inputs, num_classes) - eval_inputs = tf.random_uniform((eval_batch_size, height, width, 3)) - logits, _ = inception.inception_resnet_v2(eval_inputs, - num_classes, - is_training=False, - reuse=True) - predictions = tf.argmax(logits, 1) - sess.run(tf.initialize_all_variables()) - output = sess.run(predictions) - self.assertEquals(output.shape, (eval_batch_size,)) - - -if __name__ == '__main__': - tf.test.main() diff --git a/nets/inception_v1.py b/nets/inception_v1.py deleted file mode 100644 index 400eff6..0000000 --- a/nets/inception_v1.py +++ /dev/null @@ -1,340 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Contains the definition for inception v1 classification network.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import tensorflow as tf - -slim = tf.contrib.slim -trunc_normal = lambda stddev: tf.truncated_normal_initializer(0.0, stddev) - - -def inception_v1_base(inputs, - final_endpoint='Mixed_5c', - scope='InceptionV1'): - """Defines the Inception V1 base architecture. - - This architecture is defined in: - Going deeper with convolutions - Christian Szegedy, Wei Liu, Yangqing Jia, Pierre Sermanet, Scott Reed, - Dragomir Anguelov, Dumitru Erhan, Vincent Vanhoucke, Andrew Rabinovich. - http://arxiv.org/pdf/1409.4842v1.pdf. - - Args: - inputs: a tensor of size [batch_size, height, width, channels]. - final_endpoint: specifies the endpoint to construct the network up to. It - can be one of ['Conv2d_1a_7x7', 'MaxPool_2a_3x3', 'Conv2d_2b_1x1', - 'Conv2d_2c_3x3', 'MaxPool_3a_3x3', 'Mixed_3b', 'Mixed_3c', - 'MaxPool_4a_3x3', 'Mixed_4b', 'Mixed_4c', 'Mixed_4d', 'Mixed_4e', - 'Mixed_4f', 'MaxPool_5a_2x2', 'Mixed_5b', 'Mixed_5c'] - scope: Optional variable_scope. - - Returns: - A dictionary from components of the network to the corresponding activation. - - Raises: - ValueError: if final_endpoint is not set to one of the predefined values. - """ - end_points = {} - with tf.variable_scope(scope, 'InceptionV1', [inputs]): - with slim.arg_scope( - [slim.conv2d, slim.fully_connected], - weights_initializer=trunc_normal(0.01)): - with slim.arg_scope([slim.conv2d, slim.max_pool2d], - stride=1, padding='SAME'): - end_point = 'Conv2d_1a_7x7' - net = slim.conv2d(inputs, 64, [7, 7], stride=2, scope=end_point) - end_points[end_point] = net - if final_endpoint == end_point: return net, end_points - end_point = 'MaxPool_2a_3x3' - net = slim.max_pool2d(net, [3, 3], stride=2, scope=end_point) - end_points[end_point] = net - if final_endpoint == end_point: return net, end_points - end_point = 'Conv2d_2b_1x1' - net = slim.conv2d(net, 64, [1, 1], scope=end_point) - end_points[end_point] = net - if final_endpoint == end_point: return net, end_points - end_point = 'Conv2d_2c_3x3' - net = slim.conv2d(net, 192, [3, 3], scope=end_point) - end_points[end_point] = net - if final_endpoint == end_point: return net, end_points - end_point = 'MaxPool_3a_3x3' - net = slim.max_pool2d(net, [3, 3], stride=2, scope=end_point) - end_points[end_point] = net - if final_endpoint == end_point: return net, end_points - - end_point = 'Mixed_3b' - with tf.variable_scope(end_point): - with tf.variable_scope('Branch_0'): - branch_0 = slim.conv2d(net, 64, [1, 1], scope='Conv2d_0a_1x1') - with tf.variable_scope('Branch_1'): - branch_1 = slim.conv2d(net, 96, [1, 1], scope='Conv2d_0a_1x1') - branch_1 = slim.conv2d(branch_1, 128, [3, 3], scope='Conv2d_0b_3x3') - with tf.variable_scope('Branch_2'): - branch_2 = slim.conv2d(net, 16, [1, 1], scope='Conv2d_0a_1x1') - branch_2 = slim.conv2d(branch_2, 32, [3, 3], scope='Conv2d_0b_3x3') - with tf.variable_scope('Branch_3'): - branch_3 = slim.max_pool2d(net, [3, 3], scope='MaxPool_0a_3x3') - branch_3 = slim.conv2d(branch_3, 32, [1, 1], scope='Conv2d_0b_1x1') - net = tf.concat(3, [branch_0, branch_1, branch_2, branch_3]) - end_points[end_point] = net - if final_endpoint == end_point: return net, end_points - - end_point = 'Mixed_3c' - with tf.variable_scope(end_point): - with tf.variable_scope('Branch_0'): - branch_0 = slim.conv2d(net, 128, [1, 1], scope='Conv2d_0a_1x1') - with tf.variable_scope('Branch_1'): - branch_1 = slim.conv2d(net, 128, [1, 1], scope='Conv2d_0a_1x1') - branch_1 = slim.conv2d(branch_1, 192, [3, 3], scope='Conv2d_0b_3x3') - with tf.variable_scope('Branch_2'): - branch_2 = slim.conv2d(net, 32, [1, 1], scope='Conv2d_0a_1x1') - branch_2 = slim.conv2d(branch_2, 96, [3, 3], scope='Conv2d_0b_3x3') - with tf.variable_scope('Branch_3'): - branch_3 = slim.max_pool2d(net, [3, 3], scope='MaxPool_0a_3x3') - branch_3 = slim.conv2d(branch_3, 64, [1, 1], scope='Conv2d_0b_1x1') - net = tf.concat(3, [branch_0, branch_1, branch_2, branch_3]) - end_points[end_point] = net - if final_endpoint == end_point: return net, end_points - - end_point = 'MaxPool_4a_3x3' - net = slim.max_pool2d(net, [3, 3], stride=2, scope=end_point) - end_points[end_point] = net - if final_endpoint == end_point: return net, end_points - - end_point = 'Mixed_4b' - with tf.variable_scope(end_point): - with tf.variable_scope('Branch_0'): - branch_0 = slim.conv2d(net, 192, [1, 1], scope='Conv2d_0a_1x1') - with tf.variable_scope('Branch_1'): - branch_1 = slim.conv2d(net, 96, [1, 1], scope='Conv2d_0a_1x1') - branch_1 = slim.conv2d(branch_1, 208, [3, 3], scope='Conv2d_0b_3x3') - with tf.variable_scope('Branch_2'): - branch_2 = slim.conv2d(net, 16, [1, 1], scope='Conv2d_0a_1x1') - branch_2 = slim.conv2d(branch_2, 48, [3, 3], scope='Conv2d_0b_3x3') - with tf.variable_scope('Branch_3'): - branch_3 = slim.max_pool2d(net, [3, 3], scope='MaxPool_0a_3x3') - branch_3 = slim.conv2d(branch_3, 64, [1, 1], scope='Conv2d_0b_1x1') - net = tf.concat(3, [branch_0, branch_1, branch_2, branch_3]) - end_points[end_point] = net - if final_endpoint == end_point: return net, end_points - - end_point = 'Mixed_4c' - with tf.variable_scope(end_point): - with tf.variable_scope('Branch_0'): - branch_0 = slim.conv2d(net, 160, [1, 1], scope='Conv2d_0a_1x1') - with tf.variable_scope('Branch_1'): - branch_1 = slim.conv2d(net, 112, [1, 1], scope='Conv2d_0a_1x1') - branch_1 = slim.conv2d(branch_1, 224, [3, 3], scope='Conv2d_0b_3x3') - with tf.variable_scope('Branch_2'): - branch_2 = slim.conv2d(net, 24, [1, 1], scope='Conv2d_0a_1x1') - branch_2 = slim.conv2d(branch_2, 64, [3, 3], scope='Conv2d_0b_3x3') - with tf.variable_scope('Branch_3'): - branch_3 = slim.max_pool2d(net, [3, 3], scope='MaxPool_0a_3x3') - branch_3 = slim.conv2d(branch_3, 64, [1, 1], scope='Conv2d_0b_1x1') - net = tf.concat(3, [branch_0, branch_1, branch_2, branch_3]) - end_points[end_point] = net - if final_endpoint == end_point: return net, end_points - - end_point = 'Mixed_4d' - with tf.variable_scope(end_point): - with tf.variable_scope('Branch_0'): - branch_0 = slim.conv2d(net, 128, [1, 1], scope='Conv2d_0a_1x1') - with tf.variable_scope('Branch_1'): - branch_1 = slim.conv2d(net, 128, [1, 1], scope='Conv2d_0a_1x1') - branch_1 = slim.conv2d(branch_1, 256, [3, 3], scope='Conv2d_0b_3x3') - with tf.variable_scope('Branch_2'): - branch_2 = slim.conv2d(net, 24, [1, 1], scope='Conv2d_0a_1x1') - branch_2 = slim.conv2d(branch_2, 64, [3, 3], scope='Conv2d_0b_3x3') - with tf.variable_scope('Branch_3'): - branch_3 = slim.max_pool2d(net, [3, 3], scope='MaxPool_0a_3x3') - branch_3 = slim.conv2d(branch_3, 64, [1, 1], scope='Conv2d_0b_1x1') - net = tf.concat(3, [branch_0, branch_1, branch_2, branch_3]) - end_points[end_point] = net - if final_endpoint == end_point: return net, end_points - - end_point = 'Mixed_4e' - with tf.variable_scope(end_point): - with tf.variable_scope('Branch_0'): - branch_0 = slim.conv2d(net, 112, [1, 1], scope='Conv2d_0a_1x1') - with tf.variable_scope('Branch_1'): - branch_1 = slim.conv2d(net, 144, [1, 1], scope='Conv2d_0a_1x1') - branch_1 = slim.conv2d(branch_1, 288, [3, 3], scope='Conv2d_0b_3x3') - with tf.variable_scope('Branch_2'): - branch_2 = slim.conv2d(net, 32, [1, 1], scope='Conv2d_0a_1x1') - branch_2 = slim.conv2d(branch_2, 64, [3, 3], scope='Conv2d_0b_3x3') - with tf.variable_scope('Branch_3'): - branch_3 = slim.max_pool2d(net, [3, 3], scope='MaxPool_0a_3x3') - branch_3 = slim.conv2d(branch_3, 64, [1, 1], scope='Conv2d_0b_1x1') - net = tf.concat(3, [branch_0, branch_1, branch_2, branch_3]) - end_points[end_point] = net - if final_endpoint == end_point: return net, end_points - - end_point = 'Mixed_4f' - with tf.variable_scope(end_point): - with tf.variable_scope('Branch_0'): - branch_0 = slim.conv2d(net, 256, [1, 1], scope='Conv2d_0a_1x1') - with tf.variable_scope('Branch_1'): - branch_1 = slim.conv2d(net, 160, [1, 1], scope='Conv2d_0a_1x1') - branch_1 = slim.conv2d(branch_1, 320, [3, 3], scope='Conv2d_0b_3x3') - with tf.variable_scope('Branch_2'): - branch_2 = slim.conv2d(net, 32, [1, 1], scope='Conv2d_0a_1x1') - branch_2 = slim.conv2d(branch_2, 128, [3, 3], scope='Conv2d_0b_3x3') - with tf.variable_scope('Branch_3'): - branch_3 = slim.max_pool2d(net, [3, 3], scope='MaxPool_0a_3x3') - branch_3 = slim.conv2d(branch_3, 128, [1, 1], scope='Conv2d_0b_1x1') - net = tf.concat(3, [branch_0, branch_1, branch_2, branch_3]) - end_points[end_point] = net - if final_endpoint == end_point: return net, end_points - - end_point = 'MaxPool_5a_2x2' - net = slim.max_pool2d(net, [2, 2], stride=2, scope=end_point) - end_points[end_point] = net - if final_endpoint == end_point: return net, end_points - - end_point = 'Mixed_5b' - with tf.variable_scope(end_point): - with tf.variable_scope('Branch_0'): - branch_0 = slim.conv2d(net, 256, [1, 1], scope='Conv2d_0a_1x1') - with tf.variable_scope('Branch_1'): - branch_1 = slim.conv2d(net, 160, [1, 1], scope='Conv2d_0a_1x1') - branch_1 = slim.conv2d(branch_1, 320, [3, 3], scope='Conv2d_0b_3x3') - with tf.variable_scope('Branch_2'): - branch_2 = slim.conv2d(net, 32, [1, 1], scope='Conv2d_0a_1x1') - branch_2 = slim.conv2d(branch_2, 128, [3, 3], scope='Conv2d_0a_3x3') - with tf.variable_scope('Branch_3'): - branch_3 = slim.max_pool2d(net, [3, 3], scope='MaxPool_0a_3x3') - branch_3 = slim.conv2d(branch_3, 128, [1, 1], scope='Conv2d_0b_1x1') - net = tf.concat(3, [branch_0, branch_1, branch_2, branch_3]) - end_points[end_point] = net - if final_endpoint == end_point: return net, end_points - - end_point = 'Mixed_5c' - with tf.variable_scope(end_point): - with tf.variable_scope('Branch_0'): - branch_0 = slim.conv2d(net, 384, [1, 1], scope='Conv2d_0a_1x1') - with tf.variable_scope('Branch_1'): - branch_1 = slim.conv2d(net, 192, [1, 1], scope='Conv2d_0a_1x1') - branch_1 = slim.conv2d(branch_1, 384, [3, 3], scope='Conv2d_0b_3x3') - with tf.variable_scope('Branch_2'): - branch_2 = slim.conv2d(net, 48, [1, 1], scope='Conv2d_0a_1x1') - branch_2 = slim.conv2d(branch_2, 128, [3, 3], scope='Conv2d_0b_3x3') - with tf.variable_scope('Branch_3'): - branch_3 = slim.max_pool2d(net, [3, 3], scope='MaxPool_0a_3x3') - branch_3 = slim.conv2d(branch_3, 128, [1, 1], scope='Conv2d_0b_1x1') - net = tf.concat(3, [branch_0, branch_1, branch_2, branch_3]) - end_points[end_point] = net - if final_endpoint == end_point: return net, end_points - raise ValueError('Unknown final endpoint %s' % final_endpoint) - - -def inception_v1(inputs, - num_classes=1000, - is_training=True, - dropout_keep_prob=0.8, - prediction_fn=slim.softmax, - spatial_squeeze=True, - reuse=None, - scope='InceptionV1'): - """Defines the Inception V1 architecture. - - This architecture is defined in: - - Going deeper with convolutions - Christian Szegedy, Wei Liu, Yangqing Jia, Pierre Sermanet, Scott Reed, - Dragomir Anguelov, Dumitru Erhan, Vincent Vanhoucke, Andrew Rabinovich. - http://arxiv.org/pdf/1409.4842v1.pdf. - - The default image size used to train this network is 224x224. - - Args: - inputs: a tensor of size [batch_size, height, width, channels]. - num_classes: number of predicted classes. - is_training: whether is training or not. - dropout_keep_prob: the percentage of activation values that are retained. - prediction_fn: a function to get predictions out of logits. - spatial_squeeze: if True, logits is of shape is [B, C], if false logits is - of shape [B, 1, 1, C], where B is batch_size and C is number of classes. - reuse: whether or not the network and its variables should be reused. To be - able to reuse 'scope' must be given. - scope: Optional variable_scope. - - Returns: - logits: the pre-softmax activations, a tensor of size - [batch_size, num_classes] - end_points: a dictionary from components of the network to the corresponding - activation. - """ - # Final pooling and prediction - with tf.variable_scope(scope, 'InceptionV1', [inputs, num_classes], - reuse=reuse) as scope: - with slim.arg_scope([slim.batch_norm, slim.dropout], - is_training=is_training): - net, end_points = inception_v1_base(inputs, scope=scope) - with tf.variable_scope('Logits'): - net = slim.avg_pool2d(net, [7, 7], stride=1, scope='MaxPool_0a_7x7') - net = slim.dropout(net, - dropout_keep_prob, scope='Dropout_0b') - logits = slim.conv2d(net, num_classes, [1, 1], activation_fn=None, - normalizer_fn=None, scope='Conv2d_0c_1x1') - if spatial_squeeze: - logits = tf.squeeze(logits, [1, 2], name='SpatialSqueeze') - - end_points['Logits'] = logits - end_points['Predictions'] = prediction_fn(logits, scope='Predictions') - return logits, end_points -inception_v1.default_image_size = 224 - - -def inception_v1_arg_scope(weight_decay=0.00004, - use_batch_norm=True): - """Defines the default InceptionV1 arg scope. - - Note: Althougth the original paper didn't use batch_norm we found it useful. - - Args: - weight_decay: The weight decay to use for regularizing the model. - use_batch_norm: "If `True`, batch_norm is applied after each convolution. - - Returns: - An `arg_scope` to use for the inception v3 model. - """ - batch_norm_params = { - # Decay for the moving averages. - 'decay': 0.9997, - # epsilon to prevent 0s in variance. - 'epsilon': 0.001, - # collection containing update_ops. - 'updates_collections': tf.GraphKeys.UPDATE_OPS, - } - if use_batch_norm: - normalizer_fn = slim.batch_norm - normalizer_params = batch_norm_params - else: - normalizer_fn = None - normalizer_params = {} - # Set weight_decay for weights in Conv and FC layers. - with slim.arg_scope([slim.conv2d, slim.fully_connected], - weights_regularizer=slim.l2_regularizer(weight_decay)): - with slim.arg_scope( - [slim.conv2d], - weights_initializer=slim.variance_scaling_initializer(), - activation_fn=tf.nn.relu, - normalizer_fn=normalizer_fn, - normalizer_params=normalizer_params) as sc: - return sc diff --git a/nets/inception_v1_test.py b/nets/inception_v1_test.py deleted file mode 100644 index a769538..0000000 --- a/nets/inception_v1_test.py +++ /dev/null @@ -1,210 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for nets.inception_v1.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -import tensorflow as tf - -from nets import inception - -slim = tf.contrib.slim - - -class InceptionV1Test(tf.test.TestCase): - - def testBuildClassificationNetwork(self): - batch_size = 5 - height, width = 224, 224 - num_classes = 1000 - - inputs = tf.random_uniform((batch_size, height, width, 3)) - logits, end_points = inception.inception_v1(inputs, num_classes) - self.assertTrue(logits.op.name.startswith('InceptionV1/Logits')) - self.assertListEqual(logits.get_shape().as_list(), - [batch_size, num_classes]) - self.assertTrue('Predictions' in end_points) - self.assertListEqual(end_points['Predictions'].get_shape().as_list(), - [batch_size, num_classes]) - - def testBuildBaseNetwork(self): - batch_size = 5 - height, width = 224, 224 - - inputs = tf.random_uniform((batch_size, height, width, 3)) - mixed_6c, end_points = inception.inception_v1_base(inputs) - self.assertTrue(mixed_6c.op.name.startswith('InceptionV1/Mixed_5c')) - self.assertListEqual(mixed_6c.get_shape().as_list(), - [batch_size, 7, 7, 1024]) - expected_endpoints = ['Conv2d_1a_7x7', 'MaxPool_2a_3x3', 'Conv2d_2b_1x1', - 'Conv2d_2c_3x3', 'MaxPool_3a_3x3', 'Mixed_3b', - 'Mixed_3c', 'MaxPool_4a_3x3', 'Mixed_4b', 'Mixed_4c', - 'Mixed_4d', 'Mixed_4e', 'Mixed_4f', 'MaxPool_5a_2x2', - 'Mixed_5b', 'Mixed_5c'] - self.assertItemsEqual(end_points.keys(), expected_endpoints) - - def testBuildOnlyUptoFinalEndpoint(self): - batch_size = 5 - height, width = 224, 224 - endpoints = ['Conv2d_1a_7x7', 'MaxPool_2a_3x3', 'Conv2d_2b_1x1', - 'Conv2d_2c_3x3', 'MaxPool_3a_3x3', 'Mixed_3b', 'Mixed_3c', - 'MaxPool_4a_3x3', 'Mixed_4b', 'Mixed_4c', 'Mixed_4d', - 'Mixed_4e', 'Mixed_4f', 'MaxPool_5a_2x2', 'Mixed_5b', - 'Mixed_5c'] - for index, endpoint in enumerate(endpoints): - with tf.Graph().as_default(): - inputs = tf.random_uniform((batch_size, height, width, 3)) - out_tensor, end_points = inception.inception_v1_base( - inputs, final_endpoint=endpoint) - self.assertTrue(out_tensor.op.name.startswith( - 'InceptionV1/' + endpoint)) - self.assertItemsEqual(endpoints[:index+1], end_points) - - def testBuildAndCheckAllEndPointsUptoMixed5c(self): - batch_size = 5 - height, width = 224, 224 - - inputs = tf.random_uniform((batch_size, height, width, 3)) - _, end_points = inception.inception_v1_base(inputs, - final_endpoint='Mixed_5c') - endpoints_shapes = {'Conv2d_1a_7x7': [5, 112, 112, 64], - 'MaxPool_2a_3x3': [5, 56, 56, 64], - 'Conv2d_2b_1x1': [5, 56, 56, 64], - 'Conv2d_2c_3x3': [5, 56, 56, 192], - 'MaxPool_3a_3x3': [5, 28, 28, 192], - 'Mixed_3b': [5, 28, 28, 256], - 'Mixed_3c': [5, 28, 28, 480], - 'MaxPool_4a_3x3': [5, 14, 14, 480], - 'Mixed_4b': [5, 14, 14, 512], - 'Mixed_4c': [5, 14, 14, 512], - 'Mixed_4d': [5, 14, 14, 512], - 'Mixed_4e': [5, 14, 14, 528], - 'Mixed_4f': [5, 14, 14, 832], - 'MaxPool_5a_2x2': [5, 7, 7, 832], - 'Mixed_5b': [5, 7, 7, 832], - 'Mixed_5c': [5, 7, 7, 1024]} - - self.assertItemsEqual(endpoints_shapes.keys(), end_points.keys()) - for endpoint_name in endpoints_shapes: - expected_shape = endpoints_shapes[endpoint_name] - self.assertTrue(endpoint_name in end_points) - self.assertListEqual(end_points[endpoint_name].get_shape().as_list(), - expected_shape) - - def testModelHasExpectedNumberOfParameters(self): - batch_size = 5 - height, width = 224, 224 - inputs = tf.random_uniform((batch_size, height, width, 3)) - with slim.arg_scope(inception.inception_v1_arg_scope()): - inception.inception_v1_base(inputs) - total_params, _ = slim.model_analyzer.analyze_vars( - slim.get_model_variables()) - self.assertAlmostEqual(5607184, total_params) - - def testHalfSizeImages(self): - batch_size = 5 - height, width = 112, 112 - - inputs = tf.random_uniform((batch_size, height, width, 3)) - mixed_5c, _ = inception.inception_v1_base(inputs) - self.assertTrue(mixed_5c.op.name.startswith('InceptionV1/Mixed_5c')) - self.assertListEqual(mixed_5c.get_shape().as_list(), - [batch_size, 4, 4, 1024]) - - def testUnknownImageShape(self): - tf.reset_default_graph() - batch_size = 2 - height, width = 224, 224 - num_classes = 1000 - input_np = np.random.uniform(0, 1, (batch_size, height, width, 3)) - with self.test_session() as sess: - inputs = tf.placeholder(tf.float32, shape=(batch_size, None, None, 3)) - logits, end_points = inception.inception_v1(inputs, num_classes) - self.assertTrue(logits.op.name.startswith('InceptionV1/Logits')) - self.assertListEqual(logits.get_shape().as_list(), - [batch_size, num_classes]) - pre_pool = end_points['Mixed_5c'] - feed_dict = {inputs: input_np} - tf.initialize_all_variables().run() - pre_pool_out = sess.run(pre_pool, feed_dict=feed_dict) - self.assertListEqual(list(pre_pool_out.shape), [batch_size, 7, 7, 1024]) - - def testUnknowBatchSize(self): - batch_size = 1 - height, width = 224, 224 - num_classes = 1000 - - inputs = tf.placeholder(tf.float32, (None, height, width, 3)) - logits, _ = inception.inception_v1(inputs, num_classes) - self.assertTrue(logits.op.name.startswith('InceptionV1/Logits')) - self.assertListEqual(logits.get_shape().as_list(), - [None, num_classes]) - images = tf.random_uniform((batch_size, height, width, 3)) - - with self.test_session() as sess: - sess.run(tf.initialize_all_variables()) - output = sess.run(logits, {inputs: images.eval()}) - self.assertEquals(output.shape, (batch_size, num_classes)) - - def testEvaluation(self): - batch_size = 2 - height, width = 224, 224 - num_classes = 1000 - - eval_inputs = tf.random_uniform((batch_size, height, width, 3)) - logits, _ = inception.inception_v1(eval_inputs, num_classes, - is_training=False) - predictions = tf.argmax(logits, 1) - - with self.test_session() as sess: - sess.run(tf.initialize_all_variables()) - output = sess.run(predictions) - self.assertEquals(output.shape, (batch_size,)) - - def testTrainEvalWithReuse(self): - train_batch_size = 5 - eval_batch_size = 2 - height, width = 224, 224 - num_classes = 1000 - - train_inputs = tf.random_uniform((train_batch_size, height, width, 3)) - inception.inception_v1(train_inputs, num_classes) - eval_inputs = tf.random_uniform((eval_batch_size, height, width, 3)) - logits, _ = inception.inception_v1(eval_inputs, num_classes, reuse=True) - predictions = tf.argmax(logits, 1) - - with self.test_session() as sess: - sess.run(tf.initialize_all_variables()) - output = sess.run(predictions) - self.assertEquals(output.shape, (eval_batch_size,)) - - def testLogitsNotSqueezed(self): - num_classes = 25 - images = tf.random_uniform([1, 224, 224, 3]) - logits, _ = inception.inception_v1(images, - num_classes=num_classes, - spatial_squeeze=False) - - with self.test_session() as sess: - tf.initialize_all_variables().run() - logits_out = sess.run(logits) - self.assertListEqual(list(logits_out.shape), [1, 1, 1, num_classes]) - - -if __name__ == '__main__': - tf.test.main() diff --git a/nets/inception_v2.py b/nets/inception_v2.py deleted file mode 100644 index 914c4b2..0000000 --- a/nets/inception_v2.py +++ /dev/null @@ -1,545 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Contains the definition for inception v2 classification network.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import tensorflow as tf - -slim = tf.contrib.slim -trunc_normal = lambda stddev: tf.truncated_normal_initializer(0.0, stddev) - - -def inception_v2_base(inputs, - final_endpoint='Mixed_5c', - min_depth=16, - depth_multiplier=1.0, - scope=None): - """Inception v2 (6a2). - - Constructs an Inception v2 network from inputs to the given final endpoint. - This method can construct the network up to the layer inception(5b) as - described in http://arxiv.org/abs/1502.03167. - - Args: - inputs: a tensor of shape [batch_size, height, width, channels]. - final_endpoint: specifies the endpoint to construct the network up to. It - can be one of ['Conv2d_1a_7x7', 'MaxPool_2a_3x3', 'Conv2d_2b_1x1', - 'Conv2d_2c_3x3', 'MaxPool_3a_3x3', 'Mixed_3b', 'Mixed_3c', 'Mixed_4a', - 'Mixed_4b', 'Mixed_4c', 'Mixed_4d', 'Mixed_4e', 'Mixed_5a', 'Mixed_5b', - 'Mixed_5c']. - min_depth: Minimum depth value (number of channels) for all convolution ops. - Enforced when depth_multiplier < 1, and not an active constraint when - depth_multiplier >= 1. - depth_multiplier: Float multiplier for the depth (number of channels) - for all convolution ops. The value must be greater than zero. Typical - usage will be to set this value in (0, 1) to reduce the number of - parameters or computation cost of the model. - scope: Optional variable_scope. - - Returns: - tensor_out: output tensor corresponding to the final_endpoint. - end_points: a set of activations for external use, for example summaries or - losses. - - Raises: - ValueError: if final_endpoint is not set to one of the predefined values, - or depth_multiplier <= 0 - """ - - # end_points will collect relevant activations for external use, for example - # summaries or losses. - end_points = {} - - # Used to find thinned depths for each layer. - if depth_multiplier <= 0: - raise ValueError('depth_multiplier is not greater than zero.') - depth = lambda d: max(int(d * depth_multiplier), min_depth) - - with tf.variable_scope(scope, 'InceptionV2', [inputs]): - with slim.arg_scope( - [slim.conv2d, slim.max_pool2d, slim.avg_pool2d, slim.separable_conv2d], - stride=1, padding='SAME'): - - # Note that sizes in the comments below assume an input spatial size of - # 224x224, however, the inputs can be of any size greater 32x32. - - # 224 x 224 x 3 - end_point = 'Conv2d_1a_7x7' - # depthwise_multiplier here is different from depth_multiplier. - # depthwise_multiplier determines the output channels of the initial - # depthwise conv (see docs for tf.nn.separable_conv2d), while - # depth_multiplier controls the # channels of the subsequent 1x1 - # convolution. Must have - # in_channels * depthwise_multipler <= out_channels - # so that the separable convolution is not overparameterized. - depthwise_multiplier = min(int(depth(64) / 3), 8) - net = slim.separable_conv2d( - inputs, depth(64), [7, 7], depth_multiplier=depthwise_multiplier, - stride=2, weights_initializer=trunc_normal(1.0), - scope=end_point) - end_points[end_point] = net - if end_point == final_endpoint: return net, end_points - # 112 x 112 x 64 - end_point = 'MaxPool_2a_3x3' - net = slim.max_pool2d(net, [3, 3], scope=end_point, stride=2) - end_points[end_point] = net - if end_point == final_endpoint: return net, end_points - # 56 x 56 x 64 - end_point = 'Conv2d_2b_1x1' - net = slim.conv2d(net, depth(64), [1, 1], scope=end_point, - weights_initializer=trunc_normal(0.1)) - end_points[end_point] = net - if end_point == final_endpoint: return net, end_points - # 56 x 56 x 64 - end_point = 'Conv2d_2c_3x3' - net = slim.conv2d(net, depth(192), [3, 3], scope=end_point) - end_points[end_point] = net - if end_point == final_endpoint: return net, end_points - # 56 x 56 x 192 - end_point = 'MaxPool_3a_3x3' - net = slim.max_pool2d(net, [3, 3], scope=end_point, stride=2) - end_points[end_point] = net - if end_point == final_endpoint: return net, end_points - # 28 x 28 x 192 - # Inception module. - end_point = 'Mixed_3b' - with tf.variable_scope(end_point): - with tf.variable_scope('Branch_0'): - branch_0 = slim.conv2d(net, depth(64), [1, 1], scope='Conv2d_0a_1x1') - with tf.variable_scope('Branch_1'): - branch_1 = slim.conv2d( - net, depth(64), [1, 1], - weights_initializer=trunc_normal(0.09), - scope='Conv2d_0a_1x1') - branch_1 = slim.conv2d(branch_1, depth(64), [3, 3], - scope='Conv2d_0b_3x3') - with tf.variable_scope('Branch_2'): - branch_2 = slim.conv2d( - net, depth(64), [1, 1], - weights_initializer=trunc_normal(0.09), - scope='Conv2d_0a_1x1') - branch_2 = slim.conv2d(branch_2, depth(96), [3, 3], - scope='Conv2d_0b_3x3') - branch_2 = slim.conv2d(branch_2, depth(96), [3, 3], - scope='Conv2d_0c_3x3') - with tf.variable_scope('Branch_3'): - branch_3 = slim.avg_pool2d(net, [3, 3], scope='AvgPool_0a_3x3') - branch_3 = slim.conv2d( - branch_3, depth(32), [1, 1], - weights_initializer=trunc_normal(0.1), - scope='Conv2d_0b_1x1') - net = tf.concat(3, [branch_0, branch_1, branch_2, branch_3]) - end_points[end_point] = net - if end_point == final_endpoint: return net, end_points - # 28 x 28 x 256 - end_point = 'Mixed_3c' - with tf.variable_scope(end_point): - with tf.variable_scope('Branch_0'): - branch_0 = slim.conv2d(net, depth(64), [1, 1], scope='Conv2d_0a_1x1') - with tf.variable_scope('Branch_1'): - branch_1 = slim.conv2d( - net, depth(64), [1, 1], - weights_initializer=trunc_normal(0.09), - scope='Conv2d_0a_1x1') - branch_1 = slim.conv2d(branch_1, depth(96), [3, 3], - scope='Conv2d_0b_3x3') - with tf.variable_scope('Branch_2'): - branch_2 = slim.conv2d( - net, depth(64), [1, 1], - weights_initializer=trunc_normal(0.09), - scope='Conv2d_0a_1x1') - branch_2 = slim.conv2d(branch_2, depth(96), [3, 3], - scope='Conv2d_0b_3x3') - branch_2 = slim.conv2d(branch_2, depth(96), [3, 3], - scope='Conv2d_0c_3x3') - with tf.variable_scope('Branch_3'): - branch_3 = slim.avg_pool2d(net, [3, 3], scope='AvgPool_0a_3x3') - branch_3 = slim.conv2d( - branch_3, depth(64), [1, 1], - weights_initializer=trunc_normal(0.1), - scope='Conv2d_0b_1x1') - net = tf.concat(3, [branch_0, branch_1, branch_2, branch_3]) - end_points[end_point] = net - if end_point == final_endpoint: return net, end_points - # 28 x 28 x 320 - end_point = 'Mixed_4a' - with tf.variable_scope(end_point): - with tf.variable_scope('Branch_0'): - branch_0 = slim.conv2d( - net, depth(128), [1, 1], - weights_initializer=trunc_normal(0.09), - scope='Conv2d_0a_1x1') - branch_0 = slim.conv2d(branch_0, depth(160), [3, 3], stride=2, - scope='Conv2d_1a_3x3') - with tf.variable_scope('Branch_1'): - branch_1 = slim.conv2d( - net, depth(64), [1, 1], - weights_initializer=trunc_normal(0.09), - scope='Conv2d_0a_1x1') - branch_1 = slim.conv2d( - branch_1, depth(96), [3, 3], scope='Conv2d_0b_3x3') - branch_1 = slim.conv2d( - branch_1, depth(96), [3, 3], stride=2, scope='Conv2d_1a_3x3') - with tf.variable_scope('Branch_2'): - branch_2 = slim.max_pool2d( - net, [3, 3], stride=2, scope='MaxPool_1a_3x3') - net = tf.concat(3, [branch_0, branch_1, branch_2]) - end_points[end_point] = net - if end_point == final_endpoint: return net, end_points - # 14 x 14 x 576 - end_point = 'Mixed_4b' - with tf.variable_scope(end_point): - with tf.variable_scope('Branch_0'): - branch_0 = slim.conv2d(net, depth(224), [1, 1], scope='Conv2d_0a_1x1') - with tf.variable_scope('Branch_1'): - branch_1 = slim.conv2d( - net, depth(64), [1, 1], - weights_initializer=trunc_normal(0.09), - scope='Conv2d_0a_1x1') - branch_1 = slim.conv2d( - branch_1, depth(96), [3, 3], scope='Conv2d_0b_3x3') - with tf.variable_scope('Branch_2'): - branch_2 = slim.conv2d( - net, depth(96), [1, 1], - weights_initializer=trunc_normal(0.09), - scope='Conv2d_0a_1x1') - branch_2 = slim.conv2d(branch_2, depth(128), [3, 3], - scope='Conv2d_0b_3x3') - branch_2 = slim.conv2d(branch_2, depth(128), [3, 3], - scope='Conv2d_0c_3x3') - with tf.variable_scope('Branch_3'): - branch_3 = slim.avg_pool2d(net, [3, 3], scope='AvgPool_0a_3x3') - branch_3 = slim.conv2d( - branch_3, depth(128), [1, 1], - weights_initializer=trunc_normal(0.1), - scope='Conv2d_0b_1x1') - net = tf.concat(3, [branch_0, branch_1, branch_2, branch_3]) - end_points[end_point] = net - if end_point == final_endpoint: return net, end_points - # 14 x 14 x 576 - end_point = 'Mixed_4c' - with tf.variable_scope(end_point): - with tf.variable_scope('Branch_0'): - branch_0 = slim.conv2d(net, depth(192), [1, 1], scope='Conv2d_0a_1x1') - with tf.variable_scope('Branch_1'): - branch_1 = slim.conv2d( - net, depth(96), [1, 1], - weights_initializer=trunc_normal(0.09), - scope='Conv2d_0a_1x1') - branch_1 = slim.conv2d(branch_1, depth(128), [3, 3], - scope='Conv2d_0b_3x3') - with tf.variable_scope('Branch_2'): - branch_2 = slim.conv2d( - net, depth(96), [1, 1], - weights_initializer=trunc_normal(0.09), - scope='Conv2d_0a_1x1') - branch_2 = slim.conv2d(branch_2, depth(128), [3, 3], - scope='Conv2d_0b_3x3') - branch_2 = slim.conv2d(branch_2, depth(128), [3, 3], - scope='Conv2d_0c_3x3') - with tf.variable_scope('Branch_3'): - branch_3 = slim.avg_pool2d(net, [3, 3], scope='AvgPool_0a_3x3') - branch_3 = slim.conv2d( - branch_3, depth(128), [1, 1], - weights_initializer=trunc_normal(0.1), - scope='Conv2d_0b_1x1') - net = tf.concat(3, [branch_0, branch_1, branch_2, branch_3]) - end_points[end_point] = net - if end_point == final_endpoint: return net, end_points - # 14 x 14 x 576 - end_point = 'Mixed_4d' - with tf.variable_scope(end_point): - with tf.variable_scope('Branch_0'): - branch_0 = slim.conv2d(net, depth(160), [1, 1], scope='Conv2d_0a_1x1') - with tf.variable_scope('Branch_1'): - branch_1 = slim.conv2d( - net, depth(128), [1, 1], - weights_initializer=trunc_normal(0.09), - scope='Conv2d_0a_1x1') - branch_1 = slim.conv2d(branch_1, depth(160), [3, 3], - scope='Conv2d_0b_3x3') - with tf.variable_scope('Branch_2'): - branch_2 = slim.conv2d( - net, depth(128), [1, 1], - weights_initializer=trunc_normal(0.09), - scope='Conv2d_0a_1x1') - branch_2 = slim.conv2d(branch_2, depth(160), [3, 3], - scope='Conv2d_0b_3x3') - branch_2 = slim.conv2d(branch_2, depth(160), [3, 3], - scope='Conv2d_0c_3x3') - with tf.variable_scope('Branch_3'): - branch_3 = slim.avg_pool2d(net, [3, 3], scope='AvgPool_0a_3x3') - branch_3 = slim.conv2d( - branch_3, depth(96), [1, 1], - weights_initializer=trunc_normal(0.1), - scope='Conv2d_0b_1x1') - net = tf.concat(3, [branch_0, branch_1, branch_2, branch_3]) - end_points[end_point] = net - if end_point == final_endpoint: return net, end_points - - # 14 x 14 x 576 - end_point = 'Mixed_4e' - with tf.variable_scope(end_point): - with tf.variable_scope('Branch_0'): - branch_0 = slim.conv2d(net, depth(96), [1, 1], scope='Conv2d_0a_1x1') - with tf.variable_scope('Branch_1'): - branch_1 = slim.conv2d( - net, depth(128), [1, 1], - weights_initializer=trunc_normal(0.09), - scope='Conv2d_0a_1x1') - branch_1 = slim.conv2d(branch_1, depth(192), [3, 3], - scope='Conv2d_0b_3x3') - with tf.variable_scope('Branch_2'): - branch_2 = slim.conv2d( - net, depth(160), [1, 1], - weights_initializer=trunc_normal(0.09), - scope='Conv2d_0a_1x1') - branch_2 = slim.conv2d(branch_2, depth(192), [3, 3], - scope='Conv2d_0b_3x3') - branch_2 = slim.conv2d(branch_2, depth(192), [3, 3], - scope='Conv2d_0c_3x3') - with tf.variable_scope('Branch_3'): - branch_3 = slim.avg_pool2d(net, [3, 3], scope='AvgPool_0a_3x3') - branch_3 = slim.conv2d( - branch_3, depth(96), [1, 1], - weights_initializer=trunc_normal(0.1), - scope='Conv2d_0b_1x1') - net = tf.concat(3, [branch_0, branch_1, branch_2, branch_3]) - end_points[end_point] = net - if end_point == final_endpoint: return net, end_points - # 14 x 14 x 576 - end_point = 'Mixed_5a' - with tf.variable_scope(end_point): - with tf.variable_scope('Branch_0'): - branch_0 = slim.conv2d( - net, depth(128), [1, 1], - weights_initializer=trunc_normal(0.09), - scope='Conv2d_0a_1x1') - branch_0 = slim.conv2d(branch_0, depth(192), [3, 3], stride=2, - scope='Conv2d_1a_3x3') - with tf.variable_scope('Branch_1'): - branch_1 = slim.conv2d( - net, depth(192), [1, 1], - weights_initializer=trunc_normal(0.09), - scope='Conv2d_0a_1x1') - branch_1 = slim.conv2d(branch_1, depth(256), [3, 3], - scope='Conv2d_0b_3x3') - branch_1 = slim.conv2d(branch_1, depth(256), [3, 3], stride=2, - scope='Conv2d_1a_3x3') - with tf.variable_scope('Branch_2'): - branch_2 = slim.max_pool2d(net, [3, 3], stride=2, - scope='MaxPool_1a_3x3') - net = tf.concat(3, [branch_0, branch_1, branch_2]) - end_points[end_point] = net - if end_point == final_endpoint: return net, end_points - # 7 x 7 x 1024 - end_point = 'Mixed_5b' - with tf.variable_scope(end_point): - with tf.variable_scope('Branch_0'): - branch_0 = slim.conv2d(net, depth(352), [1, 1], scope='Conv2d_0a_1x1') - with tf.variable_scope('Branch_1'): - branch_1 = slim.conv2d( - net, depth(192), [1, 1], - weights_initializer=trunc_normal(0.09), - scope='Conv2d_0a_1x1') - branch_1 = slim.conv2d(branch_1, depth(320), [3, 3], - scope='Conv2d_0b_3x3') - with tf.variable_scope('Branch_2'): - branch_2 = slim.conv2d( - net, depth(160), [1, 1], - weights_initializer=trunc_normal(0.09), - scope='Conv2d_0a_1x1') - branch_2 = slim.conv2d(branch_2, depth(224), [3, 3], - scope='Conv2d_0b_3x3') - branch_2 = slim.conv2d(branch_2, depth(224), [3, 3], - scope='Conv2d_0c_3x3') - with tf.variable_scope('Branch_3'): - branch_3 = slim.avg_pool2d(net, [3, 3], scope='AvgPool_0a_3x3') - branch_3 = slim.conv2d( - branch_3, depth(128), [1, 1], - weights_initializer=trunc_normal(0.1), - scope='Conv2d_0b_1x1') - net = tf.concat(3, [branch_0, branch_1, branch_2, branch_3]) - end_points[end_point] = net - if end_point == final_endpoint: return net, end_points - - # 7 x 7 x 1024 - end_point = 'Mixed_5c' - with tf.variable_scope(end_point): - with tf.variable_scope('Branch_0'): - branch_0 = slim.conv2d(net, depth(352), [1, 1], scope='Conv2d_0a_1x1') - with tf.variable_scope('Branch_1'): - branch_1 = slim.conv2d( - net, depth(192), [1, 1], - weights_initializer=trunc_normal(0.09), - scope='Conv2d_0a_1x1') - branch_1 = slim.conv2d(branch_1, depth(320), [3, 3], - scope='Conv2d_0b_3x3') - with tf.variable_scope('Branch_2'): - branch_2 = slim.conv2d( - net, depth(192), [1, 1], - weights_initializer=trunc_normal(0.09), - scope='Conv2d_0a_1x1') - branch_2 = slim.conv2d(branch_2, depth(224), [3, 3], - scope='Conv2d_0b_3x3') - branch_2 = slim.conv2d(branch_2, depth(224), [3, 3], - scope='Conv2d_0c_3x3') - with tf.variable_scope('Branch_3'): - branch_3 = slim.max_pool2d(net, [3, 3], scope='MaxPool_0a_3x3') - branch_3 = slim.conv2d( - branch_3, depth(128), [1, 1], - weights_initializer=trunc_normal(0.1), - scope='Conv2d_0b_1x1') - net = tf.concat(3, [branch_0, branch_1, branch_2, branch_3]) - end_points[end_point] = net - if end_point == final_endpoint: return net, end_points - raise ValueError('Unknown final endpoint %s' % final_endpoint) - - -def inception_v2(inputs, - num_classes=1000, - is_training=True, - dropout_keep_prob=0.8, - min_depth=16, - depth_multiplier=1.0, - prediction_fn=slim.softmax, - spatial_squeeze=True, - reuse=None, - scope='InceptionV2'): - """Inception v2 model for classification. - - Constructs an Inception v2 network for classification as described in - http://arxiv.org/abs/1502.03167. - - The default image size used to train this network is 224x224. - - Args: - inputs: a tensor of shape [batch_size, height, width, channels]. - num_classes: number of predicted classes. - is_training: whether is training or not. - dropout_keep_prob: the percentage of activation values that are retained. - min_depth: Minimum depth value (number of channels) for all convolution ops. - Enforced when depth_multiplier < 1, and not an active constraint when - depth_multiplier >= 1. - depth_multiplier: Float multiplier for the depth (number of channels) - for all convolution ops. The value must be greater than zero. Typical - usage will be to set this value in (0, 1) to reduce the number of - parameters or computation cost of the model. - prediction_fn: a function to get predictions out of logits. - spatial_squeeze: if True, logits is of shape is [B, C], if false logits is - of shape [B, 1, 1, C], where B is batch_size and C is number of classes. - reuse: whether or not the network and its variables should be reused. To be - able to reuse 'scope' must be given. - scope: Optional variable_scope. - - Returns: - logits: the pre-softmax activations, a tensor of size - [batch_size, num_classes] - end_points: a dictionary from components of the network to the corresponding - activation. - - Raises: - ValueError: if final_endpoint is not set to one of the predefined values, - or depth_multiplier <= 0 - """ - if depth_multiplier <= 0: - raise ValueError('depth_multiplier is not greater than zero.') - - # Final pooling and prediction - with tf.variable_scope(scope, 'InceptionV2', [inputs, num_classes], - reuse=reuse) as scope: - with slim.arg_scope([slim.batch_norm, slim.dropout], - is_training=is_training): - net, end_points = inception_v2_base( - inputs, scope=scope, min_depth=min_depth, - depth_multiplier=depth_multiplier) - with tf.variable_scope('Logits'): - kernel_size = _reduced_kernel_size_for_small_input(net, [7, 7]) - net = slim.avg_pool2d(net, kernel_size, padding='VALID', - scope='AvgPool_1a_{}x{}'.format(*kernel_size)) - # 1 x 1 x 1024 - net = slim.dropout(net, keep_prob=dropout_keep_prob, scope='Dropout_1b') - logits = slim.conv2d(net, num_classes, [1, 1], activation_fn=None, - normalizer_fn=None, scope='Conv2d_1c_1x1') - if spatial_squeeze: - logits = tf.squeeze(logits, [1, 2], name='SpatialSqueeze') - end_points['Logits'] = logits - end_points['Predictions'] = prediction_fn(logits, scope='Predictions') - return logits, end_points -inception_v2.default_image_size = 224 - - -def _reduced_kernel_size_for_small_input(input_tensor, kernel_size): - """Define kernel size which is automatically reduced for small input. - - If the shape of the input images is unknown at graph construction time this - function assumes that the input images are is large enough. - - Args: - input_tensor: input tensor of size [batch_size, height, width, channels]. - kernel_size: desired kernel size of length 2: [kernel_height, kernel_width] - - Returns: - a tensor with the kernel size. - - TODO(jrru): Make this function work with unknown shapes. Theoretically, this - can be done with the code below. Problems are two-fold: (1) If the shape was - known, it will be lost. (2) inception.slim.ops._two_element_tuple cannot - handle tensors that define the kernel size. - shape = tf.shape(input_tensor) - return = tf.pack([tf.minimum(shape[1], kernel_size[0]), - tf.minimum(shape[2], kernel_size[1])]) - - """ - shape = input_tensor.get_shape().as_list() - if shape[1] is None or shape[2] is None: - kernel_size_out = kernel_size - else: - kernel_size_out = [min(shape[1], kernel_size[0]), - min(shape[2], kernel_size[1])] - return kernel_size_out - - -def inception_v2_arg_scope(weight_decay=0.00004): - """Defines the default InceptionV2 arg scope. - - Args: - weight_decay: The weight decay to use for regularizing the model. - - Returns: - An `arg_scope` to use for the inception v3 model. - """ - batch_norm_params = { - # Decay for the moving averages. - 'decay': 0.9997, - # epsilon to prevent 0s in variance. - 'epsilon': 0.001, - # collection containing update_ops. - 'updates_collections': tf.GraphKeys.UPDATE_OPS, - } - - # Set weight_decay for weights in Conv and FC layers. - with slim.arg_scope([slim.conv2d, slim.fully_connected], - weights_regularizer=slim.l2_regularizer(weight_decay)): - with slim.arg_scope( - [slim.conv2d], - weights_initializer=slim.variance_scaling_initializer(), - activation_fn=tf.nn.relu, - normalizer_fn=slim.batch_norm, - normalizer_params=batch_norm_params) as sc: - return sc diff --git a/nets/inception_v2_test.py b/nets/inception_v2_test.py deleted file mode 100644 index 0ff850c..0000000 --- a/nets/inception_v2_test.py +++ /dev/null @@ -1,262 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for nets.inception_v2.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -import tensorflow as tf - -from nets import inception - -slim = tf.contrib.slim - - -class InceptionV2Test(tf.test.TestCase): - - def testBuildClassificationNetwork(self): - batch_size = 5 - height, width = 224, 224 - num_classes = 1000 - - inputs = tf.random_uniform((batch_size, height, width, 3)) - logits, end_points = inception.inception_v2(inputs, num_classes) - self.assertTrue(logits.op.name.startswith('InceptionV2/Logits')) - self.assertListEqual(logits.get_shape().as_list(), - [batch_size, num_classes]) - self.assertTrue('Predictions' in end_points) - self.assertListEqual(end_points['Predictions'].get_shape().as_list(), - [batch_size, num_classes]) - - def testBuildBaseNetwork(self): - batch_size = 5 - height, width = 224, 224 - - inputs = tf.random_uniform((batch_size, height, width, 3)) - mixed_5c, end_points = inception.inception_v2_base(inputs) - self.assertTrue(mixed_5c.op.name.startswith('InceptionV2/Mixed_5c')) - self.assertListEqual(mixed_5c.get_shape().as_list(), - [batch_size, 7, 7, 1024]) - expected_endpoints = ['Mixed_3b', 'Mixed_3c', 'Mixed_4a', 'Mixed_4b', - 'Mixed_4c', 'Mixed_4d', 'Mixed_4e', 'Mixed_5a', - 'Mixed_5b', 'Mixed_5c', 'Conv2d_1a_7x7', - 'MaxPool_2a_3x3', 'Conv2d_2b_1x1', 'Conv2d_2c_3x3', - 'MaxPool_3a_3x3'] - self.assertItemsEqual(end_points.keys(), expected_endpoints) - - def testBuildOnlyUptoFinalEndpoint(self): - batch_size = 5 - height, width = 224, 224 - endpoints = ['Conv2d_1a_7x7', 'MaxPool_2a_3x3', 'Conv2d_2b_1x1', - 'Conv2d_2c_3x3', 'MaxPool_3a_3x3', 'Mixed_3b', 'Mixed_3c', - 'Mixed_4a', 'Mixed_4b', 'Mixed_4c', 'Mixed_4d', 'Mixed_4e', - 'Mixed_5a', 'Mixed_5b', 'Mixed_5c'] - for index, endpoint in enumerate(endpoints): - with tf.Graph().as_default(): - inputs = tf.random_uniform((batch_size, height, width, 3)) - out_tensor, end_points = inception.inception_v2_base( - inputs, final_endpoint=endpoint) - self.assertTrue(out_tensor.op.name.startswith( - 'InceptionV2/' + endpoint)) - self.assertItemsEqual(endpoints[:index+1], end_points) - - def testBuildAndCheckAllEndPointsUptoMixed5c(self): - batch_size = 5 - height, width = 224, 224 - - inputs = tf.random_uniform((batch_size, height, width, 3)) - _, end_points = inception.inception_v2_base(inputs, - final_endpoint='Mixed_5c') - endpoints_shapes = {'Mixed_3b': [batch_size, 28, 28, 256], - 'Mixed_3c': [batch_size, 28, 28, 320], - 'Mixed_4a': [batch_size, 14, 14, 576], - 'Mixed_4b': [batch_size, 14, 14, 576], - 'Mixed_4c': [batch_size, 14, 14, 576], - 'Mixed_4d': [batch_size, 14, 14, 576], - 'Mixed_4e': [batch_size, 14, 14, 576], - 'Mixed_5a': [batch_size, 7, 7, 1024], - 'Mixed_5b': [batch_size, 7, 7, 1024], - 'Mixed_5c': [batch_size, 7, 7, 1024], - 'Conv2d_1a_7x7': [batch_size, 112, 112, 64], - 'MaxPool_2a_3x3': [batch_size, 56, 56, 64], - 'Conv2d_2b_1x1': [batch_size, 56, 56, 64], - 'Conv2d_2c_3x3': [batch_size, 56, 56, 192], - 'MaxPool_3a_3x3': [batch_size, 28, 28, 192]} - self.assertItemsEqual(endpoints_shapes.keys(), end_points.keys()) - for endpoint_name in endpoints_shapes: - expected_shape = endpoints_shapes[endpoint_name] - self.assertTrue(endpoint_name in end_points) - self.assertListEqual(end_points[endpoint_name].get_shape().as_list(), - expected_shape) - - def testModelHasExpectedNumberOfParameters(self): - batch_size = 5 - height, width = 224, 224 - inputs = tf.random_uniform((batch_size, height, width, 3)) - with slim.arg_scope(inception.inception_v2_arg_scope()): - inception.inception_v2_base(inputs) - total_params, _ = slim.model_analyzer.analyze_vars( - slim.get_model_variables()) - self.assertAlmostEqual(10173112, total_params) - - def testBuildEndPointsWithDepthMultiplierLessThanOne(self): - batch_size = 5 - height, width = 224, 224 - num_classes = 1000 - - inputs = tf.random_uniform((batch_size, height, width, 3)) - _, end_points = inception.inception_v2(inputs, num_classes) - - endpoint_keys = [key for key in end_points.keys() - if key.startswith('Mixed') or key.startswith('Conv')] - - _, end_points_with_multiplier = inception.inception_v2( - inputs, num_classes, scope='depth_multiplied_net', - depth_multiplier=0.5) - - for key in endpoint_keys: - original_depth = end_points[key].get_shape().as_list()[3] - new_depth = end_points_with_multiplier[key].get_shape().as_list()[3] - self.assertEqual(0.5 * original_depth, new_depth) - - def testBuildEndPointsWithDepthMultiplierGreaterThanOne(self): - batch_size = 5 - height, width = 224, 224 - num_classes = 1000 - - inputs = tf.random_uniform((batch_size, height, width, 3)) - _, end_points = inception.inception_v2(inputs, num_classes) - - endpoint_keys = [key for key in end_points.keys() - if key.startswith('Mixed') or key.startswith('Conv')] - - _, end_points_with_multiplier = inception.inception_v2( - inputs, num_classes, scope='depth_multiplied_net', - depth_multiplier=2.0) - - for key in endpoint_keys: - original_depth = end_points[key].get_shape().as_list()[3] - new_depth = end_points_with_multiplier[key].get_shape().as_list()[3] - self.assertEqual(2.0 * original_depth, new_depth) - - def testRaiseValueErrorWithInvalidDepthMultiplier(self): - batch_size = 5 - height, width = 224, 224 - num_classes = 1000 - - inputs = tf.random_uniform((batch_size, height, width, 3)) - with self.assertRaises(ValueError): - _ = inception.inception_v2(inputs, num_classes, depth_multiplier=-0.1) - with self.assertRaises(ValueError): - _ = inception.inception_v2(inputs, num_classes, depth_multiplier=0.0) - - def testHalfSizeImages(self): - batch_size = 5 - height, width = 112, 112 - num_classes = 1000 - - inputs = tf.random_uniform((batch_size, height, width, 3)) - logits, end_points = inception.inception_v2(inputs, num_classes) - self.assertTrue(logits.op.name.startswith('InceptionV2/Logits')) - self.assertListEqual(logits.get_shape().as_list(), - [batch_size, num_classes]) - pre_pool = end_points['Mixed_5c'] - self.assertListEqual(pre_pool.get_shape().as_list(), - [batch_size, 4, 4, 1024]) - - def testUnknownImageShape(self): - tf.reset_default_graph() - batch_size = 2 - height, width = 224, 224 - num_classes = 1000 - input_np = np.random.uniform(0, 1, (batch_size, height, width, 3)) - with self.test_session() as sess: - inputs = tf.placeholder(tf.float32, shape=(batch_size, None, None, 3)) - logits, end_points = inception.inception_v2(inputs, num_classes) - self.assertTrue(logits.op.name.startswith('InceptionV2/Logits')) - self.assertListEqual(logits.get_shape().as_list(), - [batch_size, num_classes]) - pre_pool = end_points['Mixed_5c'] - feed_dict = {inputs: input_np} - tf.initialize_all_variables().run() - pre_pool_out = sess.run(pre_pool, feed_dict=feed_dict) - self.assertListEqual(list(pre_pool_out.shape), [batch_size, 7, 7, 1024]) - - def testUnknowBatchSize(self): - batch_size = 1 - height, width = 224, 224 - num_classes = 1000 - - inputs = tf.placeholder(tf.float32, (None, height, width, 3)) - logits, _ = inception.inception_v2(inputs, num_classes) - self.assertTrue(logits.op.name.startswith('InceptionV2/Logits')) - self.assertListEqual(logits.get_shape().as_list(), - [None, num_classes]) - images = tf.random_uniform((batch_size, height, width, 3)) - - with self.test_session() as sess: - sess.run(tf.initialize_all_variables()) - output = sess.run(logits, {inputs: images.eval()}) - self.assertEquals(output.shape, (batch_size, num_classes)) - - def testEvaluation(self): - batch_size = 2 - height, width = 224, 224 - num_classes = 1000 - - eval_inputs = tf.random_uniform((batch_size, height, width, 3)) - logits, _ = inception.inception_v2(eval_inputs, num_classes, - is_training=False) - predictions = tf.argmax(logits, 1) - - with self.test_session() as sess: - sess.run(tf.initialize_all_variables()) - output = sess.run(predictions) - self.assertEquals(output.shape, (batch_size,)) - - def testTrainEvalWithReuse(self): - train_batch_size = 5 - eval_batch_size = 2 - height, width = 150, 150 - num_classes = 1000 - - train_inputs = tf.random_uniform((train_batch_size, height, width, 3)) - inception.inception_v2(train_inputs, num_classes) - eval_inputs = tf.random_uniform((eval_batch_size, height, width, 3)) - logits, _ = inception.inception_v2(eval_inputs, num_classes, reuse=True) - predictions = tf.argmax(logits, 1) - - with self.test_session() as sess: - sess.run(tf.initialize_all_variables()) - output = sess.run(predictions) - self.assertEquals(output.shape, (eval_batch_size,)) - - def testLogitsNotSqueezed(self): - num_classes = 25 - images = tf.random_uniform([1, 224, 224, 3]) - logits, _ = inception.inception_v2(images, - num_classes=num_classes, - spatial_squeeze=False) - - with self.test_session() as sess: - tf.initialize_all_variables().run() - logits_out = sess.run(logits) - self.assertListEqual(list(logits_out.shape), [1, 1, 1, num_classes]) - - -if __name__ == '__main__': - tf.test.main() diff --git a/nets/inception_v3.py b/nets/inception_v3.py deleted file mode 100644 index 2897a4b..0000000 --- a/nets/inception_v3.py +++ /dev/null @@ -1,591 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Contains the definition for inception v3 classification network.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import tensorflow as tf - -slim = tf.contrib.slim -trunc_normal = lambda stddev: tf.truncated_normal_initializer(0.0, stddev) - - -def inception_v3_base(inputs, - final_endpoint='Mixed_7c', - min_depth=16, - depth_multiplier=1.0, - scope=None): - """Inception model from http://arxiv.org/abs/1512.00567. - - Constructs an Inception v3 network from inputs to the given final endpoint. - This method can construct the network up to the final inception block - Mixed_7c. - - Note that the names of the layers in the paper do not correspond to the names - of the endpoints registered by this function although they build the same - network. - - Here is a mapping from the old_names to the new names: - Old name | New name - ======================================= - conv0 | Conv2d_1a_3x3 - conv1 | Conv2d_2a_3x3 - conv2 | Conv2d_2b_3x3 - pool1 | MaxPool_3a_3x3 - conv3 | Conv2d_3b_1x1 - conv4 | Conv2d_4a_3x3 - pool2 | MaxPool_5a_3x3 - mixed_35x35x256a | Mixed_5b - mixed_35x35x288a | Mixed_5c - mixed_35x35x288b | Mixed_5d - mixed_17x17x768a | Mixed_6a - mixed_17x17x768b | Mixed_6b - mixed_17x17x768c | Mixed_6c - mixed_17x17x768d | Mixed_6d - mixed_17x17x768e | Mixed_6e - mixed_8x8x1280a | Mixed_7a - mixed_8x8x2048a | Mixed_7b - mixed_8x8x2048b | Mixed_7c - - Args: - inputs: a tensor of size [batch_size, height, width, channels]. - final_endpoint: specifies the endpoint to construct the network up to. It - can be one of ['Conv2d_1a_3x3', 'Conv2d_2a_3x3', 'Conv2d_2b_3x3', - 'MaxPool_3a_3x3', 'Conv2d_3b_1x1', 'Conv2d_4a_3x3', 'MaxPool_5a_3x3', - 'Mixed_5b', 'Mixed_5c', 'Mixed_5d', 'Mixed_6a', 'Mixed_6b', 'Mixed_6c', - 'Mixed_6d', 'Mixed_6e', 'Mixed_7a', 'Mixed_7b', 'Mixed_7c']. - min_depth: Minimum depth value (number of channels) for all convolution ops. - Enforced when depth_multiplier < 1, and not an active constraint when - depth_multiplier >= 1. - depth_multiplier: Float multiplier for the depth (number of channels) - for all convolution ops. The value must be greater than zero. Typical - usage will be to set this value in (0, 1) to reduce the number of - parameters or computation cost of the model. - scope: Optional variable_scope. - - Returns: - tensor_out: output tensor corresponding to the final_endpoint. - end_points: a set of activations for external use, for example summaries or - losses. - - Raises: - ValueError: if final_endpoint is not set to one of the predefined values, - or depth_multiplier <= 0 - """ - # end_points will collect relevant activations for external use, for example - # summaries or losses. - end_points = {} - - if depth_multiplier <= 0: - raise ValueError('depth_multiplier is not greater than zero.') - depth = lambda d: max(int(d * depth_multiplier), min_depth) - - #Backported to 0.10.0 - #with tf.variable_scope(scope, 'InceptionV3', [inputs]): - with tf.variable_scope(scope or 'InceptionV3'): - with slim.arg_scope([slim.conv2d, slim.max_pool2d, slim.avg_pool2d], - stride=1, padding='VALID'): - # 299 x 299 x 3 - end_point = 'Conv2d_1a_3x3' - net = slim.conv2d(inputs, depth(32), [3, 3], stride=2, scope=end_point) - end_points[end_point] = net - if end_point == final_endpoint: return net, end_points - # 149 x 149 x 32 - end_point = 'Conv2d_2a_3x3' - net = slim.conv2d(net, depth(32), [3, 3], scope=end_point) - end_points[end_point] = net - if end_point == final_endpoint: return net, end_points - # 147 x 147 x 32 - end_point = 'Conv2d_2b_3x3' - net = slim.conv2d(net, depth(64), [3, 3], padding='SAME', scope=end_point) - end_points[end_point] = net - if end_point == final_endpoint: return net, end_points - # 147 x 147 x 64 - end_point = 'MaxPool_3a_3x3' - net = slim.max_pool2d(net, [3, 3], stride=2, scope=end_point) - end_points[end_point] = net - if end_point == final_endpoint: return net, end_points - # 73 x 73 x 64 - end_point = 'Conv2d_3b_1x1' - net = slim.conv2d(net, depth(80), [1, 1], scope=end_point) - end_points[end_point] = net - if end_point == final_endpoint: return net, end_points - # 73 x 73 x 80. - end_point = 'Conv2d_4a_3x3' - net = slim.conv2d(net, depth(192), [3, 3], scope=end_point) - end_points[end_point] = net - if end_point == final_endpoint: return net, end_points - # 71 x 71 x 192. - end_point = 'MaxPool_5a_3x3' - net = slim.max_pool2d(net, [3, 3], stride=2, scope=end_point) - end_points[end_point] = net - if end_point == final_endpoint: return net, end_points - # 35 x 35 x 192. - - # Inception blocks - with slim.arg_scope([slim.conv2d, slim.max_pool2d, slim.avg_pool2d], - stride=1, padding='SAME'): - # mixed: 35 x 35 x 256. - end_point = 'Mixed_5b' - with tf.variable_scope(end_point): - with tf.variable_scope('Branch_0'): - branch_0 = slim.conv2d(net, depth(64), [1, 1], scope='Conv2d_0a_1x1') - with tf.variable_scope('Branch_1'): - branch_1 = slim.conv2d(net, depth(48), [1, 1], scope='Conv2d_0a_1x1') - branch_1 = slim.conv2d(branch_1, depth(64), [5, 5], - scope='Conv2d_0b_5x5') - with tf.variable_scope('Branch_2'): - branch_2 = slim.conv2d(net, depth(64), [1, 1], scope='Conv2d_0a_1x1') - branch_2 = slim.conv2d(branch_2, depth(96), [3, 3], - scope='Conv2d_0b_3x3') - branch_2 = slim.conv2d(branch_2, depth(96), [3, 3], - scope='Conv2d_0c_3x3') - with tf.variable_scope('Branch_3'): - branch_3 = slim.avg_pool2d(net, [3, 3], scope='AvgPool_0a_3x3') - branch_3 = slim.conv2d(branch_3, depth(32), [1, 1], - scope='Conv2d_0b_1x1') - net = tf.concat(3, [branch_0, branch_1, branch_2, branch_3]) - end_points[end_point] = net - if end_point == final_endpoint: return net, end_points - - # mixed_1: 35 x 35 x 288. - end_point = 'Mixed_5c' - with tf.variable_scope(end_point): - with tf.variable_scope('Branch_0'): - branch_0 = slim.conv2d(net, depth(64), [1, 1], scope='Conv2d_0a_1x1') - with tf.variable_scope('Branch_1'): - branch_1 = slim.conv2d(net, depth(48), [1, 1], scope='Conv2d_0b_1x1') - branch_1 = slim.conv2d(branch_1, depth(64), [5, 5], - scope='Conv_1_0c_5x5') - with tf.variable_scope('Branch_2'): - branch_2 = slim.conv2d(net, depth(64), [1, 1], - scope='Conv2d_0a_1x1') - branch_2 = slim.conv2d(branch_2, depth(96), [3, 3], - scope='Conv2d_0b_3x3') - branch_2 = slim.conv2d(branch_2, depth(96), [3, 3], - scope='Conv2d_0c_3x3') - with tf.variable_scope('Branch_3'): - branch_3 = slim.avg_pool2d(net, [3, 3], scope='AvgPool_0a_3x3') - branch_3 = slim.conv2d(branch_3, depth(64), [1, 1], - scope='Conv2d_0b_1x1') - net = tf.concat(3, [branch_0, branch_1, branch_2, branch_3]) - end_points[end_point] = net - if end_point == final_endpoint: return net, end_points - - # mixed_2: 35 x 35 x 288. - end_point = 'Mixed_5d' - with tf.variable_scope(end_point): - with tf.variable_scope('Branch_0'): - branch_0 = slim.conv2d(net, depth(64), [1, 1], scope='Conv2d_0a_1x1') - with tf.variable_scope('Branch_1'): - branch_1 = slim.conv2d(net, depth(48), [1, 1], scope='Conv2d_0a_1x1') - branch_1 = slim.conv2d(branch_1, depth(64), [5, 5], - scope='Conv2d_0b_5x5') - with tf.variable_scope('Branch_2'): - branch_2 = slim.conv2d(net, depth(64), [1, 1], scope='Conv2d_0a_1x1') - branch_2 = slim.conv2d(branch_2, depth(96), [3, 3], - scope='Conv2d_0b_3x3') - branch_2 = slim.conv2d(branch_2, depth(96), [3, 3], - scope='Conv2d_0c_3x3') - with tf.variable_scope('Branch_3'): - branch_3 = slim.avg_pool2d(net, [3, 3], scope='AvgPool_0a_3x3') - branch_3 = slim.conv2d(branch_3, depth(64), [1, 1], - scope='Conv2d_0b_1x1') - net = tf.concat(3, [branch_0, branch_1, branch_2, branch_3]) - end_points[end_point] = net - if end_point == final_endpoint: return net, end_points - - # mixed_3: 17 x 17 x 768. - end_point = 'Mixed_6a' - with tf.variable_scope(end_point): - with tf.variable_scope('Branch_0'): - branch_0 = slim.conv2d(net, depth(384), [3, 3], stride=2, - padding='VALID', scope='Conv2d_1a_1x1') - with tf.variable_scope('Branch_1'): - branch_1 = slim.conv2d(net, depth(64), [1, 1], scope='Conv2d_0a_1x1') - branch_1 = slim.conv2d(branch_1, depth(96), [3, 3], - scope='Conv2d_0b_3x3') - branch_1 = slim.conv2d(branch_1, depth(96), [3, 3], stride=2, - padding='VALID', scope='Conv2d_1a_1x1') - with tf.variable_scope('Branch_2'): - branch_2 = slim.max_pool2d(net, [3, 3], stride=2, padding='VALID', - scope='MaxPool_1a_3x3') - net = tf.concat(3, [branch_0, branch_1, branch_2]) - end_points[end_point] = net - if end_point == final_endpoint: return net, end_points - - # mixed4: 17 x 17 x 768. - end_point = 'Mixed_6b' - with tf.variable_scope(end_point): - with tf.variable_scope('Branch_0'): - branch_0 = slim.conv2d(net, depth(192), [1, 1], scope='Conv2d_0a_1x1') - with tf.variable_scope('Branch_1'): - branch_1 = slim.conv2d(net, depth(128), [1, 1], scope='Conv2d_0a_1x1') - branch_1 = slim.conv2d(branch_1, depth(128), [1, 7], - scope='Conv2d_0b_1x7') - branch_1 = slim.conv2d(branch_1, depth(192), [7, 1], - scope='Conv2d_0c_7x1') - with tf.variable_scope('Branch_2'): - branch_2 = slim.conv2d(net, depth(128), [1, 1], scope='Conv2d_0a_1x1') - branch_2 = slim.conv2d(branch_2, depth(128), [7, 1], - scope='Conv2d_0b_7x1') - branch_2 = slim.conv2d(branch_2, depth(128), [1, 7], - scope='Conv2d_0c_1x7') - branch_2 = slim.conv2d(branch_2, depth(128), [7, 1], - scope='Conv2d_0d_7x1') - branch_2 = slim.conv2d(branch_2, depth(192), [1, 7], - scope='Conv2d_0e_1x7') - with tf.variable_scope('Branch_3'): - branch_3 = slim.avg_pool2d(net, [3, 3], scope='AvgPool_0a_3x3') - branch_3 = slim.conv2d(branch_3, depth(192), [1, 1], - scope='Conv2d_0b_1x1') - net = tf.concat(3, [branch_0, branch_1, branch_2, branch_3]) - end_points[end_point] = net - if end_point == final_endpoint: return net, end_points - - # mixed_5: 17 x 17 x 768. - end_point = 'Mixed_6c' - with tf.variable_scope(end_point): - with tf.variable_scope('Branch_0'): - branch_0 = slim.conv2d(net, depth(192), [1, 1], scope='Conv2d_0a_1x1') - with tf.variable_scope('Branch_1'): - branch_1 = slim.conv2d(net, depth(160), [1, 1], scope='Conv2d_0a_1x1') - branch_1 = slim.conv2d(branch_1, depth(160), [1, 7], - scope='Conv2d_0b_1x7') - branch_1 = slim.conv2d(branch_1, depth(192), [7, 1], - scope='Conv2d_0c_7x1') - with tf.variable_scope('Branch_2'): - branch_2 = slim.conv2d(net, depth(160), [1, 1], scope='Conv2d_0a_1x1') - branch_2 = slim.conv2d(branch_2, depth(160), [7, 1], - scope='Conv2d_0b_7x1') - branch_2 = slim.conv2d(branch_2, depth(160), [1, 7], - scope='Conv2d_0c_1x7') - branch_2 = slim.conv2d(branch_2, depth(160), [7, 1], - scope='Conv2d_0d_7x1') - branch_2 = slim.conv2d(branch_2, depth(192), [1, 7], - scope='Conv2d_0e_1x7') - with tf.variable_scope('Branch_3'): - branch_3 = slim.avg_pool2d(net, [3, 3], scope='AvgPool_0a_3x3') - branch_3 = slim.conv2d(branch_3, depth(192), [1, 1], - scope='Conv2d_0b_1x1') - net = tf.concat(3, [branch_0, branch_1, branch_2, branch_3]) - end_points[end_point] = net - if end_point == final_endpoint: return net, end_points - # mixed_6: 17 x 17 x 768. - end_point = 'Mixed_6d' - with tf.variable_scope(end_point): - with tf.variable_scope('Branch_0'): - branch_0 = slim.conv2d(net, depth(192), [1, 1], scope='Conv2d_0a_1x1') - with tf.variable_scope('Branch_1'): - branch_1 = slim.conv2d(net, depth(160), [1, 1], scope='Conv2d_0a_1x1') - branch_1 = slim.conv2d(branch_1, depth(160), [1, 7], - scope='Conv2d_0b_1x7') - branch_1 = slim.conv2d(branch_1, depth(192), [7, 1], - scope='Conv2d_0c_7x1') - with tf.variable_scope('Branch_2'): - branch_2 = slim.conv2d(net, depth(160), [1, 1], scope='Conv2d_0a_1x1') - branch_2 = slim.conv2d(branch_2, depth(160), [7, 1], - scope='Conv2d_0b_7x1') - branch_2 = slim.conv2d(branch_2, depth(160), [1, 7], - scope='Conv2d_0c_1x7') - branch_2 = slim.conv2d(branch_2, depth(160), [7, 1], - scope='Conv2d_0d_7x1') - branch_2 = slim.conv2d(branch_2, depth(192), [1, 7], - scope='Conv2d_0e_1x7') - with tf.variable_scope('Branch_3'): - branch_3 = slim.avg_pool2d(net, [3, 3], scope='AvgPool_0a_3x3') - branch_3 = slim.conv2d(branch_3, depth(192), [1, 1], - scope='Conv2d_0b_1x1') - net = tf.concat(3, [branch_0, branch_1, branch_2, branch_3]) - end_points[end_point] = net - if end_point == final_endpoint: return net, end_points - - # mixed_7: 17 x 17 x 768. - end_point = 'Mixed_6e' - with tf.variable_scope(end_point): - with tf.variable_scope('Branch_0'): - branch_0 = slim.conv2d(net, depth(192), [1, 1], scope='Conv2d_0a_1x1') - with tf.variable_scope('Branch_1'): - branch_1 = slim.conv2d(net, depth(192), [1, 1], scope='Conv2d_0a_1x1') - branch_1 = slim.conv2d(branch_1, depth(192), [1, 7], - scope='Conv2d_0b_1x7') - branch_1 = slim.conv2d(branch_1, depth(192), [7, 1], - scope='Conv2d_0c_7x1') - with tf.variable_scope('Branch_2'): - branch_2 = slim.conv2d(net, depth(192), [1, 1], scope='Conv2d_0a_1x1') - branch_2 = slim.conv2d(branch_2, depth(192), [7, 1], - scope='Conv2d_0b_7x1') - branch_2 = slim.conv2d(branch_2, depth(192), [1, 7], - scope='Conv2d_0c_1x7') - branch_2 = slim.conv2d(branch_2, depth(192), [7, 1], - scope='Conv2d_0d_7x1') - branch_2 = slim.conv2d(branch_2, depth(192), [1, 7], - scope='Conv2d_0e_1x7') - with tf.variable_scope('Branch_3'): - branch_3 = slim.avg_pool2d(net, [3, 3], scope='AvgPool_0a_3x3') - branch_3 = slim.conv2d(branch_3, depth(192), [1, 1], - scope='Conv2d_0b_1x1') - net = tf.concat(3, [branch_0, branch_1, branch_2, branch_3]) - end_points[end_point] = net - if end_point == final_endpoint: return net, end_points - - # mixed_8: 8 x 8 x 1280. - end_point = 'Mixed_7a' - with tf.variable_scope(end_point): - with tf.variable_scope('Branch_0'): - branch_0 = slim.conv2d(net, depth(192), [1, 1], scope='Conv2d_0a_1x1') - branch_0 = slim.conv2d(branch_0, depth(320), [3, 3], stride=2, - padding='VALID', scope='Conv2d_1a_3x3') - with tf.variable_scope('Branch_1'): - branch_1 = slim.conv2d(net, depth(192), [1, 1], scope='Conv2d_0a_1x1') - branch_1 = slim.conv2d(branch_1, depth(192), [1, 7], - scope='Conv2d_0b_1x7') - branch_1 = slim.conv2d(branch_1, depth(192), [7, 1], - scope='Conv2d_0c_7x1') - branch_1 = slim.conv2d(branch_1, depth(192), [3, 3], stride=2, - padding='VALID', scope='Conv2d_1a_3x3') - with tf.variable_scope('Branch_2'): - branch_2 = slim.max_pool2d(net, [3, 3], stride=2, padding='VALID', - scope='MaxPool_1a_3x3') - net = tf.concat(3, [branch_0, branch_1, branch_2]) - end_points[end_point] = net - if end_point == final_endpoint: return net, end_points - # mixed_9: 8 x 8 x 2048. - end_point = 'Mixed_7b' - with tf.variable_scope(end_point): - with tf.variable_scope('Branch_0'): - branch_0 = slim.conv2d(net, depth(320), [1, 1], scope='Conv2d_0a_1x1') - with tf.variable_scope('Branch_1'): - branch_1 = slim.conv2d(net, depth(384), [1, 1], scope='Conv2d_0a_1x1') - branch_1 = tf.concat(3, [ - slim.conv2d(branch_1, depth(384), [1, 3], scope='Conv2d_0b_1x3'), - slim.conv2d(branch_1, depth(384), [3, 1], scope='Conv2d_0b_3x1')]) - with tf.variable_scope('Branch_2'): - branch_2 = slim.conv2d(net, depth(448), [1, 1], scope='Conv2d_0a_1x1') - branch_2 = slim.conv2d( - branch_2, depth(384), [3, 3], scope='Conv2d_0b_3x3') - branch_2 = tf.concat(3, [ - slim.conv2d(branch_2, depth(384), [1, 3], scope='Conv2d_0c_1x3'), - slim.conv2d(branch_2, depth(384), [3, 1], scope='Conv2d_0d_3x1')]) - with tf.variable_scope('Branch_3'): - branch_3 = slim.avg_pool2d(net, [3, 3], scope='AvgPool_0a_3x3') - branch_3 = slim.conv2d( - branch_3, depth(192), [1, 1], scope='Conv2d_0b_1x1') - net = tf.concat(3, [branch_0, branch_1, branch_2, branch_3]) - end_points[end_point] = net - if end_point == final_endpoint: return net, end_points - - # mixed_10: 8 x 8 x 2048. - end_point = 'Mixed_7c' - with tf.variable_scope(end_point): - with tf.variable_scope('Branch_0'): - branch_0 = slim.conv2d(net, depth(320), [1, 1], scope='Conv2d_0a_1x1') - with tf.variable_scope('Branch_1'): - branch_1 = slim.conv2d(net, depth(384), [1, 1], scope='Conv2d_0a_1x1') - branch_1 = tf.concat(3, [ - slim.conv2d(branch_1, depth(384), [1, 3], scope='Conv2d_0b_1x3'), - slim.conv2d(branch_1, depth(384), [3, 1], scope='Conv2d_0c_3x1')]) - with tf.variable_scope('Branch_2'): - branch_2 = slim.conv2d(net, depth(448), [1, 1], scope='Conv2d_0a_1x1') - branch_2 = slim.conv2d( - branch_2, depth(384), [3, 3], scope='Conv2d_0b_3x3') - branch_2 = tf.concat(3, [ - slim.conv2d(branch_2, depth(384), [1, 3], scope='Conv2d_0c_1x3'), - slim.conv2d(branch_2, depth(384), [3, 1], scope='Conv2d_0d_3x1')]) - with tf.variable_scope('Branch_3'): - branch_3 = slim.avg_pool2d(net, [3, 3], scope='AvgPool_0a_3x3') - branch_3 = slim.conv2d( - branch_3, depth(192), [1, 1], scope='Conv2d_0b_1x1') - net = tf.concat(3, [branch_0, branch_1, branch_2, branch_3]) - end_points[end_point] = net - if end_point == final_endpoint: return net, end_points - raise ValueError('Unknown final endpoint %s' % final_endpoint) - - -def inception_v3(inputs, - num_classes=1000, - is_training=True, - dropout_keep_prob=0.8, - min_depth=16, - depth_multiplier=1.0, - prediction_fn=slim.softmax, - spatial_squeeze=True, - reuse=None, - scope='InceptionV3'): - """Inception model from http://arxiv.org/abs/1512.00567. - - "Rethinking the Inception Architecture for Computer Vision" - - Christian Szegedy, Vincent Vanhoucke, Sergey Ioffe, Jonathon Shlens, - Zbigniew Wojna. - - With the default arguments this method constructs the exact model defined in - the paper. However, one can experiment with variations of the inception_v3 - network by changing arguments dropout_keep_prob, min_depth and - depth_multiplier. - - The default image size used to train this network is 299x299. - - Args: - inputs: a tensor of size [batch_size, height, width, channels]. - num_classes: number of predicted classes. - is_training: whether is training or not. - dropout_keep_prob: the percentage of activation values that are retained. - min_depth: Minimum depth value (number of channels) for all convolution ops. - Enforced when depth_multiplier < 1, and not an active constraint when - depth_multiplier >= 1. - depth_multiplier: Float multiplier for the depth (number of channels) - for all convolution ops. The value must be greater than zero. Typical - usage will be to set this value in (0, 1) to reduce the number of - parameters or computation cost of the model. - prediction_fn: a function to get predictions out of logits. - spatial_squeeze: if True, logits is of shape is [B, C], if false logits is - of shape [B, 1, 1, C], where B is batch_size and C is number of classes. - reuse: whether or not the network and its variables should be reused. To be - able to reuse 'scope' must be given. - scope: Optional variable_scope. - - Returns: - logits: the pre-softmax activations, a tensor of size - [batch_size, num_classes] - end_points: a dictionary from components of the network to the corresponding - activation. - - Raises: - ValueError: if 'depth_multiplier' is less than or equal to zero. - """ - if depth_multiplier <= 0: - raise ValueError('depth_multiplier is not greater than zero.') - depth = lambda d: max(int(d * depth_multiplier), min_depth) - - #Backported to 0.10.0 - #with tf.variable_scope(scope, 'InceptionV3', [inputs, num_classes], - # reuse=reuse) as scope: - with tf.variable_scope(scope or 'InceptionV3', reuse=reuse) as scope: - with slim.arg_scope([slim.batch_norm, slim.dropout], - is_training=is_training): - net, end_points = inception_v3_base( - inputs, scope=scope, min_depth=min_depth, - depth_multiplier=depth_multiplier) - - # Auxiliary Head logits - with slim.arg_scope([slim.conv2d, slim.max_pool2d, slim.avg_pool2d], - stride=1, padding='SAME'): - aux_logits = end_points['Mixed_6e'] - with tf.variable_scope('AuxLogits'): - aux_logits = slim.avg_pool2d( - aux_logits, [5, 5], stride=3, padding='VALID', - scope='AvgPool_1a_5x5') - aux_logits = slim.conv2d(aux_logits, depth(128), [1, 1], - scope='Conv2d_1b_1x1') - - # Shape of feature map before the final layer. - kernel_size = _reduced_kernel_size_for_small_input( - aux_logits, [5, 5]) - aux_logits = slim.conv2d( - aux_logits, depth(768), kernel_size, - weights_initializer=trunc_normal(0.01), - padding='VALID', scope='Conv2d_2a_{}x{}'.format(*kernel_size)) - aux_logits = slim.conv2d( - aux_logits, num_classes, [1, 1], activation_fn=None, - normalizer_fn=None, weights_initializer=trunc_normal(0.001), - scope='Conv2d_2b_1x1') - if spatial_squeeze: - aux_logits = tf.squeeze(aux_logits, [1, 2], name='SpatialSqueeze') - end_points['AuxLogits'] = aux_logits - - # Final pooling and prediction - with tf.variable_scope('Logits'): - kernel_size = _reduced_kernel_size_for_small_input(net, [8, 8]) - net = slim.avg_pool2d(net, kernel_size, padding='VALID', - scope='AvgPool_1a_{}x{}'.format(*kernel_size)) - # 1 x 1 x 2048 - net = slim.dropout(net, keep_prob=dropout_keep_prob, scope='Dropout_1b') - end_points['PreLogits'] = net - # 2048 - logits = slim.conv2d(net, num_classes, [1, 1], activation_fn=None, - normalizer_fn=None, scope='Conv2d_1c_1x1') - if spatial_squeeze: - logits = tf.squeeze(logits, [1, 2], name='SpatialSqueeze') - # 1000 - end_points['Logits'] = logits - end_points['Predictions'] = prediction_fn(logits, scope='Predictions') - return logits, end_points -inception_v3.default_image_size = 299 - - -def _reduced_kernel_size_for_small_input(input_tensor, kernel_size): - """Define kernel size which is automatically reduced for small input. - - If the shape of the input images is unknown at graph construction time this - function assumes that the input images are is large enough. - - Args: - input_tensor: input tensor of size [batch_size, height, width, channels]. - kernel_size: desired kernel size of length 2: [kernel_height, kernel_width] - - Returns: - a tensor with the kernel size. - - TODO(jrru): Make this function work with unknown shapes. Theoretically, this - can be done with the code below. Problems are two-fold: (1) If the shape was - known, it will be lost. (2) inception.slim.ops._two_element_tuple cannot - handle tensors that define the kernel size. - shape = tf.shape(input_tensor) - return = tf.pack([tf.minimum(shape[1], kernel_size[0]), - tf.minimum(shape[2], kernel_size[1])]) - - """ - shape = input_tensor.get_shape().as_list() - if shape[1] is None or shape[2] is None: - kernel_size_out = kernel_size - else: - kernel_size_out = [min(shape[1], kernel_size[0]), - min(shape[2], kernel_size[1])] - return kernel_size_out - - -def inception_v3_arg_scope(weight_decay=0.00004, - stddev=0.1): - """Defines the default InceptionV3 arg scope. - - Args: - weight_decay: The weight decay to use for regularizing the model. - stddev: The standard deviation of the trunctated normal weight initializer. - - Returns: - An `arg_scope` to use for the inception v3 model. - """ - batch_norm_params = { - # Decay for the moving averages. - 'decay': 0.9997, - # epsilon to prevent 0s in variance. - 'epsilon': 0.001, - # collection containing update_ops. - 'updates_collections': tf.GraphKeys.UPDATE_OPS, - } - - # Set weight_decay for weights in Conv and FC layers. - with slim.arg_scope([slim.conv2d, slim.fully_connected], - weights_regularizer=slim.l2_regularizer(weight_decay)): - with slim.arg_scope( - [slim.conv2d], - weights_initializer=tf.truncated_normal_initializer(stddev=stddev), - activation_fn=tf.nn.relu, - normalizer_fn=slim.batch_norm, - normalizer_params=batch_norm_params) as sc: - return sc diff --git a/nets/inception_v3_test.py b/nets/inception_v3_test.py deleted file mode 100644 index c82b265..0000000 --- a/nets/inception_v3_test.py +++ /dev/null @@ -1,292 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for nets.inception_v1.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -import tensorflow as tf - -from nets import inception - -slim = tf.contrib.slim - - -class InceptionV3Test(tf.test.TestCase): - - def testBuildClassificationNetwork(self): - batch_size = 5 - height, width = 299, 299 - num_classes = 1000 - - inputs = tf.random_uniform((batch_size, height, width, 3)) - logits, end_points = inception.inception_v3(inputs, num_classes) - self.assertTrue(logits.op.name.startswith('InceptionV3/Logits')) - self.assertListEqual(logits.get_shape().as_list(), - [batch_size, num_classes]) - self.assertTrue('Predictions' in end_points) - self.assertListEqual(end_points['Predictions'].get_shape().as_list(), - [batch_size, num_classes]) - - def testBuildBaseNetwork(self): - batch_size = 5 - height, width = 299, 299 - - inputs = tf.random_uniform((batch_size, height, width, 3)) - final_endpoint, end_points = inception.inception_v3_base(inputs) - self.assertTrue(final_endpoint.op.name.startswith( - 'InceptionV3/Mixed_7c')) - self.assertListEqual(final_endpoint.get_shape().as_list(), - [batch_size, 8, 8, 2048]) - expected_endpoints = ['Conv2d_1a_3x3', 'Conv2d_2a_3x3', 'Conv2d_2b_3x3', - 'MaxPool_3a_3x3', 'Conv2d_3b_1x1', 'Conv2d_4a_3x3', - 'MaxPool_5a_3x3', 'Mixed_5b', 'Mixed_5c', 'Mixed_5d', - 'Mixed_6a', 'Mixed_6b', 'Mixed_6c', 'Mixed_6d', - 'Mixed_6e', 'Mixed_7a', 'Mixed_7b', 'Mixed_7c'] - self.assertItemsEqual(end_points.keys(), expected_endpoints) - - def testBuildOnlyUptoFinalEndpoint(self): - batch_size = 5 - height, width = 299, 299 - endpoints = ['Conv2d_1a_3x3', 'Conv2d_2a_3x3', 'Conv2d_2b_3x3', - 'MaxPool_3a_3x3', 'Conv2d_3b_1x1', 'Conv2d_4a_3x3', - 'MaxPool_5a_3x3', 'Mixed_5b', 'Mixed_5c', 'Mixed_5d', - 'Mixed_6a', 'Mixed_6b', 'Mixed_6c', 'Mixed_6d', - 'Mixed_6e', 'Mixed_7a', 'Mixed_7b', 'Mixed_7c'] - - for index, endpoint in enumerate(endpoints): - with tf.Graph().as_default(): - inputs = tf.random_uniform((batch_size, height, width, 3)) - out_tensor, end_points = inception.inception_v3_base( - inputs, final_endpoint=endpoint) - self.assertTrue(out_tensor.op.name.startswith( - 'InceptionV3/' + endpoint)) - self.assertItemsEqual(endpoints[:index+1], end_points) - - def testBuildAndCheckAllEndPointsUptoMixed7c(self): - batch_size = 5 - height, width = 299, 299 - - inputs = tf.random_uniform((batch_size, height, width, 3)) - _, end_points = inception.inception_v3_base( - inputs, final_endpoint='Mixed_7c') - endpoints_shapes = {'Conv2d_1a_3x3': [batch_size, 149, 149, 32], - 'Conv2d_2a_3x3': [batch_size, 147, 147, 32], - 'Conv2d_2b_3x3': [batch_size, 147, 147, 64], - 'MaxPool_3a_3x3': [batch_size, 73, 73, 64], - 'Conv2d_3b_1x1': [batch_size, 73, 73, 80], - 'Conv2d_4a_3x3': [batch_size, 71, 71, 192], - 'MaxPool_5a_3x3': [batch_size, 35, 35, 192], - 'Mixed_5b': [batch_size, 35, 35, 256], - 'Mixed_5c': [batch_size, 35, 35, 288], - 'Mixed_5d': [batch_size, 35, 35, 288], - 'Mixed_6a': [batch_size, 17, 17, 768], - 'Mixed_6b': [batch_size, 17, 17, 768], - 'Mixed_6c': [batch_size, 17, 17, 768], - 'Mixed_6d': [batch_size, 17, 17, 768], - 'Mixed_6e': [batch_size, 17, 17, 768], - 'Mixed_7a': [batch_size, 8, 8, 1280], - 'Mixed_7b': [batch_size, 8, 8, 2048], - 'Mixed_7c': [batch_size, 8, 8, 2048]} - self.assertItemsEqual(endpoints_shapes.keys(), end_points.keys()) - for endpoint_name in endpoints_shapes: - expected_shape = endpoints_shapes[endpoint_name] - self.assertTrue(endpoint_name in end_points) - self.assertListEqual(end_points[endpoint_name].get_shape().as_list(), - expected_shape) - - def testModelHasExpectedNumberOfParameters(self): - batch_size = 5 - height, width = 299, 299 - inputs = tf.random_uniform((batch_size, height, width, 3)) - with slim.arg_scope(inception.inception_v3_arg_scope()): - inception.inception_v3_base(inputs) - total_params, _ = slim.model_analyzer.analyze_vars( - slim.get_model_variables()) - self.assertAlmostEqual(21802784, total_params) - - def testBuildEndPoints(self): - batch_size = 5 - height, width = 299, 299 - num_classes = 1000 - - inputs = tf.random_uniform((batch_size, height, width, 3)) - _, end_points = inception.inception_v3(inputs, num_classes) - self.assertTrue('Logits' in end_points) - logits = end_points['Logits'] - self.assertListEqual(logits.get_shape().as_list(), - [batch_size, num_classes]) - self.assertTrue('AuxLogits' in end_points) - aux_logits = end_points['AuxLogits'] - self.assertListEqual(aux_logits.get_shape().as_list(), - [batch_size, num_classes]) - self.assertTrue('Mixed_7c' in end_points) - pre_pool = end_points['Mixed_7c'] - self.assertListEqual(pre_pool.get_shape().as_list(), - [batch_size, 8, 8, 2048]) - self.assertTrue('PreLogits' in end_points) - pre_logits = end_points['PreLogits'] - self.assertListEqual(pre_logits.get_shape().as_list(), - [batch_size, 1, 1, 2048]) - - def testBuildEndPointsWithDepthMultiplierLessThanOne(self): - batch_size = 5 - height, width = 299, 299 - num_classes = 1000 - - inputs = tf.random_uniform((batch_size, height, width, 3)) - _, end_points = inception.inception_v3(inputs, num_classes) - - endpoint_keys = [key for key in end_points.keys() - if key.startswith('Mixed') or key.startswith('Conv')] - - _, end_points_with_multiplier = inception.inception_v3( - inputs, num_classes, scope='depth_multiplied_net', - depth_multiplier=0.5) - - for key in endpoint_keys: - original_depth = end_points[key].get_shape().as_list()[3] - new_depth = end_points_with_multiplier[key].get_shape().as_list()[3] - self.assertEqual(0.5 * original_depth, new_depth) - - def testBuildEndPointsWithDepthMultiplierGreaterThanOne(self): - batch_size = 5 - height, width = 299, 299 - num_classes = 1000 - - inputs = tf.random_uniform((batch_size, height, width, 3)) - _, end_points = inception.inception_v3(inputs, num_classes) - - endpoint_keys = [key for key in end_points.keys() - if key.startswith('Mixed') or key.startswith('Conv')] - - _, end_points_with_multiplier = inception.inception_v3( - inputs, num_classes, scope='depth_multiplied_net', - depth_multiplier=2.0) - - for key in endpoint_keys: - original_depth = end_points[key].get_shape().as_list()[3] - new_depth = end_points_with_multiplier[key].get_shape().as_list()[3] - self.assertEqual(2.0 * original_depth, new_depth) - - def testRaiseValueErrorWithInvalidDepthMultiplier(self): - batch_size = 5 - height, width = 299, 299 - num_classes = 1000 - - inputs = tf.random_uniform((batch_size, height, width, 3)) - with self.assertRaises(ValueError): - _ = inception.inception_v3(inputs, num_classes, depth_multiplier=-0.1) - with self.assertRaises(ValueError): - _ = inception.inception_v3(inputs, num_classes, depth_multiplier=0.0) - - def testHalfSizeImages(self): - batch_size = 5 - height, width = 150, 150 - num_classes = 1000 - - inputs = tf.random_uniform((batch_size, height, width, 3)) - logits, end_points = inception.inception_v3(inputs, num_classes) - self.assertTrue(logits.op.name.startswith('InceptionV3/Logits')) - self.assertListEqual(logits.get_shape().as_list(), - [batch_size, num_classes]) - pre_pool = end_points['Mixed_7c'] - self.assertListEqual(pre_pool.get_shape().as_list(), - [batch_size, 3, 3, 2048]) - - def testUnknownImageShape(self): - tf.reset_default_graph() - batch_size = 2 - height, width = 299, 299 - num_classes = 1000 - input_np = np.random.uniform(0, 1, (batch_size, height, width, 3)) - with self.test_session() as sess: - inputs = tf.placeholder(tf.float32, shape=(batch_size, None, None, 3)) - logits, end_points = inception.inception_v3(inputs, num_classes) - self.assertListEqual(logits.get_shape().as_list(), - [batch_size, num_classes]) - pre_pool = end_points['Mixed_7c'] - feed_dict = {inputs: input_np} - tf.initialize_all_variables().run() - pre_pool_out = sess.run(pre_pool, feed_dict=feed_dict) - self.assertListEqual(list(pre_pool_out.shape), [batch_size, 8, 8, 2048]) - - def testUnknowBatchSize(self): - batch_size = 1 - height, width = 299, 299 - num_classes = 1000 - - inputs = tf.placeholder(tf.float32, (None, height, width, 3)) - logits, _ = inception.inception_v3(inputs, num_classes) - self.assertTrue(logits.op.name.startswith('InceptionV3/Logits')) - self.assertListEqual(logits.get_shape().as_list(), - [None, num_classes]) - images = tf.random_uniform((batch_size, height, width, 3)) - - with self.test_session() as sess: - sess.run(tf.initialize_all_variables()) - output = sess.run(logits, {inputs: images.eval()}) - self.assertEquals(output.shape, (batch_size, num_classes)) - - def testEvaluation(self): - batch_size = 2 - height, width = 299, 299 - num_classes = 1000 - - eval_inputs = tf.random_uniform((batch_size, height, width, 3)) - logits, _ = inception.inception_v3(eval_inputs, num_classes, - is_training=False) - predictions = tf.argmax(logits, 1) - - with self.test_session() as sess: - sess.run(tf.initialize_all_variables()) - output = sess.run(predictions) - self.assertEquals(output.shape, (batch_size,)) - - def testTrainEvalWithReuse(self): - train_batch_size = 5 - eval_batch_size = 2 - height, width = 150, 150 - num_classes = 1000 - - train_inputs = tf.random_uniform((train_batch_size, height, width, 3)) - inception.inception_v3(train_inputs, num_classes) - eval_inputs = tf.random_uniform((eval_batch_size, height, width, 3)) - logits, _ = inception.inception_v3(eval_inputs, num_classes, - is_training=False, reuse=True) - predictions = tf.argmax(logits, 1) - - with self.test_session() as sess: - sess.run(tf.initialize_all_variables()) - output = sess.run(predictions) - self.assertEquals(output.shape, (eval_batch_size,)) - - def testLogitsNotSqueezed(self): - num_classes = 25 - images = tf.random_uniform([1, 299, 299, 3]) - logits, _ = inception.inception_v3(images, - num_classes=num_classes, - spatial_squeeze=False) - - with self.test_session() as sess: - tf.initialize_all_variables().run() - logits_out = sess.run(logits) - self.assertListEqual(list(logits_out.shape), [1, 1, 1, num_classes]) - - -if __name__ == '__main__': - tf.test.main() diff --git a/nets/lenet.py b/nets/lenet.py deleted file mode 100644 index 789d2bd..0000000 --- a/nets/lenet.py +++ /dev/null @@ -1,93 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Contains a variant of the LeNet model definition.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import tensorflow as tf - -slim = tf.contrib.slim - - -def lenet(images, num_classes=10, is_training=False, - dropout_keep_prob=0.5, - prediction_fn=slim.softmax, - scope='LeNet'): - """Creates a variant of the LeNet model. - - Note that since the output is a set of 'logits', the values fall in the - interval of (-infinity, infinity). Consequently, to convert the outputs to a - probability distribution over the characters, one will need to convert them - using the softmax function: - - logits = lenet.lenet(images, is_training=False) - probabilities = tf.nn.softmax(logits) - predictions = tf.argmax(logits, 1) - - Args: - images: A batch of `Tensors` of size [batch_size, height, width, channels]. - num_classes: the number of classes in the dataset. - is_training: specifies whether or not we're currently training the model. - This variable will determine the behaviour of the dropout layer. - dropout_keep_prob: the percentage of activation values that are retained. - prediction_fn: a function to get predictions out of logits. - scope: Optional variable_scope. - - Returns: - logits: the pre-softmax activations, a tensor of size - [batch_size, `num_classes`] - end_points: a dictionary from components of the network to the corresponding - activation. - """ - end_points = {} - - with tf.variable_scope(scope, 'LeNet', [images, num_classes]): - net = slim.conv2d(images, 32, [5, 5], scope='conv1') - net = slim.max_pool2d(net, [2, 2], 2, scope='pool1') - net = slim.conv2d(net, 64, [5, 5], scope='conv2') - net = slim.max_pool2d(net, [2, 2], 2, scope='pool2') - net = slim.flatten(net) - end_points['Flatten'] = net - - net = slim.fully_connected(net, 1024, scope='fc3') - net = slim.dropout(net, dropout_keep_prob, is_training=is_training, - scope='dropout3') - logits = slim.fully_connected(net, num_classes, activation_fn=None, - scope='fc4') - - end_points['Logits'] = logits - end_points['Predictions'] = prediction_fn(logits, scope='Predictions') - - return logits, end_points -lenet.default_image_size = 28 - - -def lenet_arg_scope(weight_decay=0.0): - """Defines the default lenet argument scope. - - Args: - weight_decay: The weight decay to use for regularizing the model. - - Returns: - An `arg_scope` to use for the inception v3 model. - """ - with slim.arg_scope( - [slim.conv2d, slim.fully_connected], - weights_regularizer=slim.l2_regularizer(weight_decay), - weights_initializer=tf.truncated_normal_initializer(stddev=0.1), - activation_fn=tf.nn.relu) as sc: - return sc diff --git a/nets/nets_factory.py b/nets/nets_factory.py deleted file mode 100644 index 5268372..0000000 --- a/nets/nets_factory.py +++ /dev/null @@ -1,107 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Contains a factory for building various models.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -import functools - -import tensorflow as tf - -from nets import alexnet -from nets import cifarnet -from nets import inception -from nets import lenet -from nets import overfeat -from nets import resnet_v1 -from nets import resnet_v2 -from nets import vgg - -slim = tf.contrib.slim - -networks_map = {'alexnet_v2': alexnet.alexnet_v2, - 'cifarnet': cifarnet.cifarnet, - 'overfeat': overfeat.overfeat, - 'vgg_a': vgg.vgg_a, - 'vgg_16': vgg.vgg_16, - 'vgg_19': vgg.vgg_19, - 'inception_v1': inception.inception_v1, - 'inception_v2': inception.inception_v2, - 'inception_v3': inception.inception_v3, - 'inception_resnet_v2': inception.inception_resnet_v2, - 'lenet': lenet.lenet, - 'resnet_v1_50': resnet_v1.resnet_v1_50, - 'resnet_v1_101': resnet_v1.resnet_v1_101, - 'resnet_v1_152': resnet_v1.resnet_v1_152, - 'resnet_v1_200': resnet_v1.resnet_v1_200, - 'resnet_v2_50': resnet_v2.resnet_v2_50, - 'resnet_v2_101': resnet_v2.resnet_v2_101, - 'resnet_v2_152': resnet_v2.resnet_v2_152, - 'resnet_v2_200': resnet_v2.resnet_v2_200, - } - -arg_scopes_map = {'alexnet_v2': alexnet.alexnet_v2_arg_scope, - 'cifarnet': cifarnet.cifarnet_arg_scope, - 'overfeat': overfeat.overfeat_arg_scope, - 'vgg_a': vgg.vgg_arg_scope, - 'vgg_16': vgg.vgg_arg_scope, - 'vgg_19': vgg.vgg_arg_scope, - 'inception_v1': inception.inception_v3_arg_scope, - 'inception_v2': inception.inception_v3_arg_scope, - 'inception_v3': inception.inception_v3_arg_scope, - 'inception_resnet_v2': - inception.inception_resnet_v2_arg_scope, - 'lenet': lenet.lenet_arg_scope, - 'resnet_v1_50': resnet_v1.resnet_arg_scope, - 'resnet_v1_101': resnet_v1.resnet_arg_scope, - 'resnet_v1_152': resnet_v1.resnet_arg_scope, - 'resnet_v1_200': resnet_v1.resnet_arg_scope, - 'resnet_v2_50': resnet_v2.resnet_arg_scope, - 'resnet_v2_101': resnet_v2.resnet_arg_scope, - 'resnet_v2_152': resnet_v2.resnet_arg_scope, - 'resnet_v2_200': resnet_v2.resnet_arg_scope, - } - - -def get_network_fn(name, num_classes, weight_decay=0.0, is_training=False): - """Returns a network_fn such as `logits, end_points = network_fn(images)`. - - Args: - name: The name of the network. - num_classes: The number of classes to use for classification. - weight_decay: The l2 coefficient for the model weights. - is_training: `True` if the model is being used for training and `False` - otherwise. - - Returns: - network_fn: A function that applies the model to a batch of images. It has - the following signature: - logits, end_points = network_fn(images) - Raises: - ValueError: If network `name` is not recognized. - """ - if name not in networks_map: - raise ValueError('Name of network unknown %s' % name) - arg_scope = arg_scopes_map[name](weight_decay=weight_decay) - func = networks_map[name] - @functools.wraps(func) - def network_fn(images): - with slim.arg_scope(arg_scope): - return func(images, num_classes, is_training=is_training) - if hasattr(func, 'default_image_size'): - network_fn.default_image_size = func.default_image_size - - return network_fn diff --git a/nets/nets_factory_test.py b/nets/nets_factory_test.py deleted file mode 100644 index 6ac723b..0000000 --- a/nets/nets_factory_test.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Tests for slim.inception.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - - -import tensorflow as tf - -from nets import nets_factory - - -class NetworksTest(tf.test.TestCase): - - def testGetNetworkFn(self): - batch_size = 5 - num_classes = 1000 - for net in nets_factory.networks_map: - with self.test_session(): - net_fn = nets_factory.get_network_fn(net, num_classes) - # Most networks use 224 as their default_image_size - image_size = getattr(net_fn, 'default_image_size', 224) - inputs = tf.random_uniform((batch_size, image_size, image_size, 3)) - logits, end_points = net_fn(inputs) - self.assertTrue(isinstance(logits, tf.Tensor)) - self.assertTrue(isinstance(end_points, dict)) - self.assertEqual(logits.get_shape().as_list()[0], batch_size) - self.assertEqual(logits.get_shape().as_list()[-1], num_classes) - -if __name__ == '__main__': - tf.test.main() diff --git a/nets/overfeat.py b/nets/overfeat.py deleted file mode 100644 index 8ef566b..0000000 --- a/nets/overfeat.py +++ /dev/null @@ -1,118 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Contains the model definition for the OverFeat network. - -The definition for the network was obtained from: - OverFeat: Integrated Recognition, Localization and Detection using - Convolutional Networks - Pierre Sermanet, David Eigen, Xiang Zhang, Michael Mathieu, Rob Fergus and - Yann LeCun, 2014 - http://arxiv.org/abs/1312.6229 - -Usage: - with slim.arg_scope(overfeat.overfeat_arg_scope()): - outputs, end_points = overfeat.overfeat(inputs) - -@@overfeat -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import tensorflow as tf - -slim = tf.contrib.slim -trunc_normal = lambda stddev: tf.truncated_normal_initializer(0.0, stddev) - - -def overfeat_arg_scope(weight_decay=0.0005): - with slim.arg_scope([slim.conv2d, slim.fully_connected], - activation_fn=tf.nn.relu, - weights_regularizer=slim.l2_regularizer(weight_decay), - biases_initializer=tf.zeros_initializer): - with slim.arg_scope([slim.conv2d], padding='SAME'): - with slim.arg_scope([slim.max_pool2d], padding='VALID') as arg_sc: - return arg_sc - - -def overfeat(inputs, - num_classes=1000, - is_training=True, - dropout_keep_prob=0.5, - spatial_squeeze=True, - scope='overfeat'): - """Contains the model definition for the OverFeat network. - - The definition for the network was obtained from: - OverFeat: Integrated Recognition, Localization and Detection using - Convolutional Networks - Pierre Sermanet, David Eigen, Xiang Zhang, Michael Mathieu, Rob Fergus and - Yann LeCun, 2014 - http://arxiv.org/abs/1312.6229 - - Note: All the fully_connected layers have been transformed to conv2d layers. - To use in classification mode, resize input to 231x231. To use in fully - convolutional mode, set spatial_squeeze to false. - - Args: - inputs: a tensor of size [batch_size, height, width, channels]. - num_classes: number of predicted classes. - is_training: whether or not the model is being trained. - dropout_keep_prob: the probability that activations are kept in the dropout - layers during training. - spatial_squeeze: whether or not should squeeze the spatial dimensions of the - outputs. Useful to remove unnecessary dimensions for classification. - scope: Optional scope for the variables. - - Returns: - the last op containing the log predictions and end_points dict. - - """ - with tf.variable_scope(scope, 'overfeat', [inputs]) as sc: - end_points_collection = sc.name + '_end_points' - # Collect outputs for conv2d, fully_connected and max_pool2d - with slim.arg_scope([slim.conv2d, slim.fully_connected, slim.max_pool2d], - outputs_collections=end_points_collection): - net = slim.conv2d(inputs, 64, [11, 11], 4, padding='VALID', - scope='conv1') - net = slim.max_pool2d(net, [2, 2], scope='pool1') - net = slim.conv2d(net, 256, [5, 5], padding='VALID', scope='conv2') - net = slim.max_pool2d(net, [2, 2], scope='pool2') - net = slim.conv2d(net, 512, [3, 3], scope='conv3') - net = slim.conv2d(net, 1024, [3, 3], scope='conv4') - net = slim.conv2d(net, 1024, [3, 3], scope='conv5') - net = slim.max_pool2d(net, [2, 2], scope='pool5') - with slim.arg_scope([slim.conv2d], - weights_initializer=trunc_normal(0.005), - biases_initializer=tf.constant_initializer(0.1)): - # Use conv2d instead of fully_connected layers. - net = slim.conv2d(net, 3072, [6, 6], padding='VALID', scope='fc6') - net = slim.dropout(net, dropout_keep_prob, is_training=is_training, - scope='dropout6') - net = slim.conv2d(net, 4096, [1, 1], scope='fc7') - net = slim.dropout(net, dropout_keep_prob, is_training=is_training, - scope='dropout7') - net = slim.conv2d(net, num_classes, [1, 1], - activation_fn=None, - normalizer_fn=None, - biases_initializer=tf.zeros_initializer, - scope='fc8') - # Convert end_points_collection into a end_point dict. - end_points = dict(tf.get_collection(end_points_collection)) - if spatial_squeeze: - net = tf.squeeze(net, [1, 2], name='fc8/squeezed') - end_points[sc.name + '/fc8'] = net - return net, end_points -overfeat.default_image_size = 231 diff --git a/nets/overfeat_test.py b/nets/overfeat_test.py deleted file mode 100644 index 71dbc9a..0000000 --- a/nets/overfeat_test.py +++ /dev/null @@ -1,145 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for slim.nets.overfeat.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import tensorflow as tf - -from nets import overfeat - -slim = tf.contrib.slim - - -class OverFeatTest(tf.test.TestCase): - - def testBuild(self): - batch_size = 5 - height, width = 231, 231 - num_classes = 1000 - with self.test_session(): - inputs = tf.random_uniform((batch_size, height, width, 3)) - logits, _ = overfeat.overfeat(inputs, num_classes) - self.assertEquals(logits.op.name, 'overfeat/fc8/squeezed') - self.assertListEqual(logits.get_shape().as_list(), - [batch_size, num_classes]) - - def testFullyConvolutional(self): - batch_size = 1 - height, width = 281, 281 - num_classes = 1000 - with self.test_session(): - inputs = tf.random_uniform((batch_size, height, width, 3)) - logits, _ = overfeat.overfeat(inputs, num_classes, spatial_squeeze=False) - self.assertEquals(logits.op.name, 'overfeat/fc8/BiasAdd') - self.assertListEqual(logits.get_shape().as_list(), - [batch_size, 2, 2, num_classes]) - - def testEndPoints(self): - batch_size = 5 - height, width = 231, 231 - num_classes = 1000 - with self.test_session(): - inputs = tf.random_uniform((batch_size, height, width, 3)) - _, end_points = overfeat.overfeat(inputs, num_classes) - expected_names = ['overfeat/conv1', - 'overfeat/pool1', - 'overfeat/conv2', - 'overfeat/pool2', - 'overfeat/conv3', - 'overfeat/conv4', - 'overfeat/conv5', - 'overfeat/pool5', - 'overfeat/fc6', - 'overfeat/fc7', - 'overfeat/fc8' - ] - self.assertSetEqual(set(end_points.keys()), set(expected_names)) - - def testModelVariables(self): - batch_size = 5 - height, width = 231, 231 - num_classes = 1000 - with self.test_session(): - inputs = tf.random_uniform((batch_size, height, width, 3)) - overfeat.overfeat(inputs, num_classes) - expected_names = ['overfeat/conv1/weights', - 'overfeat/conv1/biases', - 'overfeat/conv2/weights', - 'overfeat/conv2/biases', - 'overfeat/conv3/weights', - 'overfeat/conv3/biases', - 'overfeat/conv4/weights', - 'overfeat/conv4/biases', - 'overfeat/conv5/weights', - 'overfeat/conv5/biases', - 'overfeat/fc6/weights', - 'overfeat/fc6/biases', - 'overfeat/fc7/weights', - 'overfeat/fc7/biases', - 'overfeat/fc8/weights', - 'overfeat/fc8/biases', - ] - model_variables = [v.op.name for v in slim.get_model_variables()] - self.assertSetEqual(set(model_variables), set(expected_names)) - - def testEvaluation(self): - batch_size = 2 - height, width = 231, 231 - num_classes = 1000 - with self.test_session(): - eval_inputs = tf.random_uniform((batch_size, height, width, 3)) - logits, _ = overfeat.overfeat(eval_inputs, is_training=False) - self.assertListEqual(logits.get_shape().as_list(), - [batch_size, num_classes]) - predictions = tf.argmax(logits, 1) - self.assertListEqual(predictions.get_shape().as_list(), [batch_size]) - - def testTrainEvalWithReuse(self): - train_batch_size = 2 - eval_batch_size = 1 - train_height, train_width = 231, 231 - eval_height, eval_width = 281, 281 - num_classes = 1000 - with self.test_session(): - train_inputs = tf.random_uniform( - (train_batch_size, train_height, train_width, 3)) - logits, _ = overfeat.overfeat(train_inputs) - self.assertListEqual(logits.get_shape().as_list(), - [train_batch_size, num_classes]) - tf.get_variable_scope().reuse_variables() - eval_inputs = tf.random_uniform( - (eval_batch_size, eval_height, eval_width, 3)) - logits, _ = overfeat.overfeat(eval_inputs, is_training=False, - spatial_squeeze=False) - self.assertListEqual(logits.get_shape().as_list(), - [eval_batch_size, 2, 2, num_classes]) - logits = tf.reduce_mean(logits, [1, 2]) - predictions = tf.argmax(logits, 1) - self.assertEquals(predictions.get_shape().as_list(), [eval_batch_size]) - - def testForward(self): - batch_size = 1 - height, width = 231, 231 - with self.test_session() as sess: - inputs = tf.random_uniform((batch_size, height, width, 3)) - logits, _ = overfeat.overfeat(inputs) - sess.run(tf.initialize_all_variables()) - output = sess.run(logits) - self.assertTrue(output.any()) - -if __name__ == '__main__': - tf.test.main() diff --git a/nets/resnet_utils.py b/nets/resnet_utils.py deleted file mode 100644 index 1e1dd82..0000000 --- a/nets/resnet_utils.py +++ /dev/null @@ -1,254 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Contains building blocks for various versions of Residual Networks. - -Residual networks (ResNets) were proposed in: - Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun - Deep Residual Learning for Image Recognition. arXiv:1512.03385, 2015 - -More variants were introduced in: - Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun - Identity Mappings in Deep Residual Networks. arXiv: 1603.05027, 2016 - -We can obtain different ResNet variants by changing the network depth, width, -and form of residual unit. This module implements the infrastructure for -building them. Concrete ResNet units and full ResNet networks are implemented in -the accompanying resnet_v1.py and resnet_v2.py modules. - -Compared to https://github.com/KaimingHe/deep-residual-networks, in the current -implementation we subsample the output activations in the last residual unit of -each block, instead of subsampling the input activations in the first residual -unit of each block. The two implementations give identical results but our -implementation is more memory efficient. -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import collections -import tensorflow as tf - -slim = tf.contrib.slim - - -class Block(collections.namedtuple('Block', ['scope', 'unit_fn', 'args'])): - """A named tuple describing a ResNet block. - - Its parts are: - scope: The scope of the `Block`. - unit_fn: The ResNet unit function which takes as input a `Tensor` and - returns another `Tensor` with the output of the ResNet unit. - args: A list of length equal to the number of units in the `Block`. The list - contains one (depth, depth_bottleneck, stride) tuple for each unit in the - block to serve as argument to unit_fn. - """ - - -def subsample(inputs, factor, scope=None): - """Subsamples the input along the spatial dimensions. - - Args: - inputs: A `Tensor` of size [batch, height_in, width_in, channels]. - factor: The subsampling factor. - scope: Optional variable_scope. - - Returns: - output: A `Tensor` of size [batch, height_out, width_out, channels] with the - input, either intact (if factor == 1) or subsampled (if factor > 1). - """ - if factor == 1: - return inputs - else: - return slim.max_pool2d(inputs, [1, 1], stride=factor, scope=scope) - - -def conv2d_same(inputs, num_outputs, kernel_size, stride, rate=1, scope=None): - """Strided 2-D convolution with 'SAME' padding. - - When stride > 1, then we do explicit zero-padding, followed by conv2d with - 'VALID' padding. - - Note that - - net = conv2d_same(inputs, num_outputs, 3, stride=stride) - - is equivalent to - - net = slim.conv2d(inputs, num_outputs, 3, stride=1, padding='SAME') - net = subsample(net, factor=stride) - - whereas - - net = slim.conv2d(inputs, num_outputs, 3, stride=stride, padding='SAME') - - is different when the input's height or width is even, which is why we add the - current function. For more details, see ResnetUtilsTest.testConv2DSameEven(). - - Args: - inputs: A 4-D tensor of size [batch, height_in, width_in, channels]. - num_outputs: An integer, the number of output filters. - kernel_size: An int with the kernel_size of the filters. - stride: An integer, the output stride. - rate: An integer, rate for atrous convolution. - scope: Scope. - - Returns: - output: A 4-D tensor of size [batch, height_out, width_out, channels] with - the convolution output. - """ - if stride == 1: - return slim.conv2d(inputs, num_outputs, kernel_size, stride=1, rate=rate, - padding='SAME', scope=scope) - else: - kernel_size_effective = kernel_size + (kernel_size - 1) * (rate - 1) - pad_total = kernel_size_effective - 1 - pad_beg = pad_total // 2 - pad_end = pad_total - pad_beg - inputs = tf.pad(inputs, - [[0, 0], [pad_beg, pad_end], [pad_beg, pad_end], [0, 0]]) - return slim.conv2d(inputs, num_outputs, kernel_size, stride=stride, - rate=rate, padding='VALID', scope=scope) - - -@slim.add_arg_scope -def stack_blocks_dense(net, blocks, output_stride=None, - outputs_collections=None): - """Stacks ResNet `Blocks` and controls output feature density. - - First, this function creates scopes for the ResNet in the form of - 'block_name/unit_1', 'block_name/unit_2', etc. - - Second, this function allows the user to explicitly control the ResNet - output_stride, which is the ratio of the input to output spatial resolution. - This is useful for dense prediction tasks such as semantic segmentation or - object detection. - - Most ResNets consist of 4 ResNet blocks and subsample the activations by a - factor of 2 when transitioning between consecutive ResNet blocks. This results - to a nominal ResNet output_stride equal to 8. If we set the output_stride to - half the nominal network stride (e.g., output_stride=4), then we compute - responses twice. - - Control of the output feature density is implemented by atrous convolution. - - Args: - net: A `Tensor` of size [batch, height, width, channels]. - blocks: A list of length equal to the number of ResNet `Blocks`. Each - element is a ResNet `Block` object describing the units in the `Block`. - output_stride: If `None`, then the output will be computed at the nominal - network stride. If output_stride is not `None`, it specifies the requested - ratio of input to output spatial resolution, which needs to be equal to - the product of unit strides from the start up to some level of the ResNet. - For example, if the ResNet employs units with strides 1, 2, 1, 3, 4, 1, - then valid values for the output_stride are 1, 2, 6, 24 or None (which - is equivalent to output_stride=24). - outputs_collections: Collection to add the ResNet block outputs. - - Returns: - net: Output tensor with stride equal to the specified output_stride. - - Raises: - ValueError: If the target output_stride is not valid. - """ - # The current_stride variable keeps track of the effective stride of the - # activations. This allows us to invoke atrous convolution whenever applying - # the next residual unit would result in the activations having stride larger - # than the target output_stride. - current_stride = 1 - - # The atrous convolution rate parameter. - rate = 1 - - for block in blocks: - with tf.variable_scope(block.scope, 'block', [net]) as sc: - for i, unit in enumerate(block.args): - if output_stride is not None and current_stride > output_stride: - raise ValueError('The target output_stride cannot be reached.') - - with tf.variable_scope('unit_%d' % (i + 1), values=[net]): - unit_depth, unit_depth_bottleneck, unit_stride = unit - - # If we have reached the target output_stride, then we need to employ - # atrous convolution with stride=1 and multiply the atrous rate by the - # current unit's stride for use in subsequent layers. - if output_stride is not None and current_stride == output_stride: - net = block.unit_fn(net, - depth=unit_depth, - depth_bottleneck=unit_depth_bottleneck, - stride=1, - rate=rate) - rate *= unit_stride - - else: - net = block.unit_fn(net, - depth=unit_depth, - depth_bottleneck=unit_depth_bottleneck, - stride=unit_stride, - rate=1) - current_stride *= unit_stride - net = slim.utils.collect_named_outputs(outputs_collections, sc.name, net) - - if output_stride is not None and current_stride != output_stride: - raise ValueError('The target output_stride cannot be reached.') - - return net - - -def resnet_arg_scope(weight_decay=0.0001, - batch_norm_decay=0.997, - batch_norm_epsilon=1e-5, - batch_norm_scale=True): - """Defines the default ResNet arg scope. - - TODO(gpapan): The batch-normalization related default values above are - appropriate for use in conjunction with the reference ResNet models - released at https://github.com/KaimingHe/deep-residual-networks. When - training ResNets from scratch, they might need to be tuned. - - Args: - weight_decay: The weight decay to use for regularizing the model. - batch_norm_decay: The moving average decay when estimating layer activation - statistics in batch normalization. - batch_norm_epsilon: Small constant to prevent division by zero when - normalizing activations by their variance in batch normalization. - batch_norm_scale: If True, uses an explicit `gamma` multiplier to scale the - activations in the batch normalization layer. - - Returns: - An `arg_scope` to use for the resnet models. - """ - batch_norm_params = { - 'decay': batch_norm_decay, - 'epsilon': batch_norm_epsilon, - 'scale': batch_norm_scale, - 'updates_collections': tf.GraphKeys.UPDATE_OPS, - } - - with slim.arg_scope( - [slim.conv2d], - weights_regularizer=slim.l2_regularizer(weight_decay), - weights_initializer=slim.variance_scaling_initializer(), - activation_fn=tf.nn.relu, - normalizer_fn=slim.batch_norm, - normalizer_params=batch_norm_params): - with slim.arg_scope([slim.batch_norm], **batch_norm_params): - # The following implies padding='SAME' for pool1, which makes feature - # alignment easier for dense prediction tasks. This is also used in - # https://github.com/facebook/fb.resnet.torch. However the accompanying - # code of 'Deep Residual Learning for Image Recognition' uses - # padding='VALID' for pool1. You can switch to that choice by setting - # slim.arg_scope([slim.max_pool2d], padding='VALID'). - with slim.arg_scope([slim.max_pool2d], padding='SAME') as arg_sc: - return arg_sc diff --git a/nets/resnet_v1.py b/nets/resnet_v1.py deleted file mode 100644 index ce7f147..0000000 --- a/nets/resnet_v1.py +++ /dev/null @@ -1,295 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Contains definitions for the original form of Residual Networks. - -The 'v1' residual networks (ResNets) implemented in this module were proposed -by: -[1] Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun - Deep Residual Learning for Image Recognition. arXiv:1512.03385 - -Other variants were introduced in: -[2] Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun - Identity Mappings in Deep Residual Networks. arXiv: 1603.05027 - -The networks defined in this module utilize the bottleneck building block of -[1] with projection shortcuts only for increasing depths. They employ batch -normalization *after* every weight layer. This is the architecture used by -MSRA in the Imagenet and MSCOCO 2016 competition models ResNet-101 and -ResNet-152. See [2; Fig. 1a] for a comparison between the current 'v1' -architecture and the alternative 'v2' architecture of [2] which uses batch -normalization *before* every weight layer in the so-called full pre-activation -units. - -Typical use: - - from tensorflow.contrib.slim.nets import resnet_v1 - -ResNet-101 for image classification into 1000 classes: - - # inputs has shape [batch, 224, 224, 3] - with slim.arg_scope(resnet_v1.resnet_arg_scope()): - net, end_points = resnet_v1.resnet_v1_101(inputs, 1000, is_training=False) - -ResNet-101 for semantic segmentation into 21 classes: - - # inputs has shape [batch, 513, 513, 3] - with slim.arg_scope(resnet_v1.resnet_arg_scope()): - net, end_points = resnet_v1.resnet_v1_101(inputs, - 21, - is_training=False, - global_pool=False, - output_stride=16) -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import tensorflow as tf - -from nets import resnet_utils - - -resnet_arg_scope = resnet_utils.resnet_arg_scope -slim = tf.contrib.slim - - -@slim.add_arg_scope -def bottleneck(inputs, depth, depth_bottleneck, stride, rate=1, - outputs_collections=None, scope=None): - """Bottleneck residual unit variant with BN after convolutions. - - This is the original residual unit proposed in [1]. See Fig. 1(a) of [2] for - its definition. Note that we use here the bottleneck variant which has an - extra bottleneck layer. - - When putting together two consecutive ResNet blocks that use this unit, one - should use stride = 2 in the last unit of the first block. - - Args: - inputs: A tensor of size [batch, height, width, channels]. - depth: The depth of the ResNet unit output. - depth_bottleneck: The depth of the bottleneck layers. - stride: The ResNet unit's stride. Determines the amount of downsampling of - the units output compared to its input. - rate: An integer, rate for atrous convolution. - outputs_collections: Collection to add the ResNet unit output. - scope: Optional variable_scope. - - Returns: - The ResNet unit's output. - """ - with tf.variable_scope(scope, 'bottleneck_v1', [inputs]) as sc: - depth_in = slim.utils.last_dimension(inputs.get_shape(), min_rank=4) - if depth == depth_in: - shortcut = resnet_utils.subsample(inputs, stride, 'shortcut') - else: - shortcut = slim.conv2d(inputs, depth, [1, 1], stride=stride, - activation_fn=None, scope='shortcut') - - residual = slim.conv2d(inputs, depth_bottleneck, [1, 1], stride=1, - scope='conv1') - residual = resnet_utils.conv2d_same(residual, depth_bottleneck, 3, stride, - rate=rate, scope='conv2') - residual = slim.conv2d(residual, depth, [1, 1], stride=1, - activation_fn=None, scope='conv3') - - output = tf.nn.relu(shortcut + residual) - - return slim.utils.collect_named_outputs(outputs_collections, - sc.original_name_scope, - output) - - -def resnet_v1(inputs, - blocks, - num_classes=None, - is_training=True, - global_pool=True, - output_stride=None, - include_root_block=True, - reuse=None, - scope=None): - """Generator for v1 ResNet models. - - This function generates a family of ResNet v1 models. See the resnet_v1_*() - methods for specific model instantiations, obtained by selecting different - block instantiations that produce ResNets of various depths. - - Training for image classification on Imagenet is usually done with [224, 224] - inputs, resulting in [7, 7] feature maps at the output of the last ResNet - block for the ResNets defined in [1] that have nominal stride equal to 32. - However, for dense prediction tasks we advise that one uses inputs with - spatial dimensions that are multiples of 32 plus 1, e.g., [321, 321]. In - this case the feature maps at the ResNet output will have spatial shape - [(height - 1) / output_stride + 1, (width - 1) / output_stride + 1] - and corners exactly aligned with the input image corners, which greatly - facilitates alignment of the features to the image. Using as input [225, 225] - images results in [8, 8] feature maps at the output of the last ResNet block. - - For dense prediction tasks, the ResNet needs to run in fully-convolutional - (FCN) mode and global_pool needs to be set to False. The ResNets in [1, 2] all - have nominal stride equal to 32 and a good choice in FCN mode is to use - output_stride=16 in order to increase the density of the computed features at - small computational and memory overhead, cf. http://arxiv.org/abs/1606.00915. - - Args: - inputs: A tensor of size [batch, height_in, width_in, channels]. - blocks: A list of length equal to the number of ResNet blocks. Each element - is a resnet_utils.Block object describing the units in the block. - num_classes: Number of predicted classes for classification tasks. If None - we return the features before the logit layer. - is_training: whether is training or not. - global_pool: If True, we perform global average pooling before computing the - logits. Set to True for image classification, False for dense prediction. - output_stride: If None, then the output will be computed at the nominal - network stride. If output_stride is not None, it specifies the requested - ratio of input to output spatial resolution. - include_root_block: If True, include the initial convolution followed by - max-pooling, if False excludes it. - reuse: whether or not the network and its variables should be reused. To be - able to reuse 'scope' must be given. - scope: Optional variable_scope. - - Returns: - net: A rank-4 tensor of size [batch, height_out, width_out, channels_out]. - If global_pool is False, then height_out and width_out are reduced by a - factor of output_stride compared to the respective height_in and width_in, - else both height_out and width_out equal one. If num_classes is None, then - net is the output of the last ResNet block, potentially after global - average pooling. If num_classes is not None, net contains the pre-softmax - activations. - end_points: A dictionary from components of the network to the corresponding - activation. - - Raises: - ValueError: If the target output_stride is not valid. - """ - with tf.variable_scope(scope, 'resnet_v1', [inputs], reuse=reuse) as sc: - end_points_collection = sc.name + '_end_points' - with slim.arg_scope([slim.conv2d, bottleneck, - resnet_utils.stack_blocks_dense], - outputs_collections=end_points_collection): - with slim.arg_scope([slim.batch_norm], is_training=is_training): - net = inputs - if include_root_block: - if output_stride is not None: - if output_stride % 4 != 0: - raise ValueError('The output_stride needs to be a multiple of 4.') - output_stride /= 4 - net = resnet_utils.conv2d_same(net, 64, 7, stride=2, scope='conv1') - net = slim.max_pool2d(net, [3, 3], stride=2, scope='pool1') - net = resnet_utils.stack_blocks_dense(net, blocks, output_stride) - if global_pool: - # Global average pooling. - net = tf.reduce_mean(net, [1, 2], name='pool5', keep_dims=True) - if num_classes is not None: - net = slim.conv2d(net, num_classes, [1, 1], activation_fn=None, - normalizer_fn=None, scope='logits') - # Convert end_points_collection into a dictionary of end_points. - end_points = dict(tf.get_collection(end_points_collection)) - if num_classes is not None: - end_points['predictions'] = slim.softmax(net, scope='predictions') - return net, end_points -resnet_v1.default_image_size = 224 - - -def resnet_v1_50(inputs, - num_classes=None, - is_training=True, - global_pool=True, - output_stride=None, - reuse=None, - scope='resnet_v1_50'): - """ResNet-50 model of [1]. See resnet_v1() for arg and return description.""" - blocks = [ - resnet_utils.Block( - 'block1', bottleneck, [(256, 64, 1)] * 2 + [(256, 64, 2)]), - resnet_utils.Block( - 'block2', bottleneck, [(512, 128, 1)] * 3 + [(512, 128, 2)]), - resnet_utils.Block( - 'block3', bottleneck, [(1024, 256, 1)] * 5 + [(1024, 256, 2)]), - resnet_utils.Block( - 'block4', bottleneck, [(2048, 512, 1)] * 3) - ] - return resnet_v1(inputs, blocks, num_classes, is_training, - global_pool=global_pool, output_stride=output_stride, - include_root_block=True, reuse=reuse, scope=scope) - - -def resnet_v1_101(inputs, - num_classes=None, - is_training=True, - global_pool=True, - output_stride=None, - reuse=None, - scope='resnet_v1_101'): - """ResNet-101 model of [1]. See resnet_v1() for arg and return description.""" - blocks = [ - resnet_utils.Block( - 'block1', bottleneck, [(256, 64, 1)] * 2 + [(256, 64, 2)]), - resnet_utils.Block( - 'block2', bottleneck, [(512, 128, 1)] * 3 + [(512, 128, 2)]), - resnet_utils.Block( - 'block3', bottleneck, [(1024, 256, 1)] * 22 + [(1024, 256, 2)]), - resnet_utils.Block( - 'block4', bottleneck, [(2048, 512, 1)] * 3) - ] - return resnet_v1(inputs, blocks, num_classes, is_training, - global_pool=global_pool, output_stride=output_stride, - include_root_block=True, reuse=reuse, scope=scope) - - -def resnet_v1_152(inputs, - num_classes=None, - is_training=True, - global_pool=True, - output_stride=None, - reuse=None, - scope='resnet_v1_152'): - """ResNet-152 model of [1]. See resnet_v1() for arg and return description.""" - blocks = [ - resnet_utils.Block( - 'block1', bottleneck, [(256, 64, 1)] * 2 + [(256, 64, 2)]), - resnet_utils.Block( - 'block2', bottleneck, [(512, 128, 1)] * 7 + [(512, 128, 2)]), - resnet_utils.Block( - 'block3', bottleneck, [(1024, 256, 1)] * 35 + [(1024, 256, 2)]), - resnet_utils.Block( - 'block4', bottleneck, [(2048, 512, 1)] * 3)] - return resnet_v1(inputs, blocks, num_classes, is_training, - global_pool=global_pool, output_stride=output_stride, - include_root_block=True, reuse=reuse, scope=scope) - - -def resnet_v1_200(inputs, - num_classes=None, - is_training=True, - global_pool=True, - output_stride=None, - reuse=None, - scope='resnet_v1_200'): - """ResNet-200 model of [2]. See resnet_v1() for arg and return description.""" - blocks = [ - resnet_utils.Block( - 'block1', bottleneck, [(256, 64, 1)] * 2 + [(256, 64, 2)]), - resnet_utils.Block( - 'block2', bottleneck, [(512, 128, 1)] * 23 + [(512, 128, 2)]), - resnet_utils.Block( - 'block3', bottleneck, [(1024, 256, 1)] * 35 + [(1024, 256, 2)]), - resnet_utils.Block( - 'block4', bottleneck, [(2048, 512, 1)] * 3)] - return resnet_v1(inputs, blocks, num_classes, is_training, - global_pool=global_pool, output_stride=output_stride, - include_root_block=True, reuse=reuse, scope=scope) diff --git a/nets/resnet_v1_test.py b/nets/resnet_v1_test.py deleted file mode 100644 index 6198212..0000000 --- a/nets/resnet_v1_test.py +++ /dev/null @@ -1,450 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for slim.nets.resnet_v1.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -import tensorflow as tf - -from nets import resnet_utils -from nets import resnet_v1 - -slim = tf.contrib.slim - - -def create_test_input(batch_size, height, width, channels): - """Create test input tensor. - - Args: - batch_size: The number of images per batch or `None` if unknown. - height: The height of each image or `None` if unknown. - width: The width of each image or `None` if unknown. - channels: The number of channels per image or `None` if unknown. - - Returns: - Either a placeholder `Tensor` of dimension - [batch_size, height, width, channels] if any of the inputs are `None` or a - constant `Tensor` with the mesh grid values along the spatial dimensions. - """ - if None in [batch_size, height, width, channels]: - return tf.placeholder(tf.float32, (batch_size, height, width, channels)) - else: - return tf.to_float( - np.tile( - np.reshape( - np.reshape(np.arange(height), [height, 1]) + - np.reshape(np.arange(width), [1, width]), - [1, height, width, 1]), - [batch_size, 1, 1, channels])) - - -class ResnetUtilsTest(tf.test.TestCase): - - def testSubsampleThreeByThree(self): - x = tf.reshape(tf.to_float(tf.range(9)), [1, 3, 3, 1]) - x = resnet_utils.subsample(x, 2) - expected = tf.reshape(tf.constant([0, 2, 6, 8]), [1, 2, 2, 1]) - with self.test_session(): - self.assertAllClose(x.eval(), expected.eval()) - - def testSubsampleFourByFour(self): - x = tf.reshape(tf.to_float(tf.range(16)), [1, 4, 4, 1]) - x = resnet_utils.subsample(x, 2) - expected = tf.reshape(tf.constant([0, 2, 8, 10]), [1, 2, 2, 1]) - with self.test_session(): - self.assertAllClose(x.eval(), expected.eval()) - - def testConv2DSameEven(self): - n, n2 = 4, 2 - - # Input image. - x = create_test_input(1, n, n, 1) - - # Convolution kernel. - w = create_test_input(1, 3, 3, 1) - w = tf.reshape(w, [3, 3, 1, 1]) - - tf.get_variable('Conv/weights', initializer=w) - tf.get_variable('Conv/biases', initializer=tf.zeros([1])) - tf.get_variable_scope().reuse_variables() - - y1 = slim.conv2d(x, 1, [3, 3], stride=1, scope='Conv') - y1_expected = tf.to_float([[14, 28, 43, 26], - [28, 48, 66, 37], - [43, 66, 84, 46], - [26, 37, 46, 22]]) - y1_expected = tf.reshape(y1_expected, [1, n, n, 1]) - - y2 = resnet_utils.subsample(y1, 2) - y2_expected = tf.to_float([[14, 43], - [43, 84]]) - y2_expected = tf.reshape(y2_expected, [1, n2, n2, 1]) - - y3 = resnet_utils.conv2d_same(x, 1, 3, stride=2, scope='Conv') - y3_expected = y2_expected - - y4 = slim.conv2d(x, 1, [3, 3], stride=2, scope='Conv') - y4_expected = tf.to_float([[48, 37], - [37, 22]]) - y4_expected = tf.reshape(y4_expected, [1, n2, n2, 1]) - - with self.test_session() as sess: - sess.run(tf.initialize_all_variables()) - self.assertAllClose(y1.eval(), y1_expected.eval()) - self.assertAllClose(y2.eval(), y2_expected.eval()) - self.assertAllClose(y3.eval(), y3_expected.eval()) - self.assertAllClose(y4.eval(), y4_expected.eval()) - - def testConv2DSameOdd(self): - n, n2 = 5, 3 - - # Input image. - x = create_test_input(1, n, n, 1) - - # Convolution kernel. - w = create_test_input(1, 3, 3, 1) - w = tf.reshape(w, [3, 3, 1, 1]) - - tf.get_variable('Conv/weights', initializer=w) - tf.get_variable('Conv/biases', initializer=tf.zeros([1])) - tf.get_variable_scope().reuse_variables() - - y1 = slim.conv2d(x, 1, [3, 3], stride=1, scope='Conv') - y1_expected = tf.to_float([[14, 28, 43, 58, 34], - [28, 48, 66, 84, 46], - [43, 66, 84, 102, 55], - [58, 84, 102, 120, 64], - [34, 46, 55, 64, 30]]) - y1_expected = tf.reshape(y1_expected, [1, n, n, 1]) - - y2 = resnet_utils.subsample(y1, 2) - y2_expected = tf.to_float([[14, 43, 34], - [43, 84, 55], - [34, 55, 30]]) - y2_expected = tf.reshape(y2_expected, [1, n2, n2, 1]) - - y3 = resnet_utils.conv2d_same(x, 1, 3, stride=2, scope='Conv') - y3_expected = y2_expected - - y4 = slim.conv2d(x, 1, [3, 3], stride=2, scope='Conv') - y4_expected = y2_expected - - with self.test_session() as sess: - sess.run(tf.initialize_all_variables()) - self.assertAllClose(y1.eval(), y1_expected.eval()) - self.assertAllClose(y2.eval(), y2_expected.eval()) - self.assertAllClose(y3.eval(), y3_expected.eval()) - self.assertAllClose(y4.eval(), y4_expected.eval()) - - def _resnet_plain(self, inputs, blocks, output_stride=None, scope=None): - """A plain ResNet without extra layers before or after the ResNet blocks.""" - with tf.variable_scope(scope, values=[inputs]): - with slim.arg_scope([slim.conv2d], outputs_collections='end_points'): - net = resnet_utils.stack_blocks_dense(inputs, blocks, output_stride) - end_points = dict(tf.get_collection('end_points')) - return net, end_points - - def testEndPointsV1(self): - """Test the end points of a tiny v1 bottleneck network.""" - bottleneck = resnet_v1.bottleneck - blocks = [resnet_utils.Block('block1', bottleneck, [(4, 1, 1), (4, 1, 2)]), - resnet_utils.Block('block2', bottleneck, [(8, 2, 1), (8, 2, 1)])] - inputs = create_test_input(2, 32, 16, 3) - with slim.arg_scope(resnet_utils.resnet_arg_scope()): - _, end_points = self._resnet_plain(inputs, blocks, scope='tiny') - expected = [ - 'tiny/block1/unit_1/bottleneck_v1/shortcut', - 'tiny/block1/unit_1/bottleneck_v1/conv1', - 'tiny/block1/unit_1/bottleneck_v1/conv2', - 'tiny/block1/unit_1/bottleneck_v1/conv3', - 'tiny/block1/unit_2/bottleneck_v1/conv1', - 'tiny/block1/unit_2/bottleneck_v1/conv2', - 'tiny/block1/unit_2/bottleneck_v1/conv3', - 'tiny/block2/unit_1/bottleneck_v1/shortcut', - 'tiny/block2/unit_1/bottleneck_v1/conv1', - 'tiny/block2/unit_1/bottleneck_v1/conv2', - 'tiny/block2/unit_1/bottleneck_v1/conv3', - 'tiny/block2/unit_2/bottleneck_v1/conv1', - 'tiny/block2/unit_2/bottleneck_v1/conv2', - 'tiny/block2/unit_2/bottleneck_v1/conv3'] - self.assertItemsEqual(expected, end_points) - - def _stack_blocks_nondense(self, net, blocks): - """A simplified ResNet Block stacker without output stride control.""" - for block in blocks: - with tf.variable_scope(block.scope, 'block', [net]): - for i, unit in enumerate(block.args): - depth, depth_bottleneck, stride = unit - with tf.variable_scope('unit_%d' % (i + 1), values=[net]): - net = block.unit_fn(net, - depth=depth, - depth_bottleneck=depth_bottleneck, - stride=stride, - rate=1) - return net - - def _atrousValues(self, bottleneck): - """Verify the values of dense feature extraction by atrous convolution. - - Make sure that dense feature extraction by stack_blocks_dense() followed by - subsampling gives identical results to feature extraction at the nominal - network output stride using the simple self._stack_blocks_nondense() above. - - Args: - bottleneck: The bottleneck function. - """ - blocks = [ - resnet_utils.Block('block1', bottleneck, [(4, 1, 1), (4, 1, 2)]), - resnet_utils.Block('block2', bottleneck, [(8, 2, 1), (8, 2, 2)]), - resnet_utils.Block('block3', bottleneck, [(16, 4, 1), (16, 4, 2)]), - resnet_utils.Block('block4', bottleneck, [(32, 8, 1), (32, 8, 1)]) - ] - nominal_stride = 8 - - # Test both odd and even input dimensions. - height = 30 - width = 31 - with slim.arg_scope(resnet_utils.resnet_arg_scope()): - with slim.arg_scope([slim.batch_norm], is_training=False): - for output_stride in [1, 2, 4, 8, None]: - with tf.Graph().as_default(): - with self.test_session() as sess: - tf.set_random_seed(0) - inputs = create_test_input(1, height, width, 3) - # Dense feature extraction followed by subsampling. - output = resnet_utils.stack_blocks_dense(inputs, - blocks, - output_stride) - if output_stride is None: - factor = 1 - else: - factor = nominal_stride // output_stride - - output = resnet_utils.subsample(output, factor) - # Make the two networks use the same weights. - tf.get_variable_scope().reuse_variables() - # Feature extraction at the nominal network rate. - expected = self._stack_blocks_nondense(inputs, blocks) - sess.run(tf.initialize_all_variables()) - output, expected = sess.run([output, expected]) - self.assertAllClose(output, expected, atol=1e-4, rtol=1e-4) - - def testAtrousValuesBottleneck(self): - self._atrousValues(resnet_v1.bottleneck) - - -class ResnetCompleteNetworkTest(tf.test.TestCase): - """Tests with complete small ResNet v1 networks.""" - - def _resnet_small(self, - inputs, - num_classes=None, - is_training=True, - global_pool=True, - output_stride=None, - include_root_block=True, - reuse=None, - scope='resnet_v1_small'): - """A shallow and thin ResNet v1 for faster tests.""" - bottleneck = resnet_v1.bottleneck - blocks = [ - resnet_utils.Block( - 'block1', bottleneck, [(4, 1, 1)] * 2 + [(4, 1, 2)]), - resnet_utils.Block( - 'block2', bottleneck, [(8, 2, 1)] * 2 + [(8, 2, 2)]), - resnet_utils.Block( - 'block3', bottleneck, [(16, 4, 1)] * 2 + [(16, 4, 2)]), - resnet_utils.Block( - 'block4', bottleneck, [(32, 8, 1)] * 2)] - return resnet_v1.resnet_v1(inputs, blocks, num_classes, - is_training=is_training, - global_pool=global_pool, - output_stride=output_stride, - include_root_block=include_root_block, - reuse=reuse, - scope=scope) - - def testClassificationEndPoints(self): - global_pool = True - num_classes = 10 - inputs = create_test_input(2, 224, 224, 3) - with slim.arg_scope(resnet_utils.resnet_arg_scope()): - logits, end_points = self._resnet_small(inputs, num_classes, - global_pool=global_pool, - scope='resnet') - self.assertTrue(logits.op.name.startswith('resnet/logits')) - self.assertListEqual(logits.get_shape().as_list(), [2, 1, 1, num_classes]) - self.assertTrue('predictions' in end_points) - self.assertListEqual(end_points['predictions'].get_shape().as_list(), - [2, 1, 1, num_classes]) - - def testClassificationShapes(self): - global_pool = True - num_classes = 10 - inputs = create_test_input(2, 224, 224, 3) - with slim.arg_scope(resnet_utils.resnet_arg_scope()): - _, end_points = self._resnet_small(inputs, num_classes, - global_pool=global_pool, - scope='resnet') - endpoint_to_shape = { - 'resnet/block1': [2, 28, 28, 4], - 'resnet/block2': [2, 14, 14, 8], - 'resnet/block3': [2, 7, 7, 16], - 'resnet/block4': [2, 7, 7, 32]} - for endpoint in endpoint_to_shape: - shape = endpoint_to_shape[endpoint] - self.assertListEqual(end_points[endpoint].get_shape().as_list(), shape) - - def testFullyConvolutionalEndpointShapes(self): - global_pool = False - num_classes = 10 - inputs = create_test_input(2, 321, 321, 3) - with slim.arg_scope(resnet_utils.resnet_arg_scope()): - _, end_points = self._resnet_small(inputs, num_classes, - global_pool=global_pool, - scope='resnet') - endpoint_to_shape = { - 'resnet/block1': [2, 41, 41, 4], - 'resnet/block2': [2, 21, 21, 8], - 'resnet/block3': [2, 11, 11, 16], - 'resnet/block4': [2, 11, 11, 32]} - for endpoint in endpoint_to_shape: - shape = endpoint_to_shape[endpoint] - self.assertListEqual(end_points[endpoint].get_shape().as_list(), shape) - - def testRootlessFullyConvolutionalEndpointShapes(self): - global_pool = False - num_classes = 10 - inputs = create_test_input(2, 128, 128, 3) - with slim.arg_scope(resnet_utils.resnet_arg_scope()): - _, end_points = self._resnet_small(inputs, num_classes, - global_pool=global_pool, - include_root_block=False, - scope='resnet') - endpoint_to_shape = { - 'resnet/block1': [2, 64, 64, 4], - 'resnet/block2': [2, 32, 32, 8], - 'resnet/block3': [2, 16, 16, 16], - 'resnet/block4': [2, 16, 16, 32]} - for endpoint in endpoint_to_shape: - shape = endpoint_to_shape[endpoint] - self.assertListEqual(end_points[endpoint].get_shape().as_list(), shape) - - def testAtrousFullyConvolutionalEndpointShapes(self): - global_pool = False - num_classes = 10 - output_stride = 8 - inputs = create_test_input(2, 321, 321, 3) - with slim.arg_scope(resnet_utils.resnet_arg_scope()): - _, end_points = self._resnet_small(inputs, - num_classes, - global_pool=global_pool, - output_stride=output_stride, - scope='resnet') - endpoint_to_shape = { - 'resnet/block1': [2, 41, 41, 4], - 'resnet/block2': [2, 41, 41, 8], - 'resnet/block3': [2, 41, 41, 16], - 'resnet/block4': [2, 41, 41, 32]} - for endpoint in endpoint_to_shape: - shape = endpoint_to_shape[endpoint] - self.assertListEqual(end_points[endpoint].get_shape().as_list(), shape) - - def testAtrousFullyConvolutionalValues(self): - """Verify dense feature extraction with atrous convolution.""" - nominal_stride = 32 - for output_stride in [4, 8, 16, 32, None]: - with slim.arg_scope(resnet_utils.resnet_arg_scope()): - with tf.Graph().as_default(): - with self.test_session() as sess: - tf.set_random_seed(0) - inputs = create_test_input(2, 81, 81, 3) - # Dense feature extraction followed by subsampling. - output, _ = self._resnet_small(inputs, None, is_training=False, - global_pool=False, - output_stride=output_stride) - if output_stride is None: - factor = 1 - else: - factor = nominal_stride // output_stride - output = resnet_utils.subsample(output, factor) - # Make the two networks use the same weights. - tf.get_variable_scope().reuse_variables() - # Feature extraction at the nominal network rate. - expected, _ = self._resnet_small(inputs, None, is_training=False, - global_pool=False) - sess.run(tf.initialize_all_variables()) - self.assertAllClose(output.eval(), expected.eval(), - atol=1e-4, rtol=1e-4) - - def testUnknownBatchSize(self): - batch = 2 - height, width = 65, 65 - global_pool = True - num_classes = 10 - inputs = create_test_input(None, height, width, 3) - with slim.arg_scope(resnet_utils.resnet_arg_scope()): - logits, _ = self._resnet_small(inputs, num_classes, - global_pool=global_pool, - scope='resnet') - self.assertTrue(logits.op.name.startswith('resnet/logits')) - self.assertListEqual(logits.get_shape().as_list(), - [None, 1, 1, num_classes]) - images = create_test_input(batch, height, width, 3) - with self.test_session() as sess: - sess.run(tf.initialize_all_variables()) - output = sess.run(logits, {inputs: images.eval()}) - self.assertEqual(output.shape, (batch, 1, 1, num_classes)) - - def testFullyConvolutionalUnknownHeightWidth(self): - batch = 2 - height, width = 65, 65 - global_pool = False - inputs = create_test_input(batch, None, None, 3) - with slim.arg_scope(resnet_utils.resnet_arg_scope()): - output, _ = self._resnet_small(inputs, None, global_pool=global_pool) - self.assertListEqual(output.get_shape().as_list(), - [batch, None, None, 32]) - images = create_test_input(batch, height, width, 3) - with self.test_session() as sess: - sess.run(tf.initialize_all_variables()) - output = sess.run(output, {inputs: images.eval()}) - self.assertEqual(output.shape, (batch, 3, 3, 32)) - - def testAtrousFullyConvolutionalUnknownHeightWidth(self): - batch = 2 - height, width = 65, 65 - global_pool = False - output_stride = 8 - inputs = create_test_input(batch, None, None, 3) - with slim.arg_scope(resnet_utils.resnet_arg_scope()): - output, _ = self._resnet_small(inputs, - None, - global_pool=global_pool, - output_stride=output_stride) - self.assertListEqual(output.get_shape().as_list(), - [batch, None, None, 32]) - images = create_test_input(batch, height, width, 3) - with self.test_session() as sess: - sess.run(tf.initialize_all_variables()) - output = sess.run(output, {inputs: images.eval()}) - self.assertEqual(output.shape, (batch, 9, 9, 32)) - - -if __name__ == '__main__': - tf.test.main() diff --git a/nets/resnet_v2.py b/nets/resnet_v2.py deleted file mode 100644 index e8b7d30..0000000 --- a/nets/resnet_v2.py +++ /dev/null @@ -1,302 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Contains definitions for the preactivation form of Residual Networks. - -Residual networks (ResNets) were originally proposed in: -[1] Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun - Deep Residual Learning for Image Recognition. arXiv:1512.03385 - -The full preactivation 'v2' ResNet variant implemented in this module was -introduced by: -[2] Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun - Identity Mappings in Deep Residual Networks. arXiv: 1603.05027 - -The key difference of the full preactivation 'v2' variant compared to the -'v1' variant in [1] is the use of batch normalization before every weight layer. -Another difference is that 'v2' ResNets do not include an activation function in -the main pathway. Also see [2; Fig. 4e]. - -Typical use: - - from tensorflow.contrib.slim.nets import resnet_v2 - -ResNet-101 for image classification into 1000 classes: - - # inputs has shape [batch, 224, 224, 3] - with slim.arg_scope(resnet_v2.resnet_arg_scope()): - net, end_points = resnet_v2.resnet_v2_101(inputs, 1000, is_training=False) - -ResNet-101 for semantic segmentation into 21 classes: - - # inputs has shape [batch, 513, 513, 3] - with slim.arg_scope(resnet_v2.resnet_arg_scope(is_training)): - net, end_points = resnet_v2.resnet_v2_101(inputs, - 21, - is_training=False, - global_pool=False, - output_stride=16) -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import tensorflow as tf - -from nets import resnet_utils - -slim = tf.contrib.slim -resnet_arg_scope = resnet_utils.resnet_arg_scope - - -@slim.add_arg_scope -def bottleneck(inputs, depth, depth_bottleneck, stride, rate=1, - outputs_collections=None, scope=None): - """Bottleneck residual unit variant with BN before convolutions. - - This is the full preactivation residual unit variant proposed in [2]. See - Fig. 1(b) of [2] for its definition. Note that we use here the bottleneck - variant which has an extra bottleneck layer. - - When putting together two consecutive ResNet blocks that use this unit, one - should use stride = 2 in the last unit of the first block. - - Args: - inputs: A tensor of size [batch, height, width, channels]. - depth: The depth of the ResNet unit output. - depth_bottleneck: The depth of the bottleneck layers. - stride: The ResNet unit's stride. Determines the amount of downsampling of - the units output compared to its input. - rate: An integer, rate for atrous convolution. - outputs_collections: Collection to add the ResNet unit output. - scope: Optional variable_scope. - - Returns: - The ResNet unit's output. - """ - with tf.variable_scope(scope, 'bottleneck_v2', [inputs]) as sc: - depth_in = slim.utils.last_dimension(inputs.get_shape(), min_rank=4) - preact = slim.batch_norm(inputs, activation_fn=tf.nn.relu, scope='preact') - if depth == depth_in: - shortcut = resnet_utils.subsample(inputs, stride, 'shortcut') - else: - shortcut = slim.conv2d(preact, depth, [1, 1], stride=stride, - normalizer_fn=None, activation_fn=None, - scope='shortcut') - - residual = slim.conv2d(preact, depth_bottleneck, [1, 1], stride=1, - scope='conv1') - residual = resnet_utils.conv2d_same(residual, depth_bottleneck, 3, stride, - rate=rate, scope='conv2') - residual = slim.conv2d(residual, depth, [1, 1], stride=1, - normalizer_fn=None, activation_fn=None, - scope='conv3') - - output = shortcut + residual - - return slim.utils.collect_named_outputs(outputs_collections, - sc.original_name_scope, - output) - - -def resnet_v2(inputs, - blocks, - num_classes=None, - is_training=True, - global_pool=True, - output_stride=None, - include_root_block=True, - reuse=None, - scope=None): - """Generator for v2 (preactivation) ResNet models. - - This function generates a family of ResNet v2 models. See the resnet_v2_*() - methods for specific model instantiations, obtained by selecting different - block instantiations that produce ResNets of various depths. - - Training for image classification on Imagenet is usually done with [224, 224] - inputs, resulting in [7, 7] feature maps at the output of the last ResNet - block for the ResNets defined in [1] that have nominal stride equal to 32. - However, for dense prediction tasks we advise that one uses inputs with - spatial dimensions that are multiples of 32 plus 1, e.g., [321, 321]. In - this case the feature maps at the ResNet output will have spatial shape - [(height - 1) / output_stride + 1, (width - 1) / output_stride + 1] - and corners exactly aligned with the input image corners, which greatly - facilitates alignment of the features to the image. Using as input [225, 225] - images results in [8, 8] feature maps at the output of the last ResNet block. - - For dense prediction tasks, the ResNet needs to run in fully-convolutional - (FCN) mode and global_pool needs to be set to False. The ResNets in [1, 2] all - have nominal stride equal to 32 and a good choice in FCN mode is to use - output_stride=16 in order to increase the density of the computed features at - small computational and memory overhead, cf. http://arxiv.org/abs/1606.00915. - - Args: - inputs: A tensor of size [batch, height_in, width_in, channels]. - blocks: A list of length equal to the number of ResNet blocks. Each element - is a resnet_utils.Block object describing the units in the block. - num_classes: Number of predicted classes for classification tasks. If None - we return the features before the logit layer. - is_training: whether is training or not. - global_pool: If True, we perform global average pooling before computing the - logits. Set to True for image classification, False for dense prediction. - output_stride: If None, then the output will be computed at the nominal - network stride. If output_stride is not None, it specifies the requested - ratio of input to output spatial resolution. - include_root_block: If True, include the initial convolution followed by - max-pooling, if False excludes it. If excluded, `inputs` should be the - results of an activation-less convolution. - reuse: whether or not the network and its variables should be reused. To be - able to reuse 'scope' must be given. - scope: Optional variable_scope. - - - Returns: - net: A rank-4 tensor of size [batch, height_out, width_out, channels_out]. - If global_pool is False, then height_out and width_out are reduced by a - factor of output_stride compared to the respective height_in and width_in, - else both height_out and width_out equal one. If num_classes is None, then - net is the output of the last ResNet block, potentially after global - average pooling. If num_classes is not None, net contains the pre-softmax - activations. - end_points: A dictionary from components of the network to the corresponding - activation. - - Raises: - ValueError: If the target output_stride is not valid. - """ - with tf.variable_scope(scope, 'resnet_v2', [inputs], reuse=reuse) as sc: - end_points_collection = sc.name + '_end_points' - with slim.arg_scope([slim.conv2d, bottleneck, - resnet_utils.stack_blocks_dense], - outputs_collections=end_points_collection): - with slim.arg_scope([slim.batch_norm], is_training=is_training): - net = inputs - if include_root_block: - if output_stride is not None: - if output_stride % 4 != 0: - raise ValueError('The output_stride needs to be a multiple of 4.') - output_stride /= 4 - # We do not include batch normalization or activation functions in - # conv1 because the first ResNet unit will perform these. Cf. - # Appendix of [2]. - with slim.arg_scope([slim.conv2d], - activation_fn=None, normalizer_fn=None): - net = resnet_utils.conv2d_same(net, 64, 7, stride=2, scope='conv1') - net = slim.max_pool2d(net, [3, 3], stride=2, scope='pool1') - net = resnet_utils.stack_blocks_dense(net, blocks, output_stride) - # This is needed because the pre-activation variant does not have batch - # normalization or activation functions in the residual unit output. See - # Appendix of [2]. - net = slim.batch_norm(net, activation_fn=tf.nn.relu, scope='postnorm') - if global_pool: - # Global average pooling. - net = tf.reduce_mean(net, [1, 2], name='pool5', keep_dims=True) - if num_classes is not None: - net = slim.conv2d(net, num_classes, [1, 1], activation_fn=None, - normalizer_fn=None, scope='logits') - # Convert end_points_collection into a dictionary of end_points. - end_points = dict(tf.get_collection(end_points_collection)) - if num_classes is not None: - end_points['predictions'] = slim.softmax(net, scope='predictions') - return net, end_points -resnet_v2.default_image_size = 224 - - -def resnet_v2_50(inputs, - num_classes=None, - is_training=True, - global_pool=True, - output_stride=None, - reuse=None, - scope='resnet_v2_50'): - """ResNet-50 model of [1]. See resnet_v2() for arg and return description.""" - blocks = [ - resnet_utils.Block( - 'block1', bottleneck, [(256, 64, 1)] * 2 + [(256, 64, 2)]), - resnet_utils.Block( - 'block2', bottleneck, [(512, 128, 1)] * 3 + [(512, 128, 2)]), - resnet_utils.Block( - 'block3', bottleneck, [(1024, 256, 1)] * 5 + [(1024, 256, 2)]), - resnet_utils.Block( - 'block4', bottleneck, [(2048, 512, 1)] * 3)] - return resnet_v2(inputs, blocks, num_classes, is_training=is_training, - global_pool=global_pool, output_stride=output_stride, - include_root_block=True, reuse=reuse, scope=scope) - - -def resnet_v2_101(inputs, - num_classes=None, - is_training=True, - global_pool=True, - output_stride=None, - reuse=None, - scope='resnet_v2_101'): - """ResNet-101 model of [1]. See resnet_v2() for arg and return description.""" - blocks = [ - resnet_utils.Block( - 'block1', bottleneck, [(256, 64, 1)] * 2 + [(256, 64, 2)]), - resnet_utils.Block( - 'block2', bottleneck, [(512, 128, 1)] * 3 + [(512, 128, 2)]), - resnet_utils.Block( - 'block3', bottleneck, [(1024, 256, 1)] * 22 + [(1024, 256, 2)]), - resnet_utils.Block( - 'block4', bottleneck, [(2048, 512, 1)] * 3)] - return resnet_v2(inputs, blocks, num_classes, is_training=is_training, - global_pool=global_pool, output_stride=output_stride, - include_root_block=True, reuse=reuse, scope=scope) - - -def resnet_v2_152(inputs, - num_classes=None, - is_training=True, - global_pool=True, - output_stride=None, - reuse=None, - scope='resnet_v2_152'): - """ResNet-152 model of [1]. See resnet_v2() for arg and return description.""" - blocks = [ - resnet_utils.Block( - 'block1', bottleneck, [(256, 64, 1)] * 2 + [(256, 64, 2)]), - resnet_utils.Block( - 'block2', bottleneck, [(512, 128, 1)] * 7 + [(512, 128, 2)]), - resnet_utils.Block( - 'block3', bottleneck, [(1024, 256, 1)] * 35 + [(1024, 256, 2)]), - resnet_utils.Block( - 'block4', bottleneck, [(2048, 512, 1)] * 3)] - return resnet_v2(inputs, blocks, num_classes, is_training=is_training, - global_pool=global_pool, output_stride=output_stride, - include_root_block=True, reuse=reuse, scope=scope) - - -def resnet_v2_200(inputs, - num_classes=None, - is_training=True, - global_pool=True, - output_stride=None, - reuse=None, - scope='resnet_v2_200'): - """ResNet-200 model of [2]. See resnet_v2() for arg and return description.""" - blocks = [ - resnet_utils.Block( - 'block1', bottleneck, [(256, 64, 1)] * 2 + [(256, 64, 2)]), - resnet_utils.Block( - 'block2', bottleneck, [(512, 128, 1)] * 23 + [(512, 128, 2)]), - resnet_utils.Block( - 'block3', bottleneck, [(1024, 256, 1)] * 35 + [(1024, 256, 2)]), - resnet_utils.Block( - 'block4', bottleneck, [(2048, 512, 1)] * 3)] - return resnet_v2(inputs, blocks, num_classes, is_training=is_training, - global_pool=global_pool, output_stride=output_stride, - include_root_block=True, reuse=reuse, scope=scope) diff --git a/nets/resnet_v2_test.py b/nets/resnet_v2_test.py deleted file mode 100644 index 1b4fa75..0000000 --- a/nets/resnet_v2_test.py +++ /dev/null @@ -1,453 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for slim.nets.resnet_v2.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -import tensorflow as tf - -from nets import resnet_utils -from nets import resnet_v2 - -slim = tf.contrib.slim - - -def create_test_input(batch_size, height, width, channels): - """Create test input tensor. - - Args: - batch_size: The number of images per batch or `None` if unknown. - height: The height of each image or `None` if unknown. - width: The width of each image or `None` if unknown. - channels: The number of channels per image or `None` if unknown. - - Returns: - Either a placeholder `Tensor` of dimension - [batch_size, height, width, channels] if any of the inputs are `None` or a - constant `Tensor` with the mesh grid values along the spatial dimensions. - """ - if None in [batch_size, height, width, channels]: - return tf.placeholder(tf.float32, (batch_size, height, width, channels)) - else: - return tf.to_float( - np.tile( - np.reshape( - np.reshape(np.arange(height), [height, 1]) + - np.reshape(np.arange(width), [1, width]), - [1, height, width, 1]), - [batch_size, 1, 1, channels])) - - -class ResnetUtilsTest(tf.test.TestCase): - - def testSubsampleThreeByThree(self): - x = tf.reshape(tf.to_float(tf.range(9)), [1, 3, 3, 1]) - x = resnet_utils.subsample(x, 2) - expected = tf.reshape(tf.constant([0, 2, 6, 8]), [1, 2, 2, 1]) - with self.test_session(): - self.assertAllClose(x.eval(), expected.eval()) - - def testSubsampleFourByFour(self): - x = tf.reshape(tf.to_float(tf.range(16)), [1, 4, 4, 1]) - x = resnet_utils.subsample(x, 2) - expected = tf.reshape(tf.constant([0, 2, 8, 10]), [1, 2, 2, 1]) - with self.test_session(): - self.assertAllClose(x.eval(), expected.eval()) - - def testConv2DSameEven(self): - n, n2 = 4, 2 - - # Input image. - x = create_test_input(1, n, n, 1) - - # Convolution kernel. - w = create_test_input(1, 3, 3, 1) - w = tf.reshape(w, [3, 3, 1, 1]) - - tf.get_variable('Conv/weights', initializer=w) - tf.get_variable('Conv/biases', initializer=tf.zeros([1])) - tf.get_variable_scope().reuse_variables() - - y1 = slim.conv2d(x, 1, [3, 3], stride=1, scope='Conv') - y1_expected = tf.to_float([[14, 28, 43, 26], - [28, 48, 66, 37], - [43, 66, 84, 46], - [26, 37, 46, 22]]) - y1_expected = tf.reshape(y1_expected, [1, n, n, 1]) - - y2 = resnet_utils.subsample(y1, 2) - y2_expected = tf.to_float([[14, 43], - [43, 84]]) - y2_expected = tf.reshape(y2_expected, [1, n2, n2, 1]) - - y3 = resnet_utils.conv2d_same(x, 1, 3, stride=2, scope='Conv') - y3_expected = y2_expected - - y4 = slim.conv2d(x, 1, [3, 3], stride=2, scope='Conv') - y4_expected = tf.to_float([[48, 37], - [37, 22]]) - y4_expected = tf.reshape(y4_expected, [1, n2, n2, 1]) - - with self.test_session() as sess: - sess.run(tf.initialize_all_variables()) - self.assertAllClose(y1.eval(), y1_expected.eval()) - self.assertAllClose(y2.eval(), y2_expected.eval()) - self.assertAllClose(y3.eval(), y3_expected.eval()) - self.assertAllClose(y4.eval(), y4_expected.eval()) - - def testConv2DSameOdd(self): - n, n2 = 5, 3 - - # Input image. - x = create_test_input(1, n, n, 1) - - # Convolution kernel. - w = create_test_input(1, 3, 3, 1) - w = tf.reshape(w, [3, 3, 1, 1]) - - tf.get_variable('Conv/weights', initializer=w) - tf.get_variable('Conv/biases', initializer=tf.zeros([1])) - tf.get_variable_scope().reuse_variables() - - y1 = slim.conv2d(x, 1, [3, 3], stride=1, scope='Conv') - y1_expected = tf.to_float([[14, 28, 43, 58, 34], - [28, 48, 66, 84, 46], - [43, 66, 84, 102, 55], - [58, 84, 102, 120, 64], - [34, 46, 55, 64, 30]]) - y1_expected = tf.reshape(y1_expected, [1, n, n, 1]) - - y2 = resnet_utils.subsample(y1, 2) - y2_expected = tf.to_float([[14, 43, 34], - [43, 84, 55], - [34, 55, 30]]) - y2_expected = tf.reshape(y2_expected, [1, n2, n2, 1]) - - y3 = resnet_utils.conv2d_same(x, 1, 3, stride=2, scope='Conv') - y3_expected = y2_expected - - y4 = slim.conv2d(x, 1, [3, 3], stride=2, scope='Conv') - y4_expected = y2_expected - - with self.test_session() as sess: - sess.run(tf.initialize_all_variables()) - self.assertAllClose(y1.eval(), y1_expected.eval()) - self.assertAllClose(y2.eval(), y2_expected.eval()) - self.assertAllClose(y3.eval(), y3_expected.eval()) - self.assertAllClose(y4.eval(), y4_expected.eval()) - - def _resnet_plain(self, inputs, blocks, output_stride=None, scope=None): - """A plain ResNet without extra layers before or after the ResNet blocks.""" - with tf.variable_scope(scope, values=[inputs]): - with slim.arg_scope([slim.conv2d], outputs_collections='end_points'): - net = resnet_utils.stack_blocks_dense(inputs, blocks, output_stride) - end_points = dict(tf.get_collection('end_points')) - return net, end_points - - def testEndPointsV2(self): - """Test the end points of a tiny v2 bottleneck network.""" - bottleneck = resnet_v2.bottleneck - blocks = [resnet_utils.Block('block1', bottleneck, [(4, 1, 1), (4, 1, 2)]), - resnet_utils.Block('block2', bottleneck, [(8, 2, 1), (8, 2, 1)])] - inputs = create_test_input(2, 32, 16, 3) - with slim.arg_scope(resnet_utils.resnet_arg_scope()): - _, end_points = self._resnet_plain(inputs, blocks, scope='tiny') - expected = [ - 'tiny/block1/unit_1/bottleneck_v2/shortcut', - 'tiny/block1/unit_1/bottleneck_v2/conv1', - 'tiny/block1/unit_1/bottleneck_v2/conv2', - 'tiny/block1/unit_1/bottleneck_v2/conv3', - 'tiny/block1/unit_2/bottleneck_v2/conv1', - 'tiny/block1/unit_2/bottleneck_v2/conv2', - 'tiny/block1/unit_2/bottleneck_v2/conv3', - 'tiny/block2/unit_1/bottleneck_v2/shortcut', - 'tiny/block2/unit_1/bottleneck_v2/conv1', - 'tiny/block2/unit_1/bottleneck_v2/conv2', - 'tiny/block2/unit_1/bottleneck_v2/conv3', - 'tiny/block2/unit_2/bottleneck_v2/conv1', - 'tiny/block2/unit_2/bottleneck_v2/conv2', - 'tiny/block2/unit_2/bottleneck_v2/conv3'] - self.assertItemsEqual(expected, end_points) - - def _stack_blocks_nondense(self, net, blocks): - """A simplified ResNet Block stacker without output stride control.""" - for block in blocks: - with tf.variable_scope(block.scope, 'block', [net]): - for i, unit in enumerate(block.args): - depth, depth_bottleneck, stride = unit - with tf.variable_scope('unit_%d' % (i + 1), values=[net]): - net = block.unit_fn(net, - depth=depth, - depth_bottleneck=depth_bottleneck, - stride=stride, - rate=1) - return net - - def _atrousValues(self, bottleneck): - """Verify the values of dense feature extraction by atrous convolution. - - Make sure that dense feature extraction by stack_blocks_dense() followed by - subsampling gives identical results to feature extraction at the nominal - network output stride using the simple self._stack_blocks_nondense() above. - - Args: - bottleneck: The bottleneck function. - """ - blocks = [ - resnet_utils.Block('block1', bottleneck, [(4, 1, 1), (4, 1, 2)]), - resnet_utils.Block('block2', bottleneck, [(8, 2, 1), (8, 2, 2)]), - resnet_utils.Block('block3', bottleneck, [(16, 4, 1), (16, 4, 2)]), - resnet_utils.Block('block4', bottleneck, [(32, 8, 1), (32, 8, 1)]) - ] - nominal_stride = 8 - - # Test both odd and even input dimensions. - height = 30 - width = 31 - with slim.arg_scope(resnet_utils.resnet_arg_scope()): - with slim.arg_scope([slim.batch_norm], is_training=False): - for output_stride in [1, 2, 4, 8, None]: - with tf.Graph().as_default(): - with self.test_session() as sess: - tf.set_random_seed(0) - inputs = create_test_input(1, height, width, 3) - # Dense feature extraction followed by subsampling. - output = resnet_utils.stack_blocks_dense(inputs, - blocks, - output_stride) - if output_stride is None: - factor = 1 - else: - factor = nominal_stride // output_stride - - output = resnet_utils.subsample(output, factor) - # Make the two networks use the same weights. - tf.get_variable_scope().reuse_variables() - # Feature extraction at the nominal network rate. - expected = self._stack_blocks_nondense(inputs, blocks) - sess.run(tf.initialize_all_variables()) - output, expected = sess.run([output, expected]) - self.assertAllClose(output, expected, atol=1e-4, rtol=1e-4) - - def testAtrousValuesBottleneck(self): - self._atrousValues(resnet_v2.bottleneck) - - -class ResnetCompleteNetworkTest(tf.test.TestCase): - """Tests with complete small ResNet v2 networks.""" - - def _resnet_small(self, - inputs, - num_classes=None, - is_training=True, - global_pool=True, - output_stride=None, - include_root_block=True, - reuse=None, - scope='resnet_v2_small'): - """A shallow and thin ResNet v2 for faster tests.""" - bottleneck = resnet_v2.bottleneck - blocks = [ - resnet_utils.Block( - 'block1', bottleneck, [(4, 1, 1)] * 2 + [(4, 1, 2)]), - resnet_utils.Block( - 'block2', bottleneck, [(8, 2, 1)] * 2 + [(8, 2, 2)]), - resnet_utils.Block( - 'block3', bottleneck, [(16, 4, 1)] * 2 + [(16, 4, 2)]), - resnet_utils.Block( - 'block4', bottleneck, [(32, 8, 1)] * 2)] - return resnet_v2.resnet_v2(inputs, blocks, num_classes, - is_training=is_training, - global_pool=global_pool, - output_stride=output_stride, - include_root_block=include_root_block, - reuse=reuse, - scope=scope) - - def testClassificationEndPoints(self): - global_pool = True - num_classes = 10 - inputs = create_test_input(2, 224, 224, 3) - with slim.arg_scope(resnet_utils.resnet_arg_scope()): - logits, end_points = self._resnet_small(inputs, num_classes, - global_pool=global_pool, - scope='resnet') - self.assertTrue(logits.op.name.startswith('resnet/logits')) - self.assertListEqual(logits.get_shape().as_list(), [2, 1, 1, num_classes]) - self.assertTrue('predictions' in end_points) - self.assertListEqual(end_points['predictions'].get_shape().as_list(), - [2, 1, 1, num_classes]) - - def testClassificationShapes(self): - global_pool = True - num_classes = 10 - inputs = create_test_input(2, 224, 224, 3) - with slim.arg_scope(resnet_utils.resnet_arg_scope()): - _, end_points = self._resnet_small(inputs, num_classes, - global_pool=global_pool, - scope='resnet') - endpoint_to_shape = { - 'resnet/block1': [2, 28, 28, 4], - 'resnet/block2': [2, 14, 14, 8], - 'resnet/block3': [2, 7, 7, 16], - 'resnet/block4': [2, 7, 7, 32]} - for endpoint in endpoint_to_shape: - shape = endpoint_to_shape[endpoint] - self.assertListEqual(end_points[endpoint].get_shape().as_list(), shape) - - def testFullyConvolutionalEndpointShapes(self): - global_pool = False - num_classes = 10 - inputs = create_test_input(2, 321, 321, 3) - with slim.arg_scope(resnet_utils.resnet_arg_scope()): - _, end_points = self._resnet_small(inputs, num_classes, - global_pool=global_pool, - scope='resnet') - endpoint_to_shape = { - 'resnet/block1': [2, 41, 41, 4], - 'resnet/block2': [2, 21, 21, 8], - 'resnet/block3': [2, 11, 11, 16], - 'resnet/block4': [2, 11, 11, 32]} - for endpoint in endpoint_to_shape: - shape = endpoint_to_shape[endpoint] - self.assertListEqual(end_points[endpoint].get_shape().as_list(), shape) - - def testRootlessFullyConvolutionalEndpointShapes(self): - global_pool = False - num_classes = 10 - inputs = create_test_input(2, 128, 128, 3) - with slim.arg_scope(resnet_utils.resnet_arg_scope()): - _, end_points = self._resnet_small(inputs, num_classes, - global_pool=global_pool, - include_root_block=False, - scope='resnet') - endpoint_to_shape = { - 'resnet/block1': [2, 64, 64, 4], - 'resnet/block2': [2, 32, 32, 8], - 'resnet/block3': [2, 16, 16, 16], - 'resnet/block4': [2, 16, 16, 32]} - for endpoint in endpoint_to_shape: - shape = endpoint_to_shape[endpoint] - self.assertListEqual(end_points[endpoint].get_shape().as_list(), shape) - - def testAtrousFullyConvolutionalEndpointShapes(self): - global_pool = False - num_classes = 10 - output_stride = 8 - inputs = create_test_input(2, 321, 321, 3) - with slim.arg_scope(resnet_utils.resnet_arg_scope()): - _, end_points = self._resnet_small(inputs, - num_classes, - global_pool=global_pool, - output_stride=output_stride, - scope='resnet') - endpoint_to_shape = { - 'resnet/block1': [2, 41, 41, 4], - 'resnet/block2': [2, 41, 41, 8], - 'resnet/block3': [2, 41, 41, 16], - 'resnet/block4': [2, 41, 41, 32]} - for endpoint in endpoint_to_shape: - shape = endpoint_to_shape[endpoint] - self.assertListEqual(end_points[endpoint].get_shape().as_list(), shape) - - def testAtrousFullyConvolutionalValues(self): - """Verify dense feature extraction with atrous convolution.""" - nominal_stride = 32 - for output_stride in [4, 8, 16, 32, None]: - with slim.arg_scope(resnet_utils.resnet_arg_scope()): - with tf.Graph().as_default(): - with self.test_session() as sess: - tf.set_random_seed(0) - inputs = create_test_input(2, 81, 81, 3) - # Dense feature extraction followed by subsampling. - output, _ = self._resnet_small(inputs, None, - is_training=False, - global_pool=False, - output_stride=output_stride) - if output_stride is None: - factor = 1 - else: - factor = nominal_stride // output_stride - output = resnet_utils.subsample(output, factor) - # Make the two networks use the same weights. - tf.get_variable_scope().reuse_variables() - # Feature extraction at the nominal network rate. - expected, _ = self._resnet_small(inputs, None, - is_training=False, - global_pool=False) - sess.run(tf.initialize_all_variables()) - self.assertAllClose(output.eval(), expected.eval(), - atol=1e-4, rtol=1e-4) - - def testUnknownBatchSize(self): - batch = 2 - height, width = 65, 65 - global_pool = True - num_classes = 10 - inputs = create_test_input(None, height, width, 3) - with slim.arg_scope(resnet_utils.resnet_arg_scope()): - logits, _ = self._resnet_small(inputs, num_classes, - global_pool=global_pool, - scope='resnet') - self.assertTrue(logits.op.name.startswith('resnet/logits')) - self.assertListEqual(logits.get_shape().as_list(), - [None, 1, 1, num_classes]) - images = create_test_input(batch, height, width, 3) - with self.test_session() as sess: - sess.run(tf.initialize_all_variables()) - output = sess.run(logits, {inputs: images.eval()}) - self.assertEqual(output.shape, (batch, 1, 1, num_classes)) - - def testFullyConvolutionalUnknownHeightWidth(self): - batch = 2 - height, width = 65, 65 - global_pool = False - inputs = create_test_input(batch, None, None, 3) - with slim.arg_scope(resnet_utils.resnet_arg_scope()): - output, _ = self._resnet_small(inputs, None, - global_pool=global_pool) - self.assertListEqual(output.get_shape().as_list(), - [batch, None, None, 32]) - images = create_test_input(batch, height, width, 3) - with self.test_session() as sess: - sess.run(tf.initialize_all_variables()) - output = sess.run(output, {inputs: images.eval()}) - self.assertEqual(output.shape, (batch, 3, 3, 32)) - - def testAtrousFullyConvolutionalUnknownHeightWidth(self): - batch = 2 - height, width = 65, 65 - global_pool = False - output_stride = 8 - inputs = create_test_input(batch, None, None, 3) - with slim.arg_scope(resnet_utils.resnet_arg_scope()): - output, _ = self._resnet_small(inputs, - None, - global_pool=global_pool, - output_stride=output_stride) - self.assertListEqual(output.get_shape().as_list(), - [batch, None, None, 32]) - images = create_test_input(batch, height, width, 3) - with self.test_session() as sess: - sess.run(tf.initialize_all_variables()) - output = sess.run(output, {inputs: images.eval()}) - self.assertEqual(output.shape, (batch, 9, 9, 32)) - - -if __name__ == '__main__': - tf.test.main() diff --git a/nets/vgg.py b/nets/vgg.py deleted file mode 100644 index 8c1821e..0000000 --- a/nets/vgg.py +++ /dev/null @@ -1,244 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Contains model definitions for versions of the Oxford VGG network. - -These model definitions were introduced in the following technical report: - - Very Deep Convolutional Networks For Large-Scale Image Recognition - Karen Simonyan and Andrew Zisserman - arXiv technical report, 2015 - PDF: http://arxiv.org/pdf/1409.1556.pdf - ILSVRC 2014 Slides: http://www.robots.ox.ac.uk/~karen/pdf/ILSVRC_2014.pdf - CC-BY-4.0 - -More information can be obtained from the VGG website: -www.robots.ox.ac.uk/~vgg/research/very_deep/ - -Usage: - with slim.arg_scope(vgg.vgg_arg_scope()): - outputs, end_points = vgg.vgg_a(inputs) - - with slim.arg_scope(vgg.vgg_arg_scope()): - outputs, end_points = vgg.vgg_16(inputs) - -@@vgg_a -@@vgg_16 -@@vgg_19 -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import tensorflow as tf - -slim = tf.contrib.slim - - -def vgg_arg_scope(weight_decay=0.0005): - """Defines the VGG arg scope. - - Args: - weight_decay: The l2 regularization coefficient. - - Returns: - An arg_scope. - """ - with slim.arg_scope([slim.conv2d, slim.fully_connected], - activation_fn=tf.nn.relu, - weights_regularizer=slim.l2_regularizer(weight_decay), - biases_initializer=tf.zeros_initializer): - with slim.arg_scope([slim.conv2d], padding='SAME') as arg_sc: - return arg_sc - - -def vgg_a(inputs, - num_classes=1000, - is_training=True, - dropout_keep_prob=0.5, - spatial_squeeze=True, - scope='vgg_a'): - """Oxford Net VGG 11-Layers version A Example. - - Note: All the fully_connected layers have been transformed to conv2d layers. - To use in classification mode, resize input to 224x224. - - Args: - inputs: a tensor of size [batch_size, height, width, channels]. - num_classes: number of predicted classes. - is_training: whether or not the model is being trained. - dropout_keep_prob: the probability that activations are kept in the dropout - layers during training. - spatial_squeeze: whether or not should squeeze the spatial dimensions of the - outputs. Useful to remove unnecessary dimensions for classification. - scope: Optional scope for the variables. - - Returns: - the last op containing the log predictions and end_points dict. - """ - with tf.variable_scope(scope, 'vgg_a', [inputs]) as sc: - end_points_collection = sc.name + '_end_points' - # Collect outputs for conv2d, fully_connected and max_pool2d. - with slim.arg_scope([slim.conv2d, slim.max_pool2d], - outputs_collections=end_points_collection): - net = slim.repeat(inputs, 1, slim.conv2d, 64, [3, 3], scope='conv1') - net = slim.max_pool2d(net, [2, 2], scope='pool1') - net = slim.repeat(net, 1, slim.conv2d, 128, [3, 3], scope='conv2') - net = slim.max_pool2d(net, [2, 2], scope='pool2') - net = slim.repeat(net, 2, slim.conv2d, 256, [3, 3], scope='conv3') - net = slim.max_pool2d(net, [2, 2], scope='pool3') - net = slim.repeat(net, 2, slim.conv2d, 512, [3, 3], scope='conv4') - net = slim.max_pool2d(net, [2, 2], scope='pool4') - net = slim.repeat(net, 2, slim.conv2d, 512, [3, 3], scope='conv5') - net = slim.max_pool2d(net, [2, 2], scope='pool5') - # Use conv2d instead of fully_connected layers. - net = slim.conv2d(net, 4096, [7, 7], padding='VALID', scope='fc6') - net = slim.dropout(net, dropout_keep_prob, is_training=is_training, - scope='dropout6') - net = slim.conv2d(net, 4096, [1, 1], scope='fc7') - net = slim.dropout(net, dropout_keep_prob, is_training=is_training, - scope='dropout7') - net = slim.conv2d(net, num_classes, [1, 1], - activation_fn=None, - normalizer_fn=None, - scope='fc8') - # Convert end_points_collection into a end_point dict. - end_points = dict(tf.get_collection(end_points_collection)) - if spatial_squeeze: - net = tf.squeeze(net, [1, 2], name='fc8/squeezed') - end_points[sc.name + '/fc8'] = net - return net, end_points -vgg_a.default_image_size = 224 - - -def vgg_16(inputs, - num_classes=1000, - is_training=True, - dropout_keep_prob=0.5, - spatial_squeeze=True, - scope='vgg_16'): - """Oxford Net VGG 16-Layers version D Example. - - Note: All the fully_connected layers have been transformed to conv2d layers. - To use in classification mode, resize input to 224x224. - - Args: - inputs: a tensor of size [batch_size, height, width, channels]. - num_classes: number of predicted classes. - is_training: whether or not the model is being trained. - dropout_keep_prob: the probability that activations are kept in the dropout - layers during training. - spatial_squeeze: whether or not should squeeze the spatial dimensions of the - outputs. Useful to remove unnecessary dimensions for classification. - scope: Optional scope for the variables. - - Returns: - the last op containing the log predictions and end_points dict. - """ - with tf.variable_scope(scope, 'vgg_16', [inputs]) as sc: - end_points_collection = sc.name + '_end_points' - # Collect outputs for conv2d, fully_connected and max_pool2d. - with slim.arg_scope([slim.conv2d, slim.fully_connected, slim.max_pool2d], - outputs_collections=end_points_collection): - net = slim.repeat(inputs, 2, slim.conv2d, 64, [3, 3], scope='conv1') - net = slim.max_pool2d(net, [2, 2], scope='pool1') - net = slim.repeat(net, 2, slim.conv2d, 128, [3, 3], scope='conv2') - net = slim.max_pool2d(net, [2, 2], scope='pool2') - net = slim.repeat(net, 3, slim.conv2d, 256, [3, 3], scope='conv3') - net = slim.max_pool2d(net, [2, 2], scope='pool3') - net = slim.repeat(net, 3, slim.conv2d, 512, [3, 3], scope='conv4') - net = slim.max_pool2d(net, [2, 2], scope='pool4') - net = slim.repeat(net, 3, slim.conv2d, 512, [3, 3], scope='conv5') - net = slim.max_pool2d(net, [2, 2], scope='pool5') - # Use conv2d instead of fully_connected layers. - net = slim.conv2d(net, 4096, [7, 7], padding='VALID', scope='fc6') - net = slim.dropout(net, dropout_keep_prob, is_training=is_training, - scope='dropout6') - net = slim.conv2d(net, 4096, [1, 1], scope='fc7') - net = slim.dropout(net, dropout_keep_prob, is_training=is_training, - scope='dropout7') - net = slim.conv2d(net, num_classes, [1, 1], - activation_fn=None, - normalizer_fn=None, - scope='fc8') - # Convert end_points_collection into a end_point dict. - end_points = dict(tf.get_collection(end_points_collection)) - if spatial_squeeze: - net = tf.squeeze(net, [1, 2], name='fc8/squeezed') - end_points[sc.name + '/fc8'] = net - return net, end_points -vgg_16.default_image_size = 224 - - -def vgg_19(inputs, - num_classes=1000, - is_training=True, - dropout_keep_prob=0.5, - spatial_squeeze=True, - scope='vgg_19'): - """Oxford Net VGG 19-Layers version E Example. - - Note: All the fully_connected layers have been transformed to conv2d layers. - To use in classification mode, resize input to 224x224. - - Args: - inputs: a tensor of size [batch_size, height, width, channels]. - num_classes: number of predicted classes. - is_training: whether or not the model is being trained. - dropout_keep_prob: the probability that activations are kept in the dropout - layers during training. - spatial_squeeze: whether or not should squeeze the spatial dimensions of the - outputs. Useful to remove unnecessary dimensions for classification. - scope: Optional scope for the variables. - - Returns: - the last op containing the log predictions and end_points dict. - """ - with tf.variable_scope(scope, 'vgg_19', [inputs]) as sc: - end_points_collection = sc.name + '_end_points' - # Collect outputs for conv2d, fully_connected and max_pool2d. - with slim.arg_scope([slim.conv2d, slim.fully_connected, slim.max_pool2d], - outputs_collections=end_points_collection): - net = slim.repeat(inputs, 2, slim.conv2d, 64, [3, 3], scope='conv1') - net = slim.max_pool2d(net, [2, 2], scope='pool1') - net = slim.repeat(net, 2, slim.conv2d, 128, [3, 3], scope='conv2') - net = slim.max_pool2d(net, [2, 2], scope='pool2') - net = slim.repeat(net, 4, slim.conv2d, 256, [3, 3], scope='conv3') - net = slim.max_pool2d(net, [2, 2], scope='pool3') - net = slim.repeat(net, 4, slim.conv2d, 512, [3, 3], scope='conv4') - net = slim.max_pool2d(net, [2, 2], scope='pool4') - net = slim.repeat(net, 4, slim.conv2d, 512, [3, 3], scope='conv5') - net = slim.max_pool2d(net, [2, 2], scope='pool5') - # Use conv2d instead of fully_connected layers. - net = slim.conv2d(net, 4096, [7, 7], padding='VALID', scope='fc6') - net = slim.dropout(net, dropout_keep_prob, is_training=is_training, - scope='dropout6') - net = slim.conv2d(net, 4096, [1, 1], scope='fc7') - net = slim.dropout(net, dropout_keep_prob, is_training=is_training, - scope='dropout7') - net = slim.conv2d(net, num_classes, [1, 1], - activation_fn=None, - normalizer_fn=None, - scope='fc8') - # Convert end_points_collection into a end_point dict. - end_points = dict(tf.get_collection(end_points_collection)) - if spatial_squeeze: - net = tf.squeeze(net, [1, 2], name='fc8/squeezed') - end_points[sc.name + '/fc8'] = net - return net, end_points -vgg_19.default_image_size = 224 - -# Alias -vgg_d = vgg_16 -vgg_e = vgg_19 diff --git a/nets/vgg_test.py b/nets/vgg_test.py deleted file mode 100644 index 5927146..0000000 --- a/nets/vgg_test.py +++ /dev/null @@ -1,455 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for slim.nets.vgg.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import tensorflow as tf - -from nets import vgg - -slim = tf.contrib.slim - - -class VGGATest(tf.test.TestCase): - - def testBuild(self): - batch_size = 5 - height, width = 224, 224 - num_classes = 1000 - with self.test_session(): - inputs = tf.random_uniform((batch_size, height, width, 3)) - logits, _ = vgg.vgg_a(inputs, num_classes) - self.assertEquals(logits.op.name, 'vgg_a/fc8/squeezed') - self.assertListEqual(logits.get_shape().as_list(), - [batch_size, num_classes]) - - def testFullyConvolutional(self): - batch_size = 1 - height, width = 256, 256 - num_classes = 1000 - with self.test_session(): - inputs = tf.random_uniform((batch_size, height, width, 3)) - logits, _ = vgg.vgg_a(inputs, num_classes, spatial_squeeze=False) - self.assertEquals(logits.op.name, 'vgg_a/fc8/BiasAdd') - self.assertListEqual(logits.get_shape().as_list(), - [batch_size, 2, 2, num_classes]) - - def testEndPoints(self): - batch_size = 5 - height, width = 224, 224 - num_classes = 1000 - with self.test_session(): - inputs = tf.random_uniform((batch_size, height, width, 3)) - _, end_points = vgg.vgg_a(inputs, num_classes) - expected_names = ['vgg_a/conv1/conv1_1', - 'vgg_a/pool1', - 'vgg_a/conv2/conv2_1', - 'vgg_a/pool2', - 'vgg_a/conv3/conv3_1', - 'vgg_a/conv3/conv3_2', - 'vgg_a/pool3', - 'vgg_a/conv4/conv4_1', - 'vgg_a/conv4/conv4_2', - 'vgg_a/pool4', - 'vgg_a/conv5/conv5_1', - 'vgg_a/conv5/conv5_2', - 'vgg_a/pool5', - 'vgg_a/fc6', - 'vgg_a/fc7', - 'vgg_a/fc8' - ] - self.assertSetEqual(set(end_points.keys()), set(expected_names)) - - def testModelVariables(self): - batch_size = 5 - height, width = 224, 224 - num_classes = 1000 - with self.test_session(): - inputs = tf.random_uniform((batch_size, height, width, 3)) - vgg.vgg_a(inputs, num_classes) - expected_names = ['vgg_a/conv1/conv1_1/weights', - 'vgg_a/conv1/conv1_1/biases', - 'vgg_a/conv2/conv2_1/weights', - 'vgg_a/conv2/conv2_1/biases', - 'vgg_a/conv3/conv3_1/weights', - 'vgg_a/conv3/conv3_1/biases', - 'vgg_a/conv3/conv3_2/weights', - 'vgg_a/conv3/conv3_2/biases', - 'vgg_a/conv4/conv4_1/weights', - 'vgg_a/conv4/conv4_1/biases', - 'vgg_a/conv4/conv4_2/weights', - 'vgg_a/conv4/conv4_2/biases', - 'vgg_a/conv5/conv5_1/weights', - 'vgg_a/conv5/conv5_1/biases', - 'vgg_a/conv5/conv5_2/weights', - 'vgg_a/conv5/conv5_2/biases', - 'vgg_a/fc6/weights', - 'vgg_a/fc6/biases', - 'vgg_a/fc7/weights', - 'vgg_a/fc7/biases', - 'vgg_a/fc8/weights', - 'vgg_a/fc8/biases', - ] - model_variables = [v.op.name for v in slim.get_model_variables()] - self.assertSetEqual(set(model_variables), set(expected_names)) - - def testEvaluation(self): - batch_size = 2 - height, width = 224, 224 - num_classes = 1000 - with self.test_session(): - eval_inputs = tf.random_uniform((batch_size, height, width, 3)) - logits, _ = vgg.vgg_a(eval_inputs, is_training=False) - self.assertListEqual(logits.get_shape().as_list(), - [batch_size, num_classes]) - predictions = tf.argmax(logits, 1) - self.assertListEqual(predictions.get_shape().as_list(), [batch_size]) - - def testTrainEvalWithReuse(self): - train_batch_size = 2 - eval_batch_size = 1 - train_height, train_width = 224, 224 - eval_height, eval_width = 256, 256 - num_classes = 1000 - with self.test_session(): - train_inputs = tf.random_uniform( - (train_batch_size, train_height, train_width, 3)) - logits, _ = vgg.vgg_a(train_inputs) - self.assertListEqual(logits.get_shape().as_list(), - [train_batch_size, num_classes]) - tf.get_variable_scope().reuse_variables() - eval_inputs = tf.random_uniform( - (eval_batch_size, eval_height, eval_width, 3)) - logits, _ = vgg.vgg_a(eval_inputs, is_training=False, - spatial_squeeze=False) - self.assertListEqual(logits.get_shape().as_list(), - [eval_batch_size, 2, 2, num_classes]) - logits = tf.reduce_mean(logits, [1, 2]) - predictions = tf.argmax(logits, 1) - self.assertEquals(predictions.get_shape().as_list(), [eval_batch_size]) - - def testForward(self): - batch_size = 1 - height, width = 224, 224 - with self.test_session() as sess: - inputs = tf.random_uniform((batch_size, height, width, 3)) - logits, _ = vgg.vgg_a(inputs) - sess.run(tf.initialize_all_variables()) - output = sess.run(logits) - self.assertTrue(output.any()) - - -class VGG16Test(tf.test.TestCase): - - def testBuild(self): - batch_size = 5 - height, width = 224, 224 - num_classes = 1000 - with self.test_session(): - inputs = tf.random_uniform((batch_size, height, width, 3)) - logits, _ = vgg.vgg_16(inputs, num_classes) - self.assertEquals(logits.op.name, 'vgg_16/fc8/squeezed') - self.assertListEqual(logits.get_shape().as_list(), - [batch_size, num_classes]) - - def testFullyConvolutional(self): - batch_size = 1 - height, width = 256, 256 - num_classes = 1000 - with self.test_session(): - inputs = tf.random_uniform((batch_size, height, width, 3)) - logits, _ = vgg.vgg_16(inputs, num_classes, spatial_squeeze=False) - self.assertEquals(logits.op.name, 'vgg_16/fc8/BiasAdd') - self.assertListEqual(logits.get_shape().as_list(), - [batch_size, 2, 2, num_classes]) - - def testEndPoints(self): - batch_size = 5 - height, width = 224, 224 - num_classes = 1000 - with self.test_session(): - inputs = tf.random_uniform((batch_size, height, width, 3)) - _, end_points = vgg.vgg_16(inputs, num_classes) - expected_names = ['vgg_16/conv1/conv1_1', - 'vgg_16/conv1/conv1_2', - 'vgg_16/pool1', - 'vgg_16/conv2/conv2_1', - 'vgg_16/conv2/conv2_2', - 'vgg_16/pool2', - 'vgg_16/conv3/conv3_1', - 'vgg_16/conv3/conv3_2', - 'vgg_16/conv3/conv3_3', - 'vgg_16/pool3', - 'vgg_16/conv4/conv4_1', - 'vgg_16/conv4/conv4_2', - 'vgg_16/conv4/conv4_3', - 'vgg_16/pool4', - 'vgg_16/conv5/conv5_1', - 'vgg_16/conv5/conv5_2', - 'vgg_16/conv5/conv5_3', - 'vgg_16/pool5', - 'vgg_16/fc6', - 'vgg_16/fc7', - 'vgg_16/fc8' - ] - self.assertSetEqual(set(end_points.keys()), set(expected_names)) - - def testModelVariables(self): - batch_size = 5 - height, width = 224, 224 - num_classes = 1000 - with self.test_session(): - inputs = tf.random_uniform((batch_size, height, width, 3)) - vgg.vgg_16(inputs, num_classes) - expected_names = ['vgg_16/conv1/conv1_1/weights', - 'vgg_16/conv1/conv1_1/biases', - 'vgg_16/conv1/conv1_2/weights', - 'vgg_16/conv1/conv1_2/biases', - 'vgg_16/conv2/conv2_1/weights', - 'vgg_16/conv2/conv2_1/biases', - 'vgg_16/conv2/conv2_2/weights', - 'vgg_16/conv2/conv2_2/biases', - 'vgg_16/conv3/conv3_1/weights', - 'vgg_16/conv3/conv3_1/biases', - 'vgg_16/conv3/conv3_2/weights', - 'vgg_16/conv3/conv3_2/biases', - 'vgg_16/conv3/conv3_3/weights', - 'vgg_16/conv3/conv3_3/biases', - 'vgg_16/conv4/conv4_1/weights', - 'vgg_16/conv4/conv4_1/biases', - 'vgg_16/conv4/conv4_2/weights', - 'vgg_16/conv4/conv4_2/biases', - 'vgg_16/conv4/conv4_3/weights', - 'vgg_16/conv4/conv4_3/biases', - 'vgg_16/conv5/conv5_1/weights', - 'vgg_16/conv5/conv5_1/biases', - 'vgg_16/conv5/conv5_2/weights', - 'vgg_16/conv5/conv5_2/biases', - 'vgg_16/conv5/conv5_3/weights', - 'vgg_16/conv5/conv5_3/biases', - 'vgg_16/fc6/weights', - 'vgg_16/fc6/biases', - 'vgg_16/fc7/weights', - 'vgg_16/fc7/biases', - 'vgg_16/fc8/weights', - 'vgg_16/fc8/biases', - ] - model_variables = [v.op.name for v in slim.get_model_variables()] - self.assertSetEqual(set(model_variables), set(expected_names)) - - def testEvaluation(self): - batch_size = 2 - height, width = 224, 224 - num_classes = 1000 - with self.test_session(): - eval_inputs = tf.random_uniform((batch_size, height, width, 3)) - logits, _ = vgg.vgg_16(eval_inputs, is_training=False) - self.assertListEqual(logits.get_shape().as_list(), - [batch_size, num_classes]) - predictions = tf.argmax(logits, 1) - self.assertListEqual(predictions.get_shape().as_list(), [batch_size]) - - def testTrainEvalWithReuse(self): - train_batch_size = 2 - eval_batch_size = 1 - train_height, train_width = 224, 224 - eval_height, eval_width = 256, 256 - num_classes = 1000 - with self.test_session(): - train_inputs = tf.random_uniform( - (train_batch_size, train_height, train_width, 3)) - logits, _ = vgg.vgg_16(train_inputs) - self.assertListEqual(logits.get_shape().as_list(), - [train_batch_size, num_classes]) - tf.get_variable_scope().reuse_variables() - eval_inputs = tf.random_uniform( - (eval_batch_size, eval_height, eval_width, 3)) - logits, _ = vgg.vgg_16(eval_inputs, is_training=False, - spatial_squeeze=False) - self.assertListEqual(logits.get_shape().as_list(), - [eval_batch_size, 2, 2, num_classes]) - logits = tf.reduce_mean(logits, [1, 2]) - predictions = tf.argmax(logits, 1) - self.assertEquals(predictions.get_shape().as_list(), [eval_batch_size]) - - def testForward(self): - batch_size = 1 - height, width = 224, 224 - with self.test_session() as sess: - inputs = tf.random_uniform((batch_size, height, width, 3)) - logits, _ = vgg.vgg_16(inputs) - sess.run(tf.initialize_all_variables()) - output = sess.run(logits) - self.assertTrue(output.any()) - - -class VGG19Test(tf.test.TestCase): - - def testBuild(self): - batch_size = 5 - height, width = 224, 224 - num_classes = 1000 - with self.test_session(): - inputs = tf.random_uniform((batch_size, height, width, 3)) - logits, _ = vgg.vgg_19(inputs, num_classes) - self.assertEquals(logits.op.name, 'vgg_19/fc8/squeezed') - self.assertListEqual(logits.get_shape().as_list(), - [batch_size, num_classes]) - - def testFullyConvolutional(self): - batch_size = 1 - height, width = 256, 256 - num_classes = 1000 - with self.test_session(): - inputs = tf.random_uniform((batch_size, height, width, 3)) - logits, _ = vgg.vgg_19(inputs, num_classes, spatial_squeeze=False) - self.assertEquals(logits.op.name, 'vgg_19/fc8/BiasAdd') - self.assertListEqual(logits.get_shape().as_list(), - [batch_size, 2, 2, num_classes]) - - def testEndPoints(self): - batch_size = 5 - height, width = 224, 224 - num_classes = 1000 - with self.test_session(): - inputs = tf.random_uniform((batch_size, height, width, 3)) - _, end_points = vgg.vgg_19(inputs, num_classes) - expected_names = [ - 'vgg_19/conv1/conv1_1', - 'vgg_19/conv1/conv1_2', - 'vgg_19/pool1', - 'vgg_19/conv2/conv2_1', - 'vgg_19/conv2/conv2_2', - 'vgg_19/pool2', - 'vgg_19/conv3/conv3_1', - 'vgg_19/conv3/conv3_2', - 'vgg_19/conv3/conv3_3', - 'vgg_19/conv3/conv3_4', - 'vgg_19/pool3', - 'vgg_19/conv4/conv4_1', - 'vgg_19/conv4/conv4_2', - 'vgg_19/conv4/conv4_3', - 'vgg_19/conv4/conv4_4', - 'vgg_19/pool4', - 'vgg_19/conv5/conv5_1', - 'vgg_19/conv5/conv5_2', - 'vgg_19/conv5/conv5_3', - 'vgg_19/conv5/conv5_4', - 'vgg_19/pool5', - 'vgg_19/fc6', - 'vgg_19/fc7', - 'vgg_19/fc8' - ] - self.assertSetEqual(set(end_points.keys()), set(expected_names)) - - def testModelVariables(self): - batch_size = 5 - height, width = 224, 224 - num_classes = 1000 - with self.test_session(): - inputs = tf.random_uniform((batch_size, height, width, 3)) - vgg.vgg_19(inputs, num_classes) - expected_names = [ - 'vgg_19/conv1/conv1_1/weights', - 'vgg_19/conv1/conv1_1/biases', - 'vgg_19/conv1/conv1_2/weights', - 'vgg_19/conv1/conv1_2/biases', - 'vgg_19/conv2/conv2_1/weights', - 'vgg_19/conv2/conv2_1/biases', - 'vgg_19/conv2/conv2_2/weights', - 'vgg_19/conv2/conv2_2/biases', - 'vgg_19/conv3/conv3_1/weights', - 'vgg_19/conv3/conv3_1/biases', - 'vgg_19/conv3/conv3_2/weights', - 'vgg_19/conv3/conv3_2/biases', - 'vgg_19/conv3/conv3_3/weights', - 'vgg_19/conv3/conv3_3/biases', - 'vgg_19/conv3/conv3_4/weights', - 'vgg_19/conv3/conv3_4/biases', - 'vgg_19/conv4/conv4_1/weights', - 'vgg_19/conv4/conv4_1/biases', - 'vgg_19/conv4/conv4_2/weights', - 'vgg_19/conv4/conv4_2/biases', - 'vgg_19/conv4/conv4_3/weights', - 'vgg_19/conv4/conv4_3/biases', - 'vgg_19/conv4/conv4_4/weights', - 'vgg_19/conv4/conv4_4/biases', - 'vgg_19/conv5/conv5_1/weights', - 'vgg_19/conv5/conv5_1/biases', - 'vgg_19/conv5/conv5_2/weights', - 'vgg_19/conv5/conv5_2/biases', - 'vgg_19/conv5/conv5_3/weights', - 'vgg_19/conv5/conv5_3/biases', - 'vgg_19/conv5/conv5_4/weights', - 'vgg_19/conv5/conv5_4/biases', - 'vgg_19/fc6/weights', - 'vgg_19/fc6/biases', - 'vgg_19/fc7/weights', - 'vgg_19/fc7/biases', - 'vgg_19/fc8/weights', - 'vgg_19/fc8/biases', - ] - model_variables = [v.op.name for v in slim.get_model_variables()] - self.assertSetEqual(set(model_variables), set(expected_names)) - - def testEvaluation(self): - batch_size = 2 - height, width = 224, 224 - num_classes = 1000 - with self.test_session(): - eval_inputs = tf.random_uniform((batch_size, height, width, 3)) - logits, _ = vgg.vgg_19(eval_inputs, is_training=False) - self.assertListEqual(logits.get_shape().as_list(), - [batch_size, num_classes]) - predictions = tf.argmax(logits, 1) - self.assertListEqual(predictions.get_shape().as_list(), [batch_size]) - - def testTrainEvalWithReuse(self): - train_batch_size = 2 - eval_batch_size = 1 - train_height, train_width = 224, 224 - eval_height, eval_width = 256, 256 - num_classes = 1000 - with self.test_session(): - train_inputs = tf.random_uniform( - (train_batch_size, train_height, train_width, 3)) - logits, _ = vgg.vgg_19(train_inputs) - self.assertListEqual(logits.get_shape().as_list(), - [train_batch_size, num_classes]) - tf.get_variable_scope().reuse_variables() - eval_inputs = tf.random_uniform( - (eval_batch_size, eval_height, eval_width, 3)) - logits, _ = vgg.vgg_19(eval_inputs, is_training=False, - spatial_squeeze=False) - self.assertListEqual(logits.get_shape().as_list(), - [eval_batch_size, 2, 2, num_classes]) - logits = tf.reduce_mean(logits, [1, 2]) - predictions = tf.argmax(logits, 1) - self.assertEquals(predictions.get_shape().as_list(), [eval_batch_size]) - - def testForward(self): - batch_size = 1 - height, width = 224, 224 - with self.test_session() as sess: - inputs = tf.random_uniform((batch_size, height, width, 3)) - logits, _ = vgg.vgg_19(inputs) - sess.run(tf.initialize_all_variables()) - output = sess.run(logits) - self.assertTrue(output.any()) - -if __name__ == '__main__': - tf.test.main() diff --git a/requirements.txt b/requirements.txt index 6f1841f..f3576b0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -30,12 +30,12 @@ psutil==5.0.0 scikit-learn==0.18.1 scipy==0.18.1 sympy==1.0 -tensorflow==0.12.1 +tensorflow==1.0.0 # Optional: OpenAI gym is only needed for the Reinforcement Learning chapter. # There are a few dependencies you need to install first, check out: # https://github.com/openai/gym#installing-everything -#gym[all]==0.5.4 +gym[all]==0.5.4 # If you only want to install the Atari dependency, uncomment this line instead: #gym[atari]==0.5.4