schuetoliver
/
handson-ml
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add notebooks for chapters 5 to 14

main
Aurélien Geron 2016-09-27 23:31:21 +02:00
parent 68fb1971d7
commit d7d6c121e3
30 changed files with 9741 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
.gitignore vendored
View File

@ -1,5 +1,10 @@
.ipynb_checkpoints *.bak
.DS_Store *.ckpt
my_*
images/**/*.png
*.pyc *.pyc
.DS_Store
.ipynb_checkpoints
checkpoint
logs/*
tf_logs/*
images/**/*.png
my_*

2
fundamentals.ipynb → 01_the_machine_learning_landscape.ipynb
View File

@ -4,7 +4,7 @@
"cell_type": "markdown", "cell_type": "markdown",
"metadata": {}, "metadata": {},
"source": [ "source": [
"**Chapter 1 – Fundamentals of Machine Learning**\n", "**Chapter 1 – The Machine Learning landscape**\n",
"\n", "\n",
"_This is the code used to generate some of the figures in chapter 1._" "_This is the code used to generate some of the figures in chapter 1._"
] ]

2
end_to_end_project.ipynb → 02_end_to_end_machine_learning_project.ipynb
View File

@ -4,7 +4,7 @@
"cell_type": "markdown", "cell_type": "markdown",
"metadata": {}, "metadata": {},
"source": [ "source": [
"**Chapter 2 – End to end Machine Learning project**\n", "**Chapter 2 – End-to-end Machine Learning project**\n",
"\n", "\n",
"*Welcome to Machine Learning Housing Corp.! Your task is to predict median house values in Californian districts, given a number of features from these districts.*\n", "*Welcome to Machine Learning Housing Corp.! Your task is to predict median house values in Californian districts, given a number of features from these districts.*\n",
"\n", "\n",

0
training_linear_models.ipynb → 04_training_linear_models.ipynb
View File

1248
05_support_vector_machines.ipynb Normal file
View File

File diff suppressed because it is too large Load Diff

506
06_decision_trees.ipynb Normal file
View File

@ -0,0 +1,506 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"**Chapter 6 – Decision Trees**"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"_This notebook contains all the sample code and solutions to the exercices in chapter 6._"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Setup"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"First, let's make sure this notebook works well in both python 2 and 3, import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures:"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"# To support both python 2 and python 3\n",
"from __future__ import division, print_function, unicode_literals\n",
"\n",
"# Common imports\n",
"import numpy as np\n",
"import numpy.random as rnd\n",
"import os\n",
"\n",
"# to make this notebook's output stable across runs\n",
"rnd.seed(42)\n",
"\n",
"# To plot pretty figures\n",
"%matplotlib inline\n",
"import matplotlib\n",
"import matplotlib.pyplot as plt\n",
"plt.rcParams['axes.labelsize'] = 14\n",
"plt.rcParams['xtick.labelsize'] = 12\n",
"plt.rcParams['ytick.labelsize'] = 12\n",
"\n",
"# Where to save the figures\n",
"PROJECT_ROOT_DIR = \".\"\n",
"CHAPTER_ID = \"decision_trees\"\n",
"\n",
"def image_path(fig_id):\n",
" return os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID, fig_id)\n",
"\n",
"def save_fig(fig_id, tight_layout=True):\n",
" print(\"Saving figure\", fig_id)\n",
" if tight_layout:\n",
" plt.tight_layout()\n",
" plt.savefig(image_path(fig_id) + \".png\", format='png', dpi=300)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Training and visualizing"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"from sklearn.datasets import load_iris\n",
"from sklearn.tree import DecisionTreeClassifier, export_graphviz\n",
"\n",
"iris = load_iris()\n",
"X = iris.data[:, 2:] # petal length and width\n",
"y = iris.target\n",
"\n",
"tree_clf = DecisionTreeClassifier(max_depth=2, random_state=42)\n",
"tree_clf.fit(X, y)"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"export_graphviz(\n",
" tree_clf,\n",
" out_file=image_path(\"iris_tree.dot\"),\n",
" feature_names=iris.feature_names[2:],\n",
" class_names=iris.target_names,\n",
" rounded=True,\n",
" filled=True\n",
" )"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"from matplotlib.colors import ListedColormap\n",
"\n",
"def plot_decision_boundary(clf, X, y, axes=[0, 7.5, 0, 3], iris=True, legend=False, plot_training=True):\n",
" x1s = np.linspace(axes[0], axes[1], 100)\n",
" x2s = np.linspace(axes[2], axes[3], 100)\n",
" x1, x2 = np.meshgrid(x1s, x2s)\n",
" X_new = np.c_[x1.ravel(), x2.ravel()]\n",
" y_pred = clf.predict(X_new).reshape(x1.shape)\n",
" custom_cmap = ListedColormap(['#fafab0','#9898ff','#a0faa0'])\n",
" plt.contourf(x1, x2, y_pred, alpha=0.3, cmap=custom_cmap, linewidth=10)\n",
" if not iris:\n",
" custom_cmap2 = ListedColormap(['#7d7d58','#4c4c7f','#507d50'])\n",
" plt.contour(x1, x2, y_pred, cmap=custom_cmap2, alpha=0.8)\n",
" if plot_training:\n",
" plt.plot(X[:, 0][y==0], X[:, 1][y==0], \"yo\", label=\"Iris-Setosa\")\n",
" plt.plot(X[:, 0][y==1], X[:, 1][y==1], \"bs\", label=\"Iris-Versicolour\")\n",
" plt.plot(X[:, 0][y==2], X[:, 1][y==2], \"g^\", label=\"Iris-Virginica\")\n",
" plt.axis(axes)\n",
" if iris:\n",
" plt.xlabel(\"Petal length\", fontsize=14)\n",
" plt.ylabel(\"Petal width\", fontsize=14)\n",
" else:\n",
" plt.xlabel(r\"$x_1$\", fontsize=18)\n",
" plt.ylabel(r\"$x_2$\", fontsize=18, rotation=0)\n",
" if legend:\n",
" plt.legend(loc=\"lower right\", fontsize=14)\n",
"\n",
"plt.figure(figsize=(8, 4))\n",
"plot_decision_boundary(tree_clf, X, y)\n",
"plt.plot([2.45, 2.45], [0, 3], \"k-\", linewidth=2)\n",
"plt.plot([2.45, 7.5], [1.75, 1.75], \"k--\", linewidth=2)\n",
"plt.plot([4.95, 4.95], [0, 1.75], \"k:\", linewidth=2)\n",
"plt.plot([4.85, 4.85], [1.75, 3], \"k:\", linewidth=2)\n",
"plt.text(1.40, 1.0, \"Depth=0\", fontsize=15)\n",
"plt.text(3.2, 1.80, \"Depth=1\", fontsize=13)\n",
"plt.text(4.05, 0.5, \"(Depth=2)\", fontsize=11)\n",
"\n",
"save_fig(\"decision_tree_decision_boundaries_plot\")\n",
"plt.show()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Predicting classes and class probabilities"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"tree_clf.predict_proba([[5, 1.5]])"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"tree_clf.predict([[5, 1.5]])"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Sensitivity to training set details"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"X[(X[:, 1]==X[:, 1][y==1].max()) & (y==1)] # widest Iris-Versicolour flower"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"not_widest_versicolour = (X[:, 1]!=1.8) | (y==2)\n",
"X_tweaked = X[not_widest_versicolour]\n",
"y_tweaked = y[not_widest_versicolour]\n",
"\n",
"tree_clf_tweaked = DecisionTreeClassifier(max_depth=2, random_state=40)\n",
"tree_clf_tweaked.fit(X_tweaked, y_tweaked)"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"plt.figure(figsize=(8, 4))\n",
"plot_decision_boundary(tree_clf_tweaked, X_tweaked, y_tweaked, legend=False)\n",
"plt.plot([0, 7.5], [0.8, 0.8], \"k-\", linewidth=2)\n",
"plt.plot([0, 7.5], [1.75, 1.75], \"k--\", linewidth=2)\n",
"plt.text(1.0, 0.9, \"Depth=0\", fontsize=15)\n",
"plt.text(1.0, 1.80, \"Depth=1\", fontsize=13)\n",
"\n",
"save_fig(\"decision_tree_instability_plot\")\n",
"plt.show()"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"from sklearn.datasets import make_moons\n",
"Xm, ym = make_moons(n_samples=100, noise=0.25, random_state=53)\n",
"\n",
"deep_tree_clf1 = DecisionTreeClassifier(random_state=42)\n",
"deep_tree_clf2 = DecisionTreeClassifier(min_samples_leaf=4, random_state=42)\n",
"deep_tree_clf1.fit(Xm, ym)\n",
"deep_tree_clf2.fit(Xm, ym)\n",
"\n",
"plt.figure(figsize=(11, 4))\n",
"plt.subplot(121)\n",
"plot_decision_boundary(deep_tree_clf1, Xm, ym, axes=[-1.5, 2.5, -1, 1.5], iris=False)\n",
"plt.title(\"No restrictions\", fontsize=16)\n",
"plt.subplot(122)\n",
"plot_decision_boundary(deep_tree_clf2, Xm, ym, axes=[-1.5, 2.5, -1, 1.5], iris=False)\n",
"plt.title(\"min_samples_leaf = {}\".format(deep_tree_clf2.min_samples_leaf), fontsize=14)\n",
"\n",
"save_fig(\"min_samples_leaf_plot\")\n",
"plt.show()"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"angle = np.pi / 180 * 20\n",
"rotation_matrix = np.array([[np.cos(angle), -np.sin(angle)], [np.sin(angle), np.cos(angle)]])\n",
"Xr = X.dot(rotation_matrix)\n",
"\n",
"tree_clf_r = DecisionTreeClassifier(random_state=42)\n",
"tree_clf_r.fit(Xr, y)\n",
"\n",
"plt.figure(figsize=(8, 3))\n",
"plot_decision_boundary(tree_clf_r, Xr, y, axes=[0.5, 7.5, -1.0, 1], iris=False)\n",
"\n",
"plt.show()"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"rnd.seed(6)\n",
"Xs = rnd.rand(100, 2) - 0.5\n",
"ys = (Xs[:, 0] > 0).astype(np.float32) * 2\n",
"\n",
"angle = np.pi / 4\n",
"rotation_matrix = np.array([[np.cos(angle), -np.sin(angle)], [np.sin(angle), np.cos(angle)]])\n",
"Xsr = Xs.dot(rotation_matrix)\n",
"\n",
"tree_clf_s = DecisionTreeClassifier(random_state=42)\n",
"tree_clf_s.fit(Xs, ys)\n",
"tree_clf_sr = DecisionTreeClassifier(random_state=42)\n",
"tree_clf_sr.fit(Xsr, ys)\n",
"\n",
"plt.figure(figsize=(11, 4))\n",
"plt.subplot(121)\n",
"plot_decision_boundary(tree_clf_s, Xs, ys, axes=[-0.7, 0.7, -0.7, 0.7], iris=False)\n",
"plt.subplot(122)\n",
"plot_decision_boundary(tree_clf_sr, Xsr, ys, axes=[-0.7, 0.7, -0.7, 0.7], iris=False)\n",
"\n",
"save_fig(\"sensitivity_to_rotation_plot\")\n",
"plt.show()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Regression trees"
]
},
{
"cell_type": "code",
"execution_count": 145,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"from sklearn.tree import DecisionTreeRegressor\n",
"\n",
"# Quadratic training set + noise\n",
"rnd.seed(42)\n",
"m = 200\n",
"X = rnd.rand(m, 1)\n",
"y = 4 * (X - 0.5) ** 2\n",
"y = y + rnd.randn(m, 1) / 10\n",
"\n",
"tree_reg1 = DecisionTreeRegressor(random_state=42, max_depth=2)\n",
"tree_reg2 = DecisionTreeRegressor(random_state=42, max_depth=3)\n",
"tree_reg1.fit(X, y)\n",
"tree_reg2.fit(X, y)\n",
"\n",
"def plot_regression_predictions(tree_reg, X, y, axes=[0, 1, -0.2, 1], ylabel=\"$y$\"):\n",
" x1 = np.linspace(axes[0], axes[1], 500).reshape(-1, 1)\n",
" y_pred = tree_reg.predict(x1)\n",
" plt.axis(axes)\n",
" plt.xlabel(\"$x_1$\", fontsize=18)\n",
" if ylabel:\n",
" plt.ylabel(ylabel, fontsize=18, rotation=0)\n",
" plt.plot(X, y, \"b.\")\n",
" plt.plot(x1, y_pred, \"r.-\", linewidth=2, label=r\"$\\hat{y}$\")\n",
"\n",
"plt.figure(figsize=(11, 4))\n",
"plt.subplot(121)\n",
"plot_regression_predictions(tree_reg1, X, y)\n",
"for split, style in ((0.1973, \"k-\"), (0.0917, \"k--\"), (0.7718, \"k--\")):\n",
" plt.plot([split, split], [-0.2, 1], style, linewidth=2)\n",
"plt.text(0.21, 0.65, \"Depth=0\", fontsize=15)\n",
"plt.text(0.01, 0.2, \"Depth=1\", fontsize=13)\n",
"plt.text(0.65, 0.8, \"Depth=1\", fontsize=13)\n",
"plt.legend(loc=\"upper center\", fontsize=18)\n",
"plt.title(\"max_depth=2\", fontsize=14)\n",
"\n",
"plt.subplot(122)\n",
"plot_regression_predictions(tree_reg2, X, y, ylabel=None)\n",
"for split, style in ((0.1973, \"k-\"), (0.0917, \"k--\"), (0.7718, \"k--\")):\n",
" plt.plot([split, split], [-0.2, 1], style, linewidth=2)\n",
"for split in (0.0458, 0.1298, 0.2873, 0.9040):\n",
" plt.plot([split, split], [-0.2, 1], \"k:\", linewidth=1)\n",
"plt.text(0.3, 0.5, \"Depth=2\", fontsize=13)\n",
"plt.title(\"max_depth=3\", fontsize=14)\n",
"\n",
"save_fig(\"tree_regression_plot\")\n",
"plt.show()"
]
},
{
"cell_type": "code",
"execution_count": 131,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"export_graphviz(\n",
" tree_reg1,\n",
" out_file=image_path(\"regression_tree.dot\"),\n",
" feature_names=[\"x1\"],\n",
" rounded=True,\n",
" filled=True\n",
" )"
]
},
{
"cell_type": "code",
"execution_count": 144,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"tree_reg1 = DecisionTreeRegressor(random_state=42)\n",
"tree_reg2 = DecisionTreeRegressor(random_state=42, min_samples_leaf=10)\n",
"tree_reg1.fit(X, y)\n",
"tree_reg2.fit(X, y)\n",
"\n",
"x1 = np.linspace(0, 1, 500).reshape(-1, 1)\n",
"y_pred1 = tree_reg1.predict(x1)\n",
"y_pred2 = tree_reg2.predict(x1)\n",
"\n",
"plt.figure(figsize=(11, 4))\n",
"\n",
"plt.subplot(121)\n",
"plt.plot(X, y, \"b.\")\n",
"plt.plot(x1, y_pred1, \"r.-\", linewidth=2, label=r\"$\\hat{y}$\")\n",
"plt.axis([0, 1, -0.2, 1.1])\n",
"plt.xlabel(\"$x_1$\", fontsize=18)\n",
"plt.ylabel(\"$y$\", fontsize=18, rotation=0)\n",
"plt.legend(loc=\"upper center\", fontsize=18)\n",
"plt.title(\"No restrictions\", fontsize=14)\n",
"\n",
"plt.subplot(122)\n",
"plt.plot(X, y, \"b.\")\n",
"plt.plot(x1, y_pred2, \"r.-\", linewidth=2, label=r\"$\\hat{y}$\")\n",
"plt.axis([0, 1, -0.2, 1.1])\n",
"plt.xlabel(\"$x_1$\", fontsize=18)\n",
"plt.title(\"min_samples_leaf={}\".format(tree_reg2.min_samples_leaf), fontsize=14)\n",
"\n",
"save_fig(\"tree_regression_regularization_plot\")\n",
"plt.show()"
]
},
{
"cell_type": "markdown",
"metadata": {
"collapsed": true
},
"source": [
"# Exercise solutions"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"**Coming soon**"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.5.1"
},
"nav_menu": {
"height": "309px",
"width": "468px"
},
"toc": {
"navigate_menu": true,
"number_sections": true,
"sideBar": true,
"threshold": 6,
"toc_cell": false,
"toc_section_display": "block",
"toc_window_display": false
}
},
"nbformat": 4,
"nbformat_minor": 0
}

788
07_ensemble_learning_and_random_forests.ipynb Normal file
View File

@ -0,0 +1,788 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"**Chapter 7 – Ensemble Learning and Random Forests**"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"_This notebook contains all the sample code and solutions to the exercices in chapter 7._"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Setup"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"First, let's make sure this notebook works well in both python 2 and 3, import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures:"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"# To support both python 2 and python 3\n",
"from __future__ import division, print_function, unicode_literals\n",
"\n",
"# Common imports\n",
"import numpy as np\n",
"import numpy.random as rnd\n",
"import os\n",
"\n",
"# to make this notebook's output stable across runs\n",
"rnd.seed(42)\n",
"\n",
"# To plot pretty figures\n",
"%matplotlib inline\n",
"import matplotlib\n",
"import matplotlib.pyplot as plt\n",
"plt.rcParams['axes.labelsize'] = 14\n",
"plt.rcParams['xtick.labelsize'] = 12\n",
"plt.rcParams['ytick.labelsize'] = 12\n",
"\n",
"# Where to save the figures\n",
"PROJECT_ROOT_DIR = \".\"\n",
"CHAPTER_ID = \"ensembles\"\n",
"\n",
"def image_path(fig_id):\n",
" return os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID, fig_id)\n",
"\n",
"def save_fig(fig_id, tight_layout=True):\n",
" print(\"Saving figure\", fig_id)\n",
" if tight_layout:\n",
" plt.tight_layout()\n",
" plt.savefig(image_path(fig_id) + \".png\", format='png', dpi=300)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Voting classifiers"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"heads_proba = 0.51\n",
"coin_tosses = (rnd.rand(10000, 10) < heads_proba).astype(np.int32)\n",
"cumulative_heads_ratio = np.cumsum(coin_tosses, axis=0) / np.arange(1, 10001).reshape(-1, 1)"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"plt.figure(figsize=(8,3.5))\n",
"plt.plot(cumulative_heads_ratio)\n",
"plt.plot([0, 10000], [0.51, 0.51], \"k--\", linewidth=2, label=\"51%\")\n",
"plt.plot([0, 10000], [0.5, 0.5], \"k-\", label=\"50%\")\n",
"plt.xlabel(\"Number of coin tosses\")\n",
"plt.ylabel(\"Heads ratio\")\n",
"plt.legend(loc=\"lower right\")\n",
"plt.axis([0, 10000, 0.42, 0.58])\n",
"save_fig(\"law_of_large_numbers_plot\")\n",
"plt.show()"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"from sklearn.cross_validation import train_test_split\n",
"from sklearn.datasets import make_moons\n",
"\n",
"X, y = make_moons(n_samples=500, noise=0.30, random_state=42)\n",
"X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)\n",
"\n",
"from sklearn.ensemble import RandomForestClassifier\n",
"from sklearn.ensemble import VotingClassifier\n",
"from sklearn.linear_model import LogisticRegression\n",
"from sklearn.svm import SVC\n",
"\n",
"log_clf = LogisticRegression(random_state=42)\n",
"rnd_clf = RandomForestClassifier(random_state=42)\n",
"svm_clf = SVC(probability=True, random_state=42)\n",
"\n",
"voting_clf = VotingClassifier(\n",
" estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],\n",
" voting='soft'\n",
" )\n",
"voting_clf.fit(X_train, y_train)\n",
"\n",
"from sklearn.metrics import accuracy_score\n",
"\n",
"for clf in (log_clf, rnd_clf, svm_clf, voting_clf):\n",
" clf.fit(X_train, y_train)\n",
" y_pred = clf.predict(X_test)\n",
" print(clf.__class__.__name__, accuracy_score(y_test, y_pred))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Bagging ensembles"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"from sklearn.datasets import make_moons\n",
"from sklearn.ensemble import BaggingClassifier\n",
"from sklearn.metrics import accuracy_score\n",
"from sklearn.tree import DecisionTreeClassifier\n",
"\n",
"bag_clf = BaggingClassifier(\n",
" DecisionTreeClassifier(random_state=42), n_estimators=500,\n",
" max_samples=100, bootstrap=True, n_jobs=-1, random_state=42\n",
" )\n",
"bag_clf.fit(X_train, y_train)\n",
"y_pred = bag_clf.predict(X_test)\n",
"print(accuracy_score(y_test, y_pred))"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"tree_clf = DecisionTreeClassifier(random_state=42)\n",
"tree_clf.fit(X_train, y_train)\n",
"y_pred_tree = tree_clf.predict(X_test)\n",
"print(accuracy_score(y_test, y_pred_tree))"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"from matplotlib.colors import ListedColormap\n",
"\n",
"def plot_decision_boundary(clf, X, y, axes=[-1.5, 2.5, -1, 1.5], alpha=0.5, contour=True):\n",
" x1s = np.linspace(axes[0], axes[1], 100)\n",
" x2s = np.linspace(axes[2], axes[3], 100)\n",
" x1, x2 = np.meshgrid(x1s, x2s)\n",
" X_new = np.c_[x1.ravel(), x2.ravel()]\n",
" y_pred = clf.predict(X_new).reshape(x1.shape)\n",
" custom_cmap = ListedColormap(['#fafab0','#9898ff','#a0faa0'])\n",
" plt.contourf(x1, x2, y_pred, alpha=0.3, cmap=custom_cmap, linewidth=10)\n",
" if contour:\n",
" custom_cmap2 = ListedColormap(['#7d7d58','#4c4c7f','#507d50'])\n",
" plt.contour(x1, x2, y_pred, cmap=custom_cmap2, alpha=0.8)\n",
" plt.plot(X[:, 0][y==0], X[:, 1][y==0], \"yo\", alpha=alpha)\n",
" plt.plot(X[:, 0][y==1], X[:, 1][y==1], \"bs\", alpha=alpha)\n",
" plt.axis(axes)\n",
" plt.xlabel(r\"$x_1$\", fontsize=18)\n",
" plt.ylabel(r\"$x_2$\", fontsize=18, rotation=0)"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"plt.figure(figsize=(11,4))\n",
"plt.subplot(121)\n",
"plot_decision_boundary(tree_clf, X, y)\n",
"plt.title(\"Decision Tree\", fontsize=14)\n",
"plt.subplot(122)\n",
"plot_decision_boundary(bag_clf, X, y)\n",
"plt.title(\"Decision Trees with Bagging\", fontsize=14)\n",
"save_fig(\"decision_tree_without_and_with_bagging_plot\")\n",
"plt.show()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Random Forests"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"bag_clf = BaggingClassifier(\n",
" DecisionTreeClassifier(splitter=\"random\", max_leaf_nodes=16, random_state=42),\n",
" n_estimators=500, max_samples=1.0, bootstrap=True,\n",
" n_jobs=-1, random_state=42\n",
" )\n",
"bag_clf.fit(X_train, y_train)\n",
"y_pred = bag_clf.predict(X_test)"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"from sklearn.ensemble import RandomForestClassifier\n",
"\n",
"rnd_clf = RandomForestClassifier(n_estimators=500, max_leaf_nodes=16, n_jobs=-1, random_state=42)\n",
"rnd_clf.fit(X_train, y_train)\n",
"\n",
"y_pred_rf = rnd_clf.predict(X_test)"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"np.sum(y_pred == y_pred_rf) / len(y_pred) # almost identical predictions"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"from sklearn.datasets import load_iris\n",
"iris = load_iris()\n",
"rnd_clf = RandomForestClassifier(n_estimators=500, n_jobs=-1, random_state=42)\n",
"rnd_clf.fit(iris[\"data\"], iris[\"target\"])\n",
"for name, importance in zip(iris[\"feature_names\"], rnd_clf.feature_importances_):\n",
" print(name, \"=\", importance)"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"rnd_clf.feature_importances_"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"plt.figure(figsize=(6, 4))\n",
"\n",
"for i in range(15):\n",
" tree_clf = DecisionTreeClassifier(max_leaf_nodes=16, random_state=42+i)\n",
" indices_with_replacement = rnd.randint(0, len(X_train), len(X_train))\n",
" tree_clf.fit(X[indices_with_replacement], y[indices_with_replacement])\n",
" plot_decision_boundary(tree_clf, X, y, axes=[-1.5, 2.5, -1, 1.5], alpha=0.02, contour=False)\n",
"\n",
"plt.show()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Out-of-Bag evaluation"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"bag_clf = BaggingClassifier(\n",
" DecisionTreeClassifier(random_state=42), n_estimators=500,\n",
" bootstrap=True, n_jobs=-1, oob_score=True, random_state=40\n",
")\n",
"bag_clf.fit(X_train, y_train)\n",
"bag_clf.oob_score_"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"bag_clf.oob_decision_function_[:10]"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"from sklearn.metrics import accuracy_score\n",
"y_pred = bag_clf.predict(X_test)\n",
"accuracy_score(y_test, y_pred)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Feature importance"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"from sklearn.datasets import fetch_mldata\n",
"mnist = fetch_mldata('MNIST original')\n",
"rnd_clf = RandomForestClassifier(random_state=42)\n",
"rnd_clf.fit(mnist[\"data\"], mnist[\"target\"])"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"def plot_digit(data):\n",
" image = data.reshape(28, 28)\n",
" plt.imshow(image, cmap = matplotlib.cm.hot,\n",
" interpolation=\"nearest\")\n",
" plt.axis(\"off\")"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"plot_digit(rnd_clf.feature_importances_)\n",
"\n",
"cbar = plt.colorbar(ticks=[rnd_clf.feature_importances_.min(), rnd_clf.feature_importances_.max()])\n",
"cbar.ax.set_yticklabels(['Not important', 'Very important'])\n",
"\n",
"save_fig(\"mnist_feature_importance_plot\")\n",
"plt.show()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# AdaBoost"
]
},
{
"cell_type": "code",
"execution_count": 21,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"from sklearn.ensemble import AdaBoostClassifier\n",
"\n",
"ada_clf = AdaBoostClassifier(\n",
" DecisionTreeClassifier(max_depth=2), n_estimators=200,\n",
" algorithm=\"SAMME.R\", learning_rate=0.5, random_state=42\n",
" )\n",
"ada_clf.fit(X_train, y_train)\n",
"plot_decision_boundary(ada_clf, X, y)"
]
},
{
"cell_type": "code",
"execution_count": 22,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"m = len(X_train)\n",
"\n",
"plt.figure(figsize=(11, 4))\n",
"for subplot, learning_rate in ((121, 1), (122, 0.5)):\n",
" sample_weights = np.ones(m)\n",
" for i in range(5):\n",
" plt.subplot(subplot)\n",
" svm_clf = SVC(kernel=\"rbf\", C=0.05)\n",
" svm_clf.fit(X_train, y_train, sample_weight=sample_weights)\n",
" y_pred = svm_clf.predict(X_train)\n",
" sample_weights[y_pred != y_train] *= (1 + learning_rate)\n",
" plot_decision_boundary(svm_clf, X, y, alpha=0.2)\n",
" plt.title(\"learning_rate = {}\".format(learning_rate - 1), fontsize=16)\n",
"\n",
"plt.subplot(121)\n",
"plt.text(-0.7, -0.65, \"1\", fontsize=14)\n",
"plt.text(-0.6, -0.10, \"2\", fontsize=14)\n",
"plt.text(-0.5, 0.10, \"3\", fontsize=14)\n",
"plt.text(-0.4, 0.55, \"4\", fontsize=14)\n",
"plt.text(-0.3, 0.90, \"5\", fontsize=14)\n",
"save_fig(\"boosting_plot\")\n",
"plt.show()"
]
},
{
"cell_type": "code",
"execution_count": 23,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"list(m for m in dir(ada_clf) if not m.startswith(\"_\") and m.endswith(\"_\"))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Gradient Boosting"
]
},
{
"cell_type": "code",
"execution_count": 24,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"from sklearn.tree import DecisionTreeRegressor\n",
"\n",
"rnd.seed(42)\n",
"X = rnd.rand(100, 1) - 0.5\n",
"y = 3*X[:, 0]**2 + 0.05 * rnd.randn(100)\n",
"\n",
"tree_reg1 = DecisionTreeRegressor(max_depth=2, random_state=42)\n",
"tree_reg1.fit(X, y)\n",
"\n",
"y2 = y - tree_reg1.predict(X)\n",
"tree_reg2 = DecisionTreeRegressor(max_depth=2, random_state=42)\n",
"tree_reg2.fit(X, y2)\n",
"\n",
"y3 = y2 - tree_reg2.predict(X)\n",
"tree_reg3 = DecisionTreeRegressor(max_depth=2, random_state=42)\n",
"tree_reg3.fit(X, y3)\n",
"\n",
"X_new = np.array([[0.8]])\n",
"y_pred = sum(tree.predict(X_new) for tree in (tree_reg1, tree_reg2, tree_reg3))\n",
"print(y_pred)"
]
},
{
"cell_type": "code",
"execution_count": 25,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"def plot_predictions(regressors, X, y, axes, label=None, style=\"r-\", data_style=\"b.\", data_label=None):\n",
" x1 = np.linspace(axes[0], axes[1], 500)\n",
" y_pred = sum(regressor.predict(x1.reshape(-1, 1)) for regressor in regressors)\n",
" plt.plot(X[:, 0], y, data_style, label=data_label)\n",
" plt.plot(x1, y_pred, style, linewidth=2, label=label)\n",
" if label or data_label:\n",
" plt.legend(loc=\"upper center\", fontsize=16)\n",
" plt.axis(axes)\n",
"\n",
"plt.figure(figsize=(11,11))\n",
"\n",
"plt.subplot(321)\n",
"plot_predictions([tree_reg1], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label=\"$h_1(x_1)$\", style=\"g-\", data_label=\"Training set\")\n",
"plt.ylabel(\"$y$\", fontsize=16, rotation=0)\n",
"plt.title(\"Residuals and tree predictions\", fontsize=16)\n",
"\n",
"plt.subplot(322)\n",
"plot_predictions([tree_reg1], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label=\"$h(x_1) = h_1(x_1)$\", data_label=\"Training set\")\n",
"plt.ylabel(\"$y$\", fontsize=16, rotation=0)\n",
"plt.title(\"Ensemble predictions\", fontsize=16)\n",
"\n",
"plt.subplot(323)\n",
"plot_predictions([tree_reg2], X, y2, axes=[-0.5, 0.5, -0.5, 0.5], label=\"$h_2(x_1)$\", style=\"g-\", data_style=\"k+\", data_label=\"Residuals\")\n",
"plt.ylabel(\"$y - h_1(x_1)$\", fontsize=16)\n",
"\n",
"plt.subplot(324)\n",
"plot_predictions([tree_reg1, tree_reg2], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label=\"$h(x_1) = h_1(x_1) + h_2(x_1)$\")\n",
"plt.ylabel(\"$y$\", fontsize=16, rotation=0)\n",
"\n",
"plt.subplot(325)\n",
"plot_predictions([tree_reg3], X, y3, axes=[-0.5, 0.5, -0.5, 0.5], label=\"$h_3(x_1)$\", style=\"g-\", data_style=\"k+\")\n",
"plt.ylabel(\"$y - h_1(x_1) - h_2(x_1)$\", fontsize=16)\n",
"plt.xlabel(\"$x_1$\", fontsize=16)\n",
"\n",
"plt.subplot(326)\n",
"plot_predictions([tree_reg1, tree_reg2, tree_reg3], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label=\"$h(x_1) = h_1(x_1) + h_2(x_1) + h_3(x_1)$\")\n",
"plt.xlabel(\"$x_1$\", fontsize=16)\n",
"plt.ylabel(\"$y$\", fontsize=16, rotation=0)\n",
"\n",
"save_fig(\"gradient_boosting_plot\")\n",
"plt.show()"
]
},
{
"cell_type": "code",
"execution_count": 26,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"from sklearn.ensemble import GradientBoostingRegressor\n",
"\n",
"gbrt = GradientBoostingRegressor(max_depth=2, n_estimators=3, learning_rate=0.1, random_state=42)\n",
"gbrt.fit(X, y)\n",
"\n",
"gbrt_slow = GradientBoostingRegressor(max_depth=2, n_estimators=200, learning_rate=0.1, random_state=42)\n",
"gbrt_slow.fit(X, y)\n",
"\n",
"plt.figure(figsize=(11,4))\n",
"\n",
"plt.subplot(121)\n",
"plot_predictions([gbrt], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label=\"Ensemble predictions\")\n",
"plt.title(\"learning_rate={}, n_estimators={}\".format(gbrt.learning_rate, gbrt.n_estimators), fontsize=14)\n",
"\n",
"plt.subplot(122)\n",
"plot_predictions([gbrt_slow], X, y, axes=[-0.5, 0.5, -0.1, 0.8])\n",
"plt.title(\"learning_rate={}, n_estimators={}\".format(gbrt_slow.learning_rate, gbrt_slow.n_estimators), fontsize=14)\n",
"\n",
"save_fig(\"gbrt_learning_rate_plot\")\n",
"plt.show()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Gradient Boosting with Early stopping"
]
},
{
"cell_type": "code",
"execution_count": 27,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"from sklearn.cross_validation import train_test_split\n",
"from sklearn.metrics import mean_squared_error\n",
"\n",
"X_train, X_val, y_train, y_val = train_test_split(X, y)\n",
"\n",
"gbrt = GradientBoostingRegressor(max_depth=2, n_estimators=120, learning_rate=0.1, random_state=42)\n",
"gbrt.fit(X_train, y_train)\n",
"\n",
"errors = [mean_squared_error(y_val, y_pred) for y_pred in gbrt.staged_predict(X_val)]"
]
},
{
"cell_type": "code",
"execution_count": 28,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"best_n_estimators = np.argmin(errors)\n",
"min_error = errors[best_n_estimators]\n",
"\n",
"gbrt_best = GradientBoostingRegressor(max_depth=2, n_estimators=best_n_estimators, learning_rate=0.1, random_state=42)\n",
"gbrt_best.fit(X_train, y_train)"
]
},
{
"cell_type": "code",
"execution_count": 29,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"plt.figure(figsize=(11, 4))\n",
"\n",
"plt.subplot(121)\n",
"plt.plot(errors, \"b.-\")\n",
"plt.plot([best_n_estimators, best_n_estimators], [0, min_error], \"k--\")\n",
"plt.plot([0, 120], [min_error, min_error], \"k--\")\n",
"plt.plot(best_n_estimators, min_error, \"ko\")\n",
"plt.text(best_n_estimators, min_error*1.2, \"Minimum\", ha=\"center\", fontsize=14)\n",
"plt.axis([0, 120, 0, 0.01])\n",
"plt.xlabel(\"Number of trees\")\n",
"plt.title(\"Validation error\", fontsize=14)\n",
"\n",
"plt.subplot(122)\n",
"plot_predictions([gbrt_best], X, y, axes=[-0.5, 0.5, -0.1, 0.8])\n",
"plt.title(\"Best model (55 trees)\", fontsize=14)\n",
"\n",
"save_fig(\"early_stopping_gbrt_plot\")\n",
"plt.show()"
]
},
{
"cell_type": "code",
"execution_count": 30,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"gbrt = GradientBoostingRegressor(max_depth=2, n_estimators=1, learning_rate=0.1, random_state=42, warm_start=True)\n",
"\n",
"min_val_error = float(\"inf\")\n",
"error_going_up = 0\n",
"for n_estimators in range(1, 120):\n",
" gbrt.n_estimators = n_estimators\n",
" gbrt.fit(X_train, y_train)\n",
" y_pred = gbrt.predict(X_val)\n",
" val_error = mean_squared_error(y_val, y_pred)\n",
" if val_error < min_val_error:\n",
" min_val_error = val_error\n",
" error_going_up = 0\n",
" else:\n",
" error_going_up += 1\n",
" if error_going_up == 5:\n",
" break # early stopping"
]
},
{
"cell_type": "code",
"execution_count": 31,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"print(gbrt.n_estimators)"
]
},
{
"cell_type": "markdown",
"metadata": {
"collapsed": true
},
"source": [
"# Exercise solutions"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"**Coming soon**"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.5.1"
},
"nav_menu": {
"height": "252px",
"width": "333px"
},
"toc": {
"navigate_menu": true,
"number_sections": true,
"sideBar": true,
"threshold": 6,
"toc_cell": false,
"toc_section_display": "block",
"toc_window_display": false
}
},
"nbformat": 4,
"nbformat_minor": 0
}

1343
08_dimensionality_reduction.ipynb Normal file
View File

File diff suppressed because it is too large Load Diff

1709
09_up_and_running_with_tensorflow.ipynb Normal file
View File

File diff suppressed because it is too large Load Diff

660
10_introduction_to_artificial_neural_networks.ipynb Normal file
View File

@ -0,0 +1,660 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"**Chapter 10 – Introduction to Artificial Neural Networks**"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"_This notebook contains all the sample code and solutions to the exercices in chapter 10._"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Setup"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"First, let's make sure this notebook works well in both python 2 and 3, import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures:"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"# To support both python 2 and python 3\n",
"from __future__ import division, print_function, unicode_literals\n",
"\n",
"# Common imports\n",
"import numpy as np\n",
"import numpy.random as rnd\n",
"import os\n",
"\n",
"# to make this notebook's output stable across runs\n",
"rnd.seed(42)\n",
"\n",
"# To plot pretty figures\n",
"%matplotlib inline\n",
"import matplotlib\n",
"import matplotlib.pyplot as plt\n",
"plt.rcParams['axes.labelsize'] = 14\n",
"plt.rcParams['xtick.labelsize'] = 12\n",
"plt.rcParams['ytick.labelsize'] = 12\n",
"\n",
"# Where to save the figures\n",
"PROJECT_ROOT_DIR = \".\"\n",
"CHAPTER_ID = \"ann\"\n",
"\n",
"def save_fig(fig_id, tight_layout=True):\n",
" path = os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID, fig_id + \".png\")\n",
" print(\"Saving figure\", fig_id)\n",
" if tight_layout:\n",
" plt.tight_layout()\n",
" plt.savefig(path, format='png', dpi=300)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Perceptrons"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"from sklearn.datasets import load_iris\n",
"iris = load_iris()\n",
"X = iris.data[:, (2, 3)] # petal length, petal width\n",
"y = (iris.target == 0).astype(np.int)"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"from sklearn.linear_model import Perceptron\n",
"\n",
"per_clf = Perceptron(random_state=42)\n",
"per_clf.fit(X, y)\n",
"\n",
"y_pred = per_clf.predict([[2, 0.5]])\n",
"y_pred"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"a = -per_clf.coef_[0][0] / per_clf.coef_[0][1]\n",
"b = -per_clf.intercept_ / per_clf.coef_[0][1]\n",
"\n",
"axes = [0, 5, 0, 2]\n",
"\n",
"x0, x1 = np.meshgrid(\n",
" np.linspace(axes[0], axes[1], 500).reshape(-1, 1),\n",
" np.linspace(axes[2], axes[3], 200).reshape(-1, 1),\n",
" )\n",
"X_new = np.c_[x0.ravel(), x1.ravel()]\n",
"y_predict = per_clf.predict(X_new)\n",
"zz = y_predict.reshape(x0.shape)\n",
"\n",
"plt.figure(figsize=(10, 4))\n",
"plt.plot(X[y==0, 0], X[y==0, 1], \"bs\", label=\"Not Iris-Setosa\")\n",
"plt.plot(X[y==1, 0], X[y==1, 1], \"yo\", label=\"Iris-Setosa\")\n",
"\n",
"plt.plot([axes[0], axes[1]], [a * axes[0] + b, a * axes[1] + b], \"k-\", linewidth=3)\n",
"from matplotlib.colors import ListedColormap\n",
"custom_cmap = ListedColormap(['#9898ff', '#fafab0'])\n",
"\n",
"plt.contourf(x0, x1, zz, cmap=custom_cmap, linewidth=5)\n",
"plt.xlabel(\"Petal length\", fontsize=14)\n",
"plt.ylabel(\"Petal width\", fontsize=14)\n",
"plt.legend(loc=\"lower right\", fontsize=14)\n",
"plt.axis(axes)\n",
"\n",
"save_fig(\"perceptron_iris_plot\")\n",
"plt.show()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Activation functions"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"def logit(z):\n",
" return 1 / (1 + np.exp(-z))\n",
"\n",
"def relu(z):\n",
" return np.maximum(0, z)\n",
"\n",
"def derivative(f, z, eps=0.000001):\n",
" return (f(z + eps) - f(z - eps))/(2 * eps)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"z = np.linspace(-5, 5, 200)\n",
"\n",
"plt.figure(figsize=(11,4))\n",
"\n",
"plt.subplot(121)\n",
"plt.plot(z, np.sign(z), \"r-\", linewidth=2, label=\"Step\")\n",
"plt.plot(z, logit(z), \"g--\", linewidth=2, label=\"Logit\")\n",
"plt.plot(z, np.tanh(z), \"b-\", linewidth=2, label=\"Tanh\")\n",
"plt.plot(z, relu(z), \"m-.\", linewidth=2, label=\"ReLU\")\n",
"plt.grid(True)\n",
"plt.legend(loc=\"center right\", fontsize=14)\n",
"plt.title(\"Activation functions\", fontsize=14)\n",
"plt.axis([-5, 5, -1.2, 1.2])\n",
"\n",
"plt.subplot(122)\n",
"plt.plot(z, derivative(np.sign, z), \"r-\", linewidth=2, label=\"Step\")\n",
"plt.plot(0, 0, \"ro\", markersize=5)\n",
"plt.plot(0, 0, \"rx\", markersize=10)\n",
"plt.plot(z, derivative(logit, z), \"g--\", linewidth=2, label=\"Logit\")\n",
"plt.plot(z, derivative(np.tanh, z), \"b-\", linewidth=2, label=\"Tanh\")\n",
"plt.plot(z, derivative(relu, z), \"m-.\", linewidth=2, label=\"ReLU\")\n",
"plt.grid(True)\n",
"#plt.legend(loc=\"center right\", fontsize=14)\n",
"plt.title(\"Derivatives\", fontsize=14)\n",
"plt.axis([-5, 5, -0.2, 1.2])\n",
"\n",
"save_fig(\"activation_functions_plot\")\n",
"plt.show()"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"def heaviside(z):\n",
" return (z >= 0).astype(z.dtype)\n",
"\n",
"def sigmoid(z):\n",
" return 1/(1+np.exp(-z))\n",
"\n",
"def mlp_xor(x1, x2, activation=heaviside):\n",
" return activation(-activation(x1 + x2 - 1.5) + activation(x1 + x2 - 0.5) - 0.5)"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"x1s = np.linspace(-0.2, 1.2, 100)\n",
"x2s = np.linspace(-0.2, 1.2, 100)\n",
"x1, x2 = np.meshgrid(x1s, x2s)\n",
"\n",
"z1 = mlp_xor(x1, x2, activation=heaviside)\n",
"z2 = mlp_xor(x1, x2, activation=sigmoid)\n",
"\n",
"plt.figure(figsize=(10,4))\n",
"\n",
"plt.subplot(121)\n",
"plt.contourf(x1, x2, z1)\n",
"plt.plot([0, 1], [0, 1], \"gs\", markersize=20)\n",
"plt.plot([0, 1], [1, 0], \"y^\", markersize=20)\n",
"plt.title(\"Activation function: heaviside\", fontsize=14)\n",
"plt.grid(True)\n",
"\n",
"plt.subplot(122)\n",
"plt.contourf(x1, x2, z2)\n",
"plt.plot([0, 1], [0, 1], \"gs\", markersize=20)\n",
"plt.plot([0, 1], [1, 0], \"y^\", markersize=20)\n",
"plt.title(\"Activation function: sigmoid\", fontsize=14)\n",
"plt.grid(True)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# FNN for MNIST"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## using tf.learn"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"from tensorflow.examples.tutorials.mnist import input_data\n",
"mnist = input_data.read_data_sets(\"/tmp/data/\")\n",
"X_train = mnist.train.images\n",
"X_test = mnist.test.images\n",
"y_train = mnist.train.labels.astype(\"int\")\n",
"y_test = mnist.test.labels.astype(\"int\")"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"import tensorflow as tf\n",
"\n",
"feature_columns = tf.contrib.learn.infer_real_valued_columns_from_input(X_train)\n",
"dnn_clf = tf.contrib.learn.DNNClassifier(hidden_units=[300, 100], n_classes=10,\n",
" feature_columns=feature_columns)\n",
"dnn_clf.fit(x=X_train, y=y_train, batch_size=50, steps=40000)"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"from sklearn.metrics import accuracy_score\n",
"\n",
"y_pred = dnn_clf.predict(X_test)\n",
"accuracy = accuracy_score(y_test, y_pred)\n",
"accuracy"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"from sklearn.metrics import log_loss\n",
"\n",
"y_pred_proba = dnn_clf.predict_proba(X_test)\n",
"log_loss(y_test, y_pred_proba)"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"dnn_clf.evaluate(X_test, y_test)"
]
},
{
"cell_type": "markdown",
"metadata": {
"collapsed": true
},
"source": [
"## Using plain TensorFlow"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"import tensorflow as tf\n",
"\n",
"def neuron_layer(X, n_neurons, name, activation=None):\n",
" with tf.name_scope(name):\n",
" n_inputs = int(X.get_shape()[1])\n",
" stddev = 1 / np.sqrt(n_inputs)\n",
" init = tf.truncated_normal((n_inputs, n_neurons), stddev=stddev)\n",
" W = tf.Variable(init, name=\"weights\")\n",
" b = tf.Variable(tf.zeros([n_neurons]), name=\"biases\")\n",
" Z = tf.matmul(X, W) + b\n",
" if activation==\"relu\":\n",
" return tf.nn.relu(Z)\n",
" else:\n",
" return Z"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"tf.reset_default_graph()\n",
"\n",
"n_inputs = 28*28 # MNIST\n",
"n_hidden1 = 300\n",
"n_hidden2 = 100\n",
"n_outputs = 10\n",
"learning_rate = 0.01\n",
"\n",
"X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
"y = tf.placeholder(tf.int64, shape=(None), name=\"y\")\n",
"\n",
"with tf.name_scope(\"dnn\"):\n",
" hidden1 = neuron_layer(X, n_hidden1, \"hidden1\", activation=\"relu\")\n",
" hidden2 = neuron_layer(hidden1, n_hidden2, \"hidden2\", activation=\"relu\")\n",
" logits = neuron_layer(hidden2, n_outputs, \"output\")\n",
"\n",
"with tf.name_scope(\"loss\"):\n",
" xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits, y)\n",
" loss = tf.reduce_mean(xentropy, name=\"loss\")\n",
"\n",
"with tf.name_scope(\"train\"):\n",
" optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
" training_op = optimizer.minimize(loss)\n",
"\n",
"with tf.name_scope(\"eval\"):\n",
" correct = tf.nn.in_top_k(logits, y, 1)\n",
" accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
" \n",
"init = tf.initialize_all_variables()\n",
"saver = tf.train.Saver()"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"n_epochs = 20\n",
"batch_size = 50\n",
"\n",
"with tf.Session() as sess:\n",
" init.run()\n",
" for epoch in range(n_epochs):\n",
" for iteration in range(len(mnist.test.labels)//batch_size):\n",
" X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
" sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
" acc_train = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n",
" acc_test = accuracy.eval(feed_dict={X: mnist.test.images, y: mnist.test.labels})\n",
" print(epoch, \"Train accuracy:\", acc_train, \"Test accuracy:\", acc_test)\n",
"\n",
" save_path = saver.save(sess, \"my_model_final.ckpt\")"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"with tf.Session() as sess:\n",
" saver.restore(sess, \"my_model_final.ckpt\")\n",
" X_new_scaled = mnist.test.images[:20]\n",
" Z = logits.eval(feed_dict={X: X_new_scaled})\n",
" print(np.argmax(Z, axis=1))\n",
" print(mnist.test.labels[:20])"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"from IPython.display import clear_output, Image, display, HTML\n",
"\n",
"def strip_consts(graph_def, max_const_size=32):\n",
" \"\"\"Strip large constant values from graph_def.\"\"\"\n",
" strip_def = tf.GraphDef()\n",
" for n0 in graph_def.node:\n",
" n = strip_def.node.add() \n",
" n.MergeFrom(n0)\n",
" if n.op == 'Const':\n",
" tensor = n.attr['value'].tensor\n",
" size = len(tensor.tensor_content)\n",
" if size > max_const_size:\n",
" tensor.tensor_content = b\"<stripped %d bytes>\"%size\n",
" return strip_def\n",
"\n",
"def show_graph(graph_def, max_const_size=32):\n",
" \"\"\"Visualize TensorFlow graph.\"\"\"\n",
" if hasattr(graph_def, 'as_graph_def'):\n",
" graph_def = graph_def.as_graph_def()\n",
" strip_def = strip_consts(graph_def, max_const_size=max_const_size)\n",
" code = \"\"\"\n",
" <script>\n",
" function load() {{\n",
" document.getElementById(\"{id}\").pbtxt = {data};\n",
" }}\n",
" </script>\n",
" <link rel=\"import\" href=\"https://tensorboard.appspot.com/tf-graph-basic.build.html\" onload=load()>\n",
" <div style=\"height:600px\">\n",
" <tf-graph-basic id=\"{id}\"></tf-graph-basic>\n",
" </div>\n",
" \"\"\".format(data=repr(str(strip_def)), id='graph'+str(np.random.rand()))\n",
"\n",
" iframe = \"\"\"\n",
" <iframe seamless style=\"width:1200px;height:620px;border:0\" srcdoc=\"{}\"></iframe>\n",
" \"\"\".format(code.replace('\"', '&quot;'))\n",
" display(HTML(iframe))"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"show_graph(tf.get_default_graph())"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Using `fully_connected` instead of `neuron_layer()`"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"tf.reset_default_graph()\n",
"\n",
"from tensorflow.contrib.layers import fully_connected\n",
"\n",
"n_inputs = 28*28 # MNIST\n",
"n_hidden1 = 300\n",
"n_hidden2 = 100\n",
"n_outputs = 10\n",
"learning_rate = 0.01\n",
"\n",
"X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
"y = tf.placeholder(tf.int64, shape=(None), name=\"y\")\n",
"\n",
"with tf.name_scope(\"dnn\"):\n",
" hidden1 = fully_connected(X, n_hidden1, scope=\"hidden1\")\n",
" hidden2 = fully_connected(hidden1, n_hidden2, scope=\"hidden2\")\n",
" logits = fully_connected(hidden2, n_outputs, activation_fn=None, scope=\"outputs\")\n",
"\n",
"with tf.name_scope(\"loss\"):\n",
" xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits, y)\n",
" loss = tf.reduce_mean(xentropy, name=\"loss\")\n",
"\n",
"with tf.name_scope(\"train\"):\n",
" optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
" training_op = optimizer.minimize(loss)\n",
"\n",
"with tf.name_scope(\"eval\"):\n",
" correct = tf.nn.in_top_k(logits, y, 1)\n",
" accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
" \n",
"init = tf.initialize_all_variables()\n",
"saver = tf.train.Saver()"
]
},
{
"cell_type": "code",
"execution_count": 21,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"n_epochs = 20\n",
"n_batches = 50\n",
"\n",
"with tf.Session() as sess:\n",
" init.run()\n",
" for epoch in range(n_epochs):\n",
" for iteration in range(len(mnist.test.labels)//batch_size):\n",
" X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
" sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
" acc_train = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n",
" acc_test = accuracy.eval(feed_dict={X: mnist.test.images, y: mnist.test.labels})\n",
" print(epoch, \"Train accuracy:\", acc_train, \"Test accuracy:\", acc_test)\n",
"\n",
" save_path = saver.save(sess, \"my_model_final.ckpt\")"
]
},
{
"cell_type": "code",
"execution_count": 22,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"show_graph(tf.get_default_graph())"
]
},
{
"cell_type": "markdown",
"metadata": {
"collapsed": true
},
"source": [
"# Exercise solutions"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"**Coming soon**"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.5.1"
},
"nav_menu": {
"height": "264px",
"width": "369px"
},
"toc": {
"navigate_menu": true,
"number_sections": true,
"sideBar": true,
"threshold": 6,
"toc_cell": false,
"toc_section_display": "block",
"toc_window_display": false
}
},
"nbformat": 4,
"nbformat_minor": 0
}

931
11_deep_learning.ipynb Normal file
View File

@ -0,0 +1,931 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"**Chapter 11 – Deep Learning**"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"_This notebook contains all the sample code and solutions to the exercices in chapter 11._"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Setup"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"First, let's make sure this notebook works well in both python 2 and 3, import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures:"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"# To support both python 2 and python 3\n",
"from __future__ import division, print_function, unicode_literals\n",
"\n",
"# Common imports\n",
"import numpy as np\n",
"import numpy.random as rnd\n",
"import os\n",
"\n",
"# to make this notebook's output stable across runs\n",
"rnd.seed(42)\n",
"\n",
"# To plot pretty figures\n",
"%matplotlib inline\n",
"import matplotlib\n",
"import matplotlib.pyplot as plt\n",
"plt.rcParams['axes.labelsize'] = 14\n",
"plt.rcParams['xtick.labelsize'] = 12\n",
"plt.rcParams['ytick.labelsize'] = 12\n",
"\n",
"# Where to save the figures\n",
"PROJECT_ROOT_DIR = \".\"\n",
"CHAPTER_ID = \"deep\"\n",
"\n",
"def save_fig(fig_id, tight_layout=True):\n",
" path = os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID, fig_id + \".png\")\n",
" print(\"Saving figure\", fig_id)\n",
" if tight_layout:\n",
" plt.tight_layout()\n",
" plt.savefig(path, format='png', dpi=300)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Activation functions"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"def logit(z):\n",
" return 1 / (1 + np.exp(-z))"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"z = np.linspace(-5, 5, 200)\n",
"\n",
"plt.plot([-5, 5], [0, 0], 'k-')\n",
"plt.plot([-5, 5], [1, 1], 'k--')\n",
"plt.plot([0, 0], [-0.2, 1.2], 'k-')\n",
"plt.plot([-5, 5], [-3/4, 7/4], 'g--')\n",
"plt.plot(z, logit(z), \"b-\", linewidth=2)\n",
"props = dict(facecolor='black', shrink=0.1)\n",
"plt.annotate('Saturating', xytext=(3.5, 0.7), xy=(5, 1), arrowprops=props, fontsize=14, ha=\"center\")\n",
"plt.annotate('Saturating', xytext=(-3.5, 0.3), xy=(-5, 0), arrowprops=props, fontsize=14, ha=\"center\")\n",
"plt.annotate('Linear', xytext=(2, 0.2), xy=(0, 0.5), arrowprops=props, fontsize=14, ha=\"center\")\n",
"plt.grid(True)\n",
"plt.title(\"Sigmoid activation function\", fontsize=14)\n",
"plt.axis([-5, 5, -0.2, 1.2])\n",
"\n",
"save_fig(\"sigmoid_saturation_plot\")\n",
"plt.show()"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"def leaky_relu(z, alpha=0.01):\n",
" return np.maximum(alpha*z, z)"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"plt.plot(z, leaky_relu(z, 0.05), \"b-\", linewidth=2)\n",
"plt.plot([-5, 5], [0, 0], 'k-')\n",
"plt.plot([0, 0], [-0.5, 4.2], 'k-')\n",
"plt.grid(True)\n",
"props = dict(facecolor='black', shrink=0.1)\n",
"plt.annotate('Leak', xytext=(-3.5, 0.5), xy=(-5, -0.2), arrowprops=props, fontsize=14, ha=\"center\")\n",
"plt.title(\"Leaky ReLU activation function\", fontsize=14)\n",
"plt.axis([-5, 5, -0.5, 4.2])\n",
"\n",
"save_fig(\"leaky_relu_plot\")\n",
"plt.show()"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"def elu(z, alpha=1):\n",
" return np.where(z<0, alpha*(np.exp(z)-1), z)"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"plt.plot(z, elu(z), \"b-\", linewidth=2)\n",
"plt.plot([-5, 5], [0, 0], 'k-')\n",
"plt.plot([-5, 5], [-1, -1], 'k--')\n",
"plt.plot([0, 0], [-2.2, 3.2], 'k-')\n",
"plt.grid(True)\n",
"props = dict(facecolor='black', shrink=0.1)\n",
"plt.title(r\"ELU activation function ($\\alpha=1$)\", fontsize=14)\n",
"plt.axis([-5, 5, -2.2, 3.2])\n",
"\n",
"save_fig(\"elu_plot\")\n",
"plt.show()"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"from tensorflow.examples.tutorials.mnist import input_data\n",
"mnist = input_data.read_data_sets(\"/tmp/data/\")"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"def leaky_relu(z, name=None):\n",
" return tf.maximum(0.01 * z, z, name=name)"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"import tensorflow as tf"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"from IPython.display import clear_output, Image, display, HTML\n",
"\n",
"def strip_consts(graph_def, max_const_size=32):\n",
" \"\"\"Strip large constant values from graph_def.\"\"\"\n",
" strip_def = tf.GraphDef()\n",
" for n0 in graph_def.node:\n",
" n = strip_def.node.add() \n",
" n.MergeFrom(n0)\n",
" if n.op == 'Const':\n",
" tensor = n.attr['value'].tensor\n",
" size = len(tensor.tensor_content)\n",
" if size > max_const_size:\n",
" tensor.tensor_content = b\"<stripped %d bytes>\"%size\n",
" return strip_def\n",
"\n",
"def show_graph(graph_def, max_const_size=32):\n",
" \"\"\"Visualize TensorFlow graph.\"\"\"\n",
" if hasattr(graph_def, 'as_graph_def'):\n",
" graph_def = graph_def.as_graph_def()\n",
" strip_def = strip_consts(graph_def, max_const_size=max_const_size)\n",
" code = \"\"\"\n",
" <script>\n",
" function load() {{\n",
" document.getElementById(\"{id}\").pbtxt = {data};\n",
" }}\n",
" </script>\n",
" <link rel=\"import\" href=\"https://tensorboard.appspot.com/tf-graph-basic.build.html\" onload=load()>\n",
" <div style=\"height:600px\">\n",
" <tf-graph-basic id=\"{id}\"></tf-graph-basic>\n",
" </div>\n",
" \"\"\".format(data=repr(str(strip_def)), id='graph'+str(np.random.rand()))\n",
"\n",
" iframe = \"\"\"\n",
" <iframe seamless style=\"width:1200px;height:620px;border:0\" srcdoc=\"{}\"></iframe>\n",
" \"\"\".format(code.replace('\"', '&quot;'))\n",
" display(HTML(iframe))"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"from tensorflow.contrib.layers import fully_connected\n",
"\n",
"tf.reset_default_graph()\n",
"\n",
"n_inputs = 28*28 # MNIST\n",
"n_hidden1 = 300\n",
"n_hidden2 = 100\n",
"n_outputs = 10\n",
"learning_rate = 0.01\n",
"\n",
"X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
"y = tf.placeholder(tf.int64, shape=(None), name=\"y\")\n",
"\n",
"with tf.name_scope(\"dnn\"):\n",
" hidden1 = fully_connected(X, n_hidden1, activation_fn=leaky_relu, scope=\"hidden1\")\n",
" hidden2 = fully_connected(hidden1, n_hidden2, activation_fn=leaky_relu, scope=\"hidden2\")\n",
" logits = fully_connected(hidden2, n_outputs, activation_fn=None, scope=\"outputs\")\n",
"\n",
"with tf.name_scope(\"loss\"):\n",
" xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits, y)\n",
" loss = tf.reduce_mean(xentropy, name=\"loss\")\n",
"\n",
"with tf.name_scope(\"train\"):\n",
" optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
" training_op = optimizer.minimize(loss)\n",
"\n",
"with tf.name_scope(\"eval\"):\n",
" correct = tf.nn.in_top_k(logits, y, 1)\n",
" accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
" \n",
"init = tf.initialize_all_variables()\n",
"saver = tf.train.Saver()"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"n_epochs = 20\n",
"batch_size = 100\n",
"\n",
"with tf.Session() as sess:\n",
" init.run()\n",
" for epoch in range(n_epochs):\n",
" for iteration in range(len(mnist.test.labels)//batch_size):\n",
" X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
" sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
" acc_train = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n",
" acc_test = accuracy.eval(feed_dict={X: mnist.test.images, y: mnist.test.labels})\n",
" print(epoch, \"Train accuracy:\", acc_train, \"Test accuracy:\", acc_test)\n",
"\n",
" save_path = saver.save(sess, \"my_model_final.ckpt\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Batch Normalization"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"from tensorflow.contrib.layers import fully_connected, batch_norm\n",
"from tensorflow.contrib.framework import arg_scope\n",
"\n",
"tf.reset_default_graph()\n",
"\n",
"n_inputs = 28 * 28 # MNIST\n",
"n_hidden1 = 300\n",
"n_hidden2 = 100\n",
"n_outputs = 10\n",
"learning_rate = 0.01\n",
"momentum = 0.25\n",
"\n",
"X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
"y = tf.placeholder(tf.int64, shape=(None), name=\"y\")\n",
"is_training = tf.placeholder(tf.bool, shape=(), name='is_training')\n",
"\n",
"with tf.name_scope(\"dnn\"):\n",
" he_init = tf.contrib.layers.variance_scaling_initializer()\n",
" batch_norm_params = {\n",
" 'is_training': is_training,\n",
" 'decay': 0.9,\n",
" 'updates_collections': None,\n",
" 'scale': True,\n",
" }\n",
"\n",
" with arg_scope(\n",
" [fully_connected],\n",
" activation_fn=tf.nn.elu,\n",
" weights_initializer=he_init,\n",
" normalizer_fn=batch_norm,\n",
" normalizer_params=batch_norm_params):\n",
" hidden1 = fully_connected(X, n_hidden1, scope=\"hidden1\")\n",
" hidden2 = fully_connected(hidden1, n_hidden2, scope=\"hidden2\")\n",
" logits = fully_connected(hidden2, n_outputs, activation_fn=None, scope=\"outputs\")\n",
"\n",
"with tf.name_scope(\"loss\"):\n",
" xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits, y)\n",
" loss = tf.reduce_mean(xentropy, name=\"loss\")\n",
"\n",
"with tf.name_scope(\"train\"):\n",
" optimizer = tf.train.MomentumOptimizer(learning_rate, momentum)\n",
" training_op = optimizer.minimize(loss)\n",
"\n",
"with tf.name_scope(\"eval\"):\n",
" correct = tf.nn.in_top_k(logits, y, 1)\n",
" accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
" \n",
"init = tf.initialize_all_variables()\n",
"saver = tf.train.Saver()"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"n_epochs = 20\n",
"batch_size = 50\n",
"\n",
"with tf.Session() as sess:\n",
" init.run()\n",
" for epoch in range(n_epochs):\n",
" for iteration in range(len(mnist.test.labels)//batch_size):\n",
" X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
" sess.run(training_op, feed_dict={is_training: True, X: X_batch, y: y_batch})\n",
" acc_train = accuracy.eval(feed_dict={is_training: False, X: X_batch, y: y_batch})\n",
" acc_test = accuracy.eval(feed_dict={is_training: False, X: mnist.test.images, y: mnist.test.labels})\n",
" print(epoch, \"Train accuracy:\", acc_train, \"Test accuracy:\", acc_test)\n",
"\n",
" save_path = saver.save(sess, \"my_model_final.ckpt\")"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"tf.reset_default_graph()\n",
"\n",
"X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
"y = tf.placeholder(tf.int64, shape=(None), name=\"y\")\n",
"is_training = tf.placeholder(tf.bool, shape=(), name='is_training')\n",
"\n",
"with tf.name_scope(\"dnn\"):\n",
" he_init = tf.contrib.layers.variance_scaling_initializer()\n",
" batch_norm_params = {\n",
" 'is_training': is_training,\n",
" 'decay': 0.9,\n",
" 'updates_collections': None,\n",
" 'scale': True,\n",
" }\n",
"\n",
" with arg_scope(\n",
" [fully_connected],\n",
" activation_fn=tf.nn.elu,\n",
" weights_initializer=he_init,\n",
" normalizer_fn=batch_norm,\n",
" normalizer_params=batch_norm_params,\n",
" weights_regularizer=tf.contrib.layers.l1_regularizer(0.01)):\n",
" hidden1 = fully_connected(X, n_hidden1, scope=\"hidden1\")\n",
" hidden2 = fully_connected(hidden1, n_hidden2, scope=\"hidden2\")\n",
" logits = fully_connected(hidden2, n_outputs, activation_fn=None, scope=\"outputs\")\n",
"\n",
"with tf.name_scope(\"loss\"):\n",
" xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits, y)\n",
" reg_losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)\n",
" base_loss = tf.reduce_mean(xentropy, name=\"base_loss\")\n",
" loss = tf.add(base_loss, reg_losses, name=\"loss\")\n",
"\n",
"with tf.name_scope(\"train\"):\n",
" optimizer = tf.train.MomentumOptimizer(learning_rate, momentum)\n",
" training_op = optimizer.minimize(loss)\n",
"\n",
"with tf.name_scope(\"eval\"):\n",
" correct = tf.nn.in_top_k(logits, y, 1)\n",
" accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
" \n",
"init = tf.initialize_all_variables()\n",
"saver = tf.train.Saver()"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"n_epochs = 20\n",
"batch_size = 50\n",
"\n",
"with tf.Session() as sess:\n",
" init.run()\n",
" for epoch in range(n_epochs):\n",
" for iteration in range(len(mnist.test.labels)//batch_size):\n",
" X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
" sess.run(training_op, feed_dict={is_training: True, X: X_batch, y: y_batch})\n",
" acc_train = accuracy.eval(feed_dict={is_training: False, X: X_batch, y: y_batch})\n",
" acc_test = accuracy.eval(feed_dict={is_training: False, X: mnist.test.images, y: mnist.test.labels})\n",
" print(epoch, \"Train accuracy:\", acc_train, \"Test accuracy:\", acc_test)\n",
"\n",
" save_path = saver.save(sess, \"my_model_final.ckpt\")"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"[v.name for v in tf.all_variables()]"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"with tf.variable_scope(\"\", reuse=True):\n",
" weights1 = tf.get_variable(\"hidden1/weights\")\n",
" weights2 = tf.get_variable(\"hidden2/weights\")\n",
" "
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"tf.reset_default_graph()\n",
"\n",
"x = tf.constant([0., 0., 3., 4., 30., 40., 300., 400.], shape=(4, 2))\n",
"c = tf.clip_by_norm(x, clip_norm=10)\n",
"c0 = tf.clip_by_norm(x, clip_norm=350, axes=0)\n",
"c1 = tf.clip_by_norm(x, clip_norm=10, axes=1)\n",
"\n",
"with tf.Session() as sess:\n",
" xv = x.eval()\n",
" cv = c.eval()\n",
" c0v = c0.eval()\n",
" c1v = c1.eval()\n",
"\n",
"print(xv)"
]
},
{
"cell_type": "code",
"execution_count": 21,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"print(cv)"
]
},
{
"cell_type": "code",
"execution_count": 22,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"print(np.linalg.norm(cv))"
]
},
{
"cell_type": "code",
"execution_count": 23,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"print(c0v)"
]
},
{
"cell_type": "code",
"execution_count": 24,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"print(np.linalg.norm(c0v, axis=0))"
]
},
{
"cell_type": "code",
"execution_count": 25,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"print(c1v)"
]
},
{
"cell_type": "code",
"execution_count": 26,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"print(np.linalg.norm(c1v, axis=1))"
]
},
{
"cell_type": "code",
"execution_count": 27,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"tf.reset_default_graph()\n",
"\n",
"X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
"y = tf.placeholder(tf.int64, shape=(None), name=\"y\")\n",
"is_training = tf.placeholder(tf.bool, shape=(), name='is_training')\n",
"\n",
"def max_norm_regularizer(threshold, axes=1, name=\"max_norm\", collection=\"max_norm\"):\n",
" def max_norm(weights):\n",
" clip_weights = tf.assign(weights, tf.clip_by_norm(weights, clip_norm=threshold, axes=axes), name=name)\n",
" tf.add_to_collection(collection, clip_weights)\n",
" return None # there is no regularization loss term\n",
" return max_norm\n",
"\n",
"with tf.name_scope(\"dnn\"):\n",
" with arg_scope(\n",
" [fully_connected],\n",
" weights_regularizer=max_norm_regularizer(1.5)):\n",
" hidden1 = fully_connected(X, n_hidden1, scope=\"hidden1\")\n",
" hidden2 = fully_connected(hidden1, n_hidden2, scope=\"hidden2\")\n",
" logits = fully_connected(hidden2, n_outputs, activation_fn=None, scope=\"outputs\")\n",
"\n",
"clip_all_weights = tf.get_collection(\"max_norm\")\n",
" \n",
"with tf.name_scope(\"loss\"):\n",
" xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits, y)\n",
" loss = tf.reduce_mean(xentropy, name=\"loss\")\n",
"\n",
"with tf.name_scope(\"train\"):\n",
" optimizer = tf.train.MomentumOptimizer(learning_rate, momentum)\n",
" threshold = 1.0\n",
" grads_and_vars = optimizer.compute_gradients(loss)\n",
" capped_gvs = [(tf.clip_by_value(grad, -threshold, threshold), var)\n",
" for grad, var in grads_and_vars]\n",
" training_op = optimizer.apply_gradients(capped_gvs)\n",
"\n",
"with tf.name_scope(\"eval\"):\n",
" correct = tf.nn.in_top_k(logits, y, 1)\n",
" accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
" \n",
"init = tf.initialize_all_variables()\n",
"saver = tf.train.Saver()"
]
},
{
"cell_type": "code",
"execution_count": 28,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"n_epochs = 20\n",
"batch_size = 50\n",
"\n",
"with tf.Session() as sess:\n",
" init.run()\n",
" for epoch in range(n_epochs):\n",
" for iteration in range(len(mnist.test.labels)//batch_size):\n",
" X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
" sess.run(training_op, feed_dict={is_training: True, X: X_batch, y: y_batch})\n",
" acc_train = accuracy.eval(feed_dict={is_training: False, X: X_batch, y: y_batch})\n",
" acc_test = accuracy.eval(feed_dict={is_training: False, X: mnist.test.images, y: mnist.test.labels})\n",
" print(epoch, \"Train accuracy:\", acc_train, \"Test accuracy:\", acc_test)\n",
"\n",
" save_path = saver.save(sess, \"my_model_final.ckpt\")"
]
},
{
"cell_type": "code",
"execution_count": 29,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"show_graph(tf.get_default_graph())"
]
},
{
"cell_type": "code",
"execution_count": 30,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"from tensorflow.contrib.layers import dropout\n",
"\n",
"tf.reset_default_graph()\n",
"\n",
"X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
"y = tf.placeholder(tf.int64, shape=(None), name=\"y\")\n",
"is_training = tf.placeholder(tf.bool, shape=(), name='is_training')\n",
"\n",
"initial_learning_rate = 0.1\n",
"decay_steps = 10000\n",
"decay_rate = 1/10\n",
"global_step = tf.Variable(0, trainable=False)\n",
"learning_rate = tf.train.exponential_decay(initial_learning_rate, global_step,\n",
" decay_steps, decay_rate)\n",
"\n",
"keep_prob = 0.5\n",
"\n",
"with tf.name_scope(\"dnn\"):\n",
" he_init = tf.contrib.layers.variance_scaling_initializer()\n",
" with arg_scope(\n",
" [fully_connected],\n",
" activation_fn=tf.nn.elu,\n",
" weights_initializer=he_init):\n",
" X_drop = dropout(X, keep_prob, is_training=is_training)\n",
" hidden1 = fully_connected(X_drop, n_hidden1, scope=\"hidden1\")\n",
" hidden1_drop = dropout(hidden1, keep_prob, is_training=is_training)\n",
" hidden2 = fully_connected(hidden1_drop, n_hidden2, scope=\"hidden2\")\n",
" hidden2_drop = dropout(hidden2, keep_prob, is_training=is_training)\n",
" logits = fully_connected(hidden2_drop, n_outputs, activation_fn=None, scope=\"outputs\")\n",
"\n",
"with tf.name_scope(\"loss\"):\n",
" xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits, y)\n",
" loss = tf.reduce_mean(xentropy, name=\"loss\")\n",
"\n",
"with tf.name_scope(\"train\"):\n",
" optimizer = tf.train.MomentumOptimizer(learning_rate, momentum)\n",
" training_op = optimizer.minimize(loss, global_step=global_step) \n",
"\n",
"with tf.name_scope(\"eval\"):\n",
" correct = tf.nn.in_top_k(logits, y, 1)\n",
" accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
" \n",
"init = tf.initialize_all_variables()\n",
"saver = tf.train.Saver()"
]
},
{
"cell_type": "code",
"execution_count": 31,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"n_epochs = 20\n",
"batch_size = 50\n",
"\n",
"with tf.Session() as sess:\n",
" init.run()\n",
" for epoch in range(n_epochs):\n",
" for iteration in range(len(mnist.test.labels)//batch_size):\n",
" X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
" sess.run(training_op, feed_dict={is_training: True, X: X_batch, y: y_batch})\n",
" acc_train = accuracy.eval(feed_dict={is_training: False, X: X_batch, y: y_batch})\n",
" acc_test = accuracy.eval(feed_dict={is_training: False, X: mnist.test.images, y: mnist.test.labels})\n",
" print(epoch, \"Train accuracy:\", acc_train, \"Test accuracy:\", acc_test)\n",
"\n",
" save_path = saver.save(sess, \"my_model_final.ckpt\")"
]
},
{
"cell_type": "code",
"execution_count": 32,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"train_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,\n",
" scope=\"hidden[2]|outputs\")"
]
},
{
"cell_type": "code",
"execution_count": 33,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"training_op2 = optimizer.minimize(loss, var_list=train_vars)"
]
},
{
"cell_type": "code",
"execution_count": 34,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"for i in tf.all_variables():\n",
" print(i.name)"
]
},
{
"cell_type": "code",
"execution_count": 35,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"for i in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES):\n",
" print(i.name)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"for i in train_vars:\n",
" print(i.name)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"X_train = mnist.train.images\n",
"y_train = mnist.train.labels.astype(\"int\")\n",
"X_val = mnist.test.images[8000:]\n",
"y_val = mnist.test.labels[8000:].astype(\"int\")\n",
"\n",
"feature_columns = tf.contrib.learn.infer_real_valued_columns_from_input(X_train)\n",
"dnn_clf = tf.contrib.learn.DNNClassifier(\n",
" feature_columns = feature_columns,\n",
" hidden_units=[300, 100],\n",
" n_classes=10,\n",
" model_dir=\"/tmp/my_model\",\n",
" config=tf.contrib.learn.RunConfig(save_checkpoints_secs=60)\n",
" )\n",
"\n",
"validation_monitor = tf.contrib.learn.monitors.ValidationMonitor(\n",
" X_val,\n",
" y_val,\n",
" every_n_steps=50,\n",
" early_stopping_metric=\"loss\",\n",
" early_stopping_metric_minimize=True,\n",
" early_stopping_rounds=2000\n",
" )\n",
"\n",
"dnn_clf.fit(x=X_train,\n",
" y=y_train,\n",
" steps=40000,\n",
" monitors=[validation_monitor]\n",
" )\n"
]
},
{
"cell_type": "markdown",
"metadata": {
"collapsed": true
},
"source": [
"# Exercise solutions"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"**Coming soon**"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.5.1"
},
"nav_menu": {
"height": "360px",
"width": "416px"
},
"toc": {
"navigate_menu": true,
"number_sections": true,
"sideBar": true,
"threshold": 6,
"toc_cell": false,
"toc_section_display": "block",
"toc_window_display": false
}
},
"nbformat": 4,
"nbformat_minor": 0
}

494
12_distributed_tensorflow.ipynb Normal file
View File

@ -0,0 +1,494 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"**Chapter 12 – Distributed TensorFlow**"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"_This notebook contains all the sample code and solutions to the exercices in chapter 12._"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Setup"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"First, let's make sure this notebook works well in both python 2 and 3, import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures:"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"# To support both python 2 and python 3\n",
"from __future__ import division, print_function, unicode_literals\n",
"\n",
"# Common imports\n",
"import numpy as np\n",
"import numpy.random as rnd\n",
"import os\n",
"\n",
"# to make this notebook's output stable across runs\n",
"rnd.seed(42)\n",
"\n",
"# To plot pretty figures\n",
"%matplotlib inline\n",
"import matplotlib\n",
"import matplotlib.pyplot as plt\n",
"plt.rcParams['axes.labelsize'] = 14\n",
"plt.rcParams['xtick.labelsize'] = 12\n",
"plt.rcParams['ytick.labelsize'] = 12\n",
"\n",
"# Where to save the figures\n",
"PROJECT_ROOT_DIR = \".\"\n",
"CHAPTER_ID = \"distributed\"\n",
"\n",
"def save_fig(fig_id, tight_layout=True):\n",
" path = os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID, fig_id + \".png\")\n",
" print(\"Saving figure\", fig_id)\n",
" if tight_layout:\n",
" plt.tight_layout()\n",
" plt.savefig(path, format='png', dpi=300)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Local server"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"import tensorflow as tf"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"c = tf.constant(\"Hello distributed TensorFlow!\")\n",
"server = tf.train.Server.create_local_server()"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"with tf.Session(server.target) as sess:\n",
" print(sess.run(c))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Cluster"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"cluster_spec = tf.train.ClusterSpec({\n",
" \"ps\": [\n",
" \"127.0.0.1:2221\", # /job:ps/task:0\n",
" \"127.0.0.1:2222\", # /job:ps/task:1\n",
" ],\n",
" \"worker\": [\n",
" \"127.0.0.1:2223\", # /job:worker/task:0\n",
" \"127.0.0.1:2224\", # /job:worker/task:1\n",
" \"127.0.0.1:2225\", # /job:worker/task:2\n",
" ]})"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"task_ps0 = tf.train.Server(cluster_spec, job_name=\"ps\", task_index=0)\n",
"task_ps1 = tf.train.Server(cluster_spec, job_name=\"ps\", task_index=1)\n",
"task_worker0 = tf.train.Server(cluster_spec, job_name=\"worker\", task_index=0)\n",
"task_worker1 = tf.train.Server(cluster_spec, job_name=\"worker\", task_index=1)\n",
"task_worker2 = tf.train.Server(cluster_spec, job_name=\"worker\", task_index=2)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Pinning operations across devices and servers"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"tf.reset_default_graph()\n",
"\n",
"with tf.device(\"/job:ps\"):\n",
" a = tf.Variable(1.0, name=\"a\")\n",
"\n",
"with tf.device(\"/job:worker\"):\n",
" b = a + 2\n",
"\n",
"with tf.device(\"/job:worker/task:1\"):\n",
" c = a + b"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"with tf.Session(\"grpc://127.0.0.1:2221\") as sess:\n",
" sess.run(a.initializer)\n",
" print(c.eval())"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"tf.reset_default_graph()\n",
"\n",
"with tf.device(tf.train.replica_device_setter(\n",
" ps_tasks=2,\n",
" ps_device=\"/job:ps\",\n",
" worker_device=\"/job:worker\")):\n",
" v1 = tf.Variable(1.0, name=\"v1\") # pinned to /job:ps/task:0 (defaults to /cpu:0)\n",
" v2 = tf.Variable(2.0, name=\"v2\") # pinned to /job:ps/task:1 (defaults to /cpu:0)\n",
" v3 = tf.Variable(3.0, name=\"v3\") # pinned to /job:ps/task:0 (defaults to /cpu:0)\n",
" s = v1 + v2 # pinned to /job:worker (defaults to task:0/cpu:0)\n",
" with tf.device(\"/task:1\"):\n",
" p1 = 2 * s # pinned to /job:worker/task:1 (defaults to /cpu:0)\n",
" with tf.device(\"/cpu:0\"):\n",
" p2 = 3 * s # pinned to /job:worker/task:1/cpu:0\n",
"\n",
"config = tf.ConfigProto()\n",
"config.log_device_placement = True\n",
"\n",
"with tf.Session(\"grpc://127.0.0.1:2221\", config=config) as sess:\n",
" v1.initializer.run()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Readers"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"tf.reset_default_graph()\n",
"\n",
"test_csv = open(\"my_test.csv\", \"w\")\n",
"test_csv.write(\"x1, x2 , target\\n\")\n",
"test_csv.write(\"1., , 0\\n\")\n",
"test_csv.write(\"4., 5. , 1\\n\")\n",
"test_csv.write(\"7., 8. , 0\\n\")\n",
"test_csv.close()\n",
"\n",
"filename_queue = tf.FIFOQueue(capacity=10, dtypes=[tf.string], shapes=[()])\n",
"filename = tf.placeholder(tf.string)\n",
"enqueue_filename = filename_queue.enqueue([filename])\n",
"close_filename_queue = filename_queue.close()\n",
"\n",
"reader = tf.TextLineReader(skip_header_lines=1)\n",
"key, value = reader.read(filename_queue)\n",
"\n",
"x1, x2, target = tf.decode_csv(value, record_defaults=[[-1.], [-1.], [-1]])\n",
"features = tf.pack([x1, x2])\n",
"\n",
"instance_queue = tf.RandomShuffleQueue(\n",
" capacity=10, min_after_dequeue=2,\n",
" dtypes=[tf.float32, tf.int32], shapes=[[2],[]],\n",
" name=\"instance_q\", shared_name=\"shared_instance_q\")\n",
"enqueue_instance = instance_queue.enqueue([features, target])\n",
"close_instance_queue = instance_queue.close()\n",
"\n",
"minibatch_instances, minibatch_targets = instance_queue.dequeue_up_to(2)\n",
"\n",
"with tf.Session() as sess:\n",
" sess.run(enqueue_filename, feed_dict={filename: \"my_test.csv\"})\n",
" sess.run(close_filename_queue)\n",
" try:\n",
" while True:\n",
" sess.run(enqueue_instance)\n",
" except tf.errors.OutOfRangeError as ex:\n",
" print(\"No more files to read\")\n",
" sess.run(close_instance_queue)\n",
" try:\n",
" while True:\n",
" print(sess.run([minibatch_instances, minibatch_targets]))\n",
" except tf.errors.OutOfRangeError as ex:\n",
" print(\"No more training instances\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"#coord = tf.train.Coordinator()\n",
"#threads = tf.train.start_queue_runners(coord=coord)\n",
"#filename_queue = tf.train.string_input_producer([\"test.csv\"])\n",
"#coord.request_stop()\n",
"#coord.join(threads)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Queue runners and coordinators"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"tf.reset_default_graph()\n",
"\n",
"filename_queue = tf.FIFOQueue(capacity=10, dtypes=[tf.string], shapes=[()])\n",
"filename = tf.placeholder(tf.string)\n",
"enqueue_filename = filename_queue.enqueue([filename])\n",
"close_filename_queue = filename_queue.close()\n",
"\n",
"reader = tf.TextLineReader(skip_header_lines=1)\n",
"key, value = reader.read(filename_queue)\n",
"\n",
"x1, x2, target = tf.decode_csv(value, record_defaults=[[-1.], [-1.], [-1]])\n",
"features = tf.pack([x1, x2])\n",
"\n",
"instance_queue = tf.RandomShuffleQueue(\n",
" capacity=10, min_after_dequeue=2,\n",
" dtypes=[tf.float32, tf.int32], shapes=[[2],[]],\n",
" name=\"instance_q\", shared_name=\"shared_instance_q\")\n",
"enqueue_instance = instance_queue.enqueue([features, target])\n",
"close_instance_queue = instance_queue.close()\n",
"\n",
"minibatch_instances, minibatch_targets = instance_queue.dequeue_up_to(2)\n",
"\n",
"n_threads = 5\n",
"queue_runner = tf.train.QueueRunner(instance_queue, [enqueue_instance] * n_threads)\n",
"coord = tf.train.Coordinator()\n",
"\n",
"with tf.Session() as sess:\n",
" sess.run(enqueue_filename, feed_dict={filename: \"my_test.csv\"})\n",
" sess.run(close_filename_queue)\n",
" enqueue_threads = queue_runner.create_threads(sess, coord=coord, start=True)\n",
" try:\n",
" while True:\n",
" print(sess.run([minibatch_instances, minibatch_targets]))\n",
" except tf.errors.OutOfRangeError as ex:\n",
" print(\"No more training instances\")"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"tf.reset_default_graph()\n",
"\n",
"def read_and_push_instance(filename_queue, instance_queue):\n",
" reader = tf.TextLineReader(skip_header_lines=1)\n",
" key, value = reader.read(filename_queue)\n",
" x1, x2, target = tf.decode_csv(value, record_defaults=[[-1.], [-1.], [-1]])\n",
" features = tf.pack([x1, x2])\n",
" enqueue_instance = instance_queue.enqueue([features, target])\n",
" return enqueue_instance\n",
"\n",
"filename_queue = tf.FIFOQueue(capacity=10, dtypes=[tf.string], shapes=[()])\n",
"filename = tf.placeholder(tf.string)\n",
"enqueue_filename = filename_queue.enqueue([filename])\n",
"close_filename_queue = filename_queue.close()\n",
"\n",
"instance_queue = tf.RandomShuffleQueue(\n",
" capacity=10, min_after_dequeue=2,\n",
" dtypes=[tf.float32, tf.int32], shapes=[[2],[]],\n",
" name=\"instance_q\", shared_name=\"shared_instance_q\")\n",
"\n",
"minibatch_instances, minibatch_targets = instance_queue.dequeue_up_to(2)\n",
"\n",
"read_and_enqueue_ops = [read_and_push_instance(filename_queue, instance_queue) for i in range(5)]\n",
"queue_runner = tf.train.QueueRunner(instance_queue, read_and_enqueue_ops)\n",
"\n",
"with tf.Session() as sess:\n",
" sess.run(enqueue_filename, feed_dict={filename: \"my_test.csv\"})\n",
" sess.run(close_filename_queue)\n",
" coord = tf.train.Coordinator()\n",
" enqueue_threads = queue_runner.create_threads(sess, coord=coord, start=True)\n",
" try:\n",
" while True:\n",
" print(sess.run([minibatch_instances, minibatch_targets]))\n",
" except tf.errors.OutOfRangeError as ex:\n",
" print(\"No more training instances\")\n",
"\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Setting a timeout"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"tf.reset_default_graph()\n",
"\n",
"q = tf.FIFOQueue(capacity=10, dtypes=[tf.float32], shapes=[()])\n",
"v = tf.placeholder(tf.float32)\n",
"enqueue = q.enqueue([v])\n",
"dequeue = q.dequeue()\n",
"output = dequeue + 1\n",
"\n",
"config = tf.ConfigProto()\n",
"config.operation_timeout_in_ms = 1000\n",
"\n",
"with tf.Session(config=config) as sess:\n",
" sess.run(enqueue, feed_dict={v: 1.0})\n",
" sess.run(enqueue, feed_dict={v: 2.0})\n",
" sess.run(enqueue, feed_dict={v: 3.0})\n",
" print(sess.run(output))\n",
" print(sess.run(output, feed_dict={dequeue: 5}))\n",
" print(sess.run(output))\n",
" print(sess.run(output))\n",
" try:\n",
" print(sess.run(output))\n",
" except tf.errors.DeadlineExceededError as ex:\n",
" print(\"Timed out while dequeuing\")\n"
]
},
{
"cell_type": "markdown",
"metadata": {
"collapsed": true
},
"source": [
"# Exercise solutions"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"**Coming soon**"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.5.1"
},
"nav_menu": {},
"toc": {
"navigate_menu": true,
"number_sections": true,
"sideBar": true,
"threshold": 6,
"toc_cell": false,
"toc_section_display": "block",
"toc_window_display": false
}
},
"nbformat": 4,
"nbformat_minor": 0
}

613
13_convolutional_neural_networks.ipynb Normal file
View File

@ -0,0 +1,613 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"**Chapter 13 – Convolutional Neural Networks**"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"_This notebook contains all the sample code and solutions to the exercices in chapter 13._"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Setup"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"First, let's make sure this notebook works well in both python 2 and 3, import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures:"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"# To support both python 2 and python 3\n",
"from __future__ import division, print_function, unicode_literals\n",
"\n",
"# Common imports\n",
"import numpy as np\n",
"import numpy.random as rnd\n",
"import os\n",
"\n",
"# to make this notebook's output stable across runs\n",
"rnd.seed(42)\n",
"\n",
"# To plot pretty figures\n",
"%matplotlib inline\n",
"import matplotlib\n",
"import matplotlib.pyplot as plt\n",
"plt.rcParams['axes.labelsize'] = 14\n",
"plt.rcParams['xtick.labelsize'] = 12\n",
"plt.rcParams['ytick.labelsize'] = 12\n",
"\n",
"# Where to save the figures\n",
"PROJECT_ROOT_DIR = \".\"\n",
"CHAPTER_ID = \"cnn\"\n",
"\n",
"def save_fig(fig_id, tight_layout=True):\n",
" path = os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID, fig_id + \".png\")\n",
" print(\"Saving figure\", fig_id)\n",
" if tight_layout:\n",
" plt.tight_layout()\n",
" plt.savefig(path, format='png', dpi=300)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"A couple utility functions to plot grayscale and RGB images:"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"def plot_image(image):\n",
" plt.imshow(image, cmap=\"gray\", interpolation=\"nearest\")\n",
" plt.axis(\"off\")\n",
"\n",
"def plot_color_image(image):\n",
" plt.imshow(image.astype(np.uint8),interpolation=\"nearest\")\n",
" plt.axis(\"off\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"And of course we will need TensorFlow:"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"import tensorflow as tf"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Convolutional layer"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"from sklearn.datasets import load_sample_images\n",
"dataset = load_sample_images()\n",
"china, flower = dataset.images\n",
"image = china[150:220, 130:250]\n",
"height, width, channels = image.shape\n",
"image_grayscale = image.mean(axis=2).astype(np.float32)\n",
"images = image_grayscale.reshape(1, height, width, 1)"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"fmap = np.zeros(shape=(7, 7, 1, 2), dtype=np.float32)\n",
"fmap[:, 3, 0, 0] = 1\n",
"fmap[3, :, 0, 1] = 1\n",
"fmap[:, :, 0, 0]\n",
"plot_image(fmap[:, :, 0, 0])\n",
"plt.show()\n",
"plot_image(fmap[:, :, 0, 1])\n",
"plt.show()"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"tf.reset_default_graph()\n",
"\n",
"X = tf.placeholder(tf.float32, shape=(None, height, width, 1))\n",
"feature_maps = tf.constant(fmap)\n",
"convolution = tf.nn.conv2d(X, feature_maps, strides=[1,1,1,1], padding=\"SAME\", use_cudnn_on_gpu=False)"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"with tf.Session() as sess:\n",
" output = convolution.eval(feed_dict={X: images})"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"plot_image(images[0, :, :, 0])\n",
"save_fig(\"china_original\", tight_layout=False)\n",
"plt.show()"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"plot_image(output[0, :, :, 0])\n",
"save_fig(\"china_vertical\", tight_layout=False)\n",
"plt.show()"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"plot_image(output[0, :, :, 1])\n",
"save_fig(\"china_horizontal\", tight_layout=False)\n",
"plt.show()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Simple example"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"from sklearn.datasets import load_sample_images\n",
"dataset = np.array(load_sample_images().images, dtype=np.float32)\n",
"batch_size, height, width, channels = dataset.shape\n",
"\n",
"filters = np.zeros(shape=(7, 7, channels, 2), dtype=np.float32)\n",
"filters[:, 3, :, 0] = 1 # vertical line\n",
"filters[3, :, :, 1] = 1 # horizontal line\n",
"\n",
"X = tf.placeholder(tf.float32, shape=(None, height, width, channels))\n",
"convolution = tf.nn.conv2d(X, filters, strides=[1,2,2,1], padding=\"SAME\")\n",
"\n",
"with tf.Session() as sess:\n",
" output = sess.run(convolution, feed_dict={X: dataset})\n",
"\n",
"for image_index in (0, 1):\n",
" for feature_map_index in (0, 1):\n",
" plot_image(output[image_index, :, :, feature_map_index])\n",
" plt.show()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## VALID vs SAME padding"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"tf.reset_default_graph()\n",
"\n",
"filter_primes = np.array([2., 3., 5., 7., 11., 13.], dtype=np.float32)\n",
"x = tf.constant(np.arange(1, 13+1, dtype=np.float32).reshape([1, 1, 13, 1]))\n",
"filters = tf.constant(filter_primes.reshape(1, 6, 1, 1))\n",
"\n",
"valid_conv = tf.nn.conv2d(x, filters, strides=[1, 1, 5, 1], padding='VALID')\n",
"same_conv = tf.nn.conv2d(x, filters, strides=[1, 1, 5, 1], padding='SAME')\n",
"\n",
"with tf.Session() as sess:\n",
" print(\"VALID:\\n\", valid_conv.eval())\n",
" print(\"SAME:\\n\", same_conv.eval())"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"print(\"VALID:\")\n",
"print(np.array([1,2,3,4,5,6]).T.dot(filter_primes))\n",
"print(np.array([6,7,8,9,10,11]).T.dot(filter_primes))\n",
"print(\"SAME:\")\n",
"print(np.array([0,1,2,3,4,5]).T.dot(filter_primes))\n",
"print(np.array([5,6,7,8,9,10]).T.dot(filter_primes))\n",
"print(np.array([10,11,12,13,0,0]).T.dot(filter_primes))\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Pooling layer"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"from sklearn.datasets import load_sample_images\n",
"dataset = np.array(load_sample_images().images, dtype=np.float32)\n",
"batch_size, height, width, channels = dataset.shape\n",
"\n",
"filters = np.zeros(shape=(7, 7, channels, 2), dtype=np.float32)\n",
"filters[:, 3, :, 0] = 1 # vertical line\n",
"filters[3, :, :, 1] = 1 # horizontal line\n",
"\n",
"X = tf.placeholder(tf.float32, shape=(None, height, width, channels))\n",
"max_pool = tf.nn.max_pool(X, ksize=[1, 2, 2, 1], strides=[1,2,2,1], padding=\"VALID\")\n",
"\n",
"with tf.Session() as sess:\n",
" output = sess.run(max_pool, feed_dict={X: dataset})\n",
"\n",
"plot_color_image(dataset[0])\n",
"save_fig(\"china_original\")\n",
"plt.show()\n",
" \n",
"plot_color_image(output[0])\n",
"save_fig(\"china_max_pool\")\n",
"plt.show()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# MNIST"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"from sklearn.datasets import fetch_mldata\n",
"\n",
"mnist = fetch_mldata('MNIST original')\n",
"X_train, X_test = mnist[\"data\"][:60000].astype(np.float64), mnist[\"data\"][60000:].astype(np.float64)\n",
"y_train, y_test = mnist[\"target\"][:60000].astype(np.int64), mnist[\"target\"][60000:].astype(np.int64)"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"height, width = 28, 28\n",
"images = X_test[5000].reshape(1, height, width, 1)\n",
"plot_image(images[0, :, :, 0])"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Inception v3"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"import os\n",
"import sys\n",
"import tarfile\n",
"import urllib.request\n",
"\n",
"TF_MODELS_URL = \"http://download.tensorflow.org/models\"\n",
"INCEPTION_V3_URL = TF_MODELS_URL + \"/inception_v3_2016_08_28.tar.gz\"\n",
"INCEPTION_PATH = os.path.join(\"datasets\", \"inception\")\n",
"INCEPTION_V3_CHECKPOINT_PATH = os.path.join(INCEPTION_PATH, \"inception_v3.ckpt\")\n",
"\n",
"def download_progress(count, block_size, total_size):\n",
" percent = count * block_size * 100 // total_size\n",
" sys.stdout.write(\"\\rDownloading: {}%\".format(percent))\n",
" sys.stdout.flush()\n",
"\n",
"def fetch_pretrained_inception_v3(url=INCEPTION_V3_URL, path=INCEPTION_PATH):\n",
" if os.path.exists(INCEPTION_V3_CHECKPOINT_PATH):\n",
" return\n",
" os.makedirs(path, exist_ok=True)\n",
" tgz_path = os.path.join(path, \"inception_v3.tgz\")\n",
" urllib.request.urlretrieve(url, tgz_path, reporthook=download_progress)\n",
" inception_tgz = tarfile.open(tgz_path)\n",
" inception_tgz.extractall(path=path)\n",
" inception_tgz.close()\n",
" os.remove(tgz_path)"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"fetch_pretrained_inception_v3()"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"import re\n",
"\n",
"CLASS_NAME_REGEX = re.compile(r\"^n\\d+\\s+(.*)\\s*$\", re.M | re.U)\n",
"\n",
"def load_class_names():\n",
" with open(os.path.join(\"datasets\",\"inception\",\"imagenet_class_names.txt\"), \"rb\") as f:\n",
" content = f.read().decode(\"utf-8\")\n",
" return CLASS_NAME_REGEX.findall(content)"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"class_names = load_class_names()"
]
},
{
"cell_type": "code",
"execution_count": 21,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"width = 299\n",
"height = 299\n",
"channels = 3"
]
},
{
"cell_type": "code",
"execution_count": 22,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"import matplotlib.image as mpimg\n",
"test_image = mpimg.imread(os.path.join(\"images\",\"cnn\",\"test_image.png\"))[:, :, :channels]\n",
"plt.imshow(test_image)\n",
"plt.axis(\"off\")\n",
"plt.show()"
]
},
{
"cell_type": "code",
"execution_count": 23,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"import tensorflow as tf\n",
"from nets.inception_v3 import inception_v3, inception_v3_arg_scope\n",
"import tensorflow.contrib.slim as slim\n",
"\n",
"tf.reset_default_graph()\n",
"\n",
"X = tf.placeholder(tf.float32, shape=[None, height, width, channels], name=\"X\")\n",
"with slim.arg_scope(inception_v3_arg_scope()):\n",
" logits, end_points = inception_v3(X, num_classes=1001, is_training=False)\n",
"predictions = end_points[\"Predictions\"]\n",
"saver = tf.train.Saver()"
]
},
{
"cell_type": "code",
"execution_count": 24,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"X_test = test_image.reshape(-1, height, width, channels)\n",
"\n",
"with tf.Session() as sess:\n",
" saver.restore(sess, INCEPTION_V3_CHECKPOINT_PATH)\n",
" predictions_val = predictions.eval(feed_dict={X: X_test})"
]
},
{
"cell_type": "code",
"execution_count": 25,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"class_names[np.argmax(predictions_val[0])]"
]
},
{
"cell_type": "code",
"execution_count": 26,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"np.argmax(predictions_val, axis=1)"
]
},
{
"cell_type": "code",
"execution_count": 27,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"top_5 = np.argpartition(predictions_val[0], -5)[-5:]\n",
"top_5 = top_5[np.argsort(predictions_val[0][top_5])]\n",
"for i in top_5:\n",
" print(\"{0}: {1:.2f}%\".format(class_names[i], 100*predictions_val[0][i]))"
]
},
{
"cell_type": "markdown",
"metadata": {
"collapsed": true
},
"source": [
"# Exercise solutions"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"**Coming soon**"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.5.1"
},
"nav_menu": {},
"toc": {
"navigate_menu": true,
"number_sections": true,
"sideBar": true,
"threshold": 6,
"toc_cell": false,
"toc_section_display": "block",
"toc_window_display": false
}
},
"nbformat": 4,
"nbformat_minor": 0
}

1326
14_recurrent_neural_networks.ipynb Normal file
View File

File diff suppressed because it is too large Load Diff

68
classification.ipynb
View File

@ -4,7 +4,23 @@
"cell_type": "markdown", "cell_type": "markdown",
"metadata": {}, "metadata": {},
"source": [ "source": [
"**Classification**" "**Chapter 3 – Classification**\n",
"\n",
"_This notebook contains all the sample code and solutions to the exercices in chapter 3._"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Setup"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"First, let's make sure this notebook works well in both python 2 and 3, import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures:"
] ]
}, },
{ {
@ -15,14 +31,18 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"# To support both python 2 and python 3\n",
"from __future__ import division, print_function, unicode_literals\n", "from __future__ import division, print_function, unicode_literals\n",
"\n", "\n",
"# Common imports\n",
"import numpy as np\n", "import numpy as np\n",
"import numpy.random as rnd\n", "import numpy.random as rnd\n",
"rnd.seed(42) # to make this notebook's output stable across runs\n",
"\n",
"import os\n", "import os\n",
"\n", "\n",
"# to make this notebook's output stable across runs\n",
"rnd.seed(42)\n",
"\n",
"# To plot pretty figures\n",
"%matplotlib inline\n", "%matplotlib inline\n",
"import matplotlib\n", "import matplotlib\n",
"import matplotlib.pyplot as plt\n", "import matplotlib.pyplot as plt\n",
@ -30,6 +50,7 @@
"plt.rcParams['xtick.labelsize'] = 12\n", "plt.rcParams['xtick.labelsize'] = 12\n",
"plt.rcParams['ytick.labelsize'] = 12\n", "plt.rcParams['ytick.labelsize'] = 12\n",
"\n", "\n",
"# Where to save the figures\n",
"PROJECT_ROOT_DIR = \".\"\n", "PROJECT_ROOT_DIR = \".\"\n",
"CHAPTER_ID = \"classification\"\n", "CHAPTER_ID = \"classification\"\n",
"\n", "\n",
@ -122,7 +143,7 @@
"some_digit_index = 36000\n", "some_digit_index = 36000\n",
"some_digit = X[some_digit_index]\n", "some_digit = X[some_digit_index]\n",
"plot_digit(some_digit)\n", "plot_digit(some_digit)\n",
"save_fig(\"some_digit\")\n", "save_fig(\"some_digit_plot\")\n",
"plt.show()" "plt.show()"
] ]
}, },
@ -153,7 +174,7 @@
"plt.figure(figsize=(9,9))\n", "plt.figure(figsize=(9,9))\n",
"example_images = np.r_[X[:12000:600], X[13000:30600:600], X[30600:60000:590]]\n", "example_images = np.r_[X[:12000:600], X[13000:30600:600], X[30600:60000:590]]\n",
"plot_digits(example_images, images_per_row=10)\n", "plot_digits(example_images, images_per_row=10)\n",
"save_fig(\"more_digits\")\n", "save_fig(\"more_digits_plot\")\n",
"plt.show()" "plt.show()"
] ]
}, },
@ -980,7 +1001,7 @@
"some_index = 5500\n", "some_index = 5500\n",
"plt.subplot(121); plot_digit(X_test_mod[some_index])\n", "plt.subplot(121); plot_digit(X_test_mod[some_index])\n",
"plt.subplot(122); plot_digit(y_test_mod[some_index])\n", "plt.subplot(122); plot_digit(y_test_mod[some_index])\n",
"save_fig(\"noisy_digit_example\")\n", "save_fig(\"noisy_digit_example_plot\")\n",
"plt.show()" "plt.show()"
] ]
}, },
@ -1005,7 +1026,7 @@
"source": [ "source": [
"clean_digit = knn_clf.predict([X_test_mod[some_index]])\n", "clean_digit = knn_clf.predict([X_test_mod[some_index]])\n",
"plot_digit(clean_digit)\n", "plot_digit(clean_digit)\n",
"save_fig(\"cleaned_digit_example\")\n", "save_fig(\"cleaned_digit_example_plot\")\n",
"plt.show()" "plt.show()"
] ]
}, },
@ -1183,6 +1204,31 @@
"source": [ "source": [
"plot_digit(ambiguous_digit)" "plot_digit(ambiguous_digit)"
] ]
},
{
"cell_type": "markdown",
"metadata": {
"collapsed": true
},
"source": [
"# Exercise solutions"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"**Coming soon**"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": []
} }
], ],
"metadata": { "metadata": {
@ -1203,10 +1249,14 @@
"pygments_lexer": "ipython3", "pygments_lexer": "ipython3",
"version": "3.5.1" "version": "3.5.1"
}, },
"nav_menu": {},
"toc": { "toc": {
"navigate_menu": true,
"number_sections": true,
"sideBar": true,
"threshold": 6,
"toc_cell": false, "toc_cell": false,
"toc_number_sections": true, "toc_section_display": "block",
"toc_threshold": 6,
"toc_window_display": false "toc_window_display": false
} }
}, },

1
images/ann/README Normal file
View File

@ -0,0 +1 @@
Images generated by the notebooks

1
images/autoencoders/README Normal file
View File

@ -0,0 +1 @@
Images generated by the notebooks

1
images/cnn/README Normal file
View File

@ -0,0 +1 @@
Images generated by the notebooks

BIN
images/cnn/test_image.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 178 KiB

1
images/decision_trees/README Normal file
View File

@ -0,0 +1 @@
Images generated by the notebooks

1
images/deep/README Normal file
View File

@ -0,0 +1 @@
Images generated by the notebooks

1
images/dim_reduction/README Normal file
View File

@ -0,0 +1 @@
Images generated by the notebooks

1
images/distributed/README Normal file
View File

@ -0,0 +1 @@
Images generated by the notebooks

1
images/ensembles/README Normal file
View File

@ -0,0 +1 @@
Images generated by the notebooks

1
images/rl/README Normal file
View File

@ -0,0 +1 @@
Images generated by the notebooks

1
images/rnn/README Normal file
View File

@ -0,0 +1 @@
Images generated by the notebooks

1
images/svm/README Normal file
View File

@ -0,0 +1 @@
Images generated by the notebooks

1
images/tensorflow/README Normal file
View File

@ -0,0 +1 @@
Images generated by the notebooks

45
index.ipynb
View File

@ -16,19 +16,29 @@
"\n", "\n",
"### To run the examples\n", "### To run the examples\n",
"* **Jupyter** – These notebooks are based on Jupyter. If you just plan to read without running any code, there's really nothing more to know, just keep reading! But if you want to experiment with the code examples you need to:\n", "* **Jupyter** – These notebooks are based on Jupyter. If you just plan to read without running any code, there's really nothing more to know, just keep reading! But if you want to experiment with the code examples you need to:\n",
" * open these notebooks in Jupyter. If you clicked on the \"launch binder\" button in github or followed the Installation instructions, then you are good to go. If not you will need to go back to the project [home page](https://github.com/ageron/ml-notebooks/) and click on \"launch binder\" or follow the installation instructions.\n", " * open these notebooks in Jupyter. If you clicked on the \"launch binder\" button in github or followed the Installation instructions, then you are good to go. If not you will need to go back to the project [home page](https://github.com/ageron/handson-ml/) and click on \"launch binder\" or follow the installation instructions.\n",
" * learn how to use Jupyter. Start the User Interface Tour from the Help menu.\n", " * learn how to use Jupyter. Start the User Interface Tour from the Help menu.\n",
"\n", "\n",
"### To activate extensions\n", "### To activate extensions\n",
"* If this is an interactive session (see above), you may want to turn on a few Jupyter extensions by going to the [Extension Configuration](../nbextensions/) page. In particular the \"*table of contents (2)*\" extension is quite useful.\n", "* If this is an interactive session (see above), you may want to turn on a few Jupyter extensions by going to the [Extension Configuration](../nbextensions/) page. In particular the \"*Table of Contents (2)*\" extension is quite useful.\n",
"\n", "\n",
"## Chapters\n", "## Notebooks\n",
"1. [Fundamentals](fundamentals.ipynb)\n", "1. [The Machine Learning landscape](01_the_machine_learning_landscape.ipynb)\n",
"2. [End-to-end project](end_to_end_project.ipynb)\n", "2. [End-to-end Machine Learning project](02_end_to_end_machine_learning_project.ipynb)\n",
"3. [Classification](classification.ipynb)\n", "3. [Classification](03_classification.ipynb)\n",
"4. [Training Linear Models](training_linear_models.ipynb)\n", "4. [Training Linear Models](04_training_linear_models.ipynb)\n",
"\n", "5. [Support Vector Machines](05_support_vector_machines.ipynb)\n",
"More explanations and chapters coming soon.\n", "6. [Decision Trees](06_decision_trees.ipynb)\n",
"7. [Ensemble Learning and Random Forests](07_ensemble_learning_and_random_forests.ipynb)\n",
"8. [Dimensionality Reduction](08_dimensionality_reduction.ipynb)\n",
"9. [Up and running with TensorFlow](09_up_and_running_with_tensorflow.ipynb)\n",
"10. [Introduction to Artificial Neural Networks](10_introduction_to_artificial_neural_networks.ipynb)\n",
"11. [Deep Learning](11_deep_learning.ipynb)\n",
"12. [Distributed TensorFlow](12_distributed_tensorflow.ipynb)\n",
"13. [Convolutional Neural Networks](13_convolutional_neural_networks.ipynb)\n",
"14. [Recurrent Neural Networks](14_recurrent_neural_networks.ipynb)\n",
"15. Autoencoders (coming soon)\n",
"16. Reinforcement Learning (coming soon)\n",
"\n", "\n",
"## Scientific Python tutorials\n", "## Scientific Python tutorials\n",
"* [NumPy](tools_numpy.ipynb)\n", "* [NumPy](tools_numpy.ipynb)\n",
@ -39,6 +49,15 @@
"* [Linear Algebra](math_linear_algebra.ipynb)\n", "* [Linear Algebra](math_linear_algebra.ipynb)\n",
"* Calculus (coming soon)" "* Calculus (coming soon)"
] ]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": []
} }
], ],
"metadata": { "metadata": {
@ -59,10 +78,14 @@
"pygments_lexer": "ipython2", "pygments_lexer": "ipython2",
"version": "2.7.11" "version": "2.7.11"
}, },
"nav_menu": {},
"toc": { "toc": {
"navigate_menu": true,
"number_sections": true,
"sideBar": true,
"threshold": 6,
"toc_cell": false, "toc_cell": false,
"toc_number_sections": true, "toc_section_display": "block",
"toc_threshold": 6,
"toc_window_display": false "toc_window_display": false
} }
}, },

10
nets/inception_v3.py
View File

@ -94,7 +94,9 @@ def inception_v3_base(inputs,
raise ValueError('depth_multiplier is not greater than zero.') raise ValueError('depth_multiplier is not greater than zero.')
depth = lambda d: max(int(d * depth_multiplier), min_depth) depth = lambda d: max(int(d * depth_multiplier), min_depth)
with tf.variable_scope(scope, 'InceptionV3', [inputs]): #Backported to 0.10.0
#with tf.variable_scope(scope, 'InceptionV3', [inputs]):
with tf.variable_scope(scope or 'InceptionV3'):
with slim.arg_scope([slim.conv2d, slim.max_pool2d, slim.avg_pool2d], with slim.arg_scope([slim.conv2d, slim.max_pool2d, slim.avg_pool2d],
stride=1, padding='VALID'): stride=1, padding='VALID'):
# 299 x 299 x 3 # 299 x 299 x 3
@ -470,8 +472,10 @@ def inception_v3(inputs,
raise ValueError('depth_multiplier is not greater than zero.') raise ValueError('depth_multiplier is not greater than zero.')
depth = lambda d: max(int(d * depth_multiplier), min_depth) depth = lambda d: max(int(d * depth_multiplier), min_depth)
with tf.variable_scope(scope, 'InceptionV3', [inputs, num_classes], #Backported to 0.10.0
reuse=reuse) as scope: #with tf.variable_scope(scope, 'InceptionV3', [inputs, num_classes],
# reuse=reuse) as scope:
with tf.variable_scope(scope or 'InceptionV3', reuse=reuse) as scope:
with slim.arg_scope([slim.batch_norm, slim.dropout], with slim.arg_scope([slim.batch_norm, slim.dropout],
is_training=is_training): is_training=is_training):
net, end_points = inception_v3_base( net, end_points = inception_v3_base(