Fix titles in figure 7-8 (learning rates should be 1 and 0.5)

main
Aurélien Geron 2017-09-15 16:41:15 +02:00
parent 43c95561da
commit 46c05c8ed3
1 changed files with 65 additions and 276 deletions

View File

@ -2,40 +2,28 @@
"cells": [
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"metadata": {},
"source": [
"**Chapter 7 Ensemble Learning and Random Forests**"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"metadata": {},
"source": [
"_This notebook contains all the sample code and solutions to the exercices in chapter 7._"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"metadata": {},
"source": [
"# Setup"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"metadata": {},
"source": [
"First, let's make sure this notebook works well in both python 2 and 3, import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures:"
]
@ -44,9 +32,7 @@
"cell_type": "code",
"execution_count": 1,
"metadata": {
"collapsed": true,
"deletable": true,
"editable": true
"collapsed": true
},
"outputs": [],
"source": [
@ -84,10 +70,7 @@
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"metadata": {},
"source": [
"# Voting classifiers"
]
@ -95,11 +78,7 @@
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"metadata": {},
"outputs": [],
"source": [
"heads_proba = 0.51\n",
@ -110,11 +89,7 @@
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"metadata": {},
"outputs": [],
"source": [
"plt.figure(figsize=(8,3.5))\n",
@ -133,9 +108,7 @@
"cell_type": "code",
"execution_count": 4,
"metadata": {
"collapsed": true,
"deletable": true,
"editable": true
"collapsed": true
},
"outputs": [],
"source": [
@ -149,11 +122,7 @@
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"metadata": {},
"outputs": [],
"source": [
"from sklearn.ensemble import RandomForestClassifier\n",
@ -174,11 +143,7 @@
{
"cell_type": "code",
"execution_count": 6,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"metadata": {},
"outputs": [],
"source": [
"from sklearn.metrics import accuracy_score\n",
@ -192,11 +157,7 @@
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"metadata": {},
"outputs": [],
"source": [
"log_clf = LogisticRegression(random_state=42)\n",
@ -212,11 +173,7 @@
{
"cell_type": "code",
"execution_count": 8,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"metadata": {},
"outputs": [],
"source": [
"from sklearn.metrics import accuracy_score\n",
@ -229,10 +186,7 @@
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"metadata": {},
"source": [
"# Bagging ensembles"
]
@ -241,9 +195,7 @@
"cell_type": "code",
"execution_count": 9,
"metadata": {
"collapsed": true,
"deletable": true,
"editable": true
"collapsed": true
},
"outputs": [],
"source": [
@ -260,11 +212,7 @@
{
"cell_type": "code",
"execution_count": 10,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"metadata": {},
"outputs": [],
"source": [
"from sklearn.metrics import accuracy_score\n",
@ -274,11 +222,7 @@
{
"cell_type": "code",
"execution_count": 11,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"metadata": {},
"outputs": [],
"source": [
"tree_clf = DecisionTreeClassifier(random_state=42)\n",
@ -291,9 +235,7 @@
"cell_type": "code",
"execution_count": 12,
"metadata": {
"collapsed": true,
"deletable": true,
"editable": true
"collapsed": true
},
"outputs": [],
"source": [
@ -320,11 +262,7 @@
{
"cell_type": "code",
"execution_count": 13,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"metadata": {},
"outputs": [],
"source": [
"plt.figure(figsize=(11,4))\n",
@ -340,10 +278,7 @@
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"metadata": {},
"source": [
"# Random Forests"
]
@ -351,11 +286,7 @@
{
"cell_type": "code",
"execution_count": 14,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"metadata": {},
"outputs": [],
"source": [
"bag_clf = BaggingClassifier(\n",
@ -366,11 +297,7 @@
{
"cell_type": "code",
"execution_count": 15,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"metadata": {},
"outputs": [],
"source": [
"bag_clf.fit(X_train, y_train)\n",
@ -381,9 +308,7 @@
"cell_type": "code",
"execution_count": 16,
"metadata": {
"collapsed": true,
"deletable": true,
"editable": true
"collapsed": true
},
"outputs": [],
"source": [
@ -398,11 +323,7 @@
{
"cell_type": "code",
"execution_count": 17,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"metadata": {},
"outputs": [],
"source": [
"np.sum(y_pred == y_pred_rf) / len(y_pred) # almost identical predictions"
@ -411,11 +332,7 @@
{
"cell_type": "code",
"execution_count": 18,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"metadata": {},
"outputs": [],
"source": [
"from sklearn.datasets import load_iris\n",
@ -429,11 +346,7 @@
{
"cell_type": "code",
"execution_count": 19,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"metadata": {},
"outputs": [],
"source": [
"rnd_clf.feature_importances_"
@ -442,11 +355,7 @@
{
"cell_type": "code",
"execution_count": 20,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"metadata": {},
"outputs": [],
"source": [
"plt.figure(figsize=(6, 4))\n",
@ -462,10 +371,7 @@
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"metadata": {},
"source": [
"## Out-of-Bag evaluation"
]
@ -473,11 +379,7 @@
{
"cell_type": "code",
"execution_count": 21,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"metadata": {},
"outputs": [],
"source": [
"bag_clf = BaggingClassifier(\n",
@ -490,11 +392,7 @@
{
"cell_type": "code",
"execution_count": 22,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"metadata": {},
"outputs": [],
"source": [
"bag_clf.oob_decision_function_"
@ -503,11 +401,7 @@
{
"cell_type": "code",
"execution_count": 23,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"metadata": {},
"outputs": [],
"source": [
"from sklearn.metrics import accuracy_score\n",
@ -517,10 +411,7 @@
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"metadata": {},
"source": [
"## Feature importance"
]
@ -529,9 +420,7 @@
"cell_type": "code",
"execution_count": 24,
"metadata": {
"collapsed": true,
"deletable": true,
"editable": true
"collapsed": true
},
"outputs": [],
"source": [
@ -542,11 +431,7 @@
{
"cell_type": "code",
"execution_count": 25,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"metadata": {},
"outputs": [],
"source": [
"rnd_clf = RandomForestClassifier(random_state=42)\n",
@ -557,9 +442,7 @@
"cell_type": "code",
"execution_count": 26,
"metadata": {
"collapsed": true,
"deletable": true,
"editable": true
"collapsed": true
},
"outputs": [],
"source": [
@ -573,11 +456,7 @@
{
"cell_type": "code",
"execution_count": 27,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"metadata": {},
"outputs": [],
"source": [
"plot_digit(rnd_clf.feature_importances_)\n",
@ -591,10 +470,7 @@
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"metadata": {},
"source": [
"# AdaBoost"
]
@ -602,11 +478,7 @@
{
"cell_type": "code",
"execution_count": 28,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"metadata": {},
"outputs": [],
"source": [
"from sklearn.ensemble import AdaBoostClassifier\n",
@ -620,11 +492,7 @@
{
"cell_type": "code",
"execution_count": 29,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"metadata": {},
"outputs": [],
"source": [
"plot_decision_boundary(ada_clf, X, y)"
@ -633,11 +501,7 @@
{
"cell_type": "code",
"execution_count": 30,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"metadata": {},
"outputs": [],
"source": [
"m = len(X_train)\n",
@ -652,7 +516,7 @@
" y_pred = svm_clf.predict(X_train)\n",
" sample_weights[y_pred != y_train] *= (1 + learning_rate)\n",
" plot_decision_boundary(svm_clf, X, y, alpha=0.2)\n",
" plt.title(\"learning_rate = {}\".format(learning_rate - 1), fontsize=16)\n",
" plt.title(\"learning_rate = {}\".format(learning_rate), fontsize=16)\n",
"\n",
"plt.subplot(121)\n",
"plt.text(-0.7, -0.65, \"1\", fontsize=14)\n",
@ -667,11 +531,7 @@
{
"cell_type": "code",
"execution_count": 31,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"metadata": {},
"outputs": [],
"source": [
"list(m for m in dir(ada_clf) if not m.startswith(\"_\") and m.endswith(\"_\"))"
@ -679,10 +539,7 @@
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"metadata": {},
"source": [
"# Gradient Boosting"
]
@ -691,9 +548,7 @@
"cell_type": "code",
"execution_count": 32,
"metadata": {
"collapsed": true,
"deletable": true,
"editable": true
"collapsed": true
},
"outputs": [],
"source": [
@ -705,11 +560,7 @@
{
"cell_type": "code",
"execution_count": 33,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"metadata": {},
"outputs": [],
"source": [
"from sklearn.tree import DecisionTreeRegressor\n",
@ -721,11 +572,7 @@
{
"cell_type": "code",
"execution_count": 34,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"metadata": {},
"outputs": [],
"source": [
"y2 = y - tree_reg1.predict(X)\n",
@ -736,11 +583,7 @@
{
"cell_type": "code",
"execution_count": 35,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"metadata": {},
"outputs": [],
"source": [
"y3 = y2 - tree_reg2.predict(X)\n",
@ -752,9 +595,7 @@
"cell_type": "code",
"execution_count": 36,
"metadata": {
"collapsed": true,
"deletable": true,
"editable": true
"collapsed": true
},
"outputs": [],
"source": [
@ -764,11 +605,7 @@
{
"cell_type": "code",
"execution_count": 37,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"metadata": {},
"outputs": [],
"source": [
"y_pred = sum(tree.predict(X_new) for tree in (tree_reg1, tree_reg2, tree_reg3))"
@ -777,11 +614,7 @@
{
"cell_type": "code",
"execution_count": 38,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"metadata": {},
"outputs": [],
"source": [
"y_pred"
@ -790,11 +623,7 @@
{
"cell_type": "code",
"execution_count": 39,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"metadata": {},
"outputs": [],
"source": [
"def plot_predictions(regressors, X, y, axes, label=None, style=\"r-\", data_style=\"b.\", data_label=None):\n",
@ -843,11 +672,7 @@
{
"cell_type": "code",
"execution_count": 40,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"metadata": {},
"outputs": [],
"source": [
"from sklearn.ensemble import GradientBoostingRegressor\n",
@ -859,11 +684,7 @@
{
"cell_type": "code",
"execution_count": 41,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"metadata": {},
"outputs": [],
"source": [
"gbrt_slow = GradientBoostingRegressor(max_depth=2, n_estimators=200, learning_rate=0.1, random_state=42)\n",
@ -873,11 +694,7 @@
{
"cell_type": "code",
"execution_count": 42,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"metadata": {},
"outputs": [],
"source": [
"plt.figure(figsize=(11,4))\n",
@ -896,10 +713,7 @@
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"metadata": {},
"source": [
"## Gradient Boosting with Early stopping"
]
@ -907,11 +721,7 @@
{
"cell_type": "code",
"execution_count": 43,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"metadata": {},
"outputs": [],
"source": [
"import numpy as np\n",
@ -935,9 +745,7 @@
"cell_type": "code",
"execution_count": 44,
"metadata": {
"collapsed": true,
"deletable": true,
"editable": true
"collapsed": true
},
"outputs": [],
"source": [
@ -947,11 +755,7 @@
{
"cell_type": "code",
"execution_count": 45,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"metadata": {},
"outputs": [],
"source": [
"plt.figure(figsize=(11, 4))\n",
@ -977,11 +781,7 @@
{
"cell_type": "code",
"execution_count": 46,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"metadata": {},
"outputs": [],
"source": [
"gbrt = GradientBoostingRegressor(max_depth=2, warm_start=True, random_state=42)\n",
@ -1005,11 +805,7 @@
{
"cell_type": "code",
"execution_count": 47,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"metadata": {},
"outputs": [],
"source": [
"print(gbrt.n_estimators)"
@ -1018,9 +814,7 @@
{
"cell_type": "markdown",
"metadata": {
"collapsed": true,
"deletable": true,
"editable": true
"collapsed": true
},
"source": [
"# Exercise solutions"
@ -1028,10 +822,7 @@
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"metadata": {},
"source": [
"**Coming soon**"
]
@ -1040,9 +831,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true,
"deletable": true,
"editable": true
"collapsed": true
},
"outputs": [],
"source": []
@ -1064,7 +853,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.5.3"
"version": "3.5.2"
},
"nav_menu": {
"height": "252px",
@ -1081,5 +870,5 @@
}
},
"nbformat": 4,
"nbformat_minor": 0
"nbformat_minor": 1
}