handson-ml/05_decision_trees.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Chapter 5 – Decision Trees**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_This notebook contains all the sample code and solutions to the exercises in chapter 5._"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<table align=\"left\">\n",
    "  <td>\n",
    "    <a href=\"https://colab.research.google.com/github/ageron/handson-ml2/blob/master/06_decision_trees.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>\n",
    "  </td>\n",
    "  <td>\n",
    "    <a target=\"_blank\" href=\"https://kaggle.com/kernels/welcome?src=https://github.com/ageron/handson-ml2/blob/master/06_decision_trees.ipynb\"><img src=\"https://kaggle.com/static/images/open-in-kaggle.svg\" /></a>\n",
    "  </td>\n",
    "</table>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Setup"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First, let's import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Python ≥3.8 is required\n",
    "import sys\n",
    "assert sys.version_info >= (3, 8)\n",
    "\n",
    "# Scikit-Learn ≥1.0 is required\n",
    "import sklearn\n",
    "assert sklearn.__version__ >= \"1.0\"\n",
    "\n",
    "# Common imports\n",
    "import numpy as np\n",
    "from pathlib import Path\n",
    "\n",
    "# to make this notebook's output stable across runs\n",
    "np.random.seed(42)\n",
    "\n",
    "# To plot pretty figures\n",
    "%matplotlib inline\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "mpl.rc('axes', labelsize=14)\n",
    "mpl.rc('xtick', labelsize=12)\n",
    "mpl.rc('ytick', labelsize=12)\n",
    "\n",
    "# Where to save the figures\n",
    "IMAGES_PATH = Path() / \"images\" / \"decision_trees\"\n",
    "IMAGES_PATH.mkdir(parents=True, exist_ok=True)\n",
    "\n",
    "def save_fig(fig_id, tight_layout=True, fig_extension=\"png\", resolution=300):\n",
    "    path = IMAGES_PATH / f\"{fig_id}.{fig_extension}\"\n",
    "    if tight_layout:\n",
    "        plt.tight_layout()\n",
    "    plt.savefig(path, format=fig_extension, dpi=resolution)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training and Visualizing a Decision Tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import load_iris\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "\n",
    "iris = load_iris()\n",
    "X = iris.data[:, 2:] # petal length and width\n",
    "y = iris.target\n",
    "\n",
    "tree_clf = DecisionTreeClassifier(max_depth=2, random_state=42)\n",
    "tree_clf.fit(X, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**This code example generates Figure 6–1. Iris Decision Tree:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from graphviz import Source\n",
    "from sklearn.tree import export_graphviz\n",
    "\n",
    "export_graphviz(\n",
    "        tree_clf,\n",
    "        out_file=IMAGES_PATH / \"iris_tree.dot\",\n",
    "        feature_names=iris.feature_names[2:],\n",
    "        class_names=iris.target_names,\n",
    "        rounded=True,\n",
    "        filled=True\n",
    "    )\n",
    "\n",
    "Source.from_file(IMAGES_PATH / \"iris_tree.dot\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Making Predictions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Code to generate Figure 6–2. Decision Tree decision boundaries**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from matplotlib.colors import ListedColormap\n",
    "\n",
    "def plot_decision_boundary(clf, X, y, axes=[0, 7.5, 0, 3], iris=True, legend=False, plot_training=True):\n",
    "    x1s = np.linspace(axes[0], axes[1], 100)\n",
    "    x2s = np.linspace(axes[2], axes[3], 100)\n",
    "    x1, x2 = np.meshgrid(x1s, x2s)\n",
    "    X_new = np.c_[x1.ravel(), x2.ravel()]\n",
    "    y_pred = clf.predict(X_new).reshape(x1.shape)\n",
    "    custom_cmap = ListedColormap(['#fafab0','#9898ff','#a0faa0'])\n",
    "    plt.contourf(x1, x2, y_pred, alpha=0.3, cmap=custom_cmap)\n",
    "    if not iris:\n",
    "        custom_cmap2 = ListedColormap(['#7d7d58','#4c4c7f','#507d50'])\n",
    "        plt.contour(x1, x2, y_pred, cmap=custom_cmap2, alpha=0.8)\n",
    "    if plot_training:\n",
    "        plt.plot(X[:, 0][y==0], X[:, 1][y==0], \"yo\", label=\"Iris setosa\")\n",
    "        plt.plot(X[:, 0][y==1], X[:, 1][y==1], \"bs\", label=\"Iris versicolor\")\n",
    "        plt.plot(X[:, 0][y==2], X[:, 1][y==2], \"g^\", label=\"Iris virginica\")\n",
    "        plt.axis(axes)\n",
    "    if iris:\n",
    "        plt.xlabel(\"Petal length\", fontsize=14)\n",
    "        plt.ylabel(\"Petal width\", fontsize=14)\n",
    "    else:\n",
    "        plt.xlabel(r\"$x_1$\", fontsize=18)\n",
    "        plt.ylabel(r\"$x_2$\", fontsize=18, rotation=0)\n",
    "    if legend:\n",
    "        plt.legend(loc=\"lower right\", fontsize=14)\n",
    "\n",
    "plt.figure(figsize=(8, 4))\n",
    "plot_decision_boundary(tree_clf, X, y)\n",
    "plt.plot([2.45, 2.45], [0, 3], \"k-\", linewidth=2)\n",
    "plt.plot([2.45, 7.5], [1.75, 1.75], \"k--\", linewidth=2)\n",
    "plt.plot([4.95, 4.95], [0, 1.75], \"k:\", linewidth=2)\n",
    "plt.plot([4.85, 4.85], [1.75, 3], \"k:\", linewidth=2)\n",
    "plt.text(1.40, 1.0, \"Depth=0\", fontsize=15)\n",
    "plt.text(3.2, 1.80, \"Depth=1\", fontsize=13)\n",
    "plt.text(4.05, 0.5, \"(Depth=2)\", fontsize=11)\n",
    "\n",
    "save_fig(\"decision_tree_decision_boundaries_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Estimating Class Probabilities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "tree_clf.predict_proba([[5, 1.5]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "tree_clf.predict([[5, 1.5]])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Regularization Hyperparameters"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We've seen that small changes in the dataset (such as a rotation) may produce a very different Decision Tree.\n",
    "Now let's show that training the same model on the same data may produce a very different model every time, since the CART training algorithm used by Scikit-Learn is stochastic. To show this, we will set `random_state` to a different value than earlier:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "tree_clf_tweaked = DecisionTreeClassifier(max_depth=2, random_state=40)\n",
    "tree_clf_tweaked.fit(X, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Code to generate Figure 6–8. Sensitivity to training set details:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(8, 4))\n",
    "plot_decision_boundary(tree_clf_tweaked, X, y, legend=False)\n",
    "plt.plot([0, 7.5], [0.8, 0.8], \"k-\", linewidth=2)\n",
    "plt.plot([0, 7.5], [1.75, 1.75], \"k--\", linewidth=2)\n",
    "plt.text(1.0, 0.9, \"Depth=0\", fontsize=15)\n",
    "plt.text(1.0, 1.80, \"Depth=1\", fontsize=13)\n",
    "\n",
    "save_fig(\"decision_tree_instability_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Code to generate Figure 6–3. Regularization using min_samples_leaf:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import make_moons\n",
    "Xm, ym = make_moons(n_samples=100, noise=0.25, random_state=53)\n",
    "\n",
    "deep_tree_clf1 = DecisionTreeClassifier(random_state=42)\n",
    "deep_tree_clf2 = DecisionTreeClassifier(min_samples_leaf=4, random_state=42)\n",
    "deep_tree_clf1.fit(Xm, ym)\n",
    "deep_tree_clf2.fit(Xm, ym)\n",
    "\n",
    "fig, axes = plt.subplots(ncols=2, figsize=(10, 4), sharey=True)\n",
    "plt.sca(axes[0])\n",
    "plot_decision_boundary(deep_tree_clf1, Xm, ym, axes=[-1.5, 2.4, -1, 1.5], iris=False)\n",
    "plt.title(\"No restrictions\", fontsize=16)\n",
    "plt.sca(axes[1])\n",
    "plot_decision_boundary(deep_tree_clf2, Xm, ym, axes=[-1.5, 2.4, -1, 1.5], iris=False)\n",
    "plt.title(\"min_samples_leaf = {}\".format(deep_tree_clf2.min_samples_leaf), fontsize=14)\n",
    "plt.ylabel(\"\")\n",
    "\n",
    "save_fig(\"min_samples_leaf_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Rotating the dataset also leads to completely different decision boundaries:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "angle = np.pi / 180 * 20\n",
    "rotation_matrix = np.array([[np.cos(angle), -np.sin(angle)], [np.sin(angle), np.cos(angle)]])\n",
    "Xr = X.dot(rotation_matrix)\n",
    "\n",
    "tree_clf_r = DecisionTreeClassifier(random_state=42)\n",
    "tree_clf_r.fit(Xr, y)\n",
    "\n",
    "plt.figure(figsize=(8, 3))\n",
    "plot_decision_boundary(tree_clf_r, Xr, y, axes=[0.5, 7.5, -1.0, 1], iris=False)\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Code to generate Figure 6–7. Sensitivity to training set rotation**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(6)\n",
    "Xs = np.random.rand(100, 2) - 0.5\n",
    "ys = (Xs[:, 0] > 0).astype(np.float32) * 2\n",
    "\n",
    "angle = np.pi / 4\n",
    "rotation_matrix = np.array([[np.cos(angle), -np.sin(angle)], [np.sin(angle), np.cos(angle)]])\n",
    "Xsr = Xs.dot(rotation_matrix)\n",
    "\n",
    "tree_clf_s = DecisionTreeClassifier(random_state=42)\n",
    "tree_clf_s.fit(Xs, ys)\n",
    "tree_clf_sr = DecisionTreeClassifier(random_state=42)\n",
    "tree_clf_sr.fit(Xsr, ys)\n",
    "\n",
    "fig, axes = plt.subplots(ncols=2, figsize=(10, 4), sharey=True)\n",
    "plt.sca(axes[0])\n",
    "plot_decision_boundary(tree_clf_s, Xs, ys, axes=[-0.7, 0.7, -0.7, 0.7], iris=False)\n",
    "plt.sca(axes[1])\n",
    "plot_decision_boundary(tree_clf_sr, Xsr, ys, axes=[-0.7, 0.7, -0.7, 0.7], iris=False)\n",
    "plt.ylabel(\"\")\n",
    "\n",
    "save_fig(\"sensitivity_to_rotation_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Regression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's prepare a simple linear dataset:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Quadratic training set + noise\n",
    "np.random.seed(42)\n",
    "m = 200\n",
    "X = np.random.rand(m, 1)\n",
    "y = 4 * (X - 0.5) ** 2\n",
    "y = y + np.random.randn(m, 1) / 10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Code example:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.tree import DecisionTreeRegressor\n",
    "\n",
    "tree_reg = DecisionTreeRegressor(max_depth=2, random_state=42)\n",
    "tree_reg.fit(X, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Code to generate Figure 6–5. Predictions of two Decision Tree regression models:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.tree import DecisionTreeRegressor\n",
    "\n",
    "tree_reg1 = DecisionTreeRegressor(random_state=42, max_depth=2)\n",
    "tree_reg2 = DecisionTreeRegressor(random_state=42, max_depth=3)\n",
    "tree_reg1.fit(X, y)\n",
    "tree_reg2.fit(X, y)\n",
    "\n",
    "def plot_regression_predictions(tree_reg, X, y, axes=[0, 1, -0.2, 1], ylabel=\"$y$\"):\n",
    "    x1 = np.linspace(axes[0], axes[1], 500).reshape(-1, 1)\n",
    "    y_pred = tree_reg.predict(x1)\n",
    "    plt.axis(axes)\n",
    "    plt.xlabel(\"$x_1$\", fontsize=18)\n",
    "    if ylabel:\n",
    "        plt.ylabel(ylabel, fontsize=18, rotation=0)\n",
    "    plt.plot(X, y, \"b.\")\n",
    "    plt.plot(x1, y_pred, \"r.-\", linewidth=2, label=r\"$\\hat{y}$\")\n",
    "\n",
    "fig, axes = plt.subplots(ncols=2, figsize=(10, 4), sharey=True)\n",
    "plt.sca(axes[0])\n",
    "plot_regression_predictions(tree_reg1, X, y)\n",
    "for split, style in ((0.1973, \"k-\"), (0.0917, \"k--\"), (0.7718, \"k--\")):\n",
    "    plt.plot([split, split], [-0.2, 1], style, linewidth=2)\n",
    "plt.text(0.21, 0.65, \"Depth=0\", fontsize=15)\n",
    "plt.text(0.01, 0.2, \"Depth=1\", fontsize=13)\n",
    "plt.text(0.65, 0.8, \"Depth=1\", fontsize=13)\n",
    "plt.legend(loc=\"upper center\", fontsize=18)\n",
    "plt.title(\"max_depth=2\", fontsize=14)\n",
    "\n",
    "plt.sca(axes[1])\n",
    "plot_regression_predictions(tree_reg2, X, y, ylabel=None)\n",
    "for split, style in ((0.1973, \"k-\"), (0.0917, \"k--\"), (0.7718, \"k--\")):\n",
    "    plt.plot([split, split], [-0.2, 1], style, linewidth=2)\n",
    "for split in (0.0458, 0.1298, 0.2873, 0.9040):\n",
    "    plt.plot([split, split], [-0.2, 1], \"k:\", linewidth=1)\n",
    "plt.text(0.3, 0.5, \"Depth=2\", fontsize=13)\n",
    "plt.title(\"max_depth=3\", fontsize=14)\n",
    "\n",
    "save_fig(\"tree_regression_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Code to generate Figure 6-4. A Decision Tree for regression:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "export_graphviz(\n",
    "        tree_reg1,\n",
    "        out_file=IMAGES_PATH / \"regression_tree.dot\",\n",
    "        feature_names=[\"x1\"],\n",
    "        rounded=True,\n",
    "        filled=True\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "Source.from_file(IMAGES_PATH / \"regression_tree.dot\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Code to generate Figure 6–6. Regularizing a Decision Tree regressor:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "tree_reg1 = DecisionTreeRegressor(random_state=42)\n",
    "tree_reg2 = DecisionTreeRegressor(random_state=42, min_samples_leaf=10)\n",
    "tree_reg1.fit(X, y)\n",
    "tree_reg2.fit(X, y)\n",
    "\n",
    "x1 = np.linspace(0, 1, 500).reshape(-1, 1)\n",
    "y_pred1 = tree_reg1.predict(x1)\n",
    "y_pred2 = tree_reg2.predict(x1)\n",
    "\n",
    "fig, axes = plt.subplots(ncols=2, figsize=(10, 4), sharey=True)\n",
    "\n",
    "plt.sca(axes[0])\n",
    "plt.plot(X, y, \"b.\")\n",
    "plt.plot(x1, y_pred1, \"r.-\", linewidth=2, label=r\"$\\hat{y}$\")\n",
    "plt.axis([0, 1, -0.2, 1.1])\n",
    "plt.xlabel(\"$x_1$\", fontsize=18)\n",
    "plt.ylabel(\"$y$\", fontsize=18, rotation=0)\n",
    "plt.legend(loc=\"upper center\", fontsize=18)\n",
    "plt.title(\"No restrictions\", fontsize=14)\n",
    "\n",
    "plt.sca(axes[1])\n",
    "plt.plot(X, y, \"b.\")\n",
    "plt.plot(x1, y_pred2, \"r.-\", linewidth=2, label=r\"$\\hat{y}$\")\n",
    "plt.axis([0, 1, -0.2, 1.1])\n",
    "plt.xlabel(\"$x_1$\", fontsize=18)\n",
    "plt.title(\"min_samples_leaf={}\".format(tree_reg2.min_samples_leaf), fontsize=14)\n",
    "\n",
    "save_fig(\"tree_regression_regularization_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Exercise solutions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. to 6."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "See appendix A."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 7."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_Exercise: train and fine-tune a Decision Tree for the moons dataset._"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "a. Generate a moons dataset using `make_moons(n_samples=10000, noise=0.4)`."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Adding `random_state=42` to make this notebook's output constant:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import make_moons\n",
    "\n",
    "X, y = make_moons(n_samples=10000, noise=0.4, random_state=42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "b. Split it into a training set and a test set using `train_test_split()`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "c. Use grid search with cross-validation (with the help of the `GridSearchCV` class) to find good hyperparameter values for a `DecisionTreeClassifier`. Hint: try various values for `max_leaf_nodes`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "params = {'max_leaf_nodes': list(range(2, 100)), 'min_samples_split': [2, 3, 4]}\n",
    "grid_search_cv = GridSearchCV(DecisionTreeClassifier(random_state=42), params, verbose=1, cv=3)\n",
    "\n",
    "grid_search_cv.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "grid_search_cv.best_estimator_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "d. Train it on the full training set using these hyperparameters, and measure your model's performance on the test set. You should get roughly 85% to 87% accuracy."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "By default, `GridSearchCV` trains the best model found on the whole training set (you can change this by setting `refit=False`), so we don't need to do it again. We can simply evaluate the model's accuracy:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "y_pred = grid_search_cv.predict(X_test)\n",
    "accuracy_score(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 8."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_Exercise: Grow a forest._"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "a. Continuing the previous exercise, generate 1,000 subsets of the training set, each containing 100 instances selected randomly. Hint: you can use Scikit-Learn's `ShuffleSplit` class for this."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import ShuffleSplit\n",
    "\n",
    "n_trees = 1000\n",
    "n_instances = 100\n",
    "\n",
    "mini_sets = []\n",
    "\n",
    "rs = ShuffleSplit(n_splits=n_trees, test_size=len(X_train) - n_instances, random_state=42)\n",
    "for mini_train_index, mini_test_index in rs.split(X_train):\n",
    "    X_mini_train = X_train[mini_train_index]\n",
    "    y_mini_train = y_train[mini_train_index]\n",
    "    mini_sets.append((X_mini_train, y_mini_train))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "b. Train one Decision Tree on each subset, using the best hyperparameter values found above. Evaluate these 1,000 Decision Trees on the test set. Since they were trained on smaller sets, these Decision Trees will likely perform worse than the first Decision Tree, achieving only about 80% accuracy."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.base import clone\n",
    "\n",
    "forest = [clone(grid_search_cv.best_estimator_) for _ in range(n_trees)]\n",
    "\n",
    "accuracy_scores = []\n",
    "\n",
    "for tree, (X_mini_train, y_mini_train) in zip(forest, mini_sets):\n",
    "    tree.fit(X_mini_train, y_mini_train)\n",
    "    \n",
    "    y_pred = tree.predict(X_test)\n",
    "    accuracy_scores.append(accuracy_score(y_test, y_pred))\n",
    "\n",
    "np.mean(accuracy_scores)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "c. Now comes the magic. For each test set instance, generate the predictions of the 1,000 Decision Trees, and keep only the most frequent prediction (you can use SciPy's `mode()` function for this). This gives you _majority-vote predictions_ over the test set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y_pred = np.empty([n_trees, len(X_test)], dtype=np.uint8)\n",
    "\n",
    "for tree_index, tree in enumerate(forest):\n",
    "    Y_pred[tree_index] = tree.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.stats import mode\n",
    "\n",
    "y_pred_majority_votes, n_votes = mode(Y_pred, axis=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "d. Evaluate these predictions on the test set: you should obtain a slightly higher accuracy than your first model (about 0.5 to 1.5% higher). Congratulations, you have trained a Random Forest classifier!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "accuracy_score(y_test, y_pred_majority_votes.reshape([-1]))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  },
  "nav_menu": {
   "height": "309px",
   "width": "468px"
  },
  "toc": {
   "navigate_menu": true,
   "number_sections": true,
   "sideBar": true,
   "threshold": 6,
   "toc_cell": false,
   "toc_section_display": "block",
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}