diff --git a/01_the_machine_learning_landscape.ipynb b/01_the_machine_learning_landscape.ipynb index 6a080af..8b99fce 100644 --- a/01_the_machine_learning_landscape.ipynb +++ b/01_the_machine_learning_landscape.ipynb @@ -2,10 +2,7 @@ "cells": [ { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "**Chapter 1 – The Machine Learning landscape**\n", "\n", @@ -14,20 +11,14 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "# Setup" ] }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "First, let's make sure this notebook works well in both python 2 and 3, import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures:" ] @@ -36,9 +27,6 @@ "cell_type": "code", "execution_count": 1, "metadata": { - "collapsed": false, - "deletable": true, - "editable": true, "slideshow": { "slide_type": "-" } @@ -50,11 +38,10 @@ "\n", "# Common imports\n", "import numpy as np\n", - "import numpy.random as rnd\n", "import os\n", "\n", "# to make this notebook's output stable across runs\n", - "rnd.seed(42)\n", + "np.random.seed(42)\n", "\n", "# To plot pretty figures\n", "%matplotlib inline\n", @@ -73,35 +60,173 @@ " print(\"Saving figure\", fig_id)\n", " if tight_layout:\n", " plt.tight_layout()\n", - " plt.savefig(path, format='png', dpi=300)" + " plt.savefig(path, format='png', dpi=300)\n", + "\n", + "# Ignore useless warnings (see SciPy issue #5998)\n", + "import warnings\n", + "warnings.filterwarnings(action=\"ignore\", module=\"scipy\", message=\"^internal gelsd\")" ] }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ - "# Load and prepare Life satisfaction data" + "# Code example 1-1" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This function just merges the OECD's life satisfaction data and the IMF's GDP per capita data. It's a bit too long and boring and it's not specific to Machine Learning, which is why I left it out of the book." ] }, { "cell_type": "code", "execution_count": 2, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ + "def prepare_country_stats(oecd_bli, gdp_per_capita):\n", + " oecd_bli = oecd_bli[oecd_bli[\"INEQUALITY\"]==\"TOT\"]\n", + " oecd_bli = oecd_bli.pivot(index=\"Country\", columns=\"Indicator\", values=\"Value\")\n", + " gdp_per_capita.rename(columns={\"2015\": \"GDP per capita\"}, inplace=True)\n", + " gdp_per_capita.set_index(\"Country\", inplace=True)\n", + " full_country_stats = pd.merge(left=oecd_bli, right=gdp_per_capita,\n", + " left_index=True, right_index=True)\n", + " full_country_stats.sort_values(by=\"GDP per capita\", inplace=True)\n", + " remove_indices = [0, 1, 6, 8, 33, 34, 35]\n", + " keep_indices = list(set(range(36)) - set(remove_indices))\n", + " return full_country_stats[[\"GDP per capita\", 'Life satisfaction']].iloc[keep_indices]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The code in the book expects the data files to be located in the current directory. I just tweaked it here to fetch the files in datasets/lifesat." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "datapath = os.path.join(\"datasets\", \"lifesat\", \"\")" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "# Code example\n", + "import matplotlib\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", "import pandas as pd\n", + "import sklearn.linear_model\n", "\n", - "# Download CSV from http://stats.oecd.org/index.aspx?DataSetCode=BLI\n", - "datapath = \"datasets/lifesat/\"\n", + "# Load the data\n", + "oecd_bli = pd.read_csv(datapath + \"oecd_bli_2015.csv\", thousands=',')\n", + "gdp_per_capita = pd.read_csv(datapath + \"gdp_per_capita.csv\",thousands=',',delimiter='\\t',\n", + " encoding='latin1', na_values=\"n/a\")\n", "\n", - "oecd_bli = pd.read_csv(datapath+\"oecd_bli_2015.csv\", thousands=',')\n", + "# Prepare the data\n", + "country_stats = prepare_country_stats(oecd_bli, gdp_per_capita)\n", + "X = np.c_[country_stats[\"GDP per capita\"]]\n", + "y = np.c_[country_stats[\"Life satisfaction\"]]\n", + "\n", + "# Visualize the data\n", + "country_stats.plot(kind='scatter', x=\"GDP per capita\", y='Life satisfaction')\n", + "plt.show()\n", + "\n", + "# Select a linear model\n", + "model = sklearn.linear_model.LinearRegression()\n", + "\n", + "# Train the model\n", + "model.fit(X, y)\n", + "\n", + "# Make a prediction for Cyprus\n", + "X_new = [[22587]] # Cyprus' GDP per capita\n", + "print(model.predict(X_new)) # outputs [[ 5.96242338]]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Note: you can ignore the rest of this notebook, it just generates many of the figures in chapter 1." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Load and prepare Life satisfaction data" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If you want, you can get fresh data from the OECD's website.\n", + "Download the CSV from http://stats.oecd.org/index.aspx?DataSetCode=BLI\n", + "and save it to `datasets/lifesat/`." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "oecd_bli = pd.read_csv(datapath + \"oecd_bli_2015.csv\", thousands=',')\n", "oecd_bli = oecd_bli[oecd_bli[\"INEQUALITY\"]==\"TOT\"]\n", "oecd_bli = oecd_bli.pivot(index=\"Country\", columns=\"Indicator\", values=\"Value\")\n", "oecd_bli.head(2)" @@ -109,12 +234,8 @@ }, { "cell_type": "code", - "execution_count": 3, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "execution_count": 6, + "metadata": {}, "outputs": [], "source": [ "oecd_bli[\"Life satisfaction\"].head()" @@ -122,25 +243,24 @@ }, { "cell_type": "markdown", - "metadata": { - "deletable": true, - "editable": true - }, + "metadata": {}, "source": [ "# Load and prepare GDP per capita data" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Just like above, you can update the GDP per capita data if you want. Just download data from http://goo.gl/j1MSKe (=> imf.org) and save it to `datasets/lifesat/`." + ] + }, { "cell_type": "code", - "execution_count": 4, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "execution_count": 7, + "metadata": {}, "outputs": [], "source": [ - "# Download data from http://goo.gl/j1MSKe (=> imf.org)\n", "gdp_per_capita = pd.read_csv(datapath+\"gdp_per_capita.csv\", thousands=',', delimiter='\\t',\n", " encoding='latin1', na_values=\"n/a\")\n", "gdp_per_capita.rename(columns={\"2015\": \"GDP per capita\"}, inplace=True)\n", @@ -150,12 +270,8 @@ }, { "cell_type": "code", - "execution_count": 5, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "execution_count": 8, + "metadata": {}, "outputs": [], "source": [ "full_country_stats = pd.merge(left=oecd_bli, right=gdp_per_capita, left_index=True, right_index=True)\n", @@ -165,12 +281,8 @@ }, { "cell_type": "code", - "execution_count": 6, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "execution_count": 9, + "metadata": {}, "outputs": [], "source": [ "full_country_stats[[\"GDP per capita\", 'Life satisfaction']].loc[\"United States\"]" @@ -178,12 +290,8 @@ }, { "cell_type": "code", - "execution_count": 7, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "execution_count": 10, + "metadata": {}, "outputs": [], "source": [ "remove_indices = [0, 1, 6, 8, 33, 34, 35]\n", @@ -195,12 +303,8 @@ }, { "cell_type": "code", - "execution_count": 8, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "execution_count": 11, + "metadata": {}, "outputs": [], "source": [ "sample_data.plot(kind='scatter', x=\"GDP per capita\", y='Life satisfaction', figsize=(5,3))\n", @@ -224,25 +328,17 @@ }, { "cell_type": "code", - "execution_count": 9, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "execution_count": 12, + "metadata": {}, "outputs": [], "source": [ - "sample_data.to_csv(\"life_satisfaction_vs_gdp_per_capita.csv\")" + "sample_data.to_csv(os.path.join(\"datasets\", \"lifesat\", \"lifesat.csv\"))" ] }, { "cell_type": "code", - "execution_count": 10, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "execution_count": 13, + "metadata": {}, "outputs": [], "source": [ "sample_data.loc[list(position_text.keys())]" @@ -250,12 +346,8 @@ }, { "cell_type": "code", - "execution_count": 11, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "execution_count": 14, + "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", @@ -278,12 +370,8 @@ }, { "cell_type": "code", - "execution_count": 12, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "execution_count": 15, + "metadata": {}, "outputs": [], "source": [ "from sklearn import linear_model\n", @@ -297,12 +385,8 @@ }, { "cell_type": "code", - "execution_count": 13, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "execution_count": 16, + "metadata": {}, "outputs": [], "source": [ "sample_data.plot(kind='scatter', x=\"GDP per capita\", y='Life satisfaction', figsize=(5,3))\n", @@ -317,12 +401,8 @@ }, { "cell_type": "code", - "execution_count": 14, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "execution_count": 17, + "metadata": {}, "outputs": [], "source": [ "cyprus_gdp_per_capita = gdp_per_capita.loc[\"Cyprus\"][\"GDP per capita\"]\n", @@ -333,12 +413,8 @@ }, { "cell_type": "code", - "execution_count": 15, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "execution_count": 18, + "metadata": {}, "outputs": [], "source": [ "sample_data.plot(kind='scatter', x=\"GDP per capita\", y='Life satisfaction', figsize=(5,3), s=1)\n", @@ -356,12 +432,8 @@ }, { "cell_type": "code", - "execution_count": 16, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "execution_count": 19, + "metadata": {}, "outputs": [], "source": [ "sample_data[7:10]" @@ -369,12 +441,8 @@ }, { "cell_type": "code", - "execution_count": 17, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "execution_count": 20, + "metadata": {}, "outputs": [], "source": [ "(5.1+5.7+6.5)/3" @@ -382,28 +450,29 @@ }, { "cell_type": "code", - "execution_count": 18, - "metadata": { - "collapsed": true, - "deletable": true, - "editable": true - }, + "execution_count": 21, + "metadata": {}, "outputs": [], "source": [ "backup = oecd_bli, gdp_per_capita\n", "\n", "def prepare_country_stats(oecd_bli, gdp_per_capita):\n", - " return sample_data" + " oecd_bli = oecd_bli[oecd_bli[\"INEQUALITY\"]==\"TOT\"]\n", + " oecd_bli = oecd_bli.pivot(index=\"Country\", columns=\"Indicator\", values=\"Value\")\n", + " gdp_per_capita.rename(columns={\"2015\": \"GDP per capita\"}, inplace=True)\n", + " gdp_per_capita.set_index(\"Country\", inplace=True)\n", + " full_country_stats = pd.merge(left=oecd_bli, right=gdp_per_capita,\n", + " left_index=True, right_index=True)\n", + " full_country_stats.sort_values(by=\"GDP per capita\", inplace=True)\n", + " remove_indices = [0, 1, 6, 8, 33, 34, 35]\n", + " keep_indices = list(set(range(36)) - set(remove_indices))\n", + " return full_country_stats[[\"GDP per capita\", 'Life satisfaction']].iloc[keep_indices]" ] }, { "cell_type": "code", - "execution_count": 19, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "execution_count": 22, + "metadata": {}, "outputs": [], "source": [ "# Code example\n", @@ -440,12 +509,8 @@ }, { "cell_type": "code", - "execution_count": 20, - "metadata": { - "collapsed": true, - "deletable": true, - "editable": true - }, + "execution_count": 23, + "metadata": {}, "outputs": [], "source": [ "oecd_bli, gdp_per_capita = backup" @@ -453,12 +518,8 @@ }, { "cell_type": "code", - "execution_count": 21, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "execution_count": 24, + "metadata": {}, "outputs": [], "source": [ "missing_data" @@ -466,12 +527,8 @@ }, { "cell_type": "code", - "execution_count": 22, - "metadata": { - "collapsed": true, - "deletable": true, - "editable": true - }, + "execution_count": 25, + "metadata": {}, "outputs": [], "source": [ "position_text2 = {\n", @@ -487,12 +544,8 @@ }, { "cell_type": "code", - "execution_count": 23, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "execution_count": 26, + "metadata": {}, "outputs": [], "source": [ "sample_data.plot(kind='scatter', x=\"GDP per capita\", y='Life satisfaction', figsize=(8,3))\n", @@ -522,12 +575,8 @@ }, { "cell_type": "code", - "execution_count": 24, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "execution_count": 27, + "metadata": {}, "outputs": [], "source": [ "full_country_stats.plot(kind='scatter', x=\"GDP per capita\", y='Life satisfaction', figsize=(8,3))\n", @@ -550,12 +599,8 @@ }, { "cell_type": "code", - "execution_count": 25, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "execution_count": 28, + "metadata": {}, "outputs": [], "source": [ "full_country_stats.loc[[c for c in full_country_stats.index if \"W\" in c.upper()]][\"Life satisfaction\"]" @@ -563,12 +608,8 @@ }, { "cell_type": "code", - "execution_count": 26, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "execution_count": 29, + "metadata": {}, "outputs": [], "source": [ "gdp_per_capita.loc[[c for c in gdp_per_capita.index if \"W\" in c.upper()]].head()" @@ -576,12 +617,8 @@ }, { "cell_type": "code", - "execution_count": 27, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "execution_count": 30, + "metadata": {}, "outputs": [], "source": [ "plt.figure(figsize=(8,3))\n", @@ -611,12 +648,8 @@ }, { "cell_type": "code", - "execution_count": 28, - "metadata": { - "collapsed": true, - "deletable": true, - "editable": true - }, + "execution_count": 31, + "metadata": {}, "outputs": [], "source": [ "backup = oecd_bli, gdp_per_capita\n", @@ -627,12 +660,8 @@ }, { "cell_type": "code", - "execution_count": 29, - "metadata": { - "collapsed": true, - "deletable": true, - "editable": true - }, + "execution_count": 32, + "metadata": {}, "outputs": [], "source": [ "# Replace this linear model:\n", @@ -641,12 +670,8 @@ }, { "cell_type": "code", - "execution_count": 30, - "metadata": { - "collapsed": true, - "deletable": true, - "editable": true - }, + "execution_count": 33, + "metadata": {}, "outputs": [], "source": [ "# with this k-neighbors regression model:\n", @@ -655,12 +680,8 @@ }, { "cell_type": "code", - "execution_count": 31, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "execution_count": 34, + "metadata": {}, "outputs": [], "source": [ "X = np.c_[country_stats[\"GDP per capita\"]]\n", @@ -677,11 +698,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [] } @@ -702,7 +719,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.5.3" + "version": "3.6.3" }, "nav_menu": {}, "toc": { @@ -723,5 +740,5 @@ } }, "nbformat": 4, - "nbformat_minor": 0 + "nbformat_minor": 1 }