From d7d6c121e33b368bdc21a6bab9f3166eb1d4bf67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Geron?= Date: Tue, 27 Sep 2016 23:31:21 +0200 Subject: [PATCH] Add notebooks for chapters 5 to 14 --- .gitignore | 13 +- ...=> 01_the_machine_learning_landscape.ipynb | 2 +- ..._end_to_end_machine_learning_project.ipynb | 2 +- ...s.ipynb => 04_training_linear_models.ipynb | 0 05_support_vector_machines.ipynb | 1248 ++++++++++++ 06_decision_trees.ipynb | 506 +++++ 07_ensemble_learning_and_random_forests.ipynb | 788 ++++++++ 08_dimensionality_reduction.ipynb | 1343 +++++++++++++ 09_up_and_running_with_tensorflow.ipynb | 1709 +++++++++++++++++ ...uction_to_artificial_neural_networks.ipynb | 660 +++++++ 11_deep_learning.ipynb | 931 +++++++++ 12_distributed_tensorflow.ipynb | 494 +++++ 13_convolutional_neural_networks.ipynb | 613 ++++++ 14_recurrent_neural_networks.ipynb | 1326 +++++++++++++ classification.ipynb | 68 +- images/ann/README | 1 + images/autoencoders/README | 1 + images/cnn/README | 1 + images/cnn/test_image.png | Bin 0 -> 181822 bytes images/decision_trees/README | 1 + images/deep/README | 1 + images/dim_reduction/README | 1 + images/distributed/README | 1 + images/ensembles/README | 1 + images/rl/README | 1 + images/rnn/README | 1 + images/svm/README | 1 + images/tensorflow/README | 1 + index.ipynb | 45 +- nets/inception_v3.py | 10 +- 30 files changed, 9741 insertions(+), 29 deletions(-) rename fundamentals.ipynb => 01_the_machine_learning_landscape.ipynb (99%) rename end_to_end_project.ipynb => 02_end_to_end_machine_learning_project.ipynb (99%) rename training_linear_models.ipynb => 04_training_linear_models.ipynb (100%) create mode 100644 05_support_vector_machines.ipynb create mode 100644 06_decision_trees.ipynb create mode 100644 07_ensemble_learning_and_random_forests.ipynb create mode 100644 08_dimensionality_reduction.ipynb create mode 100644 09_up_and_running_with_tensorflow.ipynb create mode 100644 10_introduction_to_artificial_neural_networks.ipynb create mode 100644 11_deep_learning.ipynb create mode 100644 12_distributed_tensorflow.ipynb create mode 100644 13_convolutional_neural_networks.ipynb create mode 100644 14_recurrent_neural_networks.ipynb create mode 100644 images/ann/README create mode 100644 images/autoencoders/README create mode 100644 images/cnn/README create mode 100644 images/cnn/test_image.png create mode 100644 images/decision_trees/README create mode 100644 images/deep/README create mode 100644 images/dim_reduction/README create mode 100644 images/distributed/README create mode 100644 images/ensembles/README create mode 100644 images/rl/README create mode 100644 images/rnn/README create mode 100644 images/svm/README create mode 100644 images/tensorflow/README diff --git a/.gitignore b/.gitignore index a5684d8..89c7162 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,10 @@ -.ipynb_checkpoints -.DS_Store -my_* -images/**/*.png +*.bak +*.ckpt *.pyc +.DS_Store +.ipynb_checkpoints +checkpoint +logs/* +tf_logs/* +images/**/*.png +my_* diff --git a/fundamentals.ipynb b/01_the_machine_learning_landscape.ipynb similarity index 99% rename from fundamentals.ipynb rename to 01_the_machine_learning_landscape.ipynb index df6d6dc..9647fa6 100644 --- a/fundamentals.ipynb +++ b/01_the_machine_learning_landscape.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "**Chapter 1 – Fundamentals of Machine Learning**\n", + "**Chapter 1 – The Machine Learning landscape**\n", "\n", "_This is the code used to generate some of the figures in chapter 1._" ] diff --git a/end_to_end_project.ipynb b/02_end_to_end_machine_learning_project.ipynb similarity index 99% rename from end_to_end_project.ipynb rename to 02_end_to_end_machine_learning_project.ipynb index 875a1be..25278be 100644 --- a/end_to_end_project.ipynb +++ b/02_end_to_end_machine_learning_project.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "**Chapter 2 – End to end Machine Learning project**\n", + "**Chapter 2 – End-to-end Machine Learning project**\n", "\n", "*Welcome to Machine Learning Housing Corp.! Your task is to predict median house values in Californian districts, given a number of features from these districts.*\n", "\n", diff --git a/training_linear_models.ipynb b/04_training_linear_models.ipynb similarity index 100% rename from training_linear_models.ipynb rename to 04_training_linear_models.ipynb diff --git a/05_support_vector_machines.ipynb b/05_support_vector_machines.ipynb new file mode 100644 index 0000000..42937b0 --- /dev/null +++ b/05_support_vector_machines.ipynb @@ -0,0 +1,1248 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Chapter 5 – Support Vector Machines**\n", + "\n", + "_This notebook contains all the sample code and solutions to the exercices in chapter 5._" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Setup" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "First, let's make sure this notebook works well in both python 2 and 3, import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures:" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# To support both python 2 and python 3\n", + "from __future__ import division, print_function, unicode_literals\n", + "\n", + "# Common imports\n", + "import numpy as np\n", + "import numpy.random as rnd\n", + "import os\n", + "\n", + "# to make this notebook's output stable across runs\n", + "rnd.seed(42)\n", + "\n", + "# To plot pretty figures\n", + "%matplotlib inline\n", + "import matplotlib\n", + "import matplotlib.pyplot as plt\n", + "plt.rcParams['axes.labelsize'] = 14\n", + "plt.rcParams['xtick.labelsize'] = 12\n", + "plt.rcParams['ytick.labelsize'] = 12\n", + "\n", + "# Where to save the figures\n", + "PROJECT_ROOT_DIR = \".\"\n", + "CHAPTER_ID = \"svm\"\n", + "\n", + "def save_fig(fig_id, tight_layout=True):\n", + " path = os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID, fig_id + \".png\")\n", + " print(\"Saving figure\", fig_id)\n", + " if tight_layout:\n", + " plt.tight_layout()\n", + " plt.savefig(path, format='png', dpi=300)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Large margin classification" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from sklearn.svm import SVC\n", + "from sklearn import datasets\n", + "\n", + "iris = datasets.load_iris()\n", + "X = iris[\"data\"][:, (2, 3)] # petal length, petal width\n", + "y = iris[\"target\"]\n", + "\n", + "setosa_or_versicolour = (y == 0) | (y == 1)\n", + "X = X[setosa_or_versicolour]\n", + "y = y[setosa_or_versicolour]\n", + "\n", + "# SVM Classifier model\n", + "svm_clf = SVC(kernel=\"linear\", C=float(\"inf\"))\n", + "svm_clf.fit(X, y)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# Bad models\n", + "x0 = np.linspace(0, 5.5, 200)\n", + "pred_1 = 5*x0 - 20\n", + "pred_2 = x0 - 1.8\n", + "pred_3 = 0.1 * x0 + 0.5\n", + "\n", + "def plot_svc_decision_boundary(svm_clf, xmin, xmax):\n", + " w = svm_clf.coef_[0]\n", + " b = svm_clf.intercept_[0]\n", + "\n", + " # At the decision boundary, w0*x0 + w1*x1 + b = 0\n", + " # => x1 = -w0/w1 * x0 - b/w1\n", + " x0 = np.linspace(xmin, xmax, 200)\n", + " decision_boundary = -w[0]/w[1] * x0 - b/w[1]\n", + "\n", + " margin = 1/w[1]\n", + " gutter_up = decision_boundary + margin\n", + " gutter_down = decision_boundary - margin\n", + "\n", + " svs = svm_clf.support_vectors_\n", + " plt.scatter(svs[:, 0], svs[:, 1], s=180, facecolors='#FFAAAA')\n", + " plt.plot(x0, decision_boundary, \"k-\", linewidth=2)\n", + " plt.plot(x0, gutter_up, \"k--\", linewidth=2)\n", + " plt.plot(x0, gutter_down, \"k--\", linewidth=2)\n", + "\n", + "plt.figure(figsize=(12,2.7))\n", + "\n", + "plt.subplot(121)\n", + "plt.plot(x0, pred_1, \"g--\", linewidth=2)\n", + "plt.plot(x0, pred_2, \"m-\", linewidth=2)\n", + "plt.plot(x0, pred_3, \"r-\", linewidth=2)\n", + "plt.plot(X[:, 0][y==1], X[:, 1][y==1], \"bs\", label=\"Iris-Versicolour\")\n", + "plt.plot(X[:, 0][y==0], X[:, 1][y==0], \"yo\", label=\"Iris-Setosa\")\n", + "plt.xlabel(\"Petal length\", fontsize=14)\n", + "plt.ylabel(\"Petal width\", fontsize=14)\n", + "plt.legend(loc=\"upper left\", fontsize=14)\n", + "plt.axis([0, 5.5, 0, 2])\n", + "\n", + "plt.subplot(122)\n", + "plot_svc_decision_boundary(svm_clf, 0, 5.5)\n", + "plt.plot(X[:, 0][y==1], X[:, 1][y==1], \"bs\")\n", + "plt.plot(X[:, 0][y==0], X[:, 1][y==0], \"yo\")\n", + "plt.xlabel(\"Petal length\", fontsize=14)\n", + "plt.axis([0, 5.5, 0, 2])\n", + "\n", + "save_fig(\"large_margin_classification_plot\")\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Sensitivity to feature scales" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "Xs = np.array([[1, 50], [5, 20], [3, 80], [5, 60]]).astype(np.float64)\n", + "ys = np.array([0, 0, 1, 1])\n", + "svm_clf = SVC(kernel=\"linear\", C=100)\n", + "svm_clf.fit(Xs, ys)\n", + "\n", + "plt.figure(figsize=(12,3.2))\n", + "plt.subplot(121)\n", + "plt.plot(Xs[:, 0][ys==1], Xs[:, 1][ys==1], \"bo\")\n", + "plt.plot(Xs[:, 0][ys==0], Xs[:, 1][ys==0], \"ms\")\n", + "plot_svc_decision_boundary(svm_clf, 0, 6)\n", + "plt.xlabel(\"$x_0$\", fontsize=20)\n", + "plt.ylabel(\"$x_1$ \", fontsize=20, rotation=0)\n", + "plt.title(\"Unscaled\", fontsize=16)\n", + "plt.axis([0, 6, 0, 90])\n", + "\n", + "from sklearn.preprocessing import StandardScaler\n", + "scaler = StandardScaler()\n", + "X_scaled = scaler.fit_transform(Xs)\n", + "svm_clf.fit(X_scaled, ys)\n", + "\n", + "plt.subplot(122)\n", + "plt.plot(X_scaled[:, 0][ys==1], X_scaled[:, 1][ys==1], \"bo\")\n", + "plt.plot(X_scaled[:, 0][ys==0], X_scaled[:, 1][ys==0], \"ms\")\n", + "plot_svc_decision_boundary(svm_clf, -2, 2)\n", + "plt.xlabel(\"$x_0$\", fontsize=20)\n", + "plt.title(\"Scaled\", fontsize=16)\n", + "plt.axis([-2, 2, -2, 2])\n", + "\n", + "save_fig(\"sensitivity_to_feature_scales_plot\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Sensitivity to outliers" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "X_outliers = np.array([[3.4, 1.3], [3.2, 0.8]])\n", + "y_outliers = np.array([0, 0])\n", + "Xo1 = np.concatenate([X, X_outliers[:1]], axis=0)\n", + "yo1 = np.concatenate([y, y_outliers[:1]], axis=0)\n", + "Xo2 = np.concatenate([X, X_outliers[1:]], axis=0)\n", + "yo2 = np.concatenate([y, y_outliers[1:]], axis=0)\n", + "\n", + "svm_clf2 = SVC(kernel=\"linear\", C=10**9)#float(\"inf\"))\n", + "svm_clf2.fit(Xo2, yo2)\n", + "\n", + "plt.figure(figsize=(12,2.7))\n", + "\n", + "plt.subplot(121)\n", + "plt.plot(Xo1[:, 0][yo1==1], Xo1[:, 1][yo1==1], \"bs\")\n", + "plt.plot(Xo1[:, 0][yo1==0], Xo1[:, 1][yo1==0], \"yo\")\n", + "plt.text(0.3, 1.0, \"Impossible!\", fontsize=24, color=\"red\")\n", + "plt.xlabel(\"Petal length\", fontsize=14)\n", + "plt.ylabel(\"Petal width\", fontsize=14)\n", + "plt.annotate(\"Outlier\",\n", + " xy=(X_outliers[0][0], X_outliers[0][1]),\n", + " xytext=(2.5, 1.7),\n", + " ha=\"center\",\n", + " arrowprops=dict(facecolor='black', shrink=0.1),\n", + " fontsize=16,\n", + " )\n", + "plt.axis([0, 5.5, 0, 2])\n", + "\n", + "plt.subplot(122)\n", + "plt.plot(Xo2[:, 0][yo2==1], Xo2[:, 1][yo2==1], \"bs\")\n", + "plt.plot(Xo2[:, 0][yo2==0], Xo2[:, 1][yo2==0], \"yo\")\n", + "plot_svc_decision_boundary(svm_clf2, 0, 5.5)\n", + "plt.xlabel(\"Petal length\", fontsize=14)\n", + "plt.annotate(\"Outlier\",\n", + " xy=(X_outliers[1][0], X_outliers[1][1]),\n", + " xytext=(3.2, 0.08),\n", + " ha=\"center\",\n", + " arrowprops=dict(facecolor='black', shrink=0.1),\n", + " fontsize=16,\n", + " )\n", + "plt.axis([0, 5.5, 0, 2])\n", + "\n", + "save_fig(\"sensitivity_to_outliers_plot\")\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Large margin *vs* margin violations" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from sklearn import datasets\n", + "from sklearn.pipeline import Pipeline\n", + "from sklearn.preprocessing import StandardScaler\n", + "from sklearn.svm import LinearSVC\n", + "\n", + "iris = datasets.load_iris()\n", + "X = iris[\"data\"][:, (2, 3)] # petal length, petal width\n", + "y = (iris[\"target\"] == 2).astype(np.float64) # Iris-Virginica\n", + "\n", + "scaler = StandardScaler()\n", + "svm_clf1 = LinearSVC(C=100, loss=\"hinge\")\n", + "svm_clf2 = LinearSVC(C=1, loss=\"hinge\")\n", + "\n", + "scaled_svm_clf1 = Pipeline((\n", + " (\"scaler\", scaler),\n", + " (\"linear_svc\", svm_clf1),\n", + " ))\n", + "scaled_svm_clf2 = Pipeline((\n", + " (\"scaler\", scaler),\n", + " (\"linear_svc\", svm_clf2),\n", + " ))\n", + "\n", + "scaled_svm_clf1.fit(X, y)\n", + "scaled_svm_clf2.fit(X, y)\n", + "\n", + "scaled_svm_clf2.predict([[5.5, 1.7]])" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# Convert to unscaled parameters\n", + "b1 = svm_clf1.decision_function([-scaler.mean_ / scaler.scale_])\n", + "b2 = svm_clf2.decision_function([-scaler.mean_ / scaler.scale_])\n", + "w1 = svm_clf1.coef_[0] / scaler.scale_\n", + "w2 = svm_clf2.coef_[0] / scaler.scale_\n", + "svm_clf1.intercept_ = np.array([b1])\n", + "svm_clf2.intercept_ = np.array([b2])\n", + "svm_clf1.coef_ = np.array([w1])\n", + "svm_clf2.coef_ = np.array([w2])\n", + "\n", + "# Find support vectors (LinearSVC does not do this automatically)\n", + "t = y * 2 - 1\n", + "support_vectors_idx1 = (t * (X.dot(w1) + b1) < 1).ravel()\n", + "support_vectors_idx2 = (t * (X.dot(w2) + b2) < 1).ravel()\n", + "svm_clf1.support_vectors_ = X[support_vectors_idx1]\n", + "svm_clf2.support_vectors_ = X[support_vectors_idx2]" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "plt.figure(figsize=(12,3.2))\n", + "plt.subplot(121)\n", + "plt.plot(X[:, 0][y==1], X[:, 1][y==1], \"g^\", label=\"Iris-Virginica\")\n", + "plt.plot(X[:, 0][y==0], X[:, 1][y==0], \"bs\", label=\"Iris-Versicolour\")\n", + "plot_svc_decision_boundary(svm_clf1, 4, 6)\n", + "plt.xlabel(\"Petal length\", fontsize=14)\n", + "plt.ylabel(\"Petal width\", fontsize=14)\n", + "plt.legend(loc=\"upper left\", fontsize=14)\n", + "plt.title(\"$C = {}$\".format(svm_clf1.C), fontsize=16)\n", + "plt.axis([4, 6, 0.8, 2.8])\n", + "\n", + "plt.subplot(122)\n", + "plt.plot(X[:, 0][y==1], X[:, 1][y==1], \"g^\")\n", + "plt.plot(X[:, 0][y==0], X[:, 1][y==0], \"bs\")\n", + "plot_svc_decision_boundary(svm_clf2, 4, 6)\n", + "plt.xlabel(\"Petal length\", fontsize=14)\n", + "plt.title(\"$C = {}$\".format(svm_clf2.C), fontsize=16)\n", + "plt.axis([4, 6, 0.8, 2.8])\n", + "\n", + "save_fig(\"regularization_plot\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": true + }, + "source": [ + "# Non-linear classification" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "X1D = np.linspace(-4, 4, 9).reshape(-1, 1)\n", + "X2D = np.c_[X1D, X1D**2]\n", + "y = np.array([0, 0, 1, 1, 1, 1, 1, 0, 0])\n", + "\n", + "plt.figure(figsize=(11, 4))\n", + "\n", + "plt.subplot(121)\n", + "plt.grid(True, which='both')\n", + "plt.axhline(y=0, color='k')\n", + "plt.plot(X1D[:, 0][y==0], np.zeros(4), \"bs\")\n", + "plt.plot(X1D[:, 0][y==1], np.zeros(5), \"g^\")\n", + "plt.gca().get_yaxis().set_ticks([])\n", + "plt.xlabel(r\"$x_1$\", fontsize=20)\n", + "plt.axis([-4.5, 4.5, -0.2, 0.2])\n", + "\n", + "plt.subplot(122)\n", + "plt.grid(True, which='both')\n", + "plt.axhline(y=0, color='k')\n", + "plt.axvline(x=0, color='k')\n", + "plt.plot(X2D[:, 0][y==0], X2D[:, 1][y==0], \"bs\")\n", + "plt.plot(X2D[:, 0][y==1], X2D[:, 1][y==1], \"g^\")\n", + "plt.xlabel(r\"$x_1$\", fontsize=20)\n", + "plt.ylabel(r\"$x_2$\", fontsize=20, rotation=0)\n", + "plt.gca().get_yaxis().set_ticks([0, 4, 8, 12, 16])\n", + "plt.plot([-4.5, 4.5], [6.5, 6.5], \"r--\", linewidth=3)\n", + "plt.axis([-4.5, 4.5, -1, 17])\n", + "\n", + "plt.subplots_adjust(right=1)\n", + "\n", + "save_fig(\"higher_dimensions_plot\", tight_layout=False)\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from sklearn.datasets import make_moons\n", + "X, y = make_moons(n_samples=100, noise=0.15, random_state=42)\n", + "\n", + "def plot_dataset(X, y, axes):\n", + " plt.plot(X[:, 0][y==0], X[:, 1][y==0], \"bs\")\n", + " plt.plot(X[:, 0][y==1], X[:, 1][y==1], \"g^\")\n", + " plt.axis(axes)\n", + " plt.grid(True, which='both')\n", + " plt.xlabel(r\"$x_1$\", fontsize=20)\n", + " plt.ylabel(r\"$x_2$\", fontsize=20, rotation=0)\n", + "\n", + "plot_dataset(X, y, [-1.5, 2.5, -1, 1.5])\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from sklearn.pipeline import Pipeline\n", + "from sklearn.preprocessing import PolynomialFeatures\n", + "\n", + "polynomial_svm_clf = Pipeline((\n", + " (\"poly_features\", PolynomialFeatures(degree=3)),\n", + " (\"scaler\", StandardScaler()),\n", + " (\"svm_clf\", LinearSVC(C=10, loss=\"hinge\"))\n", + " ))\n", + "\n", + "polynomial_svm_clf.fit(X, y)\n", + "\n", + "def plot_predictions(clf, axes):\n", + " x0s = np.linspace(axes[0], axes[1], 100)\n", + " x1s = np.linspace(axes[2], axes[3], 100)\n", + " x0, x1 = np.meshgrid(x0s, x1s)\n", + " X = np.c_[x0.ravel(), x1.ravel()]\n", + " y_pred = clf.predict(X).reshape(x0.shape)\n", + " y_decision = clf.decision_function(X).reshape(x0.shape)\n", + " plt.contourf(x0, x1, y_pred, cmap=plt.cm.brg, alpha=0.2)\n", + " plt.contourf(x0, x1, y_decision, cmap=plt.cm.brg, alpha=0.1)\n", + "\n", + "plot_predictions(polynomial_svm_clf, [-1.5, 2.5, -1, 1.5])\n", + "plot_dataset(X, y, [-1.5, 2.5, -1, 1.5])\n", + "\n", + "save_fig(\"moons_polynomial_svc_plot\")\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from sklearn.svm import SVC\n", + "poly_kernel_svm_clf = Pipeline((\n", + " (\"scaler\", StandardScaler()),\n", + " (\"svm_clf\", SVC(kernel=\"poly\", degree=3, coef0=1, C=5))\n", + " ))\n", + "poly100_kernel_svm_clf = Pipeline((\n", + " (\"scaler\", StandardScaler()),\n", + " (\"svm_clf\", SVC(kernel=\"poly\", degree=10, coef0=100, C=5))\n", + " ))\n", + "\n", + "poly_kernel_svm_clf.fit(X, y)\n", + "poly100_kernel_svm_clf.fit(X, y)\n", + "\n", + "plt.figure(figsize=(11, 4))\n", + "\n", + "plt.subplot(121)\n", + "plot_predictions(poly_kernel_svm_clf, [-1.5, 2.5, -1, 1.5])\n", + "plot_dataset(X, y, [-1.5, 2.5, -1, 1.5])\n", + "plt.title(r\"$d=3, r=1, C=5$\", fontsize=18)\n", + "\n", + "plt.subplot(122)\n", + "plot_predictions(poly100_kernel_svm_clf, [-1.5, 2.5, -1, 1.5])\n", + "plot_dataset(X, y, [-1.5, 2.5, -1, 1.5])\n", + "plt.title(r\"$d=10, r=100, C=5$\", fontsize=18)\n", + "\n", + "save_fig(\"moons_kernelized_polynomial_svc_plot\")\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "collapsed": false, + "scrolled": true + }, + "outputs": [], + "source": [ + "def gaussian_rbf(x, landmark, gamma):\n", + " return np.exp(-gamma * np.linalg.norm(x - landmark, axis=1)**2)\n", + "\n", + "gamma = 0.3\n", + "\n", + "x1s = np.linspace(-4.5, 4.5, 200).reshape(-1, 1)\n", + "x2s = gaussian_rbf(x1s, -2, gamma)\n", + "x3s = gaussian_rbf(x1s, 1, gamma)\n", + "\n", + "XK = np.c_[gaussian_rbf(X1D, -2, gamma), gaussian_rbf(X1D, 1, gamma)]\n", + "yk = np.array([0, 0, 1, 1, 1, 1, 1, 0, 0])\n", + "\n", + "plt.figure(figsize=(11, 4))\n", + "\n", + "plt.subplot(121)\n", + "plt.grid(True, which='both')\n", + "plt.axhline(y=0, color='k')\n", + "plt.scatter(x=[-2, 1], y=[0, 0], s=150, alpha=0.5, c=\"red\")\n", + "plt.plot(X1D[:, 0][yk==0], np.zeros(4), \"bs\")\n", + "plt.plot(X1D[:, 0][yk==1], np.zeros(5), \"g^\")\n", + "plt.plot(x1s, x2s, \"g--\")\n", + "plt.plot(x1s, x3s, \"b:\")\n", + "plt.gca().get_yaxis().set_ticks([0, 0.25, 0.5, 0.75, 1])\n", + "plt.xlabel(r\"$x_1$\", fontsize=20)\n", + "plt.ylabel(r\"Similarity\", fontsize=14)\n", + "plt.annotate(r'$\\mathbf{x}$',\n", + " xy=(X1D[3, 0], 0),\n", + " xytext=(-0.5, 0.20),\n", + " ha=\"center\",\n", + " arrowprops=dict(facecolor='black', shrink=0.1),\n", + " fontsize=18,\n", + " )\n", + "plt.text(-2, 0.9, \"$x_2$\", ha=\"center\", fontsize=20)\n", + "plt.text(1, 0.9, \"$x_3$\", ha=\"center\", fontsize=20)\n", + "plt.axis([-4.5, 4.5, -0.1, 1.1])\n", + "\n", + "plt.subplot(122)\n", + "plt.grid(True, which='both')\n", + "plt.axhline(y=0, color='k')\n", + "plt.axvline(x=0, color='k')\n", + "plt.plot(XK[:, 0][yk==0], XK[:, 1][yk==0], \"bs\")\n", + "plt.plot(XK[:, 0][yk==1], XK[:, 1][yk==1], \"g^\")\n", + "plt.xlabel(r\"$x_2$\", fontsize=20)\n", + "plt.ylabel(r\"$x_3$ \", fontsize=20, rotation=0)\n", + "plt.annotate(r'$\\phi\\left(\\mathbf{x}\\right)$',\n", + " xy=(XK[3, 0], XK[3, 1]),\n", + " xytext=(0.65, 0.50),\n", + " ha=\"center\",\n", + " arrowprops=dict(facecolor='black', shrink=0.1),\n", + " fontsize=18,\n", + " )\n", + "plt.plot([-0.1, 1.1], [0.57, -0.1], \"r--\", linewidth=3)\n", + "plt.axis([-0.1, 1.1, -0.1, 1.1])\n", + " \n", + "plt.subplots_adjust(right=1)\n", + "\n", + "save_fig(\"kernel_method_plot\")\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "x1_example = X1D[3, 0]\n", + "for landmark in (-2, 1):\n", + " k = gaussian_rbf(np.array([[x1_example]]), np.array([[landmark]]), gamma)\n", + " print(\"Phi({}, {}) = {}\".format(x1_example, landmark, k))" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "rbf_kernel_svm_clf = Pipeline((\n", + " (\"scaler\", StandardScaler()),\n", + " (\"svm_clf\", SVC(kernel=\"rbf\", gamma=5, C=0.001))\n", + " ))\n", + "rbf_kernel_svm_clf.fit(X, y)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "collapsed": false, + "scrolled": true + }, + "outputs": [], + "source": [ + "from sklearn.svm import SVC\n", + "\n", + "gamma1, gamma2 = 0.1, 5\n", + "C1, C2 = 0.001, 1000\n", + "hyperparams = (gamma1, C1), (gamma1, C2), (gamma2, C1), (gamma2, C2)\n", + "\n", + "svm_clfs = []\n", + "for gamma, C in hyperparams:\n", + " rbf_kernel_svm_clf = Pipeline((\n", + " (\"scaler\", StandardScaler()),\n", + " (\"svm_clf\", SVC(kernel=\"rbf\", gamma=gamma, C=C))\n", + " ))\n", + " rbf_kernel_svm_clf.fit(X, y)\n", + " svm_clfs.append(rbf_kernel_svm_clf)\n", + "\n", + "plt.figure(figsize=(11, 7))\n", + "\n", + "for i, svm_clf in enumerate(svm_clfs):\n", + " plt.subplot(221 + i)\n", + " plot_predictions(svm_clf, [-1.5, 2.5, -1, 1.5])\n", + " plot_dataset(X, y, [-1.5, 2.5, -1, 1.5])\n", + " gamma, C = hyperparams[i]\n", + " plt.title(r\"$\\gamma = {}, C = {}$\".format(gamma, C), fontsize=16)\n", + "\n", + "save_fig(\"moons_rbf_svc_plot\")\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Regression\n" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from sklearn.svm import LinearSVR\n", + "\n", + "rnd.seed(42)\n", + "m = 50\n", + "X = 2 * rnd.rand(m, 1)\n", + "y = (4 + 3 * X + rnd.randn(m, 1)).ravel()\n", + "\n", + "svm_reg1 = LinearSVR(epsilon=1.5)\n", + "svm_reg2 = LinearSVR(epsilon=0.5)\n", + "svm_reg1.fit(X, y)\n", + "svm_reg2.fit(X, y)\n", + "\n", + "def find_support_vectors(svm_reg, X, y):\n", + " y_pred = svm_reg.predict(X)\n", + " off_margin = (np.abs(y - y_pred) >= svm_reg.epsilon)\n", + " return np.argwhere(off_margin)\n", + "\n", + "svm_reg1.support_ = find_support_vectors(svm_reg1, X, y)\n", + "svm_reg2.support_ = find_support_vectors(svm_reg2, X, y)\n", + "\n", + "eps_x1 = 1\n", + "eps_y_pred = svm_reg1.predict([[eps_x1]])" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "def plot_svm_regression(svm_reg, X, y, axes):\n", + " x1s = np.linspace(axes[0], axes[1], 100).reshape(100, 1)\n", + " y_pred = svm_reg.predict(x1s)\n", + " plt.plot(x1s, y_pred, \"k-\", linewidth=2, label=r\"$\\hat{y}$\")\n", + " plt.plot(x1s, y_pred + svm_reg.epsilon, \"k--\")\n", + " plt.plot(x1s, y_pred - svm_reg.epsilon, \"k--\")\n", + " plt.scatter(X[svm_reg.support_], y[svm_reg.support_], s=180, facecolors='#FFAAAA')\n", + " plt.plot(X, y, \"bo\")\n", + " plt.xlabel(r\"$x_1$\", fontsize=18)\n", + " plt.legend(loc=\"upper left\", fontsize=18)\n", + " plt.axis(axes)\n", + "\n", + "plt.figure(figsize=(9, 4))\n", + "plt.subplot(121)\n", + "plot_svm_regression(svm_reg1, X, y, [0, 2, 3, 11])\n", + "plt.title(r\"$\\epsilon = {}$\".format(svm_reg1.epsilon), fontsize=18)\n", + "plt.ylabel(r\"$y$\", fontsize=18, rotation=0)\n", + "#plt.plot([eps_x1, eps_x1], [eps_y_pred, eps_y_pred - svm_reg1.epsilon], \"k-\", linewidth=2)\n", + "plt.annotate(\n", + " '', xy=(eps_x1, eps_y_pred), xycoords='data',\n", + " xytext=(eps_x1, eps_y_pred - svm_reg1.epsilon),\n", + " textcoords='data', arrowprops={'arrowstyle': '<->', 'linewidth': 1.5}\n", + " )\n", + "plt.text(0.91, 5.6, r\"$\\epsilon$\", fontsize=20)\n", + "plt.subplot(122)\n", + "plot_svm_regression(svm_reg2, X, y, [0, 2, 3, 11])\n", + "plt.title(r\"$\\epsilon = {}$\".format(svm_reg2.epsilon), fontsize=18)\n", + "save_fig(\"svm_regression_plot\")\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from sklearn.svm import SVR\n", + "\n", + "rnd.seed(42)\n", + "m = 100\n", + "X = 2 * rnd.rand(m, 1) - 1\n", + "y = (0.2 + 0.1 * X + 0.5 * X**2 + rnd.randn(m, 1)/10).ravel()\n", + "\n", + "svm_poly_reg1 = SVR(kernel=\"poly\", degree=2, C=100, epsilon=0.1)\n", + "svm_poly_reg2 = SVR(kernel=\"poly\", degree=2, C=0.01, epsilon=0.1)\n", + "svm_poly_reg1.fit(X, y)\n", + "svm_poly_reg2.fit(X, y)" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "plt.figure(figsize=(9, 4))\n", + "plt.subplot(121)\n", + "plot_svm_regression(svm_poly_reg1, X, y, [-1, 1, 0, 1])\n", + "plt.title(r\"$degree={}, C={}, \\epsilon = {}$\".format(svm_poly_reg1.degree, svm_poly_reg1.C, svm_poly_reg1.epsilon), fontsize=18)\n", + "plt.ylabel(r\"$y$\", fontsize=18, rotation=0)\n", + "plt.subplot(122)\n", + "plot_svm_regression(svm_poly_reg2, X, y, [-1, 1, 0, 1])\n", + "plt.title(r\"$degree={}, C={}, \\epsilon = {}$\".format(svm_poly_reg2.degree, svm_poly_reg2.C, svm_poly_reg2.epsilon), fontsize=18)\n", + "save_fig(\"svm_with_polynomial_kernel_plot\")\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Under the hood" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "iris = datasets.load_iris()\n", + "X = iris[\"data\"][:, (2, 3)] # petal length, petal width\n", + "y = (iris[\"target\"] == 2).astype(np.float64) # Iris-Virginica" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from mpl_toolkits.mplot3d import Axes3D\n", + "\n", + "def plot_3D_decision_function(ax, w, b, x1_lim=[4, 6], x2_lim=[0.8, 2.8]):\n", + " x1_in_bounds = (X[:, 0] > x1_lim[0]) & (X[:, 0] < x1_lim[1])\n", + " X_crop = X[x1_in_bounds]\n", + " y_crop = y[x1_in_bounds]\n", + " x1s = np.linspace(x1_lim[0], x1_lim[1], 20)\n", + " x2s = np.linspace(x2_lim[0], x2_lim[1], 20)\n", + " x1, x2 = np.meshgrid(x1s, x2s)\n", + " xs = np.c_[x1.ravel(), x2.ravel()]\n", + " df = (xs.dot(w) + b).reshape(x1.shape)\n", + " m = 1 / np.linalg.norm(w)\n", + " boundary_x2s = -x1s*(w[0]/w[1])-b/w[1]\n", + " margin_x2s_1 = -x1s*(w[0]/w[1])-(b-1)/w[1]\n", + " margin_x2s_2 = -x1s*(w[0]/w[1])-(b+1)/w[1]\n", + " ax.plot_surface(x1s, x2, 0, color=\"b\", alpha=0.2, cstride=100, rstride=100)\n", + " ax.plot(x1s, boundary_x2s, 0, \"k-\", linewidth=2, label=r\"$h=0$\")\n", + " ax.plot(x1s, margin_x2s_1, 0, \"k--\", linewidth=2, label=r\"$h=\\pm 1$\")\n", + " ax.plot(x1s, margin_x2s_2, 0, \"k--\", linewidth=2)\n", + " ax.plot(X_crop[:, 0][y_crop==1], X_crop[:, 1][y_crop==1], 0, \"g^\")\n", + " ax.plot_wireframe(x1, x2, df, alpha=0.3, color=\"k\")\n", + " ax.plot(X_crop[:, 0][y_crop==0], X_crop[:, 1][y_crop==0], 0, \"bs\")\n", + " ax.axis(x1_lim + x2_lim)\n", + " ax.text(4.5, 2.5, 3.8, \"Decision function $h$\", fontsize=15)\n", + " ax.set_xlabel(r\"Petal length\", fontsize=15)\n", + " ax.set_ylabel(r\"Petal width\", fontsize=15)\n", + " ax.set_zlabel(r\"$h = \\mathbf{w}^t \\cdot \\mathbf{x} + b$\", fontsize=18)\n", + " ax.legend(loc=\"upper left\", fontsize=16)\n", + "\n", + "fig = plt.figure(figsize=(11, 6))\n", + "ax1 = fig.add_subplot(111, projection='3d')\n", + "plot_3D_decision_function(ax1, w=svm_clf2.coef_[0], b=svm_clf2.intercept_[0])\n", + "\n", + "save_fig(\"iris_3D_plot\")\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Small weight vector results in a large margin" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "def plot_2D_decision_function(w, b, ylabel=True, x1_lim=[-3, 3]):\n", + " x1 = np.linspace(x1_lim[0], x1_lim[1], 200)\n", + " y = w * x1 + b\n", + " m = 1 / w\n", + "\n", + " plt.plot(x1, y)\n", + " plt.plot(x1_lim, [1, 1], \"k:\")\n", + " plt.plot(x1_lim, [-1, -1], \"k:\")\n", + " plt.axhline(y=0, color='k')\n", + " plt.axvline(x=0, color='k')\n", + " plt.plot([m, m], [0, 1], \"k--\")\n", + " plt.plot([-m, -m], [0, -1], \"k--\")\n", + " plt.plot([-m, m], [0, 0], \"k-o\", linewidth=3)\n", + " plt.axis(x1_lim + [-2, 2])\n", + " plt.xlabel(r\"$x_1$\", fontsize=16)\n", + " if ylabel:\n", + " plt.ylabel(r\"$w_1 x_1$ \", rotation=0, fontsize=16)\n", + " plt.title(r\"$w_1 = {}$\".format(w), fontsize=16)\n", + "\n", + "plt.figure(figsize=(12, 3.2))\n", + "plt.subplot(121)\n", + "plot_2D_decision_function(1, 0)\n", + "plt.subplot(122)\n", + "plot_2D_decision_function(0.5, 0, ylabel=False)\n", + "save_fig(\"small_w_large_margin_plot\")\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from sklearn.svm import SVC\n", + "from sklearn import datasets\n", + "\n", + "iris = datasets.load_iris()\n", + "X = iris[\"data\"][:, (2, 3)] # petal length, petal width\n", + "y = (iris[\"target\"] == 2).astype(np.float64) # Iris-Virginica\n", + "\n", + "svm_clf = SVC(kernel=\"linear\", C=1)\n", + "svm_clf.fit(X, y)\n", + "svm_clf.predict([[5.3, 1.3]])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Hinge loss" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "t = np.linspace(-2, 4, 200)\n", + "h = np.where(1 - t < 0, 0, 1 - t) # max(0, 1-t)\n", + "\n", + "plt.figure(figsize=(5,2.8))\n", + "plt.plot(t, h, \"b-\", linewidth=2, label=\"$max(0, 1 - t)$\")\n", + "plt.grid(True, which='both')\n", + "plt.axhline(y=0, color='k')\n", + "plt.axvline(x=0, color='k')\n", + "plt.yticks(np.arange(-1, 2.5, 1))\n", + "plt.xlabel(\"$t$\", fontsize=16)\n", + "plt.axis([-2, 4, -1, 2.5])\n", + "plt.legend(loc=\"upper right\", fontsize=16)\n", + "save_fig(\"hinge_plot\")\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Extra material" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Training time" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "X, y = make_moons(n_samples=1000, noise=0.4)\n", + "plt.plot(X[:, 0][y==0], X[:, 1][y==0], \"bs\")\n", + "plt.plot(X[:, 0][y==1], X[:, 1][y==1], \"g^\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "import time\n", + "\n", + "tol = 0.1\n", + "tols = []\n", + "times = []\n", + "for i in range(10):\n", + " svm_clf = SVC(kernel=\"poly\", gamma=3, C=10, tol=tol, verbose=1)\n", + " t1 = time.time()\n", + " svm_clf.fit(X, y)\n", + " t2 = time.time()\n", + " times.append(t2-t1)\n", + " tols.append(tol)\n", + " print(i, tol, t2-t1)\n", + " tol /= 10\n", + "plt.semilogx(tols, times)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Identical linear classifiers" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from sklearn.svm import SVC, LinearSVC\n", + "from sklearn.linear_model import SGDClassifier\n", + "from sklearn.datasets import make_moons\n", + "from sklearn.preprocessing import StandardScaler\n", + "\n", + "X, y = make_moons(n_samples=100, noise=0.15, random_state=42)\n", + "\n", + "C = 5\n", + "alpha = 1 / (C * len(X))\n", + "\n", + "sgd_clf = SGDClassifier(loss=\"hinge\", learning_rate=\"constant\", eta0=0.001, alpha=alpha, n_iter=100000, random_state=42)\n", + "svm_clf = SVC(kernel=\"linear\", C=C)\n", + "lin_clf = LinearSVC(loss=\"hinge\", C=C)\n", + "\n", + "X_scaled = StandardScaler().fit_transform(X)\n", + "sgd_clf.fit(X_scaled, y)\n", + "svm_clf.fit(X_scaled, y)\n", + "lin_clf.fit(X_scaled, y)\n", + "\n", + "print(\"SGDClassifier(alpha={}): \".format(sgd_clf1.alpha), sgd_clf.intercept_, sgd_clf.coef_)\n", + "print(\"SVC: \", svm_clf.intercept_, svm_clf.coef_)\n", + "print(\"LinearSVC: \", lin_clf.intercept_, lin_clf.coef_)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Linear SVM classifier implementation using Batch Gradient Descent" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# Training set\n", + "X = iris[\"data\"][:, (2, 3)] # petal length, petal width\n", + "y = (iris[\"target\"] == 2).astype(np.float64).reshape(-1, 1) # Iris-Virginica" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from sklearn.base import BaseEstimator\n", + "\n", + "class MyLinearSVC(BaseEstimator):\n", + " def __init__(self, C=1, eta0=1, eta_d=10000, n_epochs=1000, random_state=None):\n", + " self.C = C\n", + " self.eta0 = eta0\n", + " self.n_epochs = n_epochs\n", + " self.random_state = random_state\n", + " self.eta_d = eta_d\n", + "\n", + " def eta(self, epoch):\n", + " return self.eta0 / (epoch + self.eta_d)\n", + " \n", + " def fit(self, X, y):\n", + " # Random initialization\n", + " if self.random_state:\n", + " rnd.seed(self.random_state)\n", + " w = rnd.randn(X.shape[1], 1) # n feature weights\n", + " b = 0\n", + "\n", + " m = len(X)\n", + " t = y * 2 - 1 # -1 if t==0, +1 if t==1\n", + " X_t = X * t\n", + " self.Js=[]\n", + "\n", + " # Training\n", + " for epoch in range(self.n_epochs):\n", + " support_vectors_idx = (X_t.dot(w) + t * b < 1).ravel()\n", + " X_t_sv = X_t[support_vectors_idx]\n", + " t_sv = t[support_vectors_idx]\n", + "\n", + " J = 1/2 * np.sum(w * w) + self.C * (np.sum(1 - X_t_sv.dot(w)) - b * np.sum(t_sv))\n", + " self.Js.append(J)\n", + "\n", + " w_gradient_vector = w - self.C * np.sum(X_t_sv, axis=0).reshape(-1, 1)\n", + " b_derivative = -C * np.sum(t_sv)\n", + " \n", + " w = w - self.eta(epoch) * w_gradient_vector\n", + " b = b - self.eta(epoch) * b_derivative\n", + " \n", + "\n", + " self.intercept_ = np.array([b])\n", + " self.coef_ = np.array([w])\n", + " support_vectors_idx = (X_t.dot(w) + b < 1).ravel()\n", + " self.support_vectors_ = X[support_vectors_idx]\n", + " return self\n", + "\n", + " def decision_function(self, X):\n", + " return X.dot(self.coef_[0]) + self.intercept_[0]\n", + "\n", + " def predict(self, X):\n", + " return (self.decision_function(X) >= 0).astype(np.float64)\n", + "\n", + "C=2\n", + "svm_clf = MyLinearSVC(C=C, eta0 = 10, eta_d = 1000, n_epochs=60000, random_state=2)\n", + "svm_clf.fit(X, y)\n", + "svm_clf.predict(np.array([[5, 2], [4, 1]]))" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "plt.plot(range(svm_clf.n_epochs), svm_clf.Js)\n", + "plt.axis([0, svm_clf.n_epochs, 0, 100])" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "print(svm_clf.intercept_, svm_clf.coef_)" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "svm_clf2 = SVC(kernel=\"linear\", C=C)\n", + "svm_clf2.fit(X, y.ravel())\n", + "print(svm_clf2.intercept_, svm_clf2.coef_)" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "yr = y.ravel()\n", + "plt.figure(figsize=(12,3.2))\n", + "plt.subplot(121)\n", + "plt.plot(X[:, 0][yr==1], X[:, 1][yr==1], \"g^\", label=\"Iris-Virginica\")\n", + "plt.plot(X[:, 0][yr==0], X[:, 1][yr==0], \"bs\", label=\"Not Iris-Virginica\")\n", + "plot_svc_decision_boundary(svm_clf, 4, 6)\n", + "plt.xlabel(\"Petal length\", fontsize=14)\n", + "plt.ylabel(\"Petal width\", fontsize=14)\n", + "plt.title(\"MyLinearSVC\", fontsize=14)\n", + "plt.axis([4, 6, 0.8, 2.8])\n", + "\n", + "plt.subplot(122)\n", + "plt.plot(X[:, 0][yr==1], X[:, 1][yr==1], \"g^\")\n", + "plt.plot(X[:, 0][yr==0], X[:, 1][yr==0], \"bs\")\n", + "plot_svc_decision_boundary(svm_clf2, 4, 6)\n", + "plt.xlabel(\"Petal length\", fontsize=14)\n", + "plt.title(\"SVC\", fontsize=14)\n", + "plt.axis([4, 6, 0.8, 2.8])\n" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": { + "collapsed": false, + "scrolled": true + }, + "outputs": [], + "source": [ + "from sklearn.linear_model import SGDClassifier\n", + "\n", + "sgd_clf = SGDClassifier(loss=\"hinge\", alpha = 0.017, n_iter = 50, random_state=42)\n", + "sgd_clf.fit(X, y.ravel())\n", + "\n", + "m = len(X)\n", + "t = y * 2 - 1 # -1 if t==0, +1 if t==1\n", + "X_b = np.c_[np.ones((m, 1)), X] # Add bias input x0=1\n", + "X_b_t = X_b * t\n", + "sgd_theta = np.r_[sgd_clf.intercept_[0], sgd_clf.coef_[0]]\n", + "print(sgd_theta)\n", + "support_vectors_idx = (X_b_t.dot(sgd_theta) < 1).ravel()\n", + "sgd_clf.support_vectors_ = X[support_vectors_idx]\n", + "sgd_clf.C = C\n", + "\n", + "plt.figure(figsize=(5.5,3.2))\n", + "plt.plot(X[:, 0][yr==1], X[:, 1][yr==1], \"g^\")\n", + "plt.plot(X[:, 0][yr==0], X[:, 1][yr==0], \"bs\")\n", + "plot_svc_decision_boundary(sgd_clf, 4, 6)\n", + "plt.xlabel(\"Petal length\", fontsize=14)\n", + "plt.ylabel(\"Petal width\", fontsize=14)\n", + "plt.title(\"SGDClassifier\", fontsize=14)\n", + "plt.axis([4, 6, 0.8, 2.8])\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Exercise solutions" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Coming soon**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.5.1" + }, + "nav_menu": {}, + "toc": { + "navigate_menu": true, + "number_sections": true, + "sideBar": true, + "threshold": 6, + "toc_cell": false, + "toc_section_display": "block", + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/06_decision_trees.ipynb b/06_decision_trees.ipynb new file mode 100644 index 0000000..8e417ab --- /dev/null +++ b/06_decision_trees.ipynb @@ -0,0 +1,506 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Chapter 6 – Decision Trees**" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "_This notebook contains all the sample code and solutions to the exercices in chapter 6._" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Setup" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "First, let's make sure this notebook works well in both python 2 and 3, import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures:" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# To support both python 2 and python 3\n", + "from __future__ import division, print_function, unicode_literals\n", + "\n", + "# Common imports\n", + "import numpy as np\n", + "import numpy.random as rnd\n", + "import os\n", + "\n", + "# to make this notebook's output stable across runs\n", + "rnd.seed(42)\n", + "\n", + "# To plot pretty figures\n", + "%matplotlib inline\n", + "import matplotlib\n", + "import matplotlib.pyplot as plt\n", + "plt.rcParams['axes.labelsize'] = 14\n", + "plt.rcParams['xtick.labelsize'] = 12\n", + "plt.rcParams['ytick.labelsize'] = 12\n", + "\n", + "# Where to save the figures\n", + "PROJECT_ROOT_DIR = \".\"\n", + "CHAPTER_ID = \"decision_trees\"\n", + "\n", + "def image_path(fig_id):\n", + " return os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID, fig_id)\n", + "\n", + "def save_fig(fig_id, tight_layout=True):\n", + " print(\"Saving figure\", fig_id)\n", + " if tight_layout:\n", + " plt.tight_layout()\n", + " plt.savefig(image_path(fig_id) + \".png\", format='png', dpi=300)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Training and visualizing" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from sklearn.datasets import load_iris\n", + "from sklearn.tree import DecisionTreeClassifier, export_graphviz\n", + "\n", + "iris = load_iris()\n", + "X = iris.data[:, 2:] # petal length and width\n", + "y = iris.target\n", + "\n", + "tree_clf = DecisionTreeClassifier(max_depth=2, random_state=42)\n", + "tree_clf.fit(X, y)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "export_graphviz(\n", + " tree_clf,\n", + " out_file=image_path(\"iris_tree.dot\"),\n", + " feature_names=iris.feature_names[2:],\n", + " class_names=iris.target_names,\n", + " rounded=True,\n", + " filled=True\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from matplotlib.colors import ListedColormap\n", + "\n", + "def plot_decision_boundary(clf, X, y, axes=[0, 7.5, 0, 3], iris=True, legend=False, plot_training=True):\n", + " x1s = np.linspace(axes[0], axes[1], 100)\n", + " x2s = np.linspace(axes[2], axes[3], 100)\n", + " x1, x2 = np.meshgrid(x1s, x2s)\n", + " X_new = np.c_[x1.ravel(), x2.ravel()]\n", + " y_pred = clf.predict(X_new).reshape(x1.shape)\n", + " custom_cmap = ListedColormap(['#fafab0','#9898ff','#a0faa0'])\n", + " plt.contourf(x1, x2, y_pred, alpha=0.3, cmap=custom_cmap, linewidth=10)\n", + " if not iris:\n", + " custom_cmap2 = ListedColormap(['#7d7d58','#4c4c7f','#507d50'])\n", + " plt.contour(x1, x2, y_pred, cmap=custom_cmap2, alpha=0.8)\n", + " if plot_training:\n", + " plt.plot(X[:, 0][y==0], X[:, 1][y==0], \"yo\", label=\"Iris-Setosa\")\n", + " plt.plot(X[:, 0][y==1], X[:, 1][y==1], \"bs\", label=\"Iris-Versicolour\")\n", + " plt.plot(X[:, 0][y==2], X[:, 1][y==2], \"g^\", label=\"Iris-Virginica\")\n", + " plt.axis(axes)\n", + " if iris:\n", + " plt.xlabel(\"Petal length\", fontsize=14)\n", + " plt.ylabel(\"Petal width\", fontsize=14)\n", + " else:\n", + " plt.xlabel(r\"$x_1$\", fontsize=18)\n", + " plt.ylabel(r\"$x_2$\", fontsize=18, rotation=0)\n", + " if legend:\n", + " plt.legend(loc=\"lower right\", fontsize=14)\n", + "\n", + "plt.figure(figsize=(8, 4))\n", + "plot_decision_boundary(tree_clf, X, y)\n", + "plt.plot([2.45, 2.45], [0, 3], \"k-\", linewidth=2)\n", + "plt.plot([2.45, 7.5], [1.75, 1.75], \"k--\", linewidth=2)\n", + "plt.plot([4.95, 4.95], [0, 1.75], \"k:\", linewidth=2)\n", + "plt.plot([4.85, 4.85], [1.75, 3], \"k:\", linewidth=2)\n", + "plt.text(1.40, 1.0, \"Depth=0\", fontsize=15)\n", + "plt.text(3.2, 1.80, \"Depth=1\", fontsize=13)\n", + "plt.text(4.05, 0.5, \"(Depth=2)\", fontsize=11)\n", + "\n", + "save_fig(\"decision_tree_decision_boundaries_plot\")\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Predicting classes and class probabilities" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "tree_clf.predict_proba([[5, 1.5]])" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "tree_clf.predict([[5, 1.5]])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Sensitivity to training set details" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "X[(X[:, 1]==X[:, 1][y==1].max()) & (y==1)] # widest Iris-Versicolour flower" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "not_widest_versicolour = (X[:, 1]!=1.8) | (y==2)\n", + "X_tweaked = X[not_widest_versicolour]\n", + "y_tweaked = y[not_widest_versicolour]\n", + "\n", + "tree_clf_tweaked = DecisionTreeClassifier(max_depth=2, random_state=40)\n", + "tree_clf_tweaked.fit(X_tweaked, y_tweaked)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "plt.figure(figsize=(8, 4))\n", + "plot_decision_boundary(tree_clf_tweaked, X_tweaked, y_tweaked, legend=False)\n", + "plt.plot([0, 7.5], [0.8, 0.8], \"k-\", linewidth=2)\n", + "plt.plot([0, 7.5], [1.75, 1.75], \"k--\", linewidth=2)\n", + "plt.text(1.0, 0.9, \"Depth=0\", fontsize=15)\n", + "plt.text(1.0, 1.80, \"Depth=1\", fontsize=13)\n", + "\n", + "save_fig(\"decision_tree_instability_plot\")\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from sklearn.datasets import make_moons\n", + "Xm, ym = make_moons(n_samples=100, noise=0.25, random_state=53)\n", + "\n", + "deep_tree_clf1 = DecisionTreeClassifier(random_state=42)\n", + "deep_tree_clf2 = DecisionTreeClassifier(min_samples_leaf=4, random_state=42)\n", + "deep_tree_clf1.fit(Xm, ym)\n", + "deep_tree_clf2.fit(Xm, ym)\n", + "\n", + "plt.figure(figsize=(11, 4))\n", + "plt.subplot(121)\n", + "plot_decision_boundary(deep_tree_clf1, Xm, ym, axes=[-1.5, 2.5, -1, 1.5], iris=False)\n", + "plt.title(\"No restrictions\", fontsize=16)\n", + "plt.subplot(122)\n", + "plot_decision_boundary(deep_tree_clf2, Xm, ym, axes=[-1.5, 2.5, -1, 1.5], iris=False)\n", + "plt.title(\"min_samples_leaf = {}\".format(deep_tree_clf2.min_samples_leaf), fontsize=14)\n", + "\n", + "save_fig(\"min_samples_leaf_plot\")\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "angle = np.pi / 180 * 20\n", + "rotation_matrix = np.array([[np.cos(angle), -np.sin(angle)], [np.sin(angle), np.cos(angle)]])\n", + "Xr = X.dot(rotation_matrix)\n", + "\n", + "tree_clf_r = DecisionTreeClassifier(random_state=42)\n", + "tree_clf_r.fit(Xr, y)\n", + "\n", + "plt.figure(figsize=(8, 3))\n", + "plot_decision_boundary(tree_clf_r, Xr, y, axes=[0.5, 7.5, -1.0, 1], iris=False)\n", + "\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "rnd.seed(6)\n", + "Xs = rnd.rand(100, 2) - 0.5\n", + "ys = (Xs[:, 0] > 0).astype(np.float32) * 2\n", + "\n", + "angle = np.pi / 4\n", + "rotation_matrix = np.array([[np.cos(angle), -np.sin(angle)], [np.sin(angle), np.cos(angle)]])\n", + "Xsr = Xs.dot(rotation_matrix)\n", + "\n", + "tree_clf_s = DecisionTreeClassifier(random_state=42)\n", + "tree_clf_s.fit(Xs, ys)\n", + "tree_clf_sr = DecisionTreeClassifier(random_state=42)\n", + "tree_clf_sr.fit(Xsr, ys)\n", + "\n", + "plt.figure(figsize=(11, 4))\n", + "plt.subplot(121)\n", + "plot_decision_boundary(tree_clf_s, Xs, ys, axes=[-0.7, 0.7, -0.7, 0.7], iris=False)\n", + "plt.subplot(122)\n", + "plot_decision_boundary(tree_clf_sr, Xsr, ys, axes=[-0.7, 0.7, -0.7, 0.7], iris=False)\n", + "\n", + "save_fig(\"sensitivity_to_rotation_plot\")\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Regression trees" + ] + }, + { + "cell_type": "code", + "execution_count": 145, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from sklearn.tree import DecisionTreeRegressor\n", + "\n", + "# Quadratic training set + noise\n", + "rnd.seed(42)\n", + "m = 200\n", + "X = rnd.rand(m, 1)\n", + "y = 4 * (X - 0.5) ** 2\n", + "y = y + rnd.randn(m, 1) / 10\n", + "\n", + "tree_reg1 = DecisionTreeRegressor(random_state=42, max_depth=2)\n", + "tree_reg2 = DecisionTreeRegressor(random_state=42, max_depth=3)\n", + "tree_reg1.fit(X, y)\n", + "tree_reg2.fit(X, y)\n", + "\n", + "def plot_regression_predictions(tree_reg, X, y, axes=[0, 1, -0.2, 1], ylabel=\"$y$\"):\n", + " x1 = np.linspace(axes[0], axes[1], 500).reshape(-1, 1)\n", + " y_pred = tree_reg.predict(x1)\n", + " plt.axis(axes)\n", + " plt.xlabel(\"$x_1$\", fontsize=18)\n", + " if ylabel:\n", + " plt.ylabel(ylabel, fontsize=18, rotation=0)\n", + " plt.plot(X, y, \"b.\")\n", + " plt.plot(x1, y_pred, \"r.-\", linewidth=2, label=r\"$\\hat{y}$\")\n", + "\n", + "plt.figure(figsize=(11, 4))\n", + "plt.subplot(121)\n", + "plot_regression_predictions(tree_reg1, X, y)\n", + "for split, style in ((0.1973, \"k-\"), (0.0917, \"k--\"), (0.7718, \"k--\")):\n", + " plt.plot([split, split], [-0.2, 1], style, linewidth=2)\n", + "plt.text(0.21, 0.65, \"Depth=0\", fontsize=15)\n", + "plt.text(0.01, 0.2, \"Depth=1\", fontsize=13)\n", + "plt.text(0.65, 0.8, \"Depth=1\", fontsize=13)\n", + "plt.legend(loc=\"upper center\", fontsize=18)\n", + "plt.title(\"max_depth=2\", fontsize=14)\n", + "\n", + "plt.subplot(122)\n", + "plot_regression_predictions(tree_reg2, X, y, ylabel=None)\n", + "for split, style in ((0.1973, \"k-\"), (0.0917, \"k--\"), (0.7718, \"k--\")):\n", + " plt.plot([split, split], [-0.2, 1], style, linewidth=2)\n", + "for split in (0.0458, 0.1298, 0.2873, 0.9040):\n", + " plt.plot([split, split], [-0.2, 1], \"k:\", linewidth=1)\n", + "plt.text(0.3, 0.5, \"Depth=2\", fontsize=13)\n", + "plt.title(\"max_depth=3\", fontsize=14)\n", + "\n", + "save_fig(\"tree_regression_plot\")\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 131, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "export_graphviz(\n", + " tree_reg1,\n", + " out_file=image_path(\"regression_tree.dot\"),\n", + " feature_names=[\"x1\"],\n", + " rounded=True,\n", + " filled=True\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": 144, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "tree_reg1 = DecisionTreeRegressor(random_state=42)\n", + "tree_reg2 = DecisionTreeRegressor(random_state=42, min_samples_leaf=10)\n", + "tree_reg1.fit(X, y)\n", + "tree_reg2.fit(X, y)\n", + "\n", + "x1 = np.linspace(0, 1, 500).reshape(-1, 1)\n", + "y_pred1 = tree_reg1.predict(x1)\n", + "y_pred2 = tree_reg2.predict(x1)\n", + "\n", + "plt.figure(figsize=(11, 4))\n", + "\n", + "plt.subplot(121)\n", + "plt.plot(X, y, \"b.\")\n", + "plt.plot(x1, y_pred1, \"r.-\", linewidth=2, label=r\"$\\hat{y}$\")\n", + "plt.axis([0, 1, -0.2, 1.1])\n", + "plt.xlabel(\"$x_1$\", fontsize=18)\n", + "plt.ylabel(\"$y$\", fontsize=18, rotation=0)\n", + "plt.legend(loc=\"upper center\", fontsize=18)\n", + "plt.title(\"No restrictions\", fontsize=14)\n", + "\n", + "plt.subplot(122)\n", + "plt.plot(X, y, \"b.\")\n", + "plt.plot(x1, y_pred2, \"r.-\", linewidth=2, label=r\"$\\hat{y}$\")\n", + "plt.axis([0, 1, -0.2, 1.1])\n", + "plt.xlabel(\"$x_1$\", fontsize=18)\n", + "plt.title(\"min_samples_leaf={}\".format(tree_reg2.min_samples_leaf), fontsize=14)\n", + "\n", + "save_fig(\"tree_regression_regularization_plot\")\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": true + }, + "source": [ + "# Exercise solutions" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Coming soon**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.5.1" + }, + "nav_menu": { + "height": "309px", + "width": "468px" + }, + "toc": { + "navigate_menu": true, + "number_sections": true, + "sideBar": true, + "threshold": 6, + "toc_cell": false, + "toc_section_display": "block", + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/07_ensemble_learning_and_random_forests.ipynb b/07_ensemble_learning_and_random_forests.ipynb new file mode 100644 index 0000000..0c716b8 --- /dev/null +++ b/07_ensemble_learning_and_random_forests.ipynb @@ -0,0 +1,788 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Chapter 7 – Ensemble Learning and Random Forests**" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "_This notebook contains all the sample code and solutions to the exercices in chapter 7._" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Setup" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "First, let's make sure this notebook works well in both python 2 and 3, import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures:" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# To support both python 2 and python 3\n", + "from __future__ import division, print_function, unicode_literals\n", + "\n", + "# Common imports\n", + "import numpy as np\n", + "import numpy.random as rnd\n", + "import os\n", + "\n", + "# to make this notebook's output stable across runs\n", + "rnd.seed(42)\n", + "\n", + "# To plot pretty figures\n", + "%matplotlib inline\n", + "import matplotlib\n", + "import matplotlib.pyplot as plt\n", + "plt.rcParams['axes.labelsize'] = 14\n", + "plt.rcParams['xtick.labelsize'] = 12\n", + "plt.rcParams['ytick.labelsize'] = 12\n", + "\n", + "# Where to save the figures\n", + "PROJECT_ROOT_DIR = \".\"\n", + "CHAPTER_ID = \"ensembles\"\n", + "\n", + "def image_path(fig_id):\n", + " return os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID, fig_id)\n", + "\n", + "def save_fig(fig_id, tight_layout=True):\n", + " print(\"Saving figure\", fig_id)\n", + " if tight_layout:\n", + " plt.tight_layout()\n", + " plt.savefig(image_path(fig_id) + \".png\", format='png', dpi=300)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Voting classifiers" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "heads_proba = 0.51\n", + "coin_tosses = (rnd.rand(10000, 10) < heads_proba).astype(np.int32)\n", + "cumulative_heads_ratio = np.cumsum(coin_tosses, axis=0) / np.arange(1, 10001).reshape(-1, 1)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "plt.figure(figsize=(8,3.5))\n", + "plt.plot(cumulative_heads_ratio)\n", + "plt.plot([0, 10000], [0.51, 0.51], \"k--\", linewidth=2, label=\"51%\")\n", + "plt.plot([0, 10000], [0.5, 0.5], \"k-\", label=\"50%\")\n", + "plt.xlabel(\"Number of coin tosses\")\n", + "plt.ylabel(\"Heads ratio\")\n", + "plt.legend(loc=\"lower right\")\n", + "plt.axis([0, 10000, 0.42, 0.58])\n", + "save_fig(\"law_of_large_numbers_plot\")\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from sklearn.cross_validation import train_test_split\n", + "from sklearn.datasets import make_moons\n", + "\n", + "X, y = make_moons(n_samples=500, noise=0.30, random_state=42)\n", + "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)\n", + "\n", + "from sklearn.ensemble import RandomForestClassifier\n", + "from sklearn.ensemble import VotingClassifier\n", + "from sklearn.linear_model import LogisticRegression\n", + "from sklearn.svm import SVC\n", + "\n", + "log_clf = LogisticRegression(random_state=42)\n", + "rnd_clf = RandomForestClassifier(random_state=42)\n", + "svm_clf = SVC(probability=True, random_state=42)\n", + "\n", + "voting_clf = VotingClassifier(\n", + " estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],\n", + " voting='soft'\n", + " )\n", + "voting_clf.fit(X_train, y_train)\n", + "\n", + "from sklearn.metrics import accuracy_score\n", + "\n", + "for clf in (log_clf, rnd_clf, svm_clf, voting_clf):\n", + " clf.fit(X_train, y_train)\n", + " y_pred = clf.predict(X_test)\n", + " print(clf.__class__.__name__, accuracy_score(y_test, y_pred))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Bagging ensembles" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from sklearn.datasets import make_moons\n", + "from sklearn.ensemble import BaggingClassifier\n", + "from sklearn.metrics import accuracy_score\n", + "from sklearn.tree import DecisionTreeClassifier\n", + "\n", + "bag_clf = BaggingClassifier(\n", + " DecisionTreeClassifier(random_state=42), n_estimators=500,\n", + " max_samples=100, bootstrap=True, n_jobs=-1, random_state=42\n", + " )\n", + "bag_clf.fit(X_train, y_train)\n", + "y_pred = bag_clf.predict(X_test)\n", + "print(accuracy_score(y_test, y_pred))" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "tree_clf = DecisionTreeClassifier(random_state=42)\n", + "tree_clf.fit(X_train, y_train)\n", + "y_pred_tree = tree_clf.predict(X_test)\n", + "print(accuracy_score(y_test, y_pred_tree))" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "from matplotlib.colors import ListedColormap\n", + "\n", + "def plot_decision_boundary(clf, X, y, axes=[-1.5, 2.5, -1, 1.5], alpha=0.5, contour=True):\n", + " x1s = np.linspace(axes[0], axes[1], 100)\n", + " x2s = np.linspace(axes[2], axes[3], 100)\n", + " x1, x2 = np.meshgrid(x1s, x2s)\n", + " X_new = np.c_[x1.ravel(), x2.ravel()]\n", + " y_pred = clf.predict(X_new).reshape(x1.shape)\n", + " custom_cmap = ListedColormap(['#fafab0','#9898ff','#a0faa0'])\n", + " plt.contourf(x1, x2, y_pred, alpha=0.3, cmap=custom_cmap, linewidth=10)\n", + " if contour:\n", + " custom_cmap2 = ListedColormap(['#7d7d58','#4c4c7f','#507d50'])\n", + " plt.contour(x1, x2, y_pred, cmap=custom_cmap2, alpha=0.8)\n", + " plt.plot(X[:, 0][y==0], X[:, 1][y==0], \"yo\", alpha=alpha)\n", + " plt.plot(X[:, 0][y==1], X[:, 1][y==1], \"bs\", alpha=alpha)\n", + " plt.axis(axes)\n", + " plt.xlabel(r\"$x_1$\", fontsize=18)\n", + " plt.ylabel(r\"$x_2$\", fontsize=18, rotation=0)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "plt.figure(figsize=(11,4))\n", + "plt.subplot(121)\n", + "plot_decision_boundary(tree_clf, X, y)\n", + "plt.title(\"Decision Tree\", fontsize=14)\n", + "plt.subplot(122)\n", + "plot_decision_boundary(bag_clf, X, y)\n", + "plt.title(\"Decision Trees with Bagging\", fontsize=14)\n", + "save_fig(\"decision_tree_without_and_with_bagging_plot\")\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Random Forests" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "bag_clf = BaggingClassifier(\n", + " DecisionTreeClassifier(splitter=\"random\", max_leaf_nodes=16, random_state=42),\n", + " n_estimators=500, max_samples=1.0, bootstrap=True,\n", + " n_jobs=-1, random_state=42\n", + " )\n", + "bag_clf.fit(X_train, y_train)\n", + "y_pred = bag_clf.predict(X_test)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "from sklearn.ensemble import RandomForestClassifier\n", + "\n", + "rnd_clf = RandomForestClassifier(n_estimators=500, max_leaf_nodes=16, n_jobs=-1, random_state=42)\n", + "rnd_clf.fit(X_train, y_train)\n", + "\n", + "y_pred_rf = rnd_clf.predict(X_test)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "np.sum(y_pred == y_pred_rf) / len(y_pred) # almost identical predictions" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from sklearn.datasets import load_iris\n", + "iris = load_iris()\n", + "rnd_clf = RandomForestClassifier(n_estimators=500, n_jobs=-1, random_state=42)\n", + "rnd_clf.fit(iris[\"data\"], iris[\"target\"])\n", + "for name, importance in zip(iris[\"feature_names\"], rnd_clf.feature_importances_):\n", + " print(name, \"=\", importance)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "rnd_clf.feature_importances_" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "plt.figure(figsize=(6, 4))\n", + "\n", + "for i in range(15):\n", + " tree_clf = DecisionTreeClassifier(max_leaf_nodes=16, random_state=42+i)\n", + " indices_with_replacement = rnd.randint(0, len(X_train), len(X_train))\n", + " tree_clf.fit(X[indices_with_replacement], y[indices_with_replacement])\n", + " plot_decision_boundary(tree_clf, X, y, axes=[-1.5, 2.5, -1, 1.5], alpha=0.02, contour=False)\n", + "\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Out-of-Bag evaluation" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "bag_clf = BaggingClassifier(\n", + " DecisionTreeClassifier(random_state=42), n_estimators=500,\n", + " bootstrap=True, n_jobs=-1, oob_score=True, random_state=40\n", + ")\n", + "bag_clf.fit(X_train, y_train)\n", + "bag_clf.oob_score_" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "bag_clf.oob_decision_function_[:10]" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from sklearn.metrics import accuracy_score\n", + "y_pred = bag_clf.predict(X_test)\n", + "accuracy_score(y_test, y_pred)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Feature importance" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from sklearn.datasets import fetch_mldata\n", + "mnist = fetch_mldata('MNIST original')\n", + "rnd_clf = RandomForestClassifier(random_state=42)\n", + "rnd_clf.fit(mnist[\"data\"], mnist[\"target\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "def plot_digit(data):\n", + " image = data.reshape(28, 28)\n", + " plt.imshow(image, cmap = matplotlib.cm.hot,\n", + " interpolation=\"nearest\")\n", + " plt.axis(\"off\")" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "plot_digit(rnd_clf.feature_importances_)\n", + "\n", + "cbar = plt.colorbar(ticks=[rnd_clf.feature_importances_.min(), rnd_clf.feature_importances_.max()])\n", + "cbar.ax.set_yticklabels(['Not important', 'Very important'])\n", + "\n", + "save_fig(\"mnist_feature_importance_plot\")\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# AdaBoost" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from sklearn.ensemble import AdaBoostClassifier\n", + "\n", + "ada_clf = AdaBoostClassifier(\n", + " DecisionTreeClassifier(max_depth=2), n_estimators=200,\n", + " algorithm=\"SAMME.R\", learning_rate=0.5, random_state=42\n", + " )\n", + "ada_clf.fit(X_train, y_train)\n", + "plot_decision_boundary(ada_clf, X, y)" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "m = len(X_train)\n", + "\n", + "plt.figure(figsize=(11, 4))\n", + "for subplot, learning_rate in ((121, 1), (122, 0.5)):\n", + " sample_weights = np.ones(m)\n", + " for i in range(5):\n", + " plt.subplot(subplot)\n", + " svm_clf = SVC(kernel=\"rbf\", C=0.05)\n", + " svm_clf.fit(X_train, y_train, sample_weight=sample_weights)\n", + " y_pred = svm_clf.predict(X_train)\n", + " sample_weights[y_pred != y_train] *= (1 + learning_rate)\n", + " plot_decision_boundary(svm_clf, X, y, alpha=0.2)\n", + " plt.title(\"learning_rate = {}\".format(learning_rate - 1), fontsize=16)\n", + "\n", + "plt.subplot(121)\n", + "plt.text(-0.7, -0.65, \"1\", fontsize=14)\n", + "plt.text(-0.6, -0.10, \"2\", fontsize=14)\n", + "plt.text(-0.5, 0.10, \"3\", fontsize=14)\n", + "plt.text(-0.4, 0.55, \"4\", fontsize=14)\n", + "plt.text(-0.3, 0.90, \"5\", fontsize=14)\n", + "save_fig(\"boosting_plot\")\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "list(m for m in dir(ada_clf) if not m.startswith(\"_\") and m.endswith(\"_\"))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Gradient Boosting" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from sklearn.tree import DecisionTreeRegressor\n", + "\n", + "rnd.seed(42)\n", + "X = rnd.rand(100, 1) - 0.5\n", + "y = 3*X[:, 0]**2 + 0.05 * rnd.randn(100)\n", + "\n", + "tree_reg1 = DecisionTreeRegressor(max_depth=2, random_state=42)\n", + "tree_reg1.fit(X, y)\n", + "\n", + "y2 = y - tree_reg1.predict(X)\n", + "tree_reg2 = DecisionTreeRegressor(max_depth=2, random_state=42)\n", + "tree_reg2.fit(X, y2)\n", + "\n", + "y3 = y2 - tree_reg2.predict(X)\n", + "tree_reg3 = DecisionTreeRegressor(max_depth=2, random_state=42)\n", + "tree_reg3.fit(X, y3)\n", + "\n", + "X_new = np.array([[0.8]])\n", + "y_pred = sum(tree.predict(X_new) for tree in (tree_reg1, tree_reg2, tree_reg3))\n", + "print(y_pred)" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "def plot_predictions(regressors, X, y, axes, label=None, style=\"r-\", data_style=\"b.\", data_label=None):\n", + " x1 = np.linspace(axes[0], axes[1], 500)\n", + " y_pred = sum(regressor.predict(x1.reshape(-1, 1)) for regressor in regressors)\n", + " plt.plot(X[:, 0], y, data_style, label=data_label)\n", + " plt.plot(x1, y_pred, style, linewidth=2, label=label)\n", + " if label or data_label:\n", + " plt.legend(loc=\"upper center\", fontsize=16)\n", + " plt.axis(axes)\n", + "\n", + "plt.figure(figsize=(11,11))\n", + "\n", + "plt.subplot(321)\n", + "plot_predictions([tree_reg1], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label=\"$h_1(x_1)$\", style=\"g-\", data_label=\"Training set\")\n", + "plt.ylabel(\"$y$\", fontsize=16, rotation=0)\n", + "plt.title(\"Residuals and tree predictions\", fontsize=16)\n", + "\n", + "plt.subplot(322)\n", + "plot_predictions([tree_reg1], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label=\"$h(x_1) = h_1(x_1)$\", data_label=\"Training set\")\n", + "plt.ylabel(\"$y$\", fontsize=16, rotation=0)\n", + "plt.title(\"Ensemble predictions\", fontsize=16)\n", + "\n", + "plt.subplot(323)\n", + "plot_predictions([tree_reg2], X, y2, axes=[-0.5, 0.5, -0.5, 0.5], label=\"$h_2(x_1)$\", style=\"g-\", data_style=\"k+\", data_label=\"Residuals\")\n", + "plt.ylabel(\"$y - h_1(x_1)$\", fontsize=16)\n", + "\n", + "plt.subplot(324)\n", + "plot_predictions([tree_reg1, tree_reg2], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label=\"$h(x_1) = h_1(x_1) + h_2(x_1)$\")\n", + "plt.ylabel(\"$y$\", fontsize=16, rotation=0)\n", + "\n", + "plt.subplot(325)\n", + "plot_predictions([tree_reg3], X, y3, axes=[-0.5, 0.5, -0.5, 0.5], label=\"$h_3(x_1)$\", style=\"g-\", data_style=\"k+\")\n", + "plt.ylabel(\"$y - h_1(x_1) - h_2(x_1)$\", fontsize=16)\n", + "plt.xlabel(\"$x_1$\", fontsize=16)\n", + "\n", + "plt.subplot(326)\n", + "plot_predictions([tree_reg1, tree_reg2, tree_reg3], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label=\"$h(x_1) = h_1(x_1) + h_2(x_1) + h_3(x_1)$\")\n", + "plt.xlabel(\"$x_1$\", fontsize=16)\n", + "plt.ylabel(\"$y$\", fontsize=16, rotation=0)\n", + "\n", + "save_fig(\"gradient_boosting_plot\")\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from sklearn.ensemble import GradientBoostingRegressor\n", + "\n", + "gbrt = GradientBoostingRegressor(max_depth=2, n_estimators=3, learning_rate=0.1, random_state=42)\n", + "gbrt.fit(X, y)\n", + "\n", + "gbrt_slow = GradientBoostingRegressor(max_depth=2, n_estimators=200, learning_rate=0.1, random_state=42)\n", + "gbrt_slow.fit(X, y)\n", + "\n", + "plt.figure(figsize=(11,4))\n", + "\n", + "plt.subplot(121)\n", + "plot_predictions([gbrt], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label=\"Ensemble predictions\")\n", + "plt.title(\"learning_rate={}, n_estimators={}\".format(gbrt.learning_rate, gbrt.n_estimators), fontsize=14)\n", + "\n", + "plt.subplot(122)\n", + "plot_predictions([gbrt_slow], X, y, axes=[-0.5, 0.5, -0.1, 0.8])\n", + "plt.title(\"learning_rate={}, n_estimators={}\".format(gbrt_slow.learning_rate, gbrt_slow.n_estimators), fontsize=14)\n", + "\n", + "save_fig(\"gbrt_learning_rate_plot\")\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Gradient Boosting with Early stopping" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from sklearn.cross_validation import train_test_split\n", + "from sklearn.metrics import mean_squared_error\n", + "\n", + "X_train, X_val, y_train, y_val = train_test_split(X, y)\n", + "\n", + "gbrt = GradientBoostingRegressor(max_depth=2, n_estimators=120, learning_rate=0.1, random_state=42)\n", + "gbrt.fit(X_train, y_train)\n", + "\n", + "errors = [mean_squared_error(y_val, y_pred) for y_pred in gbrt.staged_predict(X_val)]" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "best_n_estimators = np.argmin(errors)\n", + "min_error = errors[best_n_estimators]\n", + "\n", + "gbrt_best = GradientBoostingRegressor(max_depth=2, n_estimators=best_n_estimators, learning_rate=0.1, random_state=42)\n", + "gbrt_best.fit(X_train, y_train)" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "plt.figure(figsize=(11, 4))\n", + "\n", + "plt.subplot(121)\n", + "plt.plot(errors, \"b.-\")\n", + "plt.plot([best_n_estimators, best_n_estimators], [0, min_error], \"k--\")\n", + "plt.plot([0, 120], [min_error, min_error], \"k--\")\n", + "plt.plot(best_n_estimators, min_error, \"ko\")\n", + "plt.text(best_n_estimators, min_error*1.2, \"Minimum\", ha=\"center\", fontsize=14)\n", + "plt.axis([0, 120, 0, 0.01])\n", + "plt.xlabel(\"Number of trees\")\n", + "plt.title(\"Validation error\", fontsize=14)\n", + "\n", + "plt.subplot(122)\n", + "plot_predictions([gbrt_best], X, y, axes=[-0.5, 0.5, -0.1, 0.8])\n", + "plt.title(\"Best model (55 trees)\", fontsize=14)\n", + "\n", + "save_fig(\"early_stopping_gbrt_plot\")\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "gbrt = GradientBoostingRegressor(max_depth=2, n_estimators=1, learning_rate=0.1, random_state=42, warm_start=True)\n", + "\n", + "min_val_error = float(\"inf\")\n", + "error_going_up = 0\n", + "for n_estimators in range(1, 120):\n", + " gbrt.n_estimators = n_estimators\n", + " gbrt.fit(X_train, y_train)\n", + " y_pred = gbrt.predict(X_val)\n", + " val_error = mean_squared_error(y_val, y_pred)\n", + " if val_error < min_val_error:\n", + " min_val_error = val_error\n", + " error_going_up = 0\n", + " else:\n", + " error_going_up += 1\n", + " if error_going_up == 5:\n", + " break # early stopping" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "print(gbrt.n_estimators)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": true + }, + "source": [ + "# Exercise solutions" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Coming soon**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.5.1" + }, + "nav_menu": { + "height": "252px", + "width": "333px" + }, + "toc": { + "navigate_menu": true, + "number_sections": true, + "sideBar": true, + "threshold": 6, + "toc_cell": false, + "toc_section_display": "block", + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/08_dimensionality_reduction.ipynb b/08_dimensionality_reduction.ipynb new file mode 100644 index 0000000..fa135f9 --- /dev/null +++ b/08_dimensionality_reduction.ipynb @@ -0,0 +1,1343 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Chapter 8 – Dimensionality Reduction**" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "_This notebook contains all the sample code and solutions to the exercices in chapter 8._" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Setup" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "First, let's make sure this notebook works well in both python 2 and 3, import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures:" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# To support both python 2 and python 3\n", + "from __future__ import division, print_function, unicode_literals\n", + "\n", + "# Common imports\n", + "import numpy as np\n", + "import numpy.random as rnd\n", + "import os\n", + "\n", + "# to make this notebook's output stable across runs\n", + "rnd.seed(42)\n", + "\n", + "# To plot pretty figures\n", + "%matplotlib inline\n", + "import matplotlib\n", + "import matplotlib.pyplot as plt\n", + "plt.rcParams['axes.labelsize'] = 14\n", + "plt.rcParams['xtick.labelsize'] = 12\n", + "plt.rcParams['ytick.labelsize'] = 12\n", + "\n", + "# Where to save the figures\n", + "PROJECT_ROOT_DIR = \".\"\n", + "CHAPTER_ID = \"dim_reduction\"\n", + "\n", + "def save_fig(fig_id, tight_layout=True):\n", + " path = os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID, fig_id + \".png\")\n", + " print(\"Saving figure\", fig_id)\n", + " if tight_layout:\n", + " plt.tight_layout()\n", + " plt.savefig(path, format='png', dpi=300)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Projection methods\n", + "Build 3D dataset:" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "rnd.seed(4)\n", + "m = 60\n", + "w1, w2 = 0.1, 0.3\n", + "noise = 0.1\n", + "\n", + "angles = rnd.rand(m) * 3 * np.pi / 2 - 0.5\n", + "X = np.empty((m, 3))\n", + "X[:, 0] = np.cos(angles) + np.sin(angles)/2 + noise * rnd.randn(m) / 2\n", + "X[:, 1] = np.sin(angles) * 0.7 + noise * rnd.randn(m) / 2\n", + "X[:, 2] = X[:, 0] * w1 + X[:, 1] * w2 + noise * rnd.randn(m)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Mean normalize the data:" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "X = X - X.mean(axis=0)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Apply PCA to reduce to 2D." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from sklearn.decomposition import PCA\n", + "\n", + "pca = PCA(n_components = 2)\n", + "X2D = pca.fit_transform(X)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Recover the 3D points projected on the plane (PCA 2D subspace)." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "X2D_inv = pca.inverse_transform(X2D)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Utility class to draw 3D arrows (copied from http://stackoverflow.com/questions/11140163)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "from matplotlib.patches import FancyArrowPatch\n", + "from mpl_toolkits.mplot3d import proj3d\n", + "\n", + "class Arrow3D(FancyArrowPatch):\n", + " def __init__(self, xs, ys, zs, *args, **kwargs):\n", + " FancyArrowPatch.__init__(self, (0,0), (0,0), *args, **kwargs)\n", + " self._verts3d = xs, ys, zs\n", + "\n", + " def draw(self, renderer):\n", + " xs3d, ys3d, zs3d = self._verts3d\n", + " xs, ys, zs = proj3d.proj_transform(xs3d, ys3d, zs3d, renderer.M)\n", + " self.set_positions((xs[0],ys[0]),(xs[1],ys[1]))\n", + " FancyArrowPatch.draw(self, renderer)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Express the plane as a function of x and y." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "axes = [-1.8, 1.8, -1.3, 1.3, -1.0, 1.0]\n", + "\n", + "x1s = np.linspace(axes[0], axes[1], 10)\n", + "x2s = np.linspace(axes[2], axes[3], 10)\n", + "x1, x2 = np.meshgrid(x1s, x2s)\n", + "\n", + "C = pca.components_\n", + "R = C.T.dot(C)\n", + "z = (R[0, 2] * x1 + R[1, 2] * x2) / (1 - R[2, 2])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Plot the 3D dataset, the plane and the projections on that plane." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from mpl_toolkits.mplot3d import Axes3D\n", + "\n", + "fig = plt.figure(figsize=(6, 3.8))\n", + "ax = fig.add_subplot(111, projection='3d')\n", + "\n", + "X3D_above = X[X[:, 2] > X2D_inv[:, 2]]\n", + "X3D_below = X[X[:, 2] <= X2D_inv[:, 2]]\n", + "\n", + "ax.plot(X3D_below[:, 0], X3D_below[:, 1], X3D_below[:, 2], \"bo\", alpha=0.5)\n", + "\n", + "ax.plot_surface(x1, x2, z, alpha=0.2, color=\"k\")\n", + "np.linalg.norm(C, axis=0)\n", + "ax.add_artist(Arrow3D([0, C[0, 0]],[0, C[0, 1]],[0, C[0, 2]], mutation_scale=15, lw=1, arrowstyle=\"-|>\", color=\"k\"))\n", + "ax.add_artist(Arrow3D([0, C[1, 0]],[0, C[1, 1]],[0, C[1, 2]], mutation_scale=15, lw=1, arrowstyle=\"-|>\", color=\"k\"))\n", + "ax.plot([0], [0], [0], \"k.\")\n", + "\n", + "for i in range(m):\n", + " if X[i, 2] > X2D_inv[i, 2]:\n", + " ax.plot([X[i][0], X2D_inv[i][0]], [X[i][1], X2D_inv[i][1]], [X[i][2], X2D_inv[i][2]], \"k-\")\n", + " else:\n", + " ax.plot([X[i][0], X2D_inv[i][0]], [X[i][1], X2D_inv[i][1]], [X[i][2], X2D_inv[i][2]], \"k-\", color=\"#505050\")\n", + " \n", + "ax.plot(X2D_inv[:, 0], X2D_inv[:, 1], X2D_inv[:, 2], \"k+\")\n", + "ax.plot(X2D_inv[:, 0], X2D_inv[:, 1], X2D_inv[:, 2], \"k.\")\n", + "ax.plot(X3D_above[:, 0], X3D_above[:, 1], X3D_above[:, 2], \"bo\")\n", + "ax.set_xlabel(\"$x_1$\", fontsize=18)\n", + "ax.set_ylabel(\"$x_2$\", fontsize=18)\n", + "ax.set_zlabel(\"$x_3$\", fontsize=18)\n", + "ax.set_xlim(axes[0:2])\n", + "ax.set_ylim(axes[2:4])\n", + "ax.set_zlim(axes[4:6])\n", + "\n", + "save_fig(\"dataset_3d_plot\")\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "fig = plt.figure()\n", + "ax = fig.add_subplot(111, aspect='equal')\n", + "\n", + "ax.plot(X2D[:, 0], X2D[:, 1], \"k+\")\n", + "ax.plot(X2D[:, 0], X2D[:, 1], \"k.\")\n", + "ax.plot([0], [0], \"ko\")\n", + "ax.arrow(0, 0, 0, 1, head_width=0.05, length_includes_head=True, head_length=0.1, fc='k', ec='k')\n", + "ax.arrow(0, 0, 1, 0, head_width=0.05, length_includes_head=True, head_length=0.1, fc='k', ec='k')\n", + "ax.set_xlabel(\"$z_1$\", fontsize=18)\n", + "ax.set_ylabel(\"$z_2$\", fontsize=18, rotation=0)\n", + "ax.axis([-1.5, 1.3, -1.2, 1.2])\n", + "ax.grid(True)\n", + "save_fig(\"dataset_2d_plot\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "PCA using SVD decomposition" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "m, n = X.shape\n", + "\n", + "X_centered = X - X.mean(axis=0)\n", + "U, s, V = np.linalg.svd(X_centered)\n", + "c1 = V.T[:, 0]\n", + "c2 = V.T[:, 1]\n", + "\n", + "S = np.zeros(X.shape)\n", + "S[:n, :n] = np.diag(s)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "np.allclose(X, U.dot(S).dot(V))" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "T = X.dot(V.T[:, :2])" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "np.allclose(T, U.dot(S)[:, :2])" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from sklearn.decomposition import PCA\n", + "pca = PCA(n_components = 2)\n", + "X2D_p = pca.fit_transform(X)\n", + "np.allclose(X2D_p, T)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "X3D_recover = T.dot(V[:2, :])" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "np.allclose(X3D_recover, pca.inverse_transform(X2D_p))" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "V" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "pca.components_" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "R = pca.components_.T.dot(pca.components_)" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "S[:3]" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "pca.explained_variance_ratio_" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "1 - pca.explained_variance_ratio_.sum()" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "S[0,0]**2/(S**2).sum(), S[1,1]**2/(S**2).sum()" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "np.sqrt((T[:, 1]**2).sum())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Manifold learning\n", + "Swiss roll:" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "from sklearn.datasets import make_swiss_roll\n", + "X, t = make_swiss_roll(n_samples=1000, noise=0.2, random_state=42)" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "axes = [-11.5, 14, -2, 23, -12, 15]\n", + "\n", + "fig = plt.figure(figsize=(6, 5))\n", + "ax = fig.add_subplot(111, projection='3d')\n", + "\n", + "ax.scatter(X[:, 0], X[:, 1], X[:, 2], c=t, cmap=plt.cm.hot)\n", + "ax.view_init(10, -70)\n", + "ax.set_xlabel(\"$x_1$\", fontsize=18)\n", + "ax.set_ylabel(\"$x_2$\", fontsize=18)\n", + "ax.set_zlabel(\"$x_3$\", fontsize=18)\n", + "ax.set_xlim(axes[0:2])\n", + "ax.set_ylim(axes[2:4])\n", + "ax.set_zlim(axes[4:6])\n", + "\n", + "save_fig(\"swiss_roll_plot\")\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "plt.figure(figsize=(11, 4))\n", + "\n", + "plt.subplot(121)\n", + "plt.scatter(X[:, 0], X[:, 1], c=t, cmap=plt.cm.hot)\n", + "plt.axis(axes[:4])\n", + "plt.xlabel(\"$x_1$\", fontsize=18)\n", + "plt.ylabel(\"$x_2$\", fontsize=18, rotation=0)\n", + "plt.grid(True)\n", + "\n", + "plt.subplot(122)\n", + "plt.scatter(t, X[:, 1], c=t, cmap=plt.cm.hot)\n", + "plt.axis([4, 15, axes[2], axes[3]])\n", + "plt.xlabel(\"$z_1$\", fontsize=18)\n", + "plt.grid(True)\n", + "\n", + "save_fig(\"squished_swiss_roll_plot\")\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from matplotlib import gridspec\n", + "\n", + "axes = [-11.5, 14, -2, 23, -12, 15]\n", + "\n", + "x2s = np.linspace(axes[2], axes[3], 10)\n", + "x3s = np.linspace(axes[4], axes[5], 10)\n", + "x2, x3 = np.meshgrid(x2s, x3s)\n", + "\n", + "fig = plt.figure(figsize=(6, 5))\n", + "ax = plt.subplot(111, projection='3d')\n", + "\n", + "positive_class = X[:, 0] > 5\n", + "X_pos = X[positive_class]\n", + "X_neg = X[~positive_class]\n", + "ax.view_init(10, -70)\n", + "ax.plot(X_neg[:, 0], X_neg[:, 1], X_neg[:, 2], \"y^\")\n", + "ax.plot_wireframe(5, x2, x3, alpha=0.5)\n", + "ax.plot(X_pos[:, 0], X_pos[:, 1], X_pos[:, 2], \"gs\")\n", + "ax.set_xlabel(\"$x_1$\", fontsize=18)\n", + "ax.set_ylabel(\"$x_2$\", fontsize=18)\n", + "ax.set_zlabel(\"$x_3$\", fontsize=18)\n", + "ax.set_xlim(axes[0:2])\n", + "ax.set_ylim(axes[2:4])\n", + "ax.set_zlim(axes[4:6])\n", + "\n", + "save_fig(\"manifold_decision_boundary_plot1\")\n", + "plt.show()\n", + "\n", + "fig = plt.figure(figsize=(5, 4))\n", + "ax = plt.subplot(111)\n", + "\n", + "plt.plot(t[positive_class], X[positive_class, 1], \"gs\")\n", + "plt.plot(t[~positive_class], X[~positive_class, 1], \"y^\")\n", + "plt.axis([4, 15, axes[2], axes[3]])\n", + "plt.xlabel(\"$z_1$\", fontsize=18)\n", + "plt.ylabel(\"$z_2$\", fontsize=18, rotation=0)\n", + "plt.grid(True)\n", + "\n", + "save_fig(\"manifold_decision_boundary_plot2\")\n", + "plt.show()\n", + "\n", + "fig = plt.figure(figsize=(6, 5))\n", + "ax = plt.subplot(111, projection='3d')\n", + "\n", + "positive_class = 2 * (t[:] - 4) > X[:, 1]\n", + "X_pos = X[positive_class]\n", + "X_neg = X[~positive_class]\n", + "ax.view_init(10, -70)\n", + "ax.plot(X_neg[:, 0], X_neg[:, 1], X_neg[:, 2], \"y^\")\n", + "ax.plot(X_pos[:, 0], X_pos[:, 1], X_pos[:, 2], \"gs\")\n", + "ax.set_xlabel(\"$x_1$\", fontsize=18)\n", + "ax.set_ylabel(\"$x_2$\", fontsize=18)\n", + "ax.set_zlabel(\"$x_3$\", fontsize=18)\n", + "ax.set_xlim(axes[0:2])\n", + "ax.set_ylim(axes[2:4])\n", + "ax.set_zlim(axes[4:6])\n", + "\n", + "save_fig(\"manifold_decision_boundary_plot3\")\n", + "plt.show()\n", + "\n", + "fig = plt.figure(figsize=(5, 4))\n", + "ax = plt.subplot(111)\n", + "\n", + "plt.plot(t[positive_class], X[positive_class, 1], \"gs\")\n", + "plt.plot(t[~positive_class], X[~positive_class, 1], \"y^\")\n", + "plt.plot([4, 15], [0, 22], \"b-\", linewidth=2)\n", + "plt.axis([4, 15, axes[2], axes[3]])\n", + "plt.xlabel(\"$z_1$\", fontsize=18)\n", + "plt.ylabel(\"$z_2$\", fontsize=18, rotation=0)\n", + "plt.grid(True)\n", + "\n", + "save_fig(\"manifold_decision_boundary_plot4\")\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# PCA" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "angle = np.pi / 5\n", + "stretch = 5\n", + "m = 200\n", + "\n", + "rnd.seed(3)\n", + "X = rnd.randn(m, 2) / 10\n", + "X = X.dot(np.array([[stretch, 0],[0, 1]])) # stretch\n", + "X = X.dot([[np.cos(angle), np.sin(angle)], [-np.sin(angle), np.cos(angle)]]) # rotate\n", + "\n", + "u1 = np.array([np.cos(angle), np.sin(angle)])\n", + "u2 = np.array([np.cos(angle - 2 * np.pi/6), np.sin(angle - 2 * np.pi/6)])\n", + "u3 = np.array([np.cos(angle - np.pi/2), np.sin(angle - np.pi/2)])\n", + "\n", + "X_proj1 = X.dot(u1.reshape(-1, 1))\n", + "X_proj2 = X.dot(u2.reshape(-1, 1))\n", + "X_proj3 = X.dot(u3.reshape(-1, 1))\n", + "\n", + "plt.figure(figsize=(8,4))\n", + "plt.subplot2grid((3,2), (0, 0), rowspan=3)\n", + "plt.plot([-1.4, 1.4], [-1.4*u1[1]/u1[0], 1.4*u1[1]/u1[0]], \"k-\", linewidth=1)\n", + "plt.plot([-1.4, 1.4], [-1.4*u2[1]/u2[0], 1.4*u2[1]/u2[0]], \"k--\", linewidth=1)\n", + "plt.plot([-1.4, 1.4], [-1.4*u3[1]/u3[0], 1.4*u3[1]/u3[0]], \"k:\", linewidth=2)\n", + "plt.plot(X[:, 0], X[:, 1], \"bo\", alpha=0.5)\n", + "plt.axis([-1.4, 1.4, -1.4, 1.4])\n", + "plt.arrow(0, 0, u1[0], u1[1], head_width=0.1, linewidth=5, length_includes_head=True, head_length=0.1, fc='k', ec='k')\n", + "plt.arrow(0, 0, u3[0], u3[1], head_width=0.1, linewidth=5, length_includes_head=True, head_length=0.1, fc='k', ec='k')\n", + "plt.text(u1[0] + 0.1, u1[1] - 0.05, r\"$\\mathbf{c_1}$\", fontsize=22)\n", + "plt.text(u3[0] + 0.1, u3[1], r\"$\\mathbf{c_2}$\", fontsize=22)\n", + "plt.xlabel(\"$x_1$\", fontsize=18)\n", + "plt.ylabel(\"$x_2$\", fontsize=18, rotation=0)\n", + "plt.grid(True)\n", + "\n", + "plt.subplot2grid((3,2), (0, 1))\n", + "plt.plot([-2, 2], [0, 0], \"k-\", linewidth=1)\n", + "plt.plot(X_proj1[:, 0], np.zeros(m), \"bo\", alpha=0.3)\n", + "plt.gca().get_yaxis().set_ticks([])\n", + "plt.gca().get_xaxis().set_ticklabels([])\n", + "plt.axis([-2, 2, -1, 1])\n", + "plt.grid(True)\n", + "\n", + "plt.subplot2grid((3,2), (1, 1))\n", + "plt.plot([-2, 2], [0, 0], \"k--\", linewidth=1)\n", + "plt.plot(X_proj2[:, 0], np.zeros(m), \"bo\", alpha=0.3)\n", + "plt.gca().get_yaxis().set_ticks([])\n", + "plt.gca().get_xaxis().set_ticklabels([])\n", + "plt.axis([-2, 2, -1, 1])\n", + "plt.grid(True)\n", + "\n", + "plt.subplot2grid((3,2), (2, 1))\n", + "plt.plot([-2, 2], [0, 0], \"k:\", linewidth=2)\n", + "plt.plot(X_proj3[:, 0], np.zeros(m), \"bo\", alpha=0.3)\n", + "plt.gca().get_yaxis().set_ticks([])\n", + "plt.axis([-2, 2, -1, 1])\n", + "plt.xlabel(\"$z_1$\", fontsize=18)\n", + "plt.grid(True)\n", + "\n", + "save_fig(\"pca_best_projection\")\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# MNIST compression" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "from sklearn.cross_validation import train_test_split\n", + "from sklearn.datasets import fetch_mldata\n", + "\n", + "mnist = fetch_mldata('MNIST original')\n", + "X = mnist[\"data\"]\n", + "y = mnist[\"target\"]\n", + "\n", + "X_train, X_test, y_train, y_test = train_test_split(X, y)" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "X = X_train\n", + "\n", + "pca = PCA()\n", + "pca.fit(X)\n", + "d = np.argmax(np.cumsum(pca.explained_variance_ratio_) >= 0.95) + 1\n", + "d" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "pca = PCA(n_components=0.95)\n", + "X_reduced = pca.fit_transform(X)\n", + "pca.n_components_" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "np.sum(pca.explained_variance_ratio_)" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "X_mnist = X_train\n", + "\n", + "pca = PCA(n_components = 154)\n", + "X_mnist_reduced = pca.fit_transform(X_mnist)\n", + "X_mnist_recovered = pca.inverse_transform(X_mnist_reduced)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "def plot_digits(instances, images_per_row=5, **options):\n", + " size = 28\n", + " images_per_row = min(len(instances), images_per_row)\n", + " images = [instance.reshape(size,size) for instance in instances]\n", + " n_rows = (len(instances) - 1) // images_per_row + 1\n", + " row_images = []\n", + " n_empty = n_rows * images_per_row - len(instances)\n", + " images.append(np.zeros((size, size * n_empty)))\n", + " for row in range(n_rows):\n", + " rimages = images[row * images_per_row : (row + 1) * images_per_row]\n", + " row_images.append(np.concatenate(rimages, axis=1))\n", + " image = np.concatenate(row_images, axis=0)\n", + " plt.imshow(image, cmap = matplotlib.cm.binary, **options)\n", + " plt.axis(\"off\")" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "plt.figure(figsize=(7, 4))\n", + "plt.subplot(121)\n", + "plot_digits(X_mnist[::2100])\n", + "plt.title(\"Original\", fontsize=16)\n", + "plt.subplot(122)\n", + "plot_digits(X_mnist_recovered[::2100])\n", + "plt.title(\"Compressed\", fontsize=16)\n", + "\n", + "save_fig(\"mnist_compression_plot\")" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from sklearn.decomposition import IncrementalPCA\n", + "\n", + "n_batches = 100\n", + "inc_pca = IncrementalPCA(n_components=154)\n", + "for X_batch in np.array_split(X_mnist, n_batches):\n", + " print(\".\", end=\"\")\n", + " inc_pca.partial_fit(X_batch)\n", + "\n", + "X_mnist_reduced_inc = inc_pca.transform(X_mnist)" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "X_mnist_recovered_inc = inc_pca.inverse_transform(X_mnist_reduced_inc)" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "plt.figure(figsize=(7, 4))\n", + "plt.subplot(121)\n", + "plot_digits(X_mnist[::2100])\n", + "plt.subplot(122)\n", + "plot_digits(X_mnist_recovered_inc[::2100])\n", + "plt.tight_layout()" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "np.allclose(pca.mean_, inc_pca.mean_)" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "np.allclose(X_mnist_reduced, X_mnist_reduced_inc)" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "filename = \"my_mnist.data\"\n", + "\n", + "X_mm = np.memmap(filename, dtype='float32', mode='write', shape=X_mnist.shape)\n", + "X_mm[:] = X_mnist\n", + "del X_mm" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "X_mm = np.memmap(filename, dtype='float32', mode='readonly', shape=X_mnist.shape)\n", + "\n", + "batch_size = len(X_mnist) // n_batches\n", + "inc_pca = IncrementalPCA(n_components=154, batch_size=batch_size)\n", + "inc_pca.fit(X_mm)" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from sklearn.decomposition import RandomizedPCA\n", + "\n", + "rnd_pca = RandomizedPCA(n_components=154, random_state=42)\n", + "X_reduced = rnd_pca.fit_transform(X_mnist)" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "import time\n", + "\n", + "for n_components in (2, 10, 154):\n", + " print(\"n_components =\", n_components)\n", + " regular_pca = PCA(n_components=n_components)\n", + " inc_pca = IncrementalPCA(n_components=154, batch_size=500)\n", + " rnd_pca = RandomizedPCA(n_components=154, random_state=42)\n", + "\n", + " for pca in (regular_pca, inc_pca, rnd_pca):\n", + " t1 = time.time()\n", + " pca.fit(X_mnist)\n", + " t2 = time.time()\n", + " print(pca.__class__.__name__, t2 - t1, \"seconds\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Kernel PCA" + ] + }, + { + "cell_type": "code", + "execution_count": 46, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from sklearn.decomposition import KernelPCA\n", + "\n", + "X, t = make_swiss_roll(n_samples=1000, noise=0.2, random_state=42)\n", + "\n", + "lin_pca = KernelPCA(n_components = 2, kernel=\"linear\", fit_inverse_transform=True)\n", + "rbf_pca = KernelPCA(n_components = 2, kernel=\"rbf\", gamma=0.0433, fit_inverse_transform=True)\n", + "sig_pca = KernelPCA(n_components = 2, kernel=\"sigmoid\", gamma=0.001, coef0=1, fit_inverse_transform=True)\n", + "\n", + "y = t > 6.9\n", + "\n", + "plt.figure(figsize=(11, 4))\n", + "for subplot, pca, title in ((131, lin_pca, \"Linear kernel\"), (132, rbf_pca, \"RBF kernel, $\\gamma=0.04$\"), (133, sig_pca, \"Sigmoid kernel, $\\gamma=10^{-3}, r=1$\")):\n", + " X_reduced = pca.fit_transform(X)\n", + " if subplot == 132:\n", + " X_reduced_rbf = X_reduced\n", + " \n", + " plt.subplot(subplot)\n", + " #plt.plot(X_reduced[y, 0], X_reduced[y, 1], \"gs\")\n", + " #plt.plot(X_reduced[~y, 0], X_reduced[~y, 1], \"y^\")\n", + " plt.title(title, fontsize=14)\n", + " plt.scatter(X_reduced[:, 0], X_reduced[:, 1], c=t, cmap=plt.cm.hot)\n", + " plt.xlabel(\"$z_1$\", fontsize=18)\n", + " if subplot == 131:\n", + " plt.ylabel(\"$z_2$\", fontsize=18, rotation=0)\n", + " plt.grid(True)\n", + "\n", + "save_fig(\"kernel_pca_plot\")\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 47, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "plt.figure(figsize=(6, 5))\n", + "\n", + "X_inverse = pca.inverse_transform(X_reduced_rbf)\n", + "\n", + "ax = plt.subplot(111, projection='3d')\n", + "ax.view_init(10, -70)\n", + "ax.scatter(X_inverse[:, 0], X_inverse[:, 1], X_inverse[:, 2], c=t, cmap=plt.cm.hot, marker=\"x\")\n", + "ax.set_xlabel(\"\")\n", + "ax.set_ylabel(\"\")\n", + "ax.set_zlabel(\"\")\n", + "ax.set_xticklabels([])\n", + "ax.set_yticklabels([])\n", + "ax.set_zticklabels([])\n", + "\n", + "save_fig(\"preimage_plot\", tight_layout=False)\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 48, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "X_reduced = rbf_pca.fit_transform(X)\n", + "\n", + "plt.figure(figsize=(11, 4))\n", + "plt.subplot(132)\n", + "plt.scatter(X_reduced[:, 0], X_reduced[:, 1], c=t, cmap=plt.cm.hot, marker=\"x\")\n", + "plt.xlabel(\"$z_1$\", fontsize=18)\n", + "plt.ylabel(\"$z_2$\", fontsize=18, rotation=0)\n", + "plt.grid(True)" + ] + }, + { + "cell_type": "code", + "execution_count": 49, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from sklearn.pipeline import Pipeline\n", + "from sklearn.linear_model import LogisticRegression\n", + "from sklearn.grid_search import GridSearchCV\n", + "\n", + "clf = Pipeline([\n", + " (\"kpca\", KernelPCA(n_components=2)),\n", + " (\"log_reg\", LogisticRegression())\n", + " ])\n", + "\n", + "param_grid = [\n", + " {\"kpca__gamma\": np.linspace(0.03, 0.05, 10), \"kpca__kernel\": [\"rbf\", \"sigmoid\"]}\n", + " ]\n", + "\n", + "grid_search = GridSearchCV(clf, param_grid, cv=3)\n", + "grid_search.fit(X, y)\n", + "grid_search.best_params_" + ] + }, + { + "cell_type": "code", + "execution_count": 50, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "rbf_pca = KernelPCA(n_components = 2, kernel=\"rbf\", gamma=0.0433,\n", + " fit_inverse_transform=True)\n", + "X_reduced = rbf_pca.fit_transform(X)\n", + "X_preimage = rbf_pca.inverse_transform(X_reduced)" + ] + }, + { + "cell_type": "code", + "execution_count": 51, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from sklearn.metrics import mean_squared_error\n", + "mean_squared_error(X, X_preimage)" + ] + }, + { + "cell_type": "code", + "execution_count": 52, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "times_rpca = []\n", + "times_pca = []\n", + "sizes = [1000, 10000, 20000, 30000, 40000, 50000, 70000, 100000, 200000, 500000]\n", + "for n_samples in sizes:\n", + " X = rnd.randn(n_samples, 5)\n", + " pca = RandomizedPCA(n_components = 2, random_state=42)\n", + " t1 = time.time()\n", + " pca.fit(X)\n", + " t2 = time.time()\n", + " times_rpca.append(t2 - t1)\n", + " pca = PCA(n_components = 2)\n", + " t1 = time.time()\n", + " pca.fit(X)\n", + " t2 = time.time()\n", + " times_pca.append(t2 - t1)\n", + "\n", + "plt.plot(sizes, times_rpca, \"b-o\", label=\"RPCA\")\n", + "plt.plot(sizes, times_pca, \"r-s\", label=\"PCA\")\n", + "plt.xlabel(\"n_samples\")\n", + "plt.ylabel(\"Training time\")\n", + "plt.legend(loc=\"upper left\")\n", + "plt.title(\"PCA and Randomized PCA time complexity \")" + ] + }, + { + "cell_type": "code", + "execution_count": 55, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "times_rpca = []\n", + "times_pca = []\n", + "sizes = [1000, 2000, 3000, 4000, 5000, 6000]\n", + "for n_features in sizes:\n", + " X = rnd.randn(2000, n_features)\n", + " pca = RandomizedPCA(n_components = 2, random_state=42)\n", + " t1 = time.time()\n", + " pca.fit(X)\n", + " t2 = time.time()\n", + " times_rpca.append(t2 - t1)\n", + " pca = PCA(n_components = 2)\n", + " t1 = time.time()\n", + " pca.fit(X)\n", + " t2 = time.time()\n", + " times_pca.append(t2 - t1)\n", + "\n", + "plt.plot(sizes, times_rpca, \"b-o\", label=\"RPCA\")\n", + "plt.plot(sizes, times_pca, \"r-s\", label=\"PCA\")\n", + "plt.xlabel(\"n_features\")\n", + "plt.ylabel(\"Training time\")\n", + "plt.legend(loc=\"upper left\")\n", + "plt.title(\"PCA and Randomized PCA time complexity \")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# LLE" + ] + }, + { + "cell_type": "code", + "execution_count": 72, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from sklearn.manifold import LocallyLinearEmbedding\n", + "\n", + "X, t = make_swiss_roll(n_samples=1000, noise=0.2, random_state=41)\n", + "\n", + "lle = LocallyLinearEmbedding(n_neighbors=10, n_components=2, random_state=42)\n", + "X_reduced = lle.fit_transform(X)\n", + "\n", + "plt.title(\"Unrolled swiss roll using LLE\", fontsize=14)\n", + "plt.scatter(X_reduced[:, 0], X_reduced[:, 1], c=t, cmap=plt.cm.hot)\n", + "plt.xlabel(\"$z_1$\", fontsize=18)\n", + "plt.ylabel(\"$z_2$\", fontsize=18)\n", + "plt.axis([-0.065, 0.055, -0.1, 0.12])\n", + "plt.grid(True)\n", + "\n", + "save_fig(\"lle_unrolling_plot\")\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# MDS, Isomap and t-SNE" + ] + }, + { + "cell_type": "code", + "execution_count": 107, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from sklearn.manifold import MDS\n", + "mds = MDS(n_components=2, random_state=42)\n", + "X_reduced_mds = mds.fit_transform(X)" + ] + }, + { + "cell_type": "code", + "execution_count": 108, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from sklearn.manifold import Isomap\n", + "isomap = Isomap(n_components=2)\n", + "X_reduced_isomap = isomap.fit_transform(X)" + ] + }, + { + "cell_type": "code", + "execution_count": 109, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "from sklearn.manifold import TSNE\n", + "tsne = TSNE(n_components=2)\n", + "X_reduced_tsne = tsne.fit_transform(X)" + ] + }, + { + "cell_type": "code", + "execution_count": 128, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from sklearn.discriminant_analysis import LinearDiscriminantAnalysis\n", + "lda = LinearDiscriminantAnalysis(n_components=2)\n", + "X_mnist = mnist[\"data\"]\n", + "y_mnist = mnist[\"target\"]\n", + "lda.fit(X_mnist, y_mnist)\n", + "X_reduced_lda = lda.transform(X_mnist)" + ] + }, + { + "cell_type": "code", + "execution_count": 136, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "titles = [\"MDS\", \"Isomap\", \"t-SNE\"]\n", + "\n", + "plt.figure(figsize=(11,4))\n", + "\n", + "for subplot, title, X_reduced in zip((131, 132, 133), titles,\n", + " (X_reduced_mds, X_reduced_isomap, X_reduced_tsne)):\n", + " plt.subplot(subplot)\n", + " plt.title(title, fontsize=14)\n", + " plt.scatter(X_reduced[:, 0], X_reduced[:, 1], c=t, cmap=plt.cm.hot)\n", + " plt.xlabel(\"$z_1$\", fontsize=18)\n", + " if subplot == 131:\n", + " plt.ylabel(\"$z_2$\", fontsize=18, rotation=0)\n", + " plt.grid(True)\n", + "\n", + "save_fig(\"other_dim_reduction_plot\")\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": true + }, + "source": [ + "# Exercise solutions" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Coming soon**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.5.1" + }, + "nav_menu": { + "height": "352px", + "width": "458px" + }, + "toc": { + "navigate_menu": true, + "number_sections": true, + "sideBar": true, + "threshold": 6, + "toc_cell": false, + "toc_section_display": "block", + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/09_up_and_running_with_tensorflow.ipynb b/09_up_and_running_with_tensorflow.ipynb new file mode 100644 index 0000000..138ad7d --- /dev/null +++ b/09_up_and_running_with_tensorflow.ipynb @@ -0,0 +1,1709 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Chapter 9 – Up and running with TensorFlow**" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "_This notebook contains all the sample code and solutions to the exercices in chapter 9._" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Setup" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "First, let's make sure this notebook works well in both python 2 and 3, import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures:" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# To support both python 2 and python 3\n", + "from __future__ import division, print_function, unicode_literals\n", + "\n", + "# Common imports\n", + "import numpy as np\n", + "import numpy.random as rnd\n", + "import os\n", + "\n", + "# to make this notebook's output stable across runs\n", + "rnd.seed(42)\n", + "\n", + "# To plot pretty figures\n", + "%matplotlib inline\n", + "import matplotlib\n", + "import matplotlib.pyplot as plt\n", + "plt.rcParams['axes.labelsize'] = 14\n", + "plt.rcParams['xtick.labelsize'] = 12\n", + "plt.rcParams['ytick.labelsize'] = 12\n", + "\n", + "# Where to save the figures\n", + "PROJECT_ROOT_DIR = \".\"\n", + "CHAPTER_ID = \"tensorflow\"\n", + "\n", + "def save_fig(fig_id, tight_layout=True):\n", + " path = os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID, fig_id + \".png\")\n", + " print(\"Saving figure\", fig_id)\n", + " if tight_layout:\n", + " plt.tight_layout()\n", + " plt.savefig(path, format='png', dpi=300)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Creating and running a graph" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "import tensorflow as tf" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "\n", + "x = tf.Variable(3, name=\"x\")\n", + "y = tf.Variable(4, name=\"y\")\n", + "f = x*x*y + y + 2\n", + "\n", + "f" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "sess = tf.Session()\n", + "sess.run(x.initializer)\n", + "sess.run(y.initializer)\n", + "print(sess.run(f))\n", + "sess.close()" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "with tf.Session() as sess:\n", + " x.initializer.run()\n", + " y.initializer.run()\n", + " result = f.eval()\n", + "\n", + "result" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "init = tf.initialize_all_variables()\n", + "\n", + "with tf.Session():\n", + " init.run()\n", + " result = f.eval()\n", + "\n", + "result" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "init = tf.initialize_all_variables()\n", + "\n", + "sess = tf.InteractiveSession()\n", + "init.run()\n", + "result = f.eval()\n", + "sess.close()\n", + "\n", + "result" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Managing graphs" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "\n", + "x1 = tf.Variable(1)\n", + "x1.graph is tf.get_default_graph()" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "graph = tf.Graph()\n", + "with graph.as_default():\n", + " x2 = tf.Variable(2)\n", + "\n", + "x2.graph is tf.get_default_graph()" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "collapsed": false, + "scrolled": true + }, + "outputs": [], + "source": [ + "x2.graph is graph" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "w = tf.constant(3)\n", + "x = w + 2\n", + "y = x + 5\n", + "z = x * 3\n", + "\n", + "with tf.Session() as sess:\n", + " print(y.eval()) # 10\n", + " print(z.eval()) # 15" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "with tf.Session() as sess:\n", + " y_val, z_val = sess.run([y, z])\n", + " print(y) # 10\n", + " print(z) # 15" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Linear Regression" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using the Normal Equation" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from sklearn.datasets import fetch_california_housing\n", + "\n", + "housing = fetch_california_housing()\n", + "m, n = housing.data.shape\n", + "housing_data_plus_bias = np.c_[np.ones((m, 1)), housing.data]" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "\n", + "X = tf.constant(housing_data_plus_bias, dtype=tf.float64, name=\"X\")\n", + "y = tf.constant(housing.target.reshape(-1, 1), dtype=tf.float64, name=\"y\")\n", + "XT = tf.transpose(X)\n", + "theta = tf.matmul(tf.matmul(tf.matrix_inverse(tf.matmul(XT, X)), XT), y)\n", + "\n", + "with tf.Session() as sess:\n", + " result = theta.eval()\n", + "\n", + "print(result)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Compare with pure NumPy" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "X = housing_data_plus_bias\n", + "y = housing.target.reshape(-1, 1)\n", + "theta_numpy = np.linalg.inv(X.T.dot(X)).dot(X.T).dot(y)\n", + "\n", + "print(theta_numpy)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Compare with Scikit-Learn" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from sklearn.linear_model import LinearRegression\n", + "lin_reg = LinearRegression()\n", + "lin_reg.fit(housing.data, housing.target.reshape(-1, 1))\n", + "\n", + "print(np.r_[lin_reg.intercept_.reshape(-1, 1), lin_reg.coef_.T])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using Batch Gradient Descent" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Gradient Descent requires scaling the feature vectors first. We could do this using TF, but let's just use Scikit-Learn for now." + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "from sklearn.preprocessing import StandardScaler\n", + "scaler = StandardScaler()\n", + "scaled_housing_data = scaler.fit_transform(housing.data)\n", + "scaled_housing_data_plus_bias = np.c_[np.ones((m, 1)), scaled_housing_data]" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "print(scaled_housing_data_plus_bias.mean(axis=0))\n", + "print(scaled_housing_data_plus_bias.mean(axis=1))\n", + "print(scaled_housing_data_plus_bias.mean())\n", + "print(scaled_housing_data_plus_bias.shape)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Manually computing the gradients" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "\n", + "n_epochs = 1000\n", + "learning_rate = 0.01\n", + "\n", + "X = tf.constant(scaled_housing_data_plus_bias, dtype=tf.float32, name=\"X\")\n", + "y = tf.constant(housing.target.reshape(-1, 1), dtype=tf.float32, name=\"y\")\n", + "theta = tf.Variable(tf.random_uniform([n + 1, 1], -1.0, 1.0, seed=42), name=\"theta\")\n", + "y_pred = tf.matmul(X, theta, name=\"predictions\")\n", + "error = y_pred - y\n", + "mse = tf.reduce_mean(tf.square(error), name=\"mse\")\n", + "gradients = 2/m * tf.matmul(tf.transpose(X), error)\n", + "training_op = tf.assign(theta, theta - learning_rate * gradients)\n", + "\n", + "init = tf.initialize_all_variables()\n", + "\n", + "with tf.Session() as sess:\n", + " sess.run(init)\n", + "\n", + " for epoch in range(n_epochs):\n", + " if epoch % 100 == 0:\n", + " print(\"Epoch\", epoch, \"MSE =\", mse.eval())\n", + " sess.run(training_op)\n", + " \n", + " best_theta = theta.eval()\n", + "\n", + "print(\"Best theta:\")\n", + "print(best_theta)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Using autodiff\n", + "Same as above except for the `gradients = ...` line." + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "\n", + "n_epochs = 1000\n", + "learning_rate = 0.01\n", + "\n", + "X = tf.constant(scaled_housing_data_plus_bias, dtype=tf.float32, name=\"X\")\n", + "y = tf.constant(housing.target.reshape(-1, 1), dtype=tf.float32, name=\"y\")\n", + "theta = tf.Variable(tf.random_uniform([n + 1, 1], -1.0, 1.0, seed=42), name=\"theta\")\n", + "y_pred = tf.matmul(X, theta, name=\"predictions\")\n", + "error = y_pred - y\n", + "mse = tf.reduce_mean(tf.square(error), name=\"mse\")\n", + "gradients = tf.gradients(mse, [theta])[0]\n", + "training_op = tf.assign(theta, theta - learning_rate * gradients)\n", + "\n", + "init = tf.initialize_all_variables()\n", + "\n", + "with tf.Session() as sess:\n", + " sess.run(init)\n", + "\n", + " for epoch in range(n_epochs):\n", + " if epoch % 100 == 0:\n", + " print(\"Epoch\", epoch, \"MSE =\", mse.eval())\n", + " sess.run(training_op)\n", + " \n", + " best_theta = theta.eval()\n", + "\n", + "print(\"Best theta:\")\n", + "print(best_theta)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Using a `GradientDescentOptimizer`" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "\n", + "n_epochs = 1000\n", + "learning_rate = 0.01\n", + "\n", + "X = tf.constant(scaled_housing_data_plus_bias, dtype=tf.float32, name=\"X\")\n", + "y = tf.constant(housing.target.reshape(-1, 1), dtype=tf.float32, name=\"y\")\n", + "theta = tf.Variable(tf.random_uniform([n + 1, 1], -1.0, 1.0, seed=42), name=\"theta\")\n", + "y_pred = tf.matmul(X, theta, name=\"predictions\")\n", + "error = y_pred - y\n", + "mse = tf.reduce_mean(tf.square(error), name=\"mse\")\n", + "optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)\n", + "training_op = optimizer.minimize(mse)\n", + "\n", + "init = tf.initialize_all_variables()\n", + "\n", + "with tf.Session() as sess:\n", + " sess.run(init)\n", + "\n", + " for epoch in range(n_epochs):\n", + " if epoch % 100 == 0:\n", + " print(\"Epoch\", epoch, \"MSE =\", mse.eval())\n", + " sess.run(training_op)\n", + " \n", + " best_theta = theta.eval()\n", + "\n", + "print(\"Best theta:\")\n", + "print(best_theta)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Using a momentum optimizer" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "\n", + "n_epochs = 1000\n", + "learning_rate = 0.01\n", + "\n", + "X = tf.constant(scaled_housing_data_plus_bias, dtype=tf.float32, name=\"X\")\n", + "y = tf.constant(housing.target.reshape(-1, 1), dtype=tf.float32, name=\"y\")\n", + "theta = tf.Variable(tf.random_uniform([n + 1, 1], -1.0, 1.0, seed=42), name=\"theta\")\n", + "y_pred = tf.matmul(X, theta, name=\"predictions\")\n", + "error = y_pred - y\n", + "mse = tf.reduce_mean(tf.square(error), name=\"mse\")\n", + "optimizer = tf.train.MomentumOptimizer(learning_rate=learning_rate, momentum=0.25)\n", + "training_op = optimizer.minimize(mse)\n", + "\n", + "init = tf.initialize_all_variables()" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "with tf.Session() as sess:\n", + " sess.run(init)\n", + "\n", + " for epoch in range(n_epochs):\n", + " sess.run(training_op)\n", + " \n", + " best_theta = theta.eval()\n", + "\n", + "print(\"Best theta:\")\n", + "print(best_theta)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Feeding data to the training algorithm" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Placeholder nodes" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + ">>> tf.reset_default_graph()\n", + "\n", + ">>> A = tf.placeholder(tf.float32, shape=(None, 3))\n", + ">>> B = A + 5\n", + ">>> with tf.Session() as sess:\n", + "... B_val_1 = B.eval(feed_dict={A: [[1, 2, 3]]})\n", + "... B_val_2 = B.eval(feed_dict={A: [[4, 5, 6], [7, 8, 9]]})\n", + "...\n", + ">>> print(B_val_1)\n", + ">>> print(B_val_2)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Mini-batch Gradient Descent" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "\n", + "n_epochs = 1000\n", + "learning_rate = 0.01\n", + "\n", + "X = tf.placeholder(tf.float32, shape=(None, n + 1), name=\"X\")\n", + "y = tf.placeholder(tf.float32, shape=(None, 1), name=\"y\")\n", + "theta = tf.Variable(tf.random_uniform([n + 1, 1], -1.0, 1.0, seed=42), name=\"theta\")\n", + "y_pred = tf.matmul(X, theta, name=\"predictions\")\n", + "error = y_pred - y\n", + "mse = tf.reduce_mean(tf.square(error), name=\"mse\")\n", + "optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)\n", + "training_op = optimizer.minimize(mse)\n", + "\n", + "init = tf.initialize_all_variables()" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "def fetch_batch(epoch, batch_index, batch_size):\n", + " rnd.seed(epoch * n_batches + batch_index)\n", + " indices = rnd.randint(m, size=batch_size)\n", + " X_batch = scaled_housing_data_plus_bias[indices]\n", + " y_batch = housing.target.reshape(-1, 1)[indices]\n", + " return X_batch, y_batch\n", + "\n", + "n_epochs = 10\n", + "batch_size = 100\n", + "n_batches = int(np.ceil(m / batch_size))\n", + "\n", + "with tf.Session() as sess:\n", + " sess.run(init)\n", + "\n", + " for epoch in range(n_epochs):\n", + " for batch_index in range(n_batches):\n", + " X_batch, y_batch = fetch_batch(epoch, batch_index, batch_size)\n", + " sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n", + "\n", + " best_theta = theta.eval()\n", + " \n", + "print(\"Best theta:\")\n", + "print(best_theta)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Saving and restoring a model" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "\n", + "n_epochs = 1000\n", + "learning_rate = 0.01\n", + "\n", + "X = tf.constant(scaled_housing_data_plus_bias, dtype=tf.float32, name=\"X\")\n", + "y = tf.constant(housing.target.reshape(-1, 1), dtype=tf.float32, name=\"y\")\n", + "theta = tf.Variable(tf.random_uniform([n + 1, 1], -1.0, 1.0, seed=42), name=\"theta\")\n", + "y_pred = tf.matmul(X, theta, name=\"predictions\")\n", + "error = y_pred - y\n", + "mse = tf.reduce_mean(tf.square(error), name=\"mse\")\n", + "optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)\n", + "training_op = optimizer.minimize(mse)\n", + "\n", + "init = tf.initialize_all_variables()\n", + "saver = tf.train.Saver()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "with tf.Session() as sess:\n", + " sess.run(init)\n", + "\n", + " for epoch in range(n_epochs):\n", + " if epoch % 100 == 0:\n", + " print(\"Epoch\", epoch, \"MSE =\", mse.eval())\n", + " save_path = saver.save(sess, \"/tmp/my_model.ckpt\")\n", + " sess.run(training_op)\n", + " \n", + " best_theta = theta.eval()\n", + " save_path = saver.save(sess, \"my_model_final.ckpt\")\n", + "\n", + "print(\"Best theta:\")\n", + "print(best_theta)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Visualizing the graph\n", + "## inside Jupyter" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "from IPython.display import clear_output, Image, display, HTML\n", + "\n", + "def strip_consts(graph_def, max_const_size=32):\n", + " \"\"\"Strip large constant values from graph_def.\"\"\"\n", + " strip_def = tf.GraphDef()\n", + " for n0 in graph_def.node:\n", + " n = strip_def.node.add() \n", + " n.MergeFrom(n0)\n", + " if n.op == 'Const':\n", + " tensor = n.attr['value'].tensor\n", + " size = len(tensor.tensor_content)\n", + " if size > max_const_size:\n", + " tensor.tensor_content = b\"\"%size\n", + " return strip_def\n", + "\n", + "def show_graph(graph_def, max_const_size=32):\n", + " \"\"\"Visualize TensorFlow graph.\"\"\"\n", + " if hasattr(graph_def, 'as_graph_def'):\n", + " graph_def = graph_def.as_graph_def()\n", + " strip_def = strip_consts(graph_def, max_const_size=max_const_size)\n", + " code = \"\"\"\n", + " \n", + " \n", + "
\n", + " \n", + "
\n", + " \"\"\".format(data=repr(str(strip_def)), id='graph'+str(np.random.rand()))\n", + "\n", + " iframe = \"\"\"\n", + " \n", + " \"\"\".format(code.replace('\"', '"'))\n", + " display(HTML(iframe))" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": { + "collapsed": false, + "scrolled": true + }, + "outputs": [], + "source": [ + "show_graph(tf.get_default_graph())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using TensorBoard" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "\n", + "from datetime import datetime\n", + "\n", + "now = datetime.utcnow().strftime(\"%Y%m%d%H%M%S\")\n", + "root_logdir = \"tf_logs\"\n", + "logdir = \"{}/run-{}/\".format(root_logdir, now)\n", + "\n", + "n_epochs = 1000\n", + "learning_rate = 0.01\n", + "\n", + "X = tf.placeholder(tf.float32, shape=(None, n + 1), name=\"X\")\n", + "y = tf.placeholder(tf.float32, shape=(None, 1), name=\"y\")\n", + "theta = tf.Variable(tf.random_uniform([n + 1, 1], -1.0, 1.0, seed=42), name=\"theta\")\n", + "y_pred = tf.matmul(X, theta, name=\"predictions\")\n", + "error = y_pred - y\n", + "mse = tf.reduce_mean(tf.square(error), name=\"mse\")\n", + "optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)\n", + "training_op = optimizer.minimize(mse)\n", + "\n", + "init = tf.initialize_all_variables()\n", + "\n", + "mse_summary = tf.scalar_summary('MSE', mse)\n", + "summary_writer = tf.train.SummaryWriter(logdir, tf.get_default_graph())" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "n_epochs = 10\n", + "batch_size = 100\n", + "n_batches = int(np.ceil(m / batch_size))\n", + "\n", + "with tf.Session() as sess:\n", + " sess.run(init)\n", + "\n", + " for epoch in range(n_epochs):\n", + " for batch_index in range(n_batches):\n", + " X_batch, y_batch = fetch_batch(epoch, batch_index, batch_size)\n", + " if batch_index % 10 == 0:\n", + " summary_str = mse_summary.eval(feed_dict={X: X_batch, y: y_batch})\n", + " step = epoch * n_batches + batch_index\n", + " summary_writer.add_summary(summary_str, step)\n", + " sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n", + "\n", + " best_theta = theta.eval()\n", + "\n", + "summary_writer.flush()\n", + "summary_writer.close()\n", + "print(\"Best theta:\")\n", + "print(best_theta)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Name scopes" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "\n", + "now = datetime.utcnow().strftime(\"%Y%m%d%H%M%S\")\n", + "root_logdir = \"tf_logs\"\n", + "logdir = \"{}/run-{}/\".format(root_logdir, now)\n", + "\n", + "n_epochs = 1000\n", + "learning_rate = 0.01\n", + "\n", + "X = tf.placeholder(tf.float32, shape=(None, n + 1), name=\"X\")\n", + "y = tf.placeholder(tf.float32, shape=(None, 1), name=\"y\")\n", + "theta = tf.Variable(tf.random_uniform([n + 1, 1], -1.0, 1.0, seed=42), name=\"theta\")\n", + "y_pred = tf.matmul(X, theta, name=\"predictions\")\n", + "with tf.name_scope('loss') as scope:\n", + " error = y_pred - y\n", + " mse = tf.reduce_mean(tf.square(error), name=\"mse\")\n", + "optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)\n", + "training_op = optimizer.minimize(mse)\n", + "\n", + "init = tf.initialize_all_variables()\n", + "\n", + "mse_summary = tf.scalar_summary('MSE', mse)\n", + "summary_writer = tf.train.SummaryWriter(logdir, tf.get_default_graph())" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "n_epochs = 10\n", + "batch_size = 100\n", + "n_batches = int(np.ceil(m / batch_size))\n", + "\n", + "with tf.Session() as sess:\n", + " sess.run(init)\n", + "\n", + " for epoch in range(n_epochs):\n", + " for batch_index in range(n_batches):\n", + " X_batch, y_batch = fetch_batch(epoch, batch_index, batch_size)\n", + " if batch_index % 10 == 0:\n", + " summary_str = mse_summary.eval(feed_dict={X: X_batch, y: y_batch})\n", + " step = epoch * n_batches + batch_index\n", + " summary_writer.add_summary(summary_str, step)\n", + " sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n", + "\n", + " best_theta = theta.eval()\n", + "\n", + "summary_writer.flush()\n", + "summary_writer.close()\n", + "print(\"Best theta:\")\n", + "print(best_theta)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "print(error.op.name)" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "print(mse.op.name)" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "\n", + "a1 = tf.Variable(0, name=\"a\") # name == \"a\"\n", + "a2 = tf.Variable(0, name=\"a\") # name == \"a_1\"\n", + "\n", + "with tf.name_scope(\"param\"): # name == \"param\"\n", + " a3 = tf.Variable(0, name=\"a\") # name == \"param/a\"\n", + "\n", + "with tf.name_scope(\"param\"): # name == \"param_1\"\n", + " a4 = tf.Variable(0, name=\"a\") # name == \"param_1/a\"\n", + "\n", + "for node in (a1, a2, a3, a4):\n", + " print(node.op.name)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Modularity" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "An ugly flat code:" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "\n", + "n_features = 3\n", + "X = tf.placeholder(tf.float32, shape=(None, n_features), name=\"X\")\n", + "\n", + "w1 = tf.Variable(tf.random_normal((n_features, 1)), name=\"weights1\")\n", + "w2 = tf.Variable(tf.random_normal((n_features, 1)), name=\"weights2\")\n", + "b1 = tf.Variable(0.0, name=\"bias1\")\n", + "b2 = tf.Variable(0.0, name=\"bias2\")\n", + "\n", + "linear1 = tf.add(tf.matmul(X, w1), b1, name=\"linear1\")\n", + "linear2 = tf.add(tf.matmul(X, w2), b2, name=\"linear2\")\n", + "\n", + "relu1 = tf.maximum(linear1, 0, name=\"relu1\")\n", + "relu2 = tf.maximum(linear1, 0, name=\"relu2\") # Oops, cut&paste error! Did you spot it?\n", + "\n", + "output = tf.add_n([relu1, relu2], name=\"output\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Much better, using a function to build the ReLUs:" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "\n", + "def relu(X):\n", + " w_shape = int(X.get_shape()[1]), 1\n", + " w = tf.Variable(tf.random_normal(w_shape), name=\"weights\")\n", + " b = tf.Variable(0.0, name=\"bias\")\n", + " linear = tf.add(tf.matmul(X, w), b, name=\"linear\")\n", + " return tf.maximum(linear, 0, name=\"relu\")\n", + "\n", + "n_features = 3\n", + "X = tf.placeholder(tf.float32, shape=(None, n_features), name=\"X\")\n", + "relus = [relu(X) for i in range(5)]\n", + "output = tf.add_n(relus, name=\"output\")\n", + "summary_writer = tf.train.SummaryWriter(\"logs/relu1\", tf.get_default_graph())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Even better using name scopes:" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "\n", + "def relu(X):\n", + " with tf.name_scope(\"relu\"):\n", + " w_shape = int(X.get_shape()[1]), 1\n", + " w = tf.Variable(tf.random_normal(w_shape), name=\"weights\")\n", + " b = tf.Variable(0.0, name=\"bias\")\n", + " linear = tf.add(tf.matmul(X, w), b, name=\"linear\")\n", + " return tf.maximum(linear, 0, name=\"max\")\n", + "\n", + "n_features = 3\n", + "X = tf.placeholder(tf.float32, shape=(None, n_features), name=\"X\")\n", + "relus = [relu(X) for i in range(5)]\n", + "output = tf.add_n(relus, name=\"output\")\n", + "\n", + "summary_writer = tf.train.SummaryWriter(\"logs/relu2\", tf.get_default_graph())" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "summary_writer.close()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Sharing a `threshold` variable the classic way, by defining it outside of the `relu()` function then passing it as a parameter:" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "\n", + "def relu(X, threshold):\n", + " with tf.name_scope(\"relu\"):\n", + " w_shape = int(X.get_shape()[1]), 1\n", + " w = tf.Variable(tf.random_normal(w_shape), name=\"weights\")\n", + " b = tf.Variable(0.0, name=\"bias\")\n", + " linear = tf.add(tf.matmul(X, w), b, name=\"linear\")\n", + " return tf.maximum(linear, threshold, name=\"max\")\n", + "\n", + "threshold = tf.Variable(0.0, name=\"threshold\")\n", + "X = tf.placeholder(tf.float32, shape=(None, n_features), name=\"X\")\n", + "relus = [relu(X, threshold) for i in range(5)]\n", + "output = tf.add_n(relus, name=\"output\")" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "\n", + "def relu(X):\n", + " with tf.name_scope(\"relu\"):\n", + " if not hasattr(relu, \"threshold\"):\n", + " relu.threshold = tf.Variable(0.0, name=\"threshold\")\n", + " w_shape = int(X.get_shape()[1]), 1\n", + " w = tf.Variable(tf.random_normal(w_shape), name=\"weights\")\n", + " b = tf.Variable(0.0, name=\"bias\")\n", + " linear = tf.add(tf.matmul(X, w), b, name=\"linear\")\n", + " return tf.maximum(linear, relu.threshold, name=\"max\")\n", + "\n", + "X = tf.placeholder(tf.float32, shape=(None, n_features), name=\"X\")\n", + "relus = [relu(X) for i in range(5)]\n", + "output = tf.add_n(relus, name=\"output\")" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "\n", + "def relu(X):\n", + " with tf.variable_scope(\"relu\", reuse=True):\n", + " threshold = tf.get_variable(\"threshold\", shape=(), initializer=tf.constant_initializer(0.0))\n", + " w_shape = int(X.get_shape()[1]), 1\n", + " w = tf.Variable(tf.random_normal(w_shape), name=\"weights\")\n", + " b = tf.Variable(0.0, name=\"bias\")\n", + " linear = tf.add(tf.matmul(X, w), b, name=\"linear\")\n", + " return tf.maximum(linear, threshold, name=\"max\")\n", + "\n", + "X = tf.placeholder(tf.float32, shape=(None, n_features), name=\"X\")\n", + "with tf.variable_scope(\"relu\"):\n", + " threshold = tf.get_variable(\"threshold\", shape=(), initializer=tf.constant_initializer(0.0))\n", + "relus = [relu(X) for i in range(5)]\n", + "output = tf.add_n(relus, name=\"output\")\n", + "\n", + "summary_writer = tf.train.SummaryWriter(\"logs/relu6\", tf.get_default_graph())\n", + "summary_writer.close()" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "\n", + "def relu(X):\n", + " with tf.variable_scope(\"relu\"):\n", + " threshold = tf.get_variable(\"threshold\", shape=(), initializer=tf.constant_initializer(0.0))\n", + " w_shape = int(X.get_shape()[1]), 1\n", + " w = tf.Variable(tf.random_normal(w_shape), name=\"weights\")\n", + " b = tf.Variable(0.0, name=\"bias\")\n", + " linear = tf.add(tf.matmul(X, w), b, name=\"linear\")\n", + " return tf.maximum(linear, threshold, name=\"max\")\n", + "\n", + "X = tf.placeholder(tf.float32, shape=(None, n_features), name=\"X\")\n", + "with tf.variable_scope(\"\") as scope:\n", + " first_relu = relu(X) # create the shared variable\n", + " scope.reuse_variables() # then reuse it\n", + " relus = [first_relu] + [relu(X) for i in range(4)]\n", + "output = tf.add_n(relus, name=\"output\")\n", + "\n", + "summary_writer = tf.train.SummaryWriter(\"logs/relu8\", tf.get_default_graph())\n", + "summary_writer.close()" + ] + }, + { + "cell_type": "code", + "execution_count": 46, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "\n", + "with tf.variable_scope(\"param\"):\n", + " x = tf.get_variable(\"x\", shape=(), initializer=tf.constant_initializer(0.))\n", + " #x = tf.Variable(0., name=\"x\")\n", + "with tf.variable_scope(\"param\", reuse=True):\n", + " y = tf.get_variable(\"x\")\n", + "\n", + "with tf.variable_scope(\"\", reuse=True):\n", + " z = tf.get_variable(\"param/x\", shape=(), initializer=tf.constant_initializer(0.))\n", + "\n", + "print(x is y)\n", + "print(x.op.name)\n", + "print(y.op.name)\n", + "print(z.op.name)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Extra material" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Strings" + ] + }, + { + "cell_type": "code", + "execution_count": 47, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "\n", + "text = np.array(\"Do you want some café?\".split())\n", + "text_tensor = tf.constant(text)\n", + "\n", + "with tf.Session() as sess:\n", + " print(text_tensor.eval())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Distributed TensorFlow" + ] + }, + { + "cell_type": "code", + "execution_count": 48, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "server = tf.train.Server.create_local_server()" + ] + }, + { + "cell_type": "code", + "execution_count": 49, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "x = tf.constant(2) + tf.constant(3)\n", + "with tf.Session(server.target) as sess:\n", + " print(sess.run(x))" + ] + }, + { + "cell_type": "code", + "execution_count": 50, + "metadata": { + "collapsed": false, + "scrolled": true + }, + "outputs": [], + "source": [ + "server.target" + ] + }, + { + "cell_type": "code", + "execution_count": 51, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "class Const(object):\n", + " def __init__(self, value):\n", + " self.value = value\n", + " def evaluate(self, **variables):\n", + " return self.value\n", + " def __str__(self):\n", + " return str(self.value)\n", + "\n", + "class Var(object):\n", + " def __init__(self, name):\n", + " self.name = name\n", + " def evaluate(self, **variables):\n", + " return variables[self.name]\n", + " def __str__(self):\n", + " return self.name\n", + "\n", + "class BinaryOperator(object):\n", + " def __init__(self, a, b):\n", + " self.a = a\n", + " self.b = b\n", + "\n", + "class Add(BinaryOperator):\n", + " def evaluate(self, **variables):\n", + " return self.a.evaluate(**variables) + self.b.evaluate(**variables)\n", + " def __str__(self):\n", + " return \"{} + {}\".format(self.a, self.b)\n", + "\n", + "class Mul(BinaryOperator):\n", + " def evaluate(self, **variables):\n", + " return self.a.evaluate(**variables) * self.b.evaluate(**variables)\n", + " def __str__(self):\n", + " return \"({}) * ({})\".format(self.a, self.b)\n", + "\n", + "x = Var(\"x\")\n", + "y = Var(\"y\")\n", + "f = Add(Mul(Mul(x, x), y), Add(y, Const(2))) # f(x,y) = x²y + y + 2\n", + "print(\"f(x,y) =\", f)\n", + "print(\"f(3,4) =\", f.evaluate(x=3, y=4))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Computing gradients\n", + "### Mathematical differentiation" + ] + }, + { + "cell_type": "code", + "execution_count": 52, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "df_dx = Mul(Const(2), Mul(Var(\"x\"), Var(\"y\"))) # df/dx = 2xy\n", + "df_dy = Add(Mul(Var(\"x\"), Var(\"x\")), Const(1)) # df/dy = x² + 1\n", + "print(\"df/dx(3,4) =\", df_dx.evaluate(x=3, y=4))\n", + "print(\"df/dy(3,4) =\", df_dy.evaluate(x=3, y=4))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Numerical differentiation" + ] + }, + { + "cell_type": "code", + "execution_count": 53, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "def derivative(f, x, y, x_eps, y_eps):\n", + " return (f.evaluate(x = x + x_eps, y = y + y_eps) - f.evaluate(x = x, y = y)) / (x_eps + y_eps)\n", + "\n", + "df_dx_34 = derivative(f, x=3, y=4, x_eps=0.0001, y_eps=0)\n", + "df_dy_34 = derivative(f, x=3, y=4, x_eps=0, y_eps=0.0001)\n", + "print(\"df/dx(3,4) =\", df_dx_34)\n", + "print(\"df/dy(3,4) =\", df_dy_34)" + ] + }, + { + "cell_type": "code", + "execution_count": 54, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "def f(x, y):\n", + " return x**2*y + y + 2\n", + "\n", + "def derivative(f, x, y, x_eps, y_eps):\n", + " return (f(x + x_eps, y + y_eps) - f(x, y)) / (x_eps + y_eps)\n", + "\n", + "df_dx = derivative(f, 3, 4, 0.00001, 0)\n", + "df_dy = derivative(f, 3, 4, 0, 0.00001)" + ] + }, + { + "cell_type": "code", + "execution_count": 55, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "print(df_dx)\n", + "print(df_dy)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Symbolic differentiation" + ] + }, + { + "cell_type": "code", + "execution_count": 56, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "Const.derive = lambda self, var: Const(0)\n", + "Var.derive = lambda self, var: Const(1) if self.name==var else Const(0)\n", + "Add.derive = lambda self, var: Add(self.a.derive(var), self.b.derive(var))\n", + "Mul.derive = lambda self, var: Add(Mul(self.a, self.b.derive(var)), Mul(self.a.derive(var), self.b))\n", + "\n", + "x = Var(\"x\")\n", + "y = Var(\"y\")\n", + "f = Add(Mul(Mul(x, x), y), Add(y, Const(2))) # f(x,y) = x²y + y + 2\n", + "\n", + "df_dx = f.derive(\"x\") # 2xy\n", + "df_dy = f.derive(\"y\") # x² + 1\n", + "print(\"df/dx(3,4) =\", df_dx.evaluate(x=3, y=4))\n", + "print(\"df/dy(3,4) =\", df_dy.evaluate(x=3, y=4))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Automatic differentiation (autodiff) – forward mode" + ] + }, + { + "cell_type": "code", + "execution_count": 57, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "class Const(object):\n", + " def __init__(self, value):\n", + " self.value = value\n", + " def evaluate(self, derive, **variables):\n", + " return self.value, 0\n", + " def __str__(self):\n", + " return str(self.value)\n", + "\n", + "class Var(object):\n", + " def __init__(self, name):\n", + " self.name = name\n", + " def evaluate(self, derive, **variables):\n", + " return variables[self.name], (1 if derive == self.name else 0)\n", + " def __str__(self):\n", + " return self.name\n", + "\n", + "class BinaryOperator(object):\n", + " def __init__(self, a, b):\n", + " self.a = a\n", + " self.b = b\n", + "\n", + "class Add(BinaryOperator):\n", + " def evaluate(self, derive, **variables):\n", + " a, da = self.a.evaluate(derive, **variables)\n", + " b, db = self.b.evaluate(derive, **variables)\n", + " return a + b, da + db\n", + " def __str__(self):\n", + " return \"{} + {}\".format(self.a, self.b)\n", + "\n", + "class Mul(BinaryOperator):\n", + " def evaluate(self, derive, **variables):\n", + " a, da = self.a.evaluate(derive, **variables)\n", + " b, db = self.b.evaluate(derive, **variables)\n", + " return a * b, a * db + da * b\n", + " def __str__(self):\n", + " return \"({}) * ({})\".format(self.a, self.b)\n", + "\n", + "x = Var(\"x\")\n", + "y = Var(\"y\")\n", + "f = Add(Mul(Mul(x, x), y), Add(y, Const(2))) # f(x,y) = x²y + y + 2\n", + "f34, df_dx_34 = f.evaluate(x=3, y=4, derive=\"x\")\n", + "f34, df_dy_34 = f.evaluate(x=3, y=4, derive=\"y\")\n", + "print(\"f(3,4) =\", f34)\n", + "print(\"df/dx(3,4) =\", df_dx_34)\n", + "print(\"df/dy(3,4) =\", df_dy_34)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Autodiff – Reverse mode" + ] + }, + { + "cell_type": "code", + "execution_count": 58, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "class Const(object):\n", + " def __init__(self, value):\n", + " self.derivative = 0\n", + " self.value = value\n", + " def evaluate(self, **variables):\n", + " return self.value\n", + " def backpropagate(self, derivative):\n", + " pass\n", + " def __str__(self):\n", + " return str(self.value)\n", + "\n", + "class Var(object):\n", + " def __init__(self, name):\n", + " self.name = name\n", + " def evaluate(self, **variables):\n", + " self.derivative = 0\n", + " self.value = variables[self.name]\n", + " return self.value\n", + " def backpropagate(self, derivative):\n", + " self.derivative += derivative\n", + " def __str__(self):\n", + " return self.name\n", + "\n", + "class BinaryOperator(object):\n", + " def __init__(self, a, b):\n", + " self.a = a\n", + " self.b = b\n", + "\n", + "class Add(BinaryOperator):\n", + " def evaluate(self, **variables):\n", + " self.derivative = 0\n", + " self.value = self.a.evaluate(**variables) + self.b.evaluate(**variables)\n", + " return self.value\n", + " def backpropagate(self, derivative):\n", + " self.derivative += derivative\n", + " self.a.backpropagate(derivative)\n", + " self.b.backpropagate(derivative)\n", + " def __str__(self):\n", + " return \"{} + {}\".format(self.a, self.b)\n", + "\n", + "class Mul(BinaryOperator):\n", + " def evaluate(self, **variables):\n", + " self.derivative = 0\n", + " self.value = self.a.evaluate(**variables) * self.b.evaluate(**variables)\n", + " return self.value\n", + " def backpropagate(self, derivative):\n", + " self.derivative += derivative\n", + " self.a.backpropagate(derivative * self.b.value)\n", + " self.b.backpropagate(derivative * self.a.value)\n", + " def __str__(self):\n", + " return \"({}) * ({})\".format(self.a, self.b)" + ] + }, + { + "cell_type": "code", + "execution_count": 59, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "x = Var(\"x\")\n", + "y = Var(\"y\")\n", + "f = Add(Mul(Mul(x, x), y), Add(y, Const(2))) # f(x,y) = x²y + y + 2\n", + "f34 = f.evaluate(x=3, y=4)\n", + "f.backpropagate(1)\n", + "print(\"f(3,4) =\", f34)\n", + "print(\"df/dx(3,4) =\", x.derivative)\n", + "print(\"df/dy(3,4) =\", y.derivative)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Autodiff – reverse mode (using TensorFlow)" + ] + }, + { + "cell_type": "code", + "execution_count": 60, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "\n", + "x = tf.Variable(3., name=\"x\")\n", + "y = tf.Variable(4., name=\"x\")\n", + "f = x*x*y + y + 2\n", + "\n", + "gradients = tf.gradients(f, [x, y])\n", + "\n", + "init = tf.initialize_all_variables()\n", + "\n", + "with tf.Session() as sess:\n", + " init.run()\n", + " f_val, gradients_val = sess.run([f, gradients])\n", + "\n", + "f_val, gradients_val" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Exercise solutions" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Coming soon**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.5.1" + }, + "nav_menu": { + "height": "603px", + "width": "616px" + }, + "toc": { + "navigate_menu": true, + "number_sections": true, + "sideBar": true, + "threshold": 6, + "toc_cell": false, + "toc_section_display": "block", + "toc_window_display": true + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/10_introduction_to_artificial_neural_networks.ipynb b/10_introduction_to_artificial_neural_networks.ipynb new file mode 100644 index 0000000..2a512a5 --- /dev/null +++ b/10_introduction_to_artificial_neural_networks.ipynb @@ -0,0 +1,660 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Chapter 10 – Introduction to Artificial Neural Networks**" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "_This notebook contains all the sample code and solutions to the exercices in chapter 10._" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Setup" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "First, let's make sure this notebook works well in both python 2 and 3, import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures:" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# To support both python 2 and python 3\n", + "from __future__ import division, print_function, unicode_literals\n", + "\n", + "# Common imports\n", + "import numpy as np\n", + "import numpy.random as rnd\n", + "import os\n", + "\n", + "# to make this notebook's output stable across runs\n", + "rnd.seed(42)\n", + "\n", + "# To plot pretty figures\n", + "%matplotlib inline\n", + "import matplotlib\n", + "import matplotlib.pyplot as plt\n", + "plt.rcParams['axes.labelsize'] = 14\n", + "plt.rcParams['xtick.labelsize'] = 12\n", + "plt.rcParams['ytick.labelsize'] = 12\n", + "\n", + "# Where to save the figures\n", + "PROJECT_ROOT_DIR = \".\"\n", + "CHAPTER_ID = \"ann\"\n", + "\n", + "def save_fig(fig_id, tight_layout=True):\n", + " path = os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID, fig_id + \".png\")\n", + " print(\"Saving figure\", fig_id)\n", + " if tight_layout:\n", + " plt.tight_layout()\n", + " plt.savefig(path, format='png', dpi=300)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Perceptrons" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "from sklearn.datasets import load_iris\n", + "iris = load_iris()\n", + "X = iris.data[:, (2, 3)] # petal length, petal width\n", + "y = (iris.target == 0).astype(np.int)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from sklearn.linear_model import Perceptron\n", + "\n", + "per_clf = Perceptron(random_state=42)\n", + "per_clf.fit(X, y)\n", + "\n", + "y_pred = per_clf.predict([[2, 0.5]])\n", + "y_pred" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "a = -per_clf.coef_[0][0] / per_clf.coef_[0][1]\n", + "b = -per_clf.intercept_ / per_clf.coef_[0][1]\n", + "\n", + "axes = [0, 5, 0, 2]\n", + "\n", + "x0, x1 = np.meshgrid(\n", + " np.linspace(axes[0], axes[1], 500).reshape(-1, 1),\n", + " np.linspace(axes[2], axes[3], 200).reshape(-1, 1),\n", + " )\n", + "X_new = np.c_[x0.ravel(), x1.ravel()]\n", + "y_predict = per_clf.predict(X_new)\n", + "zz = y_predict.reshape(x0.shape)\n", + "\n", + "plt.figure(figsize=(10, 4))\n", + "plt.plot(X[y==0, 0], X[y==0, 1], \"bs\", label=\"Not Iris-Setosa\")\n", + "plt.plot(X[y==1, 0], X[y==1, 1], \"yo\", label=\"Iris-Setosa\")\n", + "\n", + "plt.plot([axes[0], axes[1]], [a * axes[0] + b, a * axes[1] + b], \"k-\", linewidth=3)\n", + "from matplotlib.colors import ListedColormap\n", + "custom_cmap = ListedColormap(['#9898ff', '#fafab0'])\n", + "\n", + "plt.contourf(x0, x1, zz, cmap=custom_cmap, linewidth=5)\n", + "plt.xlabel(\"Petal length\", fontsize=14)\n", + "plt.ylabel(\"Petal width\", fontsize=14)\n", + "plt.legend(loc=\"lower right\", fontsize=14)\n", + "plt.axis(axes)\n", + "\n", + "save_fig(\"perceptron_iris_plot\")\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Activation functions" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "def logit(z):\n", + " return 1 / (1 + np.exp(-z))\n", + "\n", + "def relu(z):\n", + " return np.maximum(0, z)\n", + "\n", + "def derivative(f, z, eps=0.000001):\n", + " return (f(z + eps) - f(z - eps))/(2 * eps)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "z = np.linspace(-5, 5, 200)\n", + "\n", + "plt.figure(figsize=(11,4))\n", + "\n", + "plt.subplot(121)\n", + "plt.plot(z, np.sign(z), \"r-\", linewidth=2, label=\"Step\")\n", + "plt.plot(z, logit(z), \"g--\", linewidth=2, label=\"Logit\")\n", + "plt.plot(z, np.tanh(z), \"b-\", linewidth=2, label=\"Tanh\")\n", + "plt.plot(z, relu(z), \"m-.\", linewidth=2, label=\"ReLU\")\n", + "plt.grid(True)\n", + "plt.legend(loc=\"center right\", fontsize=14)\n", + "plt.title(\"Activation functions\", fontsize=14)\n", + "plt.axis([-5, 5, -1.2, 1.2])\n", + "\n", + "plt.subplot(122)\n", + "plt.plot(z, derivative(np.sign, z), \"r-\", linewidth=2, label=\"Step\")\n", + "plt.plot(0, 0, \"ro\", markersize=5)\n", + "plt.plot(0, 0, \"rx\", markersize=10)\n", + "plt.plot(z, derivative(logit, z), \"g--\", linewidth=2, label=\"Logit\")\n", + "plt.plot(z, derivative(np.tanh, z), \"b-\", linewidth=2, label=\"Tanh\")\n", + "plt.plot(z, derivative(relu, z), \"m-.\", linewidth=2, label=\"ReLU\")\n", + "plt.grid(True)\n", + "#plt.legend(loc=\"center right\", fontsize=14)\n", + "plt.title(\"Derivatives\", fontsize=14)\n", + "plt.axis([-5, 5, -0.2, 1.2])\n", + "\n", + "save_fig(\"activation_functions_plot\")\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "def heaviside(z):\n", + " return (z >= 0).astype(z.dtype)\n", + "\n", + "def sigmoid(z):\n", + " return 1/(1+np.exp(-z))\n", + "\n", + "def mlp_xor(x1, x2, activation=heaviside):\n", + " return activation(-activation(x1 + x2 - 1.5) + activation(x1 + x2 - 0.5) - 0.5)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "x1s = np.linspace(-0.2, 1.2, 100)\n", + "x2s = np.linspace(-0.2, 1.2, 100)\n", + "x1, x2 = np.meshgrid(x1s, x2s)\n", + "\n", + "z1 = mlp_xor(x1, x2, activation=heaviside)\n", + "z2 = mlp_xor(x1, x2, activation=sigmoid)\n", + "\n", + "plt.figure(figsize=(10,4))\n", + "\n", + "plt.subplot(121)\n", + "plt.contourf(x1, x2, z1)\n", + "plt.plot([0, 1], [0, 1], \"gs\", markersize=20)\n", + "plt.plot([0, 1], [1, 0], \"y^\", markersize=20)\n", + "plt.title(\"Activation function: heaviside\", fontsize=14)\n", + "plt.grid(True)\n", + "\n", + "plt.subplot(122)\n", + "plt.contourf(x1, x2, z2)\n", + "plt.plot([0, 1], [0, 1], \"gs\", markersize=20)\n", + "plt.plot([0, 1], [1, 0], \"y^\", markersize=20)\n", + "plt.title(\"Activation function: sigmoid\", fontsize=14)\n", + "plt.grid(True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# FNN for MNIST" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## using tf.learn" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from tensorflow.examples.tutorials.mnist import input_data\n", + "mnist = input_data.read_data_sets(\"/tmp/data/\")\n", + "X_train = mnist.train.images\n", + "X_test = mnist.test.images\n", + "y_train = mnist.train.labels.astype(\"int\")\n", + "y_test = mnist.test.labels.astype(\"int\")" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "import tensorflow as tf\n", + "\n", + "feature_columns = tf.contrib.learn.infer_real_valued_columns_from_input(X_train)\n", + "dnn_clf = tf.contrib.learn.DNNClassifier(hidden_units=[300, 100], n_classes=10,\n", + " feature_columns=feature_columns)\n", + "dnn_clf.fit(x=X_train, y=y_train, batch_size=50, steps=40000)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from sklearn.metrics import accuracy_score\n", + "\n", + "y_pred = dnn_clf.predict(X_test)\n", + "accuracy = accuracy_score(y_test, y_pred)\n", + "accuracy" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from sklearn.metrics import log_loss\n", + "\n", + "y_pred_proba = dnn_clf.predict_proba(X_test)\n", + "log_loss(y_test, y_pred_proba)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "dnn_clf.evaluate(X_test, y_test)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": true + }, + "source": [ + "## Using plain TensorFlow" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "import tensorflow as tf\n", + "\n", + "def neuron_layer(X, n_neurons, name, activation=None):\n", + " with tf.name_scope(name):\n", + " n_inputs = int(X.get_shape()[1])\n", + " stddev = 1 / np.sqrt(n_inputs)\n", + " init = tf.truncated_normal((n_inputs, n_neurons), stddev=stddev)\n", + " W = tf.Variable(init, name=\"weights\")\n", + " b = tf.Variable(tf.zeros([n_neurons]), name=\"biases\")\n", + " Z = tf.matmul(X, W) + b\n", + " if activation==\"relu\":\n", + " return tf.nn.relu(Z)\n", + " else:\n", + " return Z" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "\n", + "n_inputs = 28*28 # MNIST\n", + "n_hidden1 = 300\n", + "n_hidden2 = 100\n", + "n_outputs = 10\n", + "learning_rate = 0.01\n", + "\n", + "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n", + "y = tf.placeholder(tf.int64, shape=(None), name=\"y\")\n", + "\n", + "with tf.name_scope(\"dnn\"):\n", + " hidden1 = neuron_layer(X, n_hidden1, \"hidden1\", activation=\"relu\")\n", + " hidden2 = neuron_layer(hidden1, n_hidden2, \"hidden2\", activation=\"relu\")\n", + " logits = neuron_layer(hidden2, n_outputs, \"output\")\n", + "\n", + "with tf.name_scope(\"loss\"):\n", + " xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits, y)\n", + " loss = tf.reduce_mean(xentropy, name=\"loss\")\n", + "\n", + "with tf.name_scope(\"train\"):\n", + " optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n", + " training_op = optimizer.minimize(loss)\n", + "\n", + "with tf.name_scope(\"eval\"):\n", + " correct = tf.nn.in_top_k(logits, y, 1)\n", + " accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n", + " \n", + "init = tf.initialize_all_variables()\n", + "saver = tf.train.Saver()" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "n_epochs = 20\n", + "batch_size = 50\n", + "\n", + "with tf.Session() as sess:\n", + " init.run()\n", + " for epoch in range(n_epochs):\n", + " for iteration in range(len(mnist.test.labels)//batch_size):\n", + " X_batch, y_batch = mnist.train.next_batch(batch_size)\n", + " sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n", + " acc_train = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n", + " acc_test = accuracy.eval(feed_dict={X: mnist.test.images, y: mnist.test.labels})\n", + " print(epoch, \"Train accuracy:\", acc_train, \"Test accuracy:\", acc_test)\n", + "\n", + " save_path = saver.save(sess, \"my_model_final.ckpt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "with tf.Session() as sess:\n", + " saver.restore(sess, \"my_model_final.ckpt\")\n", + " X_new_scaled = mnist.test.images[:20]\n", + " Z = logits.eval(feed_dict={X: X_new_scaled})\n", + " print(np.argmax(Z, axis=1))\n", + " print(mnist.test.labels[:20])" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "from IPython.display import clear_output, Image, display, HTML\n", + "\n", + "def strip_consts(graph_def, max_const_size=32):\n", + " \"\"\"Strip large constant values from graph_def.\"\"\"\n", + " strip_def = tf.GraphDef()\n", + " for n0 in graph_def.node:\n", + " n = strip_def.node.add() \n", + " n.MergeFrom(n0)\n", + " if n.op == 'Const':\n", + " tensor = n.attr['value'].tensor\n", + " size = len(tensor.tensor_content)\n", + " if size > max_const_size:\n", + " tensor.tensor_content = b\"\"%size\n", + " return strip_def\n", + "\n", + "def show_graph(graph_def, max_const_size=32):\n", + " \"\"\"Visualize TensorFlow graph.\"\"\"\n", + " if hasattr(graph_def, 'as_graph_def'):\n", + " graph_def = graph_def.as_graph_def()\n", + " strip_def = strip_consts(graph_def, max_const_size=max_const_size)\n", + " code = \"\"\"\n", + " \n", + " \n", + "
\n", + " \n", + "
\n", + " \"\"\".format(data=repr(str(strip_def)), id='graph'+str(np.random.rand()))\n", + "\n", + " iframe = \"\"\"\n", + " \n", + " \"\"\".format(code.replace('\"', '"'))\n", + " display(HTML(iframe))" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "show_graph(tf.get_default_graph())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using `fully_connected` instead of `neuron_layer()`" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "\n", + "from tensorflow.contrib.layers import fully_connected\n", + "\n", + "n_inputs = 28*28 # MNIST\n", + "n_hidden1 = 300\n", + "n_hidden2 = 100\n", + "n_outputs = 10\n", + "learning_rate = 0.01\n", + "\n", + "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n", + "y = tf.placeholder(tf.int64, shape=(None), name=\"y\")\n", + "\n", + "with tf.name_scope(\"dnn\"):\n", + " hidden1 = fully_connected(X, n_hidden1, scope=\"hidden1\")\n", + " hidden2 = fully_connected(hidden1, n_hidden2, scope=\"hidden2\")\n", + " logits = fully_connected(hidden2, n_outputs, activation_fn=None, scope=\"outputs\")\n", + "\n", + "with tf.name_scope(\"loss\"):\n", + " xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits, y)\n", + " loss = tf.reduce_mean(xentropy, name=\"loss\")\n", + "\n", + "with tf.name_scope(\"train\"):\n", + " optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n", + " training_op = optimizer.minimize(loss)\n", + "\n", + "with tf.name_scope(\"eval\"):\n", + " correct = tf.nn.in_top_k(logits, y, 1)\n", + " accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n", + " \n", + "init = tf.initialize_all_variables()\n", + "saver = tf.train.Saver()" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "n_epochs = 20\n", + "n_batches = 50\n", + "\n", + "with tf.Session() as sess:\n", + " init.run()\n", + " for epoch in range(n_epochs):\n", + " for iteration in range(len(mnist.test.labels)//batch_size):\n", + " X_batch, y_batch = mnist.train.next_batch(batch_size)\n", + " sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n", + " acc_train = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n", + " acc_test = accuracy.eval(feed_dict={X: mnist.test.images, y: mnist.test.labels})\n", + " print(epoch, \"Train accuracy:\", acc_train, \"Test accuracy:\", acc_test)\n", + "\n", + " save_path = saver.save(sess, \"my_model_final.ckpt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "show_graph(tf.get_default_graph())" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": true + }, + "source": [ + "# Exercise solutions" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Coming soon**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.5.1" + }, + "nav_menu": { + "height": "264px", + "width": "369px" + }, + "toc": { + "navigate_menu": true, + "number_sections": true, + "sideBar": true, + "threshold": 6, + "toc_cell": false, + "toc_section_display": "block", + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/11_deep_learning.ipynb b/11_deep_learning.ipynb new file mode 100644 index 0000000..761a60f --- /dev/null +++ b/11_deep_learning.ipynb @@ -0,0 +1,931 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Chapter 11 – Deep Learning**" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "_This notebook contains all the sample code and solutions to the exercices in chapter 11._" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Setup" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "First, let's make sure this notebook works well in both python 2 and 3, import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures:" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# To support both python 2 and python 3\n", + "from __future__ import division, print_function, unicode_literals\n", + "\n", + "# Common imports\n", + "import numpy as np\n", + "import numpy.random as rnd\n", + "import os\n", + "\n", + "# to make this notebook's output stable across runs\n", + "rnd.seed(42)\n", + "\n", + "# To plot pretty figures\n", + "%matplotlib inline\n", + "import matplotlib\n", + "import matplotlib.pyplot as plt\n", + "plt.rcParams['axes.labelsize'] = 14\n", + "plt.rcParams['xtick.labelsize'] = 12\n", + "plt.rcParams['ytick.labelsize'] = 12\n", + "\n", + "# Where to save the figures\n", + "PROJECT_ROOT_DIR = \".\"\n", + "CHAPTER_ID = \"deep\"\n", + "\n", + "def save_fig(fig_id, tight_layout=True):\n", + " path = os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID, fig_id + \".png\")\n", + " print(\"Saving figure\", fig_id)\n", + " if tight_layout:\n", + " plt.tight_layout()\n", + " plt.savefig(path, format='png', dpi=300)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Activation functions" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "def logit(z):\n", + " return 1 / (1 + np.exp(-z))" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "z = np.linspace(-5, 5, 200)\n", + "\n", + "plt.plot([-5, 5], [0, 0], 'k-')\n", + "plt.plot([-5, 5], [1, 1], 'k--')\n", + "plt.plot([0, 0], [-0.2, 1.2], 'k-')\n", + "plt.plot([-5, 5], [-3/4, 7/4], 'g--')\n", + "plt.plot(z, logit(z), \"b-\", linewidth=2)\n", + "props = dict(facecolor='black', shrink=0.1)\n", + "plt.annotate('Saturating', xytext=(3.5, 0.7), xy=(5, 1), arrowprops=props, fontsize=14, ha=\"center\")\n", + "plt.annotate('Saturating', xytext=(-3.5, 0.3), xy=(-5, 0), arrowprops=props, fontsize=14, ha=\"center\")\n", + "plt.annotate('Linear', xytext=(2, 0.2), xy=(0, 0.5), arrowprops=props, fontsize=14, ha=\"center\")\n", + "plt.grid(True)\n", + "plt.title(\"Sigmoid activation function\", fontsize=14)\n", + "plt.axis([-5, 5, -0.2, 1.2])\n", + "\n", + "save_fig(\"sigmoid_saturation_plot\")\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "def leaky_relu(z, alpha=0.01):\n", + " return np.maximum(alpha*z, z)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "plt.plot(z, leaky_relu(z, 0.05), \"b-\", linewidth=2)\n", + "plt.plot([-5, 5], [0, 0], 'k-')\n", + "plt.plot([0, 0], [-0.5, 4.2], 'k-')\n", + "plt.grid(True)\n", + "props = dict(facecolor='black', shrink=0.1)\n", + "plt.annotate('Leak', xytext=(-3.5, 0.5), xy=(-5, -0.2), arrowprops=props, fontsize=14, ha=\"center\")\n", + "plt.title(\"Leaky ReLU activation function\", fontsize=14)\n", + "plt.axis([-5, 5, -0.5, 4.2])\n", + "\n", + "save_fig(\"leaky_relu_plot\")\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "def elu(z, alpha=1):\n", + " return np.where(z<0, alpha*(np.exp(z)-1), z)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "plt.plot(z, elu(z), \"b-\", linewidth=2)\n", + "plt.plot([-5, 5], [0, 0], 'k-')\n", + "plt.plot([-5, 5], [-1, -1], 'k--')\n", + "plt.plot([0, 0], [-2.2, 3.2], 'k-')\n", + "plt.grid(True)\n", + "props = dict(facecolor='black', shrink=0.1)\n", + "plt.title(r\"ELU activation function ($\\alpha=1$)\", fontsize=14)\n", + "plt.axis([-5, 5, -2.2, 3.2])\n", + "\n", + "save_fig(\"elu_plot\")\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from tensorflow.examples.tutorials.mnist import input_data\n", + "mnist = input_data.read_data_sets(\"/tmp/data/\")" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "def leaky_relu(z, name=None):\n", + " return tf.maximum(0.01 * z, z, name=name)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "import tensorflow as tf" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "from IPython.display import clear_output, Image, display, HTML\n", + "\n", + "def strip_consts(graph_def, max_const_size=32):\n", + " \"\"\"Strip large constant values from graph_def.\"\"\"\n", + " strip_def = tf.GraphDef()\n", + " for n0 in graph_def.node:\n", + " n = strip_def.node.add() \n", + " n.MergeFrom(n0)\n", + " if n.op == 'Const':\n", + " tensor = n.attr['value'].tensor\n", + " size = len(tensor.tensor_content)\n", + " if size > max_const_size:\n", + " tensor.tensor_content = b\"\"%size\n", + " return strip_def\n", + "\n", + "def show_graph(graph_def, max_const_size=32):\n", + " \"\"\"Visualize TensorFlow graph.\"\"\"\n", + " if hasattr(graph_def, 'as_graph_def'):\n", + " graph_def = graph_def.as_graph_def()\n", + " strip_def = strip_consts(graph_def, max_const_size=max_const_size)\n", + " code = \"\"\"\n", + " \n", + " \n", + "
\n", + " \n", + "
\n", + " \"\"\".format(data=repr(str(strip_def)), id='graph'+str(np.random.rand()))\n", + "\n", + " iframe = \"\"\"\n", + " \n", + " \"\"\".format(code.replace('\"', '"'))\n", + " display(HTML(iframe))" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from tensorflow.contrib.layers import fully_connected\n", + "\n", + "tf.reset_default_graph()\n", + "\n", + "n_inputs = 28*28 # MNIST\n", + "n_hidden1 = 300\n", + "n_hidden2 = 100\n", + "n_outputs = 10\n", + "learning_rate = 0.01\n", + "\n", + "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n", + "y = tf.placeholder(tf.int64, shape=(None), name=\"y\")\n", + "\n", + "with tf.name_scope(\"dnn\"):\n", + " hidden1 = fully_connected(X, n_hidden1, activation_fn=leaky_relu, scope=\"hidden1\")\n", + " hidden2 = fully_connected(hidden1, n_hidden2, activation_fn=leaky_relu, scope=\"hidden2\")\n", + " logits = fully_connected(hidden2, n_outputs, activation_fn=None, scope=\"outputs\")\n", + "\n", + "with tf.name_scope(\"loss\"):\n", + " xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits, y)\n", + " loss = tf.reduce_mean(xentropy, name=\"loss\")\n", + "\n", + "with tf.name_scope(\"train\"):\n", + " optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n", + " training_op = optimizer.minimize(loss)\n", + "\n", + "with tf.name_scope(\"eval\"):\n", + " correct = tf.nn.in_top_k(logits, y, 1)\n", + " accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n", + " \n", + "init = tf.initialize_all_variables()\n", + "saver = tf.train.Saver()" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "n_epochs = 20\n", + "batch_size = 100\n", + "\n", + "with tf.Session() as sess:\n", + " init.run()\n", + " for epoch in range(n_epochs):\n", + " for iteration in range(len(mnist.test.labels)//batch_size):\n", + " X_batch, y_batch = mnist.train.next_batch(batch_size)\n", + " sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n", + " acc_train = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n", + " acc_test = accuracy.eval(feed_dict={X: mnist.test.images, y: mnist.test.labels})\n", + " print(epoch, \"Train accuracy:\", acc_train, \"Test accuracy:\", acc_test)\n", + "\n", + " save_path = saver.save(sess, \"my_model_final.ckpt\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Batch Normalization" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from tensorflow.contrib.layers import fully_connected, batch_norm\n", + "from tensorflow.contrib.framework import arg_scope\n", + "\n", + "tf.reset_default_graph()\n", + "\n", + "n_inputs = 28 * 28 # MNIST\n", + "n_hidden1 = 300\n", + "n_hidden2 = 100\n", + "n_outputs = 10\n", + "learning_rate = 0.01\n", + "momentum = 0.25\n", + "\n", + "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n", + "y = tf.placeholder(tf.int64, shape=(None), name=\"y\")\n", + "is_training = tf.placeholder(tf.bool, shape=(), name='is_training')\n", + "\n", + "with tf.name_scope(\"dnn\"):\n", + " he_init = tf.contrib.layers.variance_scaling_initializer()\n", + " batch_norm_params = {\n", + " 'is_training': is_training,\n", + " 'decay': 0.9,\n", + " 'updates_collections': None,\n", + " 'scale': True,\n", + " }\n", + "\n", + " with arg_scope(\n", + " [fully_connected],\n", + " activation_fn=tf.nn.elu,\n", + " weights_initializer=he_init,\n", + " normalizer_fn=batch_norm,\n", + " normalizer_params=batch_norm_params):\n", + " hidden1 = fully_connected(X, n_hidden1, scope=\"hidden1\")\n", + " hidden2 = fully_connected(hidden1, n_hidden2, scope=\"hidden2\")\n", + " logits = fully_connected(hidden2, n_outputs, activation_fn=None, scope=\"outputs\")\n", + "\n", + "with tf.name_scope(\"loss\"):\n", + " xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits, y)\n", + " loss = tf.reduce_mean(xentropy, name=\"loss\")\n", + "\n", + "with tf.name_scope(\"train\"):\n", + " optimizer = tf.train.MomentumOptimizer(learning_rate, momentum)\n", + " training_op = optimizer.minimize(loss)\n", + "\n", + "with tf.name_scope(\"eval\"):\n", + " correct = tf.nn.in_top_k(logits, y, 1)\n", + " accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n", + " \n", + "init = tf.initialize_all_variables()\n", + "saver = tf.train.Saver()" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "n_epochs = 20\n", + "batch_size = 50\n", + "\n", + "with tf.Session() as sess:\n", + " init.run()\n", + " for epoch in range(n_epochs):\n", + " for iteration in range(len(mnist.test.labels)//batch_size):\n", + " X_batch, y_batch = mnist.train.next_batch(batch_size)\n", + " sess.run(training_op, feed_dict={is_training: True, X: X_batch, y: y_batch})\n", + " acc_train = accuracy.eval(feed_dict={is_training: False, X: X_batch, y: y_batch})\n", + " acc_test = accuracy.eval(feed_dict={is_training: False, X: mnist.test.images, y: mnist.test.labels})\n", + " print(epoch, \"Train accuracy:\", acc_train, \"Test accuracy:\", acc_test)\n", + "\n", + " save_path = saver.save(sess, \"my_model_final.ckpt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "\n", + "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n", + "y = tf.placeholder(tf.int64, shape=(None), name=\"y\")\n", + "is_training = tf.placeholder(tf.bool, shape=(), name='is_training')\n", + "\n", + "with tf.name_scope(\"dnn\"):\n", + " he_init = tf.contrib.layers.variance_scaling_initializer()\n", + " batch_norm_params = {\n", + " 'is_training': is_training,\n", + " 'decay': 0.9,\n", + " 'updates_collections': None,\n", + " 'scale': True,\n", + " }\n", + "\n", + " with arg_scope(\n", + " [fully_connected],\n", + " activation_fn=tf.nn.elu,\n", + " weights_initializer=he_init,\n", + " normalizer_fn=batch_norm,\n", + " normalizer_params=batch_norm_params,\n", + " weights_regularizer=tf.contrib.layers.l1_regularizer(0.01)):\n", + " hidden1 = fully_connected(X, n_hidden1, scope=\"hidden1\")\n", + " hidden2 = fully_connected(hidden1, n_hidden2, scope=\"hidden2\")\n", + " logits = fully_connected(hidden2, n_outputs, activation_fn=None, scope=\"outputs\")\n", + "\n", + "with tf.name_scope(\"loss\"):\n", + " xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits, y)\n", + " reg_losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)\n", + " base_loss = tf.reduce_mean(xentropy, name=\"base_loss\")\n", + " loss = tf.add(base_loss, reg_losses, name=\"loss\")\n", + "\n", + "with tf.name_scope(\"train\"):\n", + " optimizer = tf.train.MomentumOptimizer(learning_rate, momentum)\n", + " training_op = optimizer.minimize(loss)\n", + "\n", + "with tf.name_scope(\"eval\"):\n", + " correct = tf.nn.in_top_k(logits, y, 1)\n", + " accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n", + " \n", + "init = tf.initialize_all_variables()\n", + "saver = tf.train.Saver()" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "n_epochs = 20\n", + "batch_size = 50\n", + "\n", + "with tf.Session() as sess:\n", + " init.run()\n", + " for epoch in range(n_epochs):\n", + " for iteration in range(len(mnist.test.labels)//batch_size):\n", + " X_batch, y_batch = mnist.train.next_batch(batch_size)\n", + " sess.run(training_op, feed_dict={is_training: True, X: X_batch, y: y_batch})\n", + " acc_train = accuracy.eval(feed_dict={is_training: False, X: X_batch, y: y_batch})\n", + " acc_test = accuracy.eval(feed_dict={is_training: False, X: mnist.test.images, y: mnist.test.labels})\n", + " print(epoch, \"Train accuracy:\", acc_train, \"Test accuracy:\", acc_test)\n", + "\n", + " save_path = saver.save(sess, \"my_model_final.ckpt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "[v.name for v in tf.all_variables()]" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "with tf.variable_scope(\"\", reuse=True):\n", + " weights1 = tf.get_variable(\"hidden1/weights\")\n", + " weights2 = tf.get_variable(\"hidden2/weights\")\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "\n", + "x = tf.constant([0., 0., 3., 4., 30., 40., 300., 400.], shape=(4, 2))\n", + "c = tf.clip_by_norm(x, clip_norm=10)\n", + "c0 = tf.clip_by_norm(x, clip_norm=350, axes=0)\n", + "c1 = tf.clip_by_norm(x, clip_norm=10, axes=1)\n", + "\n", + "with tf.Session() as sess:\n", + " xv = x.eval()\n", + " cv = c.eval()\n", + " c0v = c0.eval()\n", + " c1v = c1.eval()\n", + "\n", + "print(xv)" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "print(cv)" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "print(np.linalg.norm(cv))" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "print(c0v)" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "print(np.linalg.norm(c0v, axis=0))" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "print(c1v)" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "print(np.linalg.norm(c1v, axis=1))" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "\n", + "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n", + "y = tf.placeholder(tf.int64, shape=(None), name=\"y\")\n", + "is_training = tf.placeholder(tf.bool, shape=(), name='is_training')\n", + "\n", + "def max_norm_regularizer(threshold, axes=1, name=\"max_norm\", collection=\"max_norm\"):\n", + " def max_norm(weights):\n", + " clip_weights = tf.assign(weights, tf.clip_by_norm(weights, clip_norm=threshold, axes=axes), name=name)\n", + " tf.add_to_collection(collection, clip_weights)\n", + " return None # there is no regularization loss term\n", + " return max_norm\n", + "\n", + "with tf.name_scope(\"dnn\"):\n", + " with arg_scope(\n", + " [fully_connected],\n", + " weights_regularizer=max_norm_regularizer(1.5)):\n", + " hidden1 = fully_connected(X, n_hidden1, scope=\"hidden1\")\n", + " hidden2 = fully_connected(hidden1, n_hidden2, scope=\"hidden2\")\n", + " logits = fully_connected(hidden2, n_outputs, activation_fn=None, scope=\"outputs\")\n", + "\n", + "clip_all_weights = tf.get_collection(\"max_norm\")\n", + " \n", + "with tf.name_scope(\"loss\"):\n", + " xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits, y)\n", + " loss = tf.reduce_mean(xentropy, name=\"loss\")\n", + "\n", + "with tf.name_scope(\"train\"):\n", + " optimizer = tf.train.MomentumOptimizer(learning_rate, momentum)\n", + " threshold = 1.0\n", + " grads_and_vars = optimizer.compute_gradients(loss)\n", + " capped_gvs = [(tf.clip_by_value(grad, -threshold, threshold), var)\n", + " for grad, var in grads_and_vars]\n", + " training_op = optimizer.apply_gradients(capped_gvs)\n", + "\n", + "with tf.name_scope(\"eval\"):\n", + " correct = tf.nn.in_top_k(logits, y, 1)\n", + " accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n", + " \n", + "init = tf.initialize_all_variables()\n", + "saver = tf.train.Saver()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "n_epochs = 20\n", + "batch_size = 50\n", + "\n", + "with tf.Session() as sess:\n", + " init.run()\n", + " for epoch in range(n_epochs):\n", + " for iteration in range(len(mnist.test.labels)//batch_size):\n", + " X_batch, y_batch = mnist.train.next_batch(batch_size)\n", + " sess.run(training_op, feed_dict={is_training: True, X: X_batch, y: y_batch})\n", + " acc_train = accuracy.eval(feed_dict={is_training: False, X: X_batch, y: y_batch})\n", + " acc_test = accuracy.eval(feed_dict={is_training: False, X: mnist.test.images, y: mnist.test.labels})\n", + " print(epoch, \"Train accuracy:\", acc_train, \"Test accuracy:\", acc_test)\n", + "\n", + " save_path = saver.save(sess, \"my_model_final.ckpt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "show_graph(tf.get_default_graph())" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from tensorflow.contrib.layers import dropout\n", + "\n", + "tf.reset_default_graph()\n", + "\n", + "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n", + "y = tf.placeholder(tf.int64, shape=(None), name=\"y\")\n", + "is_training = tf.placeholder(tf.bool, shape=(), name='is_training')\n", + "\n", + "initial_learning_rate = 0.1\n", + "decay_steps = 10000\n", + "decay_rate = 1/10\n", + "global_step = tf.Variable(0, trainable=False)\n", + "learning_rate = tf.train.exponential_decay(initial_learning_rate, global_step,\n", + " decay_steps, decay_rate)\n", + "\n", + "keep_prob = 0.5\n", + "\n", + "with tf.name_scope(\"dnn\"):\n", + " he_init = tf.contrib.layers.variance_scaling_initializer()\n", + " with arg_scope(\n", + " [fully_connected],\n", + " activation_fn=tf.nn.elu,\n", + " weights_initializer=he_init):\n", + " X_drop = dropout(X, keep_prob, is_training=is_training)\n", + " hidden1 = fully_connected(X_drop, n_hidden1, scope=\"hidden1\")\n", + " hidden1_drop = dropout(hidden1, keep_prob, is_training=is_training)\n", + " hidden2 = fully_connected(hidden1_drop, n_hidden2, scope=\"hidden2\")\n", + " hidden2_drop = dropout(hidden2, keep_prob, is_training=is_training)\n", + " logits = fully_connected(hidden2_drop, n_outputs, activation_fn=None, scope=\"outputs\")\n", + "\n", + "with tf.name_scope(\"loss\"):\n", + " xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits, y)\n", + " loss = tf.reduce_mean(xentropy, name=\"loss\")\n", + "\n", + "with tf.name_scope(\"train\"):\n", + " optimizer = tf.train.MomentumOptimizer(learning_rate, momentum)\n", + " training_op = optimizer.minimize(loss, global_step=global_step) \n", + "\n", + "with tf.name_scope(\"eval\"):\n", + " correct = tf.nn.in_top_k(logits, y, 1)\n", + " accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n", + " \n", + "init = tf.initialize_all_variables()\n", + "saver = tf.train.Saver()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "n_epochs = 20\n", + "batch_size = 50\n", + "\n", + "with tf.Session() as sess:\n", + " init.run()\n", + " for epoch in range(n_epochs):\n", + " for iteration in range(len(mnist.test.labels)//batch_size):\n", + " X_batch, y_batch = mnist.train.next_batch(batch_size)\n", + " sess.run(training_op, feed_dict={is_training: True, X: X_batch, y: y_batch})\n", + " acc_train = accuracy.eval(feed_dict={is_training: False, X: X_batch, y: y_batch})\n", + " acc_test = accuracy.eval(feed_dict={is_training: False, X: mnist.test.images, y: mnist.test.labels})\n", + " print(epoch, \"Train accuracy:\", acc_train, \"Test accuracy:\", acc_test)\n", + "\n", + " save_path = saver.save(sess, \"my_model_final.ckpt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "train_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,\n", + " scope=\"hidden[2]|outputs\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "training_op2 = optimizer.minimize(loss, var_list=train_vars)" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "for i in tf.all_variables():\n", + " print(i.name)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "for i in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES):\n", + " print(i.name)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "for i in train_vars:\n", + " print(i.name)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "X_train = mnist.train.images\n", + "y_train = mnist.train.labels.astype(\"int\")\n", + "X_val = mnist.test.images[8000:]\n", + "y_val = mnist.test.labels[8000:].astype(\"int\")\n", + "\n", + "feature_columns = tf.contrib.learn.infer_real_valued_columns_from_input(X_train)\n", + "dnn_clf = tf.contrib.learn.DNNClassifier(\n", + " feature_columns = feature_columns,\n", + " hidden_units=[300, 100],\n", + " n_classes=10,\n", + " model_dir=\"/tmp/my_model\",\n", + " config=tf.contrib.learn.RunConfig(save_checkpoints_secs=60)\n", + " )\n", + "\n", + "validation_monitor = tf.contrib.learn.monitors.ValidationMonitor(\n", + " X_val,\n", + " y_val,\n", + " every_n_steps=50,\n", + " early_stopping_metric=\"loss\",\n", + " early_stopping_metric_minimize=True,\n", + " early_stopping_rounds=2000\n", + " )\n", + "\n", + "dnn_clf.fit(x=X_train,\n", + " y=y_train,\n", + " steps=40000,\n", + " monitors=[validation_monitor]\n", + " )\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": true + }, + "source": [ + "# Exercise solutions" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Coming soon**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.5.1" + }, + "nav_menu": { + "height": "360px", + "width": "416px" + }, + "toc": { + "navigate_menu": true, + "number_sections": true, + "sideBar": true, + "threshold": 6, + "toc_cell": false, + "toc_section_display": "block", + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/12_distributed_tensorflow.ipynb b/12_distributed_tensorflow.ipynb new file mode 100644 index 0000000..c438d48 --- /dev/null +++ b/12_distributed_tensorflow.ipynb @@ -0,0 +1,494 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Chapter 12 – Distributed TensorFlow**" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "_This notebook contains all the sample code and solutions to the exercices in chapter 12._" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Setup" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "First, let's make sure this notebook works well in both python 2 and 3, import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures:" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# To support both python 2 and python 3\n", + "from __future__ import division, print_function, unicode_literals\n", + "\n", + "# Common imports\n", + "import numpy as np\n", + "import numpy.random as rnd\n", + "import os\n", + "\n", + "# to make this notebook's output stable across runs\n", + "rnd.seed(42)\n", + "\n", + "# To plot pretty figures\n", + "%matplotlib inline\n", + "import matplotlib\n", + "import matplotlib.pyplot as plt\n", + "plt.rcParams['axes.labelsize'] = 14\n", + "plt.rcParams['xtick.labelsize'] = 12\n", + "plt.rcParams['ytick.labelsize'] = 12\n", + "\n", + "# Where to save the figures\n", + "PROJECT_ROOT_DIR = \".\"\n", + "CHAPTER_ID = \"distributed\"\n", + "\n", + "def save_fig(fig_id, tight_layout=True):\n", + " path = os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID, fig_id + \".png\")\n", + " print(\"Saving figure\", fig_id)\n", + " if tight_layout:\n", + " plt.tight_layout()\n", + " plt.savefig(path, format='png', dpi=300)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Local server" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "import tensorflow as tf" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "c = tf.constant(\"Hello distributed TensorFlow!\")\n", + "server = tf.train.Server.create_local_server()" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "with tf.Session(server.target) as sess:\n", + " print(sess.run(c))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Cluster" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "cluster_spec = tf.train.ClusterSpec({\n", + " \"ps\": [\n", + " \"127.0.0.1:2221\", # /job:ps/task:0\n", + " \"127.0.0.1:2222\", # /job:ps/task:1\n", + " ],\n", + " \"worker\": [\n", + " \"127.0.0.1:2223\", # /job:worker/task:0\n", + " \"127.0.0.1:2224\", # /job:worker/task:1\n", + " \"127.0.0.1:2225\", # /job:worker/task:2\n", + " ]})" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "task_ps0 = tf.train.Server(cluster_spec, job_name=\"ps\", task_index=0)\n", + "task_ps1 = tf.train.Server(cluster_spec, job_name=\"ps\", task_index=1)\n", + "task_worker0 = tf.train.Server(cluster_spec, job_name=\"worker\", task_index=0)\n", + "task_worker1 = tf.train.Server(cluster_spec, job_name=\"worker\", task_index=1)\n", + "task_worker2 = tf.train.Server(cluster_spec, job_name=\"worker\", task_index=2)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Pinning operations across devices and servers" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "\n", + "with tf.device(\"/job:ps\"):\n", + " a = tf.Variable(1.0, name=\"a\")\n", + "\n", + "with tf.device(\"/job:worker\"):\n", + " b = a + 2\n", + "\n", + "with tf.device(\"/job:worker/task:1\"):\n", + " c = a + b" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "with tf.Session(\"grpc://127.0.0.1:2221\") as sess:\n", + " sess.run(a.initializer)\n", + " print(c.eval())" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "\n", + "with tf.device(tf.train.replica_device_setter(\n", + " ps_tasks=2,\n", + " ps_device=\"/job:ps\",\n", + " worker_device=\"/job:worker\")):\n", + " v1 = tf.Variable(1.0, name=\"v1\") # pinned to /job:ps/task:0 (defaults to /cpu:0)\n", + " v2 = tf.Variable(2.0, name=\"v2\") # pinned to /job:ps/task:1 (defaults to /cpu:0)\n", + " v3 = tf.Variable(3.0, name=\"v3\") # pinned to /job:ps/task:0 (defaults to /cpu:0)\n", + " s = v1 + v2 # pinned to /job:worker (defaults to task:0/cpu:0)\n", + " with tf.device(\"/task:1\"):\n", + " p1 = 2 * s # pinned to /job:worker/task:1 (defaults to /cpu:0)\n", + " with tf.device(\"/cpu:0\"):\n", + " p2 = 3 * s # pinned to /job:worker/task:1/cpu:0\n", + "\n", + "config = tf.ConfigProto()\n", + "config.log_device_placement = True\n", + "\n", + "with tf.Session(\"grpc://127.0.0.1:2221\", config=config) as sess:\n", + " v1.initializer.run()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Readers" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "\n", + "test_csv = open(\"my_test.csv\", \"w\")\n", + "test_csv.write(\"x1, x2 , target\\n\")\n", + "test_csv.write(\"1., , 0\\n\")\n", + "test_csv.write(\"4., 5. , 1\\n\")\n", + "test_csv.write(\"7., 8. , 0\\n\")\n", + "test_csv.close()\n", + "\n", + "filename_queue = tf.FIFOQueue(capacity=10, dtypes=[tf.string], shapes=[()])\n", + "filename = tf.placeholder(tf.string)\n", + "enqueue_filename = filename_queue.enqueue([filename])\n", + "close_filename_queue = filename_queue.close()\n", + "\n", + "reader = tf.TextLineReader(skip_header_lines=1)\n", + "key, value = reader.read(filename_queue)\n", + "\n", + "x1, x2, target = tf.decode_csv(value, record_defaults=[[-1.], [-1.], [-1]])\n", + "features = tf.pack([x1, x2])\n", + "\n", + "instance_queue = tf.RandomShuffleQueue(\n", + " capacity=10, min_after_dequeue=2,\n", + " dtypes=[tf.float32, tf.int32], shapes=[[2],[]],\n", + " name=\"instance_q\", shared_name=\"shared_instance_q\")\n", + "enqueue_instance = instance_queue.enqueue([features, target])\n", + "close_instance_queue = instance_queue.close()\n", + "\n", + "minibatch_instances, minibatch_targets = instance_queue.dequeue_up_to(2)\n", + "\n", + "with tf.Session() as sess:\n", + " sess.run(enqueue_filename, feed_dict={filename: \"my_test.csv\"})\n", + " sess.run(close_filename_queue)\n", + " try:\n", + " while True:\n", + " sess.run(enqueue_instance)\n", + " except tf.errors.OutOfRangeError as ex:\n", + " print(\"No more files to read\")\n", + " sess.run(close_instance_queue)\n", + " try:\n", + " while True:\n", + " print(sess.run([minibatch_instances, minibatch_targets]))\n", + " except tf.errors.OutOfRangeError as ex:\n", + " print(\"No more training instances\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "#coord = tf.train.Coordinator()\n", + "#threads = tf.train.start_queue_runners(coord=coord)\n", + "#filename_queue = tf.train.string_input_producer([\"test.csv\"])\n", + "#coord.request_stop()\n", + "#coord.join(threads)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Queue runners and coordinators" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "\n", + "filename_queue = tf.FIFOQueue(capacity=10, dtypes=[tf.string], shapes=[()])\n", + "filename = tf.placeholder(tf.string)\n", + "enqueue_filename = filename_queue.enqueue([filename])\n", + "close_filename_queue = filename_queue.close()\n", + "\n", + "reader = tf.TextLineReader(skip_header_lines=1)\n", + "key, value = reader.read(filename_queue)\n", + "\n", + "x1, x2, target = tf.decode_csv(value, record_defaults=[[-1.], [-1.], [-1]])\n", + "features = tf.pack([x1, x2])\n", + "\n", + "instance_queue = tf.RandomShuffleQueue(\n", + " capacity=10, min_after_dequeue=2,\n", + " dtypes=[tf.float32, tf.int32], shapes=[[2],[]],\n", + " name=\"instance_q\", shared_name=\"shared_instance_q\")\n", + "enqueue_instance = instance_queue.enqueue([features, target])\n", + "close_instance_queue = instance_queue.close()\n", + "\n", + "minibatch_instances, minibatch_targets = instance_queue.dequeue_up_to(2)\n", + "\n", + "n_threads = 5\n", + "queue_runner = tf.train.QueueRunner(instance_queue, [enqueue_instance] * n_threads)\n", + "coord = tf.train.Coordinator()\n", + "\n", + "with tf.Session() as sess:\n", + " sess.run(enqueue_filename, feed_dict={filename: \"my_test.csv\"})\n", + " sess.run(close_filename_queue)\n", + " enqueue_threads = queue_runner.create_threads(sess, coord=coord, start=True)\n", + " try:\n", + " while True:\n", + " print(sess.run([minibatch_instances, minibatch_targets]))\n", + " except tf.errors.OutOfRangeError as ex:\n", + " print(\"No more training instances\")" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "\n", + "def read_and_push_instance(filename_queue, instance_queue):\n", + " reader = tf.TextLineReader(skip_header_lines=1)\n", + " key, value = reader.read(filename_queue)\n", + " x1, x2, target = tf.decode_csv(value, record_defaults=[[-1.], [-1.], [-1]])\n", + " features = tf.pack([x1, x2])\n", + " enqueue_instance = instance_queue.enqueue([features, target])\n", + " return enqueue_instance\n", + "\n", + "filename_queue = tf.FIFOQueue(capacity=10, dtypes=[tf.string], shapes=[()])\n", + "filename = tf.placeholder(tf.string)\n", + "enqueue_filename = filename_queue.enqueue([filename])\n", + "close_filename_queue = filename_queue.close()\n", + "\n", + "instance_queue = tf.RandomShuffleQueue(\n", + " capacity=10, min_after_dequeue=2,\n", + " dtypes=[tf.float32, tf.int32], shapes=[[2],[]],\n", + " name=\"instance_q\", shared_name=\"shared_instance_q\")\n", + "\n", + "minibatch_instances, minibatch_targets = instance_queue.dequeue_up_to(2)\n", + "\n", + "read_and_enqueue_ops = [read_and_push_instance(filename_queue, instance_queue) for i in range(5)]\n", + "queue_runner = tf.train.QueueRunner(instance_queue, read_and_enqueue_ops)\n", + "\n", + "with tf.Session() as sess:\n", + " sess.run(enqueue_filename, feed_dict={filename: \"my_test.csv\"})\n", + " sess.run(close_filename_queue)\n", + " coord = tf.train.Coordinator()\n", + " enqueue_threads = queue_runner.create_threads(sess, coord=coord, start=True)\n", + " try:\n", + " while True:\n", + " print(sess.run([minibatch_instances, minibatch_targets]))\n", + " except tf.errors.OutOfRangeError as ex:\n", + " print(\"No more training instances\")\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Setting a timeout" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "\n", + "q = tf.FIFOQueue(capacity=10, dtypes=[tf.float32], shapes=[()])\n", + "v = tf.placeholder(tf.float32)\n", + "enqueue = q.enqueue([v])\n", + "dequeue = q.dequeue()\n", + "output = dequeue + 1\n", + "\n", + "config = tf.ConfigProto()\n", + "config.operation_timeout_in_ms = 1000\n", + "\n", + "with tf.Session(config=config) as sess:\n", + " sess.run(enqueue, feed_dict={v: 1.0})\n", + " sess.run(enqueue, feed_dict={v: 2.0})\n", + " sess.run(enqueue, feed_dict={v: 3.0})\n", + " print(sess.run(output))\n", + " print(sess.run(output, feed_dict={dequeue: 5}))\n", + " print(sess.run(output))\n", + " print(sess.run(output))\n", + " try:\n", + " print(sess.run(output))\n", + " except tf.errors.DeadlineExceededError as ex:\n", + " print(\"Timed out while dequeuing\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": true + }, + "source": [ + "# Exercise solutions" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Coming soon**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.5.1" + }, + "nav_menu": {}, + "toc": { + "navigate_menu": true, + "number_sections": true, + "sideBar": true, + "threshold": 6, + "toc_cell": false, + "toc_section_display": "block", + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/13_convolutional_neural_networks.ipynb b/13_convolutional_neural_networks.ipynb new file mode 100644 index 0000000..b4ea324 --- /dev/null +++ b/13_convolutional_neural_networks.ipynb @@ -0,0 +1,613 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Chapter 13 – Convolutional Neural Networks**" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "_This notebook contains all the sample code and solutions to the exercices in chapter 13._" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Setup" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "First, let's make sure this notebook works well in both python 2 and 3, import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures:" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# To support both python 2 and python 3\n", + "from __future__ import division, print_function, unicode_literals\n", + "\n", + "# Common imports\n", + "import numpy as np\n", + "import numpy.random as rnd\n", + "import os\n", + "\n", + "# to make this notebook's output stable across runs\n", + "rnd.seed(42)\n", + "\n", + "# To plot pretty figures\n", + "%matplotlib inline\n", + "import matplotlib\n", + "import matplotlib.pyplot as plt\n", + "plt.rcParams['axes.labelsize'] = 14\n", + "plt.rcParams['xtick.labelsize'] = 12\n", + "plt.rcParams['ytick.labelsize'] = 12\n", + "\n", + "# Where to save the figures\n", + "PROJECT_ROOT_DIR = \".\"\n", + "CHAPTER_ID = \"cnn\"\n", + "\n", + "def save_fig(fig_id, tight_layout=True):\n", + " path = os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID, fig_id + \".png\")\n", + " print(\"Saving figure\", fig_id)\n", + " if tight_layout:\n", + " plt.tight_layout()\n", + " plt.savefig(path, format='png', dpi=300)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "A couple utility functions to plot grayscale and RGB images:" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "def plot_image(image):\n", + " plt.imshow(image, cmap=\"gray\", interpolation=\"nearest\")\n", + " plt.axis(\"off\")\n", + "\n", + "def plot_color_image(image):\n", + " plt.imshow(image.astype(np.uint8),interpolation=\"nearest\")\n", + " plt.axis(\"off\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "And of course we will need TensorFlow:" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "import tensorflow as tf" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Convolutional layer" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "from sklearn.datasets import load_sample_images\n", + "dataset = load_sample_images()\n", + "china, flower = dataset.images\n", + "image = china[150:220, 130:250]\n", + "height, width, channels = image.shape\n", + "image_grayscale = image.mean(axis=2).astype(np.float32)\n", + "images = image_grayscale.reshape(1, height, width, 1)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "fmap = np.zeros(shape=(7, 7, 1, 2), dtype=np.float32)\n", + "fmap[:, 3, 0, 0] = 1\n", + "fmap[3, :, 0, 1] = 1\n", + "fmap[:, :, 0, 0]\n", + "plot_image(fmap[:, :, 0, 0])\n", + "plt.show()\n", + "plot_image(fmap[:, :, 0, 1])\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "\n", + "X = tf.placeholder(tf.float32, shape=(None, height, width, 1))\n", + "feature_maps = tf.constant(fmap)\n", + "convolution = tf.nn.conv2d(X, feature_maps, strides=[1,1,1,1], padding=\"SAME\", use_cudnn_on_gpu=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "with tf.Session() as sess:\n", + " output = convolution.eval(feed_dict={X: images})" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "plot_image(images[0, :, :, 0])\n", + "save_fig(\"china_original\", tight_layout=False)\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "plot_image(output[0, :, :, 0])\n", + "save_fig(\"china_vertical\", tight_layout=False)\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "plot_image(output[0, :, :, 1])\n", + "save_fig(\"china_horizontal\", tight_layout=False)\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Simple example" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from sklearn.datasets import load_sample_images\n", + "dataset = np.array(load_sample_images().images, dtype=np.float32)\n", + "batch_size, height, width, channels = dataset.shape\n", + "\n", + "filters = np.zeros(shape=(7, 7, channels, 2), dtype=np.float32)\n", + "filters[:, 3, :, 0] = 1 # vertical line\n", + "filters[3, :, :, 1] = 1 # horizontal line\n", + "\n", + "X = tf.placeholder(tf.float32, shape=(None, height, width, channels))\n", + "convolution = tf.nn.conv2d(X, filters, strides=[1,2,2,1], padding=\"SAME\")\n", + "\n", + "with tf.Session() as sess:\n", + " output = sess.run(convolution, feed_dict={X: dataset})\n", + "\n", + "for image_index in (0, 1):\n", + " for feature_map_index in (0, 1):\n", + " plot_image(output[image_index, :, :, feature_map_index])\n", + " plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## VALID vs SAME padding" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "\n", + "filter_primes = np.array([2., 3., 5., 7., 11., 13.], dtype=np.float32)\n", + "x = tf.constant(np.arange(1, 13+1, dtype=np.float32).reshape([1, 1, 13, 1]))\n", + "filters = tf.constant(filter_primes.reshape(1, 6, 1, 1))\n", + "\n", + "valid_conv = tf.nn.conv2d(x, filters, strides=[1, 1, 5, 1], padding='VALID')\n", + "same_conv = tf.nn.conv2d(x, filters, strides=[1, 1, 5, 1], padding='SAME')\n", + "\n", + "with tf.Session() as sess:\n", + " print(\"VALID:\\n\", valid_conv.eval())\n", + " print(\"SAME:\\n\", same_conv.eval())" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "print(\"VALID:\")\n", + "print(np.array([1,2,3,4,5,6]).T.dot(filter_primes))\n", + "print(np.array([6,7,8,9,10,11]).T.dot(filter_primes))\n", + "print(\"SAME:\")\n", + "print(np.array([0,1,2,3,4,5]).T.dot(filter_primes))\n", + "print(np.array([5,6,7,8,9,10]).T.dot(filter_primes))\n", + "print(np.array([10,11,12,13,0,0]).T.dot(filter_primes))\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Pooling layer" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from sklearn.datasets import load_sample_images\n", + "dataset = np.array(load_sample_images().images, dtype=np.float32)\n", + "batch_size, height, width, channels = dataset.shape\n", + "\n", + "filters = np.zeros(shape=(7, 7, channels, 2), dtype=np.float32)\n", + "filters[:, 3, :, 0] = 1 # vertical line\n", + "filters[3, :, :, 1] = 1 # horizontal line\n", + "\n", + "X = tf.placeholder(tf.float32, shape=(None, height, width, channels))\n", + "max_pool = tf.nn.max_pool(X, ksize=[1, 2, 2, 1], strides=[1,2,2,1], padding=\"VALID\")\n", + "\n", + "with tf.Session() as sess:\n", + " output = sess.run(max_pool, feed_dict={X: dataset})\n", + "\n", + "plot_color_image(dataset[0])\n", + "save_fig(\"china_original\")\n", + "plt.show()\n", + " \n", + "plot_color_image(output[0])\n", + "save_fig(\"china_max_pool\")\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# MNIST" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "from sklearn.datasets import fetch_mldata\n", + "\n", + "mnist = fetch_mldata('MNIST original')\n", + "X_train, X_test = mnist[\"data\"][:60000].astype(np.float64), mnist[\"data\"][60000:].astype(np.float64)\n", + "y_train, y_test = mnist[\"target\"][:60000].astype(np.int64), mnist[\"target\"][60000:].astype(np.int64)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "height, width = 28, 28\n", + "images = X_test[5000].reshape(1, height, width, 1)\n", + "plot_image(images[0, :, :, 0])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Inception v3" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "import os\n", + "import sys\n", + "import tarfile\n", + "import urllib.request\n", + "\n", + "TF_MODELS_URL = \"http://download.tensorflow.org/models\"\n", + "INCEPTION_V3_URL = TF_MODELS_URL + \"/inception_v3_2016_08_28.tar.gz\"\n", + "INCEPTION_PATH = os.path.join(\"datasets\", \"inception\")\n", + "INCEPTION_V3_CHECKPOINT_PATH = os.path.join(INCEPTION_PATH, \"inception_v3.ckpt\")\n", + "\n", + "def download_progress(count, block_size, total_size):\n", + " percent = count * block_size * 100 // total_size\n", + " sys.stdout.write(\"\\rDownloading: {}%\".format(percent))\n", + " sys.stdout.flush()\n", + "\n", + "def fetch_pretrained_inception_v3(url=INCEPTION_V3_URL, path=INCEPTION_PATH):\n", + " if os.path.exists(INCEPTION_V3_CHECKPOINT_PATH):\n", + " return\n", + " os.makedirs(path, exist_ok=True)\n", + " tgz_path = os.path.join(path, \"inception_v3.tgz\")\n", + " urllib.request.urlretrieve(url, tgz_path, reporthook=download_progress)\n", + " inception_tgz = tarfile.open(tgz_path)\n", + " inception_tgz.extractall(path=path)\n", + " inception_tgz.close()\n", + " os.remove(tgz_path)" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "fetch_pretrained_inception_v3()" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "import re\n", + "\n", + "CLASS_NAME_REGEX = re.compile(r\"^n\\d+\\s+(.*)\\s*$\", re.M | re.U)\n", + "\n", + "def load_class_names():\n", + " with open(os.path.join(\"datasets\",\"inception\",\"imagenet_class_names.txt\"), \"rb\") as f:\n", + " content = f.read().decode(\"utf-8\")\n", + " return CLASS_NAME_REGEX.findall(content)" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "class_names = load_class_names()" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "width = 299\n", + "height = 299\n", + "channels = 3" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "import matplotlib.image as mpimg\n", + "test_image = mpimg.imread(os.path.join(\"images\",\"cnn\",\"test_image.png\"))[:, :, :channels]\n", + "plt.imshow(test_image)\n", + "plt.axis(\"off\")\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "import tensorflow as tf\n", + "from nets.inception_v3 import inception_v3, inception_v3_arg_scope\n", + "import tensorflow.contrib.slim as slim\n", + "\n", + "tf.reset_default_graph()\n", + "\n", + "X = tf.placeholder(tf.float32, shape=[None, height, width, channels], name=\"X\")\n", + "with slim.arg_scope(inception_v3_arg_scope()):\n", + " logits, end_points = inception_v3(X, num_classes=1001, is_training=False)\n", + "predictions = end_points[\"Predictions\"]\n", + "saver = tf.train.Saver()" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "X_test = test_image.reshape(-1, height, width, channels)\n", + "\n", + "with tf.Session() as sess:\n", + " saver.restore(sess, INCEPTION_V3_CHECKPOINT_PATH)\n", + " predictions_val = predictions.eval(feed_dict={X: X_test})" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "class_names[np.argmax(predictions_val[0])]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "np.argmax(predictions_val, axis=1)" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "top_5 = np.argpartition(predictions_val[0], -5)[-5:]\n", + "top_5 = top_5[np.argsort(predictions_val[0][top_5])]\n", + "for i in top_5:\n", + " print(\"{0}: {1:.2f}%\".format(class_names[i], 100*predictions_val[0][i]))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": true + }, + "source": [ + "# Exercise solutions" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Coming soon**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.5.1" + }, + "nav_menu": {}, + "toc": { + "navigate_menu": true, + "number_sections": true, + "sideBar": true, + "threshold": 6, + "toc_cell": false, + "toc_section_display": "block", + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/14_recurrent_neural_networks.ipynb b/14_recurrent_neural_networks.ipynb new file mode 100644 index 0000000..7f873cd --- /dev/null +++ b/14_recurrent_neural_networks.ipynb @@ -0,0 +1,1326 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Chapter 14 – Recurrent Neural Networks**" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "_This notebook contains all the sample code and solutions to the exercices in chapter 14._" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Setup" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "First, let's make sure this notebook works well in both python 2 and 3, import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures:" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# To support both python 2 and python 3\n", + "from __future__ import division, print_function, unicode_literals\n", + "\n", + "# Common imports\n", + "import numpy as np\n", + "import numpy.random as rnd\n", + "import os\n", + "\n", + "# to make this notebook's output stable across runs\n", + "rnd.seed(42)\n", + "\n", + "# To plot pretty figures\n", + "%matplotlib inline\n", + "import matplotlib\n", + "import matplotlib.pyplot as plt\n", + "plt.rcParams['axes.labelsize'] = 14\n", + "plt.rcParams['xtick.labelsize'] = 12\n", + "plt.rcParams['ytick.labelsize'] = 12\n", + "\n", + "# Where to save the figures\n", + "PROJECT_ROOT_DIR = \".\"\n", + "CHAPTER_ID = \"rnn\"\n", + "\n", + "def save_fig(fig_id, tight_layout=True):\n", + " path = os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID, fig_id + \".png\")\n", + " print(\"Saving figure\", fig_id)\n", + " if tight_layout:\n", + " plt.tight_layout()\n", + " plt.savefig(path, format='png', dpi=300)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then of course we will need TensorFlow:" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "import tensorflow as tf" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Basic RNNs" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Manual RNN" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "\n", + "n_inputs = 3\n", + "n_neurons = 5\n", + "\n", + "X0 = tf.placeholder(tf.float32, [None, n_inputs])\n", + "X1 = tf.placeholder(tf.float32, [None, n_inputs])\n", + "\n", + "Wx = tf.Variable(tf.random_normal(shape=[n_inputs, n_neurons], dtype=tf.float32))\n", + "Wy = tf.Variable(tf.random_normal(shape=[n_neurons, n_neurons], dtype=tf.float32))\n", + "b = tf.Variable(tf.zeros([1, n_neurons], dtype=tf.float32))\n", + "\n", + "Y0 = tf.tanh(tf.matmul(X0, Wx) + b)\n", + "Y1 = tf.tanh(tf.matmul(Y0, Wy) + tf.matmul(X1, Wx) + b)\n", + "\n", + "init = tf.initialize_all_variables()" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "X0_batch = np.array([[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 0, 1]]) # t = 0\n", + "X1_batch = np.array([[9, 8, 7], [0, 0, 0], [6, 5, 4], [3, 2, 1]]) # t = 1\n", + "\n", + "with tf.Session() as sess:\n", + " init.run()\n", + " Y0_val, Y1_val = sess.run([Y0, Y1], feed_dict={X0: X0_batch, X1: X1_batch})" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "print(Y0_val)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "print(Y1_val)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using `rnn()`" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "\n", + "n_inputs = 3\n", + "n_neurons = 5\n", + "\n", + "X0 = tf.placeholder(tf.float32, [None, n_inputs])\n", + "X1 = tf.placeholder(tf.float32, [None, n_inputs])\n", + "\n", + "basic_cell = tf.nn.rnn_cell.BasicRNNCell(num_units=n_neurons)\n", + "output_seqs, states = tf.nn.rnn(basic_cell, [X0, X1], dtype=tf.float32)\n", + "Y0, Y1 = output_seqs\n", + "\n", + "init = tf.initialize_all_variables()" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "X0_batch = np.array([[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 0, 1]])\n", + "X1_batch = np.array([[9, 8, 7], [0, 0, 0], [6, 5, 4], [3, 2, 1]])\n", + "\n", + "with tf.Session() as sess:\n", + " init.run()\n", + " Y0_val, Y1_val = sess.run([Y0, Y1], feed_dict={X0: X0_batch, X1: X1_batch})" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "Y0_val" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "Y1_val" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "from IPython.display import clear_output, Image, display, HTML\n", + "\n", + "def strip_consts(graph_def, max_const_size=32):\n", + " \"\"\"Strip large constant values from graph_def.\"\"\"\n", + " strip_def = tf.GraphDef()\n", + " for n0 in graph_def.node:\n", + " n = strip_def.node.add() \n", + " n.MergeFrom(n0)\n", + " if n.op == 'Const':\n", + " tensor = n.attr['value'].tensor\n", + " size = len(tensor.tensor_content)\n", + " if size > max_const_size:\n", + " tensor.tensor_content = \"b\"%size\n", + " return strip_def\n", + "\n", + "def show_graph(graph_def, max_const_size=32):\n", + " \"\"\"Visualize TensorFlow graph.\"\"\"\n", + " if hasattr(graph_def, 'as_graph_def'):\n", + " graph_def = graph_def.as_graph_def()\n", + " strip_def = strip_consts(graph_def, max_const_size=max_const_size)\n", + " code = \"\"\"\n", + " \n", + " \n", + "
\n", + " \n", + "
\n", + " \"\"\".format(data=repr(str(strip_def)), id='graph'+str(np.random.rand()))\n", + "\n", + " iframe = \"\"\"\n", + " \n", + " \"\"\".format(code.replace('\"', '"'))\n", + " display(HTML(iframe))" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "show_graph(tf.get_default_graph())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Packing sequences" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "\n", + "n_steps = 2\n", + "n_inputs = 3\n", + "n_neurons = 5\n", + "\n", + "X = tf.placeholder(tf.float32, [None, n_steps, n_inputs])\n", + "X_seqs = tf.unpack(tf.transpose(X, perm=[1, 0, 2]))\n", + "\n", + "basic_cell = tf.nn.rnn_cell.BasicRNNCell(num_units=n_neurons)\n", + "output_seqs, states = tf.nn.rnn(basic_cell, X_seqs, dtype=tf.float32)\n", + "outputs = tf.transpose(tf.pack(output_seqs), perm=[1, 0, 2])\n", + "\n", + "init = tf.initialize_all_variables()" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "X_batch = np.array([\n", + " # t = 0 t = 1 \n", + " [[0, 1, 2], [9, 8, 7]], # instance 1\n", + " [[3, 4, 5], [0, 0, 0]], # instance 2\n", + " [[6, 7, 8], [6, 5, 4]], # instance 3\n", + " [[9, 0, 1], [3, 2, 1]], # instance 4\n", + " ])\n", + "\n", + "with tf.Session() as sess:\n", + " init.run()\n", + " outputs_val = outputs.eval(feed_dict={X: X_batch})" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "print(np.transpose(outputs_val, axes=[1, 0, 2])[1])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using `dynamic_rnn()`" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "\n", + "n_steps = 2\n", + "n_inputs = 3\n", + "n_neurons = 5\n", + "\n", + "X = tf.placeholder(tf.float32, [None, n_steps, n_inputs])\n", + "\n", + "basic_cell = tf.nn.rnn_cell.BasicRNNCell(num_units=n_neurons)\n", + "outputs, states = tf.nn.dynamic_rnn(basic_cell, X, dtype=tf.float32)\n", + "\n", + "init = tf.initialize_all_variables()" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "X_batch = np.array([\n", + " [[0, 1, 2], [9, 8, 7]], # instance 1\n", + " [[3, 4, 5], [0, 0, 0]], # instance 2\n", + " [[6, 7, 8], [6, 5, 4]], # instance 3\n", + " [[9, 0, 1], [3, 2, 1]], # instance 4\n", + " ])\n", + "\n", + "with tf.Session() as sess:\n", + " init.run()\n", + " print(\"outputs =\", outputs.eval(feed_dict={X: X_batch}))" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "show_graph(tf.get_default_graph())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setting the sequence lengths" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "\n", + "n_steps = 2\n", + "n_inputs = 3\n", + "n_neurons = 5\n", + "\n", + "X = tf.placeholder(tf.float32, [None, n_steps, n_inputs])\n", + "seq_length = tf.placeholder(tf.int32, [None])\n", + "\n", + "basic_cell = tf.nn.rnn_cell.BasicRNNCell(num_units=n_neurons)\n", + "outputs, states = tf.nn.dynamic_rnn(basic_cell, X, sequence_length=seq_length, dtype=tf.float32)\n", + "\n", + "init = tf.initialize_all_variables()" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "X_batch = np.array([\n", + " # step 0 step 1\n", + " [[0, 1, 2], [9, 8, 7]], # instance 1\n", + " [[3, 4, 5], [0, 0, 0]], # instance 2 (padded with zero vectors)\n", + " [[6, 7, 8], [6, 5, 4]], # instance 3\n", + " [[9, 0, 1], [3, 2, 1]], # instance 4\n", + " ])\n", + "seq_length_batch = np.array([2, 1, 2, 2])\n", + "\n", + "with tf.Session() as sess:\n", + " init.run()\n", + " outputs_val, states_val = sess.run(\n", + " [outputs, states], feed_dict={X: X_batch, seq_length: seq_length_batch})" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "print(outputs_val)" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "print(states_val)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Training a sequence classifier" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "\n", + "from tensorflow.contrib.layers import fully_connected\n", + "\n", + "n_steps = 28\n", + "n_inputs = 28\n", + "n_neurons = 150\n", + "n_outputs = 10\n", + "\n", + "learning_rate = 0.001\n", + "\n", + "X = tf.placeholder(tf.float32, [None, n_steps, n_inputs])\n", + "y = tf.placeholder(tf.int32, [None])\n", + "\n", + "with tf.variable_scope(\"\", initializer=tf.contrib.layers.variance_scaling_initializer()):\n", + " basic_cell = tf.nn.rnn_cell.BasicRNNCell(num_units=n_neurons, activation=tf.nn.relu)\n", + " outputs, states = tf.nn.dynamic_rnn(basic_cell, X, dtype=tf.float32)\n", + "\n", + "logits = fully_connected(states, n_outputs, activation_fn=None)\n", + "xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits, y)\n", + "loss = tf.reduce_mean(xentropy)\n", + "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n", + "training_op = optimizer.minimize(loss)\n", + "correct = tf.nn.in_top_k(logits, y, 1)\n", + "accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n", + "\n", + "init = tf.initialize_all_variables()" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from tensorflow.examples.tutorials.mnist import input_data\n", + "mnist = input_data.read_data_sets(\"/tmp/data/\")\n", + "X_test = mnist.test.images.reshape((-1, n_steps, n_inputs))\n", + "y_test = mnist.test.labels" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "n_epochs = 100\n", + "batch_size = 150\n", + "\n", + "with tf.Session() as sess:\n", + " init.run()\n", + " for epoch in range(n_epochs):\n", + " for iteration in range(len(mnist.test.labels)//batch_size):\n", + " X_batch, y_batch = mnist.train.next_batch(batch_size)\n", + " X_batch = X_batch.reshape((-1, n_steps, n_inputs))\n", + " sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n", + " acc_train = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n", + " acc_test = accuracy.eval(feed_dict={X: X_test, y: y_test})\n", + " print(epoch, \"Train accuracy:\", acc_train, \"Test accuracy:\", acc_test)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Multi-layer RNN" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "\n", + "from tensorflow.contrib.layers import fully_connected\n", + "\n", + "n_steps = 28\n", + "n_inputs = 28\n", + "n_neurons1 = 150\n", + "n_neurons2 = 100\n", + "n_outputs = 10\n", + "\n", + "learning_rate = 0.001\n", + "\n", + "X = tf.placeholder(tf.float32, [None, n_steps, n_inputs])\n", + "y = tf.placeholder(tf.int32, [None])\n", + "\n", + "hidden1 = tf.nn.rnn_cell.BasicRNNCell(num_units=n_neurons1, activation=tf.nn.relu)\n", + "hidden2 = tf.nn.rnn_cell.BasicRNNCell(num_units=n_neurons2, activation=tf.nn.relu)\n", + "multi_layer_cell = tf.nn.rnn_cell.MultiRNNCell([hidden1, hidden2])\n", + "outputs, states = tf.nn.dynamic_rnn(multi_layer_cell, X, dtype=tf.float32)\n", + "\n", + "logits = fully_connected(states, n_outputs, activation_fn=None)\n", + "xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits, y)\n", + "loss = tf.reduce_mean(xentropy)\n", + "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n", + "training_op = optimizer.minimize(loss)\n", + "correct = tf.nn.in_top_k(logits, y, 1)\n", + "accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n", + "\n", + "init = tf.initialize_all_variables()" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "n_epochs = 100\n", + "batch_size = 150\n", + "\n", + "with tf.Session() as sess:\n", + " init.run()\n", + " for epoch in range(n_epochs):\n", + " for iteration in range(len(mnist.test.labels)//batch_size):\n", + " X_batch, y_batch = mnist.train.next_batch(batch_size)\n", + " X_batch = X_batch.reshape((-1, n_steps, n_inputs))\n", + " sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n", + " acc_train = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n", + " acc_test = accuracy.eval(feed_dict={X: X_test, y: y_test})\n", + " print(epoch, \"Train accuracy:\", acc_train, \"Test accuracy:\", acc_test)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Time series" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "t_min, t_max = 0, 30\n", + "resolution = 0.1\n", + "\n", + "def time_series(t):\n", + " return t * np.sin(t) / 3 + 2 * np.sin(t*5)\n", + "\n", + "def next_batch(batch_size, n_steps):\n", + " t0 = np.random.rand(batch_size, 1) * (t_max - t_min - n_steps * resolution)\n", + " Ts = t0 + np.arange(0., n_steps + 1) * resolution\n", + " ys = time_series(Ts)\n", + " return ys[:, :-1].reshape(-1, n_steps, 1), ys[:, 1:].reshape(-1, n_steps, 1)" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "t = np.linspace(t_min, t_max, (t_max - t_min) // resolution)\n", + "\n", + "n_steps = 20\n", + "t_instance = np.linspace(12.2, 12.2 + resolution * (n_steps + 1), n_steps + 1)\n", + "\n", + "plt.figure(figsize=(11,4))\n", + "plt.subplot(121)\n", + "plt.title(\"A time series (generated)\", fontsize=14)\n", + "plt.plot(t, time_series(t), label=r\"$t . \\sin(t) / 3 + 2 . \\sin(5t)$\")\n", + "plt.plot(t_instance[:-1], time_series(t_instance[:-1]), \"b-\", linewidth=3, label=\"A training instance\")\n", + "plt.legend(loc=\"lower left\", fontsize=14)\n", + "plt.axis([0, 30, -17, 13])\n", + "plt.xlabel(\"Time\")\n", + "plt.ylabel(\"Value\")\n", + "\n", + "plt.subplot(122)\n", + "plt.title(\"A training instance\", fontsize=14)\n", + "plt.plot(t_instance[:-1], time_series(t_instance[:-1]), \"bo\", markersize=10, label=\"instance\")\n", + "plt.plot(t_instance[1:], time_series(t_instance[1:]), \"w*\", markersize=10, label=\"target\")\n", + "plt.legend(loc=\"upper left\")\n", + "plt.xlabel(\"Time\")\n", + "\n", + "\n", + "save_fig(\"time_series_plot\")\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "X_batch, y_batch = next_batch(1, n_steps)" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "np.c_[X_batch[0], y_batch[0]]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using an `OuputProjectionWrapper`" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "\n", + "from tensorflow.contrib.layers import fully_connected\n", + "\n", + "n_steps = 20\n", + "n_inputs = 1\n", + "n_neurons = 100\n", + "n_outputs = 1\n", + "\n", + "X = tf.placeholder(tf.float32, [None, n_steps, n_inputs])\n", + "y = tf.placeholder(tf.float32, [None, n_steps, n_outputs])\n", + "\n", + "cell = tf.nn.rnn_cell.OutputProjectionWrapper(\n", + " tf.nn.rnn_cell.BasicRNNCell(num_units=n_neurons, activation=tf.nn.relu),\n", + " output_size=n_outputs)\n", + "outputs, states = tf.nn.dynamic_rnn(cell, X, dtype=tf.float32)\n", + "\n", + "n_outputs = 1\n", + "learning_rate = 0.001\n", + "\n", + "loss = tf.reduce_sum(tf.square(outputs - y))\n", + "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n", + "training_op = optimizer.minimize(loss)\n", + "\n", + "init = tf.initialize_all_variables()" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "n_iterations = 1000\n", + "batch_size = 50\n", + "\n", + "with tf.Session() as sess:\n", + " init.run()\n", + " for iteration in range(n_iterations):\n", + " X_batch, y_batch = next_batch(batch_size, n_steps)\n", + " sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n", + " if iteration % 100 == 0:\n", + " mse = loss.eval(feed_dict={X: X_batch, y: y_batch})\n", + " print(iteration, \"\\tMSE:\", mse)\n", + " \n", + " X_new = time_series(np.array(t_instance[:-1].reshape(-1, n_steps, n_inputs)))\n", + " y_pred = sess.run(outputs, feed_dict={X: X_new})\n", + " print(y_pred)" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "plt.title(\"Testing the model\", fontsize=14)\n", + "plt.plot(t_instance[:-1], time_series(t_instance[:-1]), \"bo\", markersize=10, label=\"instance\")\n", + "plt.plot(t_instance[1:], time_series(t_instance[1:]), \"w*\", markersize=10, label=\"target\")\n", + "plt.plot(t_instance[1:], y_pred[0,:,0], \"r.\", markersize=10, label=\"prediction\")\n", + "plt.legend(loc=\"upper left\")\n", + "plt.xlabel(\"Time\")\n", + "\n", + "save_fig(\"time_series_pred_plot\")\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Without using an `OutputProjectionWrapper`" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "\n", + "from tensorflow.contrib.layers import fully_connected\n", + "\n", + "n_steps = 20\n", + "n_inputs = 1\n", + "n_neurons = 100\n", + "\n", + "X = tf.placeholder(tf.float32, [None, n_steps, n_inputs])\n", + "y = tf.placeholder(tf.float32, [None, n_steps, n_outputs])\n", + "\n", + "basic_cell = tf.nn.rnn_cell.BasicRNNCell(num_units=n_neurons, activation=tf.nn.relu)\n", + "rnn_outputs, states = tf.nn.dynamic_rnn(basic_cell, X, dtype=tf.float32)\n", + "\n", + "n_outputs = 1\n", + "learning_rate = 0.001\n", + "\n", + "stacked_rnn_outputs = tf.reshape(rnn_outputs, [-1, n_neurons])\n", + "stacked_outputs = fully_connected(stacked_rnn_outputs, n_outputs, activation_fn=None)\n", + "outputs = tf.reshape(stacked_outputs, [-1, n_steps, n_outputs])\n", + "\n", + "loss = tf.reduce_sum(tf.square(outputs - y))\n", + "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n", + "training_op = optimizer.minimize(loss)\n", + "\n", + "init = tf.initialize_all_variables()" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "n_iterations = 1000\n", + "batch_size = 50\n", + "\n", + "with tf.Session() as sess:\n", + " init.run()\n", + " for iteration in range(n_iterations):\n", + " X_batch, y_batch = next_batch(batch_size, n_steps)\n", + " sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n", + " if iteration % 100 == 0:\n", + " mse = loss.eval(feed_dict={X: X_batch, y: y_batch})\n", + " print(iteration, \"\\tMSE:\", mse)\n", + " \n", + " X_new = time_series(np.array(t_instance[:-1].reshape(-1, n_steps, n_inputs)))\n", + " y_pred = sess.run(outputs, feed_dict={X: X_new})\n", + " print(y_pred)" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "plt.title(\"Testing the model\", fontsize=14)\n", + "plt.plot(t_instance[:-1], time_series(t_instance[:-1]), \"bo\", markersize=10, label=\"instance\")\n", + "plt.plot(t_instance[1:], time_series(t_instance[1:]), \"w*\", markersize=10, label=\"target\")\n", + "plt.plot(t_instance[1:], y_pred[0,:,0], \"r.\", markersize=10, label=\"prediction\")\n", + "plt.legend(loc=\"upper left\")\n", + "plt.xlabel(\"Time\")\n", + "\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Generating a creative new sequence" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "n_iterations = 2000\n", + "batch_size = 50\n", + "with tf.Session() as sess:\n", + " init.run()\n", + " for iteration in range(n_iterations):\n", + " X_batch, y_batch = next_batch(batch_size, n_steps)\n", + " sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n", + " if iteration % 100 == 0:\n", + " mse = loss.eval(feed_dict={X: X_batch, y: y_batch})\n", + " print(iteration, \"\\tMSE:\", mse)\n", + "\n", + " sequence1 = [0. for i in range(n_steps)]\n", + " for iteration in range(len(t) - n_steps):\n", + " X_batch = np.array(sequence1[-n_steps:]).reshape(1, n_steps, 1)\n", + " y_pred = sess.run(outputs, feed_dict={X: X_batch})\n", + " sequence1.append(y_pred[0, -1, 0])\n", + "\n", + " sequence2 = [time_series(i * resolution + t_min + (t_max-t_min/3)) for i in range(n_steps)]\n", + " for iteration in range(len(t) - n_steps):\n", + " X_batch = np.array(sequence2[-n_steps:]).reshape(1, n_steps, 1)\n", + " y_pred = sess.run(outputs, feed_dict={X: X_batch})\n", + " sequence2.append(y_pred[0, -1, 0])\n", + "\n", + "plt.figure(figsize=(11,4))\n", + "plt.subplot(121)\n", + "plt.plot(t, sequence1, \"b-\")\n", + "plt.plot(t[:n_steps], sequence1[:n_steps], \"b-\", linewidth=3)\n", + "plt.xlabel(\"Time\")\n", + "plt.ylabel(\"Value\")\n", + "\n", + "plt.subplot(122)\n", + "plt.plot(t, sequence2, \"b-\")\n", + "plt.plot(t[:n_steps], sequence2[:n_steps], \"b-\", linewidth=3)\n", + "plt.xlabel(\"Time\")\n", + "#save_fig(\"creative_sequence_plot\")\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Deep RNN" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## MultiRNNCell" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "\n", + "n_inputs = 2\n", + "n_neurons = 100\n", + "n_layers = 3\n", + "n_steps = 5\n", + "keep_prob = 0.5\n", + "\n", + "X = tf.placeholder(tf.float32, [None, n_steps, n_inputs])\n", + "basic_cell = tf.nn.rnn_cell.BasicRNNCell(num_units=n_neurons)\n", + "multi_layer_cell = tf.nn.rnn_cell.MultiRNNCell([basic_cell] * n_layers)\n", + "outputs, states = tf.nn.dynamic_rnn(multi_layer_cell, X, dtype=tf.float32)\n", + "\n", + "init = tf.initialize_all_variables()" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "X_batch = rnd.rand(2, n_steps, n_inputs)" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "with tf.Session() as sess:\n", + " init.run()\n", + " outputs_val, states_val = sess.run([outputs, states], feed_dict={X: X_batch})" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "outputs_val.shape" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Dropout" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "from tensorflow.contrib.layers import fully_connected\n", + "\n", + "n_inputs = 1\n", + "n_neurons = 100\n", + "n_layers = 3\n", + "n_steps = 20\n", + "n_outputs = 1\n", + "\n", + "keep_prob = 0.5\n", + "learning_rate = 0.001\n", + "\n", + "is_training = True\n", + "\n", + "def deep_rnn_with_dropout(X, y, is_training):\n", + " cell = tf.nn.rnn_cell.BasicRNNCell(num_units=n_neurons)\n", + " if is_training:\n", + " cell = tf.nn.rnn_cell.DropoutWrapper(cell, input_keep_prob=keep_prob)\n", + " multi_layer_cell = tf.nn.rnn_cell.MultiRNNCell([cell] * n_layers)\n", + " rnn_outputs, states = tf.nn.dynamic_rnn(multi_layer_cell, X, dtype=tf.float32)\n", + "\n", + " stacked_rnn_outputs = tf.reshape(rnn_outputs, [-1, n_neurons])\n", + " stacked_outputs = fully_connected(stacked_rnn_outputs, n_outputs, activation_fn=None)\n", + " outputs = tf.reshape(stacked_outputs, [-1, n_steps, n_outputs])\n", + "\n", + " loss = tf.reduce_sum(tf.square(outputs - y))\n", + " optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n", + " training_op = optimizer.minimize(loss)\n", + "\n", + " return outputs, loss, training_op\n", + "\n", + "X = tf.placeholder(tf.float32, [None, n_steps, n_inputs])\n", + "y = tf.placeholder(tf.float32, [None, n_steps, n_outputs])\n", + "outputs, loss, training_op = deep_rnn_with_dropout(X, y, is_training)\n", + "init = tf.initialize_all_variables()\n", + "saver = tf.train.Saver()" + ] + }, + { + "cell_type": "code", + "execution_count": 46, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "n_iterations = 2000\n", + "batch_size = 50\n", + "\n", + "with tf.Session() as sess:\n", + " if is_training:\n", + " init.run()\n", + " for iteration in range(n_iterations):\n", + " X_batch, y_batch = next_batch(batch_size, n_steps)\n", + " sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n", + " if iteration % 100 == 0:\n", + " mse = loss.eval(feed_dict={X: X_batch, y: y_batch})\n", + " print(iteration, \"\\tMSE:\", mse)\n", + " save_path = saver.save(sess, \"/tmp/my_model.ckpt\")\n", + " else:\n", + " saver.restore(sess, \"/tmp/my_model.ckpt\")\n", + " X_new = time_series(np.array(t_instance[:-1].reshape(-1, n_steps, n_inputs)))\n", + " y_pred = sess.run(outputs, feed_dict={X: X_new})\n", + " \n", + " plt.title(\"Testing the model\", fontsize=14)\n", + " plt.plot(t_instance[:-1], time_series(t_instance[:-1]), \"bo\", markersize=10, label=\"instance\")\n", + " plt.plot(t_instance[1:], time_series(t_instance[1:]), \"w*\", markersize=10, label=\"target\")\n", + " plt.plot(t_instance[1:], y_pred[0,:,0], \"r.\", markersize=10, label=\"prediction\")\n", + " plt.legend(loc=\"upper left\")\n", + " plt.xlabel(\"Time\")\n", + " plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# LSTM" + ] + }, + { + "cell_type": "code", + "execution_count": 55, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "\n", + "from tensorflow.contrib.layers import fully_connected\n", + "\n", + "n_steps = 28\n", + "n_inputs = 28\n", + "n_neurons = 150\n", + "n_outputs = 10\n", + "\n", + "learning_rate = 0.001\n", + "\n", + "X = tf.placeholder(tf.float32, [None, n_steps, n_inputs])\n", + "y = tf.placeholder(tf.int32, [None])\n", + "\n", + "lstm_cell = tf.nn.rnn_cell.BasicLSTMCell(num_units=n_neurons, state_is_tuple=True)\n", + "multi_cell = tf.nn.rnn_cell.MultiRNNCell([lstm_cell]*3, state_is_tuple=True)\n", + "outputs, states = tf.nn.dynamic_rnn(multi_cell, X, dtype=tf.float32)\n", + "top_layer_h_state = states[-1][1]\n", + "logits = fully_connected(top_layer_h_state, n_outputs, activation_fn=None, scope=\"softmax\")\n", + "xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits, y)\n", + "loss = tf.reduce_mean(xentropy, name=\"loss\")\n", + "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n", + "training_op = optimizer.minimize(loss)\n", + "correct = tf.nn.in_top_k(logits, y, 1)\n", + "accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n", + " \n", + "init = tf.initialize_all_variables()" + ] + }, + { + "cell_type": "code", + "execution_count": 56, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "states" + ] + }, + { + "cell_type": "code", + "execution_count": 57, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "top_layer_h_state" + ] + }, + { + "cell_type": "code", + "execution_count": 58, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "n_epochs = 10\n", + "batch_size = 150\n", + "\n", + "with tf.Session() as sess:\n", + " init.run()\n", + " for epoch in range(n_epochs):\n", + " for iteration in range(len(mnist.test.labels)//batch_size):\n", + " X_batch, y_batch = mnist.train.next_batch(batch_size)\n", + " X_batch = X_batch.reshape((batch_size, n_steps, n_inputs))\n", + " sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n", + " acc_train = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n", + " acc_test = accuracy.eval(feed_dict={X: X_test, y: y_test})\n", + " print(\"Epoch\", epoch, \"Train accuracy =\", acc_train, \"Test accuracy =\", acc_test)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Distributing layers across devices" + ] + }, + { + "cell_type": "code", + "execution_count": 59, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "import tensorflow as tf\n", + "\n", + "class DeviceCellWrapper(tf.nn.rnn_cell.RNNCell):\n", + " def __init__(self, device, cell):\n", + " self._cell = cell\n", + " self._device = device\n", + "\n", + " @property\n", + " def state_size(self):\n", + " return self._cell.state_size\n", + "\n", + " @property\n", + " def output_size(self):\n", + " return self._cell.output_size\n", + "\n", + " def __call__(self, inputs, state, scope=None):\n", + " with tf.device(self._device):\n", + " return self._cell(inputs, state, scope)" + ] + }, + { + "cell_type": "code", + "execution_count": 60, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "\n", + "n_inputs = 5\n", + "n_neurons = 100\n", + "devices = [\"/cpu:0\"]*5\n", + "n_steps = 20\n", + "X = tf.placeholder(tf.float32, shape=[None, n_steps, n_inputs])\n", + "lstm_cells = [DeviceCellWrapper(device, tf.nn.rnn_cell.BasicRNNCell(num_units=n_neurons))\n", + " for device in devices]\n", + "multi_layer_cell = tf.nn.rnn_cell.MultiRNNCell(lstm_cells)\n", + "outputs, states = tf.nn.dynamic_rnn(multi_layer_cell, X, dtype=tf.float32)\n", + "init = tf.initialize_all_variables()" + ] + }, + { + "cell_type": "code", + "execution_count": 61, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "with tf.Session() as sess:\n", + " init.run()\n", + " print(sess.run(outputs, feed_dict={X: rnd.rand(2, n_steps, n_inputs)}))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": true + }, + "source": [ + "# Exercise solutions" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Coming soon**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.5.1" + }, + "nav_menu": {}, + "toc": { + "navigate_menu": true, + "number_sections": true, + "sideBar": true, + "threshold": 6, + "toc_cell": false, + "toc_section_display": "block", + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/classification.ipynb b/classification.ipynb index 4c7cc98..ea7f79b 100644 --- a/classification.ipynb +++ b/classification.ipynb @@ -4,7 +4,23 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "**Classification**" + "**Chapter 3 – Classification**\n", + "\n", + "_This notebook contains all the sample code and solutions to the exercices in chapter 3._" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Setup" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "First, let's make sure this notebook works well in both python 2 and 3, import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures:" ] }, { @@ -15,14 +31,18 @@ }, "outputs": [], "source": [ + "# To support both python 2 and python 3\n", "from __future__ import division, print_function, unicode_literals\n", "\n", + "# Common imports\n", "import numpy as np\n", "import numpy.random as rnd\n", - "rnd.seed(42) # to make this notebook's output stable across runs\n", - "\n", "import os\n", "\n", + "# to make this notebook's output stable across runs\n", + "rnd.seed(42)\n", + "\n", + "# To plot pretty figures\n", "%matplotlib inline\n", "import matplotlib\n", "import matplotlib.pyplot as plt\n", @@ -30,6 +50,7 @@ "plt.rcParams['xtick.labelsize'] = 12\n", "plt.rcParams['ytick.labelsize'] = 12\n", "\n", + "# Where to save the figures\n", "PROJECT_ROOT_DIR = \".\"\n", "CHAPTER_ID = \"classification\"\n", "\n", @@ -122,7 +143,7 @@ "some_digit_index = 36000\n", "some_digit = X[some_digit_index]\n", "plot_digit(some_digit)\n", - "save_fig(\"some_digit\")\n", + "save_fig(\"some_digit_plot\")\n", "plt.show()" ] }, @@ -153,7 +174,7 @@ "plt.figure(figsize=(9,9))\n", "example_images = np.r_[X[:12000:600], X[13000:30600:600], X[30600:60000:590]]\n", "plot_digits(example_images, images_per_row=10)\n", - "save_fig(\"more_digits\")\n", + "save_fig(\"more_digits_plot\")\n", "plt.show()" ] }, @@ -980,7 +1001,7 @@ "some_index = 5500\n", "plt.subplot(121); plot_digit(X_test_mod[some_index])\n", "plt.subplot(122); plot_digit(y_test_mod[some_index])\n", - "save_fig(\"noisy_digit_example\")\n", + "save_fig(\"noisy_digit_example_plot\")\n", "plt.show()" ] }, @@ -1005,7 +1026,7 @@ "source": [ "clean_digit = knn_clf.predict([X_test_mod[some_index]])\n", "plot_digit(clean_digit)\n", - "save_fig(\"cleaned_digit_example\")\n", + "save_fig(\"cleaned_digit_example_plot\")\n", "plt.show()" ] }, @@ -1183,6 +1204,31 @@ "source": [ "plot_digit(ambiguous_digit)" ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": true + }, + "source": [ + "# Exercise solutions" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Coming soon**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] } ], "metadata": { @@ -1203,10 +1249,14 @@ "pygments_lexer": "ipython3", "version": "3.5.1" }, + "nav_menu": {}, "toc": { + "navigate_menu": true, + "number_sections": true, + "sideBar": true, + "threshold": 6, "toc_cell": false, - "toc_number_sections": true, - "toc_threshold": 6, + "toc_section_display": "block", "toc_window_display": false } }, diff --git a/images/ann/README b/images/ann/README new file mode 100644 index 0000000..1c4283b --- /dev/null +++ b/images/ann/README @@ -0,0 +1 @@ +Images generated by the notebooks diff --git a/images/autoencoders/README b/images/autoencoders/README new file mode 100644 index 0000000..1c4283b --- /dev/null +++ b/images/autoencoders/README @@ -0,0 +1 @@ +Images generated by the notebooks diff --git a/images/cnn/README b/images/cnn/README new file mode 100644 index 0000000..1c4283b --- /dev/null +++ b/images/cnn/README @@ -0,0 +1 @@ +Images generated by the notebooks diff --git a/images/cnn/test_image.png b/images/cnn/test_image.png new file mode 100644 index 0000000000000000000000000000000000000000..2d53756e0fa56f5c2b1d605f7c18e2282084d135 GIT binary patch literal 181822 zcmZ^}W0WRMw=G)ivTfV8UG7y+qP}nw#_cPtM2=Ld*5^RIQQHfBSuEfHRqZu za>cL6j8Kr1fP=<{1_Aws|z17@xMm@>-w)cP2Da3x09Xoe=qBw1v2~_VPK+X zWca_yTrAE0e`Nng{!8|sb^Vte@4v~o6)fFNZ8SwKZB6Z*|0#{1nTw0}KQ{lL$p0?% ze<(Hnf0h3u@_#Bt>}~9wlpPF>P5GJr+w6Zp|AYN+-P}s1&h|F0|Gqa>J4+XSW?qK> z8~lH&ny5_39 za-v3O{hB%E=+kPur6Q)yvDMgjd&OEa(p~r1XnVrySNo~f;<{F6x{*BPfjRrE@FcSI zB&({r_3iR^H|?r`W~;|NW#zDMsu%FcE;qvb=Vr5dq&F=qPhC~n&ZmPtq@Y>WmT-$b zcIj*Tv_dCsu423AN-dY5wx_W=bI;GW&5j_PiBr0Jv8fv z=sF)=ZiE%K+Mb%Dw{rY8r>(3P9XIib)WzSw`n!F-s?TRfk85#pTh8Aal>{PfFFPfbb64Bb%J=OIabwmQ zF?GE9lWh#DOSkyd8eLWd)yM2^Qy-o)Z60}%R?nh;6?AHEDrjZjrq-(puO|yP+Zr|6 z4t1VJRyE>0yi#YZtJ+E|B0sj-m3i_eE2`TzZEBupVp=;ity7&gM^D-KBOg>>o%ybB z6jEdrZOeRat<{!Q>Kr|NQKy>dv$U5q=6!NUZM8!SaCT>9)n8>7n%7Eu>+@ZG`Ev8( zF5an6YqsnDzAyJalQ;0kJU(UO1D>9<^~C1@m8EtsMg(o9RBbQY-ZgS_aTjf4)2MYL zyL*1sKW}*gy+3c5ayN5*-G8U{cKf{PUT+>>{d~QD&QN)>JoUZ3to^<%Ey8}&t+i}o zoqZd9xGFN#WdyAGco-3Ct7>lA(vXW_ooVRqIVKr>4`jYnJ_XBew25hFcBPQqY-OTR z#e6SkWON#r!&!T8r>+@Wj*dZy=zUHsYJ&FJU-lTbZ8g)M3^qaAtpC)g(t@i$^|)@Vmh84e54k1P=+?Zybi%In@?f1zYwH7yud-6RRK(PF1){rVM^-&| zUTy1M%=9r^`yK}qQ`Eob^7;eW7$R%T^i7*Ei^o3I+LlzDc6g1qx20cnmtLM#ZXX|+ z`cB6>O~qw6jOe#kTc5e>iaMV9zqL_o*KppRX&_jcF#W zqN3DlC{BxUfHb*$LZ`}kW5Vvd|3=(g(+(wJn>Mti{Q0zRsMbFHU3*$c)44f_pZ1J| zpqt?)lBM_f(AlQydiCLn30t}iJZhAW*JYiaEpbEJK9G**rtyk}ys~v2a0$(d8ncjD zZP^Q&yLQD?Hw7FULS>a%>yPQGy>inHl5EM4wz~FFsrTWynJa%}Q@Q#m+udGycG=my zqoS_cp>jLIUajBKW(r>S5Yfn$cG{+$X;;}h^&(Cdo&u$Ya_4giqS^w*{180lg?Eeh^jA$96x>!^!vuN zKDW5l!q#3%erWy$^lUSdT$!U|UoCg7AUDcVzy0?X*uwAHiRwxyVn|J1dJ^LY`eeeT zYdjMDDvw!kXaa*M*)@*A#Z~^a-!Vt2N&EBfn+o)J}R3gA>=i@~*)-n_oHbiy08 zS1|DD0((_q?CeIBtC*33tgsFK{^3wLU~0;+8}spS#P!m%b1RvR5mz42zm2?0!OY9& z1~*yx7y~-B(utITf$ALV+{^D0A#^5NVEdsXq9cK84rLXqXDsWj+5sITV5V6o&GbH+ z&yu6S)WnopDEk{(y_a*VwOf@N-Cz*U>mRV2&KCV!hDAfou#j?hLFWD?)#l3DL6l`_nV}_Vwm{za(L@sIz@}CZpjm z=FXkf>3{Zyseu!gR}p=M_mGdinNK^v#yJ%IWt&-Y&MmuS26b~*ZZ4~}Z(Cc`q(kY_ zY>%M}8101uiPT03pG9n{|K1V3E&W|JnQuo_QS;4QxDJythoY3}_SI zY@>f}&3V@aL7eQjda1qAjnZw^j=W`FYJGTR3Uz4w2SA z>7&`l@aWd0o%Z)L?PJPoL|t^ibujv_v|&BX_7`^`U46L--*h{fNz0r%hl?{$jLuDS z2%z?khDKkNj|WyLiXcieH#2QGHoq^-5h0fucs7`hR`oiV*gF(%sGLdO9Pg99R;ubp5Er8hEAJA%diaW-Jeu)Vhj&Z{rknM#P69%gJz14N}Z znoh)cE-c2`$l<0jqjz}t_qzNa8;b*olcvMYdYg*-+~P!h#X{s)iJ8-$>(It@AgaO9 zO4vl8(hXw@5GG9%Lo7ab%|Y9~Y^*edsg=kRtu=cJ+LLJhth=t2mY2h@`9yP@ z1%+7k4=uo;#-zGFd*NsFRVZ|@=#NyVAaK|{C^?G1a5ScCgS=#D#yOvXuAFZ(XlX=4 z%qjUi4dZEtf!a<>*;Gjn!*kQ6xE)^YR*f*SfNcoo8ln+3R6)lp+Vap^Vl#!Rq@a*g zhZC72)~{fZ(tTh0`GdDmbGeV&Y`tmxOb49oIquBfS>)_B5Z}ycIeIL17>Y4ZD?ag5 zS;i~oP?CY5rf)wz>s|(NV4gV@4jKJ>i(_|>dwtZ!OBWty0cIe|s3pj;i0qjc?6`8D zbW)j=or8V)vB~PUZWmV%?wmfjzgzE%uOKVGw^rlUlm)d%EJ1BB6jQ1)RibU+le4a3 z16KUGk_+9JSJo;?4ZERK;!Hc_2@*jgGW$VecEJM}MoJs;a{eGq8A370>5V~enXz-vj=+zE1xXx2 z{QWJhHZ2nno##xc$k>~ytyz2r$Kr)aGS(WXx{zbdPm!TNwmo!7N^Uk7B* zLZ*ZC>*$8xlnfaJ@RG-#E@e(l;|Kxl4`vid#dgR>Inod-Df5epJFcHg7k8{-9)Esl zx-m-DTyd7GJ*hHhwWvU`8>)|AInA~i)0X6Tr)4~ zMsK<^QGPV`K<)^CeT^wMt~a1TAUS5(w+az+C(?VcvF_x^h$xeL2VkSI<_PXN&`*ux zh`dhqv|gP6)nvT~@MeqE572P{7hJF1+=94cQ)**{n?{0*H*K{YZeZkk9S zaT+Z%DKNSEC(4rHbg7I0$>fKw&yE{29$?x|yQ6<1h|-vCkHwoUOw+y$qS``)eCcYS zSXDkqT_ByrV=Si0yl1G^tC1ssF$&#>8uxH6i3^`|vqXW;ukb>)?j3Og%?3l#_vQfN z4{)IarQB?*uNqYY4Z6j@~At;a?a@d<^{GiuoWdKjz}!~#YaMQ0hY3#C})e9d>9 zR_eC_*QGsC?;>{A$r z6baSCA35%a*a&Zi&k5x@Rr8$dKw7>-HB(7e-flcFI&X{u<9L`OF)k@_JXiWfW~Z6X zd46kM;&6@ydgFxM__T?j6nb}J>Bo%U0ZU^9F&@1K*)d_zYUJJ6nV$p|1kp}Mz=W?K zp`{qzyWZ@U1f_9&e?JxVcKBjqG+w?TfGjHGpc9k3Wnn#LS1Si=P!7+EKb zFfr}o$$TEg36*|k`18{Gb;CfibLRK;_vq8@g+0hh$~2dPG8+B?2rPmRatIof5rp5v zTUdF(euFEaTO*TO8fZl_li8V;<6|j2zqXm@CA#RSl_(@Ctf(^}g4U~?IQ)Ly-MnpT z|F~iLs$WHXqOll>m_&tqT`Rzc;GIvB$VF5T<{(ZIO}_#0p)hfBEs0*Xskse+JQ(8D z4D((b8m}Lq1W09p8)5AV24-1q-@?6^H;Hh@XG}W954Z~k4ghLlLvBTy68X_iqNck4 z`Wjls439HwG1^6La=if<_PBMG2sI>I`)5-J3PhLGFao~wJP<1n_Z_t`_J&;>`5ijtqj-}z9RUJ?MpB3d zN&SOmu)96@0f<~ESAn-sAB8bpr0|kp!N!DC^AsTi@)_u##&p*OdJ@wUNj0D2Psjem z=6eq<&!$we*eGyEjIXG}%F+D@GZXa5lN7$nHi2&%)uNx%oc0z_nsGv8X|uuBqW30nK8*l(I2V zIGLgWK@)GJ4gEyfjqcODL&;ZdhG}RB(E3T~NS6j;262$x zbGz4{p3Ks92Px1Ecf!-ASVxHy+`ytvMzLBibzDSzcCq_Jw~&+HDs`fUHzN*R<3Vl9 zRFrO0`VW!vS)!|&Rc;5iskWlKY|;p8vjjch+tiPus6ms7r*Q|QeW@3rSEnqJj~3H0 zYUOcVbK{w$3=A7my%@&H>d$ay$b zfM623-~Yf|iJ_&)#p3Fc^06%}$Rh0d&qfh4%efJ{^sb(LGo&ex8C9i0_%IalrJT{(2cD`(#9@`#x>zs?9mmX z!4(s6fC`_9R70WX7l#YH&FX7U`MBY0Zn$#i{&w`;PCQZt2k7ooX8m|e3$`BYe-Op{ z4kREVu%)61Q02XnM9^=1wQuYH{M|eIIi9eNT7-K0&Ls8{l@J{O%Wb5m85O-XP7em= z?pD;MMq7V(-d-=NxKr$lKEGW^RvUKm5Vp__Z?QN+Fs1bn22)xfC=|q`h4*~3v{C@X zzV@U$x{%f)nq^{D&W$HJ>Dj{yvt+>$Gh;6&=>-lDG>U2v&(*4S8LB7eZ6Ec|o0vi6 z@Ge9bPaiT@vK@M%3)+e>u>jP8L=@vZw2}dbL~TbDrHA`ys{)?pg#=3l>XnU<;RjMgzl~mjX`yB^ z@;aJe;9M*Id zYdaNU<5NoGNs0x5(7`pFPPF1g&ZSXcIYBC1&3NJ*m0Cwf;WGr=PueB|2Yr~fkDc4= ziL3(4C+|CacIOBzWU5fi&gB|DIS-$jBO4HR0~ZQ`G)d@zDB~^q@Iqi%M$&at=gp8z zjq2bw|12kJXe`hLy1ubzaW3PE|ADHP7R3BI)SsxGpAFtZg&wtB&I92m2@hwCz$U`p zgap`G(INBXn&9bJZ3P;xVhuka10AT>NeOZX`aMkU;~a@7VZww4RycxGS&5+>4}f-S zgFODSDb_)QkxDfeJQufv1sTa=-S3xP-bxn*h6$TG33ATQU-5UO6+W~W4?HoMh=09% z;iV*rML&!prNmG<0h}KMuJ@9PIIc|tX{(_)!0ThMC{HyrS66t02Y|L;&_j!cp`Crw z6PRTj@98Jzk?Aq3<;^~vd1^u)0H-$yI2mfRc)h7Ab&+IK+%I#0m_BivoSGJg2@Ks2 z<)JPKqUXgh+{^@fBv|5%W`3NbAfa8bwDjV3JG|Fpm)0Q)S#TswT*vJ7r`uR%Bbh8I z8%-qLb#Hc(x#voLB|F^nD^HJisDF0=!6Y1jwZ8H|QKp*27w4!ECzg>ZmY2Z+Bgy!o z9tzsQ+GeJX`Sc2m#3|6(D#lYz|5dn@4oz^pu;|>K9Y7~2$#x_Zq{hKyM zKtK?u4X0YGKJrAGH{}UIVQDDIcg$~@d&tR}_Exl#r&qSwo&|HSc^Q=4YWhj5&oF~h z?`9BGL7)Y|t}yDP&Yu0}b5J=jd6YEiAHC-N;9OSp7IP${ti7<;7TQr#&XY$qN1G_$2u3SsIhzT|}6BOuZR^r6%uL7v+)D3k7AU>PL0%&361 zYrI7{Gax{K_-?KQ)kllDUq7uX@~Sk6v~s~ykaDOJ6bxi4eYGTDwBQ2#vA_e5Jue)j zM)AA{C!7|Wjx=GRm3l?5s%90X4eAlnOa2{sHOnThoM5gwjK3I{DCM=n*lH*mOa)CP zX(G5RMT9&FA%#l`xbu|U4EZ=S1mE$=4XSKdB`OKR#r&9t*&r%?(Na&|N7t;9))Dx| zKd#KP!l@#d+0@Y(+S8a?%<-^o=6ltZcGBe#ovJ-F43w+`$dEj}qNCyn>2*zp0m1Sm z-o@ltK9|M;9}o0O6(Jw2dAPm2ZQrKP$)KU&}srBIgS>ecMZ7{MPLCy2s z^oN&)JwbZYV5dI;N=a>w1LVUq*9d`}5DyzjCYI339hi*;P;cOq{RD=Zbf|lTxm4otvROV zYO#1hc0$;Ir5HFd7>*OE>n4_{K}SD@#B3*+M6rF|5`y@WRpH!;cP1STV`I82v` zBU3_ri>UbY-Ewwa;;PvK>$|u2+G=s%M6#-OUPat4R2gO{+F5PboCj)5-$TsXcW3mc z@{*`FY}$8O-p_|`2mQx&>K^_Q47AQ?Qa&|*WdAotfkW*gm<1#K_&J@ZmMy2J=cEo6 zzXvsNDuk9n29&uWyaX`w6(<$wOk?w9u_|v2h^KM$N_T{01$hms)i@^I)c8g~6yubE ztcCHU(tUh8K`(QN$13<;a7uY+-l<+KH69x{IiayhGnPXRGq*|NaOv@DS-bhWN1k=L zap`sXal}$6Z$o+^W=QxOauDRu%ty{h5oGnTIP8okL;aHpTb`W0q_G!{ML~$3*MO%L1&$&v z&JkMT_h$F`bbYIKzNWREZN2~L`};&i@KbDda z04+L5d*!6tR-kwK@E*?TfI72%E9oX;kSOv&FT;FvUb@2id$ck-4Z155^g8y9l7SzD z-z&wXY)nv7Vm^Z6`p7w(O=lZpWC&%(spPqK%G}Cnxh3K$U`xB*qCkPp4bDxvXOUnI zq=;<(TX)omav`-w&EWKoRj5y~hd5RoR5W^nV9pRe8T^tRO25^q-Q0#|B3~>1!xRYI z(tNV}DBb`dA6ZVHI|3>tg;xxidjaiPYAO{vOrM!< zv&a^(G*37Ye>;`OvJ{q5*z=3sW9k!y5QsD$M2a7s)exb+?X@9Vl0?t@by1q78(E8<0OdhJ~fzYA6DyJ4Zd1Q<5wJ!cd+Ir`;Uk5 zK8UPZJ?nG--K#U(g{l4O#<`NVp1B}(Pgq+)0S;U^yN!@MY{l09pz1wn&3EZj6- zb~YBMW`*`~a+6m3Zgey<=J&0?`v8dp&P6{D7x|hhJ~VbaF4Bm8`?1`^p~$g71-S_Ocx1i!-3SO5QAml*k}?8(a4bb zra#p@-}+9r?b(?M{T?4h41Az~9rQgygF#S)Q@Fk*x@H7*FsbuH6jOe;y-#V{*n-MX zOD87tJxUEmUG}Jf=`C;q!3eYW(pt>PIg7}V3&`k@1db`WzBPI9o^gZi6$rB7`p7s$ z`1BzkB^+Kja161Yc+mtDM9Mzm!(?IP8g+_#Dt+T=?anVK3S=DUK5h9``3Hh+;i%=} zVO^Fn4aaOx^nJ*)=%|AQ}8M?$L|yDeZy^3z`D9gqfFb zF#%RZLgcWi+ei#s{Cq?9_2uRJ`OK=8-}lpdH!s7y$6x5uTTP^nt!)F;a{fjt4*h=J z@2MH`$NSTw9hEE{JXF@dv3Q{=3C1=$)f%>y6cKn3zw5EYayRm_lFVyUpl|Qz9oNsL zxJV8rU5ZYAro^6JgJ(yuzjm7;nYqB6cs z=RW%DnpgK+oYivGW_skX>Iuj;GFVf2gxGwG14T10);E*yTnfG`U3KdT74$8&S?_?Z zSR6)}Ni0){vdVhR#T!?Yz8LZXQ-iPmEK|3>Lq273txl)4BIDEq&KMKM!I<| zZ{4|58yzDEsqhR$yfqsPi+v-X`ylD}NyUn=vZ8b&{bH|);dt2nctbEU)s5U-bijM^ zUF0D$TK&@yO>tNf8rjZBdPxuetO$}}07v&CD`c7FWp|QqaS}8Z+_J0K0rL}ZwPRUv znsm#?^g#C6LkpbqUrkxRN{iY6a?D0x3rETM)t`-RKAsRnR3V|vc5}=oL(r^xq!9QT zcB&=!Ztj>1DWl|@^sjNAh z#3%q2)<{GWAR|~62O`lTbSbM#!vWDo`P@f)i`d085-CD0_zyLbEGM{842|q1!u*B35B2aeiQs+B`Y)NldAWGgeGBiS5fmoc=n~{3d#Ky zFbK^|S|qc3l}DTTF#URLSy=zV%$}lwZ!QcPtB_eq9hw_UQFV~}&v76w@jdB|7(l@o zkZZ~AO^pPo<)JW|>@BjFJZZs|^g)pUd2GP{iHAg1v)H-;1uMx-oR4W)p*HpPofwDMg58!i4acY_V5-;1j0Gk=sx|J$pf7~sXG?& zc#o^eJIH`IIng~($I;@V>hA|+2?73)N&N$lDl%XyHd@Ul63E!WHyf&Ktp>LUvL4DR9nUgH%{mfHA~ODfHQhdG5N2(VEhP zdkeD*VxW$T?xksC2A<|EfLy$~()N1Dy(c|BgF~K>Who9g0SiR~zym7-)c8T_jKL(n z%COelHHfK8t{b2L47Wm%a7{(5%{bd{Sdx%wg4HN+9_OThC@r>kN|xtyP>!n{9(XXYjZyqDrOXJ(#Ezs+@*PC5`7Ex@>Q1a z;m^^HN1tClR2UT$+>EATMjCE36xTfwLw!hP7O zF}=j!x-v>}`P&N^sShGLusg;xg7i|qc1hI4KK4NdvHQkd;raVF(6}tQgVp$`0mCj5 zuYvjWz)uOpLg(_g<_}NaQuRcE{o#^gICog!p)Q@I4J6`&_ClN$cAZ3ytUb`hMah*Y z6<@>`VRPdC2WY(sc(uRP8Vpb{Xo4WCJ2G-Vk3{Z{0`Bcnb1Ge_5etr?8BTM6JMzk7 z{C`aVr_ejxIq=p8L@X9+M0#7&$@vIq96Nf%QzuQARz0y-QIMHs_y!1V9WEV9B^)9} z(_%;KLwDRGsO~=})K(6w+nM#kC#KC*^YYVQDnYa`>QIH=l3ZCUPJi=J_kkzFI8JKx z3^T8T4o~6Q8-!t4OgLrCmCsw!aedJaQsxF;1-IODwh~E~vIpRX`zK~T{mL}EO>S%g zMHx@XlvZsAb?IdLl9vm|O^i~u2Y|`?w-TG8x@XQKp_w8}NiDHbR5e~^@U3WC8e3r) z9#uST$`$zFIh3I-x+vE#WiSp&B9pABs|b_Z`+${@+r>T~Y=0yt2vEeNBe=X#e&I_s>Fur0@EthWQP$%&3mXe2)j_%Ou2)C=?8<-gxwKFNKja2%eHF4aUX($Bh^?(Y*9zq+A#dDnK(WU@zA#IoL3X99yT2vWkpy*xxr4M;|f zKivQk=EQ|yR15+V2NRya1Toyh7R7ru8S`&6!Ny2y@QxYVCD7Ms8p1D@NoGg_xA9l-+V6h`YR-c6hFiv;{2kxHRd;MV(~Obi(_ASr>(f2HOd z8`vOF2b*SkY}V~mxsU@Vi4z(lkcCIVEcZV2+1cGeVt9qYEBLzuP5H!9h0=erdC-CQ zMY3p!$>A`=jRVGMp{7_aEs@|bgV6Xm=;%}6mydoU^H|omRB!-ynKpkL82-Xh2F8bA zj(Fn-8xJ9~Ong7#77W}fn?Y3~3WbOhx(5C7e|4RID|Q}-aRY zL4#zqSiT(=Xp#j;0Xgy@691U1n>ewE$mEZWDNUpBmRDNeO@C!7;`|7vhBOuo*eaPz zjo=&AGh}4)Z(Y3~oAW_6k1B|mwqA<#wQRr_HHTY4coN>oUF3Y2bd0qThhxQO@%$>8m;(fG}Hy)%h`&eOvUCL1C8EBSw{1q2epe&0OuYevGIpK3a`(N-8z!t74FsN7oUvZ>>>4PIYTQ&)(2*p zW>JStM`^Ff0M39%jZkmOcEFRjg`$n?5!RX3q5%6_A+KeYHV#WccHsBUeSw?1&y^ql zLde0dlY}=Gjc|Ml533mF*Da_yp)W^^W~5c_h}`A&k^gfh9;qCS$=c zqlLmQJK8RMO|0}|$DWs6T%(VfO8j*In*=WI79)|CE{Fm;8!3I zQ124p3kWj~W|SyDiIWflg)t$dETb@fiWps#tOtLXPK|XMFFFtuYV2NKQAOyWC5q_b zxlw(&0)W^9$H@9ab03K z$gMGSX3h_lV{!s2oK8v=sbuB3%1LFjNVmw64u!R#IS99i%M!P?w2~O;Rh7B34o~mJ z=g0N4)pj+vJmvW5z#ql7FQV4}fu+&@s)*U>KB3lM7z1O5_9y0@#UvpR%VV%!*mrbR zsznIpu_Q5wn;F+m;x5EYNa%iv9EM7BeMWKMp&Y8t^3grA!Lv(mrHD5NN`E%*z<3ZK zoL}*0Np<`eFRkM2&zmxcqm`awsu19bC{rC9A7KNV8yIF-B|Ls&CN69elv0xPkV8Ca zdL(!N`;3I?LiE~L5N2j{6gOLYieKRgEJ{Sr?7cNOff%DLS`9bp7u99J^lXPdI{TIs z;(mVD4azJ?!~9beC_r}nUJS!W;!R-Qgt5AzgRK5KPLNt38U+egGKz8Tew+Sc)1Uwz2=8ShSA28NKxc{+-pX_D@R7TLe77)tSVkGgA9 zmYja?d*3&GJ$*mHv|s=HsnF`v*MmnqbHudQCNG;kJ=u61nh6jQe1P~19W}eVoz#D^ z8hYLbMN4U4{v`^y>FqUDxj213?%i|r5kP95XST?LM_LkE2>iihWf?lG}~3Rp)uAAnHa z2jz=Sa4ka(Msf6Sc6qPJkv0LLGN%uJK;q@75BkB<*T8zt0}nTZM&F4*Fqtgnkw<3; zy*=Dc&t^eMcNgE7!NZE6$L%MLYmvLe2e1~86_sVXD5&8R3>tKbNHk#}mNzU{8yl&c zWz{IrCvy=0HfzeW3`Em>HxY|S+E}t!SDU35TmlhnvLxzppYS#P!6$t*cS6VzWlmw7 z3Go*&VX9ZT2kHIZlh|3!@Vcc_0XHWMMO#~6u~mW4CUlki1<)ObD>gC^us;7tKvwzv z?F~KWj8E@PgeF-)CFWsHHz9{|wj<>SrmhIfJY@sE4y_e{(bPIvv)xJwZ&bs@{8x>ALWXEn0=7L07r<6UYa z2zGE`Bd%K`Fbg`rh09e$eNWRnPY8`702&=f|=*w{`aCs)+Z>str+r zg-lC?(QpBZmo+)!%jiUsWO}%`v4;$`NagZgfjGYtbS*S4+5MuvXC-14( zX(;qMhBHePjY>ZG;<*H%VB!phixyMCC}|pK^WNd{Iwkgj%{;D{Mi=|bgv!r4);riG zOMPSYIuVhFaZ#5Lq(H=)d1;EOZ@}bjN^XZF*ip7Rm_0;%aJz(`#uQH~9DC!^0k7-) z3*qx&%*BmZZOX;S_nW3vjXRiP)HD_sp#TcJ7vC0T_0P7NJusyuxmF9m{s)o%J{H%+ zfxHV|>J8S>--fGTND$A-5@IQh%<`>A=ZLslNmC0T$6=;EDZ0xRllj2V7F3AhV$*7Qftb$*S!*k~-ALdYz zF9~K0n4GW14~7ql?@Zm0Dbm)7oYY0eD6tvi@lLC;e7N!w-o*};2DcV|>o zTHBm^bhFOz#HL4akoG0l;Fs8i$WdCiyny@-VavTR`CV9km@U9##UA?qRDy@_4lO-H7UrT;MZ=Tphv#UJWWz?B}wrJ56^isPRgtkOWM+tl%$Mbw8nAM zx&*ht*PxsCL|;V4#Fk14R+2$P)}%eJt~*NglA&XCVheMVOn#}HP!0x?A$$YI(HaCr0_Wle>3Sk-2dmH0 zk4Awz$?7mv(ftU$A>rYTBXV2Z(;Pwqx*f6SH*Ml$5AhqQ|df#4JjJO&b2>9VmOrVesB>2hDTV(n4WaWEwPJ79zs6%mCxi zBa~&b8Ru!X+otC@yZ0jr-`tv7MdA%*=uxwjiO9K3hG8Ktgvn2HAV-F|Z?iOuFzJ)q zrCku!1orG6Z&Y|;+p?{%qsO74Z$1;omoDOkv&SsZv&)8_9(*H9^_(Oa=2m~hTph^g z$8F&$?n@M|NjBwoL>SE03WW&Fsj?YMFuC$rj!F4Ye8P!dBh03cwED!M#OW5#wD**; z26pCdH2JlVU?J=7>wOn=6_39L_jt$^&zL`s%0)o3awDGnCRy-#5!adGP2tIB(khe;KQrB`EQ+i*SO(+YrVk<2Cg%NiM~Xa)yCgpbl1+Cpb=m;i^Sp zVp;hv=+~8z+%9Z98AFo@3uB+VRB7hhkjTKBwY7eP1)OOmi0osEHaNJ2>1lUTl36-} z479|8;Fr`SNYEbLoQmi2r-UAN(xjsU(RNx?r$g2^)b+zJEboX4h)*Xf>|}S-$sJDY z{|#8-G|ozyC{#7N{!=>4q+LLs^=RCBsn7@_VaX?X`KleOd(lr|IQVzHep$U+$;=28 zB%2G-HAuYOz7t?k?oGmn(rowJv{g7LPz@GtC_G15j;r-gBdta?LVak)Wou9bX@uZts*+jWy9zf4EF4o) z%b!<_RKiS6cNAWW5O74R`_M2fLdsm(NK(WMp<;DmRXvqm=A0HayyD$}Q)Ivu2-*gu z9;=0HV^+!nR>d%X#0L_1vAkt??u+x4{{us&0qpWXC~1uxj#X#wrtLQWUpnoGEf#I3 zMy6Ju1i5g3CyT+;>e&f?kiXJ&-sw&0sB2`aG@tFporpEYF9wR{mO{1WrLIFTFabte zW^9P0RwbK;FHR4|IgkJe%DcC~-;I%wdfbYrI#BI3;qOjwZJUpZ$gmw!KI!FxjITud zC>nCMyb{uu{Aw0wPC`o-dA&TMi`a*R#eVjSXeO8X)@W{tpqQ&EwQm5g~8obf!v;Hxkn;8vc;9`6fAD|4o(q$nq5ur7jo z!xIlPCD7j!ljDhz3D=Pj>toQ=OyQBiIgcIh#t;Vqbgf_LdOFhjJX=P9;oEGSnkC4B zIOqiaE>vY@So&O_I`8DBkm(qxi9O-0W1elIbNBO$WzB z0?wt`Vmp+d`p2Rn{!uL5**02i>7#7c*3?=AO-U6=zhLcmw4!LaQL@I-yS+1&GB82- z_fAJcb8?FDHcOLR2K~L=*`3ds0>L8z-;k^X!ASgB) z5#Gedm#1HR4hUj621(`9k3)r&*o>lBH*>VS@79q5{5yGE4UM!lI+BQXVN35WonV%> z$tzN&r6guZ+#bE}Q7y&x8h%BBXZQ7K>!8A*l9Za?M#S$Y0q~0N*>$F^Nbe4ymI1*- zc|id>Ro1{<)E~RX+@b>A^nB4z@YW}Rtsay(kmUv5T(itJBlF-KA6DOZ1^Hm*&Xhb6 z*pEk#M@J;Iq)LYerY;vzsOnKuV%kuMbL@@zbsk|EAPn}xZ{6`n7Ki>5Y2Vji8%;+H z9@K;*IPE!D^|0OSl%OuKi-5~7C$+R)v_-A2SPx)JgjuNzE#2dH&|e~Fv3gKAy>h{= zzwgW-c$D*E_geg_1o~_V{J;#*M1^2Q(JA?mHIobGn<~PboU?B@VJ8nuO;Gn|u^7@D z{EU?voM~w0-$sOl2Z`Y?2f&bto$cZ=sW&mM&|Y6iy>&##7lP|~iIKyCNhDGKj9inN z4l$ZNw{u$& zcIo{YtBEp*Tb6J?f3s`B#0b|!je}``;lw<_sV`z?TC!b5Yo^wH31z^Ggs!a)@E&^n zqVC-daYfwa;Vl}gWVs>dDF_bvH4T*<79Qv(ZW!&GIg`lzMm%FkE_a`|e>I$T6tHGT zKbumm+r{21i6kvyBJ3vz%t+-Rda-M!JG=S^KI$7fvn4_^I-MUf{Bles{mIZP9|A-a z?DbdytOwC#=xrJX6@G#pytu%l{*y{!_?X)jlkPk-fU9`I0CJGj6oJVO_pY>@A5*9j zg~?}>tdJbI-wOivqa>~lKrO$>+`MQQ z1kSEKPIGSPFK`n(p`Bx$f2Sg?h`{uI@&te0=wg3$@8Fvi;a57o`~E11qA&L_?@4gAaquOcgLnl90^d7 z5gN<#S3$DzwvAlsLg~CFT%r;ILJTI!d!qQW2u1ht^#$0paK&=e?CbjxQ@!}^UK$j3 z>j+^5Z;B*$1Yo>IbdwG`i;t~7@{b@(f!mBLTGlg&9~Pz;OTx(U0ssIT#6DtKPQfEd zn`44Q_woM+H9*S0vq+WViO@SebgpqafRMcb_OiIl{NR!x#mj>%=ELcCy90I&*kb3c z0Q&0{eVitB9Pvp_RC3Bu72!2$Lrcb=8Zj!by9tsSQc%TMIh;^H0;_0K99|V9Vy?#J z6sHVGctG?BeV~dhfgy{I=42}!FG(zA6B?3ii#|5=nGT?uy=Ba~L^Lzew5#f_!2ijUx!3BDVQ)XC+GcjPqpc zD{4ifZL~T8(&Ok8ZIDaxpbCmmoJ&A;y|s1_s1HcYSr3Sk9;bl-PopDB)>(1xx<~Fw z7>!6E>F!>9h{mXa`0X81M+ZH- zQl*ds#F%oX-mhgaaxqP1S6RCea6Wk1jW06+jqL_t));%`y8vjf#FhIau9`|8^P*Ba;?mqWpoKMA<;5xnVq}(ZMezv$8?lJ0 zylVn8ndAwARB{G1K3u{BrRs+Z(g;xggFTBPrX1cJr##T*0&RMK^Sm0K%(*)%Zt(`K zu+~JHjBPg+Ln6yWXm*n*G9i}Z=PO<@ROf5V>s;C+AFIm4;+J%P7YHk_ zt;+W~Azl6`AEr!nocrjfmW(x7Y!53>2*9Ktz&@>nz}}fVTD(GbPo$|(2J?XG2QvJJ z3(_~#6Fi8Us)b7flRT!@DlTwtrf+6NQ+wh$jEudrUD6EqkU7>R_>DPY5T8iXW} zB~EH@GF1746R~7qME-A4A@?V(#1;*W^&Ap-oUTemFxO$n(F&q|zRK}I@c5JuW3CrE zj@iKNS!>n)^!{Idep|2oum7z7r-ndHNfZxc;?bb&Pg$WQQs{FhM8LGG^d2hPXkf?j z0!)T3COiV|uu%DFvMrNkY4^OdAq_U^6u|Us_L00wq_voa23r*8p5bEpx&%bwq1=Yo zEh&kvsBgba?_cZ&Dt`Rl-R?TA*5_)s5JOj~NkHKi^_7ugK#~5LsM&)0n6+D?<^B;h zPAxkfjmTbiy8fUll6_(;-Tufaw!xSo`tEZ(6p<@r)Xi`PuflfTCuC zC!{l!VRx)GydY)>wc#Uk+F6cXC%2JWCtYWUDP)7hv|{hiG|cQ@-#il14By}(-{UrU z+m#Kmv_u0Zy=P8!sh+>~U%%0N9IUR>Bo!yGYXDKsX7BILf*cJ>pepXjWI~2M*Kp~Z z$FEOL?$lA3))bRf>a>iSorgnO+~@WbM^m%sz^4nev-S>z^PUiNVe7Zf8jy;NBUHlc zgD$L1S}i;X`l5lLbZ)dPi7ft}w^zU@=QlI4r*x0=XrZ*a?T}6S{DctkLx}@3%`P-1 z)hk?-+8}Kllv7XXn5*_%q%0@i>|ck|n+uN5f8ylh_>MoTN+@uI=rqsACq*CB*alE6 zH@nRC_jCI`T$jtg4*jD)f1ajK{ydHRfY8l5uzfT{!NJj&IB{@Y^ww?z&5THC0Kmn# zQpaSp>ZxfyKq*;5y*;%{SQ^#^*7Rerf#$q*?|9Oy_%JZ1Gr+cq%xFWVIFt!G?bp-z zm%rS9{#j)0|DzrM{eQ@ekaFG21iS52%4#CAIKv(iG~VzN*%pVAnMo%V)O0rBWd~%(4T??no(dJ9_IbO50&1+nyO7J-e9?>VsN!F|v#L)v zXwB7m#EfJn$d<|%sHgm9d$Nc~g7UUabEeVTxi891tM@~;^+{)l(suSOJO~1>Q=ByC z)yv><85ia#5Zo-u{ynN@b}!ZxL;jpf$^Bi7ew~DJ8>DQ7y?%41Mv^13(b078-=Z)& z^)xWdF^vy((Zi#kEP~loDYYgP#__{?TB&^+nP7x2Y5Q}XRD`rTOy{<=5=33-)Nl%o zjZR8MAOh(#CkjykPS%ng?*Ia1P<`bh-ZRK}La8jAUI+C#HhmiYCD|1I4!DF}vD$!k zuQHWWpV zYmSUzwE7egv3)xwt2MIJZ*?c9+1Tct%ZV^zFfS=tHWQ7gP{P?Y+p8l_d%hN#(3oAV z*URht)=Ov79+JwNL>-#tgf2KKl|UWMG*cA>3DaaV6)X(Lj(edsF-N@eQ*pp&I1w5# z97SVM26hNoUT@>Ya_Po*?R%KG*DAVwgHz^33GF~fM_ZoF>uzV??bIMp6Lm_ z*+WAd^Nr_(J~73ouqpfzd-4X!^rXm|o|?915$NpRX+F1?lV%5EkIsp=pWZvCVm&@!HHbmckU_{r{T-?y zhhsRZVP9@Gej&4YS);%&XJwf_ zcEt3eakd0AOO<3-h!5mw`TF6|h2Kpt)l^}2armWJg%Rjo{`lBW|BzD{U3>o8QM-$~ zS4ZU>KJj02)`1cRLay<2I)&JBj?yw~8iwqxW@Kpsfz%Qud@e7Xa0IBF*}!z(SRVzR z^rqSqBLlX`a!2ZnXZ5aSo4d`Y+%o4rVQr)ylBvWyuD<)Z;U~A|%3N1e@CV*Wd5I9t zz;sEVO6Dkcc=OOOwyC>u|MjL2=RJDBuBV z1+na7m&9eOBs)N;xbh*Bd!I%+XrOYC!vdwSla2+6c_d*}4MYh#>Gj79L|yWw!R&h9 zNzVKtsIBg*cKmewj-y2DTEZL@1%f8BvFOS6p$`r!ZK^MC>J5Wscggy4-^))` zQ6}r?8aGv%fkVrzqV=Xx?c`O=?L!!<#2H&2f^d>X4R`DB3OXNN6sZe@)rk3WMi<;K zOL0>OO2dLQHobhZPpzY}!uP8;Fki~qP9^7>y)=Db0iw}FEeG%3iHVXn!`tcSufvxE zHMiT-Td_YMBMH!9g!}aH9`Mh`x%eDX%uiF4Ajaj`6yB0OLY5XL*X^i1Tf8lcZ{FvO z-Z-aR2N86D=2qj(N+7Sy+-79@A>dE9rlVsVZ=2$X17A(VWq?W??DOyc{QPkv;SXMb zop*|TwvYAZ&`PF6?U9)gJ;OS=?~!*3Te>fVDLv$HNshow@N|7G58HkOA0m`lk5pmz z$@W9Dm?K#12i1T7Ht_V|1*Li=jaTpQO2NEzdW;B2P8_J#2SG|c$;AYRF*GvUHQjs*eM> zh4!RQ5+uYSk=|>2d4bty?wm(A(kC$nOAc(Pl4av5&+%FVIicvlnJ+>eYv7lxk`U;} ziIQN?37|TkluBfK(ZNlk0eW1lLWwYa{dkI>P|TDSgT}TfdEZ!FUa5Y!{U77>d3JUP zU0H*=AcK|ZP*LMD19=hxj|c@elhEYTX+ zm!p}SD|%WFn-BUK0HNMV(n2jsTsIY=)z?C6L0K0*C+l z<(s3yAk^rU@^Uw4W#^sdQgxq3Sx0Uw=!KP647VSCW>Vc_xqZL=JGs(!!*qC9RxU+= zcoa8$&MTzA1>5Mh3!BGJ*K7J8mHG@>w;uw8LQCEByS4hOO2ommB6#LkOmQ?%a&p0f zsUBOjWGeVb&Ui>|&`Y9kY=*{SV*MA7fT(ay6+!%=BCu9ji)#sYq){V+*GhsUjHNv% zwR8`9j4F?F(HX%jd;7fn^XvOJm5m^O0E?vMcEY^dPEGeZU*@+j_!#ncSw=4MkgxVIcuXdaRzPql80h z67^&-;z0yu;ogEW0SkvL3%tUZ>tO+@|7!g@b`m$M3bkU4x4oZ#Hk-@qpEiS}IIFix z6LTIf^`(gE)CW90n*nAuD!H+)ip)Ux+8nRR1-s8 zauSmh+=?IHZTK|<%;Ia!k;vYIIkteH``cF)w(>6C@(hOO{pmdCXisz+#jAYY zwoh-gvkqQqO8FE=6t#4RrO5D$sR~z$kMn0+=BNId;?Klw!zY$AALt5J}-?YUnuQRkA zmrp5+%Q<;oT?1_*u{{3xL7jH;vA2{|^4?|3an8;N_%zDkFVdsYi&C%SQBCJD5Bcj% z$U!JRxSF#t1F*VfSdENRq7ho^?VHZFqELeAsq3HT6UDkXJ-Z)|@eB0FT}&c4fPR znI>Ng0_j76gL)!FC3Yy-*+%3lC~JIYHRU`><^n?_x_Ieo}YI+%8V|Uu?&YF=F zjDUOaWht&WA~(v`Q8Y@VTh9(8m>2)AH#oOF|~GIt3xV zacbYcU7(;R5@sC7c)rwHK~a_SsfV2iDwNg4rb*24`7{xjAi0_fK=-&1W5=y?uDC3b^$BLlD@Tfqhr5h5z?-a<2y6fIK%Lf_8Wc>%} zs*CG2Elzn_HCOsaaakUhTm2fkyO?4t0_M&M(>UaqH?MBP2sp1z^UQ(jV4l(SoT`~) zQ@o>UqiB@Px3?YUBB3; zP2J1dAw=m4!mSZ2AynLrDsLDJUqs=h%~^-1T)@l1p7iV&I~ko6@#{8zf4J}E`W_~d zID4afcoK$UQ<<|?y(G|qx(v&eJqcHZy$s{EJS8tx^S;$*$Y{&TpRCS_5cS)a)GFNE zbDp$|NL|L1?mQe>tIsRV(t`OK26a{V@b`RR6dzK<%@ezF_TfdDcRD%RuU@$vhoQ(dSs{iXZ)%k{5+ z`*ywN__L4O$Mj)8q3kPUcf-LQRoY$8?_byPLtdkBhyOyJ>Qr9elr)z!2pM6slCyBv zy3y#cNjk3sV^k@K3&JFFU={*{w(=t6ojXA7=Y!lzC(#xF{nE%G3^oI`!Y+Y+wtcz3 z0u++nJok^+51ostJYT+ff%>_x#>e;j=i~FnqHQj`I&G5?-W98++iB_Ur|<8S`{Ebt z+Mb>{58?J>-5>W`w)wc|N(T*O`^^?H=Cf*d~56Llm^O9(dp2Aqzxiut3 zVoxk0dR#Tj@m8;KcZ?>s9;JiOuH)5q$<(k&yE(K9afZ0abDT^pWg)RRlj_Z7skkbi zD?jjd&=Eld*VC<337zl!IH$UB3|VZe?%!E(zF4W! zq_MtSPYHQupt~f?)LV^fc)Lr&vG?An6k4E6pZ?>&eCw~>>AbF=d zr%!*HXGk^h2+vA=2y$wl@lqIxfr^XEqS@C=D>8(0{~jr)OrU$rALIJK5v(+(nk5mS zWpFpgfy{q=AHBp0G>X;E6u__?(KkKH`;Q0OL#e*l1MAQ3trCM42Q@%hvx=M6H1*bB ze*T;I^2hh*{e}`#6a*4IY6DH`bGwbtn<2hm-~aVr|MhQw`|J5Chxz+!|8e_{BCB<4 zJ(8Z5&(BoNhA;0w{q;>`C4C&ZP}rk=s02qmTKoaFzB@09C0Kx5M;(l*k@UQ{`(!IU z#uw?Hqe;I~suj;~0jp@UxCNY7*iz=U>FFft^_qf=^TLN;DJXBzw58YJC>w49M-v<}0-@efa z_S1PhzdQD3)56(*sMN~&O{7TJNK#!J7R!Z&VQ9~LB=Q=(MFy*n2_>s%SKF8ZA6j*? zQxTG8f0Isa0s>GRI%#J?IQP*GAWzFDJ*7p?}G(4)&M4MyRJ(XS+%msCmjuAyX}7 zYsslKiC|6=O8xw))T9q0&66AonMl{iaWtL$D(4PF@k`B~T7#<@3(ltrM;3rU8&|{) zzrv}oWo!itLV!&V6h+V4U5);VHwo3j4|wr;Q!xGd{>4@9`$yW8WFqRD>6uTFOrdA` z@tw1oAjearULv$4`;}O${*ekM^>=pG_1;s|dDVrq$OEO%NizVk>vZ|*Jl{o8Hx1?* zyC_Ik=V8CD<8S}*kLmFnM?rH*or#M1kG)EqIp%^L^E$a;GB+(*M^~s;9BvFgG&rZE zMz$6}@gseJInuVi)Y)c2=~l4SjC7}e+%)6wj^x#`c=9?dqE^v=uq8UY>{`{61q91Yg0&9d zZx6nbbNf7-WG3YWJIgTm!n=WdIom&S9)&7{Ln!=G*_x)N0KwE;l^SW`Slq7op%5e6 z^y3^M1qRr?lmJ^+1Pu~*mc{%0ynWpM;43MOHb5mTC@j!<-T^f!Gkz+PTxdGB z3OqEt{f7pQ`9>E;?w#GH`gd2L^=b;35_hB(h}I86H;>1@5b|slPLav31SDd7-L5%R z$)wtwKbEDnh(drWgI(%1zzlXE5vVsAXqKViC9FRMG4AP%BED>r(kyD4R!J2VN5InP2&`g8`R zsEJ`BL4Akydj}mTcc_c$z^8&>{%}3MXUc_Z*?dx7>pI`{ax0nR=>Zg*_Rj0qAVhvozFl2@_w;3QS)u%)Zok8 zmoL(bXd_7*LO& z9!x(-W=qD*NWEmE`^(Rc;9%VA=ZBUWWvi&CIT*q#6mKt~_3C^BsTGkzhAv3sIp)Gr zAnO`=BS>p{kw2#YQ;tnt%L@xF;hr8k-;|v?*@2Rla-MekRAjb z@=j&Ee|UShXX5pbAK%BD{x@o${8rDH{>4i+Z#WVI%wUHwC1ORgEbv-yC#&`6jy!=e z0pPUPBXL%h0^3Fqm2gwu=W)r`Q}mjZ(d8f(SGiJu`AZr-AEDIjNSEf+*UhvOI)Ai90Eb&80}DU>juN^f|eT^Biqbine$y$(lz&GW<;4 zSPXyjP(j-xJ&pCU*tylws$oT{>&!?yO}@zRA(vuIqccID5BX1e*j|YK=mf2mAK(#@ z9n+2@dyaalc}mb0&0Q81ApH0tnJa(vPNv5tox`B%NBOIrr)mMq4l7bpwX9#Zxkp9f zBh=@B5Y7Um1}O}|ounO;R?X*peggz4D5PmA0pJep#_yS9&YVR=GI7k(Z_)v!jC)8| zHMy3Y_5LLHPcMYSo3|xOj=QcjAOj~hYhzfNx6a>wslQ#)k_NN;i;k09wTXEQFwkUx0FpWvPwkz?s$a0{stoC zc^1r`DK~o``fp$1e)Z#XynTE=KXZKIsk>fISN8RszW*-*7M)UGrtaIc(u-P}Pe}{W zBU!V~9+9gr&Zl(2J2ReZYV9bCujU)n%ADuSf)qOx&yi}rHE_v&R&s9 z5m>?;JEl}KD;d*G(gel_(SwBxLA=%W0$SX$PcYmSe@VpAiDPks=&AKFR4xIPXmx3N zHwT#rE|!>wFy;up{f0zZmtnYSpL)N3QAhpp@vDT-W3q09$6I*0+~?;X2&tFm+C0^E zN(sa&yE#Ta5}lbxhIOPR&EWsMuVCnhf0m%!`B(?Dh36nCN_&Jsvc4)j@{=CN$th;k z9OZD-jh^X+h+F$F-xr@1{AybE2cG4YUt3qO4VI7u*!~hQ=jM-R63zt(8QpAfk z&^B*xKb@~X|Ne(I;ZKc3Drr%Mh3`41J-mNQc=-O2UEfz|WOGB-YNohI0lS)=ye$sQ zp>b^9aOTXjQu3dAG|iDMYx7LAQ!oGkB*WJ^r&-1cORRO8zJK2CqnH6Al$(|>7qRH8 z2QBM|bl|9EIk24MIsIlC?P}mBO0-n%b+>ftghFc+^q2$N^I*EUSKX!& z;6sU7va&Ao=f%ngs3vJK=VBG?hrBam#|8IFr9$e+j<+#gEJMKj7ZVj!bNU22$1WACn^O@&+Pd@7~Xj&$Li*tUz z@38mUYp=cc)isJEw8YeooxZ5O0$QfibtCT{U|(qutBJTO-z z3y-U-5skCOF#rex?+~VzdlD|hZH2DZK&9R?iq$4$jG&EnC-C#>sZc|XF>Mf;gzD8Xyn@I=3e#fnYFo}(%0z(KnwW6XVYU47KQ1g7EW~8Z@Rj{S0GyP zQ$#FTo3c$$v>7E;DC6iwtxnQv5kedgpKHbtpc6??RTyQ6hVu~+4zL$2_Q`3j7BkIp)lZ%N?)+yb9 zE!M(O|I4oPQSL8SA&3fjsQaOviaE2XDfJNHtT&UWr(v-&VUn#~P+ z)T^EKmyopwW=?_OI@yU$bWUBkVXhyS>Vh9K<$jFK0Y>g6dTOZWT7T7W*AUk*w8B?L zCxLz@x7{1jmoU<{vLB8RzR%sJ0`y=x_6+5r-Gsjq4RU;npceASPT2^6KP(-f8#STD zz2}8#nxeCEKMvN?;`06)7nbWifj(W-`ubrzv$D_(zmsm6=-o=gE{XqjNKw1h2N&2! z=iKF&`*frdA0LT&ruWMt3UtZ=ac|;fyTvXu z!(fhiOu{g({=j>6y2s~rqNsLfED^gEV3yN9j_~U z@3TV)%ynTRC=hH2++=$ttDSVpW=q8E5NWsq2W#s1SQ)1eog=2iW?7R#zAH8Ku&43AJiQv)@c1Bs|%Vy~Y2s~?(gOWjx~A<__0 zhXLpbBILT|!uXo|eZ3{f_d9eo-W*CKhU9Y7lqOIqorknSr@LLzZ+wfC4{rW|J8=NHy37rsY)ly1yHZ-r6tf?WUxEUJofElP zOqu-XF3B1co%(8#fqUp;!x9o+LNL5>JWJ|8;F0@wQ!Ag=BB$2rS|Wyh7J8{!--7L; zJ63EuCNr)%6`%^+0d24!Zj^oIFtH+b!5N4~r6~q-0%$$vgxMaMe8NWio27sLgYS64 z5N*6f;&)5rRLPcFKCA|mLiTo0Q3Dcik8ROZSXnlENf)4|Xg0#N6Y>!FliO>EP@UJf9JjB~`G<;h+=F#y>V32_d)ulhmPPiMMjDZIPzi8-pBgP8b=h z!{LZ}?QG^W>oSOZM*9l1wB+%HVmW6!8Q2Uj;QE*$ZXnuv1~r*L`ztax$@J?EVvT;Y zoj8VN6^*J)4Ez?(Gidfi4u=JeQhzfMOotUrt2H^g_%KX2k%1aC!4)EKuACU-4^pLP z^MQ;2zd(k7K66up?*x6IpEu!g93+v^T8!eQoDz(pTHWZFUFDR?f(d@9$EZp`3i#ma zD5WS+Y+yzSmbGGf*mu--I4dri)KUfcEf}oAhC*8`6kUV#q=*@^_cM|)So`={jvIb( z!^!dl?aU8p5#*5y@Vc7xw$s1f%=18LAt`up&ZMVgi-Jq(HP{l%bh((ZfRUZc0T(Og(%N6hR@Q{)_4ZmhQAwdd`Xo1w z=6pSkNHv8mP(ms&07yo}g;IrMp4siqXgZ(X-QA|waB2}|i|yTfbGzD3)4vuRA$CvV zJv4C=$!7XOq@34bP*mi95BC#_pR`nB-lCXrGbxNB#5Au5){qbp{7b*EyNDx4idNkQJhLzD`{kV^-O zi=YczL4AOTT@#@t9u4e|F@qsBYReZ}oe=0U6}gpZlCQaupp_Tm9+-8u%a6HF8^eyl ze#QH+XeNSgknE5bzBKJ9qQAFs$eTmk+6O=Ylko2a|lkqPF6{< zh`yK{kEpyr!#s>QYwE`-FI-@S0@-wL0q?9ZK#g`2N1T{D#s$;hTDbyvBiZN_!PtH4 z_WVTM&;^yjf7=P~zqW2Y6g3Q2|rDyTrxs)Gp{snrZagkWGDn35F7lZT;7dv;6$@D(@~BFrQPZjJZXzo00pY8z-=<`JZR?0q?E=H zo=@56H2Ho(0sfl* zAs)amK$=XNI4VcS79TE!7R2OI_6T5&wayKVrq=%~f0Q6H*dtfc#&D!-`ae!A zUywiJwhnr;Ob=k7wGU1ChOKZot+SKlfq|fKsWQzL*fF!t)keM5gu`dJ+>nl9ZW0=Q z%046+9Hb@1(vsN(C~Y@LvCuVBBQ8Ef=E9Eh-kuu!k@B0>a&$9t5|YzKE%^H48bE?; z1rFocUfu*I&{`V-WThlGSp(INVg>gZ+!VTFOdxZG5)Zk4>P8hTD+>%f($YYdDn^Cb zf=p5c&uGSG0BA?u4Fukvb|LXhRSFh$@-P~Q?+`j(uj!zb4 zwEKe(oV>lo@#Zkfu_Z*bD+$W@C$AKX13jX#Lz&oYI#UnKY`dt!CXmHqX<%nN{CrZr zERkO4KxQSq6_5{zlbPn>@Ls|oun-}h1LP~If)G6cVdzl>L-!+Wk>?eZV;UO09=ki6 z%oWlR6tSM#G$a|&0*}TzC}JL+!&G?9BJa|XaGlU?-|R6W4wZ9?r?mSZI_Oy57H^>@ zL6)dJJL`mn@7$E@ZD9)@TMaK?o~X~n2&HNd3MpH1iJ_Cyyu>|bh-S_+JK`eoY0j|J z4aDJ@3il?oI-+1JRgSESJiaC2t3tL=+c^7dkmNU(j`7MFCWnh%SUK~oxh0jL7P+{h zQ51>TDl(4>JAJs&6R|6pXMdAO^ouV~-H~Y|VjeS|*Gu2!Xx9|CNrs!aFaP-Em;d))|LXt!@ke1D zlW7BC5D10ief{@-=Yv1`gKu@ZJP4NlXf~TYI#cm2p8-s{Bvhsya~Iekm$zB%Sr_=3 zTWHtYXsTIZG7<*zaV(51Z7>xm@7JLSCfZQZZK6l}k+B(~dxv!?Sw`@l`)b2;{m5Q6 z>(n(suI_^)^|cXp4CPzZfe}j2qmdV%gm6L@eLZX_{!-fS-poepCDQhz@*AjKHQ&V^ zxw2$&hdhRBC-WnL#8Hy{@fQyZXXyO+9FKbo4wv=|d3kY_lg1thz?>bC3#O8k2q6te zIJ<1hs&K_RpkCZOV$fQrx1wB%>PFQMGAB|J%Nz4ur-e`-6fZCtvmH|QL3r7|aY7Lw1FSxZujlaVtj2)~idsN1iYw-zx=**` z36C-8wd<{3#pjUM94d_cx%?AsEw9pSl304siEa%mfsfvam@gqbAE0+g>elg0QR z3BklR!k4funm9bncghIW%&oO^Mlp!6!9*@Eb2h9%+DZuBH9VS6^rP`R0*?gdsorOh z=qV3C(OKJoJyxGV=QyEm?~&XUn_EwmL4GFQPmisgbyAw@P@<9uJfc|r8aD+aG0@iD z-qn-susX{74$10mbv~Ut3;wg+sjtm9vq;1S70H!q5H6(UOPBiwA64(DU23hDgDx;4 z_`jVmZ-G8j_;8j^XFA$?1{aJ|fP!JHS>U;C5>#k?S~g9K09qb1g(*?IsB-Yy2NJ>t zVY>+5!_Ei2@WS^jLDF31mTblxA;*{|ILR6GIwGdJRHz73JsrFd3yk=K>=9=`Cty&? zOddBzZ(BZId5{xqh;Cf2}n0mLM8- z5QcPYapDY2VY@6Yjo@025DIpnxot6}c{z4fwgoAn5 zYG@U@EI|i1Bp4KTFs~p9i!8R{y)md7I@SYPFMz2I0s-rXW)C7~}K`RG!?x6?tBJKr8H^$l^ujU2W> z?IS%M3<}^gXSeh1_|5D`KmNtP{MUcAC{*8h8(tW?HcTU~K2g6c^mbC~c1b1Pycz%S zZ~oy2|JjFcKPJ*<)8WC9Zo6b|Zq#Zsnv4VlX|u!VEzC$}Lj}u`wQ!`Q2m%Z4!gF&j znz(X*n%RPvB7}sg?4}zgEwir7gKc~E%yhj1*{&$YluW>gl# zvZ0Mp)Q0C_DKk|d5}sgr&`*_y@X+ja0ahF>vzHQJtrpf1*Bq$Ih<{C~X$k<4dbEK# z?hfYOeC!8h;D;W&tY4HHE7*l{bp4xGpZ@ytFaP6je)i(!>7(JZZU?HasNI6O zJqxH-Q%}s~X^YR(?9QJ&)}0BKmDf;aB(q*w&#ajgDz)!!&EVT3>Hp4Dr8vW``9xg; zd&A36P(?`ax#8(eRHXZ zY_Wg<*WF5=+OU**;J{pe1!!0ij1$-Zzq@|g zHr~yqslp-6CsYw&gXC;Vo9EZIl?1I>aqXAW3FLo}w-kwi!!c`>V*E>`u+Xo1y(kx3 z02?+}3KWL&Cn2_&P&zS9O{wb@U1idqulfZ((Hz19jy_u^_M zTT3-5j{#jX&a3e#g7QSZU=qI)^!-pv$53?%S~Tu!ZL|{xr9xiH_&W09Tig;`F})o* zbzIH<@H3!yo)3e}QjsoFwscNT2ld8!I^`Z&Me&H5TPO3rj45y}d4g?irxd3wD)mNm z?V5rx5P$(^i`q-EkW`;7789V6cMn(NNw-C%ylt37?LNd|)HUS9f?#YBUWi*e68n}M z`+m9}fAY)Q|NdY8+fujm&2PSS04z>E3WRS?1Fu>UF5yoCQ#CKERE3+scXMtzg(#pcfLQbGa_Dj|SIVRp)( zFujv;NL>*C?1XQ(pry4$aIxAD83+gHMmZU3P?rWq7gcj#3_dIBLNNXCJzpA)1e=ez z^S1lC?E$~m5B!xoe%kJL>MiwKp2Awg3* zlJq4!k#qs-MxI@Fj9R5fnv2SA^-h}IX7}W@T>I$e>IH+4$rHFC3SkLTZ5DB;)(15s z>`2ap4<9tz_oeBZ{et?pp)Z9*Y<1U}E^jB>YtU-D(wDA2+itb)*4v3&vWvV)N|R-X zi0&dZg1}E3wO*|-Ev@Q^NVNjqorj*xmbGqs*gh40xeUuOd^9_1j6*i!Riv07?u1yT zs_q-(`(pEc)lQaDGl9*7^q^kuXn+uC&%YH4v$gO<jyc2w5Jon6F&S?p!$O5~tk6aJrZk2@|;#0Rth zIB_gfvXfxA58*5cqs3R`QAi0z+3rtaF&!yW&dUnOZ5b+FAT-Ap9SsBj!b{gf5XbZ22R_y&FBDHVwtNEJ!tJn0}iMe%ddLVI4c`LC_rn*cFK{e$Wb+y^sisqWzx7QksGDD z#>C@6yQM;G%{7GVh5PdEQM#%sN?;roGuRsniu9bPu*IX9I>kdn2&GjL;d}?I0g<{h zsrZXwO{Kj<5dW9R4yA?lpGYO)wiMs)A4QE!m)?9KP*4eoUkQG4bLEv~`@o4rq0?dTM5=$W_!hcB(fY%J2 z#OMFt3)H5Qwma-O=Mw2T`K+6@O(k)XAqW7?mb3}ix5P)LaGVY(f`PyaMFK)T>*^uo zZW5h{=M#DyrB*W1WAE5@6&I;Q?)fbDMmNYsHU?EaMlYxK;Q1{K~f!p=frF*m-3|kIL za;BpE1Igk?v5-Qu97wJf0?ueE??jfp9FK43v+KnY_Rvtw*924|6sbZccPavA`{K&7 zVm{)P;>jE!bcjVyiA`j5GBj`&Mk+->MLx%&Ps@2Z4~BrWMPn-p1B7N&U+f7zM;Lhs ztluA=9iMd?%(I+$c$&yEJpg1to4*)iOK6vhBB&(~<0~SwOphW-6gN7VJ(3KlOz9}+ zLA<|Q-hsZp3v&yq(1F;MB<|HSvr`OGX(#S=;xOAV{A#fqYc)Yd=^z{Lgc?txhR`Mk z)n+nujtq0)-Q79jhw=DEu|7FznA8OsNO*t#@9Ty09Ig=B9yyv*BG(&^kq==Nj5Oo zj(?2+{cyGgovFl-pij4=U~>kWLXRw5>wC#W+aFgjSk9PA^_DN`SZ<94@xHVJm z002M$Nklk;T}^!OT?ss%-z_>h`by?#o~Q>PnXM z^tAQtt>NRdL9cHKXZO4A>&uMe4%kDfQA3A;T2=2|M@j*}1sd8zp6kBJ0-O8%QZ|ePn!f#DgT&m=fWp~ zFZ0dJXA2DsHv20w-qJa9#hYsx5cF)wRU|Y-yxYAugnnsS3{&<(n!Gi$`LU zJtyfZsnIfvF6j;M00UxE5tN4th**hYyIo7CK=BO2xY?v_57a5+z;S4kjOR&XTkmw5>UT#WS zbcY;5a$rf)2OT@(_nDhYse95phJ;tkshzWNnYFVBC2r*@N+|q*_*_3f?#pciJ&Qa| z(g`()kX99VuprFNaHunw{nPivH6$p5EHu0XTeUK-malI&<+sU@4fx(`EWbGn~b-pX#|I+Rco$$;IoQ7K@ZDyWj`2|xcj-(HGa%HZXwmgzfDo&Pp;6sSzeJ@}?NUVEYyBfx zo>QR!QURjuwa_7G3HXM!`6pq%(|7dHNG?s1u_-BG4<(G;h8$XV%IHVFZ?c|vUI^<@ zv*8MRf1ZxBFX@Dqx;$3c4}wc*d;LaZJz>*jsnrE2H9*ijG3E_jRS?}eOB5{%@I9{Y zjQo>A0(3<`yQ@q*GK>%Be&eg{1?(g^=5rS$wgg*yJY21oHbt&PGPOxIfxjArR#gpA zBC^)(jm#?k(BG&@x2SBUWnavM4sMFGiI0_x5ziWp;DWcbiHK89|0s1o2jzv8!%{G< z> zcKPaNTTCxK#IJ}`w{BRprPoYkLzq8eJT>y^aG~U(i>hm=8&o(+Sw=P~zsI?iUcbIN z8&;pbd)ggnUkM>IRlJ&&z9l=PCy3<|%EO7OyL0`hLD};ZJ;HkECe|wb(J}wfMygCa zKgGGE+mH=5_inoz9RvmROJSt^d}x2mFClyIhR)@oTfNWXG{PiV2l$XHa57Acg6cvLn@U$HJ5E743xI* z5dy@zlL1;5uwQ1_F0f`u4>p_%M(WL(00L51<*S`^>*-Xo^^P=Jz~zfe@fX54ps2O5 zw;TKx#YTkI*g%uT1=n#`2QDP$J9sBbPoEO#k3?j6*tLVZ1K(p;2f?>Pr5J{@|i?T#Ilr=ide#=Q^56M^v4`^v(?J1v}2KBQC!VO0H zvp@c(%!QC0B1(!*e+if=IbuK~KD3UqP746JZ+g*$sA4mVA-q&azR>1B-{(M}yHwe4 z9auTIcE-3;@zL1p4~FKV9(uqVOedwk{*Rv-zt;_bjqrl`71*HQKl;T>t+<|^wY5+9 z>f)i>*RgP+-$$)k77LZ4d zg&BTs1*6Mcts-@XGNN5}no}jg;)gLdY)G8An9&@=BGx=rd6kAg!L&zZ7Wz9_m`BnF z2&V@nkaBQe79}?jZ6Y4gEJov7RTxG_rTabL0_R3e4!m+Q)OU=2{<=y`!$Cq&h^76J zJr(g<>Q)4p2!Iw z#`}qrwY_3+cqad-ZORQ4g;CCkd}*DM(h^u7HGADq9nhQ;1%+fbgDi|EPK^^gAgni; z+&U{gTv(5o8%|bp@w9xDlZp3mDpTI*BF`8ZqywKlA58)&$tZwunV0o`>Xj3{iciHN zSU{GJPqpI~i`iiU3

jbQC%p7#LzNn_)BzT#!bf_^Vg{{ZD?;``!1B`-9nfB#GDU zpVaD#M7*AA6!VNzuqOuQ@qjIM$t6TqouYBq-N|_EkvJq+hB*AySBs0w+1XGjUrJ-O zKuy0LP(q~dD%ZLalZI+XvPct?{{cZPD}eo&+AjU? zKYjhj-#&ittTmmRL4(gB87ca7HpEGj6b9uWDCjEcGs($H>rh3xx*JdKMl%W&-llxC zd9%Ms*d{V!z5@)Fj*r;1W&nh9uLl$4J@e{!U}wda)xDv)(KG zUkKmubfUHGTHnZJX<2O+k6YbSiHJ;G&?!LU0U~@n>|)bu_RA+Htfa@#;xcx@)D^6f z-cmFyAgqLxI4kWlT$LI86+i8Usok6z5;GQ284^c6#c}v_R)s>7t8}DP04u9vM~H+O zOGeo#0Ngq#11k{Zqy-+miN=P=Q>>pA^0$`2Iqe51;=l>)cqBRWXav?wTcgu!c-Ksyx^>S4x&=x3My_F;rLQa778O})rsQF`d53sq6bNm~~Pt6h@9 z8r`OJa`IL(y3$lyvWVM&TK~iOt7oDo(=h4-4l5FDd@=YquLa2?wu@ z{1l{Ojux)*5_p=v+Ag&Rc1_hp&QrgA7#JjY06O%M!H)wD;CM`3C{PS%M8jcf)IB<# z2ZNCj!G4BQCWGR=SukIG$TKI_W765U_*Ld6F%N-4p453#fUMysRg4{AjiqqF(?A3s z-6z<&=sN=g8D#%68@;Kz-SpJ({~d{lTwJ3S3!0;{Xu`u(Q?Ayxmsh02MiCnuqXb_9 z);~p3s!f619V5D)am)-r?zN%;r!$SCH^u3l_KAuKbzw_ebT6)JE_WSSnleJOjP0e^ z`Tzu2BIXOT%XC3wdKd8mUZzam5Fko!0Sq-5-+K+YzW(0Ul;=w3!y#s8Vv>&7hm4`Qh$| zt8@yBZ9Z-T`YC$giq#s6<8&mKdXQZgJjL|7))g~WrGSAqhl~HPw&4^xyMUHt0b2<9 zay1GsAG3Urdit{YIz3ID!zijd&Ab8d zXNIr~v|%pl@C3#Q$8^PQo)?Ng`TPyF>$B&N%LQTyn$i%FonBiRj-Sk`X{%E%UtC;x z2Mw-tQw`TJhrW{@jr+!39U9;|AE8fubl_F$cqy%^p9BlyHWO*ptSFRST3oCvy2?4$ zSFPSL4otQw7~ImtvScs_NV&n{F10pH&;uaXifNf?DT2==LeY2NA|u8GIX5(XKG6z6 z`D=`^j__!1YJtqH)8E0FUTm(VDFlF|Af%)SkFZH~;7SVK>}8_})?>tsn}P|Iriq~2 zX^UXhAxZrrw+X?L$+HrY{A%P85l8ZV^|#N@K`c2OZUyPjrnzj!4Z4ZiP`EkbjHHg> z17Te5Q-#->s$i6qfp4Jw3{G}pO`v7kFvhrG=^wQod(6>pqX`o3%_2n&r82NwrSt3i zd(+(;)Ijn@dRWN2;~KW$tgMx@ojgEly43|+8>=IrBy(Vil{cnXnN%+LPFEv-Hb0+m zCJ8slK-~xt4i8n8Xw6`7URY}_cDh6ZzEb7khgCJ2^JQQOa$J1ox`WeE;V=Q2K?HfR zXAMjNJ%T#soyH4xzBtHDU79r%c;%v}bI1=qsC~vphEfPDqPOyqNCf3Iml8VNI zi4=rk45$r*GShs6aY<2Jx{|Bq7wd^-*XiLD@)RtYAA9>{hRNbG z$V4J@t;po00?A`wJd-D6Bd}Yvy%?gN_tf16&h~A}O?|hs5Jt!Jjk#{$4=00^M|3(0 zL>Ag!iu73%%MpJO0F7VW6?bHQ+tbp~Uw-uE@6IRR`_4BM`hEU#>Vs*NY@49S-$3wi z&Luhu5UlXf>kVWJuUF&n7Gcr&qZFuGEEfF$GO=s=lxQL)+v@^QW+wEJq=M}nN{P(n zJn=cCdZZ0TVWFXuw~ai;b>@IW#`wzvTzbc9hcIOdXC(4ll5W6G2AC!MYiR`sL7yh~&Q zbe19CGIybCBNXE$NnL@CN?SR!?O82fUONz2C9n~B06hNTLcxclk65@chtw>a^rI|OT&rc&2h&h z^fp$nC57n%VJ0y3rCKafZz0UYPI9cf(!xO&L`pC1_p_NU^NWo`q3ti}Ja>_spZGD2 z$Qo(@#;K~5w75@C6~_`FVTY|N4Kq24%~zC=P7>=?quXZiEU|juL_>`Jk`h%a#NP;n za`!Oa>~ga7jv6bVXBg)wk%Fl*hS^j^z4;KjvUv_7XYqqSc`vD#gWD9ubei?!E^$2k zsR@sZp~l_@_;j(8^B4959L>%-9>^T=hCa9~q@pu!%uV&I&~BiH^Hmiep!pC>PJF8) z>0J5Z)io**u5$VjC~BAaaMIzkcq1Gn#Etdo(`I|=WjjfwyKxcH$MN-qtg_% zQ7s;hm?(zAyR*GR)6bJQ()(lnOS)ekCm1lflS^Y)O&2OsKZ{f zXjCK2^C1PB#JMXV-7H&8mIAL)y9B5Pbx5}gkboVnm4rW7G*6EYpaSq7~7%icj#z6jKp`pxI<`(a0k#j4JkvO*CjLqPRdP9 zkrdsMDJhH(*lwv$b?j_kyQd>da0Cy8+9k7bv)~8|M-!y=C^;{qXyO36pL&*sgPyR7 z2j)=4u@gH^x|K8~DO2rey*L!Bu$|yjiXF3VH=I@abq7l+0b&Qf3iBL9Z}McrwWZ36 z_y+QDOgytxD@tCIRq|__LceC|B9~zpv=L%6g{TAovYRNtB<+HUtRxrsaD>9hHMLtF zCM^-}C+JBs`Y`-T&J;;1ctiB3U1%UOkPy%;LUwBoQL!ae-vmD(1)TrHD1|oOgk*^DjwLctIB%3Uw6IZiDso`e}Rx-*Ob<;$~6fk(Hyyt5{|cArx!KIE$Kj z39O(#CDuzxKbwh5^Q{b;Wl^kP=*4OxbB=KdAQo%#X616HTRT3D@b_ka1s#a_GQtynbKBEXo-e&=$Lh9O?YKt5Z_p^_lnc+%>v*@tbsQ&!RHyG8&r^f)9syqwGrE|mpp%=wsF7^40XJk`+ z1vDm&v%i_lzWc$W_ulDqN3p|k`;IYp!BS&oIJ~T}a|4j^qOn<(UaLLSklVrL_2V{i z#oP%|=L=Q$$>@jGVN*8v4sAjU=F0kS2+QtD1N zCDo+;j)O-L9(y1@%&cb}!aWYSIme_Rm7-=weOO#J7+z%BmbA=)a8@8n;ta;4C$iV2HE`czll4GTzLIs8e12i|f&v9W$gQ&-p3 z;X!z&H3Q?6qDVUub`6U55@;u>XMWAul$WGTKtav@t-u-^EotOwU`zH{yiW?7fPz+2 z?oagSfV5gtV&Vl?kO46|^#S~`JBiIWJXbgrpT#F4Ai#^O^c$%)P5~~PqWmb1T&C_o zei5#;Ei`M%Ggxc0g)>%I<|jCx3;f#Vi1e^8T2r-KNERFAG`f>kkT+vWVgsFEIwudK zPDPEzd@A~c!P4Olzsx54xV;0GA!s6IdS#8hYwDbq< zT)!=(=gC0@`4d{zU<0MPvowq&PbwoZf_sk|rY0n+$zh2Z++(WKrY@lhrT}tq$U>`F zWJW?1{Gc2GXh^dnER;=V)!iB$w5*Z^WZdnsZE+J3mM%wlMGXL|j~G4U6iW9%2WpOr zklNaC!cA%I_4#ntllWLiGTZ1XYtt6$X+9qtJCA0FEPZPIvhAz@VGcXP;#JMB94 zXYkcjCbR<#qzxsCGF3!M(7!t#&cJ+}j$M_i(p6b78YwJ96lE46j^U^@H{V#zyz9*w z>d~C$Vb!Q=gG%L6d~6OEJ)PjXomLmlaCoJT1ndIqg0A7p6m`Q^ z;y_xjEjJ1VVTFRklgou_Ok6goG{bbHWIV7X3yp|9ib*(_ZsBrJYZ6$&A~rc4JLP0( zT-XLhKrEJ4^sUkl{*!leO%ouqwk;SY1_oE*{BSo(IahdAR+S;^2;w*%6Go~jq}r1UAH3+Vq%n@E2@v5NZxBF6#|#f#jV0KX|zB2 z{qy0O8@e!#i5(S-i$9cACDWe1CV>ee?DUt*v{&}g(VHyClO&j}Lf#&TL$IJu+@-km z@P;WbtKQXbII}O`Vt6*_UDGO{AS%i-S${0K*k!kY_AZJsBz)_eoIWRBREHtWh zi3=0kvF)rtRx$=iSW6ofLFAE+(mMYX>&@Qz$uS+}uArbfB?wf1j*`cKr&3-DEAnJg z%@Izg6sIExWW3`60vd8Gtw2l#%(}6PmYFS4gI=BGE`aJ`S?RW&fnH51a*fogD6A1* zItv0SbWw!OJPh}RedcmJnS+6((}UlIKD`hk*+$q;ClTn$^iZ{$wC1|yG;l%kC%}0>hPHt)?Vg)IZJOBj5V~`Kcr(^ZS2^--V zsv!EgPFGi#=z$e>_;B_&;lGyYbqCPWCX12|=VX`EMA~^KZu1VQSpp*H8KDR+VrR-H zuE$g=QCO}u*F)ygGsgL^kC%iv^Lne%m`DfZ~a{&IM=<)>C7h*pn;}|thjr& zAd*bW<~HQ#tQ?A-R@FDSvQ{7>(Ll@X98C$7`FE$pRc49(viK1?Emy)F$U4LdFj_3n zQVM-^d|dv^zxv7P@Z^tw`4FZJgJJL84{QVgCegonjYh}G9A*-=c?duK*Y_?1ST{gkf684VtU2FH{1Je zUptblQE$fa+3u4 zzR4;BI3Da#l;=j^%~n0=+7vYK}@h_lj;n5P(67vPL$3dAotYiDP&k zR=2}UCX=k1q0{Q{2w;)_bQt7kRHmxOg%T*IYp^7=pEv z{os35s!$+Btr_78{LL0hziP85C~yVrC-G-Wn`VnqOSd_Rh@?>xTDWsKEpkLbwIX3h zif_${3Hd0>%yF}JNg~60I_q%snczB3I(B>1r00WP`F2!+8ym%=7=kQjijYl5BdsYS zOwl%LEwS#=v>P7H0(#+7Mxfm+kW~DS|I5F5`tFnOz4z?NY3I$=;%|TUn{iA{#`z_A zleh*N32OZC!+yV`A4H?qX=@P9_X-Y>Q0eLrm#O9`#dMf-T~We;L_$v)#XtdU2D;A# zr+lAu?e>0Vf|SsgO$0PR#PsQ$IT5v2Zp6!#K^Y7Gs*1|Vdl!m*ulyv@> zj3M)06ig}n3wWt>1#vSHdR%}8BU(qxmo*t|N@_FUE z0qJ$@g6qM$iRtV#Td$4`%%?Txgl&eOMZGcw(#j%u={Bu8TxRVg?qJHOa7uznSrZlh zk``diR#M2>QjG%OthWF;UUf*f?X<_-KEls9qJnC z(sq19it^+lPkN{BIi}H;PbXkVoppBZqhEFiU?^+i<3a@M6MSIB(_+bvF~#s32eI$A zhSei?lcG+J$DolqfglBS=a>SXe@1fxo+2A40W}-=FBr-7afoTgGeQr-N-o-0HKF^M z6J|y9<_O7$CgAV{#6$bS91*)UhFK4DFX4#%lYjgc)77jSYl8+iz^es5p#lU8LPowX zFf4Hcyb+}C?f+H6m332=7~>p^y4u`J5w^3B~B zmp8p0W1EK+t7*K#73;3wZYG=Lk|uASOp2|IFBP|(KRLe|P3+M3zWv>|&xb}?h-L(0 zSCg7A5n8}Y9+ojVXM7GYltd>iQXllBfqK@`;SjGRSw(f|9o!s9uA|St&Upl}G8_O` z;4gGnt3SvQhESnwo2~2uH|rcD9opTF`wrIz!zMCzJd(I@&76I=#K1zi2e#B^5{g9G z-Wr?@2R0oA5?C)N61w;wur=r?h$SJT_hrUGt&&bnxncCvy^g-or1E}S;;@BbGS!nQPc{J4L7E}_RoL{R1bV_jqWay$+cu*RE*eqX#3%8Pj;!b?|eYrAVJ3Nb0C zTs|2NOtf|4QUx9XIslj&3w|_7v1atyGy`IEnIS>7mtfq=4PY<2L`$ICy*x{zU*n|p z*6<^#36{Q-&txF<6kT0iiB9Ad<$qdS%y^=|%g#V?L6pC!V4yd6GB`eS9eka;BKa=Z zo14gcB!F+#nr@X;Rm!z;&B##GKghM$R27V}j zv$szBz3K-azCG;Jr(+6J=NlqFa>1Uy-T7yK{vD|`6><`)V2itE8F&SGG51>Hi~(jJ zMc5)&a}EStnPipe2oz>b_(1(Uy3Zr1ClyPD#ZI~ePh>Srt#tV~t;E>)fuLoEiV&fs zi-}^m{DyGdq{9Pu?Ra6X_&3)hKvcto<-_7&Tf|;!{4-6Z)sl;)zIDW5HYKynQ5q0Q z2$IqGhv(KU!VFi}FC`;n$fOPT(?eyH34o#dZB|paaIt}-kz^wIX>2Ih;wWM)gb=9< z2t_FTW#w%6VRbkT+)rD}98GcNUf9ul~a{@tOdrD z1!pv@o|uN1NYX7O{HgFy{?HVP)b`iADF7n|Q{5O_kwU7FK$Nkrdq&{!r@qX5LYA0* zU@$5AUU*3nJ47u<@8pO>JY!AnfE=DBsBuxcOUTHTdxPOy z#PN%TwsB*I*cHoEN1!!efVLKfS~@DoJ}DATSf#$Y9oea50+SuK-DP1}i3Ea(AnwYY zU@vfj^^RE>@4olWTTjm3|K{^2PfiUsESHuy7mKSq)%}HbwfN`%={xOK;rjB{(t6TJ z7B+K)5=VwaR_5Yn1FOucNX4}a7|+$PahV+FKr$*;$=DufTU)bm-O8Fc4!2rz(_D zk(o}wc}RJ=wz=DW5sRS$_*8$MQh^m-9aAA0qbkM2O4IrCGQmu!NN}qSR>2ITyRka- zBKqByvo(yqbKicDsXvuWvP~n zqmvV@EZeSbw~haSp_tROeDhX^4Rdw$IssguX4EjCMh6L$=Jov&9lKkMl2Pk=mF-=d>B znF`SyR|ckLVX#@EL?~bE5sw_wM7~z*F@pl)FI*KX?fY{Py-mvZb!iRuvhs`;fm$>$ z5%KyQ6nupRl25bq-c*}VlOJf?la&#!C;6=^d){cAcKeSWJ(|~=KewBAvq|L+k8uEZ z6s+3f~y$4u6Xe|YohOVX;e6ic`_Ts&bowF00B;srUufFD$K$*pdv6A`_y$s zuLy*ymzPbJkpJ5C!#tUBf`sy6$zevf0{+#iT~qB!yE&VM7M2xE0wa}ax!}52M7rVr za|Fo2xTstQ5f1o4a_+-XRtP}62%G2gS7K5>0?Kam@W-kFyw}ybb_rdsvsxY~R?)CR zDU-&&xIp&>x@Nnwfk`#{E-#4V^lz9ipP4Y%yCzh?jVb1`G6^DtWjhWw&#Sv02NV$@}u| z`X(_%wRQdGHNawFlD*W4cQffwib+*$cMVoYiw)6#fx;Gj7=`n6^p(EG;(rTlBd{{w zvP#6-=2M(0a$%}V1ZoQ4CG@CT@h-sNM!qJKfLw(gO`aN~$P&aT%T1F()h@HzSWFyj zTVLp&8al+qC0(RXl^Tfx#20vOu@5tkAI9COJ@O*zjGZ(| zl#@rpM(r^*%V>0C6qN6B$i}7~Yy^QZ+Uv#K+k#qN$}tu2GT;yu->@sM`h!RN&&ikS zP|rfy&AQ$_KYjY_+aJFFlV5(Os_@~_i8J3kJ?@taOLwAN>q@;Y_qFTW36fkgr<%T! zux`2AR<{7|sT&uWoL~6t2})4>0%Q7M!pkY$O&9vq(P5E-za;k=L`xq*x} zf|3Vgjh*-UM~@#rzPr1U zy*z*GZJITd$iYuLQ(^~0*i}yH4oT!TF%rYji_?Bn0q|;GUCc|P*}UBv@KNXmr3Z?4 zE1+SgMeHp8(Ci=WD0Xjj8^^~ji#M80nGGxKnUl$8XR^g~rZXNNs^eHt|6ESJ6E(h9 z&>YD8FqpYnFZKsCJTs=V)#w&fdkLXeY1$x$QF2Ip*=WK2kb?yY5xHATq~~1v@N|Af zD3ELr^T_ydT{0f1XWg12OnaEiej|P!#*=9WEJ(Umwsh$l2>D{=twgehO1MZ+lb9F0 z8X(8SnvcU2C^!l!oL-u+6Y%EJ><&7+oCOBA);xax?)%$S>*K%w{ARY+FaczvP>OaQ z+g0@-a^ly!(EubxPQ{y)~lKs#TAVGeXU%92Q6}*|-zKtA>z-LSL=;Rh~jM2)|Bb zXhGL{$9e$+l0`jWl}J*x7PyE{O>A!=`5Dc1IQ7{fEFT-4^XN1ilZNyKqN!q6fG|X!+ zwun11XCY+_?E#fdurj`k1%X}o@`J5*yK+u+%qln9t?qI$k)2-`?#vobvIj6V2m`=R zAk^sh9(Q_!>1iT(qUjtrufh&gInmf%W|Sv#+LiV~_m7cfRGiuBVgv7!+?FzjfXp4AC)U0FDO> z8^w6(L6BWg8oyrMUA-BNZeL%#xk2{x;sOn$RPeO`N(T`yiaQblaUX51Q-`bokFcGM z_|Ad}0Aiw}MPV}o^=Y%^-=Vq#AUs;s?nSnR$;Ff<7Kh$H^r~fT9hJt`R4Kq0@?BCv zxFA7?`8KE;mQ;ST^k4kb50ry0ruJ21;7U`+Rw+|is-C2ZyaLo^x=Gr*3#9CaQ=#py zZdY!XfZy87{xj8bHM|@QEZRUD`eQ!!>n2&SU%%HpJ$+m$4OXP&#mXl?|GTfQM+7c{ zBH0XVPg@M#Oeg(h<8TBzv5-3`h!U@I|17Gaq+0P*bwxeS`Fr2}@Y%B`1e`qd@X;d_ z^l`U2Jod-s&Ek&9*Ip@P8>B9v7B0jHB_oUuPoXRIZ(bUn)~o;Zf=K?shb~d-K)xm#;4( zAu3g$arog#J+e$hs7N2F#=70^xfEh@+A;yVNp@^6gQt`ATCW#N z1G$3K>;@Vsm|pfj8Kf9_zpx63wA?EqMq*lyC74wum1C;4@O~zhyF=r`pydWRt8>hW zL>j(7KY9CXa6Y=8-CSQ;4#VyQ_2*{;M`AQHp}tR-1cymFV3yr#=vvI+kSjQ{!5G@Y z{Q_%g^bO`8P+2W!Kl!^Kf9pHnXw_Pyo7=aaJ~?~#!JqwL{o>P42;8P;ov*&UyxS~< zG_w8ZK^Fkoj3#lDnS+EdvcdP|-td@hesi&%Otr|+a03XBLP~D}jdA5Gp)M5;=|ylD zjQ|dWu93#3uv?VbN_r96C8_R6ERu+zKBV4{I43TgH>4|ZGh>t^H6w4a-qY=>)8i)V z$mlBRP_;3cFKDq-q~^Y=uG#JF_kp}}%H2&}ol*wDsVUKqmEgS>OH7Z;#&mHq2!@hb z;aFbm8-TSt(tbFYS#>SH=NCkoP+SWWBjd>fYWP0W8(1hNLo^dvv`n!ys zBj#nTOQSJx)PP#A6fMt065i3Qx8yKGcWXM9s%7Dl<0Luz@W%8CQHg6`NZ)bj?BTvQ z=$}36E!nI;_8bRpNk=% zz5Bt-&%OwvtBM0loqkUjP+X-SyQgQqn_Z5ls;szNo-k{a`h)#MGcx38sjQ^PfUU!5 zs)_(aJkFdwl$Sc%D~VvTvqVC2wI8;Aj#FF(t&U3I_T>` zfannfI1GVc(ZV5m{ElPT9+0T-QkWbPzS61h&BVkM1#d<4FMioReee9y+1u4(_p8sZ zUcY+t{MqpItpDQEFZ@uq*Sxq&m-wjZI1_}bnlysVT2pPi9C41EctwDbw1nyHI)`N1 zI6axy_~}QV{NVPs`}p~CdKDE57QEko@7w!e+aRevIRx+rUivoOFXt z^yLboc~%}dYIUa}@FYqW@+S0&fP6~F;XmaP()XgGk-vB;GO|h$n{TJQre-gBZk;EY zfNIA&a2z-pxgI^sd?~&^P3gT8_6#+O@~Twk)2Tr#o+b4PEX=*`0{Vc$@Qz*tN~QtT z!3)_64bGuc6rNrd>5EfOm28Ar;48!JMGK3K0S>+dd6gQ@dcY%BcUk7fFd3ttoeA;F`u5Wm-%xbtBC3_r$ zC|ZdlOo$}lv`xzSA)qqlzk`U?Sugs7=6YUu_q}tE_UUIoA9g$My!*7%?P}Y|i#wl` zJ}>^^Rk-2wr&+?uy?*D(*-)GOZl~5iZg++!z2RBse z#Fi0)N2Sj2*hK4x)iq`j>1)j7#7&|}qj{Z~H zSd<1ydS-wyzzv~M-WE`4I-|%#n6ZVK#>-e~nZOixU_N+&TsFi5-htylGVj)z1|#>H zTPl5ykdBNLR)y3s0t}ldjssTTUha{iMSIOz`Wt~5UN`X4r=tdpnbXn(_I4ayQ%)&U zZdoIr7#m>|Jk!^TCmW$GNbJO6Vpk-rZLknPlMr`GwWAU%8Bw<~ZT=k&jYJMKAyP3) z5mXiPiUi}j&|h#dv79Ohd;d_Jju*E#QyHyBy&oyqL&tT(-S2nfxTJeWJQkWi<30n3 zpgIi%h$z?(|E&j-f=Ov{p-oFs0x_*QlXto9zAbqa~&>N+To9j=0{p;iY z$!~h~w;vC``|WS`pFVG$JjKa~$Dh4hIvxzVtzW!&@p2v2N_}d^%|c>ELDZOn$&qoX zJpjaGQ$Yth681@N^yMyN19EXh(ke|xcga=Qa~Wi)R!+s7nzbtv#WoqFRwMM%!r;m$wqJr zmfqb6b-x57qwF%X#@oZ&!a=C_>G|^yKm0>|jDGs#AI%pd{U%gMytx=%UrpXz&feT` zxifwXWCEXPOy=`Q2RYN_`W8P0lX9Vk8c|tX-pX?*>lZO3D4kBmw-apu)b&g?dR;8{ zpZxtVf9Fqrzt$Yy7hkwqXO9NU=BhT4JHdp7_lFL?&OO6FcC$-8(3OfSr`_9#WYkbD z+GuB%=vEI3Vg(6!>4`KW@F@XFBAQ5JH3><+#4q^;Gsc~M(Yo*w>tiDw6tQoTz44h) zXUf(D*f?iEr{Un~nBuYmuRsFOyWJW9FIwOBWXL^Y{pGLd9qzNUy=y13EuchD^eYP;)=MlbnW!eipZiI&x z*57o>MfcE`6o^)o*g?UJb%4dpZyJrR4;ip6TK43$Q>$5$V1(Ew!``Tv#u=6&`|EpHVP`VkjTGMOc7|*D4r2BJ>5oBYZKDBoY?M0MNK2dzwWd$Ijb!ErLnN>n z(L@&aH8a(v^-h|D?l-@C+&O*z@>jne-;I&v(jL>>2{GD(uE0DRZH=6^H3!PwMHVW# zPgQ2O^l^x2?Q&K1AQ5OYERRB)t(Ms}YM4%q4sryg-?u6!+C|FGZ(Gzpb0Mpu`XsPq zd9D-t zD~#q#t){sbzW@2-56({C`|9P@Pk!{%?dHw_2*HcZ4G}$qek0gxz!}D`3aU-7dAcV^?eNQ!0~Pns_lf+blgui<06iST5h2)P{C` z2WeCia_M7%oaE2NK#Fn_>^Y`M*D@j4nZ&*>pv->QfY8ui%MuPZ^8k@EKV4<*uzzen zp@99|r_E))39XXbp*ZFFSWwkAK;r;jK%u|t=ukso(v^G(_iTKH&m<+uyGtKHsvIEL zFV48Vyqk_?p?GucB(=u7L)~59U8%hCfr3x>gxGjCRgsP-J;aF{ZODUB0do0OgpZH~ za(?{ev_C*3naGNdL}h4BXLc9ctuK{d?r)f{EM~KT112$LkVLk}$H$}5;^NKdYIY?Q z5=V=;#4Fk_-WS(b*Ei*I*WOJtvT>(j<9vPF*|Y_CtQBW(o+88Q$o6|GKU5Zy2a_qJ zPRW;*n&ws`TbfUrC+|G}LuUhR3;+N?07*naRH?#OUwnLh{l%--pMUk{D*c;}n$j_` zdY03+vrqof52M|%9hKq|OQII6Bj#@?dZGMa+5jt61od<_p~>Bup{u!U=IIb(XMMqb(tbu&?E+wHedrZZZUzu(LA6r!;-Dpu zoN+m90h-j`kn|}nrwbW{(+}Ps4BmbDn>T;)fBx%QyZl?&dsrlKPuTAJ*pp_4n=Ob`bZtL5f}Lk!?mN(u|53XTnUiQIro z(KT>?Z?+pOvEEJZrt`_v%DSBU)rQ+dqj?IZUNB|u^{vO}*Vos~jVo^b=`vgwmzry$ zg@}#F9a*PkL}sbz4ABZ2WEjA+W=j%CZHw>4h)UE7@eC;AbHN0+a6=1~3e`lW2ly8l z%U&qF(fz5>$}yrk;5)aIJA|DN7kqYz!Qjr_vR#0zYL<%IY_Xv4jQ;*7v zi@O&uUR+#FW)m(o{uenr81`$;;=^eFR`LAFThFBk%y_+7+`pMlIoe|x9%(SGBX|3o z>Fjc3mb#9I)r$!=T>a_Ow~1;Qj; zwsXIzY|GOH+dj_wB2IFg3FgTacTPK0Qoi?jN*A?Sq62VCHB!d*5QC7=S`KshIuSia zX<%}$M5j(~%&M4`a)y0)m>JBwZWPKh4GwB2r;po>lb`phSXr3)&MWU#Z6e><;Bf&bdJPH*^F3Mn49vi(+7pGpMUc6di$Z-?vg8SnQ2-dCObS^ zvSZ!{`vDX=)*W_H7$jl|(Sr{uuNEVxfm(_oOC7HRuVv?S0Z0kesJSD@R_e;9SqUGU zHZYBJcd3|Z(FQFtRSKHHi;_cw{!?)Q-q4*}3j*eE@qPzvaqhsy1kkH}0? ztMjmJw;v2_9u^8%hBuP5b_;&2mNpZewRR6o$zn1UUo=R11Bng}vfeamCt(J08+jq9 zgOe5IqevNAfMljCm#A_lWu*V}ZZ@BkDl6%K4#tfI>@Y5eo>b}%CZ zwd+h9S;`Q0IvuW#2NQIA5IR0@C{rE7#x1c)jpmS;Mq?>y$xgd}JYcPFuSd7qLlrAy zZI0I~eHMquJvysg29Za(MuNltm<*7{s@t{hK7O$(a}^mGO+k6=uEwPq0wRo7&+aG9 zT!9yOl3t?+-8%Y%N3Gnk{k`LkCm-L9ECtfd-$zuW*?T2km+4@s^g8B%HE3j3b#6fS ziQPxzo4GW3tDTnp#0ciSL2juI7)Z%5NikxSj60t`dD1(1eExhgp5ES$klv#2o2whY zFUqwY4vP1X!07g&ljI}Bp-|;S^WVG4n+vu)z}THi-mpy1Wc!w#MpaH-)}h^QrU|*I zQd68(`|Rw*dPZFDg-zV``NTFrBWamoi)>f%Yc9;LOd#*V9v2T)9LZWNcofO|4puQ)n? z(mn3C&IU@|KE0?f9=(13AWgL~U`Ktm2F>@)Qs?Qjb4GYJd3AgBsZ!u&GrX7Xq~HtiYncx@aK>rBoh(P;RRv}dC2>u#i$bo|Sim9V7oR51 z5~<5QNGYbpk)e=cB_*nj+J=Cr$g1mlZNSmSR`z@SC@|JjrX>hLV=9gQh>25-YZXyT`{}Ce|obFeBx=mAV1;KnPgR z;|V5UJXVt*xVc7=?YVeH#dT%8B?eKvC@-V~KHY9<%iR>-^P8uS zpR`)NL!3xw1+*t(1e6YjDKyUA?MyARexL$o|7a&BiB_E=N^)fQ{Vn~Z5+hnTbDHbr&t810+*)yQ z_w;GkHuR3!QV!J31wJ8LQct%XjXHo(3;=Gm(rC1B)E}NMRMWPWyNgS$v+zySA;aMb zsRQFBVcj;Bez&}BV4*9&{`D8P*OPFng$Jkn^;fS{QTjAfq_L)2@=2rVLmU~cqE?}} z)KqaZnkYh9HXA+HNphT^(UD-oTpM{6izUX>HUk05Ei!9x*UiZI#g)72T(W%`93V)I z4A*Sj%6t{?hZzr@Jyw$iJd`f4?wAn@s?lit)mImve0nh$3_kvcFEore9t=tJPLJgc zglLJl4S|1Vn2CCZ>@)|x?g!s`_T;TWzf-%pnZJH@*`rze|ERjp zC)?8NJnWs*>65zqbh>@xWDG!%040E=NkI}tkd{OjB~{twKf!;J%AahP%?~bC$(E{E zW)(q#0U$vTVFG6E%+2?9JUMo5|DN5DtK2gfbU0_9z2Em;=~>TOivZ~!b#Ym3tgAjV zJ@RxsU9IeH-+60r=V<@sFZSPjbi6cNS*i#NPYo^<=}j+NL%59iN%1ah3P7r^g_DXt zi=w1=I|Ln%A_`{y0<8!ps}Y+b?W@ayuo*JP3_vyPmrob>TNeMD8=Uq@;H+#C7sMiA zyHHWRUV=1^i4OTi1L-~;yME%NP&~;bWtM$9%$?QM;;G;kOU;pI( zdk^3J@c!DWzpPNmz=e$cP5{L5ZQHGkCKowK0g?64N#>+SuSOC_OKN=CAsKl=8X4$x zdOTYoIuBraUf0?i75>MatAF*czG=AXCm(+aWj(#vG&*H98rv^j$A+>|R-0pa|zds=~62x@_`L`xvY(XxVUBl4Q= z2+SBUu!6#R!x|dfd-aWqBP%qu6(J&EvIvxsP;>A8yKC#)jP{JKk*P8<(bn^a%PnC8NYxBklLKcI zzaN;~ZELnS-dNvd5~V9JepXkn%Cif#dIMr2FY)~Oa~0y|#owrSx#AQ|j*efQv^ogB z3elU#M-1ymt>kG@zhnZ4B?>y)5R=Qd#WrwH$Z7PCTKQrV4-?S++D4N8DC{oeTXvGnhT5{~lH%!9r4a(n(*JUBr{m#E+-j{ADs zrQga8P7MEJxZgWFKkf}n?|xuiH{QH@LaKM~-oIFws3y5;c311GhF4y$jGN7uC&zAK zw%*u=y}%Rmot4A2_B^^Hw)%$-tv84>;xp}EI(?C=Hbw?!U+cev1%1)% zU3~4sd+f`z=ZDA5)31N?-ar4&w>P%7zWmuo?Stm(YF)A!fU~rke1i(3R;LB@syW8y~gg{a=pRHRQ=!T^44yM!QTBxckbPfq%CO#T63PM?SuW};C^bsRP2WY ztITx8mQtuC3JOhK=E_XblcNs36k;6bxya3k;<&D?oF6G~+sJD3avF^_Gf4^Z+L{h` zRSMX00zd8L)YYiY2MH2OzVe|q$ zKgFV2Mpz@3t5RAa7!}k1!(oJ00cqB=GtfxK9zTM-KG05msHmF2APbg zy99dg==>NRRM#>t#N`PzQLqAHHU_ErP;NDa89Ls{H!HdX7QjOj>zJ*&!a zf(PsA$sd7XK&DD8q$!UQw_};R(>aXt8zZnd4GZPPQIFK=#ehr~jsdoYG?vP?a4+pU z=S4}8&cpm3J!fj`D^vX|5e~arThs0EdT8Y(U(ZGhwT*IZ4H*6GY4hVxp8W8Ke-Dx0 zy0^KrN9$GlfX~4g;54}CcY7y|#-_YKrJjuxkf+l>WFojUD8w&m0+!G|r8;YV5#?*W zE{{3SJH?h-U0IZ2q9f2g+MWJ84{K2F*KfLSTK$d2UCc1EnnfWuSEGJs-{4D+hP?+V znOYpKt*Hd6U0iptE>=oc^|kdgQ~Cz1Lh=%YMneZ4-eX{X1~`FhNePs2E^Ax)u=MzrBs5@#6VY+X>zM^0WQT`ufh++l~4T z!p(5d91qMho@&zxlg*A`#>8+di2iG=$26`jnf+YI&Q|8&6sp^f}vwz?N&;4KA?*Lb7G^ zlTiU;N9W*IfdmFKpZwyJPd<=w zq(0M#!N)lntK~Y1CQozFSFU~@qPkkmvGHcYoU|L%Rm_;Hr4%I?bIWUOiy6^O*n(VS zJceYxc>1(^B0OpJTI~!o^Fhi4W-mAD8|z9{7Y&#m0C?bxzW3~t$$_J08nDbYZjl0? zcJl~3RcJx+FaaVm%Z^mR(7F?GCa%|=-8+KYFyf|79?>y;9LAjK2PFe@igGi|0X3=; z%cQ+KK*?~2a1|L}r${Bk*3yo&DT^wRnMiG}rF8~#}yOQI2bv9e4 zg8|q}l^^F2tY9k80+@8f%BisGPq0_5Tdsf&cu|AOGT`nKd55?)n{N2&X4%=I#xkbY!(|B#X2lGfRcGkgN&E4aCz^d$x7U^y>fH2V222`h z56&jt>*=ySv-nYH^TJk+gYlpMeKpBnCusj|m`3 z=|-3lj?06|1#rXAfzIHVH;x(#UkRG#kWQesS28)3>V+O4y0(~zM&;1oPS4T7VcPah zu1=2Ixa}(|r`_SFr0nhY-hK4W+wxz;_2A9^(=VQ?)qojJFbTv*G;kUioUT?%d%GK( z4L!NFw9zA>7asRucZ?Xn1_cd%yJ2bPmr`2n20q-`!uq9}-%XS1KE8 z>zgW5WoA@YqA%;oBc9gyM#g?{sz050Awv;Cj!cQoF7ht3m?v*gF`LF$IsxjM2W`U>K7*(!Wzz?QuP(-O?%X_~$7M)c3pF-j5}bFm zgG0tiCvCly=2rUW)LHrZYGqMNViQ;<7qrm?BpZdo6{7mQO${MF9LfmlGuUaIfsUB$ z*SdzSYpi@WeF{V}unfThVN`fwu$fuGczjqh$x*?y6rqoXoik{(-NYh%qM91|?6dv_n*|4Mm%^ZEC_|2O~p|Mv8se(Tp7Ys*s+ zY0Va-*_L|UV`sxbDwPr!Y|nA;rjL44=o%75Qz!m-wxsL>LyQ1&#AgSX6b#pOPbz&NWsjG5VQy}zOA;%qXlDwVHN_+^e9j*DX5DIPlL z8&;9di_);ZC?vev)2o*Uqh|-5I~9c| z2$LVvyCnnWfGr#n2cTf+rt&SsXaWj`0e%2WLp@bT40*z~PQ{=!=pz58i*a{3&lMky zhQI#@zx&ps`#iMc!~LK9!;jD;$a#R%)F7n_3>=Ts(dkH5>!)BcPDQ(NF4S-^=C;9N zeLg>30i`%qbPRp+h{6wFrVX&DDPRcr&J_h7`08gAquExeoG`Q{C6yK!-DCgN%h%7p zlx7BlNj%nkxiB|9$a%JdjsFb3n z0Pg+^Bo#~P`=vyUlPO;*=2ydx`y6(MSKw8sZv%zIoCBeR06mS?JKYYYizS>A1wF^o z@(%lPcyw^`;>(wR`@}$J7)9Njxc<$K=_U2eJijyH>4A zMY3+#eu*EU$a20qyc5Nde*38qPU8~5N2sK zNIl2Q5rC{(7b`>D0&Qs0g5)aqm@ZNk{4j0@_eB{&;*42+orF&S55_YE2@yn`?vdKD zlzIq)F=L#Dr8wph{BHNOH$2_k+2#RzaLM_#jqL~b*AAY4^tXTZF(H|nD{bx7E7hs% z@sX6q*+?e`9oVY8GAst+03^)%aBoh}M~3eqDk`crn2thslx&JSrcaZH(%eZOiLi62$<0E}sG!llEuWnwIFUx!s`Jv>s&`P9w zXgRf2d{9oNRu~K;y%~(IQn@%CdL0V7@Q-3u%?Q{aKMR%0s5dCr7Hjo&nA>m&7Atc@ zfO~HQSZsBaHDcOLC*ZOCl3 zS|?o$54{F>*E6N9tvY54OjGGMqbe}dpph2xSzG`g>K?9oy+wrf)zlwQwEG!M}SEtpKBv$2&v4!DGuDKRbu5ubgf`_fS1*SPLxOe{L z%ih6JU-`Yfx+4w!!p1Q9+1_(gQKInRb?Jm4z^B(pLO-DDabQHTWcS0~tbZrp66;npwha+d~{JhF6%uu9!G2Vvayl^nW zOpy4YiOpvqFCa$USyQ$lrVjocf;_bl6iVSCvNcQ@R*;MGjEKL0K<^SHAzQO|T2VX;=PH`Z2~M>n0ekss}ARs1}?*|H7-=)a3Tq(Tra zJogSC5D}NrOY>Lx_p5obz&`A1rPIEP=Qc?v>qQqtNmMV2z)oE}r-G&D!7#N@7 zl!V2%syRxU{u(-`jyaiZAyC`a>Xehslj_{O95sQk4jgD!b(}P;oZU zvT8N=Htvs)kDc{yzjc(j!RgQ%_{!jciJl87%hW$dqF2mSV+g;3=q6B7(6yJUAI{2(Fe%1#b~hE*Q>nATYWka|v#SgI<>{P+h>? zGL+`Lab7-Y^LROve56soujES}9CJa&7Irbl*Ck#V;E`yY(lNfm(X41AGlyn5)>a5* z+GdJKPcd!H$aLHWvX?W9?d&?0h#yz4N`lV{9N7E%R26Q44yuPO_RNv8%T7;WDWldM z^wqWWM^m4E_UfqB-Me?^{)6q>s)C^N#Z1N0SX#uD_AgvYCL`)VDLNZAJp*(*8A$w! zbtvM6vwjz?p4Q?3%hAj(FTDRNcf_Bce)5-z+LXA3k~Z(t1kTo2Ef={Pd&8`};@R+m(8C z?cw&0Gg7KAZQWgK?T;&E8#6jNp285kb7yPN$-vXSo$bYybfc+kZf_oZ_Q@B|E^o?K z7@UM(af3`gcS<3|Ka}*PS2NgER&v4fQHyD}tzM;=GZyl?0u%+*NToAY{(Sbkzw-|7 zY<;t_TCUPszW9ozG_pWYz!KGPA>GSC5cCtLCc-;2)9jA_?&D^AINn@OTUZSx27Lv2 zlV-E!qS0NDGSXJ(cEFamTKw9&JuAScNu?e@t@lZxpLM;%^u%q{0R z^drZsRH9gQ_?_3{h^?j$o;-f?^N)W1>c#VZx8G^;22VY(K%UCmXslz0%0v@*c{8>G zH^6GiTqA-NNN^ZPA=6(e!Zd*QrrT+yS%n7M;1-jw&rFwx@9~6E%$0WF>4>H}>3X1I zU*j|DHa{o5aMWVeXO7#Q&mSM)@$B7I1fUU_W-S?_NE<@ytfX8=4!9EA3HB5}FwA@v zffR~jaB9e!(&(H>&K?Jqi%)M7TU5fdr&+)oSS5Q}m}$_;&7~pjp%fVQu_Kx28ev*K zQJV{W3tzIlWn3QR1ZkUuAWv*6v(Py3a=k zKRr8!c&4(5(NDiPA&r>S-Qd0=ajE3w^p%nYLrN9~>1eu^o{Ja-wpZ|E7`+sP(4>(~rE4PMq*Uqn)wV`oq-3)Tx`kmr#xnzkHz#Bqf^ynemU zqC@=b1IZM($`k5kp&jD~UwL%z-d0(ki>>wb`s&f)(UT{SMLrxn=D5DG@$UQYy9cE+ zx`t?ev{WkJyZ-=3vC{#f%vZ`Akj#oX%WHM~8@>{?!nY)V@pWt(=gC6O%)(=8^2YSApHhICU+4fe7IL@QMl;1;TN0^4}<Gj$Ze5IBB+>IXPzs9j=49 zWzl7q?(FS3yQ0vI4arL=`r5sFcX#)8@9gc?*Oc!nXRBE&%e1~O>4**?ox9XQ*Ople zzfbX+$_%ZYZ=jypm88o~o82T>IywMcPNf}Lt;V@5S2n)(l}GFA%ey=E^=fG#HFVgX zU?2x%Or)znPAw!B_l+u72Au2HL?(aYcRTGMl+=ohNY0#>0m;NtMj zaktgyCyp~Z#v#m55;~QoW}I3rgi!((Y7vo4dR_fVVFxlE6G)xB1cggxIALa?2A$$SFxF2*UrhP+&;yf9E{f6Y6<%%9u(0cd zPo7tf*U1C!Uezj%TDiIh9o?ld}vt?j)X(^Sj#O0VB( z9v-J$OO2HMSJ3qSuYdTBKe+qeuddx+Gldt)W#MWF!$6wA)Wqr+-^p#I4;IRcPNYmW z{9j>>PA-BmA;syXqEs3~yZ_n${=fUJ?K=-Q8}~^Wr{~h(-ZRDl$xEh4wgY2mz>!G3 zX?oaeiFc`d?;n5B`(nSnv7T|2d>GKb)r4N)xL8MajTBL(`W^>1;45VAEQ=AvQ&JI_ zOhE8XcF7A!q_?Od59)rQzN(adG@6*q%(dDrStcn0E!j6WHq23e|NTe*{9pXeR%2_Z z?T1MH=2T3J(Ip($IqR#x@^uYwsMzhDEse$RJ$SpWGs)U!eXSwzlK@eSq!q@J@$9%h zCUOLfjkRWn;>Md6!rXvdqbnu4R`8RNI|9z9J(kXWQ4x6hQB`_8nEytVU~;XjR5`b` znmYj?S=+XuqP$10vCK{uFb~M2W4$0cIu;gcb92+@ym>?2q(5|1+!(uBs_`;c>wzzv zHTLI<7ZfMC`< z7_;3zHk6irRGGR(nfu?-D`jTc3wVzt$23os@-q4+}$d6Y-N>8|Ni^G_LUF6_V!zEtyStyO)Ou=dP0|Ema3Af z1g|jXkbIsCptl%BEX$D|beuNr8W!SgJQ;nVu;6UivARY`UR(~3nmtTpt;ggH98im{ zv5;EMtI^z({g*%e=}~4aOIC@nPQgr5uzU`>)dZt)&xn>~o(QBV+v{ru()5|lV5Pb5 zjukYIN5RG7ygJdo;s7HKq)`3l&fboiah4~w7X)MaMvr=1sH{e7bcq4#M4D=Oqk8|r zF8ehZ?Ksd*)xmM=@Zex?XRp4FV6a%**r;u6=}hJjXz$ZHI-Fb#K?;TmSM5b*jjt

b|=YseK zQ&R-ViG%N@KaV`Q!WhLevXT7(2R7{w48*-q9$8}w-y9smLa10z%|JP>=Fg;y|K(-3 zJyBN~a8d>4nFfiKI6hQ*3|&OW zzJhM8-Rhhd`QeAtX9U(R3B)ex*TpSa!${|;rf#L)b zsv*+PUuqHq_qv>F8wqK5uxF)VLFTRIz>+pQZ3*2E-l?@(M?d)M!>MnB`Wg)BYX4ZC zXLY9`?ah)H%(egKXLHjHn>^`HB$MP7C1V#$J2wW`10=A7_(vs78U#meB~?LbeYSKp z?%HMA2DewwOsS;GDlsYQ{)OR|?~i`(8`uexvo5w&ug$TAh9_v`PmV77XvT{2N_y*o zVRSiJSv~Fy|Magq&zl!Dr&k%>b(M%uZbsTj1zT1S0IN#t8HI_b>VH&4h(mz({xU<4{5ZIVb6#G#t8)WQS&FoLm%s3zgFmtPCP6RaKUhVCEEQX+*xllU6Pi zTN8#PTj2GM5PGH{n?0zOD~tqWUqn+-U*@ZIO-9x?*DBPWCL8`L!5|=L9vr>+;;B9W zl2omh;8(>x4zBKW0_S=Cyv>u^Xf*2U4KjXo{JIq|Q?pjtg=txxPP4BGixjqU4vv{U z6U@fUDcpPRDRidlh$%14tWinq({3sVXo_=i=*8nGHeB%3n}AxxkoZQ2v2% zM~e|+Yqhk7Ps|VJ6}ab9ocPQVbNVB$c(q(DHi>UR8aYS2@`y?dK4%~1LXmkrlA`d% zr`rdsZlm%Ej){|qg=nA5o>L-|eu^%Vc}%F|iz}u&b4JjKb$tXU>-?F|P8PScz%|XF zt_Zw6I$uc(i}3X6)6%j}ktiCS93BqF4r@vVDVmLqk}i8y?6&vr8m7#1^%RS%%UtwN zKl)U;I&;+>(-PBTHG_3nAH7sB*UHDk*3+N<@RJ|?sN0)dFIQCcjCz<=(n;NGPNc(4 zX$bT)3qY9-;6rf)uNW`#eD=4$d5^@$xekh|bBRwtj+|5R!k5Cj0hHpB=>UZ+aj<{# zr+>Ep@n;8(JBx_ddd1oXt*l_jU@oidXO5hE6#T>tCO=ErOH9m@3NKK4AhfWuS!gwT zIUuQ^<-rCCF@M6OK5ZJ#C$7ys8uo^Ml-u#2?d|QoyLay2zrzO*l6edoQ?)f~BQCP= zzSvUCdMO*{J7dVT>5BAmxrFaSQ9o)3z)A4QWLipFfU+P&R9!R&GNgqG^To<9G@%V#g2K6(23)gkQ!je-{O zpQ=^rAnm`mx3|`kJB6g)KrCh6)yoY>t;3Vo2ZzUyW0cE@`H_qUai3kPR5xoB&&o>3 zUfOT)7r4Yoh$h!xoMz6!<+&0;VUk}WOpfH7x~HE%+5g!`PkE#&ecQdhn9J(Y*w%ws zGdhgaRgl!N06RgpDn~w^67DhFd``D9COV)g3obxX9T>OaZakVy zZQ=M!-lWrqlWKRNviXM9<~)csQ=64OSw17~1Q=G`>!q5*`D%^3tT_}YLo?zeJg5-HOVPc?g-4&bMe{^@FuI&{ff5?X0=tm2>yf+n6 z@%4CN#=>9_DTp1Pw=K#2x8J}2;n%m{dU$o-Iqw|MISxOkGsgcM(-kv`iNY(m!E$HF z)wQ@02pUr~`y1cbL$>!GNd=&jzut!a9xXwmsB-FR9W`ia`Pu9K_x^1Ehd*y_?5x_+ zjGMd0V8sABm9w;bg*%b54aX}Y$>>Nn6vYs(MhhplqI@g~625F>#&H!L3JU)x$yI7r z@{K#`>`T!KPN*$Sz-^e{-CRF<>!BdZ86;iwBglad$%{q@v(tmVo|5nn9XXZkb<#G^ z4Akg}6xWf)6BR>8&C#e|bR0@M^;_NK1(YpZGX%DvZ9vE#z)ssFQJp&=Z#K?<#i&|! zj##Kul;VEj$)DqUQ(?6#Uhk{NpS+?P3ioP^qE1ydTggD-8c6z-IRfU*LEGZRQFMSbspxk17cPJzpq>)Ugm&5MW z7th|D>t}g+mKG0JQ{6$92T?br@Z3HpR(V;{MTX=6RBa1_ppc0RAYF9c8msq}n-KAY zD<*}Wg;y7&b&xCyk6Sr>XVB`6)BRCfcG(nsfDj>yf8!DGc)l#5VdqPQ;3q5T(n>iS*O;%<3;b zIr)?S>$8uZk2iLu>|mHeK$UxTWNK9*HnP|VvzB(rmnCpYETS)zP7#xBuwd*S=s9b` z{3SP?7RB^E-YV<_$+OkP#^hp+_{UC4kkaBTJ7}xrWv38!4@dECE)=^8sb$=e*qZGbB+G&)nBWeT(O#;8ir_p4?AhI@Ebh zYHp@s8zut1e(@?qe|>} zA!ZJ|4dp<_BeElmm79!X=tu?O)JC2w7Y{}ao*#Lb&ZgrCOCeMgbhKimmOzPL=FR;? zL^0xm^|Sxk4y@Oph&+|l+hG-P;KFD-yVN6LZlUA`_@zPI`M}ZKZb_3k50k)?4YK)C z-)-!e3v`q6)>e&gB;BUFA%^p~*G7-pz0;7#kbPgPH+6*4!TuYaztM8e&S$sp zu6J6m)EjD+#i<_lUM$E0F09lx>kl7&?Vb02``!1y*(fa^_Kul)OCRRhI4KHt-mPI& ziCDqfv;X>Eeh|f=;ivptoO?V@?&rNINBrVStYWfW5p%R?@3l zar6-hAqO{eco;|($rdX3km%6`d~V7lIfsdrQ^@BZ*#)3Bq9)`P!cnwc(C!!!;Ns{w z^>Xp#g2C8}`L+!1>FN6V22zOqNRCa+=yYRg!pM|E4nwMj%1e%U8%b88PDA5Hx_4MU z{!NiSY%`C&F8eAs2$@G$8*BH#zGwpd>g7w#aMYryCAaZH7wX&Aukyf+m6(i$x74pP zC7pOU|L`$4lYa113otiB**zSKk3vkupv9f+0_cnwUX9%pVWvKUiI{5A8~6H3 z0sJ1gya?TCY*ul}!HUtAfEOcdOo!8xo2V=*-5gOf%8o_ZEN}>R%S6sixt=06>t~S!-ts!~0Y(Lyn%`ME%DIxS#OA$g_uamWZU}BO|1iEJ^0^d1*DN z?;eur@M3p+{n5i6L)vgNcJ}r%HwD4CtQwB*Wm;hC@h?6`K{uKL+H&X4?CURHzJ75u zy?6mdxg568(-fJPmyz|>c6RsP`P#31^_>T%)&Rl{2vC~L0RT?1bK1hv4V2I9Z+>H! zOQw_Bis{=Sn`0}$_!k`HUU%;C%jSRio5R2O!AWBiLVcl1VEUT+2O@>mf)+@S(6YpJ zM1(n}ykfj8f1CjT7#$?3kYoc}4UUV08THBCPumE~YB4jqFkJ@J7AHL?;y_ncCB$zL zBoHb~2A`A@lO8D6%OVO!xNz;m{R86o`qk_G{R7U`;o)&d#b1B)^uv%FylDD5V4K_I|tF#06`%+9boH@L)wyLA|3YY;#()3g_J;_Uwg0 z7E|Z=$C^owPcNIM7hYZ+pX~qD4?aS_mH%@%Y<#^|<9H${nUP2($@iFs3rYdi#E9@d zu2piR9W{xy{?e7ZdwUyORlb<0V`1KyVeaBUn_jm-g%ts?&;rvuhcdps41viEjjcOd zJ35BFIygef)m=W_uC7WJj~fpRpe}3fg7Miq?EKvcNgA&Af?e&1qqXaptC{pHM zLgmpW>(Y`wLi!Mc!^u^t>~iWKYGiWWbgvLFf_ThpR{=j>bfZo_d*fKs6C!6mUVI{& zTNXi_ac;~#Sc{;TF>)pMuxiGGV4j@iL}bfiKs{;#UQ9b{8RZVF!Ij}0^M<5kBHRvH zdah@zv|VC2k?)e3Tq&lWAWpcIOaRoRT+TMRO-F)`=Vxnd!Bn$K9ch}O!~BY|Nh&L-g@Vq-JM;-fMuL#zw?{#@~AD0Y^S)9 ze8@H}Os}4_#((kS7ys$c54*$btxct>^&;_3MKh>7DO7wDh= zaBNJi85|!TbWfTTgl$l?{^arJx~H@iN{zJ}Gv}iB;^mwDH-`kj|P> zK(q`!O8gn9h4&n1BoR>&C5bIogP%;Hn=qjyhtpd{Ix$#G0bb^s1%%@>hLyk7Xnga7 zP2)KVW`-t6<^`VNI7w-Q{ly)1Zp@sM!bu#U(wND^iqWPR_)N$FF!Y`ZBQ2QOJ5{UU z%A|9$fSp7{ldur9GgcWT_Kc<*482homX&Hr*RE9kQimcc?yj>h zqJiw9k8$ID9?dS6mPUn<+9}u;pM#yuDP=IRb9AZ^2WplrXE?F@)1~DyRdsLg?%fAl zNLI~*W0_-15Xc1dKQq6xQ|-17UOs(Ot}j1%GMHIJ0Y827yg3{!+`YT2K1@Dtsf>FJ z-<>*3Hq}$xCc#K$htxIhtZ(iZGSg5CMNr!EaDP0a1 z1^Z>*Dd&D}P8NP=vpnpd9UdInM+H74P}W!vD4S!!bxGBDOoxyh$&yrpIa+5&&Ers2 zOu8%xaNw4eRm7kRo+U#;>oa{8{+)Lpz4!iGo#ylZ{GWdOYJYTwkbgOQK28V0Q*9Vk z!H_0q$~23tbAUuYa`h!qfJ4lnX>Qf>;_h}zcQ!ZXaZZ=h4RwArJk`72oG|pa#+E6z zRo#&e4?0G0a$P%O(@SX>Qmtu1iI*ljs;e{XjT{+{gUTa{1F{!EiQP|7VM_wLr-Ct~ zoWn2h6nG*8jgxH8(0bK%d2nG8?^dcdbuJh@68(LPoLEt2B@r*!> zlADW~Fjg=M&1Nx7P6L3;|L0Ht_}R0w>C(-M*G)Da$E4YOeRR~*lWREYUJi8kTe(cH zN|3vHh?Mn>d!wEXD`#QM(Ev%DDEy>0{_S@@yiOO$OzGyyEAacl$FKkAKY7|49M{&$ z$_eBOlcgd^@gL(U!DuCmnS4$VKMzyNss#%rHgO_sG!TV;3Qi+fi9_z=NJvPAoDUPc z2&oJK2SM)ioTp&<2`X@++i6>FJHZWeb`zH0Xau&QYXr{|QBQc&>UD7e(^=f$AzwIL z1=?v5RW_}rS)~S34s(^7Z{e`QjE&A5P8@E_jW9j#54oZqg^7DrD|hF88krH{o?_RUv-b%XStqRbO``1c-4{^xhbJY@VDgDNjrBeW)QA7 zHFJ2_+LXPkF7NN}OYqAUN>@-_lmJgau)h(TmD1Mvg}nz4_wMedHQ1zQup?&`{lhwV zKx9GTr2=fnIgZh^samV+Yn!Tjjr45M83PmMl9~=HF%w$V>B{TmG3&V@m)4KmbWZK~y~G zi=cqyG>ZpQ>h6enTac}Ae+amUEnEVeLdZ)3tqge9D)KtjU~`VQJr&S`USw;+G!jpA%E?@}fM+yduTcVlda>uIC7>~a|T z^NZqhP9b$qN{O@+2YPf+6H;WgYV|X7Xh_0To3NA%kp3?Gy55ttsRCzZHr5z zL6cycz2(^iz0)_N#zyJq|M>Xmbx-DPq&3YH#l1oljt)$WS}a;OtB2A_!a*8GN zttr%q*6CPHg61icoItVefBP@L2G~rNptRl@FaPw@{qOzdi^|rGtc(>V3^pxLp$*Ig zytq?-<3Cp0V=x*{j^FskZ5>V#%=4Q3DD*NtJEQ;P+)F;e1+#_+!Eh_IY@h-hp!tvU&g&qw9It*(^$tF~DS4IoO6o&3za@{4$=?hVo+Y9&_;n@6@q@D|3_r@3G< zT~Z4O5VEUCkAMNe^DG0dtYpFc_IsnO0!>QmWZBXOT~kb@wLv01JhTBZg8=WZ$81^~!n$lGyiqCyb|`SM?4YeoIb zN;m*w3hP;*EIJrzO(e8#HKRk5NwE_*RHEyObWbeG-t%^0a7lfcoX6Tf!SOU_*_F?02veaFcPP+_JdA zGEd;#@BZ!w_1f0y#q5(eXFq*>^uwR*FP3$u4_n&Vu_QSfh14-?zI8bK(l7in&yz*- zkBy9R^pmL4N|C|h1^G?Um&roKvk#6Qx^CVJPs9v%rX4A7&ij>+305evqPrFmcPS$5 z?D`CHm5K_u_S}qgfqx4-BQ7Pa`NE?JND7G2U`AtpHG$XDiWf+=YeU$4?isbEEClqaIr%rGPHix2M_;h)w}asZJg!sanjV^zLmxDfv+9=EA>wLV0a;?+ z5=&#@_(iK&JVirP%3g}qO%P&v5qi~0z=5iiR)x52)x-L_8%rw7uT4vNl}9aE&w4Oq za6woR#ldH$lo&v3BvQqmWjTbv4ztaG9yOaov=ydCAZcIy%g^F@+$f)VPPsoLOZ9Ovt6yrm!vbOqz*E0}K0O;)_$7gi~yWn|3$@C|uC%=1d{< zlJnb7;h^wz+)r@kt0{#@U-`;g_wR2TC#UvG`57bb&CGr_XlfPny1Y7feDM7HfBs`D z)@)vyvn1V$Lrd3-%AD%Va!>ff-}@D#qyFM= z4*u?^FAiFri>dMKRFSpQ;3Tr=;j^Xj9_<^28C4Y3YUc`(@k`f?L&=kj?F$irk;v8k z;9b6KfaQon4uqV)`=6f(FQt=1Ok0Hs$1Yw=2_Uu5w23gtB^S6**tXs95BsDWS#idI z>>8`(i$xuw+Uf412Oe(&q=gh(tNR#vhvH6hx2O_K#PRWoqvOLArL%19R+k7#PH2+v zf;Lq9J!;r1qiUv+gM%?UU(PH9qR5HkE*MjR-i5=nV$*X$cWb$@2trFN8rhM&43KmP zykttM*AFa(UwWU-^8b#BZDO?)YS0G+gu?PHsMtA_*OyJKQ0)d(*TNa5u9a325xGK%kEM_an!RhN5}%7lkG|CeO^))E+I& ztyPzIwvA)1Zf$R|qxgeD52$>*t=w!fzLaSLu&nMX_?etgH?53<1w$31j8#H_@^gzP0`IuK#&-&+jeHrL*FjI7f_RA%tFG zmn|0cPk>V&qa~R|AB98gm2q@MFi~R8G$E(g!jG4mC+GO&r0MPGeu|rH;qo%)KF?_l zavFj~VXm3EI}m+3$YhwN;VwA$?v7N@Je`eVTsHRP@pLpfUXp6BLJxzD5HqWf(q z@syx@A$q)-!~)Dx(rna{{05SnIjMpE4e)R#sXxt2vKtP$GtJ{Bbo8A-9?`(C2&M|a zXi2RF00pDL*-k)}q(huMzji1b02@v}csg=wJ(W#J%u1f~O(||#0IfY-YEH8UrmqUS z06Ma@!A3-Rg^Q7jUKXC1V*g!DM0CWGuFxaY9T&kK_D$r?R*FQpqp0m7Izm1f)4B1^ ze1fgF5H{MN=kOxa4uWelW52XZ^hd=Z*2B`cj7i)aO2COjLJ7*=>B-RR9=&>fba>cd z2@HUR#^Dqr!(W_r{`i0S!TRo+fpNiCr)I@a&MmQ1<4zC+q8{+Vh(QiYhZ8{?M%lVs zsgUN;iIuYsv%Bl7fBNTdw%_t*ijFhj1n_;+O4YIU7ztbpGfvo?M4pJRTOsq8Si1s= zc1HcWcx>{PxF%|ptWc?i{qrZH0<86?T%sV3hUJj+M%=tNqiZt?#j&(ee#b$I$CA<& z^2{I=Bd@Z%+NJRN5`EDVj%}bV8xPfu%|s$Tp5;XSkt~50($=7{bdd@1Ntomihln9-FZ0bfVQVE5$Pz8it3pc0 z6;DWnoyQ!eF+E9WQVnIUaj08v39r-+$>?U3Uo31ga@d;o(srk*CqM@lC95Za&Pq|y ziKymglnZkq*i1L6{80+g%E#7;v`k+}+^D=g4WZ&iL83(>1++zT=ceL=g^9*kX$8Xn z6q9piDG`6Pyv~|ur!2h7x#4HHH&-QQPM_07>sfc{8ysD~v4dVuI|zn+Lj|}Y&*Etm zvm1x4*Z*6O-oeB>WF8klsBI`A zAlTj88aEuXR+u}B6BXynig?5rhe%E=q3>=bmg@E6?Qin;Y;WPnk|JOH;6gda9gU!J zuaW)7cqvf{wnFwk1t_c^ts3j+roGSS6skUBACXts8Amh1X7nRO@>DMm7yhp%n4G9j zSy*s!_wNr(EGNwO{r-ToP|RFAeiRN`wm@STiYs9fd5~mJcPU4f&h}8um}g|um=MoJ z#GOs|PHWPgv|Z*bPn2Bo&9WwtST`m!X0fRckKwo%6RZWv1TsSCy;(!d1d4q*5dtA} z@i>-o`zZK1<;7GAbiwjNE<7^p;ZN0i)p0J4ojb8GDGTyfd=a;i1?Moh%d|mLYm!jU zNexO2Fl8h{ai*8l4b%Ne)`{^62r_$0Us9TJIO!D!``n5_tSKg&opZq1E@rXXXTRh$_;2i$K%>AEY!DB~qd_w% zF}(wl?(u52kzO{SMHiC^n{0T!V;DZCgA9$dbv{c#0}CO0cGl|JN_%Q4EH&Y_m4!qL zOwO74xGaoBoO3TTR=uJu5UgT_E7khW9VO0zeB?HtJ$){k&L;f` z%j_x;Wa`qAgefX$%(*Nq0CiGG0*|iv@`JE0{%O7}q~CfL-+NTw7jhcSMCxwWH^0m- zAkakmNO%5Z!5B&ZS6ICaM9GK@X9)K&8jds;C`Ae>SY_ApCymmWa`y-OOn>` zN!iDzoRzWl=j_@oCtg;TE)WO0JwO&n(DGX)mdanOL!m%qJ67zzVhh&@-8a1vW{nA4 ztw^6{v*-<1C(Kd++?oWwA!^Vj**lieno`Z&Y+Xi5|7am7^Zs5BP-vgp(N|EX+(>`bf2S8n}zyU=^5fckc&=l@C zwmCwv()p5acIa#k?r6|lK&YrO^;8Uk$O3$+z=b^8`9$~(6qc+Y9E)#lz4Y|i6D1Uz zJ3BzC7yB)fNr#=|kAC*~_;gt*i_}>vD=){$+Z=81lAW{wEbWT-?E6X_h?|%DhfBVy?%PO*T443GXfjPTel6y?|5s3@`=x>;_<&ee#lHOQ6|N4(( zo?WA7o!bNfPF5t0((@vB>)^&DIuPFNFS&@1evRpyyMACF*;S>>O z%Vm=y6^SI#C^>b~kz7_-j-a48Z8>I%NU8tmcMA!(OK1vXv_K?t76&xHSxm;3*)ueN zPDO5H$55iI1+^R|2b>!D3!{YyO3uk;Vf_Tr)x(Jh_&)|Cl@>7@ zI7lFZq`%V%NmR&xzR|~hVOlS=k_Cz#$aGEzvIwu!D^qyx5hJb!6}sSpnW?jvirhJ> z&=QUq$P`>BDMiCjM0)M(Mapur?}aBI-eZ0+NS(+;wl)GEMfQ}=qZUb>jvH(1xKpHL ziq+yM7E~BT2CSl@ds3G%=XMSJAPJ@su$zRYJ0!Lu2f=cOgM-WIcaiL@Qs{4)dq=qQ zTj^AlZ=e=;Tacd;{LsSLrO>|8x`GG_3zIa)Wl0Z9=R{BALIuavI!^(eqOZw`vo~E? zGs41jopfikPK=n_Q5Yjw5&KKPVvMRL*WN>&2I7&{n>iiH61xJXs5Bx$O97AvEKVpT z=n2g(XJatml#UjPm{K`Y7PxGHR50V|izknd)@oZ@cgn`_adM6KK6%!vHP&X9M?e0% zztgi%H*-x}<=s6Xt)YhvR-a{)`ct8iuu2{>DR>s$WSh{wuFvhVq(Q`T_J900?}MLz z@~o#WWNKo5H@J1Ebj7+wMn&rpz8r8;;m8TygI7kDtfRACoNpGv4JVW!DIPXFjvT3| zJ+?SsWQPd3V63J~M>j>5z2NrC@2K8eYs1KRi#_+2yd0mG4Ca!t8h-6MzWgF^fTd>U zbF046>&_QPh)MHS^2pfxQgY&u6j0*Cw}G#~ zPL{X8c!U$aVGo%q+&|ecxik~V91D-YfhZP_=ovnh6q$A6+4v@|F`vgc8s{|n>z6C3 zx+;umKH9vN@6u+AR+AB+XkZ>LNbP0iZ?1*bkuHK!ux$#1#)fhe6zMF%E3+5b$KX3f z+~VjG$t=D9#j`aIhCiHUNx=dvf}665Oj+Eb=uZm-NGLkrT`FCIU+)KO{P2zP-)j;CeJ#kGiumoO&+c5k=Jiz`vMX-t!h z6i->aK6q(XQXvC0J*{lhW`{I788RinL{-?@lLQG)rnTg_q>CDXkKjo#TtuXb)TqrM zPl8H!#U45ne3HB>C*qg!w%2+(hDk~;b5U^Cl-ZQ!Q)J|QqEdr|vdlHdfsgI!_-szI zTn^t~7Dfir!vB$nEE51n6!;Tz=U|= zyA={jw5+yEK~1yXiK{sLnLL%=rR0zXVNhKPj6^&=v-<*nCV4_sU_oemG_nIJmNfh$ z`NJ5W+hs2z3!zF(pP1BU@NtAos)r1zX8@gJmC@_#Ytk6qc3XBXK9f_(M0y?)9iA7v z!XHy|iMM#A$G!z{v&D(B;4mkVwpNHn^A#$+u*i;z7^Fx=)2&J5@e7iexe3i!&uX?= z$9MO$e<)*hy1r5BwtKC?nZc+_i{=+yKYz17Qd%aTY@P=`Ws!EF>THdb_e2RiB}j}p z_a&X9Oedj73aP|WLJ9cb>>vLAx1c~+W+vU58-gXcLfR z+ad~)8GGx|qvWXb*g2e$WP7Y}9B`{{N@LjQ&Z4i|*B6H^qV@l?e!0}`)p%3>bvV5~ zFZ<>D99|uTT%T==n`wo~QCzQE;faDNqE@~T|IrgObl^>vd!T>_ zZQNxSvSg09kE9pD${yU|uq!GC(&ob$4fB(Ry!@zh4}yk4{N0_C=K&L~Q4iJYIesXFlrFVlh(JVM(G$_d&X9k#OvOK&I%71Q%?fl~KZGM=0-Hz^N^qX1^D!7N~_t z+BP{~**6V)Xdlljf~#$mcp?h=+1hSP@$3m#}I{|GsNJlS1WclL{0DlSkQ z%eZ}HDwp*2OF7TWp@^}cerZV}+|kX^4msa$HSszL^n!#{*<(CBq<9`pSD8C@%W1;*u7eb+bU<#5puZQsEw!BsRAcR z>W@P30p7?XiF%2ac5Z`wf@}EK@svbA$cpY610(j|$w<6ZXbomrWfmG{n%JZ5yDQfe zwVs@`37+EpUazUIMF5zLd7?{+ee|1SWAC7du6bJ!&4aM(O1V1SK(kdwM)Rovj;IwyGjI#c|UM8|z!u1pthm9d69c z#X~FdqOOAK6!rt$!Lw3&p3pa}a83oVp7%inXaDH8-diqd`?*rzu5Z;FUwrcMv*y%V zqfF2#BuAXGxV1Amsctz)N=~rNzWs_+cvbObBZ-LIDe6Li(O}55QeJ4mJZQcZh(`kz zzx8`Z=)cO`?f>5j;O)f=kuOA!PwR^VS6rmHbkQ;-l`jc0bB>E^+UV#BmuKmfWAe*n z{|oYW>Uw*9Iq_^*d^RMQd}U5xn6hGe+g3I666BqF&{-4Pr7Nai?H@Q=h~1=!O`u6> z8cwTw@sxgY&w+Cae2C6>G;Icj=fvBE^{Syx=~GP4Bb0b3qEjGPKoU0@ql%W8h2Yl( zN(x&9J-~u6Q0Vz03e8}s^j6zg)uf){;CW+_mav@E4hTM8Krt`c!#jB%Er$_jz!Kc9 zdvS0nVNelU$HJePucfEm#GGoVhT>f(mH3JJ!hw$N<=?sYY+6*90LCjhqZE0%ml6n? z&!ff>@R@NbDKC9Qz04n}q#Tx_CX8>F&bdX6pH)Q<^QgQrUP-ZmZW1!W03w%xm(#|A zBk#&65PSjQ9GSJGA&@0!NS{(MCOy&d;=QG~(qp%G>nP2EZ>*S0xd%pvl6ALqvBp@SmN9Q2h7~o!buchEZP0Lm6OiF)2C-|-_^fLofL5bo9ip8 z+A5x~{FpE&>^0^N{8j{3iII&W%EU&$P{)^mB6>DsO+0I1#r~Ub?983(hc+*NvAMB+ ze{-+ZeD%ePHZ!Wcz@pP|Zk^b1FgOFwR-qQ-g*xSwG^Ig5h90fJrs$RCgi?5g#HWcB zXF*`;#Sx2`saR^F=c_QF`6kB8DfWU8nn`QEa=sHx$GQ|}Hw}E~s=`qa@Hu!1-EQ4N zk664i`i%EN?BSLfFFw$q>C#@?A@=9su<2NFha67-k$fvomIOwU_|8x)8x-D#oB5`m zH9ahp`CZ6&yt>V$SN~!q(QhhjTSv z0F;uPc;uK|E(*sqh1;RRQFCdA$^UR4GqY$y37)5>PnrffB@7dT=msEWV~(j|Gz0dH zPwLQHw2<0WnG|~7^_@SCPy&y1$2EC9Fr;%FrJAZZl?N!-()+0fV!S*EUO*lqI9z-T zB1C(W$^yU?SVlObCp@fJ&jlUd^lK>`NKX?jCP`z$U!WQCOMnwkctRBLfW`5P96flL z+oOHrwqDUGwlfDIA{~yqS$gb|0Sswq352$xDLPR>r#eaDPda76V9^YGP%JZh=NzgZ z3hGb1H%7^ZIcoF8ywOM97_QJL)tLqGJb622hV$Hp@hDn%``)$ z*KA{}^zwPLR+&+n^8D3cYim;~%A+}KeswzTwNB6>j7Si0ra_TQGX^Pqp_`l|w_A9B zf+K5fi|uR@Lf`$)TbkfkD!VET^cvTP+F*h8bu-KsU+?SQb%D)Ta8)?9wntz0cumpn z;4d7ws7W_qj!ITK&l4?}g|(Q|=pgJ*A*ey06LH03B!1-0m+KTi1}}}wQJaOCv>nc( z;L(44Bb?af;Q~`K=8gTj7g@r*5ATq(^ZU98ODTSDanPo0G`&a9kk?Y%&w) zY8)+h=-9&~lG!8zmu3=8=y)bHXFWP1j8@JFp9nzfQ?l~YyyY2C?a+88E~0nZ#Gsb~ zwhK($KqfbQ)ZB?D@X^6k%#)xPLkx^1OZ-B+P#-nFs?-ym%g72Y(6#9kgA{v1v``~w}R@QNxsfeZe7Sp zF!K<2iuh&2t5qeOoH<<)S5};m^zsDNT&hR8xp%)bJJs2LqoZear7ll0Y@G~mt`}`h zlvF&A=m^;}MuY0->I=D9s92k+`9vA9g0r*#!yo*XRyP})Rr8?|s;F}+Ep3$NzIMO- zaHr0D(>TOS4O5T)E<_02;;|C{76F)u&P$!ey{=%DxJTZ-5Q&32#|4V6p(`$L%pkQ$@&VMSMogxg*pBRo29nHI$`N@r zf_93L@XgrzNCK3PaZL?<){7@6R$MPkyOqoDW z%gaE$Nw}H-AznqSTq=e`+!i@ojMNV$xf*@O)AMypZy;mSEW+}QN!ThDRu&Cj)kr`! z1CLso0Uj^^wrFIRh=@bh{uA!CM2IJHySCOglX zMhHW|APdV;ZQ^&&)$fkpw6Wpm-s6v!=M^VQ-4R*Fnz&h8jts_(y(4B=SkPQ1++LeM z4J`}FF`%Y-I0i79<20K6!+-IsuFhuhm3e$pPc*yL$KBoD)yuZ@y?=bJcG`G~mFeZH zl94?WC*4j5aTeivuEy;#p(-PXU}QiCYC|k&2N5vvhnOk}N$73v(*x3FrzvB1uWSa- z(>6!HI%{dJJRgrKP%=t^4GNhG-+{1(8ALnyQ0w7)B4icaN0eBQ zzTidz5>IU}q;ELV-Pw-8**<2HD*7NC zav^%dS5-pc$GIm*PYN!vZtC|~En+6kN!;AC2-Iigd>3Rr{`kuBYz;)jTo-!OYM{9J zd_a$5VWNZAixY3jq=k=@)ngBN(~gkxGEF!)8)fN}(7sN!;!Jl1@|@KuawqPjg~=Gz zsPI)MSnZ$;5=}j%*T*Oz{ZPBa68Cs=+BT&lzKE0OFv6L1MrJR$Bvi@etJht*)A!$d zu)fx)svR%Q8~uc%?xdx@awL6 z=|~50gsx3`^eOsD3UG9;$FFljH9fO-)SV+k?czHji)ZMb3IOsKRC(dE1tr&;qB}RX$ZX^3&H`N;;0nK}zhgr&AW;^tz_b zzZ6e|JIaKrz1e82Ymp`w1_hx#sOU}?eL)7cmh28_G+%1*Gh`-{y!-&65+#j=<;+7iQMH z3jGdoN<^2Sf--YDtw69`4tS)}36W?OX2%*6ZgUf`?IFf$ zVTsU4&0c1?ZLEi(Rv^KOfRYNQe7K-(uc?cQG~B z07J+;MLm&$4l-eLLU9o(wFsrk(Jo?s-_L&cx8CDySXqn05Q23qki7zI8^%^$ys9nF ze*b4L0fVI_Wg=WlaD_vd<0k|!A~$8MGsg?4C$5x*jME7jhtS zGGvp3HD2>}T+ZtHeIP_GaYTDc;6&;;{pkM*3*$G&$IBro%mh(n&FBL=2TEJiqeVB~ zE44pDkkylp!I$a}rjAaUKFt;Mhxg#jgXIPcP^k1db;WgLi#-Yfa8Lrs!~`iMB44)7 zTccul9j@;&-4zpVg(*4a+@16Bf}v5EHYO2Q>b;E*`X8U&S>Vm4Fxs>UZG3brXefOkM6oLM`kACe;hz@^||(5hta z*6_LTt#Kxmg?XD{HJnjTS(p?aYHqb52YqZj+IFv zLUu&I0uj2VLE5OvB{CaVa0iJ@bqRb%O)J@Yx-SpPONzoz(7`29$HG{2+e1{Ab(6i- z=tgrT-XnC)w&nvc8>3v^1OIbm4Y}(gWH0pzf(ZXQLPCPH+KfAsbHleNd>X5PB(z+w zX$*(Yd##Z(ZT0f?n-?$4p z=WRXoE>0JwmrOI%UI2U7Z6&T`k1wf!gq4mXINDhxLb0>)grcYE5o)%O%;+szJD%__ zqr_aov2*URRUA8q&mmytm`aJ-#MxZoqT5eV=NamNLom4prC+HQD5As#Pm~;j9t{`g zRee$CmcDrQ3muG9h?@t5*NL9uZq=|}UIsy(bh_0GUF;PWTo&*}fCzifP4qI@mk*^E zIa1`rpI`vOEQ@S$Zf!CzsL&De=7y+UYP2Ah;_Rdx=!Uu(Q#NJ^^Akx^xn9ywm4_ly zO?ECWAN3?J$JkQmWZVZQq+!NBm9$P24@?mcTS3O0F*|{9 z0iagT9VRp_Y238Te$!I&HPkF4BmBIGXMs!X!x zInZ_r{c>Y(@lLyRaBzXKu~J?ZT(`Tel~NTN;tXg}SAZRKqV`P*NoN1ucfVoDf&92y z7A`dhBV=W5cGNQ`e4$9!H|pPfv{PN0`Qc};TLaUo%}3FQ>p~aKJC*rwe`V))zxDR- z{l>5V=C?k0_+Udj7qG`#b?L3W^{+hGHIw@W6?t)SZKZs?-@)`S7Di94DGk?>TaS|o zEMQ@Yj@9-XDw%?0BBFgcMNCS-IQrXm!=9b6ig zZ&TzR<{kLx*u?1gnPgB>`%Lo)SMwVP4u8jSJk)Zu*`Hj@DNe=3x9?1)jxA2CPn1al zRdPq2cV{jkhsaGQI3*DUol4bK)Kj4GiOM~cVhloME49IiZo3d7DYD|afq z1BOmZq1kcZA{UD~xV5ymPi>ZW96IR8s4CD<;r$>)xwDcQ&no>We{1Ei>P zz`RULk!taFK#yZ4EMU-CY|2Jd;)Vr^!~qniTltbPfT>!Rcj4~%qxjTKspWnn!1=eUAUUP7_$>!{n|T~y^XJb<fW6#_#r(I zDw; z&Vis(;vV+4&pYkt%7luF@|Zz#gh!4AuTq?vCxE{RU1C_W^_1kyEFZw*#J2mv5*FU6 zuFQo8s6VfD2Q~^*mvCY@UuFd6{$sV4RA+KX|NO82p?4mUR^gsbJ-TsK%IIN+$nrv& z7b@xSxD^vdEN28cquEmuc+L$}!^EA|aEhzL905+#(}6ab*3s3$NCyBqCgK-koL*RX z*vuXPIfU>IT)7a%>!Mvg97D^#kd$>+MB`}9H9P&QY3OT!2+P%qtXk1no-`uZY~ zo27KPl9ii8S+k!82Wq_Prg^Y_%m~o{Hkva8uXGY-CMNA6yPi`yxO1`_)e=F`cp;*v z6g&w%Fz5BnTYDR;x~Mu#^D+Q)lT!l-k{r^*dE%MpPlYrJL|2xlFK3r)Yb8bU*EjW} z=0F7NfUza~QYwYoO+=5u_aDCdPn=|)e-O3_BS6;!28Y+D9jqaJrIiPH+V1tJR+@XT zegCUp|FyRs+}UZYZqydHYQ~J3W4AJUJ=-6dpPmJ`ccy*lIYFHZ*)C=Pzp#_Q(Nqg% z8zXyjx^g|cWnxXGqGghnM(b;}-OYwYjY7IiV`g^4)nfgkk}X|8c2_EFoita1Rhz-r zvX?&9k2a(D|8OjM3=BdDC@~;TcJ@+pGPoB2ifF0U@q6gJKjoay3T&vfG86RT&70$1 zGAkS@#tKDdXLpMRJKjHZI0TSMW(BCD%b+r_`_O0$IlAyR00|>BLZku9F?Vp)i11k~DE( zP<QF2&_jgU{Mned^2WC@-OoWm!HO85n; zye&7$9s@i`Lbsb%^PmMSY#GM?m#I5@k}bQ=!(LTcQ_f>eckbJbZU6*`7DS4MNQwYN z%^}-iJ^97|oE`S7BcumMIAnz)Y|0K=bV!s*0*xN-SW^y_b4{7$-**a%a1y+IZ&hZV zbM{_)4d43KTCff)%WV#ho?^cOrzcEI90d{NF95FqR>p!n;cMn<59Eq@fiJjI`5TM6 zPSTr-#0%q(hG*c*$YhbqW&))=S9e5%@W?d zZ%P^mU=w~%x@Z#AFasl0T6!i=7O4ytmSGV{1qIHN_1G_Ou=!gi=c23c-Y7 z!EfLYn`_XUg3syXuH8HS!4ID`+bqboPboYo^PxCsdk=>PPi5y2Gvq3}=Ks`pZ|1jV z^6J36-kMp!)%6yCzt>KI1bzoBBbpE!i9qswt-OCL3rG+Juk$EO7!7X6io$13f=<~lde5yy&=3mO?zJ5B zyG`B4Na?Nop@c3(Mq6k;5_b&D*C_ZIfnj!~6Xipoppuf%Ub+2TQYcDZ^X zWt}dC6&l_~>>NA-Yd8kpGs0@8*9FOOJ+)$VG(-f%$bpZWx43c7MZrY!e#N6n!0;>g zg)czjjJHhnO}349Z5pa0>TOgae^9$6oqxRUXM{K=oSW{qev+>urLi$WYx=%I#p(v` zu`Hwol)x|@?l|?3h4m6dsq@W7q<@(-n2T^V9Tle1!#W(Ef(u9VxyPQG9`JtaFZ}gw z{6sy3JrSpaYwaGGXnVoQ)HjJIEU>VW*3g5yQzzBou!gDOFS-p`7W&t=dihpo($Bqt zcjep#eM=QsP)A>=HYR>kVq_dWPC&t&Ok4hNiOHBzYm5#r!*^?E;Q` zeS#Dh7)&>RX4?G5Ei0#zN&F?De=)t%L%+lUz1{4zj{*T1FiHd=7u%bQ>AJSF4;|TB zL4!daJ4O&ZlO{0(DLmmkg5$@xTMJ!8VOB7#6XrPnKJO@L6g(jvP1(KWY+`x0c6!9W z)jLp4s6%|q*jv=ne@XF|1!asmWHpL7cn7a@T)bV^Hv zo6km2r_eUv5@FvnSCcKad|QD?DZOs{9c=9T-`S{%8&_WApTMVo6W3L zW{6?pWx12#Wxv&G8DIy)>2|t#@R@*f-{gnN?7D1BI}W;*AnG?;99cQ~rELFk z&nRVunZga|=KYf=4}-o@3)`iC_{TrtP3awNV|Z=kL?lSK)O=`#N$=Pdu>kFu%x>mj z(TjL=Rq0O5K`tMkO;=;AbdJ>~$J4bNt?fN3dB*M7-bnVwA(sQBWr)Xf3Qi(E#* z7n;NuA(3({B6(DDe(#*yVaOo`7J@+7U{VElli`H_Mt`NA+V|3M`A)iJdPwO9H;DGBy9sII1+bBv5@P)r!d121(!#}ymZEDI3o8?e z53X7Gz_vW$180Vo;HTNUY_R71X$C1sv5FQykL%C0XS37MT$*_&%iONl^pNf}xqKnk zI{u!3&ZG@Fnbwd8?#@A%iO1IZ4$=jQ3TPP)vY;=82w_*~Eh5!llua6Bof-8pJ5H2m z@l>paxXVQ|&Sje+0?y@|>@1MebtyWIrj8Vt-+T%;Z8zzcF-Ms(rJk4=Y7s#!PfwP8 zA{cNqlYAy!kMskEEo&FB>rd{WUpbsqMW9rLRDvvJ%B8(pl|7Keat4tiH`)!YqD!N} z&F4Qb!!hP_cbRxIX<;A5MJdfoo!9&~u4+=%^567dadORg? z86?CS^e=~h^Q*7)P*N}|U}gj?T0Gz&Dk5Lra70R_md_tY3vDkGDId=i=a$` zG7ezQfMZNFpe+$?FBPuK8~v?;sP5CVMA3;i3;)o`qChD{=R^`RUH}X;HxrhohxY|8 z5olDvUuXdwB56jJ8;6FLi$W=8kQC8x%8SCM(R)rXoJR@OD~S(tXB^a+^36yu;g|({r5F6Qcb{pwORN>*K5toq#F&64q28yO<9!@GYd16 zuVn}X_goIUsr>7E-~W6udl&@;iLSri1Q7+BR-k%(PB2G znb|bU?UnOkyqg6g8M|d3i4J}KaxUYCH)UYVL_llBDQwJta%{rwjnTaGj;`L z*WFITYIs>3FZ#m{Qj-Llq!$|>+sO1emF|RdZ_fi;hsd-g2SDD)YZ0I9L8k8YDh^V& z^H+cU)zgzF4w|_%y2^?_eXCAjn4%Pf1Zs$9K3wo7iIx}LgOZDb?{V@TdK3&R&XIFX zVCgU`$Z$5`3h^G@VK`Eh2W#o(e#_CZ1IaOX*c9)Cx1zvAu!2&bU{Wj;#iZ2KDU96^ zNy(;Vj=b;ln8aVQgc&Rp4{{O|!;#fHi8l1PIMKAVEClb}gNvvFbAZsrrit{daliWlo%l)u5BeLTaoWD&T;Ak$eoAD}<;O;8C?5~PX{*k@DLd*k?cA3P5Er|var z-`YWfQM6h2XJ?=Q06+jqL_t&?jxI)5Z%u^^ekb0@fX>ab8K`?5liFsZOBWO{FVcI9BLG9{$*qv*2wNpDYsFUp)f$sg zL@C^lMXg=eGV&hG$XKMxUKW?B|KRR`8X%R zDp6lIz~R(nyH!0rdRC;ufy8JNMNrHmpdv4Q~ zA-vGv`tj*FLZKmHGt0n?SjQOtwYbaUXj?1s!^(*b(_(mSe#?EF+hRT9#1_yH@_2~4}vs6&n?-5Mrlk{xFJ1XzBcEptB z#pq!y&g<|>>HYuqpZyW(w7?EwlK0DHK|9`8c|N?la!V9a|MHT)4!hu+ujbr}LTB0y zEmp1WNwe9pzWhUbOiSA;>jH;kd#$e@>`9c1lvH-ahNN>k?Z>Gng>hMm4n;8w9J~ZG z>fGzX9m@&nErmSH4T}rim&cV%=R)EHnFdp@@k=O6DJb5%Uf4rz5$^6k|Kigp&yQ$v zjk*wKDCw$-qm3|k6_L`=6bs}%wccrL;cjgq=C`|ZFx*a5CYctma^3moTr%7)N6G%# zLFGo8#meCea*QaT)Pbo)#Vl?y0mZC_#b=ov1~2eVV3iaSvA1?d3zOK8sCy^~idGr` z5lv<=8>(GbRwkeNtx_V7nN-BO2X^;XcSklQ()=QhWN^GRO7`+8u5yn3{WkP+k+DjO z=O}@EFuDpx$B)es)Q=qsl}7lqlHikUP{@y8k}%ZE_X9e7cY%-z|NO;rx$Q>1Eg9|| z)UJ8j2}jfLDtaeQc3ytTiI&B|dv=dWN+ky2RTQb>prb4}$8qg^g8P-2=MX+>nfRee zz-&Q%bkBiW!HfzABJmI9;_lG186_dQN7xhLvegfF^Luq920`2(KChp=KJK9e(8F~L)F-SS^CF+ z`uh}yDnwpI;XJH{7Z>yK)U&uJAbeplZ0Ux%pQ`__r%J`_CCFGCyeeN+t(IBySmt}B zRe9&3Y`RDDFlhcA&jq7oqjUW1>C?|mM|3`wH{!i%Y;!&L)a?KDc-j{b^NcM|xCU{g zuBl&S8Sy8t`1s_R+AjbYUk%P7>+8(LeX%>rw<42L@M!x!WR)qh=+10z3U`Aq=W_)i zGeORFs;koEhZx-<8>|I8J0)CQz8+1qAx zrA=EURz^eDB-yp%3NHL>X*dBZHE9@t1^&Jpw&) zb4sG!?OolvB|_+ZTN!Oixdu>H5g+B?-K&U&I5R}Vk~ng4Db`w@K9E^x1+2YN?crQvv$@_?22+|+sy5rei+$`S`NPaRgw+L; zx?^BU|J}d%h1Ck*iX{$#F(n9E^ZWLF6vlc@4V9xJ|5|wc*B~@?UMvp;k3f51Ya_dF ziR`aa1Wlp2V!Fh&Fa~dX*r9ST;;7W9p{T=zd=|qG9ZR{rNUEf)Rwa0qWPtl9Zi!j~ z3@MN`UkYz7Qy&7F-|46utO1Fn0Oc503NBfPcrWDD8d|-g0>yLCp>eE=Xa{oWsbx~- z1HB(kRo-t*<`QUYbFc(Z6oepNMTK>L_Ts5zpa(Rj_u^uJ%~UAk=xsj~7OA`{I?wS~ zf(P0#KlI4XaNuv&BS$f^g#QJFr4H;Z7HEW0bd+*RbOMaGYlwxLWcUMeSb#u563P;s z1z;P0CC>#da3Y*$YB=pbf^DP}Gx0~!D$a57xM4XKjR{#asdLXoNbO8<<2jGs64xz5 z#dd)cvP3vF61`T=E)eLLk@%qg!le5IKNL?b=b52P<56Gg{TVQ>vs1Vxatcnd2Xnw+ zrKyTd;Huq|8WGq9C5rJT(P@#29GlX+ZR~0_(5WT%u-3D{uHyJ)M}uAE%o7;)%5!;Y z34Llhpy2_Dn_7Dj90qi^8z@*@$-`vwC@&;gXV61jhk^r$<=rbWE#Q<+t1?r zfP2=I1e(n|i_<`uRo0qT@w-2KU2pDpx`*{T`rO(uNS4illL^k|}w*p~m30h*Z9K`({4XL5eiUwA;wJ@qM=$N1B*|O#+f+PB*Lx z_<5`0DPlB_*M=5gS_aMVH@dxJso$&Vj47+tn-HqZ?lF-wbD+kAnWE-<`hzzilefQL zgYRa0tQ?;txF3W}DZ1D%7)`?*aGJ}-5sr9^65NZs@n}p3GvdRELGY=>Y1W%6yU45D z3909kQ(U~bGG`+5H|H1v{6mrvHzn?${bewdZA)Idn0Fb7LLiL%DFsJqb4|%HyxVat zB0s?$Yf#X(>^xhYexU>_Ua7B6cSiIuhP4$|^|OJkpTw-Hhe{ZYH-u>B$l8DliW-JGdRwKrI(2Eg#T6!ebvz znZJWNnI+ze0LLf2mtW){=S;_zu5t5l5cZ`EQCQ!5C%d1bQrg(C;3{-$@Ze?b(y4iy8~k{l6sTo zhZ+g_EPUFtIW2G(z?d{5@HOm81`|OhGc>@GhN@Q+(p+D|_nYmGnhs5!u`F0m*ce4{ zaemI`2$1Z8INMq{^EVS60H$nwqt*7%giIqTz=Il&@g!_?a_FS_OZJXl%LJ7)=c6D9 zrH^nj{8I{QXzb#RS+m%>H>0WN)@)NBE|anFh1@K6jz`G53k6iAnY@qfzZ{HcL_$k$ zyvL4GpYeF_k_XpQE#wbo* zWYxWpS=V$qHZTs4BOaNiV-q`xh!px0Y!)bI;PFwNgQ)H}_TI>LI(*I~uRCzb>c;8P zjk}3_=+W-(aE7GXXNfcCl+~gm%6Zu;t3~7lJn6~psO4^OW>c5a>0|6=}pLK5qvGn zZ087!XQUjNk$qO%hejLhh3gmyhPa{OvCy_w#))8;gzXBq}J!}C&J&QxcJlIIO!LguCoDVllK-`ICe3aW%%O{4) z+3~XXK1h7Cj)gsQ7_5rz;k*$(ZE6`+_=YD*03~xxE1+~(ouYFo`6cs#4=EJa%BUlL z030+KLMi5`nYd1lXh0gsGLcJ?oU`s|r6VJ&#>1owywBTQ%9%{?$e<(_>%qiOyb+F^ zdvZOUXIMhG4wyt;SdKto9}gpGE)L+!AmS-?cjufM{1c%M)y4*Me^euSKQ4X@mS=xR z;@*yh^g=fz@&X#mGp9LnPGAnHCrFx?@KIn6A-`f(~smw1bT0@3CdZ$d-e3_ zxZBX!;^4r{IdueMD{qO|1Jodyz$~EAShuFq&j8yYSM`&i9bw?=@gkzshs?=Rf}uds448I08;D8|4l?*kU@#t=Jn}X7MAwh?RHBX)o)S#t@8$ z8C|l5+L=Ueg5Fb+MKaJ8YfS;1^dlQ+<3mm_I5%{F4&WrAb=0vUB-g1EW0^gzR*Wfe zjYPvKO{_Rh)UZ8>E${&QRq{^OF}VO=(T+&>JB%UA4x2QCA>FBkP8>xVs2ISK)_@%9 zgx8qzRGW*Ml9=+NLgphbaf@9wiafe3M~Bjhr&|2nDN1S%3c`5MfFzKhGK92B z7LUh5AveL4`izP@&JT)b;#IV@_(D<$ zMQ?APA=7Ri4r9lmaw$L3<0Pt!aJX5ooQB+sGnre<6=6F&qDN}c6-!3)aoe1H6Rh; z$e@Y7v%03x09te2w~^%B$w~L{)w7es!-|B+ba}D`^PrX3{nhX$eA0JEhgXB23PFwOIwi8HIHXRpa=&P>M3;105Jt1nP+CNOl>d zxqgs#w+3#PC@hmg-Mk4pLs9ZZW-@V1tiOl;dwYj_Q*Ly&_ZrlK=f0ZE-4EIF276*t z{e3hVl+|SG-L-y5Bb7bOhMRP*DeZ0CUhh^cILS&fayFTGm|MLdOf*pE7Guil!iuKg zEbRjy6k@Hn4mxc^fv$PyD2Kb-phdCHeS-#Ze(zN;Lkj5GR=u*`UA;foLUgmJXcHh& zCcEGdK2?oC6$!?7A1?<3W62k|XHoSwKdxPfris>zlS9+S+(a}}LPak}w^F~MeDI$9 zD>#1Cb%ClEhYh}(;3sD*CLvJ`spn|(6ks5XDlza$rpAB#O3+5-^8^T$x5wXzznw6c z#R+#}waO4L?F{F)<(fs4h!cI&VmeZW01$3P1TZ87F0>XYY?Va1g#k7s5fspjOe<`M zz__`HDZ>-3x9eS7GUJKO(wWPBv)O)C#GXED2u$)Kc@z{Td8POy!A=~wRIINobb-DC z*?0-wZv&mo9A(P|bG1Y~EcmmZW@+=C+%Dg31^Dh+FK_p9Zz1crlMsON4(z<$63L;( zL=J?_gQ6fZV}*Lm+FCHjnSO{Hn-fNm)66dT0e&RXerAQC$T1S}DNf8}DVOH6r4dN8 z+ijUp0h;^=ID%~LZ2?T3ReN7*w+`i1);C-^*f^qBjJ4hx40*W{B{3acQ`%|&<#&;y zP$b;7Q$}dY4?8)_*Nf6W`J*4&0PDpYAUQH-_%RuceYe~(*SFoaK%TI$Dvl&C9hKkN zh?N3gx*(@HMIUQvdcVyi~;863d;^(X@!|%e6#+L|vi-C1Eli3^O1Ky1 z`^l@Pdv+qMb0FQkwY2VC2Q(4nZ_Qrr#IznW+V{P zfjgA8_N=K;H+v-Ia4U}j$Iu!pWKs6g`DfYOe_^yZ)DEyDnIhKou;5@aG>OvUMX-PO zcE?{_#1q94qr_5PnCz;bkUKZ&JFK?AK*$1wquFV^MQGwahImyput<=rUMc;=ELZFV z)h3?Pet{<)lbNbWggeKtO6Q}o+^+`{L4&qYd#_&|x7$_Nq()z(v0fgnmroBWJ8)FD z258-b@w$@)tJ2cD5nhag7}hXdvC(m(rEt>+m44uVadNq3p1oRaQ4?$yV7lF?!+iLV zK@I=QpZ}CQX@)!9CJdm`n1wV9SzPT`>QuTDWCPeKyGM!c*VRLHA*J$;-dV2W-Ry8# zuNM;oyC}5lnPYUDIc-SUfUX9jx(q?aE!1i~M;h177hc1M!<4nYSZU#BUN5G&u)S_4 z7C*B zZo>D47%nUA#m=S44fV|vN`&yRnV6ejw!YOc0&owqNPZ3K3jwDfVZIaC=Bh-H7ZoCrK6-F}M?5WQ6CP+s1rI7AqSv?ue!lFbq?Cyh&l%>KH z2J?GUm=ro>@@_hhMFF{SWfVm{QsK=8rDDxIJp&glB;2Hq-4WN_4JTtAFD#z~v$~A? zl{)Tw04kcedK$__sO>Tiwx@PkdjUmyG{AHN z{UD0zbmSCQ2$E{QOmj8m`uX#d<0Fl?9pcq!GMY{$z&5`4-qTl4n&pC;GF^xERFXZ% zTbv6?wvJ9Zf%uXZ4VtjKSLyX=-o)gXn9f5J(z;0}u4@$|C)Sh~EHbkl9%Pavckfk; z?~m7}|0LS>KdOL|@Grra&anFKKqKvYt=;xG$f1>u3V#sBnKX262llX5h^-mtPBs1v zzlVPzuG1N(*=S?>Zyl6|qf1($nB^9Oz1!QL-^>O5gvR0mTFT{<*$mNtAMJgx9<-%E z8;$g_1l0su@{s;lG#ZN;+|z7EkPhsfjOMBe$Q=YOD2ZYSZ2TRQ=vv-SYJ~!X#~7vP zJMy-=IXOPi2!>@qNf}SCh`W|k&z~Hqwi#6PLWTdHTZGz0^%kQojdSGU&=+yk;uwY}o_Y9BNYPMSOA&9Ui7!nxJfQ71(HhllaTc4K*eqaCp{MCmS|@qK6n zfRs(mFK{~5nrk06xH<#VYPa927$en$?{ar_L2O!x#jBfNBQIXle6%1=i|E0b=!r zy;M|TWeYvVi%7&mJgjp9bxuK`0x>9%%IY~Mx^4ol?wq&CP4Oz)-OYLjy>`2s{5xkO z*{xJIc@Gt*W+&TI^w^$G5D`k9Ue7kUF&fj65ewO40)PYSF&#Ub5;u@Zxx1+}>V~!> z_k!O*uS!xH8n1R#8aSgRw$yg|Y*Gto*5oi{zA`bQdNg&e3nfA-W5*<#I1>Dz#4kKY zCUe)c+7>Ux>0sCGmdD3xNN`>p&f*X5XhDCHxI_2|9htNw`c6WC zA`g(q_tiM>D)v-|g-zw-MR1TzU2svsfj7I=nr5M5Z7tJRBE|$u#nFv8=ebtSWz?Ew z2^b=xwT_{~v5tvI>1W5!gZqBJA$<-R2bFXehm9eVrVqAi9pGX1lhEh4ex3Y*{Ye7d z^9CJpjjRxnw+uO}9^8K&OUr9%oB{vvBpwWE#<_4aRtc2udAai$zp=gFYd1gp^y!nQ zP0eYCSi1W8l%zmHrE;hKj0$2iql$ehy#m-$T`_Wvyb=!B3P#0gCXqU(6lEvdPoDM; zPb##@+3Ap9Y8${-Klt?cxLu{DRJ>#0FxiW%naV*+!?%?hpF$OHxu;qy*8e=zY*YiT1TXzxe5^7ePK+i6cE%P8d!5Uy)b4Jh)gg6IVDuDU%H8yJxH5#5J&|ZY zjypU$!o-E>YIJR{0}oj8LXkWl5DLhTuj4@24O7$8z`KMg7ym-6wP#9@orI|rGTN!sU|0XO&;L735)K|Bk z6sVi>u!Xk^4NM0taz)SjuaM1hVFJo_9UitCO~ZE~BDWX)!A1X?OPspkSWyoWC)+FM zOq&4c0A(q1fWlFh1Sa=mbHGB<)wsv;v&EH%y`z(pHfn(BpTnVHAUB=`tGn0Bi5rsrzP$mb zo2t7D9U6<)lB`+*l+AX|G!L~&B#ityY6u~l+k3V8u2n~wvJh62&9dYXQ!Ek>Yo-$C z;6V@qamsm?ZOe&3Wp6>oqiDF>RH0#a^ZNl zVmDwj_z$5}e*(rI?O{*Cz+ee1C{vZ`D4a?Q8 zLj9&==ASO%bTg$y+aW|9btmp7hp>2Y2Edo|jMvV!@={V;m`XpOKAn88D3PexqMR$4 z`amE-KlDCU*pp|J(nyZST|ZhrBEgQbBtGt0$P)j;Raix;JkML?R2k)MQ>?Y|X2yBK z_jjB?bPZ!J8`Yi9zIP-p?IQt?l?Z0TQ53| z9#=Dj6J3q0x*(8&0vAE5gj)Khm+$t=jc%`r7CY9axsS3eU`|Vx?!ig-=*f?N`23{T z`gr>7+ly&=r}40V?fdQ8VXvoyk1pEqduWWVYUquxKY`MB5M}~k{pylBgeh=Oua{hj1{4luJXwTLEBVxn^KPAp~^!YW3u#t<9G5>Uyh65nj#AxQ^B~ zA*-U0v#b5;fOg;1e}jncz+?sGLUqi6r~oj zz7CZ!$8={;a?P7WHqMCaib9=O&z=j#CKPFayun>D@ZnENUioc~THb$k!0Iuu+f+?u`34^`PDC{M|qN>653W>D9Dad5RRZwPysd>Gt@8Dn$7^ z#kvBiHZv0_c$GwoJ0;~dRN{{-0hf)&^W)^4iKvDsy-gXxNb2;wug@iVbO7%qNz{&8r3?DMfFg?Mo$=mt(95 zkUGOCUF^{$7at=Y@?^bOio%B2s=ya&R8{TuQ!PP}7xs5srZRjJGgr~Y6RGD=Z*+3ghd1jl2~smj#ex!QMRvK zAl6==NKauHy(m}w>xv}4=$05H)+GwR&tafC$!H3igiEB8-M+)i>0S0MH8w%W%hT4bV}nLd9z7+$=2cRC!IONDV&e*N_D z)h7oUK97bU-+ucWqgh^m@5@(TJkuT#^#`yr8jRg|Vvao0PMXb0-|7&%TcP7bn68Zv zbA#)FcyLEj-_iKvJ$rvwtzJ0qK zE0b44zM~+pR%>eJM2pO?N9t4D9*yNH1q8`ox)nu2kel6}BO<*aEMsJeVzbdvUZm5X zwluV=h8UhpY^~3QEaKe_AaT`gD|AS!d5>0rjt^Yz#mv}tElyWgH8xcC1oi1-09Xs&5h}Y32 z+>*dQ1?9uTl5b07=zyGT7&E#JMdifKQ$8HUn=`q$)%D4<*RLDak^`!KB8Yb>eK*GQ z#le&E)5DhWE9-K3zHGFQUVrwtx`VIo|IPpQXLSSqn(eLojkH<=?H`|P?QdO;&sATboH!Ce(vrF-~N6vyOhTBtFa2_ZAz$WeLw z_Pu^ydt1B9$>RL>w5&*@Ug2!ZRdQOzjcBZ5Gd&F9_%^c9?l}*R`Q6~0Oo+iWZO3Jq zP5M))|K;lF;Lu^#GB=F7t(|M6pCvNU)?!Q3o*|u4mFk^~p$y2IOa=pa*9}U)j0%@> z(!em-M%$%ctA!IYNT1}J_VbfAGV0ak@@Ad^c?yVCC)X>@gQu(O`3Tf?yOzBpAB2V` z+}^1ZU``NGH2=6!z19^`en0Vu)H)$>1T)P8Rec`wt{jmuVu(NzY=kCaiySCc9F;LA zB9=VvfZ|q{Q2Wn$%Ml7^H73Y;@Qkvu{^E%GAFb~ol7(pRP#*c50BHHu^s~utd`Ouo zC&b<)?V_4~x7BPi6xO-m#K*st&Oy}ha$Nx-1sVtUDnoIL3W!ktU%LZUm~D!d$uzKpSteN^F7NUCEqhhdWjr91nJ%_y?W z2@1&0X(nPAl}UAp{usUD|8mab*>GHf%vl82$W4lictHL_@YS~o0=)qrrGGyA2fzH; zmp_3}CME=0fUGNx)_PvLy&5!j$~$<@4~ra`+Q!ZV0IKeDMF(g@L+L4k!f@a%FdB4I zx)rA}6H6QJ87&sr;d(weU7UTC9DaR#T)R;{_ORdHFSQ$c$5O|Q?b|tlk!onI9+GADs1>F$=w$o+}T^b5sO<7!1^p$Gmf8^bQj(`E6Va`j9u+7Dls* zbflO^Di)2-pu}vj`3HLy)Sby_$ZP=YL{ZGy@@o3c@G_lxv5l3pK`wx@dpeolVxqO` z4U8InQ~Q_a#n{C4LXbHgF-GNOU~YT19NjcRJt;ank?%tNr!S`@60zPZtGc{ju4Lug zPoBJf^>TP|e);~L?i;!kGZKVAAF_Kk(upD>6r+KxvW3jct#g|!DS>5OJ-GT0=fQV# zjX#(q(`{4%;dfi6SB#^7=Fmp{S)jN{UVcKJ z{WIs1wMzT?po>nm+)bfzco`Hz>|tI|uc*lcakGZklv&>A)>61ZoK?i#&4KKNlq6?$*i&7!Q)T^?5SBfD z2L08_Vxba1P8XiP`4HtBuo)%V`?2w%`&ItzJuxhJwR7#&Utg#6J$3 zIx#ulSOYMN;6xL9sJJ8&u8)E^CSEQ&i&vyWRYBMzn9KkBm;Z4x>hT9bk<^v3*u1Bw zo!;@Y6VNlJ)A(|Fb@uUL;Qvkm8i$!tjHDEq-K3F2`(on6f~R!C8euwwPK zxHVn0Kf2Zi{$X#sc6ihgP?+JC_1REhG*Uid@3!|Lp5+BM$SuN>x<`jr_U+ei7fX6B zu|^6SRytcwcsO~6mhagI0}vO#4x=iCg%zwIV&ue{PcXgeE1m$-zJ2$N3q5^yaC~%t zhkE|;!L5!)_NyL+8Th3xJ~+zTxI42jH`6?^f@-)d^vH-ti8rA6#3{q!cJgsGJISaxhyg5yKle2TWczR>a-%byK@K=r_GK{m-lIk zQ`SUeem9?y2!0zKR@PFks@(>04Byf|#ewv_qJ*29`0Pn}QPmr27S@z0xd@>qxko=N z@(nR}Y>!_F{IVe8lL8x$C>AeY>mxi5e__v=D<4-J>k|Nn-mHwAEf7AjdFP5=>Qp2 zRvp9IFilE^J6AJ(RNMSr)R{X!DW|lJf&dB2Ch;`I^di<0Vw8E?96Otw-#2YJ<`kuwB7JSnO9ohs_K zoS!o<*;V`Y>^pjN|N7d=@3aqx_3`lPB0+x$diKjJ#GU)C z2Mx`}wmB@)EZ4Wo?(yNlv*YUan2*;VD^XeE9+bDMkO`qp_pm2jHn=#S?dz~V_d${% zw`6GE=1zXC(SWWk2#`oGCNyA>Z*DW2Lo}PYu{{iNYT4z|6SiHKu(KoeVxlm_X*B@(VkM|Ef zD$Wl(N01Ut1qO`o#%8PemDEVYYh8F9HZ{thFIk|bJ{UmPE5G^edGHN{QM}~GIZ_1; z(lIH_lsVbabM`voS-iY00b~*oIZlOd$YL*rFHt2$wO0;aUcXI8#;Zy1YPK|}1gR=c zZ4vY4EN}=37VnX?`LBm4fsfR+dK)y^9772%rckmGErAaPA_U&}I>{SLEhJNCCoos` zAhO_>L#;q%+QxU!KX4i9yO}d%XSmO}|2}JBVgU0va@T$%tz{z95YwcQSZ&DNvol@PMU7bpGCv-S()ObvW zs0NP9omAcEv>M9^xJ0E;=QcG3Fs~y)I6B8@qCaN4+i|d34#I&&SR0_g;MRivRra-Ftj? z0U53(FcWoKXas-}WNM|OjfFbDphVTR_OfIGvJ83Z9v!oFgb?=y?7ds5Jvlz%E`Im* zclXPwHYJ)0;XmF!+&1>q)OTJerk^L+JeTOPXB4xEqYc6UDHNzt9e(1Q0{k_Lt z(olYq(<^@BZWYGMsUTm4ocW1EkE@g?%0DUT!i)CAK9a>K9+q>ceb*;14*%}Ye&@+?^TXSB!oKZB11afl zqqW+EG>kWIK`F+l^M5lhUKWkf)^6gmXf){y5t`D;3wKa^6wt0bmS@yKPY{Sf5Tl#~ zh368qTO70!pJKO#<#srmEGHH&bkv}>-qC%qy7->Q^T0c>gCb%=bwD~Vm6-~9vuuWFZ6Q3EpT&EiY7k; zT3#NkSL!^=V&G07MhjaOl_$D+Rgkk^o|%YQboRyUu4~!bpKy|++Dj5z(=?A-ALyxCAXkNz-OW1uBN%=IJOX@1i7DvJ} z=1Um5oI|o6?A7i0yVUG-hB{e*k0H6F`d*=xOcVeM!7l$X+| z8)yJJ8vg35uYl*x+RiisLVX<+Cgob2CH#0@ZKHRR8QU!&E9CKF(LhFC?uQ%#ogSA6 z!oRr}BDkbA3b2E1C9w}TYdDxI9ki~`rl)Ui4qhCt_BZDWYX);z9iDD!^SWAFSL+*k z(Vv_gs^Vo7b+D#7rGg~=c@!^SN@=j+W+-v?35{jhZxf5!f4_(>KIjt;?+ z{fkT3f%3@dba{gF(>SoN;{MLTvEU4!*z6Z_h#<6mt-?@vU&e{Uvgc<=Ns&3}C>+tR zfBluVyVCs~?C{&0@u*LXVMRrQUjT-&*ua+-6QbbJ{mf+ujf71#MT29hDbI3&1BBGp zvFEeFm5l1s7f*^}sr7f?d@FKzaoicr1_r8tYI+BqLO9kJgMouWh{zrQKGa?gWRm%X z_fk-w?V<;lg7FbQ&)Iv3V;_+#&YQyqZjj6ILGK}YzAo{kZT zKTAk$Zquy{wK$ehH=^BqT*jZ7jHBsMJzDMu4CC0vL(XGasd$Ddv&zI(n!?lDs$cx^ zPrvu-L>G(Y6|K6d=0g`_USVBd+eYc)?A>}}K7QRHgL{w0X`!f% zZL?GEHW&Iq!DI=NF|n49f-HDR%) zQ<<8~ZkCz{t^MjA0zGnl5$@jCVvGxkb?ce@0=WT8`sy9 z{)IW)TBW!tprNWgB|t&bWu}Z;#iQINaO%PodvQtD?FIl@N`7#9`uX4b@ejZMgLl9A z=Au&J_0sHNnapHY($Cj*)V#m>?wwc!3Axqf)Tjf_oKh3mISE|k-AY9=aCyntYx!R5 z^$x}I_!yby{;Teyy!zx>quo&Rl#o^htyz|AAiL*$v4`;h>HYXx zjtV*RD?f?k5km6r{1oOcLPp%pGrG@I>c_Jc?<)dXpj@diFigAVp9~r>*-AXdPz2jy z0|WbJzm4`bXhkGmNHKX~=WAjY@l=$91`VlkA!S7?sV%7n&d4WUd?tq^FX~nStuV2A zy%`r;Uoc+2hVP{IVU6HE99+^MNbfQn?Y7xadgfn=J4Aq7@7D2g-3+d!D%Dc0v%j2} zi1p@%em=toY$$LBwihJr^eJ^8c|1BO*D1~+16YKD46${vCjKPf&f$1G{=MJ(y_e5B z+Oy(Vc}QI?E=Q~S^x(;}%J%K~#aHHIX+c!E=)ajD z&u&=4-StA_!7w6M=ZbsGgD{h!(e4~%6fj1PD#95=Y?I#(!YVbk%;K5dtWf_^L@v&j z-FnRfN>=o=P*WKHu(!Ion9VLXx4Kh&(UKNVQQnBK@Q!uoFy6^%U>4`P37Sp2+uh@* z2OXWj7Zc@}8#)E6(xeNwtG6@x-)pWj2vujZO4(>w8sv6oV|#dZNpVXOT^<3jc`%j&T&^tv>g>M$YH zGDEA~kgwNEh8%BIHutu6WBJn&wIEuV#O~SheRHdxr{!HNq4cwJO{9+vBUg&9$CzMbw$IGhK{n@l>Yf z92Hsk^y$_HkF)E)_*itj{;h!&bA+G}3?T4P2pz8x$g6j%UUS@4#?{oO8PXiRCZQq2 z5lozZb;fH2tGK{DCE9WrnacHQyuRD>;K(*xyaj$`(Oq6prz%p@0C5thodG#GI)480 zwN7ywn&AYx2WnxV>~b<|HIFmidt*beExu@XbU-m{eX2TIx&h9X(E%ovKiu*##|(I_ zg$qSC38){-@m8AZ)2~PGmVFvF`Y1}7(+s;_-NXj^Vd^aCl;Y)FJ^nqJRS6P(F7H;A zy0dY8eE9PBfA8h(&EVUwKh|rAz#SoFzdztTG3c|4hU&E^FP~H@_s!FtZ-4#m-A#Wm zSlUH(QQZosU|5oS+qoSM?;Ew7HTA7mC`V&+Q=eui;sWZkPifMUL{jo7D$?oK>yv+O zI8b$;Q1GASrS6t@KwyP<*LPY1l}q>O>?KEzy$n*VO&0zB#i^X86PRx&wwLD* z(7(C-_O9&kwzkuE>%oUJ*D@zec^m zy^nb0QbXeH$<^Sm{?o*`4X4onIT<_><88nx;wS+K|K`w;#c$rcry_I;veY4|sh8L- z&u?!pM^~y38iGX-zeE!_XY-+b)UAE`;^Oq9C9w@Ah)G(sYTej3L?jxLya! z2TgQsz^Re%AglXP8RfP+506(}}mcIPs=aP8+w`to$O<5}=P#nd} zOHIoM^ZoJB;WIs+m-Eeb@6&Gg#rto*e)ILOWu1=?K6!QY{9BtFyD1DSRTh^qxI3aQWr2pv$p-E8WGDU2$}Pl+CY$(l++;~p0C zF`@*`m zGC1mCtKL$NQ5g+^`B%IxX*UXgIPBAry5mglCmyxiZQ2wFtrITTwX%&pgA1*}eWhKy zQ)h)kEt46D;P&X~0Hpi%*Kf{0T=cq3tl&-n06+jqL_t(-3dwc`qFAqwO}p-O#9tbQ zYUegr!LU2oktbGiyg6wo5ddsJlfPnEH#h2?2Ie?g#;cc~fAaeC|NGDX9|svk;oh}J zg)mVNaR{Sd`>meIy%j5MekvD+FLZL;wjR^TWYLk4O=P7{0gFYPt@5K71wap-_$I}o zMUKu*dPSucKBqtF2p7sOhQ*3arflM|=s6w?0srJ#=jGE@uifCvp7#gBYJ^%}RzxOZ zCX@FV{)NN9qSdR$F~bZMPp3Z>4Chu!DF4$is22EFP#Azyu@p>j9y*pe*T!`0>e zmp}gDr?1-Ief6vN-<=x75p(*0+G+%!;aPl|Rb{hrvlw2yohY$9dGYMYi|5b3|K(?2 z{!yv1-`uU534c`@{Ez>`Uv3+MohBhHE#H#0^F0n*8p&{1?iQMiI2^3Lvdq+cWy1>6 zgOU^xy1$Cvygjvb!N3C(D~h@}d$>HG)UdRdXQ2TwlGA=#3-ME^YL()QMYtJ!61<4C z+~@x^gxXcYem_{u2bZRh>sZuLM{T4C_+v*;ahD`wDT}$=ZnhgP3eN!bijVz;G#njt z>&+fe&Y*bBMNiK@a#I^s%e;m3Y)XN&k6t|eMMVpnX;42Jw^}7F`SErbZc(3JIovlP zEPiPMe7)O(aAX!aW~Mw#X_~4>ZSkR>GHAU6Lm>!d*(BQ8_0<=zpPdaZhdgk_Rs8{9 zkal0{%sDX`cfCz~RwIH&@~8k{G;U2^tRb=gVGlfiLh3fBvN?&$EwEnZjX=$Xq)a6M}=p zkco)~2WYJ(SE~4_VL2~;^VJo0As2P4XVDdsO2lv+_4o>@4R)tgE(LTsE!v{OD{uyI z-!(y*;Du;MI39r+?MT@HSINrh7>#fbtF8j4dVJnT?E{uoJQ#iCD==0-WRI{=k+69* zi7wlywtn*C7y3I~oe#+0X6feW_z-gV@x$Be*^SIKKyF`rYhvy}yH zw93c#m#3$Le+LT~zB~Qmb@OI-v#gLI6E&o2lJwVxe2q$T zSHqbq`~xyfcKdHjdQtD!RtLL|nRGgHHyzKUemZU06on-8;D+o#>7V||4^{0_1bD}2 zAD|5xYR4G@0g*R25=btDc3%pCL!>~@tJE#AlOQwWAk&!WCcZPr+XNU8OxfxKDwxd) zP@yG7l9455(-NCTr`2KNg1Z%QvFOHSDN0Bhd1j@KpFHavod}O+qYH^DwUvD|mgztT zHvqHx|8}=)gO~F=yjFRni@rm3kIkbg&1f|F`0?YLci%oNk@!&J_KuEi>oK3x&S+%4 zv#+WJ6k#l;ijwfuAt9)Y=Ixwa&cA(gs-;xjS=`0swomFpcje*~=POC}n6wpiiP8+g ztT=N1EKZ$7UA(WyGikpy1NRQDW=fmWeP`8eHJ==JUcTscyAUbZ{;Gm&i-uU$OFGJch;rDmwM0|1o>TKMJ{xeq2-Q#c>9I}DQrG0j#$i3VG=Ly}Djlm;qJ%o5;XWj90sr6una{f-EE-C}v#JMt)!DL9YC^@XWJXmPHk%GogFf zbWr*){@{n7zwW;M=GXJlUGMnl<+EpUMMei4cDu%XI}8FJ)v3v3D^~6FurMypzWw%B ze>MK_;T90|>%aKs*S`|ppepyTA2`)v+oAA;lbYqluK=z?VQe=eT*U2hj;T&aD^nJ< zS+3X#M8>2P5s^~bZ++5sTDq7?C3G>TSTg*JyI46Pu_6Hm)g2=19zW?F8tBrj*)6&K zW(`a=ee>(L!~VoT032+kaIZV$tzTw750W{($RFpwHb|MVBXD_MmWFZzTJwV;4tVAipgLiKR6*#~R8KEE8uE(%c&jt&HbvtqWy^B14k*Q>!Lj7k$U_~Yh|d*AjZ zT=>ydxCa!(us=4SX|g1#M!n}MmQBAPl4Ph1MSwr02d~1VX0L-}q+O%IV`W85n;Tcd zz8ip6Q6X5$v+MO&-;BO{^TDyA5df^Tk{Mb8A!|qH#kQn5`C>SlCFV)Bp1u67|H)aR zO@;N3T9#sA5NeGYb*t&PWn)?Jf)6`wDcu*(juI8$-8<>z8)I8VCZ3S~x5mo#&p#ri zPDYsYO3Ic-DS_LR2i6D)SR zzJ$1FJgZ|ZH$jl!Y*Z}{jR3@o!Bc$gePVB)k$l&{FIs~N7E)`BPx){VZl)Y~X}b(~ zt7{#?3($H-2y;1{U5bkH`FH~KXCPM2G~=0&%bE}wx;ns&m%lrpPN;9D3y!-_{_tl% z*egx0`g1a8sZj!%bhraG6cy*=iRr?YA41ss{ba5r1n%$nA|n~5bc@*1J9=`vcze1a z`s!sQqU=jL&{4P{3i9}fa@rIyfQ7E~#o0K1xF4zl7qqtU4(eroOhE>W32*Z8db$p- zh!IO^K!(;T+@bDaTN)JvQ_!Th5*lQFy+jgvvmCDCUfR{Afcs2i4prHXHWG$8deUXH9d56bg%Z}QkzJWNqp<<;xY?rzMW4>w=U zZs+jGCeci%-+tTw@TSqMx~ONU)XN}Ir@^qu(Umf88CE5;OTmqNnVqxYg9|ML0{Mj@ zvU;c;c6`>esWBSOwA-Dk(-wlL78KPqx@~8)nCiOo{&f29|Ke}X2iGd)A9Qe9-+OF5 z3x!_aeD|Fc8lw%=X*e$kS%nWnm*ct9O^+|znUq)oHn1T6=>+;GrAT2V6nWU_)Z4FL zK0)|+_Uwt30eArgFHS$qr%P=s`GuqLK*B^HnvKPtqGj$9Tv-3s zEwzn@MTD0F=;zC@s2W&mW}w@Ed1>lM4-Q2rw`ZX1J+po`lv$fM$729X@2)^odE~_u z^O4LYx0!Dw&IthXBIJaJ5ErT{^-;D1-lW9bsFUCQ!>6m;>EC?w*R^sdZI@PpKbUa7O?w|MNlDN$u-_2lcyi=pPu1Dv050$O^J&A}iZ*GNsA>XrwU&Xof zS#%SR?byNWi3O~~-FimnFrCJOhv{;#F{PuxO=N=%G0qf4e0$?1x#v|K$s{Hlb^W+_ zfktZufRW&J&QL%ZAt0)tRw`(*xI&!;fY~_y-N}=l6&UnCz8{>)?KAE5N%i#f9Nyh& z_Fx#;7tURJ6#mKYy`~C`uv5o@ejV#e^L z<bd zXGwX7+nx{3&0?ob6z^$cj|G1@z5H-SUgw~2wNxtlK>CZhO6m$0xG;}THM@o`5~eLi z`0@fscGuGb^yGNbAAb1mgHpiZWO?@fW3#U6y6u3K_Nw1}d-1RS)t{Yy#D3TnLZ}bn zH;P!fmJ}y+&V>epD||hP8II*}kdcFMj;QGW{GwU7D^XR-O>VH9O6+-L367r+@T^KYns#k+f{xk!}Gh>UrW^@eVGr?*|wW+jW#9GeP8i5GIy4SM%HV7w-Cg zsPWqU!|Ts~*s83wQY|`|8<~n&tOb0D3Z9NNA&^&^q-gONKV{>g*J(rhivH9qzP|@!Vle7u=>yY**pR)$6+qSWok$A^ zF3gW!n9MzM<%v|T38rF7aUOIDTPfW&$&3QLy`3+#2lrK*#1*@<<*!=#84*N z*zD-RCZSquH=H*X^foi-P|P+@-@QLO&D1-=)qb^!iJ;=#+(QjKDZs(y6;=+Y0L|v$ zuq)p=9{2Sw6I#F=^xe3gT*xjPTx3F`%{J|s8>V9)x9Q>by1K^)=p;`(n!#YSpK2ZO ztx5)&LwU6pg23zDG#?GGrt`3hcXmgY7iXs*Jd~~~dV$n;yz8NVF{ss^Zfy4c&A+ly7GzhNDfA8|jD?^h(;IMe!zfQKf{F_B(M~^}V)Rj9WKDz+&KWp%n)KJDs@aR1=3R=dIq&A~>hZa~r?AZg-RmT)Bn%&JucQ!kc;*Vg7sV26 z>5Tz{%rBSR5sgta{$O}9E}62C>2Fn+D<{ADK-_)!9^J z2mavg02HfK^5ANGHPJ>2CojP#kbp;E!zgXF9>s_c>=QjmwI>jZ-(RYWv^$UrnBT+Z zCqMY&)%QN@%VgZGP7Zr_x1i2DeM+{RtV+`w=q1zdk81qJgN}el48vQ{<<16D#oTPE z=l5shqo=>~?9(sLNAuH*GpjZqkD|;tsg}Q>bZNoFrp1CWCSZ0$Ld;jvORsF&!2>1j zVQC1*!MaDbgbsXxkk_cPba@INWR;#e2s1za{>#sw9|}#;36z%%^;ezD6Sdtmo0GskX z7)3gEYb?@|Q*h%hAiA<+eUGu9k4-9N1kchC0eF>a`$} zw%KzXIYf=V+w*p){JIK;;#^aL`E$GoGp6@41MCp1q{<#(_q(1TdZd&fMF91YZq8JWyYtB?X9cb=U(ebr`nX)rl z>>36KgDFylHk2Dk`V2Nd=rltuMSqj?&42Y@P+h*K1LbUXwE)NgK7k~lRbv`axS(Hg z#+*NTX?rc3%gggu5=cQ~DNt`*LS`DmoXpnG|KR)a0PX?xTRjAU?d{>^kR$8F@Wx)e zdV#(6=FM*w*UQ7>XE0Z~O$y@d;~Cq|HKyonn{?vt!`A2$34MxvcqrrGN7|%>1KJ2zwT0qdz zlasfnr~lWV|D_6VL8gp~HTlhV?@nHQ@$-N1&-4`e`m0~Rdi}jmzW9MW-`Ux@h=po~ zb5Mc2T%P5J!Q#^|e)9W&`1e0t_WzH6_h*B_5MhL>PN#zh^K@CtbZ4ZcsBLZoH;RH_ z62s0}-J9&GEY@Q>&pCncPr~R}_(0GGXt@2lNxi%RmFVF6=g*HnecnFt^oJc?31V-Q zair;*lcCrtHOi@|*TO$JE7aZ_{jxAC>?ahU+1Ppc5}e!O^H85sP4(KInIv>DB^I15 zHQyL6>1uT~oY)%iyZb>t#doP0-I}7!DfbV#2~etR5)!DXeyi1F@Z=cXO8Q}PI#Va1 zK4-t#)Kmnhcx?naJRB(0A1%K-8_uU5YG+gSczJVvHdtu#8yq1HpL0kTTL!Tpvtv<( zS4qJrU`_E9p;azix}EL)qkr_%moGvetLThg zUxf)$gY(OU_!~H+9GqWbvU1qNJ>1a?U0?N&r<$EJ;-(?E88k13$g$y~C(q*1>~E5h z^c-lDlZkHGYxg*Wnz-yH$;3q!y3?ob3Modd&Wjf>54y&Tv(>05QL?9(S0B!%X_cd{ z^)4L6MJ(dOc0HY!)nUM;4R@5pmuCZ7-SHNTLfP5F`;hU5SHt*bvnkKXib(>N{=v_G zN`Fw0j;TJ|wn|4T@QJviT(3)oe|-1t|M_SCJ5&R!>1r^&{P+=B5~{<|I66AEa}?^c zr_U90+FP0NN6&d%ds2k%n(6T*XHWTknt9Q@a7^sBO_jk!sLamV_>g9_T>>OTW5ays zc1tjIb~*lVc8&vg|8Vo}+xIdm2gk>HIt5S}k0g=-%}}fjG4@{N^=F?mKG+*S`N@}} z(05;b?LmcWFiHQTL50>AmsfxLPyXwl{+*xw=IdYo>wo>f{`#+f4GK#^U!as^^rF1; z@>dhQm*QM{Hw+nX$RWHI%CdVYnJ@bPkv;B%Ulqtp`l)ydN8Kr-5GC4u$tN$LHTJ|6 zHzv`^lB3oO-xa*!bShHe9>ddIk7BYQ&2Gn+L07n7T0^#4)#Jl1TCWE2ydAYGs`oK7 z9jfQAKK<q)uQMmf6jnUd1wa!Dh74UO&ud zQ6-__P|1ox@7ELE7q4aj9B7e5)J|1L2=0L7rLI{Ivnj)St*54}g$i*Iy>DSZt$;`H zVu&9hM`v3m)}5L*prX$~mEL>v_4H4E@v~q4@^@SHozY-=Wtuti!h`Aad;1L%A`7r5 zb!K^sq``)iUAJ>YaGivg&!6sPUf^P)+uxO9E;3!_$f*IMP?}9CvC;dtupD*U2Hfgo z2mad>OBaRmQIr&o3IkeLD$`kHlX!L{!y?Ovr3q!qxa3Z#cgK-ap#2%-1&vOTKs=qj z-TLA2(aDRKkOW~KUasCX(0-i|_1P`!mc)ZDrd+nW2Y`2H>h@+P0%Wo&WDnBrwx2$G z^8Cdy^+AB)f>%R#tkn@VwfMrj7sj=7SbXjY|E?AALC=-TW{A z@_%l0>aBLWq9qKadpQ)sF6Vy(G^4r2h>ib`r8|3)G(E4wUX@vum22gmS!;K7_4GE= zv(ErC0EQUM03-p5a!`~-DI_ln`^FcJ(1mY(;eWyrj_{SQ9bsEyXd%cDArcUF&H$LL z*Q%~H_kFF+zqcp^2!fpMs?6_u-)A|`Ip@h`+2eEu%`m05s5>~Co1856J3Z=89lucz z@L+8{uZcN5e4%QQd0@xpGlml#;pBXpPs0RRm;1w;o#ApcQF!gc2c*N*V$SSY4~=;) zKUfB!6@2~vR-@H>vH!GK%Dw#XRoKSLxA)AOSAC1NYCf2G9o<{iU_L;nKNgSad?6yq(Wy4AvFQ*Dvh9 z;4#`>r`8@MAy$QBHsOkDW!cwFhO^_z_}dn7n6h7AZIBydk4bzHBmUQH*pT62F-UxOjlKB@OEu0}-IR{3Ru zPGYH8E)WLbPO!hR{7%=uRxNc~5%eaL51%t`p(g_3TSnuh5sc{$Xr!&E1Jz#)2iUt)4T}xKbiL?={-(hLymP_4Vbm zX&d+2{arXm*0X^%qmP-Jh+i={U5>Gz#n99kB@&KGDnt4xdznJ#7;OSkp-+aqkt^WV zqTd_+{&(K-mAQ0<0~aWB!KXw+BQ5OSeo4_-J3f(9)hwsS&BKFZRjdYB6x4a6e*^2G zfSpe7>0aGVFuJhkd&f_oA2b@>TCF9zBHssU_>7+^oePb{f+|b2^FURzY97&-sim?_ z8=#+*hymzO1C|jxx9{xU*?DRA=GMk~d1GTV={$Y*+eYo=t1q9{>fJz&243R^%$cwH zDd&1r!;;47N$dE?E_S^M#7UTWad@z$(tbDk->=6!=Dp1F0_wl0CjjWyN;6^x*C~eYH+5nazcypMsa@kW5^MctzsAQh@olD}e}z_9J79)7L>E zU@wMzdS;EE>3k|-47GGzZ$96vvz~Km;I!CzP&dYNDvZu>zLK&drvTterIe^b`+O#H z1^Qv!N&=V(M8!Kcds6QK_hc_XTCr5!Tw9~KQY^_ zve6sLGHzRlPU?6=(ol{DTQqktOjW~915XuvM60@V^rHR2w?6pjgFC~{e(U6zm#*Yv ze4qd&6NebeM4_WR4_lcev3MHA_q@}sL4i#uVT{Ns)Jf7Y_Um}o9SmTA%jR}L(mCOP z6fNYFpFjT1Y!ug1@Lj>VkjFKrUQ1BXbCFa*2kuLl?)tZqQ}gWUDHyLIE~(e|@O^|9 zwE+(Q1&M4ngTju>Y{DvQ^+5$ID;e|^U(lZ0<(1Xy8r4_XU+_uH&ahJy0_q|ZDo+NJ zhQum?U>NmvT5xrYAAuN)odZVn55My&Ul(fz@7%i!4L^9kHyd^r=Hs>9-Ho-a{k<3U zRvT^^Z2eN{0lUxhStduqsN+3T7x|KOoa7sT+RA!WZQ_n;aMss~R)wMSq6K?9a0)L( zRbQ{yTasZsjvONh=@8Z`6`uyKpiYn>Eq(X$Fg*3y(2?`0on}LJZ^*cZ`=70{#l6ny zFAi$YUo-;A;%wek+Z{JiJ~p;CG*_R0_SM1ukrA@5)9Cc%>UY2S;oVnWKdHC>-~Z>| z{MBFn2;+lKti7D*R|;3h#rAFXa=Q=xwDj;2!9z$Gsv48N*uRXYTXB*ADy1s!>rquzx0NQy;X_);` z<)Y*6hDaLPLehj}yioS?tBRQ*V^-=9ddf2`5lAKoVGRP2ur)iXnZ{kbNx6I;& zEW5kMN5=@ju9ipuIAO>NqGkK<31D@3O0u7Sg=FXsDHTmJq*nI>TFZ;V+Eo4OJoe4s zdytM&yFSN-b6NehPQTx5wX=RxCJm20Iy$K}Fr3aHt|2r434<`&iYS$E4C?D(7t2Kz zy5H6Aj*9t8VKv@upPF743e}xkRU0CH_0z|pkW6y2LE)Nn#>=KneLOV(UJCA-?x1XV zX~V)f{)yZWz^veT>n$irMuz5cu3rHhhHx8kc_||^k`&WtKo?gL?#w}O71x+oHaE7a z7Z-x#41c^w&lPx*K`)HUEqg;H!<{D97ZV0OEks}-EQXcL#y`=I-hH47Fp)6^s;qA* zFAZV1{k{6myR0`q|KbU*M<$oMv9+z>dwg)f5dz4+R?JrNrFb-PT5o^$=-Kmw(~yX+ zgCXyU&5gAZ#Fve^g?^{yzYfn@1H{2Hb3=}o`l}g!ZJ*n5B z9HBoXKT&Q3Fx@8#_Atc)<>0F1*6Z|@6pA{nqPXeUxt8vqOru}hSl?pO_UUgv`?LT0 zkN)%@en{@jKh=FZ+COeJYJf-Y+l@Bka%}(@z%+1|*H0(Lx28LuA>69iLzf;vOH=-c zbrF%IoKINo%P`mV<`>jCpy&7>3G`s4vv=cK?`mb0wEMgnz7Vg-f2I zn}Kp!!h%?sVK{J>o+C_;#tqLWWEp0gm_zdsoswO=tE-i@8(T7@Dsh0*c@?qMi84;7 z94?>O<3B<^^shjB0|dt++B@V-s~QtXavRmn_r7^66`31~K#QRUwQfJEFk0K7ib0X5 zV_%ME6Jj~6iP}?j9#(G~fLvw5CtQptv#-7ODw!Q~al{7|%9sn%j8y{iCK;(LIp0}Z z-`Fnv_;0?XybaqlJ)Z|@mf@?oD27I3h0+_mA z$O_;%Jc^TB)NpU2L}T%YX&8O`^(488UMPGkCL7sjpK9f8&31}h7rW^0o(j!*WwT(GJs zRnyr#AAmei*)l*J!Y7ZO;R@T@3!8Vw?!Wf%!K)9|CqMj)zx=De{2PR_>7}K-uwx)D zf6Ix|%dAQ)aL7657)r5%+T6XHUS6RrlrC_`aoIwkhzQAq8mgd+?VB&fR{?@ceYSAD zzSdVs;nsxY9adq&zN5q-rTmS=;^bfa*&qC~fBb!XvnTsUnw|4e<7n^EXueR{eLY>; zmiY)$KpD7kd>ge;mGzy<+WP6~3so_d|Ixv*DULq5)2fdRrqn$PRB1@^ImjV2f;INx zaRb(00d%au@dfOx6?oRxkvOs1lFn3xiFEvucZ9SZ%uAEi>UM^AE5~c}CVX8`Mlhsg zk5(|lM3YAWTf9i@-nvHw(`waMmO&E1z&;U_*u02f2!pVTAsl5nRvjUUI8R|;gRo@? zrt5<81Ir~ekFRlX_RX*FJlHLPj#P2VFJ}7-m#O=AcbClt(e9ni@b86GE*Hl?N*bLG zn}k2v`V|Izbc=k4+F&r&t!Q47<_A zVlj8~)|x^Ip5~v!>lTn+nG7Mvkj!;EbNh064+i)@%B#FL#LiIa}c+e)`LUU|Jx z&4GZStH*~YEVMxL>IbSY>77lOH(_eqWe{aUC-=b)s?m{=Otn&yTU`AHhSB^C07WYZ zqrN6gxlv%_MOQAJ4Zaf4FxO(|<__ux9!ikPNH-GD2=%sm22F^c?m>B zYnlRE&lan#N--ZA1~z)UxL`x$sgklmG z4zKs%@Z_KWv+uwC#_q}C^I!k^XYAyC7kz~Ry#;C3{{{v^zZd2p*P&4*=5t(6ttBqi z5xaZU3$=9u>({_a)D0MNEJi#Ws2A7%yhi51O#-SKThtx+gf4=r8HaIxKJ6kNT`5pC zoiURv*A*`i_hUXH%~l7hqJ4ID2ksK@+Z74J9XFM>VqoXi4U>$+!vl%&;lqbc+l#~f zUU#gx-nmtpO#9FGT5cmLg1-%|QXO~;&Y*5Wdgm+Tk$T7?opgll{H<@kjE8*k^r`EB z*_~QV7AjfZkw-6%#+^%M&xwWQi}AS_QnARdn2siv=U356u39Z-=syU#QHg7(`z<5f z2o4?9-;&hfH`wnIRRf`fl?;lwo~~?e=PJbrdExY$_8AO{&dVP^iCr?{+c0I&3xp)li*cId^mSCQ+}oQD{|Ce86KeDUk2!X&v^kwpLoJ zlnj@ZZJvKXdou~(gz8S;ia2IOu6Z!-QLjir>l>R$80~f!_@l`{6<=GgcuX5Nw=s9X zDk3y=es6z2q}vd(G}HoNa;01@>%v-{f$~+v`|clp_u*@=G&=pC{O6zj zhrj&m5!rJjW~LQ@Z@NpD0A(CJ>P#82hv)~W5W!W%3)s>dTibNKiQo;tpiYzVRDh(- zV1rCLZWfR4vbMIqcy$IkYPLFljC#en)6WZ^s4DXzHHwzt{EvU|?aVUH-Qd+%-t?m` z7ZZXM6J7fhwZ!ey6a7bFyoQNRHnSW zy@41hiO(l9L%~X!v;y8k)*KXu&Y4CG$4Jw!&FERCg?E!!B7z_#G1ta@MQQ1^4$;3g z?dC7W$8HXV{K3HsUlVdh^s>2m!>IYi!Bcw4N~LNOA#y^#P)errZhkJm1W|bO;a$*UHEJ@I z2Q&z3*j;4ps(u--+iBx1#HfPt&qA4ywk2Bk?FTDc0t?+HkeKmFlgBnuWVr%&gQE}9 z1pF4YT%48j#k;plM5)jVzeQHXSHTldJxU>L2ZB9T=+tzxiUTD@+@Zv$Q`h)l!nEgn z-5H*(l{Q|wTa~_Ar+t*|G?{GHupuD9^y?Eoz8By%OU2LF$~slXMcf>ne@>jOcK3Q~ zg>OG%8Q5yL0^;I|z7Jd-{m$FB5t5*`a-wIK2IWQ&^R$%X{-}d`74%1RbkVB`W!<31 z2({I0nCux{JHp2CcvYw#Mmk$t)iOV0+ne+H=4hH5BA0tjj(vaw%OECr4IDOfFX&d# zq8yn{rR;ucX7cIPP>bsZInRg4q%`v)V<)F4jyLf{=y*^l^+}4PYs!u@Venen+%)go ze{t~m$+P#r@r^(D!S}U0zy9<$KmN(jp6?wQ{13wLQj4;{F%A>~^b+TN(|{Rh#sdfz zh7H1yi*4FNh2Scxx~$z~VOrT*vj*D;zFs>KKs{{BuFP>c(?VIBjka5@uW*C4f6kC= zsl7z3(|f=3+U}j3TiHyyc61`*1l8lJ%9$}P{k1*;(Sv@!m|n5iBdwn@4MCvlb~@;| z`V;k_hR7*7I@oVC>VWT3<%Urq(Y+#O`-VCF8d<&)Ls<=21D^)k=ZcoXps%m5f)n^G zSW-Zu0duSIC>#_1*DFyNuwguNDmX5xDvN%L1@Mkw5)@44dPhh5YV>k(Q%102$Qv?p;GW?#o?UWY^(){Vjy4wp$H_*9_usqw>Vs_zWI)^u zq-#{t>1x3>W9|R|KAyJToFZ(9&}*wlAn*)^YCY@88nqU=mAJrp=(=T{KyeT<+&}yf z0iYKsL<=Z~xT3Av{`FF^!EB;*h6fvIFzaN&K5{j}d&hTCFU$6z^stQ2d5L*zAbS6Q zov3mQ2nQ@YJ43qTp@!J7)a!8a-rU?N(l-}Mo7=nV3S-M=w2o)f*3q+rR?}w)%$txM zmXf7P6`y-NYLc`e6x;kRlMY9%PHTuZuw-lH1@#{%jA-=Z_g|BdU0io5j4hT@*wTRq zI%pf&8vxU7U{B*7cGbW&WAEZx0OeI?UGX3a*c)oSwIEf@34 z%O;IB>?JqXOE+(9z>PL{9w_7;N4sA>{OH41UVXLRY5(xAe)Re0PXjptgl%NuXau)U z;IFE4!SNIF!<7qQmO8|J_difC4lT=&C9LcKL%&Vk^g@`o8 z5)G;>53eRy5tJ^^={8iy(N4?C*JF7RPuK{GDw)QkRL(BN?Pl@90LIs|w&}FpX*Zf} zvyE&<6kZ9ww2@e;xQW}lY>{QLteH}9LJCxvcn?QZm?u5mgGVV;s@6jJEkjLa-X;~V zxI~o#apPLxxqkTVm$%k)tWNT|j0&tb8s2$ukNR?d?PzC*-F>keJ5~)(UPG-bQ>!*Bd^~9G?AOFP<|GLu= zcVke+Ak>7;p;5Ys;fP@B>F7O*YlqEiyTSi*8x>LK7v(}lTHJg3tW>RF$`I;WpJMzu zV&F$ixLRm;8_X1yfDmWEtall{5lTLD9KPDXE;R!lX<%?(I2T7R4y{s!T5y1Pc0AvD z$#}pBI$9|dF(*WC*ZBCfXJajkBx$pPHQE4EM#<<#N;^H(1M5 zwOWH-V>)QVz%=Jf8J&}b2xUNAR9O7PlBB>y#<*Sv$SK8(|R9|Eyz2# zr!6n{+r!|ek;pkG95cksle31ts9f4qR1(mbW~%`eeo%T=JZqKec%m0Nwoeu53m|Dj z4pv_9XS{axb}`&ib*ZoAKr^%7TDLhb?TYtXlN&GHyPZj|Zc}KxND02z zSy&7YdO^tqa0XB`y>jnvYJczewANQLhm$M*c=~9xVg0q&xS}3Prm_^hHh-Zsz+95- z`3tk=@jMx>>)F5m;lI`U{8xYS@XnpcXTScfF5Z;WCeGaYE*{;odcv`2)?8Y^T)o+j zD=o-TQxN<)I>NAfsqTjE>}*$pI3C=$k^oyo|H=2?3nO@(NnELgbKhLK1rA5cNyNkB zuzLq1)4;k^I0(=q~_B~Uf2AL%;DT&VLFg_fB%5Vhte@DthGG`9sb%*`_)$; zLco<+>M!a#Y1T8EjInYSnaUe5eeviKGct7TKuHVMddpRMhBr#qH#$MXT=zZAiTp`(%-}?D#=1}C1w@YC+}VW;3vj*!Oh;7=V(}u? zv)=f4Ha150_5J?s4Nj0NREo{UkOlPJd-u%ft?Ut!_6|?A0se+kNX$dxL>vh_072p6 z1}2JRLB6t_;KeV)nnTi0_(h;Ep&WBc@_|VPqF<0IMd>!W6gq8WCR<9-julq(Hrm%Y zd#L1eepM<}w|8y^%rXGLQJgbee5(X}sio`vyN)4j-k>4}VooPDb$<3#88qs)yRD=B zL;H9OdEQ*c&^=0&r_UN5fjUE}0C*6@^hHhrZXJ@BBB(!vibcE4+1K8>|L&W+ng{^0 z;+KbWg3=nZdot?UpOYSOYEK`h==WM7U;7D`n#FWWEm=cCbStFAqcL z@`_zZPyk||KjX;3`J6b35!j%}_g|!3jq1;S^2OH1&3C`{N-iI7G@HJ;lF$h{tv`MI z)$?|z3tRQN7{Ceo9&DgU#F5v4(xE&`uNDE$wyIg)zJB9IHqVNGA^Ixrc5;93Z%#LcW$eBWUBX3lFIJee%5gloWXQl05d6DdL7ki@QfsT?M)Vf8R%LRfLbcs*{q(25_?sX9;^`5QNHzmltD24mL#ZH>nW{uV2!hs^_53whZMsmFjy8N~g0^X7UGpKm_CiUrgp z5GC&>SR%g`p$MajaYkjmB>I`)z5C|Fg6Y$lGBu2b%#Xp9qLaR1*zTAFgL;><=~6x) z(6fL_!EHTUf5gjUO=d(8nGF~yGOJ2SCfaL%vE4n}THi1k1J$6nrdKn}mOg*<=tqD3 z%TGV~Jdlq#+yu`hwvbrJW8DK{5)zeT>Zt-SB)fnu0}Fk<0DN+fMBmeYfe4M3iqVBB zDk={Q%Jk}bRWsv0f+&n}$LP=0k-dTRp#eZ+DbUQR-kd5se&*9wvjr4^r(RAL$e(rA zocEZ#WmcsuCV~b9VcW*R)4gAQ@@4-rLC#_WLMD1uLrQD)yF?S|6dxE0DJuPkU+KmOqL;wta9;P@UOxv3D9 zPQeCB+1aG|yH9^TpU{FAVQm*P+?yldA%DNp+L|tj#FA)~^+TA>&j<7(GM~oTkS!Xl zD5Xe{O}FWD)laS8Qi|Z9;+OI^!je??R;rz4D4^rv1`>)){u3p-ttD@qU z8@w5!E0-7umtCAHc1H{G1>{z?V)Qk32#!# zGOx}-X~>ddsI|EF^2@ngUV1q`ZGHNiFMs@#Um@~40va%h-@66P`77dCaJs%;vMYUE z&!vK@UE;6geE<$GX&R-^7UH9%5G}xy~+*-*Hf+^j&)RqrHcSXoZ4s z90!Uz#LQsaUV~N|>&8jV*3vA0Nf#7xkDY`Z;aVc_ADA#z?DT9r$kVJ zb4y{ycfqkVV0U)uOb!oDsCKiffz?r|Y?ez!$l7?=QyXBwf&2V?B%4ycOw~nCVRpg0 zU4eil1H40ucNme*%xpbD0a8p1vrDC2WI>oU^fRAeiECFTsw{|2vL(d4Qd$u&} zU%a%t{@Tk%!NXNcL9aEQcEQ%Bs8)b!WgXD~<1rIpO{0TZ zm^GWNbShmfIPd@wVvoH5v95<}TK`k{|4|qJ= z2^%ejlRg&-eO+ux`3Y1Em3~n1@tm-3k2aqrwx@l0v|rPT+`qm3#_M-;#q<|ny*NFo z1q|9+J-xUgT=3f!dU=&EYKD%EPZhTT$B`An)iL)|XcfeV#_x7{%_5+8o?8|AD%-oeUjWzUVxx9TV;K+J|Ha_|eD6w-CzSbAIvu?5uR)lPHp^>=pe-zm_wO=gRDr-Tuc(GhEE9m^t;YWa5{C=R#)TJ>sMsCCkp&Oat45QN4@&3i%^T~cFf0RVxZ|NwK#o(`;VHVC zbHX;DyP+3%U8TB+&&NK?MJy5Rb?YaGM~%}Kgqcns5~{j{wy574&6Ek)Bzta`b7)@u_$2uBB~}1@K!d+SON@9n?Vr+!Uz15ST;=K( zO0yL1FFhCS2>z7XU280tA|9i2S= z?c-kCmOZ5=%s*)^+TjusE>b`G+D$L~lAM;JQ!4k%)WTt>4YVPdf?+Vp6!t^46)a`X zPflJK9RckG2mJ!*MwZvwIqjhfxZV;>Y+(#+WTSl(Qq%wv1s`tIVX`tuigyqtsOrgz zTg5jp`HeT?7_QxEYF7)`Y`?FvE?%=p(2cSMYOqbFAP?hWm z4!h$$7ixnEVv%2`SRC~xsNt>MZRL^ert;KszRukh^qn0W=ke~VQf9N zg!$Zr_W3h-BNL?Hd~D9-`wPB?nL?=yssVs1;Z%tN4|dC_k=4qM1thp^$H(=?@rWZw zvv=HXo#C%oeqw7B0~V~5U{-o%I){*H0g0O0jO>a}IztB662p^MGUN;v&0s>m_st;SF#ODB-t5rOeL~F)#2B{-lsna)zz}3X_l%nUF`PfH@KjeS3mvvPZP@(fjL1b${e`u%UbbS>zD^YY}4D7D?3Kp0~M zV_M*2ju9Gzt-3Bnl0>OlTDxFf&&=F@qtES%HtIAe8ce&M;x}W*88AeZlhN;Pl(yc>VXk^WIypSKod2 z&8>|MRVKZ`_&hq8Ewp+A5>HuO9l+FdXJ>aakiSx5bpV?5qu1*@!J)3GQx-UPerrgU zRK-mr3D-|6#dE@@EH_`BQOKH|aQh0D_-h-KC49Q)?e4^`!fc+4@D?pqzGSo#q`g&} zEz$`v9j^i+Eo$IV=;lR%Dv{D^zUcbduVP9hJ%yfUkufCu2?T%Qg0jj*Byz7ZSu)VF zhC*@xHTl&3aIM~!(%BjF0jgBIt2G-D294&N<*}0(DzGy+p7DxsV@ZMoKr4e)hrb%e zWl8Urw1x`YtgJCV8;cwJ2R*PeU_t`(`TZCDQDBGi;hBUre_o5sv$0mZcXORk!m|LT zz(0m(wsr;f<8maswRRhmGnQoZuw*fWEfg}j@E;PDEQv8=&SHjN z%LzniD;$iCfhgPCC5Aa-hpCfN+%8~9Tuv8}&(!Nj=#y4o!C8`N3+6AG+xVQ|4r3u3 zV~gu+xdM@7i~$`Tf;o0*utc`S%fK{d4|^4%xp5K^i5zJaOR%tB;(oL!F?;y}!kh2D z_wv29pZxrX$0rkjJjvVnxdl=~I;61N(uP4^Zx^HGE3U39XJb^MvvZhm^kR5E#)-KX z0O(x)8i5UWpHxFNjIhIpOA3&?{D1M1bWv%X71agBm-(@yFq)vlG7GeE&J%UhELA)@ zI%I+4tq6yLza6=IPoBCT{(>8FT5q15Hc_xE{^|(>8WoU&>G_KTU75nyC`|DQBmge+w+v5&hM7^f$26bnwnNrl zpv~{T_4>m<{YM{v^W9fA*X)3^_>S-_SeXd@B>7(*=uyrn)Dqc*TM;OBs->mH08+L( zfyK?Ss$=_#A~)Dlg^VV>1n-zIW|R*N>pUU}AZ(qzI=gdgXBU%vjXx&BYkP7PI zpu=yax~5kwt#4LnLF`nU_Ue2*pn8}IYAtek?*gs3R4A@ht8fj~j_8FaUC3q!9W?PU zE;4!*nGIpNat1{AQt4uxi#%=oum@rci5XR;+Ssowk54+qc(em#v1?3ey3@cqB!0 zLIVVLX}DX))Zqy&B~3N45n3W{pg?mgAYI;DQ*tIO#)3uEs5SByHL>0jaJ=2KyZK9 zpRx(S{lB_WF2leA`3S5Om#-Xx6Z2|MOUdH2d6W_yI8Fi3JF0bu{am@^N*?aLXq+_3 z_wT;;#_fCCEPU<3uz{{e@IGJg6L1KsU`Y)DsN-zZY&TA8?ce-nFAzu;&z?TttJUf* zk27QV8;E9te{|VgIy?|=JBwFF1sL?G$;2^7hbQ@5^4s5j=R4nd_wMZt^&8pEBEdlf z-z{wsJA&oolh(82Q?p({+x*N|qjl0R5vnse1how&69&GGdQ-_4M%h8Ob%`oeLgzF% z@yIL~ipNP&_MRQJP6i);^xoEH)yfHtvA;3GVq0Q7;q^}o8BPj#DZ#Y{6`QQ2)&R%qiD2J0-MuE673~2 z(^sx8_Hn2!a{0~5`i)$<3LcEFum&wCnj2#_cQ_fLXCqx3yV; zRe11_Fg2mVRH6+NKoTlGl8X!iPs75g&Oo*?h!i*&w;*bVo$ywh=*fcaQLJ=%Qkxs= z&U(Mq!`4=kYI&9u88(N?1rDg1f6+qZ(H9`Tk=}#!;>%f=!kYz&sB*_l*skW=TZL$3 ze0V%mS^4=9)=fYN?|s9=oDdN)SFawyW7f z@GhyhVL^Ng0Tav9!CCa7x#{`m2z9$a@7Ton9_*4gKUADCW!`Sn*GynHub zNRP21>RnqZVlKAOr@*l5%V?*?1aWaVoHgoWVr8JCyUw&&(ncf zm9m)^Kkj$pK}-^PdeC6<{Ou3l_~7^Myz}*4CmX#KeJm_SEs4Wf6{9&*dwBW9^UiO6 z`xP%(Ag4(aGlJ)jz5*9901qL3na`S%)1bwU(9rj42~ZSDtI1qC^r*PuR$ww8R1W;$ z55D%NfBNq2`(+#~!aV3uSn$ACOsP~_wZllZ;d#ol2MHXX1_Fx1kz2WQyi zYFVYRxP(*}P73~@tsGb$M@I+c!mW@0;BEe+)GDcLInbe~lNOg&gGcA8x`IV9TnwYupcchl#we1J%^HHUpb_Bmm91X(t`b7}Dd}z^RQVGk_&s zUVy9?IOndDkK5+~RD&K!d_Ylr+mJ=r#NwH8R$579U`$uKyQ>Ee?`*AQYt{-3d7LUW zP5y}uWmY4T;V=Gnk4*8kx88XDjax^@2Pz~qhiRL=|B|o;ZnNqW9YJfO7`&KHu9q=H z$X}wr_x5eSiEC8t;0S3m;N3$7$=GLLHv|l|2u-2k%`t`YQSiD5puFcgfb zG%+s;E43By$%(4eZe8t!D;2m4;gjMzsqbXh&@N`OGJLn*?s2T3ssPm(-Nt$Jo>@4f zicna(t{l)bjDjD0#E3!@pwvVz3QFvN_!%n(LW4o6tY{0kTL^^1hWot_Uj4@VFTMWi zU9Fv-lMjxiXh`?kB9voRbasKINkAL;6Pc+ZMh##l{P%Npj5r05YdOfw3 zfEQLbL?blsu=e2ou^fe1t1)maHMAeT|Mfrm?l<0gcw=kL`VKor0m$ygg7FFC88Saz zOtl0(4?oZo$!1D^9*Mo0Dw}1o<;(|dEA@?SYcwDRvFT!#P|Q@)CiGUP(d%`?)E1u; z?hKx_{H1smECg6u+04dNO6`#kj(aL8JCz)E&%Ky#ZJk}a{4jMKk zo6b%2m6??Yz*0>Lq2+XVo?1%2^Y#q{eozq6y&7H-VH8sDboLHUm;z~)8@2Y)kp^dR zavl$BWN`4VQgwcdjZOyEE}A0K(}$;WP8=UH0JxKx-LyZKHsR7AaVWIFVIik+paxF~ zu}Xac8&8DFD|K~Jr`vAG?Minjy~x``*B)SvU&NEqurWQJx_#&F){QNbbWUYX-Ou6S5uO#O`{Nbnh+}0~E?{079 zPfibUfMd%TI(A(yl?rKY8gLXFV1m!G@Qfi7TVb0U{cry9cae(BI^-0sI;)=b&Fx^c zW;kQA)@>^^?X}?ntwp6Ua^_`%mi5>$(y$1Qbii-}@#7rmLR79NX5VB*wxU4++-mWs z?>aEJa2gxHny!YI+Qjh-eyJCQViAIh1zOCd13G=as0H%JY($!bai5;Vs5ESxnoJD| z8KhyW5#vZ&TIRqN)W3PM)9wg~>;5aZ{^U=-_pNu|V8g+ymr8ilQ|>QwDc2zY?i?r$ zfL^peU3&by^^2eX3V3M92XP7R{r+W%bQs5&a>I|j6ex_z&XQ9Y2H&`i!GR`lFB{F? zn-A~&^MCq-Kl$VDJiNO-?6;mg`qE_%>X*<3&B80k>KKpEL38ArAba6bArZ43QOvno z->B;Dc+8o1xtw~0 zwS4B`9lB~b8HgGz3(yVz1X@&FA~3NxiS;%+*HNLCD$I~a7N=SqYu*faf;TZ!<14J| zk)?vOUP$bAJEAG9_=|dQ8(?Q_3i#$2^!Oh{YatHLX3)Wpm1r}S4a;V-Y>M^nlt)6C z;cB^B%oey;4r?N4e}=uR0(X)`f0rbzL5$44`tlh!hF~UialV@6W48F8e)LObXB)ig z7B3skVHknRj(|CfO@o+@A9VERKRMjxEbvMODngQ`1t>b4R1s;jxs`ZHMwsW;4&*Q>$GGq$7?7}k+A zl~%p>_^UmxmQD!Ml;EnYe_$4L9%-~O5lMkko$7U+XV_ zD^tc#Rh}W^D8#KSQ9=?DE}nCzx5R)M+z-C{jX(JATlZfo8vfTCC+MBYoORF&i!0XJ z=z(KAZ;1PR7WjG5Gv!>ZhHrD?(UExto3eD$wXw^jP~L?8gD?X}_3HBQ=*dZ~9@wjt z1uU9nmnAr-&U4wIKfFk5khP3qh$lC|=%)UlvS>YUCsx>cp?{xE&j;OMDp%QDU*|%h zvy#jW&CaIX`stpxw`A_+@k?2Y3tX3~RlfMeg^PLKzG4jtKNaC!n-{+v&V!>81`r?0 zFQ4{&V6tS-(l8hdLZpXXD1_$CJ7H%rCy-2eymr&}haS5CCd1=yx#9RU+>>yYR5l{I z=t@k7=W}rIq}x*6TNH&vmP=<=ij}3G{`Ao%Sx0VJ>^hIc?H2cDMGv17Ar2fMXn5?Q z9$)M>j%qz_;3MLzd8>=Cvs&K76M#H=ci0G%U;>)SML+n)y^$tnIK0M@@NTEZeJ1<1 z$fzq&K*bVeqy&_7m*=p$u>IJVVBKaEZgl2dZQs~o1w-B_iL*;k1RJgBf2@j(lmgKoc}{HPQwdK@5BfYBwlu(i{W!XzjWkQB9%c@X7oZ;}1-NA#vAE(*kKBrO zvxysy)L?}RXhA^=m9(KvAkZlUI;h{E^gQQ3n`EhuBw^?pvZCc=xguLNqd3GMRf1^Y zsag9}F63Dzg+7_QIFn%an!+_m2TSuxbxVt@@MhY&R^0|~@zGJ&0tC%MisiDj!wDEJ z2I>?6pY1h%{>!8EYM8zXe7t7>iOLbrPKE}FDMI0w7Nw+8xw|_UGzpHY*qM-R_o2+k zb|u<6V1c6Mby?Bi21`lxZB#cEdSfMK*gMWN=fZ&LA~I=SOs&*dPG-vN07+gZlWJux zV}1Gp3TglEC1(H%55xpS+cul`PV0ls?fYN*`q$DaEZIYzHv;+Ml7ASc2nzTwKY#J; z$)WPicu-9OQ(ugP4q4%#$nxdc_~ZTz1}F2GHY6il`fM5#_~*fKFB>!~4(G%T!cRb3 zG|6*x1UPwfArDo@LWu! zt9jU-OLN3&Xwa;CRUG6>#j>`~$8zg5ZpfTG9x_4Ecc#;YcZq$@r zAnBpXl2c=)ym9;1O%Abgh}XqNJ9v>^oe?pj|KQ2VdlZRuA&Oz*r;B-?4-50)7ssYp zn01Jl)(-{xZvfrG@R*vPvd((Y33$NWh)3pcy}tY4)!pDy;51+`#G*I0ckbS~rItkX zU8R1>6tHVZ4Wtz;?bA+okLhf^l}ttQ0$kW0r%Pt{#HBSX-1M8mumPzlfYVApm-6C9 zc>{(5b^FG*-@9{f9VgZpVm^UC@lgkin$rZKGd?(!GLuLvOjwd!IlI?nOJqqeiH=e0 zWQsSq&TcL9I5)Csct5?xFR(W&V>5f_wNu}CC=vONWcr0diQ?BAF=*{}4071mscZCd zi%Y2$QP5Mu5kW@N?j!qBtGb8yjGunWrqrnfx*`2qWd~9U@5NYBY-DTo_d`;w%u+V- z>iwb|(dbVSxkU78+^L<=@(pyqlc;gSA~u4L*iDi{>Y=U9TyXc2OS zRn+4|a((+|6|D*$E4Fg*r~$yQtz(kkqBvppSmQOSqB zvB@wqeYHgNzx^j4sh9O?{ch`8Y7DgLv7>_#plE?SVX&IW z_4_?`8-e>nVmz82?H%-5b*4k<#Ht3stIHQP{pKFHPYv4;^2G`*ymO)$Ffuc!A!S$V z5{AKi6jc0=`jd;3HpZTby}k*y8Lr>od+8{fq)hFGbQpTqt1Aob-+%Z1&6{QQgPhEU z5JgKG+}?(R0jkPi;{gUbS1p)6)~jU6>D_D z1K_&raIO)h2u=Rs-T_|TzxM3Dh>Y4gKMAQ!+7rIR4)Gdof8>>YM~^3!Ac(7>&fTmm8H)OBm1 zvBk!wRP@5a`o#G-dgs>G!~0n!Vj?W081@=_%CX5jq1j-_ajHxe{Dp%Yn2UeUk5{7_C=Vxsl8x(aTD+x3GbY|RvM_OL zKty$RsoI55&f+nJVeoV{==TDXqTZc-`Se8D;NypV8%k}j$gOZArN1*sz4~gZQgnh} zx{+RA-{4Smdbr=V#eSKglfExyXacZNop@x|Sc|pBz`$5<&^v8jq_W#Y^v|9?SqvI# zt%_@Beu`Z=!s695>%Np0nq~SR1djD$;b`y0>ubg)kQw|ogLqRru;%vm4TUYM8zqET(CVJ>wu2KI-w(#t!;N81#dJAxY4I(9AW%xw z83vXjbfH)!*$$fxq6@-#53jn9kr&M@Cp5q0-zqjQZ8jfctUh_P4+oYp?9`!*RR{q4 zP7e3Dqp7Y$VbJ=SLH&709;J^nNsL`);vCYKAl$guVcI}%Ng7s4B4kEd?H6X4a2;yH zHd&D_r8E5GvpvJ@$Z|irPz?KI!*V=++ZN>!nFr2VAm<8N@b2ltTW{}dRu@{wEk+IT z=y1uEQ|FYona}D{5)5MCnE6cFV75Iy+&jMNLGC!z2BJh|^`*7-wej2_d4f8l!nN@J za=L(3f+PZr33SMKVr{+5XkyT;MeM3tOj$>6|6nm?r>7mSw7n^hx-+7SKcyOgkn49; zRD9tXh3cz)dLLJ-8)C^t4jFbHzv#ISlU5)BBWk4%tHL&aFH7OogQWGBQ~>TkLVsUbNPac!o?1Gc<4$$?cW40<*)Ya#002 z0zoU$ths1MNA17->{uCg6Kq*s<`A{7#|^Ui8KJ@2-D{ivfATQvAsI1l@Xv2rG}2JpS_x+;?G zcZ>T}4yibFjYnKcZNqMF`=$H;%m4Pje*DMZVNBMk)fi?xYeyqgocBI zy=JFbsg~9^%6Nr!6=16=MKzmyzCBOFuFa*RfjOfpOw3|KDo7||jfOa>MNh663CxWf|*3$oSVxWnH8ol~C<(b3&&n|w%s zvlyYmtalE2=k-j1e#Zc|Kqduty^6qf7E&=#)IylYH47m zt6$fcP)lM!|5aLDML2g2Q>Z}8Mq#sJ)Nn#Yv`{WpUVEjoG;crs^zr_l3V5)*T0jlH zoM4^Ei*xBD#rS|3Vn3<0{8J)>Fvf<9n z^#0zLQyh?0&Zzy-^l~sRd1LyGe_U*N; zEgL@BPMf@AHz|q`S*=(L=4gu%3()z%<8|RE=qO6bC14J&DvViiiIs3qR2O)oMS0vZg{ zpyJsYAnyd>~r|BziX_mT;$#g(-oWFaYTC)B4tEVjJO(4|lVac!vBN5sO~=602+5Y8%anPboxhbxmv>*f0S@c6I=6PVUw-)(B90<$ z0V8WZ4BzhzJa*P-`I^b#D58fu6a9?J&zbg{dEeZrxGcZ@DTE4lKkFK<|F z7D^o+I>WBW8t99HlOj`|Lhp%)HOh)Mch~55-+xu}AQv|{Otf0(GdCv)!f}YCQ7EP6 zP^qY9a0ryJbR|??xlF-aU%nDt+3Z>Q#+$%L*IyVS5_IcgaThYFJQ#yMaPP&wVHEh? z@zIHSZ9oh-D%Mp#Tf%WMyyN_vO&6ZOIQpyq{KTfgaTI^b-L+RVA8S@NMvmfKB&zCkCRI?*p9i3Mi1!BvO4xqFo8E8iza5H?jL^qwcYLf$^I9O!>6_5gX0&63{|jW z27S|uGkq`QP<+M13+6Q|FWe1IRN}$cE)|jv$-JjQGCHQL!s223Tmt&`4zjsT(O9_Qq=5BW+oD zJ+}O=Z<66p7Agg(yq%36glNurogNj^uy&OXatnsUX*B;!m-RpYzjKu>j3r3nuq8C}<*a=%Lm3D5H zH#TGFT;o}!yiUz9e)8p4=0m`(rKK}lR7+Ai=U#7Uu@w=~ z4_+;cqkIh@h zoij_Y7f~xm=eWq{Q&LaYLFH;c1t`D7?=_LQmOjATfOv7|%S2M?r7lqu%ZHG#FCV@5 zkH6TveS4dgj7D^BUl#6cu*@R*3D#JeA_CWgTK9tw-o3f8&O#k2E~pl!Q?xoetwRT| zfjD;zFS3>GN~MIZSlXy|+GoG~@-&@?f^e_#Ank|2wTVc}X5~1p|HXTokQQTN!jv(> zuH25ikH7QIJ8$1fES)dH;qXu>fK&n~+Q`dX0)mm16C1YVAbC0N2mt2zC=r%JCKH)r zsjRisRb(qgBN>hxbTtZdGkWFl(rf`HYkX&*=FV8!t-|5k3LaIufFKueLfEslyjY@)xMvdD|tJrRdC zVtkqjXykCu&9V)F!J#es1dShv#?Din#Zm-j*Y5Q{`J%BBw|!7M5mq~fidU>#jHab0 zBWb*aYr8iG)7`S`YWGIkjSl!QcjAk_fqx7DPpRiPMItwE-^>~n+EO#NPRKHC7V1l8 zCPIo26iiK4k^!rQ2&rya>X`D`SQ^!9upmqf1`Kze5Lg?BdRl|7Gk8fdFLIch37+&M z%x*9=;w!nzS|yddXtwvmC`hl8TqYuymMYcCauV12P%LK7(B`mx%DtQ8851a;@(?9* z>9yTE>jscOon$(z`}L&)^VOHV!Z^c-ZuM#%nhU!oy6tnU-q&9)`E^&9K~ci6*Yi+w z2|s2NS^_^oKJecZ^S-5mKKh+^Zkp#9uoxIqRMt~v6zqk3tcd9%AgX(1Zs9zD&gxpu>SK77fu7VJG>zuw#*i5G zUp}ef&A3p8m%eF;V%KCek`5JzKoI(rfDQ$k_^q9_N}d^nWU3M(3n~}ZbmM1a8*-;` zf_w*dF_J*U!zn1_)ZU}{2t6Q)PqSj$p&%WRGR8BNT!M^jWn%ad+E-&p-}3zIU~kVP zA`JO2Rw~u1IVhJqU3CI6CbO&_R`vOPkb_h{O&*kt*<8KQ=(z1b(9M#a#PR9D{&R3f zs3#}3BL}lq90t%og{9WXE)gF_TxDOzUYygqPk_`AOz7nVbv!Q`bE_~aHo(TfzIt%? z_Ug2wB0ISdpiwM@9(p}~3!+aZD4!<-^Y>D@s++wQ-Nrk? z%bfi$JIWMvsAVeI>0V^hreu6$gS62N4l7xLJ^&{nUA9$E4Uc`6$B<}EjHYKX2VrtdmCk)?^00wP#J+5{|snx4ltj%i^DZ>-{(7^}S~{p4&3J zQemmGV2}Lh`1r*YxMV(I8SYY&eXIHdHW5JO8IY5J0lWh~0!7MpDiH$OS-&%ktPHy? zB$iN8Lw=``LRM9oY@NQin2p(c*f@a7%wvlS1J1s>LXF_`K7M5{oG4qPU;k4Q2IH~dp} z8&p@*@u_+Nf0G$5y!vuV#2@^Em9~%=m1bln<8+Mj{H+lS_kc*n_`RtJmWr{VvTCJR z&aW;566+V2lgnjVdFt-|(4`JP;363b`ZllZ;(UOA-C4;l@sJ$!&r*qUX}vgW*gkZK z;htEm#PjK!Mb!FBKUM8zQh)rcMHr?F)zP05%>J9756)`Ah-PxGNAj zG|sY#oe(}oumw;l%*Lp$Fj5t~vC;}wZvrWlU~!2#4y0w!YjZ&{*Fqp+g@F6U{mEhC zvi>h^=Jn>X@1o7rmCa@JYTKMU`c zv04!6H99TLgKkqVK{oJz8$B7>bWKo$L@PnxHqIiQh944EI5y;U zmJDKtiXFKOU0beT7WC*{`7!Iv2s_NCqeV3mQ0H6$_7^9slf?k8v@jLk~)~ z1FlZB-VY^+>{Ib;U!3Y^5eoA)A;(ALf!E0D&6!x1t`_?DwwI!}eAb(f|>&3i`J zZHpUQGiGfX%&Fmh37yGDIll|Z|j&FsU0 z08IM=W*UW16FCJ-UR-UiuXzl$)(Ik}yj<^}EQAT&!s=xrnuyhY_tpRTzyIH#_ZL3Q zuMW-S!n_L6B*gMSbV1AsymF~g(4qruY_F}Mad{c)5* zaMVGVE$%iMCqWR@OMAg0%1<+Y#-PM9kx3=?W z_!%AYg|X9O+lczK{=5f9?U8tOqp4hVZQCnJD0ou&Yo@8Sq!}z1t>&U(dmrd3Hd|QERH`@Le)#&)$&<#TAvg48;}4bR<%HztVxAz*7|pv$ z0};$%PDLd_m0wGeuw1bbBH5pO_z)@$`hrJ@MrWM$)e>@TXo`5(7zY{qiNF1FnLYfv z{@#%a8vt>8Sh%mm^O+?40N1U!T3ES?vF`4Vy2M&YPKo46e7Yog1ckk#3z*o){Eqbv34Q{fKASRywr%xXJyI-AFZ{92cDDsv@NG|{_6XcP; zg?!2NBiL!zJID17KcaHha{03dFW-6lotGF`LVudvjn1rONC(8-6PZZfczK*qnGWXN zF(QS)3ql=KMHF)kdV-QCpwVr8k$HP9%*)5%8u6Pz-MZ6%tLomR}au+N%%vw5pKJB1C8?(tXv=r8W zYq*xLq6o|sa@ES}c-$C@Yf+0OxQU{+3$g4AC&Ay7b?xOmP`h|Aicn{CSp1DsldJR3 zzCtA*a$bO5D_H|qPp4}!t!E!x;pKRN)Ho(sC98{9BI)R5d}lX%qfAQTGxB7Zhd*Zx z@VrOm<-tHB^f%CHZWRb9Hje5wV0|{F7qccS%lIO=mSA_htjZQpWkqZTHN;a3SY0L- zloj}`3Z+UZ5WcjFjDV2Y0yZ?e>#5i%O()anB23I#n!S~sr;Cs^cMt9XF{LY#tn zMmbB~btysUmu=f2hAg7CL$M75^exzo!VZSIe;b80Y3e3cW@Xb=KskaVx#bgV1 z=j_E8#zmQI{JqnP)kDcluxS8zrQ!kprf>_cjnk1e;_we6CG$~JAEctRxwK~-F{kER z8w0v3e+{%-J^tbhaWlb-W#J-~0dZL%*AKO&CWI>(AJmH}y%+5iv`5}DK0%`a$7w9A zLDPYY9~Vo}!+Q72Cnw#(RHDLC2ab4bfq{1ILFOoe;8F?NeY9f#95*Vq_w;x@v+}{) zH)T_*Vq8auTEl~^b*5*=1zon`RU0W*1-M|OAPgIxf-t~&bo~0A zX3I<%w+ZDBQX>#Bp46F1R7xQ_nIL8HW=)bNrsEsV$x2+w-M&BK zF;x<}R?Z_-b)^JL5lc1E+8Bi(30Y4TsMPF+xGWBZkHsqA&E2*CpQQVImh?=|!`|tf z?mnHv>663E027cG3t)G-%uA6nMOtQ+ZAo_7<h$0c9kucEQ@wYRbr+|F0oiF z;>43X=N$Zdj+dhq78uOwe%~kD_jO-^RT>_sI^R0sR`@`d0^`*iK#*{am?lX7c>Ue4 zRAormE*5(bSJZq%%^p4(6Jr^L!QNbm7pQZt^`@1IAsUnFR5y(IqIz6KZ;QMwY!R!L!8jEK_6E1)-3eC?sG7xlO z4qR$nqMYe$k`Rz3o030TVuRrCVUi#kg=xKK-ST49NW;R=~t5)yZmyOGt;Lzz4 zdN?+cD@=3f69$2QngS-eUVw)n+dS40Ok+QO|G}pp?dQ`hD#U3pm7pbenV>=V1A?$a zGw)&g{I)tSRjb)B$hUh{t8G?YXvU$GMVAQ6C5$8||Kn=bhF}D9E(GF#Z zNM@y*D+GbxreLvo@1EvKX>q--%y6uRj~yNFN#AJQUBfaBp3H0t)gofbgzRZC=Sk#G zb7Z>Z`BA1#ClH4uY%T(N2^ns7=Y8rluoiTNltZrp4K@6XafQwAmP`QTE#KjS+gH{< zc)=ADjV*9oJuZi#+dizZRJ75&dh`0VH{vP^6Fyr4`@yb~T5$gG5zxicxp?(@k;_rd zxB*E&oI|#ahyr)C2NNjs@wlQCCs;q@GcEe#+h!wp8fuW8v9LjjOYK!LmtZs)1fZ*B zI%3Yc4a;%k`E3KF1NMwDvXpga5+BE@WwzPTL_|h~qtC#_<>?n0#QLp=ryqRqDEzyF zIh`}7Z6vKqjhkR*jm31Ipa7!ov_=Vm@)oWdZ5jWIU)}ycfAt#MbTt;VL_8HpJUZrp z+O4Fl_0V)6!jMKMB?SO1;o;H#Pk!<&Ux+pD>Lyfayu(TAwt4GI9UmV!H9;M|yR9JM z=H{yIg*m!CUCoO72fC@BbPS{U>*7?P1_y+{3dZen9>CUbH_eNyd*{{4;W{?!_Ue&k za`!M#F>6STA%1uLGP9#?JKIi&5xO$TKCzm>s{Y{Z&AL#mY7Fc)XA=qnM$T-F22=x& z=oiYxJa3m=IbtBf^ZHKN=2eD zN{Fl5cDQ%V3bkKKN@W*~6qOem=#q7X@2=^y6LysTrJeOz%^adouH|N{6v4lj2THK6 z34%OKR+-*rCuoJ#PlR+LmcoflSGAML&oS0^e9+*Q;N>}z?0{%KzK|L~?}CJCvI$C( ztLF;qi}D4vdLNPxjhz<*Hq?_?zF4wJNw~V5rs_DoUBLK7(viaf7o$xPnv=oUsSj8TWdZ2}N>X6- zOx}8Uh!AN|4q?BaS&enDI+Y;Qn8lXk0E95qP@Ec%hng&(eD&t#|Mb6n5wwtzG+y8N zV_svtuI{`qH4N5^3((OLCbSNWxceavO8Jw2{yQK4;Pm3G@ZiCk3E0KR{sCzRw~c_X`s8>t=tIFSfsv2K-;eB1W%}2NU&%POHi@5n`0T@{yXC?bT@ssej2~5p zy4QWrFTiu=s1eM4$v)*d^~W?)#4*@I*mFo;08+p!%W!rN;r=jf3}ZCa#LU|BRBZG* zm;C$tN_`Xr%9#wjh?>HX;Q#bX4unQ zs^b6Zk3UsXnui5?Q=sT4IZzm^dubLN5x0?<9pwNuDAey!u{bXSd>ak*Cp7^a#g*~ufZ_#_Nio2^?uIDw0SMrCpP|JeNmYleGIGyMx-U@ILo6?QcfD7;gei2mIC zM&+hTwYHRwCIv~L$Z8@Al(Z7J;b89oIB5cytnTT-u>zDr3O?J;r*O2{A^a@S1T865 zhM^VJnIpX`vT8XMQGLIUJpT|gqx9i5YCdjm=K*b{Lgq}M&1Q9&>mvJ6*2WY`I5iHE z;=D)3ja=5TdQCyZ#^@9JTce-0U?l&iyzcg8vSX#-7v7n2Ax^iSVEtLE^QrU>qB zS=p;ZZaWD_QGbWkA5>#D64*EH%{sUXg%aCeC4&hEw8H~*qhxb?W}!q|XaSy|TL>7% z+=K|r3(`PB7o2gdP;q5#6LmsqGX}f|dX(sQkmq!~mpU3LZ|j#N92b7WI%e!YvH`f-ab#g_CN-5Kf}Vw!#tO zqL@}f!lImfm1@PtgM6qo)Iug^CU6$CW3|)nnOYKz(N9hz>G6s?;-Y?gXH$C9P)dFz zFssF6+Pt~?Z~nzUd;6`U#e~PKG-C!9NjU+KYjbjgK59<=H+)fCD)L~(xD4P z@WNCt+8g8y8{fW#xr+#hEaN1(fmR;S4qcYGwNNQF?rz{&)ykeeM8V=kTez-G{VLL2 zugl-LgA3w$h7(ntm&`S%@#j3OIAl5PoH@aWB5;b)iH+m^d@KPL64=h#eFW|I-aa}$ z%q7xH`1+HDsjSm~M5N-ShbCuh8|e%Y+|_|oEEXck9IjgCguo48jbl%#O^gD^3H&Pp zqTq!cB+|ya!Sn*~!zn7beOW^Owjxm{!k_nvtoMtq?Q?vZuJ+cyj+2Q-$Y~u!<#9QF0!N}Ons{?*KddHP6MQ1 zG;QSOPYeE1Aq~P%bBTvNQ6-nlWyf|H$jAxXEjo$9Ek|24XD>y;X%EIGJ%Ru?s()*N zP_6bR7V^jqUR9+E%t@Ch2^U^9l$M=jmA-!c8rH=j8}kr~q-7#V8?Cn*-QiAtYifUj zHzY-=UcKnW!Ue5=_~erhL5|SENOXdET;4JiA}3Wx!$J`W2r|)pq197|8~=Rtp3V>i z+KBB`G+_wc+;wUN7E%N4zOjnx%ZDUgx3Oq>eRr=*>G079Pbw&*qrT1;5y>kr7xpze zYGFt>iOdRK-DM3t-!i+}&NCwRP;vYGw*>Y8O%Tf<38mDfe#T7OK)z=%q!ahZqRYm5 zRa(W%tC;i^;g*e@HYv7^#F<#)sddam%ucHho+Jv%%bTxXeg6&KlXD2ER|bd}kxHqe zrYWcRBYQx@bR6_$LG|a;o~b1A8uiinm0npa8vT4_{t4-BZN>IGN^tb1a{TSTk2&n5`tXrY`$Aa_;u_ z<0psd*y64}OyrUa{U+@{H7T<7XquZ%_~N>X`3>cZ<@(H2VoZM08Q72jf+*gkpWt{| zcDqJb`!chpv97^nugNPbO#3~aD2sNr-j?Oq4v-c%tvP!zw+ZY6O(z;7w~b<&lGJeD zKm{DMA|g9VWvY&7<4*5q?mFJA@-UjiLz1hJBR-SN_gHz1@V-o8s8+M;VP8+YkuHiI zwTGkQ1STOufm&zdwS+(YziJw1^q%r&X#`LRV{(>`qu0pZ+^iT_x@#MRm-;Uo%Bhx$ z?YQB4LMgSlWXI`9p~}l8HRw%EUVV}H&wg-->ue0E;VIK-Wz)vx&JxOB2hoUVwdwF$ z1mAG#8yKr-P)2=2_bCVU09%o`K%fv~Yzx)@9CcyDlPlxAK%;}r+fS=`&me#|Z*$N? z_`q8OubF$^_8?*GmbVb^#nIg-S1azPAh&F~Kv(~xPv3s{pyWRC_%Zk4uc3V~wRS9{ zxN+voi{*>c`)tQ^ys+z?!N2>{FTZLI^P-y8k<*hsMyaTKkZSB1pClM1ggNwmLjt0l zaZla)ou9t@!)MIY;)(z>Hl}YHxv&#$bc_MukX7T7_q>(R=zI|D!SMFZ6nl4%0>wmBQjNB!*=~V4GNO;i;^SdQ+1{q`O?~ji%S$73{AfIuW~Wa! z?177Gb1)|3Zccli&?0hND!%{pv{VLTfpx=TtT#v8Nklw__C~W0JD3}pBv5*sdLvg# zPD0XDKS=oI zMBUnu!f}~ORh&{dSqAvVdvZRrY`T*PX?a>WX<){Pc20adBzQBtAP_q^FB^Mlpu+PY zlHUz+UD7_1JmMiFkR&sAdFANMyqexVY(AvDZp_UJWpJz-3V+n;pC$F*{yWF7y& zJ9!^i(>f~@P*y?@_#__eDdjctrn(L&Gb4q(vzX9Kdt!v*f84@D8%=C&JHrciFQN+1 zhfRnazY482&-MDIET~3wcDfBjQ&rKHYtwal;#&Ld-Z< zi|%5?5`5#r5BYH3-Zm{*R)udxYgZ({vM#kksC_ECRbW-?u17wjJ`!MHgv;xYGf-|{ z7S@A$V{q5#sOQ*T2FV`x0ar)2?;0^}W_ktkaW-o@Qt85#d0%g21!l^J+xf$**7WNa zS2%to2u@aQFeAX}T((ItI3{v*nJc?!bbs*Sqj#S2ds0y>?I`h~;NR1;BcV~Ia}Y|1 zPyqKgbNTbk6*=L!JB(EdS!w`m0moqQpw!#xZ-?X2ZmEiJ>ETd#`j>UN4h2uZtcL;r zn$#?t)1m@|>2xi&u=d-$LB^SE$()&X0Q^XWP~NM-$^0hnkkRYt+=z}Y4nuV@w5|6i-C{k@LGhp0|>CA z2>EOK;dwE-^d;nKsqH*kqu4JDE*uF;(CpPU_?pWsv|ewmLND=0r%67KKGqKL-- z>1R*3)I%U(kXD4x961M;CZM zdsU@`w$d78vg>KaE%%TH?vCdrBf>^+skBJ5`D_vT&|^fmF0O7Q7OFRH>h=4chXO6K zvx7aqdG_e=t+PYa4U%A{8cd1;Ffa)$mT&+BiUsH#*sB;De)a9^|M`FY`;`|GiP0B6 zcyK`aXwQb0+42i2Np8pn?*P?cJ=p2bfAHQr|KvB{DQDyEC}d8JuH*Uo`r7=RdC@)h z!6nU4F%12h?Yc%3#1gEb`ELa{u&|;X7JhAy5Wafa;~C^j==H1gZn$fUr#dEF8$I7%&H}Ph(pu0ib2j7mFN-s+e8J~~ zrXAZ;(6a6xU=djQ%)u0ZI|Fd6z=0bHpn=W}l=16A`9vq(_u=eYGwkuVg^7mhVcXh$f3!Lc{pfYPx zk;AA5RRd;9{(bQhP}7M|>H9dbpl^nx7Idj3w7^8?-il?l_2ALNlShZk=+>LY$ZfQa zr!Fq3C@;yC!Aho*K@TsIIwqn`gE~Dut{fj$1$rogO|-o)Q+Gwn4ny(ycI&ob^=ov{a{|=>*Y*Ujp!W1!IJA={PW-Z$q(N% z=VtMb+0`3%mEYCf6Lxii7FOKT*T&qe{^*cKcQ=UJL#MkcNH&A(&C!SF2`X&?nnu*j z8RIu&OwM?c6EvRaCOTU|NMyf?C`lv1l6)}(M{H;FO2qjlMc_iBkB;V2bP|UNfIDEs zNmk6su2b2(M{SH=G3Fnk(BH{en_d_J0Z$x7TzKr(u?nSI$4F#`2vsvaS7jOc!)z48 zdiKz{qjx6-cX9F^@B})2{FoH#^Y32%@|%~2UTAs>Y3ms0KNtgYmf>+Y6fA{^pJEC5 z(Yue{d*@V*Z3aF??pF+#(|dcRI8K4e3W*?{oZ`lp$7>YHyUjxlu-nzyxXTtK{)eO zaQLNgDtxsDnz2L(oJ@0*x17Ma(bH!y@cpDLXWI{rJKom(`{X85GpJ*}rAQxyOiO&7AY z+TPqvJRI%UsxF{Vbj_y7lLoielZ9;cn+CRB{KFw95@rYPeG>wwyBf$f;22JVvLXv* zHSrtiA=F-7`n*0|Znoq$zJC42dk@@pl=o2J1}3UMLfvo2A05&C1qNF-WKc4Y?wOox zK69DE{d7eh=!Vt0LFMOPxBuOrzl!H#hlS$9^OL=Od6<;420C30nMTDHbkLY9!r9s# zKzZ~JfA-n4hy2tw-60zLSjW^5Vcr{NcT9~$8A!&$u*EtOuFnfADzx#2aT20Scg z&^(2<(YK@-fK7S?N>=r@*sMx1Z+Z-g2QZ>&fNy)WNr zX0lIfh`}}qH9~nN{^{;oEVc;F_|5f|83+Fuw)!ToAdLj4lk&mg(XQsX*__B*PT0U( z{6B>XPR&|y#LYcW9CNt?w!TQu#d5*i-H-D!FebM%LR1TcVV{5f&Bbj4V;ATEM++AC zGgPclEYU?nkR;w9loit1^MmryZu(FRZgGbN2Iimigq=*zl0!L=o?L|^S10Ft(>Lr0qaiPUdY^t5&hs3jwn5 zS@buAm9RRe(;N%Rl493<+$W_S;f!hm#LFCOEO-+D^i0=wXk+u1aoJSBp%844wzmpp zm42Wux)Q@prnL5>NxD_Oh~L350BSI0g54UgzrSqt$Fsl;)A@)wq^;U=-H~!`-hc1x zaDR&t0~w~_VYk}}KLV>dXPrs9Mm|@&#Rua>b$8zsK;c&6Cu8zDUuz3dZ=9s@|N0Mq ztyC=Et4pl=z(ZFjaH$wh8u1Y1KtVk=mAY5dFLZdGmT`(ts&2c zEv3?IIW6&?%S?^Vk|&4z3fEW9zq`D8>V7rWpMnpJ=Uc$tjaJjT4T-!_pl4Ck5>RBqTEXF?3MNo zAz{y7-@UxN0Tf6*v=F^iDsc&@u@Wd~|;B_W539t)0tMs#X3@u4%?5 zqVTK@)I#Zrc2gowqs3H#apx*f0ck<`N^DUu*mH(@sd70nS@v%l_2+L|Mjogm_rl+J zdx+-{tXVa~N_$=}kQdjHt$BN6s?w1NT`uhlzL)tU(UCU7^iPQPrmv3SEWLFnb&U4FjrF>jark$%rB#oyp zSS8k$PFIF2(f4)&N+~)Rcw}5*1K8)TQ*1Q{i)bm}d+3{|OKD9yhf8U?Jf7drQR1hw zdvK2H+nc}r#kDk@-M#pJWp{7CWN)hb+~b}avuOvNBiF61OlR(6l>PYavyb0B+%Hm{ zNlgOUbF2v`mW*<5gY42yyS;5VpH=3uP2d+QP|f^d{jGN0bab=5c7g2g9TrlB+a8{} z-;_K`?=IHpod9jNXR+PdA$-`F8li7*XFup8xGpbOZKhQ!M|jX24>Ku*r8^HK1#Nfj z-~_99ww$$E4Lk=Lznhzt;(iLH@AdPAH2kd74WXR7d(}#^G}31;n3r!U!|ewel#f38 zKyTJ;-U)8^BvUE18Z5RAF1?fIn%#Q*>=_2pU^@8ft6$a$rpzNH7tKS;0HHl87ziZP zqgn+3n;?hD_`%uH4?Z|O+p8>RyW5!(&o|fWF_t*%*+i(aJ0;IGz}IP182IkvxsANnBM9j6orQ; zXCNO(4~D>Mk$n94bh+C4;uqge$6LF*r7@z$V#5a2&?H1d z2s7~i#9j073FlUKTI1#xPVaudx{-uCK%szFIXSwRJRClfwx$S{IaAYeN`JL{b7w`l zoJ<`Y6*^bkT)m2i9I~%fXE;#$;E&#W^48ftX`Qnj4@%)~2X?#T+qx{Fc1t`Ekg*)z8@gQ2N88mg60@X?)9dYvud4(c4*IlKs0{zBrd9QxWboVkx3y>#H z#8rrf_Q5o3B|}>RiVy;>~p%Hp-USVeuZM4OG8m>q5~h_Krxh(uS@?_XSNMvOq0 z(^WoO7810*Q#pO>?aEQDJMF%F{aQ9oPiq(|&pAXjSS&d@MsuD+UN$ILZ7hz+dZ zA%r5y#vV569i~SZY(f~K@pQ4uU3>ohWvAm)J5cRBKq5bZ;0nj;JMs3}(c5RG$g*|u z;uT|>uF#(E>zY~Pmg#tv-mYQ=i|lrU=o)k%?&JaoBaUt$;`K%}0QLxEn7}gr+aK=Z z3Sb7g?wGx}^n3jgw>YlG5OAPYsG)!IVG9gzRUBOGu_)tbKfzEclVFKC+! z-r_T!25F*-930dL=Aa%N8uLjGCCl4p`{q^m&8sdMB#1IhG^9xwYWSX^_dst6L?E+) z$LD*1P^>Jhh5le17oxX9_G?Z1Y%lf-`z*);?EkCpZ~xE#^zAxTRq;GHF7EEi2~KV% zTud4M#ZXHzJG2($ZK7b?4TSW<)AQ14B`FkqAj^Ic<7BT|l*2}A2e1ZV7qew2sZ-$Q z&D~_mRszv+Le-eT?jLv6Tn@OKc28r2jj3XnGnx&?$@3=n+@8y@CKY|L8OK!g5=*AD z-t8OaKtUD|Jjf;v#z1Ppg#eFi<~zJU4Dwy~tf6(?n|OZjK;3%Vs4LO2j)-=!;&4M8 zYFqp{s{01t!tQKs=JRxMmNbeXMwQ5_)soWRP|LNCj-24ec2jS|+#fu9a`NcZJN)K* z#=EE|##E55() z4l@DcR=9G27P1hCgc=Rl#v9$!YusK7a$zVJSq%kI&+z-Zy>h7x%?#6EFDaO0tNTQN zX|LQ0<8Gt)Ps`)}pNGF}flm=80Y_P<^i4a*r zOV$FL^f|<_G<|d3bHG@cs&Z^Pf1rtpUfU)Q7v&%P=&}4GbBA`Psd%+hHSob`rs(ob z8jJ=>k=HFl(_$eFlB75qdfhsOtLv+;zxl?_WaAXnYRSCbOm+idASjOopoeX60~JM! zF}QPH8gTQX`kOCqK-TxgVQ84_JsZ&K^cT(vW4M!pGJX1P#l0djSjnc24--qHB-66J z<2`Vz;@hs%z=;gp%ct!CZu zeSQAu`0T-+k;v};p<-sUwSI9?|BJu+vd6M~HmYAX(3hw=!W1?N;4!Dg>{?|Au-5T| zOi|iMP zPda3*C}6X#lFAd>=r(;kjsMK7J(1K`n_5}EMg#F0cRDF=uAx+l0ubL$Vc;w8*5FVj zg&;^0{JUSPWy^xt^(;t8g4Xc4*BC`GOb9}6Fw@03A0unYUn}G<^ zL%j_`L+?EU@Hvw?UVxsX-tWbkabY0tUQ6A97LYjQkAj(Yk);Tih_#ge|yJ1j9 zklybi%leq0QO%f~!{$v}6@}b<@11N$FD@&Ck4yfU*vXDYi$SA9%xe;eF}+u-C>05Z zBN4#ScB!zZWe0~+H2YT0GA&Sj)Zaby_w)bTpZ=ZS@8{}3)cU4sc9;L?=P8dhUNXDi zI@_yNgWHrUUuQ1~MpCjco}D7c)%^Lw^kbZ_Af~M+7f}PT* zu}EVmxUE!^6#+%+n!CllS|XJ!{77Yy$sS7dP3A=-kN)ky|A&A0$2Wy4Wx%r8=yQkk zJIp!l526z;B!(JF-gnL_Kl;f>k3Tv{WLLehI=OIoQspF#_eU>^=EA@QXzU!;5dhC8 z{RKf2I}Kz`_y+&@odU0Od`b09zuv4j+x?(rP2k7&8|~XLVzYt65Mt>LM6%L!kW;!A zL?y$z0?10_68P{Ou?@%(wYLpC3MEvz%>Kc_spGktC58tgz_|S!C412M2DOkFi5Xs# zL6N;<;%KtrTwoSrkaMr7q`?x1!t!D}G!mdU=YclcI=~?|gBZ^)>w|iqK^pcM-<*(3 z1BP)ugpFY`seXyA-+8eA_Ic&{&BgU~+m{>8*iY({whULC3m{w=o57E1$K3S81&b4t zL%!Xv*PF`3V!2{BNq_KX$N#h6c!WZ2<#z%B0`b(sMv)mrUCA6o1DAxr2^mbfa?BMm znm}2Euau3j)zz~NU(F33?rg1<;B3!-VJ`XL`1ly@D9~jyczg;1SXQENP~(vGd@PH5Etwf@_7H9F(la!@kiau*ZpoMlHCDE<&YG`C*-t? zg5#UW@fD7;hbM_`Y7#$ZwHxe)q>F4-DmAYy$L$Pmi*pE9mmLuBTN1}`h9=XcOpyof zxY0sAk0HCH3;E%!_0_jmKmX!QbSop>lghAh$Bx?iRQDmZ_|!AJFt!H=)$)5!pX}@J zs_ENl>vg^T=Fspx9FG1#QMRRi#Zxdy5JiU?@6)MY;F1_NPB|MZ zutR;_Aw4kU81Spz_{a z-i2WbPPH4DDH)Y?+G0>!O#~5vxJi+nYK*roK-wEF8>7g?M%q z18PHJn5I~|B%vvXwxKg$IJ}?AUTrWG)R?<`z;!Oz?FbI0o&p|<9g zqu%0!ci&;W$Vysq&!x{1GG7cfk?z&S#NjJpKy&zKKYv*t#6Ea>Kw`dweIu^W1R#vy zG;t=mMpfqMxVTmqpRL7%(}N#>_AHa0-u2rqbAtnN7eKDp&jDbA{pnxB(#Aest!^%E zrTqFwN5{JweTb`*#oeVZEmh0b5Cp>j4m{(AY&XVQpzxGeZ27UMd;%hDZ$z%hSY((} z`1<1R^!V*hKmB;8uz3CVyLVTY!JNY5;#C!iwA(IBqyhc4W?j2a2!lwGQUSKRt#tMP zK1aG`-7A`0*;kiWSnOy`$|*d%+|GRfI%J&-Yx=bGHmBWnqucoTmtVYi{!+?@K??ho z%iD|k%{a|~eKR*Fkyr=dqFkg-K6rc{xDszYko#~~zZtyY@|Aq@pd>Tq`L}QAy#x8N zP~!;>UdW?u%9AoGuC?BMK_`EIY-NhPqIM)lhkn73Rh z`y#=_C$Zyh);6l;t}ZTFR8rOW6Y8zL%Xv2LxLquo_<#Rj{Y<#zblPRi#Tk#v57KOQ zffa@A&*2=NXO+N9;7z$oSrOQ{zWe!K{_U_81Q?oUEX$~vXDrM zl#x7Ejo_EF0jr0--P6O9@>O&A58t$^506TmlH&|xBv+z4t3lA_EQchJT9ssU##2hA zXQcbvzy6UK;JA;*o;W=_rP~Pk8FX6de6B{c*@Aj6@5sV{->zN=bz#AqKrZK=eDJY+=(%_#~`xZq(P@blvT&e_(MG&UwaYLq?|!ver{8(!2@Mmb zDDNq^)}Yy{=2DLz?7#QyI1p?5O{bwY#UiT}w~9LgsK*0f)n?Vb>uP-;zx_6=dGiYd zAoqi(7cs~g5sr!G4Q)I^t>n7XYC+qafZh)x^*E-y~^xfw!4F*2P7 zx{y&b@jN`jS?^wVTFHmBs#S^c|MpLQ-Hr{eJ_yHXiKf@9C6>J(>I-W>T?s{Tag&(PYVEMHdvgBpd=+2)`QLwi z+a70N)O>!FI8l&B&)p??%0h6}``T!T&PnWw`d$CiPu~CSpT1wt1dgU@m@|gFJFzWK zBJCm5HSGWi)=>hf+1%wL_15Smh8MP9ldc=Tuj!dB=YdkepM4!adw6cX@#5u6r!R!Y zfo>2?a>YaxE^>5yWUQ#$DOB>V&CczOJXZ(A#rH4aY#b%@oH{up>cFFo6YNUB!&V7( zr)RDf&}JHhW(eR@l8FKj+wbvrj0*HO#d0=`ZP#ecV^3=T_&Af&y3en!FTQ*A{r~=d z{kwnjZ@-AH^ZR?GF|u4P8G$pm?27gFK9bD5Y4J=FMh}i_k00$meo!ssl5Fk!ZD*!e zM>5Y1WG%bB)tmD31sO1N=SUJlY+|dMX6Ndr)A42JW1lOyK{ z);g8A8j0q1z(~QiT7O5ZGEk6mtQgI@_g=>66E?6N2z&PVT@~&^Cm><02JGRPSmI=- zdHoN6_czjj!USQF1t(1hLAbOKB;C<4VNVsyoS z_a`U3hS;|mbf|z-&t%bBB)mht4NZ`f2qN3OaUG3Gc2O7{9jYp+2PC~|D zh*$}}p5YvzOA6E(+bI&w4ko?GmH;T!*+Q>A{qi|eLV3LkIqC%J61Zv@xVUtk4Y>8+ zegEy-k4{gIk9eNy#3255daK_p@1G#bu|fI%W&Nuccg~2YxKx80sC*b=K!VZ2#8$-& ze*=kqsq}9`hj(@gzyGse6IGzaB$RP4;hppbjWvCTnFGa|mPAj!T~LyaMm(90E*zCJ8M-qP&BxO=?n>K|AI3BcC=dy8b`o#^Z1>eLnU@-p}%DHv27Z&o!$aXguC10z#$oA}L}D>V1r za`C8`KHe{X`pMJ%T_R&pvuP|x@bjbdhv_uzU15?vJ~X~o^GQn^mX*@kRJVKkD;dyN zb<;ITc#NBKLBydM zOzOEVQ2|3bIEcz-_hihbNSL8nM3&ta>!vwNVW}*%GQJBO{_3j!-SgKfk>JOdK^~O0 ztDl-T8lB*a;}}mw`y;Pb@UN#BB|CI%Aq3|t*}#h(FLs6Kkak%!b9_dX&yNAV)S%BL5RKe_IvS9e&TCTif1}K)!FurDKsc2 zQeLyokMV5Q-GDR$7_@)556MJV2_y=0&E+Cr-4T8qq~GBDoQ#2(L8`+;UkqFyjG2f` zXmR55~e1O6AtXbAy5_Q>>P>6t#f8+ zG{?XH+dus5!`*mf(UKd|Hi|K4mFt9hrTPn24;VL-UIIRz&SxCkwf!Sv_QiB~+q%1a zSqJo`5)o5BXf-CE{XUwsn>Q_79QRLce^2FWumQCegG*(@f`JnR@B!S{?M_<_&}AC< zkT0uxaSzg&Vl9P)v{~F|hR)D_>~14O+gzaGj3dh=C}RD^3UDaHAc;~q{q9BMra$YB zV#SJG8Na-^`?J6L`CViF{8gjX8YxrI#fk@ye)0V3?>~R>^z=bwVIH$gg<31Bdmc@D zdbL`yn%gl+U*~g)Nv?bpeB*N?Jm5HvFD0b zwjpi823>-tW0N5gTwu~ZIX^3wQkV6sj!G#=V@+aI06jMYUt8E{t8q2%_ThVa(NjGvn3h~*$eyx5`>W&^F*n*`_7a7a&=7%Z77y6=J*XIvnfmW;!VeW5(8P5 ziWu4|?2Zw$%=fM3#l_{#^_@wPL!NJ-PX{$)diwBAg$c zRSt_WIp@fVqvea2^;d8D?44ETdLxWWn;RY0;HBF)T^av(|MDk4{)k(I!+VN|WlrEs zRo>d+qh)6pj475Ew*#KI=4LK>lm+OEnmz!=?!z#>ql%t1gOI3SZ%y8R@7@3WkA50m z_ujnvrc^*PPTbzzFyJ6@P`tR91QX#@0G^nS=ribKjJDZGhC%pdoyk|_5CKKNMA)HZ z+Eynnb|Mmc_3HJ@SC{5d2m1#(W!*+KOB%i6zaLbmGoG_Zp4sz9Z=obJ%nRKMLEd!c zdIv?eWaQmmL*#-9#CFOt9<1E_wlvsUAaRI`V#gqY!fSGLe)Vnrs!!T1^anB z7825lm=4P)M`e62#C2v#zyEt5{^YZ>`G}^uFGNs9C;+D2S@b$u?lp-+xnu#-I;cEi zLZ%EPbu!u~pD>hsSzn%3VJJ zfIPaY_w`U*{Q)ihCR2MI3bjp`QQ&3r1!B;pUzSV9y(RXALy z?!gp?uSQfmrVymA2m%o?b^841PQFqwhyVI@@LI)N{AWLUVvz^Rztrg2#v=jXD$@o{Ic#l}k--|rNOdhFwcTl5U*DPw z**BcA`Lo!0<^w)~p*g9Ca z1)C#-pN$0gKmOex{lTw4J1mz$Oeia1KEYx?DS~%P8s~GNfj9kjo63fVp1TnzX1m_+ z${)wrG7$1^or0QyFc|3#4Hj^M{)<2Q?f0J^3ac6RZ)uth&?+@fK{_sRr?wNdj$Vsn z0GSy;C+T|cb$TSo8d7^B+|p*AAOhoFPt4h+%=MGuX!v^YjA*=8t@;;Ttx8ZGH4&4w z;sM{TE5shtP><^UV8jmzm?+NGKZmWq_gnne1VHISG{6YZ% zYnq*xZ|(-J{VgN`2SYG#4dX!)hoq~9hOlR~iMqaga|y27-z`f0f}I8$W3aa3D7EG+ zP%Q;L0bPz?Iz!_YUgH;6($jiwRe)q)ddD5t4_j#2==Zsc_)Sv?Y2NX zkQ@=nE1@5WS2OwZvm>9fJDL`!EpkzkoxJKS|^U)Os#tzmc6yJ+-Y-!9|H z>h7U~vNve=oHKZ+dLw}k#9-juqvK=t)TSqkS=SgeF8~-3-(xi~lzYp?JQ^=*4~uzL zyth{^Pl>>Vy6}zwsIBo7s-l?v!G-shy*j zy>ARKRDvQl=s1FVr6tQ3AjCVWz>&NSV3}cuq+4Tv`4Sa?z=YO>o)pLlUkOjGHzpsT zzx~B8UUa*4{_eMzA%aUGl-bT@6Y4%}E#61*fAVYZ|EoX#?5JFAclFq}-9865&Cv$K zougFyuG<~P1`~j?>&L72fW6%4O|x}*b=~dAhPB`*8?@TNsZRGZ8m*tab@c2(e%7r+ z8Z5i!o4$Er;9G2nz8RT1Zt|uVDkaU5%wO|j$CkRjd0pqLs|+S8=z!hocL+z}Q43Gu zMiX_<2j>%YCJHmO+Ad&th+93z5Zjhc5!*XFK0ZFJ?v|O=27i{ttkXe~VqlR9o=dKy zJg^ikMnV%#KJ*LI1@IK6{1}(w+FBhqz0*MmN%FeU(81U8g)*JPa>GED;sgI1mVjB_ z=%S^GtVvm|msjmCe(?gtYOq7$*&sZ!V{|G&9yq7qXk~3=t-}v!YM6%_TLm%!K?W^4 zF%L<5e>4uB+|slWk~X};Qf5qm1}emWA4C!@JCmdmj)gEGh!lPmC;Bd(9#&-fwR;1` zP1-4^1YQ?Lhy5NdZ4!Bn6?rADPnfntBQz8G?X6))vePiQ8TnfGwi~1I>QnnhnWwG9JgjitR?ues>M0U^>VV6e5pxm$MHfsFr4}bTg zN4x29yRp=F45sO=g09lYUM&t2PGeaVtJ)-_6GGc!wy1m$ZbbSw9Gc~PcfU4n8zK*N zPJRzRFO+3@PE5@nDibM2{3!FN{+*bua?K|zeuYUO*NfA)a z3$KoeK|2C$4= zkz|IZy@UDirgevPL^43ytNk&x3i_$_g6!WpYfM6GDG2`L_`m>H*PSQ9@>^he*R>DK#Eq+nwz077m|uT1&NBb+v#0j-&i9AGVB8y zjk|WY>6~y$0<`%VTS=kxoKqh&q+4s&JDo0I!m0P-yTzMV+V3?_s0|)iFlC+Nf_(Vy zTl?2U#ponpyR$k@c3O&C;UMbUyMdI3ERp@~v%6uk<1TiR7=<001q$R6Hf<+J zQG*_r0jZ_RZe_nxDYshn=P#bK)<<8~k-4J1Gq2?QJ7?0qXr1Sk5tFmAQ2KdfZG3H7BEG;zQG}oA$zer{f4d zMzlfZLdt}@A}3_N#5<)Y4EY`4+_g9~Gjd{1yo)uA3^>vE-v(mJt&kb+d%(e?$a`i} zf%pL%MOtTLJYPZ=s}4^(qcnHt8Y*(WW~EM z7381e6r(|l|C8T+Oc&Q}huJBOBLh=JYt|2|)Hv7(_}K9ju~(^5+{k2z1q~yb35JdN zvO|u+1X;QA%*@d%(Y9euQ*)7;;{wv;#T7ZR=!-WOoxzih#diPzKmbWZK~z9=d47Lb ztl+jMZeR3*6A^=eMCuRz=yyN(;RDUSbTmk&=K;9PRTGL_*;-e_AP!uP@) z*cuO>D-OBY*bj2EDJ+eK!dT zem_h$!W7Qkpt_dL;3+igZC)n+a6Yc~x-$@m&_^;oR}QERc;V`JU%kAr0vwA%L%56t z8Xg#1yx0VstzR8lhU`)}1nOyb_p1ogrU*>E*{Yk%uX);Ja=L3;BzY{Fc+aHGL1?p` zDH=+xhdq#ev|8i$mj}kkHutf(5>W|gDqOreQn>}lqU>mf@c)77=d*LEH_rc*YyD) z{&}@S09HV$zc#PRYlY1$^-$@yDwHH^CmQupyDbPPy2Zkl%ph2rXhg%{+^{yXMyLx^ zFvB-D4~nw`D6rB^1GkB#x%C*DdsD>WYoulwIn}bdN}_E*L|6twPodY=G;dsdSydA1 zA0w58qP>A1I!VB%ili{+0TR5Wlj_zM5U%`A;pyXZg1unE3#FwUV`ddw5~e5#F`Z$e zItY-J7nsfKA|2{5Z7agHsmSzF2)sdYu(a^hg88b7RTi>Z(Zn2km3z#b){G&yt{Z)i zd-_D_&OZZ;r)mq_W0PL4hj_y4Ek8I|Km6{G_RKC7ENb@(=*kbb&L@k;lEEI^x(%Pn ztyBz-8p?66FsO*bIcj=Ewe3VyV3-}5WntV?7&_INWrek<$c9;()+NF|8L2F6o2||M zVe;K~PCxnd{Ow28<8t=tTgS)m?$6_s=HG-qgBpmt5QWe)>VWX0?@~H(L$THVX)4kn?{HA zk{_*JM`Va4vb(#-wcTA2$K}GVOwM0}^!8;JzqXRQrmNdg8C(=H2-0UYOCW9pYuX6UM}rtog4#yn zE>sEM1xiE(4Mmh=Eei+x&I|W&HN3Ov-065vDBtnTG;4?#7{(#CU6&ADV!G+QjcdS8 zx<$x{R;<_T!UG(3p=C_yWc<~uOKiV`QnttoWqXVLw-FL_M&q;Gq#<_d zkpTD&FbcWypxv*9&ht(`Ju-{i2OtBS?GF4ktQzG2Uk>G;b9=vLX1*uoyl?# z%|tV$84Dt1OME9XT}5t;Uxqeja7hgceYOP56X2s^hQJSHn`48&)JEMmfGKP>W3g@M zrcGiLZEVO_D{p)1*G=Y8*eQG9GV0PwCYRqotd*%+N z29u#FeX^M9cIpN;1QVp^Y37FO9fQQv{R00(!U{7~jnrb&+sCLW<|88&_oUq!-;l#D zH^t(~dXszpx*m!N#7%)78Z}x>#BeR5n+f+>v6M)#R6*eJfn`9XQ>Ay`KH009J&YU~ zrj!_lvd>*>63RkQvX*(R#HYCo)TADpz`1L7!@Z-jTck5a>M!r|V%cG~NTYjjbO3a- z@kALOo|#M*dptp=Q%qv+C!jN|REVEfrZj*TwwiYoHypCMeeKO0zXlUZia1SuB@EmK zZ088)A3Zo;X+9RD=2VTjv$x+V9ab|z>lFF!`{&oqZX~gzP3pFpezai;BhNaq6?Chq zZC~BM%W+IFTPakPQchNOAnp@C(T4%{AZ`j!)9TO)WOs5$`_kqxn_cQV^T^mkw#WsH zpz*qdFzp+i!7`fOEAW1Stp&n{(WC+dPQJw4dgYR< zfWUH+y9`|B^0h9nk?iH#@%+aWU+YrM0zjkw4vhvw!o)n?X)P!oPtW)A8D3k(gWX*n zsNg9BN4$KoiZtcGWm}jS+e@d5r%MXDQD%W@d3z{tX|g_Mv|xgzF+TYj)cl@yh~^s8 zcxs6tK>WFOVj4&hE%Eo>J*P^&HDL?h-iFfsO&nIKrvoV@SL0N_;p@v`0`wURw@k>Y zm71~p-CZ5xa$nsVtU1S#kysdQDUoq(&_~Qjd6EWy|HLYl+cpMKKvGAUaq(sy^PhrH zP2;`AqF9yxlvxfZ-Fm+Re;7|P#md8HPmUiv#3cCQ%kKlj75Um#<-6)v?Fq*zluqRw z3^`WJ7@%qnCqN&p@a@Ov|NQrVE57O9UA|(u)odDVbGBmxEu~}#Ki2JJG)_582Ccp^ z&qg@7ZG3#ZUvJdE{pLG2t%@kv)7b!nu^JXy=rE?hRSSWh!IeS#9xt$8FJ-!Fk`5OAA}_tia8@3j%ws{^Znflk`x^sJOC*yfD5Y)ga$WE1!&$Ow+Sab z5*OB$-v*TW>9>Y$TufAA%U}jHlL-TQC5>CTn7GQMI3a>lD2qa=>+^?!3_FdO>cr8F>aoBDvayZ$LbbmZ zRWK-wOex3Hivr^gsfIqqz{r>aHlVn)IzRMkrp2K27i(6uEgjVRmCW46MS6E zJg+V|Id(Qn$?C~WIZWD>Z-rc`QaXC;$$KAv^7QnivW@yQFuG97;%;H;84ntRadb{N zj-6->pbz#cW-8Xn(Y{GuO+}@!%%)KMRESx@gT#&H-7sG21w|qiuksW6z0KHLo#?3> zvbLKOe`mK=$rq9=b6~fa#j-fy&#+lOgPP2KucbYS|LEfffua_VJ$mct@L(6>WbUP? zORREUl8K$K6NnH&2MAh#mD6S?Xu+vFlNo|!Rult0kKU7r*@+yC>D=>b!7%7sUtYIH zt5LdyjmTaq$P||ov%OV%3+$<1&SoklRls7<>D2+!K?}H@$-no59~_>&rDgxeUw)5E z?nwp%g+QDOnlwz8V3#lsjHeLhYHf3?Fp5iHdu7khySZD4i@RaDkDMzlLq9LGasy zMv85@t>>X5OCV6iR&1aOTNx3?22MP2L*{MD76RVDTc|bQL7e;yO?94bW7B4B7$ce0 z#p|Xe?h}xJAz2wnR#_HFuimk=?8I2BD~fKan>{>W&`N`1nmaZRv%7q8Nt2iW_NwV) z0L&H%$9*bUdG_QzH|pR1)!$NASkNd$PD?p2@}3Oa;_F$vJrX5ffRm=S4{CcK{qRXC z7pHjzbSnzDQeX-3*l;J?Dtu^puS;hRCt$yCS=sq$U0f2)!;}rej=-3Rd33E5vj84S zT%i}4_VkNKZ(^wDs#+Yyh%c1N$j(f7SZ$J}hyw zKRwzn)1>kD? zUjY#ky`lpLc1!pQm?u%#d{ZFaOcuEOs#~PbgEc@*K|?Yfs<-p0;)Xd>HviVMr|Dej z<;Bha`}bc0$1EvB_HLJDT{!Km$j}~n*>j_nB^U;A&QKD|l9vYW_k1S$TfhEkc4za; zU;YD3(iXpK^)mUw{?VcKwclSI9-fGbZZ*3lyz3gh3;=!I8}+aOS4)=s90x`{%3&ZKW-Itsaq;Wi!yhZFyOawt7=(wg>sd&;HMT4Qu=3(Xt4q-SeR($4y z+}UW<;K`zP10#ex!?+hcKRW*CgWcKz^~^L^Ngg~>J22pm&ftg(0HRSFR=?Ael6EE0 z6bbC=wvSr*;B0q*;W&dE1ot`;WfL4EjcywWdy~p!D&@i&r9c^`DYixt9b>#Wq%z2} z9&bjA`knG4etv#H?cMJ+KmX+~C~FU@+{1Vh%yY$~gZ(2%D6u2Z662+q_lR-0Aufrk zPn;OuHZN}*9SEJAt_^Fuad)=v2Elz;;86F%(eXh#gV-g$J7aRE-0+I&d>Cv}(Q9}S zFyuq%DFANH6KzDm+Xz+tcV9NMSks;OPd|FbCYRlE6Cq=a8f6*Hf*XSDG>qJ(j*Br2 z$p=(>BG2B&qc?QT89FpUcb+wa(l8v@fV6^9qnM^3mF8EKayRk`N)>{6PCi+xmG@Q8 z))gA3f9&Bdr$%jlFX|LT&T_4A(+A97+}wTs~0|7Dr4$w6e!yd zIpv-$kqmdBGcf4WK_9jtXwJDMDZ7}Tjt%1Y8ci*TeWq-I*_@$$v%X20h3>^~Hv{ti6|O!9)MT^R)}np-!-pS!aGuPlN_#tn7^g5Z z0`JcRPHh!5;K?{22Wir9a$ge6Y!+8!U$B+n=fdZJx0nf0G^!jAv6wDR_)Izzy}kba z`(J%aa+4}<1NgNM`7F)MybLzZg@vb4IMje9#jRk1yFqpsilIj6k1Pp6bmutu4^IJ} zu4|PL*3M>5nV-w=mVF8<7yVUhFYVCD8;KgZH%xr{O+%AakxP=@oF@zJkOj7z(@OPJ zk8gaCrSiiWKckXN+6^O}p%>>?gw+|oIrF5Bs5^m&0HDCsiN@W3 z7EC&~$QQVPEXMt10P?g8lte?F@tv4yroRTQrSe{}R7?A*$sN7PcQ4;uT;BP&>y5VP zIimwuc@VNf;avIxeWME}yyh)PEs;S(jTfe{#>(topS=ALVEX*|Hwre681RaEtIbA^ z#1P#F^;~FNuiGybOUGxsPM)v6xm+cdTk6|L43Eb$y}WB$-?>7@AL*>XCvirLzj=9a z!HPh;=yVN!iqi@6i(uR$T{W1;h|5{G<-=??Cr8Kbn#+sVxbVtLHKvM=9Ydv>Hy!<( zzyA9!W5HEIIR^!_fQ|m#24XE8aP<}jr!!XrsbryBrO*#b4=^^inqELn>z9h2vr-Td z@Qc;vTOxVVJ{ns}wqCncR3XV2%2W)sav^p2xq9;0>Ci5b*w%YHG3Zr^lG zSXd)?KBPAxLVFGZ$QCN@^9?PlycCue4xP{$rBlUHsDo)F@d8e;4Gv4xZ-YK<#Xxna zQp{(~6<2RwUQvCk0A$Y&`^0;}#fRXKl+1V6L}19Sw?#tV4cY?jj6(O!y*BH9DzipK z0gVSQ<3LoTD@;jxy1Bk_j{!n6Ibn@CHr`E6{pwf=O?>xuFae8Z&X-qLbgEgwWnw_G zIt!<(NhB*OIr2X}3260}fbX=U+}Ph!W7U!1i4NdeZHQ+Jd*uQaLTamnl*w3X5xwN| zqa;#AD}mAW5Q}wiE(g9H%5kmklx!S{BqltO18E^yN<@P!-OyF_!j{b--uTv-ZVR0} zU*(2ktaRDveD(DUs&bPX5R$JNr$+|!AfncZ_%P?kd*l50X*=-fdeg_BASe@Z|`n9 zE$uJCC$Y=^e(~X>Gs)q|E1puX53h4KEj1=`BkZq0G{ivit$ZVZwCiya!sY#l05ifQ z&6UYI6s3VqJvV%F4Dr$pVi6R+^80zc;zP1rJ}H1yh8E~ zr)|05F2U81X1eO3MXgq*c^aQHamlcxLe8+Y5>eo!5Z8k@`KS zPQ$@y_+CkZ<@Ku9XuxFpTQ#7qo4ge!h z4m_>@zr4DY(X?CJb#%r*e)rU5Ho|rw;EKvuj#69g+paZmbbLr|vr|Ya)rND+JNDIEyPai+#dDB9R z#9pC)=Bu3DI#eFy()hMF*Y#H~-Y5jm&(9?c++JNn#+lQ;dF}X^06&+D9r=XSYPxj? z+M5X(S9GaGO+oY}%p_xKL&j!{m4I@Eh1JbU%j{OWi>VZ(n~q0$WRtnx%vwj=%rSQZ zByTp%m!yIQ%5n$7vUH%p$GjdOJAlDC)$fB1CN@5v<0#k_r;*8V=hQ_YA@_?a+sPjd zY7ajva9$0`(LMbiyaOU*TLyAn5E8O+Q902Bg`8J^j{{AZr*9|6bK?-5B&*ff$VSt5 z4tRKOl`GpYK2R##%=^s8O`lIG_{43xjO@yg!Xtrg&R6VnDmCf6bB1UD3AoE$xBdN< zGC~`iukL2Ki94J^e20}SP(Vz`VtOZFo>Ng|K{ry}t86Ey*>a2{(|Q!E<@N%CxQ^#g zD`$P!=6*7>Nd`MZ7a2mViNCR%%7#hXD3m&pVs^Jsi4GX#jwX3Dl$d#=D?sU+7HZz83c<;mk;1xR^wwEIWybjtX+04c5a+fClpa)+R|| zM^Fy6@?1m6wZ+Y$Qnq$J_#NGQGH(TKcbxsKQzOfL*dZHP?LczWW5HWqN!kj7me4lx z2f!Pitvanb&H(5DfFgPaL+aw9b-4HB?bLA}d|Ld#U%Uc%iMet6yJ~Fy=UbwYw4=DUx>8aoe#@6mA(i};q@~eC?0ipmO zxvk8Y1KgaUB&Nf#y;jP?uXX~u`MuBXvTsf-c6f{8s& zTE<`nCLfZDBAY~HU16G*+xh=T)}~ICs)mQC~a)|tU^SWnC_v)k$rf4kwW z6caEkFw<*#sUV>q+R7hDYr$o=g(g>M67-B)x`kh zuXPS9L{Eqw>%rRh8fu-Mi!Ly^)cB@%?!-_6{C_0f*K@4-c_;QpH-OGLAvD5CGiOMO zLuo0lwD#`St}K`A3t!nc_MOZBqwT8oh3?i>D~FXyaX2$4;UIQI=Kys3^FUJ+i#me? zpx^fwp68n$jVSGdS{CweR|@mecJId^5d*A>diR)M7-Jz!)r#mABj(MC_ZLH!6C|>_ zG(^u~i6tnHP+_rRdR>wU&(`B#45t0@V^ln!VGt?bm&qe_?#>ksw?&AFP z?c0jMDt9+MlfyV%LEg(g&0GXPZ8Z^8qv~o1r}1K59EwHwPXX&ldn{OtUlddh(UA|h+^`G&8^bC-v1 zY~<#0jV|YZ`~mg-$)AY91pT>h?s}$G?p4b*-MUSJ*U~*Iq+ks%-p>^EjB`d0>I!r^ z1{f8%i#QOHNXdgMYfm+Hb$8wE4H%k?K#4sy0tv@6}rs)KthWpI14q0?I0h z>j-IOxI zYlS}eb~Fg@ESwC??uac%pvm?gPBwkUVdoqBawh~`ET`tTr4W3`C$p$k9~cbKhvr(j z>eL{%C>qX~$;B{w#I^G|B)CZ$Yk%VPFrzufSI>S8X{*`Pzx~}WLa%HbV`2|L-|Y!( z8}al$d>M$}c!`cRYh_4*kGXYRkiEN6oDPCoBac>@Xi+PswT`$aODn21KgcvpV$~S( z7~|Fo1%GtXK05E5ha7_ZhX8EIZmX0wF=PkhB{pSKm1GG2bqZs|cl&*r1GhLAR zOQv^_%itiFB$`TdFjg72xz7)Qnb;%Qds(vkZlOw}TCEmu?rxYdO0^bbQZ|_~DRkQF zjeQ`UIzsZEw$h{B@P3lKN7{by5!~Oi)1E;>=^E8t)`s6-v92sR4hA4!*k+pQ^zW^oM47>A2@AP zltCow1yhHRcJ3p4+ij+E59!^Od;3Y+?{d1Xlqw;=h`?+%tyJV*E<Rc*|>-+BY zc+o*Vnm4w2VtA9YVvx>so4%;#UbX8dVuNhe-L7_iSZK1`ed4M*Gj( zR^w2Cx7$Dhi``*&_B1CJJn0~XP=l1(WW8ME2t8xl>a}{dTB8x9D{)AH2+2XE*QoZX zM$-r`pVIFdt|Roqlt;z-KxvlEc6s~V>-KTUNP6kBT49Z4u|Rlew`i}mf%uXTS2@~@ z<#rx{R~&AN2e>XAz*s-C{K$8S28kBERLTtci)^ZJa(v1}lTPD9B|;2T{-@?AnFl6b zq9%}zYQkOgOh)IF2g#rwJa(F7??|Bi1f8+ag)#PLfAP!jC_Yoa`*jEZYjXG3tGiF1 zu7&?VX}Y(Q@Bn1?Z~FIil_dT0Vy~~SW1e*!v2HtC%ma@$M0JGWfj#&DfRoS&$5hg2 zrC_8=PWCn`QAAKR*ciM@jO6f{ta)GEdzp&|4|Jbs5zVmnsp2j*E>-Rrkn{P;FHkHd zNp(*eIR!R`PHqCafQ*sYLysP(x0SU`z_q9oA+8sfES2>iIiIJLl^OPB6Ly2lqcm1R z#%8iabbYSZ2r0~%Jz_!PNfdp`K|tcO(Z9MH`V(lU*Tcr7#CFwfLIJ=LU){whd)w3k zr_gY%5HyR&jB#Iz08u;9k^!~gLs}-#RB(pKR@x|pFKGZdi;LX}scP8US=Y9tcG{kF zc`lt~Y#_cn0!Q0nJKFLtTm32EUGJOP2y`G57k{%e(9EaM3ft5gN5^c$9h99Cyfe~) z+s6$;N|RV0dbCZ6iU^-w3LWC00t34LknpSqBw#J_z%rgwq#t($LC*?;$vc@%2?fhe zS4`padb5P~4%QQQ5#vnOB3VCNjUUEqnlaGE)s?s)GF%3h=YkdTq?07~NTfH})m*~* zLAIPM))9Mq1t9cNvNe4sX^xF=?n3eHn~SqowaFCKI*AF&p;^}jr+iO*f3Z+yP--^w z66Sh$Bd^dF*gRAJkcB*v?FdXE_`)`4a%DYvPDbcH+g^&qMx>)L;{;9cWkpO3Qn3pa zmPKC~TD6KD+~i_gY3wcK2xC;jB@8ZV2*|261a>5bBXrJ!w-_ou?mrIh9|*Zf6WHP& z?jFnq2@cAu3>Gh=8;75(5GWDcnamc)CuQ^SfI9vrKJXRsGPQ|y8E-i3svcNonb-{A zfH3gHe!=ZS!62Y~k9#m8sGyX1R7a6cB+xr85N))sFHj}b1Lh!xR~nzBPgzb^FN{h` z%lH5O@sAi0qA-UM85}_-5GUH+O$?Vpb=paY4HqEYfFe%=>riXm)`it0x1;O({A#UA0@LthR+Y%{m2%ZqfDZ!++25Hr7wErXJZ z##zig^RB)OoNlklpCX^L6X*v03R)5Enw{cT^$(ab;yfBpLa+UQn-$w;Afeb!+)Px9 zb|r&@IaSfwFNeK8F_;rKT7MlFGY5=dB|hC<8Zu42E(d^JKl9A--NqJ$v0X20n?(RA zoqANI4jXlu(Y3_nqVFOL4geT3B&WGw3A`SdF+q}oEP^wkugYT`gZ1F@nR~}v`81qv z-7{iWq7#IB?jQy=Prn70|JP81jPBn9Fn($oPaG2;k43a5I!^Dq+cY!JxW- zoz6LdWkKUC&4KLR-wQ{e7NUCfO7VLzbGVrpe8Cu$KxA41U-W)yzh%a8W4NQzDF9|Y zu6qh(FdyIF-`wUvNx%Em>rUqsx&$*v@3bWHmohsst`RYywPVHGTd9WDD;5FI(PViy z1|Z(*v6BSyxcGE*9`1?nf7L)t&* zkV@7*ah@_DTDbmECn#*qu~22g;D=_W;+TDOBVzn72!^M~Ip6~k|B#Kz+j8WX?;BGu z!yxBEE0*%ORcUCx2?1@Xl8?wqB*&g7?t!~g;vc<*a^1<|MkmO#DPHs>*h&sdB3}KP zJ?uhq|3GTgEf6r1h_v`f&cu37f9Eg8?#_KRjdp-&x(N~(fecr1QP8Y zQ%VJj5t3-u;p}Cz(j@w4@7=CT#KVTypuGJIo7kU}YJc|KP?cZ#r7G>S>t}+K^U_D$q?MBw_Ee zS%%_hWSfna;oHLgR|Na~bp<3X2w{4tyyOj4ng-1+K{Z)EmQ(j zv#=1A9#=VdhJ%snLxG6Qv!T`Px?}YyBj7-7&O#mg^fS!$EJ=2ot>*gY=%>i&`cR8jzQ{yD#aSMXXtZb6V$d&qd+8l zR{tHoFh8COnB#bV9D;CVCxfgaB}YDa-yb09OalPxsbd3B2Ps(HRm+o= zh~Ujq4KH_0B#RwfhNizU%EBbB{EF? z5Y~a%;mU)U8y*V2gN(o*pAm59=n2(bnC@Mau7UWE-_6^loCx+vde&xnkhj+NjOa~x zcLy}bHVwqg_Haf~$AnZB0OvVn4vj8A;knDa6g%{Vg_jMX__fW+TpUq`_oVl0wXy?I zb)ja~Y(~kuUemHik3wS(pm0}__$#v#f{;OhDZeE>6#J>a>!~x(0*%5Pv5B#@qL~`C zGO(fF8@e{j+#ZX0_hBqO0_$&vM736RSVX3}OFM?TCdio;Ye1QUZ3=8-vG$3Co}ci9 zzCR(N?jMQ>tv70c*3D95+8}mH=k}iuwv)}uB=A~Rj_e`73XlM#=}vCcN_N{^*!n^` zqwY9T>njx*D7#Nz$0RamougDzPWgVlQiD-4M_WTW;D}HGaoQx?LqadNykJiPSSOwh z=LyWFdn~WAPMXbH=d_N-9zH67gkG%4JHZ_CVbeuw0FyfdZmQMEjHxn2El1=~hO7pe zW@5SDEhHWA!Yf_4r3e8{s_~q8eSE+e+ zDtBvmlUk?5PnqzxK3LN9ZPub9Y)RLk-h{(4+Cd7V*qcfwzn@>o2DID|=Mhc{{0*K; zVvohcE=Xx<9H)rewIC+rQ}AQ7kS?%r_`4BYh;Ff1e{g-b8xZ8h$!S3JyEo)zdFetn zp*$!R38#SFqQ2sZ@oN(+(pf(AyWRdA4`#uuH1^<=o0z<%s&K}$|ARkR+HDdrz6a>n zgNbeYbzG@nEnH zA<3br0edZhkJ++_)7_zZ7*9?bba%6KUX-%tc%v7#bXhz`MmS{iNmJb7Z zoSh(Eq90OyyjEI6HyO(M?M-s~lmJo)f?KXMTVgkDS%KWU!tSuTU#Jl;o_zo7@7t}4 zNs_EUt`H|&YR_%FxxT->>yK)!^Iv}d7bllT@Kw28i^V>X@NC4~E`pFO`;oCQR3$2m zCF4tmYl1U2y69cd64^$G+vR{Z+b&I^1PqEX0NzH6I%&HkjM0+lQytS}ChC6@vj_rM zEuHJGLa$fRTdU??#y{cS4S3R8ks$6EMv}?AQ9mRLslWQoYhD;3LIe)s8d}B zi{4>L-8r7No2}Q!b%n7?+ohrq0QZA-&^6>M?phDiJ|>e_DY|Jnsu(h71?cwdp2Nt$*>Yja z1NC}@KWRH%d%CUgUW6*vN5aQNP6BuZUWThPjL^k|vmo#cxPLD@7%{vi8%2}-szh_W z(ZbdO#hV_B_h)RQ`y*%!rF<0ET0d@jlfcL+>DQs29wN1Z2HIho5CJ3nr%~K~x|VeL zd~%xa@Yl_7a!+Pp#f)CRk=x)oU1`_f{_6PMyH+um?hfwekMmr%Hidpgmx9ocEW|j! z{CLxsjc_FSB=dBTjib%k>J@1vDAAAvbQc=Wy{(6)*iM?)=V65*JmJ_jCQpccQXi(6 zU;?%(3RRP4Hxd)si;dr6VuY+zY46gNmdx6w{4+(fk76}Wy)`_uqy%MBYQKuD=CI%J zB;rhB&W{%IS$s#O`0|P^s|i)cCMhY{oV^Co z=k_9T7GCva&RWPp6H*3>yobjvDUyThEBnKvQ|M5XJ}VzsWK`**VVPw-CtcA-_5VE8 z=wp8CH6BsvLM~jHO#YJD;>$ z76wXL5#_*@6V4(jIn~wd4$y9af0*$+b+Rb%fcnhI308EIz#`luHjLjAi)F0|?Rn9S zCKbW{alHIK3K>bd761S@dL%p7wQpMzdXoxk` zG@*JZyrsXwd?Lx%lok|UWH3Q5Rgq(?%rBuLN*xb@gu$%2#fK6 zH-!17wE-bbX^y07<9MDo?PlZjw0hJAoiKUyKYzFug{*sXx@y}s1@1B+jw;v2=?#~E z_@^rY2D(AuA$E}dKBzzN*m5RZ4se^_YwAlg&tCxbc zb(q3bffJNDLhO~RjxuE>wP#rjPOApk0;u}5P&EY`*`?(XJ=w-E%>gb{nWM2Mvvx{L zqyr|0{?$QRYjYPX`}Xq{iJ+WZd?%VkG|zZCfidbuh%gITpk#A@Z+>SDZ9|mg^oFN~svdQCfS;u!&TrLevR0HY!9mgbG(I4q0G?c_jDmd_%21diba|Lcv z;x^7M>jz1aNgGn^*xEP|OR-oy6qCR=MfT*}_1#{OXLN|>MOJS}2oV{Z-7=FuAiM<{ zgHkMv=?uXKzaaY7FRn%r$_myIU}4-(Huc@>V^S%VnqBAs(-DyyY4e~uR;#91t!Sa! z_Rt4?ql%eFftuVhRc^i5kpd))WwlKRcXQjSXyo0ij5#22OfNQ>*qPxZQ&2}<{law+ z{F+%?hCx#Af!N(Wc1pPZ&6nf5A*n&HAk8ch_wYWeSIkk_5U+YQs>E%?b;-^k{v^kG zqSlY;3ocUV6cEf2tCsV7)#_C;w0_Ub8nHSgLRv|8GETviBA?RvL~gRR>~QWno3Nhl z`2E1GWMT^@1_XKtSlqhL@ijiWrOn1;c1IP$6Xy3kbNpgN)SMWPLeyj=g!YZH*(Gdm zJ0h`wZd*h{iuOu9vP2kIgq$d8iCVUS-^VAdRkA0NJ+&R=lI!)=RS$P~dd88G`ThTQ zwU;cteSO*Ka3H38_oLtc)++u4Fak^S=V(ICCK#{tsZt@P$kSMI zlu!lX?+hy0&&UW5lO&YV&8O~TPg<`g>NxAf5vSCX*6y^_K6>CFD~DsPS-*I7tSle) zdz5qKS{W1#O9F(25!eu5Sz_)ivj^h{2Rk@hyy$Ja*}Qyv)TZD=Bs(AQbbu>=};)al90~|Gi51HDAb(@j9Qt$bbfNemq54(ZzL{R z&W0Z6KZ}DRdhDqB1inuTuS@81|eTL11evua-eeN%ol$?RkI`wWxUW2@<4TI0tDb=qsGLPAQ{rvyMw<^%K$Rw4EM z`%b=Gz)`!nZJRKq!mY?DUZgY*N7^oeb|4CUqud=|7G_T%lfR{@r zY87*l#Taj47x+it5O0M10M?GT+eR+xZ* z@ui75hwyMAd9e)9aa5@eYHWv_&Orc+aD0SI-BKo8uo%oG2vMMww4tSRjJN8isx97a znOB4E*yI2jJ-5lxaTzGhfvV8uSW+^GpF$sLjMO6T5Wj)>A^e1@H&B-3B_FeMta0>M z!kNUP&ILy(tm@(=aT50KdL;T6ug_p_H&<6m(}E))JhIAsifvU-I6QEHid0#gf=Sg6 z(h1+czkm;odqeriJlzx#k}W7bR@2qZ^_6%VAzHK@zPWzg&E{0`3VC6U>up`42cnM9 zT}v?0*{c|3W?CuakHL0Q(#NA@FX(Ws%*e(U&8|jA572u#YvXg(EACz` zxcir@)XQ+&dyEJ=pDX4|8yDcQrAb&KCDL)>3LxpR&)qp7kR1PODrp4xeEKw%5^3-5 zyS*_Jy8C&yKsuAQrg=5HOzd;k-ZYVvj=xc)e*F6MAUQSo&s4~CD(iBW z&f<-d5n~mErrh54AcawFNy@>7t)@Uhcd-r^36 z7PC+-xFgBGY6~;3EXhl!Mbwz5JU>i}ewhx33q%Dx3ip@^A4fun;!Cf+ z1-PhqMH84CQxn{VmP6(L`r8w<2@yv4u2RBa6u{A=D=w|MFp5W^800cB&g-`)%>8o@ z+EnVmap?C2q@v8o8aed?!M3@gbu&=fw^5mdpwUrdPeq++p=dzWn?x9HMYXF}9>kuS zOyPG(8e$-*2JVU2OwzCy3%m2A7_=M5ZOx>C3T9?XtrGTDtsEW|Gxfs0+&3-j!+pQs zAJ$`mWMnHi9i8#5ct#DkjW1FOt$Sja{m|y zr-Xi-jeXR(yP#9SB>E1dnCg6HXP*tMz~iS$+0Z`F3Y&XHKpXS8o>!*Rc)?~uu(z@V z7&boQ6*xe7Q5e!N?hhW!Uf$Rw#qcd{WUo=4@Dk~yiF(hDnvNuBJDkE*2uOZ=(aC1Y zH`gOvyKom+SZ@H>%DB9&4fCK*o7KyUb3znsiVCA}kS*0pCe18nV<(R&b&Zf2w^Y{v z9>P6$M#aFN@CKoI)ov{CVTd+}F6dm_Cn;ObX95=`tgYn!@{rSy{M=wbQbgw#F3FAJ zgmRxQaLp0{BS_3~Pppq9mkp!3%;sZ#aZ?Pg538BYat$|I0VjO9>ws}iV-eLGYbYm~0|5HXU?#2C=j z)+y=w@ii)8?@=Kg%s~1uz@n{r4GPK7+_FM|@uKq$1J+jPQ!1qqkho?R!=5Hq6m_s2 z$RrQ)=>QiaHpkFh+Bby{>qg%Z8==~2Y)JL%3Qb6 zghDx$`u^KXB&1Wytwtp+>?f`ygvZXXJ2Mt{`4U3RDE0`9kcuwD7#i!gLfYY-gx%8( zsy{*9vOL@+{wfX>)hq5Q)Ky~eydK107-}K|%KB5!EaS3FxS>KL8n|f`s7?XXhiK;N z<$~eF61rm&ZCdNKy1E&Nh2)HucC!*IYv4m3<@EHnykE(fg>YD@HjVG%PM7=_w zBYjVyN@GB9wsC|Rh0Pp^;&YX%etE2%9sSW1+*7O;;;J4RDV*ZW?$*@PY8@V*pParai%fYOPF2blkc5lt2V+ zuC5#v6^+x}z9kcPhbdWY4|f3Ao_ZqwC#j1O1|dTPjKMmV25(7Aas(%Bf-N*&ZKs2? z=l6Ba`s5H_nv@Xwd=qo13(+sj@hIv!9Kivg#ZdSIB4_4d04}6Y9Q}H~6q?8RBQWB5 zuXr$~-cSg>{-(VGK7|Iv$sVFRpo2d*f;gI`kYc3$i}l9b=1RBw@b&7ESW)1IJPv%3 zYE6HmEi7?;c(toFmP~&*x;Monv)UQ0jG1%Z&~J)T4#W~c8STn~x$Y%iSQ!+QW4f+h z%F5eqnh_EJ06+jqL_t(ce|K>X0b3>4H4^?@-Uz*9=9zf`pzUHVl_Vmk#H!lfMjUk? zpMSbHqb0&}E?=Gg?ce^5XZ+#EKRt_%@@%Jw{b&9FnFuUcViJCNDk;C z8nudykC%wvB{3hjUN$0S#bKGu7&BH?3312c@Byjhmgc`jVP(hgFUHJzEl0`m)XW>b zM>%e!ndf47iNE5?!amLnn;#olu@>)tagIhvXnh9{>wLvXnJSzgm2TK0tQJqho?cR% zrP8(y-HORerXl^}+!?d~ho5`?jTlJ9f(dhGOvUaw+^=RcXKmRfO$&!7z@~4ui}|YT z9c9(r;HpZiZJjgYG%AH^tw9HXn0;gznTI3E*kI`c)+5AC0R#6V0nd8Wa#bxsg&pUL zoxoMyta1Yb-n9~%78&*gMB5dAW-3+K@OI)6J&3$f?=GICU0N`%aGDf5 zBB?<~qCXN^mpEJ-0Bfpmp0mAFYBni+4NjR8_d8?=?*ipQo`&)S)9s3Zb#Jo(RY0o0 z)mQR)H3ihbM`uMLiQ|3@`2iu@4uqi+cSn!3I}rP@K7@ou<`^>oP#CqnOqBSk>t-Li zX!eMJq_p|y)`m{Ofv6c$bz|6E+5|0Pk9=5#y-G?G7nczE)DGi~T(q7^UDOxu(F{gR zdwiG$5J%8k;%VG}V3Jzoo!RX__8*4zYT>lgsFcD} z^TOLhYO3A4X?NIXO{cg<9>F(FO94Ce?{COP=nDYKs;hdXp??TQ|9Ne!ds~^)t{~P` z|5Dy0N{I$C3pLL5`2FC(bI;iwXbOdZIzJA?b%7%~-1Y`STs$5M6;}nNqJt5gK#9AB zZRiuAWgN$d?Dw;wDO5=nv&MS~7Z2V=h9BS(?55FB>^~ol zyjwp)+X7bQ!sjW8k@8{>X%eKbxQw!ZAG+{a0v%?m0S1x!Zxtgog^sy1v9s-yi_2H0 z(9Pu4G#LJ4UTVxmYBq}wT2W+7CtPWjT8+g?rx4b;6k$e;wThseSg49kXu^tjrHQbr zAb}a_4;?Zipe_BJF0Bm5!@&CfkRo8ir*GeG9Z9zA>p zQfH^Dm2Tg!0>zd8*REILJ&aVoL#NXsm-_JYwNsa^1_Gk=hP>rV)mFW!F;`Y-(1|vj zGr*NX3OPW+KhDZpU984CjU3Z|KPBtq+W4OznCtArnqtYi*VPf2O81D0FP+z@P0zD&O zMNz3(Lc*R(a2b?a74#f5!6thi zOyO){n8ZB96!0+mzis&+|8xIOe_S_O%!s~Kw_uqRxh|mxayYlb(XOvbmnoPl>l}Cen zgM|k)wCTf|VN~s!Ndk7xYAH?-T7#w3FGyh7RovKejOO$3cRDuxBWdnYYR5!cY_m?Q zjWZUvC3D^Hak#fHbk*oYMW7O)1}HK5kqKz&8;ydcK;0k_R>-K42LwY3SJ4Tf zpLgG8vV14@d$mTBV8H0I)jmEwZ&?eJj889~Gc{)F*Y8g79zQDy?M6g2PdQ+X%nlDU?ya0D2O;L} zOYU`=^>(A)6Cd)lHFFJak=)E11@RzhX$WVht@ER*(8|xBhA#HjseDe10x=4M)UHwE zAvPxgb`)2ELDQ9b(<)J}oHmc1X+b4Cv#el}mJ%qFw1*F9Y*Ub}N#I&PIf z08)e0^@?Hac948e90_1aKZynlt4{(4*$a>~5zT4jl(U4Xkim~!m5u}Y2mb2=MvFog z0RzrDKG8>6BzBI&HTqdGkjLjuH=AfEB#e$byn-n{;M%v}z0!t!{P7muZZ6Lj9&teO zl6UI|tLiFUtVSl6J_`gWyKE7&F|=>HfGzO>G7`VOtT-XPs8Ln&lYncl+e{?lDWyXv%7>0_{Yz1A9mWK#^{^Fhj{& zbU1TlYDNHvd`bC?(tu#o(QLGU;s6j$NX$-xQqwr*%V?HC9eF2AG^u;QgV!-?_2PwvNZJ*B1JAd|l$6xv3_qS#Q ziB(tg%8DjHsMIS&D1;crZ22%#Kpv^9$*zUZaLZ=+OJqn1m6Er32y_129WgsO!os&Y;+zX1R1} z++TtRwO@R3RD_@pEuy-*W4Rnx=%7tk&|?$F6mu27iTw#>#v!}0-C{6sGoQtd8GJb% zKNAeTjKzaAP2myHdDW9I7i8c(IXby;rp-ou=^RHVZE@a?GgJ!I4hSL;qr?H^PI;mz zec9fT=8#NtGerF1)z_=>1l~rK44Q>j?8uX?ke8fj3tnpsHK&jF!zCipn)%tF)TizM z2kM#cWO<>|ha=i)GbaH4&XM8pag(n$&2K80nd?ATb{L(lnp&{e8YN)RkokeXeFxev z2jQ7lv+Ky;yhXmP=Z{+ti*WOLIJ_MKF|(t|NX}x%%spKp&Tu-BM*Ls=i(eNqncM3- z?^tEXD@EeYzKenyID0H(!RM zp4U>$AKGk4;@hwHmMg$ea}t@XK6Gt1(bzvi6uPE)K$qrOYutwOVd;I;Xjk4HAlZREgzFWfsGopAltI z>n?1eVBcJxgeox-Ze%Dz(k6*|0~0Tx0{mys!kp|iS|?y8Oc7S0p6Ohi>e%3x#tO!8$*ug=?U$9tJ1U3>dR!66IwMC7MBl{W;yB~h~6R3RiC%2Oy+t zWlW_O)Flsth?s{(BuZoojLn{9ZZg{Nnws2>2`gPiW02zP})$<4F;9@mZs%XL4 znXQL73RB;lw)8KK{6@JZXqB#y5`eE&{h)up*X=D?IFq;cgR86SdOKe))L}K2HPF~2 zDpgXr-A1MI;pf}WU++QoUb)NAm?R=r=>@e$PVgqJU2zFo2OOgBzkRRLqarBR>m4~C8aP*3M6z$3!2kgj;b;PZ++B1|NxJ^m|NZZ) zjkdzL)iN5qO2$tMcKPNEX8Rxi<3AYMUMv+~Uz}6PHJVkg2HWJY_qy4gz_l(ZjmqoS z-=L(O!WVB}>GS1Yxz(EWV*xDj#JRj5} z^4!M;LWLa94kf4{3lWw1yo8&}!h?<~g{icOt3pZ$ z-tZL>AqP%{tm|LBe$6arI0F$Oa=C0NgJ;~_j0->j5OrqM58-SsS0EEoyr4RKiLy|m zrSjDKNu~0IYH--S6nV-Dic<+}4gwXZP{`x+w%OoHWpT-f0AN-FMZ_cnhi^Sd}XY&QFrq!dh`8Ho=PgJvPkHM_wwAaBy2D=#B_;3ZuJT}U;p zyMdpH;S&88<}R{y=TkbreO`*l@^nzF9;bSOgo`==P^w3Ul{4*fetr3RCxai@;wh~r zn{1+7J#NSiDKT{X?w7xAo)Xk#*3;$ZkDt2zfs?mcl@gdVmqBBoO*%dc?xyPub`^j6daKCcz2utx{Av98s&~<8yg3)>UWr*C`>6-&gh5ZX;PH`R@yPd` zp;COkCdCl|HwqcG`rr}D7f}oQnk)oLYGp{q+3WXk{`KOnbJ1`D8K71r{IPGf;?o{% zrdB`F2*d4I)F40)j}P~^x8+Pt2Z2pV8pD<=@^HL)^IE<4_3FnL9SnX0ExGKR9vz>y znYC_iZ`21)O~CgEnI%>-^?cSjVVc7T=1c8*zD})JsmCgzMtF7Jd7KOtU<%}4{^~t9 ziFQNhy$8R}7Q9!7n(Ftob1b*j4UefmJ?Th|l11iCt3*}{GD|BWP46#cp;&vOgLVqI z71hj;1NKBRZhmh~tdbG`Z!nv1;6q=9E#=S~Ns&tyDuy~59Jr#rgv8uDpm$wkw9^po zJ(Mh1!)#y9k(tR9vk_RKR07%2_?o3r@?wE+dvU{o8BK(pG9zV#uvPewR0{3@Ee0Si zI%Agv=^1X;2bE67liusz084$m(6*YBoGe~kzVXgi3rVw@f{i;5qrr@iX&1Nd_lHv{ z9Mt?Sd|gl~o30-<+BQ~zB$zbcd~=%2 zjXr+)`Sx0JSwdJw1h<&(iZ(znZ!zXewk|2NjvMe#sbX3^d?S!gvDso{cObNCN6d~FQq_+f-)h^TcT7F zcO3cA%%=voZcM9`^k^U#V5VRekZ?#sP2V~4jdaXeX(`L`c$E5||EGU<(yZ%8uO3F9 zuV$2p#Y}u2IHFJ9Y>2$ZSSH@SJA3y{lgAgXCY>^BPlo*Wl{^PBf< zsV-n~tK_ds{oC_S6g`Yc5_=Uwo<*GP5F;6&IzGg|7j3uM-S_&=GYEmoKJ?r?&4hXA z3zgg6A&3BZk;~x}iaZMv;}SxF5BC}-HlCg^CVu?-Ne~D|R+<|0xGq$^+3xig*Ejb> zt%9(adMd4NIv3vnSg6f_@09n@o6nyf9`2?NRzOMm@G0*)svfKECF#+O8;(IRij7>Y z`8)e+yK1%ExjZ{MIw|By$__q#`e3=7b-t-qTM+h2IV0m}n|9~qVAHnQ<=M&QW!rPU zzx(Q7rJ{Z`$m7{)=-40VZ-V1S6Ze*B9End3nY#vvUcnq1-V%d8?Rn^)pn}TT$tULcF23{G!;_)E9I8l`yfDPtI^!>YBmfN$0*uzk%CMLY2h@p`=P)+ zXRq4^REm?fzgSM6pySPG{wN{q)`^44NSYA!iJMF0}#WCn<#buijXj$Ooh z2+xQ}DyJ8(-~ICShOeRfH3S_SXPhgbMr9ELFK&Cf1Aj`HD3C6D=w!%)td^rCI>%Y6 z5Ct4qgHh^9wVck_yZU!zw&XNM{A%19jVAlcCPBu)NzTJEM_)&V3{$PzKTElxR7@#N z+RJ7zIq|!ZT4X7(?^)AS`=`@+TE;;wqKn z+4P}1ztKG5G$PMJ8>If-zj`k>P%bDYuvAW-Ulw}Z$N%#mZm82KU_=O55O=qO51+3Q zT$N^B1?p^ofPs_M6~Fz>U$pBrL5M^Vm1gDbFD~+Bn3c?n)gOPl5vo-zHd-f*K&m7f z$VW?WI7D0+&QBOQ95Hd)zr|y*_C% zXghS@y?b|le!>$#mv$gz*0cKgqc9(uEll-Ggn&m&*o;#$CPv!wXfu3Q(Yc6B*j;LXEMONwDqlE~JHl2lvk z8i9fqUuJ1BLKi|>?ggY`RF=BbrKJlK(I$*XuXlHQ^ZDcTVG82q%A&4b>)@ z$XyPtuS;J8Vr9#r@pJ##KodylKj*<73LkOd^T=~uZ66;O+)0}~97C7i@9W2|kY!bm zxM=t!J+I$Z0zbj1B~3CU(JzQz4=dd6>YHV@1h`J%;ZlG7+t=**-neE!8|vA02_>HC zl!SjK2Hm^3vh&$HBTDqw2E5{K4WLR3r6TZqI)Z5LZ6EH08tOa0n#)sf)cKL&N1COh zvkql|slxeuK@-R^4f8ERoLy>D=T(GN=@DDahAjiWt5<|@V5)E$kz{mxePz+hBhW}d z9BEC)lR;fhUAZE60u4k6G`{am3?Fm5hcFWABc6qw3iOu+ocfUNMIY_U)n~ywn(Of( zJaPh5yiT$YEumfaI0f`2k-q@Z#lp!lRWVwtBr2oj@b3Of2!+yC1Mev!RYg?#D2=B1 zRVUbaV3GxF%q{LKv(L{StJCZ^z=Z^8*@_{>%c}aI*>d|!?W2bGAel{-%1wbF(HKkn zrBw^C4n(}1lVMqlG$N&TZE9Y^{-No|UGB0WH%1I31JTf^F7{T#sAe5ES!FJeR@_4r z$;nl%W|o_PO@0U}h9{#*qM&5va_yV z0BY7uq^%LAzQ~mMEzuH`7h$b^ z$?>ddQvr6y9Zon2U{jsNK_q8%zK@}GB>D^f(`Z*j4^qV% z?%sO;P5JZ|{o(jlUGWT7Y0yV^&Q6X`S|(zEjP^nd5rv4P}TeMMJHOdg4@mN(P>pSTj*rtb$i{9X>C%iLBInA(1r_8=BQMKE+73 z{fv=DNPOdj?G_>0INj%DzT7w_<>3y=2!c2|p>DHy1P;1I9rht%jlis=ZgaNiQo|d& zn&XTVTww^s^8WhjemtWWGszb51;WW1_SJ0H)&_nq6RqQ>1va0%=L~*y(V?}2NpdOa z3RWWIL($1BLDg2%wJKK~MA%E93#!Qtte>iD{U+h36#L;5(?jQI3 zrlDD#vmDXu0?Aaf%I$Kwd4AF6r5W@_Ng<3R0d8HnJ3^l#wvm_GM!=J4{|YKw2Rfn% zC#jWF6CSmpS*WBZ768G*@&dSW`RZKjN(&Xci7p{+6D{X4r6Qikyt(6(i&t;XmD>Tr z%5>B1Zl9uS``ds0D=sxUGXyzAeJQtfjQ|{zkipc5hZgIEf2%pq@ALNfV>OIEii(1% z744IH|K0n1zI1={m8RZoD%*@z-&9n37N%^WQ}5osW*@t`5&*TheASV?Tu-=4HM zPRrRxBc=rNgF7L@iM^?tj$zzktrAg|=_IrtnH(htHA9ARms^D^XybAxsEN-A^; zIE-OTkTp%RMa<GdO!aB!`0OlGd{FjgGIitDROlJY_FU!Hk_hs+u^gt2BC|UsB($&{NwtV#4fAyEA#}%4A+%seM!nAGPDtq}r42xP( z5N%cjHE%Xd0`Tey8M8SKEtB3Wp^9;0c&icq@WS*cKGA`puFh%olij=S{BcUHMwYLw z)o`xoV^9hajJ6GUDuOhEYceJLL9tS3w6p8o;PcfNwH3yPqsV!v@Md{Z)WHt8e(VQ` zF4cqQrs>rj;XPrRst-LXX17|c+YKJ9^Mafv@oWZ*7)8}4Js<7!Dhz6e+0n4S76$>z zi>OyOJj~qk4fUthtK;**+U=GV|2Xc2_3i0NFXOmFnupN*VAB^LAh=5(Or$5dD$<)@ z62k;xIqYNXP++-=Tb=6kQBh_D8fcS0hGftDA_oWDXV+@g==}RJs#A2(!OCpvk(>#Q zn2wan(sk-w&J=5QH1^UA%tKI}QdyCuHc9=rfAimYeRoqM-|DA~$W$ z@Ty8#J&XA%>cUVWC=?U|Rma7}DN~t613|^iAS%r(w*)RO4w29M78F1sJ5QkdD zxbnr2B-DsG7I*iz>OF;xI$qVm4Lut7h$lj-vR*t+=hqM8%kytfTNihCHx-G1CoOgB zPe1(7zrP38>mlf|EKHh4XORQ;^_y>vhINFkC+nlu5fb_Hr;k7X^s_PIm=4B_BTl_c zR^pwIEYc9y59g8QNCWH%YZ<1|I5GroincAy)}3wW?rU6lG)ZK#S_QZjyt(g@8{1y{TP>B z;xP1`dQpUw5fNh#6fdJaH?>MIOw8sxIlkzey+&Tkkzx&tY`LfB^UGH!=V$Rr=d1hc z0o4QfXkyE80~K@SW4{D^blG4KjkH6~4qE}$0F34uOdX2p$n<;nB*Y+@$MHxQP&QZ3 zMY5BaE|gYA1IhYo2R}acXt)_$RZ^ae+syLxOri%Bk!3l`ZWZ*3oY~WoNiiRw_oK&A z<%R*!C!}FkVeZPsN{D+R*}8aqd_fL88gy4PFhA+?Rsa41NnLP2ONomlJs}!8t;*^x3|Zgo=4uO71!;Mj({ipxJIo!7_PPDBjtt#^~|x4}bWRbqOD)W`V93N+O)L zQtMw!-z5dSznjtOVsXlOsG9keiii?bpwWU@j&R>f9?zwNYTDbQGJ?5YT$~xy%{M{& zB#4e`L3t2SAX_S4$)owR)6P-7LD#EkCR!)`QmXL>`423Cz|_MP1@5Z0RH;wg1G)Wd z%;@w^o}QEhuDgm1aoJ-v3)%qUkrr`0;kel_kgj_ZvU6rkmw;#I&Q@X>l>~uDznaK4 z>yK1@wiR>Ut5+9PUmt$DB7fDr_wMdlPI9Ufq^%lwM@LI~>l)9JATpl$aUQ?J_4ugr z`Rb>C`uzv_rc9qIOiZ-mHl|z{?aa=QpxtXUT1}-^Azkd8cOVD1w>KeO5FQ~xrrLA{ zi|kAa0>?TpCaAfB4}aVHcZNk%1Dhs#_m^ z5NY>pdJiX<&a}Ja^#|SVovM#wKq8+-=VW-CK1w+}YB$?QHNrxlj{krR&n`XgP$`~J zCcvCg)=C?4e}%|Nr&TFtXS0D*)S3gH#mPrjIvJR>!HkXj0w)d!Q2ogZrt3Y>uF-Zy zuqD+#>M~4}DqF{j>`g&c3Gh8582Fx(FJcHBTaDV$>+?6Ig0(jtKZ;9o5E=^94P|fK zcOrkVwbSm+fQJLFWNC@+S9CIA+xgUQm1+`)nN-vp-?x)fDa|08FT{9OKOKkU*O0Ly z7cifz9-SN$x?A0~nz7151-5x%5ix-o3L?Qw#JN(=xFKs)`Q!v?*6ZW~x*O3(WJ2b2 z7=@EK$eo;>noDQ28qSo&IRfoAJToRk>=?kgxNvyQqk32@0Z3uV2`!DOAu=7w>@-X3 z6;d~XksK4L%j3r3A#tDzg9>U@$ZUNxu!ICh2dX5|`Ff0#_4d``^}6{=IhCvpBI+hw z4H1!3hVAtqZ!}OKgj%IKrvrQ(zr8#^He{G0r)Wv>6D>fd{6T=wqxLzibM%TFF<=V4 zP9UF=zRe@5cy~9_!50b*))SwcSN?G(J+(`NX0xDcjj%W66PCJ+cI2Wlq;HabQn1!<@Ck0BIKPhY=& z5fvxQd%o^{`lx>1Yt&n52!&kEw6A|;ybmK-f>DtZe|DztmYg@~i;JF&hi+PTtlEyE zw^B!AW+JJt)&lUFGQZtr99xdpO0kv^MW-~JO(2L;wh03q@Zk=~=kM=6{r%tnhwHn0 zK@_~7HuGe%hBLO$8^Yox#%rJsI4HR!&K@- zeD$&0zyI+0N5Pp7V=eX&c!Ir<#6Yn^1I=S;)TWOE-!oV};%XP`799l_@}r;>Mq4O8 z8?d%DLy+ll5Y0y5_++Go!+Am`2ZIIc<7xYd>WgF1m_8MOTSB}u`AZ((>MGj_7e#)k zRJA6L&K-vXQ;)Lqq3d|Huw@Oy$jJzEgwCok7ikLy?>+QVl;Q)h~>(MBykz zH&13YOG_}C4(FNDAUe5NQ^{p1De2wNmBYsei!PU|s#8KNx*|{=8n3WOkV{RO(0}ZWa+>+Nd4V^{T=_V>=uSYs1wOyP#=36qGxW%2)IdPF=lL5 zTH`RuI9Wq?-M4ZuY=hj!#w~mgeJZFs$sY;DhQYNb+u-Cos^+vLGNOrN( zs9u|Iv0bPUv>SO?K=9Qf!KAVrt``^zO$zC8=KQjQXRZq^lFBT`fqOj$GvIr9ZGg){ zsolPwbmR-o^cr)MiA>S0`87nm6f-3;LA4eu?P2%k%A`gi75jWuhiFDe+E@%0Gly2A zd-HPyo-fx#^ru$6Oq9j2Sg&%MB^+c}8b38KFBK90fFZGz6>FTOw}j2L2ITeVpAC!2 zR1CgKREkZ;ZH78y8?^%Aos%k`Dj}=bZk?fFNC2_mJ;=cx#`gxxp%Bb;%+(x8HoWH6 zI~jC)>bI;BiCQO_Js|b|`uG3$|KmQ1aZ*S+7ngjYEYsw?`}GU0r0*Gx7@Q~co|;=x zJ(BriVZ>N&!{1@rihOsBQ8&>1guMyYKFY2aatixmR#*CLV4kSS8sXMPa05us)-pM83U`vo zwBP8y@Yca5_!)rGsy^wR`zF3d!ew02fE|Y{P-FDiM^7}XN5y&<9msM z>HQMYXgY&NWohfHB^@T*lw)_PzuYcOGS&%s=Cj_z4=_~yBpiD+GL@CsA^BuH*>DW& z6%TFc>J`MxTHcKJ0FT~M%W~Dt>&S~R~7nmgrFK8IG#Nlo;>c|^3p-3J3r*#!=*eUx{PJL=9RS?dm}`0gs|_C=?mvoq2w;9y7bJ5EjO{ zMI@@~FZ6Wo)8{EVQyCIrAKdsmaJQ%z^S6_hM99OnttmNlNazF8N(49E-dsJ&U3XQl zo$m~dn83N-l=WBtjj&NF>v#O~qDB#B7zRZTT$p;y65vBZsp{VJaETmIa8`bhX)(P4 z!V8h6<=Iasaz;D*R4By#B$2&K-jH-~hNKiKiO}Xd%AZ3Il*x0xZy}7kZNY0M%m@S5 zE=BaOW~L&Vj3VoV!B73a&)`))pnYagbo8=g@ITHM5w@w~AX!pCnc!K1R6Ex-oyzqn zEV3zO?$Jq0i2>xZO)t()JQCUzETXi?*RMJ!r?v3)0)r`;ay8-bAbb}i=eWXvb{Vcp z+|z6wyO#z%Dryx+?e^-LRY9^Do!afse?o?Yna9L>iYvuqF2z+4-q1(LpF#u(wV5x5 zlS$9@?LwBygUhPdYuJlMy$GCASaO=vm6qxiGl~K6lm~C#zR@#~?x%blJ68PeLAlkw z;B&tD`GbPW`+{g)@b%nNoJEHLtD18kBYnH)6LbDggcJS z;jC-c2j!6LLRSyvE|y&^BCx8ID_JoELrEU@WM#T63Um@a#35TyPJsl2F(76->59aJ za2|JFQGaFW&w7ve_z1 zz#H}MurJJ9L;(umfRy%%t)n)kVJ^WbLBnwC>6m&|CaX;+in*q zVlXH@NmBx^v-!!I?3=}@O3aJ~1cj1P0rDo}2~h?5+YxK@tRRm#DI!;0m#M>GinT2# zH($EoAAA=o&<*DBh^*+H&@nF51Tt2`>9tEf(^eBxVU;~|vJy%vjx)57%fmC!2?EAd zQWJ>4e&y9S@2L4k{Tqp{e+cCjH7&0a_%JXhg3O39ZEy-;tq8c*c)u3 zOEYbKZ*PQju~Ys`G_jz@&^MkH?Ysg@5^jTp<{TCRtEw;j$t?1Y>fii zH#Ey~N|>iLQn1>VQhLOn=t0dx9C^UdQ>~|7*{k~e(?{4K&AtF1S`rMh6QNSBYw?hz z%0kdiwO-}*qP2I<@krl1T#v^RSJUE!&N>$m%isV0$LW|e?Qmi==X&0%3&w1Ln1tsr zSt+_yEz*um>-*KOe)p?iy)$4_-cCB3?LL5~$(+DRh5;wdxU`Yr02=4#Ev$rm;y~%)t&^GJUT!DTeaJfjc**pu*XMurSKqhWHDy{;FZSl7bJ-H;RBK^IMb4=r zmlGB3_Sqvdep^}IzrXuW|LGs@``0YK(C_iRHdc^n(if?t6~oR>!zQ^mo7Hm`+a7#x zvoY+;6{pFOcMUX&5$^?5m{iUcVS4Ad4OerxUVXZP4mFPJ0@>P-sGq(%D&&%cOj`xJNhsUY zcx`x2K}{<6K&lkq4B6>JuXp=p02_8pI3VoGyY5hB%#6cqLSL`k^0Ts3vx}PWBI8X?WXZ* zyvP6}6kW{t$fyJ{Em_yPXQu08u-1D->k9@j7-2qq7Q*s8f4J=`ERZ=dI`Vm2XGTiv zKUV^Bq1emywA!p`8%#~2VwNF18OuA#S$tYBZdY+;uP)C4bt+rp;N<@E(NV*uA{1nK zhn0a3>51u<+}p<|bzAPkr=Q8Z8g;8voy$7H5zjr45j?v8wB(J3;1%lS*74DI-@QNF zH#hg*TyPyhz=oLL!e~(Wft0oI@5+w|jec{Z#~@3I%iyKrzDz z`@#T6iFWQwgBV`kfHVLzH9pVET(yZ6b}kPW2+PS=MD^r#;RlQ^+i&g=+P{Zf?Y&| zpti&fG#bXy(Be3g!|Or_dAOe=WJ=SVnwVYE);dWZ!QB#%a9BVGj=Ps|WX+@*Z*GDL zM>rBMkdK697qB0PF4QX#bAZr24n~c;#1w01qAo)IXtFjN zXGS`|r%W3>bXmjk?1Q1mBo)4H_N9`vp7#)e;2PowxR8qrIIIg3qLT#t8rm)KQGq03 zL~kiX$q)*qG#M77EVB-;t|-}?8)is^}-%8-if(|FcXP}tAIfoO4o z471d~{&mMJAaBc7dl=rdI>*2L>wjg=q7)z+T%X5>QtmBRV+Rp9?(5ALVwh^P3UY?X zDB^u~asj-8d$1khB?Tj==KO^_TNDP7ClwCY3$?{^d4Kx_PRda23bs03%zgyp+vKtT zIP6VjyRD3fk+Ly};Rd@zyLEJSapvjI7p^U^OnA(^64;d1tAIHPHN9aP&kLKHSyG>l z(pd`sm~)PCjoOELjK}`i?jngztRJ~_{Ow|%`@QU*PzkH(q&8XiiJeCIkkW-j{OsaQ zrB?d#(~q8(bptP2t`qK%KmYzW7jHV%@_zsB?*I7T{=?U=-PeRz_4;x=A3d0?j^ncB zltVFE1A|^IRsm|G>8&h(ePzB(fFe6^D{w^ed|ehyrJ-3tP198aIq7pOP2o6FSe*3X zIj`j76-;I{QG7KEd)fzpotPlozq|VKN#rryg!Cme&IkFty_7;XwejS^MDcVy=oZ*%I`S$b!tw zc~KlDTg~>X*KMFE3dpWw)djP}YnZB5cGQ28cv)fOMCq=*++H6aNU-SQLW$|^LfFYB zPU(iV(OM~_1Ig8cuLTPrhXo&RsPJCoA%0i$0t0KMiIpM~K2JX~pYyF$(dawoPg79s z1m#bZ0Md!56Fe8E3pz4ICDJwI!@uH72nKq`DTWR!l7$Iw3_QqOIRI{i*7-*phxp7t znUp}msIkl#kP(6))rzN@1{qNQM$mnkWlrvf6VZ5-7~*Vbfgiko&^f>4-Ink=91XY> z_V$finn;{xc_r(K5iI}9|M*`yKVS}ycIj|s$)lOw!|hkO=t2!#)O3}@Aq?Uxu9~)| zUOI6|W0h@vGb?OAu@nXcIKly^HlgJLBbo9ix(P5P=9(I-O+m3yLnn#)aWC?0RR9oq zD3#9(T%13ydOb2fl0~axTP;;QrYR5g@^SCr3A9tMw>%$nS#;DOkGqGi@2s73Erdc4 zmHFhQCoO-Mj1G+%oQ^nL=wa$*7ujnxPAtTQV`sL`=jzx$Kp(osihdDcd-(~KuqHO# zPRuRoe7#w8#9n>=Sv^;4HqXw^9QE|lrbYF8S998K6R@O&iub9Oj_NMhm|FuoS%~N<43Y^SJKej8`xQ3SgsNy6fE9`YHI`s6ybqZd=fcoj zTz-AjvRCT zNH|RvyVG+*h)1vA7w3f@)%R)_(ZXWRoZ9eRDotV!iVFr`TyaU_bTa00#_&lST3L}R z)>};pHN8P!7kzqCH_^nmrQCHa*vRIKV0|Mo-rjmnRr4_6Ld<~>sf+1xSE?FVUk^3* zHdp*dw1jN~_5nd|(Cj`vnQ9mgWxs||y@YdUh+lV>R4f6RLd#qe3 z;iyK=<#y}zWvn%|?K4e+oIiHiOlHrpcze~~6|k^*Qh!1LI6wAV?Y81y-jDYLykY|g zdXl+kF*m{l$!Ze#@o6>n-~HXcg@K?KX`?VUtRQ#S9}Z&{>7iE+(}BS`a34no@UAk2 zbgOCH_Ll6;laCL(oN;p_ZiwPtCVW#?Gx@}*k5}E*s%_9UP2+U2d;RTiv+3&BPamh_ zf%DTT;`i}@kaAS%UODBlxHQ6XH=Wd7*Agj{_QH;l#H@eSSS*+3bw`ZVki2OvoetIq zDVGo>#uKd0HB(TOmu!M;9f@DeX{I;5;r)8MIMN`^Fm<=s5nEdL9A=Mi4RGvK6qL7I zj{Fa}DA@+(0Bi$@Gbj4GyGJNPrIpZ&Y5(J=KmGB?KYaf1aC3FX_(Kza{`z}>sxh&% z6^@9{WdY89O6~4P-NjaKDfngRk0TdWj!}x)Y3Q;-uPdBJqh~WP0x?sraxU%&Ze*_+ z5E)KS6}u`VM#0RJ&00%guL)VXKGle$4Q|IrcXk?qk~3?V9AdtXkyqpX%+C-*1krjn z&Q0B>N1H#*B=i^s!DvhDq=X~t9=i2#y%IQ8m4p7U-v{pRRcmg!`DiHHKXZE4GUtYV zb^SD@%oI;_uaSlvf(wUnVqk!2kps{9`gqbjK07~FJF@)2XuJbE5S=?4E=d8fls|v^ zJerTJM!)=i)(oGCJd4xI*SV5IbA5Y@k*AhCAWL>4l5KHg=)&PYe#(nCTYZ~u2@{7( zx>x6;+ncX8B}l@lsVZh3E>2w{gG!!Kr0R;3ZQ}^ZriwN;B4Q^$XSGMT;O4Bfs6@edeg26XCB35!h3p1*?MOR;F-6l$!MR*K-1W3K)Fh+XOy31q?wN(U z6cO*4j#p#mzh8-|b^!N*RQVnE=052R1YupsoTi?t)4SorM~SPu`D0~c zit}>M`Ce`s+)NBF)_y5q0gwceN5L73@o<0dQK;AOw&hx;+$ackc8er=WVXrrNo_xq zG?OC5F|eKA-zo|cK6EsbMwf+w#Z|e*czPQ1`CLe^x(k)#zEMyUl*^fxjH&QZso5LK zEa)QiwO9@A13ru2_L4MKYa`Nb{%IzHi@-W@X%$c2`7EMySE)64WL$17F8U)kvaYe zFD^_}DyJ3@bdCVXYA!a((G0yFsvU#YBR!Xz&7_+|0Je1VIAK;@t`>PF4f=VKhYdfVgonC+x@!R;O!E`u1 zbIx9CueCPqpeDGGT9QwO=MbQ1{;BLyk02&H$V$ZxGJW8V%^db6b0A7H6Pst=AQzD_ znY1*xcotVuuhW7u+cOOltg+Qh@?dvm!!@@nEXed>M5gK#n_4L`8p$-Mw?7`d-%Hj2 zSgb$&YB89YLG#=0*=cg+nBfWaaF zCW;4Ulid&@pDvUjHkq)He~t_b`ecL0%u>#N1+h( z1=zfnmgVX*V=OJOHxo72|h6Dv_;dz}c@Dn^T`rlf=V zhVgM)o!EDMGtF1mr(U>au}|aFCnb3UH+n#TeAYZy#JUC+Ke1{m720JTl`;X)Lqd&; zjr_KlU6In&(h51tBJf0;xk%Mc9-gsA1fKos^s+Q+8Qii6TN%M)BaM^9@Ib~91ggy! z=J%PG?G{_ia+qjvH$#-Bdn2mL}dxp-EgX$VxU_i8fmOI)7Mb8P{wg3g!r+y1<_QV8ir`+b0&P`9&&@jcD=Wg%f+@kh8xyvNG5A2h zT=~oA55+iG7BTlHC*R%aFigY?`(GGIe+avZ@qL;Hr>Ew3eMBYgh6iT;u?TO}HZWRz z_$(lRZ_&?ma!`Ws06I(&XZ{Rxz>lB5f1RB=-8~SWrTi!lN{+s|s*S*Pu3=iz7c`*J zF7YiX0HAWehq|0{36y2sl6b~7A19%+d}(>_pncRn0Q@PlxgvIV-l?gy?0i2Cki6e> zSd&MZimdTUtwtqWT~F8+s5g4|IpTq3$tkYR^&6WvVA{mh)upq?LnuTQ9%1vG7-YXV z2p+A=ntt8QRvML7t2%wkF8-W(NHp(Ykn*&9+4B1W!Rv0h>h?=E+OpWZ$E|wnUE{5t zCq>(;lWmhHk+7bgUz}fGE0f%FYCjz_bG9|;T@KZT_`Wp(m;^mLq=P2l3Lsrytqbnl zO1yreMvGUAbF}6kKgd`swt%%<;rtRhOp_@9QjtrCx$m~&Rr2b zGW$}S2k3)%bd2G1M=+)!H2Tcsnglb@J03zpB(BpZjueUr{XB{~4psEL^BiQ-bg4I+ ztr+R4p=eIHaiI~XY}0v_7Wan>xV6ClyWTth|JGB~Dl2vl(_j46tM|vXNf#OAYgZ{keK|_YR{l7Pt=e;|+4c}dy bB6i?^tMTovQB6cm00000NkvXXu0mjf)U=3L literal 0 HcmV?d00001 diff --git a/images/decision_trees/README b/images/decision_trees/README new file mode 100644 index 0000000..1c4283b --- /dev/null +++ b/images/decision_trees/README @@ -0,0 +1 @@ +Images generated by the notebooks diff --git a/images/deep/README b/images/deep/README new file mode 100644 index 0000000..1c4283b --- /dev/null +++ b/images/deep/README @@ -0,0 +1 @@ +Images generated by the notebooks diff --git a/images/dim_reduction/README b/images/dim_reduction/README new file mode 100644 index 0000000..1c4283b --- /dev/null +++ b/images/dim_reduction/README @@ -0,0 +1 @@ +Images generated by the notebooks diff --git a/images/distributed/README b/images/distributed/README new file mode 100644 index 0000000..1c4283b --- /dev/null +++ b/images/distributed/README @@ -0,0 +1 @@ +Images generated by the notebooks diff --git a/images/ensembles/README b/images/ensembles/README new file mode 100644 index 0000000..1c4283b --- /dev/null +++ b/images/ensembles/README @@ -0,0 +1 @@ +Images generated by the notebooks diff --git a/images/rl/README b/images/rl/README new file mode 100644 index 0000000..1c4283b --- /dev/null +++ b/images/rl/README @@ -0,0 +1 @@ +Images generated by the notebooks diff --git a/images/rnn/README b/images/rnn/README new file mode 100644 index 0000000..1c4283b --- /dev/null +++ b/images/rnn/README @@ -0,0 +1 @@ +Images generated by the notebooks diff --git a/images/svm/README b/images/svm/README new file mode 100644 index 0000000..1c4283b --- /dev/null +++ b/images/svm/README @@ -0,0 +1 @@ +Images generated by the notebooks diff --git a/images/tensorflow/README b/images/tensorflow/README new file mode 100644 index 0000000..1c4283b --- /dev/null +++ b/images/tensorflow/README @@ -0,0 +1 @@ +Images generated by the notebooks diff --git a/index.ipynb b/index.ipynb index c9878a6..a1c72fe 100644 --- a/index.ipynb +++ b/index.ipynb @@ -16,19 +16,29 @@ "\n", "### To run the examples\n", "* **Jupyter** – These notebooks are based on Jupyter. If you just plan to read without running any code, there's really nothing more to know, just keep reading! But if you want to experiment with the code examples you need to:\n", - " * open these notebooks in Jupyter. If you clicked on the \"launch binder\" button in github or followed the Installation instructions, then you are good to go. If not you will need to go back to the project [home page](https://github.com/ageron/ml-notebooks/) and click on \"launch binder\" or follow the installation instructions.\n", + " * open these notebooks in Jupyter. If you clicked on the \"launch binder\" button in github or followed the Installation instructions, then you are good to go. If not you will need to go back to the project [home page](https://github.com/ageron/handson-ml/) and click on \"launch binder\" or follow the installation instructions.\n", " * learn how to use Jupyter. Start the User Interface Tour from the Help menu.\n", "\n", "### To activate extensions\n", - "* If this is an interactive session (see above), you may want to turn on a few Jupyter extensions by going to the [Extension Configuration](../nbextensions/) page. In particular the \"*table of contents (2)*\" extension is quite useful.\n", + "* If this is an interactive session (see above), you may want to turn on a few Jupyter extensions by going to the [Extension Configuration](../nbextensions/) page. In particular the \"*Table of Contents (2)*\" extension is quite useful.\n", "\n", - "## Chapters\n", - "1. [Fundamentals](fundamentals.ipynb)\n", - "2. [End-to-end project](end_to_end_project.ipynb)\n", - "3. [Classification](classification.ipynb)\n", - "4. [Training Linear Models](training_linear_models.ipynb)\n", - "\n", - "More explanations and chapters coming soon.\n", + "## Notebooks\n", + "1. [The Machine Learning landscape](01_the_machine_learning_landscape.ipynb)\n", + "2. [End-to-end Machine Learning project](02_end_to_end_machine_learning_project.ipynb)\n", + "3. [Classification](03_classification.ipynb)\n", + "4. [Training Linear Models](04_training_linear_models.ipynb)\n", + "5. [Support Vector Machines](05_support_vector_machines.ipynb)\n", + "6. [Decision Trees](06_decision_trees.ipynb)\n", + "7. [Ensemble Learning and Random Forests](07_ensemble_learning_and_random_forests.ipynb)\n", + "8. [Dimensionality Reduction](08_dimensionality_reduction.ipynb)\n", + "9. [Up and running with TensorFlow](09_up_and_running_with_tensorflow.ipynb)\n", + "10. [Introduction to Artificial Neural Networks](10_introduction_to_artificial_neural_networks.ipynb)\n", + "11. [Deep Learning](11_deep_learning.ipynb)\n", + "12. [Distributed TensorFlow](12_distributed_tensorflow.ipynb)\n", + "13. [Convolutional Neural Networks](13_convolutional_neural_networks.ipynb)\n", + "14. [Recurrent Neural Networks](14_recurrent_neural_networks.ipynb)\n", + "15. Autoencoders (coming soon)\n", + "16. Reinforcement Learning (coming soon)\n", "\n", "## Scientific Python tutorials\n", "* [NumPy](tools_numpy.ipynb)\n", @@ -39,6 +49,15 @@ "* [Linear Algebra](math_linear_algebra.ipynb)\n", "* Calculus (coming soon)" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] } ], "metadata": { @@ -59,10 +78,14 @@ "pygments_lexer": "ipython2", "version": "2.7.11" }, + "nav_menu": {}, "toc": { + "navigate_menu": true, + "number_sections": true, + "sideBar": true, + "threshold": 6, "toc_cell": false, - "toc_number_sections": true, - "toc_threshold": 6, + "toc_section_display": "block", "toc_window_display": false } }, diff --git a/nets/inception_v3.py b/nets/inception_v3.py index d5a1fe3..2897a4b 100644 --- a/nets/inception_v3.py +++ b/nets/inception_v3.py @@ -94,7 +94,9 @@ def inception_v3_base(inputs, raise ValueError('depth_multiplier is not greater than zero.') depth = lambda d: max(int(d * depth_multiplier), min_depth) - with tf.variable_scope(scope, 'InceptionV3', [inputs]): + #Backported to 0.10.0 + #with tf.variable_scope(scope, 'InceptionV3', [inputs]): + with tf.variable_scope(scope or 'InceptionV3'): with slim.arg_scope([slim.conv2d, slim.max_pool2d, slim.avg_pool2d], stride=1, padding='VALID'): # 299 x 299 x 3 @@ -470,8 +472,10 @@ def inception_v3(inputs, raise ValueError('depth_multiplier is not greater than zero.') depth = lambda d: max(int(d * depth_multiplier), min_depth) - with tf.variable_scope(scope, 'InceptionV3', [inputs, num_classes], - reuse=reuse) as scope: + #Backported to 0.10.0 + #with tf.variable_scope(scope, 'InceptionV3', [inputs, num_classes], + # reuse=reuse) as scope: + with tf.variable_scope(scope or 'InceptionV3', reuse=reuse) as scope: with slim.arg_scope([slim.batch_norm, slim.dropout], is_training=is_training): net, end_points = inception_v3_base(