From 3dd82863d1099d1d18ca9aee81d82e8f1e2c3702 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Geron?= Date: Thu, 1 Jul 2021 13:19:38 +1200 Subject: [PATCH] Add explanations for the models using the custom layers --- ..._models_and_training_with_tensorflow.ipynb | 561 +++++++++--------- 1 file changed, 290 insertions(+), 271 deletions(-) diff --git a/12_custom_models_and_training_with_tensorflow.ipynb b/12_custom_models_and_training_with_tensorflow.ipynb index c3c34af..bdd8aa5 100644 --- a/12_custom_models_and_training_with_tensorflow.ipynb +++ b/12_custom_models_and_training_with_tensorflow.ipynb @@ -1855,7 +1855,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 144, "metadata": {}, "outputs": [], "source": [ @@ -1871,19 +1871,35 @@ ] }, { - "cell_type": "code", - "execution_count": 3, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "keras.backend.clear_session()\n", - "np.random.seed(42)\n", - "tf.random.set_seed(42)" + "Our custom layer can be called using the functional API like this:" ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 145, + "metadata": {}, + "outputs": [], + "source": [ + "inputs1 = keras.layers.Input(shape=[2])\n", + "inputs2 = keras.layers.Input(shape=[2])\n", + "outputs1, outputs2 = MyMultiLayer()((inputs1, inputs2))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note that the `call()` method receives symbolic inputs, whose shape is only partially specified (at this stage, we don't know the batch size, which is why the first dimension is `None`):\n", + "\n", + "We can also pass actual data to the custom layer. To test this, let's split each dataset's inputs into two parts, with four features each:" + ] + }, + { + "cell_type": "code", + "execution_count": 146, "metadata": {}, "outputs": [], "source": [ @@ -1901,31 +1917,38 @@ ] }, { - "cell_type": "code", - "execution_count": 5, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "inputs1 = keras.layers.Input(shape=[2])\n", - "inputs2 = keras.layers.Input(shape=[2])\n", - "outputs1, outputs2 = MyMultiLayer()((inputs1, inputs2))" + "Now notice that the shapes are fully specified:" ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 147, "metadata": {}, "outputs": [], "source": [ "outputs1, outputs2 = MyMultiLayer()((X_train_scaled_A, X_train_scaled_B))" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's build a more complete model using the functional API (this is just a toy example, don't expect awesome performance):" + ] + }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 148, "metadata": {}, "outputs": [], "source": [ + "keras.backend.clear_session()\n", + "np.random.seed(42)\n", + "tf.random.set_seed(42)\n", + "\n", "input_A = keras.layers.Input(shape=X_train_scaled_A.shape[-1])\n", "input_B = keras.layers.Input(shape=X_train_scaled_B.shape[-1])\n", "hidden_A, hidden_B = MyMultiLayer()((input_A, input_B))\n", @@ -1933,12 +1956,12 @@ "hidden_B = keras.layers.Dense(30, activation='selu')(hidden_B)\n", "concat = keras.layers.Concatenate()((hidden_A, hidden_B))\n", "output = keras.layers.Dense(1)(concat)\n", - "model = keras.Model(inputs=[input_A, input_B], outputs=[output])" + "model = keras.models.Model(inputs=[input_A, input_B], outputs=[output])" ] }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 149, "metadata": {}, "outputs": [], "source": [ @@ -1947,23 +1970,24 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 150, "metadata": {}, "outputs": [], "source": [ - "model.fit((X_train_scaled_A, X_train_scaled_B), y_train, epochs=2, validation_data=((X_valid_scaled_A, X_valid_scaled_B), y_valid))" + "model.fit((X_train_scaled_A, X_train_scaled_B), y_train, epochs=2,\n", + " validation_data=((X_valid_scaled_A, X_valid_scaled_B), y_valid))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Let's create a layer with a different behavior during training and testing:" + "Now let's create a layer with a different behavior during training and testing:" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 151, "metadata": {}, "outputs": [], "source": [ @@ -1984,37 +2008,32 @@ ] }, { - "cell_type": "code", - "execution_count": 3, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "class GaussianNoiseModel(keras.Model):\n", - " def __init__(self, units, activation, **kwargs):\n", - " super(GaussianNoiseModel, self).__init__(**kwargs)\n", - " self.noise = AddGaussianNoise(stddev=1.0)\n", - " self.hidden = keras.layers.Dense(units, activation=activation)\n", - " self.out = keras.layers.Dense(1)\n", - " \n", - " def call(self, inputs):\n", - " noise = self.noise(inputs)\n", - " hidden = self.hidden(noise)\n", - " out = self.out(hidden)\n", - " return out" + "Here's a simple model that uses this custom layer:" ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 152, "metadata": {}, "outputs": [], "source": [ - "model = GaussianNoiseModel(units=30, activation='selu')" + "keras.backend.clear_session()\n", + "np.random.seed(42)\n", + "tf.random.set_seed(42)\n", + "\n", + "model = keras.models.Sequential([\n", + " AddGaussianNoise(stddev=1.0),\n", + " keras.layers.Dense(30, activation=\"selu\"),\n", + " keras.layers.Dense(1)\n", + "])" ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 153, "metadata": {}, "outputs": [], "source": [ @@ -2033,7 +2052,7 @@ }, { "cell_type": "code", - "execution_count": 149, + "execution_count": 154, "metadata": {}, "outputs": [], "source": [ @@ -2042,7 +2061,7 @@ }, { "cell_type": "code", - "execution_count": 150, + "execution_count": 155, "metadata": {}, "outputs": [], "source": [ @@ -2062,7 +2081,7 @@ }, { "cell_type": "code", - "execution_count": 151, + "execution_count": 156, "metadata": {}, "outputs": [], "source": [ @@ -2083,64 +2102,6 @@ " return self.out(Z)" ] }, - { - "cell_type": "code", - "execution_count": 152, - "metadata": {}, - "outputs": [], - "source": [ - "keras.backend.clear_session()\n", - "np.random.seed(42)\n", - "tf.random.set_seed(42)" - ] - }, - { - "cell_type": "code", - "execution_count": 153, - "metadata": {}, - "outputs": [], - "source": [ - "model = ResidualRegressor(1)\n", - "model.compile(loss=\"mse\", optimizer=\"nadam\")\n", - "history = model.fit(X_train_scaled, y_train, epochs=5)\n", - "score = model.evaluate(X_test_scaled, y_test)\n", - "y_pred = model.predict(X_new_scaled)" - ] - }, - { - "cell_type": "code", - "execution_count": 154, - "metadata": {}, - "outputs": [], - "source": [ - "model.save(\"my_custom_model.ckpt\")" - ] - }, - { - "cell_type": "code", - "execution_count": 155, - "metadata": {}, - "outputs": [], - "source": [ - "model = keras.models.load_model(\"my_custom_model.ckpt\")" - ] - }, - { - "cell_type": "code", - "execution_count": 156, - "metadata": {}, - "outputs": [], - "source": [ - "history = model.fit(X_train_scaled, y_train, epochs=5)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We could have defined the model using the sequential API instead:" - ] - }, { "cell_type": "code", "execution_count": 157, @@ -2157,6 +2118,64 @@ "execution_count": 158, "metadata": {}, "outputs": [], + "source": [ + "model = ResidualRegressor(1)\n", + "model.compile(loss=\"mse\", optimizer=\"nadam\")\n", + "history = model.fit(X_train_scaled, y_train, epochs=5)\n", + "score = model.evaluate(X_test_scaled, y_test)\n", + "y_pred = model.predict(X_new_scaled)" + ] + }, + { + "cell_type": "code", + "execution_count": 159, + "metadata": {}, + "outputs": [], + "source": [ + "model.save(\"my_custom_model.ckpt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 160, + "metadata": {}, + "outputs": [], + "source": [ + "model = keras.models.load_model(\"my_custom_model.ckpt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 161, + "metadata": {}, + "outputs": [], + "source": [ + "history = model.fit(X_train_scaled, y_train, epochs=5)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We could have defined the model using the sequential API instead:" + ] + }, + { + "cell_type": "code", + "execution_count": 162, + "metadata": {}, + "outputs": [], + "source": [ + "keras.backend.clear_session()\n", + "np.random.seed(42)\n", + "tf.random.set_seed(42)" + ] + }, + { + "cell_type": "code", + "execution_count": 163, + "metadata": {}, + "outputs": [], "source": [ "block1 = ResidualBlock(2, 30)\n", "model = keras.models.Sequential([\n", @@ -2169,7 +2188,7 @@ }, { "cell_type": "code", - "execution_count": 159, + "execution_count": 164, "metadata": {}, "outputs": [], "source": [ @@ -2195,7 +2214,7 @@ }, { "cell_type": "code", - "execution_count": 160, + "execution_count": 165, "metadata": {}, "outputs": [], "source": [ @@ -2230,7 +2249,7 @@ }, { "cell_type": "code", - "execution_count": 161, + "execution_count": 166, "metadata": {}, "outputs": [], "source": [ @@ -2241,7 +2260,7 @@ }, { "cell_type": "code", - "execution_count": 162, + "execution_count": 167, "metadata": {}, "outputs": [], "source": [ @@ -2260,7 +2279,7 @@ }, { "cell_type": "code", - "execution_count": 163, + "execution_count": 168, "metadata": {}, "outputs": [], "source": [ @@ -2270,7 +2289,7 @@ }, { "cell_type": "code", - "execution_count": 164, + "execution_count": 169, "metadata": {}, "outputs": [], "source": [ @@ -2281,7 +2300,7 @@ }, { "cell_type": "code", - "execution_count": 165, + "execution_count": 170, "metadata": {}, "outputs": [], "source": [ @@ -2290,7 +2309,7 @@ }, { "cell_type": "code", - "execution_count": 166, + "execution_count": 171, "metadata": {}, "outputs": [], "source": [ @@ -2301,67 +2320,6 @@ "gradients = tape.gradient(z, [w1, w2])" ] }, - { - "cell_type": "code", - "execution_count": 167, - "metadata": {}, - "outputs": [], - "source": [ - "gradients" - ] - }, - { - "cell_type": "code", - "execution_count": 168, - "metadata": {}, - "outputs": [], - "source": [ - "with tf.GradientTape() as tape:\n", - " z = f(w1, w2)\n", - "\n", - "dz_dw1 = tape.gradient(z, w1)\n", - "try:\n", - " dz_dw2 = tape.gradient(z, w2)\n", - "except RuntimeError as ex:\n", - " print(ex)" - ] - }, - { - "cell_type": "code", - "execution_count": 169, - "metadata": {}, - "outputs": [], - "source": [ - "with tf.GradientTape(persistent=True) as tape:\n", - " z = f(w1, w2)\n", - "\n", - "dz_dw1 = tape.gradient(z, w1)\n", - "dz_dw2 = tape.gradient(z, w2) # works now!\n", - "del tape" - ] - }, - { - "cell_type": "code", - "execution_count": 170, - "metadata": {}, - "outputs": [], - "source": [ - "dz_dw1, dz_dw2" - ] - }, - { - "cell_type": "code", - "execution_count": 171, - "metadata": {}, - "outputs": [], - "source": [ - "c1, c2 = tf.constant(5.), tf.constant(3.)\n", - "with tf.GradientTape() as tape:\n", - " z = f(c1, c2)\n", - "\n", - "gradients = tape.gradient(z, [c1, c2])" - ] - }, { "cell_type": "code", "execution_count": 172, @@ -2376,6 +2334,67 @@ "execution_count": 173, "metadata": {}, "outputs": [], + "source": [ + "with tf.GradientTape() as tape:\n", + " z = f(w1, w2)\n", + "\n", + "dz_dw1 = tape.gradient(z, w1)\n", + "try:\n", + " dz_dw2 = tape.gradient(z, w2)\n", + "except RuntimeError as ex:\n", + " print(ex)" + ] + }, + { + "cell_type": "code", + "execution_count": 174, + "metadata": {}, + "outputs": [], + "source": [ + "with tf.GradientTape(persistent=True) as tape:\n", + " z = f(w1, w2)\n", + "\n", + "dz_dw1 = tape.gradient(z, w1)\n", + "dz_dw2 = tape.gradient(z, w2) # works now!\n", + "del tape" + ] + }, + { + "cell_type": "code", + "execution_count": 175, + "metadata": {}, + "outputs": [], + "source": [ + "dz_dw1, dz_dw2" + ] + }, + { + "cell_type": "code", + "execution_count": 176, + "metadata": {}, + "outputs": [], + "source": [ + "c1, c2 = tf.constant(5.), tf.constant(3.)\n", + "with tf.GradientTape() as tape:\n", + " z = f(c1, c2)\n", + "\n", + "gradients = tape.gradient(z, [c1, c2])" + ] + }, + { + "cell_type": "code", + "execution_count": 177, + "metadata": {}, + "outputs": [], + "source": [ + "gradients" + ] + }, + { + "cell_type": "code", + "execution_count": 178, + "metadata": {}, + "outputs": [], "source": [ "with tf.GradientTape() as tape:\n", " tape.watch(c1)\n", @@ -2387,7 +2406,7 @@ }, { "cell_type": "code", - "execution_count": 174, + "execution_count": 179, "metadata": {}, "outputs": [], "source": [ @@ -2396,7 +2415,7 @@ }, { "cell_type": "code", - "execution_count": 175, + "execution_count": 180, "metadata": {}, "outputs": [], "source": [ @@ -2410,7 +2429,7 @@ }, { "cell_type": "code", - "execution_count": 176, + "execution_count": 181, "metadata": {}, "outputs": [], "source": [ @@ -2425,7 +2444,7 @@ }, { "cell_type": "code", - "execution_count": 177, + "execution_count": 182, "metadata": {}, "outputs": [], "source": [ @@ -2440,7 +2459,7 @@ }, { "cell_type": "code", - "execution_count": 178, + "execution_count": 183, "metadata": {}, "outputs": [], "source": [ @@ -2449,7 +2468,7 @@ }, { "cell_type": "code", - "execution_count": 179, + "execution_count": 184, "metadata": {}, "outputs": [], "source": [ @@ -2458,7 +2477,7 @@ }, { "cell_type": "code", - "execution_count": 180, + "execution_count": 185, "metadata": {}, "outputs": [], "source": [ @@ -2473,7 +2492,7 @@ }, { "cell_type": "code", - "execution_count": 181, + "execution_count": 186, "metadata": {}, "outputs": [], "source": [ @@ -2486,7 +2505,7 @@ }, { "cell_type": "code", - "execution_count": 182, + "execution_count": 187, "metadata": {}, "outputs": [], "source": [ @@ -2495,7 +2514,7 @@ }, { "cell_type": "code", - "execution_count": 183, + "execution_count": 188, "metadata": {}, "outputs": [], "source": [ @@ -2508,7 +2527,7 @@ }, { "cell_type": "code", - "execution_count": 184, + "execution_count": 189, "metadata": {}, "outputs": [], "source": [ @@ -2522,7 +2541,7 @@ }, { "cell_type": "code", - "execution_count": 185, + "execution_count": 190, "metadata": {}, "outputs": [], "source": [ @@ -2532,7 +2551,7 @@ }, { "cell_type": "code", - "execution_count": 186, + "execution_count": 191, "metadata": {}, "outputs": [], "source": [ @@ -2552,7 +2571,7 @@ }, { "cell_type": "code", - "execution_count": 187, + "execution_count": 192, "metadata": {}, "outputs": [], "source": [ @@ -2563,7 +2582,7 @@ }, { "cell_type": "code", - "execution_count": 188, + "execution_count": 193, "metadata": {}, "outputs": [], "source": [ @@ -2577,7 +2596,7 @@ }, { "cell_type": "code", - "execution_count": 189, + "execution_count": 194, "metadata": {}, "outputs": [], "source": [ @@ -2588,7 +2607,7 @@ }, { "cell_type": "code", - "execution_count": 190, + "execution_count": 195, "metadata": {}, "outputs": [], "source": [ @@ -2602,7 +2621,7 @@ }, { "cell_type": "code", - "execution_count": 191, + "execution_count": 196, "metadata": {}, "outputs": [], "source": [ @@ -2627,7 +2646,7 @@ }, { "cell_type": "code", - "execution_count": 192, + "execution_count": 197, "metadata": {}, "outputs": [], "source": [ @@ -2642,7 +2661,7 @@ }, { "cell_type": "code", - "execution_count": 193, + "execution_count": 198, "metadata": {}, "outputs": [], "source": [ @@ -2651,7 +2670,7 @@ }, { "cell_type": "code", - "execution_count": 194, + "execution_count": 199, "metadata": {}, "outputs": [], "source": [ @@ -2664,7 +2683,7 @@ }, { "cell_type": "code", - "execution_count": 195, + "execution_count": 200, "metadata": {}, "outputs": [], "source": [ @@ -2680,7 +2699,7 @@ }, { "cell_type": "code", - "execution_count": 196, + "execution_count": 201, "metadata": {}, "outputs": [], "source": [ @@ -2691,7 +2710,7 @@ }, { "cell_type": "code", - "execution_count": 197, + "execution_count": 202, "metadata": {}, "outputs": [], "source": [ @@ -2706,7 +2725,7 @@ }, { "cell_type": "code", - "execution_count": 198, + "execution_count": 203, "metadata": {}, "outputs": [], "source": [ @@ -2734,7 +2753,7 @@ }, { "cell_type": "code", - "execution_count": 199, + "execution_count": 204, "metadata": {}, "outputs": [], "source": [ @@ -2777,7 +2796,7 @@ }, { "cell_type": "code", - "execution_count": 200, + "execution_count": 205, "metadata": {}, "outputs": [], "source": [ @@ -2787,7 +2806,7 @@ }, { "cell_type": "code", - "execution_count": 201, + "execution_count": 206, "metadata": {}, "outputs": [], "source": [ @@ -2796,7 +2815,7 @@ }, { "cell_type": "code", - "execution_count": 202, + "execution_count": 207, "metadata": {}, "outputs": [], "source": [ @@ -2805,7 +2824,7 @@ }, { "cell_type": "code", - "execution_count": 203, + "execution_count": 208, "metadata": {}, "outputs": [], "source": [ @@ -2815,7 +2834,7 @@ }, { "cell_type": "code", - "execution_count": 204, + "execution_count": 209, "metadata": {}, "outputs": [], "source": [ @@ -2824,7 +2843,7 @@ }, { "cell_type": "code", - "execution_count": 205, + "execution_count": 210, "metadata": {}, "outputs": [], "source": [ @@ -2840,7 +2859,7 @@ }, { "cell_type": "code", - "execution_count": 206, + "execution_count": 211, "metadata": {}, "outputs": [], "source": [ @@ -2850,7 +2869,7 @@ }, { "cell_type": "code", - "execution_count": 207, + "execution_count": 212, "metadata": {}, "outputs": [], "source": [ @@ -2859,7 +2878,7 @@ }, { "cell_type": "code", - "execution_count": 208, + "execution_count": 213, "metadata": {}, "outputs": [], "source": [ @@ -2875,7 +2894,7 @@ }, { "cell_type": "code", - "execution_count": 209, + "execution_count": 214, "metadata": {}, "outputs": [], "source": [ @@ -2884,7 +2903,7 @@ }, { "cell_type": "code", - "execution_count": 210, + "execution_count": 215, "metadata": {}, "outputs": [], "source": [ @@ -2894,7 +2913,7 @@ }, { "cell_type": "code", - "execution_count": 211, + "execution_count": 216, "metadata": {}, "outputs": [], "source": [ @@ -2904,7 +2923,7 @@ }, { "cell_type": "code", - "execution_count": 212, + "execution_count": 217, "metadata": {}, "outputs": [], "source": [ @@ -2913,7 +2932,7 @@ }, { "cell_type": "code", - "execution_count": 213, + "execution_count": 218, "metadata": {}, "outputs": [], "source": [ @@ -2922,7 +2941,7 @@ }, { "cell_type": "code", - "execution_count": 214, + "execution_count": 219, "metadata": {}, "outputs": [], "source": [ @@ -2931,7 +2950,7 @@ }, { "cell_type": "code", - "execution_count": 215, + "execution_count": 220, "metadata": {}, "outputs": [], "source": [ @@ -2947,7 +2966,7 @@ }, { "cell_type": "code", - "execution_count": 216, + "execution_count": 221, "metadata": {}, "outputs": [], "source": [ @@ -2959,7 +2978,7 @@ }, { "cell_type": "code", - "execution_count": 217, + "execution_count": 222, "metadata": {}, "outputs": [], "source": [ @@ -2968,7 +2987,7 @@ }, { "cell_type": "code", - "execution_count": 218, + "execution_count": 223, "metadata": {}, "outputs": [], "source": [ @@ -2977,7 +2996,7 @@ }, { "cell_type": "code", - "execution_count": 219, + "execution_count": 224, "metadata": {}, "outputs": [], "source": [ @@ -2997,7 +3016,7 @@ }, { "cell_type": "code", - "execution_count": 220, + "execution_count": 225, "metadata": {}, "outputs": [], "source": [ @@ -3009,7 +3028,7 @@ }, { "cell_type": "code", - "execution_count": 221, + "execution_count": 226, "metadata": {}, "outputs": [], "source": [ @@ -3020,7 +3039,7 @@ }, { "cell_type": "code", - "execution_count": 222, + "execution_count": 227, "metadata": {}, "outputs": [], "source": [ @@ -3032,7 +3051,7 @@ }, { "cell_type": "code", - "execution_count": 223, + "execution_count": 228, "metadata": {}, "outputs": [], "source": [ @@ -3059,7 +3078,7 @@ }, { "cell_type": "code", - "execution_count": 224, + "execution_count": 229, "metadata": {}, "outputs": [], "source": [ @@ -3072,7 +3091,7 @@ }, { "cell_type": "code", - "execution_count": 225, + "execution_count": 230, "metadata": {}, "outputs": [], "source": [ @@ -3081,7 +3100,7 @@ }, { "cell_type": "code", - "execution_count": 226, + "execution_count": 231, "metadata": {}, "outputs": [], "source": [ @@ -3097,7 +3116,7 @@ }, { "cell_type": "code", - "execution_count": 227, + "execution_count": 232, "metadata": {}, "outputs": [], "source": [ @@ -3111,7 +3130,7 @@ }, { "cell_type": "code", - "execution_count": 228, + "execution_count": 233, "metadata": {}, "outputs": [], "source": [ @@ -3120,7 +3139,7 @@ }, { "cell_type": "code", - "execution_count": 229, + "execution_count": 234, "metadata": {}, "outputs": [], "source": [ @@ -3136,7 +3155,7 @@ }, { "cell_type": "code", - "execution_count": 230, + "execution_count": 235, "metadata": {}, "outputs": [], "source": [ @@ -3149,7 +3168,7 @@ }, { "cell_type": "code", - "execution_count": 231, + "execution_count": 236, "metadata": {}, "outputs": [], "source": [ @@ -3165,7 +3184,7 @@ }, { "cell_type": "code", - "execution_count": 232, + "execution_count": 237, "metadata": {}, "outputs": [], "source": [ @@ -3178,7 +3197,7 @@ }, { "cell_type": "code", - "execution_count": 233, + "execution_count": 238, "metadata": {}, "outputs": [], "source": [ @@ -3188,7 +3207,7 @@ }, { "cell_type": "code", - "execution_count": 234, + "execution_count": 239, "metadata": {}, "outputs": [], "source": [ @@ -3198,7 +3217,7 @@ }, { "cell_type": "code", - "execution_count": 235, + "execution_count": 240, "metadata": {}, "outputs": [], "source": [ @@ -3211,7 +3230,7 @@ }, { "cell_type": "code", - "execution_count": 236, + "execution_count": 241, "metadata": {}, "outputs": [], "source": [ @@ -3221,7 +3240,7 @@ }, { "cell_type": "code", - "execution_count": 237, + "execution_count": 242, "metadata": {}, "outputs": [], "source": [ @@ -3231,7 +3250,7 @@ }, { "cell_type": "code", - "execution_count": 238, + "execution_count": 243, "metadata": {}, "outputs": [], "source": [ @@ -3246,7 +3265,7 @@ }, { "cell_type": "code", - "execution_count": 239, + "execution_count": 244, "metadata": {}, "outputs": [], "source": [ @@ -3257,7 +3276,7 @@ }, { "cell_type": "code", - "execution_count": 240, + "execution_count": 245, "metadata": { "scrolled": true }, @@ -3274,7 +3293,7 @@ }, { "cell_type": "code", - "execution_count": 241, + "execution_count": 246, "metadata": {}, "outputs": [], "source": [ @@ -3288,7 +3307,7 @@ }, { "cell_type": "code", - "execution_count": 242, + "execution_count": 247, "metadata": {}, "outputs": [], "source": [ @@ -3312,7 +3331,7 @@ }, { "cell_type": "code", - "execution_count": 243, + "execution_count": 248, "metadata": {}, "outputs": [], "source": [ @@ -3324,7 +3343,7 @@ }, { "cell_type": "code", - "execution_count": 244, + "execution_count": 249, "metadata": {}, "outputs": [], "source": [ @@ -3336,7 +3355,7 @@ }, { "cell_type": "code", - "execution_count": 245, + "execution_count": 250, "metadata": {}, "outputs": [], "source": [ @@ -3365,7 +3384,7 @@ }, { "cell_type": "code", - "execution_count": 246, + "execution_count": 251, "metadata": {}, "outputs": [], "source": [ @@ -3376,7 +3395,7 @@ }, { "cell_type": "code", - "execution_count": 247, + "execution_count": 252, "metadata": {}, "outputs": [], "source": [ @@ -3401,7 +3420,7 @@ }, { "cell_type": "code", - "execution_count": 248, + "execution_count": 253, "metadata": {}, "outputs": [], "source": [ @@ -3410,7 +3429,7 @@ }, { "cell_type": "code", - "execution_count": 249, + "execution_count": 254, "metadata": {}, "outputs": [], "source": [ @@ -3428,7 +3447,7 @@ }, { "cell_type": "code", - "execution_count": 250, + "execution_count": 255, "metadata": {}, "outputs": [], "source": [ @@ -3439,7 +3458,7 @@ }, { "cell_type": "code", - "execution_count": 251, + "execution_count": 256, "metadata": {}, "outputs": [], "source": [ @@ -3448,7 +3467,7 @@ }, { "cell_type": "code", - "execution_count": 252, + "execution_count": 257, "metadata": {}, "outputs": [], "source": [ @@ -3464,7 +3483,7 @@ }, { "cell_type": "code", - "execution_count": 253, + "execution_count": 258, "metadata": {}, "outputs": [], "source": [ @@ -3482,7 +3501,7 @@ }, { "cell_type": "code", - "execution_count": 254, + "execution_count": 259, "metadata": {}, "outputs": [], "source": [ @@ -3493,7 +3512,7 @@ }, { "cell_type": "code", - "execution_count": 255, + "execution_count": 260, "metadata": {}, "outputs": [], "source": [ @@ -3502,7 +3521,7 @@ }, { "cell_type": "code", - "execution_count": 256, + "execution_count": 261, "metadata": {}, "outputs": [], "source": [ @@ -3511,7 +3530,7 @@ }, { "cell_type": "code", - "execution_count": 257, + "execution_count": 262, "metadata": {}, "outputs": [], "source": [ @@ -3536,7 +3555,7 @@ }, { "cell_type": "code", - "execution_count": 258, + "execution_count": 263, "metadata": {}, "outputs": [], "source": [ @@ -3582,7 +3601,7 @@ }, { "cell_type": "code", - "execution_count": 259, + "execution_count": 264, "metadata": {}, "outputs": [], "source": [ @@ -3593,7 +3612,7 @@ }, { "cell_type": "code", - "execution_count": 260, + "execution_count": 265, "metadata": {}, "outputs": [], "source": [ @@ -3650,7 +3669,7 @@ }, { "cell_type": "code", - "execution_count": 261, + "execution_count": 266, "metadata": {}, "outputs": [], "source": [ @@ -3704,7 +3723,7 @@ }, { "cell_type": "code", - "execution_count": 262, + "execution_count": 267, "metadata": {}, "outputs": [], "source": [ @@ -3726,7 +3745,7 @@ }, { "cell_type": "code", - "execution_count": 263, + "execution_count": 268, "metadata": {}, "outputs": [], "source": [ @@ -3765,7 +3784,7 @@ }, { "cell_type": "code", - "execution_count": 264, + "execution_count": 269, "metadata": {}, "outputs": [], "source": [ @@ -3778,7 +3797,7 @@ }, { "cell_type": "code", - "execution_count": 265, + "execution_count": 270, "metadata": {}, "outputs": [], "source": [ @@ -3789,7 +3808,7 @@ }, { "cell_type": "code", - "execution_count": 266, + "execution_count": 271, "metadata": {}, "outputs": [], "source": [ @@ -3802,7 +3821,7 @@ }, { "cell_type": "code", - "execution_count": 267, + "execution_count": 272, "metadata": {}, "outputs": [], "source": [ @@ -3817,7 +3836,7 @@ }, { "cell_type": "code", - "execution_count": 268, + "execution_count": 273, "metadata": {}, "outputs": [], "source": [ @@ -3861,7 +3880,7 @@ }, { "cell_type": "code", - "execution_count": 269, + "execution_count": 274, "metadata": {}, "outputs": [], "source": [ @@ -3872,7 +3891,7 @@ }, { "cell_type": "code", - "execution_count": 270, + "execution_count": 275, "metadata": {}, "outputs": [], "source": [ @@ -3890,7 +3909,7 @@ }, { "cell_type": "code", - "execution_count": 271, + "execution_count": 276, "metadata": {}, "outputs": [], "source": [ @@ -3900,7 +3919,7 @@ }, { "cell_type": "code", - "execution_count": 272, + "execution_count": 277, "metadata": {}, "outputs": [], "source": [ @@ -3914,7 +3933,7 @@ }, { "cell_type": "code", - "execution_count": 273, + "execution_count": 278, "metadata": {}, "outputs": [], "source": [ @@ -3975,7 +3994,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.10" + "version": "3.7.10" } }, "nbformat": 4,