diff --git a/02_end_to_end_machine_learning_project.ipynb b/02_end_to_end_machine_learning_project.ipynb index 7acc3d8..95fcd36 100644 --- a/02_end_to_end_machine_learning_project.ipynb +++ b/02_end_to_end_machine_learning_project.ipynb @@ -81,7 +81,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -105,7 +105,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -114,7 +114,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -127,7 +127,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -137,7 +137,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -146,7 +146,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ @@ -155,7 +155,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ @@ -164,7 +164,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -177,7 +177,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ @@ -187,7 +187,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 11, "metadata": {}, "outputs": [], "source": [ @@ -204,7 +204,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 12, "metadata": {}, "outputs": [], "source": [ @@ -214,7 +214,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 13, "metadata": {}, "outputs": [], "source": [ @@ -223,7 +223,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 14, "metadata": {}, "outputs": [], "source": [ @@ -247,7 +247,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 15, "metadata": {}, "outputs": [], "source": [ @@ -266,7 +266,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 16, "metadata": {}, "outputs": [], "source": [ @@ -276,7 +276,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 17, "metadata": {}, "outputs": [], "source": [ @@ -286,7 +286,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 18, "metadata": {}, "outputs": [], "source": [ @@ -296,7 +296,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 19, "metadata": {}, "outputs": [], "source": [ @@ -305,7 +305,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 20, "metadata": {}, "outputs": [], "source": [ @@ -316,7 +316,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 21, "metadata": {}, "outputs": [], "source": [ @@ -325,7 +325,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 22, "metadata": {}, "outputs": [], "source": [ @@ -334,7 +334,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 23, "metadata": {}, "outputs": [], "source": [ @@ -345,7 +345,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 24, "metadata": {}, "outputs": [], "source": [ @@ -354,7 +354,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 25, "metadata": {}, "outputs": [], "source": [ @@ -363,7 +363,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 26, "metadata": {}, "outputs": [], "source": [ @@ -377,7 +377,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 27, "metadata": {}, "outputs": [], "source": [ @@ -386,7 +386,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 28, "metadata": {}, "outputs": [], "source": [ @@ -395,7 +395,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 29, "metadata": {}, "outputs": [], "source": [ @@ -415,7 +415,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 30, "metadata": {}, "outputs": [], "source": [ @@ -424,7 +424,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 31, "metadata": {}, "outputs": [], "source": [ @@ -441,7 +441,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 32, "metadata": {}, "outputs": [], "source": [ @@ -450,7 +450,7 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 33, "metadata": {}, "outputs": [], "source": [ @@ -460,7 +460,7 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 34, "metadata": {}, "outputs": [], "source": [ @@ -477,7 +477,7 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 35, "metadata": {}, "outputs": [], "source": [ @@ -491,7 +491,7 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 36, "metadata": {}, "outputs": [], "source": [ @@ -520,7 +520,7 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 37, "metadata": {}, "outputs": [], "source": [ @@ -529,7 +529,7 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 38, "metadata": {}, "outputs": [], "source": [ @@ -538,7 +538,7 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 39, "metadata": {}, "outputs": [], "source": [ @@ -553,7 +553,7 @@ }, { "cell_type": "code", - "execution_count": 41, + "execution_count": 40, "metadata": {}, "outputs": [], "source": [ @@ -565,7 +565,7 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": 41, "metadata": {}, "outputs": [], "source": [ @@ -576,7 +576,7 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": 42, "metadata": {}, "outputs": [], "source": [ @@ -586,7 +586,7 @@ }, { "cell_type": "code", - "execution_count": 44, + "execution_count": 43, "metadata": {}, "outputs": [], "source": [ @@ -598,7 +598,7 @@ }, { "cell_type": "code", - "execution_count": 45, + "execution_count": 44, "metadata": {}, "outputs": [], "source": [ @@ -614,7 +614,7 @@ }, { "cell_type": "code", - "execution_count": 46, + "execution_count": 45, "metadata": {}, "outputs": [], "source": [ @@ -624,7 +624,7 @@ }, { "cell_type": "code", - "execution_count": 47, + "execution_count": 46, "metadata": {}, "outputs": [], "source": [ @@ -634,7 +634,7 @@ }, { "cell_type": "code", - "execution_count": 48, + "execution_count": 47, "metadata": {}, "outputs": [], "source": [ @@ -643,7 +643,7 @@ }, { "cell_type": "code", - "execution_count": 49, + "execution_count": 48, "metadata": {}, "outputs": [], "source": [ @@ -652,7 +652,7 @@ }, { "cell_type": "code", - "execution_count": 50, + "execution_count": 49, "metadata": {}, "outputs": [], "source": [ @@ -662,7 +662,7 @@ }, { "cell_type": "code", - "execution_count": 51, + "execution_count": 50, "metadata": {}, "outputs": [], "source": [ @@ -671,7 +671,7 @@ }, { "cell_type": "code", - "execution_count": 52, + "execution_count": 51, "metadata": {}, "outputs": [], "source": [ @@ -688,7 +688,7 @@ }, { "cell_type": "code", - "execution_count": 53, + "execution_count": 52, "metadata": {}, "outputs": [], "source": [ @@ -698,7 +698,7 @@ }, { "cell_type": "code", - "execution_count": 54, + "execution_count": 53, "metadata": {}, "outputs": [], "source": [ @@ -707,7 +707,7 @@ }, { "cell_type": "code", - "execution_count": 55, + "execution_count": 54, "metadata": {}, "outputs": [], "source": [ @@ -723,7 +723,7 @@ }, { "cell_type": "code", - "execution_count": 56, + "execution_count": 55, "metadata": {}, "outputs": [], "source": [ @@ -739,7 +739,7 @@ }, { "cell_type": "code", - "execution_count": 57, + "execution_count": 56, "metadata": {}, "outputs": [], "source": [ @@ -748,7 +748,7 @@ }, { "cell_type": "code", - "execution_count": 58, + "execution_count": 57, "metadata": {}, "outputs": [], "source": [ @@ -758,7 +758,7 @@ }, { "cell_type": "code", - "execution_count": 59, + "execution_count": 58, "metadata": {}, "outputs": [], "source": [ @@ -767,7 +767,7 @@ }, { "cell_type": "code", - "execution_count": 60, + "execution_count": 59, "metadata": {}, "outputs": [], "source": [ @@ -776,7 +776,7 @@ }, { "cell_type": "code", - "execution_count": 61, + "execution_count": 60, "metadata": {}, "outputs": [], "source": [ @@ -785,7 +785,7 @@ }, { "cell_type": "code", - "execution_count": 62, + "execution_count": 61, "metadata": {}, "outputs": [], "source": [ @@ -801,7 +801,7 @@ }, { "cell_type": "code", - "execution_count": 63, + "execution_count": 62, "metadata": {}, "outputs": [], "source": [ @@ -811,7 +811,7 @@ }, { "cell_type": "code", - "execution_count": 64, + "execution_count": 63, "metadata": {}, "outputs": [], "source": [ @@ -824,7 +824,7 @@ }, { "cell_type": "code", - "execution_count": 65, + "execution_count": 64, "metadata": {}, "outputs": [], "source": [ @@ -833,7 +833,7 @@ }, { "cell_type": "code", - "execution_count": 66, + "execution_count": 65, "metadata": {}, "outputs": [], "source": [ @@ -853,7 +853,7 @@ }, { "cell_type": "code", - "execution_count": 67, + "execution_count": 66, "metadata": {}, "outputs": [], "source": [ @@ -869,7 +869,7 @@ }, { "cell_type": "code", - "execution_count": 68, + "execution_count": 67, "metadata": {}, "outputs": [], "source": [ @@ -880,7 +880,7 @@ }, { "cell_type": "code", - "execution_count": 69, + "execution_count": 68, "metadata": {}, "outputs": [], "source": [ @@ -896,7 +896,7 @@ }, { "cell_type": "code", - "execution_count": 70, + "execution_count": 69, "metadata": {}, "outputs": [], "source": [ @@ -926,7 +926,7 @@ }, { "cell_type": "code", - "execution_count": 71, + "execution_count": 70, "metadata": {}, "outputs": [], "source": [ @@ -945,7 +945,7 @@ }, { "cell_type": "code", - "execution_count": 72, + "execution_count": 71, "metadata": {}, "outputs": [], "source": [ @@ -963,7 +963,7 @@ }, { "cell_type": "code", - "execution_count": 73, + "execution_count": 72, "metadata": {}, "outputs": [], "source": [ @@ -972,7 +972,7 @@ }, { "cell_type": "code", - "execution_count": 74, + "execution_count": 73, "metadata": {}, "outputs": [], "source": [ @@ -991,7 +991,7 @@ }, { "cell_type": "code", - "execution_count": 75, + "execution_count": 74, "metadata": {}, "outputs": [], "source": [ @@ -1000,7 +1000,7 @@ }, { "cell_type": "code", - "execution_count": 76, + "execution_count": 75, "metadata": {}, "outputs": [], "source": [ @@ -1016,7 +1016,7 @@ }, { "cell_type": "code", - "execution_count": 77, + "execution_count": 76, "metadata": {}, "outputs": [], "source": [ @@ -1041,7 +1041,7 @@ }, { "cell_type": "code", - "execution_count": 78, + "execution_count": 77, "metadata": {}, "outputs": [], "source": [ @@ -1063,7 +1063,7 @@ }, { "cell_type": "code", - "execution_count": 79, + "execution_count": 78, "metadata": {}, "outputs": [], "source": [ @@ -1077,7 +1077,7 @@ }, { "cell_type": "code", - "execution_count": 80, + "execution_count": 79, "metadata": {}, "outputs": [], "source": [ @@ -1094,7 +1094,7 @@ }, { "cell_type": "code", - "execution_count": 81, + "execution_count": 80, "metadata": {}, "outputs": [], "source": [ @@ -1110,7 +1110,7 @@ }, { "cell_type": "code", - "execution_count": 82, + "execution_count": 81, "metadata": {}, "outputs": [], "source": [ @@ -1122,7 +1122,7 @@ }, { "cell_type": "code", - "execution_count": 83, + "execution_count": 82, "metadata": {}, "outputs": [], "source": [ @@ -1143,7 +1143,7 @@ }, { "cell_type": "code", - "execution_count": 84, + "execution_count": 83, "metadata": {}, "outputs": [], "source": [ @@ -1152,7 +1152,7 @@ }, { "cell_type": "code", - "execution_count": 85, + "execution_count": 84, "metadata": {}, "outputs": [], "source": [ @@ -1161,7 +1161,7 @@ }, { "cell_type": "code", - "execution_count": 86, + "execution_count": 85, "metadata": {}, "outputs": [], "source": [ @@ -1175,7 +1175,7 @@ }, { "cell_type": "code", - "execution_count": 87, + "execution_count": 86, "metadata": {}, "outputs": [], "source": [ @@ -1187,7 +1187,7 @@ }, { "cell_type": "code", - "execution_count": 88, + "execution_count": 87, "metadata": {}, "outputs": [], "source": [ @@ -1199,7 +1199,7 @@ }, { "cell_type": "code", - "execution_count": 89, + "execution_count": 88, "metadata": {}, "outputs": [], "source": [ @@ -1218,7 +1218,7 @@ }, { "cell_type": "code", - "execution_count": 90, + "execution_count": 89, "metadata": {}, "outputs": [], "source": [ @@ -1231,7 +1231,7 @@ }, { "cell_type": "code", - "execution_count": 91, + "execution_count": 90, "metadata": {}, "outputs": [], "source": [ @@ -1245,7 +1245,7 @@ }, { "cell_type": "code", - "execution_count": 92, + "execution_count": 91, "metadata": {}, "outputs": [], "source": [ @@ -1264,7 +1264,7 @@ }, { "cell_type": "code", - "execution_count": 93, + "execution_count": 92, "metadata": {}, "outputs": [], "source": [ @@ -1276,7 +1276,7 @@ }, { "cell_type": "code", - "execution_count": 94, + "execution_count": 93, "metadata": {}, "outputs": [], "source": [ @@ -1288,7 +1288,7 @@ }, { "cell_type": "code", - "execution_count": 95, + "execution_count": 94, "metadata": {}, "outputs": [], "source": [ @@ -1302,7 +1302,7 @@ }, { "cell_type": "code", - "execution_count": 96, + "execution_count": 95, "metadata": {}, "outputs": [], "source": [ @@ -1312,7 +1312,7 @@ }, { "cell_type": "code", - "execution_count": 97, + "execution_count": 96, "metadata": {}, "outputs": [], "source": [ @@ -1328,7 +1328,7 @@ }, { "cell_type": "code", - "execution_count": 98, + "execution_count": 97, "metadata": {}, "outputs": [], "source": [ @@ -1358,7 +1358,7 @@ }, { "cell_type": "code", - "execution_count": 99, + "execution_count": 98, "metadata": {}, "outputs": [], "source": [ @@ -1367,7 +1367,7 @@ }, { "cell_type": "code", - "execution_count": 100, + "execution_count": 99, "metadata": {}, "outputs": [], "source": [ @@ -1383,7 +1383,7 @@ }, { "cell_type": "code", - "execution_count": 101, + "execution_count": 100, "metadata": {}, "outputs": [], "source": [ @@ -1394,7 +1394,7 @@ }, { "cell_type": "code", - "execution_count": 102, + "execution_count": 101, "metadata": {}, "outputs": [], "source": [ @@ -1403,7 +1403,7 @@ }, { "cell_type": "code", - "execution_count": 103, + "execution_count": 102, "metadata": {}, "outputs": [], "source": [ @@ -1423,7 +1423,7 @@ }, { "cell_type": "code", - "execution_count": 104, + "execution_count": 103, "metadata": {}, "outputs": [], "source": [ @@ -1434,7 +1434,7 @@ }, { "cell_type": "code", - "execution_count": 105, + "execution_count": 104, "metadata": {}, "outputs": [], "source": [ @@ -1444,7 +1444,7 @@ }, { "cell_type": "code", - "execution_count": 106, + "execution_count": 105, "metadata": {}, "outputs": [], "source": [ @@ -1458,7 +1458,7 @@ }, { "cell_type": "code", - "execution_count": 107, + "execution_count": 106, "metadata": {}, "outputs": [], "source": [ @@ -1476,7 +1476,7 @@ }, { "cell_type": "code", - "execution_count": 108, + "execution_count": 107, "metadata": {}, "outputs": [], "source": [ @@ -1492,7 +1492,7 @@ }, { "cell_type": "code", - "execution_count": 109, + "execution_count": 108, "metadata": {}, "outputs": [], "source": [ @@ -1514,7 +1514,7 @@ }, { "cell_type": "code", - "execution_count": 110, + "execution_count": 109, "metadata": {}, "outputs": [], "source": [ @@ -1534,7 +1534,7 @@ }, { "cell_type": "code", - "execution_count": 111, + "execution_count": 110, "metadata": {}, "outputs": [], "source": [ @@ -1559,7 +1559,7 @@ }, { "cell_type": "code", - "execution_count": 112, + "execution_count": 111, "metadata": {}, "outputs": [], "source": [ @@ -1581,7 +1581,7 @@ }, { "cell_type": "code", - "execution_count": 113, + "execution_count": 112, "metadata": {}, "outputs": [], "source": [ @@ -1590,7 +1590,7 @@ }, { "cell_type": "code", - "execution_count": 114, + "execution_count": 113, "metadata": {}, "outputs": [], "source": [ @@ -1609,7 +1609,7 @@ }, { "cell_type": "code", - "execution_count": 115, + "execution_count": 114, "metadata": {}, "outputs": [], "source": [ @@ -1647,7 +1647,7 @@ }, { "cell_type": "code", - "execution_count": 116, + "execution_count": 115, "metadata": {}, "outputs": [], "source": [ @@ -1673,7 +1673,7 @@ }, { "cell_type": "code", - "execution_count": 117, + "execution_count": 116, "metadata": {}, "outputs": [], "source": [ @@ -1691,7 +1691,7 @@ }, { "cell_type": "code", - "execution_count": 118, + "execution_count": 117, "metadata": {}, "outputs": [], "source": [ @@ -1721,7 +1721,7 @@ }, { "cell_type": "code", - "execution_count": 119, + "execution_count": 118, "metadata": {}, "outputs": [], "source": [ @@ -1754,7 +1754,7 @@ }, { "cell_type": "code", - "execution_count": 120, + "execution_count": 119, "metadata": {}, "outputs": [], "source": [ @@ -1772,7 +1772,7 @@ }, { "cell_type": "code", - "execution_count": 121, + "execution_count": 120, "metadata": {}, "outputs": [], "source": [ @@ -1795,7 +1795,7 @@ }, { "cell_type": "code", - "execution_count": 122, + "execution_count": 121, "metadata": {}, "outputs": [], "source": [ @@ -1820,7 +1820,7 @@ }, { "cell_type": "code", - "execution_count": 123, + "execution_count": 122, "metadata": {}, "outputs": [], "source": [ @@ -1859,7 +1859,7 @@ }, { "cell_type": "code", - "execution_count": 124, + "execution_count": 123, "metadata": {}, "outputs": [], "source": [ @@ -1895,7 +1895,7 @@ }, { "cell_type": "code", - "execution_count": 125, + "execution_count": 124, "metadata": {}, "outputs": [], "source": [ @@ -1911,7 +1911,7 @@ }, { "cell_type": "code", - "execution_count": 126, + "execution_count": 125, "metadata": {}, "outputs": [], "source": [ @@ -1921,7 +1921,7 @@ }, { "cell_type": "code", - "execution_count": 127, + "execution_count": 126, "metadata": {}, "outputs": [], "source": [ @@ -1937,7 +1937,7 @@ }, { "cell_type": "code", - "execution_count": 128, + "execution_count": 127, "metadata": {}, "outputs": [], "source": [ @@ -1953,7 +1953,7 @@ }, { "cell_type": "code", - "execution_count": 129, + "execution_count": 128, "metadata": {}, "outputs": [], "source": [ @@ -1965,7 +1965,7 @@ }, { "cell_type": "code", - "execution_count": 130, + "execution_count": 129, "metadata": {}, "outputs": [], "source": [ @@ -1981,7 +1981,7 @@ }, { "cell_type": "code", - "execution_count": 131, + "execution_count": 130, "metadata": {}, "outputs": [], "source": [ @@ -1997,7 +1997,7 @@ }, { "cell_type": "code", - "execution_count": 132, + "execution_count": 131, "metadata": {}, "outputs": [], "source": [ @@ -2027,7 +2027,7 @@ }, { "cell_type": "code", - "execution_count": 133, + "execution_count": 132, "metadata": {}, "outputs": [], "source": [ @@ -2040,7 +2040,7 @@ }, { "cell_type": "code", - "execution_count": 134, + "execution_count": 133, "metadata": {}, "outputs": [], "source": [ @@ -2056,7 +2056,7 @@ }, { "cell_type": "code", - "execution_count": 135, + "execution_count": 134, "metadata": {}, "outputs": [], "source": [ @@ -2090,7 +2090,7 @@ }, { "cell_type": "code", - "execution_count": 136, + "execution_count": 135, "metadata": {}, "outputs": [], "source": [ @@ -2106,7 +2106,7 @@ }, { "cell_type": "code", - "execution_count": 137, + "execution_count": 136, "metadata": {}, "outputs": [], "source": [ diff --git a/03_classification.ipynb b/03_classification.ipynb index b9f3650..852d0c1 100644 --- a/03_classification.ipynb +++ b/03_classification.ipynb @@ -935,7 +935,7 @@ }, { "cell_type": "code", - "execution_count": 73, + "execution_count": 72, "metadata": {}, "outputs": [], "source": [ @@ -952,7 +952,7 @@ }, { "cell_type": "code", - "execution_count": 74, + "execution_count": 73, "metadata": {}, "outputs": [], "source": [ @@ -966,7 +966,7 @@ }, { "cell_type": "code", - "execution_count": 75, + "execution_count": 74, "metadata": {}, "outputs": [], "source": [ @@ -979,7 +979,7 @@ }, { "cell_type": "code", - "execution_count": 76, + "execution_count": 75, "metadata": {}, "outputs": [], "source": [ @@ -1005,7 +1005,7 @@ }, { "cell_type": "code", - "execution_count": 77, + "execution_count": 76, "metadata": {}, "outputs": [], "source": [ @@ -1017,7 +1017,7 @@ }, { "cell_type": "code", - "execution_count": 78, + "execution_count": 77, "metadata": { "scrolled": true }, @@ -1036,7 +1036,7 @@ }, { "cell_type": "code", - "execution_count": 79, + "execution_count": 78, "metadata": {}, "outputs": [], "source": [ @@ -1047,7 +1047,7 @@ }, { "cell_type": "code", - "execution_count": 80, + "execution_count": 79, "metadata": {}, "outputs": [], "source": [ @@ -1056,7 +1056,7 @@ }, { "cell_type": "code", - "execution_count": 81, + "execution_count": 80, "metadata": {}, "outputs": [], "source": [ @@ -1066,7 +1066,7 @@ }, { "cell_type": "code", - "execution_count": 82, + "execution_count": 81, "metadata": {}, "outputs": [], "source": [ @@ -1079,7 +1079,7 @@ }, { "cell_type": "code", - "execution_count": 83, + "execution_count": 82, "metadata": {}, "outputs": [], "source": [ @@ -1097,7 +1097,7 @@ }, { "cell_type": "code", - "execution_count": 84, + "execution_count": 83, "metadata": {}, "outputs": [], "source": [ @@ -1106,7 +1106,7 @@ }, { "cell_type": "code", - "execution_count": 85, + "execution_count": 84, "metadata": {}, "outputs": [], "source": [ @@ -1115,7 +1115,7 @@ }, { "cell_type": "code", - "execution_count": 86, + "execution_count": 85, "metadata": {}, "outputs": [], "source": [ @@ -1124,7 +1124,7 @@ }, { "cell_type": "code", - "execution_count": 87, + "execution_count": 86, "metadata": {}, "outputs": [], "source": [ @@ -1134,7 +1134,7 @@ }, { "cell_type": "code", - "execution_count": 88, + "execution_count": 87, "metadata": {}, "outputs": [], "source": [ @@ -1166,7 +1166,7 @@ }, { "cell_type": "code", - "execution_count": 89, + "execution_count": 88, "metadata": {}, "outputs": [], "source": [ @@ -1181,7 +1181,7 @@ }, { "cell_type": "code", - "execution_count": 90, + "execution_count": 89, "metadata": {}, "outputs": [], "source": [ @@ -1190,7 +1190,7 @@ }, { "cell_type": "code", - "execution_count": 91, + "execution_count": 90, "metadata": {}, "outputs": [], "source": [ @@ -1199,7 +1199,7 @@ }, { "cell_type": "code", - "execution_count": 92, + "execution_count": 91, "metadata": {}, "outputs": [], "source": [ @@ -1218,7 +1218,7 @@ }, { "cell_type": "code", - "execution_count": 93, + "execution_count": 92, "metadata": {}, "outputs": [], "source": [ @@ -1227,7 +1227,7 @@ }, { "cell_type": "code", - "execution_count": 94, + "execution_count": 93, "metadata": {}, "outputs": [], "source": [ @@ -1239,7 +1239,7 @@ }, { "cell_type": "code", - "execution_count": 95, + "execution_count": 94, "metadata": {}, "outputs": [], "source": [ @@ -1262,7 +1262,7 @@ }, { "cell_type": "code", - "execution_count": 96, + "execution_count": 95, "metadata": {}, "outputs": [], "source": [ @@ -1280,7 +1280,7 @@ }, { "cell_type": "code", - "execution_count": 97, + "execution_count": 96, "metadata": {}, "outputs": [], "source": [ @@ -1291,7 +1291,7 @@ }, { "cell_type": "code", - "execution_count": 98, + "execution_count": 97, "metadata": {}, "outputs": [], "source": [ @@ -1300,7 +1300,7 @@ }, { "cell_type": "code", - "execution_count": 99, + "execution_count": 98, "metadata": {}, "outputs": [], "source": [ @@ -1309,7 +1309,7 @@ }, { "cell_type": "code", - "execution_count": 100, + "execution_count": 99, "metadata": {}, "outputs": [], "source": [ @@ -1354,7 +1354,7 @@ }, { "cell_type": "code", - "execution_count": 101, + "execution_count": 100, "metadata": {}, "outputs": [], "source": [ @@ -1365,7 +1365,7 @@ }, { "cell_type": "code", - "execution_count": 102, + "execution_count": 101, "metadata": {}, "outputs": [], "source": [ @@ -1378,7 +1378,7 @@ }, { "cell_type": "code", - "execution_count": 104, + "execution_count": 102, "metadata": {}, "outputs": [], "source": [ @@ -1402,7 +1402,7 @@ }, { "cell_type": "code", - "execution_count": 105, + "execution_count": 103, "metadata": {}, "outputs": [], "source": [ @@ -1434,7 +1434,7 @@ }, { "cell_type": "code", - "execution_count": 106, + "execution_count": 104, "metadata": {}, "outputs": [], "source": [ @@ -1464,7 +1464,7 @@ }, { "cell_type": "code", - "execution_count": 107, + "execution_count": 105, "metadata": {}, "outputs": [], "source": [ @@ -1489,7 +1489,7 @@ }, { "cell_type": "code", - "execution_count": 108, + "execution_count": 106, "metadata": {}, "outputs": [], "source": [ @@ -1505,7 +1505,7 @@ }, { "cell_type": "code", - "execution_count": 109, + "execution_count": 107, "metadata": {}, "outputs": [], "source": [ @@ -1514,7 +1514,7 @@ }, { "cell_type": "code", - "execution_count": 110, + "execution_count": 108, "metadata": {}, "outputs": [], "source": [ @@ -1523,7 +1523,7 @@ }, { "cell_type": "code", - "execution_count": 111, + "execution_count": 109, "metadata": {}, "outputs": [], "source": [ @@ -1553,7 +1553,7 @@ }, { "cell_type": "code", - "execution_count": 112, + "execution_count": 110, "metadata": {}, "outputs": [], "source": [ @@ -1577,7 +1577,7 @@ }, { "cell_type": "code", - "execution_count": 113, + "execution_count": 111, "metadata": {}, "outputs": [], "source": [ @@ -1592,7 +1592,7 @@ }, { "cell_type": "code", - "execution_count": 114, + "execution_count": 112, "metadata": {}, "outputs": [], "source": [ @@ -1608,7 +1608,7 @@ }, { "cell_type": "code", - "execution_count": 115, + "execution_count": 113, "metadata": {}, "outputs": [], "source": [ @@ -1624,7 +1624,7 @@ }, { "cell_type": "code", - "execution_count": 116, + "execution_count": 114, "metadata": {}, "outputs": [], "source": [ @@ -1640,7 +1640,7 @@ }, { "cell_type": "code", - "execution_count": 117, + "execution_count": 115, "metadata": {}, "outputs": [], "source": [ @@ -1653,7 +1653,7 @@ }, { "cell_type": "code", - "execution_count": 118, + "execution_count": 116, "metadata": {}, "outputs": [], "source": [ @@ -1669,7 +1669,7 @@ }, { "cell_type": "code", - "execution_count": 119, + "execution_count": 117, "metadata": {}, "outputs": [], "source": [ @@ -1689,7 +1689,7 @@ }, { "cell_type": "code", - "execution_count": 120, + "execution_count": 118, "metadata": {}, "outputs": [], "source": [ @@ -1706,7 +1706,7 @@ }, { "cell_type": "code", - "execution_count": 121, + "execution_count": 119, "metadata": {}, "outputs": [], "source": [ @@ -1722,7 +1722,7 @@ }, { "cell_type": "code", - "execution_count": 122, + "execution_count": 120, "metadata": {}, "outputs": [], "source": [ @@ -1741,7 +1741,7 @@ }, { "cell_type": "code", - "execution_count": 123, + "execution_count": 121, "metadata": {}, "outputs": [], "source": [ @@ -1758,7 +1758,7 @@ }, { "cell_type": "code", - "execution_count": 124, + "execution_count": 122, "metadata": {}, "outputs": [], "source": [ @@ -1784,7 +1784,7 @@ }, { "cell_type": "code", - "execution_count": 125, + "execution_count": 123, "metadata": {}, "outputs": [], "source": [ @@ -1811,7 +1811,7 @@ }, { "cell_type": "code", - "execution_count": 126, + "execution_count": 124, "metadata": {}, "outputs": [], "source": [ @@ -1837,7 +1837,7 @@ }, { "cell_type": "code", - "execution_count": 127, + "execution_count": 125, "metadata": {}, "outputs": [], "source": [ @@ -1847,7 +1847,7 @@ }, { "cell_type": "code", - "execution_count": 128, + "execution_count": 126, "metadata": {}, "outputs": [], "source": [ @@ -1871,7 +1871,7 @@ }, { "cell_type": "code", - "execution_count": 129, + "execution_count": 127, "metadata": {}, "outputs": [], "source": [ @@ -1898,7 +1898,7 @@ }, { "cell_type": "code", - "execution_count": 130, + "execution_count": 128, "metadata": {}, "outputs": [], "source": [ @@ -1914,7 +1914,7 @@ }, { "cell_type": "code", - "execution_count": 131, + "execution_count": 129, "metadata": {}, "outputs": [], "source": [ @@ -1926,7 +1926,7 @@ }, { "cell_type": "code", - "execution_count": 132, + "execution_count": 130, "metadata": {}, "outputs": [], "source": [ @@ -1935,7 +1935,7 @@ }, { "cell_type": "code", - "execution_count": 133, + "execution_count": 131, "metadata": {}, "outputs": [], "source": [ @@ -1951,7 +1951,7 @@ }, { "cell_type": "code", - "execution_count": 134, + "execution_count": 132, "metadata": {}, "outputs": [], "source": [ @@ -1966,7 +1966,7 @@ }, { "cell_type": "code", - "execution_count": 135, + "execution_count": 133, "metadata": {}, "outputs": [], "source": [ @@ -1983,7 +1983,7 @@ }, { "cell_type": "code", - "execution_count": 136, + "execution_count": 134, "metadata": {}, "outputs": [], "source": [ @@ -1992,7 +1992,7 @@ }, { "cell_type": "code", - "execution_count": 137, + "execution_count": 135, "metadata": {}, "outputs": [], "source": [ @@ -2008,7 +2008,7 @@ }, { "cell_type": "code", - "execution_count": 138, + "execution_count": 136, "metadata": {}, "outputs": [], "source": [ @@ -2027,7 +2027,7 @@ }, { "cell_type": "code", - "execution_count": 139, + "execution_count": 137, "metadata": {}, "outputs": [], "source": [ @@ -2043,7 +2043,7 @@ }, { "cell_type": "code", - "execution_count": 140, + "execution_count": 138, "metadata": {}, "outputs": [], "source": [ @@ -2052,7 +2052,7 @@ }, { "cell_type": "code", - "execution_count": 141, + "execution_count": 139, "metadata": {}, "outputs": [], "source": [ @@ -2075,7 +2075,7 @@ }, { "cell_type": "code", - "execution_count": 142, + "execution_count": 140, "metadata": {}, "outputs": [], "source": [ @@ -2092,7 +2092,7 @@ }, { "cell_type": "code", - "execution_count": 143, + "execution_count": 141, "metadata": {}, "outputs": [], "source": [ @@ -2108,7 +2108,7 @@ }, { "cell_type": "code", - "execution_count": 144, + "execution_count": 142, "metadata": {}, "outputs": [], "source": [ @@ -2130,7 +2130,7 @@ }, { "cell_type": "code", - "execution_count": 145, + "execution_count": 143, "metadata": {}, "outputs": [], "source": [ @@ -2154,7 +2154,7 @@ }, { "cell_type": "code", - "execution_count": 146, + "execution_count": 144, "metadata": {}, "outputs": [], "source": [ @@ -2173,7 +2173,7 @@ }, { "cell_type": "code", - "execution_count": 147, + "execution_count": 145, "metadata": {}, "outputs": [], "source": [ @@ -2189,7 +2189,7 @@ }, { "cell_type": "code", - "execution_count": 148, + "execution_count": 146, "metadata": {}, "outputs": [], "source": [ @@ -2213,7 +2213,7 @@ }, { "cell_type": "code", - "execution_count": 149, + "execution_count": 147, "metadata": {}, "outputs": [], "source": [ @@ -2231,7 +2231,7 @@ }, { "cell_type": "code", - "execution_count": 150, + "execution_count": 148, "metadata": {}, "outputs": [], "source": [ @@ -2257,7 +2257,7 @@ }, { "cell_type": "code", - "execution_count": 151, + "execution_count": 149, "metadata": {}, "outputs": [], "source": [ @@ -2280,7 +2280,7 @@ }, { "cell_type": "code", - "execution_count": 152, + "execution_count": 150, "metadata": {}, "outputs": [], "source": [ @@ -2332,7 +2332,7 @@ }, { "cell_type": "code", - "execution_count": 153, + "execution_count": 151, "metadata": {}, "outputs": [], "source": [ @@ -2357,7 +2357,7 @@ }, { "cell_type": "code", - "execution_count": 154, + "execution_count": 152, "metadata": {}, "outputs": [], "source": [ @@ -2389,7 +2389,7 @@ }, { "cell_type": "code", - "execution_count": 155, + "execution_count": 153, "metadata": {}, "outputs": [], "source": [ @@ -2400,7 +2400,7 @@ }, { "cell_type": "code", - "execution_count": 156, + "execution_count": 154, "metadata": {}, "outputs": [], "source": [ @@ -2416,7 +2416,7 @@ }, { "cell_type": "code", - "execution_count": 157, + "execution_count": 155, "metadata": {}, "outputs": [], "source": [ @@ -2432,7 +2432,7 @@ }, { "cell_type": "code", - "execution_count": 158, + "execution_count": 156, "metadata": {}, "outputs": [], "source": [ @@ -2455,7 +2455,7 @@ }, { "cell_type": "code", - "execution_count": 159, + "execution_count": 157, "metadata": {}, "outputs": [], "source": [ @@ -2478,7 +2478,7 @@ }, { "cell_type": "code", - "execution_count": 160, + "execution_count": 158, "metadata": {}, "outputs": [], "source": [ diff --git a/04_training_linear_models.ipynb b/04_training_linear_models.ipynb index 21b0bfe..cfbb9f7 100644 --- a/04_training_linear_models.ipynb +++ b/04_training_linear_models.ipynb @@ -84,7 +84,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -96,7 +96,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -110,7 +110,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -120,7 +120,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -129,7 +129,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -141,7 +141,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ @@ -160,7 +160,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ @@ -176,7 +176,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -189,7 +189,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ @@ -205,7 +205,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 11, "metadata": {}, "outputs": [], "source": [ @@ -222,7 +222,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 12, "metadata": {}, "outputs": [], "source": [ @@ -238,7 +238,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 13, "metadata": {}, "outputs": [], "source": [ @@ -255,7 +255,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 14, "metadata": {}, "outputs": [], "source": [ @@ -264,7 +264,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 15, "metadata": {}, "outputs": [], "source": [ @@ -273,7 +273,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 16, "metadata": {}, "outputs": [], "source": [ @@ -299,7 +299,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 17, "metadata": {}, "outputs": [], "source": [ @@ -325,7 +325,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 18, "metadata": {}, "outputs": [], "source": [ @@ -336,7 +336,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 19, "metadata": {}, "outputs": [], "source": [ @@ -372,7 +372,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 20, "metadata": { "scrolled": true }, @@ -383,7 +383,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 21, "metadata": {}, "outputs": [], "source": [ @@ -395,7 +395,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 22, "metadata": {}, "outputs": [], "source": [ @@ -411,7 +411,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 23, "metadata": {}, "outputs": [], "source": [ @@ -444,7 +444,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 24, "metadata": {}, "outputs": [], "source": [ @@ -453,7 +453,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 25, "metadata": {}, "outputs": [], "source": [ @@ -464,7 +464,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 26, "metadata": {}, "outputs": [], "source": [ @@ -489,7 +489,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 27, "metadata": {}, "outputs": [], "source": [ @@ -501,7 +501,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 28, "metadata": {}, "outputs": [], "source": [ @@ -512,7 +512,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 29, "metadata": {}, "outputs": [], "source": [ @@ -526,7 +526,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 30, "metadata": {}, "outputs": [], "source": [ @@ -538,7 +538,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 31, "metadata": {}, "outputs": [], "source": [ @@ -547,7 +547,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 32, "metadata": {}, "outputs": [], "source": [ @@ -558,7 +558,7 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 33, "metadata": {}, "outputs": [], "source": [ @@ -577,7 +577,7 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 34, "metadata": {}, "outputs": [], "source": [ @@ -608,7 +608,7 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 35, "metadata": {}, "outputs": [], "source": [ @@ -634,7 +634,7 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 36, "metadata": {}, "outputs": [], "source": [ @@ -647,7 +647,7 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 37, "metadata": {}, "outputs": [], "source": [ @@ -673,7 +673,7 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 38, "metadata": {}, "outputs": [], "source": [ @@ -686,7 +686,7 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 39, "metadata": {}, "outputs": [], "source": [ @@ -698,7 +698,7 @@ }, { "cell_type": "code", - "execution_count": 41, + "execution_count": 40, "metadata": {}, "outputs": [], "source": [ @@ -709,7 +709,7 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": 41, "metadata": {}, "outputs": [], "source": [ @@ -753,7 +753,7 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": 42, "metadata": {}, "outputs": [], "source": [ @@ -764,7 +764,7 @@ }, { "cell_type": "code", - "execution_count": 44, + "execution_count": 43, "metadata": {}, "outputs": [], "source": [ @@ -783,7 +783,7 @@ }, { "cell_type": "code", - "execution_count": 45, + "execution_count": 44, "metadata": {}, "outputs": [], "source": [ @@ -795,7 +795,7 @@ }, { "cell_type": "code", - "execution_count": 46, + "execution_count": 45, "metadata": {}, "outputs": [], "source": [ @@ -807,7 +807,7 @@ }, { "cell_type": "code", - "execution_count": 47, + "execution_count": 46, "metadata": {}, "outputs": [], "source": [ @@ -828,7 +828,7 @@ }, { "cell_type": "code", - "execution_count": 48, + "execution_count": 47, "metadata": { "scrolled": true }, @@ -869,7 +869,7 @@ }, { "cell_type": "code", - "execution_count": 49, + "execution_count": 48, "metadata": {}, "outputs": [], "source": [ @@ -909,7 +909,7 @@ }, { "cell_type": "code", - "execution_count": 50, + "execution_count": 49, "metadata": {}, "outputs": [], "source": [ @@ -918,7 +918,7 @@ }, { "cell_type": "code", - "execution_count": 51, + "execution_count": 50, "metadata": {}, "outputs": [], "source": [ @@ -929,7 +929,7 @@ }, { "cell_type": "code", - "execution_count": 52, + "execution_count": 51, "metadata": {}, "outputs": [], "source": [ @@ -956,7 +956,7 @@ }, { "cell_type": "code", - "execution_count": 53, + "execution_count": 52, "metadata": {}, "outputs": [], "source": [ @@ -1024,7 +1024,7 @@ }, { "cell_type": "code", - "execution_count": 54, + "execution_count": 53, "metadata": {}, "outputs": [], "source": [ @@ -1045,7 +1045,7 @@ }, { "cell_type": "code", - "execution_count": 55, + "execution_count": 54, "metadata": {}, "outputs": [], "source": [ @@ -1056,7 +1056,7 @@ }, { "cell_type": "code", - "execution_count": 56, + "execution_count": 55, "metadata": {}, "outputs": [], "source": [ @@ -1065,7 +1065,7 @@ }, { "cell_type": "code", - "execution_count": 57, + "execution_count": 56, "metadata": {}, "outputs": [], "source": [ @@ -1082,7 +1082,7 @@ }, { "cell_type": "code", - "execution_count": 58, + "execution_count": 57, "metadata": {}, "outputs": [], "source": [ @@ -1093,7 +1093,7 @@ }, { "cell_type": "code", - "execution_count": 59, + "execution_count": 58, "metadata": {}, "outputs": [], "source": [ @@ -1113,7 +1113,7 @@ }, { "cell_type": "code", - "execution_count": 60, + "execution_count": 59, "metadata": {}, "outputs": [], "source": [ @@ -1140,7 +1140,7 @@ }, { "cell_type": "code", - "execution_count": 61, + "execution_count": 60, "metadata": {}, "outputs": [], "source": [ @@ -1149,7 +1149,7 @@ }, { "cell_type": "code", - "execution_count": 62, + "execution_count": 61, "metadata": {}, "outputs": [], "source": [ @@ -1158,7 +1158,7 @@ }, { "cell_type": "code", - "execution_count": 63, + "execution_count": 62, "metadata": {}, "outputs": [], "source": [ @@ -1202,7 +1202,7 @@ }, { "cell_type": "code", - "execution_count": 64, + "execution_count": 63, "metadata": {}, "outputs": [], "source": [ @@ -1215,7 +1215,7 @@ }, { "cell_type": "code", - "execution_count": 65, + "execution_count": 64, "metadata": {}, "outputs": [], "source": [ @@ -1253,7 +1253,7 @@ }, { "cell_type": "code", - "execution_count": 66, + "execution_count": 65, "metadata": {}, "outputs": [], "source": [ @@ -1262,7 +1262,7 @@ }, { "cell_type": "code", - "execution_count": 67, + "execution_count": 66, "metadata": {}, "outputs": [], "source": [ @@ -1307,7 +1307,7 @@ }, { "cell_type": "code", - "execution_count": 68, + "execution_count": 67, "metadata": {}, "outputs": [], "source": [ @@ -1324,7 +1324,7 @@ }, { "cell_type": "code", - "execution_count": 69, + "execution_count": 68, "metadata": {}, "outputs": [], "source": [ @@ -1340,7 +1340,7 @@ }, { "cell_type": "code", - "execution_count": 70, + "execution_count": 69, "metadata": {}, "outputs": [], "source": [ @@ -1356,7 +1356,7 @@ }, { "cell_type": "code", - "execution_count": 71, + "execution_count": 70, "metadata": {}, "outputs": [], "source": [ @@ -1387,7 +1387,7 @@ }, { "cell_type": "code", - "execution_count": 72, + "execution_count": 71, "metadata": {}, "outputs": [], "source": [ @@ -1408,7 +1408,7 @@ }, { "cell_type": "code", - "execution_count": 73, + "execution_count": 72, "metadata": {}, "outputs": [], "source": [ @@ -1417,7 +1417,7 @@ }, { "cell_type": "code", - "execution_count": 74, + "execution_count": 73, "metadata": {}, "outputs": [], "source": [ @@ -1433,7 +1433,7 @@ }, { "cell_type": "code", - "execution_count": 75, + "execution_count": 74, "metadata": {}, "outputs": [], "source": [ @@ -1453,7 +1453,7 @@ }, { "cell_type": "code", - "execution_count": 76, + "execution_count": 75, "metadata": {}, "outputs": [], "source": [ @@ -1472,7 +1472,7 @@ }, { "cell_type": "code", - "execution_count": 77, + "execution_count": 76, "metadata": {}, "outputs": [], "source": [ @@ -1500,7 +1500,7 @@ }, { "cell_type": "code", - "execution_count": 78, + "execution_count": 77, "metadata": {}, "outputs": [], "source": [ @@ -1531,7 +1531,7 @@ }, { "cell_type": "code", - "execution_count": 79, + "execution_count": 78, "metadata": {}, "outputs": [], "source": [ @@ -1547,7 +1547,7 @@ }, { "cell_type": "code", - "execution_count": 80, + "execution_count": 79, "metadata": {}, "outputs": [], "source": [ @@ -1568,7 +1568,7 @@ }, { "cell_type": "code", - "execution_count": 81, + "execution_count": 80, "metadata": {}, "outputs": [], "source": [ @@ -1602,7 +1602,7 @@ }, { "cell_type": "code", - "execution_count": 82, + "execution_count": 81, "metadata": {}, "outputs": [], "source": [ @@ -1630,7 +1630,7 @@ }, { "cell_type": "code", - "execution_count": 83, + "execution_count": 82, "metadata": {}, "outputs": [], "source": [ @@ -1670,7 +1670,7 @@ }, { "cell_type": "code", - "execution_count": 84, + "execution_count": 83, "metadata": {}, "outputs": [], "source": [ @@ -1698,7 +1698,7 @@ }, { "cell_type": "code", - "execution_count": 85, + "execution_count": 84, "metadata": {}, "outputs": [], "source": [ @@ -1743,7 +1743,7 @@ }, { "cell_type": "code", - "execution_count": 86, + "execution_count": 85, "metadata": {}, "outputs": [], "source": [ diff --git a/05_support_vector_machines.ipynb b/05_support_vector_machines.ipynb index 50c8a01..91f4978 100644 --- a/05_support_vector_machines.ipynb +++ b/05_support_vector_machines.ipynb @@ -1,12 +1,5 @@ { "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, { "cell_type": "markdown", "metadata": {}, @@ -89,7 +82,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -111,7 +104,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -173,7 +166,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -217,7 +210,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -282,7 +275,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -306,7 +299,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ @@ -322,7 +315,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ @@ -345,7 +338,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -369,7 +362,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ @@ -406,7 +399,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 11, "metadata": {}, "outputs": [], "source": [ @@ -445,7 +438,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 12, "metadata": {}, "outputs": [], "source": [ @@ -466,7 +459,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 13, "metadata": {}, "outputs": [], "source": [ @@ -485,7 +478,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 14, "metadata": {}, "outputs": [], "source": [ @@ -508,7 +501,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 15, "metadata": {}, "outputs": [], "source": [ @@ -523,7 +516,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 16, "metadata": {}, "outputs": [], "source": [ @@ -536,7 +529,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 17, "metadata": {}, "outputs": [], "source": [ @@ -558,7 +551,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 18, "metadata": { "scrolled": true }, @@ -626,7 +619,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 19, "metadata": {}, "outputs": [], "source": [ @@ -638,7 +631,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 20, "metadata": {}, "outputs": [], "source": [ @@ -651,7 +644,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 21, "metadata": { "scrolled": true }, @@ -694,7 +687,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 22, "metadata": {}, "outputs": [], "source": [ @@ -706,7 +699,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 23, "metadata": {}, "outputs": [], "source": [ @@ -718,7 +711,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 24, "metadata": {}, "outputs": [], "source": [ @@ -741,7 +734,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 25, "metadata": {}, "outputs": [], "source": [ @@ -778,7 +771,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 26, "metadata": {}, "outputs": [], "source": [ @@ -797,7 +790,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 27, "metadata": {}, "outputs": [], "source": [ @@ -809,7 +802,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 28, "metadata": {}, "outputs": [], "source": [ @@ -823,7 +816,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 29, "metadata": {}, "outputs": [], "source": [ @@ -848,7 +841,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 30, "metadata": {}, "outputs": [], "source": [ @@ -859,7 +852,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 31, "metadata": {}, "outputs": [], "source": [ @@ -910,7 +903,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 32, "metadata": {}, "outputs": [], "source": [ @@ -944,7 +937,7 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 33, "metadata": {}, "outputs": [], "source": [ @@ -969,7 +962,7 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 34, "metadata": {}, "outputs": [], "source": [ @@ -1005,7 +998,7 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 35, "metadata": {}, "outputs": [], "source": [ @@ -1016,7 +1009,7 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 36, "metadata": {}, "outputs": [], "source": [ @@ -1050,7 +1043,7 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 37, "metadata": {}, "outputs": [], "source": [ @@ -1061,7 +1054,7 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 38, "metadata": {}, "outputs": [], "source": [ @@ -1126,7 +1119,7 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 39, "metadata": {}, "outputs": [], "source": [ @@ -1136,7 +1129,7 @@ }, { "cell_type": "code", - "execution_count": 41, + "execution_count": 40, "metadata": {}, "outputs": [], "source": [ @@ -1145,7 +1138,7 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": 41, "metadata": {}, "outputs": [], "source": [ @@ -1156,7 +1149,7 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": 42, "metadata": {}, "outputs": [], "source": [ @@ -1182,7 +1175,7 @@ }, { "cell_type": "code", - "execution_count": 44, + "execution_count": 43, "metadata": { "scrolled": true }, @@ -1259,7 +1252,7 @@ }, { "cell_type": "code", - "execution_count": 45, + "execution_count": 44, "metadata": {}, "outputs": [], "source": [ @@ -1276,7 +1269,7 @@ }, { "cell_type": "code", - "execution_count": 46, + "execution_count": 45, "metadata": {}, "outputs": [], "source": [ @@ -1313,7 +1306,7 @@ }, { "cell_type": "code", - "execution_count": 47, + "execution_count": 46, "metadata": {}, "outputs": [], "source": [ @@ -1375,7 +1368,7 @@ }, { "cell_type": "code", - "execution_count": 59, + "execution_count": 47, "metadata": {}, "outputs": [], "source": [ @@ -1409,7 +1402,7 @@ }, { "cell_type": "code", - "execution_count": 60, + "execution_count": 48, "metadata": {}, "outputs": [], "source": [ @@ -1426,7 +1419,7 @@ }, { "cell_type": "code", - "execution_count": 61, + "execution_count": 49, "metadata": {}, "outputs": [], "source": [ @@ -1445,7 +1438,7 @@ }, { "cell_type": "code", - "execution_count": 62, + "execution_count": 50, "metadata": {}, "outputs": [], "source": [ @@ -1463,7 +1456,7 @@ }, { "cell_type": "code", - "execution_count": 63, + "execution_count": 51, "metadata": {}, "outputs": [], "source": [ @@ -1473,7 +1466,7 @@ }, { "cell_type": "code", - "execution_count": 64, + "execution_count": 52, "metadata": {}, "outputs": [], "source": [ @@ -1497,7 +1490,7 @@ }, { "cell_type": "code", - "execution_count": 77, + "execution_count": 53, "metadata": {}, "outputs": [], "source": [ @@ -1507,7 +1500,7 @@ }, { "cell_type": "code", - "execution_count": 78, + "execution_count": 54, "metadata": {}, "outputs": [], "source": [ @@ -1524,7 +1517,7 @@ }, { "cell_type": "code", - "execution_count": 56, + "execution_count": 55, "metadata": {}, "outputs": [], "source": [ @@ -1538,7 +1531,7 @@ }, { "cell_type": "code", - "execution_count": 57, + "execution_count": 56, "metadata": {}, "outputs": [], "source": [ @@ -1547,7 +1540,7 @@ }, { "cell_type": "code", - "execution_count": 58, + "execution_count": 57, "metadata": {}, "outputs": [], "source": [ @@ -1563,7 +1556,7 @@ }, { "cell_type": "code", - "execution_count": 60, + "execution_count": 58, "metadata": {}, "outputs": [], "source": [ @@ -1572,7 +1565,7 @@ }, { "cell_type": "code", - "execution_count": 61, + "execution_count": 59, "metadata": {}, "outputs": [], "source": [ @@ -1589,7 +1582,7 @@ }, { "cell_type": "code", - "execution_count": 62, + "execution_count": 60, "metadata": {}, "outputs": [], "source": [ @@ -1627,7 +1620,7 @@ }, { "cell_type": "code", - "execution_count": 63, + "execution_count": 61, "metadata": {}, "outputs": [], "source": [ @@ -1647,7 +1640,7 @@ }, { "cell_type": "code", - "execution_count": 64, + "execution_count": 62, "metadata": {}, "outputs": [], "source": [ @@ -1665,7 +1658,7 @@ }, { "cell_type": "code", - "execution_count": 65, + "execution_count": 63, "metadata": {}, "outputs": [], "source": [ @@ -1685,7 +1678,7 @@ }, { "cell_type": "code", - "execution_count": 66, + "execution_count": 64, "metadata": {}, "outputs": [], "source": [ @@ -1704,7 +1697,7 @@ }, { "cell_type": "code", - "execution_count": 67, + "execution_count": 65, "metadata": {}, "outputs": [], "source": [ @@ -1724,7 +1717,7 @@ }, { "cell_type": "code", - "execution_count": 68, + "execution_count": 66, "metadata": {}, "outputs": [], "source": [ @@ -1740,7 +1733,7 @@ }, { "cell_type": "code", - "execution_count": 69, + "execution_count": 67, "metadata": {}, "outputs": [], "source": [ @@ -1755,7 +1748,7 @@ }, { "cell_type": "code", - "execution_count": 70, + "execution_count": 68, "metadata": {}, "outputs": [], "source": [ @@ -1771,7 +1764,7 @@ }, { "cell_type": "code", - "execution_count": 71, + "execution_count": 69, "metadata": {}, "outputs": [], "source": [ @@ -1789,7 +1782,7 @@ }, { "cell_type": "code", - "execution_count": 72, + "execution_count": 70, "metadata": {}, "outputs": [], "source": [ diff --git a/07_ensemble_learning_and_random_forests.ipynb b/07_ensemble_learning_and_random_forests.ipynb index a47f0a5..311b6a6 100644 --- a/07_ensemble_learning_and_random_forests.ipynb +++ b/07_ensemble_learning_and_random_forests.ipynb @@ -80,7 +80,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -91,7 +91,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -109,7 +109,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -129,7 +129,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -149,7 +149,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -158,7 +158,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ @@ -179,7 +179,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ @@ -195,7 +195,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -216,7 +216,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ @@ -232,7 +232,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 11, "metadata": {}, "outputs": [], "source": [ @@ -242,7 +242,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 12, "metadata": {}, "outputs": [], "source": [ @@ -254,7 +254,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 13, "metadata": {}, "outputs": [], "source": [ @@ -280,7 +280,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 14, "metadata": {}, "outputs": [], "source": [ @@ -304,7 +304,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 15, "metadata": {}, "outputs": [], "source": [ @@ -315,7 +315,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 16, "metadata": {}, "outputs": [], "source": [ @@ -325,7 +325,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 17, "metadata": {}, "outputs": [], "source": [ @@ -339,7 +339,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 18, "metadata": {}, "outputs": [], "source": [ @@ -348,7 +348,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 19, "metadata": {}, "outputs": [], "source": [ @@ -362,7 +362,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 20, "metadata": {}, "outputs": [], "source": [ @@ -371,7 +371,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 21, "metadata": {}, "outputs": [], "source": [ @@ -395,7 +395,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 22, "metadata": {}, "outputs": [], "source": [ @@ -408,7 +408,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 23, "metadata": {}, "outputs": [], "source": [ @@ -417,7 +417,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 24, "metadata": {}, "outputs": [], "source": [ @@ -435,7 +435,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 25, "metadata": {}, "outputs": [], "source": [ @@ -447,7 +447,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 26, "metadata": {}, "outputs": [], "source": [ @@ -457,7 +457,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 27, "metadata": {}, "outputs": [], "source": [ @@ -470,7 +470,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 28, "metadata": {}, "outputs": [], "source": [ @@ -492,7 +492,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 29, "metadata": {}, "outputs": [], "source": [ @@ -506,7 +506,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 30, "metadata": {}, "outputs": [], "source": [ @@ -515,7 +515,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 31, "metadata": {}, "outputs": [], "source": [ @@ -545,7 +545,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 32, "metadata": {}, "outputs": [], "source": [ @@ -561,7 +561,7 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 33, "metadata": {}, "outputs": [], "source": [ @@ -572,7 +572,7 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 34, "metadata": {}, "outputs": [], "source": [ @@ -584,7 +584,7 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 35, "metadata": {}, "outputs": [], "source": [ @@ -595,7 +595,7 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 36, "metadata": {}, "outputs": [], "source": [ @@ -606,7 +606,7 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 37, "metadata": {}, "outputs": [], "source": [ @@ -615,7 +615,7 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 38, "metadata": {}, "outputs": [], "source": [ @@ -624,7 +624,7 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 39, "metadata": {}, "outputs": [], "source": [ @@ -633,7 +633,7 @@ }, { "cell_type": "code", - "execution_count": 41, + "execution_count": 40, "metadata": {}, "outputs": [], "source": [ @@ -682,7 +682,7 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": 41, "metadata": {}, "outputs": [], "source": [ @@ -694,7 +694,7 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": 42, "metadata": {}, "outputs": [], "source": [ @@ -704,7 +704,7 @@ }, { "cell_type": "code", - "execution_count": 44, + "execution_count": 43, "metadata": {}, "outputs": [], "source": [ @@ -731,7 +731,7 @@ }, { "cell_type": "code", - "execution_count": 45, + "execution_count": 44, "metadata": {}, "outputs": [], "source": [ @@ -754,7 +754,7 @@ }, { "cell_type": "code", - "execution_count": 46, + "execution_count": 45, "metadata": {}, "outputs": [], "source": [ @@ -763,7 +763,7 @@ }, { "cell_type": "code", - "execution_count": 47, + "execution_count": 46, "metadata": {}, "outputs": [], "source": [ @@ -789,7 +789,7 @@ }, { "cell_type": "code", - "execution_count": 48, + "execution_count": 47, "metadata": {}, "outputs": [], "source": [ @@ -813,7 +813,7 @@ }, { "cell_type": "code", - "execution_count": 49, + "execution_count": 48, "metadata": {}, "outputs": [], "source": [ @@ -822,7 +822,7 @@ }, { "cell_type": "code", - "execution_count": 50, + "execution_count": 49, "metadata": {}, "outputs": [], "source": [ @@ -838,7 +838,7 @@ }, { "cell_type": "code", - "execution_count": 51, + "execution_count": 50, "metadata": {}, "outputs": [], "source": [ @@ -851,7 +851,7 @@ }, { "cell_type": "code", - "execution_count": 52, + "execution_count": 51, "metadata": {}, "outputs": [], "source": [ @@ -865,7 +865,7 @@ }, { "cell_type": "code", - "execution_count": 53, + "execution_count": 52, "metadata": {}, "outputs": [], "source": [ @@ -879,7 +879,7 @@ }, { "cell_type": "code", - "execution_count": 54, + "execution_count": 53, "metadata": {}, "outputs": [], "source": [ @@ -888,7 +888,7 @@ }, { "cell_type": "code", - "execution_count": 55, + "execution_count": 54, "metadata": {}, "outputs": [], "source": [ @@ -941,7 +941,7 @@ }, { "cell_type": "code", - "execution_count": 56, + "execution_count": 55, "metadata": {}, "outputs": [], "source": [ @@ -950,7 +950,7 @@ }, { "cell_type": "code", - "execution_count": 57, + "execution_count": 56, "metadata": {}, "outputs": [], "source": [ @@ -969,7 +969,7 @@ }, { "cell_type": "code", - "execution_count": 58, + "execution_count": 57, "metadata": {}, "outputs": [], "source": [ @@ -980,7 +980,7 @@ }, { "cell_type": "code", - "execution_count": 59, + "execution_count": 58, "metadata": {}, "outputs": [], "source": [ @@ -992,7 +992,7 @@ }, { "cell_type": "code", - "execution_count": 60, + "execution_count": 59, "metadata": {}, "outputs": [], "source": [ @@ -1004,7 +1004,7 @@ }, { "cell_type": "code", - "execution_count": 61, + "execution_count": 60, "metadata": {}, "outputs": [], "source": [ @@ -1027,7 +1027,7 @@ }, { "cell_type": "code", - "execution_count": 62, + "execution_count": 61, "metadata": {}, "outputs": [], "source": [ @@ -1036,7 +1036,7 @@ }, { "cell_type": "code", - "execution_count": 63, + "execution_count": 62, "metadata": {}, "outputs": [], "source": [ @@ -1050,7 +1050,7 @@ }, { "cell_type": "code", - "execution_count": 64, + "execution_count": 63, "metadata": {}, "outputs": [], "source": [ @@ -1059,7 +1059,7 @@ }, { "cell_type": "code", - "execution_count": 65, + "execution_count": 64, "metadata": {}, "outputs": [], "source": [ @@ -1068,7 +1068,7 @@ }, { "cell_type": "code", - "execution_count": 66, + "execution_count": 65, "metadata": {}, "outputs": [], "source": [ @@ -1077,7 +1077,7 @@ }, { "cell_type": "code", - "execution_count": 67, + "execution_count": 66, "metadata": {}, "outputs": [], "source": [ @@ -1093,7 +1093,7 @@ }, { "cell_type": "code", - "execution_count": 68, + "execution_count": 67, "metadata": {}, "outputs": [], "source": [ @@ -1109,7 +1109,7 @@ }, { "cell_type": "code", - "execution_count": 69, + "execution_count": 68, "metadata": {}, "outputs": [], "source": [ @@ -1125,7 +1125,7 @@ }, { "cell_type": "code", - "execution_count": 70, + "execution_count": 69, "metadata": {}, "outputs": [], "source": [ @@ -1141,7 +1141,7 @@ }, { "cell_type": "code", - "execution_count": 71, + "execution_count": 70, "metadata": {}, "outputs": [], "source": [ @@ -1157,7 +1157,7 @@ }, { "cell_type": "code", - "execution_count": 72, + "execution_count": 71, "metadata": {}, "outputs": [], "source": [ @@ -1173,7 +1173,7 @@ }, { "cell_type": "code", - "execution_count": 73, + "execution_count": 72, "metadata": {}, "outputs": [], "source": [ @@ -1182,7 +1182,7 @@ }, { "cell_type": "code", - "execution_count": 74, + "execution_count": 73, "metadata": {}, "outputs": [], "source": [ @@ -1205,7 +1205,7 @@ }, { "cell_type": "code", - "execution_count": 75, + "execution_count": 74, "metadata": {}, "outputs": [], "source": [ @@ -1214,7 +1214,7 @@ }, { "cell_type": "code", - "execution_count": 76, + "execution_count": 75, "metadata": {}, "outputs": [], "source": [ @@ -1244,7 +1244,7 @@ }, { "cell_type": "code", - "execution_count": 77, + "execution_count": 76, "metadata": {}, "outputs": [], "source": [ @@ -1256,7 +1256,7 @@ }, { "cell_type": "code", - "execution_count": 78, + "execution_count": 77, "metadata": {}, "outputs": [], "source": [ @@ -1265,7 +1265,7 @@ }, { "cell_type": "code", - "execution_count": 79, + "execution_count": 78, "metadata": {}, "outputs": [], "source": [ @@ -1275,7 +1275,7 @@ }, { "cell_type": "code", - "execution_count": 80, + "execution_count": 79, "metadata": {}, "outputs": [], "source": [ @@ -1298,7 +1298,7 @@ }, { "cell_type": "code", - "execution_count": 81, + "execution_count": 80, "metadata": {}, "outputs": [], "source": [ @@ -1310,7 +1310,7 @@ }, { "cell_type": "code", - "execution_count": 82, + "execution_count": 81, "metadata": {}, "outputs": [], "source": [ @@ -1319,7 +1319,7 @@ }, { "cell_type": "code", - "execution_count": 83, + "execution_count": 82, "metadata": {}, "outputs": [], "source": [ @@ -1328,7 +1328,7 @@ }, { "cell_type": "code", - "execution_count": 84, + "execution_count": 83, "metadata": {}, "outputs": [], "source": [ diff --git a/08_dimensionality_reduction.ipynb b/08_dimensionality_reduction.ipynb index 0a5634a..e515aef 100644 --- a/08_dimensionality_reduction.ipynb +++ b/08_dimensionality_reduction.ipynb @@ -80,7 +80,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -105,7 +105,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -117,7 +117,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -129,7 +129,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -138,7 +138,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -148,7 +148,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ @@ -171,7 +171,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ @@ -183,7 +183,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -192,7 +192,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ @@ -210,7 +210,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 11, "metadata": {}, "outputs": [], "source": [ @@ -226,7 +226,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 12, "metadata": {}, "outputs": [], "source": [ @@ -242,7 +242,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 13, "metadata": {}, "outputs": [], "source": [ @@ -258,7 +258,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 14, "metadata": {}, "outputs": [], "source": [ @@ -274,7 +274,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 15, "metadata": {}, "outputs": [], "source": [ @@ -290,7 +290,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 16, "metadata": {}, "outputs": [], "source": [ @@ -306,7 +306,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 17, "metadata": {}, "outputs": [], "source": [ @@ -322,7 +322,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 18, "metadata": {}, "outputs": [], "source": [ @@ -345,7 +345,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 19, "metadata": {}, "outputs": [], "source": [ @@ -368,7 +368,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 20, "metadata": {}, "outputs": [], "source": [ @@ -384,7 +384,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 21, "metadata": {}, "outputs": [], "source": [ @@ -407,7 +407,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 22, "metadata": {}, "outputs": [], "source": [ @@ -435,7 +435,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 23, "metadata": {}, "outputs": [], "source": [ @@ -459,7 +459,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 24, "metadata": {}, "outputs": [], "source": [ @@ -509,7 +509,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 25, "metadata": {}, "outputs": [], "source": [ @@ -538,7 +538,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 26, "metadata": {}, "outputs": [], "source": [ @@ -548,7 +548,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 27, "metadata": {}, "outputs": [], "source": [ @@ -572,7 +572,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 28, "metadata": {}, "outputs": [], "source": [ @@ -597,7 +597,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 29, "metadata": {}, "outputs": [], "source": [ @@ -685,7 +685,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 30, "metadata": {}, "outputs": [], "source": [ @@ -758,7 +758,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 31, "metadata": {}, "outputs": [], "source": [ @@ -770,7 +770,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 32, "metadata": {}, "outputs": [], "source": [ @@ -784,7 +784,7 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 33, "metadata": {}, "outputs": [], "source": [ @@ -796,7 +796,7 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 34, "metadata": {}, "outputs": [], "source": [ @@ -805,7 +805,7 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 35, "metadata": {}, "outputs": [], "source": [ @@ -1516,7 +1516,7 @@ }, { "cell_type": "code", - "execution_count": 72, + "execution_count": 73, "metadata": {}, "outputs": [], "source": [ @@ -1536,7 +1536,7 @@ }, { "cell_type": "code", - "execution_count": 73, + "execution_count": 74, "metadata": {}, "outputs": [], "source": [ @@ -1547,7 +1547,7 @@ }, { "cell_type": "code", - "execution_count": 74, + "execution_count": 75, "metadata": {}, "outputs": [], "source": [ @@ -1560,7 +1560,7 @@ }, { "cell_type": "code", - "execution_count": 75, + "execution_count": 76, "metadata": {}, "outputs": [], "source": [ @@ -1569,7 +1569,7 @@ }, { "cell_type": "code", - "execution_count": 76, + "execution_count": 77, "metadata": {}, "outputs": [], "source": [ @@ -1588,7 +1588,7 @@ }, { "cell_type": "code", - "execution_count": 77, + "execution_count": 78, "metadata": {}, "outputs": [], "source": [ @@ -1607,7 +1607,7 @@ }, { "cell_type": "code", - "execution_count": 78, + "execution_count": 79, "metadata": {}, "outputs": [], "source": [ @@ -1619,7 +1619,7 @@ }, { "cell_type": "code", - "execution_count": 79, + "execution_count": 80, "metadata": {}, "outputs": [], "source": [ @@ -1642,7 +1642,7 @@ }, { "cell_type": "code", - "execution_count": 80, + "execution_count": 81, "metadata": {}, "outputs": [], "source": [ @@ -1663,7 +1663,7 @@ }, { "cell_type": "code", - "execution_count": 81, + "execution_count": 82, "metadata": {}, "outputs": [], "source": [ @@ -1677,7 +1677,7 @@ }, { "cell_type": "code", - "execution_count": 82, + "execution_count": 83, "metadata": {}, "outputs": [], "source": [ @@ -1686,7 +1686,7 @@ }, { "cell_type": "code", - "execution_count": 83, + "execution_count": 84, "metadata": {}, "outputs": [], "source": [ @@ -1703,7 +1703,7 @@ }, { "cell_type": "code", - "execution_count": 84, + "execution_count": 85, "metadata": {}, "outputs": [], "source": [ @@ -1715,7 +1715,7 @@ }, { "cell_type": "code", - "execution_count": 85, + "execution_count": 86, "metadata": {}, "outputs": [], "source": [ @@ -1731,7 +1731,7 @@ }, { "cell_type": "code", - "execution_count": 86, + "execution_count": 87, "metadata": {}, "outputs": [], "source": [ @@ -1783,7 +1783,7 @@ }, { "cell_type": "code", - "execution_count": 87, + "execution_count": 88, "metadata": {}, "outputs": [], "source": [ @@ -1805,7 +1805,7 @@ }, { "cell_type": "code", - "execution_count": 88, + "execution_count": 89, "metadata": {}, "outputs": [], "source": [ @@ -1824,7 +1824,7 @@ }, { "cell_type": "code", - "execution_count": 89, + "execution_count": 90, "metadata": {}, "outputs": [], "source": [ @@ -1851,7 +1851,7 @@ }, { "cell_type": "code", - "execution_count": 90, + "execution_count": 91, "metadata": {}, "outputs": [], "source": [ @@ -1872,7 +1872,7 @@ }, { "cell_type": "code", - "execution_count": 91, + "execution_count": 92, "metadata": {}, "outputs": [], "source": [ @@ -1886,7 +1886,7 @@ }, { "cell_type": "code", - "execution_count": 92, + "execution_count": 93, "metadata": {}, "outputs": [], "source": [ @@ -1922,7 +1922,7 @@ }, { "cell_type": "code", - "execution_count": 93, + "execution_count": 94, "metadata": {}, "outputs": [], "source": [ @@ -1966,7 +1966,7 @@ }, { "cell_type": "code", - "execution_count": 94, + "execution_count": 95, "metadata": {}, "outputs": [], "source": [ @@ -1982,7 +1982,7 @@ }, { "cell_type": "code", - "execution_count": 95, + "execution_count": 96, "metadata": {}, "outputs": [], "source": [ @@ -1991,7 +1991,7 @@ }, { "cell_type": "code", - "execution_count": 96, + "execution_count": 97, "metadata": {}, "outputs": [], "source": [ @@ -2014,7 +2014,7 @@ }, { "cell_type": "code", - "execution_count": 97, + "execution_count": 98, "metadata": {}, "outputs": [], "source": [ @@ -2038,7 +2038,7 @@ }, { "cell_type": "code", - "execution_count": 98, + "execution_count": 99, "metadata": {}, "outputs": [], "source": [ @@ -2061,7 +2061,7 @@ }, { "cell_type": "code", - "execution_count": 99, + "execution_count": 100, "metadata": {}, "outputs": [], "source": [ @@ -2095,7 +2095,7 @@ }, { "cell_type": "code", - "execution_count": 100, + "execution_count": 101, "metadata": {}, "outputs": [], "source": [ @@ -2119,7 +2119,7 @@ }, { "cell_type": "code", - "execution_count": 101, + "execution_count": 102, "metadata": {}, "outputs": [], "source": [ @@ -2153,7 +2153,7 @@ }, { "cell_type": "code", - "execution_count": 102, + "execution_count": 103, "metadata": {}, "outputs": [], "source": [ @@ -2183,7 +2183,7 @@ }, { "cell_type": "code", - "execution_count": 103, + "execution_count": 104, "metadata": {}, "outputs": [], "source": [ @@ -2206,7 +2206,7 @@ }, { "cell_type": "code", - "execution_count": 104, + "execution_count": 105, "metadata": {}, "outputs": [], "source": [ diff --git a/10_neural_nets_with_keras.ipynb b/10_neural_nets_with_keras.ipynb index 4419f6a..6aca606 100644 --- a/10_neural_nets_with_keras.ipynb +++ b/10_neural_nets_with_keras.ipynb @@ -1423,7 +1423,7 @@ }, { "cell_type": "code", - "execution_count": 95, + "execution_count": 94, "metadata": {}, "outputs": [], "source": [ @@ -1442,7 +1442,7 @@ }, { "cell_type": "code", - "execution_count": 96, + "execution_count": 95, "metadata": {}, "outputs": [], "source": [ @@ -1451,7 +1451,7 @@ }, { "cell_type": "code", - "execution_count": 97, + "execution_count": 96, "metadata": {}, "outputs": [], "source": [ @@ -1467,7 +1467,7 @@ }, { "cell_type": "code", - "execution_count": 98, + "execution_count": 97, "metadata": {}, "outputs": [], "source": [ @@ -1477,7 +1477,7 @@ }, { "cell_type": "code", - "execution_count": 99, + "execution_count": 98, "metadata": {}, "outputs": [], "source": [ @@ -1495,7 +1495,7 @@ }, { "cell_type": "code", - "execution_count": 100, + "execution_count": 99, "metadata": {}, "outputs": [], "source": [ @@ -1504,7 +1504,7 @@ }, { "cell_type": "code", - "execution_count": 101, + "execution_count": 100, "metadata": {}, "outputs": [], "source": [ @@ -1515,7 +1515,7 @@ }, { "cell_type": "code", - "execution_count": 102, + "execution_count": 101, "metadata": {}, "outputs": [], "source": [ @@ -1524,7 +1524,7 @@ }, { "cell_type": "code", - "execution_count": 103, + "execution_count": 102, "metadata": {}, "outputs": [], "source": [ @@ -1533,7 +1533,7 @@ }, { "cell_type": "code", - "execution_count": 104, + "execution_count": 103, "metadata": {}, "outputs": [], "source": [ @@ -1543,7 +1543,7 @@ }, { "cell_type": "code", - "execution_count": 105, + "execution_count": 104, "metadata": {}, "outputs": [], "source": [ @@ -1564,7 +1564,7 @@ }, { "cell_type": "code", - "execution_count": 106, + "execution_count": 105, "metadata": {}, "outputs": [], "source": [ @@ -1573,7 +1573,7 @@ }, { "cell_type": "code", - "execution_count": 107, + "execution_count": 106, "metadata": {}, "outputs": [], "source": [ @@ -1582,7 +1582,7 @@ }, { "cell_type": "code", - "execution_count": 108, + "execution_count": 107, "metadata": {}, "outputs": [], "source": [ @@ -1591,7 +1591,7 @@ }, { "cell_type": "code", - "execution_count": 109, + "execution_count": 108, "metadata": {}, "outputs": [], "source": [ @@ -1600,7 +1600,7 @@ }, { "cell_type": "code", - "execution_count": 110, + "execution_count": 109, "metadata": {}, "outputs": [], "source": [ @@ -1610,7 +1610,7 @@ }, { "cell_type": "code", - "execution_count": 111, + "execution_count": 110, "metadata": {}, "outputs": [], "source": [ diff --git a/12_custom_models_with_tensorflow_2.ipynb b/12_custom_models_with_tensorflow_2.ipynb index 7f26747..895719d 100644 --- a/12_custom_models_with_tensorflow_2.ipynb +++ b/12_custom_models_with_tensorflow_2.ipynb @@ -1489,7 +1489,7 @@ }, { "cell_type": "code", - "execution_count": 118, + "execution_count": 117, "metadata": {}, "outputs": [], "source": [ @@ -1500,7 +1500,7 @@ }, { "cell_type": "code", - "execution_count": 119, + "execution_count": 118, "metadata": {}, "outputs": [], "source": [ @@ -1509,7 +1509,7 @@ }, { "cell_type": "code", - "execution_count": 120, + "execution_count": 119, "metadata": {}, "outputs": [], "source": [ @@ -1525,7 +1525,7 @@ }, { "cell_type": "code", - "execution_count": 121, + "execution_count": 120, "metadata": {}, "outputs": [], "source": [ @@ -1551,7 +1551,7 @@ }, { "cell_type": "code", - "execution_count": 122, + "execution_count": 121, "metadata": {}, "outputs": [], "source": [ @@ -1564,7 +1564,7 @@ }, { "cell_type": "code", - "execution_count": 123, + "execution_count": 122, "metadata": {}, "outputs": [], "source": [ @@ -1573,7 +1573,7 @@ }, { "cell_type": "code", - "execution_count": 124, + "execution_count": 123, "metadata": { "scrolled": true }, @@ -1585,7 +1585,7 @@ }, { "cell_type": "code", - "execution_count": 125, + "execution_count": 124, "metadata": {}, "outputs": [], "source": [ @@ -1594,7 +1594,7 @@ }, { "cell_type": "code", - "execution_count": 126, + "execution_count": 125, "metadata": {}, "outputs": [], "source": [ @@ -1603,7 +1603,7 @@ }, { "cell_type": "code", - "execution_count": 128, + "execution_count": 126, "metadata": {}, "outputs": [], "source": [ @@ -1613,7 +1613,7 @@ }, { "cell_type": "code", - "execution_count": 129, + "execution_count": 127, "metadata": {}, "outputs": [], "source": [ @@ -1622,7 +1622,7 @@ }, { "cell_type": "code", - "execution_count": 130, + "execution_count": 128, "metadata": { "scrolled": true }, @@ -1640,7 +1640,7 @@ }, { "cell_type": "code", - "execution_count": 131, + "execution_count": 129, "metadata": {}, "outputs": [], "source": [ @@ -1649,7 +1649,7 @@ }, { "cell_type": "code", - "execution_count": 132, + "execution_count": 130, "metadata": {}, "outputs": [], "source": [ @@ -1665,7 +1665,7 @@ }, { "cell_type": "code", - "execution_count": 133, + "execution_count": 131, "metadata": {}, "outputs": [], "source": [ @@ -1682,7 +1682,7 @@ }, { "cell_type": "code", - "execution_count": 134, + "execution_count": 132, "metadata": {}, "outputs": [], "source": [ @@ -1714,7 +1714,7 @@ }, { "cell_type": "code", - "execution_count": 135, + "execution_count": 133, "metadata": {}, "outputs": [], "source": [ @@ -1726,7 +1726,7 @@ }, { "cell_type": "code", - "execution_count": 136, + "execution_count": 134, "metadata": { "scrolled": false }, @@ -1740,7 +1740,7 @@ }, { "cell_type": "code", - "execution_count": 137, + "execution_count": 135, "metadata": {}, "outputs": [], "source": [ @@ -1749,7 +1749,7 @@ }, { "cell_type": "code", - "execution_count": 138, + "execution_count": 136, "metadata": {}, "outputs": [], "source": [ @@ -1759,7 +1759,7 @@ }, { "cell_type": "code", - "execution_count": 139, + "execution_count": 137, "metadata": {}, "outputs": [], "source": [ @@ -1775,7 +1775,7 @@ }, { "cell_type": "code", - "execution_count": 140, + "execution_count": 138, "metadata": {}, "outputs": [], "source": [ @@ -1793,7 +1793,7 @@ }, { "cell_type": "code", - "execution_count": 141, + "execution_count": 139, "metadata": {}, "outputs": [], "source": [ @@ -1817,7 +1817,7 @@ }, { "cell_type": "code", - "execution_count": 142, + "execution_count": 140, "metadata": {}, "outputs": [], "source": [ @@ -1836,7 +1836,7 @@ }, { "cell_type": "code", - "execution_count": 143, + "execution_count": 141, "metadata": {}, "outputs": [], "source": [ @@ -1845,7 +1845,7 @@ }, { "cell_type": "code", - "execution_count": 144, + "execution_count": 142, "metadata": {}, "outputs": [], "source": [ @@ -1872,7 +1872,7 @@ }, { "cell_type": "code", - "execution_count": 145, + "execution_count": 143, "metadata": {}, "outputs": [], "source": [ @@ -1901,7 +1901,7 @@ }, { "cell_type": "code", - "execution_count": 146, + "execution_count": 144, "metadata": {}, "outputs": [], "source": [ @@ -1914,7 +1914,7 @@ }, { "cell_type": "code", - "execution_count": 147, + "execution_count": 145, "metadata": {}, "outputs": [], "source": [ @@ -1936,7 +1936,7 @@ }, { "cell_type": "code", - "execution_count": 148, + "execution_count": 146, "metadata": {}, "outputs": [], "source": [ @@ -1951,7 +1951,7 @@ }, { "cell_type": "code", - "execution_count": 149, + "execution_count": 147, "metadata": {}, "outputs": [], "source": [ @@ -2012,7 +2012,7 @@ }, { "cell_type": "code", - "execution_count": 150, + "execution_count": 148, "metadata": {}, "outputs": [], "source": [ @@ -2041,7 +2041,7 @@ }, { "cell_type": "code", - "execution_count": 151, + "execution_count": 149, "metadata": {}, "outputs": [], "source": [ @@ -2061,7 +2061,7 @@ }, { "cell_type": "code", - "execution_count": 152, + "execution_count": 150, "metadata": {}, "outputs": [], "source": [ @@ -2071,7 +2071,7 @@ }, { "cell_type": "code", - "execution_count": 153, + "execution_count": 151, "metadata": {}, "outputs": [], "source": [ @@ -2082,7 +2082,7 @@ }, { "cell_type": "code", - "execution_count": 154, + "execution_count": 152, "metadata": {}, "outputs": [], "source": [ @@ -2091,7 +2091,7 @@ }, { "cell_type": "code", - "execution_count": 155, + "execution_count": 153, "metadata": {}, "outputs": [], "source": [ @@ -2104,7 +2104,7 @@ }, { "cell_type": "code", - "execution_count": 156, + "execution_count": 154, "metadata": {}, "outputs": [], "source": [ @@ -2113,7 +2113,7 @@ }, { "cell_type": "code", - "execution_count": 157, + "execution_count": 155, "metadata": {}, "outputs": [], "source": [ @@ -2129,7 +2129,7 @@ }, { "cell_type": "code", - "execution_count": 158, + "execution_count": 156, "metadata": {}, "outputs": [], "source": [ @@ -2143,7 +2143,7 @@ }, { "cell_type": "code", - "execution_count": 159, + "execution_count": 157, "metadata": {}, "outputs": [], "source": [ @@ -2152,7 +2152,7 @@ }, { "cell_type": "code", - "execution_count": 160, + "execution_count": 158, "metadata": {}, "outputs": [], "source": [ @@ -2163,6 +2163,29 @@ "gradients = tape.gradient(z, [c1, c2])" ] }, + { + "cell_type": "code", + "execution_count": 159, + "metadata": {}, + "outputs": [], + "source": [ + "gradients" + ] + }, + { + "cell_type": "code", + "execution_count": 160, + "metadata": {}, + "outputs": [], + "source": [ + "with tf.GradientTape() as tape:\n", + " tape.watch(c1)\n", + " tape.watch(c2)\n", + " z = f(c1, c2)\n", + "\n", + "gradients = tape.gradient(z, [c1, c2])" + ] + }, { "cell_type": "code", "execution_count": 161, @@ -2177,29 +2200,6 @@ "execution_count": 162, "metadata": {}, "outputs": [], - "source": [ - "with tf.GradientTape() as tape:\n", - " tape.watch(c1)\n", - " tape.watch(c2)\n", - " z = f(c1, c2)\n", - "\n", - "gradients = tape.gradient(z, [c1, c2])" - ] - }, - { - "cell_type": "code", - "execution_count": 163, - "metadata": {}, - "outputs": [], - "source": [ - "gradients" - ] - }, - { - "cell_type": "code", - "execution_count": 164, - "metadata": {}, - "outputs": [], "source": [ "with tf.GradientTape() as tape:\n", " z1 = f(w1, w2 + 2.)\n", @@ -2211,7 +2211,7 @@ }, { "cell_type": "code", - "execution_count": 165, + "execution_count": 163, "metadata": {}, "outputs": [], "source": [ @@ -2226,7 +2226,7 @@ }, { "cell_type": "code", - "execution_count": 166, + "execution_count": 164, "metadata": {}, "outputs": [], "source": [ @@ -2241,7 +2241,7 @@ }, { "cell_type": "code", - "execution_count": 167, + "execution_count": 165, "metadata": {}, "outputs": [], "source": [ @@ -2250,7 +2250,7 @@ }, { "cell_type": "code", - "execution_count": 168, + "execution_count": 166, "metadata": {}, "outputs": [], "source": [ @@ -2259,7 +2259,7 @@ }, { "cell_type": "code", - "execution_count": 169, + "execution_count": 167, "metadata": {}, "outputs": [], "source": [ @@ -2274,7 +2274,7 @@ }, { "cell_type": "code", - "execution_count": 170, + "execution_count": 168, "metadata": {}, "outputs": [], "source": [ @@ -2287,7 +2287,7 @@ }, { "cell_type": "code", - "execution_count": 171, + "execution_count": 169, "metadata": {}, "outputs": [], "source": [ @@ -2296,7 +2296,7 @@ }, { "cell_type": "code", - "execution_count": 172, + "execution_count": 170, "metadata": {}, "outputs": [], "source": [ @@ -2309,7 +2309,7 @@ }, { "cell_type": "code", - "execution_count": 173, + "execution_count": 171, "metadata": {}, "outputs": [], "source": [ @@ -2323,7 +2323,7 @@ }, { "cell_type": "code", - "execution_count": 174, + "execution_count": 172, "metadata": {}, "outputs": [], "source": [ @@ -2333,7 +2333,7 @@ }, { "cell_type": "code", - "execution_count": 175, + "execution_count": 173, "metadata": {}, "outputs": [], "source": [ @@ -2353,7 +2353,7 @@ }, { "cell_type": "code", - "execution_count": 176, + "execution_count": 174, "metadata": {}, "outputs": [], "source": [ @@ -2367,7 +2367,7 @@ }, { "cell_type": "code", - "execution_count": 177, + "execution_count": 175, "metadata": {}, "outputs": [], "source": [ @@ -2378,7 +2378,7 @@ }, { "cell_type": "code", - "execution_count": 178, + "execution_count": 176, "metadata": {}, "outputs": [], "source": [ @@ -2392,7 +2392,7 @@ }, { "cell_type": "code", - "execution_count": 179, + "execution_count": 177, "metadata": {}, "outputs": [], "source": [ @@ -2417,7 +2417,7 @@ }, { "cell_type": "code", - "execution_count": 180, + "execution_count": 178, "metadata": {}, "outputs": [], "source": [ @@ -2432,7 +2432,7 @@ }, { "cell_type": "code", - "execution_count": 181, + "execution_count": 179, "metadata": {}, "outputs": [], "source": [ @@ -2441,7 +2441,7 @@ }, { "cell_type": "code", - "execution_count": 182, + "execution_count": 180, "metadata": {}, "outputs": [], "source": [ @@ -2454,7 +2454,7 @@ }, { "cell_type": "code", - "execution_count": 183, + "execution_count": 181, "metadata": {}, "outputs": [], "source": [ @@ -2470,7 +2470,7 @@ }, { "cell_type": "code", - "execution_count": 184, + "execution_count": 182, "metadata": {}, "outputs": [], "source": [ @@ -2485,7 +2485,7 @@ }, { "cell_type": "code", - "execution_count": 185, + "execution_count": 183, "metadata": {}, "outputs": [], "source": [ @@ -2513,7 +2513,7 @@ }, { "cell_type": "code", - "execution_count": 186, + "execution_count": 184, "metadata": {}, "outputs": [], "source": [ @@ -2556,7 +2556,7 @@ }, { "cell_type": "code", - "execution_count": 187, + "execution_count": 185, "metadata": {}, "outputs": [], "source": [ @@ -2566,7 +2566,7 @@ }, { "cell_type": "code", - "execution_count": 188, + "execution_count": 186, "metadata": {}, "outputs": [], "source": [ @@ -2575,7 +2575,7 @@ }, { "cell_type": "code", - "execution_count": 189, + "execution_count": 187, "metadata": {}, "outputs": [], "source": [ @@ -2584,7 +2584,7 @@ }, { "cell_type": "code", - "execution_count": 190, + "execution_count": 188, "metadata": {}, "outputs": [], "source": [ @@ -2594,7 +2594,7 @@ }, { "cell_type": "code", - "execution_count": 191, + "execution_count": 189, "metadata": {}, "outputs": [], "source": [ @@ -2603,7 +2603,7 @@ }, { "cell_type": "code", - "execution_count": 192, + "execution_count": 190, "metadata": {}, "outputs": [], "source": [ @@ -2619,7 +2619,7 @@ }, { "cell_type": "code", - "execution_count": 193, + "execution_count": 191, "metadata": {}, "outputs": [], "source": [ @@ -2629,7 +2629,7 @@ }, { "cell_type": "code", - "execution_count": 194, + "execution_count": 192, "metadata": {}, "outputs": [], "source": [ @@ -2638,7 +2638,7 @@ }, { "cell_type": "code", - "execution_count": 195, + "execution_count": 193, "metadata": {}, "outputs": [], "source": [ @@ -2654,7 +2654,7 @@ }, { "cell_type": "code", - "execution_count": 196, + "execution_count": 194, "metadata": {}, "outputs": [], "source": [ @@ -2663,7 +2663,7 @@ }, { "cell_type": "code", - "execution_count": 197, + "execution_count": 195, "metadata": {}, "outputs": [], "source": [ @@ -2673,7 +2673,7 @@ }, { "cell_type": "code", - "execution_count": 198, + "execution_count": 196, "metadata": {}, "outputs": [], "source": [ @@ -2683,7 +2683,7 @@ }, { "cell_type": "code", - "execution_count": 199, + "execution_count": 197, "metadata": {}, "outputs": [], "source": [ @@ -2692,7 +2692,7 @@ }, { "cell_type": "code", - "execution_count": 200, + "execution_count": 198, "metadata": {}, "outputs": [], "source": [ @@ -2701,7 +2701,7 @@ }, { "cell_type": "code", - "execution_count": 201, + "execution_count": 199, "metadata": {}, "outputs": [], "source": [ @@ -2710,7 +2710,7 @@ }, { "cell_type": "code", - "execution_count": 202, + "execution_count": 200, "metadata": {}, "outputs": [], "source": [ @@ -2726,7 +2726,7 @@ }, { "cell_type": "code", - "execution_count": 203, + "execution_count": 201, "metadata": {}, "outputs": [], "source": [ @@ -2738,7 +2738,7 @@ }, { "cell_type": "code", - "execution_count": 204, + "execution_count": 202, "metadata": {}, "outputs": [], "source": [ @@ -2747,7 +2747,7 @@ }, { "cell_type": "code", - "execution_count": 205, + "execution_count": 203, "metadata": {}, "outputs": [], "source": [ @@ -2756,7 +2756,7 @@ }, { "cell_type": "code", - "execution_count": 206, + "execution_count": 204, "metadata": {}, "outputs": [], "source": [ @@ -2776,7 +2776,7 @@ }, { "cell_type": "code", - "execution_count": 207, + "execution_count": 205, "metadata": {}, "outputs": [], "source": [ @@ -2788,7 +2788,7 @@ }, { "cell_type": "code", - "execution_count": 208, + "execution_count": 206, "metadata": {}, "outputs": [], "source": [ @@ -2800,7 +2800,7 @@ }, { "cell_type": "code", - "execution_count": 209, + "execution_count": 207, "metadata": {}, "outputs": [], "source": [ @@ -2827,7 +2827,7 @@ }, { "cell_type": "code", - "execution_count": 210, + "execution_count": 208, "metadata": {}, "outputs": [], "source": [ @@ -2840,7 +2840,7 @@ }, { "cell_type": "code", - "execution_count": 211, + "execution_count": 209, "metadata": {}, "outputs": [], "source": [ @@ -2849,7 +2849,7 @@ }, { "cell_type": "code", - "execution_count": 212, + "execution_count": 210, "metadata": {}, "outputs": [], "source": [ @@ -2865,7 +2865,7 @@ }, { "cell_type": "code", - "execution_count": 213, + "execution_count": 211, "metadata": {}, "outputs": [], "source": [ @@ -2879,7 +2879,7 @@ }, { "cell_type": "code", - "execution_count": 214, + "execution_count": 212, "metadata": {}, "outputs": [], "source": [ @@ -2888,7 +2888,7 @@ }, { "cell_type": "code", - "execution_count": 215, + "execution_count": 213, "metadata": {}, "outputs": [], "source": [ @@ -2904,7 +2904,7 @@ }, { "cell_type": "code", - "execution_count": 216, + "execution_count": 214, "metadata": {}, "outputs": [], "source": [ @@ -2917,7 +2917,7 @@ }, { "cell_type": "code", - "execution_count": 217, + "execution_count": 215, "metadata": {}, "outputs": [], "source": [ @@ -2933,7 +2933,7 @@ }, { "cell_type": "code", - "execution_count": 218, + "execution_count": 216, "metadata": {}, "outputs": [], "source": [ @@ -2946,7 +2946,7 @@ }, { "cell_type": "code", - "execution_count": 219, + "execution_count": 217, "metadata": {}, "outputs": [], "source": [ @@ -2956,7 +2956,7 @@ }, { "cell_type": "code", - "execution_count": 220, + "execution_count": 218, "metadata": {}, "outputs": [], "source": [ @@ -2966,7 +2966,7 @@ }, { "cell_type": "code", - "execution_count": 221, + "execution_count": 219, "metadata": {}, "outputs": [], "source": [ @@ -2979,7 +2979,7 @@ }, { "cell_type": "code", - "execution_count": 222, + "execution_count": 220, "metadata": {}, "outputs": [], "source": [ @@ -2989,7 +2989,7 @@ }, { "cell_type": "code", - "execution_count": 223, + "execution_count": 221, "metadata": {}, "outputs": [], "source": [ @@ -2999,7 +2999,7 @@ }, { "cell_type": "code", - "execution_count": 224, + "execution_count": 222, "metadata": {}, "outputs": [], "source": [ @@ -3014,7 +3014,7 @@ }, { "cell_type": "code", - "execution_count": 225, + "execution_count": 223, "metadata": {}, "outputs": [], "source": [ @@ -3025,7 +3025,7 @@ }, { "cell_type": "code", - "execution_count": 226, + "execution_count": 224, "metadata": { "scrolled": true }, @@ -3044,7 +3044,7 @@ }, { "cell_type": "code", - "execution_count": 227, + "execution_count": 225, "metadata": {}, "outputs": [], "source": [ @@ -3058,7 +3058,7 @@ }, { "cell_type": "code", - "execution_count": 228, + "execution_count": 226, "metadata": {}, "outputs": [], "source": [ @@ -3082,7 +3082,7 @@ }, { "cell_type": "code", - "execution_count": 229, + "execution_count": 227, "metadata": {}, "outputs": [], "source": [ @@ -3094,7 +3094,7 @@ }, { "cell_type": "code", - "execution_count": 230, + "execution_count": 228, "metadata": {}, "outputs": [], "source": [ @@ -3106,7 +3106,7 @@ }, { "cell_type": "code", - "execution_count": 231, + "execution_count": 229, "metadata": {}, "outputs": [], "source": [ @@ -3135,7 +3135,7 @@ }, { "cell_type": "code", - "execution_count": 232, + "execution_count": 230, "metadata": {}, "outputs": [], "source": [ @@ -3160,7 +3160,7 @@ }, { "cell_type": "code", - "execution_count": 233, + "execution_count": 231, "metadata": {}, "outputs": [], "source": [ @@ -3169,7 +3169,7 @@ }, { "cell_type": "code", - "execution_count": 234, + "execution_count": 232, "metadata": {}, "outputs": [], "source": [ @@ -3187,7 +3187,7 @@ }, { "cell_type": "code", - "execution_count": 235, + "execution_count": 233, "metadata": {}, "outputs": [], "source": [ @@ -3196,7 +3196,7 @@ }, { "cell_type": "code", - "execution_count": 236, + "execution_count": 234, "metadata": {}, "outputs": [], "source": [ @@ -3212,7 +3212,7 @@ }, { "cell_type": "code", - "execution_count": 237, + "execution_count": 235, "metadata": {}, "outputs": [], "source": [ @@ -3230,7 +3230,7 @@ }, { "cell_type": "code", - "execution_count": 238, + "execution_count": 236, "metadata": {}, "outputs": [], "source": [ @@ -3239,7 +3239,7 @@ }, { "cell_type": "code", - "execution_count": 239, + "execution_count": 237, "metadata": {}, "outputs": [], "source": [ @@ -3248,7 +3248,7 @@ }, { "cell_type": "code", - "execution_count": 240, + "execution_count": 238, "metadata": {}, "outputs": [], "source": [ @@ -3273,7 +3273,7 @@ }, { "cell_type": "code", - "execution_count": 241, + "execution_count": 239, "metadata": {}, "outputs": [], "source": [ @@ -3319,7 +3319,7 @@ }, { "cell_type": "code", - "execution_count": 242, + "execution_count": 240, "metadata": { "scrolled": false }, diff --git a/13_loading_and_preprocessing_data.ipynb b/13_loading_and_preprocessing_data.ipynb index c84fc19..901c074 100644 --- a/13_loading_and_preprocessing_data.ipynb +++ b/13_loading_and_preprocessing_data.ipynb @@ -1286,7 +1286,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 80, "metadata": {}, "outputs": [], "source": [ @@ -1634,7 +1634,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 106, "metadata": {}, "outputs": [], "source": [ diff --git a/15_recurrent_neural_networks.ipynb b/15_recurrent_neural_networks.ipynb new file mode 100644 index 0000000..3c5d541 --- /dev/null +++ b/15_recurrent_neural_networks.ipynb @@ -0,0 +1,2317 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Chapter 15 – Recurrent Neural Networks**" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "_This notebook contains all the sample code in chapter 15._" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Setup" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "First, let's import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures. We also check that Python 3.5 or later is installed (although Python 2.x may work, it is deprecated so we strongly recommend you use Python 3 instead), as well as Scikit-Learn ≥0.20 and TensorFlow ≥2.0-preview." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "# Python ≥3.5 is required\n", + "import sys\n", + "assert sys.version_info >= (3, 5)\n", + "\n", + "# Scikit-Learn ≥0.20 is required\n", + "import sklearn\n", + "assert sklearn.__version__ >= \"0.20\"\n", + "\n", + "# TensorFlow ≥2.0-preview is required\n", + "import tensorflow as tf\n", + "from tensorflow import keras\n", + "assert tf.__version__ >= \"2.0\"\n", + "\n", + "# Common imports\n", + "import numpy as np\n", + "import os\n", + "\n", + "# to make this notebook's output stable across runs\n", + "np.random.seed(42)\n", + "tf.random.set_seed(42)\n", + "\n", + "# To plot pretty figures\n", + "%matplotlib inline\n", + "import matplotlib as mpl\n", + "import matplotlib.pyplot as plt\n", + "mpl.rc('axes', labelsize=14)\n", + "mpl.rc('xtick', labelsize=12)\n", + "mpl.rc('ytick', labelsize=12)\n", + "\n", + "# Where to save the figures\n", + "PROJECT_ROOT_DIR = \".\"\n", + "CHAPTER_ID = \"rnn\"\n", + "IMAGES_PATH = os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID)\n", + "os.makedirs(IMAGES_PATH, exist_ok=True)\n", + "\n", + "def save_fig(fig_id, tight_layout=True, fig_extension=\"png\", resolution=300):\n", + " path = os.path.join(IMAGES_PATH, fig_id + \".\" + fig_extension)\n", + " print(\"Saving figure\", fig_id)\n", + " if tight_layout:\n", + " plt.tight_layout()\n", + " plt.savefig(path, format=fig_extension, dpi=resolution)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Basic RNNs" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Generate the Dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "def generate_time_series(batch_size, n_steps):\n", + " freq1, freq2, offsets1, offsets2 = np.random.rand(4, batch_size, 1)\n", + " time = np.linspace(0, 1, n_steps)\n", + " series = 0.5 * np.sin((time - offsets1) * (freq1 * 10 + 10)) # wave 1\n", + " series += 0.2 * np.sin((time - offsets2) * (freq2 * 20 + 20)) # + wave 2\n", + " series += 0.1 * (np.random.rand(batch_size, n_steps) - 0.5) # + noise\n", + " return series[..., np.newaxis].astype(np.float32)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "np.random.seed(42)\n", + "\n", + "n_steps = 50\n", + "series = generate_time_series(10000, n_steps + 1)\n", + "X_train, y_train = series[:7000, :n_steps], series[:7000, -1]\n", + "X_valid, y_valid = series[7000:9000, :n_steps], series[7000:9000, -1]\n", + "X_test, y_test = series[9000:, :n_steps], series[9000:, -1]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "X_train.shape, y_train.shape" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "def plot_series(series, y=None, y_pred=None, x_label=\"$t$\", y_label=\"$x(t)$\"):\n", + " plt.plot(series, \".-\")\n", + " if y is not None:\n", + " plt.plot(n_steps, y, \"bx\", markersize=10)\n", + " if y_pred is not None:\n", + " plt.plot(n_steps, y_pred, \"ro\")\n", + " plt.grid(True)\n", + " if x_label:\n", + " plt.xlabel(x_label, fontsize=16)\n", + " if y_label:\n", + " plt.ylabel(y_label, fontsize=16, rotation=0)\n", + " plt.hlines(0, 0, 100, linewidth=1)\n", + " plt.axis([0, n_steps + 1, -1, 1])\n", + "\n", + "fig, axes = plt.subplots(nrows=1, ncols=3, sharey=True, figsize=(12, 4))\n", + "for col in range(3):\n", + " plt.sca(axes[col])\n", + " plot_series(X_valid[col, :, 0], y_valid[col, 0],\n", + " y_label=(\"$x(t)$\" if col==0 else None))\n", + "save_fig(\"time_series_plot\")\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Computing Some Baselines" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Naive predictions (just predict the last observed value):" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "y_pred = X_valid[:, -1]\n", + "np.mean(keras.losses.mean_squared_error(y_valid, y_pred))" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "plot_series(X_valid[0, :, 0], y_valid[0, 0], y_pred[0, 0])\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Linear predictions:" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "np.random.seed(42)\n", + "tf.random.set_seed(42)\n", + "\n", + "model = keras.models.Sequential([\n", + " keras.layers.Flatten(input_shape=[50, 1]),\n", + " keras.layers.Dense(1)\n", + "])\n", + "\n", + "model.compile(loss=\"mse\", optimizer=\"adam\")\n", + "history = model.fit(X_train, y_train, epochs=20,\n", + " validation_data=(X_valid, y_valid))" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "model.evaluate(X_valid, y_valid)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "def plot_learning_curves(loss, val_loss):\n", + " plt.plot(np.arange(len(loss)) + 0.5, loss, \"b.-\", label=\"Training loss\")\n", + " plt.plot(np.arange(len(val_loss)) + 1, val_loss, \"r.-\", label=\"Validation loss\")\n", + " plt.gca().xaxis.set_major_locator(mpl.ticker.MaxNLocator(integer=True))\n", + " plt.axis([1, 20, 0, 0.05])\n", + " plt.legend(fontsize=14)\n", + " plt.xlabel(\"Epochs\")\n", + " plt.ylabel(\"Loss\")\n", + " plt.grid(True)\n", + "\n", + "plot_learning_curves(history.history[\"loss\"], history.history[\"val_loss\"])\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "y_pred = model.predict(X_valid)\n", + "plot_series(X_valid[0, :, 0], y_valid[0, 0], y_pred[0, 0])\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Using a Simple RNN" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "np.random.seed(42)\n", + "tf.random.set_seed(42)\n", + "\n", + "model = keras.models.Sequential([\n", + " keras.layers.SimpleRNN(1, input_shape=[None, 1])\n", + "])\n", + "\n", + "optimizer = keras.optimizers.Adam(lr=0.005)\n", + "model.compile(loss=\"mse\", optimizer=optimizer)\n", + "history = model.fit(X_train, y_train, epochs=20,\n", + " validation_data=(X_valid, y_valid))" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "model.evaluate(X_valid, y_valid)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "plot_learning_curves(history.history[\"loss\"], history.history[\"val_loss\"])\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "y_pred = model.predict(X_valid)\n", + "plot_series(X_valid[0, :, 0], y_valid[0, 0], y_pred[0, 0])\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Deep RNNs" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "np.random.seed(42)\n", + "tf.random.set_seed(42)\n", + "\n", + "model = keras.models.Sequential([\n", + " keras.layers.SimpleRNN(20, return_sequences=True, input_shape=[None, 1]),\n", + " keras.layers.SimpleRNN(20, return_sequences=True),\n", + " keras.layers.SimpleRNN(1)\n", + "])\n", + "\n", + "model.compile(loss=\"mse\", optimizer=\"adam\")\n", + "history = model.fit(X_train, y_train, epochs=20,\n", + " validation_data=(X_valid, y_valid))" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "model.evaluate(X_valid, y_valid)" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "plot_learning_curves(history.history[\"loss\"], history.history[\"val_loss\"])\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [], + "source": [ + "y_pred = model.predict(X_valid)\n", + "plot_series(X_valid[0, :, 0], y_valid[0, 0], y_pred[0, 0])\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Make the second `SimpleRNN` layer return only the last output:" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [], + "source": [ + "np.random.seed(42)\n", + "tf.random.set_seed(42)\n", + "\n", + "model = keras.models.Sequential([\n", + " keras.layers.SimpleRNN(20, return_sequences=True, input_shape=[None, 1]),\n", + " keras.layers.SimpleRNN(20),\n", + " keras.layers.Dense(1)\n", + "])\n", + "\n", + "model.compile(loss=\"mse\", optimizer=\"adam\")\n", + "history = model.fit(X_train, y_train, epochs=20,\n", + " validation_data=(X_valid, y_valid))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [], + "source": [ + "model.evaluate(X_valid, y_valid)" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [], + "source": [ + "plot_learning_curves(history.history[\"loss\"], history.history[\"val_loss\"])\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [], + "source": [ + "y_pred = model.predict(X_valid)\n", + "plot_series(X_valid[0, :, 0], y_valid[0, 0], y_pred[0, 0])\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Forecasting Several Steps Ahead" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [], + "source": [ + "np.random.seed(43) # not 42, as it would give the first series in the train set\n", + "\n", + "series = generate_time_series(1, n_steps + 10)\n", + "X_new, Y_new = series[:, :n_steps], series[:, n_steps:]\n", + "X = X_new\n", + "for step_ahead in range(10):\n", + " y_pred_one = model.predict(X[:, step_ahead:])[:, np.newaxis, :]\n", + " X = np.concatenate([X, y_pred_one], axis=1)\n", + "\n", + "Y_pred = X[:, n_steps:]" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [], + "source": [ + "Y_pred.shape" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [], + "source": [ + "def plot_multiple_forecasts(X, Y, Y_pred):\n", + " n_steps = X.shape[1]\n", + " ahead = Y.shape[1]\n", + " plot_series(X[0, :, 0])\n", + " plt.plot(np.arange(n_steps, n_steps + ahead), Y_pred[0, :, 0], \"ro-\")\n", + " plt.plot(np.arange(n_steps, n_steps + ahead), Y[0, :, 0], \"bx-\", markersize=10)\n", + " plt.axis([0, n_steps + ahead, -1, 1])\n", + "\n", + "plot_multiple_forecasts(X_new, Y_new, Y_pred)\n", + "save_fig(\"forecast_ahead_plot\")\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now let's create an RNN that predicts all 10 next values at once:" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [], + "source": [ + "np.random.seed(42)\n", + "\n", + "n_steps = 50\n", + "series = generate_time_series(10000, n_steps + 10)\n", + "X_train, Y_train = series[:7000, :n_steps], series[:7000, -10:]\n", + "X_valid, Y_valid = series[7000:9000, :n_steps], series[7000:9000, -10:]\n", + "X_test, Y_test = series[9000:, :n_steps], series[9000:, -10:]" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [], + "source": [ + "np.random.seed(42)\n", + "tf.random.set_seed(42)\n", + "\n", + "model = keras.models.Sequential([\n", + " keras.layers.SimpleRNN(20, return_sequences=True, input_shape=[None, 1]),\n", + " keras.layers.SimpleRNN(20, return_sequences=True),\n", + " keras.layers.TimeDistributed(keras.layers.Dense(1)),\n", + " keras.layers.Lambda(lambda Y_pred: Y_pred[:, -10:])\n", + "])\n", + "\n", + "model.compile(loss=\"mse\", optimizer=\"adam\")\n", + "history = model.fit(X_train, Y_train, epochs=20,\n", + " validation_data=(X_valid, Y_valid))" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [], + "source": [ + "np.random.seed(43)\n", + "\n", + "series = generate_time_series(1, 50 + 10)\n", + "X_new, Y_new = series[:, :50, :], series[:, -10:, :]\n", + "Y_pred = model.predict(X_new)[:, -10:, :]" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [], + "source": [ + "plot_multiple_forecasts(X_new, Y_new, Y_pred)\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now let's create an RNN that predicts the input sequence, shifted 10 steps into the future. That is, instead of just forecasting time steps 50 to 59 based on time steps 0 to 49, it will forecast time steps 10 to 59 based on time steps 0 to 49 (the time steps 10 to 49 are in the input, but the model is causal so at any time step it cannot see the future inputs):" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [], + "source": [ + "np.random.seed(42)\n", + "\n", + "n_steps = 50\n", + "series = generate_time_series(10000, n_steps + 10)\n", + "X_train, Y_train = series[:7000, :n_steps], series[:7000, 10:]\n", + "X_valid, Y_valid = series[7000:9000, :n_steps], series[7000:9000, 10:]\n", + "X_test, Y_test = series[9000:, :n_steps], series[9000:, 10:]" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, + "outputs": [], + "source": [ + "X_train.shape, Y_train.shape" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": {}, + "outputs": [], + "source": [ + "np.random.seed(42)\n", + "tf.random.set_seed(42)\n", + "\n", + "model = keras.models.Sequential([\n", + " keras.layers.SimpleRNN(20, return_sequences=True, input_shape=[None, 1]),\n", + " keras.layers.SimpleRNN(20, return_sequences=True),\n", + " keras.layers.TimeDistributed(keras.layers.Dense(1))\n", + "])\n", + "\n", + "def last_10_time_steps_mse(Y_true, Y_pred):\n", + " return keras.metrics.mean_squared_error(Y_true[:, -10:], Y_pred[:, -10:])\n", + "\n", + "model.compile(loss=\"mse\", optimizer=\"adam\", metrics=[last_10_time_steps_mse])\n", + "history = model.fit(X_train, Y_train, epochs=20,\n", + " validation_data=(X_valid, Y_valid))" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": {}, + "outputs": [], + "source": [ + "np.random.seed(43)\n", + "\n", + "series = generate_time_series(1, 50 + 10)\n", + "X_new, Y_new = series[:, :50, :], series[:, 50:, :]\n", + "Y_pred = model.predict(X_new)[:, -10:, :]" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": {}, + "outputs": [], + "source": [ + "plot_multiple_forecasts(X_new, Y_new, Y_pred)\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Deep RNN with Batch Norm" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": {}, + "outputs": [], + "source": [ + "np.random.seed(42)\n", + "tf.random.set_seed(42)\n", + "\n", + "\n", + "model = keras.models.Sequential([\n", + " keras.layers.SimpleRNN(20, return_sequences=True, input_shape=[None, 1]),\n", + " keras.layers.BatchNormalization(),\n", + " keras.layers.SimpleRNN(20, return_sequences=True),\n", + " keras.layers.BatchNormalization(),\n", + " keras.layers.TimeDistributed(keras.layers.Dense(1))\n", + "])\n", + "\n", + "model.compile(loss=\"mse\", optimizer=\"adam\", metrics=[last_10_time_steps_mse])\n", + "history = model.fit(X_train, Y_train, epochs=20,\n", + " validation_data=(X_valid, Y_valid))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Deep RNNs with Layer Norm" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "metadata": {}, + "outputs": [], + "source": [ + "from tensorflow.keras.layers.experimental import LayerNormalization" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": {}, + "outputs": [], + "source": [ + "class LNSimpleRNNCell(keras.layers.Layer):\n", + " def __init__(self, units, activation=\"tanh\", **kwargs):\n", + " super().__init__(**kwargs)\n", + " self.state_size = units\n", + " self.output_size = units\n", + " self.simple_rnn_cell = keras.layers.SimpleRNNCell(units,\n", + " activation=None)\n", + " self.layer_norm = LayerNormalization()\n", + " self.activation = keras.activations.get(activation)\n", + " def get_initial_state(self, inputs=None, batch_size=None, dtype=None):\n", + " if inputs is not None:\n", + " batch_size = tf.shape(inputs)[0]\n", + " dtype = inputs.dtype\n", + " return [tf.zeros([batch_size, self.state_size], dtype=dtype)]\n", + " def call(self, inputs, states):\n", + " outputs, new_states = self.simple_rnn_cell(inputs, states)\n", + " norm_outputs = self.activation(self.layer_norm(outputs))\n", + " return norm_outputs, [norm_outputs]" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "metadata": {}, + "outputs": [], + "source": [ + "np.random.seed(42)\n", + "tf.random.set_seed(42)\n", + "\n", + "model = keras.models.Sequential([\n", + " keras.layers.RNN(LNSimpleRNNCell(20), return_sequences=True,\n", + " input_shape=[None, 1]),\n", + " keras.layers.RNN(LNSimpleRNNCell(20), return_sequences=True),\n", + " keras.layers.TimeDistributed(keras.layers.Dense(1))\n", + "])\n", + "\n", + "model.compile(loss=\"mse\", optimizer=\"adam\", metrics=[last_10_time_steps_mse])\n", + "history = model.fit(X_train, Y_train, epochs=20,\n", + " validation_data=(X_valid, Y_valid))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Creating a Custom RNN Class" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "metadata": {}, + "outputs": [], + "source": [ + "class MyRNN(keras.layers.Layer):\n", + " def __init__(self, cell, return_sequences=False, **kwargs):\n", + " super().__init__(**kwargs)\n", + " self.cell = cell\n", + " self.return_sequences = return_sequences\n", + " self.get_initial_state = getattr(\n", + " self.cell, \"get_initial_state\", self.fallback_initial_state)\n", + " def fallback_initial_state(self, inputs):\n", + " return [tf.zeros([self.cell.state_size], dtype=inputs.dtype)]\n", + " @tf.function\n", + " def call(self, inputs):\n", + " states = self.get_initial_state(inputs)\n", + " n_steps = tf.shape(inputs)[1]\n", + " if self.return_sequences:\n", + " sequences = tf.TensorArray(inputs.dtype, size=n_steps)\n", + " outputs = tf.zeros(shape=[n_steps, self.cell.output_size], dtype=inputs.dtype)\n", + " for step in tf.range(n_steps):\n", + " outputs, states = self.cell(inputs[:, step], states)\n", + " if self.return_sequences:\n", + " sequences = sequences.write(step, outputs)\n", + " if self.return_sequences:\n", + " return sequences.stack()\n", + " else:\n", + " return outputs" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "metadata": {}, + "outputs": [], + "source": [ + "np.random.seed(42)\n", + "tf.random.set_seed(42)\n", + "\n", + "model = keras.models.Sequential([\n", + " MyRNN(LNSimpleRNNCell(20), return_sequences=True,\n", + " input_shape=[None, 1]),\n", + " MyRNN(LNSimpleRNNCell(20), return_sequences=True),\n", + " keras.layers.TimeDistributed(keras.layers.Dense(1))\n", + "])\n", + "\n", + "model.compile(loss=\"mse\", optimizer=\"adam\", metrics=[last_10_time_steps_mse])\n", + "history = model.fit(X_train, Y_train, epochs=20,\n", + " validation_data=(X_valid, Y_valid))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# LSTMs" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "np.random.seed(42)\n", + "tf.random.set_seed(42)\n", + "\n", + "model = keras.models.Sequential([\n", + " keras.layers.LSTM(20, return_sequences=True, input_shape=[None, 1]),\n", + " keras.layers.LSTM(20, return_sequences=True),\n", + " keras.layers.TimeDistributed(keras.layers.Dense(1))\n", + "])\n", + "\n", + "model.compile(loss=\"mse\", optimizer=\"adam\", metrics=[last_10_time_steps_mse])\n", + "history = model.fit(X_train, Y_train, epochs=20,\n", + " validation_data=(X_valid, Y_valid))" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "metadata": {}, + "outputs": [], + "source": [ + "model.evaluate(X_valid, Y_valid)" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "metadata": {}, + "outputs": [], + "source": [ + "plot_learning_curves(history.history[\"loss\"], history.history[\"val_loss\"])\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "metadata": {}, + "outputs": [], + "source": [ + "np.random.seed(43)\n", + "\n", + "series = generate_time_series(1, 50 + 10)\n", + "X_new, Y_new = series[:, :50, :], series[:, 50:, :]\n", + "Y_pred = model.predict(X_new)[:, -10:, :]" + ] + }, + { + "cell_type": "code", + "execution_count": 46, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "plot_multiple_forecasts(X_new, Y_new, Y_pred)\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# GRUs" + ] + }, + { + "cell_type": "code", + "execution_count": 47, + "metadata": { + "scrolled": false + }, + "outputs": [], + "source": [ + "np.random.seed(42)\n", + "tf.random.set_seed(42)\n", + "\n", + "model = keras.models.Sequential([\n", + " keras.layers.GRU(20, return_sequences=True, input_shape=[None, 1]),\n", + " keras.layers.GRU(20, return_sequences=True),\n", + " keras.layers.TimeDistributed(keras.layers.Dense(1))\n", + "])\n", + "\n", + "model.compile(loss=\"mse\", optimizer=\"adam\", metrics=[last_10_time_steps_mse])\n", + "history = model.fit(X_train, Y_train, epochs=20,\n", + " validation_data=(X_valid, Y_valid))" + ] + }, + { + "cell_type": "code", + "execution_count": 48, + "metadata": {}, + "outputs": [], + "source": [ + "model.evaluate(X_valid, Y_valid)" + ] + }, + { + "cell_type": "code", + "execution_count": 49, + "metadata": {}, + "outputs": [], + "source": [ + "plot_learning_curves(history.history[\"loss\"], history.history[\"val_loss\"])\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 50, + "metadata": {}, + "outputs": [], + "source": [ + "np.random.seed(43)\n", + "\n", + "series = generate_time_series(1, 50 + 10)\n", + "X_new, Y_new = series[:, :50, :], series[:, 50:, :]\n", + "Y_pred = model.predict(X_new)[:, -10:, :]" + ] + }, + { + "cell_type": "code", + "execution_count": 51, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "plot_multiple_forecasts(X_new, Y_new, Y_pred)\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using One-Dimensional Convolutional Layers to Process Sequences" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "```\n", + "1D conv layer with kernel size 4, stride 2, VALID padding:\n", + "\n", + " |-----2----| |-----5---... |----23-----|\n", + " |-----1----| |-----4-----| ... |-----22----|\n", + " |-----0----| |-----3----| |---...-21---|\n", + "X: 0 1 2 3 4 5 6 7 8 9 10 11 12 ... 43 44 45 46 47 48 49\n", + "Y: 10 11 12 13 14 15 16 17 18 19 20 21 22 ... 53 54 55 56 57 58 59\n", + "\n", + "Output:\n", + "\n", + "X: 0 1 2 3 4 5 ... 19 20 21 22 23\n", + "Y: 13 15 17 19 21 23 ... 51 53 55 57 59\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": 52, + "metadata": {}, + "outputs": [], + "source": [ + "np.random.seed(42)\n", + "tf.random.set_seed(42)\n", + "\n", + "def last_5_time_steps_mse(Y_true, Y_pred):\n", + " return keras.metrics.mean_squared_error(Y_true[:, -5:], Y_pred[:, -5:])\n", + "\n", + "model = keras.models.Sequential([\n", + " keras.layers.Conv1D(filters=20, kernel_size=4, strides=2, padding=\"VALID\",\n", + " input_shape=[None, 1]),\n", + " keras.layers.GRU(20, return_sequences=True),\n", + " keras.layers.GRU(20, return_sequences=True),\n", + " keras.layers.TimeDistributed(keras.layers.Dense(1))\n", + "])\n", + "\n", + "model.compile(loss=\"mse\", optimizer=\"adam\", metrics=[last_5_time_steps_mse])\n", + "history = model.fit(X_train, Y_train[:, 3::2], epochs=20,\n", + " validation_data=(X_valid, Y_valid[:, 3::2]))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## WaveNet" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "```\n", + "C2 /\\ /\\ /\\ /\\ /\\ /\\ /\\ /\\ /\\ /\\ /\\ /\\.../\\ /\\ /\\ /\\ /\\ /\\ \n", + " / \\ / \\ / \\ / \\ / \\ / \\ / \\ / \\ / \\\n", + " / \\ / \\ / \\ / \\\n", + "C1 /\\ /\\ /\\ /\\ /\\ /\\ /\\ /\\ /\\ /\\ /\\ /\\ /.../\\ /\\ /\\ /\\ /\\ /\\ /\\\n", + "X: 0 1 2 3 4 5 6 7 8 9 10 11 12 ... 43 44 45 46 47 48 49\n", + "Y: 10 11 12 13 14 15 16 17 18 19 20 21 22 ... 53 54 55 56 57 58 59\n", + "\n", + "Output:\n", + "\n", + "X: 0 1 2 3 4 5 ... 19 20 21 22 23\n", + "Y: 13 15 17 19 21 23 ... 51 53 55 57 59\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": 53, + "metadata": {}, + "outputs": [], + "source": [ + "np.random.seed(42)\n", + "tf.random.set_seed(42)\n", + "\n", + "model = keras.models.Sequential()\n", + "model.add(keras.layers.InputLayer(input_shape=[None, 1]))\n", + "for rate in (1, 2, 4, 8) * 2:\n", + " model.add(keras.layers.Lambda(\n", + " lambda inputs: keras.backend.temporal_padding(inputs, (rate, 0))))\n", + " model.add(keras.layers.Conv1D(filters=20, kernel_size=2, padding=\"VALID\",\n", + " activation=\"relu\", dilation_rate=rate))\n", + "model.add(keras.layers.Conv1D(filters=1, kernel_size=1))\n", + "model.compile(loss=\"mse\", optimizer=\"adam\", metrics=[last_10_time_steps_mse])\n", + "history = model.fit(X_train, Y_train, epochs=20,\n", + " validation_data=(X_valid, Y_valid))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here is the original WaveNet defined in the paper: it uses Gated Activation Units instead of ReLU and parametrized skip connections, plus it pads with zeros on the left to avoid getting shorter and shorter sequences:" + ] + }, + { + "cell_type": "code", + "execution_count": 54, + "metadata": {}, + "outputs": [], + "source": [ + "from tensorflow import keras\n", + "\n", + "class GatedActivationUnit(keras.layers.Layer):\n", + " def __init__(self, activation=\"tanh\", **kwargs):\n", + " super().__init__(**kwargs)\n", + " self.activation = keras.activations.get(activation)\n", + " def call(self, inputs):\n", + " n_filters = inputs.shape[-1] // 2\n", + " linear_output = self.activation(inputs[..., :n_filters])\n", + " gate = keras.activations.sigmoid(inputs[..., n_filters:])\n", + " return self.activation(linear_output) * gate" + ] + }, + { + "cell_type": "code", + "execution_count": 55, + "metadata": {}, + "outputs": [], + "source": [ + "def wavenet_residual_block(inputs, n_filters, dilation_rate):\n", + " z = keras.backend.temporal_padding(inputs, (dilation_rate, 0))\n", + " z = keras.layers.Conv1D(2 * n_filters, kernel_size=2,\n", + " dilation_rate=dilation_rate)(z)\n", + " z = GatedActivationUnit()(z)\n", + " z = keras.layers.Conv1D(n_filters, kernel_size=1)(z)\n", + " return keras.layers.Add()([z, inputs]), z" + ] + }, + { + "cell_type": "code", + "execution_count": 56, + "metadata": {}, + "outputs": [], + "source": [ + "np.random.seed(42)\n", + "tf.random.set_seed(42)\n", + "\n", + "n_layers_per_block = 10\n", + "n_blocks = 3\n", + "n_filters = 128\n", + "n_outputs = 256\n", + "\n", + "inputs = keras.layers.Input(shape=[None, 1])\n", + "z = keras.backend.temporal_padding(inputs, (1, 0))\n", + "z = keras.layers.Conv1D(n_filters, kernel_size=2)(z)\n", + "skip_to_last = []\n", + "for dilation_rate in [2**i for i in range(n_layers_per_block)] * n_blocks:\n", + " z, skip = wavenet_residual_block(z, n_filters, dilation_rate)\n", + " skip_to_last.append(skip)\n", + "z = keras.activations.relu(keras.layers.Add()(skip_to_last))\n", + "z = keras.layers.Conv1D(n_filters, kernel_size=1, activation=\"relu\")(z)\n", + "Y_proba = keras.layers.Conv1D(n_outputs, kernel_size=1, activation=\"softmax\")(z)\n", + "\n", + "model = keras.models.Model(inputs=[inputs], outputs=[Y_proba])" + ] + }, + { + "cell_type": "code", + "execution_count": 57, + "metadata": {}, + "outputs": [], + "source": [ + "model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=\"adam\")\n", + "history = model.fit(X_train, Y_train, epochs=2, validation_data=(X_valid, Y_valid))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Splitting a sequence into batches of shuffled windows" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "For example, let's split the sequence 0 to 14 into windows of length 5, each shifted by 2 (e.g.,`[0, 1, 2, 3, 4]`, `[2, 3, 4, 5, 6]`, etc.), then shuffle them, and split them into inputs (the first 4 steps) and targets (the last 4 steps) (e.g., `[2, 3, 4, 5, 6]` would be split into `[[2, 3, 4, 5], [3, 4, 5, 6]]`), then create batches of 3 such input/target pairs:" + ] + }, + { + "cell_type": "code", + "execution_count": 58, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "np.random.seed(42)\n", + "tf.random.set_seed(42)\n", + "\n", + "n_steps = 5\n", + "dataset = tf.data.Dataset.from_tensor_slices(tf.range(15))\n", + "dataset = dataset.window(n_steps, shift=2, drop_remainder=True)\n", + "dataset = dataset.flat_map(lambda window: window.batch(n_steps))\n", + "dataset = dataset.shuffle(10).map(lambda window: (window[:-1], window[1:]))\n", + "dataset = dataset.batch(3).prefetch(1)\n", + "for index, (X_batch, Y_batch) in enumerate(dataset):\n", + " print(\"_\" * 20, \"Batch\", index, \"\\nX_batch\")\n", + " print(X_batch.numpy())\n", + " print(\"=\" * 5, \"\\nY_batch\")\n", + " print(Y_batch.numpy())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Char-RNN" + ] + }, + { + "cell_type": "code", + "execution_count": 59, + "metadata": {}, + "outputs": [], + "source": [ + "shakespeare_url = \"https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt\"\n", + "filepath = keras.utils.get_file(\"shakespeare.txt\", shakespeare_url)\n", + "with open(filepath) as f:\n", + " shakespeare_text = f.read()" + ] + }, + { + "cell_type": "code", + "execution_count": 60, + "metadata": {}, + "outputs": [], + "source": [ + "print(shakespeare_text[:148])" + ] + }, + { + "cell_type": "code", + "execution_count": 61, + "metadata": {}, + "outputs": [], + "source": [ + "\"\".join(sorted(set(shakespeare_text.lower())))" + ] + }, + { + "cell_type": "code", + "execution_count": 62, + "metadata": {}, + "outputs": [], + "source": [ + "tokenizer = keras.preprocessing.text.Tokenizer(char_level=True)\n", + "tokenizer.fit_on_texts(shakespeare_text)" + ] + }, + { + "cell_type": "code", + "execution_count": 63, + "metadata": {}, + "outputs": [], + "source": [ + "tokenizer.texts_to_sequences([\"First\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 64, + "metadata": {}, + "outputs": [], + "source": [ + "tokenizer.sequences_to_texts([[20, 6, 9, 8, 3]])" + ] + }, + { + "cell_type": "code", + "execution_count": 65, + "metadata": {}, + "outputs": [], + "source": [ + "max_id = len(tokenizer.word_index) # number of distinct characters\n", + "dataset_size = tokenizer.document_count # total number of characters" + ] + }, + { + "cell_type": "code", + "execution_count": 66, + "metadata": {}, + "outputs": [], + "source": [ + "[encoded] = np.array(tokenizer.texts_to_sequences([shakespeare_text])) - 1\n", + "train_size = dataset_size * 90 // 100\n", + "dataset = tf.data.Dataset.from_tensor_slices(encoded[:train_size])" + ] + }, + { + "cell_type": "code", + "execution_count": 67, + "metadata": {}, + "outputs": [], + "source": [ + "n_steps = 100\n", + "window_length = n_steps + 1 # target = input shifted 1 character ahead\n", + "dataset = dataset.repeat().window(window_length, shift=1, drop_remainder=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 68, + "metadata": {}, + "outputs": [], + "source": [ + "dataset = dataset.flat_map(lambda window: window.batch(window_length))" + ] + }, + { + "cell_type": "code", + "execution_count": 69, + "metadata": {}, + "outputs": [], + "source": [ + "np.random.seed(42)\n", + "tf.random.set_seed(42)" + ] + }, + { + "cell_type": "code", + "execution_count": 70, + "metadata": {}, + "outputs": [], + "source": [ + "batch_size = 32\n", + "dataset = dataset.shuffle(10000).batch(batch_size)\n", + "dataset = dataset.map(lambda windows: (windows[:, :-1], windows[:, 1:]))" + ] + }, + { + "cell_type": "code", + "execution_count": 71, + "metadata": {}, + "outputs": [], + "source": [ + "dataset = dataset.map(\n", + " lambda X_batch, Y_batch: (tf.one_hot(X_batch, depth=max_id), Y_batch))" + ] + }, + { + "cell_type": "code", + "execution_count": 72, + "metadata": {}, + "outputs": [], + "source": [ + "dataset = dataset.prefetch(1)" + ] + }, + { + "cell_type": "code", + "execution_count": 73, + "metadata": {}, + "outputs": [], + "source": [ + "for X_batch, Y_batch in dataset.take(1):\n", + " print(X_batch.shape, Y_batch.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": 74, + "metadata": {}, + "outputs": [], + "source": [ + "model = keras.models.Sequential([\n", + " keras.layers.GRU(128, return_sequences=True, input_shape=[None, max_id],\n", + " dropout=0.2, recurrent_dropout=0.2),\n", + " keras.layers.GRU(128, return_sequences=True,\n", + " dropout=0.2, recurrent_dropout=0.2),\n", + " keras.layers.TimeDistributed(keras.layers.Dense(max_id,\n", + " activation=\"softmax\"))\n", + "])\n", + "model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=\"adam\")\n", + "history = model.fit(dataset, steps_per_epoch=train_size // batch_size,\n", + " epochs=10)" + ] + }, + { + "cell_type": "code", + "execution_count": 75, + "metadata": {}, + "outputs": [], + "source": [ + "def preprocess(texts):\n", + " X = np.array(tokenizer.texts_to_sequences(texts)) - 1\n", + " return tf.one_hot(X, max_id)" + ] + }, + { + "cell_type": "code", + "execution_count": 76, + "metadata": {}, + "outputs": [], + "source": [ + "X_new = preprocess([\"How are yo\"])\n", + "Y_pred = model.predict_classes(X_new)\n", + "tokenizer.sequences_to_texts(Y_pred + 1)[0][-1] # 1st sentence, last char" + ] + }, + { + "cell_type": "code", + "execution_count": 77, + "metadata": {}, + "outputs": [], + "source": [ + "tf.random.set_seed(42)\n", + "\n", + "tf.random.categorical([[np.log(0.5), np.log(0.4), np.log(0.1)]], num_samples=40).numpy()" + ] + }, + { + "cell_type": "code", + "execution_count": 78, + "metadata": {}, + "outputs": [], + "source": [ + "def next_char(text, temperature=1):\n", + " X_new = preprocess([text])\n", + " y_proba = model.predict(X_new)[0, -1:, :]\n", + " rescaled_logits = tf.math.log(y_proba) / temperature\n", + " char_id = tf.random.categorical(rescaled_logits, num_samples=1) + 1\n", + " return tokenizer.sequences_to_texts(char_id.numpy())[0]" + ] + }, + { + "cell_type": "code", + "execution_count": 79, + "metadata": {}, + "outputs": [], + "source": [ + "tf.random.set_seed(42)\n", + "\n", + "next_char(\"How are yo\", temperature=1)" + ] + }, + { + "cell_type": "code", + "execution_count": 80, + "metadata": {}, + "outputs": [], + "source": [ + "def complete_text(text, n_chars=50, temperature=1):\n", + " for _ in range(n_chars):\n", + " text += next_char(text, temperature)\n", + " return text" + ] + }, + { + "cell_type": "code", + "execution_count": 81, + "metadata": {}, + "outputs": [], + "source": [ + "tf.random.set_seed(42)\n", + "\n", + "print(complete_text(\"t\", temperature=0.2))" + ] + }, + { + "cell_type": "code", + "execution_count": 82, + "metadata": {}, + "outputs": [], + "source": [ + "print(complete_text(\"t\", temperature=1))" + ] + }, + { + "cell_type": "code", + "execution_count": 83, + "metadata": {}, + "outputs": [], + "source": [ + "print(complete_text(\"t\", temperature=2))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Stateful RNN" + ] + }, + { + "cell_type": "code", + "execution_count": 84, + "metadata": {}, + "outputs": [], + "source": [ + "tf.random.set_seed(42)" + ] + }, + { + "cell_type": "code", + "execution_count": 85, + "metadata": {}, + "outputs": [], + "source": [ + "dataset = tf.data.Dataset.from_tensor_slices(encoded[:train_size])\n", + "dataset = dataset.window(window_length, shift=n_steps, drop_remainder=True)\n", + "dataset = dataset.flat_map(lambda window: window.batch(window_length))\n", + "dataset = dataset.repeat().batch(1)\n", + "dataset = dataset.map(lambda windows: (windows[:, :-1], windows[:, 1:]))\n", + "dataset = dataset.map(\n", + " lambda X_batch, Y_batch: (tf.one_hot(X_batch, depth=max_id), Y_batch))\n", + "dataset = dataset.prefetch(1)" + ] + }, + { + "cell_type": "code", + "execution_count": 86, + "metadata": {}, + "outputs": [], + "source": [ + "batch_size = 32\n", + "encoded_parts = np.array_split(encoded[:train_size], batch_size)\n", + "datasets = []\n", + "for encoded_part in encoded_parts:\n", + " dataset = tf.data.Dataset.from_tensor_slices(encoded_part)\n", + " dataset = dataset.window(window_length, shift=n_steps, drop_remainder=True)\n", + " dataset = dataset.flat_map(lambda window: window.batch(window_length))\n", + " datasets.append(dataset)\n", + "dataset = tf.data.Dataset.zip(tuple(datasets)).map(lambda *windows: tf.stack(windows))\n", + "dataset = dataset.repeat().map(lambda windows: (windows[:, :-1], windows[:, 1:]))\n", + "dataset = dataset.map(\n", + " lambda X_batch, Y_batch: (tf.one_hot(X_batch, depth=max_id), Y_batch))\n", + "dataset = dataset.prefetch(1)" + ] + }, + { + "cell_type": "code", + "execution_count": 87, + "metadata": {}, + "outputs": [], + "source": [ + "model = keras.models.Sequential([\n", + " keras.layers.GRU(128, return_sequences=True, stateful=True,\n", + "# dropout=0.2, recurrent_dropout=0.2, # see TF issue #27829\n", + " batch_input_shape=[batch_size, None, max_id]),\n", + " keras.layers.GRU(128, return_sequences=True, stateful=True\n", + "# dropout=0.2, recurrent_dropout=0.2 # see TF issue #27829\n", + " ),\n", + " keras.layers.TimeDistributed(keras.layers.Dense(max_id,\n", + " activation=\"softmax\"))\n", + "])" + ] + }, + { + "cell_type": "code", + "execution_count": 88, + "metadata": {}, + "outputs": [], + "source": [ + "class ResetStatesCallback(keras.callbacks.Callback):\n", + " def on_epoch_begin(self, epoch, logs):\n", + " self.model.reset_states()" + ] + }, + { + "cell_type": "code", + "execution_count": 89, + "metadata": {}, + "outputs": [], + "source": [ + "model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=\"adam\")\n", + "steps_per_epoch = train_size // batch_size // n_steps\n", + "model.fit(dataset, steps_per_epoch=steps_per_epoch, epochs=50,\n", + " callbacks=[ResetStatesCallback()])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To use the model with different batch sizes, we need to create a stateless copy. We can get rid of dropout since it is only used during training:" + ] + }, + { + "cell_type": "code", + "execution_count": 90, + "metadata": {}, + "outputs": [], + "source": [ + "stateless_model = keras.models.Sequential([\n", + " keras.layers.GRU(128, return_sequences=True, input_shape=[None, max_id]),\n", + " keras.layers.GRU(128, return_sequences=True),\n", + " keras.layers.TimeDistributed(keras.layers.Dense(max_id,\n", + " activation=\"softmax\"))\n", + "])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To set the weights, we first need to build the model (so the weights get created):" + ] + }, + { + "cell_type": "code", + "execution_count": 91, + "metadata": {}, + "outputs": [], + "source": [ + "stateless_model.build(tf.TensorShape([None, None, max_id]))" + ] + }, + { + "cell_type": "code", + "execution_count": 92, + "metadata": {}, + "outputs": [], + "source": [ + "stateless_model.set_weights(model.get_weights())\n", + "model = stateless_model" + ] + }, + { + "cell_type": "code", + "execution_count": 93, + "metadata": {}, + "outputs": [], + "source": [ + "tf.random.set_seed(42)\n", + "\n", + "print(complete_text(\"t\"))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Sentiment Analysis" + ] + }, + { + "cell_type": "code", + "execution_count": 94, + "metadata": {}, + "outputs": [], + "source": [ + "tf.random.set_seed(42)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can load the IMDB dataset easily:" + ] + }, + { + "cell_type": "code", + "execution_count": 95, + "metadata": {}, + "outputs": [], + "source": [ + "(X_train, y_test), (X_valid, y_test) = keras.datasets.imdb.load_data()" + ] + }, + { + "cell_type": "code", + "execution_count": 96, + "metadata": {}, + "outputs": [], + "source": [ + "X_train[0][:10]" + ] + }, + { + "cell_type": "code", + "execution_count": 97, + "metadata": {}, + "outputs": [], + "source": [ + "word_index = keras.datasets.imdb.get_word_index()\n", + "id_to_word = {id_ + 3: word for word, id_ in word_index.items()}\n", + "for id_, token in enumerate((\"\", \"\", \"\")):\n", + " id_to_word[id_] = token\n", + "\" \".join([id_to_word[id_] for id_ in X_train[0][:10]])" + ] + }, + { + "cell_type": "code", + "execution_count": 98, + "metadata": {}, + "outputs": [], + "source": [ + "import tensorflow_datasets as tfds\n", + "\n", + "datasets, info = tfds.load(\"imdb_reviews\", as_supervised=True, with_info=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 99, + "metadata": {}, + "outputs": [], + "source": [ + "datasets.keys()" + ] + }, + { + "cell_type": "code", + "execution_count": 100, + "metadata": {}, + "outputs": [], + "source": [ + "train_size = info.splits[\"train\"].num_examples\n", + "test_size = info.splits[\"test\"].num_examples" + ] + }, + { + "cell_type": "code", + "execution_count": 101, + "metadata": {}, + "outputs": [], + "source": [ + "train_size, test_size" + ] + }, + { + "cell_type": "code", + "execution_count": 102, + "metadata": {}, + "outputs": [], + "source": [ + "for X_batch, y_batch in datasets[\"train\"].batch(2).take(1):\n", + " for review, label in zip(X_batch.numpy(), y_batch.numpy()):\n", + " print(\"Review:\", review.decode(\"utf-8\")[:200], \"...\")\n", + " print(\"Label:\", label, \"= Positive\" if label else \"= Negative\")\n", + " print()" + ] + }, + { + "cell_type": "code", + "execution_count": 103, + "metadata": {}, + "outputs": [], + "source": [ + "def preprocess(X_batch, y_batch):\n", + " X_batch = tf.strings.substr(X_batch, 0, 300)\n", + " X_batch = tf.strings.regex_replace(X_batch, rb\"\", b\" \")\n", + " X_batch = tf.strings.regex_replace(X_batch, b\"[^a-zA-Z']\", b\" \")\n", + " X_batch = tf.strings.split(X_batch)\n", + " return X_batch.to_tensor(default_value=b\"\"), y_batch" + ] + }, + { + "cell_type": "code", + "execution_count": 104, + "metadata": {}, + "outputs": [], + "source": [ + "preprocess(X_batch, y_batch)" + ] + }, + { + "cell_type": "code", + "execution_count": 105, + "metadata": {}, + "outputs": [], + "source": [ + "from collections import Counter\n", + "\n", + "vocabulary = Counter()\n", + "for X_batch, y_batch in datasets[\"train\"].batch(32).map(preprocess):\n", + " for review in X_batch:\n", + " vocabulary.update(list(review.numpy()))" + ] + }, + { + "cell_type": "code", + "execution_count": 106, + "metadata": {}, + "outputs": [], + "source": [ + "vocabulary.most_common()[:3]" + ] + }, + { + "cell_type": "code", + "execution_count": 107, + "metadata": {}, + "outputs": [], + "source": [ + "len(vocabulary)" + ] + }, + { + "cell_type": "code", + "execution_count": 108, + "metadata": {}, + "outputs": [], + "source": [ + "vocab_size = 10000\n", + "truncated_vocabulary = [\n", + " word for word, count in vocabulary.most_common()[:vocab_size]]" + ] + }, + { + "cell_type": "code", + "execution_count": 109, + "metadata": {}, + "outputs": [], + "source": [ + "word_to_id = {word: index for index, word in enumerate(truncated_vocabulary)}\n", + "for word in b\"This movie was faaaaaantastic\".split():\n", + " print(word_to_id.get(word) or vocab_size)" + ] + }, + { + "cell_type": "code", + "execution_count": 110, + "metadata": {}, + "outputs": [], + "source": [ + "words = tf.constant(truncated_vocabulary)\n", + "word_ids = tf.range(len(truncated_vocabulary), dtype=tf.int64)\n", + "vocab_init = tf.lookup.KeyValueTensorInitializer(words, word_ids)\n", + "num_oov_buckets = 1000\n", + "table = tf.lookup.StaticVocabularyTable(vocab_init, num_oov_buckets)" + ] + }, + { + "cell_type": "code", + "execution_count": 111, + "metadata": {}, + "outputs": [], + "source": [ + "table.lookup(tf.constant([b\"This movie was faaaaaantastic\".split()]))" + ] + }, + { + "cell_type": "code", + "execution_count": 112, + "metadata": {}, + "outputs": [], + "source": [ + "def encode_words(X_batch, y_batch):\n", + " return table.lookup(X_batch), y_batch\n", + "\n", + "train_set = datasets[\"train\"].repeat().batch(32).map(preprocess)\n", + "train_set = train_set.map(encode_words).prefetch(1)" + ] + }, + { + "cell_type": "code", + "execution_count": 113, + "metadata": {}, + "outputs": [], + "source": [ + "for X_batch, y_batch in train_set.take(1):\n", + " print(X_batch)\n", + " print(y_batch)" + ] + }, + { + "cell_type": "code", + "execution_count": 114, + "metadata": {}, + "outputs": [], + "source": [ + "embed_size = 128\n", + "model = keras.models.Sequential([\n", + " keras.layers.Embedding(vocab_size + num_oov_buckets, embed_size,\n", + " mask_zero=True, # not shown in the book\n", + " input_shape=[None]),\n", + " keras.layers.GRU(128, return_sequences=True),\n", + " keras.layers.GRU(128),\n", + " keras.layers.Dense(1, activation=\"sigmoid\")\n", + "])\n", + "model.compile(loss=\"binary_crossentropy\", optimizer=\"adam\", metrics=[\"accuracy\"])\n", + "history = model.fit(train_set, steps_per_epoch=train_size // 32, epochs=5)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Or using manual masking:" + ] + }, + { + "cell_type": "code", + "execution_count": 115, + "metadata": {}, + "outputs": [], + "source": [ + "K = keras.backend\n", + "embed_size = 128\n", + "inputs = keras.layers.Input(shape=[None])\n", + "mask = keras.layers.Lambda(lambda inputs: K.not_equal(inputs, 0))(inputs)\n", + "z = keras.layers.Embedding(vocab_size + num_oov_buckets, embed_size)(inputs)\n", + "z = keras.layers.GRU(128, return_sequences=True)(z, mask=mask)\n", + "z = keras.layers.GRU(128)(z, mask=mask)\n", + "outputs = keras.layers.Dense(1, activation=\"sigmoid\")(z)\n", + "model = keras.models.Model(inputs=[inputs], outputs=[outputs])\n", + "model.compile(loss=\"binary_crossentropy\", optimizer=\"adam\", metrics=[\"accuracy\"])\n", + "history = model.fit(train_set, steps_per_epoch=train_size // 32, epochs=5)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Reusing Pretrained Embeddings" + ] + }, + { + "cell_type": "code", + "execution_count": 116, + "metadata": {}, + "outputs": [], + "source": [ + "tf.random.set_seed(42)" + ] + }, + { + "cell_type": "code", + "execution_count": 117, + "metadata": {}, + "outputs": [], + "source": [ + "TFHUB_CACHE_DIR = os.path.join(os.curdir, \"my_tfhub_cache\")\n", + "os.environ[\"TFHUB_CACHE_DIR\"] = TFHUB_CACHE_DIR" + ] + }, + { + "cell_type": "code", + "execution_count": 118, + "metadata": {}, + "outputs": [], + "source": [ + "import tensorflow_hub as hub\n", + "\n", + "model = keras.Sequential([\n", + " hub.KerasLayer(\"https://tfhub.dev/google/tf2-preview/nnlm-en-dim50/1\",\n", + " dtype=tf.string, input_shape=[], output_shape=[50]),\n", + " keras.layers.Dense(128, activation=\"relu\"),\n", + " keras.layers.Dense(1, activation=\"sigmoid\")\n", + "])\n", + "model.compile(loss=\"binary_crossentropy\", optimizer=\"adam\",\n", + " metrics=[\"accuracy\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 119, + "metadata": {}, + "outputs": [], + "source": [ + "for dirpath, dirnames, filenames in os.walk(TFHUB_CACHE_DIR):\n", + " for filename in filenames:\n", + " print(os.path.join(dirpath, filename))" + ] + }, + { + "cell_type": "code", + "execution_count": 120, + "metadata": {}, + "outputs": [], + "source": [ + "import tensorflow_datasets as tfds\n", + "\n", + "datasets, info = tfds.load(\"imdb_reviews\", as_supervised=True, with_info=True)\n", + "train_size = info.splits[\"train\"].num_examples\n", + "batch_size = 32\n", + "train_set = datasets[\"train\"].repeat().batch(batch_size).prefetch(1)\n", + "history = model.fit(train_set, steps_per_epoch=train_size // batch_size, epochs=5)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Automatic Translation" + ] + }, + { + "cell_type": "code", + "execution_count": 121, + "metadata": {}, + "outputs": [], + "source": [ + "tf.random.set_seed(42)" + ] + }, + { + "cell_type": "code", + "execution_count": 122, + "metadata": {}, + "outputs": [], + "source": [ + "vocab_size = 100\n", + "embed_size = 10" + ] + }, + { + "cell_type": "code", + "execution_count": 123, + "metadata": {}, + "outputs": [], + "source": [ + "import tensorflow_addons as tfa\n", + "\n", + "encoder_inputs = keras.layers.Input(shape=[None], dtype=np.int32)\n", + "decoder_inputs = keras.layers.Input(shape=[None], dtype=np.int32)\n", + "sequence_lengths = keras.layers.Input(shape=[], dtype=np.int32)\n", + "\n", + "embeddings = keras.layers.Embedding(vocab_size, embed_size)\n", + "encoder_embeddings = embeddings(encoder_inputs)\n", + "decoder_embeddings = embeddings(decoder_inputs)\n", + "\n", + "encoder = keras.layers.LSTM(512, return_state=True)\n", + "encoder_outputs, state_h, state_c = encoder(encoder_embeddings)\n", + "encoder_state = [state_h, state_c]\n", + "\n", + "sampler = tfa.seq2seq.sampler.TrainingSampler()\n", + "\n", + "decoder_cell = keras.layers.LSTMCell(512)\n", + "output_layer = keras.layers.Dense(vocab_size)\n", + "decoder = tfa.seq2seq.basic_decoder.BasicDecoder(decoder_cell, sampler,\n", + " output_layer=output_layer)\n", + "final_outputs, final_state, final_sequence_lengths = decoder(\n", + " decoder_embeddings, initial_state=encoder_state,\n", + " sequence_length=sequence_lengths)\n", + "Y_proba = tf.nn.softmax(final_outputs.rnn_output)\n", + "\n", + "model = keras.models.Model(\n", + " inputs=[encoder_inputs, decoder_inputs, sequence_lengths],\n", + " outputs=[Y_proba])" + ] + }, + { + "cell_type": "code", + "execution_count": 124, + "metadata": {}, + "outputs": [], + "source": [ + "model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=\"adam\")" + ] + }, + { + "cell_type": "code", + "execution_count": 125, + "metadata": {}, + "outputs": [], + "source": [ + "X = np.random.randint(100, size=10*1000).reshape(1000, 10)\n", + "Y = np.random.randint(100, size=15*1000).reshape(1000, 15)\n", + "X_decoder = np.c_[np.zeros((1000, 1)), Y[:, :-1]]\n", + "seq_lengths = np.full([1000], 15)\n", + "\n", + "history = model.fit([X, X_decoder, seq_lengths], Y, epochs=2)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Bidirectional Recurrent Layers" + ] + }, + { + "cell_type": "code", + "execution_count": 126, + "metadata": {}, + "outputs": [], + "source": [ + "model = keras.models.Sequential([\n", + " keras.layers.GRU(10, return_sequences=True, input_shape=[None, 10]),\n", + " keras.layers.Bidirectional(keras.layers.GRU(10, return_sequences=True))\n", + "])\n", + "\n", + "model.summary()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Positional Encoding" + ] + }, + { + "cell_type": "code", + "execution_count": 127, + "metadata": {}, + "outputs": [], + "source": [ + "class PositionalEncoding(keras.layers.Layer):\n", + " def __init__(self, max_steps, max_dims, dtype=tf.float32, **kwargs):\n", + " super().__init__(dtype=dtype, **kwargs)\n", + " if max_dims % 2 == 1: max_dims += 1 # max_dims must be even\n", + " p, i = np.meshgrid(np.arange(max_steps), np.arange(max_dims // 2))\n", + " pos_emb = np.empty((1, max_steps, max_dims))\n", + " pos_emb[0, :, ::2] = np.sin(p / 10000**(2 * i / max_dims)).T\n", + " pos_emb[0, :, 1::2] = np.cos(p / 10000**(2 * i / max_dims)).T\n", + " self.positional_embedding = tf.constant(pos_emb.astype(self.dtype))\n", + " def call(self, inputs):\n", + " shape = tf.shape(inputs)\n", + " return inputs + self.positional_embedding[:, :shape[-2], :shape[-1]]" + ] + }, + { + "cell_type": "code", + "execution_count": 128, + "metadata": {}, + "outputs": [], + "source": [ + "max_steps = 201\n", + "max_dims = 512\n", + "pos_emb = PositionalEncoding(max_steps, max_dims)\n", + "PE = pos_emb(np.zeros((1, max_steps, max_dims), np.float32))[0].numpy()" + ] + }, + { + "cell_type": "code", + "execution_count": 129, + "metadata": {}, + "outputs": [], + "source": [ + "i1, i2, crop_i = 100, 101, 150\n", + "p1, p2, p3 = 22, 60, 35\n", + "fig, (ax1, ax2) = plt.subplots(nrows=2, ncols=1, sharex=True, figsize=(9, 5))\n", + "ax1.plot([p1, p1], [-1, 1], \"k--\", label=\"$p = {}$\".format(p1))\n", + "ax1.plot([p2, p2], [-1, 1], \"k--\", label=\"$p = {}$\".format(p2), alpha=0.5)\n", + "ax1.plot(p3, PE[p3, i1], \"bx\", label=\"$p = {}$\".format(p3))\n", + "ax1.plot(PE[:,i1], \"b-\", label=\"$i = {}$\".format(i1))\n", + "ax1.plot(PE[:,i2], \"r-\", label=\"$i = {}$\".format(i2))\n", + "ax1.plot([p1, p2], [PE[p1, i1], PE[p2, i1]], \"bo\")\n", + "ax1.plot([p1, p2], [PE[p1, i2], PE[p2, i2]], \"ro\")\n", + "ax1.legend(loc=\"center right\", fontsize=14, framealpha=0.95)\n", + "ax1.set_ylabel(\"$P_{(p,i)}$\", rotation=0, fontsize=16)\n", + "ax1.grid(True, alpha=0.3)\n", + "ax1.hlines(0, 0, max_steps - 1, color=\"k\", linewidth=1, alpha=0.3)\n", + "ax1.axis([0, max_steps - 1, -1, 1])\n", + "ax2.imshow(PE.T[:crop_i], cmap=\"gray\", interpolation=\"bilinear\", aspect=\"auto\")\n", + "ax2.hlines(i1, 0, max_steps - 1, color=\"b\")\n", + "cheat = 2 # need to raise the red line a bit, or else it hides the blue one\n", + "ax2.hlines(i2+cheat, 0, max_steps - 1, color=\"r\")\n", + "ax2.plot([p1, p1], [0, crop_i], \"k--\")\n", + "ax2.plot([p2, p2], [0, crop_i], \"k--\", alpha=0.5)\n", + "ax2.plot([p1, p2], [i2+cheat, i2+cheat], \"ro\")\n", + "ax2.plot([p1, p2], [i1, i1], \"bo\")\n", + "ax2.axis([0, max_steps - 1, 0, crop_i])\n", + "ax2.set_xlabel(\"$p$\", fontsize=16)\n", + "ax2.set_ylabel(\"$i$\", rotation=0, fontsize=16)\n", + "plt.savefig(\"positional_embedding_plot\")\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 130, + "metadata": {}, + "outputs": [], + "source": [ + "embed_size = 512; max_steps = 500; vocab_size = 10000\n", + "encoder_inputs = keras.layers.Input(shape=[None], dtype=np.int32)\n", + "decoder_inputs = keras.layers.Input(shape=[None], dtype=np.int32)\n", + "embeddings = keras.layers.Embedding(vocab_size, embed_size)\n", + "encoder_embeddings = embeddings(encoder_inputs)\n", + "decoder_embeddings = embeddings(decoder_inputs)\n", + "positional_encoding = PositionalEncoding(max_steps, max_dims=embed_size)\n", + "encoder_in = positional_encoding(encoder_embeddings)\n", + "decoder_in = positional_encoding(decoder_embeddings)" + ] + }, + { + "cell_type": "code", + "execution_count": 131, + "metadata": {}, + "outputs": [], + "source": [ + "for N in range(6):\n", + " encoder_attn = keras.layers.Attention(use_scale=True)\n", + " encoder_in = encoder_attn([encoder_in, encoder_in])\n", + " masked_decoder_attn = keras.layers.Attention(use_scale=True, causal=True)\n", + " decoder_in = masked_decoder_attn([decoder_in, decoder_in])\n", + " decoder_attn = keras.layers.Attention(use_scale=True)\n", + " final_enc = decoder_attn([decoder_in, encoder_in])\n", + "\n", + "output_layer = keras.layers.TimeDistributed(\n", + " keras.layers.Dense(vocab_size, activation=\"softmax\"))\n", + "outputs = output_layer(final_enc)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": true + }, + "source": [ + "# Exercise solutions" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. to 6." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "See Appendix A." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 7. Embedded Reber Grammars" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "First we need to build a function that generates strings based on a grammar. The grammar will be represented as a list of possible transitions for each state. A transition specifies the string to output (or a grammar to generate it) and the next state." + ] + }, + { + "cell_type": "code", + "execution_count": 132, + "metadata": {}, + "outputs": [], + "source": [ + "np.random.seed(42)\n", + "\n", + "default_reber_grammar = [\n", + " [(\"B\", 1)], # (state 0) =B=>(state 1)\n", + " [(\"T\", 2), (\"P\", 3)], # (state 1) =T=>(state 2) or =P=>(state 3)\n", + " [(\"S\", 2), (\"X\", 4)], # (state 2) =S=>(state 2) or =X=>(state 4)\n", + " [(\"T\", 3), (\"V\", 5)], # and so on...\n", + " [(\"X\", 3), (\"S\", 6)],\n", + " [(\"P\", 4), (\"V\", 6)],\n", + " [(\"E\", None)]] # (state 6) =E=>(terminal state)\n", + "\n", + "embedded_reber_grammar = [\n", + " [(\"B\", 1)],\n", + " [(\"T\", 2), (\"P\", 3)],\n", + " [(default_reber_grammar, 4)],\n", + " [(default_reber_grammar, 5)],\n", + " [(\"T\", 6)],\n", + " [(\"P\", 6)],\n", + " [(\"E\", None)]]\n", + "\n", + "def generate_string(grammar):\n", + " state = 0\n", + " output = []\n", + " while state is not None:\n", + " index = np.random.randint(len(grammar[state]))\n", + " production, state = grammar[state][index]\n", + " if isinstance(production, list):\n", + " production = generate_string(grammar=production)\n", + " output.append(production)\n", + " return \"\".join(output)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's generate a few strings based on the default Reber grammar:" + ] + }, + { + "cell_type": "code", + "execution_count": 133, + "metadata": {}, + "outputs": [], + "source": [ + "for _ in range(25):\n", + " print(generate_string(default_reber_grammar), end=\" \")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Looks good. Now let's generate a few strings based on the embedded Reber grammar:" + ] + }, + { + "cell_type": "code", + "execution_count": 134, + "metadata": {}, + "outputs": [], + "source": [ + "for _ in range(25):\n", + " print(generate_string(embedded_reber_grammar), end=\" \")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Okay, now we need a function to generate strings that do not respect the grammar. We could generate a random string, but the task would be a bit too easy, so instead we will generate a string that respects the grammar, and we will corrupt it by changing just one character:" + ] + }, + { + "cell_type": "code", + "execution_count": 135, + "metadata": {}, + "outputs": [], + "source": [ + "def generate_corrupted_string(grammar, chars=\"BEPSTVX\"):\n", + " good_string = generate_string(grammar)\n", + " index = np.random.randint(len(good_string))\n", + " good_char = good_string[index]\n", + " bad_char = np.random.choice(sorted(set(chars) - set(good_char)))\n", + " return good_string[:index] + bad_char + good_string[index + 1:]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's look at a few corrupted strings:" + ] + }, + { + "cell_type": "code", + "execution_count": 136, + "metadata": {}, + "outputs": [], + "source": [ + "for _ in range(25):\n", + " print(generate_corrupted_string(embedded_reber_grammar), end=\" \")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To be continued..." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.8" + }, + "nav_menu": {}, + "toc": { + "navigate_menu": true, + "number_sections": true, + "sideBar": true, + "threshold": 6, + "toc_cell": false, + "toc_section_display": "block", + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 1 +} diff --git a/work_in_progress/14_recurrent_neural_networks.ipynb b/work_in_progress/14_recurrent_neural_networks.ipynb deleted file mode 100644 index d84171e..0000000 --- a/work_in_progress/14_recurrent_neural_networks.ipynb +++ /dev/null @@ -1,2500 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**Chapter 14 – Recurrent Neural Networks**" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "_This notebook contains all the sample code and solutions to the exercises in chapter 14._" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Setup" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "First, let's make sure this notebook works well in both python 2 and 3, import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures:" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "# To support both python 2 and python 3\n", - "from __future__ import division, print_function, unicode_literals\n", - "\n", - "# Common imports\n", - "import numpy as np\n", - "import os\n", - "\n", - "# to make this notebook's output stable across runs\n", - "def reset_graph(seed=42):\n", - " tf.reset_default_graph()\n", - " tf.set_random_seed(seed)\n", - " np.random.seed(seed)\n", - "\n", - "# To plot pretty figures\n", - "%matplotlib inline\n", - "import matplotlib\n", - "import matplotlib.pyplot as plt\n", - "plt.rcParams['axes.labelsize'] = 14\n", - "plt.rcParams['xtick.labelsize'] = 12\n", - "plt.rcParams['ytick.labelsize'] = 12\n", - "\n", - "# Where to save the figures\n", - "PROJECT_ROOT_DIR = \".\"\n", - "CHAPTER_ID = \"rnn\"\n", - "\n", - "def save_fig(fig_id, tight_layout=True):\n", - " path = os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID, fig_id + \".png\")\n", - " print(\"Saving figure\", fig_id)\n", - " if tight_layout:\n", - " plt.tight_layout()\n", - " plt.savefig(path, format='png', dpi=300)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Then of course we will need TensorFlow:" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "import tensorflow as tf" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Basic RNNs" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Manual RNN" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "reset_graph()\n", - "\n", - "n_inputs = 3\n", - "n_neurons = 5\n", - "\n", - "X0 = tf.placeholder(tf.float32, [None, n_inputs])\n", - "X1 = tf.placeholder(tf.float32, [None, n_inputs])\n", - "\n", - "Wx = tf.Variable(tf.random_normal(shape=[n_inputs, n_neurons],dtype=tf.float32))\n", - "Wy = tf.Variable(tf.random_normal(shape=[n_neurons,n_neurons],dtype=tf.float32))\n", - "b = tf.Variable(tf.zeros([1, n_neurons], dtype=tf.float32))\n", - "\n", - "Y0 = tf.tanh(tf.matmul(X0, Wx) + b)\n", - "Y1 = tf.tanh(tf.matmul(Y0, Wy) + tf.matmul(X1, Wx) + b)\n", - "\n", - "init = tf.global_variables_initializer()" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np\n", - "\n", - "X0_batch = np.array([[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 0, 1]]) # t = 0\n", - "X1_batch = np.array([[9, 8, 7], [0, 0, 0], [6, 5, 4], [3, 2, 1]]) # t = 1\n", - "\n", - "with tf.Session() as sess:\n", - " init.run()\n", - " Y0_val, Y1_val = sess.run([Y0, Y1], feed_dict={X0: X0_batch, X1: X1_batch})" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "print(Y0_val)" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "print(Y1_val)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Using `static_rnn()`" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**Note**: `tf.contrib.rnn` was partially moved to the core API in TensorFlow 1.2. Most of the `*Cell` and `*Wrapper` classes are now available in `tf.nn.rnn_cell`, and the `tf.contrib.rnn.static_rnn()` function is available as `tf.nn.static_rnn()`." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "n_inputs = 3\n", - "n_neurons = 5" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "reset_graph()\n", - "\n", - "X0 = tf.placeholder(tf.float32, [None, n_inputs])\n", - "X1 = tf.placeholder(tf.float32, [None, n_inputs])\n", - "\n", - "basic_cell = tf.nn.rnn_cell.BasicRNNCell(num_units=n_neurons)\n", - "output_seqs, states = tf.nn.static_rnn(basic_cell, [X0, X1],\n", - " dtype=tf.float32)\n", - "Y0, Y1 = output_seqs" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "init = tf.global_variables_initializer()" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "X0_batch = np.array([[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 0, 1]])\n", - "X1_batch = np.array([[9, 8, 7], [0, 0, 0], [6, 5, 4], [3, 2, 1]])\n", - "\n", - "with tf.Session() as sess:\n", - " init.run()\n", - " Y0_val, Y1_val = sess.run([Y0, Y1], feed_dict={X0: X0_batch, X1: X1_batch})" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "Y0_val" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "Y1_val" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "from tensorflow_graph_in_jupyter import show_graph" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [], - "source": [ - "show_graph(tf.get_default_graph())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Packing sequences" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [], - "source": [ - "n_steps = 2\n", - "n_inputs = 3\n", - "n_neurons = 5" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [], - "source": [ - "reset_graph()\n", - "\n", - "X = tf.placeholder(tf.float32, [None, n_steps, n_inputs])\n", - "X_seqs = tf.unstack(tf.transpose(X, perm=[1, 0, 2]))\n", - "\n", - "basic_cell = tf.nn.rnn_cell.BasicRNNCell(num_units=n_neurons)\n", - "output_seqs, states = tf.nn.static_rnn(basic_cell, X_seqs,\n", - " dtype=tf.float32)\n", - "outputs = tf.transpose(tf.stack(output_seqs), perm=[1, 0, 2])" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [], - "source": [ - "init = tf.global_variables_initializer()" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [], - "source": [ - "X_batch = np.array([\n", - " # t = 0 t = 1 \n", - " [[0, 1, 2], [9, 8, 7]], # instance 1\n", - " [[3, 4, 5], [0, 0, 0]], # instance 2\n", - " [[6, 7, 8], [6, 5, 4]], # instance 3\n", - " [[9, 0, 1], [3, 2, 1]], # instance 4\n", - " ])\n", - "\n", - "with tf.Session() as sess:\n", - " init.run()\n", - " outputs_val = outputs.eval(feed_dict={X: X_batch})" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [], - "source": [ - "print(outputs_val)" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [], - "source": [ - "print(np.transpose(outputs_val, axes=[1, 0, 2])[1])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Using `dynamic_rnn()`" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [], - "source": [ - "n_steps = 2\n", - "n_inputs = 3\n", - "n_neurons = 5" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [], - "source": [ - "reset_graph()\n", - "\n", - "X = tf.placeholder(tf.float32, [None, n_steps, n_inputs])\n", - "\n", - "basic_cell = tf.nn.rnn_cell.BasicRNNCell(num_units=n_neurons)\n", - "outputs, states = tf.nn.dynamic_rnn(basic_cell, X, dtype=tf.float32)" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [], - "source": [ - "init = tf.global_variables_initializer()" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": {}, - "outputs": [], - "source": [ - "X_batch = np.array([\n", - " [[0, 1, 2], [9, 8, 7]], # instance 1\n", - " [[3, 4, 5], [0, 0, 0]], # instance 2\n", - " [[6, 7, 8], [6, 5, 4]], # instance 3\n", - " [[9, 0, 1], [3, 2, 1]], # instance 4\n", - " ])\n", - "\n", - "with tf.Session() as sess:\n", - " init.run()\n", - " outputs_val = outputs.eval(feed_dict={X: X_batch})" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": {}, - "outputs": [], - "source": [ - "print(outputs_val)" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "metadata": {}, - "outputs": [], - "source": [ - "show_graph(tf.get_default_graph())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Setting the sequence lengths" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "metadata": {}, - "outputs": [], - "source": [ - "n_steps = 2\n", - "n_inputs = 3\n", - "n_neurons = 5\n", - "\n", - "reset_graph()\n", - "\n", - "X = tf.placeholder(tf.float32, [None, n_steps, n_inputs])\n", - "basic_cell = tf.nn.rnn_cell.BasicRNNCell(num_units=n_neurons)" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "metadata": {}, - "outputs": [], - "source": [ - "seq_length = tf.placeholder(tf.int32, [None])\n", - "outputs, states = tf.nn.dynamic_rnn(basic_cell, X, dtype=tf.float32,\n", - " sequence_length=seq_length)" - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "metadata": {}, - "outputs": [], - "source": [ - "init = tf.global_variables_initializer()" - ] - }, - { - "cell_type": "code", - "execution_count": 30, - "metadata": {}, - "outputs": [], - "source": [ - "X_batch = np.array([\n", - " # step 0 step 1\n", - " [[0, 1, 2], [9, 8, 7]], # instance 1\n", - " [[3, 4, 5], [0, 0, 0]], # instance 2 (padded with zero vectors)\n", - " [[6, 7, 8], [6, 5, 4]], # instance 3\n", - " [[9, 0, 1], [3, 2, 1]], # instance 4\n", - " ])\n", - "seq_length_batch = np.array([2, 1, 2, 2])" - ] - }, - { - "cell_type": "code", - "execution_count": 31, - "metadata": {}, - "outputs": [], - "source": [ - "with tf.Session() as sess:\n", - " init.run()\n", - " outputs_val, states_val = sess.run(\n", - " [outputs, states], feed_dict={X: X_batch, seq_length: seq_length_batch})" - ] - }, - { - "cell_type": "code", - "execution_count": 32, - "metadata": {}, - "outputs": [], - "source": [ - "print(outputs_val)" - ] - }, - { - "cell_type": "code", - "execution_count": 33, - "metadata": {}, - "outputs": [], - "source": [ - "print(states_val)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Training a sequence classifier" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Note: the book uses `tensorflow.contrib.layers.fully_connected()` rather than `tf.layers.dense()` (which did not exist when this chapter was written). It is now preferable to use `tf.layers.dense()`, because anything in the contrib module may change or be deleted without notice. The `dense()` function is almost identical to the `fully_connected()` function. The main differences relevant to this chapter are:\n", - "* several parameters are renamed: `scope` becomes `name`, `activation_fn` becomes `activation` (and similarly the `_fn` suffix is removed from other parameters such as `normalizer_fn`), `weights_initializer` becomes `kernel_initializer`, etc.\n", - "* the default `activation` is now `None` rather than `tf.nn.relu`." - ] - }, - { - "cell_type": "code", - "execution_count": 34, - "metadata": {}, - "outputs": [], - "source": [ - "reset_graph()\n", - "\n", - "n_steps = 28\n", - "n_inputs = 28\n", - "n_neurons = 150\n", - "n_outputs = 10\n", - "\n", - "learning_rate = 0.001\n", - "\n", - "X = tf.placeholder(tf.float32, [None, n_steps, n_inputs])\n", - "y = tf.placeholder(tf.int32, [None])\n", - "\n", - "basic_cell = tf.nn.rnn_cell.BasicRNNCell(num_units=n_neurons)\n", - "outputs, states = tf.nn.dynamic_rnn(basic_cell, X, dtype=tf.float32)\n", - "\n", - "logits = tf.layers.dense(states, n_outputs)\n", - "xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y,\n", - " logits=logits)\n", - "loss = tf.reduce_mean(xentropy)\n", - "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n", - "training_op = optimizer.minimize(loss)\n", - "correct = tf.nn.in_top_k(logits, y, 1)\n", - "accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n", - "\n", - "init = tf.global_variables_initializer()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**Warning**: `tf.examples.tutorials.mnist` is deprecated. We will use `tf.keras.datasets.mnist` instead." - ] - }, - { - "cell_type": "code", - "execution_count": 35, - "metadata": {}, - "outputs": [], - "source": [ - "(X_train, y_train), (X_test, y_test) = tf.keras.datasets.mnist.load_data()\n", - "X_train = X_train.astype(np.float32).reshape(-1, 28*28) / 255.0\n", - "X_test = X_test.astype(np.float32).reshape(-1, 28*28) / 255.0\n", - "y_train = y_train.astype(np.int32)\n", - "y_test = y_test.astype(np.int32)\n", - "X_valid, X_train = X_train[:5000], X_train[5000:]\n", - "y_valid, y_train = y_train[:5000], y_train[5000:]" - ] - }, - { - "cell_type": "code", - "execution_count": 36, - "metadata": {}, - "outputs": [], - "source": [ - "def shuffle_batch(X, y, batch_size):\n", - " rnd_idx = np.random.permutation(len(X))\n", - " n_batches = len(X) // batch_size\n", - " for batch_idx in np.array_split(rnd_idx, n_batches):\n", - " X_batch, y_batch = X[batch_idx], y[batch_idx]\n", - " yield X_batch, y_batch" - ] - }, - { - "cell_type": "code", - "execution_count": 37, - "metadata": {}, - "outputs": [], - "source": [ - "X_test = X_test.reshape((-1, n_steps, n_inputs))" - ] - }, - { - "cell_type": "code", - "execution_count": 38, - "metadata": {}, - "outputs": [], - "source": [ - "n_epochs = 100\n", - "batch_size = 150\n", - "\n", - "with tf.Session() as sess:\n", - " init.run()\n", - " for epoch in range(n_epochs):\n", - " for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size):\n", - " X_batch = X_batch.reshape((-1, n_steps, n_inputs))\n", - " sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n", - " acc_batch = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n", - " acc_test = accuracy.eval(feed_dict={X: X_test, y: y_test})\n", - " print(epoch, \"Last batch accuracy:\", acc_batch, \"Test accuracy:\", acc_test)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Multi-layer RNN" - ] - }, - { - "cell_type": "code", - "execution_count": 39, - "metadata": {}, - "outputs": [], - "source": [ - "reset_graph()\n", - "\n", - "n_steps = 28\n", - "n_inputs = 28\n", - "n_outputs = 10\n", - "\n", - "learning_rate = 0.001\n", - "\n", - "X = tf.placeholder(tf.float32, [None, n_steps, n_inputs])\n", - "y = tf.placeholder(tf.int32, [None])" - ] - }, - { - "cell_type": "code", - "execution_count": 40, - "metadata": {}, - "outputs": [], - "source": [ - "n_neurons = 100\n", - "n_layers = 3\n", - "\n", - "layers = [tf.nn.rnn_cell.BasicRNNCell(num_units=n_neurons,\n", - " activation=tf.nn.relu)\n", - " for layer in range(n_layers)]\n", - "multi_layer_cell = tf.nn.rnn_cell.MultiRNNCell(layers)\n", - "outputs, states = tf.nn.dynamic_rnn(multi_layer_cell, X, dtype=tf.float32)" - ] - }, - { - "cell_type": "code", - "execution_count": 41, - "metadata": {}, - "outputs": [], - "source": [ - "states_concat = tf.concat(axis=1, values=states)\n", - "logits = tf.layers.dense(states_concat, n_outputs)\n", - "xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n", - "loss = tf.reduce_mean(xentropy)\n", - "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n", - "training_op = optimizer.minimize(loss)\n", - "correct = tf.nn.in_top_k(logits, y, 1)\n", - "accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n", - "\n", - "init = tf.global_variables_initializer()" - ] - }, - { - "cell_type": "code", - "execution_count": 42, - "metadata": {}, - "outputs": [], - "source": [ - "n_epochs = 10\n", - "batch_size = 150\n", - "\n", - "with tf.Session() as sess:\n", - " init.run()\n", - " for epoch in range(n_epochs):\n", - " for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size):\n", - " X_batch = X_batch.reshape((-1, n_steps, n_inputs))\n", - " sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n", - " acc_batch = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n", - " acc_test = accuracy.eval(feed_dict={X: X_test, y: y_test})\n", - " print(epoch, \"Last batch accuracy:\", acc_batch, \"Test accuracy:\", acc_test)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Time series" - ] - }, - { - "cell_type": "code", - "execution_count": 43, - "metadata": {}, - "outputs": [], - "source": [ - "t_min, t_max = 0, 30\n", - "resolution = 0.1\n", - "\n", - "def time_series(t):\n", - " return t * np.sin(t) / 3 + 2 * np.sin(t*5)\n", - "\n", - "def next_batch(batch_size, n_steps):\n", - " t0 = np.random.rand(batch_size, 1) * (t_max - t_min - n_steps * resolution)\n", - " Ts = t0 + np.arange(0., n_steps + 1) * resolution\n", - " ys = time_series(Ts)\n", - " return ys[:, :-1].reshape(-1, n_steps, 1), ys[:, 1:].reshape(-1, n_steps, 1)" - ] - }, - { - "cell_type": "code", - "execution_count": 44, - "metadata": {}, - "outputs": [], - "source": [ - "t = np.linspace(t_min, t_max, int((t_max - t_min) / resolution))\n", - "\n", - "n_steps = 20\n", - "t_instance = np.linspace(12.2, 12.2 + resolution * (n_steps + 1), n_steps + 1)\n", - "\n", - "plt.figure(figsize=(11,4))\n", - "plt.subplot(121)\n", - "plt.title(\"A time series (generated)\", fontsize=14)\n", - "plt.plot(t, time_series(t), label=r\"$t . \\sin(t) / 3 + 2 . \\sin(5t)$\")\n", - "plt.plot(t_instance[:-1], time_series(t_instance[:-1]), \"b-\", linewidth=3, label=\"A training instance\")\n", - "plt.legend(loc=\"lower left\", fontsize=14)\n", - "plt.axis([0, 30, -17, 13])\n", - "plt.xlabel(\"Time\")\n", - "plt.ylabel(\"Value\")\n", - "\n", - "plt.subplot(122)\n", - "plt.title(\"A training instance\", fontsize=14)\n", - "plt.plot(t_instance[:-1], time_series(t_instance[:-1]), \"bo\", markersize=10, label=\"instance\")\n", - "plt.plot(t_instance[1:], time_series(t_instance[1:]), \"w*\", markersize=10, label=\"target\")\n", - "plt.legend(loc=\"upper left\")\n", - "plt.xlabel(\"Time\")\n", - "\n", - "\n", - "save_fig(\"time_series_plot\")\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": 45, - "metadata": {}, - "outputs": [], - "source": [ - "X_batch, y_batch = next_batch(1, n_steps)" - ] - }, - { - "cell_type": "code", - "execution_count": 46, - "metadata": {}, - "outputs": [], - "source": [ - "np.c_[X_batch[0], y_batch[0]]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Using an `OuputProjectionWrapper`" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's create the RNN. It will contain 100 recurrent neurons and we will unroll it over 20 time steps since each traiing instance will be 20 inputs long. Each input will contain only one feature (the value at that time). The targets are also sequences of 20 inputs, each containing a sigle value:" - ] - }, - { - "cell_type": "code", - "execution_count": 47, - "metadata": {}, - "outputs": [], - "source": [ - "reset_graph()\n", - "\n", - "n_steps = 20\n", - "n_inputs = 1\n", - "n_neurons = 100\n", - "n_outputs = 1\n", - "\n", - "X = tf.placeholder(tf.float32, [None, n_steps, n_inputs])\n", - "y = tf.placeholder(tf.float32, [None, n_steps, n_outputs])\n", - "\n", - "cell = tf.nn.rnn_cell.BasicRNNCell(num_units=n_neurons, activation=tf.nn.relu)\n", - "outputs, states = tf.nn.dynamic_rnn(cell, X, dtype=tf.float32)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "At each time step we now have an output vector of size 100. But what we actually want is a single output value at each time step. The simplest solution is to wrap the cell in an `OutputProjectionWrapper`." - ] - }, - { - "cell_type": "code", - "execution_count": 48, - "metadata": {}, - "outputs": [], - "source": [ - "reset_graph()\n", - "\n", - "n_steps = 20\n", - "n_inputs = 1\n", - "n_neurons = 100\n", - "n_outputs = 1\n", - "\n", - "X = tf.placeholder(tf.float32, [None, n_steps, n_inputs])\n", - "y = tf.placeholder(tf.float32, [None, n_steps, n_outputs])" - ] - }, - { - "cell_type": "code", - "execution_count": 50, - "metadata": {}, - "outputs": [], - "source": [ - "cell = tf.contrib.rnn.OutputProjectionWrapper(\n", - " tf.nn.rnn_cell.BasicRNNCell(num_units=n_neurons, activation=tf.nn.relu),\n", - " output_size=n_outputs)" - ] - }, - { - "cell_type": "code", - "execution_count": 51, - "metadata": {}, - "outputs": [], - "source": [ - "outputs, states = tf.nn.dynamic_rnn(cell, X, dtype=tf.float32)" - ] - }, - { - "cell_type": "code", - "execution_count": 52, - "metadata": {}, - "outputs": [], - "source": [ - "learning_rate = 0.001\n", - "\n", - "loss = tf.reduce_mean(tf.square(outputs - y)) # MSE\n", - "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n", - "training_op = optimizer.minimize(loss)\n", - "\n", - "init = tf.global_variables_initializer()" - ] - }, - { - "cell_type": "code", - "execution_count": 53, - "metadata": {}, - "outputs": [], - "source": [ - "saver = tf.train.Saver()" - ] - }, - { - "cell_type": "code", - "execution_count": 54, - "metadata": {}, - "outputs": [], - "source": [ - "n_iterations = 1500\n", - "batch_size = 50\n", - "\n", - "with tf.Session() as sess:\n", - " init.run()\n", - " for iteration in range(n_iterations):\n", - " X_batch, y_batch = next_batch(batch_size, n_steps)\n", - " sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n", - " if iteration % 100 == 0:\n", - " mse = loss.eval(feed_dict={X: X_batch, y: y_batch})\n", - " print(iteration, \"\\tMSE:\", mse)\n", - " \n", - " saver.save(sess, \"./my_time_series_model\") # not shown in the book" - ] - }, - { - "cell_type": "code", - "execution_count": 55, - "metadata": {}, - "outputs": [], - "source": [ - "with tf.Session() as sess: # not shown in the book\n", - " saver.restore(sess, \"./my_time_series_model\") # not shown\n", - "\n", - " X_new = time_series(np.array(t_instance[:-1].reshape(-1, n_steps, n_inputs)))\n", - " y_pred = sess.run(outputs, feed_dict={X: X_new})" - ] - }, - { - "cell_type": "code", - "execution_count": 56, - "metadata": {}, - "outputs": [], - "source": [ - "y_pred" - ] - }, - { - "cell_type": "code", - "execution_count": 57, - "metadata": {}, - "outputs": [], - "source": [ - "plt.title(\"Testing the model\", fontsize=14)\n", - "plt.plot(t_instance[:-1], time_series(t_instance[:-1]), \"bo\", markersize=10, label=\"instance\")\n", - "plt.plot(t_instance[1:], time_series(t_instance[1:]), \"w*\", markersize=10, label=\"target\")\n", - "plt.plot(t_instance[1:], y_pred[0,:,0], \"r.\", markersize=10, label=\"prediction\")\n", - "plt.legend(loc=\"upper left\")\n", - "plt.xlabel(\"Time\")\n", - "\n", - "save_fig(\"time_series_pred_plot\")\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Without using an `OutputProjectionWrapper`" - ] - }, - { - "cell_type": "code", - "execution_count": 58, - "metadata": {}, - "outputs": [], - "source": [ - "reset_graph()\n", - "\n", - "n_steps = 20\n", - "n_inputs = 1\n", - "n_neurons = 100\n", - "\n", - "X = tf.placeholder(tf.float32, [None, n_steps, n_inputs])\n", - "y = tf.placeholder(tf.float32, [None, n_steps, n_outputs])" - ] - }, - { - "cell_type": "code", - "execution_count": 59, - "metadata": {}, - "outputs": [], - "source": [ - "cell = tf.nn.rnn_cell.BasicRNNCell(num_units=n_neurons, activation=tf.nn.relu)\n", - "rnn_outputs, states = tf.nn.dynamic_rnn(cell, X, dtype=tf.float32)" - ] - }, - { - "cell_type": "code", - "execution_count": 60, - "metadata": {}, - "outputs": [], - "source": [ - "n_outputs = 1\n", - "learning_rate = 0.001" - ] - }, - { - "cell_type": "code", - "execution_count": 61, - "metadata": {}, - "outputs": [], - "source": [ - "stacked_rnn_outputs = tf.reshape(rnn_outputs, [-1, n_neurons])\n", - "stacked_outputs = tf.layers.dense(stacked_rnn_outputs, n_outputs)\n", - "outputs = tf.reshape(stacked_outputs, [-1, n_steps, n_outputs])" - ] - }, - { - "cell_type": "code", - "execution_count": 62, - "metadata": {}, - "outputs": [], - "source": [ - "loss = tf.reduce_mean(tf.square(outputs - y))\n", - "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n", - "training_op = optimizer.minimize(loss)\n", - "\n", - "init = tf.global_variables_initializer()\n", - "saver = tf.train.Saver()" - ] - }, - { - "cell_type": "code", - "execution_count": 63, - "metadata": {}, - "outputs": [], - "source": [ - "n_iterations = 1500\n", - "batch_size = 50\n", - "\n", - "with tf.Session() as sess:\n", - " init.run()\n", - " for iteration in range(n_iterations):\n", - " X_batch, y_batch = next_batch(batch_size, n_steps)\n", - " sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n", - " if iteration % 100 == 0:\n", - " mse = loss.eval(feed_dict={X: X_batch, y: y_batch})\n", - " print(iteration, \"\\tMSE:\", mse)\n", - " \n", - " X_new = time_series(np.array(t_instance[:-1].reshape(-1, n_steps, n_inputs)))\n", - " y_pred = sess.run(outputs, feed_dict={X: X_new})\n", - " \n", - " saver.save(sess, \"./my_time_series_model\")" - ] - }, - { - "cell_type": "code", - "execution_count": 64, - "metadata": {}, - "outputs": [], - "source": [ - "y_pred" - ] - }, - { - "cell_type": "code", - "execution_count": 65, - "metadata": {}, - "outputs": [], - "source": [ - "plt.title(\"Testing the model\", fontsize=14)\n", - "plt.plot(t_instance[:-1], time_series(t_instance[:-1]), \"bo\", markersize=10, label=\"instance\")\n", - "plt.plot(t_instance[1:], time_series(t_instance[1:]), \"w*\", markersize=10, label=\"target\")\n", - "plt.plot(t_instance[1:], y_pred[0,:,0], \"r.\", markersize=10, label=\"prediction\")\n", - "plt.legend(loc=\"upper left\")\n", - "plt.xlabel(\"Time\")\n", - "\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Generating a creative new sequence" - ] - }, - { - "cell_type": "code", - "execution_count": 66, - "metadata": {}, - "outputs": [], - "source": [ - "with tf.Session() as sess: # not shown in the book\n", - " saver.restore(sess, \"./my_time_series_model\") # not shown\n", - "\n", - " sequence = [0.] * n_steps\n", - " for iteration in range(300):\n", - " X_batch = np.array(sequence[-n_steps:]).reshape(1, n_steps, 1)\n", - " y_pred = sess.run(outputs, feed_dict={X: X_batch})\n", - " sequence.append(y_pred[0, -1, 0])" - ] - }, - { - "cell_type": "code", - "execution_count": 67, - "metadata": {}, - "outputs": [], - "source": [ - "plt.figure(figsize=(8,4))\n", - "plt.plot(np.arange(len(sequence)), sequence, \"b-\")\n", - "plt.plot(t[:n_steps], sequence[:n_steps], \"b-\", linewidth=3)\n", - "plt.xlabel(\"Time\")\n", - "plt.ylabel(\"Value\")\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": 68, - "metadata": {}, - "outputs": [], - "source": [ - "with tf.Session() as sess:\n", - " saver.restore(sess, \"./my_time_series_model\")\n", - "\n", - " sequence1 = [0. for i in range(n_steps)]\n", - " for iteration in range(len(t) - n_steps):\n", - " X_batch = np.array(sequence1[-n_steps:]).reshape(1, n_steps, 1)\n", - " y_pred = sess.run(outputs, feed_dict={X: X_batch})\n", - " sequence1.append(y_pred[0, -1, 0])\n", - "\n", - " sequence2 = [time_series(i * resolution + t_min + (t_max-t_min/3)) for i in range(n_steps)]\n", - " for iteration in range(len(t) - n_steps):\n", - " X_batch = np.array(sequence2[-n_steps:]).reshape(1, n_steps, 1)\n", - " y_pred = sess.run(outputs, feed_dict={X: X_batch})\n", - " sequence2.append(y_pred[0, -1, 0])\n", - "\n", - "plt.figure(figsize=(11,4))\n", - "plt.subplot(121)\n", - "plt.plot(t, sequence1, \"b-\")\n", - "plt.plot(t[:n_steps], sequence1[:n_steps], \"b-\", linewidth=3)\n", - "plt.xlabel(\"Time\")\n", - "plt.ylabel(\"Value\")\n", - "\n", - "plt.subplot(122)\n", - "plt.plot(t, sequence2, \"b-\")\n", - "plt.plot(t[:n_steps], sequence2[:n_steps], \"b-\", linewidth=3)\n", - "plt.xlabel(\"Time\")\n", - "save_fig(\"creative_sequence_plot\")\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Deep RNN" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## MultiRNNCell" - ] - }, - { - "cell_type": "code", - "execution_count": 69, - "metadata": {}, - "outputs": [], - "source": [ - "reset_graph()\n", - "\n", - "n_inputs = 2\n", - "n_steps = 5\n", - "\n", - "X = tf.placeholder(tf.float32, [None, n_steps, n_inputs])" - ] - }, - { - "cell_type": "code", - "execution_count": 70, - "metadata": {}, - "outputs": [], - "source": [ - "n_neurons = 100\n", - "n_layers = 3\n", - "\n", - "layers = [tf.nn.rnn_cell.BasicRNNCell(num_units=n_neurons)\n", - " for layer in range(n_layers)]\n", - "multi_layer_cell = tf.nn.rnn_cell.MultiRNNCell(layers)\n", - "outputs, states = tf.nn.dynamic_rnn(multi_layer_cell, X, dtype=tf.float32)" - ] - }, - { - "cell_type": "code", - "execution_count": 71, - "metadata": {}, - "outputs": [], - "source": [ - "init = tf.global_variables_initializer()" - ] - }, - { - "cell_type": "code", - "execution_count": 72, - "metadata": {}, - "outputs": [], - "source": [ - "X_batch = np.random.rand(2, n_steps, n_inputs)" - ] - }, - { - "cell_type": "code", - "execution_count": 73, - "metadata": {}, - "outputs": [], - "source": [ - "with tf.Session() as sess:\n", - " init.run()\n", - " outputs_val, states_val = sess.run([outputs, states], feed_dict={X: X_batch})" - ] - }, - { - "cell_type": "code", - "execution_count": 74, - "metadata": {}, - "outputs": [], - "source": [ - "outputs_val.shape" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Distributing a Deep RNN Across Multiple GPUs" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Do **NOT** do this:" - ] - }, - { - "cell_type": "code", - "execution_count": 75, - "metadata": {}, - "outputs": [], - "source": [ - "with tf.device(\"/gpu:0\"): # BAD! This is ignored.\n", - " layer1 = tf.nn.rnn_cell.BasicRNNCell(num_units=n_neurons)\n", - "\n", - "with tf.device(\"/gpu:1\"): # BAD! Ignored again.\n", - " layer2 = tf.nn.rnn_cell.BasicRNNCell(num_units=n_neurons)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Instead, you need a `DeviceCellWrapper`:" - ] - }, - { - "cell_type": "code", - "execution_count": 76, - "metadata": {}, - "outputs": [], - "source": [ - "import tensorflow as tf\n", - "\n", - "class DeviceCellWrapper(tf.nn.rnn_cell.RNNCell):\n", - " def __init__(self, device, cell):\n", - " self._cell = cell\n", - " self._device = device\n", - "\n", - " @property\n", - " def state_size(self):\n", - " return self._cell.state_size\n", - "\n", - " @property\n", - " def output_size(self):\n", - " return self._cell.output_size\n", - "\n", - " def __call__(self, inputs, state, scope=None):\n", - " with tf.device(self._device):\n", - " return self._cell(inputs, state, scope)" - ] - }, - { - "cell_type": "code", - "execution_count": 77, - "metadata": {}, - "outputs": [], - "source": [ - "reset_graph()\n", - "\n", - "n_inputs = 5\n", - "n_steps = 20\n", - "n_neurons = 100\n", - "\n", - "X = tf.placeholder(tf.float32, shape=[None, n_steps, n_inputs])" - ] - }, - { - "cell_type": "code", - "execution_count": 78, - "metadata": {}, - "outputs": [], - "source": [ - "devices = [\"/cpu:0\", \"/cpu:0\", \"/cpu:0\"] # replace with [\"/gpu:0\", \"/gpu:1\", \"/gpu:2\"] if you have 3 GPUs\n", - "cells = [DeviceCellWrapper(dev,tf.nn.rnn_cell.BasicRNNCell(num_units=n_neurons))\n", - " for dev in devices]\n", - "multi_layer_cell = tf.nn.rnn_cell.MultiRNNCell(cells)\n", - "outputs, states = tf.nn.dynamic_rnn(multi_layer_cell, X, dtype=tf.float32)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Alternatively, since TensorFlow 1.1, you can use the `tf.contrib.rnn.DeviceWrapper` class (alias `tf.nn.rnn_cell.DeviceWrapper` since TF 1.2)." - ] - }, - { - "cell_type": "code", - "execution_count": 79, - "metadata": {}, - "outputs": [], - "source": [ - "init = tf.global_variables_initializer()" - ] - }, - { - "cell_type": "code", - "execution_count": 80, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "with tf.Session() as sess:\n", - " init.run()\n", - " print(sess.run(outputs, feed_dict={X: np.random.rand(2, n_steps, n_inputs)}))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Dropout" - ] - }, - { - "cell_type": "code", - "execution_count": 81, - "metadata": {}, - "outputs": [], - "source": [ - "reset_graph()\n", - "\n", - "n_inputs = 1\n", - "n_neurons = 100\n", - "n_layers = 3\n", - "n_steps = 20\n", - "n_outputs = 1" - ] - }, - { - "cell_type": "code", - "execution_count": 82, - "metadata": {}, - "outputs": [], - "source": [ - "X = tf.placeholder(tf.float32, [None, n_steps, n_inputs])\n", - "y = tf.placeholder(tf.float32, [None, n_steps, n_outputs])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Note: the `input_keep_prob` parameter can be a placeholder, making it possible to set it to any value you want during training, and to 1.0 during testing (effectively turning dropout off). This is a much more elegant solution than what was recommended in earlier versions of the book (i.e., writing your own wrapper class or having a separate model for training and testing). Thanks to Shen Cheng for bringing this to my attention." - ] - }, - { - "cell_type": "code", - "execution_count": 83, - "metadata": {}, - "outputs": [], - "source": [ - "keep_prob = tf.placeholder_with_default(1.0, shape=())\n", - "cells = [tf.nn.rnn_cell.BasicRNNCell(num_units=n_neurons)\n", - " for layer in range(n_layers)]\n", - "cells_drop = [tf.nn.rnn_cell.DropoutWrapper(cell, input_keep_prob=keep_prob)\n", - " for cell in cells]\n", - "multi_layer_cell = tf.nn.rnn_cell.MultiRNNCell(cells_drop)\n", - "rnn_outputs, states = tf.nn.dynamic_rnn(multi_layer_cell, X, dtype=tf.float32)" - ] - }, - { - "cell_type": "code", - "execution_count": 84, - "metadata": {}, - "outputs": [], - "source": [ - "learning_rate = 0.01\n", - "\n", - "stacked_rnn_outputs = tf.reshape(rnn_outputs, [-1, n_neurons])\n", - "stacked_outputs = tf.layers.dense(stacked_rnn_outputs, n_outputs)\n", - "outputs = tf.reshape(stacked_outputs, [-1, n_steps, n_outputs])\n", - "\n", - "loss = tf.reduce_mean(tf.square(outputs - y))\n", - "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n", - "training_op = optimizer.minimize(loss)\n", - "\n", - "init = tf.global_variables_initializer()\n", - "saver = tf.train.Saver()" - ] - }, - { - "cell_type": "code", - "execution_count": 85, - "metadata": {}, - "outputs": [], - "source": [ - "n_iterations = 1500\n", - "batch_size = 50\n", - "train_keep_prob = 0.5\n", - "\n", - "with tf.Session() as sess:\n", - " init.run()\n", - " for iteration in range(n_iterations):\n", - " X_batch, y_batch = next_batch(batch_size, n_steps)\n", - " _, mse = sess.run([training_op, loss],\n", - " feed_dict={X: X_batch, y: y_batch,\n", - " keep_prob: train_keep_prob})\n", - " if iteration % 100 == 0: # not shown in the book\n", - " print(iteration, \"Training MSE:\", mse) # not shown\n", - " \n", - " saver.save(sess, \"./my_dropout_time_series_model\")" - ] - }, - { - "cell_type": "code", - "execution_count": 86, - "metadata": {}, - "outputs": [], - "source": [ - "with tf.Session() as sess:\n", - " saver.restore(sess, \"./my_dropout_time_series_model\")\n", - "\n", - " X_new = time_series(np.array(t_instance[:-1].reshape(-1, n_steps, n_inputs)))\n", - " y_pred = sess.run(outputs, feed_dict={X: X_new})" - ] - }, - { - "cell_type": "code", - "execution_count": 87, - "metadata": {}, - "outputs": [], - "source": [ - "plt.title(\"Testing the model\", fontsize=14)\n", - "plt.plot(t_instance[:-1], time_series(t_instance[:-1]), \"bo\", markersize=10, label=\"instance\")\n", - "plt.plot(t_instance[1:], time_series(t_instance[1:]), \"w*\", markersize=10, label=\"target\")\n", - "plt.plot(t_instance[1:], y_pred[0,:,0], \"r.\", markersize=10, label=\"prediction\")\n", - "plt.legend(loc=\"upper left\")\n", - "plt.xlabel(\"Time\")\n", - "\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Oops, it seems that Dropout does not help at all in this particular case. :/" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# LSTM" - ] - }, - { - "cell_type": "code", - "execution_count": 88, - "metadata": {}, - "outputs": [], - "source": [ - "reset_graph()\n", - "\n", - "lstm_cell = tf.nn.rnn_cell.BasicLSTMCell(num_units=n_neurons)" - ] - }, - { - "cell_type": "code", - "execution_count": 89, - "metadata": {}, - "outputs": [], - "source": [ - "n_steps = 28\n", - "n_inputs = 28\n", - "n_neurons = 150\n", - "n_outputs = 10\n", - "n_layers = 3\n", - "\n", - "learning_rate = 0.001\n", - "\n", - "X = tf.placeholder(tf.float32, [None, n_steps, n_inputs])\n", - "y = tf.placeholder(tf.int32, [None])\n", - "\n", - "lstm_cells = [tf.nn.rnn_cell.BasicLSTMCell(num_units=n_neurons)\n", - " for layer in range(n_layers)]\n", - "multi_cell = tf.nn.rnn_cell.MultiRNNCell(lstm_cells)\n", - "outputs, states = tf.nn.dynamic_rnn(multi_cell, X, dtype=tf.float32)\n", - "top_layer_h_state = states[-1][1]\n", - "logits = tf.layers.dense(top_layer_h_state, n_outputs, name=\"softmax\")\n", - "xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n", - "loss = tf.reduce_mean(xentropy, name=\"loss\")\n", - "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n", - "training_op = optimizer.minimize(loss)\n", - "correct = tf.nn.in_top_k(logits, y, 1)\n", - "accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n", - " \n", - "init = tf.global_variables_initializer()" - ] - }, - { - "cell_type": "code", - "execution_count": 90, - "metadata": {}, - "outputs": [], - "source": [ - "states" - ] - }, - { - "cell_type": "code", - "execution_count": 91, - "metadata": {}, - "outputs": [], - "source": [ - "top_layer_h_state" - ] - }, - { - "cell_type": "code", - "execution_count": 92, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "n_epochs = 10\n", - "batch_size = 150\n", - "\n", - "with tf.Session() as sess:\n", - " init.run()\n", - " for epoch in range(n_epochs):\n", - " for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size):\n", - " X_batch = X_batch.reshape((-1, n_steps, n_inputs))\n", - " sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n", - " acc_batch = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n", - " acc_test = accuracy.eval(feed_dict={X: X_test, y: y_test})\n", - " print(epoch, \"Last batch accuracy:\", acc_batch, \"Test accuracy:\", acc_test)" - ] - }, - { - "cell_type": "code", - "execution_count": 93, - "metadata": {}, - "outputs": [], - "source": [ - "lstm_cell = tf.nn.rnn_cell.LSTMCell(num_units=n_neurons, use_peepholes=True)" - ] - }, - { - "cell_type": "code", - "execution_count": 94, - "metadata": {}, - "outputs": [], - "source": [ - "gru_cell = tf.nn.rnn_cell.GRUCell(num_units=n_neurons)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Embeddings" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "This section is based on TensorFlow's [Word2Vec tutorial](https://www.tensorflow.org/versions/r0.11/tutorials/word2vec/index.html)." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Fetch the data" - ] - }, - { - "cell_type": "code", - "execution_count": 95, - "metadata": {}, - "outputs": [], - "source": [ - "from six.moves import urllib\n", - "\n", - "import errno\n", - "import os\n", - "import zipfile\n", - "\n", - "WORDS_PATH = \"datasets/words\"\n", - "WORDS_URL = 'http://mattmahoney.net/dc/text8.zip'\n", - "\n", - "def mkdir_p(path):\n", - " \"\"\"Create directories, ok if they already exist.\n", - " \n", - " This is for python 2 support. In python >=3.2, simply use:\n", - " >>> os.makedirs(path, exist_ok=True)\n", - " \"\"\"\n", - " try:\n", - " os.makedirs(path)\n", - " except OSError as exc:\n", - " if exc.errno == errno.EEXIST and os.path.isdir(path):\n", - " pass\n", - " else:\n", - " raise\n", - "\n", - "def fetch_words_data(words_url=WORDS_URL, words_path=WORDS_PATH):\n", - " os.makedirs(words_path, exist_ok=True)\n", - " zip_path = os.path.join(words_path, \"words.zip\")\n", - " if not os.path.exists(zip_path):\n", - " urllib.request.urlretrieve(words_url, zip_path)\n", - " with zipfile.ZipFile(zip_path) as f:\n", - " data = f.read(f.namelist()[0])\n", - " return data.decode(\"ascii\").split()" - ] - }, - { - "cell_type": "code", - "execution_count": 96, - "metadata": {}, - "outputs": [], - "source": [ - "words = fetch_words_data()" - ] - }, - { - "cell_type": "code", - "execution_count": 97, - "metadata": {}, - "outputs": [], - "source": [ - "words[:5]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Build the dictionary" - ] - }, - { - "cell_type": "code", - "execution_count": 98, - "metadata": {}, - "outputs": [], - "source": [ - "from collections import Counter\n", - "\n", - "vocabulary_size = 50000\n", - "\n", - "vocabulary = [(\"UNK\", None)] + Counter(words).most_common(vocabulary_size - 1)\n", - "vocabulary = np.array([word for word, _ in vocabulary])\n", - "dictionary = {word: code for code, word in enumerate(vocabulary)}\n", - "data = np.array([dictionary.get(word, 0) for word in words])" - ] - }, - { - "cell_type": "code", - "execution_count": 99, - "metadata": {}, - "outputs": [], - "source": [ - "\" \".join(words[:9]), data[:9]" - ] - }, - { - "cell_type": "code", - "execution_count": 100, - "metadata": {}, - "outputs": [], - "source": [ - "\" \".join([vocabulary[word_index] for word_index in [5241, 3081, 12, 6, 195, 2, 3134, 46, 59]])" - ] - }, - { - "cell_type": "code", - "execution_count": 101, - "metadata": {}, - "outputs": [], - "source": [ - "words[24], data[24]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Generate batches" - ] - }, - { - "cell_type": "code", - "execution_count": 102, - "metadata": {}, - "outputs": [], - "source": [ - "from collections import deque\n", - "\n", - "def generate_batch(batch_size, num_skips, skip_window):\n", - " global data_index\n", - " assert batch_size % num_skips == 0\n", - " assert num_skips <= 2 * skip_window\n", - " batch = np.ndarray(shape=[batch_size], dtype=np.int32)\n", - " labels = np.ndarray(shape=[batch_size, 1], dtype=np.int32)\n", - " span = 2 * skip_window + 1 # [ skip_window target skip_window ]\n", - " buffer = deque(maxlen=span)\n", - " for _ in range(span):\n", - " buffer.append(data[data_index])\n", - " data_index = (data_index + 1) % len(data)\n", - " for i in range(batch_size // num_skips):\n", - " target = skip_window # target label at the center of the buffer\n", - " targets_to_avoid = [ skip_window ]\n", - " for j in range(num_skips):\n", - " while target in targets_to_avoid:\n", - " target = np.random.randint(0, span)\n", - " targets_to_avoid.append(target)\n", - " batch[i * num_skips + j] = buffer[skip_window]\n", - " labels[i * num_skips + j, 0] = buffer[target]\n", - " buffer.append(data[data_index])\n", - " data_index = (data_index + 1) % len(data)\n", - " return batch, labels" - ] - }, - { - "cell_type": "code", - "execution_count": 103, - "metadata": {}, - "outputs": [], - "source": [ - "np.random.seed(42)" - ] - }, - { - "cell_type": "code", - "execution_count": 104, - "metadata": {}, - "outputs": [], - "source": [ - "data_index = 0\n", - "batch, labels = generate_batch(8, 2, 1)" - ] - }, - { - "cell_type": "code", - "execution_count": 105, - "metadata": {}, - "outputs": [], - "source": [ - "batch, [vocabulary[word] for word in batch]" - ] - }, - { - "cell_type": "code", - "execution_count": 106, - "metadata": {}, - "outputs": [], - "source": [ - "labels, [vocabulary[word] for word in labels[:, 0]]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Build the model" - ] - }, - { - "cell_type": "code", - "execution_count": 107, - "metadata": {}, - "outputs": [], - "source": [ - "batch_size = 128\n", - "embedding_size = 128 # Dimension of the embedding vector.\n", - "skip_window = 1 # How many words to consider left and right.\n", - "num_skips = 2 # How many times to reuse an input to generate a label.\n", - "\n", - "# We pick a random validation set to sample nearest neighbors. Here we limit the\n", - "# validation samples to the words that have a low numeric ID, which by\n", - "# construction are also the most frequent.\n", - "valid_size = 16 # Random set of words to evaluate similarity on.\n", - "valid_window = 100 # Only pick dev samples in the head of the distribution.\n", - "valid_examples = np.random.choice(valid_window, valid_size, replace=False)\n", - "num_sampled = 64 # Number of negative examples to sample.\n", - "\n", - "learning_rate = 0.01" - ] - }, - { - "cell_type": "code", - "execution_count": 108, - "metadata": {}, - "outputs": [], - "source": [ - "reset_graph()\n", - "\n", - "# Input data.\n", - "train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1])\n", - "valid_dataset = tf.constant(valid_examples, dtype=tf.int32)" - ] - }, - { - "cell_type": "code", - "execution_count": 109, - "metadata": {}, - "outputs": [], - "source": [ - "vocabulary_size = 50000\n", - "embedding_size = 150\n", - "\n", - "# Look up embeddings for inputs.\n", - "init_embeds = tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0)\n", - "embeddings = tf.Variable(init_embeds)" - ] - }, - { - "cell_type": "code", - "execution_count": 110, - "metadata": {}, - "outputs": [], - "source": [ - "train_inputs = tf.placeholder(tf.int32, shape=[None])\n", - "embed = tf.nn.embedding_lookup(embeddings, train_inputs)" - ] - }, - { - "cell_type": "code", - "execution_count": 111, - "metadata": {}, - "outputs": [], - "source": [ - "# Construct the variables for the NCE loss\n", - "nce_weights = tf.Variable(\n", - " tf.truncated_normal([vocabulary_size, embedding_size],\n", - " stddev=1.0 / np.sqrt(embedding_size)))\n", - "nce_biases = tf.Variable(tf.zeros([vocabulary_size]))\n", - "\n", - "# Compute the average NCE loss for the batch.\n", - "# tf.nce_loss automatically draws a new sample of the negative labels each\n", - "# time we evaluate the loss.\n", - "loss = tf.reduce_mean(\n", - " tf.nn.nce_loss(nce_weights, nce_biases, train_labels, embed,\n", - " num_sampled, vocabulary_size))\n", - "\n", - "# Construct the Adam optimizer\n", - "optimizer = tf.train.AdamOptimizer(learning_rate)\n", - "training_op = optimizer.minimize(loss)\n", - "\n", - "# Compute the cosine similarity between minibatch examples and all embeddings.\n", - "norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), axis=1, keepdims=True))\n", - "normalized_embeddings = embeddings / norm\n", - "valid_embeddings = tf.nn.embedding_lookup(normalized_embeddings, valid_dataset)\n", - "similarity = tf.matmul(valid_embeddings, normalized_embeddings, transpose_b=True)\n", - "\n", - "# Add variable initializer.\n", - "init = tf.global_variables_initializer()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Train the model" - ] - }, - { - "cell_type": "code", - "execution_count": 112, - "metadata": {}, - "outputs": [], - "source": [ - "num_steps = 10001\n", - "\n", - "with tf.Session() as session:\n", - " init.run()\n", - "\n", - " average_loss = 0\n", - " for step in range(num_steps):\n", - " print(\"\\rIteration: {}\".format(step), end=\"\\t\")\n", - " batch_inputs, batch_labels = generate_batch(batch_size, num_skips, skip_window)\n", - " feed_dict = {train_inputs : batch_inputs, train_labels : batch_labels}\n", - "\n", - " # We perform one update step by evaluating the training op (including it\n", - " # in the list of returned values for session.run()\n", - " _, loss_val = session.run([training_op, loss], feed_dict=feed_dict)\n", - " average_loss += loss_val\n", - "\n", - " if step % 2000 == 0:\n", - " if step > 0:\n", - " average_loss /= 2000\n", - " # The average loss is an estimate of the loss over the last 2000 batches.\n", - " print(\"Average loss at step \", step, \": \", average_loss)\n", - " average_loss = 0\n", - "\n", - " # Note that this is expensive (~20% slowdown if computed every 500 steps)\n", - " if step % 10000 == 0:\n", - " sim = similarity.eval()\n", - " for i in range(valid_size):\n", - " valid_word = vocabulary[valid_examples[i]]\n", - " top_k = 8 # number of nearest neighbors\n", - " nearest = (-sim[i, :]).argsort()[1:top_k+1]\n", - " log_str = \"Nearest to %s:\" % valid_word\n", - " for k in range(top_k):\n", - " close_word = vocabulary[nearest[k]]\n", - " log_str = \"%s %s,\" % (log_str, close_word)\n", - " print(log_str)\n", - "\n", - " final_embeddings = normalized_embeddings.eval()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's save the final embeddings (of course you can use a TensorFlow `Saver` if you prefer):" - ] - }, - { - "cell_type": "code", - "execution_count": 113, - "metadata": {}, - "outputs": [], - "source": [ - "np.save(\"./my_final_embeddings.npy\", final_embeddings)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Plot the embeddings" - ] - }, - { - "cell_type": "code", - "execution_count": 114, - "metadata": {}, - "outputs": [], - "source": [ - "def plot_with_labels(low_dim_embs, labels):\n", - " assert low_dim_embs.shape[0] >= len(labels), \"More labels than embeddings\"\n", - " plt.figure(figsize=(18, 18)) #in inches\n", - " for i, label in enumerate(labels):\n", - " x, y = low_dim_embs[i,:]\n", - " plt.scatter(x, y)\n", - " plt.annotate(label,\n", - " xy=(x, y),\n", - " xytext=(5, 2),\n", - " textcoords='offset points',\n", - " ha='right',\n", - " va='bottom')" - ] - }, - { - "cell_type": "code", - "execution_count": 115, - "metadata": {}, - "outputs": [], - "source": [ - "from sklearn.manifold import TSNE\n", - "\n", - "tsne = TSNE(perplexity=30, n_components=2, init='pca', n_iter=5000)\n", - "plot_only = 500\n", - "low_dim_embs = tsne.fit_transform(final_embeddings[:plot_only,:])\n", - "labels = [vocabulary[i] for i in range(plot_only)]\n", - "plot_with_labels(low_dim_embs, labels)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Machine Translation" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The `basic_rnn_seq2seq()` function creates a simple Encoder/Decoder model: it first runs an RNN to encode `encoder_inputs` into a state vector, then runs a decoder initialized with the last encoder state on `decoder_inputs`. Encoder and decoder use the same RNN cell type but they don't share parameters." - ] - }, - { - "cell_type": "code", - "execution_count": 116, - "metadata": {}, - "outputs": [], - "source": [ - "import tensorflow as tf\n", - "reset_graph()\n", - "\n", - "n_steps = 50\n", - "n_neurons = 200\n", - "n_layers = 3\n", - "num_encoder_symbols = 20000\n", - "num_decoder_symbols = 20000\n", - "embedding_size = 150\n", - "learning_rate = 0.01\n", - "\n", - "X = tf.placeholder(tf.int32, [None, n_steps]) # English sentences\n", - "Y = tf.placeholder(tf.int32, [None, n_steps]) # French translations\n", - "W = tf.placeholder(tf.float32, [None, n_steps - 1, 1])\n", - "Y_input = Y[:, :-1]\n", - "Y_target = Y[:, 1:]\n", - "\n", - "encoder_inputs = tf.unstack(tf.transpose(X)) # list of 1D tensors\n", - "decoder_inputs = tf.unstack(tf.transpose(Y_input)) # list of 1D tensors\n", - "\n", - "lstm_cells = [tf.nn.rnn_cell.BasicLSTMCell(num_units=n_neurons)\n", - " for layer in range(n_layers)]\n", - "cell = tf.nn.rnn_cell.MultiRNNCell(lstm_cells)\n", - "\n", - "output_seqs, states = tf.contrib.legacy_seq2seq.embedding_rnn_seq2seq(\n", - " encoder_inputs,\n", - " decoder_inputs,\n", - " cell,\n", - " num_encoder_symbols,\n", - " num_decoder_symbols,\n", - " embedding_size)\n", - "\n", - "logits = tf.transpose(tf.unstack(output_seqs), perm=[1, 0, 2])" - ] - }, - { - "cell_type": "code", - "execution_count": 117, - "metadata": {}, - "outputs": [], - "source": [ - "logits_flat = tf.reshape(logits, [-1, num_decoder_symbols])\n", - "Y_target_flat = tf.reshape(Y_target, [-1])\n", - "W_flat = tf.reshape(W, [-1])\n", - "xentropy = W_flat * tf.nn.sparse_softmax_cross_entropy_with_logits(labels=Y_target_flat, logits=logits_flat)\n", - "loss = tf.reduce_mean(xentropy)\n", - "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n", - "training_op = optimizer.minimize(loss)\n", - "\n", - "init = tf.global_variables_initializer()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "collapsed": true - }, - "source": [ - "# Exercise solutions" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 1. to 6." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "See Appendix A." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 7. Embedded Reber Grammars" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "First we need to build a function that generates strings based on a grammar. The grammar will be represented as a list of possible transitions for each state. A transition specifies the string to output (or a grammar to generate it) and the next state." - ] - }, - { - "cell_type": "code", - "execution_count": 118, - "metadata": {}, - "outputs": [], - "source": [ - "np.random.seed(42)\n", - "\n", - "default_reber_grammar = [\n", - " [(\"B\", 1)], # (state 0) =B=>(state 1)\n", - " [(\"T\", 2), (\"P\", 3)], # (state 1) =T=>(state 2) or =P=>(state 3)\n", - " [(\"S\", 2), (\"X\", 4)], # (state 2) =S=>(state 2) or =X=>(state 4)\n", - " [(\"T\", 3), (\"V\", 5)], # and so on...\n", - " [(\"X\", 3), (\"S\", 6)],\n", - " [(\"P\", 4), (\"V\", 6)],\n", - " [(\"E\", None)]] # (state 6) =E=>(terminal state)\n", - "\n", - "embedded_reber_grammar = [\n", - " [(\"B\", 1)],\n", - " [(\"T\", 2), (\"P\", 3)],\n", - " [(default_reber_grammar, 4)],\n", - " [(default_reber_grammar, 5)],\n", - " [(\"T\", 6)],\n", - " [(\"P\", 6)],\n", - " [(\"E\", None)]]\n", - "\n", - "def generate_string(grammar):\n", - " state = 0\n", - " output = []\n", - " while state is not None:\n", - " index = np.random.randint(len(grammar[state]))\n", - " production, state = grammar[state][index]\n", - " if isinstance(production, list):\n", - " production = generate_string(grammar=production)\n", - " output.append(production)\n", - " return \"\".join(output)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's generate a few strings based on the default Reber grammar:" - ] - }, - { - "cell_type": "code", - "execution_count": 119, - "metadata": {}, - "outputs": [], - "source": [ - "for _ in range(25):\n", - " print(generate_string(default_reber_grammar), end=\" \")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Looks good. Now let's generate a few strings based on the embedded Reber grammar:" - ] - }, - { - "cell_type": "code", - "execution_count": 120, - "metadata": {}, - "outputs": [], - "source": [ - "for _ in range(25):\n", - " print(generate_string(embedded_reber_grammar), end=\" \")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Okay, now we need a function to generate strings that do not respect the grammar. We could generate a random string, but the task would be a bit too easy, so instead we will generate a string that respects the grammar, and we will corrupt it by changing just one character:" - ] - }, - { - "cell_type": "code", - "execution_count": 121, - "metadata": {}, - "outputs": [], - "source": [ - "def generate_corrupted_string(grammar, chars=\"BEPSTVX\"):\n", - " good_string = generate_string(grammar)\n", - " index = np.random.randint(len(good_string))\n", - " good_char = good_string[index]\n", - " bad_char = np.random.choice(sorted(set(chars) - set(good_char)))\n", - " return good_string[:index] + bad_char + good_string[index + 1:]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's look at a few corrupted strings:" - ] - }, - { - "cell_type": "code", - "execution_count": 122, - "metadata": {}, - "outputs": [], - "source": [ - "for _ in range(25):\n", - " print(generate_corrupted_string(embedded_reber_grammar), end=\" \")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "It's not possible to feed a string directly to an RNN: we need to convert it to a sequence of vectors, first. Each vector will represent a single letter, using a one-hot encoding. For example, the letter \"B\" will be represented as the vector `[1, 0, 0, 0, 0, 0, 0]`, the letter E will be represented as `[0, 1, 0, 0, 0, 0, 0]` and so on. Let's write a function that converts a string to a sequence of such one-hot vectors. Note that if the string is shorted than `n_steps`, it will be padded with zero vectors (later, we will tell TensorFlow how long each string actually is using the `sequence_length` parameter)." - ] - }, - { - "cell_type": "code", - "execution_count": 123, - "metadata": {}, - "outputs": [], - "source": [ - "def string_to_one_hot_vectors(string, n_steps, chars=\"BEPSTVX\"):\n", - " char_to_index = {char: index for index, char in enumerate(chars)}\n", - " output = np.zeros((n_steps, len(chars)), dtype=np.int32)\n", - " for index, char in enumerate(string):\n", - " output[index, char_to_index[char]] = 1.\n", - " return output" - ] - }, - { - "cell_type": "code", - "execution_count": 124, - "metadata": {}, - "outputs": [], - "source": [ - "string_to_one_hot_vectors(\"BTBTXSETE\", 12)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can now generate the dataset, with 50% good strings, and 50% bad strings:" - ] - }, - { - "cell_type": "code", - "execution_count": 125, - "metadata": {}, - "outputs": [], - "source": [ - "def generate_dataset(size):\n", - " good_strings = [generate_string(embedded_reber_grammar)\n", - " for _ in range(size // 2)]\n", - " bad_strings = [generate_corrupted_string(embedded_reber_grammar)\n", - " for _ in range(size - size // 2)]\n", - " all_strings = good_strings + bad_strings\n", - " n_steps = max([len(string) for string in all_strings])\n", - " X = np.array([string_to_one_hot_vectors(string, n_steps)\n", - " for string in all_strings])\n", - " seq_length = np.array([len(string) for string in all_strings])\n", - " y = np.array([[1] for _ in range(len(good_strings))] +\n", - " [[0] for _ in range(len(bad_strings))])\n", - " rnd_idx = np.random.permutation(size)\n", - " return X[rnd_idx], seq_length[rnd_idx], y[rnd_idx]" - ] - }, - { - "cell_type": "code", - "execution_count": 126, - "metadata": {}, - "outputs": [], - "source": [ - "X_train, l_train, y_train = generate_dataset(10000)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's take a look at the first training instances:" - ] - }, - { - "cell_type": "code", - "execution_count": 127, - "metadata": {}, - "outputs": [], - "source": [ - "X_train[0]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "It's padded with a lot of zeros because the longest string in the dataset is that long. How long is this particular string?" - ] - }, - { - "cell_type": "code", - "execution_count": 128, - "metadata": {}, - "outputs": [], - "source": [ - "l_train[0]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "What class is it?" - ] - }, - { - "cell_type": "code", - "execution_count": 129, - "metadata": {}, - "outputs": [], - "source": [ - "y_train[0]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Perfect! We are ready to create the RNN to identify good strings. We build a sequence classifier very similar to the one we built earlier to classify MNIST images, with two main differences:\n", - "* First, the input strings have variable length, so we need to specify the `sequence_length` when calling the `dynamic_rnn()` function.\n", - "* Second, this is a binary classifier, so we only need one output neuron that will output, for each input string, the estimated log probability that it is a good string. For multiclass classification, we used `sparse_softmax_cross_entropy_with_logits()` but for binary classification we use `sigmoid_cross_entropy_with_logits()`.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 130, - "metadata": {}, - "outputs": [], - "source": [ - "reset_graph()\n", - "\n", - "possible_chars = \"BEPSTVX\"\n", - "n_inputs = len(possible_chars)\n", - "n_neurons = 30\n", - "n_outputs = 1\n", - "\n", - "learning_rate = 0.02\n", - "momentum = 0.95\n", - "\n", - "X = tf.placeholder(tf.float32, [None, None, n_inputs], name=\"X\")\n", - "seq_length = tf.placeholder(tf.int32, [None], name=\"seq_length\")\n", - "y = tf.placeholder(tf.float32, [None, 1], name=\"y\")\n", - "\n", - "gru_cell = tf.nn.rnn_cell.GRUCell(num_units=n_neurons)\n", - "outputs, states = tf.nn.dynamic_rnn(gru_cell, X, dtype=tf.float32,\n", - " sequence_length=seq_length)\n", - "\n", - "logits = tf.layers.dense(states, n_outputs, name=\"logits\")\n", - "y_pred = tf.cast(tf.greater(logits, 0.), tf.float32, name=\"y_pred\")\n", - "y_proba = tf.nn.sigmoid(logits, name=\"y_proba\")\n", - "\n", - "xentropy = tf.nn.sigmoid_cross_entropy_with_logits(labels=y, logits=logits)\n", - "loss = tf.reduce_mean(xentropy, name=\"loss\")\n", - "optimizer = tf.train.MomentumOptimizer(learning_rate=learning_rate,\n", - " momentum=momentum,\n", - " use_nesterov=True)\n", - "training_op = optimizer.minimize(loss)\n", - "\n", - "correct = tf.equal(y_pred, y, name=\"correct\")\n", - "accuracy = tf.reduce_mean(tf.cast(correct, tf.float32), name=\"accuracy\")\n", - "\n", - "init = tf.global_variables_initializer()\n", - "saver = tf.train.Saver()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now let's generate a validation set so we can track progress during training:" - ] - }, - { - "cell_type": "code", - "execution_count": 131, - "metadata": {}, - "outputs": [], - "source": [ - "X_val, l_val, y_val = generate_dataset(5000)" - ] - }, - { - "cell_type": "code", - "execution_count": 132, - "metadata": {}, - "outputs": [], - "source": [ - "n_epochs = 50\n", - "batch_size = 50\n", - "\n", - "with tf.Session() as sess:\n", - " init.run()\n", - " for epoch in range(n_epochs):\n", - " X_batches = np.array_split(X_train, len(X_train) // batch_size)\n", - " l_batches = np.array_split(l_train, len(l_train) // batch_size)\n", - " y_batches = np.array_split(y_train, len(y_train) // batch_size)\n", - " for X_batch, l_batch, y_batch in zip(X_batches, l_batches, y_batches):\n", - " loss_val, _ = sess.run(\n", - " [loss, training_op],\n", - " feed_dict={X: X_batch, seq_length: l_batch, y: y_batch})\n", - " acc_train = accuracy.eval(feed_dict={X: X_batch, seq_length: l_batch, y: y_batch})\n", - " acc_val = accuracy.eval(feed_dict={X: X_val, seq_length: l_val, y: y_val})\n", - " print(\"{:4d} Train loss: {:.4f}, accuracy: {:.2f}% Validation accuracy: {:.2f}%\".format(\n", - " epoch, loss_val, 100 * acc_train, 100 * acc_val))\n", - " saver.save(sess, \"./my_reber_classifier\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now let's test our RNN on two tricky strings: the first one is bad while the second one is good. They only differ by the second to last character. If the RNN gets this right, it shows that it managed to notice the pattern that the second letter should always be equal to the second to last letter. That requires a fairly long short-term memory (which is the reason why we used a GRU cell)." - ] - }, - { - "cell_type": "code", - "execution_count": 133, - "metadata": {}, - "outputs": [], - "source": [ - "test_strings = [\n", - " \"BPBTSSSSSSSXXTTVPXVPXTTTTTVVETE\",\n", - " \"BPBTSSSSSSSXXTTVPXVPXTTTTTVVEPE\"]\n", - "l_test = np.array([len(s) for s in test_strings])\n", - "max_length = l_test.max()\n", - "X_test = [string_to_one_hot_vectors(s, n_steps=max_length)\n", - " for s in test_strings]\n", - "\n", - "with tf.Session() as sess:\n", - " saver.restore(sess, \"./my_reber_classifier\")\n", - " y_proba_val = y_proba.eval(feed_dict={X: X_test, seq_length: l_test})\n", - "\n", - "print()\n", - "print(\"Estimated probability that these are Reber strings:\")\n", - "for index, string in enumerate(test_strings):\n", - " print(\"{}: {:.2f}%\".format(string, 100 * y_proba_val[index][0]))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Ta-da! It worked fine. The RNN found the correct answers with high confidence. :)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 8. and 9." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Coming soon..." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.6" - }, - "nav_menu": {}, - "toc": { - "navigate_menu": true, - "number_sections": true, - "sideBar": true, - "threshold": 6, - "toc_cell": false, - "toc_section_display": "block", - "toc_window_display": false - } - }, - "nbformat": 4, - "nbformat_minor": 1 -}