schuetoliver
/
handson-ml
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Replace keras with tf.keras

main
Aurélien Geron 2021-10-17 15:04:08 +13:00
parent 0253e950ea
commit 9145dfbab1
10 changed files with 1122 additions and 1122 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

232
09_neural_nets_with_keras.ipynb
View File

@ -310,14 +310,14 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"keras.__version__" "tf.keras.__version__"
] ]
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",
"metadata": {}, "metadata": {},
"source": [ "source": [
"Let's start by loading the fashion MNIST dataset. Keras has a number of functions to load popular datasets in `keras.datasets`. The dataset is already split for you between a training set and a test set, but it can be useful to split the training set further to have a validation set:" "Let's start by loading the fashion MNIST dataset. Keras has a number of functions to load popular datasets in `tf.keras.datasets`. The dataset is already split for you between a training set and a test set, but it can be useful to split the training set further to have a validation set:"
] ]
}, },
{ {
@ -326,7 +326,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"fashion_mnist = keras.datasets.fashion_mnist\n", "fashion_mnist = tf.keras.datasets.fashion_mnist\n",
"(X_train_full, y_train_full), (X_test, y_test) = fashion_mnist.load_data()" "(X_train_full, y_train_full), (X_test, y_test) = fashion_mnist.load_data()"
] ]
}, },
@ -507,11 +507,11 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"model = keras.models.Sequential()\n", "model = tf.keras.Sequential()\n",
"model.add(keras.layers.Flatten(input_shape=[28, 28]))\n", "model.add(tf.keras.layers.Flatten(input_shape=[28, 28]))\n",
"model.add(keras.layers.Dense(300, activation=\"relu\"))\n", "model.add(tf.keras.layers.Dense(300, activation=\"relu\"))\n",
"model.add(keras.layers.Dense(100, activation=\"relu\"))\n", "model.add(tf.keras.layers.Dense(100, activation=\"relu\"))\n",
"model.add(keras.layers.Dense(10, activation=\"softmax\"))" "model.add(tf.keras.layers.Dense(10, activation=\"softmax\"))"
] ]
}, },
{ {
@ -520,7 +520,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"keras.backend.clear_session()\n", "tf.keras.backend.clear_session()\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)" "tf.random.set_seed(42)"
] ]
@ -531,11 +531,11 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"model = keras.models.Sequential([\n", "model = tf.keras.Sequential([\n",
" keras.layers.Flatten(input_shape=[28, 28]),\n", " tf.keras.layers.Flatten(input_shape=[28, 28]),\n",
" keras.layers.Dense(300, activation=\"relu\"),\n", " tf.keras.layers.Dense(300, activation=\"relu\"),\n",
" keras.layers.Dense(100, activation=\"relu\"),\n", " tf.keras.layers.Dense(100, activation=\"relu\"),\n",
" keras.layers.Dense(10, activation=\"softmax\")\n", " tf.keras.layers.Dense(10, activation=\"softmax\")\n",
"])" "])"
] ]
}, },
@ -563,7 +563,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"keras.utils.plot_model(model, \"my_fashion_mnist_model.png\", show_shapes=True)" "tf.keras.utils.plot_model(model, \"my_fashion_mnist_model.png\", show_shapes=True)"
] ]
}, },
{ {
@ -653,9 +653,9 @@
"metadata": {}, "metadata": {},
"source": [ "source": [
"```python\n", "```python\n",
"model.compile(loss=keras.losses.sparse_categorical_crossentropy,\n", "model.compile(loss=tf.keras.losses.sparse_categorical_crossentropy,\n",
" optimizer=keras.optimizers.SGD(),\n", " optimizer=tf.keras.optimizers.SGD(),\n",
" metrics=[keras.metrics.sparse_categorical_accuracy])\n", " metrics=[tf.keras.metrics.sparse_categorical_accuracy])\n",
"```" "```"
] ]
}, },
@ -836,11 +836,11 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"model = keras.models.Sequential([\n", "model = tf.keras.Sequential([\n",
" keras.layers.Dense(30, activation=\"relu\", input_shape=X_train.shape[1:]),\n", " tf.keras.layers.Dense(30, activation=\"relu\", input_shape=X_train.shape[1:]),\n",
" keras.layers.Dense(1)\n", " tf.keras.layers.Dense(1)\n",
"])\n", "])\n",
"model.compile(loss=\"mean_squared_error\", optimizer=keras.optimizers.SGD(learning_rate=1e-3))\n", "model.compile(loss=\"mean_squared_error\", optimizer=tf.keras.optimizers.SGD(learning_rate=1e-3))\n",
"history = model.fit(X_train, y_train, epochs=20, validation_data=(X_valid, y_valid))\n", "history = model.fit(X_train, y_train, epochs=20, validation_data=(X_valid, y_valid))\n",
"mse_test = model.evaluate(X_test, y_test)\n", "mse_test = model.evaluate(X_test, y_test)\n",
"X_new = X_test[:3]\n", "X_new = X_test[:3]\n",
@ -898,12 +898,12 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"input_ = keras.layers.Input(shape=X_train.shape[1:])\n", "input_ = tf.keras.layers.Input(shape=X_train.shape[1:])\n",
"hidden1 = keras.layers.Dense(30, activation=\"relu\")(input_)\n", "hidden1 = tf.keras.layers.Dense(30, activation=\"relu\")(input_)\n",
"hidden2 = keras.layers.Dense(30, activation=\"relu\")(hidden1)\n", "hidden2 = tf.keras.layers.Dense(30, activation=\"relu\")(hidden1)\n",
"concat = keras.layers.concatenate([input_, hidden2])\n", "concat = tf.keras.layers.concatenate([input_, hidden2])\n",
"output = keras.layers.Dense(1)(concat)\n", "output = tf.keras.layers.Dense(1)(concat)\n",
"model = keras.models.Model(inputs=[input_], outputs=[output])" "model = tf.keras.Model(inputs=[input_], outputs=[output])"
] ]
}, },
{ {
@ -921,7 +921,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"model.compile(loss=\"mean_squared_error\", optimizer=keras.optimizers.SGD(learning_rate=1e-3))\n", "model.compile(loss=\"mean_squared_error\", optimizer=tf.keras.optimizers.SGD(learning_rate=1e-3))\n",
"history = model.fit(X_train, y_train, epochs=20,\n", "history = model.fit(X_train, y_train, epochs=20,\n",
" validation_data=(X_valid, y_valid))\n", " validation_data=(X_valid, y_valid))\n",
"mse_test = model.evaluate(X_test, y_test)\n", "mse_test = model.evaluate(X_test, y_test)\n",
@ -951,13 +951,13 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"input_A = keras.layers.Input(shape=[5], name=\"wide_input\")\n", "input_A = tf.keras.layers.Input(shape=[5], name=\"wide_input\")\n",
"input_B = keras.layers.Input(shape=[6], name=\"deep_input\")\n", "input_B = tf.keras.layers.Input(shape=[6], name=\"deep_input\")\n",
"hidden1 = keras.layers.Dense(30, activation=\"relu\")(input_B)\n", "hidden1 = tf.keras.layers.Dense(30, activation=\"relu\")(input_B)\n",
"hidden2 = keras.layers.Dense(30, activation=\"relu\")(hidden1)\n", "hidden2 = tf.keras.layers.Dense(30, activation=\"relu\")(hidden1)\n",
"concat = keras.layers.concatenate([input_A, hidden2])\n", "concat = tf.keras.layers.concatenate([input_A, hidden2])\n",
"output = keras.layers.Dense(1, name=\"output\")(concat)\n", "output = tf.keras.layers.Dense(1, name=\"output\")(concat)\n",
"model = keras.models.Model(inputs=[input_A, input_B], outputs=[output])" "model = tf.keras.Model(inputs=[input_A, input_B], outputs=[output])"
] ]
}, },
{ {
@ -966,7 +966,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"model.compile(loss=\"mse\", optimizer=keras.optimizers.SGD(learning_rate=1e-3))\n", "model.compile(loss=\"mse\", optimizer=tf.keras.optimizers.SGD(learning_rate=1e-3))\n",
"\n", "\n",
"X_train_A, X_train_B = X_train[:, :5], X_train[:, 2:]\n", "X_train_A, X_train_B = X_train[:, :5], X_train[:, 2:]\n",
"X_valid_A, X_valid_B = X_valid[:, :5], X_valid[:, 2:]\n", "X_valid_A, X_valid_B = X_valid[:, :5], X_valid[:, 2:]\n",
@ -1002,14 +1002,14 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"input_A = keras.layers.Input(shape=[5], name=\"wide_input\")\n", "input_A = tf.keras.layers.Input(shape=[5], name=\"wide_input\")\n",
"input_B = keras.layers.Input(shape=[6], name=\"deep_input\")\n", "input_B = tf.keras.layers.Input(shape=[6], name=\"deep_input\")\n",
"hidden1 = keras.layers.Dense(30, activation=\"relu\")(input_B)\n", "hidden1 = tf.keras.layers.Dense(30, activation=\"relu\")(input_B)\n",
"hidden2 = keras.layers.Dense(30, activation=\"relu\")(hidden1)\n", "hidden2 = tf.keras.layers.Dense(30, activation=\"relu\")(hidden1)\n",
"concat = keras.layers.concatenate([input_A, hidden2])\n", "concat = tf.keras.layers.concatenate([input_A, hidden2])\n",
"output = keras.layers.Dense(1, name=\"main_output\")(concat)\n", "output = tf.keras.layers.Dense(1, name=\"main_output\")(concat)\n",
"aux_output = keras.layers.Dense(1, name=\"aux_output\")(hidden2)\n", "aux_output = tf.keras.layers.Dense(1, name=\"aux_output\")(hidden2)\n",
"model = keras.models.Model(inputs=[input_A, input_B],\n", "model = tf.keras.Model(inputs=[input_A, input_B],\n",
" outputs=[output, aux_output])" " outputs=[output, aux_output])"
] ]
}, },
@ -1019,7 +1019,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"model.compile(loss=[\"mse\", \"mse\"], loss_weights=[0.9, 0.1], optimizer=keras.optimizers.SGD(learning_rate=1e-3))" "model.compile(loss=[\"mse\", \"mse\"], loss_weights=[0.9, 0.1], optimizer=tf.keras.optimizers.SGD(learning_rate=1e-3))"
] ]
}, },
{ {
@ -1056,19 +1056,19 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"class WideAndDeepModel(keras.models.Model):\n", "class WideAndDeepModel(tf.keras.Model):\n",
" def __init__(self, units=30, activation=\"relu\", **kwargs):\n", " def __init__(self, units=30, activation=\"relu\", **kwargs):\n",
" super().__init__(**kwargs)\n", " super().__init__(**kwargs)\n",
" self.hidden1 = keras.layers.Dense(units, activation=activation)\n", " self.hidden1 = tf.keras.layers.Dense(units, activation=activation)\n",
" self.hidden2 = keras.layers.Dense(units, activation=activation)\n", " self.hidden2 = tf.keras.layers.Dense(units, activation=activation)\n",
" self.main_output = keras.layers.Dense(1)\n", " self.main_output = tf.keras.layers.Dense(1)\n",
" self.aux_output = keras.layers.Dense(1)\n", " self.aux_output = tf.keras.layers.Dense(1)\n",
" \n", " \n",
" def call(self, inputs):\n", " def call(self, inputs):\n",
" input_A, input_B = inputs\n", " input_A, input_B = inputs\n",
" hidden1 = self.hidden1(input_B)\n", " hidden1 = self.hidden1(input_B)\n",
" hidden2 = self.hidden2(hidden1)\n", " hidden2 = self.hidden2(hidden1)\n",
" concat = keras.layers.concatenate([input_A, hidden2])\n", " concat = tf.keras.layers.concatenate([input_A, hidden2])\n",
" main_output = self.main_output(concat)\n", " main_output = self.main_output(concat)\n",
" aux_output = self.aux_output(hidden2)\n", " aux_output = self.aux_output(hidden2)\n",
" return main_output, aux_output\n", " return main_output, aux_output\n",
@ -1082,7 +1082,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"model.compile(loss=\"mse\", loss_weights=[0.9, 0.1], optimizer=keras.optimizers.SGD(learning_rate=1e-3))\n", "model.compile(loss=\"mse\", loss_weights=[0.9, 0.1], optimizer=tf.keras.optimizers.SGD(learning_rate=1e-3))\n",
"history = model.fit((X_train_A, X_train_B), (y_train, y_train), epochs=10,\n", "history = model.fit((X_train_A, X_train_B), (y_train, y_train), epochs=10,\n",
" validation_data=((X_valid_A, X_valid_B), (y_valid, y_valid)))\n", " validation_data=((X_valid_A, X_valid_B), (y_valid, y_valid)))\n",
"total_loss, main_loss, aux_loss = model.evaluate((X_test_A, X_test_B), (y_test, y_test))\n", "total_loss, main_loss, aux_loss = model.evaluate((X_test_A, X_test_B), (y_test, y_test))\n",
@ -1112,10 +1112,10 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"model = keras.models.Sequential([\n", "model = tf.keras.Sequential([\n",
" keras.layers.Dense(30, activation=\"relu\", input_shape=[8]),\n", " tf.keras.layers.Dense(30, activation=\"relu\", input_shape=[8]),\n",
" keras.layers.Dense(30, activation=\"relu\"),\n", " tf.keras.layers.Dense(30, activation=\"relu\"),\n",
" keras.layers.Dense(1)\n", " tf.keras.layers.Dense(1)\n",
"]) " "]) "
] ]
}, },
@ -1125,7 +1125,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"model.compile(loss=\"mse\", optimizer=keras.optimizers.SGD(learning_rate=1e-3))\n", "model.compile(loss=\"mse\", optimizer=tf.keras.optimizers.SGD(learning_rate=1e-3))\n",
"history = model.fit(X_train, y_train, epochs=10, validation_data=(X_valid, y_valid))\n", "history = model.fit(X_train, y_train, epochs=10, validation_data=(X_valid, y_valid))\n",
"mse_test = model.evaluate(X_test, y_test)" "mse_test = model.evaluate(X_test, y_test)"
] ]
@ -1145,7 +1145,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"model = keras.models.load_model(\"my_keras_model.h5\")" "model = tf.keras.models.load_model(\"my_keras_model.h5\")"
] ]
}, },
{ {
@ -1188,7 +1188,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"keras.backend.clear_session()\n", "tf.keras.backend.clear_session()\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)" "tf.random.set_seed(42)"
] ]
@ -1199,10 +1199,10 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"model = keras.models.Sequential([\n", "model = tf.keras.Sequential([\n",
" keras.layers.Dense(30, activation=\"relu\", input_shape=[8]),\n", " tf.keras.layers.Dense(30, activation=\"relu\", input_shape=[8]),\n",
" keras.layers.Dense(30, activation=\"relu\"),\n", " tf.keras.layers.Dense(30, activation=\"relu\"),\n",
" keras.layers.Dense(1)\n", " tf.keras.layers.Dense(1)\n",
"]) " "]) "
] ]
}, },
@ -1212,12 +1212,12 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"model.compile(loss=\"mse\", optimizer=keras.optimizers.SGD(learning_rate=1e-3))\n", "model.compile(loss=\"mse\", optimizer=tf.keras.optimizers.SGD(learning_rate=1e-3))\n",
"checkpoint_cb = keras.callbacks.ModelCheckpoint(\"my_keras_model.h5\", save_best_only=True)\n", "checkpoint_cb = tf.keras.callbacks.ModelCheckpoint(\"my_keras_model.h5\", save_best_only=True)\n",
"history = model.fit(X_train, y_train, epochs=10,\n", "history = model.fit(X_train, y_train, epochs=10,\n",
" validation_data=(X_valid, y_valid),\n", " validation_data=(X_valid, y_valid),\n",
" callbacks=[checkpoint_cb])\n", " callbacks=[checkpoint_cb])\n",
"model = keras.models.load_model(\"my_keras_model.h5\") # rollback to best model\n", "model = tf.keras.models.load_model(\"my_keras_model.h5\") # rollback to best model\n",
"mse_test = model.evaluate(X_test, y_test)" "mse_test = model.evaluate(X_test, y_test)"
] ]
}, },
@ -1227,8 +1227,8 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"model.compile(loss=\"mse\", optimizer=keras.optimizers.SGD(learning_rate=1e-3))\n", "model.compile(loss=\"mse\", optimizer=tf.keras.optimizers.SGD(learning_rate=1e-3))\n",
"early_stopping_cb = keras.callbacks.EarlyStopping(patience=10,\n", "early_stopping_cb = tf.keras.callbacks.EarlyStopping(patience=10,\n",
" restore_best_weights=True)\n", " restore_best_weights=True)\n",
"history = model.fit(X_train, y_train, epochs=100,\n", "history = model.fit(X_train, y_train, epochs=100,\n",
" validation_data=(X_valid, y_valid),\n", " validation_data=(X_valid, y_valid),\n",
@ -1242,7 +1242,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"class PrintValTrainRatioCallback(keras.callbacks.Callback):\n", "class PrintValTrainRatioCallback(tf.keras.callbacks.Callback):\n",
" def on_epoch_end(self, epoch, logs):\n", " def on_epoch_end(self, epoch, logs):\n",
" print(\"\\nval/train: {:.2f}\".format(logs[\"val_loss\"] / logs[\"loss\"]))" " print(\"\\nval/train: {:.2f}\".format(logs[\"val_loss\"] / logs[\"loss\"]))"
] ]
@ -1296,7 +1296,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"keras.backend.clear_session()\n", "tf.keras.backend.clear_session()\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)" "tf.random.set_seed(42)"
] ]
@ -1307,12 +1307,12 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"model = keras.models.Sequential([\n", "model = tf.keras.Sequential([\n",
" keras.layers.Dense(30, activation=\"relu\", input_shape=[8]),\n", " tf.keras.layers.Dense(30, activation=\"relu\", input_shape=[8]),\n",
" keras.layers.Dense(30, activation=\"relu\"),\n", " tf.keras.layers.Dense(30, activation=\"relu\"),\n",
" keras.layers.Dense(1)\n", " tf.keras.layers.Dense(1)\n",
"]) \n", "]) \n",
"model.compile(loss=\"mse\", optimizer=keras.optimizers.SGD(learning_rate=1e-3))" "model.compile(loss=\"mse\", optimizer=tf.keras.optimizers.SGD(learning_rate=1e-3))"
] ]
}, },
{ {
@ -1321,7 +1321,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"tensorboard_cb = keras.callbacks.TensorBoard(run_logdir)\n", "tensorboard_cb = tf.keras.callbacks.TensorBoard(run_logdir)\n",
"history = model.fit(X_train, y_train, epochs=30,\n", "history = model.fit(X_train, y_train, epochs=30,\n",
" validation_data=(X_valid, y_valid),\n", " validation_data=(X_valid, y_valid),\n",
" callbacks=[checkpoint_cb, tensorboard_cb])" " callbacks=[checkpoint_cb, tensorboard_cb])"
@ -1368,7 +1368,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"keras.backend.clear_session()\n", "tf.keras.backend.clear_session()\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)" "tf.random.set_seed(42)"
] ]
@ -1379,12 +1379,12 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"model = keras.models.Sequential([\n", "model = tf.keras.Sequential([\n",
" keras.layers.Dense(30, activation=\"relu\", input_shape=[8]),\n", " tf.keras.layers.Dense(30, activation=\"relu\", input_shape=[8]),\n",
" keras.layers.Dense(30, activation=\"relu\"),\n", " tf.keras.layers.Dense(30, activation=\"relu\"),\n",
" keras.layers.Dense(1)\n", " tf.keras.layers.Dense(1)\n",
"]) \n", "]) \n",
"model.compile(loss=\"mse\", optimizer=keras.optimizers.SGD(learning_rate=0.05))" "model.compile(loss=\"mse\", optimizer=tf.keras.optimizers.SGD(learning_rate=0.05))"
] ]
}, },
{ {
@ -1393,7 +1393,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"tensorboard_cb = keras.callbacks.TensorBoard(run_logdir2)\n", "tensorboard_cb = tf.keras.callbacks.TensorBoard(run_logdir2)\n",
"history = model.fit(X_train, y_train, epochs=30,\n", "history = model.fit(X_train, y_train, epochs=30,\n",
" validation_data=(X_valid, y_valid),\n", " validation_data=(X_valid, y_valid),\n",
" callbacks=[checkpoint_cb, tensorboard_cb])" " callbacks=[checkpoint_cb, tensorboard_cb])"
@ -1419,7 +1419,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"help(keras.callbacks.TensorBoard.__init__)" "help(tf.keras.callbacks.TensorBoard.__init__)"
] ]
}, },
{ {
@ -1435,7 +1435,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"keras.backend.clear_session()\n", "tf.keras.backend.clear_session()\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)" "tf.random.set_seed(42)"
] ]
@ -1447,12 +1447,12 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"def build_model(n_hidden=1, n_neurons=30, learning_rate=3e-3, input_shape=[8]):\n", "def build_model(n_hidden=1, n_neurons=30, learning_rate=3e-3, input_shape=[8]):\n",
" model = keras.models.Sequential()\n", " model = tf.keras.Sequential()\n",
" model.add(keras.layers.InputLayer(input_shape=input_shape))\n", " model.add(tf.keras.layers.InputLayer(input_shape=input_shape))\n",
" for layer in range(n_hidden):\n", " for layer in range(n_hidden):\n",
" model.add(keras.layers.Dense(n_neurons, activation=\"relu\"))\n", " model.add(tf.keras.layers.Dense(n_neurons, activation=\"relu\"))\n",
" model.add(keras.layers.Dense(1))\n", " model.add(tf.keras.layers.Dense(1))\n",
" optimizer = keras.optimizers.SGD(learning_rate=learning_rate)\n", " optimizer = tf.keras.optimizers.SGD(learning_rate=learning_rate)\n",
" model.compile(loss=\"mse\", optimizer=optimizer)\n", " model.compile(loss=\"mse\", optimizer=optimizer)\n",
" return model" " return model"
] ]
@ -1463,7 +1463,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"keras_reg = keras.wrappers.scikit_learn.KerasRegressor(build_model)" "keras_reg = tf.keras.wrappers.scikit_learn.KerasRegressor(build_model)"
] ]
}, },
{ {
@ -1474,7 +1474,7 @@
"source": [ "source": [
"keras_reg.fit(X_train, y_train, epochs=100,\n", "keras_reg.fit(X_train, y_train, epochs=100,\n",
" validation_data=(X_valid, y_valid),\n", " validation_data=(X_valid, y_valid),\n",
" callbacks=[keras.callbacks.EarlyStopping(patience=10)])" " callbacks=[tf.keras.callbacks.EarlyStopping(patience=10)])"
] ]
}, },
{ {
@ -1530,7 +1530,7 @@
"rnd_search_cv = RandomizedSearchCV(keras_reg, param_distribs, n_iter=10, cv=3, verbose=2)\n", "rnd_search_cv = RandomizedSearchCV(keras_reg, param_distribs, n_iter=10, cv=3, verbose=2)\n",
"rnd_search_cv.fit(X_train, y_train, epochs=100,\n", "rnd_search_cv.fit(X_train, y_train, epochs=100,\n",
" validation_data=(X_valid, y_valid),\n", " validation_data=(X_valid, y_valid),\n",
" callbacks=[keras.callbacks.EarlyStopping(patience=10)])" " callbacks=[tf.keras.callbacks.EarlyStopping(patience=10)])"
] ]
}, },
{ {
@ -1622,7 +1622,7 @@
"cell_type": "markdown", "cell_type": "markdown",
"metadata": {}, "metadata": {},
"source": [ "source": [
"*Exercise: Train a deep MLP on the MNIST dataset (you can load it using `keras.datasets.mnist.load_data()`. See if you can get over 98% precision. Try searching for the optimal learning rate by using the approach presented in this chapter (i.e., by growing the learning rate exponentially, plotting the loss, and finding the point where the loss shoots up). Try adding all the bells and whistles—save checkpoints, use early stopping, and plot learning curves using TensorBoard.*" "*Exercise: Train a deep MLP on the MNIST dataset (you can load it using `tf.keras.datasets.mnist.load_data()`. See if you can get over 98% precision. Try searching for the optimal learning rate by using the approach presented in this chapter (i.e., by growing the learning rate exponentially, plotting the loss, and finding the point where the loss shoots up). Try adding all the bells and whistles—save checkpoints, use early stopping, and plot learning curves using TensorBoard.*"
] ]
}, },
{ {
@ -1638,7 +1638,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"(X_train_full, y_train_full), (X_test, y_test) = keras.datasets.mnist.load_data()" "(X_train_full, y_train_full), (X_test, y_test) = tf.keras.datasets.mnist.load_data()"
] ]
}, },
{ {
@ -1791,9 +1791,9 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"K = keras.backend\n", "K = tf.keras.backend\n",
"\n", "\n",
"class ExponentialLearningRate(keras.callbacks.Callback):\n", "class ExponentialLearningRate(tf.keras.callbacks.Callback):\n",
" def __init__(self, factor):\n", " def __init__(self, factor):\n",
" self.factor = factor\n", " self.factor = factor\n",
" self.rates = []\n", " self.rates = []\n",
@ -1810,7 +1810,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"keras.backend.clear_session()\n", "tf.keras.backend.clear_session()\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)" "tf.random.set_seed(42)"
] ]
@ -1821,11 +1821,11 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"model = keras.models.Sequential([\n", "model = tf.keras.Sequential([\n",
" keras.layers.Flatten(input_shape=[28, 28]),\n", " tf.keras.layers.Flatten(input_shape=[28, 28]),\n",
" keras.layers.Dense(300, activation=\"relu\"),\n", " tf.keras.layers.Dense(300, activation=\"relu\"),\n",
" keras.layers.Dense(100, activation=\"relu\"),\n", " tf.keras.layers.Dense(100, activation=\"relu\"),\n",
" keras.layers.Dense(10, activation=\"softmax\")\n", " tf.keras.layers.Dense(10, activation=\"softmax\")\n",
"])" "])"
] ]
}, },
@ -1843,7 +1843,7 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"model.compile(loss=\"sparse_categorical_crossentropy\",\n", "model.compile(loss=\"sparse_categorical_crossentropy\",\n",
" optimizer=keras.optimizers.SGD(learning_rate=1e-3),\n", " optimizer=tf.keras.optimizers.SGD(learning_rate=1e-3),\n",
" metrics=[\"accuracy\"])\n", " metrics=[\"accuracy\"])\n",
"expon_lr = ExponentialLearningRate(factor=1.005)" "expon_lr = ExponentialLearningRate(factor=1.005)"
] ]
@ -1901,7 +1901,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"keras.backend.clear_session()\n", "tf.keras.backend.clear_session()\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)" "tf.random.set_seed(42)"
] ]
@ -1912,11 +1912,11 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"model = keras.models.Sequential([\n", "model = tf.keras.Sequential([\n",
" keras.layers.Flatten(input_shape=[28, 28]),\n", " tf.keras.layers.Flatten(input_shape=[28, 28]),\n",
" keras.layers.Dense(300, activation=\"relu\"),\n", " tf.keras.layers.Dense(300, activation=\"relu\"),\n",
" keras.layers.Dense(100, activation=\"relu\"),\n", " tf.keras.layers.Dense(100, activation=\"relu\"),\n",
" keras.layers.Dense(10, activation=\"softmax\")\n", " tf.keras.layers.Dense(10, activation=\"softmax\")\n",
"])" "])"
] ]
}, },
@ -1927,7 +1927,7 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"model.compile(loss=\"sparse_categorical_crossentropy\",\n", "model.compile(loss=\"sparse_categorical_crossentropy\",\n",
" optimizer=keras.optimizers.SGD(learning_rate=3e-1),\n", " optimizer=tf.keras.optimizers.SGD(learning_rate=3e-1),\n",
" metrics=[\"accuracy\"])" " metrics=[\"accuracy\"])"
] ]
}, },
@ -1948,9 +1948,9 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"early_stopping_cb = keras.callbacks.EarlyStopping(patience=20)\n", "early_stopping_cb = tf.keras.callbacks.EarlyStopping(patience=20)\n",
"checkpoint_cb = keras.callbacks.ModelCheckpoint(\"my_mnist_model.h5\", save_best_only=True)\n", "checkpoint_cb = tf.keras.callbacks.ModelCheckpoint(\"my_mnist_model.h5\", save_best_only=True)\n",
"tensorboard_cb = keras.callbacks.TensorBoard(run_logdir)\n", "tensorboard_cb = tf.keras.callbacks.TensorBoard(run_logdir)\n",
"\n", "\n",
"history = model.fit(X_train, y_train, epochs=100,\n", "history = model.fit(X_train, y_train, epochs=100,\n",
" validation_data=(X_valid, y_valid),\n", " validation_data=(X_valid, y_valid),\n",
@ -1963,7 +1963,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"model = keras.models.load_model(\"my_mnist_model.h5\") # rollback to best model\n", "model = tf.keras.models.load_model(\"my_mnist_model.h5\") # rollback to best model\n",
"model.evaluate(X_test, y_test)" "model.evaluate(X_test, y_test)"
] ]
}, },

488
10_training_deep_neural_networks.ipynb
View File

File diff suppressed because it is too large Load Diff

284
11_custom_models_and_training_with_tensorflow.ipynb
View File

@ -200,7 +200,7 @@
"cell_type": "markdown", "cell_type": "markdown",
"metadata": {}, "metadata": {},
"source": [ "source": [
"### Using `keras.backend`" "### Using `tf.keras.backend`"
] ]
}, },
{ {
@ -210,7 +210,7 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"from tensorflow import keras\n", "from tensorflow import keras\n",
"K = keras.backend\n", "K = tf.keras.backend\n",
"K.square(K.transpose(t)) + 10" "K.square(K.transpose(t)) + 10"
] ]
}, },
@ -818,10 +818,10 @@
"source": [ "source": [
"input_shape = X_train.shape[1:]\n", "input_shape = X_train.shape[1:]\n",
"\n", "\n",
"model = keras.models.Sequential([\n", "model = tf.keras.Sequential([\n",
" keras.layers.Dense(30, activation=\"selu\", kernel_initializer=\"lecun_normal\",\n", " tf.keras.layers.Dense(30, activation=\"selu\", kernel_initializer=\"lecun_normal\",\n",
" input_shape=input_shape),\n", " input_shape=input_shape),\n",
" keras.layers.Dense(1),\n", " tf.keras.layers.Dense(1),\n",
"])" "])"
] ]
}, },
@ -868,7 +868,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"model = keras.models.load_model(\"my_model_with_a_custom_loss.h5\",\n", "model = tf.keras.models.load_model(\"my_model_with_a_custom_loss.h5\",\n",
" custom_objects={\"huber_fn\": huber_fn})" " custom_objects={\"huber_fn\": huber_fn})"
] ]
}, },
@ -934,7 +934,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"model = keras.models.load_model(\"my_model_with_a_custom_loss_threshold_2.h5\",\n", "model = tf.keras.models.load_model(\"my_model_with_a_custom_loss_threshold_2.h5\",\n",
" custom_objects={\"huber_fn\": create_huber(2.0)})" " custom_objects={\"huber_fn\": create_huber(2.0)})"
] ]
}, },
@ -954,7 +954,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"class HuberLoss(keras.losses.Loss):\n", "class HuberLoss(tf.keras.losses.Loss):\n",
" def __init__(self, threshold=1.0, **kwargs):\n", " def __init__(self, threshold=1.0, **kwargs):\n",
" self.threshold = threshold\n", " self.threshold = threshold\n",
" super().__init__(**kwargs)\n", " super().__init__(**kwargs)\n",
@ -975,10 +975,10 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"model = keras.models.Sequential([\n", "model = tf.keras.Sequential([\n",
" keras.layers.Dense(30, activation=\"selu\", kernel_initializer=\"lecun_normal\",\n", " tf.keras.layers.Dense(30, activation=\"selu\", kernel_initializer=\"lecun_normal\",\n",
" input_shape=input_shape),\n", " input_shape=input_shape),\n",
" keras.layers.Dense(1),\n", " tf.keras.layers.Dense(1),\n",
"])" "])"
] ]
}, },
@ -1016,7 +1016,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"model = keras.models.load_model(\"my_model_with_a_custom_loss_class.h5\",\n", "model = tf.keras.models.load_model(\"my_model_with_a_custom_loss_class.h5\",\n",
" custom_objects={\"HuberLoss\": HuberLoss})" " custom_objects={\"HuberLoss\": HuberLoss})"
] ]
}, },
@ -1052,7 +1052,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"keras.backend.clear_session()\n", "tf.keras.backend.clear_session()\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)" "tf.random.set_seed(42)"
] ]
@ -1083,7 +1083,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"layer = keras.layers.Dense(1, activation=my_softplus,\n", "layer = tf.keras.layers.Dense(1, activation=my_softplus,\n",
" kernel_initializer=my_glorot_initializer,\n", " kernel_initializer=my_glorot_initializer,\n",
" kernel_regularizer=my_l1_regularizer,\n", " kernel_regularizer=my_l1_regularizer,\n",
" kernel_constraint=my_positive_weights)" " kernel_constraint=my_positive_weights)"
@ -1095,7 +1095,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"keras.backend.clear_session()\n", "tf.keras.backend.clear_session()\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)" "tf.random.set_seed(42)"
] ]
@ -1106,10 +1106,10 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"model = keras.models.Sequential([\n", "model = tf.keras.Sequential([\n",
" keras.layers.Dense(30, activation=\"selu\", kernel_initializer=\"lecun_normal\",\n", " tf.keras.layers.Dense(30, activation=\"selu\", kernel_initializer=\"lecun_normal\",\n",
" input_shape=input_shape),\n", " input_shape=input_shape),\n",
" keras.layers.Dense(1, activation=my_softplus,\n", " tf.keras.layers.Dense(1, activation=my_softplus,\n",
" kernel_regularizer=my_l1_regularizer,\n", " kernel_regularizer=my_l1_regularizer,\n",
" kernel_constraint=my_positive_weights,\n", " kernel_constraint=my_positive_weights,\n",
" kernel_initializer=my_glorot_initializer),\n", " kernel_initializer=my_glorot_initializer),\n",
@ -1150,7 +1150,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"model = keras.models.load_model(\n", "model = tf.keras.models.load_model(\n",
" \"my_model_with_many_custom_parts.h5\",\n", " \"my_model_with_many_custom_parts.h5\",\n",
" custom_objects={\n", " custom_objects={\n",
" \"my_l1_regularizer\": my_l1_regularizer,\n", " \"my_l1_regularizer\": my_l1_regularizer,\n",
@ -1166,7 +1166,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"class MyL1Regularizer(keras.regularizers.Regularizer):\n", "class MyL1Regularizer(tf.keras.regularizers.Regularizer):\n",
" def __init__(self, factor):\n", " def __init__(self, factor):\n",
" self.factor = factor\n", " self.factor = factor\n",
" def __call__(self, weights):\n", " def __call__(self, weights):\n",
@ -1181,7 +1181,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"keras.backend.clear_session()\n", "tf.keras.backend.clear_session()\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)" "tf.random.set_seed(42)"
] ]
@ -1192,10 +1192,10 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"model = keras.models.Sequential([\n", "model = tf.keras.Sequential([\n",
" keras.layers.Dense(30, activation=\"selu\", kernel_initializer=\"lecun_normal\",\n", " tf.keras.layers.Dense(30, activation=\"selu\", kernel_initializer=\"lecun_normal\",\n",
" input_shape=input_shape),\n", " input_shape=input_shape),\n",
" keras.layers.Dense(1, activation=my_softplus,\n", " tf.keras.layers.Dense(1, activation=my_softplus,\n",
" kernel_regularizer=MyL1Regularizer(0.01),\n", " kernel_regularizer=MyL1Regularizer(0.01),\n",
" kernel_constraint=my_positive_weights,\n", " kernel_constraint=my_positive_weights,\n",
" kernel_initializer=my_glorot_initializer),\n", " kernel_initializer=my_glorot_initializer),\n",
@ -1236,7 +1236,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"model = keras.models.load_model(\n", "model = tf.keras.models.load_model(\n",
" \"my_model_with_many_custom_parts.h5\",\n", " \"my_model_with_many_custom_parts.h5\",\n",
" custom_objects={\n", " custom_objects={\n",
" \"MyL1Regularizer\": MyL1Regularizer,\n", " \"MyL1Regularizer\": MyL1Regularizer,\n",
@ -1259,7 +1259,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"keras.backend.clear_session()\n", "tf.keras.backend.clear_session()\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)" "tf.random.set_seed(42)"
] ]
@ -1270,10 +1270,10 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"model = keras.models.Sequential([\n", "model = tf.keras.Sequential([\n",
" keras.layers.Dense(30, activation=\"selu\", kernel_initializer=\"lecun_normal\",\n", " tf.keras.layers.Dense(30, activation=\"selu\", kernel_initializer=\"lecun_normal\",\n",
" input_shape=input_shape),\n", " input_shape=input_shape),\n",
" keras.layers.Dense(1),\n", " tf.keras.layers.Dense(1),\n",
"])" "])"
] ]
}, },
@ -1347,7 +1347,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"precision = keras.metrics.Precision()\n", "precision = tf.keras.metrics.Precision()\n",
"precision([0, 1, 1, 1, 0, 1, 0, 1], [1, 1, 0, 1, 0, 1, 0, 1])" "precision([0, 1, 1, 1, 0, 1, 0, 1], [1, 1, 0, 1, 0, 1, 0, 1])"
] ]
}, },
@ -1400,7 +1400,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"class HuberMetric(keras.metrics.Metric):\n", "class HuberMetric(tf.keras.metrics.Metric):\n",
" def __init__(self, threshold=1.0, **kwargs):\n", " def __init__(self, threshold=1.0, **kwargs):\n",
" super().__init__(**kwargs) # handles base args (e.g., dtype)\n", " super().__init__(**kwargs) # handles base args (e.g., dtype)\n",
" self.threshold = threshold\n", " self.threshold = threshold\n",
@ -1478,7 +1478,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"keras.backend.clear_session()\n", "tf.keras.backend.clear_session()\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)" "tf.random.set_seed(42)"
] ]
@ -1489,10 +1489,10 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"model = keras.models.Sequential([\n", "model = tf.keras.Sequential([\n",
" keras.layers.Dense(30, activation=\"selu\", kernel_initializer=\"lecun_normal\",\n", " tf.keras.layers.Dense(30, activation=\"selu\", kernel_initializer=\"lecun_normal\",\n",
" input_shape=input_shape),\n", " input_shape=input_shape),\n",
" keras.layers.Dense(1),\n", " tf.keras.layers.Dense(1),\n",
"])" "])"
] ]
}, },
@ -1529,7 +1529,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"model = keras.models.load_model(\"my_model_with_a_custom_metric.h5\",\n", "model = tf.keras.models.load_model(\"my_model_with_a_custom_metric.h5\",\n",
" custom_objects={\"huber_fn\": create_huber(2.0),\n", " custom_objects={\"huber_fn\": create_huber(2.0),\n",
" \"HuberMetric\": HuberMetric})" " \"HuberMetric\": HuberMetric})"
] ]
@ -1572,7 +1572,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"class HuberMetric(keras.metrics.Mean):\n", "class HuberMetric(tf.keras.metrics.Mean):\n",
" def __init__(self, threshold=1.0, name='HuberMetric', dtype=None):\n", " def __init__(self, threshold=1.0, name='HuberMetric', dtype=None):\n",
" self.threshold = threshold\n", " self.threshold = threshold\n",
" self.huber_fn = create_huber(threshold)\n", " self.huber_fn = create_huber(threshold)\n",
@ -1598,7 +1598,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"keras.backend.clear_session()\n", "tf.keras.backend.clear_session()\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)" "tf.random.set_seed(42)"
] ]
@ -1609,10 +1609,10 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"model = keras.models.Sequential([\n", "model = tf.keras.Sequential([\n",
" keras.layers.Dense(30, activation=\"selu\", kernel_initializer=\"lecun_normal\",\n", " tf.keras.layers.Dense(30, activation=\"selu\", kernel_initializer=\"lecun_normal\",\n",
" input_shape=input_shape),\n", " input_shape=input_shape),\n",
" keras.layers.Dense(1),\n", " tf.keras.layers.Dense(1),\n",
"])" "])"
] ]
}, },
@ -1622,7 +1622,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"model.compile(loss=keras.losses.Huber(2.0), optimizer=\"nadam\", weighted_metrics=[HuberMetric(2.0)])" "model.compile(loss=tf.keras.losses.Huber(2.0), optimizer=\"nadam\", weighted_metrics=[HuberMetric(2.0)])"
] ]
}, },
{ {
@ -1662,7 +1662,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"model = keras.models.load_model(\"my_model_with_a_custom_metric_v2.h5\",\n", "model = tf.keras.models.load_model(\"my_model_with_a_custom_metric_v2.h5\",\n",
" custom_objects={\"HuberMetric\": HuberMetric})" " custom_objects={\"HuberMetric\": HuberMetric})"
] ]
}, },
@ -1699,7 +1699,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"exponential_layer = keras.layers.Lambda(lambda x: tf.exp(x))" "exponential_layer = tf.keras.layers.Lambda(lambda x: tf.exp(x))"
] ]
}, },
{ {
@ -1724,7 +1724,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"keras.backend.clear_session()\n", "tf.keras.backend.clear_session()\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)" "tf.random.set_seed(42)"
] ]
@ -1735,9 +1735,9 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"model = keras.models.Sequential([\n", "model = tf.keras.Sequential([\n",
" keras.layers.Dense(30, activation=\"relu\", input_shape=input_shape),\n", " tf.keras.layers.Dense(30, activation=\"relu\", input_shape=input_shape),\n",
" keras.layers.Dense(1),\n", " tf.keras.layers.Dense(1),\n",
" exponential_layer\n", " exponential_layer\n",
"])\n", "])\n",
"model.compile(loss=\"mse\", optimizer=\"sgd\")\n", "model.compile(loss=\"mse\", optimizer=\"sgd\")\n",
@ -1752,11 +1752,11 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"class MyDense(keras.layers.Layer):\n", "class MyDense(tf.keras.layers.Layer):\n",
" def __init__(self, units, activation=None, **kwargs):\n", " def __init__(self, units, activation=None, **kwargs):\n",
" super().__init__(**kwargs)\n", " super().__init__(**kwargs)\n",
" self.units = units\n", " self.units = units\n",
" self.activation = keras.activations.get(activation)\n", " self.activation = tf.keras.activations.get(activation)\n",
"\n", "\n",
" def build(self, batch_input_shape):\n", " def build(self, batch_input_shape):\n",
" self.kernel = self.add_weight(\n", " self.kernel = self.add_weight(\n",
@ -1775,7 +1775,7 @@
" def get_config(self):\n", " def get_config(self):\n",
" base_config = super().get_config()\n", " base_config = super().get_config()\n",
" return {**base_config, \"units\": self.units,\n", " return {**base_config, \"units\": self.units,\n",
" \"activation\": keras.activations.serialize(self.activation)}" " \"activation\": tf.keras.activations.serialize(self.activation)}"
] ]
}, },
{ {
@ -1784,7 +1784,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"keras.backend.clear_session()\n", "tf.keras.backend.clear_session()\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)" "tf.random.set_seed(42)"
] ]
@ -1795,7 +1795,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"model = keras.models.Sequential([\n", "model = tf.keras.Sequential([\n",
" MyDense(30, activation=\"relu\", input_shape=input_shape),\n", " MyDense(30, activation=\"relu\", input_shape=input_shape),\n",
" MyDense(1)\n", " MyDense(1)\n",
"])" "])"
@ -1828,7 +1828,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"model = keras.models.load_model(\"my_model_with_a_custom_layer.h5\",\n", "model = tf.keras.models.load_model(\"my_model_with_a_custom_layer.h5\",\n",
" custom_objects={\"MyDense\": MyDense})" " custom_objects={\"MyDense\": MyDense})"
] ]
}, },
@ -1838,7 +1838,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"class MyMultiLayer(keras.layers.Layer):\n", "class MyMultiLayer(tf.keras.layers.Layer):\n",
" def call(self, X):\n", " def call(self, X):\n",
" X1, X2 = X\n", " X1, X2 = X\n",
" print(\"X1.shape: \", X1.shape ,\" X2.shape: \", X2.shape) # Debugging of custom layer\n", " print(\"X1.shape: \", X1.shape ,\" X2.shape: \", X2.shape) # Debugging of custom layer\n",
@ -1862,8 +1862,8 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"inputs1 = keras.layers.Input(shape=[2])\n", "inputs1 = tf.keras.layers.Input(shape=[2])\n",
"inputs2 = keras.layers.Input(shape=[2])\n", "inputs2 = tf.keras.layers.Input(shape=[2])\n",
"outputs1, outputs2 = MyMultiLayer()((inputs1, inputs2))" "outputs1, outputs2 = MyMultiLayer()((inputs1, inputs2))"
] ]
}, },
@ -1924,18 +1924,18 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"keras.backend.clear_session()\n", "tf.keras.backend.clear_session()\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)\n", "tf.random.set_seed(42)\n",
"\n", "\n",
"input_A = keras.layers.Input(shape=X_train_scaled_A.shape[-1])\n", "input_A = tf.keras.layers.Input(shape=X_train_scaled_A.shape[-1])\n",
"input_B = keras.layers.Input(shape=X_train_scaled_B.shape[-1])\n", "input_B = tf.keras.layers.Input(shape=X_train_scaled_B.shape[-1])\n",
"hidden_A, hidden_B = MyMultiLayer()((input_A, input_B))\n", "hidden_A, hidden_B = MyMultiLayer()((input_A, input_B))\n",
"hidden_A = keras.layers.Dense(30, activation='selu')(hidden_A)\n", "hidden_A = tf.keras.layers.Dense(30, activation='selu')(hidden_A)\n",
"hidden_B = keras.layers.Dense(30, activation='selu')(hidden_B)\n", "hidden_B = tf.keras.layers.Dense(30, activation='selu')(hidden_B)\n",
"concat = keras.layers.Concatenate()((hidden_A, hidden_B))\n", "concat = tf.keras.layers.Concatenate()((hidden_A, hidden_B))\n",
"output = keras.layers.Dense(1)(concat)\n", "output = tf.keras.layers.Dense(1)(concat)\n",
"model = keras.models.Model(inputs=[input_A, input_B], outputs=[output])" "model = tf.keras.Model(inputs=[input_A, input_B], outputs=[output])"
] ]
}, },
{ {
@ -1970,7 +1970,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"class AddGaussianNoise(keras.layers.Layer):\n", "class AddGaussianNoise(tf.keras.layers.Layer):\n",
" def __init__(self, stddev, **kwargs):\n", " def __init__(self, stddev, **kwargs):\n",
" super().__init__(**kwargs)\n", " super().__init__(**kwargs)\n",
" self.stddev = stddev\n", " self.stddev = stddev\n",
@ -1999,14 +1999,14 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"keras.backend.clear_session()\n", "tf.keras.backend.clear_session()\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)\n", "tf.random.set_seed(42)\n",
"\n", "\n",
"model = keras.models.Sequential([\n", "model = tf.keras.Sequential([\n",
" AddGaussianNoise(stddev=1.0),\n", " AddGaussianNoise(stddev=1.0),\n",
" keras.layers.Dense(30, activation=\"selu\"),\n", " tf.keras.layers.Dense(30, activation=\"selu\"),\n",
" keras.layers.Dense(1)\n", " tf.keras.layers.Dense(1)\n",
"])" "])"
] ]
}, },
@ -2044,10 +2044,10 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"class ResidualBlock(keras.layers.Layer):\n", "class ResidualBlock(tf.keras.layers.Layer):\n",
" def __init__(self, n_layers, n_neurons, **kwargs):\n", " def __init__(self, n_layers, n_neurons, **kwargs):\n",
" super().__init__(**kwargs)\n", " super().__init__(**kwargs)\n",
" self.hidden = [keras.layers.Dense(n_neurons, activation=\"elu\",\n", " self.hidden = [tf.keras.layers.Dense(n_neurons, activation=\"elu\",\n",
" kernel_initializer=\"he_normal\")\n", " kernel_initializer=\"he_normal\")\n",
" for _ in range(n_layers)]\n", " for _ in range(n_layers)]\n",
"\n", "\n",
@ -2064,14 +2064,14 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"class ResidualRegressor(keras.models.Model):\n", "class ResidualRegressor(tf.keras.Model):\n",
" def __init__(self, output_dim, **kwargs):\n", " def __init__(self, output_dim, **kwargs):\n",
" super().__init__(**kwargs)\n", " super().__init__(**kwargs)\n",
" self.hidden1 = keras.layers.Dense(30, activation=\"elu\",\n", " self.hidden1 = tf.keras.layers.Dense(30, activation=\"elu\",\n",
" kernel_initializer=\"he_normal\")\n", " kernel_initializer=\"he_normal\")\n",
" self.block1 = ResidualBlock(2, 30)\n", " self.block1 = ResidualBlock(2, 30)\n",
" self.block2 = ResidualBlock(2, 30)\n", " self.block2 = ResidualBlock(2, 30)\n",
" self.out = keras.layers.Dense(output_dim)\n", " self.out = tf.keras.layers.Dense(output_dim)\n",
"\n", "\n",
" def call(self, inputs):\n", " def call(self, inputs):\n",
" Z = self.hidden1(inputs)\n", " Z = self.hidden1(inputs)\n",
@ -2087,7 +2087,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"keras.backend.clear_session()\n", "tf.keras.backend.clear_session()\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)" "tf.random.set_seed(42)"
] ]
@ -2120,7 +2120,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"model = keras.models.load_model(\"my_custom_model.ckpt\")" "model = tf.keras.models.load_model(\"my_custom_model.ckpt\")"
] ]
}, },
{ {
@ -2145,7 +2145,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"keras.backend.clear_session()\n", "tf.keras.backend.clear_session()\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)" "tf.random.set_seed(42)"
] ]
@ -2157,11 +2157,11 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"block1 = ResidualBlock(2, 30)\n", "block1 = ResidualBlock(2, 30)\n",
"model = keras.models.Sequential([\n", "model = tf.keras.Sequential([\n",
" keras.layers.Dense(30, activation=\"elu\", kernel_initializer=\"he_normal\"),\n", " tf.keras.layers.Dense(30, activation=\"elu\", kernel_initializer=\"he_normal\"),\n",
" block1, block1, block1, block1,\n", " block1, block1, block1, block1,\n",
" ResidualBlock(2, 30),\n", " ResidualBlock(2, 30),\n",
" keras.layers.Dense(1)\n", " tf.keras.layers.Dense(1)\n",
"])" "])"
] ]
}, },
@ -2189,7 +2189,7 @@
"metadata": {}, "metadata": {},
"source": [ "source": [
"**Note**: the following code has two differences with the code in the book:\n", "**Note**: the following code has two differences with the code in the book:\n",
"1. It creates a `keras.metrics.Mean()` metric in the constructor and uses it in the `call()` method to track the mean reconstruction loss. Since we only want to do this during training, we add a `training` argument to the `call()` method, and if `training` is `True`, then we update `reconstruction_mean` and we call `self.add_metric()` to ensure it's displayed properly.\n", "1. It creates a `tf.keras.metrics.Mean()` metric in the constructor and uses it in the `call()` method to track the mean reconstruction loss. Since we only want to do this during training, we add a `training` argument to the `call()` method, and if `training` is `True`, then we update `reconstruction_mean` and we call `self.add_metric()` to ensure it's displayed properly.\n",
"2. Due to an issue introduced in TF 2.2 ([#46858](https://github.com/tensorflow/tensorflow/issues/46858)), we must not call `super().build()` inside the `build()` method." "2. Due to an issue introduced in TF 2.2 ([#46858](https://github.com/tensorflow/tensorflow/issues/46858)), we must not call `super().build()` inside the `build()` method."
] ]
}, },
@ -2199,18 +2199,18 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"class ReconstructingRegressor(keras.Model):\n", "class ReconstructingRegressor(tf.keras.Model):\n",
" def __init__(self, output_dim, **kwargs):\n", " def __init__(self, output_dim, **kwargs):\n",
" super().__init__(**kwargs)\n", " super().__init__(**kwargs)\n",
" self.hidden = [keras.layers.Dense(30, activation=\"selu\",\n", " self.hidden = [tf.keras.layers.Dense(30, activation=\"selu\",\n",
" kernel_initializer=\"lecun_normal\")\n", " kernel_initializer=\"lecun_normal\")\n",
" for _ in range(5)]\n", " for _ in range(5)]\n",
" self.out = keras.layers.Dense(output_dim)\n", " self.out = tf.keras.layers.Dense(output_dim)\n",
" self.reconstruction_mean = keras.metrics.Mean(name=\"reconstruction_error\")\n", " self.reconstruction_mean = tf.keras.metrics.Mean(name=\"reconstruction_error\")\n",
"\n", "\n",
" def build(self, batch_input_shape):\n", " def build(self, batch_input_shape):\n",
" n_inputs = batch_input_shape[-1]\n", " n_inputs = batch_input_shape[-1]\n",
" self.reconstruct = keras.layers.Dense(n_inputs)\n", " self.reconstruct = tf.keras.layers.Dense(n_inputs)\n",
" #super().build(batch_input_shape)\n", " #super().build(batch_input_shape)\n",
"\n", "\n",
" def call(self, inputs, training=None):\n", " def call(self, inputs, training=None):\n",
@ -2232,7 +2232,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"keras.backend.clear_session()\n", "tf.keras.backend.clear_session()\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)" "tf.random.set_seed(42)"
] ]
@ -2554,7 +2554,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"keras.backend.clear_session()\n", "tf.keras.backend.clear_session()\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)" "tf.random.set_seed(42)"
] ]
@ -2565,11 +2565,11 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"l2_reg = keras.regularizers.l2(0.05)\n", "l2_reg = tf.keras.regularizers.l2(0.05)\n",
"model = keras.models.Sequential([\n", "model = tf.keras.Sequential([\n",
" keras.layers.Dense(30, activation=\"elu\", kernel_initializer=\"he_normal\",\n", " tf.keras.layers.Dense(30, activation=\"elu\", kernel_initializer=\"he_normal\",\n",
" kernel_regularizer=l2_reg),\n", " kernel_regularizer=l2_reg),\n",
" keras.layers.Dense(1, kernel_regularizer=l2_reg)\n", " tf.keras.layers.Dense(1, kernel_regularizer=l2_reg)\n",
"])" "])"
] ]
}, },
@ -2606,8 +2606,8 @@
"source": [ "source": [
"import time\n", "import time\n",
"\n", "\n",
"mean_loss = keras.metrics.Mean(name=\"loss\")\n", "mean_loss = tf.keras.metrics.Mean(name=\"loss\")\n",
"mean_square = keras.metrics.Mean(name=\"mean_square\")\n", "mean_square = tf.keras.metrics.Mean(name=\"mean_square\")\n",
"for i in range(1, 50 + 1):\n", "for i in range(1, 50 + 1):\n",
" loss = 1 / i\n", " loss = 1 / i\n",
" mean_loss(loss)\n", " mean_loss(loss)\n",
@ -2666,8 +2666,8 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"mean_loss = keras.metrics.Mean(name=\"loss\")\n", "mean_loss = tf.keras.metrics.Mean(name=\"loss\")\n",
"mean_square = keras.metrics.Mean(name=\"mean_square\")\n", "mean_square = tf.keras.metrics.Mean(name=\"mean_square\")\n",
"for i in range(1, 50 + 1):\n", "for i in range(1, 50 + 1):\n",
" loss = 1 / i\n", " loss = 1 / i\n",
" mean_loss(loss)\n", " mean_loss(loss)\n",
@ -2682,7 +2682,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"keras.backend.clear_session()\n", "tf.keras.backend.clear_session()\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)" "tf.random.set_seed(42)"
] ]
@ -2696,10 +2696,10 @@
"n_epochs = 5\n", "n_epochs = 5\n",
"batch_size = 32\n", "batch_size = 32\n",
"n_steps = len(X_train) // batch_size\n", "n_steps = len(X_train) // batch_size\n",
"optimizer = keras.optimizers.Nadam(learning_rate=0.01)\n", "optimizer = tf.keras.optimizers.Nadam(learning_rate=0.01)\n",
"loss_fn = keras.losses.mean_squared_error\n", "loss_fn = tf.keras.losses.mean_squared_error\n",
"mean_loss = keras.metrics.Mean()\n", "mean_loss = tf.keras.metrics.Mean()\n",
"metrics = [keras.metrics.MeanAbsoluteError()]" "metrics = [tf.keras.metrics.MeanAbsoluteError()]"
] ]
}, },
{ {
@ -3013,7 +3013,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"keras.backend.clear_session()\n", "tf.keras.backend.clear_session()\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)" "tf.random.set_seed(42)"
] ]
@ -3341,11 +3341,11 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"# Custom layer\n", "# Custom layer\n",
"class MyDense(keras.layers.Layer):\n", "class MyDense(tf.keras.layers.Layer):\n",
" def __init__(self, units, activation=None, **kwargs):\n", " def __init__(self, units, activation=None, **kwargs):\n",
" super().__init__(**kwargs)\n", " super().__init__(**kwargs)\n",
" self.units = units\n", " self.units = units\n",
" self.activation = keras.activations.get(activation)\n", " self.activation = tf.keras.activations.get(activation)\n",
"\n", "\n",
" def build(self, input_shape):\n", " def build(self, input_shape):\n",
" self.kernel = self.add_weight(name='kernel', \n", " self.kernel = self.add_weight(name='kernel', \n",
@ -3369,7 +3369,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"keras.backend.clear_session()\n", "tf.keras.backend.clear_session()\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)" "tf.random.set_seed(42)"
] ]
@ -3381,7 +3381,7 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"# Custom model\n", "# Custom model\n",
"class MyModel(keras.models.Model):\n", "class MyModel(tf.keras.Model):\n",
" def __init__(self, **kwargs):\n", " def __init__(self, **kwargs):\n",
" super().__init__(**kwargs)\n", " super().__init__(**kwargs)\n",
" self.hidden1 = MyDense(30, activation=\"relu\")\n", " self.hidden1 = MyDense(30, activation=\"relu\")\n",
@ -3392,7 +3392,7 @@
" print(\"Tracing MyModel.call()\")\n", " print(\"Tracing MyModel.call()\")\n",
" hidden1 = self.hidden1(input)\n", " hidden1 = self.hidden1(input)\n",
" hidden2 = self.hidden2(hidden1)\n", " hidden2 = self.hidden2(hidden1)\n",
" concat = keras.layers.concatenate([input, hidden2])\n", " concat = tf.keras.layers.concatenate([input, hidden2])\n",
" output = self.output_(concat)\n", " output = self.output_(concat)\n",
" return output\n", " return output\n",
"\n", "\n",
@ -3432,7 +3432,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"keras.backend.clear_session()\n", "tf.keras.backend.clear_session()\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)" "tf.random.set_seed(42)"
] ]
@ -3486,7 +3486,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"keras.backend.clear_session()\n", "tf.keras.backend.clear_session()\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)" "tf.random.set_seed(42)"
] ]
@ -3540,7 +3540,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"class MyMomentumOptimizer(keras.optimizers.Optimizer):\n", "class MyMomentumOptimizer(tf.keras.optimizers.Optimizer):\n",
" def __init__(self, learning_rate=0.001, momentum=0.9, name=\"MyMomentumOptimizer\", **kwargs):\n", " def __init__(self, learning_rate=0.001, momentum=0.9, name=\"MyMomentumOptimizer\", **kwargs):\n",
" \"\"\"Call super().__init__() and use _set_hyper() to store hyperparameters\"\"\"\n", " \"\"\"Call super().__init__() and use _set_hyper() to store hyperparameters\"\"\"\n",
" super().__init__(name, **kwargs)\n", " super().__init__(name, **kwargs)\n",
@ -3586,7 +3586,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"keras.backend.clear_session()\n", "tf.keras.backend.clear_session()\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)" "tf.random.set_seed(42)"
] ]
@ -3597,7 +3597,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"model = keras.models.Sequential([keras.layers.Dense(1, input_shape=[8])])\n", "model = tf.keras.Sequential([tf.keras.layers.Dense(1, input_shape=[8])])\n",
"model.compile(loss=\"mse\", optimizer=MyMomentumOptimizer())\n", "model.compile(loss=\"mse\", optimizer=MyMomentumOptimizer())\n",
"model.fit(X_train_scaled, y_train, epochs=5)" "model.fit(X_train_scaled, y_train, epochs=5)"
] ]
@ -3654,7 +3654,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"class LayerNormalization(keras.layers.Layer):\n", "class LayerNormalization(tf.keras.layers.Layer):\n",
" def __init__(self, eps=0.001, **kwargs):\n", " def __init__(self, eps=0.001, **kwargs):\n",
" super().__init__(**kwargs)\n", " super().__init__(**kwargs)\n",
" self.eps = eps\n", " self.eps = eps\n",
@ -3692,7 +3692,7 @@
"metadata": {}, "metadata": {},
"source": [ "source": [
"### c.\n", "### c.\n",
"_Exercise: Ensure that your custom layer produces the same (or very nearly the same) output as the `keras.layers.LayerNormalization` layer._" "_Exercise: Ensure that your custom layer produces the same (or very nearly the same) output as the `tf.keras.layers.LayerNormalization` layer._"
] ]
}, },
{ {
@ -3711,9 +3711,9 @@
"X = X_train.astype(np.float32)\n", "X = X_train.astype(np.float32)\n",
"\n", "\n",
"custom_layer_norm = LayerNormalization()\n", "custom_layer_norm = LayerNormalization()\n",
"keras_layer_norm = keras.layers.LayerNormalization()\n", "keras_layer_norm = tf.keras.layers.LayerNormalization()\n",
"\n", "\n",
"tf.reduce_mean(keras.losses.mean_absolute_error(\n", "tf.reduce_mean(tf.keras.losses.mean_absolute_error(\n",
" keras_layer_norm(X), custom_layer_norm(X)))" " keras_layer_norm(X), custom_layer_norm(X)))"
] ]
}, },
@ -3736,7 +3736,7 @@
"custom_layer_norm.set_weights([random_alpha, random_beta])\n", "custom_layer_norm.set_weights([random_alpha, random_beta])\n",
"keras_layer_norm.set_weights([random_alpha, random_beta])\n", "keras_layer_norm.set_weights([random_alpha, random_beta])\n",
"\n", "\n",
"tf.reduce_mean(keras.losses.mean_absolute_error(\n", "tf.reduce_mean(tf.keras.losses.mean_absolute_error(\n",
" keras_layer_norm(X), custom_layer_norm(X)))" " keras_layer_norm(X), custom_layer_norm(X)))"
] ]
}, },
@ -3769,7 +3769,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"(X_train_full, y_train_full), (X_test, y_test) = keras.datasets.fashion_mnist.load_data()\n", "(X_train_full, y_train_full), (X_test, y_test) = tf.keras.datasets.fashion_mnist.load_data()\n",
"X_train_full = X_train_full.astype(np.float32) / 255.\n", "X_train_full = X_train_full.astype(np.float32) / 255.\n",
"X_valid, X_train = X_train_full[:5000], X_train_full[5000:]\n", "X_valid, X_train = X_train_full[:5000], X_train_full[5000:]\n",
"y_valid, y_train = y_train_full[:5000], y_train_full[5000:]\n", "y_valid, y_train = y_train_full[:5000], y_train_full[5000:]\n",
@ -3782,7 +3782,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"keras.backend.clear_session()\n", "tf.keras.backend.clear_session()\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)" "tf.random.set_seed(42)"
] ]
@ -3793,10 +3793,10 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"model = keras.models.Sequential([\n", "model = tf.keras.Sequential([\n",
" keras.layers.Flatten(input_shape=[28, 28]),\n", " tf.keras.layers.Flatten(input_shape=[28, 28]),\n",
" keras.layers.Dense(100, activation=\"relu\"),\n", " tf.keras.layers.Dense(100, activation=\"relu\"),\n",
" keras.layers.Dense(10, activation=\"softmax\"),\n", " tf.keras.layers.Dense(10, activation=\"softmax\"),\n",
"])" "])"
] ]
}, },
@ -3809,10 +3809,10 @@
"n_epochs = 5\n", "n_epochs = 5\n",
"batch_size = 32\n", "batch_size = 32\n",
"n_steps = len(X_train) // batch_size\n", "n_steps = len(X_train) // batch_size\n",
"optimizer = keras.optimizers.Nadam(learning_rate=0.01)\n", "optimizer = tf.keras.optimizers.Nadam(learning_rate=0.01)\n",
"loss_fn = keras.losses.sparse_categorical_crossentropy\n", "loss_fn = tf.keras.losses.sparse_categorical_crossentropy\n",
"mean_loss = keras.metrics.Mean()\n", "mean_loss = tf.keras.metrics.Mean()\n",
"metrics = [keras.metrics.SparseCategoricalAccuracy()]" "metrics = [tf.keras.metrics.SparseCategoricalAccuracy()]"
] ]
}, },
{ {
@ -3844,7 +3844,7 @@
" steps.set_postfix(status)\n", " steps.set_postfix(status)\n",
" y_pred = model(X_valid)\n", " y_pred = model(X_valid)\n",
" status[\"val_loss\"] = np.mean(loss_fn(y_valid, y_pred))\n", " status[\"val_loss\"] = np.mean(loss_fn(y_valid, y_pred))\n",
" status[\"val_accuracy\"] = np.mean(keras.metrics.sparse_categorical_accuracy(\n", " status[\"val_accuracy\"] = np.mean(tf.keras.metrics.sparse_categorical_accuracy(\n",
" tf.constant(y_valid, dtype=np.float32), y_pred))\n", " tf.constant(y_valid, dtype=np.float32), y_pred))\n",
" steps.set_postfix(status)\n", " steps.set_postfix(status)\n",
" for metric in [mean_loss] + metrics:\n", " for metric in [mean_loss] + metrics:\n",
@ -3865,7 +3865,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"keras.backend.clear_session()\n", "tf.keras.backend.clear_session()\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)" "tf.random.set_seed(42)"
] ]
@ -3876,14 +3876,14 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"lower_layers = keras.models.Sequential([\n", "lower_layers = tf.keras.Sequential([\n",
" keras.layers.Flatten(input_shape=[28, 28]),\n", " tf.keras.layers.Flatten(input_shape=[28, 28]),\n",
" keras.layers.Dense(100, activation=\"relu\"),\n", " tf.keras.layers.Dense(100, activation=\"relu\"),\n",
"])\n", "])\n",
"upper_layers = keras.models.Sequential([\n", "upper_layers = tf.keras.Sequential([\n",
" keras.layers.Dense(10, activation=\"softmax\"),\n", " tf.keras.layers.Dense(10, activation=\"softmax\"),\n",
"])\n", "])\n",
"model = keras.models.Sequential([\n", "model = tf.keras.Sequential([\n",
" lower_layers, upper_layers\n", " lower_layers, upper_layers\n",
"])" "])"
] ]
@ -3894,8 +3894,8 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"lower_optimizer = keras.optimizers.SGD(learning_rate=1e-4)\n", "lower_optimizer = tf.keras.optimizers.SGD(learning_rate=1e-4)\n",
"upper_optimizer = keras.optimizers.Nadam(learning_rate=1e-3)" "upper_optimizer = tf.keras.optimizers.Nadam(learning_rate=1e-3)"
] ]
}, },
{ {
@ -3907,9 +3907,9 @@
"n_epochs = 5\n", "n_epochs = 5\n",
"batch_size = 32\n", "batch_size = 32\n",
"n_steps = len(X_train) // batch_size\n", "n_steps = len(X_train) // batch_size\n",
"loss_fn = keras.losses.sparse_categorical_crossentropy\n", "loss_fn = tf.keras.losses.sparse_categorical_crossentropy\n",
"mean_loss = keras.metrics.Mean()\n", "mean_loss = tf.keras.metrics.Mean()\n",
"metrics = [keras.metrics.SparseCategoricalAccuracy()]" "metrics = [tf.keras.metrics.SparseCategoricalAccuracy()]"
] ]
}, },
{ {
@ -3944,7 +3944,7 @@
" steps.set_postfix(status)\n", " steps.set_postfix(status)\n",
" y_pred = model(X_valid)\n", " y_pred = model(X_valid)\n",
" status[\"val_loss\"] = np.mean(loss_fn(y_valid, y_pred))\n", " status[\"val_loss\"] = np.mean(loss_fn(y_valid, y_pred))\n",
" status[\"val_accuracy\"] = np.mean(keras.metrics.sparse_categorical_accuracy(\n", " status[\"val_accuracy\"] = np.mean(tf.keras.metrics.sparse_categorical_accuracy(\n",
" tf.constant(y_valid, dtype=np.float32), y_pred))\n", " tf.constant(y_valid, dtype=np.float32), y_pred))\n",
" steps.set_postfix(status)\n", " steps.set_postfix(status)\n",
" for metric in [mean_loss] + metrics:\n", " for metric in [mean_loss] + metrics:\n",

100
12_loading_and_preprocessing_data.ipynb
View File

@ -548,13 +548,13 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"keras.backend.clear_session()\n", "tf.keras.backend.clear_session()\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)\n", "tf.random.set_seed(42)\n",
"\n", "\n",
"model = keras.models.Sequential([\n", "model = tf.keras.Sequential([\n",
" keras.layers.Dense(30, activation=\"relu\", input_shape=X_train.shape[1:]),\n", " tf.keras.layers.Dense(30, activation=\"relu\", input_shape=X_train.shape[1:]),\n",
" keras.layers.Dense(1),\n", " tf.keras.layers.Dense(1),\n",
"])" "])"
] ]
}, },
@ -564,7 +564,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"model.compile(loss=\"mse\", optimizer=keras.optimizers.SGD(learning_rate=1e-3))" "model.compile(loss=\"mse\", optimizer=tf.keras.optimizers.SGD(learning_rate=1e-3))"
] ]
}, },
{ {
@ -606,8 +606,8 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"optimizer = keras.optimizers.Nadam(learning_rate=0.01)\n", "optimizer = tf.keras.optimizers.Nadam(learning_rate=0.01)\n",
"loss_fn = keras.losses.mean_squared_error\n", "loss_fn = tf.keras.losses.mean_squared_error\n",
"\n", "\n",
"n_epochs = 5\n", "n_epochs = 5\n",
"batch_size = 32\n", "batch_size = 32\n",
@ -631,7 +631,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"keras.backend.clear_session()\n", "tf.keras.backend.clear_session()\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)" "tf.random.set_seed(42)"
] ]
@ -642,8 +642,8 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"optimizer = keras.optimizers.Nadam(learning_rate=0.01)\n", "optimizer = tf.keras.optimizers.Nadam(learning_rate=0.01)\n",
"loss_fn = keras.losses.mean_squared_error\n", "loss_fn = tf.keras.losses.mean_squared_error\n",
"\n", "\n",
"@tf.function\n", "@tf.function\n",
"def train(model, n_epochs, batch_size=32,\n", "def train(model, n_epochs, batch_size=32,\n",
@ -668,7 +668,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"keras.backend.clear_session()\n", "tf.keras.backend.clear_session()\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)" "tf.random.set_seed(42)"
] ]
@ -679,8 +679,8 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"optimizer = keras.optimizers.Nadam(learning_rate=0.01)\n", "optimizer = tf.keras.optimizers.Nadam(learning_rate=0.01)\n",
"loss_fn = keras.losses.mean_squared_error\n", "loss_fn = tf.keras.losses.mean_squared_error\n",
"\n", "\n",
"@tf.function\n", "@tf.function\n",
"def train(model, n_epochs, batch_size=32,\n", "def train(model, n_epochs, batch_size=32,\n",
@ -1632,7 +1632,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"keras.backend.clear_session()\n", "tf.keras.backend.clear_session()\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)" "tf.random.set_seed(42)"
] ]
@ -1667,12 +1667,12 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"columns_without_target = columns[:-1]\n", "columns_without_target = columns[:-1]\n",
"model = keras.models.Sequential([\n", "model = tf.keras.Sequential([\n",
" keras.layers.DenseFeatures(feature_columns=columns_without_target),\n", " tf.keras.layers.DenseFeatures(feature_columns=columns_without_target),\n",
" keras.layers.Dense(1)\n", " tf.keras.layers.Dense(1)\n",
"])\n", "])\n",
"model.compile(loss=\"mse\",\n", "model.compile(loss=\"mse\",\n",
" optimizer=keras.optimizers.SGD(learning_rate=1e-3),\n", " optimizer=tf.keras.optimizers.SGD(learning_rate=1e-3),\n",
" metrics=[\"accuracy\"])\n", " metrics=[\"accuracy\"])\n",
"model.fit(dataset, steps_per_epoch=len(X_train) // batch_size, epochs=5)" "model.fit(dataset, steps_per_epoch=len(X_train) // batch_size, epochs=5)"
] ]
@ -1684,7 +1684,7 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"some_columns = [ocean_proximity_embed, bucketized_income]\n", "some_columns = [ocean_proximity_embed, bucketized_income]\n",
"dense_features = keras.layers.DenseFeatures(some_columns)\n", "dense_features = tf.keras.layers.DenseFeatures(some_columns)\n",
"dense_features({\n", "dense_features({\n",
" \"ocean_proximity\": [[\"NEAR OCEAN\"], [\"INLAND\"], [\"INLAND\"]],\n", " \"ocean_proximity\": [[\"NEAR OCEAN\"], [\"INLAND\"], [\"INLAND\"]],\n",
" \"median_income\": [[3.], [7.2], [1.]]\n", " \"median_income\": [[3.], [7.2], [1.]]\n",
@ -1791,7 +1791,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"keras.backend.clear_session()\n", "tf.keras.backend.clear_session()\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)" "tf.random.set_seed(42)"
] ]
@ -1804,12 +1804,12 @@
"source": [ "source": [
"datasets = tfds.load(name=\"mnist\", batch_size=32, as_supervised=True)\n", "datasets = tfds.load(name=\"mnist\", batch_size=32, as_supervised=True)\n",
"mnist_train = datasets[\"train\"].repeat().prefetch(1)\n", "mnist_train = datasets[\"train\"].repeat().prefetch(1)\n",
"model = keras.models.Sequential([\n", "model = tf.keras.Sequential([\n",
" keras.layers.Flatten(input_shape=[28, 28, 1]),\n", " tf.keras.layers.Flatten(input_shape=[28, 28, 1]),\n",
" keras.layers.Lambda(lambda images: tf.cast(images, tf.float32)),\n", " tf.keras.layers.Lambda(lambda images: tf.cast(images, tf.float32)),\n",
" keras.layers.Dense(10, activation=\"softmax\")])\n", " tf.keras.layers.Dense(10, activation=\"softmax\")])\n",
"model.compile(loss=\"sparse_categorical_crossentropy\",\n", "model.compile(loss=\"sparse_categorical_crossentropy\",\n",
" optimizer=keras.optimizers.SGD(learning_rate=1e-3),\n", " optimizer=tf.keras.optimizers.SGD(learning_rate=1e-3),\n",
" metrics=[\"accuracy\"])\n", " metrics=[\"accuracy\"])\n",
"model.fit(mnist_train, steps_per_epoch=60000 // 32, epochs=5)" "model.fit(mnist_train, steps_per_epoch=60000 // 32, epochs=5)"
] ]
@ -1827,7 +1827,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"keras.backend.clear_session()\n", "tf.keras.backend.clear_session()\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)" "tf.random.set_seed(42)"
] ]
@ -1843,10 +1843,10 @@
"hub_layer = hub.KerasLayer(\"https://tfhub.dev/google/nnlm-en-dim50/2\",\n", "hub_layer = hub.KerasLayer(\"https://tfhub.dev/google/nnlm-en-dim50/2\",\n",
" output_shape=[50], input_shape=[], dtype=tf.string)\n", " output_shape=[50], input_shape=[], dtype=tf.string)\n",
"\n", "\n",
"model = keras.Sequential()\n", "model = tf.keras.Sequential()\n",
"model.add(hub_layer)\n", "model.add(hub_layer)\n",
"model.add(keras.layers.Dense(16, activation='relu'))\n", "model.add(tf.keras.layers.Dense(16, activation='relu'))\n",
"model.add(keras.layers.Dense(1, activation='sigmoid'))\n", "model.add(tf.keras.layers.Dense(1, activation='sigmoid'))\n",
"\n", "\n",
"model.summary()" "model.summary()"
] ]
@ -1890,7 +1890,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"(X_train_full, y_train_full), (X_test, y_test) = keras.datasets.fashion_mnist.load_data()\n", "(X_train_full, y_train_full), (X_test, y_test) = tf.keras.datasets.fashion_mnist.load_data()\n",
"X_valid, X_train = X_train_full[:5000], X_train_full[5000:]\n", "X_valid, X_train = X_train_full[:5000], X_train_full[5000:]\n",
"y_valid, y_train = y_train_full[:5000], y_train_full[5000:]" "y_valid, y_train = y_train_full[:5000], y_train_full[5000:]"
] ]
@ -1901,7 +1901,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"keras.backend.clear_session()\n", "tf.keras.backend.clear_session()\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)" "tf.random.set_seed(42)"
] ]
@ -2052,31 +2052,31 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"keras.backend.clear_session()\n", "tf.keras.backend.clear_session()\n",
"tf.random.set_seed(42)\n", "tf.random.set_seed(42)\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"\n", "\n",
"class Standardization(keras.layers.Layer):\n", "class Standardization(tf.keras.layers.Layer):\n",
" def adapt(self, data_sample):\n", " def adapt(self, data_sample):\n",
" self.means_ = np.mean(data_sample, axis=0, keepdims=True)\n", " self.means_ = np.mean(data_sample, axis=0, keepdims=True)\n",
" self.stds_ = np.std(data_sample, axis=0, keepdims=True)\n", " self.stds_ = np.std(data_sample, axis=0, keepdims=True)\n",
" def call(self, inputs):\n", " def call(self, inputs):\n",
" return (inputs - self.means_) / (self.stds_ + keras.backend.epsilon())\n", " return (inputs - self.means_) / (self.stds_ + tf.keras.backend.epsilon())\n",
"\n", "\n",
"standardization = Standardization(input_shape=[28, 28])\n", "standardization = Standardization(input_shape=[28, 28])\n",
"# or perhaps soon:\n", "# or perhaps soon:\n",
"#standardization = keras.layers.Normalization()\n", "#standardization = tf.keras.layers.Normalization()\n",
"\n", "\n",
"sample_image_batches = train_set.take(100).map(lambda image, label: image)\n", "sample_image_batches = train_set.take(100).map(lambda image, label: image)\n",
"sample_images = np.concatenate(list(sample_image_batches.as_numpy_iterator()),\n", "sample_images = np.concatenate(list(sample_image_batches.as_numpy_iterator()),\n",
" axis=0).astype(np.float32)\n", " axis=0).astype(np.float32)\n",
"standardization.adapt(sample_images)\n", "standardization.adapt(sample_images)\n",
"\n", "\n",
"model = keras.models.Sequential([\n", "model = tf.keras.Sequential([\n",
" standardization,\n", " standardization,\n",
" keras.layers.Flatten(),\n", " tf.keras.layers.Flatten(),\n",
" keras.layers.Dense(100, activation=\"relu\"),\n", " tf.keras.layers.Dense(100, activation=\"relu\"),\n",
" keras.layers.Dense(10, activation=\"softmax\")\n", " tf.keras.layers.Dense(10, activation=\"softmax\")\n",
"])\n", "])\n",
"model.compile(loss=\"sparse_categorical_crossentropy\",\n", "model.compile(loss=\"sparse_categorical_crossentropy\",\n",
" optimizer=\"nadam\", metrics=[\"accuracy\"])" " optimizer=\"nadam\", metrics=[\"accuracy\"])"
@ -2137,7 +2137,7 @@
"\n", "\n",
"root = \"http://ai.stanford.edu/~amaas/data/sentiment/\"\n", "root = \"http://ai.stanford.edu/~amaas/data/sentiment/\"\n",
"filename = \"aclImdb_v1.tar.gz\"\n", "filename = \"aclImdb_v1.tar.gz\"\n",
"filepath = keras.utils.get_file(filename, root + filename, extract=True)\n", "filepath = tf.keras.utils.get_file(filename, root + filename, extract=True)\n",
"path = Path(filepath).with_name(\"aclImdb\")\n", "path = Path(filepath).with_name(\"aclImdb\")\n",
"path" "path"
] ]
@ -2416,7 +2416,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"class TextVectorization(keras.layers.Layer):\n", "class TextVectorization(tf.keras.layers.Layer):\n",
" def __init__(self, max_vocabulary_size=1000, n_oov_buckets=100, dtype=tf.string, **kwargs):\n", " def __init__(self, max_vocabulary_size=1000, n_oov_buckets=100, dtype=tf.string, **kwargs):\n",
" super().__init__(dtype=dtype, **kwargs)\n", " super().__init__(dtype=dtype, **kwargs)\n",
" self.max_vocabulary_size = max_vocabulary_size\n", " self.max_vocabulary_size = max_vocabulary_size\n",
@ -2549,7 +2549,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"class BagOfWords(keras.layers.Layer):\n", "class BagOfWords(tf.keras.layers.Layer):\n",
" def __init__(self, n_tokens, dtype=tf.int32, **kwargs):\n", " def __init__(self, n_tokens, dtype=tf.int32, **kwargs):\n",
" super().__init__(dtype=dtype, **kwargs)\n", " super().__init__(dtype=dtype, **kwargs)\n",
" self.n_tokens = n_tokens\n", " self.n_tokens = n_tokens\n",
@ -2605,11 +2605,11 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"model = keras.models.Sequential([\n", "model = tf.keras.Sequential([\n",
" text_vectorization,\n", " text_vectorization,\n",
" bag_of_words,\n", " bag_of_words,\n",
" keras.layers.Dense(100, activation=\"relu\"),\n", " tf.keras.layers.Dense(100, activation=\"relu\"),\n",
" keras.layers.Dense(1, activation=\"sigmoid\"),\n", " tf.keras.layers.Dense(1, activation=\"sigmoid\"),\n",
"])\n", "])\n",
"model.compile(loss=\"binary_crossentropy\", optimizer=\"nadam\",\n", "model.compile(loss=\"binary_crossentropy\", optimizer=\"nadam\",\n",
" metrics=[\"accuracy\"])\n", " metrics=[\"accuracy\"])\n",
@ -2702,14 +2702,14 @@
"source": [ "source": [
"embedding_size = 20\n", "embedding_size = 20\n",
"\n", "\n",
"model = keras.models.Sequential([\n", "model = tf.keras.Sequential([\n",
" text_vectorization,\n", " text_vectorization,\n",
" keras.layers.Embedding(input_dim=n_tokens,\n", " tf.keras.layers.Embedding(input_dim=n_tokens,\n",
" output_dim=embedding_size,\n", " output_dim=embedding_size,\n",
" mask_zero=True), # <pad> tokens => zero vectors\n", " mask_zero=True), # <pad> tokens => zero vectors\n",
" keras.layers.Lambda(compute_mean_embedding),\n", " tf.keras.layers.Lambda(compute_mean_embedding),\n",
" keras.layers.Dense(100, activation=\"relu\"),\n", " tf.keras.layers.Dense(100, activation=\"relu\"),\n",
" keras.layers.Dense(1, activation=\"sigmoid\"),\n", " tf.keras.layers.Dense(1, activation=\"sigmoid\"),\n",
"])" "])"
] ]
}, },

126
13_deep_computer_vision_with_cnns.ipynb
View File

@ -217,7 +217,7 @@
"cell_type": "markdown", "cell_type": "markdown",
"metadata": {}, "metadata": {},
"source": [ "source": [
"Let's create a 2D convolutional layer, using `keras.layers.Conv2D()`:" "Let's create a 2D convolutional layer, using `tf.keras.layers.Conv2D()`:"
] ]
}, },
{ {
@ -229,7 +229,7 @@
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)\n", "tf.random.set_seed(42)\n",
"\n", "\n",
"conv = keras.layers.Conv2D(filters=2, kernel_size=7, strides=1,\n", "conv = tf.keras.layers.Conv2D(filters=2, kernel_size=7, strides=1,\n",
" padding=\"SAME\", activation=\"relu\", input_shape=outputs.shape)" " padding=\"SAME\", activation=\"relu\", input_shape=outputs.shape)"
] ]
}, },
@ -399,8 +399,8 @@
"kernel_size = 7\n", "kernel_size = 7\n",
"strides = 2\n", "strides = 2\n",
"\n", "\n",
"conv_valid = keras.layers.Conv2D(filters=1, kernel_size=kernel_size, strides=strides, padding=\"VALID\")\n", "conv_valid = tf.keras.layers.Conv2D(filters=1, kernel_size=kernel_size, strides=strides, padding=\"VALID\")\n",
"conv_same = keras.layers.Conv2D(filters=1, kernel_size=kernel_size, strides=strides, padding=\"SAME\")\n", "conv_same = tf.keras.layers.Conv2D(filters=1, kernel_size=kernel_size, strides=strides, padding=\"SAME\")\n",
"\n", "\n",
"valid_output = conv_valid(manual_same_padding(images, kernel_size, strides))\n", "valid_output = conv_valid(manual_same_padding(images, kernel_size, strides))\n",
"\n", "\n",
@ -435,7 +435,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"max_pool = keras.layers.MaxPool2D(pool_size=2)" "max_pool = tf.keras.layers.MaxPool2D(pool_size=2)"
] ]
}, },
{ {
@ -482,7 +482,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"class DepthMaxPool(keras.layers.Layer):\n", "class DepthMaxPool(tf.keras.layers.Layer):\n",
" def __init__(self, pool_size, strides=None, padding=\"VALID\", **kwargs):\n", " def __init__(self, pool_size, strides=None, padding=\"VALID\", **kwargs):\n",
" super().__init__(**kwargs)\n", " super().__init__(**kwargs)\n",
" if strides is None:\n", " if strides is None:\n",
@ -522,7 +522,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"depth_pool = keras.layers.Lambda(lambda X: tf.nn.max_pool(\n", "depth_pool = tf.keras.layers.Lambda(lambda X: tf.nn.max_pool(\n",
" X, ksize=(1, 1, 1, 3), strides=(1, 1, 1, 3), padding=\"VALID\"))\n", " X, ksize=(1, 1, 1, 3), strides=(1, 1, 1, 3), padding=\"VALID\"))\n",
"with tf.device(\"/cpu:0\"): # there is no GPU-kernel yet\n", "with tf.device(\"/cpu:0\"): # there is no GPU-kernel yet\n",
" depth_output = depth_pool(cropped_images)\n", " depth_output = depth_pool(cropped_images)\n",
@ -559,7 +559,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"avg_pool = keras.layers.AvgPool2D(pool_size=2)" "avg_pool = tf.keras.layers.AvgPool2D(pool_size=2)"
] ]
}, },
{ {
@ -604,7 +604,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"global_avg_pool = keras.layers.GlobalAvgPool2D()\n", "global_avg_pool = tf.keras.layers.GlobalAvgPool2D()\n",
"global_avg_pool(cropped_images)" "global_avg_pool(cropped_images)"
] ]
}, },
@ -614,7 +614,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"output_global_avg2 = keras.layers.Lambda(lambda X: tf.reduce_mean(X, axis=[1, 2]))\n", "output_global_avg2 = tf.keras.layers.Lambda(lambda X: tf.reduce_mean(X, axis=[1, 2]))\n",
"output_global_avg2(cropped_images)" "output_global_avg2(cropped_images)"
] ]
}, },
@ -631,7 +631,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"(X_train_full, y_train_full), (X_test, y_test) = keras.datasets.fashion_mnist.load_data()\n", "(X_train_full, y_train_full), (X_test, y_test) = tf.keras.datasets.fashion_mnist.load_data()\n",
"X_train, X_valid = X_train_full[:-5000], X_train_full[-5000:]\n", "X_train, X_valid = X_train_full[:-5000], X_train_full[-5000:]\n",
"y_train, y_valid = y_train_full[:-5000], y_train_full[-5000:]\n", "y_train, y_valid = y_train_full[:-5000], y_train_full[-5000:]\n",
"\n", "\n",
@ -654,24 +654,24 @@
"source": [ "source": [
"from functools import partial\n", "from functools import partial\n",
"\n", "\n",
"DefaultConv2D = partial(keras.layers.Conv2D,\n", "DefaultConv2D = partial(tf.keras.layers.Conv2D,\n",
" kernel_size=3, activation='relu', padding=\"SAME\")\n", " kernel_size=3, activation='relu', padding=\"SAME\")\n",
"\n", "\n",
"model = keras.models.Sequential([\n", "model = tf.keras.Sequential([\n",
" DefaultConv2D(filters=64, kernel_size=7, input_shape=[28, 28, 1]),\n", " DefaultConv2D(filters=64, kernel_size=7, input_shape=[28, 28, 1]),\n",
" keras.layers.MaxPooling2D(pool_size=2),\n", " tf.keras.layers.MaxPooling2D(pool_size=2),\n",
" DefaultConv2D(filters=128),\n", " DefaultConv2D(filters=128),\n",
" DefaultConv2D(filters=128),\n", " DefaultConv2D(filters=128),\n",
" keras.layers.MaxPooling2D(pool_size=2),\n", " tf.keras.layers.MaxPooling2D(pool_size=2),\n",
" DefaultConv2D(filters=256),\n", " DefaultConv2D(filters=256),\n",
" DefaultConv2D(filters=256),\n", " DefaultConv2D(filters=256),\n",
" keras.layers.MaxPooling2D(pool_size=2),\n", " tf.keras.layers.MaxPooling2D(pool_size=2),\n",
" keras.layers.Flatten(),\n", " tf.keras.layers.Flatten(),\n",
" keras.layers.Dense(units=128, activation='relu'),\n", " tf.keras.layers.Dense(units=128, activation='relu'),\n",
" keras.layers.Dropout(0.5),\n", " tf.keras.layers.Dropout(0.5),\n",
" keras.layers.Dense(units=64, activation='relu'),\n", " tf.keras.layers.Dense(units=64, activation='relu'),\n",
" keras.layers.Dropout(0.5),\n", " tf.keras.layers.Dropout(0.5),\n",
" keras.layers.Dense(units=10, activation='softmax'),\n", " tf.keras.layers.Dense(units=10, activation='softmax'),\n",
"])" "])"
] ]
}, },
@ -701,24 +701,24 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"DefaultConv2D = partial(keras.layers.Conv2D, kernel_size=3, strides=1,\n", "DefaultConv2D = partial(tf.keras.layers.Conv2D, kernel_size=3, strides=1,\n",
" padding=\"SAME\", use_bias=False)\n", " padding=\"SAME\", use_bias=False)\n",
"\n", "\n",
"class ResidualUnit(keras.layers.Layer):\n", "class ResidualUnit(tf.keras.layers.Layer):\n",
" def __init__(self, filters, strides=1, activation=\"relu\", **kwargs):\n", " def __init__(self, filters, strides=1, activation=\"relu\", **kwargs):\n",
" super().__init__(**kwargs)\n", " super().__init__(**kwargs)\n",
" self.activation = keras.activations.get(activation)\n", " self.activation = tf.keras.activations.get(activation)\n",
" self.main_layers = [\n", " self.main_layers = [\n",
" DefaultConv2D(filters, strides=strides),\n", " DefaultConv2D(filters, strides=strides),\n",
" keras.layers.BatchNormalization(),\n", " tf.keras.layers.BatchNormalization(),\n",
" self.activation,\n", " self.activation,\n",
" DefaultConv2D(filters),\n", " DefaultConv2D(filters),\n",
" keras.layers.BatchNormalization()]\n", " tf.keras.layers.BatchNormalization()]\n",
" self.skip_layers = []\n", " self.skip_layers = []\n",
" if strides > 1:\n", " if strides > 1:\n",
" self.skip_layers = [\n", " self.skip_layers = [\n",
" DefaultConv2D(filters, kernel_size=1, strides=strides),\n", " DefaultConv2D(filters, kernel_size=1, strides=strides),\n",
" keras.layers.BatchNormalization()]\n", " tf.keras.layers.BatchNormalization()]\n",
"\n", "\n",
" def call(self, inputs):\n", " def call(self, inputs):\n",
" Z = inputs\n", " Z = inputs\n",
@ -736,20 +736,20 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"model = keras.models.Sequential()\n", "model = tf.keras.Sequential()\n",
"model.add(DefaultConv2D(64, kernel_size=7, strides=2,\n", "model.add(DefaultConv2D(64, kernel_size=7, strides=2,\n",
" input_shape=[224, 224, 3]))\n", " input_shape=[224, 224, 3]))\n",
"model.add(keras.layers.BatchNormalization())\n", "model.add(tf.keras.layers.BatchNormalization())\n",
"model.add(keras.layers.Activation(\"relu\"))\n", "model.add(tf.keras.layers.Activation(\"relu\"))\n",
"model.add(keras.layers.MaxPool2D(pool_size=3, strides=2, padding=\"SAME\"))\n", "model.add(tf.keras.layers.MaxPool2D(pool_size=3, strides=2, padding=\"SAME\"))\n",
"prev_filters = 64\n", "prev_filters = 64\n",
"for filters in [64] * 3 + [128] * 4 + [256] * 6 + [512] * 3:\n", "for filters in [64] * 3 + [128] * 4 + [256] * 6 + [512] * 3:\n",
" strides = 1 if filters == prev_filters else 2\n", " strides = 1 if filters == prev_filters else 2\n",
" model.add(ResidualUnit(filters, strides=strides))\n", " model.add(ResidualUnit(filters, strides=strides))\n",
" prev_filters = filters\n", " prev_filters = filters\n",
"model.add(keras.layers.GlobalAvgPool2D())\n", "model.add(tf.keras.layers.GlobalAvgPool2D())\n",
"model.add(keras.layers.Flatten())\n", "model.add(tf.keras.layers.Flatten())\n",
"model.add(keras.layers.Dense(10, activation=\"softmax\"))" "model.add(tf.keras.layers.Dense(10, activation=\"softmax\"))"
] ]
}, },
{ {
@ -774,7 +774,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"model = keras.applications.resnet50.ResNet50(weights=\"imagenet\")" "model = tf.keras.applications.resnet50.ResNet50(weights=\"imagenet\")"
] ]
}, },
{ {
@ -830,7 +830,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"inputs = keras.applications.resnet50.preprocess_input(images_resized * 255)\n", "inputs = tf.keras.applications.resnet50.preprocess_input(images_resized * 255)\n",
"Y_proba = model.predict(inputs)" "Y_proba = model.predict(inputs)"
] ]
}, },
@ -849,7 +849,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"top_K = keras.applications.resnet50.decode_predictions(Y_proba, top=3)\n", "top_K = tf.keras.applications.resnet50.decode_predictions(Y_proba, top=3)\n",
"for image_index in range(len(images)):\n", "for image_index in range(len(images)):\n",
" print(\"Image #{}\".format(image_index))\n", " print(\"Image #{}\".format(image_index))\n",
" for class_id, name, y_proba in top_K[image_index]:\n", " for class_id, name, y_proba in top_K[image_index]:\n",
@ -974,7 +974,7 @@
"source": [ "source": [
"def preprocess(image, label):\n", "def preprocess(image, label):\n",
" resized_image = tf.image.resize(image, [224, 224])\n", " resized_image = tf.image.resize(image, [224, 224])\n",
" final_image = keras.applications.xception.preprocess_input(resized_image)\n", " final_image = tf.keras.applications.xception.preprocess_input(resized_image)\n",
" return final_image, label" " return final_image, label"
] ]
}, },
@ -1012,7 +1012,7 @@
" else:\n", " else:\n",
" cropped_image = central_crop(image)\n", " cropped_image = central_crop(image)\n",
" resized_image = tf.image.resize(cropped_image, [224, 224])\n", " resized_image = tf.image.resize(cropped_image, [224, 224])\n",
" final_image = keras.applications.xception.preprocess_input(resized_image)\n", " final_image = tf.keras.applications.xception.preprocess_input(resized_image)\n",
" return final_image, label\n", " return final_image, label\n",
"\n", "\n",
"batch_size = 32\n", "batch_size = 32\n",
@ -1062,11 +1062,11 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"base_model = keras.applications.xception.Xception(weights=\"imagenet\",\n", "base_model = tf.keras.applications.xception.Xception(weights=\"imagenet\",\n",
" include_top=False)\n", " include_top=False)\n",
"avg = keras.layers.GlobalAveragePooling2D()(base_model.output)\n", "avg = tf.keras.layers.GlobalAveragePooling2D()(base_model.output)\n",
"output = keras.layers.Dense(n_classes, activation=\"softmax\")(avg)\n", "output = tf.keras.layers.Dense(n_classes, activation=\"softmax\")(avg)\n",
"model = keras.models.Model(inputs=base_model.input, outputs=output)" "model = tf.keras.Model(inputs=base_model.input, outputs=output)"
] ]
}, },
{ {
@ -1088,7 +1088,7 @@
"for layer in base_model.layers:\n", "for layer in base_model.layers:\n",
" layer.trainable = False\n", " layer.trainable = False\n",
"\n", "\n",
"optimizer = keras.optimizers.SGD(learning_rate=0.2, momentum=0.9, decay=0.01)\n", "optimizer = tf.keras.optimizers.SGD(learning_rate=0.2, momentum=0.9, decay=0.01)\n",
"model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=optimizer,\n", "model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=optimizer,\n",
" metrics=[\"accuracy\"])\n", " metrics=[\"accuracy\"])\n",
"history = model.fit(train_set,\n", "history = model.fit(train_set,\n",
@ -1107,7 +1107,7 @@
"for layer in base_model.layers:\n", "for layer in base_model.layers:\n",
" layer.trainable = True\n", " layer.trainable = True\n",
"\n", "\n",
"optimizer = keras.optimizers.SGD(learning_rate=0.01, momentum=0.9,\n", "optimizer = tf.keras.optimizers.SGD(learning_rate=0.01, momentum=0.9,\n",
" nesterov=True, decay=0.001)\n", " nesterov=True, decay=0.001)\n",
"model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=optimizer,\n", "model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=optimizer,\n",
" metrics=[\"accuracy\"])\n", " metrics=[\"accuracy\"])\n",
@ -1131,12 +1131,12 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"base_model = keras.applications.xception.Xception(weights=\"imagenet\",\n", "base_model = tf.keras.applications.xception.Xception(weights=\"imagenet\",\n",
" include_top=False)\n", " include_top=False)\n",
"avg = keras.layers.GlobalAveragePooling2D()(base_model.output)\n", "avg = tf.keras.layers.GlobalAveragePooling2D()(base_model.output)\n",
"class_output = keras.layers.Dense(n_classes, activation=\"softmax\")(avg)\n", "class_output = tf.keras.layers.Dense(n_classes, activation=\"softmax\")(avg)\n",
"loc_output = keras.layers.Dense(4)(avg)\n", "loc_output = tf.keras.layers.Dense(4)(avg)\n",
"model = keras.models.Model(inputs=base_model.input,\n", "model = tf.keras.Model(inputs=base_model.input,\n",
" outputs=[class_output, loc_output])\n", " outputs=[class_output, loc_output])\n",
"model.compile(loss=[\"sparse_categorical_crossentropy\", \"mse\"],\n", "model.compile(loss=[\"sparse_categorical_crossentropy\", \"mse\"],\n",
" loss_weights=[0.8, 0.2], # depends on what you care most about\n", " loss_weights=[0.8, 0.2], # depends on what you care most about\n",
@ -1220,7 +1220,7 @@
"tf.random.set_seed(42)\n", "tf.random.set_seed(42)\n",
"X = images_resized.numpy()\n", "X = images_resized.numpy()\n",
"\n", "\n",
"conv_transpose = keras.layers.Conv2DTranspose(filters=5, kernel_size=3, strides=2, padding=\"VALID\")\n", "conv_transpose = tf.keras.layers.Conv2DTranspose(filters=5, kernel_size=3, strides=2, padding=\"VALID\")\n",
"output = conv_transpose(X)\n", "output = conv_transpose(X)\n",
"output.shape" "output.shape"
] ]
@ -1360,7 +1360,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"(X_train_full, y_train_full), (X_test, y_test) = keras.datasets.mnist.load_data()\n", "(X_train_full, y_train_full), (X_test, y_test) = tf.keras.datasets.mnist.load_data()\n",
"X_train_full = X_train_full / 255.\n", "X_train_full = X_train_full / 255.\n",
"X_test = X_test / 255.\n", "X_test = X_test / 255.\n",
"X_train, X_valid = X_train_full[:-5000], X_train_full[-5000:]\n", "X_train, X_valid = X_train_full[:-5000], X_train_full[-5000:]\n",
@ -1377,19 +1377,19 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"keras.backend.clear_session()\n", "tf.keras.backend.clear_session()\n",
"tf.random.set_seed(42)\n", "tf.random.set_seed(42)\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"\n", "\n",
"model = keras.models.Sequential([\n", "model = tf.keras.Sequential([\n",
" keras.layers.Conv2D(32, kernel_size=3, padding=\"same\", activation=\"relu\"),\n", " tf.keras.layers.Conv2D(32, kernel_size=3, padding=\"same\", activation=\"relu\"),\n",
" keras.layers.Conv2D(64, kernel_size=3, padding=\"same\", activation=\"relu\"),\n", " tf.keras.layers.Conv2D(64, kernel_size=3, padding=\"same\", activation=\"relu\"),\n",
" keras.layers.MaxPool2D(),\n", " tf.keras.layers.MaxPool2D(),\n",
" keras.layers.Flatten(),\n", " tf.keras.layers.Flatten(),\n",
" keras.layers.Dropout(0.25),\n", " tf.keras.layers.Dropout(0.25),\n",
" keras.layers.Dense(128, activation=\"relu\"),\n", " tf.keras.layers.Dense(128, activation=\"relu\"),\n",
" keras.layers.Dropout(0.5),\n", " tf.keras.layers.Dropout(0.5),\n",
" keras.layers.Dense(10, activation=\"softmax\")\n", " tf.keras.layers.Dense(10, activation=\"softmax\")\n",
"])\n", "])\n",
"model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=\"nadam\",\n", "model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=\"nadam\",\n",
" metrics=[\"accuracy\"])\n", " metrics=[\"accuracy\"])\n",

206
14_processing_sequences_using_rnns_and_cnns.ipynb
View File

@ -211,7 +211,7 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"y_pred = X_valid[:, -1]\n", "y_pred = X_valid[:, -1]\n",
"np.mean(keras.losses.mean_squared_error(y_valid, y_pred))" "np.mean(tf.keras.losses.mean_squared_error(y_valid, y_pred))"
] ]
}, },
{ {
@ -240,9 +240,9 @@
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)\n", "tf.random.set_seed(42)\n",
"\n", "\n",
"model = keras.models.Sequential([\n", "model = tf.keras.Sequential([\n",
" keras.layers.Flatten(input_shape=[50, 1]),\n", " tf.keras.layers.Flatten(input_shape=[50, 1]),\n",
" keras.layers.Dense(1)\n", " tf.keras.layers.Dense(1)\n",
"])\n", "])\n",
"\n", "\n",
"model.compile(loss=\"mse\", optimizer=\"adam\")\n", "model.compile(loss=\"mse\", optimizer=\"adam\")\n",
@ -306,11 +306,11 @@
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)\n", "tf.random.set_seed(42)\n",
"\n", "\n",
"model = keras.models.Sequential([\n", "model = tf.keras.Sequential([\n",
" keras.layers.SimpleRNN(1, input_shape=[None, 1])\n", " tf.keras.layers.SimpleRNN(1, input_shape=[None, 1])\n",
"])\n", "])\n",
"\n", "\n",
"optimizer = keras.optimizers.Adam(learning_rate=0.005)\n", "optimizer = tf.keras.optimizers.Adam(learning_rate=0.005)\n",
"model.compile(loss=\"mse\", optimizer=optimizer)\n", "model.compile(loss=\"mse\", optimizer=optimizer)\n",
"history = model.fit(X_train, y_train, epochs=20,\n", "history = model.fit(X_train, y_train, epochs=20,\n",
" validation_data=(X_valid, y_valid))" " validation_data=(X_valid, y_valid))"
@ -362,10 +362,10 @@
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)\n", "tf.random.set_seed(42)\n",
"\n", "\n",
"model = keras.models.Sequential([\n", "model = tf.keras.Sequential([\n",
" keras.layers.SimpleRNN(20, return_sequences=True, input_shape=[None, 1]),\n", " tf.keras.layers.SimpleRNN(20, return_sequences=True, input_shape=[None, 1]),\n",
" keras.layers.SimpleRNN(20, return_sequences=True),\n", " tf.keras.layers.SimpleRNN(20, return_sequences=True),\n",
" keras.layers.SimpleRNN(1)\n", " tf.keras.layers.SimpleRNN(1)\n",
"])\n", "])\n",
"\n", "\n",
"model.compile(loss=\"mse\", optimizer=\"adam\")\n", "model.compile(loss=\"mse\", optimizer=\"adam\")\n",
@ -419,10 +419,10 @@
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)\n", "tf.random.set_seed(42)\n",
"\n", "\n",
"model = keras.models.Sequential([\n", "model = tf.keras.Sequential([\n",
" keras.layers.SimpleRNN(20, return_sequences=True, input_shape=[None, 1]),\n", " tf.keras.layers.SimpleRNN(20, return_sequences=True, input_shape=[None, 1]),\n",
" keras.layers.SimpleRNN(20),\n", " tf.keras.layers.SimpleRNN(20),\n",
" keras.layers.Dense(1)\n", " tf.keras.layers.Dense(1)\n",
"])\n", "])\n",
"\n", "\n",
"model.compile(loss=\"mse\", optimizer=\"adam\")\n", "model.compile(loss=\"mse\", optimizer=\"adam\")\n",
@ -572,7 +572,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"np.mean(keras.metrics.mean_squared_error(Y_valid, Y_pred))" "np.mean(tf.keras.metrics.mean_squared_error(Y_valid, Y_pred))"
] ]
}, },
{ {
@ -589,7 +589,7 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"Y_naive_pred = np.tile(X_valid[:, -1], 10) # take the last time step value, and repeat it 10 times\n", "Y_naive_pred = np.tile(X_valid[:, -1], 10) # take the last time step value, and repeat it 10 times\n",
"np.mean(keras.metrics.mean_squared_error(Y_valid, Y_naive_pred))" "np.mean(tf.keras.metrics.mean_squared_error(Y_valid, Y_naive_pred))"
] ]
}, },
{ {
@ -601,9 +601,9 @@
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)\n", "tf.random.set_seed(42)\n",
"\n", "\n",
"model = keras.models.Sequential([\n", "model = tf.keras.Sequential([\n",
" keras.layers.Flatten(input_shape=[50, 1]),\n", " tf.keras.layers.Flatten(input_shape=[50, 1]),\n",
" keras.layers.Dense(10)\n", " tf.keras.layers.Dense(10)\n",
"])\n", "])\n",
"\n", "\n",
"model.compile(loss=\"mse\", optimizer=\"adam\")\n", "model.compile(loss=\"mse\", optimizer=\"adam\")\n",
@ -627,10 +627,10 @@
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)\n", "tf.random.set_seed(42)\n",
"\n", "\n",
"model = keras.models.Sequential([\n", "model = tf.keras.Sequential([\n",
" keras.layers.SimpleRNN(20, return_sequences=True, input_shape=[None, 1]),\n", " tf.keras.layers.SimpleRNN(20, return_sequences=True, input_shape=[None, 1]),\n",
" keras.layers.SimpleRNN(20),\n", " tf.keras.layers.SimpleRNN(20),\n",
" keras.layers.Dense(10)\n", " tf.keras.layers.Dense(10)\n",
"])\n", "])\n",
"\n", "\n",
"model.compile(loss=\"mse\", optimizer=\"adam\")\n", "model.compile(loss=\"mse\", optimizer=\"adam\")\n",
@ -707,16 +707,16 @@
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)\n", "tf.random.set_seed(42)\n",
"\n", "\n",
"model = keras.models.Sequential([\n", "model = tf.keras.Sequential([\n",
" keras.layers.SimpleRNN(20, return_sequences=True, input_shape=[None, 1]),\n", " tf.keras.layers.SimpleRNN(20, return_sequences=True, input_shape=[None, 1]),\n",
" keras.layers.SimpleRNN(20, return_sequences=True),\n", " tf.keras.layers.SimpleRNN(20, return_sequences=True),\n",
" keras.layers.TimeDistributed(keras.layers.Dense(10))\n", " tf.keras.layers.TimeDistributed(tf.keras.layers.Dense(10))\n",
"])\n", "])\n",
"\n", "\n",
"def last_time_step_mse(Y_true, Y_pred):\n", "def last_time_step_mse(Y_true, Y_pred):\n",
" return keras.metrics.mean_squared_error(Y_true[:, -1], Y_pred[:, -1])\n", " return tf.keras.metrics.mean_squared_error(Y_true[:, -1], Y_pred[:, -1])\n",
"\n", "\n",
"model.compile(loss=\"mse\", optimizer=keras.optimizers.Adam(learning_rate=0.01), metrics=[last_time_step_mse])\n", "model.compile(loss=\"mse\", optimizer=tf.keras.optimizers.Adam(learning_rate=0.01), metrics=[last_time_step_mse])\n",
"history = model.fit(X_train, Y_train, epochs=20,\n", "history = model.fit(X_train, Y_train, epochs=20,\n",
" validation_data=(X_valid, Y_valid))" " validation_data=(X_valid, Y_valid))"
] ]
@ -760,12 +760,12 @@
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)\n", "tf.random.set_seed(42)\n",
"\n", "\n",
"model = keras.models.Sequential([\n", "model = tf.keras.Sequential([\n",
" keras.layers.SimpleRNN(20, return_sequences=True, input_shape=[None, 1]),\n", " tf.keras.layers.SimpleRNN(20, return_sequences=True, input_shape=[None, 1]),\n",
" keras.layers.BatchNormalization(),\n", " tf.keras.layers.BatchNormalization(),\n",
" keras.layers.SimpleRNN(20, return_sequences=True),\n", " tf.keras.layers.SimpleRNN(20, return_sequences=True),\n",
" keras.layers.BatchNormalization(),\n", " tf.keras.layers.BatchNormalization(),\n",
" keras.layers.TimeDistributed(keras.layers.Dense(10))\n", " tf.keras.layers.TimeDistributed(tf.keras.layers.Dense(10))\n",
"])\n", "])\n",
"\n", "\n",
"model.compile(loss=\"mse\", optimizer=\"adam\", metrics=[last_time_step_mse])\n", "model.compile(loss=\"mse\", optimizer=\"adam\", metrics=[last_time_step_mse])\n",
@ -795,15 +795,15 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"class LNSimpleRNNCell(keras.layers.Layer):\n", "class LNSimpleRNNCell(tf.keras.layers.Layer):\n",
" def __init__(self, units, activation=\"tanh\", **kwargs):\n", " def __init__(self, units, activation=\"tanh\", **kwargs):\n",
" super().__init__(**kwargs)\n", " super().__init__(**kwargs)\n",
" self.state_size = units\n", " self.state_size = units\n",
" self.output_size = units\n", " self.output_size = units\n",
" self.simple_rnn_cell = keras.layers.SimpleRNNCell(units,\n", " self.simple_rnn_cell = tf.keras.layers.SimpleRNNCell(units,\n",
" activation=None)\n", " activation=None)\n",
" self.layer_norm = LayerNormalization()\n", " self.layer_norm = LayerNormalization()\n",
" self.activation = keras.activations.get(activation)\n", " self.activation = tf.keras.activations.get(activation)\n",
" def get_initial_state(self, inputs=None, batch_size=None, dtype=None):\n", " def get_initial_state(self, inputs=None, batch_size=None, dtype=None):\n",
" if inputs is not None:\n", " if inputs is not None:\n",
" batch_size = tf.shape(inputs)[0]\n", " batch_size = tf.shape(inputs)[0]\n",
@ -824,11 +824,11 @@
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)\n", "tf.random.set_seed(42)\n",
"\n", "\n",
"model = keras.models.Sequential([\n", "model = tf.keras.Sequential([\n",
" keras.layers.RNN(LNSimpleRNNCell(20), return_sequences=True,\n", " tf.keras.layers.RNN(LNSimpleRNNCell(20), return_sequences=True,\n",
" input_shape=[None, 1]),\n", " input_shape=[None, 1]),\n",
" keras.layers.RNN(LNSimpleRNNCell(20), return_sequences=True),\n", " tf.keras.layers.RNN(LNSimpleRNNCell(20), return_sequences=True),\n",
" keras.layers.TimeDistributed(keras.layers.Dense(10))\n", " tf.keras.layers.TimeDistributed(tf.keras.layers.Dense(10))\n",
"])\n", "])\n",
"\n", "\n",
"model.compile(loss=\"mse\", optimizer=\"adam\", metrics=[last_time_step_mse])\n", "model.compile(loss=\"mse\", optimizer=\"adam\", metrics=[last_time_step_mse])\n",
@ -849,7 +849,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"class MyRNN(keras.layers.Layer):\n", "class MyRNN(tf.keras.layers.Layer):\n",
" def __init__(self, cell, return_sequences=False, **kwargs):\n", " def __init__(self, cell, return_sequences=False, **kwargs):\n",
" super().__init__(**kwargs)\n", " super().__init__(**kwargs)\n",
" self.cell = cell\n", " self.cell = cell\n",
@ -887,11 +887,11 @@
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)\n", "tf.random.set_seed(42)\n",
"\n", "\n",
"model = keras.models.Sequential([\n", "model = tf.keras.Sequential([\n",
" MyRNN(LNSimpleRNNCell(20), return_sequences=True,\n", " MyRNN(LNSimpleRNNCell(20), return_sequences=True,\n",
" input_shape=[None, 1]),\n", " input_shape=[None, 1]),\n",
" MyRNN(LNSimpleRNNCell(20), return_sequences=True),\n", " MyRNN(LNSimpleRNNCell(20), return_sequences=True),\n",
" keras.layers.TimeDistributed(keras.layers.Dense(10))\n", " tf.keras.layers.TimeDistributed(tf.keras.layers.Dense(10))\n",
"])\n", "])\n",
"\n", "\n",
"model.compile(loss=\"mse\", optimizer=\"adam\", metrics=[last_time_step_mse])\n", "model.compile(loss=\"mse\", optimizer=\"adam\", metrics=[last_time_step_mse])\n",
@ -917,10 +917,10 @@
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)\n", "tf.random.set_seed(42)\n",
"\n", "\n",
"model = keras.models.Sequential([\n", "model = tf.keras.Sequential([\n",
" keras.layers.LSTM(20, return_sequences=True, input_shape=[None, 1]),\n", " tf.keras.layers.LSTM(20, return_sequences=True, input_shape=[None, 1]),\n",
" keras.layers.LSTM(20, return_sequences=True),\n", " tf.keras.layers.LSTM(20, return_sequences=True),\n",
" keras.layers.TimeDistributed(keras.layers.Dense(10))\n", " tf.keras.layers.TimeDistributed(tf.keras.layers.Dense(10))\n",
"])\n", "])\n",
"\n", "\n",
"model.compile(loss=\"mse\", optimizer=\"adam\", metrics=[last_time_step_mse])\n", "model.compile(loss=\"mse\", optimizer=\"adam\", metrics=[last_time_step_mse])\n",
@ -988,10 +988,10 @@
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)\n", "tf.random.set_seed(42)\n",
"\n", "\n",
"model = keras.models.Sequential([\n", "model = tf.keras.Sequential([\n",
" keras.layers.GRU(20, return_sequences=True, input_shape=[None, 1]),\n", " tf.keras.layers.GRU(20, return_sequences=True, input_shape=[None, 1]),\n",
" keras.layers.GRU(20, return_sequences=True),\n", " tf.keras.layers.GRU(20, return_sequences=True),\n",
" keras.layers.TimeDistributed(keras.layers.Dense(10))\n", " tf.keras.layers.TimeDistributed(tf.keras.layers.Dense(10))\n",
"])\n", "])\n",
"\n", "\n",
"model.compile(loss=\"mse\", optimizer=\"adam\", metrics=[last_time_step_mse])\n", "model.compile(loss=\"mse\", optimizer=\"adam\", metrics=[last_time_step_mse])\n",
@ -1080,12 +1080,12 @@
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)\n", "tf.random.set_seed(42)\n",
"\n", "\n",
"model = keras.models.Sequential([\n", "model = tf.keras.Sequential([\n",
" keras.layers.Conv1D(filters=20, kernel_size=4, strides=2, padding=\"valid\",\n", " tf.keras.layers.Conv1D(filters=20, kernel_size=4, strides=2, padding=\"valid\",\n",
" input_shape=[None, 1]),\n", " input_shape=[None, 1]),\n",
" keras.layers.GRU(20, return_sequences=True),\n", " tf.keras.layers.GRU(20, return_sequences=True),\n",
" keras.layers.GRU(20, return_sequences=True),\n", " tf.keras.layers.GRU(20, return_sequences=True),\n",
" keras.layers.TimeDistributed(keras.layers.Dense(10))\n", " tf.keras.layers.TimeDistributed(tf.keras.layers.Dense(10))\n",
"])\n", "])\n",
"\n", "\n",
"model.compile(loss=\"mse\", optimizer=\"adam\", metrics=[last_time_step_mse])\n", "model.compile(loss=\"mse\", optimizer=\"adam\", metrics=[last_time_step_mse])\n",
@ -1124,12 +1124,12 @@
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)\n", "tf.random.set_seed(42)\n",
"\n", "\n",
"model = keras.models.Sequential()\n", "model = tf.keras.Sequential()\n",
"model.add(keras.layers.InputLayer(input_shape=[None, 1]))\n", "model.add(tf.keras.layers.InputLayer(input_shape=[None, 1]))\n",
"for rate in (1, 2, 4, 8) * 2:\n", "for rate in (1, 2, 4, 8) * 2:\n",
" model.add(keras.layers.Conv1D(filters=20, kernel_size=2, padding=\"causal\",\n", " model.add(tf.keras.layers.Conv1D(filters=20, kernel_size=2, padding=\"causal\",\n",
" activation=\"relu\", dilation_rate=rate))\n", " activation=\"relu\", dilation_rate=rate))\n",
"model.add(keras.layers.Conv1D(filters=10, kernel_size=1))\n", "model.add(tf.keras.layers.Conv1D(filters=10, kernel_size=1))\n",
"model.compile(loss=\"mse\", optimizer=\"adam\", metrics=[last_time_step_mse])\n", "model.compile(loss=\"mse\", optimizer=\"adam\", metrics=[last_time_step_mse])\n",
"history = model.fit(X_train, Y_train, epochs=20,\n", "history = model.fit(X_train, Y_train, epochs=20,\n",
" validation_data=(X_valid, Y_valid))" " validation_data=(X_valid, Y_valid))"
@ -1148,14 +1148,14 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"class GatedActivationUnit(keras.layers.Layer):\n", "class GatedActivationUnit(tf.keras.layers.Layer):\n",
" def __init__(self, activation=\"tanh\", **kwargs):\n", " def __init__(self, activation=\"tanh\", **kwargs):\n",
" super().__init__(**kwargs)\n", " super().__init__(**kwargs)\n",
" self.activation = keras.activations.get(activation)\n", " self.activation = tf.keras.activations.get(activation)\n",
" def call(self, inputs):\n", " def call(self, inputs):\n",
" n_filters = inputs.shape[-1] // 2\n", " n_filters = inputs.shape[-1] // 2\n",
" linear_output = self.activation(inputs[..., :n_filters])\n", " linear_output = self.activation(inputs[..., :n_filters])\n",
" gate = keras.activations.sigmoid(inputs[..., n_filters:])\n", " gate = tf.keras.activations.sigmoid(inputs[..., n_filters:])\n",
" return self.activation(linear_output) * gate" " return self.activation(linear_output) * gate"
] ]
}, },
@ -1166,11 +1166,11 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"def wavenet_residual_block(inputs, n_filters, dilation_rate):\n", "def wavenet_residual_block(inputs, n_filters, dilation_rate):\n",
" z = keras.layers.Conv1D(2 * n_filters, kernel_size=2, padding=\"causal\",\n", " z = tf.keras.layers.Conv1D(2 * n_filters, kernel_size=2, padding=\"causal\",\n",
" dilation_rate=dilation_rate)(inputs)\n", " dilation_rate=dilation_rate)(inputs)\n",
" z = GatedActivationUnit()(z)\n", " z = GatedActivationUnit()(z)\n",
" z = keras.layers.Conv1D(n_filters, kernel_size=1)(z)\n", " z = tf.keras.layers.Conv1D(n_filters, kernel_size=1)(z)\n",
" return keras.layers.Add()([z, inputs]), z" " return tf.keras.layers.Add()([z, inputs]), z"
] ]
}, },
{ {
@ -1179,7 +1179,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"keras.backend.clear_session()\n", "tf.keras.backend.clear_session()\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)\n", "tf.random.set_seed(42)\n",
"\n", "\n",
@ -1188,17 +1188,17 @@
"n_filters = 32 # 128 in the paper\n", "n_filters = 32 # 128 in the paper\n",
"n_outputs = 10 # 256 in the paper\n", "n_outputs = 10 # 256 in the paper\n",
"\n", "\n",
"inputs = keras.layers.Input(shape=[None, 1])\n", "inputs = tf.keras.layers.Input(shape=[None, 1])\n",
"z = keras.layers.Conv1D(n_filters, kernel_size=2, padding=\"causal\")(inputs)\n", "z = tf.keras.layers.Conv1D(n_filters, kernel_size=2, padding=\"causal\")(inputs)\n",
"skip_to_last = []\n", "skip_to_last = []\n",
"for dilation_rate in [2**i for i in range(n_layers_per_block)] * n_blocks:\n", "for dilation_rate in [2**i for i in range(n_layers_per_block)] * n_blocks:\n",
" z, skip = wavenet_residual_block(z, n_filters, dilation_rate)\n", " z, skip = wavenet_residual_block(z, n_filters, dilation_rate)\n",
" skip_to_last.append(skip)\n", " skip_to_last.append(skip)\n",
"z = keras.activations.relu(keras.layers.Add()(skip_to_last))\n", "z = tf.keras.activations.relu(tf.keras.layers.Add()(skip_to_last))\n",
"z = keras.layers.Conv1D(n_filters, kernel_size=1, activation=\"relu\")(z)\n", "z = tf.keras.layers.Conv1D(n_filters, kernel_size=1, activation=\"relu\")(z)\n",
"Y_proba = keras.layers.Conv1D(n_outputs, kernel_size=1, activation=\"softmax\")(z)\n", "Y_proba = tf.keras.layers.Conv1D(n_outputs, kernel_size=1, activation=\"softmax\")(z)\n",
"\n", "\n",
"model = keras.models.Model(inputs=[inputs], outputs=[Y_proba])" "model = tf.keras.Model(inputs=[inputs], outputs=[Y_proba])"
] ]
}, },
{ {
@ -1269,7 +1269,7 @@
"source": [ "source": [
"DOWNLOAD_ROOT = \"http://download.tensorflow.org/data/\"\n", "DOWNLOAD_ROOT = \"http://download.tensorflow.org/data/\"\n",
"FILENAME = \"quickdraw_tutorial_dataset_v1.tar.gz\"\n", "FILENAME = \"quickdraw_tutorial_dataset_v1.tar.gz\"\n",
"filepath = keras.utils.get_file(FILENAME,\n", "filepath = tf.keras.utils.get_file(FILENAME,\n",
" DOWNLOAD_ROOT + FILENAME,\n", " DOWNLOAD_ROOT + FILENAME,\n",
" cache_subdir=\"datasets/quickdraw\",\n", " cache_subdir=\"datasets/quickdraw\",\n",
" extract=True)" " extract=True)"
@ -1472,18 +1472,18 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"model = keras.models.Sequential([\n", "model = tf.keras.Sequential([\n",
" keras.layers.Conv1D(32, kernel_size=5, strides=2, activation=\"relu\"),\n", " tf.keras.layers.Conv1D(32, kernel_size=5, strides=2, activation=\"relu\"),\n",
" keras.layers.BatchNormalization(),\n", " tf.keras.layers.BatchNormalization(),\n",
" keras.layers.Conv1D(64, kernel_size=5, strides=2, activation=\"relu\"),\n", " tf.keras.layers.Conv1D(64, kernel_size=5, strides=2, activation=\"relu\"),\n",
" keras.layers.BatchNormalization(),\n", " tf.keras.layers.BatchNormalization(),\n",
" keras.layers.Conv1D(128, kernel_size=3, strides=2, activation=\"relu\"),\n", " tf.keras.layers.Conv1D(128, kernel_size=3, strides=2, activation=\"relu\"),\n",
" keras.layers.BatchNormalization(),\n", " tf.keras.layers.BatchNormalization(),\n",
" keras.layers.LSTM(128, return_sequences=True),\n", " tf.keras.layers.LSTM(128, return_sequences=True),\n",
" keras.layers.LSTM(128),\n", " tf.keras.layers.LSTM(128),\n",
" keras.layers.Dense(len(class_names), activation=\"softmax\")\n", " tf.keras.layers.Dense(len(class_names), activation=\"softmax\")\n",
"])\n", "])\n",
"optimizer = keras.optimizers.SGD(learning_rate=1e-2, clipnorm=1.)\n", "optimizer = tf.keras.optimizers.SGD(learning_rate=1e-2, clipnorm=1.)\n",
"model.compile(loss=\"sparse_categorical_crossentropy\",\n", "model.compile(loss=\"sparse_categorical_crossentropy\",\n",
" optimizer=optimizer,\n", " optimizer=optimizer,\n",
" metrics=[\"accuracy\", \"sparse_top_k_categorical_accuracy\"])\n", " metrics=[\"accuracy\", \"sparse_top_k_categorical_accuracy\"])\n",
@ -1507,7 +1507,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"np.mean(keras.metrics.sparse_top_k_categorical_accuracy(y_test, y_probas))" "np.mean(tf.keras.metrics.sparse_top_k_categorical_accuracy(y_test, y_probas))"
] ]
}, },
{ {
@ -1557,7 +1557,7 @@
"source": [ "source": [
"DOWNLOAD_ROOT = \"https://github.com/ageron/handson-ml2/raw/master/datasets/jsb_chorales/\"\n", "DOWNLOAD_ROOT = \"https://github.com/ageron/handson-ml2/raw/master/datasets/jsb_chorales/\"\n",
"FILENAME = \"jsb_chorales.tgz\"\n", "FILENAME = \"jsb_chorales.tgz\"\n",
"filepath = keras.utils.get_file(FILENAME,\n", "filepath = tf.keras.utils.get_file(FILENAME,\n",
" DOWNLOAD_ROOT + FILENAME,\n", " DOWNLOAD_ROOT + FILENAME,\n",
" cache_subdir=\"datasets/jsb_chorales\",\n", " cache_subdir=\"datasets/jsb_chorales\",\n",
" extract=True)" " extract=True)"
@ -1792,19 +1792,19 @@
"source": [ "source": [
"n_embedding_dims = 5\n", "n_embedding_dims = 5\n",
"\n", "\n",
"model = keras.models.Sequential([\n", "model = tf.keras.Sequential([\n",
" keras.layers.Embedding(input_dim=n_notes, output_dim=n_embedding_dims,\n", " tf.keras.layers.Embedding(input_dim=n_notes, output_dim=n_embedding_dims,\n",
" input_shape=[None]),\n", " input_shape=[None]),\n",
" keras.layers.Conv1D(32, kernel_size=2, padding=\"causal\", activation=\"relu\"),\n", " tf.keras.layers.Conv1D(32, kernel_size=2, padding=\"causal\", activation=\"relu\"),\n",
" keras.layers.BatchNormalization(),\n", " tf.keras.layers.BatchNormalization(),\n",
" keras.layers.Conv1D(48, kernel_size=2, padding=\"causal\", activation=\"relu\", dilation_rate=2),\n", " tf.keras.layers.Conv1D(48, kernel_size=2, padding=\"causal\", activation=\"relu\", dilation_rate=2),\n",
" keras.layers.BatchNormalization(),\n", " tf.keras.layers.BatchNormalization(),\n",
" keras.layers.Conv1D(64, kernel_size=2, padding=\"causal\", activation=\"relu\", dilation_rate=4),\n", " tf.keras.layers.Conv1D(64, kernel_size=2, padding=\"causal\", activation=\"relu\", dilation_rate=4),\n",
" keras.layers.BatchNormalization(),\n", " tf.keras.layers.BatchNormalization(),\n",
" keras.layers.Conv1D(96, kernel_size=2, padding=\"causal\", activation=\"relu\", dilation_rate=8),\n", " tf.keras.layers.Conv1D(96, kernel_size=2, padding=\"causal\", activation=\"relu\", dilation_rate=8),\n",
" keras.layers.BatchNormalization(),\n", " tf.keras.layers.BatchNormalization(),\n",
" keras.layers.LSTM(256, return_sequences=True),\n", " tf.keras.layers.LSTM(256, return_sequences=True),\n",
" keras.layers.Dense(n_notes, activation=\"softmax\")\n", " tf.keras.layers.Dense(n_notes, activation=\"softmax\")\n",
"])\n", "])\n",
"\n", "\n",
"model.summary()" "model.summary()"
@ -1823,7 +1823,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"optimizer = keras.optimizers.Nadam(learning_rate=1e-3)\n", "optimizer = tf.keras.optimizers.Nadam(learning_rate=1e-3)\n",
"model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=optimizer,\n", "model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=optimizer,\n",
" metrics=[\"accuracy\"])\n", " metrics=[\"accuracy\"])\n",
"model.fit(train_set, epochs=20, validation_data=valid_set)" "model.fit(train_set, epochs=20, validation_data=valid_set)"

234
15_nlp_with_rnns_and_attention.ipynb
View File

@ -161,7 +161,7 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"shakespeare_url = \"https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt\"\n", "shakespeare_url = \"https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt\"\n",
"filepath = keras.utils.get_file(\"shakespeare.txt\", shakespeare_url)\n", "filepath = tf.keras.utils.get_file(\"shakespeare.txt\", shakespeare_url)\n",
"with open(filepath) as f:\n", "with open(filepath) as f:\n",
" shakespeare_text = f.read()" " shakespeare_text = f.read()"
] ]
@ -190,7 +190,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"tokenizer = keras.preprocessing.text.Tokenizer(char_level=True)\n", "tokenizer = tf.keras.preprocessing.text.Tokenizer(char_level=True)\n",
"tokenizer.fit_on_texts(shakespeare_text)" "tokenizer.fit_on_texts(shakespeare_text)"
] ]
}, },
@ -337,14 +337,14 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"model = keras.models.Sequential([\n", "model = tf.keras.Sequential([\n",
" keras.layers.GRU(128, return_sequences=True, input_shape=[None, max_id],\n", " tf.keras.layers.GRU(128, return_sequences=True, input_shape=[None, max_id],\n",
" #dropout=0.2, recurrent_dropout=0.2),\n", " #dropout=0.2, recurrent_dropout=0.2),\n",
" dropout=0.2),\n", " dropout=0.2),\n",
" keras.layers.GRU(128, return_sequences=True,\n", " tf.keras.layers.GRU(128, return_sequences=True,\n",
" #dropout=0.2, recurrent_dropout=0.2),\n", " #dropout=0.2, recurrent_dropout=0.2),\n",
" dropout=0.2),\n", " dropout=0.2),\n",
" keras.layers.TimeDistributed(keras.layers.Dense(max_id,\n", " tf.keras.layers.TimeDistributed(tf.keras.layers.Dense(max_id,\n",
" activation=\"softmax\"))\n", " activation=\"softmax\"))\n",
"])\n", "])\n",
"model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=\"adam\")\n", "model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=\"adam\")\n",
@ -531,15 +531,15 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"model = keras.models.Sequential([\n", "model = tf.keras.Sequential([\n",
" keras.layers.GRU(128, return_sequences=True, stateful=True,\n", " tf.keras.layers.GRU(128, return_sequences=True, stateful=True,\n",
" #dropout=0.2, recurrent_dropout=0.2,\n", " #dropout=0.2, recurrent_dropout=0.2,\n",
" dropout=0.2,\n", " dropout=0.2,\n",
" batch_input_shape=[batch_size, None, max_id]),\n", " batch_input_shape=[batch_size, None, max_id]),\n",
" keras.layers.GRU(128, return_sequences=True, stateful=True,\n", " tf.keras.layers.GRU(128, return_sequences=True, stateful=True,\n",
" #dropout=0.2, recurrent_dropout=0.2),\n", " #dropout=0.2, recurrent_dropout=0.2),\n",
" dropout=0.2),\n", " dropout=0.2),\n",
" keras.layers.TimeDistributed(keras.layers.Dense(max_id,\n", " tf.keras.layers.TimeDistributed(tf.keras.layers.Dense(max_id,\n",
" activation=\"softmax\"))\n", " activation=\"softmax\"))\n",
"])" "])"
] ]
@ -550,7 +550,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"class ResetStatesCallback(keras.callbacks.Callback):\n", "class ResetStatesCallback(tf.keras.callbacks.Callback):\n",
" def on_epoch_begin(self, epoch, logs):\n", " def on_epoch_begin(self, epoch, logs):\n",
" self.model.reset_states()" " self.model.reset_states()"
] ]
@ -579,10 +579,10 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"stateless_model = keras.models.Sequential([\n", "stateless_model = tf.keras.Sequential([\n",
" keras.layers.GRU(128, return_sequences=True, input_shape=[None, max_id]),\n", " tf.keras.layers.GRU(128, return_sequences=True, input_shape=[None, max_id]),\n",
" keras.layers.GRU(128, return_sequences=True),\n", " tf.keras.layers.GRU(128, return_sequences=True),\n",
" keras.layers.TimeDistributed(keras.layers.Dense(max_id,\n", " tf.keras.layers.TimeDistributed(tf.keras.layers.Dense(max_id,\n",
" activation=\"softmax\"))\n", " activation=\"softmax\"))\n",
"])" "])"
] ]
@ -653,7 +653,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"(X_train, y_train), (X_test, y_test) = keras.datasets.imdb.load_data()" "(X_train, y_train), (X_test, y_test) = tf.keras.datasets.imdb.load_data()"
] ]
}, },
{ {
@ -671,7 +671,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"word_index = keras.datasets.imdb.get_word_index()\n", "word_index = tf.keras.datasets.imdb.get_word_index()\n",
"id_to_word = {id_ + 3: word for word, id_ in word_index.items()}\n", "id_to_word = {id_ + 3: word for word, id_ in word_index.items()}\n",
"for id_, token in enumerate((\"<pad>\", \"<sos>\", \"<unk>\")):\n", "for id_, token in enumerate((\"<pad>\", \"<sos>\", \"<unk>\")):\n",
" id_to_word[id_] = token\n", " id_to_word[id_] = token\n",
@ -860,13 +860,13 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"embed_size = 128\n", "embed_size = 128\n",
"model = keras.models.Sequential([\n", "model = tf.keras.Sequential([\n",
" keras.layers.Embedding(vocab_size + num_oov_buckets, embed_size,\n", " tf.keras.layers.Embedding(vocab_size + num_oov_buckets, embed_size,\n",
" mask_zero=True, # not shown in the book\n", " mask_zero=True, # not shown in the book\n",
" input_shape=[None]),\n", " input_shape=[None]),\n",
" keras.layers.GRU(128, return_sequences=True),\n", " tf.keras.layers.GRU(128, return_sequences=True),\n",
" keras.layers.GRU(128),\n", " tf.keras.layers.GRU(128),\n",
" keras.layers.Dense(1, activation=\"sigmoid\")\n", " tf.keras.layers.Dense(1, activation=\"sigmoid\")\n",
"])\n", "])\n",
"model.compile(loss=\"binary_crossentropy\", optimizer=\"adam\", metrics=[\"accuracy\"])\n", "model.compile(loss=\"binary_crossentropy\", optimizer=\"adam\", metrics=[\"accuracy\"])\n",
"history = model.fit(train_set, epochs=5)" "history = model.fit(train_set, epochs=5)"
@ -885,15 +885,15 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"K = keras.backend\n", "K = tf.keras.backend\n",
"embed_size = 128\n", "embed_size = 128\n",
"inputs = keras.layers.Input(shape=[None])\n", "inputs = tf.keras.layers.Input(shape=[None])\n",
"mask = keras.layers.Lambda(lambda inputs: K.not_equal(inputs, 0))(inputs)\n", "mask = tf.keras.layers.Lambda(lambda inputs: K.not_equal(inputs, 0))(inputs)\n",
"z = keras.layers.Embedding(vocab_size + num_oov_buckets, embed_size)(inputs)\n", "z = tf.keras.layers.Embedding(vocab_size + num_oov_buckets, embed_size)(inputs)\n",
"z = keras.layers.GRU(128, return_sequences=True)(z, mask=mask)\n", "z = tf.keras.layers.GRU(128, return_sequences=True)(z, mask=mask)\n",
"z = keras.layers.GRU(128)(z, mask=mask)\n", "z = tf.keras.layers.GRU(128)(z, mask=mask)\n",
"outputs = keras.layers.Dense(1, activation=\"sigmoid\")(z)\n", "outputs = tf.keras.layers.Dense(1, activation=\"sigmoid\")(z)\n",
"model = keras.models.Model(inputs=[inputs], outputs=[outputs])\n", "model = tf.keras.Model(inputs=[inputs], outputs=[outputs])\n",
"model.compile(loss=\"binary_crossentropy\", optimizer=\"adam\", metrics=[\"accuracy\"])\n", "model.compile(loss=\"binary_crossentropy\", optimizer=\"adam\", metrics=[\"accuracy\"])\n",
"history = model.fit(train_set, epochs=5)" "history = model.fit(train_set, epochs=5)"
] ]
@ -932,11 +932,11 @@
"source": [ "source": [
"import tensorflow_hub as hub\n", "import tensorflow_hub as hub\n",
"\n", "\n",
"model = keras.Sequential([\n", "model = tf.keras.Sequential([\n",
" hub.KerasLayer(\"https://tfhub.dev/google/tf2-preview/nnlm-en-dim50/1\",\n", " hub.KerasLayer(\"https://tfhub.dev/google/tf2-preview/nnlm-en-dim50/1\",\n",
" dtype=tf.string, input_shape=[], output_shape=[50]),\n", " dtype=tf.string, input_shape=[], output_shape=[50]),\n",
" keras.layers.Dense(128, activation=\"relu\"),\n", " tf.keras.layers.Dense(128, activation=\"relu\"),\n",
" keras.layers.Dense(1, activation=\"sigmoid\")\n", " tf.keras.layers.Dense(1, activation=\"sigmoid\")\n",
"])\n", "])\n",
"model.compile(loss=\"binary_crossentropy\", optimizer=\"adam\",\n", "model.compile(loss=\"binary_crossentropy\", optimizer=\"adam\",\n",
" metrics=[\"accuracy\"])" " metrics=[\"accuracy\"])"
@ -1028,22 +1028,22 @@
"source": [ "source": [
"import tensorflow_addons as tfa\n", "import tensorflow_addons as tfa\n",
"\n", "\n",
"encoder_inputs = keras.layers.Input(shape=[None], dtype=np.int32)\n", "encoder_inputs = tf.keras.layers.Input(shape=[None], dtype=np.int32)\n",
"decoder_inputs = keras.layers.Input(shape=[None], dtype=np.int32)\n", "decoder_inputs = tf.keras.layers.Input(shape=[None], dtype=np.int32)\n",
"sequence_lengths = keras.layers.Input(shape=[], dtype=np.int32)\n", "sequence_lengths = tf.keras.layers.Input(shape=[], dtype=np.int32)\n",
"\n", "\n",
"embeddings = keras.layers.Embedding(vocab_size, embed_size)\n", "embeddings = tf.keras.layers.Embedding(vocab_size, embed_size)\n",
"encoder_embeddings = embeddings(encoder_inputs)\n", "encoder_embeddings = embeddings(encoder_inputs)\n",
"decoder_embeddings = embeddings(decoder_inputs)\n", "decoder_embeddings = embeddings(decoder_inputs)\n",
"\n", "\n",
"encoder = keras.layers.LSTM(512, return_state=True)\n", "encoder = tf.keras.layers.LSTM(512, return_state=True)\n",
"encoder_outputs, state_h, state_c = encoder(encoder_embeddings)\n", "encoder_outputs, state_h, state_c = encoder(encoder_embeddings)\n",
"encoder_state = [state_h, state_c]\n", "encoder_state = [state_h, state_c]\n",
"\n", "\n",
"sampler = tfa.seq2seq.sampler.TrainingSampler()\n", "sampler = tfa.seq2seq.sampler.TrainingSampler()\n",
"\n", "\n",
"decoder_cell = keras.layers.LSTMCell(512)\n", "decoder_cell = tf.keras.layers.LSTMCell(512)\n",
"output_layer = keras.layers.Dense(vocab_size)\n", "output_layer = tf.keras.layers.Dense(vocab_size)\n",
"decoder = tfa.seq2seq.basic_decoder.BasicDecoder(decoder_cell, sampler,\n", "decoder = tfa.seq2seq.basic_decoder.BasicDecoder(decoder_cell, sampler,\n",
" output_layer=output_layer)\n", " output_layer=output_layer)\n",
"final_outputs, final_state, final_sequence_lengths = decoder(\n", "final_outputs, final_state, final_sequence_lengths = decoder(\n",
@ -1051,7 +1051,7 @@
" sequence_length=sequence_lengths)\n", " sequence_length=sequence_lengths)\n",
"Y_proba = tf.nn.softmax(final_outputs.rnn_output)\n", "Y_proba = tf.nn.softmax(final_outputs.rnn_output)\n",
"\n", "\n",
"model = keras.models.Model(\n", "model = tf.keras.Model(\n",
" inputs=[encoder_inputs, decoder_inputs, sequence_lengths],\n", " inputs=[encoder_inputs, decoder_inputs, sequence_lengths],\n",
" outputs=[Y_proba])" " outputs=[Y_proba])"
] ]
@ -1092,9 +1092,9 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"model = keras.models.Sequential([\n", "model = tf.keras.Sequential([\n",
" keras.layers.GRU(10, return_sequences=True, input_shape=[None, 10]),\n", " tf.keras.layers.GRU(10, return_sequences=True, input_shape=[None, 10]),\n",
" keras.layers.Bidirectional(keras.layers.GRU(10, return_sequences=True))\n", " tf.keras.layers.Bidirectional(tf.keras.layers.GRU(10, return_sequences=True))\n",
"])\n", "])\n",
"\n", "\n",
"model.summary()" "model.summary()"
@ -1113,7 +1113,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"class PositionalEncoding(keras.layers.Layer):\n", "class PositionalEncoding(tf.keras.layers.Layer):\n",
" def __init__(self, max_steps, max_dims, dtype=tf.float32, **kwargs):\n", " def __init__(self, max_steps, max_dims, dtype=tf.float32, **kwargs):\n",
" super().__init__(dtype=dtype, **kwargs)\n", " super().__init__(dtype=dtype, **kwargs)\n",
" if max_dims % 2 == 1: max_dims += 1 # max_dims must be even\n", " if max_dims % 2 == 1: max_dims += 1 # max_dims must be even\n",
@ -1182,9 +1182,9 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"embed_size = 512; max_steps = 500; vocab_size = 10000\n", "embed_size = 512; max_steps = 500; vocab_size = 10000\n",
"encoder_inputs = keras.layers.Input(shape=[None], dtype=np.int32)\n", "encoder_inputs = tf.keras.layers.Input(shape=[None], dtype=np.int32)\n",
"decoder_inputs = keras.layers.Input(shape=[None], dtype=np.int32)\n", "decoder_inputs = tf.keras.layers.Input(shape=[None], dtype=np.int32)\n",
"embeddings = keras.layers.Embedding(vocab_size, embed_size)\n", "embeddings = tf.keras.layers.Embedding(vocab_size, embed_size)\n",
"encoder_embeddings = embeddings(encoder_inputs)\n", "encoder_embeddings = embeddings(encoder_inputs)\n",
"decoder_embeddings = embeddings(decoder_inputs)\n", "decoder_embeddings = embeddings(decoder_inputs)\n",
"positional_encoding = PositionalEncoding(max_steps, max_dims=embed_size)\n", "positional_encoding = PositionalEncoding(max_steps, max_dims=embed_size)\n",
@ -1207,23 +1207,23 @@
"source": [ "source": [
"Z = encoder_in\n", "Z = encoder_in\n",
"for N in range(6):\n", "for N in range(6):\n",
" Z = keras.layers.Attention(use_scale=True)([Z, Z])\n", " Z = tf.keras.layers.Attention(use_scale=True)([Z, Z])\n",
"\n", "\n",
"encoder_outputs = Z\n", "encoder_outputs = Z\n",
"Z = decoder_in\n", "Z = decoder_in\n",
"for N in range(6):\n", "for N in range(6):\n",
" Z = keras.layers.Attention(use_scale=True, causal=True)([Z, Z])\n", " Z = tf.keras.layers.Attention(use_scale=True, causal=True)([Z, Z])\n",
" Z = keras.layers.Attention(use_scale=True)([Z, encoder_outputs])\n", " Z = tf.keras.layers.Attention(use_scale=True)([Z, encoder_outputs])\n",
"\n", "\n",
"outputs = keras.layers.TimeDistributed(\n", "outputs = tf.keras.layers.TimeDistributed(\n",
" keras.layers.Dense(vocab_size, activation=\"softmax\"))(Z)" " tf.keras.layers.Dense(vocab_size, activation=\"softmax\"))(Z)"
] ]
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",
"metadata": {}, "metadata": {},
"source": [ "source": [
"Here's a basic implementation of the `MultiHeadAttention` layer. One will likely be added to `keras.layers` in the near future. Note that `Conv1D` layers with `kernel_size=1` (and the default `padding=\"valid\"` and `strides=1`) is equivalent to a `TimeDistributed(Dense(...))` layer." "Here's a basic implementation of the `MultiHeadAttention` layer. One will likely be added to `tf.keras.layers` in the near future. Note that `Conv1D` layers with `kernel_size=1` (and the default `padding=\"valid\"` and `strides=1`) is equivalent to a `TimeDistributed(Dense(...))` layer."
] ]
}, },
{ {
@ -1232,9 +1232,9 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"K = keras.backend\n", "K = tf.keras.backend\n",
"\n", "\n",
"class MultiHeadAttention(keras.layers.Layer):\n", "class MultiHeadAttention(tf.keras.layers.Layer):\n",
" def __init__(self, n_heads, causal=False, use_scale=False, **kwargs):\n", " def __init__(self, n_heads, causal=False, use_scale=False, **kwargs):\n",
" self.n_heads = n_heads\n", " self.n_heads = n_heads\n",
" self.causal = causal\n", " self.causal = causal\n",
@ -1243,11 +1243,11 @@
" def build(self, batch_input_shape):\n", " def build(self, batch_input_shape):\n",
" self.dims = batch_input_shape[0][-1]\n", " self.dims = batch_input_shape[0][-1]\n",
" self.q_dims, self.v_dims, self.k_dims = [self.dims // self.n_heads] * 3 # could be hyperparameters instead\n", " self.q_dims, self.v_dims, self.k_dims = [self.dims // self.n_heads] * 3 # could be hyperparameters instead\n",
" self.q_linear = keras.layers.Conv1D(self.n_heads * self.q_dims, kernel_size=1, use_bias=False)\n", " self.q_linear = tf.keras.layers.Conv1D(self.n_heads * self.q_dims, kernel_size=1, use_bias=False)\n",
" self.v_linear = keras.layers.Conv1D(self.n_heads * self.v_dims, kernel_size=1, use_bias=False)\n", " self.v_linear = tf.keras.layers.Conv1D(self.n_heads * self.v_dims, kernel_size=1, use_bias=False)\n",
" self.k_linear = keras.layers.Conv1D(self.n_heads * self.k_dims, kernel_size=1, use_bias=False)\n", " self.k_linear = tf.keras.layers.Conv1D(self.n_heads * self.k_dims, kernel_size=1, use_bias=False)\n",
" self.attention = keras.layers.Attention(causal=self.causal, use_scale=self.use_scale)\n", " self.attention = tf.keras.layers.Attention(causal=self.causal, use_scale=self.use_scale)\n",
" self.out_linear = keras.layers.Conv1D(self.dims, kernel_size=1, use_bias=False)\n", " self.out_linear = tf.keras.layers.Conv1D(self.dims, kernel_size=1, use_bias=False)\n",
" super().build(batch_input_shape)\n", " super().build(batch_input_shape)\n",
" def _multi_head_linear(self, inputs, linear):\n", " def _multi_head_linear(self, inputs, linear):\n",
" shape = K.concatenate([K.shape(inputs)[:-1], [self.n_heads, -1]])\n", " shape = K.concatenate([K.shape(inputs)[:-1], [self.n_heads, -1]])\n",
@ -1547,13 +1547,13 @@
"\n", "\n",
"embedding_size = 5\n", "embedding_size = 5\n",
"\n", "\n",
"model = keras.models.Sequential([\n", "model = tf.keras.Sequential([\n",
" keras.layers.InputLayer(input_shape=[None], dtype=tf.int32, ragged=True),\n", " tf.keras.layers.InputLayer(input_shape=[None], dtype=tf.int32, ragged=True),\n",
" keras.layers.Embedding(input_dim=len(POSSIBLE_CHARS), output_dim=embedding_size),\n", " tf.keras.layers.Embedding(input_dim=len(POSSIBLE_CHARS), output_dim=embedding_size),\n",
" keras.layers.GRU(30),\n", " tf.keras.layers.GRU(30),\n",
" keras.layers.Dense(1, activation=\"sigmoid\")\n", " tf.keras.layers.Dense(1, activation=\"sigmoid\")\n",
"])\n", "])\n",
"optimizer = keras.optimizers.SGD(learning_rate=0.02, momentum = 0.95, nesterov=True)\n", "optimizer = tf.keras.optimizers.SGD(learning_rate=0.02, momentum = 0.95, nesterov=True)\n",
"model.compile(loss=\"binary_crossentropy\", optimizer=optimizer, metrics=[\"accuracy\"])\n", "model.compile(loss=\"binary_crossentropy\", optimizer=optimizer, metrics=[\"accuracy\"])\n",
"history = model.fit(X_train, y_train, epochs=20, validation_data=(X_valid, y_valid))" "history = model.fit(X_train, y_train, epochs=20, validation_data=(X_valid, y_valid))"
] ]
@ -1785,25 +1785,25 @@
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)\n", "tf.random.set_seed(42)\n",
"\n", "\n",
"encoder = keras.models.Sequential([\n", "encoder = tf.keras.Sequential([\n",
" keras.layers.Embedding(input_dim=len(INPUT_CHARS) + 1,\n", " tf.keras.layers.Embedding(input_dim=len(INPUT_CHARS) + 1,\n",
" output_dim=embedding_size,\n", " output_dim=embedding_size,\n",
" input_shape=[None]),\n", " input_shape=[None]),\n",
" keras.layers.LSTM(128)\n", " tf.keras.layers.LSTM(128)\n",
"])\n", "])\n",
"\n", "\n",
"decoder = keras.models.Sequential([\n", "decoder = tf.keras.Sequential([\n",
" keras.layers.LSTM(128, return_sequences=True),\n", " tf.keras.layers.LSTM(128, return_sequences=True),\n",
" keras.layers.Dense(len(OUTPUT_CHARS) + 1, activation=\"softmax\")\n", " tf.keras.layers.Dense(len(OUTPUT_CHARS) + 1, activation=\"softmax\")\n",
"])\n", "])\n",
"\n", "\n",
"model = keras.models.Sequential([\n", "model = tf.keras.Sequential([\n",
" encoder,\n", " encoder,\n",
" keras.layers.RepeatVector(max_output_length),\n", " tf.keras.layers.RepeatVector(max_output_length),\n",
" decoder\n", " decoder\n",
"])\n", "])\n",
"\n", "\n",
"optimizer = keras.optimizers.Nadam()\n", "optimizer = tf.keras.optimizers.Nadam()\n",
"model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=optimizer,\n", "model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=optimizer,\n",
" metrics=[\"accuracy\"])\n", " metrics=[\"accuracy\"])\n",
"history = model.fit(X_train, Y_train, epochs=20,\n", "history = model.fit(X_train, Y_train, epochs=20,\n",
@ -2017,27 +2017,27 @@
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)\n", "tf.random.set_seed(42)\n",
"\n", "\n",
"encoder_input = keras.layers.Input(shape=[None], dtype=tf.int32)\n", "encoder_input = tf.keras.layers.Input(shape=[None], dtype=tf.int32)\n",
"encoder_embedding = keras.layers.Embedding(\n", "encoder_embedding = tf.keras.layers.Embedding(\n",
" input_dim=len(INPUT_CHARS) + 1,\n", " input_dim=len(INPUT_CHARS) + 1,\n",
" output_dim=encoder_embedding_size)(encoder_input)\n", " output_dim=encoder_embedding_size)(encoder_input)\n",
"_, encoder_state_h, encoder_state_c = keras.layers.LSTM(\n", "_, encoder_state_h, encoder_state_c = tf.keras.layers.LSTM(\n",
" lstm_units, return_state=True)(encoder_embedding)\n", " lstm_units, return_state=True)(encoder_embedding)\n",
"encoder_state = [encoder_state_h, encoder_state_c]\n", "encoder_state = [encoder_state_h, encoder_state_c]\n",
"\n", "\n",
"decoder_input = keras.layers.Input(shape=[None], dtype=tf.int32)\n", "decoder_input = tf.keras.layers.Input(shape=[None], dtype=tf.int32)\n",
"decoder_embedding = keras.layers.Embedding(\n", "decoder_embedding = tf.keras.layers.Embedding(\n",
" input_dim=len(OUTPUT_CHARS) + 2,\n", " input_dim=len(OUTPUT_CHARS) + 2,\n",
" output_dim=decoder_embedding_size)(decoder_input)\n", " output_dim=decoder_embedding_size)(decoder_input)\n",
"decoder_lstm_output = keras.layers.LSTM(lstm_units, return_sequences=True)(\n", "decoder_lstm_output = tf.keras.layers.LSTM(lstm_units, return_sequences=True)(\n",
" decoder_embedding, initial_state=encoder_state)\n", " decoder_embedding, initial_state=encoder_state)\n",
"decoder_output = keras.layers.Dense(len(OUTPUT_CHARS) + 1,\n", "decoder_output = tf.keras.layers.Dense(len(OUTPUT_CHARS) + 1,\n",
" activation=\"softmax\")(decoder_lstm_output)\n", " activation=\"softmax\")(decoder_lstm_output)\n",
"\n", "\n",
"model = keras.models.Model(inputs=[encoder_input, decoder_input],\n", "model = tf.keras.Model(inputs=[encoder_input, decoder_input],\n",
" outputs=[decoder_output])\n", " outputs=[decoder_output])\n",
"\n", "\n",
"optimizer = keras.optimizers.Nadam()\n", "optimizer = tf.keras.optimizers.Nadam()\n",
"model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=optimizer,\n", "model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=optimizer,\n",
" metrics=[\"accuracy\"])\n", " metrics=[\"accuracy\"])\n",
"history = model.fit([X_train, X_train_decoder], Y_train, epochs=10,\n", "history = model.fit([X_train, X_train_decoder], Y_train, epochs=10,\n",
@ -2123,25 +2123,25 @@
"decoder_embedding_size = 32\n", "decoder_embedding_size = 32\n",
"units = 128\n", "units = 128\n",
"\n", "\n",
"encoder_inputs = keras.layers.Input(shape=[None], dtype=np.int32)\n", "encoder_inputs = tf.keras.layers.Input(shape=[None], dtype=np.int32)\n",
"decoder_inputs = keras.layers.Input(shape=[None], dtype=np.int32)\n", "decoder_inputs = tf.keras.layers.Input(shape=[None], dtype=np.int32)\n",
"sequence_lengths = keras.layers.Input(shape=[], dtype=np.int32)\n", "sequence_lengths = tf.keras.layers.Input(shape=[], dtype=np.int32)\n",
"\n", "\n",
"encoder_embeddings = keras.layers.Embedding(\n", "encoder_embeddings = tf.keras.layers.Embedding(\n",
" len(INPUT_CHARS) + 1, encoder_embedding_size)(encoder_inputs)\n", " len(INPUT_CHARS) + 1, encoder_embedding_size)(encoder_inputs)\n",
"\n", "\n",
"decoder_embedding_layer = keras.layers.Embedding(\n", "decoder_embedding_layer = tf.keras.layers.Embedding(\n",
" len(OUTPUT_CHARS) + 2, decoder_embedding_size)\n", " len(OUTPUT_CHARS) + 2, decoder_embedding_size)\n",
"decoder_embeddings = decoder_embedding_layer(decoder_inputs)\n", "decoder_embeddings = decoder_embedding_layer(decoder_inputs)\n",
"\n", "\n",
"encoder = keras.layers.LSTM(units, return_state=True)\n", "encoder = tf.keras.layers.LSTM(units, return_state=True)\n",
"encoder_outputs, state_h, state_c = encoder(encoder_embeddings)\n", "encoder_outputs, state_h, state_c = encoder(encoder_embeddings)\n",
"encoder_state = [state_h, state_c]\n", "encoder_state = [state_h, state_c]\n",
"\n", "\n",
"sampler = tfa.seq2seq.sampler.TrainingSampler()\n", "sampler = tfa.seq2seq.sampler.TrainingSampler()\n",
"\n", "\n",
"decoder_cell = keras.layers.LSTMCell(units)\n", "decoder_cell = tf.keras.layers.LSTMCell(units)\n",
"output_layer = keras.layers.Dense(len(OUTPUT_CHARS) + 1)\n", "output_layer = tf.keras.layers.Dense(len(OUTPUT_CHARS) + 1)\n",
"\n", "\n",
"decoder = tfa.seq2seq.basic_decoder.BasicDecoder(decoder_cell,\n", "decoder = tfa.seq2seq.basic_decoder.BasicDecoder(decoder_cell,\n",
" sampler,\n", " sampler,\n",
@ -2149,11 +2149,11 @@
"final_outputs, final_state, final_sequence_lengths = decoder(\n", "final_outputs, final_state, final_sequence_lengths = decoder(\n",
" decoder_embeddings,\n", " decoder_embeddings,\n",
" initial_state=encoder_state)\n", " initial_state=encoder_state)\n",
"Y_proba = keras.layers.Activation(\"softmax\")(final_outputs.rnn_output)\n", "Y_proba = tf.keras.layers.Activation(\"softmax\")(final_outputs.rnn_output)\n",
"\n", "\n",
"model = keras.models.Model(inputs=[encoder_inputs, decoder_inputs],\n", "model = tf.keras.Model(inputs=[encoder_inputs, decoder_inputs],\n",
" outputs=[Y_proba])\n", " outputs=[Y_proba])\n",
"optimizer = keras.optimizers.Nadam()\n", "optimizer = tf.keras.optimizers.Nadam()\n",
"model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=optimizer,\n", "model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=optimizer,\n",
" metrics=[\"accuracy\"])\n", " metrics=[\"accuracy\"])\n",
"history = model.fit([X_train, X_train_decoder], Y_train, epochs=15,\n", "history = model.fit([X_train, X_train_decoder], Y_train, epochs=15,\n",
@ -2206,7 +2206,7 @@
" start_tokens=start_tokens,\n", " start_tokens=start_tokens,\n",
" end_token=0)\n", " end_token=0)\n",
"\n", "\n",
"inference_model = keras.models.Model(inputs=[encoder_inputs],\n", "inference_model = tf.keras.Model(inputs=[encoder_inputs],\n",
" outputs=[final_outputs.sample_id])" " outputs=[final_outputs.sample_id])"
] ]
}, },
@ -2320,18 +2320,18 @@
"decoder_embedding_size = 32\n", "decoder_embedding_size = 32\n",
"units = 128\n", "units = 128\n",
"\n", "\n",
"encoder_inputs = keras.layers.Input(shape=[None], dtype=np.int32)\n", "encoder_inputs = tf.keras.layers.Input(shape=[None], dtype=np.int32)\n",
"decoder_inputs = keras.layers.Input(shape=[None], dtype=np.int32)\n", "decoder_inputs = tf.keras.layers.Input(shape=[None], dtype=np.int32)\n",
"sequence_lengths = keras.layers.Input(shape=[], dtype=np.int32)\n", "sequence_lengths = tf.keras.layers.Input(shape=[], dtype=np.int32)\n",
"\n", "\n",
"encoder_embeddings = keras.layers.Embedding(\n", "encoder_embeddings = tf.keras.layers.Embedding(\n",
" len(INPUT_CHARS) + 1, encoder_embedding_size)(encoder_inputs)\n", " len(INPUT_CHARS) + 1, encoder_embedding_size)(encoder_inputs)\n",
"\n", "\n",
"decoder_embedding_layer = keras.layers.Embedding(\n", "decoder_embedding_layer = tf.keras.layers.Embedding(\n",
" len(OUTPUT_CHARS) + 2, decoder_embedding_size)\n", " len(OUTPUT_CHARS) + 2, decoder_embedding_size)\n",
"decoder_embeddings = decoder_embedding_layer(decoder_inputs)\n", "decoder_embeddings = decoder_embedding_layer(decoder_inputs)\n",
"\n", "\n",
"encoder = keras.layers.LSTM(units, return_state=True)\n", "encoder = tf.keras.layers.LSTM(units, return_state=True)\n",
"encoder_outputs, state_h, state_c = encoder(encoder_embeddings)\n", "encoder_outputs, state_h, state_c = encoder(encoder_embeddings)\n",
"encoder_state = [state_h, state_c]\n", "encoder_state = [state_h, state_c]\n",
"\n", "\n",
@ -2342,8 +2342,8 @@
"# (see https://github.com/tensorflow/addons/pull/1714)\n", "# (see https://github.com/tensorflow/addons/pull/1714)\n",
"sampler.sampling_probability = tf.Variable(0.)\n", "sampler.sampling_probability = tf.Variable(0.)\n",
"\n", "\n",
"decoder_cell = keras.layers.LSTMCell(units)\n", "decoder_cell = tf.keras.layers.LSTMCell(units)\n",
"output_layer = keras.layers.Dense(len(OUTPUT_CHARS) + 1)\n", "output_layer = tf.keras.layers.Dense(len(OUTPUT_CHARS) + 1)\n",
"\n", "\n",
"decoder = tfa.seq2seq.basic_decoder.BasicDecoder(decoder_cell,\n", "decoder = tfa.seq2seq.basic_decoder.BasicDecoder(decoder_cell,\n",
" sampler,\n", " sampler,\n",
@ -2351,11 +2351,11 @@
"final_outputs, final_state, final_sequence_lengths = decoder(\n", "final_outputs, final_state, final_sequence_lengths = decoder(\n",
" decoder_embeddings,\n", " decoder_embeddings,\n",
" initial_state=encoder_state)\n", " initial_state=encoder_state)\n",
"Y_proba = keras.layers.Activation(\"softmax\")(final_outputs.rnn_output)\n", "Y_proba = tf.keras.layers.Activation(\"softmax\")(final_outputs.rnn_output)\n",
"\n", "\n",
"model = keras.models.Model(inputs=[encoder_inputs, decoder_inputs],\n", "model = tf.keras.Model(inputs=[encoder_inputs, decoder_inputs],\n",
" outputs=[Y_proba])\n", " outputs=[Y_proba])\n",
"optimizer = keras.optimizers.Nadam()\n", "optimizer = tf.keras.optimizers.Nadam()\n",
"model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=optimizer,\n", "model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=optimizer,\n",
" metrics=[\"accuracy\"])\n", " metrics=[\"accuracy\"])\n",
"\n", "\n",
@ -2363,7 +2363,7 @@
" proba = min(1.0, epoch / (n_epochs - 10))\n", " proba = min(1.0, epoch / (n_epochs - 10))\n",
" sampler.sampling_probability.assign(proba)\n", " sampler.sampling_probability.assign(proba)\n",
"\n", "\n",
"sampling_probability_cb = keras.callbacks.LambdaCallback(\n", "sampling_probability_cb = tf.keras.callbacks.LambdaCallback(\n",
" on_epoch_begin=update_sampling_probability)\n", " on_epoch_begin=update_sampling_probability)\n",
"history = model.fit([X_train, X_train_decoder], Y_train, epochs=n_epochs,\n", "history = model.fit([X_train, X_train_decoder], Y_train, epochs=n_epochs,\n",
" validation_data=([X_valid, X_valid_decoder], Y_valid),\n", " validation_data=([X_valid, X_valid_decoder], Y_valid),\n",
@ -2407,7 +2407,7 @@
" start_tokens=start_tokens,\n", " start_tokens=start_tokens,\n",
" end_token=0)\n", " end_token=0)\n",
"\n", "\n",
"inference_model = keras.models.Model(inputs=[encoder_inputs],\n", "inference_model = tf.keras.Model(inputs=[encoder_inputs],\n",
" outputs=[final_outputs.sample_id])" " outputs=[final_outputs.sample_id])"
] ]
}, },
@ -2490,25 +2490,25 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"class DateTranslation(keras.models.Model):\n", "class DateTranslation(tf.keras.Model):\n",
" def __init__(self, units=128, encoder_embedding_size=32,\n", " def __init__(self, units=128, encoder_embedding_size=32,\n",
" decoder_embedding_size=32, **kwargs):\n", " decoder_embedding_size=32, **kwargs):\n",
" super().__init__(**kwargs)\n", " super().__init__(**kwargs)\n",
" self.encoder_embedding = keras.layers.Embedding(\n", " self.encoder_embedding = tf.keras.layers.Embedding(\n",
" input_dim=len(INPUT_CHARS) + 1,\n", " input_dim=len(INPUT_CHARS) + 1,\n",
" output_dim=encoder_embedding_size)\n", " output_dim=encoder_embedding_size)\n",
" self.encoder = keras.layers.LSTM(units,\n", " self.encoder = tf.keras.layers.LSTM(units,\n",
" return_sequences=True,\n", " return_sequences=True,\n",
" return_state=True)\n", " return_state=True)\n",
" self.decoder_embedding = keras.layers.Embedding(\n", " self.decoder_embedding = tf.keras.layers.Embedding(\n",
" input_dim=len(OUTPUT_CHARS) + 2,\n", " input_dim=len(OUTPUT_CHARS) + 2,\n",
" output_dim=decoder_embedding_size)\n", " output_dim=decoder_embedding_size)\n",
" self.attention = tfa.seq2seq.LuongAttention(units)\n", " self.attention = tfa.seq2seq.LuongAttention(units)\n",
" decoder_inner_cell = keras.layers.LSTMCell(units)\n", " decoder_inner_cell = tf.keras.layers.LSTMCell(units)\n",
" self.decoder_cell = tfa.seq2seq.AttentionWrapper(\n", " self.decoder_cell = tfa.seq2seq.AttentionWrapper(\n",
" cell=decoder_inner_cell,\n", " cell=decoder_inner_cell,\n",
" attention_mechanism=self.attention)\n", " attention_mechanism=self.attention)\n",
" output_layer = keras.layers.Dense(len(OUTPUT_CHARS) + 1)\n", " output_layer = tf.keras.layers.Dense(len(OUTPUT_CHARS) + 1)\n",
" self.decoder = tfa.seq2seq.BasicDecoder(\n", " self.decoder = tfa.seq2seq.BasicDecoder(\n",
" cell=self.decoder_cell,\n", " cell=self.decoder_cell,\n",
" sampler=tfa.seq2seq.sampler.TrainingSampler(),\n", " sampler=tfa.seq2seq.sampler.TrainingSampler(),\n",
@ -2564,7 +2564,7 @@
"tf.random.set_seed(42)\n", "tf.random.set_seed(42)\n",
"\n", "\n",
"model = DateTranslation()\n", "model = DateTranslation()\n",
"optimizer = keras.optimizers.Nadam()\n", "optimizer = tf.keras.optimizers.Nadam()\n",
"model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=optimizer,\n", "model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=optimizer,\n",
" metrics=[\"accuracy\"])\n", " metrics=[\"accuracy\"])\n",
"history = model.fit([X_train, X_train_decoder], Y_train, epochs=25,\n", "history = model.fit([X_train, X_train_decoder], Y_train, epochs=25,\n",

380
16_autoencoders_and_gans.ipynb
View File

@ -166,11 +166,11 @@
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)\n", "tf.random.set_seed(42)\n",
"\n", "\n",
"encoder = keras.models.Sequential([keras.layers.Dense(2, input_shape=[3])])\n", "encoder = tf.keras.Sequential([tf.keras.layers.Dense(2, input_shape=[3])])\n",
"decoder = keras.models.Sequential([keras.layers.Dense(3, input_shape=[2])])\n", "decoder = tf.keras.Sequential([tf.keras.layers.Dense(3, input_shape=[2])])\n",
"autoencoder = keras.models.Sequential([encoder, decoder])\n", "autoencoder = tf.keras.Sequential([encoder, decoder])\n",
"\n", "\n",
"autoencoder.compile(loss=\"mse\", optimizer=keras.optimizers.SGD(learning_rate=1.5))" "autoencoder.compile(loss=\"mse\", optimizer=tf.keras.optimizers.SGD(learning_rate=1.5))"
] ]
}, },
{ {
@ -226,7 +226,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"(X_train_full, y_train_full), (X_test, y_test) = keras.datasets.fashion_mnist.load_data()\n", "(X_train_full, y_train_full), (X_test, y_test) = tf.keras.datasets.fashion_mnist.load_data()\n",
"X_train_full = X_train_full.astype(np.float32) / 255\n", "X_train_full = X_train_full.astype(np.float32) / 255\n",
"X_test = X_test.astype(np.float32) / 255\n", "X_test = X_test.astype(np.float32) / 255\n",
"X_train, X_valid = X_train_full[:-5000], X_train_full[-5000:]\n", "X_train, X_valid = X_train_full[:-5000], X_train_full[-5000:]\n",
@ -254,7 +254,7 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"def rounded_accuracy(y_true, y_pred):\n", "def rounded_accuracy(y_true, y_pred):\n",
" return keras.metrics.binary_accuracy(tf.round(y_true), tf.round(y_pred))" " return tf.keras.metrics.binary_accuracy(tf.round(y_true), tf.round(y_pred))"
] ]
}, },
{ {
@ -266,19 +266,19 @@
"tf.random.set_seed(42)\n", "tf.random.set_seed(42)\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"\n", "\n",
"stacked_encoder = keras.models.Sequential([\n", "stacked_encoder = tf.keras.Sequential([\n",
" keras.layers.Flatten(input_shape=[28, 28]),\n", " tf.keras.layers.Flatten(input_shape=[28, 28]),\n",
" keras.layers.Dense(100, activation=\"selu\"),\n", " tf.keras.layers.Dense(100, activation=\"selu\"),\n",
" keras.layers.Dense(30, activation=\"selu\"),\n", " tf.keras.layers.Dense(30, activation=\"selu\"),\n",
"])\n", "])\n",
"stacked_decoder = keras.models.Sequential([\n", "stacked_decoder = tf.keras.Sequential([\n",
" keras.layers.Dense(100, activation=\"selu\", input_shape=[30]),\n", " tf.keras.layers.Dense(100, activation=\"selu\", input_shape=[30]),\n",
" keras.layers.Dense(28 * 28, activation=\"sigmoid\"),\n", " tf.keras.layers.Dense(28 * 28, activation=\"sigmoid\"),\n",
" keras.layers.Reshape([28, 28])\n", " tf.keras.layers.Reshape([28, 28])\n",
"])\n", "])\n",
"stacked_ae = keras.models.Sequential([stacked_encoder, stacked_decoder])\n", "stacked_ae = tf.keras.Sequential([stacked_encoder, stacked_decoder])\n",
"stacked_ae.compile(loss=\"binary_crossentropy\",\n", "stacked_ae.compile(loss=\"binary_crossentropy\",\n",
" optimizer=keras.optimizers.SGD(learning_rate=1.5), metrics=[rounded_accuracy])\n", " optimizer=tf.keras.optimizers.SGD(learning_rate=1.5), metrics=[rounded_accuracy])\n",
"history = stacked_ae.fit(X_train, X_train, epochs=20,\n", "history = stacked_ae.fit(X_train, X_train, epochs=20,\n",
" validation_data=(X_valid, X_valid))" " validation_data=(X_valid, X_valid))"
] ]
@ -401,10 +401,10 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"class DenseTranspose(keras.layers.Layer):\n", "class DenseTranspose(tf.keras.layers.Layer):\n",
" def __init__(self, dense, activation=None, **kwargs):\n", " def __init__(self, dense, activation=None, **kwargs):\n",
" self.dense = dense\n", " self.dense = dense\n",
" self.activation = keras.activations.get(activation)\n", " self.activation = tf.keras.activations.get(activation)\n",
" super().__init__(**kwargs)\n", " super().__init__(**kwargs)\n",
" def build(self, batch_input_shape):\n", " def build(self, batch_input_shape):\n",
" self.biases = self.add_weight(name=\"bias\",\n", " self.biases = self.add_weight(name=\"bias\",\n",
@ -422,29 +422,29 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"keras.backend.clear_session()\n", "tf.keras.backend.clear_session()\n",
"tf.random.set_seed(42)\n", "tf.random.set_seed(42)\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"\n", "\n",
"dense_1 = keras.layers.Dense(100, activation=\"selu\")\n", "dense_1 = tf.keras.layers.Dense(100, activation=\"selu\")\n",
"dense_2 = keras.layers.Dense(30, activation=\"selu\")\n", "dense_2 = tf.keras.layers.Dense(30, activation=\"selu\")\n",
"\n", "\n",
"tied_encoder = keras.models.Sequential([\n", "tied_encoder = tf.keras.Sequential([\n",
" keras.layers.Flatten(input_shape=[28, 28]),\n", " tf.keras.layers.Flatten(input_shape=[28, 28]),\n",
" dense_1,\n", " dense_1,\n",
" dense_2\n", " dense_2\n",
"])\n", "])\n",
"\n", "\n",
"tied_decoder = keras.models.Sequential([\n", "tied_decoder = tf.keras.Sequential([\n",
" DenseTranspose(dense_2, activation=\"selu\"),\n", " DenseTranspose(dense_2, activation=\"selu\"),\n",
" DenseTranspose(dense_1, activation=\"sigmoid\"),\n", " DenseTranspose(dense_1, activation=\"sigmoid\"),\n",
" keras.layers.Reshape([28, 28])\n", " tf.keras.layers.Reshape([28, 28])\n",
"])\n", "])\n",
"\n", "\n",
"tied_ae = keras.models.Sequential([tied_encoder, tied_decoder])\n", "tied_ae = tf.keras.Sequential([tied_encoder, tied_decoder])\n",
"\n", "\n",
"tied_ae.compile(loss=\"binary_crossentropy\",\n", "tied_ae.compile(loss=\"binary_crossentropy\",\n",
" optimizer=keras.optimizers.SGD(learning_rate=1.5), metrics=[rounded_accuracy])\n", " optimizer=tf.keras.optimizers.SGD(learning_rate=1.5), metrics=[rounded_accuracy])\n",
"history = tied_ae.fit(X_train, X_train, epochs=10,\n", "history = tied_ae.fit(X_train, X_train, epochs=10,\n",
" validation_data=(X_valid, X_valid))" " validation_data=(X_valid, X_valid))"
] ]
@ -477,13 +477,13 @@
"def train_autoencoder(n_neurons, X_train, X_valid, loss, optimizer,\n", "def train_autoencoder(n_neurons, X_train, X_valid, loss, optimizer,\n",
" n_epochs=10, output_activation=None, metrics=None):\n", " n_epochs=10, output_activation=None, metrics=None):\n",
" n_inputs = X_train.shape[-1]\n", " n_inputs = X_train.shape[-1]\n",
" encoder = keras.models.Sequential([\n", " encoder = tf.keras.Sequential([\n",
" keras.layers.Dense(n_neurons, activation=\"selu\", input_shape=[n_inputs])\n", " tf.keras.layers.Dense(n_neurons, activation=\"selu\", input_shape=[n_inputs])\n",
" ])\n", " ])\n",
" decoder = keras.models.Sequential([\n", " decoder = tf.keras.Sequential([\n",
" keras.layers.Dense(n_inputs, activation=output_activation),\n", " tf.keras.layers.Dense(n_inputs, activation=output_activation),\n",
" ])\n", " ])\n",
" autoencoder = keras.models.Sequential([encoder, decoder])\n", " autoencoder = tf.keras.Sequential([encoder, decoder])\n",
" autoencoder.compile(optimizer, loss, metrics=metrics)\n", " autoencoder.compile(optimizer, loss, metrics=metrics)\n",
" autoencoder.fit(X_train, X_train, epochs=n_epochs,\n", " autoencoder.fit(X_train, X_train, epochs=n_epochs,\n",
" validation_data=(X_valid, X_valid))\n", " validation_data=(X_valid, X_valid))\n",
@ -499,15 +499,15 @@
"tf.random.set_seed(42)\n", "tf.random.set_seed(42)\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"\n", "\n",
"K = keras.backend\n", "K = tf.keras.backend\n",
"X_train_flat = K.batch_flatten(X_train) # equivalent to .reshape(-1, 28 * 28)\n", "X_train_flat = K.batch_flatten(X_train) # equivalent to .reshape(-1, 28 * 28)\n",
"X_valid_flat = K.batch_flatten(X_valid)\n", "X_valid_flat = K.batch_flatten(X_valid)\n",
"enc1, dec1, X_train_enc1, X_valid_enc1 = train_autoencoder(\n", "enc1, dec1, X_train_enc1, X_valid_enc1 = train_autoencoder(\n",
" 100, X_train_flat, X_valid_flat, \"binary_crossentropy\",\n", " 100, X_train_flat, X_valid_flat, \"binary_crossentropy\",\n",
" keras.optimizers.SGD(learning_rate=1.5), output_activation=\"sigmoid\",\n", " tf.keras.optimizers.SGD(learning_rate=1.5), output_activation=\"sigmoid\",\n",
" metrics=[rounded_accuracy])\n", " metrics=[rounded_accuracy])\n",
"enc2, dec2, _, _ = train_autoencoder(\n", "enc2, dec2, _, _ = train_autoencoder(\n",
" 30, X_train_enc1, X_valid_enc1, \"mse\", keras.optimizers.SGD(learning_rate=0.05),\n", " 30, X_train_enc1, X_valid_enc1, \"mse\", tf.keras.optimizers.SGD(learning_rate=0.05),\n",
" output_activation=\"selu\")" " output_activation=\"selu\")"
] ]
}, },
@ -517,10 +517,10 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"stacked_ae_1_by_1 = keras.models.Sequential([\n", "stacked_ae_1_by_1 = tf.keras.Sequential([\n",
" keras.layers.Flatten(input_shape=[28, 28]),\n", " tf.keras.layers.Flatten(input_shape=[28, 28]),\n",
" enc1, enc2, dec2, dec1,\n", " enc1, enc2, dec2, dec1,\n",
" keras.layers.Reshape([28, 28])\n", " tf.keras.layers.Reshape([28, 28])\n",
"])" "])"
] ]
}, },
@ -541,7 +541,7 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"stacked_ae_1_by_1.compile(loss=\"binary_crossentropy\",\n", "stacked_ae_1_by_1.compile(loss=\"binary_crossentropy\",\n",
" optimizer=keras.optimizers.SGD(learning_rate=0.1), metrics=[rounded_accuracy])\n", " optimizer=tf.keras.optimizers.SGD(learning_rate=0.1), metrics=[rounded_accuracy])\n",
"history = stacked_ae_1_by_1.fit(X_train, X_train, epochs=10,\n", "history = stacked_ae_1_by_1.fit(X_train, X_train, epochs=10,\n",
" validation_data=(X_valid, X_valid))" " validation_data=(X_valid, X_valid))"
] ]
@ -579,25 +579,25 @@
"tf.random.set_seed(42)\n", "tf.random.set_seed(42)\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"\n", "\n",
"conv_encoder = keras.models.Sequential([\n", "conv_encoder = tf.keras.Sequential([\n",
" keras.layers.Reshape([28, 28, 1], input_shape=[28, 28]),\n", " tf.keras.layers.Reshape([28, 28, 1], input_shape=[28, 28]),\n",
" keras.layers.Conv2D(16, kernel_size=3, padding=\"SAME\", activation=\"selu\"),\n", " tf.keras.layers.Conv2D(16, kernel_size=3, padding=\"SAME\", activation=\"selu\"),\n",
" keras.layers.MaxPool2D(pool_size=2),\n", " tf.keras.layers.MaxPool2D(pool_size=2),\n",
" keras.layers.Conv2D(32, kernel_size=3, padding=\"SAME\", activation=\"selu\"),\n", " tf.keras.layers.Conv2D(32, kernel_size=3, padding=\"SAME\", activation=\"selu\"),\n",
" keras.layers.MaxPool2D(pool_size=2),\n", " tf.keras.layers.MaxPool2D(pool_size=2),\n",
" keras.layers.Conv2D(64, kernel_size=3, padding=\"SAME\", activation=\"selu\"),\n", " tf.keras.layers.Conv2D(64, kernel_size=3, padding=\"SAME\", activation=\"selu\"),\n",
" keras.layers.MaxPool2D(pool_size=2)\n", " tf.keras.layers.MaxPool2D(pool_size=2)\n",
"])\n", "])\n",
"conv_decoder = keras.models.Sequential([\n", "conv_decoder = tf.keras.Sequential([\n",
" keras.layers.Conv2DTranspose(32, kernel_size=3, strides=2, padding=\"VALID\", activation=\"selu\",\n", " tf.keras.layers.Conv2DTranspose(32, kernel_size=3, strides=2, padding=\"VALID\", activation=\"selu\",\n",
" input_shape=[3, 3, 64]),\n", " input_shape=[3, 3, 64]),\n",
" keras.layers.Conv2DTranspose(16, kernel_size=3, strides=2, padding=\"SAME\", activation=\"selu\"),\n", " tf.keras.layers.Conv2DTranspose(16, kernel_size=3, strides=2, padding=\"SAME\", activation=\"selu\"),\n",
" keras.layers.Conv2DTranspose(1, kernel_size=3, strides=2, padding=\"SAME\", activation=\"sigmoid\"),\n", " tf.keras.layers.Conv2DTranspose(1, kernel_size=3, strides=2, padding=\"SAME\", activation=\"sigmoid\"),\n",
" keras.layers.Reshape([28, 28])\n", " tf.keras.layers.Reshape([28, 28])\n",
"])\n", "])\n",
"conv_ae = keras.models.Sequential([conv_encoder, conv_decoder])\n", "conv_ae = tf.keras.Sequential([conv_encoder, conv_decoder])\n",
"\n", "\n",
"conv_ae.compile(loss=\"binary_crossentropy\", optimizer=keras.optimizers.SGD(learning_rate=1.0),\n", "conv_ae.compile(loss=\"binary_crossentropy\", optimizer=tf.keras.optimizers.SGD(learning_rate=1.0),\n",
" metrics=[rounded_accuracy])\n", " metrics=[rounded_accuracy])\n",
"history = conv_ae.fit(X_train, X_train, epochs=5,\n", "history = conv_ae.fit(X_train, X_train, epochs=5,\n",
" validation_data=(X_valid, X_valid))" " validation_data=(X_valid, X_valid))"
@ -636,17 +636,17 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"recurrent_encoder = keras.models.Sequential([\n", "recurrent_encoder = tf.keras.Sequential([\n",
" keras.layers.LSTM(100, return_sequences=True, input_shape=[28, 28]),\n", " tf.keras.layers.LSTM(100, return_sequences=True, input_shape=[28, 28]),\n",
" keras.layers.LSTM(30)\n", " tf.keras.layers.LSTM(30)\n",
"])\n", "])\n",
"recurrent_decoder = keras.models.Sequential([\n", "recurrent_decoder = tf.keras.Sequential([\n",
" keras.layers.RepeatVector(28, input_shape=[30]),\n", " tf.keras.layers.RepeatVector(28, input_shape=[30]),\n",
" keras.layers.LSTM(100, return_sequences=True),\n", " tf.keras.layers.LSTM(100, return_sequences=True),\n",
" keras.layers.TimeDistributed(keras.layers.Dense(28, activation=\"sigmoid\"))\n", " tf.keras.layers.TimeDistributed(tf.keras.layers.Dense(28, activation=\"sigmoid\"))\n",
"])\n", "])\n",
"recurrent_ae = keras.models.Sequential([recurrent_encoder, recurrent_decoder])\n", "recurrent_ae = tf.keras.Sequential([recurrent_encoder, recurrent_decoder])\n",
"recurrent_ae.compile(loss=\"binary_crossentropy\", optimizer=keras.optimizers.SGD(0.1),\n", "recurrent_ae.compile(loss=\"binary_crossentropy\", optimizer=tf.keras.optimizers.SGD(0.1),\n",
" metrics=[rounded_accuracy])" " metrics=[rounded_accuracy])"
] ]
}, },
@ -692,19 +692,19 @@
"tf.random.set_seed(42)\n", "tf.random.set_seed(42)\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"\n", "\n",
"denoising_encoder = keras.models.Sequential([\n", "denoising_encoder = tf.keras.Sequential([\n",
" keras.layers.Flatten(input_shape=[28, 28]),\n", " tf.keras.layers.Flatten(input_shape=[28, 28]),\n",
" keras.layers.GaussianNoise(0.2),\n", " tf.keras.layers.GaussianNoise(0.2),\n",
" keras.layers.Dense(100, activation=\"selu\"),\n", " tf.keras.layers.Dense(100, activation=\"selu\"),\n",
" keras.layers.Dense(30, activation=\"selu\")\n", " tf.keras.layers.Dense(30, activation=\"selu\")\n",
"])\n", "])\n",
"denoising_decoder = keras.models.Sequential([\n", "denoising_decoder = tf.keras.Sequential([\n",
" keras.layers.Dense(100, activation=\"selu\", input_shape=[30]),\n", " tf.keras.layers.Dense(100, activation=\"selu\", input_shape=[30]),\n",
" keras.layers.Dense(28 * 28, activation=\"sigmoid\"),\n", " tf.keras.layers.Dense(28 * 28, activation=\"sigmoid\"),\n",
" keras.layers.Reshape([28, 28])\n", " tf.keras.layers.Reshape([28, 28])\n",
"])\n", "])\n",
"denoising_ae = keras.models.Sequential([denoising_encoder, denoising_decoder])\n", "denoising_ae = tf.keras.Sequential([denoising_encoder, denoising_decoder])\n",
"denoising_ae.compile(loss=\"binary_crossentropy\", optimizer=keras.optimizers.SGD(learning_rate=1.0),\n", "denoising_ae.compile(loss=\"binary_crossentropy\", optimizer=tf.keras.optimizers.SGD(learning_rate=1.0),\n",
" metrics=[rounded_accuracy])\n", " metrics=[rounded_accuracy])\n",
"history = denoising_ae.fit(X_train, X_train, epochs=10,\n", "history = denoising_ae.fit(X_train, X_train, epochs=10,\n",
" validation_data=(X_valid, X_valid))" " validation_data=(X_valid, X_valid))"
@ -719,7 +719,7 @@
"tf.random.set_seed(42)\n", "tf.random.set_seed(42)\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"\n", "\n",
"noise = keras.layers.GaussianNoise(0.2)\n", "noise = tf.keras.layers.GaussianNoise(0.2)\n",
"show_reconstructions(denoising_ae, noise(X_valid, training=True))\n", "show_reconstructions(denoising_ae, noise(X_valid, training=True))\n",
"plt.show()" "plt.show()"
] ]
@ -740,19 +740,19 @@
"tf.random.set_seed(42)\n", "tf.random.set_seed(42)\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"\n", "\n",
"dropout_encoder = keras.models.Sequential([\n", "dropout_encoder = tf.keras.Sequential([\n",
" keras.layers.Flatten(input_shape=[28, 28]),\n", " tf.keras.layers.Flatten(input_shape=[28, 28]),\n",
" keras.layers.Dropout(0.5),\n", " tf.keras.layers.Dropout(0.5),\n",
" keras.layers.Dense(100, activation=\"selu\"),\n", " tf.keras.layers.Dense(100, activation=\"selu\"),\n",
" keras.layers.Dense(30, activation=\"selu\")\n", " tf.keras.layers.Dense(30, activation=\"selu\")\n",
"])\n", "])\n",
"dropout_decoder = keras.models.Sequential([\n", "dropout_decoder = tf.keras.Sequential([\n",
" keras.layers.Dense(100, activation=\"selu\", input_shape=[30]),\n", " tf.keras.layers.Dense(100, activation=\"selu\", input_shape=[30]),\n",
" keras.layers.Dense(28 * 28, activation=\"sigmoid\"),\n", " tf.keras.layers.Dense(28 * 28, activation=\"sigmoid\"),\n",
" keras.layers.Reshape([28, 28])\n", " tf.keras.layers.Reshape([28, 28])\n",
"])\n", "])\n",
"dropout_ae = keras.models.Sequential([dropout_encoder, dropout_decoder])\n", "dropout_ae = tf.keras.Sequential([dropout_encoder, dropout_decoder])\n",
"dropout_ae.compile(loss=\"binary_crossentropy\", optimizer=keras.optimizers.SGD(learning_rate=1.0),\n", "dropout_ae.compile(loss=\"binary_crossentropy\", optimizer=tf.keras.optimizers.SGD(learning_rate=1.0),\n",
" metrics=[rounded_accuracy])\n", " metrics=[rounded_accuracy])\n",
"history = dropout_ae.fit(X_train, X_train, epochs=10,\n", "history = dropout_ae.fit(X_train, X_train, epochs=10,\n",
" validation_data=(X_valid, X_valid))" " validation_data=(X_valid, X_valid))"
@ -767,7 +767,7 @@
"tf.random.set_seed(42)\n", "tf.random.set_seed(42)\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"\n", "\n",
"dropout = keras.layers.Dropout(0.5)\n", "dropout = tf.keras.layers.Dropout(0.5)\n",
"show_reconstructions(dropout_ae, dropout(X_valid, training=True))\n", "show_reconstructions(dropout_ae, dropout(X_valid, training=True))\n",
"save_fig(\"dropout_denoising_plot\", tight_layout=False)" "save_fig(\"dropout_denoising_plot\", tight_layout=False)"
] ]
@ -795,18 +795,18 @@
"tf.random.set_seed(42)\n", "tf.random.set_seed(42)\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"\n", "\n",
"simple_encoder = keras.models.Sequential([\n", "simple_encoder = tf.keras.Sequential([\n",
" keras.layers.Flatten(input_shape=[28, 28]),\n", " tf.keras.layers.Flatten(input_shape=[28, 28]),\n",
" keras.layers.Dense(100, activation=\"selu\"),\n", " tf.keras.layers.Dense(100, activation=\"selu\"),\n",
" keras.layers.Dense(30, activation=\"sigmoid\"),\n", " tf.keras.layers.Dense(30, activation=\"sigmoid\"),\n",
"])\n", "])\n",
"simple_decoder = keras.models.Sequential([\n", "simple_decoder = tf.keras.Sequential([\n",
" keras.layers.Dense(100, activation=\"selu\", input_shape=[30]),\n", " tf.keras.layers.Dense(100, activation=\"selu\", input_shape=[30]),\n",
" keras.layers.Dense(28 * 28, activation=\"sigmoid\"),\n", " tf.keras.layers.Dense(28 * 28, activation=\"sigmoid\"),\n",
" keras.layers.Reshape([28, 28])\n", " tf.keras.layers.Reshape([28, 28])\n",
"])\n", "])\n",
"simple_ae = keras.models.Sequential([simple_encoder, simple_decoder])\n", "simple_ae = tf.keras.Sequential([simple_encoder, simple_decoder])\n",
"simple_ae.compile(loss=\"binary_crossentropy\", optimizer=keras.optimizers.SGD(learning_rate=1.),\n", "simple_ae.compile(loss=\"binary_crossentropy\", optimizer=tf.keras.optimizers.SGD(learning_rate=1.),\n",
" metrics=[rounded_accuracy])\n", " metrics=[rounded_accuracy])\n",
"history = simple_ae.fit(X_train, X_train, epochs=10,\n", "history = simple_ae.fit(X_train, X_train, epochs=10,\n",
" validation_data=(X_valid, X_valid))" " validation_data=(X_valid, X_valid))"
@ -905,21 +905,21 @@
"tf.random.set_seed(42)\n", "tf.random.set_seed(42)\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"\n", "\n",
"sparse_l1_encoder = keras.models.Sequential([\n", "sparse_l1_encoder = tf.keras.Sequential([\n",
" keras.layers.Flatten(input_shape=[28, 28]),\n", " tf.keras.layers.Flatten(input_shape=[28, 28]),\n",
" keras.layers.Dense(100, activation=\"selu\"),\n", " tf.keras.layers.Dense(100, activation=\"selu\"),\n",
" keras.layers.Dense(300, activation=\"sigmoid\"),\n", " tf.keras.layers.Dense(300, activation=\"sigmoid\"),\n",
" keras.layers.ActivityRegularization(l1=1e-3) # Alternatively, you could add\n", " tf.keras.layers.ActivityRegularization(l1=1e-3) # Alternatively, you could add\n",
" # activity_regularizer=keras.regularizers.l1(1e-3)\n", " # activity_regularizer=tf.keras.regularizers.l1(1e-3)\n",
" # to the previous layer.\n", " # to the previous layer.\n",
"])\n", "])\n",
"sparse_l1_decoder = keras.models.Sequential([\n", "sparse_l1_decoder = tf.keras.Sequential([\n",
" keras.layers.Dense(100, activation=\"selu\", input_shape=[300]),\n", " tf.keras.layers.Dense(100, activation=\"selu\", input_shape=[300]),\n",
" keras.layers.Dense(28 * 28, activation=\"sigmoid\"),\n", " tf.keras.layers.Dense(28 * 28, activation=\"sigmoid\"),\n",
" keras.layers.Reshape([28, 28])\n", " tf.keras.layers.Reshape([28, 28])\n",
"])\n", "])\n",
"sparse_l1_ae = keras.models.Sequential([sparse_l1_encoder, sparse_l1_decoder])\n", "sparse_l1_ae = tf.keras.Sequential([sparse_l1_encoder, sparse_l1_decoder])\n",
"sparse_l1_ae.compile(loss=\"binary_crossentropy\", optimizer=keras.optimizers.SGD(learning_rate=1.0),\n", "sparse_l1_ae.compile(loss=\"binary_crossentropy\", optimizer=tf.keras.optimizers.SGD(learning_rate=1.0),\n",
" metrics=[rounded_accuracy])\n", " metrics=[rounded_accuracy])\n",
"history = sparse_l1_ae.fit(X_train, X_train, epochs=10,\n", "history = sparse_l1_ae.fit(X_train, X_train, epochs=10,\n",
" validation_data=(X_valid, X_valid))" " validation_data=(X_valid, X_valid))"
@ -980,10 +980,10 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"K = keras.backend\n", "K = tf.keras.backend\n",
"kl_divergence = keras.losses.kullback_leibler_divergence\n", "kl_divergence = tf.keras.losses.kullback_leibler_divergence\n",
"\n", "\n",
"class KLDivergenceRegularizer(keras.regularizers.Regularizer):\n", "class KLDivergenceRegularizer(tf.keras.regularizers.Regularizer):\n",
" def __init__(self, weight, target=0.1):\n", " def __init__(self, weight, target=0.1):\n",
" self.weight = weight\n", " self.weight = weight\n",
" self.target = target\n", " self.target = target\n",
@ -1004,18 +1004,18 @@
"np.random.seed(42)\n", "np.random.seed(42)\n",
"\n", "\n",
"kld_reg = KLDivergenceRegularizer(weight=0.05, target=0.1)\n", "kld_reg = KLDivergenceRegularizer(weight=0.05, target=0.1)\n",
"sparse_kl_encoder = keras.models.Sequential([\n", "sparse_kl_encoder = tf.keras.Sequential([\n",
" keras.layers.Flatten(input_shape=[28, 28]),\n", " tf.keras.layers.Flatten(input_shape=[28, 28]),\n",
" keras.layers.Dense(100, activation=\"selu\"),\n", " tf.keras.layers.Dense(100, activation=\"selu\"),\n",
" keras.layers.Dense(300, activation=\"sigmoid\", activity_regularizer=kld_reg)\n", " tf.keras.layers.Dense(300, activation=\"sigmoid\", activity_regularizer=kld_reg)\n",
"])\n", "])\n",
"sparse_kl_decoder = keras.models.Sequential([\n", "sparse_kl_decoder = tf.keras.Sequential([\n",
" keras.layers.Dense(100, activation=\"selu\", input_shape=[300]),\n", " tf.keras.layers.Dense(100, activation=\"selu\", input_shape=[300]),\n",
" keras.layers.Dense(28 * 28, activation=\"sigmoid\"),\n", " tf.keras.layers.Dense(28 * 28, activation=\"sigmoid\"),\n",
" keras.layers.Reshape([28, 28])\n", " tf.keras.layers.Reshape([28, 28])\n",
"])\n", "])\n",
"sparse_kl_ae = keras.models.Sequential([sparse_kl_encoder, sparse_kl_decoder])\n", "sparse_kl_ae = tf.keras.Sequential([sparse_kl_encoder, sparse_kl_decoder])\n",
"sparse_kl_ae.compile(loss=\"binary_crossentropy\", optimizer=keras.optimizers.SGD(learning_rate=1.0),\n", "sparse_kl_ae.compile(loss=\"binary_crossentropy\", optimizer=tf.keras.optimizers.SGD(learning_rate=1.0),\n",
" metrics=[rounded_accuracy])\n", " metrics=[rounded_accuracy])\n",
"history = sparse_kl_ae.fit(X_train, X_train, epochs=10,\n", "history = sparse_kl_ae.fit(X_train, X_train, epochs=10,\n",
" validation_data=(X_valid, X_valid))" " validation_data=(X_valid, X_valid))"
@ -1054,7 +1054,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"class Sampling(keras.layers.Layer):\n", "class Sampling(tf.keras.layers.Layer):\n",
" def call(self, inputs):\n", " def call(self, inputs):\n",
" mean, log_var = inputs\n", " mean, log_var = inputs\n",
" return K.random_normal(tf.shape(log_var)) * K.exp(log_var / 2) + mean " " return K.random_normal(tf.shape(log_var)) * K.exp(log_var / 2) + mean "
@ -1071,26 +1071,26 @@
"\n", "\n",
"codings_size = 10\n", "codings_size = 10\n",
"\n", "\n",
"inputs = keras.layers.Input(shape=[28, 28])\n", "inputs = tf.keras.layers.Input(shape=[28, 28])\n",
"z = keras.layers.Flatten()(inputs)\n", "z = tf.keras.layers.Flatten()(inputs)\n",
"z = keras.layers.Dense(150, activation=\"selu\")(z)\n", "z = tf.keras.layers.Dense(150, activation=\"selu\")(z)\n",
"z = keras.layers.Dense(100, activation=\"selu\")(z)\n", "z = tf.keras.layers.Dense(100, activation=\"selu\")(z)\n",
"codings_mean = keras.layers.Dense(codings_size)(z)\n", "codings_mean = tf.keras.layers.Dense(codings_size)(z)\n",
"codings_log_var = keras.layers.Dense(codings_size)(z)\n", "codings_log_var = tf.keras.layers.Dense(codings_size)(z)\n",
"codings = Sampling()([codings_mean, codings_log_var])\n", "codings = Sampling()([codings_mean, codings_log_var])\n",
"variational_encoder = keras.models.Model(\n", "variational_encoder = tf.keras.Model(\n",
" inputs=[inputs], outputs=[codings_mean, codings_log_var, codings])\n", " inputs=[inputs], outputs=[codings_mean, codings_log_var, codings])\n",
"\n", "\n",
"decoder_inputs = keras.layers.Input(shape=[codings_size])\n", "decoder_inputs = tf.keras.layers.Input(shape=[codings_size])\n",
"x = keras.layers.Dense(100, activation=\"selu\")(decoder_inputs)\n", "x = tf.keras.layers.Dense(100, activation=\"selu\")(decoder_inputs)\n",
"x = keras.layers.Dense(150, activation=\"selu\")(x)\n", "x = tf.keras.layers.Dense(150, activation=\"selu\")(x)\n",
"x = keras.layers.Dense(28 * 28, activation=\"sigmoid\")(x)\n", "x = tf.keras.layers.Dense(28 * 28, activation=\"sigmoid\")(x)\n",
"outputs = keras.layers.Reshape([28, 28])(x)\n", "outputs = tf.keras.layers.Reshape([28, 28])(x)\n",
"variational_decoder = keras.models.Model(inputs=[decoder_inputs], outputs=[outputs])\n", "variational_decoder = tf.keras.Model(inputs=[decoder_inputs], outputs=[outputs])\n",
"\n", "\n",
"_, _, codings = variational_encoder(inputs)\n", "_, _, codings = variational_encoder(inputs)\n",
"reconstructions = variational_decoder(codings)\n", "reconstructions = variational_decoder(codings)\n",
"variational_ae = keras.models.Model(inputs=[inputs], outputs=[reconstructions])\n", "variational_ae = tf.keras.Model(inputs=[inputs], outputs=[reconstructions])\n",
"\n", "\n",
"latent_loss = -0.5 * K.sum(\n", "latent_loss = -0.5 * K.sum(\n",
" 1 + codings_log_var - K.exp(codings_log_var) - K.square(codings_mean),\n", " 1 + codings_log_var - K.exp(codings_log_var) - K.square(codings_mean),\n",
@ -1210,19 +1210,19 @@
"\n", "\n",
"codings_size = 30\n", "codings_size = 30\n",
"\n", "\n",
"generator = keras.models.Sequential([\n", "generator = tf.keras.Sequential([\n",
" keras.layers.Dense(100, activation=\"selu\", input_shape=[codings_size]),\n", " tf.keras.layers.Dense(100, activation=\"selu\", input_shape=[codings_size]),\n",
" keras.layers.Dense(150, activation=\"selu\"),\n", " tf.keras.layers.Dense(150, activation=\"selu\"),\n",
" keras.layers.Dense(28 * 28, activation=\"sigmoid\"),\n", " tf.keras.layers.Dense(28 * 28, activation=\"sigmoid\"),\n",
" keras.layers.Reshape([28, 28])\n", " tf.keras.layers.Reshape([28, 28])\n",
"])\n", "])\n",
"discriminator = keras.models.Sequential([\n", "discriminator = tf.keras.Sequential([\n",
" keras.layers.Flatten(input_shape=[28, 28]),\n", " tf.keras.layers.Flatten(input_shape=[28, 28]),\n",
" keras.layers.Dense(150, activation=\"selu\"),\n", " tf.keras.layers.Dense(150, activation=\"selu\"),\n",
" keras.layers.Dense(100, activation=\"selu\"),\n", " tf.keras.layers.Dense(100, activation=\"selu\"),\n",
" keras.layers.Dense(1, activation=\"sigmoid\")\n", " tf.keras.layers.Dense(1, activation=\"sigmoid\")\n",
"])\n", "])\n",
"gan = keras.models.Sequential([generator, discriminator])" "gan = tf.keras.Sequential([generator, discriminator])"
] ]
}, },
{ {
@ -1325,28 +1325,28 @@
"\n", "\n",
"codings_size = 100\n", "codings_size = 100\n",
"\n", "\n",
"generator = keras.models.Sequential([\n", "generator = tf.keras.Sequential([\n",
" keras.layers.Dense(7 * 7 * 128, input_shape=[codings_size]),\n", " tf.keras.layers.Dense(7 * 7 * 128, input_shape=[codings_size]),\n",
" keras.layers.Reshape([7, 7, 128]),\n", " tf.keras.layers.Reshape([7, 7, 128]),\n",
" keras.layers.BatchNormalization(),\n", " tf.keras.layers.BatchNormalization(),\n",
" keras.layers.Conv2DTranspose(64, kernel_size=5, strides=2, padding=\"SAME\",\n", " tf.keras.layers.Conv2DTranspose(64, kernel_size=5, strides=2, padding=\"SAME\",\n",
" activation=\"selu\"),\n", " activation=\"selu\"),\n",
" keras.layers.BatchNormalization(),\n", " tf.keras.layers.BatchNormalization(),\n",
" keras.layers.Conv2DTranspose(1, kernel_size=5, strides=2, padding=\"SAME\",\n", " tf.keras.layers.Conv2DTranspose(1, kernel_size=5, strides=2, padding=\"SAME\",\n",
" activation=\"tanh\"),\n", " activation=\"tanh\"),\n",
"])\n", "])\n",
"discriminator = keras.models.Sequential([\n", "discriminator = tf.keras.Sequential([\n",
" keras.layers.Conv2D(64, kernel_size=5, strides=2, padding=\"SAME\",\n", " tf.keras.layers.Conv2D(64, kernel_size=5, strides=2, padding=\"SAME\",\n",
" activation=keras.layers.LeakyReLU(0.2),\n", " activation=tf.keras.layers.LeakyReLU(0.2),\n",
" input_shape=[28, 28, 1]),\n", " input_shape=[28, 28, 1]),\n",
" keras.layers.Dropout(0.4),\n", " tf.keras.layers.Dropout(0.4),\n",
" keras.layers.Conv2D(128, kernel_size=5, strides=2, padding=\"SAME\",\n", " tf.keras.layers.Conv2D(128, kernel_size=5, strides=2, padding=\"SAME\",\n",
" activation=keras.layers.LeakyReLU(0.2)),\n", " activation=tf.keras.layers.LeakyReLU(0.2)),\n",
" keras.layers.Dropout(0.4),\n", " tf.keras.layers.Dropout(0.4),\n",
" keras.layers.Flatten(),\n", " tf.keras.layers.Flatten(),\n",
" keras.layers.Dense(1, activation=\"sigmoid\")\n", " tf.keras.layers.Dense(1, activation=\"sigmoid\")\n",
"])\n", "])\n",
"gan = keras.models.Sequential([generator, discriminator])" "gan = tf.keras.Sequential([generator, discriminator])"
] ]
}, },
{ {
@ -1432,7 +1432,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"(X_train_full, y_train_full), (X_test, y_test) = keras.datasets.fashion_mnist.load_data()\n", "(X_train_full, y_train_full), (X_test, y_test) = tf.keras.datasets.fashion_mnist.load_data()\n",
"X_train_full = X_train_full.astype(np.float32) / 255\n", "X_train_full = X_train_full.astype(np.float32) / 255\n",
"X_test = X_test.astype(np.float32) / 255\n", "X_test = X_test.astype(np.float32) / 255\n",
"X_train, X_valid = X_train_full[:-5000], X_train_full[-5000:]\n", "X_train, X_valid = X_train_full[:-5000], X_train_full[-5000:]\n",
@ -1455,19 +1455,19 @@
"tf.random.set_seed(42)\n", "tf.random.set_seed(42)\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"\n", "\n",
"hashing_encoder = keras.models.Sequential([\n", "hashing_encoder = tf.keras.Sequential([\n",
" keras.layers.Flatten(input_shape=[28, 28]),\n", " tf.keras.layers.Flatten(input_shape=[28, 28]),\n",
" keras.layers.Dense(100, activation=\"selu\"),\n", " tf.keras.layers.Dense(100, activation=\"selu\"),\n",
" keras.layers.GaussianNoise(15.),\n", " tf.keras.layers.GaussianNoise(15.),\n",
" keras.layers.Dense(16, activation=\"sigmoid\"),\n", " tf.keras.layers.Dense(16, activation=\"sigmoid\"),\n",
"])\n", "])\n",
"hashing_decoder = keras.models.Sequential([\n", "hashing_decoder = tf.keras.Sequential([\n",
" keras.layers.Dense(100, activation=\"selu\", input_shape=[16]),\n", " tf.keras.layers.Dense(100, activation=\"selu\", input_shape=[16]),\n",
" keras.layers.Dense(28 * 28, activation=\"sigmoid\"),\n", " tf.keras.layers.Dense(28 * 28, activation=\"sigmoid\"),\n",
" keras.layers.Reshape([28, 28])\n", " tf.keras.layers.Reshape([28, 28])\n",
"])\n", "])\n",
"hashing_ae = keras.models.Sequential([hashing_encoder, hashing_decoder])\n", "hashing_ae = tf.keras.Sequential([hashing_encoder, hashing_decoder])\n",
"hashing_ae.compile(loss=\"binary_crossentropy\", optimizer=keras.optimizers.Nadam(),\n", "hashing_ae.compile(loss=\"binary_crossentropy\", optimizer=tf.keras.optimizers.Nadam(),\n",
" metrics=[rounded_accuracy])\n", " metrics=[rounded_accuracy])\n",
"history = hashing_ae.fit(X_train, X_train, epochs=10,\n", "history = hashing_ae.fit(X_train, X_train, epochs=10,\n",
" validation_data=(X_valid, X_valid))" " validation_data=(X_valid, X_valid))"
@ -1590,7 +1590,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"[X_train, y_train], [X_test, y_test] = keras.datasets.cifar10.load_data()\n", "[X_train, y_train], [X_test, y_test] = tf.keras.datasets.cifar10.load_data()\n",
"X_train = X_train / 255\n", "X_train = X_train / 255\n",
"X_test = X_test / 255" "X_test = X_test / 255"
] ]
@ -1604,12 +1604,12 @@
"tf.random.set_seed(42)\n", "tf.random.set_seed(42)\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"\n", "\n",
"denoising_encoder = keras.models.Sequential([\n", "denoising_encoder = tf.keras.Sequential([\n",
" keras.layers.GaussianNoise(0.1, input_shape=[32, 32, 3]),\n", " tf.keras.layers.GaussianNoise(0.1, input_shape=[32, 32, 3]),\n",
" keras.layers.Conv2D(32, kernel_size=3, padding=\"same\", activation=\"relu\"),\n", " tf.keras.layers.Conv2D(32, kernel_size=3, padding=\"same\", activation=\"relu\"),\n",
" keras.layers.MaxPool2D(),\n", " tf.keras.layers.MaxPool2D(),\n",
" keras.layers.Flatten(),\n", " tf.keras.layers.Flatten(),\n",
" keras.layers.Dense(512, activation=\"relu\"),\n", " tf.keras.layers.Dense(512, activation=\"relu\"),\n",
"])" "])"
] ]
}, },
@ -1628,10 +1628,10 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"denoising_decoder = keras.models.Sequential([\n", "denoising_decoder = tf.keras.Sequential([\n",
" keras.layers.Dense(16 * 16 * 32, activation=\"relu\", input_shape=[512]),\n", " tf.keras.layers.Dense(16 * 16 * 32, activation=\"relu\", input_shape=[512]),\n",
" keras.layers.Reshape([16, 16, 32]),\n", " tf.keras.layers.Reshape([16, 16, 32]),\n",
" keras.layers.Conv2DTranspose(filters=3, kernel_size=3, strides=2,\n", " tf.keras.layers.Conv2DTranspose(filters=3, kernel_size=3, strides=2,\n",
" padding=\"same\", activation=\"sigmoid\")\n", " padding=\"same\", activation=\"sigmoid\")\n",
"])" "])"
] ]
@ -1651,8 +1651,8 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"denoising_ae = keras.models.Sequential([denoising_encoder, denoising_decoder])\n", "denoising_ae = tf.keras.Sequential([denoising_encoder, denoising_decoder])\n",
"denoising_ae.compile(loss=\"binary_crossentropy\", optimizer=keras.optimizers.Nadam(),\n", "denoising_ae.compile(loss=\"binary_crossentropy\", optimizer=tf.keras.optimizers.Nadam(),\n",
" metrics=[\"mse\"])\n", " metrics=[\"mse\"])\n",
"history = denoising_ae.fit(X_train, X_train, epochs=10,\n", "history = denoising_ae.fit(X_train, X_train, epochs=10,\n",
" validation_data=(X_test, X_test))" " validation_data=(X_test, X_test))"

102
17_reinforcement_learning.ipynb
View File

@ -575,15 +575,15 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"keras.backend.clear_session()\n", "tf.keras.backend.clear_session()\n",
"tf.random.set_seed(42)\n", "tf.random.set_seed(42)\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"\n", "\n",
"n_inputs = 4 # == env.observation_space.shape[0]\n", "n_inputs = 4 # == env.observation_space.shape[0]\n",
"\n", "\n",
"model = keras.models.Sequential([\n", "model = tf.keras.Sequential([\n",
" keras.layers.Dense(5, activation=\"elu\", input_shape=[n_inputs]),\n", " tf.keras.layers.Dense(5, activation=\"elu\", input_shape=[n_inputs]),\n",
" keras.layers.Dense(1, activation=\"sigmoid\"),\n", " tf.keras.layers.Dense(1, activation=\"sigmoid\"),\n",
"])" "])"
] ]
}, },
@ -676,8 +676,8 @@
" env.seed(index)\n", " env.seed(index)\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"observations = [env.reset() for env in envs]\n", "observations = [env.reset() for env in envs]\n",
"optimizer = keras.optimizers.RMSprop()\n", "optimizer = tf.keras.optimizers.RMSprop()\n",
"loss_fn = keras.losses.binary_crossentropy\n", "loss_fn = tf.keras.losses.binary_crossentropy\n",
"\n", "\n",
"for iteration in range(n_iterations):\n", "for iteration in range(n_iterations):\n",
" # if angle < 0, we want proba(left) = 1., or else proba(left) = 0.\n", " # if angle < 0, we want proba(left) = 1., or else proba(left) = 0.\n",
@ -874,8 +874,8 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"optimizer = keras.optimizers.Adam(learning_rate=0.01)\n", "optimizer = tf.keras.optimizers.Adam(learning_rate=0.01)\n",
"loss_fn = keras.losses.binary_crossentropy" "loss_fn = tf.keras.losses.binary_crossentropy"
] ]
}, },
{ {
@ -884,13 +884,13 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"keras.backend.clear_session()\n", "tf.keras.backend.clear_session()\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)\n", "tf.random.set_seed(42)\n",
"\n", "\n",
"model = keras.models.Sequential([\n", "model = tf.keras.Sequential([\n",
" keras.layers.Dense(5, activation=\"elu\", input_shape=[4]),\n", " tf.keras.layers.Dense(5, activation=\"elu\", input_shape=[4]),\n",
" keras.layers.Dense(1, activation=\"sigmoid\"),\n", " tf.keras.layers.Dense(1, activation=\"sigmoid\"),\n",
"])" "])"
] ]
}, },
@ -1277,7 +1277,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"keras.backend.clear_session()\n", "tf.keras.backend.clear_session()\n",
"tf.random.set_seed(42)\n", "tf.random.set_seed(42)\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"\n", "\n",
@ -1285,10 +1285,10 @@
"input_shape = [4] # == env.observation_space.shape\n", "input_shape = [4] # == env.observation_space.shape\n",
"n_outputs = 2 # == env.action_space.n\n", "n_outputs = 2 # == env.action_space.n\n",
"\n", "\n",
"model = keras.models.Sequential([\n", "model = tf.keras.Sequential([\n",
" keras.layers.Dense(32, activation=\"elu\", input_shape=input_shape),\n", " tf.keras.layers.Dense(32, activation=\"elu\", input_shape=input_shape),\n",
" keras.layers.Dense(32, activation=\"elu\"),\n", " tf.keras.layers.Dense(32, activation=\"elu\"),\n",
" keras.layers.Dense(n_outputs)\n", " tf.keras.layers.Dense(n_outputs)\n",
"])" "])"
] ]
}, },
@ -1392,8 +1392,8 @@
"source": [ "source": [
"batch_size = 32\n", "batch_size = 32\n",
"discount_rate = 0.95\n", "discount_rate = 0.95\n",
"optimizer = keras.optimizers.Adam(learning_rate=1e-2)\n", "optimizer = tf.keras.optimizers.Adam(learning_rate=1e-2)\n",
"loss_fn = keras.losses.mean_squared_error\n", "loss_fn = tf.keras.losses.mean_squared_error\n",
"\n", "\n",
"def training_step(batch_size):\n", "def training_step(batch_size):\n",
" experiences = sample_experiences(batch_size)\n", " experiences = sample_experiences(batch_size)\n",
@ -1515,17 +1515,17 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"keras.backend.clear_session()\n", "tf.keras.backend.clear_session()\n",
"tf.random.set_seed(42)\n", "tf.random.set_seed(42)\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"\n", "\n",
"model = keras.models.Sequential([\n", "model = tf.keras.Sequential([\n",
" keras.layers.Dense(32, activation=\"elu\", input_shape=[4]),\n", " tf.keras.layers.Dense(32, activation=\"elu\", input_shape=[4]),\n",
" keras.layers.Dense(32, activation=\"elu\"),\n", " tf.keras.layers.Dense(32, activation=\"elu\"),\n",
" keras.layers.Dense(n_outputs)\n", " tf.keras.layers.Dense(n_outputs)\n",
"])\n", "])\n",
"\n", "\n",
"target = keras.models.clone_model(model)\n", "target = tf.keras.models.clone_model(model)\n",
"target.set_weights(model.get_weights())" "target.set_weights(model.get_weights())"
] ]
}, },
@ -1537,8 +1537,8 @@
"source": [ "source": [
"batch_size = 32\n", "batch_size = 32\n",
"discount_rate = 0.95\n", "discount_rate = 0.95\n",
"optimizer = keras.optimizers.Adam(learning_rate=6e-3)\n", "optimizer = tf.keras.optimizers.Adam(learning_rate=6e-3)\n",
"loss_fn = keras.losses.Huber()\n", "loss_fn = tf.keras.losses.Huber()\n",
"\n", "\n",
"def training_step(batch_size):\n", "def training_step(batch_size):\n",
" experiences = sample_experiences(batch_size)\n", " experiences = sample_experiences(batch_size)\n",
@ -1659,21 +1659,21 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"keras.backend.clear_session()\n", "tf.keras.backend.clear_session()\n",
"tf.random.set_seed(42)\n", "tf.random.set_seed(42)\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"\n", "\n",
"K = keras.backend\n", "K = tf.keras.backend\n",
"input_states = keras.layers.Input(shape=[4])\n", "input_states = tf.keras.layers.Input(shape=[4])\n",
"hidden1 = keras.layers.Dense(32, activation=\"elu\")(input_states)\n", "hidden1 = tf.keras.layers.Dense(32, activation=\"elu\")(input_states)\n",
"hidden2 = keras.layers.Dense(32, activation=\"elu\")(hidden1)\n", "hidden2 = tf.keras.layers.Dense(32, activation=\"elu\")(hidden1)\n",
"state_values = keras.layers.Dense(1)(hidden2)\n", "state_values = tf.keras.layers.Dense(1)(hidden2)\n",
"raw_advantages = keras.layers.Dense(n_outputs)(hidden2)\n", "raw_advantages = tf.keras.layers.Dense(n_outputs)(hidden2)\n",
"advantages = raw_advantages - K.max(raw_advantages, axis=1, keepdims=True)\n", "advantages = raw_advantages - K.max(raw_advantages, axis=1, keepdims=True)\n",
"Q_values = state_values + advantages\n", "Q_values = state_values + advantages\n",
"model = keras.models.Model(inputs=[input_states], outputs=[Q_values])\n", "model = tf.keras.Model(inputs=[input_states], outputs=[Q_values])\n",
"\n", "\n",
"target = keras.models.clone_model(model)\n", "target = tf.keras.models.clone_model(model)\n",
"target.set_weights(model.get_weights())" "target.set_weights(model.get_weights())"
] ]
}, },
@ -1685,8 +1685,8 @@
"source": [ "source": [
"batch_size = 32\n", "batch_size = 32\n",
"discount_rate = 0.95\n", "discount_rate = 0.95\n",
"optimizer = keras.optimizers.Adam(learning_rate=7.5e-3)\n", "optimizer = tf.keras.optimizers.Adam(learning_rate=7.5e-3)\n",
"loss_fn = keras.losses.Huber()\n", "loss_fn = tf.keras.losses.Huber()\n",
"\n", "\n",
"def training_step(batch_size):\n", "def training_step(batch_size):\n",
" experiences = sample_experiences(batch_size)\n", " experiences = sample_experiences(batch_size)\n",
@ -2180,7 +2180,7 @@
"source": [ "source": [
"from tf_agents.networks.q_network import QNetwork\n", "from tf_agents.networks.q_network import QNetwork\n",
"\n", "\n",
"preprocessing_layer = keras.layers.Lambda(\n", "preprocessing_layer = tf.keras.layers.Lambda(\n",
" lambda obs: tf.cast(obs, np.float32) / 255.)\n", " lambda obs: tf.cast(obs, np.float32) / 255.)\n",
"conv_layer_params=[(32, (8, 8), 4), (64, (4, 4), 2), (64, (3, 3), 1)]\n", "conv_layer_params=[(32, (8, 8), 4), (64, (4, 4), 2), (64, (3, 3), 1)]\n",
"fc_layer_params=[512]\n", "fc_layer_params=[512]\n",
@ -2210,9 +2210,9 @@
"\n", "\n",
"train_step = tf.Variable(0)\n", "train_step = tf.Variable(0)\n",
"update_period = 4 # run a training step every 4 collect steps\n", "update_period = 4 # run a training step every 4 collect steps\n",
"optimizer = keras.optimizers.RMSprop(learning_rate=2.5e-4, rho=0.95, momentum=0.0,\n", "optimizer = tf.keras.optimizers.RMSprop(learning_rate=2.5e-4, rho=0.95, momentum=0.0,\n",
" epsilon=0.00001, centered=True)\n", " epsilon=0.00001, centered=True)\n",
"epsilon_fn = keras.optimizers.schedules.PolynomialDecay(\n", "epsilon_fn = tf.keras.optimizers.schedules.PolynomialDecay(\n",
" initial_learning_rate=1.0, # initial ε\n", " initial_learning_rate=1.0, # initial ε\n",
" decay_steps=250000 // update_period, # <=> 1,000,000 ALE frames\n", " decay_steps=250000 // update_period, # <=> 1,000,000 ALE frames\n",
" end_learning_rate=0.01) # final ε\n", " end_learning_rate=0.01) # final ε\n",
@ -2221,7 +2221,7 @@
" q_network=q_net,\n", " q_network=q_net,\n",
" optimizer=optimizer,\n", " optimizer=optimizer,\n",
" target_update_period=2000, # <=> 32,000 ALE frames\n", " target_update_period=2000, # <=> 32,000 ALE frames\n",
" td_errors_loss_fn=keras.losses.Huber(reduction=\"none\"),\n", " td_errors_loss_fn=tf.keras.losses.Huber(reduction=\"none\"),\n",
" gamma=0.99, # discount factor\n", " gamma=0.99, # discount factor\n",
" train_step_counter=train_step,\n", " train_step_counter=train_step,\n",
" epsilon_greedy=lambda: epsilon_fn(train_step))\n", " epsilon_greedy=lambda: epsilon_fn(train_step))\n",
@ -2911,17 +2911,17 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"keras.backend.clear_session()\n", "tf.keras.backend.clear_session()\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)\n", "tf.random.set_seed(42)\n",
"\n", "\n",
"n_inputs = env.observation_space.shape[0]\n", "n_inputs = env.observation_space.shape[0]\n",
"n_outputs = env.action_space.n\n", "n_outputs = env.action_space.n\n",
"\n", "\n",
"model = keras.models.Sequential([\n", "model = tf.keras.Sequential([\n",
" keras.layers.Dense(32, activation=\"relu\", input_shape=[n_inputs]),\n", " tf.keras.layers.Dense(32, activation=\"relu\", input_shape=[n_inputs]),\n",
" keras.layers.Dense(32, activation=\"relu\"),\n", " tf.keras.layers.Dense(32, activation=\"relu\"),\n",
" keras.layers.Dense(n_outputs, activation=\"softmax\"),\n", " tf.keras.layers.Dense(n_outputs, activation=\"softmax\"),\n",
"])" "])"
] ]
}, },
@ -2949,7 +2949,7 @@
"def lander_play_one_step(env, obs, model, loss_fn):\n", "def lander_play_one_step(env, obs, model, loss_fn):\n",
" with tf.GradientTape() as tape:\n", " with tf.GradientTape() as tape:\n",
" probas = model(obs[np.newaxis])\n", " probas = model(obs[np.newaxis])\n",
" logits = tf.math.log(probas + keras.backend.epsilon())\n", " logits = tf.math.log(probas + tf.keras.backend.epsilon())\n",
" action = tf.random.categorical(logits, num_samples=1)\n", " action = tf.random.categorical(logits, num_samples=1)\n",
" loss = tf.reduce_mean(loss_fn(action, probas))\n", " loss = tf.reduce_mean(loss_fn(action, probas))\n",
" grads = tape.gradient(loss, model.trainable_variables)\n", " grads = tape.gradient(loss, model.trainable_variables)\n",
@ -3035,8 +3035,8 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"optimizer = keras.optimizers.Nadam(learning_rate=0.005)\n", "optimizer = tf.keras.optimizers.Nadam(learning_rate=0.005)\n",
"loss_fn = keras.losses.sparse_categorical_crossentropy" "loss_fn = tf.keras.losses.sparse_categorical_crossentropy"
] ]
}, },
{ {
@ -3120,7 +3120,7 @@
" for step in range(n_max_steps):\n", " for step in range(n_max_steps):\n",
" frames.append(env.render(mode=\"rgb_array\"))\n", " frames.append(env.render(mode=\"rgb_array\"))\n",
" probas = model(obs[np.newaxis])\n", " probas = model(obs[np.newaxis])\n",
" logits = tf.math.log(probas + keras.backend.epsilon())\n", " logits = tf.math.log(probas + tf.keras.backend.epsilon())\n",
" action = tf.random.categorical(logits, num_samples=1)\n", " action = tf.random.categorical(logits, num_samples=1)\n",
" obs, reward, done, info = env.step(action[0, 0].numpy())\n", " obs, reward, done, info = env.step(action[0, 0].numpy())\n",
" if done:\n", " if done:\n",

92
18_training_and_deploying_at_scale.ipynb
View File

@ -127,7 +127,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"(X_train_full, y_train_full), (X_test, y_test) = keras.datasets.mnist.load_data()\n", "(X_train_full, y_train_full), (X_test, y_test) = tf.keras.datasets.mnist.load_data()\n",
"X_train_full = X_train_full[..., np.newaxis].astype(np.float32) / 255.\n", "X_train_full = X_train_full[..., np.newaxis].astype(np.float32) / 255.\n",
"X_test = X_test[..., np.newaxis].astype(np.float32) / 255.\n", "X_test = X_test[..., np.newaxis].astype(np.float32) / 255.\n",
"X_valid, X_train = X_train_full[:5000], X_train_full[5000:]\n", "X_valid, X_train = X_train_full[:5000], X_train_full[5000:]\n",
@ -144,13 +144,13 @@
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)\n", "tf.random.set_seed(42)\n",
"\n", "\n",
"model = keras.models.Sequential([\n", "model = tf.keras.Sequential([\n",
" keras.layers.Flatten(input_shape=[28, 28, 1]),\n", " tf.keras.layers.Flatten(input_shape=[28, 28, 1]),\n",
" keras.layers.Dense(100, activation=\"relu\"),\n", " tf.keras.layers.Dense(100, activation=\"relu\"),\n",
" keras.layers.Dense(10, activation=\"softmax\")\n", " tf.keras.layers.Dense(10, activation=\"softmax\")\n",
"])\n", "])\n",
"model.compile(loss=\"sparse_categorical_crossentropy\",\n", "model.compile(loss=\"sparse_categorical_crossentropy\",\n",
" optimizer=keras.optimizers.SGD(learning_rate=1e-2),\n", " optimizer=tf.keras.optimizers.SGD(learning_rate=1e-2),\n",
" metrics=[\"accuracy\"])\n", " metrics=[\"accuracy\"])\n",
"model.fit(X_train, y_train, epochs=10, validation_data=(X_valid, y_valid))" "model.fit(X_train, y_train, epochs=10, validation_data=(X_valid, y_valid))"
] ]
@ -560,14 +560,14 @@
"np.random.seed(42)\n", "np.random.seed(42)\n",
"tf.random.set_seed(42)\n", "tf.random.set_seed(42)\n",
"\n", "\n",
"model = keras.models.Sequential([\n", "model = tf.keras.Sequential([\n",
" keras.layers.Flatten(input_shape=[28, 28, 1]),\n", " tf.keras.layers.Flatten(input_shape=[28, 28, 1]),\n",
" keras.layers.Dense(50, activation=\"relu\"),\n", " tf.keras.layers.Dense(50, activation=\"relu\"),\n",
" keras.layers.Dense(50, activation=\"relu\"),\n", " tf.keras.layers.Dense(50, activation=\"relu\"),\n",
" keras.layers.Dense(10, activation=\"softmax\")\n", " tf.keras.layers.Dense(10, activation=\"softmax\")\n",
"])\n", "])\n",
"model.compile(loss=\"sparse_categorical_crossentropy\",\n", "model.compile(loss=\"sparse_categorical_crossentropy\",\n",
" optimizer=keras.optimizers.SGD(learning_rate=1e-2),\n", " optimizer=tf.keras.optimizers.SGD(learning_rate=1e-2),\n",
" metrics=[\"accuracy\"])\n", " metrics=[\"accuracy\"])\n",
"history = model.fit(X_train, y_train, epochs=10, validation_data=(X_valid, y_valid))" "history = model.fit(X_train, y_train, epochs=10, validation_data=(X_valid, y_valid))"
] ]
@ -774,7 +774,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"keras.backend.clear_session()\n", "tf.keras.backend.clear_session()\n",
"tf.random.set_seed(42)\n", "tf.random.set_seed(42)\n",
"np.random.seed(42)" "np.random.seed(42)"
] ]
@ -786,19 +786,19 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"def create_model():\n", "def create_model():\n",
" return keras.models.Sequential([\n", " return tf.keras.Sequential([\n",
" keras.layers.Conv2D(filters=64, kernel_size=7, activation=\"relu\",\n", " tf.keras.layers.Conv2D(filters=64, kernel_size=7, activation=\"relu\",\n",
" padding=\"same\", input_shape=[28, 28, 1]),\n", " padding=\"same\", input_shape=[28, 28, 1]),\n",
" keras.layers.MaxPooling2D(pool_size=2),\n", " tf.keras.layers.MaxPooling2D(pool_size=2),\n",
" keras.layers.Conv2D(filters=128, kernel_size=3, activation=\"relu\",\n", " tf.keras.layers.Conv2D(filters=128, kernel_size=3, activation=\"relu\",\n",
" padding=\"same\"), \n", " padding=\"same\"), \n",
" keras.layers.Conv2D(filters=128, kernel_size=3, activation=\"relu\",\n", " tf.keras.layers.Conv2D(filters=128, kernel_size=3, activation=\"relu\",\n",
" padding=\"same\"),\n", " padding=\"same\"),\n",
" keras.layers.MaxPooling2D(pool_size=2),\n", " tf.keras.layers.MaxPooling2D(pool_size=2),\n",
" keras.layers.Flatten(),\n", " tf.keras.layers.Flatten(),\n",
" keras.layers.Dense(units=64, activation='relu'),\n", " tf.keras.layers.Dense(units=64, activation='relu'),\n",
" keras.layers.Dropout(0.5),\n", " tf.keras.layers.Dropout(0.5),\n",
" keras.layers.Dense(units=10, activation='softmax'),\n", " tf.keras.layers.Dense(units=10, activation='softmax'),\n",
" ])" " ])"
] ]
}, },
@ -811,7 +811,7 @@
"batch_size = 100\n", "batch_size = 100\n",
"model = create_model()\n", "model = create_model()\n",
"model.compile(loss=\"sparse_categorical_crossentropy\",\n", "model.compile(loss=\"sparse_categorical_crossentropy\",\n",
" optimizer=keras.optimizers.SGD(learning_rate=1e-2),\n", " optimizer=tf.keras.optimizers.SGD(learning_rate=1e-2),\n",
" metrics=[\"accuracy\"])\n", " metrics=[\"accuracy\"])\n",
"model.fit(X_train, y_train, epochs=10,\n", "model.fit(X_train, y_train, epochs=10,\n",
" validation_data=(X_valid, y_valid), batch_size=batch_size)" " validation_data=(X_valid, y_valid), batch_size=batch_size)"
@ -823,7 +823,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"keras.backend.clear_session()\n", "tf.keras.backend.clear_session()\n",
"tf.random.set_seed(42)\n", "tf.random.set_seed(42)\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"\n", "\n",
@ -851,7 +851,7 @@
"with distribution.scope():\n", "with distribution.scope():\n",
" model = create_model()\n", " model = create_model()\n",
" model.compile(loss=\"sparse_categorical_crossentropy\",\n", " model.compile(loss=\"sparse_categorical_crossentropy\",\n",
" optimizer=keras.optimizers.SGD(learning_rate=1e-2),\n", " optimizer=tf.keras.optimizers.SGD(learning_rate=1e-2),\n",
" metrics=[\"accuracy\"])" " metrics=[\"accuracy\"])"
] ]
}, },
@ -888,17 +888,17 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"keras.backend.clear_session()\n", "tf.keras.backend.clear_session()\n",
"tf.random.set_seed(42)\n", "tf.random.set_seed(42)\n",
"np.random.seed(42)\n", "np.random.seed(42)\n",
"\n", "\n",
"K = keras.backend\n", "K = tf.keras.backend\n",
"\n", "\n",
"distribution = tf.distribute.MirroredStrategy()\n", "distribution = tf.distribute.MirroredStrategy()\n",
"\n", "\n",
"with distribution.scope():\n", "with distribution.scope():\n",
" model = create_model()\n", " model = create_model()\n",
" optimizer = keras.optimizers.SGD()\n", " optimizer = tf.keras.optimizers.SGD()\n",
"\n", "\n",
"with distribution.scope():\n", "with distribution.scope():\n",
" dataset = tf.data.Dataset.from_tensor_slices((X_train, y_train)).repeat().batch(batch_size)\n", " dataset = tf.data.Dataset.from_tensor_slices((X_train, y_train)).repeat().batch(batch_size)\n",
@ -910,7 +910,7 @@
" X, y = inputs\n", " X, y = inputs\n",
" with tf.GradientTape() as tape:\n", " with tf.GradientTape() as tape:\n",
" Y_proba = model(X)\n", " Y_proba = model(X)\n",
" loss = K.sum(keras.losses.sparse_categorical_crossentropy(y, Y_proba)) / batch_size\n", " loss = K.sum(tf.keras.losses.sparse_categorical_crossentropy(y, Y_proba)) / batch_size\n",
"\n", "\n",
" grads = tape.gradient(loss, model.trainable_variables)\n", " grads = tape.gradient(loss, model.trainable_variables)\n",
" optimizer.apply_gradients(zip(grads, model.trainable_variables))\n", " optimizer.apply_gradients(zip(grads, model.trainable_variables))\n",
@ -1072,36 +1072,36 @@
" run_id = time.strftime(\"run_%Y_%m_%d-%H_%M_%S\")\n", " run_id = time.strftime(\"run_%Y_%m_%d-%H_%M_%S\")\n",
" run_dir = root_logdir / run_id\n", " run_dir = root_logdir / run_id\n",
" callbacks = [\n", " callbacks = [\n",
" keras.callbacks.TensorBoard(run_dir),\n", " tf.keras.callbacks.TensorBoard(run_dir),\n",
" keras.callbacks.ModelCheckpoint(\"my_mnist_multiworker_model.h5\",\n", " tf.keras.callbacks.ModelCheckpoint(\"my_mnist_multiworker_model.h5\",\n",
" save_best_only=True),\n", " save_best_only=True),\n",
" ]\n", " ]\n",
"else:\n", "else:\n",
" callbacks = []\n", " callbacks = []\n",
"\n", "\n",
"# Load and prepare the MNIST dataset\n", "# Load and prepare the MNIST dataset\n",
"(X_train_full, y_train_full), (X_test, y_test) = keras.datasets.mnist.load_data()\n", "(X_train_full, y_train_full), (X_test, y_test) = tf.keras.datasets.mnist.load_data()\n",
"X_train_full = X_train_full[..., np.newaxis] / 255.\n", "X_train_full = X_train_full[..., np.newaxis] / 255.\n",
"X_valid, X_train = X_train_full[:5000], X_train_full[5000:]\n", "X_valid, X_train = X_train_full[:5000], X_train_full[5000:]\n",
"y_valid, y_train = y_train_full[:5000], y_train_full[5000:]\n", "y_valid, y_train = y_train_full[:5000], y_train_full[5000:]\n",
"\n", "\n",
"with distribution.scope():\n", "with distribution.scope():\n",
" model = keras.models.Sequential([\n", " model = tf.keras.Sequential([\n",
" keras.layers.Conv2D(filters=64, kernel_size=7, activation=\"relu\",\n", " tf.keras.layers.Conv2D(filters=64, kernel_size=7, activation=\"relu\",\n",
" padding=\"same\", input_shape=[28, 28, 1]),\n", " padding=\"same\", input_shape=[28, 28, 1]),\n",
" keras.layers.MaxPooling2D(pool_size=2),\n", " tf.keras.layers.MaxPooling2D(pool_size=2),\n",
" keras.layers.Conv2D(filters=128, kernel_size=3, activation=\"relu\",\n", " tf.keras.layers.Conv2D(filters=128, kernel_size=3, activation=\"relu\",\n",
" padding=\"same\"), \n", " padding=\"same\"), \n",
" keras.layers.Conv2D(filters=128, kernel_size=3, activation=\"relu\",\n", " tf.keras.layers.Conv2D(filters=128, kernel_size=3, activation=\"relu\",\n",
" padding=\"same\"),\n", " padding=\"same\"),\n",
" keras.layers.MaxPooling2D(pool_size=2),\n", " tf.keras.layers.MaxPooling2D(pool_size=2),\n",
" keras.layers.Flatten(),\n", " tf.keras.layers.Flatten(),\n",
" keras.layers.Dense(units=64, activation='relu'),\n", " tf.keras.layers.Dense(units=64, activation='relu'),\n",
" keras.layers.Dropout(0.5),\n", " tf.keras.layers.Dropout(0.5),\n",
" keras.layers.Dense(units=10, activation='softmax'),\n", " tf.keras.layers.Dense(units=10, activation='softmax'),\n",
" ])\n", " ])\n",
" model.compile(loss=\"sparse_categorical_crossentropy\",\n", " model.compile(loss=\"sparse_categorical_crossentropy\",\n",
" optimizer=keras.optimizers.SGD(learning_rate=1e-2),\n", " optimizer=tf.keras.optimizers.SGD(learning_rate=1e-2),\n",
" metrics=[\"accuracy\"])\n", " metrics=[\"accuracy\"])\n",
"\n", "\n",
"model.fit(X_train, y_train, validation_data=(X_valid, y_valid),\n", "model.fit(X_train, y_train, validation_data=(X_valid, y_valid),\n",
@ -1172,7 +1172,7 @@
"cell_type": "markdown", "cell_type": "markdown",
"metadata": {}, "metadata": {},
"source": [ "source": [
"That's it! Once training is over, the best checkpoint of the model will be available in the `my_mnist_multiworker_model.h5` file. You can load it using `keras.models.load_model()` and use it for predictions, as usual:" "That's it! Once training is over, the best checkpoint of the model will be available in the `my_mnist_multiworker_model.h5` file. You can load it using `tf.keras.models.load_model()` and use it for predictions, as usual:"
] ]
}, },
{ {
@ -1183,7 +1183,7 @@
"source": [ "source": [
"from tensorflow import keras\n", "from tensorflow import keras\n",
"\n", "\n",
"model = keras.models.load_model(\"my_mnist_multiworker_model.h5\")\n", "model = tf.keras.models.load_model(\"my_mnist_multiworker_model.h5\")\n",
"Y_pred = model.predict(X_new)\n", "Y_pred = model.predict(X_new)\n",
"np.argmax(Y_pred, axis=-1)" "np.argmax(Y_pred, axis=-1)"
] ]