schuetoliver
/
handson-ml
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Replace lr with learning_rate in Keras optimizers, fixes #456

main
Aurélien Geron 2021-08-31 20:54:35 +12:00
parent 1568ac3b94
commit 108fe1fa53
10 changed files with 94 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
10_neural_nets_with_keras.ipynb
View File

@ -843,7 +843,7 @@
" keras.layers.Dense(30, activation=\"relu\", input_shape=X_train.shape[1:]),\n",
" keras.layers.Dense(1)\n",
"])\n",
"model.compile(loss=\"mean_squared_error\", optimizer=keras.optimizers.SGD(lr=1e-3))\n",
"model.compile(loss=\"mean_squared_error\", optimizer=keras.optimizers.SGD(learning_rate=1e-3))\n",
"history = model.fit(X_train, y_train, epochs=20, validation_data=(X_valid, y_valid))\n",
"mse_test = model.evaluate(X_test, y_test)\n",
"X_new = X_test[:3]\n",
@ -924,7 +924,7 @@
"metadata": {},
"outputs": [],
"source": [
"model.compile(loss=\"mean_squared_error\", optimizer=keras.optimizers.SGD(lr=1e-3))\n",
"model.compile(loss=\"mean_squared_error\", optimizer=keras.optimizers.SGD(learning_rate=1e-3))\n",
"history = model.fit(X_train, y_train, epochs=20,\n",
" validation_data=(X_valid, y_valid))\n",
"mse_test = model.evaluate(X_test, y_test)\n",
@ -969,7 +969,7 @@
"metadata": {},
"outputs": [],
"source": [
"model.compile(loss=\"mse\", optimizer=keras.optimizers.SGD(lr=1e-3))\n",
"model.compile(loss=\"mse\", optimizer=keras.optimizers.SGD(learning_rate=1e-3))\n",
"\n",
"X_train_A, X_train_B = X_train[:, :5], X_train[:, 2:]\n",
"X_valid_A, X_valid_B = X_valid[:, :5], X_valid[:, 2:]\n",
@ -1022,7 +1022,7 @@
"metadata": {},
"outputs": [],
"source": [
"model.compile(loss=[\"mse\", \"mse\"], loss_weights=[0.9, 0.1], optimizer=keras.optimizers.SGD(lr=1e-3))"
"model.compile(loss=[\"mse\", \"mse\"], loss_weights=[0.9, 0.1], optimizer=keras.optimizers.SGD(learning_rate=1e-3))"
]
},
{
@ -1085,7 +1085,7 @@
"metadata": {},
"outputs": [],
"source": [
"model.compile(loss=\"mse\", loss_weights=[0.9, 0.1], optimizer=keras.optimizers.SGD(lr=1e-3))\n",
"model.compile(loss=\"mse\", loss_weights=[0.9, 0.1], optimizer=keras.optimizers.SGD(learning_rate=1e-3))\n",
"history = model.fit((X_train_A, X_train_B), (y_train, y_train), epochs=10,\n",
" validation_data=((X_valid_A, X_valid_B), (y_valid, y_valid)))\n",
"total_loss, main_loss, aux_loss = model.evaluate((X_test_A, X_test_B), (y_test, y_test))\n",
@ -1128,7 +1128,7 @@
"metadata": {},
"outputs": [],
"source": [
"model.compile(loss=\"mse\", optimizer=keras.optimizers.SGD(lr=1e-3))\n",
"model.compile(loss=\"mse\", optimizer=keras.optimizers.SGD(learning_rate=1e-3))\n",
"history = model.fit(X_train, y_train, epochs=10, validation_data=(X_valid, y_valid))\n",
"mse_test = model.evaluate(X_test, y_test)"
]
@ -1215,7 +1215,7 @@
"metadata": {},
"outputs": [],
"source": [
"model.compile(loss=\"mse\", optimizer=keras.optimizers.SGD(lr=1e-3))\n",
"model.compile(loss=\"mse\", optimizer=keras.optimizers.SGD(learning_rate=1e-3))\n",
"checkpoint_cb = keras.callbacks.ModelCheckpoint(\"my_keras_model.h5\", save_best_only=True)\n",
"history = model.fit(X_train, y_train, epochs=10,\n",
" validation_data=(X_valid, y_valid),\n",
@ -1230,7 +1230,7 @@
"metadata": {},
"outputs": [],
"source": [
"model.compile(loss=\"mse\", optimizer=keras.optimizers.SGD(lr=1e-3))\n",
"model.compile(loss=\"mse\", optimizer=keras.optimizers.SGD(learning_rate=1e-3))\n",
"early_stopping_cb = keras.callbacks.EarlyStopping(patience=10,\n",
" restore_best_weights=True)\n",
"history = model.fit(X_train, y_train, epochs=100,\n",
@ -1315,7 +1315,7 @@
" keras.layers.Dense(30, activation=\"relu\"),\n",
" keras.layers.Dense(1)\n",
"]) \n",
"model.compile(loss=\"mse\", optimizer=keras.optimizers.SGD(lr=1e-3))"
"model.compile(loss=\"mse\", optimizer=keras.optimizers.SGD(learning_rate=1e-3))"
]
},
{
@ -1387,7 +1387,7 @@
" keras.layers.Dense(30, activation=\"relu\"),\n",
" keras.layers.Dense(1)\n",
"]) \n",
"model.compile(loss=\"mse\", optimizer=keras.optimizers.SGD(lr=0.05))"
"model.compile(loss=\"mse\", optimizer=keras.optimizers.SGD(learning_rate=0.05))"
]
},
{
@ -1455,7 +1455,7 @@
" for layer in range(n_hidden):\n",
" model.add(keras.layers.Dense(n_neurons, activation=\"relu\"))\n",
" model.add(keras.layers.Dense(1))\n",
" optimizer = keras.optimizers.SGD(lr=learning_rate)\n",
" optimizer = keras.optimizers.SGD(learning_rate=learning_rate)\n",
" model.compile(loss=\"mse\", optimizer=optimizer)\n",
" return model"
]
@ -1802,9 +1802,9 @@
" self.rates = []\n",
" self.losses = []\n",
" def on_batch_end(self, batch, logs):\n",
" self.rates.append(K.get_value(self.model.optimizer.lr))\n",
" self.rates.append(K.get_value(self.model.optimizer.learning_rate))\n",
" self.losses.append(logs[\"loss\"])\n",
" K.set_value(self.model.optimizer.lr, self.model.optimizer.lr * self.factor)"
" K.set_value(self.model.optimizer.learning_rate, self.model.optimizer.learning_rate * self.factor)"
]
},
{
@ -1846,7 +1846,7 @@
"outputs": [],
"source": [
"model.compile(loss=\"sparse_categorical_crossentropy\",\n",
" optimizer=keras.optimizers.SGD(lr=1e-3),\n",
" optimizer=keras.optimizers.SGD(learning_rate=1e-3),\n",
" metrics=[\"accuracy\"])\n",
"expon_lr = ExponentialLearningRate(factor=1.005)"
]
@ -1930,7 +1930,7 @@
"outputs": [],
"source": [
"model.compile(loss=\"sparse_categorical_crossentropy\",\n",
" optimizer=keras.optimizers.SGD(lr=3e-1),\n",
" optimizer=keras.optimizers.SGD(learning_rate=3e-1),\n",
" metrics=[\"accuracy\"])"
]
},

80
11_training_deep_neural_networks.ipynb
View File

@ -283,7 +283,7 @@
"outputs": [],
"source": [
"model.compile(loss=\"sparse_categorical_crossentropy\",\n",
" optimizer=keras.optimizers.SGD(lr=1e-3),\n",
" optimizer=keras.optimizers.SGD(learning_rate=1e-3),\n",
" metrics=[\"accuracy\"])"
]
},
@ -332,7 +332,7 @@
"outputs": [],
"source": [
"model.compile(loss=\"sparse_categorical_crossentropy\",\n",
" optimizer=keras.optimizers.SGD(lr=1e-3),\n",
" optimizer=keras.optimizers.SGD(learning_rate=1e-3),\n",
" metrics=[\"accuracy\"])"
]
},
@ -534,7 +534,7 @@
"outputs": [],
"source": [
"model.compile(loss=\"sparse_categorical_crossentropy\",\n",
" optimizer=keras.optimizers.SGD(lr=1e-3),\n",
" optimizer=keras.optimizers.SGD(learning_rate=1e-3),\n",
" metrics=[\"accuracy\"])"
]
},
@ -606,7 +606,7 @@
"outputs": [],
"source": [
"model.compile(loss=\"sparse_categorical_crossentropy\",\n",
" optimizer=keras.optimizers.SGD(lr=1e-3),\n",
" optimizer=keras.optimizers.SGD(learning_rate=1e-3),\n",
" metrics=[\"accuracy\"])"
]
},
@ -686,7 +686,7 @@
"outputs": [],
"source": [
"model.compile(loss=\"sparse_categorical_crossentropy\",\n",
" optimizer=keras.optimizers.SGD(lr=1e-3),\n",
" optimizer=keras.optimizers.SGD(learning_rate=1e-3),\n",
" metrics=[\"accuracy\"])"
]
},
@ -733,7 +733,7 @@
"outputs": [],
"source": [
"model.compile(loss=\"sparse_categorical_crossentropy\",\n",
" optimizer=keras.optimizers.SGD(lr=1e-3),\n",
" optimizer=keras.optimizers.SGD(learning_rate=1e-3),\n",
" metrics=[\"accuracy\"])"
]
},
@ -893,7 +893,7 @@
"outputs": [],
"source": [
"model_A.compile(loss=\"sparse_categorical_crossentropy\",\n",
" optimizer=keras.optimizers.SGD(lr=1e-3),\n",
" optimizer=keras.optimizers.SGD(learning_rate=1e-3),\n",
" metrics=[\"accuracy\"])"
]
},
@ -936,7 +936,7 @@
"outputs": [],
"source": [
"model_B.compile(loss=\"binary_crossentropy\",\n",
" optimizer=keras.optimizers.SGD(lr=1e-3),\n",
" optimizer=keras.optimizers.SGD(learning_rate=1e-3),\n",
" metrics=[\"accuracy\"])"
]
},
@ -990,7 +990,7 @@
" layer.trainable = False\n",
"\n",
"model_B_on_A.compile(loss=\"binary_crossentropy\",\n",
" optimizer=keras.optimizers.SGD(lr=1e-3),\n",
" optimizer=keras.optimizers.SGD(learning_rate=1e-3),\n",
" metrics=[\"accuracy\"])"
]
},
@ -1007,7 +1007,7 @@
" layer.trainable = True\n",
"\n",
"model_B_on_A.compile(loss=\"binary_crossentropy\",\n",
" optimizer=keras.optimizers.SGD(lr=1e-3),\n",
" optimizer=keras.optimizers.SGD(learning_rate=1e-3),\n",
" metrics=[\"accuracy\"])\n",
"history = model_B_on_A.fit(X_train_B, y_train_B, epochs=16,\n",
" validation_data=(X_valid_B, y_valid_B))"
@ -1074,7 +1074,7 @@
"metadata": {},
"outputs": [],
"source": [
"optimizer = keras.optimizers.SGD(lr=0.001, momentum=0.9)"
"optimizer = keras.optimizers.SGD(learning_rate=0.001, momentum=0.9)"
]
},
{
@ -1090,7 +1090,7 @@
"metadata": {},
"outputs": [],
"source": [
"optimizer = keras.optimizers.SGD(lr=0.001, momentum=0.9, nesterov=True)"
"optimizer = keras.optimizers.SGD(learning_rate=0.001, momentum=0.9, nesterov=True)"
]
},
{
@ -1106,7 +1106,7 @@
"metadata": {},
"outputs": [],
"source": [
"optimizer = keras.optimizers.Adagrad(lr=0.001)"
"optimizer = keras.optimizers.Adagrad(learning_rate=0.001)"
]
},
{
@ -1122,7 +1122,7 @@
"metadata": {},
"outputs": [],
"source": [
"optimizer = keras.optimizers.RMSprop(lr=0.001, rho=0.9)"
"optimizer = keras.optimizers.RMSprop(learning_rate=0.001, rho=0.9)"
]
},
{
@ -1138,7 +1138,7 @@
"metadata": {},
"outputs": [],
"source": [
"optimizer = keras.optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999)"
"optimizer = keras.optimizers.Adam(learning_rate=0.001, beta_1=0.9, beta_2=0.999)"
]
},
{
@ -1154,7 +1154,7 @@
"metadata": {},
"outputs": [],
"source": [
"optimizer = keras.optimizers.Adamax(lr=0.001, beta_1=0.9, beta_2=0.999)"
"optimizer = keras.optimizers.Adamax(learning_rate=0.001, beta_1=0.9, beta_2=0.999)"
]
},
{
@ -1170,7 +1170,7 @@
"metadata": {},
"outputs": [],
"source": [
"optimizer = keras.optimizers.Nadam(lr=0.001, beta_1=0.9, beta_2=0.999)"
"optimizer = keras.optimizers.Nadam(learning_rate=0.001, beta_1=0.9, beta_2=0.999)"
]
},
{
@ -1201,7 +1201,7 @@
"metadata": {},
"outputs": [],
"source": [
"optimizer = keras.optimizers.SGD(lr=0.01, decay=1e-4)"
"optimizer = keras.optimizers.SGD(learning_rate=0.01, decay=1e-4)"
]
},
{
@ -1374,12 +1374,12 @@
"\n",
" def on_batch_begin(self, batch, logs=None):\n",
" # Note: the `batch` argument is reset at each epoch\n",
" lr = K.get_value(self.model.optimizer.lr)\n",
" K.set_value(self.model.optimizer.lr, lr * 0.1**(1 / s))\n",
" lr = K.get_value(self.model.optimizer.learning_rate)\n",
" K.set_value(self.model.optimizer.learning_rate, lr * 0.1**(1 / s))\n",
"\n",
" def on_epoch_end(self, epoch, logs=None):\n",
" logs = logs or {}\n",
" logs['lr'] = K.get_value(self.model.optimizer.lr)\n",
" logs['lr'] = K.get_value(self.model.optimizer.learning_rate)\n",
"\n",
"model = keras.models.Sequential([\n",
" keras.layers.Flatten(input_shape=[28, 28]),\n",
@ -1388,7 +1388,7 @@
" keras.layers.Dense(10, activation=\"softmax\")\n",
"])\n",
"lr0 = 0.01\n",
"optimizer = keras.optimizers.Nadam(lr=lr0)\n",
"optimizer = keras.optimizers.Nadam(learning_rate=lr0)\n",
"model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=optimizer, metrics=[\"accuracy\"])\n",
"n_epochs = 25\n",
"\n",
@ -1532,7 +1532,7 @@
" keras.layers.Dense(100, activation=\"selu\", kernel_initializer=\"lecun_normal\"),\n",
" keras.layers.Dense(10, activation=\"softmax\")\n",
"])\n",
"optimizer = keras.optimizers.SGD(lr=0.02, momentum=0.9)\n",
"optimizer = keras.optimizers.SGD(learning_rate=0.02, momentum=0.9)\n",
"model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=optimizer, metrics=[\"accuracy\"])\n",
"n_epochs = 25\n",
"history = model.fit(X_train_scaled, y_train, epochs=n_epochs,\n",
@ -1629,20 +1629,20 @@
" self.rates = []\n",
" self.losses = []\n",
" def on_batch_end(self, batch, logs):\n",
" self.rates.append(K.get_value(self.model.optimizer.lr))\n",
" self.rates.append(K.get_value(self.model.optimizer.learning_rate))\n",
" self.losses.append(logs[\"loss\"])\n",
" K.set_value(self.model.optimizer.lr, self.model.optimizer.lr * self.factor)\n",
" K.set_value(self.model.optimizer.learning_rate, self.model.optimizer.learning_rate * self.factor)\n",
"\n",
"def find_learning_rate(model, X, y, epochs=1, batch_size=32, min_rate=10**-5, max_rate=10):\n",
" init_weights = model.get_weights()\n",
" iterations = math.ceil(len(X) / batch_size) * epochs\n",
" factor = np.exp(np.log(max_rate / min_rate) / iterations)\n",
" init_lr = K.get_value(model.optimizer.lr)\n",
" K.set_value(model.optimizer.lr, min_rate)\n",
" init_lr = K.get_value(model.optimizer.learning_rate)\n",
" K.set_value(model.optimizer.learning_rate, min_rate)\n",
" exp_lr = ExponentialLearningRate(factor)\n",
" history = model.fit(X, y, epochs=epochs, batch_size=batch_size,\n",
" callbacks=[exp_lr])\n",
" K.set_value(model.optimizer.lr, init_lr)\n",
" K.set_value(model.optimizer.learning_rate, init_lr)\n",
" model.set_weights(init_weights)\n",
" return exp_lr.rates, exp_lr.losses\n",
"\n",
@ -1672,9 +1672,9 @@
" def on_batch_end(self, batch, logs=None):\n",
" batch_loss = logs[\"loss\"] * (batch + 1) - self.prev_loss * batch\n",
" self.prev_loss = logs[\"loss\"]\n",
" self.rates.append(K.get_value(self.model.optimizer.lr))\n",
" self.rates.append(K.get_value(self.model.optimizer.learning_rate))\n",
" self.losses.append(batch_loss)\n",
" K.set_value(self.model.optimizer.lr, self.model.optimizer.lr * self.factor)\n",
" K.set_value(self.model.optimizer.learning_rate, self.model.optimizer.learning_rate * self.factor)\n",
"```"
]
},
@ -1694,7 +1694,7 @@
" keras.layers.Dense(10, activation=\"softmax\")\n",
"])\n",
"model.compile(loss=\"sparse_categorical_crossentropy\",\n",
" optimizer=keras.optimizers.SGD(lr=1e-3),\n",
" optimizer=keras.optimizers.SGD(learning_rate=1e-3),\n",
" metrics=[\"accuracy\"])"
]
},
@ -1738,7 +1738,7 @@
" rate = self._interpolate(2 * self.half_iteration, self.iterations,\n",
" self.start_rate, self.last_rate)\n",
" self.iteration += 1\n",
" K.set_value(self.model.optimizer.lr, rate)"
" K.set_value(self.model.optimizer.learning_rate, rate)"
]
},
{
@ -1889,7 +1889,7 @@
" keras.layers.AlphaDropout(rate=0.2),\n",
" keras.layers.Dense(10, activation=\"softmax\")\n",
"])\n",
"optimizer = keras.optimizers.SGD(lr=0.01, momentum=0.9, nesterov=True)\n",
"optimizer = keras.optimizers.SGD(learning_rate=0.01, momentum=0.9, nesterov=True)\n",
"model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=optimizer, metrics=[\"accuracy\"])\n",
"n_epochs = 20\n",
"history = model.fit(X_train_scaled, y_train, epochs=n_epochs,\n",
@ -2060,7 +2060,7 @@
"metadata": {},
"outputs": [],
"source": [
"optimizer = keras.optimizers.SGD(lr=0.01, momentum=0.9, nesterov=True)\n",
"optimizer = keras.optimizers.SGD(learning_rate=0.01, momentum=0.9, nesterov=True)\n",
"mc_model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=optimizer, metrics=[\"accuracy\"])"
]
},
@ -2219,7 +2219,7 @@
"metadata": {},
"outputs": [],
"source": [
"optimizer = keras.optimizers.Nadam(lr=5e-5)\n",
"optimizer = keras.optimizers.Nadam(learning_rate=5e-5)\n",
"model.compile(loss=\"sparse_categorical_crossentropy\",\n",
" optimizer=optimizer,\n",
" metrics=[\"accuracy\"])"
@ -2342,7 +2342,7 @@
" model.add(keras.layers.Activation(\"elu\"))\n",
"model.add(keras.layers.Dense(10, activation=\"softmax\"))\n",
"\n",
"optimizer = keras.optimizers.Nadam(lr=5e-4)\n",
"optimizer = keras.optimizers.Nadam(learning_rate=5e-4)\n",
"model.compile(loss=\"sparse_categorical_crossentropy\",\n",
" optimizer=optimizer,\n",
" metrics=[\"accuracy\"])\n",
@ -2399,7 +2399,7 @@
" activation=\"selu\"))\n",
"model.add(keras.layers.Dense(10, activation=\"softmax\"))\n",
"\n",
"optimizer = keras.optimizers.Nadam(lr=7e-4)\n",
"optimizer = keras.optimizers.Nadam(learning_rate=7e-4)\n",
"model.compile(loss=\"sparse_categorical_crossentropy\",\n",
" optimizer=optimizer,\n",
" metrics=[\"accuracy\"])\n",
@ -2470,7 +2470,7 @@
"model.add(keras.layers.AlphaDropout(rate=0.1))\n",
"model.add(keras.layers.Dense(10, activation=\"softmax\"))\n",
"\n",
"optimizer = keras.optimizers.Nadam(lr=5e-4)\n",
"optimizer = keras.optimizers.Nadam(learning_rate=5e-4)\n",
"model.compile(loss=\"sparse_categorical_crossentropy\",\n",
" optimizer=optimizer,\n",
" metrics=[\"accuracy\"])\n",
@ -2621,7 +2621,7 @@
"model.add(keras.layers.AlphaDropout(rate=0.1))\n",
"model.add(keras.layers.Dense(10, activation=\"softmax\"))\n",
"\n",
"optimizer = keras.optimizers.SGD(lr=1e-3)\n",
"optimizer = keras.optimizers.SGD(learning_rate=1e-3)\n",
"model.compile(loss=\"sparse_categorical_crossentropy\",\n",
" optimizer=optimizer,\n",
" metrics=[\"accuracy\"])"
@ -2659,7 +2659,7 @@
"model.add(keras.layers.AlphaDropout(rate=0.1))\n",
"model.add(keras.layers.Dense(10, activation=\"softmax\"))\n",
"\n",
"optimizer = keras.optimizers.SGD(lr=1e-2)\n",
"optimizer = keras.optimizers.SGD(learning_rate=1e-2)\n",
"model.compile(loss=\"sparse_categorical_crossentropy\",\n",
" optimizer=optimizer,\n",
" metrics=[\"accuracy\"])"

8
12_custom_models_and_training_with_tensorflow.ipynb
View File

@ -2717,7 +2717,7 @@
"n_epochs = 5\n",
"batch_size = 32\n",
"n_steps = len(X_train) // batch_size\n",
"optimizer = keras.optimizers.Nadam(lr=0.01)\n",
"optimizer = keras.optimizers.Nadam(learning_rate=0.01)\n",
"loss_fn = keras.losses.mean_squared_error\n",
"mean_loss = keras.metrics.Mean()\n",
"metrics = [keras.metrics.MeanAbsoluteError()]"
@ -3828,7 +3828,7 @@
"n_epochs = 5\n",
"batch_size = 32\n",
"n_steps = len(X_train) // batch_size\n",
"optimizer = keras.optimizers.Nadam(lr=0.01)\n",
"optimizer = keras.optimizers.Nadam(learning_rate=0.01)\n",
"loss_fn = keras.losses.sparse_categorical_crossentropy\n",
"mean_loss = keras.metrics.Mean()\n",
"metrics = [keras.metrics.SparseCategoricalAccuracy()]"
@ -3913,8 +3913,8 @@
"metadata": {},
"outputs": [],
"source": [
"lower_optimizer = keras.optimizers.SGD(lr=1e-4)\n",
"upper_optimizer = keras.optimizers.Nadam(lr=1e-3)"
"lower_optimizer = keras.optimizers.SGD(learning_rate=1e-4)\n",
"upper_optimizer = keras.optimizers.Nadam(learning_rate=1e-3)"
]
},
{

12
13_loading_and_preprocessing_data.ipynb
View File

@ -559,7 +559,7 @@
"metadata": {},
"outputs": [],
"source": [
"model.compile(loss=\"mse\", optimizer=keras.optimizers.SGD(lr=1e-3))"
"model.compile(loss=\"mse\", optimizer=keras.optimizers.SGD(learning_rate=1e-3))"
]
},
{
@ -601,7 +601,7 @@
"metadata": {},
"outputs": [],
"source": [
"optimizer = keras.optimizers.Nadam(lr=0.01)\n",
"optimizer = keras.optimizers.Nadam(learning_rate=0.01)\n",
"loss_fn = keras.losses.mean_squared_error\n",
"\n",
"n_epochs = 5\n",
@ -637,7 +637,7 @@
"metadata": {},
"outputs": [],
"source": [
"optimizer = keras.optimizers.Nadam(lr=0.01)\n",
"optimizer = keras.optimizers.Nadam(learning_rate=0.01)\n",
"loss_fn = keras.losses.mean_squared_error\n",
"\n",
"@tf.function\n",
@ -674,7 +674,7 @@
"metadata": {},
"outputs": [],
"source": [
"optimizer = keras.optimizers.Nadam(lr=0.01)\n",
"optimizer = keras.optimizers.Nadam(learning_rate=0.01)\n",
"loss_fn = keras.losses.mean_squared_error\n",
"\n",
"@tf.function\n",
@ -1687,7 +1687,7 @@
" keras.layers.Dense(1)\n",
"])\n",
"model.compile(loss=\"mse\",\n",
" optimizer=keras.optimizers.SGD(lr=1e-3),\n",
" optimizer=keras.optimizers.SGD(learning_rate=1e-3),\n",
" metrics=[\"accuracy\"])\n",
"model.fit(dataset, steps_per_epoch=len(X_train) // batch_size, epochs=5)"
]
@ -1824,7 +1824,7 @@
" keras.layers.Lambda(lambda images: tf.cast(images, tf.float32)),\n",
" keras.layers.Dense(10, activation=\"softmax\")])\n",
"model.compile(loss=\"sparse_categorical_crossentropy\",\n",
" optimizer=keras.optimizers.SGD(lr=1e-3),\n",
" optimizer=keras.optimizers.SGD(learning_rate=1e-3),\n",
" metrics=[\"accuracy\"])\n",
"model.fit(mnist_train, steps_per_epoch=60000 // 32, epochs=5)"
]

2
14_deep_computer_vision_with_cnns.ipynb
View File

@ -1092,7 +1092,7 @@
"for layer in base_model.layers:\n",
" layer.trainable = False\n",
"\n",
"optimizer = keras.optimizers.SGD(lr=0.2, momentum=0.9, decay=0.01)\n",
"optimizer = keras.optimizers.SGD(learning_rate=0.2, momentum=0.9, decay=0.01)\n",
"model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=optimizer,\n",
" metrics=[\"accuracy\"])\n",
"history = model.fit(train_set,\n",

8
15_processing_sequences_using_rnns_and_cnns.ipynb
View File

@ -305,7 +305,7 @@
" keras.layers.SimpleRNN(1, input_shape=[None, 1])\n",
"])\n",
"\n",
"optimizer = keras.optimizers.Adam(lr=0.005)\n",
"optimizer = keras.optimizers.Adam(learning_rate=0.005)\n",
"model.compile(loss=\"mse\", optimizer=optimizer)\n",
"history = model.fit(X_train, y_train, epochs=20,\n",
" validation_data=(X_valid, y_valid))"
@ -711,7 +711,7 @@
"def last_time_step_mse(Y_true, Y_pred):\n",
" return keras.metrics.mean_squared_error(Y_true[:, -1], Y_pred[:, -1])\n",
"\n",
"model.compile(loss=\"mse\", optimizer=keras.optimizers.Adam(lr=0.01), metrics=[last_time_step_mse])\n",
"model.compile(loss=\"mse\", optimizer=keras.optimizers.Adam(learning_rate=0.01), metrics=[last_time_step_mse])\n",
"history = model.fit(X_train, Y_train, epochs=20,\n",
" validation_data=(X_valid, Y_valid))"
]
@ -1478,7 +1478,7 @@
" keras.layers.LSTM(128),\n",
" keras.layers.Dense(len(class_names), activation=\"softmax\")\n",
"])\n",
"optimizer = keras.optimizers.SGD(lr=1e-2, clipnorm=1.)\n",
"optimizer = keras.optimizers.SGD(learning_rate=1e-2, clipnorm=1.)\n",
"model.compile(loss=\"sparse_categorical_crossentropy\",\n",
" optimizer=optimizer,\n",
" metrics=[\"accuracy\", \"sparse_top_k_categorical_accuracy\"])\n",
@ -1818,7 +1818,7 @@
"metadata": {},
"outputs": [],
"source": [
"optimizer = keras.optimizers.Nadam(lr=1e-3)\n",
"optimizer = keras.optimizers.Nadam(learning_rate=1e-3)\n",
"model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=optimizer,\n",
" metrics=[\"accuracy\"])\n",
"model.fit(train_set, epochs=20, validation_data=valid_set)"

2
16_nlp_with_rnns_and_attention.ipynb
View File

@ -1531,7 +1531,7 @@
" keras.layers.GRU(30),\n",
" keras.layers.Dense(1, activation=\"sigmoid\")\n",
"])\n",
"optimizer = keras.optimizers.SGD(lr=0.02, momentum = 0.95, nesterov=True)\n",
"optimizer = keras.optimizers.SGD(learning_rate=0.02, momentum = 0.95, nesterov=True)\n",
"model.compile(loss=\"binary_crossentropy\", optimizer=optimizer, metrics=[\"accuracy\"])\n",
"history = model.fit(X_train, y_train, epochs=20, validation_data=(X_valid, y_valid))"
]

24
17_autoencoders_and_gans.ipynb
View File

@ -174,7 +174,7 @@
"decoder = keras.models.Sequential([keras.layers.Dense(3, input_shape=[2])])\n",
"autoencoder = keras.models.Sequential([encoder, decoder])\n",
"\n",
"autoencoder.compile(loss=\"mse\", optimizer=keras.optimizers.SGD(lr=1.5))"
"autoencoder.compile(loss=\"mse\", optimizer=keras.optimizers.SGD(learning_rate=1.5))"
]
},
{
@ -282,7 +282,7 @@
"])\n",
"stacked_ae = keras.models.Sequential([stacked_encoder, stacked_decoder])\n",
"stacked_ae.compile(loss=\"binary_crossentropy\",\n",
" optimizer=keras.optimizers.SGD(lr=1.5), metrics=[rounded_accuracy])\n",
" optimizer=keras.optimizers.SGD(learning_rate=1.5), metrics=[rounded_accuracy])\n",
"history = stacked_ae.fit(X_train, X_train, epochs=20,\n",
" validation_data=(X_valid, X_valid))"
]
@ -448,7 +448,7 @@
"tied_ae = keras.models.Sequential([tied_encoder, tied_decoder])\n",
"\n",
"tied_ae.compile(loss=\"binary_crossentropy\",\n",
" optimizer=keras.optimizers.SGD(lr=1.5), metrics=[rounded_accuracy])\n",
" optimizer=keras.optimizers.SGD(learning_rate=1.5), metrics=[rounded_accuracy])\n",
"history = tied_ae.fit(X_train, X_train, epochs=10,\n",
" validation_data=(X_valid, X_valid))"
]
@ -508,10 +508,10 @@
"X_valid_flat = K.batch_flatten(X_valid)\n",
"enc1, dec1, X_train_enc1, X_valid_enc1 = train_autoencoder(\n",
" 100, X_train_flat, X_valid_flat, \"binary_crossentropy\",\n",
" keras.optimizers.SGD(lr=1.5), output_activation=\"sigmoid\",\n",
" keras.optimizers.SGD(learning_rate=1.5), output_activation=\"sigmoid\",\n",
" metrics=[rounded_accuracy])\n",
"enc2, dec2, _, _ = train_autoencoder(\n",
" 30, X_train_enc1, X_valid_enc1, \"mse\", keras.optimizers.SGD(lr=0.05),\n",
" 30, X_train_enc1, X_valid_enc1, \"mse\", keras.optimizers.SGD(learning_rate=0.05),\n",
" output_activation=\"selu\")"
]
},
@ -545,7 +545,7 @@
"outputs": [],
"source": [
"stacked_ae_1_by_1.compile(loss=\"binary_crossentropy\",\n",
" optimizer=keras.optimizers.SGD(lr=0.1), metrics=[rounded_accuracy])\n",
" optimizer=keras.optimizers.SGD(learning_rate=0.1), metrics=[rounded_accuracy])\n",
"history = stacked_ae_1_by_1.fit(X_train, X_train, epochs=10,\n",
" validation_data=(X_valid, X_valid))"
]
@ -601,7 +601,7 @@
"])\n",
"conv_ae = keras.models.Sequential([conv_encoder, conv_decoder])\n",
"\n",
"conv_ae.compile(loss=\"binary_crossentropy\", optimizer=keras.optimizers.SGD(lr=1.0),\n",
"conv_ae.compile(loss=\"binary_crossentropy\", optimizer=keras.optimizers.SGD(learning_rate=1.0),\n",
" metrics=[rounded_accuracy])\n",
"history = conv_ae.fit(X_train, X_train, epochs=5,\n",
" validation_data=(X_valid, X_valid))"
@ -708,7 +708,7 @@
" keras.layers.Reshape([28, 28])\n",
"])\n",
"denoising_ae = keras.models.Sequential([denoising_encoder, denoising_decoder])\n",
"denoising_ae.compile(loss=\"binary_crossentropy\", optimizer=keras.optimizers.SGD(lr=1.0),\n",
"denoising_ae.compile(loss=\"binary_crossentropy\", optimizer=keras.optimizers.SGD(learning_rate=1.0),\n",
" metrics=[rounded_accuracy])\n",
"history = denoising_ae.fit(X_train, X_train, epochs=10,\n",
" validation_data=(X_valid, X_valid))"
@ -756,7 +756,7 @@
" keras.layers.Reshape([28, 28])\n",
"])\n",
"dropout_ae = keras.models.Sequential([dropout_encoder, dropout_decoder])\n",
"dropout_ae.compile(loss=\"binary_crossentropy\", optimizer=keras.optimizers.SGD(lr=1.0),\n",
"dropout_ae.compile(loss=\"binary_crossentropy\", optimizer=keras.optimizers.SGD(learning_rate=1.0),\n",
" metrics=[rounded_accuracy])\n",
"history = dropout_ae.fit(X_train, X_train, epochs=10,\n",
" validation_data=(X_valid, X_valid))"
@ -810,7 +810,7 @@
" keras.layers.Reshape([28, 28])\n",
"])\n",
"simple_ae = keras.models.Sequential([simple_encoder, simple_decoder])\n",
"simple_ae.compile(loss=\"binary_crossentropy\", optimizer=keras.optimizers.SGD(lr=1.),\n",
"simple_ae.compile(loss=\"binary_crossentropy\", optimizer=keras.optimizers.SGD(learning_rate=1.),\n",
" metrics=[rounded_accuracy])\n",
"history = simple_ae.fit(X_train, X_train, epochs=10,\n",
" validation_data=(X_valid, X_valid))"
@ -923,7 +923,7 @@
" keras.layers.Reshape([28, 28])\n",
"])\n",
"sparse_l1_ae = keras.models.Sequential([sparse_l1_encoder, sparse_l1_decoder])\n",
"sparse_l1_ae.compile(loss=\"binary_crossentropy\", optimizer=keras.optimizers.SGD(lr=1.0),\n",
"sparse_l1_ae.compile(loss=\"binary_crossentropy\", optimizer=keras.optimizers.SGD(learning_rate=1.0),\n",
" metrics=[rounded_accuracy])\n",
"history = sparse_l1_ae.fit(X_train, X_train, epochs=10,\n",
" validation_data=(X_valid, X_valid))"
@ -1019,7 +1019,7 @@
" keras.layers.Reshape([28, 28])\n",
"])\n",
"sparse_kl_ae = keras.models.Sequential([sparse_kl_encoder, sparse_kl_decoder])\n",
"sparse_kl_ae.compile(loss=\"binary_crossentropy\", optimizer=keras.optimizers.SGD(lr=1.0),\n",
"sparse_kl_ae.compile(loss=\"binary_crossentropy\", optimizer=keras.optimizers.SGD(learning_rate=1.0),\n",
" metrics=[rounded_accuracy])\n",
"history = sparse_kl_ae.fit(X_train, X_train, epochs=10,\n",
" validation_data=(X_valid, X_valid))"

12
18_reinforcement_learning.ipynb
View File

@ -871,7 +871,7 @@
"metadata": {},
"outputs": [],
"source": [
"optimizer = keras.optimizers.Adam(lr=0.01)\n",
"optimizer = keras.optimizers.Adam(learning_rate=0.01)\n",
"loss_fn = keras.losses.binary_crossentropy"
]
},
@ -1389,7 +1389,7 @@
"source": [
"batch_size = 32\n",
"discount_rate = 0.95\n",
"optimizer = keras.optimizers.Adam(lr=1e-2)\n",
"optimizer = keras.optimizers.Adam(learning_rate=1e-2)\n",
"loss_fn = keras.losses.mean_squared_error\n",
"\n",
"def training_step(batch_size):\n",
@ -1534,7 +1534,7 @@
"source": [
"batch_size = 32\n",
"discount_rate = 0.95\n",
"optimizer = keras.optimizers.Adam(lr=6e-3)\n",
"optimizer = keras.optimizers.Adam(learning_rate=6e-3)\n",
"loss_fn = keras.losses.Huber()\n",
"\n",
"def training_step(batch_size):\n",
@ -1682,7 +1682,7 @@
"source": [
"batch_size = 32\n",
"discount_rate = 0.95\n",
"optimizer = keras.optimizers.Adam(lr=7.5e-3)\n",
"optimizer = keras.optimizers.Adam(learning_rate=7.5e-3)\n",
"loss_fn = keras.losses.Huber()\n",
"\n",
"def training_step(batch_size):\n",
@ -2207,7 +2207,7 @@
"\n",
"train_step = tf.Variable(0)\n",
"update_period = 4 # run a training step every 4 collect steps\n",
"optimizer = keras.optimizers.RMSprop(lr=2.5e-4, rho=0.95, momentum=0.0,\n",
"optimizer = keras.optimizers.RMSprop(learning_rate=2.5e-4, rho=0.95, momentum=0.0,\n",
" epsilon=0.00001, centered=True)\n",
"epsilon_fn = keras.optimizers.schedules.PolynomialDecay(\n",
" initial_learning_rate=1.0, # initial ε\n",
@ -3032,7 +3032,7 @@
"metadata": {},
"outputs": [],
"source": [
"optimizer = keras.optimizers.Nadam(lr=0.005)\n",
"optimizer = keras.optimizers.Nadam(learning_rate=0.005)\n",
"loss_fn = keras.losses.sparse_categorical_crossentropy"
]
},

10
19_training_and_deploying_at_scale.ipynb
View File

@ -146,7 +146,7 @@
" keras.layers.Dense(10, activation=\"softmax\")\n",
"])\n",
"model.compile(loss=\"sparse_categorical_crossentropy\",\n",
" optimizer=keras.optimizers.SGD(lr=1e-2),\n",
" optimizer=keras.optimizers.SGD(learning_rate=1e-2),\n",
" metrics=[\"accuracy\"])\n",
"model.fit(X_train, y_train, epochs=10, validation_data=(X_valid, y_valid))"
]
@ -535,7 +535,7 @@
" keras.layers.Dense(10, activation=\"softmax\")\n",
"])\n",
"model.compile(loss=\"sparse_categorical_crossentropy\",\n",
" optimizer=keras.optimizers.SGD(lr=1e-2),\n",
" optimizer=keras.optimizers.SGD(learning_rate=1e-2),\n",
" metrics=[\"accuracy\"])\n",
"history = model.fit(X_train, y_train, epochs=10, validation_data=(X_valid, y_valid))"
]
@ -783,7 +783,7 @@
"batch_size = 100\n",
"model = create_model()\n",
"model.compile(loss=\"sparse_categorical_crossentropy\",\n",
" optimizer=keras.optimizers.SGD(lr=1e-2),\n",
" optimizer=keras.optimizers.SGD(learning_rate=1e-2),\n",
" metrics=[\"accuracy\"])\n",
"model.fit(X_train, y_train, epochs=10,\n",
" validation_data=(X_valid, y_valid), batch_size=batch_size)"
@ -823,7 +823,7 @@
"with distribution.scope():\n",
" model = create_model()\n",
" model.compile(loss=\"sparse_categorical_crossentropy\",\n",
" optimizer=keras.optimizers.SGD(lr=1e-2),\n",
" optimizer=keras.optimizers.SGD(learning_rate=1e-2),\n",
" metrics=[\"accuracy\"])"
]
},
@ -1075,7 +1075,7 @@
" keras.layers.Dense(units=10, activation='softmax'),\n",
" ])\n",
" model.compile(loss=\"sparse_categorical_crossentropy\",\n",
" optimizer=keras.optimizers.SGD(lr=1e-2),\n",
" optimizer=keras.optimizers.SGD(learning_rate=1e-2),\n",
" metrics=[\"accuracy\"])\n",
"\n",
"model.fit(X_train, y_train, validation_data=(X_valid, y_valid),\n",