1301 lines
39 KiB
Plaintext
1301 lines
39 KiB
Plaintext
|
{
|
|||
|
|
"cells": [
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {},
|
|||
|
|
"source": [
|
|||
|
|
"**Chapter 17 – Autoencoders**"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {},
|
|||
|
|
"source": [
|
|||
|
|
"_This notebook contains all the sample code in chapter 17._"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {},
|
|||
|
|
"source": [
|
|||
|
|
"# Setup"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {},
|
|||
|
|
"source": [
|
|||
|
|
"First, let's import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures. We also check that Python 3.5 or later is installed (although Python 2.x may work, it is deprecated so we strongly recommend you use Python 3 instead), as well as Scikit-Learn ≥0.20 and TensorFlow ≥2.0-preview."
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 1,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"# Python ≥3.5 is required\n",
|
|||
|
|
"import sys\n",
|
|||
|
|
"assert sys.version_info >= (3, 5)\n",
|
|||
|
|
"\n",
|
|||
|
|
"# Scikit-Learn ≥0.20 is required\n",
|
|||
|
|
"import sklearn\n",
|
|||
|
|
"assert sklearn.__version__ >= \"0.20\"\n",
|
|||
|
|
"\n",
|
|||
|
|
"# TensorFlow ≥2.0-preview is required\n",
|
|||
|
|
"import tensorflow as tf\n",
|
|||
|
|
"from tensorflow import keras\n",
|
|||
|
|
"assert tf.__version__ >= \"2.0\"\n",
|
|||
|
|
"\n",
|
|||
|
|
"# Common imports\n",
|
|||
|
|
"import numpy as np\n",
|
|||
|
|
"import os\n",
|
|||
|
|
"\n",
|
|||
|
|
"# to make this notebook's output stable across runs\n",
|
|||
|
|
"np.random.seed(42)\n",
|
|||
|
|
"tf.random.set_seed(42)\n",
|
|||
|
|
"\n",
|
|||
|
|
"# To plot pretty figures\n",
|
|||
|
|
"%matplotlib inline\n",
|
|||
|
|
"import matplotlib as mpl\n",
|
|||
|
|
"import matplotlib.pyplot as plt\n",
|
|||
|
|
"mpl.rc('axes', labelsize=14)\n",
|
|||
|
|
"mpl.rc('xtick', labelsize=12)\n",
|
|||
|
|
"mpl.rc('ytick', labelsize=12)\n",
|
|||
|
|
"\n",
|
|||
|
|
"# Where to save the figures\n",
|
|||
|
|
"PROJECT_ROOT_DIR = \".\"\n",
|
|||
|
|
"CHAPTER_ID = \"autoencoders\"\n",
|
|||
|
|
"IMAGES_PATH = os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID)\n",
|
|||
|
|
"os.makedirs(IMAGES_PATH, exist_ok=True)\n",
|
|||
|
|
"\n",
|
|||
|
|
"def save_fig(fig_id, tight_layout=True, fig_extension=\"png\", resolution=300):\n",
|
|||
|
|
" path = os.path.join(IMAGES_PATH, fig_id + \".\" + fig_extension)\n",
|
|||
|
|
" print(\"Saving figure\", fig_id)\n",
|
|||
|
|
" if tight_layout:\n",
|
|||
|
|
" plt.tight_layout()\n",
|
|||
|
|
" plt.savefig(path, format=fig_extension, dpi=resolution)"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {},
|
|||
|
|
"source": [
|
|||
|
|
"A couple utility functions to plot grayscale 28x28 image:"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 2,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"def plot_image(image):\n",
|
|||
|
|
" plt.imshow(image, cmap=\"Greys\", interpolation=\"nearest\")\n",
|
|||
|
|
" plt.axis(\"off\")"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {},
|
|||
|
|
"source": [
|
|||
|
|
"# PCA with a linear Autoencoder"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {},
|
|||
|
|
"source": [
|
|||
|
|
"Build 3D dataset:"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 3,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"np.random.seed(4)\n",
|
|||
|
|
"\n",
|
|||
|
|
"m = 200\n",
|
|||
|
|
"w1, w2 = 0.1, 0.3\n",
|
|||
|
|
"noise = 0.1\n",
|
|||
|
|
"\n",
|
|||
|
|
"angles = np.random.rand(m) * 3 * np.pi / 2 - 0.5\n",
|
|||
|
|
"data = np.empty((m, 3))\n",
|
|||
|
|
"data[:, 0] = np.cos(angles) + np.sin(angles)/2 + noise * np.random.randn(m) / 2\n",
|
|||
|
|
"data[:, 1] = np.sin(angles) * 0.7 + noise * np.random.randn(m) / 2\n",
|
|||
|
|
"data[:, 2] = data[:, 0] * w1 + data[:, 1] * w2 + noise * np.random.randn(m)"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {},
|
|||
|
|
"source": [
|
|||
|
|
"Normalize the data:"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 4,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"from sklearn.preprocessing import StandardScaler\n",
|
|||
|
|
"scaler = StandardScaler()\n",
|
|||
|
|
"X_train = scaler.fit_transform(data[:100])\n",
|
|||
|
|
"X_test = scaler.transform(data[100:])"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {},
|
|||
|
|
"source": [
|
|||
|
|
"Now let's build the Autoencoder..."
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 5,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"tf.random.set_seed(42)\n",
|
|||
|
|
"\n",
|
|||
|
|
"encoder = keras.models.Sequential([keras.layers.Dense(2, input_shape=[3])])\n",
|
|||
|
|
"decoder = keras.models.Sequential([keras.layers.Dense(3, input_shape=[2])])\n",
|
|||
|
|
"autoencoder = keras.models.Sequential([encoder, decoder])\n",
|
|||
|
|
"\n",
|
|||
|
|
"autoencoder.compile(loss=\"mse\", optimizer=keras.optimizers.SGD(lr=0.1))\n",
|
|||
|
|
"history = autoencoder.fit(X_train, X_train, epochs=20, validation_data=[X_test, X_test])"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 6,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"codings_test = encoder.predict(X_test)"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 7,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"fig = plt.figure(figsize=(4,3))\n",
|
|||
|
|
"plt.plot(codings_test[:,0], codings_test[:, 1], \"b.\")\n",
|
|||
|
|
"plt.xlabel(\"$z_1$\", fontsize=18)\n",
|
|||
|
|
"plt.ylabel(\"$z_2$\", fontsize=18, rotation=0)\n",
|
|||
|
|
"save_fig(\"linear_autoencoder_pca_plot\")\n",
|
|||
|
|
"plt.show()"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {},
|
|||
|
|
"source": [
|
|||
|
|
"# Stacked Autoencoders"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {},
|
|||
|
|
"source": [
|
|||
|
|
"Let's use MNIST:"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 8,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"(X_train_full, y_train_full), (X_test, y_test) = keras.datasets.fashion_mnist.load_data()\n",
|
|||
|
|
"X_train_full = X_train_full / 255\n",
|
|||
|
|
"X_test = X_test / 255\n",
|
|||
|
|
"X_train, X_valid = X_train_full[:-5000], X_train_full[-5000:]\n",
|
|||
|
|
"y_train, y_valid = y_train_full[:-5000], y_train_full[-5000:]"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {},
|
|||
|
|
"source": [
|
|||
|
|
"## Train all layers at once"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {},
|
|||
|
|
"source": [
|
|||
|
|
"Let's build a stacked Autoencoder with 3 hidden layers and 1 output layer (i.e., 2 stacked Autoencoders)."
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 9,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"tf.random.set_seed(42)\n",
|
|||
|
|
"np.random.seed(42)\n",
|
|||
|
|
"\n",
|
|||
|
|
"stacked_encoder = keras.models.Sequential([\n",
|
|||
|
|
" keras.layers.Flatten(input_shape=[28, 28]),\n",
|
|||
|
|
" keras.layers.Dense(100, activation=\"selu\"),\n",
|
|||
|
|
" keras.layers.Dense(30, activation=\"selu\"),\n",
|
|||
|
|
"])\n",
|
|||
|
|
"stacked_decoder = keras.models.Sequential([\n",
|
|||
|
|
" keras.layers.Dense(100, activation=\"selu\", input_shape=[30]),\n",
|
|||
|
|
" keras.layers.Dense(28 * 28, activation=\"sigmoid\"),\n",
|
|||
|
|
" keras.layers.Reshape([28, 28])\n",
|
|||
|
|
"])\n",
|
|||
|
|
"stacked_ae = keras.models.Sequential([stacked_encoder, stacked_decoder])\n",
|
|||
|
|
"stacked_ae.compile(loss=\"binary_crossentropy\", optimizer=keras.optimizers.SGD(lr=1.),\n",
|
|||
|
|
" metrics=[\"accuracy\"])\n",
|
|||
|
|
"history = stacked_ae.fit(X_train, X_train, epochs=10,\n",
|
|||
|
|
" validation_data=[X_valid, X_valid])"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {},
|
|||
|
|
"source": [
|
|||
|
|
"This function processes a few test images through the autoencoder and displays the original images and their reconstructions:"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 10,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"def show_reconstructions(model, n_test_images = 2):\n",
|
|||
|
|
" reconstructions = model.predict(X_test[:n_test_images])\n",
|
|||
|
|
" fig = plt.figure(figsize=(8, 3 * n_test_images))\n",
|
|||
|
|
" for image_index in range(n_test_images):\n",
|
|||
|
|
" plt.subplot(n_test_images, 2, image_index * 2 + 1)\n",
|
|||
|
|
" plot_image(X_test[image_index])\n",
|
|||
|
|
" plt.subplot(n_test_images, 2, image_index * 2 + 2)\n",
|
|||
|
|
" plot_image(reconstructions[image_index])"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 11,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"show_reconstructions(stacked_ae)\n",
|
|||
|
|
"save_fig(\"reconstruction_plot\")"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {},
|
|||
|
|
"source": [
|
|||
|
|
"## Tying weights"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {},
|
|||
|
|
"source": [
|
|||
|
|
"It is common to tie the weights of the encoder and the decoder, by simply using the transpose of the encoder's weights as the decoder weights. For this, we need to use a custom layer."
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 12,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"K = keras.backend\n",
|
|||
|
|
"\n",
|
|||
|
|
"class DenseTranspose(keras.layers.Layer):\n",
|
|||
|
|
" def __init__(self, dense, activation=None, **kwargs):\n",
|
|||
|
|
" self.dense = dense\n",
|
|||
|
|
" self.activation = keras.activations.get(activation)\n",
|
|||
|
|
" super().__init__(**kwargs)\n",
|
|||
|
|
" def build(self, batch_input_shape):\n",
|
|||
|
|
" self.biases = self.add_weight(name=\"bias\", shape=[self.dense.input_shape[-1]],\n",
|
|||
|
|
" initializer=\"zeros\")\n",
|
|||
|
|
" super().build(batch_input_shape)\n",
|
|||
|
|
" def call(self, inputs):\n",
|
|||
|
|
" return self.activation(inputs @ K.transpose(self.dense.weights[0]) + self.biases)"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 13,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"tf.random.set_seed(42)\n",
|
|||
|
|
"np.random.seed(42)\n",
|
|||
|
|
"\n",
|
|||
|
|
"tied_encoder = keras.models.Sequential([\n",
|
|||
|
|
" keras.layers.Flatten(input_shape=[28, 28]),\n",
|
|||
|
|
" keras.layers.Dense(100, activation=\"selu\"),\n",
|
|||
|
|
" keras.layers.Dense(30, activation=\"selu\"),\n",
|
|||
|
|
"])\n",
|
|||
|
|
"tied_decoder = keras.models.Sequential([\n",
|
|||
|
|
" DenseTranspose(tied_encoder.layers[2], activation=\"selu\"),\n",
|
|||
|
|
" DenseTranspose(tied_encoder.layers[1], activation=\"sigmoid\"),\n",
|
|||
|
|
" keras.layers.Reshape([28, 28])\n",
|
|||
|
|
"])\n",
|
|||
|
|
"tied_ae = keras.models.Sequential([tied_encoder, tied_decoder])\n",
|
|||
|
|
"tied_ae.compile(loss=\"binary_crossentropy\", optimizer=keras.optimizers.SGD(lr=1.),\n",
|
|||
|
|
" metrics=[\"accuracy\"])\n",
|
|||
|
|
"history = tied_ae.fit(X_train, X_train, epochs=10, validation_data=[X_valid, X_valid])"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 14,
|
|||
|
|
"metadata": {
|
|||
|
|
"scrolled": true
|
|||
|
|
},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"show_reconstructions(tied_ae)\n",
|
|||
|
|
"plt.show()"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {},
|
|||
|
|
"source": [
|
|||
|
|
"## Training one Autoencoder at a time"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 15,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"def train_autoencoder(n_neurons, X_train, X_valid, loss, optimizer, metrics=None,\n",
|
|||
|
|
" n_epochs=10, output_activation=None):\n",
|
|||
|
|
" n_inputs = X_train.shape[-1]\n",
|
|||
|
|
" encoder = keras.models.Sequential([\n",
|
|||
|
|
" keras.layers.Dense(n_neurons, activation=\"selu\", input_shape=[n_inputs])\n",
|
|||
|
|
" ])\n",
|
|||
|
|
" decoder = keras.models.Sequential([\n",
|
|||
|
|
" keras.layers.Dense(n_inputs, activation=output_activation),\n",
|
|||
|
|
" ])\n",
|
|||
|
|
" autoencoder = keras.models.Sequential([encoder, decoder])\n",
|
|||
|
|
" autoencoder.compile(optimizer, loss, metrics=metrics)\n",
|
|||
|
|
" autoencoder.fit(X_train, X_train, epochs=n_epochs,\n",
|
|||
|
|
" validation_data=[X_valid, X_valid])\n",
|
|||
|
|
" return encoder, decoder, encoder(X_train), encoder(X_valid)"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 16,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"tf.random.set_seed(42)\n",
|
|||
|
|
"np.random.seed(42)\n",
|
|||
|
|
"\n",
|
|||
|
|
"X_train_flat = keras.layers.Flatten()(X_train)\n",
|
|||
|
|
"X_valid_flat = keras.layers.Flatten()(X_valid)\n",
|
|||
|
|
"enc1, dec1, X_train_enc1, X_valid_enc1 = train_autoencoder(\n",
|
|||
|
|
" 100, X_train_flat, X_valid_flat, \"binary_crossentropy\", keras.optimizers.SGD(lr=0.1),\n",
|
|||
|
|
" output_activation=\"sigmoid\", metrics=[\"accuracy\"])\n",
|
|||
|
|
"enc2, dec2, _, _ = train_autoencoder(\n",
|
|||
|
|
" 30, X_train_enc1, X_valid_enc1, \"mse\", keras.optimizers.Adam(),\n",
|
|||
|
|
" output_activation=\"selu\")"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 17,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"stacked_ae_1_by_1 = keras.models.Sequential([\n",
|
|||
|
|
" keras.layers.Flatten(input_shape=[28, 28]),\n",
|
|||
|
|
" enc1,\n",
|
|||
|
|
" enc2,\n",
|
|||
|
|
" dec2,\n",
|
|||
|
|
" dec1,\n",
|
|||
|
|
" keras.layers.Reshape([28, 28])\n",
|
|||
|
|
"])"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 18,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"show_reconstructions(stacked_ae_1_by_1)\n",
|
|||
|
|
"plt.show()"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 19,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"stacked_ae_1_by_1.compile(loss=\"binary_crossentropy\", optimizer=keras.optimizers.SGD(lr=0.5),\n",
|
|||
|
|
" metrics=[\"accuracy\"])\n",
|
|||
|
|
"history = stacked_ae_1_by_1.fit(X_train, X_train, epochs=10, validation_data=[X_valid, X_valid])"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 20,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"show_reconstructions(stacked_ae_1_by_1)\n",
|
|||
|
|
"plt.show()"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {},
|
|||
|
|
"source": [
|
|||
|
|
"## Visualizing the extracted features"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 21,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"weights1 = stacked_ae_1_by_1.layers[1].get_weights()[0]\n",
|
|||
|
|
"plt.figure(figsize=(8, 2))\n",
|
|||
|
|
"n_rows, n_cols = 2, 8\n",
|
|||
|
|
"for row in range(n_rows):\n",
|
|||
|
|
" for col in range(n_cols):\n",
|
|||
|
|
" index = row * n_cols + col\n",
|
|||
|
|
" plt.subplot(n_rows, n_cols, index + 1)\n",
|
|||
|
|
" plt.imshow(weights1[:, index].reshape(28, 28), cmap=\"Greys\")\n",
|
|||
|
|
" plt.axis(\"off\")\n",
|
|||
|
|
"\n",
|
|||
|
|
"save_fig(\"extracted_features_plot\", tight_layout=False) # not shown\n",
|
|||
|
|
"plt.show() # not shown"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {},
|
|||
|
|
"source": [
|
|||
|
|
"## Using Convolutional Layers Instead of Dense Layers"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {},
|
|||
|
|
"source": [
|
|||
|
|
"Let's build a stacked Autoencoder with 3 hidden layers and 1 output layer (i.e., 2 stacked Autoencoders)."
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 22,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"tf.random.set_seed(42)\n",
|
|||
|
|
"np.random.seed(42)\n",
|
|||
|
|
"\n",
|
|||
|
|
"conv_encoder = keras.models.Sequential([\n",
|
|||
|
|
" keras.layers.Reshape([28, 28, 1], input_shape=[28, 28]),\n",
|
|||
|
|
" keras.layers.Conv2D(16, kernel_size=3, padding=\"SAME\", activation=\"selu\"),\n",
|
|||
|
|
" keras.layers.MaxPool2D(pool_size=2),\n",
|
|||
|
|
" keras.layers.Conv2D(32, kernel_size=3, padding=\"SAME\", activation=\"selu\"),\n",
|
|||
|
|
" keras.layers.MaxPool2D(pool_size=2),\n",
|
|||
|
|
" keras.layers.Conv2D(64, kernel_size=3, padding=\"SAME\", activation=\"selu\"),\n",
|
|||
|
|
" keras.layers.MaxPool2D(pool_size=2)\n",
|
|||
|
|
"])\n",
|
|||
|
|
"conv_decoder = keras.models.Sequential([\n",
|
|||
|
|
" keras.layers.Conv2DTranspose(32, kernel_size=3, strides=2, padding=\"VALID\", activation=\"selu\",\n",
|
|||
|
|
" input_shape=[3, 3, 64]),\n",
|
|||
|
|
" keras.layers.Conv2DTranspose(16, kernel_size=3, strides=2, padding=\"SAME\", activation=\"selu\"),\n",
|
|||
|
|
" keras.layers.Conv2DTranspose(1, kernel_size=3, strides=2, padding=\"SAME\", activation=\"sigmoid\"),\n",
|
|||
|
|
" keras.layers.Reshape([28, 28])\n",
|
|||
|
|
"])\n",
|
|||
|
|
"conv_ae = keras.models.Sequential([conv_encoder, conv_decoder])\n",
|
|||
|
|
"\n",
|
|||
|
|
"conv_ae.compile(loss=\"binary_crossentropy\", optimizer=keras.optimizers.SGD(lr=1.0),\n",
|
|||
|
|
" metrics=[\"accuracy\"])\n",
|
|||
|
|
"history = conv_ae.fit(X_train, X_train, epochs=5,\n",
|
|||
|
|
" validation_data=[X_valid, X_valid])"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 23,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"conv_encoder.summary()\n",
|
|||
|
|
"conv_decoder.summary()"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 24,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"show_reconstructions(conv_ae)\n",
|
|||
|
|
"plt.show()"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {},
|
|||
|
|
"source": [
|
|||
|
|
"## Unsupervised pretraining"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {},
|
|||
|
|
"source": [
|
|||
|
|
"Let's create a small neural network for MNIST classification:"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 25,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"tf.random.set_seed(42)\n",
|
|||
|
|
"np.random.seed(42)\n",
|
|||
|
|
"\n",
|
|||
|
|
"X_train_small = X_train[:500]\n",
|
|||
|
|
"y_train_small = y_train[:500]\n",
|
|||
|
|
"\n",
|
|||
|
|
"classifier = keras.models.Sequential([\n",
|
|||
|
|
" keras.layers.Reshape([28, 28, 1], input_shape=[28, 28]),\n",
|
|||
|
|
" keras.layers.Conv2D(16, kernel_size=3, padding=\"SAME\", activation=\"selu\"),\n",
|
|||
|
|
" keras.layers.MaxPool2D(pool_size=2),\n",
|
|||
|
|
" keras.layers.Conv2D(32, kernel_size=3, padding=\"SAME\", activation=\"selu\"),\n",
|
|||
|
|
" keras.layers.MaxPool2D(pool_size=2),\n",
|
|||
|
|
" keras.layers.Conv2D(64, kernel_size=3, padding=\"SAME\", activation=\"selu\"),\n",
|
|||
|
|
" keras.layers.MaxPool2D(pool_size=2),\n",
|
|||
|
|
" keras.layers.Flatten(),\n",
|
|||
|
|
" keras.layers.Dense(20, activation=\"selu\"),\n",
|
|||
|
|
" keras.layers.Dense(10, activation=\"softmax\")\n",
|
|||
|
|
"])\n",
|
|||
|
|
"classifier.compile(loss=\"sparse_categorical_crossentropy\", optimizer=keras.optimizers.SGD(lr=0.02), metrics=[\"accuracy\"])\n",
|
|||
|
|
"history = classifier.fit(X_train_small, y_train_small, epochs=20, validation_data=[X_valid, y_valid])"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 26,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"import pandas as pd\n",
|
|||
|
|
"pd.DataFrame(history.history).plot()\n",
|
|||
|
|
"plt.show()"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 27,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"tf.random.set_seed(42)\n",
|
|||
|
|
"np.random.seed(42)\n",
|
|||
|
|
"\n",
|
|||
|
|
"conv_encoder_clone = keras.models.clone_model(conv_encoder)\n",
|
|||
|
|
"\n",
|
|||
|
|
"pretrained_clf = keras.models.Sequential([\n",
|
|||
|
|
" conv_encoder_clone,\n",
|
|||
|
|
" keras.layers.Flatten(),\n",
|
|||
|
|
" keras.layers.Dense(20, activation=\"selu\"),\n",
|
|||
|
|
" keras.layers.Dense(10, activation=\"softmax\")\n",
|
|||
|
|
"])"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 28,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"conv_encoder_clone.trainable = False\n",
|
|||
|
|
"pretrained_clf.compile(loss=\"sparse_categorical_crossentropy\",\n",
|
|||
|
|
" optimizer=keras.optimizers.SGD(lr=0.02),\n",
|
|||
|
|
" metrics=[\"accuracy\"])\n",
|
|||
|
|
"history = pretrained_clf.fit(X_train_small, y_train_small, epochs=30,\n",
|
|||
|
|
" validation_data=[X_valid, y_valid])"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 29,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"conv_encoder_clone.trainable = True\n",
|
|||
|
|
"pretrained_clf.compile(loss=\"sparse_categorical_crossentropy\",\n",
|
|||
|
|
" optimizer=keras.optimizers.SGD(lr=0.02),\n",
|
|||
|
|
" metrics=[\"accuracy\"])\n",
|
|||
|
|
"history = pretrained_clf.fit(X_train_small, y_train_small, epochs=20,\n",
|
|||
|
|
" validation_data=[X_valid, y_valid])"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {},
|
|||
|
|
"source": [
|
|||
|
|
"# Stacked denoising Autoencoder"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {},
|
|||
|
|
"source": [
|
|||
|
|
"Using Gaussian noise:"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 30,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"tf.random.set_seed(42)\n",
|
|||
|
|
"np.random.seed(42)\n",
|
|||
|
|
"\n",
|
|||
|
|
"denoising_encoder = keras.models.Sequential([\n",
|
|||
|
|
" keras.layers.Flatten(input_shape=[28, 28]),\n",
|
|||
|
|
" keras.layers.GaussianNoise(1.0),\n",
|
|||
|
|
" keras.layers.Dense(100, activation=\"selu\"),\n",
|
|||
|
|
" keras.layers.Dense(30, activation=\"selu\")\n",
|
|||
|
|
"])\n",
|
|||
|
|
"denoising_decoder = keras.models.Sequential([\n",
|
|||
|
|
" keras.layers.Dense(100, activation=\"selu\", input_shape=[30]),\n",
|
|||
|
|
" keras.layers.Dense(28 * 28, activation=\"sigmoid\"),\n",
|
|||
|
|
" keras.layers.Reshape([28, 28])\n",
|
|||
|
|
"])\n",
|
|||
|
|
"denoising_ae = keras.models.Sequential([denoising_encoder, denoising_decoder])\n",
|
|||
|
|
"denoising_ae.compile(loss=\"binary_crossentropy\", optimizer=keras.optimizers.SGD(lr=1.0),\n",
|
|||
|
|
" metrics=[\"accuracy\"])\n",
|
|||
|
|
"history = denoising_ae.fit(X_train, X_train, epochs=10,\n",
|
|||
|
|
" validation_data=[X_valid, X_valid])"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 31,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"show_reconstructions(denoising_ae)\n",
|
|||
|
|
"plt.show()"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {},
|
|||
|
|
"source": [
|
|||
|
|
"Using dropout:"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 32,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"tf.random.set_seed(42)\n",
|
|||
|
|
"np.random.seed(42)\n",
|
|||
|
|
"\n",
|
|||
|
|
"dropout_encoder = keras.models.Sequential([\n",
|
|||
|
|
" keras.layers.Flatten(input_shape=[28, 28]),\n",
|
|||
|
|
" keras.layers.Dropout(0.5),\n",
|
|||
|
|
" keras.layers.Dense(100, activation=\"selu\"),\n",
|
|||
|
|
" keras.layers.Dense(30, activation=\"selu\")\n",
|
|||
|
|
"])\n",
|
|||
|
|
"dropout_decoder = keras.models.Sequential([\n",
|
|||
|
|
" keras.layers.Dense(100, activation=\"selu\", input_shape=[30]),\n",
|
|||
|
|
" keras.layers.Dense(28 * 28, activation=\"sigmoid\"),\n",
|
|||
|
|
" keras.layers.Reshape([28, 28])\n",
|
|||
|
|
"])\n",
|
|||
|
|
"dropout_ae = keras.models.Sequential([dropout_encoder, dropout_decoder])\n",
|
|||
|
|
"dropout_ae.compile(loss=\"binary_crossentropy\", optimizer=keras.optimizers.SGD(lr=1.0),\n",
|
|||
|
|
" metrics=[\"accuracy\"])\n",
|
|||
|
|
"history = dropout_ae.fit(X_train, X_train, epochs=10,\n",
|
|||
|
|
" validation_data=[X_valid, X_valid])"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 33,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"show_reconstructions(dropout_ae)\n",
|
|||
|
|
"plt.show()"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {},
|
|||
|
|
"source": [
|
|||
|
|
"# Sparse Autoencoder"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {},
|
|||
|
|
"source": [
|
|||
|
|
"Let's build a simple stacked autoencoder, but this time we will use the sigmoid activation function for the coding layer, to ensure that the coding values range from 0 to 1:"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 34,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"tf.random.set_seed(42)\n",
|
|||
|
|
"np.random.seed(42)\n",
|
|||
|
|
"\n",
|
|||
|
|
"simple_encoder = keras.models.Sequential([\n",
|
|||
|
|
" keras.layers.Flatten(input_shape=[28, 28]),\n",
|
|||
|
|
" keras.layers.Dense(100, activation=\"selu\"),\n",
|
|||
|
|
" keras.layers.Dense(30, activation=\"sigmoid\"),\n",
|
|||
|
|
"])\n",
|
|||
|
|
"simple_decoder = keras.models.Sequential([\n",
|
|||
|
|
" keras.layers.Dense(100, activation=\"selu\", input_shape=[30]),\n",
|
|||
|
|
" keras.layers.Dense(28 * 28, activation=\"sigmoid\"),\n",
|
|||
|
|
" keras.layers.Reshape([28, 28])\n",
|
|||
|
|
"])\n",
|
|||
|
|
"simple_ae = keras.models.Sequential([simple_encoder, simple_decoder])\n",
|
|||
|
|
"simple_ae.compile(loss=\"binary_crossentropy\", optimizer=keras.optimizers.SGD(lr=1.),\n",
|
|||
|
|
" metrics=[\"accuracy\"])\n",
|
|||
|
|
"history = simple_ae.fit(X_train, X_train, epochs=10,\n",
|
|||
|
|
" validation_data=[X_valid, X_valid])"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 35,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"show_reconstructions(simple_ae)\n",
|
|||
|
|
"plt.show()"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {},
|
|||
|
|
"source": [
|
|||
|
|
"Let's create a couple functions to print nice activation histograms:"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 36,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"def plot_percent_hist(ax, data, bins):\n",
|
|||
|
|
" counts, _ = np.histogram(data, bins=bins)\n",
|
|||
|
|
" widths = bins[1:] - bins[:-1]\n",
|
|||
|
|
" x = bins[:-1] + widths / 2\n",
|
|||
|
|
" ax.bar(x, counts / len(data), width=widths*0.8)\n",
|
|||
|
|
" ax.xaxis.set_ticks(bins)\n",
|
|||
|
|
" ax.yaxis.set_major_formatter(mpl.ticker.FuncFormatter(\n",
|
|||
|
|
" lambda y, position: \"{}%\".format(int(np.round(100 * y)))))\n",
|
|||
|
|
" ax.grid(True)"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 37,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"def plot_activations_histogram(encoder, height=1, n_bins=10):\n",
|
|||
|
|
" X_valid_codings = encoder(X_valid).numpy()\n",
|
|||
|
|
" activation_means = X_valid_codings.mean(axis=0)\n",
|
|||
|
|
" mean = activation_means.mean()\n",
|
|||
|
|
" bins = np.linspace(0, 1, n_bins + 1)\n",
|
|||
|
|
"\n",
|
|||
|
|
" fig, [ax1, ax2] = plt.subplots(figsize=(10, 3), nrows=1, ncols=2, sharey=True)\n",
|
|||
|
|
" plot_percent_hist(ax1, X_valid_codings.ravel(), bins)\n",
|
|||
|
|
" ax1.plot([mean, mean], [0, height], \"k--\", label=\"Overall Mean = {:.2f}\".format(mean))\n",
|
|||
|
|
" ax1.legend(loc=\"upper center\", fontsize=14)\n",
|
|||
|
|
" ax1.set_xlabel(\"Activation\")\n",
|
|||
|
|
" ax1.set_ylabel(\"% Activations\")\n",
|
|||
|
|
" ax1.axis([0, 1, 0, height])\n",
|
|||
|
|
" plot_percent_hist(ax2, activation_means, bins)\n",
|
|||
|
|
" ax2.plot([mean, mean], [0, height], \"k--\")\n",
|
|||
|
|
" ax2.set_xlabel(\"Neuron Mean Activation\")\n",
|
|||
|
|
" ax2.set_ylabel(\"% Neurons\")\n",
|
|||
|
|
" ax2.axis([0, 1, 0, height])"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {},
|
|||
|
|
"source": [
|
|||
|
|
"Let's use these functions to plot histograms of the activations of the encoding layer. The histogram on the left shows the distribution of all the activations. You can see that values close to 0 or 1 are more frequent overall, which is consistent with the saturating nature of the sigmoid function. The histogram on the right shows the distribution of mean neuron activations: you can see that most neurons have a mean activation close to 0.5. Both histograms tell us that each neuron tends to either fire close to 0 or 1, with about 50% probability each. However, some neurons fire almost all the time (right side of the right histogram)."
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 38,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"plot_activations_histogram(simple_encoder, height=0.35)\n",
|
|||
|
|
"plt.show()"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {},
|
|||
|
|
"source": [
|
|||
|
|
"Now let's add $\\ell_1$ regularization to the coding layer:"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 39,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"tf.random.set_seed(42)\n",
|
|||
|
|
"np.random.seed(42)\n",
|
|||
|
|
"\n",
|
|||
|
|
"sparse_l1_encoder = keras.models.Sequential([\n",
|
|||
|
|
" keras.layers.Flatten(input_shape=[28, 28]),\n",
|
|||
|
|
" keras.layers.Dense(100, activation=\"selu\"),\n",
|
|||
|
|
" keras.layers.Dense(300, activation=\"sigmoid\"),\n",
|
|||
|
|
" keras.layers.ActivityRegularization(l1=1e-3) # Alternatively, you could add\n",
|
|||
|
|
" # activity_regularizer=keras.regularizers.l1(1e-3)\n",
|
|||
|
|
" # to the previous layer.\n",
|
|||
|
|
"])\n",
|
|||
|
|
"sparse_l1_decoder = keras.models.Sequential([\n",
|
|||
|
|
" keras.layers.Dense(100, activation=\"selu\", input_shape=[300]),\n",
|
|||
|
|
" keras.layers.Dense(28 * 28, activation=\"sigmoid\"),\n",
|
|||
|
|
" keras.layers.Reshape([28, 28])\n",
|
|||
|
|
"])\n",
|
|||
|
|
"sparse_l1_ae = keras.models.Sequential([sparse_l1_encoder, sparse_l1_decoder])\n",
|
|||
|
|
"sparse_l1_ae.compile(loss=\"binary_crossentropy\", optimizer=keras.optimizers.SGD(lr=1.0),\n",
|
|||
|
|
" metrics=[\"accuracy\"])\n",
|
|||
|
|
"history = sparse_l1_ae.fit(X_train, X_train, epochs=10,\n",
|
|||
|
|
" validation_data=[X_valid, X_valid])"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 40,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"show_reconstructions(sparse_l1_ae)"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 41,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"plot_activations_histogram(sparse_l1_encoder, height=1.)\n",
|
|||
|
|
"plt.show()"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {},
|
|||
|
|
"source": [
|
|||
|
|
"Let's use the KL Divergence loss instead to ensure sparsity, and target 10% sparsity rather than 0%:"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 42,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"p = 0.1\n",
|
|||
|
|
"q = np.linspace(0.001, 0.999, 500)\n",
|
|||
|
|
"kl_div = p * np.log(p / q) + (1 - p) * np.log((1 - p) / (1 - q))\n",
|
|||
|
|
"mse = (p - q)**2\n",
|
|||
|
|
"mae = np.abs(p - q)\n",
|
|||
|
|
"plt.plot([p, p], [0, 0.3], \"k:\")\n",
|
|||
|
|
"plt.text(0.05, 0.32, \"Target\\nsparsity\", fontsize=14)\n",
|
|||
|
|
"plt.plot(q, kl_div, \"b-\", label=\"KL divergence\")\n",
|
|||
|
|
"plt.plot(q, mae, \"g--\", label=r\"MAE ($\\ell_1$)\")\n",
|
|||
|
|
"plt.plot(q, mse, \"r--\", linewidth=1, label=r\"MSE ($\\ell_2$)\")\n",
|
|||
|
|
"plt.legend(loc=\"upper left\", fontsize=14)\n",
|
|||
|
|
"plt.xlabel(\"Actual sparsity\")\n",
|
|||
|
|
"plt.ylabel(\"Cost\", rotation=0)\n",
|
|||
|
|
"plt.axis([0, 1, 0, 0.95])\n",
|
|||
|
|
"save_fig(\"sparsity_loss_plot\")"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 43,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"K = keras.backend\n",
|
|||
|
|
"\n",
|
|||
|
|
"class KLDivergenceRegularizer(keras.regularizers.Regularizer):\n",
|
|||
|
|
" def __init__(self, weight, target=0.1):\n",
|
|||
|
|
" self.weight = weight\n",
|
|||
|
|
" self.target = target\n",
|
|||
|
|
" def __call__(self, inputs):\n",
|
|||
|
|
" mean_activities = K.mean(inputs, axis=0)\n",
|
|||
|
|
" return self.weight * (\n",
|
|||
|
|
" keras.losses.kullback_leibler_divergence(self.target, mean_activities) +\n",
|
|||
|
|
" keras.losses.kullback_leibler_divergence(1. - self.target, 1. - mean_activities))"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 44,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"tf.random.set_seed(42)\n",
|
|||
|
|
"np.random.seed(42)\n",
|
|||
|
|
"\n",
|
|||
|
|
"kld_reg = KLDivergenceRegularizer(weight=0.05, target=0.1)\n",
|
|||
|
|
"sparse_kl_encoder = keras.models.Sequential([\n",
|
|||
|
|
" keras.layers.Flatten(input_shape=[28, 28]),\n",
|
|||
|
|
" keras.layers.Dense(100, activation=\"selu\"),\n",
|
|||
|
|
" keras.layers.Dense(300, activation=\"sigmoid\", activity_regularizer=kld_reg)\n",
|
|||
|
|
"])\n",
|
|||
|
|
"sparse_kl_decoder = keras.models.Sequential([\n",
|
|||
|
|
" keras.layers.Dense(100, activation=\"selu\", input_shape=[300]),\n",
|
|||
|
|
" keras.layers.Dense(28 * 28, activation=\"sigmoid\"),\n",
|
|||
|
|
" keras.layers.Reshape([28, 28])\n",
|
|||
|
|
"])\n",
|
|||
|
|
"sparse_kl_ae = keras.models.Sequential([sparse_kl_encoder, sparse_kl_decoder])\n",
|
|||
|
|
"sparse_kl_ae.compile(loss=\"binary_crossentropy\", optimizer=keras.optimizers.SGD(lr=1.0),\n",
|
|||
|
|
" metrics=[\"accuracy\"])\n",
|
|||
|
|
"history = sparse_kl_ae.fit(X_train, X_train, epochs=10,\n",
|
|||
|
|
" validation_data=[X_valid, X_valid])"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 45,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"show_reconstructions(sparse_kl_ae)"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 46,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"plot_activations_histogram(sparse_kl_encoder)\n",
|
|||
|
|
"plt.show()"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {},
|
|||
|
|
"source": [
|
|||
|
|
"## Hashing Autoencoder"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 47,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"tf.random.set_seed(42)\n",
|
|||
|
|
"np.random.seed(42)\n",
|
|||
|
|
"\n",
|
|||
|
|
"hashing_encoder = keras.models.Sequential([\n",
|
|||
|
|
" keras.layers.Flatten(input_shape=[28, 28]),\n",
|
|||
|
|
" keras.layers.Dense(100, activation=\"selu\"),\n",
|
|||
|
|
" keras.layers.GaussianNoise(15.),\n",
|
|||
|
|
" keras.layers.Dense(16, activation=\"sigmoid\"),\n",
|
|||
|
|
"])\n",
|
|||
|
|
"hashing_decoder = keras.models.Sequential([\n",
|
|||
|
|
" keras.layers.Dense(100, activation=\"selu\", input_shape=[16]),\n",
|
|||
|
|
" keras.layers.Dense(28 * 28, activation=\"sigmoid\"),\n",
|
|||
|
|
" keras.layers.Reshape([28, 28])\n",
|
|||
|
|
"])\n",
|
|||
|
|
"hashing_ae = keras.models.Sequential([hashing_encoder, hashing_decoder])\n",
|
|||
|
|
"hashing_ae.compile(loss=\"binary_crossentropy\", optimizer=keras.optimizers.SGD(lr=1.0),\n",
|
|||
|
|
" metrics=[\"accuracy\"])\n",
|
|||
|
|
"history = hashing_ae.fit(X_train, X_train, epochs=10,\n",
|
|||
|
|
" validation_data=[X_valid, X_valid])"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 48,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"show_reconstructions(hashing_ae)\n",
|
|||
|
|
"plt.show()"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 49,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"plot_activations_histogram(hashing_encoder)\n",
|
|||
|
|
"plt.show()"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 50,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"hashes = np.round(hashing_encoder.predict(X_valid)).astype(np.int32)\n",
|
|||
|
|
"hashes *= np.array([[2**bit for bit in range(16)]])\n",
|
|||
|
|
"hashes = hashes.sum(axis=1)\n",
|
|||
|
|
"for h in hashes[:5]:\n",
|
|||
|
|
" print(\"{:016b}\".format(h))\n",
|
|||
|
|
"print(\"...\")"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 51,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"n_bits = 4\n",
|
|||
|
|
"n_images = 8\n",
|
|||
|
|
"plt.figure(figsize=(n_images, n_bits))\n",
|
|||
|
|
"for bit_index in range(n_bits):\n",
|
|||
|
|
" in_bucket = (hashes & 2**bit_index != 0)\n",
|
|||
|
|
" for index, image in zip(range(n_images), X_valid[in_bucket]):\n",
|
|||
|
|
" plt.subplot(n_bits, n_images, bit_index * n_images + index + 1)\n",
|
|||
|
|
" plt.imshow(image, cmap=\"binary\")\n",
|
|||
|
|
" plt.axis(\"off\")"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {},
|
|||
|
|
"source": [
|
|||
|
|
"# Variational Autoencoder"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 52,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"class Sampling(keras.layers.Layer):\n",
|
|||
|
|
" def call(self, inputs):\n",
|
|||
|
|
" mean, log_var = inputs\n",
|
|||
|
|
" return mean + K.exp(log_var / 2) * K.random_normal(shape=tf.shape(log_var))"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 53,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"tf.random.set_seed(42)\n",
|
|||
|
|
"np.random.seed(42)\n",
|
|||
|
|
"\n",
|
|||
|
|
"codings_size = 30\n",
|
|||
|
|
"\n",
|
|||
|
|
"inputs = keras.layers.Input(shape=[28, 28])\n",
|
|||
|
|
"z = keras.layers.Flatten()(inputs)\n",
|
|||
|
|
"z = keras.layers.Dense(150, activation=\"selu\")(z)\n",
|
|||
|
|
"z = keras.layers.Dense(100, activation=\"selu\")(z)\n",
|
|||
|
|
"codings_mean = keras.layers.Dense(codings_size)(z)\n",
|
|||
|
|
"codings_log_var = keras.layers.Dense(codings_size)(z)\n",
|
|||
|
|
"codings = Sampling()([codings_mean, codings_log_var])\n",
|
|||
|
|
"variational_encoder = keras.models.Model(inputs=[inputs], outputs=[codings_mean, codings_log_var, codings])\n",
|
|||
|
|
"\n",
|
|||
|
|
"decoder_inputs = keras.layers.Input(shape=[codings_size])\n",
|
|||
|
|
"x = keras.layers.Dense(100, activation=\"selu\")(decoder_inputs)\n",
|
|||
|
|
"x = keras.layers.Dense(150, activation=\"selu\")(x)\n",
|
|||
|
|
"x = keras.layers.Dense(28 * 28, activation=\"sigmoid\")(x)\n",
|
|||
|
|
"outputs = keras.layers.Reshape([28, 28])(x)\n",
|
|||
|
|
"variational_decoder = keras.models.Model(inputs=[decoder_inputs], outputs=[outputs])\n",
|
|||
|
|
"\n",
|
|||
|
|
"_, _, codings = variational_encoder(inputs)\n",
|
|||
|
|
"reconstructions = variational_decoder(codings)\n",
|
|||
|
|
"variational_ae = keras.models.Model(inputs=[inputs], outputs=[reconstructions])\n",
|
|||
|
|
"\n",
|
|||
|
|
"kld_loss = -0.5 * K.sum(1 + codings_log_var - K.exp(codings_log_var) - K.square(codings_mean), axis=-1)\n",
|
|||
|
|
"variational_ae.add_loss(K.mean(kld_loss) / 784.)\n",
|
|||
|
|
"variational_ae.compile(loss=\"binary_crossentropy\", optimizer=\"rmsprop\", metrics=[\"accuracy\"])\n",
|
|||
|
|
"history = variational_ae.fit(X_train, X_train, epochs=50,\n",
|
|||
|
|
" validation_data=[X_valid, X_valid],\n",
|
|||
|
|
" batch_size=128)"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 54,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"show_reconstructions(variational_ae)\n",
|
|||
|
|
"plt.show()"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {},
|
|||
|
|
"source": [
|
|||
|
|
"## Generate Fashion Images"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {},
|
|||
|
|
"source": [
|
|||
|
|
"Let's train the model and generate a few random fashion images:"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 55,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"tf.random.set_seed(42)\n",
|
|||
|
|
"np.random.seed(42)\n",
|
|||
|
|
"\n",
|
|||
|
|
"n_rows = 6\n",
|
|||
|
|
"n_cols = 10\n",
|
|||
|
|
"codings_rnd = np.random.normal(size=[n_rows * n_cols, codings_size])\n",
|
|||
|
|
"images = variational_decoder.predict(codings_rnd)"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 56,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"def plot_multiple_images(images, n_rows, n_cols, pad=2):\n",
|
|||
|
|
" images = images - images.min() # make the minimum == 0, so the padding looks white\n",
|
|||
|
|
" w,h = images.shape[1:]\n",
|
|||
|
|
" image = np.zeros(((w+pad)*n_rows+pad, (h+pad)*n_cols+pad))\n",
|
|||
|
|
" for y in range(n_rows):\n",
|
|||
|
|
" for x in range(n_cols):\n",
|
|||
|
|
" image[(y*(h+pad)+pad):(y*(h+pad)+pad+h),(x*(w+pad)+pad):(x*(w+pad)+pad+w)] = images[y*n_cols+x]\n",
|
|||
|
|
" plt.imshow(image, cmap=\"Greys\", interpolation=\"nearest\")\n",
|
|||
|
|
" plt.axis(\"off\")"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 57,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"plt.figure(figsize=(8, 5))\n",
|
|||
|
|
"plot_multiple_images(images, n_rows, n_cols)\n",
|
|||
|
|
"save_fig(\"generated_fashion_images_plot\")"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {},
|
|||
|
|
"source": [
|
|||
|
|
"## Encode & Decode"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 58,
|
|||
|
|
"metadata": {
|
|||
|
|
"scrolled": true
|
|||
|
|
},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"n_iterations = 3\n",
|
|||
|
|
"n_images = 10\n",
|
|||
|
|
"source_codings = np.random.normal(size=[n_images, codings_size])\n",
|
|||
|
|
"target_codings = np.roll(source_codings, -1, axis=0)\n",
|
|||
|
|
"images = []\n",
|
|||
|
|
"for iteration in range(n_iterations):\n",
|
|||
|
|
" codings_interpolate = source_codings + (target_codings - source_codings) * iteration / n_iterations\n",
|
|||
|
|
" images.append(variational_decoder(codings_interpolate).numpy())\n",
|
|||
|
|
"images = np.concatenate(images)\n",
|
|||
|
|
"\n",
|
|||
|
|
"plt.figure(figsize=(8, 3))\n",
|
|||
|
|
"plot_multiple_images(images, n_iterations, n_cols)"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": null,
|
|||
|
|
"metadata": {},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": []
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"metadata": {
|
|||
|
|
"kernelspec": {
|
|||
|
|
"display_name": "Python 3",
|
|||
|
|
"language": "python",
|
|||
|
|
"name": "python3"
|
|||
|
|
},
|
|||
|
|
"language_info": {
|
|||
|
|
"codemirror_mode": {
|
|||
|
|
"name": "ipython",
|
|||
|
|
"version": 3
|
|||
|
|
},
|
|||
|
|
"file_extension": ".py",
|
|||
|
|
"mimetype": "text/x-python",
|
|||
|
|
"name": "python",
|
|||
|
|
"nbconvert_exporter": "python",
|
|||
|
|
"pygments_lexer": "ipython3",
|
|||
|
|
"version": "3.6.8"
|
|||
|
|
},
|
|||
|
|
"nav_menu": {
|
|||
|
|
"height": "381px",
|
|||
|
|
"width": "453px"
|
|||
|
|
},
|
|||
|
|
"toc": {
|
|||
|
|
"navigate_menu": true,
|
|||
|
|
"number_sections": true,
|
|||
|
|
"sideBar": true,
|
|||
|
|
"threshold": 6,
|
|||
|
|
"toc_cell": false,
|
|||
|
|
"toc_section_display": "block",
|
|||
|
|
"toc_window_display": false
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"nbformat": 4,
|
|||
|
|
"nbformat_minor": 1
|
|||
|
|
}
|