diff --git a/16_nlp_with_rnns_and_attention.ipynb b/16_nlp_with_rnns_and_attention.ipynb index 901a069..9bbef1f 100644 --- a/16_nlp_with_rnns_and_attention.ipynb +++ b/16_nlp_with_rnns_and_attention.ipynb @@ -28,16 +28,6 @@ "" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# WORK IN PROGRESS\n", - "\n", - "\n", - "**I'm still working on updating this chapter to the 3rd edition. Please come back in a few weeks.**" - ] - }, { "cell_type": "markdown", "metadata": { @@ -59,7 +49,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": { "id": "TFSU3FCOpKzu" }, @@ -81,7 +71,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": { "id": "YqCwW7cMpKzw" }, @@ -103,7 +93,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": { "id": "0Piq5se2pKzx" }, @@ -125,7 +115,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": { "id": "8d4TH3NbpKzx" }, @@ -151,7 +141,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": { "id": "PQFH5Y9PpKzy" }, @@ -180,7 +170,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": { "id": "Ekxzo6pOpKzy" }, @@ -199,86 +189,42 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "This notebooks uses the TensorFlow Addons library, and the Transformers library. If you're running on Colab, then we need to install them now:" + "# Generating Shakespearean Text Using a Character RNN" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Creating the Training Dataset" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's download the Shakespeare data from Andrej Karpathy's [char-rnn project](https://github.com/karpathy/char-rnn/)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Downloading data from https://homl.info/shakespeare\n", + "1122304/1115394 [==============================] - 0s 0us/step\n", + "1130496/1115394 [==============================] - 0s 0us/step\n" + ] + } + ], "source": [ - "if \"google.colab\" in sys.modules:\n", - " %pip install -q -U tensorflow-addons\n", - " %pip install -q -U transformers" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "First, let's import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Char-RNN" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Splitting a sequence into batches of shuffled windows" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "For example, let's split the sequence 0 to 14 into windows of length 5, each shifted by 2 (e.g.,`[0, 1, 2, 3, 4]`, `[2, 3, 4, 5, 6]`, etc.), then shuffle them, and split them into inputs (the first 4 steps) and targets (the last 4 steps) (e.g., `[2, 3, 4, 5, 6]` would be split into `[[2, 3, 4, 5], [3, 4, 5, 6]]`), then create batches of 3 such input/target pairs:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "np.random.seed(42)\n", - "tf.random.set_seed(42)\n", + "import tensorflow as tf\n", "\n", - "n_steps = 5\n", - "dataset = tf.data.Dataset.from_tensor_slices(tf.range(15))\n", - "dataset = dataset.window(n_steps, shift=2, drop_remainder=True)\n", - "dataset = dataset.flat_map(lambda window: window.batch(n_steps))\n", - "dataset = dataset.shuffle(10).map(lambda window: (window[:-1], window[1:]))\n", - "dataset = dataset.batch(3).prefetch(1)\n", - "for index, (X_batch, Y_batch) in enumerate(dataset):\n", - " print(\"_\" * 20, \"Batch\", index, \"\\nX_batch\")\n", - " print(X_batch.numpy())\n", - " print(\"=\" * 5, \"\\nY_batch\")\n", - " print(Y_batch.numpy())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Loading the Data and Preparing the Dataset" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "shakespeare_url = \"https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt\"\n", + "shakespeare_url = \"https://homl.info/shakespeare\" # shortcut URL\n", "filepath = tf.keras.utils.get_file(\"shakespeare.txt\", shakespeare_url)\n", "with open(filepath) as f:\n", " shakespeare_text = f.read()" @@ -286,269 +232,440 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "First Citizen:\n", + "Before we proceed any further, hear me speak.\n", + "\n", + "All:\n", + "Speak, speak.\n" + ] + } + ], "source": [ - "print(shakespeare_text[:148])" + "# extra code – shows a short text sample\n", + "print(shakespeare_text[:80])" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "\"\\n !$&',-.3:;?abcdefghijklmnopqrstuvwxyz\"" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ + "# extra code – shows all 39 distinct characters (after converting to lower case)\n", "\"\".join(sorted(set(shakespeare_text.lower())))" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ - "tokenizer = tf.keras.preprocessing.text.Tokenizer(char_level=True)\n", - "tokenizer.fit_on_texts(shakespeare_text)" + "text_vec_layer = tf.keras.layers.TextVectorization(split=\"character\",\n", + " standardize=\"lower\")\n", + "text_vec_layer.adapt([shakespeare_text])\n", + "encoded = text_vec_layer([shakespeare_text])[0]" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "metadata": {}, "outputs": [], "source": [ - "tokenizer.texts_to_sequences([\"First\"])" + "encoded -= 2 # drop tokens 0 (pad) and 1 (unknown), which we will not use\n", + "n_tokens = text_vec_layer.vocabulary_size() - 2 # number of distinct chars = 39\n", + "dataset_size = len(encoded) # total number of chars = 1,115,394" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "39" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "tokenizer.sequences_to_texts([[20, 6, 9, 8, 3]])" + "n_tokens" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "1115394" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "max_id = len(tokenizer.word_index) # number of distinct characters\n", - "dataset_size = tokenizer.document_count # total number of characters" + "dataset_size" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "metadata": {}, "outputs": [], "source": [ - "[encoded] = np.array(tokenizer.texts_to_sequences([shakespeare_text])) - 1\n", - "train_size = dataset_size * 90 // 100\n", - "dataset = tf.data.Dataset.from_tensor_slices(encoded[:train_size])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**Note**: in previous versions of this code, we used `dataset.repeat()` now to make the dataset \"infinite\", and later in the notebook we set the `steps_per_epoch` argument when calling the `model.fit()` method. This was needed to work around some TensorFlow bugs. However, since these bugs have now been fixed, we can simplify the code: no need for `dataset.repeat()` or `steps_per_epoch` anymore." + "def to_dataset(sequence, length, shuffle=False, seed=None, batch_size=32):\n", + " ds = tf.data.Dataset.from_tensor_slices(sequence)\n", + " ds = ds.window(length + 1, shift=1, drop_remainder=True)\n", + " ds = ds.flat_map(lambda window_ds: window_ds.batch(length + 1))\n", + " if shuffle:\n", + " ds = ds.shuffle(100_000, seed=seed)\n", + " ds = ds.batch(batch_size)\n", + " return ds.map(lambda window: (window[:, :-1], window[:, 1:])).prefetch(1)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "[(,\n", + " )]" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "n_steps = 100\n", - "window_length = n_steps + 1 # target = input shifted 1 character ahead\n", - "dataset = dataset.window(window_length, shift=1, drop_remainder=True)" + "# extra code – a simple example using to_dataset()\n", + "# There's just one sample in this dataset: the input represents \"to b\" and the\n", + "# output represents \"o be\"\n", + "list(to_dataset(text_vec_layer([\"To be\"])[0], length=4))" ] }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "dataset = dataset.flat_map(lambda window: window.batch(window_length))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "np.random.seed(42)\n", - "tf.random.set_seed(42)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "batch_size = 32\n", - "dataset = dataset.shuffle(10000).batch(batch_size)\n", - "dataset = dataset.map(lambda windows: (windows[:, :-1], windows[:, 1:]))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "dataset = dataset.map(\n", - " lambda X_batch, Y_batch: (tf.one_hot(X_batch, depth=max_id), Y_batch))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "dataset = dataset.prefetch(1)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "for X_batch, Y_batch in dataset.take(1):\n", - " print(X_batch.shape, Y_batch.shape)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Creating and Training the Model" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**Warning**: the following code may take up to 24 hours to run, depending on your hardware. If you use a GPU, it may take just 1 or 2 hours, or less." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**Note**: the `GRU` class will only use the GPU (if you have one) when using the default values for the following arguments: `activation`, `recurrent_activation`, `recurrent_dropout`, `unroll`, `use_bias` and `reset_after`. This is why I commented out `recurrent_dropout=0.2` (compared to the book)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "model = tf.keras.Sequential([\n", - " tf.keras.layers.GRU(128, return_sequences=True, input_shape=[None, max_id],\n", - " #dropout=0.2, recurrent_dropout=0.2),\n", - " dropout=0.2),\n", - " tf.keras.layers.GRU(128, return_sequences=True,\n", - " #dropout=0.2, recurrent_dropout=0.2),\n", - " dropout=0.2),\n", - " tf.keras.layers.TimeDistributed(tf.keras.layers.Dense(max_id,\n", - " activation=\"softmax\"))\n", - "])\n", - "model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=\"adam\")\n", - "history = model.fit(dataset, epochs=10)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Using the Model to Generate Text" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def preprocess(texts):\n", - " X = np.array(tokenizer.texts_to_sequences(texts)) - 1\n", - " return tf.one_hot(X, max_id)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**Warning**: the `predict_classes()` method is deprecated. Instead, we must use `model(X_new).argmax(axis=-1)`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "X_new = preprocess([\"How are yo\"])\n", - "#Y_pred = model.predict_classes(X_new)\n", - "Y_pred = model(X_new).argmax(axis=-1)\n", - "tokenizer.sequences_to_texts(Y_pred + 1)[0][-1] # 1st sentence, last char" - ] - }, - { - "cell_type": "code", - "execution_count": null, + "execution_count": 16, "metadata": {}, "outputs": [], "source": [ + "length = 100\n", "tf.random.set_seed(42)\n", - "\n", - "tf.random.categorical([[np.log(0.5), np.log(0.4), np.log(0.1)]], num_samples=40).numpy()" + "train_set = to_dataset(encoded[:1_000_000], length=length, shuffle=True,\n", + " seed=42)\n", + "valid_set = to_dataset(encoded[1_000_000:1_060_000], length=length)\n", + "test_set = to_dataset(encoded[1_060_000:], length=length)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Building and Training the Char-RNN Model" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Warning**: the following code may one or two hours to run, depending on your GPU. Without a GPU, it may take over 24 hours. If you don't want to wait, just skip the next two code cells and run the code below to download a pretrained model." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Note**: the `GRU` class will only use cuDNN acceleration (assuming you have a GPU) when using the default values for the following arguments: `activation`, `recurrent_activation`, `recurrent_dropout`, `unroll`, `use_bias` and `reset_after`." ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 1/10\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Assets written to: my_shakespeare_model/assets\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "31247/31247 [==============================] - 1407s 45ms/step - loss: 1.3873 - accuracy: 0.5754 - val_loss: 1.6155 - val_accuracy: 0.5333\n", + "Epoch 2/10\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Assets written to: my_shakespeare_model/assets\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "31247/31247 [==============================] - 1376s 44ms/step - loss: 1.2921 - accuracy: 0.5973 - val_loss: 1.5881 - val_accuracy: 0.5401\n", + "Epoch 3/10\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Assets written to: my_shakespeare_model/assets\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "31247/31247 [==============================] - 1379s 44ms/step - loss: 1.2743 - accuracy: 0.6015 - val_loss: 1.5885 - val_accuracy: 0.5407\n", + "Epoch 4/10\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Assets written to: my_shakespeare_model/assets\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "31247/31247 [==============================] - 1381s 44ms/step - loss: 1.2654 - accuracy: 0.6031 - val_loss: 1.5701 - val_accuracy: 0.5418\n", + "Epoch 5/10\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Assets written to: my_shakespeare_model/assets\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "31247/31247 [==============================] - 1379s 44ms/step - loss: 1.2594 - accuracy: 0.6045 - val_loss: 1.5674 - val_accuracy: 0.5450\n", + "Epoch 6/10\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Assets written to: my_shakespeare_model/assets\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "31247/31247 [==============================] - 1386s 44ms/step - loss: 1.2545 - accuracy: 0.6058 - val_loss: 1.5587 - val_accuracy: 0.5492\n", + "Epoch 7/10\n", + "31247/31247 [==============================] - 1381s 44ms/step - loss: 1.2514 - accuracy: 0.6062 - val_loss: 1.5532 - val_accuracy: 0.5460\n", + "Epoch 8/10\n", + "31247/31247 [==============================] - 1381s 44ms/step - loss: 1.2485 - accuracy: 0.6067 - val_loss: 1.5522 - val_accuracy: 0.5479\n", + "Epoch 9/10\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Assets written to: my_shakespeare_model/assets\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "31247/31247 [==============================] - 1382s 44ms/step - loss: 1.2460 - accuracy: 0.6073 - val_loss: 1.5521 - val_accuracy: 0.5497\n", + "Epoch 10/10\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Assets written to: my_shakespeare_model/assets\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "31247/31247 [==============================] - 1385s 44ms/step - loss: 1.2436 - accuracy: 0.6080 - val_loss: 1.5477 - val_accuracy: 0.5513\n" + ] + } + ], + "source": [ + "tf.random.set_seed(42) # extra code – ensures reproducibility on CPU\n", + "model = tf.keras.Sequential([\n", + " tf.keras.layers.Embedding(input_dim=n_tokens, output_dim=16),\n", + " tf.keras.layers.GRU(128, return_sequences=True),\n", + " tf.keras.layers.Dense(n_tokens, activation=\"softmax\")\n", + "])\n", + "model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=\"nadam\",\n", + " metrics=[\"accuracy\"])\n", + "model_ckpt = tf.keras.callbacks.ModelCheckpoint(\n", + " \"my_shakespeare_model\", monitor=\"val_accuracy\", save_best_only=True)\n", + "history = model.fit(train_set, validation_data=valid_set, epochs=10,\n", + " callbacks=[model_ckpt])" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "shakespeare_model = tf.keras.Sequential([\n", + " text_vec_layer,\n", + " tf.keras.layers.Lambda(lambda X: X - 2), # no or tokens\n", + " model\n", + "])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If you don't want to wait for training to complete, I've pretrained a model for you. The following code will download it. Uncomment the last line if you want to use it instead of the model trained above." + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [], + "source": [ + "# extra code – downloads a pretrained model\n", + "url = \"https://github.com/ageron/data/raw/main/shakespeare_model.tgz\"\n", + "path = tf.keras.utils.get_file(\"shakespeare_model.tgz\", url, extract=True)\n", + "model_path = Path(path).with_name(\"shakespeare_model\")\n", + "#shakespeare_model = tf.keras.models.load_model(model_path)" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'e'" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "y_proba = shakespeare_model.predict([\"To be or not to b\"])[0, -1]\n", + "y_pred = tf.argmax(y_proba) # choose the most probable character ID\n", + "text_vec_layer.get_vocabulary()[y_pred + 2]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Generating Fake Shakespearean Text" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "log_probas = tf.math.log([[0.5, 0.4, 0.1]]) # probas = 50%, 40%, and 10%\n", + "tf.random.set_seed(42)\n", + "tf.random.categorical(log_probas, num_samples=8) # draw 8 samples" + ] + }, + { + "cell_type": "code", + "execution_count": 22, "metadata": {}, "outputs": [], "source": [ "def next_char(text, temperature=1):\n", - " X_new = preprocess([text])\n", - " y_proba = model(X_new)[0, -1:, :]\n", + " y_proba = shakespeare_model.predict([text])[0, -1:]\n", " rescaled_logits = tf.math.log(y_proba) / temperature\n", - " char_id = tf.random.categorical(rescaled_logits, num_samples=1) + 1\n", - " return tokenizer.sequences_to_texts(char_id.numpy())[0]" + " char_id = tf.random.categorical(rescaled_logits, num_samples=1)[0, 0]\n", + " return text_vec_layer.get_vocabulary()[char_id + 2]" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 23, "metadata": {}, "outputs": [], "source": [ - "tf.random.set_seed(42)\n", - "\n", - "next_char(\"How are yo\", temperature=1)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def complete_text(text, n_chars=50, temperature=1):\n", + "def extend_text(text, n_chars=50, temperature=1):\n", " for _ in range(n_chars):\n", " text += next_char(text, temperature)\n", " return text" @@ -556,31 +673,68 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 24, "metadata": {}, "outputs": [], "source": [ - "tf.random.set_seed(42)\n", - "\n", - "print(complete_text(\"t\", temperature=0.2))" + "tf.random.set_seed(42) # extra code – ensures reproducibility on CPU" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 25, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "To be or not to be the duke\n", + "as it is a proper strange death,\n", + "and the\n" + ] + } + ], "source": [ - "print(complete_text(\"t\", temperature=1))" + "print(extend_text(\"To be or not to be\", temperature=0.01))" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 26, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "To be or not to behold?\n", + "\n", + "second push:\n", + "gremio, lord all, a sistermen,\n" + ] + } + ], "source": [ - "print(complete_text(\"t\", temperature=2))" + "print(extend_text(\"To be or not to be\", temperature=1))" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "To be or not to bef ,mt'&o3fpadm!$\n", + "wh!nse?bws3est--vgerdjw?c-y-ewznq\n" + ] + } + ], + "source": [ + "print(extend_text(\"To be or not to be\", temperature=100))" ] }, { @@ -592,79 +746,124 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 28, "metadata": {}, "outputs": [], "source": [ - "tf.random.set_seed(42)" + "def to_dataset_for_stateful_rnn(sequence, length):\n", + " ds = tf.data.Dataset.from_tensor_slices(sequence)\n", + " ds = ds.window(length + 1, shift=length, drop_remainder=True)\n", + " ds = ds.flat_map(lambda window: window.batch(length + 1)).batch(1)\n", + " return ds.map(lambda window: (window[:, :-1], window[:, 1:])).prefetch(1)\n", + "\n", + "stateful_train_set = to_dataset_for_stateful_rnn(encoded[:1_000_000], length)\n", + "stateful_valid_set = to_dataset_for_stateful_rnn(encoded[:1_000_000], length)\n", + "stateful_test_set = to_dataset_for_stateful_rnn(encoded[1_060_000:], length)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 29, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "[(,\n", + " ),\n", + " (,\n", + " ),\n", + " (,\n", + " )]" + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "dataset = tf.data.Dataset.from_tensor_slices(encoded[:train_size])\n", - "dataset = dataset.window(window_length, shift=n_steps, drop_remainder=True)\n", - "dataset = dataset.flat_map(lambda window: window.batch(window_length))\n", - "dataset = dataset.batch(1)\n", - "dataset = dataset.map(lambda windows: (windows[:, :-1], windows[:, 1:]))\n", - "dataset = dataset.map(\n", - " lambda X_batch, Y_batch: (tf.one_hot(X_batch, depth=max_id), Y_batch))\n", - "dataset = dataset.prefetch(1)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "batch_size = 32\n", - "encoded_parts = np.array_split(encoded[:train_size], batch_size)\n", - "datasets = []\n", - "for encoded_part in encoded_parts:\n", - " dataset = tf.data.Dataset.from_tensor_slices(encoded_part)\n", - " dataset = dataset.window(window_length, shift=n_steps, drop_remainder=True)\n", - " dataset = dataset.flat_map(lambda window: window.batch(window_length))\n", - " datasets.append(dataset)\n", - "dataset = tf.data.Dataset.zip(tuple(datasets)).map(lambda *windows: tf.stack(windows))\n", - "dataset = dataset.map(lambda windows: (windows[:, :-1], windows[:, 1:]))\n", - "dataset = dataset.map(\n", - " lambda X_batch, Y_batch: (tf.one_hot(X_batch, depth=max_id), Y_batch))\n", - "dataset = dataset.prefetch(1)" + "# extra code – simple example using to_dataset_for_stateful_rnn()\n", + "list(to_dataset_for_stateful_rnn(tf.range(10), 3))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "**Note**: once again, I commented out `recurrent_dropout=0.2` (compared to the book) so you can get GPU acceleration (if you have one)." + "If you'd like to have more than one window per batch, you can use the `to_batched_dataset_for_stateful_rnn()` function instead of `to_dataset_for_stateful_rnn()`:" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 30, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[(,\n", + " ),\n", + " (,\n", + " ),\n", + " (,\n", + " )]" + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# extra code – shows one way to prepare a batched dataset for a stateful RNN\n", + "\n", + "import numpy as np\n", + "\n", + "def to_non_overlapping_windows(sequence, length):\n", + " ds = tf.data.Dataset.from_tensor_slices(sequence)\n", + " ds = ds.window(length + 1, shift=length, drop_remainder=True)\n", + " return ds.flat_map(lambda window: window.batch(length + 1))\n", + "\n", + "def to_batched_dataset_for_stateful_rnn(sequence, length, batch_size=32):\n", + " parts = np.array_split(sequence, batch_size)\n", + " datasets = tuple(to_non_overlapping_windows(part, length) for part in parts)\n", + " ds = tf.data.Dataset.zip(datasets).map(lambda *windows: tf.stack(windows))\n", + " return ds.map(lambda window: (window[:, :-1], window[:, 1:])).prefetch(1)\n", + "\n", + "list(to_batched_dataset_for_stateful_rnn(tf.range(20), length=3, batch_size=2))" + ] + }, + { + "cell_type": "code", + "execution_count": 31, "metadata": {}, "outputs": [], "source": [ + "tf.random.set_seed(42) # extra code – ensures reproducibility on CPU\n", "model = tf.keras.Sequential([\n", - " tf.keras.layers.GRU(128, return_sequences=True, stateful=True,\n", - " #dropout=0.2, recurrent_dropout=0.2,\n", - " dropout=0.2,\n", - " batch_input_shape=[batch_size, None, max_id]),\n", - " tf.keras.layers.GRU(128, return_sequences=True, stateful=True,\n", - " #dropout=0.2, recurrent_dropout=0.2),\n", - " dropout=0.2),\n", - " tf.keras.layers.TimeDistributed(tf.keras.layers.Dense(max_id,\n", - " activation=\"softmax\"))\n", + " tf.keras.layers.Embedding(input_dim=n_tokens, output_dim=16,\n", + " batch_input_shape=[1, None]),\n", + " tf.keras.layers.GRU(128, return_sequences=True, stateful=True),\n", + " tf.keras.layers.Dense(n_tokens, activation=\"softmax\")\n", "])" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 32, "metadata": {}, "outputs": [], "source": [ @@ -675,33 +874,197 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 33, "metadata": {}, "outputs": [], "source": [ - "model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=\"adam\")\n", - "history = model.fit(dataset, epochs=50,\n", - " callbacks=[ResetStatesCallback()])" + "# extra code – use a different directory to save the checkpoints\n", + "model_ckpt = tf.keras.callbacks.ModelCheckpoint(\n", + " \"my_stateful_shakespeare_model\",\n", + " monitor=\"val_accuracy\",\n", + " save_best_only=True)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "To use the model with different batch sizes, we need to create a stateless copy. We can get rid of dropout since it is only used during training:" + "**Warning**: the following cell will take a while to run (possibly an hour if you are not using a GPU)." ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 34, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Assets written to: my_stateful_shakespeare_model/assets\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "9999/9999 [==============================] - 213s 21ms/step - loss: 1.8690 - accuracy: 0.4494 - val_loss: 1.7632 - val_accuracy: 0.4672\n", + "Epoch 2/10\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Assets written to: my_stateful_shakespeare_model/assets\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "9999/9999 [==============================] - 211s 21ms/step - loss: 1.5635 - accuracy: 0.5284 - val_loss: 1.6334 - val_accuracy: 0.4994\n", + "Epoch 3/10\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Assets written to: my_stateful_shakespeare_model/assets\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "9999/9999 [==============================] - 209s 21ms/step - loss: 1.4875 - accuracy: 0.5478 - val_loss: 1.5788 - val_accuracy: 0.5153\n", + "Epoch 4/10\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Assets written to: my_stateful_shakespeare_model/assets\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "9999/9999 [==============================] - 208s 21ms/step - loss: 1.4483 - accuracy: 0.5579 - val_loss: 1.5471 - val_accuracy: 0.5236\n", + "Epoch 5/10\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Assets written to: my_stateful_shakespeare_model/assets\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "9999/9999 [==============================] - 213s 21ms/step - loss: 1.4241 - accuracy: 0.5643 - val_loss: 1.5270 - val_accuracy: 0.5286\n", + "Epoch 6/10\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Assets written to: my_stateful_shakespeare_model/assets\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "9999/9999 [==============================] - 215s 21ms/step - loss: 1.4074 - accuracy: 0.5686 - val_loss: 1.5109 - val_accuracy: 0.5338\n", + "Epoch 7/10\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Assets written to: my_stateful_shakespeare_model/assets\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "9999/9999 [==============================] - 210s 21ms/step - loss: 1.3953 - accuracy: 0.5714 - val_loss: 1.5008 - val_accuracy: 0.5361\n", + "Epoch 8/10\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Assets written to: my_stateful_shakespeare_model/assets\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "9999/9999 [==============================] - 212s 21ms/step - loss: 1.3863 - accuracy: 0.5737 - val_loss: 1.4938 - val_accuracy: 0.5381\n", + "Epoch 9/10\n", + "9999/9999 [==============================] - 207s 21ms/step - loss: 1.3790 - accuracy: 0.5757 - val_loss: 1.4890 - val_accuracy: 0.5380\n", + "Epoch 10/10\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Assets written to: my_stateful_shakespeare_model/assets\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "9999/9999 [==============================] - 208s 21ms/step - loss: 1.3729 - accuracy: 0.5770 - val_loss: 1.4786 - val_accuracy: 0.5420\n" + ] + } + ], + "source": [ + "model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=\"nadam\",\n", + " metrics=[\"accuracy\"])\n", + "history = model.fit(stateful_train_set, validation_data=stateful_valid_set,\n", + " epochs=10, callbacks=[ResetStatesCallback(), model_ckpt])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Extra Material: converting the stateful RNN to a stateless RNN and using it**" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To use the model with different batch sizes, we need to create a stateless copy:" + ] + }, + { + "cell_type": "code", + "execution_count": 35, "metadata": {}, "outputs": [], "source": [ "stateless_model = tf.keras.Sequential([\n", - " tf.keras.layers.GRU(128, return_sequences=True, input_shape=[None, max_id]),\n", + " tf.keras.layers.Embedding(input_dim=n_tokens, output_dim=16),\n", " tf.keras.layers.GRU(128, return_sequences=True),\n", - " tf.keras.layers.TimeDistributed(tf.keras.layers.Dense(max_id,\n", - " activation=\"softmax\"))\n", + " tf.keras.layers.Dense(n_tokens, activation=\"softmax\")\n", "])" ] }, @@ -714,32 +1077,53 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 36, "metadata": {}, "outputs": [], "source": [ - "stateless_model.build(tf.TensorShape([None, None, max_id]))" + "stateless_model.build(tf.TensorShape([None, None]))" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 37, "metadata": {}, "outputs": [], "source": [ - "stateless_model.set_weights(model.get_weights())\n", - "model = stateless_model" + "stateless_model.set_weights(model.get_weights())" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 38, "metadata": {}, "outputs": [], + "source": [ + "shakespeare_model = tf.keras.Sequential([\n", + " text_vec_layer,\n", + " tf.keras.layers.Lambda(lambda X: X - 2), # no or tokens\n", + " stateless_model\n", + "])" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "to be or not to be so in the world and the strangeness\n", + "to see the wo\n" + ] + } + ], "source": [ "tf.random.set_seed(42)\n", "\n", - "print(complete_text(\"t\"))" + "print(extend_text(\"to be or not to be\", temperature=0.01))" ] }, { @@ -751,243 +1135,286 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 40, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[1mDownloading and preparing dataset 80.23 MiB (download: 80.23 MiB, generated: Unknown size, total: 80.23 MiB) to /home/ageron/tensorflow_datasets/imdb_reviews/plain_text/1.0.0...\u001b[0m\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "055c0f544ac349d9a14da8f843651df0", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Dl Completed...: 0 url [00:00, ? url/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "e2abc244f4844d56919979b33cc2fa79", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Dl Size...: 0 MiB [00:00, ? MiB/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "af507eed124c4ff6900538205b1b00fd", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Generating splits...: 0%| | 0/3 [00:00
But come on Hollywood - a Moun ...\n", + "Label: 0\n", + "This is the kind of film for a snowy Sunday afternoon when the rest of the world can go ahead with its own business as you descend into a big arm-chair and mellow for a couple of hours. Wonderful perf ...\n", + "Label: 1\n" + ] + } + ], + "source": [ + "for review, label in raw_train_set.take(4):\n", + " print(review.numpy().decode(\"utf-8\")[:200], \"...\")\n", + " print(\"Label:\", label.numpy())" + ] + }, + { + "cell_type": "code", + "execution_count": 42, "metadata": {}, "outputs": [], "source": [ - "tf.random.set_seed(42)" + "vocab_size = 1000\n", + "text_vec_layer = tf.keras.layers.TextVectorization(max_tokens=vocab_size)\n", + "text_vec_layer.adapt(train_set.map(lambda reviews, labels: reviews))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "You can load the IMDB dataset easily:" + "**Warning**: the following cell will take a few minutes to run and the model will probably not learn anything because we didn't mask the padding tokens (that's the point of the next section)." ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 43, "metadata": {}, - "outputs": [], - "source": [ - "(X_train, y_train), (X_test, y_test) = tf.keras.datasets.imdb.load_data()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "X_train[0][:10]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "word_index = tf.keras.datasets.imdb.get_word_index()\n", - "id_to_word = {id_ + 3: word for word, id_ in word_index.items()}\n", - "for id_, token in enumerate((\"\", \"\", \"\")):\n", - " id_to_word[id_] = token\n", - "\" \".join([id_to_word[id_] for id_ in X_train[0][:10]])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import tensorflow_datasets as tfds\n", - "\n", - "datasets, info = tfds.load(\"imdb_reviews\", as_supervised=True, with_info=True)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "datasets.keys()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "train_size = info.splits[\"train\"].num_examples\n", - "test_size = info.splits[\"test\"].num_examples" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "train_size, test_size" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "for X_batch, y_batch in datasets[\"train\"].batch(2).take(1):\n", - " for review, label in zip(X_batch.numpy(), y_batch.numpy()):\n", - " print(\"Review:\", review.decode(\"utf-8\")[:200], \"...\")\n", - " print(\"Label:\", label, \"= Positive\" if label else \"= Negative\")\n", - " print()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def preprocess(X_batch, y_batch):\n", - " X_batch = tf.strings.substr(X_batch, 0, 300)\n", - " X_batch = tf.strings.regex_replace(X_batch, rb\"\", b\" \")\n", - " X_batch = tf.strings.regex_replace(X_batch, b\"[^a-zA-Z']\", b\" \")\n", - " X_batch = tf.strings.split(X_batch)\n", - " return X_batch.to_tensor(default_value=b\"\"), y_batch" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "preprocess(X_batch, y_batch)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from collections import Counter\n", - "\n", - "vocabulary = Counter()\n", - "for X_batch, y_batch in datasets[\"train\"].batch(32).map(preprocess):\n", - " for review in X_batch:\n", - " vocabulary.update(list(review.numpy()))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "vocabulary.most_common()[:3]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "len(vocabulary)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "vocab_size = 10000\n", - "truncated_vocabulary = [\n", - " word for word, count in vocabulary.most_common()[:vocab_size]]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "word_to_id = {word: index for index, word in enumerate(truncated_vocabulary)}\n", - "for word in b\"This movie was faaaaaantastic\".split():\n", - " print(word_to_id.get(word) or vocab_size)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "words = tf.constant(truncated_vocabulary)\n", - "word_ids = tf.range(len(truncated_vocabulary), dtype=tf.int64)\n", - "vocab_init = tf.lookup.KeyValueTensorInitializer(words, word_ids)\n", - "num_oov_buckets = 1000\n", - "table = tf.lookup.StaticVocabularyTable(vocab_init, num_oov_buckets)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "table.lookup(tf.constant([b\"This movie was faaaaaantastic\".split()]))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def encode_words(X_batch, y_batch):\n", - " return table.lookup(X_batch), y_batch\n", - "\n", - "train_set = datasets[\"train\"].batch(32).map(preprocess)\n", - "train_set = train_set.map(encode_words).prefetch(1)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "for X_batch, y_batch in train_set.take(1):\n", - " print(X_batch)\n", - " print(y_batch)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 1/2\n", + "704/704 [==============================] - 255s 359ms/step - loss: 0.6934 - accuracy: 0.4990 - val_loss: 0.6931 - val_accuracy: 0.5016\n", + "Epoch 2/2\n", + "704/704 [==============================] - 250s 355ms/step - loss: 0.6934 - accuracy: 0.5042 - val_loss: 0.6942 - val_accuracy: 0.5008\n" + ] + } + ], "source": [ "embed_size = 128\n", + "tf.random.set_seed(42)\n", "model = tf.keras.Sequential([\n", - " tf.keras.layers.Embedding(vocab_size + num_oov_buckets, embed_size,\n", - " mask_zero=True, # not shown in the book\n", - " input_shape=[None]),\n", - " tf.keras.layers.GRU(128, return_sequences=True),\n", + " text_vec_layer,\n", + " tf.keras.layers.Embedding(vocab_size, embed_size),\n", " tf.keras.layers.GRU(128),\n", " tf.keras.layers.Dense(1, activation=\"sigmoid\")\n", "])\n", - "model.compile(loss=\"binary_crossentropy\", optimizer=\"adam\", metrics=[\"accuracy\"])\n", - "history = model.fit(train_set, epochs=5)" + "model.compile(loss=\"binary_crossentropy\", optimizer=\"nadam\",\n", + " metrics=[\"accuracy\"])\n", + "history = model.fit(train_set, validation_data=valid_set, epochs=2)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Masking" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Warning**: the following cell will take a while to run (possibly 30 minutes if you are not using a GPU)." + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 1/5\n", + "704/704 [==============================] - 303s 426ms/step - loss: 0.5296 - accuracy: 0.7234 - val_loss: 0.4045 - val_accuracy: 0.8244\n", + "Epoch 2/5\n", + "704/704 [==============================] - 295s 419ms/step - loss: 0.3702 - accuracy: 0.8418 - val_loss: 0.3390 - val_accuracy: 0.8532\n", + "Epoch 3/5\n", + "704/704 [==============================] - 298s 423ms/step - loss: 0.3057 - accuracy: 0.8747 - val_loss: 0.3196 - val_accuracy: 0.8696\n", + "Epoch 4/5\n", + "704/704 [==============================] - 294s 418ms/step - loss: 0.2784 - accuracy: 0.8871 - val_loss: 0.3162 - val_accuracy: 0.8596\n", + "Epoch 5/5\n", + "704/704 [==============================] - 293s 417ms/step - loss: 0.2597 - accuracy: 0.8961 - val_loss: 0.3209 - val_accuracy: 0.8548\n" + ] + } + ], + "source": [ + "embed_size = 128\n", + "tf.random.set_seed(42)\n", + "model = tf.keras.Sequential([\n", + " text_vec_layer,\n", + " tf.keras.layers.Embedding(vocab_size, embed_size, mask_zero=True),\n", + " tf.keras.layers.GRU(128),\n", + " tf.keras.layers.Dense(1, activation=\"sigmoid\")\n", + "])\n", + "model.compile(loss=\"binary_crossentropy\", optimizer=\"nadam\",\n", + " metrics=[\"accuracy\"])\n", + "history = model.fit(train_set, validation_data=valid_set, epochs=5)" ] }, { @@ -999,405 +1426,1497 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 45, "metadata": {}, "outputs": [], "source": [ - "K = tf.keras.backend\n", - "embed_size = 128\n", - "inputs = tf.keras.layers.Input(shape=[None])\n", - "mask = tf.keras.layers.Lambda(lambda inputs: K.not_equal(inputs, 0))(inputs)\n", - "z = tf.keras.layers.Embedding(vocab_size + num_oov_buckets, embed_size)(inputs)\n", - "z = tf.keras.layers.GRU(128, return_sequences=True)(z, mask=mask)\n", - "z = tf.keras.layers.GRU(128)(z, mask=mask)\n", - "outputs = tf.keras.layers.Dense(1, activation=\"sigmoid\")(z)\n", - "model = tf.keras.Model(inputs=[inputs], outputs=[outputs])\n", - "model.compile(loss=\"binary_crossentropy\", optimizer=\"adam\", metrics=[\"accuracy\"])\n", - "history = model.fit(train_set, epochs=5)" + "tf.random.set_seed(42) # extra code – ensures reproducibility on the CPU\n", + "inputs = tf.keras.layers.Input(shape=[], dtype=tf.string)\n", + "token_ids = text_vec_layer(inputs)\n", + "mask = tf.math.not_equal(token_ids, 0)\n", + "Z = tf.keras.layers.Embedding(vocab_size, embed_size)(token_ids)\n", + "Z = tf.keras.layers.GRU(128, dropout=0.2)(Z, mask=mask)\n", + "outputs = tf.keras.layers.Dense(1, activation=\"sigmoid\")(Z)\n", + "model = tf.keras.Model(inputs=[inputs], outputs=[outputs])" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Reusing Pretrained Embeddings" + "**Warning**: the following cell will take a while to run (possibly 30 minutes if you are not using a GPU)." ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 46, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 1/5\n", + "704/704 [==============================] - 303s 427ms/step - loss: 0.5447 - accuracy: 0.7198 - val_loss: 0.4604 - val_accuracy: 0.7720\n", + "Epoch 2/5\n", + "704/704 [==============================] - 301s 427ms/step - loss: 0.3469 - accuracy: 0.8512 - val_loss: 0.3214 - val_accuracy: 0.8608\n", + "Epoch 3/5\n", + "704/704 [==============================] - 295s 419ms/step - loss: 0.3054 - accuracy: 0.8713 - val_loss: 0.3069 - val_accuracy: 0.8672\n", + "Epoch 4/5\n", + "704/704 [==============================] - 295s 420ms/step - loss: 0.2798 - accuracy: 0.8828 - val_loss: 0.3028 - val_accuracy: 0.8672\n", + "Epoch 5/5\n", + "704/704 [==============================] - 298s 423ms/step - loss: 0.2622 - accuracy: 0.8920 - val_loss: 0.2953 - val_accuracy: 0.8700\n" + ] + } + ], "source": [ - "tf.random.set_seed(42)" + "# extra code – compiles and trains the model, as usual\n", + "model.compile(loss=\"binary_crossentropy\", optimizer=\"nadam\",\n", + " metrics=[\"accuracy\"])\n", + "history = model.fit(train_set, validation_data=valid_set, epochs=5)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Extra material: using ragged tensors**" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 47, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 47, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "tfhub_cache_dir = Path() / \"my_tfhub_cache\"\n", - "os.environ[\"TFHUB_CACHE_DIR\"] = str(tfhub_cache_dir)" + "text_vec_layer_ragged = tf.keras.layers.TextVectorization(\n", + " max_tokens=vocab_size, ragged=True)\n", + "text_vec_layer_ragged.adapt(train_set.map(lambda reviews, labels: reviews))\n", + "text_vec_layer_ragged([\"Great movie!\", \"This is DiCaprio's best role.\"])" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 48, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 48, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "import tensorflow_hub as hub\n", - "\n", + "text_vec_layer([\"Great movie!\", \"This is DiCaprio's best role.\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Warning**: the following cell will take a while to run (possibly 30 minutes if you are not using a GPU)." + ] + }, + { + "cell_type": "code", + "execution_count": 49, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 1/5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "704/704 [==============================] - 280s 395ms/step - loss: 0.5038 - accuracy: 0.7496 - val_loss: 0.6706 - val_accuracy: 0.6752\n", + "Epoch 2/5\n", + "704/704 [==============================] - 277s 393ms/step - loss: 0.4499 - accuracy: 0.7892 - val_loss: 0.3494 - val_accuracy: 0.8500\n", + "Epoch 3/5\n", + "704/704 [==============================] - 276s 392ms/step - loss: 0.3270 - accuracy: 0.8592 - val_loss: 0.3855 - val_accuracy: 0.8260\n", + "Epoch 4/5\n", + "704/704 [==============================] - 277s 394ms/step - loss: 0.2935 - accuracy: 0.8760 - val_loss: 0.3401 - val_accuracy: 0.8520\n", + "Epoch 5/5\n", + "704/704 [==============================] - 275s 390ms/step - loss: 0.2742 - accuracy: 0.8854 - val_loss: 0.3971 - val_accuracy: 0.8208\n" + ] + } + ], + "source": [ + "embed_size = 128\n", + "tf.random.set_seed(42)\n", "model = tf.keras.Sequential([\n", - " hub.KerasLayer(\"https://tfhub.dev/google/tf2-preview/nnlm-en-dim50/1\",\n", - " dtype=tf.string, input_shape=[], output_shape=[50]),\n", - " tf.keras.layers.Dense(128, activation=\"relu\"),\n", + " text_vec_layer_ragged,\n", + " tf.keras.layers.Embedding(vocab_size, embed_size),\n", + " tf.keras.layers.GRU(128),\n", " tf.keras.layers.Dense(1, activation=\"sigmoid\")\n", "])\n", - "model.compile(loss=\"binary_crossentropy\", optimizer=\"adam\",\n", - " metrics=[\"accuracy\"])" + "model.compile(loss=\"binary_crossentropy\", optimizer=\"nadam\",\n", + " metrics=[\"accuracy\"])\n", + "history = model.fit(train_set, validation_data=valid_set, epochs=5)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Let's define a `tree()` function to view the structure of the cache directory TF Hub just created:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def tree(path, level=0, indent=4):\n", - " if level == 0:\n", - " print(f\"{path}/\")\n", - " level += 1\n", - " sub_paths = sorted(path.iterdir())\n", - " sub_dirs = [sub_path for sub_path in sub_paths if sub_path.is_dir()]\n", - " filepaths = [sub_path for sub_path in sub_paths if not sub_path in sub_dirs]\n", - " indent_str = \" \" * indent * level\n", - " for sub_dir in sub_dirs:\n", - " print(f\"{indent_str}{sub_dir.name}/\")\n", - " tree(sub_dir, level + 1, indent)\n", - " for filepath in filepaths:\n", - " print(f\"{indent_str}{filepath.name}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "tree(tfhub_cache_dir)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import tensorflow_datasets as tfds\n", - "\n", - "datasets, info = tfds.load(\"imdb_reviews\", as_supervised=True, with_info=True)\n", - "train_size = info.splits[\"train\"].num_examples\n", - "batch_size = 32\n", - "train_set = datasets[\"train\"].batch(batch_size).prefetch(1)\n", - "history = model.fit(train_set, epochs=5)" + "## Reusing Pretrained Embeddings and Language Models" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Automatic Translation" + "**Warning**: the following cell will take a while to run (possibly an hour if you are not using a GPU)." ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 50, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 1/10\n", + "704/704 [==============================] - 224s 303ms/step - loss: 0.3141 - accuracy: 0.8648 - val_loss: 0.2397 - val_accuracy: 0.9008\n", + "Epoch 2/10\n", + "704/704 [==============================] - 205s 291ms/step - loss: 0.0489 - accuracy: 0.9852 - val_loss: 0.3257 - val_accuracy: 0.8936\n", + "Epoch 3/10\n", + "704/704 [==============================] - 204s 290ms/step - loss: 0.0061 - accuracy: 0.9988 - val_loss: 0.3963 - val_accuracy: 0.8944\n", + "Epoch 4/10\n", + "704/704 [==============================] - 204s 290ms/step - loss: 9.4918e-04 - accuracy: 0.9999 - val_loss: 0.4291 - val_accuracy: 0.8924\n", + "Epoch 5/10\n", + "704/704 [==============================] - 203s 289ms/step - loss: 5.1920e-04 - accuracy: 1.0000 - val_loss: 0.4691 - val_accuracy: 0.8932\n", + "Epoch 6/10\n", + "704/704 [==============================] - 204s 289ms/step - loss: 5.0053e-04 - accuracy: 1.0000 - val_loss: 0.4687 - val_accuracy: 0.8912\n", + "Epoch 7/10\n", + "704/704 [==============================] - 208s 296ms/step - loss: 3.7360e-04 - accuracy: 1.0000 - val_loss: 0.5034 - val_accuracy: 0.8984\n", + "Epoch 8/10\n", + "704/704 [==============================] - 209s 297ms/step - loss: 2.3907e-05 - accuracy: 1.0000 - val_loss: 0.5773 - val_accuracy: 0.8924\n", + "Epoch 9/10\n", + "704/704 [==============================] - 204s 290ms/step - loss: 9.0970e-06 - accuracy: 1.0000 - val_loss: 0.6163 - val_accuracy: 0.8972\n", + "Epoch 10/10\n", + "704/704 [==============================] - 205s 291ms/step - loss: 5.2528e-06 - accuracy: 1.0000 - val_loss: 0.6455 - val_accuracy: 0.8956\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 50, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "tf.random.set_seed(42)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "vocab_size = 100\n", - "embed_size = 10" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import tensorflow_addons as tfa\n", + "import os\n", + "import tensorflow_hub as hub\n", "\n", - "encoder_inputs = tf.keras.layers.Input(shape=[None], dtype=np.int32)\n", - "decoder_inputs = tf.keras.layers.Input(shape=[None], dtype=np.int32)\n", - "sequence_lengths = tf.keras.layers.Input(shape=[], dtype=np.int32)\n", - "\n", - "embeddings = tf.keras.layers.Embedding(vocab_size, embed_size)\n", - "encoder_embeddings = embeddings(encoder_inputs)\n", - "decoder_embeddings = embeddings(decoder_inputs)\n", - "\n", - "encoder = tf.keras.layers.LSTM(512, return_state=True)\n", - "encoder_outputs, state_h, state_c = encoder(encoder_embeddings)\n", - "encoder_state = [state_h, state_c]\n", - "\n", - "sampler = tfa.seq2seq.sampler.TrainingSampler()\n", - "\n", - "decoder_cell = tf.keras.layers.LSTMCell(512)\n", - "output_layer = tf.keras.layers.Dense(vocab_size)\n", - "decoder = tfa.seq2seq.basic_decoder.BasicDecoder(decoder_cell, sampler,\n", - " output_layer=output_layer)\n", - "final_outputs, final_state, final_sequence_lengths = decoder(\n", - " decoder_embeddings, initial_state=encoder_state,\n", - " sequence_length=sequence_lengths)\n", - "Y_proba = tf.nn.softmax(final_outputs.rnn_output)\n", - "\n", - "model = tf.keras.Model(\n", - " inputs=[encoder_inputs, decoder_inputs, sequence_lengths],\n", - " outputs=[Y_proba])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=\"adam\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "X = np.random.randint(100, size=1000 * 10).reshape(1000, 10)\n", - "Y = np.random.randint(100, size=1000 * 15).reshape(1000, 15)\n", - "X_decoder = np.c_[np.zeros((1000, 1)), Y[:, :-1]]\n", - "seq_lengths = np.full([1000], 10)\n", - "\n", - "history = model.fit([X, X_decoder, seq_lengths], Y, epochs=2)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Bidirectional Recurrent Layers" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ + "os.environ[\"TFHUB_CACHE_DIR\"] = \"my_tfhub_cache\"\n", + "tf.random.set_seed(42) # extra code – ensures reproducibility on CPU\n", "model = tf.keras.Sequential([\n", - " tf.keras.layers.GRU(10, return_sequences=True, input_shape=[None, 10]),\n", - " tf.keras.layers.Bidirectional(tf.keras.layers.GRU(10, return_sequences=True))\n", + " hub.KerasLayer(\"https://tfhub.dev/google/universal-sentence-encoder/4\",\n", + " trainable=True, dtype=tf.string, input_shape=[]),\n", + " tf.keras.layers.Dense(64, activation=\"relu\"),\n", + " tf.keras.layers.Dense(1, activation=\"sigmoid\")\n", "])\n", - "\n", - "model.summary()" + "model.compile(loss=\"binary_crossentropy\", optimizer=\"nadam\",\n", + " metrics=[\"accuracy\"])\n", + "model.fit(train_set, validation_data=valid_set, epochs=10)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### Positional Encoding" + "# An Encoder–Decoder Network for Neural Machine Translation" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 51, + "metadata": {}, + "outputs": [], + "source": [ + "url = \"https://storage.googleapis.com/download.tensorflow.org/data/spa-eng.zip\"\n", + "path = tf.keras.utils.get_file(\"spa-eng.zip\", origin=url, cache_dir=\"datasets\",\n", + " extract=True)\n", + "text = (Path(path).with_name(\"spa-eng\") / \"spa.txt\").read_text()" + ] + }, + { + "cell_type": "code", + "execution_count": 52, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "\n", + "text = text.replace(\"¡\", \"\").replace(\"¿\", \"\")\n", + "pairs = [line.split(\"\\t\") for line in text.splitlines()]\n", + "np.random.seed(42) # extra code – ensures reproducibility on CPU\n", + "np.random.shuffle(pairs)\n", + "sentences_en, sentences_es = zip(*pairs) # separates the pairs into 2 lists" + ] + }, + { + "cell_type": "code", + "execution_count": 53, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "How boring! => Qué aburrimiento!\n", + "I love sports. => Adoro el deporte.\n", + "Would you like to swap jobs? => Te gustaría que intercambiemos los trabajos?\n" + ] + } + ], + "source": [ + "for i in range(3):\n", + " print(sentences_en[i], \"=>\", sentences_es[i])" + ] + }, + { + "cell_type": "code", + "execution_count": 54, + "metadata": {}, + "outputs": [], + "source": [ + "vocab_size = 1000\n", + "max_length = 50\n", + "text_vec_layer_en = tf.keras.layers.TextVectorization(\n", + " vocab_size, output_sequence_length=max_length)\n", + "text_vec_layer_es = tf.keras.layers.TextVectorization(\n", + " vocab_size, output_sequence_length=max_length)\n", + "text_vec_layer_en.adapt(sentences_en)\n", + "text_vec_layer_es.adapt([f\"startofseq {s} endofseq\" for s in sentences_es])" + ] + }, + { + "cell_type": "code", + "execution_count": 55, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['', '[UNK]', 'the', 'i', 'to', 'you', 'tom', 'a', 'is', 'he']" + ] + }, + "execution_count": 55, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "text_vec_layer_en.get_vocabulary()[:10]" + ] + }, + { + "cell_type": "code", + "execution_count": 56, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['', '[UNK]', 'startofseq', 'endofseq', 'de', 'que', 'a', 'no', 'tom', 'la']" + ] + }, + "execution_count": 56, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "text_vec_layer_es.get_vocabulary()[:10]" + ] + }, + { + "cell_type": "code", + "execution_count": 57, + "metadata": {}, + "outputs": [], + "source": [ + "X_train = tf.constant(sentences_en[:100_000])\n", + "X_valid = tf.constant(sentences_en[100_000:])\n", + "X_train_dec = tf.constant([f\"startofseq {s}\" for s in sentences_es[:100_000]])\n", + "X_valid_dec = tf.constant([f\"startofseq {s}\" for s in sentences_es[100_000:]])\n", + "Y_train = text_vec_layer_es([f\"{s} endofseq\" for s in sentences_es[:100_000]])\n", + "Y_valid = text_vec_layer_es([f\"{s} endofseq\" for s in sentences_es[100_000:]])" + ] + }, + { + "cell_type": "code", + "execution_count": 58, + "metadata": {}, + "outputs": [], + "source": [ + "tf.random.set_seed(42) # extra code – ensures reproducibility on CPU\n", + "encoder_inputs = tf.keras.layers.Input(shape=[], dtype=tf.string)\n", + "decoder_inputs = tf.keras.layers.Input(shape=[], dtype=tf.string)" + ] + }, + { + "cell_type": "code", + "execution_count": 59, + "metadata": {}, + "outputs": [], + "source": [ + "embed_size = 128\n", + "encoder_input_ids = text_vec_layer_en(encoder_inputs)\n", + "decoder_input_ids = text_vec_layer_es(decoder_inputs)\n", + "encoder_embedding_layer = tf.keras.layers.Embedding(vocab_size, embed_size,\n", + " mask_zero=True)\n", + "decoder_embedding_layer = tf.keras.layers.Embedding(vocab_size, embed_size,\n", + " mask_zero=True)\n", + "encoder_embeddings = encoder_embedding_layer(encoder_input_ids)\n", + "decoder_embeddings = decoder_embedding_layer(decoder_input_ids)" + ] + }, + { + "cell_type": "code", + "execution_count": 60, + "metadata": {}, + "outputs": [], + "source": [ + "encoder = tf.keras.layers.LSTM(512, return_state=True)\n", + "encoder_outputs, *encoder_state = encoder(encoder_embeddings)" + ] + }, + { + "cell_type": "code", + "execution_count": 61, + "metadata": {}, + "outputs": [], + "source": [ + "decoder = tf.keras.layers.LSTM(512, return_sequences=True)\n", + "decoder_outputs = decoder(decoder_embeddings, initial_state=encoder_state)" + ] + }, + { + "cell_type": "code", + "execution_count": 62, + "metadata": {}, + "outputs": [], + "source": [ + "output_layer = tf.keras.layers.Dense(vocab_size, activation=\"softmax\")\n", + "Y_proba = output_layer(decoder_outputs)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Warning**: the following cell will take a while to run (possibly a couple hours if you are not using a GPU)." + ] + }, + { + "cell_type": "code", + "execution_count": 63, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 1/10\n", + "3125/3125 [==============================] - 698s 221ms/step - loss: 0.4154 - accuracy: 0.4256 - val_loss: 0.3069 - val_accuracy: 0.5246\n", + "Epoch 2/10\n", + "3125/3125 [==============================] - 686s 219ms/step - loss: 0.2631 - accuracy: 0.5745 - val_loss: 0.2367 - val_accuracy: 0.6055\n", + "Epoch 3/10\n", + "3125/3125 [==============================] - 686s 220ms/step - loss: 0.2066 - accuracy: 0.6457 - val_loss: 0.2061 - val_accuracy: 0.6500\n", + "Epoch 4/10\n", + "3125/3125 [==============================] - 682s 218ms/step - loss: 0.1740 - accuracy: 0.6907 - val_loss: 0.1920 - val_accuracy: 0.6691\n", + "Epoch 5/10\n", + "3125/3125 [==============================] - 676s 216ms/step - loss: 0.1507 - accuracy: 0.7237 - val_loss: 0.1865 - val_accuracy: 0.6767\n", + "Epoch 6/10\n", + "3125/3125 [==============================] - 675s 216ms/step - loss: 0.1316 - accuracy: 0.7522 - val_loss: 0.1847 - val_accuracy: 0.6804\n", + "Epoch 7/10\n", + "3125/3125 [==============================] - 675s 216ms/step - loss: 0.1154 - accuracy: 0.7774 - val_loss: 0.1866 - val_accuracy: 0.6822\n", + "Epoch 8/10\n", + "3125/3125 [==============================] - 673s 215ms/step - loss: 0.1011 - accuracy: 0.8007 - val_loss: 0.1907 - val_accuracy: 0.6829\n", + "Epoch 9/10\n", + "3125/3125 [==============================] - 673s 215ms/step - loss: 0.0888 - accuracy: 0.8215 - val_loss: 0.1961 - val_accuracy: 0.6792\n", + "Epoch 10/10\n", + "3125/3125 [==============================] - 673s 215ms/step - loss: 0.0782 - accuracy: 0.8402 - val_loss: 0.2027 - val_accuracy: 0.6763\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 63, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model = tf.keras.Model(inputs=[encoder_inputs, decoder_inputs],\n", + " outputs=[Y_proba])\n", + "model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=\"nadam\",\n", + " metrics=[\"accuracy\"])\n", + "model.fit((X_train, X_train_dec), Y_train, epochs=10,\n", + " validation_data=((X_valid, X_valid_dec), Y_valid))" + ] + }, + { + "cell_type": "code", + "execution_count": 64, + "metadata": {}, + "outputs": [], + "source": [ + "def translate(sentence_en):\n", + " translation = \"\"\n", + " for word_idx in range(max_length):\n", + " X = np.array([sentence_en]) # encoder input \n", + " X_dec = np.array([\"startofseq \" + translation]) # decoder input\n", + " y_proba = model.predict((X, X_dec))[0, word_idx] # last token's probas\n", + " predicted_word_id = np.argmax(y_proba)\n", + " predicted_word = text_vec_layer_es.get_vocabulary()[predicted_word_id]\n", + " if predicted_word == \"endofseq\":\n", + " break\n", + " translation += \" \" + predicted_word\n", + " return translation.strip()" + ] + }, + { + "cell_type": "code", + "execution_count": 65, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'me gusta el fútbol'" + ] + }, + "execution_count": 65, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "translate(\"I like soccer\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Nice! However, the model struggles with longer sentences:" + ] + }, + { + "cell_type": "code", + "execution_count": 66, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'me gusta el fútbol y a veces mismo al bus'" + ] + }, + "execution_count": 66, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "translate(\"I like soccer and also going to the beach\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Bidirectional RNNs" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To create a bidirectional recurrent layer, just wrap a regular recurrent layer in a `Bidirectional` layer:" + ] + }, + { + "cell_type": "code", + "execution_count": 67, + "metadata": {}, + "outputs": [], + "source": [ + "tf.random.set_seed(42) # extra code – ensures reproducibility on CPU\n", + "encoder = tf.keras.layers.Bidirectional(\n", + " tf.keras.layers.LSTM(256, return_state=True))" + ] + }, + { + "cell_type": "code", + "execution_count": 68, + "metadata": {}, + "outputs": [], + "source": [ + "encoder_outputs, *encoder_state = encoder(encoder_embeddings)\n", + "encoder_state = [tf.concat(encoder_state[::2], axis=-1), # short-term (0 & 2)\n", + " tf.concat(encoder_state[1::2], axis=-1)] # long-term (1 & 3)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Warning**: the following cell will take a while to run (possibly a couple hours if you are not using a GPU)." + ] + }, + { + "cell_type": "code", + "execution_count": 69, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 1/10\n", + "3125/3125 [==============================] - 574s 181ms/step - loss: 0.3075 - accuracy: 0.5393 - val_loss: 0.2192 - val_accuracy: 0.6319\n", + "Epoch 2/10\n", + "3125/3125 [==============================] - 564s 180ms/step - loss: 0.1916 - accuracy: 0.6689 - val_loss: 0.1880 - val_accuracy: 0.6731\n", + "Epoch 3/10\n", + "3125/3125 [==============================] - 566s 181ms/step - loss: 0.1602 - accuracy: 0.7119 - val_loss: 0.1751 - val_accuracy: 0.6916\n", + "Epoch 4/10\n", + "3125/3125 [==============================] - 566s 181ms/step - loss: 0.1395 - accuracy: 0.7415 - val_loss: 0.1715 - val_accuracy: 0.6979\n", + "Epoch 5/10\n", + "3125/3125 [==============================] - 566s 181ms/step - loss: 0.1227 - accuracy: 0.7666 - val_loss: 0.1707 - val_accuracy: 0.7025\n", + "Epoch 6/10\n", + "3125/3125 [==============================] - 567s 181ms/step - loss: 0.1085 - accuracy: 0.7887 - val_loss: 0.1730 - val_accuracy: 0.6995\n", + "Epoch 7/10\n", + "3125/3125 [==============================] - 571s 183ms/step - loss: 0.0961 - accuracy: 0.8089 - val_loss: 0.1764 - val_accuracy: 0.7000\n", + "Epoch 8/10\n", + "3125/3125 [==============================] - 567s 181ms/step - loss: 0.0852 - accuracy: 0.8273 - val_loss: 0.1821 - val_accuracy: 0.6981\n", + "Epoch 9/10\n", + "3125/3125 [==============================] - 565s 181ms/step - loss: 0.0759 - accuracy: 0.8438 - val_loss: 0.1881 - val_accuracy: 0.6956\n", + "Epoch 10/10\n", + "3125/3125 [==============================] - 565s 181ms/step - loss: 0.0682 - accuracy: 0.8577 - val_loss: 0.1951 - val_accuracy: 0.6906\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 69, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# extra code — completes the model and trains it\n", + "decoder = tf.keras.layers.LSTM(512, return_sequences=True)\n", + "decoder_outputs = decoder(decoder_embeddings, initial_state=encoder_state)\n", + "output_layer = tf.keras.layers.Dense(vocab_size, activation=\"softmax\")\n", + "Y_proba = output_layer(decoder_outputs)\n", + "model = tf.keras.Model(inputs=[encoder_inputs, decoder_inputs],\n", + " outputs=[Y_proba])\n", + "model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=\"nadam\",\n", + " metrics=[\"accuracy\"])\n", + "model.fit((X_train, X_train_dec), Y_train, epochs=10,\n", + " validation_data=((X_valid, X_valid_dec), Y_valid))" + ] + }, + { + "cell_type": "code", + "execution_count": 70, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'me gusta el fútbol'" + ] + }, + "execution_count": 70, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "translate(\"I like soccer\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Beam Search" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This is a very basic implementation of beam search. I tried to make it readable and understandable, but it's definitely not optimized for speed! The function first uses the model to find the top _k_ words to start the translations (where _k_ is the beam width). For each of the top _k_ translations, it evaluates the conditional probabilities of all possible words it could add to that translation. These extended translations and their probabilities are added to the list of candidates. Once we've gone through all top _k_ translations and all words that could complete them, we keep only the top _k_ candidates with the highest probability, and we iterate over and over until they all finish with an EOS token. The top translation is then returned (after removing its EOS token).\n", + "\n", + "* Note: If p(S) is the probability of sentence S, and p(W|S) is the conditional probability of the word W given that the translation starts with S, then the probability of the sentence S' = concat(S, W) is p(S') = p(S) * p(W|S). As we add more words, the probability gets smaller and smaller. To avoid the risk of it getting too small, which could cause floating point precision errors, the function keeps track of log probabilities instead of probabilities: recall that log(a\\*b) = log(a) + log(b), therefore log(p(S')) = log(p(S)) + log(p(W|S))." + ] + }, + { + "cell_type": "code", + "execution_count": 71, + "metadata": {}, + "outputs": [], + "source": [ + "# extra code – a basic implementation of beam search\n", + "\n", + "def beam_search(sentence_en, beam_width, verbose=False):\n", + " X = np.array([sentence_en]) # encoder input\n", + " X_dec = np.array([\"startofseq\"]) # decoder input\n", + " y_proba = model.predict((X, X_dec))[0, 0] # first token's probas\n", + " top_k = tf.math.top_k(y_proba, k=beam_width)\n", + " top_translations = [ # list of best (log_proba, translation)\n", + " (np.log(word_proba), text_vec_layer_es.get_vocabulary()[word_id])\n", + " for word_proba, word_id in zip(top_k.values, top_k.indices)\n", + " ]\n", + " \n", + " # extra code – displays the top first words in verbose mode\n", + " if verbose:\n", + " print(\"Top first words:\", top_translations)\n", + "\n", + " for idx in range(1, max_length):\n", + " candidates = []\n", + " for log_proba, translation in top_translations:\n", + " if translation.endswith(\"endofseq\"):\n", + " candidates.append((log_proba, translation))\n", + " continue # translation is finished, so don't try to extend it\n", + " X = np.array([sentence_en]) # encoder input\n", + " X_dec = np.array([\"startofseq \" + translation]) # decoder input\n", + " y_proba = model.predict((X, X_dec))[0, idx] # last token's proba\n", + " for word_id, word_proba in enumerate(y_proba):\n", + " word = text_vec_layer_es.get_vocabulary()[word_id]\n", + " candidates.append((log_proba + np.log(word_proba),\n", + " f\"{translation} {word}\"))\n", + " top_translations = sorted(candidates, reverse=True)[:beam_width]\n", + "\n", + " # extra code – displays the top translation so far in verbose mode\n", + " if verbose:\n", + " print(\"Top translations so far:\", top_translations)\n", + "\n", + " if all([tr.endswith(\"endofseq\") for _, tr in top_translations]):\n", + " return top_translations[0][1].replace(\"endofseq\", \"\").strip()" + ] + }, + { + "cell_type": "code", + "execution_count": 72, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'me [UNK] los gatos y los gatos'" + ] + }, + "execution_count": 72, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# extra code – shows how the model making an error\n", + "sentence_en = \"I love cats and dogs\"\n", + "translate(sentence_en)" + ] + }, + { + "cell_type": "code", + "execution_count": 73, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Top first words: [(-0.012974381, 'me'), (-4.592527, '[UNK]'), (-6.314033, 'yo')]\n", + "Top translations so far: [(-0.4831518, 'me [UNK]'), (-1.4920667, 'me encanta'), (-1.986235, 'me gustan')]\n", + "Top translations so far: [(-0.6793061, 'me [UNK] los'), (-1.9889652, 'me gustan los'), (-2.0470557, 'me encanta los')]\n", + "Top translations so far: [(-0.7609749, 'me [UNK] los gatos'), (-2.0677316, 'me gustan los gatos'), (-2.26029, 'me encanta los gatos')]\n", + "Top translations so far: [(-0.76985043, 'me [UNK] los gatos y'), (-2.0701222, 'me gustan los gatos y'), (-2.2649746, 'me encanta los gatos y')]\n", + "Top translations so far: [(-0.81283045, 'me [UNK] los gatos y los'), (-2.118244, 'me gustan los gatos y los'), (-2.96167, 'me encanta los gatos y los')]\n", + "Top translations so far: [(-1.2259341, 'me [UNK] los gatos y los gatos'), (-1.9556838, 'me [UNK] los gatos y los perros'), (-2.7524388, 'me gustan los gatos y los perros')]\n", + "Top translations so far: [(-1.2261332, 'me [UNK] los gatos y los gatos endofseq'), (-1.9560521, 'me [UNK] los gatos y los perros endofseq'), (-2.7566314, 'me gustan los gatos y los perros endofseq')]\n" + ] + }, + { + "data": { + "text/plain": [ + "'me [UNK] los gatos y los gatos'" + ] + }, + "execution_count": 73, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# extra code – shows how beam search can help\n", + "beam_search(sentence_en, beam_width=3, verbose=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The correct translation is in the top 3 sentences found by beam search, but it's not the first. Since we're using a small vocabulary, the \\[UNK] token is quite frequent, so you may want to penalize it (e.g., divide its probability by 2 in the beam search function): this will discourage beam search from using it too much." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Attention Mechanisms" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We need to feed all the encoder's outputs to the `Attention` layer, so we must add `return_sequences=True` to the encoder:" + ] + }, + { + "cell_type": "code", + "execution_count": 74, + "metadata": {}, + "outputs": [], + "source": [ + "tf.random.set_seed(42) # extra code – ensures reproducibility on CPU\n", + "encoder = tf.keras.layers.Bidirectional(\n", + " tf.keras.layers.LSTM(256, return_sequences=True, return_state=True))" + ] + }, + { + "cell_type": "code", + "execution_count": 75, + "metadata": {}, + "outputs": [], + "source": [ + "# extra code – this part of the model is exactly the same as earlier\n", + "encoder_outputs, *encoder_state = encoder(encoder_embeddings)\n", + "encoder_state = [tf.concat(encoder_state[::2], axis=-1), # short-term (0 & 2)\n", + " tf.concat(encoder_state[1::2], axis=-1)] # long-term (1 & 3)\n", + "decoder = tf.keras.layers.LSTM(512, return_sequences=True)\n", + "decoder_outputs = decoder(decoder_embeddings, initial_state=encoder_state)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "And finally, let's add the `Attention` layer and the output layer:" + ] + }, + { + "cell_type": "code", + "execution_count": 76, + "metadata": {}, + "outputs": [], + "source": [ + "attention_layer = tf.keras.layers.Attention()\n", + "attention_outputs = attention_layer([decoder_outputs, encoder_outputs])\n", + "output_layer = tf.keras.layers.Dense(vocab_size, activation=\"softmax\")\n", + "Y_proba = output_layer(attention_outputs)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Warning**: the following cell will take a while to run (possibly a couple hours if you are not using a GPU)." + ] + }, + { + "cell_type": "code", + "execution_count": 77, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 1/10\n", + "3125/3125 [==============================] - 597s 189ms/step - loss: 0.3074 - accuracy: 0.5469 - val_loss: 0.2106 - val_accuracy: 0.6487\n", + "Epoch 2/10\n", + "3125/3125 [==============================] - 585s 187ms/step - loss: 0.1902 - accuracy: 0.6789 - val_loss: 0.1865 - val_accuracy: 0.6830\n", + "Epoch 3/10\n", + "3125/3125 [==============================] - 585s 187ms/step - loss: 0.1659 - accuracy: 0.7123 - val_loss: 0.1759 - val_accuracy: 0.7005\n", + "Epoch 4/10\n", + "3125/3125 [==============================] - 584s 187ms/step - loss: 0.1493 - accuracy: 0.7359 - val_loss: 0.1728 - val_accuracy: 0.7060\n", + "Epoch 5/10\n", + "3125/3125 [==============================] - 582s 186ms/step - loss: 0.1358 - accuracy: 0.7548 - val_loss: 0.1724 - val_accuracy: 0.7084\n", + "Epoch 6/10\n", + "3125/3125 [==============================] - 583s 186ms/step - loss: 0.1245 - accuracy: 0.7712 - val_loss: 0.1738 - val_accuracy: 0.7103\n", + "Epoch 7/10\n", + "3125/3125 [==============================] - 582s 186ms/step - loss: 0.1148 - accuracy: 0.7863 - val_loss: 0.1770 - val_accuracy: 0.7111\n", + "Epoch 8/10\n", + "3125/3125 [==============================] - 582s 186ms/step - loss: 0.1064 - accuracy: 0.7992 - val_loss: 0.1806 - val_accuracy: 0.7110\n", + "Epoch 9/10\n", + "3125/3125 [==============================] - 582s 186ms/step - loss: 0.0991 - accuracy: 0.8101 - val_loss: 0.1862 - val_accuracy: 0.7088\n", + "Epoch 10/10\n", + "3125/3125 [==============================] - 581s 186ms/step - loss: 0.0929 - accuracy: 0.8205 - val_loss: 0.1903 - val_accuracy: 0.7077\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 77, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model = tf.keras.Model(inputs=[encoder_inputs, decoder_inputs],\n", + " outputs=[Y_proba])\n", + "model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=\"nadam\",\n", + " metrics=[\"accuracy\"])\n", + "model.fit((X_train, X_train_dec), Y_train, epochs=10,\n", + " validation_data=((X_valid, X_valid_dec), Y_valid))" + ] + }, + { + "cell_type": "code", + "execution_count": 78, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'me gusta el fútbol y también ir a la playa'" + ] + }, + "execution_count": 78, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "translate(\"I like soccer and also going to the beach\")" + ] + }, + { + "cell_type": "code", + "execution_count": 79, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Top first words: [(-0.26210824, 'me'), (-2.553061, 'prefiero'), (-3.2005944, 'yo')]\n", + "Top translations so far: [(-0.32478744, 'me gusta'), (-3.0608056, 'prefiero el'), (-3.1685317, 'me gustan')]\n", + "Top translations so far: [(-0.7464272, 'me gusta el'), (-2.4712462, 'me gusta fútbol'), (-2.9149299, 'me gusta al')]\n", + "Top translations so far: [(-1.0369574, 'me gusta el fútbol'), (-2.3301778, 'me gusta el el'), (-2.9658434, 'me gusta fútbol y')]\n", + "Top translations so far: [(-1.0404125, 'me gusta el fútbol y'), (-2.5983238, 'me gusta el el fútbol'), (-2.9736564, 'me gusta fútbol y también')]\n", + "Top translations so far: [(-1.0520902, 'me gusta el fútbol y también'), (-2.6003318, 'me gusta el el fútbol y'), (-3.128903, 'me gusta fútbol y también me')]\n", + "Top translations so far: [(-1.9568634, 'me gusta el fútbol y también ir'), (-2.6169589, 'me gusta el el fútbol y también'), (-2.6949644, 'me gusta el fútbol y también fuera')]\n", + "Top translations so far: [(-1.9676423, 'me gusta el fútbol y también ir a'), (-2.8482866, 'me gusta el fútbol y también fuera a'), (-3.7197533, 'me gusta el el fútbol y también ir')]\n", + "Top translations so far: [(-1.9692448, 'me gusta el fútbol y también ir a la'), (-2.8501132, 'me gusta el fútbol y también fuera a la'), (-3.7309551, 'me gusta el el fútbol y también ir a')]\n", + "Top translations so far: [(-1.9733216, 'me gusta el fútbol y también ir a la playa'), (-2.851697, 'me gusta el fútbol y también fuera a la playa'), (-3.7333717, 'me gusta el el fútbol y también ir a la')]\n", + "Top translations so far: [(-1.9737166, 'me gusta el fútbol y también ir a la playa endofseq'), (-2.8547554, 'me gusta el fútbol y también fuera a la playa endofseq'), (-3.737218, 'me gusta el el fútbol y también ir a la playa')]\n", + "Top translations so far: [(-1.9737166, 'me gusta el fútbol y también ir a la playa endofseq'), (-2.8547554, 'me gusta el fútbol y también fuera a la playa endofseq'), (-3.7375438, 'me gusta el el fútbol y también ir a la playa endofseq')]\n" + ] + }, + { + "data": { + "text/plain": [ + "'me gusta el fútbol y también ir a la playa'" + ] + }, + "execution_count": 79, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "beam_search(\"I like soccer and also going to the beach\", beam_width=3,\n", + " verbose=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Attention Is All You Need: The Transformer Architecture\n", + "### Positional encodings" + ] + }, + { + "cell_type": "code", + "execution_count": 80, + "metadata": {}, + "outputs": [], + "source": [ + "max_length = 50 # max length in the whole training set\n", + "embed_size = 128\n", + "tf.random.set_seed(42) # extra code – ensures reproducibility on CPU\n", + "pos_embed_layer = tf.keras.layers.Embedding(max_length, embed_size)\n", + "batch_max_len_enc = tf.shape(encoder_embeddings)[1]\n", + "encoder_in = encoder_embeddings + pos_embed_layer(tf.range(batch_max_len_enc))\n", + "batch_max_len_dec = tf.shape(decoder_embeddings)[1]\n", + "decoder_in = decoder_embeddings + pos_embed_layer(tf.range(batch_max_len_dec))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Alternatively, we can use fixed, non-trainable positional encodings:" + ] + }, + { + "cell_type": "code", + "execution_count": 81, "metadata": {}, "outputs": [], "source": [ "class PositionalEncoding(tf.keras.layers.Layer):\n", - " def __init__(self, max_steps, max_dims, dtype=tf.float32, **kwargs):\n", + " def __init__(self, max_length, embed_size, dtype=tf.float32, **kwargs):\n", " super().__init__(dtype=dtype, **kwargs)\n", - " if max_dims % 2 == 1: max_dims += 1 # max_dims must be even\n", - " p, i = np.meshgrid(np.arange(max_steps), np.arange(max_dims // 2))\n", - " pos_emb = np.empty((1, max_steps, max_dims))\n", - " pos_emb[0, :, ::2] = np.sin(p / 10000**(2 * i / max_dims)).T\n", - " pos_emb[0, :, 1::2] = np.cos(p / 10000**(2 * i / max_dims)).T\n", - " self.positional_embedding = tf.constant(pos_emb.astype(self.dtype))\n", + " max_dims = (embed_size + 1) // 2 * 2 # round up to nearest even number\n", + " p, i = np.meshgrid(np.arange(max_length), 2 * np.arange(max_dims // 2))\n", + " pos_emb = np.empty((1, max_length, max_dims))\n", + " pos_emb[0, :, ::2] = np.sin(p / 10_000 ** (i / max_dims)).T\n", + " pos_emb[0, :, 1::2] = np.cos(p / 10_000 ** (i / max_dims)).T\n", + " self.pos_encodings = tf.constant(pos_emb.astype(self.dtype))\n", + " self.supports_masking = True\n", + "\n", " def call(self, inputs):\n", - " shape = tf.shape(inputs)\n", - " return inputs + self.positional_embedding[:, :shape[-2], :shape[-1]]" + " batch_max_length = tf.shape(inputs)[1]\n", + " return inputs + self.pos_encodings[:, :batch_max_length]" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 82, "metadata": {}, "outputs": [], "source": [ - "max_steps = 201\n", - "max_dims = 512\n", - "pos_emb = PositionalEncoding(max_steps, max_dims)\n", - "PE = pos_emb(np.zeros((1, max_steps, max_dims), np.float32))[0].numpy()" + "pos_embed_layer = PositionalEncoding(max_length, embed_size)\n", + "encoder_in = pos_embed_layer(encoder_embeddings)\n", + "decoder_in = pos_embed_layer(decoder_embeddings)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 83, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAngAAAFYCAYAAADA04GRAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAEAAElEQVR4nOxdeZwUxfX/9uzJLfclIgoiioAIKl4ICwoo4C3emkTjkRhNTOIRk5hfvHKoSbxiEkHjrSgIghy7qHhFFBZRkUNBuW+XhT1n5v3+eFu7tbVV3T0zXb0L9vfz6c/M9PR09VS9evV9r169cogIESJEiBAhQoQIEfYfxBr7ASJEiBAhQoQIESIEi4jgRYgQIUKECBEi7GeICF6ECBEiRIgQIcJ+hojgRYgQIUKECBEi7GeICF6ECBEiRIgQIcJ+hojgRYgQIUKECBEi7GdosgTPcZwnHcfZ6jjOZ4bvHcdx/u44zmrHcT51HGdw2M8YIUKECBEiRIjQFNFkCR6AKQDGuHw/FkCfmuMaAI+F8EwRIkSIECFChAhNHk2W4BHROwB2ulwyEcDTxPgQwAGO43QN5+kiRIgQIUKECBGaLposwfOB7gDWSZ/X15yLECFChAgRIkT4XiO7sR8gAziac9p91xzHuQY8jYt2wDG9+CSobVskDjwQcHS3Sg1EwJYtMWzdGoO6+1ubNoTu3RPIycm4mJSxevVqAEDv3r1dr0skEsjKygrjkRqCCFnr1sHZtYs/Z2UBiQR/1awZkj16gJo1C6So0lIH69dnoaqq/vmcHKBHjwRataprvDDrZOdOdla3a9culPLSgfPdd8hatw5IJvmEaKfsbCQ7dUKyY8dAyqmuBjZuzMJ33zXsl506JdGlSzKILhsoGrX/yIjHkbVmDZyyMv4s96U2bVjfZWeu9omAHTti2Lw5Jm5fi2bNCD17JpCd3UTqpIkhkUggZ/t2xDZv5hOOw0cyCcrPR7JrV1Dr1oGUVVbmYMOGGMrKHDgOasemWAzo3j2Jdu2SgZSTKZpM/5HglJcja80aVkhAXV/KykKyfXsku3QJhDvE48wdduxgf1sdf/hkOxFlplSJqMkeAA4G8Jnhu38CuEj6vAJAV697Dhw4kOj994muu44IIJowgaiigjLB2rVE/frx7S69lGjLFqJ4nG97331EeXlEbdsSzZ6dUTFpYfjw4TR8+HDP67Zt22b/YXTYs4do9GiuvJtvJlq0iCiRINqwgeihh4i6dSNq145o8eKMikkmiW64gYvp3Zto3jw+V1FB9N57RIcfzt9dfz0XTxRunUyePJkmT54cWnkp41//IorFqGrIEKLXXiMqKSGqrCSaPp3o9NO58n7724yLeestolatuM/89rdEZWVE1dVEW7cSXX01FzNgANHKlZn/pSDRaP1HxurVRIceStSsGfedVatYyD//nOiWW/h8//5EGT5rWRnRiSdyWxQUEC1bxsWUlBC9+CLruubNif7+990B/bH9CNXVVHbFFVx5F1xAtHAhK6Hdu4n++U8eSLKyiF59NeOiHn2Ui+nShejf/+YxqbycaPlyohEj+LtJk7joxkaT6D8yZs8matmSqHt3oldeIdq8mYX8nXeIzj+fK+/ss1k5ZYBvviHq2pUoFiP60Y+I1q/ndtq4kQjAx5Qph8r0BjYPD4J3BoDZYE/e8QA+8nPPgQMH1tXuI49wFZx2GtHevem0D5WVEQ0eTNSmDdGsWfprvvySaOBAHri+/DKtYtJGkyZ4333HI0UsRmQiN6tXEx10ENEBBzD5SxMPPshN/dOfspJTUVZGdNNNfM1dd/G5iODV4E9/4ooZO5a2ffNNw+8TCaIf/ICv+d3v0i5mzRqi9u2ZbK9apb9mxgy+pm9fotLStIsKHI0+QC1dStS5MxtDH3ygv2b+fKL8fKJBg4h27EirmGSS6KKLiByHaMoU/qxi3bo6AjFzZlrF7J+oqiIaP54r5tZb9ZW3ezfR8ccT5eRk5BGYM4d54pln6glcPE509918zVln6R8lTDR6/5Hx7LNcMYMGMePS4W9/43a8/PI6j0CKKCkhOuoootatiT7+uOH3+zXBA/A8gE0AqsHxdT8EcC2Aa2u+dwA8AuArAMsADPFz33oEj4joySdZW118sa9GkZFMEl15JdfijBnu1377LVGHDkRHHBHuwNRkCV4yyZolJ4fo5Zfdr12zhujgg5nkmTqcC4qK6hSZW19MJtkD6zisICOCR0RvvMECfuGFRJWV5jqRSd7f/55yMXv2sBF0wAHe3rmiIrYJLr648QcmgUYdoPbuZbd09+5EX3zhfu2bbxLl5hINHcoe2BRxzz3cxPfc435deTnRkUdWU7t2rPsiENEf/kAEUKlX5e3aRXT00UzG33sv5WKWL2eHw4AB3t65Bx5Iu8sGiiZD8FatYvfzKad4V15Ne9KNN6ZcTHU10bhxPC7NmaO/Zr8meLaOBgSPiOj3v+eqMLngDHjsMf7ZnXf6u37+fB6YLrwwvIHp6quvpquvvtrzutA72CuvcOX96U/+rl+1ihXeOeekVMw33zCx7tfP31TEnj1ERx7JXqLi4vS8HOng9ddfp9dffz208nyhpISoRw+2SmrCGFzlJJFgrdW8OVe8TySTPFsVi/l3Wvzxjyw+jz3muxiraNQB6pZbuDIKC/1d//LLfP1996VUzMyZdbawH/314Yc7qGVLdtJXVaVU1P6HpUvZmL3oIn+ysm0bG7WHH54SEd+9m2fpO3Xi0CEvJJPs5cvNJfrkE9/FBI4mQfASCaKTTmJ2vG6d9/XJJJO7VPpeDW69lX/2+OPmayKCFxTBq6hgBtCzp2/32ldfcacYM4bd3X5x331c60895f83YSDUDrZzJweGDB6cWgzDvfdy5U2f7vsnY8emPjX+5Zf8m2HDqpqMh6hRcP31PKJLU36ecrJ2LRO8iRN9FzNtGjfrvff6f7REgvtebi7RZ5/5/50tNNoA9eGHzIx//OPUfnfWWdxOflgAcQhDjx7sFSor81fEtm3b6LnnuG3vuCO1x9uvUFXFHrlOnYi2bfMvKzNnpkzEb7+df7Jwof/H27aNnb+9ezde2EOTIHjCnTlliv/flJUR9erF/MEnEV+xgig7myc83BAEwXP4Pt8fDBo0iIqLixt+8e67wMknAz//OfDXv3re55JLgFdfBVavBrqnkJyFCDj2WGDrVmDlSiAvz/9vbWL79u3o0KFDOIVdfTUweTKwaBFw9NH+f1ddDQweDJSUAF98AbRs6Xr5ggXAyJHAn/4E/PKXqT3i448D110HzJgBnHlmar/dL7BwIXDKKcBNNwEPPlh72pec/PnPwK9+BUybBkyc6HppPA7078+L0ZYtS22B57ZtQJ8+wKmnclGNiVD7j0BlJfeH3buBzz8HUll5+e23QL9+wOjRvirv/vuBW2/lPnXqqf6KEHVy+eXAiy8Cq1YBBx3k/xH3G/zxj8CddwJTpwLnnJOarJx1FjBvHrB8uWflrVsHHHYYcO65wDPPuN92165d2LRpE6qqqkBESCZZvWZn82LR7x2IgKoqXl6carqLFCuvuhpIJBy0adMKhx/eC7GYPlud4zifENGQ1B5GQaYMcV87tB48gR//mK1hjxWbS5ZQbZxsOpg3j3//0EPp/T4VNLkp2vfe4z//61+n9/v33+ff//znrpclkxxmdOCB/j0OMqqqiHr1itOAAWnH0KaEJjVFm0jwtOzBB/OctQRfclJVxas1e/TwdAk8/njKTtl6EFO1H36Y3u+DQqN4IP7yF0ontKQW99/Pv/eQu61bORB8/PjUbi/q5Jtv2NPqQw3tf9i0iUNLLrig9lRKsrJ2La9+9hGacumlvPrcyym7c+dOWrZsGe3Zs4cSknLbu5ejMhpj1iIRhpJ1g/jz6T6Hz99XV/NlZWUJ+uqrr2jDhg3GaxFN0QZM8Hbt4ihvj840ZgynAti1y/UyV4waxbFhJSXp38MPmtwii7FjeaoizVXLRET0wx/yiLFpk/GSl15i6c5k3cITT5QQwIuqbKNJLbKYOpUr7/nnG3zlW07efZfvcf/9xktKS3nh50knpT+olJYSdezI6ToaE6ETvPJyzq+QyR+vqmIif9hhrgPTT3/KweBe6zdUyHUi7mFaHb3f4pe/ZKeB9MdTlhWxsuWtt4yXfPyxf6fD559/TnsUw42IQ41KSvRZBmyjUQleEH88kRDMzXhJMsn6avdukaKrgpYtW2a8PgiCty/vZBE8DjgAuP564LXXgBUrtJe89Rbw5pvAbbfx5eni3nuB7duBBx5I/x77HIqLgdmzedqvefP073PrrTy397e/ab+urgZuv52n/i67LP1iJk6swqBBPLuiJkbeb0HEwnnoocB556V/nxNPBAoKuI0MlffXvwJbtvCMbrr5Qlu2BO64Aygs5ON7g6eeAjZtYkFPFzk5wG9/y7EiM2ZoL1m1CnjsMY6q6Ncv/aJuvx3IzQV+//v077HPYedOrrwLLwQ8Es274uabgY4dgb/8xXjJL38JdOjAqtELVVVVaKZJHJ+VxbOMVVV1ucy/FxD6KTc3/XvEYvz76mpj5Ymv8vNZ3+Xm5iIej6dfpp/Hsnr3fRE33sgNZehMv/kNcOCBwE9+klkxQ4YA55/Pg9yOHZnda5/B/fcDrVpxcFsm6N2bA00ee4zjjxQ88wzHRt57b2bxJLEYcPfdwNdfA08+mcHz7ksoLAQ+/hj49a8z3/Hgl78ENm4EnnuuwVd79rBxc845wPHHZ1bMj38M9OjBRI++DyHF8Tj3peOOA0aMyOxe554LHHwws2wN/vIXFoNMiVmXLsBPf8qi8Nlnmd1rn8Hf/86CngkJB5gR/OQnwMyZHIun4OOPOTbyttuANm28b0dExrgvERP+vTFoRfxcbi4r/EwgCKKm8kgK8RNq1XEcnka1iIjgqejcGfjBD4Cnn2YLWcKSJcB77wG/+AUQxM5Zd97J/X/y5Mzv1eTx1VfASy8xucvE9Snwy1/yYot//aveaSLg4YeBI48Ezjgj82LGjuVFMQ899D0hD/feC3TrBlx+eeb3Ou00YMAAJg+KVfvMM8zNb7kl82Ly89nw+t//eK3Ufo8XXgDWrGHikOlWSdnZ7CF67z3ggw/qfbVrF7fTxRezWswUv/oV0KIFL3ra71FaygRv4kSeSsgU113Hgq6Z8vnHP7hef/jDzIvJymLHblXV90TfVVbyaybeOwHB3jSVl0iwCszNDWR3M/+PFF5R+xBuuYWt5Iceqnf60UeZ2F15ZTDFHHUUL9x9/HF7LvFBgwZh0KBBdm6eCoQr4Kabgrnf0KHsvXjwwXoW0//+ByxeDNxwQzAdyXH4XitWsJVsC126dEGXLl3sFeAH//sfUFTEK8mDWN7tOEzEv/iCp+ZrIEj44MGZe+8ELrmEvRePPRbM/Zoskkkm4f37B7e8+wc/YKNLmbWYMgUoK8t8tkKgfXu2G1566Xswa/H448yQ77gjmPt17MgDz3//y3ENNdi6lfn+FVf48975geA6YgvW/RbCe5eTk7n3TsDgAq2qYnUY+n70mQbx7WuH6yILGRdeyEvHalZB7NrFi5l+9CN/P/eL55/n4Ng33wz2vqnCapD41q28vOuaa4K97+zZDVZSXHop57ALYn9FUSfl5bwD1HnnZX7PJo0LLuDVQy6Vl7KcVFXxatpTTqk99dZb3GxPPpnug+px442cS3bLlmDv6wehLbKYO5cr77//Dfa+t93GOQ9rFgMkEpww98QT07+lrk6WLuXHf+CB9O/b5JFI8Ar0ESO0X6ctKytWcBtJmfXFKvJUFsAs9rGvd2kpH2GtqG2URRYVFZmtnDVhz566lRRUt/5Ct4bDrS0QLbKwiJtv5jmkl14CwNZseTmvwQgS55wDdOrE3sH9Fs89x67wn/402PuefjrPxT7+OAC2Zl96ia3ZVq2CKyY/H7jqKk4XtnFjcPdtUtixg//gZZcFW3k5Oey1fecdTnQH9t61awdMmhRcMQDH4lVX7+chD08+yZV3/vnB3venP+W2+sc/APBCsq++Cs57JzBgAHDCCdxl99spwAULgLVreWVKkDjsMGDCBOCRR4CKClRXs8d69OjMFsDokJvLDq5EItj7NhmIoLisLK337t5778XQoUPRunVrdOzYEePHj8dnSvCo8ZrcXL5/zQKKINZwpIuI4Jlw7LHca6ZMQTLJBGzYsNTy8vpBbi7rgZkzgW++CfbeAHDppZfi0ksvDf7GqWDyZF5VEkQsigzHYeb1v/8By5fjP//hzhQ0CQeYPMTjwL//Hfy9AeDVV1/Fq6++aufmfvDcc1x5P/hB8Pe+/HImD089hfXreZH6j34UTByrjCOOAIYPB/75z/10FeDOnVx5F18cfIb0rl2ZPNTIwT/+wafOOSfYYgDg2mt54e5bbwV/7yaBJ5/kKe+zzgr+3tdfz3IwcyZeew3YsCF4uxmom0rcb6dpEwkmYQbW9dZbb+H666/H+++/j6KiImRnZ2PUqFHYuXOn9zW7d/PYVFUForocyEHNAqeETF2A+9rhe4qWqDYR6LuTV1qZFRH45htOlXT77cHfu9Hz4C1ezHMIDz9s5/6bNhFlZVHil7+mgw4yzoqkBbVORo/mLX1S2V3NLxo9D97RR/PhgbTl5KyziDp3pt/dUU2OQ/T11+ndxgsvvMDi5ndP26AQyhTtww/zn/MxxZYWZswgAmjjY9MI4C26M4GpTsrKOORByv27/2DnTg5HueEG4yUZyUo8TtStG9H48TRyJO+SlcpWmUT+pmiJuJ1szGDq4HeKdtu2bQSAHnjgARoyZAjl5eVRnz59aM6cOakVKP6czzno0tJSisVirsno611TXk5UUkJVlQkqKTGPGdEUbWPi0kuBWAxb/vQU2rcPflZE4KCDOF76P/+p9eruP5gyha2kiy6yc/8uXYCxY1H15H+x/ttExhlY3HDddWwxz5plr4xGQXExLxG/6ip7ZVxxBbBlC9b+ay5OOw3o1ctOMWefzSEP++Vii8mTgUGDgp9GEDj9dKBTJ3z3j6cB2BMHsVDt1VfrrRfYP/D88xyOYsMTDvCU4qWXgmbPxmdFW3HFFfa2FmuKiy2WLFkCAHj44Ydx33334dNPP8WAAQNw8cUXo7y8vN6199xzD1q2bKk/OnZEy27dsNDnsvvS0lIkk0m0bdvW3zU1LlCqqobjNN72bxkmutrP0a0b4qNOx7HznsKkH9+FvDx7rXT55cDrr/O0xahR1ooJF1VVwLPP8lRFu3b2yrnySuTPnIkJzebjzDNPt1bMmWdyMtHnnuPZrP0GkyezNr/4YntljBuH6jbtMXbrU6i6ZJy1YnJzuS899BDPZNkUu1CxdCnwySecesMWcnJAF1+CQx96GGcO24GDDmpvrairr+aMH88/H9zC+iaB//yHSfjgwfbKuOwyOH/6EybheVx88c8Cu+2pmk2GJ068AFdffT0SiTKMG9ew31555ZW48sorsX37dpynSYx+3XXX4cILL8S6detwmZJ1/q005uiLi4uRlZWFWbNmoW/fvgCA+++/H71798aXX36JoyXj59prr8UFF1zQ8CZVVUzCmzdHd5+bI//sZz/DoEGDMGzYMH/XZGWBYlnISlYjJzcv1NQoMiIPngc+7HslDqT1+HGfIqvljBvHse3PP2+1mHAxYwYH79v0DAGoHH0mdjlt8evOUwKP65KRk8ObO8yYAezda6+cUCFI+MSJnMfCFnJz8V7PizER03HW8F32ygE7i+Nx9hDtNwiDhANYefzlyEU1ft3zBavlHH44OyJffNFqMeGiuJhzNNny3gn074/lzQbj2uZPo08fu0VlZTWtxRbFxcUYP358LbkDeEcIHdq1a4fevXs3PA46CL379EHvvn21O3qo+PnPf453330XU6dORZbBFae7JhHLQRaSyMlqxMrLdI53XztSisEjorPHltMupy0lL7o4pd+lg8su461wKyqCu+ett95Kt/rYoNBKDNGZZ3K8SKpBIinitdeI/oEbKJ6bn9kGwQp0dSJSfGi2ac0I8+bNo3nz5gV7Uz8Q+8763LA+XTmpriYa1bZmw8zHHkvrHn6RTBL17h3u/rRWY/Cqqnjj6vPPt1dGDX75S6KlGEBVg4/N+F5edXLffSwOa9ZkXFTTwC9+wXl6duxwvSxTWVm+nOhGPMSV57KXqQl+Y/CI3FN8BAm/MXhHHHEE3XXXXfXOTZ06lfLz82mvsr/53XffTS1atHA93nnnHdfybrrpJurSpQstX7485Wv2liYoWVJCSZfKsx2D1+iEK+wjFYK3Ywf31/cGXstJ8DQbNAeJWbO4RaZPt1qMFoEPUDt3EmVn84hhGRdcQDTqgEVcef/6V2D31dVJIsGcdeLEwIppXFx4IVHHjr5XjqQrJ3PmEAFJKulxJNGwYWndIxX85je8cGnzZutFEZFlgjdvHsv2q6/aK4NYtg88kOhfR/yVy3MZ1PzAq06+/pqLuf/+jIppGkgmiXr2JBo3zvPSTGXlzjuJOjtbKJmdTfSrX6X8+1QIHhEPe7Zz4vkheOXl5ZSVlUW/+93v6p0/+eST6corr2xw/Y4dO2jVqlX1j88/p1WLF9OqFSto1apVVFZWZizvxhtvpE6dOtEXLkkGTdcIYhwv3VsvJ56KaJFFI2LqVA4w7XDt+ZwE7803rZY3ahTPkr1gd3YkHMyYwfNkmWxY7wN79nBRh110DHDIIdbn5WIx3jt89mzgu++sFmUf5eXAG29wjGSm+8564LnngDZtHDT70cW8JdaGDVbLmzSJp5ZeecVqMeFg6lTei2rMGKvFvPsusH490P4nF3Oah5ocoLbQqxdno9ovpmkXL+Y8V5b1HRFHVBxV0AnOmDEc00N2Ewrm5HBfauzUQ8tq8mg+//zzWLhwIVasWIHLLrsMq1evxr333tvgeu0U7UEHofdhh/HRu7dxivaGG27A5MmT8fzzz6Nt27bYvHkzNm/ejD179vi6RixMcXJzuH0aaY47IngueOEFoE8foM8PT2HmZZk8iBiv6dODi/E699xzce655wZzs1QwdSrvAD90qNVipk9nnnLRxQ4n7Zo/3zrzmjSJQ9emTQvuni+++CJeDHukmzuXGbJl+Sgv565zzjlAzgU1idWCrDwNjjySj32ePCQSXHnjxgWfOFDBc88BzZsDp13ehbMRhxDEeOGFzI1WrbJelF288gobSRMnWi3mo4+Ar7+uCcU891xg3TpefGMRwvZr7NW0xcXF6NOnD+666y5cdNFFOProo1FaWopFixb52+ZRbAjrw5h99NFHUVpaioKCAnTt2rX2+Iu0nZ/pmj//+S+orq7JoZzTuJUXETwDNm3ihOSTJgFOTjZ7OWbOrNuc2BImTeL9H2fODOZ+O3bswI6wN34sLQXmzOER3fLyoeeeYx55wgng8qqr2StlEUOHsvchSE9reXl5g2X+1jF1KtC2LTBypNViZs9mkbj4YnB0fb9+oZCHSZOAhQvZK7XP4t13eYsWy56heJw5ysSJ7CzEOefwyt2vv7ZarljkuE8TcSKuvJEjrS/bfuklXmtzzjkAxo9nFvHaa1bLjMW4mOrqxt19pLi4GEcddRQmTZqE9evXo6ysDNOmTUP37t393UDkIPOxIaxpyvP3v/+95zW//e3vkUzWFOM4TCjj8UapvIjgGfDqq9wetdspnXMOb102f77Vck8+mTPI79MKb9YsJsKWPUO7dwPz5vEgEYsBOO44oFs36+TBcVgu5s8Htm+3WpQ9VFVxXp4JE6zvgP3aa+wAr83CcO65wNtvW6+8Cy/kV8szjXYxdSrvladJUREk3n2XF7zX8sizz+ZXy+ThwAOBk07ax/Xdp58Cq1eHMj07bRpQUAC0aQPuVMOHW28jgFUEUeNO0xYXF2PAgAHp36DWrWaX9ggeWesozGm8adqI4Bnw+uu89d8RR9ScKCgAWre2Th6ysthZOGcOUFFhtSh7mDoV6Ny5xq1mD3PmcJ+t3REoFuOBafZsdoNaxHnncX+17Cy0h6IioKTE+qAkHKpnnikpvHPO4cqbMcNq2X36cEqyEMY/O0gmuS+NGQO0bGm1qOnTefez006rOdGrF1deCJ7WCy4APvsMWLHCelF28MorrHtsbE0m4fPP2aFabxb47LOB5cuBL7+0WnZjb11GRFi2bFn6BE8EEVqONQaY4MViEo8UZTbCLgYRwdNg926enq2XzDYvj0ep6dOtN9T48cxPiuym3rOD8nL24J11lvX03dOnsxFbL/fkOefwM8yZY7Xso48Gune3zlHs4ZVXOPHi6NFWi3nvPWDXLqUvDRoEHHxwKORhwgTg/ffZO7XP4X//AzZuDMUzNH0627D1eOQ55/CCmE2brJYvZGOf7EtEwMsvs3u6Y0erRU2fzq/jx0snBam0bMWI3Rgaa6clx3Gwe/duTEg3w7xgppZnK0TOwHrFyJUX8jRtRPA0EJ6hBrJ07rk8UrzzjtXyR4zgOJggFF5BQQEKCgoyv5FfzJnDK0QsT8/KnqF6PPKUmgUxU6daLd9xWNG++WYwntZevXqhl639u1TE4zzXc+aZwW9ar6CBZwjgyjvnHF7kUVpqtfzx41np7pPby73yCo8UZ55ptZjPPwfWrNGsDzj77Dr2ZxE9ewIDBuyjBG/5cnY9WibhADfDscdyFEotDjyQg4JDmqZtSkmPU0IDt5q9YgCNo7CRliI3aYLnOM4Yx3FWOI6z2nGcWzXfn+o4TonjOMU1x2+DKPf11zWeIYD3amzWzDp5yM/nol5/PXPCf+edd+LOO+8M5sH84LXXOHBfs+1NkHjvPV4s22BQEivZZszgODOLGD+euWwaO+40wPDhwzF8+PDMb+QH773Hhso551gtxugZArjsqirrzGvwYI5pff11q8XYweuvSwFX9qD1DAG8DLlPn1A8rePHs1ju3Gm9qGAhWKnlvQs3bgQWLTIs0j3nHP5y3Tqrz9CIM42ZQetWs4N4nO3XBjyykSqvyRI8x3GyADwCYCyAIwBc5DjOEZpLFxLRoJrjD5mWG4+zZ+iMMzQsvEULdkXMnGnd1TphAnfqxYutFhMshKtk3Djrnen119kzpJ1hPOssnmd/+22rzzByJKeV2Oc8D7Nmcfucbm/fXsDFMwSw9dS5cygrAM88kx3Llvl+sFi5kgP3LXvvgDrPUNeuyheOw168BQt4nt0ixo/nMXj2bKvFBI833uCQA78rOdOEMFC0fUksiLGcekg4wPY5gmd0qwULIi4qJ0eTPEJeihwimizBA3AsgNVE9DURVQF4AYDdJEMwxAzJOOMM4NtvgS++sPoc48axTGTqeRg7dizGjh0bzEN5YdEiXhl5xhlWiyGqc25oY89HjmT2Z3m0yM9nvh+Ep/WZZ57BM888E8yDeeGNN3gqu1Urq8UI2dVylFiMhXzOHOsjxoQJPBNsme8HC7F6x3JfcvUMAVx58TgvV7eIoUOZ7+9TxtKuXRzgabmNACbhhx4qLfqT0bcvpx8KKreWC7KzmYg3ZrqUlGF0qwVfDODCI7OzQ5+mbcoErzsA2ee8vuacimGO4yx1HGe24zhHZlro669znqF6MUMyBFmyPLXUsSMvQs1U4YWaX23WLO5ExsoLBsuXA1995ULCW7TgKeIQAq8mTOA8a8XFmd0nHo8jHoZp/M037FqznHYD4L7UIGZIxrhxPM/+v/9ZfY6CAo6s2KfIwxtv8Gh+8MFWi/GcYTzuOA65sGwsxWLMk958s/ET6vrGnDnMdiwTvNJSXnA3caJLWtGxYzlWJKgM+Qbsc9O0wq2WnW09J6vgkca1hY1QefbXDKcPXWuodsNiAD2JaI/jOOMATAPQp8GNHOcaANcAQLdu3bDdkH+LCHjttbY46aQEKit363Ma5+fjgCOPRHL6dOy+6qoU/k7qGDmyGf7whxZYunQnundPj/VX12hL038WKCkpSev+Mtq8/jpwzDEoIbKa4+y555oBaIETT9yJ7dv19ZJ/yiloOWcOdi5ahGSaixf81MnxxztwnHZ48cUy9OiRPpEurVls4NVOmSL/pZfQEsCuE05AIo2y/MrJli0O/ve/9rjttr3Yvl1fL87RR6NdVhbKX3kFZX37pvwsqeCUU1ph2rRs3HnnLit6Poj+I+Ds2YN277yD8muuQZlleXjlldY4+OAsdO68y9hlWw0fjpw33sDOrVtT8oKkWifDh+fiySdbY+bMEpx8ctNneS1ffRW57dtj5yGHpKTvUq2XmTNzUVXVGqec8h22b9eTg5wTTkCbBx9EyfTpqPZhYCfT9CLFYryilVPKBevGS/eZ3OAkEnAAUFYWyLLnLB53kJUlEiDrHsaB4zhAPA6S3HxWdX6mm9naOgAMAzBH+nwbgNs8frMWQAe3awYOHGjc3PfLL3nz60ceMV7CuPVWouxsou++87gwMyxf7vN5XDB8+HAaPny453UZb5a+aRM/7N13Z3YfHzjxRKJjjvG4aNUqfp5//CPtcvzWybBhPp7HA5MnT6bJkydndhM/OPNMokMOSXvncL91MnkyV39xsceFp5xCNGhQWs+SCp54gp/n00/t3D/j/iPj1Vf5YRcsCO6eGpSXEzVrRvTTn3pc+NRT/DyffJLS/VOtkz17iPLyiG66KaWfNQ7icaL27YkuvTTln6ZaL1dfTdS6NVFVlctFFRVEzZsTXX+95/3cNrj3g7Iyot2701YhWiQSieBuJqO8nKikJNiH1SAe52Jc24iIK096Hre2APAxZcijmvIU7SIAfRzH6eU4Ti6ASQDqRaQ5jtPFcdgedxznWPCUc9oZr958k189Q9bGjQslLqVvX843ajmlWzAQlWd56u+774APP/Sx73rv3rwCMISo7TPO4O0gt2yxXlRmKC8HCgv5gS1PV7z5JtClC6e/cMW4cTy/vXGj1ecRcYD7RGLqN97glbMnnmi1mIULWSQ819qICyyHPLRoweGz+8RU+kcf8Ur0EOKN33wTGDXKY91aXh7HIsyaZT1ALju70TZmSB3xOM+ZhjA9C/hI/So8dyFVXpMleEQUB/ATAHMALAfwEhF97jjOtY7jXFtz2XkAPnMcZymAvwOYVMN808KcObx7heeM3rBhwAEHWFd4jsO6tago/RWAZ555Js4MYSUeZs3iZXgDB1otprCQ+4YnwQOYPBQV8ShmEWL8y4TvH3bYYTjssMOCeSAT3n6b68IyCU8kuC5OP92HXhXPIgwES+jalclmkzeWiLgvnXaa9ZXoc+ZwvLFnRqPOnYEhQ0IxlsaO5fjar76yXlRmeOMNHs0tr0T/8kvOfuJL340dC6xda31LkH0mDi/k3St87YIWcuU1WYIHAEQ0i4gOI6JDiejumnOPE9HjNe8fJqIjiWggER1PRO+nW1ZFBceo+uqv2dl84ezZ1lfEnH46sGcPL9ZKB7fccgtuueWWYB9KRXU1J60dN866pTRnDu8Yd9xxPi4eN66uYS1i8GCgQ4fMyMMJJ5yAEyxv7YY33uDVBpZzFH78Mecz8zUo9e/PKSZCWBBz+um8Sn7PHutFpY/iYt45IoRFMG++yYupW7TwcfG4cew6t5yoTujfuXOtFpM53niDV8G1bWu1GGH3+BqXxNSTZSLe2Lta+IZ4QMuGkvBm+tq4KeTKa9IEL0z4nq4QGDcO2Lw58+WTHhg5kvlkk/Y8vP8+72vaVKYrBE45hRPVWSYPsRjn45s7t3E343aF8AwVFHB+F4uYM4f1mK9d0ByH+9K8edaXT55+Ohdhme9nBiGrllMbrVvHi6l9kXDxPMmkdebVpw/vbNGk9d2mTaz3QyLh/foBBx3k4+KDD+aLQzCWGiHjR+rIcPeKTZs24YorrkDHjh2Rn5+PI444Am8ruZYeffRRHHJIL3TqlI8TTjgGCxcu9L5xiJUXEbwa+J6uEBCa0XJnat2aDcV0Fd6pp56KUy17bDBnDgut5S3RxHSFbxKen88MOSTv0NatwNKl6f1+ypQpmDJlSqDPVA9ffcU7lfse0dPHm29yXrP27X3+YNw4Tkz93ntWn+ukk5jvN2nyMHcub3TcubP1YoAUxEE0aIhhKU02Xcr8+fxqOR1UWRlHVaTUZceO5a00LbupQw4lSx0iPUqa+6F/9913OPHEE0FEeOONN7B8+XL84x//QKdOnWqvefHFF/Gzn/0Mt9xyO959dwlOOOEEjB07Ft9++637zUOcpo0IXg3mzAFOPtnndAUAdOrEitjyQguAFd6SJU04iL+oiBOetW5ttRgxMKcU9jJmDBMby0E9Qtc3WfIg5NTyoLRrF6e1S2lQKihgpWfZO5SXxwZck22jPXuADz7w6frMDG++yTPj2sS5Ooh4szlzrAfxn34653778EOrxaSPuXM5JmPQIKvFvP02UFmZBsGrquLdRyyC06WEP027fft2OI6DBx98EEOHDkV+fj4OO+wwzFV1h2Ceacbf/elPf0LXrl3x9NNP49hjj0WvXr1QUFCAfv361V7zwAMP4IorrsTll1+NI47oh4cf/ge6du2Kxx57zP3mIVZeRPDAiWo/+yyNeNnRo1khW7aWmnRcSkkJp8K37L0DeFA6/HCewvGNUaP41TIRb/JB/PPmccX17m21mPnzeeYhpb7UqhUbCIWF1p5L4PTTgVWreAu1Joe332a3lWUSLhIAjBmTYsjs6NHspv7sM2vPBrDTPSurifYlIhbyUaOs74zw5pscMnvKKSn86KST+EfCy2gJcihZmLtaLFmyBADw8MMP47777sOnn36KAQMG4OKLL66f0D8exz1/+Qtatm2Lli1bGg/TlOq0adNw3HHH4cILL0SnTp0waNAgPPzwwyIdG6qqqvDJJ59g9OjTQFTHI0877TS87xUw7zj8gxAIXlNOdBwaBHFKi+D96U/sErcYj3H00byzxZw5wGWXWSsmPbzzDo/oI0daLaa8nMe/H/84xR8edhjQowePaNde6319Bjj9dOChh5jva7dQayzE4+xlPf/8UNKjHHAA87WUMGoU8Mc/ch6cAw4I/sFqIPr4nDnWxSF1zJ3LYQWW06N89BHbZSnP1gsjrrAQOOqowJ9L4IADeBHVnDksEk0Kn33GsdeWSTjAfenUU1MMmc3PZ5KXhrGkCxE58sgjMXToUFRXV+PZZ5+t9108Dhx++CAMGzYIlZVleOmllxr8fsiQIejfvz9KSkrwmrLv9JVXXpnyMxYXFyMrKwuzZs1C35rk6Pfffz969+6NL7/8EkcffXTtw1179dW44PLLXe/X3bCH8Ndff41HH30UN998M2699VYUFxfjpz/9KQDgJz/5CbZv345EIoH27TmUQhC8zp07Y74fcp2dzcacZXYcefDAiqRr1zR01kkncYey7B1q0kH8hYVcB8OGWS3mnXd4QWzKJFxE+xcVWQ8YEUH8lmdHUsfHH/OIbnnqj4j70qhRacyMFBSwcFteAXHYYRyw3iS9Q/PmsbsmhEUwsVgaTvcePXgVhGXvEMB96ZNPrG6Ikx6EN8ByX/rmG2DlyjR5ZEEBr6DZvDnw55IhHJhhTtMWFxdj/PjxteQOAHJzc+tfVLOAoV2nTujdu7fr0axZM205yWQSgwcPxr333oujjz4aV111FW688UY88sgjynVOvXUcRATHjxEtYgMtD+jfew9eIsH6asKENJwb+fkcuBdCHN6YMcBzz/HircGD/f/uggsusPZMAJjgnXQSBzhZxLx5vAhm+PA0fjx6NPDkkzxipOxa8g85iH/8+NR+e+SRGW+jbMbcuSzclqfRV6wANmxIc+w7/niuvMJC4Kyzgn60Wogg/hdeYDJuOYOCf6xfz5ss//CH1osqLOQ1E2ll+CgoAJ55xnrlnX468LvfsW6eNMlaMalj3jxeqXrggVaLKSriVxFhkhJEPy8qAi6+2PfP3DxqOTk52u/37GGC16JFc9fft2nTJi2PnYri4mJceOGF9c4tWrQI+fn5daSvxpC/589/xj333ed6v9mzZ+Pkk09ucL5r1644QglQ7devH/72t78BADp06ICsrCxs2rS5njG7detWdPazQEqwQssE73vvwVu6lFM7pdWRAB7NPv/ceiZ+uc+mguuvvx7XX3998A8E8KqPzz4LJf6uqIhXEzdvnsaPxfNZJuJ5eUxA0wklGzp0KIYOHRr8QwH8vwcPTmFZa3oQspmWOOTmsvcqpDi80lKeqmwyEF4xy56hPXt4EUzaERUFBXyTRYsCfS4VQ4YwAW1ScccVFTyVEMIimMJCXseXlt139NE8zx2CpzU7m/lUGHF4FRUVWLFiRYM9ax966CFMmjQJzcXgUONSvPa661BcXOx6DBkyRFvWiSeeiBVKwuiVK1eiZ00AeG5uLgYPPgYLFsyrt1B33rx5/vOZinQpFhPxf+8JnhiURoxI8wais1vuTN268QKDVMe/srIylJWV2XkoMRdpmeDt2MGey7QHpY4decVbCJ7WkSM5nUuqfL+6uhrVNvJCiOWIIQ1KBx0EHHJImjcoKGAv1oYNgT6XilNPZU9eqsaSVcydy6lRLMa2AZzvMx7PoMuOGMGVZ5mIZ2VxXyosDDeI3xXvvceDseX4OyKWzZEj0wyZzcridgqh8gS5CSNdyrJlywAAzz//PBYuXIgVK1bgsssuw+rVq3HvvffyRSI9SnY22rVvn/YU7c0334wPP/wQd999N1avXo2XX34Zf//733HDDTfUXvPTn/4czz03BVOm/BvLly/Hz372M2zcuBHX+g3uFZVnMT2Ub4LnOM7VjuOQdFQ4jvOZ4zhXWHu6EFBUxMSpW7c0bzBgABOIEMhDQQEr6FS2LRs3bhzG2VoAUlTEe2amMmecBt56i/ttRjxy9GhOyGx5xbMgoamSh2effbZBEHMgeOstVniWCV4yyXy/oCCDdRzpuqlTRPv2zPebDMFLJtlAHD3a+iKYwkL2NKe9aUr79uwhCsE7NHIk8O23nOWoSWDePJ6WTitOxD++/JJzKWek70aN4sqznB4qzJ23iouL0adPH9x111246KKLcPTRR6O0tBSLFi1Cly5d+KJkEvWWtaaJoUOHYtq0aXjppZfQv39/3HHHHfi///u/erNh55xzIe6//yHcffcfMWjQILz77ruYNWtWrZfPE+IZLRpLqXjwBgGoADCs5jgbwG4AUxzHSdf/1aioqmKPe0YdSUQrz59v3VoaORLYu9f67Ih/FBayOyTNZJJ+UVTE+QkzmsEcPZrjht55J7Dn0mHQIJ5aajLkYe5cTptgeWVmcTHnwMtoMfXAgUwgQpimHTmS+b7lbYr94dNPgW3bQvGyilAHg+PCHwoKOD3U3r2BPZcO6RpL1jBvHi8ms7xEXvzfjPqSvOLZIkS6lDA8eMXFxTjqqKMwadIkrF+/HmVlZZg2bVr9lbAZ5r+TccYZZ2Dp0qWoqKjAypUrceONN9YuoBDbk1133fVYu3YtKisr8cknn+CUVHLaOA7zhyZE8L4gog9rjtkARESw/T1bLGDRItZRGWf4GD2aVyxZzg81fHgTmlpau5ZNa8vpUQD+v6eckmFMt1gIEsKK55BmR/yhsJArz/IimEAGpViMbxCCsVRQwAZeuns8B4qMghf9I+NQB4GCAjaW3n03iMcyom9fzm7QJPTdzp2cbT6EeOPCQt51LO1QB4CXi3fvHoqxJAiebX1XXFyMAQMGuF8Uj/MgadkTLjyWGfPIWIwX/333XaaPpL+9n4scpq0DACxTvtpd85qJPdhoKCpiOch4Jy+xQiOkqaUQ+qw3QhqUNmzgKYuMi2nWLO38UKlCTC01ejLdzZs5pi0EEl5YmGGog0BBATf6ypWBPJcJJ53EyrnJ9KW+fXlAtohAQh0Arrzc3FC8QyNHcvU0urH09tv8EJb7UiLBoQ4ZFyNWzRcV2U/FEcI0LRFh2bJl7gRPuNWys60TPOEozHjyynGspofy68HrA6AlgE+V8yIY4ZPAnihEFBZyOEm7dhne6KCDeIeAEEYLMTtia92EbxQWclC4772O0oNYxxGIXi0oAJYt42z8FiGetdHJQ6CVZ0ZVFceGBsL1hbFkufLE5hmN7h2qrmbykPYqL/8oLOTZRcPCQf9o0YKnKkOKw9u6FfjiC+tFuWPBAl7CbzHNEsAe1u++C6gvFRSw2/ZTddgOFmEstHAcB7t378aECRPMF4n4O8shQ0DdNrcZ88hYrC49lAX4JXiDal6/cBwn23Gcto7jnAPgQQBfAnjexsPZRFkZE6XAxr6CAlbUlqNNR45MbWrpyiuvDCT/UD1kvMzLPwoLmYAPHBjAzYTWtJyJ+PDDU59aGjRoEAYFvbelWAQjsrtbwkcfBRTqAPC8VM+eoZGHRYs4B3Sj4ZNPeOFPSF7W4cMDSl9XUMBsZMeOAG5mRpOJwysq4pynalLdgCHG+cAMWvmmliBvW9aoCGze1B01eZSDK+bkkxud4IkRYjaAagA7AbwA4C0AI4ioAgAcx3nZcZy0o7lriOMCh/Gu4ziZRCG44v33mSgFNsM4ciSwezcrbIs4+eTUppasELzly3n6z/L0rOCRI0YEtO3j4MFA69ZNcmrJGsEbPjyURTCBhDoAdVNLCxZYj9weOZIVtWE7ynAgjI1AKs+M9et51jtQg5bIurF08MFAr16N7A3fsoVznYbgZS0q4kkRsSg0I3Tvnl5urTQgUro16k5LiQTqbSthsRggQLUq0kNZyKWbigdvPYChAIYAOBJAGyK6kIg2A4DjOEMAtCOitJO6ENEuIhpBvKPvAwDuSvdeXigsZKE86aSAbig6v+XO1LIl79Po16Ldvn07tge934/4j5YJ3tdfcyxbYMVkZ/NAGoI7QEwtff65v+sDz1f4zTehLYIJLNRBoKCA56lqNha3hWHDeDOaRvUOFRXVpVqyXAwQYF8aOpSVUUhhKW+9Fc5KTS32xVAHATGzlEpurTQQZj48LUT+u5CmZ4GACR5gRRGlQvA+JqKPiegTIvqCiNQEA9cAeE4+4TjO7xzHmec4zkLHcb6oee1gKsRxnD84jvPbmo8zAIxxHOcAn8+YEoqKmCgFtuK9Y0dW1CGRB7G9qBfOO+88nHfeecE+QFERm9UHHxzsfRUEOl0hUFDAuaG++SbAm+qLAfyLw0svvaTdrDtthDQoiVCHwAclwPo0bX4+Z49pNO9QZSWvRA3JM9ShQ4B5lHNy2FgKaSq9pMQ63zcjpFCH//2P+1Pg+q6sjG9uEYLsNNo0rXAdWp6eBSys4xg0iK1jC4rIk+A5jtMZQBcAXt2rAIAqRUPAK2zPJKIjAGwBE0ETjkHNgg0iqgav2g3Kx1aLkhImSIE7oAoKOCt1RUXAN64PMbVkOaWbHokEm9MhbU/WrRuv+A8MIa2A6NmTw8kazTtUVMRGh809bsH8pLo64EGpc2egf//QVjyLNHSh43//Y11hmYQTcVUGFuogUFAArF7NbnaLEPy30frSggUc6mCZPBQWcvsEOlt/6qnWc60BTHbEtmWNgsDdanqIaehAi7GYW8tPdxdmixfBOxBM4GQMAXATEQlf0zIARg8emOAtlj5vrrlvoHj7bW6kwPXqyJGssD/4IOAb18fxx7P3oVE8D0uW8PSZ5UEpmWSFntHOCDoceSQTiJA8rY0ytSRio4Ryt4iiIlbsmv26M0NBAbPHEIwlwFqWAncUFXH7pJIcNQ2sXs0xeIF32ZCC+Lt04bi0RiF4337LFRhSqMMxx/A2soGhbVu+aUj58BotDi8eDyX+zto6jlGjgHXrgFWrAr2tn9oYVPPqRfDKIOXDcxznQADtUJ+wHQ/gY92Pa65PEtEm6XQ+gMBzzRcVcVq0448P+MannMJSbrkz5edz7GCjKDwr86YN8fnn7FUJvBixAiKETMRiamnxYu9rA4W1Eb0hCgu5H7VoEfCNR41icmc5E/GQIZwypVH6UlGRhRG9IayFzPbvD3TqFJqnNdVtGgOBCHWwPI2+dy9vGW2lyxYU8M0tb9MY5rZl9SDnv7OMRKJuA4pAYclY8nxMIrqPiBwiWu9x6acADpc+DwWQC6A3ADiOcy6A7gBervn8tOM4Z0vX107PSugHYKnXM6aKoqK6jQ0CRevWnCcpJIUXQkq3higsZMXeubPVYgLZGcGEkSPrkgBbRKOleLBaeXXYtYvJq5XZ+pCMpexsnn0LvY3KyiyO6PVRVAQceCCn6gwUYsVzSMZSWRmn5AkVInixf3+rxSxcyMTISl8qKOCbW14uHouxSIQ+YxH4slY95HUcgWcH690b6NEjfIKXAl4BMFb6PATA3wE86jjOMgBXABhTE1snvpdJY73pWcdxegHIQsAELx5nYmRNr4rkWrt3e1+bYTGAd5aC6667Dtddd10whYqg8JA8Q717cw7pwBHSpvadO/OMsJ9ihgwZgiEZZ6CtQVERp0jo0yeY+xlgLdQBCN1YWrmSnZ6h4b33OHjRsmfIWqiDQEEBG0uWMxGLbRpDDUsJPE+TGUVFnGLPypbRJ57I3owQ8+GFuvNIgPvPukHkUbZSjJweKsA57iCldjKA0xzHEetShwJ4g4hGEdFRRDSBiDYCgOM4HQFsIKJF4sdE9Fsi+p10v+sA/KkmZUpg2LOHtZw1jlJQwAJneQXEMcfwGOhFHi688EJceOGFwRT64Ye8O7vlBRbxOJMHa8WIFcBNaGqpf//+6B+El0DE340YYT0JtbVQB4GCglAyETeKp1UELwaWp0mPZcs4F7FVfQdY70vt2nEay1DbKORQh2HDeFODwNGsGXDCCaGseM7OZhUUKsET8XchbU9mjUcWFPCex8XFgd0yMIJHRHsA3AhAJCceDEO8HRFtI6LRHrdcD2CD4zgrHMdZ7TjOreoFNQmR/17z/aeO4wz2es49exy0acPKwgpCSq7ld2pp3bp1WLduXTCFhhQUvngxO0Ct6tWQkmsVFDAn/vBD9+tKSkpQEgSRsRa82BCFhZaT+4e0XPyoo3gWLnSCZyV4sT6sh8yKTMQhLVoKdZvGkFIN7dzJa9es2s0FBcDSpdZ3Hgk9XYql+LtHHnkEAwYMQOvWrdG6dWsMGzYMM2e+Acep45G///3v4ThOvaNLJhmqLViagfqdiaiQiD6ted+BiHZmcLtHADwMnvY9AsBFjuOoG5+OBe+T2wecfuUxr5uWlsbsrngPMbnWyJHeWQouu+wyXHbZZcEUWFjIUekhBYVbnb0KKZnu8OHMib367GuvvYbXXnst8wJDir8Ts3JWB6Vhw3hqyfJuCSJLQWib2os8TSGR8MMO4xg8axg5kl3uIew8Ul3Ns9uhQORpshzq8NZbLHdWxSGk5eLCkRYawbPkVjvwwANx//33Y/Hixfj4448xYsRIXHjhWVi+/NN6jsK+ffti06ZNtceyZcvSL7RbN6Bv30D1nd3AgsxwLIDVRPQ1EVWBt0abqFwzEcDTxPgQwAGO43R1u2lVVQh6NaTkWqFOLe3Zw3m7Qsp/Zz25f0g7jxxwQMhTSwsWsEelZ0+rxYTCI4WxFJJ3aN06zoFtHQsXsmfScvxddTU7P6132ZEj2VgKcGpJh5NO4nE8lL4U4n7bRUXsyD32WIuFDBnCWf0Dqrw//akhD1mwAPjzn+vy4dkwlrZv3w7HcfDggw9i6NChyG/ZEocNHoy5AevxiRMnYuzYsejduzcOO+ww/N//3Y2WLVvh44/rp0DLzs5Gly5dao+OmQ5aI0Zwp62u9r7WB5oywesOQJ5bXF9zLtVrGsC6wgtpU/v+/ZkEhRJ4LJZ5WWbHFRUhrePo0sX/CogMMXIkT9Hu3Wu5IJGEOqSVmQccYD25P/+XpUuBoLfb0xQDhEQeioqYvFoLXmR8/DHbZdY3yggpE3Gq2zRmhBBDHYqKOOolJ8diITk5XEhAlTd0KHDBBXVD3IIF/HnoUJ6mJbKTD29JzYzLww8/jPvuuw+ffvghBvTvj4svuQTl5fUzqt1zzz1o2bKl67HQx8riRCKB5557AXv37sGJJ55Q77uvv/4a3bt3R69evTBp0iR8/fXXmf3BkSO50wa0p739xDHpQ2c2qTaBn2vgOM41qNlBIze3D4qLn8XSwJOvSEgk0LJZM1Q//jgqA2LiJhxySDPMnBnDM8/s1RqaW7Zw7ulnn33W9T579+5FC5d4oLznnkNudjZKv/0W8LhXJvjiiyxUVLRAVtZePPus3SmfvO7dkfvWWyidMkWrXb3qxD+yUF3dAnfdtRcDB+r/0/s1+d5yMtDysTVr0PK771Cel4dqS20k6uT111vg0EOTeOGFwNNU1kOsuhotAZT94Q+IH3ectXKIgHbtWuDJJ5No0SL1/5SKrLSYOhXJQw9F+dSpKZeTCqZNywWQj+3bd9vssgCAFl27IvnMMyjv1q32XHD9pw4dO+Zh+vRcPPFEqdXwxZw330QzAKUlJaCAK0+ul127HCxf3gpHH12BZ5+1m+Qv94ADkP/ll+iWSKA6w3HppJOA555zcMEFWbjmmiSeeCKG555L4KSTqJbYVVX5nzklIjg+PKWffPIJsrKyMH36dPQ97DCgogJ//OMf0e+oo7Bs2TIcLVmcP/zhD3H22We73A3o3r27sS6WLVuGU045BRUVFWjZsiWeffZlHHHE4bXXH3PMMfj3v/+Nvn37Ytu2bbj33ntxwgknoLi4GO3bt/f1v5PJZP2xefdutAZQ8cADvn7vCSJqkgeAYQDmSJ9vA3Cbcs0/AVwkfV4BoKvbfQcOHEihYPx4ot69rRfz+OO8ZunLL/XfDx8+nIYPH+55n23btrlfMGgQ0amnpv6AKeLOO4mysohKSqwXRTRtGlfe229rv/asE5/Ys4coJ4foV78yXzN58mSaPHlyZgX96U/8fzZuzOw+Lti2bRt9/TUX8/e/WyumDlVVRC1aEF1/vfWiLr+cqGNHokQi9d/6lpVt27jy7r479UJSxIgR3G1DwfXXcztVVdWeCqr/yFiwgKtv+vTAb10fEycSHXKIlVvL9fLMM/x/Fi+2UlR9fPIJEUCLP/44sFveeSc//5131j+/ezfR3r3+75Pw2ekuvvhiOuuss/hDVRVRSQl9+/XXBIAWB1yJlZWVtGrVKvroo0V08823Uvv27WnZsmXG60tLS6ljx47017/+1XcZ2mfu359o1CgC8DFlyKOa8hTtIgB9HMfp5ThOLoBJAF5XrnkdwOU1q2mPB1BC9XfCaDyEtE+jV0q3X/ziF/jFL36RWSE7dnB8TQjxd2IdR+vW1ouqWwFheY67RQteL+BWzLBhwzBs2LDMClqwADj8cKCraxhqxghpHQcj4KklN4wcybNyn31msZC33+ZXy/Om5eW8CUgobQRwQXv3clobixBJCqx22USC28n63DaLddu2wMCB1oviQtq2DWzudMEC4LHHgDvv5Fc5Iik7204+vOLi4jovXc0Ci0WLFyM/Px99+/atd22mU7S5ubno3bs3jj56CH7/+3sxcOAgPPjgg8brW7ZsiSOPPBKrMt1ubMSIwFYSNVmCR0RxAD8BMAfAcgAvEdHnjuNc6zjOtTWXzQLwNYDVAP4F4PpGeVgdQgrqOfRQ9wTY48ePx/jx4zMrRPRcywSvtJQz1YfAIxkHHBDaPo0jR3L6l1279N/37du3gYJKCSKiPqSYoc6deX/QUDByJPDll8DGjdaLASx32aIiDiYLKqm1AR98wHnJQyN4p57Kr5b1XV5eCNs0FheHst82EFoeZUZWFrdTAARPxNy99BLwhz/wqxyTJ6Zmg4zDq6iowIoVK5AUN63ZVuKhv/0NkyZNQnMlieC1116L4uJi18NPcnmxUJcoicrKStfn+/LLL9E1UwN75Ei20AJAkyV4AEBEs4joMCI6lIjurjn3OBE9XvOeiOiGmu+PIiJt3r1GQUgrILwSYK9YsQIrVqzIrJCQBqV33w1lHUd9FBTw6mDL+zQWFLA1a8pSsH37dmzPZCHBokXsQbHsdQhxcWEdQkrx0KMHZ8Sw2mWLijh5oNWIei4mK4uLCgXt27OHKARPa0EBe1lrwouDRyh5moA1a4C1a0PWdyNHBrICYtEiJnWiikaM4M/CgWsjH55IQfL8889j4dtvY8WKFbjs6quxevVq3HvvvQ2ub9euHXr37u16NGvWTFvWrbfeioULF2Lt2rUoLl6Gu+66DW+99RYuueSS2mtuueUWvP3221izZg3+97//4bzzzsPevXtxxRVXZPZHxbYtAaBJE7x9GmJT+xCSa40cyckydQtHfvzjH+PHP/5xZgUUFrLQhTAo5eZy0vXQENI+jccey1nqTePfzJkzMXPmzPQLKCxkmbM8KK1alYXNm0MelAYOZG9rSOTh7bct5fHasIE9kaNGWbh5fRQV8YrGUEIdBEaO5HnhigqrxVhPUlBYyO7p/SnUQUDohwwF/Fe/aqhqRozg8wB7JGOxYPtRcXEx+vTpg7vuugsXXXIJjj75ZJTu3YtFixZllmBYg82bN+PSSy9F3759ceaZBViyZBFmz56NsWPrdmNdv349LrroIvTt2xfnnHMO8vLy8OGHH6Jnpimq2rYFBg3K7B41iAieTRQU8LRSph40D1idWlq/njfqDCkp6wkn8M46oeGEE5hVWva05uayN8UaRyksZKXgc/VWuli4kEl+qIOSmFoKKQ6vtJRTjAQOIWOWYxB27w451EFg5EieF/7gA+9rM8DgwUCbNpa6bGUlG3sh5fvs0oXDZkPDEUewIWg5KTUQfD684uJiHHXUUZg0aRLWr1qFss2bMW3aNHTv7pkZLWVMmTIF33zzDfburcRXX23FnDnzcfrpp9e75oUXXsDGjRtRVVWFDRs2YOrUqTgiqLiVgAz1iODZREj7NHbvzgmwrYx/4qaWFV6I6zjqo3lzJnkheYe++ALYFPQyoL172XMSgmdo4cIcHHwwcMghnpcGi5Ej6+a0LMJqSrfCQt4T7aijLNy8DgsX8sAaKgkHeDFMVpb1vmSV73/wAcc/We5LjRLqAKB2ry0bKyAUiGnaoLhkcXExBgwYwB/icWaQlitPeCCt7XxlQkCdNyJ4NiF2FAjJ8xBgAuw6hDQovf12CNv1mFBQwOzS8j6N4r8FPrX07rvc8JbZcSIBvPdeTuO0kbXKq48OHXhGOHCbjIg3ex850npEfVERL0bIdFF2ymjdmhcthaTvvv7aAt8vLOT2GT484BvXx/LlCD/UQSAWs5eJWIIgRUFM0xIRli1bxgQvmeTnD4F1JRJMVEMl4QAwZkwgt4kInk3IKyBC2NR+z56AsxQIMzOEZV6FhZxOZOhQq8XoIQKPLQfxDxrE4RWBk4fCQp4DPumkgG9cH0uXAt99F2ucQemII4BOnULztL73XsChZCtWcLhGSPF3oYc6CIwcyfPDISxaAiz1paFDeQ7YIkKaGNFD6HLLY5LjMDkKohjHcbB7925MmDChjjEKF6ElENURvNARUKERwbONkSM5N4bVrTPqshSoCu83v/kNfvOb36R301WrOAYvpHgU69v1mDB0KK8StjyV7ja1dMopp+CUU05J78bz57O7xmZqf9Q9dwjpwRpCLCAJadFSZSXPegeGkOLvGi3UQWDkSB6A333XajFHHMGpegLl+yEGLxYV8QTPwQdbL6oh5GlayxAEL9AuG4/z81t2OjTa9GyAiAiebQh3h2Xy0L49e4hUhTdq1CiMStdrIJ7Zsstm40ZeXNhog5JIphtCPryCAp5WUrcsPOSQQ3BIOoFt27eHNqIXFQF9+sQh7UYVLkaOZGFZudJqMSefbCGUbP58hBG8KGawG8XLCgAnnsj9ybKn1UqSgnfeYTYSQqhDSFtGmxH0CgiXYoAAuaRwq4UQfyc8j43iwQsIEcGzja5dgX79Qptaev/9+jkSRULHtFBUxMnBevcO5PncigEaWeEVFDBxWL/eajGmFc+bN2/G5s2bU7/hggWs9CxP/VVV8fh38sl291Z2RUhxeK1bc1qbwPi+GNFDIuEhpKw0o3lz9iaHpO82b+Z4tkAwfz5vk2E5T9Nnn2Vh165G1ndZWaHE4QW90KI2/i4E1lWTRzn8+LsAERG8MFBQwKNjld3NpEeO5CLkXU5uuukm3HTTTanfLJnkgbSgwLqEFxUB7dqFtF2PCV57vgUEsZOYWsybb76JN998M/UbFhYCrVpZD14UeZQbleCJbVtCCuJftIhn7TLG4sW8M0JI8XeNFuogMGIEsHgxnO++s1pM4JMjhYUcx5qfH9AN9Vi4MBdAI4U6CATuWtNDxOEFVkxI86bJJB/78vQsEBG8cFBQAJSV8Y4JFnHyySyQgYx/n37KAT2WzUwi1quhbddjwlFH8RLKEHYeCXRqSSShtqyJior42U84oREJnojDM23bEiAKCtjr8M47AdwspFCHDRt4LUejeoaA2kVLOZbz4fXqxUcg+m7LFt4eIwQv68KFOejXz3oeZXeITMQh5cMThCljJBJ1z24R+8P0LBARvHAgNrW37Hlo1YqnlgIpJqSg8K+/Br79tgkMSrFYaEH8BQU8nnzxRYY3+uYbYPXq0DxDgwYB7drZrRtPjBzJcYeffWa1GLGpfSB9af58NiA6dQrgZmaEtGW0N447DmjWDDmWd4cBWBzeeisAnhLSstaqKuDDDxsp1ZAK4VrbV/LhEdXNm1pGSAt1rSMieGGgbVtOvx7SpvaLFgElJRneaN48nk+0HFHfJOLvBAoKOAZv1SqrxQQ2tRQSCS8v59jORicOgOVMxHXIz+f1Ahm3UUUFx0yEUHmFhRzqIHLBNhry8oCTTkKO5ZW0AFfrd9/xLHhGKCzk7fAGDw7gqcxYtAgoK3Oahr4TXv+Q4vAynqYVDDGk/HchrOOwjojghYWRI4EPP+RAJosoKOD+mtHUUmUl32D06MCey4SiIp6q6NvXelHesLrnWx169uRwsoyLKSzkXBFHHhnIc5nw3nvseWgSg9JBBwF9+rBXzDJGjuRIha1bM7iJ2JvVMsELMWWlP4wciWyRzdduMQAy7EsiCfWIEdZdNhzqQLVprRoVgTEvdzhO3aLdjBDSvKmYTt7XvXdARPDCQ0EB7zZg2ao9/nj2PgjPwz333IN77rkntZuIpbiWCZ4YlEJYx+EPvXtzEH9InlZ5aqmgoAAFqZAAEbwY0iKY7GzreZT9Y/RorjzLi5ZEc2SU/3r+fB4pLO+M0GRCHQSE7rBMxIV9k1GX/fprDncIaZXzUUcl0K6d9aK8IWLZQsqHl3EcXjweSvzd/pD/TiAieGEhpPxQ+fk8EItiTjjhBJyQ6rL/efNYui2bmZ9/zt6RJjMoiRUQIQXxl5TUTS316NEDPXr08H+Dzz/nQL6QBqVjj+UYzyaB0aPZE245iP+YYzhlSkbkobCQY9IsV16TCnUAgKOPRrJdO9YlllFQwHZzZWWaNwg51OHkk+0aJilhX8mHJ+W/u+GGG3DOOecE9mwqEolQ8iiHgv3gL+wjaNGCI7dD8g4tW8bk6f3338f7qabknzePXYHft0EJYCW/YwfPzVmE4M6iDtatW4d169b5v4GQI8sLLEpKgI8/bmJtJKbSLJOH7Gx2vKVtk333HVdeCCR8/nwOl20SoQ4AEIuh+pRTuI1C2HmkvJwjYNJCYWEolScyZZ10UiOuRFchmJfl1bSxGJOmtIuR4u/++Mc/4umnnw7s2d555x1MmDAB3bt3h+M4eOqpKQ3y3z366KPo1asX8vPzccwxx2ChZgGRn2vCRkTwwkRBAbtsdu60XgzAA9Ptt9+O22+/3f+Pd+wAPvkklPi7uXM5Fq1nT+tF+UdIO4907gz0719XTGFhIQpTKXP+fJ5SPuggOw9Yg6Ii1q0hiIN/tGnDXrEQvEMjR/JC5W+/TePHb73FnuAQdkaYNw847bQmEupQg6rhw4FNm9jbbBEiSUFaXTaZZCEfNcp65c2dy+tPGjXVkIoQ4/AyWrQrLWtt27YtWrZsGdiz7dmzB/3798ff/vY3NGvWDET1p2dffPFF/OxnP8Ptt9+OJUuW4IQTTsDYsWPxraQU/FzTGIgIXpgQm9q//bbVYo45hhfuzpmTxo9FmhDLI3plJc+EjhljtZjU0b07W/IhJNMdPRpYuLD+ziO+EI+zDIXgGZozhx25w4ZZLyo1jB7N3jHLxpJwkKbFJQsLeWeH448P9JlUfPIJb3d92mlWi0kZ1cJNbZmIH3AA79yRVrjfsmWcdiekvnTyySwSTQaBZyI2Izs7g80zatKjrN+wAY7jYMWKFYE917hx43DPPffgvPPOQ6xmXlYmeA888ACuvPJKXH311ejXrx/+8Y9/oGvXrnjsscdSuqYxEBG8MHHssTxVG8Km9qNHs0JJ2VqaN48DjyzvjPDee5z7+fTTrRaTHsTOI9V2Le0xY3iBZcp8f9EioLTU+vQsEcvQyJGNvDOCDqNH13lfLOLII3n2Li1jqbCQR/S8vMCfS8acOTxONykvK4DkgQeysTR3rvWyTj+d88jv2pXiD0OKv1u/nh2ZTVLfBZqJ2L0YII1pWsEKs7NRXFyM5s2bo0+fPvUuueeee9CyZUvXw++UqRx/V1VVhU8++QSnKdbTaaedVhv65OeaxsJ+sE5kH0JuLiv8ELxDp58OvPQS0K3bIWjZ8mvvHwDckebN4xgny0uI5sxh0tCo2/WYUFAAPPooshcvBs44w1oxJ5/Mi2LmzElxm7b58+t2dbCIVauAtWuBX/3KajHpQaz6mDcPOO88a8U4DhPxV19lJ4LvbrFhA2+SetVV1p5NYM4c9tp36GC9qNRx2mnAv//NLnuLRPf004H/+z/uGuefn8IP589nEtq9u7VnA+qcmE3NywoAyM7GTTcDxZ8DsDzFLy9gGDQIeOghHz+SpmeXLl2KAQMG1HraBK699lpccMEFrrfp7tHGwhkip0fZvn07EokEOnfuXO/azp07Y36Ny9jPNY2FyIMXNgoKWPFv3Gi1GKFIdu1KwRP31Vc8oofgCpgzhxcWBxhKERxOPRVwHOuJWps14/ihlLegLSxk7di+vY3HqoXwWjVJr0NODrsW5861HsR/+um8XuKjj1L4UUieoZISXlzQJNsIYF0ilo9axHHHcWhmSp7Wykp2n4ewE8ycOZzv86ijrBeVOmIxAI71fgQwuUu5GIngFRcXY9CgQQ0uadeuHXr37u16NGvWzLUY4VnUrZ51lPhMImpwzs81YSPy4IUNofDnzwcuv9xaMQceyNNLLVpcioce8hlAJbSjZTNz0yZg6VLg3nutFpM+2rUDjj4auZZjJQH2Dt18M3DkkePQo4ePKZLdu3l++xe/sP5sc+bwOo5DDrFeVHoYPRqYPp0Nk969rRUzahQr/TlzAN8Zh2bP5q3JNINRkBCLYJqkZwhgYyk7u25mwBKys1kc3nyTCYSvcfWddzhOZOxYa88F1C2CGT++aS2CqYXj4KG/JoB4JVvcFh8yHucqb97cpzdcbE9Ws63E0qVL8QuN7vOT73X27Nk4+eSTXZ8NqE/wOnTogKysLGxWEnZv3bq11mPn55rGQuTBCxsDB/ISytmzrRc1ZgxQXNwKffoM8veDWbN4oFTiG4KGCMlpsl4HADjtNGQHsuebO0QdLFnSCV26dPH+wfz5rInGjbP6XGIRTJNuI/FwKbtAU0O7duwh8l1MPM5scOxY68m0muwiGAHxcJbbCGB9t2FDCot2Z8/maWPLoQ4icUKTJeEAz0umvQIitWKAFNZ0JJMQy1r37t2Lr776SuvBu/baa1FcXOx6DBkyxLUo8Uwyv83NzcUxxxyDecpCoXnz5tXml/VzTWMhInhhIxZjxT9njvWVS9nZnHepZUvCwQcDzz7rcnF5ObsDLBGHP/2pbjP0OXPYubFzJ59vkjjjDDjxuPUVgIcfzplOpk7di6+/9hErOWsWz0VZHtGb9CIYAWGMzJplvajTT+e1LTt2+Lj4o4842t8yCW/Si2BkjBsHLFnCrnuLSJnvz57NHkbLy1rFxEhTWwRTDxlnIvaHlBftSttKfFqTm/QozTx3JlO0e/bsweLFTAKTySS+/fZbFBcX16Y4+fnPf44pU6bg3//+N5YvX46f/exn2LhxI6699trae/i5plFARE3uANAOwDwAq2pe2xquWwtgGYBiAB/7uffAgQOp0fHyy0QA0bvvWivimWeImjfnYsTRvDmf1+G755/ni95808rzFBURdehANH8+v44axa9FRVaKyxzV1ZRo04boqqusFzViBJHjJAlIUs+e5jaiZJKoa1ei88+3/ky/+hVRTg5RaWn989u2bbNedkr42c+I8vOJ9u61WsyHH3L3eP55/ff16uWOO4hiMaKdO60+08qV/EyPPGK1mLRRWyeffsoP+u9/Wy+zf3/WLZ74+mt+pocesv5MJ51ENHhw3efG7kOLFy/Wf1FaSrRnj/XyKyqISkqIEom6cwn5g4w9e2qV0GOPPUaHH3544M+zYMECAtDguOKKK2qveeSRR6hnz56Um5tLgwcPprfffrvBffxco8LYFkTkl9O4HY1O5rQPBfwJwK01728FcL/hurUAOqRy7yZB8L77jigri+i226wV0bNnfXInjp499deX/fCHRM2aEZWXW3umoiKiNm34OVq2bMLkrgYVZ51F1LlzfU0UMJ55hig31ycRX7KEL5g82drzCAwYQHTqqQ3PN/bg1ABz53KdzJhhtZh4nKhdOyJJ59dDvXoZPJhHdcv429/4r69ebb2otFBbJ8kkUY8eRGefbb3MX/yC+5MnT3nkEa68lSutPs/Onazqb7+97lxj9yEjqSgvb8i8LCAe52IqK+vOaQleMskXWhyTBPbuJdq9m4sME7YJXlOdop0I4Kma908BOKvxHsUC2rThDWMtTi2ZEmhrzxMhV2xcn59v7ZlGjKhLB3LNNU00RYqEqtGjeb/XJUuslXHHHTyNLqOsjM83gJAXy9mhv/2Wd2qzPMMYDE45hXNLWp6mzcriGCrP3JKbNnHQleXAfQCYMYOn+A891HpRmcFxWJjmzctgw1h/GDOG+9Nbb3lcOGsWV5zleOPZs3mRxfjxVosJBmKeP6RtyzynaaXpWZug+us49is0VYLXmYg2AUDNayfDdQRgruM4nziOc01oTxcExo3jpaQbNli5vWkHK+35lSuRtXat9RF9wQKO7TrwQODpp+ti8poqqkaO5B7/xhvWykiJiM+aBQweDPhZjJEBZszg1wkTrBYTDPLyeJnrG29YT/MwdiyweTPzNyNEAJjlvrR7N2f42CeIA8D5JPfsASynHhI7Rbh22YoKq/HGMmbO5HjjY4+1XlTmEMzLcoJ3x2Ey5bltmZQexSakbW73OzTaX3IcZz4A3Uil812YcCIRbXQcpxOAeY7jfElE72jKugbANQDQrVs3bN++Pa1nDhJZw4ahLYDSl15C5WWXBX7/W2/Nxc9/3grl5XUmSbNmhFtvLcX27fVdRvkvv4yWAHYedxySlurm3XdzcNVVrZBIxHDVVXsxeHAc55/fCv/+d2nT2nxbQklODloPHgy8/jpKrr/eShndu7fF+vUNFVj37gls316Xlt/ZtQvtPvgA5TfdhDLL8jt1amscckgW2rffBbWoEsuritNB3vDhaDV9Ona9+y4S/fpZK+e44xzEYu3w/PPl6NmzrN53ol5aTZuG7M6dsat7dzSovAAxfXouqqtb4+STv8P27fa3mUoH9WRlwAC0z8tDxSuvYG9KWb1Tx4gRrTB9ejbuumuX1iOTU1SENuXlKDnhBFRbbKPqamDWrHYYN64KO3fuqT3fFPpQ0rBa1qlZAUHWd7VwUF3tIB4nZGWR9pmcGrcaiegVS6iuZiHJyiLbi4i1sMpHMp3jtXEAWAGga837rgBW+PjN7wHc4nVdk4jBI+LJ/oMOIjrrLGtFPPMMUbduVQQkCSD65z8NF44aRdV9+1p7DiKi++8nuukm7qmff87nior4fFPFtm3biO66i8hxiLZutVKG78UwYhHM++9beQ6B3bs5hunnP9d/39jxQ1qsX891E4IwnXIKxyeq2LZtG1F1NQeZ/uAH1p/jssuI2rfnIpsqGsjKmDFEhx1mvdwpU1gcPv7YcIFYmFNWZvU5Fizg55g6tf75xu5DbnFfVFXFcW+WBUuE14kmaBCDpwvUs/Qcu3dbX6NlxPc1Bu91AFfUvL8CwHT1AsdxWjiO00q8B3AagM9Ce8JMEUJcyiWXABs25OCDD9hC0e4aUVoKvPMOqixnc//Vr4AVKzjsRThZRoxoottgyTjjDOZdlvJ4XXIJ8MQTQM+e/Dkvjz9fcoly4RtvcEI2y3M9c+dyDNM+MT0r0L07B3danEoXmDCB4xPXrtV8uXAh5020PPUXj/NfHTduH5tWOuMMYOVKYPVq68XEYpwDuwGIgGnTON7YY2eDTDFjBu9O2aTz36kIMV2K6zRtSPF3Upq9/RJNleDdB2C04zirAIyu+QzHcbo5jiOiqTsDeNdxnKUAPgLwBhHZz6YZJM48E9i71+retDNmzMCWLTPQqRPw+uuaC2bPBqqqUGVZC+3Zw39zwoR9LJD16KM55k1becHgkkuAOXNW4Oqrd+hzGFdV8WgxYYL1eJQZM4C2bXkbuX0KZ5zBAZ47d1otRhBfrTi8+iovUrK8COaDD/hv7jPxdwJiX2fLRLxDB5ZfLcErLga++QY45xyrzwBwXxoxoolux2hCyonq0kd2tktu5epqfg7LicJD4pGNhiZJ8IhoBxEVEFGfmtedNec3EtG4mvdfE9HAmuNIIrq7cZ86DYwaxZnep061VsRf//pXPPjgXzF+fC2Xq4+pU4FOnRA/7jhrzwDUOSr3uUEpFgPOOosXOJSVeV6eLj744AN07vwhEgnNJicLFrBnyPKglEjso54hgNsokahbIWIJffqwB7oBwUsmgddeY3LXooXVZ5gxgxc8Nukk1Dr06gX078/1ZBkTJ7Kndc0a5YtXX+U+bVkRrVwJrFq1D+o7gDt/Mml9VwuhYxqs6RBlh6CE4nEWB8s8stGwn/6tfQR5eezFmz7dusU0cSKvvHtHXoJSUcEj+tlnh+IZOuAAzg6zz+Hcc5ncpbSTeero1Wub3tP66qtMGiynwv/wQ14XsE8OSkOG8BLxV16xXtTEibyC9bvv6s5lL1nCK+JD8gwNHw60bm29qOBx3nmshLZssVrMxIn82qAvvfYap9bp2NFq+cLOOPNMq8XYgUiXYnk1bSxmcBaKci1vz5JMsk24zxmzKSAieI2Nc8/lUXXhQqvFiJCTegpv7lyeIg7BMzRzJqeZaNJbKpkwfDjHv1n0tAJ1joV6ntZEgmOGzjjDao5CgGUjO9v6DKMdOA73pblz2ZKxiAkTeFCSU+/lvvEGV57lEX3lSuDLL/dREg5wGxFZ9+L17g0ccYQyTbtyJW9Ue/bZVssG+O8NHFgXW7tPQbi0LBM8oM5ZSCTF7YTkVhPEcp8ck3wiIniNjTFjmHlZJg/Nm7MDaNo0yfP+6qvsVrOccfjdd4Ft2+qs6n0OOTn88DNmWE/UKjyttYla338f2LqVB0aLIAJefpn3NW3TxmpR9nDuucyMZ860WsxxxwGdO0vkgQh5M2dy5bVta7Xsl17i1xAchXZw5JFA376heVrfeYe3BQZQRyrPOstquevWcTjo+edbLcYucnJCmaYV5KrWiyfcaiGwrupqtgv31+lZICJ4jY8WLZjkvfqq9c50/vmsfD74ACzdr7/O7gjLnen555lg7pPTFQLnncfMq7DQajEiLPPFF2tOvPoqT+Vb3hlh0SKOV7roIqvF2MWwYUDXrqF4WidO5OiGsjIAn3+OrDVrQvEMvfAChzkceKD1ouzAcbgvvfUWW30WIcIya4n4q6/WTeVbxMsv8+uFF1otxi5CnKaNxSSC52PVww033IBzMrRwZB65Ty36SxERwWsKOPdc3uLoww8Dv/V///tf/Pe//wXAg1J+PhMuvPUWm7aWPUPV1WysT5xoPfbcLgoKOOjJEnk4++yzcfbZZ6NZM+YJU6cClRXEg9JppzHrs4gXXuCUDpadG3YRi7Fra/ZsDj2wiIsu4iJmzADw6qsgx7Huov7sM55hnDTJajH2cd55CvOyg6FDeV3H888DWL8e+OijUEj4iy/yhjO9e1svyh4aMC97yMkBEgmH/RvxuKdb7Y9//COefvrpjMqUp2ffeecdTJgwAd27d4fjOJgyZUqD6x999FH06tUL+fn5OOaYY7BQCanyc4/GQETwmgLOPJMlzQJ56NGjB3r06AGAOcL48WxhJl8JJ3C/sBDYsWM/GJTy8rjyLC2IadOmDdrUzI1edBEvmv3g4U94zzLL83HJJE/9jRnDM/b7NM47Dygv1yxFDhYnn8zp9557DsCrryI+dCh7Dy3ixRd53DvvPKvF2MfAgcAhh1ifpnUc4OKLgfnzgd1PvconLRO8NWuYR+7T3jsBZl7hTdNW12wK6+FWa9u2LVpmmHumurqOw+7Zswf9+/fH3/72NzTT5EZ88cUX8bOf/Qy33347lixZghNOOAFjx47Ft9J+kl73aCxEBK8poE0bJlqvvBJ4Z3rxxRfxYu18HxOtnVurEX9xKufDsCyMzz/PpGGfS+mgw7nnMlv13Mk8dXz22Wf47DPO011QwLm89vzreVZ2liPq33uPF4Du8yQcYObVsaN18pCVxfX17azPgKVLUWnZe0fEBG/ECI7/26chpmkLC6UAOTu4+GJWqeVP/BcYNKguy7oliBjJCy6wWkw4MOYxCRZMtAhU5b16dv369XAcBytWrEi7PHn1LO83MA733HMPzjvvPMQ0nsMHHngAV155Ja6++mr069cP//jHP9C1a1c89thjtdd43aOx0HSe5PuOiy9mb83bbwd628cee0wRRODcZrORW7INuPzyQMtSUVHBcc3nnMMOsH0eY8bwNG2G0wM6fPzxx/j4448BsH678JxqDF35DOJjxwPt2wdenowXXmCev8+uzJSRlcVE/PXXra+mvegi4OL4U0hmZaPSspd1yRLOq7ZfkHCAA4LjcetE/IgjgLP7foHO335sXd8BTMKPOw44+GDrRdmHSDQc0jRtFlWDPFbPFhcXo3nz5ujTp0+98/fccw9atmzpeohp1VRWz1ZVVeGTTz7BacpGAKeddhref//91P5kI2A/zgCzj+Hss5k8TJlidVVrfj5wS8cp2LKuM9oMPx02E2/Mns07oe03g1KzZjyqP/008PDDVhORXXfIHHTGVrx16BU41VoprOxefpnJ3T6Vcd8NV14JPP44u1N+9CNrxQweEEePrGfwftszcHiHDtbKAZiEZ2fvw6tnVRxzDLOvyZOBq6+2WtSvuz6N+IosrD/hYhxssZyVK5mIP/CAxUJs4aabeJcPFWIvr1gs+NUIgwYBDz0EAMiJJRFDAtVOHnJcylm6dCkGDBjQwEt27bXX4gIPt2n37t0B1E3P+kn9un37diQSCXRW3OadO3fG/PnzvW/QyIg8eE0FzZtz4MYrrzArsoVt2zB4wwz8ly7Fm4X2V8926mQ9C0u4uOoqjvESczGWcMRHU7A91hEPLLe7enbBAl7MuN+QcID36z3iCODJJ60W48yfh06JzXhg+xXYtMmeKk0k2DN02mmcjnG/gOMAP/gBL+lfvtxeOYkEjvnyGbyJMXhmnt257Wef5b+1T6dHUSGIlHbD2ODgJKpBACqTOa5FFRcXY9CgQQ3Ot2vXDr1793Y9mjVrlnYWFkchnUTU4FxTROTBa0q46irgX/9il8oPfmCnjOefRywRx4y2V6D90/ZWTW7fzrNkV1+9n2UKP/ZYjuOZPNmed2jnTjgzZ2D50ddj9vwcbN3KRNkG/vUvTt1mOQtLuHAc7ku//CWTB1txV089hcQB7THzuzNw9LQqHHWUnWLmzuXojT/9yc79Gw2XXgrceisT8T//2U4ZCxYge/MGfNzvAbz4LHDHHXbSYsTjwL//zbHG+2QKmxpPmhZlZcyKWra0U3lEQHU1KJaFZDLmurvE0qVL8Ytf/KLB+XvuuQf33HOPazGzZ8/G0KEnA/BP8Dp06ICsrCxs3ry53vmtW7c28Oo1RUQevKaE44/nJKA2l1hPmQIccwyO+9FReP11zh5gA5Mnc07gH//Yzv0bDYI8vP8+kEGgryteeAGoqkK3269EPA785z92itm0iWMkr7rK+iYZ4eOyy3gOxlZf+u47YNo0ZF16EY45PhdPP51vbbHh44/zwooQMnyEi86dOYPA00/bC+R/6imgTRsc9JMJ+PLLmhygFvDGG8DGjcC119q5f6MiJ4dJmK1YvEQCDhGc3FwAZlHYu3cvvvrqK60H79prr0VxcbHrccwxQ1BdzeTR7zqI3NxcHHPMMZg3b1698/PmzcMJJ5yQyr9sHBDR9+oYOHAgNWncey8RQLRqVSC327ZtG23bto0/FBfzvf/xD/r6ayLHIbrzzrrrgkIiQXTIIUQnnxzYLRsFxjrZtIkoK4vo1lsDK2vv3r20d+9e/jB0KFGNnI4YQXTQQUTxeGBF1eL//o/FYeVK/78JUk6sY+JEoi5diKqrg7/3P//JlbdoEf33v/x27tzgi/n2W6JYjOj224O/t234kpUZM7jypk0L/gF27yZq3pzommuotJSoTRuiSZOCL4aIaMwYou7d/YlaY/ehxYsXp/aDZJLrUuinoFFWRsmSEqJkksrKiEpKeAxR8f7771MsFqvTkymiqorvXVVV/3xpaSktWbKElixZQs2aNaO77rqLlixZQt988w0REb3wwguUk5ND//rXv+iLL76gG2+8kVq0aEFr1671fQ8T3NoCwMeUId9pdMIV9tHkCd769fY0+o03EuXkEG3fTkREZ5zB419lZbBKZ/Zslqznnw/slo0C1zo580yirl2DJw9Ll3LlPfAAERFNnWpn/IvHiXr0IBo1KrXfNfbglBKmTePKmzEj2Psmk0SDBxMdeSRRMkkVFUQdOiRowoRgiyFiA8xxiNasCf7etuFLVqqrWQnZqLx//IPb/4MPiIjo5puJsrOJNmwIthhhLP/ud/6ub+w+lDLBIyIqLzczr0yQSBCVlFCyrIyIWC+VlBBVVDS89LHHHqPDDz887aL27GGemkzWP79gwQIC0OC44ooraq955JFHqGfPnpSbm0uDBw+mt99+O+V76BARvO8bwSMiGj+eqH17lsgMMXnyZJo8eTLRzp1ELVsSXXxx7XdvvMES8MILwSqdCROIOnVi4rgvw7VOBHl48cVAyhLWH112GVGLFtxexOPfgQcSjR4dSDG1eP11fvypU1P7XWMPTimhqoqoc2d2rwSJwkKuvCeeqD118817AydiVVVsQ4wbF9w9w4RvWfn1r9kj7uHtSAnxOFGvXkTDhtWeWr06NSLmF7fdxjb5unX+rm/sPpQWwXNjXpmgooIJnmQom4hYJqjhkYE/fqawTfCiGLymiF//mhPqBrAKcMqUKbxtymOPAXv2AL/6Ve13Y8bwVj6PPJJxMbX45hve6/1HP+Ktr/ZbjB/P8ZL33hvICrPi4mKsKizkpcfXXFO7aX12Nsf1zJsXbMjfY48B3brtJ7nvTMjJAW68EXjzTc5fERT+/GeOH7vsstpTV1xRgViM6zUozJjBcZLXXRfcPZskrr+eY1v/+tfg7vnaa7ytxC231J469FDOA/rPfwJVVcEUU1XFavrMM/fRxRV+kZXFR1VVcCtqifh+2dmc/64GubnBh/yJ9ra87XqTQ0TwmiJOPJGPv/41kODj3EQC+NvfmNENHFh7PhbjwWPhQuDzz30kBfKBxx/nznnNNYHcrukiFmOyXFzMyxwDwBHiPjfdVO/8j37EiunRRwMpBitXMucR992vcf31nK/wvvuCud+yZVx5P/1pvZUp3bsncdZZvJKyvDzzYoh4YWOPHvvZCmcdDjqIyfK//gVs3Zr5/YiYhB96aIP9gX/yE2Dz5uDyKz/3HLBly366uEJFXl7titdAUF3N91M8AWKHiaBIuMQjfS+u2G+QqQtwXzv2iSlaorrg4//+N/17PPMMbcrLo5pUlUR33NHgku3bOQ554sTMfddbtvAs8HnnZXyrJgHPqZTKSp4/PfXUzAp65hna07Ytt1OLFkTPPNPgkksu4brdvDmzoog40Lx58/Tu1djTS2nh1lt5bm7FiszvdfnlXHk7dtQ7vW3bNnrrLe5mDz+ceTFz5pBYD7XPIiVZ+fJLbqPbbsu84IULufIeeaTBV4kEUZ8+REOGZD4FWFlJdPDBRMcck9q9GrsPpTVFKxDU/GkySVRaykcySQkltq9m5jaQxWUifNDGQrVMEcXgfV8JXiJB1L8/H+l0pmee4YFIkDuAP2vIw513klgQmBF+9jOORVm+PLP7NBX4UsQPPkhyIHfK8NlOX37JYUo33JBeMQJLlnAR6a7haezBKS1s3kyUn0/0wx9mdp916zhK/6c/bfDVtm3bKJkkOuUUjj/dvTv9YpJJJiA9eza9mKFUkLKsnH8+UevWRLt2ZVbwxIkcw2xYbfnkkxRI+Oyjj/J9Zs9O7XeN3YcyInjV1cyWMg2wVpa0qgQvmeSvM124K2LvbC0AzhQRwfu+EjwioqefprSXUPbsWZ80iKNnzwaXlpQQtW+foJEj0zfM1q4lys3NfAxtSvCliEtLidq144Ux6SCFdrr2WuYXqaQ1UXHGGURt26Y/hjb24JQ2briBV5B/+23697jmGmbZX3/d4CtRL//7HzefSD+UDl59le8xeXL692gKSFlWhPXxxz+mX+i77/I9fvtb4yXxONGAAbwGI10CXVZG1K0b0Uknpa4zG7sPLVmypAGh8g3hecvEi6e5h+55hBcvk0QFTdl7l0wmeWGdARHB298JXlUV0eGH8zxAqitqHUdPHBxHe/ndd5cSwFND6eDKK4ny8vyvJNsX4FsR33031+2sWakXkkI7bdrEM7jpToGLse+++9L7PVHjD05pQ1gg55+f3u/ff58r7+abtV/L9XLhheyETScdRzxOdMQRRH372knfFybSkpUzzuCEdelUXnU1M7cePZhAuGDuXG7Ov/419WKI+HcA0Vtvpf7bxu5Dn3/+Oe3JJEOD8L6l68UTzE1KSKcjeCL9Xs0sbsoQ3ruaDCxNDhUVFbRs2TLj9xHB298JHhHRO+9wM/3iF6n9LgXPEBHR+vXbqFcvokGDUk91tGwZT83ecktqv2vq8K2IKyp4VO7RI/W5uc6dU2qn3/2Ov/7ww9SKSSSITjyRU45lMl3R2INTRvjjH7nyXn01td9VV3Pi6e7dje0r18tXX7Gz8Ec/Sv0R//1vfsSXXkr9t00NacnKypU8nT5+fOqjugiX8Nm+Y8YQHXBAg3BKT2zeTNShQ/qpixq7D+3cuZOWLVtGe/bsSc+Tl0yywyGdvHiGOVPTc2TCJd2SJjc2EokEffXVV7TBxZAJguA5fJ/vDwYNGkTFxcWN/Rip4dpreYXZRx8Bxxzj7ze//S3wf/9X/1zz5sATTwCXXNLg8u3bt2PevA64+GL+2W9+46+YvXuBYcN4m54VK4D27f39bl/A9u3b0aFDB38Xf/ABr3y+/nrg4Yf9/aa6GjjkEGDDhvqpB1zaqbQU6NOHs3S8/z7QooW/ov74R+DOO3nbs0y2OU6pTpoaqqt5L+HNm4EvvqhNReOJBx8Efv5zYOpU4JxztJeo9XLzzcDf/w4UFQHDh/sr5ssvgSFDuIsvWLDvr/hLW1YeeAD4xS+AZ57R9gEtNmwADj8cOOUUztPkY8/Uzz7jpAJXXsmrn/1ss5pMcjKChQuBRYuA/v39PZ6MptCHdu3ahU2bNqGqqgppcQCqWZoai6W2FL+6misxN9f3vraGxbauSCZRuy1ZVjAJIgKF4zho1aoVevXqhZihozuO8wkRDcmooEwZ4r527HMePCKi777jjKeDBjXcZ0WHvXuJ+vUjat+eNuflUUJ4hDQLLAREkPgll/DsoJ/ZxmSS6NJL+fp0p3abMlK2tG+8kSvj3Xf9XS/2Crv5Zipt355X0Xq0ExHRm2+yx/SCC/w5OebP58e65JLMF781tvchYyxezHF0V17p7/oVK3j58hlnuFaeWi8lJRxd0aGDv+THe/fyeqqOHYPfaaGxkLasxOOcoLhtW45L8EJ1NbdPfj67T1PAbbdRSquV77mH1BzXKWOf70MCTzzBlfHoo/6uf+EFvv73v2/wlVudLFlC1KwZ0QkncEydF1at4i570kn7dpgD9tcpWgDnA/gcQBLAEJfrxgBYAWA1gFv93HufJHhEdZHX48e7S/mePbyBqeMQzZ5Nw4cPp+HDh3veXnSwvXuZRx5wgPd2uGIV2R/+kML/2IeQsiIuLeV4yU6dWCu54eGHufJqNses3XHEJ+6/n3zF061fz49zxBGeYUm+sF8MTrffzpV3773ujHfVKo6k79jRk6Xp6mXFCg4nGzDAu+6vumr/M5QykpXlyzmod9gw9znUeJx35zGkRfFCPM4qNSvLey/hhQv5ukmTMjOU9os+RMSVcNppHHBaWOh+7YwZvELspJO045dXnbz8Mjexl5FaUcFpa9q2DXZjlMbA/kzw+gHoC+AtE8EDkAXgKwCHAMgFsBTAEV733mcJHlEdoxo1Sr/ooqSEO1AsVusFSpXgEfEiwXbtOND7k08aXptIEP3tbxxnNHZs04xxCAJpKeIVKzgWr00bsydP7JE5YUJtcEmqBC+Z5GB+x+Hb6VaJLVrEXqQWLYi++CL1v6LDfjE4VVXVkYKf/1wvwKtXc47DDh04yNQDpnoR3taxYzlPpIrdu5ncAUS/+U2qf6RpI2NZeeUVXhjTr59+tE4k6irv3nvTLmb3bqKjjuIuq2wxWotnniFq1Yro0ENZzWaC/aIPCWzcyHsy5+Rw1gcd5s9nsn7MMTwbpYGfOhFr2X71K31M3urVnF4onTDbpoj9luDVPpw7wRsGYI70+TYAt3ndc58meEREU6bwiDFoENFjj/F8zvr1zLiOOIJNTClCOx2CR8SKrksXLurnP+dFiGvXEn38MdHJJ7PkjB1bu2Xqfom0FfE33xAddhjPK9xxB+fIKy/nDWAvuIArb+LEeloqVYJHxBx/9Gi+3bHHcjHr1nGc+m9/y6LQvbu3cZ0K9pvBKZHgKXXhFX/lFRbmL7/kebuOHdnKKS72dTu3ennsMXZetGlD9NBDTPRWr+YwiF69uI/dcUfTTOWQCQKRlQULuOK6dWNL5vPP2R365JNExx3H7RfA5rJr1tStSzv/fO5D69axI/6yy/j8iSeyDswU+00fEti1i2eNAM4RWVjIyumDD7jycnOZBG7fbryFnzpJJjkNF0DUu3ddl/38c6L//IcJ+AEH7B/kjigieOcB+Lf0+TIAD3vdc58neEQswb17U72UGgCboTNm1Ls0XYJHxP32xz+uK0YcBxxA9NRTwW4G3RSRkSLesoXo9NN59AbqXtu3Z8asmKDpEDwiboNnntEvxr300uAJ+H41OCWTPMfdunX9NorFiMaNI/r0U9+38qqX5ct5Nktto169/Ids7msITFY+/ZSnE0SliXY6/HCixx8PTBHt3cvhYc2a1W+jWIzPBxXPtV/1IYHKSqIf/KBuLBKvrVoRXX+957Y5fuskmWTD6MgjG/al448PhoA3FQRB8BptFa3jOPMBdNF8dQcRTa+55i0AtxDRx5rfnw/gdCL6Uc3nywAcS0Q/1Vx7DQCxO2p/AJ8F8if2L3QAsL2xH6KJIaqThojqRI+oXhoiqhM9onppiKhOGqIvEbXK5AbZQT1JqiCiURneYj2AHtLnAwFsNJT1BIAnAMBxnI8p06XH+yGiemmIqE4aIqoTPaJ6aYioTvSI6qUhojppCMdxGji2UsW+nGlpEYA+juP0chwnF8AkAK838jNFiBAhQoQIESI0OpokwXMc52zHcdaDF1K84TjOnJrz3RzHmQUARBQH8BMAcwAsB/ASEX3eWM8cIUKECBEiRIjQVNBoU7RuIKLXALymOb8RwDjp8ywAs1K8/ROZPd1+i6heGiKqk4aI6kSPqF4aIqoTPaJ6aYioThoi4zr53m1VFiFChAgRIkSIsL+jSU7RRogQIUKECBEiREgfEcGLECFChAgRIkTYzxARvAgRIkSIECFChP0MEcGLECFChAgRIkTYzxARvAgRIkSIECFChP0MEcGLECFChAgRIkTYzxARvAgRIkSIECFChP0MEcGLECFChAgRIkTYzxARvAgRIkSIECFChP0MEcGLECFChAgRIkTYzxARvAgRIkSIECFChP0MEcGLECFChAgRIkTYzxARvAgRIkSIECFChP0MEcGLECFChAgRIkTYzxARvAgRIkSIECFChP0MTYrgOY7zpOM4Wx3H+Uw693vHcTY4jlNcc4yTvrvNcZzVjuOscBzn9MZ56ggRIkSIECFChKYFh4ga+xlq4TjOKQD2AHiaiPrXnPs9gD1E9Bfl2iMAPA/gWADdAMwHcBgRJUJ96AgRIkSIECFChCaGJuXBI6J3AOz0eflEAC8QUSURrQGwGkz2IkSIECFChAgRvtdoUgTPBT9xHOfTminctjXnugNYJ12zvuZchAgRIkSIECHC9xrZjf0APvAYgP8DQDWvfwXwAwCO5lrtfLPjONcAuAYAmjdvfkyfPn3gOLqfa27oMYWtfu/3s3xe997tOj+v4v3mzZsBAJ06dWrwnZ/fp1NmJv/BrU5015nOuZ1PFSZZUc+7fda9l8/F43E4joOcnBzP6x3H0Z7zek33u0zKdPvPbufU96ZzXv04rH6unnOTeb/nguqntvu5n995nVPf+/nsdT5V+O3n6jk3mU2nb4TR3/1cE8Szp/Pez2cVTamfy+8z6d9bt25FaWmpvz9mQJMneES0Rbx3HOdfAGbWfFwPoId06YEANhru8QSAJwCgV69edMsttyA7OxuxWKz2yMrKQiwWg+M4te91h/jecZx659RX9b36ueb/+O5I6mcvQiQ+T5o0CQDw7LPP1vuOiJBMJrXv5c/JZLLe+0SCQxzFebcjkUi4nhPvvc7F43EQUe059bPX79yeR5xPJBLa/637z6Js9Zz4nVzPps9qO5WWliKZTKJVq1Y6ETYiFosZZUaVN/WzkGP5vSq/qoync+j6ltzHRNlZWVkNzqf7Xn5uta+b+rXps1v9yH1Y7c/ie9EufgdBE0yDhfpeJ2Nq/1Zl3fSqvpf7iakvmb5X+5/X53T7r+453Pq3Tre56UdTfxevpj4un08HmfR1k9yaxjBTn1DHx1Q+u42rbuOv2+/S7dOm/55KH9eN337Hcj+48MIL0xGTemjyBM9xnK5EtKnm49kAxArb1wE85zjOA+BFFn0AfOR1v+bNm2PQoEEA0KDxasrTvhef/cCNmasDvnzOjWR5KSeTkt2zZw+ICJ988okv5WoiTSbl6/Y7VUn7IXxeA4tcDyrJUt+bSJa4Tm0v9b0bTB03XUVLRHAcB506ddIaEKkoVTcFKStblVSZyJfX73QKOh2lLA8gfo0mtU5TNZpM8EumdDKmI/1exMmNNLn1lXQMJvWcl3Gk/lZHnryeXf2vuv7sh0jJda8SJlXfesGk21ViLp9zM47k925Ew+/h1afd+qncV4WB4+d6P8RKR8K8iJQbeVLrN6yx2GQcqX3br+Fvusarr5jGztLSUl//0Q1NiuA5jvM8gFMBdHAcZz2A3wE41XGcQQAIwFoAPwYAIvrccZyXAHwBIA7gBvKxgjYej2Pbtm1aQUyFvQvolIGbIOq+y8rKciWD4rNJ6NyIT3Y2N3H79u21g4hfa1s3OKgDjG4AMA0sut/6IX3qwOA2WMrnRH14kT9dnYvfZoJYrL71Ld8zFoshmUzCcRzE4/Ha917yqFOifhSuF1n08nilQi79Eju355VJn4n4uQ0Yot79Ej+vwcNE/tRX0wCi+15+RllWhGwkk0lkZWVp+6xMmFIhjJkeunJNHn4TwXUjdepvdPUmn5dfdf01HeInILeJTv/rCKFu/PAz1sjPK/+PZDJZe99kMlkb1iEbV7r+BaBBv/SrK1Ihbl6ETpZt09gq17GJ6OnaKVXCJ7839V35vWn81V1jkmF1bNYZf6pOyBRNiuAR0UWa0/9xuf5uAHenWAaqq6u1gqsKKhEZiZ4QVNHpMh04vARPR0BUoRDfZ2dn137Xs2dPEBFyc3NTUvxiMJEJWyKRqFUuMimRlbuoG/He9L/FvcT/8KontdP5GXzUz26DiW2i5zVAiDKqqqp8DQZ+lLV8iPtnZWXVKhmgzrgwKRW1TJO3z6+HQQxIqRJBE4l1G0jkwVb+L02hj5oIj1cfFf1Nfg71ucTv0/Xe6b5TjTKT1yFVgthU+qiQESEDbiTOD0kzhfGYDvkZZLk09SHZCPPy5qXjhfcT1mDqp+r/b8p91ETcdH3V7b3pELpW7ju6fmN6nymaFMELA7m5uejZs2daVoIKkzC5WQByg7spey8h8issZ511FpLJJJYvX45kMlnrIXJT5OI68bym6Vjd1I3pWd08cfLgAejj2vwqefGaaufQKVcB2XskrtVZom6KTlXWus9ucaGqQs/k1e297rP6v9y8ATqL3mQkmZS5rj28IPdFVSaE/IrPOrky9UUdCZHl102xm86lMr3qda1u4EiXZOn+s6hPU7/z6ocyTH1SJVdqu6v9SPzGrU/qjAJdHzX1Rd1nU7/w4wHX/TaI2DLdezdCpas7L1KVTh9UZUDV+eI7tQ+mQrJM42U6caJ++1Aq8Z5u47kfT97u3bt91bsbvncEb8+ePfjggw9qO6xuEPQTn2A6ANQbmE1Wi85KBFLrVLpOJH/nZyAzTal4DVpu5FD3XhBFtaOoh7jGdA/dOd31pleVUOpe1Y4p149atzpCL7/K7aFrU5OBoSNAbrE+8jndqx/rX+0Duj4h/zY7O7vetfI1unvp7unW79y8g14Dn58BT9cH0x3U1D5okhGdMpcHCN2gYFpsoBpV8nm3vubW//xcZzL+/BqCXjpG1xdNukwl7qn2QR2ZMfVDkyylQs7cYuR0/UvX10z9yqufmcY8NUZP1yfl8zojT/1e7YdeRp7of7o2SaUPmsbAVIij19iXyRhoOqeOe4lEAsXFxb7+vxu+dwSvTZs2GDt2LAB3QTJZIzphchMcWenpBChV0uSleNXvXnnlFRARzjzzTN+kyq9C1z2rF4EC3D10qRInHfySJjdFLWIXTdMZJgPAD4nRkaG1a9ciFouhb9++vhS3Sq5M93d7NSlqWTHLRNKvJ05V1KY+pmtXvwpa9yr6WyoGiizPOoLkhxCZ+k+qxoofQ0Xue/JhIkWiz6nvVZ2WSv/y08eEHMheI9UQUfubFxnSya8X8fHqd17Ex8sQSdcYUfuTySgx9THVOPEDnS71S4B0YxcRe8fVsU2WT9144fbeT58Jul/pjCivw8v4kHWZW/+SP6vtuWvXrpTaV4fvHcErKSnBrFmztJ3YT6f102Hlz0JxBWm5pOI1eO2110BEuOSSS9K2WNwGHK8O5kUe/ZJKP8rBy+Jy66wy+YzH4/XqU1e/unYwQSU6OiW9d+9eAMCnn36qJZ2Au4dOJmt+iKefwdGPt8BrUPXqU34HRN0gKOpE/k9eg6Kf/qVrWzcDRByyDPntZ6a+pvYz00CpElSZuLr1UdNn02Aon9eRTNWYU/+zyfg19TOdfvOCSY/K/Un9bDLwxDkvQ8jU51LxzKnvvYxEt/6s60fqd+oh9yO1n8l1JJeZSv+SP5sIjsnoUHWvSZb89LNU+5wXMdR5yXXjkXyd39XpiUQCCxcu9CX3bvjeEbyWLVvipJNO8i2gqQin+lkWxnQEMhXBMxGt7du3AwCmTZvmSrhUIfQzOGRCrtQBUe7I4rzOGlLbQW0TN6+RH0UvK0LAnVQFSaaWL1+OWCyGgQMHuhIq2TBR7yOXo3s23bPqBi+TkvfrWfBqB137mRS8SakLGauqqsqoD3n1H7e+4nZdKkaJiTiZ+pFMpOS+YnovXsPqQ6kYJm59SEeSdP3JrzHixzAxES3dZ5NhYiKL8ndNoQ+ZxiCdF9zPWOSnP+nGGz/XePUnmTz5GY90xohu7JHHJbc+pH52I7lufUjuL9XV1cgU3zuCV1lZia+++ko70Jm8BjoFZmoc9b1Qcir8dFBZuLw6qEqWxNGiRQsQEfr161evM5s6SapEUlUIOmvFNLCqg5t8zu1zJp3SVPfy9fI9hZdBwG/nlN+bFL98bNmyBY7joFmzZvUGOBMx87LeTQOT273U934P3X809RnTgOY2mLlBLtttgDPJiZvB5WaA6QiZTp5Nsm4icm6DUyr9JJX+4ncgk/uK2qfUPqK2pc5QlgdzVSZkj6wqV376UyqHydjx0xfcjCM/fcRE+nT9Ru4rbmQwVej6jfyeiBo8azJZl61ByIMbedT1JfWcKs+mPud2zu33cp+U5dzUT9T36v9yqy9Zzt36ihu++eabtNtU4HtH8GKxGJo3b16rhHSWl66Tid+mMzC5kTmdsOiEzEt4VVIlBFl8Li0t9bSw1M/yuXQ8EjoC6TVoyf9VJa1eg1EqA5E6CPm1sISsCNkxGQZuUzc6L8KuXbsQi8XQoUOHBqQslWBr9ZxK5LwIXrpEzlRfqQxEOrJtame5j4h21pEdHdFy+6zKrlcf0U2RevUlk8GjXmMyyOQ+4XdwkvtCKgOQkHVd+6n9xGTcqOf8eMFMHnDTe7/9QfXUqdfIxpKpf+ieX/wvrz4i6lOul3T7iYmUye2skwedjtXpYDfD3y12VdcfUu1LfuK+df1Y7f9eXjqTc8BWP3F7L8uN4zj19GG6+N4RvKysLBxwwAG+LCX5VX2vVr6JyZs6XKoWjJv3y21A6tChA4gIGzZs0HY6mwOSF3Fz62A6heUFWYGq7ZdqJ9NZ9F7ELZ2YG3Ef8YyHHnqodnDSDXq6KSPTIKoOPLrPbodbX1D7gJd1qzt0RMVvH3Ejazojw02OU/U66wYY2cjyY7zIdSauUetPN/Co72U5N7WTl9GqIyeq/KjnvbxesofMLd2P6Vq/99c9j+7ZgfS2nvNTv2obqu2kIxFe/cJt3PBj9LuNJ179yGRopHLonlf3X+T60Y2fOp0i17kKIUOi3dQ2czPs1SMVYuY2K5jK8dVXXzX4T6nie0fwKioq8MUXX7haem5KST2AhspCbXDAX549neL22/lNnX3o0KHaAcpECN0O1YsXj8e15/z+Xve9aXBVLTKThSaulevORDh09a5rG50ikEmZTg50h0q+dLL3wQcfGFf4yR498VnIpnrO9Huv2CX5fSpTuCYC6UYW3QZKtX3c+oNpwJA/p0IC/fYHN5lP5ZypfN17t8FTJo9q3ZiMp0z6gqrjdKRJlnn51UT03DxvOvk3ybWpD3il9vHqA7o+bCK9XqQxiDEBMG9xqcqBiby5jQeqbjf1Dz/y7kf+TWXr5F/NUGEaE7wcCen2A/nVDwF0O3R6uKysDJnie0fwWrVqhVNPPRWA2R3uNriYOpMfa8iNVLktftARKnHe69X0nY6oiY5gShFh8li4DSqqMpLfm6AOJjqFqJIp+Vx+fr7Ws+U2eHgNHLK3LScnp8Fg4XadXIbunO5Z1M5uOnSeFq8BxA3pECnR/uqCh1RWfqYzSJjem+7ll1Cp8i3iMHVybyJRJlkXn90GDSC1AcMtxYiJxHsZEaqcqgTIr2ynS5y8ZN0k76b6NMm6lyFt8jzJsuFmQJv0vd9DlXFxL5Mu90OSdM9qOmRyJM7J9aXKu/yqwkRi3WTdzVDQLSjTvXoZtNnZ2Q3k1K1/uPUbv9P+skzrzsViMVxyySWu8usH3zuCV1JSgtmzZ/u25kwKyU0p5eTkNBBUwBx3YYLO0hYdSjfAqISSiHDrrbeCiPCHP/zBk2R6KRc3wqi+ug241dXVnp48VfmYVkjJSqi6utqVYJoUEBFplY/O4yS3peqxUpWQW0Jh9di0aROysrLQs2dPawTTTd51hFJHMMV/k8mFH0+FDrFYrN4AIbeHl1GlGhYmAyudwTYdUpkuwdQ9p86okl9VwinqSQzEujpUdYoq56q8yzKuvtcNtjq5dyOZJn1r8kjrzqny7vYbU9k6gmkacOXzqRJMN6+ciWTqDCmdrMv60a/M+5lN8UM2TQQ0SJJp0u2y7Mu6wg+ELJs80l5Gle5wk3eT3ldlV5bXKA9eGmjdujUKCgq0A49odAFVWPwOOiYhTWXAMQ0eghz59eAVFxeDiPDPf/5T690zdU7ZUjQRKz8eDZ3nTmfhmQYbk9dOJVX5+fkNFLQ84Hh1LHVQUDuc+E4lV6ZXN1KlO4TRMX78eFcrz6/HLh2Z1ilQP+QpFQIVlNFg8kq7DTim/peqTMv1ZSJPfmXaL3Fyk2l18PAr0+IwybTXObdDNgDUATBsmXbzPrsRinSNAr+62STT8ndy2fI5IeupyLRsJKjOgsaQadUIzs3NNZIhP7Mrqox66exU5dpkrKsyLZPCTGR65syZyBTfO4JXVlaG4uLiBkKkU0Lq4aWIRCOJwE43D52fqQE/A606Baa+FhYWgogwatSolKw1r+/SmRrQEQSd0nGbFhCv4py4ThBRN2+RaBOVMMrvTR45VT7cPHSmVa9uBPPLL79ELBZDq1atGigP9R4mazEVGfZSPjqPhKhHWRn59UboZNprsFXl28tAUuXR5E1IZ0W4zsiRv1P/i8nzIteNTq5lyN/L6XrUdpHbUP1sanOVZMmfdfKuI5gmEicOmYiqcmkqR723eoi+6qWPZRlV60eVaZO+EM8iy7WODJl0tZuHWZVxVUcKOXWTf5OxpfPm6eRVLV98Fv9Tp4t1fVqtD7meUoEoU6d73NpZJwsmmfHSjzo5TfV7r8PtueX/Vl5ennIdqmhSBM9xnCcBnAlgKxH1rzn3ZwDjAVQB+ArAVUT0neM4BwNYDmBFzc8/JKJrvcrIyclB586dXSsYqD/4mwY7FbJwy0pbdBAvS9IPYUuFZMXj8dpAzZUrV2Yc9Cq+NykdL8tR1IFOMYh6M0EmaPKrOpjJHVo+p5I1E1HSkS8TOfOaAk0ljmPnzp1wHAdHHXWU0TqUp0NVhaEO6DqZTcfgkK18lZy5DVri1RRv52U06Dxz6cQg6cicPAiaArV1xob8Pl3ZldtA1jG69szJyWng7dB59HTy6vZZndZUpzNN055uh0rm/BoaMhnTGRtecqu2g18jWbyXyZCXAaHKtZvMpfLZj/41jQc62dWRNDeDIlXZ1RkQOr1kIkaqvOiMV5P+9NKxqSxQMxnMuvHCD1+Q6ykdnSu/TyaTyM3NdZV7P2hSBA/AFAAPA3haOjcPwG1EFHcc534AtwH4dc13XxHRoFQLMTWY+h5wj5Xz8l6oFo/Ja2Gy0LyUjZuyEOdF5y0vL8+YzJme06RU1DoX34t6Vq9RoXqOVCVjInNy27p1YJXY6Yiel2Lw49Y3lSV/JwZvEb+pDvbivwr5kuXIL5FTFbvOyybLptyeqtyqpN6vDOs+u5FAr77g51l0xFSci8U4DjArK6uBB9jkWTPJqh95lQcE1Zo3DYp+PWum927nZI+ZOpiZCJqbZ1D8P1lvpErYUpFXE4nz6zXzI8OqUWAybnXXmsqUP+vGBvGfZONCV8epesx0hrKOVOvGRj/eMZUUqZ9NMiX3B79Ggq7/qDLpNr6L/yrqPhaLobq62iincr27yancFm7yamp/8VlsX5kJmhTBI6J3ajxz8rm50scPAZyXSRlVVVXYsGGDq+XgJlw6AfLynCj/p8GrrsFVwuSmeNy8IgMHDkQymUTfvn0beD7kVzm2z088iR9LVPWQmBSum6Upf/aCblBNxbqUvSR+U42kEwei+734fteuXZ6EUTdI65SuTi79kEEvAuhmrLjJppeB4jfWNNVFPTqjRz4PwNUb4ubRE69+ZFQeXGWip743veoWDLgZGTrvhiynbnGkqsya7iv+jxuJ9KtD/XrvwtahJjnNRIfGYjEt6dtfdKibwRy0DpVlJZX4UPVzU9Oh+fn5vtrMDU2K4PnADwC8KH3u5TjOEgC7AfyGiDx3523RogWGDBnSwJJRobJxN9bt13thGoTi8XgD5aAOXvIhBwCr36kDnlDCL7/8spGQyc+oKhLdIKfWh1xfKnTeUD9KQ/Uu6AYyXTC5TgmI97rPbgOeGyHz8piYrE2TMTB8+HCtPLoNZm4eC5GyROftSncAM8mlH0ImfyfO6/qJThZ1g7WoC3FelUH5vU4Ru1n48pGXlxf64GW6Xr2n+gx+iZXsEfZrmOrkUdUJJm9UdXW1Ni7Mjfy7EXY3Y1NnJKi60qSfde+9yL5OJ4q6MEFH9HUyCNQn/bp4RpMsmoi/1znTAh2/pMltUY1JFlMh925y6DZGy5/lhSrqZ7n9TTNcXrNmqo7T6T3d/VXdt3HjRuP/94t9huA5jnMHgDiAZ2tObQJwEBHtcBznGADTHMc5koh2a357DYBrAKBdu3aYN2+e7wHeJMyy8MrTaukoz1SUpskK9TNou5FG0yHIpNurarXqlK1skarv1U5mcme7kUmVRKqHl+WpI2uyTPghj6YjK4s9dLrPJgKajhyqwenyfxfyqNaVSQ79GDaZyKKOJMrypsqfmyHjRTZ1sqgqVnHI6RhUedTJolxvcn3q5FHnGZDJo5sc6gZkVX5kGTW9usmoSR7dZFFHNnJzc42k0u8ArsqliVjJMuhGIN1k0mSU+NGZbte7yaJ4PlUPJhKcT9KPcePXwFblUOgCHYnUyaFq2KjyYjJQ/BrVpnHY77gs6z71s/g/XsTSSx7VutaNU6ocehkzqsGhys7y5ct99Q83OCbBaCw4PEU7k2oWWdScuwLAtQAKiEib3tlxnLcA3EJEH7vdf+DAgfTmm2/Wa1gVKuHyIlsmFm4a0NyUiRjUdAQrFWIm7v3JJ58AAPr161evXDfLQVYoOqIl15HSBrWvOoKlI1peg5qus/s9ZFLldd6kTNyUisnNL/9fQL+HsSpv06dPBxFhwoQJ9ereJG8m5aFajTrZM5Eok4xVVVX5Jvm6wU1H8L1kTpY7Vd5UQmXyEOsGMLWt5PZVSb38nRdxkl9NcueXXLmRetWrrfYrVeZ08qbTc351nKrvvAiTF0FSZc6PvpNlT1deJjKXiZ4z6TidAeaXtKer7/wYleqrrj94yZzOoaHWk5u8qTpOJel+DUeTo8GPzJm+N5F3WfZUuVOdFbL8pSpz27dvR3V1tT8ryIAm78FzHGcMeFHFcJncOY7TEcBOIko4jnMIgD4Avva6X2lpKd57772ULQJZsFULPDs7u0GnF+8B/eDkx6XsNZj7GdS//vprEBHOP//8eh1AvkY35WsipDpviO7ZZKWq84aI99XV1bWd0w9UsiS3hTivU0penhFZwZmUn+oFcfPq6ZS3mwdkzZo1iMViWLt2bYOBQWd1iv+ene2eksdLqeqUq1BKqXjn3DxmbjLkJVdqGbK8qc+qypmsTE0DSjwe19abSd5Uz6jqLTV5tXSGjB9ZTFWedPeUyzQRXd3/Uj2POg9QTk6Oq8yZDqEjTDrPJIMmfaPKh4kImGTJdG+VbJgOtR+p/U1+lfWwKmtyXzbJnNpG6tikjls6fShkQ/bOmXSTjpC6Gedq2V7jp8mrJus0VQaFvs3Ly6s3vsrjiJvnV+el1xkyJh1okh8/cuUmX/L4OWfOHFfd5AdNiuA5jvM8gFMBdHAcZz2A34FXzeYBmFfT4CIdyikA/uA4ThxAAsC1RLTTq4z8/Hz07t1b66I2CZf8XtdZVWWlmz7QWb7iszqweXldvCwLmbCVlJSAiPDee+8ZB1FZ4emsXJ1CVpWYWj+6ulMHS0GMTRavSeHoBkM/06de57yImpsC1A2YqchWu3btQETo1KmT1oMsv+oGHBMBk9tWZwCosqR+1sWGmu7jtkOJmydF7S/qf09HtnTERSX8XoObadopHdlSZVb1lIgwD5N8mUi/jvjrPHeqkemXcMn6xA+B0nlz3bwqbud099PpMB3xdyP8fnSXajDpCLDcv3XkWbSpTsZMHjtZ7tym1cV93LxyOoPAjYyZCJeO4Lt553SyBTRc7KKSebfxya/eSmWc1BF/E/lX9Zeb0yJT2RLvKyoqkCmaFMEjoos0p/9juHYqgKmplpFMJrFnzx6thSMLOOA/hk4daJLJZO39k0lefi2EIysry5X0uSlPL0HXvc/O5kSdrVu3NipIlQzoLGi1k/pRmCa4eUVUhanzgiaTdakXxDOI2Cn5WdT/lp2dXftqUrRuXhK/lqr6X3RKUpUnkdRy9+66EFKT90NndboRPpOMqaRKVTJZWbxAJ5FI1PsuKysLiUQC2dnZDZSfmwzpBlw3K1uGSu7UfqeTK7X+3Q4dYVffm+RA1Rvi+R3HqVcf4vp4PG4sV2cg+PFyuOknL3Ln5dF1k690PRtyvYg+LMucTJZMz6A+p0oo5P8mf9bVjc7oUutXR3z8ypeXvJk8bSZZE/Ih12UikXA1ArzkSNef3KAzvtzkSjVS3Ty4fg9VDlVPnFym+E+5ubmuBE0nP6ocmcY6InOyfVG+KlNq/Ys227Rpk2v9+0GTInhhIJFIoLS01OhyVjugOlgD/pIZ6hSLH6WpG5Blz4iJyJniqrKysmoJnpfVLJcjBvF4PF77X+WOIuCX8Ll5XwD9vpemQ54+1xEyk/cklXgWMSALgiw/pzwouHldVKUMmD16Yqu1Nm3aNKgzVbG4KUbTwCpbp26WsDgnyHB1dXU9pSQ/h/xsfrwupufTET+/cqTKktxPVcNNN5CKNjJ5WUyyI55VvJcJvCjDj7fYVLYf4udF9nRy5Ne74mYY6ORIfS+TW3F/1VOiO3SeGV3Zfrwq6eojk6GQqRzJ7S5kRzam5GvU+DmTLAUpR6oRqiJVOfIyMnVeWZ0cyAZjqrKjkyPx2Y8ciUP+b266SJalVORIbafs7GxUV1dr758KvncELz8/H3379nW1XATcCJqwDlQLwq8HTnfoAo7dPrsJuCg7FmMv4vvvv28cVE1KUX5VoQ4mJuE1CbGbFy0VIuZ1qIozFmu4A4UbKdNZ624yI8uLOrCongtZZpo1a4ZkMom1a9calV86qXTcfmMifiZjQ7a8VYWejszIA43YtUE3eMqDqO7QTX96TavqvjPJpZesiAFa5ykR/zdVmVE90apHwmvwVGXIpIdMhEp3jY60y55b3SApntnkofUjMyr5UsmJSGUje/3UdpLPu02ZqoRJnTY1yYXu3rIcq1441bsm/lsqOkaVG7V+q6urUVlZ6erxEp9l4uwlR26ypSPjqt4TsqI+j3xO6BYvWZGNK53MqHUq6lxMn+vaxk1+/MiSG9l2k091zFyzZo2x/f3ie0fwSktLsXDhwnoWkslaUju0PNDodhxQGbsfK8jk1dMpbj9EUZBA3auf915kUiUFqrIX+dfUTqtzc8swTYXInVCnwHUK2XSIdhbKOjc3t/az/J0qG/J3fsik3JFVC1+VF4CV0KGHHtpAblLx4PkZ6OW2FQOAqd3Fa1VVVb3vvGRFXjSjkxv5GeV0EG7yIsuNX5lxMzBMxoVJZtxkxe2cTr/4JZUq0dUZHGod6HSMbFxkIjOp6hqd7JhkRRgiqkGikxmZSMqvJj3jV2ZUHaMaFbLcqOODKiM6fSPOyXLk9urHWNXpGZ3sq4RHNzaJ+pLJlElmTHKTiszIK/P9yJJJVvzIjMlINekYWXZ0s3YmPeMmNyZD001XlJaWamU4FXzvCF6bNm0wZswYV6+dahXpyJdKtlTvmc77plNu8qGe96sg5cFUfg6ddS3/J68BFNCnAlBJl6xYmjVr1kCQZQtYVoKqUlMHUR3xMilEoWjdlKHobKqnTiVbfqYoVLe9SQlWV1fXbhOnI1qmAVNWgCb5cCNmfsi5zqPoRrK85EM3WMqyIdpCeFxkAmVSdl5E3C/JMsmHybo2DZLqQClDnb5JZYCsrKxs0E5u8uFmqHnpjVQMN93gqP4/uW+osiFedUawzliTB8S8vDy0aNHCl8GmGmSy7OjkQsiDm85Q5UPn4dfJh9eMkM5IE3Wr88zK7VNZWYm9e/caiZObnHgZcjrypNNbqnzoDvF/Vd1BRPXkQu1TqRhnog3z8/N9GWZehNqU4shrFyLVWyt0nknu1f5hmvqeN2+eVo5SwfeO4O3evRuFhYVGpaESElPn1jWim0sd8E4aKw+2fr12Xh37ySefBBHhggsuMCp8XefWkUY3xV9VVQXAe8pON02nkke/FpGb58XNMvLysqRCFk2KPzubpxabNWvmy6s7bdo0AMCECRPqyYdMyt08Ln4taC9y6ecwTdGoUzCqxSy8u3JfEDIjIHsYdDKiell0/VM3nab2a7dXVR5M8mbyvnlNvYj/I8pR/7dOPuR6cjM+Vf2hto+bTtGlS/KSA533xCSbRGSUDVnGZV3iBlXvqt5O3ZSbzvuvkxWTgapb2Sr/3ksuVFnVyYc6cyF0iUmHmAiCaqypnltZHnTOC1WvmFZIu8mUeg/doeo2+VmEHMj/IR6Pa/+7WjfyeyEbOjlJZ2pfyIWuzb1kwGRYquXvd6tow0CLFi1w9NFH+7a8TLEOsqIyKVW1c+iUqYmYeVlZXkpXlL9582YQEebOnVvvWUUn0VlcOsUq6gpAvc4BoNba1pEytSO4kbF0LC4dGVPTW7h1It30l5cVrvPwyha4XP9+ZWH58uVIJBJo0aJFYLIgW9yy4pXlVjybKgvqqx9ZkJWlLAvNmjWzKgtuXtywZUEl43Idizbzoxf8eGTk940pC0IO/MiCaAehL9xSgTSmLOiM93RkQdSx2xihGth+ZcFrGtMPQdcZaCZZMBllqizIr35kQSbR6jT3/iILok69ZEHWC6ZZk1TwvSN4VVVV2Lx5s9Gqc1P0QJ0iy8ri5fziVTSm2incpme8rB+T10VHDk3HypUrQUTo0aNHA0tePdxiWtTpJwFRLzJJdBxecWuarlPrXGcZeSl+3SHuIXdsnTXl1ZFlC0/t0H47tWxRqh6n7OzsBhasWGTRtm3bWkvZzUrWeVl10yryPUweWJ2smtrfTenoPG6ij6j1IGRGHgDFf6iqqkJWVhYqKip8W826Nta1uc6zJreN29SKmxfWTQ78eNxE/YrBUbwX/yMejyMvL8/oBTEdbu2uGqgqQXWTa7cpe1nu1bpTvSemKTnxXpQp762clZWFyspK17bVeWNMHjP1mdR2lw0auY38TMWa6lD1qqneetVIENeo8moyJlRvuumeuudQ+73XGCA+q/Wh1plOJtS+pr43HQDqybCQCR2h1B1++rvc7qpRa/IU6uRAlgHxXicbav1XV0eraDOGicCp35kGd8dx6jVkVhanF3H7ndrYJravDtAqoVPjKnSETwiLHOMjW/E6hS8TVTevHmAezOVX0flVYqgqZFP8QyrWmxv5cxvwdcolVVKnI/dCFkQ7iN/J9S6TtIqKCm0769rby3LXeRPdBnz52TOx1tXBM5lM1hIzIQty25tiovx6cU3trxoKurbXKX7xfEJOde0ut6PJe6cz6OQ6Vr01JmPOq+3dPDSqp0bnrZEJRqp9Xn6vGnKizUVdi+loL0+uWzylKazCq8+L8nQkT372dPq8St5V401noAnvnkmvx+MNvfQ6r73a/up4IcuZSde7ETlTuwPmPZV1fU1tE51nTW1Pv3JganP5OXQkX6fr3fq8l8Gm9nvVI6cztr36e6b43hG83NxcdOnSxdVDp2Pebo2qdmS/Xjg3l7suQFbt3OpArnbmZDJZO4+/bt06rTdGZ4mrnVgMdn4UuB+vm46s6cibH2+d/F59NtOg7QaheFXL1tSBTd4yr0Ntx6+++gqJBKe10SlrcX+5neXn0ZFz2TMk/pvazuI1Ly/PlaTLA6Tc1qpCVT2p6qvXYKwSM1lBuw3Qbv1ZbV9RN+rgbOrfcn/StbfO06obbE1eVvlVRxC8vC5qX1YHaJmsynVs8pboPJ0mkmwaVN36qV/PquqFSdWjKpOweJwXKHh5Uk193OQdTeWzifjrZFEm26rBJfdjYTzJfVocubm5rl4yNy+nrn1M7eanPd08qOrzCZn1Q7p0si+3s6lPq95rXfu4yYLpNyqJVss0eUp17VtWVtbgP6eK7x3B27t3L5YsWaK1DlSrQie0ubm5DYQSqK9ETS56N0WiGxBUT53XUVVV1eBzLBZDIpHAYYcd5uoJMA00urQnOoE0eT3VgUO28EwWnW6lpLpqMjc3t3aFnIjdcLtetQhNnj0TgVDbWW5rr3b2MgCqq6vRqVMnJBIJ9O7d23fbitWSJnlQvXtuVr6OQKjtLLe1Vzuryt/Nate1t9qepvZVV9jKhoJbO5sGGl07C8jtrA4eJgKgi6vy06fl9lbfq/cxxXCp8mZqZ/l/qXCbSlPbWW1r1TMj911dO/ppbzdPjl9PrdrOqlGva2cdGTB5aEzGu6kvm/q37rdyeTpvkGoMqF46WU+p04xqO8sGntrOalvr+rBoaz9tLr83ee90ZauEVCWNprFZ186m8dlrbJb7oalv69rXpK/lds0U3zuC17p1axQUFBgte5Vlq0SssrKyXqML5erVsLpX9ZzfhtcRMlUJqQP0hg0btERM7gyq5ZWTk1Ob9kTnRk+l0wpClp3NCzJMyl4uQ+cJMLnZgYaxMSrR9hqU5bZ1866aOqtMulQCJu6ttqvOG6t6cnTTKfL/E9ARLtVSl72tctvqpsL9DsB+B2OZ4MmDsZfn1WvKxK2/ijqVU9Wo/cmPQhZtaLpe10fVgcGrTVULXvVeqPWjetHlQVhOL6LzkOv6n9tgrLanStTVvmqaEnObBvU7W6K2bVVVFcrKyrRtmgrB0vVP2UASrypplo04+ZD/kwqVZPoxhEWd5+fno1WrVq5t6mYgy4dp5kQmUOI6tS1N7eo2rvoxgNV+s2fPHk8DSda7aju6TW2r5clta5oVkV9VqDNGboavziASeln0x0zxvSN4JSUlmDt3rqunR2ctiIYQq790UwYqVOVtUlaygMnKRCfMOjKoI43V1dU4evlynPvJJ2i3dy+2N2uG//brh8LOnY2DkWzly0lovaYJ3Lw56iCkGwy8Bh4/5FE3CJnaVLXwc3JytO0q/psKuS78DEI6q1tu0x4LF2LwK6+g5c6dKG3bFm+ffjqWHXWU64Ckkw3Vk6Ba9WIwNBFGuV3l/y3qBYDWK+LmvfHrwdH1RTWHmUz8vTw34rnkXQ5kj3um5MJk0bsNQmIAkl9117iRxerq+lP0YiAS01JiINK1rYxUBiIdyVA9pWp/07Wdm/fNy6iTCYZuFsWvISD6r+qlMXlr3IwCtf1UgqHz5pkMAFW2xPNUVlaioqJCq4PVttV5J910cSp919TOJkPOjfyb2lWEiej6qwlqvZhmyEzjranvmtpOp8NlvSv6p0oW1WcSberWrnv37nX9737g6KyL/RlHHXUUTZ8+3agY1Hl9VdGrVptJSLy8An7emyxJnXJQvQNnlJTg3h070Fxq3zLHwa/btsXrLVsCaBggr1qOqhLQdWx1QFa9dDIB8xrYVY+DziNgcsPL7ai2oWl6XO2EJutfKG6/RFvXjqrCEGUWbNmCO9auRTOpncodB7d37IjXW7Y0EmuTJ9Yv+dINvn6JtKkddYO3n2kzU0yVm/WvDoSmwThVI8n0XtxD9RAIfeDH+tdNeav/XZ4WU/uiaDvhVZGJlt929ON51Q3scjsKuVLbUEegdfrUzYuu9g8vr436XnhZdSEM6mcvb6toV9NA7NYXvTw3JiLl17D1akc/oQqyR85PX5QJiKgPE5Hya/S49UFTaIKXQSSP03Jbyp5VdRZEbUO1L8r15ddJ4dZ26nipvs/JycGvfvUrrFq1yp3heqBJETzHcZ4EcCaArUTUv+ZcOwAvAjgYwFoAFxDRrprvbgPwQwAJADcS0RyvMvq3b0+f7dxp5fkjRIgQIUKECBEyRbeuXbFx48aMCJ73ssJwMQXAGOXcrQAKiagPgMKaz3Ac5wgAkwAcWfObRx3HyfIqIObh8t2f8CwuwsFYgxgSOBhr8CwuauxHiqBB1E4RIkSIEEFGu3btMr5HkyJ4RPQOANW9NhHAUzXvnwJwlnT+BSKqJKI1AFYDONarDK85/f0Fz+IiXIN/4RscDEIM3+BgXIN/ReShiSFqpwgRIkSIoGL37t0Z32NfWGTRmYg2AQARbXIcp1PN+e4APpSuW19zzhXfxjvBwbbgn7KJoTn2oAwt6p0rQwtcgydwKZ5rpKeKoCJqpwgRIkSIoOLI1v0zvkeT8uClCJ0rThtQ6DjONY7jfOw4zsci4Hl/RxlapnQ+QuMgaqcIESJEiKBizZo1Gd9jXyB4WxzH6QoANa9ba86vB9BDuu5AABt1NyCiJ4hoCBEN0SUwjRAhQoQIESJEaCpo27ZtxvfYF6ZoXwdwBYD7al6nS+efcxznAQDdAPQB8JHXzQ46KBuzZq1vkGPHlCfJLY2Gupy7srKy9rN4b3rVHbrl37r0C2oyYxnif5WVnY1du+4DUfPa7xynHD16/AGdOs3XJq5Vj7y8PF+vbukz1GSZujQnujQ1pjxkbu0gt4da15WVldo2kM/p0i/oUinI6QHUJLVcz/XzAsr/WU1bsGvXOKxZcxuSyfza32dlVeDYY/+Nvn0/8d0W6qFrB1O+OFPOKTWthZxeRs3tp+sTurZw6w/ye3EveQ9l9dC1gyxDcuoDWea80sbI9SfXbyr9QdcnTOlj5BRF3P5Zru2g001u6WDcZN6PblLvqWsDOdGvWzoYNbG62ha6dvCrm0x9Qe0Tujxtqm6SdamuHeQ+oct3qUvRkkl/0CXZFulg1DQ9ftpB7hOmNC6qHhGpr/y0ga4t5N+K+6spXPyMEQDqJR7W6Sb50KVfEfperueKiop647j4rPuNLl2Lmq7Jbax2y1Uo6uOrrzLP9tHU0qQ8D+BUAB0AbAHwOwDTALwE4CAA3wI4n4h21lx/B4AfAIgDuImIZnuV0bt3b3rwwQddla4pP1BNmQDqExEvxSu/N+VwUhWqLEy6PEGmPEDyM+zYcTrWr78eRAciO3sTOnR4AG3avKHWOQB9zh/TQCgrR3lXClkByJ1Zl6/JT840eeDzk1DaLTeTXE+69nDLsaUjffJ3cqJLNbmlfMg50WTs3DkGGzbcgESiG3Jzt+DAAx9Fp07ztRntZbLsVpemHGdy3jRdPi5TIlKvZMFqnizxP/3myXLLXyeUtC7nlY6A65LGyv1T7r+q/lPzYLklitUlcDXlIJPbTP6N+tkrZ6ApX5mpP6jkUM7R55YA1q2N1PqWZV9tA7k/qn1BNox045Bu5ws1T6BOX+n6iq6OverdlPNPbQM/CbRFvzDlV1VztunaRffZ9F6tf10b6Iign34h/19ZP8uEUX11q1vT9+o4oBuXTQmvTe2h6x9qPxH5K1XZ1dWpaazRtanc9/y0xbx587Bz5879Jw9eGOjXrx8988wztcIje1h0ClMlDqpy1HmU5ESbbl4NHbFzS3RsSsQJNNxYHuCOuG7dOgBAr169tFaClwdDvHfzFKnETiUZOgIhd1bTYOVW9zqyZrLWdB5W1bvhlgxVLkeXyFb2Hsl178dSFvW1efNmZGVloXfv3rVk2c2DYbKO3bwUssdIlyxarXs/XlQdUTZ5U9X6Vq9T75NI1O2F7OahcKt7t4TBuiTB6da5ru5lwieTBNVoAer2ydR5sHWDu87Q0HkbTB4InYfORKJNXjqdsSJ7wNyMRZ1BovP0eOkek7daZ7CkoutlXWPyWsu6Qk2wHLaul+ve5BHS1b1Ohr3qW+cRddP1qpEoH7pE2DpdL9e7qCOdrnfTO6aZGpPRqJK3oHS9KquqZ1Qc//nPf7Bhw4aMCN6+MEUbOJLJJGKxGJLJJLKysmobzTTQqZaNmxUmM3SdFWBi7Oo0n4DqUhfTN+L34n+IZxO/ERD/qVmzZg0GOtXjYPL+uA1msqdCJQ/yc6pWkcn1LtexOsjJA72b58evV04lckKZinoU/yMnJ6e2o5o8QKqFK/5/dnZ2Ay+c6hXNzs5GRUUFYrEYunTpolUGOu+pzjshlKbsQauurk5ppwFVjk1eHlMbuL3XedvEZ1F34n8lk0nk5eU1mN6Q+6raP9QpcdUboPPeyP1A59lUf6fzMAColZ/q6ura9lT7oyzjcr/V6Qadl83N2ybaWiXEKkmT25WIaus8Ozu73jO56SNZznV1b/LemLw2budNXkxR54kEb+ll8tzodLcq6zpdI783ecRMnkqZFKhlEPGe4EKPys/oVt+yMeZW9+Kz6nlUv5frWPXCqdPpcj8TdV5eXu6qv3V1LusmXd27jauyzKpEXL6f2sZyXebn5yMvL69BHcvQEVHVs68LOTDVmem9ztsovxdjfSb43hG86upqbNmypd7Uop8pWqDhti1ug6Cbde03HsPPvL/qllc9S2Lg3LVrV+1/UDu3SjjcrAs/MUkmK1uQHZ2F4zb1JHdcub7lwc3Lg+cWg6TGW1RXV9dTDjpCI7e5eEb51c2q01nUO3fuRHZ2NjZu3GiMOVLrOZFI1Cor2UMh6lslLLopP7muZaXs5rmTZU/nCXWTa7mOhIfOFOfl5bkT91KnjLy8F7JxovNayPUsBmMxYOo8GMLI0YUa6Ora5LmQp3Tc9Igq27LsiroW9WUK/9AZNzqvhZdsmzykqlGi8xCpdS5+r7aDm1Gpm5bV6RF1mtTNMyrqxM0rJ2J6hY5NxVskH0KnCT2tEhOdHpF1pirbcj2pMdJqnKI8U6AeqqHvxzMnxhdVj5i8oW6zXiadLdpI1StuBrxqBLnNvqTigXbzwJnGxpycHOTn52u9oup9W7bMPJNCoxE8x3GOBfA/ACcQ0Qdhldu8eXMMHDjQ1aqWlZxwl7sRNVnQ1KOioqLedzpFoE6PuHV8ATeXcLNmzWo76I4dOxCLxdC3b1+jYlUPE2lTBzPZgySTRfnZxLPL9at65CoqKrR1bCJpXq/q700eJK9pVnkqTa7j5s2bawmDqjDdBjOVKLz77rvIzs7GmDFj6hEFUceqt0hYd6rRoRvAqqqqUF5e7kkWTFN4bkpUlltZicqya5pS0tWx+N95eXmuUxumaSN18NJNn6reTz8hAybPvey1E3XsNW2t6/teRpxqWMjy66Yf5DpWPZnNmjVDq1at6tWHaYDXkVo377KO5MptLcuul/zK8iSTUzd51p0zkVpVdnWeIh3cPPXi/+fl5aFFixYN6khXjyavvWgXkydZZ6zJs1GqV1ZnuMnvy8rKGtSVWo9yfcpTpuqYKRtpqpfTq25VUqvzxosjPz9fW78mGdV56mV5VR09Oo+bqhfUulUdAbK8VVZWYu/evUY5jsfj2LFjh7ZuUkFjevDWABgGHytfg8Tu3bvx9ttvNyA0KpMWh7AsZRc4UH9gVae0ZCUvWxw6C1Alg+JVEEP1VVyrU16y50MeVBOJBBYvXlz73HLnUTuLPE1rskjy8vJqXd2m96rXSVVkoozmzZs3UP6ycjIpJTkmw414m+pYfS+uF94Q1dMhiKisrNQpAVG/ah3rFIlav3v37kVeXh5KSkqQm5tbr07VOvbymMpkUyWH6gCgThOqA6qqwEW9mOpYllO5fnUrNFUZ1nnsdPUrnt/NU6cjgKpcis9q3crndXUriLtMwE1eUVWG1VADnQzLdSyv6POqbxNR15FE8aobZOWBTPbQqfWr1ocqk171KuteeUZF9vjrvEWyfhAeIx0x1NWxm4dIV6+qcS7rB1mGZXIvD+6qx1muXwD1vI6mOjZ5mVVd4Fc/yHXt5tmXofPGqaRF1cM6OTY5PdQVrHLdinO62RN5nFPrWBc2oI5z6rgv9/lU6leuV/EqG+cyYdQRRaGLZRl+4436CyLTwfdukcXAgQNpzpw5DRSwOiWls8TlKRFVKQhSJr+XiZppoNO5kuXBTY0hME35qcrRjZQJIW7WrFm9cyZvnjrNqlo76jS2atH4UQamOvVTn6Z0Aqqyla1yHSHzo2h1nV3UZ35+Ppo1a2YkvbJ73lSnKtnVyalOyarTHG6eZV196siYH6IgP5Nap14EQefl1NWpOPwQXVmxyiRBllFRr3L/Uvu+6rl3q1OdnJpkVCYIqgdUfgZVTuXnlgcrIa86wmXqz37rVGf0qh5l1Vsv6lIOqTAZZG7GQip1KsuqGkKgm94XfckrhMLkmXfr/7o6NXntTX1fR7DEM6tT6DojzM3ATbVOZceBrk69DFyZQKqkSjdOqfWm06eq40CuU3ksdDMMRH3qDC95jDLNJJkcBqqBqzMkVO+oOn0s6nXLli2oqqraN1fROo6zFMBiIroqzHIPOeQQ+stf/qKd0lFjaHQWjUpg1JgwHYGRB0pTp9MNrqry0k0veCkwMYDl5ubW63Bq7IbJUnSzXtw8SLpBVvaEmuKRvGK/VI+lyWunGzxMU2E64qIG+MpQvUiqlWaabtFZfOK943DQdcuWLbV1qSPXuuBzFW5TBzp51U0pqspetdBVhaV643R1Kp5NrVedZ06uW7mvqkaNburWbUpRV5/yACvLqa5OTZ5lVRfIU1duU4nqNKI45Gkvtf3kZ5D7kSyncr2q8bc677I6xWWaJneb7jJNf5umwE31aqpbUR+y/OqmbtX6VKcQdfdW69RrGtEUKqPTtbp61r2apg913iC30AJT3cqxaLo60Bk6Jpl2m5ZV69REDMV4pdYtUD+uVp2mletX6F7Zs67OyLnJaE5OTgNZ9SOz4lWES5jGMFUnqh5fWV4TiQSmTJmCLVu27HuraB3HyQXQD8B/wi67WbNmOOyww4yxC0Ad0ZA9P6bBUHU1u00Nqt+5kQ1TnI2AX6/T4sWLEYvFMGLECFd3s1/PiJtr31SPsrBXV1c38GKonjxBHPwQYd2UiW4AcKtHwDxVkpubWxsTJteF2/STzsqUPSJqfE12djZef/11xONxjBo1yliPpukRIVsmS1PnoTMZE6pnWSW+omyTPMoDjiqTap3IHk2dV8TNmDARX13cjE4eiaiBspWn27ymot1kMtN6BBoGeKukQDUGTB47t6klHfFVya7Jk+zl+ZD7ZHl5eUp1qDPQ3OpR7h+pyKNsEMjeolTqMFWvnKke5XFGJlS6fl1WVlav7+r6tqwbdV5OnUEbj9fFe3vJo84jpzoFWrRo4bs+VcPMyxPnJY8mx4DOIbBnzx7jGC3O66aeU5VHUY+qo0Xn2czJyaltj0zQWDF4/QHkAFgSdsHxeBy7d+/WkhR1qkENCFbd9/F4vFaY3QZXtTOqykslebrpRdX7IUPnwVGneqqqquopE1n4c3NzUVVV5am4dApMCKuO5MneTjfPnDzdJ1tqOTk59bwRcv3n5uaiuroa+fn5Dax34UHy4z0yxXTJg5xqDYo6kOMw4/E4srKyautZlF1ZWWn0wMnW43fffYdYLIaNGzfWqz9xf7UOdfWoToPJyk3IgqhH4dEV/0+1KIUlLnuKZG+GzvtmintRg8NNXgzxPhaL1RtwkklexKBOS5osbdGeMuSpWHVQ1Rlzar2ajAZRl8K7KHvUZc+67LUweYTEc7t53EweItkLIZ9zHAeJRKJ2cFdJh1pvWVn1F++o/RhAPWKqyqQ6yKrv5XNyv8/Kyqol+y1atGhQb3KZsuzpvJWyDOhCMHSH6g1T60fo0Hg8jvLy8gb3lj08at2pz6p6rtT/pnoP1fem70Qd5uXlNdB1urLl51I9U2p96upUPUzfyfIl+lNZWVmDe6uvXtDJgfpfxavc59yudZyGqVR05aQDr/+lfv/hhx+mXZZAYxG8owEQgKVhF5xMJlFeXt5gikEoTkEy3Kx/Na5EF/8k2LgY4CsrK2vLqKysbFCGfG/xnRrHonqiZJisVaFM9+7di5ycnNpnFYORbGnIA4ggBPJ53VSNOoXmZqmqyl3UmRg8Kysr6w1Mol50A4r4rRwMnYq1r8LLY5KXl4dEIoHc3Fwkk8nauga4Y7rFPpk8eLKV37ZtWziOg27duhmtU5WE6Dyg4rfywCq8piaPsl8LX2dgyO2tkjjT1L98rS5G1C3ORpB73bSqm2Uv+o7qadJNRQtCKWRSjbPRee3l+/itOyF3qvEiSLc65SzqRMioW72Z5M7kgVeNW1knqcaEybsk6kbUtyk2STV+ddPUKkH0qjtV7oScmEJNxPcmr52p7mQCrXrnVLmTQ3i8ZoDUuhH1Z/J+mpwDslGi1p1M5ExTyrpYTrXu5JkK08yP6iXWeeTUUB25/kxGrC6ExM0TbJq5UD1ysgHnZ6w1zZ7J/1mVO3WGx6TvxOsLL7zQoOxU0ZgE7ysi2h12wfn5+Tj44IMbEBF1GsxtIFCFRXRCdaGFKmBug4GXl0l0BjGtpbp21elAIUybNm1CLBbDSSed5Noh/Q4CqpWhegTdyIff6VeVeKjKy20KW+eRE/XlZwBQX1OZklG9KbL3VzyfrPjFdEsikcCuXbuQSCSwYsWKBkpMrgtdzjO3eDi5vlIhamJqWo5pkQ9VkakyqIuFU5W8zgsnnkG1vsWrmPJTSagqbzryJsuSSa5UciEPNrr6kgfK3NxctG7dWitrulhAtxjBVOpKhs7bK6addXFVfg+5ntXZBbWuVC+R6s2R60ykHBLeR1NMmkwSvGL/TFN6boanjlSUlpZq5UznIVe/V/WUzqtr8ujKzyxe5f+Wl5eH5s2bN4g3M8mPbEzJsqSrJy+50s3GCDkQ3k213tQ6casjnQdcLtskV7JzQ52BEXWg9k1dPel0ucmQVPW7Wx90myUQ44Cqy3bvzpweNSbBC316FgBKS0vx0Ucf1VvxpJIcYR23aNGinvteJYOqhSEImzpNK5O98vJyVFRU1MZRyJ9Nq3DkwUgMbibLVgieEEaRB2/atGn1BmHZAhOHWFWrvqqr7tQ603nyxHOoUxGygMvkWSYwstUvDlFP6quOSMukSEdwRAdUlYM6hSj+n2pxpVNfauyZrEhisVit527IkCH16ks1OFTSrHpJTPUlv9d5VcT9TN4nGbKiU5WkqDN1xXZ+fj6aN29eb6Wx+Cwfwqsik2rdYKUz0OQ4HK8+qdaZrp5UGdMZaqrlr/OYyH1S5+GU+5Ncb7I8+e2TsoypxFCdftUN1Lo+maqMmeRLll15wJNnEdxkTNcn5f6Yl5eXtg7TeYZVg0N1AqixcjoZE/Uh15Gs602r2YUO03nl5NkVecZJlTHVkJV1v+6Q+6Poo+oqVjfjVp7VEBDGt65Pyno6FRlTD1nvuzlOVHLo1idVr2Uq/VKWSV2Yk0oYdTKWTCYxY8aMdChOfT6QyZxyWgU6TgxACYB7iOjeUAsHp0mZO3dug4FB7kA697lO8ORDnNN1XnkqQg7OVC3fmvoBAK3wqd4mVampHbV58+b4/PPPkZubi1NOOUVLauWpL3UwkC0T1RKRp1hlxSZ7M9W6ceusqudOVgQyORRtpsZ7yVNbOve4GynzQ8x0HVSdmhHPJpMMnSdTR/rXrFmDyspKtGnTxjhgqgRDF6+pegRkgiHXkSxPgpCZ5Mk0WOrqSJYlleyrdaQSC5OXXB4k1cHSpOxVUqHrc7LuU6dc1HpSvbq6AdIkR2JQVaea5TrSGZE6wqrKkk6edKRVZwypU6Tq9Kibd0k3JarzfPutIxNJVQ1H3XSoG0lV5clkQPqRI1l3uxlAstGo8xSZCL3pUAmWyViU60jtb26yZCJabvWk00nyDIIfwxqoHxrjNj2sqyeVgLoZO7LuVvubSq7UECw3Au9HnlQjWjWmZWNH1knr169HRUXFvpUmxXGcvgC+BDCWiN4MtXAAvXr1ovvuu6/BXLguPkqdahNQpz90ZCddT5ROsajubN0gLp5RdTXLFq+scHXWiPxZN51rqiNRT+JZAHfFa/JA6aa7TYpX/r18X9WTIk9zA/4HKJNi0SkTnVJRp9hkpaJ6Nt28weoUrUm5qOd1nhO5jvwOUGp8jpgikom0Wg+mGBOT0tUZF/JzuMU0CSWsDuZyKIApTkdVuLLSFf1OJTtu8iSe32062zT9r4ZYuIVMqGTHFHcoD1SmqWz1qKhouAhM1KdcR6p33CteThe3JJNotb5M4RNuIQAmsuPHGNMZryrx1dWX7pDlUfxObQs1tETue0KPq3WlTl3L/9utftTZKdOsi59QHDUmXNYnuhAJtzqSdXdlZWVtX1brSi5flSl1elb2XOvqSQ0nUd/rwkp0IQB++p8uhlDtg271NHny5H0yTcrRNa/Ffn9QQwpflE4dAuC3AA4AcDWAbTXnbyeiWW73atGiBQYPHuxqMcfj8XoCp/MqmDx6Xq53VTmaBlo1JionJ6fWWhHCqiMcqrelvLwc+fn56NatW4NBVyaAOlKr1o1QhKITlpWVGUltKtMSqXqlAP2iCOElkevGaxra5EmQFaPaub08LXLnFXXk5Y3atWsXKioqQEQNZMZUN6YBVTyjrJhEvTRv3tyXF8rNWycPwm5yA+g9v2p84e7du+vViXiv9idRl16eulTrRiX3oo78yo06fSUPKGpAuTCE/HroKioq6qVxkOvC5M1Uia0cL5Zp3QgjsU2bNml5ef0aiG4zBkJu5H6yZ88eo/dSDk1QDzePUyp1o3p3W7VqlZLcqCRM1Tey3AANF1HJ/UqedZJnCXbt2uU5iyL/Vq4b1SiU4eb51k11iqNVq1aucqN6vVViapIbOaxFHcPdpoNLSkqM9aMzClW50S3KkKeuVdKpm42T60Kd7k4H+9xOFo7jZAHYAOA4AFcB2ENEf/H7+8MOO4yeeOKJBopGtgL8TpP4cW3LHimTh86N1KgeA52i0VkmokPNnDkTsVgMV1xxhW+vimzZqVadLlZAdWerpFhHkOXPsudAtX51Hifd9JqoG/mQ21cmJLqpDlXJimtN0yCmODCgvrWrTv2rA5X4v++//z4SiQT69+/fwAsgXyfuI9e7aiCIV3UhgBysrQ5UslI2eaDU4Hb5ProAZBMRNsmOPJjL/1UeaNQpV9W7rRpMOnlR5UZuT1WGTPWlO9wC2NUpIb9B7KonTtZD6nvdOVVO5Olp2XDSeZHk6X11QYnOw6sO9Go9qDIi9yFVXlSviFpHajycqd683ssyosqmWrYK9Vnl/6C+V6dO5bpQ61dXJzrZUWVIhane1PpT/7vu1eu9rq68uIUqZwKqd8xUp7pXr3oz1Z9O3uS6k9/L/10nh7IMqfKpfmdqFwB45ZVXsHXr1n3Og5cpCsArcL/xEnAdZHKkC3ZULSW5QXSue51Xz0+MkC5mQSZ3ckPLCpWIGgTS6qbGxCEUb7t27YyWo+wVVN326iCtTrvK7+VpMdmL5xY35RXroipi2WJTPXhA3RS1IGmmuBa/ngZRJ/LgL8oH6qe30XlhEomE0WKU5WXr1q2orq5GTk5Obb1UV1cbLWlZRtSpCl2QteyVdJuid6sT2cMgypAVqkpw3QL34/F4g+l44ZUyxfvorOeqqrq8g//f3psHeZZV953n5Z6V+77Vnl1VvYPZ5AljI1nDYuFwG3lAyBMWHtA0jhGOsayIsdHYM4Rsh5CEHTLISPRAo8UWCOjGIEBIjDRqQGFjuoHAvda+ZOW+7/ubPyo/t7735H1ZVepWl6h8N+IXVZ2d9Xv33XvuWb7ne85VB9cj4B7NVKRXOYg34oqBKuiaeAcvFQylnP0U4pKiJ9wMZ9VnBVIyYnbdmc2yLErZaaBThH6DbvLfNTXX24ykuHPe8efc7EVLuJlsSQqZS62JPzfeAHsZ8efGUzVu9dzcCHlKoXJKQShCnvya7GVv9MyoXvX0Az03PpWumZ69EDk9O54f59fEB4rK+fYgguo9BVZ0v/eSj5StQb+mkH9F4m5FRorWo6GhYVcRRiojwveyHl//+tdv2b/x4wfRwXunmX1K/vt9WZb9lJk9aWY/l+f5zF7/uKqqyjo6OnYhMOpd34j4nRKim6m80wOrys0sbvHhHRc9YJomKRIor5ArKirsxIkTUcoRPlURwVvnWOTYptbEr4/+DgfTIzCqaBR9qq2tLTxY3kh7qNsbqhS3SddXHUaz60qG+XvibYp8m1LE3gin+F5bW1s2OjoanHcfEfO+HmHyBjoF/d8MH8cHO+q0MZCDpaWlyLnXc6JOftHaeF6TR+BSpGz2I8uysMee8wZi67l9GrykuH+es5PibCET7Avv5tE1f0482q+ObgqBvBn0UY1gfX29NTQ07EIci9BZ/6dHHz3alpID5JKzur5+rW2NXwtF3j0SX4S8Kjct9f6qIxWtqaystIaGBmtsbIwM5F7oq///3vFSB6yIX4xM8ME5T6Gp/j397yjC6lFF1QVF6KruW0NDgzU3N+/ipHkZ8QhsClW8FbTZp41ZC4/Op9BoBVE8Mla09x5V9vqxqqrKGhoarKWlZdd76/um3j11/hkevfQoud/fzc1Nm5mZKZSD1P7neW4jIyP2YscPlIOXXbvi7O+Y2ft3fvTrZvav7FrT5H9lZv/WzN6d+HcPm9nDZmadnZ323HPPRZEnDoIaAjX2DBVCFHYqivAVtsvLy9Gfvto25QSosjOLK0dTzp++i0YLc3NzVllZad/+9rftwIEDoWqUD06gRyUU/fNcABVqn65OcYV419Q6pNqdoBx0HXimHjrm5p0+HDvP7yCSSq2DOsbqDGIQU/wy9kejS3XyfBDg91//nJ6eto2Na7eK+Mjb84NQLkWoXYoT5N91r7VIpbA9IuPXoQjh9jwg/kQuvGwUcab4TjUIfh38uVB0W2VCPyBSrId+VJbUgfbOsF+HFCLlkbmUHOhaFCFUe6FS/lywBioTvsWDXwsvE8r9LUL7i9Zhe3s7cmyLsh17rUUKhdFzgUxoWnVH50eGP0Vh0bXQd06tQwqh02DBo3KMVGBYxMkt0g1Fa+GzHZ7GooCBXwflKdfU1EQtjFLvr39PoXIaNPh1QD5TwIlHrfVdb2Q3vTyorlQH2TuEHsn3KC3ysNc6KFLpuYLqQHrgxNsMDwzoufjmN7/pXZlbHj9QHLwsyx4ys5/J8/xNif931My+lOf5/Xt9h7ZJUYWsfJ8iJaRGaXl52ZaWlqKf6yHVQ4wiUg9dHZWiw+fTIghd6lPkqDz88MOWZZl96lOfCoeP5+oh8IrYGyR/yPRT5Kx5grdGe0XvjzORgv298tG/qyOXSgf4Awdq6532IrQ29e7eQPl390pHHXbenfdfWFiwyspK6+np2YVG6r6mHBHd+5s1xPr+qdSyT7OnAhff+kZTqSmkVhEJfX/mpyibvotvaVPkmPu99ykP75gqYlDEq00h9freewVqHrUyi5GYomCtKOWj75wyMkUGVykeOlLOR5EjVpSt0N9RLq1H7XxqFD20V+o8Jf9Ffcf8u3uHg/VW+VOURdHlvVLDe7XBKqLdaBr0ZvY+laFJrUHq3XXvPWWAsVemqijt6Z1L376p6MynHM69gpC90r83Ou/Km1Zwwr+7dzbV0dpr729UmOLReY9C+4Dc23wFaerq6uxLX/rSi+bg/aA5eJ82sz/M8/yTO//dl+f5yM7ff9bMfijP83fu9R1Hjx7Nf+mXfmmXgVAP3BtEzxspMoZ7OYPqEPqPL0bwqZqd9wvCkUIo9EAoOjM0NGS1tbX2qle9ahdaowfFo5fqDPr3TzmDKUdAHcClpaUoMruRM+SdQQ6pNwieV5ZyALwTrJFoyjD4vTfbzcW8WQS3CJXxEejc3Jxtbm5aVVXVLuXAHnijoKhSET9IgwNvJFLvfrPvn0Isi1DLmzGMyqtTZ9CnptQZVKfAK0f//p4X5JFK7wz7NJXuR5FhKEIsi/a9CKnFMVDDwBr4NFuKP5f6eKdA0Zub2X8zi97fo5RFlI2UQ6DnHofQF894/WeWbnFUhMTsRV+52ffX/de0ZBHfNiUDRR9dq5T8p94/hdh7PeBtgdeN+qc//55fq46xf3+vB72Tq/uc2nvvGPr317QlslcUGKSyFqmzoMFQyjlMBQZFWZvU+3sk7kYycKMz8Nxzz9ny8vL+cPCyLDtgZlfM7Hie53M7P/sdM3ulXUvRXjSz9+LwFY0HHnggp7KUkYKPU0bbpxBSH795GtFpKsEjWD6S8YKpqaSGhoZCCFkPtCqtVKpVI3gV0BRydzPOqhotlL6ilrKXhWnmIkf1ZlFLRYFSaROfXtZUgU+V4JgWIZbst99rnyZgKFqpXLgi51T3OYVYpoITTR/6vcY5TyEV3ilNfYqcc09UVt5QUSrdG2H/jkUyrsrZB2SgFUV7rWnj9fX1Xekg5LooRUYRyF4pIaVSwB/16UGPSN4ImS9yyJU/6rnEKVQOvpx/55Scp9DZ1F77NNhetAmPRN/suVZEOkWZ8IR81eG612trazfU4XoG1AakkOjUXu9FmeH96urqrLGxcdf76n6rPlQkmvf2Dhdz806W12X+3TXw9pQAdWBTAbe3Xakg42bkO5WBSDnb3l6ngoybAVp81snbrlTWRYOsVPHJXtkmL+tFe61OZkVFhT300EP2/e9/f384eC/VOHHiRP7Rj340GMa9BMkshvF9CtdH6inBUfTCIxZeaWj6zhNg1UgUpW9Sf05NTVltba0dPXo0QuoUqcAZ0Goqn7b2DuDNRicpzkzqff17p9J2+u4+LeUNvzpQN5uuKoLuiwpLfATmo/FUROrTlCiMzc1Nq6ystPr6+l2RuToHvrDEfzRQ0LVMEebVUOkcU0UCvlBC01G6twQynjyt72yWvstV98gXQ3iOrBp7v7+eMK/KeS+CuCJTune+qEblw6NuGsSpo+uRuBQ5XpETLRyprLxeUFNUKJAqEtiLIK/0DCV4+3Xw5yFVHJB6b11nHeqM8lG0RN8jVTDh10l/n7PtCwRSKbqUDHiyvF8PlenU7+n37VUkwDqg7/QspIoFlMZQ9NF11DXwBSKpdUitgb5X0Rp4mVcZUAS6SA6K9ADv4oshfOq5SP5TZ1//rnbOo5KpAhBPN9jrPHidp89InYPUnvNOn/3sZ38gGx3f1lFZWWmtra2FaIdGB8rNUMfGR8GeCOo5WZqCxVCoAKhC2ouPoqm2IiKwInk1NTX2kY98xLIss09+8pM3TDtr2k1JwD4i2osIrU6dR7N0FEV/nnfCexUVBfho92aj/L1Qy70Iz6lUm5KdVblpWkGjvpSjPjw8bBUVFXby5MkbEr15T33fItTOO23qmHp53avoQf+dd1xfjBynuJU+peq5RT59UiTHKS5p0Z6qLHsu4a3IMfzRIv5gEV3Ao1WeP+h1VEqOQaJT77qXjvJpolSasEiOlS+ZelfdU4/CEnD+eeQ4lQ5XpCoVcPv0oDryqSyDyrFy5FI0AFCaojNLQKbverNynOLEFumoIiqE54d5pyMlx6qjUvLa2tq6J9KcQuJScqxBFfrFv6uez4WFhei9U6nvP48cp0CTokITb39SRRZFnF//vilbu7y8bFVVL94923cOHumxosMF4fpmNiBlGIuUpqZozSxytvi7ohaeO+Ghfg4SytPzCRA0Bs9eX1/flbryaWmPzpHe8AZwr3QVz1ZB12i0CJkr4o3o+3lHjnXk+YwUmVgVpt/f1McrDEWotre3wzzq6+t3RaK6p54r5PkiKIV77rknyQdRdBmFvbS0ZGbXEVevLPWj+6upaUUbNR2jitHMwpybmpp2IQ6eE+aNhO61vpc36j6lzpyL+D8+ovY8lvX1uFmyIvGpKNvMgnJvbW0N59QjKjpnv083g655ZInzpDrIIyP+fT2you+kaUR9N3VgOGMvFj3yiGEKMeL5vOfNoGYeFfSp0RQq7PU8JPa6ujprb29PojxFf3okTL+X4RFa5oTzOTU1FWUr9D084qlIT+q9vNyoTkXW2trarLOzM3Jg/Dv59ypCOLEZKaRbP7OzszY5OblrH/VdU2gmMuLfqUgOVQZbWlpCqzOVUX3HvZBLPWOq64p0ytbWls3Nzdn09PSuc6d7l8pI+Tl59LFIb2LXXuzYdw7e2tqaDQ0N7eK06GLX19eH3/dGM4XkeU6D/umRH89vQFB0qBPkEQGNIoga/Z8aNSuy4quLGHvxOFJR4tLSki0uLu7iLilywPfoYdGhUHwRj0F5C/4deU+ge0U+cJhUianS0VQ7e5Li5vi9XFlZCQ6Ed3C9Q8TYi2eoHI2ZmRmrqamx6enp8H5mtgvVVDRLgwSGKiB1ZlPcK/9+29vbuxCCoogYudF0isorMqvcUb9/BFueV5jij/pAzMtrilenZ5C/c349Guu5sil5RSmrk+7bS6Rk1Z9bjfqLuHTq8HiOEXuknCrdSy/PqneKqvzUwGo1YoofmtI7HvFQhE6Lt3yqTA2r1zs3ktci/Vp0JvM8j5wHRQ+L+IFFutUXaik9wusdz4n0NuRm9A5/pnjOfL+mgX3QpYFziuOb+rv+born651dzXppZkT5j/qO/nzqudzLTvKe6D4NIuG4pvitfi/hQSp33fPg+Kij6gtLPIqc0juc1fn5+UhHafbHy+vY2NhN+TR7jX3HwXvFK16Rf/WrXw3GQpEtT071wri4uGiLi4vRf6egcUV7VLl4ZEeNvTcQRR81hvzblJEwu2bs3/nOd1qe5/aJT3wikKxRLKkD5w9ekXOqhl6hb5+auxHZNkU21hSzr2pmqEJRhMpzIosUp08ta5oLA69nQyPFonSyGoAbkca9IvniF79oZmZve9vbzMyiaFLRxhuhyCnHJYWwqiOq+8b7eW6jT7XdiAxf5Jx5MvxeqVQfXHiEvCiFyrspypFCUfdKGafei3+n/LAU3cEb8xQq7lOHRcg4OgRZ16F8vSK5TNEdioo2PG3FBw6s782+WypVyLulyPtmu6ulKysrdyHfqRSp/1mqIEUdE8/H9Ah/yqHW9yviG2sg5FOhqbSv8mu9E6Z/6nuh84tSgrea7vVgRVHKcy95VIRKUW2dc0oW/TnzTpZmL3xg4ANZXxjpz9nN7ht+gaKAirqlKCdeFvfS+z4YKNKP73rXu+z5558vOXi3Mubm5uwb3/hGcKQ0QqmtrbWmpqYI0k4pABUidRrUCcQRXFxcjDx4DNbGxoYtLCzY9PR0EFazuErHIyIYWXX4Ghsbw584Sxptbm5uBqSksbExOIGVlZW70tIaXaacXH0n7wyCHHBwFhcXo3SwIokeCdGDrlXC+m6p6mE9OHSxTyk5Td/p3hU5tykHXpUC76ZpI7PrDoUqKK/YUu/X0NBg58+ft5qaGjt79uyutLt26Ec+VIGr8r1ZB95X1Gm0XYTwpCLmFO9K33Gvfauru3bhOHKue6cIljdMXjaLEAEfnCwtLRUaXw2+UMDe0CrimkI8NDipq6uzlpaWyJh7hwnZTPHnioIT/2663+rsFqEdniKQQq1411TwhVy2tLREhSAetVKE1fNcfVDiETkfgC0sLBSiyOydBl+eT5UKutD9vq9kbW2tNTc3h2wAht0HzimUXPdur/fzfLKpqaldnGUd6shz7m5170CRGxoaIlqI7h3y8lLt3crKis3MzAS9q+lQnlWEqKaQ8dTeqU5pbW1NOvWKNHp+fari2L9bCsgBjVO+uSL/eu6QzRuhxfX19cEmHDhwwNbX11+0v7NvEbyKiopdsLJy7dhMdWoWFxdtYWEh+m9FTjzCBWxudj0KuJHThqPW2NhoTU1NkROgykgFWXkLysdaX1+3J554wjY2NuzBBx/clWIt+ug7aaSTQrbUgKnhVgOv78R7qYOTSndotJ1Kc/j0hnew5+fnbWlpyRYWFqL/r1Gc8rXU2KeKP7yCYe7sER81/hxkFLJPbeg7nT9/3tbX162zszMYiVTQgPzpe6kjg7OAItV3wkgwn5RTlpI9HzhopK1kcYYqO9K8PhjiXC0sLCTlTx3ropSbGnZN86eQcL9PqkgVASpKfytPR1GrIpQ/pSsUIVEkS7m5OPDKYVR0wCP6zc3Nhe+kDpkaPHWi2SfVf2qw99onj4h7RFWDVtV/Gvhg1NAHqQ8y6B0xZFlRkL10OumzlE5PBa0pna7OlwYFyhvWQMC/Q+pcFel0SPY+00QwwJlPyZ7fJz4enStySngn1RWpwJt30nOmjib/XnU6wTfnSmkvGry9FDpds0seDU7p9KKPd5ZTmTPVf8xBkdK99ITqPt7pu9/9ri0tLZVtUm5l3HXXXflHPvKRyPiqoQLZMrte9eKRu5Sn79Esn7ZF+BT5UfTAzCJD5RW7R7eKUoAKA/vKJYwIytdzJYpSmymYOwVtq0OR4vR4DppPiXkOjzoS3vAqfK5O7V7pI58SS8H12vJDSbQ6fIpFU5o+HaGFIurE8u8U+lcO1l4pFt+uxqcl9N19elaJ0Do0kvZpdkVa9T3VAGiaT99JSd2+WMIXv+i++aIffW9N8fmCA4bnlSmHTlPIivj4v/vCECVIK5Gb4QskNCBRpM6/h75P0V7p2fLnK1XoknpH/2dRoYsWS+i7eaK9GjKdf+rvPnWZejdfzJB6t73ez3/032gQrGicL5C4mXfz76WOuk9h+mpVnu2RK01tamDp/65nTG2WfzctaEgVYBXJo5dNT4FAz6coA8iOnpOis+Z1h3/HondLyWMqva66wxfMeQqE1yM+iOTdvG5UXaEOvv+7p+SoLlGdrwVEm5ub9jM/8zN25syZMkV7K6OmpsYGBgZ25b9xDhQx8NyEFAJRlDIpIk76NJcn3aeidP2ZOnOKDHEYzK4rrY2NDXvmmWdsc3PTjhw5sss51YjBp+6I9oocOUaqiEA5CCliq4fYiwiuPl2nKQPPP1M4PRWJa9GAR058+pg5eHK58gZT+8N74wx5/qA6OKpkV1dXbWRkxFZXV622tnbPtLEiqx7d8uiC5wym0FWPtPrAgX+jykkNpBoOT6z29AWf+tbom3PjK3n1XXwhB/Pbi7vqU1dFAVCKRO0J1MvLywGV8/QLTXnrvytCfjwHy+uAlpaWXWTwG6Fzfm806ClKRaV0gO6d5wSur69HFYOee8u76N4w7/b29l3np4h3q4Zdg4KiAikNtqenp5M0C9UBmpHQvVFdqg6I8p95B/bI74sGdFoRmaKO8C7wo1NZiaWlJRsfHy/kEKsjpnujhWxFnGivC9rb25N7ow6YBm4eifOofRFFZHx8fBd3mD31Ve9qa9gf5S3q+fbnv62tLdJpGnj7vcF2popFUrZGdcDi4qKNjY3tCeqk/AC1eV6fzc3NvWh/Z985eHmeh9y2ller0+JbQBRF/pqK8pFolmW2sbFhVVVVhWklRYBSKRTmq9V0KowIokaDnqj5wQ9+0PI8t1/8xV+MFHwqTaSFILW1tWZmwahqepbn8x78jirEFPEUhUdUBISN4vbvwfvrOmtkmUKv9L14n5qamnCYUqicpiVQ7Bo5a/ovReBmr3HcUgiPR0AUHVhfX7dvfOMbtrm5aQ8++GAhWlVZWRkcx8bGxl08K/YkJbcadaZQAX7PzELqZmlpaReak2qb4CN83knfk/9m/aurq4Oh9ygH76H7orKmQYX/mU/tYzz8mWJPPHqj6ETRh99h3ihmj9SoLvHIjSJKRR9de+RgdnY2egcNGvz7pH6unFG/7lVVVdbS0mItLS3R/HU/fFsLXVd+T/+dzpU1Jy2lc/cfPZ9FWaYU4sd8MPjd3d3RWur8/Xek1lVRWS/76J6JiYmIR61rn5qz7qvqPD27aptaW1uto6Njl2ykzqZHyb08e8Sbz+Lios3Ozkb6UfWLfwc/Zz1/isCpbjlw4IA1Nzcn0WIvLx6h8zozhaDyGR8f33VmfVDi196jwz574VHGtrY26+7ujgKC1H4oOOHRYJ9lUruGvnyxY985eJubmzY3NxeirDzPQ8QIsqaRb0NDQ5TCBHlIcTn4uedZacqPw6dOHoKjhQZLS0sRL0CjGS3C0BQofBR1AKurqy3Pc+vu7t7FW6urq7OlpaUI/cORRDg9twHn2HO7fISoqAoHlcNPVOmrqNTRVSWsB7yoSAIUloOxvr4epcpTbSL0PfSAa2pVia+8Jw5wim+niArvoVGu8p0UGaqrq7O1tTWrqqqK+F0pnqciqsiUVh/6Ki+PpDI3RVMUgVCntggZ5qMInKKovKPndSoqBMKVKjJSRavpfI3OiXqVv6URPe/hydYYYE3nKHriz7fnzfgUv98L3kMDH863J1N7nhbv4pF6zniqyEaDAeUvpubukXqPAvlAFYdCgxxFslLv4DnDnCUNJhTN2osH7ZF5nEPlCLIfKTSb91DnQ7lyHpUv4tNqwKrID3utjogGn4rsevRX0VNFfkDjQLI9jxvd6VFfz2NM8Wc1QFWnjLXyyKLqWt2LVMEdv6P0Eb5LdZQGnkUc0wMHDgRuH+/S0dER3lODbD3frFWKr53iNvs/p6eno0ycchVTuhZ9qzKV4p/re7S0tER8PvQUzibn4sknn3yx7s7+4+A9+OCD+e///u8HYcCZKyJOQ4DUP32KlkOQSsfcqAgBIfYkcD20ytvSaIHD7w2uGqpf+IVfsK2tLXvXu96VJHJiEDS68ClYFCPCuBck7gmpKYWizq9Wuapi8JVM/k9fWIAySVV+psjdvoIpxWtMzR3isxonRal8GsxXe/pyfeRuaGjI8jy3zs5O296O73tU5Fjnr8qd/VCn2ae8dO3N0j0eU5zFFHfRcxa9o0ag4VORHt1Vh8zzFIvmjuwrsq1Ebf8Oft5ajKKEbI2YfUrYcxHVeVZk13NGFVVkqN7Rs6tpR49KqzOZokwg84p8Fq29Os0pVNqn4nTuZruvb/ScUI+m32jungyfCn5T/FY/d3SkrrtPJfrCkr3m7t/D8+0UsUNXavDt+WbeUfTvoLLmeWfqCHsUyAccRTpGz2qqCCHLsl1cOR/s+hYn/D/NengUnTXSANfrSrW7/h1S/GhFodWBT51TryNVv6eK+9R5J6j1XHz1FTwf31fcKpVGgR4v87ru9fX19sQTT9jU1FTJwbuVsbCwYE8++WQUMbPJtKPQQ6XRjK/EWlxctLm5uVDhs7CwEP7U8moEYX5+PihGVcRK0tTIuLm52ZqamqypqSlUy/EzjRJqa6+V9pOKxAHc3Ny0X/u1X7M8z+3Hf/zHg+NEBLOwsBA+8/PzNj8/X1jNuLi4GAScgUJQ59VHLsyd9/CVwSiM1tbWyBgqrM7hXl1djaJIdbpZd52/8rz84TKLm4HqgfdVb37uuv51dddafSjxGdnB+VYehyIPzH1hYcH+y3/5L7a6umoDAwMRQrG8vBwhExhzTV+q46coEB9df1+ViPx7VMLMIufVI9i+AkxlSfkptLhQB1wdEfg0pHC8/OseFFW1oRQ9B80jKqlKSuavsq8Rvipo5q/pPXVAFGXUKmuVG9U7TU1Nu7i0GEZN7ShHy8uPrr0/u3Nzc7a8vBw5BD4lq6i1R1F0zvoeilJoRaE6sspnVD6zl5/U/D3XTDlmZtf7biqlYi/59/Kj8u/5WBoAeV4ZZ8CjiTp35H9ubi4KQNfX16PUPkbdO9+pwD81/5aWlogf5/mxarvQnR7B0rmjQ/n/s7OzkZOoDrkicRp4qhPF/NVu6dmFG6fOuTqaKf6ozxJoVS0f/ntmZibiXHIGkE10jw+c1TaldD//v729fVfWRvm8Hl33oIXqzpTtmp6ettXV1Rft7+w7BO8Vr3hF/od/+IdBiPQA49DowvuP3wit+tPqxJTyUeXS3Ny866NCpA6H8oxSKQ1NZ3iH7ROf+IRtbm7aG97whugAA0GjABgppw2lo4dV56zzVocZo8V6KO9DHQZVjt5R5hCrw4nSxAD6VJJXlsytpaUlOJzqPPiUmKIXDI08vaHVNedPRXt1vVE0OIA4asvLy1ZdXW19fX27nGTWWWVEWxMoeqQwv/L9tAWGV/DM2xsrItDl5eu98TY3NyMOkKbuNOhQxZiat2+jUGRcFZX2jrEGVz51h2OMcdKImbOkJG3v1PigRNFpRV2US4RSZ71JmWKUdI2ZP/ug2QBF1BVVRJ9wNlNpOR8M+hS8IhzeGVZnQNebOXo54U7QIkfYO2I4s+rEp4Ko5ubmXeuNI58Kom5kRFXOfQDImmv2ooi2oen11Ly9A69FPVrUp7xiRW01uEittTqWut7KDfXFVQQe6jj6c+nlG12oaKjyVtXpQidrKxPvtKh863qvra1FnEHlP/tCEK/D/dlMZbt0vVkj1uxGcuLtvKK52B3l2ysVw6doda4tLS2R3dHiD0XOzcze+ta32ve///390SYly7KLZrZgZltmtpnn+WuyLGs3s98zs6NmdtHM3pHn+cxe33P8+PH8wx/+8K6KVCVLcshvZNTVCVRFrdWBPnrYeZewkRr5KxekKOpUeBmB0MIMTZ+srq7aU089ZWtra3b06NFd/A8fMasDxZw1FeErgDXNowbSp2tZa5/yLOJNKJ/IK23leyhC6rmCWu2bqij1aQc9lKmKOE1f+eprkDqF8DViTkH4mppYX18PBGcKWnSkUj7KS/MtZrRXk/JU1IlSlEjTnMi68lZSqVpNm6jTypojj5ra3itFm0pZ+TSnpqx8wUqKtOxTV74IR2U8lV5OpWiL0oQ+RevTVakUbVHhUGq+qVSPyrkGDCrvKjc3StH6uWuKNpUq1NT+zaQ5NSWqVZJFqcKbSY3r+fTtKLzMoGN8MVBKZrzc+ApPz6nWdVfHKCUnek7155oeLUrR+spbRUY1EPLV6crd1cJA1Y+eiuPpLD7F6eVHHXBNjSsNZy8etdeVnlahKCI6xsyiQo4UBcdXCvuKWi8zfHeKV+nnXFRRz9orfQg9owVXe1GfHn/8cRsdHd1XDt5r8jyflJ/9splN53n+wSzL/rmZteV5/s/2+p777rsvf+yxxyICvKbSUoiYj0zUmWPDVQA9EdOnoOrr63eln3wKKoUoaXWpKig92D5dkEp/qBPHnLVaTpXqXtGIphFAkjRlcLNrrOidX2M99GocbmaN/boSPb0Ua+xTA36NcUw1zePXGAOA8ixCBlhjVaw3WuNUlOplQlOqN7PGyMTNpmJ8QEKErmuszqXOI7XGRfSBm1ljTZ2m1lkDqVtdY0Uwis6dIkY3s8apdF3RGquTf7NrzJx9iu5W11hTXFoE41O7N7PGDOUVK0qUQltSug39hvN8M2vsZaJojevq6iKeXNEaYxe8TLwUa5yabyqTosVGN1pjr4+L5qxrrLxEHTe7xn7OL+Uaq+1+sWvskXH+W/WgBtF/njVO6Ypvf/vbtrRfGh0XOHgvmNkP53k+kmVZn5n9aZ7np/b6nlOnTuW/9Vu/FYSI6FphV4XNlQ/iNwHh0ZSQJ/ubxYUKGoF650mRL1XaWkWnkZGSbn31GZ/nnnvOVldXra2tLTikGlF7tMssrnLyUVGqGOFGkRxDI1AfOStX0UfP/K4iihrFaQsNTxDei5StEbOW/SuBV8nkWgii81c0y6MrioAyNG3FGlNB29HREZAiJbx78riW5eN4sMaejK3Rs/8oksK6ppA45fzpHIh6lYOov6PrqilBTfn4titFbR0UafOtQFSPqeLWuStC5OeG3KfagfjBs/TZfLS1BB9N++nPir7Xj9QceD/9/37OqfcoahVS9IzUe2pxgcqLbyWi767vn5qzks31fPiPlydFF7WYgnmqnvTtQdAnpGYVZVEdow6i2fViChwDlX89q6oH9cyovKkspM6uBhOKsvszrFWeusapNiCaBtUCFqVL+POsMuPRc4+geyRXC5vU5mBvma/XizdCbr0+9wVZivazPh759FkWDeR1zr64g/mmshNqG9XGayGcUjxSRR3r6+v2nve8x1544YV9U2SRm9kfZVmWm9nH8jx/xMx68jwfMTPbcfK6b/glYrQVjuWwesOhyiCVMjKL00U4LyiMiooK29zcjCIRUhAeblbBVifQ85Q8iXZtbS3Mg2iKw/e1r33Ntra27I1vfGMUgeGQFqWPFcGDg0FUpVwOjcopE/dcGSWcctgpAqGCOUWW5aPpEQ6qpqHUmTOzoFxTJHeF6VN8JA4XykDRDuaR4mlomlsVr19XlJHykOrr621oaMjq6urswQcf3NUeQFMJ2tYAI6TOpvJ41tfXCxE7/j//Tg23Ki1NwWukrRGt58GoY6qGEUV5MwiuX1fldyE3GB5Nn2oqZi+OVGNjY1Qg4NfVp9sJ5opQUKUOqGFQh47984YghXb5ggZFNlLripFVOdyLV6RIPmiXtjPSdU2hMOx9a2vrLu6ZGl1PnGddea4veGGeMzMzEQKj3C30K7pFg6AidNlznD1ihO7VgEkdFw30U5yt0dHRIB/q2Ph11bSc53/6/SfjoMWAyLk66mp3PH8yJa9TU1O7OHHKnWQgq1VVVdH5LuIftra27qI+eSqOFg55+k0RXxJZUEBAg7xUsZnPOqnM8mdnZ2e0rtBvWFe1/R4x9AUSioTTlFqzespdVkdbHUHN2LS0tOyvIossy/rzPB/eceK+Zmb/2My+mOd5q/zOTJ7nbYl/+7CZPWxm1t/f/+onnngiOFxKvtQqKTU6+lHkbmlpKRwsNea+L1yRctSUoSpxFA4ODAeP4gTPA1Dh8jDvn/3Zn9n29radPHlyVxWUCpoihupg6Px8+xDPR9PoEwWnlU+pvkkcHJ8y3tk3M7OIF6QcFq90PKKoPChtzbJX3y2dr/5MIz9FkzyfwvO0PD+O+Ssfqqqqyv7kT/7EsiyzN7/5zWZ23UFPzdNz49SgeG6Wn6PuuY+OfXTpeXBE2Hu1PtCKPR/B+/kpSsK50eIkdYiVU+iRB0VJFI3i/TUto2ircgmVo6TcPEVYvILWyF3nmWrboSixopiKHKWQEJ2nZgj03+hQpCi1hp6v5hF3RWsUjdN1UQTbryFGTYsWtG2RIl/Km0q1QfHc171aznhumsqfnhE9S2p8+S5Fqvfi/3n+nM5RET7eXffX8xQVOdL0p+qb1D6zfrpuvh2I6h0/T0W5fDZIeefoRNXZODBayIRM6lnRNUxxzfWj6+rTysqJ0yCRIMHzy5UXp2uNrKg8atCla+crmdVOE3ijS32mylN7blQ563n7m5ubdu7cOVtdXd0fCF6e58M7f45nWfZ5M3udmY1lWdYnKdrxgn/7iJk9YmZ27Nix/JlnngmVWnog2GyFnxEQjTDn5uZsbm7OZmdnw99TbRamp6ej9IxC4xpdtLS0WHNzs7W2tlpLS4u1trZGUSfza2lpiRScOqcIklZyzs7O2tNPP22bm5t233332dzcXEAbmKdyKBB45Voh1HS4Z378qcKvKWUMoR5KTR3jMM/Oztrs7Gy0rlRHokiY49bWVkTe1yiYOTA3nSeOdHNzc6RE1OAq306LONhb9pu5anENShAnQpWHKl9dq9bW1mgdl5aWgjFrbGyMFLKitlpZqtWwzHN2dtZmZmZ2BSZKI1DepVY4KvKhctnW1hbtvyo7dbKQIb+WyJvOkfX0a8l7ITc+QmfvtApQ15E5aiUmRHyfClGHVBWwP9/8ydlSI6wpeBwQNbRacc651nkq+qmGjTmCcKP80TF6ZlQPMX9fAa1oB46BosjM0a8jHw2iNP2oTjTOnvIn/fyY9+zsbECT+XfqDPIMsggYWuaY2m+/lsqJwllXSoiiRal5qgHWIBSED0cV54V5+rVknpwlZJJAD9nW1jiaKkTumBvnm/+enJwMconzqu2gNBDxXQWQRz7MnWvlkGNNL+PIqQOTWks+/BzkkOIDz49URxr9ovute85+9/b2RnYHG64ZLqUt+bVM6aKRkRFbXFyM0uEaJKfsjj/X/B39dPDgwfBuGgB4lFCrqH/u537uRftNPxAIXpZlDWZWkef5ws7fv2Zmv2BmP2pmU1Jk0Z7n+f+x13fRJsXMooheoWwVTj6qQDUqRAlratMLqBogf5Bw4FRAfUsR3Xg1kjo3fqZtCzY3N214eNjMzHp6eoJwYiR1Djo3rzA1Wqmuro64OD7VgiH3ChOnDcdNuQ07exwpIoXWvbHR+akzVFtbuyvVShTsDbh3KBVmV2WOolS+XMoweiPu91aVpE+rrKys2OOPP25ra2v2ute9LpqbtudBiaccNdZPUQZtg6DrxkcrzDw53xttRYp1Xjo/rSADCWAQOKAcvTFUo+jbZKgTqYiDEq5VMRa1NuLs8vFpaV9YhOJWxyJFvudseD6vN9RqBP38fLFLysH11A01LnommCPGWVE6TRnfbNYChxHjjOHzaSfOrm/xk2oHlUo7KrKZOrvatUDPrWYtlK+rLat4f+VE+b3VQNCncDVToQGMFtmwlz5wYY7IgqZDFVnXwEAdMN1fPbt8tEJeEdeU7Kle0aBF6TCsPec2y7KgX7UYzDtf/LdmfQA0tMBDOdg+APQf3wIJhEs57soVV72SCv5UJ2pRoGYnCNTZVwUk1HlVPajopnIiceZwrr1eaW1ttba2tmA3VP+9+93vtmeeeebOL7LIsuy4mX1+5z+rzOx38zz/N1mWdZjZZ8zssJldNrO353k+vdd3HTt2LP/VX/3V6BBzSLxzpZWIHoHwyJP2mFPIGiQny7IQWXqUJOW8KIStcLVyA73xxQHUA/yZz3zGtra27FWvelVwXpR3xdBUjo+YvQH2vDsMnLYO0UPiOUGKMvq0dwpKV5KwwvzKXUo181UDp8gnBk7TJVq1uxdnCY6dGjgzC2jdjSp5PQ8Q1IbejA899FBkRLyi9tWaPuWtaSgcGN8mBidQ0yM+Hc/8Uo1sNe1U1D5AlXIR58uTlrUaWvdZ+Yi6t0WcPxwFTedoOkyNiqZyNB3PHFNrmOq1qGvoKQMYR89HUkTWp8R0bsimpvCUP6trqHSGvdYwVWmpTjTOAudwr5QYukedQH5fCe5apIOuUeRLZdBXYe9V1azOYIq6omdFswg4gpqixRjjMCgdJNV4W9OL2vsTvaC8Q3+OlXOmPRG10Aw0jjkiQ5oZ0DSi9vhUZxo50HS8Fsd4Z3+vYEkDOdbey6CeD+8QqnOjulFT8lrQkcpaqN1VBE45sYpgc449v5j5eWRYA3V1qKFPYUs0EGZvFUkvAokU6EAGVcfU19fbpUuXbHl5+c538F7K8cADD+Rf+MIXkigZisGjOz4i0wPoUzTa+sKnFXxqk4NIJIGTBHSLYN9MmksRHkUoVlZWzMysubk5KK0U/O3TRtr6BKHm0IHObGxsRA5HUepakTEiRB9dc+A8aufTRapUlcjNUKdNo/5UKlibKCuXhMFe1tbWRg5aKl2gc1PSLoNUmy/tZy5DQ0M2P3/tphOv7FECyJknFKvRSUWsRISsmSKxRMJaUYZyUjRbFWkKzTG73k9LUWI1OqkUpRYSKIdJnQvlRabSQD7lh6Or/CpFSXAYdH18uq+lpSVyJJX/5dFXnVcqyk+hrym9ofwenUdRKlKRYdZeHUZN4XtqiabOOJs+navtI7RlixpqTd8jg1rpqA63VqF7Q+330xtqLQRhzThj6hD6tKMGzKA26A1FDJV35s+m12sakCqfj6F9QtWB8bQM3/RWOa8MnC+cVw2SfYYEWcM+eecL5xW9oQhmKi2vAZ8HGTwnXB3WVPYGOUutGbKbclhTNkABEOymRzA1feyR35R9Qr+owwUyXVlZuYubV0QT4IYR5obeUY66OoO6l7qn8/Pz9pWvfMXm5uZKB+9WxsmTJ/NHH310VzSjLRmUGFmUVtFCBoTct0bRSEHTAhpdaUGDjwKJpNXZ86RXX1zBz7VPERGM2fXrlZQrpJVcnvzqo2dVPma2C4FQlET/VHTOE8Vx0JQ47MvidS6KioA6YIw0fawFAEr298UJvhxeja8vovAIjpL/92q2qqgNUakn1isBXH+mZHBNK5rFhQmsnz7fE/21IEHTTmYWrZ2mGVX+WE8tAvEtVlSn6DPUSdJ5+DYSRW0wdI4321oFp8C3VvFtSopaqKRaw/h56r/1rTr0eX5eKqP+HXw1P7Koc+T5yJnusVYGK3Lo210oipPab9+Sgz1XR1DbXGjKMSWLqfY/KXn0BR5ayOOLOviZzlXX8GaKjHzRhJ5jECUtJmM/dU6+7YbXM8ol5v18YZGmQj1SrWimnu0Uaq2FJopaq93Q+aljw2CtQNS9rfBdFHxFKmlZzZiog+8rvZVKoSCKItVa4e2Db08JwHlNpbMJ1JAv7ILafA/uFNFk4IarM8h6qOOngYcig+wpssA+PvTQQ/b000/vjyKLl2poKg2looRMHBUf+fqNVs4Hh8PMoo1WUqtPx7LpNEwkHaZRklZ7ki723BiFqf2h3drassXFRTOzcNcrKcEDBw5EgpfitpH+wuFkaKXa+vp6khvjScqK9qDYME7qRCpcrkRVz4vR/VMjoKnKFIzv+VgYCrPrVXSaPkrxTTxXgv1DGaqBUsWvBF+/f6OjowFtXV5ejgy9GkuNIjVt7gsgQIaRKWTTIzyKcvq90xQwCMr6+npkMLUyWNNEKb6kynmqLYmmeVNy7tMbms7HEdKqPa2C83vn0WD+jbYz8mjwXjwmTUNrAKNOFrKrhigl5yBN7J9WZGowlULBpqendyFg0ApI2+MMaICS4vO1t7fvQoD92VMdxZ54lAmCPT9jnVZWViJHUeVceZBKVm9pabHu7u4oRaoOGUPpAyn+I+s1Pj6+i17DGSmSc8/dw2D39vYWGu0sy4Jzp05Xak5zc3OB6K+8Qi2cUAdPqSopG9Pf359E4nDOtW1NSs5V1qempiIqA/unPeH24sIpAtfZ2RkF6yn+tGZi/JzQU0tLSzYxMREF6l7OtdDtRvq8o6MjOIOcXy0s8bzLIj73wsK1ljTMy+z6PcrsnYIqfj5LS0sv2t/Zdwjefffdl3/uc58LaI+mgTAu3gD7tIEiURgWiiyUBO0VuUYWmtNHkMyuN2VUToQKjM7DG5UUF+KZZ56xiooK++Ef/uFd/ZY0PeBbtAArc4AVBfMl6ZoaA4LXg6ZKKVUyr4ihrovntIA6aDm/Rqn+o+ic7/fHfJSgq9G88vc8z8v3oFKeUqpNCHNRREQduIqKCrt8+bJlWWanTp2KUsPKtdHUD1GoFh6YWYh0ixAP/ZlHBhWJ0cpV3TdFYIjSFanUFiUaOOl8WANFKJm3bz9TNBePUCrR3Lci0d6XHp3yc9Df05Gah87Fz0NbjjAUbUzNQfdGEVK/NxhTnYNv1+IRMv4d89BzxFlSVEyNpaLIKfRY06V6jlRutUgLmcWY+0BWg6JU41lkV/cJ+dP9UKfMZwBwVBVpUs4XDqLyDlOZCU0fa+GEFp74LE4qU6I6RgsSfOW8csI1WE3xcFkflRltp6IOmAbRzE9lSeej6WzsAPbS2yjVxSkuodoBX1TiiyLQx3V1dbvS6+yVDy6UKpGil7Bn6px6hwvUjWIIHGlkiiIc5qSZInUAZ2ZmbHp62qanpyOaBHYUHQRXcHx83NbW1soU7a2MwcHB/KMf/WiItBB4Brw1DjUbNLvTfmJmZiYSHIW9MaxmcWUPjpRWqra3t0fVMzh9CDBzyfM8go+Vg6fCws84cCidkZERMzM7dOiQVVVV7Wopwhza2toiAQb90Ao3DIM6VOoEI8B6oLSNiPY901YsyrfQefj5cMBJmWCkNKWuHIvp6emwX1rJy/w1BabGSSNh5sNclBhMlRionXIstOoZWVH5UeRndXXVxsfHbXt72xobGy3P811Iq8pPe3v7ropsDIUadIyOVjn7+aj8sD44qWrM1WBqlJlaHzVemn5TZ0KRQ9aE9aENhHI3cTIwnn4+ul+6PsolUi4d88Fh4FnMx89J56Prg2LXtJoaKM6556oRQCjvVh0tNVS6LsxJ58O/o5ABRwcDpOmhovmo0+O5tswHXYjMFBkqdXhY7+rq6iAXKssqP2rgtU0Qg0Dcz8evD+dLK0NxCDV97Lmre81H+y/6QjfQHNWF6B4FCJSDpoUnOMh+Pl43K8rLXGpra6NgSos4mI9fH+aEztSqVG2Tpbxjr3cU3QVx5t9pAKEOKc9WvaO8S3XkzWK0i0BbObNe9yin0Xd9YL21BZbOR9dHK8eVz25mUUWxdgEomg/yznw0O0fGCd2s6zEzM2OPPvqoTUxMlA7erYxXvOIV+R/8wR+YmUWNY3Vxp6ambHp62qampsJia5SkfCN1DDC2GBkOBEpVHQNF7TRCQ7GrkvCGGB4AHDuz6xGRVsy1trba17/+dauqqrL3ve991t7eHikthceVf6UQdJEDxyFQpIpB5Ipzy/NUYXljRwQLKqDohiKYuk8cUPaGKIioTDlzil4WkZ3VaWM92etU+hDl4KNXokPfJkQJ/j5Sff75562+vt5+7Md+bFeKTlExDK5vW6KpFJURyNZaRabtQJScrlFzKv0MguXbRKRSTfypAZDuDU4fijtVWa4VganiFdLgivboPHA81BlK7Q1ViciJTzWrIdO2PAQaZtfbtmhKUIuiNMBQKoU2A1ZUTrm5qb3RSl11WrVyWPfGp9p0b0jh+rYsytFNFYvpmtxobzQzok69OrLKuWJgUEn7KcfKt5fwjqr2uESX3KgdFlkRXRN/bvZyfAgA2Tv+nRbnaBbEB1wpB16LJUBhtX+rPtsHWy0tLVE1tyKnCmSkAj91mBUd1MyDtrViX7B9GkwQ+KHblPuO/dMgC/BCP6rvFWFWDqOeGfamo6PDOjo6dgV+WtyoFdnoeV2PyclJm5qasqmpqWCTcQLRbcg8tq++vj7al87OTmtvb7fOzs4wHz1Dim6vr6/bm970Jvv+979fOni3Mo4dO5Z/+MMfDg0nUWBm1+8GxGFRwUfo2WCcP22zQNSiHbxVKSHsCJ0KPlEZCl5J7r4aVA/g9PR0pDg1FVdRUWGjo6Mh9VcUPeP4qZJkLgzlaaiRSEVjRT2BFPL3BkOVEmvC/qjge1RqL2WtpFicYVohVFRU7EoPeEje88d8esBXxirHThU1H41Qtf1GTU2Nzc3NWU1NjZ04cSJSUJ6XgQMPjw0lifFQQrWvsFMOG0gLyhXDob3WUgiHVkuqs6OpYoyY7y+Z4vSo4VByvKZg1TlPcek01YWTpAivts8o4jz5ysii6kPPNVQ5UVlJVZEqp0+NmaLOWqkPJwxdQNovVQ2vsuIrDXFGFc3Y3t7e1TkgVZ2stBTlq/rKxxSv0J8h3+8T1Jv1TjlfnCHOM7LtnS9FwXRNvPOjHFXlXaZkRYNb5qJnmjPEHqXQZV0D1fkaWKKDNI3u03ypIBsHTIvrNE2M3tRzrDpf0W7kWatSFWnSbgnqcAGGIL8a9CtflypxzrG3hR0dHbucUnS/VmETGOm+TE1N2eTkZGQTlReregXUE2QbWcXpwvHq6OiI0sOaqVFARlFtnQfOIPNULiW+RmVlZRRkd3Z2Bkews7PTPvrRj9rFixdLB+9WxoMPPph/6UtfCl4yxt8LDILLoVY+gfJiFKlSZwVhUdhWK42UU0Ykpc4BQuIdOeBl7aiuHDJVrh0dHfbYY49ZdXW1/et//a9DtIuToJW6PtJGgTAHXQtNA2s1mE9xagSlKenW1taobQdKgBQDTqTfD5/CAwnR6Ek5Jroffh7sB3vCOmhkq1wO1kDnwcFFhkB1QT90P1paWiLlwRxAyr761a9aRUWFveMd77CKioqIg1kkmyhWRcm08bYaX01hosw0utb90IauzAOjC9GajxphNb6paNbLpo/ycY4aGhqiymOcC0Ur9Xwo2qA9ulL70djYCbWjigAAVuJJREFUGBBknQPzUCda06fK8/HBTUpXcEa0YtfrCk3hotwxuFrUgGzigOh+6DlFNjFwimqDEmrGwae1/fnQno3a2025uLoWKpucmRvpCjINOEC6J15X4BhWVFREzrI6YhhZrysUaUEuvK5gLVK6W1FkLXrRIhw9HyqXrAdz0Fs7lBenDsdeukL3A9lUx0f3w89jL12Bg8w58GsBEKAcPU8vUABAHUCdh+oKlU2vKxQN9LKpNBDVFdrXVHWFOoDeMdYOGMp/9rQYRQE7Ojp2UXX20hVFfoUGlso/rqmpsaGhIVtZWSkdvFsZJ06cyD/+8Y8HASUi1rRTET9IYXwcPk07QZAF/VD43Kc21KCSYlFCrTaM9ek3EAc4FD71pb2rtra27MCBA3bo0KEIZVAekK/2Yi00mtXoWqM5nBAKTiD1Ej1qWlLL2bU1jKJAHBIloyvZWknOiv5obyUcX20hocpc0xtaTEFE7wnpWtThW634VgPKr9Fef8yliIy+ublptbW11t7eHlAF3xQXArimr5UUnyrk8IUl2vZDiyUUXeVPbWFCFK17kyoWSBVyaPpP90YrqRWx8wUdSs5XR4OhhHs/D/bsRuui1a7a0iP10UIKTXmprGgVoO6RL3ZBTkBzOYda1KNcYV+ApE28tQ8YKKqeHyW8+3Oja4KDr+vAGfaFAVqEhKyovOqZUf6TZj6ghGgbEW2tw/vieGuBmFI8Uj3RSLtxRrUwLEUj0NQf5yXLskhfaL89RSiZm8qstnpRXeqL+BRdhxen1aCqS32fUIJYRde1iKSI86VOsAYGrI0W2ChKqr1KlY7Ex6cd2Y8U8uUdH4IUbBw2liwD76S85I6OjgiBAw3kdzTbwTup84UDODk5aRMTExH6hs3TAhFNS6vD193dbV1dXWEunZ2dAdmmJx4UF2wmgdDMzIyNj4/b+Pi4TUxM2OTkpH31q1+1hYWF0sG7lXHPPffkn/70p6PCAYj6KbKjkmXZbFXQCI7240kRmoHBcfwgHptZMDyacvSHDyWAc0MKKc/zSAlq6kqheE1nYSQwlKBWmprw70/KZn5+PhgG0sAcPC2N1xSwRl2kjjzkzR6g5DzJ3cPuELlxarU1hnJBUgUAKGpt3qwVr8qvU/SUdVGnFseHAhZSeKk0uHLKUBIYoluVQZw5HGpP1PYyqCkqnGv+HVEnis8XFvkUPPOEd6kymCpS2UsGCWxA6qhwS72/pg21clyrOXk3Jarrh/VvamqKKm8xZNrmwxfqeCQGGcQIcwY971T3AIcAh0ErJjV4SBULKd+TVPLq6uouGVQObkoGkQ9NUSo/DCfIpyb5u0cdNH2sPMYiGdQsgiIfWkFbJIO+eEu5v0q7aGhoCOudksGiwgCQyRvpQQJ8bZGFLdAqWdVBXgZJEyO7WvSjdA99fy+DFK95GQRc0ExKSgZxBBnoAPTgzcggwT1Bxl4y6B1R7dOp1fbYgZuRQa3uVlpHkQz6NLBmkW6GiqS0Di+DZhaQXmRQ18Dz4LVYRntRLi4u2rvf/W47ffp06eDdyrj33nuDg6ftE7RpcKpqzRP5McqaHlXnQqOqoqIGM4sqsVJcGHVqlHuiB0o7ZysJuampyc6dO2f19fX2+te/PvDqUKiKeChi6NuwkFbQyJa0kUb4Gt1rSoP3VqSBNSSi0zVAcWm/Jd+wWZEfqk1BOHimtmfQvmuKhinS4j/aXkWbwnq0Rw8pH+ZVhMRpQ9n19XUbHh62jY0Na25ujtpSaCNZs+sIXKqxLX/XFh2KRGo1mSJrioBpARHDtytRZC/V9Jc15rmp5r6+WS7PVKTwRk2R+R0d/nnaFkVbtqRaw+gztcmtIjK+FQv/3rdgwQHX1jmp1ie+/UqqNY7KlJdjUH+VYwyvcj4VHfN959Q513YeylFVeda1VkREESo9x5xv9l11lyKTWrygOsTzPBUNIuPi23XgADAHz3/KsixC8lVv4gT6anD2yVeDe96gdjrQ/WC/fEDuK9J9pke5v+wfRXBLS0u7ULipqakIiVP51+4OWl2t6fEUZUKbY6OXFxYWguNHatwXIWgbGd1z7BPPhXcG+sXcCIJqamqi3obq+E1MTISPzoH10fOmdlKf293dbd3d3aEQgl54yB46hWfT425iYsLGx8dtbGws/J256dqbXadoEPiB+vX09FhPT491dXVZV1dX4P+97W1vs+9973tlo+NbGevr6zY9PR28ZxQnEQ98vBR8PD09bTU1NUFwtIVAlmVBeDyJtLOz09bW1qylpcXMbFd6RXvxoSyU8zU1NRWiC003aCNcz/lCgL785S9bdXW1/dW/+lcDyobiNrveE0sVJIpieno6ICtEVUD3GEldP89j0XdtbW2NetwxKAzwVUtTU1PRYVb0FI4XBk/7OaEwKisro1QGyppoDdQKx1Kj46mpqbAXzAnEZnV1NXI8+F6iRCXoeqKu9q3TIoT5+Xm7cOGCLS4uWm9vb5gPAQbrQ/NV1hUkDKOAgmYPVEkqyR6lAyKonLYUj0urxPI8j1ASNUooJ42SNf2tShLDyLrrGZuamor6e+EIElAod4t1xzAoSsQ8FSX0hpFnwtFBMWOcNEWFs1fElWL/tbO/GkacIA0idd0nJycDOoCzpukxLfDgjGGQOzo67NChQ1FbBg2mkN0Un1N5c9rnE4dbyeBNTU27UnIdHR3W398fIYPMWR3cIr7e9PS0jY6ORi2VkHOezbvDB9N0XHNzsx0+fDikR6luVroLDo+ut6bGzp8/H6HSyldEt5GOUw5ra2urHT16NHLENJDSYhov42QGxsbGbGFhIQq2UlxiLUhQ+zIwMBDOuDr9nF1FIPXd5+fnbXh42M6cORM1d8aZ14bAnDHPVz127Jjdf//9EY1Dq3MV/VYeHHI4MTERdKG21EFveu4yz29qarLOzk47cuRIRNsAEFD0z+s2nPDh4WE7f/58SP/ifCJrivzy7uiW5uZmO378uJ08edKqq6ujPo5avKVnTDm6i4uLNjo6as8880ywhUoBaGtrs6tXr75of2ffIXgPPvhg/uUvf9nMLCj7xcXFSMmOj49HeXhIuhw+DkBdXV1kXDs7O4ORUyI/hgZ0YXNzM4LgvaIlAmNuWnHJdxVVImkVXU1Njf30T/+0mZk9+uijURoYR04Nm6bgiPq1GziK1qeftRqY9KNGu558q5A3B91Xu5LyqaqqirqPKzKKE+GdVjMLBtW3e1EDhwOhXEqz6xyLVOsO5Xb4SryiPni+ipV3xYCbmY2Pj1tlZaUdO3Ys7K/yTEhreMI5nI5UGwit4NKecuyJIpGaVlHUWVMJOGt+jXlfrRjm3KjRVgRWq9l8W5KitjXw/7SKWtdYn7+4uBhQLNLYkJdBblhbfS78JVpcaM+8VI86jdZ5X3UMFelVCofKkjrDGHYzi9oW+d5del4VMQMd1spf7c3nU4WkC7XfJXpO271o5wCMJnKME87wukKDH63aJEWHUWeNtXLUF7LwfO1RBiIHRYPz6ltSeGK8L3TDEcQJUV6WInPKj1NnSCsysQVaQKNpaR9wgAjpM+GDaXUqKLGivPp+ikRhE+A6Y4NA81QXKwrFR+UaIAAgg0bAcM4UidLAAxClsrIy0lHI08TEhI2Njdno6GjEP2NNtJ0RyFtTU5N1dXVFz+zt7bXu7u5gD9GRVVVVUYDDGk9MTNjo6Gj4gL5pxszsesZEA7ru7m7r7e21vr4+6+3ttd7e3ii4VN6ypvt5v9HRURsZGbHh4eEIedRAnuBoc3OzTNHeyjh27Fj+H/7Df7COjo6oUs07XzhAbMD4+Hg4fBz0hYWFqDUKSk0rwhD+rq6uoOhaW1uDE4Hgb2xsRI1o/SGfmJiwmZmZiIOH86UFDCg1FMwjjzxiVVVV9uu//uvBiCCAyjdQREHJpig7ijrg/KjzhaHEuVXlgtGE80WqEkPoFSrPJcqbm5uL2laApCjHRN9ZI0wUqka2pExUyfDOGtmqUw+KoigpTq2+qzr1OESakkVpqVPPs7/3ve/ZysqKNTY22uLiYlCoZhaiSuX0eNSKtcBQa786bzB9MKGcHlLwZhbWOYVQah8nTaVUVVUFLpNGsx4xIpJWZ5cebFmWRVxCjd4VpUSuNCWtLY48eqByRfqMgEJTlZ4/p3KF8UBvmF2/Ao41BBFV5ECDRargQQ4IEtTxU/3BPLSgQAs7lFqhz9NKZ/QVKWKPEiG7mrZSZ1B7/6mTnWoToc/Wvn/IM6h7fX19FKBqmk6DGi0W0MbUOHsYyImJiTAPOGK0rwJJVSe7o6MjOCjqkLH/OAiVlZXRjRq6t9gGbZ+lBTX8e22Z0t7eHtJx+nxknrZdZhbRh7ALvCsOg9KIyPLQrqW2tjbiIfJMHDHWWnnJZIY05c3zsIO8M/oM/by1tRXS3o2NjdG+9vT0RGlQdCXOKnZQ7dHU1JSNjY0FB3RqasrGx8ejFDx2kP6RPFfflWejK5ubm0PghaNN5gTZHRsbCx9kGwAEuVI7yHfzLP7s7u6OdCUIc5ZlUQGj+hpjY2P2yU9+0qampu58By/LskNm9ttm1mtm22b2SJ7n/z7Lsg+Y2f9qZhM7v/rzeZ5/Za/vevDBB/OvfOUrlud51BdLF5eoQnseaRsMvYoMxYAA83c2W7kzWs2GEPmDg5MFgXdtbS04g0RrRIV6UHGulDi6ublpb3/72217e9s+/OEPh4PKAVXFiBIhPYKCIGJqb28PjqpWC6mBV6IqiIJGpf49cSzUiSLS1HYe+lwUlRYqaJNZTbPq81hrUKzl5eVQZFBVVRURYTmUqny1r5g2hVYyuEa/fDQYwCBryqulpcU6OzttYmLCmpqa7M1vfnNQRCBpWp2o7RC8DI2NjQWHBqWLw6bpRYIMFBCyg0POHmjfQTWoyI6urSJJGNSKiooIsVKjxtpiUCl6QIb4rvn5+V2oxvj4eJROVW4ZKFgqyOru7g4OMdXKGHFtII0B1+dp+w3O8cbGRkQTQIY8koJBU/QVB14LCvQd9T2VGoHskab2OohndnV1BVSSdC3vCS9NHTLQG20xApqwvr4eIa7ID2dSESPeU6vDQebU4eZsarZEmwxr5bm2zEBe9UOQ2dDQgL0IATPZGdVB6Hfl6xGEabGUpub0rChHDeeTZuirq6vhHXnP0dHRqD8ahWoEy8iQEvGLdBAggtl1hFcpRVqJqUgcqLNyPxXZVdlRZwgZQn482ukdTuwm76lop3LQcOqxYZxNzifyo5xHdJBWnCrwQgsvEGVkSHWQrit/Ejyhg8yucXK16NGfS56LDIHY4+Cqf8B5xOlTEAKOeE1NTVToNzU1ZQ8//LCdO3duXzh4fWbWl+f5d7IsazKzp8zs75rZO8xsMc/zD93sd9111135I488EoQI0qkS7RXNUuImwqWl9MrN4UAoR0Srd7TFgd6/6Kv2tGJpeno6HCit2lR0RZt66vMaGhrs/e9/v1VUVNjHPvaxXb3NfKk+aQgfdZPy0fJw36hY05baWJV/v7W1FaWSUik1bduAofeVcZqu1RJ0FIK2rWCftK0KKSeq4JTPqJVYoDnKi9Dmy6T5tBKTFKTyJLUtAYgCzjuOiZnZlStXrLKy0k6cOBH2lv31bSlSLWVwxHgnLVjhZ9qdnz3Vthz6vtqzkShbbyogtaWtSPT9tO0EH0Ultfpcif16Q4O/lUDRBT2vGs1roAKKqFeuaSEDz9G2GyBLexUk6XNw3vX5ylHVYiDlkvmWRXpuQKs4O75JM2dH6SM4nQwcFS3+Qj+hKxSJ1N5uuq6axlMOEY27ceAICEFRtM8fH62iRp7MLHyHNqFO9QzjzMKTY86er+U/2pYK/ZnnecSPQ8erwwKlQlOUZhbkhudhwDHiiqgj52aWDM5wFsbGxqJKXWRXHSN4Z21tbdbb2xuQKeUEanCmXFtsCSlJUDEyQ8iuXp8Git3a2hqlI9UhU8cImdVsgaZB+ZAFm52djbi9Sknp6+uznp4e6+vrCx+c6/b29qATKioqwvmfm5sL70YK9OrVq5EN11ZJ6LrW1tbg+PX399vAwID19/dHwYsW+iiflTXkWVevXg2o38TERNR/UYNsHD+eNzAwENK+XV1d1tjYaG95y1vsO9/5zp3v4PmRZdkXzOzXzOyv2S06ePfee2/+uc99LhAjMU6qwDjgQLLA/erU4ahp+b9GshxyvcJH+Wg+JapOJDAw6SNF8DQ1qGkFfq7O1fr6ul26dMlWV1eturo6SldxsJkDc9PCCUj0oBKp52lKkPlqegwlpiTT2dnZ4BjAY4GjpH2FFOUB6cE406h6Y2Mj4kKxloqG6tU2PmLW5r8+NQVHqK6uLupBh0ykZIVmoigB7R+lxlVlpa2tLTh1KBOtsNXn+UoxCkI0JYR8plLY2rRUK41x/JSI72WT9cSZ0DZBWmSjspIKcLQlgt4YABqpaAfcI1IioK5aQONlxSOROGgEGTwPOoC+nzpMpGBwInierqWmqpUHSsCAk+DTa1rIwvtR0IBxVeoDKKuSzHGMtf+eVuDr3rGeOPzLy8vh3UgPY9D1/bSiEKcbJ1vbeXhZ4Xma+lciu6bf/XqSuqNwQNFOHC7/PPaP5ylxXrlbmjJUxAhHl5Qd1aK8nz5LecusJzQd2oTgcPpnIZvapJfnaX8+/zwQMW1SzPNwsuCmaUYH2QRRMrPQPUIpI34tJycng0OtaVDOHrKpPd+U84cTjH8Boqpnwaf0FVhQIIPv06yOPo930+I90vgg8v5Z09PT4XnK89N2Z576g+3TFjcMDTSRD2/XATLgq5INZD0149DR0WEf+tCH9geCpyPLsqNm9nUzu9/M/qmZ/UMzmzezJ83s5/I8n9nr399999357/7u7wZ+FIZG0Sw2hkNMhINzpkZbFb8idvwcBw+jhpOoZHQtK9d0jJa24+TBS1HSvxKHmRt9pUA6QAkVBVCkBSWT53nUjFcbeCKI2oYEojLIFc/T5qGgV9p0VluOgORomwaP4JBKY4B+KjHbN5XF6JEyNLOAuPFMRabYJ9Ai3kvb6ajj5VtvaHsRRSC1mMG3FuH39FmK3vB3bSUCksBgD1IfHdoWRJ+l7UmYO9/Jn77tis4XB0NblGizZ9oU6PfrGrC32urFtyLR1jbaxJdAAedbkVgMvKLs2tZH917fT9F1ZEk5WMgYvCPkS51rlWOMufLYFH31zc0VadZWI5qi5Z00VavIoBLrNfWt58OjgegfHE5FPvV9CFIUqVK9oC2BitpPEehh0JeXl83sOqldm+mqo0lw4h1O9kv506lgT1sv+WBIA0sQKp6re6d9SzXTAxKnAYo2/FYdjvNA2g6ivgZ8FDRpb0QCII+IKS0D51Yr3UHdQIn6+vqiwhPOHe+Fo+IRsdHR0ZBlWlxcjPicOJm8T19fn/X391t/f39ITYIyIv/QMObm5mxkZCQ8Z3h42IaGhiIgAroJaB/ryDNA3vr7+wPnvbW1NUIXsedTU1M2NDQUIW9Xr16Nep7SzqeysjLIQnd3tw0MDNjBgwdtYGDA+vr6bGBgIOJuarcAHMuxsTG7cuWKDQ0N2dWrV21kZMSuXr0aAoilpaWg/+rr662rq8suX768vxodZ1nWaGZPmNm/yfP88SzLesxs0sxyM/tXdi2N++7Ev3vYzB42M+vu7n715z//+cBtwrD5fLvvbE3qUqMnFBGOlnKLNJpBGaHMNzc3o2o0LaZAMajDgoMHN43ISSN7UETP+fviF79oq6ur9spXvjJEFUrO5aOGWFO/GlGAOsGJaWxsDIYDdITiBV07rYwi4sVR4EoYr/R4Lsocg4yTpY1IU2RnSPRUnrEuRGZtbW0Rj0dRQiV1swfKq9HKJ01r4+hpXz4Ug98vrU7NsszOnj1ra2tr1tfXt4s3pNwo1lgrnNW4I3/Kq1MStTowinzqOyEn2jKDSkc1ULpf8E7ZL0WttYIN5YoManSLIdzc3IwKSzRqZw2Vo0iqRrl7IIJ+DdmrhYUFdEM4x4pKKMIKAd4XZGkRiZcLPceKdtbU1ETnWDmmWpmuRVjaUkfbuvA8TWFqylTvQ1V0gDXESFIgA9KJk+krSfm7VmQzOMeKkqkuhJunqBytoXAyfS8z5AUHB8eWNLDqQuXHgXDizGvRj6ff6NmCE0yKnb0qOseadibYpSCDNVQUR5+VQnP0HGuRQIoHbHa952OKB+yRMZxoznFVVVVYP9ApdWoV4a+trY2aUGtlM8/z6WWCLe0RqwVaqjP4aF9LvSPZr5+mzxcXF8P6gS4q2qfrlzrHeZ4HJBpqgNpjbLJSazRr4Xm3Ku/KJaQtF1QE5FC5fdpIemNjw86ePWurq6v7w8HLsqzazL5kZn+Y5/m/S/z/o2b2pTzP79/rex544IH8P//n/xwqWPCglbSpC45DomiTthtQ0q9W2fm0LAdDIXAOBhGMIlt6LY86WcpnoR2B2fWUEJA7Bu4jH/mIbW5u2utf//qA2oEY4DhqBMuB5xkcGEUBlGelEbl2mVcSMaiV8o04AFoNqWRwFIpHGlQ5YjxBtGhe6rmJynXSFDZGmnSFpix4Bu+qXBFFG7XFh3ICicC1JxaylOJuPfvss7a1tWUDAwO2tbUVoU16LZNW2vH/GUqA1oCFCNHzwxTJVBRISesYOL0eTNEs3kO5jRjTjY2NCLlUNFiDBPYMJ0OrYbUIQT++TQbGAQWu7WWUHwqdABnDgU/xUTUgUWRWW8r45rTQMtgbDB3okjqEWpyCUgcJRr5AYLQHGw4Mc9Bmy8rn9Y4SfF4NHD31A5SCdDc6QRs86403morSwiLmgc5Ab2qxhL/aCdSRikr0pjoTtNTQSl2cWh+cajUj/DH0AT1JzSwE3KwZSBKI3OjoaHRrgXIqcVh6e3sDeqTvxTmuqKgI6zYzMxN4WqBVoEc4STgfFJlQINDf3x+QMZA4LXRTKgL7z/cPDQ0FRG5sbCyiPmhxG2vV399vhw4dsoMHD0aBFfoc5xIu2tWrVwPyBjqmN2+AkOu6gYYdOnQoQvvU7hDgTE9Ph/24cuVK+PCzmZmZqKce+qy7u9sOHjwY3ufw4cPW398fnD64mSCK7APvwXPg13GGtcgKJ7mnpyc8g3fq6+uzjo6OgIJr66OxsTG7evWqjY6O2uXLl+3y5csBuXz22WdtY2PjznfwsmtQxW+Z2XSe5/9Eft6X5/nIzt9/1sx+KM/zd+71XcePH89/4zd+w7q6ukLapLa2NhiTpaWlqFIGWFqry/RmA4y7El+VANu60x+uqakpGPqNjY2oRcbIyEg4eChKVcRKfqdSt7u7OxBftUUHRtLMbG5uzn7qp37KNjc37Wd/9mejsu/R0dHIMGuXcUW4OOxanafpLe01x9z5fhQxsDgpuyzLonUjdaDl7BjlhoaGqL+ctlRh3RRdUNSO6KmxsTGqnOKdMJqtra3RXZ7aJxAlr6X66pgB4WMktQoXwiyKBFnzRN2pqSn7whe+YPPz8zYwMBDSIB5d1dQVEaMvycc5U64UwQUR48jISCRr2tpBG7wqh0jlTdFOvV8RBw9FzJrx0WvWQM7q6+sjxwI5YN1AEjQKZt3YczVayAfEeBxZrTj16waaBeqj7UcwuHpuNADknEJtIFhi3VQnKLqkTqa2OFE5g6iNUwuPzcyiIEkdGEWklROo3F8CTHVkNO2Gk6D8J10z3SOcCwqqKF5g3ej5paR+DP7KykrgANbW1kZtPEjxadWhUlCU8+T1Gvqaddvc3Iz4aug3LRzQVlaa+sVhUCeWdaN9BggPgROFQjgXPIOPdh/Qm2bQk9PT0+EZ9IejRxvrprQWzUTgUGhVKoFzdXV11AdV10v1NU4b1bYVFRVRGx0cTD1DygllzVZWVqLK0+Hh4bBuOO3cg7y5uRn1MPRr1tfXFzVvN7sOaiiXnRSvniMFA1izhoaGqI2KpnYBUkDQKyoqottKWCv2Z2RkJKJY6fWdyC4yPTAwELVP0SBai5smJibsve99r50/f35fOHivN7NvmNl/t2ttUszMft7MftLMXmnXUrQXzey9OHxF4xWveEX+pS99KUQfRIEIBgoPxaqFFaAN6vgQReGU0OcG46odzDm0IyMjUcpInwFfxzsjKCF9BoZfb75QxTA5OWl/+qd/apubm9bf3x+4GaAcGDtfHaXGTpFBrcTiGRgL+tXRU4x0jJbdq2HQrveslV5945UOafLUWmmKxj+D7va8h6a5aGHgn6E9yjBw2vpCHQMUuT5D10oNjxYOgDqAOi0sLFhjY6O99rWvjZ7De0Dq1Qid9/DPgEeCwdG2LKn3IHWhz4CLw5wVzdBnoJx5DxwP73hqegRki2dwPvh+NQDz8/PRMxQB5kxoCwtkgmdQ9aq9rXiGpvO1TQZ7qs6gfwZrpS0c/DOQK57Be2gRBe+h74Jc8R6goZD9eYZyonStNBgg3adnnHfx74GziW7SZ7BHVD5S8AIC39nZGRwmbT6rrWHgIPMeBIMgGIo2+vcg2FCnjGdpg3XeQ58xOjoakDItAkF/aiVuV1dX0OnazFZvCPFrhdMHUqbP4LxmWRa1JMFJ0mdoBgC6QtEzNA3LOc+yLOyHOhRaiappXuUSahNeeGKa7tVG8NggqkAVUeRGEwpJ9Jzre/AMzZihd8wskl2cLz4gsTwDuVKHUt9DgzL2TsEZzgccPv7U7I8WhalcgfJpMIuN2t7eDjYIriCcPZ6jWQaesbq6apOTk/b3//7ftzNnztz5Dt5LOQYHB/NHHnnEOjs7A4GfyIZIWg2m9vbh/yuhWq8M0ihaeQzwRjR1hsApkkK6ZnFxMSKIK18Hx0/5C/oMTdFNTk7aBz7wAdvY2LC3vvWtwREjXWcWd+r2ndhJ08IlAI4mOiN9otfgkHImaiqq9tJml9qvi/3Q1LZyBeHC6S0USgZHeaXSZ77oBTQ1VfQCL8xf1UU6wxPO+X44TaTotJBHb5ngGezHc889Z1tbW9bT02NbW1tRGli/W296IPKjZQmoE2uj70EqlZssfIoepa1NnEkFU+3H0HuLQR14l7m5uahFCZEsDVf1NhL2HodKU7R6ewPOMnuvNACqQZF/lWNt0aGBETKvz9DbEvg7VYRwRkGbtBpa6Rn8DJQWPhYtTvbiBPoqWr2lRh1xfkbLBq9XtEchCKDqFbPrLWqUR6mOGVwp9GNVVVVELVAnWR1yrx+rq6sjPh4GjqxDZ2dn1CBbb0PRYJgPe7K4uBiq4aurq6N+fEp+R4/pM/SGF1KK+ie6BSSGc6yp3kOHDtmhQ4cCQk9WA7kC/Z+ZmbHLly+H9CjP0fcA/a+rq4sKE3jGwYMHo8wJKW8FJzS1x7PYj6Wlpai3H47ewYMH7ciRI3b48GEbGBiIsgwMOH2Tk5N26dIlu3TpUkhVXr16NVRJLy8vB5S8oaEhpF0PHjxohw8ftiNHjtjAwEBAFOFF5nkeFahcvHjRLl26FNaMFC/ZNW4qaW5utoGBgbA+R44csSNHjlh/f3+wKfSB3draCgHR2NiYXbp0yS5evGiXL18ODrOmdtFTLS0tIXV8+PDh8NHMD3aKjBzAB2t16dKlEOjNzs6G39UOB3w/74FcNTc321vf+tb92SblxYz7778/f+yxx4LyRcGqstIUDLwLvRhbHS0OvU/zeFQKFIQNVyNCZK9EZZSupl1wSqlC0r596jDimM7Nzdl3v/td29rasiNHjoRO7hgPTe1oCT/KUDk9ekiIViYnJ0OVHylRDoimqDAe6pxwwLVljK6Pj4JxFnAGNCWuiJT2i1NyuqIS7DOw+vz8fMQV0kauugesEdwl7cFEGg/DqmlJnET4f6DAiqA+//zz1tDQYO94xzustbU1yOvW1lZU4KFX62haGuVBeqilpWUXitbT0xP1BGNdfQpKn4ExVCdauZrqGGC0lYOqTpRfH/YAZ1F7xikxmn5YWhkH9xBjp323kFHeR69owxAdOHAgQn1ZH23zoL3ENLWpvcRwcEnTwgNDgStlgz3nxhEuMcdBU6RUHTScAdA/WvxoYEk6U29TUT6jkvC1DxrUE3SJmYW0kRaAqJ5TzqwGLxSXcI6Hh4ejgIwghCIJgkk9x5wzJafrLQOKNLHP6CiKCGiYrpWPOJbatoeAgkwLTpPvEzc+Ph7WZ319PbouUnWP6jmQMq2o1CItXwELEgefWHsXKsKrKLKm9qAX6ZzVueCMKHCghSMenOjo6Ihu8tDuC7o+qufYJ2S7rq5uFzWGM6YpauRCgQLmDn2AgiXkDUQVrqW3AwTa8BKxBehNzUhQLKfp/CzLQiDqUXbWDI4gwAq2QNPq7AN0DoAVbKUW7ugetLa22nvf+1575plnSgfvVsapU6fy//gf/6M1NDSEtMzGxkYQIuXBUDFGqtbMgpOBs6LOFwLBlS+gXShwfYb2aAOp0mbC2mQSIVOhxQHTfk04GkoSn56eDoRqSLsoKEVtfKsVqpmAmzHSROnaQwgiOXwF37xW20Rog1Oz6+1O4AiByqlC1RYhWs0J+kSUDapBJK0tPDDUvI9vc2JmgazOe2hLFf2gJEG1itqFoLC1hYq2UjG73pKEprjwZVQR++fpNVUUcWgbFz4YF32ebx3CWuJYafsSv36sG1WhpJHYf70yiO+kmlibfHNVD8pd24T4DzdGQKLX1hYqXxhUDItywpBfkDS9J1eLqEg7aoNgRWeVVoCDD1K5tLQUnT29DYLWKjiwIKZUI2tDXoJE9hkqCfwfrdLVHpbsgfbVUt4szhprRhqQtVEHH8cJLuPc3FzU8gcnSdtuKHdJK0q1UbLSR9Q546zrDQCaiiWlRUsPWlEgk2qsNWXGOum1jlpdCUIDL6q3tzfQLcjqYPhpcQFKRnoR3U7xAChZf39/QLAg2+N8a2ofWs3o6GhAfS5fvhz2YX5+PpxDvRkBZAx0jOIOvUGEuQ8PD9vly5cDAgfah1O7sbERbEJnZ2dA344cOWJHjx61o0ePRpW8SkXBEQMZu3DhQqA6kXbd2toKDiVp3WPHjtnRo0cjpJL1q6qqioofQSh5xsWLF0PWYGFhIcpC0DT48OHDYe5w67q7u4OuVrm5evWqXbhwwS5cuGBXrlwJyB72Z2trK0LTWXcQt0OHDgWufWNjYwhY5+bmQpGGzn98fDzoB81wgK729/fb0aNH7dFHH90fHLyXcuDg0fQRQ69l2CnPHt4XfA3SG4pQcYC5SqaysjK0bPBEbeWu6VVdGCrPJ6PSjANsZtHdiNoRnT+16SeGRRW/cmV8JRaOYwp91KpZMwuOo7Yt0LljwHC+QF1ShHyQSG30jFOrlXGKTDFvJfguLy9H0Zqidjjt6sRpGwklQ4MYVVVVhehaU8WpSFNvxtDrgBTNUYWJw4CjAOKoiKmS+ukZhpOjbQ7YU+UH0t+Krvbam0mRKJwe9ogKTo/2amqdtKK2DFEkWdv/6NVpIIkdHR27OG5qqLSQRzvH69pouh7UwV8TpAgX+6HOq/Iyx8fHQ0oQWSdFD8+J/myKPrEPOOk1NTVRZbbfT5APDElNTU0wcp4nydxx0PQWEW1HpMVaWmSgHQA8asPtC6S2GMvLy8FZVV7hjbIPyutl/bX1C9dEetRSsyd7ZQY0cwLBH5I/QRtOa0p3pVB7vatV546eJ+gys/DvlW+pV2Ypog5aTDGM6q3u7u6oX6Heo4yMq6x7Co+mqH3GBN3CNWMEMJoNoBhBG2+DRG9tbVljY2PQf6mbM+ifV11dHSrRQREVxeL7eT+9nYj9TPF1cfbQG7PSvFuDBIIQKuorKysD/xs+MOui9BMzC0EaOl3RVV+pTRCuTbO97urq6gpyrvQJqA2pDA9BIHIOB//rX/+6jY+Plw7erYz7778//9znPhdKon21F44dfdtwdriKrLm5eRfcTNoFJbq5uRnSBjgYeuciBoN+VSBn2ptN+UMgInCTEBjt08NHkan6+nqbnJy02tpae/WrXx1V28Krw7mgGlCVOkIPr4rvxlmk0lJRDpwtDr6mAf31ZAg3yhCjSNsOf5WTcmm0sbJHFjH2RI9mFg6dXjumV4CpAtcWDXwvjhWcP86NNs7lO7VJM86S2fVKRr5XrxEbHh627e1ta2trMzML64tTznfrHY1Eo9qIV9dAGz6DwNTW1kZtPlhfomC4ifQOA+Viv/QezfX19ahZMVXPKgvaFFvTVdrOQ9vr8B4EA3rLgrYnwjDi7PhGt7Nyabimyc2u806RW3i0OPTIB60gtGeg0iHgmy4sLIS1gEyPw6RpNeWaaiPgFEWBZ8GbA931DjctGOAZkpYlrcz3UmSgqX0MuVZJdnV1BURIHREc5yzLgl6YnJwMnC+qI0dGRqKrHPXKMm2HATJH4UJdXV3g+M3Ozgbkg++Hu4aOUoeS7wUJOnjwYHCeSJNyJvg+UJUrV67Y8PBwkEVS7PX19dbb2xs4fUePHg2IDU5PTU1N0BNTU1NhviA1ly5dim5N0Kuq+N7Dhw/b8ePH7dixY1F7Gk19s3eXL1+28+fPB5SMIEHPtnLsjh8/boODg4GT2NvbG2R0a2srOEhXr161c+fO2fnz58N6XL16NSD029vbwS719fXZ8ePH7fjx43b06NGAmGn/UO2Dev78eTt//rxdvHgxtBtRZw/KUHt7ux07dix8L5w3UOcDBw5ELbMuXbpkFy5cCOt86dIlGxkZCfpaOd/K0wPVO3jwYOQAox/GxsYCYghaeOnSpYiKATe6o6MjyB2I4fHjxyPeLyj53NxcQH4vX74c5o5fMD09HcAGUOAXXnjBZmZmSgfvVsbg4GD+8Y9/fBcxlshmamoqUio4Z9o+gNRNe3t7IJQCkYOGKQEaJ0k7Znvy5fr6ekRE93fUgZzglGkPIg4p34uy5aoyM7MTJ06ERrhakYRC0Lv2lHfHYVeyM6Tw9fX1qLEkc9YKMU0r6z202rGc7ydCJZrBGdOO6Hz0ihoi9pWVlV3d0FGESs7mgMLNobJpYGAgGOKmpqaoLQdRIxVgVBdSGs/+cX2d9pLiu7XQAwPI/j3++OM2OztrXV1dNjQ0FKXZSac2NjaGiBElfvDgwYjgT+CwuroaIkWtWtOGmqTRqquro0pn5qstKrRBsvI9SVspskPKmw7wRLrMly76kLpxdrT1ifbS0oCLAcqtvbQgi8OHBaHBAPueYIou4LTD/2MPcRpwdDo7O6N7cUmVshacQfQGwZaZ7XKiWAvQC21Szv6r48C5Hh0dDedvY2Mj6KO2trawDpw/eJfKiURvqJ5TfUR7Jm1CrmdEHUtNCRIA0BNN9QaoPNdQoTfUodSqUpwsNb6Tk5NRUQQ8P1DvjY2NEGChj9AdPIMg5MCBA0Fn+FsUmLvey619CTWVxnfT/oaKS4IuQAPVddoyiDt8a2pqos4MegZxRLidiKbhoFh6Bn2PNlp1oDfUXmmRHnKKPmIPkWkQPhB+UHL0Butw8ODBACQoSr6yshLOG/poaGgo4lprIZPy0rQXH/qE21K2traCMzk+Ph6tBXqD82dmEaqPPoL3hj4iIOf8oY+QO94DagcVudAtVB9pU3Et7lIUlTnzDlR4Dw0N2draWung3cp48MEH8y9+8YshLcvB1ogRRIEedJAhtcwZ4VAERCtL9UDzpyIritqpgkNxKIlW+7Lpd6LsFxcXg8MArM73fvWrX7Wamhr70Ic+FIoQ4FpRoTgzMxMpZAy4CjAtNtrb24NzodfdNDQ0RP3DtNcWa6GoBEic9iPCIerr64sa1CrPSVPR7N3sTl8tFDF9jnzLBgjQoEogVFpkw/tTpIKDB8eDFBTGSInzvJM2ztV5aloe5QQqMzs7ay0tLfY3/+bfDOsK0ooDxNU3pJ34XlqWzM7OhrRzXV1dUGTqyOqVOoqu+nUlnYWB4zup9tN1JfWhpGaQZk3taZsVvfZPrzhinnqhuVZWpxqronAnJyejnoHKFdMgDCWPIsdIkHLDsGl6eXt7OyhoTYejyEmhtu505AeV1pQP36sp8e3t7RAYaIsO/V4NONAxisqp8eEckGJTJ4rvRN/gRMF54jYJbezKM0D4Nzc3g37p6uoKHDO+E+eMPWP9JiYmArIFRwtHlcBBCeeKtBB40JDWzAIySfXlhQsXgtM+NjYWeHG0cgHB4nsPHz4czq62wZidnY0QM1Aczt38/Hx05RjrqegQt8e0traG/VIeH4jTxYsXQyp9bm4unJ+mpqZQ4ap8L9KiTU1N4frJ+fn5qGoT9Ik0N8VdBJeKZIE+dXd3RzoRm3DlypWANGEfx8fHgz5WasvBgwft2LFjduzYsSBrvb29UVYLuzI0NBRQPWwOGQwuFNCqaP1e9I32D9TzdeHCBTt//nxwpMfHxwNKXVlZGQX0+r3oBPQMHD3sIjJG9TDcVz27nIMitBdZ4IwNDQ1F3EjO4NramjU0NNjExMT+ucnipRrHjx/PP/7xjwfoF6Wt3dJRcKA/Owsd0nJUWCohFVSJ9gLKeeA7EBatLtO7bUn/4kCos4NjotedKLeMQwJSBR/p4sWLVlVVZW95y1t2OTukUxsaGqKGxalqR62SUnTN9yEjWiHlp2k+5dqw1qBfykGCPwU/SEnoauyB4zUNTsNLuEcYe7iNyvkAncIoa3d2LjTX78WRUkK+kuX1eznYFKuYWUByQXz43tnZWTtz5oxtbW1Zd3d34HlhSNXZwfHVS9GZqzZmVsNB+lv5o1qFrH0JaS/A+iqPSTkvpJdBnaj81lQO/CgQE5w6OIb6naOjo0F2V1ZWQnDT2NiYdNRJ/x44cCBUp1HBhvPAB6WJo076X/t3aWNy0C5oBbOzs+HuSDUcU1NToYimpqYmyJOiqyoTpE+pWgZRRN8ojwv0EeNBM1ZNXXEumMP6+nqERGE8kDNQW4ySoi44E1rtrjejXL16NcwTg0RwSUoMvimpR4zc4cOHra+vL+xXnucBoRweHo4cNPQuTkSWZcHxU6OsTiX6c3t7O+yNph0V8QPNqaioiIz94OCgHTt2zA4fPhzOm6bo2fsrV67Y+fPn7dy5cwGBmpiYCK2NWFf2iXTmoUOHomb0nB/en5Tj+fPn7fLlyyH1v7m5GQIyihMoUMCR0MI7rUJlntriBG7z1tZWhIAPDg7a4OCgHTlyJMguVJDKyspgZ4aHh+3s2bPhe5E3KrCzLLPu7u6wN3fddVf4XtZVr5xUBPLs2bPBoaRAhjNLqxrWlXTukSNHIv4otoaAgnXFodZm66xrd3d3WEs+hw8fDrSgmpqaiJ7BHEnxA95gxwA9+F7kSsEb1tUHKnzv2NiYPfXUU6WDd6vjgQceyB977LGgZIj+FcGDf4bTQdqNKFAjYW3kCdcHbopGwGNjY1FVqKZr4JCACHV3d1tVVVWYM4p/bOz6hcUgLNPT02ZmobO5Tzt+7GMfs+rqavvN3/zNcK2R8g0UucOhBdWBpwTJFsWNU9DV1RVdYq5Eb4yBtgEg1VhXVxeMNetIqlFRO9AIj4SC2sHjoACjtbU1KCftGE56A94lRhSFTXNSuDKkIJqamoJDoagtikEd41npq6UfHBYUivKRCAx6enrsW9/6ltXU1Njb3/72cPBB/DTgAElgf7Isi5pJk87QwEBvziDNpQgzSAqIrRb66PexZ6QR6WNFoQyyqS1KFPlScrw6QHDh4BOur68HlEcdK9rygCgyT3VUtO0PnCBQCU2JXLlyJeLSgQhw0beiXocOHYoKS1DkS0tLu1J7Q0NDoXp2cXExutVA9QbrScuU2traUCmr/dO0GSwp6u3t7VDM5KtBtdALFBEHFZSDeerVctoahQpQzSjAo6yqqgpOr0+l47BT0WsWp8To9wXdpKOjI9yCAsKDU+6vhlpcXLQ8z4OTiMOrlBDS3PX19SHlD4qusklhAc5bVVVVKBxANkH41ImGP8p3aKqRYHVubm5X3znN0NCrjfeAVzY/Px/0Ec4oSC/OkLZ+4fzQJ4+iCoLPxcXFqEqUeRJkLiwsRBxfXUdkVK8r1C4NuucEVHCe19fXI4QXm6E9A5WWoQ4Oeg797m2wz6IxZ+3+QMbBAyrIP2c3z/Po5iGljzBPgvPt7e2IlqKyRIBuZqGwgjS79jSk6I0WYfSt9DZ4ZGQkFOqcP3++TNHe6jh58mT+O7/zO6Hcm6rLVCpJr3CCA+WrxKi8UididXU1IiJrFZRWKnJVE5G/NgVVsj7og1a1EpFjTKgGJD3Z0NBg7e3t9vnPf96qqqrsX/yLfxGQH5AE7d2jBRXanoLfVUMF2kNqGgI2BoXecxRR8AzmCNeJtJ8WO2hxAg6MtuhgbuwL0SgGBYRPezHBW+QeUpxSkCo4M8gCKKneP0vqkzkTrYP2VFdX72r9gXxpUQLIhKYSq6qq7OzZs1ZVVWWvfe1rA3rFc/heLSzRljK6HswXrqGirUSk8InMLKCA2rwYQ6XtarS5N0Uf2niZIIfonHXEGaB6kcbOeh8u+0+xA6mo1tbWcNcy746MatU7aAfFGRCVqaJTdFkbRFPUoc3NCaagKHjeLYEeDgVnt66uLuypBo4alGjLJWQdHq8aWOXEVlRUhHfGuNCkFT2FUSd9jIEBjcDhpYAHOdS7NhXtQ0dVV1eHoATURBEeUlpwjXnnixcvBvSIdYVEDppO4AAx/fjx44H72NzcHHTG1NRUaGEBcnLx4sVQ+LWyshJVkN5111121113hfTuwMCAHThwIKR2NVV+5swZO3PmjF28eDG66QD56O/vDwUQinChA7MsCwHOlStX7MyZM3b27NmAyF25ciXqkMDeHj161E6ePGknT54MDm9fX1/QRysrK+E9L1y4EOZJUD87OxvSrs3NzXbixIkwP+YLn6yuri7wIsfGxuzs2bPh+3AsZmdng45pa2sLqdvBwUE7ceKE3XXXXWGNGxsbg06fnJwMRR/nzp0L+478rK2tBbvW19cX0DzSw6CE2E2lS4ESXr58OciV3tGtKLGmWgm+cfBJjSM/yDvOHlxF9MOhQ4fC9xGEdXZ2RlxCgmPQbOSHAiFtQ8Z7EoAdPHgw8B5BHDUlrs7jE088YUtLS6WDdyvj7rvvzj/1qU8FxwmlqygRhx2OjNl1Ujd9fLR6iPQpxpCIWfs+AWNTmQQyRuoBNINIktL2ra2tiB+lcySlpVeD6fU0FDlQFl5VVRU4gsrnA3Wg7xUOG21JfESCowDqQL89TYUQJePMmFnkwOo7g76ZWXBelGPG+hHZ0LIG5EeRQBQR3DXSTNp0UknU3O+orViIEDV1p7wi1koLcuCWcVcxzhDoGvxCeGDa7gIHm7YiqTQgldyKNPDOpIRBl3CwNV3JWmq6klSldzK0qz36gUgbtJt95vzQfBQETPmUrKEif6whMn3lypXofmZ1pjU9yxrSzBcUldSLOlTIIAVSkKFZQ5Cqvr6+KM2DE65tcEDpRkdHo3t7MSjINMgxck7PQopT/DmG1zU/Px+Qn/r6+sDx1MKDlpaWELxwaT3IsX7f5OSkrayshLQ8LXrgOHrko6amJjj3/hynUDltIKxkcnSamYWgTNPlrCF3t66uriYLOehbpsUyZhZdd6UItN6ggnPW2toadCBr2NvbG91kQ+Ch51j75xGAZVkWtTjROWqlLpkRLapgDemcAO9W0/noQr3jGeeDc6zrp4Vuq6urUVNk9kMb5BMkZlkW5EP5ZdpqJM/zQK/RbBBr2dnZGYJbOIs0P+b74K7OzMxEd05DV1CkUHuwErBij/kgg5xj7CfBoPI/QdrpE6u3TJBi5jsJ8MwsumCANdTCKtZvY2MjZIAIuJBp5JAuE1qQo1ffKYVJwRVF8iYnJ0sH788z7r333vz3fu/3QnduBN7z4zCEREG+YzybD2KA0QK1U6I6joSmKHEWtVEyBRA4Ykp85tBgyCC5wn8C/dCmyGpQtfmiNlkGuaHHnzZhJUJEIEHVQBOXl5fDu3FbAIYPA51qS8KVO1RmgkChtJaWloKjsrW1Fd6DFh8YQQ4JRpQUHxVs/KnNkisrKwPyBpIF0Zw1QInQ1Jfvwmhx2PWaIb4LTpq2q8CxZ820T5bZddSRKj3aofB9vBt3OiraRvUgBlhRO/ZOe23RIoFGpdq2hshbuYvsG6lldaxQ8hRV6HViOHDr6+uh1YJeJo7Mt7W1hZ/zjihlvdILpI49UDlVRJ2ARlOU2ptPkXqVVxxkAi4UPEaEK+c2NjYig6tXQ1FAVXQVETqjqakpyCMNgEEEIMnTMmRzczOcxZ6ensAVwjHt7+8P6eXNzc0IXYDTNT4+HgyxOlQgFXDaQBdA2ggCh4eH7fz584Ejxd5osVhvb29AkOCHHT16NDikeZ4HqsqlS5cCiqT3eNbU1IQzwHfR6uP48eOh80F9fb3NSl/R06dP2+nTp+3s2bPBAaIBbk1NjXV3d4fvAOEbHBwMGYmKioroSq4XXnjBXnjhhVAEcOXKlcDRra+vD+92/PhxO3nypJ06dSqSPb296Pnnnw/vCkpKSi/P81BQcvjwYTtx4oTdfffdNjg4GF14j+MwNDRkp0+ftjNnzkTcOkXDcRaPHTtmp06dspMnT0bVnPBV5+bmwvfwuXDhQnRtF4FkX19fQBwHBwdDsACoQbUwaB7o7eXLlwOHE24eKNmJEyfsxIkTERih99ZqY2C+c2pqKug87eeIjBw5ciSciQMHDoQzhjOriNuVK1eC/iPwIyiAN6cBqra64vxTMHP58uUQqM3NzUUtqLQIibXklpmKioqo7RKI+sjIiM3NzdlTTz1ly8vLpYN3K+Ouu+7Kf/M3f9O6urqiajfl+gCTUlzB9SvKfwAexklramqKDIpyXYCFcRbW19ejKJjvAj3p6uoys+tRMIiJcsa4umV5eTmkt9rb2+3QoUOB3NzV1WV//Md/bLW1tfYTP/EToTdWam44bXATKK0nzaMNl2myur6+HrV+4bv0ijcuhm9vbw/vSck790TynkTSypOiMhASL04GKSgI50RaOGarq6u73hOkEsPObSHt7e1BySqnpa6uLqqGVPlgntq8k++CZH3kyJGoapOofHl5Oewp83r22WdD5d/Kykpow9PR0RF1rOe79IJ439LgypUrgfNJ2pIAIDW3tra28J4LCwvJPVUuofaNQ9YwIrQR0XOlskulI+tGG4eic9XS0hLec25uLmpjwfcRiUNwhjbBusGrIRJnbipr2raB91Skheq4w4cPhz3QM68RuH4fsru5uRlkV88VqEhnZ6eZWaAHpPYA0rme+Y6OjkA858wTLLJucA/1u65evRrmxpnHoGvBBd+nN87o3Ip6m6HfUmeeNkGgIewD+3n16tXo6jQCVz3zOjfec2VlJToHyjvTM8+5oncZ54q2L3yfnnk1vnrm2VMqadFHfJ8icX4POFecU+Vl6x2uWuHJ9yl3S2/vQHb1XPX39++aW2tra3hPPfMquxq0Ix9F9gpbur6+Hpwf/b5bsaWKYBbZUmT3RraUM0/mLGVLCVb0zNPjjrOlKWreEz6i6hBsqZ55b0v1zO9lSx977DGbn59/UQ5e1Y1/5c4aEHTxxLW0HIInToIWA8AXoQwedAvkQ6uhIK6T2iBVon2T4DkAoXOn4NTUlH3ve9+LSqhJJWgqkfQSkTzFABBDaWT5+OOP2/r6eoCn+R7SGW1tbRGvhvQX0SPR3OTkpH3nO9+J2rPQlkJL2w8fPmyve93rwntSjHDgwIHoyqXvfOc7IRUJ4kM1Yk1NTYj0Xvva1wbEtK2tLaCI2tvsu9/9btSjD1QL5BAl9+pXvzo0kW5ra4sQI9CAp556KqrqpEKxoaEhcLr6+vrsla98pfX09AQEg7YzcFNGRkbs29/+dtTcGpSwvr4+RME9PT123333WW9vr9XU1NhDDz0U0sCkK4eHh+273/2u/dEf/VHUz49Ikaq1gYEBu/vuu6MrhbgPWautn332WfvjP/7joCA3Nzd3pd0OHjxoJ0+eDHKurU+UZH3mzBn75je/GebEHY5akHTw4EH7kR/5kYC+mFnY85GRkXD+Lly4YE8++aRNT08HBLOpqSnisfT399vrX//6KArW9i5aMfjss8+GCk/Sh6AbR48etZ6eHnvNa15jb3rTmyKDwnz0iqTnn38+rJWm6kCZ+vv77f7777c3vOENUaNz0AdaLFy5csWee+65sFYEZTSmhUN16tQpe/WrX21VVVUBXYKbBH+KvQSNbmxsDN8Bknbs2DG766677J577rHq6urIaTlz5oydPn06tAG5cOGCbW9vByTqyJEjduLECTt58mQwdlQuIleKsMBBY1+XlpaCjPf399upU6fs1KlTduzYseBIHj161O69915bXl4O33HmzJnwnVAfqCZFDninU6dOhfPd0NBghw4dCg7t888/b21tbXb+/PlIjw4NDVlNTY2NjIzY+Ph44JrBm6KRNj0KSWebXbs9iMwMqHVDQ0O4kYg+bu3t7aFZNvxkilToN0c2BB4f6Pz6+npoH5XneWilRAsiEHboNKSH+Y6lpaVwXRwZgNra2pDWBJnmd8mUkNHCQVtZWQloP1e4UXCH80lrIAqZ5ufnQ2GY5/mR7YLSAHoMuqr3rJPavnLlip0+fdrOnTsX0afIUnR0dETnhkCHW3bMLOolSeNlrVTO8zw4mwMDA3bXXXfZ8ePHgw7r6uqKbtNAJ9BGB0ed3+nu7g7/njM4MDAQaCB5ngedrG1zsGGktsnwvJhxRyB4WZa9xcz+vZlVmtnH8zz/YNHvHj9+PP/kJz9p/f394b5KyuDxwHHUiIiWl5fDtSqgFginXrDO7wL1X7p0KUQyV69eDYfF9+PBswcVgMhKbl5hZXg82qJC27VoA9Wamhp7z3veY3me26/8yq8EZwEBnZqaCiR4Uqmtra3B+YRI3tvbG64wyvM8NKv0FUV6CwEGlSqiQ4cORe1OtMO38i0QcpCM7e3tsO7aLBd0p7m5OSBPtHPA+YCDRUNRz0dS3pkqF71aTiNQOGI4oVolp1wLLqGmYS2pRiX4AuebXeN3zs3NWUtLi73hDW+I0nlU8HG9E+kB7X9GgQGRNvdtYpSVZE8R0Nra2q4eagQAGBF4p6w70SwFNlTXQSPAIWLtx8fHzcxCM2WtIkUmKFCADE77EEjmVGKTOiLFr0UE2oAXx2l1dTVKfdJ3i95+S0tL4boxjA3tLHDg6GkIasU51IIEnD7SlCh10kbsB04AXEs1NKw9gVJ1dXVAXPiuo0ePhivcWHcCkXPnztnZs2ft/Pnz0XVZBDgYLdJYrL3SGtTJPnfunJ05cyZUlS8uLgbnv7e3Nzh9x44dC2epqqoqUBAuX74cSPLs48jISAg4GhoagiE+fvx4SNdBLWloaIj6aJ4+fdpeeOGFKHjG8amtrQ0OgqZ1u7q6Iq6dFlaQziWIWltbC/t08OBBO3HihJ06dSoEvgMDA4HPtby8HJxPUpFnz54N6Orq6mrgaff394fUJoUPINx6IwNOrbafIXhpbGwMAQkp5sHBwbBOBw4cCHIwPDwcUriXL18OnF5a2PBdODCKSHNuZmdnA1VA07ekMzc2NgI6efDgQRscHIyKRXp6emxzczOkgrVAhvMzOzsbdAxZBd+qpXXn1iguIWBerDkBzvj4eDgzpERB8aAzKOdNm7+zh2R2pqenQxNuAktAHXh+7e3twQ7Oz88HW6O9CHGcNzY2ImScfVR0cWtrKyCbgEw4o1/+8pdtcXFxf6dosyyrNLPTZvZGMxsys2+b2U/mef5s6vcfeOCB/HOf+1wgAGvET3UaXe3r6+vDxqpTBypVV1cXjDeK9vz584HsDGoDCRZvngodbr3A85+cnIyiAz44oVRiYSD5LqLFAwcO2OxO373x8XE7f/68ffjDH7a1tTUbHBwMDVsh1/MdRBt0p6eCFOdSI5XLly8HuNvMIsIsSgNkq7GxMVTCTU1NhfW5dOlSMESKtPX19YX5KDcDZbC0tBStjUZP8KO0E7y+G+9MOxfWmkMOSrK0tBQ4ZkRix44dC+hmf39/IM2vra2FQ8m+gRSACFFBpu+mrVa2t7dtenraPv3pT9v09LT19vbahQsXgnOK8020SiWjOjZ616RewUSgMjk5GdJrFKX09PREvaRwlumlRSqf9UGZTkxMhBYPzc3NQWnpfGhTAe9KA6eLFy+G9DEcV/hzR44cCQYaDlt9fX10nSDzgcA+MTERUNEDBw5E/cG0GTlDuS7qRGKcq6qqQtTOfDDOGB34bHoVk7bgwPmoqakJc6GqE6d2R3eFdSaSxxASLG1vbwcEW1E+2su0t7eHqueZmZnIcQS11YbtXGSujl53d3dwKBYXF8N8OB+XLl2y+fn5UFnvr60aHBwMTbTb29ujRt84QyAUo6OjtrS0FIKknp6e4MgqXwnOG841gSnOwqzcekOqtre3N6zPwMBAQKVBGxVxvHDhQuBRgi6BvFHlqc2A4QFvb28neY7wwwgcMOA4nYcPHw4BLmg/qVG9fmx4eDgqMqKCGYefD7zhysrKiAbBd1GUsLCwEHQH+oNKVqWiMPeRkZGwPhr48az6+vpoLugivZcWBwruJu8GYqbpY87Y4OBgRHcCzfR953hPMwtoKXvEnh05ciS6+1iLpZjPpUuXwrkxs4gmwp5BA2hvbw9Zr5mZmWg+oPJ6e49encd86P/H/dTMh8rjixcvhjOzvr5ubW1tONb73sH7H8zsA3mev3nnv99vZpbn+S+mfv/EiRP5b//2b1t7e3uIyBYXF4NhVEXN/YFVVVVRxRPkSzpqQ8zHsdJqJ0VX9I5KnAUEWlMACDYKmuhieXk5XDINhA+CSPqQQgGza411/+W//Je2sbFhf+/v/b1w3ROoBBcbc+k7KA8/g0sFTwMCPdHcwsKCmVloIaIkej6kJaqrq4PRojiD/lZEMGYWFJveRaukfi1eQAkwH4ozKDigpJ5Un/Z1ouGq9o3yd9pyjyvfRXEH5H69R5NrhEATOaz0RiRVmGVZSFPwaW1tte9///tWXV1tb3zjGwMawlwgbFMpuLy8bGYW0hQgdkodaGhoiAogQMZwbqempkJ/LeUS6QXa3JtaV1cXXXEFykr1HcUArI826tZiCnXU9Yxos1SqXbVyFmUJZ5PmxrwPhh8uGJwrZBFnVgOrnp6eqOJY750kiubu2jzPra+vL0KdIGFTCLO2thbeQ1ONVNctLy9HvCNQIvhkvb294V7lhYWFUDgAmZ6UEuejv7/fBgcHQ9EAwZW2LSHoOHv2rD333HN2+vTpUKi1uLgY9em699577Z577glIIWkpdM9zzz1nzz77rD3//PMBYdDgo7+/3+6++267++67A8o0ODgYHP2VlRU7e/ZsSDE///zz9sILL4RWVCsrKyGQJm1777332vHjxwO6BpVidHTUTp8+bU8//bQ988wzwdmf3bkNpqWlxfr7++2+++4L34HjT7ud5eXlgAyy1uwXSKMGZXfffbfdc889Njg4GPQRnC7S3c8++6w999xzUUU9+rm/v9/uueeeMB/sSJ7nwbk6c+ZMmI/eoAHFp6enJzhDWixCAdrW1lYI7i5cuBDei44QCwsLIUVL4QRIJefUzMI5v3jxYpQepVoUvYrDSNCJ08edxVy3lbq9g4CKIqnu7u6o4EevZwP5BimjLQnIvt6LDt1Br+qjCDLP86izgL4TAAGZOpBFvZ2FXnvod70JS28Tosr4wIEDIRuAfe7o6AhFR/TiRb+jl2d3ri/95je/+aI5eHeCg/c/mdlb8jz/6Z3//gdm9kN5nr8v9fv33HNP/qlPfSpcz4MzByLEtTw4UxxQvY6GSL6pqSnwOiYnJ4OBII9O7ycQKo3AaGhMs8vV1dUozaXtBZS7hQCDloCYwDeAa8fB+uxnP2sbGxs2ODgYUkkYT1J4R44cCU6PltIrwjk5ORkEm6pdIjCKE3ASsiyLIlTWFv7E3NxcOJRKEtd2BmYWuJCkgeFIQgA2u2Z8NSVC9NTX1xccS9Ba7ipkjXG+19bWojSwRl2grzQznpmZia4F4oCCRNIcGVSip6cnpAFx9LleiAhwdHTUnn766YDWVFVVRY2VQUjpY0YqRZEE9huHHCQK/hP7RA+m+vr6cIUUvcsuXboU0scLCwuhOIPUlUbHcIJARvQqKnpIamUk6BqyglPDPaYYby2uoeUCst3Q0BA1NWavtXJZU9js89TUVCBlw/9RIrteOWhmAcFUXi2IPE4EZwDHQQs5cGhWVlaiyli9WpAz7RsVY1CQbV/UwPpoL0GtImafteKU65G4pQfZ04p8gju+4+jRo6H1SVtbWwhSQPVBZfS+T/ZZC3n0hhD2GXK6vw5ta2srVKfrlWo4+FwMX1VVFaEyqjMViWU/KDxD/sl2wH3z8s8+U3Hc1NQUoZ6KfIHsYEP4DhB4HO7GxsZdXNL+/v7I2GuhnxbWsM+0rdI1OXToUNClZhZABqXiaKq9ubk52DTeiTZBtJYiMFXHjL6TNO+mz6QWNYA0Uziwuroa5J99vnr1ariP2MwCf1gbN3OLCN0ksIWqu7V4CVuK7OKwkpaGnzg9PR10FGnZ+fn5wDGkVQ/rwj63tbUF/jEcPN4J/4ErB6urq8M+Y9NocQTVhlZEvmCPbgM0S/7Wt75lExMT+97Be7uZvdk5eK/L8/wfy+88bGYP7/zn/Wb29Ms+0b/8o9PMJm/3JP6SjXJNdo9yTdKjXJfdo1yT9CjXZfco12T3OJXnedONf6143AlVtENmdkj++6CZDesv5Hn+iJk9YmaWZdmTeZ6/5uWb3g/GKNdl9yjXZPco1yQ9ynXZPco1SY9yXXaPck12jyzLnnyx31HxUkzkNo9vm9mJLMuOZVlWY2bvNLMv3uY5laMc5ShHOcpRjnLctvEDj+Dleb6ZZdn7zOwP7VqblEfzPH/mNk+rHOUoRznKUY5ylOO2jR94B8/MLM/zr5jZV27y1x/5i5zLD/Ao12X3KNdk9yjXJD3Kddk9yjVJj3Jddo9yTXaPF70mP/BFFuUoRznKUY5ylKMc5YjHncDBK0c5ylGOcpSjHOUoh4x95eBlWfaWLMteyLLsbJZl//x2z+d2jCzLDmVZ9v9lWfZclmXPZFn2v+/8/ANZll3Nsux7O58fu91zfblHlmUXsyz77zvv/+TOz9qzLPtalmVndv5su93zfLlGlmWnRB6+l2XZfJZl/2S/yUqWZY9mWTaeZdnT8rNCuciy7P07OuaFLMvefHtm/Rc/CtblV7Isez7Lsu9nWfb5LMtad35+NMuyFZGZ37htE/8LHAVrUnhe9rms/J6sycUsy7638/P9IitFtvgl0y37JkWb3eKVZnfqyLKsz8z68jz/TpZlTWb2lJn9XTN7h5kt5nn+ods5v9s5siy7aGavyfN8Un72y2Y2nef5B3eCgrY8z//Z7Zrj7Ro75+eqmf2Qmf0vto9kJcuyv2Fmi2b223me37/zs6RcZFl2r5l9ysxeZ2b9Zvb/mtnJPM+3btP0/8JGwbq8ycz+ZKf47ZfMzHbW5aiZfYnfu1NHwZp8wBLnZb/Livv//9bM5vI8/4V9JCtFtvgf2kukW/YTgvc6Mzub5/n5PM/XzezTZvbQbZ7Tyz7yPB/J8/w7O39fMLPnzGzg9s7qL/V4yMx+a+fvv2XXDuB+HD9qZufyPL90uyfyco88z79uZtPux0Vy8ZCZfTrP87U8zy+Y2Vm7pnvuuJFalzzP/yjP882d//yvdq0v6b4ZBbJSNPa1rDCyLMvsGsDwqZd1Urd57GGLXzLdsp8cvAEzuyL/PWT73LHZiZT+ipl9a+dH79tJrTy6n1KRMnIz+6Msy57Krt1+YmbWk+f5iNm1A2lm3bdtdrd3vNNiBbzfZaVILko9c32828z+QP77WJZl382y7Iksy/767ZrUbRqp81LKyrXx181sLM/zM/KzfSUrzha/ZLplPzl4qTvd9kd+OjGyLGs0s8fM7J/keT5vZr9uZoNm9kozGzGzf3v7Znfbxl/L8/xVZva3zOxndtIK+35k1xqI/x0z++zOj0pZKR6lnjGzLMv+TzPbNLP/tPOjETM7nOf5XzGzf2pmv5tlWfPtmt/LPIrOSykr18ZPWhw87itZSdjiwl9N/GxPedlPDt4NrzTbLyPLsmq7JlD/Kc/zx83M8jwfy/N8K8/zbTP7f+wOTRXsNfI8H975c9zMPm/X1mBshysBZ2L89s3wto2/ZWbfyfN8zKyUlZ1RJBf7Xs9kWfYuM/vbZvY/5zsk75200tTO358ys3NmdvL2zfLlG3ucl1JWsqzKzH7czH6Pn+0nWUnZYnsJdct+cvDKK80s8B0+YWbP5Xn+7+TnffJrbzOzp/2/vZNHlmUNO0RXy7KswczeZNfW4Itm9q6dX3uXmX3h9szwto4owt7vsrIziuTii2b2zizLarMsO2ZmJ8zsv92G+d2WkWXZW8zsn5nZ38nzfFl+3rVTqGNZlh23a+ty/vbM8uUde5yXfS0rO+N/NLPn8zwf4gf7RVaKbLG9hLrljrjJ4mZGeaVZGH/NzP6Bmf13ytLN7OfN7CezLHulXYN8L5rZe2/H5G7j6DGzz187c1ZlZr+b5/lXsyz7tpl9Jsuy95jZZTN7+22c48s+siw7YNcqz1Uefnk/yUqWZZ8ysx82s84sy4bM7P82sw9aQi7yPH8my7LPmNmzdi1F+TN3YlWkWeG6vN/Mas3saztn6b/mef6PzOxvmNkvZFm2aWZbZvaP8jy/2WKEH5hRsCY/nDov+11W8jz/hO3m9prtE1mxYlv8kumWfdMmpRzlKEc5ylGOcpRjv4z9lKItRznKUY5ylKMc5dgXo3TwylGOcpSjHOUoRznusFE6eOUoRznKUY5ylKMcd9goHbxylKMc5ShHOcpRjjtslA5eOcpRjnKUoxzlKMcdNkoHrxzlKEc5ylGOcpTjDhulg1eOcpSjHOUoRznKcYeN0sErRznKUY4/x8iy7L9lWfaZLMt+Icuyc1mWre5cKP+jt3tu5ShHOcpRNjouRznKUY5bHDt3aC6Y2baZfcvMftWu3ZDzb8xswMwG8zyfvG0TLEc5yrHvx765qqwc5ShHOV7Cca+Z1ZnZ183sjVwZlGXZtJn9qZn9dTP7/G2bXTnKUY59P8oUbTnKUY5y3Pp49c6fP+/ug3x+58+Ol3k+5ShHOcoRjdLBK0c5ylGOWx+vMrPhPM//zP28f+fPoZd5PuUoRznKEY3SwStHOcpRjlsfrzKzq4mf/4SZLZvZN17e6ZSjHOUoRzxKDl45ylGOctzCyLKswsxeYWZLWZZV5Xm+ufPzfjP738zs1/I8X7qdcyxHOcpRjrKKthzlKEc5bmFkWXavmT1jZlfsWpHFJ83soJn9X2Y2ZWZ/I8/z1ds3w3KUoxzlKFO05ShHOcpxq+NVO3/+mJm1mtnvm9kvm9lXzOxHS+euHOUox1+GUaZoy1GOcpTj1sarzWwoz/Onzexv3+7JlKMc5ShHapQIXjnKUY5y3Np4lZk9dbsnUY5ylKMce43SwStHOcpRjpscWZZlZvZKKx28cpSjHH/JR1lkUY5ylKMc5ShHOcpxh40SwStHOcpRjnKUoxzluMNG6eCVoxzlKEc5ylGOctxho3TwylGOcpSjHOUoRznusFE6eOUoRznKUY5ylKMcd9goHbxylKMc5ShHOcpRjjtslA5eOcpRjnKUoxzlKMcdNkoHrxzlKEc5ylGOcpTjDhulg1eOcpSjHOUoRznKcYeN/x9nCNXqAA12hAAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], "source": [ + "# extra code – this cells generates and saves Figure 16–9\n", + "figure_max_length = 201\n", + "figure_embed_size = 512\n", + "pos_emb = PositionalEncoding(figure_max_length, figure_embed_size)\n", + "zeros = np.zeros((1, figure_max_length, figure_embed_size), np.float32)\n", + "P = pos_emb(zeros)[0].numpy()\n", "i1, i2, crop_i = 100, 101, 150\n", "p1, p2, p3 = 22, 60, 35\n", "fig, (ax1, ax2) = plt.subplots(nrows=2, ncols=1, sharex=True, figsize=(9, 5))\n", "ax1.plot([p1, p1], [-1, 1], \"k--\", label=\"$p = {}$\".format(p1))\n", "ax1.plot([p2, p2], [-1, 1], \"k--\", label=\"$p = {}$\".format(p2), alpha=0.5)\n", - "ax1.plot(p3, PE[p3, i1], \"bx\", label=\"$p = {}$\".format(p3))\n", - "ax1.plot(PE[:,i1], \"b-\", label=\"$i = {}$\".format(i1))\n", - "ax1.plot(PE[:,i2], \"r-\", label=\"$i = {}$\".format(i2))\n", - "ax1.plot([p1, p2], [PE[p1, i1], PE[p2, i1]], \"bo\")\n", - "ax1.plot([p1, p2], [PE[p1, i2], PE[p2, i2]], \"ro\")\n", + "ax1.plot(p3, P[p3, i1], \"bx\", label=\"$p = {}$\".format(p3))\n", + "ax1.plot(P[:,i1], \"b-\", label=\"$i = {}$\".format(i1))\n", + "ax1.plot(P[:,i2], \"r-\", label=\"$i = {}$\".format(i2))\n", + "ax1.plot([p1, p2], [P[p1, i1], P[p2, i1]], \"bo\")\n", + "ax1.plot([p1, p2], [P[p1, i2], P[p2, i2]], \"ro\")\n", "ax1.legend(loc=\"center right\", fontsize=14, framealpha=0.95)\n", "ax1.set_ylabel(\"$P_{(p,i)}$\", rotation=0, fontsize=16)\n", "ax1.grid(True, alpha=0.3)\n", - "ax1.hlines(0, 0, max_steps - 1, color=\"k\", linewidth=1, alpha=0.3)\n", - "ax1.axis([0, max_steps - 1, -1, 1])\n", - "ax2.imshow(PE.T[:crop_i], cmap=\"gray\", interpolation=\"bilinear\", aspect=\"auto\")\n", - "ax2.hlines(i1, 0, max_steps - 1, color=\"b\")\n", - "cheat = 2 # need to raise the red line a bit, or else it hides the blue one\n", - "ax2.hlines(i2+cheat, 0, max_steps - 1, color=\"r\")\n", + "ax1.hlines(0, 0, figure_max_length - 1, color=\"k\", linewidth=1, alpha=0.3)\n", + "ax1.axis([0, figure_max_length - 1, -1, 1])\n", + "ax2.imshow(P.T[:crop_i], cmap=\"gray\", interpolation=\"bilinear\", aspect=\"auto\")\n", + "ax2.hlines(i1, 0, figure_max_length - 1, color=\"b\", linewidth=3)\n", + "cheat = 2 # need to raise the red line a bit, or else it hides the blue one\n", + "ax2.hlines(i2+cheat, 0, figure_max_length - 1, color=\"r\", linewidth=3)\n", "ax2.plot([p1, p1], [0, crop_i], \"k--\")\n", "ax2.plot([p2, p2], [0, crop_i], \"k--\", alpha=0.5)\n", "ax2.plot([p1, p2], [i2+cheat, i2+cheat], \"ro\")\n", "ax2.plot([p1, p2], [i1, i1], \"bo\")\n", - "ax2.axis([0, max_steps - 1, 0, crop_i])\n", + "ax2.axis([0, figure_max_length - 1, 0, crop_i])\n", "ax2.set_xlabel(\"$p$\", fontsize=16)\n", "ax2.set_ylabel(\"$i$\", rotation=0, fontsize=16)\n", "save_fig(\"positional_embedding_plot\")\n", "plt.show()" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "embed_size = 512; max_steps = 500; vocab_size = 10000\n", - "encoder_inputs = tf.keras.layers.Input(shape=[None], dtype=np.int32)\n", - "decoder_inputs = tf.keras.layers.Input(shape=[None], dtype=np.int32)\n", - "embeddings = tf.keras.layers.Embedding(vocab_size, embed_size)\n", - "encoder_embeddings = embeddings(encoder_inputs)\n", - "decoder_embeddings = embeddings(decoder_inputs)\n", - "positional_encoding = PositionalEncoding(max_steps, max_dims=embed_size)\n", - "encoder_in = positional_encoding(encoder_embeddings)\n", - "decoder_in = positional_encoding(decoder_embeddings)" - ] - }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Here is a (very) simplified Transformer (the actual architecture has skip connections, layer norm, dense nets, and most importantly it uses Multi-Head Attention instead of regular Attention):" + "### Multi-Head Attention" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 84, "metadata": {}, "outputs": [], "source": [ + "N = 2 # instead of 6\n", + "num_heads = 8\n", + "dropout_rate = 0.1\n", + "n_units = 128 # for the first Dense layer in each Feed Forward block\n", + "encoder_pad_mask = tf.math.not_equal(encoder_input_ids, 0)[:, tf.newaxis]\n", "Z = encoder_in\n", - "for N in range(6):\n", - " Z = tf.keras.layers.Attention(use_scale=True)([Z, Z])\n", - "\n", - "encoder_outputs = Z\n", - "Z = decoder_in\n", - "for N in range(6):\n", - " Z = tf.keras.layers.Attention(use_scale=True, causal=True)([Z, Z])\n", - " Z = tf.keras.layers.Attention(use_scale=True)([Z, encoder_outputs])\n", - "\n", - "outputs = tf.keras.layers.TimeDistributed(\n", - " tf.keras.layers.Dense(vocab_size, activation=\"softmax\"))(Z)" + "for _ in range(N):\n", + " skip = Z\n", + " attn_layer = tf.keras.layers.MultiHeadAttention(\n", + " num_heads=num_heads, key_dim=embed_size, dropout=dropout_rate)\n", + " Z = attn_layer(Z, value=Z, attention_mask=encoder_pad_mask)\n", + " Z = tf.keras.layers.LayerNormalization()(tf.keras.layers.Add()([Z, skip]))\n", + " skip = Z\n", + " Z = tf.keras.layers.Dense(n_units, activation=\"relu\")(Z)\n", + " Z = tf.keras.layers.Dense(embed_size)(Z)\n", + " Z = tf.keras.layers.Dropout(dropout_rate)(Z)\n", + " Z = tf.keras.layers.LayerNormalization()(tf.keras.layers.Add()([Z, skip]))" + ] + }, + { + "cell_type": "code", + "execution_count": 85, + "metadata": {}, + "outputs": [], + "source": [ + "decoder_pad_mask = tf.math.not_equal(decoder_input_ids, 0)[:, tf.newaxis]\n", + "causal_mask = tf.linalg.band_part( # creates a lower triangular matrix\n", + " tf.ones((batch_max_len_dec, batch_max_len_dec), tf.bool), -1, 0)" + ] + }, + { + "cell_type": "code", + "execution_count": 86, + "metadata": {}, + "outputs": [], + "source": [ + "encoder_outputs = Z # let's save the encoder's final outputs\n", + "Z = decoder_in # the decoder starts with its own inputs\n", + "for _ in range(N):\n", + " skip = Z\n", + " attn_layer = tf.keras.layers.MultiHeadAttention(\n", + " num_heads=num_heads, key_dim=embed_size, dropout=dropout_rate)\n", + " Z = attn_layer(Z, value=Z, attention_mask=causal_mask & decoder_pad_mask)\n", + " Z = tf.keras.layers.LayerNormalization()(tf.keras.layers.Add()([Z, skip]))\n", + " skip = Z\n", + " attn_layer = tf.keras.layers.MultiHeadAttention(\n", + " num_heads=num_heads, key_dim=embed_size, dropout=dropout_rate)\n", + " Z = attn_layer(Z, value=encoder_outputs, attention_mask=encoder_pad_mask)\n", + " Z = tf.keras.layers.LayerNormalization()(tf.keras.layers.Add()([Z, skip]))\n", + " skip = Z\n", + " Z = tf.keras.layers.Dense(n_units, activation=\"relu\")(Z)\n", + " Z = tf.keras.layers.Dense(embed_size)(Z)\n", + " Z = tf.keras.layers.LayerNormalization()(tf.keras.layers.Add()([Z, skip]))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Here's a basic implementation of the `MultiHeadAttention` layer. One will likely be added to `tf.keras.layers` in the near future. Note that `Conv1D` layers with `kernel_size=1` (and the default `padding=\"valid\"` and `strides=1`) is equivalent to a `TimeDistributed(Dense(...))` layer." + "**Warning**: the following cell will take a while to run (possibly 2 or 3 hours if you are not using a GPU)." ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 87, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 1/10\n", + "3125/3125 [==============================] - 828s 263ms/step - loss: 0.2982 - accuracy: 0.5545 - val_loss: 0.2105 - val_accuracy: 0.6476\n", + "Epoch 2/10\n", + "3125/3125 [==============================] - 820s 262ms/step - loss: 0.2006 - accuracy: 0.6601 - val_loss: 0.1876 - val_accuracy: 0.6802\n", + "Epoch 3/10\n", + "3125/3125 [==============================] - 820s 263ms/step - loss: 0.1842 - accuracy: 0.6816 - val_loss: 0.1766 - val_accuracy: 0.6975\n", + "Epoch 4/10\n", + "3125/3125 [==============================] - 820s 262ms/step - loss: 0.1748 - accuracy: 0.6942 - val_loss: 0.1704 - val_accuracy: 0.7055\n", + "Epoch 5/10\n", + "3125/3125 [==============================] - 820s 262ms/step - loss: 0.1683 - accuracy: 0.7021 - val_loss: 0.1657 - val_accuracy: 0.7102\n", + "Epoch 6/10\n", + "3125/3125 [==============================] - 821s 263ms/step - loss: 0.1628 - accuracy: 0.7096 - val_loss: 0.1628 - val_accuracy: 0.7130\n", + "Epoch 7/10\n", + "3125/3125 [==============================] - 826s 264ms/step - loss: 0.1588 - accuracy: 0.7154 - val_loss: 0.1595 - val_accuracy: 0.7205\n", + "Epoch 8/10\n", + "3125/3125 [==============================] - 822s 263ms/step - loss: 0.1550 - accuracy: 0.7205 - val_loss: 0.1590 - val_accuracy: 0.7199\n", + "Epoch 9/10\n", + "3125/3125 [==============================] - 821s 263ms/step - loss: 0.1518 - accuracy: 0.7249 - val_loss: 0.1547 - val_accuracy: 0.7258\n", + "Epoch 10/10\n", + "3125/3125 [==============================] - 821s 263ms/step - loss: 0.1492 - accuracy: 0.7279 - val_loss: 0.1538 - val_accuracy: 0.7281\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 87, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "Y_proba = tf.keras.layers.Dense(vocab_size, activation=\"softmax\")(Z)\n", + "model = tf.keras.Model(inputs=[encoder_inputs, decoder_inputs],\n", + " outputs=[Y_proba])\n", + "model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=\"nadam\",\n", + " metrics=[\"accuracy\"])\n", + "model.fit((X_train, X_train_dec), Y_train, epochs=10,\n", + " validation_data=((X_valid, X_valid_dec), Y_valid))" + ] + }, + { + "cell_type": "code", + "execution_count": 88, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'me gusta el fútbol y yo también voy a la playa'" + ] + }, + "execution_count": 88, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "translate(\"I like soccer and also going to the beach\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# HuggingFace" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Install the Transformers and Datasets libraries if we're running on Colab:" + ] + }, + { + "cell_type": "code", + "execution_count": 89, "metadata": {}, "outputs": [], "source": [ - "K = tf.keras.backend\n", + "if \"google.colab\" in sys.modules:\n", + " %pip install -q -U transformers\n", + " %pip install -q -U datasets" + ] + }, + { + "cell_type": "code", + "execution_count": 90, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "No model was supplied, defaulted to distilbert-base-uncased-finetuned-sst-2-english (https://huggingface.co/distilbert-base-uncased-finetuned-sst-2-english)\n", + "All model checkpoint layers were used when initializing TFDistilBertForSequenceClassification.\n", + "\n", + "All the layers of TFDistilBertForSequenceClassification were initialized from the model checkpoint at distilbert-base-uncased-finetuned-sst-2-english.\n", + "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFDistilBertForSequenceClassification for predictions without further training.\n" + ] + } + ], + "source": [ + "from transformers import pipeline\n", "\n", - "class MultiHeadAttention(tf.keras.layers.Layer):\n", - " def __init__(self, n_heads, causal=False, use_scale=False, **kwargs):\n", - " self.n_heads = n_heads\n", - " self.causal = causal\n", - " self.use_scale = use_scale\n", - " super().__init__(**kwargs)\n", - " def build(self, batch_input_shape):\n", - " self.dims = batch_input_shape[0][-1]\n", - " self.q_dims, self.v_dims, self.k_dims = [self.dims // self.n_heads] * 3 # could be hyperparameters instead\n", - " self.q_linear = tf.keras.layers.Conv1D(self.n_heads * self.q_dims, kernel_size=1, use_bias=False)\n", - " self.v_linear = tf.keras.layers.Conv1D(self.n_heads * self.v_dims, kernel_size=1, use_bias=False)\n", - " self.k_linear = tf.keras.layers.Conv1D(self.n_heads * self.k_dims, kernel_size=1, use_bias=False)\n", - " self.attention = tf.keras.layers.Attention(causal=self.causal, use_scale=self.use_scale)\n", - " self.out_linear = tf.keras.layers.Conv1D(self.dims, kernel_size=1, use_bias=False)\n", - " super().build(batch_input_shape)\n", - " def _multi_head_linear(self, inputs, linear):\n", - " shape = K.concatenate([K.shape(inputs)[:-1], [self.n_heads, -1]])\n", - " projected = K.reshape(linear(inputs), shape)\n", - " perm = K.permute_dimensions(projected, [0, 2, 1, 3])\n", - " return K.reshape(perm, [shape[0] * self.n_heads, shape[1], -1])\n", - " def call(self, inputs):\n", - " q = inputs[0]\n", - " v = inputs[1]\n", - " k = inputs[2] if len(inputs) > 2 else v\n", - " shape = K.shape(q)\n", - " q_proj = self._multi_head_linear(q, self.q_linear)\n", - " v_proj = self._multi_head_linear(v, self.v_linear)\n", - " k_proj = self._multi_head_linear(k, self.k_linear)\n", - " multi_attended = self.attention([q_proj, v_proj, k_proj])\n", - " shape_attended = K.shape(multi_attended)\n", - " reshaped_attended = K.reshape(multi_attended, [shape[0], self.n_heads, shape_attended[1], shape_attended[2]])\n", - " perm = K.permute_dimensions(reshaped_attended, [0, 2, 1, 3])\n", - " concat = K.reshape(perm, [shape[0], shape_attended[1], -1])\n", - " return self.out_linear(concat)" + "classifier = pipeline(\"sentiment-analysis\") # many other tasks are available\n", + "result = classifier(\"The actors were very convincing.\")" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 91, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "[{'label': 'POSITIVE', 'score': 0.9896161556243896},\n", + " {'label': 'NEGATIVE', 'score': 0.9811071157455444}]" + ] + }, + "execution_count": 91, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "Q = np.random.rand(2, 50, 512)\n", - "V = np.random.rand(2, 80, 512)\n", - "multi_attn = MultiHeadAttention(8)\n", - "multi_attn([Q, V]).shape" + "classifier([\"I am from India.\", \"I am from Iraq.\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 92, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Some layers from the model checkpoint at huggingface/distilbert-base-uncased-finetuned-mnli were not used when initializing TFDistilBertForSequenceClassification: ['dropout_19']\n", + "- This IS expected if you are initializing TFDistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", + "- This IS NOT expected if you are initializing TFDistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n", + "Some layers of TFDistilBertForSequenceClassification were not initialized from the model checkpoint at huggingface/distilbert-base-uncased-finetuned-mnli and are newly initialized: ['dropout_39']\n", + "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n" + ] + }, + { + "data": { + "text/plain": [ + "[{'label': 'contradiction', 'score': 0.9790192246437073}]" + ] + }, + "execution_count": 92, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model_name = \"huggingface/distilbert-base-uncased-finetuned-mnli\"\n", + "classifier_mnli = pipeline(\"text-classification\", model=model_name)\n", + "classifier_mnli(\"She loves me. [SEP] She loves me not.\")" + ] + }, + { + "cell_type": "code", + "execution_count": 93, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Some layers from the model checkpoint at huggingface/distilbert-base-uncased-finetuned-mnli were not used when initializing TFDistilBertForSequenceClassification: ['dropout_19']\n", + "- This IS expected if you are initializing TFDistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", + "- This IS NOT expected if you are initializing TFDistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n", + "Some layers of TFDistilBertForSequenceClassification were not initialized from the model checkpoint at huggingface/distilbert-base-uncased-finetuned-mnli and are newly initialized: ['dropout_59']\n", + "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n" + ] + } + ], + "source": [ + "from transformers import AutoTokenizer, TFAutoModelForSequenceClassification\n", + "\n", + "tokenizer = AutoTokenizer.from_pretrained(model_name)\n", + "model = TFAutoModelForSequenceClassification.from_pretrained(model_name)" + ] + }, + { + "cell_type": "code", + "execution_count": 94, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'input_ids': , 'attention_mask': }" + ] + }, + "execution_count": 94, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "token_ids = tokenizer([\"I like soccer. [SEP] We all love soccer!\",\n", + " \"Joe lived for a very long time. [SEP] Joe is old.\"],\n", + " padding=True, return_tensors=\"tf\")\n", + "token_ids" + ] + }, + { + "cell_type": "code", + "execution_count": 95, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'input_ids': , 'attention_mask': }" + ] + }, + "execution_count": 95, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "token_ids = tokenizer([(\"I like soccer.\", \"We all love soccer!\"),\n", + " (\"Joe lived for a very long time.\", \"Joe is old.\")],\n", + " padding=True, return_tensors=\"tf\")\n", + "token_ids" + ] + }, + { + "cell_type": "code", + "execution_count": 96, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "TFSequenceClassifierOutput(loss=None, logits=, hidden_states=None, attentions=None)" + ] + }, + "execution_count": 96, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "outputs = model(token_ids)\n", + "outputs" + ] + }, + { + "cell_type": "code", + "execution_count": 97, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 97, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "Y_probas = tf.keras.activations.softmax(outputs.logits)\n", + "Y_probas" + ] + }, + { + "cell_type": "code", + "execution_count": 98, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 98, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "Y_pred = tf.argmax(Y_probas, axis=1)\n", + "Y_pred # 0 = contradiction, 1 = entailment, 2 = neutral" + ] + }, + { + "cell_type": "code", + "execution_count": 99, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 1/2\n", + "1/1 [==============================] - 10s 10s/step - loss: 1.1190 - accuracy: 0.5000\n", + "Epoch 2/2\n", + "1/1 [==============================] - 0s 491ms/step - loss: 0.6666 - accuracy: 0.5000\n" + ] + } + ], + "source": [ + "sentences = [(\"Sky is blue\", \"Sky is red\"), (\"I love her\", \"She loves me\")]\n", + "X_train = tokenizer(sentences, padding=True, return_tensors=\"tf\").data\n", + "y_train = tf.constant([0, 2]) # contradiction, neutral\n", + "loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n", + "model.compile(loss=loss, optimizer=\"nadam\", metrics=[\"accuracy\"])\n", + "history = model.fit(X_train, y_train, epochs=2)" ] }, { @@ -1420,8 +2939,8 @@ "source": [ "1. Stateless RNNs can only capture patterns whose length is less than, or equal to, the size of the windows the RNN is trained on. Conversely, stateful RNNs can capture longer-term patterns. However, implementing a stateful RNN is much harder⁠—especially preparing the dataset properly. Moreover, stateful RNNs do not always work better, in part because consecutive batches are not independent and identically distributed (IID). Gradient Descent is not fond of non-IID datasets.\n", "2. In general, if you translate a sentence one word at a time, the result will be terrible. For example, the French sentence \"Je vous en prie\" means \"You are welcome,\" but if you translate it one word at a time, you get \"I you in pray.\" Huh? It is much better to read the whole sentence first and then translate it. A plain sequence-to-sequence RNN would start translating a sentence immediately after reading the first word, while an Encoder–Decoder RNN will first read the whole sentence and then translate it. That said, one could imagine a plain sequence-to-sequence RNN that would output silence whenever it is unsure about what to say next (just like human translators do when they must translate a live broadcast).\n", - "3. Variable-length input sequences can be handled by padding the shorter sequences so that all sequences in a batch have the same length, and using masking to ensure the RNN ignores the padding token. For better performance, you may also want to create batches containing sequences of similar sizes. Ragged tensors can hold sequences of variable lengths, and tf.keras will likely support them eventually, which will greatly simplify handling variable-length input sequences (at the time of this writing, it is not the case yet). Regarding variable-length output sequences, if the length of the output sequence is known in advance (e.g., if you know that it is the same as the input sequence), then you just need to configure the loss function so that it ignores tokens that come after the end of the sequence. Similarly, the code that will use the model should ignore tokens beyond the end of the sequence. But generally the length of the output sequence is not known ahead of time, so the solution is to train the model so that it outputs an end-of-sequence token at the end of each sequence.\n", - "4. Beam search is a technique used to improve the performance of a trained Encoder–Decoder model, for example in a neural machine translation system. The algorithm keeps track of a short list of the _k_ most promising output sentences (say, the top three), and at each decoder step it tries to extend them by one word; then it keeps only the _k_ most likely sentences. The parameter _k_ is called the _beam width_: the larger it is, the more CPU and RAM will be used, but also the more accurate the system will be. Instead of greedily choosing the most likely next word at each step to extend a single sentence, this technique allows the system to explore several promising sentences simultaneously. Moreover, this technique lends itself well to parallelization. You can implement beam search fairly easily using TensorFlow Addons.\n", + "3. Variable-length input sequences can be handled by padding the shorter sequences so that all sequences in a batch have the same length, and using masking to ensure the RNN ignores the padding token. For better performance, you may also want to create batches containing sequences of similar sizes. Ragged tensors can hold sequences of variable lengths, and Keras now supports them, which simplifies handling variable-length input sequences (at the time of this writing, it still does not handle ragged tensors as targets on the GPU, though). Regarding variable-length output sequences, if the length of the output sequence is known in advance (e.g., if you know that it is the same as the input sequence), then you just need to configure the loss function so that it ignores tokens that come after the end of the sequence. Similarly, the code that will use the model should ignore tokens beyond the end of the sequence. But generally the length of the output sequence is not known ahead of time, so the solution is to train the model so that it outputs an end-of-sequence token at the end of each sequence.\n", + "4. Beam search is a technique used to improve the performance of a trained Encoder–Decoder model, for example in a neural machine translation system. The algorithm keeps track of a short list of the _k_ most promising output sentences (say, the top three), and at each decoder step it tries to extend them by one word; then it keeps only the _k_ most likely sentences. The parameter _k_ is called the _beam width_: the larger it is, the more CPU and RAM will be used, but also the more accurate the system will be. Instead of greedily choosing the most likely next word at each step to extend a single sentence, this technique allows the system to explore several promising sentences simultaneously. Moreover, this technique lends itself well to parallelization. You can implement beam search by writing a custom memory cell. Alternatively, TensorFlow Addons's seq2seq API provides an implementation.\n", "5. An attention mechanism is a technique initially used in Encoder–Decoder models to give the decoder more direct access to the input sequence, allowing it to deal with longer input sequences. At each decoder time step, the current decoder's state and the full output of the encoder are processed by an alignment model that outputs an alignment score for each input time step. This score indicates which part of the input is most relevant to the current decoder time step. The weighted sum of the encoder output (weighted by their alignment score) is then fed to the decoder, which produces the next decoder state and the output for this time step. The main benefit of using an attention mechanism is the fact that the Encoder–Decoder model can successfully process longer input sequences. Another benefit is that the alignment scores make the model easier to debug and interpret: for example, if the model makes a mistake, you can look at which part of the input it was paying attention to, and this can help diagnose the issue. An attention mechanism is also at the core of the Transformer architecture, in the Multi-Head Attention layers. See the next answer.\n", "6. The most important layer in the Transformer architecture is the Multi-Head Attention layer (the original Transformer architecture contains 18 of them, including 6 Masked Multi-Head Attention layers). It is at the core of language models such as BERT and GPT-2. Its purpose is to allow the model to identify which words are most aligned with each other, and then improve each word's representation using these contextual clues.\n", "7. Sampled softmax is used when training a classification model when there are many classes (e.g., thousands). It computes an approximation of the cross-entropy loss based on the logit predicted by the model for the correct class, and the predicted logits for a sample of incorrect words. This speeds up training considerably compared to computing the softmax over all logits and then estimating the cross-entropy loss. After training, the model can be used normally, using the regular softmax function to compute all the class probabilities based on all the logits." @@ -1444,7 +2963,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 100, "metadata": {}, "outputs": [], "source": [ @@ -1487,9 +3006,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 101, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "BTXXTTVPXTVPXTTVPSE BPVPSE BTXSE BPVVE BPVVE BTSXSE BPTVPXTTTVVE BPVVE BTXSE BTXXVPSE BPTTTTTTTTVVE BTXSE BPVPSE BTXSE BPTVPSE BTXXTVPSE BPVVE BPVVE BPVVE BPTTVVE BPVVE BPVVE BTXXVVE BTXXVVE BTXXVPXVVE " + ] + } + ], "source": [ "np.random.seed(42)\n", "\n", @@ -1506,9 +3033,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 102, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "BTBPTTTVPXTVPXTTVPSETE BPBPTVPSEPE BPBPVVEPE BPBPVPXVVEPE BPBTXXTTTTVVEPE BPBPVPSEPE BPBTXXVPSEPE BPBTSSSSSSSXSEPE BTBPVVETE BPBTXXVVEPE BPBTXXVPSEPE BTBTXXVVETE BPBPVVEPE BPBPVVEPE BPBTSXSEPE BPBPVVEPE BPBPTVPSEPE BPBTXXVVEPE BTBPTVPXVVETE BTBPVVETE BTBTSSSSSSSXXVVETE BPBTSSSXXTTTTVPSEPE BTBPTTVVETE BPBTXXTVVEPE BTBTXSETE " + ] + } + ], "source": [ "np.random.seed(42)\n", "\n", @@ -1525,7 +3060,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 103, "metadata": {}, "outputs": [], "source": [ @@ -1548,9 +3083,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 104, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "BTBPTTTPPXTVPXTTVPSETE BPBTXEEPE BPBPTVVVEPE BPBTSSSSXSETE BPTTXSEPE BTBPVPXTTTTTTEVETE BPBTXXSVEPE BSBPTTVPSETE BPBXVVEPE BEBTXSETE BPBPVPSXPE BTBPVVVETE BPBTSXSETE BPBPTTTPTTTTTVPSEPE BTBTXXTTSTVPSETE BBBTXSETE BPBTPXSEPE BPBPVPXTTTTVPXTVPXVPXTTTVVEVE BTBXXXTVPSETE BEBTSSSSSXXVPXTVVETE BTBXTTVVETE BPBTXSTPE BTBTXXTTTVPSBTE BTBTXSETX BTBTSXSSTE " + ] + } + ], "source": [ "np.random.seed(42)\n", "\n", @@ -1567,7 +3110,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 105, "metadata": {}, "outputs": [], "source": [ @@ -1577,9 +3120,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 106, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "[0, 4, 4, 4, 6, 6, 5, 5, 1, 4, 1]" + ] + }, + "execution_count": 106, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "string_to_ids(\"BTTTXXVVETE\")" ] @@ -1593,15 +3147,19 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 107, "metadata": {}, "outputs": [], "source": [ "def generate_dataset(size):\n", - " good_strings = [string_to_ids(generate_string(embedded_reber_grammar))\n", - " for _ in range(size // 2)]\n", - " bad_strings = [string_to_ids(generate_corrupted_string(embedded_reber_grammar))\n", - " for _ in range(size - size // 2)]\n", + " good_strings = [\n", + " string_to_ids(generate_string(embedded_reber_grammar))\n", + " for _ in range(size // 2)\n", + " ]\n", + " bad_strings = [\n", + " string_to_ids(generate_corrupted_string(embedded_reber_grammar))\n", + " for _ in range(size - size // 2)\n", + " ]\n", " all_strings = good_strings + bad_strings\n", " X = tf.ragged.constant(all_strings, ragged_rank=1)\n", " y = np.array([[1.] for _ in range(len(good_strings))] +\n", @@ -1611,7 +3169,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 108, "metadata": {}, "outputs": [], "source": [ @@ -1630,9 +3188,22 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 109, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 109, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "X_train[0]" ] @@ -1646,9 +3217,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 110, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "array([1.])" + ] + }, + "execution_count": 110, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "y_train[0]" ] @@ -1662,9 +3244,62 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 111, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 1/20\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "313/313 [==============================] - 4s 8ms/step - loss: 0.6910 - accuracy: 0.5095 - val_loss: 0.6825 - val_accuracy: 0.5645\n", + "Epoch 2/20\n", + "313/313 [==============================] - 2s 7ms/step - loss: 0.6678 - accuracy: 0.5659 - val_loss: 0.6635 - val_accuracy: 0.6105\n", + "Epoch 3/20\n", + "313/313 [==============================] - 2s 7ms/step - loss: 0.6504 - accuracy: 0.5766 - val_loss: 0.6521 - val_accuracy: 0.6110\n", + "Epoch 4/20\n", + "313/313 [==============================] - 2s 8ms/step - loss: 0.6347 - accuracy: 0.5980 - val_loss: 0.6224 - val_accuracy: 0.6445\n", + "Epoch 5/20\n", + "313/313 [==============================] - 2s 7ms/step - loss: 0.6054 - accuracy: 0.6361 - val_loss: 0.5779 - val_accuracy: 0.6980\n", + "Epoch 6/20\n", + "313/313 [==============================] - 2s 7ms/step - loss: 0.5414 - accuracy: 0.7093 - val_loss: 0.4695 - val_accuracy: 0.7795\n", + "Epoch 7/20\n", + "313/313 [==============================] - 2s 7ms/step - loss: 0.3756 - accuracy: 0.8418 - val_loss: 0.2685 - val_accuracy: 0.9115\n", + "Epoch 8/20\n", + "313/313 [==============================] - 2s 7ms/step - loss: 0.2601 - accuracy: 0.9044 - val_loss: 0.1534 - val_accuracy: 0.9615\n", + "Epoch 9/20\n", + "313/313 [==============================] - 2s 7ms/step - loss: 0.1774 - accuracy: 0.9427 - val_loss: 0.1063 - val_accuracy: 0.9735\n", + "Epoch 10/20\n", + "313/313 [==============================] - 2s 7ms/step - loss: 0.0624 - accuracy: 0.9826 - val_loss: 0.0219 - val_accuracy: 0.9975\n", + "Epoch 11/20\n", + "313/313 [==============================] - 2s 7ms/step - loss: 0.0371 - accuracy: 0.9914 - val_loss: 0.0055 - val_accuracy: 1.0000\n", + "Epoch 12/20\n", + "313/313 [==============================] - 2s 7ms/step - loss: 0.0029 - accuracy: 0.9995 - val_loss: 8.7265e-04 - val_accuracy: 1.0000\n", + "Epoch 13/20\n", + "313/313 [==============================] - 2s 7ms/step - loss: 6.7552e-04 - accuracy: 1.0000 - val_loss: 4.9408e-04 - val_accuracy: 1.0000\n", + "Epoch 14/20\n", + "313/313 [==============================] - 2s 7ms/step - loss: 4.4514e-04 - accuracy: 1.0000 - val_loss: 3.6322e-04 - val_accuracy: 1.0000\n", + "Epoch 15/20\n", + "313/313 [==============================] - 2s 7ms/step - loss: 3.3943e-04 - accuracy: 1.0000 - val_loss: 2.8524e-04 - val_accuracy: 1.0000\n", + "Epoch 16/20\n", + "313/313 [==============================] - 2s 7ms/step - loss: 2.7723e-04 - accuracy: 1.0000 - val_loss: 2.3880e-04 - val_accuracy: 1.0000\n", + "Epoch 17/20\n", + "313/313 [==============================] - 2s 7ms/step - loss: 2.3477e-04 - accuracy: 1.0000 - val_loss: 2.0363e-04 - val_accuracy: 1.0000\n", + "Epoch 18/20\n", + "313/313 [==============================] - 2s 7ms/step - loss: 2.0382e-04 - accuracy: 1.0000 - val_loss: 1.7760e-04 - val_accuracy: 1.0000\n", + "Epoch 19/20\n", + "313/313 [==============================] - 2s 7ms/step - loss: 1.8077e-04 - accuracy: 1.0000 - val_loss: 1.5916e-04 - val_accuracy: 1.0000\n", + "Epoch 20/20\n", + "313/313 [==============================] - 2s 8ms/step - loss: 1.6246e-04 - accuracy: 1.0000 - val_loss: 1.4362e-04 - val_accuracy: 1.0000\n" + ] + } + ], "source": [ "np.random.seed(42)\n", "tf.random.set_seed(42)\n", @@ -1673,13 +3308,17 @@ "\n", "model = tf.keras.Sequential([\n", " tf.keras.layers.InputLayer(input_shape=[None], dtype=tf.int32, ragged=True),\n", - " tf.keras.layers.Embedding(input_dim=len(POSSIBLE_CHARS), output_dim=embedding_size),\n", + " tf.keras.layers.Embedding(input_dim=len(POSSIBLE_CHARS),\n", + " output_dim=embedding_size),\n", " tf.keras.layers.GRU(30),\n", " tf.keras.layers.Dense(1, activation=\"sigmoid\")\n", "])\n", - "optimizer = tf.keras.optimizers.SGD(learning_rate=0.02, momentum = 0.95, nesterov=True)\n", - "model.compile(loss=\"binary_crossentropy\", optimizer=optimizer, metrics=[\"accuracy\"])\n", - "history = model.fit(X_train, y_train, epochs=20, validation_data=(X_valid, y_valid))" + "optimizer = tf.keras.optimizers.SGD(learning_rate=0.02, momentum = 0.95,\n", + " nesterov=True)\n", + "model.compile(loss=\"binary_crossentropy\", optimizer=optimizer,\n", + " metrics=[\"accuracy\"])\n", + "history = model.fit(X_train, y_train, epochs=20,\n", + " validation_data=(X_valid, y_valid))" ] }, { @@ -1691,9 +3330,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 112, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Estimated probability that these are Reber strings:\n", + "BPBTSSSSSSSXXTTVPXVPXTTTTTVVETE: 0.02%\n", + "BPBTSSSSSSSXXTTVPXVPXTTTTTVVEPE: 99.99%\n" + ] + } + ], "source": [ "test_strings = [\"BPBTSSSSSSSXXTTVPXVPXTTTTTVVETE\",\n", " \"BPBTSSSSSSSXXTTVPXVPXTTTTTVVEPE\"]\n", @@ -1730,7 +3380,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 113, "metadata": {}, "outputs": [], "source": [ @@ -1761,9 +3411,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 114, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Input Target \n", + "--------------------------------------------------\n", + "September 20, 7075 7075-09-20 \n", + "May 15, 8579 8579-05-15 \n", + "January 11, 7103 7103-01-11 \n" + ] + } + ], "source": [ "np.random.seed(42)\n", "\n", @@ -1784,9 +3446,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 115, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "' ,0123456789ADFJMNOSabceghilmnoprstuvy'" + ] + }, + "execution_count": 115, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "INPUT_CHARS = \"\".join(sorted(set(\"\".join(MONTHS) + \"0123456789, \")))\n", "INPUT_CHARS" @@ -1801,7 +3474,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 116, "metadata": {}, "outputs": [], "source": [ @@ -1817,7 +3490,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 117, "metadata": {}, "outputs": [], "source": [ @@ -1827,25 +3500,47 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 118, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "[19, 23, 31, 34, 23, 28, 21, 23, 32, 0, 4, 2, 1, 0, 9, 2, 9, 7]" + ] + }, + "execution_count": 118, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "date_str_to_ids(x_example[0], INPUT_CHARS)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 119, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "[7, 0, 7, 5, 10, 0, 9, 10, 2, 0]" + ] + }, + "execution_count": 119, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "date_str_to_ids(y_example[0], OUTPUT_CHARS)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 120, "metadata": {}, "outputs": [], "source": [ @@ -1861,7 +3556,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 121, "metadata": {}, "outputs": [], "source": [ @@ -1874,9 +3569,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 122, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 122, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "Y_train[0]" ] @@ -1899,9 +3605,56 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 123, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 1/20\n", + "313/313 [==============================] - 10s 23ms/step - loss: 1.8150 - accuracy: 0.3489 - val_loss: 1.3726 - val_accuracy: 0.4939\n", + "Epoch 2/20\n", + "313/313 [==============================] - 7s 22ms/step - loss: 1.2447 - accuracy: 0.5510 - val_loss: 1.0725 - val_accuracy: 0.6115\n", + "Epoch 3/20\n", + "313/313 [==============================] - 7s 23ms/step - loss: 1.0937 - accuracy: 0.6125 - val_loss: 1.0548 - val_accuracy: 0.6130\n", + "Epoch 4/20\n", + "313/313 [==============================] - 7s 23ms/step - loss: 1.0032 - accuracy: 0.6413 - val_loss: 3.8747 - val_accuracy: 0.1788\n", + "Epoch 5/20\n", + "313/313 [==============================] - 8s 26ms/step - loss: 0.8159 - accuracy: 0.7023 - val_loss: 0.6623 - val_accuracy: 0.7474\n", + "Epoch 6/20\n", + "313/313 [==============================] - 8s 26ms/step - loss: 0.5645 - accuracy: 0.7795 - val_loss: 0.5005 - val_accuracy: 0.8032\n", + "Epoch 7/20\n", + "313/313 [==============================] - 8s 26ms/step - loss: 0.5037 - accuracy: 0.8103 - val_loss: 0.3798 - val_accuracy: 0.8500\n", + "Epoch 8/20\n", + "313/313 [==============================] - 8s 26ms/step - loss: 0.3131 - accuracy: 0.8795 - val_loss: 0.2582 - val_accuracy: 0.9043\n", + "Epoch 9/20\n", + "313/313 [==============================] - 8s 26ms/step - loss: 0.2141 - accuracy: 0.9280 - val_loss: 0.1637 - val_accuracy: 0.9498\n", + "Epoch 10/20\n", + "313/313 [==============================] - 9s 28ms/step - loss: 0.1282 - accuracy: 0.9650 - val_loss: 0.0918 - val_accuracy: 0.9774\n", + "Epoch 11/20\n", + "313/313 [==============================] - 9s 28ms/step - loss: 0.0669 - accuracy: 0.9871 - val_loss: 0.3368 - val_accuracy: 0.8871\n", + "Epoch 12/20\n", + "313/313 [==============================] - 10s 32ms/step - loss: 0.1551 - accuracy: 0.9662 - val_loss: 0.0398 - val_accuracy: 0.9949\n", + "Epoch 13/20\n", + "313/313 [==============================] - 9s 29ms/step - loss: 0.0291 - accuracy: 0.9969 - val_loss: 0.0240 - val_accuracy: 0.9984\n", + "Epoch 14/20\n", + "313/313 [==============================] - 9s 30ms/step - loss: 0.0182 - accuracy: 0.9986 - val_loss: 0.0161 - val_accuracy: 0.9993\n", + "Epoch 15/20\n", + "313/313 [==============================] - 9s 30ms/step - loss: 0.0119 - accuracy: 0.9995 - val_loss: 0.0112 - val_accuracy: 0.9997\n", + "Epoch 16/20\n", + "313/313 [==============================] - 10s 32ms/step - loss: 0.0082 - accuracy: 0.9998 - val_loss: 0.0083 - val_accuracy: 0.9999\n", + "Epoch 17/20\n", + "313/313 [==============================] - 10s 33ms/step - loss: 0.0059 - accuracy: 0.9999 - val_loss: 0.0058 - val_accuracy: 0.9999\n", + "Epoch 18/20\n", + "313/313 [==============================] - 11s 34ms/step - loss: 0.0042 - accuracy: 1.0000 - val_loss: 0.0043 - val_accuracy: 0.9999\n", + "Epoch 19/20\n", + "313/313 [==============================] - 10s 33ms/step - loss: 0.0031 - accuracy: 1.0000 - val_loss: 0.0034 - val_accuracy: 0.9999\n", + "Epoch 20/20\n", + "313/313 [==============================] - 12s 40ms/step - loss: 0.0024 - accuracy: 1.0000 - val_loss: 0.0026 - val_accuracy: 1.0000\n" + ] + } + ], "source": [ "embedding_size = 32\n", "max_output_length = Y_train.shape[1]\n", @@ -1943,7 +3696,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 124, "metadata": {}, "outputs": [], "source": [ @@ -1961,7 +3714,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 125, "metadata": {}, "outputs": [], "source": [ @@ -1970,11 +3723,19 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 126, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2009-09-17\n", + "1789-07-14\n" + ] + } + ], "source": [ - "#ids = model.predict_classes(X_new)\n", "ids = model.predict(X_new).argmax(axis=-1)\n", "for date_str in ids_to_date_strs(ids):\n", " print(date_str)" @@ -1996,7 +3757,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 127, "metadata": {}, "outputs": [], "source": [ @@ -2005,11 +3766,19 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 128, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2020-02-02\n", + "1789-01-14\n" + ] + } + ], "source": [ - "#ids = model.predict_classes(X_new)\n", "ids = model.predict(X_new).argmax(axis=-1)\n", "for date_str in ids_to_date_strs(ids):\n", " print(date_str)" @@ -2024,7 +3793,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 129, "metadata": {}, "outputs": [], "source": [ @@ -2038,16 +3807,26 @@ "\n", "def convert_date_strs(date_strs):\n", " X = prepare_date_strs_padded(date_strs)\n", - " #ids = model.predict_classes(X)\n", " ids = model.predict(X).argmax(axis=-1)\n", " return ids_to_date_strs(ids)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 130, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "['2020-05-02', '1789-07-14']" + ] + }, + "execution_count": 130, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "convert_date_strs([\"May 02, 2020\", \"July 14, 1789\"])" ] @@ -2090,7 +3869,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 131, "metadata": {}, "outputs": [], "source": [ @@ -2114,9 +3893,27 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 132, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 132, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "X_train_decoder" ] @@ -2130,9 +3927,36 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 133, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 1/10\n", + "313/313 [==============================] - 11s 27ms/step - loss: 1.6824 - accuracy: 0.3734 - val_loss: 1.4054 - val_accuracy: 0.4681\n", + "Epoch 2/10\n", + "313/313 [==============================] - 8s 26ms/step - loss: 1.1935 - accuracy: 0.5550 - val_loss: 0.8868 - val_accuracy: 0.6750\n", + "Epoch 3/10\n", + "313/313 [==============================] - 8s 26ms/step - loss: 0.6403 - accuracy: 0.7700 - val_loss: 0.3493 - val_accuracy: 0.8978\n", + "Epoch 4/10\n", + "313/313 [==============================] - 8s 26ms/step - loss: 0.2292 - accuracy: 0.9423 - val_loss: 0.1254 - val_accuracy: 0.9782\n", + "Epoch 5/10\n", + "313/313 [==============================] - 8s 26ms/step - loss: 0.0694 - accuracy: 0.9932 - val_loss: 0.0441 - val_accuracy: 0.9982\n", + "Epoch 6/10\n", + "313/313 [==============================] - 9s 29ms/step - loss: 0.0576 - accuracy: 0.9923 - val_loss: 0.0280 - val_accuracy: 0.9988\n", + "Epoch 7/10\n", + "313/313 [==============================] - 8s 26ms/step - loss: 0.0179 - accuracy: 0.9998 - val_loss: 0.0143 - val_accuracy: 0.9999\n", + "Epoch 8/10\n", + "313/313 [==============================] - 6s 18ms/step - loss: 0.0107 - accuracy: 0.9999 - val_loss: 0.0092 - val_accuracy: 0.9999\n", + "Epoch 9/10\n", + "313/313 [==============================] - 6s 20ms/step - loss: 0.0070 - accuracy: 1.0000 - val_loss: 0.0065 - val_accuracy: 0.9999\n", + "Epoch 10/10\n", + "313/313 [==============================] - 6s 18ms/step - loss: 0.0050 - accuracy: 1.0000 - val_loss: 0.0047 - val_accuracy: 0.9999\n" + ] + } + ], "source": [ "encoder_embedding_size = 32\n", "decoder_embedding_size = 32\n", @@ -2184,7 +4008,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 134, "metadata": {}, "outputs": [], "source": [ @@ -2204,9 +4028,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 135, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "['1789-07-14', '2020-05-01']" + ] + }, + "execution_count": 135, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "predict_date_strs([\"July 14, 1789\", \"May 01, 2020\"])" ] @@ -2215,526 +4050,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Works fine! :)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Third version: using TF-Addons's seq2seq implementation" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's build exactly the same model, but using TF-Addon's seq2seq API. The implementation below is almost very similar to the TFA example higher in this notebook, except without the model input to specify the output sequence length, for simplicity (but you can easily add it back in if you need it for your projects, when the output sequences have very different lengths)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import tensorflow_addons as tfa\n", - "\n", - "np.random.seed(42)\n", - "tf.random.set_seed(42)\n", - "\n", - "encoder_embedding_size = 32\n", - "decoder_embedding_size = 32\n", - "units = 128\n", - "\n", - "encoder_inputs = tf.keras.layers.Input(shape=[None], dtype=np.int32)\n", - "decoder_inputs = tf.keras.layers.Input(shape=[None], dtype=np.int32)\n", - "sequence_lengths = tf.keras.layers.Input(shape=[], dtype=np.int32)\n", - "\n", - "encoder_embeddings = tf.keras.layers.Embedding(\n", - " len(INPUT_CHARS) + 1, encoder_embedding_size)(encoder_inputs)\n", - "\n", - "decoder_embedding_layer = tf.keras.layers.Embedding(\n", - " len(OUTPUT_CHARS) + 2, decoder_embedding_size)\n", - "decoder_embeddings = decoder_embedding_layer(decoder_inputs)\n", - "\n", - "encoder = tf.keras.layers.LSTM(units, return_state=True)\n", - "encoder_outputs, state_h, state_c = encoder(encoder_embeddings)\n", - "encoder_state = [state_h, state_c]\n", - "\n", - "sampler = tfa.seq2seq.sampler.TrainingSampler()\n", - "\n", - "decoder_cell = tf.keras.layers.LSTMCell(units)\n", - "output_layer = tf.keras.layers.Dense(len(OUTPUT_CHARS) + 1)\n", - "\n", - "decoder = tfa.seq2seq.basic_decoder.BasicDecoder(decoder_cell,\n", - " sampler,\n", - " output_layer=output_layer)\n", - "final_outputs, final_state, final_sequence_lengths = decoder(\n", - " decoder_embeddings,\n", - " initial_state=encoder_state)\n", - "Y_proba = tf.keras.layers.Activation(\"softmax\")(final_outputs.rnn_output)\n", - "\n", - "model = tf.keras.Model(inputs=[encoder_inputs, decoder_inputs],\n", - " outputs=[Y_proba])\n", - "optimizer = tf.keras.optimizers.Nadam()\n", - "model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=optimizer,\n", - " metrics=[\"accuracy\"])\n", - "history = model.fit([X_train, X_train_decoder], Y_train, epochs=15,\n", - " validation_data=([X_valid, X_valid_decoder], Y_valid))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "And once again, 100% validation accuracy! To use the model, we can just reuse the `predict_date_strs()` function:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "predict_date_strs([\"July 14, 1789\", \"May 01, 2020\"])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "However, there's a much more efficient way to perform inference. Until now, during inference, we've run the model once for each new character. Instead, we can create a new decoder, based on the previously trained layers, but using a `GreedyEmbeddingSampler` instead of a `TrainingSampler`.\n", - "\n", - "At each time step, the `GreedyEmbeddingSampler` will compute the argmax of the decoder's outputs, and run the resulting token IDs through the decoder's embedding layer. Then it will feed the resulting embeddings to the decoder's LSTM cell at the next time step. This way, we only need to run the decoder once to get the full prediction." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "inference_sampler = tfa.seq2seq.sampler.GreedyEmbeddingSampler(\n", - " embedding_fn=decoder_embedding_layer)\n", - "inference_decoder = tfa.seq2seq.basic_decoder.BasicDecoder(\n", - " decoder_cell, inference_sampler, output_layer=output_layer,\n", - " maximum_iterations=max_output_length)\n", - "batch_size = tf.shape(encoder_inputs)[:1]\n", - "start_tokens = tf.fill(dims=batch_size, value=sos_id)\n", - "final_outputs, final_state, final_sequence_lengths = inference_decoder(\n", - " start_tokens,\n", - " initial_state=encoder_state,\n", - " start_tokens=start_tokens,\n", - " end_token=0)\n", - "\n", - "inference_model = tf.keras.Model(inputs=[encoder_inputs],\n", - " outputs=[final_outputs.sample_id])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "A few notes:\n", - "* The `GreedyEmbeddingSampler` needs the `start_tokens` (a vector containing the start-of-sequence ID for each decoder sequence), and the `end_token` (the decoder will stop decoding a sequence once the model outputs this token).\n", - "* We must set `maximum_iterations` when creating the `BasicDecoder`, or else it may run into an infinite loop (if the model never outputs the end token for at least one of the sequences). This would force you would to restart the Jupyter kernel.\n", - "* The decoder inputs are not needed anymore, since all the decoder inputs are generated dynamically based on the outputs from the previous time step.\n", - "* The model's outputs are `final_outputs.sample_id` instead of the softmax of `final_outputs.rnn_outputs`. This allows us to directly get the argmax of the model's outputs. If you prefer to have access to the logits, you can replace `final_outputs.sample_id` with `final_outputs.rnn_outputs`." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now we can write a simple function that uses the model to perform the date format conversion:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def fast_predict_date_strs(date_strs):\n", - " X = prepare_date_strs_padded(date_strs)\n", - " Y_pred = inference_model.predict(X)\n", - " return ids_to_date_strs(Y_pred)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "fast_predict_date_strs([\"July 14, 1789\", \"May 01, 2020\"])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's check that it really is faster:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%timeit predict_date_strs([\"July 14, 1789\", \"May 01, 2020\"])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%timeit fast_predict_date_strs([\"July 14, 1789\", \"May 01, 2020\"])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "That's more than a 10x speedup! And it would be even more if we were handling longer sequences." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Fourth version: using TF-Addons's seq2seq implementation with a scheduled sampler" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**Warning**: due to a TF bug, this version only works using TensorFlow 2.2 or above." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "When we trained the previous model, at each time step _t_ we gave the model the target token for time step _t_ - 1. However, at inference time, the model did not get the previous target at each time step. Instead, it got the previous prediction. So there is a discrepancy between training and inference, which may lead to disappointing performance. To alleviate this, we can gradually replace the targets with the predictions, during training. For this, we just need to replace the `TrainingSampler` with a `ScheduledEmbeddingTrainingSampler`, and use a Keras callback to gradually increase the `sampling_probability` (i.e., the probability that the decoder will use the prediction from the previous time step rather than the target for the previous time step)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import tensorflow_addons as tfa\n", - "\n", - "np.random.seed(42)\n", - "tf.random.set_seed(42)\n", - "\n", - "n_epochs = 20\n", - "encoder_embedding_size = 32\n", - "decoder_embedding_size = 32\n", - "units = 128\n", - "\n", - "encoder_inputs = tf.keras.layers.Input(shape=[None], dtype=np.int32)\n", - "decoder_inputs = tf.keras.layers.Input(shape=[None], dtype=np.int32)\n", - "sequence_lengths = tf.keras.layers.Input(shape=[], dtype=np.int32)\n", - "\n", - "encoder_embeddings = tf.keras.layers.Embedding(\n", - " len(INPUT_CHARS) + 1, encoder_embedding_size)(encoder_inputs)\n", - "\n", - "decoder_embedding_layer = tf.keras.layers.Embedding(\n", - " len(OUTPUT_CHARS) + 2, decoder_embedding_size)\n", - "decoder_embeddings = decoder_embedding_layer(decoder_inputs)\n", - "\n", - "encoder = tf.keras.layers.LSTM(units, return_state=True)\n", - "encoder_outputs, state_h, state_c = encoder(encoder_embeddings)\n", - "encoder_state = [state_h, state_c]\n", - "\n", - "sampler = tfa.seq2seq.sampler.ScheduledEmbeddingTrainingSampler(\n", - " sampling_probability=0.,\n", - " embedding_fn=decoder_embedding_layer)\n", - "# we must set the sampling_probability after creating the sampler\n", - "# (see https://github.com/tensorflow/addons/pull/1714)\n", - "sampler.sampling_probability = tf.Variable(0.)\n", - "\n", - "decoder_cell = tf.keras.layers.LSTMCell(units)\n", - "output_layer = tf.keras.layers.Dense(len(OUTPUT_CHARS) + 1)\n", - "\n", - "decoder = tfa.seq2seq.basic_decoder.BasicDecoder(decoder_cell,\n", - " sampler,\n", - " output_layer=output_layer)\n", - "final_outputs, final_state, final_sequence_lengths = decoder(\n", - " decoder_embeddings,\n", - " initial_state=encoder_state)\n", - "Y_proba = tf.keras.layers.Activation(\"softmax\")(final_outputs.rnn_output)\n", - "\n", - "model = tf.keras.Model(inputs=[encoder_inputs, decoder_inputs],\n", - " outputs=[Y_proba])\n", - "optimizer = tf.keras.optimizers.Nadam()\n", - "model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=optimizer,\n", - " metrics=[\"accuracy\"])\n", - "\n", - "def update_sampling_probability(epoch, logs):\n", - " proba = min(1.0, epoch / (n_epochs - 10))\n", - " sampler.sampling_probability.assign(proba)\n", - "\n", - "sampling_probability_cb = tf.keras.callbacks.LambdaCallback(\n", - " on_epoch_begin=update_sampling_probability)\n", - "history = model.fit([X_train, X_train_decoder], Y_train, epochs=n_epochs,\n", - " validation_data=([X_valid, X_valid_decoder], Y_valid),\n", - " callbacks=[sampling_probability_cb])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Not quite 100% validation accuracy, but close enough!" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "For inference, we could do the exact same thing as earlier, using a `GreedyEmbeddingSampler`. However, just for the sake of completeness, let's use a `SampleEmbeddingSampler` instead. It's almost the same thing, except that instead of using the argmax of the model's output to find the token ID, it treats the outputs as logits and uses them to sample a token ID randomly. This can be useful when you want to generate text. The `softmax_temperature` argument serves the \n", - "same purpose as when we generated Shakespeare-like text (the higher this argument, the more random the generated text will be)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "softmax_temperature = tf.Variable(1.)\n", - "\n", - "inference_sampler = tfa.seq2seq.sampler.SampleEmbeddingSampler(\n", - " embedding_fn=decoder_embedding_layer,\n", - " softmax_temperature=softmax_temperature)\n", - "inference_decoder = tfa.seq2seq.basic_decoder.BasicDecoder(\n", - " decoder_cell, inference_sampler, output_layer=output_layer,\n", - " maximum_iterations=max_output_length)\n", - "batch_size = tf.shape(encoder_inputs)[:1]\n", - "start_tokens = tf.fill(dims=batch_size, value=sos_id)\n", - "final_outputs, final_state, final_sequence_lengths = inference_decoder(\n", - " start_tokens,\n", - " initial_state=encoder_state,\n", - " start_tokens=start_tokens,\n", - " end_token=0)\n", - "\n", - "inference_model = tf.keras.Model(inputs=[encoder_inputs],\n", - " outputs=[final_outputs.sample_id])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def creative_predict_date_strs(date_strs, temperature=1.0):\n", - " softmax_temperature.assign(temperature)\n", - " X = prepare_date_strs_padded(date_strs)\n", - " Y_pred = inference_model.predict(X)\n", - " return ids_to_date_strs(Y_pred)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "tf.random.set_seed(42)\n", - "\n", - "creative_predict_date_strs([\"July 14, 1789\", \"May 01, 2020\"])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Dates look good at room temperature. Now let's heat things up a bit:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "tf.random.set_seed(42)\n", - "\n", - "creative_predict_date_strs([\"July 14, 1789\", \"May 01, 2020\"],\n", - " temperature=5.)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Oops, the dates are overcooked, now. Let's call them \"creative\" dates." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Fifth version: using TFA seq2seq, the Keras subclassing API and attention mechanisms" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The sequences in this problem are pretty short, but if we wanted to tackle longer sequences, we would probably have to use attention mechanisms. While it's possible to code our own implementation, it's simpler and more efficient to use TF-Addons's implementation instead. Let's do that now, this time using Keras' subclassing API.\n", - "\n", - "**Warning**: due to a TensorFlow bug (see [this issue](https://github.com/tensorflow/addons/issues/1153) for details), the `get_initial_state()` method fails in eager mode, so for now we have to use the subclassing API, as Keras automatically calls `tf.function()` on the `call()` method (so it runs in graph mode)." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In this implementation, we've reverted back to using the `TrainingSampler`, for simplicity (but you can easily tweak it to use a `ScheduledEmbeddingTrainingSampler` instead). We also use a `GreedyEmbeddingSampler` during inference, so this class is pretty easy to use:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "class DateTranslation(tf.keras.Model):\n", - " def __init__(self, units=128, encoder_embedding_size=32,\n", - " decoder_embedding_size=32, **kwargs):\n", - " super().__init__(**kwargs)\n", - " self.encoder_embedding = tf.keras.layers.Embedding(\n", - " input_dim=len(INPUT_CHARS) + 1,\n", - " output_dim=encoder_embedding_size)\n", - " self.encoder = tf.keras.layers.LSTM(units,\n", - " return_sequences=True,\n", - " return_state=True)\n", - " self.decoder_embedding = tf.keras.layers.Embedding(\n", - " input_dim=len(OUTPUT_CHARS) + 2,\n", - " output_dim=decoder_embedding_size)\n", - " self.attention = tfa.seq2seq.LuongAttention(units)\n", - " decoder_inner_cell = tf.keras.layers.LSTMCell(units)\n", - " self.decoder_cell = tfa.seq2seq.AttentionWrapper(\n", - " cell=decoder_inner_cell,\n", - " attention_mechanism=self.attention)\n", - " output_layer = tf.keras.layers.Dense(len(OUTPUT_CHARS) + 1)\n", - " self.decoder = tfa.seq2seq.BasicDecoder(\n", - " cell=self.decoder_cell,\n", - " sampler=tfa.seq2seq.sampler.TrainingSampler(),\n", - " output_layer=output_layer)\n", - " self.inference_decoder = tfa.seq2seq.BasicDecoder(\n", - " cell=self.decoder_cell,\n", - " sampler=tfa.seq2seq.sampler.GreedyEmbeddingSampler(\n", - " embedding_fn=self.decoder_embedding),\n", - " output_layer=output_layer,\n", - " maximum_iterations=max_output_length)\n", - "\n", - " def call(self, inputs, training=None):\n", - " encoder_input, decoder_input = inputs\n", - " encoder_embeddings = self.encoder_embedding(encoder_input)\n", - " encoder_outputs, encoder_state_h, encoder_state_c = self.encoder(\n", - " encoder_embeddings,\n", - " training=training)\n", - " encoder_state = [encoder_state_h, encoder_state_c]\n", - "\n", - " self.attention(encoder_outputs,\n", - " setup_memory=True)\n", - " \n", - " decoder_embeddings = self.decoder_embedding(decoder_input)\n", - "\n", - " decoder_initial_state = self.decoder_cell.get_initial_state(\n", - " decoder_embeddings)\n", - " decoder_initial_state = decoder_initial_state.clone(\n", - " cell_state=encoder_state)\n", - " \n", - " if training:\n", - " decoder_outputs, _, _ = self.decoder(\n", - " decoder_embeddings,\n", - " initial_state=decoder_initial_state,\n", - " training=training)\n", - " else:\n", - " start_tokens = tf.zeros_like(encoder_input[:, 0]) + sos_id\n", - " decoder_outputs, _, _ = self.inference_decoder(\n", - " decoder_embeddings,\n", - " initial_state=decoder_initial_state,\n", - " start_tokens=start_tokens,\n", - " end_token=0)\n", - "\n", - " return tf.nn.softmax(decoder_outputs.rnn_output)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "np.random.seed(42)\n", - "tf.random.set_seed(42)\n", - "\n", - "model = DateTranslation()\n", - "optimizer = tf.keras.optimizers.Nadam()\n", - "model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=optimizer,\n", - " metrics=[\"accuracy\"])\n", - "history = model.fit([X_train, X_train_decoder], Y_train, epochs=25,\n", - " validation_data=([X_valid, X_valid_decoder], Y_valid))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Not quite 100% validation accuracy, but close. It took a bit longer to converge this time, but there were also more parameters and more computations per iteration. And we did not use a scheduled sampler.\n", - "\n", - "To use the model, we can write yet another little function:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def fast_predict_date_strs_v2(date_strs):\n", - " X = prepare_date_strs_padded(date_strs)\n", - " X_decoder = tf.zeros(shape=(len(X), max_output_length), dtype=tf.int32)\n", - " Y_probas = model.predict([X, X_decoder])\n", - " Y_pred = tf.argmax(Y_probas, axis=-1)\n", - " return ids_to_date_strs(Y_pred)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "fast_predict_date_strs_v2([\"July 14, 1789\", \"May 01, 2020\"])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "There are still a few interesting features from TF-Addons that you may want to look at:\n", - "* Using a `BeamSearchDecoder` rather than a `BasicDecoder` for inference. Instead of outputing the character with the highest probability, this decoder keeps track of the several candidates, and keeps only the most likely sequences of candidates (see chapter 16 in the book for more details).\n", - "* Setting masks or specifying `sequence_length` if the input or target sequences may have very different lengths.\n", - "* Using a `ScheduledOutputTrainingSampler`, which gives you more flexibility than the `ScheduledEmbeddingTrainingSampler` to decide how to feed the output at time _t_ to the cell at time _t_+1. By default it feeds the outputs directly to cell, without computing the argmax ID and passing it through an embedding layer. Alternatively, you specify a `next_inputs_fn` function that will be used to convert the cell outputs to inputs at the next step." + "Works fine! Next, feel free to write a Transformer version. :)" ] }, { @@ -2742,14 +4058,14 @@ "metadata": {}, "source": [ "## 10.\n", - "_Exercise: Go through TensorFlow's [Neural Machine Translation with Attention tutorial](https://homl.info/nmttuto)._" + "_Exercise: Go through Keras's tutorial for [Natural language image search with a Dual Encoder](https://homl.info/dualtuto). You will learn how to build a model capable of representing both images and text within the same embedding space. This makes it possible to search for images using a text prompt, like in the [CLIP model](https://openai.com/blog/clip/) by OpenAI._ " ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Simply open the Colab and follow its instructions. Alternatively, if you want a simpler example of using TF-Addons's seq2seq implementation for Neural Machine Translation (NMT), look at the solution to the previous question. The last model implementation will give you a simpler example of using TF-Addons to build an NMT model using attention mechanisms." + "Just click the link and follow the instructions." ] }, { @@ -2757,14 +4073,7 @@ "metadata": {}, "source": [ "## 11.\n", - "_Exercise: Use one of the recent language models (e.g., GPT) to generate more convincing Shakespearean text._" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The simplest way to use recent language models is to use the excellent [transformers library](https://huggingface.co/transformers/), open sourced by Hugging Face. It provides many modern neural net architectures (including BERT, GPT-2, RoBERTa, XLM, DistilBert, XLNet and more) for Natural Language Processing (NLP), including many pretrained models. It relies on either TensorFlow or PyTorch. Best of all: it's amazingly simple to use." + "_Exercise: Use the Transformers library to download a pretrained language model capable of generating text (e.g., GPT), and try generating more convincing Shakespearean text. You will need to use the model's `generate()` method—see Hugging Face's documentation for more details._" ] }, { @@ -2776,9 +4085,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 136, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "All model checkpoint layers were used when initializing TFOpenAIGPTLMHeadModel.\n", + "\n", + "All the layers of TFOpenAIGPTLMHeadModel were initialized from the model checkpoint at openai-gpt.\n", + "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFOpenAIGPTLMHeadModel for predictions without further training.\n" + ] + } + ], "source": [ "from transformers import TFOpenAIGPTLMHeadModel\n", "\n", @@ -2794,9 +4114,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 137, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy.\n" + ] + } + ], "source": [ "from transformers import OpenAIGPTTokenizer\n", "\n", @@ -2812,9 +4140,42 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 138, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "{'input_ids': [3570, 1473], 'attention_mask': [1, 1]}" + ] + }, + "execution_count": 138, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "tokenizer(\"hello everyone\")" + ] + }, + { + "cell_type": "code", + "execution_count": 139, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 139, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "prompt_text = \"This royal throne of kings, this sceptred isle\"\n", "encoded_prompt = tokenizer.encode(prompt_text,\n", @@ -2832,9 +4193,50 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 140, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 140, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "num_sequences = 5\n", "length = 40\n", @@ -2862,9 +4264,32 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 141, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "this royal throne of kings, this sceptred isle of the necronomicon is the only place that can unlock it from this dark world. \n", + " i am surprised that i've been able to see it, \" the man named dallon says to\n", + "--------------------------------------------------------------------------------\n", + "this royal throne of kings, this sceptred isle was home to many beloved possessors, such as the mighty astaroth. their wives had been husband and wife to lord teixiara for many generations. \n", + " the high king had his own\n", + "--------------------------------------------------------------------------------\n", + "this royal throne of kings, this sceptred isle is now our home and the land of our fathers!'this was made the standard of the coates, which is at king celebrant's command. \n", + " this was the longest story the coates\n", + "--------------------------------------------------------------------------------\n", + "this royal throne of kings, this sceptred isle has a powerful spirit that can not be severed or erased. it will reign until there is no army in our realm or the light will fade from the sky, and the lands will be stripped of its\n", + "--------------------------------------------------------------------------------\n", + "this royal throne of kings, this sceptred isle will be your final gift to king dragomir. \n", + " good luck, my guards. \n", + " * * * \n", + " a light touch on her arm caused aleria to jolt. \" come on. i think you\n", + "--------------------------------------------------------------------------------\n" + ] + } + ], "source": [ "for sequence in generated_sequences:\n", " text = tokenizer.decode(sequence, clean_up_tokenization_spaces=True)\n", @@ -2903,7 +4328,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.12" + "version": "3.9.10" }, "nav_menu": {}, "toc": {