schuetoliver
/
handson-ml
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix the transformer (use final encoder outputs)

main
Aurélien Geron 2019-05-10 21:30:18 +08:00
parent 7b1c890195
commit f0f83903bd
1 changed files with 10 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
16_nlp_with_rnns_and_attention.ipynb
View File

@ -1116,17 +1116,18 @@
"metadata": {},
"outputs": [],
"source": [
"Z = encoder_in\n",
"for N in range(6):\n",
" encoder_attn = keras.layers.Attention(use_scale=True)\n",
" encoder_in = encoder_attn([encoder_in, encoder_in])\n",
" masked_decoder_attn = keras.layers.Attention(use_scale=True, causal=True)\n",
" decoder_in = masked_decoder_attn([decoder_in, decoder_in])\n",
" decoder_attn = keras.layers.Attention(use_scale=True)\n",
" final_enc = decoder_attn([decoder_in, encoder_in])\n",
" Z = keras.layers.Attention(use_scale=True)([Z, Z])\n",
"\n",
"output_layer = keras.layers.TimeDistributed(\n",
" keras.layers.Dense(vocab_size, activation=\"softmax\"))\n",
"outputs = output_layer(final_enc)"
"encoder_outputs = Z\n",
"Z = decoder_in\n",
"for N in range(6):\n",
" Z = keras.layers.Attention(use_scale=True, causal=True)([Z, Z])\n",
" Z = keras.layers.Attention(use_scale=True)([Z, encoder_outputs])\n",
"\n",
"outputs = keras.layers.TimeDistributed(\n",
" keras.layers.Dense(vocab_size, activation=\"softmax\"))(Z)"
]
},
{