From fb47fab0c8cf7d07e1c6282622829fa112ae7273 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Geron?= Date: Tue, 4 Jun 2019 16:43:14 +0800 Subject: [PATCH] Update docker image for 2nd edition --- docker/.env | 2 +- docker/Dockerfile | 68 +++++++++++++++++------------------ docker/Makefile | 4 +-- docker/README.md | 8 ++--- docker/docker-compose.yml | 10 +++--- docker/nbdime-1-details.patch | 17 --------- docker/nbdime-2-toc.patch | 11 ------ 7 files changed, 45 insertions(+), 75 deletions(-) delete mode 100644 docker/nbdime-1-details.patch delete mode 100644 docker/nbdime-2-toc.patch diff --git a/docker/.env b/docker/.env index 16adf41..fea151a 100644 --- a/docker/.env +++ b/docker/.env @@ -1 +1 @@ -COMPOSE_PROJECT_NAME=handson-ml +COMPOSE_PROJECT_NAME=handson-ml2 diff --git a/docker/Dockerfile b/docker/Dockerfile index 6e9976b..9857df7 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM continuumio/anaconda3:5.2.0 +FROM continuumio/anaconda3:2019.03 RUN apt-get update && apt-get upgrade -y \ && apt-get install -y \ @@ -7,20 +7,37 @@ RUN apt-get update && apt-get upgrade -y \ git \ sudo \ cmake zlib1g-dev libjpeg-dev xvfb libav-tools xorg-dev libboost-all-dev libsdl2-dev swig \ + unzip zip \ && rm -rf /var/lib/apt/lists/* RUN conda update -n base conda +RUN conda install -y \ + joblib \ + PyYAML==3.13 RUN conda install -y -c conda-forge \ - tensorflow \ - jupyter_contrib_nbextensions \ - pyopengl + pyopengl \ + xgboost \ + nbdime +RUN pip install "urlextract" RUN pip install "gym[atari,box2d,classic_control]" +RUN pip install "tensorflow-hub" +RUN pip install "tensorflow-serving-api" +RUN pip install "tfx" +#RUN pip install "tensorflow-addons" +RUN pip install "tf-agents-nightly" +RUN pip install "tfds-nightly" +RUN pip install "tfp-nightly" +RUN pip uninstall -y tensorflow +RUN pip uninstall -y tensorboard +RUN pip install "tf-nightly-2.0-preview" +RUN pip install "tb-nightly" + ARG username ARG userid ARG home=/home/${username} -ARG workdir=${home}/handson-ml +ARG workdir=${home}/handson-ml2 RUN adduser ${username} --uid ${userid} --gecos '' --disabled-password \ && echo "${username} ALL=(root) NOPASSWD:ALL" > /etc/sudoers.d/${username} \ @@ -29,25 +46,17 @@ RUN adduser ${username} --uid ${userid} --gecos '' --disabled-password \ WORKDIR ${workdir} RUN chown ${username}:${username} ${workdir} -USER ${username} - -RUN jupyter contrib nbextension install --user -RUN jupyter nbextension enable toc2/main - - -# INFO: Jupyter and nbdime extension are not totally integrated (anaconda image is py36, -# nbdime checks for py35 at the moment, still the config below enables diffing -# notebooks with nbdiff (and nbdiff support in git diff command) after connecting -# to the container by "make exec" (or "docker-compose exec handson-ml bash") -# You may also try running: -# nbd NOTEBOOK_NAME.ipynb -# to get nbdiff between checkpointed version and current version of the given notebook -USER root -WORKDIR / -RUN conda install -y -c conda-forge nbdime USER ${username} WORKDIR ${workdir} +# The config below enables diffing notebooks with nbdiff (and nbdiff support +# in git diff command) after connecting to the container by "make exec" (or +# "docker-compose exec handson-ml2 bash") +# You may also try running: +# nbdiff NOTEBOOK_NAME.ipynb +# to get nbdiff between checkpointed version and current version of the +# given notebook. + RUN git-nbdiffdriver config --enable --global # INFO: Optionally uncomment any (one) of the following RUN commands below to ignore either @@ -56,19 +65,6 @@ RUN git-nbdiffdriver config --enable --global RUN git config --global diff.jupyternotebook.command 'git-nbdiffdriver diff --ignore-details' -# INFO: Dirty nbdime patching (ignored if not matching) -COPY docker/nbdime-*.patch /tmp/ -USER root -WORKDIR / -RUN patch -d /opt/conda/lib/python3.6/site-packages -p1 --forward --reject-file=- < \ - /tmp/nbdime-1-details.patch || true \ - && patch -d /opt/conda/lib/python3.6/site-packages -p1 --forward --reject-file=- < \ - /tmp/nbdime-2-toc.patch || true -RUN rm /tmp/nbdime-*.patch -USER ${username} -WORKDIR ${workdir} - - COPY docker/bashrc.bash /tmp/ RUN cat /tmp/bashrc.bash >> ${home}/.bashrc RUN echo "export PATH=\"${workdir}/docker/bin:$PATH\"" >> ${home}/.bashrc @@ -81,11 +77,13 @@ RUN sudo rm /tmp/bashrc.bash #RUN cat /tmp/jupyter_notebook_config.py >> ${home}/.jupyter/jupyter_notebook_config.py #RUN sudo rm /tmp/jupyter_notebook_config.py + # INFO: Uncomment the RUN command below to disable git diff paging #RUN git config --global core.pager '' + # INFO: Uncomment the RUN command below for easy and constant notebook URL (just localhost:8888) -# That will switch jupyter to using empty password instead of a token. +# That will switch Jupyter to using empty password instead of a token. # To avoid making a security hole you SHOULD in fact not only uncomment but # regenerate the hash for your own non-empty password and replace the hash below. # You can compute a password hash in any notebook, just run the code: diff --git a/docker/Makefile b/docker/Makefile index f85c49a..c5a98cc 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -4,11 +4,11 @@ help: run: docker-compose up exec: - docker-compose exec handson-ml bash + docker-compose exec handson-ml2 bash build: stop .FORCE docker-compose build rebuild: stop .FORCE docker-compose build --force-rm stop: - docker stop handson-ml || true; docker rm handson-ml || true; + docker stop handson-ml2 || true; docker rm handson-ml2 || true; .FORCE: diff --git a/docker/README.md b/docker/README.md index 037ae22..d7d67da 100644 --- a/docker/README.md +++ b/docker/README.md @@ -18,11 +18,11 @@ Some general knowledge about `docker` infrastructure might be useful (that's an Switch to `docker` directory here and run `make build` (or `docker-compose build`) to build your docker image. That may take some time but is only required once. Or perhaps a few times after you tweak something in a `Dockerfile`. -After the process is finished you have a `handson-ml` image, that will be the base for your experiments. You can confirm that looking on results of `docker images` command. +After the process is finished you have a `handson-ml2` image, that will be the base for your experiments. You can confirm that looking on results of `docker images` command. ### Run the notebooks -Run `make run` (or just `docker-compose up`) to start the jupyter server inside the container (also named `handson-ml`, same as image). Just point your browser to the URL printed on the screen (or just if you enabled password authentication) and you're ready to play with the book's code! +Run `make run` (or just `docker-compose up`) to start the jupyter server inside the container (also named `handson-ml2`, same as image). Just point your browser to the URL printed on the screen (or just if you enabled password authentication) and you're ready to play with the book's code! The server runs in the directory containing the notebooks, and the changes you make from the browser will be persisted there. @@ -30,9 +30,9 @@ You can close the server just by pressing `Ctrl-C` in terminal window. ### Run additional commands in container -Run `make exec` (or `docker-compose exec handson-ml bash`) while the server is running to run an additional `bash` shell inside the `handson-ml` container. Now you're inside the environment prepared within the image. +Run `make exec` (or `docker-compose exec handson-ml2 bash`) while the server is running to run an additional `bash` shell inside the `handson-ml2` container. Now you're inside the environment prepared within the image. -One of the usefull things that can be done there would be starting TensorBoard (for example with simple `tb` command, see bashrc file). +One of the useful things that can be done there would be starting TensorBoard (for example with simple `tb` command, see bashrc file). Another one may be comparing versions of the notebooks using the `nbdiff` command if you haven't got `nbdime` installed locally (it is **way** better than plain `diff` for notebooks). See [Tools for diffing and merging of Jupyter notebooks](https://github.com/jupyter/nbdime) for more details. diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index a5c94d7..12eb355 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,15 +1,15 @@ version: "3" services: - handson-ml: + handson-ml2: build: context: ../ dockerfile: ./docker/Dockerfile args: - username=devel - userid=1000 - container_name: handson-ml - image: handson-ml - restart: unless-stopped + container_name: handson-ml2 + image: handson-ml2 + restart: unless-stopped logging: driver: json-file options: @@ -18,5 +18,5 @@ services: - "8888:8888" - "6006:6006" volumes: - - ../:/home/devel/handson-ml + - ../:/home/devel/handson-ml2 command: /opt/conda/bin/jupyter notebook --ip='0.0.0.0' --port=8888 --no-browser diff --git a/docker/nbdime-1-details.patch b/docker/nbdime-1-details.patch deleted file mode 100644 index 98f76d6..0000000 --- a/docker/nbdime-1-details.patch +++ /dev/null @@ -1,17 +0,0 @@ ---- a/nbdime/diffing/notebooks.py -+++ b/nbdime/diffing/notebooks.py -@@ -548,8 +548,12 @@ def set_notebook_diff_targets(sources=True, outputs=True, attachments=True, meta - metadata_keys = ("/cells/*/metadata", "/metadata", "/cells/*/outputs/*/metadata") - if metadata: - for key in metadata_keys: -- if key in notebook_differs: -- del notebook_differs[key] -+ if details: -+ if key in notebook_differs: -+ del notebook_differs[key] -+ else: -+ notebook_differs[key] = diff_ignore_keys( -+ inner_differ=diff, ignore_keys=['collapsed', 'autoscroll', 'deletable', 'editable']) - else: - for key in metadata_keys: - notebook_differs[key] = diff_ignore diff --git a/docker/nbdime-2-toc.patch b/docker/nbdime-2-toc.patch deleted file mode 100644 index 4924e66..0000000 --- a/docker/nbdime-2-toc.patch +++ /dev/null @@ -1,11 +0,0 @@ ---- a/nbdime/diffing/notebooks.py -+++ b/nbdime/diffing/notebooks.py -@@ -553,7 +553,7 @@ - del notebook_differs[key] - else: - notebook_differs[key] = diff_ignore_keys( -- inner_differ=diff, ignore_keys=['collapsed', 'autoscroll', 'deletable', 'editable']) -+ inner_differ=diff, ignore_keys=['toc', 'collapsed', 'autoscroll', 'deletable', 'editable']) - else: - for key in metadata_keys: - notebook_differs[key] = diff_ignore