From 4a39541c3aec2cfb6d60366d4d24da3a6f8ef267 Mon Sep 17 00:00:00 2001 From: Josh Taylor Date: Fri, 1 Mar 2019 21:31:17 +1300 Subject: [PATCH 1/3] Upgrade Tensorflow to version 1.13.1 * Upgrade Tensorflow to version 1.13.1. * Upgrade to CUDA 10 as Tensorflow GPU binaries are now built against this version. See #2117 --- containers/base/Dockerfile | 4 ++-- containers/base/Dockerfile.gpu | 4 ++-- containers/base/build.gpu.sh | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/containers/base/Dockerfile b/containers/base/Dockerfile index 4f66ded6a..edf33a7c0 100644 --- a/containers/base/Dockerfile +++ b/containers/base/Dockerfile @@ -118,7 +118,7 @@ RUN echo "deb-src http://ftp.us.debian.org/debian testing main" >> /etc/apt/sour google-cloud-monitoring==0.28.0 \ lime==0.1.1.23 \ protobuf==3.6.1 \ - tensorflow==1.8.0 && \ + tensorflow==1.13.1 && \ source deactivate && \ # Clean up before setting up the Python3 env. conda clean -tipsy && \ @@ -230,7 +230,7 @@ RUN source activate $PYTHON_3_ENV && \ google-cloud-monitoring==0.28.0 \ lime==0.1.1.23 \ protobuf==3.6.1 \ - tensorflow==1.8.0 + tensorflow==1.13.1 ENV LANG en_US.UTF-8 diff --git a/containers/base/Dockerfile.gpu b/containers/base/Dockerfile.gpu index 3800fce50..81204e7a1 100644 --- a/containers/base/Dockerfile.gpu +++ b/containers/base/Dockerfile.gpu @@ -20,8 +20,8 @@ SHELL ["/bin/bash", "-c"] # Download and Install GPU specific packages RUN source activate py2env && \ - pip install -U --upgrade-strategy only-if-needed --no-cache-dir tensorflow-gpu==1.8.0 tflearn h5py && \ + pip install -U --upgrade-strategy only-if-needed --no-cache-dir tensorflow-gpu==1.13.1 tflearn h5py && \ source deactivate && \ source activate py3env && \ - pip install -U --upgrade-strategy only-if-needed --no-cache-dir tensorflow-gpu==1.8.0 tflearn h5py && \ + pip install -U --upgrade-strategy only-if-needed --no-cache-dir tensorflow-gpu==1.13.1 tflearn h5py && \ source deactivate diff --git a/containers/base/build.gpu.sh b/containers/base/build.gpu.sh index 5d38f30d9..6432e1284 100755 --- a/containers/base/build.gpu.sh +++ b/containers/base/build.gpu.sh @@ -32,11 +32,11 @@ fi trap 'rm -rf pydatalab' exit -docker pull nvidia/cuda:9.0-cudnn7-devel-ubuntu16.04 +docker pull nvidia/cuda:10.0-cudnn7-devel-ubuntu16.04 # Docker tag flags changed in an incompatible way between versions. # The Datalab Jenkins build still uses the old one, so try it both ways. -if ! $(docker tag -f nvidia/cuda:9.0-cudnn7-devel-ubuntu16.04 datalab-external-base-image 2> /dev/null); then - docker tag nvidia/cuda:9.0-cudnn7-devel-ubuntu16.04 datalab-external-base-image +if ! $(docker tag -f nvidia/cuda:10.0-cudnn7-devel-ubuntu16.04 datalab-external-base-image 2> /dev/null); then + docker tag nvidia/cuda:10.0-cudnn7-devel-ubuntu16.04 datalab-external-base-image fi docker build ${DOCKER_BUILD_ARGS} -t datalab-core-gpu . docker build ${DOCKER_BUILD_ARGS} -f Dockerfile.gpu -t datalab-base-gpu . From a46cb21ff14cedaa50f91b414093f5d80b722fca Mon Sep 17 00:00:00 2001 From: Josh Taylor Date: Sat, 2 Mar 2019 21:41:18 +1300 Subject: [PATCH 2/3] Update CUDA cudnn7 version in cloud build config See: #2117 --- containers/base/Dockerfile | 2 +- tools/release/cloudbuild.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/containers/base/Dockerfile b/containers/base/Dockerfile index edf33a7c0..5d8ef1428 100644 --- a/containers/base/Dockerfile +++ b/containers/base/Dockerfile @@ -15,7 +15,7 @@ # We use different base images for GPU vs CPU Dockerfiles, so we expect # that the appropriate image is pulled and tagged locally. # CPU should use ubuntu:16.04 -# and GPU uses nvidia/cuda:9.1-cudnn7-devel-ubuntu16.04 +# and GPU uses nvidia/cuda:10.0-cudnn7-devel-ubuntu16.04 FROM datalab-external-base-image MAINTAINER Google Cloud DataLab diff --git a/tools/release/cloudbuild.yaml b/tools/release/cloudbuild.yaml index 4f0c2ab68..143b09bf7 100644 --- a/tools/release/cloudbuild.yaml +++ b/tools/release/cloudbuild.yaml @@ -81,10 +81,10 @@ steps: ## Second, we build the GPU base image - name: 'gcr.io/cloud-builders/docker' - args: ['pull', 'nvidia/cuda:9.0-cudnn7-devel-ubuntu16.04'] + args: ['pull', 'nvidia/cuda:10.0-cudnn7-devel-ubuntu16.04'] id: 'pullNvidiaUbuntu' - name: 'gcr.io/cloud-builders/docker' - args: ['tag', 'nvidia/cuda:9.0-cudnn7-devel-ubuntu16.04', 'datalab-external-base-image'] + args: ['tag', 'nvidia/cuda:10.0-cudnn7-devel-ubuntu16.04', 'datalab-external-base-image'] id: 'tagNvidiaUbuntu' waitFor: ['buildBase', 'pullNvidiaUbuntu'] - name: 'gcr.io/cloud-builders/docker' From 2f599689172dd7ba0e52097891c4033897a59643 Mon Sep 17 00:00:00 2001 From: Josh Taylor Date: Sat, 2 Mar 2019 22:31:53 +1300 Subject: [PATCH 3/3] Add missing Tensorflow and Keras licenses See: #2117 --- containers/base/third_party_licenses.csv | 2 ++ 1 file changed, 2 insertions(+) diff --git a/containers/base/third_party_licenses.csv b/containers/base/third_party_licenses.csv index fd903efa2..3df390d28 100644 --- a/containers/base/third_party_licenses.csv +++ b/containers/base/third_party_licenses.csv @@ -71,6 +71,7 @@ tensorflow,https://raw.githubusercontent.com/tensorflow/tensorflow/master/LICENS tensorboard,https://raw.githubusercontent.com/tensorflow/tensorboard/master/LICENSE,Apache 2.0 keras,https://raw.githubusercontent.com/keras-team/keras/master/LICENSE,MIT Keras-Applications,https://raw.githubusercontent.com/keras-team/keras-applications/master/LICENSE,MIT +Keras-Preprocessing,https://raw.githubusercontent.com/keras-team/keras-preprocessing/master/LICENSE,MIT jupyter-tensorboard,https://raw.githubusercontent.com/lspvic/jupyter_tensorboard/master/LICENSE,MIT torch,https://raw.githubusercontent.com/pytorch/pytorch/master/LICENSE,BSD-3-clause torchvision,https://raw.githubusercontent.com/pytorch/vision/master/LICENSE,BSD-3-clause @@ -168,6 +169,7 @@ scipy,https://raw.githubusercontent.com/scipy/scipy/master/LICENSE.txt,BSD tensorflow-tensorboard,https://raw.githubusercontent.com/tensorflow/tensorboard/master/LICENSE,Apache 2.0 tensorflow-data-validation,https://raw.githubusercontent.com/tensorflow/data-validation/master/LICENSE,Apache 2.0 tensorflow-metadata,https://raw.githubusercontent.com/tensorflow/metadata/master/LICENSE,Apache 2.0 +tensorflow-estimator,https://raw.githubusercontent.com/tensorflow/estimator/master/LICENSE,Apache 2.0 defusedxml,https://raw.githubusercontent.com/tiran/defusedxml/master/LICENSE,PSF conda,https://raw.githubusercontent.com/conda/conda/master/LICENSE.txt,3-Clause BSD packaging,https://raw.githubusercontent.com/pypa/packaging/master/LICENSE,BSD or Apache 2.0