From 5959ea73300332cf0106844aed32287641391e3f Mon Sep 17 00:00:00 2001 From: Kyle Kelley Date: Sat, 26 Jul 2014 09:58:24 -0500 Subject: [PATCH 001/362] Add An Article --- runtime/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/README.md b/runtime/README.md index 98ad08fd..ec732081 100644 --- a/runtime/README.md +++ b/runtime/README.md @@ -1,6 +1,6 @@ # google/python-runtime -[`google/python-runtime`](https://index.docker.io/u/google/python-runtime) is a [docker](https://docker.io) base image that makes it easy to dockerize standard [Python](http://python.org) application. +[`google/python-runtime`](https://index.docker.io/u/google/python-runtime) is a [docker](https://docker.io) base image that makes it easy to dockerize a standard [Python](http://python.org) application. It can automatically bundle a Python application and its dependencies with a single line Dockerfile. From d77930e89e30ec6158ed198ce00f72833be93a30 Mon Sep 17 00:00:00 2001 From: henryvps Date: Thu, 19 Mar 2015 13:33:14 +0200 Subject: [PATCH 002/362] Missing -r from pip install command --- base/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/README.md b/base/README.md index 7664c1d6..38ab1501 100644 --- a/base/README.md +++ b/base/README.md @@ -13,7 +13,7 @@ It serves as a base for the [`google/python-runtime`](https://index.docker.io/u/ WORKDIR /app RUN virtualenv /env ADD requirements.txt /app/requirements.txt - RUN /env/bin/pip install requirements.txt + RUN /env/bin/pip install -r requirements.txt ADD . /app CMD [] From 9e5909574da4a2d45bfa293b2bb50f1ddff80322 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Mon, 24 Aug 2015 16:20:58 -0700 Subject: [PATCH 003/362] Initial commit for the new runtime. --- base/Dockerfile | 5 ----- base/README.md | 24 ------------------------ runtime/Dockerfile | 29 ++++++++++++++++++++++++++--- 3 files changed, 26 insertions(+), 32 deletions(-) delete mode 100644 base/Dockerfile delete mode 100644 base/README.md diff --git a/base/Dockerfile b/base/Dockerfile deleted file mode 100644 index 9e464eb3..00000000 --- a/base/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM google/debian:wheezy - -RUN apt-get update -y && apt-get install --no-install-recommends -y -q build-essential python2.7 python2.7-dev python-pip git -RUN pip install -U pip -RUN pip install virtualenv diff --git a/base/README.md b/base/README.md deleted file mode 100644 index 38ab1501..00000000 --- a/base/README.md +++ /dev/null @@ -1,24 +0,0 @@ -# google/python - -[`google/python`](https://index.docker.io/u/google/python) is a [docker](https://docker.io) base image that bundles the stable version of [python](http://python.org) installed from [debian stable](https://packages.debian.org/stable/) and [pip](https://pip.pypa.io/en/latest/) and [virtualenv](https://virtualenv.pypa.io/) installed from [PyPI](https://pypi.python.org/pypi). - -It serves as a base for the [`google/python-runtime`](https://index.docker.io/u/google/python-runtime) image. - -## Usage - -- Create a Dockerfile in your python application directory with the following content: - - FROM google/python - - WORKDIR /app - RUN virtualenv /env - ADD requirements.txt /app/requirements.txt - RUN /env/bin/pip install -r requirements.txt - ADD . /app - - CMD [] - ENTRYPOINT ["/env/bin/python", "/app/main.py"] - -- Run the following command in your application directory: - - docker build -t my/app . diff --git a/runtime/Dockerfile b/runtime/Dockerfile index 90523697..206f2264 100644 --- a/runtime/Dockerfile +++ b/runtime/Dockerfile @@ -1,11 +1,34 @@ -FROM google/python +# The Google App Engine base image is debian (wheezy) with ca-certificates +# installed. +FROM gcr.io/google_appengine/base +# Install Python, pip, and C dev libraries necessary to compile the most popular +# Python libraries. +RUN apt-get -q update && \ + apt-get install --no-install-recommends -y -q \ + python2.7 python-pip python-dev build-essential git mercurial \ + libffi-dev libssl-dev libxml2-dev \ + libxslt1-dev libpq-dev libmysqlclient-dev libcurl4-openssl-dev \ + libjpeg-dev zlib1g-dev libpng12-dev && \ + apt-get clean && rm /var/lib/apt/lists/*_* + +# Upgrade pip (debian package version tends to run a few version behind) and +# install virtualenv system-wide. +RUN pip install --upgrade pip virtualenv + +EXPOSE 8080 + +# Install all application dependencies into a new virtualenv. WORKDIR /app ONBUILD RUN virtualenv /env ONBUILD ADD requirements.txt /app/requirements.txt ONBUILD RUN /env/bin/pip install -r /app/requirements.txt ONBUILD ADD . /app -EXPOSE 8080 +# Set virtualenv environment variables. This is equivalent to running source +# /env/bin/activate +ENV VIRTUAL_ENV /env +ENV PATH /env/bin:$PATH + CMD [] -ENTRYPOINT ["/env/bin/python", "main.py"] +# The user's Dockerfile must specify an entrypoint. From 564b5148f11826cb7f076d8fda4f2ad46c845abf Mon Sep 17 00:00:00 2001 From: Andrew Gorcester Date: Mon, 24 Aug 2015 16:25:56 -0700 Subject: [PATCH 004/362] Add port number to Dockerfile --- runtime/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/runtime/Dockerfile b/runtime/Dockerfile index 206f2264..54679d19 100644 --- a/runtime/Dockerfile +++ b/runtime/Dockerfile @@ -29,6 +29,7 @@ ONBUILD ADD . /app # /env/bin/activate ENV VIRTUAL_ENV /env ENV PATH /env/bin:$PATH +ENV PORT 8080 CMD [] # The user's Dockerfile must specify an entrypoint. From 7a99ce2a242ccfe6b082a0b989086d94c799a39d Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Wed, 26 Aug 2015 10:41:18 -0700 Subject: [PATCH 005/362] Adding basic python3.4 runtime --- runtime-python3/Dockerfile | 41 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 runtime-python3/Dockerfile diff --git a/runtime-python3/Dockerfile b/runtime-python3/Dockerfile new file mode 100644 index 00000000..48a477e8 --- /dev/null +++ b/runtime-python3/Dockerfile @@ -0,0 +1,41 @@ +# The Google App Engine base image is debian (wheezy) with ca-certificates +# installed. +#FROM gcr.io/google_appengine/base +# Will switch back to google_appengine/base when it switches to jessie. +FROM debian:jessie +ENV DEBIAN_FRONTEND noninteractive +ENV PORT 8080 + +# Ideally, this should be moved into the base image +ENV LC_ALL=C.UTF-8 +ENV LANG=C.UTF-8 +EXPOSE 8080 + +# Install Python, pip, and C dev libraries necessary to compile the most popular +# Python libraries. +RUN apt-get -q update && \ + apt-get install --no-install-recommends -y -q \ + python3 python3-dev python3-pip build-essential git mercurial \ + libffi-dev libssl-dev libxml2-dev \ + libxslt1-dev libpq-dev libmysqlclient-dev libcurl4-openssl-dev \ + libjpeg-dev zlib1g-dev libpng12-dev \ + && apt-get clean && rm /var/lib/apt/lists/*_* + +# Upgrade pip (debian package version tends to run a few versions behind) and +# install virtualenv system-wide. +RUN pip3 install --upgrade pip virtualenv + +# Install all application dependencies into a new virtualenv. +WORKDIR /app +RUN virtualenv /env +ONBUILD ADD requirements.txt /app/requirements.txt +ONBUILD RUN /env/bin/pip install -r /app/requirements.txt +ONBUILD ADD . /app + +# Set virtualenv environment variables. This is equivalent to running source +# /env/bin/activate +ENV VIRTUAL_ENV /env +ENV PATH /env/bin:$PATH + +CMD [] +# The user's Dockerfile must specify an entrypoint. From 0473c5a6da5fd030d949a159ba23e777d2bf16fd Mon Sep 17 00:00:00 2001 From: Andrew Gorcester Date: Tue, 13 Oct 2015 11:01:30 -0700 Subject: [PATCH 006/362] Remove ONBUILD to delegate responsibility to user Dockerfile; add Python 3 installation step --- runtime/Dockerfile | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/runtime/Dockerfile b/runtime/Dockerfile index 54679d19..8031420a 100644 --- a/runtime/Dockerfile +++ b/runtime/Dockerfile @@ -1,4 +1,4 @@ -# The Google App Engine base image is debian (wheezy) with ca-certificates +# The Google App Engine base image is debian (jessie) with ca-certificates # installed. FROM gcr.io/google_appengine/base @@ -6,7 +6,7 @@ FROM gcr.io/google_appengine/base # Python libraries. RUN apt-get -q update && \ apt-get install --no-install-recommends -y -q \ - python2.7 python-pip python-dev build-essential git mercurial \ + python2.7 python3.4 python-pip python-dev build-essential git mercurial \ libffi-dev libssl-dev libxml2-dev \ libxslt1-dev libpq-dev libmysqlclient-dev libcurl4-openssl-dev \ libjpeg-dev zlib1g-dev libpng12-dev && \ @@ -18,18 +18,10 @@ RUN pip install --upgrade pip virtualenv EXPOSE 8080 -# Install all application dependencies into a new virtualenv. +RUN ln -s /home/vmagent/app /app WORKDIR /app -ONBUILD RUN virtualenv /env -ONBUILD ADD requirements.txt /app/requirements.txt -ONBUILD RUN /env/bin/pip install -r /app/requirements.txt -ONBUILD ADD . /app -# Set virtualenv environment variables. This is equivalent to running source -# /env/bin/activate -ENV VIRTUAL_ENV /env -ENV PATH /env/bin:$PATH ENV PORT 8080 CMD [] -# The user's Dockerfile must specify an entrypoint. +# The user's Dockerfile must specify an entrypoint with ENTRYPOINT or CMD. From ac103b0582b6b94ba450b9449716293726dd29be Mon Sep 17 00:00:00 2001 From: Andrew Gorcester Date: Thu, 29 Oct 2015 13:15:53 -0700 Subject: [PATCH 007/362] Move runtime directory contents into root and delete outdated samples --- runtime/Dockerfile => Dockerfile | 0 README.md | 41 +++++++++++++++++++++++++++----- hello/Dockerfile | 1 - hello/README.md | 11 --------- hello/main.py | 9 ------- hello/requirements.txt | 1 - runtime-python3/Dockerfile | 41 -------------------------------- runtime/README.md | 36 ---------------------------- 8 files changed, 35 insertions(+), 105 deletions(-) rename runtime/Dockerfile => Dockerfile (100%) delete mode 100644 hello/Dockerfile delete mode 100644 hello/README.md delete mode 100644 hello/main.py delete mode 100644 hello/requirements.txt delete mode 100644 runtime-python3/Dockerfile delete mode 100644 runtime/README.md diff --git a/runtime/Dockerfile b/Dockerfile similarity index 100% rename from runtime/Dockerfile rename to Dockerfile diff --git a/README.md b/README.md index 8f96b8e5..ec732081 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,36 @@ -python-docker -============= +# google/python-runtime + +[`google/python-runtime`](https://index.docker.io/u/google/python-runtime) is a [docker](https://docker.io) base image that makes it easy to dockerize a standard [Python](http://python.org) application. + +It can automatically bundle a Python application and its dependencies with a single line Dockerfile. + +It is based on [`google/python`](https://index.docker.io/u/google/python) base image. + +## Usage + +- Create a Dockerfile in your python application directory with the following content: + + FROM google/python-runtime + +- Run the following command in your application directory: + + docker build -t app . + +## Sample + +See the [sources](/hello) for [`google/python-hello`](https://index.docker.io/u/google/python-hello) based on this image. + +## Notes + +The image assumes that your application: + +- has a [`requirements.txt`](https://pip.pypa.io/en/latest/user_guide.html#requirements-files) file to specify its dependencies +- listens on port `8080` +- either has a `main.py` script as entrypoint or defines `ENTRYPOINT ["/env/bin/python", "/app/some_other_file.py"]` in its `Dockerfile` + +When building your application docker image, `ONBUILD` triggers: + +- Create a new virtualenv under the `/env` directory in the container +- Fetch the dependencies listed in `requirements.txt` into the virtualenv using `pip install` and leverage docker caching appropriately +- Copy the application sources under the `/app` directory in the container -This repository contains the sources for the following [docker](https://docker.io) base images: -- [`google/python`](/base) -- [`google/python-runtime`](/runtime) -- [`google/python-hello`](/hello) diff --git a/hello/Dockerfile b/hello/Dockerfile deleted file mode 100644 index acbdb568..00000000 --- a/hello/Dockerfile +++ /dev/null @@ -1 +0,0 @@ -FROM google/python-runtime diff --git a/hello/README.md b/hello/README.md deleted file mode 100644 index 46359905..00000000 --- a/hello/README.md +++ /dev/null @@ -1,11 +0,0 @@ -# google/python-hello - -[`google/python-hello`](https://index.docker.io/u/google/python-hello) is a [docker](https://docker.io) image for the [Flask microframework](http://flask.pocoo.org/) hello world application. - -It is based on [`google/python-runtime`](https://index.docker.io/u/google/python-runtime) base image and listen on port `8080`. - -## Usage - -- Run the following command - - docker run -p 8080 google/python-hello diff --git a/hello/main.py b/hello/main.py deleted file mode 100644 index cf978895..00000000 --- a/hello/main.py +++ /dev/null @@ -1,9 +0,0 @@ -from flask import Flask -app = Flask(__name__) - -@app.route("/") -def hello(): - return "Hello World!" - -if __name__ == "__main__": - app.run(host='0.0.0.0', port=8080, debug=True) diff --git a/hello/requirements.txt b/hello/requirements.txt deleted file mode 100644 index 880a7bc4..00000000 --- a/hello/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -Flask==0.10 diff --git a/runtime-python3/Dockerfile b/runtime-python3/Dockerfile deleted file mode 100644 index 48a477e8..00000000 --- a/runtime-python3/Dockerfile +++ /dev/null @@ -1,41 +0,0 @@ -# The Google App Engine base image is debian (wheezy) with ca-certificates -# installed. -#FROM gcr.io/google_appengine/base -# Will switch back to google_appengine/base when it switches to jessie. -FROM debian:jessie -ENV DEBIAN_FRONTEND noninteractive -ENV PORT 8080 - -# Ideally, this should be moved into the base image -ENV LC_ALL=C.UTF-8 -ENV LANG=C.UTF-8 -EXPOSE 8080 - -# Install Python, pip, and C dev libraries necessary to compile the most popular -# Python libraries. -RUN apt-get -q update && \ - apt-get install --no-install-recommends -y -q \ - python3 python3-dev python3-pip build-essential git mercurial \ - libffi-dev libssl-dev libxml2-dev \ - libxslt1-dev libpq-dev libmysqlclient-dev libcurl4-openssl-dev \ - libjpeg-dev zlib1g-dev libpng12-dev \ - && apt-get clean && rm /var/lib/apt/lists/*_* - -# Upgrade pip (debian package version tends to run a few versions behind) and -# install virtualenv system-wide. -RUN pip3 install --upgrade pip virtualenv - -# Install all application dependencies into a new virtualenv. -WORKDIR /app -RUN virtualenv /env -ONBUILD ADD requirements.txt /app/requirements.txt -ONBUILD RUN /env/bin/pip install -r /app/requirements.txt -ONBUILD ADD . /app - -# Set virtualenv environment variables. This is equivalent to running source -# /env/bin/activate -ENV VIRTUAL_ENV /env -ENV PATH /env/bin:$PATH - -CMD [] -# The user's Dockerfile must specify an entrypoint. diff --git a/runtime/README.md b/runtime/README.md deleted file mode 100644 index ec732081..00000000 --- a/runtime/README.md +++ /dev/null @@ -1,36 +0,0 @@ -# google/python-runtime - -[`google/python-runtime`](https://index.docker.io/u/google/python-runtime) is a [docker](https://docker.io) base image that makes it easy to dockerize a standard [Python](http://python.org) application. - -It can automatically bundle a Python application and its dependencies with a single line Dockerfile. - -It is based on [`google/python`](https://index.docker.io/u/google/python) base image. - -## Usage - -- Create a Dockerfile in your python application directory with the following content: - - FROM google/python-runtime - -- Run the following command in your application directory: - - docker build -t app . - -## Sample - -See the [sources](/hello) for [`google/python-hello`](https://index.docker.io/u/google/python-hello) based on this image. - -## Notes - -The image assumes that your application: - -- has a [`requirements.txt`](https://pip.pypa.io/en/latest/user_guide.html#requirements-files) file to specify its dependencies -- listens on port `8080` -- either has a `main.py` script as entrypoint or defines `ENTRYPOINT ["/env/bin/python", "/app/some_other_file.py"]` in its `Dockerfile` - -When building your application docker image, `ONBUILD` triggers: - -- Create a new virtualenv under the `/env` directory in the container -- Fetch the dependencies listed in `requirements.txt` into the virtualenv using `pip install` and leverage docker caching appropriately -- Copy the application sources under the `/app` directory in the container - From 8659af91ab088fd362414f64589f98f748e61324 Mon Sep 17 00:00:00 2001 From: Jonathan Wayne Parrott Date: Thu, 29 Oct 2015 14:35:47 -0700 Subject: [PATCH 008/362] Update README.md --- README.md | 60 +++++++++++++++++++++++++++++-------------------------- 1 file changed, 32 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index ec732081..65582e14 100644 --- a/README.md +++ b/README.md @@ -1,36 +1,40 @@ -# google/python-runtime +# Google Cloud Platform Python Docker Image -[`google/python-runtime`](https://index.docker.io/u/google/python-runtime) is a [docker](https://docker.io) base image that makes it easy to dockerize a standard [Python](http://python.org) application. +This repository contains the source for the `gcr.io/google_appengine/python` [docker](https://docker.io) base image. This image can be used as the base image for running applications on [Google App Engine Managed VMs](https://cloud.google.com/appengine), [Google Container Engine](https://cloud.google.com/container-engine), or any other Docker host. -It can automatically bundle a Python application and its dependencies with a single line Dockerfile. +This image is based on Debian Jessie and contains packages required to build most of the popular Python libraries. -It is based on [`google/python`](https://index.docker.io/u/google/python) base image. +## App Engine -## Usage +To generate a Dockerfile that uses this image as a base, use the [`Cloud SDK`](https://cloud.google.com/sdk/gcloud/reference/preview/app/gen-config): -- Create a Dockerfile in your python application directory with the following content: + gcloud preview app gen-config --custom - FROM google/python-runtime +You can then modify the `Dockerfile` and `.dockerignore` as needed for you application. -- Run the following command in your application directory: - - docker build -t app . - -## Sample +## Container Engine & other Docker hosts. -See the [sources](/hello) for [`google/python-hello`](https://index.docker.io/u/google/python-hello) based on this image. - -## Notes - -The image assumes that your application: - -- has a [`requirements.txt`](https://pip.pypa.io/en/latest/user_guide.html#requirements-files) file to specify its dependencies -- listens on port `8080` -- either has a `main.py` script as entrypoint or defines `ENTRYPOINT ["/env/bin/python", "/app/some_other_file.py"]` in its `Dockerfile` - -When building your application docker image, `ONBUILD` triggers: - -- Create a new virtualenv under the `/env` directory in the container -- Fetch the dependencies listed in `requirements.txt` into the virtualenv using `pip install` and leverage docker caching appropriately -- Copy the application sources under the `/app` directory in the container - +For other docker hosts, you'll need to create a `Dockerfile` based on this image that copies your application code, installs dependencies, and declares an command or entrypoint. For example: + + FROM gcr.io/google_appengine/python + + # Create a virtualenv for dependencies. This isolates these packages from + # system-level packages. + RUN virtualenv /env + + # Setting these environment variables are the same as running + # source /env/bin/activate + ENV VIRTUAL_ENV /env + ENV PATH /env/bin:$PATH + + # Copy the application's requirements.txt and run pip to install all + # dependencies into the virtualenv. + ADD requirements.txt /app/requirements.txt + RUN pip install -r /app/requirements.txt + + # Add the application source code. + ADD . /app + + # Run a WSGI server to serve the application. gunicorn must be declared as + # a dependency in requirements.txt. + CMD gunicorn -b :$PORT main:app From b08b47464fba4fb174171181a83d71e90dea4e46 Mon Sep 17 00:00:00 2001 From: Jonathan Wayne Parrott Date: Wed, 11 Nov 2015 12:11:01 -0800 Subject: [PATCH 009/362] Include python 3.4 dev headers These are required to build binary packages for python 3. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 8031420a..9893bb88 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ FROM gcr.io/google_appengine/base # Python libraries. RUN apt-get -q update && \ apt-get install --no-install-recommends -y -q \ - python2.7 python3.4 python-pip python-dev build-essential git mercurial \ + python2.7 python3.4 python2.7-dev python3.4-dev python-pip build-essential git mercurial \ libffi-dev libssl-dev libxml2-dev \ libxslt1-dev libpq-dev libmysqlclient-dev libcurl4-openssl-dev \ libjpeg-dev zlib1g-dev libpng12-dev && \ From 3c6e5ccbb4b33f062a13eff15764625038b75eea Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Thu, 3 Dec 2015 10:22:22 -0800 Subject: [PATCH 010/362] Adding language environment variable to fix IO encoding for Python 3 --- Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile b/Dockerfile index 9893bb88..64ae4043 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,6 +12,9 @@ RUN apt-get -q update && \ libjpeg-dev zlib1g-dev libpng12-dev && \ apt-get clean && rm /var/lib/apt/lists/*_* +# Setup locale. This prevents Python 3 IO encoding issues. +ENV LANG C.UTF-8 + # Upgrade pip (debian package version tends to run a few version behind) and # install virtualenv system-wide. RUN pip install --upgrade pip virtualenv From d72ad968a0dd01c9e5c3f41f59c83870472d1ff7 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Thu, 25 Feb 2016 15:39:26 -0800 Subject: [PATCH 011/362] Updating readme with contributing and license information. --- CONTRIB.md | 20 -------------------- CONTRIBUTING.md | 35 +++++++++++++++++++++++++++++++++++ README.md | 8 ++++++++ 3 files changed, 43 insertions(+), 20 deletions(-) delete mode 100644 CONTRIB.md create mode 100644 CONTRIBUTING.md diff --git a/CONTRIB.md b/CONTRIB.md deleted file mode 100644 index 0e104db5..00000000 --- a/CONTRIB.md +++ /dev/null @@ -1,20 +0,0 @@ -# How to become a contributor and submit your own code - -## Contributor License Agreements - -We'd love to accept your patches! Before we can take them, we have to jump a couple of legal hurdles. - -Please fill out either the individual or corporate Contributor License Agreement (CLA). - - * If you are an individual writing original source code and you're sure you own the intellectual property, then you'll need to sign an [individual CLA](http://code.google.com/legal/individual-cla-v1.0.html). - * If you work for a company that wants to allow you to contribute your work, then you'll need to sign a [corporate CLA](http://code.google.com/legal/corporate-cla-v1.0.html). - -Follow either of the two links above to access the appropriate CLA and instructions for how to sign and return it. Once we receive it, we'll be able to accept your pull requests. - -## Contributing A Patch - -1. Submit an issue describing your proposed change to the repo in question. -1. The repo owner will respond to your issue promptly. -1. If your proposed change is accepted, and you haven't already done so, sign a Contributor License Agreement (see details above). -1. Fork the desired repo, develop and test your code changes. -1. Submit a pull request. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..bb1ec7cb --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,35 @@ +# How to become a contributor and submit your own code + +## Contributor License Agreements + +We'd love to accept your sample apps and patches! Before we can take them, we +have to jump a couple of legal hurdles. + +Please fill out either the individual or corporate Contributor License +Agreement (CLA). + + * If you are an individual writing original source code and you're sure you + own the intellectual property, then you'll need to sign an [individual CLA] + (https://developers.google.com/open-source/cla/individual). + * If you work for a company that wants to allow you to contribute your work, + then you'll need to sign a [corporate CLA] + (https://developers.google.com/open-source/cla/corporate). + +Follow either of the two links above to access the appropriate CLA and +instructions for how to sign and return it. Once we receive it, we'll +be able to accept your pull requests. + +## Contributing A Patch + +1. Submit an issue describing your proposed change to the repo in question. +1. The repo owner will respond to your issue promptly. +1. If your proposed change is accepted, and you haven't already done so, sign a + Contributor License Agreement (see details above). +1. Fork the desired repo, develop and test your code changes. +1. Ensure that your code adheres to the existing style in the sample to which + you are contributing. Refer to the + [Google Cloud Platform Samples Style Guide] + (https://github.com/GoogleCloudPlatform/Template/wiki/style.html) for the + recommended coding standards for this organization. +1. Ensure that your code has an appropriate set of unit tests which all pass. +1. Submit a pull request. diff --git a/README.md b/README.md index 65582e14..10c41cdb 100644 --- a/README.md +++ b/README.md @@ -38,3 +38,11 @@ For other docker hosts, you'll need to create a `Dockerfile` based on this image # Run a WSGI server to serve the application. gunicorn must be declared as # a dependency in requirements.txt. CMD gunicorn -b :$PORT main:app + +## Contributing changes + +* See [CONTRIBUTING.md](CONTRIBUTING.md) + +## Licensing + +* See [LICENSE](LICENSE) From a9e8f4394177179fd5a47632f75abefbd1b07a1a Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Thu, 25 Feb 2016 15:52:33 -0800 Subject: [PATCH 012/362] Adding scipy dependencies. --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 64ae4043..e5a9394c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,9 @@ RUN apt-get -q update && \ python2.7 python3.4 python2.7-dev python3.4-dev python-pip build-essential git mercurial \ libffi-dev libssl-dev libxml2-dev \ libxslt1-dev libpq-dev libmysqlclient-dev libcurl4-openssl-dev \ - libjpeg-dev zlib1g-dev libpng12-dev && \ + libjpeg-dev zlib1g-dev libpng12-dev \ + gfortran libblas-dev liblapack-dev libatlas-dev libquadmath0 \ + && \ apt-get clean && rm /var/lib/apt/lists/*_* # Setup locale. This prevents Python 3 IO encoding issues. From 9aa0f084a2340847aefdc3bd21c47055d7949f0d Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Thu, 25 Feb 2016 16:27:14 -0800 Subject: [PATCH 013/362] Adding basic test setup, adding dependencies required to get tests to pass. --- .travis.yml | 6 + Dockerfile | 1 + tests/python2-libraries/Dockerfile | 7 + tests/python2-libraries/requirements.txt | 211 +++++++++++++++++++++++ 4 files changed, 225 insertions(+) create mode 100644 .travis.yml create mode 100644 tests/python2-libraries/Dockerfile create mode 100644 tests/python2-libraries/requirements.txt diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..91985fe5 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,6 @@ +sudo: required +services: +- docker +script: +- docker build -t google/python . +- docker build -t google/python-libraries tests/python2-libraries diff --git a/Dockerfile b/Dockerfile index e5a9394c..38fa3663 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,6 +11,7 @@ RUN apt-get -q update && \ libxslt1-dev libpq-dev libmysqlclient-dev libcurl4-openssl-dev \ libjpeg-dev zlib1g-dev libpng12-dev \ gfortran libblas-dev liblapack-dev libatlas-dev libquadmath0 \ + libfreetype6-dev pkg-config swig \ && \ apt-get clean && rm /var/lib/apt/lists/*_* diff --git a/tests/python2-libraries/Dockerfile b/tests/python2-libraries/Dockerfile new file mode 100644 index 00000000..11a7dc25 --- /dev/null +++ b/tests/python2-libraries/Dockerfile @@ -0,0 +1,7 @@ +FROM google/python + +RUN virtualenv /env +ENV VIRTUAL_ENV /env +ENV PATH /env/bin:$PATH +ADD requirements.txt /app/requirements.txt +RUN pip install -r /app/requirements.txt diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt new file mode 100644 index 00000000..f2f261e4 --- /dev/null +++ b/tests/python2-libraries/requirements.txt @@ -0,0 +1,211 @@ +simplejson +setuptools +six +requests +virtualenv +pip +distribute +python-dateutil +certifi +boto +pbr +#wincertstore +docutils +pyasn1 +pyyaml +jinja2 +markupsafe +pytz +nose +lxml +pycrypto +rsa +colorama +botocore +cffi +awscli +coverage +jmespath +pycparser +pika +django +psycopg2 +paramiko +ecdsa +sqlalchemy +mock +redis +werkzeug +selenium +bcdoc +supervisor +pep8 +httplib2 +flask +pymongo +zc.buildout +psutil +mysql-python +argparse +carbon +pygments +babel +paste +anyjson +meld3 +# ssl (already included in standard library) +cryptography +py +tornado +pyopenssl +greenlet +kombu +graphite-web +docopt +mako +itsdangerous +pillow +wheel +beautifulsoup4 +enum34 +pyflakes +zope.interface +decorator +futures +pastedeploy +ordereddict +setuptools-git +fabric +backports.ssl_match_hostname +amqp +numpy +sphinx +iso8601 +flake8 +celery +pyparsing +mccabe +stevedore +pytest +webob +gunicorn +urllib3 +billiard +jsonschema +msgpack-python +gevent +logilab-common +unittest2 +prettytable +pylint +blessings +south +mozrunner +netaddr +oslo.config +twisted +ipaddress +ujson +moznetwork +mozdevice +mozprofile +mozprocess +mozfile +mozinfo +html5lib +mozlog +mozcrash +oauthlib +idna +ipython +tox +astroid +google-api-python-client +pycurl +isodate +python-keystoneclient +websocket-client +markdown +python-mimeparse +python-daemon +raven +suds +oauth2client +cython +eventlet +netifaces +repoze.lru +thrift +sqlparse +ndg-httpsclient +djangorestframework +python-novaclient +testtools +alembic +uritemplate +statsd +python-memcached +coveralls +funcsigs +configobj +linecache2 +extras +beautifulsoup +# scikit-learn +cliff +oauth2 +# pycups +cmd2 +unidecode +newrelic +python-gflags +cov-core +pytest-cov +fixtures +pyasn1-modules +python-swiftclient +django-debug-toolbar +elasticsearch +webtest +docker-py +python-subunit +retrying +django-extensions +pystache +waitress +pexpect +blinker +scipy +requests-oauthlib +protobuf +manifestparser +passlib +ansible +click +versiontools +django_compress +pyzmq +chardet +xlrd +snowballstemmer +testrepository +pandas +functools32 +python-cjson +pastescript +warlock +sqlalchemy-migrate +django-celery +uwsgi +cssselect +# Hand selected +matplotlib +pymysql +amqplib +sh +m2crypto +apache-libcloud +hiredis +bottle +pyramid +pyjwt From 6cc694e33101a1b9ac2e9be89d6004bfa960ad61 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Fri, 26 Feb 2016 15:18:45 -0800 Subject: [PATCH 014/362] Adding tests for virtualenv usage. --- .travis.yml | 5 ++++- Dockerfile | 6 +++--- README.md | 2 +- tests/no-virtualenv/Dockerfile | 5 +++++ tests/virtualenv/Dockerfile | 28 ++++++++++++++++++++++++++++ 5 files changed, 41 insertions(+), 5 deletions(-) create mode 100644 tests/no-virtualenv/Dockerfile create mode 100644 tests/virtualenv/Dockerfile diff --git a/.travis.yml b/.travis.yml index 91985fe5..41768984 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,4 +3,7 @@ services: - docker script: - docker build -t google/python . -- docker build -t google/python-libraries tests/python2-libraries +- docker build tests/virtualenv +- docker build tests/no-virtualenv +# Disabled temporarily. +#- docker build -t google/python-libraries tests/python2-libraries diff --git a/Dockerfile b/Dockerfile index 38fa3663..959814b6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,12 +22,12 @@ ENV LANG C.UTF-8 # install virtualenv system-wide. RUN pip install --upgrade pip virtualenv -EXPOSE 8080 - RUN ln -s /home/vmagent/app /app WORKDIR /app +# Port 8080 is the port used by Google App Engine for serving HTTP traffic. +EXPOSE 8080 ENV PORT 8080 -CMD [] # The user's Dockerfile must specify an entrypoint with ENTRYPOINT or CMD. +CMD [] diff --git a/README.md b/README.md index 10c41cdb..c97c6bf5 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ For other docker hosts, you'll need to create a `Dockerfile` based on this image RUN virtualenv /env # Setting these environment variables are the same as running - # source /env/bin/activate + # source /env/bin/activate. ENV VIRTUAL_ENV /env ENV PATH /env/bin:$PATH diff --git a/tests/no-virtualenv/Dockerfile b/tests/no-virtualenv/Dockerfile new file mode 100644 index 00000000..68089c81 --- /dev/null +++ b/tests/no-virtualenv/Dockerfile @@ -0,0 +1,5 @@ +FROM google/python + +RUN if [ "$(which python)" != "/usr/bin/python" ]; then exit 1; fi; +RUN pip install gunicorn +RUN if [ "$(which gunicorn)" != "/usr/local/bin/gunicorn" ]; then exit 1; fi; diff --git a/tests/virtualenv/Dockerfile b/tests/virtualenv/Dockerfile new file mode 100644 index 00000000..d4effb54 --- /dev/null +++ b/tests/virtualenv/Dockerfile @@ -0,0 +1,28 @@ +FROM google/python + +ENV VIRTUAL_ENV /env +ENV PATH /env/bin:$PATH + +RUN if [ "$(which python)" != "/usr/bin/python" ]; then exit 1; fi; +RUN virtualenv /env +# All commands from this point on should use the virtualenv +RUN if [ "$(which python)" != "/env/bin/python" ]; then exit 1; fi; +RUN if [ "$(python --version 2>&1)" != "Python 2.7.9" ]; then exit 1; fi; +RUN if [ "$(which pip)" != "/env/bin/pip" ]; then exit 1; fi; +RUN pip install gunicorn flask +RUN if [ "$(which gunicorn)" != "/env/bin/gunicorn" ]; then exit 1; fi; +RUN python -c "import sys; import flask; sys.exit(0 if flask.__file__.startswith('/env') else 1)" + +# Python 3 +RUN rm -rf /env +RUN if [ "$(which python3.4)" != "/usr/bin/python3.4" ]; then exit 1; fi; +RUN virtualenv -p python3.4 /env +# All commands from this point on should use the virtualenv +RUN if [ "$(which python)" != "/env/bin/python" ]; then exit 1; fi; +RUN if [ "$(which python3)" != "/env/bin/python3" ]; then exit 1; fi; +RUN if [ "$(python --version 2>&1)" != "Python 3.4.2" ]; then exit 1; fi; +RUN if [ "$(which pip)" != "/env/bin/pip" ]; then exit 1; fi; +RUN if [ "$(which pip3)" != "/env/bin/pip3" ]; then exit 1; fi; +RUN pip install gunicorn flask +RUN if [ "$(which gunicorn)" != "/env/bin/gunicorn" ]; then exit 1; fi; +RUN python -c "import sys; import flask; sys.exit(0 if flask.__file__.startswith('/env') else 1)" From d340361e189c36f84692a9deeb2761777db0fdf2 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Wed, 4 May 2016 10:41:46 -0700 Subject: [PATCH 015/362] Updated readme with Flex changes --- README.md | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index c97c6bf5..c9c99fdd 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,24 @@ -# Google Cloud Platform Python Docker Image +# Google Cloud Platform - Python Runtime Docker Image -This repository contains the source for the `gcr.io/google_appengine/python` [docker](https://docker.io) base image. This image can be used as the base image for running applications on [Google App Engine Managed VMs](https://cloud.google.com/appengine), [Google Container Engine](https://cloud.google.com/container-engine), or any other Docker host. +This repository contains the source for the `gcr.io/google_appengine/python` [docker](https://docker.io) base image. This image can be used as the base image for running applications on [Google App Engine Flexible](https://cloud.google.com/appengine/docs/flexible/), [Google Container Engine](https://cloud.google.com/container-engine), or any other Docker host. -This image is based on Debian Jessie and contains packages required to build most of the popular Python libraries. +This image is based on Debian Jessie and contains packages required to build most of the popular Python libraries. For more information about this runtime, see the [documentation](https://cloud.google.com/appengine/docs/flexible/python/runtime). ## App Engine -To generate a Dockerfile that uses this image as a base, use the [`Cloud SDK`](https://cloud.google.com/sdk/gcloud/reference/preview/app/gen-config): +When using App Engine Flexible, you can use the runtime without worrying about docker by specifying `runtime: python` in your `app.yaml`: + +```yaml +runtime: python +vm: true +entrypoint: gunicorn -b :$PORT main:app + +runtime_config: + # You can also specify 2 for Python 2.7 + python_version: 3 +``` + +If you want to use this runtime as a base and customize it, you can generate a Dockerfile using the [`Cloud SDK`](https://cloud.google.com/sdk/gcloud/reference/preview/app/gen-config): gcloud preview app gen-config --custom From e1c974ed7bbb4087ed4c1698ded9f383ee705545 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Wed, 25 May 2016 14:12:11 -0700 Subject: [PATCH 016/362] Adding PYTHONUNBUFFERED --- Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile b/Dockerfile index 959814b6..0e4e7f01 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,6 +17,9 @@ RUN apt-get -q update && \ # Setup locale. This prevents Python 3 IO encoding issues. ENV LANG C.UTF-8 +# Make stdout/stderr unbuffered. This prevents delay between output and cloud +# logging collection. +ENV PYTHONUNBUFFERED 1 # Upgrade pip (debian package version tends to run a few version behind) and # install virtualenv system-wide. From bdb8ddba98a36f8807cc125dd4b2021818a41ddd Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Tue, 12 Jul 2016 10:28:28 -0700 Subject: [PATCH 017/362] Update README.md to use gcloud beta. Fixes #24 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c9c99fdd..4562c7b1 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ runtime_config: If you want to use this runtime as a base and customize it, you can generate a Dockerfile using the [`Cloud SDK`](https://cloud.google.com/sdk/gcloud/reference/preview/app/gen-config): - gcloud preview app gen-config --custom + gcloud beta app gen-config --custom You can then modify the `Dockerfile` and `.dockerignore` as needed for you application. From 6fc0bc98c78a773fff7af4ec3261e52317e3a365 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Mon, 29 Aug 2016 15:17:54 -0700 Subject: [PATCH 018/362] Add basic Python unified benchmark test --- .travis.yml | 8 +++----- Makefile | 11 +++++++++++ tests/Makefile | 9 +++++++++ tests/benchmark/Dockerfile | 4 ++++ tests/benchmark/Dockerfile.2vs3 | 1 + tests/benchmark/Makefile | 7 +++++++ tests/no-virtualenv/Makefile | 3 +++ tests/python2-libraries/Makefile | 3 +++ tests/virtualenv/Makefile | 3 +++ 9 files changed, 44 insertions(+), 5 deletions(-) create mode 100644 Makefile create mode 100644 tests/Makefile create mode 100644 tests/benchmark/Dockerfile create mode 100644 tests/benchmark/Dockerfile.2vs3 create mode 100644 tests/benchmark/Makefile create mode 100644 tests/no-virtualenv/Makefile create mode 100644 tests/python2-libraries/Makefile create mode 100644 tests/virtualenv/Makefile diff --git a/.travis.yml b/.travis.yml index 41768984..5f7debaf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,8 +2,6 @@ sudo: required services: - docker script: -- docker build -t google/python . -- docker build tests/virtualenv -- docker build tests/no-virtualenv -# Disabled temporarily. -#- docker build -t google/python-libraries tests/python2-libraries +- make build +- make tests/virtualenv +- make tests/no-virtualenv diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..96e165e8 --- /dev/null +++ b/Makefile @@ -0,0 +1,11 @@ +.PHONY: build +build: + docker build -t google/python . + +.PHONY: tests +tests: + make -C tests all + +.PHONY: benchmarks +benchmarks: + make -C tests benchmarks diff --git a/tests/Makefile b/tests/Makefile new file mode 100644 index 00000000..03d4753a --- /dev/null +++ b/tests/Makefile @@ -0,0 +1,9 @@ +.PHONY: all +all: + make -C no-virtualenv + make -C virtualenv + make -C python2-libraries + +.PHONY: benchmarks +benchmarks: + make -C benchmark all diff --git a/tests/benchmark/Dockerfile b/tests/benchmark/Dockerfile new file mode 100644 index 00000000..e00792d6 --- /dev/null +++ b/tests/benchmark/Dockerfile @@ -0,0 +1,4 @@ +FROM google/python + +RUN hg clone https://hg.python.org/benchmarks /app/benchmarks +WORKDIR /app/benchmarks diff --git a/tests/benchmark/Dockerfile.2vs3 b/tests/benchmark/Dockerfile.2vs3 new file mode 100644 index 00000000..507cd4bb --- /dev/null +++ b/tests/benchmark/Dockerfile.2vs3 @@ -0,0 +1 @@ +RUN python perf.py -r -b default /usr/bin/python2.7 /usr/bin/python3.4 diff --git a/tests/benchmark/Makefile b/tests/benchmark/Makefile new file mode 100644 index 00000000..caa51bb4 --- /dev/null +++ b/tests/benchmark/Makefile @@ -0,0 +1,7 @@ +.PHONY: all +all: 2vs3 + +# Tests Python 2.7 against Python 3.4. +.PHONY: 2vs3 +2vs3: + cat Dockerfile Dockerfile.2vs3 | docker build - diff --git a/tests/no-virtualenv/Makefile b/tests/no-virtualenv/Makefile new file mode 100644 index 00000000..7241697f --- /dev/null +++ b/tests/no-virtualenv/Makefile @@ -0,0 +1,3 @@ +.PHONY: all +all: + docker build . diff --git a/tests/python2-libraries/Makefile b/tests/python2-libraries/Makefile new file mode 100644 index 00000000..7241697f --- /dev/null +++ b/tests/python2-libraries/Makefile @@ -0,0 +1,3 @@ +.PHONY: all +all: + docker build . diff --git a/tests/virtualenv/Makefile b/tests/virtualenv/Makefile new file mode 100644 index 00000000..7241697f --- /dev/null +++ b/tests/virtualenv/Makefile @@ -0,0 +1,3 @@ +.PHONY: all +all: + docker build . From bbc55c2d30e7c98723d742330be04816f5094312 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Thu, 8 Sep 2016 12:39:30 -0700 Subject: [PATCH 019/362] Add Python 3.5 interpreter under /opt/python3.5 --- Dockerfile | 18 +++++++-------- resources/apt-packages.txt | 37 +++++++++++++++++++++++++++++++ scripts/build-python-3.5.sh | 21 ++++++++++++++++++ scripts/install-apt-packages.sh | 12 ++++++++++ tests/benchmark/Dockerfile.34vs35 | 1 + tests/benchmark/Makefile | 9 ++++++-- tests/virtualenv/Dockerfile | 16 ++++++++++++- 7 files changed, 101 insertions(+), 13 deletions(-) create mode 100644 resources/apt-packages.txt create mode 100755 scripts/build-python-3.5.sh create mode 100755 scripts/install-apt-packages.sh create mode 100644 tests/benchmark/Dockerfile.34vs35 diff --git a/Dockerfile b/Dockerfile index 0e4e7f01..8b6677e7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,18 +2,12 @@ # installed. FROM gcr.io/google_appengine/base +ADD resources /resources +ADD scripts /scripts + # Install Python, pip, and C dev libraries necessary to compile the most popular # Python libraries. -RUN apt-get -q update && \ - apt-get install --no-install-recommends -y -q \ - python2.7 python3.4 python2.7-dev python3.4-dev python-pip build-essential git mercurial \ - libffi-dev libssl-dev libxml2-dev \ - libxslt1-dev libpq-dev libmysqlclient-dev libcurl4-openssl-dev \ - libjpeg-dev zlib1g-dev libpng12-dev \ - gfortran libblas-dev liblapack-dev libatlas-dev libquadmath0 \ - libfreetype6-dev pkg-config swig \ - && \ - apt-get clean && rm /var/lib/apt/lists/*_* +RUN /scripts/install-apt-packages.sh # Setup locale. This prevents Python 3 IO encoding issues. ENV LANG C.UTF-8 @@ -25,6 +19,10 @@ ENV PYTHONUNBUFFERED 1 # install virtualenv system-wide. RUN pip install --upgrade pip virtualenv +# Install the Python 3.5 interpreter +RUN /scripts/build-python-3.5.sh + +# Setup the app working directory RUN ln -s /home/vmagent/app /app WORKDIR /app diff --git a/resources/apt-packages.txt b/resources/apt-packages.txt new file mode 100644 index 00000000..5bc58837 --- /dev/null +++ b/resources/apt-packages.txt @@ -0,0 +1,37 @@ +# utilities +wget +git +mercurial +pkg-config +# debian-provided interpreters +python2.7 +python3.4 +python2.7-dev +python3.4-dev +python-pip +# Dependenies for Python source builds and third-party Python packages +# with C-extensions +build-essential +swig +libffi-dev +libssl-dev +libxml2-dev +libxslt1-dev +libpq-dev +libmysqlclient-dev +libcurl4-openssl-dev +libjpeg-dev +zlib1g-dev +libpng12-dev +libbz2-dev +libsqlite3-dev +libreadline-dev +libncurses5-dev +libgdbm-dev +# Needed by scipy/numpy +gfortran +libblas-dev +liblapack-dev +libatlas-dev +libquadmath0 +libfreetype6-dev diff --git a/scripts/build-python-3.5.sh b/scripts/build-python-3.5.sh new file mode 100755 index 00000000..4a442814 --- /dev/null +++ b/scripts/build-python-3.5.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -e + +# Get the source +mkdir -p /opt/sources +cd /opt/sources +wget -nv https://www.python.org/ftp/python/3.5.2/Python-3.5.2.tgz +tar xzf Python-3.5.2.tgz + +# Build +cd Python-3.5.2 +# Use /opt/python{X}.{Y} for the prefix. +./configure --prefix=/opt/python3.5 +# Explicitly build the profile-guided-optimized interpreter +make profile-opt +make install + +# Clean-up sources +rm /opt/sources/Python-3.5.2.tgz +rm -r /opt/sources/Python-3.5.2 diff --git a/scripts/install-apt-packages.sh b/scripts/install-apt-packages.sh new file mode 100755 index 00000000..9d23cab7 --- /dev/null +++ b/scripts/install-apt-packages.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +set -e + +apt-get -q update + +xargs -a <(awk '/^\s*[^#]/' '/resources/apt-packages.txt') -r -- \ + apt-get install --no-install-recommends -yq + +# Remove unneeded files. +apt-get clean +rm /var/lib/apt/lists/*_* diff --git a/tests/benchmark/Dockerfile.34vs35 b/tests/benchmark/Dockerfile.34vs35 new file mode 100644 index 00000000..d319d14d --- /dev/null +++ b/tests/benchmark/Dockerfile.34vs35 @@ -0,0 +1 @@ +RUN python perf.py -r -b default /usr/bin/python3.4 /opt/python3.5/bin/python3.5 diff --git a/tests/benchmark/Makefile b/tests/benchmark/Makefile index caa51bb4..64290f94 100644 --- a/tests/benchmark/Makefile +++ b/tests/benchmark/Makefile @@ -1,7 +1,12 @@ .PHONY: all -all: 2vs3 +all: 2vs3 34vs35 -# Tests Python 2.7 against Python 3.4. +# Tests Python 3.4 against Python 2.7. .PHONY: 2vs3 2vs3: cat Dockerfile Dockerfile.2vs3 | docker build - + +# Tests Python 3.5 against Python 3.4. +.PHONY: 2vs3 +34vs35: + cat Dockerfile Dockerfile.34vs35 | docker build - diff --git a/tests/virtualenv/Dockerfile b/tests/virtualenv/Dockerfile index d4effb54..86e0d487 100644 --- a/tests/virtualenv/Dockerfile +++ b/tests/virtualenv/Dockerfile @@ -13,7 +13,7 @@ RUN pip install gunicorn flask RUN if [ "$(which gunicorn)" != "/env/bin/gunicorn" ]; then exit 1; fi; RUN python -c "import sys; import flask; sys.exit(0 if flask.__file__.startswith('/env') else 1)" -# Python 3 +# Python 3.4 RUN rm -rf /env RUN if [ "$(which python3.4)" != "/usr/bin/python3.4" ]; then exit 1; fi; RUN virtualenv -p python3.4 /env @@ -26,3 +26,17 @@ RUN if [ "$(which pip3)" != "/env/bin/pip3" ]; then exit 1; fi; RUN pip install gunicorn flask RUN if [ "$(which gunicorn)" != "/env/bin/gunicorn" ]; then exit 1; fi; RUN python -c "import sys; import flask; sys.exit(0 if flask.__file__.startswith('/env') else 1)" + +# Python 3.5 +RUN rm -rf /env +RUN virtualenv -p /opt/python3.5/bin/python3.5 /env +# All commands from this point on should use the virtualenv +RUN if [ "$(which python)" != "/env/bin/python" ]; then exit 1; fi; +RUN if [ "$(which python3)" != "/env/bin/python3" ]; then exit 1; fi; +RUN if [ "$(which python3.5)" != "/env/bin/python3.5" ]; then exit 1; fi; +RUN if [ "$(python --version 2>&1)" != "Python 3.5.2" ]; then exit 1; fi; +RUN if [ "$(which pip)" != "/env/bin/pip" ]; then exit 1; fi; +RUN if [ "$(which pip3)" != "/env/bin/pip3" ]; then exit 1; fi; +RUN pip install gunicorn flask +RUN if [ "$(which gunicorn)" != "/env/bin/gunicorn" ]; then exit 1; fi; +RUN python -c "import sys; import flask; sys.exit(0 if flask.__file__.startswith('/env') else 1)" From 38a3b5af32279e18093c763f3fb30f5a6b2d51f2 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Thu, 8 Sep 2016 14:37:26 -0700 Subject: [PATCH 020/362] Adding liblzma --- resources/apt-packages.txt | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/resources/apt-packages.txt b/resources/apt-packages.txt index 5bc58837..f423b9ec 100644 --- a/resources/apt-packages.txt +++ b/resources/apt-packages.txt @@ -1,37 +1,38 @@ # utilities -wget git mercurial pkg-config +wget # debian-provided interpreters +python-pip python2.7 -python3.4 python2.7-dev +python3.4 python3.4-dev -python-pip # Dependenies for Python source builds and third-party Python packages # with C-extensions build-essential -swig +libbz2-dev +libcurl4-openssl-dev libffi-dev +libgdbm-dev +libjpeg-dev +liblzma-dev +libmysqlclient-dev +libncurses5-dev +libpng12-dev +libpq-dev +libreadline-dev +libsqlite3-dev libssl-dev libxml2-dev libxslt1-dev -libpq-dev -libmysqlclient-dev -libcurl4-openssl-dev -libjpeg-dev +swig zlib1g-dev -libpng12-dev -libbz2-dev -libsqlite3-dev -libreadline-dev -libncurses5-dev -libgdbm-dev # Needed by scipy/numpy gfortran +libatlas-dev libblas-dev +libfreetype6-dev liblapack-dev -libatlas-dev libquadmath0 -libfreetype6-dev From d9472942e17f96d8023833e061cc856360f5768c Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Thu, 15 Sep 2016 10:40:04 -0700 Subject: [PATCH 021/362] Add separate interpreter builder --- Dockerfile | 4 +-- Makefile | 6 +++- python-interpreter-builder/.dockerignore | 1 + python-interpreter-builder/.gitignore | 1 + python-interpreter-builder/Dockerfile | 30 +++++++++++++++++++ python-interpreter-builder/Makefile | 9 ++++++ python-interpreter-builder/README.md | 20 +++++++++++++ .../scripts}/build-python-3.5.sh | 2 +- resources/apt-packages.txt | 8 +---- 9 files changed, 70 insertions(+), 11 deletions(-) create mode 100644 python-interpreter-builder/.dockerignore create mode 100644 python-interpreter-builder/.gitignore create mode 100644 python-interpreter-builder/Dockerfile create mode 100644 python-interpreter-builder/Makefile create mode 100644 python-interpreter-builder/README.md rename {scripts => python-interpreter-builder/scripts}/build-python-3.5.sh (89%) diff --git a/Dockerfile b/Dockerfile index 8b6677e7..11ca579a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,8 +19,8 @@ ENV PYTHONUNBUFFERED 1 # install virtualenv system-wide. RUN pip install --upgrade pip virtualenv -# Install the Python 3.5 interpreter -RUN /scripts/build-python-3.5.sh +# Install the Google-built interpreters +ADD python-interpreter-builder/output/interpreters.tar.gz / # Setup the app working directory RUN ln -s /home/vmagent/app /app diff --git a/Makefile b/Makefile index 96e165e8..34922d0a 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,11 @@ .PHONY: build -build: +build: build-interpreters docker build -t google/python . +.PHONY: build-interpreters +build-interpreters: + make -C python-interpreter-builder build + .PHONY: tests tests: make -C tests all diff --git a/python-interpreter-builder/.dockerignore b/python-interpreter-builder/.dockerignore new file mode 100644 index 00000000..53752db2 --- /dev/null +++ b/python-interpreter-builder/.dockerignore @@ -0,0 +1 @@ +output diff --git a/python-interpreter-builder/.gitignore b/python-interpreter-builder/.gitignore new file mode 100644 index 00000000..53752db2 --- /dev/null +++ b/python-interpreter-builder/.gitignore @@ -0,0 +1 @@ +output diff --git a/python-interpreter-builder/Dockerfile b/python-interpreter-builder/Dockerfile new file mode 100644 index 00000000..3458c5e4 --- /dev/null +++ b/python-interpreter-builder/Dockerfile @@ -0,0 +1,30 @@ +# The Google App Engine base image is debian (jessie) with ca-certificates +# installed. +FROM gcr.io/google_appengine/base + +# Install Python build dependencies +RUN apt-get update && apt-get install -yq \ + build-essential \ + wget \ + pkg-config \ + libbz2-dev \ + libgdbm-dev \ + liblzma-dev \ + libncurses5-dev \ + libreadline-dev \ + libsqlite3-dev \ + libssl-dev \ + zlib1g-dev + +# Setup locale. This prevents Python 3 IO encoding issues. +ENV LANG C.UTF-8 + +# Add build scripts +ADD scripts /scripts + +# Build the Python 3.5 interpreter +RUN /scripts/build-python-3.5.sh + +# Tar the interpreters. Tarring is needed because docker cp doesn't handle +# links correctly. +RUN tar czf /interpreters.tar.gz /opt diff --git a/python-interpreter-builder/Makefile b/python-interpreter-builder/Makefile new file mode 100644 index 00000000..a57ff192 --- /dev/null +++ b/python-interpreter-builder/Makefile @@ -0,0 +1,9 @@ +.PHONY: build +build: + docker build -t google/python-interpreter-builder . + # Extract the built interpreters + # This is needed because `docker cp` doesn't work on images, just containers. + docker run --name python-interpreter-builder google/python-interpreter-builder /bin/bash + mkdir -p output + docker cp --follow-link python-interpreter-builder:/interpreters.tar.gz output/interpreters.tar.gz + docker rm python-interpreter-builder diff --git a/python-interpreter-builder/README.md b/python-interpreter-builder/README.md new file mode 100644 index 00000000..3c1fc588 --- /dev/null +++ b/python-interpreter-builder/README.md @@ -0,0 +1,20 @@ +# Python Interpreter Builder + +This is a Docker-based Python interpreter builder. It builds Python interpreters +using a Debian-based Docker image. These interpreters are suitable to be moved +to another Debian-based Docker image. This avoids needing to install build +dependencies in the final container. + + +## Building + +Use make: + + make build + +The interpreters will be outputted to `output/interpreters.tar.gz`, this is +suitable to be added directly to a Docker container: + + ADD interpreters.tar.gz / + +Docker will automatically un-tar the interpreters into `/opt`. diff --git a/scripts/build-python-3.5.sh b/python-interpreter-builder/scripts/build-python-3.5.sh similarity index 89% rename from scripts/build-python-3.5.sh rename to python-interpreter-builder/scripts/build-python-3.5.sh index 4a442814..4b8cffc1 100755 --- a/scripts/build-python-3.5.sh +++ b/python-interpreter-builder/scripts/build-python-3.5.sh @@ -11,7 +11,7 @@ tar xzf Python-3.5.2.tgz # Build cd Python-3.5.2 # Use /opt/python{X}.{Y} for the prefix. -./configure --prefix=/opt/python3.5 +./configure --prefix=/opt/python3.5 --with-lto # Explicitly build the profile-guided-optimized interpreter make profile-opt make install diff --git a/resources/apt-packages.txt b/resources/apt-packages.txt index f423b9ec..13ad4ff1 100644 --- a/resources/apt-packages.txt +++ b/resources/apt-packages.txt @@ -9,21 +9,15 @@ python2.7 python2.7-dev python3.4 python3.4-dev -# Dependenies for Python source builds and third-party Python packages +# Dependenies for third-party Python packages # with C-extensions build-essential -libbz2-dev libcurl4-openssl-dev libffi-dev -libgdbm-dev libjpeg-dev -liblzma-dev libmysqlclient-dev -libncurses5-dev libpng12-dev libpq-dev -libreadline-dev -libsqlite3-dev libssl-dev libxml2-dev libxslt1-dev From e53f1fb04e1b7269dbcb0454703b4b5c90b760f9 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Mon, 19 Sep 2016 10:19:20 -0700 Subject: [PATCH 022/362] Add libraries test for Python 3 --- tests/Makefile | 1 + tests/python3-libraries/Dockerfile | 7 + tests/python3-libraries/Makefile | 3 + tests/python3-libraries/requirements.txt | 171 +++++++++++++++++++++++ 4 files changed, 182 insertions(+) create mode 100644 tests/python3-libraries/Dockerfile create mode 100644 tests/python3-libraries/Makefile create mode 100644 tests/python3-libraries/requirements.txt diff --git a/tests/Makefile b/tests/Makefile index 03d4753a..c79127f6 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -3,6 +3,7 @@ all: make -C no-virtualenv make -C virtualenv make -C python2-libraries + make -C python3-libraries .PHONY: benchmarks benchmarks: diff --git a/tests/python3-libraries/Dockerfile b/tests/python3-libraries/Dockerfile new file mode 100644 index 00000000..c618cfb1 --- /dev/null +++ b/tests/python3-libraries/Dockerfile @@ -0,0 +1,7 @@ +FROM google/python + +RUN virtualenv -p /opt/python3.5/bin/python3.5 /env +ENV VIRTUAL_ENV /env +ENV PATH /env/bin:$PATH +ADD requirements.txt /app/requirements.txt +RUN pip install -r /app/requirements.txt diff --git a/tests/python3-libraries/Makefile b/tests/python3-libraries/Makefile new file mode 100644 index 00000000..7241697f --- /dev/null +++ b/tests/python3-libraries/Makefile @@ -0,0 +1,3 @@ +.PHONY: all +all: + docker build . diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt new file mode 100644 index 00000000..d14d83f1 --- /dev/null +++ b/tests/python3-libraries/requirements.txt @@ -0,0 +1,171 @@ +simplejson +six +requests +virtualenv +pip +certifi +docutils +pyasn1 +pyyaml +jinja2 +markupsafe +pytz +nose +lxml +pycrypto +rsa +colorama +cffi +pycparser +django +psycopg2 +ecdsa +sqlalchemy +mock +redis +werkzeug +pep8 +httplib2 +flask +pymongo +zc.buildout +psutil +argparse +carbon +pygments +babel +paste +anyjson +cryptography +py +tornado +pyopenssl +greenlet +kombu +docopt +mako +itsdangerous +pillow +wheel +beautifulsoup4 +pyflakes +zope.interface +decorator +futures +pastedeploy +setuptools-git +amqp +numpy +sphinx +iso8601 +flake8 +celery +pyparsing +mccabe +stevedore +pytest +webob +gunicorn +urllib3 +billiard +jsonschema +msgpack-python +gevent +logilab-common +prettytable +pylint +blessings +south +netaddr +oslo.config +twisted +ipaddress +ujson +html5lib +oauthlib +idna +ipython +tox +astroid +google-api-python-client +pycurl +isodate +python-keystoneclient +websocket-client +markdown +python-mimeparse +python-daemon +raven +oauth2client +cython +eventlet +netifaces +repoze.lru +thrift +sqlparse +ndg-httpsclient +djangorestframework +python-novaclient +testtools +alembic +uritemplate +statsd +python-memcached +coveralls +funcsigs +configobj +linecache2 +extras +cliff +oauth2 +cmd2 +unidecode +newrelic +python-gflags +cov-core +pytest-cov +fixtures +pyasn1-modules +python-swiftclient +django-debug-toolbar +elasticsearch +webtest +docker-py +python-subunit +retrying +django-extensions +pystache +waitress +pexpect +blinker +scipy +requests-oauthlib +protobuf +manifestparser +passlib +ansible +click +versiontools +django_compress +pyzmq +chardet +xlrd +snowballstemmer +testrepository +pandas +pastescript +warlock +sqlalchemy-migrate +django-celery +cssselect +# Hand selected +matplotlib +pymysql +amqplib +sh +m2crypto +apache-libcloud +hiredis +bottle +pyramid +pyjwt From b34ebc309d9a0b5f4e95628469d964ce2afd23ec Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Mon, 19 Sep 2016 11:02:33 -0700 Subject: [PATCH 023/362] Add FORCE_REBUILD make option --- Makefile | 7 ++++++- python-interpreter-builder/Makefile | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 34922d0a..f6974073 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,14 @@ +ifdef FORCE_REBUILD + DOCKER_FLAGS = --no-cache --pull +endif + .PHONY: build build: build-interpreters - docker build -t google/python . + docker build $(DOCKER_FLAGS) -t google/python . .PHONY: build-interpreters build-interpreters: + export DOCKER_FLAGS make -C python-interpreter-builder build .PHONY: tests diff --git a/python-interpreter-builder/Makefile b/python-interpreter-builder/Makefile index a57ff192..240775b5 100644 --- a/python-interpreter-builder/Makefile +++ b/python-interpreter-builder/Makefile @@ -1,6 +1,6 @@ .PHONY: build build: - docker build -t google/python-interpreter-builder . + docker build $(DOCKER_FLAGS) -t google/python-interpreter-builder . # Extract the built interpreters # This is needed because `docker cp` doesn't work on images, just containers. docker run --name python-interpreter-builder google/python-interpreter-builder /bin/bash From 7f670b2153a0f808f3e491ed6c4e3042ff927058 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Mon, 19 Sep 2016 11:04:17 -0700 Subject: [PATCH 024/362] Add IMAGE_NAME make variable --- Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index f6974073..1fdd410b 100644 --- a/Makefile +++ b/Makefile @@ -2,9 +2,11 @@ ifdef FORCE_REBUILD DOCKER_FLAGS = --no-cache --pull endif +IMAGE_NAME ?= google/python + .PHONY: build build: build-interpreters - docker build $(DOCKER_FLAGS) -t google/python . + docker build $(DOCKER_FLAGS) -t "$(IMAGE_NAME)" . .PHONY: build-interpreters build-interpreters: From 5197b325cabca068bd0107c878a6345f783b0d68 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Mon, 19 Sep 2016 11:21:08 -0700 Subject: [PATCH 025/362] Switch to make altinstall --- python-interpreter-builder/scripts/build-python-3.5.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-interpreter-builder/scripts/build-python-3.5.sh b/python-interpreter-builder/scripts/build-python-3.5.sh index 4b8cffc1..263312af 100755 --- a/python-interpreter-builder/scripts/build-python-3.5.sh +++ b/python-interpreter-builder/scripts/build-python-3.5.sh @@ -14,7 +14,7 @@ cd Python-3.5.2 ./configure --prefix=/opt/python3.5 --with-lto # Explicitly build the profile-guided-optimized interpreter make profile-opt -make install +make altinstall # Clean-up sources rm /opt/sources/Python-3.5.2.tgz From 2b7f2f2baa5d22121c3dd036c75259d7110bb0fa Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Mon, 19 Sep 2016 12:33:15 -0700 Subject: [PATCH 026/362] Remove follow-link argument --- python-interpreter-builder/Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python-interpreter-builder/Makefile b/python-interpreter-builder/Makefile index 240775b5..ddd5a15a 100644 --- a/python-interpreter-builder/Makefile +++ b/python-interpreter-builder/Makefile @@ -3,7 +3,8 @@ build: docker build $(DOCKER_FLAGS) -t google/python-interpreter-builder . # Extract the built interpreters # This is needed because `docker cp` doesn't work on images, just containers. + -docker rm python-interpreter-builder docker run --name python-interpreter-builder google/python-interpreter-builder /bin/bash mkdir -p output - docker cp --follow-link python-interpreter-builder:/interpreters.tar.gz output/interpreters.tar.gz + docker cp python-interpreter-builder:/interpreters.tar.gz output/interpreters.tar.gz docker rm python-interpreter-builder From 8a3f9f6de972141996836bb9f7f4954046244f7e Mon Sep 17 00:00:00 2001 From: sharifelgamal Date: Wed, 28 Sep 2016 13:44:44 -0700 Subject: [PATCH 027/362] Switch base image from base to debian8 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 11ca579a..2b5d38a6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ # The Google App Engine base image is debian (jessie) with ca-certificates # installed. -FROM gcr.io/google_appengine/base +FROM gcr.io/google_appengine/debian8 ADD resources /resources ADD scripts /scripts From 6c7f170ad05a80a9bd9aa5748b86eb51628c728f Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Thu, 6 Oct 2016 10:08:33 -0700 Subject: [PATCH 028/362] Update README.md to be explicit about using gen-config against existing apps, fixes #31 --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4562c7b1..d6147015 100644 --- a/README.md +++ b/README.md @@ -18,11 +18,11 @@ runtime_config: python_version: 3 ``` -If you want to use this runtime as a base and customize it, you can generate a Dockerfile using the [`Cloud SDK`](https://cloud.google.com/sdk/gcloud/reference/preview/app/gen-config): +If you have an existing App Engine application using this runtime and want to customize it, you can use the [`Cloud SDK`](https://cloud.google.com/sdk/gcloud/reference/preview/app/gen-config) to create a custom runtime: gcloud beta app gen-config --custom -You can then modify the `Dockerfile` and `.dockerignore` as needed for you application. +You can then modify the `Dockerfile` and `.dockerignore` as needed for you application. ## Container Engine & other Docker hosts. From 5fe6c716d50f7071d712c722fe6d2d13622f218d Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 7 Oct 2016 14:30:27 -0700 Subject: [PATCH 029/362] Add link to source repository of base image. --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 2b5d38a6..a63cb7f3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,6 @@ # The Google App Engine base image is debian (jessie) with ca-certificates # installed. +# Source: https://github.com/GoogleCloudPlatform/debian-docker FROM gcr.io/google_appengine/debian8 ADD resources /resources From 227db8260df588b0e16dc8acfbd4742294a9d006 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Thu, 20 Oct 2016 15:21:33 -0700 Subject: [PATCH 030/362] Add required packages for pylibmc --- resources/apt-packages.txt | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/resources/apt-packages.txt b/resources/apt-packages.txt index 13ad4ff1..5c1c2e6a 100644 --- a/resources/apt-packages.txt +++ b/resources/apt-packages.txt @@ -30,3 +30,11 @@ libblas-dev libfreetype6-dev liblapack-dev libquadmath0 +# Needed by pylibmc +libmemcached +libmemcached-dev +libsasl2-2 +sasl2-bin +libsasl2-2 +libsasl2-dev +libsasl1-modules From 49f3e37baa0ecb312971e91008e05a4a3e2ce5f5 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Fri, 21 Oct 2016 10:25:01 -0700 Subject: [PATCH 031/362] Fix package names --- resources/apt-packages.txt | 3 +-- tests/python2-libraries/requirements.txt | 1 + tests/python3-libraries/requirements.txt | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/resources/apt-packages.txt b/resources/apt-packages.txt index 5c1c2e6a..2ef0214f 100644 --- a/resources/apt-packages.txt +++ b/resources/apt-packages.txt @@ -31,10 +31,9 @@ libfreetype6-dev liblapack-dev libquadmath0 # Needed by pylibmc -libmemcached libmemcached-dev libsasl2-2 sasl2-bin libsasl2-2 libsasl2-dev -libsasl1-modules +libsasl2-modules diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index f2f261e4..36dcbc16 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -209,3 +209,4 @@ hiredis bottle pyramid pyjwt +pylibmc diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index d14d83f1..aeef827e 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -169,3 +169,4 @@ hiredis bottle pyramid pyjwt +pylibmc From 6ecc377fae2c9a0f1a8828482feaf7c2cc7cbca3 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 21 Oct 2016 13:11:29 -0700 Subject: [PATCH 032/362] Sort packages and remove a duplicate --- resources/apt-packages.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/resources/apt-packages.txt b/resources/apt-packages.txt index 2ef0214f..4bb6d6fc 100644 --- a/resources/apt-packages.txt +++ b/resources/apt-packages.txt @@ -33,7 +33,6 @@ libquadmath0 # Needed by pylibmc libmemcached-dev libsasl2-2 -sasl2-bin -libsasl2-2 libsasl2-dev libsasl2-modules +sasl2-bin From 892998ffffc95c1dc58bd4a1d0acf64a80201854 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Tue, 25 Oct 2016 10:08:43 -0700 Subject: [PATCH 033/362] Add tests for google-cloud-python --- tests/Makefile | 1 + tests/google-cloud-python/Dockerfile | 21 +++++++++++++++++++++ tests/google-cloud-python/Makefile | 3 +++ 3 files changed, 25 insertions(+) create mode 100644 tests/google-cloud-python/Dockerfile create mode 100644 tests/google-cloud-python/Makefile diff --git a/tests/Makefile b/tests/Makefile index c79127f6..b848eb19 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -2,6 +2,7 @@ all: make -C no-virtualenv make -C virtualenv + make -C google-cloud-python make -C python2-libraries make -C python3-libraries diff --git a/tests/google-cloud-python/Dockerfile b/tests/google-cloud-python/Dockerfile new file mode 100644 index 00000000..06a2c004 --- /dev/null +++ b/tests/google-cloud-python/Dockerfile @@ -0,0 +1,21 @@ +FROM google/python + +# Install tox +RUN pip install --upgrade tox + +# Get the source. +RUN git clone https://github.com/GoogleCloudPlatform/google-cloud-python.git +WORKDIR google-cloud-python + +# Checkout the latest release. +RUN git checkout $(git describe --tags) + +# Run Python 2.7 unit tests +RUN python2.7 scripts/run_unit_tests.py + +# Run Python 3.4 unit tests +RUN python3.4 scripts/run_unit_tests.py + +# Run Python 3.5 unit tests +ENV PATH /opt/python3.5/bin:$PATH +RUN python3.5 scripts/run_unit_tests.py diff --git a/tests/google-cloud-python/Makefile b/tests/google-cloud-python/Makefile new file mode 100644 index 00000000..69a1e9e2 --- /dev/null +++ b/tests/google-cloud-python/Makefile @@ -0,0 +1,3 @@ +.PHONY: all +all: + docker build --no-cache . From dc304a0e3174a402bf62b93874cf6c2d3f26e3ce Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Tue, 25 Oct 2016 14:17:56 -0700 Subject: [PATCH 034/362] Address review comments --- tests/google-cloud-python/Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/google-cloud-python/Makefile b/tests/google-cloud-python/Makefile index 69a1e9e2..6f9c5522 100644 --- a/tests/google-cloud-python/Makefile +++ b/tests/google-cloud-python/Makefile @@ -1,3 +1,5 @@ .PHONY: all all: + # Use no-cache to prevent layer caching because there is a layer that does + # a `git clone` which can not be cached. docker build --no-cache . From dd68108320f5989e8a7fb0bcfbb426a1d217c9bf Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Tue, 1 Nov 2016 14:55:58 -0700 Subject: [PATCH 035/362] Ensure 'make build|tests|benchmarks' are using the same image --- Makefile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 1fdd410b..7cd94ca7 100644 --- a/Makefile +++ b/Makefile @@ -2,11 +2,16 @@ ifdef FORCE_REBUILD DOCKER_FLAGS = --no-cache --pull endif -IMAGE_NAME ?= google/python +# Note: 'make build/tests/benchmarks' share images by retagging the +# candidate as 'google/python'. So this could cause trouble with +# concurrent builds on the same machine. +CANDIDATE_NAME ?= $(shell date +%Y-%m-%d_%H_%M) +IMAGE_NAME ?= google/python:$(CANDIDATE_NAME) .PHONY: build build: build-interpreters docker build $(DOCKER_FLAGS) -t "$(IMAGE_NAME)" . + docker tag -f "$(IMAGE_NAME)" "google/python" .PHONY: build-interpreters build-interpreters: From 9aeedd3344bd36c3e92d50527f978464253eeacc Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Wed, 2 Nov 2016 19:59:18 -0700 Subject: [PATCH 036/362] Run google-cloud-python system tests against new runtime images. --- system_tests/.gitignore | 2 ++ system_tests/Dockerfile | 40 ++++++++++++++++++++++++++++++++++++++++ system_tests/Makefile | 7 +++++++ 3 files changed, 49 insertions(+) create mode 100644 system_tests/.gitignore create mode 100644 system_tests/Dockerfile create mode 100644 system_tests/Makefile diff --git a/system_tests/.gitignore b/system_tests/.gitignore new file mode 100644 index 00000000..457b0368 --- /dev/null +++ b/system_tests/.gitignore @@ -0,0 +1,2 @@ +data/ +secrets.tar \ No newline at end of file diff --git a/system_tests/Dockerfile b/system_tests/Dockerfile new file mode 100644 index 00000000..8e9ad3ff --- /dev/null +++ b/system_tests/Dockerfile @@ -0,0 +1,40 @@ +FROM google/python + +# Inject secrets +# TODO: Encrypt/decrypt secrets +ADD data/ data/ +ENV GOOGLE_APPLICATION_CREDENTIALS /home/vmagent/app/data/cloud-python-runtime-qa-credentials.json + +# Get the source. +RUN git clone https://github.com/GoogleCloudPlatform/google-cloud-python.git +WORKDIR google-cloud-python + +# Checkout the latest release. +RUN git checkout $(git describe --tags --abbrev 0) + +# Install gcloud so we can do test setup +RUN apt-get update && apt-get install curl +RUN curl https://sdk.cloud.google.com | bash +ENV PATH ${PATH}:/root/google-cloud-sdk/bin + +# System test setup as per google-cloud-python/CONTRIBUTING.rst +RUN gcloud config set project cloud-python-runtime-qa +RUN gcloud components install app-engine-python +RUN gcloud auth activate-service-account \ + --key-file=${GOOGLE_APPLICATION_CREDENTIALS} +RUN gcloud preview datastore create-indexes system_tests/data/index.yaml +# TODO: python system_tests/clear_datastore.py +# TODO: python system_tests/populate_datastore.py + +# Install tox for running the system tests +RUN pip install --upgrade tox + +# List of system tests to run. Ideally would be all of them. But +# some of them get permission or other errors that are probably +# related to incomplete test setup. +ENV MODULES="datastore storage speech bigquery pubsub language bigtable" + +# Run Python 2.7, 3.4 system tests +# TODO(https://github.com/GoogleCloudPlatform/google-cloud-python/issues/2670) +RUN GOOGLE_CLOUD_TESTS_API_KEY=$(cat /home/vmagent/app/data/cloud-python-runtime-qa-api-key.txt) \ + tox -e system-tests,system-tests3 -- ${MODULES} diff --git a/system_tests/Makefile b/system_tests/Makefile new file mode 100644 index 00000000..4dbde7d1 --- /dev/null +++ b/system_tests/Makefile @@ -0,0 +1,7 @@ +# Use no-cache to prevent layer caching because there is a layer that does +# a `git clone` which can not be cached. +CACHE ?= --no-cache + +.PHONY: all +all: + docker build $(CACHE) . From 95c653c7edd77fe95ee7e6c5ee3507db5f919a5a Mon Sep 17 00:00:00 2001 From: dlorenc Date: Thu, 27 Oct 2016 14:20:42 -0700 Subject: [PATCH 037/362] Second version of switching things to cloud build. --- .gitignore | 2 ++ Dockerfile | 2 +- Makefile | 5 +++++ cloudbuild.yaml.in | 9 +++++++++ python-interpreter-builder/Makefile | 2 +- 5 files changed, 18 insertions(+), 2 deletions(-) create mode 100644 .gitignore create mode 100644 cloudbuild.yaml.in diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..b2c26136 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +cloudbuild.yaml +interpreters.tar.gz \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index a63cb7f3..96837525 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,7 @@ ENV PYTHONUNBUFFERED 1 RUN pip install --upgrade pip virtualenv # Install the Google-built interpreters -ADD python-interpreter-builder/output/interpreters.tar.gz / +ADD interpreters.tar.gz / # Setup the app working directory RUN ln -s /home/vmagent/app /app diff --git a/Makefile b/Makefile index 7cd94ca7..d12e6cd3 100644 --- a/Makefile +++ b/Makefile @@ -25,3 +25,8 @@ tests: .PHONY: benchmarks benchmarks: make -C tests benchmarks + +.PHONY: cloudbuild +cloudbuild: + envsubst cloudbuild.yaml + gcloud alpha container builds create . --config=cloudbuild.yaml \ No newline at end of file diff --git a/cloudbuild.yaml.in b/cloudbuild.yaml.in new file mode 100644 index 00000000..aebfa2ca --- /dev/null +++ b/cloudbuild.yaml.in @@ -0,0 +1,9 @@ +steps: +- name: gcr.io/cloud-builders/docker + args: ['build', '--tag=interpreter', 'python-interpreter-builder'] +- name: interpreter + args: ['cp', '/interpreters.tar.gz', '/workspace/'] +- name: gcr.io/cloud-builders/docker + args: ['build', '--tag=${IMAGE_NAME}', '.'] +images: + ['${IMAGE_NAME}'] \ No newline at end of file diff --git a/python-interpreter-builder/Makefile b/python-interpreter-builder/Makefile index ddd5a15a..6acfe6bb 100644 --- a/python-interpreter-builder/Makefile +++ b/python-interpreter-builder/Makefile @@ -6,5 +6,5 @@ build: -docker rm python-interpreter-builder docker run --name python-interpreter-builder google/python-interpreter-builder /bin/bash mkdir -p output - docker cp python-interpreter-builder:/interpreters.tar.gz output/interpreters.tar.gz + docker cp python-interpreter-builder:/interpreters.tar.gz ../interpreters.tar.gz docker rm python-interpreter-builder From 3b2abef4a943abd11f1f628404a62f69d5bef7d9 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Thu, 3 Nov 2016 13:01:41 -0700 Subject: [PATCH 038/362] Add missing newline --- system_tests/.gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system_tests/.gitignore b/system_tests/.gitignore index 457b0368..2fc08f4d 100644 --- a/system_tests/.gitignore +++ b/system_tests/.gitignore @@ -1,2 +1,2 @@ data/ -secrets.tar \ No newline at end of file +secrets.tar From e3a7cf856971a43526063bd34d011a00e0b25a6a Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Thu, 3 Nov 2016 13:03:31 -0700 Subject: [PATCH 039/362] Ignore Emacs autosave files --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..21adbdb4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +# Editor +*~ +\#*\# From 87db4d3a483b6f3eff4c6a27fd57da41435ff3b4 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Thu, 3 Nov 2016 13:06:26 -0700 Subject: [PATCH 040/362] Shorten paths (/home/vmagent/app is symlinked to /app) --- system_tests/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/system_tests/Dockerfile b/system_tests/Dockerfile index 8e9ad3ff..4ffc08a3 100644 --- a/system_tests/Dockerfile +++ b/system_tests/Dockerfile @@ -3,7 +3,7 @@ FROM google/python # Inject secrets # TODO: Encrypt/decrypt secrets ADD data/ data/ -ENV GOOGLE_APPLICATION_CREDENTIALS /home/vmagent/app/data/cloud-python-runtime-qa-credentials.json +ENV GOOGLE_APPLICATION_CREDENTIALS /app/data/cloud-python-runtime-qa-credentials.json # Get the source. RUN git clone https://github.com/GoogleCloudPlatform/google-cloud-python.git @@ -36,5 +36,5 @@ ENV MODULES="datastore storage speech bigquery pubsub language bigtable" # Run Python 2.7, 3.4 system tests # TODO(https://github.com/GoogleCloudPlatform/google-cloud-python/issues/2670) -RUN GOOGLE_CLOUD_TESTS_API_KEY=$(cat /home/vmagent/app/data/cloud-python-runtime-qa-api-key.txt) \ +RUN GOOGLE_CLOUD_TESTS_API_KEY=$(cat /app/data/cloud-python-runtime-qa-api-key.txt) \ tox -e system-tests,system-tests3 -- ${MODULES} From c0933d0d75b5f5e7caf26851e28747bea73d8cf5 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Thu, 3 Nov 2016 13:07:04 -0700 Subject: [PATCH 041/362] Add top-level system-tests target --- Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Makefile b/Makefile index 7cd94ca7..ef3ff347 100644 --- a/Makefile +++ b/Makefile @@ -25,3 +25,7 @@ tests: .PHONY: benchmarks benchmarks: make -C tests benchmarks + +.PHONY: system-tests +system-tests: + make -C system_tests From 5439e04db2fc33adb32b92f9d1bb045c623b9a9a Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Thu, 3 Nov 2016 13:31:06 -0700 Subject: [PATCH 042/362] Fix "checkout latest release", then disable it to always checkout head. Previous code was always checking out head because of missing --abbrev flag. Once I added --abbrev, I found out the latest release, 0.20, is missing some things we need. We can reenable this when 0.21+ is released. --- system_tests/Dockerfile | 3 ++- tests/google-cloud-python/Dockerfile | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/system_tests/Dockerfile b/system_tests/Dockerfile index 4ffc08a3..77bce574 100644 --- a/system_tests/Dockerfile +++ b/system_tests/Dockerfile @@ -10,7 +10,8 @@ RUN git clone https://github.com/GoogleCloudPlatform/google-cloud-python.git WORKDIR google-cloud-python # Checkout the latest release. -RUN git checkout $(git describe --tags --abbrev 0) +# TODO: Enable once the latest release works +#RUN git checkout $(git describe --tags --abbrev=0) # Install gcloud so we can do test setup RUN apt-get update && apt-get install curl diff --git a/tests/google-cloud-python/Dockerfile b/tests/google-cloud-python/Dockerfile index 06a2c004..7fb6936c 100644 --- a/tests/google-cloud-python/Dockerfile +++ b/tests/google-cloud-python/Dockerfile @@ -8,7 +8,8 @@ RUN git clone https://github.com/GoogleCloudPlatform/google-cloud-python.git WORKDIR google-cloud-python # Checkout the latest release. -RUN git checkout $(git describe --tags) +# TODO: Enable once run_unit_tests.py is released +#RUN git checkout $(git describe --tags --abbrev=0) # Run Python 2.7 unit tests RUN python2.7 scripts/run_unit_tests.py From 29c73aa1a6249d0e5f2efa76d5d0d9745c4abda4 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Thu, 3 Nov 2016 14:13:10 -0700 Subject: [PATCH 043/362] Enable system tests that work now. Currently, everything but 'logging' works. See: https://github.com/GoogleCloudPlatform/google-cloud-python/issues/2669 --- system_tests/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system_tests/Dockerfile b/system_tests/Dockerfile index 77bce574..ac1f427d 100644 --- a/system_tests/Dockerfile +++ b/system_tests/Dockerfile @@ -33,7 +33,7 @@ RUN pip install --upgrade tox # List of system tests to run. Ideally would be all of them. But # some of them get permission or other errors that are probably # related to incomplete test setup. -ENV MODULES="datastore storage speech bigquery pubsub language bigtable" +ENV MODULES="datastore storage speech bigquery pubsub language translate monitoring bigtable" # Run Python 2.7, 3.4 system tests # TODO(https://github.com/GoogleCloudPlatform/google-cloud-python/issues/2670) From 6960bb4a694b71563c8956288b3f7c0a00595e6b Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Thu, 3 Nov 2016 14:23:22 -0700 Subject: [PATCH 044/362] Revert "Ignore Emacs autosave files" This reverts commit e3a7cf856971a43526063bd34d011a00e0b25a6a. --- .gitignore | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 .gitignore diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 21adbdb4..00000000 --- a/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -# Editor -*~ -\#*\# From 2e6f7041fb2c591b927619e82c5321cfb5834617 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Thu, 3 Nov 2016 14:49:31 -0700 Subject: [PATCH 045/362] Use google-cloud-python from head, not the released version. See https://github.com/GoogleCloudPlatform/python-runtime/issues/41 --- system_tests/Dockerfile | 4 ---- tests/google-cloud-python/Dockerfile | 4 ---- 2 files changed, 8 deletions(-) diff --git a/system_tests/Dockerfile b/system_tests/Dockerfile index ac1f427d..1edfc50c 100644 --- a/system_tests/Dockerfile +++ b/system_tests/Dockerfile @@ -9,10 +9,6 @@ ENV GOOGLE_APPLICATION_CREDENTIALS /app/data/cloud-python-runtime-qa-credentials RUN git clone https://github.com/GoogleCloudPlatform/google-cloud-python.git WORKDIR google-cloud-python -# Checkout the latest release. -# TODO: Enable once the latest release works -#RUN git checkout $(git describe --tags --abbrev=0) - # Install gcloud so we can do test setup RUN apt-get update && apt-get install curl RUN curl https://sdk.cloud.google.com | bash diff --git a/tests/google-cloud-python/Dockerfile b/tests/google-cloud-python/Dockerfile index 7fb6936c..ade8d818 100644 --- a/tests/google-cloud-python/Dockerfile +++ b/tests/google-cloud-python/Dockerfile @@ -7,10 +7,6 @@ RUN pip install --upgrade tox RUN git clone https://github.com/GoogleCloudPlatform/google-cloud-python.git WORKDIR google-cloud-python -# Checkout the latest release. -# TODO: Enable once run_unit_tests.py is released -#RUN git checkout $(git describe --tags --abbrev=0) - # Run Python 2.7 unit tests RUN python2.7 scripts/run_unit_tests.py From d24cb129c88aaafde817394d5284045d492d3b4a Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Thu, 3 Nov 2016 15:09:55 -0700 Subject: [PATCH 046/362] Don't install or use the Google Cloud SDK for system tests. Test setup will be handled elsewhere. --- system_tests/Dockerfile | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/system_tests/Dockerfile b/system_tests/Dockerfile index 1edfc50c..75a0830b 100644 --- a/system_tests/Dockerfile +++ b/system_tests/Dockerfile @@ -4,22 +4,23 @@ FROM google/python # TODO: Encrypt/decrypt secrets ADD data/ data/ ENV GOOGLE_APPLICATION_CREDENTIALS /app/data/cloud-python-runtime-qa-credentials.json +ENV GCLOUD_PROJECT cloud-python-runtime-qa # Get the source. RUN git clone https://github.com/GoogleCloudPlatform/google-cloud-python.git WORKDIR google-cloud-python # Install gcloud so we can do test setup -RUN apt-get update && apt-get install curl -RUN curl https://sdk.cloud.google.com | bash -ENV PATH ${PATH}:/root/google-cloud-sdk/bin +#RUN apt-get update && apt-get install curl +#RUN curl https://sdk.cloud.google.com | bash +#ENV PATH ${PATH}:/root/google-cloud-sdk/bin # System test setup as per google-cloud-python/CONTRIBUTING.rst -RUN gcloud config set project cloud-python-runtime-qa -RUN gcloud components install app-engine-python -RUN gcloud auth activate-service-account \ - --key-file=${GOOGLE_APPLICATION_CREDENTIALS} -RUN gcloud preview datastore create-indexes system_tests/data/index.yaml +#RUN gcloud config set project cloud-python-runtime-qa +#RUN gcloud components install app-engine-python +#RUN gcloud auth activate-service-account \ +# --key-file=${GOOGLE_APPLICATION_CREDENTIALS} +# RUN gcloud preview datastore create-indexes system_tests/data/index.yaml # TODO: python system_tests/clear_datastore.py # TODO: python system_tests/populate_datastore.py From d108c5aba6411fc697f1d9965bf3acfc9cdd4ebc Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Thu, 3 Nov 2016 15:18:54 -0700 Subject: [PATCH 047/362] Remove cruft --- system_tests/Dockerfile | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/system_tests/Dockerfile b/system_tests/Dockerfile index 75a0830b..64c69717 100644 --- a/system_tests/Dockerfile +++ b/system_tests/Dockerfile @@ -10,20 +10,6 @@ ENV GCLOUD_PROJECT cloud-python-runtime-qa RUN git clone https://github.com/GoogleCloudPlatform/google-cloud-python.git WORKDIR google-cloud-python -# Install gcloud so we can do test setup -#RUN apt-get update && apt-get install curl -#RUN curl https://sdk.cloud.google.com | bash -#ENV PATH ${PATH}:/root/google-cloud-sdk/bin - -# System test setup as per google-cloud-python/CONTRIBUTING.rst -#RUN gcloud config set project cloud-python-runtime-qa -#RUN gcloud components install app-engine-python -#RUN gcloud auth activate-service-account \ -# --key-file=${GOOGLE_APPLICATION_CREDENTIALS} -# RUN gcloud preview datastore create-indexes system_tests/data/index.yaml -# TODO: python system_tests/clear_datastore.py -# TODO: python system_tests/populate_datastore.py - # Install tox for running the system tests RUN pip install --upgrade tox From 6ad7fb2e7ffd445223890da80fc259aca4c6a0bb Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Thu, 3 Nov 2016 15:25:31 -0700 Subject: [PATCH 048/362] Rename system-tests target to google-cloud-system-tests --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index ef3ff347..0d79ffd6 100644 --- a/Makefile +++ b/Makefile @@ -26,6 +26,6 @@ tests: benchmarks: make -C tests benchmarks -.PHONY: system-tests -system-tests: +.PHONY: google-cloud-system-tests +google-cloud-system-tests: make -C system_tests From 5632d0e2c324df77578e3a986ddc84c26df41ef6 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Thu, 3 Nov 2016 15:30:44 -0700 Subject: [PATCH 049/362] Remove TODOs in favor of opening issues https://github.com/GoogleCloudPlatform/python-runtime/issues/43 https://github.com/GoogleCloudPlatform/python-runtime/issues/44 --- system_tests/Dockerfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/system_tests/Dockerfile b/system_tests/Dockerfile index 64c69717..8a306f64 100644 --- a/system_tests/Dockerfile +++ b/system_tests/Dockerfile @@ -1,7 +1,6 @@ FROM google/python # Inject secrets -# TODO: Encrypt/decrypt secrets ADD data/ data/ ENV GOOGLE_APPLICATION_CREDENTIALS /app/data/cloud-python-runtime-qa-credentials.json ENV GCLOUD_PROJECT cloud-python-runtime-qa @@ -19,6 +18,5 @@ RUN pip install --upgrade tox ENV MODULES="datastore storage speech bigquery pubsub language translate monitoring bigtable" # Run Python 2.7, 3.4 system tests -# TODO(https://github.com/GoogleCloudPlatform/google-cloud-python/issues/2670) RUN GOOGLE_CLOUD_TESTS_API_KEY=$(cat /app/data/cloud-python-runtime-qa-api-key.txt) \ tox -e system-tests,system-tests3 -- ${MODULES} From a33b58d10bae3d90e220dc641f8f2c0427fa1b9b Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Mon, 7 Nov 2016 13:30:16 -0800 Subject: [PATCH 050/362] Add /opt/python3.5/bin to $PATH Contents of this directory are: 2to3-3.5 easy_install-3.5 idle3.5 pip3.5 pydoc3.5 python3.5 python3.5m python3.5m-config pyvenv-3.5 For completeness, the current interpreters exposed to users: $ which python ; python -V /usr/bin/python Python 2.7.9 $ which python2 ; python2 -V /usr/bin/python2 Python 2.7.9 $which python2.7 ; python2.7 -V /usr/bin/python2.7 Python 2.7.9 $ which python3 ; python3 -V bash: python3: command not found $ which python3.4 ; python3.4 -V /usr/bin/python3.4 Python 3.4.2 $ which python3.5 ; python3.5 -V /opt/python3.5/bin/python3.5 Python 3.5.2 --- Dockerfile | 3 +++ tests/python3-libraries/Dockerfile | 2 +- tests/virtualenv/Dockerfile | 3 ++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 96837525..85d10374 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,6 +23,9 @@ RUN pip install --upgrade pip virtualenv # Install the Google-built interpreters ADD interpreters.tar.gz / +# Add Google-built interpreters to the path +ENV PATH /opt/python3.5/bin:$PATH + # Setup the app working directory RUN ln -s /home/vmagent/app /app WORKDIR /app diff --git a/tests/python3-libraries/Dockerfile b/tests/python3-libraries/Dockerfile index c618cfb1..e39b89dc 100644 --- a/tests/python3-libraries/Dockerfile +++ b/tests/python3-libraries/Dockerfile @@ -1,6 +1,6 @@ FROM google/python -RUN virtualenv -p /opt/python3.5/bin/python3.5 /env +RUN virtualenv -p python3.5 /env ENV VIRTUAL_ENV /env ENV PATH /env/bin:$PATH ADD requirements.txt /app/requirements.txt diff --git a/tests/virtualenv/Dockerfile b/tests/virtualenv/Dockerfile index 86e0d487..416a773b 100644 --- a/tests/virtualenv/Dockerfile +++ b/tests/virtualenv/Dockerfile @@ -29,7 +29,8 @@ RUN python -c "import sys; import flask; sys.exit(0 if flask.__file__.startswith # Python 3.5 RUN rm -rf /env -RUN virtualenv -p /opt/python3.5/bin/python3.5 /env +RUN if [ "$(which python3.5)" != "/opt/python3.5/bin/python3.5" ]; then exit 1; fi; +RUN virtualenv -p python3.5 /env # All commands from this point on should use the virtualenv RUN if [ "$(which python)" != "/env/bin/python" ]; then exit 1; fi; RUN if [ "$(which python3)" != "/env/bin/python3" ]; then exit 1; fi; From aa075362c756304332558ff0338d196df6279de4 Mon Sep 17 00:00:00 2001 From: Nick Kubala Date: Thu, 10 Nov 2016 16:10:16 -0800 Subject: [PATCH 051/362] wip; converting dockerfile tests to structure tests --- cloudbuild.yaml.in | 4 ++- tests/virtualenv/virtualenv_test.json | 38 +++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 tests/virtualenv/virtualenv_test.json diff --git a/cloudbuild.yaml.in b/cloudbuild.yaml.in index aebfa2ca..83589e54 100644 --- a/cloudbuild.yaml.in +++ b/cloudbuild.yaml.in @@ -5,5 +5,7 @@ steps: args: ['cp', '/interpreters.tar.gz', '/workspace/'] - name: gcr.io/cloud-builders/docker args: ['build', '--tag=${IMAGE_NAME}', '.'] +- name: gcr.io/gcp-runtimes/structure_test + args: ['-i', 'google/python', '--config', '/workspace/tests/virtualenv/virtualenv_test.json', '-v'] images: - ['${IMAGE_NAME}'] \ No newline at end of file + ['${IMAGE_NAME}'] diff --git a/tests/virtualenv/virtualenv_test.json b/tests/virtualenv/virtualenv_test.json new file mode 100644 index 00000000..5e93c587 --- /dev/null +++ b/tests/virtualenv/virtualenv_test.json @@ -0,0 +1,38 @@ +{ + "schemaVersion": "1.0.0", + "commandTests": [ + { + "name": "env var setup", + "envVars": [ + ["VIRTUAL_ENV", "/env"], + ["PATH", "/env/bin:$PATH"] + ], + "command": ["echo", "'setup complete'"] + },{ + "name": "python installation", + "command": ["which", "python"], + "expectedOutput": ["/usr/bin/python\n"] + },{ + "name": "virtualenv installation", + "setup": [["virtualenv", "/env"]], + "command": ["which", "python"], + "expectedOutput": ["/env/bin/python\n"] + },{ + "name": "python version", + "command": ["python", "--version"], + "expectedError": ["Python 2.7.9\n"] + },{ + "name": "pip installation", + "command": ["which", "pip"], + "expectedOutput": ["/env/bin/pip\n"] + },{ + "name": "gunicorn flask", + "setup": [["pip", "install", "gunicorn", "flask"]], + "command": ["which", "gunicorn"], + "expectedOutput": ["/env/bin/gunicorn"] + },{ + "name": "flask integration", + "command": ["python", "-c", "\"import sys; import flask; sys.exit(0 if flask.__file__.startswith('/env') else 1)\""] + } + ] +} From cab53035144b84e7c8bad5517cea5df6c732fae7 Mon Sep 17 00:00:00 2001 From: Nick Kubala Date: Mon, 14 Nov 2016 10:23:03 -0800 Subject: [PATCH 052/362] switch tests from json to yaml. fix makefile --- Makefile | 1 + cloudbuild.yaml.in | 16 +++++------ tests/virtualenv/virtualenv_test.json | 38 --------------------------- tests/virtualenv/virtualenv_test.yaml | 33 +++++++++++++++++++++++ 4 files changed, 42 insertions(+), 46 deletions(-) delete mode 100644 tests/virtualenv/virtualenv_test.json create mode 100644 tests/virtualenv/virtualenv_test.yaml diff --git a/Makefile b/Makefile index aa703b57..82364f70 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,7 @@ endif # concurrent builds on the same machine. CANDIDATE_NAME ?= $(shell date +%Y-%m-%d_%H_%M) IMAGE_NAME ?= google/python:$(CANDIDATE_NAME) +export IMAGE_NAME .PHONY: build build: build-interpreters diff --git a/cloudbuild.yaml.in b/cloudbuild.yaml.in index 83589e54..e10ff986 100644 --- a/cloudbuild.yaml.in +++ b/cloudbuild.yaml.in @@ -1,11 +1,11 @@ steps: -- name: gcr.io/cloud-builders/docker - args: ['build', '--tag=interpreter', 'python-interpreter-builder'] -- name: interpreter - args: ['cp', '/interpreters.tar.gz', '/workspace/'] -- name: gcr.io/cloud-builders/docker - args: ['build', '--tag=${IMAGE_NAME}', '.'] -- name: gcr.io/gcp-runtimes/structure_test - args: ['-i', 'google/python', '--config', '/workspace/tests/virtualenv/virtualenv_test.json', '-v'] +# - name: gcr.io/cloud-builders/docker +# args: ['build', '--tag=interpreter', 'python-interpreter-builder'] +# - name: interpreter +# args: ['cp', '/interpreters.tar.gz', '/workspace/'] +# - name: gcr.io/cloud-builders/docker +# args: ['build', '--tag=${IMAGE_NAME}', '.'] +- name: gcr.io/nick-cloudbuild/structure_test:env_var + args: ['-i', 'google/python', '--config', '/workspace/tests/virtualenv/virtualenv_test.yaml', '-v'] images: ['${IMAGE_NAME}'] diff --git a/tests/virtualenv/virtualenv_test.json b/tests/virtualenv/virtualenv_test.json deleted file mode 100644 index 5e93c587..00000000 --- a/tests/virtualenv/virtualenv_test.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "schemaVersion": "1.0.0", - "commandTests": [ - { - "name": "env var setup", - "envVars": [ - ["VIRTUAL_ENV", "/env"], - ["PATH", "/env/bin:$PATH"] - ], - "command": ["echo", "'setup complete'"] - },{ - "name": "python installation", - "command": ["which", "python"], - "expectedOutput": ["/usr/bin/python\n"] - },{ - "name": "virtualenv installation", - "setup": [["virtualenv", "/env"]], - "command": ["which", "python"], - "expectedOutput": ["/env/bin/python\n"] - },{ - "name": "python version", - "command": ["python", "--version"], - "expectedError": ["Python 2.7.9\n"] - },{ - "name": "pip installation", - "command": ["which", "pip"], - "expectedOutput": ["/env/bin/pip\n"] - },{ - "name": "gunicorn flask", - "setup": [["pip", "install", "gunicorn", "flask"]], - "command": ["which", "gunicorn"], - "expectedOutput": ["/env/bin/gunicorn"] - },{ - "name": "flask integration", - "command": ["python", "-c", "\"import sys; import flask; sys.exit(0 if flask.__file__.startswith('/env') else 1)\""] - } - ] -} diff --git a/tests/virtualenv/virtualenv_test.yaml b/tests/virtualenv/virtualenv_test.yaml new file mode 100644 index 00000000..f904027f --- /dev/null +++ b/tests/virtualenv/virtualenv_test.yaml @@ -0,0 +1,33 @@ +schemaVersion: "1.0.0" +commandTests: + - name: "env var setup" + envVars: [ + ["VIRTUAL_ENV", "/env"], + ["PATH", "/env/bin:$PATH"] + ] + command: ["echo", "'setup complete'"] + + - name: "python installation" + command: ["which", "python"] + expectedOutput: ["/usr/bin/python\n"] + + - name: "virtualenv installation" + setup: [["virtualenv", "/env"]] + command: ["which", "python"] + expectedOutput": ["/env/bin/python\n"] + + - name: "python version" + command: ["python", "--version"] + expectedError: ["Python 2.7.9\n"] + + - name: "pip installation" + command: ["which", "pip"] + expectedOutput: ["/env/bin/pip\n"] + + - name: "gunicorn flask" + setup: [["pip", "install", "gunicorn", "flask"]] + command: ["which", "gunicorn"] + expectedOutput: ["/env/bin/gunicorn"] + + - name: "flask integration" + command: ["python", "-c", "\"import sys; import flask; sys.exit(0 if flask.__file__.startswith('/env') else 1)\""] From 6141fcbdc267f57089f70749d7a4839e6a44e3ff Mon Sep 17 00:00:00 2001 From: Nick Kubala Date: Mon, 14 Nov 2016 11:48:23 -0800 Subject: [PATCH 053/362] finish translating virtualenv tests to framework --- cloudbuild.yaml.in | 18 ++-- tests/virtualenv/virtualenv.yaml | 119 ++++++++++++++++++++++++++ tests/virtualenv/virtualenv_test.yaml | 33 ------- 3 files changed, 130 insertions(+), 40 deletions(-) create mode 100644 tests/virtualenv/virtualenv.yaml delete mode 100644 tests/virtualenv/virtualenv_test.yaml diff --git a/cloudbuild.yaml.in b/cloudbuild.yaml.in index e10ff986..e1e918e8 100644 --- a/cloudbuild.yaml.in +++ b/cloudbuild.yaml.in @@ -1,11 +1,15 @@ steps: -# - name: gcr.io/cloud-builders/docker -# args: ['build', '--tag=interpreter', 'python-interpreter-builder'] -# - name: interpreter -# args: ['cp', '/interpreters.tar.gz', '/workspace/'] -# - name: gcr.io/cloud-builders/docker -# args: ['build', '--tag=${IMAGE_NAME}', '.'] +- name: gcr.io/cloud-builders/docker + args: ['build', '--tag=interpreter', 'python-interpreter-builder'] +- name: interpreter + args: ['cp', '/interpreters.tar.gz', '/workspace/'] +- name: gcr.io/cloud-builders/docker + args: ['build', '--tag=${IMAGE_NAME}', '.'] - name: gcr.io/nick-cloudbuild/structure_test:env_var - args: ['-i', 'google/python', '--config', '/workspace/tests/virtualenv/virtualenv_test.yaml', '-v'] + args: [ + '-i', 'google/python', + '--config', '/workspace/tests/virtualenv/virtualenv.yaml', + '-v' + ] images: ['${IMAGE_NAME}'] diff --git a/tests/virtualenv/virtualenv.yaml b/tests/virtualenv/virtualenv.yaml new file mode 100644 index 00000000..49bae181 --- /dev/null +++ b/tests/virtualenv/virtualenv.yaml @@ -0,0 +1,119 @@ +schemaVersion: "1.0.0" +commandTests: + - name: "env var setup" + envVars: [ + ["VIRTUAL_ENV", "/env"], + ["PATH", "/env/bin:$PATH"] + ] + command: ["echo", "'setup complete'"] + +##################################################### +# Default Python Tests +##################################################### + + - name: "python installation" + command: ["which", "python"] + expectedOutput: ["/usr/bin/python\n"] + + - name: "virtualenv installation" + setup: [["virtualenv", "/env"]] + command: ["which", "python"] + expectedOutput": ["/env/bin/python\n"] + + - name: "python version" + command: ["python", "--version"] + # we check stderr instead of stdout for Python versions < 3.4 + # https://bugs.python.org/issue18338 + expectedError: ["Python 2.7.9\n"] + + - name: "pip installation" + command: ["which", "pip"] + expectedOutput: ["/env/bin/pip\n"] + + - name: "gunicorn flask" + setup: [["pip", "install", "gunicorn", "flask"]] + command: ["which", "gunicorn"] + expectedOutput: ["/env/bin/gunicorn"] + + - name: "flask integration" + command: ["python", "-c", "\"import sys; import flask; sys.exit(0 if flask.__file__.startswith('/env') else 1)\""] + +##################################################### +# Python 3.4 Tests +##################################################### + + - name: "virtual env teardown" + command: ["rm", "-rf", "/env"] + + - name: "python installation" + command: ["which", "python3.4"] + expectedOutput: ["/usr/bin/python3.4\n"] + + - name: "virtualenv python installation" + setup: [["virtualenv", "-p", "python3.4", "/env"]] + command: ["which", "python"] + expectedOutput: ["/env/bin/python\n"] + + - name: "virtualenv python3 installation" + command: ["which", "python3"] + expectedOutput: ["/env/bin/python3\n"] + + - name: "python version" + command: ["python", "--version"] + expectedOutput: ["Python 3.4.2\n"] + + - name: "pip installation" + command: ["which", "pip"] + expectedOutput: ["/env/bin/pip\n"] + + - name: "pip3 installation" + command: ["which", "pip3"] + expectedOutput: ["/env/bin/pip3\n"] + + - name: "gunicorn flask" + setup: [["pip", "install", "gunicorn", "flask"]] + command: ["which", "gunicorn"] + expectedOutput: ["/env/bin/gunicorn"] + + - name: "flask integration" + command: ["python", "-c", "\"import sys; import flask; sys.exit(0 if flask.__file__.startswith('/env') else 1)\""] + +##################################################### +# Python 3.5 Tests +##################################################### + + - name: "virtual env teardown" + command: ["rm", "-rf", "/env"] + + - name: "virtualenv python installation" + setup: [["virtualenv", "-p", "/opt/python3.5/bin/python3.5", "/env"]] + command: ["which", "python"] + expectedOutput: ["/env/bin/python\n"] + + - name: "virtualenv python3 installation" + command: ["which", "python3"] + expectedOutput: ["/env/bin/python3\n"] + + - name: "virtualenv python3.5 installation" + command: ["which", "python3.5"] + expectedOutput: ["/env/bin/python3.5\n"] + + - name: "python version" + command: ["python", "--version"] + expectedOutput: ["Python 3.5.2\n"] + + - name: "pip installation" + command: ["which", "pip"] + expectedOutput: ["/env/bin/pip\n"] + + - name: "pip3 installation" + command: ["which", "pip3"] + expectedOutput: ["/env/bin/pip3\n"] + + - name: "gunicorn flask" + setup: [["pip", "install", "gunicorn", "flask"]] + command: ["which", "gunicorn"] + expectedOutput: ["/env/bin/gunicorn"] + + - name: "flask integration" + command: ["python", "-c", "\"import sys; import flask; sys.exit(0 if flask.__file__.startswith('/env') else 1)\""] diff --git a/tests/virtualenv/virtualenv_test.yaml b/tests/virtualenv/virtualenv_test.yaml deleted file mode 100644 index f904027f..00000000 --- a/tests/virtualenv/virtualenv_test.yaml +++ /dev/null @@ -1,33 +0,0 @@ -schemaVersion: "1.0.0" -commandTests: - - name: "env var setup" - envVars: [ - ["VIRTUAL_ENV", "/env"], - ["PATH", "/env/bin:$PATH"] - ] - command: ["echo", "'setup complete'"] - - - name: "python installation" - command: ["which", "python"] - expectedOutput: ["/usr/bin/python\n"] - - - name: "virtualenv installation" - setup: [["virtualenv", "/env"]] - command: ["which", "python"] - expectedOutput": ["/env/bin/python\n"] - - - name: "python version" - command: ["python", "--version"] - expectedError: ["Python 2.7.9\n"] - - - name: "pip installation" - command: ["which", "pip"] - expectedOutput: ["/env/bin/pip\n"] - - - name: "gunicorn flask" - setup: [["pip", "install", "gunicorn", "flask"]] - command: ["which", "gunicorn"] - expectedOutput: ["/env/bin/gunicorn"] - - - name: "flask integration" - command: ["python", "-c", "\"import sys; import flask; sys.exit(0 if flask.__file__.startswith('/env') else 1)\""] From 108d1028b1c01beec56cbbb5170100413e1c61ab Mon Sep 17 00:00:00 2001 From: Nick Kubala Date: Mon, 14 Nov 2016 11:58:02 -0800 Subject: [PATCH 054/362] no-virtualenv tests --- cloudbuild.yaml.in | 1 + tests/no-virtualenv/no-virtualenv.yaml | 10 ++++++++++ 2 files changed, 11 insertions(+) create mode 100644 tests/no-virtualenv/no-virtualenv.yaml diff --git a/cloudbuild.yaml.in b/cloudbuild.yaml.in index e1e918e8..8399432e 100644 --- a/cloudbuild.yaml.in +++ b/cloudbuild.yaml.in @@ -9,6 +9,7 @@ steps: args: [ '-i', 'google/python', '--config', '/workspace/tests/virtualenv/virtualenv.yaml', + '--config', '/workspace/tests/no-virtualenv/no-virtualenv.yaml', '-v' ] images: diff --git a/tests/no-virtualenv/no-virtualenv.yaml b/tests/no-virtualenv/no-virtualenv.yaml new file mode 100644 index 00000000..de932617 --- /dev/null +++ b/tests/no-virtualenv/no-virtualenv.yaml @@ -0,0 +1,10 @@ +schemaVersion: "1.0.0" +commandTests: + - name: "default python installation" + command: ["which", "python"] + expectedOutput: ["/usr/bin/python\n"] + + - name: "default gunicorn installation" + setup: [["pip", "install", "gunicorn"]] + command: ["which", "gunicorn"] + expectedOutput: ["/usr/local/bin/gunicorn\n"] From 2566a1a5541bd6f1fc2c3635aaec4eb5648a5e05 Mon Sep 17 00:00:00 2001 From: Nick Kubala Date: Mon, 14 Nov 2016 15:20:06 -0800 Subject: [PATCH 055/362] add tests for installing requirements on python 2 and 3 --- cloudbuild.yaml.in | 4 +++- tests/python2-libraries/python2-libraries.yaml | 13 +++++++++++++ tests/python3-libraries/python3-libraries.yaml | 13 +++++++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 tests/python2-libraries/python2-libraries.yaml create mode 100644 tests/python3-libraries/python3-libraries.yaml diff --git a/cloudbuild.yaml.in b/cloudbuild.yaml.in index 8399432e..6a941236 100644 --- a/cloudbuild.yaml.in +++ b/cloudbuild.yaml.in @@ -7,9 +7,11 @@ steps: args: ['build', '--tag=${IMAGE_NAME}', '.'] - name: gcr.io/nick-cloudbuild/structure_test:env_var args: [ - '-i', 'google/python', + '-i', '${IMAGE_NAME}', '--config', '/workspace/tests/virtualenv/virtualenv.yaml', '--config', '/workspace/tests/no-virtualenv/no-virtualenv.yaml', + '--config', '/workspace/tests/python2-libraries/python2-libraries.yaml', + '--config', '/workspace/tests/python2-libraries/python3-libraries.yaml', '-v' ] images: diff --git a/tests/python2-libraries/python2-libraries.yaml b/tests/python2-libraries/python2-libraries.yaml new file mode 100644 index 00000000..711bfcbc --- /dev/null +++ b/tests/python2-libraries/python2-libraries.yaml @@ -0,0 +1,13 @@ +schemaVersion: "1.0.0" +commandTests: + - name: "env var setup" + envVars: [ + ["VIRTUAL_ENV", "/env"], + ["PATH", "/env/bin:$PATH"] + ] + command: ["echo", "'setup complete'"] + + - name: "requirements" + setup: [["virtualenv", "/env"]] + command: ["pip", "install", "-r", "/workspace/tests/python2-libraries/requirements.txt"] + exitCode: 0 diff --git a/tests/python3-libraries/python3-libraries.yaml b/tests/python3-libraries/python3-libraries.yaml new file mode 100644 index 00000000..1154ef05 --- /dev/null +++ b/tests/python3-libraries/python3-libraries.yaml @@ -0,0 +1,13 @@ +schemaVersion: "1.0.0" +commandTests: + - name: "env var setup" + envVars: [ + ["VIRTUAL_ENV", "/env"], + ["PATH", "/env/bin:$PATH"] + ] + command: ["echo", "'setup complete'"] + + - name: "requirements" + setup: [["virtualenv", "-p", "/opt/python3.5/bin/python3.5", "/env"]] + command: ["pip", "install", "-r", "/workspace/tests/python3-libraries/requirements.txt"] + exitCode: 0 From 5dd943c93d192905557baf1e0c816ef1ce8c2d04 Mon Sep 17 00:00:00 2001 From: Nick Kubala Date: Mon, 14 Nov 2016 15:20:36 -0800 Subject: [PATCH 056/362] fix cloudbuild --- cloudbuild.yaml.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudbuild.yaml.in b/cloudbuild.yaml.in index 6a941236..b79d7d12 100644 --- a/cloudbuild.yaml.in +++ b/cloudbuild.yaml.in @@ -5,7 +5,7 @@ steps: args: ['cp', '/interpreters.tar.gz', '/workspace/'] - name: gcr.io/cloud-builders/docker args: ['build', '--tag=${IMAGE_NAME}', '.'] -- name: gcr.io/nick-cloudbuild/structure_test:env_var +- name: gcr.io/gcp-runtimes/structure_test args: [ '-i', '${IMAGE_NAME}', '--config', '/workspace/tests/virtualenv/virtualenv.yaml', From 5fbb9a46ec689a1f54abda2719a83c89fa3d9e27 Mon Sep 17 00:00:00 2001 From: Nick Kubala Date: Mon, 14 Nov 2016 16:18:47 -0800 Subject: [PATCH 057/362] update env var schema definition --- tests/python2-libraries/python2-libraries.yaml | 9 +++++---- tests/python3-libraries/python3-libraries.yaml | 9 +++++---- tests/virtualenv/virtualenv.yaml | 9 +++++---- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/tests/python2-libraries/python2-libraries.yaml b/tests/python2-libraries/python2-libraries.yaml index 711bfcbc..83c98972 100644 --- a/tests/python2-libraries/python2-libraries.yaml +++ b/tests/python2-libraries/python2-libraries.yaml @@ -1,10 +1,11 @@ schemaVersion: "1.0.0" commandTests: - name: "env var setup" - envVars: [ - ["VIRTUAL_ENV", "/env"], - ["PATH", "/env/bin:$PATH"] - ] + envVars: + - key: "VIRTUAL_ENV" + value: "/env" + - key: "PATH" + value: "/env/bin:$PATH" command: ["echo", "'setup complete'"] - name: "requirements" diff --git a/tests/python3-libraries/python3-libraries.yaml b/tests/python3-libraries/python3-libraries.yaml index 1154ef05..3ddf6e21 100644 --- a/tests/python3-libraries/python3-libraries.yaml +++ b/tests/python3-libraries/python3-libraries.yaml @@ -1,10 +1,11 @@ schemaVersion: "1.0.0" commandTests: - name: "env var setup" - envVars: [ - ["VIRTUAL_ENV", "/env"], - ["PATH", "/env/bin:$PATH"] - ] + envVars: + - key: "VIRTUAL_ENV" + value: "/env" + - key: "PATH" + value: "/env/bin:$PATH" command: ["echo", "'setup complete'"] - name: "requirements" diff --git a/tests/virtualenv/virtualenv.yaml b/tests/virtualenv/virtualenv.yaml index 49bae181..9b5c0401 100644 --- a/tests/virtualenv/virtualenv.yaml +++ b/tests/virtualenv/virtualenv.yaml @@ -1,10 +1,11 @@ schemaVersion: "1.0.0" commandTests: - name: "env var setup" - envVars: [ - ["VIRTUAL_ENV", "/env"], - ["PATH", "/env/bin:$PATH"] - ] + envVars: + - key: "VIRTUAL_ENV" + value: "/env" + - key: "PATH" + value: "/env/bin:$PATH" command: ["echo", "'setup complete'"] ##################################################### From 3cd086f98364070c542f51b61ff7b38d84f35338 Mon Sep 17 00:00:00 2001 From: Nick Kubala Date: Wed, 16 Nov 2016 11:50:44 -0800 Subject: [PATCH 058/362] remove old dockerfiles. update makefiles to run structure tests through ext_run script. --- Makefile | 2 + tests/no-virtualenv/Dockerfile | 5 --- tests/no-virtualenv/Makefile | 2 +- tests/python2-libraries/Dockerfile | 7 --- tests/python2-libraries/Makefile | 2 +- .../python2-libraries/python2-libraries.yaml | 15 +++---- tests/python3-libraries/Dockerfile | 7 --- tests/python3-libraries/Makefile | 2 +- .../python3-libraries/python3-libraries.yaml | 15 +++---- tests/virtualenv/Dockerfile | 43 ------------------- tests/virtualenv/Makefile | 2 +- tests/virtualenv/virtualenv.yaml | 20 +++++---- 12 files changed, 32 insertions(+), 90 deletions(-) delete mode 100644 tests/no-virtualenv/Dockerfile delete mode 100644 tests/python2-libraries/Dockerfile delete mode 100644 tests/python3-libraries/Dockerfile delete mode 100644 tests/virtualenv/Dockerfile diff --git a/Makefile b/Makefile index 82364f70..974a6d6c 100644 --- a/Makefile +++ b/Makefile @@ -21,6 +21,8 @@ build-interpreters: .PHONY: tests tests: + curl https://raw.githubusercontent.com/GoogleCloudPlatform/runtimes-common/master/structure_tests/ext_run.sh > ext_run.sh + chmod +x ext_run.sh make -C tests all .PHONY: benchmarks diff --git a/tests/no-virtualenv/Dockerfile b/tests/no-virtualenv/Dockerfile deleted file mode 100644 index 68089c81..00000000 --- a/tests/no-virtualenv/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM google/python - -RUN if [ "$(which python)" != "/usr/bin/python" ]; then exit 1; fi; -RUN pip install gunicorn -RUN if [ "$(which gunicorn)" != "/usr/local/bin/gunicorn" ]; then exit 1; fi; diff --git a/tests/no-virtualenv/Makefile b/tests/no-virtualenv/Makefile index 7241697f..2d9ec8e3 100644 --- a/tests/no-virtualenv/Makefile +++ b/tests/no-virtualenv/Makefile @@ -1,3 +1,3 @@ .PHONY: all all: - docker build . + ../../ext_run.sh -i "${IMAGE_NAME}" -c no-virtualenv.yaml -v diff --git a/tests/python2-libraries/Dockerfile b/tests/python2-libraries/Dockerfile deleted file mode 100644 index 11a7dc25..00000000 --- a/tests/python2-libraries/Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -FROM google/python - -RUN virtualenv /env -ENV VIRTUAL_ENV /env -ENV PATH /env/bin:$PATH -ADD requirements.txt /app/requirements.txt -RUN pip install -r /app/requirements.txt diff --git a/tests/python2-libraries/Makefile b/tests/python2-libraries/Makefile index 7241697f..13ab239e 100644 --- a/tests/python2-libraries/Makefile +++ b/tests/python2-libraries/Makefile @@ -1,3 +1,3 @@ .PHONY: all all: - docker build . + ../../ext_run.sh -i "${IMAGE_NAME}" -c python2-libraries.yaml -w ../.. -v diff --git a/tests/python2-libraries/python2-libraries.yaml b/tests/python2-libraries/python2-libraries.yaml index 83c98972..fb4f977a 100644 --- a/tests/python2-libraries/python2-libraries.yaml +++ b/tests/python2-libraries/python2-libraries.yaml @@ -1,13 +1,12 @@ schemaVersion: "1.0.0" -commandTests: - - name: "env var setup" - envVars: - - key: "VIRTUAL_ENV" - value: "/env" - - key: "PATH" - value: "/env/bin:$PATH" - command: ["echo", "'setup complete'"] +globalEnvVars: + - key: "VIRTUAL_ENV" + value: "/env" + - key: "PATH" + value: "/env/bin:$PATH" + +commandTests: - name: "requirements" setup: [["virtualenv", "/env"]] command: ["pip", "install", "-r", "/workspace/tests/python2-libraries/requirements.txt"] diff --git a/tests/python3-libraries/Dockerfile b/tests/python3-libraries/Dockerfile deleted file mode 100644 index e39b89dc..00000000 --- a/tests/python3-libraries/Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -FROM google/python - -RUN virtualenv -p python3.5 /env -ENV VIRTUAL_ENV /env -ENV PATH /env/bin:$PATH -ADD requirements.txt /app/requirements.txt -RUN pip install -r /app/requirements.txt diff --git a/tests/python3-libraries/Makefile b/tests/python3-libraries/Makefile index 7241697f..92fb683b 100644 --- a/tests/python3-libraries/Makefile +++ b/tests/python3-libraries/Makefile @@ -1,3 +1,3 @@ .PHONY: all all: - docker build . + ../../ext_run.sh -i "${IMAGE_NAME}" -c python3-libraries.yaml -w ../.. -v diff --git a/tests/python3-libraries/python3-libraries.yaml b/tests/python3-libraries/python3-libraries.yaml index 3ddf6e21..cd407604 100644 --- a/tests/python3-libraries/python3-libraries.yaml +++ b/tests/python3-libraries/python3-libraries.yaml @@ -1,13 +1,12 @@ schemaVersion: "1.0.0" -commandTests: - - name: "env var setup" - envVars: - - key: "VIRTUAL_ENV" - value: "/env" - - key: "PATH" - value: "/env/bin:$PATH" - command: ["echo", "'setup complete'"] +globalEnvVars: + - key: "VIRTUAL_ENV" + value: "/env" + - key: "PATH" + value: "/env/bin:$PATH" + +commandTests: - name: "requirements" setup: [["virtualenv", "-p", "/opt/python3.5/bin/python3.5", "/env"]] command: ["pip", "install", "-r", "/workspace/tests/python3-libraries/requirements.txt"] diff --git a/tests/virtualenv/Dockerfile b/tests/virtualenv/Dockerfile deleted file mode 100644 index 416a773b..00000000 --- a/tests/virtualenv/Dockerfile +++ /dev/null @@ -1,43 +0,0 @@ -FROM google/python - -ENV VIRTUAL_ENV /env -ENV PATH /env/bin:$PATH - -RUN if [ "$(which python)" != "/usr/bin/python" ]; then exit 1; fi; -RUN virtualenv /env -# All commands from this point on should use the virtualenv -RUN if [ "$(which python)" != "/env/bin/python" ]; then exit 1; fi; -RUN if [ "$(python --version 2>&1)" != "Python 2.7.9" ]; then exit 1; fi; -RUN if [ "$(which pip)" != "/env/bin/pip" ]; then exit 1; fi; -RUN pip install gunicorn flask -RUN if [ "$(which gunicorn)" != "/env/bin/gunicorn" ]; then exit 1; fi; -RUN python -c "import sys; import flask; sys.exit(0 if flask.__file__.startswith('/env') else 1)" - -# Python 3.4 -RUN rm -rf /env -RUN if [ "$(which python3.4)" != "/usr/bin/python3.4" ]; then exit 1; fi; -RUN virtualenv -p python3.4 /env -# All commands from this point on should use the virtualenv -RUN if [ "$(which python)" != "/env/bin/python" ]; then exit 1; fi; -RUN if [ "$(which python3)" != "/env/bin/python3" ]; then exit 1; fi; -RUN if [ "$(python --version 2>&1)" != "Python 3.4.2" ]; then exit 1; fi; -RUN if [ "$(which pip)" != "/env/bin/pip" ]; then exit 1; fi; -RUN if [ "$(which pip3)" != "/env/bin/pip3" ]; then exit 1; fi; -RUN pip install gunicorn flask -RUN if [ "$(which gunicorn)" != "/env/bin/gunicorn" ]; then exit 1; fi; -RUN python -c "import sys; import flask; sys.exit(0 if flask.__file__.startswith('/env') else 1)" - -# Python 3.5 -RUN rm -rf /env -RUN if [ "$(which python3.5)" != "/opt/python3.5/bin/python3.5" ]; then exit 1; fi; -RUN virtualenv -p python3.5 /env -# All commands from this point on should use the virtualenv -RUN if [ "$(which python)" != "/env/bin/python" ]; then exit 1; fi; -RUN if [ "$(which python3)" != "/env/bin/python3" ]; then exit 1; fi; -RUN if [ "$(which python3.5)" != "/env/bin/python3.5" ]; then exit 1; fi; -RUN if [ "$(python --version 2>&1)" != "Python 3.5.2" ]; then exit 1; fi; -RUN if [ "$(which pip)" != "/env/bin/pip" ]; then exit 1; fi; -RUN if [ "$(which pip3)" != "/env/bin/pip3" ]; then exit 1; fi; -RUN pip install gunicorn flask -RUN if [ "$(which gunicorn)" != "/env/bin/gunicorn" ]; then exit 1; fi; -RUN python -c "import sys; import flask; sys.exit(0 if flask.__file__.startswith('/env') else 1)" diff --git a/tests/virtualenv/Makefile b/tests/virtualenv/Makefile index 7241697f..f5df5ce4 100644 --- a/tests/virtualenv/Makefile +++ b/tests/virtualenv/Makefile @@ -1,3 +1,3 @@ .PHONY: all all: - docker build . + ../../ext_run.sh -i "${IMAGE_NAME}" -c virtualenv.yaml -v diff --git a/tests/virtualenv/virtualenv.yaml b/tests/virtualenv/virtualenv.yaml index 9b5c0401..1448f3d1 100644 --- a/tests/virtualenv/virtualenv.yaml +++ b/tests/virtualenv/virtualenv.yaml @@ -1,12 +1,12 @@ schemaVersion: "1.0.0" + +globalEnvVars: + - key: "VIRTUAL_ENV" + value: "/env" + - key: "PATH" + value: "/env/bin:$PATH" + commandTests: - - name: "env var setup" - envVars: - - key: "VIRTUAL_ENV" - value: "/env" - - key: "PATH" - value: "/env/bin:$PATH" - command: ["echo", "'setup complete'"] ##################################################### # Default Python Tests @@ -87,10 +87,14 @@ commandTests: command: ["rm", "-rf", "/env"] - name: "virtualenv python installation" - setup: [["virtualenv", "-p", "/opt/python3.5/bin/python3.5", "/env"]] + setup: [["virtualenv", "-p", "python3.5", "/env"]] command: ["which", "python"] expectedOutput: ["/env/bin/python\n"] + - name: "python installation" + command: ["which", "python3.5"] + expectedOutput: ["/opt/python3.5/bin/python3.5\n"] + - name: "virtualenv python3 installation" command: ["which", "python3"] expectedOutput: ["/env/bin/python3\n"] From 725699ab4d82ea1a12328a1a4520bc22c0876f47 Mon Sep 17 00:00:00 2001 From: Nick Kubala Date: Wed, 16 Nov 2016 11:55:28 -0800 Subject: [PATCH 059/362] update makefile with local targets --- Makefile | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 974a6d6c..8a53071e 100644 --- a/Makefile +++ b/Makefile @@ -9,22 +9,22 @@ CANDIDATE_NAME ?= $(shell date +%Y-%m-%d_%H_%M) IMAGE_NAME ?= google/python:$(CANDIDATE_NAME) export IMAGE_NAME -.PHONY: build -build: build-interpreters +.PHONY: local-image +local-image: build-interpreters docker build $(DOCKER_FLAGS) -t "$(IMAGE_NAME)" . docker tag -f "$(IMAGE_NAME)" "google/python" +.PHONY: local-tests +local-tests: local-image + curl https://raw.githubusercontent.com/GoogleCloudPlatform/runtimes-common/master/structure_tests/ext_run.sh > ext_run.sh + chmod +x ext_run.sh + make -C tests all + .PHONY: build-interpreters build-interpreters: export DOCKER_FLAGS make -C python-interpreter-builder build -.PHONY: tests -tests: - curl https://raw.githubusercontent.com/GoogleCloudPlatform/runtimes-common/master/structure_tests/ext_run.sh > ext_run.sh - chmod +x ext_run.sh - make -C tests all - .PHONY: benchmarks benchmarks: make -C tests benchmarks From 19cb998b8b9eda89b35410852079163df0b0aa08 Mon Sep 17 00:00:00 2001 From: Nick Kubala Date: Wed, 16 Nov 2016 17:09:28 -0800 Subject: [PATCH 060/362] split virtualenv tests into individual files for each python version --- tests/virtualenv/Makefile | 4 +- tests/virtualenv/virtualenv.yaml | 124 ---------------------- tests/virtualenv/virtualenv_default.yaml | 35 ++++++ tests/virtualenv/virtualenv_python34.yaml | 44 ++++++++ tests/virtualenv/virtualenv_python35.yaml | 48 +++++++++ 5 files changed, 130 insertions(+), 125 deletions(-) delete mode 100644 tests/virtualenv/virtualenv.yaml create mode 100644 tests/virtualenv/virtualenv_default.yaml create mode 100644 tests/virtualenv/virtualenv_python34.yaml create mode 100644 tests/virtualenv/virtualenv_python35.yaml diff --git a/tests/virtualenv/Makefile b/tests/virtualenv/Makefile index f5df5ce4..66a19642 100644 --- a/tests/virtualenv/Makefile +++ b/tests/virtualenv/Makefile @@ -1,3 +1,5 @@ .PHONY: all all: - ../../ext_run.sh -i "${IMAGE_NAME}" -c virtualenv.yaml -v + ../../ext_run.sh -i "${IMAGE_NAME}" -c virtualenv_default.yaml -v + ../../ext_run.sh -i "${IMAGE_NAME}" -c virtualenv_python34.yaml -v + ../../ext_run.sh -i "${IMAGE_NAME}" -c virtualenv_python35.yaml -v diff --git a/tests/virtualenv/virtualenv.yaml b/tests/virtualenv/virtualenv.yaml deleted file mode 100644 index 1448f3d1..00000000 --- a/tests/virtualenv/virtualenv.yaml +++ /dev/null @@ -1,124 +0,0 @@ -schemaVersion: "1.0.0" - -globalEnvVars: - - key: "VIRTUAL_ENV" - value: "/env" - - key: "PATH" - value: "/env/bin:$PATH" - -commandTests: - -##################################################### -# Default Python Tests -##################################################### - - - name: "python installation" - command: ["which", "python"] - expectedOutput: ["/usr/bin/python\n"] - - - name: "virtualenv installation" - setup: [["virtualenv", "/env"]] - command: ["which", "python"] - expectedOutput": ["/env/bin/python\n"] - - - name: "python version" - command: ["python", "--version"] - # we check stderr instead of stdout for Python versions < 3.4 - # https://bugs.python.org/issue18338 - expectedError: ["Python 2.7.9\n"] - - - name: "pip installation" - command: ["which", "pip"] - expectedOutput: ["/env/bin/pip\n"] - - - name: "gunicorn flask" - setup: [["pip", "install", "gunicorn", "flask"]] - command: ["which", "gunicorn"] - expectedOutput: ["/env/bin/gunicorn"] - - - name: "flask integration" - command: ["python", "-c", "\"import sys; import flask; sys.exit(0 if flask.__file__.startswith('/env') else 1)\""] - -##################################################### -# Python 3.4 Tests -##################################################### - - - name: "virtual env teardown" - command: ["rm", "-rf", "/env"] - - - name: "python installation" - command: ["which", "python3.4"] - expectedOutput: ["/usr/bin/python3.4\n"] - - - name: "virtualenv python installation" - setup: [["virtualenv", "-p", "python3.4", "/env"]] - command: ["which", "python"] - expectedOutput: ["/env/bin/python\n"] - - - name: "virtualenv python3 installation" - command: ["which", "python3"] - expectedOutput: ["/env/bin/python3\n"] - - - name: "python version" - command: ["python", "--version"] - expectedOutput: ["Python 3.4.2\n"] - - - name: "pip installation" - command: ["which", "pip"] - expectedOutput: ["/env/bin/pip\n"] - - - name: "pip3 installation" - command: ["which", "pip3"] - expectedOutput: ["/env/bin/pip3\n"] - - - name: "gunicorn flask" - setup: [["pip", "install", "gunicorn", "flask"]] - command: ["which", "gunicorn"] - expectedOutput: ["/env/bin/gunicorn"] - - - name: "flask integration" - command: ["python", "-c", "\"import sys; import flask; sys.exit(0 if flask.__file__.startswith('/env') else 1)\""] - -##################################################### -# Python 3.5 Tests -##################################################### - - - name: "virtual env teardown" - command: ["rm", "-rf", "/env"] - - - name: "virtualenv python installation" - setup: [["virtualenv", "-p", "python3.5", "/env"]] - command: ["which", "python"] - expectedOutput: ["/env/bin/python\n"] - - - name: "python installation" - command: ["which", "python3.5"] - expectedOutput: ["/opt/python3.5/bin/python3.5\n"] - - - name: "virtualenv python3 installation" - command: ["which", "python3"] - expectedOutput: ["/env/bin/python3\n"] - - - name: "virtualenv python3.5 installation" - command: ["which", "python3.5"] - expectedOutput: ["/env/bin/python3.5\n"] - - - name: "python version" - command: ["python", "--version"] - expectedOutput: ["Python 3.5.2\n"] - - - name: "pip installation" - command: ["which", "pip"] - expectedOutput: ["/env/bin/pip\n"] - - - name: "pip3 installation" - command: ["which", "pip3"] - expectedOutput: ["/env/bin/pip3\n"] - - - name: "gunicorn flask" - setup: [["pip", "install", "gunicorn", "flask"]] - command: ["which", "gunicorn"] - expectedOutput: ["/env/bin/gunicorn"] - - - name: "flask integration" - command: ["python", "-c", "\"import sys; import flask; sys.exit(0 if flask.__file__.startswith('/env') else 1)\""] diff --git a/tests/virtualenv/virtualenv_default.yaml b/tests/virtualenv/virtualenv_default.yaml new file mode 100644 index 00000000..56669d60 --- /dev/null +++ b/tests/virtualenv/virtualenv_default.yaml @@ -0,0 +1,35 @@ +schemaVersion: "1.0.0" + +globalEnvVars: + - key: "VIRTUAL_ENV" + value: "/env" + - key: "PATH" + value: "/env/bin:$PATH" + +commandTests: + - name: "python installation" + command: ["which", "python"] + expectedOutput: ["/usr/bin/python\n"] + + - name: "virtualenv installation" + setup: [["virtualenv", "/env"]] + command: ["which", "python"] + expectedOutput": ["/env/bin/python\n"] + + - name: "python version" + command: ["python", "--version"] + # we check stderr instead of stdout for Python versions < 3.4 + # https://bugs.python.org/issue18338 + expectedError: ["Python 2.7.9\n"] + + - name: "pip installation" + command: ["which", "pip"] + expectedOutput: ["/env/bin/pip\n"] + + - name: "gunicorn flask" + setup: [["pip", "install", "gunicorn", "flask"]] + command: ["which", "gunicorn"] + expectedOutput: ["/env/bin/gunicorn"] + + - name: "flask integration" + command: ["python", "-c", "\"import sys; import flask; sys.exit(0 if flask.__file__.startswith('/env') else 1)\""] diff --git a/tests/virtualenv/virtualenv_python34.yaml b/tests/virtualenv/virtualenv_python34.yaml new file mode 100644 index 00000000..0a498186 --- /dev/null +++ b/tests/virtualenv/virtualenv_python34.yaml @@ -0,0 +1,44 @@ +schemaVersion: "1.0.0" + +globalEnvVars: + - key: "VIRTUAL_ENV" + value: "/env" + - key: "PATH" + value: "/env/bin:$PATH" + +commandTests: + - name: "virtual env teardown" + command: ["rm", "-rf", "/env"] + + - name: "python installation" + command: ["which", "python3.4"] + expectedOutput: ["/usr/bin/python3.4\n"] + + - name: "virtualenv python installation" + setup: [["virtualenv", "-p", "python3.4", "/env"]] + command: ["which", "python"] + expectedOutput: ["/env/bin/python\n"] + + - name: "virtualenv python3 installation" + command: ["which", "python3"] + expectedOutput: ["/env/bin/python3\n"] + + - name: "python version" + command: ["python", "--version"] + expectedOutput: ["Python 3.4.2\n"] + + - name: "pip installation" + command: ["which", "pip"] + expectedOutput: ["/env/bin/pip\n"] + + - name: "pip3 installation" + command: ["which", "pip3"] + expectedOutput: ["/env/bin/pip3\n"] + + - name: "gunicorn flask" + setup: [["pip", "install", "gunicorn", "flask"]] + command: ["which", "gunicorn"] + expectedOutput: ["/env/bin/gunicorn"] + + - name: "flask integration" + command: ["python", "-c", "\"import sys; import flask; sys.exit(0 if flask.__file__.startswith('/env') else 1)\""] diff --git a/tests/virtualenv/virtualenv_python35.yaml b/tests/virtualenv/virtualenv_python35.yaml new file mode 100644 index 00000000..070ea174 --- /dev/null +++ b/tests/virtualenv/virtualenv_python35.yaml @@ -0,0 +1,48 @@ +schemaVersion: "1.0.0" + +globalEnvVars: + - key: "VIRTUAL_ENV" + value: "/env" + - key: "PATH" + value: "/env/bin:$PATH" + +commandTests: + - name: "virtual env teardown" + command: ["rm", "-rf", "/env"] + + - name: "virtualenv python installation" + setup: [["virtualenv", "-p", "python3.5", "/env"]] + command: ["which", "python"] + expectedOutput: ["/env/bin/python\n"] + + - name: "python installation" + command: ["which", "python3.5"] + expectedOutput: ["/opt/python3.5/bin/python3.5\n"] + + - name: "virtualenv python3 installation" + command: ["which", "python3"] + expectedOutput: ["/env/bin/python3\n"] + + - name: "virtualenv python3.5 installation" + command: ["which", "python3.5"] + expectedOutput: ["/env/bin/python3.5\n"] + + - name: "python version" + command: ["python", "--version"] + expectedOutput: ["Python 3.5.2\n"] + + - name: "pip installation" + command: ["which", "pip"] + expectedOutput: ["/env/bin/pip\n"] + + - name: "pip3 installation" + command: ["which", "pip3"] + expectedOutput: ["/env/bin/pip3\n"] + + - name: "gunicorn flask" + setup: [["pip", "install", "gunicorn", "flask"]] + command: ["which", "gunicorn"] + expectedOutput: ["/env/bin/gunicorn"] + + - name: "flask integration" + command: ["python", "-c", "\"import sys; import flask; sys.exit(0 if flask.__file__.startswith('/env') else 1)\""] From 5284f3807efa094de7b8974ad3890eee614f9ec3 Mon Sep 17 00:00:00 2001 From: Nick Kubala Date: Thu, 17 Nov 2016 09:43:00 -0800 Subject: [PATCH 061/362] fix cloudbuild yaml --- cloudbuild.yaml.in | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cloudbuild.yaml.in b/cloudbuild.yaml.in index b79d7d12..445e8986 100644 --- a/cloudbuild.yaml.in +++ b/cloudbuild.yaml.in @@ -8,7 +8,9 @@ steps: - name: gcr.io/gcp-runtimes/structure_test args: [ '-i', '${IMAGE_NAME}', - '--config', '/workspace/tests/virtualenv/virtualenv.yaml', + '--config', '/workspace/tests/virtualenv/virtualenv_default.yaml', + '--config', '/workspace/tests/virtualenv/virtualenv_python34.yaml', + '--config', '/workspace/tests/virtualenv/virtualenv_python35.yaml', '--config', '/workspace/tests/no-virtualenv/no-virtualenv.yaml', '--config', '/workspace/tests/python2-libraries/python2-libraries.yaml', '--config', '/workspace/tests/python2-libraries/python3-libraries.yaml', From 13da3b11bb9f83ae6704e8bcb515c665511780e5 Mon Sep 17 00:00:00 2001 From: Nick Kubala Date: Thu, 17 Nov 2016 15:17:22 -0800 Subject: [PATCH 062/362] add build targets for build/build-local, and structure/integration tests. parameterize dockerfiles --- Makefile | 35 +++++++++++++------ cloudbuild.yaml.in | 18 +++++----- jenkins_build.sh | 16 +++++++++ system_tests/.gitignore | 1 + system_tests/{Dockerfile => Dockerfile.in} | 2 +- system_tests/Makefile | 1 + tests/Makefile | 9 +++-- tests/benchmark/.gitignore | 1 + tests/benchmark/{Dockerfile => Dockerfile.in} | 2 +- tests/benchmark/Makefile | 2 ++ tests/google-cloud-python/.gitignore | 1 + .../{Dockerfile => Dockerfile.in} | 2 +- 12 files changed, 66 insertions(+), 24 deletions(-) create mode 100755 jenkins_build.sh rename system_tests/{Dockerfile => Dockerfile.in} (97%) create mode 100644 tests/benchmark/.gitignore rename tests/benchmark/{Dockerfile => Dockerfile.in} (82%) create mode 100644 tests/google-cloud-python/.gitignore rename tests/google-cloud-python/{Dockerfile => Dockerfile.in} (95%) diff --git a/Makefile b/Makefile index 8a53071e..b5aeccdc 100644 --- a/Makefile +++ b/Makefile @@ -14,26 +14,41 @@ local-image: build-interpreters docker build $(DOCKER_FLAGS) -t "$(IMAGE_NAME)" . docker tag -f "$(IMAGE_NAME)" "google/python" -.PHONY: local-tests -local-tests: local-image - curl https://raw.githubusercontent.com/GoogleCloudPlatform/runtimes-common/master/structure_tests/ext_run.sh > ext_run.sh - chmod +x ext_run.sh - make -C tests all - .PHONY: build-interpreters build-interpreters: export DOCKER_FLAGS make -C python-interpreter-builder build +.PHONY: cloudbuild +cloudbuild: + envsubst < cloudbuild.yaml.in > cloudbuild.yaml + gcloud alpha container builds create . --config=cloudbuild.yaml + +.PHONY: build +# no structure tests since they are implicit in cloudbuild +build: cloudbuild integration-tests + +.PHONY: build-local +build-local: local-image structure-tests integration-tests + + +.PHONY: structure-tests +structure-tests: local-image + curl https://raw.githubusercontent.com/GoogleCloudPlatform/runtimes-common/master/structure_tests/ext_run.sh > ext_run.sh + chmod +x ext_run.sh + make -C tests structure-tests + .PHONY: benchmarks benchmarks: make -C tests benchmarks +.PHONY: google-cloud-python +google-cloud-python: + make -C tests google-cloud-python + .PHONY: google-cloud-system-tests google-cloud-system-tests: make -C system_tests -.PHONY: cloudbuild -cloudbuild: - envsubst cloudbuild.yaml - gcloud alpha container builds create . --config=cloudbuild.yaml +.PHONY: integration-tests +tests: benchmarks google-cloud-system-tests google-cloud-python diff --git a/cloudbuild.yaml.in b/cloudbuild.yaml.in index 445e8986..02cf1709 100644 --- a/cloudbuild.yaml.in +++ b/cloudbuild.yaml.in @@ -1,13 +1,13 @@ steps: -- name: gcr.io/cloud-builders/docker - args: ['build', '--tag=interpreter', 'python-interpreter-builder'] -- name: interpreter - args: ['cp', '/interpreters.tar.gz', '/workspace/'] -- name: gcr.io/cloud-builders/docker - args: ['build', '--tag=${IMAGE_NAME}', '.'] +# - name: gcr.io/cloud-builders/docker +# args: ['build', '--tag=interpreter', 'python-interpreter-builder'] +# - name: interpreter +# args: ['cp', '/interpreters.tar.gz', '/workspace/'] +# - name: gcr.io/cloud-builders/docker +# args: ['build', '--tag=${IMAGE_NAME}', '.'] - name: gcr.io/gcp-runtimes/structure_test args: [ - '-i', '${IMAGE_NAME}', + '-i', 'gcr.io/nick-cloudbuild/python', '--config', '/workspace/tests/virtualenv/virtualenv_default.yaml', '--config', '/workspace/tests/virtualenv/virtualenv_python34.yaml', '--config', '/workspace/tests/virtualenv/virtualenv_python35.yaml', @@ -16,5 +16,7 @@ steps: '--config', '/workspace/tests/python2-libraries/python3-libraries.yaml', '-v' ] +- name: gcr.io/gcp-runtimes/make + args: ['-C', 'tests', 'benchmarks'] images: - ['${IMAGE_NAME}'] + ['gcr.io/nick-cloudbuild/python'] diff --git a/jenkins_build.sh b/jenkins_build.sh new file mode 100755 index 00000000..a1226e18 --- /dev/null +++ b/jenkins_build.sh @@ -0,0 +1,16 @@ +RUNTIME_NAME="python" + +CANDIDATE_NAME=`date +%Y-%m-%d_%H_%M` +echo "CANDIDATE_NAME:${CANDIDATE_NAME}" + +IMAGE_NAME="${DOCKER_NAMESPACE}/${RUNTIME_NAME}:${CANDIDATE_NAME}" + +export IMAGE_NAME +export FORCE_REBUILD +make build + +if [ "${UPLOAD_TO_STAGING}" = "true" ]; then + STAGING="${DOCKER_NAMESPACE}/${RUNTIME_NAME}:staging" + docker tag -f "${IMAGE_NAME}" "${STAGING}" + gcloud docker push "${STAGING}" +fi diff --git a/system_tests/.gitignore b/system_tests/.gitignore index 2fc08f4d..3a8c6aa8 100644 --- a/system_tests/.gitignore +++ b/system_tests/.gitignore @@ -1,2 +1,3 @@ data/ secrets.tar +Dockerfile diff --git a/system_tests/Dockerfile b/system_tests/Dockerfile.in similarity index 97% rename from system_tests/Dockerfile rename to system_tests/Dockerfile.in index 8a306f64..6cbf6db3 100644 --- a/system_tests/Dockerfile +++ b/system_tests/Dockerfile.in @@ -1,4 +1,4 @@ -FROM google/python +FROM ${IMAGE_NAME} # Inject secrets ADD data/ data/ diff --git a/system_tests/Makefile b/system_tests/Makefile index 4dbde7d1..f3c554f3 100644 --- a/system_tests/Makefile +++ b/system_tests/Makefile @@ -4,4 +4,5 @@ CACHE ?= --no-cache .PHONY: all all: + envsubst < Dockerfile.in > Dockerfile docker build $(CACHE) . diff --git a/tests/Makefile b/tests/Makefile index b848eb19..1320f1a9 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -1,11 +1,14 @@ -.PHONY: all -all: +.PHONY: structure-tests +structure-tests: make -C no-virtualenv make -C virtualenv - make -C google-cloud-python make -C python2-libraries make -C python3-libraries .PHONY: benchmarks benchmarks: make -C benchmark all + +.PHONY: google-cloud-python +google-cloud-python: + make -C google-cloud-python all diff --git a/tests/benchmark/.gitignore b/tests/benchmark/.gitignore new file mode 100644 index 00000000..94143827 --- /dev/null +++ b/tests/benchmark/.gitignore @@ -0,0 +1 @@ +Dockerfile diff --git a/tests/benchmark/Dockerfile b/tests/benchmark/Dockerfile.in similarity index 82% rename from tests/benchmark/Dockerfile rename to tests/benchmark/Dockerfile.in index e00792d6..c202cb2f 100644 --- a/tests/benchmark/Dockerfile +++ b/tests/benchmark/Dockerfile.in @@ -1,4 +1,4 @@ -FROM google/python +FROM ${IMAGE_NAME} RUN hg clone https://hg.python.org/benchmarks /app/benchmarks WORKDIR /app/benchmarks diff --git a/tests/benchmark/Makefile b/tests/benchmark/Makefile index 64290f94..dd56208a 100644 --- a/tests/benchmark/Makefile +++ b/tests/benchmark/Makefile @@ -4,9 +4,11 @@ all: 2vs3 34vs35 # Tests Python 3.4 against Python 2.7. .PHONY: 2vs3 2vs3: + envsubst < Dockerfile.in > Dockerfile cat Dockerfile Dockerfile.2vs3 | docker build - # Tests Python 3.5 against Python 3.4. .PHONY: 2vs3 34vs35: + envsubst < Dockerfile.in > Dockerfile cat Dockerfile Dockerfile.34vs35 | docker build - diff --git a/tests/google-cloud-python/.gitignore b/tests/google-cloud-python/.gitignore new file mode 100644 index 00000000..94143827 --- /dev/null +++ b/tests/google-cloud-python/.gitignore @@ -0,0 +1 @@ +Dockerfile diff --git a/tests/google-cloud-python/Dockerfile b/tests/google-cloud-python/Dockerfile.in similarity index 95% rename from tests/google-cloud-python/Dockerfile rename to tests/google-cloud-python/Dockerfile.in index ade8d818..a952f865 100644 --- a/tests/google-cloud-python/Dockerfile +++ b/tests/google-cloud-python/Dockerfile.in @@ -1,4 +1,4 @@ -FROM google/python +FROM ${IMAGE_NAME} # Install tox RUN pip install --upgrade tox From 592c964328430faa3aa41263a2bdc9f04195efc6 Mon Sep 17 00:00:00 2001 From: Nick Kubala Date: Thu, 17 Nov 2016 15:51:30 -0800 Subject: [PATCH 063/362] fix cloudbuild.yaml (again) --- cloudbuild.yaml.in | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/cloudbuild.yaml.in b/cloudbuild.yaml.in index 02cf1709..445e8986 100644 --- a/cloudbuild.yaml.in +++ b/cloudbuild.yaml.in @@ -1,13 +1,13 @@ steps: -# - name: gcr.io/cloud-builders/docker -# args: ['build', '--tag=interpreter', 'python-interpreter-builder'] -# - name: interpreter -# args: ['cp', '/interpreters.tar.gz', '/workspace/'] -# - name: gcr.io/cloud-builders/docker -# args: ['build', '--tag=${IMAGE_NAME}', '.'] +- name: gcr.io/cloud-builders/docker + args: ['build', '--tag=interpreter', 'python-interpreter-builder'] +- name: interpreter + args: ['cp', '/interpreters.tar.gz', '/workspace/'] +- name: gcr.io/cloud-builders/docker + args: ['build', '--tag=${IMAGE_NAME}', '.'] - name: gcr.io/gcp-runtimes/structure_test args: [ - '-i', 'gcr.io/nick-cloudbuild/python', + '-i', '${IMAGE_NAME}', '--config', '/workspace/tests/virtualenv/virtualenv_default.yaml', '--config', '/workspace/tests/virtualenv/virtualenv_python34.yaml', '--config', '/workspace/tests/virtualenv/virtualenv_python35.yaml', @@ -16,7 +16,5 @@ steps: '--config', '/workspace/tests/python2-libraries/python3-libraries.yaml', '-v' ] -- name: gcr.io/gcp-runtimes/make - args: ['-C', 'tests', 'benchmarks'] images: - ['gcr.io/nick-cloudbuild/python'] + ['${IMAGE_NAME}'] From 610870a77d84142662ff6fc637dd67ec551bcbe8 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Sun, 4 Dec 2016 20:38:33 -0800 Subject: [PATCH 064/362] Fix order of tests. --- tests/virtualenv/virtualenv_python35.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/virtualenv/virtualenv_python35.yaml b/tests/virtualenv/virtualenv_python35.yaml index 070ea174..23dc1367 100644 --- a/tests/virtualenv/virtualenv_python35.yaml +++ b/tests/virtualenv/virtualenv_python35.yaml @@ -10,15 +10,15 @@ commandTests: - name: "virtual env teardown" command: ["rm", "-rf", "/env"] + - name: "python installation" + command: ["which", "python3.5"] + expectedOutput: ["/opt/python3.5/bin/python3.5\n"] + - name: "virtualenv python installation" setup: [["virtualenv", "-p", "python3.5", "/env"]] command: ["which", "python"] expectedOutput: ["/env/bin/python\n"] - - name: "python installation" - command: ["which", "python3.5"] - expectedOutput: ["/opt/python3.5/bin/python3.5\n"] - - name: "virtualenv python3 installation" command: ["which", "python3"] expectedOutput: ["/env/bin/python3\n"] From f6bc007704f49a1d7ea175753a0c5c23f30c1854 Mon Sep 17 00:00:00 2001 From: Douglas Greiman Date: Mon, 5 Dec 2016 09:28:36 -0800 Subject: [PATCH 065/362] Fix "docker tag" usage for Docker 1.12 (#50) Docker 1.12 removed support for the "-f" flag to "docker tag". Replace it with code that should work for at least Docker 1.09 and above. --- Makefile | 3 ++- jenkins_build.sh | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index b5aeccdc..c44a3436 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,8 @@ export IMAGE_NAME .PHONY: local-image local-image: build-interpreters docker build $(DOCKER_FLAGS) -t "$(IMAGE_NAME)" . - docker tag -f "$(IMAGE_NAME)" "google/python" + -docker rmi "google/python" 2>/dev/null + docker tag "$(IMAGE_NAME)" "google/python" .PHONY: build-interpreters build-interpreters: diff --git a/jenkins_build.sh b/jenkins_build.sh index a1226e18..1e51c06f 100755 --- a/jenkins_build.sh +++ b/jenkins_build.sh @@ -11,6 +11,7 @@ make build if [ "${UPLOAD_TO_STAGING}" = "true" ]; then STAGING="${DOCKER_NAMESPACE}/${RUNTIME_NAME}:staging" - docker tag -f "${IMAGE_NAME}" "${STAGING}" + docker rmi "${STAGING}" 2>/dev/null || true # Ignore if tag not present + docker tag "${IMAGE_NAME}" "${STAGING}" gcloud docker push "${STAGING}" fi From b75fb62aad4b3bd476c2f3be3c8693dca9714fef Mon Sep 17 00:00:00 2001 From: Nick Kubala Date: Mon, 5 Dec 2016 12:05:52 -0800 Subject: [PATCH 066/362] prevent container build from using cached images (#51) --- cloudbuild.yaml.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cloudbuild.yaml.in b/cloudbuild.yaml.in index 445e8986..78f1636b 100644 --- a/cloudbuild.yaml.in +++ b/cloudbuild.yaml.in @@ -1,10 +1,10 @@ steps: - name: gcr.io/cloud-builders/docker - args: ['build', '--tag=interpreter', 'python-interpreter-builder'] + args: ['build', '--tag=interpreter', '--no-cache', 'python-interpreter-builder'] - name: interpreter args: ['cp', '/interpreters.tar.gz', '/workspace/'] - name: gcr.io/cloud-builders/docker - args: ['build', '--tag=${IMAGE_NAME}', '.'] + args: ['build', '--tag=${IMAGE_NAME}', '--no-cache', '.'] - name: gcr.io/gcp-runtimes/structure_test args: [ '-i', '${IMAGE_NAME}', From 6ebb53a950cf042a8279decd0cde3a10ad8d32c1 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Thu, 8 Dec 2016 11:44:43 -0800 Subject: [PATCH 067/362] Fix Jenkins script to exit on error --- jenkins_build.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/jenkins_build.sh b/jenkins_build.sh index 1e51c06f..345cb735 100755 --- a/jenkins_build.sh +++ b/jenkins_build.sh @@ -1,3 +1,7 @@ +#!/bin/sh + +set -eu + RUNTIME_NAME="python" CANDIDATE_NAME=`date +%Y-%m-%d_%H_%M` From a1a048ca7326fe4862c9db0b0baaed07744d5416 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Thu, 8 Dec 2016 14:06:23 -0800 Subject: [PATCH 068/362] Increase cloudbuild timeout to 1 hour --- cloudbuild.yaml.in | 1 + 1 file changed, 1 insertion(+) diff --git a/cloudbuild.yaml.in b/cloudbuild.yaml.in index 78f1636b..6049f549 100644 --- a/cloudbuild.yaml.in +++ b/cloudbuild.yaml.in @@ -1,3 +1,4 @@ +timeout: 3600s steps: - name: gcr.io/cloud-builders/docker args: ['build', '--tag=interpreter', '--no-cache', 'python-interpreter-builder'] From 12651f0c3a9d326d53338b36514cfd60caee31f6 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Thu, 8 Dec 2016 15:04:11 -0800 Subject: [PATCH 069/362] Fix typo in directory name --- cloudbuild.yaml.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudbuild.yaml.in b/cloudbuild.yaml.in index 6049f549..1404b4a8 100644 --- a/cloudbuild.yaml.in +++ b/cloudbuild.yaml.in @@ -14,7 +14,7 @@ steps: '--config', '/workspace/tests/virtualenv/virtualenv_python35.yaml', '--config', '/workspace/tests/no-virtualenv/no-virtualenv.yaml', '--config', '/workspace/tests/python2-libraries/python2-libraries.yaml', - '--config', '/workspace/tests/python2-libraries/python3-libraries.yaml', + '--config', '/workspace/tests/python3-libraries/python3-libraries.yaml', '-v' ] images: From e0fa299947ff7c4a039a9026d2ad87ebdc5bc13f Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Thu, 8 Dec 2016 16:52:01 -0800 Subject: [PATCH 070/362] Explicitly nuke old virtualenvs before creating new one. This fixes: OSError: [Errno 26] Text file busy: '/env/bin/python3.5' --- tests/python2-libraries/python2-libraries.yaml | 3 +++ tests/python3-libraries/python3-libraries.yaml | 3 +++ tests/virtualenv/virtualenv_default.yaml | 3 +++ 3 files changed, 9 insertions(+) diff --git a/tests/python2-libraries/python2-libraries.yaml b/tests/python2-libraries/python2-libraries.yaml index fb4f977a..5d298b58 100644 --- a/tests/python2-libraries/python2-libraries.yaml +++ b/tests/python2-libraries/python2-libraries.yaml @@ -7,6 +7,9 @@ globalEnvVars: value: "/env/bin:$PATH" commandTests: + - name: "virtual env teardown" + command: ["rm", "-rf", "/env"] + - name: "requirements" setup: [["virtualenv", "/env"]] command: ["pip", "install", "-r", "/workspace/tests/python2-libraries/requirements.txt"] diff --git a/tests/python3-libraries/python3-libraries.yaml b/tests/python3-libraries/python3-libraries.yaml index cd407604..28613555 100644 --- a/tests/python3-libraries/python3-libraries.yaml +++ b/tests/python3-libraries/python3-libraries.yaml @@ -7,6 +7,9 @@ globalEnvVars: value: "/env/bin:$PATH" commandTests: + - name: "virtual env teardown" + command: ["rm", "-rf", "/env"] + - name: "requirements" setup: [["virtualenv", "-p", "/opt/python3.5/bin/python3.5", "/env"]] command: ["pip", "install", "-r", "/workspace/tests/python3-libraries/requirements.txt"] diff --git a/tests/virtualenv/virtualenv_default.yaml b/tests/virtualenv/virtualenv_default.yaml index 56669d60..e9187f13 100644 --- a/tests/virtualenv/virtualenv_default.yaml +++ b/tests/virtualenv/virtualenv_default.yaml @@ -7,6 +7,9 @@ globalEnvVars: value: "/env/bin:$PATH" commandTests: + - name: "virtual env teardown" + command: ["rm", "-rf", "/env"] + - name: "python installation" command: ["which", "python"] expectedOutput: ["/usr/bin/python\n"] From 662c6eec696d4cc7893e0fb25054ad35c015a159 Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Fri, 9 Dec 2016 14:15:16 -0500 Subject: [PATCH 071/362] adding license tests to structure tests (#53) * adding license tests to structure tests * adding license-test to structure-tests Makefile --- cloudbuild.yaml.in | 1 + tests/Makefile | 1 + tests/license-test/Makefile | 3 +++ tests/license-test/license-test.yaml | 5 +++++ 4 files changed, 10 insertions(+) create mode 100644 tests/license-test/Makefile create mode 100644 tests/license-test/license-test.yaml diff --git a/cloudbuild.yaml.in b/cloudbuild.yaml.in index 1404b4a8..564a1dcd 100644 --- a/cloudbuild.yaml.in +++ b/cloudbuild.yaml.in @@ -15,6 +15,7 @@ steps: '--config', '/workspace/tests/no-virtualenv/no-virtualenv.yaml', '--config', '/workspace/tests/python2-libraries/python2-libraries.yaml', '--config', '/workspace/tests/python3-libraries/python3-libraries.yaml', + '--config', '/workspace/tests/license-test.yaml', '-v' ] images: diff --git a/tests/Makefile b/tests/Makefile index 1320f1a9..5953a62d 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -4,6 +4,7 @@ structure-tests: make -C virtualenv make -C python2-libraries make -C python3-libraries + make -C license-test .PHONY: benchmarks benchmarks: diff --git a/tests/license-test/Makefile b/tests/license-test/Makefile new file mode 100644 index 00000000..b56b60ea --- /dev/null +++ b/tests/license-test/Makefile @@ -0,0 +1,3 @@ +.PHONY: all +all: + ../../ext_run.sh -i "${IMAGE_NAME}" -c license-test.yaml -v diff --git a/tests/license-test/license-test.yaml b/tests/license-test/license-test.yaml new file mode 100644 index 00000000..d156688a --- /dev/null +++ b/tests/license-test/license-test.yaml @@ -0,0 +1,5 @@ +schemaVersion: "1.0.0" + +licenseTests: + - debian: true + files: [] From 568b3aeebec9b407ed0561070d1a5d7e9f0f7073 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 9 Dec 2016 14:11:10 -0800 Subject: [PATCH 072/362] Fix typo in path --- cloudbuild.yaml.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudbuild.yaml.in b/cloudbuild.yaml.in index 564a1dcd..79ee93ba 100644 --- a/cloudbuild.yaml.in +++ b/cloudbuild.yaml.in @@ -15,7 +15,7 @@ steps: '--config', '/workspace/tests/no-virtualenv/no-virtualenv.yaml', '--config', '/workspace/tests/python2-libraries/python2-libraries.yaml', '--config', '/workspace/tests/python3-libraries/python3-libraries.yaml', - '--config', '/workspace/tests/license-test.yaml', + '--config', '/workspace/tests/license-test/license-test.yaml', '-v' ] images: From ff5a79ae9bb8a466e5048bf027abf60ba777ddbe Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 9 Dec 2016 14:47:28 -0800 Subject: [PATCH 073/362] Ignore another generated file --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index b2c26136..2c260f13 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ cloudbuild.yaml -interpreters.tar.gz \ No newline at end of file +ext_run.sh +interpreters.tar.gz From 6cbb27c89ebcd21c35c6c2cc51c90aa7072bf77f Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 9 Dec 2016 14:48:07 -0800 Subject: [PATCH 074/362] Decouple some Makefile targets that didn't need to be coupled --- Makefile | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index c44a3436..528def03 100644 --- a/Makefile +++ b/Makefile @@ -20,9 +20,11 @@ build-interpreters: export DOCKER_FLAGS make -C python-interpreter-builder build -.PHONY: cloudbuild -cloudbuild: +cloudbuild.yaml: cloudbuild.yaml.in envsubst < cloudbuild.yaml.in > cloudbuild.yaml + +.PHONY: cloudbuild +cloudbuild: cloudbuild.yaml gcloud alpha container builds create . --config=cloudbuild.yaml .PHONY: build @@ -32,11 +34,13 @@ build: cloudbuild integration-tests .PHONY: build-local build-local: local-image structure-tests integration-tests - -.PHONY: structure-tests -structure-tests: local-image +.PHONY: ext_run.sh # Force refetch every time +ext_run.sh: curl https://raw.githubusercontent.com/GoogleCloudPlatform/runtimes-common/master/structure_tests/ext_run.sh > ext_run.sh chmod +x ext_run.sh + +.PHONY: structure-tests +structure-tests: local-image ext_run.sh make -C tests structure-tests .PHONY: benchmarks From 649b3c55daf4ddee46d2d49809fc25dea1c3bacb Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 9 Dec 2016 14:50:36 -0800 Subject: [PATCH 075/362] Add link to documentation --- tests/license-test/license-test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/license-test/license-test.yaml b/tests/license-test/license-test.yaml index d156688a..cb2d67ce 100644 --- a/tests/license-test/license-test.yaml +++ b/tests/license-test/license-test.yaml @@ -1,5 +1,6 @@ schemaVersion: "1.0.0" +# See https://github.com/GoogleCloudPlatform/runtimes-common/blob/master/structure_tests/README.md#license-tests licenseTests: - debian: true files: [] From 6550ea43b5466680267d7d51b46be7f557436190 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 9 Dec 2016 16:35:37 -0800 Subject: [PATCH 076/362] Fix non-hermetic target that was being treated like it was hermetic --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 528def03..3a2a526c 100644 --- a/Makefile +++ b/Makefile @@ -20,6 +20,7 @@ build-interpreters: export DOCKER_FLAGS make -C python-interpreter-builder build +.PHONY: cloudbuild.yaml # Force reevaluation of env vars every time cloudbuild.yaml: cloudbuild.yaml.in envsubst < cloudbuild.yaml.in > cloudbuild.yaml From 64bcd0768709625fbefeabc15fe76694af828d75 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Thu, 19 Jan 2017 14:47:16 -0800 Subject: [PATCH 077/362] Require explicit IMAGE_NAME to avoid errors and confusion. --- Makefile | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 3a2a526c..84c4651d 100644 --- a/Makefile +++ b/Makefile @@ -2,12 +2,10 @@ ifdef FORCE_REBUILD DOCKER_FLAGS = --no-cache --pull endif -# Note: 'make build/tests/benchmarks' share images by retagging the -# candidate as 'google/python'. So this could cause trouble with -# concurrent builds on the same machine. -CANDIDATE_NAME ?= $(shell date +%Y-%m-%d_%H_%M) -IMAGE_NAME ?= google/python:$(CANDIDATE_NAME) -export IMAGE_NAME +ifndef IMAGE_NAME +$(error IMAGE_NAME is not set; invoke make with something like IMAGE_NAME=google/python:2017-01-02_03_45) +endif + .PHONY: local-image local-image: build-interpreters From 7ee56fd69f19e80cb5814b827dbe5929185321b7 Mon Sep 17 00:00:00 2001 From: Nick Kubala Date: Mon, 5 Dec 2016 11:45:47 -0800 Subject: [PATCH 078/362] adding python integration sample app --- build.sh | 14 ++++ tests/integration/Dockerfile.in | 11 +++ tests/integration/requirements.txt | 5 ++ tests/integration/server.py | 117 +++++++++++++++++++++++++++++ 4 files changed, 147 insertions(+) create mode 100755 build.sh create mode 100644 tests/integration/Dockerfile.in create mode 100644 tests/integration/requirements.txt create mode 100644 tests/integration/server.py diff --git a/build.sh b/build.sh new file mode 100755 index 00000000..aedc0bb9 --- /dev/null +++ b/build.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -e + +export IMAGE_NAME=$1 + +if [ -z "$1" ]; then + echo "Usage: ./build.sh [image_path]" + echo "Please provide fully qualified path to target image." + exit 1 +fi + +envsubst < cloudbuild.yaml.in > cloudbuild.yaml +gcloud alpha container builds create . --config=cloudbuild.yaml diff --git a/tests/integration/Dockerfile.in b/tests/integration/Dockerfile.in new file mode 100644 index 00000000..be4f5870 --- /dev/null +++ b/tests/integration/Dockerfile.in @@ -0,0 +1,11 @@ +FROM ${STAGING_IMAGE} + +COPY . /app +WORKDIR /app + +RUN pip install -r requirements.txt + +ENV GOOGLE_APPLICATION_CREDENTIALS=/app/auth.json + +ENTRYPOINT ["python"] +CMD ["server.py"] diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt new file mode 100644 index 00000000..62de41cf --- /dev/null +++ b/tests/integration/requirements.txt @@ -0,0 +1,5 @@ +Flask +google-cloud +google-cloud-logging +google-cloud-monitoring +requests diff --git a/tests/integration/server.py b/tests/integration/server.py new file mode 100644 index 00000000..96180f4b --- /dev/null +++ b/tests/integration/server.py @@ -0,0 +1,117 @@ +#!/usr/bin/python + +from google.cloud import logging as gcloud_logging +from google.cloud import monitoring +from google.cloud.monitoring import MetricKind, ValueType +from oauth2client.client import GoogleCredentials +import logging +from flask import Flask, request, abort, jsonify +app = Flask(__name__) + + +@app.route('/') +def hello_world(): + return 'Hello World!' + + +@app.route('/logging', methods=['POST']) +def logging(): + request_data = request.get_json() + if request_data is None: + raise ErrorResponse("Unable to parse request JSON: did you set the Content-type header?") + log_name = request_data.get('log_name', '') + if log_name == '': + raise ErrorResponse("please provide log name") + token = request_data.get('token', '') + if token == '': + raise ErrorResponse("please provide token name") + + _log("log name is {0}, token is {1}".format(log_name, token)) + _log(token, log_name) + + return ('', 204) + + +def _log(token, log_name='stdout'): + # TODO (nkubala): write token to 'log_name' log, instead of stdout + # is this possible in non-standard (flex)??? + + # try: + # client = gcloud_logging.Client(credentials=GoogleCredentials.get_application_default()) + # gcloud_logger = client.logger(log_name) + # gcloud_logger.log_text(token) + # except Exception as e: + # logging.error("error while writing logs") + # raise ErrorResponse("error while writing logs: {0}".format(e)) + + # logging.info(token) + print token + + +@app.route('/monitoring', methods=['POST']) +def monitoring(): + request_data = request.get_json() + if request_data is None: + raise ErrorResponse("Unable to parse request JSON: did you set the Content-type header?") + name = request_data.get('name', '') + if name == '': + raise ErrorResponse("please provide name") + token = request_data.get('token', '') + if token == '': + raise ErrorResponse("please provide metric token") + + client = monitoring.Client(credentials=GoogleCredentials.get_application_default()) + + descriptor = client.metric_descriptor( + 'custom.googleapis.com/{0}'.format(name), + metric_kind=MetricKind.GAUGE, + value_type=ValueType.DOUBLE, + description=token + ) + descriptor.create() + + # resource = client.resource() + + metric = client.metric( + type='custom.googleapis.com/{0}'.format(name)) + + client.write_point(metric=metric, value=token) + + print 'OK' + + +@app.route('/exception', methods=['POST']) +def exception(): + print '' + + +@app.route('/trace', methods=['POST']) +def trace(): + print'' + + +class ErrorResponse(Exception): + status_code = 400 + + def __init__(self, message, status_code=None, payload=None): + Exception.__init__(self) + self.message = message + if status_code is not None: + self.status_code = status_code + self.payload = payload + + def to_dict(self): + rv = dict(self.payload or ()) + rv['message'] = self.message + return rv + + +@app.errorhandler(ErrorResponse) +def handle_invalid_usage(error): + response = jsonify(error.to_dict()) + response.status_code = error.status_code + return response + + +if __name__ == '__main__': + app.run(debug=True, host='0.0.0.0', port=8080) From bf03aa3e080a7ac37675c16fb41bde6469818e7e Mon Sep 17 00:00:00 2001 From: Nick Kubala Date: Wed, 7 Dec 2016 13:44:06 -0800 Subject: [PATCH 079/362] added support for writing custom metric to stackdriver monitoring --- tests/integration/server.py | 67 ++++++++++++++++++++++++------------- 1 file changed, 43 insertions(+), 24 deletions(-) diff --git a/tests/integration/server.py b/tests/integration/server.py index 96180f4b..2ee27635 100644 --- a/tests/integration/server.py +++ b/tests/integration/server.py @@ -1,8 +1,10 @@ #!/usr/bin/python from google.cloud import logging as gcloud_logging -from google.cloud import monitoring +from google.cloud import monitoring as gcloud_monitoring from google.cloud.monitoring import MetricKind, ValueType +from google.cloud.exceptions import Forbidden as ForbiddenException +from google.cloud.exceptions import NotFound as NotFoundException from oauth2client.client import GoogleCredentials import logging from flask import Flask, request, abort, jsonify @@ -15,7 +17,7 @@ def hello_world(): @app.route('/logging', methods=['POST']) -def logging(): +def _logging(): request_data = request.get_json() if request_data is None: raise ErrorResponse("Unable to parse request JSON: did you set the Content-type header?") @@ -32,24 +34,25 @@ def logging(): return ('', 204) +# TODO (nkubala): just as a note, currently the client logging API is broken def _log(token, log_name='stdout'): # TODO (nkubala): write token to 'log_name' log, instead of stdout # is this possible in non-standard (flex)??? - # try: - # client = gcloud_logging.Client(credentials=GoogleCredentials.get_application_default()) - # gcloud_logger = client.logger(log_name) - # gcloud_logger.log_text(token) - # except Exception as e: - # logging.error("error while writing logs") - # raise ErrorResponse("error while writing logs: {0}".format(e)) + try: + client = gcloud_logging.Client(credentials=GoogleCredentials.get_application_default()) + gcloud_logger = client.logger(log_name) + gcloud_logger.log_text(token) + except Exception as e: + logging.error("error while writing logs") + raise ErrorResponse("error while writing logs: {0}".format(e)) # logging.info(token) print token @app.route('/monitoring', methods=['POST']) -def monitoring(): +def _monitoring(): request_data = request.get_json() if request_data is None: raise ErrorResponse("Unable to parse request JSON: did you set the Content-type header?") @@ -60,33 +63,49 @@ def monitoring(): if token == '': raise ErrorResponse("please provide metric token") - client = monitoring.Client(credentials=GoogleCredentials.get_application_default()) + try: + client = gcloud_monitoring.Client(credentials=GoogleCredentials.get_application_default()) + + try: + descriptor = client.fetch_metric_descriptor(name) + if descriptor is None: + _create_descriptor(name, client) + except (ForbiddenException, NotFoundException) as ignored: + # print "forbidden" + _create_descriptor(name, client) + + metric = client.metric(name, {}) + resource = client.resource('global', labels={}) + client.write_point(metric, resource, token) + except Exception as e: + logging.error(e) + raise ErrorResponse("error while writing custom metric: {0}".format(e)) + # finally: + # if descriptor is not None: + # descriptor.delete() + print 'OK' + + +def _create_descriptor(name, client): + logging.info("no descriptor found with name {0}: creating".format(name)) descriptor = client.metric_descriptor( - 'custom.googleapis.com/{0}'.format(name), + name, metric_kind=MetricKind.GAUGE, value_type=ValueType.DOUBLE, - description=token + description="this is a test metric" ) descriptor.create() - - # resource = client.resource() - - metric = client.metric( - type='custom.googleapis.com/{0}'.format(name)) - - client.write_point(metric=metric, value=token) - - print 'OK' + time.sleep(5) @app.route('/exception', methods=['POST']) -def exception(): +def _exception(): print '' @app.route('/trace', methods=['POST']) -def trace(): +def _trace(): print'' From 9f1e7e36a07fe8e66551fd3035ac337adea1c3c5 Mon Sep 17 00:00:00 2001 From: Nick Kubala Date: Wed, 7 Dec 2016 15:45:21 -0800 Subject: [PATCH 080/362] fix broken response and import --- tests/integration/server.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/tests/integration/server.py b/tests/integration/server.py index 2ee27635..eb7b2f8d 100644 --- a/tests/integration/server.py +++ b/tests/integration/server.py @@ -6,8 +6,11 @@ from google.cloud.exceptions import Forbidden as ForbiddenException from google.cloud.exceptions import NotFound as NotFoundException from oauth2client.client import GoogleCredentials + import logging -from flask import Flask, request, abort, jsonify +import time + +from flask import Flask, request, jsonify app = Flask(__name__) @@ -31,7 +34,7 @@ def _logging(): _log("log name is {0}, token is {1}".format(log_name, token)) _log(token, log_name) - return ('', 204) + return ('OK', 200) # TODO (nkubala): just as a note, currently the client logging API is broken @@ -58,7 +61,7 @@ def _monitoring(): raise ErrorResponse("Unable to parse request JSON: did you set the Content-type header?") name = request_data.get('name', '') if name == '': - raise ErrorResponse("please provide name") + raise ErrorResponse("please provide metric name") token = request_data.get('token', '') if token == '': raise ErrorResponse("please provide metric token") @@ -84,7 +87,7 @@ def _monitoring(): # if descriptor is not None: # descriptor.delete() - print 'OK' + return ('OK', 200) def _create_descriptor(name, client): @@ -92,7 +95,7 @@ def _create_descriptor(name, client): descriptor = client.metric_descriptor( name, metric_kind=MetricKind.GAUGE, - value_type=ValueType.DOUBLE, + value_type=ValueType.INT64, description="this is a test metric" ) descriptor.create() @@ -101,12 +104,12 @@ def _create_descriptor(name, client): @app.route('/exception', methods=['POST']) def _exception(): - print '' + return ('', 204) @app.route('/trace', methods=['POST']) def _trace(): - print'' + return ('', 204) class ErrorResponse(Exception): From e14e435b3296bacdc8f7d62c67c95458b2ef8372 Mon Sep 17 00:00:00 2001 From: Nick Kubala Date: Wed, 7 Dec 2016 16:49:00 -0800 Subject: [PATCH 081/362] formatting and general cleanup --- tests/integration/Dockerfile.in | 14 +++ tests/integration/requirements.txt | 24 +++- tests/integration/server.py | 184 +++++++++++++++-------------- 3 files changed, 131 insertions(+), 91 deletions(-) diff --git a/tests/integration/Dockerfile.in b/tests/integration/Dockerfile.in index be4f5870..14501974 100644 --- a/tests/integration/Dockerfile.in +++ b/tests/integration/Dockerfile.in @@ -1,3 +1,17 @@ +# Copyright 2016 Google Inc. All rights reserved. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + FROM ${STAGING_IMAGE} COPY . /app diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index 62de41cf..0506252c 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -1,5 +1,19 @@ -Flask -google-cloud -google-cloud-logging -google-cloud-monitoring -requests +# Copyright 2016 Google Inc. All rights reserved. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Flask==0.11.1 +google-cloud==0.21.0 +google-cloud-logging==0.21.0 +google-cloud-monitoring==0.21.0 +requests==2.2.1 diff --git a/tests/integration/server.py b/tests/integration/server.py index eb7b2f8d..ae204577 100644 --- a/tests/integration/server.py +++ b/tests/integration/server.py @@ -1,139 +1,151 @@ #!/usr/bin/python +# Copyright 2016 Google Inc. All rights reserved. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import time + from google.cloud import logging as gcloud_logging from google.cloud import monitoring as gcloud_monitoring from google.cloud.monitoring import MetricKind, ValueType from google.cloud.exceptions import Forbidden as ForbiddenException from google.cloud.exceptions import NotFound as NotFoundException -from oauth2client.client import GoogleCredentials - -import logging -import time from flask import Flask, request, jsonify + app = Flask(__name__) @app.route('/') def hello_world(): - return 'Hello World!' + return 'Hello World!' @app.route('/logging', methods=['POST']) def _logging(): - request_data = request.get_json() - if request_data is None: - raise ErrorResponse("Unable to parse request JSON: did you set the Content-type header?") - log_name = request_data.get('log_name', '') - if log_name == '': - raise ErrorResponse("please provide log name") - token = request_data.get('token', '') - if token == '': - raise ErrorResponse("please provide token name") + request_data = request.get_json() + if request_data is None: + raise ErrorResponse('Unable to parse request JSON: ' + 'did you set the Content-type header?') + log_name = request_data.get('log_name', '') + if log_name == '': + raise ErrorResponse('Please provide log name') + token = request_data.get('token', '') + if token == '': + raise ErrorResponse('Please provide token name') - _log("log name is {0}, token is {1}".format(log_name, token)) - _log(token, log_name) + _log("log name is {0}, token is {1}".format(log_name, token)) + _log(token, log_name) - return ('OK', 200) + return 'OK', 200 # TODO (nkubala): just as a note, currently the client logging API is broken def _log(token, log_name='stdout'): - # TODO (nkubala): write token to 'log_name' log, instead of stdout - # is this possible in non-standard (flex)??? + # TODO (nkubala): write token to 'log_name' log, instead of stdout + # is this possible in non-standard (flex)??? - try: - client = gcloud_logging.Client(credentials=GoogleCredentials.get_application_default()) - gcloud_logger = client.logger(log_name) - gcloud_logger.log_text(token) - except Exception as e: - logging.error("error while writing logs") - raise ErrorResponse("error while writing logs: {0}".format(e)) + try: + client = gcloud_logging.Client() + gcloud_logger = client.logger(log_name) + gcloud_logger.log_text(token) + except Exception as e: + logging.error('Error while writing logs: {0}'.format(e)) + raise ErrorResponse('Error while writing logs: {0}'.format(e)) - # logging.info(token) - print token + # logging.info(token) + print token @app.route('/monitoring', methods=['POST']) def _monitoring(): - request_data = request.get_json() - if request_data is None: - raise ErrorResponse("Unable to parse request JSON: did you set the Content-type header?") - name = request_data.get('name', '') - if name == '': - raise ErrorResponse("please provide metric name") - token = request_data.get('token', '') - if token == '': - raise ErrorResponse("please provide metric token") - - try: - client = gcloud_monitoring.Client(credentials=GoogleCredentials.get_application_default()) - - try: - descriptor = client.fetch_metric_descriptor(name) - if descriptor is None: - _create_descriptor(name, client) - except (ForbiddenException, NotFoundException) as ignored: - # print "forbidden" - _create_descriptor(name, client) - - metric = client.metric(name, {}) - resource = client.resource('global', labels={}) - client.write_point(metric, resource, token) - except Exception as e: - logging.error(e) - raise ErrorResponse("error while writing custom metric: {0}".format(e)) - # finally: - # if descriptor is not None: - # descriptor.delete() - - return ('OK', 200) + request_data = request.get_json() + if request_data is None: + raise ErrorResponse('Unable to parse request JSON: ' + 'did you set the Content-type header?') + name = request_data.get('name', '') + if name == '': + raise ErrorResponse('Please provide metric name') + token = request_data.get('token', '') + if token == '': + raise ErrorResponse('Please provide metric token') + + try: + client = gcloud_monitoring.Client() + + try: + descriptor = client.fetch_metric_descriptor(name) + if descriptor is None: + _create_descriptor(name, client) + except (ForbiddenException, NotFoundException) as ignored: + _create_descriptor(name, client) + + metric = client.metric(name, {}) + resource = client.resource('global', labels={}) + client.write_point(metric, resource, token) + except Exception as e: + logging.error('Error while writing custom metric: {0}'.format(e)) + raise ErrorResponse('Error while writing custom metric: {0}'.format(e)) + + return 'OK', 200 def _create_descriptor(name, client): - logging.info("no descriptor found with name {0}: creating".format(name)) - descriptor = client.metric_descriptor( - name, - metric_kind=MetricKind.GAUGE, - value_type=ValueType.INT64, - description="this is a test metric" - ) - descriptor.create() - time.sleep(5) + logging.info('No descriptor found with name {0}: Creating...'.format(name)) + descriptor = client.metric_descriptor( + name, + metric_kind=MetricKind.GAUGE, + value_type=ValueType.INT64, + description='Test Metric' + ) + descriptor.create() + time.sleep(5) @app.route('/exception', methods=['POST']) def _exception(): - return ('', 204) + return ('', 204) @app.route('/trace', methods=['POST']) def _trace(): - return ('', 204) + return ('', 204) class ErrorResponse(Exception): - status_code = 400 + status_code = 400 - def __init__(self, message, status_code=None, payload=None): - Exception.__init__(self) - self.message = message - if status_code is not None: - self.status_code = status_code - self.payload = payload + def __init__(self, message, status_code=None, payload=None): + Exception.__init__(self) + self.message = message + if status_code is not None: + self.status_code = status_code + self.payload = payload - def to_dict(self): - rv = dict(self.payload or ()) - rv['message'] = self.message - return rv + def to_dict(self): + rv = dict(self.payload or ()) + rv['message'] = self.message + return rv @app.errorhandler(ErrorResponse) def handle_invalid_usage(error): - response = jsonify(error.to_dict()) - response.status_code = error.status_code - return response + response = jsonify(error.to_dict()) + response.status_code = error.status_code + return response if __name__ == '__main__': - app.run(debug=True, host='0.0.0.0', port=8080) + app.run(debug=True, host='0.0.0.0', port=8080) From 157b6e5e1688b2b45c15d4619ebcb2545956638f Mon Sep 17 00:00:00 2001 From: Nick Kubala Date: Wed, 7 Dec 2016 16:50:05 -0800 Subject: [PATCH 082/362] add license header to build script --- build.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/build.sh b/build.sh index aedc0bb9..e5d476eb 100755 --- a/build.sh +++ b/build.sh @@ -1,5 +1,19 @@ #!/bin/bash +# Copyright 2016 Google Inc. All rights reserved. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + set -e export IMAGE_NAME=$1 From ea319520da6cf862c48899624499fbc2bbc971c4 Mon Sep 17 00:00:00 2001 From: Nick Kubala Date: Wed, 7 Dec 2016 16:50:38 -0800 Subject: [PATCH 083/362] use beta container builds command --- build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sh b/build.sh index e5d476eb..92a009a7 100755 --- a/build.sh +++ b/build.sh @@ -25,4 +25,4 @@ if [ -z "$1" ]; then fi envsubst < cloudbuild.yaml.in > cloudbuild.yaml -gcloud alpha container builds create . --config=cloudbuild.yaml +gcloud beta container builds submit . --config=cloudbuild.yaml From a6b77d419323dec50a0c50bd42748f1fbd832821 Mon Sep 17 00:00:00 2001 From: Nick Kubala Date: Thu, 8 Dec 2016 11:25:24 -0800 Subject: [PATCH 084/362] addressing code review --- tests/integration/server.py | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/tests/integration/server.py b/tests/integration/server.py index ae204577..91886744 100644 --- a/tests/integration/server.py +++ b/tests/integration/server.py @@ -17,11 +17,9 @@ import logging import time -from google.cloud import logging as gcloud_logging -from google.cloud import monitoring as gcloud_monitoring -from google.cloud.monitoring import MetricKind, ValueType -from google.cloud.exceptions import Forbidden as ForbiddenException -from google.cloud.exceptions import NotFound as NotFoundException +import google.cloud.logging +import google.cloud.monitoring +import google.cloud.exceptions from flask import Flask, request, jsonify @@ -39,11 +37,11 @@ def _logging(): if request_data is None: raise ErrorResponse('Unable to parse request JSON: ' 'did you set the Content-type header?') - log_name = request_data.get('log_name', '') - if log_name == '': + log_name = request_data.get('log_name') + if not log_name: raise ErrorResponse('Please provide log name') - token = request_data.get('token', '') - if token == '': + token = request_data.get('token') + if not token: raise ErrorResponse('Please provide token name') _log("log name is {0}, token is {1}".format(log_name, token)) @@ -58,7 +56,7 @@ def _log(token, log_name='stdout'): # is this possible in non-standard (flex)??? try: - client = gcloud_logging.Client() + client = google.cloud.logging.Client() gcloud_logger = client.logger(log_name) gcloud_logger.log_text(token) except Exception as e: @@ -66,7 +64,7 @@ def _log(token, log_name='stdout'): raise ErrorResponse('Error while writing logs: {0}'.format(e)) # logging.info(token) - print token + print(token) @app.route('/monitoring', methods=['POST']) @@ -75,21 +73,22 @@ def _monitoring(): if request_data is None: raise ErrorResponse('Unable to parse request JSON: ' 'did you set the Content-type header?') - name = request_data.get('name', '') - if name == '': + name = request_data.get('name') + if not name: raise ErrorResponse('Please provide metric name') - token = request_data.get('token', '') - if token == '': + token = request_data.get('token') + if not token: raise ErrorResponse('Please provide metric token') try: - client = gcloud_monitoring.Client() + client = google.cloud.monitoring.Client() try: descriptor = client.fetch_metric_descriptor(name) if descriptor is None: _create_descriptor(name, client) - except (ForbiddenException, NotFoundException) as ignored: + except (google.cloud.exceptions.Forbidden, + google.cloud.exceptions.NotFound) as ignored: _create_descriptor(name, client) metric = client.metric(name, {}) @@ -106,8 +105,8 @@ def _create_descriptor(name, client): logging.info('No descriptor found with name {0}: Creating...'.format(name)) descriptor = client.metric_descriptor( name, - metric_kind=MetricKind.GAUGE, - value_type=ValueType.INT64, + metric_kind=google.cloud.monitoring.MetricKind.GAUGE, + value_type=google.cloud.monitoring.ValueType.INT64, description='Test Metric' ) descriptor.create() From 11406767a704b2e1e8fdc093121e8ff7c8994ceb Mon Sep 17 00:00:00 2001 From: Nick Kubala Date: Wed, 14 Dec 2016 13:43:09 -0800 Subject: [PATCH 085/362] serve with gunicorn. create write_metric helper. update requirements. --- tests/integration/Dockerfile.in | 3 +-- tests/integration/requirements.txt | 4 +++- tests/integration/server.py | 13 ++++++++++--- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/tests/integration/Dockerfile.in b/tests/integration/Dockerfile.in index 14501974..3aa54232 100644 --- a/tests/integration/Dockerfile.in +++ b/tests/integration/Dockerfile.in @@ -21,5 +21,4 @@ RUN pip install -r requirements.txt ENV GOOGLE_APPLICATION_CREDENTIALS=/app/auth.json -ENTRYPOINT ["python"] -CMD ["server.py"] +ENTRYPOINT ["gunicorn", "-b", ":8080", "server:app"] diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index 0506252c..369de14a 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -13,7 +13,9 @@ # limitations under the License. Flask==0.11.1 -google-cloud==0.21.0 google-cloud-logging==0.21.0 google-cloud-monitoring==0.21.0 +google-cloud-error-reporting==0.21.0 +gunicorn==19.6.0 +retrying==1.3.3 requests==2.2.1 diff --git a/tests/integration/server.py b/tests/integration/server.py index 91886744..3afdd084 100644 --- a/tests/integration/server.py +++ b/tests/integration/server.py @@ -16,6 +16,7 @@ import logging import time +from retrying import retry import google.cloud.logging import google.cloud.monitoring @@ -91,9 +92,8 @@ def _monitoring(): google.cloud.exceptions.NotFound) as ignored: _create_descriptor(name, client) - metric = client.metric(name, {}) - resource = client.resource('global', labels={}) - client.write_point(metric, resource, token) + _write_metric(name, client) + except Exception as e: logging.error('Error while writing custom metric: {0}'.format(e)) raise ErrorResponse('Error while writing custom metric: {0}'.format(e)) @@ -101,6 +101,13 @@ def _monitoring(): return 'OK', 200 +@retry(wait_exponential_multiplier=1000, stop_max_attempt_number=10) +def _write_metric(name, client): + metric = client.metric(name, {}) + resource = client.resource('global', labels={}) + client.write_point(metric, resource, token) + + def _create_descriptor(name, client): logging.info('No descriptor found with name {0}: Creating...'.format(name)) descriptor = client.metric_descriptor( From 880cf8bf0be6796aa9e656477d0e5f3f3cec310d Mon Sep 17 00:00:00 2001 From: Nick Kubala Date: Thu, 15 Dec 2016 15:22:49 -0800 Subject: [PATCH 086/362] fix metric writing. remove time.sleep --- tests/integration/server.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tests/integration/server.py b/tests/integration/server.py index 3afdd084..3ea7c366 100644 --- a/tests/integration/server.py +++ b/tests/integration/server.py @@ -15,7 +15,6 @@ # limitations under the License. import logging -import time from retrying import retry import google.cloud.logging @@ -92,7 +91,7 @@ def _monitoring(): google.cloud.exceptions.NotFound) as ignored: _create_descriptor(name, client) - _write_metric(name, client) + _write_metric(name, client, token) except Exception as e: logging.error('Error while writing custom metric: {0}'.format(e)) @@ -101,8 +100,7 @@ def _monitoring(): return 'OK', 200 -@retry(wait_exponential_multiplier=1000, stop_max_attempt_number=10) -def _write_metric(name, client): +def _write_metric(name, client, token): metric = client.metric(name, {}) resource = client.resource('global', labels={}) client.write_point(metric, resource, token) @@ -117,7 +115,6 @@ def _create_descriptor(name, client): description='Test Metric' ) descriptor.create() - time.sleep(5) @app.route('/exception', methods=['POST']) From 459f7d227607382eab003d635c77af3748369118 Mon Sep 17 00:00:00 2001 From: Nick Kubala Date: Wed, 28 Dec 2016 14:25:54 -0600 Subject: [PATCH 087/362] implement error reporting --- tests/integration/server.py | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/tests/integration/server.py b/tests/integration/server.py index 3ea7c366..053c5028 100644 --- a/tests/integration/server.py +++ b/tests/integration/server.py @@ -19,6 +19,7 @@ import google.cloud.logging import google.cloud.monitoring +import google.cloud.error_reporting import google.cloud.exceptions from flask import Flask, request, jsonify @@ -119,12 +120,32 @@ def _create_descriptor(name, client): @app.route('/exception', methods=['POST']) def _exception(): - return ('', 204) + request_data = request.get_json() + if request_data is None: + raise ErrorResponse('Unable to parse request JSON: ' + 'did you set the Content-type header?') + token = request_data.get('token') + if not token: + raise ErrorResponse('Please provide token') + + try: + client = google.cloud.error_reporting.Client() + try: + raise NameError + except Exception: + client.report_exception() + + client.report(token) + except Exception as e: + logging.error('Error while reporting exception: {0}'.format(e)) + raise ErrorResponse('Error while reporting exception: {0}'.format(e)) + + return 'OK', 200 @app.route('/trace', methods=['POST']) def _trace(): - return ('', 204) + return 'OK', 204 class ErrorResponse(Exception): From 771f1fe5a4b08c3189fdcf9640ba19a2719cd0ae Mon Sep 17 00:00:00 2001 From: Nick Kubala Date: Mon, 9 Jan 2017 15:50:35 -0800 Subject: [PATCH 088/362] add docstrings for helper methods. minor build script changes. fix flake --- build.sh | 4 ++-- tests/integration/server.py | 42 ++++++++++++++++++++++++++++++++----- 2 files changed, 39 insertions(+), 7 deletions(-) diff --git a/build.sh b/build.sh index 92a009a7..2eae6ce3 100755 --- a/build.sh +++ b/build.sh @@ -14,11 +14,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -set -e +set -euo pipefail export IMAGE_NAME=$1 -if [ -z "$1" ]; then +if [ -z "$IMAGE_NAME" ]; then echo "Usage: ./build.sh [image_path]" echo "Please provide fully qualified path to target image." exit 1 diff --git a/tests/integration/server.py b/tests/integration/server.py index 053c5028..f370ba4f 100644 --- a/tests/integration/server.py +++ b/tests/integration/server.py @@ -15,7 +15,6 @@ # limitations under the License. import logging -from retrying import retry import google.cloud.logging import google.cloud.monitoring @@ -51,8 +50,23 @@ def _logging(): return 'OK', 200 -# TODO (nkubala): just as a note, currently the client logging API is broken def _log(token, log_name='stdout'): + """ + Write a log entry to Stackdriver. + + Keyword arguments: + token -- 16-character (8-byte) hexadecimal token, to be written + as a log entry. + log_name -- The name of the logging group to be written to. + + Once the entry is written to Stackdriver, the test driver will retrieve + all entries with the name 'log_name', and verify there is an entry with + the same value as 'token', indicating the entry was written successfully. + """ + + # TODO (nkubala): just as a note, currently the client logging API is + # broken + # TODO (nkubala): write token to 'log_name' log, instead of stdout # is this possible in non-standard (flex)??? @@ -64,7 +78,7 @@ def _log(token, log_name='stdout'): logging.error('Error while writing logs: {0}'.format(e)) raise ErrorResponse('Error while writing logs: {0}'.format(e)) - # logging.info(token) + logging.debug(token) print(token) @@ -88,8 +102,8 @@ def _monitoring(): descriptor = client.fetch_metric_descriptor(name) if descriptor is None: _create_descriptor(name, client) - except (google.cloud.exceptions.Forbidden, - google.cloud.exceptions.NotFound) as ignored: + except (google.cloud.exceptions.Forbidden, + google.cloud.exceptions.NotFound) as ignored: # noqa: F841 _create_descriptor(name, client) _write_metric(name, client, token) @@ -102,12 +116,30 @@ def _monitoring(): def _write_metric(name, client, token): + """ + Write a metric to Stackdriver Monitoring. + + Keyword arguments: + name -- The name of the metric to write. Takes the form + 'custom.googleapis.com/{metric_name}' + client -- the authenticated instance of a Google Cloud Client + token -- a random 64-bit integer token. The metric value to be written. + + Once the metric is written, the test driver will retrieve all metrics + written with the provided name, and verify there is an entry with the + same value as the provided token. + """ metric = client.metric(name, {}) resource = client.resource('global', labels={}) client.write_point(metric, resource, token) def _create_descriptor(name, client): + """ + Create a new metric descriptor. + This descriptor is implicitly used to write a point-value metric to + Stackdriver. + """ logging.info('No descriptor found with name {0}: Creating...'.format(name)) descriptor = client.metric_descriptor( name, From a28cc06627174f1ed3372a66158ced5c5b650a16 Mon Sep 17 00:00:00 2001 From: Nick Kubala Date: Wed, 11 Jan 2017 16:49:47 -0800 Subject: [PATCH 089/362] remove manual auth. remove unnecessary log statement --- tests/integration/Dockerfile.in | 2 -- tests/integration/server.py | 1 - 2 files changed, 3 deletions(-) diff --git a/tests/integration/Dockerfile.in b/tests/integration/Dockerfile.in index 3aa54232..e18d60ef 100644 --- a/tests/integration/Dockerfile.in +++ b/tests/integration/Dockerfile.in @@ -19,6 +19,4 @@ WORKDIR /app RUN pip install -r requirements.txt -ENV GOOGLE_APPLICATION_CREDENTIALS=/app/auth.json - ENTRYPOINT ["gunicorn", "-b", ":8080", "server:app"] diff --git a/tests/integration/server.py b/tests/integration/server.py index f370ba4f..448ea597 100644 --- a/tests/integration/server.py +++ b/tests/integration/server.py @@ -44,7 +44,6 @@ def _logging(): if not token: raise ErrorResponse('Please provide token name') - _log("log name is {0}, token is {1}".format(log_name, token)) _log(token, log_name) return 'OK', 200 From 0abb857c43f1d0475d6e423e48764d7ef45bd6a4 Mon Sep 17 00:00:00 2001 From: Nick Kubala Date: Thu, 12 Jan 2017 12:50:59 -0800 Subject: [PATCH 090/362] move request verification to decorator. catch GoogleCloudError in test functions. remove 0.0.0.0 host in app.run --- tests/integration/server.py | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) mode change 100644 => 100755 tests/integration/server.py diff --git a/tests/integration/server.py b/tests/integration/server.py old mode 100644 new mode 100755 index 448ea597..e2ff8375 --- a/tests/integration/server.py +++ b/tests/integration/server.py @@ -14,6 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from functools import wraps import logging import google.cloud.logging @@ -26,17 +27,26 @@ app = Flask(__name__) +def verify_request(f): + @wraps(f) + def verified_func(*args, **kwargs): + request_data = request.get_json() + if request_data is None: + raise ErrorResponse('Unable to parse request JSON: ' + 'did you set the Content-type header?') + return f(*args, **kwargs) + return verified_func + + @app.route('/') def hello_world(): return 'Hello World!' @app.route('/logging', methods=['POST']) +@verify_request def _logging(): request_data = request.get_json() - if request_data is None: - raise ErrorResponse('Unable to parse request JSON: ' - 'did you set the Content-type header?') log_name = request_data.get('log_name') if not log_name: raise ErrorResponse('Please provide log name') @@ -73,7 +83,7 @@ def _log(token, log_name='stdout'): client = google.cloud.logging.Client() gcloud_logger = client.logger(log_name) gcloud_logger.log_text(token) - except Exception as e: + except google.cloud.exceptions.GoogleCloudError as e: logging.error('Error while writing logs: {0}'.format(e)) raise ErrorResponse('Error while writing logs: {0}'.format(e)) @@ -82,11 +92,9 @@ def _log(token, log_name='stdout'): @app.route('/monitoring', methods=['POST']) +@verify_request def _monitoring(): request_data = request.get_json() - if request_data is None: - raise ErrorResponse('Unable to parse request JSON: ' - 'did you set the Content-type header?') name = request_data.get('name') if not name: raise ErrorResponse('Please provide metric name') @@ -107,7 +115,7 @@ def _monitoring(): _write_metric(name, client, token) - except Exception as e: + except google.cloud.exceptions.GoogleCloudError as e: logging.error('Error while writing custom metric: {0}'.format(e)) raise ErrorResponse('Error while writing custom metric: {0}'.format(e)) @@ -144,17 +152,14 @@ def _create_descriptor(name, client): name, metric_kind=google.cloud.monitoring.MetricKind.GAUGE, value_type=google.cloud.monitoring.ValueType.INT64, - description='Test Metric' - ) + description='Test Metric') descriptor.create() @app.route('/exception', methods=['POST']) +@verify_request def _exception(): request_data = request.get_json() - if request_data is None: - raise ErrorResponse('Unable to parse request JSON: ' - 'did you set the Content-type header?') token = request_data.get('token') if not token: raise ErrorResponse('Please provide token') @@ -167,7 +172,7 @@ def _exception(): client.report_exception() client.report(token) - except Exception as e: + except google.cloud.exceptions.GoogleCloudError as e: logging.error('Error while reporting exception: {0}'.format(e)) raise ErrorResponse('Error while reporting exception: {0}'.format(e)) @@ -203,4 +208,4 @@ def handle_invalid_usage(error): if __name__ == '__main__': - app.run(debug=True, host='0.0.0.0', port=8080) + app.run(debug=True, port=8080) From df371b0def73e4f86854fc69cbf08b3f3e922f6e Mon Sep 17 00:00:00 2001 From: Nick Kubala Date: Thu, 12 Jan 2017 13:40:15 -0800 Subject: [PATCH 091/362] move token check into decorator. pass request_data back to wrapped function --- tests/integration/server.py | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/tests/integration/server.py b/tests/integration/server.py index e2ff8375..528ebefa 100755 --- a/tests/integration/server.py +++ b/tests/integration/server.py @@ -34,7 +34,10 @@ def verified_func(*args, **kwargs): if request_data is None: raise ErrorResponse('Unable to parse request JSON: ' 'did you set the Content-type header?') - return f(*args, **kwargs) + token = request_data.get('token') + if not token: + raise ErrorResponse('Please provide token name') + return f(*args, request_data=request_data, token=token, **kwargs) return verified_func @@ -45,14 +48,10 @@ def hello_world(): @app.route('/logging', methods=['POST']) @verify_request -def _logging(): - request_data = request.get_json() +def _logging(request_data, token): log_name = request_data.get('log_name') if not log_name: raise ErrorResponse('Please provide log name') - token = request_data.get('token') - if not token: - raise ErrorResponse('Please provide token name') _log(token, log_name) @@ -93,14 +92,10 @@ def _log(token, log_name='stdout'): @app.route('/monitoring', methods=['POST']) @verify_request -def _monitoring(): - request_data = request.get_json() +def _monitoring(request_data, token): name = request_data.get('name') if not name: raise ErrorResponse('Please provide metric name') - token = request_data.get('token') - if not token: - raise ErrorResponse('Please provide metric token') try: client = google.cloud.monitoring.Client() @@ -158,12 +153,7 @@ def _create_descriptor(name, client): @app.route('/exception', methods=['POST']) @verify_request -def _exception(): - request_data = request.get_json() - token = request_data.get('token') - if not token: - raise ErrorResponse('Please provide token') - +def _exception(request_data, token): try: client = google.cloud.error_reporting.Client() try: From 09e796ca3d13831010a211a0a38ec68694bf8562 Mon Sep 17 00:00:00 2001 From: Nick Kubala Date: Thu, 12 Jan 2017 13:43:53 -0800 Subject: [PATCH 092/362] add app.yaml to directory --- tests/integration/app.yaml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 tests/integration/app.yaml diff --git a/tests/integration/app.yaml b/tests/integration/app.yaml new file mode 100644 index 00000000..ce2a1243 --- /dev/null +++ b/tests/integration/app.yaml @@ -0,0 +1,2 @@ +runtime: custom +env: flex From bbd99153031bb25ccabee755ef50659baef4dbce Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Tue, 24 Jan 2017 12:57:26 -0800 Subject: [PATCH 093/362] Remove $UPLOAD_TO_STAGING, this is now handled separately. --- jenkins_build.sh | 7 ------- 1 file changed, 7 deletions(-) diff --git a/jenkins_build.sh b/jenkins_build.sh index 345cb735..16179231 100755 --- a/jenkins_build.sh +++ b/jenkins_build.sh @@ -12,10 +12,3 @@ IMAGE_NAME="${DOCKER_NAMESPACE}/${RUNTIME_NAME}:${CANDIDATE_NAME}" export IMAGE_NAME export FORCE_REBUILD make build - -if [ "${UPLOAD_TO_STAGING}" = "true" ]; then - STAGING="${DOCKER_NAMESPACE}/${RUNTIME_NAME}:staging" - docker rmi "${STAGING}" 2>/dev/null || true # Ignore if tag not present - docker tag "${IMAGE_NAME}" "${STAGING}" - gcloud docker push "${STAGING}" -fi From c9830b63e84f5f9550837051cd8918502040f8eb Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Tue, 24 Jan 2017 14:02:19 -0800 Subject: [PATCH 094/362] Remove redundant $PATH manipulation --- tests/google-cloud-python/Dockerfile.in | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/google-cloud-python/Dockerfile.in b/tests/google-cloud-python/Dockerfile.in index a952f865..94bf4648 100644 --- a/tests/google-cloud-python/Dockerfile.in +++ b/tests/google-cloud-python/Dockerfile.in @@ -14,5 +14,4 @@ RUN python2.7 scripts/run_unit_tests.py RUN python3.4 scripts/run_unit_tests.py # Run Python 3.5 unit tests -ENV PATH /opt/python3.5/bin:$PATH RUN python3.5 scripts/run_unit_tests.py From 8396fe4c4f5c8afcbc76b7d2f85468c466a87e1d Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Tue, 24 Jan 2017 15:01:55 -0800 Subject: [PATCH 095/362] Reorganize Makefile. All targets using the Google Container Builder are prefixed with 'cloud-' and all targets that use the local Docker daemon are prefixed with 'local-'. Fixed the 'google-cloud-python' unit test target, and incorporated it into the Container Builder workflow. --- Makefile | 79 +++++++++++++++++++++++++++------------------- README.md | 61 +++++++++++++++++++++++++++++++---- cloudbuild.yaml.in | 2 ++ jenkins_build.sh | 6 +++- 4 files changed, 107 insertions(+), 41 deletions(-) diff --git a/Makefile b/Makefile index 84c4651d..aaf03888 100644 --- a/Makefile +++ b/Makefile @@ -6,53 +6,66 @@ ifndef IMAGE_NAME $(error IMAGE_NAME is not set; invoke make with something like IMAGE_NAME=google/python:2017-01-02_03_45) endif - -.PHONY: local-image -local-image: build-interpreters - docker build $(DOCKER_FLAGS) -t "$(IMAGE_NAME)" . - -docker rmi "google/python" 2>/dev/null - docker tag "$(IMAGE_NAME)" "google/python" - -.PHONY: build-interpreters -build-interpreters: - export DOCKER_FLAGS - make -C python-interpreter-builder build +## Files that must be refreshed every build .PHONY: cloudbuild.yaml # Force reevaluation of env vars every time cloudbuild.yaml: cloudbuild.yaml.in - envsubst < cloudbuild.yaml.in > cloudbuild.yaml - -.PHONY: cloudbuild -cloudbuild: cloudbuild.yaml - gcloud alpha container builds create . --config=cloudbuild.yaml + envsubst < $< > $@ -.PHONY: build -# no structure tests since they are implicit in cloudbuild -build: cloudbuild integration-tests -.PHONY: build-local -build-local: local-image structure-tests integration-tests +.PHONY: tests/google-cloud-python/Dockerfile # Force reevaluation of env vars every time +tests/google-cloud-python/Dockerfile: tests/google-cloud-python/Dockerfile.in + envsubst < $< > $@ .PHONY: ext_run.sh # Force refetch every time ext_run.sh: curl https://raw.githubusercontent.com/GoogleCloudPlatform/runtimes-common/master/structure_tests/ext_run.sh > ext_run.sh chmod +x ext_run.sh -.PHONY: structure-tests -structure-tests: local-image ext_run.sh - make -C tests structure-tests +## Build using Google Container Builder service -.PHONY: benchmarks -benchmarks: - make -C tests benchmarks +.PHONY: cloud-build +cloud-build: cloudbuild.yaml tests/google-cloud-python/Dockerfile + gcloud alpha container builds create . --config=cloudbuild.yaml -.PHONY: google-cloud-python -google-cloud-python: +.PHONY: cloud-test +# structure-tests and google-cloud-python-tests are implicit in cloud-build +cloud-test: cloud-build integration-tests + +## Build using local Docker daemon + +.PHONY: local-build +local-build: local-build-interpreters + docker build $(DOCKER_FLAGS) -t "$(IMAGE_NAME)" . + +.PHONY: local-build-interpreters +local-build-interpreters: + export DOCKER_FLAGS + make -C python-interpreter-builder build + +.PHONY: local-test +local-test: local-build local-structure-tests local-google-cloud-python-tests integration-tests + +.PHONY: local-structure-tests +local-structure-tests: local-build ext_run.sh + make -C tests structure-tests + +# Unit tests for Google Cloud Client Library for Python +.PHONY: local-google-cloud-python-tests +local-google-cloud-python-tests: tests/google-cloud-python/Dockerfile make -C tests google-cloud-python -.PHONY: google-cloud-system-tests -google-cloud-system-tests: - make -C system_tests +## Always local .PHONY: integration-tests -tests: benchmarks google-cloud-system-tests google-cloud-python +integration-tests: google-cloud-python-system-tests benchmarks + +# System tests for Google Cloud Client Library for Python. +# They require gcloud auth and network access. +.PHONY: google-cloud-python-system-tests +google-cloud-python-system-tests: + make -C system_tests + +.PHONY: benchmarks +benchmarks: + make -C tests benchmarks diff --git a/README.md b/README.md index d6147015..ab7fc243 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,22 @@ # Google Cloud Platform - Python Runtime Docker Image -This repository contains the source for the `gcr.io/google_appengine/python` [docker](https://docker.io) base image. This image can be used as the base image for running applications on [Google App Engine Flexible](https://cloud.google.com/appengine/docs/flexible/), [Google Container Engine](https://cloud.google.com/container-engine), or any other Docker host. +This repository contains the source for the +[`gcr.io/google-appengine/python`](https://gcr.io/google-appengine/python) +[docker](https://docker.io) base image. This image can be used as the base image +for running applications on +[Google App Engine Flexible](https://cloud.google.com/appengine/docs/flexible/), +[Google Container Engine](https://cloud.google.com/container-engine), or any +other Docker host. -This image is based on Debian Jessie and contains packages required to build most of the popular Python libraries. For more information about this runtime, see the [documentation](https://cloud.google.com/appengine/docs/flexible/python/runtime). +This image is based on Debian Jessie and contains packages required to build +most of the popular Python libraries. For more information about this runtime, +see the +[documentation](https://cloud.google.com/appengine/docs/flexible/python/runtime). ## App Engine -When using App Engine Flexible, you can use the runtime without worrying about docker by specifying `runtime: python` in your `app.yaml`: +When using App Engine Flexible, you can use the runtime without worrying about +docker by specifying `runtime: python` in your `app.yaml`: ```yaml runtime: python @@ -18,17 +28,23 @@ runtime_config: python_version: 3 ``` -If you have an existing App Engine application using this runtime and want to customize it, you can use the [`Cloud SDK`](https://cloud.google.com/sdk/gcloud/reference/preview/app/gen-config) to create a custom runtime: +If you have an existing App Engine application using this runtime and want to +customize it, you can use the +[`Cloud SDK`](https://cloud.google.com/sdk/gcloud/reference/preview/app/gen-config) +to create a custom runtime: gcloud beta app gen-config --custom -You can then modify the `Dockerfile` and `.dockerignore` as needed for you application. +You can then modify the `Dockerfile` and `.dockerignore` as needed for you +application. ## Container Engine & other Docker hosts. -For other docker hosts, you'll need to create a `Dockerfile` based on this image that copies your application code, installs dependencies, and declares an command or entrypoint. For example: +For other docker hosts, you'll need to create a `Dockerfile` based on this image +that copies your application code, installs dependencies, and declares an +command or entrypoint. For example: - FROM gcr.io/google_appengine/python + FROM gcr.io/google-appengine/python # Create a virtualenv for dependencies. This isolates these packages from # system-level packages. @@ -51,6 +67,37 @@ For other docker hosts, you'll need to create a `Dockerfile` based on this image # a dependency in requirements.txt. CMD gunicorn -b :$PORT main:app +## Building the image + +Google regularly builds and releases this image at +[`gcr.io/google-appengine/python`](https://gcr.io/google-appengine/python). + +To rebuild the image yourself, first set the following variables in your +shell. You need to be authenticated to a Google Cloud Project to invoke the +Google Container Builder service, and also to run the system tests. + +```shell +$ export GCLOUD_PROJECT=YOUR-PROJECT-NAME +$ DOCKER_NAMESPACE=gcr.io/${GCLOUD_PROJECT} +$ CANDIDATE_NAME=`date +%Y-%m-%d_%H_%M` +$ export IMAGE_NAME=${DOCKER_NAMESPACE}/python:${CANDIDATE_NAME} +``` + +To rebuild the image using the Google Container Builder service, do the +following: + +```shell +$ make cloud-build +$ make cloud-test +``` + +To rebuild the image using your local Docker daemon, do the following: + +``` shell +$ make local-build +$ make local-test +``` + ## Contributing changes * See [CONTRIBUTING.md](CONTRIBUTING.md) diff --git a/cloudbuild.yaml.in b/cloudbuild.yaml.in index 79ee93ba..776bc6a2 100644 --- a/cloudbuild.yaml.in +++ b/cloudbuild.yaml.in @@ -18,5 +18,7 @@ steps: '--config', '/workspace/tests/license-test/license-test.yaml', '-v' ] +- name: gcr.io/cloud-builders/docker + args: ['build', '--tag=google-cloud-python-tests', '--no-cache', '/workspace/tests/google-cloud-python/'] images: ['${IMAGE_NAME}'] diff --git a/jenkins_build.sh b/jenkins_build.sh index 16179231..b081255f 100755 --- a/jenkins_build.sh +++ b/jenkins_build.sh @@ -7,8 +7,12 @@ RUNTIME_NAME="python" CANDIDATE_NAME=`date +%Y-%m-%d_%H_%M` echo "CANDIDATE_NAME:${CANDIDATE_NAME}" +if [ -z "${DOCKER_NAMESPACE+set}" ] ; then + echo "Error: DOCKER_NAMESPACE is not set; invoke with something like DOCKER_NAMESPACE=gcr.io/YOUR-PROJECT-NAME" >&2 + exit 1 +fi IMAGE_NAME="${DOCKER_NAMESPACE}/${RUNTIME_NAME}:${CANDIDATE_NAME}" export IMAGE_NAME export FORCE_REBUILD -make build +make cloud-test From d87aff9fe8597d281917013707664b8fddff2d5d Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Tue, 24 Jan 2017 16:29:04 -0800 Subject: [PATCH 096/362] Fix Makefile for Google Cloud Client Library for Python system tests. Add rule to build Dockerfile, and change variable name to be consistent with top-level Makefile. --- system_tests/Makefile | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/system_tests/Makefile b/system_tests/Makefile index f3c554f3..7fd52f9a 100644 --- a/system_tests/Makefile +++ b/system_tests/Makefile @@ -1,8 +1,11 @@ # Use no-cache to prevent layer caching because there is a layer that does # a `git clone` which can not be cached. -CACHE ?= --no-cache +DOCKER_FLAGS ?= --no-cache + +.PHONY: Dockerfile +Dockerfile: Dockerfile.in + envsubst < $< > $@ .PHONY: all all: - envsubst < Dockerfile.in > Dockerfile - docker build $(CACHE) . + docker build $(DOCKER_FLAGS) . From 0339ba9ff82b7ef1ce9128a728214f48abb8ce66 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Tue, 24 Jan 2017 17:40:49 -0800 Subject: [PATCH 097/362] Give each Makefile a reasonable default target. --- Makefile | 4 ++++ system_tests/Makefile | 8 ++++---- tests/Makefile | 3 +++ 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index aaf03888..ca03ce30 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,10 @@ ifndef IMAGE_NAME $(error IMAGE_NAME is not set; invoke make with something like IMAGE_NAME=google/python:2017-01-02_03_45) endif +.PHONY: all +all: + cloud-test + ## Files that must be refreshed every build .PHONY: cloudbuild.yaml # Force reevaluation of env vars every time diff --git a/system_tests/Makefile b/system_tests/Makefile index 7fd52f9a..ecfba4ac 100644 --- a/system_tests/Makefile +++ b/system_tests/Makefile @@ -2,10 +2,10 @@ # a `git clone` which can not be cached. DOCKER_FLAGS ?= --no-cache +.PHONY: all +all: Dockerfile + docker build $(DOCKER_FLAGS) . + .PHONY: Dockerfile Dockerfile: Dockerfile.in envsubst < $< > $@ - -.PHONY: all -all: - docker build $(DOCKER_FLAGS) . diff --git a/tests/Makefile b/tests/Makefile index 5953a62d..0535885b 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -1,3 +1,6 @@ +.PHONY: all +all: structure-tests benchmarks google-cloud-python + .PHONY: structure-tests structure-tests: make -C no-virtualenv From cbc359cd4fad03ef54e4ef18d5d4f734210ef12a Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Mon, 30 Jan 2017 18:32:55 -0800 Subject: [PATCH 098/362] Fix typo in Makefile --- Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index ca03ce30..252adbba 100644 --- a/Makefile +++ b/Makefile @@ -7,8 +7,7 @@ $(error IMAGE_NAME is not set; invoke make with something like IMAGE_NAME=google endif .PHONY: all -all: - cloud-test +all: cloud-test ## Files that must be refreshed every build From 188d7bcc5202392556f6571618b6ba92b2405fe1 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Mon, 30 Jan 2017 18:35:24 -0800 Subject: [PATCH 099/362] Improve system tests 1) Allow the project name and service account to be configured 2) Don't store credentials inside Docker images --- system_tests/Dockerfile.in | 19 ++++++------------- system_tests/Makefile | 13 ++++++++++++- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/system_tests/Dockerfile.in b/system_tests/Dockerfile.in index 6cbf6db3..97f8e004 100644 --- a/system_tests/Dockerfile.in +++ b/system_tests/Dockerfile.in @@ -1,22 +1,15 @@ FROM ${IMAGE_NAME} -# Inject secrets -ADD data/ data/ -ENV GOOGLE_APPLICATION_CREDENTIALS /app/data/cloud-python-runtime-qa-credentials.json -ENV GCLOUD_PROJECT cloud-python-runtime-qa +# Secrets injected at runtime +ENV GOOGLE_APPLICATION_CREDENTIALS=/app/credentials/credentials.json +ENV GOOGLE_CLOUD_PROJECT=${GOOGLE_CLOUD_PROJECT} # Get the source. -RUN git clone https://github.com/GoogleCloudPlatform/google-cloud-python.git +RUN git clone --depth 1 https://github.com/GoogleCloudPlatform/google-cloud-python.git WORKDIR google-cloud-python # Install tox for running the system tests RUN pip install --upgrade tox -# List of system tests to run. Ideally would be all of them. But -# some of them get permission or other errors that are probably -# related to incomplete test setup. -ENV MODULES="datastore storage speech bigquery pubsub language translate monitoring bigtable" - -# Run Python 2.7, 3.4 system tests -RUN GOOGLE_CLOUD_TESTS_API_KEY=$(cat /app/data/cloud-python-runtime-qa-api-key.txt) \ - tox -e system-tests,system-tests3 -- ${MODULES} +# Run Python 2.7, 3.5 system tests +ENTRYPOINT ["tox", "-e", "system-tests,system-tests3"] diff --git a/system_tests/Makefile b/system_tests/Makefile index ecfba4ac..cec219ca 100644 --- a/system_tests/Makefile +++ b/system_tests/Makefile @@ -2,9 +2,20 @@ # a `git clone` which can not be cached. DOCKER_FLAGS ?= --no-cache +ifndef GOOGLE_APPLICATION_CREDENTIALS +$(error GOOGLE_APPLICATION_CREDENTIALS is not set; download service account credentials in JSON format from the Google Cloud Console and invoke make with something like GOOGLE_APPLICATION_CREDENTIALS=/path/to/credentials.json) +endif + +ifndef GOOGLE_CLOUD_PROJECT +$(error GOOGLE_CLOUD_PROJECT is not set; invoke make with something like GOOGLE_CLOUD_PROJECT=my-project-name) +endif + + .PHONY: all all: Dockerfile - docker build $(DOCKER_FLAGS) . + @echo "Running system tests in project ${GOOGLE_CLOUD_PROJECT} using service account credentials from ${GOOGLE_APPLICATION_CREDENTIALS}" + docker build --tag google-cloud-python-system-tests $(DOCKER_FLAGS) . + docker run --rm -v $(GOOGLE_APPLICATION_CREDENTIALS):/app/credentials/credentials.json google-cloud-python-system-tests .PHONY: Dockerfile Dockerfile: Dockerfile.in From 237ffbfa79580313a396ed4f79a3b950c7a8ba1c Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Tue, 31 Jan 2017 14:14:04 -0800 Subject: [PATCH 100/362] Increase timeout for cloudbuild path to 2 hours. --- cloudbuild.yaml.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudbuild.yaml.in b/cloudbuild.yaml.in index 776bc6a2..b8ed6baa 100644 --- a/cloudbuild.yaml.in +++ b/cloudbuild.yaml.in @@ -1,4 +1,4 @@ -timeout: 3600s +timeout: 7200s steps: - name: gcr.io/cloud-builders/docker args: ['build', '--tag=interpreter', '--no-cache', 'python-interpreter-builder'] From a6d2a7d9f35330b42ceefb5059e53ca78110aa5f Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Mon, 30 Jan 2017 18:36:51 -0800 Subject: [PATCH 101/362] Add documentation for running system tests --- README.md | 44 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ab7fc243..4a35d047 100644 --- a/README.md +++ b/README.md @@ -77,10 +77,11 @@ shell. You need to be authenticated to a Google Cloud Project to invoke the Google Container Builder service, and also to run the system tests. ```shell -$ export GCLOUD_PROJECT=YOUR-PROJECT-NAME +$ export GOOGLE_CLOUD_PROJECT=YOUR-PROJECT-NAME $ DOCKER_NAMESPACE=gcr.io/${GCLOUD_PROJECT} $ CANDIDATE_NAME=`date +%Y-%m-%d_%H_%M` $ export IMAGE_NAME=${DOCKER_NAMESPACE}/python:${CANDIDATE_NAME} +$ gcloud config set project ${GOOGLE_CLOUD_PROJECT} ``` To rebuild the image using the Google Container Builder service, do the @@ -98,6 +99,47 @@ $ make local-build $ make local-test ``` +To open an interactive shell session to this image after building it, do the following: + +``` shell +docker run --it --entrypoint /bin/bash ${IMAGE_NAME} +``` + +## Running the system tests + +To run the system tests, you need a Google Cloud Project with a service account. +From the [Google Cloud Console](https://console.cloud.google.com/), either +create a new project or switch to an existing one. Next, +[create a service account]( +https://cloud.google.com/iam/docs/creating-managing-service-accounts) that will +be used to run the system tests. Once you have a service account, +[create and download a service account key](https://cloud.google.com/iam/docs/managing-service-account-keys). + +In the +[IAM & Admin](https://console.cloud.google.com/permissions/projectpermissions) +section, grant the `Owner` role to the service account you created above. Also +grant the `Editor` role to the `cloud-logs@google.com` service account. + + +Then, follow the +[system test setup instructions](https://github.com/GoogleCloudPlatform/google-cloud-python/blob/master/CONTRIBUTING.rst#running-system-tests). It +describes various steps, including running some scripts to populate and/or +delete datastore example data and indexes (populate_datastore.py, +clear_datastore.py, and `gcloud preview datastore create-indexes +system_tests/data/index.yaml`). + +From the cloud console, you will need to enable the following APIs for your project: + +- Bigquery API +- Cloud Bigtable Admin API +- Google Cloud Natural Language API +- Google Cloud Pub/Sub API +- Google Cloud Storage JSON API +- Google Cloud Vision API +- Google Translate API +- Stackdriver Logging API +- Stackdriver Monitoring API + ## Contributing changes * See [CONTRIBUTING.md](CONTRIBUTING.md) From 46956ec6f0f73803d736371f43e1fcf4938b98b0 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Mon, 30 Jan 2017 18:33:27 -0800 Subject: [PATCH 102/362] Run system tests under Jenkins --- jenkins_build.sh | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/jenkins_build.sh b/jenkins_build.sh index b081255f..53062fa4 100755 --- a/jenkins_build.sh +++ b/jenkins_build.sh @@ -12,7 +12,21 @@ if [ -z "${DOCKER_NAMESPACE+set}" ] ; then exit 1 fi IMAGE_NAME="${DOCKER_NAMESPACE}/${RUNTIME_NAME}:${CANDIDATE_NAME}" - export IMAGE_NAME + +if [ -z "${GOOGLE_CLOUD_PROJECT+set}" ] ; then + echo "Error: GOOGLE_CLOUD_PROJECT is not set; invoke with something like GOOGLE_CLOUD_PROJECT=YOUR-PROJECT-NAME" >&2 + exit 1 +fi + export FORCE_REBUILD -make cloud-test + +make cloud-build +# We explicitly pull the image using 'gcloud', instead of letting +# Docker do it, so that we have the right credentials. +gcloud docker -- pull "${IMAGE_NAME}" +# Note that system test failures might be caused environment factors +# outside our control. Also, the images will be pushed to GCR by the +# previous build step regardless of system test failures. +make integration-tests || \ + echo "ERROR: System test failure, please examine the logs" From 3d559e6eb7b2c6204a95090d02384c34995f107e Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Tue, 31 Jan 2017 17:24:43 -0800 Subject: [PATCH 103/362] Be more verbose during Jenkins build --- jenkins_build.sh | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/jenkins_build.sh b/jenkins_build.sh index 53062fa4..f28196a7 100755 --- a/jenkins_build.sh +++ b/jenkins_build.sh @@ -11,8 +11,7 @@ if [ -z "${DOCKER_NAMESPACE+set}" ] ; then echo "Error: DOCKER_NAMESPACE is not set; invoke with something like DOCKER_NAMESPACE=gcr.io/YOUR-PROJECT-NAME" >&2 exit 1 fi -IMAGE_NAME="${DOCKER_NAMESPACE}/${RUNTIME_NAME}:${CANDIDATE_NAME}" -export IMAGE_NAME +export IMAGE_NAME="${DOCKER_NAMESPACE}/${RUNTIME_NAME}:${CANDIDATE_NAME}" if [ -z "${GOOGLE_CLOUD_PROJECT+set}" ] ; then echo "Error: GOOGLE_CLOUD_PROJECT is not set; invoke with something like GOOGLE_CLOUD_PROJECT=YOUR-PROJECT-NAME" >&2 @@ -21,10 +20,20 @@ fi export FORCE_REBUILD +echo "==================================================================" +echo "Building image ${IMAGE_NAME} and pushing to ${DOCKER_NAMESPACE} using ${GOOGLE_CLOUD_PROJECT}" +echo "==================================================================" + make cloud-build + # We explicitly pull the image using 'gcloud', instead of letting # Docker do it, so that we have the right credentials. +echo "==================================================================" +gcloud info +echo gcloud docker -- pull "${IMAGE_NAME}" gcloud docker -- pull "${IMAGE_NAME}" +echo "==================================================================" + # Note that system test failures might be caused environment factors # outside our control. Also, the images will be pushed to GCR by the # previous build step regardless of system test failures. From 21462276ba9c1c4eb45502feae16720d674055c3 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Wed, 1 Feb 2017 16:49:21 -0800 Subject: [PATCH 104/362] Do not cache Docker layers and base images by default Partially fixes #58. --- Makefile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 252adbba..9db5dc93 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ -ifdef FORCE_REBUILD - DOCKER_FLAGS = --no-cache --pull +ifneq ($(FORCE_REBUILD),0) + export DOCKER_FLAGS=--no-cache --pull endif ifndef IMAGE_NAME @@ -43,7 +43,6 @@ local-build: local-build-interpreters .PHONY: local-build-interpreters local-build-interpreters: - export DOCKER_FLAGS make -C python-interpreter-builder build .PHONY: local-test From b12d3d72e0a5e82ba633f5b53c9abce30cfd6d8b Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Wed, 1 Feb 2017 15:39:37 -0800 Subject: [PATCH 105/362] python-interpreter-builder/Dockerfile FROM debian8 not base Fixes #60 --- python-interpreter-builder/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-interpreter-builder/Dockerfile b/python-interpreter-builder/Dockerfile index 3458c5e4..36f5c47c 100644 --- a/python-interpreter-builder/Dockerfile +++ b/python-interpreter-builder/Dockerfile @@ -1,6 +1,6 @@ # The Google App Engine base image is debian (jessie) with ca-certificates # installed. -FROM gcr.io/google_appengine/base +FROM gcr.io/google_appengine/debian8 # Install Python build dependencies RUN apt-get update && apt-get install -yq \ From c885ddb00dcb104c4ca04a88118a4cac0c98b3c7 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Wed, 1 Feb 2017 15:42:47 -0800 Subject: [PATCH 106/362] Use google-appengine instead of google_appengine Fixes #59 --- Dockerfile | 2 +- python-interpreter-builder/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 85d10374..03cd19fc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # The Google App Engine base image is debian (jessie) with ca-certificates # installed. # Source: https://github.com/GoogleCloudPlatform/debian-docker -FROM gcr.io/google_appengine/debian8 +FROM gcr.io/google-appengine/debian8 ADD resources /resources ADD scripts /scripts diff --git a/python-interpreter-builder/Dockerfile b/python-interpreter-builder/Dockerfile index 36f5c47c..e3dffd4c 100644 --- a/python-interpreter-builder/Dockerfile +++ b/python-interpreter-builder/Dockerfile @@ -1,6 +1,6 @@ # The Google App Engine base image is debian (jessie) with ca-certificates # installed. -FROM gcr.io/google_appengine/debian8 +FROM gcr.io/google-appengine/debian8 # Install Python build dependencies RUN apt-get update && apt-get install -yq \ From 8e35a5afd7cc9a6406873742129df12ab5e88909 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Tue, 7 Feb 2017 16:22:53 -0800 Subject: [PATCH 107/362] Fix typo in structure test --- tests/virtualenv/virtualenv_default.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/virtualenv/virtualenv_default.yaml b/tests/virtualenv/virtualenv_default.yaml index e9187f13..314a3d0c 100644 --- a/tests/virtualenv/virtualenv_default.yaml +++ b/tests/virtualenv/virtualenv_default.yaml @@ -17,7 +17,7 @@ commandTests: - name: "virtualenv installation" setup: [["virtualenv", "/env"]] command: ["which", "python"] - expectedOutput": ["/env/bin/python\n"] + expectedOutput: ["/env/bin/python\n"] - name: "python version" command: ["python", "--version"] From fccdcd65ff5791d340f830dc3e5ada06a6aa7eb1 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Tue, 7 Feb 2017 16:28:01 -0800 Subject: [PATCH 108/362] Switch from 'gcloud alpha container' to 'gcloud beta container' --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 9db5dc93..080b9d8d 100644 --- a/Makefile +++ b/Makefile @@ -29,7 +29,7 @@ ext_run.sh: .PHONY: cloud-build cloud-build: cloudbuild.yaml tests/google-cloud-python/Dockerfile - gcloud alpha container builds create . --config=cloudbuild.yaml + gcloud beta container builds submit . --config=cloudbuild.yaml .PHONY: cloud-test # structure-tests and google-cloud-python-tests are implicit in cloud-build From bc562810d198a44d764e3f054ac3fcb5261517ef Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Tue, 7 Feb 2017 12:37:26 -0800 Subject: [PATCH 109/362] Move main Dockerfile and supporting files to a subdirectory. Partially addresses #57 --- .gitignore | 1 - Makefile | 2 +- cloudbuild.yaml.in | 6 +++--- python-interpreter-builder/Makefile | 2 +- runtime-image/.gitignore | 1 + Dockerfile => runtime-image/Dockerfile | 0 {resources => runtime-image/resources}/apt-packages.txt | 0 {scripts => runtime-image/scripts}/install-apt-packages.sh | 0 8 files changed, 6 insertions(+), 6 deletions(-) create mode 100644 runtime-image/.gitignore rename Dockerfile => runtime-image/Dockerfile (100%) rename {resources => runtime-image/resources}/apt-packages.txt (100%) rename {scripts => runtime-image/scripts}/install-apt-packages.sh (100%) diff --git a/.gitignore b/.gitignore index 2c260f13..05626c51 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,2 @@ cloudbuild.yaml ext_run.sh -interpreters.tar.gz diff --git a/Makefile b/Makefile index 080b9d8d..395d43b6 100644 --- a/Makefile +++ b/Makefile @@ -39,7 +39,7 @@ cloud-test: cloud-build integration-tests .PHONY: local-build local-build: local-build-interpreters - docker build $(DOCKER_FLAGS) -t "$(IMAGE_NAME)" . + docker build $(DOCKER_FLAGS) -t "$(IMAGE_NAME)" runtime-image .PHONY: local-build-interpreters local-build-interpreters: diff --git a/cloudbuild.yaml.in b/cloudbuild.yaml.in index b8ed6baa..4c505105 100644 --- a/cloudbuild.yaml.in +++ b/cloudbuild.yaml.in @@ -1,11 +1,11 @@ timeout: 7200s steps: - name: gcr.io/cloud-builders/docker - args: ['build', '--tag=interpreter', '--no-cache', 'python-interpreter-builder'] + args: ['build', '--tag=interpreter', '--no-cache', '/workspace/python-interpreter-builder/'] - name: interpreter - args: ['cp', '/interpreters.tar.gz', '/workspace/'] + args: ['cp', '/interpreters.tar.gz', '/workspace/runtime-image/interpreters.tar.gz'] - name: gcr.io/cloud-builders/docker - args: ['build', '--tag=${IMAGE_NAME}', '--no-cache', '.'] + args: ['build', '--tag=${IMAGE_NAME}', '--no-cache', '/workspace/runtime-image/'] - name: gcr.io/gcp-runtimes/structure_test args: [ '-i', '${IMAGE_NAME}', diff --git a/python-interpreter-builder/Makefile b/python-interpreter-builder/Makefile index 6acfe6bb..d125b1c8 100644 --- a/python-interpreter-builder/Makefile +++ b/python-interpreter-builder/Makefile @@ -6,5 +6,5 @@ build: -docker rm python-interpreter-builder docker run --name python-interpreter-builder google/python-interpreter-builder /bin/bash mkdir -p output - docker cp python-interpreter-builder:/interpreters.tar.gz ../interpreters.tar.gz + docker cp python-interpreter-builder:/interpreters.tar.gz ../runtime-image/interpreters.tar.gz docker rm python-interpreter-builder diff --git a/runtime-image/.gitignore b/runtime-image/.gitignore new file mode 100644 index 00000000..cb1afebc --- /dev/null +++ b/runtime-image/.gitignore @@ -0,0 +1 @@ +interpreters.tar.gz diff --git a/Dockerfile b/runtime-image/Dockerfile similarity index 100% rename from Dockerfile rename to runtime-image/Dockerfile diff --git a/resources/apt-packages.txt b/runtime-image/resources/apt-packages.txt similarity index 100% rename from resources/apt-packages.txt rename to runtime-image/resources/apt-packages.txt diff --git a/scripts/install-apt-packages.sh b/runtime-image/scripts/install-apt-packages.sh similarity index 100% rename from scripts/install-apt-packages.sh rename to runtime-image/scripts/install-apt-packages.sh From 7e5075618b578161eb15c37eb5c1afbcaf4bf431 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Tue, 7 Feb 2017 16:32:26 -0800 Subject: [PATCH 110/362] Add tool to run cloudbuild.yaml steps locally. Supports just enough Container Builder functionality to build this image. --- local-cloudbuild.py | 148 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 148 insertions(+) create mode 100755 local-cloudbuild.py diff --git a/local-cloudbuild.py b/local-cloudbuild.py new file mode 100755 index 00000000..31a2b903 --- /dev/null +++ b/local-cloudbuild.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python +# +# Run the steps of cloudbuild.yaml locally, emulating Google Container +# Builder functionality. Does not actually push images to the +# Container Registry. Not all functionality is supported. +# +# Based on https://cloud.google.com/container-builder/docs/api/build-steps + +# System packages +import argparse +import getpass +import os +import shutil +import subprocess +import sys +import tempfile + +# Third party packages +import yaml + + +# Types +class CloudBuildError(Exception): + pass + + +def main(argv): + """Main entrypoint for cli""" + parser = argparse.ArgumentParser( + description='Process cloudbuild.yaml locally to build Docker images') + parser.add_argument( + 'cloudbuild', type=str, help='Path to cloudbuild.yaml input file') + parser.add_argument( + '--keep_workspace', + type=bool, + default=False, + help='Retain workspace directory after building') + args = parser.parse_args(argv[1:]) + + # Load and parse cloudbuild.yaml + cloudbuild = None + with open(args.cloudbuild, 'rb') as infile: + cloudbuild = yaml.safe_load(infile) + + host_workspace_parent = tempfile.mkdtemp(prefix='local-cloudbuild_%s_' % + getpass.getuser()) + host_workspace = os.path.join(host_workspace_parent, 'workspace') + try: + # Prepare workspace + shutil.copytree('.', host_workspace, symlinks=True) + + # Execute a series of 'docker run' commands locally + print('Running cloudbuild locally. Host workspace directory is %s' % + host_workspace) + run_steps(cloudbuild, host_workspace) + finally: + if not args.keep_workspace: + shutil.rmtree(host_workspace_parent, ignore_errors=True) + + +def run_steps(cloudbuild, host_workspace): + """Run the steps listed in a cloudbuild.yaml file. + + Args: + cloudbuild (dict): The decoded contents of a cloudbuild.yaml + host_workspace (str): Scratch directory + + Raises: + CloudBuildError if the yaml contents are invalid + """ + steps = cloudbuild.get('steps', {}) + if not steps: + raise CloudBuildError('No steps defined in cloudbuild.yaml') + + for step in steps: + run_one_step(step, host_workspace) + + +def run_one_step(step, host_workspace): + """Run a single step listed in a cloudbuild.yaml file. + + Args: + step (dict): A single step to perform + host_workspace (str): Scratch directory + """ + name = get(step, 'name', str) + dir_ = get(step, 'dir', list) + env = get(step, 'env', list) + args = get(step, 'args', list) + run_docker(name, dir_, env, args, host_workspace) + + +def get(container, field_name, field_type): + """Fetch a field from a container with typechecking and default values. + + If the field is not present, a instance of `field_type` is + constructed with no arguments and used as the default value. + + Args: + container (dict): Object decoded from yaml + field_name (str): Field that should be present in `container` + field_type (type): Expected type for field value + + Returns: + fetched or default value of field + + Raises: + CloudBuildError if field value is present but is the wrong type. + """ + value = container.get(field_name) + if value is None: + return field_type() + if not isinstance(value, field_type): + raise CloudBuildError( + 'Syntax error: Expected "%d" to be of type "%d", but found "%d"', + field_name, field_type, type(value)) + return value + + +def run_docker(name, dir_, env_args, args, host_workspace): + """Construct and execute a single 'docker run' command""" + workdir = '/workspace' + if dir_: + workdir += '/' + dir_ + + env_pairs = [] + for env_arg in env_args: + env_pairs.append('--env', env_arg) + + process_args = [ + 'docker', + 'run', + '--volume', + '/var/run/docker.sock:/var/run/docker.sock', + '--volume', + '/root/.docker:/root/.docker', + '--volume', + '%s:/workspace' % host_workspace, + '--workdir', + workdir, + ] + env_args + [name] + args + + print('Executing ' + ' '.join(process_args)) + subprocess.check_call(process_args) + + +if __name__ == '__main__': + sys.exit(main(sys.argv)) From e84e14e4712b0f3fac41b9b07110877b6287d88c Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Thu, 9 Feb 2017 14:19:22 -0800 Subject: [PATCH 111/362] Minor review changes --- local-cloudbuild.py | 56 ++++++++++++++++++++++++++++----------------- 1 file changed, 35 insertions(+), 21 deletions(-) diff --git a/local-cloudbuild.py b/local-cloudbuild.py index 31a2b903..51bd2d79 100755 --- a/local-cloudbuild.py +++ b/local-cloudbuild.py @@ -1,12 +1,29 @@ #!/usr/bin/env python + +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# Run the steps of cloudbuild.yaml locally, emulating Google Container -# Builder functionality. Does not actually push images to the -# Container Registry. Not all functionality is supported. +# http://www.apache.org/licenses/LICENSE-2.0 # -# Based on https://cloud.google.com/container-builder/docs/api/build-steps +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Emulate the Google Container Builder locally. + +The input is a cloudbuild.yaml file locally, which is processed using +a locally installed Docker daemon. The output images are not pushed +to the Google Container Registry. Not all functionality is supported. + +See https://cloud.google.com/container-builder/docs/api/build-steps +for more information. +""" -# System packages import argparse import getpass import os @@ -15,12 +32,11 @@ import sys import tempfile -# Third party packages import yaml -# Types class CloudBuildError(Exception): + """Syntax error in cloudbuild.yaml or other user error""" pass @@ -38,20 +54,18 @@ def main(argv): args = parser.parse_args(argv[1:]) # Load and parse cloudbuild.yaml - cloudbuild = None with open(args.cloudbuild, 'rb') as infile: cloudbuild = yaml.safe_load(infile) - host_workspace_parent = tempfile.mkdtemp(prefix='local-cloudbuild_%s_' % - getpass.getuser()) + host_workspace_parent = tempfile.mkdtemp(prefix='local-cloudbuild_') host_workspace = os.path.join(host_workspace_parent, 'workspace') try: # Prepare workspace + print('Running cloudbuild locally. Host workspace directory is %s' % + host_workspace) shutil.copytree('.', host_workspace, symlinks=True) # Execute a series of 'docker run' commands locally - print('Running cloudbuild locally. Host workspace directory is %s' % - host_workspace) run_steps(cloudbuild, host_workspace) finally: if not args.keep_workspace: @@ -68,7 +82,7 @@ def run_steps(cloudbuild, host_workspace): Raises: CloudBuildError if the yaml contents are invalid """ - steps = cloudbuild.get('steps', {}) + steps = cloudbuild.get_field_value('steps', {}) if not steps: raise CloudBuildError('No steps defined in cloudbuild.yaml') @@ -83,14 +97,14 @@ def run_one_step(step, host_workspace): step (dict): A single step to perform host_workspace (str): Scratch directory """ - name = get(step, 'name', str) - dir_ = get(step, 'dir', list) - env = get(step, 'env', list) - args = get(step, 'args', list) + name = get_field_value(step, 'name', str) + dir_ = get_field_value(step, 'dir', list) + env = get_field_value(step, 'env', list) + args = get_field_value(step, 'args', list) run_docker(name, dir_, env, args, host_workspace) -def get(container, field_name, field_type): +def get_field_value(container, field_name, field_type): """Fetch a field from a container with typechecking and default values. If the field is not present, a instance of `field_type` is @@ -102,7 +116,7 @@ def get(container, field_name, field_type): field_type (type): Expected type for field value Returns: - fetched or default value of field + (any) fetched or default value of field Raises: CloudBuildError if field value is present but is the wrong type. @@ -112,7 +126,7 @@ def get(container, field_name, field_type): return field_type() if not isinstance(value, field_type): raise CloudBuildError( - 'Syntax error: Expected "%d" to be of type "%d", but found "%d"', + 'Expected "%d" to be of type "%d", but found "%d"', field_name, field_type, type(value)) return value @@ -121,7 +135,7 @@ def run_docker(name, dir_, env_args, args, host_workspace): """Construct and execute a single 'docker run' command""" workdir = '/workspace' if dir_: - workdir += '/' + dir_ + workdir = os.path.join(workdir, dir_) env_pairs = [] for env_arg in env_args: From c2aabc49fea8567a88ee50fba92c98ecd40b582b Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Thu, 9 Feb 2017 15:30:07 -0800 Subject: [PATCH 112/362] Move main() to bottom of file --- local-cloudbuild.py | 64 ++++++++++++++++++++++----------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/local-cloudbuild.py b/local-cloudbuild.py index 51bd2d79..b44446ff 100755 --- a/local-cloudbuild.py +++ b/local-cloudbuild.py @@ -40,38 +40,6 @@ class CloudBuildError(Exception): pass -def main(argv): - """Main entrypoint for cli""" - parser = argparse.ArgumentParser( - description='Process cloudbuild.yaml locally to build Docker images') - parser.add_argument( - 'cloudbuild', type=str, help='Path to cloudbuild.yaml input file') - parser.add_argument( - '--keep_workspace', - type=bool, - default=False, - help='Retain workspace directory after building') - args = parser.parse_args(argv[1:]) - - # Load and parse cloudbuild.yaml - with open(args.cloudbuild, 'rb') as infile: - cloudbuild = yaml.safe_load(infile) - - host_workspace_parent = tempfile.mkdtemp(prefix='local-cloudbuild_') - host_workspace = os.path.join(host_workspace_parent, 'workspace') - try: - # Prepare workspace - print('Running cloudbuild locally. Host workspace directory is %s' % - host_workspace) - shutil.copytree('.', host_workspace, symlinks=True) - - # Execute a series of 'docker run' commands locally - run_steps(cloudbuild, host_workspace) - finally: - if not args.keep_workspace: - shutil.rmtree(host_workspace_parent, ignore_errors=True) - - def run_steps(cloudbuild, host_workspace): """Run the steps listed in a cloudbuild.yaml file. @@ -158,5 +126,37 @@ def run_docker(name, dir_, env_args, args, host_workspace): subprocess.check_call(process_args) +def main(argv): + """Main entrypoint for cli""" + parser = argparse.ArgumentParser( + description='Process cloudbuild.yaml locally to build Docker images') + parser.add_argument( + 'cloudbuild', type=str, help='Path to cloudbuild.yaml input file') + parser.add_argument( + '--keep_workspace', + type=bool, + default=False, + help='Retain workspace directory after building') + args = parser.parse_args(argv[1:]) + + # Load and parse cloudbuild.yaml + with open(args.cloudbuild, 'rb') as infile: + cloudbuild = yaml.safe_load(infile) + + host_workspace_parent = tempfile.mkdtemp(prefix='local-cloudbuild_') + host_workspace = os.path.join(host_workspace_parent, 'workspace') + try: + # Prepare workspace + print('Running cloudbuild locally. Host workspace directory is %s' % + host_workspace) + shutil.copytree('.', host_workspace, symlinks=True) + + # Execute a series of 'docker run' commands locally + run_steps(cloudbuild, host_workspace) + finally: + if not args.keep_workspace: + shutil.rmtree(host_workspace_parent, ignore_errors=True) + + if __name__ == '__main__': sys.exit(main(sys.argv)) From 6dc37bdee5f75d909eafdca1df1e1bfb39494e17 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Thu, 16 Feb 2017 17:25:38 -0800 Subject: [PATCH 113/362] Improve tool to run cloudbuild.yaml steps locally. The tool was moved to scripts/ and renamed. It now produces a shell script of Docker commands that can be stored and examined without necessarily running them (by passing --no-run). Command line flags changed to match 'gcloud container builds submit'. Added tests. Various other bugfixes and improvements. --- .gitignore | 6 +- local-cloudbuild.py | 162 --------- scripts/local_cloudbuild.py | 322 ++++++++++++++++++ scripts/local_cloudbuild_test.py | 322 ++++++++++++++++++ .../testdata/cloudbuild_err_not_found.yaml | 3 + scripts/testdata/cloudbuild_err_rc1.yaml | 3 + scripts/testdata/cloudbuild_ok.yaml | 7 + scripts/testdata/cloudbuild_ok.yaml_golden.sh | 28 ++ 8 files changed, 689 insertions(+), 164 deletions(-) delete mode 100755 local-cloudbuild.py create mode 100755 scripts/local_cloudbuild.py create mode 100755 scripts/local_cloudbuild_test.py create mode 100644 scripts/testdata/cloudbuild_err_not_found.yaml create mode 100644 scripts/testdata/cloudbuild_err_rc1.yaml create mode 100644 scripts/testdata/cloudbuild_ok.yaml create mode 100755 scripts/testdata/cloudbuild_ok.yaml_golden.sh diff --git a/.gitignore b/.gitignore index 05626c51..0c354cc9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ -cloudbuild.yaml -ext_run.sh +/cloudbuild.yaml +/cloudbuild.yaml_local.sh +/ext_run.sh +__pycache__ diff --git a/local-cloudbuild.py b/local-cloudbuild.py deleted file mode 100755 index b44446ff..00000000 --- a/local-cloudbuild.py +++ /dev/null @@ -1,162 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Emulate the Google Container Builder locally. - -The input is a cloudbuild.yaml file locally, which is processed using -a locally installed Docker daemon. The output images are not pushed -to the Google Container Registry. Not all functionality is supported. - -See https://cloud.google.com/container-builder/docs/api/build-steps -for more information. -""" - -import argparse -import getpass -import os -import shutil -import subprocess -import sys -import tempfile - -import yaml - - -class CloudBuildError(Exception): - """Syntax error in cloudbuild.yaml or other user error""" - pass - - -def run_steps(cloudbuild, host_workspace): - """Run the steps listed in a cloudbuild.yaml file. - - Args: - cloudbuild (dict): The decoded contents of a cloudbuild.yaml - host_workspace (str): Scratch directory - - Raises: - CloudBuildError if the yaml contents are invalid - """ - steps = cloudbuild.get_field_value('steps', {}) - if not steps: - raise CloudBuildError('No steps defined in cloudbuild.yaml') - - for step in steps: - run_one_step(step, host_workspace) - - -def run_one_step(step, host_workspace): - """Run a single step listed in a cloudbuild.yaml file. - - Args: - step (dict): A single step to perform - host_workspace (str): Scratch directory - """ - name = get_field_value(step, 'name', str) - dir_ = get_field_value(step, 'dir', list) - env = get_field_value(step, 'env', list) - args = get_field_value(step, 'args', list) - run_docker(name, dir_, env, args, host_workspace) - - -def get_field_value(container, field_name, field_type): - """Fetch a field from a container with typechecking and default values. - - If the field is not present, a instance of `field_type` is - constructed with no arguments and used as the default value. - - Args: - container (dict): Object decoded from yaml - field_name (str): Field that should be present in `container` - field_type (type): Expected type for field value - - Returns: - (any) fetched or default value of field - - Raises: - CloudBuildError if field value is present but is the wrong type. - """ - value = container.get(field_name) - if value is None: - return field_type() - if not isinstance(value, field_type): - raise CloudBuildError( - 'Expected "%d" to be of type "%d", but found "%d"', - field_name, field_type, type(value)) - return value - - -def run_docker(name, dir_, env_args, args, host_workspace): - """Construct and execute a single 'docker run' command""" - workdir = '/workspace' - if dir_: - workdir = os.path.join(workdir, dir_) - - env_pairs = [] - for env_arg in env_args: - env_pairs.append('--env', env_arg) - - process_args = [ - 'docker', - 'run', - '--volume', - '/var/run/docker.sock:/var/run/docker.sock', - '--volume', - '/root/.docker:/root/.docker', - '--volume', - '%s:/workspace' % host_workspace, - '--workdir', - workdir, - ] + env_args + [name] + args - - print('Executing ' + ' '.join(process_args)) - subprocess.check_call(process_args) - - -def main(argv): - """Main entrypoint for cli""" - parser = argparse.ArgumentParser( - description='Process cloudbuild.yaml locally to build Docker images') - parser.add_argument( - 'cloudbuild', type=str, help='Path to cloudbuild.yaml input file') - parser.add_argument( - '--keep_workspace', - type=bool, - default=False, - help='Retain workspace directory after building') - args = parser.parse_args(argv[1:]) - - # Load and parse cloudbuild.yaml - with open(args.cloudbuild, 'rb') as infile: - cloudbuild = yaml.safe_load(infile) - - host_workspace_parent = tempfile.mkdtemp(prefix='local-cloudbuild_') - host_workspace = os.path.join(host_workspace_parent, 'workspace') - try: - # Prepare workspace - print('Running cloudbuild locally. Host workspace directory is %s' % - host_workspace) - shutil.copytree('.', host_workspace, symlinks=True) - - # Execute a series of 'docker run' commands locally - run_steps(cloudbuild, host_workspace) - finally: - if not args.keep_workspace: - shutil.rmtree(host_workspace_parent, ignore_errors=True) - - -if __name__ == '__main__': - sys.exit(main(sys.argv)) diff --git a/scripts/local_cloudbuild.py b/scripts/local_cloudbuild.py new file mode 100755 index 00000000..53ae86e0 --- /dev/null +++ b/scripts/local_cloudbuild.py @@ -0,0 +1,322 @@ +#!/usr/bin/env python3 + +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Emulate the Google Container Builder locally. + +The input is a local cloudbuild.yaml file. This is translated into a +series of commands for the locally installed Docker daemon. These +commands are output as a shell script and optionally executed. + +The output images are not pushed to the Google Container Registry. +Not all cloudbuild.yaml functionality is supported. + +See https://cloud.google.com/container-builder/docs/api/build-steps +for more information. +""" + +import argparse +import collections +import collections.abc +import functools +import io +import os +import re +import shlex +import subprocess +import sys + +import yaml + + +# Exclude non-printable control characters (including newlines) +PRINTABLE_REGEX = re.compile(r"""^[^\x00-\x1f]*$""") + +# File template +BUILD_SCRIPT_HEADER = """\ +#!/bin/bash +# This is a generated file. Do not edit. + +set -euo pipefail + +SOURCE_DIR=. + +# Setup staging directory +HOST_WORKSPACE=$(mktemp -d) +function cleanup { + if [ "${HOST_WORKSPACE}" != '/' -a -d "${HOST_WORKSPACE}" ]; then + rm -rf "${HOST_WORKSPACE}" + fi +} +trap cleanup EXIT + +# Copy source to staging directory +echo "Copying source to staging directory ${HOST_WORKSPACE}" +rsync -avzq --exclude=.git "${SOURCE_DIR}" "${HOST_WORKSPACE}" + +# Build commands +""" + +BUILD_SCRIPT_FOOTER = """\ +# End of build commands + +echo "Build completed successfully" +""" + + +# Validated cloudbuild recipe + flags +CloudBuild = collections.namedtuple('CloudBuild', 'output_script run steps') + +# Single validated step in a cloudbuild recipe +Step = collections.namedtuple('Step', 'args dir_ env name') + + +def get_field_value(container, field_name, field_type): + """Fetch a field from a container with typechecking and default values. + + The field value is coerced to the desired type. If the field is + not present, a instance of `field_type` is constructed with no + arguments and used as the default value. + + Args: + container (dict): Object decoded from yaml + field_name (str): Field that should be present in `container` + field_type (type): Expected type for field value + + Returns: + any: Fetched or default value of field + + Raises: + ValueError: if field value cannot be converted to the desired type + """ + try: + value = container[field_name] + except (IndexError, KeyError): + return field_type() + + msg = 'Expected "{}" field to be of type "{}", but found type "{}"' + if not isinstance(value, field_type): + # list('some string') is a successful type cast as far as Python + # is concerned, but doesn't exactly produce the results we want. + # We have a whitelist of conversions we will attempt. + whitelist = ( + (float, str), + (int, str), + (str, float), + (str, int), + (int, float), + ) + if (type(value), field_type) not in whitelist: + raise ValueError(msg.format(field_name, field_type, type(value))) + + try: + value = field_type(value) + except ValueError as e: + e.message = msg.format(field_name, field_type, type(value)) + raise + return value + + +def get_cloudbuild(raw_config, args): + """Read and validate a cloudbuild recipe + + Args: + raw_config (dict): deserialized cloudbuild.yaml + args (argparse.Namespace): ccommand line flags + + Returns: + CloudBuild: valid configuration + """ + if not isinstance(raw_config, dict): + raise ValueError( + 'Expected {} contents to be of type "dict", but found type "{}"'. + format(args.config, type(raw_config))) + raw_steps = get_field_value(raw_config, 'steps', list) + if not raw_steps: + raise ValueError('No steps defined in {}'.format(args.config)) + steps = [get_step(raw_step) for raw_step in raw_steps] + return CloudBuild( + output_script=args.output_script, + run=args.run, + steps=steps, + ) + + +def get_step(raw_step): + """Read and validate a single cloudbuild step + + Args: + raw_step (dict): deserialized step + + Returns: + Step: valid build step + """ + if not isinstance(raw_step, dict): + raise ValueError( + 'Expected step to be of type "dict", but found type "{}"'. + format(type(raw_step))) + raw_args = get_field_value(raw_step, 'args', list) + args = [get_field_value(raw_args, i, str) + for i in range(len(raw_args))] + dir_ = get_field_value(raw_step, 'dir', str) + raw_env = get_field_value(raw_step, 'env', list) + env = [get_field_value(raw_env, i, str) + for i in range(len(raw_env))] + name = get_field_value(raw_step, 'name', str) + return Step( + args=args, + dir_=dir_, + env=env, + name=name, + ) + + +def generate_command(step): + """Generate a single shell command to run for a single cloudbuild step + + Args: + step (Step): Valid build step + + Returns: + [str]: A single shell command, expressed as a list of quoted tokens. + """ + quoted_args = [shlex.quote(arg) for arg in step.args] + quoted_env = [] + for env in step.env: + quoted_env.extend(['--env', shlex.quote(env)]) + quoted_name = shlex.quote(step.name) + workdir = '/workspace' + if step.dir_: + workdir = os.path.join(workdir, shlex.quote(step.dir_)) + process_args = [ + 'docker', + 'run', + '--volume', + '/var/run/docker.sock:/var/run/docker.sock', + '--volume', + '/root/.docker:/root/.docker', + '--volume', + '${HOST_WORKSPACE}:/workspace', + '--workdir', + workdir, + ] + quoted_env + [quoted_name] + quoted_args + return process_args + + +def generate_script(cloudbuild): + """Generate the contents of a shell script + + Args: + cloudbuild (CloudBuild): Valid cloudbuild configuration + + Returns: + (str): Contents of shell script + """ + outfile = io.StringIO() + outfile.write(BUILD_SCRIPT_HEADER) + docker_commands = [generate_command(step) for step in cloudbuild.steps] + for docker_command in docker_commands: + line = ' '.join(docker_command) + '\n\n' + outfile.write(line) + outfile.write(BUILD_SCRIPT_FOOTER) + s = outfile.getvalue() + outfile.close() + return s + + +def make_executable(path): + """Set executable bit(s) on file""" + # http://stackoverflow.com/questions/12791997 + mode = os.stat(path).st_mode + mode |= (mode & 0o444) >> 2 # copy R bits to X + os.chmod(path, mode) + + +def write_script(cloudbuild, contents): + """Write a shell script to a file.""" + print('Writing build script to {}'.format(cloudbuild.output_script)) + with open(cloudbuild.output_script, 'w', encoding='utf8') as outfile: + outfile.write(contents) + make_executable(cloudbuild.output_script) + + +def local_cloudbuild(args): + """Execute the steps of a cloudbuild.yaml locally + + Args: + args: command line flags as per parse_args + """ + # Load and parse cloudbuild.yaml + with open(args.config, 'r', encoding='utf8') as cloudbuild_file: + raw_config = yaml.safe_load(cloudbuild_file) + + # Determine configuration + cloudbuild = get_cloudbuild(raw_config, args) + + # Create shell script + contents = generate_script(cloudbuild) + write_script(cloudbuild, contents) + + # Run shell script + if cloudbuild.run: + args = [os.path.abspath(cloudbuild.output_script)] + subprocess.check_call(args) + + +def validate_arg_regex(flag_value, flag_regex): + """Check a named command line flag against a regular expression""" + if not re.match(flag_regex, flag_value): + raise argparse.ArgumentTypeError( + 'Value "{}" does not match pattern "{}"'.format( + flag_value, flag_regex.pattern)) + return flag_value + + +def parse_args(argv): + """Parse and validate command line flags""" + parser = argparse.ArgumentParser( + description='Process cloudbuild.yaml locally to build Docker images') + parser.add_argument( + '--config', + type=functools.partial( + validate_arg_regex, flag_regex=PRINTABLE_REGEX), + default='cloudbuild.yaml', + help='Path to cloudbuild.yaml file' + ) + parser.add_argument( + '--output_script', + type=functools.partial( + validate_arg_regex, flag_regex=PRINTABLE_REGEX), + help='Filename to write shell script to', + ) + parser.add_argument( + '--no-run', + action='store_false', + help='Create shell script but don\'t execute it', + dest='run', + ) + args = parser.parse_args(argv[1:]) + if not args.output_script: + args.output_script = args.config + "_local.sh" + return args + + +def main(): + args = parse_args(sys.argv) + local_cloudbuild(args) + + +if __name__ == '__main__': + main() diff --git a/scripts/local_cloudbuild_test.py b/scripts/local_cloudbuild_test.py new file mode 100755 index 00000000..481e5ae9 --- /dev/null +++ b/scripts/local_cloudbuild_test.py @@ -0,0 +1,322 @@ +#!/usr/bin/env python3 + +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit test for local_cloudbuild.py""" + +import argparse +import os +import re +import shutil +import subprocess +import tempfile +import unittest +import unittest.mock + +import yaml + +import local_cloudbuild + + +class ValidationUtilsTest(unittest.TestCase): + + def test_get_field_value(self): + valid_cases = ( + # Normal case, field present and correct type + ({ 'present': 1 }, 'present', int, 1), + ({ 'present': '1' }, 'present', str, '1'), + ({ 'present': [1] }, 'present', list, [1]), + ({ 'present': {1: 2} }, 'present', dict, {1: 2}), + # Missing field replaced by default + ({}, 'missing', str, ''), + # Valid conversions + ({ 'str_to_int': '1' }, 'str_to_int', int, 1), + ({ 'int_to_str': 1 }, 'int_to_str', str, '1'), + ) + for valid_case in valid_cases: + with self.subTest(valid_case=valid_case): + container, field_name, field_type, expected = valid_case + self.assertEqual( + local_cloudbuild.get_field_value( + container, field_name, field_type), + expected) + + invalid_cases = ( + # Type conversion failures + ({ 'bad_list_to_dict': [1] }, 'bad_list_to_dict', dict), + ({ 'bad_list_to_str': [1] }, 'bad_list_to_str', str), + ({ 'bad_dict_to_list': {1: 2} }, 'bad_dict_to_list', list), + ({ 'bad_str_to_int': 'not_an_int' }, 'bad_str_to_int', int), + ({ 'bad_str_to_list': 'abc' }, 'bad_str_to_list', list), + ) + for invalid_case in invalid_cases: + with self.subTest(invalid_case=invalid_case): + container, field_name, field_type = invalid_case + with self.assertRaises(ValueError): + local_cloudbuild.get_field_value( + container, field_name, field_type) + + def test_validate_arg_regex(self): + self.assertEqual( + local_cloudbuild.validate_arg_regex('abc', re.compile('a[b]c')), + 'abc') + with self.assertRaises(argparse.ArgumentTypeError): + local_cloudbuild.validate_arg_regex('abc', re.compile('a[d]c')) + + +class LocalCloudbuildTest(unittest.TestCase): + + def setUp(self): + self.testdata_dir = 'testdata' + assert os.path.isdir(self.testdata_dir), 'Could not run test: testdata directory not found' + + def test_get_cloudbuild(self): + args = argparse.Namespace( + config='some_config_file', + output_script='some_output_script', + run=False, + ) + # Basic valid case + valid_case = 'steps:\n- name: step1\n- name: step2\n' + raw_config = yaml.safe_load(valid_case) + actual = local_cloudbuild.get_cloudbuild(raw_config, args) + self.assertEqual(len(actual.steps), 2) + + invalid_cases = ( + # Empty cloud build + '', + # No steps + 'foo: bar\n', + # Steps not a list + 'steps: astring\n', + ) + for invalid_case in invalid_cases: + with self.subTest(invalid_case=invalid_case): + raw_config = yaml.safe_load(invalid_case) + with self.assertRaises(ValueError): + local_cloudbuild.get_cloudbuild(raw_config, args) + + def test_get_step(self): + valid_cases = ( + # Empty step + ({}, local_cloudbuild.Step( + args=[], + dir_='', + env=[], + name='', + )), + # Full step + ({'name' : 'aname', + 'args' : [ 'arg1', 2, 'arg3 with \n newline', ], + 'env' : [ 'ENV1=value1', 'ENV2=space in value2' ], + 'dir' : 'adir', + }, local_cloudbuild.Step( + args = [ 'arg1', '2', 'arg3 with \n newline', ], + env = [ 'ENV1=value1', 'ENV2=space in value2' ], + dir_ = 'adir', + name = 'aname', + )), + ) + for valid_case in valid_cases: + with self.subTest(valid_case=valid_case): + raw_step, expected = valid_case + actual = local_cloudbuild.get_step(raw_step) + self.assertEqual(actual, expected) + + invalid_cases = ( + # Wrong type + [], + # More wrong types + {'args': 'not_a_list'}, + {'args': [ [] ]}, + {'env': 'not_a_list'}, + {'env': [ {} ]}, + {'dir': {}}, + {'name': []}, + ) + for invalid_case in invalid_cases: + with self.subTest(invalid_case=invalid_case): + with self.assertRaises(ValueError): + local_cloudbuild.get_step(invalid_case) + + def test_generate_command(self): + # Basic valid case + base_step = local_cloudbuild.Step( + args = ['arg1','arg2'], + dir_ = '', + env = ['ENV1=value1', 'ENV2=value2'], + name = 'aname', + ) + command = local_cloudbuild.generate_command(base_step) + self.assertEqual(command, [ + 'docker', + 'run', + '--volume', + '/var/run/docker.sock:/var/run/docker.sock', + '--volume', + '/root/.docker:/root/.docker', + '--volume', + '${HOST_WORKSPACE}:/workspace', + '--workdir', + '/workspace', + '--env', + 'ENV1=value1', + '--env', + 'ENV2=value2', + 'aname', + 'arg1', + 'arg2', + ]) + + # dir specified + step = base_step._replace(dir_='adir') + command = local_cloudbuild.generate_command(step) + self.assertIn('--workdir', command) + self.assertIn('/workspace/adir', command) + + # Shell quoting + step = base_step._replace(args=['arg with \n newline']) + command = local_cloudbuild.generate_command(step) + self.assertIn("'arg with \n newline'", command) + + step = base_step._replace(dir_='dir/ with space/') + command = local_cloudbuild.generate_command(step) + self.assertIn("/workspace/'dir/ with space/'", command) + + step = base_step._replace(env=['env with space']) + command = local_cloudbuild.generate_command(step) + self.assertIn("'env with space'", command) + + step = base_step._replace(name='a name') + command = local_cloudbuild.generate_command(step) + self.assertIn("'a name'", command) + + def test_generate_script(self): + config_name = 'cloudbuild_ok.yaml' + config = os.path.join(self.testdata_dir, config_name) + expected_output_script = os.path.join(self.testdata_dir, config_name + '_golden.sh') + cloudbuild = local_cloudbuild.CloudBuild( + output_script='test_generate_script', + run=False, + steps=[ + local_cloudbuild.Step( + args=['/bin/sh', '-c', 'echo "${MESSAGE}"'], + dir_='', + env=['MESSAGE=Hello World!'], + name='debian', + ), + local_cloudbuild.Step( + args=['/bin/sh', '-c', 'echo "${MESSAGE}"'], + dir_='', + env=['MESSAGE=Goodbye\\n And Farewell!', 'UNUSED=unused'], + name='debian', + ) + ] + ) + actual = local_cloudbuild.generate_script(cloudbuild) + self.maxDiff = 2**16 + # Compare output against golden + with open(expected_output_script, 'r', encoding='utf8') as expected: + self.assertEqual(actual, expected.read()) + + def test_make_executable(self): + with tempfile.TemporaryDirectory( + prefix='local_cloudbuild_test_') as tempdir: + test_script_filename = os.path.join(tempdir, 'test_make_executable.sh') + with open(test_script_filename, 'w', encoding='utf8') as test_script: + test_script.write('#!/bin/sh\necho "Output from test_make_executable"') + local_cloudbuild.make_executable(test_script_filename) + output = subprocess.check_output([test_script_filename]) + self.assertEqual(output.decode('utf8'), "Output from test_make_executable\n") + + def test_write_script(self): + with tempfile.TemporaryDirectory( + prefix='local_cloudbuild_test_') as tempdir: + contents = 'The contents\n' + output_script_filename = os.path.join(tempdir, 'test_write_script') + cloudbuild = local_cloudbuild.CloudBuild( + output_script=output_script_filename, + run=False, + steps=[], + ) + local_cloudbuild.write_script(cloudbuild, contents) + with open(output_script_filename, 'r', encoding='utf8') as output_script: + actual = output_script.read() + self.assertEqual(actual, contents) + + def test_local_cloudbuild(self): + # Actually run it if we can find a docker command. + should_run = False + if ((shutil.which('docker') is not None) and + (subprocess.call(['docker', 'info'], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) == 0)): + should_run = True + + # Read cloudbuild.yaml from testdata file, write output to + # tempdir, and maybe try to run it + with tempfile.TemporaryDirectory( + prefix='local_cloudbuild_test_') as tempdir: + cases = ( + # Everything is ok + ('cloudbuild_ok.yaml', True), + # Exit code 1 (failure) + ('cloudbuild_err_rc1.yaml', False), + # Command not found + ('cloudbuild_err_not_found.yaml', False), + ) + for case in cases: + with self.subTest(case=cases): + config_name, should_succeed = case + config = os.path.join(self.testdata_dir, config_name) + actual_output_script = os.path.join( + tempdir, config_name + '_local.sh') + args = argparse.Namespace( + config=config, + output_script=actual_output_script, + run=should_run) + if should_run: + print("Executing docker commands in {}".format(actual_output_script)) + if should_succeed: + local_cloudbuild.local_cloudbuild(args) + else: + with self.assertRaises(subprocess.CalledProcessError): + local_cloudbuild.local_cloudbuild(args) + else: + # Generate but don't execute script + local_cloudbuild.local_cloudbuild(args) + + + def test_parse_args(self): + # Test explicit output_script + argv = ['argv0', '--output_script=my_output'] + args = local_cloudbuild.parse_args(argv) + self.assertEqual(args.output_script, 'my_output') + # Test implicit output_script + argv = ['argv0', '--config=my_config'] + args = local_cloudbuild.parse_args(argv) + self.assertEqual(args.output_script, 'my_config_local.sh') + + # Test run flag (default and --no-run) + argv = ['argv0'] + args = local_cloudbuild.parse_args(argv) + self.assertEqual(args.run, True) + argv = ['argv0', '--no-run'] + args = local_cloudbuild.parse_args(argv) + self.assertEqual(args.run, False) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/testdata/cloudbuild_err_not_found.yaml b/scripts/testdata/cloudbuild_err_not_found.yaml new file mode 100644 index 00000000..c7eb070d --- /dev/null +++ b/scripts/testdata/cloudbuild_err_not_found.yaml @@ -0,0 +1,3 @@ +steps: +- name: debian + args: ['/expected file not found'] diff --git a/scripts/testdata/cloudbuild_err_rc1.yaml b/scripts/testdata/cloudbuild_err_rc1.yaml new file mode 100644 index 00000000..3953a586 --- /dev/null +++ b/scripts/testdata/cloudbuild_err_rc1.yaml @@ -0,0 +1,3 @@ +steps: +- name: debian + args: ['/bin/sh', '-c', 'exit 1'] diff --git a/scripts/testdata/cloudbuild_ok.yaml b/scripts/testdata/cloudbuild_ok.yaml new file mode 100644 index 00000000..8478269b --- /dev/null +++ b/scripts/testdata/cloudbuild_ok.yaml @@ -0,0 +1,7 @@ +steps: +- name: debian + args: ['/bin/sh', '-c', 'echo "${MESSAGE}"'] + env: ['MESSAGE=Hello World!'] +- name: debian + args: ['/bin/sh', '-c', 'echo "${MESSAGE}"'] + env: ['MESSAGE=Goodbye\n And Farewell!', 'UNUSED=unused'] diff --git a/scripts/testdata/cloudbuild_ok.yaml_golden.sh b/scripts/testdata/cloudbuild_ok.yaml_golden.sh new file mode 100755 index 00000000..996c436e --- /dev/null +++ b/scripts/testdata/cloudbuild_ok.yaml_golden.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# This is a generated file. Do not edit. + +set -euo pipefail + +SOURCE_DIR=. + +# Setup staging directory +HOST_WORKSPACE=$(mktemp -d) +function cleanup { + if [ "${HOST_WORKSPACE}" != '/' -a -d "${HOST_WORKSPACE}" ]; then + rm -rf "${HOST_WORKSPACE}" + fi +} +trap cleanup EXIT + +# Copy source to staging directory +echo "Copying source to staging directory ${HOST_WORKSPACE}" +rsync -avzq --exclude=.git "${SOURCE_DIR}" "${HOST_WORKSPACE}" + +# Build commands +docker run --volume /var/run/docker.sock:/var/run/docker.sock --volume /root/.docker:/root/.docker --volume ${HOST_WORKSPACE}:/workspace --workdir /workspace --env 'MESSAGE=Hello World!' debian /bin/sh -c 'echo "${MESSAGE}"' + +docker run --volume /var/run/docker.sock:/var/run/docker.sock --volume /root/.docker:/root/.docker --volume ${HOST_WORKSPACE}:/workspace --workdir /workspace --env 'MESSAGE=Goodbye\n And Farewell!' --env UNUSED=unused debian /bin/sh -c 'echo "${MESSAGE}"' + +# End of build commands + +echo "Build completed successfully" From 2d128fcea89362652ab25b12aa37123dc54aa47a Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Thu, 16 Feb 2017 17:33:04 -0800 Subject: [PATCH 114/362] Revert "Improve tool to run cloudbuild.yaml steps locally." This reverts commit 6dc37bdee5f75d909eafdca1df1e1bfb39494e17. --- .gitignore | 6 +- local-cloudbuild.py | 162 +++++++++ scripts/local_cloudbuild.py | 322 ------------------ scripts/local_cloudbuild_test.py | 322 ------------------ .../testdata/cloudbuild_err_not_found.yaml | 3 - scripts/testdata/cloudbuild_err_rc1.yaml | 3 - scripts/testdata/cloudbuild_ok.yaml | 7 - scripts/testdata/cloudbuild_ok.yaml_golden.sh | 28 -- 8 files changed, 164 insertions(+), 689 deletions(-) create mode 100755 local-cloudbuild.py delete mode 100755 scripts/local_cloudbuild.py delete mode 100755 scripts/local_cloudbuild_test.py delete mode 100644 scripts/testdata/cloudbuild_err_not_found.yaml delete mode 100644 scripts/testdata/cloudbuild_err_rc1.yaml delete mode 100644 scripts/testdata/cloudbuild_ok.yaml delete mode 100755 scripts/testdata/cloudbuild_ok.yaml_golden.sh diff --git a/.gitignore b/.gitignore index 0c354cc9..05626c51 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,2 @@ -/cloudbuild.yaml -/cloudbuild.yaml_local.sh -/ext_run.sh -__pycache__ +cloudbuild.yaml +ext_run.sh diff --git a/local-cloudbuild.py b/local-cloudbuild.py new file mode 100755 index 00000000..b44446ff --- /dev/null +++ b/local-cloudbuild.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Emulate the Google Container Builder locally. + +The input is a cloudbuild.yaml file locally, which is processed using +a locally installed Docker daemon. The output images are not pushed +to the Google Container Registry. Not all functionality is supported. + +See https://cloud.google.com/container-builder/docs/api/build-steps +for more information. +""" + +import argparse +import getpass +import os +import shutil +import subprocess +import sys +import tempfile + +import yaml + + +class CloudBuildError(Exception): + """Syntax error in cloudbuild.yaml or other user error""" + pass + + +def run_steps(cloudbuild, host_workspace): + """Run the steps listed in a cloudbuild.yaml file. + + Args: + cloudbuild (dict): The decoded contents of a cloudbuild.yaml + host_workspace (str): Scratch directory + + Raises: + CloudBuildError if the yaml contents are invalid + """ + steps = cloudbuild.get_field_value('steps', {}) + if not steps: + raise CloudBuildError('No steps defined in cloudbuild.yaml') + + for step in steps: + run_one_step(step, host_workspace) + + +def run_one_step(step, host_workspace): + """Run a single step listed in a cloudbuild.yaml file. + + Args: + step (dict): A single step to perform + host_workspace (str): Scratch directory + """ + name = get_field_value(step, 'name', str) + dir_ = get_field_value(step, 'dir', list) + env = get_field_value(step, 'env', list) + args = get_field_value(step, 'args', list) + run_docker(name, dir_, env, args, host_workspace) + + +def get_field_value(container, field_name, field_type): + """Fetch a field from a container with typechecking and default values. + + If the field is not present, a instance of `field_type` is + constructed with no arguments and used as the default value. + + Args: + container (dict): Object decoded from yaml + field_name (str): Field that should be present in `container` + field_type (type): Expected type for field value + + Returns: + (any) fetched or default value of field + + Raises: + CloudBuildError if field value is present but is the wrong type. + """ + value = container.get(field_name) + if value is None: + return field_type() + if not isinstance(value, field_type): + raise CloudBuildError( + 'Expected "%d" to be of type "%d", but found "%d"', + field_name, field_type, type(value)) + return value + + +def run_docker(name, dir_, env_args, args, host_workspace): + """Construct and execute a single 'docker run' command""" + workdir = '/workspace' + if dir_: + workdir = os.path.join(workdir, dir_) + + env_pairs = [] + for env_arg in env_args: + env_pairs.append('--env', env_arg) + + process_args = [ + 'docker', + 'run', + '--volume', + '/var/run/docker.sock:/var/run/docker.sock', + '--volume', + '/root/.docker:/root/.docker', + '--volume', + '%s:/workspace' % host_workspace, + '--workdir', + workdir, + ] + env_args + [name] + args + + print('Executing ' + ' '.join(process_args)) + subprocess.check_call(process_args) + + +def main(argv): + """Main entrypoint for cli""" + parser = argparse.ArgumentParser( + description='Process cloudbuild.yaml locally to build Docker images') + parser.add_argument( + 'cloudbuild', type=str, help='Path to cloudbuild.yaml input file') + parser.add_argument( + '--keep_workspace', + type=bool, + default=False, + help='Retain workspace directory after building') + args = parser.parse_args(argv[1:]) + + # Load and parse cloudbuild.yaml + with open(args.cloudbuild, 'rb') as infile: + cloudbuild = yaml.safe_load(infile) + + host_workspace_parent = tempfile.mkdtemp(prefix='local-cloudbuild_') + host_workspace = os.path.join(host_workspace_parent, 'workspace') + try: + # Prepare workspace + print('Running cloudbuild locally. Host workspace directory is %s' % + host_workspace) + shutil.copytree('.', host_workspace, symlinks=True) + + # Execute a series of 'docker run' commands locally + run_steps(cloudbuild, host_workspace) + finally: + if not args.keep_workspace: + shutil.rmtree(host_workspace_parent, ignore_errors=True) + + +if __name__ == '__main__': + sys.exit(main(sys.argv)) diff --git a/scripts/local_cloudbuild.py b/scripts/local_cloudbuild.py deleted file mode 100755 index 53ae86e0..00000000 --- a/scripts/local_cloudbuild.py +++ /dev/null @@ -1,322 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2017 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Emulate the Google Container Builder locally. - -The input is a local cloudbuild.yaml file. This is translated into a -series of commands for the locally installed Docker daemon. These -commands are output as a shell script and optionally executed. - -The output images are not pushed to the Google Container Registry. -Not all cloudbuild.yaml functionality is supported. - -See https://cloud.google.com/container-builder/docs/api/build-steps -for more information. -""" - -import argparse -import collections -import collections.abc -import functools -import io -import os -import re -import shlex -import subprocess -import sys - -import yaml - - -# Exclude non-printable control characters (including newlines) -PRINTABLE_REGEX = re.compile(r"""^[^\x00-\x1f]*$""") - -# File template -BUILD_SCRIPT_HEADER = """\ -#!/bin/bash -# This is a generated file. Do not edit. - -set -euo pipefail - -SOURCE_DIR=. - -# Setup staging directory -HOST_WORKSPACE=$(mktemp -d) -function cleanup { - if [ "${HOST_WORKSPACE}" != '/' -a -d "${HOST_WORKSPACE}" ]; then - rm -rf "${HOST_WORKSPACE}" - fi -} -trap cleanup EXIT - -# Copy source to staging directory -echo "Copying source to staging directory ${HOST_WORKSPACE}" -rsync -avzq --exclude=.git "${SOURCE_DIR}" "${HOST_WORKSPACE}" - -# Build commands -""" - -BUILD_SCRIPT_FOOTER = """\ -# End of build commands - -echo "Build completed successfully" -""" - - -# Validated cloudbuild recipe + flags -CloudBuild = collections.namedtuple('CloudBuild', 'output_script run steps') - -# Single validated step in a cloudbuild recipe -Step = collections.namedtuple('Step', 'args dir_ env name') - - -def get_field_value(container, field_name, field_type): - """Fetch a field from a container with typechecking and default values. - - The field value is coerced to the desired type. If the field is - not present, a instance of `field_type` is constructed with no - arguments and used as the default value. - - Args: - container (dict): Object decoded from yaml - field_name (str): Field that should be present in `container` - field_type (type): Expected type for field value - - Returns: - any: Fetched or default value of field - - Raises: - ValueError: if field value cannot be converted to the desired type - """ - try: - value = container[field_name] - except (IndexError, KeyError): - return field_type() - - msg = 'Expected "{}" field to be of type "{}", but found type "{}"' - if not isinstance(value, field_type): - # list('some string') is a successful type cast as far as Python - # is concerned, but doesn't exactly produce the results we want. - # We have a whitelist of conversions we will attempt. - whitelist = ( - (float, str), - (int, str), - (str, float), - (str, int), - (int, float), - ) - if (type(value), field_type) not in whitelist: - raise ValueError(msg.format(field_name, field_type, type(value))) - - try: - value = field_type(value) - except ValueError as e: - e.message = msg.format(field_name, field_type, type(value)) - raise - return value - - -def get_cloudbuild(raw_config, args): - """Read and validate a cloudbuild recipe - - Args: - raw_config (dict): deserialized cloudbuild.yaml - args (argparse.Namespace): ccommand line flags - - Returns: - CloudBuild: valid configuration - """ - if not isinstance(raw_config, dict): - raise ValueError( - 'Expected {} contents to be of type "dict", but found type "{}"'. - format(args.config, type(raw_config))) - raw_steps = get_field_value(raw_config, 'steps', list) - if not raw_steps: - raise ValueError('No steps defined in {}'.format(args.config)) - steps = [get_step(raw_step) for raw_step in raw_steps] - return CloudBuild( - output_script=args.output_script, - run=args.run, - steps=steps, - ) - - -def get_step(raw_step): - """Read and validate a single cloudbuild step - - Args: - raw_step (dict): deserialized step - - Returns: - Step: valid build step - """ - if not isinstance(raw_step, dict): - raise ValueError( - 'Expected step to be of type "dict", but found type "{}"'. - format(type(raw_step))) - raw_args = get_field_value(raw_step, 'args', list) - args = [get_field_value(raw_args, i, str) - for i in range(len(raw_args))] - dir_ = get_field_value(raw_step, 'dir', str) - raw_env = get_field_value(raw_step, 'env', list) - env = [get_field_value(raw_env, i, str) - for i in range(len(raw_env))] - name = get_field_value(raw_step, 'name', str) - return Step( - args=args, - dir_=dir_, - env=env, - name=name, - ) - - -def generate_command(step): - """Generate a single shell command to run for a single cloudbuild step - - Args: - step (Step): Valid build step - - Returns: - [str]: A single shell command, expressed as a list of quoted tokens. - """ - quoted_args = [shlex.quote(arg) for arg in step.args] - quoted_env = [] - for env in step.env: - quoted_env.extend(['--env', shlex.quote(env)]) - quoted_name = shlex.quote(step.name) - workdir = '/workspace' - if step.dir_: - workdir = os.path.join(workdir, shlex.quote(step.dir_)) - process_args = [ - 'docker', - 'run', - '--volume', - '/var/run/docker.sock:/var/run/docker.sock', - '--volume', - '/root/.docker:/root/.docker', - '--volume', - '${HOST_WORKSPACE}:/workspace', - '--workdir', - workdir, - ] + quoted_env + [quoted_name] + quoted_args - return process_args - - -def generate_script(cloudbuild): - """Generate the contents of a shell script - - Args: - cloudbuild (CloudBuild): Valid cloudbuild configuration - - Returns: - (str): Contents of shell script - """ - outfile = io.StringIO() - outfile.write(BUILD_SCRIPT_HEADER) - docker_commands = [generate_command(step) for step in cloudbuild.steps] - for docker_command in docker_commands: - line = ' '.join(docker_command) + '\n\n' - outfile.write(line) - outfile.write(BUILD_SCRIPT_FOOTER) - s = outfile.getvalue() - outfile.close() - return s - - -def make_executable(path): - """Set executable bit(s) on file""" - # http://stackoverflow.com/questions/12791997 - mode = os.stat(path).st_mode - mode |= (mode & 0o444) >> 2 # copy R bits to X - os.chmod(path, mode) - - -def write_script(cloudbuild, contents): - """Write a shell script to a file.""" - print('Writing build script to {}'.format(cloudbuild.output_script)) - with open(cloudbuild.output_script, 'w', encoding='utf8') as outfile: - outfile.write(contents) - make_executable(cloudbuild.output_script) - - -def local_cloudbuild(args): - """Execute the steps of a cloudbuild.yaml locally - - Args: - args: command line flags as per parse_args - """ - # Load and parse cloudbuild.yaml - with open(args.config, 'r', encoding='utf8') as cloudbuild_file: - raw_config = yaml.safe_load(cloudbuild_file) - - # Determine configuration - cloudbuild = get_cloudbuild(raw_config, args) - - # Create shell script - contents = generate_script(cloudbuild) - write_script(cloudbuild, contents) - - # Run shell script - if cloudbuild.run: - args = [os.path.abspath(cloudbuild.output_script)] - subprocess.check_call(args) - - -def validate_arg_regex(flag_value, flag_regex): - """Check a named command line flag against a regular expression""" - if not re.match(flag_regex, flag_value): - raise argparse.ArgumentTypeError( - 'Value "{}" does not match pattern "{}"'.format( - flag_value, flag_regex.pattern)) - return flag_value - - -def parse_args(argv): - """Parse and validate command line flags""" - parser = argparse.ArgumentParser( - description='Process cloudbuild.yaml locally to build Docker images') - parser.add_argument( - '--config', - type=functools.partial( - validate_arg_regex, flag_regex=PRINTABLE_REGEX), - default='cloudbuild.yaml', - help='Path to cloudbuild.yaml file' - ) - parser.add_argument( - '--output_script', - type=functools.partial( - validate_arg_regex, flag_regex=PRINTABLE_REGEX), - help='Filename to write shell script to', - ) - parser.add_argument( - '--no-run', - action='store_false', - help='Create shell script but don\'t execute it', - dest='run', - ) - args = parser.parse_args(argv[1:]) - if not args.output_script: - args.output_script = args.config + "_local.sh" - return args - - -def main(): - args = parse_args(sys.argv) - local_cloudbuild(args) - - -if __name__ == '__main__': - main() diff --git a/scripts/local_cloudbuild_test.py b/scripts/local_cloudbuild_test.py deleted file mode 100755 index 481e5ae9..00000000 --- a/scripts/local_cloudbuild_test.py +++ /dev/null @@ -1,322 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2017 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Unit test for local_cloudbuild.py""" - -import argparse -import os -import re -import shutil -import subprocess -import tempfile -import unittest -import unittest.mock - -import yaml - -import local_cloudbuild - - -class ValidationUtilsTest(unittest.TestCase): - - def test_get_field_value(self): - valid_cases = ( - # Normal case, field present and correct type - ({ 'present': 1 }, 'present', int, 1), - ({ 'present': '1' }, 'present', str, '1'), - ({ 'present': [1] }, 'present', list, [1]), - ({ 'present': {1: 2} }, 'present', dict, {1: 2}), - # Missing field replaced by default - ({}, 'missing', str, ''), - # Valid conversions - ({ 'str_to_int': '1' }, 'str_to_int', int, 1), - ({ 'int_to_str': 1 }, 'int_to_str', str, '1'), - ) - for valid_case in valid_cases: - with self.subTest(valid_case=valid_case): - container, field_name, field_type, expected = valid_case - self.assertEqual( - local_cloudbuild.get_field_value( - container, field_name, field_type), - expected) - - invalid_cases = ( - # Type conversion failures - ({ 'bad_list_to_dict': [1] }, 'bad_list_to_dict', dict), - ({ 'bad_list_to_str': [1] }, 'bad_list_to_str', str), - ({ 'bad_dict_to_list': {1: 2} }, 'bad_dict_to_list', list), - ({ 'bad_str_to_int': 'not_an_int' }, 'bad_str_to_int', int), - ({ 'bad_str_to_list': 'abc' }, 'bad_str_to_list', list), - ) - for invalid_case in invalid_cases: - with self.subTest(invalid_case=invalid_case): - container, field_name, field_type = invalid_case - with self.assertRaises(ValueError): - local_cloudbuild.get_field_value( - container, field_name, field_type) - - def test_validate_arg_regex(self): - self.assertEqual( - local_cloudbuild.validate_arg_regex('abc', re.compile('a[b]c')), - 'abc') - with self.assertRaises(argparse.ArgumentTypeError): - local_cloudbuild.validate_arg_regex('abc', re.compile('a[d]c')) - - -class LocalCloudbuildTest(unittest.TestCase): - - def setUp(self): - self.testdata_dir = 'testdata' - assert os.path.isdir(self.testdata_dir), 'Could not run test: testdata directory not found' - - def test_get_cloudbuild(self): - args = argparse.Namespace( - config='some_config_file', - output_script='some_output_script', - run=False, - ) - # Basic valid case - valid_case = 'steps:\n- name: step1\n- name: step2\n' - raw_config = yaml.safe_load(valid_case) - actual = local_cloudbuild.get_cloudbuild(raw_config, args) - self.assertEqual(len(actual.steps), 2) - - invalid_cases = ( - # Empty cloud build - '', - # No steps - 'foo: bar\n', - # Steps not a list - 'steps: astring\n', - ) - for invalid_case in invalid_cases: - with self.subTest(invalid_case=invalid_case): - raw_config = yaml.safe_load(invalid_case) - with self.assertRaises(ValueError): - local_cloudbuild.get_cloudbuild(raw_config, args) - - def test_get_step(self): - valid_cases = ( - # Empty step - ({}, local_cloudbuild.Step( - args=[], - dir_='', - env=[], - name='', - )), - # Full step - ({'name' : 'aname', - 'args' : [ 'arg1', 2, 'arg3 with \n newline', ], - 'env' : [ 'ENV1=value1', 'ENV2=space in value2' ], - 'dir' : 'adir', - }, local_cloudbuild.Step( - args = [ 'arg1', '2', 'arg3 with \n newline', ], - env = [ 'ENV1=value1', 'ENV2=space in value2' ], - dir_ = 'adir', - name = 'aname', - )), - ) - for valid_case in valid_cases: - with self.subTest(valid_case=valid_case): - raw_step, expected = valid_case - actual = local_cloudbuild.get_step(raw_step) - self.assertEqual(actual, expected) - - invalid_cases = ( - # Wrong type - [], - # More wrong types - {'args': 'not_a_list'}, - {'args': [ [] ]}, - {'env': 'not_a_list'}, - {'env': [ {} ]}, - {'dir': {}}, - {'name': []}, - ) - for invalid_case in invalid_cases: - with self.subTest(invalid_case=invalid_case): - with self.assertRaises(ValueError): - local_cloudbuild.get_step(invalid_case) - - def test_generate_command(self): - # Basic valid case - base_step = local_cloudbuild.Step( - args = ['arg1','arg2'], - dir_ = '', - env = ['ENV1=value1', 'ENV2=value2'], - name = 'aname', - ) - command = local_cloudbuild.generate_command(base_step) - self.assertEqual(command, [ - 'docker', - 'run', - '--volume', - '/var/run/docker.sock:/var/run/docker.sock', - '--volume', - '/root/.docker:/root/.docker', - '--volume', - '${HOST_WORKSPACE}:/workspace', - '--workdir', - '/workspace', - '--env', - 'ENV1=value1', - '--env', - 'ENV2=value2', - 'aname', - 'arg1', - 'arg2', - ]) - - # dir specified - step = base_step._replace(dir_='adir') - command = local_cloudbuild.generate_command(step) - self.assertIn('--workdir', command) - self.assertIn('/workspace/adir', command) - - # Shell quoting - step = base_step._replace(args=['arg with \n newline']) - command = local_cloudbuild.generate_command(step) - self.assertIn("'arg with \n newline'", command) - - step = base_step._replace(dir_='dir/ with space/') - command = local_cloudbuild.generate_command(step) - self.assertIn("/workspace/'dir/ with space/'", command) - - step = base_step._replace(env=['env with space']) - command = local_cloudbuild.generate_command(step) - self.assertIn("'env with space'", command) - - step = base_step._replace(name='a name') - command = local_cloudbuild.generate_command(step) - self.assertIn("'a name'", command) - - def test_generate_script(self): - config_name = 'cloudbuild_ok.yaml' - config = os.path.join(self.testdata_dir, config_name) - expected_output_script = os.path.join(self.testdata_dir, config_name + '_golden.sh') - cloudbuild = local_cloudbuild.CloudBuild( - output_script='test_generate_script', - run=False, - steps=[ - local_cloudbuild.Step( - args=['/bin/sh', '-c', 'echo "${MESSAGE}"'], - dir_='', - env=['MESSAGE=Hello World!'], - name='debian', - ), - local_cloudbuild.Step( - args=['/bin/sh', '-c', 'echo "${MESSAGE}"'], - dir_='', - env=['MESSAGE=Goodbye\\n And Farewell!', 'UNUSED=unused'], - name='debian', - ) - ] - ) - actual = local_cloudbuild.generate_script(cloudbuild) - self.maxDiff = 2**16 - # Compare output against golden - with open(expected_output_script, 'r', encoding='utf8') as expected: - self.assertEqual(actual, expected.read()) - - def test_make_executable(self): - with tempfile.TemporaryDirectory( - prefix='local_cloudbuild_test_') as tempdir: - test_script_filename = os.path.join(tempdir, 'test_make_executable.sh') - with open(test_script_filename, 'w', encoding='utf8') as test_script: - test_script.write('#!/bin/sh\necho "Output from test_make_executable"') - local_cloudbuild.make_executable(test_script_filename) - output = subprocess.check_output([test_script_filename]) - self.assertEqual(output.decode('utf8'), "Output from test_make_executable\n") - - def test_write_script(self): - with tempfile.TemporaryDirectory( - prefix='local_cloudbuild_test_') as tempdir: - contents = 'The contents\n' - output_script_filename = os.path.join(tempdir, 'test_write_script') - cloudbuild = local_cloudbuild.CloudBuild( - output_script=output_script_filename, - run=False, - steps=[], - ) - local_cloudbuild.write_script(cloudbuild, contents) - with open(output_script_filename, 'r', encoding='utf8') as output_script: - actual = output_script.read() - self.assertEqual(actual, contents) - - def test_local_cloudbuild(self): - # Actually run it if we can find a docker command. - should_run = False - if ((shutil.which('docker') is not None) and - (subprocess.call(['docker', 'info'], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL) == 0)): - should_run = True - - # Read cloudbuild.yaml from testdata file, write output to - # tempdir, and maybe try to run it - with tempfile.TemporaryDirectory( - prefix='local_cloudbuild_test_') as tempdir: - cases = ( - # Everything is ok - ('cloudbuild_ok.yaml', True), - # Exit code 1 (failure) - ('cloudbuild_err_rc1.yaml', False), - # Command not found - ('cloudbuild_err_not_found.yaml', False), - ) - for case in cases: - with self.subTest(case=cases): - config_name, should_succeed = case - config = os.path.join(self.testdata_dir, config_name) - actual_output_script = os.path.join( - tempdir, config_name + '_local.sh') - args = argparse.Namespace( - config=config, - output_script=actual_output_script, - run=should_run) - if should_run: - print("Executing docker commands in {}".format(actual_output_script)) - if should_succeed: - local_cloudbuild.local_cloudbuild(args) - else: - with self.assertRaises(subprocess.CalledProcessError): - local_cloudbuild.local_cloudbuild(args) - else: - # Generate but don't execute script - local_cloudbuild.local_cloudbuild(args) - - - def test_parse_args(self): - # Test explicit output_script - argv = ['argv0', '--output_script=my_output'] - args = local_cloudbuild.parse_args(argv) - self.assertEqual(args.output_script, 'my_output') - # Test implicit output_script - argv = ['argv0', '--config=my_config'] - args = local_cloudbuild.parse_args(argv) - self.assertEqual(args.output_script, 'my_config_local.sh') - - # Test run flag (default and --no-run) - argv = ['argv0'] - args = local_cloudbuild.parse_args(argv) - self.assertEqual(args.run, True) - argv = ['argv0', '--no-run'] - args = local_cloudbuild.parse_args(argv) - self.assertEqual(args.run, False) - - -if __name__ == '__main__': - unittest.main() diff --git a/scripts/testdata/cloudbuild_err_not_found.yaml b/scripts/testdata/cloudbuild_err_not_found.yaml deleted file mode 100644 index c7eb070d..00000000 --- a/scripts/testdata/cloudbuild_err_not_found.yaml +++ /dev/null @@ -1,3 +0,0 @@ -steps: -- name: debian - args: ['/expected file not found'] diff --git a/scripts/testdata/cloudbuild_err_rc1.yaml b/scripts/testdata/cloudbuild_err_rc1.yaml deleted file mode 100644 index 3953a586..00000000 --- a/scripts/testdata/cloudbuild_err_rc1.yaml +++ /dev/null @@ -1,3 +0,0 @@ -steps: -- name: debian - args: ['/bin/sh', '-c', 'exit 1'] diff --git a/scripts/testdata/cloudbuild_ok.yaml b/scripts/testdata/cloudbuild_ok.yaml deleted file mode 100644 index 8478269b..00000000 --- a/scripts/testdata/cloudbuild_ok.yaml +++ /dev/null @@ -1,7 +0,0 @@ -steps: -- name: debian - args: ['/bin/sh', '-c', 'echo "${MESSAGE}"'] - env: ['MESSAGE=Hello World!'] -- name: debian - args: ['/bin/sh', '-c', 'echo "${MESSAGE}"'] - env: ['MESSAGE=Goodbye\n And Farewell!', 'UNUSED=unused'] diff --git a/scripts/testdata/cloudbuild_ok.yaml_golden.sh b/scripts/testdata/cloudbuild_ok.yaml_golden.sh deleted file mode 100755 index 996c436e..00000000 --- a/scripts/testdata/cloudbuild_ok.yaml_golden.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -# This is a generated file. Do not edit. - -set -euo pipefail - -SOURCE_DIR=. - -# Setup staging directory -HOST_WORKSPACE=$(mktemp -d) -function cleanup { - if [ "${HOST_WORKSPACE}" != '/' -a -d "${HOST_WORKSPACE}" ]; then - rm -rf "${HOST_WORKSPACE}" - fi -} -trap cleanup EXIT - -# Copy source to staging directory -echo "Copying source to staging directory ${HOST_WORKSPACE}" -rsync -avzq --exclude=.git "${SOURCE_DIR}" "${HOST_WORKSPACE}" - -# Build commands -docker run --volume /var/run/docker.sock:/var/run/docker.sock --volume /root/.docker:/root/.docker --volume ${HOST_WORKSPACE}:/workspace --workdir /workspace --env 'MESSAGE=Hello World!' debian /bin/sh -c 'echo "${MESSAGE}"' - -docker run --volume /var/run/docker.sock:/var/run/docker.sock --volume /root/.docker:/root/.docker --volume ${HOST_WORKSPACE}:/workspace --workdir /workspace --env 'MESSAGE=Goodbye\n And Farewell!' --env UNUSED=unused debian /bin/sh -c 'echo "${MESSAGE}"' - -# End of build commands - -echo "Build completed successfully" From c4b43870ac19cddb0d4dec9a63ace2914c51633c Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Thu, 16 Feb 2017 17:25:38 -0800 Subject: [PATCH 115/362] Improve tool to run cloudbuild.yaml steps locally. The tool was moved to scripts/ and renamed. It now produces a shell script of Docker commands that can be stored and examined without necessarily running them (by passing --no-run). Command line flags changed to match 'gcloud container builds submit'. Added tests. Various other bugfixes and improvements. --- .gitignore | 6 +- local-cloudbuild.py | 162 --------- scripts/local_cloudbuild.py | 322 ++++++++++++++++++ scripts/local_cloudbuild_test.py | 322 ++++++++++++++++++ .../testdata/cloudbuild_err_not_found.yaml | 3 + scripts/testdata/cloudbuild_err_rc1.yaml | 3 + scripts/testdata/cloudbuild_ok.yaml | 7 + scripts/testdata/cloudbuild_ok.yaml_golden.sh | 28 ++ 8 files changed, 689 insertions(+), 164 deletions(-) delete mode 100755 local-cloudbuild.py create mode 100755 scripts/local_cloudbuild.py create mode 100755 scripts/local_cloudbuild_test.py create mode 100644 scripts/testdata/cloudbuild_err_not_found.yaml create mode 100644 scripts/testdata/cloudbuild_err_rc1.yaml create mode 100644 scripts/testdata/cloudbuild_ok.yaml create mode 100755 scripts/testdata/cloudbuild_ok.yaml_golden.sh diff --git a/.gitignore b/.gitignore index 05626c51..0c354cc9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ -cloudbuild.yaml -ext_run.sh +/cloudbuild.yaml +/cloudbuild.yaml_local.sh +/ext_run.sh +__pycache__ diff --git a/local-cloudbuild.py b/local-cloudbuild.py deleted file mode 100755 index b44446ff..00000000 --- a/local-cloudbuild.py +++ /dev/null @@ -1,162 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Emulate the Google Container Builder locally. - -The input is a cloudbuild.yaml file locally, which is processed using -a locally installed Docker daemon. The output images are not pushed -to the Google Container Registry. Not all functionality is supported. - -See https://cloud.google.com/container-builder/docs/api/build-steps -for more information. -""" - -import argparse -import getpass -import os -import shutil -import subprocess -import sys -import tempfile - -import yaml - - -class CloudBuildError(Exception): - """Syntax error in cloudbuild.yaml or other user error""" - pass - - -def run_steps(cloudbuild, host_workspace): - """Run the steps listed in a cloudbuild.yaml file. - - Args: - cloudbuild (dict): The decoded contents of a cloudbuild.yaml - host_workspace (str): Scratch directory - - Raises: - CloudBuildError if the yaml contents are invalid - """ - steps = cloudbuild.get_field_value('steps', {}) - if not steps: - raise CloudBuildError('No steps defined in cloudbuild.yaml') - - for step in steps: - run_one_step(step, host_workspace) - - -def run_one_step(step, host_workspace): - """Run a single step listed in a cloudbuild.yaml file. - - Args: - step (dict): A single step to perform - host_workspace (str): Scratch directory - """ - name = get_field_value(step, 'name', str) - dir_ = get_field_value(step, 'dir', list) - env = get_field_value(step, 'env', list) - args = get_field_value(step, 'args', list) - run_docker(name, dir_, env, args, host_workspace) - - -def get_field_value(container, field_name, field_type): - """Fetch a field from a container with typechecking and default values. - - If the field is not present, a instance of `field_type` is - constructed with no arguments and used as the default value. - - Args: - container (dict): Object decoded from yaml - field_name (str): Field that should be present in `container` - field_type (type): Expected type for field value - - Returns: - (any) fetched or default value of field - - Raises: - CloudBuildError if field value is present but is the wrong type. - """ - value = container.get(field_name) - if value is None: - return field_type() - if not isinstance(value, field_type): - raise CloudBuildError( - 'Expected "%d" to be of type "%d", but found "%d"', - field_name, field_type, type(value)) - return value - - -def run_docker(name, dir_, env_args, args, host_workspace): - """Construct and execute a single 'docker run' command""" - workdir = '/workspace' - if dir_: - workdir = os.path.join(workdir, dir_) - - env_pairs = [] - for env_arg in env_args: - env_pairs.append('--env', env_arg) - - process_args = [ - 'docker', - 'run', - '--volume', - '/var/run/docker.sock:/var/run/docker.sock', - '--volume', - '/root/.docker:/root/.docker', - '--volume', - '%s:/workspace' % host_workspace, - '--workdir', - workdir, - ] + env_args + [name] + args - - print('Executing ' + ' '.join(process_args)) - subprocess.check_call(process_args) - - -def main(argv): - """Main entrypoint for cli""" - parser = argparse.ArgumentParser( - description='Process cloudbuild.yaml locally to build Docker images') - parser.add_argument( - 'cloudbuild', type=str, help='Path to cloudbuild.yaml input file') - parser.add_argument( - '--keep_workspace', - type=bool, - default=False, - help='Retain workspace directory after building') - args = parser.parse_args(argv[1:]) - - # Load and parse cloudbuild.yaml - with open(args.cloudbuild, 'rb') as infile: - cloudbuild = yaml.safe_load(infile) - - host_workspace_parent = tempfile.mkdtemp(prefix='local-cloudbuild_') - host_workspace = os.path.join(host_workspace_parent, 'workspace') - try: - # Prepare workspace - print('Running cloudbuild locally. Host workspace directory is %s' % - host_workspace) - shutil.copytree('.', host_workspace, symlinks=True) - - # Execute a series of 'docker run' commands locally - run_steps(cloudbuild, host_workspace) - finally: - if not args.keep_workspace: - shutil.rmtree(host_workspace_parent, ignore_errors=True) - - -if __name__ == '__main__': - sys.exit(main(sys.argv)) diff --git a/scripts/local_cloudbuild.py b/scripts/local_cloudbuild.py new file mode 100755 index 00000000..53ae86e0 --- /dev/null +++ b/scripts/local_cloudbuild.py @@ -0,0 +1,322 @@ +#!/usr/bin/env python3 + +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Emulate the Google Container Builder locally. + +The input is a local cloudbuild.yaml file. This is translated into a +series of commands for the locally installed Docker daemon. These +commands are output as a shell script and optionally executed. + +The output images are not pushed to the Google Container Registry. +Not all cloudbuild.yaml functionality is supported. + +See https://cloud.google.com/container-builder/docs/api/build-steps +for more information. +""" + +import argparse +import collections +import collections.abc +import functools +import io +import os +import re +import shlex +import subprocess +import sys + +import yaml + + +# Exclude non-printable control characters (including newlines) +PRINTABLE_REGEX = re.compile(r"""^[^\x00-\x1f]*$""") + +# File template +BUILD_SCRIPT_HEADER = """\ +#!/bin/bash +# This is a generated file. Do not edit. + +set -euo pipefail + +SOURCE_DIR=. + +# Setup staging directory +HOST_WORKSPACE=$(mktemp -d) +function cleanup { + if [ "${HOST_WORKSPACE}" != '/' -a -d "${HOST_WORKSPACE}" ]; then + rm -rf "${HOST_WORKSPACE}" + fi +} +trap cleanup EXIT + +# Copy source to staging directory +echo "Copying source to staging directory ${HOST_WORKSPACE}" +rsync -avzq --exclude=.git "${SOURCE_DIR}" "${HOST_WORKSPACE}" + +# Build commands +""" + +BUILD_SCRIPT_FOOTER = """\ +# End of build commands + +echo "Build completed successfully" +""" + + +# Validated cloudbuild recipe + flags +CloudBuild = collections.namedtuple('CloudBuild', 'output_script run steps') + +# Single validated step in a cloudbuild recipe +Step = collections.namedtuple('Step', 'args dir_ env name') + + +def get_field_value(container, field_name, field_type): + """Fetch a field from a container with typechecking and default values. + + The field value is coerced to the desired type. If the field is + not present, a instance of `field_type` is constructed with no + arguments and used as the default value. + + Args: + container (dict): Object decoded from yaml + field_name (str): Field that should be present in `container` + field_type (type): Expected type for field value + + Returns: + any: Fetched or default value of field + + Raises: + ValueError: if field value cannot be converted to the desired type + """ + try: + value = container[field_name] + except (IndexError, KeyError): + return field_type() + + msg = 'Expected "{}" field to be of type "{}", but found type "{}"' + if not isinstance(value, field_type): + # list('some string') is a successful type cast as far as Python + # is concerned, but doesn't exactly produce the results we want. + # We have a whitelist of conversions we will attempt. + whitelist = ( + (float, str), + (int, str), + (str, float), + (str, int), + (int, float), + ) + if (type(value), field_type) not in whitelist: + raise ValueError(msg.format(field_name, field_type, type(value))) + + try: + value = field_type(value) + except ValueError as e: + e.message = msg.format(field_name, field_type, type(value)) + raise + return value + + +def get_cloudbuild(raw_config, args): + """Read and validate a cloudbuild recipe + + Args: + raw_config (dict): deserialized cloudbuild.yaml + args (argparse.Namespace): ccommand line flags + + Returns: + CloudBuild: valid configuration + """ + if not isinstance(raw_config, dict): + raise ValueError( + 'Expected {} contents to be of type "dict", but found type "{}"'. + format(args.config, type(raw_config))) + raw_steps = get_field_value(raw_config, 'steps', list) + if not raw_steps: + raise ValueError('No steps defined in {}'.format(args.config)) + steps = [get_step(raw_step) for raw_step in raw_steps] + return CloudBuild( + output_script=args.output_script, + run=args.run, + steps=steps, + ) + + +def get_step(raw_step): + """Read and validate a single cloudbuild step + + Args: + raw_step (dict): deserialized step + + Returns: + Step: valid build step + """ + if not isinstance(raw_step, dict): + raise ValueError( + 'Expected step to be of type "dict", but found type "{}"'. + format(type(raw_step))) + raw_args = get_field_value(raw_step, 'args', list) + args = [get_field_value(raw_args, i, str) + for i in range(len(raw_args))] + dir_ = get_field_value(raw_step, 'dir', str) + raw_env = get_field_value(raw_step, 'env', list) + env = [get_field_value(raw_env, i, str) + for i in range(len(raw_env))] + name = get_field_value(raw_step, 'name', str) + return Step( + args=args, + dir_=dir_, + env=env, + name=name, + ) + + +def generate_command(step): + """Generate a single shell command to run for a single cloudbuild step + + Args: + step (Step): Valid build step + + Returns: + [str]: A single shell command, expressed as a list of quoted tokens. + """ + quoted_args = [shlex.quote(arg) for arg in step.args] + quoted_env = [] + for env in step.env: + quoted_env.extend(['--env', shlex.quote(env)]) + quoted_name = shlex.quote(step.name) + workdir = '/workspace' + if step.dir_: + workdir = os.path.join(workdir, shlex.quote(step.dir_)) + process_args = [ + 'docker', + 'run', + '--volume', + '/var/run/docker.sock:/var/run/docker.sock', + '--volume', + '/root/.docker:/root/.docker', + '--volume', + '${HOST_WORKSPACE}:/workspace', + '--workdir', + workdir, + ] + quoted_env + [quoted_name] + quoted_args + return process_args + + +def generate_script(cloudbuild): + """Generate the contents of a shell script + + Args: + cloudbuild (CloudBuild): Valid cloudbuild configuration + + Returns: + (str): Contents of shell script + """ + outfile = io.StringIO() + outfile.write(BUILD_SCRIPT_HEADER) + docker_commands = [generate_command(step) for step in cloudbuild.steps] + for docker_command in docker_commands: + line = ' '.join(docker_command) + '\n\n' + outfile.write(line) + outfile.write(BUILD_SCRIPT_FOOTER) + s = outfile.getvalue() + outfile.close() + return s + + +def make_executable(path): + """Set executable bit(s) on file""" + # http://stackoverflow.com/questions/12791997 + mode = os.stat(path).st_mode + mode |= (mode & 0o444) >> 2 # copy R bits to X + os.chmod(path, mode) + + +def write_script(cloudbuild, contents): + """Write a shell script to a file.""" + print('Writing build script to {}'.format(cloudbuild.output_script)) + with open(cloudbuild.output_script, 'w', encoding='utf8') as outfile: + outfile.write(contents) + make_executable(cloudbuild.output_script) + + +def local_cloudbuild(args): + """Execute the steps of a cloudbuild.yaml locally + + Args: + args: command line flags as per parse_args + """ + # Load and parse cloudbuild.yaml + with open(args.config, 'r', encoding='utf8') as cloudbuild_file: + raw_config = yaml.safe_load(cloudbuild_file) + + # Determine configuration + cloudbuild = get_cloudbuild(raw_config, args) + + # Create shell script + contents = generate_script(cloudbuild) + write_script(cloudbuild, contents) + + # Run shell script + if cloudbuild.run: + args = [os.path.abspath(cloudbuild.output_script)] + subprocess.check_call(args) + + +def validate_arg_regex(flag_value, flag_regex): + """Check a named command line flag against a regular expression""" + if not re.match(flag_regex, flag_value): + raise argparse.ArgumentTypeError( + 'Value "{}" does not match pattern "{}"'.format( + flag_value, flag_regex.pattern)) + return flag_value + + +def parse_args(argv): + """Parse and validate command line flags""" + parser = argparse.ArgumentParser( + description='Process cloudbuild.yaml locally to build Docker images') + parser.add_argument( + '--config', + type=functools.partial( + validate_arg_regex, flag_regex=PRINTABLE_REGEX), + default='cloudbuild.yaml', + help='Path to cloudbuild.yaml file' + ) + parser.add_argument( + '--output_script', + type=functools.partial( + validate_arg_regex, flag_regex=PRINTABLE_REGEX), + help='Filename to write shell script to', + ) + parser.add_argument( + '--no-run', + action='store_false', + help='Create shell script but don\'t execute it', + dest='run', + ) + args = parser.parse_args(argv[1:]) + if not args.output_script: + args.output_script = args.config + "_local.sh" + return args + + +def main(): + args = parse_args(sys.argv) + local_cloudbuild(args) + + +if __name__ == '__main__': + main() diff --git a/scripts/local_cloudbuild_test.py b/scripts/local_cloudbuild_test.py new file mode 100755 index 00000000..481e5ae9 --- /dev/null +++ b/scripts/local_cloudbuild_test.py @@ -0,0 +1,322 @@ +#!/usr/bin/env python3 + +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit test for local_cloudbuild.py""" + +import argparse +import os +import re +import shutil +import subprocess +import tempfile +import unittest +import unittest.mock + +import yaml + +import local_cloudbuild + + +class ValidationUtilsTest(unittest.TestCase): + + def test_get_field_value(self): + valid_cases = ( + # Normal case, field present and correct type + ({ 'present': 1 }, 'present', int, 1), + ({ 'present': '1' }, 'present', str, '1'), + ({ 'present': [1] }, 'present', list, [1]), + ({ 'present': {1: 2} }, 'present', dict, {1: 2}), + # Missing field replaced by default + ({}, 'missing', str, ''), + # Valid conversions + ({ 'str_to_int': '1' }, 'str_to_int', int, 1), + ({ 'int_to_str': 1 }, 'int_to_str', str, '1'), + ) + for valid_case in valid_cases: + with self.subTest(valid_case=valid_case): + container, field_name, field_type, expected = valid_case + self.assertEqual( + local_cloudbuild.get_field_value( + container, field_name, field_type), + expected) + + invalid_cases = ( + # Type conversion failures + ({ 'bad_list_to_dict': [1] }, 'bad_list_to_dict', dict), + ({ 'bad_list_to_str': [1] }, 'bad_list_to_str', str), + ({ 'bad_dict_to_list': {1: 2} }, 'bad_dict_to_list', list), + ({ 'bad_str_to_int': 'not_an_int' }, 'bad_str_to_int', int), + ({ 'bad_str_to_list': 'abc' }, 'bad_str_to_list', list), + ) + for invalid_case in invalid_cases: + with self.subTest(invalid_case=invalid_case): + container, field_name, field_type = invalid_case + with self.assertRaises(ValueError): + local_cloudbuild.get_field_value( + container, field_name, field_type) + + def test_validate_arg_regex(self): + self.assertEqual( + local_cloudbuild.validate_arg_regex('abc', re.compile('a[b]c')), + 'abc') + with self.assertRaises(argparse.ArgumentTypeError): + local_cloudbuild.validate_arg_regex('abc', re.compile('a[d]c')) + + +class LocalCloudbuildTest(unittest.TestCase): + + def setUp(self): + self.testdata_dir = 'testdata' + assert os.path.isdir(self.testdata_dir), 'Could not run test: testdata directory not found' + + def test_get_cloudbuild(self): + args = argparse.Namespace( + config='some_config_file', + output_script='some_output_script', + run=False, + ) + # Basic valid case + valid_case = 'steps:\n- name: step1\n- name: step2\n' + raw_config = yaml.safe_load(valid_case) + actual = local_cloudbuild.get_cloudbuild(raw_config, args) + self.assertEqual(len(actual.steps), 2) + + invalid_cases = ( + # Empty cloud build + '', + # No steps + 'foo: bar\n', + # Steps not a list + 'steps: astring\n', + ) + for invalid_case in invalid_cases: + with self.subTest(invalid_case=invalid_case): + raw_config = yaml.safe_load(invalid_case) + with self.assertRaises(ValueError): + local_cloudbuild.get_cloudbuild(raw_config, args) + + def test_get_step(self): + valid_cases = ( + # Empty step + ({}, local_cloudbuild.Step( + args=[], + dir_='', + env=[], + name='', + )), + # Full step + ({'name' : 'aname', + 'args' : [ 'arg1', 2, 'arg3 with \n newline', ], + 'env' : [ 'ENV1=value1', 'ENV2=space in value2' ], + 'dir' : 'adir', + }, local_cloudbuild.Step( + args = [ 'arg1', '2', 'arg3 with \n newline', ], + env = [ 'ENV1=value1', 'ENV2=space in value2' ], + dir_ = 'adir', + name = 'aname', + )), + ) + for valid_case in valid_cases: + with self.subTest(valid_case=valid_case): + raw_step, expected = valid_case + actual = local_cloudbuild.get_step(raw_step) + self.assertEqual(actual, expected) + + invalid_cases = ( + # Wrong type + [], + # More wrong types + {'args': 'not_a_list'}, + {'args': [ [] ]}, + {'env': 'not_a_list'}, + {'env': [ {} ]}, + {'dir': {}}, + {'name': []}, + ) + for invalid_case in invalid_cases: + with self.subTest(invalid_case=invalid_case): + with self.assertRaises(ValueError): + local_cloudbuild.get_step(invalid_case) + + def test_generate_command(self): + # Basic valid case + base_step = local_cloudbuild.Step( + args = ['arg1','arg2'], + dir_ = '', + env = ['ENV1=value1', 'ENV2=value2'], + name = 'aname', + ) + command = local_cloudbuild.generate_command(base_step) + self.assertEqual(command, [ + 'docker', + 'run', + '--volume', + '/var/run/docker.sock:/var/run/docker.sock', + '--volume', + '/root/.docker:/root/.docker', + '--volume', + '${HOST_WORKSPACE}:/workspace', + '--workdir', + '/workspace', + '--env', + 'ENV1=value1', + '--env', + 'ENV2=value2', + 'aname', + 'arg1', + 'arg2', + ]) + + # dir specified + step = base_step._replace(dir_='adir') + command = local_cloudbuild.generate_command(step) + self.assertIn('--workdir', command) + self.assertIn('/workspace/adir', command) + + # Shell quoting + step = base_step._replace(args=['arg with \n newline']) + command = local_cloudbuild.generate_command(step) + self.assertIn("'arg with \n newline'", command) + + step = base_step._replace(dir_='dir/ with space/') + command = local_cloudbuild.generate_command(step) + self.assertIn("/workspace/'dir/ with space/'", command) + + step = base_step._replace(env=['env with space']) + command = local_cloudbuild.generate_command(step) + self.assertIn("'env with space'", command) + + step = base_step._replace(name='a name') + command = local_cloudbuild.generate_command(step) + self.assertIn("'a name'", command) + + def test_generate_script(self): + config_name = 'cloudbuild_ok.yaml' + config = os.path.join(self.testdata_dir, config_name) + expected_output_script = os.path.join(self.testdata_dir, config_name + '_golden.sh') + cloudbuild = local_cloudbuild.CloudBuild( + output_script='test_generate_script', + run=False, + steps=[ + local_cloudbuild.Step( + args=['/bin/sh', '-c', 'echo "${MESSAGE}"'], + dir_='', + env=['MESSAGE=Hello World!'], + name='debian', + ), + local_cloudbuild.Step( + args=['/bin/sh', '-c', 'echo "${MESSAGE}"'], + dir_='', + env=['MESSAGE=Goodbye\\n And Farewell!', 'UNUSED=unused'], + name='debian', + ) + ] + ) + actual = local_cloudbuild.generate_script(cloudbuild) + self.maxDiff = 2**16 + # Compare output against golden + with open(expected_output_script, 'r', encoding='utf8') as expected: + self.assertEqual(actual, expected.read()) + + def test_make_executable(self): + with tempfile.TemporaryDirectory( + prefix='local_cloudbuild_test_') as tempdir: + test_script_filename = os.path.join(tempdir, 'test_make_executable.sh') + with open(test_script_filename, 'w', encoding='utf8') as test_script: + test_script.write('#!/bin/sh\necho "Output from test_make_executable"') + local_cloudbuild.make_executable(test_script_filename) + output = subprocess.check_output([test_script_filename]) + self.assertEqual(output.decode('utf8'), "Output from test_make_executable\n") + + def test_write_script(self): + with tempfile.TemporaryDirectory( + prefix='local_cloudbuild_test_') as tempdir: + contents = 'The contents\n' + output_script_filename = os.path.join(tempdir, 'test_write_script') + cloudbuild = local_cloudbuild.CloudBuild( + output_script=output_script_filename, + run=False, + steps=[], + ) + local_cloudbuild.write_script(cloudbuild, contents) + with open(output_script_filename, 'r', encoding='utf8') as output_script: + actual = output_script.read() + self.assertEqual(actual, contents) + + def test_local_cloudbuild(self): + # Actually run it if we can find a docker command. + should_run = False + if ((shutil.which('docker') is not None) and + (subprocess.call(['docker', 'info'], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) == 0)): + should_run = True + + # Read cloudbuild.yaml from testdata file, write output to + # tempdir, and maybe try to run it + with tempfile.TemporaryDirectory( + prefix='local_cloudbuild_test_') as tempdir: + cases = ( + # Everything is ok + ('cloudbuild_ok.yaml', True), + # Exit code 1 (failure) + ('cloudbuild_err_rc1.yaml', False), + # Command not found + ('cloudbuild_err_not_found.yaml', False), + ) + for case in cases: + with self.subTest(case=cases): + config_name, should_succeed = case + config = os.path.join(self.testdata_dir, config_name) + actual_output_script = os.path.join( + tempdir, config_name + '_local.sh') + args = argparse.Namespace( + config=config, + output_script=actual_output_script, + run=should_run) + if should_run: + print("Executing docker commands in {}".format(actual_output_script)) + if should_succeed: + local_cloudbuild.local_cloudbuild(args) + else: + with self.assertRaises(subprocess.CalledProcessError): + local_cloudbuild.local_cloudbuild(args) + else: + # Generate but don't execute script + local_cloudbuild.local_cloudbuild(args) + + + def test_parse_args(self): + # Test explicit output_script + argv = ['argv0', '--output_script=my_output'] + args = local_cloudbuild.parse_args(argv) + self.assertEqual(args.output_script, 'my_output') + # Test implicit output_script + argv = ['argv0', '--config=my_config'] + args = local_cloudbuild.parse_args(argv) + self.assertEqual(args.output_script, 'my_config_local.sh') + + # Test run flag (default and --no-run) + argv = ['argv0'] + args = local_cloudbuild.parse_args(argv) + self.assertEqual(args.run, True) + argv = ['argv0', '--no-run'] + args = local_cloudbuild.parse_args(argv) + self.assertEqual(args.run, False) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/testdata/cloudbuild_err_not_found.yaml b/scripts/testdata/cloudbuild_err_not_found.yaml new file mode 100644 index 00000000..c7eb070d --- /dev/null +++ b/scripts/testdata/cloudbuild_err_not_found.yaml @@ -0,0 +1,3 @@ +steps: +- name: debian + args: ['/expected file not found'] diff --git a/scripts/testdata/cloudbuild_err_rc1.yaml b/scripts/testdata/cloudbuild_err_rc1.yaml new file mode 100644 index 00000000..3953a586 --- /dev/null +++ b/scripts/testdata/cloudbuild_err_rc1.yaml @@ -0,0 +1,3 @@ +steps: +- name: debian + args: ['/bin/sh', '-c', 'exit 1'] diff --git a/scripts/testdata/cloudbuild_ok.yaml b/scripts/testdata/cloudbuild_ok.yaml new file mode 100644 index 00000000..8478269b --- /dev/null +++ b/scripts/testdata/cloudbuild_ok.yaml @@ -0,0 +1,7 @@ +steps: +- name: debian + args: ['/bin/sh', '-c', 'echo "${MESSAGE}"'] + env: ['MESSAGE=Hello World!'] +- name: debian + args: ['/bin/sh', '-c', 'echo "${MESSAGE}"'] + env: ['MESSAGE=Goodbye\n And Farewell!', 'UNUSED=unused'] diff --git a/scripts/testdata/cloudbuild_ok.yaml_golden.sh b/scripts/testdata/cloudbuild_ok.yaml_golden.sh new file mode 100755 index 00000000..996c436e --- /dev/null +++ b/scripts/testdata/cloudbuild_ok.yaml_golden.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# This is a generated file. Do not edit. + +set -euo pipefail + +SOURCE_DIR=. + +# Setup staging directory +HOST_WORKSPACE=$(mktemp -d) +function cleanup { + if [ "${HOST_WORKSPACE}" != '/' -a -d "${HOST_WORKSPACE}" ]; then + rm -rf "${HOST_WORKSPACE}" + fi +} +trap cleanup EXIT + +# Copy source to staging directory +echo "Copying source to staging directory ${HOST_WORKSPACE}" +rsync -avzq --exclude=.git "${SOURCE_DIR}" "${HOST_WORKSPACE}" + +# Build commands +docker run --volume /var/run/docker.sock:/var/run/docker.sock --volume /root/.docker:/root/.docker --volume ${HOST_WORKSPACE}:/workspace --workdir /workspace --env 'MESSAGE=Hello World!' debian /bin/sh -c 'echo "${MESSAGE}"' + +docker run --volume /var/run/docker.sock:/var/run/docker.sock --volume /root/.docker:/root/.docker --volume ${HOST_WORKSPACE}:/workspace --workdir /workspace --env 'MESSAGE=Goodbye\n And Farewell!' --env UNUSED=unused debian /bin/sh -c 'echo "${MESSAGE}"' + +# End of build commands + +echo "Build completed successfully" From 5b68d6b903a41057d8b8a2ccaeab1abd54933416 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 17 Feb 2017 11:01:12 -0800 Subject: [PATCH 116/362] Style fixes --- scripts/local_cloudbuild.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/local_cloudbuild.py b/scripts/local_cloudbuild.py index 53ae86e0..d67d5727 100755 --- a/scripts/local_cloudbuild.py +++ b/scripts/local_cloudbuild.py @@ -96,7 +96,7 @@ def get_field_value(container, field_name, field_type): field_type (type): Expected type for field value Returns: - any: Fetched or default value of field + Any: Fetched or default value of field Raises: ValueError: if field value cannot be converted to the desired type @@ -143,9 +143,11 @@ def get_cloudbuild(raw_config, args): raise ValueError( 'Expected {} contents to be of type "dict", but found type "{}"'. format(args.config, type(raw_config))) + raw_steps = get_field_value(raw_config, 'steps', list) if not raw_steps: raise ValueError('No steps defined in {}'.format(args.config)) + steps = [get_step(raw_step) for raw_step in raw_steps] return CloudBuild( output_script=args.output_script, @@ -180,7 +182,7 @@ def get_step(raw_step): dir_=dir_, env=env, name=name, - ) + ) def generate_command(step): From 3437eac89095d3280a3fcde2c87b6dfc85b5d0d3 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 17 Feb 2017 11:19:07 -0800 Subject: [PATCH 117/362] Fix style nits --- scripts/local_cloudbuild.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/local_cloudbuild.py b/scripts/local_cloudbuild.py index d67d5727..f7904570 100755 --- a/scripts/local_cloudbuild.py +++ b/scripts/local_cloudbuild.py @@ -170,12 +170,12 @@ def get_step(raw_step): 'Expected step to be of type "dict", but found type "{}"'. format(type(raw_step))) raw_args = get_field_value(raw_step, 'args', list) - args = [get_field_value(raw_args, i, str) - for i in range(len(raw_args))] + args = [get_field_value(raw_args, index, str) + for index in range(len(raw_args))] dir_ = get_field_value(raw_step, 'dir', str) raw_env = get_field_value(raw_step, 'env', list) - env = [get_field_value(raw_env, i, str) - for i in range(len(raw_env))] + env = [get_field_value(raw_env, index, str) + for index in range(len(raw_env))] name = get_field_value(raw_step, 'name', str) return Step( args=args, From e44b310c2f2b20c2137d454f458868c18a8b51a5 Mon Sep 17 00:00:00 2001 From: Nick Kubala Date: Thu, 2 Mar 2017 13:58:20 -0800 Subject: [PATCH 118/362] add support for logging at an explicit level passed from the client. update requirement versions --- tests/integration/requirements.txt | 8 ++++---- tests/integration/server.py | 21 ++++++++++----------- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index 369de14a..dbd26d30 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -13,9 +13,9 @@ # limitations under the License. Flask==0.11.1 -google-cloud-logging==0.21.0 -google-cloud-monitoring==0.21.0 -google-cloud-error-reporting==0.21.0 +google-cloud-logging==0.22.0 +google-cloud-monitoring==0.22.0 +google-cloud-error-reporting==0.22.0 gunicorn==19.6.0 retrying==1.3.3 -requests==2.2.1 +requests==2.13.0 diff --git a/tests/integration/server.py b/tests/integration/server.py index 528ebefa..8083cbf7 100755 --- a/tests/integration/server.py +++ b/tests/integration/server.py @@ -52,13 +52,16 @@ def _logging(request_data, token): log_name = request_data.get('log_name') if not log_name: raise ErrorResponse('Please provide log name') + level = request_data.get('level') + if not level: + raise ErrorResponse('Please provide log level') - _log(token, log_name) + _log(token, log_name, level) return 'OK', 200 -def _log(token, log_name='stdout'): +def _log(token, log_name, level): """ Write a log entry to Stackdriver. @@ -66,22 +69,18 @@ def _log(token, log_name='stdout'): token -- 16-character (8-byte) hexadecimal token, to be written as a log entry. log_name -- The name of the logging group to be written to. + level -- enum(LogSeverity), level of the log to write Once the entry is written to Stackdriver, the test driver will retrieve - all entries with the name 'log_name', and verify there is an entry with - the same value as 'token', indicating the entry was written successfully. + all entries with the name 'log_name' at level 'level', and verify there + is an entry with the same value as 'token', indicating the entry + was written successfully. """ - # TODO (nkubala): just as a note, currently the client logging API is - # broken - - # TODO (nkubala): write token to 'log_name' log, instead of stdout - # is this possible in non-standard (flex)??? - try: client = google.cloud.logging.Client() gcloud_logger = client.logger(log_name) - gcloud_logger.log_text(token) + gcloud_logger.log_text(token, severity=str(level)) except google.cloud.exceptions.GoogleCloudError as e: logging.error('Error while writing logs: {0}'.format(e)) raise ErrorResponse('Error while writing logs: {0}'.format(e)) From 87a55c3824eda4460efaf84b47eda90f335e3807 Mon Sep 17 00:00:00 2001 From: slhawkins Date: Tue, 7 Mar 2017 03:11:17 -0600 Subject: [PATCH 119/362] Update README.md Correction to the docker command that opens an interactive shell session. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4a35d047..3cef3315 100644 --- a/README.md +++ b/README.md @@ -102,7 +102,7 @@ $ make local-test To open an interactive shell session to this image after building it, do the following: ``` shell -docker run --it --entrypoint /bin/bash ${IMAGE_NAME} +docker run -it --entrypoint /bin/bash ${IMAGE_NAME} ``` ## Running the system tests From 03dd7e65df4babc4ffea32d6ebca61e764b7e710 Mon Sep 17 00:00:00 2001 From: Nick Kubala Date: Thu, 9 Mar 2017 14:51:09 -0800 Subject: [PATCH 120/362] add support for testing logging through standard module and custom client --- tests/integration/server.py | 74 +++++++++++++++++++++++++++++++------ 1 file changed, 62 insertions(+), 12 deletions(-) diff --git a/tests/integration/server.py b/tests/integration/server.py index 8083cbf7..eaeb992c 100755 --- a/tests/integration/server.py +++ b/tests/integration/server.py @@ -1,6 +1,6 @@ #!/usr/bin/python -# Copyright 2016 Google Inc. All rights reserved. +# Copyright 2017 Google Inc. All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -24,6 +24,19 @@ from flask import Flask, request, jsonify +# set up logging module to write to Stackdriver +client = google.cloud.logging.Client() +client.setup_logging(log_level=logging.DEBUG) +logging.getLogger().setLevel(logging.DEBUG) + +log_funcs = { + 'DEBUG': (logging.debug, 'stderr'), + 'INFO': (logging.info, 'stderr'), + 'WARNING': (logging.warn, 'stderr'), + 'ERROR': (logging.error, 'stderr'), + 'CRITICAL': (logging.critical, 'stderr') +} + app = Flask(__name__) @@ -46,9 +59,9 @@ def hello_world(): return 'Hello World!' -@app.route('/logging', methods=['POST']) +@app.route('/logging_custom', methods=['POST']) @verify_request -def _logging(request_data, token): +def _logging_custom(request_data, token): log_name = request_data.get('log_name') if not log_name: raise ErrorResponse('Please provide log name') @@ -56,14 +69,26 @@ def _logging(request_data, token): if not level: raise ErrorResponse('Please provide log level') - _log(token, log_name, level) + log_source = _log_custom(token, log_name, level) + + return log_source, 200 - return 'OK', 200 +@app.route('/logging_standard', methods=['POST']) +@verify_request +def _logging_standard(request_data, token): + level = request_data.get('level') + if not level: + raise ErrorResponse('Please provide log level') -def _log(token, log_name, level): + log_source = _log_default(token, level) + + return log_source, 200 + + +def _log_custom(token, log_name, level): """ - Write a log entry to Stackdriver. + Write a custom log entry to Stackdriver using a client library. Keyword arguments: token -- 16-character (8-byte) hexadecimal token, to be written @@ -72,11 +97,10 @@ def _log(token, log_name, level): level -- enum(LogSeverity), level of the log to write Once the entry is written to Stackdriver, the test driver will retrieve - all entries with the name 'log_name' at level 'level', and verify there - is an entry with the same value as 'token', indicating the entry + all entries with the name 'log_name' at level 'level', and verify there + is an entry with the same value as 'token', indicating the entry was written successfully. """ - try: client = google.cloud.logging.Client() gcloud_logger = client.logger(log_name) @@ -85,8 +109,34 @@ def _log(token, log_name, level): logging.error('Error while writing logs: {0}'.format(e)) raise ErrorResponse('Error while writing logs: {0}'.format(e)) - logging.debug(token) - print(token) + return log_name + + +def _log_default(token, level): + """ + Write a log entry to Stackdriver through the default logging module. + + Keyword arguments: + token -- 16-character (8-byte) hexadecimal token, to be written + as a log entry. + level -- enum(LogSeverity), level of the log to write + + Once the entry is written to Stackdriver, the test driver will retrieve + all entries from the default log stream (sent back to the driver) at level + 'level', and verify there is an entry with the same value as 'token', + indicating the entry was written successfully. + """ + + try: + func_pair = log_funcs[level] + f = func_pair[0] + source = func_pair[1] + f(token) + except google.cloud.exceptions.GoogleCloudError as e: + logging.error('Error while writing logs: {0}'.format(e)) + raise ErrorResponse('Error while writing logs: {0}'.format(e)) + + return 'appengine.googleapis.com%2F{0}'.format(source) @app.route('/monitoring', methods=['POST']) From 4dae3bb3a92a7f0c3b9397dad9b7b53458f24d27 Mon Sep 17 00:00:00 2001 From: Nick Kubala Date: Tue, 14 Mar 2017 14:19:44 -0700 Subject: [PATCH 121/362] move client setup code to __name__ check block --- tests/integration/server.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/integration/server.py b/tests/integration/server.py index eaeb992c..ff675046 100755 --- a/tests/integration/server.py +++ b/tests/integration/server.py @@ -24,11 +24,6 @@ from flask import Flask, request, jsonify -# set up logging module to write to Stackdriver -client = google.cloud.logging.Client() -client.setup_logging(log_level=logging.DEBUG) -logging.getLogger().setLevel(logging.DEBUG) - log_funcs = { 'DEBUG': (logging.debug, 'stderr'), 'INFO': (logging.info, 'stderr'), @@ -247,4 +242,8 @@ def handle_invalid_usage(error): if __name__ == '__main__': + # set up logging module to write to Stackdriver + client = google.cloud.logging.Client() + client.setup_logging(log_level=logging.DEBUG) + logging.getLogger().setLevel(logging.DEBUG) app.run(debug=True, port=8080) From 895dc4540b248a78fc8af6f087f14dc6369753d7 Mon Sep 17 00:00:00 2001 From: Nick Kubala Date: Tue, 14 Mar 2017 14:02:44 -0700 Subject: [PATCH 122/362] add sample custom integration test --- tests/integration/server.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/integration/server.py b/tests/integration/server.py index ff675046..cc82fb93 100755 --- a/tests/integration/server.py +++ b/tests/integration/server.py @@ -15,6 +15,7 @@ # limitations under the License. from functools import wraps +import json import logging import google.cloud.logging @@ -218,6 +219,18 @@ def _trace(): return 'OK', 204 +@app.route('/custom', methods=['GET']) +def _custom(): + tests = [ + { + 'name': 'foo', + 'path': '/', + 'timeout': 500 + } + ] + return json.dumps(tests), 200 + + class ErrorResponse(Exception): status_code = 400 From 2ed27358d02f78175cdb258dda61fdf16ce388c1 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Mon, 6 Mar 2017 18:22:31 -0800 Subject: [PATCH 123/362] Support the --substitutions flag as per 'gcloud container builds submit' --- scripts/local_cloudbuild.py | 92 ++++++++++- scripts/local_cloudbuild_test.py | 151 +++++++++++++++--- .../cloudbuild_builtin_substitutions.yaml | 11 ++ scripts/testdata/cloudbuild_ok.yaml | 4 +- scripts/testdata/cloudbuild_ok.yaml_golden.sh | 4 +- .../cloudbuild_user_substitutions.yaml | 11 ++ 6 files changed, 239 insertions(+), 34 deletions(-) create mode 100644 scripts/testdata/cloudbuild_builtin_substitutions.yaml create mode 100644 scripts/testdata/cloudbuild_user_substitutions.yaml diff --git a/scripts/local_cloudbuild.py b/scripts/local_cloudbuild.py index f7904570..78759ffb 100755 --- a/scripts/local_cloudbuild.py +++ b/scripts/local_cloudbuild.py @@ -21,7 +21,9 @@ commands are output as a shell script and optionally executed. The output images are not pushed to the Google Container Registry. -Not all cloudbuild.yaml functionality is supported. +Not all cloudbuild.yaml functionality is supported. In particular, +substitutions are a simplified subset that doesn't include all the +corner cases and error conditions. See https://cloud.google.com/container-builder/docs/api/build-steps for more information. @@ -44,6 +46,28 @@ # Exclude non-printable control characters (including newlines) PRINTABLE_REGEX = re.compile(r"""^[^\x00-\x1f]*$""") +# Container Builder substitutions +# https://cloud.google.com/container-builder/docs/api/build-requests#substitutions +SUBSTITUTION_REGEX = re.compile(r"""(?x) + (? Date: Tue, 7 Mar 2017 17:10:48 -0800 Subject: [PATCH 124/362] Add BUILD_ID and COMMIT_SHA builtin substitutions. --- scripts/local_cloudbuild.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/local_cloudbuild.py b/scripts/local_cloudbuild.py index 78759ffb..7bdd4fda 100755 --- a/scripts/local_cloudbuild.py +++ b/scripts/local_cloudbuild.py @@ -61,11 +61,13 @@ # Default builtin substitutions DEFAULT_SUBSTITUTIONS = { + 'BRANCH_NAME': '', + 'BUILD_ID': 'abcdef12-3456-7890-abcd-ef0123456789', + 'COMMIT_SHA': '', 'PROJECT_ID': 'dummy-project-id', 'REPO_NAME': '', - 'BRANCH_NAME': '', - 'TAG_NAME': '', 'REVISION_ID': '', + 'TAG_NAME': '', } # File template From e6dfe0cd53edfbf679546f5d15de7c5ce27c423f Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Tue, 7 Mar 2017 17:21:40 -0800 Subject: [PATCH 125/362] Substitition names must begin with a letter or underscore. --- scripts/local_cloudbuild.py | 13 ++++++++----- scripts/local_cloudbuild_test.py | 5 +++++ 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/scripts/local_cloudbuild.py b/scripts/local_cloudbuild.py index 7bdd4fda..08b9558e 100755 --- a/scripts/local_cloudbuild.py +++ b/scripts/local_cloudbuild.py @@ -49,15 +49,18 @@ # Container Builder substitutions # https://cloud.google.com/container-builder/docs/api/build-requests#substitutions SUBSTITUTION_REGEX = re.compile(r"""(?x) - (? Date: Tue, 7 Mar 2017 18:30:10 -0800 Subject: [PATCH 126/362] Validate that all user substitutions are used at least once --- scripts/local_cloudbuild.py | 58 +++++++++++++++++-------- scripts/local_cloudbuild_test.py | 74 +++++++++++++++++++------------- 2 files changed, 86 insertions(+), 46 deletions(-) diff --git a/scripts/local_cloudbuild.py b/scripts/local_cloudbuild.py index 08b9558e..21908543 100755 --- a/scripts/local_cloudbuild.py +++ b/scripts/local_cloudbuild.py @@ -112,8 +112,15 @@ Step = collections.namedtuple('Step', 'args dir_ env name') -def sub_and_quote(s, substitutions): - """Return a shell-escaped, variable substituted, version of the string s.""" +def sub_and_quote(s, substitutions, substitutions_used): + """Return a shell-escaped, variable substituted, version of the string s. + + Args: + s (str): Any string + subs (dict): Substitution map to apply + subs_used (set): Updated with names from `subs.keys()` when those + substitutions are encountered in `s` + """ def sub(match): """Perform a single substitution.""" @@ -133,6 +140,7 @@ def sub(match): value = '' else: value = substitutions.get(variable_name) + substitutions_used.add(variable_name) return value substituted_s = re.sub(SUBSTITUTION_REGEX, sub, s) @@ -242,24 +250,29 @@ def get_step(raw_step): ) -def generate_command(step, subs): +def generate_command(step, substitutions, substitutions_used): """Generate a single shell command to run for a single cloudbuild step Args: step (Step): Valid build step subs (dict): Substitution map to apply + subs_used (set): Updated with names from `subs.keys()` when those + substitutions are encountered in an element of `step` Returns: [str]: A single shell command, expressed as a list of quoted tokens. """ - quoted_args = [sub_and_quote(arg, subs) for arg in step.args] + quoted_args = [sub_and_quote(arg, substitutions, substitutions_used) + for arg in step.args] quoted_env = [] for env in step.env: - quoted_env.extend(['--env', sub_and_quote(env, subs)]) - quoted_name = sub_and_quote(step.name, subs) + quoted_env.extend(['--env', sub_and_quote(env, substitutions, + substitutions_used)]) + quoted_name = sub_and_quote(step.name, substitutions, substitutions_used) workdir = '/workspace' if step.dir_: - workdir = os.path.join(workdir, sub_and_quote(step.dir_, subs)) + workdir = os.path.join(workdir, sub_and_quote(step.dir_, substitutions, + substitutions_used)) process_args = [ 'docker', 'run', @@ -284,16 +297,27 @@ def generate_script(cloudbuild): Returns: (str): Contents of shell script """ - outfile = io.StringIO() - outfile.write(BUILD_SCRIPT_HEADER) - docker_commands = [generate_command(step, cloudbuild.substitutions) - for step in cloudbuild.steps] - for docker_command in docker_commands: - line = ' '.join(docker_command) + '\n\n' - outfile.write(line) - outfile.write(BUILD_SCRIPT_FOOTER) - s = outfile.getvalue() - outfile.close() + with io.StringIO() as outfile: + outfile.write(BUILD_SCRIPT_HEADER) + subs_used = set() + docker_commands = [ + generate_command(step, cloudbuild.substitutions, subs_used) + for step in cloudbuild.steps] + for docker_command in docker_commands: + line = ' '.join(docker_command) + '\n\n' + outfile.write(line) + outfile.write(BUILD_SCRIPT_FOOTER) + s = outfile.getvalue() + + # Check that all user variables were referenced at least once + user_subs_unused = [name for name in cloudbuild.substitutions.keys() + if name not in subs_used and name[0] == '_'] + if user_subs_unused: + nice_list = '"' + '", "'.join(sorted(user_subs_unused)) + '"' + raise ValueError( + 'User substitution variables {} were defined in the --substitution ' + 'flag but never used in the cloudbuild file.'.format(nice_list)) + return s diff --git a/scripts/local_cloudbuild_test.py b/scripts/local_cloudbuild_test.py index abb47da9..f6138fbe 100755 --- a/scripts/local_cloudbuild_test.py +++ b/scripts/local_cloudbuild_test.py @@ -120,32 +120,37 @@ def setUp(self): def test_sub_and_quote(self): valid_cases = ( # Empty string - ('', {}, "''"), + ('', {}, "''", []), # No substitutions - ('a', {}, 'a'), - # Unused substitutions - ('a', {'FOO':'foo'}, 'a'), + ('a', {}, 'a', []), + # Unused builtin substitutions are fine + ('a', {'FOO':'foo'}, 'a', []), + # Unused user substitition (ok here but error in generate_script) + ('a', {'_FOO':'_foo'}, 'a', []), # Defined builtin substitution - ('a$FOOb', {'FOO':'foo'}, 'afoob'), - ('a${FOO}b', {'FOO':'foo'}, 'afoob'), + ('a$FOOb', {'FOO':'foo'}, 'afoob', ['FOO']), + ('a${FOO}b', {'FOO':'foo'}, 'afoob', ['FOO']), # Undefined builtin substitution - ('a$FOOb', {}, 'ab'), - ('a${FOO}b', {}, 'ab'), + ('a$FOOb', {}, 'ab', ['FOO']), + ('a${FOO}b', {}, 'ab', ['FOO']), # Defined user substitution - ('a$_FOOb', {'_FOO':'_foo'}, 'a_foob'), - ('a${_FOO}b', {'_FOO':'_foo'}, 'a_foob'), + ('a$_FOOb', {'_FOO':'_foo'}, 'a_foob', ['_FOO']), + ('a${_FOO}b', {'_FOO':'_foo'}, 'a_foob', ['_FOO']), # Multiple substitutions - ('$FOO${FOO}${BAR}$FOO', {'FOO':'foo', 'BAR':'bar'}, 'foofoobarfoo'), + ('$FOO${FOO}${BAR}$FOO', {'FOO':'foo', 'BAR':'bar'}, + 'foofoobarfoo', ['FOO', 'BAR']), # Invalid names - ('a $ b', {}, "'a $ b'"), - ('a$foo b', {}, "'a$foo b'"), - ('a$0FOO b', {}, "'a$0FOO b'"), + ('a $ b', {}, "'a $ b'", []), + ('a$foo b', {}, "'a$foo b'", []), + ('a$0FOO b', {}, "'a$0FOO b'", []), ) for valid_case in valid_cases: with self.subTest(valid_case=valid_case): - s, subs, expected = valid_case - actual = local_cloudbuild.sub_and_quote(s, subs) + s, subs, expected, expected_used = valid_case + used = set() + actual = local_cloudbuild.sub_and_quote(s, subs, used) self.assertEqual(actual, expected) + self.assertEqual(used, set(expected_used)) invalid_cases = ( # Undefined user substitution @@ -156,7 +161,8 @@ def test_sub_and_quote(self): with self.subTest(invalid_case=invalid_case): s, subs = invalid_case with self.assertRaises(ValueError): - local_cloudbuild.sub_and_quote(s, subs) + used = set() + local_cloudbuild.sub_and_quote(s, subs, used) def test_get_cloudbuild(self): args = argparse.Namespace( @@ -237,7 +243,7 @@ def test_generate_command(self): name = 'aname', ) subs = {'BUILTIN':'builtin', '_USER':'_user'} - command = local_cloudbuild.generate_command(base_step, subs) + command = local_cloudbuild.generate_command(base_step, subs, set()) self.assertEqual(command, [ 'docker', 'run', @@ -260,49 +266,49 @@ def test_generate_command(self): # dir specified step = base_step._replace(dir_='adir') - command = local_cloudbuild.generate_command(step, subs) + command = local_cloudbuild.generate_command(step, subs, set()) self.assertIn('--workdir', command) self.assertIn('/workspace/adir', command) # Shell quoting step = base_step._replace(args=['arg with \n newline']) - command = local_cloudbuild.generate_command(step, subs) + command = local_cloudbuild.generate_command(step, subs, set()) self.assertIn("'arg with \n newline'", command) step = base_step._replace(dir_='dir/ with space/') - command = local_cloudbuild.generate_command(step, subs) + command = local_cloudbuild.generate_command(step, subs, set()) self.assertIn("/workspace/'dir/ with space/'", command) step = base_step._replace(env=['env with space']) - command = local_cloudbuild.generate_command(step, subs) + command = local_cloudbuild.generate_command(step, subs, set()) self.assertIn("'env with space'", command) step = base_step._replace(name='a name') - command = local_cloudbuild.generate_command(step, subs) + command = local_cloudbuild.generate_command(step, subs, set()) self.assertIn("'a name'", command) # Variable substitution step = base_step._replace(name='a $BUILTIN substitution') - command = local_cloudbuild.generate_command(step, subs) + command = local_cloudbuild.generate_command(step, subs, set()) self.assertIn("'a builtin substitution'", command) step = base_step._replace(name='a $UNSET_BUILTIN substitution') - command = local_cloudbuild.generate_command(step, subs) + command = local_cloudbuild.generate_command(step, subs, set()) self.assertIn("'a substitution'", command) step = base_step._replace(name='a $_USER substitution') - command = local_cloudbuild.generate_command(step, subs) + command = local_cloudbuild.generate_command(step, subs, set()) self.assertIn("'a _user substitution'", command) step = base_step._replace(name='a $_UNSET_USER substitution') with self.assertRaises(ValueError): - local_cloudbuild.generate_command(step, subs) + local_cloudbuild.generate_command(step, subs, set()) step = base_step._replace(name='a curly brace ${BUILTIN} substitution') - command = local_cloudbuild.generate_command(step, subs) + command = local_cloudbuild.generate_command(step, subs, set()) self.assertIn("'a curly brace builtin substitution'", command) - def test_generate_script(self): + def test_generate_script_golden(self): config_name = 'cloudbuild_ok.yaml' config = os.path.join(self.testdata_dir, config_name) expected_output_script = os.path.join(self.testdata_dir, config_name + '_golden.sh') @@ -331,6 +337,16 @@ def test_generate_script(self): with open(expected_output_script, 'r', encoding='utf8') as expected: self.assertEqual(actual, expected.read()) + def test_generate_script_unused_user_substitution(self): + cloudbuild = local_cloudbuild.CloudBuild( + output_script='', + run=False, + steps=[], + substitutions={'_FOO':'_foo'}, + ) + with self.assertRaisesRegexp(ValueError, 'User substitution variables'): + actual = local_cloudbuild.generate_script(cloudbuild) + def test_make_executable(self): with tempfile.TemporaryDirectory( prefix='local_cloudbuild_test_') as tempdir: From 4ed577487b3444681e50a94548f6317bd98d8dc8 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Wed, 15 Mar 2017 16:52:02 -0700 Subject: [PATCH 127/362] Update cloudbuild substitution rules * Escaping $ is done with $$ not \$ * Undefined builtin substitutions are now an error, instead of being replaced by the empty string --- scripts/local_cloudbuild.py | 23 +++++++++++------------ scripts/local_cloudbuild_test.py | 17 ++++++++++------- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/scripts/local_cloudbuild.py b/scripts/local_cloudbuild.py index 21908543..894be5ac 100755 --- a/scripts/local_cloudbuild.py +++ b/scripts/local_cloudbuild.py @@ -49,12 +49,13 @@ # Container Builder substitutions # https://cloud.google.com/container-builder/docs/api/build-requests#substitutions SUBSTITUTION_REGEX = re.compile(r"""(?x) - (? Date: Wed, 15 Mar 2017 17:37:35 -0700 Subject: [PATCH 128/362] Minor test cleanup --- scripts/local_cloudbuild_test.py | 86 +++++++++++++++----------------- 1 file changed, 39 insertions(+), 47 deletions(-) diff --git a/scripts/local_cloudbuild_test.py b/scripts/local_cloudbuild_test.py index f22001c8..b33065db 100755 --- a/scripts/local_cloudbuild_test.py +++ b/scripts/local_cloudbuild_test.py @@ -263,53 +263,45 @@ def test_generate_command(self): 'arg2', ]) - # dir specified - step = base_step._replace(dir_='adir') - command = local_cloudbuild.generate_command(step, subs, set()) - self.assertIn('--workdir', command) - self.assertIn('/workspace/adir', command) - - # Shell quoting - step = base_step._replace(args=['arg with \n newline']) - command = local_cloudbuild.generate_command(step, subs, set()) - self.assertIn("'arg with \n newline'", command) - - step = base_step._replace(dir_='dir/ with space/') - command = local_cloudbuild.generate_command(step, subs, set()) - self.assertIn("/workspace/'dir/ with space/'", command) - - step = base_step._replace(env=['env with space']) - command = local_cloudbuild.generate_command(step, subs, set()) - self.assertIn("'env with space'", command) - - step = base_step._replace(name='a name') - command = local_cloudbuild.generate_command(step, subs, set()) - self.assertIn("'a name'", command) - - # Variable substitution - step = base_step._replace(name='a $BUILTIN substitution') - command = local_cloudbuild.generate_command(step, subs, set()) - self.assertIn("'a builtin substitution'", command) - - step = base_step._replace(name='a $UNSET_BUILTIN substitution') - with self.assertRaises(ValueError): - command = local_cloudbuild.generate_command(step, subs, set()) - - step = base_step._replace(name='a $_USER substitution') - command = local_cloudbuild.generate_command(step, subs, set()) - self.assertIn("'a _user substitution'", command) - - step = base_step._replace(name='a $_UNSET_USER substitution') - with self.assertRaises(ValueError): - local_cloudbuild.generate_command(step, subs, set()) - - step = base_step._replace(name='a curly brace ${BUILTIN} substitution') - command = local_cloudbuild.generate_command(step, subs, set()) - self.assertIn("'a curly brace builtin substitution'", command) - - step = base_step._replace(name='an escaped $$ or $$$$ or $$BUILTIN or $${BUILTIN} is unescaped') - command = local_cloudbuild.generate_command(step, subs, set()) - self.assertIn("'an escaped $ or $$ or $BUILTIN or ${BUILTIN} is unescaped'", command) + valid_cases = ( + # dir specified + (base_step._replace(dir_='adir'), + ['--workdir', '/workspace/adir']), + # Shell quoting + (base_step._replace(args=['arg with \n newline']), + ["'arg with \n newline'"]), + (base_step._replace(dir_='dir/ with space/'), + ["/workspace/'dir/ with space/'"]), + (base_step._replace(env=['env with space']), + ["'env with space'"]), + (base_step._replace(name='a name'), + ["'a name'"]), + # Variable substitution + (base_step._replace(name='a $BUILTIN substitution'), + ["'a builtin substitution'"]), + (base_step._replace(name='a $_USER substitution'), + ["'a _user substitution'"]), + (base_step._replace(name='a curly brace ${BUILTIN} substitution'), + ["'a curly brace builtin substitution'"]), + (base_step._replace(name='an escaped $$ or $$$$ or $$FOO or $${_FOO} is unescaped'), + ["'an escaped $ or $$ or $FOO or ${_FOO} is unescaped'"]), + ) + for valid_case in valid_cases: + with self.subTest(valid_case=valid_case): + step, args = valid_case + command = local_cloudbuild.generate_command(step, subs, set()) + for arg in args: + self.assertIn(arg, command) + + invalid_cases = ( + base_step._replace(name='a $UNSET_BUILTIN substitution'), + base_step._replace(name='a $_UNSET_USER substitution'), + ) + for invalid_case in invalid_cases: + with self.subTest(invalid_case=invalid_case): + step = invalid_case + with self.assertRaises(ValueError): + local_cloudbuild.generate_command(step, subs, set()) def test_generate_script_golden(self): config_name = 'cloudbuild_ok.yaml' From 428e39f6ddb2017d56f0ea0e712ce9b4a3fdbc48 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Wed, 15 Mar 2017 19:13:47 -0700 Subject: [PATCH 129/362] Fix staging directory cleanup --- scripts/local_cloudbuild.py | 54 ++++++++++++------- scripts/local_cloudbuild_test.py | 31 +++++++---- .../cloudbuild_difficult_cleanup.yaml | 3 ++ scripts/testdata/cloudbuild_ok.yaml_golden.sh | 7 +-- 4 files changed, 64 insertions(+), 31 deletions(-) create mode 100644 scripts/testdata/cloudbuild_difficult_cleanup.yaml diff --git a/scripts/local_cloudbuild.py b/scripts/local_cloudbuild.py index f7904570..12edd125 100755 --- a/scripts/local_cloudbuild.py +++ b/scripts/local_cloudbuild.py @@ -44,8 +44,11 @@ # Exclude non-printable control characters (including newlines) PRINTABLE_REGEX = re.compile(r"""^[^\x00-\x1f]*$""") +# Use this image for cleanup actions +DEBIAN_IMAGE='gcr.io/google-appengine/debian8' + # File template -BUILD_SCRIPT_HEADER = """\ +BUILD_SCRIPT_TEMPLATE = """\ #!/bin/bash # This is a generated file. Do not edit. @@ -54,24 +57,22 @@ SOURCE_DIR=. # Setup staging directory -HOST_WORKSPACE=$(mktemp -d) -function cleanup { - if [ "${HOST_WORKSPACE}" != '/' -a -d "${HOST_WORKSPACE}" ]; then - rm -rf "${HOST_WORKSPACE}" +HOST_WORKSPACE=$(mktemp -d -t local_cloudbuild_XXXXXXXXXX) +function cleanup {{ + if [ "${{HOST_WORKSPACE}}" != '/' -a -d "${{HOST_WORKSPACE}}" ]; then + {cleanup_str} 2>/dev/null || true + rmdir "${{HOST_WORKSPACE}}" fi -} +}} trap cleanup EXIT # Copy source to staging directory -echo "Copying source to staging directory ${HOST_WORKSPACE}" -rsync -avzq --exclude=.git "${SOURCE_DIR}" "${HOST_WORKSPACE}" +echo "Copying source to staging directory ${{HOST_WORKSPACE}}" +rsync -avzq --exclude=.git "${{SOURCE_DIR}}" "${{HOST_WORKSPACE}}" # Build commands -""" - -BUILD_SCRIPT_FOOTER = """\ +{docker_str} # End of build commands - echo "Build completed successfully" """ @@ -226,15 +227,25 @@ def generate_script(cloudbuild): Returns: (str): Contents of shell script """ - outfile = io.StringIO() - outfile.write(BUILD_SCRIPT_HEADER) + # This deletes everything in /workspace including hidden files, + # but not /workspace itself + cleanup_step = Step( + args=['rm', '-rf', '/workspace'], + dir_='', + env=[], + name=DEBIAN_IMAGE, + ) + cleanup_command = generate_command(cleanup_step) + cleanup_str = ' '.join(cleanup_command) docker_commands = [generate_command(step) for step in cloudbuild.steps] + docker_lines = [] for docker_command in docker_commands: line = ' '.join(docker_command) + '\n\n' - outfile.write(line) - outfile.write(BUILD_SCRIPT_FOOTER) - s = outfile.getvalue() - outfile.close() + docker_lines.append(line) + docker_str = ''.join(docker_lines) + + s = BUILD_SCRIPT_TEMPLATE.format(cleanup_str=cleanup_str, + docker_str=docker_str) return s @@ -259,6 +270,9 @@ def local_cloudbuild(args): Args: args: command line flags as per parse_args + + Returns: + str: Output of build, or None if build not run """ # Load and parse cloudbuild.yaml with open(args.config, 'r', encoding='utf8') as cloudbuild_file: @@ -274,7 +288,9 @@ def local_cloudbuild(args): # Run shell script if cloudbuild.run: args = [os.path.abspath(cloudbuild.output_script)] - subprocess.check_call(args) + return subprocess.check_output(args) + else: + return None def validate_arg_regex(flag_value, flag_regex): diff --git a/scripts/local_cloudbuild_test.py b/scripts/local_cloudbuild_test.py index 481e5ae9..0cbb0ecc 100755 --- a/scripts/local_cloudbuild_test.py +++ b/scripts/local_cloudbuild_test.py @@ -82,6 +82,15 @@ def setUp(self): self.testdata_dir = 'testdata' assert os.path.isdir(self.testdata_dir), 'Could not run test: testdata directory not found' + def have_docker(self): + """Determine if the Docker daemon is present and usable""" + if ((shutil.which('docker') is not None) and + (subprocess.call(['docker', 'info'], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) == 0)): + return True + return False + def test_get_cloudbuild(self): args = argparse.Namespace( config='some_config_file', @@ -258,12 +267,7 @@ def test_write_script(self): def test_local_cloudbuild(self): # Actually run it if we can find a docker command. - should_run = False - if ((shutil.which('docker') is not None) and - (subprocess.call(['docker', 'info'], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL) == 0)): - should_run = True + should_run = self.have_docker() # Read cloudbuild.yaml from testdata file, write output to # tempdir, and maybe try to run it @@ -276,9 +280,11 @@ def test_local_cloudbuild(self): ('cloudbuild_err_rc1.yaml', False), # Command not found ('cloudbuild_err_not_found.yaml', False), + # Cleaning up files owned by root + ('cloudbuild_difficult_cleanup.yaml', True), ) for case in cases: - with self.subTest(case=cases): + with self.subTest(case=case): config_name, should_succeed = case config = os.path.join(self.testdata_dir, config_name) actual_output_script = os.path.join( @@ -288,9 +294,16 @@ def test_local_cloudbuild(self): output_script=actual_output_script, run=should_run) if should_run: - print("Executing docker commands in {}".format(actual_output_script)) + print('Executing docker commands in {}'.format(actual_output_script)) if should_succeed: - local_cloudbuild.local_cloudbuild(args) + output = local_cloudbuild.local_cloudbuild(args) + # Check that staging dir was cleaned up + staging_regex = re.compile( + b'(?m)Copying source to staging directory (.+)$') + match = re.search(staging_regex, output) + self.assertTrue(match, output) + staging_dir = match.group(1) + self.assertFalse(os.path.isdir(staging_dir), staging_dir) else: with self.assertRaises(subprocess.CalledProcessError): local_cloudbuild.local_cloudbuild(args) diff --git a/scripts/testdata/cloudbuild_difficult_cleanup.yaml b/scripts/testdata/cloudbuild_difficult_cleanup.yaml new file mode 100644 index 00000000..e76846d5 --- /dev/null +++ b/scripts/testdata/cloudbuild_difficult_cleanup.yaml @@ -0,0 +1,3 @@ +steps: +- name: debian + args: ['/bin/sh', '-c', 'mkdir root; umask 0000; touch root/deny_all.txt'] diff --git a/scripts/testdata/cloudbuild_ok.yaml_golden.sh b/scripts/testdata/cloudbuild_ok.yaml_golden.sh index 996c436e..a6d5d12e 100755 --- a/scripts/testdata/cloudbuild_ok.yaml_golden.sh +++ b/scripts/testdata/cloudbuild_ok.yaml_golden.sh @@ -6,10 +6,11 @@ set -euo pipefail SOURCE_DIR=. # Setup staging directory -HOST_WORKSPACE=$(mktemp -d) +HOST_WORKSPACE=$(mktemp -d -t local_cloudbuild_XXXXXXXXXX) function cleanup { if [ "${HOST_WORKSPACE}" != '/' -a -d "${HOST_WORKSPACE}" ]; then - rm -rf "${HOST_WORKSPACE}" + docker run --volume /var/run/docker.sock:/var/run/docker.sock --volume /root/.docker:/root/.docker --volume ${HOST_WORKSPACE}:/workspace --workdir /workspace gcr.io/google-appengine/debian8 rm -rf /workspace 2>/dev/null || true + rmdir "${HOST_WORKSPACE}" fi } trap cleanup EXIT @@ -23,6 +24,6 @@ docker run --volume /var/run/docker.sock:/var/run/docker.sock --volume /root/.do docker run --volume /var/run/docker.sock:/var/run/docker.sock --volume /root/.docker:/root/.docker --volume ${HOST_WORKSPACE}:/workspace --workdir /workspace --env 'MESSAGE=Goodbye\n And Farewell!' --env UNUSED=unused debian /bin/sh -c 'echo "${MESSAGE}"' -# End of build commands +# End of build commands echo "Build completed successfully" From 42e58f1d4514f65247822c9bba3d1bbccaaf3712 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Thu, 16 Mar 2017 15:18:27 -0700 Subject: [PATCH 130/362] Restore previous behavior of printing docker output. --- scripts/local_cloudbuild.py | 11 +-- scripts/local_cloudbuild_test.py | 91 +++++++++++-------- scripts/testdata/cloudbuild_ok.yaml_golden.sh | 4 +- 3 files changed, 59 insertions(+), 47 deletions(-) diff --git a/scripts/local_cloudbuild.py b/scripts/local_cloudbuild.py index 12edd125..b577aa9d 100755 --- a/scripts/local_cloudbuild.py +++ b/scripts/local_cloudbuild.py @@ -60,8 +60,10 @@ HOST_WORKSPACE=$(mktemp -d -t local_cloudbuild_XXXXXXXXXX) function cleanup {{ if [ "${{HOST_WORKSPACE}}" != '/' -a -d "${{HOST_WORKSPACE}}" ]; then + # Expect a single error message about /workspace busy {cleanup_str} 2>/dev/null || true - rmdir "${{HOST_WORKSPACE}}" + # Do not expect error messages here. Display but ignore any that happen. + rmdir "${{HOST_WORKSPACE}}" || true fi }} trap cleanup EXIT @@ -270,9 +272,6 @@ def local_cloudbuild(args): Args: args: command line flags as per parse_args - - Returns: - str: Output of build, or None if build not run """ # Load and parse cloudbuild.yaml with open(args.config, 'r', encoding='utf8') as cloudbuild_file: @@ -288,9 +287,7 @@ def local_cloudbuild(args): # Run shell script if cloudbuild.run: args = [os.path.abspath(cloudbuild.output_script)] - return subprocess.check_output(args) - else: - return None + subprocess.check_call(args) def validate_arg_regex(flag_value, flag_regex): diff --git a/scripts/local_cloudbuild_test.py b/scripts/local_cloudbuild_test.py index 0cbb0ecc..26ac4c0b 100755 --- a/scripts/local_cloudbuild_test.py +++ b/scripts/local_cloudbuild_test.py @@ -30,6 +30,10 @@ import local_cloudbuild +# Matches script boilerplate +STAGING_DIR_REGEX = re.compile( + b'(?m)Copying source to staging directory (.+)$') + class ValidationUtilsTest(unittest.TestCase): def test_get_field_value(self): @@ -82,6 +86,16 @@ def setUp(self): self.testdata_dir = 'testdata' assert os.path.isdir(self.testdata_dir), 'Could not run test: testdata directory not found' + def check_call_with_capture(self, *args, **kw_args): + """Act like subprocess.check_call but capture stdout""" + try: + self.check_call_output = subprocess.check_output(*args, **kw_args) + print(self.check_call_output) + except subprocess.CalledProcessError as e: + self.check_call_output = e.output + print(self.check_call_output) + raise + def have_docker(self): """Determine if the Docker daemon is present and usable""" if ((shutil.which('docker') is not None) and @@ -271,46 +285,45 @@ def test_local_cloudbuild(self): # Read cloudbuild.yaml from testdata file, write output to # tempdir, and maybe try to run it - with tempfile.TemporaryDirectory( - prefix='local_cloudbuild_test_') as tempdir: - cases = ( - # Everything is ok - ('cloudbuild_ok.yaml', True), - # Exit code 1 (failure) - ('cloudbuild_err_rc1.yaml', False), - # Command not found - ('cloudbuild_err_not_found.yaml', False), - # Cleaning up files owned by root - ('cloudbuild_difficult_cleanup.yaml', True), - ) - for case in cases: - with self.subTest(case=case): - config_name, should_succeed = case - config = os.path.join(self.testdata_dir, config_name) - actual_output_script = os.path.join( - tempdir, config_name + '_local.sh') - args = argparse.Namespace( - config=config, - output_script=actual_output_script, - run=should_run) - if should_run: - print('Executing docker commands in {}'.format(actual_output_script)) - if should_succeed: - output = local_cloudbuild.local_cloudbuild(args) - # Check that staging dir was cleaned up - staging_regex = re.compile( - b'(?m)Copying source to staging directory (.+)$') - match = re.search(staging_regex, output) - self.assertTrue(match, output) - staging_dir = match.group(1) - self.assertFalse(os.path.isdir(staging_dir), staging_dir) - else: - with self.assertRaises(subprocess.CalledProcessError): - local_cloudbuild.local_cloudbuild(args) - else: - # Generate but don't execute script + cases = ( + # Everything is ok + ('cloudbuild_ok.yaml', True), + # Exit code 1 (failure) + ('cloudbuild_err_rc1.yaml', False), + # Command not found + ('cloudbuild_err_not_found.yaml', False), + # Cleaning up files owned by root + ('cloudbuild_difficult_cleanup.yaml', True), + ) + for case in cases: + with self.subTest(case=case), \ + tempfile.TemporaryDirectory(prefix='local_cloudbuild_test_') as tempdir, \ + unittest.mock.patch('subprocess.check_call', self.check_call_with_capture): + config_name, should_succeed = case + config = os.path.join(self.testdata_dir, config_name) + actual_output_script = os.path.join( + tempdir, config_name + '_local.sh') + args = argparse.Namespace( + config=config, + output_script=actual_output_script, + run=should_run) + + if should_run: + print('Executing docker commands in {}'.format(actual_output_script)) + if should_succeed: local_cloudbuild.local_cloudbuild(args) - + else: + with self.assertRaises(subprocess.CalledProcessError): + local_cloudbuild.local_cloudbuild(args) + + # Check that staging dir was cleaned up + match = re.search(STAGING_DIR_REGEX, self.check_call_output) + self.assertTrue(match) + staging_dir = match.group(1) + self.assertFalse(os.path.isdir(staging_dir), staging_dir) + else: + # Generate but don't execute script + local_cloudbuild.local_cloudbuild(args) def test_parse_args(self): # Test explicit output_script diff --git a/scripts/testdata/cloudbuild_ok.yaml_golden.sh b/scripts/testdata/cloudbuild_ok.yaml_golden.sh index a6d5d12e..0b945fa9 100755 --- a/scripts/testdata/cloudbuild_ok.yaml_golden.sh +++ b/scripts/testdata/cloudbuild_ok.yaml_golden.sh @@ -9,8 +9,10 @@ SOURCE_DIR=. HOST_WORKSPACE=$(mktemp -d -t local_cloudbuild_XXXXXXXXXX) function cleanup { if [ "${HOST_WORKSPACE}" != '/' -a -d "${HOST_WORKSPACE}" ]; then + # Expect a single error message about /workspace busy docker run --volume /var/run/docker.sock:/var/run/docker.sock --volume /root/.docker:/root/.docker --volume ${HOST_WORKSPACE}:/workspace --workdir /workspace gcr.io/google-appengine/debian8 rm -rf /workspace 2>/dev/null || true - rmdir "${HOST_WORKSPACE}" + # Do not expect error messages here. Display but ignore any that happen. + rmdir "${HOST_WORKSPACE}" || true fi } trap cleanup EXIT From 7480ba51653ef4580b1c8a77423c27ed12c8d19a Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Tue, 21 Mar 2017 13:58:11 -0700 Subject: [PATCH 131/362] adding support for TAG environment variable (#82) --- jenkins_build.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/jenkins_build.sh b/jenkins_build.sh index f28196a7..253d5ca0 100755 --- a/jenkins_build.sh +++ b/jenkins_build.sh @@ -4,7 +4,11 @@ set -eu RUNTIME_NAME="python" -CANDIDATE_NAME=`date +%Y-%m-%d_%H_%M` +if [ -z "${TAG}" ] ; then + TAG=`date +%Y-%m-%d_%H_%M` +fi + +CANDIDATE_NAME="${TAG}" echo "CANDIDATE_NAME:${CANDIDATE_NAME}" if [ -z "${DOCKER_NAMESPACE+set}" ] ; then From 4f16a4714442a2c971e83ad4fcc66d1f275c7993 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Tue, 21 Mar 2017 14:55:33 -0700 Subject: [PATCH 132/362] Add nox.py to update requirements (#83) --- .gitignore | 2 ++ nox.py | 43 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 nox.py diff --git a/.gitignore b/.gitignore index 0c354cc9..c9aa6307 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ /cloudbuild.yaml_local.sh /ext_run.sh __pycache__ +.nox +*.pyc diff --git a/nox.py b/nox.py new file mode 100644 index 00000000..6d2e6723 --- /dev/null +++ b/nox.py @@ -0,0 +1,43 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import fnmatch +import os + +# Location of our common testing utilities. This isn't published to PyPI. +GCP_REPO_TOOLS_REQ =\ + 'git+https://github.com/GoogleCloudPlatform/python-repo-tools.git' + + +def _list_files(folder, pattern): + """Lists all files below the given folder that match the pattern.""" + for root, folders, files in os.walk(folder): + for filename in files: + if fnmatch.fnmatch(filename, pattern): + yield os.path.join(root, filename) + + +def session_check_requirements(session): + """Checks for out of date requirements and optionally updates them.""" + session.install(GCP_REPO_TOOLS_REQ) + + if 'update' in session.posargs: + command = 'update-requirements' + else: + command = 'check-requirements' + + reqfiles = list(_list_files('.', 'requirements*.txt')) + + for reqfile in reqfiles: + session.run('gcprepotools', command, reqfile) From 8613c33f32617da799df75b858cc418292812d19 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Tue, 21 Mar 2017 22:17:53 +0000 Subject: [PATCH 133/362] Auto-update dependencies. --- tests/integration/requirements.txt | 24 +- tests/python2-libraries/requirements.txt | 415 +++++++++++------------ tests/python3-libraries/requirements.txt | 339 +++++++++--------- 3 files changed, 379 insertions(+), 399 deletions(-) diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index dbd26d30..acc54ff8 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -1,21 +1,7 @@ -# Copyright 2016 Google Inc. All rights reserved. - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -Flask==0.11.1 -google-cloud-logging==0.22.0 -google-cloud-monitoring==0.22.0 -google-cloud-error-reporting==0.22.0 -gunicorn==19.6.0 +Flask==0.12 +google-cloud-logging==0.23.1 +google-cloud-monitoring==0.23.0 +google-cloud-error-reporting==0.23.1 +gunicorn==19.7.1 retrying==1.3.3 requests==2.13.0 diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 36dcbc16..7acbab47 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -1,212 +1,207 @@ -simplejson -setuptools -six -requests -virtualenv -pip -distribute -python-dateutil -certifi -boto -pbr -#wincertstore -docutils -pyasn1 -pyyaml -jinja2 -markupsafe -pytz -nose -lxml -pycrypto -rsa -colorama -botocore -cffi -awscli -coverage -jmespath -pycparser -pika -django -psycopg2 -paramiko -ecdsa -sqlalchemy -mock -redis -werkzeug -selenium -bcdoc -supervisor -pep8 -httplib2 -flask -pymongo -zc.buildout -psutil -mysql-python -argparse -carbon -pygments -babel -paste -anyjson -meld3 -# ssl (already included in standard library) -cryptography -py -tornado -pyopenssl -greenlet -kombu -graphite-web -docopt -mako -itsdangerous -pillow -wheel -beautifulsoup4 -enum34 -pyflakes -zope.interface -decorator -futures -pastedeploy -ordereddict -setuptools-git -fabric -backports.ssl_match_hostname -amqp -numpy -sphinx -iso8601 -flake8 -celery -pyparsing -mccabe -stevedore -pytest -webob -gunicorn -urllib3 -billiard -jsonschema -msgpack-python -gevent -logilab-common -unittest2 +simplejson==3.10.0 +setuptools==34.3.2 +six==1.10.0 +requests==2.13.0 +virtualenv==15.1.0 +pip==9.0.1 +distribute==0.7.3 +python-dateutil==2.6.0 +certifi==2017.1.23 +boto==2.46.1 +pbr==2.0.0 +docutils==0.13.1 +pyasn1==0.2.3 +pyyaml==3.12 +jinja2==2.9.5 +markupsafe==1.0 +pytz==2016.10 +nose==1.3.7 +lxml==3.7.3 +pycrypto==2.6.1 +rsa==3.4.2 +colorama==0.3.7 +botocore==1.5.27 +cffi==1.10.0 +awscli==1.11.64 +coverage==4.3.4 +jmespath==0.9.2 +pycparser==2.17 +pika==0.10.0 +django==1.10.6 +psycopg2==2.7.1 +paramiko==2.1.2 +ecdsa==0.13 +sqlalchemy==1.1.6 +mock==2.0.0 +redis==2.10.5 +werkzeug==0.12.1 +selenium==3.3.1 +bcdoc==0.16.0 +supervisor==3.3.1 +pep8==1.7.0 +httplib2==0.10.3 +flask==0.12 +pymongo==3.4.0 +zc.buildout==2.9.2 +psutil==5.2.0 +mysql-python==1.2.5 +argparse==1.4.0 +carbon==0.9.15 +pygments==2.2.0 +babel==2.3.4 +paste==2.0.3 +anyjson==0.3.3 +meld3==1.0.2 +cryptography==1.8.1 +py==1.4.33 +tornado==4.4.2 +pyopenssl==16.2.0 +greenlet==0.4.12 +kombu==4.0.2 +graphite-web==0.9.15 +docopt==0.6.2 +mako==1.0.6 +itsdangerous==0.24 +pillow==4.0.0 +wheel==0.29.0 +beautifulsoup4==4.5.3 +enum34==1.1.6 +pyflakes==1.5.0 +zope.interface==4.3.3 +decorator==4.0.11 +futures==3.0.5 +pastedeploy==1.5.2 +ordereddict==1.1 +setuptools-git==1.2 +fabric==1.13.1 +backports.ssl_match_hostname==3.5.0.1 +amqp==2.1.4 +numpy==1.12.1 +sphinx==1.5.3 +iso8601==0.1.11 +flake8==3.3.0 +celery==4.0.2 +pyparsing==2.2.0 +mccabe==0.6.1 +stevedore==1.21.0 +pytest==3.0.7 +webob==1.7.2 +gunicorn==19.7.1 +urllib3==1.20 +billiard==3.5.0.2 +jsonschema==2.6.0 +msgpack-python==0.4.8 +gevent==1.2.1 +logilab-common==1.4.0 +unittest2==1.1.0 prettytable -pylint -blessings -south -mozrunner -netaddr -oslo.config -twisted -ipaddress -ujson -moznetwork -mozdevice -mozprofile -mozprocess -mozfile -mozinfo +pylint==1.6.5 +blessings==1.6 +south==1.0.2 +mozrunner==6.13 +netaddr==0.7.19 +oslo.config==3.23.0 +twisted==17.1.0 +ipaddress==1.0.18 +ujson==1.35 +moznetwork==0.27 +mozdevice==0.48 +mozprofile==0.28 +mozprocess==0.25 +mozfile==1.2 +mozinfo==0.9 html5lib -mozlog -mozcrash -oauthlib -idna -ipython -tox -astroid -google-api-python-client -pycurl -isodate -python-keystoneclient -websocket-client -markdown -python-mimeparse -python-daemon -raven -suds -oauth2client -cython -eventlet -netifaces -repoze.lru -thrift -sqlparse -ndg-httpsclient -djangorestframework -python-novaclient -testtools -alembic -uritemplate -statsd -python-memcached -coveralls -funcsigs -configobj -linecache2 -extras -beautifulsoup -# scikit-learn -cliff -oauth2 -# pycups -cmd2 -unidecode -newrelic -python-gflags -cov-core -pytest-cov -fixtures -pyasn1-modules -python-swiftclient -django-debug-toolbar -elasticsearch -webtest -docker-py -python-subunit -retrying -django-extensions -pystache -waitress -pexpect -blinker -scipy -requests-oauthlib -protobuf -manifestparser -passlib -ansible -click -versiontools -django_compress -pyzmq -chardet -xlrd -snowballstemmer -testrepository -pandas -functools32 -python-cjson -pastescript -warlock -sqlalchemy-migrate -django-celery -uwsgi -cssselect -# Hand selected -matplotlib -pymysql -amqplib -sh -m2crypto -apache-libcloud -hiredis -bottle -pyramid -pyjwt -pylibmc +mozlog==3.4 +mozcrash==1.0 +oauthlib==2.0.2 +idna==2.5 +ipython==5.3.0 +tox==2.6.0 +astroid==1.4.9 +google-api-python-client==1.6.2 +pycurl==7.43.0 +isodate==0.5.4 +python-keystoneclient==3.10.0 +websocket-client==0.40.0 +markdown==2.6.8 +python-mimeparse==1.6.0 +python-daemon==2.1.2 +raven==6.0.0 +suds==0.4 +oauth2client==4.0.0 +cython==0.25.2 +eventlet==0.20.1 +netifaces==0.10.5 +repoze.lru==0.6 +thrift==0.10.0 +sqlparse==0.2.3 +ndg-httpsclient==0.4.2 +djangorestframework==3.6.2 +python-novaclient==7.1.0 +testtools==2.2.0 +alembic==0.9.1 +uritemplate==3.0.0 +statsd==3.2.1 +python-memcached==1.58 +coveralls==1.1 +funcsigs==1.0.2 +configobj==5.0.6 +linecache2==1.0.0 +extras==1.0.0 +beautifulsoup==3.2.1 +cliff==2.4.0 +oauth2==1.9.0.post1 +cmd2==0.7.0 +unidecode==0.4.20 +newrelic==2.82.0.62 +python-gflags==3.1.1 +cov-core==1.15.0 +pytest-cov==2.4.0 +fixtures==3.0.0 +pyasn1-modules==0.0.8 +python-swiftclient==3.3.0 +django-debug-toolbar==1.7 +elasticsearch==5.2.0 +webtest==2.0.27 +docker-py==1.10.6 +python-subunit==1.2.0 +retrying==1.3.3 +django-extensions==1.7.7 +pystache==0.5.4 +waitress==1.0.2 +pexpect==4.2.1 +blinker==1.4 +scipy==0.19.0 +requests-oauthlib==0.8.0 +protobuf==3.2.0 +manifestparser==1.1 +passlib==1.7.1 +ansible==2.2.1.0 +click==6.7 +versiontools==1.9.1 +django_compress==1.0.1 +pyzmq==16.0.2 +chardet==2.3.0 +xlrd==1.0.0 +snowballstemmer==1.2.1 +testrepository==0.0.20 +pandas==0.19.2 +functools32==3.2.3.post2 +python-cjson==1.2.1 +pastescript==2.0.2 +warlock==1.3.0 +sqlalchemy-migrate==0.11.0 +django-celery==3.2.1 +uwsgi==2.0.14 +cssselect==1.0.1 +matplotlib==2.0.0 +pymysql==0.7.10 +amqplib==1.0.2 +sh==1.12.11 +m2crypto==0.25.1 +apache-libcloud==1.5.0 +hiredis==0.2.0 +bottle==0.12.13 +pyramid==1.8.3 +pyjwt==1.4.2 +pylibmc==1.5.2 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index aeef827e..7f0b4ae0 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -1,172 +1,171 @@ -simplejson -six -requests -virtualenv -pip -certifi -docutils -pyasn1 -pyyaml -jinja2 -markupsafe -pytz -nose -lxml -pycrypto -rsa -colorama -cffi -pycparser -django -psycopg2 -ecdsa -sqlalchemy -mock -redis -werkzeug -pep8 -httplib2 -flask -pymongo -zc.buildout -psutil -argparse -carbon -pygments -babel -paste -anyjson -cryptography -py -tornado -pyopenssl -greenlet -kombu -docopt -mako -itsdangerous -pillow -wheel -beautifulsoup4 -pyflakes -zope.interface -decorator -futures -pastedeploy -setuptools-git -amqp -numpy -sphinx -iso8601 -flake8 -celery -pyparsing -mccabe -stevedore -pytest -webob -gunicorn -urllib3 -billiard -jsonschema -msgpack-python -gevent -logilab-common +simplejson==3.10.0 +six==1.10.0 +requests==2.13.0 +virtualenv==15.1.0 +pip==9.0.1 +certifi==2017.1.23 +docutils==0.13.1 +pyasn1==0.2.3 +pyyaml==3.12 +jinja2==2.9.5 +markupsafe==1.0 +pytz==2016.10 +nose==1.3.7 +lxml==3.7.3 +pycrypto==2.6.1 +rsa==3.4.2 +colorama==0.3.7 +cffi==1.10.0 +pycparser==2.17 +django==1.10.6 +psycopg2==2.7.1 +ecdsa==0.13 +sqlalchemy==1.1.6 +mock==2.0.0 +redis==2.10.5 +werkzeug==0.12.1 +pep8==1.7.0 +httplib2==0.10.3 +flask==0.12 +pymongo==3.4.0 +zc.buildout==2.9.2 +psutil==5.2.0 +argparse==1.4.0 +carbon==0.9.15 +pygments==2.2.0 +babel==2.3.4 +paste==2.0.3 +anyjson==0.3.3 +cryptography==1.8.1 +py==1.4.33 +tornado==4.4.2 +pyopenssl==16.2.0 +greenlet==0.4.12 +kombu==4.0.2 +docopt==0.6.2 +mako==1.0.6 +itsdangerous==0.24 +pillow==4.0.0 +wheel==0.29.0 +beautifulsoup4==4.5.3 +pyflakes==1.5.0 +zope.interface==4.3.3 +decorator==4.0.11 +futures==3.0.5 +pastedeploy==1.5.2 +setuptools-git==1.2 +amqp==2.1.4 +numpy==1.12.1 +sphinx==1.5.3 +iso8601==0.1.11 +flake8==3.3.0 +celery==4.0.2 +pyparsing==2.2.0 +mccabe==0.6.1 +stevedore==1.21.0 +pytest==3.0.7 +webob==1.7.2 +gunicorn==19.7.1 +urllib3==1.20 +billiard==3.5.0.2 +jsonschema==2.6.0 +msgpack-python==0.4.8 +gevent==1.2.1 +logilab-common==1.4.0 prettytable -pylint -blessings -south -netaddr -oslo.config -twisted -ipaddress -ujson +pylint==1.6.5 +blessings==1.6 +south==1.0.2 +netaddr==0.7.19 +oslo.config==3.23.0 +twisted==17.1.0 +ipaddress==1.0.18 +ujson==1.35 html5lib -oauthlib -idna -ipython -tox -astroid -google-api-python-client -pycurl -isodate -python-keystoneclient -websocket-client -markdown -python-mimeparse -python-daemon -raven -oauth2client -cython -eventlet -netifaces -repoze.lru -thrift -sqlparse -ndg-httpsclient -djangorestframework -python-novaclient -testtools -alembic -uritemplate -statsd -python-memcached -coveralls -funcsigs -configobj -linecache2 -extras -cliff -oauth2 -cmd2 -unidecode -newrelic -python-gflags -cov-core -pytest-cov -fixtures -pyasn1-modules -python-swiftclient -django-debug-toolbar -elasticsearch -webtest -docker-py -python-subunit -retrying -django-extensions -pystache -waitress -pexpect -blinker -scipy -requests-oauthlib -protobuf -manifestparser -passlib -ansible -click -versiontools -django_compress -pyzmq -chardet -xlrd -snowballstemmer -testrepository -pandas -pastescript -warlock -sqlalchemy-migrate -django-celery -cssselect -# Hand selected -matplotlib -pymysql -amqplib -sh -m2crypto -apache-libcloud -hiredis -bottle -pyramid -pyjwt -pylibmc +oauthlib==2.0.2 +idna==2.5 +ipython==5.3.0 +tox==2.6.0 +astroid==1.4.9 +google-api-python-client==1.6.2 +pycurl==7.43.0 +isodate==0.5.4 +python-keystoneclient==3.10.0 +websocket-client==0.40.0 +markdown==2.6.8 +python-mimeparse==1.6.0 +python-daemon==2.1.2 +raven==6.0.0 +oauth2client==4.0.0 +cython==0.25.2 +eventlet==0.20.1 +netifaces==0.10.5 +repoze.lru==0.6 +thrift==0.10.0 +sqlparse==0.2.3 +ndg-httpsclient==0.4.2 +djangorestframework==3.6.2 +python-novaclient==7.1.0 +testtools==2.2.0 +alembic==0.9.1 +uritemplate==3.0.0 +statsd==3.2.1 +python-memcached==1.58 +coveralls==1.1 +funcsigs==1.0.2 +configobj==5.0.6 +linecache2==1.0.0 +extras==1.0.0 +cliff==2.4.0 +oauth2==1.9.0.post1 +cmd2==0.7.0 +unidecode==0.4.20 +newrelic==2.82.0.62 +python-gflags==3.1.1 +cov-core==1.15.0 +pytest-cov==2.4.0 +fixtures==3.0.0 +pyasn1-modules==0.0.8 +python-swiftclient==3.3.0 +django-debug-toolbar==1.7 +elasticsearch==5.2.0 +webtest==2.0.27 +docker-py==1.10.6 +python-subunit==1.2.0 +retrying==1.3.3 +django-extensions==1.7.7 +pystache==0.5.4 +waitress==1.0.2 +pexpect==4.2.1 +blinker==1.4 +scipy==0.19.0 +requests-oauthlib==0.8.0 +protobuf==3.2.0 +manifestparser==1.1 +passlib==1.7.1 +ansible==2.2.1.0 +click==6.7 +versiontools==1.9.1 +django_compress==1.0.1 +pyzmq==16.0.2 +chardet==2.3.0 +xlrd==1.0.0 +snowballstemmer==1.2.1 +testrepository==0.0.20 +pandas==0.19.2 +pastescript==2.0.2 +warlock==1.3.0 +sqlalchemy-migrate==0.11.0 +django-celery==3.2.1 +cssselect==1.0.1 +matplotlib==2.0.0 +pymysql==0.7.10 +amqplib==1.0.2 +sh==1.12.11 +m2crypto==0.25.1 +apache-libcloud==1.5.0 +hiredis==0.2.0 +bottle==0.12.13 +pyramid==1.8.3 +pyjwt==1.4.2 +pylibmc==1.5.2 From 53926b59fa1eaba937526eb3b20104cd980d202f Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Tue, 21 Mar 2017 15:41:33 -0700 Subject: [PATCH 134/362] Sort requirements.txt files --- tests/integration/requirements.txt | 4 +- tests/python2-libraries/requirements.txt | 360 +++++++++++------------ tests/python3-libraries/requirements.txt | 298 +++++++++---------- 3 files changed, 331 insertions(+), 331 deletions(-) diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index acc54ff8..1a8489db 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -1,7 +1,7 @@ Flask==0.12 +google-cloud-error-reporting==0.23.1 google-cloud-logging==0.23.1 google-cloud-monitoring==0.23.0 -google-cloud-error-reporting==0.23.1 gunicorn==19.7.1 -retrying==1.3.3 requests==2.13.0 +retrying==1.3.3 diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 7acbab47..d64b5d2c 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -1,207 +1,207 @@ -simplejson==3.10.0 -setuptools==34.3.2 -six==1.10.0 -requests==2.13.0 -virtualenv==15.1.0 -pip==9.0.1 -distribute==0.7.3 -python-dateutil==2.6.0 -certifi==2017.1.23 +alembic==0.9.1 +amqp==2.1.4 +amqplib==1.0.2 +ansible==2.2.1.0 +anyjson==0.3.3 +apache-libcloud==1.5.0 +argparse==1.4.0 +astroid==1.4.9 +awscli==1.11.64 +babel==2.3.4 +backports.ssl_match_hostname==3.5.0.1 +bcdoc==0.16.0 +beautifulsoup4==4.5.3 +beautifulsoup==3.2.1 +billiard==3.5.0.2 +blessings==1.6 +blinker==1.4 boto==2.46.1 -pbr==2.0.0 -docutils==0.13.1 -pyasn1==0.2.3 -pyyaml==3.12 -jinja2==2.9.5 -markupsafe==1.0 -pytz==2016.10 -nose==1.3.7 -lxml==3.7.3 -pycrypto==2.6.1 -rsa==3.4.2 -colorama==0.3.7 botocore==1.5.27 +bottle==0.12.13 +carbon==0.9.15 +celery==4.0.2 +certifi==2017.1.23 cffi==1.10.0 -awscli==1.11.64 +chardet==2.3.0 +click==6.7 +cliff==2.4.0 +cmd2==0.7.0 +colorama==0.3.7 +configobj==5.0.6 +cov-core==1.15.0 coverage==4.3.4 -jmespath==0.9.2 -pycparser==2.17 -pika==0.10.0 -django==1.10.6 -psycopg2==2.7.1 -paramiko==2.1.2 -ecdsa==0.13 -sqlalchemy==1.1.6 -mock==2.0.0 -redis==2.10.5 -werkzeug==0.12.1 -selenium==3.3.1 -bcdoc==0.16.0 -supervisor==3.3.1 -pep8==1.7.0 -httplib2==0.10.3 -flask==0.12 -pymongo==3.4.0 -zc.buildout==2.9.2 -psutil==5.2.0 -mysql-python==1.2.5 -argparse==1.4.0 -carbon==0.9.15 -pygments==2.2.0 -babel==2.3.4 -paste==2.0.3 -anyjson==0.3.3 -meld3==1.0.2 +coveralls==1.1 cryptography==1.8.1 -py==1.4.33 -tornado==4.4.2 -pyopenssl==16.2.0 -greenlet==0.4.12 -kombu==4.0.2 -graphite-web==0.9.15 +cssselect==1.0.1 +cython==0.25.2 +decorator==4.0.11 +distribute==0.7.3 +django-celery==3.2.1 +django-debug-toolbar==1.7 +django-extensions==1.7.7 +django==1.10.6 +django_compress==1.0.1 +djangorestframework==3.6.2 +docker-py==1.10.6 docopt==0.6.2 -mako==1.0.6 -itsdangerous==0.24 -pillow==4.0.0 -wheel==0.29.0 -beautifulsoup4==4.5.3 +docutils==0.13.1 +ecdsa==0.13 +elasticsearch==5.2.0 enum34==1.1.6 -pyflakes==1.5.0 -zope.interface==4.3.3 -decorator==4.0.11 -futures==3.0.5 -pastedeploy==1.5.2 -ordereddict==1.1 -setuptools-git==1.2 +eventlet==0.20.1 +extras==1.0.0 fabric==1.13.1 -backports.ssl_match_hostname==3.5.0.1 -amqp==2.1.4 -numpy==1.12.1 -sphinx==1.5.3 -iso8601==0.1.11 +fixtures==3.0.0 flake8==3.3.0 -celery==4.0.2 -pyparsing==2.2.0 -mccabe==0.6.1 -stevedore==1.21.0 -pytest==3.0.7 -webob==1.7.2 +flask==0.12 +funcsigs==1.0.2 +functools32==3.2.3.post2 +futures==3.0.5 +gevent==1.2.1 +google-api-python-client==1.6.2 +graphite-web==0.9.15 +greenlet==0.4.12 gunicorn==19.7.1 -urllib3==1.20 -billiard==3.5.0.2 +hiredis==0.2.0 +html5lib +httplib2==0.10.3 +idna==2.5 +ipaddress==1.0.18 +ipython==5.3.0 +iso8601==0.1.11 +isodate==0.5.4 +itsdangerous==0.24 +jinja2==2.9.5 +jmespath==0.9.2 jsonschema==2.6.0 -msgpack-python==0.4.8 -gevent==1.2.1 +kombu==4.0.2 +linecache2==1.0.0 logilab-common==1.4.0 -unittest2==1.1.0 -prettytable -pylint==1.6.5 -blessings==1.6 -south==1.0.2 -mozrunner==6.13 -netaddr==0.7.19 -oslo.config==3.23.0 -twisted==17.1.0 -ipaddress==1.0.18 -ujson==1.35 -moznetwork==0.27 +lxml==3.7.3 +m2crypto==0.25.1 +mako==1.0.6 +manifestparser==1.1 +markdown==2.6.8 +markupsafe==1.0 +matplotlib==2.0.0 +mccabe==0.6.1 +meld3==1.0.2 +mock==2.0.0 +mozcrash==1.0 mozdevice==0.48 -mozprofile==0.28 -mozprocess==0.25 mozfile==1.2 mozinfo==0.9 -html5lib mozlog==3.4 -mozcrash==1.0 +moznetwork==0.27 +mozprocess==0.25 +mozprofile==0.28 +mozrunner==6.13 +msgpack-python==0.4.8 +mysql-python==1.2.5 +ndg-httpsclient==0.4.2 +netaddr==0.7.19 +netifaces==0.10.5 +newrelic==2.82.0.62 +nose==1.3.7 +numpy==1.12.1 +oauth2==1.9.0.post1 +oauth2client==4.0.0 oauthlib==2.0.2 -idna==2.5 -ipython==5.3.0 -tox==2.6.0 -astroid==1.4.9 -google-api-python-client==1.6.2 +ordereddict==1.1 +oslo.config==3.23.0 +pandas==0.19.2 +paramiko==2.1.2 +passlib==1.7.1 +paste==2.0.3 +pastedeploy==1.5.2 +pastescript==2.0.2 +pbr==2.0.0 +pep8==1.7.0 +pexpect==4.2.1 +pika==0.10.0 +pillow==4.0.0 +pip==9.0.1 +prettytable +protobuf==3.2.0 +psutil==5.2.0 +psycopg2==2.7.1 +py==1.4.33 +pyasn1-modules==0.0.8 +pyasn1==0.2.3 +pycparser==2.17 +pycrypto==2.6.1 pycurl==7.43.0 -isodate==0.5.4 +pyflakes==1.5.0 +pygments==2.2.0 +pyjwt==1.4.2 +pylibmc==1.5.2 +pylint==1.6.5 +pymongo==3.4.0 +pymysql==0.7.10 +pyopenssl==16.2.0 +pyparsing==2.2.0 +pyramid==1.8.3 +pystache==0.5.4 +pytest-cov==2.4.0 +pytest==3.0.7 +python-cjson==1.2.1 +python-daemon==2.1.2 +python-dateutil==2.6.0 +python-gflags==3.1.1 python-keystoneclient==3.10.0 -websocket-client==0.40.0 -markdown==2.6.8 +python-memcached==1.58 python-mimeparse==1.6.0 -python-daemon==2.1.2 -raven==6.0.0 -suds==0.4 -oauth2client==4.0.0 -cython==0.25.2 -eventlet==0.20.1 -netifaces==0.10.5 -repoze.lru==0.6 -thrift==0.10.0 -sqlparse==0.2.3 -ndg-httpsclient==0.4.2 -djangorestframework==3.6.2 python-novaclient==7.1.0 -testtools==2.2.0 -alembic==0.9.1 -uritemplate==3.0.0 -statsd==3.2.1 -python-memcached==1.58 -coveralls==1.1 -funcsigs==1.0.2 -configobj==5.0.6 -linecache2==1.0.0 -extras==1.0.0 -beautifulsoup==3.2.1 -cliff==2.4.0 -oauth2==1.9.0.post1 -cmd2==0.7.0 -unidecode==0.4.20 -newrelic==2.82.0.62 -python-gflags==3.1.1 -cov-core==1.15.0 -pytest-cov==2.4.0 -fixtures==3.0.0 -pyasn1-modules==0.0.8 -python-swiftclient==3.3.0 -django-debug-toolbar==1.7 -elasticsearch==5.2.0 -webtest==2.0.27 -docker-py==1.10.6 python-subunit==1.2.0 +python-swiftclient==3.3.0 +pytz==2016.10 +pyyaml==3.12 +pyzmq==16.0.2 +raven==6.0.0 +redis==2.10.5 +repoze.lru==0.6 +requests-oauthlib==0.8.0 +requests==2.13.0 retrying==1.3.3 -django-extensions==1.7.7 -pystache==0.5.4 -waitress==1.0.2 -pexpect==4.2.1 -blinker==1.4 +rsa==3.4.2 scipy==0.19.0 -requests-oauthlib==0.8.0 -protobuf==3.2.0 -manifestparser==1.1 -passlib==1.7.1 -ansible==2.2.1.0 -click==6.7 -versiontools==1.9.1 -django_compress==1.0.1 -pyzmq==16.0.2 -chardet==2.3.0 -xlrd==1.0.0 +selenium==3.3.1 +setuptools-git==1.2 +setuptools==34.3.2 +sh==1.12.11 +simplejson==3.10.0 +six==1.10.0 snowballstemmer==1.2.1 -testrepository==0.0.20 -pandas==0.19.2 -functools32==3.2.3.post2 -python-cjson==1.2.1 -pastescript==2.0.2 -warlock==1.3.0 +south==1.0.2 +sphinx==1.5.3 sqlalchemy-migrate==0.11.0 -django-celery==3.2.1 +sqlalchemy==1.1.6 +sqlparse==0.2.3 +statsd==3.2.1 +stevedore==1.21.0 +suds==0.4 +supervisor==3.3.1 +testrepository==0.0.20 +testtools==2.2.0 +thrift==0.10.0 +tornado==4.4.2 +tox==2.6.0 +twisted==17.1.0 +ujson==1.35 +unidecode==0.4.20 +unittest2==1.1.0 +uritemplate==3.0.0 +urllib3==1.20 uwsgi==2.0.14 -cssselect==1.0.1 -matplotlib==2.0.0 -pymysql==0.7.10 -amqplib==1.0.2 -sh==1.12.11 -m2crypto==0.25.1 -apache-libcloud==1.5.0 -hiredis==0.2.0 -bottle==0.12.13 -pyramid==1.8.3 -pyjwt==1.4.2 -pylibmc==1.5.2 +versiontools==1.9.1 +virtualenv==15.1.0 +waitress==1.0.2 +warlock==1.3.0 +webob==1.7.2 +websocket-client==0.40.0 +webtest==2.0.27 +werkzeug==0.12.1 +wheel==0.29.0 +xlrd==1.0.0 +zc.buildout==2.9.2 +zope.interface==4.3.3 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 7f0b4ae0..c87fc842 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -1,171 +1,171 @@ -simplejson==3.10.0 -six==1.10.0 -requests==2.13.0 -virtualenv==15.1.0 -pip==9.0.1 +alembic==0.9.1 +amqp==2.1.4 +amqplib==1.0.2 +ansible==2.2.1.0 +anyjson==0.3.3 +apache-libcloud==1.5.0 +argparse==1.4.0 +astroid==1.4.9 +babel==2.3.4 +beautifulsoup4==4.5.3 +billiard==3.5.0.2 +blessings==1.6 +blinker==1.4 +bottle==0.12.13 +carbon==0.9.15 +celery==4.0.2 certifi==2017.1.23 -docutils==0.13.1 -pyasn1==0.2.3 -pyyaml==3.12 -jinja2==2.9.5 -markupsafe==1.0 -pytz==2016.10 -nose==1.3.7 -lxml==3.7.3 -pycrypto==2.6.1 -rsa==3.4.2 -colorama==0.3.7 cffi==1.10.0 -pycparser==2.17 +chardet==2.3.0 +click==6.7 +cliff==2.4.0 +cmd2==0.7.0 +colorama==0.3.7 +configobj==5.0.6 +cov-core==1.15.0 +coveralls==1.1 +cryptography==1.8.1 +cssselect==1.0.1 +cython==0.25.2 +decorator==4.0.11 +django-celery==3.2.1 +django-debug-toolbar==1.7 +django-extensions==1.7.7 django==1.10.6 -psycopg2==2.7.1 +django_compress==1.0.1 +djangorestframework==3.6.2 +docker-py==1.10.6 +docopt==0.6.2 +docutils==0.13.1 ecdsa==0.13 -sqlalchemy==1.1.6 -mock==2.0.0 -redis==2.10.5 -werkzeug==0.12.1 -pep8==1.7.0 -httplib2==0.10.3 +elasticsearch==5.2.0 +eventlet==0.20.1 +extras==1.0.0 +fixtures==3.0.0 +flake8==3.3.0 flask==0.12 -pymongo==3.4.0 -zc.buildout==2.9.2 -psutil==5.2.0 -argparse==1.4.0 -carbon==0.9.15 -pygments==2.2.0 -babel==2.3.4 -paste==2.0.3 -anyjson==0.3.3 -cryptography==1.8.1 -py==1.4.33 -tornado==4.4.2 -pyopenssl==16.2.0 +funcsigs==1.0.2 +futures==3.0.5 +gevent==1.2.1 +google-api-python-client==1.6.2 greenlet==0.4.12 +gunicorn==19.7.1 +hiredis==0.2.0 +html5lib +httplib2==0.10.3 +idna==2.5 +ipaddress==1.0.18 +ipython==5.3.0 +iso8601==0.1.11 +isodate==0.5.4 +itsdangerous==0.24 +jinja2==2.9.5 +jsonschema==2.6.0 kombu==4.0.2 -docopt==0.6.2 +linecache2==1.0.0 +logilab-common==1.4.0 +lxml==3.7.3 +m2crypto==0.25.1 mako==1.0.6 -itsdangerous==0.24 -pillow==4.0.0 -wheel==0.29.0 -beautifulsoup4==4.5.3 -pyflakes==1.5.0 -zope.interface==4.3.3 -decorator==4.0.11 -futures==3.0.5 -pastedeploy==1.5.2 -setuptools-git==1.2 -amqp==2.1.4 -numpy==1.12.1 -sphinx==1.5.3 -iso8601==0.1.11 -flake8==3.3.0 -celery==4.0.2 -pyparsing==2.2.0 +manifestparser==1.1 +markdown==2.6.8 +markupsafe==1.0 +matplotlib==2.0.0 mccabe==0.6.1 -stevedore==1.21.0 -pytest==3.0.7 -webob==1.7.2 -gunicorn==19.7.1 -urllib3==1.20 -billiard==3.5.0.2 -jsonschema==2.6.0 +mock==2.0.0 msgpack-python==0.4.8 -gevent==1.2.1 -logilab-common==1.4.0 -prettytable -pylint==1.6.5 -blessings==1.6 -south==1.0.2 +ndg-httpsclient==0.4.2 netaddr==0.7.19 -oslo.config==3.23.0 -twisted==17.1.0 -ipaddress==1.0.18 -ujson==1.35 -html5lib +netifaces==0.10.5 +newrelic==2.82.0.62 +nose==1.3.7 +numpy==1.12.1 +oauth2==1.9.0.post1 +oauth2client==4.0.0 oauthlib==2.0.2 -idna==2.5 -ipython==5.3.0 -tox==2.6.0 -astroid==1.4.9 -google-api-python-client==1.6.2 +oslo.config==3.23.0 +pandas==0.19.2 +passlib==1.7.1 +paste==2.0.3 +pastedeploy==1.5.2 +pastescript==2.0.2 +pep8==1.7.0 +pexpect==4.2.1 +pillow==4.0.0 +pip==9.0.1 +prettytable +protobuf==3.2.0 +psutil==5.2.0 +psycopg2==2.7.1 +py==1.4.33 +pyasn1-modules==0.0.8 +pyasn1==0.2.3 +pycparser==2.17 +pycrypto==2.6.1 pycurl==7.43.0 -isodate==0.5.4 +pyflakes==1.5.0 +pygments==2.2.0 +pyjwt==1.4.2 +pylibmc==1.5.2 +pylint==1.6.5 +pymongo==3.4.0 +pymysql==0.7.10 +pyopenssl==16.2.0 +pyparsing==2.2.0 +pyramid==1.8.3 +pystache==0.5.4 +pytest-cov==2.4.0 +pytest==3.0.7 +python-daemon==2.1.2 +python-gflags==3.1.1 python-keystoneclient==3.10.0 -websocket-client==0.40.0 -markdown==2.6.8 +python-memcached==1.58 python-mimeparse==1.6.0 -python-daemon==2.1.2 -raven==6.0.0 -oauth2client==4.0.0 -cython==0.25.2 -eventlet==0.20.1 -netifaces==0.10.5 -repoze.lru==0.6 -thrift==0.10.0 -sqlparse==0.2.3 -ndg-httpsclient==0.4.2 -djangorestframework==3.6.2 python-novaclient==7.1.0 -testtools==2.2.0 -alembic==0.9.1 -uritemplate==3.0.0 -statsd==3.2.1 -python-memcached==1.58 -coveralls==1.1 -funcsigs==1.0.2 -configobj==5.0.6 -linecache2==1.0.0 -extras==1.0.0 -cliff==2.4.0 -oauth2==1.9.0.post1 -cmd2==0.7.0 -unidecode==0.4.20 -newrelic==2.82.0.62 -python-gflags==3.1.1 -cov-core==1.15.0 -pytest-cov==2.4.0 -fixtures==3.0.0 -pyasn1-modules==0.0.8 -python-swiftclient==3.3.0 -django-debug-toolbar==1.7 -elasticsearch==5.2.0 -webtest==2.0.27 -docker-py==1.10.6 python-subunit==1.2.0 +python-swiftclient==3.3.0 +pytz==2016.10 +pyyaml==3.12 +pyzmq==16.0.2 +raven==6.0.0 +redis==2.10.5 +repoze.lru==0.6 +requests-oauthlib==0.8.0 +requests==2.13.0 retrying==1.3.3 -django-extensions==1.7.7 -pystache==0.5.4 -waitress==1.0.2 -pexpect==4.2.1 -blinker==1.4 +rsa==3.4.2 scipy==0.19.0 -requests-oauthlib==0.8.0 -protobuf==3.2.0 -manifestparser==1.1 -passlib==1.7.1 -ansible==2.2.1.0 -click==6.7 -versiontools==1.9.1 -django_compress==1.0.1 -pyzmq==16.0.2 -chardet==2.3.0 -xlrd==1.0.0 +setuptools-git==1.2 +sh==1.12.11 +simplejson==3.10.0 +six==1.10.0 snowballstemmer==1.2.1 +south==1.0.2 +sphinx==1.5.3 +sqlalchemy-migrate==0.11.0 +sqlalchemy==1.1.6 +sqlparse==0.2.3 +statsd==3.2.1 +stevedore==1.21.0 testrepository==0.0.20 -pandas==0.19.2 -pastescript==2.0.2 +testtools==2.2.0 +thrift==0.10.0 +tornado==4.4.2 +tox==2.6.0 +twisted==17.1.0 +ujson==1.35 +unidecode==0.4.20 +uritemplate==3.0.0 +urllib3==1.20 +versiontools==1.9.1 +virtualenv==15.1.0 +waitress==1.0.2 warlock==1.3.0 -sqlalchemy-migrate==0.11.0 -django-celery==3.2.1 -cssselect==1.0.1 -matplotlib==2.0.0 -pymysql==0.7.10 -amqplib==1.0.2 -sh==1.12.11 -m2crypto==0.25.1 -apache-libcloud==1.5.0 -hiredis==0.2.0 -bottle==0.12.13 -pyramid==1.8.3 -pyjwt==1.4.2 -pylibmc==1.5.2 +webob==1.7.2 +websocket-client==0.40.0 +webtest==2.0.27 +werkzeug==0.12.1 +wheel==0.29.0 +xlrd==1.0.0 +zc.buildout==2.9.2 +zope.interface==4.3.3 From f85f392fd5000628e8eb6cb58134672ff29b95e8 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Fri, 24 Mar 2017 10:17:44 -0700 Subject: [PATCH 135/362] Auto-update dependencies. (#86) --- tests/python2-libraries/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index d64b5d2c..9a1fc8a2 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -6,7 +6,7 @@ anyjson==0.3.3 apache-libcloud==1.5.0 argparse==1.4.0 astroid==1.4.9 -awscli==1.11.64 +awscli==1.11.66 babel==2.3.4 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 @@ -16,7 +16,7 @@ billiard==3.5.0.2 blessings==1.6 blinker==1.4 boto==2.46.1 -botocore==1.5.27 +botocore==1.5.29 bottle==0.12.13 carbon==0.9.15 celery==4.0.2 From d63ab6d0c7640ef9c6c54e86dc348a1c31cad495 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Sun, 26 Mar 2017 09:07:48 +0000 Subject: [PATCH 136/362] Auto-update dependencies. --- tests/python2-libraries/requirements.txt | 6 +++--- tests/python3-libraries/requirements.txt | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 9a1fc8a2..0a018109 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -7,7 +7,7 @@ apache-libcloud==1.5.0 argparse==1.4.0 astroid==1.4.9 awscli==1.11.66 -babel==2.3.4 +babel==2.4.0 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 beautifulsoup4==4.5.3 @@ -78,7 +78,7 @@ kombu==4.0.2 linecache2==1.0.0 logilab-common==1.4.0 lxml==3.7.3 -m2crypto==0.25.1 +m2crypto==0.26.0 mako==1.0.6 manifestparser==1.1 markdown==2.6.8 @@ -123,7 +123,7 @@ pillow==4.0.0 pip==9.0.1 prettytable protobuf==3.2.0 -psutil==5.2.0 +psutil==5.2.1 psycopg2==2.7.1 py==1.4.33 pyasn1-modules==0.0.8 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index c87fc842..ed1a7ffd 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -6,7 +6,7 @@ anyjson==0.3.3 apache-libcloud==1.5.0 argparse==1.4.0 astroid==1.4.9 -babel==2.3.4 +babel==2.4.0 beautifulsoup4==4.5.3 billiard==3.5.0.2 blessings==1.6 @@ -65,7 +65,7 @@ kombu==4.0.2 linecache2==1.0.0 logilab-common==1.4.0 lxml==3.7.3 -m2crypto==0.25.1 +m2crypto==0.26.0 mako==1.0.6 manifestparser==1.1 markdown==2.6.8 @@ -95,7 +95,7 @@ pillow==4.0.0 pip==9.0.1 prettytable protobuf==3.2.0 -psutil==5.2.0 +psutil==5.2.1 psycopg2==2.7.1 py==1.4.33 pyasn1-modules==0.0.8 From 9416574b7ddc61d93c5ab17873bd15deeabcf6fa Mon Sep 17 00:00:00 2001 From: DPE bot Date: Mon, 27 Mar 2017 09:07:54 +0000 Subject: [PATCH 137/362] Auto-update dependencies. --- tests/python2-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 0a018109..7d168be6 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -167,7 +167,7 @@ rsa==3.4.2 scipy==0.19.0 selenium==3.3.1 setuptools-git==1.2 -setuptools==34.3.2 +setuptools==34.3.3 sh==1.12.11 simplejson==3.10.0 six==1.10.0 From 22f420c841b4beaa1e6946e06d6e03484d309bc6 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Tue, 28 Mar 2017 09:07:49 +0000 Subject: [PATCH 138/362] Auto-update dependencies. --- tests/python2-libraries/requirements.txt | 8 ++++---- tests/python3-libraries/requirements.txt | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 7d168be6..99a95bad 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -1,12 +1,12 @@ alembic==0.9.1 amqp==2.1.4 amqplib==1.0.2 -ansible==2.2.1.0 +ansible==2.2.2.0 anyjson==0.3.3 apache-libcloud==1.5.0 argparse==1.4.0 astroid==1.4.9 -awscli==1.11.66 +awscli==1.11.67 babel==2.4.0 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 @@ -16,7 +16,7 @@ billiard==3.5.0.2 blessings==1.6 blinker==1.4 boto==2.46.1 -botocore==1.5.29 +botocore==1.5.30 bottle==0.12.13 carbon==0.9.15 celery==4.0.2 @@ -175,7 +175,7 @@ snowballstemmer==1.2.1 south==1.0.2 sphinx==1.5.3 sqlalchemy-migrate==0.11.0 -sqlalchemy==1.1.6 +sqlalchemy==1.1.7 sqlparse==0.2.3 statsd==3.2.1 stevedore==1.21.0 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index ed1a7ffd..3ea2903e 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -1,7 +1,7 @@ alembic==0.9.1 amqp==2.1.4 amqplib==1.0.2 -ansible==2.2.1.0 +ansible==2.2.2.0 anyjson==0.3.3 apache-libcloud==1.5.0 argparse==1.4.0 @@ -143,7 +143,7 @@ snowballstemmer==1.2.1 south==1.0.2 sphinx==1.5.3 sqlalchemy-migrate==0.11.0 -sqlalchemy==1.1.6 +sqlalchemy==1.1.7 sqlparse==0.2.3 statsd==3.2.1 stevedore==1.21.0 From 8a467434353fb6d8d81213ebd5b716b9f9620106 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Wed, 29 Mar 2017 09:07:49 +0000 Subject: [PATCH 139/362] Auto-update dependencies. --- tests/integration/requirements.txt | 2 +- tests/python2-libraries/requirements.txt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index 1a8489db..c5004b32 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -1,5 +1,5 @@ Flask==0.12 -google-cloud-error-reporting==0.23.1 +google-cloud-error-reporting==0.23.2 google-cloud-logging==0.23.1 google-cloud-monitoring==0.23.0 gunicorn==19.7.1 diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 99a95bad..11a2cb63 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -6,7 +6,7 @@ anyjson==0.3.3 apache-libcloud==1.5.0 argparse==1.4.0 astroid==1.4.9 -awscli==1.11.67 +awscli==1.11.68 babel==2.4.0 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 @@ -16,7 +16,7 @@ billiard==3.5.0.2 blessings==1.6 blinker==1.4 boto==2.46.1 -botocore==1.5.30 +botocore==1.5.31 bottle==0.12.13 carbon==0.9.15 celery==4.0.2 From 54a6085dab5128f8ce74eb9863a05a649b4a7016 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Thu, 30 Mar 2017 09:07:51 +0000 Subject: [PATCH 140/362] Auto-update dependencies. --- tests/python2-libraries/requirements.txt | 4 ++-- tests/python3-libraries/requirements.txt | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 11a2cb63..409342f4 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -38,7 +38,7 @@ decorator==4.0.11 distribute==0.7.3 django-celery==3.2.1 django-debug-toolbar==1.7 -django-extensions==1.7.7 +django-extensions==1.7.8 django==1.10.6 django_compress==1.0.1 djangorestframework==3.6.2 @@ -154,7 +154,7 @@ python-mimeparse==1.6.0 python-novaclient==7.1.0 python-subunit==1.2.0 python-swiftclient==3.3.0 -pytz==2016.10 +pytz==2017.2 pyyaml==3.12 pyzmq==16.0.2 raven==6.0.0 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 3ea2903e..a7269999 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -30,7 +30,7 @@ cython==0.25.2 decorator==4.0.11 django-celery==3.2.1 django-debug-toolbar==1.7 -django-extensions==1.7.7 +django-extensions==1.7.8 django==1.10.6 django_compress==1.0.1 djangorestframework==3.6.2 @@ -124,7 +124,7 @@ python-mimeparse==1.6.0 python-novaclient==7.1.0 python-subunit==1.2.0 python-swiftclient==3.3.0 -pytz==2016.10 +pytz==2017.2 pyyaml==3.12 pyzmq==16.0.2 raven==6.0.0 From 41d3d61cb8ddda83af006e7887925cead90d567d Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Thu, 30 Mar 2017 15:23:25 -0700 Subject: [PATCH 141/362] Remove 'distribute' from library test. 'pip install distribute' unpredictably fails when other packages are being updated at the same time. See https://github.com/pypa/setuptools/issues/535 for related discussion. Error message looks like: Running setup.py install for distribute: started Running setup.py install for distribute: finished with status 'error' Complete output from command /env/bin/python -u -c "import setuptools, tokenize;__file__='/tmp/pip-build-9DiKZ8/distribute/setup.py';f=getattr(tokenize, 'open', open)(__file__);code=f.read().replace('\r\n', '\n');f.close();exec(compile(code, __file__, 'exec'))" install --record /tmp/pip-zFa039-record/install-record.txt --single-version-externally-managed --compile --install-headers /env/include/site/python2.7/distribute: running install running build running install_egg_info running egg_info writing requirements to distribute.egg-info/requires.txt writing distribute.egg-info/PKG-INFO writing top-level names to distribute.egg-info/top_level.txt writing dependency_links to distribute.egg-info/dependency_links.txt Traceback (most recent call last): File "", line 1, in File "/tmp/pip-build-9DiKZ8/distribute/setup.py", line 58, in setuptools.setup(**setup_params) File "/usr/lib/python2.7/distutils/core.py", line 151, in setup dist.run_commands() File "/usr/lib/python2.7/distutils/dist.py", line 953, in run_commands self.run_command(cmd) File "/usr/lib/python2.7/distutils/dist.py", line 972, in run_command cmd_obj.run() File "setuptools/command/install.py", line 53, in run return _install.run(self) File "/usr/lib/python2.7/distutils/command/install.py", line 613, in run self.run_command(cmd_name) File "/usr/lib/python2.7/distutils/cmd.py", line 326, in run_command self.distribution.run_command(command) File "/usr/lib/python2.7/distutils/dist.py", line 972, in run_command cmd_obj.run() File "setuptools/command/install_egg_info.py", line 29, in run self.run_command('egg_info') File "/usr/lib/python2.7/distutils/cmd.py", line 326, in run_command self.distribution.run_command(command) File "/usr/lib/python2.7/distutils/dist.py", line 972, in run_command cmd_obj.run() File "setuptools/command/egg_info.py", line 177, in run writer = ep.load(installer=installer) File "pkg_resources.py", line 2241, in load if require: self.require(env, installer) File "pkg_resources.py", line 2254, in require working_set.resolve(self.dist.requires(self.extras),env,installer))) File "pkg_resources.py", line 2471, in requires dm = self._dep_map File "pkg_resources.py", line 2682, in _dep_map self.__dep_map = self._compute_dependencies() File "pkg_resources.py", line 2699, in _compute_dependencies from _markerlib import compile as compile_marker ImportError: No module named _markerlib --- tests/python2-libraries/requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 409342f4..d92af43f 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -35,7 +35,6 @@ cryptography==1.8.1 cssselect==1.0.1 cython==0.25.2 decorator==4.0.11 -distribute==0.7.3 django-celery==3.2.1 django-debug-toolbar==1.7 django-extensions==1.7.8 From e9ecfac40107413048414c7e91f5834fdc9c5fb3 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 31 Mar 2017 17:02:20 -0700 Subject: [PATCH 142/362] Update build process. build.sh is the entrypoint for all builds. cloudbuild*.yaml files are now the single source of truth for the build steps. Makefiles have been removed. Local builds are done via local_cloudbuild.py. jenkins_build.sh will be removed once the Jenkins config is updated. The benchmarking and system tests now work in Container Builder. Build instructions have been moved into their own file. gcloud substitutions are used for cloudbuild.yaml substitions. See 'gcloud beta container builds submit --help' for more info. 'envsubst' is still used for Dockerifle substitutions. Updated the google-cloud-python test logic. --- .gitignore | 8 +- Makefile | 73 --------- README.md | 68 +-------- RELEASING.md | 130 ++++++++++++++++ build.sh | 139 +++++++++++++++++- cloudbuild.yaml | 34 +++++ cloudbuild.yaml.in | 24 --- cloudbuild_benchmark.yaml | 8 + cloudbuild_system_tests.yaml | 9 ++ jenkins_build.sh | 44 +----- python-interpreter-builder/.gitignore | 2 +- .../{Dockerfile => Dockerfile.in} | 2 +- python-interpreter-builder/Makefile | 10 -- runtime-image/.gitignore | 2 +- runtime-image/{Dockerfile => Dockerfile.in} | 2 +- system_tests/.gitignore | 3 - system_tests/Dockerfile.in | 15 -- system_tests/Makefile | 22 --- tests/Makefile | 18 --- tests/benchmark/Dockerfile.2vs3 | 1 - tests/benchmark/Dockerfile.34vs35 | 1 - tests/benchmark/Dockerfile.in | 6 +- tests/benchmark/Makefile | 14 -- tests/google-cloud-python-system/.gitignore | 2 + .../google-cloud-python-system/Dockerfile.in | 16 ++ .../run_system_tests.sh | 34 +++++ tests/google-cloud-python/Dockerfile.in | 19 +-- tests/google-cloud-python/Makefile | 5 - tests/google-cloud-python/run_unit_tests.sh | 11 ++ tests/integration/.gitignore | 1 + tests/integration/Dockerfile.in | 2 +- tests/license-test/Makefile | 3 - tests/no-virtualenv/Makefile | 3 - tests/python2-libraries/Makefile | 3 - tests/python3-libraries/Makefile | 3 - tests/virtualenv/Makefile | 5 - 36 files changed, 400 insertions(+), 342 deletions(-) delete mode 100644 Makefile create mode 100644 RELEASING.md create mode 100644 cloudbuild.yaml delete mode 100644 cloudbuild.yaml.in create mode 100644 cloudbuild_benchmark.yaml create mode 100644 cloudbuild_system_tests.yaml rename python-interpreter-builder/{Dockerfile => Dockerfile.in} (94%) delete mode 100644 python-interpreter-builder/Makefile rename runtime-image/{Dockerfile => Dockerfile.in} (96%) delete mode 100644 system_tests/.gitignore delete mode 100644 system_tests/Dockerfile.in delete mode 100644 system_tests/Makefile delete mode 100644 tests/Makefile delete mode 100644 tests/benchmark/Dockerfile.2vs3 delete mode 100644 tests/benchmark/Dockerfile.34vs35 delete mode 100644 tests/benchmark/Makefile create mode 100644 tests/google-cloud-python-system/.gitignore create mode 100644 tests/google-cloud-python-system/Dockerfile.in create mode 100755 tests/google-cloud-python-system/run_system_tests.sh delete mode 100644 tests/google-cloud-python/Makefile create mode 100755 tests/google-cloud-python/run_unit_tests.sh create mode 100644 tests/integration/.gitignore delete mode 100644 tests/license-test/Makefile delete mode 100644 tests/no-virtualenv/Makefile delete mode 100644 tests/python2-libraries/Makefile delete mode 100644 tests/python3-libraries/Makefile delete mode 100644 tests/virtualenv/Makefile diff --git a/.gitignore b/.gitignore index c9aa6307..2a84c77d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ -/cloudbuild.yaml +*.pyc +.nox /cloudbuild.yaml_local.sh -/ext_run.sh +/cloudbuild_benchmark.yaml_local.sh +/cloudbuild_system_tests.yaml_local.sh __pycache__ -.nox -*.pyc diff --git a/Makefile b/Makefile deleted file mode 100644 index 395d43b6..00000000 --- a/Makefile +++ /dev/null @@ -1,73 +0,0 @@ -ifneq ($(FORCE_REBUILD),0) - export DOCKER_FLAGS=--no-cache --pull -endif - -ifndef IMAGE_NAME -$(error IMAGE_NAME is not set; invoke make with something like IMAGE_NAME=google/python:2017-01-02_03_45) -endif - -.PHONY: all -all: cloud-test - -## Files that must be refreshed every build - -.PHONY: cloudbuild.yaml # Force reevaluation of env vars every time -cloudbuild.yaml: cloudbuild.yaml.in - envsubst < $< > $@ - - -.PHONY: tests/google-cloud-python/Dockerfile # Force reevaluation of env vars every time -tests/google-cloud-python/Dockerfile: tests/google-cloud-python/Dockerfile.in - envsubst < $< > $@ - -.PHONY: ext_run.sh # Force refetch every time -ext_run.sh: - curl https://raw.githubusercontent.com/GoogleCloudPlatform/runtimes-common/master/structure_tests/ext_run.sh > ext_run.sh - chmod +x ext_run.sh - -## Build using Google Container Builder service - -.PHONY: cloud-build -cloud-build: cloudbuild.yaml tests/google-cloud-python/Dockerfile - gcloud beta container builds submit . --config=cloudbuild.yaml - -.PHONY: cloud-test -# structure-tests and google-cloud-python-tests are implicit in cloud-build -cloud-test: cloud-build integration-tests - -## Build using local Docker daemon - -.PHONY: local-build -local-build: local-build-interpreters - docker build $(DOCKER_FLAGS) -t "$(IMAGE_NAME)" runtime-image - -.PHONY: local-build-interpreters -local-build-interpreters: - make -C python-interpreter-builder build - -.PHONY: local-test -local-test: local-build local-structure-tests local-google-cloud-python-tests integration-tests - -.PHONY: local-structure-tests -local-structure-tests: local-build ext_run.sh - make -C tests structure-tests - -# Unit tests for Google Cloud Client Library for Python -.PHONY: local-google-cloud-python-tests -local-google-cloud-python-tests: tests/google-cloud-python/Dockerfile - make -C tests google-cloud-python - -## Always local - -.PHONY: integration-tests -integration-tests: google-cloud-python-system-tests benchmarks - -# System tests for Google Cloud Client Library for Python. -# They require gcloud auth and network access. -.PHONY: google-cloud-python-system-tests -google-cloud-python-system-tests: - make -C system_tests - -.PHONY: benchmarks -benchmarks: - make -C tests benchmarks diff --git a/README.md b/README.md index 3cef3315..7c14904e 100644 --- a/README.md +++ b/README.md @@ -72,73 +72,7 @@ command or entrypoint. For example: Google regularly builds and releases this image at [`gcr.io/google-appengine/python`](https://gcr.io/google-appengine/python). -To rebuild the image yourself, first set the following variables in your -shell. You need to be authenticated to a Google Cloud Project to invoke the -Google Container Builder service, and also to run the system tests. - -```shell -$ export GOOGLE_CLOUD_PROJECT=YOUR-PROJECT-NAME -$ DOCKER_NAMESPACE=gcr.io/${GCLOUD_PROJECT} -$ CANDIDATE_NAME=`date +%Y-%m-%d_%H_%M` -$ export IMAGE_NAME=${DOCKER_NAMESPACE}/python:${CANDIDATE_NAME} -$ gcloud config set project ${GOOGLE_CLOUD_PROJECT} -``` - -To rebuild the image using the Google Container Builder service, do the -following: - -```shell -$ make cloud-build -$ make cloud-test -``` - -To rebuild the image using your local Docker daemon, do the following: - -``` shell -$ make local-build -$ make local-test -``` - -To open an interactive shell session to this image after building it, do the following: - -``` shell -docker run -it --entrypoint /bin/bash ${IMAGE_NAME} -``` - -## Running the system tests - -To run the system tests, you need a Google Cloud Project with a service account. -From the [Google Cloud Console](https://console.cloud.google.com/), either -create a new project or switch to an existing one. Next, -[create a service account]( -https://cloud.google.com/iam/docs/creating-managing-service-accounts) that will -be used to run the system tests. Once you have a service account, -[create and download a service account key](https://cloud.google.com/iam/docs/managing-service-account-keys). - -In the -[IAM & Admin](https://console.cloud.google.com/permissions/projectpermissions) -section, grant the `Owner` role to the service account you created above. Also -grant the `Editor` role to the `cloud-logs@google.com` service account. - - -Then, follow the -[system test setup instructions](https://github.com/GoogleCloudPlatform/google-cloud-python/blob/master/CONTRIBUTING.rst#running-system-tests). It -describes various steps, including running some scripts to populate and/or -delete datastore example data and indexes (populate_datastore.py, -clear_datastore.py, and `gcloud preview datastore create-indexes -system_tests/data/index.yaml`). - -From the cloud console, you will need to enable the following APIs for your project: - -- Bigquery API -- Cloud Bigtable Admin API -- Google Cloud Natural Language API -- Google Cloud Pub/Sub API -- Google Cloud Storage JSON API -- Google Cloud Vision API -- Google Translate API -- Stackdriver Logging API -- Stackdriver Monitoring API +See [RELEASING.md](RELEASING.md) for more information. ## Contributing changes diff --git a/RELEASING.md b/RELEASING.md new file mode 100644 index 00000000..887ac7ce --- /dev/null +++ b/RELEASING.md @@ -0,0 +1,130 @@ +# Google Cloud Platform - Python Runtime Docker Image + +## `build.sh` + +There is a shell script called `build.sh` that builds everything in this +repository. + +### Environment variables for `build.sh` + +DOCKER_NAMESPACE +: The prefix applied to all images names created. To push images to Google +Container Registry (GCR), this should be `gcr.io/YOUR-PROJECT-NAME`. + +TAG +: The suffix applied to all images created. This should be unique. If not +specified, the current time will be used (timestamp format `YYYY-mm-dd-HHMMSS`). + +GOOGLE_APPLICATION_CREDENTIALS +: (System test only) Path to service account credentials in JSON format. + +GOOGLE_CLOUD_PROJECT +: (System test only) Name of the Google Cloud Platform project to run the system +tests under. + +## Building and Releasing + +A custom Jenkins job builds and releases this repository using scripts and job +configurations that are not yet available publicly. The control flow is as +follows: + +1. Jenkins job `python/release` is invoked by + a. Manually running the script `build_and_release.py` with arguments + b. Manually invoking the job from the GUI +2. The job runs the script `release.sh` + a. Service account credentials are read + b. `gcloud auth activate-service-account` is performed + c. `gcloud config set project` is performed +3. The script invokes `build.sh` in this repository +4. `build.sh` invokes Google Container Builder with the `cloudbuild-*.yaml` + config files. + +## Building outside Jenkins + +To build this repository outside Jenkins, authenticate and authorize yourself +with `gcloud auth`, set the variables listed above, and run: + +``` shell +./build.sh +``` + +This assumes an environment similar to the internal Jenkins environment (Linux, +Debian or Ubuntu-like). + +## Building locally + +To build this repository using local Docker commands instead of the Google +Container Builder service, add the `--local` flag as shown: + +``` shell +./build.sh --local +``` + +To open an interactive shell session to this image after building it, do the +following: + +``` shell +docker run -it --entrypoint /bin/bash YOUR-IMAGE-NAME +``` + +## Running benchmarks + +There is a benchmark suite which compares the performance of interpreters +against each other. + +``` shell +./build.sh --nobuild --benchmark +``` + +Since these benchmarks are run on cloud instances, the timings may vary from run +to run. + +## Running system tests + +**TAKE NOTE: You will incur charges for use of Google Cloud Platform services!** + +System tests perform mutating operations against the real Google Cloud services. +Since these system tests may fail or be flaky for outside reasons such as +netorking issues, configuration errors, or services outages, they are run +separately from building the images, and should be run in their own project. + +To run the system tests, you need a Google Cloud Project with a service account. +From the [Google Cloud Console](https://console.cloud.google.com/), either +create a new project or switch to an existing one. Next, +[create a service account]( +https://cloud.google.com/iam/docs/creating-managing-service-accounts) that will +be used to run the system tests. Once you have a service account, +[create and download a service account key](https://cloud.google.com/iam/docs/managing-service-account-keys). + +In the +[IAM & Admin](https://console.cloud.google.com/permissions/projectpermissions) +section, grant the `Owner` role to the service account you created above. Also +grant the `Editor` role to the `cloud-logs@google.com` service account. + +Then, follow the +[system test setup instructions](https://github.com/GoogleCloudPlatform/google-cloud-python/blob/master/CONTRIBUTING.rst#running-system-tests). It +describes various steps, including running some scripts to populate and/or +delete datastore example data and indexes (populate_datastore.py, +clear_datastore.py, and `gcloud preview datastore create-indexes +system_tests/data/index.yaml`). + +From the cloud console, you will need to enable at least the following APIs for +your project: + +- Bigquery API +- Cloud Bigtable Admin API +- Cloud Spanner API +- Google Cloud Natural Language API +- Google Cloud Pub/Sub API +- Google Cloud Speech API +- Google Cloud Storage JSON API +- Google Cloud Translation API +- Google Cloud Vision API +- Stackdriver Logging API +- Stackdriver Monitoring API + +Once all the setup has been done, run the following: + +``` shell +./build.sh --nobuild --system_tests +``` diff --git a/build.sh b/build.sh index 2eae6ce3..5e4302ba 100755 --- a/build.sh +++ b/build.sh @@ -16,13 +16,140 @@ set -euo pipefail -export IMAGE_NAME=$1 +# Actions +benchmark=0 # Should run benchmarks? +build=1 # Should build images? +system_tests=0 # Should run system tests? -if [ -z "$IMAGE_NAME" ]; then - echo "Usage: ./build.sh [image_path]" - echo "Please provide fully qualified path to target image." +local=0 # Should run using local Docker daemon instead of GCR? + +# Note that $gcloud_cmd has spaces in it +gcloud_cmd="gcloud beta container builds submit ." +local_gcloud_cmd="scripts/local_cloudbuild.py" + +# Helper functions +function fatal() { + echo "$1" >&2 exit 1 +} + +function usage { + fatal "Usage: $0 [OPTION]... +Build and test artifacts in this repository + +Options: + --benchmark: Run benchmarking suite + --[no]build: Build all images (default) + --local: Build images using local Docker daemon + --system_tests: Run system tests +" +} + +# Read environment variables +if [ -z "${DOCKER_NAMESPACE+set}" ] ; then + fatal 'Error: $DOCKER_NAMESPACE is not set; invoke with something like DOCKER_NAMESPACE=gcr.io/YOUR-PROJECT-NAME' +fi + +if [ -z "${TAG+set}" ] ; then + export TAG=`date +%Y-%m-%d-%H%M%S` +fi + +substitutions="_DOCKER_NAMESPACE=${DOCKER_NAMESPACE},_TAG=${TAG}" + +# Read command line arguments +while [ $# -gt 0 ]; do + case "$1" in + --benchmark) + benchmark=1 + shift + ;; + --build) + build=1 + shift + ;; + --nobuild) + build=0 + shift + ;; + --local) + local=1 + shift + ;; + --system_tests) + system_tests=1 + shift + ;; + *) + usage + ;; + esac +done + +# If no actions chosen, then tell the user +if [ "${benchmark}" -eq 0 -a "${build}" -eq 0 -a "${system_tests}" -eq 0 ]; then + fatal 'Error: No actions specified (for example, --build), exiting' +fi + +# Running build local or remote? +if [ "${local}" -eq 1 ]; then + gcloud_cmd="${local_gcloud_cmd}" +fi + +# Read action-specific environment variables +if [ "${system_tests}" -eq 1 ]; then + if [ -z "${GOOGLE_APPLICATION_CREDENTIALS+set}" ] ; then + fatal 'Error: $GOOGLE_APPLICATION_CREDENTIALS is not set; invoke with something like GOOGLE_APPLICATION_CREDENTIALS=/path/to/service/account/creds.json' + fi + + if [ -z "${GOOGLE_CLOUD_PROJECT+set}" ] ; then + fatal 'Error: $GOOGLE_CLOUD_PROJECT is not set; invoke with something like GOOGLE_CLOUD_PROJECT=YOUR-PROJECT-NAME' + fi +fi + +# Use latest released Debian as our base image +export DEBIAN_BASE_IMAGE="gcr.io/google-appengine/debian8:latest" +export FULL_BASE_IMAGE="${DOCKER_NAMESPACE}/python:${TAG}" +echo "Using base image name ${FULL_BASE_IMAGE}" + +# Generate Dockerfiles +for outfile in \ + python-interpreter-builder/Dockerfile \ + runtime-image/Dockerfile \ + tests/benchmark/Dockerfile \ + tests/google-cloud-python/Dockerfile \ + tests/google-cloud-python-system/Dockerfile \ + tests/integration/Dockerfile \ + ; do + envsubst <"${outfile}".in >"${outfile}" '$DEBIAN_BASE_IMAGE $FULL_BASE_IMAGE $GOOGLE_CLOUD_PROJECT' +done + +# Build images and push to GCR +if [ "${build}" -eq 1 ]; then + echo "Building images" + ${gcloud_cmd} --config cloudbuild.yaml --substitutions "${substitutions}" +fi + +# If both system tests and benchmarks are requested, run them both +# even if one or the other has errors. If the build step had errors, +# this script will have already exited. +exit_code=0 + +# Run system tests +if [ "${system_tests}" -eq 1 ]; then + echo "Running system tests using project ${GOOGLE_CLOUD_PROJECT}" + + trap "rm -f tests/google-cloud-python-system/credentials.json" EXIT + cp "${GOOGLE_APPLICATION_CREDENTIALS}" tests/google-cloud-python-system/credentials.json + ${gcloud_cmd} --config cloudbuild_system_tests.yaml --substitutions "${substitutions}" || \ + exit_code=1 + rm -f tests/google-cloud-python-system/credentials.json +fi + +# Run benchmarks +if [ "${benchmark}" -eq 1 ] ; then + echo "Running benchmark" + ${gcloud_cmd} --config cloudbuild_benchmark.yaml --substitutions "${substitutions}" || \ + exit_code=1 fi -envsubst < cloudbuild.yaml.in > cloudbuild.yaml -gcloud beta container builds submit . --config=cloudbuild.yaml +exit ${exit_code} diff --git a/cloudbuild.yaml b/cloudbuild.yaml new file mode 100644 index 00000000..32c501b2 --- /dev/null +++ b/cloudbuild.yaml @@ -0,0 +1,34 @@ +timeout: 7200s +steps: +- name: gcr.io/cloud-builders/docker:latest + # Compile Python interpreters from source + args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/interpreter-builder:${_TAG}', + '--no-cache', '/workspace/python-interpreter-builder/'] +- name: ${_DOCKER_NAMESPACE}/python/interpreter-builder:${_TAG} + # Copy interpreters back to workspace + args: ['cp', '/interpreters.tar.gz', '/workspace/runtime-image/interpreters.tar.gz'] +- name: gcr.io/cloud-builders/docker:latest + # Build base runtime image + args: ['build', '--tag=${_DOCKER_NAMESPACE}/python:${_TAG}', + '--no-cache', '/workspace/runtime-image/'] +- name: gcr.io/gcp-runtimes/structure_test:latest + # Validate structure of base runtime image + args: [ + '-i', '${_DOCKER_NAMESPACE}/python:${_TAG}', + '--config', '/workspace/tests/virtualenv/virtualenv_default.yaml', + '--config', '/workspace/tests/virtualenv/virtualenv_python34.yaml', + '--config', '/workspace/tests/virtualenv/virtualenv_python35.yaml', + '--config', '/workspace/tests/no-virtualenv/no-virtualenv.yaml', + '--config', '/workspace/tests/python2-libraries/python2-libraries.yaml', + '--config', '/workspace/tests/python3-libraries/python3-libraries.yaml', + '--config', '/workspace/tests/license-test/license-test.yaml', + '-v' + ] +- name: gcr.io/cloud-builders/docker:latest + # Run google client library unit tests against base runtime image + args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/google-cloud-python:${_TAG}', + '--no-cache', '/workspace/tests/google-cloud-python/'] +- name: ${_DOCKER_NAMESPACE}/python/tests/google-cloud-python:${_TAG} +images: [ + '${_DOCKER_NAMESPACE}/python:${_TAG}' +] diff --git a/cloudbuild.yaml.in b/cloudbuild.yaml.in deleted file mode 100644 index 4c505105..00000000 --- a/cloudbuild.yaml.in +++ /dev/null @@ -1,24 +0,0 @@ -timeout: 7200s -steps: -- name: gcr.io/cloud-builders/docker - args: ['build', '--tag=interpreter', '--no-cache', '/workspace/python-interpreter-builder/'] -- name: interpreter - args: ['cp', '/interpreters.tar.gz', '/workspace/runtime-image/interpreters.tar.gz'] -- name: gcr.io/cloud-builders/docker - args: ['build', '--tag=${IMAGE_NAME}', '--no-cache', '/workspace/runtime-image/'] -- name: gcr.io/gcp-runtimes/structure_test - args: [ - '-i', '${IMAGE_NAME}', - '--config', '/workspace/tests/virtualenv/virtualenv_default.yaml', - '--config', '/workspace/tests/virtualenv/virtualenv_python34.yaml', - '--config', '/workspace/tests/virtualenv/virtualenv_python35.yaml', - '--config', '/workspace/tests/no-virtualenv/no-virtualenv.yaml', - '--config', '/workspace/tests/python2-libraries/python2-libraries.yaml', - '--config', '/workspace/tests/python3-libraries/python3-libraries.yaml', - '--config', '/workspace/tests/license-test/license-test.yaml', - '-v' - ] -- name: gcr.io/cloud-builders/docker - args: ['build', '--tag=google-cloud-python-tests', '--no-cache', '/workspace/tests/google-cloud-python/'] -images: - ['${IMAGE_NAME}'] diff --git a/cloudbuild_benchmark.yaml b/cloudbuild_benchmark.yaml new file mode 100644 index 00000000..a960bc9b --- /dev/null +++ b/cloudbuild_benchmark.yaml @@ -0,0 +1,8 @@ +timeout: 3600s +steps: +- name: gcr.io/cloud-builders/docker:latest + args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/benchmark:${_TAG}', + '--no-cache', '/workspace/tests/benchmark/'] +images: [ + # Intentionally empty +] diff --git a/cloudbuild_system_tests.yaml b/cloudbuild_system_tests.yaml new file mode 100644 index 00000000..3cac03e2 --- /dev/null +++ b/cloudbuild_system_tests.yaml @@ -0,0 +1,9 @@ +timeout: 3600s +steps: +- name: gcr.io/cloud-builders/docker:latest + args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/google-cloud-python-system:${_TAG}', + '--no-cache', '/workspace/tests/google-cloud-python-system/'] +- name: ${_DOCKER_NAMESPACE}/python/tests/google-cloud-python-system:${_TAG} +images: [ + # Intentionally empty +] diff --git a/jenkins_build.sh b/jenkins_build.sh index 253d5ca0..f88007f5 100755 --- a/jenkins_build.sh +++ b/jenkins_build.sh @@ -1,45 +1,3 @@ #!/bin/sh -set -eu - -RUNTIME_NAME="python" - -if [ -z "${TAG}" ] ; then - TAG=`date +%Y-%m-%d_%H_%M` -fi - -CANDIDATE_NAME="${TAG}" -echo "CANDIDATE_NAME:${CANDIDATE_NAME}" - -if [ -z "${DOCKER_NAMESPACE+set}" ] ; then - echo "Error: DOCKER_NAMESPACE is not set; invoke with something like DOCKER_NAMESPACE=gcr.io/YOUR-PROJECT-NAME" >&2 - exit 1 -fi -export IMAGE_NAME="${DOCKER_NAMESPACE}/${RUNTIME_NAME}:${CANDIDATE_NAME}" - -if [ -z "${GOOGLE_CLOUD_PROJECT+set}" ] ; then - echo "Error: GOOGLE_CLOUD_PROJECT is not set; invoke with something like GOOGLE_CLOUD_PROJECT=YOUR-PROJECT-NAME" >&2 - exit 1 -fi - -export FORCE_REBUILD - -echo "==================================================================" -echo "Building image ${IMAGE_NAME} and pushing to ${DOCKER_NAMESPACE} using ${GOOGLE_CLOUD_PROJECT}" -echo "==================================================================" - -make cloud-build - -# We explicitly pull the image using 'gcloud', instead of letting -# Docker do it, so that we have the right credentials. -echo "==================================================================" -gcloud info -echo gcloud docker -- pull "${IMAGE_NAME}" -gcloud docker -- pull "${IMAGE_NAME}" -echo "==================================================================" - -# Note that system test failures might be caused environment factors -# outside our control. Also, the images will be pushed to GCR by the -# previous build step regardless of system test failures. -make integration-tests || \ - echo "ERROR: System test failure, please examine the logs" +./build.sh "$@" diff --git a/python-interpreter-builder/.gitignore b/python-interpreter-builder/.gitignore index 53752db2..94143827 100644 --- a/python-interpreter-builder/.gitignore +++ b/python-interpreter-builder/.gitignore @@ -1 +1 @@ -output +Dockerfile diff --git a/python-interpreter-builder/Dockerfile b/python-interpreter-builder/Dockerfile.in similarity index 94% rename from python-interpreter-builder/Dockerfile rename to python-interpreter-builder/Dockerfile.in index e3dffd4c..54181518 100644 --- a/python-interpreter-builder/Dockerfile +++ b/python-interpreter-builder/Dockerfile.in @@ -1,6 +1,6 @@ # The Google App Engine base image is debian (jessie) with ca-certificates # installed. -FROM gcr.io/google-appengine/debian8 +FROM ${DEBIAN_BASE_IMAGE} # Install Python build dependencies RUN apt-get update && apt-get install -yq \ diff --git a/python-interpreter-builder/Makefile b/python-interpreter-builder/Makefile deleted file mode 100644 index d125b1c8..00000000 --- a/python-interpreter-builder/Makefile +++ /dev/null @@ -1,10 +0,0 @@ -.PHONY: build -build: - docker build $(DOCKER_FLAGS) -t google/python-interpreter-builder . - # Extract the built interpreters - # This is needed because `docker cp` doesn't work on images, just containers. - -docker rm python-interpreter-builder - docker run --name python-interpreter-builder google/python-interpreter-builder /bin/bash - mkdir -p output - docker cp python-interpreter-builder:/interpreters.tar.gz ../runtime-image/interpreters.tar.gz - docker rm python-interpreter-builder diff --git a/runtime-image/.gitignore b/runtime-image/.gitignore index cb1afebc..94143827 100644 --- a/runtime-image/.gitignore +++ b/runtime-image/.gitignore @@ -1 +1 @@ -interpreters.tar.gz +Dockerfile diff --git a/runtime-image/Dockerfile b/runtime-image/Dockerfile.in similarity index 96% rename from runtime-image/Dockerfile rename to runtime-image/Dockerfile.in index 03cd19fc..15cf3ee4 100644 --- a/runtime-image/Dockerfile +++ b/runtime-image/Dockerfile.in @@ -1,7 +1,7 @@ # The Google App Engine base image is debian (jessie) with ca-certificates # installed. # Source: https://github.com/GoogleCloudPlatform/debian-docker -FROM gcr.io/google-appengine/debian8 +FROM ${DEBIAN_BASE_IMAGE} ADD resources /resources ADD scripts /scripts diff --git a/system_tests/.gitignore b/system_tests/.gitignore deleted file mode 100644 index 3a8c6aa8..00000000 --- a/system_tests/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -data/ -secrets.tar -Dockerfile diff --git a/system_tests/Dockerfile.in b/system_tests/Dockerfile.in deleted file mode 100644 index 97f8e004..00000000 --- a/system_tests/Dockerfile.in +++ /dev/null @@ -1,15 +0,0 @@ -FROM ${IMAGE_NAME} - -# Secrets injected at runtime -ENV GOOGLE_APPLICATION_CREDENTIALS=/app/credentials/credentials.json -ENV GOOGLE_CLOUD_PROJECT=${GOOGLE_CLOUD_PROJECT} - -# Get the source. -RUN git clone --depth 1 https://github.com/GoogleCloudPlatform/google-cloud-python.git -WORKDIR google-cloud-python - -# Install tox for running the system tests -RUN pip install --upgrade tox - -# Run Python 2.7, 3.5 system tests -ENTRYPOINT ["tox", "-e", "system-tests,system-tests3"] diff --git a/system_tests/Makefile b/system_tests/Makefile deleted file mode 100644 index cec219ca..00000000 --- a/system_tests/Makefile +++ /dev/null @@ -1,22 +0,0 @@ -# Use no-cache to prevent layer caching because there is a layer that does -# a `git clone` which can not be cached. -DOCKER_FLAGS ?= --no-cache - -ifndef GOOGLE_APPLICATION_CREDENTIALS -$(error GOOGLE_APPLICATION_CREDENTIALS is not set; download service account credentials in JSON format from the Google Cloud Console and invoke make with something like GOOGLE_APPLICATION_CREDENTIALS=/path/to/credentials.json) -endif - -ifndef GOOGLE_CLOUD_PROJECT -$(error GOOGLE_CLOUD_PROJECT is not set; invoke make with something like GOOGLE_CLOUD_PROJECT=my-project-name) -endif - - -.PHONY: all -all: Dockerfile - @echo "Running system tests in project ${GOOGLE_CLOUD_PROJECT} using service account credentials from ${GOOGLE_APPLICATION_CREDENTIALS}" - docker build --tag google-cloud-python-system-tests $(DOCKER_FLAGS) . - docker run --rm -v $(GOOGLE_APPLICATION_CREDENTIALS):/app/credentials/credentials.json google-cloud-python-system-tests - -.PHONY: Dockerfile -Dockerfile: Dockerfile.in - envsubst < $< > $@ diff --git a/tests/Makefile b/tests/Makefile deleted file mode 100644 index 0535885b..00000000 --- a/tests/Makefile +++ /dev/null @@ -1,18 +0,0 @@ -.PHONY: all -all: structure-tests benchmarks google-cloud-python - -.PHONY: structure-tests -structure-tests: - make -C no-virtualenv - make -C virtualenv - make -C python2-libraries - make -C python3-libraries - make -C license-test - -.PHONY: benchmarks -benchmarks: - make -C benchmark all - -.PHONY: google-cloud-python -google-cloud-python: - make -C google-cloud-python all diff --git a/tests/benchmark/Dockerfile.2vs3 b/tests/benchmark/Dockerfile.2vs3 deleted file mode 100644 index 507cd4bb..00000000 --- a/tests/benchmark/Dockerfile.2vs3 +++ /dev/null @@ -1 +0,0 @@ -RUN python perf.py -r -b default /usr/bin/python2.7 /usr/bin/python3.4 diff --git a/tests/benchmark/Dockerfile.34vs35 b/tests/benchmark/Dockerfile.34vs35 deleted file mode 100644 index d319d14d..00000000 --- a/tests/benchmark/Dockerfile.34vs35 +++ /dev/null @@ -1 +0,0 @@ -RUN python perf.py -r -b default /usr/bin/python3.4 /opt/python3.5/bin/python3.5 diff --git a/tests/benchmark/Dockerfile.in b/tests/benchmark/Dockerfile.in index c202cb2f..e39eac1c 100644 --- a/tests/benchmark/Dockerfile.in +++ b/tests/benchmark/Dockerfile.in @@ -1,4 +1,6 @@ -FROM ${IMAGE_NAME} +FROM ${FULL_BASE_IMAGE} -RUN hg clone https://hg.python.org/benchmarks /app/benchmarks +RUN hg clone -r 9923b81a1d34 https://hg.python.org/benchmarks /app/benchmarks WORKDIR /app/benchmarks +RUN python perf.py -r -b default /usr/bin/python2.7 /usr/bin/python3.4 +RUN python perf.py -r -b default /usr/bin/python3.4 /opt/python3.5/bin/python3.5 diff --git a/tests/benchmark/Makefile b/tests/benchmark/Makefile deleted file mode 100644 index dd56208a..00000000 --- a/tests/benchmark/Makefile +++ /dev/null @@ -1,14 +0,0 @@ -.PHONY: all -all: 2vs3 34vs35 - -# Tests Python 3.4 against Python 2.7. -.PHONY: 2vs3 -2vs3: - envsubst < Dockerfile.in > Dockerfile - cat Dockerfile Dockerfile.2vs3 | docker build - - -# Tests Python 3.5 against Python 3.4. -.PHONY: 2vs3 -34vs35: - envsubst < Dockerfile.in > Dockerfile - cat Dockerfile Dockerfile.34vs35 | docker build - diff --git a/tests/google-cloud-python-system/.gitignore b/tests/google-cloud-python-system/.gitignore new file mode 100644 index 00000000..470715ec --- /dev/null +++ b/tests/google-cloud-python-system/.gitignore @@ -0,0 +1,2 @@ +credentials.json +Dockerfile diff --git a/tests/google-cloud-python-system/Dockerfile.in b/tests/google-cloud-python-system/Dockerfile.in new file mode 100644 index 00000000..86a2d97d --- /dev/null +++ b/tests/google-cloud-python-system/Dockerfile.in @@ -0,0 +1,16 @@ +FROM ${FULL_BASE_IMAGE} + +# Get the source. +RUN git clone --depth 1 https://github.com/GoogleCloudPlatform/google-cloud-python.git +WORKDIR google-cloud-python + +# Install nox +RUN pip install --upgrade nox-automation + +# Secrets injected at runtime +ENV GOOGLE_APPLICATION_CREDENTIALS=/workspace/tests/google-cloud-python-system/credentials.json +ENV GOOGLE_CLOUD_PROJECT=${GOOGLE_CLOUD_PROJECT} + +# Run system tests for all supported Python versions +ADD run_system_tests.sh /run_system_tests.sh +ENTRYPOINT ["/run_system_tests.sh"] diff --git a/tests/google-cloud-python-system/run_system_tests.sh b/tests/google-cloud-python-system/run_system_tests.sh new file mode 100755 index 00000000..aab10c88 --- /dev/null +++ b/tests/google-cloud-python-system/run_system_tests.sh @@ -0,0 +1,34 @@ +#!/bin/sh +set -eu + +cd /app/google-cloud-python + +# Not all packages have system tests +packages=" +bigquery +bigtable +datastore +language +logging +monitoring +pubsub +spanner +speech +storage +vision +" + +# translate has system test but it gives error message: +# BadRequest: 400 Invalid JSON payload received. Unknown name "model": Cannot bind 'nmt'. Field 'model' could not be found in request message. (GET https://translation.googleapis.com/language/translate/v2?target=de&q=hvala+ti&q=dankon&q=Me+llamo+Jeff&q=My+name+is+Jeff&model=nmt) +disabled_packages="translate" + +# Spanner system test needs this +export GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE=1 + +exit_code=0 +for package in ${packages}; do + noxfile="${package}/nox.py" + nox -f "${noxfile}" -e "system_tests(python_version='2.7')" || exit_code=1 +done + +exit "${exit_code}" diff --git a/tests/google-cloud-python/Dockerfile.in b/tests/google-cloud-python/Dockerfile.in index 94bf4648..ada429fb 100644 --- a/tests/google-cloud-python/Dockerfile.in +++ b/tests/google-cloud-python/Dockerfile.in @@ -1,17 +1,12 @@ -FROM ${IMAGE_NAME} - -# Install tox -RUN pip install --upgrade tox +FROM ${FULL_BASE_IMAGE} # Get the source. -RUN git clone https://github.com/GoogleCloudPlatform/google-cloud-python.git +RUN git clone --depth 1 https://github.com/GoogleCloudPlatform/google-cloud-python.git WORKDIR google-cloud-python -# Run Python 2.7 unit tests -RUN python2.7 scripts/run_unit_tests.py - -# Run Python 3.4 unit tests -RUN python3.4 scripts/run_unit_tests.py +# Install nox +RUN pip install --upgrade nox-automation -# Run Python 3.5 unit tests -RUN python3.5 scripts/run_unit_tests.py +# Run unit tests for all supported Python versions +ADD run_unit_tests.sh /run_unit_tests.sh +ENTRYPOINT ["/run_unit_tests.sh"] diff --git a/tests/google-cloud-python/Makefile b/tests/google-cloud-python/Makefile deleted file mode 100644 index 6f9c5522..00000000 --- a/tests/google-cloud-python/Makefile +++ /dev/null @@ -1,5 +0,0 @@ -.PHONY: all -all: - # Use no-cache to prevent layer caching because there is a layer that does - # a `git clone` which can not be cached. - docker build --no-cache . diff --git a/tests/google-cloud-python/run_unit_tests.sh b/tests/google-cloud-python/run_unit_tests.sh new file mode 100755 index 00000000..8563606e --- /dev/null +++ b/tests/google-cloud-python/run_unit_tests.sh @@ -0,0 +1,11 @@ +#!/bin/sh +set -eu + +cd /app/google-cloud-python + +exit_code=0 +for noxfile in */nox.py; do + nox -f "${noxfile}" -e "unit_tests(python_version='2.7')" "unit_tests(python_version='3.4')" "unit_tests(python_version='3.5')" || exit_code=1 +done + +exit "${exit_code}" diff --git a/tests/integration/.gitignore b/tests/integration/.gitignore new file mode 100644 index 00000000..94143827 --- /dev/null +++ b/tests/integration/.gitignore @@ -0,0 +1 @@ +Dockerfile diff --git a/tests/integration/Dockerfile.in b/tests/integration/Dockerfile.in index e18d60ef..52f848f6 100644 --- a/tests/integration/Dockerfile.in +++ b/tests/integration/Dockerfile.in @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM ${STAGING_IMAGE} +FROM ${FULL_BASE_IMAGE} COPY . /app WORKDIR /app diff --git a/tests/license-test/Makefile b/tests/license-test/Makefile deleted file mode 100644 index b56b60ea..00000000 --- a/tests/license-test/Makefile +++ /dev/null @@ -1,3 +0,0 @@ -.PHONY: all -all: - ../../ext_run.sh -i "${IMAGE_NAME}" -c license-test.yaml -v diff --git a/tests/no-virtualenv/Makefile b/tests/no-virtualenv/Makefile deleted file mode 100644 index 2d9ec8e3..00000000 --- a/tests/no-virtualenv/Makefile +++ /dev/null @@ -1,3 +0,0 @@ -.PHONY: all -all: - ../../ext_run.sh -i "${IMAGE_NAME}" -c no-virtualenv.yaml -v diff --git a/tests/python2-libraries/Makefile b/tests/python2-libraries/Makefile deleted file mode 100644 index 13ab239e..00000000 --- a/tests/python2-libraries/Makefile +++ /dev/null @@ -1,3 +0,0 @@ -.PHONY: all -all: - ../../ext_run.sh -i "${IMAGE_NAME}" -c python2-libraries.yaml -w ../.. -v diff --git a/tests/python3-libraries/Makefile b/tests/python3-libraries/Makefile deleted file mode 100644 index 92fb683b..00000000 --- a/tests/python3-libraries/Makefile +++ /dev/null @@ -1,3 +0,0 @@ -.PHONY: all -all: - ../../ext_run.sh -i "${IMAGE_NAME}" -c python3-libraries.yaml -w ../.. -v diff --git a/tests/virtualenv/Makefile b/tests/virtualenv/Makefile deleted file mode 100644 index 66a19642..00000000 --- a/tests/virtualenv/Makefile +++ /dev/null @@ -1,5 +0,0 @@ -.PHONY: all -all: - ../../ext_run.sh -i "${IMAGE_NAME}" -c virtualenv_default.yaml -v - ../../ext_run.sh -i "${IMAGE_NAME}" -c virtualenv_python34.yaml -v - ../../ext_run.sh -i "${IMAGE_NAME}" -c virtualenv_python35.yaml -v From 7bee8a75c65c3eab1e23e87faa86492a08dbf93f Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 7 Apr 2017 14:43:43 -0700 Subject: [PATCH 143/362] Change handling of GOOGLE_CLOUD_PROJECT in build.sh. The previous behavior could possibly, conceivably be causing problems. Now, the user now specifies GOOGLE_CLOUD_PROJECT_FOR_TESTS, which should not collide with anything, and the GOOGLE_CLOUD_PROJECT environment variable is only set for the single thing that should need it, which is run_system_tests.sh, and only in the single, ephemeral Docker container where we run those system tests. --- RELEASING.md | 2 +- build.sh | 8 ++++---- tests/google-cloud-python-system/Dockerfile.in | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/RELEASING.md b/RELEASING.md index 887ac7ce..01f7077d 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -18,7 +18,7 @@ specified, the current time will be used (timestamp format `YYYY-mm-dd-HHMMSS`). GOOGLE_APPLICATION_CREDENTIALS : (System test only) Path to service account credentials in JSON format. -GOOGLE_CLOUD_PROJECT +GOOGLE_CLOUD_PROJECT_FOR_TESTS : (System test only) Name of the Google Cloud Platform project to run the system tests under. diff --git a/build.sh b/build.sh index 5e4302ba..bc80277c 100755 --- a/build.sh +++ b/build.sh @@ -101,8 +101,8 @@ if [ "${system_tests}" -eq 1 ]; then fatal 'Error: $GOOGLE_APPLICATION_CREDENTIALS is not set; invoke with something like GOOGLE_APPLICATION_CREDENTIALS=/path/to/service/account/creds.json' fi - if [ -z "${GOOGLE_CLOUD_PROJECT+set}" ] ; then - fatal 'Error: $GOOGLE_CLOUD_PROJECT is not set; invoke with something like GOOGLE_CLOUD_PROJECT=YOUR-PROJECT-NAME' + if [ -z "${GOOGLE_CLOUD_PROJECT_FOR_TESTS+set}" ] ; then + fatal 'Error: $GOOGLE_CLOUD_PROJECT_FOR_TESTS is not set; invoke with something like GOOGLE_CLOUD_PROJECT_FOR_TESTS=YOUR-PROJECT-NAME' fi fi @@ -120,7 +120,7 @@ for outfile in \ tests/google-cloud-python-system/Dockerfile \ tests/integration/Dockerfile \ ; do - envsubst <"${outfile}".in >"${outfile}" '$DEBIAN_BASE_IMAGE $FULL_BASE_IMAGE $GOOGLE_CLOUD_PROJECT' + envsubst <"${outfile}".in >"${outfile}" '$DEBIAN_BASE_IMAGE $FULL_BASE_IMAGE $GOOGLE_CLOUD_PROJECT_FOR_TESTS' done # Build images and push to GCR @@ -136,7 +136,7 @@ exit_code=0 # Run system tests if [ "${system_tests}" -eq 1 ]; then - echo "Running system tests using project ${GOOGLE_CLOUD_PROJECT}" + echo "Running system tests using project ${GOOGLE_CLOUD_PROJECT_FOR_TESTS}" trap "rm -f tests/google-cloud-python-system/credentials.json" EXIT cp "${GOOGLE_APPLICATION_CREDENTIALS}" tests/google-cloud-python-system/credentials.json diff --git a/tests/google-cloud-python-system/Dockerfile.in b/tests/google-cloud-python-system/Dockerfile.in index 86a2d97d..18506681 100644 --- a/tests/google-cloud-python-system/Dockerfile.in +++ b/tests/google-cloud-python-system/Dockerfile.in @@ -9,7 +9,7 @@ RUN pip install --upgrade nox-automation # Secrets injected at runtime ENV GOOGLE_APPLICATION_CREDENTIALS=/workspace/tests/google-cloud-python-system/credentials.json -ENV GOOGLE_CLOUD_PROJECT=${GOOGLE_CLOUD_PROJECT} +ENV GOOGLE_CLOUD_PROJECT=${GOOGLE_CLOUD_PROJECT_FOR_TESTS} # Run system tests for all supported Python versions ADD run_system_tests.sh /run_system_tests.sh From 9f6233a2be614ad136270bee44169de1d7c0c329 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 7 Apr 2017 15:55:51 -0700 Subject: [PATCH 144/362] Rename GOOGLE_APPLICATIONS_CREDENTIALS. Changed to GOOGLE_APPLICATIONS_CREDENTIALS_FOR_TESTS to match the handling of GOOGLE_CLOUD_PROJECT. --- RELEASING.md | 2 +- build.sh | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/RELEASING.md b/RELEASING.md index 01f7077d..1f92084a 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -15,7 +15,7 @@ TAG : The suffix applied to all images created. This should be unique. If not specified, the current time will be used (timestamp format `YYYY-mm-dd-HHMMSS`). -GOOGLE_APPLICATION_CREDENTIALS +GOOGLE_APPLICATION_CREDENTIALS_FOR_TESTS : (System test only) Path to service account credentials in JSON format. GOOGLE_CLOUD_PROJECT_FOR_TESTS diff --git a/build.sh b/build.sh index bc80277c..5929fbb2 100755 --- a/build.sh +++ b/build.sh @@ -97,8 +97,8 @@ fi # Read action-specific environment variables if [ "${system_tests}" -eq 1 ]; then - if [ -z "${GOOGLE_APPLICATION_CREDENTIALS+set}" ] ; then - fatal 'Error: $GOOGLE_APPLICATION_CREDENTIALS is not set; invoke with something like GOOGLE_APPLICATION_CREDENTIALS=/path/to/service/account/creds.json' + if [ -z "${GOOGLE_APPLICATION_CREDENTIALS_FOR_TESTS+set}" ] ; then + fatal 'Error: $GOOGLE_APPLICATION_CREDENTIALS_FOR_TESTS is not set; invoke with something like GOOGLE_APPLICATION_CREDENTIALS_FOR_TESTS=/path/to/service/account/creds.json' fi if [ -z "${GOOGLE_CLOUD_PROJECT_FOR_TESTS+set}" ] ; then @@ -139,7 +139,7 @@ if [ "${system_tests}" -eq 1 ]; then echo "Running system tests using project ${GOOGLE_CLOUD_PROJECT_FOR_TESTS}" trap "rm -f tests/google-cloud-python-system/credentials.json" EXIT - cp "${GOOGLE_APPLICATION_CREDENTIALS}" tests/google-cloud-python-system/credentials.json + cp "${GOOGLE_APPLICATION_CREDENTIALS_FOR_TESTS}" tests/google-cloud-python-system/credentials.json ${gcloud_cmd} --config cloudbuild_system_tests.yaml --substitutions "${substitutions}" || \ exit_code=1 rm -f tests/google-cloud-python-system/credentials.json From 092807920eeddfc37b403ff5210d8431b6fd74c6 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Mon, 10 Apr 2017 09:08:20 +0000 Subject: [PATCH 145/362] Auto-update dependencies. --- tests/integration/requirements.txt | 8 ++--- tests/python2-libraries/requirements.txt | 44 ++++++++++++------------ tests/python3-libraries/requirements.txt | 32 ++++++++--------- 3 files changed, 42 insertions(+), 42 deletions(-) diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index c5004b32..c5c38c56 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -1,7 +1,7 @@ -Flask==0.12 -google-cloud-error-reporting==0.23.2 -google-cloud-logging==0.23.1 -google-cloud-monitoring==0.23.0 +Flask==0.12.1 +google-cloud-error-reporting==0.24.0 +google-cloud-logging==1.0.0 +google-cloud-monitoring==0.24.0 gunicorn==19.7.1 requests==2.13.0 retrying==1.3.3 diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index d92af43f..48d0bf43 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -6,7 +6,7 @@ anyjson==0.3.3 apache-libcloud==1.5.0 argparse==1.4.0 astroid==1.4.9 -awscli==1.11.68 +awscli==1.11.75 babel==2.4.0 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 @@ -16,7 +16,7 @@ billiard==3.5.0.2 blessings==1.6 blinker==1.4 boto==2.46.1 -botocore==1.5.31 +botocore==1.5.38 bottle==0.12.13 carbon==0.9.15 celery==4.0.2 @@ -24,7 +24,7 @@ certifi==2017.1.23 cffi==1.10.0 chardet==2.3.0 click==6.7 -cliff==2.4.0 +cliff==2.5.0 cmd2==0.7.0 colorama==0.3.7 configobj==5.0.6 @@ -38,21 +38,21 @@ decorator==4.0.11 django-celery==3.2.1 django-debug-toolbar==1.7 django-extensions==1.7.8 -django==1.10.6 +django==1.11 django_compress==1.0.1 djangorestframework==3.6.2 docker-py==1.10.6 docopt==0.6.2 docutils==0.13.1 ecdsa==0.13 -elasticsearch==5.2.0 +elasticsearch==5.3.0 enum34==1.1.6 -eventlet==0.20.1 +eventlet==0.21.0 extras==1.0.0 fabric==1.13.1 fixtures==3.0.0 flake8==3.3.0 -flask==0.12 +flask==0.12.1 funcsigs==1.0.2 functools32==3.2.3.post2 futures==3.0.5 @@ -70,7 +70,7 @@ ipython==5.3.0 iso8601==0.1.11 isodate==0.5.4 itsdangerous==0.24 -jinja2==2.9.5 +jinja2==2.9.6 jmespath==0.9.2 jsonschema==2.6.0 kombu==4.0.2 @@ -87,7 +87,7 @@ mccabe==0.6.1 meld3==1.0.2 mock==2.0.0 mozcrash==1.0 -mozdevice==0.48 +mozdevice==0.50 mozfile==1.2 mozinfo==0.9 mozlog==3.4 @@ -107,7 +107,7 @@ oauth2==1.9.0.post1 oauth2client==4.0.0 oauthlib==2.0.2 ordereddict==1.1 -oslo.config==3.23.0 +oslo.config==3.24.0 pandas==0.19.2 paramiko==2.1.2 passlib==1.7.1 @@ -118,7 +118,7 @@ pbr==2.0.0 pep8==1.7.0 pexpect==4.2.1 pika==0.10.0 -pillow==4.0.0 +pillow==4.1.0 pip==9.0.1 prettytable protobuf==3.2.0 @@ -136,7 +136,7 @@ pyjwt==1.4.2 pylibmc==1.5.2 pylint==1.6.5 pymongo==3.4.0 -pymysql==0.7.10 +pymysql==0.7.11 pyopenssl==16.2.0 pyparsing==2.2.0 pyramid==1.8.3 @@ -150,7 +150,7 @@ python-gflags==3.1.1 python-keystoneclient==3.10.0 python-memcached==1.58 python-mimeparse==1.6.0 -python-novaclient==7.1.0 +python-novaclient==8.0.0 python-subunit==1.2.0 python-swiftclient==3.3.0 pytz==2017.2 @@ -164,17 +164,17 @@ requests==2.13.0 retrying==1.3.3 rsa==3.4.2 scipy==0.19.0 -selenium==3.3.1 +selenium==3.3.3 setuptools-git==1.2 -setuptools==34.3.3 -sh==1.12.11 +setuptools==34.4.0 +sh==1.12.13 simplejson==3.10.0 six==1.10.0 snowballstemmer==1.2.1 south==1.0.2 -sphinx==1.5.3 +sphinx==1.5.5 sqlalchemy-migrate==0.11.0 -sqlalchemy==1.1.7 +sqlalchemy==1.1.9 sqlparse==0.2.3 statsd==3.2.1 stevedore==1.21.0 @@ -183,15 +183,15 @@ supervisor==3.3.1 testrepository==0.0.20 testtools==2.2.0 thrift==0.10.0 -tornado==4.4.2 -tox==2.6.0 +tornado==4.4.3 +tox==2.7.0 twisted==17.1.0 ujson==1.35 unidecode==0.4.20 unittest2==1.1.0 uritemplate==3.0.0 urllib3==1.20 -uwsgi==2.0.14 +uwsgi==2.0.15 versiontools==1.9.1 virtualenv==15.1.0 waitress==1.0.2 @@ -202,5 +202,5 @@ webtest==2.0.27 werkzeug==0.12.1 wheel==0.29.0 xlrd==1.0.0 -zc.buildout==2.9.2 +zc.buildout==2.9.3 zope.interface==4.3.3 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index a7269999..41b23682 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -18,7 +18,7 @@ certifi==2017.1.23 cffi==1.10.0 chardet==2.3.0 click==6.7 -cliff==2.4.0 +cliff==2.5.0 cmd2==0.7.0 colorama==0.3.7 configobj==5.0.6 @@ -31,19 +31,19 @@ decorator==4.0.11 django-celery==3.2.1 django-debug-toolbar==1.7 django-extensions==1.7.8 -django==1.10.6 +django==1.11 django_compress==1.0.1 djangorestframework==3.6.2 docker-py==1.10.6 docopt==0.6.2 docutils==0.13.1 ecdsa==0.13 -elasticsearch==5.2.0 -eventlet==0.20.1 +elasticsearch==5.3.0 +eventlet==0.21.0 extras==1.0.0 fixtures==3.0.0 flake8==3.3.0 -flask==0.12 +flask==0.12.1 funcsigs==1.0.2 futures==3.0.5 gevent==1.2.1 @@ -59,7 +59,7 @@ ipython==5.3.0 iso8601==0.1.11 isodate==0.5.4 itsdangerous==0.24 -jinja2==2.9.5 +jinja2==2.9.6 jsonschema==2.6.0 kombu==4.0.2 linecache2==1.0.0 @@ -83,7 +83,7 @@ numpy==1.12.1 oauth2==1.9.0.post1 oauth2client==4.0.0 oauthlib==2.0.2 -oslo.config==3.23.0 +oslo.config==3.24.0 pandas==0.19.2 passlib==1.7.1 paste==2.0.3 @@ -91,7 +91,7 @@ pastedeploy==1.5.2 pastescript==2.0.2 pep8==1.7.0 pexpect==4.2.1 -pillow==4.0.0 +pillow==4.1.0 pip==9.0.1 prettytable protobuf==3.2.0 @@ -109,7 +109,7 @@ pyjwt==1.4.2 pylibmc==1.5.2 pylint==1.6.5 pymongo==3.4.0 -pymysql==0.7.10 +pymysql==0.7.11 pyopenssl==16.2.0 pyparsing==2.2.0 pyramid==1.8.3 @@ -121,7 +121,7 @@ python-gflags==3.1.1 python-keystoneclient==3.10.0 python-memcached==1.58 python-mimeparse==1.6.0 -python-novaclient==7.1.0 +python-novaclient==8.0.0 python-subunit==1.2.0 python-swiftclient==3.3.0 pytz==2017.2 @@ -136,22 +136,22 @@ retrying==1.3.3 rsa==3.4.2 scipy==0.19.0 setuptools-git==1.2 -sh==1.12.11 +sh==1.12.13 simplejson==3.10.0 six==1.10.0 snowballstemmer==1.2.1 south==1.0.2 -sphinx==1.5.3 +sphinx==1.5.5 sqlalchemy-migrate==0.11.0 -sqlalchemy==1.1.7 +sqlalchemy==1.1.9 sqlparse==0.2.3 statsd==3.2.1 stevedore==1.21.0 testrepository==0.0.20 testtools==2.2.0 thrift==0.10.0 -tornado==4.4.2 -tox==2.6.0 +tornado==4.4.3 +tox==2.7.0 twisted==17.1.0 ujson==1.35 unidecode==0.4.20 @@ -167,5 +167,5 @@ webtest==2.0.27 werkzeug==0.12.1 wheel==0.29.0 xlrd==1.0.0 -zc.buildout==2.9.2 +zc.buildout==2.9.3 zope.interface==4.3.3 From 25de7309a6c959c2894411a8fff535a08c19e993 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 7 Apr 2017 15:15:47 -0700 Subject: [PATCH 146/362] Harmonize Python build flags with Debian. Also check sha256 of Python tarball when downloading. --- python-interpreter-builder/.dockerignore | 1 - python-interpreter-builder/Dockerfile.in | 33 ++++- python-interpreter-builder/README.md | 9 +- .../scripts/build-python-3.5.sh | 124 +++++++++++++++++- tests/python3-libraries/requirements.txt | 37 +++++- 5 files changed, 188 insertions(+), 16 deletions(-) diff --git a/python-interpreter-builder/.dockerignore b/python-interpreter-builder/.dockerignore index 53752db2..e69de29b 100644 --- a/python-interpreter-builder/.dockerignore +++ b/python-interpreter-builder/.dockerignore @@ -1 +0,0 @@ -output diff --git a/python-interpreter-builder/Dockerfile.in b/python-interpreter-builder/Dockerfile.in index 54181518..c3b7dada 100644 --- a/python-interpreter-builder/Dockerfile.in +++ b/python-interpreter-builder/Dockerfile.in @@ -2,20 +2,43 @@ # installed. FROM ${DEBIAN_BASE_IMAGE} -# Install Python build dependencies +# Install Python build dependencies (based on Debian Build-Depends) RUN apt-get update && apt-get install -yq \ - build-essential \ - wget \ - pkg-config \ + autoconf \ + blt-dev \ + bzip2 \ + debhelper \ + dpkg-dev \ + gcc \ + libbluetooth-dev \ libbz2-dev \ + libdb-dev \ + libexpat1-dev \ + libffi-dev \ libgdbm-dev \ + libgpm2 \ liblzma-dev \ - libncurses5-dev \ + libmpdec-dev \ + libncursesw5-dev \ libreadline-dev \ libsqlite3-dev \ libssl-dev \ + locales \ + lsb-release \ + mime-support \ + net-tools \ + netbase \ + python3 \ + quilt \ + sharutils \ + time \ + tk-dev \ + wget \ + xauth \ + xvfb \ zlib1g-dev + # Setup locale. This prevents Python 3 IO encoding issues. ENV LANG C.UTF-8 diff --git a/python-interpreter-builder/README.md b/python-interpreter-builder/README.md index 3c1fc588..7b718581 100644 --- a/python-interpreter-builder/README.md +++ b/python-interpreter-builder/README.md @@ -8,12 +8,13 @@ dependencies in the final container. ## Building -Use make: +Use: - make build + docker build --tag=google/python/interpreter-builder . -The interpreters will be outputted to `output/interpreters.tar.gz`, this is -suitable to be added directly to a Docker container: +The interpreters will be stored in the image at `/interpreters.tar.gz`. This is +suitable to be extracted from this image and added directly to another Docker +image via: ADD interpreters.tar.gz / diff --git a/python-interpreter-builder/scripts/build-python-3.5.sh b/python-interpreter-builder/scripts/build-python-3.5.sh index 263312af..b5e49ac8 100755 --- a/python-interpreter-builder/scripts/build-python-3.5.sh +++ b/python-interpreter-builder/scripts/build-python-3.5.sh @@ -1,21 +1,135 @@ #!/bin/bash -set -e +set -euo pipefail # Get the source mkdir -p /opt/sources cd /opt/sources -wget -nv https://www.python.org/ftp/python/3.5.2/Python-3.5.2.tgz +wget --no-verbose https://www.python.org/ftp/python/3.5.2/Python-3.5.2.tgz +shasum --check < Date: Wed, 12 Apr 2017 09:08:14 +0000 Subject: [PATCH 147/362] Auto-update dependencies. --- tests/python2-libraries/requirements.txt | 16 ++++++++-------- tests/python3-libraries/requirements.txt | 6 +++--- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 48d0bf43..1f635b65 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -6,7 +6,7 @@ anyjson==0.3.3 apache-libcloud==1.5.0 argparse==1.4.0 astroid==1.4.9 -awscli==1.11.75 +awscli==1.11.76 babel==2.4.0 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 @@ -16,13 +16,13 @@ billiard==3.5.0.2 blessings==1.6 blinker==1.4 boto==2.46.1 -botocore==1.5.38 +botocore==1.5.39 bottle==0.12.13 -carbon==0.9.15 +carbon==1.0.0 celery==4.0.2 certifi==2017.1.23 cffi==1.10.0 -chardet==2.3.0 +chardet==3.0.1 click==6.7 cliff==2.5.0 cmd2==0.7.0 @@ -58,7 +58,7 @@ functools32==3.2.3.post2 futures==3.0.5 gevent==1.2.1 google-api-python-client==1.6.2 -graphite-web==0.9.15 +graphite-web==1.0.0 greenlet==0.4.12 gunicorn==19.7.1 hiredis==0.2.0 @@ -114,7 +114,7 @@ passlib==1.7.1 paste==2.0.3 pastedeploy==1.5.2 pastescript==2.0.2 -pbr==2.0.0 +pbr==2.1.0 pep8==1.7.0 pexpect==4.2.1 pika==0.10.0 @@ -122,7 +122,7 @@ pillow==4.1.0 pip==9.0.1 prettytable protobuf==3.2.0 -psutil==5.2.1 +psutil==5.2.2 psycopg2==2.7.1 py==1.4.33 pyasn1-modules==0.0.8 @@ -166,7 +166,7 @@ rsa==3.4.2 scipy==0.19.0 selenium==3.3.3 setuptools-git==1.2 -setuptools==34.4.0 +setuptools==34.4.1 sh==1.12.13 simplejson==3.10.0 six==1.10.0 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 41b23682..0cdacb82 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -12,11 +12,11 @@ billiard==3.5.0.2 blessings==1.6 blinker==1.4 bottle==0.12.13 -carbon==0.9.15 +carbon==1.0.0 celery==4.0.2 certifi==2017.1.23 cffi==1.10.0 -chardet==2.3.0 +chardet==3.0.1 click==6.7 cliff==2.5.0 cmd2==0.7.0 @@ -95,7 +95,7 @@ pillow==4.1.0 pip==9.0.1 prettytable protobuf==3.2.0 -psutil==5.2.1 +psutil==5.2.2 psycopg2==2.7.1 py==1.4.33 pyasn1-modules==0.0.8 From a07d390cf68597768398e338c81426c63ac5f6c5 Mon Sep 17 00:00:00 2001 From: liyanhui1228 Date: Thu, 13 Apr 2017 14:56:46 -0700 Subject: [PATCH 148/362] Update the benchmark code to use the new Python benchmark suite (#105) --- tests/benchmark/Dockerfile.in | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/tests/benchmark/Dockerfile.in b/tests/benchmark/Dockerfile.in index e39eac1c..fcdaa5f9 100644 --- a/tests/benchmark/Dockerfile.in +++ b/tests/benchmark/Dockerfile.in @@ -1,6 +1,19 @@ FROM ${FULL_BASE_IMAGE} -RUN hg clone -r 9923b81a1d34 https://hg.python.org/benchmarks /app/benchmarks -WORKDIR /app/benchmarks -RUN python perf.py -r -b default /usr/bin/python2.7 /usr/bin/python3.4 -RUN python perf.py -r -b default /usr/bin/python3.4 /opt/python3.5/bin/python3.5 +# Install performance +RUN pip install performance + +# Create virtual environment +RUN pip install --upgrade virtualenv + +# Download ensurepip module which prevents the failure when benchmarking python3, can be removed once we drop debian's 3.4 +RUN wget https://www.python.org/ftp/python/3.4.2/Python-3.4.2.tgz +RUN tar xzf Python-3.4.2.tgz +RUN cp -R Python-3.4.2/Lib/ensurepip /usr/lib/python3.4 + +# Run the benchmark and compare the performance, add the --debug-single-value flag to let the benchmark run in fastest mode +RUN pyperformance run --debug-single-value --python=python2.7 -o py2.7.json +RUN pyperformance run --debug-single-value --python=python3.4 -o py3.4.json +RUN pyperformance run --debug-single-value --python=python3.5 -o py3.5.json +RUN pyperformance compare py2.7.json py3.4.json --output_style table +RUN pyperformance compare py3.4.json py3.5.json --output_style table From 94999df2ea7008b9a90490720ab3e379c5fabb60 Mon Sep 17 00:00:00 2001 From: Angela Li Date: Tue, 18 Apr 2017 15:24:07 -0700 Subject: [PATCH 149/362] Benchmark the same interpreter from release to release. --- .../benchmark_between_releases/Dockerfile.in | 24 +++++++++++ .../benchmark_between_releases.sh | 40 +++++++++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 tests/benchmark/benchmark_between_releases/Dockerfile.in create mode 100755 tests/benchmark/benchmark_between_releases/benchmark_between_releases.sh diff --git a/tests/benchmark/benchmark_between_releases/Dockerfile.in b/tests/benchmark/benchmark_between_releases/Dockerfile.in new file mode 100644 index 00000000..0bf1f65a --- /dev/null +++ b/tests/benchmark/benchmark_between_releases/Dockerfile.in @@ -0,0 +1,24 @@ +FROM ${FULL_BASE_IMAGE} + +# Install performance +RUN pip install performance + +# Create virtual environment +RUN pip install --upgrade virtualenv + +# Download ensurepip module which prevents the failure when benchmarking python3, can be removed once we drop debian's 3.4 +RUN wget https://www.python.org/ftp/python/3.4.2/Python-3.4.2.tgz +RUN tar xzf Python-3.4.2.tgz +RUN cp -R Python-3.4.2/Lib/ensurepip /usr/lib/python3.4 + +RUN mkdir /${TAG} + +# Run the benchmark and compare the performance, add the --debug-single-value flag to let the benchmark run in fastest mode +RUN pyperformance run --debug-single-value --python=python2.7 -o /${TAG}/py2.7.json +RUN pyperformance run --debug-single-value --python=python3.4 -o /${TAG}/py3.4.json +RUN pyperformance run --debug-single-value --python=python3.5 -o /${TAG}/py3.5.json +RUN pyperformance compare /${TAG}/py2.7.json /${TAG}/py3.4.json --output_style table +RUN pyperformance compare /${TAG}/py3.4.json /${TAG}/py3.5.json --output_style table + +# Initialize mount volume +VOLUME /${TAG} diff --git a/tests/benchmark/benchmark_between_releases/benchmark_between_releases.sh b/tests/benchmark/benchmark_between_releases/benchmark_between_releases.sh new file mode 100755 index 00000000..1b4ac7fc --- /dev/null +++ b/tests/benchmark/benchmark_between_releases/benchmark_between_releases.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +# Build the benchmark image for release 1 from Dockerfile +echo "Building image for release 1" +export FULL_BASE_IMAGE="${DOCKER_NAMESPACE}/python:${TAG1}" +export TAG="${TAG1}" +envsubst <"Dockerfile".in >"Dockerfile" '$FULL_BASE_IMAGE $TAG' +docker build -t benchmark_1 . +rm Dockerfile + +# Build the benchmark image for release 2 from Dockerfile +echo "Building image for release 2" +export FULL_BASE_IMAGE="${DOCKER_NAMESPACE}/python:${TAG2}" +export TAG="${TAG2}" +envsubst <"Dockerfile".in >"Dockerfile" '$FULL_BASE_IMAGE $TAG' +docker build -t benchmark_2 . +rm Dockerfile + +echo "Successfully built images" + +# Start running the containers +docker run -it --name benchmark_1 -h CONTAINER1 -v /"${TAG1}" benchmark_1 ls +docker run -it --name benchmark_2 -h CONTAINER2 -v /"${TAG2}" benchmark_2 ls + +# Create folders to hold the files +mkdir release1 +mkdir release2 + +# Copy the benchmark result for python versions from container to host +docker cp benchmark_1:/"${TAG1}"/ release1/ +docker cp benchmark_2:/"${TAG2}"/ release2/ + +echo "Start benchmarking the python interpreter performance between the two releases" + +# Compare the performance between the interpreter in different release +pyperformance compare release1/"${TAG1}"/py2.7.json release2/"${TAG2}"/py2.7.json --output_style table > py2.7_res +pyperformance compare release1/"${TAG1}"/py3.4.json release2/"${TAG2}"/py3.4.json --output_style table > py3.4_res +pyperformance compare release1/"${TAG1}"/py3.5.json release2/"${TAG2}"/py3.5.json --output_style table > py3.5_res + +echo "Completed" \ No newline at end of file From 009fb07c4dcf449293cf502ecae0dd3a14809776 Mon Sep 17 00:00:00 2001 From: Angela Li Date: Tue, 18 Apr 2017 15:24:07 -0700 Subject: [PATCH 150/362] Benchmark the same interpreter from release to release. --- RELEASING.md | 10 +++++- tests/benchmark/Dockerfile.in | 12 ++++--- tests/benchmark/benchmark_between_releases.sh | 34 +++++++++++++++++++ 3 files changed, 50 insertions(+), 6 deletions(-) create mode 100755 tests/benchmark/benchmark_between_releases.sh diff --git a/RELEASING.md b/RELEASING.md index 1f92084a..f7d5a459 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -72,8 +72,16 @@ docker run -it --entrypoint /bin/bash YOUR-IMAGE-NAME There is a benchmark suite which compares the performance of interpreters against each other. +**Benchmark different versions of interpreter in the same release + +``` shell +DOCKER_NAMESPACE=DOCKER_NAMESPACE_EXAMPLE TAG=TAG_EXAMPLE ./build.sh --nobuild --benchmark +``` + +**Benchmark same versions of interpreter from release to release + ``` shell -./build.sh --nobuild --benchmark +DOCKER_NAMESPACE=DOCKER_NAMESPACE_EXAMPLE TAG1=TAG1_EXAMPLE TAG2=TAG2_EXAMPLE ./benchmark_between_releases.sh ``` Since these benchmarks are run on cloud instances, the timings may vary from run diff --git a/tests/benchmark/Dockerfile.in b/tests/benchmark/Dockerfile.in index fcdaa5f9..9a379a99 100644 --- a/tests/benchmark/Dockerfile.in +++ b/tests/benchmark/Dockerfile.in @@ -11,9 +11,11 @@ RUN wget https://www.python.org/ftp/python/3.4.2/Python-3.4.2.tgz RUN tar xzf Python-3.4.2.tgz RUN cp -R Python-3.4.2/Lib/ensurepip /usr/lib/python3.4 +RUN mkdir /result + # Run the benchmark and compare the performance, add the --debug-single-value flag to let the benchmark run in fastest mode -RUN pyperformance run --debug-single-value --python=python2.7 -o py2.7.json -RUN pyperformance run --debug-single-value --python=python3.4 -o py3.4.json -RUN pyperformance run --debug-single-value --python=python3.5 -o py3.5.json -RUN pyperformance compare py2.7.json py3.4.json --output_style table -RUN pyperformance compare py3.4.json py3.5.json --output_style table +RUN pyperformance run --debug-single-value --python=python2.7 -o /result/py2.7.json +RUN pyperformance run --debug-single-value --python=python3.4 -o /result/py3.4.json +RUN pyperformance run --debug-single-value --python=python3.5 -o /result/py3.5.json +RUN pyperformance compare /result/py2.7.json /result/py3.4.json --output_style table +RUN pyperformance compare /result/py3.4.json /result/py3.5.json --output_style table diff --git a/tests/benchmark/benchmark_between_releases.sh b/tests/benchmark/benchmark_between_releases.sh new file mode 100755 index 00000000..ea3d5ffe --- /dev/null +++ b/tests/benchmark/benchmark_between_releases.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# Build the benchmark image for release 1 from Dockerfile +echo "Building image for release 1" +export FULL_BASE_IMAGE="${DOCKER_NAMESPACE}/python:${TAG1}" +envsubst <"Dockerfile".in >"Dockerfile" '$FULL_BASE_IMAGE' +docker build --no-cache -t benchmark_1 . +rm Dockerfile + +# Build the benchmark image for release 2 from Dockerfile +echo "Building image for release 2" +export FULL_BASE_IMAGE="${DOCKER_NAMESPACE}/python:${TAG2}" +envsubst <"Dockerfile".in >"Dockerfile" '$FULL_BASE_IMAGE' +docker build --no-cache -t benchmark_2 . +rm Dockerfile + +echo "Successfully built images" + +# Create folders to hold the files +mkdir release1 +mkdir release2 + +# Start running the containers and copy the benchmark result for python versions from container to host +docker run -it --name benchmark_1 -h CONTAINER1 -v "${PWD}"/release1:/export benchmark_1 /bin/bash -c "cp /result/py*.json /export/" +docker run -it --name benchmark_2 -h CONTAINER2 -v "${PWD}"/release2:/export benchmark_2 /bin/bash -c "cp /result/py*.json /export/" + +echo "Start benchmarking the python interpreter performance between the two releases" + +# Compare the performance between the interpreter in different release +pyperformance compare release1/py2.7.json release2/py2.7.json --output_style table > py2.7_res +pyperformance compare release1/py3.4.json release2/py3.4.json --output_style table > py3.4_res +pyperformance compare release1/py3.5.json release2/py3.5.json --output_style table > py3.5_res + +echo "Completed" From 89bc0b95ff570e500288dc2c5dccc919c69ca5ba Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Thu, 20 Apr 2017 14:46:34 -0700 Subject: [PATCH 151/362] Update from Python 3.5.2 to 3.5.3 --- .../scripts/build-python-3.5.sh | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/python-interpreter-builder/scripts/build-python-3.5.sh b/python-interpreter-builder/scripts/build-python-3.5.sh index b5e49ac8..fa54900b 100755 --- a/python-interpreter-builder/scripts/build-python-3.5.sh +++ b/python-interpreter-builder/scripts/build-python-3.5.sh @@ -5,14 +5,15 @@ set -euo pipefail # Get the source mkdir -p /opt/sources cd /opt/sources -wget --no-verbose https://www.python.org/ftp/python/3.5.2/Python-3.5.2.tgz +wget --no-verbose https://www.python.org/ftp/python/3.5.3/Python-3.5.3.tgz +# SHA-256 generated via `shasum -a 256 [file]` shasum --check < Date: Mon, 24 Apr 2017 20:17:44 -0700 Subject: [PATCH 152/362] Tweak Debian flags based on manually compiling .deb package --- .../scripts/build-python-3.5.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/python-interpreter-builder/scripts/build-python-3.5.sh b/python-interpreter-builder/scripts/build-python-3.5.sh index fa54900b..f05bf58b 100755 --- a/python-interpreter-builder/scripts/build-python-3.5.sh +++ b/python-interpreter-builder/scripts/build-python-3.5.sh @@ -67,8 +67,6 @@ cd Python-3.5.3 # (Debian) Security hardening # CFLAGS=-g # (Debian) More debug info -# CFLAGS=-O2 -# (Debian) the default is -O3, don't know why the difference # CFLAGS=-Wformat -Werror=format-security # (Debian) Security hardening # CPPFLAGS=-D_FORTIFY_SOURCE=2 @@ -113,7 +111,6 @@ cd build-static CFLAGS="\ -fstack-protector-strong \ -g \ - -O2 \ -Wformat -Werror=format-security \ " \ CPPFLAGS="\ @@ -125,13 +122,16 @@ cd build-static RANLIB="x86_64-linux-gnu-gcc-ranlib" \ # Explicitly build the profile-guided-optimized interpreter -EXTRA_OPT_CFLAGS="-g -flto -fuse-linker-plugin -ffat-lto-objects" -NUM_JOBS=$(nproc) -make -j"$NUM_JOBS" EXTRA_CFLAGS="${EXTRA_OPT_CFLAGS}" profile-opt -make -j"$NUM_JOBS" EXTRA_CFLAGS="${EXTRA_OPT_CFLAGS}" test +NUM_JOBS="$(nproc)" +make \ + -j"${NUM_JOBS}" \ + EXTRA_CFLAGS="-g -flto -fuse-linker-plugin -ffat-lto-objects" \ + PROFILE_TASK='../Lib/test/regrtest.py -s -j 1 -unone,decimal -x test_cmd_line_script test_compiler test_concurrent_futures test_ctypes test_dbm_dumb test_dbm_ndbm test_distutils test_ensurepip test_gdb test_ioctl test_linuxaudiodev test_multiprocessing test_multiprocessing_fork test_multiprocessing_forkserver test_multiprocessing_main_handling test_multiprocessing_spawn test_ossaudiodev test_pydoc test_signal test_socket test_socketserver test_subprocess test_sundry test_thread test_threaded_import test_threadedtempfile test_threading test_threading_local test_threadsignals test_venv test_zipimport_support' \ + profile-opt + make altinstall # Clean-up sources -cd / +cd /opt rm /opt/sources/Python-3.5.3.tgz rm -r /opt/sources/Python-3.5.3 From d0c20d1c3ba3f7c97013a036f99a1354529b874d Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Mon, 24 Apr 2017 20:30:55 -0700 Subject: [PATCH 153/362] Build Python 3.6.1 interpreter --- .../scripts/build-python-3.6.sh | 145 ++++++++++++++++++ 1 file changed, 145 insertions(+) create mode 100644 python-interpreter-builder/scripts/build-python-3.6.sh diff --git a/python-interpreter-builder/scripts/build-python-3.6.sh b/python-interpreter-builder/scripts/build-python-3.6.sh new file mode 100644 index 00000000..c15aa090 --- /dev/null +++ b/python-interpreter-builder/scripts/build-python-3.6.sh @@ -0,0 +1,145 @@ +#!/bin/bash + +set -euo pipefail + +# Get the source +mkdir -p /opt/sources +cd /opt/sources +wget --no-verbose https://www.python.org/ftp/python/3.6.1/Python-3.6.1.tgz +# SHA-256 generated via `shasum -a 256 [file]` +shasum --check < Date: Mon, 1 May 2017 13:20:39 -0700 Subject: [PATCH 154/362] Further compile flag tweaks. Turn off LTO for now. --- python-interpreter-builder/Dockerfile.in | 5 +- .../scripts/build-python-3.5.sh | 43 +++++++-------- .../scripts/build-python-3.6.sh | 53 +++++++------------ tests/python3-libraries/requirements.txt | 4 +- 4 files changed, 44 insertions(+), 61 deletions(-) mode change 100644 => 100755 python-interpreter-builder/scripts/build-python-3.6.sh diff --git a/python-interpreter-builder/Dockerfile.in b/python-interpreter-builder/Dockerfile.in index c3b7dada..9323f750 100644 --- a/python-interpreter-builder/Dockerfile.in +++ b/python-interpreter-builder/Dockerfile.in @@ -45,9 +45,10 @@ ENV LANG C.UTF-8 # Add build scripts ADD scripts /scripts -# Build the Python 3.5 interpreter +# Build the Python interpreters RUN /scripts/build-python-3.5.sh +RUN /scripts/build-python-3.6.sh # Tar the interpreters. Tarring is needed because docker cp doesn't handle # links correctly. -RUN tar czf /interpreters.tar.gz /opt +RUN tar czf /interpreters.tar.gz /opt/python?.? diff --git a/python-interpreter-builder/scripts/build-python-3.5.sh b/python-interpreter-builder/scripts/build-python-3.5.sh index f05bf58b..873b7257 100755 --- a/python-interpreter-builder/scripts/build-python-3.5.sh +++ b/python-interpreter-builder/scripts/build-python-3.5.sh @@ -1,6 +1,7 @@ #!/bin/bash set -euo pipefail +set -x # Get the source mkdir -p /opt/sources @@ -26,12 +27,6 @@ cd Python-3.5.3 # (https://bugs.python.org/issue27685) # --without-ensurepip # Debian unbundles pip for their own reasons -# --with-system-expat -# (Debian) for compatibility with other Debian packages -# --with-system-ffi -# (Debian) for compatibility with other Debian packages -# --with-system-libmpdec -# (Debian) for compatibility with other Debian packages # CFLAGS=-fdebug-prefix-map # Unnecessary in our build environment # @@ -59,6 +54,12 @@ cd Python-3.5.3 # would prefer one over the other. # --with-fpectl # (Debian) Floating point exception control +# --with-system-expat +# (Debian) for compatibility with other Debian packages +# --with-system-ffi +# (Debian) for compatibility with other Debian packages +# --with-system-libmpdec +# (Debian) for compatibility with other Debian packages # AR= # (Debian) No-op # CC= @@ -83,29 +84,26 @@ cd Python-3.5.3 # # LTO (Link time optimization) # -# There is a --with-lto flag, but Debian doesn't use it. We used to -# use it, but it caused trouble with the uWGSI module. Instead, we -# pass lto related flags in EXTRA_CFLAGS (to make, rather than -# configure), as Debian does. -# -# -# Debugging: It is very helpful to view and diff sysconfig data from two -# python interpreters. For example: -# docker run -it --entrypoint=/opt/python3.5/bin/python3.5 google/python/interpreter-builder -c 'import sysconfig;print("\n".join("%s:%s"%(key,value) for key,value in sorted(sysconfig.get_config_vars().items())))' +# Currently disabled, due to unresolved compile problems. There is a +# --with-lto flag, but Debian doesn't use it. Instead, they pass lto +# related flags in EXTRA_CFLAGS (to make, rather than configure). +# Specifically EXTRA_CFLAGS="-g -flto -fuse-linker-plugin +# -ffat-lto-objects" mkdir build-static cd build-static ../configure \ - --build=x86_64-pc-linux-gnu \ --enable-ipv6 \ --enable-loadable-sqlite-extensions \ --enable-optimizations \ - --host=x86_64-pc-linux-gnu \ --prefix=/opt/python3.5 \ --with-dbmliborder=bdb:gdbm \ --with-computed-gotos \ --with-fpectl \ + --with-system-expat \ + --with-system-ffi \ + --with-system-libmpdec \ AR="x86_64-linux-gnu-gcc-ar" \ CC="x86_64-linux-gnu-gcc" \ CFLAGS="\ @@ -121,14 +119,11 @@ cd build-static LDFLAGS="-Wl,-z,relro" \ RANLIB="x86_64-linux-gnu-gcc-ranlib" \ -# Explicitly build the profile-guided-optimized interpreter -NUM_JOBS="$(nproc)" -make \ - -j"${NUM_JOBS}" \ - EXTRA_CFLAGS="-g -flto -fuse-linker-plugin -ffat-lto-objects" \ - PROFILE_TASK='../Lib/test/regrtest.py -s -j 1 -unone,decimal -x test_cmd_line_script test_compiler test_concurrent_futures test_ctypes test_dbm_dumb test_dbm_ndbm test_distutils test_ensurepip test_gdb test_ioctl test_linuxaudiodev test_multiprocessing test_multiprocessing_fork test_multiprocessing_forkserver test_multiprocessing_main_handling test_multiprocessing_spawn test_ossaudiodev test_pydoc test_signal test_socket test_socketserver test_subprocess test_sundry test_thread test_threaded_import test_threadedtempfile test_threading test_threading_local test_threadsignals test_venv test_zipimport_support' \ - profile-opt +# Due to https://bugs.python.org/issue29243, "make altinstall" +# rebuilds everything from scratch, twice. This is a workaround. +sed -i 's/^all:.*$/all: build_all/' Makefile +make profile-opt make altinstall # Clean-up sources diff --git a/python-interpreter-builder/scripts/build-python-3.6.sh b/python-interpreter-builder/scripts/build-python-3.6.sh old mode 100644 new mode 100755 index c15aa090..af4fc0b4 --- a/python-interpreter-builder/scripts/build-python-3.6.sh +++ b/python-interpreter-builder/scripts/build-python-3.6.sh @@ -1,6 +1,7 @@ #!/bin/bash set -euo pipefail +set -x # Get the source mkdir -p /opt/sources @@ -26,12 +27,6 @@ cd Python-3.6.1 # (https://bugs.python.org/issue27685) # --without-ensurepip # Debian unbundles pip for their own reasons -# --with-system-expat -# (Debian) for compatibility with other Debian packages -# --with-system-ffi -# (Debian) for compatibility with other Debian packages -# --with-system-libmpdec -# (Debian) for compatibility with other Debian packages # CFLAGS=-fdebug-prefix-map # Unnecessary in our build environment # @@ -59,6 +54,12 @@ cd Python-3.6.1 # would prefer one over the other. # --with-fpectl # (Debian) Floating point exception control +# --with-system-expat +# (Debian) for compatibility with other Debian packages +# --with-system-ffi +# (Debian) for compatibility with other Debian packages +# --with-system-libmpdec +# (Debian) for compatibility with other Debian packages # AR= # (Debian) No-op # CC= @@ -67,8 +68,6 @@ cd Python-3.6.1 # (Debian) Security hardening # CFLAGS=-g # (Debian) More debug info -# CFLAGS=-specs=/usr/share/dpkg/no-pie-link.specs -# (Debian) Temporarily disable security hardening # CFLAGS=-Wformat -Werror=format-security # (Debian) Security hardening # CPPFLAGS=-D_FORTIFY_SOURCE=2 @@ -77,8 +76,6 @@ cd Python-3.6.1 # (Debian) Warnings about non-reproducible builds # CXX= # (Debian) No-op -# LDFLAGS=-specs=/usr/share/dpkg/no-pie-link.specs -# (Debian) Temporarily disable security hardening # LDFLAGS=-Wl,-z,relro: # (Debian) Security hardening # RANLIB= @@ -87,35 +84,31 @@ cd Python-3.6.1 # # LTO (Link time optimization) # -# There is a --with-lto flag, but Debian doesn't use it. We used to -# use it, but it caused trouble with the uWGSI module. Instead, we -# pass lto related flags in EXTRA_CFLAGS (to make, rather than -# configure), as Debian does. -# -# -# Debugging: It is very helpful to view and diff sysconfig data from two -# python interpreters. For example: -# docker run -it --entrypoint=/opt/python3.6/bin/python3.6 google/python/interpreter-builder -c 'import sysconfig;print("\n".join("%s:%s"%(key,value) for key,value in sorted(sysconfig.get_config_vars().items())))' +# Currently disabled, due to unresolved compile problems. There is a +# --with-lto flag, but Debian doesn't use it. Instead, they pass lto +# related flags in EXTRA_CFLAGS (to make, rather than configure). +# Specifically EXTRA_CFLAGS="-g -flto -fuse-linker-plugin +# -ffat-lto-objects" mkdir build-static cd build-static ../configure \ - --build=x86_64-pc-linux-gnu \ --enable-ipv6 \ --enable-loadable-sqlite-extensions \ --enable-optimizations \ - --host=x86_64-pc-linux-gnu \ --prefix=/opt/python3.6 \ --with-dbmliborder=bdb:gdbm \ --with-computed-gotos \ --with-fpectl \ + --with-system-expat \ + --with-system-ffi \ + --with-system-libmpdec \ AR="x86_64-linux-gnu-gcc-ar" \ CC="x86_64-linux-gnu-gcc" \ CFLAGS="\ -fstack-protector-strong \ -g \ - -specs=/usr/share/dpkg/no-pie-compile.specs \ -Wformat -Werror=format-security \ " \ CPPFLAGS="\ @@ -123,20 +116,14 @@ cd build-static -Wdate-time \ " \ CXX="x86_64-linux-gnu-g++" \ - LDFLAGS="\ - -specs=/usr/share/dpkg/no-pie-link.specs \ - -Wl,-z,relro \ - " \ + LDFLAGS="-Wl,-z,relro" \ RANLIB="x86_64-linux-gnu-gcc-ranlib" \ -# Explicitly build the profile-guided-optimized interpreter -NUM_JOBS="$(nproc)" -make \ - -j"${NUM_JOBS}" \ - EXTRA_CFLAGS="" \ - PROFILE_TASK="../Lib/test/regrtest.py -s -j 1 -unone,decimal -x test_cmd_line_script test_compiler test_concurrent_futures test_ctypes test_dbm_dumb test_dbm_ndbm test_distutils test_ensurepip test_gdb test_ioctl test_linuxaudiodev test_multiprocessing test_ossaudiodev test_pydoc test_signal test_socket test_socketserver test_subprocess test_sundry test_thread test_threaded_import test_threadedtempfile test_threading test_threading_local test_threadsignals test_venv test_zipimport_support" \ - profile-opt +# Due to https://bugs.python.org/issue29243, "make altinstall" +# rebuilds everything from scratch, twice. This is a workaround. +sed -i 's/^all:.*$/all: build_all/' Makefile +make profile-opt make altinstall # Clean-up sources diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 784f5751..74546872 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -143,7 +143,7 @@ pyramid==1.8.3 pystache==0.5.4 pytest-cov==2.4.0 pytest==3.0.7 -python-cjson==1.2.1 +#python-cjson python-daemon==2.1.2 python-dateutil==2.6.0 python-gflags==3.1.1 @@ -191,7 +191,7 @@ unidecode==0.4.20 unittest2==1.1.0 uritemplate==3.0.0 urllib3==1.20 -uwsgi==2.0.15 +#uwsgi versiontools==1.9.1 virtualenv==15.1.0 waitress==1.0.2 From b6c4dc4dc9f5e1ef287a9d6ea21dadb03aa8f3c6 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Mon, 1 May 2017 14:44:12 -0700 Subject: [PATCH 155/362] Update expected version in structure test. --- tests/virtualenv/virtualenv_python35.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/virtualenv/virtualenv_python35.yaml b/tests/virtualenv/virtualenv_python35.yaml index 23dc1367..b37d1461 100644 --- a/tests/virtualenv/virtualenv_python35.yaml +++ b/tests/virtualenv/virtualenv_python35.yaml @@ -29,7 +29,7 @@ commandTests: - name: "python version" command: ["python", "--version"] - expectedOutput: ["Python 3.5.2\n"] + expectedOutput: ["Python 3.5.3\n"] - name: "pip installation" command: ["which", "pip"] From 3499ecd55bf699ca5389abcd0335950dd3d8466f Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Mon, 1 May 2017 16:46:23 -0700 Subject: [PATCH 156/362] Disable 'carbon' module which fails under Python 3 --- tests/python3-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 7279102f..6f971d8d 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -18,7 +18,7 @@ blinker==1.4 boto==2.46.1 botocore==1.5.31 bottle==0.12.13 -carbon==1.0.0 +#carbon celery==4.0.2 certifi==2017.1.23 cffi==1.10.0 From efd52e351eecbee5fc07d5433809c90e9031b8bc Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Thu, 4 May 2017 14:38:16 -0700 Subject: [PATCH 157/362] Add explanation of --enable-ipv6 flag --- python-interpreter-builder/scripts/build-python-3.5.sh | 6 +----- python-interpreter-builder/scripts/build-python-3.6.sh | 6 +----- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/python-interpreter-builder/scripts/build-python-3.5.sh b/python-interpreter-builder/scripts/build-python-3.5.sh index 873b7257..a80473d8 100755 --- a/python-interpreter-builder/scripts/build-python-3.5.sh +++ b/python-interpreter-builder/scripts/build-python-3.5.sh @@ -34,17 +34,13 @@ cd Python-3.5.3 # Flags that we _do_ use: # (Debian) means it was taken from Debian build rules. # -# --build -# (Debian) # --enable-ipv6 -# (Debian) +# (Debian) Ensure support is compiled in instead of relying on autodetection # --enable-loadable-sqlite-extensions # (Debian) # --enable-optimizations # Performance optimization (Enables PGO and may or may not enable # LTO based on complex logic and bugs) -# --host -# (Debian) # --prefix # Avoid possible collisions with Debian or others # --with-computed-gotos diff --git a/python-interpreter-builder/scripts/build-python-3.6.sh b/python-interpreter-builder/scripts/build-python-3.6.sh index af4fc0b4..cb4fa360 100755 --- a/python-interpreter-builder/scripts/build-python-3.6.sh +++ b/python-interpreter-builder/scripts/build-python-3.6.sh @@ -34,17 +34,13 @@ cd Python-3.6.1 # Flags that we _do_ use: # (Debian) means it was taken from Debian build rules. # -# --build -# (Debian) # --enable-ipv6 -# (Debian) +# (Debian) Ensure support is compiled in instead of relying on autodetection # --enable-loadable-sqlite-extensions # (Debian) # --enable-optimizations # Performance optimization (Enables PGO and may or may not enable # LTO based on complex logic and bugs) -# --host -# (Debian) # --prefix # Avoid possible collisions with Debian or others # --with-computed-gotos From 069c598f4763dc6b691019c955c7b96b945ee527 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Thu, 4 May 2017 15:20:50 -0700 Subject: [PATCH 158/362] Enable Python 3.6 in the base runtime image. --- runtime-image/Dockerfile.in | 2 +- tests/benchmark/Dockerfile.in | 13 +++-- tests/benchmark/benchmark_between_releases.sh | 1 + .../run_system_tests.sh | 7 ++- tests/google-cloud-python/run_unit_tests.sh | 9 +++- .../python3-libraries/python3-libraries.yaml | 14 ++++-- tests/virtualenv/virtualenv_python36.yaml | 48 +++++++++++++++++++ 7 files changed, 82 insertions(+), 12 deletions(-) create mode 100644 tests/virtualenv/virtualenv_python36.yaml diff --git a/runtime-image/Dockerfile.in b/runtime-image/Dockerfile.in index 15cf3ee4..ed194257 100644 --- a/runtime-image/Dockerfile.in +++ b/runtime-image/Dockerfile.in @@ -24,7 +24,7 @@ RUN pip install --upgrade pip virtualenv ADD interpreters.tar.gz / # Add Google-built interpreters to the path -ENV PATH /opt/python3.5/bin:$PATH +ENV PATH /opt/python3.5/bin:/opt/python3.6/bin:$PATH # Setup the app working directory RUN ln -s /home/vmagent/app /app diff --git a/tests/benchmark/Dockerfile.in b/tests/benchmark/Dockerfile.in index 9a379a99..69f41862 100644 --- a/tests/benchmark/Dockerfile.in +++ b/tests/benchmark/Dockerfile.in @@ -6,16 +6,19 @@ RUN pip install performance # Create virtual environment RUN pip install --upgrade virtualenv -# Download ensurepip module which prevents the failure when benchmarking python3, can be removed once we drop debian's 3.4 -RUN wget https://www.python.org/ftp/python/3.4.2/Python-3.4.2.tgz -RUN tar xzf Python-3.4.2.tgz -RUN cp -R Python-3.4.2/Lib/ensurepip /usr/lib/python3.4 +# Required for Python 3.4, see +# https://bugs.launchpad.net/ubuntu/+source/python3.4/+bug/1290847 +RUN apt-get update && apt-get install -y --force-yes python3-pip python3-venv RUN mkdir /result -# Run the benchmark and compare the performance, add the --debug-single-value flag to let the benchmark run in fastest mode +# Run the benchmark and compare the performance, add the +# --debug-single-value flag to let the benchmark run in fastest mode RUN pyperformance run --debug-single-value --python=python2.7 -o /result/py2.7.json RUN pyperformance run --debug-single-value --python=python3.4 -o /result/py3.4.json RUN pyperformance run --debug-single-value --python=python3.5 -o /result/py3.5.json +RUN pyperformance run --debug-single-value --python=python3.6 -o /result/py3.6.json + RUN pyperformance compare /result/py2.7.json /result/py3.4.json --output_style table RUN pyperformance compare /result/py3.4.json /result/py3.5.json --output_style table +RUN pyperformance compare /result/py3.5.json /result/py3.6.json --output_style table diff --git a/tests/benchmark/benchmark_between_releases.sh b/tests/benchmark/benchmark_between_releases.sh index ea3d5ffe..c540ae7f 100755 --- a/tests/benchmark/benchmark_between_releases.sh +++ b/tests/benchmark/benchmark_between_releases.sh @@ -30,5 +30,6 @@ echo "Start benchmarking the python interpreter performance between the two rele pyperformance compare release1/py2.7.json release2/py2.7.json --output_style table > py2.7_res pyperformance compare release1/py3.4.json release2/py3.4.json --output_style table > py3.4_res pyperformance compare release1/py3.5.json release2/py3.5.json --output_style table > py3.5_res +pyperformance compare release1/py3.6.json release2/py3.6.json --output_style table > py3.6_res echo "Completed" diff --git a/tests/google-cloud-python-system/run_system_tests.sh b/tests/google-cloud-python-system/run_system_tests.sh index aab10c88..17d3aac3 100755 --- a/tests/google-cloud-python-system/run_system_tests.sh +++ b/tests/google-cloud-python-system/run_system_tests.sh @@ -28,7 +28,12 @@ export GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE=1 exit_code=0 for package in ${packages}; do noxfile="${package}/nox.py" - nox -f "${noxfile}" -e "system_tests(python_version='2.7')" || exit_code=1 + nox \ + -f "${noxfile}" \ + -e \ + "system_tests(python_version='2.7')" \ + "system_tests(python_version='3.6')" \ + || exit_code=1 done exit "${exit_code}" diff --git a/tests/google-cloud-python/run_unit_tests.sh b/tests/google-cloud-python/run_unit_tests.sh index 8563606e..030919d8 100755 --- a/tests/google-cloud-python/run_unit_tests.sh +++ b/tests/google-cloud-python/run_unit_tests.sh @@ -5,7 +5,14 @@ cd /app/google-cloud-python exit_code=0 for noxfile in */nox.py; do - nox -f "${noxfile}" -e "unit_tests(python_version='2.7')" "unit_tests(python_version='3.4')" "unit_tests(python_version='3.5')" || exit_code=1 + nox \ + -f "${noxfile}" \ + -e \ + "unit_tests(python_version='2.7')" \ + "unit_tests(python_version='3.4')" \ + "unit_tests(python_version='3.5')" \ + "unit_tests(python_version='3.6')" \ + || exit_code=1 done exit "${exit_code}" diff --git a/tests/python3-libraries/python3-libraries.yaml b/tests/python3-libraries/python3-libraries.yaml index 28613555..ef15c0a1 100644 --- a/tests/python3-libraries/python3-libraries.yaml +++ b/tests/python3-libraries/python3-libraries.yaml @@ -7,10 +7,16 @@ globalEnvVars: value: "/env/bin:$PATH" commandTests: - - name: "virtual env teardown" - command: ["rm", "-rf", "/env"] + - name: "requirements 3.5" + setup: + - ["rm", "-rf", "/env"] + - ["virtualenv", "-p", "/opt/python3.5/bin/python3.5", "/env"] + command: ["pip", "install", "-r", "/workspace/tests/python3-libraries/requirements.txt"] + exitCode: 0 - - name: "requirements" - setup: [["virtualenv", "-p", "/opt/python3.5/bin/python3.5", "/env"]] + - name: "requirements 3.6" + setup: + - ["rm", "-rf", "/env"] + - ["virtualenv", "-p", "/opt/python3.6/bin/python3.6", "/env"] command: ["pip", "install", "-r", "/workspace/tests/python3-libraries/requirements.txt"] exitCode: 0 diff --git a/tests/virtualenv/virtualenv_python36.yaml b/tests/virtualenv/virtualenv_python36.yaml new file mode 100644 index 00000000..241099b6 --- /dev/null +++ b/tests/virtualenv/virtualenv_python36.yaml @@ -0,0 +1,48 @@ +schemaVersion: "1.0.0" + +globalEnvVars: + - key: "VIRTUAL_ENV" + value: "/env" + - key: "PATH" + value: "/env/bin:$PATH" + +commandTests: + - name: "virtual env teardown" + command: ["rm", "-rf", "/env"] + + - name: "python installation" + command: ["which", "python3.6"] + expectedOutput: ["/opt/python3.6/bin/python3.6\n"] + + - name: "virtualenv python installation" + setup: [["virtualenv", "-p", "python3.6", "/env"]] + command: ["which", "python"] + expectedOutput: ["/env/bin/python\n"] + + - name: "virtualenv python3 installation" + command: ["which", "python3"] + expectedOutput: ["/env/bin/python3\n"] + + - name: "virtualenv python3.6 installation" + command: ["which", "python3.6"] + expectedOutput: ["/env/bin/python3.6\n"] + + - name: "python version" + command: ["python", "--version"] + expectedOutput: ["Python 3.6.3\n"] + + - name: "pip installation" + command: ["which", "pip"] + expectedOutput: ["/env/bin/pip\n"] + + - name: "pip3 installation" + command: ["which", "pip3"] + expectedOutput: ["/env/bin/pip3\n"] + + - name: "gunicorn flask" + setup: [["pip", "install", "gunicorn", "flask"]] + command: ["which", "gunicorn"] + expectedOutput: ["/env/bin/gunicorn"] + + - name: "flask integration" + command: ["python", "-c", "\"import sys; import flask; sys.exit(0 if flask.__file__.startswith('/env') else 1)\""] From cb0fad1f7ef358ba0f49e27507597c8284762d6e Mon Sep 17 00:00:00 2001 From: Angela Li Date: Mon, 8 May 2017 10:46:34 -0700 Subject: [PATCH 159/362] Implemented the benchmark dashboard --- tests/benchmark/Dockerfile.in | 1 + tests/benchmark/benchmark_between_releases.sh | 46 ++++++-- tests/benchmark/generate_csv.py | 107 ++++++++++++++++++ 3 files changed, 146 insertions(+), 8 deletions(-) create mode 100644 tests/benchmark/generate_csv.py diff --git a/tests/benchmark/Dockerfile.in b/tests/benchmark/Dockerfile.in index 69f41862..5ca308f4 100644 --- a/tests/benchmark/Dockerfile.in +++ b/tests/benchmark/Dockerfile.in @@ -22,3 +22,4 @@ RUN pyperformance run --debug-single-value --python=python3.6 -o /result/py3.6.j RUN pyperformance compare /result/py2.7.json /result/py3.4.json --output_style table RUN pyperformance compare /result/py3.4.json /result/py3.5.json --output_style table RUN pyperformance compare /result/py3.5.json /result/py3.6.json --output_style table + diff --git a/tests/benchmark/benchmark_between_releases.sh b/tests/benchmark/benchmark_between_releases.sh index c540ae7f..11fda0eb 100755 --- a/tests/benchmark/benchmark_between_releases.sh +++ b/tests/benchmark/benchmark_between_releases.sh @@ -17,19 +17,49 @@ rm Dockerfile echo "Successfully built images" # Create folders to hold the files -mkdir release1 -mkdir release2 +mkdir $TAG1 +mkdir $TAG2 # Start running the containers and copy the benchmark result for python versions from container to host -docker run -it --name benchmark_1 -h CONTAINER1 -v "${PWD}"/release1:/export benchmark_1 /bin/bash -c "cp /result/py*.json /export/" -docker run -it --name benchmark_2 -h CONTAINER2 -v "${PWD}"/release2:/export benchmark_2 /bin/bash -c "cp /result/py*.json /export/" +docker run -it --name benchmark_1 -h CONTAINER1 -v "${PWD}"/"$TAG1":/export benchmark_1 /bin/bash -c "cp /result/py*.json /export/" +docker run -it --name benchmark_2 -h CONTAINER2 -v "${PWD}"/"$TAG2":/export benchmark_2 /bin/bash -c "cp /result/py*.json /export/" echo "Start benchmarking the python interpreter performance between the two releases" # Compare the performance between the interpreter in different release -pyperformance compare release1/py2.7.json release2/py2.7.json --output_style table > py2.7_res -pyperformance compare release1/py3.4.json release2/py3.4.json --output_style table > py3.4_res -pyperformance compare release1/py3.5.json release2/py3.5.json --output_style table > py3.5_res -pyperformance compare release1/py3.6.json release2/py3.6.json --output_style table > py3.6_res +pyperformance compare "$TAG1"/py2.7.json "$TAG2"/py2.7.json --output_style table > py2.7_res +pyperformance compare "$TAG1"/py3.4.json "$TAG2"/py3.4.json --output_style table > py3.4_res +pyperformance compare "$TAG1"/py3.5.json "$TAG2"/py3.5.json --output_style table > py3.5_res +pyperformance compare "$TAG1"/py3.6.json "$TAG2"/py3.6.json --output_style table > py3.6_res + +echo "Start extracting data and generating CSV file, then upload to Cloud Storage and insert to Big Query table" + +# Extracting memory usage and running time data from the performace result json, generating CSV files +for path_to_file in $TAG1/*.json; do + python generate_csv.py --filename $path_to_file --tag $TAG1 +done + +for path_to_file in $TAG2/*.json; do + python generate_csv.py --filename $path_to_file --tag $TAG2 +done + +# Get the list of existed release data on Cloud Storage and skip if the current TAG1 or TAG2 existed in the list +gsutil ls gs://python_runtime_benchmark > existed_releases + +for container_tag in $TAG1 $TAG2; do + if grep --fixed-strings --quiet "$container_tag" existed_releases; then + echo "Performace data of $container_tag existed, so skip processing it." + else + # Upload the CSV files to Cloud Storage + gsutil cp -r $container_tag gs://python_runtime_benchmark + # Load the CSV files from Cloud Storage to Big Query table + # Load the performance data of each function + for path_to_file in $container_tag/py2.7.csv $container_tag/py3.4.csv $container_tag/py3.5.csv; do + bq load benchmark_test.benchmark_functions gs://python_runtime_benchmark/"$path_to_file" container_tag:string,runtime_version:string,function_name:string,time_used:float,mem_usage:float + done + # Load the average performance data of each runtime version in a release + bq load benchmark_test.benchmark_statistics gs://python_runtime_benchmark/"$container_tag"/averages.csv container_tag:string,runtime_version:string,ave_time_used:float,ave_mem_usage:float + fi +done echo "Completed" diff --git a/tests/benchmark/generate_csv.py b/tests/benchmark/generate_csv.py new file mode 100644 index 00000000..9409dfa3 --- /dev/null +++ b/tests/benchmark/generate_csv.py @@ -0,0 +1,107 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import argparse +import sys +import json +import csv + + +def generate_csv(args): + """Extract function name, time used and memory usage from the metadata and write to the output CSV file. + + :param args: the command line parameters, including filename and tag + :type: str + """ + with open(args.filename) as input: + data = json.load(input) + benchmarks = data["benchmarks"] + runtime_version = os.path.basename(args.filename).split(".json")[0] + + # Write data to CSV file + with open("{}.csv".format(os.path.splitext(args.filename)[0]), "wb") as output: + csv_writer = csv.writer(output, delimiter=',') + for benchmark in benchmarks: + try: + # Get the function name + func_name = benchmark["metadata"]["name"] + # Get the time used for this function, convert to millisecond + time_used = benchmark["runs"][0]["values"][0] * 1000 + # Get the memory usage, convert to MB + mem_usage = benchmark["metadata"]["mem_max_rss"] / float(1<<20) + line = [args.tag, runtime_version, func_name, time_used, mem_usage] + # Write to CSV file + csv_writer.writerow(line) + except KeyError: + # Skip the benchmark result if it does not contain the fields we want + pass + output.close() + + +def get_averages(args): + """Calculate the averages of time_used and memory_usage and append to CSV file. + + :param args: the command line parameters, including filename and tag + :type: str + """ + with open("{}.csv".format(os.path.splitext(args.filename)[0]), "rb") as input: + lines = input.readlines() + # Get the two columns of times_used and mem_usage + rows_of_data = [map(float, line.split(',')[-2:]) for line in lines] + # Calculate the sum of the two columns + col_sums = map(sum, zip(*rows_of_data)) + # Calculate the average of the two columns by using the sum divided by the total number of lines + averages = [col_sum / len(lines) for col_sum in col_sums] + input.close() + + # Get the runtime version from filename + runtime_version = os.path.basename(args.filename).split(".json")[0] + + # Write the averages to CSV file in appending mode + with open("{}/averages.csv".format(args.tag), "a+") as output: + try: + csv_writer = csv.writer(output, delimiter=',') + csv_writer.writerow([args.tag, runtime_version] + averages) + except IOError: + print "Could not write averages to file." + output.close() + + +def parse_args(argv): + """Parse and validate command line flags""" + parser = argparse.ArgumentParser( + description='Read the python performance json file and extract data to genarate CSV file.' + ) + parser.add_argument( + '--filename', + help='Filename of the performance json file to read' + ) + parser.add_argument( + '--tag', + help='Tag of the docker container' + ) + args = parser.parse_args(argv[1:]) + return args + + +def main(): + args = parse_args(sys.argv) + generate_csv(args) + get_averages(args) + + +if __name__ == '__main__': + main() From 4a2e096f27cb7eda046fa5a5ea560852bce742c6 Mon Sep 17 00:00:00 2001 From: Angela Li Date: Mon, 8 May 2017 10:46:34 -0700 Subject: [PATCH 160/362] Implemented the benchmark dashboard --- tests/benchmark/Dockerfile.in | 1 + tests/benchmark/benchmark_between_releases.sh | 46 ++++++-- tests/benchmark/generate_csv.py | 109 ++++++++++++++++++ 3 files changed, 148 insertions(+), 8 deletions(-) create mode 100644 tests/benchmark/generate_csv.py diff --git a/tests/benchmark/Dockerfile.in b/tests/benchmark/Dockerfile.in index 69f41862..5ca308f4 100644 --- a/tests/benchmark/Dockerfile.in +++ b/tests/benchmark/Dockerfile.in @@ -22,3 +22,4 @@ RUN pyperformance run --debug-single-value --python=python3.6 -o /result/py3.6.j RUN pyperformance compare /result/py2.7.json /result/py3.4.json --output_style table RUN pyperformance compare /result/py3.4.json /result/py3.5.json --output_style table RUN pyperformance compare /result/py3.5.json /result/py3.6.json --output_style table + diff --git a/tests/benchmark/benchmark_between_releases.sh b/tests/benchmark/benchmark_between_releases.sh index c540ae7f..11fda0eb 100755 --- a/tests/benchmark/benchmark_between_releases.sh +++ b/tests/benchmark/benchmark_between_releases.sh @@ -17,19 +17,49 @@ rm Dockerfile echo "Successfully built images" # Create folders to hold the files -mkdir release1 -mkdir release2 +mkdir $TAG1 +mkdir $TAG2 # Start running the containers and copy the benchmark result for python versions from container to host -docker run -it --name benchmark_1 -h CONTAINER1 -v "${PWD}"/release1:/export benchmark_1 /bin/bash -c "cp /result/py*.json /export/" -docker run -it --name benchmark_2 -h CONTAINER2 -v "${PWD}"/release2:/export benchmark_2 /bin/bash -c "cp /result/py*.json /export/" +docker run -it --name benchmark_1 -h CONTAINER1 -v "${PWD}"/"$TAG1":/export benchmark_1 /bin/bash -c "cp /result/py*.json /export/" +docker run -it --name benchmark_2 -h CONTAINER2 -v "${PWD}"/"$TAG2":/export benchmark_2 /bin/bash -c "cp /result/py*.json /export/" echo "Start benchmarking the python interpreter performance between the two releases" # Compare the performance between the interpreter in different release -pyperformance compare release1/py2.7.json release2/py2.7.json --output_style table > py2.7_res -pyperformance compare release1/py3.4.json release2/py3.4.json --output_style table > py3.4_res -pyperformance compare release1/py3.5.json release2/py3.5.json --output_style table > py3.5_res -pyperformance compare release1/py3.6.json release2/py3.6.json --output_style table > py3.6_res +pyperformance compare "$TAG1"/py2.7.json "$TAG2"/py2.7.json --output_style table > py2.7_res +pyperformance compare "$TAG1"/py3.4.json "$TAG2"/py3.4.json --output_style table > py3.4_res +pyperformance compare "$TAG1"/py3.5.json "$TAG2"/py3.5.json --output_style table > py3.5_res +pyperformance compare "$TAG1"/py3.6.json "$TAG2"/py3.6.json --output_style table > py3.6_res + +echo "Start extracting data and generating CSV file, then upload to Cloud Storage and insert to Big Query table" + +# Extracting memory usage and running time data from the performace result json, generating CSV files +for path_to_file in $TAG1/*.json; do + python generate_csv.py --filename $path_to_file --tag $TAG1 +done + +for path_to_file in $TAG2/*.json; do + python generate_csv.py --filename $path_to_file --tag $TAG2 +done + +# Get the list of existed release data on Cloud Storage and skip if the current TAG1 or TAG2 existed in the list +gsutil ls gs://python_runtime_benchmark > existed_releases + +for container_tag in $TAG1 $TAG2; do + if grep --fixed-strings --quiet "$container_tag" existed_releases; then + echo "Performace data of $container_tag existed, so skip processing it." + else + # Upload the CSV files to Cloud Storage + gsutil cp -r $container_tag gs://python_runtime_benchmark + # Load the CSV files from Cloud Storage to Big Query table + # Load the performance data of each function + for path_to_file in $container_tag/py2.7.csv $container_tag/py3.4.csv $container_tag/py3.5.csv; do + bq load benchmark_test.benchmark_functions gs://python_runtime_benchmark/"$path_to_file" container_tag:string,runtime_version:string,function_name:string,time_used:float,mem_usage:float + done + # Load the average performance data of each runtime version in a release + bq load benchmark_test.benchmark_statistics gs://python_runtime_benchmark/"$container_tag"/averages.csv container_tag:string,runtime_version:string,ave_time_used:float,ave_mem_usage:float + fi +done echo "Completed" diff --git a/tests/benchmark/generate_csv.py b/tests/benchmark/generate_csv.py new file mode 100644 index 00000000..d42ca745 --- /dev/null +++ b/tests/benchmark/generate_csv.py @@ -0,0 +1,109 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import argparse +import csv +import json +import os +import sys + + +def generate_csv(filename, tag): + """Extract function name, time used and memory usage from the metadata and write to the output CSV file. + + Args: + filename (str): Filename of the performance json file to read + tag (str): Tag of the docker container + """ + with open(filename) as input: + data = json.load(input) + benchmarks = data["benchmarks"] + runtime_version = os.path.basename(filename).split(".json")[0] + + # Write data to CSV file + with open("{}.csv".format(os.path.splitext(filename)[0]), "wb") as output: + csv_writer = csv.writer(output, delimiter=',') + for benchmark in benchmarks: + try: + # Get the function name + func_name = benchmark["metadata"]["name"] + # Get the time used for this function, convert to millisecond + time_used = benchmark["runs"][0]["values"][0] * 1000 + # Get the memory usage, convert to MB + mem_usage = benchmark["metadata"]["mem_max_rss"] / float(1<<20) + line = [tag, runtime_version, func_name, time_used, mem_usage] + # Write to CSV file + csv_writer.writerow(line) + except KeyError: + # Skip the benchmark result if it does not contain the fields we want + pass + output.close() + + +def get_averages(filename, tag): + """Calculate the averages of time_used and memory_usage and append to CSV file. + + Args: + filename (str): Filename of the performance json file to read + tag (str): Tag of the docker container + """ + with open("{}.csv".format(os.path.splitext(filename)[0]), "rb") as input: + lines = input.readlines() + # Get the two columns of times_used and mem_usage + rows_of_data = [map(float, line.split(',')[-2:]) for line in lines] + # Calculate the sum of the two columns + col_sums = map(sum, zip(*rows_of_data)) + # Calculate the average of the two columns by using the sum divided by the total number of lines + averages = [col_sum / len(lines) for col_sum in col_sums] + input.close() + + # Get the runtime version from filename + runtime_version = os.path.basename(filename).split(".json")[0] + + # Write the averages to CSV file in appending mode + with open("{}/averages.csv".format(tag), "a+") as output: + try: + csv_writer = csv.writer(output, delimiter=',') + csv_writer.writerow([tag, runtime_version] + averages) + except IOError: + print "Could not write averages to file." + output.close() + + +def parse_args(argv): + """Parse and validate command line flags""" + parser = argparse.ArgumentParser( + description='Read the python performance json file and extract data to genarate CSV file.' + ) + parser.add_argument( + '--filename', + help='Filename of the performance json file to read' + ) + parser.add_argument( + '--tag', + help='Tag of the docker container' + ) + args = parser.parse_args(argv[1:]) + return args + + +def main(): + args = parse_args(sys.argv) + generate_csv(args.filename, args.tag) + get_averages(args.filename, args.tag) + + +if __name__ == '__main__': + main() From 5ee1e6c31c1a605d18679086986cfc2f58204a94 Mon Sep 17 00:00:00 2001 From: Angela Li Date: Mon, 8 May 2017 16:44:40 -0700 Subject: [PATCH 161/362] Removed the merging conflicts. --- tests/benchmark/generate_csv.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/benchmark/generate_csv.py b/tests/benchmark/generate_csv.py index e60fb439..e9882303 100644 --- a/tests/benchmark/generate_csv.py +++ b/tests/benchmark/generate_csv.py @@ -42,11 +42,7 @@ def generate_csv(filename, tag): # Get the time used for this function, convert to millisecond time_used = float(benchmark["runs"][0]["values"][0]) * 1000 # Get the memory usage, convert to MB -<<<<<<< HEAD mem_usage = float(benchmark["metadata"]["mem_max_rss"]) / float(1<<20) -======= - mem_usage = benchmark["metadata"]["mem_max_rss"] / float(1<<20) ->>>>>>> 38dcf3b17c2eb6509002f92ac99f43b665ac5005 line = [tag, runtime_version, func_name, time_used, mem_usage] # Write to CSV file csv_writer.writerow(line) From a8d033a5be4cb0ceadf06c0bcd6c1362d2046e29 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Tue, 9 May 2017 10:45:20 -0700 Subject: [PATCH 162/362] Fix the requirements updating script (#115) --- nox.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/nox.py b/nox.py index 6d2e6723..83be1ffa 100644 --- a/nox.py +++ b/nox.py @@ -15,10 +15,6 @@ import fnmatch import os -# Location of our common testing utilities. This isn't published to PyPI. -GCP_REPO_TOOLS_REQ =\ - 'git+https://github.com/GoogleCloudPlatform/python-repo-tools.git' - def _list_files(folder, pattern): """Lists all files below the given folder that match the pattern.""" @@ -30,7 +26,7 @@ def _list_files(folder, pattern): def session_check_requirements(session): """Checks for out of date requirements and optionally updates them.""" - session.install(GCP_REPO_TOOLS_REQ) + session.install('gcp-devrel-py-tools') if 'update' in session.posargs: command = 'update-requirements' @@ -40,4 +36,4 @@ def session_check_requirements(session): reqfiles = list(_list_files('.', 'requirements*.txt')) for reqfile in reqfiles: - session.run('gcprepotools', command, reqfile) + session.run('gcp-devrel-py-tools', command, reqfile) From 0ef2f0b76e0c7012a3268c8ac0242144d37ce44b Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Mon, 8 May 2017 18:06:36 -0700 Subject: [PATCH 163/362] Apply patch to fix double-build issue. Source is https://github.com/python/cpython/pull/1478, backported to 3.5 and 3.6. --- python-interpreter-builder/Dockerfile.in | 1 + .../patches/3.5/double-build.diff | 96 +++++++++++++++++++ python-interpreter-builder/patches/3.5/series | 1 + .../patches/3.6/double-build.diff | 96 +++++++++++++++++++ python-interpreter-builder/patches/3.6/series | 1 + .../scripts/build-python-3.5.sh | 7 +- .../scripts/build-python-3.6.sh | 7 +- 7 files changed, 199 insertions(+), 10 deletions(-) create mode 100644 python-interpreter-builder/patches/3.5/double-build.diff create mode 100644 python-interpreter-builder/patches/3.5/series create mode 100644 python-interpreter-builder/patches/3.6/double-build.diff create mode 100644 python-interpreter-builder/patches/3.6/series diff --git a/python-interpreter-builder/Dockerfile.in b/python-interpreter-builder/Dockerfile.in index 9323f750..4ce3bd02 100644 --- a/python-interpreter-builder/Dockerfile.in +++ b/python-interpreter-builder/Dockerfile.in @@ -44,6 +44,7 @@ ENV LANG C.UTF-8 # Add build scripts ADD scripts /scripts +ADD patches /patches # Build the Python interpreters RUN /scripts/build-python-3.5.sh diff --git a/python-interpreter-builder/patches/3.5/double-build.diff b/python-interpreter-builder/patches/3.5/double-build.diff new file mode 100644 index 00000000..cb52c64b --- /dev/null +++ b/python-interpreter-builder/patches/3.5/double-build.diff @@ -0,0 +1,96 @@ +# Source is https://github.com/python/cpython/pull/1478 + +Index: Python-3.5.3/Makefile.pre.in +=================================================================== +--- Python-3.5.3.orig/Makefile.pre.in ++++ Python-3.5.3/Makefile.pre.in +@@ -982,7 +982,7 @@ TESTTIMEOUT= 3600 + + # Run a basic set of regression tests. + # This excludes some tests that are particularly resource-intensive. +-test: all platform ++test: @DEF_MAKE_RULE@ platform + $(TESTRUNNER) $(TESTOPTS) + + # Run the full test suite twice - once without .pyc files, and once with. +@@ -992,7 +992,7 @@ test: all platform + # the bytecode read from a .pyc file had the bug, sometimes the directly + # generated bytecode. This is sometimes a very shy bug needing a lot of + # sample data. +-testall: all platform ++testall: @DEF_MAKE_RULE@ platform + -find $(srcdir)/Lib -name '*.py[co]' -print | xargs rm -f + $(TESTPYTHON) -E $(srcdir)/Lib/compileall.py + -find $(srcdir)/Lib -name '*.py[co]' -print | xargs rm -f +@@ -1001,7 +1001,7 @@ testall: all platform + + # Run the test suite for both architectures in a Universal build on OSX. + # Must be run on an Intel box. +-testuniversal: all platform ++testuniversal: @DEF_MAKE_RULE@ platform + if [ `arch` != 'i386' ];then \ + echo "This can only be used on OSX/i386" ;\ + exit 1 ;\ +@@ -1024,7 +1024,7 @@ QUICKTESTOPTS= $(TESTOPTS) -x test_subpr + test_multiprocessing_forkserver \ + test_mailbox test_socket test_poll \ + test_select test_zipfile test_concurrent_futures +-quicktest: all platform ++quicktest: @DEF_MAKE_RULE@ platform + $(TESTRUNNER) $(QUICKTESTOPTS) + + +@@ -1376,7 +1376,7 @@ LIBPL= @LIBPL@ + # pkgconfig directory + LIBPC= $(LIBDIR)/pkgconfig + +-libainstall: all python-config ++libainstall: @DEF_MAKE_RULE@ python-config + @for i in $(LIBDIR) $(LIBPL) $(LIBPC); \ + do \ + if test ! -d $(DESTDIR)$$i; then \ +@@ -1635,7 +1635,7 @@ distclean: clobber + -exec rm -f {} ';' + + # Check for smelly exported symbols (not starting with Py/_Py) +-smelly: all ++smelly: @DEF_MAKE_RULE@ + nm -p $(LIBRARY) | \ + sed -n "/ [TDB] /s/.* //p" | grep -v "^_*Py" | sort -u; \ + +@@ -1673,7 +1673,7 @@ funny: + -o -print + + # Perform some verification checks on any modified files. +-patchcheck: all ++patchcheck: @DEF_MAKE_RULE@ + $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/scripts/patchcheck.py + + # Dependencies +Index: Python-3.5.3/Misc/ACKS +=================================================================== +--- Python-3.5.3.orig/Misc/ACKS ++++ Python-3.5.3/Misc/ACKS +@@ -1092,6 +1092,7 @@ Jason Orendorff + Douglas Orr + William Orr + Michele Orrù ++Tomáš Orsava + Oleg Oshmyan + Denis S. Otkidach + Peter Otten +Index: Python-3.5.3/Misc/NEWS +=================================================================== +--- Python-3.5.3.orig/Misc/NEWS ++++ Python-3.5.3/Misc/NEWS +@@ -634,6 +634,10 @@ Windows + Build + ----- + ++- bpo-29243: Prevent unnecessary rebuilding of Python during ``make test``, ++ ``make install`` and some other make targets when configured with ++ ``--enable-optimizations``. ++ + - Issue #29080: Removes hard dependency on hg.exe from PCBuild/build.bat + + - Issue #23903: Added missed names to PC/python3.def. diff --git a/python-interpreter-builder/patches/3.5/series b/python-interpreter-builder/patches/3.5/series new file mode 100644 index 00000000..f6c2876b --- /dev/null +++ b/python-interpreter-builder/patches/3.5/series @@ -0,0 +1 @@ +double-build.diff diff --git a/python-interpreter-builder/patches/3.6/double-build.diff b/python-interpreter-builder/patches/3.6/double-build.diff new file mode 100644 index 00000000..a89a20f6 --- /dev/null +++ b/python-interpreter-builder/patches/3.6/double-build.diff @@ -0,0 +1,96 @@ +# Source is https://github.com/python/cpython/pull/1478 + +Index: Python-3.6.1/Makefile.pre.in +=================================================================== +--- Python-3.6.1.orig/Makefile.pre.in ++++ Python-3.6.1/Makefile.pre.in +@@ -1000,7 +1000,7 @@ TESTTIMEOUT= 1200 + + # Run a basic set of regression tests. + # This excludes some tests that are particularly resource-intensive. +-test: all platform ++test: @DEF_MAKE_RULE@ platform + $(TESTRUNNER) $(TESTOPTS) + + # Run the full test suite twice - once without .pyc files, and once with. +@@ -1010,7 +1010,7 @@ test: all platform + # the bytecode read from a .pyc file had the bug, sometimes the directly + # generated bytecode. This is sometimes a very shy bug needing a lot of + # sample data. +-testall: all platform ++testall: @DEF_MAKE_RULE@ platform + -find $(srcdir)/Lib -name '*.py[co]' -print | xargs rm -f + $(TESTPYTHON) -E $(srcdir)/Lib/compileall.py + -find $(srcdir)/Lib -name '*.py[co]' -print | xargs rm -f +@@ -1019,7 +1019,7 @@ testall: all platform + + # Run the test suite for both architectures in a Universal build on OSX. + # Must be run on an Intel box. +-testuniversal: all platform ++testuniversal: @DEF_MAKE_RULE@ platform + if [ `arch` != 'i386' ];then \ + echo "This can only be used on OSX/i386" ;\ + exit 1 ;\ +@@ -1042,7 +1042,7 @@ QUICKTESTOPTS= $(TESTOPTS) -x test_subpr + test_multiprocessing_forkserver \ + test_mailbox test_socket test_poll \ + test_select test_zipfile test_concurrent_futures +-quicktest: all platform ++quicktest: @DEF_MAKE_RULE@ platform + $(TESTRUNNER) $(QUICKTESTOPTS) + + +@@ -1379,7 +1379,7 @@ LIBPL= @LIBPL@ + # pkgconfig directory + LIBPC= $(LIBDIR)/pkgconfig + +-libainstall: all python-config ++libainstall: @DEF_MAKE_RULE@ python-config + @for i in $(LIBDIR) $(LIBPL) $(LIBPC); \ + do \ + if test ! -d $(DESTDIR)$$i; then \ +@@ -1639,7 +1639,7 @@ distclean: clobber + -exec rm -f {} ';' + + # Check for smelly exported symbols (not starting with Py/_Py) +-smelly: all ++smelly: @DEF_MAKE_RULE@ + nm -p $(LIBRARY) | \ + sed -n "/ [TDB] /s/.* //p" | grep -v "^_*Py" | sort -u; \ + +@@ -1676,7 +1676,7 @@ funny: + -o -print + + # Perform some verification checks on any modified files. +-patchcheck: all ++patchcheck: @DEF_MAKE_RULE@ + $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/scripts/patchcheck.py + + # Dependencies +Index: Python-3.6.1/Misc/ACKS +=================================================================== +--- Python-3.6.1.orig/Misc/ACKS ++++ Python-3.6.1/Misc/ACKS +@@ -1111,6 +1111,7 @@ Jason Orendorff + Douglas Orr + William Orr + Michele Orrù ++Tomáš Orsava + Oleg Oshmyan + Denis S. Otkidach + Peter Otten +Index: Python-3.6.1/Misc/NEWS +=================================================================== +--- Python-3.6.1.orig/Misc/NEWS ++++ Python-3.6.1/Misc/NEWS +@@ -306,6 +306,10 @@ Tests + Build + ----- + ++- bpo-29243: Prevent unnecessary rebuilding of Python during ``make test``, ++ ``make install`` and some other make targets when configured with ++ ``--enable-optimizations``. ++ + - bpo-27593: sys.version and the platform module python_build(), + python_branch(), and python_revision() functions now use + git information rather than hg when building from a repo. diff --git a/python-interpreter-builder/patches/3.6/series b/python-interpreter-builder/patches/3.6/series new file mode 100644 index 00000000..f6c2876b --- /dev/null +++ b/python-interpreter-builder/patches/3.6/series @@ -0,0 +1 @@ +double-build.diff diff --git a/python-interpreter-builder/scripts/build-python-3.5.sh b/python-interpreter-builder/scripts/build-python-3.5.sh index a80473d8..8f536144 100755 --- a/python-interpreter-builder/scripts/build-python-3.5.sh +++ b/python-interpreter-builder/scripts/build-python-3.5.sh @@ -13,8 +13,9 @@ d8890b84d773cd7059e597dbefa510340de8336ec9b9e9032bf030f19291565a Python-3.5.3.t EOF tar xzf Python-3.5.3.tgz -# Build +# Apply patches cd Python-3.5.3 +QUILT_PATCHES=/patches/3.5 quilt push -a # Explanation of flags: # @@ -115,10 +116,6 @@ cd build-static LDFLAGS="-Wl,-z,relro" \ RANLIB="x86_64-linux-gnu-gcc-ranlib" \ -# Due to https://bugs.python.org/issue29243, "make altinstall" -# rebuilds everything from scratch, twice. This is a workaround. -sed -i 's/^all:.*$/all: build_all/' Makefile - make profile-opt make altinstall diff --git a/python-interpreter-builder/scripts/build-python-3.6.sh b/python-interpreter-builder/scripts/build-python-3.6.sh index cb4fa360..05a4e00e 100755 --- a/python-interpreter-builder/scripts/build-python-3.6.sh +++ b/python-interpreter-builder/scripts/build-python-3.6.sh @@ -13,8 +13,9 @@ aa50b0143df7c89ce91be020fe41382613a817354b33acdc6641b44f8ced3828 Python-3.6.1.t EOF tar xzf Python-3.6.1.tgz -# Build +# Apply patches cd Python-3.6.1 +QUILT_PATCHES=/patches/3.6 quilt push -a # Explanation of flags: # @@ -115,10 +116,6 @@ cd build-static LDFLAGS="-Wl,-z,relro" \ RANLIB="x86_64-linux-gnu-gcc-ranlib" \ -# Due to https://bugs.python.org/issue29243, "make altinstall" -# rebuilds everything from scratch, twice. This is a workaround. -sed -i 's/^all:.*$/all: build_all/' Makefile - make profile-opt make altinstall From 3e4599c416654ad00323134707bc7d2b850c372d Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Mon, 8 May 2017 19:17:58 -0700 Subject: [PATCH 164/362] Run Python self-tests when building interpreter Fixes #107 --- python-interpreter-builder/scripts/build-python-3.5.sh | 6 ++++++ python-interpreter-builder/scripts/build-python-3.6.sh | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/python-interpreter-builder/scripts/build-python-3.5.sh b/python-interpreter-builder/scripts/build-python-3.5.sh index 8f536144..32b21ead 100755 --- a/python-interpreter-builder/scripts/build-python-3.5.sh +++ b/python-interpreter-builder/scripts/build-python-3.5.sh @@ -119,6 +119,12 @@ cd build-static make profile-opt make altinstall +# Run tests +# test___all__: Depends on Debian-specific locale changes +# test_imap: https://bugs.python.org/issue30175 +# test_shutil: https://bugs.python.org/issue29317 +make test TESTOPTS="--exclude test___all__ test_imaplib test_shutil" + # Clean-up sources cd /opt rm /opt/sources/Python-3.5.3.tgz diff --git a/python-interpreter-builder/scripts/build-python-3.6.sh b/python-interpreter-builder/scripts/build-python-3.6.sh index 05a4e00e..1d697756 100755 --- a/python-interpreter-builder/scripts/build-python-3.6.sh +++ b/python-interpreter-builder/scripts/build-python-3.6.sh @@ -119,6 +119,12 @@ cd build-static make profile-opt make altinstall +# Run tests +# test___all__: Depends on Debian-specific locale changes +# test_imap: https://bugs.python.org/issue30175 +# test_shutil: https://bugs.python.org/issue29317 +make test TESTOPTS="--exclude test___all__ test_imaplib test_shutil" + # Clean-up sources cd /opt rm /opt/sources/Python-3.6.1.tgz From 26255b491d55390f93fb225e0f3ebc93405e9ffb Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Tue, 9 May 2017 13:15:20 -0700 Subject: [PATCH 165/362] Disable test_dbm for Python 3.6 --- python-interpreter-builder/scripts/build-python-3.6.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python-interpreter-builder/scripts/build-python-3.6.sh b/python-interpreter-builder/scripts/build-python-3.6.sh index 1d697756..9ad508e0 100755 --- a/python-interpreter-builder/scripts/build-python-3.6.sh +++ b/python-interpreter-builder/scripts/build-python-3.6.sh @@ -121,9 +121,10 @@ make altinstall # Run tests # test___all__: Depends on Debian-specific locale changes +# test_dbm: https://bugs.python.org/issue28700 # test_imap: https://bugs.python.org/issue30175 # test_shutil: https://bugs.python.org/issue29317 -make test TESTOPTS="--exclude test___all__ test_imaplib test_shutil" +make test TESTOPTS="--exclude test___all__ test_dbm test_imaplib test_shutil" # Clean-up sources cd /opt From 97dec06d02c608305917036ecccea59b0dd13163 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Tue, 9 May 2017 18:18:17 -0700 Subject: [PATCH 166/362] Actually run the structure tests for Python 3.6 --- cloudbuild.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 32c501b2..f242ac8e 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -18,6 +18,7 @@ steps: '--config', '/workspace/tests/virtualenv/virtualenv_default.yaml', '--config', '/workspace/tests/virtualenv/virtualenv_python34.yaml', '--config', '/workspace/tests/virtualenv/virtualenv_python35.yaml', + '--config', '/workspace/tests/virtualenv/virtualenv_python36.yaml', '--config', '/workspace/tests/no-virtualenv/no-virtualenv.yaml', '--config', '/workspace/tests/python2-libraries/python2-libraries.yaml', '--config', '/workspace/tests/python3-libraries/python3-libraries.yaml', From 37d384d1b83d66e77db689740d5d017f65c4167a Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Tue, 9 May 2017 18:18:56 -0700 Subject: [PATCH 167/362] Perform standard Docker cleanup after running apt-get --- python-interpreter-builder/Dockerfile.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python-interpreter-builder/Dockerfile.in b/python-interpreter-builder/Dockerfile.in index 4ce3bd02..ee9fecf3 100644 --- a/python-interpreter-builder/Dockerfile.in +++ b/python-interpreter-builder/Dockerfile.in @@ -36,8 +36,8 @@ RUN apt-get update && apt-get install -yq \ wget \ xauth \ xvfb \ - zlib1g-dev - + zlib1g-dev \ + && rm -rf /var/lib/apt/lists/* # Setup locale. This prevents Python 3 IO encoding issues. ENV LANG C.UTF-8 From 8fcf1d94aee91b1dc9bf3728e87f5b08245d28de Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Tue, 9 May 2017 18:19:48 -0700 Subject: [PATCH 168/362] Remove unused files to slim down the image Add tests that the 'test' standard library module is functional. --- .../scripts/build-python-3.5.sh | 27 ++++++++++++++++--- .../scripts/build-python-3.6.sh | 24 +++++++++++++++-- tests/virtualenv/virtualenv_python34.yaml | 3 +++ tests/virtualenv/virtualenv_python35.yaml | 3 +++ tests/virtualenv/virtualenv_python36.yaml | 5 +++- 5 files changed, 56 insertions(+), 6 deletions(-) diff --git a/python-interpreter-builder/scripts/build-python-3.5.sh b/python-interpreter-builder/scripts/build-python-3.5.sh index 32b21ead..d43b4bf6 100755 --- a/python-interpreter-builder/scripts/build-python-3.5.sh +++ b/python-interpreter-builder/scripts/build-python-3.5.sh @@ -87,6 +87,8 @@ QUILT_PATCHES=/patches/3.5 quilt push -a # Specifically EXTRA_CFLAGS="-g -flto -fuse-linker-plugin # -ffat-lto-objects" +PREFIX=/opt/python3.5 + mkdir build-static cd build-static @@ -94,7 +96,7 @@ cd build-static --enable-ipv6 \ --enable-loadable-sqlite-extensions \ --enable-optimizations \ - --prefix=/opt/python3.5 \ + --prefix="$PREFIX" \ --with-dbmliborder=bdb:gdbm \ --with-computed-gotos \ --with-fpectl \ @@ -117,13 +119,32 @@ cd build-static RANLIB="x86_64-linux-gnu-gcc-ranlib" \ make profile-opt -make altinstall # Run tests # test___all__: Depends on Debian-specific locale changes +# test_dbm: https://bugs.python.org/issue28700 # test_imap: https://bugs.python.org/issue30175 # test_shutil: https://bugs.python.org/issue29317 -make test TESTOPTS="--exclude test___all__ test_imaplib test_shutil" +make test TESTOPTS="--exclude test___all__ test_dbm test_imaplib test_shutil" + +# Install +make altinstall +# We don't expect users to statically link Python into a C/C++ program +rm "$PREFIX"/lib/libpython3.5m.a \ + "$PREFIX"/lib/python3.5/config-*/libpython3.5m.a +# Remove opt-mode bytecode +find "$PREFIX"/lib/python3.5/ \ + -name \*.opt-\?.pyc \ + -exec rm {} \; +# Remove all but a few files in the 'test' subdirectory +find "$PREFIX"/lib/python3.5/test \ + -mindepth 1 -maxdepth 1 \ + \! -name support \ + -a \! -name __init__.py \ + -a \! -name pystone.\* \ + -a \! -name regrtest.\* \ + -a \! -name test_support.py \ + -exec rm -rf {} \; # Clean-up sources cd /opt diff --git a/python-interpreter-builder/scripts/build-python-3.6.sh b/python-interpreter-builder/scripts/build-python-3.6.sh index 9ad508e0..d8cdf023 100755 --- a/python-interpreter-builder/scripts/build-python-3.6.sh +++ b/python-interpreter-builder/scripts/build-python-3.6.sh @@ -87,6 +87,8 @@ QUILT_PATCHES=/patches/3.6 quilt push -a # Specifically EXTRA_CFLAGS="-g -flto -fuse-linker-plugin # -ffat-lto-objects" +PREFIX=/opt/python3.6 + mkdir build-static cd build-static @@ -94,7 +96,7 @@ cd build-static --enable-ipv6 \ --enable-loadable-sqlite-extensions \ --enable-optimizations \ - --prefix=/opt/python3.6 \ + --prefix="$PREFIX" \ --with-dbmliborder=bdb:gdbm \ --with-computed-gotos \ --with-fpectl \ @@ -117,7 +119,6 @@ cd build-static RANLIB="x86_64-linux-gnu-gcc-ranlib" \ make profile-opt -make altinstall # Run tests # test___all__: Depends on Debian-specific locale changes @@ -126,6 +127,25 @@ make altinstall # test_shutil: https://bugs.python.org/issue29317 make test TESTOPTS="--exclude test___all__ test_dbm test_imaplib test_shutil" +# Install +make altinstall +# We don't expect users to statically link Python into a C/C++ program +rm "$PREFIX"/lib/libpython3.6m.a \ + "$PREFIX"/lib/python3.6/config-*/libpython3.6m.a +# Remove opt-mode bytecode +find "$PREFIX"/lib/python3.6/ \ + -name \*.opt-\?.pyc \ + -exec rm {} \; +# Remove all but a few files in the 'test' subdirectory +find "$PREFIX"/lib/python3.6/test \ + -mindepth 1 -maxdepth 1 \ + \! -name support \ + -a \! -name __init__.py \ + -a \! -name pystone.\* \ + -a \! -name regrtest.\* \ + -a \! -name test_support.py \ + -exec rm -rf {} \; + # Clean-up sources cd /opt rm /opt/sources/Python-3.6.1.tgz diff --git a/tests/virtualenv/virtualenv_python34.yaml b/tests/virtualenv/virtualenv_python34.yaml index 0a498186..082b3b6e 100644 --- a/tests/virtualenv/virtualenv_python34.yaml +++ b/tests/virtualenv/virtualenv_python34.yaml @@ -42,3 +42,6 @@ commandTests: - name: "flask integration" command: ["python", "-c", "\"import sys; import flask; sys.exit(0 if flask.__file__.startswith('/env') else 1)\""] + + - name: "test.support" + command: ["python", "-c", "\"from test import pystone, regrtest, support\""] diff --git a/tests/virtualenv/virtualenv_python35.yaml b/tests/virtualenv/virtualenv_python35.yaml index b37d1461..a9248f88 100644 --- a/tests/virtualenv/virtualenv_python35.yaml +++ b/tests/virtualenv/virtualenv_python35.yaml @@ -46,3 +46,6 @@ commandTests: - name: "flask integration" command: ["python", "-c", "\"import sys; import flask; sys.exit(0 if flask.__file__.startswith('/env') else 1)\""] + + - name: "test.support" + command: ["python", "-c", "\"from test import pystone, regrtest, support\""] diff --git a/tests/virtualenv/virtualenv_python36.yaml b/tests/virtualenv/virtualenv_python36.yaml index 241099b6..83938fc9 100644 --- a/tests/virtualenv/virtualenv_python36.yaml +++ b/tests/virtualenv/virtualenv_python36.yaml @@ -29,7 +29,7 @@ commandTests: - name: "python version" command: ["python", "--version"] - expectedOutput: ["Python 3.6.3\n"] + expectedOutput: ["Python 3.6.1\n"] - name: "pip installation" command: ["which", "pip"] @@ -46,3 +46,6 @@ commandTests: - name: "flask integration" command: ["python", "-c", "\"import sys; import flask; sys.exit(0 if flask.__file__.startswith('/env') else 1)\""] + + - name: "test.support" + command: ["python", "-c", "\"from test import pystone, regrtest, support\""] From 1d7490665baa6e4d3031802adeab4bcf8eaf292f Mon Sep 17 00:00:00 2001 From: DPE bot Date: Sun, 14 May 2017 09:07:52 +0000 Subject: [PATCH 169/362] Auto-update dependencies. --- tests/integration/requirements.txt | 4 +- tests/python2-libraries/requirements.txt | 78 ++++++++++----------- tests/python3-libraries/requirements.txt | 88 ++++++++++-------------- 3 files changed, 79 insertions(+), 91 deletions(-) diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index c5c38c56..71d785ae 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -1,7 +1,7 @@ Flask==0.12.1 -google-cloud-error-reporting==0.24.0 +google-cloud-error-reporting==0.24.2 google-cloud-logging==1.0.0 google-cloud-monitoring==0.24.0 gunicorn==19.7.1 -requests==2.13.0 +requests==2.14.2 retrying==1.3.3 diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 1f635b65..a2f9be77 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -1,46 +1,46 @@ alembic==0.9.1 amqp==2.1.4 amqplib==1.0.2 -ansible==2.2.2.0 +ansible==2.3.0.0 anyjson==0.3.3 -apache-libcloud==1.5.0 +apache-libcloud==2.0.0 argparse==1.4.0 -astroid==1.4.9 -awscli==1.11.76 +astroid==1.5.2 +awscli==1.11.85 babel==2.4.0 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 -beautifulsoup4==4.5.3 +beautifulsoup4==4.6.0 beautifulsoup==3.2.1 billiard==3.5.0.2 blessings==1.6 blinker==1.4 boto==2.46.1 -botocore==1.5.39 +botocore==1.5.48 bottle==0.12.13 -carbon==1.0.0 +carbon==1.0.1 celery==4.0.2 -certifi==2017.1.23 +certifi==2017.4.17 cffi==1.10.0 -chardet==3.0.1 +chardet==3.0.2 click==6.7 -cliff==2.5.0 +cliff==2.7.0 cmd2==0.7.0 -colorama==0.3.7 +colorama==0.3.9 configobj==5.0.6 cov-core==1.15.0 -coverage==4.3.4 +coverage==4.4 coveralls==1.1 cryptography==1.8.1 cssselect==1.0.1 cython==0.25.2 decorator==4.0.11 django-celery==3.2.1 -django-debug-toolbar==1.7 -django-extensions==1.7.8 -django==1.11 +django-debug-toolbar==1.8 +django-extensions==1.7.9 +django==1.11.1 django_compress==1.0.1 -djangorestframework==3.6.2 +djangorestframework==3.6.3 docker-py==1.10.6 docopt==0.6.2 docutils==0.13.1 @@ -49,16 +49,16 @@ elasticsearch==5.3.0 enum34==1.1.6 eventlet==0.21.0 extras==1.0.0 -fabric==1.13.1 +fabric==1.13.2 fixtures==3.0.0 flake8==3.3.0 flask==0.12.1 funcsigs==1.0.2 functools32==3.2.3.post2 -futures==3.0.5 +futures==3.1.1 gevent==1.2.1 google-api-python-client==1.6.2 -graphite-web==1.0.0 +graphite-web==1.0.1 greenlet==0.4.12 gunicorn==19.7.1 hiredis==0.2.0 @@ -66,7 +66,7 @@ html5lib httplib2==0.10.3 idna==2.5 ipaddress==1.0.18 -ipython==5.3.0 +ipython==6.0.0 iso8601==0.1.11 isodate==0.5.4 itsdangerous==0.24 @@ -82,7 +82,7 @@ mako==1.0.6 manifestparser==1.1 markdown==2.6.8 markupsafe==1.0 -matplotlib==2.0.0 +matplotlib==2.0.2 mccabe==0.6.1 meld3==1.0.2 mock==2.0.0 @@ -100,28 +100,28 @@ mysql-python==1.2.5 ndg-httpsclient==0.4.2 netaddr==0.7.19 netifaces==0.10.5 -newrelic==2.82.0.62 +newrelic==2.86.0.65 nose==1.3.7 numpy==1.12.1 oauth2==1.9.0.post1 -oauth2client==4.0.0 +oauth2client==4.1.0 oauthlib==2.0.2 ordereddict==1.1 -oslo.config==3.24.0 -pandas==0.19.2 +oslo.config==4.1.0 +pandas==0.20.1 paramiko==2.1.2 passlib==1.7.1 paste==2.0.3 pastedeploy==1.5.2 pastescript==2.0.2 -pbr==2.1.0 +pbr==3.0.0 pep8==1.7.0 pexpect==4.2.1 pika==0.10.0 -pillow==4.1.0 +pillow==4.1.1 pip==9.0.1 prettytable -protobuf==3.2.0 +protobuf==3.3.0 psutil==5.2.2 psycopg2==2.7.1 py==1.4.33 @@ -132,16 +132,16 @@ pycrypto==2.6.1 pycurl==7.43.0 pyflakes==1.5.0 pygments==2.2.0 -pyjwt==1.4.2 +pyjwt==1.5.0 pylibmc==1.5.2 -pylint==1.6.5 +pylint==1.7.1 pymongo==3.4.0 pymysql==0.7.11 -pyopenssl==16.2.0 +pyopenssl==17.0.0 pyparsing==2.2.0 pyramid==1.8.3 pystache==0.5.4 -pytest-cov==2.4.0 +pytest-cov==2.5.1 pytest==3.0.7 python-cjson==1.2.1 python-daemon==2.1.2 @@ -160,13 +160,13 @@ raven==6.0.0 redis==2.10.5 repoze.lru==0.6 requests-oauthlib==0.8.0 -requests==2.13.0 +requests==2.14.2 retrying==1.3.3 rsa==3.4.2 scipy==0.19.0 -selenium==3.3.3 +selenium==3.4.1 setuptools-git==1.2 -setuptools==34.4.1 +setuptools==35.0.2 sh==1.12.13 simplejson==3.10.0 six==1.10.0 @@ -181,16 +181,16 @@ stevedore==1.21.0 suds==0.4 supervisor==3.3.1 testrepository==0.0.20 -testtools==2.2.0 +testtools==2.3.0 thrift==0.10.0 -tornado==4.4.3 +tornado==4.5.1 tox==2.7.0 twisted==17.1.0 ujson==1.35 unidecode==0.4.20 unittest2==1.1.0 uritemplate==3.0.0 -urllib3==1.20 +urllib3==1.21.1 uwsgi==2.0.15 versiontools==1.9.1 virtualenv==15.1.0 @@ -203,4 +203,4 @@ werkzeug==0.12.1 wheel==0.29.0 xlrd==1.0.0 zc.buildout==2.9.3 -zope.interface==4.3.3 +zope.interface==4.4.1 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 6f971d8d..2364fbed 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -1,46 +1,44 @@ alembic==0.9.1 amqp==2.1.4 amqplib==1.0.2 -ansible==2.2.2.0 +ansible==2.3.0.0 anyjson==0.3.3 -apache-libcloud==1.5.0 +apache-libcloud==2.0.0 argparse==1.4.0 -astroid==1.4.9 -awscli==1.11.68 +astroid==1.5.2 +awscli==1.11.85 babel==2.4.0 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 -#beautifulsoup -beautifulsoup4==4.5.3 +beautifulsoup4==4.6.0 billiard==3.5.0.2 blessings==1.6 blinker==1.4 boto==2.46.1 -botocore==1.5.31 +botocore==1.5.48 bottle==0.12.13 -#carbon celery==4.0.2 -certifi==2017.1.23 +certifi==2017.4.17 cffi==1.10.0 -chardet==3.0.1 +chardet==3.0.2 click==6.7 -cliff==2.5.0 +cliff==2.7.0 cmd2==0.7.0 -colorama==0.3.7 +colorama==0.3.9 configobj==5.0.6 cov-core==1.15.0 -coverage==4.3.4 +coverage==4.4 coveralls==1.1 cryptography==1.8.1 cssselect==1.0.1 cython==0.25.2 decorator==4.0.11 django-celery==3.2.1 -django-debug-toolbar==1.7 -django-extensions==1.7.8 -django==1.11 +django-debug-toolbar==1.8 +django-extensions==1.7.9 +django==1.11.1 django_compress==1.0.1 -djangorestframework==3.6.2 +djangorestframework==3.6.3 docker-py==1.10.6 docopt==0.6.2 docutils==0.13.1 @@ -49,16 +47,14 @@ elasticsearch==5.3.0 enum34==1.1.6 eventlet==0.21.0 extras==1.0.0 -fabric==1.13.1 +fabric==1.13.2 fixtures==3.0.0 flake8==3.3.0 flask==0.12.1 funcsigs==1.0.2 -#functools32 -futures==3.0.5 +futures==3.1.1 gevent==1.2.1 google-api-python-client==1.6.2 -#graphite-web greenlet==0.4.12 gunicorn==19.7.1 hiredis==0.2.0 @@ -66,7 +62,7 @@ html5lib httplib2==0.10.3 idna==2.5 ipaddress==1.0.18 -ipython==5.3.0 +ipython==6.0.0 iso8601==0.1.11 isodate==0.5.4 itsdangerous==0.24 @@ -82,46 +78,43 @@ mako==1.0.6 manifestparser==1.1 markdown==2.6.8 markupsafe==1.0 -matplotlib==2.0.0 +matplotlib==2.0.2 mccabe==0.6.1 meld3==1.0.2 mock==2.0.0 mozcrash==1.0 -mozdevice==0.48 +mozdevice==0.50 mozfile==1.2 mozinfo==0.9 mozlog==3.4 moznetwork==0.27 mozprocess==0.25 -#mozprofile -#mozrunner msgpack-python==0.4.8 -#mysql-python ndg-httpsclient==0.4.2 netaddr==0.7.19 netifaces==0.10.5 -newrelic==2.82.0.62 +newrelic==2.86.0.65 nose==1.3.7 numpy==1.12.1 oauth2==1.9.0.post1 -oauth2client==4.0.0 +oauth2client==4.1.0 oauthlib==2.0.2 ordereddict==1.1 -oslo.config==3.24.0 -pandas==0.19.2 +oslo.config==4.1.0 +pandas==0.20.1 paramiko==2.1.2 passlib==1.7.1 paste==2.0.3 pastedeploy==1.5.2 pastescript==2.0.2 -pbr==2.0.0 +pbr==3.0.0 pep8==1.7.0 pexpect==4.2.1 pika==0.10.0 -pillow==4.1.0 +pillow==4.1.1 pip==9.0.1 prettytable -protobuf==3.2.0 +protobuf==3.3.0 psutil==5.2.2 psycopg2==2.7.1 py==1.4.33 @@ -129,21 +122,19 @@ pyasn1-modules==0.0.8 pyasn1==0.2.3 pycparser==2.17 pycrypto==2.6.1 -#pycurl pyflakes==1.5.0 pygments==2.2.0 -pyjwt==1.4.2 +pyjwt==1.5.0 pylibmc==1.5.2 -pylint==1.6.5 +pylint==1.7.1 pymongo==3.4.0 pymysql==0.7.11 -pyopenssl==16.2.0 +pyopenssl==17.0.0 pyparsing==2.2.0 pyramid==1.8.3 pystache==0.5.4 -pytest-cov==2.4.0 +pytest-cov==2.5.1 pytest==3.0.7 -#python-cjson python-daemon==2.1.2 python-dateutil==2.6.0 python-gflags==3.1.1 @@ -160,13 +151,13 @@ raven==6.0.0 redis==2.10.5 repoze.lru==0.6 requests-oauthlib==0.8.0 -requests==2.13.0 +requests==2.14.2 retrying==1.3.3 rsa==3.4.2 scipy==0.19.0 -selenium==3.3.1 +selenium==3.4.1 setuptools-git==1.2 -setuptools==34.3.3 +setuptools==35.0.2 sh==1.12.13 simplejson==3.10.0 six==1.10.0 @@ -178,20 +169,17 @@ sqlalchemy==1.1.9 sqlparse==0.2.3 statsd==3.2.1 stevedore==1.21.0 -#suds -#supervisor testrepository==0.0.20 -testtools==2.2.0 +testtools==2.3.0 thrift==0.10.0 -tornado==4.4.3 +tornado==4.5.1 tox==2.7.0 twisted==17.1.0 ujson==1.35 unidecode==0.4.20 unittest2==1.1.0 uritemplate==3.0.0 -urllib3==1.20 -#uwsgi +urllib3==1.21.1 versiontools==1.9.1 virtualenv==15.1.0 waitress==1.0.2 @@ -203,4 +191,4 @@ werkzeug==0.12.1 wheel==0.29.0 xlrd==1.0.0 zc.buildout==2.9.3 -zope.interface==4.3.3 +zope.interface==4.4.1 From 5d9d6de32cc1b1071e889534bf116dd1230a4c81 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Mon, 22 May 2017 17:59:32 -0700 Subject: [PATCH 170/362] Add libpython.a back into runtime image. Needed for things that statically link the Python interpreter into a program, such as the uWSGI web server. --- python-interpreter-builder/scripts/build-python-3.5.sh | 5 ++--- python-interpreter-builder/scripts/build-python-3.6.sh | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/python-interpreter-builder/scripts/build-python-3.5.sh b/python-interpreter-builder/scripts/build-python-3.5.sh index d43b4bf6..09653669 100755 --- a/python-interpreter-builder/scripts/build-python-3.5.sh +++ b/python-interpreter-builder/scripts/build-python-3.5.sh @@ -129,9 +129,8 @@ make test TESTOPTS="--exclude test___all__ test_dbm test_imaplib test_shutil" # Install make altinstall -# We don't expect users to statically link Python into a C/C++ program -rm "$PREFIX"/lib/libpython3.5m.a \ - "$PREFIX"/lib/python3.5/config-*/libpython3.5m.a +# Remove redundant copy of libpython +rm "$PREFIX"/lib/libpython3.5m.a # Remove opt-mode bytecode find "$PREFIX"/lib/python3.5/ \ -name \*.opt-\?.pyc \ diff --git a/python-interpreter-builder/scripts/build-python-3.6.sh b/python-interpreter-builder/scripts/build-python-3.6.sh index d8cdf023..d3eb0c33 100755 --- a/python-interpreter-builder/scripts/build-python-3.6.sh +++ b/python-interpreter-builder/scripts/build-python-3.6.sh @@ -129,9 +129,8 @@ make test TESTOPTS="--exclude test___all__ test_dbm test_imaplib test_shutil" # Install make altinstall -# We don't expect users to statically link Python into a C/C++ program -rm "$PREFIX"/lib/libpython3.6m.a \ - "$PREFIX"/lib/python3.6/config-*/libpython3.6m.a +# Remove redundant copy of libpython +rm "$PREFIX"/lib/libpython3.6m.a # Remove opt-mode bytecode find "$PREFIX"/lib/python3.6/ \ -name \*.opt-\?.pyc \ From e7529122cd5862dea808fddb930cc3a6ef632590 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Mon, 22 May 2017 18:00:31 -0700 Subject: [PATCH 171/362] Test that uWSGI installs properly in Python 3.x --- tests/python3-libraries/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 2364fbed..e99f3629 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -180,6 +180,7 @@ unidecode==0.4.20 unittest2==1.1.0 uritemplate==3.0.0 urllib3==1.21.1 +uwsgi==2.0.15 versiontools==1.9.1 virtualenv==15.1.0 waitress==1.0.2 From b3ef908ef3a1f7ed3c4420394f45b0c0ff972651 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Mon, 22 May 2017 18:00:52 -0700 Subject: [PATCH 172/362] Workaround the fact that ipython 6.x does not support Python 2 --- tests/python2-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index a2f9be77..61e36d58 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -66,7 +66,7 @@ html5lib httplib2==0.10.3 idna==2.5 ipaddress==1.0.18 -ipython==6.0.0 +#ipython # No longer supports Python 2 as of version 6.0 iso8601==0.1.11 isodate==0.5.4 itsdangerous==0.24 From 6f2275b44ff4a4b083f013f014fc37849952c79e Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Thu, 15 Jun 2017 18:20:56 -0700 Subject: [PATCH 173/362] Move some utility functions into their own file --- scripts/local_cloudbuild.py | 96 ++++----------------------- scripts/local_cloudbuild_test.py | 83 +---------------------- scripts/validation_utils.py | 98 +++++++++++++++++++++++++++ scripts/validation_utils_test.py | 110 +++++++++++++++++++++++++++++++ 4 files changed, 221 insertions(+), 166 deletions(-) create mode 100644 scripts/validation_utils.py create mode 100755 scripts/validation_utils_test.py diff --git a/scripts/local_cloudbuild.py b/scripts/local_cloudbuild.py index f134278d..f0c6659d 100755 --- a/scripts/local_cloudbuild.py +++ b/scripts/local_cloudbuild.py @@ -42,6 +42,8 @@ import yaml +import validation_utils + # Exclude non-printable control characters (including newlines) PRINTABLE_REGEX = re.compile(r"""^[^\x00-\x1f]*$""") @@ -59,10 +61,6 @@ ) """) -# For easier development, we allow redefining builtins like -# --substitutions=PROJECT_ID=foo even though gcloud doesn't. -KEY_VALUE_REGEX = re.compile(r'^([A-Z_][A-Z0-9_]*)=(.*)$') - # Default builtin substitutions DEFAULT_SUBSTITUTIONS = { 'BRANCH_NAME': '', @@ -149,51 +147,6 @@ def sub(match): quoted_s = shlex.quote(substituted_s) return quoted_s -def get_field_value(container, field_name, field_type): - """Fetch a field from a container with typechecking and default values. - - The field value is coerced to the desired type. If the field is - not present, a instance of `field_type` is constructed with no - arguments and used as the default value. - - Args: - container (dict): Object decoded from yaml - field_name (str): Field that should be present in `container` - field_type (type): Expected type for field value - - Returns: - Any: Fetched or default value of field - - Raises: - ValueError: if field value cannot be converted to the desired type - """ - try: - value = container[field_name] - except (IndexError, KeyError): - return field_type() - - msg = 'Expected "{}" field to be of type "{}", but found type "{}"' - if not isinstance(value, field_type): - # list('some string') is a successful type cast as far as Python - # is concerned, but doesn't exactly produce the results we want. - # We have a whitelist of conversions we will attempt. - whitelist = ( - (float, str), - (int, str), - (str, float), - (str, int), - (int, float), - ) - if (type(value), field_type) not in whitelist: - raise ValueError(msg.format(field_name, field_type, type(value))) - - try: - value = field_type(value) - except ValueError as e: - e.message = msg.format(field_name, field_type, type(value)) - raise - return value - def get_cloudbuild(raw_config, args): """Read and validate a cloudbuild recipe @@ -210,7 +163,7 @@ def get_cloudbuild(raw_config, args): 'Expected {} contents to be of type "dict", but found type "{}"'. format(args.config, type(raw_config))) - raw_steps = get_field_value(raw_config, 'steps', list) + raw_steps = validation_utils.get_field_value(raw_config, 'steps', list) if not raw_steps: raise ValueError('No steps defined in {}'.format(args.config)) @@ -236,14 +189,14 @@ def get_step(raw_step): raise ValueError( 'Expected step to be of type "dict", but found type "{}"'. format(type(raw_step))) - raw_args = get_field_value(raw_step, 'args', list) - args = [get_field_value(raw_args, index, str) + raw_args = validation_utils.get_field_value(raw_step, 'args', list) + args = [validation_utils.get_field_value(raw_args, index, str) for index in range(len(raw_args))] - dir_ = get_field_value(raw_step, 'dir', str) - raw_env = get_field_value(raw_step, 'env', list) - env = [get_field_value(raw_env, index, str) + dir_ = validation_utils.get_field_value(raw_step, 'dir', str) + raw_env = validation_utils.get_field_value(raw_step, 'env', list) + env = [validation_utils.get_field_value(raw_env, index, str) for index in range(len(raw_env))] - name = get_field_value(raw_step, 'name', str) + name = validation_utils.get_field_value(raw_step, 'name', str) return Step( args=args, dir_=dir_, @@ -373,31 +326,6 @@ def local_cloudbuild(args): subprocess.check_call(args) -def validate_arg_regex(flag_value, flag_regex): - """Check a named command line flag against a regular expression""" - if not re.match(flag_regex, flag_value): - raise argparse.ArgumentTypeError( - 'Value "{}" does not match pattern "{}"'.format( - flag_value, flag_regex.pattern)) - return flag_value - - -def validate_arg_dict(flag_value): - """Parse a command line flag as a key=val,... dict""" - if not flag_value: - return {} - entries = flag_value.split(',') - pairs = [] - for entry in entries: - match = re.match(KEY_VALUE_REGEX, entry) - if not match: - raise argparse.ArgumentTypeError( - 'Value "{}" should be a list like _KEY1=value1,_KEY2=value2"'.format( - flag_value)) - pairs.append((match.group(1), match.group(2))) - return dict(pairs) - - def parse_args(argv): """Parse and validate command line flags""" parser = argparse.ArgumentParser( @@ -405,14 +333,14 @@ def parse_args(argv): parser.add_argument( '--config', type=functools.partial( - validate_arg_regex, flag_regex=PRINTABLE_REGEX), + validation_utils.validate_arg_regex, flag_regex=PRINTABLE_REGEX), default='cloudbuild.yaml', help='Path to cloudbuild.yaml file' ) parser.add_argument( '--output_script', type=functools.partial( - validate_arg_regex, flag_regex=PRINTABLE_REGEX), + validation_utils.validate_arg_regex, flag_regex=PRINTABLE_REGEX), help='Filename to write shell script to', ) parser.add_argument( @@ -423,7 +351,7 @@ def parse_args(argv): ) parser.add_argument( '--substitutions', - type=validate_arg_dict, + type=validation_utils.validate_arg_dict, default={}, help='Parameters to be substituted in the build specification', ) diff --git a/scripts/local_cloudbuild_test.py b/scripts/local_cloudbuild_test.py index 0409c98f..68a7ff27 100755 --- a/scripts/local_cloudbuild_test.py +++ b/scripts/local_cloudbuild_test.py @@ -34,91 +34,10 @@ STAGING_DIR_REGEX = re.compile( b'(?m)Copying source to staging directory (.+)$') -class ValidationUtilsTest(unittest.TestCase): - - def test_get_field_value(self): - valid_cases = ( - # Normal case, field present and correct type - ({ 'present': 1 }, 'present', int, 1), - ({ 'present': '1' }, 'present', str, '1'), - ({ 'present': [1] }, 'present', list, [1]), - ({ 'present': {1: 2} }, 'present', dict, {1: 2}), - # Missing field replaced by default - ({}, 'missing', str, ''), - # Valid conversions - ({ 'str_to_int': '1' }, 'str_to_int', int, 1), - ({ 'int_to_str': 1 }, 'int_to_str', str, '1'), - ) - for valid_case in valid_cases: - with self.subTest(valid_case=valid_case): - container, field_name, field_type, expected = valid_case - self.assertEqual( - local_cloudbuild.get_field_value( - container, field_name, field_type), - expected) - - invalid_cases = ( - # Type conversion failures - ({ 'bad_list_to_dict': [1] }, 'bad_list_to_dict', dict), - ({ 'bad_list_to_str': [1] }, 'bad_list_to_str', str), - ({ 'bad_dict_to_list': {1: 2} }, 'bad_dict_to_list', list), - ({ 'bad_str_to_int': 'not_an_int' }, 'bad_str_to_int', int), - ({ 'bad_str_to_list': 'abc' }, 'bad_str_to_list', list), - ) - for invalid_case in invalid_cases: - with self.subTest(invalid_case=invalid_case): - container, field_name, field_type = invalid_case - with self.assertRaises(ValueError): - local_cloudbuild.get_field_value( - container, field_name, field_type) - - def test_validate_arg_regex(self): - self.assertEqual( - local_cloudbuild.validate_arg_regex('abc', re.compile('a[b]c')), - 'abc') - with self.assertRaises(argparse.ArgumentTypeError): - local_cloudbuild.validate_arg_regex('abc', re.compile('a[d]c')) - - - def test_validate_arg_dict(self): - valid_cases = ( - # Normal case, field present and correct type - ('', {}), - ('_A=1', {'_A':'1'}), - ('_A=1,_B=2', {'_A':'1', '_B':'2'}), - # Repeated key is ok - ('_A=1,_A=2', {'_A':'2'}), - # Extra = is ok - ('_A=x=y=z,_B=2', {'_A':'x=y=z', '_B':'2'}), - # No value is ok - ('_A=', {'_A':''}), - ) - for valid_case in valid_cases: - with self.subTest(valid_case=valid_case): - s, expected = valid_case - self.assertEqual( - local_cloudbuild.validate_arg_dict(s), - expected) - - invalid_cases = ( - # No key - ',_A', - '_A,', - # Invalid variable name - '_Aa=1', - '_aA=1', - '0A=1', - ) - for invalid_case in invalid_cases: - with self.subTest(invalid_case=invalid_case): - with self.assertRaises(argparse.ArgumentTypeError): - local_cloudbuild.validate_arg_dict(invalid_case) - - class LocalCloudbuildTest(unittest.TestCase): def setUp(self): - self.testdata_dir = 'testdata' + self.testdata_dir = os.path.join(os.path.dirname(__file__), 'testdata') # Sigh assert os.path.isdir(self.testdata_dir), 'Could not run test: testdata directory not found' def test_sub_and_quote(self): diff --git a/scripts/validation_utils.py b/scripts/validation_utils.py new file mode 100644 index 00000000..7a04847e --- /dev/null +++ b/scripts/validation_utils.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 + +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Utilities for schema and command line validation""" + +import argparse +import re + + +# For easier development, we allow redefining builtins like +# --substitutions=PROJECT_ID=foo even though gcloud doesn't. +KEY_VALUE_REGEX = re.compile(r'^([A-Z_][A-Z0-9_]*)=(.*)$') + + +def get_field_value(container, field_name, field_type): + """Fetch a field from a container with typechecking and default values. + + The field value is coerced to the desired type. If the field is + not present, a instance of `field_type` is constructed with no + arguments and used as the default value. + + Args: + container (dict): Object decoded from yaml + field_name (str): Field that should be present in `container` + field_type (type): Expected type for field value + + Returns: + Any: Fetched or default value of field + + Raises: + ValueError: if field value cannot be converted to the desired type + """ + try: + value = container[field_name] + if value is None: + return field_type() + except (IndexError, KeyError): + return field_type() + + msg = 'Expected "{}" field to be of type "{}", but found type "{}"' + if not isinstance(value, field_type): + # list('some string') is a successful type cast as far as Python + # is concerned, but doesn't exactly produce the results we want. + # We have a whitelist of conversions we will attempt. + whitelist = ( + (float, str), + (int, str), + (str, float), + (str, int), + (int, float), + ) + if (type(value), field_type) not in whitelist: + raise ValueError(msg.format(field_name, field_type, type(value))) + + try: + value = field_type(value) + except ValueError as e: + e.message = msg.format(field_name, field_type, type(value)) + raise + return value + + +def validate_arg_regex(flag_value, flag_regex): + """Check a named command line flag against a regular expression""" + if not re.match(flag_regex, flag_value): + raise argparse.ArgumentTypeError( + 'Value "{}" does not match pattern "{}"'.format( + flag_value, flag_regex.pattern)) + return flag_value + + +def validate_arg_dict(flag_value): + """Parse a command line flag as a key=val,... dict""" + if not flag_value: + return {} + entries = flag_value.split(',') + pairs = [] + for entry in entries: + match = re.match(KEY_VALUE_REGEX, entry) + if not match: + raise argparse.ArgumentTypeError( + 'Value "{}" should be a list like _KEY1=value1,_KEY2=value2"'.format( + flag_value)) + pairs.append((match.group(1), match.group(2))) + return dict(pairs) diff --git a/scripts/validation_utils_test.py b/scripts/validation_utils_test.py new file mode 100755 index 00000000..9ef87096 --- /dev/null +++ b/scripts/validation_utils_test.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 + +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit test for validation_utils.py""" + +import argparse +import re +import unittest + +import validation_utils + + +class ValidationUtilsTest(unittest.TestCase): + + def test_get_field_value(self): + valid_cases = ( + # Normal case, field present and correct type + ({ 'present': 1 }, 'present', int, 1), + ({ 'present': '1' }, 'present', str, '1'), + ({ 'present': [1] }, 'present', list, [1]), + ({ 'present': {1: 2} }, 'present', dict, {1: 2}), + # Missing field replaced by default + ({}, 'missing', str, ''), + # Valid conversions + ({ 'str_to_int': '1' }, 'str_to_int', int, 1), + ({ 'int_to_str': 1 }, 'int_to_str', str, '1'), + # None + ({ 'None_to_int': None }, 'None_to_int', int, 0), + ({ 'None_to_str': None }, 'None_to_str', str, ''), + ) + for valid_case in valid_cases: + with self.subTest(valid_case=valid_case): + container, field_name, field_type, expected = valid_case + self.assertEqual( + validation_utils.get_field_value( + container, field_name, field_type), + expected) + + invalid_cases = ( + # Type conversion failures + ({ 'bad_list_to_dict': [1] }, 'bad_list_to_dict', dict), + ({ 'bad_list_to_str': [1] }, 'bad_list_to_str', str), + ({ 'bad_dict_to_list': {1: 2} }, 'bad_dict_to_list', list), + ({ 'bad_str_to_int': 'not_an_int' }, 'bad_str_to_int', int), + ({ 'bad_str_to_list': 'abc' }, 'bad_str_to_list', list), + ) + for invalid_case in invalid_cases: + with self.subTest(invalid_case=invalid_case): + container, field_name, field_type = invalid_case + with self.assertRaises(ValueError): + validation_utils.get_field_value( + container, field_name, field_type) + + def test_validate_arg_regex(self): + self.assertEqual( + validation_utils.validate_arg_regex('abc', re.compile('a[b]c')), + 'abc') + with self.assertRaises(argparse.ArgumentTypeError): + validation_utils.validate_arg_regex('abc', re.compile('a[d]c')) + + def test_validate_arg_dict(self): + valid_cases = ( + # Normal case, field present and correct type + ('', {}), + ('_A=1', {'_A':'1'}), + ('_A=1,_B=2', {'_A':'1', '_B':'2'}), + # Repeated key is ok + ('_A=1,_A=2', {'_A':'2'}), + # Extra = is ok + ('_A=x=y=z,_B=2', {'_A':'x=y=z', '_B':'2'}), + # No value is ok + ('_A=', {'_A':''}), + ) + for valid_case in valid_cases: + with self.subTest(valid_case=valid_case): + s, expected = valid_case + self.assertEqual( + validation_utils.validate_arg_dict(s), + expected) + + invalid_cases = ( + # No key + ',_A', + '_A,', + # Invalid variable name + '_Aa=1', + '_aA=1', + '0A=1', + ) + for invalid_case in invalid_cases: + with self.subTest(invalid_case=invalid_case): + with self.assertRaises(argparse.ArgumentTypeError): + validation_utils.validate_arg_dict(invalid_case) + + +if __name__ == '__main__': + unittest.main() From 58e8abb82578964a8c3997ac0e6dd590f9ab52d3 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Thu, 15 Jun 2017 18:22:06 -0700 Subject: [PATCH 174/362] Add script to generate Dockerfiles like Google Cloud SDK does. --- scripts/gen_dockerfile.py | 252 ++++++++++++++++++ scripts/gen_dockerfile_test.py | 227 ++++++++++++++++ scripts/testdata/hello_world/app.yaml | 6 + scripts/testdata/hello_world/main.py | 43 +++ scripts/testdata/hello_world/main_test.py | 24 ++ scripts/testdata/hello_world/requirements.txt | 2 + .../testdata/hello_world_golden/.dockerignore | 19 ++ .../testdata/hello_world_golden/Dockerfile | 12 + 8 files changed, 585 insertions(+) create mode 100755 scripts/gen_dockerfile.py create mode 100755 scripts/gen_dockerfile_test.py create mode 100644 scripts/testdata/hello_world/app.yaml create mode 100644 scripts/testdata/hello_world/main.py create mode 100644 scripts/testdata/hello_world/main_test.py create mode 100644 scripts/testdata/hello_world/requirements.txt create mode 100644 scripts/testdata/hello_world_golden/.dockerignore create mode 100644 scripts/testdata/hello_world_golden/Dockerfile diff --git a/scripts/gen_dockerfile.py b/scripts/gen_dockerfile.py new file mode 100755 index 00000000..07d2f8c5 --- /dev/null +++ b/scripts/gen_dockerfile.py @@ -0,0 +1,252 @@ +#!/usr/bin/env python2 + +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Generate a Dockerfile and helper files for a Python application.""" + +import argparse +import collections +import functools +import io +import os +import re +import sys + +import yaml + +import validation_utils + + +# Validate characters for dockerfile image names +# https://github.com/docker/docker/blob/master/api/names.go +IMAGE_REGEX = re.compile(r"""(?x) + ^ + [a-zA-Z0-9] # First char must be alphanumeric + [-a-zA-Z0-9_./]* # Punctuation allowed after that + ( + : # At most one colon allowed + [-a-zA-Z0-9_./]+ # Colon must be followed by other chars + )? + $ +""") + +# `entrypoint` is specified as free-form text parsed as a unix shell +# command line, which limits the sanity checking possible. We +# disallow newlines and control characters which would break the +# Dockerfile format. +PRINTABLE_REGEX = re.compile(r"""^[^\x00-\x1f]*$""") + +# Map from app.yaml "python_version" to {python_version} in Dockerfile +PYTHON_INTERPRETER_VERSION_MAP = { + '': '', # == 2.7 + '2': '', # == 2.7 + '3': '3.5', + '3.4': '3.4', + '3.5': '3.5', + '3.6': '3.6', +} + +# File templates. +# Designed to exactly match the current output of 'gcloud app gen-config' +DOCKERFILE_TEMPLATE = """\ +FROM {base_image} +LABEL python_version=python{dockerfile_python_version} +RUN virtualenv --no-download /env -p python{dockerfile_python_version} + +# Set virtualenv environment variables. This is equivalent to running +# source /env/bin/activate +ENV VIRTUAL_ENV /env +ENV PATH /env/bin:$PATH +{optional_requirements_txt}ADD . /app/ +{optional_entrypoint}""" + +DOCKERFILE_REQUIREMENTS_TXT = """\ +ADD requirements.txt /app/ +RUN pip install -r requirements.txt +""" + +DOCKERFILE_ENTRYPOINT_TEMPLATE = """\ +CMD {entrypoint} +""" + +DOCKERIGNORE = """\ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +.dockerignore +Dockerfile +.git +.hg +.svn +""" + +# Validated application configuration +AppConfig = collections.namedtuple( + 'AppConfig', + 'base_image dockerfile_python_version entrypoint has_requirements_txt' +) + + +def get_app_config(raw_config, args): + """Read and validate the application runtime configuration. + + Args: + raw_config (dict): deserialized app.yaml + args (argparse.Namespace): ccommand line flags + + Returns: + AppConfig: valid configuration + """ + # Examine app.yaml + if not isinstance(raw_config, dict): + raise ValueError( + 'Expected {} contents to be of type "dict", but found type "{}"'. + format(args.config, type(raw_config))) + + entrypoint = validation_utils.get_field_value(raw_config, 'entrypoint', str) + if not PRINTABLE_REGEX.match(entrypoint): + raise ValueError('Invalid character in "entrypoint" field of app.yaml') + raw_runtime_config = validation_utils.get_field_value(raw_config, 'runtime_config', dict) + python_version = validation_utils.get_field_value(raw_runtime_config, 'python_version', str) + dockerfile_python_version = PYTHON_INTERPRETER_VERSION_MAP.get( + python_version) + if dockerfile_python_version is None: + valid_versions = str(sorted(PYTHON_INTERPRETER_VERSION_MAP.keys())) + msg = ('Invalid "python_version" field in "runtime_config" section ' + 'of app.yaml. Valid options are:\n{}').format(valid_versions) + raise ValueError(msg) + + # Examine user's files + has_requirements_txt = os.path.isfile( + os.path.join(args.source_dir, 'requirements.txt')) + + # Compute base image name and Dockerfile contents + base_image = args.base_image + + return AppConfig( + base_image=base_image, + dockerfile_python_version=dockerfile_python_version, + entrypoint=entrypoint, + has_requirements_txt=has_requirements_txt) + + +def generate_files(app_config): + """Generate a Dockerfile and helper files for an application. + + Args: + app_config (AppConfig): Validated configuration + + Returns: + dict: Map of filename to desired file contents + """ + if app_config.has_requirements_txt: + optional_requirements_txt = DOCKERFILE_REQUIREMENTS_TXT + else: + optional_requirements_txt = '' + + if app_config.entrypoint: + # Mangle entrypoint in the same way as the Cloud SDK + # (googlecloudsdk/third_party/appengine/api/validation.py) + # + # We could handle both string ("shell form") and list ("exec + # form") but it appears that gcloud only handles string form. + entrypoint = app_config.entrypoint + if entrypoint and not entrypoint.startswith('exec '): + entrypoint = 'exec ' + entrypoint + optional_entrypoint = DOCKERFILE_ENTRYPOINT_TEMPLATE.format( + entrypoint=entrypoint) + else: + optional_entrypoint = '' + + dockerfile = DOCKERFILE_TEMPLATE.format( + base_image=app_config.base_image, + dockerfile_python_version=app_config.dockerfile_python_version, + optional_requirements_txt=optional_requirements_txt, + optional_entrypoint=optional_entrypoint) + + return { + 'Dockerfile': dockerfile, + '.dockerignore': DOCKERIGNORE, + } + + +def gen_dockerfile(args): + """Write a Dockerfile and helper files for an application. + + Args: + args (argparse.Namespace): ccommand line flags + """ + # Read yaml file. Does not currently support multiple services + # with configuration filenames besides app.yaml + with io.open(args.config, 'r', encoding='utf8') as yaml_config_file: + raw_config = yaml.load(yaml_config_file) + + # Determine configuration + app_config = get_app_config(raw_config, args) + + # Generate list of filenames and their textual contents + files = generate_files(app_config) + + # Write files + for filename, contents in files.items(): + full_filename = os.path.join(args.source_dir, filename) + with io.open(full_filename, 'w', encoding='utf8') as outfile: + outfile.write(contents) + + +def parse_args(argv): + """Parse and validate command line flags""" + parser = argparse.ArgumentParser() + parser.add_argument( + '--config', + type=functools.partial( + validation_utils.validate_arg_regex, flag_regex=PRINTABLE_REGEX), + default='app.yaml', + help='Path to application configuration file' + ) + parser.add_argument( + '--base-image', + type=functools.partial( + validation_utils.validate_arg_regex, flag_regex=IMAGE_REGEX), + default='gcr.io/google-appengine/python:latest', + help='Name of Docker image to use as base') + parser.add_argument( + '--source-dir', + type=functools.partial( + validation_utils.validate_arg_regex, flag_regex=PRINTABLE_REGEX), + default='.', + help=('Application source and output directory')) + args = parser.parse_args(argv[1:]) + return args + + +def main(): + args = parse_args(sys.argv) + gen_dockerfile(args) + + +if __name__ == '__main__': + main() diff --git a/scripts/gen_dockerfile_test.py b/scripts/gen_dockerfile_test.py new file mode 100755 index 00000000..faf44fc7 --- /dev/null +++ b/scripts/gen_dockerfile_test.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python2 + +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit test for gen_dockerfile.py""" + +import argparse +import os +import re +import shutil +import subprocess +import tempfile +import unittest +import unittest.mock + +import yaml + +import gen_dockerfile + +# Expected list of files generated +EXPECTED_OUTPUT_FILES = ['Dockerfile', '.dockerignore'] + + +class GenDockerfileTest(unittest.TestCase): + def setUp(self): + self.testdata_dir = os.path.join(os.path.dirname(__file__), 'testdata') # Sigh + assert os.path.isdir(self.testdata_dir), 'Could not run test: testdata directory not found' + + def compare_file(self, filename, dir1, dir2): + """Compare identically named files in two different directories""" + with open(os.path.join(dir1, filename), 'r', encoding='utf8') as file1: + with open(os.path.join(dir2, filename), 'r', encoding='utf8') as file2: + contents1 = file1.read() + contents2 = file2.read() + msg = 'Contents of "{}" differ between "{}" and "{}"'.format( + filename, dir1, dir2) + self.assertMultiLineEqual(contents1, contents2, msg) + + def test_get_app_config(self): + args = argparse.Namespace( + config='some_config_file', + base_image='some_image_name', + source_dir='some_source_dir') + + valid_cases = ( + # Basic app.yaml + ('env: flex', False, { + 'base_image': 'some_image_name', + 'dockerfile_python_version': '', + 'has_requirements_txt': False, + 'entrypoint': '', + }), + # All supported python versions + ('runtime_config:\n python_version:', False, { + 'dockerfile_python_version': '', + }), + ('runtime_config:\n python_version: 2', False, { + 'dockerfile_python_version': '', + }), + ('runtime_config:\n python_version: 3', False, { + 'dockerfile_python_version': '3.5', + }), + ('runtime_config:\n python_version: 3.4', False, { + 'dockerfile_python_version': '3.4', + }), + ('runtime_config:\n python_version: 3.5', False, { + 'dockerfile_python_version': '3.5', + }), + # requirements.txt present + ('env: flex', True, { + 'has_requirements_txt': True, + }), + # entrypoint present + ('entrypoint: my entrypoint', False, { + 'entrypoint': 'my entrypoint', + }), + ) + for valid_case in valid_cases: + with self.subTest(valid_case=valid_case): + app_yaml, isfile, expected = valid_case + raw_app_config = yaml.safe_load(app_yaml) + with unittest.mock.patch.object( + os.path, 'isfile', return_value=isfile): + actual = gen_dockerfile.get_app_config(raw_app_config, args) + for key, value in expected.items(): + self.assertEqual(getattr(actual, key), value) + + invalid_cases = ( + # Empty app.yaml + '', + # Invalid entrypoint + 'entrypoint: "bad \\n entrypoint"', + # Invalid python version + 'runtime_config:\n python_version: 1', + 'runtime_config:\n python_version: python2', + ) + for invalid_case in invalid_cases: + with self.subTest(invalid_case=invalid_case): + raw_app_config = yaml.safe_load(invalid_case) + with self.assertRaises(ValueError): + gen_dockerfile.get_app_config(raw_app_config, args) + + def test_generate_files(self): + base = gen_dockerfile.AppConfig( + base_image='', + dockerfile_python_version='', + entrypoint='', + has_requirements_txt=False + ) + cases = ( + # Requirements.txt + (base, False, 'ADD requirements.txt'), + (base._replace(has_requirements_txt=True), True, + 'ADD requirements.txt'), + # Entrypoint + (base, False, 'CMD'), + (base._replace(entrypoint='my entrypoint'), True, + 'CMD exec my entrypoint'), + (base._replace(entrypoint='exec my entrypoint'), True, + 'CMD exec my entrypoint'), + # Base runtime image + (base._replace(base_image='my_base_runtime_image'), True, + 'FROM my_base_runtime_image'), + # Python version + (base._replace(dockerfile_python_version='_my_version'), True, + 'python_version=python_my_version'), + ) + for case in cases: + with self.subTest(case=case): + app_config, should_find, test_string = case + result = gen_dockerfile.generate_files(app_config) + self.assertEqual( + sorted(result.keys()), sorted(EXPECTED_OUTPUT_FILES)) + dockerfile = result['Dockerfile'] + if should_find: + self.assertIn(test_string, dockerfile) + else: + self.assertNotIn(test_string, dockerfile) + + def test_gen_dockerfile(self): + """Generates output and compares against a set of golden files. + + Optionally runs 'gcloud app gen-config' and compares against that. + """ + # Sample app from https://github.com/GoogleCloudPlatform/python-docs-samples + with tempfile.TemporaryDirectory( + prefix='gen_dockerfile_test_') as parent_tempdir: + for app in ['hello_world']: + app_dir = os.path.join(self.testdata_dir, app) + temp_dir = os.path.join(parent_tempdir, app) + os.mkdir(temp_dir) + + # Copy sample app to writable temp dir, and generate Dockerfile. + config_dir = os.path.join(temp_dir, 'config') + shutil.copytree(app_dir, config_dir) + args = argparse.Namespace( + config=os.path.join(config_dir, 'app.yaml'), + base_image='gcr.io/google-appengine/python', + source_dir=config_dir, + ) + gen_dockerfile.gen_dockerfile(args) + + # Compare against golden files + golden_dir = os.path.join(self.testdata_dir, app + '_golden') + for filename in EXPECTED_OUTPUT_FILES: + with self.subTest(source='golden', filename=filename): + self.compare_file(filename, config_dir, golden_dir) + + # Copy sample app to different writable temp dir, and + # generate Dockerfile using gcloud. + if not shutil.which('gcloud'): + self.skipTest( + '"gcloud" tool not found in $PATH, skipping test') + gen_config_dir = os.path.join(temp_dir, 'gen_config') + shutil.copytree(app_dir, gen_config_dir) + app_yaml = os.path.join(gen_config_dir, 'app.yaml') + gcloud_args = [ + 'gcloud', '--quiet', 'beta', 'app', 'gen-config', + gen_config_dir, '--custom', '--config={}'.format(app_yaml) + ] + print('Invoking gcloud as {}'.format(gcloud_args)) + subprocess.check_call(gcloud_args) + for filename in EXPECTED_OUTPUT_FILES: + with self.subTest(source='gcloud', filename=filename): + self.compare_file(filename, config_dir, gen_config_dir) + + def test_parse_args(self): + valid_cases = ( + [], + ['argv0', '--base-image=nocolon'], + ['argv0', '--base-image=name:andcolon'], + ) + for argv in valid_cases: + with self.subTest(valid_argv=argv): + args = gen_dockerfile.parse_args(argv) + + def mock_error(*args): + """Prevent argparse from calling sys.exit()""" + raise AssertionError(*args) + + invalid_cases = ( + ['argv0', '--base-image=:noname'], + ['argv0', '--base-image=notag:'], + ['argv0', '--base-image=bad_:_:_multiple_colons'], + ) + for argv in invalid_cases: + with self.subTest(invalid_argv=argv): + with unittest.mock.patch.object( + argparse.ArgumentParser, 'error', mock_error): + with self.assertRaises(AssertionError): + gen_dockerfile.parse_args(argv) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/testdata/hello_world/app.yaml b/scripts/testdata/hello_world/app.yaml new file mode 100644 index 00000000..e5ac514e --- /dev/null +++ b/scripts/testdata/hello_world/app.yaml @@ -0,0 +1,6 @@ +runtime: python +env: flex +entrypoint: gunicorn -b :$PORT main:app + +runtime_config: + python_version: 3 diff --git a/scripts/testdata/hello_world/main.py b/scripts/testdata/hello_world/main.py new file mode 100644 index 00000000..97eb37d8 --- /dev/null +++ b/scripts/testdata/hello_world/main.py @@ -0,0 +1,43 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START app] +import logging + +from flask import Flask + + +app = Flask(__name__) + + +@app.route('/') +def hello(): + """Return a friendly HTTP greeting.""" + return 'Hello World!' + + +@app.errorhandler(500) +def server_error(e): + logging.exception('An error occurred during a request.') + return """ + An internal error occurred: 
{}
+ See logs for full stacktrace. + """.format(e), 500 + + +if __name__ == '__main__': + # This is used when running locally. Gunicorn is used to run the + # application on Google App Engine. See entrypoint in app.yaml. + app.run(host='127.0.0.1', port=8080, debug=True) +# [END app] diff --git a/scripts/testdata/hello_world/main_test.py b/scripts/testdata/hello_world/main_test.py new file mode 100644 index 00000000..4e230185 --- /dev/null +++ b/scripts/testdata/hello_world/main_test.py @@ -0,0 +1,24 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import main + + +def test_index(): + main.app.testing = True + client = main.app.test_client() + + r = client.get('/') + assert r.status_code == 200 + assert 'Hello World' in r.data.decode('utf-8') diff --git a/scripts/testdata/hello_world/requirements.txt b/scripts/testdata/hello_world/requirements.txt new file mode 100644 index 00000000..7861fb49 --- /dev/null +++ b/scripts/testdata/hello_world/requirements.txt @@ -0,0 +1,2 @@ +Flask==0.12 +gunicorn==19.6.0 diff --git a/scripts/testdata/hello_world_golden/.dockerignore b/scripts/testdata/hello_world_golden/.dockerignore new file mode 100644 index 00000000..8b927bb7 --- /dev/null +++ b/scripts/testdata/hello_world_golden/.dockerignore @@ -0,0 +1,19 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +.dockerignore +Dockerfile +.git +.hg +.svn diff --git a/scripts/testdata/hello_world_golden/Dockerfile b/scripts/testdata/hello_world_golden/Dockerfile new file mode 100644 index 00000000..e3e0563c --- /dev/null +++ b/scripts/testdata/hello_world_golden/Dockerfile @@ -0,0 +1,12 @@ +FROM gcr.io/google-appengine/python +LABEL python_version=python3.5 +RUN virtualenv --no-download /env -p python3.5 + +# Set virtualenv environment variables. This is equivalent to running +# source /env/bin/activate +ENV VIRTUAL_ENV /env +ENV PATH /env/bin:$PATH +ADD requirements.txt /app/ +RUN pip install -r requirements.txt +ADD . /app/ +CMD exec gunicorn -b :$PORT main:app From e4855c2c50f02704f67ee81b1b7e3db03af8acce Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Thu, 15 Jun 2017 18:23:31 -0700 Subject: [PATCH 175/362] Create a "runtime builder" image that builds other Docker images This builder will be called by the Google Cloud SDK during "gcloud app deploy" to create Docker images for AppEngine Flex applications. --- build.sh | 9 ++++++++ builder/gen-dockerfile/.gitignore | 2 ++ builder/gen-dockerfile/Dockerfile.in | 13 +++++++++++ builder/gen-dockerfile/requirements.txt | 1 + builder/python.yaml | 12 ++++++++++ cloudbuild.yaml | 30 +++++++++++++++---------- 6 files changed, 55 insertions(+), 12 deletions(-) create mode 100644 builder/gen-dockerfile/.gitignore create mode 100644 builder/gen-dockerfile/Dockerfile.in create mode 100644 builder/gen-dockerfile/requirements.txt create mode 100644 builder/python.yaml diff --git a/build.sh b/build.sh index 5929fbb2..0e01abcd 100755 --- a/build.sh +++ b/build.sh @@ -113,6 +113,7 @@ echo "Using base image name ${FULL_BASE_IMAGE}" # Generate Dockerfiles for outfile in \ + builder/gen-dockerfile/Dockerfile \ python-interpreter-builder/Dockerfile \ runtime-image/Dockerfile \ tests/benchmark/Dockerfile \ @@ -123,6 +124,14 @@ for outfile in \ envsubst <"${outfile}".in >"${outfile}" '$DEBIAN_BASE_IMAGE $FULL_BASE_IMAGE $GOOGLE_CLOUD_PROJECT_FOR_TESTS' done +# Make some files available to the runtime builder Docker context +for file in \ + gen_dockerfile.py \ + validation_utils.py \ + ; do + cp -a "scripts/${file}" "builder/gen-dockerfile/${file}" +done + # Build images and push to GCR if [ "${build}" -eq 1 ]; then echo "Building images" diff --git a/builder/gen-dockerfile/.gitignore b/builder/gen-dockerfile/.gitignore new file mode 100644 index 00000000..d8ffddc3 --- /dev/null +++ b/builder/gen-dockerfile/.gitignore @@ -0,0 +1,2 @@ +Dockerfile +*.py diff --git a/builder/gen-dockerfile/Dockerfile.in b/builder/gen-dockerfile/Dockerfile.in new file mode 100644 index 00000000..4a8de5ec --- /dev/null +++ b/builder/gen-dockerfile/Dockerfile.in @@ -0,0 +1,13 @@ +FROM ${FULL_BASE_IMAGE} +LABEL python_version=python3.5 +RUN virtualenv --no-download /env -p python3.5 + +# Set virtualenv environment variables. This is equivalent to running +# source /env/bin/activate +ENV VIRTUAL_ENV /env +ENV PATH /env/bin:$PATH +ADD requirements.txt /builder/ +RUN pip install -r /builder/requirements.txt +ADD . /builder/ +WORKDIR /workspace +ENTRYPOINT [ "python", "/builder/gen_dockerfile.py" ] diff --git a/builder/gen-dockerfile/requirements.txt b/builder/gen-dockerfile/requirements.txt new file mode 100644 index 00000000..c3726e8b --- /dev/null +++ b/builder/gen-dockerfile/requirements.txt @@ -0,0 +1 @@ +pyyaml diff --git a/builder/python.yaml b/builder/python.yaml new file mode 100644 index 00000000..3e75acad --- /dev/null +++ b/builder/python.yaml @@ -0,0 +1,12 @@ +# This is a cloudbuild.yaml template for the runtime builder +steps: +- # Generate application Dockerfile + name: 'gcr.io/gcp-runtimes/python/gen-dockerfile:latest' + args: [ + '--base-image=gcr.io/google-appengine/python:latest' + ] +- # Use that Dockerfile to create final application image + name: 'gcr.io/cloud-builders/docker:latest' + args: ['build', '-t', '$_OUTPUT_IMAGE', '.'] +images: + - '$_OUTPUT_IMAGE' diff --git a/cloudbuild.yaml b/cloudbuild.yaml index f242ac8e..3f52e41d 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -1,18 +1,18 @@ timeout: 7200s steps: -- name: gcr.io/cloud-builders/docker:latest - # Compile Python interpreters from source +- # Compile Python interpreters from source + name: gcr.io/cloud-builders/docker:latest args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/interpreter-builder:${_TAG}', '--no-cache', '/workspace/python-interpreter-builder/'] -- name: ${_DOCKER_NAMESPACE}/python/interpreter-builder:${_TAG} - # Copy interpreters back to workspace +- # Copy interpreters back to workspace + name: ${_DOCKER_NAMESPACE}/python/interpreter-builder:${_TAG} args: ['cp', '/interpreters.tar.gz', '/workspace/runtime-image/interpreters.tar.gz'] -- name: gcr.io/cloud-builders/docker:latest - # Build base runtime image +- # Build base runtime image + name: gcr.io/cloud-builders/docker:latest args: ['build', '--tag=${_DOCKER_NAMESPACE}/python:${_TAG}', '--no-cache', '/workspace/runtime-image/'] -- name: gcr.io/gcp-runtimes/structure_test:latest - # Validate structure of base runtime image +- # Validate structure of base runtime image + name: gcr.io/gcp-runtimes/structure_test:latest args: [ '-i', '${_DOCKER_NAMESPACE}/python:${_TAG}', '--config', '/workspace/tests/virtualenv/virtualenv_default.yaml', @@ -25,11 +25,17 @@ steps: '--config', '/workspace/tests/license-test/license-test.yaml', '-v' ] -- name: gcr.io/cloud-builders/docker:latest - # Run google client library unit tests against base runtime image +- # Build image to run google client library unit tests + name: gcr.io/cloud-builders/docker:latest args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/google-cloud-python:${_TAG}', '--no-cache', '/workspace/tests/google-cloud-python/'] -- name: ${_DOCKER_NAMESPACE}/python/tests/google-cloud-python:${_TAG} +- # Run google client library unit tests + name: ${_DOCKER_NAMESPACE}/python/tests/google-cloud-python:${_TAG} +- # Build runtime builder image + name: gcr.io/cloud-builders/docker:latest + args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/builder/gen-dockerfile:${_TAG}', + '--no-cache', '/workspace/builder/gen-dockerfile/'] images: [ - '${_DOCKER_NAMESPACE}/python:${_TAG}' + '${_DOCKER_NAMESPACE}/python:${_TAG}', + '${_DOCKER_NAMESPACE}/python/builder/gen-dockerfile:${_TAG}', ] From b67eff102c2e3c4dddcf5bb2e7d757691d5628d6 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 16 Jun 2017 14:18:36 -0700 Subject: [PATCH 176/362] Rename cloudbuild.yaml template file. --- builder/{python.yaml => python-latest.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename builder/{python.yaml => python-latest.yaml} (100%) diff --git a/builder/python.yaml b/builder/python-latest.yaml similarity index 100% rename from builder/python.yaml rename to builder/python-latest.yaml From 1091d19ff841e095f366a85d5ba321d04aa62999 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 16 Jun 2017 14:24:37 -0700 Subject: [PATCH 177/362] Fix shebang lines --- scripts/gen_dockerfile.py | 2 +- scripts/gen_dockerfile_test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/gen_dockerfile.py b/scripts/gen_dockerfile.py index 07d2f8c5..77c91e5b 100755 --- a/scripts/gen_dockerfile.py +++ b/scripts/gen_dockerfile.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # Copyright 2017 Google Inc. All Rights Reserved. # diff --git a/scripts/gen_dockerfile_test.py b/scripts/gen_dockerfile_test.py index faf44fc7..56655159 100755 --- a/scripts/gen_dockerfile_test.py +++ b/scripts/gen_dockerfile_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # Copyright 2017 Google Inc. All Rights Reserved. # From a42b40e67e5a85914e0a97d3de8770f2a79cd840 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 16 Jun 2017 14:46:14 -0700 Subject: [PATCH 178/362] Support Docker images names with sha256 digests --- scripts/gen_dockerfile.py | 14 ++++++-------- scripts/gen_dockerfile_test.py | 5 +++-- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/scripts/gen_dockerfile.py b/scripts/gen_dockerfile.py index 77c91e5b..2637a5f5 100755 --- a/scripts/gen_dockerfile.py +++ b/scripts/gen_dockerfile.py @@ -29,16 +29,14 @@ import validation_utils -# Validate characters for dockerfile image names -# https://github.com/docker/docker/blob/master/api/names.go +# Validate characters for dockerfile image names. +# +# This roots out obvious mistakes, the full gory details are here: +# https://github.com/docker/distribution/blob/master/reference/regexp.go IMAGE_REGEX = re.compile(r"""(?x) ^ - [a-zA-Z0-9] # First char must be alphanumeric - [-a-zA-Z0-9_./]* # Punctuation allowed after that - ( - : # At most one colon allowed - [-a-zA-Z0-9_./]+ # Colon must be followed by other chars - )? + [a-zA-Z0-9] # First char must be alphanumeric + [a-zA-Z0-9-_./:@+]* # Punctuation allowed after that $ """) diff --git a/scripts/gen_dockerfile_test.py b/scripts/gen_dockerfile_test.py index 56655159..4a99c51d 100755 --- a/scripts/gen_dockerfile_test.py +++ b/scripts/gen_dockerfile_test.py @@ -201,6 +201,7 @@ def test_parse_args(self): [], ['argv0', '--base-image=nocolon'], ['argv0', '--base-image=name:andcolon'], + ['argv0', '--base-image=name@sha256:digest'], ) for argv in valid_cases: with self.subTest(valid_argv=argv): @@ -211,9 +212,9 @@ def mock_error(*args): raise AssertionError(*args) invalid_cases = ( + ['argv0', '--base-image='], + ['argv0', '--base-image=:'], ['argv0', '--base-image=:noname'], - ['argv0', '--base-image=notag:'], - ['argv0', '--base-image=bad_:_:_multiple_colons'], ) for argv in invalid_cases: with self.subTest(invalid_argv=argv): From e5dab4cde5144edaa4e2a27e6ba9bb441c4fa2bf Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 16 Jun 2017 14:56:38 -0700 Subject: [PATCH 179/362] Fix some comments --- scripts/gen_dockerfile.py | 4 ++-- scripts/gen_dockerfile_test.py | 2 +- scripts/local_cloudbuild.py | 2 +- scripts/local_cloudbuild_test.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/gen_dockerfile.py b/scripts/gen_dockerfile.py index 2637a5f5..b02684fb 100755 --- a/scripts/gen_dockerfile.py +++ b/scripts/gen_dockerfile.py @@ -113,7 +113,7 @@ def get_app_config(raw_config, args): Args: raw_config (dict): deserialized app.yaml - args (argparse.Namespace): ccommand line flags + args (argparse.Namespace): command line flags Returns: AppConfig: valid configuration @@ -195,7 +195,7 @@ def gen_dockerfile(args): """Write a Dockerfile and helper files for an application. Args: - args (argparse.Namespace): ccommand line flags + args (argparse.Namespace): command line flags """ # Read yaml file. Does not currently support multiple services # with configuration filenames besides app.yaml diff --git a/scripts/gen_dockerfile_test.py b/scripts/gen_dockerfile_test.py index 4a99c51d..e8ad371d 100755 --- a/scripts/gen_dockerfile_test.py +++ b/scripts/gen_dockerfile_test.py @@ -35,7 +35,7 @@ class GenDockerfileTest(unittest.TestCase): def setUp(self): - self.testdata_dir = os.path.join(os.path.dirname(__file__), 'testdata') # Sigh + self.testdata_dir = os.path.join(os.path.dirname(__file__), 'testdata') assert os.path.isdir(self.testdata_dir), 'Could not run test: testdata directory not found' def compare_file(self, filename, dir1, dir2): diff --git a/scripts/local_cloudbuild.py b/scripts/local_cloudbuild.py index f0c6659d..a7c5eda9 100755 --- a/scripts/local_cloudbuild.py +++ b/scripts/local_cloudbuild.py @@ -153,7 +153,7 @@ def get_cloudbuild(raw_config, args): Args: raw_config (dict): deserialized cloudbuild.yaml - args (argparse.Namespace): ccommand line flags + args (argparse.Namespace): command line flags Returns: CloudBuild: valid configuration diff --git a/scripts/local_cloudbuild_test.py b/scripts/local_cloudbuild_test.py index 68a7ff27..fff089dc 100755 --- a/scripts/local_cloudbuild_test.py +++ b/scripts/local_cloudbuild_test.py @@ -37,7 +37,7 @@ class LocalCloudbuildTest(unittest.TestCase): def setUp(self): - self.testdata_dir = os.path.join(os.path.dirname(__file__), 'testdata') # Sigh + self.testdata_dir = os.path.join(os.path.dirname(__file__), 'testdata') assert os.path.isdir(self.testdata_dir), 'Could not run test: testdata directory not found' def test_sub_and_quote(self): From af2f40ff58a8b0c8994dae7bf13ffd4b881b2d98 Mon Sep 17 00:00:00 2001 From: Nick Kubala Date: Fri, 16 Jun 2017 16:09:27 -0700 Subject: [PATCH 180/362] add simple application for running deploy health check --- tests/health_check/app.yaml | 3 ++ tests/health_check/main.py | 43 +++++++++++++++++++++++++++++ tests/health_check/requirements.txt | 2 ++ 3 files changed, 48 insertions(+) create mode 100644 tests/health_check/app.yaml create mode 100644 tests/health_check/main.py create mode 100644 tests/health_check/requirements.txt diff --git a/tests/health_check/app.yaml b/tests/health_check/app.yaml new file mode 100644 index 00000000..c4a838e6 --- /dev/null +++ b/tests/health_check/app.yaml @@ -0,0 +1,3 @@ +runtime: python +env: flex +entrypoint: gunicorn -b :$PORT main:app diff --git a/tests/health_check/main.py b/tests/health_check/main.py new file mode 100644 index 00000000..97eb37d8 --- /dev/null +++ b/tests/health_check/main.py @@ -0,0 +1,43 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START app] +import logging + +from flask import Flask + + +app = Flask(__name__) + + +@app.route('/') +def hello(): + """Return a friendly HTTP greeting.""" + return 'Hello World!' + + +@app.errorhandler(500) +def server_error(e): + logging.exception('An error occurred during a request.') + return """ + An internal error occurred: 
{}
+ See logs for full stacktrace. + """.format(e), 500 + + +if __name__ == '__main__': + # This is used when running locally. Gunicorn is used to run the + # application on Google App Engine. See entrypoint in app.yaml. + app.run(host='127.0.0.1', port=8080, debug=True) +# [END app] diff --git a/tests/health_check/requirements.txt b/tests/health_check/requirements.txt new file mode 100644 index 00000000..e7676bba --- /dev/null +++ b/tests/health_check/requirements.txt @@ -0,0 +1,2 @@ +Flask==0.12.1 +gunicorn==19.7.1 From 0feceedcdfefc972d7c31bc9ba2a38e71f5bfc9b Mon Sep 17 00:00:00 2001 From: Nick Kubala Date: Fri, 16 Jun 2017 16:12:23 -0700 Subject: [PATCH 181/362] health_check -> deploy_check --- tests/{health_check => deploy_check}/app.yaml | 0 tests/{health_check => deploy_check}/main.py | 0 tests/{health_check => deploy_check}/requirements.txt | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename tests/{health_check => deploy_check}/app.yaml (100%) rename tests/{health_check => deploy_check}/main.py (100%) rename tests/{health_check => deploy_check}/requirements.txt (100%) diff --git a/tests/health_check/app.yaml b/tests/deploy_check/app.yaml similarity index 100% rename from tests/health_check/app.yaml rename to tests/deploy_check/app.yaml diff --git a/tests/health_check/main.py b/tests/deploy_check/main.py similarity index 100% rename from tests/health_check/main.py rename to tests/deploy_check/main.py diff --git a/tests/health_check/requirements.txt b/tests/deploy_check/requirements.txt similarity index 100% rename from tests/health_check/requirements.txt rename to tests/deploy_check/requirements.txt From 2d172e042218c926f367344b4a6064984717401b Mon Sep 17 00:00:00 2001 From: Nick Kubala Date: Mon, 19 Jun 2017 13:29:20 -0700 Subject: [PATCH 182/362] rename FULL_BASE_IMAGE to STAGING_IMAGE to match integration test framework --- build.sh | 6 +++--- tests/benchmark/Dockerfile.in | 2 +- tests/benchmark/benchmark_between_releases.sh | 8 ++++---- tests/google-cloud-python-system/Dockerfile.in | 2 +- tests/google-cloud-python/Dockerfile.in | 2 +- tests/integration/Dockerfile.in | 2 +- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/build.sh b/build.sh index 5929fbb2..b7796dd6 100755 --- a/build.sh +++ b/build.sh @@ -108,8 +108,8 @@ fi # Use latest released Debian as our base image export DEBIAN_BASE_IMAGE="gcr.io/google-appengine/debian8:latest" -export FULL_BASE_IMAGE="${DOCKER_NAMESPACE}/python:${TAG}" -echo "Using base image name ${FULL_BASE_IMAGE}" +export STAGING_IMAGE="${DOCKER_NAMESPACE}/python:${TAG}" +echo "Using base image name ${STAGING_IMAGE}" # Generate Dockerfiles for outfile in \ @@ -120,7 +120,7 @@ for outfile in \ tests/google-cloud-python-system/Dockerfile \ tests/integration/Dockerfile \ ; do - envsubst <"${outfile}".in >"${outfile}" '$DEBIAN_BASE_IMAGE $FULL_BASE_IMAGE $GOOGLE_CLOUD_PROJECT_FOR_TESTS' + envsubst <"${outfile}".in >"${outfile}" '$DEBIAN_BASE_IMAGE $STAGING_IMAGE $GOOGLE_CLOUD_PROJECT_FOR_TESTS' done # Build images and push to GCR diff --git a/tests/benchmark/Dockerfile.in b/tests/benchmark/Dockerfile.in index 6166cdd9..ca0edc24 100644 --- a/tests/benchmark/Dockerfile.in +++ b/tests/benchmark/Dockerfile.in @@ -1,4 +1,4 @@ -FROM ${FULL_BASE_IMAGE} +FROM ${STAGING_IMAGE} # Install performance RUN pip install performance diff --git a/tests/benchmark/benchmark_between_releases.sh b/tests/benchmark/benchmark_between_releases.sh index f63a2877..0e105cf3 100755 --- a/tests/benchmark/benchmark_between_releases.sh +++ b/tests/benchmark/benchmark_between_releases.sh @@ -2,15 +2,15 @@ # Build the benchmark image for release 1 from Dockerfile echo "Building image for release 1" -export FULL_BASE_IMAGE="${DOCKER_NAMESPACE}/python:${TAG1}" -envsubst <"Dockerfile".in >"Dockerfile" '$FULL_BASE_IMAGE' +export STAGING_IMAGE="${DOCKER_NAMESPACE}/python:${TAG1}" +envsubst <"Dockerfile".in >"Dockerfile" '$STAGING_IMAGE' docker build --no-cache -t benchmark_1 . rm Dockerfile # Build the benchmark image for release 2 from Dockerfile echo "Building image for release 2" -export FULL_BASE_IMAGE="${DOCKER_NAMESPACE}/python:${TAG2}" -envsubst <"Dockerfile".in >"Dockerfile" '$FULL_BASE_IMAGE' +export STAGING_IMAGE="${DOCKER_NAMESPACE}/python:${TAG2}" +envsubst <"Dockerfile".in >"Dockerfile" '$STAGING_IMAGE' docker build --no-cache -t benchmark_2 . rm Dockerfile diff --git a/tests/google-cloud-python-system/Dockerfile.in b/tests/google-cloud-python-system/Dockerfile.in index 18506681..04f28bd5 100644 --- a/tests/google-cloud-python-system/Dockerfile.in +++ b/tests/google-cloud-python-system/Dockerfile.in @@ -1,4 +1,4 @@ -FROM ${FULL_BASE_IMAGE} +FROM ${STAGING_IMAGE} # Get the source. RUN git clone --depth 1 https://github.com/GoogleCloudPlatform/google-cloud-python.git diff --git a/tests/google-cloud-python/Dockerfile.in b/tests/google-cloud-python/Dockerfile.in index ada429fb..a2873863 100644 --- a/tests/google-cloud-python/Dockerfile.in +++ b/tests/google-cloud-python/Dockerfile.in @@ -1,4 +1,4 @@ -FROM ${FULL_BASE_IMAGE} +FROM ${STAGING_IMAGE} # Get the source. RUN git clone --depth 1 https://github.com/GoogleCloudPlatform/google-cloud-python.git diff --git a/tests/integration/Dockerfile.in b/tests/integration/Dockerfile.in index 52f848f6..e18d60ef 100644 --- a/tests/integration/Dockerfile.in +++ b/tests/integration/Dockerfile.in @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM ${FULL_BASE_IMAGE} +FROM ${STAGING_IMAGE} COPY . /app WORKDIR /app From cb1b7a58503c6b90c14cca877fbdd51a0d16ad7c Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Mon, 19 Jun 2017 15:03:35 -0700 Subject: [PATCH 183/362] Explicitly pass individual arg values instead of argparse object. --- scripts/gen_dockerfile.py | 42 ++++++++++++++++++---------------- scripts/gen_dockerfile_test.py | 23 ++++++++++--------- 2 files changed, 34 insertions(+), 31 deletions(-) diff --git a/scripts/gen_dockerfile.py b/scripts/gen_dockerfile.py index b02684fb..071b1cd1 100755 --- a/scripts/gen_dockerfile.py +++ b/scripts/gen_dockerfile.py @@ -108,12 +108,14 @@ ) -def get_app_config(raw_config, args): +def get_app_config(raw_config, base_image, config_file, source_dir): """Read and validate the application runtime configuration. Args: raw_config (dict): deserialized app.yaml - args (argparse.Namespace): command line flags + base_image (str): Docker image name to build on top of + config_file (str): Path to user's app.yaml (might be .yaml) + source_dir (str): Directory container user's source code Returns: AppConfig: valid configuration @@ -122,7 +124,7 @@ def get_app_config(raw_config, args): if not isinstance(raw_config, dict): raise ValueError( 'Expected {} contents to be of type "dict", but found type "{}"'. - format(args.config, type(raw_config))) + format(config_file, type(raw_config))) entrypoint = validation_utils.get_field_value(raw_config, 'entrypoint', str) if not PRINTABLE_REGEX.match(entrypoint): @@ -139,10 +141,7 @@ def get_app_config(raw_config, args): # Examine user's files has_requirements_txt = os.path.isfile( - os.path.join(args.source_dir, 'requirements.txt')) - - # Compute base image name and Dockerfile contents - base_image = args.base_image + os.path.join(source_dir, 'requirements.txt')) return AppConfig( base_image=base_image, @@ -191,26 +190,29 @@ def generate_files(app_config): } -def gen_dockerfile(args): +def gen_dockerfile(base_image, config_file, source_dir): """Write a Dockerfile and helper files for an application. Args: - args (argparse.Namespace): command line flags + base_image (str): Docker image name to build on top of + config_file (str): Path to user's app.yaml (might be .yaml) + source_dir (str): Directory container user's source code """ # Read yaml file. Does not currently support multiple services # with configuration filenames besides app.yaml - with io.open(args.config, 'r', encoding='utf8') as yaml_config_file: + with io.open(config_file, 'r', encoding='utf8') as yaml_config_file: raw_config = yaml.load(yaml_config_file) - # Determine configuration - app_config = get_app_config(raw_config, args) + # Determine complete configuration + app_config = get_app_config(raw_config, base_image, config_file, + source_dir) # Generate list of filenames and their textual contents files = generate_files(app_config) # Write files for filename, contents in files.items(): - full_filename = os.path.join(args.source_dir, filename) + full_filename = os.path.join(source_dir, filename) with io.open(full_filename, 'w', encoding='utf8') as outfile: outfile.write(contents) @@ -218,6 +220,12 @@ def gen_dockerfile(args): def parse_args(argv): """Parse and validate command line flags""" parser = argparse.ArgumentParser() + parser.add_argument( + '--base-image', + type=functools.partial( + validation_utils.validate_arg_regex, flag_regex=IMAGE_REGEX), + default='gcr.io/google-appengine/python:latest', + help='Name of Docker image to use as base') parser.add_argument( '--config', type=functools.partial( @@ -225,12 +233,6 @@ def parse_args(argv): default='app.yaml', help='Path to application configuration file' ) - parser.add_argument( - '--base-image', - type=functools.partial( - validation_utils.validate_arg_regex, flag_regex=IMAGE_REGEX), - default='gcr.io/google-appengine/python:latest', - help='Name of Docker image to use as base') parser.add_argument( '--source-dir', type=functools.partial( @@ -243,7 +245,7 @@ def parse_args(argv): def main(): args = parse_args(sys.argv) - gen_dockerfile(args) + gen_dockerfile(args.base_image, args.config, args.source_dir) if __name__ == '__main__': diff --git a/scripts/gen_dockerfile_test.py b/scripts/gen_dockerfile_test.py index e8ad371d..8cbbd1d6 100755 --- a/scripts/gen_dockerfile_test.py +++ b/scripts/gen_dockerfile_test.py @@ -49,10 +49,9 @@ def compare_file(self, filename, dir1, dir2): self.assertMultiLineEqual(contents1, contents2, msg) def test_get_app_config(self): - args = argparse.Namespace( - config='some_config_file', - base_image='some_image_name', - source_dir='some_source_dir') + config_file = 'some_config_file' + base_image = 'some_image_name' + source_dir = 'some_source_dir' valid_cases = ( # Basic app.yaml @@ -93,7 +92,9 @@ def test_get_app_config(self): raw_app_config = yaml.safe_load(app_yaml) with unittest.mock.patch.object( os.path, 'isfile', return_value=isfile): - actual = gen_dockerfile.get_app_config(raw_app_config, args) + actual = gen_dockerfile.get_app_config( + raw_app_config, base_image, config_file, + source_dir) for key, value in expected.items(): self.assertEqual(getattr(actual, key), value) @@ -110,7 +111,9 @@ def test_get_app_config(self): with self.subTest(invalid_case=invalid_case): raw_app_config = yaml.safe_load(invalid_case) with self.assertRaises(ValueError): - gen_dockerfile.get_app_config(raw_app_config, args) + gen_dockerfile.get_app_config( + raw_app_config, base_image, config_file, + source_dir) def test_generate_files(self): base = gen_dockerfile.AppConfig( @@ -165,12 +168,10 @@ def test_gen_dockerfile(self): # Copy sample app to writable temp dir, and generate Dockerfile. config_dir = os.path.join(temp_dir, 'config') shutil.copytree(app_dir, config_dir) - args = argparse.Namespace( - config=os.path.join(config_dir, 'app.yaml'), + gen_dockerfile.gen_dockerfile( base_image='gcr.io/google-appengine/python', - source_dir=config_dir, - ) - gen_dockerfile.gen_dockerfile(args) + config_file=os.path.join(config_dir, 'app.yaml'), + source_dir=config_dir) # Compare against golden files golden_dir = os.path.join(self.testdata_dir, app + '_golden') From 6e4a63d36de781ad8a0d1e7384ccf84a593128eb Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Mon, 19 Jun 2017 15:23:51 -0700 Subject: [PATCH 184/362] Pin package version --- builder/gen-dockerfile/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/gen-dockerfile/requirements.txt b/builder/gen-dockerfile/requirements.txt index c3726e8b..59bc593a 100644 --- a/builder/gen-dockerfile/requirements.txt +++ b/builder/gen-dockerfile/requirements.txt @@ -1 +1 @@ -pyyaml +PyYAML==3.12 From 4ca86d15d6887e8ff5678b7d5d05400ff9d188f2 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Mon, 19 Jun 2017 15:24:24 -0700 Subject: [PATCH 185/362] Use filecmp module instead of handrolled comparison --- scripts/gen_dockerfile.py | 2 ++ scripts/gen_dockerfile_test.py | 13 ++++++------- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/scripts/gen_dockerfile.py b/scripts/gen_dockerfile.py index 071b1cd1..e9debe83 100755 --- a/scripts/gen_dockerfile.py +++ b/scripts/gen_dockerfile.py @@ -129,8 +129,10 @@ def get_app_config(raw_config, base_image, config_file, source_dir): entrypoint = validation_utils.get_field_value(raw_config, 'entrypoint', str) if not PRINTABLE_REGEX.match(entrypoint): raise ValueError('Invalid character in "entrypoint" field of app.yaml') + raw_runtime_config = validation_utils.get_field_value(raw_config, 'runtime_config', dict) python_version = validation_utils.get_field_value(raw_runtime_config, 'python_version', str) + dockerfile_python_version = PYTHON_INTERPRETER_VERSION_MAP.get( python_version) if dockerfile_python_version is None: diff --git a/scripts/gen_dockerfile_test.py b/scripts/gen_dockerfile_test.py index 8cbbd1d6..9ba439aa 100755 --- a/scripts/gen_dockerfile_test.py +++ b/scripts/gen_dockerfile_test.py @@ -17,6 +17,7 @@ """Unit test for gen_dockerfile.py""" import argparse +import filecmp import os import re import shutil @@ -40,13 +41,11 @@ def setUp(self): def compare_file(self, filename, dir1, dir2): """Compare identically named files in two different directories""" - with open(os.path.join(dir1, filename), 'r', encoding='utf8') as file1: - with open(os.path.join(dir2, filename), 'r', encoding='utf8') as file2: - contents1 = file1.read() - contents2 = file2.read() - msg = 'Contents of "{}" differ between "{}" and "{}"'.format( - filename, dir1, dir2) - self.assertMultiLineEqual(contents1, contents2, msg) + if not filecmp.cmp(os.path.join(dir1, filename), + os.path.join(dir2, filename)): + msg = 'Contents of "{}" differ between "{}" and "{}"'.format( + filename, dir1, dir2) + self.assertMultiLineEqual(contents1, contents2, msg) def test_get_app_config(self): config_file = 'some_config_file' From ad9a940bdf41bc433d2dbc262f77f659fd9a3cf6 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Mon, 19 Jun 2017 15:49:24 -0700 Subject: [PATCH 186/362] Fix style nit --- scripts/gen_dockerfile.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/gen_dockerfile.py b/scripts/gen_dockerfile.py index e9debe83..abb891bc 100755 --- a/scripts/gen_dockerfile.py +++ b/scripts/gen_dockerfile.py @@ -137,9 +137,9 @@ def get_app_config(raw_config, base_image, config_file, source_dir): python_version) if dockerfile_python_version is None: valid_versions = str(sorted(PYTHON_INTERPRETER_VERSION_MAP.keys())) - msg = ('Invalid "python_version" field in "runtime_config" section ' - 'of app.yaml. Valid options are:\n{}').format(valid_versions) - raise ValueError(msg) + raise ValueError( + 'Invalid "python_version" field in "runtime_config" section ' + 'of app.yaml. Valid options are:\n{}'.format(valid_versions)) # Examine user's files has_requirements_txt = os.path.isfile( From 336e42a16fd9cb29c4eff3cd69d85686edadad6c Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Mon, 19 Jun 2017 16:07:13 -0700 Subject: [PATCH 187/362] Rename function so it doesn't shadow module name. --- scripts/gen_dockerfile.py | 4 ++-- scripts/gen_dockerfile_test.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/gen_dockerfile.py b/scripts/gen_dockerfile.py index abb891bc..bdbf1802 100755 --- a/scripts/gen_dockerfile.py +++ b/scripts/gen_dockerfile.py @@ -192,7 +192,7 @@ def generate_files(app_config): } -def gen_dockerfile(base_image, config_file, source_dir): +def generate_dockerfile_command(base_image, config_file, source_dir): """Write a Dockerfile and helper files for an application. Args: @@ -247,7 +247,7 @@ def parse_args(argv): def main(): args = parse_args(sys.argv) - gen_dockerfile(args.base_image, args.config, args.source_dir) + generate_dockerfile_command(args.base_image, args.config, args.source_dir) if __name__ == '__main__': diff --git a/scripts/gen_dockerfile_test.py b/scripts/gen_dockerfile_test.py index 9ba439aa..92f70061 100755 --- a/scripts/gen_dockerfile_test.py +++ b/scripts/gen_dockerfile_test.py @@ -151,7 +151,7 @@ def test_generate_files(self): else: self.assertNotIn(test_string, dockerfile) - def test_gen_dockerfile(self): + def test_generate_dockerfile_command(self): """Generates output and compares against a set of golden files. Optionally runs 'gcloud app gen-config' and compares against that. @@ -167,7 +167,7 @@ def test_gen_dockerfile(self): # Copy sample app to writable temp dir, and generate Dockerfile. config_dir = os.path.join(temp_dir, 'config') shutil.copytree(app_dir, config_dir) - gen_dockerfile.gen_dockerfile( + gen_dockerfile.generate_dockerfile_command( base_image='gcr.io/google-appengine/python', config_file=os.path.join(config_dir, 'app.yaml'), source_dir=config_dir) From 881196ac012829edccc5e9544d8dd88f4c3e8a76 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Wed, 21 Jun 2017 13:43:46 -0700 Subject: [PATCH 188/362] Move file templates into separate files. The templates were copied from the gae-xrt-python template files in the gcloud SDK, with the same names and contents, except for Dockerfile.preamble which no longer hardcodes the base image, and Dockerfile.entrypoint.template which is new. --- build.sh | 1 + scripts/data/Dockerfile.entrypoint.template | 1 + scripts/data/Dockerfile.install_app | 1 + scripts/data/Dockerfile.preamble.template | 1 + scripts/data/Dockerfile.requirements_txt | 2 + scripts/data/Dockerfile.virtualenv.template | 7 ++ scripts/data/dockerignore | 19 +++++ scripts/gen_dockerfile.py | 88 +++++++-------------- scripts/gen_dockerfile_test.py | 4 +- 9 files changed, 61 insertions(+), 63 deletions(-) create mode 100644 scripts/data/Dockerfile.entrypoint.template create mode 100644 scripts/data/Dockerfile.install_app create mode 100644 scripts/data/Dockerfile.preamble.template create mode 100644 scripts/data/Dockerfile.requirements_txt create mode 100644 scripts/data/Dockerfile.virtualenv.template create mode 100644 scripts/data/dockerignore diff --git a/build.sh b/build.sh index 0e01abcd..7d20ef03 100755 --- a/build.sh +++ b/build.sh @@ -128,6 +128,7 @@ done for file in \ gen_dockerfile.py \ validation_utils.py \ + 'data/*' \ ; do cp -a "scripts/${file}" "builder/gen-dockerfile/${file}" done diff --git a/scripts/data/Dockerfile.entrypoint.template b/scripts/data/Dockerfile.entrypoint.template new file mode 100644 index 00000000..f6b52bf6 --- /dev/null +++ b/scripts/data/Dockerfile.entrypoint.template @@ -0,0 +1 @@ +CMD {entrypoint} diff --git a/scripts/data/Dockerfile.install_app b/scripts/data/Dockerfile.install_app new file mode 100644 index 00000000..54c3d6cc --- /dev/null +++ b/scripts/data/Dockerfile.install_app @@ -0,0 +1 @@ +ADD . /app/ diff --git a/scripts/data/Dockerfile.preamble.template b/scripts/data/Dockerfile.preamble.template new file mode 100644 index 00000000..e4d005bd --- /dev/null +++ b/scripts/data/Dockerfile.preamble.template @@ -0,0 +1 @@ +FROM {base_image} diff --git a/scripts/data/Dockerfile.requirements_txt b/scripts/data/Dockerfile.requirements_txt new file mode 100644 index 00000000..f684c45c --- /dev/null +++ b/scripts/data/Dockerfile.requirements_txt @@ -0,0 +1,2 @@ +ADD requirements.txt /app/ +RUN pip install -r requirements.txt diff --git a/scripts/data/Dockerfile.virtualenv.template b/scripts/data/Dockerfile.virtualenv.template new file mode 100644 index 00000000..557b1992 --- /dev/null +++ b/scripts/data/Dockerfile.virtualenv.template @@ -0,0 +1,7 @@ +LABEL python_version=python{python_version} +RUN virtualenv --no-download /env -p python{python_version} + +# Set virtualenv environment variables. This is equivalent to running +# source /env/bin/activate +ENV VIRTUAL_ENV /env +ENV PATH /env/bin:$PATH diff --git a/scripts/data/dockerignore b/scripts/data/dockerignore new file mode 100644 index 00000000..8b927bb7 --- /dev/null +++ b/scripts/data/dockerignore @@ -0,0 +1,19 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +.dockerignore +Dockerfile +.git +.hg +.svn diff --git a/scripts/gen_dockerfile.py b/scripts/gen_dockerfile.py index bdbf1802..30004ead 100755 --- a/scripts/gen_dockerfile.py +++ b/scripts/gen_dockerfile.py @@ -56,50 +56,6 @@ '3.6': '3.6', } -# File templates. -# Designed to exactly match the current output of 'gcloud app gen-config' -DOCKERFILE_TEMPLATE = """\ -FROM {base_image} -LABEL python_version=python{dockerfile_python_version} -RUN virtualenv --no-download /env -p python{dockerfile_python_version} - -# Set virtualenv environment variables. This is equivalent to running -# source /env/bin/activate -ENV VIRTUAL_ENV /env -ENV PATH /env/bin:$PATH -{optional_requirements_txt}ADD . /app/ -{optional_entrypoint}""" - -DOCKERFILE_REQUIREMENTS_TXT = """\ -ADD requirements.txt /app/ -RUN pip install -r requirements.txt -""" - -DOCKERFILE_ENTRYPOINT_TEMPLATE = """\ -CMD {entrypoint} -""" - -DOCKERIGNORE = """\ -# Copyright 2015 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -.dockerignore -Dockerfile -.git -.hg -.svn -""" # Validated application configuration AppConfig = collections.namedtuple( @@ -130,6 +86,14 @@ def get_app_config(raw_config, base_image, config_file, source_dir): if not PRINTABLE_REGEX.match(entrypoint): raise ValueError('Invalid character in "entrypoint" field of app.yaml') + # Mangle entrypoint in the same way as the Cloud SDK + # (googlecloudsdk/third_party/appengine/api/validation.py) + # + # We could handle both string ("shell form") and list ("exec + # form") but it appears that gcloud only handles string form. + if entrypoint and not entrypoint.startswith('exec '): + entrypoint = 'exec ' + entrypoint + raw_runtime_config = validation_utils.get_field_value(raw_config, 'runtime_config', dict) python_version = validation_utils.get_field_value(raw_runtime_config, 'python_version', str) @@ -152,6 +116,13 @@ def get_app_config(raw_config, base_image, config_file, source_dir): has_requirements_txt=has_requirements_txt) +def get_data(name): + """Return the contents of the named data resource""" + filename = os.path.join(os.path.dirname(__file__), 'data', name) + with io.open(filename, 'r', encoding='utf8') as template_file: + return template_file.read() + + def generate_files(app_config): """Generate a Dockerfile and helper files for an application. @@ -162,33 +133,28 @@ def generate_files(app_config): dict: Map of filename to desired file contents """ if app_config.has_requirements_txt: - optional_requirements_txt = DOCKERFILE_REQUIREMENTS_TXT + optional_requirements_txt = get_data('Dockerfile.requirements_txt') else: optional_requirements_txt = '' if app_config.entrypoint: - # Mangle entrypoint in the same way as the Cloud SDK - # (googlecloudsdk/third_party/appengine/api/validation.py) - # - # We could handle both string ("shell form") and list ("exec - # form") but it appears that gcloud only handles string form. - entrypoint = app_config.entrypoint - if entrypoint and not entrypoint.startswith('exec '): - entrypoint = 'exec ' + entrypoint - optional_entrypoint = DOCKERFILE_ENTRYPOINT_TEMPLATE.format( - entrypoint=entrypoint) + optional_entrypoint = get_data('Dockerfile.entrypoint.template').format( + entrypoint=app_config.entrypoint) else: optional_entrypoint = '' - dockerfile = DOCKERFILE_TEMPLATE.format( - base_image=app_config.base_image, - dockerfile_python_version=app_config.dockerfile_python_version, - optional_requirements_txt=optional_requirements_txt, - optional_entrypoint=optional_entrypoint) + dockerfile = ( + get_data('Dockerfile.preamble.template').format( + base_image=app_config.base_image) + + get_data('Dockerfile.virtualenv.template').format( + python_version=app_config.dockerfile_python_version) + + optional_requirements_txt + + get_data('Dockerfile.install_app') + + optional_entrypoint) return { 'Dockerfile': dockerfile, - '.dockerignore': DOCKERIGNORE, + '.dockerignore': get_data('dockerignore'), } diff --git a/scripts/gen_dockerfile_test.py b/scripts/gen_dockerfile_test.py index 92f70061..8169b297 100755 --- a/scripts/gen_dockerfile_test.py +++ b/scripts/gen_dockerfile_test.py @@ -82,7 +82,7 @@ def test_get_app_config(self): }), # entrypoint present ('entrypoint: my entrypoint', False, { - 'entrypoint': 'my entrypoint', + 'entrypoint': 'exec my entrypoint', }), ) for valid_case in valid_cases: @@ -129,7 +129,7 @@ def test_generate_files(self): # Entrypoint (base, False, 'CMD'), (base._replace(entrypoint='my entrypoint'), True, - 'CMD exec my entrypoint'), + 'CMD my entrypoint'), (base._replace(entrypoint='exec my entrypoint'), True, 'CMD exec my entrypoint'), # Base runtime image From c24977b57fff8be3ee228d91405670f707aa400b Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Wed, 21 Jun 2017 14:11:21 -0700 Subject: [PATCH 189/362] Relax type check and add comments about validation --- scripts/gen_dockerfile.py | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/scripts/gen_dockerfile.py b/scripts/gen_dockerfile.py index 30004ead..eb5dabbf 100755 --- a/scripts/gen_dockerfile.py +++ b/scripts/gen_dockerfile.py @@ -18,6 +18,7 @@ import argparse import collections +import collections.abc import functools import io import os @@ -73,13 +74,25 @@ def get_app_config(raw_config, base_image, config_file, source_dir): config_file (str): Path to user's app.yaml (might be .yaml) source_dir (str): Directory container user's source code + We validate the user input for security and better error messages. + + Yaml parsing rules can lead to extremely unhelpful error messages. + For example, parsing a string value where we expected a list. + Python will happily use the string as a sequence of individual + characters, leading to confusing results. + + We also try to prevent Dockerfile and Bash injection attacks. For + example, specifying entrypoint as "true\\nADD /etc/passwd /pwned" + would allow the user to inject arbitrary directives into the + Dockerfile, which is a support problem if nothing else. + Returns: AppConfig: valid configuration """ # Examine app.yaml - if not isinstance(raw_config, dict): + if not isinstance(raw_config, collections.abc.Mapping): raise ValueError( - 'Expected {} contents to be of type "dict", but found type "{}"'. + 'Expected {} contents to be a Mapping type, but found type "{}"'. format(config_file, type(raw_config))) entrypoint = validation_utils.get_field_value(raw_config, 'entrypoint', str) @@ -117,7 +130,18 @@ def get_app_config(raw_config, base_image, config_file, source_dir): def get_data(name): - """Return the contents of the named data resource""" + """Return the contents of the named data resource + + Args: + name (str): Name of file, without directory + + These templates are copied from the Google Cloud SDK at + google-cloud-sdk/platform/ext-runtime/python/data + and the two should be kept in sync. + + Returns: + str: Contents of data file + """ filename = os.path.join(os.path.dirname(__file__), 'data', name) with io.open(filename, 'r', encoding='utf8') as template_file: return template_file.read() From c34d4f04afbee13cd90d44430ac154162a34fd0c Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Wed, 21 Jun 2017 14:40:00 -0700 Subject: [PATCH 190/362] Document rationale for yaml validation --- scripts/gen_dockerfile.py | 26 +++++++++++++------------- scripts/validation_utils.py | 11 ++++++++++- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/scripts/gen_dockerfile.py b/scripts/gen_dockerfile.py index eb5dabbf..c75bf52f 100755 --- a/scripts/gen_dockerfile.py +++ b/scripts/gen_dockerfile.py @@ -68,24 +68,24 @@ def get_app_config(raw_config, base_image, config_file, source_dir): """Read and validate the application runtime configuration. - Args: - raw_config (dict): deserialized app.yaml - base_image (str): Docker image name to build on top of - config_file (str): Path to user's app.yaml (might be .yaml) - source_dir (str): Directory container user's source code - We validate the user input for security and better error messages. - Yaml parsing rules can lead to extremely unhelpful error messages. - For example, parsing a string value where we expected a list. - Python will happily use the string as a sequence of individual - characters, leading to confusing results. + Consider, parsing a yaml file which has a string value where we + expected a list. Python will happily use the string as a sequence + of individual characters, at least for a while, leading to + confusing results when it finally fails. We also try to prevent Dockerfile and Bash injection attacks. For example, specifying entrypoint as "true\\nADD /etc/passwd /pwned" would allow the user to inject arbitrary directives into the Dockerfile, which is a support problem if nothing else. + Args: + raw_config (dict): deserialized app.yaml + base_image (str): Docker image name to build on top of + config_file (str): Path to user's app.yaml (might be .yaml) + source_dir (str): Directory container user's source code + Returns: AppConfig: valid configuration """ @@ -132,13 +132,13 @@ def get_app_config(raw_config, base_image, config_file, source_dir): def get_data(name): """Return the contents of the named data resource - Args: - name (str): Name of file, without directory - These templates are copied from the Google Cloud SDK at google-cloud-sdk/platform/ext-runtime/python/data and the two should be kept in sync. + Args: + name (str): Name of file, without directory + Returns: str: Contents of data file """ diff --git a/scripts/validation_utils.py b/scripts/validation_utils.py index 7a04847e..2792dcb1 100644 --- a/scripts/validation_utils.py +++ b/scripts/validation_utils.py @@ -29,9 +29,18 @@ def get_field_value(container, field_name, field_type): """Fetch a field from a container with typechecking and default values. The field value is coerced to the desired type. If the field is - not present, a instance of `field_type` is constructed with no + not present, an instance of `field_type` is constructed with no arguments and used as the default value. + This function exists because yaml parsing can lead to surprising + outputs, and the resulting errors are confusing. For example: + entrypoint1: a string, but I can accidentally treat as an sequence + entrypoint2: [a, list, but, I, might, think, its, a, string] + version1: 3 # Parsed to int + version2: 3.1 # Parsed to float + version3: 3.1.1 # Parsed to str + feature: off # Parsed to the boolean False + Args: container (dict): Object decoded from yaml field_name (str): Field that should be present in `container` From 2fd8330f30ba7f8554cdb41c233711d22f610fbc Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Thu, 22 Jun 2017 14:40:15 -0700 Subject: [PATCH 191/362] Add more information to error messages Also use yaml "safe" mode, because "unsafe" is the default for some reason. --- scripts/gen_dockerfile.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/scripts/gen_dockerfile.py b/scripts/gen_dockerfile.py index c75bf52f..441fb153 100755 --- a/scripts/gen_dockerfile.py +++ b/scripts/gen_dockerfile.py @@ -97,7 +97,8 @@ def get_app_config(raw_config, base_image, config_file, source_dir): entrypoint = validation_utils.get_field_value(raw_config, 'entrypoint', str) if not PRINTABLE_REGEX.match(entrypoint): - raise ValueError('Invalid character in "entrypoint" field of app.yaml') + raise ValueError( + 'Invalid "entrypoint" value in app.yaml: {!r}'.format(entrypoint)) # Mangle entrypoint in the same way as the Cloud SDK # (googlecloudsdk/third_party/appengine/api/validation.py) @@ -116,7 +117,8 @@ def get_app_config(raw_config, base_image, config_file, source_dir): valid_versions = str(sorted(PYTHON_INTERPRETER_VERSION_MAP.keys())) raise ValueError( 'Invalid "python_version" field in "runtime_config" section ' - 'of app.yaml. Valid options are:\n{}'.format(valid_versions)) + 'of app.yaml: {!r}. Valid options are: {}'. + format(python_version, valid_versions)) # Examine user's files has_requirements_txt = os.path.isfile( @@ -167,14 +169,15 @@ def generate_files(app_config): else: optional_entrypoint = '' - dockerfile = ( + dockerfile = ''.join([ get_data('Dockerfile.preamble.template').format( - base_image=app_config.base_image) + + base_image=app_config.base_image), get_data('Dockerfile.virtualenv.template').format( - python_version=app_config.dockerfile_python_version) + - optional_requirements_txt + - get_data('Dockerfile.install_app') + - optional_entrypoint) + python_version=app_config.dockerfile_python_version), + optional_requirements_txt, + get_data('Dockerfile.install_app'), + optional_entrypoint , + ]) return { 'Dockerfile': dockerfile, @@ -193,7 +196,7 @@ def generate_dockerfile_command(base_image, config_file, source_dir): # Read yaml file. Does not currently support multiple services # with configuration filenames besides app.yaml with io.open(config_file, 'r', encoding='utf8') as yaml_config_file: - raw_config = yaml.load(yaml_config_file) + raw_config = yaml.safe_load(yaml_config_file) # Determine complete configuration app_config = get_app_config(raw_config, base_image, config_file, From 118871687d3eed8cbe6696ccccfd1fb29dbf268e Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Thu, 22 Jun 2017 18:40:33 -0700 Subject: [PATCH 192/362] Add nox support for running tests, lint, coverage, and requirements checking --- .coveragerc | 6 +++++ .gitignore | 2 ++ nox.py | 46 ++++++++++++++++++++++++++++++++++- scripts/requirements-test.txt | 4 +++ 4 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 .coveragerc create mode 100644 scripts/requirements-test.txt diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 00000000..b08b2574 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,6 @@ +[run] +branch = True + +[report] +exclude_lines = + pragma: no cover diff --git a/.gitignore b/.gitignore index 2a84c77d..62f3090b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ *.pyc +.cache +.coverage .nox /cloudbuild.yaml_local.sh /cloudbuild_benchmark.yaml_local.sh diff --git a/nox.py b/nox.py index 83be1ffa..fd79945e 100644 --- a/nox.py +++ b/nox.py @@ -15,6 +15,8 @@ import fnmatch import os +import nox + def _list_files(folder, pattern): """Lists all files below the given folder that match the pattern.""" @@ -24,7 +26,8 @@ def _list_files(folder, pattern): yield os.path.join(root, filename) -def session_check_requirements(session): +@nox.session +def check_requirements(session): """Checks for out of date requirements and optionally updates them.""" session.install('gcp-devrel-py-tools') @@ -37,3 +40,44 @@ def session_check_requirements(session): for reqfile in reqfiles: session.run('gcp-devrel-py-tools', command, reqfile) + + +@nox.session +def lint(session): + session.install('flake8', 'flake8-import-order') + session.run( + 'flake8', + '--import-order-style=google', + 'scripts', + 'nox.py', + ) + + +@nox.session +@nox.parametrize('version', ['3.4', '3.5', '3.6']) +def tests(session, version): + session.interpreter = 'python' + version + session.install('-r', 'scripts/requirements-test.txt') + session.run( + 'py.test', + '--ignore=scripts/testdata', + '--cov=scripts', + '--cov-append', + '--cov-config=.coveragerc', + '--cov-report=', # Report generated below + 'scripts', + env={'PYTHONPATH': ''} + ) + + +@nox.session +def cover(session): + """Run the final coverage report. + + This outputs the coverage report aggregating coverage from the unit + test runs (not system test runs), and then erases coverage data. + """ + session.interpreter = 'python3.6' + session.install('coverage', 'pytest-cov') + session.run('coverage', 'report', '--show-missing', '--fail-under=97') + session.run('coverage', 'erase') diff --git a/scripts/requirements-test.txt b/scripts/requirements-test.txt new file mode 100644 index 00000000..77b8c636 --- /dev/null +++ b/scripts/requirements-test.txt @@ -0,0 +1,4 @@ +flask +pytest +pytest-cov +pyyaml From 17359a7515e880cfa6c494d5e71d08fa8b000477 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 23 Jun 2017 18:11:33 -0700 Subject: [PATCH 193/362] Be more verbose when running a build --- scripts/local_cloudbuild.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/local_cloudbuild.py b/scripts/local_cloudbuild.py index a7c5eda9..ab02d83a 100755 --- a/scripts/local_cloudbuild.py +++ b/scripts/local_cloudbuild.py @@ -322,6 +322,7 @@ def local_cloudbuild(args): # Run shell script if cloudbuild.run: + print('Running {}'.format(cloudbuild.output_script)) args = [os.path.abspath(cloudbuild.output_script)] subprocess.check_call(args) From bb138e6f65ba4941ed1d1d3bafe681e31b3770f6 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 23 Jun 2017 18:58:25 -0700 Subject: [PATCH 194/362] Switch local_cloudbuild.py to py.test --- scripts/local_cloudbuild_test.py | 712 ++++++++++++++++--------------- 1 file changed, 365 insertions(+), 347 deletions(-) diff --git a/scripts/local_cloudbuild_test.py b/scripts/local_cloudbuild_test.py index fff089dc..00609f01 100755 --- a/scripts/local_cloudbuild_test.py +++ b/scripts/local_cloudbuild_test.py @@ -17,14 +17,14 @@ """Unit test for local_cloudbuild.py""" import argparse +import contextlib import os import re import shutil import subprocess -import tempfile -import unittest import unittest.mock +import pytest import yaml import local_cloudbuild @@ -34,355 +34,373 @@ STAGING_DIR_REGEX = re.compile( b'(?m)Copying source to staging directory (.+)$') -class LocalCloudbuildTest(unittest.TestCase): - - def setUp(self): - self.testdata_dir = os.path.join(os.path.dirname(__file__), 'testdata') - assert os.path.isdir(self.testdata_dir), 'Could not run test: testdata directory not found' - - def test_sub_and_quote(self): - valid_cases = ( - # Empty string - ('', {}, "''", []), - # No substitutions - ('a', {}, 'a', []), - # Unused substitition (ok here but error in generate_script) - ('a', {'FOO':'foo'}, 'a', []), - ('a', {'_FOO':'_foo'}, 'a', []), - # Defined builtin substitution - ('a$FOOb', {'FOO':'foo'}, 'afoob', ['FOO']), - ('a${FOO}b', {'FOO':'foo'}, 'afoob', ['FOO']), - # Defined user substitution - ('a$_FOOb', {'_FOO':'_foo'}, 'a_foob', ['_FOO']), - ('a${_FOO}b', {'_FOO':'_foo'}, 'a_foob', ['_FOO']), - # Multiple substitutions - ('$FOO${FOO}${BAR}$FOO', {'FOO':'foo', 'BAR':'bar'}, - 'foofoobarfoo', ['FOO', 'BAR']), - # Invalid names - ('a $ b', {}, "'a $ b'", []), - ('a$foo b', {}, "'a$foo b'", []), - ('a$0FOO b', {}, "'a$0FOO b'", []), - ) - for valid_case in valid_cases: - with self.subTest(valid_case=valid_case): - s, subs, expected, expected_used = valid_case - used = set() - actual = local_cloudbuild.sub_and_quote(s, subs, used) - self.assertEqual(actual, expected) - self.assertEqual(used, set(expected_used)) - - invalid_cases = ( - # Undefined builtin substitution - ('a$FOOb', {}), - ('a${FOO}b', {}), - # Undefined user substitution - ('a$_FOOb', {}), - ('a${_FOO}b', {}), - ) - for invalid_case in invalid_cases: - with self.subTest(invalid_case=invalid_case): - s, subs = invalid_case - with self.assertRaises(ValueError): - used = set() - local_cloudbuild.sub_and_quote(s, subs, used) - - def check_call_with_capture(self, *args, **kw_args): - """Act like subprocess.check_call but capture stdout""" + +@pytest.fixture +def testdata_dir(): + testdata_dir = os.path.join(os.path.dirname(__file__), 'testdata') + assert os.path.isdir(testdata_dir), ( + 'Could not run test: testdata directory not found') + return testdata_dir + + +@pytest.mark.parametrize('s, subs, expected, expected_used', [ + # Empty string + ('', {}, "''", []), + # No substitutions + ('a', {}, 'a', []), + # Unused substitition (ok here but error in generate_script) + ('a', {'FOO': 'foo'}, 'a', []), + ('a', {'_FOO': '_foo'}, 'a', []), + # Defined builtin substitution + ('a$FOOb', {'FOO': 'foo'}, 'afoob', ['FOO']), + ('a${FOO}b', {'FOO': 'foo'}, 'afoob', ['FOO']), + # Defined user substitution + ('a$_FOOb', {'_FOO': '_foo'}, 'a_foob', ['_FOO']), + ('a${_FOO}b', {'_FOO': '_foo'}, 'a_foob', ['_FOO']), + # Multiple substitutions + ('$FOO${FOO}${BAR}$FOO', + {'FOO': 'foo', 'BAR': 'bar'}, + 'foofoobarfoo', + ['FOO', 'BAR']), + # Invalid names + ('a $ b', {}, "'a $ b'", []), + ('a$foo b', {}, "'a$foo b'", []), + ('a$0FOO b', {}, "'a$0FOO b'", []), +]) +def test_sub_and_quote_valid(s, subs, expected, expected_used): + used = set() + actual = local_cloudbuild.sub_and_quote(s, subs, used) + assert actual == expected + assert used == set(expected_used) + + +@pytest.mark.parametrize('s, subs', [ + # Undefined builtin substitution + ('a$FOOb', {}), + ('a${FOO}b', {}), + # Undefined user substitution + ('a$_FOOb', {}), + ('a${_FOO}b', {}), +]) +def test_sub_and_quote_invalid(s, subs): + with pytest.raises(ValueError): + used = set() + local_cloudbuild.sub_and_quote(s, subs, used) + + +def have_docker(): + """Determine if the Docker daemon is present and usable""" + if ((shutil.which('docker') is not None) and + (subprocess.call(['docker', 'info'], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) == 0)): + return True + return False + + +_args = argparse.Namespace( + config='some_config_file', + output_script='some_output_script', + run=False, + substitutions={}, +) + + +def test_get_cloudbuild_valid(): + raw_yaml = 'steps:\n- name: step1\n- name: step2\n' + raw_config = yaml.safe_load(raw_yaml) + actual = local_cloudbuild.get_cloudbuild(raw_config, _args) + assert len(actual.steps) == 2 + + +@pytest.mark.parametrize('raw_yaml', [ + # Empty cloud build + '', + # No steps + 'foo: bar\n', + # Steps not a list + 'steps: astring\n', + ]) +def test_get_cloudbuild_invalid(raw_yaml): + raw_config = yaml.safe_load(raw_yaml) + with pytest.raises(ValueError): + local_cloudbuild.get_cloudbuild(raw_config, _args) + + +@pytest.mark.parametrize('raw_step, expected', [ + # Empty step + ({}, local_cloudbuild.Step( + args=[], + dir_='', + env=[], + name='', + )), + # Full step + ({'name': 'aname', + 'args': ['arg1', 2, 'arg3 with \n newline'], + 'env': ['ENV1=value1', 'ENV2=space in value2'], + 'dir': 'adir', + }, local_cloudbuild.Step( + args=['arg1', '2', 'arg3 with \n newline'], + env=['ENV1=value1', 'ENV2=space in value2'], + dir_='adir', + name='aname', + )), +]) +def test_get_step_valid(raw_step, expected): + actual = local_cloudbuild.get_step(raw_step) + assert actual == expected + + +@pytest.mark.parametrize('raw_step', [ + # Wrong type + [], + # More wrong types + {'args': 'not_a_list'}, + {'args': [[]]}, + {'env': 'not_a_list'}, + {'env': [{}]}, + {'dir': {}}, + {'name': []}, +]) +def test_get_step_invalid(raw_step): + with pytest.raises(ValueError): + local_cloudbuild.get_step(raw_step) + + +# Basic valid case +_base_step = local_cloudbuild.Step( + args=['arg1', 'arg2'], + dir_='', + env=['ENV1=value1', 'ENV2=value2'], + name='aname', +) +_subs = {'BUILTIN': 'builtin', '_USER': '_user'} + + +def test_generate_command_basic(): + command = local_cloudbuild.generate_command(_base_step, _subs, set()) + assert command == [ + 'docker', + 'run', + '--volume', + '/var/run/docker.sock:/var/run/docker.sock', + '--volume', + '/root/.docker:/root/.docker', + '--volume', + '${HOST_WORKSPACE}:/workspace', + '--workdir', + '/workspace', + '--env', + 'ENV1=value1', + '--env', + 'ENV2=value2', + 'aname', + 'arg1', + 'arg2', + ] + + +@pytest.mark.parametrize('step, args', [ + # dir specified + (_base_step._replace(dir_='adir'), + ['--workdir', '/workspace/adir']), + # Shell quoting + (_base_step._replace(args=['arg with \n newline']), + ["'arg with \n newline'"]), + (_base_step._replace(dir_='dir/ with space/'), + ["/workspace/'dir/ with space/'"]), + (_base_step._replace(env=['env with space']), + ["'env with space'"]), + (_base_step._replace(name='a name'), + ["'a name'"]), + # Variable substitution + (_base_step._replace(name='a $BUILTIN substitution'), + ["'a builtin substitution'"]), + (_base_step._replace(name='a $_USER substitution'), + ["'a _user substitution'"]), + (_base_step._replace(name='a curly brace ${BUILTIN} substitution'), + ["'a curly brace builtin substitution'"]), + (_base_step._replace( + name='an escaped $$ or $$$$ or $$FOO or $${_FOO} is unescaped'), + ["'an escaped $ or $$ or $FOO or ${_FOO} is unescaped'"]), +]) +def test_generate_command_valid(step, args): + command = local_cloudbuild.generate_command(step, _subs, set()) + for arg in args: + assert arg in command + + +@pytest.mark.parametrize('step', [ + _base_step._replace(name='a $UNSET_BUILTIN substitution'), + _base_step._replace(name='a $_UNSET_USER substitution'), +]) +def test_generate_command_invalid(step): + with pytest.raises(ValueError): + local_cloudbuild.generate_command(step, _subs, set()) + + +def test_generate_script_golden(testdata_dir): + config_name = 'cloudbuild_ok.yaml' + expected_output_script = os.path.join( + testdata_dir, config_name + '_golden.sh') + cloudbuild = local_cloudbuild.CloudBuild( + output_script='test_generate_script', + run=False, + steps=[ + local_cloudbuild.Step( + args=['/bin/sh', '-c', 'printenv MESSAGE'], + dir_='', + env=['MESSAGE=Hello World!'], + name='debian', + ), + local_cloudbuild.Step( + args=['/bin/sh', '-c', 'printenv MESSAGE'], + dir_='', + env=['MESSAGE=Goodbye\\n And Farewell!', 'UNUSED=unused'], + name='debian', + ) + ], + substitutions=local_cloudbuild.DEFAULT_SUBSTITUTIONS, + ) + actual = local_cloudbuild.generate_script(cloudbuild) + # Compare output against golden + with open(expected_output_script, 'r', encoding='utf8') as expected_file: + expected = expected_file.read() + assert actual == expected + + +def test_generate_script_unused_user_substitution(): + cloudbuild = local_cloudbuild.CloudBuild( + output_script='', + run=False, + steps=[], + substitutions={'_FOO': '_foo'}, + ) + with pytest.raises(ValueError, match='User substitution variables'): + local_cloudbuild.generate_script(cloudbuild) + + +def test_make_executable(tmpdir): + test_script_filename = tmpdir.join('test_make_executable.sh') + with test_script_filename.open('w', encoding='utf8') as test_script: + test_script.write('#!/bin/sh\necho "Output from test_make_executable"') + local_cloudbuild.make_executable(str(test_script_filename)) + output = subprocess.check_output([str(test_script_filename)]) + assert output.decode('utf8') == "Output from test_make_executable\n" + + +def test_write_script(tmpdir): + contents = 'The contents\n' + output_script_filename = tmpdir.join('test_write_script') + cloudbuild = local_cloudbuild.CloudBuild( + output_script=str(output_script_filename), + run=False, + steps=[], + substitutions={}, + ) + local_cloudbuild.write_script(cloudbuild, contents) + with output_script_filename.open('r', encoding='utf8') as output_script: + actual = output_script.read() + assert actual == contents + + +@contextlib.contextmanager +def chdir(new_dir): + """Not threadsafe""" + old_dir = os.getcwd() + os.chdir(new_dir) + yield + os.chdir(old_dir) + + +@pytest.mark.parametrize('config_name, substitutions, exception, cleanup', [ + # Everything is ok + ('cloudbuild_ok.yaml', None, None, True), + # Builtin substitutions like $PROJECT_ID work + ('cloudbuild_builtin_substitutions.yaml', None, None, True), + # User substitutions like $_FOO work + ('cloudbuild_user_substitutions.yaml', + {'_FOO': 'this is foo value'}, + None, True + ), + # User substitutions like $_FOO fails when undefined + ('cloudbuild_user_substitutions.yaml', None, ValueError, False), + # Exit code 1 (failure) + ('cloudbuild_err_rc1.yaml', None, subprocess.CalledProcessError, True), + # Command not found + ('cloudbuild_err_not_found.yaml', None, subprocess.CalledProcessError, + True), + # Cleaning up files owned by root + ('cloudbuild_difficult_cleanup.yaml', None, None, True), +]) +def test_local_cloudbuild(testdata_dir, tmpdir, config_name, substitutions, + exception, cleanup): + if not have_docker(): + pytest.fail('This test requires a working Docker daemon') + + check_call_output = None + + def check_call(*args, **kw_args): + """Act like subprocess.check_call but store stdout""" + nonlocal check_call_output try: - self.check_call_output = subprocess.check_output(*args, **kw_args) - print(self.check_call_output) + check_call_output = subprocess.check_output(*args, **kw_args) + print(check_call_output) except subprocess.CalledProcessError as e: - self.check_call_output = e.output - print(self.check_call_output) + check_call_output = e.output + print(check_call_output) raise - def have_docker(self): - """Determine if the Docker daemon is present and usable""" - if ((shutil.which('docker') is not None) and - (subprocess.call(['docker', 'info'], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL) == 0)): - return True - return False - - def test_get_cloudbuild(self): + # Read cloudbuild.yaml from testdata file, write output to + # tempdir, and maybe try to run it + with unittest.mock.patch('subprocess.check_call', check_call): + if substitutions is None: + substitutions = local_cloudbuild.DEFAULT_SUBSTITUTIONS + should_succeed = (exception is None) + config = os.path.join(testdata_dir, config_name) + actual_output_script = tmpdir.join(config_name + '_local.sh') args = argparse.Namespace( - config='some_config_file', - output_script='some_output_script', - run=False, - substitutions={}, - ) - # Basic valid case - valid_case = 'steps:\n- name: step1\n- name: step2\n' - raw_config = yaml.safe_load(valid_case) - actual = local_cloudbuild.get_cloudbuild(raw_config, args) - self.assertEqual(len(actual.steps), 2) - - invalid_cases = ( - # Empty cloud build - '', - # No steps - 'foo: bar\n', - # Steps not a list - 'steps: astring\n', - ) - for invalid_case in invalid_cases: - with self.subTest(invalid_case=invalid_case): - raw_config = yaml.safe_load(invalid_case) - with self.assertRaises(ValueError): - local_cloudbuild.get_cloudbuild(raw_config, args) - - def test_get_step(self): - valid_cases = ( - # Empty step - ({}, local_cloudbuild.Step( - args=[], - dir_='', - env=[], - name='', - )), - # Full step - ({'name' : 'aname', - 'args' : [ 'arg1', 2, 'arg3 with \n newline', ], - 'env' : [ 'ENV1=value1', 'ENV2=space in value2' ], - 'dir' : 'adir', - }, local_cloudbuild.Step( - args = [ 'arg1', '2', 'arg3 with \n newline', ], - env = [ 'ENV1=value1', 'ENV2=space in value2' ], - dir_ = 'adir', - name = 'aname', - )), + config=config, + output_script=str(actual_output_script), + run=True, + substitutions=substitutions, ) - for valid_case in valid_cases: - with self.subTest(valid_case=valid_case): - raw_step, expected = valid_case - actual = local_cloudbuild.get_step(raw_step) - self.assertEqual(actual, expected) - - invalid_cases = ( - # Wrong type - [], - # More wrong types - {'args': 'not_a_list'}, - {'args': [ [] ]}, - {'env': 'not_a_list'}, - {'env': [ {} ]}, - {'dir': {}}, - {'name': []}, - ) - for invalid_case in invalid_cases: - with self.subTest(invalid_case=invalid_case): - with self.assertRaises(ValueError): - local_cloudbuild.get_step(invalid_case) - - def test_generate_command(self): - # Basic valid case - base_step = local_cloudbuild.Step( - args = ['arg1','arg2'], - dir_ = '', - env = ['ENV1=value1', 'ENV2=value2'], - name = 'aname', - ) - subs = {'BUILTIN':'builtin', '_USER':'_user'} - command = local_cloudbuild.generate_command(base_step, subs, set()) - self.assertEqual(command, [ - 'docker', - 'run', - '--volume', - '/var/run/docker.sock:/var/run/docker.sock', - '--volume', - '/root/.docker:/root/.docker', - '--volume', - '${HOST_WORKSPACE}:/workspace', - '--workdir', - '/workspace', - '--env', - 'ENV1=value1', - '--env', - 'ENV2=value2', - 'aname', - 'arg1', - 'arg2', - ]) - - valid_cases = ( - # dir specified - (base_step._replace(dir_='adir'), - ['--workdir', '/workspace/adir']), - # Shell quoting - (base_step._replace(args=['arg with \n newline']), - ["'arg with \n newline'"]), - (base_step._replace(dir_='dir/ with space/'), - ["/workspace/'dir/ with space/'"]), - (base_step._replace(env=['env with space']), - ["'env with space'"]), - (base_step._replace(name='a name'), - ["'a name'"]), - # Variable substitution - (base_step._replace(name='a $BUILTIN substitution'), - ["'a builtin substitution'"]), - (base_step._replace(name='a $_USER substitution'), - ["'a _user substitution'"]), - (base_step._replace(name='a curly brace ${BUILTIN} substitution'), - ["'a curly brace builtin substitution'"]), - (base_step._replace(name='an escaped $$ or $$$$ or $$FOO or $${_FOO} is unescaped'), - ["'an escaped $ or $$ or $FOO or ${_FOO} is unescaped'"]), - ) - for valid_case in valid_cases: - with self.subTest(valid_case=valid_case): - step, args = valid_case - command = local_cloudbuild.generate_command(step, subs, set()) - for arg in args: - self.assertIn(arg, command) - - invalid_cases = ( - base_step._replace(name='a $UNSET_BUILTIN substitution'), - base_step._replace(name='a $_UNSET_USER substitution'), - ) - for invalid_case in invalid_cases: - with self.subTest(invalid_case=invalid_case): - step = invalid_case - with self.assertRaises(ValueError): - local_cloudbuild.generate_command(step, subs, set()) - - def test_generate_script_golden(self): - config_name = 'cloudbuild_ok.yaml' - config = os.path.join(self.testdata_dir, config_name) - expected_output_script = os.path.join(self.testdata_dir, config_name + '_golden.sh') - cloudbuild = local_cloudbuild.CloudBuild( - output_script='test_generate_script', - run=False, - steps=[ - local_cloudbuild.Step( - args=['/bin/sh', '-c', 'printenv MESSAGE'], - dir_='', - env=['MESSAGE=Hello World!'], - name='debian', - ), - local_cloudbuild.Step( - args=['/bin/sh', '-c', 'printenv MESSAGE'], - dir_='', - env=['MESSAGE=Goodbye\\n And Farewell!', 'UNUSED=unused'], - name='debian', - ) - ], - substitutions=local_cloudbuild.DEFAULT_SUBSTITUTIONS, - ) - actual = local_cloudbuild.generate_script(cloudbuild) - self.maxDiff = 2**16 - # Compare output against golden - with open(expected_output_script, 'r', encoding='utf8') as expected: - self.assertEqual(actual, expected.read()) - - def test_generate_script_unused_user_substitution(self): - cloudbuild = local_cloudbuild.CloudBuild( - output_script='', - run=False, - steps=[], - substitutions={'_FOO':'_foo'}, - ) - with self.assertRaisesRegex(ValueError, 'User substitution variables'): - actual = local_cloudbuild.generate_script(cloudbuild) - - def test_make_executable(self): - with tempfile.TemporaryDirectory( - prefix='local_cloudbuild_test_') as tempdir: - test_script_filename = os.path.join(tempdir, 'test_make_executable.sh') - with open(test_script_filename, 'w', encoding='utf8') as test_script: - test_script.write('#!/bin/sh\necho "Output from test_make_executable"') - local_cloudbuild.make_executable(test_script_filename) - output = subprocess.check_output([test_script_filename]) - self.assertEqual(output.decode('utf8'), "Output from test_make_executable\n") - - def test_write_script(self): - with tempfile.TemporaryDirectory( - prefix='local_cloudbuild_test_') as tempdir: - contents = 'The contents\n' - output_script_filename = os.path.join(tempdir, 'test_write_script') - cloudbuild = local_cloudbuild.CloudBuild( - output_script=output_script_filename, - run=False, - steps=[], - substitutions={}, - ) - local_cloudbuild.write_script(cloudbuild, contents) - with open(output_script_filename, 'r', encoding='utf8') as output_script: - actual = output_script.read() - self.assertEqual(actual, contents) - - def test_local_cloudbuild(self): - if not self.have_docker(): - self.fail('This test requires a working Docker daemon') - - # Read cloudbuild.yaml from testdata file, write output to - # tempdir, and maybe try to run it - cases = ( - # Everything is ok - ('cloudbuild_ok.yaml', None, None), - # Builtin substitutions like $PROJECT_ID work - ('cloudbuild_builtin_substitutions.yaml', None, None), - # User substitutions like $_FOO work - ('cloudbuild_user_substitutions.yaml', - {'_FOO':'this is foo value'}, - None - ), - # User substitutions like $_FOO fails when undefined - ('cloudbuild_user_substitutions.yaml', None, ValueError), - # Exit code 1 (failure) - ('cloudbuild_err_rc1.yaml', None, subprocess.CalledProcessError), - # Command not found - ('cloudbuild_err_not_found.yaml', None, subprocess.CalledProcessError), - # Cleaning up files owned by root - ('cloudbuild_difficult_cleanup.yaml', None, None), - ) - for case in cases: - with self.subTest(case=case), \ - tempfile.TemporaryDirectory(prefix='local_cloudbuild_test_') as tempdir, \ - unittest.mock.patch('subprocess.check_call', self.check_call_with_capture): - config_name, substitutions, exception = case - if substitutions is None: - substitutions = local_cloudbuild.DEFAULT_SUBSTITUTIONS - should_succeed = (exception is None) - config = os.path.join(self.testdata_dir, config_name) - actual_output_script = os.path.join( - tempdir, config_name + '_local.sh') - args = argparse.Namespace( - config=config, - output_script=actual_output_script, - run=True, - substitutions=substitutions, - ) - - if should_succeed: + + # The source directory of the build is currently hardcoded as + # '.', so we must chdir there. + with chdir(testdata_dir): + if should_succeed: + local_cloudbuild.local_cloudbuild(args) + else: + with pytest.raises(exception): local_cloudbuild.local_cloudbuild(args) - else: - with self.assertRaises(exception): - local_cloudbuild.local_cloudbuild(args) - - # Check that staging dir was cleaned up - match = re.search(STAGING_DIR_REGEX, self.check_call_output) - self.assertTrue(match) - staging_dir = match.group(1) - self.assertFalse(os.path.isdir(staging_dir), staging_dir) - - def test_parse_args(self): - # Test explicit output_script - argv = ['argv0', '--output_script=my_output'] - args = local_cloudbuild.parse_args(argv) - self.assertEqual(args.output_script, 'my_output') - # Test implicit output_script - argv = ['argv0', '--config=my_config'] - args = local_cloudbuild.parse_args(argv) - self.assertEqual(args.output_script, 'my_config_local.sh') - - # Test run flag (default and --no-run) - argv = ['argv0'] - args = local_cloudbuild.parse_args(argv) - self.assertEqual(args.run, True) - argv = ['argv0', '--no-run'] - args = local_cloudbuild.parse_args(argv) - self.assertEqual(args.run, False) - - -if __name__ == '__main__': - unittest.main() + + # Check that staging dir was cleaned up + if cleanup: + assert check_call_output is not None + match = re.search(STAGING_DIR_REGEX, check_call_output) + assert match + staging_dir = match.group(1) + assert not os.path.isdir(staging_dir) + + +@pytest.mark.parametrize('argv, expected', [ + # Test explicit output_script + (['argv0', '--output_script=my_output'], 'my_output'), + # Test implicit output_script + (['argv0', '--config=my_config'], 'my_config_local.sh'), +]) +def test_parse_args_output_script(argv, expected): + args = local_cloudbuild.parse_args(argv) + assert args.output_script == expected + + +@pytest.mark.parametrize('argv, expected', [ + # Test run flag (default) + (['argv0'], True), + (['argv0', '--no-run'], False), +]) +def test_parse_args_run_flag(argv, expected): + args = local_cloudbuild.parse_args(argv) + assert args.run == expected From 83380501fcab435638cccc2446207f5f1505e916 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 23 Jun 2017 21:18:25 -0700 Subject: [PATCH 195/362] Fix style violations --- scripts/gen_dockerfile.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/scripts/gen_dockerfile.py b/scripts/gen_dockerfile.py index 441fb153..c9632b04 100755 --- a/scripts/gen_dockerfile.py +++ b/scripts/gen_dockerfile.py @@ -83,7 +83,7 @@ def get_app_config(raw_config, base_image, config_file, source_dir): Args: raw_config (dict): deserialized app.yaml base_image (str): Docker image name to build on top of - config_file (str): Path to user's app.yaml (might be .yaml) + config_file (str): Path to user's app.yaml (might be .yaml) source_dir (str): Directory container user's source code Returns: @@ -95,7 +95,8 @@ def get_app_config(raw_config, base_image, config_file, source_dir): 'Expected {} contents to be a Mapping type, but found type "{}"'. format(config_file, type(raw_config))) - entrypoint = validation_utils.get_field_value(raw_config, 'entrypoint', str) + entrypoint = validation_utils.get_field_value( + raw_config, 'entrypoint', str) if not PRINTABLE_REGEX.match(entrypoint): raise ValueError( 'Invalid "entrypoint" value in app.yaml: {!r}'.format(entrypoint)) @@ -108,8 +109,10 @@ def get_app_config(raw_config, base_image, config_file, source_dir): if entrypoint and not entrypoint.startswith('exec '): entrypoint = 'exec ' + entrypoint - raw_runtime_config = validation_utils.get_field_value(raw_config, 'runtime_config', dict) - python_version = validation_utils.get_field_value(raw_runtime_config, 'python_version', str) + raw_runtime_config = validation_utils.get_field_value( + raw_config, 'runtime_config', dict) + python_version = validation_utils.get_field_value( + raw_runtime_config, 'python_version', str) dockerfile_python_version = PYTHON_INTERPRETER_VERSION_MAP.get( python_version) @@ -164,8 +167,9 @@ def generate_files(app_config): optional_requirements_txt = '' if app_config.entrypoint: - optional_entrypoint = get_data('Dockerfile.entrypoint.template').format( - entrypoint=app_config.entrypoint) + optional_entrypoint = get_data( + 'Dockerfile.entrypoint.template').format( + entrypoint=app_config.entrypoint) else: optional_entrypoint = '' @@ -176,8 +180,8 @@ def generate_files(app_config): python_version=app_config.dockerfile_python_version), optional_requirements_txt, get_data('Dockerfile.install_app'), - optional_entrypoint , - ]) + optional_entrypoint, + ]) return { 'Dockerfile': dockerfile, @@ -190,7 +194,7 @@ def generate_dockerfile_command(base_image, config_file, source_dir): Args: base_image (str): Docker image name to build on top of - config_file (str): Path to user's app.yaml (might be .yaml) + config_file (str): Path to user's app.yaml (might be .yaml) source_dir (str): Directory container user's source code """ # Read yaml file. Does not currently support multiple services From 7675e2e2a11c330ab3afd4850dcbe43cb6f4cc9d Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 23 Jun 2017 21:22:56 -0700 Subject: [PATCH 196/362] Fix style violations --- scripts/local_cloudbuild.py | 19 ++++++++++--------- scripts/testdata/cloudbuild_ok.yaml_golden.sh | 2 +- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/scripts/local_cloudbuild.py b/scripts/local_cloudbuild.py index ab02d83a..f80e0f4b 100755 --- a/scripts/local_cloudbuild.py +++ b/scripts/local_cloudbuild.py @@ -73,7 +73,7 @@ } # Use this image for cleanup actions -DEBIAN_IMAGE='gcr.io/google-appengine/debian8' +DEBIAN_IMAGE = 'gcr.io/google-appengine/debian8' # File template BUILD_SCRIPT_TEMPLATE = """\ @@ -90,7 +90,7 @@ if [ "${{HOST_WORKSPACE}}" != '/' -a -d "${{HOST_WORKSPACE}}" ]; then # Expect a single error message about /workspace busy {cleanup_str} 2>/dev/null || true - # Do not expect error messages here. Display but ignore any that happen. + # Do not expect error messages here. Display but ignore. rmdir "${{HOST_WORKSPACE}}" || true fi }} @@ -108,7 +108,8 @@ # Validated cloudbuild recipe + flags -CloudBuild = collections.namedtuple('CloudBuild', 'output_script run steps substitutions') +CloudBuild = collections.namedtuple('CloudBuild', + 'output_script run steps substitutions') # Single validated step in a cloudbuild recipe Step = collections.namedtuple('Step', 'args dir_ env name') @@ -136,8 +137,7 @@ def sub(match): # Variables must be set raise ValueError( 'Variable "{}" used without being defined. Try adding ' - 'it to the --substitutions flag'.format( - variable_name)) + 'it to the --substitutions flag'.format(variable_name)) else: value = substitutions.get(variable_name) substitutions_used.add(variable_name) @@ -272,8 +272,9 @@ def generate_script(cloudbuild): if user_subs_unused: nice_list = '"' + '", "'.join(sorted(user_subs_unused)) + '"' raise ValueError( - 'User substitution variables {} were defined in the --substitution ' - 'flag but never used in the cloudbuild file.'.format(nice_list)) + 'User substitution variables {} were defined in the ' + '--substitution flag but never used in the cloudbuild file.'. + format(nice_list)) cleanup_str = ' '.join(cleanup_command) docker_lines = [] @@ -298,7 +299,7 @@ def make_executable(path): def write_script(cloudbuild, contents): """Write a shell script to a file.""" print('Writing build script to {}'.format(cloudbuild.output_script)) - with open(cloudbuild.output_script, 'w', encoding='utf8') as outfile: + with io.open(cloudbuild.output_script, 'w', encoding='utf8') as outfile: outfile.write(contents) make_executable(cloudbuild.output_script) @@ -310,7 +311,7 @@ def local_cloudbuild(args): args: command line flags as per parse_args """ # Load and parse cloudbuild.yaml - with open(args.config, 'r', encoding='utf8') as cloudbuild_file: + with io.open(args.config, 'r', encoding='utf8') as cloudbuild_file: raw_config = yaml.safe_load(cloudbuild_file) # Determine configuration diff --git a/scripts/testdata/cloudbuild_ok.yaml_golden.sh b/scripts/testdata/cloudbuild_ok.yaml_golden.sh index ba4a52e1..bda52bd2 100755 --- a/scripts/testdata/cloudbuild_ok.yaml_golden.sh +++ b/scripts/testdata/cloudbuild_ok.yaml_golden.sh @@ -11,7 +11,7 @@ function cleanup { if [ "${HOST_WORKSPACE}" != '/' -a -d "${HOST_WORKSPACE}" ]; then # Expect a single error message about /workspace busy docker run --volume /var/run/docker.sock:/var/run/docker.sock --volume /root/.docker:/root/.docker --volume ${HOST_WORKSPACE}:/workspace --workdir /workspace gcr.io/google-appengine/debian8 rm -rf /workspace 2>/dev/null || true - # Do not expect error messages here. Display but ignore any that happen. + # Do not expect error messages here. Display but ignore. rmdir "${HOST_WORKSPACE}" || true fi } From 66a1a2b812c3637c99e26ca7b9fd1c0b7347e69f Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 23 Jun 2017 21:25:00 -0700 Subject: [PATCH 197/362] Convert tests to pytest and fix style violations --- scripts/gen_dockerfile_test.py | 382 +++++++++++++++---------------- scripts/validation_utils.py | 4 +- scripts/validation_utils_test.py | 142 +++++------- 3 files changed, 252 insertions(+), 276 deletions(-) diff --git a/scripts/gen_dockerfile_test.py b/scripts/gen_dockerfile_test.py index 8169b297..09194eb8 100755 --- a/scripts/gen_dockerfile_test.py +++ b/scripts/gen_dockerfile_test.py @@ -19,210 +19,202 @@ import argparse import filecmp import os -import re import shutil import subprocess -import tempfile -import unittest import unittest.mock +import pytest import yaml import gen_dockerfile + # Expected list of files generated EXPECTED_OUTPUT_FILES = ['Dockerfile', '.dockerignore'] -class GenDockerfileTest(unittest.TestCase): - def setUp(self): - self.testdata_dir = os.path.join(os.path.dirname(__file__), 'testdata') - assert os.path.isdir(self.testdata_dir), 'Could not run test: testdata directory not found' - - def compare_file(self, filename, dir1, dir2): - """Compare identically named files in two different directories""" - if not filecmp.cmp(os.path.join(dir1, filename), - os.path.join(dir2, filename)): - msg = 'Contents of "{}" differ between "{}" and "{}"'.format( - filename, dir1, dir2) - self.assertMultiLineEqual(contents1, contents2, msg) - - def test_get_app_config(self): - config_file = 'some_config_file' - base_image = 'some_image_name' - source_dir = 'some_source_dir' - - valid_cases = ( - # Basic app.yaml - ('env: flex', False, { - 'base_image': 'some_image_name', - 'dockerfile_python_version': '', - 'has_requirements_txt': False, - 'entrypoint': '', - }), - # All supported python versions - ('runtime_config:\n python_version:', False, { - 'dockerfile_python_version': '', - }), - ('runtime_config:\n python_version: 2', False, { - 'dockerfile_python_version': '', - }), - ('runtime_config:\n python_version: 3', False, { - 'dockerfile_python_version': '3.5', - }), - ('runtime_config:\n python_version: 3.4', False, { - 'dockerfile_python_version': '3.4', - }), - ('runtime_config:\n python_version: 3.5', False, { - 'dockerfile_python_version': '3.5', - }), - # requirements.txt present - ('env: flex', True, { - 'has_requirements_txt': True, - }), - # entrypoint present - ('entrypoint: my entrypoint', False, { - 'entrypoint': 'exec my entrypoint', - }), - ) - for valid_case in valid_cases: - with self.subTest(valid_case=valid_case): - app_yaml, isfile, expected = valid_case - raw_app_config = yaml.safe_load(app_yaml) - with unittest.mock.patch.object( - os.path, 'isfile', return_value=isfile): - actual = gen_dockerfile.get_app_config( - raw_app_config, base_image, config_file, - source_dir) - for key, value in expected.items(): - self.assertEqual(getattr(actual, key), value) - - invalid_cases = ( - # Empty app.yaml - '', - # Invalid entrypoint - 'entrypoint: "bad \\n entrypoint"', - # Invalid python version - 'runtime_config:\n python_version: 1', - 'runtime_config:\n python_version: python2', - ) - for invalid_case in invalid_cases: - with self.subTest(invalid_case=invalid_case): - raw_app_config = yaml.safe_load(invalid_case) - with self.assertRaises(ValueError): - gen_dockerfile.get_app_config( - raw_app_config, base_image, config_file, - source_dir) - - def test_generate_files(self): - base = gen_dockerfile.AppConfig( - base_image='', - dockerfile_python_version='', - entrypoint='', - has_requirements_txt=False - ) - cases = ( - # Requirements.txt - (base, False, 'ADD requirements.txt'), - (base._replace(has_requirements_txt=True), True, - 'ADD requirements.txt'), - # Entrypoint - (base, False, 'CMD'), - (base._replace(entrypoint='my entrypoint'), True, - 'CMD my entrypoint'), - (base._replace(entrypoint='exec my entrypoint'), True, - 'CMD exec my entrypoint'), - # Base runtime image - (base._replace(base_image='my_base_runtime_image'), True, - 'FROM my_base_runtime_image'), - # Python version - (base._replace(dockerfile_python_version='_my_version'), True, - 'python_version=python_my_version'), - ) - for case in cases: - with self.subTest(case=case): - app_config, should_find, test_string = case - result = gen_dockerfile.generate_files(app_config) - self.assertEqual( - sorted(result.keys()), sorted(EXPECTED_OUTPUT_FILES)) - dockerfile = result['Dockerfile'] - if should_find: - self.assertIn(test_string, dockerfile) - else: - self.assertNotIn(test_string, dockerfile) - - def test_generate_dockerfile_command(self): - """Generates output and compares against a set of golden files. - - Optionally runs 'gcloud app gen-config' and compares against that. - """ - # Sample app from https://github.com/GoogleCloudPlatform/python-docs-samples - with tempfile.TemporaryDirectory( - prefix='gen_dockerfile_test_') as parent_tempdir: - for app in ['hello_world']: - app_dir = os.path.join(self.testdata_dir, app) - temp_dir = os.path.join(parent_tempdir, app) - os.mkdir(temp_dir) - - # Copy sample app to writable temp dir, and generate Dockerfile. - config_dir = os.path.join(temp_dir, 'config') - shutil.copytree(app_dir, config_dir) - gen_dockerfile.generate_dockerfile_command( - base_image='gcr.io/google-appengine/python', - config_file=os.path.join(config_dir, 'app.yaml'), - source_dir=config_dir) - - # Compare against golden files - golden_dir = os.path.join(self.testdata_dir, app + '_golden') - for filename in EXPECTED_OUTPUT_FILES: - with self.subTest(source='golden', filename=filename): - self.compare_file(filename, config_dir, golden_dir) - - # Copy sample app to different writable temp dir, and - # generate Dockerfile using gcloud. - if not shutil.which('gcloud'): - self.skipTest( - '"gcloud" tool not found in $PATH, skipping test') - gen_config_dir = os.path.join(temp_dir, 'gen_config') - shutil.copytree(app_dir, gen_config_dir) - app_yaml = os.path.join(gen_config_dir, 'app.yaml') - gcloud_args = [ - 'gcloud', '--quiet', 'beta', 'app', 'gen-config', - gen_config_dir, '--custom', '--config={}'.format(app_yaml) - ] - print('Invoking gcloud as {}'.format(gcloud_args)) - subprocess.check_call(gcloud_args) - for filename in EXPECTED_OUTPUT_FILES: - with self.subTest(source='gcloud', filename=filename): - self.compare_file(filename, config_dir, gen_config_dir) - - def test_parse_args(self): - valid_cases = ( - [], - ['argv0', '--base-image=nocolon'], - ['argv0', '--base-image=name:andcolon'], - ['argv0', '--base-image=name@sha256:digest'], - ) - for argv in valid_cases: - with self.subTest(valid_argv=argv): - args = gen_dockerfile.parse_args(argv) - - def mock_error(*args): - """Prevent argparse from calling sys.exit()""" - raise AssertionError(*args) - - invalid_cases = ( - ['argv0', '--base-image='], - ['argv0', '--base-image=:'], - ['argv0', '--base-image=:noname'], - ) - for argv in invalid_cases: - with self.subTest(invalid_argv=argv): - with unittest.mock.patch.object( - argparse.ArgumentParser, 'error', mock_error): - with self.assertRaises(AssertionError): - gen_dockerfile.parse_args(argv) - - -if __name__ == '__main__': - unittest.main() +@pytest.fixture +def testdata_dir(): + testdata_dir = os.path.join(os.path.dirname(__file__), 'testdata') + assert os.path.isdir(testdata_dir), ( + 'Could not run test: testdata directory not found') + return testdata_dir + + +def compare_file(filename, dir1, dir2): + """Compare identically named files in two different directories""" + assert filecmp.cmp( + os.path.join(dir1, filename), os.path.join(dir2, filename)) + + +@pytest.mark.parametrize('app_yaml, isfile, expected', [ + # Basic app.yaml + ('env: flex', False, { + 'base_image': 'some_image_name', + 'dockerfile_python_version': '', + 'has_requirements_txt': False, + 'entrypoint': '', + }), + # All supported python versions + ('runtime_config:\n python_version:', False, { + 'dockerfile_python_version': '', + }), + ('runtime_config:\n python_version: 2', False, { + 'dockerfile_python_version': '', + }), + ('runtime_config:\n python_version: 3', False, { + 'dockerfile_python_version': '3.5', + }), + ('runtime_config:\n python_version: 3.4', False, { + 'dockerfile_python_version': '3.4', + }), + ('runtime_config:\n python_version: 3.5', False, { + 'dockerfile_python_version': '3.5', + }), + # requirements.txt present + ('env: flex', True, { + 'has_requirements_txt': True, + }), + # entrypoint present + ('entrypoint: my entrypoint', False, { + 'entrypoint': 'exec my entrypoint', + }), +]) +def test_get_app_config_valid(app_yaml, isfile, expected): + config_file = 'some_config_file' + base_image = 'some_image_name' + source_dir = 'some_source_dir' + raw_app_config = yaml.safe_load(app_yaml) + with unittest.mock.patch.object(os.path, 'isfile', return_value=isfile): + actual = gen_dockerfile.get_app_config( + raw_app_config, base_image, config_file, + source_dir) + for key, value in expected.items(): + assert getattr(actual, key) == value + + +@pytest.mark.parametrize('app_yaml', [ + # Empty app.yaml + '', + # Invalid entrypoint + 'entrypoint: "bad \\n entrypoint"', + # Invalid python version + 'runtime_config:\n python_version: 1', + 'runtime_config:\n python_version: python2', +]) +def test_get_app_config_invalid(app_yaml): + config_file = 'some_config_file' + base_image = 'some_image_name' + source_dir = 'some_source_dir' + raw_app_config = yaml.safe_load(app_yaml) + with pytest.raises(ValueError): + gen_dockerfile.get_app_config( + raw_app_config, base_image, config_file, source_dir) + + +# Basic AppConfig used below +_base = gen_dockerfile.AppConfig( + base_image='', + dockerfile_python_version='', + entrypoint='', + has_requirements_txt=False +) + + +@pytest.mark.parametrize('app_config, should_find, test_string', [ + # Requirements.txt + (_base, False, 'ADD requirements.txt'), + (_base._replace(has_requirements_txt=True), True, + 'ADD requirements.txt'), + # Entrypoint + (_base, False, 'CMD'), + (_base._replace(entrypoint='my entrypoint'), True, + 'CMD my entrypoint'), + (_base._replace(entrypoint='exec my entrypoint'), True, + 'CMD exec my entrypoint'), + # Base runtime image + (_base._replace(base_image='my_base_runtime_image'), True, + 'FROM my_base_runtime_image'), + # Python version + (_base._replace(dockerfile_python_version='_my_version'), True, + 'python_version=python_my_version'), +]) +def test_generate_files(app_config, should_find, test_string): + result = gen_dockerfile.generate_files(app_config) + assert sorted(result.keys()) == sorted(EXPECTED_OUTPUT_FILES) + dockerfile = result['Dockerfile'] + if should_find: + assert test_string in dockerfile + else: + assert test_string not in dockerfile + + +@pytest.mark.parametrize('app', [ + # Sample from https://github.com/GoogleCloudPlatform/python-docs-samples + 'hello_world', +]) +def test_generate_dockerfile_command(tmpdir, testdata_dir, app): + """Generates output and compares against a set of golden files. + + Optionally runs 'gcloud app gen-config' and compares against that. + """ + app_dir = os.path.join(testdata_dir, app) + temp_dir = os.path.join(str(tmpdir), app) + os.mkdir(temp_dir) + + # Copy sample app to writable temp dir, and generate Dockerfile. + config_dir = os.path.join(temp_dir, 'config') + shutil.copytree(app_dir, config_dir) + gen_dockerfile.generate_dockerfile_command( + base_image='gcr.io/google-appengine/python', + config_file=os.path.join(config_dir, 'app.yaml'), + source_dir=config_dir) + + # Compare against golden files + golden_dir = os.path.join(testdata_dir, app + '_golden') + for filename in EXPECTED_OUTPUT_FILES: + compare_file(filename, config_dir, golden_dir) + + # Copy sample app to different writable temp dir, and + # generate Dockerfile using gcloud. + if not shutil.which('gcloud'): + print('"gcloud" tool not found in $PATH, skipping rest of test') + return + gen_config_dir = os.path.join(temp_dir, 'gen_config') + shutil.copytree(app_dir, gen_config_dir) + app_yaml = os.path.join(gen_config_dir, 'app.yaml') + gcloud_args = [ + 'gcloud', '--quiet', 'beta', 'app', 'gen-config', + gen_config_dir, '--custom', '--config={}'.format(app_yaml) + ] + print('Invoking gcloud as {}'.format(gcloud_args)) + subprocess.check_call(gcloud_args) + for filename in EXPECTED_OUTPUT_FILES: + compare_file(filename, config_dir, gen_config_dir) + + +@pytest.mark.parametrize('argv', [ + [], + ['argv0', '--base-image=nocolon'], + ['argv0', '--base-image=name:andcolon'], + ['argv0', '--base-image=name@sha256:digest'], +]) +def test_parse_args_valid(argv): + args = gen_dockerfile.parse_args(argv) + assert args is not None + + +@pytest.mark.parametrize('argv', [ + ['argv0', '--base-image='], + ['argv0', '--base-image=:'], + ['argv0', '--base-image=:noname'], +]) +def test_parse_args_invalid(argv): + def mock_error(*args): + """Prevent argparse from calling sys.exit()""" + raise AssertionError(*args) + + with unittest.mock.patch.object(argparse.ArgumentParser, 'error', + mock_error): + with pytest.raises(AssertionError): + gen_dockerfile.parse_args(argv) diff --git a/scripts/validation_utils.py b/scripts/validation_utils.py index 2792dcb1..28de2d52 100644 --- a/scripts/validation_utils.py +++ b/scripts/validation_utils.py @@ -101,7 +101,7 @@ def validate_arg_dict(flag_value): match = re.match(KEY_VALUE_REGEX, entry) if not match: raise argparse.ArgumentTypeError( - 'Value "{}" should be a list like _KEY1=value1,_KEY2=value2"'.format( - flag_value)) + 'Value "{}" should be a list like _KEY1=value1,_KEY2=value2"'. + format(flag_value)) pairs.append((match.group(1), match.group(2))) return dict(pairs) diff --git a/scripts/validation_utils_test.py b/scripts/validation_utils_test.py index 9ef87096..ff853e4e 100755 --- a/scripts/validation_utils_test.py +++ b/scripts/validation_utils_test.py @@ -18,93 +18,77 @@ import argparse import re -import unittest + +import pytest import validation_utils -class ValidationUtilsTest(unittest.TestCase): +@pytest.mark.parametrize('container, field_name, field_type, expected', [ + # Normal case, field present and correct type + ({'present': 1}, 'present', int, 1), + ({'present': '1'}, 'present', str, '1'), + ({'present': [1]}, 'present', list, [1]), + ({'present': {1: 2}}, 'present', dict, {1: 2}), + # Missing field replaced by default + ({}, 'missing', str, ''), + # Valid conversions + ({'str_to_int': '1'}, 'str_to_int', int, 1), + ({'int_to_str': 1}, 'int_to_str', str, '1'), + # None + ({'None_to_int': None}, 'None_to_int', int, 0), + ({'None_to_str': None}, 'None_to_str', str, ''), +]) +def test_get_field_value_valid(container, field_name, field_type, expected): + assert validation_utils.get_field_value( + container, field_name, field_type) == expected + - def test_get_field_value(self): - valid_cases = ( - # Normal case, field present and correct type - ({ 'present': 1 }, 'present', int, 1), - ({ 'present': '1' }, 'present', str, '1'), - ({ 'present': [1] }, 'present', list, [1]), - ({ 'present': {1: 2} }, 'present', dict, {1: 2}), - # Missing field replaced by default - ({}, 'missing', str, ''), - # Valid conversions - ({ 'str_to_int': '1' }, 'str_to_int', int, 1), - ({ 'int_to_str': 1 }, 'int_to_str', str, '1'), - # None - ({ 'None_to_int': None }, 'None_to_int', int, 0), - ({ 'None_to_str': None }, 'None_to_str', str, ''), - ) - for valid_case in valid_cases: - with self.subTest(valid_case=valid_case): - container, field_name, field_type, expected = valid_case - self.assertEqual( - validation_utils.get_field_value( - container, field_name, field_type), - expected) +@pytest.mark.parametrize('container, field_name, field_type', [ + # Type conversion failures + ({'bad_list_to_dict': [1]}, 'bad_list_to_dict', dict), + ({'bad_list_to_str': [1]}, 'bad_list_to_str', str), + ({'bad_dict_to_list': {1: 2}}, 'bad_dict_to_list', list), + ({'bad_str_to_int': 'not_an_int'}, 'bad_str_to_int', int), + ({'bad_str_to_list': 'abc'}, 'bad_str_to_list', list), +]) +def test_get_field_value_invalid(container, field_name, field_type): + with pytest.raises(ValueError): + validation_utils.get_field_value(container, field_name, field_type) - invalid_cases = ( - # Type conversion failures - ({ 'bad_list_to_dict': [1] }, 'bad_list_to_dict', dict), - ({ 'bad_list_to_str': [1] }, 'bad_list_to_str', str), - ({ 'bad_dict_to_list': {1: 2} }, 'bad_dict_to_list', list), - ({ 'bad_str_to_int': 'not_an_int' }, 'bad_str_to_int', int), - ({ 'bad_str_to_list': 'abc' }, 'bad_str_to_list', list), - ) - for invalid_case in invalid_cases: - with self.subTest(invalid_case=invalid_case): - container, field_name, field_type = invalid_case - with self.assertRaises(ValueError): - validation_utils.get_field_value( - container, field_name, field_type) - def test_validate_arg_regex(self): - self.assertEqual( - validation_utils.validate_arg_regex('abc', re.compile('a[b]c')), - 'abc') - with self.assertRaises(argparse.ArgumentTypeError): - validation_utils.validate_arg_regex('abc', re.compile('a[d]c')) +def test_validate_arg_regex(): + assert validation_utils.validate_arg_regex( + 'abc', re.compile('a[b]c')) == 'abc' + with pytest.raises(argparse.ArgumentTypeError): + validation_utils.validate_arg_regex('abc', re.compile('a[d]c')) - def test_validate_arg_dict(self): - valid_cases = ( - # Normal case, field present and correct type - ('', {}), - ('_A=1', {'_A':'1'}), - ('_A=1,_B=2', {'_A':'1', '_B':'2'}), - # Repeated key is ok - ('_A=1,_A=2', {'_A':'2'}), - # Extra = is ok - ('_A=x=y=z,_B=2', {'_A':'x=y=z', '_B':'2'}), - # No value is ok - ('_A=', {'_A':''}), - ) - for valid_case in valid_cases: - with self.subTest(valid_case=valid_case): - s, expected = valid_case - self.assertEqual( - validation_utils.validate_arg_dict(s), - expected) - invalid_cases = ( - # No key - ',_A', - '_A,', - # Invalid variable name - '_Aa=1', - '_aA=1', - '0A=1', - ) - for invalid_case in invalid_cases: - with self.subTest(invalid_case=invalid_case): - with self.assertRaises(argparse.ArgumentTypeError): - validation_utils.validate_arg_dict(invalid_case) +@pytest.mark.parametrize('arg, expected', [ + # Normal case, field present and correct type + ('', {}), + ('_A=1', {'_A': '1'}), + ('_A=1,_B=2', {'_A': '1', '_B': '2'}), + # Repeated key is ok + ('_A=1,_A=2', {'_A': '2'}), + # Extra = is ok + ('_A=x=y=z,_B=2', {'_A': 'x=y=z', '_B': '2'}), + # No value is ok + ('_A=', {'_A': ''}), +]) +def test_validate_arg_dicts_valid(arg, expected): + assert validation_utils.validate_arg_dict(arg) == expected -if __name__ == '__main__': - unittest.main() +@pytest.mark.parametrize('arg', [ + # No key + ',_A', + '_A,', + # Invalid variable name + '_Aa=1', + '_aA=1', + '0A=1', +]) +def test_validate_arg_dicts_invalid(arg): + with pytest.raises(argparse.ArgumentTypeError): + validation_utils.validate_arg_dict(arg) From d9ff1ae9b452edbf179e19e0ab9afec6857b53f2 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 23 Jun 2017 21:28:28 -0700 Subject: [PATCH 198/362] Use Python3 for nox lint, and make nox.py pass lint --- nox.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/nox.py b/nox.py index fd79945e..08ffd872 100644 --- a/nox.py +++ b/nox.py @@ -17,7 +17,6 @@ import nox - def _list_files(folder, pattern): """Lists all files below the given folder that match the pattern.""" for root, folders, files in os.walk(folder): @@ -44,10 +43,13 @@ def check_requirements(session): @nox.session def lint(session): + session.interpreter = 'python3' session.install('flake8', 'flake8-import-order') session.run( 'flake8', '--import-order-style=google', + ('--application-import-names=' + 'gen_dockerfile,local_cloudbuild,validation_utils'), 'scripts', 'nox.py', ) @@ -64,9 +66,9 @@ def tests(session, version): '--cov=scripts', '--cov-append', '--cov-config=.coveragerc', - '--cov-report=', # Report generated below + '--cov-report=', # Report generated below 'scripts', - env={'PYTHONPATH': ''} + env={'PYTHONPATH':''} ) From d4b6fb1a1ac43faee5d5b51ae259dcc2fbc4255d Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 23 Jun 2017 21:52:47 -0700 Subject: [PATCH 199/362] Make nox.py actually pass lint --- nox.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nox.py b/nox.py index 08ffd872..ab2f0917 100644 --- a/nox.py +++ b/nox.py @@ -17,6 +17,7 @@ import nox + def _list_files(folder, pattern): """Lists all files below the given folder that match the pattern.""" for root, folders, files in os.walk(folder): @@ -66,9 +67,9 @@ def tests(session, version): '--cov=scripts', '--cov-append', '--cov-config=.coveragerc', - '--cov-report=', # Report generated below + '--cov-report=', # Report generated below 'scripts', - env={'PYTHONPATH':''} + env={'PYTHONPATH': ''} ) From 24e4d218f2a21d970f8a178d71ddd720bfe8f4db Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 23 Jun 2017 21:58:07 -0700 Subject: [PATCH 200/362] FULL_BASE_IMAGE -> STAGING_IMAGE to match other Dockerfiles --- builder/gen-dockerfile/Dockerfile.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/gen-dockerfile/Dockerfile.in b/builder/gen-dockerfile/Dockerfile.in index 4a8de5ec..e2fe4463 100644 --- a/builder/gen-dockerfile/Dockerfile.in +++ b/builder/gen-dockerfile/Dockerfile.in @@ -1,4 +1,4 @@ -FROM ${FULL_BASE_IMAGE} +FROM ${STAGING_IMAGE} LABEL python_version=python3.5 RUN virtualenv --no-download /env -p python3.5 From 3d14f06219e61a33a5b1b0957f7dc16ed8e21fbb Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 23 Jun 2017 22:04:36 -0700 Subject: [PATCH 201/362] Fix build.sh file copy --- build.sh | 9 +++++---- builder/gen-dockerfile/.gitignore | 1 + 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/build.sh b/build.sh index ee1d0ddf..943c5b92 100755 --- a/build.sh +++ b/build.sh @@ -125,12 +125,13 @@ for outfile in \ done # Make some files available to the runtime builder Docker context +mkdir -p builder/gen-dockerfile/data for file in \ - gen_dockerfile.py \ - validation_utils.py \ - 'data/*' \ + scripts/gen_dockerfile.py \ + scripts/validation_utils.py \ + scripts/data/* \ ; do - cp -a "scripts/${file}" "builder/gen-dockerfile/${file}" + cp -a "${file}" "builder/gen-dockerfile/${file##scripts/}" done # Build images and push to GCR diff --git a/builder/gen-dockerfile/.gitignore b/builder/gen-dockerfile/.gitignore index d8ffddc3..f45549bd 100644 --- a/builder/gen-dockerfile/.gitignore +++ b/builder/gen-dockerfile/.gitignore @@ -1,2 +1,3 @@ Dockerfile *.py +data/ From 67fbea3b4ed83c95b9b51e6d65f96813bf44f7bd Mon Sep 17 00:00:00 2001 From: Chris Heng Date: Mon, 26 Jun 2017 23:51:20 +0800 Subject: [PATCH 202/362] Replace deprecated "vm: true" in app.yaml example with "env: flex" (#126) This brings it up to date with official documentation: https://cloud.google.com/appengine/docs/flexible/python/runtime --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7c14904e..78a02c5c 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ docker by specifying `runtime: python` in your `app.yaml`: ```yaml runtime: python -vm: true +env: flex entrypoint: gunicorn -b :$PORT main:app runtime_config: From eef60bf009ec89d89e32d47c83d0ad17deb18ec6 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Mon, 26 Jun 2017 10:58:44 -0700 Subject: [PATCH 203/362] Slight tweak to nox.py format --- nox.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nox.py b/nox.py index ab2f0917..8bbe47fa 100644 --- a/nox.py +++ b/nox.py @@ -44,13 +44,13 @@ def check_requirements(session): @nox.session def lint(session): - session.interpreter = 'python3' + session.interpreter = 'python3' # So it understands Python3 syntax session.install('flake8', 'flake8-import-order') session.run( 'flake8', '--import-order-style=google', - ('--application-import-names=' - 'gen_dockerfile,local_cloudbuild,validation_utils'), + '--application-import-names', + 'gen_dockerfile,local_cloudbuild,validation_utils', 'scripts', 'nox.py', ) From 46b18dfb8fa5a4320f1d8ba3a35b9a6df877a5d5 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Mon, 26 Jun 2017 11:10:32 -0700 Subject: [PATCH 204/362] Split one test into two --- scripts/gen_dockerfile_test.py | 43 +++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/scripts/gen_dockerfile_test.py b/scripts/gen_dockerfile_test.py index 09194eb8..78520205 100755 --- a/scripts/gen_dockerfile_test.py +++ b/scripts/gen_dockerfile_test.py @@ -47,50 +47,55 @@ def compare_file(filename, dir1, dir2): os.path.join(dir1, filename), os.path.join(dir2, filename)) -@pytest.mark.parametrize('app_yaml, isfile, expected', [ +@pytest.mark.parametrize('app_yaml, expected', [ # Basic app.yaml - ('env: flex', False, { + ('env: flex', { 'base_image': 'some_image_name', 'dockerfile_python_version': '', 'has_requirements_txt': False, 'entrypoint': '', }), # All supported python versions - ('runtime_config:\n python_version:', False, { + ('runtime_config:\n python_version:', { 'dockerfile_python_version': '', }), - ('runtime_config:\n python_version: 2', False, { + ('runtime_config:\n python_version: 2', { 'dockerfile_python_version': '', }), - ('runtime_config:\n python_version: 3', False, { + ('runtime_config:\n python_version: 3', { 'dockerfile_python_version': '3.5', }), - ('runtime_config:\n python_version: 3.4', False, { + ('runtime_config:\n python_version: 3.4', { 'dockerfile_python_version': '3.4', }), - ('runtime_config:\n python_version: 3.5', False, { + ('runtime_config:\n python_version: 3.5', { 'dockerfile_python_version': '3.5', }), - # requirements.txt present - ('env: flex', True, { - 'has_requirements_txt': True, - }), # entrypoint present - ('entrypoint: my entrypoint', False, { + ('entrypoint: my entrypoint', { 'entrypoint': 'exec my entrypoint', }), ]) -def test_get_app_config_valid(app_yaml, isfile, expected): +def test_get_app_config_valid(app_yaml, expected): config_file = 'some_config_file' base_image = 'some_image_name' source_dir = 'some_source_dir' raw_app_config = yaml.safe_load(app_yaml) - with unittest.mock.patch.object(os.path, 'isfile', return_value=isfile): - actual = gen_dockerfile.get_app_config( - raw_app_config, base_image, config_file, - source_dir) - for key, value in expected.items(): - assert getattr(actual, key) == value + actual = gen_dockerfile.get_app_config( + raw_app_config, base_image, config_file, + source_dir) + for key, value in expected.items(): + assert getattr(actual, key) == value + + +def test_get_app_config_requirements_txt(): + """requirements.txt file present""" + app_yaml = 'env: flex' + expected = { + 'has_requirements_txt': True, + } + with unittest.mock.patch.object(os.path, 'isfile', return_value=True): + test_get_app_config_valid(app_yaml, expected) @pytest.mark.parametrize('app_yaml', [ From d53ac16caa8c583dee84ec2d1c058d89c8c9335d Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Mon, 26 Jun 2017 11:50:31 -0700 Subject: [PATCH 205/362] Minor restructure and tidying of test code --- scripts/gen_dockerfile_test.py | 69 +++++++++++++++++----------------- 1 file changed, 34 insertions(+), 35 deletions(-) diff --git a/scripts/gen_dockerfile_test.py b/scripts/gen_dockerfile_test.py index 78520205..705cd524 100755 --- a/scripts/gen_dockerfile_test.py +++ b/scripts/gen_dockerfile_test.py @@ -30,7 +30,7 @@ # Expected list of files generated -EXPECTED_OUTPUT_FILES = ['Dockerfile', '.dockerignore'] +EXPECTED_OUTPUT_FILES = set(('Dockerfile', '.dockerignore')) @pytest.fixture @@ -118,7 +118,7 @@ def test_get_app_config_invalid(app_yaml): # Basic AppConfig used below -_base = gen_dockerfile.AppConfig( +_BASE_APP_CONFIG = gen_dockerfile.AppConfig( base_image='', dockerfile_python_version='', entrypoint='', @@ -128,25 +128,25 @@ def test_get_app_config_invalid(app_yaml): @pytest.mark.parametrize('app_config, should_find, test_string', [ # Requirements.txt - (_base, False, 'ADD requirements.txt'), - (_base._replace(has_requirements_txt=True), True, + (_BASE_APP_CONFIG, False, 'ADD requirements.txt'), + (_BASE_APP_CONFIG._replace(has_requirements_txt=True), True, 'ADD requirements.txt'), # Entrypoint - (_base, False, 'CMD'), - (_base._replace(entrypoint='my entrypoint'), True, + (_BASE_APP_CONFIG, False, 'CMD'), + (_BASE_APP_CONFIG._replace(entrypoint='my entrypoint'), True, 'CMD my entrypoint'), - (_base._replace(entrypoint='exec my entrypoint'), True, + (_BASE_APP_CONFIG._replace(entrypoint='exec my entrypoint'), True, 'CMD exec my entrypoint'), # Base runtime image - (_base._replace(base_image='my_base_runtime_image'), True, + (_BASE_APP_CONFIG._replace(base_image='my_base_runtime_image'), True, 'FROM my_base_runtime_image'), # Python version - (_base._replace(dockerfile_python_version='_my_version'), True, + (_BASE_APP_CONFIG._replace(dockerfile_python_version='_my_version'), True, 'python_version=python_my_version'), ]) def test_generate_files(app_config, should_find, test_string): result = gen_dockerfile.generate_files(app_config) - assert sorted(result.keys()) == sorted(EXPECTED_OUTPUT_FILES) + assert set(result.keys()) == EXPECTED_OUTPUT_FILES dockerfile = result['Dockerfile'] if should_find: assert test_string in dockerfile @@ -154,38 +154,38 @@ def test_generate_files(app_config, should_find, test_string): assert test_string not in dockerfile -@pytest.mark.parametrize('app', [ - # Sample from https://github.com/GoogleCloudPlatform/python-docs-samples - 'hello_world', -]) -def test_generate_dockerfile_command(tmpdir, testdata_dir, app): - """Generates output and compares against a set of golden files. +def compare_against_golden_files(app, config_dir, testdata_dir): + golden_dir = os.path.join(testdata_dir, app + '_golden') + for filename in EXPECTED_OUTPUT_FILES: + compare_file(filename, config_dir, golden_dir) + - Optionally runs 'gcloud app gen-config' and compares against that. - """ +def test_generate_dockerfile_command(tmpdir, testdata_dir): + """Generates output and compares against a set of golden files.""" + # Sample from https://github.com/GoogleCloudPlatform/python-docs-samples + app = 'hello_world' app_dir = os.path.join(testdata_dir, app) - temp_dir = os.path.join(str(tmpdir), app) - os.mkdir(temp_dir) # Copy sample app to writable temp dir, and generate Dockerfile. - config_dir = os.path.join(temp_dir, 'config') + config_dir = os.path.join(str(tmpdir), 'config') shutil.copytree(app_dir, config_dir) gen_dockerfile.generate_dockerfile_command( base_image='gcr.io/google-appengine/python', config_file=os.path.join(config_dir, 'app.yaml'), source_dir=config_dir) + compare_against_golden_files(app, config_dir, testdata_dir) - # Compare against golden files - golden_dir = os.path.join(testdata_dir, app + '_golden') - for filename in EXPECTED_OUTPUT_FILES: - compare_file(filename, config_dir, golden_dir) - # Copy sample app to different writable temp dir, and - # generate Dockerfile using gcloud. - if not shutil.which('gcloud'): - print('"gcloud" tool not found in $PATH, skipping rest of test') - return - gen_config_dir = os.path.join(temp_dir, 'gen_config') +@pytest.mark.xfail(not shutil.which('gcloud'), + reason='Google Cloud SDK is not installed') +def test_generate_dockerfile_golden(tmpdir, testdata_dir): + """Validate our golden files against gcloud app gen-config""" + # Sample from https://github.com/GoogleCloudPlatform/python-docs-samples + app = 'hello_world' + app_dir = os.path.join(testdata_dir, app) + + # Copy sample app to writable temp dir, and generate Dockerfile. + gen_config_dir = os.path.join(str(tmpdir), 'gen_config') shutil.copytree(app_dir, gen_config_dir) app_yaml = os.path.join(gen_config_dir, 'app.yaml') gcloud_args = [ @@ -194,8 +194,7 @@ def test_generate_dockerfile_command(tmpdir, testdata_dir, app): ] print('Invoking gcloud as {}'.format(gcloud_args)) subprocess.check_call(gcloud_args) - for filename in EXPECTED_OUTPUT_FILES: - compare_file(filename, config_dir, gen_config_dir) + compare_against_golden_files(app, gen_config_dir, testdata_dir) @pytest.mark.parametrize('argv', [ @@ -219,7 +218,7 @@ def mock_error(*args): """Prevent argparse from calling sys.exit()""" raise AssertionError(*args) - with unittest.mock.patch.object(argparse.ArgumentParser, 'error', - mock_error): + with unittest.mock.patch.object( + argparse.ArgumentParser, 'error', mock_error): with pytest.raises(AssertionError): gen_dockerfile.parse_args(argv) From 17191cc674de5983fa91474c32b711a63943a794 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Mon, 26 Jun 2017 11:53:57 -0700 Subject: [PATCH 206/362] Fix nit in nox.py --- nox.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nox.py b/nox.py index 8bbe47fa..872eb88c 100644 --- a/nox.py +++ b/nox.py @@ -48,7 +48,7 @@ def lint(session): session.install('flake8', 'flake8-import-order') session.run( 'flake8', - '--import-order-style=google', + '--import-order-style', 'google', '--application-import-names', 'gen_dockerfile,local_cloudbuild,validation_utils', 'scripts', From d722f0053e59b9436eccd16196477c2289b560a9 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Mon, 26 Jun 2017 11:58:19 -0700 Subject: [PATCH 207/362] Fix lint warning --- scripts/gen_dockerfile_test.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/gen_dockerfile_test.py b/scripts/gen_dockerfile_test.py index 705cd524..046580dd 100755 --- a/scripts/gen_dockerfile_test.py +++ b/scripts/gen_dockerfile_test.py @@ -218,7 +218,8 @@ def mock_error(*args): """Prevent argparse from calling sys.exit()""" raise AssertionError(*args) - with unittest.mock.patch.object( - argparse.ArgumentParser, 'error', mock_error): + error_patch = unittest.mock.patch.object( + argparse.ArgumentParser, 'error', mock_error) + with error_patch: with pytest.raises(AssertionError): gen_dockerfile.parse_args(argv) From 65183249c9e05566b7f05636d8cfb94dafcdc9f7 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Mon, 26 Jun 2017 15:15:38 -0700 Subject: [PATCH 208/362] Fix minor typos in comments --- scripts/gen_dockerfile.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/gen_dockerfile.py b/scripts/gen_dockerfile.py index c9632b04..9bfbca2a 100755 --- a/scripts/gen_dockerfile.py +++ b/scripts/gen_dockerfile.py @@ -70,7 +70,7 @@ def get_app_config(raw_config, base_image, config_file, source_dir): We validate the user input for security and better error messages. - Consider, parsing a yaml file which has a string value where we + Consider parsing a yaml file which has a string value where we expected a list. Python will happily use the string as a sequence of individual characters, at least for a while, leading to confusing results when it finally fails. @@ -84,7 +84,7 @@ def get_app_config(raw_config, base_image, config_file, source_dir): raw_config (dict): deserialized app.yaml base_image (str): Docker image name to build on top of config_file (str): Path to user's app.yaml (might be .yaml) - source_dir (str): Directory container user's source code + source_dir (str): Directory containing user's source code Returns: AppConfig: valid configuration From 7b8c4de65a606d93045073a8ea6ef8befedb7475 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Mon, 26 Jun 2017 15:17:07 -0700 Subject: [PATCH 209/362] Add main so these tests can be run standalone --- scripts/gen_dockerfile_test.py | 4 ++++ scripts/local_cloudbuild_test.py | 4 ++++ scripts/validation_utils_test.py | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/scripts/gen_dockerfile_test.py b/scripts/gen_dockerfile_test.py index 046580dd..6a7bd33a 100755 --- a/scripts/gen_dockerfile_test.py +++ b/scripts/gen_dockerfile_test.py @@ -223,3 +223,7 @@ def mock_error(*args): with error_patch: with pytest.raises(AssertionError): gen_dockerfile.parse_args(argv) + + +if __name__ == '__main__': + pytest.main([__file__]) diff --git a/scripts/local_cloudbuild_test.py b/scripts/local_cloudbuild_test.py index 00609f01..b22d585b 100755 --- a/scripts/local_cloudbuild_test.py +++ b/scripts/local_cloudbuild_test.py @@ -404,3 +404,7 @@ def test_parse_args_output_script(argv, expected): def test_parse_args_run_flag(argv, expected): args = local_cloudbuild.parse_args(argv) assert args.run == expected + + +if __name__ == '__main__': + pytest.main([__file__]) diff --git a/scripts/validation_utils_test.py b/scripts/validation_utils_test.py index ff853e4e..d759b276 100755 --- a/scripts/validation_utils_test.py +++ b/scripts/validation_utils_test.py @@ -92,3 +92,7 @@ def test_validate_arg_dicts_valid(arg, expected): def test_validate_arg_dicts_invalid(arg): with pytest.raises(argparse.ArgumentTypeError): validation_utils.validate_arg_dict(arg) + + +if __name__ == '__main__': + pytest.main([__file__]) From 251cd92369dd750a493f8a091a924d675a957a82 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Wed, 28 Jun 2017 19:19:49 -0700 Subject: [PATCH 210/362] Rename builder image to parallel the other languages' builders. --- build.sh | 10 +++++++++- cloudbuild.yaml | 4 ++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/build.sh b/build.sh index 943c5b92..7ee1416b 100755 --- a/build.sh +++ b/build.sh @@ -50,11 +50,19 @@ if [ -z "${DOCKER_NAMESPACE+set}" ] ; then fatal 'Error: $DOCKER_NAMESPACE is not set; invoke with something like DOCKER_NAMESPACE=gcr.io/YOUR-PROJECT-NAME' fi +if [ -z "${BUILDER_DOCKER_NAMESPACE+set}" ] ; then + export BUILDER_DOCKER_NAMESPACE="${DOCKER_NAMESPACE}" +fi + if [ -z "${TAG+set}" ] ; then export TAG=`date +%Y-%m-%d-%H%M%S` fi -substitutions="_DOCKER_NAMESPACE=${DOCKER_NAMESPACE},_TAG=${TAG}" +substitutions="\ +_BUILDER_DOCKER_NAMESPACE=${BUILDER_DOCKER_NAMESPACE},\ +_DOCKER_NAMESPACE=${DOCKER_NAMESPACE},\ +_TAG=${TAG}\ +" # Read command line arguments while [ $# -gt 0 ]; do diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 3f52e41d..187884dc 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -33,9 +33,9 @@ steps: name: ${_DOCKER_NAMESPACE}/python/tests/google-cloud-python:${_TAG} - # Build runtime builder image name: gcr.io/cloud-builders/docker:latest - args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/builder/gen-dockerfile:${_TAG}', + args: ['build', '--tag=${_BUILDER_DOCKER_NAMESPACE}/python/gen-dockerfile:${_TAG}', '--no-cache', '/workspace/builder/gen-dockerfile/'] images: [ '${_DOCKER_NAMESPACE}/python:${_TAG}', - '${_DOCKER_NAMESPACE}/python/builder/gen-dockerfile:${_TAG}', + '${_BUILDER_DOCKER_NAMESPACE}/python/gen-dockerfile:${_TAG}', ] From 35c666ff7a9717fc53da17e2bc20006f706b8e66 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Wed, 12 Jul 2017 01:41:50 -0700 Subject: [PATCH 211/362] Auto-update dependencies. --- scripts/requirements-test.txt | 8 +- scripts/testdata/hello_world/requirements.txt | 4 +- tests/deploy_check/requirements.txt | 2 +- tests/integration/requirements.txt | 10 +- tests/python2-libraries/requirements.txt | 123 +++++++++--------- tests/python3-libraries/requirements.txt | 118 ++++++++--------- 6 files changed, 132 insertions(+), 133 deletions(-) diff --git a/scripts/requirements-test.txt b/scripts/requirements-test.txt index 77b8c636..d1db042b 100644 --- a/scripts/requirements-test.txt +++ b/scripts/requirements-test.txt @@ -1,4 +1,4 @@ -flask -pytest -pytest-cov -pyyaml +flask==0.12.2 +pytest==3.1.3 +pytest-cov==2.5.1 +pyyaml==3.12 diff --git a/scripts/testdata/hello_world/requirements.txt b/scripts/testdata/hello_world/requirements.txt index 7861fb49..bb84e50e 100644 --- a/scripts/testdata/hello_world/requirements.txt +++ b/scripts/testdata/hello_world/requirements.txt @@ -1,2 +1,2 @@ -Flask==0.12 -gunicorn==19.6.0 +Flask==0.12.2 +gunicorn==19.7.1 diff --git a/tests/deploy_check/requirements.txt b/tests/deploy_check/requirements.txt index e7676bba..bb84e50e 100644 --- a/tests/deploy_check/requirements.txt +++ b/tests/deploy_check/requirements.txt @@ -1,2 +1,2 @@ -Flask==0.12.1 +Flask==0.12.2 gunicorn==19.7.1 diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index 71d785ae..ee8d85c3 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -1,7 +1,7 @@ -Flask==0.12.1 -google-cloud-error-reporting==0.24.2 -google-cloud-logging==1.0.0 -google-cloud-monitoring==0.24.0 +Flask==0.12.2 +google-cloud-error-reporting==0.25.1 +google-cloud-logging==1.1.0 +google-cloud-monitoring==0.25.0 gunicorn==19.7.1 -requests==2.14.2 +requests==2.18.1 retrying==1.3.3 diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 61e36d58..2e51b07f 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -1,12 +1,12 @@ -alembic==0.9.1 +alembic==0.9.3 amqp==2.1.4 amqplib==1.0.2 -ansible==2.3.0.0 +ansible==2.3.1.0 anyjson==0.3.3 apache-libcloud==2.0.0 argparse==1.4.0 -astroid==1.5.2 -awscli==1.11.85 +astroid==1.5.3 +awscli==1.11.117 babel==2.4.0 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 @@ -15,50 +15,50 @@ beautifulsoup==3.2.1 billiard==3.5.0.2 blessings==1.6 blinker==1.4 -boto==2.46.1 -botocore==1.5.48 +boto==2.48.0 +botocore==1.5.80 bottle==0.12.13 -carbon==1.0.1 +carbon==1.0.2 celery==4.0.2 certifi==2017.4.17 cffi==1.10.0 -chardet==3.0.2 +chardet==3.0.4 click==6.7 -cliff==2.7.0 -cmd2==0.7.0 +cliff==2.8.0 +cmd2==0.7.5 colorama==0.3.9 configobj==5.0.6 cov-core==1.15.0 -coverage==4.4 +coverage==4.4.1 coveralls==1.1 -cryptography==1.8.1 +cryptography==1.9 cssselect==1.0.1 cython==0.25.2 decorator==4.0.11 django-celery==3.2.1 django-debug-toolbar==1.8 -django-extensions==1.7.9 -django==1.11.1 +django-extensions==1.8.1 +django==1.11.3 django_compress==1.0.1 djangorestframework==3.6.3 docker-py==1.10.6 docopt==0.6.2 docutils==0.13.1 ecdsa==0.13 -elasticsearch==5.3.0 +elasticsearch==5.4.0 enum34==1.1.6 eventlet==0.21.0 extras==1.0.0 fabric==1.13.2 fixtures==3.0.0 flake8==3.3.0 -flask==0.12.1 +flask==0.12.2 funcsigs==1.0.2 functools32==3.2.3.post2 futures==3.1.1 -gevent==1.2.1 +gevent==1.2.2 google-api-python-client==1.6.2 -graphite-web==1.0.1 +graphite-web==1.0.2 greenlet==0.4.12 gunicorn==19.7.1 hiredis==0.2.0 @@ -66,17 +66,16 @@ html5lib httplib2==0.10.3 idna==2.5 ipaddress==1.0.18 -#ipython # No longer supports Python 2 as of version 6.0 iso8601==0.1.11 isodate==0.5.4 itsdangerous==0.24 jinja2==2.9.6 -jmespath==0.9.2 +jmespath==0.9.3 jsonschema==2.6.0 kombu==4.0.2 linecache2==1.0.0 -logilab-common==1.4.0 -lxml==3.7.3 +logilab-common==1.4.1 +lxml==3.8.0 m2crypto==0.26.0 mako==1.0.6 manifestparser==1.1 @@ -89,8 +88,8 @@ mock==2.0.0 mozcrash==1.0 mozdevice==0.50 mozfile==1.2 -mozinfo==0.9 -mozlog==3.4 +mozinfo==0.10 +mozlog==3.5 moznetwork==0.27 mozprocess==0.25 mozprofile==0.28 @@ -99,95 +98,95 @@ msgpack-python==0.4.8 mysql-python==1.2.5 ndg-httpsclient==0.4.2 netaddr==0.7.19 -netifaces==0.10.5 -newrelic==2.86.0.65 +netifaces==0.10.6 +newrelic==2.88.0.72 nose==1.3.7 -numpy==1.12.1 +numpy==1.13.1 oauth2==1.9.0.post1 -oauth2client==4.1.0 +oauth2client==4.1.2 oauthlib==2.0.2 ordereddict==1.1 -oslo.config==4.1.0 -pandas==0.20.1 -paramiko==2.1.2 +oslo.config==4.8.0 +pandas==0.20.3 +paramiko==2.2.1 passlib==1.7.1 paste==2.0.3 pastedeploy==1.5.2 pastescript==2.0.2 -pbr==3.0.0 +pbr==3.1.1 pep8==1.7.0 pexpect==4.2.1 pika==0.10.0 -pillow==4.1.1 +pillow==4.2.1 pip==9.0.1 prettytable protobuf==3.3.0 psutil==5.2.2 psycopg2==2.7.1 -py==1.4.33 -pyasn1-modules==0.0.8 +py==1.4.34 +pyasn1-modules==0.0.9 pyasn1==0.2.3 -pycparser==2.17 +pycparser==2.18 pycrypto==2.6.1 pycurl==7.43.0 pyflakes==1.5.0 pygments==2.2.0 -pyjwt==1.5.0 +pyjwt==1.5.2 pylibmc==1.5.2 -pylint==1.7.1 +pylint==1.7.2 pymongo==3.4.0 pymysql==0.7.11 -pyopenssl==17.0.0 +pyopenssl==17.1.0 pyparsing==2.2.0 -pyramid==1.8.3 +pyramid==1.9 pystache==0.5.4 pytest-cov==2.5.1 -pytest==3.0.7 +pytest==3.1.3 python-cjson==1.2.1 python-daemon==2.1.2 -python-dateutil==2.6.0 +python-dateutil==2.6.1 python-gflags==3.1.1 -python-keystoneclient==3.10.0 +python-keystoneclient==3.12.0 python-memcached==1.58 python-mimeparse==1.6.0 -python-novaclient==8.0.0 +python-novaclient==9.0.1 python-subunit==1.2.0 python-swiftclient==3.3.0 pytz==2017.2 pyyaml==3.12 pyzmq==16.0.2 -raven==6.0.0 +raven==6.1.0 redis==2.10.5 repoze.lru==0.6 requests-oauthlib==0.8.0 -requests==2.14.2 +requests==2.18.1 retrying==1.3.3 rsa==3.4.2 -scipy==0.19.0 -selenium==3.4.1 +scipy==0.19.1 +selenium==3.4.3 setuptools-git==1.2 -setuptools==35.0.2 -sh==1.12.13 -simplejson==3.10.0 +setuptools==36.0.1 +sh==1.12.14 +simplejson==3.11.1 six==1.10.0 snowballstemmer==1.2.1 south==1.0.2 -sphinx==1.5.5 +sphinx==1.6.3 sqlalchemy-migrate==0.11.0 -sqlalchemy==1.1.9 +sqlalchemy==1.1.11 sqlparse==0.2.3 statsd==3.2.1 -stevedore==1.21.0 +stevedore==1.24.0 suds==0.4 -supervisor==3.3.1 +supervisor==3.3.2 testrepository==0.0.20 testtools==2.3.0 thrift==0.10.0 tornado==4.5.1 tox==2.7.0 -twisted==17.1.0 +twisted==17.5.0 ujson==1.35 -unidecode==0.4.20 +unidecode==0.4.21 unittest2==1.1.0 uritemplate==3.0.0 urllib3==1.21.1 @@ -196,11 +195,11 @@ versiontools==1.9.1 virtualenv==15.1.0 waitress==1.0.2 warlock==1.3.0 -webob==1.7.2 -websocket-client==0.40.0 +webob==1.7.3 +websocket-client==0.44.0 webtest==2.0.27 -werkzeug==0.12.1 +werkzeug==0.12.2 wheel==0.29.0 xlrd==1.0.0 -zc.buildout==2.9.3 -zope.interface==4.4.1 +zc.buildout==2.9.4 +zope.interface==4.4.2 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index e99f3629..92261832 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -1,12 +1,12 @@ -alembic==0.9.1 +alembic==0.9.3 amqp==2.1.4 amqplib==1.0.2 -ansible==2.3.0.0 +ansible==2.3.1.0 anyjson==0.3.3 apache-libcloud==2.0.0 argparse==1.4.0 -astroid==1.5.2 -awscli==1.11.85 +astroid==1.5.3 +awscli==1.11.117 babel==2.4.0 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 @@ -14,46 +14,46 @@ beautifulsoup4==4.6.0 billiard==3.5.0.2 blessings==1.6 blinker==1.4 -boto==2.46.1 -botocore==1.5.48 +boto==2.48.0 +botocore==1.5.80 bottle==0.12.13 celery==4.0.2 certifi==2017.4.17 cffi==1.10.0 -chardet==3.0.2 +chardet==3.0.4 click==6.7 -cliff==2.7.0 -cmd2==0.7.0 +cliff==2.8.0 +cmd2==0.7.5 colorama==0.3.9 configobj==5.0.6 cov-core==1.15.0 -coverage==4.4 +coverage==4.4.1 coveralls==1.1 -cryptography==1.8.1 +cryptography==1.9 cssselect==1.0.1 cython==0.25.2 decorator==4.0.11 django-celery==3.2.1 django-debug-toolbar==1.8 -django-extensions==1.7.9 -django==1.11.1 +django-extensions==1.8.1 +django==1.11.3 django_compress==1.0.1 djangorestframework==3.6.3 docker-py==1.10.6 docopt==0.6.2 docutils==0.13.1 ecdsa==0.13 -elasticsearch==5.3.0 +elasticsearch==5.4.0 enum34==1.1.6 eventlet==0.21.0 extras==1.0.0 fabric==1.13.2 fixtures==3.0.0 flake8==3.3.0 -flask==0.12.1 +flask==0.12.2 funcsigs==1.0.2 futures==3.1.1 -gevent==1.2.1 +gevent==1.2.2 google-api-python-client==1.6.2 greenlet==0.4.12 gunicorn==19.7.1 @@ -62,17 +62,17 @@ html5lib httplib2==0.10.3 idna==2.5 ipaddress==1.0.18 -ipython==6.0.0 +ipython==6.1.0 iso8601==0.1.11 isodate==0.5.4 itsdangerous==0.24 jinja2==2.9.6 -jmespath==0.9.2 +jmespath==0.9.3 jsonschema==2.6.0 kombu==4.0.2 linecache2==1.0.0 -logilab-common==1.4.0 -lxml==3.7.3 +logilab-common==1.4.1 +lxml==3.8.0 m2crypto==0.26.0 mako==1.0.6 manifestparser==1.1 @@ -85,98 +85,98 @@ mock==2.0.0 mozcrash==1.0 mozdevice==0.50 mozfile==1.2 -mozinfo==0.9 -mozlog==3.4 +mozinfo==0.10 +mozlog==3.5 moznetwork==0.27 mozprocess==0.25 msgpack-python==0.4.8 ndg-httpsclient==0.4.2 netaddr==0.7.19 -netifaces==0.10.5 -newrelic==2.86.0.65 +netifaces==0.10.6 +newrelic==2.88.0.72 nose==1.3.7 -numpy==1.12.1 +numpy==1.13.1 oauth2==1.9.0.post1 -oauth2client==4.1.0 +oauth2client==4.1.2 oauthlib==2.0.2 ordereddict==1.1 -oslo.config==4.1.0 -pandas==0.20.1 -paramiko==2.1.2 +oslo.config==4.8.0 +pandas==0.20.3 +paramiko==2.2.1 passlib==1.7.1 paste==2.0.3 pastedeploy==1.5.2 pastescript==2.0.2 -pbr==3.0.0 +pbr==3.1.1 pep8==1.7.0 pexpect==4.2.1 pika==0.10.0 -pillow==4.1.1 +pillow==4.2.1 pip==9.0.1 prettytable protobuf==3.3.0 psutil==5.2.2 psycopg2==2.7.1 -py==1.4.33 -pyasn1-modules==0.0.8 +py==1.4.34 +pyasn1-modules==0.0.9 pyasn1==0.2.3 -pycparser==2.17 +pycparser==2.18 pycrypto==2.6.1 pyflakes==1.5.0 pygments==2.2.0 -pyjwt==1.5.0 +pyjwt==1.5.2 pylibmc==1.5.2 -pylint==1.7.1 +pylint==1.7.2 pymongo==3.4.0 pymysql==0.7.11 -pyopenssl==17.0.0 +pyopenssl==17.1.0 pyparsing==2.2.0 -pyramid==1.8.3 +pyramid==1.9 pystache==0.5.4 pytest-cov==2.5.1 -pytest==3.0.7 +pytest==3.1.3 python-daemon==2.1.2 -python-dateutil==2.6.0 +python-dateutil==2.6.1 python-gflags==3.1.1 -python-keystoneclient==3.10.0 +python-keystoneclient==3.12.0 python-memcached==1.58 python-mimeparse==1.6.0 -python-novaclient==8.0.0 +python-novaclient==9.0.1 python-subunit==1.2.0 python-swiftclient==3.3.0 pytz==2017.2 pyyaml==3.12 pyzmq==16.0.2 -raven==6.0.0 +raven==6.1.0 redis==2.10.5 repoze.lru==0.6 requests-oauthlib==0.8.0 -requests==2.14.2 +requests==2.18.1 retrying==1.3.3 rsa==3.4.2 -scipy==0.19.0 -selenium==3.4.1 +scipy==0.19.1 +selenium==3.4.3 setuptools-git==1.2 -setuptools==35.0.2 -sh==1.12.13 -simplejson==3.10.0 +setuptools==36.0.1 +sh==1.12.14 +simplejson==3.11.1 six==1.10.0 snowballstemmer==1.2.1 south==1.0.2 -sphinx==1.5.5 +sphinx==1.6.3 sqlalchemy-migrate==0.11.0 -sqlalchemy==1.1.9 +sqlalchemy==1.1.11 sqlparse==0.2.3 statsd==3.2.1 -stevedore==1.21.0 +stevedore==1.24.0 testrepository==0.0.20 testtools==2.3.0 thrift==0.10.0 tornado==4.5.1 tox==2.7.0 -twisted==17.1.0 +twisted==17.5.0 ujson==1.35 -unidecode==0.4.20 +unidecode==0.4.21 unittest2==1.1.0 uritemplate==3.0.0 urllib3==1.21.1 @@ -185,11 +185,11 @@ versiontools==1.9.1 virtualenv==15.1.0 waitress==1.0.2 warlock==1.3.0 -webob==1.7.2 -websocket-client==0.40.0 +webob==1.7.3 +websocket-client==0.44.0 webtest==2.0.27 -werkzeug==0.12.1 +werkzeug==0.12.2 wheel==0.29.0 xlrd==1.0.0 -zc.buildout==2.9.3 -zope.interface==4.4.1 +zc.buildout==2.9.4 +zope.interface==4.4.2 From 6dd33a4378d6252addefe6793f9992fbfc98720a Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Wed, 12 Jul 2017 13:26:41 -0700 Subject: [PATCH 212/362] Add a way to run just the library install tests for the DPE Gardener bot. --- .gitignore | 4 +--- build.sh | 41 ++++++++++++++++++++++++++++++----- cloudbuild_benchmark.yaml | 4 ++++ cloudbuild_library_tests.yaml | 15 +++++++++++++ cloudbuild_system_tests.yaml | 4 ++++ 5 files changed, 60 insertions(+), 8 deletions(-) create mode 100644 cloudbuild_library_tests.yaml diff --git a/.gitignore b/.gitignore index 62f3090b..73ec28f0 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,5 @@ .cache .coverage .nox -/cloudbuild.yaml_local.sh -/cloudbuild_benchmark.yaml_local.sh -/cloudbuild_system_tests.yaml_local.sh +/*_local.sh __pycache__ diff --git a/build.sh b/build.sh index 7ee1416b..82d8c05e 100755 --- a/build.sh +++ b/build.sh @@ -19,6 +19,7 @@ set -euo pipefail # Actions benchmark=0 # Should run benchmarks? build=1 # Should build images? +library_tests=0 # Should try to install top N Python libraries system_tests=0 # Should run system tests? local=0 # Should run using local Docker daemon instead of GCR? @@ -38,10 +39,11 @@ function usage { Build and test artifacts in this repository Options: - --benchmark: Run benchmarking suite - --[no]build: Build all images (default) - --local: Build images using local Docker daemon - --system_tests: Run system tests + --[no]benchmark: Run benchmarking suite (default false) + --[no]build: Build all images (default true) + --[no]library_tests: Run library compatiblity tests (default false) + --[no]local: Build images using local Docker daemon (default false) + --[no]system_tests: Run system tests (default false) " } @@ -71,6 +73,10 @@ while [ $# -gt 0 ]; do benchmark=1 shift ;; + --nobenchmark) + benchmark=0 + shift + ;; --build) build=1 shift @@ -79,14 +85,30 @@ while [ $# -gt 0 ]; do build=0 shift ;; + --library_tests) + library_tests=1 + shift + ;; + --nolibrary_tests) + library_tests=0 + shift + ;; --local) local=1 shift ;; + --nolocal) + local=0 + shift + ;; --system_tests) system_tests=1 shift ;; + --nosystem_tests) + system_tests=0 + shift + ;; *) usage ;; @@ -94,7 +116,10 @@ while [ $# -gt 0 ]; do done # If no actions chosen, then tell the user -if [ "${benchmark}" -eq 0 -a "${build}" -eq 0 -a "${system_tests}" -eq 0 ]; then +if [ "${benchmark}" -eq 0 -a \ + "${build}" -eq 0 -a \ + "${library_tests}" -eq 0 -a \ + "${system_tests}" -eq 0 ]; then fatal 'Error: No actions specified (for example, --build), exiting' fi @@ -148,6 +173,12 @@ if [ "${build}" -eq 1 ]; then ${gcloud_cmd} --config cloudbuild.yaml --substitutions "${substitutions}" fi +# Run just the library compatibility tests (for DPE Gardener bot usually) +if [ "${library_tests}" -eq 1 ]; then + echo "Testing compatibility with popular Python libraries" + ${gcloud_cmd} --config cloudbuild_library_tests.yaml --substitutions "${substitutions}" +fi + # If both system tests and benchmarks are requested, run them both # even if one or the other has errors. If the build step had errors, # this script will have already exited. diff --git a/cloudbuild_benchmark.yaml b/cloudbuild_benchmark.yaml index a960bc9b..616991ef 100644 --- a/cloudbuild_benchmark.yaml +++ b/cloudbuild_benchmark.yaml @@ -3,6 +3,10 @@ steps: - name: gcr.io/cloud-builders/docker:latest args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/benchmark:${_TAG}', '--no-cache', '/workspace/tests/benchmark/'] + env: [ + # Avoid warning about unused substitutions + 'UNUSED1=${_BUILDER_DOCKER_NAMESPACE}', + ] images: [ # Intentionally empty ] diff --git a/cloudbuild_library_tests.yaml b/cloudbuild_library_tests.yaml new file mode 100644 index 00000000..01fe7f40 --- /dev/null +++ b/cloudbuild_library_tests.yaml @@ -0,0 +1,15 @@ +steps: +- # Check that we can install important libraries without error + name: gcr.io/gcp-runtimes/structure_test:latest + args: [ + '-i', '${_DOCKER_NAMESPACE}/python:${_TAG}', + '--config', '/workspace/tests/python2-libraries/python2-libraries.yaml', + '--config', '/workspace/tests/python3-libraries/python3-libraries.yaml', + '-v' + ] + env: [ + # Avoid warning about unused substitutions + 'UNUSED1=${_BUILDER_DOCKER_NAMESPACE}', + ] +images: [ +] diff --git a/cloudbuild_system_tests.yaml b/cloudbuild_system_tests.yaml index 3cac03e2..f01a48e7 100644 --- a/cloudbuild_system_tests.yaml +++ b/cloudbuild_system_tests.yaml @@ -4,6 +4,10 @@ steps: args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/google-cloud-python-system:${_TAG}', '--no-cache', '/workspace/tests/google-cloud-python-system/'] - name: ${_DOCKER_NAMESPACE}/python/tests/google-cloud-python-system:${_TAG} + env: [ + # Avoid warning about unused substitutions + 'UNUSED1=${_BUILDER_DOCKER_NAMESPACE}', + ] images: [ # Intentionally empty ] From e0049040e1367088bec7d04d9132ea933348df18 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Wed, 12 Jul 2017 15:15:07 -0700 Subject: [PATCH 213/362] Allow $GAE_APPLICATION_YAML_PATH as an alternative to --config In some cases, gcloud sets an environment variable to indicate the location of the application configuration file, rather than using the --config flag. --- scripts/gen_dockerfile.py | 12 +++++++++++- scripts/gen_dockerfile_test.py | 21 +++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/scripts/gen_dockerfile.py b/scripts/gen_dockerfile.py index 9bfbca2a..5514b092 100755 --- a/scripts/gen_dockerfile.py +++ b/scripts/gen_dockerfile.py @@ -57,6 +57,8 @@ '3.6': '3.6', } +# Name of environment variable potentially set by gcloud +GAE_APPLICATION_YAML_PATH = 'GAE_APPLICATION_YAML_PATH' # Validated application configuration AppConfig = collections.namedtuple( @@ -225,11 +227,19 @@ def parse_args(argv): validation_utils.validate_arg_regex, flag_regex=IMAGE_REGEX), default='gcr.io/google-appengine/python:latest', help='Name of Docker image to use as base') + # In some cases, gcloud sets an environment variable to indicate + # the location of the application configuration file, rather than + # using the --config flag. The order of precedence from highest + # to lowest is: + # + # 1) --config flag + # 2) $GAE_APPLICATION_YAML_PATH environment variable + # 3) a file named "app.yaml" in the current working directory parser.add_argument( '--config', type=functools.partial( validation_utils.validate_arg_regex, flag_regex=PRINTABLE_REGEX), - default='app.yaml', + default=(os.environ.get(GAE_APPLICATION_YAML_PATH) or 'app.yaml'), help='Path to application configuration file' ) parser.add_argument( diff --git a/scripts/gen_dockerfile_test.py b/scripts/gen_dockerfile_test.py index 6a7bd33a..31ef7d42 100755 --- a/scripts/gen_dockerfile_test.py +++ b/scripts/gen_dockerfile_test.py @@ -225,5 +225,26 @@ def mock_error(*args): gen_dockerfile.parse_args(argv) +@pytest.mark.parametrize('argv, env, expected', [ + # Explicit flag wins + (['argv0', '--config=flag/path'], 'env/path', 'flag/path'), + (['argv0', '--config=flag/path'], '', 'flag/path'), + (['argv0', '--config=flag/path'], None, 'flag/path'), + # Otherwise env var wins + (['argv0'], 'env/path', 'env/path'), + # Otherwise use default name + (['argv0'], '', 'app.yaml'), + (['argv0'], None, 'app.yaml'), +]) +def test_parse_args_config(argv, env, expected): + if env is None: + mock_environ = {} + else: + mock_environ = {gen_dockerfile.GAE_APPLICATION_YAML_PATH: env} + with unittest.mock.patch.dict('os.environ', mock_environ, clear=True): + args = gen_dockerfile.parse_args(argv) + assert args.config == expected + + if __name__ == '__main__': pytest.main([__file__]) From 7e19176d3ae08f10b601b9a24bb8943c5b997171 Mon Sep 17 00:00:00 2001 From: Angela Li Date: Thu, 13 Jul 2017 16:36:08 -0700 Subject: [PATCH 214/362] Added a script for collecting the python client libraries download data (#129) Added a script for collecting the python client libraries download data --- .../python_clientlibs_download.py | 162 ++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100644 perf-dashboard/clientlibs-download/python_clientlibs_download.py diff --git a/perf-dashboard/clientlibs-download/python_clientlibs_download.py b/perf-dashboard/clientlibs-download/python_clientlibs_download.py new file mode 100644 index 00000000..888e6070 --- /dev/null +++ b/perf-dashboard/clientlibs-download/python_clientlibs_download.py @@ -0,0 +1,162 @@ +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import time +import uuid + +from google.cloud import bigquery + +DATETIME_FORMAT = '%Y%m%d' + +DATASET_NAME = 'python_clientlibs_download_by_week' + +VENEER_TABLE_NAME = 'veneer_client_libs' +STACKDRIVER_TABLE_NAME = 'stackdriver_client_libs' +GRPC_TABLE_NAME = 'grpc_lib' +TABLES = [VENEER_TABLE_NAME, GRPC_TABLE_NAME, STACKDRIVER_TABLE_NAME] + +CLIENTLIBS = { + VENEER_TABLE_NAME: [ + 'google-cloud-core', + 'google-cloud-speech', + 'google-cloud-language', + 'google-cloud-pubsub', + 'google-cloud-bigquery', + 'google-cloud-bigtable', + 'google-cloud-datastore', + 'google-cloud-spanner', + 'google-cloud-storage', + 'google-cloud-vision', + 'google-cloud-translate', + 'google-cloud-dns', + 'google-cloud-videointelligence', + ], + STACKDRIVER_TABLE_NAME: [ + 'google-cloud-logging', + 'google-cloud-monitoring', + 'google-cloud-error_reporting', + ], + GRPC_TABLE_NAME: [ + 'grpcio', + ], +} + + +def wait_for_job(job): + """Wait for the query job to complete.""" + while True: + job.reload() # Refreshes the state via a GET request. + if job.state == 'DONE': + if job.error_result: + raise RuntimeError(job.errors) + return + time.sleep(1) + + +def get_weekly_clientlibs_downloads(clientlibs_table_name, date_str): + """Use a SQL query to collect the weekly download data of the client + libraries. + + Args: + clientlibs_table_name (str): Table name, which is the key in the + CLIENTLIBS dict. + date_str (str): A date string in "YYYYMMDD" format. + + Returns: + list: rows of the query result. + """ + client_libs = CLIENTLIBS[clientlibs_table_name] + date_time = datetime.datetime.strptime(date_str, DATETIME_FORMAT) + week_dates = [(date_time + datetime.timedelta(days=-i)) + .strftime(DATETIME_FORMAT) + for i in range(7)] + query = """ + SELECT + file.project as client_library_name, + COUNT(*) as download_count + FROM + `the-psf.pypi.downloads*` + WHERE + file.project IN UNNEST(@client_libs) + AND + _TABLE_SUFFIX IN UNNEST(@week_dates) + GROUP BY client_library_name + """ + client = bigquery.Client() + query_job = client.run_async_query( + str(uuid.uuid4()), + query, + query_parameters=( + bigquery.ArrayQueryParameter( + 'client_libs', 'STRING', + client_libs), + bigquery.ArrayQueryParameter( + 'week_dates', 'STRING', + week_dates) + )) + query_job.use_legacy_sql = False + + # Start the query job and wait it to complete + query_job.begin() + wait_for_job(query_job) + + # Fetch the results + result = query_job.results().fetch_data() + result_list = [item for item in result] + + # In case the result_list contains the metadata like total_rows, the + # actual rows will be the first element of the result_list. + if len(result_list) > 0 and isinstance(result_list[0], list): + result_list = result_list[0] + + rows = [(date_time,) + row for row in result_list] + print rows + + return rows + + +def insert_rows(dataset_name, table_name, rows): + """Insert rows to a bigquery table. + + Args: + dataset_name (str): Name of the dataset that holds the tables. + table_name (str): Name of the bigquery table. + rows (list): The rows that going to be inserted into the table. + + Returns: + list: Empty if inserted successfully, else the errors when inserting + each row. + """ + client = bigquery.Client() + dataset = client.dataset(dataset_name) + table = bigquery.Table(name=table_name, dataset=dataset) + table.reload() + error = table.insert_data(rows) + return error + + +def main(): + for table_name in CLIENTLIBS.keys(): + rows = get_weekly_clientlibs_downloads( + clientlibs_table_name=table_name, + date_str=datetime.datetime.now().strftime("%Y%m%d")) + insert_rows( + dataset_name=DATASET_NAME, + table_name=table_name, + rows=rows) + + +if __name__ == '__main__': + main() From fa68b87ddccebdd5727329093a27e26284a355f3 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Mon, 17 Jul 2017 01:41:31 -0700 Subject: [PATCH 215/362] Auto-update dependencies. --- tests/python2-libraries/requirements.txt | 18 +++++++++--------- tests/python3-libraries/requirements.txt | 18 +++++++++--------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 2e51b07f..5b879a08 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -1,22 +1,22 @@ alembic==0.9.3 -amqp==2.1.4 +amqp==2.2.1 amqplib==1.0.2 ansible==2.3.1.0 anyjson==0.3.3 apache-libcloud==2.0.0 argparse==1.4.0 astroid==1.5.3 -awscli==1.11.117 +awscli==1.11.120 babel==2.4.0 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 beautifulsoup4==4.6.0 beautifulsoup==3.2.1 -billiard==3.5.0.2 +billiard==3.5.0.3 blessings==1.6 blinker==1.4 boto==2.48.0 -botocore==1.5.80 +botocore==1.5.83 bottle==0.12.13 carbon==1.0.2 celery==4.0.2 @@ -34,7 +34,7 @@ coveralls==1.1 cryptography==1.9 cssselect==1.0.1 cython==0.25.2 -decorator==4.0.11 +decorator==4.1.1 django-celery==3.2.1 django-debug-toolbar==1.8 django-extensions==1.8.1 @@ -77,7 +77,7 @@ linecache2==1.0.0 logilab-common==1.4.1 lxml==3.8.0 m2crypto==0.26.0 -mako==1.0.6 +mako==1.0.7 manifestparser==1.1 markdown==2.6.8 markupsafe==1.0 @@ -99,7 +99,7 @@ mysql-python==1.2.5 ndg-httpsclient==0.4.2 netaddr==0.7.19 netifaces==0.10.6 -newrelic==2.88.0.72 +newrelic==2.88.1.73 nose==1.3.7 numpy==1.13.1 oauth2==1.9.0.post1 @@ -138,7 +138,7 @@ pymongo==3.4.0 pymysql==0.7.11 pyopenssl==17.1.0 pyparsing==2.2.0 -pyramid==1.9 +pyramid==1.9.1 pystache==0.5.4 pytest-cov==2.5.1 pytest==3.1.3 @@ -165,7 +165,7 @@ rsa==3.4.2 scipy==0.19.1 selenium==3.4.3 setuptools-git==1.2 -setuptools==36.0.1 +setuptools==36.2.0 sh==1.12.14 simplejson==3.11.1 six==1.10.0 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 92261832..f4559cb0 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -1,21 +1,21 @@ alembic==0.9.3 -amqp==2.1.4 +amqp==2.2.1 amqplib==1.0.2 ansible==2.3.1.0 anyjson==0.3.3 apache-libcloud==2.0.0 argparse==1.4.0 astroid==1.5.3 -awscli==1.11.117 +awscli==1.11.120 babel==2.4.0 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 beautifulsoup4==4.6.0 -billiard==3.5.0.2 +billiard==3.5.0.3 blessings==1.6 blinker==1.4 boto==2.48.0 -botocore==1.5.80 +botocore==1.5.83 bottle==0.12.13 celery==4.0.2 certifi==2017.4.17 @@ -32,7 +32,7 @@ coveralls==1.1 cryptography==1.9 cssselect==1.0.1 cython==0.25.2 -decorator==4.0.11 +decorator==4.1.1 django-celery==3.2.1 django-debug-toolbar==1.8 django-extensions==1.8.1 @@ -74,7 +74,7 @@ linecache2==1.0.0 logilab-common==1.4.1 lxml==3.8.0 m2crypto==0.26.0 -mako==1.0.6 +mako==1.0.7 manifestparser==1.1 markdown==2.6.8 markupsafe==1.0 @@ -93,7 +93,7 @@ msgpack-python==0.4.8 ndg-httpsclient==0.4.2 netaddr==0.7.19 netifaces==0.10.6 -newrelic==2.88.0.72 +newrelic==2.88.1.73 nose==1.3.7 numpy==1.13.1 oauth2==1.9.0.post1 @@ -131,7 +131,7 @@ pymongo==3.4.0 pymysql==0.7.11 pyopenssl==17.1.0 pyparsing==2.2.0 -pyramid==1.9 +pyramid==1.9.1 pystache==0.5.4 pytest-cov==2.5.1 pytest==3.1.3 @@ -157,7 +157,7 @@ rsa==3.4.2 scipy==0.19.1 selenium==3.4.3 setuptools-git==1.2 -setuptools==36.0.1 +setuptools==36.2.0 sh==1.12.14 simplejson==3.11.1 six==1.10.0 From 62e80bf702e3568df0d03646334bbe1799722d21 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Mon, 17 Jul 2017 13:08:48 -0700 Subject: [PATCH 216/362] Increase timeout for library compatibility test --- cloudbuild_library_tests.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/cloudbuild_library_tests.yaml b/cloudbuild_library_tests.yaml index 01fe7f40..846348fe 100644 --- a/cloudbuild_library_tests.yaml +++ b/cloudbuild_library_tests.yaml @@ -1,3 +1,4 @@ +timeout: 1800s steps: - # Check that we can install important libraries without error name: gcr.io/gcp-runtimes/structure_test:latest From 66e5264c2d71c8b2f8acc80fe06cbbf74ac850fa Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Mon, 17 Jul 2017 17:57:00 -0700 Subject: [PATCH 217/362] Switch image from Python 3.6.1 to 3.6.2 --- .../patches/3.6/double-build.diff | 96 ------------------- python-interpreter-builder/patches/3.6/series | 1 - .../scripts/build-python-3.6.sh | 14 ++- tests/virtualenv/virtualenv_python36.yaml | 2 +- 4 files changed, 7 insertions(+), 106 deletions(-) delete mode 100644 python-interpreter-builder/patches/3.6/double-build.diff delete mode 100644 python-interpreter-builder/patches/3.6/series diff --git a/python-interpreter-builder/patches/3.6/double-build.diff b/python-interpreter-builder/patches/3.6/double-build.diff deleted file mode 100644 index a89a20f6..00000000 --- a/python-interpreter-builder/patches/3.6/double-build.diff +++ /dev/null @@ -1,96 +0,0 @@ -# Source is https://github.com/python/cpython/pull/1478 - -Index: Python-3.6.1/Makefile.pre.in -=================================================================== ---- Python-3.6.1.orig/Makefile.pre.in -+++ Python-3.6.1/Makefile.pre.in -@@ -1000,7 +1000,7 @@ TESTTIMEOUT= 1200 - - # Run a basic set of regression tests. - # This excludes some tests that are particularly resource-intensive. --test: all platform -+test: @DEF_MAKE_RULE@ platform - $(TESTRUNNER) $(TESTOPTS) - - # Run the full test suite twice - once without .pyc files, and once with. -@@ -1010,7 +1010,7 @@ test: all platform - # the bytecode read from a .pyc file had the bug, sometimes the directly - # generated bytecode. This is sometimes a very shy bug needing a lot of - # sample data. --testall: all platform -+testall: @DEF_MAKE_RULE@ platform - -find $(srcdir)/Lib -name '*.py[co]' -print | xargs rm -f - $(TESTPYTHON) -E $(srcdir)/Lib/compileall.py - -find $(srcdir)/Lib -name '*.py[co]' -print | xargs rm -f -@@ -1019,7 +1019,7 @@ testall: all platform - - # Run the test suite for both architectures in a Universal build on OSX. - # Must be run on an Intel box. --testuniversal: all platform -+testuniversal: @DEF_MAKE_RULE@ platform - if [ `arch` != 'i386' ];then \ - echo "This can only be used on OSX/i386" ;\ - exit 1 ;\ -@@ -1042,7 +1042,7 @@ QUICKTESTOPTS= $(TESTOPTS) -x test_subpr - test_multiprocessing_forkserver \ - test_mailbox test_socket test_poll \ - test_select test_zipfile test_concurrent_futures --quicktest: all platform -+quicktest: @DEF_MAKE_RULE@ platform - $(TESTRUNNER) $(QUICKTESTOPTS) - - -@@ -1379,7 +1379,7 @@ LIBPL= @LIBPL@ - # pkgconfig directory - LIBPC= $(LIBDIR)/pkgconfig - --libainstall: all python-config -+libainstall: @DEF_MAKE_RULE@ python-config - @for i in $(LIBDIR) $(LIBPL) $(LIBPC); \ - do \ - if test ! -d $(DESTDIR)$$i; then \ -@@ -1639,7 +1639,7 @@ distclean: clobber - -exec rm -f {} ';' - - # Check for smelly exported symbols (not starting with Py/_Py) --smelly: all -+smelly: @DEF_MAKE_RULE@ - nm -p $(LIBRARY) | \ - sed -n "/ [TDB] /s/.* //p" | grep -v "^_*Py" | sort -u; \ - -@@ -1676,7 +1676,7 @@ funny: - -o -print - - # Perform some verification checks on any modified files. --patchcheck: all -+patchcheck: @DEF_MAKE_RULE@ - $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/scripts/patchcheck.py - - # Dependencies -Index: Python-3.6.1/Misc/ACKS -=================================================================== ---- Python-3.6.1.orig/Misc/ACKS -+++ Python-3.6.1/Misc/ACKS -@@ -1111,6 +1111,7 @@ Jason Orendorff - Douglas Orr - William Orr - Michele Orrù -+Tomáš Orsava - Oleg Oshmyan - Denis S. Otkidach - Peter Otten -Index: Python-3.6.1/Misc/NEWS -=================================================================== ---- Python-3.6.1.orig/Misc/NEWS -+++ Python-3.6.1/Misc/NEWS -@@ -306,6 +306,10 @@ Tests - Build - ----- - -+- bpo-29243: Prevent unnecessary rebuilding of Python during ``make test``, -+ ``make install`` and some other make targets when configured with -+ ``--enable-optimizations``. -+ - - bpo-27593: sys.version and the platform module python_build(), - python_branch(), and python_revision() functions now use - git information rather than hg when building from a repo. diff --git a/python-interpreter-builder/patches/3.6/series b/python-interpreter-builder/patches/3.6/series deleted file mode 100644 index f6c2876b..00000000 --- a/python-interpreter-builder/patches/3.6/series +++ /dev/null @@ -1 +0,0 @@ -double-build.diff diff --git a/python-interpreter-builder/scripts/build-python-3.6.sh b/python-interpreter-builder/scripts/build-python-3.6.sh index d3eb0c33..431c305d 100755 --- a/python-interpreter-builder/scripts/build-python-3.6.sh +++ b/python-interpreter-builder/scripts/build-python-3.6.sh @@ -6,16 +6,14 @@ set -x # Get the source mkdir -p /opt/sources cd /opt/sources -wget --no-verbose https://www.python.org/ftp/python/3.6.1/Python-3.6.1.tgz +wget --no-verbose https://www.python.org/ftp/python/3.6.2/Python-3.6.2.tgz # SHA-256 generated via `shasum -a 256 [file]` shasum --check < Date: Wed, 19 Jul 2017 09:37:39 -0700 Subject: [PATCH 218/362] Explicitly set the gcloud project when creating bigquery client (#134) Explicitly set the gcloud project when creating bigquery client --- .../clientlibs-download/python_clientlibs_download.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/perf-dashboard/clientlibs-download/python_clientlibs_download.py b/perf-dashboard/clientlibs-download/python_clientlibs_download.py index 888e6070..e93a6f93 100644 --- a/perf-dashboard/clientlibs-download/python_clientlibs_download.py +++ b/perf-dashboard/clientlibs-download/python_clientlibs_download.py @@ -13,11 +13,14 @@ # limitations under the License. import datetime +import os import time import uuid from google.cloud import bigquery +GCLOUD_PROJECT_ENV = 'GCLOUD_PROJECT' + DATETIME_FORMAT = '%Y%m%d' DATASET_NAME = 'python_clientlibs_download_by_week' @@ -70,7 +73,7 @@ def get_weekly_clientlibs_downloads(clientlibs_table_name, date_str): libraries. Args: - clientlibs_table_name (str): Table name, which is the key in the + clientlibs_table_name (str): Table name, which is the key in the CLIENTLIBS dict. date_str (str): A date string in "YYYYMMDD" format. @@ -139,7 +142,8 @@ def insert_rows(dataset_name, table_name, rows): list: Empty if inserted successfully, else the errors when inserting each row. """ - client = bigquery.Client() + project = os.environ.get(GCLOUD_PROJECT_ENV) + client = bigquery.Client(project=project) dataset = client.dataset(dataset_name) table = bigquery.Table(name=table_name, dataset=dataset) table.reload() From 0bdbf2b422b7627fea36c13875d575d3647790de Mon Sep 17 00:00:00 2001 From: Angela Li Date: Wed, 9 Aug 2017 13:38:57 -0700 Subject: [PATCH 219/362] Update the dashboard script to match the bigquery library changes (#137) --- .../clientlibs-download/python_clientlibs_download.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/perf-dashboard/clientlibs-download/python_clientlibs_download.py b/perf-dashboard/clientlibs-download/python_clientlibs_download.py index e93a6f93..fcfb048e 100644 --- a/perf-dashboard/clientlibs-download/python_clientlibs_download.py +++ b/perf-dashboard/clientlibs-download/python_clientlibs_download.py @@ -116,7 +116,7 @@ def get_weekly_clientlibs_downloads(clientlibs_table_name, date_str): wait_for_job(query_job) # Fetch the results - result = query_job.results().fetch_data() + result = query_job.result().fetch_data() result_list = [item for item in result] # In case the result_list contains the metadata like total_rows, the @@ -125,7 +125,7 @@ def get_weekly_clientlibs_downloads(clientlibs_table_name, date_str): result_list = result_list[0] rows = [(date_time,) + row for row in result_list] - print rows + print(rows) return rows From 7002a73e7ce80b7b594f2f22cafe420e6910f217 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Thu, 10 Aug 2017 01:41:54 -0700 Subject: [PATCH 220/362] Auto-update dependencies. --- scripts/requirements-test.txt | 2 +- tests/integration/requirements.txt | 8 +-- tests/python2-libraries/requirements.txt | 68 ++++++++++++------------ tests/python3-libraries/requirements.txt | 66 +++++++++++------------ 4 files changed, 72 insertions(+), 72 deletions(-) diff --git a/scripts/requirements-test.txt b/scripts/requirements-test.txt index d1db042b..6e01a83d 100644 --- a/scripts/requirements-test.txt +++ b/scripts/requirements-test.txt @@ -1,4 +1,4 @@ flask==0.12.2 -pytest==3.1.3 +pytest==3.2.1 pytest-cov==2.5.1 pyyaml==3.12 diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index ee8d85c3..05779ae6 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -1,7 +1,7 @@ Flask==0.12.2 -google-cloud-error-reporting==0.25.1 -google-cloud-logging==1.1.0 -google-cloud-monitoring==0.25.0 +google-cloud-error-reporting==0.26.0 +google-cloud-logging==1.2.0 +google-cloud-monitoring==0.26.0 gunicorn==19.7.1 -requests==2.18.1 +requests==2.18.3 retrying==1.3.3 diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 5b879a08..05a6dbe3 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -1,12 +1,12 @@ -alembic==0.9.3 +alembic==0.9.5 amqp==2.2.1 amqplib==1.0.2 -ansible==2.3.1.0 +ansible==2.3.2.0 anyjson==0.3.3 -apache-libcloud==2.0.0 +apache-libcloud==2.1.0 argparse==1.4.0 astroid==1.5.3 -awscli==1.11.120 +awscli==1.11.131 babel==2.4.0 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 @@ -16,11 +16,11 @@ billiard==3.5.0.3 blessings==1.6 blinker==1.4 boto==2.48.0 -botocore==1.5.83 +botocore==1.5.94 bottle==0.12.13 carbon==1.0.2 -celery==4.0.2 -certifi==2017.4.17 +celery==4.1.0 +certifi==2017.7.27.1 cffi==1.10.0 chardet==3.0.4 click==6.7 @@ -31,19 +31,19 @@ configobj==5.0.6 cov-core==1.15.0 coverage==4.4.1 coveralls==1.1 -cryptography==1.9 +cryptography==2.0.3 cssselect==1.0.1 -cython==0.25.2 -decorator==4.1.1 +cython==0.26 +decorator==4.1.2 django-celery==3.2.1 django-debug-toolbar==1.8 django-extensions==1.8.1 -django==1.11.3 +django==1.11.4 django_compress==1.0.1 djangorestframework==3.6.3 docker-py==1.10.6 docopt==0.6.2 -docutils==0.13.1 +docutils==0.14 ecdsa==0.13 elasticsearch==5.4.0 enum34==1.1.6 @@ -51,7 +51,7 @@ eventlet==0.21.0 extras==1.0.0 fabric==1.13.2 fixtures==3.0.0 -flake8==3.3.0 +flake8==3.4.1 flask==0.12.2 funcsigs==1.0.2 functools32==3.2.3.post2 @@ -64,15 +64,15 @@ gunicorn==19.7.1 hiredis==0.2.0 html5lib httplib2==0.10.3 -idna==2.5 +idna==2.6 ipaddress==1.0.18 -iso8601==0.1.11 +iso8601==0.1.12 isodate==0.5.4 itsdangerous==0.24 jinja2==2.9.6 jmespath==0.9.3 jsonschema==2.6.0 -kombu==4.0.2 +kombu==4.1.0 linecache2==1.0.0 logilab-common==1.4.1 lxml==3.8.0 @@ -99,14 +99,14 @@ mysql-python==1.2.5 ndg-httpsclient==0.4.2 netaddr==0.7.19 netifaces==0.10.6 -newrelic==2.88.1.73 +newrelic==2.90.0.75 nose==1.3.7 numpy==1.13.1 oauth2==1.9.0.post1 oauth2client==4.1.2 oauthlib==2.0.2 ordereddict==1.1 -oslo.config==4.8.0 +oslo.config==4.11.0 pandas==0.20.3 paramiko==2.2.1 passlib==1.7.1 @@ -122,36 +122,36 @@ pip==9.0.1 prettytable protobuf==3.3.0 psutil==5.2.2 -psycopg2==2.7.1 +psycopg2==2.7.3 py==1.4.34 -pyasn1-modules==0.0.9 -pyasn1==0.2.3 +pyasn1-modules==0.0.11 +pyasn1==0.3.2 pycparser==2.18 pycrypto==2.6.1 pycurl==7.43.0 -pyflakes==1.5.0 +pyflakes==1.6.0 pygments==2.2.0 pyjwt==1.5.2 pylibmc==1.5.2 pylint==1.7.2 -pymongo==3.4.0 +pymongo==3.5.0 pymysql==0.7.11 -pyopenssl==17.1.0 +pyopenssl==17.2.0 pyparsing==2.2.0 pyramid==1.9.1 pystache==0.5.4 pytest-cov==2.5.1 -pytest==3.1.3 +pytest==3.2.1 python-cjson==1.2.1 python-daemon==2.1.2 python-dateutil==2.6.1 python-gflags==3.1.1 -python-keystoneclient==3.12.0 +python-keystoneclient==3.13.0 python-memcached==1.58 python-mimeparse==1.6.0 -python-novaclient==9.0.1 +python-novaclient==9.1.0 python-subunit==1.2.0 -python-swiftclient==3.3.0 +python-swiftclient==3.4.0 pytz==2017.2 pyyaml==3.12 pyzmq==16.0.2 @@ -159,13 +159,13 @@ raven==6.1.0 redis==2.10.5 repoze.lru==0.6 requests-oauthlib==0.8.0 -requests==2.18.1 +requests==2.18.3 retrying==1.3.3 rsa==3.4.2 scipy==0.19.1 selenium==3.4.3 setuptools-git==1.2 -setuptools==36.2.0 +setuptools==36.2.7 sh==1.12.14 simplejson==3.11.1 six==1.10.0 @@ -176,9 +176,9 @@ sqlalchemy-migrate==0.11.0 sqlalchemy==1.1.11 sqlparse==0.2.3 statsd==3.2.1 -stevedore==1.24.0 +stevedore==1.25.0 suds==0.4 -supervisor==3.3.2 +supervisor==3.3.3 testrepository==0.0.20 testtools==2.3.0 thrift==0.10.0 @@ -189,7 +189,7 @@ ujson==1.35 unidecode==0.4.21 unittest2==1.1.0 uritemplate==3.0.0 -urllib3==1.21.1 +urllib3==1.22 uwsgi==2.0.15 versiontools==1.9.1 virtualenv==15.1.0 @@ -197,7 +197,7 @@ waitress==1.0.2 warlock==1.3.0 webob==1.7.3 websocket-client==0.44.0 -webtest==2.0.27 +webtest==2.0.28 werkzeug==0.12.2 wheel==0.29.0 xlrd==1.0.0 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index f4559cb0..f1e3539b 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -1,12 +1,12 @@ -alembic==0.9.3 +alembic==0.9.5 amqp==2.2.1 amqplib==1.0.2 -ansible==2.3.1.0 +ansible==2.3.2.0 anyjson==0.3.3 -apache-libcloud==2.0.0 +apache-libcloud==2.1.0 argparse==1.4.0 astroid==1.5.3 -awscli==1.11.120 +awscli==1.11.131 babel==2.4.0 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 @@ -15,10 +15,10 @@ billiard==3.5.0.3 blessings==1.6 blinker==1.4 boto==2.48.0 -botocore==1.5.83 +botocore==1.5.94 bottle==0.12.13 -celery==4.0.2 -certifi==2017.4.17 +celery==4.1.0 +certifi==2017.7.27.1 cffi==1.10.0 chardet==3.0.4 click==6.7 @@ -29,19 +29,19 @@ configobj==5.0.6 cov-core==1.15.0 coverage==4.4.1 coveralls==1.1 -cryptography==1.9 +cryptography==2.0.3 cssselect==1.0.1 -cython==0.25.2 -decorator==4.1.1 +cython==0.26 +decorator==4.1.2 django-celery==3.2.1 django-debug-toolbar==1.8 django-extensions==1.8.1 -django==1.11.3 +django==1.11.4 django_compress==1.0.1 djangorestframework==3.6.3 docker-py==1.10.6 docopt==0.6.2 -docutils==0.13.1 +docutils==0.14 ecdsa==0.13 elasticsearch==5.4.0 enum34==1.1.6 @@ -49,7 +49,7 @@ eventlet==0.21.0 extras==1.0.0 fabric==1.13.2 fixtures==3.0.0 -flake8==3.3.0 +flake8==3.4.1 flask==0.12.2 funcsigs==1.0.2 futures==3.1.1 @@ -60,16 +60,16 @@ gunicorn==19.7.1 hiredis==0.2.0 html5lib httplib2==0.10.3 -idna==2.5 +idna==2.6 ipaddress==1.0.18 ipython==6.1.0 -iso8601==0.1.11 +iso8601==0.1.12 isodate==0.5.4 itsdangerous==0.24 jinja2==2.9.6 jmespath==0.9.3 jsonschema==2.6.0 -kombu==4.0.2 +kombu==4.1.0 linecache2==1.0.0 logilab-common==1.4.1 lxml==3.8.0 @@ -93,14 +93,14 @@ msgpack-python==0.4.8 ndg-httpsclient==0.4.2 netaddr==0.7.19 netifaces==0.10.6 -newrelic==2.88.1.73 +newrelic==2.90.0.75 nose==1.3.7 numpy==1.13.1 oauth2==1.9.0.post1 oauth2client==4.1.2 oauthlib==2.0.2 ordereddict==1.1 -oslo.config==4.8.0 +oslo.config==4.11.0 pandas==0.20.3 paramiko==2.2.1 passlib==1.7.1 @@ -116,34 +116,34 @@ pip==9.0.1 prettytable protobuf==3.3.0 psutil==5.2.2 -psycopg2==2.7.1 +psycopg2==2.7.3 py==1.4.34 -pyasn1-modules==0.0.9 -pyasn1==0.2.3 +pyasn1-modules==0.0.11 +pyasn1==0.3.2 pycparser==2.18 pycrypto==2.6.1 -pyflakes==1.5.0 +pyflakes==1.6.0 pygments==2.2.0 pyjwt==1.5.2 pylibmc==1.5.2 pylint==1.7.2 -pymongo==3.4.0 +pymongo==3.5.0 pymysql==0.7.11 -pyopenssl==17.1.0 +pyopenssl==17.2.0 pyparsing==2.2.0 pyramid==1.9.1 pystache==0.5.4 pytest-cov==2.5.1 -pytest==3.1.3 +pytest==3.2.1 python-daemon==2.1.2 python-dateutil==2.6.1 python-gflags==3.1.1 -python-keystoneclient==3.12.0 +python-keystoneclient==3.13.0 python-memcached==1.58 python-mimeparse==1.6.0 -python-novaclient==9.0.1 +python-novaclient==9.1.0 python-subunit==1.2.0 -python-swiftclient==3.3.0 +python-swiftclient==3.4.0 pytz==2017.2 pyyaml==3.12 pyzmq==16.0.2 @@ -151,13 +151,13 @@ raven==6.1.0 redis==2.10.5 repoze.lru==0.6 requests-oauthlib==0.8.0 -requests==2.18.1 +requests==2.18.3 retrying==1.3.3 rsa==3.4.2 scipy==0.19.1 selenium==3.4.3 setuptools-git==1.2 -setuptools==36.2.0 +setuptools==36.2.7 sh==1.12.14 simplejson==3.11.1 six==1.10.0 @@ -168,7 +168,7 @@ sqlalchemy-migrate==0.11.0 sqlalchemy==1.1.11 sqlparse==0.2.3 statsd==3.2.1 -stevedore==1.24.0 +stevedore==1.25.0 testrepository==0.0.20 testtools==2.3.0 thrift==0.10.0 @@ -179,7 +179,7 @@ ujson==1.35 unidecode==0.4.21 unittest2==1.1.0 uritemplate==3.0.0 -urllib3==1.21.1 +urllib3==1.22 uwsgi==2.0.15 versiontools==1.9.1 virtualenv==15.1.0 @@ -187,7 +187,7 @@ waitress==1.0.2 warlock==1.3.0 webob==1.7.3 websocket-client==0.44.0 -webtest==2.0.27 +webtest==2.0.28 werkzeug==0.12.2 wheel==0.29.0 xlrd==1.0.0 From bc20cb49aeb5d018698a9ca4c870116b772e2722 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Thu, 10 Aug 2017 16:56:40 -0700 Subject: [PATCH 221/362] Add 'netbase' package needed by eventlet package --- runtime-image/resources/apt-packages.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/runtime-image/resources/apt-packages.txt b/runtime-image/resources/apt-packages.txt index 4bb6d6fc..1fa11bcb 100644 --- a/runtime-image/resources/apt-packages.txt +++ b/runtime-image/resources/apt-packages.txt @@ -36,3 +36,5 @@ libsasl2-2 libsasl2-dev libsasl2-modules sasl2-bin +# Needed by eventlet +netbase From 50e51f682cd21d874af9361efe8cb4423cb87a6a Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Thu, 10 Aug 2017 17:04:12 -0700 Subject: [PATCH 222/362] Add compatibility test for the 'eventlet' Python package. The eventlet worker for gunicorn dies if /etc/protocols isn't present, which was previously the case. We now install the Debian 'netbase' package which provides this file and others. --- build.sh | 4 ++++ cloudbuild.yaml | 4 ++++ cloudbuild_library_tests.yaml | 4 ++++ tests/eventlet/.gitignore | 2 ++ tests/eventlet/Dockerfile.in | 13 +++++++++++++ tests/eventlet/README.md | 1 + tests/eventlet/requirements.txt | 10 ++++++++++ 7 files changed, 38 insertions(+) create mode 100644 tests/eventlet/.gitignore create mode 100644 tests/eventlet/Dockerfile.in create mode 100644 tests/eventlet/README.md create mode 100644 tests/eventlet/requirements.txt diff --git a/build.sh b/build.sh index 82d8c05e..2f0595e9 100755 --- a/build.sh +++ b/build.sh @@ -150,6 +150,7 @@ for outfile in \ python-interpreter-builder/Dockerfile \ runtime-image/Dockerfile \ tests/benchmark/Dockerfile \ + tests/eventlet/Dockerfile \ tests/google-cloud-python/Dockerfile \ tests/google-cloud-python-system/Dockerfile \ tests/integration/Dockerfile \ @@ -167,6 +168,9 @@ for file in \ cp -a "${file}" "builder/gen-dockerfile/${file##scripts/}" done +# Make a file available to the eventlet test. +cp -a scripts/testdata/hello_world/main.py tests/eventlet/main.py + # Build images and push to GCR if [ "${build}" -eq 1 ]; then echo "Building images" diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 187884dc..68b5c5bb 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -25,6 +25,10 @@ steps: '--config', '/workspace/tests/license-test/license-test.yaml', '-v' ] +- # Run compatibility tests + name: gcr.io/cloud-builders/docker:latest + args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/eventlet:${_TAG}', + '--no-cache', '/workspace/tests/eventlet/'] - # Build image to run google client library unit tests name: gcr.io/cloud-builders/docker:latest args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/google-cloud-python:${_TAG}', diff --git a/cloudbuild_library_tests.yaml b/cloudbuild_library_tests.yaml index 846348fe..f547d73f 100644 --- a/cloudbuild_library_tests.yaml +++ b/cloudbuild_library_tests.yaml @@ -12,5 +12,9 @@ steps: # Avoid warning about unused substitutions 'UNUSED1=${_BUILDER_DOCKER_NAMESPACE}', ] +- # Run compatibility tests + name: gcr.io/cloud-builders/docker:latest + args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/eventlet:${_TAG}', + '--no-cache', '/workspace/tests/eventlet/'] images: [ ] diff --git a/tests/eventlet/.gitignore b/tests/eventlet/.gitignore new file mode 100644 index 00000000..14be8077 --- /dev/null +++ b/tests/eventlet/.gitignore @@ -0,0 +1,2 @@ +Dockerfile +main.py diff --git a/tests/eventlet/Dockerfile.in b/tests/eventlet/Dockerfile.in new file mode 100644 index 00000000..fa65a236 --- /dev/null +++ b/tests/eventlet/Dockerfile.in @@ -0,0 +1,13 @@ +FROM ${STAGING_IMAGE} +LABEL python_version=python3.6 +RUN virtualenv --no-download /env -p python3.6 + +# Set virtualenv environment variables. This is equivalent to running +# source /env/bin/activate +ENV VIRTUAL_ENV /env +ENV PATH /env/bin:$PATH +ADD requirements.txt /app/ +RUN pip install -r requirements.txt +ADD . /app/ +RUN gunicorn -k eventlet -b :$PORT --daemon main:app ; \ + wget --retry-connrefused --tries=5 http://localhost:$PORT/ diff --git a/tests/eventlet/README.md b/tests/eventlet/README.md new file mode 100644 index 00000000..0c2a969c --- /dev/null +++ b/tests/eventlet/README.md @@ -0,0 +1 @@ +# Test the Python base image against the 'eventlet' library diff --git a/tests/eventlet/requirements.txt b/tests/eventlet/requirements.txt new file mode 100644 index 00000000..c33e53f2 --- /dev/null +++ b/tests/eventlet/requirements.txt @@ -0,0 +1,10 @@ +click==6.7 +enum-compat==0.0.2 +eventlet==0.21.0 +Flask==0.12.2 +greenlet==0.4.12 +gunicorn==19.7.1 +itsdangerous==0.24 +Jinja2==2.9.6 +MarkupSafe==1.0 +Werkzeug==0.12.2 From d287c50143d05dee1fe3ba9b3d93fd1f61948848 Mon Sep 17 00:00:00 2001 From: Angela Li Date: Mon, 21 Aug 2017 16:57:44 -0700 Subject: [PATCH 223/362] Add pandas-gbq to clientlibs download metrics (#141) * Add pandas-gbq to clientlibs download metrics * Create table for third party client libs --- .../python_clientlibs_download.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/perf-dashboard/clientlibs-download/python_clientlibs_download.py b/perf-dashboard/clientlibs-download/python_clientlibs_download.py index fcfb048e..350eb7ed 100644 --- a/perf-dashboard/clientlibs-download/python_clientlibs_download.py +++ b/perf-dashboard/clientlibs-download/python_clientlibs_download.py @@ -28,7 +28,14 @@ VENEER_TABLE_NAME = 'veneer_client_libs' STACKDRIVER_TABLE_NAME = 'stackdriver_client_libs' GRPC_TABLE_NAME = 'grpc_lib' -TABLES = [VENEER_TABLE_NAME, GRPC_TABLE_NAME, STACKDRIVER_TABLE_NAME] +THIRD_PARTY_TABLE_NAME = 'third_party_client_libs' + +TABLES = [ + VENEER_TABLE_NAME, + GRPC_TABLE_NAME, + STACKDRIVER_TABLE_NAME, + THIRD_PARTY_TABLE_NAME, +] CLIENTLIBS = { VENEER_TABLE_NAME: [ @@ -54,6 +61,9 @@ GRPC_TABLE_NAME: [ 'grpcio', ], + THIRD_PARTY_TABLE_NAME: [ + 'pandas-gbq', + ] } From f89776254ebcf82c4770ba30e79e7d3458492d75 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Tue, 22 Aug 2017 18:02:52 -0700 Subject: [PATCH 224/362] Update Python interpreter from 3.5.3 to 3.5.4 --- python-interpreter-builder/Dockerfile.in | 1 - .../patches/3.5/double-build.diff | 96 ------------------- python-interpreter-builder/patches/3.5/series | 1 - .../scripts/build-python-3.5.sh | 14 ++- tests/virtualenv/virtualenv_python35.yaml | 2 +- 5 files changed, 7 insertions(+), 107 deletions(-) delete mode 100644 python-interpreter-builder/patches/3.5/double-build.diff delete mode 100644 python-interpreter-builder/patches/3.5/series diff --git a/python-interpreter-builder/Dockerfile.in b/python-interpreter-builder/Dockerfile.in index ee9fecf3..83640de3 100644 --- a/python-interpreter-builder/Dockerfile.in +++ b/python-interpreter-builder/Dockerfile.in @@ -29,7 +29,6 @@ RUN apt-get update && apt-get install -yq \ net-tools \ netbase \ python3 \ - quilt \ sharutils \ time \ tk-dev \ diff --git a/python-interpreter-builder/patches/3.5/double-build.diff b/python-interpreter-builder/patches/3.5/double-build.diff deleted file mode 100644 index cb52c64b..00000000 --- a/python-interpreter-builder/patches/3.5/double-build.diff +++ /dev/null @@ -1,96 +0,0 @@ -# Source is https://github.com/python/cpython/pull/1478 - -Index: Python-3.5.3/Makefile.pre.in -=================================================================== ---- Python-3.5.3.orig/Makefile.pre.in -+++ Python-3.5.3/Makefile.pre.in -@@ -982,7 +982,7 @@ TESTTIMEOUT= 3600 - - # Run a basic set of regression tests. - # This excludes some tests that are particularly resource-intensive. --test: all platform -+test: @DEF_MAKE_RULE@ platform - $(TESTRUNNER) $(TESTOPTS) - - # Run the full test suite twice - once without .pyc files, and once with. -@@ -992,7 +992,7 @@ test: all platform - # the bytecode read from a .pyc file had the bug, sometimes the directly - # generated bytecode. This is sometimes a very shy bug needing a lot of - # sample data. --testall: all platform -+testall: @DEF_MAKE_RULE@ platform - -find $(srcdir)/Lib -name '*.py[co]' -print | xargs rm -f - $(TESTPYTHON) -E $(srcdir)/Lib/compileall.py - -find $(srcdir)/Lib -name '*.py[co]' -print | xargs rm -f -@@ -1001,7 +1001,7 @@ testall: all platform - - # Run the test suite for both architectures in a Universal build on OSX. - # Must be run on an Intel box. --testuniversal: all platform -+testuniversal: @DEF_MAKE_RULE@ platform - if [ `arch` != 'i386' ];then \ - echo "This can only be used on OSX/i386" ;\ - exit 1 ;\ -@@ -1024,7 +1024,7 @@ QUICKTESTOPTS= $(TESTOPTS) -x test_subpr - test_multiprocessing_forkserver \ - test_mailbox test_socket test_poll \ - test_select test_zipfile test_concurrent_futures --quicktest: all platform -+quicktest: @DEF_MAKE_RULE@ platform - $(TESTRUNNER) $(QUICKTESTOPTS) - - -@@ -1376,7 +1376,7 @@ LIBPL= @LIBPL@ - # pkgconfig directory - LIBPC= $(LIBDIR)/pkgconfig - --libainstall: all python-config -+libainstall: @DEF_MAKE_RULE@ python-config - @for i in $(LIBDIR) $(LIBPL) $(LIBPC); \ - do \ - if test ! -d $(DESTDIR)$$i; then \ -@@ -1635,7 +1635,7 @@ distclean: clobber - -exec rm -f {} ';' - - # Check for smelly exported symbols (not starting with Py/_Py) --smelly: all -+smelly: @DEF_MAKE_RULE@ - nm -p $(LIBRARY) | \ - sed -n "/ [TDB] /s/.* //p" | grep -v "^_*Py" | sort -u; \ - -@@ -1673,7 +1673,7 @@ funny: - -o -print - - # Perform some verification checks on any modified files. --patchcheck: all -+patchcheck: @DEF_MAKE_RULE@ - $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/scripts/patchcheck.py - - # Dependencies -Index: Python-3.5.3/Misc/ACKS -=================================================================== ---- Python-3.5.3.orig/Misc/ACKS -+++ Python-3.5.3/Misc/ACKS -@@ -1092,6 +1092,7 @@ Jason Orendorff - Douglas Orr - William Orr - Michele Orrù -+Tomáš Orsava - Oleg Oshmyan - Denis S. Otkidach - Peter Otten -Index: Python-3.5.3/Misc/NEWS -=================================================================== ---- Python-3.5.3.orig/Misc/NEWS -+++ Python-3.5.3/Misc/NEWS -@@ -634,6 +634,10 @@ Windows - Build - ----- - -+- bpo-29243: Prevent unnecessary rebuilding of Python during ``make test``, -+ ``make install`` and some other make targets when configured with -+ ``--enable-optimizations``. -+ - - Issue #29080: Removes hard dependency on hg.exe from PCBuild/build.bat - - - Issue #23903: Added missed names to PC/python3.def. diff --git a/python-interpreter-builder/patches/3.5/series b/python-interpreter-builder/patches/3.5/series deleted file mode 100644 index f6c2876b..00000000 --- a/python-interpreter-builder/patches/3.5/series +++ /dev/null @@ -1 +0,0 @@ -double-build.diff diff --git a/python-interpreter-builder/scripts/build-python-3.5.sh b/python-interpreter-builder/scripts/build-python-3.5.sh index 09653669..f9113879 100755 --- a/python-interpreter-builder/scripts/build-python-3.5.sh +++ b/python-interpreter-builder/scripts/build-python-3.5.sh @@ -6,16 +6,14 @@ set -x # Get the source mkdir -p /opt/sources cd /opt/sources -wget --no-verbose https://www.python.org/ftp/python/3.5.3/Python-3.5.3.tgz +wget --no-verbose https://www.python.org/ftp/python/3.5.4/Python-3.5.4.tgz # SHA-256 generated via `shasum -a 256 [file]` shasum --check < Date: Thu, 24 Aug 2017 15:58:50 -0700 Subject: [PATCH 225/362] Fix Dockerfile reference to deleted directory. The 'patches' directory was implicitly removed in https://github.com/GoogleCloudPlatform/python-runtime/pull/142 when we deleted the last file in it. However, this wasn't found in testing because my local client still had a patches directory. --- python-interpreter-builder/Dockerfile.in | 1 - 1 file changed, 1 deletion(-) diff --git a/python-interpreter-builder/Dockerfile.in b/python-interpreter-builder/Dockerfile.in index 83640de3..eba153e9 100644 --- a/python-interpreter-builder/Dockerfile.in +++ b/python-interpreter-builder/Dockerfile.in @@ -43,7 +43,6 @@ ENV LANG C.UTF-8 # Add build scripts ADD scripts /scripts -ADD patches /patches # Build the Python interpreters RUN /scripts/build-python-3.5.sh From d4bc05472792ca5f419fdcce9eccbe85df23cd46 Mon Sep 17 00:00:00 2001 From: Angela Li Date: Mon, 28 Aug 2017 17:39:54 -0700 Subject: [PATCH 226/362] Add google-cloud-trace to clientlib metrics (#144) --- perf-dashboard/clientlibs-download/python_clientlibs_download.py | 1 + 1 file changed, 1 insertion(+) diff --git a/perf-dashboard/clientlibs-download/python_clientlibs_download.py b/perf-dashboard/clientlibs-download/python_clientlibs_download.py index 350eb7ed..53873e70 100644 --- a/perf-dashboard/clientlibs-download/python_clientlibs_download.py +++ b/perf-dashboard/clientlibs-download/python_clientlibs_download.py @@ -57,6 +57,7 @@ 'google-cloud-logging', 'google-cloud-monitoring', 'google-cloud-error_reporting', + 'google-cloud-trace', ], GRPC_TABLE_NAME: [ 'grpcio', From 3c145063fbbc0f8fd5c5a2219ce8ab529f7ba420 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Tue, 29 Aug 2017 15:20:49 -0700 Subject: [PATCH 227/362] Switch default Python 3 from 3.5 to 3.6 The gen_dockerfile_test.test_generate_dockerfile_golden fails, as expected, until the corresponding XRT change is released in gcloud. --- builder/gen-dockerfile/Dockerfile.in | 4 ++-- scripts/gen_dockerfile.py | 2 +- scripts/gen_dockerfile_test.py | 5 ++++- scripts/testdata/hello_world_golden/Dockerfile | 4 ++-- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/builder/gen-dockerfile/Dockerfile.in b/builder/gen-dockerfile/Dockerfile.in index e2fe4463..4f6447eb 100644 --- a/builder/gen-dockerfile/Dockerfile.in +++ b/builder/gen-dockerfile/Dockerfile.in @@ -1,6 +1,6 @@ FROM ${STAGING_IMAGE} -LABEL python_version=python3.5 -RUN virtualenv --no-download /env -p python3.5 +LABEL python_version=python3.6 +RUN virtualenv --no-download /env -p python3.6 # Set virtualenv environment variables. This is equivalent to running # source /env/bin/activate diff --git a/scripts/gen_dockerfile.py b/scripts/gen_dockerfile.py index 5514b092..eaff183f 100755 --- a/scripts/gen_dockerfile.py +++ b/scripts/gen_dockerfile.py @@ -51,7 +51,7 @@ PYTHON_INTERPRETER_VERSION_MAP = { '': '', # == 2.7 '2': '', # == 2.7 - '3': '3.5', + '3': '3.6', '3.4': '3.4', '3.5': '3.5', '3.6': '3.6', diff --git a/scripts/gen_dockerfile_test.py b/scripts/gen_dockerfile_test.py index 31ef7d42..b52e15e8 100755 --- a/scripts/gen_dockerfile_test.py +++ b/scripts/gen_dockerfile_test.py @@ -63,7 +63,7 @@ def compare_file(filename, dir1, dir2): 'dockerfile_python_version': '', }), ('runtime_config:\n python_version: 3', { - 'dockerfile_python_version': '3.5', + 'dockerfile_python_version': '3.6', }), ('runtime_config:\n python_version: 3.4', { 'dockerfile_python_version': '3.4', @@ -71,6 +71,9 @@ def compare_file(filename, dir1, dir2): ('runtime_config:\n python_version: 3.5', { 'dockerfile_python_version': '3.5', }), + ('runtime_config:\n python_version: 3.6', { + 'dockerfile_python_version': '3.6', + }), # entrypoint present ('entrypoint: my entrypoint', { 'entrypoint': 'exec my entrypoint', diff --git a/scripts/testdata/hello_world_golden/Dockerfile b/scripts/testdata/hello_world_golden/Dockerfile index e3e0563c..10396399 100644 --- a/scripts/testdata/hello_world_golden/Dockerfile +++ b/scripts/testdata/hello_world_golden/Dockerfile @@ -1,6 +1,6 @@ FROM gcr.io/google-appengine/python -LABEL python_version=python3.5 -RUN virtualenv --no-download /env -p python3.5 +LABEL python_version=python3.6 +RUN virtualenv --no-download /env -p python3.6 # Set virtualenv environment variables. This is equivalent to running # source /env/bin/activate From db3c95817da35d45a06f2e7d66419476873048c2 Mon Sep 17 00:00:00 2001 From: Douglas Greiman Date: Thu, 31 Aug 2017 14:20:30 -0700 Subject: [PATCH 228/362] Revert "Switch default Python 3 from 3.5 to 3.6" --- builder/gen-dockerfile/Dockerfile.in | 4 ++-- scripts/gen_dockerfile.py | 2 +- scripts/gen_dockerfile_test.py | 5 +---- scripts/testdata/hello_world_golden/Dockerfile | 4 ++-- 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/builder/gen-dockerfile/Dockerfile.in b/builder/gen-dockerfile/Dockerfile.in index 4f6447eb..e2fe4463 100644 --- a/builder/gen-dockerfile/Dockerfile.in +++ b/builder/gen-dockerfile/Dockerfile.in @@ -1,6 +1,6 @@ FROM ${STAGING_IMAGE} -LABEL python_version=python3.6 -RUN virtualenv --no-download /env -p python3.6 +LABEL python_version=python3.5 +RUN virtualenv --no-download /env -p python3.5 # Set virtualenv environment variables. This is equivalent to running # source /env/bin/activate diff --git a/scripts/gen_dockerfile.py b/scripts/gen_dockerfile.py index eaff183f..5514b092 100755 --- a/scripts/gen_dockerfile.py +++ b/scripts/gen_dockerfile.py @@ -51,7 +51,7 @@ PYTHON_INTERPRETER_VERSION_MAP = { '': '', # == 2.7 '2': '', # == 2.7 - '3': '3.6', + '3': '3.5', '3.4': '3.4', '3.5': '3.5', '3.6': '3.6', diff --git a/scripts/gen_dockerfile_test.py b/scripts/gen_dockerfile_test.py index b52e15e8..31ef7d42 100755 --- a/scripts/gen_dockerfile_test.py +++ b/scripts/gen_dockerfile_test.py @@ -63,7 +63,7 @@ def compare_file(filename, dir1, dir2): 'dockerfile_python_version': '', }), ('runtime_config:\n python_version: 3', { - 'dockerfile_python_version': '3.6', + 'dockerfile_python_version': '3.5', }), ('runtime_config:\n python_version: 3.4', { 'dockerfile_python_version': '3.4', @@ -71,9 +71,6 @@ def compare_file(filename, dir1, dir2): ('runtime_config:\n python_version: 3.5', { 'dockerfile_python_version': '3.5', }), - ('runtime_config:\n python_version: 3.6', { - 'dockerfile_python_version': '3.6', - }), # entrypoint present ('entrypoint: my entrypoint', { 'entrypoint': 'exec my entrypoint', diff --git a/scripts/testdata/hello_world_golden/Dockerfile b/scripts/testdata/hello_world_golden/Dockerfile index 10396399..e3e0563c 100644 --- a/scripts/testdata/hello_world_golden/Dockerfile +++ b/scripts/testdata/hello_world_golden/Dockerfile @@ -1,6 +1,6 @@ FROM gcr.io/google-appengine/python -LABEL python_version=python3.6 -RUN virtualenv --no-download /env -p python3.6 +LABEL python_version=python3.5 +RUN virtualenv --no-download /env -p python3.5 # Set virtualenv environment variables. This is equivalent to running # source /env/bin/activate From 34c4d0f67a18b62fc286343370da48340926a887 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Wed, 6 Sep 2017 11:18:47 -0700 Subject: [PATCH 229/362] Auto-update dependencies. (#139) --- tests/integration/requirements.txt | 8 +-- tests/python2-libraries/requirements.txt | 62 ++++++++++++------------ tests/python3-libraries/requirements.txt | 62 ++++++++++++------------ 3 files changed, 66 insertions(+), 66 deletions(-) diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index 05779ae6..6e6be47e 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -1,7 +1,7 @@ Flask==0.12.2 -google-cloud-error-reporting==0.26.0 -google-cloud-logging==1.2.0 -google-cloud-monitoring==0.26.0 +google-cloud-error-reporting==0.27.0 +google-cloud-logging==1.3.0 +google-cloud-monitoring==0.27.0 gunicorn==19.7.1 -requests==2.18.3 +requests==2.18.4 retrying==1.3.3 diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 05a6dbe3..91807712 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -3,11 +3,11 @@ amqp==2.2.1 amqplib==1.0.2 ansible==2.3.2.0 anyjson==0.3.3 -apache-libcloud==2.1.0 +apache-libcloud==2.2.0 argparse==1.4.0 astroid==1.5.3 -awscli==1.11.131 -babel==2.4.0 +awscli==1.11.146 +babel==2.5.0 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 beautifulsoup4==4.6.0 @@ -16,7 +16,7 @@ billiard==3.5.0.3 blessings==1.6 blinker==1.4 boto==2.48.0 -botocore==1.5.94 +botocore==1.7.4 bottle==0.12.13 carbon==1.0.2 celery==4.1.0 @@ -25,22 +25,22 @@ cffi==1.10.0 chardet==3.0.4 click==6.7 cliff==2.8.0 -cmd2==0.7.5 +cmd2==0.7.7 colorama==0.3.9 configobj==5.0.6 cov-core==1.15.0 coverage==4.4.1 -coveralls==1.1 +coveralls==1.2.0 cryptography==2.0.3 cssselect==1.0.1 -cython==0.26 +cython==0.26.1 decorator==4.1.2 django-celery==3.2.1 django-debug-toolbar==1.8 -django-extensions==1.8.1 -django==1.11.4 +django-extensions==1.9.0 +django==1.11.5 django_compress==1.0.1 -djangorestframework==3.6.3 +djangorestframework==3.6.4 docker-py==1.10.6 docopt==0.6.2 docutils==0.14 @@ -49,7 +49,7 @@ elasticsearch==5.4.0 enum34==1.1.6 eventlet==0.21.0 extras==1.0.0 -fabric==1.13.2 +fabric==1.14.0 fixtures==3.0.0 flake8==3.4.1 flask==0.12.2 @@ -57,7 +57,7 @@ funcsigs==1.0.2 functools32==3.2.3.post2 futures==3.1.1 gevent==1.2.2 -google-api-python-client==1.6.2 +google-api-python-client==1.6.3 graphite-web==1.0.2 greenlet==0.4.12 gunicorn==19.7.1 @@ -79,7 +79,7 @@ lxml==3.8.0 m2crypto==0.26.0 mako==1.0.7 manifestparser==1.1 -markdown==2.6.8 +markdown==2.6.9 markupsafe==1.0 matplotlib==2.0.2 mccabe==0.6.1 @@ -96,7 +96,7 @@ mozprofile==0.28 mozrunner==6.13 msgpack-python==0.4.8 mysql-python==1.2.5 -ndg-httpsclient==0.4.2 +ndg-httpsclient==0.4.3 netaddr==0.7.19 netifaces==0.10.6 newrelic==2.90.0.75 @@ -106,7 +106,7 @@ oauth2==1.9.0.post1 oauth2client==4.1.2 oauthlib==2.0.2 ordereddict==1.1 -oslo.config==4.11.0 +oslo.config==4.12.0 pandas==0.20.3 paramiko==2.2.1 passlib==1.7.1 @@ -116,25 +116,25 @@ pastescript==2.0.2 pbr==3.1.1 pep8==1.7.0 pexpect==4.2.1 -pika==0.10.0 +pika==0.11.0 pillow==4.2.1 pip==9.0.1 prettytable -protobuf==3.3.0 -psutil==5.2.2 -psycopg2==2.7.3 +protobuf==3.4.0 +psutil==5.3.0 +psycopg2==2.7.3.1 py==1.4.34 -pyasn1-modules==0.0.11 -pyasn1==0.3.2 +pyasn1-modules==0.1.1 +pyasn1==0.3.3 pycparser==2.18 pycrypto==2.6.1 pycurl==7.43.0 pyflakes==1.6.0 pygments==2.2.0 -pyjwt==1.5.2 +pyjwt==1.5.3 pylibmc==1.5.2 pylint==1.7.2 -pymongo==3.5.0 +pymongo==3.5.1 pymysql==0.7.11 pyopenssl==17.2.0 pyparsing==2.2.0 @@ -156,16 +156,16 @@ pytz==2017.2 pyyaml==3.12 pyzmq==16.0.2 raven==6.1.0 -redis==2.10.5 +redis==2.10.6 repoze.lru==0.6 requests-oauthlib==0.8.0 -requests==2.18.3 +requests==2.18.4 retrying==1.3.3 rsa==3.4.2 scipy==0.19.1 -selenium==3.4.3 +selenium==3.5.0 setuptools-git==1.2 -setuptools==36.2.7 +setuptools==36.4.0 sh==1.12.14 simplejson==3.11.1 six==1.10.0 @@ -176,14 +176,14 @@ sqlalchemy-migrate==0.11.0 sqlalchemy==1.1.11 sqlparse==0.2.3 statsd==3.2.1 -stevedore==1.25.0 +stevedore==1.26.0 suds==0.4 supervisor==3.3.3 testrepository==0.0.20 testtools==2.3.0 thrift==0.10.0 -tornado==4.5.1 -tox==2.7.0 +tornado==4.5.2 +tox==2.8.1 twisted==17.5.0 ujson==1.35 unidecode==0.4.21 @@ -200,6 +200,6 @@ websocket-client==0.44.0 webtest==2.0.28 werkzeug==0.12.2 wheel==0.29.0 -xlrd==1.0.0 +xlrd==1.1.0 zc.buildout==2.9.4 zope.interface==4.4.2 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index f1e3539b..f2d6973b 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -3,11 +3,11 @@ amqp==2.2.1 amqplib==1.0.2 ansible==2.3.2.0 anyjson==0.3.3 -apache-libcloud==2.1.0 +apache-libcloud==2.2.0 argparse==1.4.0 astroid==1.5.3 -awscli==1.11.131 -babel==2.4.0 +awscli==1.11.146 +babel==2.5.0 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 beautifulsoup4==4.6.0 @@ -15,7 +15,7 @@ billiard==3.5.0.3 blessings==1.6 blinker==1.4 boto==2.48.0 -botocore==1.5.94 +botocore==1.7.4 bottle==0.12.13 celery==4.1.0 certifi==2017.7.27.1 @@ -23,22 +23,22 @@ cffi==1.10.0 chardet==3.0.4 click==6.7 cliff==2.8.0 -cmd2==0.7.5 +cmd2==0.7.7 colorama==0.3.9 configobj==5.0.6 cov-core==1.15.0 coverage==4.4.1 -coveralls==1.1 +coveralls==1.2.0 cryptography==2.0.3 cssselect==1.0.1 -cython==0.26 +cython==0.26.1 decorator==4.1.2 django-celery==3.2.1 django-debug-toolbar==1.8 -django-extensions==1.8.1 -django==1.11.4 +django-extensions==1.9.0 +django==1.11.5 django_compress==1.0.1 -djangorestframework==3.6.3 +djangorestframework==3.6.4 docker-py==1.10.6 docopt==0.6.2 docutils==0.14 @@ -47,14 +47,14 @@ elasticsearch==5.4.0 enum34==1.1.6 eventlet==0.21.0 extras==1.0.0 -fabric==1.13.2 +fabric==1.14.0 fixtures==3.0.0 flake8==3.4.1 flask==0.12.2 funcsigs==1.0.2 futures==3.1.1 gevent==1.2.2 -google-api-python-client==1.6.2 +google-api-python-client==1.6.3 greenlet==0.4.12 gunicorn==19.7.1 hiredis==0.2.0 @@ -76,7 +76,7 @@ lxml==3.8.0 m2crypto==0.26.0 mako==1.0.7 manifestparser==1.1 -markdown==2.6.8 +markdown==2.6.9 markupsafe==1.0 matplotlib==2.0.2 mccabe==0.6.1 @@ -90,7 +90,7 @@ mozlog==3.5 moznetwork==0.27 mozprocess==0.25 msgpack-python==0.4.8 -ndg-httpsclient==0.4.2 +ndg-httpsclient==0.4.3 netaddr==0.7.19 netifaces==0.10.6 newrelic==2.90.0.75 @@ -100,7 +100,7 @@ oauth2==1.9.0.post1 oauth2client==4.1.2 oauthlib==2.0.2 ordereddict==1.1 -oslo.config==4.11.0 +oslo.config==4.12.0 pandas==0.20.3 paramiko==2.2.1 passlib==1.7.1 @@ -110,24 +110,24 @@ pastescript==2.0.2 pbr==3.1.1 pep8==1.7.0 pexpect==4.2.1 -pika==0.10.0 +pika==0.11.0 pillow==4.2.1 pip==9.0.1 prettytable -protobuf==3.3.0 -psutil==5.2.2 -psycopg2==2.7.3 +protobuf==3.4.0 +psutil==5.3.0 +psycopg2==2.7.3.1 py==1.4.34 -pyasn1-modules==0.0.11 -pyasn1==0.3.2 +pyasn1-modules==0.1.1 +pyasn1==0.3.3 pycparser==2.18 pycrypto==2.6.1 pyflakes==1.6.0 pygments==2.2.0 -pyjwt==1.5.2 +pyjwt==1.5.3 pylibmc==1.5.2 pylint==1.7.2 -pymongo==3.5.0 +pymongo==3.5.1 pymysql==0.7.11 pyopenssl==17.2.0 pyparsing==2.2.0 @@ -148,16 +148,16 @@ pytz==2017.2 pyyaml==3.12 pyzmq==16.0.2 raven==6.1.0 -redis==2.10.5 +redis==2.10.6 repoze.lru==0.6 requests-oauthlib==0.8.0 -requests==2.18.3 +requests==2.18.4 retrying==1.3.3 rsa==3.4.2 scipy==0.19.1 -selenium==3.4.3 +selenium==3.5.0 setuptools-git==1.2 -setuptools==36.2.7 +setuptools==36.4.0 sh==1.12.14 simplejson==3.11.1 six==1.10.0 @@ -168,12 +168,12 @@ sqlalchemy-migrate==0.11.0 sqlalchemy==1.1.11 sqlparse==0.2.3 statsd==3.2.1 -stevedore==1.25.0 +stevedore==1.26.0 testrepository==0.0.20 testtools==2.3.0 thrift==0.10.0 -tornado==4.5.1 -tox==2.7.0 +tornado==4.5.2 +tox==2.8.1 twisted==17.5.0 ujson==1.35 unidecode==0.4.21 @@ -190,6 +190,6 @@ websocket-client==0.44.0 webtest==2.0.28 werkzeug==0.12.2 wheel==0.29.0 -xlrd==1.0.0 +xlrd==1.1.0 zc.buildout==2.9.4 zope.interface==4.4.2 From 2e3a11e93f53f1f59a3d735410cbdbd0fbb9969e Mon Sep 17 00:00:00 2001 From: Angela Li Date: Wed, 6 Sep 2017 16:22:07 -0700 Subject: [PATCH 230/362] Update bigquery method to match the latest version (#147) --- .../clientlibs-download/python_clientlibs_download.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-dashboard/clientlibs-download/python_clientlibs_download.py b/perf-dashboard/clientlibs-download/python_clientlibs_download.py index 53873e70..1d307ac2 100644 --- a/perf-dashboard/clientlibs-download/python_clientlibs_download.py +++ b/perf-dashboard/clientlibs-download/python_clientlibs_download.py @@ -127,7 +127,7 @@ def get_weekly_clientlibs_downloads(clientlibs_table_name, date_str): wait_for_job(query_job) # Fetch the results - result = query_job.result().fetch_data() + result = query_job.query_results().fetch_data() result_list = [item for item in result] # In case the result_list contains the metadata like total_rows, the From 76d04c4c8251cf9f2b90b098365999c2606fba9e Mon Sep 17 00:00:00 2001 From: Douglas Greiman Date: Wed, 27 Sep 2017 15:54:42 -0700 Subject: [PATCH 231/362] Revert "Revert "Switch default Python 3 from 3.5 to 3.6"" --- builder/gen-dockerfile/Dockerfile.in | 4 ++-- scripts/gen_dockerfile.py | 2 +- scripts/gen_dockerfile_test.py | 5 ++++- scripts/testdata/hello_world_golden/Dockerfile | 4 ++-- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/builder/gen-dockerfile/Dockerfile.in b/builder/gen-dockerfile/Dockerfile.in index e2fe4463..4f6447eb 100644 --- a/builder/gen-dockerfile/Dockerfile.in +++ b/builder/gen-dockerfile/Dockerfile.in @@ -1,6 +1,6 @@ FROM ${STAGING_IMAGE} -LABEL python_version=python3.5 -RUN virtualenv --no-download /env -p python3.5 +LABEL python_version=python3.6 +RUN virtualenv --no-download /env -p python3.6 # Set virtualenv environment variables. This is equivalent to running # source /env/bin/activate diff --git a/scripts/gen_dockerfile.py b/scripts/gen_dockerfile.py index 5514b092..eaff183f 100755 --- a/scripts/gen_dockerfile.py +++ b/scripts/gen_dockerfile.py @@ -51,7 +51,7 @@ PYTHON_INTERPRETER_VERSION_MAP = { '': '', # == 2.7 '2': '', # == 2.7 - '3': '3.5', + '3': '3.6', '3.4': '3.4', '3.5': '3.5', '3.6': '3.6', diff --git a/scripts/gen_dockerfile_test.py b/scripts/gen_dockerfile_test.py index 31ef7d42..b52e15e8 100755 --- a/scripts/gen_dockerfile_test.py +++ b/scripts/gen_dockerfile_test.py @@ -63,7 +63,7 @@ def compare_file(filename, dir1, dir2): 'dockerfile_python_version': '', }), ('runtime_config:\n python_version: 3', { - 'dockerfile_python_version': '3.5', + 'dockerfile_python_version': '3.6', }), ('runtime_config:\n python_version: 3.4', { 'dockerfile_python_version': '3.4', @@ -71,6 +71,9 @@ def compare_file(filename, dir1, dir2): ('runtime_config:\n python_version: 3.5', { 'dockerfile_python_version': '3.5', }), + ('runtime_config:\n python_version: 3.6', { + 'dockerfile_python_version': '3.6', + }), # entrypoint present ('entrypoint: my entrypoint', { 'entrypoint': 'exec my entrypoint', diff --git a/scripts/testdata/hello_world_golden/Dockerfile b/scripts/testdata/hello_world_golden/Dockerfile index e3e0563c..10396399 100644 --- a/scripts/testdata/hello_world_golden/Dockerfile +++ b/scripts/testdata/hello_world_golden/Dockerfile @@ -1,6 +1,6 @@ FROM gcr.io/google-appengine/python -LABEL python_version=python3.5 -RUN virtualenv --no-download /env -p python3.5 +LABEL python_version=python3.6 +RUN virtualenv --no-download /env -p python3.6 # Set virtualenv environment variables. This is equivalent to running # source /env/bin/activate From 1d4e6df0f8780ea2be67c9a2fe0a01b9c7927e3c Mon Sep 17 00:00:00 2001 From: Douglas Greiman Date: Mon, 2 Oct 2017 09:37:27 -0700 Subject: [PATCH 232/362] Add missing dependency (was using a too-old version of six in some cases). (#151) --- tests/integration/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index 6e6be47e..eae15fe9 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -5,3 +5,4 @@ google-cloud-monitoring==0.27.0 gunicorn==19.7.1 requests==2.18.4 retrying==1.3.3 +six==1.11.0 From 18397afdf4f8fb7e81310ff55951063830f80546 Mon Sep 17 00:00:00 2001 From: Angela Li Date: Fri, 13 Oct 2017 10:42:29 -0700 Subject: [PATCH 233/362] Add Stackoverflow metrics (#149) --- perf_dashboard/bq_utils.py | 46 ++++++++ .../python_clientlibs_download.py | 57 +++------- perf_dashboard/stackoverflow/posts_stats.py | 107 ++++++++++++++++++ 3 files changed, 166 insertions(+), 44 deletions(-) create mode 100644 perf_dashboard/bq_utils.py rename {perf-dashboard/clientlibs-download => perf_dashboard/clientlibs_download}/python_clientlibs_download.py (71%) create mode 100644 perf_dashboard/stackoverflow/posts_stats.py diff --git a/perf_dashboard/bq_utils.py b/perf_dashboard/bq_utils.py new file mode 100644 index 00000000..d9441f07 --- /dev/null +++ b/perf_dashboard/bq_utils.py @@ -0,0 +1,46 @@ +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Common util methods for processing data in BigQuery.""" + +import uuid + +from google.cloud import bigquery + + +def insert_rows(project, dataset_name, table_name, rows): + """Insert rows to bigquery table.""" + client = bigquery.Client(project=project) + dataset = client.dataset(dataset_name) + table = bigquery.Table(table_name, dataset) + table.reload() + table.insert_data(rows) + + +def execute_query(query): + """Execute query and return the query results.""" + client = bigquery.Client() + query_job = client.run_async_query(str(uuid.uuid4()), query) + query_job.use_legacy_sql = False + + # Start the query job and wait it to complete + query_job.begin() + query_job.result() + + # Get the results + destination_table = query_job.destination + destination_table.reload() + results = destination_table.fetch_data() + + return results diff --git a/perf-dashboard/clientlibs-download/python_clientlibs_download.py b/perf_dashboard/clientlibs_download/python_clientlibs_download.py similarity index 71% rename from perf-dashboard/clientlibs-download/python_clientlibs_download.py rename to perf_dashboard/clientlibs_download/python_clientlibs_download.py index 1d307ac2..3c8f79fe 100644 --- a/perf-dashboard/clientlibs-download/python_clientlibs_download.py +++ b/perf_dashboard/clientlibs_download/python_clientlibs_download.py @@ -14,11 +14,16 @@ import datetime import os +import sys import time import uuid from google.cloud import bigquery +sys.path.insert(0, os.path.abspath(__file__+"/../../..")) +from perf_dashboard import bq_utils + + GCLOUD_PROJECT_ENV = 'GCLOUD_PROJECT' DATETIME_FORMAT = '%Y%m%d' @@ -68,17 +73,6 @@ } -def wait_for_job(job): - """Wait for the query job to complete.""" - while True: - job.reload() # Refreshes the state via a GET request. - if job.state == 'DONE': - if job.error_result: - raise RuntimeError(job.errors) - return - time.sleep(1) - - def get_weekly_clientlibs_downloads(clientlibs_table_name, date_str): """Use a SQL query to collect the weekly download data of the client libraries. @@ -124,50 +118,25 @@ def get_weekly_clientlibs_downloads(clientlibs_table_name, date_str): # Start the query job and wait it to complete query_job.begin() - wait_for_job(query_job) - - # Fetch the results - result = query_job.query_results().fetch_data() - result_list = [item for item in result] + query_job.result() - # In case the result_list contains the metadata like total_rows, the - # actual rows will be the first element of the result_list. - if len(result_list) > 0 and isinstance(result_list[0], list): - result_list = result_list[0] + # Get the results + destination_table = query_job.destination + destination_table.reload() + results = destination_table.fetch_data() - rows = [(date_time,) + row for row in result_list] - print(rows) + rows = [(date_time,) + row for row in results] return rows -def insert_rows(dataset_name, table_name, rows): - """Insert rows to a bigquery table. - - Args: - dataset_name (str): Name of the dataset that holds the tables. - table_name (str): Name of the bigquery table. - rows (list): The rows that going to be inserted into the table. - - Returns: - list: Empty if inserted successfully, else the errors when inserting - each row. - """ - project = os.environ.get(GCLOUD_PROJECT_ENV) - client = bigquery.Client(project=project) - dataset = client.dataset(dataset_name) - table = bigquery.Table(name=table_name, dataset=dataset) - table.reload() - error = table.insert_data(rows) - return error - - def main(): for table_name in CLIENTLIBS.keys(): rows = get_weekly_clientlibs_downloads( clientlibs_table_name=table_name, date_str=datetime.datetime.now().strftime("%Y%m%d")) - insert_rows( + bq_utils.insert_rows( + project=project, dataset_name=DATASET_NAME, table_name=table_name, rows=rows) diff --git a/perf_dashboard/stackoverflow/posts_stats.py b/perf_dashboard/stackoverflow/posts_stats.py new file mode 100644 index 00000000..280fd42c --- /dev/null +++ b/perf_dashboard/stackoverflow/posts_stats.py @@ -0,0 +1,107 @@ +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A script to collect the number of StackOverflow posts related to +Python and Google Cloud Platform.""" + +import datetime +import os +import sys +import time +import uuid + +from collections import Counter + +from google.cloud import bigquery + +# Need this to import the local helper function +sys.path.insert(0, os.path.abspath(__file__+"/../../..")) +from perf_dashboard import bq_utils + +GCLOUD_PROJECT_ENV = 'GCLOUD_PROJECT' +DATASET_NAME = 'stackoverflow' +TAG_COUNT_TABLE_NAME = 'tag_count_timestamp' +UNANSWERED_POSTS_TABLE_NAME = 'unanswered_posts' + + +def get_stackoverflow_tags_count(): + """Get all the tags contains python and cloud key words""" + query = """ + SELECT + SPLIT(tags, '|') tags + FROM + `bigquery-public-data.stackoverflow.posts_questions` + WHERE + tags LIKE '%python%' + AND (tags LIKE '%google-cloud-platform%' OR tags LIKE '%gcp%') + """ + + results = bq_utils.execute_query(query) + + rows = [row[0] for row in results] + + return rows + + +def get_posts_list_unanswered(): + # Get the list of posts that are unanswered + query = """ + SELECT + id, title, tags + FROM + `bigquery-public-data.stackoverflow.posts_questions` + WHERE + tags LIKE '%python%' + AND (tags LIKE '%google-cloud-platform%' OR tags LIKE '%gcp%') + AND accepted_answer_id is NULL + AND answer_count = 0; + """ + + results = bq_utils.execute_query(query) + + # Add current timestamp to the rows + date_time = datetime.datetime.now() + rows = [(date_time,) + row for row in results] + + return rows + + +def count_unique_tags(data): + flattened_tag_list = [tag for tag_list in data for tag in tag_list] + tag_count = Counter(flattened_tag_list) + + # Add current timestamp to the rows + date_time = datetime.datetime.now() + time_tag_count = [(date_time,) + item for item in tag_count.items()] + + return time_tag_count + + +def main(): + project = os.environ.get(GCLOUD_PROJECT_ENV) + + # Get the posts count for each tag + rows = get_stackoverflow_tags_count() + tag_count = count_unique_tags(rows) + bq_utils.insert_rows( + project, DATASET_NAME, TAG_COUNT_TABLE_NAME, tag_count) + + # Get the list of unanswered posts + unanswered_posts = get_posts_list_unanswered() + bq_utils.insert_rows( + project, DATASET_NAME, UNANSWERED_POSTS_TABLE_NAME, unanswered_posts) + + +if __name__ == '__main__': + main() From fed4c56915fe360cc98f6e9f144098d3ed6c4f78 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Mon, 16 Oct 2017 16:40:07 -0700 Subject: [PATCH 234/362] Auto-update dependencies. (#148) --- scripts/requirements-test.txt | 2 +- tests/python2-libraries/requirements.txt | 92 +++++++++++------------ tests/python3-libraries/requirements.txt | 94 ++++++++++++------------ 3 files changed, 94 insertions(+), 94 deletions(-) diff --git a/scripts/requirements-test.txt b/scripts/requirements-test.txt index 6e01a83d..5afb5315 100644 --- a/scripts/requirements-test.txt +++ b/scripts/requirements-test.txt @@ -1,4 +1,4 @@ flask==0.12.2 -pytest==3.2.1 +pytest==3.2.3 pytest-cov==2.5.1 pyyaml==3.12 diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 91807712..87e9e53a 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -1,13 +1,13 @@ -alembic==0.9.5 -amqp==2.2.1 +alembic==0.9.6 +amqp==2.2.2 amqplib==1.0.2 -ansible==2.3.2.0 +ansible==2.4.0.0 anyjson==0.3.3 -apache-libcloud==2.2.0 +apache-libcloud==2.2.1 argparse==1.4.0 astroid==1.5.3 -awscli==1.11.146 -babel==2.5.0 +awscli==1.11.170 +babel==2.5.1 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 beautifulsoup4==4.6.0 @@ -16,31 +16,31 @@ billiard==3.5.0.3 blessings==1.6 blinker==1.4 boto==2.48.0 -botocore==1.7.4 +botocore==1.7.28 bottle==0.12.13 carbon==1.0.2 celery==4.1.0 certifi==2017.7.27.1 -cffi==1.10.0 +cffi==1.11.2 chardet==3.0.4 click==6.7 -cliff==2.8.0 +cliff==2.9.1 cmd2==0.7.7 colorama==0.3.9 configobj==5.0.6 cov-core==1.15.0 coverage==4.4.1 coveralls==1.2.0 -cryptography==2.0.3 +cryptography==2.1.1 cssselect==1.0.1 -cython==0.26.1 +cython==0.27.1 decorator==4.1.2 django-celery==3.2.1 django-debug-toolbar==1.8 -django-extensions==1.9.0 -django==1.11.5 +django-extensions==1.9.1 +django==1.11.6 django_compress==1.0.1 -djangorestframework==3.6.4 +djangorestframework==3.7.0 docker-py==1.10.6 docopt==0.6.2 docutils==0.14 @@ -57,7 +57,7 @@ funcsigs==1.0.2 functools32==3.2.3.post2 futures==3.1.1 gevent==1.2.2 -google-api-python-client==1.6.3 +google-api-python-client==1.6.4 graphite-web==1.0.2 greenlet==0.4.12 gunicorn==19.7.1 @@ -67,7 +67,7 @@ httplib2==0.10.3 idna==2.6 ipaddress==1.0.18 iso8601==0.1.12 -isodate==0.5.4 +isodate==0.6.0 itsdangerous==0.24 jinja2==2.9.6 jmespath==0.9.3 @@ -75,13 +75,13 @@ jsonschema==2.6.0 kombu==4.1.0 linecache2==1.0.0 logilab-common==1.4.1 -lxml==3.8.0 -m2crypto==0.26.0 +lxml==4.1.0 +m2crypto==0.27.0 mako==1.0.7 manifestparser==1.1 markdown==2.6.9 markupsafe==1.0 -matplotlib==2.0.2 +matplotlib==2.1.0 mccabe==0.6.1 meld3==1.0.2 mock==2.0.0 @@ -99,16 +99,16 @@ mysql-python==1.2.5 ndg-httpsclient==0.4.3 netaddr==0.7.19 netifaces==0.10.6 -newrelic==2.90.0.75 +newrelic==2.94.0.79 nose==1.3.7 -numpy==1.13.1 +numpy==1.13.3 oauth2==1.9.0.post1 oauth2client==4.1.2 -oauthlib==2.0.2 +oauthlib==2.0.4 ordereddict==1.1 -oslo.config==4.12.0 +oslo.config==4.13.1 pandas==0.20.3 -paramiko==2.2.1 +paramiko==2.3.1 passlib==1.7.1 paste==2.0.3 pastedeploy==1.5.2 @@ -117,15 +117,15 @@ pbr==3.1.1 pep8==1.7.0 pexpect==4.2.1 pika==0.11.0 -pillow==4.2.1 +pillow==4.3.0 pip==9.0.1 prettytable protobuf==3.4.0 -psutil==5.3.0 +psutil==5.4.0 psycopg2==2.7.3.1 py==1.4.34 -pyasn1-modules==0.1.1 -pyasn1==0.3.3 +pyasn1-modules==0.1.5 +pyasn1==0.3.7 pycparser==2.18 pycrypto==2.6.1 pycurl==7.43.0 @@ -133,15 +133,15 @@ pyflakes==1.6.0 pygments==2.2.0 pyjwt==1.5.3 pylibmc==1.5.2 -pylint==1.7.2 +pylint==1.7.4 pymongo==3.5.1 pymysql==0.7.11 -pyopenssl==17.2.0 +pyopenssl==17.3.0 pyparsing==2.2.0 pyramid==1.9.1 pystache==0.5.4 pytest-cov==2.5.1 -pytest==3.2.1 +pytest==3.2.3 python-cjson==1.2.1 python-daemon==2.1.2 python-dateutil==2.6.1 @@ -155,36 +155,36 @@ python-swiftclient==3.4.0 pytz==2017.2 pyyaml==3.12 pyzmq==16.0.2 -raven==6.1.0 +raven==6.2.1 redis==2.10.6 -repoze.lru==0.6 +repoze.lru==0.7 requests-oauthlib==0.8.0 requests==2.18.4 retrying==1.3.3 rsa==3.4.2 scipy==0.19.1 -selenium==3.5.0 +selenium==3.6.0 setuptools-git==1.2 -setuptools==36.4.0 +setuptools==36.6.0 sh==1.12.14 simplejson==3.11.1 -six==1.10.0 +six==1.11.0 snowballstemmer==1.2.1 south==1.0.2 -sphinx==1.6.3 +sphinx==1.6.4 sqlalchemy-migrate==0.11.0 -sqlalchemy==1.1.11 -sqlparse==0.2.3 +sqlalchemy==1.1.14 +sqlparse==0.2.4 statsd==3.2.1 -stevedore==1.26.0 +stevedore==1.27.1 suds==0.4 supervisor==3.3.3 testrepository==0.0.20 testtools==2.3.0 thrift==0.10.0 tornado==4.5.2 -tox==2.8.1 -twisted==17.5.0 +tox==2.9.1 +twisted==17.9.0 ujson==1.35 unidecode==0.4.21 unittest2==1.1.0 @@ -193,13 +193,13 @@ urllib3==1.22 uwsgi==2.0.15 versiontools==1.9.1 virtualenv==15.1.0 -waitress==1.0.2 +waitress==1.1.0 warlock==1.3.0 webob==1.7.3 websocket-client==0.44.0 webtest==2.0.28 werkzeug==0.12.2 -wheel==0.29.0 +wheel==0.30.0 xlrd==1.1.0 -zc.buildout==2.9.4 -zope.interface==4.4.2 +zc.buildout==2.9.5 +zope.interface==4.4.3 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index f2d6973b..614f2f37 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -1,13 +1,13 @@ -alembic==0.9.5 -amqp==2.2.1 +alembic==0.9.6 +amqp==2.2.2 amqplib==1.0.2 -ansible==2.3.2.0 +ansible==2.4.0.0 anyjson==0.3.3 -apache-libcloud==2.2.0 +apache-libcloud==2.2.1 argparse==1.4.0 astroid==1.5.3 -awscli==1.11.146 -babel==2.5.0 +awscli==1.11.170 +babel==2.5.1 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 beautifulsoup4==4.6.0 @@ -15,30 +15,30 @@ billiard==3.5.0.3 blessings==1.6 blinker==1.4 boto==2.48.0 -botocore==1.7.4 +botocore==1.7.28 bottle==0.12.13 celery==4.1.0 certifi==2017.7.27.1 -cffi==1.10.0 +cffi==1.11.2 chardet==3.0.4 click==6.7 -cliff==2.8.0 +cliff==2.9.1 cmd2==0.7.7 colorama==0.3.9 configobj==5.0.6 cov-core==1.15.0 coverage==4.4.1 coveralls==1.2.0 -cryptography==2.0.3 +cryptography==2.1.1 cssselect==1.0.1 -cython==0.26.1 +cython==0.27.1 decorator==4.1.2 django-celery==3.2.1 django-debug-toolbar==1.8 -django-extensions==1.9.0 -django==1.11.5 +django-extensions==1.9.1 +django==1.11.6 django_compress==1.0.1 -djangorestframework==3.6.4 +djangorestframework==3.7.0 docker-py==1.10.6 docopt==0.6.2 docutils==0.14 @@ -54,7 +54,7 @@ flask==0.12.2 funcsigs==1.0.2 futures==3.1.1 gevent==1.2.2 -google-api-python-client==1.6.3 +google-api-python-client==1.6.4 greenlet==0.4.12 gunicorn==19.7.1 hiredis==0.2.0 @@ -62,9 +62,9 @@ html5lib httplib2==0.10.3 idna==2.6 ipaddress==1.0.18 -ipython==6.1.0 +ipython==6.2.1 iso8601==0.1.12 -isodate==0.5.4 +isodate==0.6.0 itsdangerous==0.24 jinja2==2.9.6 jmespath==0.9.3 @@ -72,13 +72,13 @@ jsonschema==2.6.0 kombu==4.1.0 linecache2==1.0.0 logilab-common==1.4.1 -lxml==3.8.0 -m2crypto==0.26.0 +lxml==4.1.0 +m2crypto==0.27.0 mako==1.0.7 manifestparser==1.1 markdown==2.6.9 markupsafe==1.0 -matplotlib==2.0.2 +matplotlib==2.1.0 mccabe==0.6.1 meld3==1.0.2 mock==2.0.0 @@ -93,16 +93,16 @@ msgpack-python==0.4.8 ndg-httpsclient==0.4.3 netaddr==0.7.19 netifaces==0.10.6 -newrelic==2.90.0.75 +newrelic==2.94.0.79 nose==1.3.7 -numpy==1.13.1 +numpy==1.13.3 oauth2==1.9.0.post1 oauth2client==4.1.2 -oauthlib==2.0.2 +oauthlib==2.0.4 ordereddict==1.1 -oslo.config==4.12.0 +oslo.config==4.13.1 pandas==0.20.3 -paramiko==2.2.1 +paramiko==2.3.1 passlib==1.7.1 paste==2.0.3 pastedeploy==1.5.2 @@ -111,30 +111,30 @@ pbr==3.1.1 pep8==1.7.0 pexpect==4.2.1 pika==0.11.0 -pillow==4.2.1 +pillow==4.3.0 pip==9.0.1 prettytable protobuf==3.4.0 -psutil==5.3.0 +psutil==5.4.0 psycopg2==2.7.3.1 py==1.4.34 -pyasn1-modules==0.1.1 -pyasn1==0.3.3 +pyasn1-modules==0.1.5 +pyasn1==0.3.7 pycparser==2.18 pycrypto==2.6.1 pyflakes==1.6.0 pygments==2.2.0 pyjwt==1.5.3 pylibmc==1.5.2 -pylint==1.7.2 +pylint==1.7.4 pymongo==3.5.1 pymysql==0.7.11 -pyopenssl==17.2.0 +pyopenssl==17.3.0 pyparsing==2.2.0 pyramid==1.9.1 pystache==0.5.4 pytest-cov==2.5.1 -pytest==3.2.1 +pytest==3.2.3 python-daemon==2.1.2 python-dateutil==2.6.1 python-gflags==3.1.1 @@ -147,34 +147,34 @@ python-swiftclient==3.4.0 pytz==2017.2 pyyaml==3.12 pyzmq==16.0.2 -raven==6.1.0 +raven==6.2.1 redis==2.10.6 -repoze.lru==0.6 +repoze.lru==0.7 requests-oauthlib==0.8.0 requests==2.18.4 retrying==1.3.3 rsa==3.4.2 scipy==0.19.1 -selenium==3.5.0 +selenium==3.6.0 setuptools-git==1.2 -setuptools==36.4.0 +setuptools==36.6.0 sh==1.12.14 simplejson==3.11.1 -six==1.10.0 +six==1.11.0 snowballstemmer==1.2.1 south==1.0.2 -sphinx==1.6.3 +sphinx==1.6.4 sqlalchemy-migrate==0.11.0 -sqlalchemy==1.1.11 -sqlparse==0.2.3 +sqlalchemy==1.1.14 +sqlparse==0.2.4 statsd==3.2.1 -stevedore==1.26.0 +stevedore==1.27.1 testrepository==0.0.20 testtools==2.3.0 thrift==0.10.0 tornado==4.5.2 -tox==2.8.1 -twisted==17.5.0 +tox==2.9.1 +twisted==17.9.0 ujson==1.35 unidecode==0.4.21 unittest2==1.1.0 @@ -183,13 +183,13 @@ urllib3==1.22 uwsgi==2.0.15 versiontools==1.9.1 virtualenv==15.1.0 -waitress==1.0.2 +waitress==1.1.0 warlock==1.3.0 webob==1.7.3 websocket-client==0.44.0 webtest==2.0.28 werkzeug==0.12.2 -wheel==0.29.0 +wheel==0.30.0 xlrd==1.1.0 -zc.buildout==2.9.4 -zope.interface==4.4.2 +zc.buildout==2.9.5 +zope.interface==4.4.3 From 8df5ad6b9fa2c99ced22302d4867fdf73cf4d60b Mon Sep 17 00:00:00 2001 From: Angela Li Date: Fri, 20 Oct 2017 11:13:09 -0700 Subject: [PATCH 235/362] Upgrade setuptools in test (#154) --- tests/google-cloud-python/Dockerfile.in | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/google-cloud-python/Dockerfile.in b/tests/google-cloud-python/Dockerfile.in index a2873863..b82c18d1 100644 --- a/tests/google-cloud-python/Dockerfile.in +++ b/tests/google-cloud-python/Dockerfile.in @@ -4,6 +4,9 @@ FROM ${STAGING_IMAGE} RUN git clone --depth 1 https://github.com/GoogleCloudPlatform/google-cloud-python.git WORKDIR google-cloud-python +# Upgrade setuptools +RUN pip install --upgrade setuptools + # Install nox RUN pip install --upgrade nox-automation From 44af51158a5ca6eb5e7ec5ee4981613980d7c629 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 20 Oct 2017 15:36:14 -0700 Subject: [PATCH 236/362] Separate building and testing phases via --build and --test flags. Sometimes the tests fails for reasons not related to the runtime. This gives us a way to build and push the image anyway. --- build.sh | 72 +++++++++++++++++------------------ cloudbuild.yaml | 25 +----------- cloudbuild_benchmark.yaml | 4 -- cloudbuild_library_tests.yaml | 20 ---------- cloudbuild_system_tests.yaml | 4 -- cloudbuild_tests.yaml | 28 ++++++++++++++ 6 files changed, 65 insertions(+), 88 deletions(-) delete mode 100644 cloudbuild_library_tests.yaml create mode 100644 cloudbuild_tests.yaml diff --git a/build.sh b/build.sh index 2f0595e9..ca651fb7 100755 --- a/build.sh +++ b/build.sh @@ -18,9 +18,9 @@ set -euo pipefail # Actions benchmark=0 # Should run benchmarks? -build=1 # Should build images? -library_tests=0 # Should try to install top N Python libraries +build=0 # Should build images? system_tests=0 # Should run system tests? +tests=0 # Should run standard test suite? local=0 # Should run using local Docker daemon instead of GCR? @@ -40,32 +40,37 @@ Build and test artifacts in this repository Options: --[no]benchmark: Run benchmarking suite (default false) - --[no]build: Build all images (default true) - --[no]library_tests: Run library compatiblity tests (default false) + --[no]build: Build all images (default true if no options set) + --[no]tests: Run basic tests (default true if no options set) --[no]local: Build images using local Docker daemon (default false) --[no]system_tests: Run system tests (default false) " } - + # Read environment variables -if [ -z "${DOCKER_NAMESPACE+set}" ] ; then +if [ -z "${DOCKER_NAMESPACE:+set}" ] ; then fatal 'Error: $DOCKER_NAMESPACE is not set; invoke with something like DOCKER_NAMESPACE=gcr.io/YOUR-PROJECT-NAME' fi -if [ -z "${BUILDER_DOCKER_NAMESPACE+set}" ] ; then +if [ -z "${BUILDER_DOCKER_NAMESPACE:+set}" ] ; then export BUILDER_DOCKER_NAMESPACE="${DOCKER_NAMESPACE}" fi -if [ -z "${TAG+set}" ] ; then +if [ -z "${TAG:+set}" ] ; then export TAG=`date +%Y-%m-%d-%H%M%S` fi -substitutions="\ +build_substitutions="\ _BUILDER_DOCKER_NAMESPACE=${BUILDER_DOCKER_NAMESPACE},\ _DOCKER_NAMESPACE=${DOCKER_NAMESPACE},\ _TAG=${TAG}\ " +substitutions="\ +_DOCKER_NAMESPACE=${DOCKER_NAMESPACE},\ +_TAG=${TAG}\ +" + # Read command line arguments while [ $# -gt 0 ]; do case "$1" in @@ -85,14 +90,6 @@ while [ $# -gt 0 ]; do build=0 shift ;; - --library_tests) - library_tests=1 - shift - ;; - --nolibrary_tests) - library_tests=0 - shift - ;; --local) local=1 shift @@ -109,6 +106,14 @@ while [ $# -gt 0 ]; do system_tests=0 shift ;; + --tests) + tests=1 + shift + ;; + --notests) + tests=0 + shift + ;; *) usage ;; @@ -118,9 +123,12 @@ done # If no actions chosen, then tell the user if [ "${benchmark}" -eq 0 -a \ "${build}" -eq 0 -a \ - "${library_tests}" -eq 0 -a \ - "${system_tests}" -eq 0 ]; then - fatal 'Error: No actions specified (for example, --build), exiting' + "${system_tests}" -eq 0 -a \ + "${tests}" -eq 0 \ +]; then + echo 'No actions specified, defaulting to --build --tests' + build=1 + tests=1 fi # Running build local or remote? @@ -155,7 +163,8 @@ for outfile in \ tests/google-cloud-python-system/Dockerfile \ tests/integration/Dockerfile \ ; do - envsubst <"${outfile}".in >"${outfile}" '$DEBIAN_BASE_IMAGE $STAGING_IMAGE $GOOGLE_CLOUD_PROJECT_FOR_TESTS' + envsubst <"${outfile}".in >"${outfile}" \ + '$DEBIAN_BASE_IMAGE $STAGING_IMAGE $GOOGLE_CLOUD_PROJECT_FOR_TESTS $TAG' done # Make some files available to the runtime builder Docker context @@ -174,36 +183,27 @@ cp -a scripts/testdata/hello_world/main.py tests/eventlet/main.py # Build images and push to GCR if [ "${build}" -eq 1 ]; then echo "Building images" - ${gcloud_cmd} --config cloudbuild.yaml --substitutions "${substitutions}" + ${gcloud_cmd} --config cloudbuild.yaml --substitutions "${build_substitutions}" fi -# Run just the library compatibility tests (for DPE Gardener bot usually) -if [ "${library_tests}" -eq 1 ]; then +# Run the tests that don't require (too many) external services +if [ "${tests}" -eq 1 ]; then echo "Testing compatibility with popular Python libraries" - ${gcloud_cmd} --config cloudbuild_library_tests.yaml --substitutions "${substitutions}" + ${gcloud_cmd} --config cloudbuild_tests.yaml --substitutions "${substitutions}" fi -# If both system tests and benchmarks are requested, run them both -# even if one or the other has errors. If the build step had errors, -# this script will have already exited. -exit_code=0 - # Run system tests if [ "${system_tests}" -eq 1 ]; then echo "Running system tests using project ${GOOGLE_CLOUD_PROJECT_FOR_TESTS}" trap "rm -f tests/google-cloud-python-system/credentials.json" EXIT cp "${GOOGLE_APPLICATION_CREDENTIALS_FOR_TESTS}" tests/google-cloud-python-system/credentials.json - ${gcloud_cmd} --config cloudbuild_system_tests.yaml --substitutions "${substitutions}" || \ - exit_code=1 + ${gcloud_cmd} --config cloudbuild_system_tests.yaml --substitutions "${substitutions}" rm -f tests/google-cloud-python-system/credentials.json fi # Run benchmarks if [ "${benchmark}" -eq 1 ] ; then echo "Running benchmark" - ${gcloud_cmd} --config cloudbuild_benchmark.yaml --substitutions "${substitutions}" || \ - exit_code=1 + ${gcloud_cmd} --config cloudbuild_benchmark.yaml --substitutions "${substitutions}" fi - -exit ${exit_code} diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 68b5c5bb..1eabb997 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -11,35 +11,12 @@ steps: name: gcr.io/cloud-builders/docker:latest args: ['build', '--tag=${_DOCKER_NAMESPACE}/python:${_TAG}', '--no-cache', '/workspace/runtime-image/'] -- # Validate structure of base runtime image - name: gcr.io/gcp-runtimes/structure_test:latest - args: [ - '-i', '${_DOCKER_NAMESPACE}/python:${_TAG}', - '--config', '/workspace/tests/virtualenv/virtualenv_default.yaml', - '--config', '/workspace/tests/virtualenv/virtualenv_python34.yaml', - '--config', '/workspace/tests/virtualenv/virtualenv_python35.yaml', - '--config', '/workspace/tests/virtualenv/virtualenv_python36.yaml', - '--config', '/workspace/tests/no-virtualenv/no-virtualenv.yaml', - '--config', '/workspace/tests/python2-libraries/python2-libraries.yaml', - '--config', '/workspace/tests/python3-libraries/python3-libraries.yaml', - '--config', '/workspace/tests/license-test/license-test.yaml', - '-v' - ] -- # Run compatibility tests - name: gcr.io/cloud-builders/docker:latest - args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/eventlet:${_TAG}', - '--no-cache', '/workspace/tests/eventlet/'] -- # Build image to run google client library unit tests - name: gcr.io/cloud-builders/docker:latest - args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/google-cloud-python:${_TAG}', - '--no-cache', '/workspace/tests/google-cloud-python/'] -- # Run google client library unit tests - name: ${_DOCKER_NAMESPACE}/python/tests/google-cloud-python:${_TAG} - # Build runtime builder image name: gcr.io/cloud-builders/docker:latest args: ['build', '--tag=${_BUILDER_DOCKER_NAMESPACE}/python/gen-dockerfile:${_TAG}', '--no-cache', '/workspace/builder/gen-dockerfile/'] images: [ + '${_DOCKER_NAMESPACE}/python/interpreter-builder:${_TAG}', '${_DOCKER_NAMESPACE}/python:${_TAG}', '${_BUILDER_DOCKER_NAMESPACE}/python/gen-dockerfile:${_TAG}', ] diff --git a/cloudbuild_benchmark.yaml b/cloudbuild_benchmark.yaml index 616991ef..a960bc9b 100644 --- a/cloudbuild_benchmark.yaml +++ b/cloudbuild_benchmark.yaml @@ -3,10 +3,6 @@ steps: - name: gcr.io/cloud-builders/docker:latest args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/benchmark:${_TAG}', '--no-cache', '/workspace/tests/benchmark/'] - env: [ - # Avoid warning about unused substitutions - 'UNUSED1=${_BUILDER_DOCKER_NAMESPACE}', - ] images: [ # Intentionally empty ] diff --git a/cloudbuild_library_tests.yaml b/cloudbuild_library_tests.yaml deleted file mode 100644 index f547d73f..00000000 --- a/cloudbuild_library_tests.yaml +++ /dev/null @@ -1,20 +0,0 @@ -timeout: 1800s -steps: -- # Check that we can install important libraries without error - name: gcr.io/gcp-runtimes/structure_test:latest - args: [ - '-i', '${_DOCKER_NAMESPACE}/python:${_TAG}', - '--config', '/workspace/tests/python2-libraries/python2-libraries.yaml', - '--config', '/workspace/tests/python3-libraries/python3-libraries.yaml', - '-v' - ] - env: [ - # Avoid warning about unused substitutions - 'UNUSED1=${_BUILDER_DOCKER_NAMESPACE}', - ] -- # Run compatibility tests - name: gcr.io/cloud-builders/docker:latest - args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/eventlet:${_TAG}', - '--no-cache', '/workspace/tests/eventlet/'] -images: [ -] diff --git a/cloudbuild_system_tests.yaml b/cloudbuild_system_tests.yaml index f01a48e7..3cac03e2 100644 --- a/cloudbuild_system_tests.yaml +++ b/cloudbuild_system_tests.yaml @@ -4,10 +4,6 @@ steps: args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/google-cloud-python-system:${_TAG}', '--no-cache', '/workspace/tests/google-cloud-python-system/'] - name: ${_DOCKER_NAMESPACE}/python/tests/google-cloud-python-system:${_TAG} - env: [ - # Avoid warning about unused substitutions - 'UNUSED1=${_BUILDER_DOCKER_NAMESPACE}', - ] images: [ # Intentionally empty ] diff --git a/cloudbuild_tests.yaml b/cloudbuild_tests.yaml new file mode 100644 index 00000000..806e8b12 --- /dev/null +++ b/cloudbuild_tests.yaml @@ -0,0 +1,28 @@ +timeout: 3600s +steps: +- # Validate structure of base runtime image + name: gcr.io/gcp-runtimes/structure_test:latest + args: [ + '-i', '${_DOCKER_NAMESPACE}/python:${_TAG}', + '--config', '/workspace/tests/virtualenv/virtualenv_default.yaml', + '--config', '/workspace/tests/virtualenv/virtualenv_python34.yaml', + '--config', '/workspace/tests/virtualenv/virtualenv_python35.yaml', + '--config', '/workspace/tests/virtualenv/virtualenv_python36.yaml', + '--config', '/workspace/tests/no-virtualenv/no-virtualenv.yaml', + '--config', '/workspace/tests/python2-libraries/python2-libraries.yaml', + '--config', '/workspace/tests/python3-libraries/python3-libraries.yaml', + '--config', '/workspace/tests/license-test/license-test.yaml', + '-v' + ] +- # Run compatibility tests + name: gcr.io/cloud-builders/docker:latest + args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/eventlet:${_TAG}', + '--no-cache', '/workspace/tests/eventlet/'] +- # Build image to run google client library unit tests + name: gcr.io/cloud-builders/docker:latest + args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/google-cloud-python:${_TAG}', + '--no-cache', '/workspace/tests/google-cloud-python/'] +- # Run google client library unit tests + name: ${_DOCKER_NAMESPACE}/python/tests/google-cloud-python:${_TAG} +images: [ +] From f6bc889f1bb99dee8277b43d6a8dea1d00c91f4e Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 20 Oct 2017 15:53:28 -0700 Subject: [PATCH 237/362] Remove --tests and --systems_tests to --test and --system_test --- build.sh | 38 +++++++++---------- ..._tests.yaml => cloudbuild_system_test.yaml | 0 cloudbuild_tests.yaml => cloudbuild_test.yaml | 0 3 files changed, 19 insertions(+), 19 deletions(-) rename cloudbuild_system_tests.yaml => cloudbuild_system_test.yaml (100%) rename cloudbuild_tests.yaml => cloudbuild_test.yaml (100%) diff --git a/build.sh b/build.sh index ca651fb7..ba20b8c8 100755 --- a/build.sh +++ b/build.sh @@ -19,8 +19,8 @@ set -euo pipefail # Actions benchmark=0 # Should run benchmarks? build=0 # Should build images? -system_tests=0 # Should run system tests? -tests=0 # Should run standard test suite? +system_test=0 # Should run system tests? +test=0 # Should run standard test suite? local=0 # Should run using local Docker daemon instead of GCR? @@ -41,9 +41,9 @@ Build and test artifacts in this repository Options: --[no]benchmark: Run benchmarking suite (default false) --[no]build: Build all images (default true if no options set) - --[no]tests: Run basic tests (default true if no options set) + --[no]test: Run basic tests (default true if no options set) --[no]local: Build images using local Docker daemon (default false) - --[no]system_tests: Run system tests (default false) + --[no]system_test: Run system tests (default false) " } @@ -98,20 +98,20 @@ while [ $# -gt 0 ]; do local=0 shift ;; - --system_tests) - system_tests=1 + --system_test) + system_test=1 shift ;; - --nosystem_tests) - system_tests=0 + --nosystem_test) + system_test=0 shift ;; - --tests) - tests=1 + --test) + test=1 shift ;; - --notests) - tests=0 + --notest) + test=0 shift ;; *) @@ -123,12 +123,12 @@ done # If no actions chosen, then tell the user if [ "${benchmark}" -eq 0 -a \ "${build}" -eq 0 -a \ - "${system_tests}" -eq 0 -a \ - "${tests}" -eq 0 \ + "${system_test}" -eq 0 -a \ + "${test}" -eq 0 \ ]; then - echo 'No actions specified, defaulting to --build --tests' + echo 'No actions specified, defaulting to --build --test' build=1 - tests=1 + test=1 fi # Running build local or remote? @@ -137,7 +137,7 @@ if [ "${local}" -eq 1 ]; then fi # Read action-specific environment variables -if [ "${system_tests}" -eq 1 ]; then +if [ "${system_test}" -eq 1 ]; then if [ -z "${GOOGLE_APPLICATION_CREDENTIALS_FOR_TESTS+set}" ] ; then fatal 'Error: $GOOGLE_APPLICATION_CREDENTIALS_FOR_TESTS is not set; invoke with something like GOOGLE_APPLICATION_CREDENTIALS_FOR_TESTS=/path/to/service/account/creds.json' fi @@ -187,13 +187,13 @@ if [ "${build}" -eq 1 ]; then fi # Run the tests that don't require (too many) external services -if [ "${tests}" -eq 1 ]; then +if [ "${test}" -eq 1 ]; then echo "Testing compatibility with popular Python libraries" ${gcloud_cmd} --config cloudbuild_tests.yaml --substitutions "${substitutions}" fi # Run system tests -if [ "${system_tests}" -eq 1 ]; then +if [ "${system_test}" -eq 1 ]; then echo "Running system tests using project ${GOOGLE_CLOUD_PROJECT_FOR_TESTS}" trap "rm -f tests/google-cloud-python-system/credentials.json" EXIT diff --git a/cloudbuild_system_tests.yaml b/cloudbuild_system_test.yaml similarity index 100% rename from cloudbuild_system_tests.yaml rename to cloudbuild_system_test.yaml diff --git a/cloudbuild_tests.yaml b/cloudbuild_test.yaml similarity index 100% rename from cloudbuild_tests.yaml rename to cloudbuild_test.yaml From b25687d8fe0348421901fb0b255417649691e2ed Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 20 Oct 2017 16:52:19 -0700 Subject: [PATCH 238/362] Fix typo in filename --- build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sh b/build.sh index ba20b8c8..7e2370d4 100755 --- a/build.sh +++ b/build.sh @@ -189,7 +189,7 @@ fi # Run the tests that don't require (too many) external services if [ "${test}" -eq 1 ]; then echo "Testing compatibility with popular Python libraries" - ${gcloud_cmd} --config cloudbuild_tests.yaml --substitutions "${substitutions}" + ${gcloud_cmd} --config cloudbuild_test.yaml --substitutions "${substitutions}" fi # Run system tests @@ -198,7 +198,7 @@ if [ "${system_test}" -eq 1 ]; then trap "rm -f tests/google-cloud-python-system/credentials.json" EXIT cp "${GOOGLE_APPLICATION_CREDENTIALS_FOR_TESTS}" tests/google-cloud-python-system/credentials.json - ${gcloud_cmd} --config cloudbuild_system_tests.yaml --substitutions "${substitutions}" + ${gcloud_cmd} --config cloudbuild_system_test.yaml --substitutions "${substitutions}" rm -f tests/google-cloud-python-system/credentials.json fi From ca918fda5a777bc7a127fa52c8d207d4f038c7dd Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 20 Oct 2017 17:07:23 -0700 Subject: [PATCH 239/362] Remove m2crypto package which doesn't work under Python 3 --- tests/python3-libraries/requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 614f2f37..edbe81f6 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -73,7 +73,6 @@ kombu==4.1.0 linecache2==1.0.0 logilab-common==1.4.1 lxml==4.1.0 -m2crypto==0.27.0 mako==1.0.7 manifestparser==1.1 markdown==2.6.9 From d4afefe05e8b5ae72faba986d9429bfbc2f6c919 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 20 Oct 2017 18:44:30 -0700 Subject: [PATCH 240/362] Add crcmod to compatilibity test suite, it's used by gsutil. --- tests/python2-libraries/requirements.txt | 1 + tests/python3-libraries/requirements.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 87e9e53a..8711bad0 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -31,6 +31,7 @@ configobj==5.0.6 cov-core==1.15.0 coverage==4.4.1 coveralls==1.2.0 +crcmod==1.7 cryptography==2.1.1 cssselect==1.0.1 cython==0.27.1 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index edbe81f6..dba60564 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -29,6 +29,7 @@ configobj==5.0.6 cov-core==1.15.0 coverage==4.4.1 coveralls==1.2.0 +crcmod==1.7 cryptography==2.1.1 cssselect==1.0.1 cython==0.27.1 From 509cb5ce2768733948f3da42d25bac3ff735bf75 Mon Sep 17 00:00:00 2001 From: Angela Li Date: Tue, 24 Oct 2017 16:32:44 -0700 Subject: [PATCH 241/362] Remove sub directories of metrics scripts (#159) --- perf_dashboard/__init__.py | 0 perf_dashboard/{stackoverflow => }/posts_stats.py | 4 +--- .../{clientlibs_download => }/python_clientlibs_download.py | 6 ++---- 3 files changed, 3 insertions(+), 7 deletions(-) create mode 100644 perf_dashboard/__init__.py rename perf_dashboard/{stackoverflow => }/posts_stats.py (95%) rename perf_dashboard/{clientlibs_download => }/python_clientlibs_download.py (97%) diff --git a/perf_dashboard/__init__.py b/perf_dashboard/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/perf_dashboard/stackoverflow/posts_stats.py b/perf_dashboard/posts_stats.py similarity index 95% rename from perf_dashboard/stackoverflow/posts_stats.py rename to perf_dashboard/posts_stats.py index 280fd42c..efb9c6fc 100644 --- a/perf_dashboard/stackoverflow/posts_stats.py +++ b/perf_dashboard/posts_stats.py @@ -25,9 +25,7 @@ from google.cloud import bigquery -# Need this to import the local helper function -sys.path.insert(0, os.path.abspath(__file__+"/../../..")) -from perf_dashboard import bq_utils +import bq_utils GCLOUD_PROJECT_ENV = 'GCLOUD_PROJECT' DATASET_NAME = 'stackoverflow' diff --git a/perf_dashboard/clientlibs_download/python_clientlibs_download.py b/perf_dashboard/python_clientlibs_download.py similarity index 97% rename from perf_dashboard/clientlibs_download/python_clientlibs_download.py rename to perf_dashboard/python_clientlibs_download.py index 3c8f79fe..3866fd92 100644 --- a/perf_dashboard/clientlibs_download/python_clientlibs_download.py +++ b/perf_dashboard/python_clientlibs_download.py @@ -20,9 +20,7 @@ from google.cloud import bigquery -sys.path.insert(0, os.path.abspath(__file__+"/../../..")) -from perf_dashboard import bq_utils - +import bq_utils GCLOUD_PROJECT_ENV = 'GCLOUD_PROJECT' @@ -136,7 +134,7 @@ def main(): clientlibs_table_name=table_name, date_str=datetime.datetime.now().strftime("%Y%m%d")) bq_utils.insert_rows( - project=project, + project=os.environ.get(GCLOUD_PROJECT_ENV), dataset_name=DATASET_NAME, table_name=table_name, rows=rows) From 2e621ba203ed43d7e562cbc35850cdc6ac97dd7a Mon Sep 17 00:00:00 2001 From: Douglas Greiman Date: Tue, 31 Oct 2017 11:05:07 -0700 Subject: [PATCH 242/362] Add CODEOWNERS file (#162) --- CODEOWNERS | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 CODEOWNERS diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 00000000..a2d4a67f --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1,4 @@ +# Code owners file. +# This file controls who is tagged for review for any given pull request. + +* @duggelz @liyanhui1228 @jonparrott From 29fe28bf4fbb8e3b02122081d5908ee7c71450e7 Mon Sep 17 00:00:00 2001 From: Douglas Greiman Date: Tue, 31 Oct 2017 11:05:45 -0700 Subject: [PATCH 243/362] Add honcho package to library test (#163) --- tests/python2-libraries/requirements.txt | 3 ++- tests/python3-libraries/requirements.txt | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 8711bad0..dee9abbf 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -63,7 +63,8 @@ graphite-web==1.0.2 greenlet==0.4.12 gunicorn==19.7.1 hiredis==0.2.0 -html5lib +honcho==1.0.1 +html5lib==0.999999999 httplib2==0.10.3 idna==2.6 ipaddress==1.0.18 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index dba60564..fe9d4156 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -59,7 +59,8 @@ google-api-python-client==1.6.4 greenlet==0.4.12 gunicorn==19.7.1 hiredis==0.2.0 -html5lib +honcho=1.0.1 +html5lib==0.999999999 httplib2==0.10.3 idna==2.6 ipaddress==1.0.18 From 0965d52c5f1b557b17ad480cd9c298cb25ba5c84 Mon Sep 17 00:00:00 2001 From: Douglas Greiman Date: Tue, 31 Oct 2017 11:06:10 -0700 Subject: [PATCH 244/362] Update pip, setuptools, virtualenv when we build the base runtime image. (#161) --- runtime-image/Dockerfile.in | 13 ++++++++----- runtime-image/resources/apt-packages.txt | 1 + runtime-image/resources/requirements.txt | 3 +++ 3 files changed, 12 insertions(+), 5 deletions(-) create mode 100644 runtime-image/resources/requirements.txt diff --git a/runtime-image/Dockerfile.in b/runtime-image/Dockerfile.in index ed194257..60a6fa6f 100644 --- a/runtime-image/Dockerfile.in +++ b/runtime-image/Dockerfile.in @@ -16,15 +16,18 @@ ENV LANG C.UTF-8 # logging collection. ENV PYTHONUNBUFFERED 1 -# Upgrade pip (debian package version tends to run a few version behind) and -# install virtualenv system-wide. -RUN pip install --upgrade pip virtualenv - # Install the Google-built interpreters ADD interpreters.tar.gz / # Add Google-built interpreters to the path -ENV PATH /opt/python3.5/bin:/opt/python3.6/bin:$PATH +ENV PATH /opt/python3.6/bin:/opt/python3.5/bin:$PATH + +# Upgrade pip (debian package version tends to run a few version behind) and +# install virtualenv system-wide. +RUN /usr/bin/pip install --upgrade -r /resources/requirements.txt && \ + /usr/bin/pip3 install --upgrade -r /resources/requirements.txt && \ + /opt/python3.5/bin/pip3.5 install --upgrade -r /resources/requirements.txt && \ + /opt/python3.6/bin/pip3.6 install --upgrade -r /resources/requirements.txt # Setup the app working directory RUN ln -s /home/vmagent/app /app diff --git a/runtime-image/resources/apt-packages.txt b/runtime-image/resources/apt-packages.txt index 1fa11bcb..a62a3195 100644 --- a/runtime-image/resources/apt-packages.txt +++ b/runtime-image/resources/apt-packages.txt @@ -7,6 +7,7 @@ wget python-pip python2.7 python2.7-dev +python3-pip python3.4 python3.4-dev # Dependenies for third-party Python packages diff --git a/runtime-image/resources/requirements.txt b/runtime-image/resources/requirements.txt new file mode 100644 index 00000000..40155447 --- /dev/null +++ b/runtime-image/resources/requirements.txt @@ -0,0 +1,3 @@ +pip==9.0.1 +setuptools==36.6.0 +virtualenv==15.1.0 From ba4e5184b1715f4ee89c68d53bd912c44f216215 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Tue, 31 Oct 2017 11:06:59 -0700 Subject: [PATCH 245/362] Auto-update dependencies. (#153) --- tests/python2-libraries/requirements.txt | 42 ++++++++++++------------ tests/python3-libraries/requirements.txt | 42 ++++++++++++------------ 2 files changed, 42 insertions(+), 42 deletions(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index dee9abbf..6db2c17c 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -1,12 +1,12 @@ alembic==0.9.6 amqp==2.2.2 amqplib==1.0.2 -ansible==2.4.0.0 +ansible==2.4.1.0 anyjson==0.3.3 apache-libcloud==2.2.1 argparse==1.4.0 astroid==1.5.3 -awscli==1.11.170 +awscli==1.11.178 babel==2.5.1 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 @@ -16,7 +16,7 @@ billiard==3.5.0.3 blessings==1.6 blinker==1.4 boto==2.48.0 -botocore==1.7.28 +botocore==1.7.36 bottle==0.12.13 carbon==1.0.2 celery==4.1.0 @@ -32,16 +32,16 @@ cov-core==1.15.0 coverage==4.4.1 coveralls==1.2.0 crcmod==1.7 -cryptography==2.1.1 +cryptography==2.1.2 cssselect==1.0.1 -cython==0.27.1 +cython==0.27.2 decorator==4.1.2 django-celery==3.2.1 django-debug-toolbar==1.8 -django-extensions==1.9.1 +django-extensions==1.9.6 django==1.11.6 django_compress==1.0.1 -djangorestframework==3.7.0 +djangorestframework==3.7.1 docker-py==1.10.6 docopt==0.6.2 docutils==0.14 @@ -52,7 +52,7 @@ eventlet==0.21.0 extras==1.0.0 fabric==1.14.0 fixtures==3.0.0 -flake8==3.4.1 +flake8==3.5.0 flask==0.12.2 funcsigs==1.0.2 functools32==3.2.3.post2 @@ -101,22 +101,22 @@ mysql-python==1.2.5 ndg-httpsclient==0.4.3 netaddr==0.7.19 netifaces==0.10.6 -newrelic==2.94.0.79 +newrelic==2.96.0.80 nose==1.3.7 numpy==1.13.3 oauth2==1.9.0.post1 oauth2client==4.1.2 -oauthlib==2.0.4 +oauthlib==2.0.6 ordereddict==1.1 -oslo.config==4.13.1 -pandas==0.20.3 +oslo.config==5.0.0 +pandas==0.21.0 paramiko==2.3.1 passlib==1.7.1 paste==2.0.3 pastedeploy==1.5.2 pastescript==2.0.2 pbr==3.1.1 -pep8==1.7.0 +pep8==1.7.1 pexpect==4.2.1 pika==0.11.0 pillow==4.3.0 @@ -124,7 +124,7 @@ pip==9.0.1 prettytable protobuf==3.4.0 psutil==5.4.0 -psycopg2==2.7.3.1 +psycopg2==2.7.3.2 py==1.4.34 pyasn1-modules==0.1.5 pyasn1==0.3.7 @@ -147,24 +147,24 @@ pytest==3.2.3 python-cjson==1.2.1 python-daemon==2.1.2 python-dateutil==2.6.1 -python-gflags==3.1.1 +python-gflags==3.1.2 python-keystoneclient==3.13.0 python-memcached==1.58 python-mimeparse==1.6.0 python-novaclient==9.1.0 python-subunit==1.2.0 python-swiftclient==3.4.0 -pytz==2017.2 +pytz==2017.3 pyyaml==3.12 -pyzmq==16.0.2 -raven==6.2.1 +pyzmq==16.0.3 +raven==6.3.0 redis==2.10.6 repoze.lru==0.7 requests-oauthlib==0.8.0 requests==2.18.4 retrying==1.3.3 rsa==3.4.2 -scipy==0.19.1 +scipy==1.0.0 selenium==3.6.0 setuptools-git==1.2 setuptools==36.6.0 @@ -173,7 +173,7 @@ simplejson==3.11.1 six==1.11.0 snowballstemmer==1.2.1 south==1.0.2 -sphinx==1.6.4 +sphinx==1.6.5 sqlalchemy-migrate==0.11.0 sqlalchemy==1.1.14 sqlparse==0.2.4 @@ -199,7 +199,7 @@ waitress==1.1.0 warlock==1.3.0 webob==1.7.3 websocket-client==0.44.0 -webtest==2.0.28 +webtest==2.0.29 werkzeug==0.12.2 wheel==0.30.0 xlrd==1.1.0 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index fe9d4156..cca4bef2 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -1,12 +1,12 @@ alembic==0.9.6 amqp==2.2.2 amqplib==1.0.2 -ansible==2.4.0.0 +ansible==2.4.1.0 anyjson==0.3.3 apache-libcloud==2.2.1 argparse==1.4.0 astroid==1.5.3 -awscli==1.11.170 +awscli==1.11.178 babel==2.5.1 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 @@ -15,7 +15,7 @@ billiard==3.5.0.3 blessings==1.6 blinker==1.4 boto==2.48.0 -botocore==1.7.28 +botocore==1.7.36 bottle==0.12.13 celery==4.1.0 certifi==2017.7.27.1 @@ -30,16 +30,16 @@ cov-core==1.15.0 coverage==4.4.1 coveralls==1.2.0 crcmod==1.7 -cryptography==2.1.1 +cryptography==2.1.2 cssselect==1.0.1 -cython==0.27.1 +cython==0.27.2 decorator==4.1.2 django-celery==3.2.1 django-debug-toolbar==1.8 -django-extensions==1.9.1 +django-extensions==1.9.6 django==1.11.6 django_compress==1.0.1 -djangorestframework==3.7.0 +djangorestframework==3.7.1 docker-py==1.10.6 docopt==0.6.2 docutils==0.14 @@ -50,7 +50,7 @@ eventlet==0.21.0 extras==1.0.0 fabric==1.14.0 fixtures==3.0.0 -flake8==3.4.1 +flake8==3.5.0 flask==0.12.2 funcsigs==1.0.2 futures==3.1.1 @@ -94,22 +94,22 @@ msgpack-python==0.4.8 ndg-httpsclient==0.4.3 netaddr==0.7.19 netifaces==0.10.6 -newrelic==2.94.0.79 +newrelic==2.96.0.80 nose==1.3.7 numpy==1.13.3 oauth2==1.9.0.post1 oauth2client==4.1.2 -oauthlib==2.0.4 +oauthlib==2.0.6 ordereddict==1.1 -oslo.config==4.13.1 -pandas==0.20.3 +oslo.config==5.0.0 +pandas==0.21.0 paramiko==2.3.1 passlib==1.7.1 paste==2.0.3 pastedeploy==1.5.2 pastescript==2.0.2 pbr==3.1.1 -pep8==1.7.0 +pep8==1.7.1 pexpect==4.2.1 pika==0.11.0 pillow==4.3.0 @@ -117,7 +117,7 @@ pip==9.0.1 prettytable protobuf==3.4.0 psutil==5.4.0 -psycopg2==2.7.3.1 +psycopg2==2.7.3.2 py==1.4.34 pyasn1-modules==0.1.5 pyasn1==0.3.7 @@ -138,24 +138,24 @@ pytest-cov==2.5.1 pytest==3.2.3 python-daemon==2.1.2 python-dateutil==2.6.1 -python-gflags==3.1.1 +python-gflags==3.1.2 python-keystoneclient==3.13.0 python-memcached==1.58 python-mimeparse==1.6.0 python-novaclient==9.1.0 python-subunit==1.2.0 python-swiftclient==3.4.0 -pytz==2017.2 +pytz==2017.3 pyyaml==3.12 -pyzmq==16.0.2 -raven==6.2.1 +pyzmq==16.0.3 +raven==6.3.0 redis==2.10.6 repoze.lru==0.7 requests-oauthlib==0.8.0 requests==2.18.4 retrying==1.3.3 rsa==3.4.2 -scipy==0.19.1 +scipy==1.0.0 selenium==3.6.0 setuptools-git==1.2 setuptools==36.6.0 @@ -164,7 +164,7 @@ simplejson==3.11.1 six==1.11.0 snowballstemmer==1.2.1 south==1.0.2 -sphinx==1.6.4 +sphinx==1.6.5 sqlalchemy-migrate==0.11.0 sqlalchemy==1.1.14 sqlparse==0.2.4 @@ -188,7 +188,7 @@ waitress==1.1.0 warlock==1.3.0 webob==1.7.3 websocket-client==0.44.0 -webtest==2.0.28 +webtest==2.0.29 werkzeug==0.12.2 wheel==0.30.0 xlrd==1.1.0 From babc14b911dc1c49d8bdc97130510f89a0bcabf7 Mon Sep 17 00:00:00 2001 From: Nick Kubala Date: Thu, 26 Oct 2017 15:53:51 -0700 Subject: [PATCH 246/362] migrate structure tests to new version --- cloudbuild_test.yaml | 29 +++++++++++++++---- tests/python2-libraries/Dockerfile | 3 ++ .../python2-libraries/python2-libraries.yaml | 5 +--- tests/python3-libraries/Dockerfile | 3 ++ .../python3-libraries/python3-libraries.yaml | 12 +++----- tests/virtualenv/virtualenv_default.yaml | 14 +++++---- tests/virtualenv/virtualenv_python34.yaml | 9 +++--- tests/virtualenv/virtualenv_python35.yaml | 7 ++++- tests/virtualenv/virtualenv_python36.yaml | 7 ++++- 9 files changed, 59 insertions(+), 30 deletions(-) create mode 100644 tests/python2-libraries/Dockerfile create mode 100644 tests/python3-libraries/Dockerfile diff --git a/cloudbuild_test.yaml b/cloudbuild_test.yaml index 806e8b12..fbad487a 100644 --- a/cloudbuild_test.yaml +++ b/cloudbuild_test.yaml @@ -1,7 +1,27 @@ timeout: 3600s steps: -- # Validate structure of base runtime image - name: gcr.io/gcp-runtimes/structure_test:latest + # Validate structure of base runtime image +- name: gcr.io/cloud-builders/docker:latest + args: ['build', '-t', 'python2-libraries-intermediate', '--build-arg', + 'intermediate_image=${_DOCKER_NAMESPACE}/python:${_TAG}', + '/workspace/tests/python2-libraries'] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.1.1 + args: [ + '-test.v', + '-image', 'python2-libraries-intermediate', + '/workspace/tests/python2-libraries/python2-libraries.yaml' + ] +- name: gcr.io/cloud-builders/docker:latest + args: ['build', '-t', 'python3-libraries-intermediate', '--build-arg', + 'intermediate_image=${_DOCKER_NAMESPACE}/python:${_TAG}', + '/workspace/tests/python3-libraries'] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.1.1 + args: [ + '-test.v', + '-image', 'python3-libraries-intermediate', + '/workspace/tests/python3-libraries/python3-libraries.yaml' + ] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.1.1 args: [ '-i', '${_DOCKER_NAMESPACE}/python:${_TAG}', '--config', '/workspace/tests/virtualenv/virtualenv_default.yaml', @@ -9,8 +29,6 @@ steps: '--config', '/workspace/tests/virtualenv/virtualenv_python35.yaml', '--config', '/workspace/tests/virtualenv/virtualenv_python36.yaml', '--config', '/workspace/tests/no-virtualenv/no-virtualenv.yaml', - '--config', '/workspace/tests/python2-libraries/python2-libraries.yaml', - '--config', '/workspace/tests/python3-libraries/python3-libraries.yaml', '--config', '/workspace/tests/license-test/license-test.yaml', '-v' ] @@ -24,5 +42,4 @@ steps: '--no-cache', '/workspace/tests/google-cloud-python/'] - # Run google client library unit tests name: ${_DOCKER_NAMESPACE}/python/tests/google-cloud-python:${_TAG} -images: [ -] +images: [] diff --git a/tests/python2-libraries/Dockerfile b/tests/python2-libraries/Dockerfile new file mode 100644 index 00000000..c2647292 --- /dev/null +++ b/tests/python2-libraries/Dockerfile @@ -0,0 +1,3 @@ +ARG intermediate_image +FROM $intermediate_image +COPY requirements.txt /requirements.txt diff --git a/tests/python2-libraries/python2-libraries.yaml b/tests/python2-libraries/python2-libraries.yaml index 5d298b58..3a2771ac 100644 --- a/tests/python2-libraries/python2-libraries.yaml +++ b/tests/python2-libraries/python2-libraries.yaml @@ -7,10 +7,7 @@ globalEnvVars: value: "/env/bin:$PATH" commandTests: - - name: "virtual env teardown" - command: ["rm", "-rf", "/env"] - - name: "requirements" setup: [["virtualenv", "/env"]] - command: ["pip", "install", "-r", "/workspace/tests/python2-libraries/requirements.txt"] + command: ["pip", "install", "-r", "/requirements.txt"] exitCode: 0 diff --git a/tests/python3-libraries/Dockerfile b/tests/python3-libraries/Dockerfile new file mode 100644 index 00000000..c2647292 --- /dev/null +++ b/tests/python3-libraries/Dockerfile @@ -0,0 +1,3 @@ +ARG intermediate_image +FROM $intermediate_image +COPY requirements.txt /requirements.txt diff --git a/tests/python3-libraries/python3-libraries.yaml b/tests/python3-libraries/python3-libraries.yaml index ef15c0a1..ace58132 100644 --- a/tests/python3-libraries/python3-libraries.yaml +++ b/tests/python3-libraries/python3-libraries.yaml @@ -8,15 +8,11 @@ globalEnvVars: commandTests: - name: "requirements 3.5" - setup: - - ["rm", "-rf", "/env"] - - ["virtualenv", "-p", "/opt/python3.5/bin/python3.5", "/env"] - command: ["pip", "install", "-r", "/workspace/tests/python3-libraries/requirements.txt"] + setup: [["virtualenv", "-p", "/opt/python3.5/bin/python3.5", "/env"]] + command: ["pip", "install", "-r", "/requirements.txt"] exitCode: 0 - name: "requirements 3.6" - setup: - - ["rm", "-rf", "/env"] - - ["virtualenv", "-p", "/opt/python3.6/bin/python3.6", "/env"] - command: ["pip", "install", "-r", "/workspace/tests/python3-libraries/requirements.txt"] + setup: [["virtualenv", "-p", "/opt/python3.6/bin/python3.6", "/env"]] + command: ["pip", "install", "-r", "/requirements.txt"] exitCode: 0 diff --git a/tests/virtualenv/virtualenv_default.yaml b/tests/virtualenv/virtualenv_default.yaml index 314a3d0c..8695e319 100644 --- a/tests/virtualenv/virtualenv_default.yaml +++ b/tests/virtualenv/virtualenv_default.yaml @@ -7,13 +7,14 @@ globalEnvVars: value: "/env/bin:$PATH" commandTests: - - name: "virtual env teardown" - command: ["rm", "-rf", "/env"] - - name: "python installation" command: ["which", "python"] expectedOutput: ["/usr/bin/python\n"] + - name: "pip installation" + command: ["which", "pip"] + expectedOutput: ["/usr/local/bin/pip\n"] + - name: "virtualenv installation" setup: [["virtualenv", "/env"]] command: ["which", "python"] @@ -25,12 +26,13 @@ commandTests: # https://bugs.python.org/issue18338 expectedError: ["Python 2.7.9\n"] - - name: "pip installation" + - name: "virtualenv pip installation" + setup: [["virtualenv", "/env"]] command: ["which", "pip"] expectedOutput: ["/env/bin/pip\n"] - - name: "gunicorn flask" - setup: [["pip", "install", "gunicorn", "flask"]] + - name: "virtualenv gunicorn flask" + setup: [["virtualenv", "/env"], ["pip", "install", "gunicorn", "flask"]] command: ["which", "gunicorn"] expectedOutput: ["/env/bin/gunicorn"] diff --git a/tests/virtualenv/virtualenv_python34.yaml b/tests/virtualenv/virtualenv_python34.yaml index 082b3b6e..e35e9693 100644 --- a/tests/virtualenv/virtualenv_python34.yaml +++ b/tests/virtualenv/virtualenv_python34.yaml @@ -7,9 +7,6 @@ globalEnvVars: value: "/env/bin:$PATH" commandTests: - - name: "virtual env teardown" - command: ["rm", "-rf", "/env"] - - name: "python installation" command: ["which", "python3.4"] expectedOutput: ["/usr/bin/python3.4\n"] @@ -20,23 +17,27 @@ commandTests: expectedOutput: ["/env/bin/python\n"] - name: "virtualenv python3 installation" + setup: [["virtualenv", "-p", "python3.4", "/env"]] command: ["which", "python3"] expectedOutput: ["/env/bin/python3\n"] - name: "python version" + setup: [["virtualenv", "-p", "python3.4", "/env"]] command: ["python", "--version"] expectedOutput: ["Python 3.4.2\n"] - name: "pip installation" + setup: [["virtualenv", "-p", "python3.4", "/env"]] command: ["which", "pip"] expectedOutput: ["/env/bin/pip\n"] - name: "pip3 installation" + setup: [["virtualenv", "-p", "python3.4", "/env"]] command: ["which", "pip3"] expectedOutput: ["/env/bin/pip3\n"] - name: "gunicorn flask" - setup: [["pip", "install", "gunicorn", "flask"]] + setup: [["virtualenv", "-p", "python3.4", "/env"], ["pip", "install", "gunicorn", "flask"]] command: ["which", "gunicorn"] expectedOutput: ["/env/bin/gunicorn"] diff --git a/tests/virtualenv/virtualenv_python35.yaml b/tests/virtualenv/virtualenv_python35.yaml index c0b461e1..4b019967 100644 --- a/tests/virtualenv/virtualenv_python35.yaml +++ b/tests/virtualenv/virtualenv_python35.yaml @@ -20,27 +20,32 @@ commandTests: expectedOutput: ["/env/bin/python\n"] - name: "virtualenv python3 installation" + setup: [["virtualenv", "-p", "python3.5", "/env"]] command: ["which", "python3"] expectedOutput: ["/env/bin/python3\n"] - name: "virtualenv python3.5 installation" + setup: [["virtualenv", "-p", "python3.5", "/env"]] command: ["which", "python3.5"] expectedOutput: ["/env/bin/python3.5\n"] - name: "python version" + setup: [["virtualenv", "-p", "python3.5", "/env"]] command: ["python", "--version"] expectedOutput: ["Python 3.5.4\n"] - name: "pip installation" + setup: [["virtualenv", "-p", "python3.5", "/env"]] command: ["which", "pip"] expectedOutput: ["/env/bin/pip\n"] - name: "pip3 installation" + setup: [["virtualenv", "-p", "python3.5", "/env"]] command: ["which", "pip3"] expectedOutput: ["/env/bin/pip3\n"] - name: "gunicorn flask" - setup: [["pip", "install", "gunicorn", "flask"]] + setup: [["virtualenv", "-p", "python3.5", "/env"], ["pip", "install", "gunicorn", "flask"]] command: ["which", "gunicorn"] expectedOutput: ["/env/bin/gunicorn"] diff --git a/tests/virtualenv/virtualenv_python36.yaml b/tests/virtualenv/virtualenv_python36.yaml index ab25c28b..deea9b5d 100644 --- a/tests/virtualenv/virtualenv_python36.yaml +++ b/tests/virtualenv/virtualenv_python36.yaml @@ -20,27 +20,32 @@ commandTests: expectedOutput: ["/env/bin/python\n"] - name: "virtualenv python3 installation" + setup: [["virtualenv", "-p", "python3.6", "/env"]] command: ["which", "python3"] expectedOutput: ["/env/bin/python3\n"] - name: "virtualenv python3.6 installation" + setup: [["virtualenv", "-p", "python3.6", "/env"]] command: ["which", "python3.6"] expectedOutput: ["/env/bin/python3.6\n"] - name: "python version" + setup: [["virtualenv", "-p", "python3.6", "/env"]] command: ["python", "--version"] expectedOutput: ["Python 3.6.2\n"] - name: "pip installation" + setup: [["virtualenv", "-p", "python3.6", "/env"]] command: ["which", "pip"] expectedOutput: ["/env/bin/pip\n"] - name: "pip3 installation" + setup: [["virtualenv", "-p", "python3.6", "/env"]] command: ["which", "pip3"] expectedOutput: ["/env/bin/pip3\n"] - name: "gunicorn flask" - setup: [["pip", "install", "gunicorn", "flask"]] + setup: [["virtualenv", "-p", "python3.6", "/env"], ["pip", "install", "gunicorn", "flask"]] command: ["which", "gunicorn"] expectedOutput: ["/env/bin/gunicorn"] From e95dfdd7e4546231f69b13a6c8d182f1b31583bb Mon Sep 17 00:00:00 2001 From: Nick Kubala Date: Mon, 30 Oct 2017 12:45:24 -0700 Subject: [PATCH 247/362] fix args --- cloudbuild_test.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cloudbuild_test.yaml b/cloudbuild_test.yaml index fbad487a..c6496dec 100644 --- a/cloudbuild_test.yaml +++ b/cloudbuild_test.yaml @@ -23,14 +23,14 @@ steps: ] - name: gcr.io/gcp-runtimes/container-structure-test:v0.1.1 args: [ - '-i', '${_DOCKER_NAMESPACE}/python:${_TAG}', - '--config', '/workspace/tests/virtualenv/virtualenv_default.yaml', - '--config', '/workspace/tests/virtualenv/virtualenv_python34.yaml', - '--config', '/workspace/tests/virtualenv/virtualenv_python35.yaml', - '--config', '/workspace/tests/virtualenv/virtualenv_python36.yaml', - '--config', '/workspace/tests/no-virtualenv/no-virtualenv.yaml', - '--config', '/workspace/tests/license-test/license-test.yaml', - '-v' + '-test.v', + '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', + '/workspace/tests/virtualenv/virtualenv_default.yaml', + '/workspace/tests/virtualenv/virtualenv_python34.yaml', + '/workspace/tests/virtualenv/virtualenv_python35.yaml', + '/workspace/tests/virtualenv/virtualenv_python36.yaml', + '/workspace/tests/no-virtualenv/no-virtualenv.yaml', + '/workspace/tests/license-test/license-test.yaml' ] - # Run compatibility tests name: gcr.io/cloud-builders/docker:latest From 8995620601d3db38a9edd9e6ddc48abf2d5739ae Mon Sep 17 00:00:00 2001 From: Nick Kubala Date: Thu, 2 Nov 2017 14:19:23 -0700 Subject: [PATCH 248/362] expose /environment endpoint for gke tests (#164) * expose /environment endpoint for gke tests * mimic logic in python logging client library for determining cloud env --- tests/integration/server.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/integration/server.py b/tests/integration/server.py index cc82fb93..8e2958cd 100755 --- a/tests/integration/server.py +++ b/tests/integration/server.py @@ -17,6 +17,7 @@ from functools import wraps import json import logging +import os import google.cloud.logging import google.cloud.monitoring @@ -33,6 +34,12 @@ 'CRITICAL': (logging.critical, 'stderr') } +_APPENGINE_FLEXIBLE_ENV_VM = 'GAE_APPENGINE_HOSTNAME' +"""Environment variable set in App Engine when vm:true is set.""" + +_APPENGINE_FLEXIBLE_ENV_FLEX = 'GAE_INSTANCE' +"""Environment variable set in App Engine when env:flex is set.""" + app = Flask(__name__) @@ -132,6 +139,7 @@ def _log_default(token, level): logging.error('Error while writing logs: {0}'.format(e)) raise ErrorResponse('Error while writing logs: {0}'.format(e)) + # this is fine regardless of environment, it's only used in GAE logs return 'appengine.googleapis.com%2F{0}'.format(source) @@ -231,6 +239,16 @@ def _custom(): return json.dumps(tests), 200 +@app.route('/environment', methods=['GET']) +def _check_environment(): + # determine what cloud env we're running in; essentially, GAE vs GKE + # for GAE, we'll check the existence env vars set on + # vm:true or env:flex + # if neither exist, assume we're in GKE + return (_APPENGINE_FLEXIBLE_ENV_VM in os.environ or + _APPENGINE_FLEXIBLE_ENV_FLEX in os.environ), 200 + + class ErrorResponse(Exception): status_code = 400 From 5aa45062d97abf6419418069bbd29e1c13b942c8 Mon Sep 17 00:00:00 2001 From: Douglas Greiman Date: Thu, 2 Nov 2017 14:20:09 -0700 Subject: [PATCH 249/362] Create Debian packages for GCP Python interpreters (#140) We build binary packages directly, rather than building from a source package as the standard Debian Python packages are. --- python-interpreter-builder/DEBIAN/control.in | 25 ++++++++ python-interpreter-builder/Dockerfile.in | 12 +++- .../scripts/package-python.sh | 59 +++++++++++++++++++ 3 files changed, 94 insertions(+), 2 deletions(-) create mode 100644 python-interpreter-builder/DEBIAN/control.in create mode 100755 python-interpreter-builder/scripts/package-python.sh diff --git a/python-interpreter-builder/DEBIAN/control.in b/python-interpreter-builder/DEBIAN/control.in new file mode 100644 index 00000000..49980654 --- /dev/null +++ b/python-interpreter-builder/DEBIAN/control.in @@ -0,0 +1,25 @@ +Package: ${DEB_PACKAGE_NAME} +Version: ${DEB_PACKAGE_VERSION} +Section: python +Priority: optional +Architecture: amd64 +Maintainer: Douglas Greiman +Description: Interactive high-level object-oriented language (version ${SHORT_VERSION}) + Python is a high-level, interactive, object-oriented language. Its ${SHORT_VERSION} version + includes an extensive class library with lots of goodies for + network programming, system administration, sounds and graphics. +Depends: libbz2-1.0, + libc6, + libdb5.3, + libexpat1, + libffi6, + liblzma5, + libmpdec2, + libncursesw5, + libreadline6, + libsqlite3-0, + libssl1.0.0, + libtinfo5, + mime-support, + zlib1g +Homepage: https://www.python.org diff --git a/python-interpreter-builder/Dockerfile.in b/python-interpreter-builder/Dockerfile.in index eba153e9..90e606eb 100644 --- a/python-interpreter-builder/Dockerfile.in +++ b/python-interpreter-builder/Dockerfile.in @@ -10,6 +10,7 @@ RUN apt-get update && apt-get install -yq \ debhelper \ dpkg-dev \ gcc \ + gettext-base \ libbluetooth-dev \ libbz2-dev \ libdb-dev \ @@ -43,10 +44,17 @@ ENV LANG C.UTF-8 # Add build scripts ADD scripts /scripts +ADD DEBIAN /DEBIAN # Build the Python interpreters -RUN /scripts/build-python-3.5.sh -RUN /scripts/build-python-3.6.sh +RUN mkdir -p /opt/packages && \ + echo -n "" > /opt/packages/packages.txt + +RUN /scripts/build-python-3.5.sh && \ + /scripts/package-python.sh 3.5.4 "1gcp~${TAG}" + +RUN /scripts/build-python-3.6.sh && \ + /scripts/package-python.sh 3.6.2 "1gcp~${TAG}" # Tar the interpreters. Tarring is needed because docker cp doesn't handle # links correctly. diff --git a/python-interpreter-builder/scripts/package-python.sh b/python-interpreter-builder/scripts/package-python.sh new file mode 100755 index 00000000..58cd02e6 --- /dev/null +++ b/python-interpreter-builder/scripts/package-python.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +set -euo pipefail +set -x + +function usage { + echo "Usage: $0 long_version tag +Create .deb package file for a Python interpreter with + long_version: (x.y.z) Interpreter version + tag: version suffix unique to this build +" >&2 + exit 1 +} + # Process command line +if [ -z "${1:+set}" -o -z "${2:+set}" ]; then + usage +fi +LONG_VERSION=$1 +BUILD_TAG=$2 +SHORT_VERSION=${1%.*} + +# Compute version specs +DEB_PACKAGE_NAME=gcp-python${SHORT_VERSION} +# Can't have - (hyphen) in debian revision as per +# https://www.debian.org/doc/debian-policy/ch-controlfields.html#s-f-Version +DEBIAN_REVISION=${BUILD_TAG//-/.} +DEB_PACKAGE_VERSION=${LONG_VERSION}-${DEBIAN_REVISION} + +PACKAGE_DIR=/opt/packages +# E.g. gcp-python3.6_3.6.2-1gcp~2017.07.25.110644_amd64.deb +DEB_FILENAME=${DEB_PACKAGE_NAME}_${DEB_PACKAGE_VERSION}_amd64.deb + +# Create directory for intermediate files +SCRATCH_DIR=$(mktemp --directory) +cd "${SCRATCH_DIR}" + +# Synthesize Debian control file. Note that the "Depends:" is +# currently Debian8-specific, and lacks version specifiers present in +# the standard Debian Python packages. +export DEB_PACKAGE_NAME DEB_PACKAGE_VERSION SHORT_VERSION +envsubst control \ + '${DEB_PACKAGE_NAME} ${DEB_PACKAGE_VERSION} ${SHORT_VERSION}' + +# Generate components of .deb archive +tar czf control.tar.gz control +tar czf data.tar.gz "/opt/python${SHORT_VERSION}" +echo "2.0" >debian-binary + +# Generate final .deb. +mkdir -p "${PACKAGE_DIR}" +ar rcD "${PACKAGE_DIR}/${DEB_FILENAME}" \ + debian-binary control.tar.gz data.tar.gz +rm debian-binary control.tar.gz data.tar.gz + +# Validate .deb +dpkg --install --dry-run "${PACKAGE_DIR}/${DEB_FILENAME}" + +# Add to list +echo "${DEB_FILENAME}" >> "${PACKAGE_DIR}/packages.txt" From 3aa916b1ddc1e116d62cea5f776379e1dcb7e6fb Mon Sep 17 00:00:00 2001 From: Douglas Greiman Date: Thu, 2 Nov 2017 14:20:59 -0700 Subject: [PATCH 250/362] Fix unit test failures (#165) * Update unit test script with new nox session names. * Fix typo --- RELEASING.md | 9 +++++++++ tests/google-cloud-python/run_unit_tests.sh | 8 ++++---- tests/python3-libraries/requirements.txt | 2 +- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/RELEASING.md b/RELEASING.md index f7d5a459..00b0fe16 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -67,6 +67,15 @@ following: docker run -it --entrypoint /bin/bash YOUR-IMAGE-NAME ``` +## Running tests against a released image + +To run compatibility tests against an existing image, such as +`gcr.io/google-appengine/python:latest`, run: + +```shell +DOCKER_NAMESPACE=gcr.io/google-appengine TAG=latest ./build.sh --nobuild --test +``` + ## Running benchmarks There is a benchmark suite which compares the performance of interpreters diff --git a/tests/google-cloud-python/run_unit_tests.sh b/tests/google-cloud-python/run_unit_tests.sh index 030919d8..81a6a389 100755 --- a/tests/google-cloud-python/run_unit_tests.sh +++ b/tests/google-cloud-python/run_unit_tests.sh @@ -8,10 +8,10 @@ for noxfile in */nox.py; do nox \ -f "${noxfile}" \ -e \ - "unit_tests(python_version='2.7')" \ - "unit_tests(python_version='3.4')" \ - "unit_tests(python_version='3.5')" \ - "unit_tests(python_version='3.6')" \ + "unit(py='2.7')" \ + "unit(py='3.4')" \ + "unit(py='3.5')" \ + "unit(py='3.6')" \ || exit_code=1 done diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index cca4bef2..b5bb7cae 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -59,7 +59,7 @@ google-api-python-client==1.6.4 greenlet==0.4.12 gunicorn==19.7.1 hiredis==0.2.0 -honcho=1.0.1 +honcho==1.0.1 html5lib==0.999999999 httplib2==0.10.3 idna==2.6 From 3a3b04c4a002f6a6d87f377dae8bbea475359ee5 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Fri, 3 Nov 2017 11:05:31 -0700 Subject: [PATCH 251/362] Auto-update dependencies. (#166) --- tests/integration/requirements.txt | 6 +++--- tests/python2-libraries/requirements.txt | 12 ++++++------ tests/python3-libraries/requirements.txt | 12 ++++++------ 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index eae15fe9..3e51c81d 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -1,7 +1,7 @@ Flask==0.12.2 -google-cloud-error-reporting==0.27.0 -google-cloud-logging==1.3.0 -google-cloud-monitoring==0.27.0 +google-cloud-error-reporting==0.28.0 +google-cloud-logging==1.4.0 +google-cloud-monitoring==0.28.0 gunicorn==19.7.1 requests==2.18.4 retrying==1.3.3 diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 6db2c17c..8c134e48 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -6,7 +6,7 @@ anyjson==0.3.3 apache-libcloud==2.2.1 argparse==1.4.0 astroid==1.5.3 -awscli==1.11.178 +awscli==1.11.180 babel==2.5.1 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 @@ -16,7 +16,7 @@ billiard==3.5.0.3 blessings==1.6 blinker==1.4 boto==2.48.0 -botocore==1.7.36 +botocore==1.7.38 bottle==0.12.13 carbon==1.0.2 celery==4.1.0 @@ -32,14 +32,14 @@ cov-core==1.15.0 coverage==4.4.1 coveralls==1.2.0 crcmod==1.7 -cryptography==2.1.2 +cryptography==2.1.3 cssselect==1.0.1 cython==0.27.2 decorator==4.1.2 django-celery==3.2.1 django-debug-toolbar==1.8 -django-extensions==1.9.6 -django==1.11.6 +django-extensions==1.9.7 +django==1.11.7 django_compress==1.0.1 djangorestframework==3.7.1 docker-py==1.10.6 @@ -165,7 +165,7 @@ requests==2.18.4 retrying==1.3.3 rsa==3.4.2 scipy==1.0.0 -selenium==3.6.0 +selenium==3.7.0 setuptools-git==1.2 setuptools==36.6.0 sh==1.12.14 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index b5bb7cae..2bdcc565 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -6,7 +6,7 @@ anyjson==0.3.3 apache-libcloud==2.2.1 argparse==1.4.0 astroid==1.5.3 -awscli==1.11.178 +awscli==1.11.180 babel==2.5.1 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 @@ -15,7 +15,7 @@ billiard==3.5.0.3 blessings==1.6 blinker==1.4 boto==2.48.0 -botocore==1.7.36 +botocore==1.7.38 bottle==0.12.13 celery==4.1.0 certifi==2017.7.27.1 @@ -30,14 +30,14 @@ cov-core==1.15.0 coverage==4.4.1 coveralls==1.2.0 crcmod==1.7 -cryptography==2.1.2 +cryptography==2.1.3 cssselect==1.0.1 cython==0.27.2 decorator==4.1.2 django-celery==3.2.1 django-debug-toolbar==1.8 -django-extensions==1.9.6 -django==1.11.6 +django-extensions==1.9.7 +django==1.11.7 django_compress==1.0.1 djangorestframework==3.7.1 docker-py==1.10.6 @@ -156,7 +156,7 @@ requests==2.18.4 retrying==1.3.3 rsa==3.4.2 scipy==1.0.0 -selenium==3.6.0 +selenium==3.7.0 setuptools-git==1.2 setuptools==36.6.0 sh==1.12.14 From f811fb8083a3fcd3e00c9c211af5c50d8b0e8076 Mon Sep 17 00:00:00 2001 From: Angela Li Date: Tue, 14 Nov 2017 10:35:37 -0800 Subject: [PATCH 252/362] Update metrics script to use the latest bigquery client library (#169) --- perf_dashboard/bq_utils.py | 22 ++++--------- perf_dashboard/python_clientlibs_download.py | 33 +++++++------------- 2 files changed, 18 insertions(+), 37 deletions(-) diff --git a/perf_dashboard/bq_utils.py b/perf_dashboard/bq_utils.py index d9441f07..cbad65b9 100644 --- a/perf_dashboard/bq_utils.py +++ b/perf_dashboard/bq_utils.py @@ -22,25 +22,15 @@ def insert_rows(project, dataset_name, table_name, rows): """Insert rows to bigquery table.""" client = bigquery.Client(project=project) - dataset = client.dataset(dataset_name) - table = bigquery.Table(table_name, dataset) - table.reload() - table.insert_data(rows) - + dataset_ref = client.dataset(dataset_name) + table_ref = dataset_ref.table(table_name) + table = client.get_table(table_ref) + client.create_rows(table, rows) def execute_query(query): """Execute query and return the query results.""" client = bigquery.Client() - query_job = client.run_async_query(str(uuid.uuid4()), query) - query_job.use_legacy_sql = False + query_job = client.query((query)) # Start the query job and wait it to complete - query_job.begin() - query_job.result() - - # Get the results - destination_table = query_job.destination - destination_table.reload() - results = destination_table.fetch_data() - - return results + return [row.values() for row in query_job.result()] diff --git a/perf_dashboard/python_clientlibs_download.py b/perf_dashboard/python_clientlibs_download.py index 3866fd92..cae3d2c4 100644 --- a/perf_dashboard/python_clientlibs_download.py +++ b/perf_dashboard/python_clientlibs_download.py @@ -101,27 +101,18 @@ def get_weekly_clientlibs_downloads(clientlibs_table_name, date_str): GROUP BY client_library_name """ client = bigquery.Client() - query_job = client.run_async_query( - str(uuid.uuid4()), - query, - query_parameters=( - bigquery.ArrayQueryParameter( - 'client_libs', 'STRING', - client_libs), - bigquery.ArrayQueryParameter( - 'week_dates', 'STRING', - week_dates) - )) - query_job.use_legacy_sql = False - - # Start the query job and wait it to complete - query_job.begin() - query_job.result() - - # Get the results - destination_table = query_job.destination - destination_table.reload() - results = destination_table.fetch_data() + query_parameters=[ + bigquery.ArrayQueryParameter( + 'client_libs', 'STRING', client_libs), + bigquery.ArrayQueryParameter( + 'week_dates', 'STRING', week_dates) + ] + job_config = bigquery.QueryJobConfig() + job_config.query_parameters = query_parameters + query_job = client.query(query, job_config=job_config) + + # Wait for the job to complete and get the results + results = [row.values() for row in query_job.result()] rows = [(date_time,) + row for row in results] From 4a5026620adc4916b17ce099f5ca642ede5a2bda Mon Sep 17 00:00:00 2001 From: Douglas Greiman Date: Wed, 15 Nov 2017 10:45:41 -0800 Subject: [PATCH 253/362] Revert to previous virtualenv behavior. (#168) * Revert to previous virtualenv behavior. A previous change caused the 'virtualenv' command change to switch its default Python version from 2.7 to 3.6 (when no -p flag is passed). Although not a bad change, it was accidental, and this reverts to the previous behavior. This also add tests for the behavior, and fixes various tests that had been subtly broken by a previous refactoring. The cause of this behavior change was that the 'virtualenv' package was installed under each Python interpreter, instead of the 'wheel' package. * Pin the virtualenv version. * Fix typos * Fix another typo --- cloudbuild_test.yaml | 1 + runtime-image/Dockerfile.in | 3 +- .../resources/requirements-virtualenv.txt | 1 + runtime-image/resources/requirements.txt | 2 +- tests/no-virtualenv/no-virtualenv.yaml | 34 ++++++++++++++ .../python2-libraries/python2-libraries.yaml | 2 +- tests/virtualenv/virtualenv_default.yaml | 25 +++++----- tests/virtualenv/virtualenv_python27.yaml | 47 +++++++++++++++++++ tests/virtualenv/virtualenv_python34.yaml | 34 ++++++++------ tests/virtualenv/virtualenv_python35.yaml | 34 +++++++------- tests/virtualenv/virtualenv_python36.yaml | 34 +++++++------- 11 files changed, 150 insertions(+), 67 deletions(-) create mode 100644 runtime-image/resources/requirements-virtualenv.txt create mode 100644 tests/virtualenv/virtualenv_python27.yaml diff --git a/cloudbuild_test.yaml b/cloudbuild_test.yaml index c6496dec..acf70bc7 100644 --- a/cloudbuild_test.yaml +++ b/cloudbuild_test.yaml @@ -26,6 +26,7 @@ steps: '-test.v', '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', '/workspace/tests/virtualenv/virtualenv_default.yaml', + '/workspace/tests/virtualenv/virtualenv_python27.yaml', '/workspace/tests/virtualenv/virtualenv_python34.yaml', '/workspace/tests/virtualenv/virtualenv_python35.yaml', '/workspace/tests/virtualenv/virtualenv_python36.yaml', diff --git a/runtime-image/Dockerfile.in b/runtime-image/Dockerfile.in index 60a6fa6f..7fafb2c4 100644 --- a/runtime-image/Dockerfile.in +++ b/runtime-image/Dockerfile.in @@ -27,7 +27,8 @@ ENV PATH /opt/python3.6/bin:/opt/python3.5/bin:$PATH RUN /usr/bin/pip install --upgrade -r /resources/requirements.txt && \ /usr/bin/pip3 install --upgrade -r /resources/requirements.txt && \ /opt/python3.5/bin/pip3.5 install --upgrade -r /resources/requirements.txt && \ - /opt/python3.6/bin/pip3.6 install --upgrade -r /resources/requirements.txt + /opt/python3.6/bin/pip3.6 install --upgrade -r /resources/requirements.txt && \ + /usr/bin/pip install --upgrade -r /resources/requirements-virtualenv.txt # Setup the app working directory RUN ln -s /home/vmagent/app /app diff --git a/runtime-image/resources/requirements-virtualenv.txt b/runtime-image/resources/requirements-virtualenv.txt new file mode 100644 index 00000000..a4ce32e5 --- /dev/null +++ b/runtime-image/resources/requirements-virtualenv.txt @@ -0,0 +1 @@ +virtualenv==15.1.0 diff --git a/runtime-image/resources/requirements.txt b/runtime-image/resources/requirements.txt index 40155447..c9f190ec 100644 --- a/runtime-image/resources/requirements.txt +++ b/runtime-image/resources/requirements.txt @@ -1,3 +1,3 @@ pip==9.0.1 setuptools==36.6.0 -virtualenv==15.1.0 +wheel==0.30.0 diff --git a/tests/no-virtualenv/no-virtualenv.yaml b/tests/no-virtualenv/no-virtualenv.yaml index de932617..f43d08a5 100644 --- a/tests/no-virtualenv/no-virtualenv.yaml +++ b/tests/no-virtualenv/no-virtualenv.yaml @@ -4,7 +4,41 @@ commandTests: command: ["which", "python"] expectedOutput: ["/usr/bin/python\n"] + - name: "default pip installation" + command: ["which", "pip"] + expectedOutput: ["/usr/local/bin/pip\n"] + + - name: "default pip python version" + command: ["pip", "-V"] + expectedOutput: ["pip .* from .*python 2[.]7"] + + - name: "default virtualenv installation" + command: ["which", "virtualenv"] + expectedOutput: ["/usr/local/bin/virtualenv\n"] + + - name: "default python2.7 installation" + command: ["which", "python2.7"] + expectedOutput: ["/usr/bin/python2.7\n"] + + - name: "default python3.4 installation" + command: ["which", "python3.4"] + expectedOutput: ["/usr/bin/python3.4\n"] + + - name: "default python3.5 installation" + command: ["which", "python3.5"] + expectedOutput: ["/opt/python3.5/bin/python3.5\n"] + + - name: "default python3.6 installation" + command: ["which", "python3.6"] + expectedOutput: ["/opt/python3.6/bin/python3.6\n"] + - name: "default gunicorn installation" setup: [["pip", "install", "gunicorn"]] command: ["which", "gunicorn"] expectedOutput: ["/usr/local/bin/gunicorn\n"] + + - name: "default flask installation" + # Checks that 'pip' and 'python' are using the same Python version + setup: [["pip", "install", "flask"]] + command: ["python", "-c", "import flask; print(flask.__file__)"] + expectedOutput: ["/usr/local/lib/python2.7/dist-packages/flask"] diff --git a/tests/python2-libraries/python2-libraries.yaml b/tests/python2-libraries/python2-libraries.yaml index 3a2771ac..f9fef5a0 100644 --- a/tests/python2-libraries/python2-libraries.yaml +++ b/tests/python2-libraries/python2-libraries.yaml @@ -8,6 +8,6 @@ globalEnvVars: commandTests: - name: "requirements" - setup: [["virtualenv", "/env"]] + setup: [["virtualenv", "-p", "python", "/env"]] command: ["pip", "install", "-r", "/requirements.txt"] exitCode: 0 diff --git a/tests/virtualenv/virtualenv_default.yaml b/tests/virtualenv/virtualenv_default.yaml index 8695e319..60714de6 100644 --- a/tests/virtualenv/virtualenv_default.yaml +++ b/tests/virtualenv/virtualenv_default.yaml @@ -7,20 +7,13 @@ globalEnvVars: value: "/env/bin:$PATH" commandTests: - - name: "python installation" - command: ["which", "python"] - expectedOutput: ["/usr/bin/python\n"] - - - name: "pip installation" - command: ["which", "pip"] - expectedOutput: ["/usr/local/bin/pip\n"] - - - name: "virtualenv installation" + - name: "virtualenv python installation" setup: [["virtualenv", "/env"]] command: ["which", "python"] expectedOutput: ["/env/bin/python\n"] - - name: "python version" + - name: "virtualenv python version" + setup: [["virtualenv", "/env"]] command: ["python", "--version"] # we check stderr instead of stdout for Python versions < 3.4 # https://bugs.python.org/issue18338 @@ -31,10 +24,14 @@ commandTests: command: ["which", "pip"] expectedOutput: ["/env/bin/pip\n"] - - name: "virtualenv gunicorn flask" - setup: [["virtualenv", "/env"], ["pip", "install", "gunicorn", "flask"]] + - name: "virtualenv gunicorn installation" + setup: [["virtualenv", "/env"], + ["pip", "install", "gunicorn"]] command: ["which", "gunicorn"] expectedOutput: ["/env/bin/gunicorn"] - - name: "flask integration" - command: ["python", "-c", "\"import sys; import flask; sys.exit(0 if flask.__file__.startswith('/env') else 1)\""] + - name: "virtualenv flask installation" + setup: [["virtualenv", "/env"], + ["pip", "install", "flask"]] + command: ["python", "-c", "import flask; print(flask.__file__)"] + expectedOutput: ["/env/local/lib/python2.7/site-packages/flask"] diff --git a/tests/virtualenv/virtualenv_python27.yaml b/tests/virtualenv/virtualenv_python27.yaml new file mode 100644 index 00000000..d28ea541 --- /dev/null +++ b/tests/virtualenv/virtualenv_python27.yaml @@ -0,0 +1,47 @@ +schemaVersion: "1.0.0" + +globalEnvVars: + - key: "VIRTUAL_ENV" + value: "/env" + - key: "PATH" + value: "/env/bin:$PATH" + +commandTests: + - name: "virtualenv27 python installation" + setup: [["virtualenv", "-p", "python", "/env"]] + command: ["which", "python"] + expectedOutput: ["/env/bin/python\n"] + + - name: "virtualenv27 python2 installation" + setup: [["virtualenv", "-p", "python", "/env"]] + command: ["which", "python2"] + expectedOutput: ["/env/bin/python2\n"] + + - name: "virtualenv27python2.7 installation" + setup: [["virtualenv", "-p", "python", "/env"]] + command: ["which", "python2.7"] + expectedOutput: ["/env/bin/python2.7\n"] + + - name: "virtualenv27 python version" + setup: [["virtualenv", "-p", "python", "/env"]] + command: ["python", "--version"] + # we check stderr instead of stdout for Python versions < 3.4 + # https://bugs.python.org/issue18338 + expectedError: ["Python 2.7.9\n"] + + - name: "virtualenv27 pip installation" + setup: [["virtualenv", "-p", "python", "/env"]] + command: ["which", "pip"] + expectedOutput: ["/env/bin/pip\n"] + + - name: "virtualenv27 gunicorn installation" + setup: [["virtualenv", "-p", "python", "/env"], + ["pip", "install", "gunicorn"]] + command: ["which", "gunicorn"] + expectedOutput: ["/env/bin/gunicorn"] + + - name: "virtualenv27 flask installation" + setup: [["virtualenv", "-p", "python", "/env"], + ["pip", "install", "flask"]] + command: ["python", "-c", "import flask; print(flask.__file__)"] + expectedOutput: ["/env/local/lib/python2.7/site-packages/flask"] diff --git a/tests/virtualenv/virtualenv_python34.yaml b/tests/virtualenv/virtualenv_python34.yaml index e35e9693..779f4d45 100644 --- a/tests/virtualenv/virtualenv_python34.yaml +++ b/tests/virtualenv/virtualenv_python34.yaml @@ -7,42 +7,48 @@ globalEnvVars: value: "/env/bin:$PATH" commandTests: - - name: "python installation" - command: ["which", "python3.4"] - expectedOutput: ["/usr/bin/python3.4\n"] - - - name: "virtualenv python installation" + - name: "virtualenv34 python installation" setup: [["virtualenv", "-p", "python3.4", "/env"]] command: ["which", "python"] expectedOutput: ["/env/bin/python\n"] - - name: "virtualenv python3 installation" + - name: "virtualenv34 python3 installation" setup: [["virtualenv", "-p", "python3.4", "/env"]] command: ["which", "python3"] expectedOutput: ["/env/bin/python3\n"] - - name: "python version" + - name: "virtualenv34 python3.4 installation" + setup: [["virtualenv", "-p", "python3.4", "/env"]] + command: ["which", "python3.4"] + expectedOutput: ["/env/bin/python3.4\n"] + + - name: "virtualenv34 python version" setup: [["virtualenv", "-p", "python3.4", "/env"]] command: ["python", "--version"] expectedOutput: ["Python 3.4.2\n"] - - name: "pip installation" + - name: "virtualenv34 pip installation" setup: [["virtualenv", "-p", "python3.4", "/env"]] command: ["which", "pip"] expectedOutput: ["/env/bin/pip\n"] - - name: "pip3 installation" + - name: "virtualenv34 pip3 installation" setup: [["virtualenv", "-p", "python3.4", "/env"]] command: ["which", "pip3"] expectedOutput: ["/env/bin/pip3\n"] - - name: "gunicorn flask" - setup: [["virtualenv", "-p", "python3.4", "/env"], ["pip", "install", "gunicorn", "flask"]] + - name: "virtualenv34 gunicorn installation" + setup: [["virtualenv", "-p", "python3.4", "/env"], + ["pip", "install", "gunicorn"]] command: ["which", "gunicorn"] expectedOutput: ["/env/bin/gunicorn"] - - name: "flask integration" - command: ["python", "-c", "\"import sys; import flask; sys.exit(0 if flask.__file__.startswith('/env') else 1)\""] + - name: "virtualenv34 flask installation" + setup: [["virtualenv", "-p", "python3.4", "/env"], + ["pip", "install", "flask"]] + command: ["python", "-c", "import flask; print(flask.__file__)"] + expectedOutput: ["/env/lib/python3.4/site-packages/flask"] - - name: "test.support" + - name: "virtualenv34 test.support availability" + setup: [["virtualenv", "-p", "python3.4", "/env"]] command: ["python", "-c", "\"from test import pystone, regrtest, support\""] diff --git a/tests/virtualenv/virtualenv_python35.yaml b/tests/virtualenv/virtualenv_python35.yaml index 4b019967..4150ba19 100644 --- a/tests/virtualenv/virtualenv_python35.yaml +++ b/tests/virtualenv/virtualenv_python35.yaml @@ -7,50 +7,48 @@ globalEnvVars: value: "/env/bin:$PATH" commandTests: - - name: "virtual env teardown" - command: ["rm", "-rf", "/env"] - - - name: "python installation" - command: ["which", "python3.5"] - expectedOutput: ["/opt/python3.5/bin/python3.5\n"] - - - name: "virtualenv python installation" + - name: "virtualenv35 python installation" setup: [["virtualenv", "-p", "python3.5", "/env"]] command: ["which", "python"] expectedOutput: ["/env/bin/python\n"] - - name: "virtualenv python3 installation" + - name: "virtualenv35 python3 installation" setup: [["virtualenv", "-p", "python3.5", "/env"]] command: ["which", "python3"] expectedOutput: ["/env/bin/python3\n"] - - name: "virtualenv python3.5 installation" + - name: "virtualenv35 python3.5 installation" setup: [["virtualenv", "-p", "python3.5", "/env"]] command: ["which", "python3.5"] expectedOutput: ["/env/bin/python3.5\n"] - - name: "python version" + - name: "virtualenv35 python version" setup: [["virtualenv", "-p", "python3.5", "/env"]] command: ["python", "--version"] expectedOutput: ["Python 3.5.4\n"] - - name: "pip installation" + - name: "virtualenv35 pip installation" setup: [["virtualenv", "-p", "python3.5", "/env"]] command: ["which", "pip"] expectedOutput: ["/env/bin/pip\n"] - - name: "pip3 installation" + - name: "virtualenv35 pip3 installation" setup: [["virtualenv", "-p", "python3.5", "/env"]] command: ["which", "pip3"] expectedOutput: ["/env/bin/pip3\n"] - - name: "gunicorn flask" - setup: [["virtualenv", "-p", "python3.5", "/env"], ["pip", "install", "gunicorn", "flask"]] + - name: "virtualenv35 gunicorn installation" + setup: [["virtualenv", "-p", "python3.5", "/env"], + ["pip", "install", "gunicorn"]] command: ["which", "gunicorn"] expectedOutput: ["/env/bin/gunicorn"] - - name: "flask integration" - command: ["python", "-c", "\"import sys; import flask; sys.exit(0 if flask.__file__.startswith('/env') else 1)\""] + - name: "virtualenv35 flask installation" + setup: [["virtualenv", "-p", "python3.5", "/env"], + ["pip", "install", "flask"]] + command: ["python", "-c", "import flask; print(flask.__file__)"] + expectedOutput: ["/env/lib/python3.5/site-packages/flask"] - - name: "test.support" + - name: "virtualenv35 test.support availability" + setup: [["virtualenv", "-p", "python3.5", "/env"]] command: ["python", "-c", "\"from test import pystone, regrtest, support\""] diff --git a/tests/virtualenv/virtualenv_python36.yaml b/tests/virtualenv/virtualenv_python36.yaml index deea9b5d..71c91a9f 100644 --- a/tests/virtualenv/virtualenv_python36.yaml +++ b/tests/virtualenv/virtualenv_python36.yaml @@ -7,50 +7,48 @@ globalEnvVars: value: "/env/bin:$PATH" commandTests: - - name: "virtual env teardown" - command: ["rm", "-rf", "/env"] - - - name: "python installation" - command: ["which", "python3.6"] - expectedOutput: ["/opt/python3.6/bin/python3.6\n"] - - - name: "virtualenv python installation" + - name: "virtualenv36 python installation" setup: [["virtualenv", "-p", "python3.6", "/env"]] command: ["which", "python"] expectedOutput: ["/env/bin/python\n"] - - name: "virtualenv python3 installation" + - name: "virtualenv36 python3 installation" setup: [["virtualenv", "-p", "python3.6", "/env"]] command: ["which", "python3"] expectedOutput: ["/env/bin/python3\n"] - - name: "virtualenv python3.6 installation" + - name: "virtualenv36 python3.6 installation" setup: [["virtualenv", "-p", "python3.6", "/env"]] command: ["which", "python3.6"] expectedOutput: ["/env/bin/python3.6\n"] - - name: "python version" + - name: "virtualenv36 python version" setup: [["virtualenv", "-p", "python3.6", "/env"]] command: ["python", "--version"] expectedOutput: ["Python 3.6.2\n"] - - name: "pip installation" + - name: "virtualenv36 pip installation" setup: [["virtualenv", "-p", "python3.6", "/env"]] command: ["which", "pip"] expectedOutput: ["/env/bin/pip\n"] - - name: "pip3 installation" + - name: "virtualenv36 pip3 installation" setup: [["virtualenv", "-p", "python3.6", "/env"]] command: ["which", "pip3"] expectedOutput: ["/env/bin/pip3\n"] - - name: "gunicorn flask" - setup: [["virtualenv", "-p", "python3.6", "/env"], ["pip", "install", "gunicorn", "flask"]] + - name: "virtualenv36 gunicorn installation" + setup: [["virtualenv", "-p", "python3.6", "/env"], + ["pip", "install", "gunicorn"]] command: ["which", "gunicorn"] expectedOutput: ["/env/bin/gunicorn"] - - name: "flask integration" - command: ["python", "-c", "\"import sys; import flask; sys.exit(0 if flask.__file__.startswith('/env') else 1)\""] + - name: "virtualenv36 flask installation" + setup: [["virtualenv", "-p", "python3.6", "/env"], + ["pip", "install", "flask"]] + command: ["python", "-c", "import flask; print(flask.__file__)"] + expectedOutput: ["/env/lib/python3.6/site-packages/flask"] - - name: "test.support" + - name: "virtualenv36 test.support availability" + setup: [["virtualenv", "-p", "python3.6", "/env"]] command: ["python", "-c", "\"from test import pystone, regrtest, support\""] From 714dcb7f83aba9e1c80b9a739294a4864109a607 Mon Sep 17 00:00:00 2001 From: Nick Kubala Date: Wed, 15 Nov 2017 14:18:04 -0800 Subject: [PATCH 254/362] return string instead of bool in integration test sample app env handler --- tests/integration/server.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/integration/server.py b/tests/integration/server.py index 8e2958cd..0b4382c6 100755 --- a/tests/integration/server.py +++ b/tests/integration/server.py @@ -245,8 +245,11 @@ def _check_environment(): # for GAE, we'll check the existence env vars set on # vm:true or env:flex # if neither exist, assume we're in GKE - return (_APPENGINE_FLEXIBLE_ENV_VM in os.environ or - _APPENGINE_FLEXIBLE_ENV_FLEX in os.environ), 200 + environment = "GKE" + if (_APPENGINE_FLEXIBLE_ENV_VM in os.environ or + _APPENGINE_FLEXIBLE_ENV_FLEX in os.environ): + environment = "GAE" + return environment, 200 class ErrorResponse(Exception): From 7b8bf15059075f06ce2c5c6e61d1445ba3c1094e Mon Sep 17 00:00:00 2001 From: Angela Li Date: Mon, 4 Dec 2017 13:34:58 -0800 Subject: [PATCH 255/362] Update system tests script (#171) --- tests/google-cloud-python-system/run_system_tests.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/google-cloud-python-system/run_system_tests.sh b/tests/google-cloud-python-system/run_system_tests.sh index 17d3aac3..f9f81a47 100755 --- a/tests/google-cloud-python-system/run_system_tests.sh +++ b/tests/google-cloud-python-system/run_system_tests.sh @@ -31,8 +31,8 @@ for package in ${packages}; do nox \ -f "${noxfile}" \ -e \ - "system_tests(python_version='2.7')" \ - "system_tests(python_version='3.6')" \ + "system(py='2.7')" \ + "system(py='3.6')" \ || exit_code=1 done From 597a87474472699e2611f0d170becd69c2b244b2 Mon Sep 17 00:00:00 2001 From: Angela Li Date: Wed, 3 Jan 2018 18:07:37 -0800 Subject: [PATCH 256/362] Skip test_xmlrpc_net when building python interpreters (#175) --- python-interpreter-builder/scripts/build-python-3.5.sh | 3 ++- python-interpreter-builder/scripts/build-python-3.6.sh | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/python-interpreter-builder/scripts/build-python-3.5.sh b/python-interpreter-builder/scripts/build-python-3.5.sh index f9113879..eeea1f76 100755 --- a/python-interpreter-builder/scripts/build-python-3.5.sh +++ b/python-interpreter-builder/scripts/build-python-3.5.sh @@ -123,7 +123,8 @@ make profile-opt # test_dbm: https://bugs.python.org/issue28700 # test_imap: https://bugs.python.org/issue30175 # test_shutil: https://bugs.python.org/issue29317 -make test TESTOPTS="--exclude test___all__ test_dbm test_imaplib test_shutil" +# test_xmlrpc_net: https://bugs.python.org/issue31724 +make test TESTOPTS="--exclude test___all__ test_dbm test_imaplib test_shutil test_xmlrpc_net" # Install make altinstall diff --git a/python-interpreter-builder/scripts/build-python-3.6.sh b/python-interpreter-builder/scripts/build-python-3.6.sh index 431c305d..ca089d59 100755 --- a/python-interpreter-builder/scripts/build-python-3.6.sh +++ b/python-interpreter-builder/scripts/build-python-3.6.sh @@ -123,7 +123,8 @@ make profile-opt # test_dbm: https://bugs.python.org/issue28700 # test_imap: https://bugs.python.org/issue30175 # test_shutil: https://bugs.python.org/issue29317 -make test TESTOPTS="--exclude test___all__ test_dbm test_imaplib test_shutil" +# test_xmlrpc_net: https://bugs.python.org/issue31724 +make test TESTOPTS="--exclude test___all__ test_dbm test_imaplib test_shutil test_xmlrpc_net" # Install make altinstall From 01a4da0bc4a429b24550912f2f2ba1e732b4ac36 Mon Sep 17 00:00:00 2001 From: Jon Donovan Date: Fri, 12 Jan 2018 12:43:31 -0800 Subject: [PATCH 257/362] Python compat (#174) * Add support for python-compat * Update app.yaml and tests * Add missing .dockerignore file * Updating per review comments * Adding missing files * Update gen_dockerfile.py * Update gen_dockerfile_test.py --- scripts/data/Dockerfile.python_compat | 3 ++ scripts/data/dockerignore.python_compat | 5 +++ scripts/gen_dockerfile.py | 43 +++++++++++++------ scripts/gen_dockerfile_test.py | 32 +++++++++++--- scripts/testdata/hello_world_compat/app.yaml | 13 ++++++ scripts/testdata/hello_world_compat/main.py | 14 ++++++ .../hello_world_compat_golden/.dockerignore | 5 +++ .../hello_world_compat_golden/Dockerfile | 3 ++ 8 files changed, 97 insertions(+), 21 deletions(-) create mode 100644 scripts/data/Dockerfile.python_compat create mode 100644 scripts/data/dockerignore.python_compat create mode 100644 scripts/testdata/hello_world_compat/app.yaml create mode 100644 scripts/testdata/hello_world_compat/main.py create mode 100644 scripts/testdata/hello_world_compat_golden/.dockerignore create mode 100644 scripts/testdata/hello_world_compat_golden/Dockerfile diff --git a/scripts/data/Dockerfile.python_compat b/scripts/data/Dockerfile.python_compat new file mode 100644 index 00000000..1e4d6352 --- /dev/null +++ b/scripts/data/Dockerfile.python_compat @@ -0,0 +1,3 @@ +FROM gcr.io/google_appengine/python-compat-multicore +ADD . /app/ +RUN if [ -s requirements.txt ]; then pip install -r requirements.txt; fi diff --git a/scripts/data/dockerignore.python_compat b/scripts/data/dockerignore.python_compat new file mode 100644 index 00000000..5ce5abfa --- /dev/null +++ b/scripts/data/dockerignore.python_compat @@ -0,0 +1,5 @@ +.dockerignore +Dockerfile +.git +.hg +.svn diff --git a/scripts/gen_dockerfile.py b/scripts/gen_dockerfile.py index eaff183f..5a4ceae2 100755 --- a/scripts/gen_dockerfile.py +++ b/scripts/gen_dockerfile.py @@ -29,7 +29,6 @@ import validation_utils - # Validate characters for dockerfile image names. # # This roots out obvious mistakes, the full gory details are here: @@ -63,7 +62,7 @@ # Validated application configuration AppConfig = collections.namedtuple( 'AppConfig', - 'base_image dockerfile_python_version entrypoint has_requirements_txt' + 'base_image dockerfile_python_version entrypoint has_requirements_txt is_python_compat' ) @@ -97,6 +96,16 @@ def get_app_config(raw_config, base_image, config_file, source_dir): 'Expected {} contents to be a Mapping type, but found type "{}"'. format(config_file, type(raw_config))) + # Short circuit for python compat. + if validation_utils.get_field_value( + raw_config, 'runtime', str) == 'python-compat': + return AppConfig( + base_image=None, + dockerfile_python_version=None, + entrypoint=None, + has_requirements_txt=None, + is_python_compat=True) + entrypoint = validation_utils.get_field_value( raw_config, 'entrypoint', str) if not PRINTABLE_REGEX.match(entrypoint): @@ -133,7 +142,8 @@ def get_app_config(raw_config, base_image, config_file, source_dir): base_image=base_image, dockerfile_python_version=dockerfile_python_version, entrypoint=entrypoint, - has_requirements_txt=has_requirements_txt) + has_requirements_txt=has_requirements_txt, + is_python_compat=False) def get_data(name): @@ -175,19 +185,24 @@ def generate_files(app_config): else: optional_entrypoint = '' - dockerfile = ''.join([ - get_data('Dockerfile.preamble.template').format( - base_image=app_config.base_image), - get_data('Dockerfile.virtualenv.template').format( - python_version=app_config.dockerfile_python_version), - optional_requirements_txt, - get_data('Dockerfile.install_app'), - optional_entrypoint, - ]) + if app_config.is_python_compat: + dockerfile = get_data('Dockerfile.python_compat') + dockerignore = get_data('dockerignore.python_compat') + else: + dockerfile = ''.join([ + get_data('Dockerfile.preamble.template').format( + base_image=app_config.base_image), + get_data('Dockerfile.virtualenv.template').format( + python_version=app_config.dockerfile_python_version), + optional_requirements_txt, + get_data('Dockerfile.install_app'), + optional_entrypoint, + ]) + dockerignore = get_data('dockerignore') return { 'Dockerfile': dockerfile, - '.dockerignore': get_data('dockerignore'), + '.dockerignore': dockerignore, } @@ -206,7 +221,7 @@ def generate_dockerfile_command(base_image, config_file, source_dir): # Determine complete configuration app_config = get_app_config(raw_config, base_image, config_file, - source_dir) + source_dir) # Generate list of filenames and their textual contents files = generate_files(app_config) diff --git a/scripts/gen_dockerfile_test.py b/scripts/gen_dockerfile_test.py index b52e15e8..8d1d0798 100755 --- a/scripts/gen_dockerfile_test.py +++ b/scripts/gen_dockerfile_test.py @@ -54,6 +54,14 @@ def compare_file(filename, dir1, dir2): 'dockerfile_python_version': '', 'has_requirements_txt': False, 'entrypoint': '', + 'is_python_compat': False, + }), + ('env: flex\nruntime: python-compat', { + 'base_image': None, + 'dockerfile_python_version': None, + 'has_requirements_txt': None, + 'entrypoint': None, + 'is_python_compat': True, }), # All supported python versions ('runtime_config:\n python_version:', { @@ -125,7 +133,8 @@ def test_get_app_config_invalid(app_yaml): base_image='', dockerfile_python_version='', entrypoint='', - has_requirements_txt=False + has_requirements_txt=False, + is_python_compat=False, ) @@ -146,6 +155,9 @@ def test_get_app_config_invalid(app_yaml): # Python version (_BASE_APP_CONFIG._replace(dockerfile_python_version='_my_version'), True, 'python_version=python_my_version'), + # python-compat runtime + (_BASE_APP_CONFIG._replace(is_python_compat=True), True, + 'FROM gcr.io/google_appengine/python-compat-multicore'), ]) def test_generate_files(app_config, should_find, test_string): result = gen_dockerfile.generate_files(app_config) @@ -163,10 +175,13 @@ def compare_against_golden_files(app, config_dir, testdata_dir): compare_file(filename, config_dir, golden_dir) -def test_generate_dockerfile_command(tmpdir, testdata_dir): +@pytest.mark.parametrize('app', [ + # Sampled from https://github.com/GoogleCloudPlatform/python-docs-samples + 'hello_world', + # From an internal source. + 'hello_world_compat']) +def test_generate_dockerfile_command(tmpdir, testdata_dir, app): """Generates output and compares against a set of golden files.""" - # Sample from https://github.com/GoogleCloudPlatform/python-docs-samples - app = 'hello_world' app_dir = os.path.join(testdata_dir, app) # Copy sample app to writable temp dir, and generate Dockerfile. @@ -179,12 +194,15 @@ def test_generate_dockerfile_command(tmpdir, testdata_dir): compare_against_golden_files(app, config_dir, testdata_dir) +@pytest.mark.parametrize('app', [ + # Sampled from https://github.com/GoogleCloudPlatform/python-docs-samples + 'hello_world', + # From an internal source. + 'hello_world_compat']) @pytest.mark.xfail(not shutil.which('gcloud'), reason='Google Cloud SDK is not installed') -def test_generate_dockerfile_golden(tmpdir, testdata_dir): +def test_generate_dockerfile_golden(tmpdir, testdata_dir, app): """Validate our golden files against gcloud app gen-config""" - # Sample from https://github.com/GoogleCloudPlatform/python-docs-samples - app = 'hello_world' app_dir = os.path.join(testdata_dir, app) # Copy sample app to writable temp dir, and generate Dockerfile. diff --git a/scripts/testdata/hello_world_compat/app.yaml b/scripts/testdata/hello_world_compat/app.yaml new file mode 100644 index 00000000..e514d42c --- /dev/null +++ b/scripts/testdata/hello_world_compat/app.yaml @@ -0,0 +1,13 @@ +service: default +runtime: python-compat +env: flex + +api_version: 1 +threadsafe: true + +beta_settings: + enable_app_engine_apis: true # Needed for compat apps. + +handlers: +- url: .* + script: main.app diff --git a/scripts/testdata/hello_world_compat/main.py b/scripts/testdata/hello_world_compat/main.py new file mode 100644 index 00000000..40302722 --- /dev/null +++ b/scripts/testdata/hello_world_compat/main.py @@ -0,0 +1,14 @@ +"""The hello world flex app!""" + +import webapp2 + + +class HelloHandler(webapp2.RequestHandler): + + def get(self): + msg = 'Hello GAE Flex (env: flex) Compat-Runtime App\n' + self.response.headers['Content-Type'] = 'text/plain' + self.response.out.write(msg) + +app = webapp2.WSGIApplication([('/', HelloHandler)], + debug=True) diff --git a/scripts/testdata/hello_world_compat_golden/.dockerignore b/scripts/testdata/hello_world_compat_golden/.dockerignore new file mode 100644 index 00000000..5ce5abfa --- /dev/null +++ b/scripts/testdata/hello_world_compat_golden/.dockerignore @@ -0,0 +1,5 @@ +.dockerignore +Dockerfile +.git +.hg +.svn diff --git a/scripts/testdata/hello_world_compat_golden/Dockerfile b/scripts/testdata/hello_world_compat_golden/Dockerfile new file mode 100644 index 00000000..1e4d6352 --- /dev/null +++ b/scripts/testdata/hello_world_compat_golden/Dockerfile @@ -0,0 +1,3 @@ +FROM gcr.io/google_appengine/python-compat-multicore +ADD . /app/ +RUN if [ -s requirements.txt ]; then pip install -r requirements.txt; fi From dfd43223fd8b40b8e90fd3b6cd8e088ca8f04e86 Mon Sep 17 00:00:00 2001 From: Julius Adorf Date: Wed, 24 Jan 2018 17:14:01 +0000 Subject: [PATCH 258/362] Replace "Container Engine" by "Kubernetes Engine". (#176) --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 78a02c5c..2cc7d173 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ This repository contains the source for the [docker](https://docker.io) base image. This image can be used as the base image for running applications on [Google App Engine Flexible](https://cloud.google.com/appengine/docs/flexible/), -[Google Container Engine](https://cloud.google.com/container-engine), or any +[Google Kubernetes Engine](https://cloud.google.com/kubernetes-engine), or any other Docker host. This image is based on Debian Jessie and contains packages required to build @@ -38,7 +38,7 @@ to create a custom runtime: You can then modify the `Dockerfile` and `.dockerignore` as needed for you application. -## Container Engine & other Docker hosts. +## Kubernetes Engine & other Docker hosts. For other docker hosts, you'll need to create a `Dockerfile` based on this image that copies your application code, installs dependencies, and declares an From 389436f9d0d62bf1144ca7cb96f55388c5603f70 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Wed, 7 Feb 2018 17:59:17 -0800 Subject: [PATCH 259/362] Auto-update dependencies. (#167) * Auto-update dependencies. * Pin some library versions to deal with Python 2 and 3 incompatibilties. --- runtime-image/resources/requirements.txt | 2 +- scripts/requirements-test.txt | 2 +- tests/eventlet/requirements.txt | 8 +- tests/integration/requirements.txt | 2 +- tests/python2-libraries/requirements.txt | 158 +++++++++++------------ tests/python3-libraries/requirements.txt | 147 +++++++++++---------- 6 files changed, 159 insertions(+), 160 deletions(-) diff --git a/runtime-image/resources/requirements.txt b/runtime-image/resources/requirements.txt index c9f190ec..6a1b2cc3 100644 --- a/runtime-image/resources/requirements.txt +++ b/runtime-image/resources/requirements.txt @@ -1,3 +1,3 @@ pip==9.0.1 -setuptools==36.6.0 +setuptools==38.5.0 wheel==0.30.0 diff --git a/scripts/requirements-test.txt b/scripts/requirements-test.txt index 5afb5315..12a6ca4a 100644 --- a/scripts/requirements-test.txt +++ b/scripts/requirements-test.txt @@ -1,4 +1,4 @@ flask==0.12.2 -pytest==3.2.3 +pytest==3.4.0 pytest-cov==2.5.1 pyyaml==3.12 diff --git a/tests/eventlet/requirements.txt b/tests/eventlet/requirements.txt index c33e53f2..a8f7b0a9 100644 --- a/tests/eventlet/requirements.txt +++ b/tests/eventlet/requirements.txt @@ -1,10 +1,10 @@ click==6.7 enum-compat==0.0.2 -eventlet==0.21.0 +eventlet==0.22.0 Flask==0.12.2 -greenlet==0.4.12 +greenlet==0.4.13 gunicorn==19.7.1 itsdangerous==0.24 -Jinja2==2.9.6 +Jinja2==2.10 MarkupSafe==1.0 -Werkzeug==0.12.2 +Werkzeug==0.14.1 diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index 3e51c81d..99a06d3b 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -1,5 +1,5 @@ Flask==0.12.2 -google-cloud-error-reporting==0.28.0 +google-cloud-error-reporting==0.29.0 google-cloud-logging==1.4.0 google-cloud-monitoring==0.28.0 gunicorn==19.7.1 diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 8c134e48..f512ca18 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -1,54 +1,54 @@ -alembic==0.9.6 +alembic==0.9.7 amqp==2.2.2 amqplib==1.0.2 -ansible==2.4.1.0 +ansible==2.4.3.0 anyjson==0.3.3 apache-libcloud==2.2.1 argparse==1.4.0 -astroid==1.5.3 -awscli==1.11.180 -babel==2.5.1 +astroid==1.6.1 +awscli==1.14.32 +babel==2.5.3 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 beautifulsoup4==4.6.0 beautifulsoup==3.2.1 billiard==3.5.0.3 -blessings==1.6 +blessings==1.6.1 blinker==1.4 boto==2.48.0 -botocore==1.7.38 +botocore==1.8.36 bottle==0.12.13 -carbon==1.0.2 +carbon<1.1.1 celery==4.1.0 -certifi==2017.7.27.1 -cffi==1.11.2 +certifi==2018.1.18 +cffi==1.11.4 chardet==3.0.4 click==6.7 -cliff==2.9.1 -cmd2==0.7.7 +cliff==2.11.0 +cmd2==0.8.0 colorama==0.3.9 configobj==5.0.6 cov-core==1.15.0 -coverage==4.4.1 +coverage==4.5 coveralls==1.2.0 crcmod==1.7 -cryptography==2.1.3 -cssselect==1.0.1 -cython==0.27.2 -decorator==4.1.2 -django-celery==3.2.1 -django-debug-toolbar==1.8 -django-extensions==1.9.7 -django==1.11.7 +cryptography==2.1.4 +cssselect==1.0.3 +cython==0.27.3 +decorator==4.2.1 +django-celery==3.2.2 +django-debug-toolbar==1.9.1 +django-extensions==1.9.9 +django<2.0 django_compress==1.0.1 -djangorestframework==3.7.1 +djangorestframework==3.7.7 docker-py==1.10.6 docopt==0.6.2 docutils==0.14 ecdsa==0.13 -elasticsearch==5.4.0 +elasticsearch==6.1.1 enum34==1.1.6 -eventlet==0.21.0 +eventlet==0.22.0 extras==1.0.0 fabric==1.14.0 fixtures==3.0.0 @@ -56,108 +56,108 @@ flake8==3.5.0 flask==0.12.2 funcsigs==1.0.2 functools32==3.2.3.post2 -futures==3.1.1 +futures==3.2.0 gevent==1.2.2 -google-api-python-client==1.6.4 -graphite-web==1.0.2 -greenlet==0.4.12 +google-api-python-client==1.6.5 +graphite-web==1.1.1 +greenlet==0.4.13 gunicorn==19.7.1 hiredis==0.2.0 honcho==1.0.1 -html5lib==0.999999999 +html5lib==1.0.1 httplib2==0.10.3 idna==2.6 -ipaddress==1.0.18 +ipaddress==1.0.19 iso8601==0.1.12 isodate==0.6.0 itsdangerous==0.24 -jinja2==2.9.6 +jinja2==2.10 jmespath==0.9.3 jsonschema==2.6.0 kombu==4.1.0 linecache2==1.0.0 logilab-common==1.4.1 -lxml==4.1.0 +lxml==4.1.1 m2crypto==0.27.0 mako==1.0.7 manifestparser==1.1 -markdown==2.6.9 +markdown==2.6.11 markupsafe==1.0 -matplotlib==2.1.0 +matplotlib==2.1.2 mccabe==0.6.1 meld3==1.0.2 mock==2.0.0 mozcrash==1.0 -mozdevice==0.50 +mozdevice==0.51 mozfile==1.2 mozinfo==0.10 -mozlog==3.5 +mozlog==3.7 moznetwork==0.27 -mozprocess==0.25 -mozprofile==0.28 -mozrunner==6.13 -msgpack-python==0.4.8 +mozprocess==0.26 +mozprofile==0.29 +mozrunner==6.14 +msgpack-python==0.5.2 mysql-python==1.2.5 -ndg-httpsclient==0.4.3 +ndg-httpsclient==0.4.4 netaddr==0.7.19 netifaces==0.10.6 -newrelic==2.96.0.80 +newrelic==2.100.0.84 nose==1.3.7 -numpy==1.13.3 +numpy==1.14.0 oauth2==1.9.0.post1 oauth2client==4.1.2 oauthlib==2.0.6 ordereddict==1.1 -oslo.config==5.0.0 -pandas==0.21.0 -paramiko==2.3.1 +oslo.config==5.2.0 +pandas==0.22.0 +paramiko==2.4.0 passlib==1.7.1 paste==2.0.3 pastedeploy==1.5.2 pastescript==2.0.2 pbr==3.1.1 pep8==1.7.1 -pexpect==4.2.1 -pika==0.11.0 -pillow==4.3.0 +pexpect==4.3.1 +pika==0.11.2 +pillow==5.0.0 pip==9.0.1 prettytable -protobuf==3.4.0 -psutil==5.4.0 +protobuf==3.5.1 +psutil==5.4.3 psycopg2==2.7.3.2 -py==1.4.34 -pyasn1-modules==0.1.5 -pyasn1==0.3.7 +py==1.5.2 +pyasn1-modules==0.2.1 +pyasn1==0.4.2 pycparser==2.18 pycrypto==2.6.1 -pycurl==7.43.0 +pycurl==7.43.0.1 pyflakes==1.6.0 pygments==2.2.0 pyjwt==1.5.3 pylibmc==1.5.2 -pylint==1.7.4 -pymongo==3.5.1 -pymysql==0.7.11 -pyopenssl==17.3.0 +pylint==1.8.2 +pymongo==3.6.0 +pymysql==0.8.0 +pyopenssl==17.5.0 pyparsing==2.2.0 pyramid==1.9.1 pystache==0.5.4 pytest-cov==2.5.1 -pytest==3.2.3 +pytest==3.4.0 python-cjson==1.2.1 python-daemon==2.1.2 python-dateutil==2.6.1 python-gflags==3.1.2 -python-keystoneclient==3.13.0 -python-memcached==1.58 +python-keystoneclient==3.15.0 +python-memcached==1.59 python-mimeparse==1.6.0 -python-novaclient==9.1.0 +python-novaclient==10.1.0 python-subunit==1.2.0 -python-swiftclient==3.4.0 +python-swiftclient==3.5.0 pytz==2017.3 pyyaml==3.12 pyzmq==16.0.3 -raven==6.3.0 +raven==6.5.0 redis==2.10.6 repoze.lru==0.7 requests-oauthlib==0.8.0 @@ -165,30 +165,30 @@ requests==2.18.4 retrying==1.3.3 rsa==3.4.2 scipy==1.0.0 -selenium==3.7.0 +selenium==3.8.1 setuptools-git==1.2 -setuptools==36.6.0 +setuptools==38.5.0 sh==1.12.14 -simplejson==3.11.1 +simplejson==3.13.2 six==1.11.0 snowballstemmer==1.2.1 south==1.0.2 -sphinx==1.6.5 +sphinx==1.6.7 sqlalchemy-migrate==0.11.0 -sqlalchemy==1.1.14 +sqlalchemy==1.2.2 sqlparse==0.2.4 -statsd==3.2.1 -stevedore==1.27.1 +statsd==3.2.2 +stevedore==1.28.0 suds==0.4 supervisor==3.3.3 testrepository==0.0.20 testtools==2.3.0 -thrift==0.10.0 -tornado==4.5.2 +thrift==0.11.0 +tornado==4.5.3 tox==2.9.1 twisted==17.9.0 ujson==1.35 -unidecode==0.4.21 +unidecode==1.0.22 unittest2==1.1.0 uritemplate==3.0.0 urllib3==1.22 @@ -197,11 +197,11 @@ versiontools==1.9.1 virtualenv==15.1.0 waitress==1.1.0 warlock==1.3.0 -webob==1.7.3 -websocket-client==0.44.0 +webob==1.7.4 +websocket-client==0.46.0 webtest==2.0.29 -werkzeug==0.12.2 +werkzeug==0.14.1 wheel==0.30.0 xlrd==1.1.0 -zc.buildout==2.9.5 +zc.buildout==2.11.0 zope.interface==4.4.3 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 2bdcc565..a8664b03 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -1,154 +1,153 @@ -alembic==0.9.6 +alembic==0.9.7 amqp==2.2.2 amqplib==1.0.2 -ansible==2.4.1.0 +ansible==2.4.3.0 anyjson==0.3.3 apache-libcloud==2.2.1 argparse==1.4.0 -astroid==1.5.3 -awscli==1.11.180 -babel==2.5.1 +astroid==1.6.1 +awscli==1.14.32 +babel==2.5.3 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 beautifulsoup4==4.6.0 billiard==3.5.0.3 -blessings==1.6 +blessings==1.6.1 blinker==1.4 boto==2.48.0 -botocore==1.7.38 +botocore==1.8.36 bottle==0.12.13 celery==4.1.0 -certifi==2017.7.27.1 -cffi==1.11.2 +certifi==2018.1.18 +cffi==1.11.4 chardet==3.0.4 click==6.7 -cliff==2.9.1 -cmd2==0.7.7 +cliff==2.11.0 +cmd2==0.8.0 colorama==0.3.9 configobj==5.0.6 cov-core==1.15.0 -coverage==4.4.1 +coverage==4.5 coveralls==1.2.0 crcmod==1.7 -cryptography==2.1.3 -cssselect==1.0.1 -cython==0.27.2 -decorator==4.1.2 -django-celery==3.2.1 -django-debug-toolbar==1.8 -django-extensions==1.9.7 -django==1.11.7 +cryptography==2.1.4 +cssselect==1.0.3 +cython==0.27.3 +decorator==4.2.1 +django-celery==3.2.2 +django-debug-toolbar==1.9.1 +django-extensions==1.9.9 +django==2.0.2 django_compress==1.0.1 -djangorestframework==3.7.1 +djangorestframework==3.7.7 docker-py==1.10.6 docopt==0.6.2 docutils==0.14 ecdsa==0.13 -elasticsearch==5.4.0 +elasticsearch==6.1.1 enum34==1.1.6 -eventlet==0.21.0 +eventlet==0.22.0 extras==1.0.0 fabric==1.14.0 fixtures==3.0.0 flake8==3.5.0 flask==0.12.2 funcsigs==1.0.2 -futures==3.1.1 gevent==1.2.2 -google-api-python-client==1.6.4 -greenlet==0.4.12 +google-api-python-client==1.6.5 +greenlet==0.4.13 gunicorn==19.7.1 hiredis==0.2.0 honcho==1.0.1 -html5lib==0.999999999 +html5lib==1.0.1 httplib2==0.10.3 idna==2.6 -ipaddress==1.0.18 +ipaddress==1.0.19 ipython==6.2.1 iso8601==0.1.12 isodate==0.6.0 itsdangerous==0.24 -jinja2==2.9.6 +jinja2==2.10 jmespath==0.9.3 jsonschema==2.6.0 kombu==4.1.0 linecache2==1.0.0 logilab-common==1.4.1 -lxml==4.1.0 +lxml==4.1.1 mako==1.0.7 manifestparser==1.1 -markdown==2.6.9 +markdown==2.6.11 markupsafe==1.0 -matplotlib==2.1.0 +matplotlib==2.1.2 mccabe==0.6.1 meld3==1.0.2 mock==2.0.0 mozcrash==1.0 -mozdevice==0.50 +mozdevice==0.51 mozfile==1.2 mozinfo==0.10 -mozlog==3.5 +mozlog==3.7 moznetwork==0.27 -mozprocess==0.25 -msgpack-python==0.4.8 -ndg-httpsclient==0.4.3 +mozprocess==0.26 +msgpack-python==0.5.2 +ndg-httpsclient==0.4.4 netaddr==0.7.19 netifaces==0.10.6 -newrelic==2.96.0.80 +newrelic==2.100.0.84 nose==1.3.7 -numpy==1.13.3 +numpy==1.14.0 oauth2==1.9.0.post1 oauth2client==4.1.2 oauthlib==2.0.6 ordereddict==1.1 -oslo.config==5.0.0 -pandas==0.21.0 -paramiko==2.3.1 +oslo.config==5.2.0 +pandas==0.22.0 +paramiko==2.4.0 passlib==1.7.1 paste==2.0.3 pastedeploy==1.5.2 pastescript==2.0.2 pbr==3.1.1 pep8==1.7.1 -pexpect==4.2.1 -pika==0.11.0 -pillow==4.3.0 +pexpect==4.3.1 +pika==0.11.2 +pillow==5.0.0 pip==9.0.1 prettytable -protobuf==3.4.0 -psutil==5.4.0 +protobuf==3.5.1 +psutil==5.4.3 psycopg2==2.7.3.2 -py==1.4.34 -pyasn1-modules==0.1.5 -pyasn1==0.3.7 +py==1.5.2 +pyasn1-modules==0.2.1 +pyasn1==0.4.2 pycparser==2.18 pycrypto==2.6.1 pyflakes==1.6.0 pygments==2.2.0 pyjwt==1.5.3 pylibmc==1.5.2 -pylint==1.7.4 -pymongo==3.5.1 -pymysql==0.7.11 -pyopenssl==17.3.0 +pylint==1.8.2 +pymongo==3.6.0 +pymysql==0.8.0 +pyopenssl==17.5.0 pyparsing==2.2.0 pyramid==1.9.1 pystache==0.5.4 pytest-cov==2.5.1 -pytest==3.2.3 +pytest==3.4.0 python-daemon==2.1.2 python-dateutil==2.6.1 python-gflags==3.1.2 -python-keystoneclient==3.13.0 -python-memcached==1.58 +python-keystoneclient==3.15.0 +python-memcached==1.59 python-mimeparse==1.6.0 -python-novaclient==9.1.0 +python-novaclient==10.1.0 python-subunit==1.2.0 -python-swiftclient==3.4.0 +python-swiftclient==3.5.0 pytz==2017.3 pyyaml==3.12 pyzmq==16.0.3 -raven==6.3.0 +raven==6.5.0 redis==2.10.6 repoze.lru==0.7 requests-oauthlib==0.8.0 @@ -156,28 +155,28 @@ requests==2.18.4 retrying==1.3.3 rsa==3.4.2 scipy==1.0.0 -selenium==3.7.0 +selenium==3.8.1 setuptools-git==1.2 -setuptools==36.6.0 +setuptools==38.5.0 sh==1.12.14 -simplejson==3.11.1 +simplejson==3.13.2 six==1.11.0 snowballstemmer==1.2.1 south==1.0.2 -sphinx==1.6.5 +sphinx==1.6.7 sqlalchemy-migrate==0.11.0 -sqlalchemy==1.1.14 +sqlalchemy==1.2.2 sqlparse==0.2.4 -statsd==3.2.1 -stevedore==1.27.1 +statsd==3.2.2 +stevedore==1.28.0 testrepository==0.0.20 testtools==2.3.0 -thrift==0.10.0 -tornado==4.5.2 +thrift==0.11.0 +tornado==4.5.3 tox==2.9.1 twisted==17.9.0 ujson==1.35 -unidecode==0.4.21 +unidecode==1.0.22 unittest2==1.1.0 uritemplate==3.0.0 urllib3==1.22 @@ -186,11 +185,11 @@ versiontools==1.9.1 virtualenv==15.1.0 waitress==1.1.0 warlock==1.3.0 -webob==1.7.3 -websocket-client==0.44.0 +webob==1.7.4 +websocket-client==0.46.0 webtest==2.0.29 -werkzeug==0.12.2 +werkzeug==0.14.1 wheel==0.30.0 xlrd==1.1.0 -zc.buildout==2.9.5 +zc.buildout==2.11.0 zope.interface==4.4.3 From 1a69141db0b08cc73eccc40a2d3ad03d71a09e73 Mon Sep 17 00:00:00 2001 From: Douglas Greiman Date: Mon, 12 Feb 2018 14:20:24 -0800 Subject: [PATCH 260/362] Update Python interpreters to 3.5.5 and 3.6.4 (#177) --- python-interpreter-builder/Dockerfile.in | 4 ++-- .../scripts/build-python-3.5.sh | 12 ++++++------ .../scripts/build-python-3.6.sh | 12 ++++++------ tests/virtualenv/virtualenv_python35.yaml | 2 +- tests/virtualenv/virtualenv_python36.yaml | 2 +- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/python-interpreter-builder/Dockerfile.in b/python-interpreter-builder/Dockerfile.in index 90e606eb..3c5fc31c 100644 --- a/python-interpreter-builder/Dockerfile.in +++ b/python-interpreter-builder/Dockerfile.in @@ -51,10 +51,10 @@ RUN mkdir -p /opt/packages && \ echo -n "" > /opt/packages/packages.txt RUN /scripts/build-python-3.5.sh && \ - /scripts/package-python.sh 3.5.4 "1gcp~${TAG}" + /scripts/package-python.sh 3.5.5 "1gcp~${TAG}" RUN /scripts/build-python-3.6.sh && \ - /scripts/package-python.sh 3.6.2 "1gcp~${TAG}" + /scripts/package-python.sh 3.6.4 "1gcp~${TAG}" # Tar the interpreters. Tarring is needed because docker cp doesn't handle # links correctly. diff --git a/python-interpreter-builder/scripts/build-python-3.5.sh b/python-interpreter-builder/scripts/build-python-3.5.sh index eeea1f76..c2a3337e 100755 --- a/python-interpreter-builder/scripts/build-python-3.5.sh +++ b/python-interpreter-builder/scripts/build-python-3.5.sh @@ -6,14 +6,14 @@ set -x # Get the source mkdir -p /opt/sources cd /opt/sources -wget --no-verbose https://www.python.org/ftp/python/3.5.4/Python-3.5.4.tgz +wget --no-verbose https://www.python.org/ftp/python/3.5.5/Python-3.5.5.tgz # SHA-256 generated via `shasum -a 256 [file]` shasum --check < Date: Mon, 26 Feb 2018 13:08:16 -0800 Subject: [PATCH 261/362] Test cleanups and fixes (#179) * Separate Google Cloud Client Library tests from other tests. Use the --client_test flag to build.sh to run them. * Fix test failure with google-cloud-dlp. See https://github.com/GoogleCloudPlatform/google-cloud-python/issues/4924 Also add better logging of exactly which files experience failures. * Remove system test which is redundant with the integration test. * Fix punctuation. --- build.sh | 46 ++++++------------- cloudbuild_client_test.yaml | 9 ++++ cloudbuild_system_test.yaml | 9 ---- cloudbuild_test.yaml | 6 --- tests/google-cloud-python-system/.gitignore | 2 - .../google-cloud-python-system/Dockerfile.in | 16 ------- .../run_system_tests.sh | 39 ---------------- tests/google-cloud-python/run_unit_tests.sh | 18 +++++++- 8 files changed, 41 insertions(+), 104 deletions(-) create mode 100644 cloudbuild_client_test.yaml delete mode 100644 cloudbuild_system_test.yaml delete mode 100644 tests/google-cloud-python-system/.gitignore delete mode 100644 tests/google-cloud-python-system/Dockerfile.in delete mode 100755 tests/google-cloud-python-system/run_system_tests.sh diff --git a/build.sh b/build.sh index 7e2370d4..62269d36 100755 --- a/build.sh +++ b/build.sh @@ -19,7 +19,7 @@ set -euo pipefail # Actions benchmark=0 # Should run benchmarks? build=0 # Should build images? -system_test=0 # Should run system tests? +client_test=0 # Should run Google Cloud Client Library tests test=0 # Should run standard test suite? local=0 # Should run using local Docker daemon instead of GCR? @@ -42,8 +42,8 @@ Options: --[no]benchmark: Run benchmarking suite (default false) --[no]build: Build all images (default true if no options set) --[no]test: Run basic tests (default true if no options set) + --[no]client_test: Run Google Cloud Client Library tests (default false) --[no]local: Build images using local Docker daemon (default false) - --[no]system_test: Run system tests (default false) " } @@ -90,20 +90,20 @@ while [ $# -gt 0 ]; do build=0 shift ;; - --local) - local=1 + --client_test) + client_test=1 shift ;; - --nolocal) - local=0 + --noclient_test) + client_test=0 shift ;; - --system_test) - system_test=1 + --local) + local=1 shift ;; - --nosystem_test) - system_test=0 + --nolocal) + local=0 shift ;; --test) @@ -123,7 +123,7 @@ done # If no actions chosen, then tell the user if [ "${benchmark}" -eq 0 -a \ "${build}" -eq 0 -a \ - "${system_test}" -eq 0 -a \ + "${client_test}" -eq 0 -a \ "${test}" -eq 0 \ ]; then echo 'No actions specified, defaulting to --build --test' @@ -136,17 +136,6 @@ if [ "${local}" -eq 1 ]; then gcloud_cmd="${local_gcloud_cmd}" fi -# Read action-specific environment variables -if [ "${system_test}" -eq 1 ]; then - if [ -z "${GOOGLE_APPLICATION_CREDENTIALS_FOR_TESTS+set}" ] ; then - fatal 'Error: $GOOGLE_APPLICATION_CREDENTIALS_FOR_TESTS is not set; invoke with something like GOOGLE_APPLICATION_CREDENTIALS_FOR_TESTS=/path/to/service/account/creds.json' - fi - - if [ -z "${GOOGLE_CLOUD_PROJECT_FOR_TESTS+set}" ] ; then - fatal 'Error: $GOOGLE_CLOUD_PROJECT_FOR_TESTS is not set; invoke with something like GOOGLE_CLOUD_PROJECT_FOR_TESTS=YOUR-PROJECT-NAME' - fi -fi - # Use latest released Debian as our base image export DEBIAN_BASE_IMAGE="gcr.io/google-appengine/debian8:latest" export STAGING_IMAGE="${DOCKER_NAMESPACE}/python:${TAG}" @@ -160,7 +149,6 @@ for outfile in \ tests/benchmark/Dockerfile \ tests/eventlet/Dockerfile \ tests/google-cloud-python/Dockerfile \ - tests/google-cloud-python-system/Dockerfile \ tests/integration/Dockerfile \ ; do envsubst <"${outfile}".in >"${outfile}" \ @@ -192,14 +180,10 @@ if [ "${test}" -eq 1 ]; then ${gcloud_cmd} --config cloudbuild_test.yaml --substitutions "${substitutions}" fi -# Run system tests -if [ "${system_test}" -eq 1 ]; then - echo "Running system tests using project ${GOOGLE_CLOUD_PROJECT_FOR_TESTS}" - - trap "rm -f tests/google-cloud-python-system/credentials.json" EXIT - cp "${GOOGLE_APPLICATION_CREDENTIALS_FOR_TESTS}" tests/google-cloud-python-system/credentials.json - ${gcloud_cmd} --config cloudbuild_system_test.yaml --substitutions "${substitutions}" - rm -f tests/google-cloud-python-system/credentials.json +# Run client library tests +if [ "${client_test}" -eq 1 ]; then + echo "Testing compatibility with Google Cloud Client Libraries" + ${gcloud_cmd} --config cloudbuild_client_test.yaml --substitutions "${substitutions}" fi # Run benchmarks diff --git a/cloudbuild_client_test.yaml b/cloudbuild_client_test.yaml new file mode 100644 index 00000000..010e1ffd --- /dev/null +++ b/cloudbuild_client_test.yaml @@ -0,0 +1,9 @@ +timeout: 3600s +steps: +- # Build image to run google client library unit tests + name: gcr.io/cloud-builders/docker:latest + args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/google-cloud-python:${_TAG}', + '--no-cache', '/workspace/tests/google-cloud-python/'] +- # Run google client library unit tests + name: ${_DOCKER_NAMESPACE}/python/tests/google-cloud-python:${_TAG} +images: [] diff --git a/cloudbuild_system_test.yaml b/cloudbuild_system_test.yaml deleted file mode 100644 index 3cac03e2..00000000 --- a/cloudbuild_system_test.yaml +++ /dev/null @@ -1,9 +0,0 @@ -timeout: 3600s -steps: -- name: gcr.io/cloud-builders/docker:latest - args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/google-cloud-python-system:${_TAG}', - '--no-cache', '/workspace/tests/google-cloud-python-system/'] -- name: ${_DOCKER_NAMESPACE}/python/tests/google-cloud-python-system:${_TAG} -images: [ - # Intentionally empty -] diff --git a/cloudbuild_test.yaml b/cloudbuild_test.yaml index acf70bc7..6efe5a45 100644 --- a/cloudbuild_test.yaml +++ b/cloudbuild_test.yaml @@ -37,10 +37,4 @@ steps: name: gcr.io/cloud-builders/docker:latest args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/eventlet:${_TAG}', '--no-cache', '/workspace/tests/eventlet/'] -- # Build image to run google client library unit tests - name: gcr.io/cloud-builders/docker:latest - args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/google-cloud-python:${_TAG}', - '--no-cache', '/workspace/tests/google-cloud-python/'] -- # Run google client library unit tests - name: ${_DOCKER_NAMESPACE}/python/tests/google-cloud-python:${_TAG} images: [] diff --git a/tests/google-cloud-python-system/.gitignore b/tests/google-cloud-python-system/.gitignore deleted file mode 100644 index 470715ec..00000000 --- a/tests/google-cloud-python-system/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -credentials.json -Dockerfile diff --git a/tests/google-cloud-python-system/Dockerfile.in b/tests/google-cloud-python-system/Dockerfile.in deleted file mode 100644 index 04f28bd5..00000000 --- a/tests/google-cloud-python-system/Dockerfile.in +++ /dev/null @@ -1,16 +0,0 @@ -FROM ${STAGING_IMAGE} - -# Get the source. -RUN git clone --depth 1 https://github.com/GoogleCloudPlatform/google-cloud-python.git -WORKDIR google-cloud-python - -# Install nox -RUN pip install --upgrade nox-automation - -# Secrets injected at runtime -ENV GOOGLE_APPLICATION_CREDENTIALS=/workspace/tests/google-cloud-python-system/credentials.json -ENV GOOGLE_CLOUD_PROJECT=${GOOGLE_CLOUD_PROJECT_FOR_TESTS} - -# Run system tests for all supported Python versions -ADD run_system_tests.sh /run_system_tests.sh -ENTRYPOINT ["/run_system_tests.sh"] diff --git a/tests/google-cloud-python-system/run_system_tests.sh b/tests/google-cloud-python-system/run_system_tests.sh deleted file mode 100755 index f9f81a47..00000000 --- a/tests/google-cloud-python-system/run_system_tests.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/sh -set -eu - -cd /app/google-cloud-python - -# Not all packages have system tests -packages=" -bigquery -bigtable -datastore -language -logging -monitoring -pubsub -spanner -speech -storage -vision -" - -# translate has system test but it gives error message: -# BadRequest: 400 Invalid JSON payload received. Unknown name "model": Cannot bind 'nmt'. Field 'model' could not be found in request message. (GET https://translation.googleapis.com/language/translate/v2?target=de&q=hvala+ti&q=dankon&q=Me+llamo+Jeff&q=My+name+is+Jeff&model=nmt) -disabled_packages="translate" - -# Spanner system test needs this -export GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE=1 - -exit_code=0 -for package in ${packages}; do - noxfile="${package}/nox.py" - nox \ - -f "${noxfile}" \ - -e \ - "system(py='2.7')" \ - "system(py='3.6')" \ - || exit_code=1 -done - -exit "${exit_code}" diff --git a/tests/google-cloud-python/run_unit_tests.sh b/tests/google-cloud-python/run_unit_tests.sh index 81a6a389..c386f1c0 100755 --- a/tests/google-cloud-python/run_unit_tests.sh +++ b/tests/google-cloud-python/run_unit_tests.sh @@ -4,7 +4,13 @@ set -eu cd /app/google-cloud-python exit_code=0 +failed_files= for noxfile in */nox.py; do + if [ "${noxfile}" = "dlp/nox.py" ]; then + echo "**** Skipping ${noxfile} ****" + continue + fi + echo "**** Starting tests in ${noxfile} ****" nox \ -f "${noxfile}" \ -e \ @@ -12,7 +18,17 @@ for noxfile in */nox.py; do "unit(py='3.4')" \ "unit(py='3.5')" \ "unit(py='3.6')" \ - || exit_code=1 + || { + echo "**** FAILED tests in ${noxfile} ****" + exit_code=1 + failed_files="${failed_files} ${noxfile}" + } + echo "**** Finished tests in ${noxfile} ****" done +if [ "${exit_code}" -eq 0 ]; then + echo "**** All tests passed ****" +else + echo "**** There were test failures:${failed_files} ****" +fi exit "${exit_code}" From a1a0c2d7efa00dacc0f197fa0a97426f783da8e3 Mon Sep 17 00:00:00 2001 From: Douglas Greiman Date: Mon, 26 Feb 2018 18:50:31 -0800 Subject: [PATCH 262/362] Remove unused file (#181) --- jenkins_build.sh | 3 --- 1 file changed, 3 deletions(-) delete mode 100755 jenkins_build.sh diff --git a/jenkins_build.sh b/jenkins_build.sh deleted file mode 100755 index f88007f5..00000000 --- a/jenkins_build.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -./build.sh "$@" From fe98d8b5b58d8dee486797d55398e38f76bdfd4d Mon Sep 17 00:00:00 2001 From: Douglas Greiman Date: Wed, 28 Feb 2018 11:32:41 -0800 Subject: [PATCH 263/362] Stop building Debian packages. (#180) We never started using these packages, and have a different internal plan for packaging in the future. --- python-interpreter-builder/Dockerfile.in | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/python-interpreter-builder/Dockerfile.in b/python-interpreter-builder/Dockerfile.in index 3c5fc31c..834047b9 100644 --- a/python-interpreter-builder/Dockerfile.in +++ b/python-interpreter-builder/Dockerfile.in @@ -50,11 +50,9 @@ ADD DEBIAN /DEBIAN RUN mkdir -p /opt/packages && \ echo -n "" > /opt/packages/packages.txt -RUN /scripts/build-python-3.5.sh && \ - /scripts/package-python.sh 3.5.5 "1gcp~${TAG}" +RUN /scripts/build-python-3.5.sh -RUN /scripts/build-python-3.6.sh && \ - /scripts/package-python.sh 3.6.4 "1gcp~${TAG}" +RUN /scripts/build-python-3.6.sh # Tar the interpreters. Tarring is needed because docker cp doesn't handle # links correctly. From 58a537c00e55a75cdfd5fe6e637d33829139b9e8 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Mon, 12 Mar 2018 17:53:31 -0700 Subject: [PATCH 264/362] Auto-update dependencies. (#178) --- runtime-image/resources/requirements.txt | 2 +- scripts/requirements-test.txt | 2 +- tests/eventlet/requirements.txt | 2 +- tests/integration/requirements.txt | 6 +- tests/python2-libraries/requirements.txt | 72 ++++++++++++------------ tests/python3-libraries/requirements.txt | 68 +++++++++++----------- 6 files changed, 76 insertions(+), 76 deletions(-) diff --git a/runtime-image/resources/requirements.txt b/runtime-image/resources/requirements.txt index 6a1b2cc3..ef675285 100644 --- a/runtime-image/resources/requirements.txt +++ b/runtime-image/resources/requirements.txt @@ -1,3 +1,3 @@ pip==9.0.1 -setuptools==38.5.0 +setuptools==38.5.2 wheel==0.30.0 diff --git a/scripts/requirements-test.txt b/scripts/requirements-test.txt index 12a6ca4a..1ae0abd8 100644 --- a/scripts/requirements-test.txt +++ b/scripts/requirements-test.txt @@ -1,4 +1,4 @@ flask==0.12.2 -pytest==3.4.0 +pytest==3.4.2 pytest-cov==2.5.1 pyyaml==3.12 diff --git a/tests/eventlet/requirements.txt b/tests/eventlet/requirements.txt index a8f7b0a9..5d104d0a 100644 --- a/tests/eventlet/requirements.txt +++ b/tests/eventlet/requirements.txt @@ -1,6 +1,6 @@ click==6.7 enum-compat==0.0.2 -eventlet==0.22.0 +eventlet==0.22.1 Flask==0.12.2 greenlet==0.4.13 gunicorn==19.7.1 diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index 99a06d3b..64e39e0c 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -1,7 +1,7 @@ Flask==0.12.2 -google-cloud-error-reporting==0.29.0 -google-cloud-logging==1.4.0 -google-cloud-monitoring==0.28.0 +google-cloud-error-reporting==0.29.1 +google-cloud-logging==1.6.0 +google-cloud-monitoring==0.28.1 gunicorn==19.7.1 requests==2.18.4 retrying==1.3.3 diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index f512ca18..f261209e 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -1,12 +1,12 @@ -alembic==0.9.7 +alembic==0.9.8 amqp==2.2.2 amqplib==1.0.2 ansible==2.4.3.0 anyjson==0.3.3 -apache-libcloud==2.2.1 +apache-libcloud==2.3.0 argparse==1.4.0 astroid==1.6.1 -awscli==1.14.32 +awscli==1.14.53 babel==2.5.3 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 @@ -16,21 +16,21 @@ billiard==3.5.0.3 blessings==1.6.1 blinker==1.4 boto==2.48.0 -botocore==1.8.36 +botocore==1.9.6 bottle==0.12.13 carbon<1.1.1 celery==4.1.0 certifi==2018.1.18 -cffi==1.11.4 +cffi==1.11.5 chardet==3.0.4 click==6.7 cliff==2.11.0 -cmd2==0.8.0 +cmd2==0.8.1 colorama==0.3.9 configobj==5.0.6 cov-core==1.15.0 -coverage==4.5 -coveralls==1.2.0 +coverage==4.5.1 +coveralls==1.3.0 crcmod==1.7 cryptography==2.1.4 cssselect==1.0.3 @@ -38,7 +38,7 @@ cython==0.27.3 decorator==4.2.1 django-celery==3.2.2 django-debug-toolbar==1.9.1 -django-extensions==1.9.9 +django-extensions==2.0.5 django<2.0 django_compress==1.0.1 djangorestframework==3.7.7 @@ -48,7 +48,7 @@ docutils==0.14 ecdsa==0.13 elasticsearch==6.1.1 enum34==1.1.6 -eventlet==0.22.0 +eventlet==0.22.1 extras==1.0.0 fabric==1.14.0 fixtures==3.0.0 @@ -59,7 +59,7 @@ functools32==3.2.3.post2 futures==3.2.0 gevent==1.2.2 google-api-python-client==1.6.5 -graphite-web==1.1.1 +graphite-web==1.1.2 greenlet==0.4.13 gunicorn==19.7.1 hiredis==0.2.0 @@ -78,17 +78,17 @@ kombu==4.1.0 linecache2==1.0.0 logilab-common==1.4.1 lxml==4.1.1 -m2crypto==0.27.0 +m2crypto==0.29.0 mako==1.0.7 manifestparser==1.1 markdown==2.6.11 markupsafe==1.0 -matplotlib==2.1.2 +matplotlib==2.2.0 mccabe==0.6.1 meld3==1.0.2 mock==2.0.0 mozcrash==1.0 -mozdevice==0.51 +mozdevice==0.52 mozfile==1.2 mozinfo==0.10 mozlog==3.7 @@ -96,14 +96,14 @@ moznetwork==0.27 mozprocess==0.26 mozprofile==0.29 mozrunner==6.14 -msgpack-python==0.5.2 +msgpack-python==0.5.6 mysql-python==1.2.5 ndg-httpsclient==0.4.4 netaddr==0.7.19 netifaces==0.10.6 -newrelic==2.100.0.84 +newrelic==2.106.1.88 nose==1.3.7 -numpy==1.14.0 +numpy==1.14.1 oauth2==1.9.0.post1 oauth2client==4.1.2 oauthlib==2.0.6 @@ -117,14 +117,14 @@ pastedeploy==1.5.2 pastescript==2.0.2 pbr==3.1.1 pep8==1.7.1 -pexpect==4.3.1 +pexpect==4.4.0 pika==0.11.2 pillow==5.0.0 pip==9.0.1 prettytable -protobuf==3.5.1 +protobuf==3.5.2 psutil==5.4.3 -psycopg2==2.7.3.2 +psycopg2==2.7.4 py==1.5.2 pyasn1-modules==0.2.1 pyasn1==0.4.2 @@ -133,20 +133,20 @@ pycrypto==2.6.1 pycurl==7.43.0.1 pyflakes==1.6.0 pygments==2.2.0 -pyjwt==1.5.3 +pyjwt==1.6.0 pylibmc==1.5.2 pylint==1.8.2 -pymongo==3.6.0 +pymongo==3.6.1 pymysql==0.8.0 pyopenssl==17.5.0 pyparsing==2.2.0 pyramid==1.9.1 pystache==0.5.4 pytest-cov==2.5.1 -pytest==3.4.0 +pytest==3.4.2 python-cjson==1.2.1 python-daemon==2.1.2 -python-dateutil==2.6.1 +python-dateutil==2.7.0 python-gflags==3.1.2 python-keystoneclient==3.15.0 python-memcached==1.59 @@ -154,10 +154,10 @@ python-mimeparse==1.6.0 python-novaclient==10.1.0 python-subunit==1.2.0 python-swiftclient==3.5.0 -pytz==2017.3 +pytz==2018.3 pyyaml==3.12 -pyzmq==16.0.3 -raven==6.5.0 +pyzmq==17.0.0 +raven==6.6.0 redis==2.10.6 repoze.lru==0.7 requests-oauthlib==0.8.0 @@ -165,26 +165,26 @@ requests==2.18.4 retrying==1.3.3 rsa==3.4.2 scipy==1.0.0 -selenium==3.8.1 +selenium==3.10.0 setuptools-git==1.2 -setuptools==38.5.0 +setuptools==38.5.2 sh==1.12.14 simplejson==3.13.2 six==1.11.0 snowballstemmer==1.2.1 south==1.0.2 -sphinx==1.6.7 +sphinx==1.7.1 sqlalchemy-migrate==0.11.0 -sqlalchemy==1.2.2 +sqlalchemy==1.2.5 sqlparse==0.2.4 statsd==3.2.2 stevedore==1.28.0 suds==0.4 -supervisor==3.3.3 +supervisor==3.3.4 testrepository==0.0.20 testtools==2.3.0 thrift==0.11.0 -tornado==4.5.3 +tornado==5.0 tox==2.9.1 twisted==17.9.0 ujson==1.35 @@ -192,16 +192,16 @@ unidecode==1.0.22 unittest2==1.1.0 uritemplate==3.0.0 urllib3==1.22 -uwsgi==2.0.15 +uwsgi==2.0.17 versiontools==1.9.1 virtualenv==15.1.0 waitress==1.1.0 warlock==1.3.0 webob==1.7.4 -websocket-client==0.46.0 +websocket-client==0.47.0 webtest==2.0.29 werkzeug==0.14.1 wheel==0.30.0 xlrd==1.1.0 -zc.buildout==2.11.0 +zc.buildout==2.11.1 zope.interface==4.4.3 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index a8664b03..30fb236e 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -1,12 +1,12 @@ -alembic==0.9.7 +alembic==0.9.8 amqp==2.2.2 amqplib==1.0.2 ansible==2.4.3.0 anyjson==0.3.3 -apache-libcloud==2.2.1 +apache-libcloud==2.3.0 argparse==1.4.0 astroid==1.6.1 -awscli==1.14.32 +awscli==1.14.53 babel==2.5.3 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 @@ -15,20 +15,20 @@ billiard==3.5.0.3 blessings==1.6.1 blinker==1.4 boto==2.48.0 -botocore==1.8.36 +botocore==1.9.6 bottle==0.12.13 celery==4.1.0 certifi==2018.1.18 -cffi==1.11.4 +cffi==1.11.5 chardet==3.0.4 click==6.7 cliff==2.11.0 -cmd2==0.8.0 +cmd2==0.8.1 colorama==0.3.9 configobj==5.0.6 cov-core==1.15.0 -coverage==4.5 -coveralls==1.2.0 +coverage==4.5.1 +coveralls==1.3.0 crcmod==1.7 cryptography==2.1.4 cssselect==1.0.3 @@ -36,8 +36,8 @@ cython==0.27.3 decorator==4.2.1 django-celery==3.2.2 django-debug-toolbar==1.9.1 -django-extensions==1.9.9 -django==2.0.2 +django-extensions==2.0.5 +django==2.0.3 django_compress==1.0.1 djangorestframework==3.7.7 docker-py==1.10.6 @@ -46,7 +46,7 @@ docutils==0.14 ecdsa==0.13 elasticsearch==6.1.1 enum34==1.1.6 -eventlet==0.22.0 +eventlet==0.22.1 extras==1.0.0 fabric==1.14.0 fixtures==3.0.0 @@ -78,24 +78,24 @@ mako==1.0.7 manifestparser==1.1 markdown==2.6.11 markupsafe==1.0 -matplotlib==2.1.2 +matplotlib==2.2.0 mccabe==0.6.1 meld3==1.0.2 mock==2.0.0 mozcrash==1.0 -mozdevice==0.51 +mozdevice==0.52 mozfile==1.2 mozinfo==0.10 mozlog==3.7 moznetwork==0.27 mozprocess==0.26 -msgpack-python==0.5.2 +msgpack-python==0.5.6 ndg-httpsclient==0.4.4 netaddr==0.7.19 netifaces==0.10.6 -newrelic==2.100.0.84 +newrelic==2.106.1.88 nose==1.3.7 -numpy==1.14.0 +numpy==1.14.1 oauth2==1.9.0.post1 oauth2client==4.1.2 oauthlib==2.0.6 @@ -109,14 +109,14 @@ pastedeploy==1.5.2 pastescript==2.0.2 pbr==3.1.1 pep8==1.7.1 -pexpect==4.3.1 +pexpect==4.4.0 pika==0.11.2 pillow==5.0.0 pip==9.0.1 prettytable -protobuf==3.5.1 +protobuf==3.5.2 psutil==5.4.3 -psycopg2==2.7.3.2 +psycopg2==2.7.4 py==1.5.2 pyasn1-modules==0.2.1 pyasn1==0.4.2 @@ -124,19 +124,19 @@ pycparser==2.18 pycrypto==2.6.1 pyflakes==1.6.0 pygments==2.2.0 -pyjwt==1.5.3 +pyjwt==1.6.0 pylibmc==1.5.2 pylint==1.8.2 -pymongo==3.6.0 +pymongo==3.6.1 pymysql==0.8.0 pyopenssl==17.5.0 pyparsing==2.2.0 pyramid==1.9.1 pystache==0.5.4 pytest-cov==2.5.1 -pytest==3.4.0 +pytest==3.4.2 python-daemon==2.1.2 -python-dateutil==2.6.1 +python-dateutil==2.7.0 python-gflags==3.1.2 python-keystoneclient==3.15.0 python-memcached==1.59 @@ -144,10 +144,10 @@ python-mimeparse==1.6.0 python-novaclient==10.1.0 python-subunit==1.2.0 python-swiftclient==3.5.0 -pytz==2017.3 +pytz==2018.3 pyyaml==3.12 -pyzmq==16.0.3 -raven==6.5.0 +pyzmq==17.0.0 +raven==6.6.0 redis==2.10.6 repoze.lru==0.7 requests-oauthlib==0.8.0 @@ -155,24 +155,24 @@ requests==2.18.4 retrying==1.3.3 rsa==3.4.2 scipy==1.0.0 -selenium==3.8.1 +selenium==3.10.0 setuptools-git==1.2 -setuptools==38.5.0 +setuptools==38.5.2 sh==1.12.14 simplejson==3.13.2 six==1.11.0 snowballstemmer==1.2.1 south==1.0.2 -sphinx==1.6.7 +sphinx==1.7.1 sqlalchemy-migrate==0.11.0 -sqlalchemy==1.2.2 +sqlalchemy==1.2.5 sqlparse==0.2.4 statsd==3.2.2 stevedore==1.28.0 testrepository==0.0.20 testtools==2.3.0 thrift==0.11.0 -tornado==4.5.3 +tornado==5.0 tox==2.9.1 twisted==17.9.0 ujson==1.35 @@ -180,16 +180,16 @@ unidecode==1.0.22 unittest2==1.1.0 uritemplate==3.0.0 urllib3==1.22 -uwsgi==2.0.15 +uwsgi==2.0.17 versiontools==1.9.1 virtualenv==15.1.0 waitress==1.1.0 warlock==1.3.0 webob==1.7.4 -websocket-client==0.46.0 +websocket-client==0.47.0 webtest==2.0.29 werkzeug==0.14.1 wheel==0.30.0 xlrd==1.1.0 -zc.buildout==2.11.0 +zc.buildout==2.11.1 zope.interface==4.4.3 From 8106a265e0ad5f44d9db43bbf44d742282a96a74 Mon Sep 17 00:00:00 2001 From: Douglas Greiman Date: Tue, 13 Mar 2018 14:05:07 -0700 Subject: [PATCH 265/362] Add support for using Ubuntu 16 as a base instead of Debian 8 (#182) * Add --os_base flag to build against different OS base images. Also, change --local to use the official gcloud tool instead of our own. * Update from Python 3.4.2 to Python 3.4.8 Contains various bugfixes and security patches. We switch from using the Debian-provided "python3.4" packages to building it from source, in preparation for moving to newer OS base images that don't have a prebuilt Python 3.4 interpreter available. * Update comments about configure flags * Temporarily disable license check * Add some necessary packages for Ubuntu * Ubuntu ships with Python 2.7.12 * Remove errant whitespace * Rename DEBIAN_BASE_IMAGE to OS_BASE_IMAGE --- build.sh | 46 ++++-- cloudbuild_test.yaml | 70 +++++---- python-interpreter-builder/Dockerfile.in | 5 +- .../scripts/build-python-3.4.sh | 138 ++++++++++++++++++ runtime-image/Dockerfile.in | 6 +- runtime-image/resources/apt-packages.txt | 3 - tests/license-test/license-test.yaml | 2 +- tests/no-virtualenv/no-virtualenv.yaml | 2 +- tests/virtualenv/virtualenv_default.yaml | 2 +- tests/virtualenv/virtualenv_python27.yaml | 2 +- tests/virtualenv/virtualenv_python34.yaml | 2 +- 11 files changed, 232 insertions(+), 46 deletions(-) create mode 100755 python-interpreter-builder/scripts/build-python-3.4.sh diff --git a/build.sh b/build.sh index 62269d36..9b3a258d 100755 --- a/build.sh +++ b/build.sh @@ -24,9 +24,12 @@ test=0 # Should run standard test suite? local=0 # Should run using local Docker daemon instead of GCR? +os_base=debian8 # Which operating system base to use + # Note that $gcloud_cmd has spaces in it -gcloud_cmd="gcloud beta container builds submit ." -local_gcloud_cmd="scripts/local_cloudbuild.py" +gcloud_cmd="gcloud container builds submit" +# May need to install via "gcloud components install container-builder-local" +local_gcloud_cmd="container-builder-local --push=false --dryrun=false" # Helper functions function fatal() { @@ -44,6 +47,7 @@ Options: --[no]test: Run basic tests (default true if no options set) --[no]client_test: Run Google Cloud Client Library tests (default false) --[no]local: Build images using local Docker daemon (default false) + --os_base: Which OS image to build on top of [debian8, ubuntu16] " } @@ -106,6 +110,14 @@ while [ $# -gt 0 ]; do local=0 shift ;; + --os_base=debian8) + os_base=debian8 + shift + ;; + --os_base=ubuntu16) + os_base=ubuntu16 + shift + ;; --test) test=1 shift @@ -136,8 +148,12 @@ if [ "${local}" -eq 1 ]; then gcloud_cmd="${local_gcloud_cmd}" fi -# Use latest released Debian as our base image -export DEBIAN_BASE_IMAGE="gcr.io/google-appengine/debian8:latest" +# Pick OS image to use as base +if [ "${os_base}" == "ubuntu16" ]; then + export OS_BASE_IMAGE="gcr.io/gcp-runtimes/ubuntu_16_0_4:latest" +else + export OS_BASE_IMAGE="gcr.io/google-appengine/debian8:latest" +fi export STAGING_IMAGE="${DOCKER_NAMESPACE}/python:${TAG}" echo "Using base image name ${STAGING_IMAGE}" @@ -152,7 +168,7 @@ for outfile in \ tests/integration/Dockerfile \ ; do envsubst <"${outfile}".in >"${outfile}" \ - '$DEBIAN_BASE_IMAGE $STAGING_IMAGE $GOOGLE_CLOUD_PROJECT_FOR_TESTS $TAG' + '$OS_BASE_IMAGE $STAGING_IMAGE $GOOGLE_CLOUD_PROJECT_FOR_TESTS $TAG' done # Make some files available to the runtime builder Docker context @@ -171,23 +187,35 @@ cp -a scripts/testdata/hello_world/main.py tests/eventlet/main.py # Build images and push to GCR if [ "${build}" -eq 1 ]; then echo "Building images" - ${gcloud_cmd} --config cloudbuild.yaml --substitutions "${build_substitutions}" + ${gcloud_cmd} \ + --config=cloudbuild.yaml \ + --substitutions="${build_substitutions}" \ + . fi # Run the tests that don't require (too many) external services if [ "${test}" -eq 1 ]; then echo "Testing compatibility with popular Python libraries" - ${gcloud_cmd} --config cloudbuild_test.yaml --substitutions "${substitutions}" + ${gcloud_cmd} \ + --config=cloudbuild_test.yaml \ + --substitutions="${substitutions}" \ + . fi # Run client library tests if [ "${client_test}" -eq 1 ]; then echo "Testing compatibility with Google Cloud Client Libraries" - ${gcloud_cmd} --config cloudbuild_client_test.yaml --substitutions "${substitutions}" + ${gcloud_cmd} \ + --config=cloudbuild_client_test.yaml \ + --substitutions="${substitutions}" \ + . fi # Run benchmarks if [ "${benchmark}" -eq 1 ] ; then echo "Running benchmark" - ${gcloud_cmd} --config cloudbuild_benchmark.yaml --substitutions "${substitutions}" + ${gcloud_cmd} \ + --config=cloudbuild_benchmark.yaml \ + --substitutions="${substitutions}" \ + . fi diff --git a/cloudbuild_test.yaml b/cloudbuild_test.yaml index 6efe5a45..ef0b48b3 100644 --- a/cloudbuild_test.yaml +++ b/cloudbuild_test.yaml @@ -1,27 +1,12 @@ timeout: 3600s steps: - # Validate structure of base runtime image -- name: gcr.io/cloud-builders/docker:latest - args: ['build', '-t', 'python2-libraries-intermediate', '--build-arg', - 'intermediate_image=${_DOCKER_NAMESPACE}/python:${_TAG}', - '/workspace/tests/python2-libraries'] -- name: gcr.io/gcp-runtimes/container-structure-test:v0.1.1 +- # Explicitly pull image into GCB so that later steps work + name: '${_DOCKER_NAMESPACE}/python:${_TAG}' args: [ - '-test.v', - '-image', 'python2-libraries-intermediate', - '/workspace/tests/python2-libraries/python2-libraries.yaml' - ] -- name: gcr.io/cloud-builders/docker:latest - args: ['build', '-t', 'python3-libraries-intermediate', '--build-arg', - 'intermediate_image=${_DOCKER_NAMESPACE}/python:${_TAG}', - '/workspace/tests/python3-libraries'] -- name: gcr.io/gcp-runtimes/container-structure-test:v0.1.1 - args: [ - '-test.v', - '-image', 'python3-libraries-intermediate', - '/workspace/tests/python3-libraries/python3-libraries.yaml' - ] -- name: gcr.io/gcp-runtimes/container-structure-test:v0.1.1 + '/bin/true', + ] +- # Validate structure of base runtime image + name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 args: [ '-test.v', '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', @@ -31,10 +16,45 @@ steps: '/workspace/tests/virtualenv/virtualenv_python35.yaml', '/workspace/tests/virtualenv/virtualenv_python36.yaml', '/workspace/tests/no-virtualenv/no-virtualenv.yaml', - '/workspace/tests/license-test/license-test.yaml' ] -- # Run compatibility tests +# Temporarily disabled because it fails on symbolic links in Ubuntu: +# https://github.com/GoogleCloudPlatform/container-structure-test/issues/77 +#- # Check license compliance +# name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 +# args: [ +# '-test.v', +# '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', +# '/workspace/tests/license-test/license-test.yaml' +# ] +- # Do third-party library compatibility tests + name: gcr.io/cloud-builders/docker:latest + args: [ + 'build', '-t', 'python2-libraries-intermediate', '--build-arg', + 'intermediate_image=${_DOCKER_NAMESPACE}/python:${_TAG}', + '/workspace/tests/python2-libraries' + ] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 + args: [ + '-test.v', + '-image', 'python2-libraries-intermediate', + '/workspace/tests/python2-libraries/python2-libraries.yaml' + ] +- name: gcr.io/cloud-builders/docker:latest + args: [ + 'build', '-t', 'python3-libraries-intermediate', '--build-arg', + 'intermediate_image=${_DOCKER_NAMESPACE}/python:${_TAG}', + '/workspace/tests/python3-libraries' + ] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 + args: [ + '-test.v', + '-image', 'python3-libraries-intermediate', + '/workspace/tests/python3-libraries/python3-libraries.yaml' + ] +- # Run other compatibility tests name: gcr.io/cloud-builders/docker:latest - args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/eventlet:${_TAG}', - '--no-cache', '/workspace/tests/eventlet/'] + args: [ + 'build', '--tag=${_DOCKER_NAMESPACE}/python/tests/eventlet:${_TAG}', + '--no-cache', '/workspace/tests/eventlet/' + ] images: [] diff --git a/python-interpreter-builder/Dockerfile.in b/python-interpreter-builder/Dockerfile.in index 834047b9..46bc092a 100644 --- a/python-interpreter-builder/Dockerfile.in +++ b/python-interpreter-builder/Dockerfile.in @@ -1,6 +1,6 @@ # The Google App Engine base image is debian (jessie) with ca-certificates # installed. -FROM ${DEBIAN_BASE_IMAGE} +FROM ${OS_BASE_IMAGE} # Install Python build dependencies (based on Debian Build-Depends) RUN apt-get update && apt-get install -yq \ @@ -29,6 +29,7 @@ RUN apt-get update && apt-get install -yq \ mime-support \ net-tools \ netbase \ + python \ python3 \ sharutils \ time \ @@ -50,6 +51,8 @@ ADD DEBIAN /DEBIAN RUN mkdir -p /opt/packages && \ echo -n "" > /opt/packages/packages.txt +RUN /scripts/build-python-3.4.sh + RUN /scripts/build-python-3.5.sh RUN /scripts/build-python-3.6.sh diff --git a/python-interpreter-builder/scripts/build-python-3.4.sh b/python-interpreter-builder/scripts/build-python-3.4.sh new file mode 100755 index 00000000..95089f75 --- /dev/null +++ b/python-interpreter-builder/scripts/build-python-3.4.sh @@ -0,0 +1,138 @@ +#!/bin/bash + +set -euo pipefail +set -x + +# Get the source +mkdir -p /opt/sources +cd /opt/sources +wget --no-verbose https://www.python.org/ftp/python/3.4.8/Python-3.4.8.tgz +# SHA-256 generated via `shasum -a 256 [file]` +shasum --check < Date: Tue, 20 Mar 2018 16:55:49 -0700 Subject: [PATCH 266/362] Increase cloudbuild timeout now that we're building Python 3.4 from source. (#183) --- cloudbuild.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 1eabb997..5554015f 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -1,4 +1,4 @@ -timeout: 7200s +timeout: 10800s steps: - # Compile Python interpreters from source name: gcr.io/cloud-builders/docker:latest From f5364134cd9115ebf47f835e81140107f67c0465 Mon Sep 17 00:00:00 2001 From: Douglas Greiman Date: Thu, 22 Mar 2018 15:02:13 -0700 Subject: [PATCH 267/362] Remove jonwayne, add dlorenc to CODEOWNERS (#185) --- CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CODEOWNERS b/CODEOWNERS index a2d4a67f..c08c2d11 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1,4 +1,4 @@ # Code owners file. # This file controls who is tagged for review for any given pull request. -* @duggelz @liyanhui1228 @jonparrott +* @duggelz @liyanhui1228 @dlorenc From dd6fe8f470466cb52490d6f011e9f232744b7975 Mon Sep 17 00:00:00 2001 From: Douglas Greiman Date: Thu, 22 Mar 2018 15:02:30 -0700 Subject: [PATCH 268/362] Add Runtime Builder template file for "staging" tag. (#184) --- builder/python-staging.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 builder/python-staging.yaml diff --git a/builder/python-staging.yaml b/builder/python-staging.yaml new file mode 100644 index 00000000..1e977a89 --- /dev/null +++ b/builder/python-staging.yaml @@ -0,0 +1,12 @@ +# This is a cloudbuild.yaml template for the runtime builder +steps: +- # Generate application Dockerfile + name: 'gcr.io/gcp-runtimes/python/gen-dockerfile:staging' + args: [ + '--base-image=gcr.io/google-appengine/python:staging' + ] +- # Use that Dockerfile to create final application image + name: 'gcr.io/cloud-builders/docker:latest' + args: ['build', '-t', '$_OUTPUT_IMAGE', '.'] +images: + - '$_OUTPUT_IMAGE' From 9b43e372a1495eceea50240dadda565f5744e225 Mon Sep 17 00:00:00 2001 From: Douglas Greiman Date: Tue, 27 Mar 2018 16:32:38 -0700 Subject: [PATCH 269/362] Build and test python interpreters in parallel instead of serially. (#186) --- cloudbuild.yaml | 23 +++++++-- cloudbuild_test.yaml | 51 ++++++++++++++++++- python-interpreter-builder/Dockerfile.in | 14 ----- .../scripts/build-python-3.4.sh | 3 ++ .../scripts/build-python-3.5.sh | 3 ++ .../scripts/build-python-3.6.sh | 3 ++ runtime-image/Dockerfile.in | 4 +- 7 files changed, 80 insertions(+), 21 deletions(-) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 5554015f..51a77480 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -1,20 +1,35 @@ timeout: 10800s steps: -- # Compile Python interpreters from source +- # Compile Python interpreters from source. This step happens first, then + # the next three in parallel. name: gcr.io/cloud-builders/docker:latest args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/interpreter-builder:${_TAG}', '--no-cache', '/workspace/python-interpreter-builder/'] -- # Copy interpreters back to workspace - name: ${_DOCKER_NAMESPACE}/python/interpreter-builder:${_TAG} - args: ['cp', '/interpreters.tar.gz', '/workspace/runtime-image/interpreters.tar.gz'] + id: interpreter-builder +- name: ${_DOCKER_NAMESPACE}/python/interpreter-builder:${_TAG} + args: ['/scripts/build-python-3.4.sh'] + id: build-3.4 + waitFor: ['interpreter-builder'] +- name: ${_DOCKER_NAMESPACE}/python/interpreter-builder:${_TAG} + args: ['/scripts/build-python-3.5.sh'] + id: build-3.5 + waitFor: ['interpreter-builder'] +- name: ${_DOCKER_NAMESPACE}/python/interpreter-builder:${_TAG} + args: ['/scripts/build-python-3.6.sh'] + id: build-3.6 + waitFor: ['interpreter-builder'] - # Build base runtime image name: gcr.io/cloud-builders/docker:latest args: ['build', '--tag=${_DOCKER_NAMESPACE}/python:${_TAG}', '--no-cache', '/workspace/runtime-image/'] + id: runtime + waitFor: ['build-3.4', 'build-3.5', 'build-3.6'] - # Build runtime builder image name: gcr.io/cloud-builders/docker:latest args: ['build', '--tag=${_BUILDER_DOCKER_NAMESPACE}/python/gen-dockerfile:${_TAG}', '--no-cache', '/workspace/builder/gen-dockerfile/'] + id: gen-dockerfile + waitFor: ['runtime'] images: [ '${_DOCKER_NAMESPACE}/python/interpreter-builder:${_TAG}', '${_DOCKER_NAMESPACE}/python:${_TAG}', diff --git a/cloudbuild_test.yaml b/cloudbuild_test.yaml index ef0b48b3..416c5204 100644 --- a/cloudbuild_test.yaml +++ b/cloudbuild_test.yaml @@ -5,18 +5,52 @@ steps: args: [ '/bin/true', ] + id: runtime + - # Validate structure of base runtime image name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 args: [ '-test.v', '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', '/workspace/tests/virtualenv/virtualenv_default.yaml', + ] + waitFor: ['runtime'] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 + args: [ + '-test.v', + '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', '/workspace/tests/virtualenv/virtualenv_python27.yaml', + ] + waitFor: ['runtime'] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 + args: [ + '-test.v', + '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', '/workspace/tests/virtualenv/virtualenv_python34.yaml', + ] + waitFor: ['runtime'] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 + args: [ + '-test.v', + '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', '/workspace/tests/virtualenv/virtualenv_python35.yaml', + ] + waitFor: ['runtime'] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 + args: [ + '-test.v', + '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', '/workspace/tests/virtualenv/virtualenv_python36.yaml', + ] + waitFor: ['runtime'] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 + args: [ + '-test.v', + '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', '/workspace/tests/no-virtualenv/no-virtualenv.yaml', ] + waitFor: ['runtime'] + # Temporarily disabled because it fails on symbolic links in Ubuntu: # https://github.com/GoogleCloudPlatform/container-structure-test/issues/77 #- # Check license compliance @@ -26,35 +60,48 @@ steps: # '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', # '/workspace/tests/license-test/license-test.yaml' # ] -- # Do third-party library compatibility tests +# waitFor: ['runtime'] + +- # Do third-party library compatibility tests for Python 2 name: gcr.io/cloud-builders/docker:latest args: [ 'build', '-t', 'python2-libraries-intermediate', '--build-arg', 'intermediate_image=${_DOCKER_NAMESPACE}/python:${_TAG}', '/workspace/tests/python2-libraries' ] + id: python2-libraries-intermediate + waitFor: ['runtime'] - name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 args: [ '-test.v', '-image', 'python2-libraries-intermediate', '/workspace/tests/python2-libraries/python2-libraries.yaml' ] -- name: gcr.io/cloud-builders/docker:latest + waitFor: ['python2-libraries-intermediate'] + +- # Do third-party library compatibility tests for Python 3 + name: gcr.io/cloud-builders/docker:latest args: [ 'build', '-t', 'python3-libraries-intermediate', '--build-arg', 'intermediate_image=${_DOCKER_NAMESPACE}/python:${_TAG}', '/workspace/tests/python3-libraries' ] + id: python3-libraries-intermediate + waitFor: ['runtime'] - name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 args: [ '-test.v', '-image', 'python3-libraries-intermediate', '/workspace/tests/python3-libraries/python3-libraries.yaml' ] + waitFor: ['python3-libraries-intermediate'] + - # Run other compatibility tests name: gcr.io/cloud-builders/docker:latest args: [ 'build', '--tag=${_DOCKER_NAMESPACE}/python/tests/eventlet:${_TAG}', '--no-cache', '/workspace/tests/eventlet/' ] + waitFor: ['runtime'] + images: [] diff --git a/python-interpreter-builder/Dockerfile.in b/python-interpreter-builder/Dockerfile.in index 46bc092a..9039fbf3 100644 --- a/python-interpreter-builder/Dockerfile.in +++ b/python-interpreter-builder/Dockerfile.in @@ -46,17 +46,3 @@ ENV LANG C.UTF-8 # Add build scripts ADD scripts /scripts ADD DEBIAN /DEBIAN - -# Build the Python interpreters -RUN mkdir -p /opt/packages && \ - echo -n "" > /opt/packages/packages.txt - -RUN /scripts/build-python-3.4.sh - -RUN /scripts/build-python-3.5.sh - -RUN /scripts/build-python-3.6.sh - -# Tar the interpreters. Tarring is needed because docker cp doesn't handle -# links correctly. -RUN tar czf /interpreters.tar.gz /opt/python?.? diff --git a/python-interpreter-builder/scripts/build-python-3.4.sh b/python-interpreter-builder/scripts/build-python-3.4.sh index 95089f75..5c0bfb8e 100755 --- a/python-interpreter-builder/scripts/build-python-3.4.sh +++ b/python-interpreter-builder/scripts/build-python-3.4.sh @@ -136,3 +136,6 @@ find "$PREFIX"/lib/python3.4/test \ cd /opt rm /opt/sources/Python-3.4.8.tgz rm -r /opt/sources/Python-3.4.8 + +# Archive and copy to persistent external volume +tar czf /workspace/runtime-image/interpreter-3.4.tar.gz /opt/python3.4 diff --git a/python-interpreter-builder/scripts/build-python-3.5.sh b/python-interpreter-builder/scripts/build-python-3.5.sh index c2a3337e..86a564c3 100755 --- a/python-interpreter-builder/scripts/build-python-3.5.sh +++ b/python-interpreter-builder/scripts/build-python-3.5.sh @@ -148,3 +148,6 @@ find "$PREFIX"/lib/python3.5/test \ cd /opt rm /opt/sources/Python-3.5.5.tgz rm -r /opt/sources/Python-3.5.5 + +# Archive and copy to persistent external volume +tar czf /workspace/runtime-image/interpreter-3.5.tar.gz /opt/python3.5 diff --git a/python-interpreter-builder/scripts/build-python-3.6.sh b/python-interpreter-builder/scripts/build-python-3.6.sh index a61b2a8b..fc19fc5e 100755 --- a/python-interpreter-builder/scripts/build-python-3.6.sh +++ b/python-interpreter-builder/scripts/build-python-3.6.sh @@ -148,3 +148,6 @@ find "$PREFIX"/lib/python3.6/test \ cd /opt rm /opt/sources/Python-3.6.4.tgz rm -r /opt/sources/Python-3.6.4 + +# Archive and copy to persistent external volume +tar czf /workspace/runtime-image/interpreter-3.6.tar.gz /opt/python3.6 diff --git a/runtime-image/Dockerfile.in b/runtime-image/Dockerfile.in index d0a2e060..1332279c 100644 --- a/runtime-image/Dockerfile.in +++ b/runtime-image/Dockerfile.in @@ -17,7 +17,9 @@ ENV LANG C.UTF-8 ENV PYTHONUNBUFFERED 1 # Install the Google-built interpreters -ADD interpreters.tar.gz / +ADD interpreter-3.4.tar.gz / +ADD interpreter-3.5.tar.gz / +ADD interpreter-3.6.tar.gz / # Add Google-built interpreters to the path ENV PATH /opt/python3.6/bin:/opt/python3.5/bin:/opt/python3.4/bin:$PATH From 8d34b501d5b08d3995e1c45c9e212dfa82387043 Mon Sep 17 00:00:00 2001 From: Douglas Greiman Date: Thu, 29 Mar 2018 14:04:02 -0700 Subject: [PATCH 270/362] Restore python3 and pip3 $PATH entries. (#189) They now invoke Python 3.6. Previously they invoked Python 3.4. --- runtime-image/Dockerfile.in | 2 ++ tests/no-virtualenv/no-virtualenv.yaml | 11 +++++++++++ 2 files changed, 13 insertions(+) diff --git a/runtime-image/Dockerfile.in b/runtime-image/Dockerfile.in index 1332279c..f0040cff 100644 --- a/runtime-image/Dockerfile.in +++ b/runtime-image/Dockerfile.in @@ -23,6 +23,8 @@ ADD interpreter-3.6.tar.gz / # Add Google-built interpreters to the path ENV PATH /opt/python3.6/bin:/opt/python3.5/bin:/opt/python3.4/bin:$PATH +RUN update-alternatives --install /usr/local/bin/python3 python3 /opt/python3.6/bin/python3.6 50 +RUN update-alternatives --install /usr/local/bin/pip3 pip3 /opt/python3.6/bin/pip3.6 50 # Upgrade pip (debian package version tends to run a few version behind) and # install virtualenv system-wide. diff --git a/tests/no-virtualenv/no-virtualenv.yaml b/tests/no-virtualenv/no-virtualenv.yaml index cf997e29..1015b0da 100644 --- a/tests/no-virtualenv/no-virtualenv.yaml +++ b/tests/no-virtualenv/no-virtualenv.yaml @@ -37,6 +37,17 @@ commandTests: command: ["which", "gunicorn"] expectedOutput: ["/usr/local/bin/gunicorn\n"] + - # Regression test for issue187 + name: "default python3 installation" + command: ["which", "python3"] + expectedOutput: ["/usr/local/bin/python3\n"] + - name: "default python3 version" + command: ["python3", "--version"] + expectedOutput: ["Python 3.6.4\n"] + - name: "default pip3 installation" + command: ["which", "pip3"] + expectedOutput: ["/usr/local/bin/pip3\n"] + - name: "default flask installation" # Checks that 'pip' and 'python' are using the same Python version setup: [["pip", "install", "flask"]] From bae048399184724052ee344ec6de9553e13fbb6b Mon Sep 17 00:00:00 2001 From: dlorenc Date: Tue, 3 Apr 2018 08:38:37 -0700 Subject: [PATCH 271/362] Update to Python 3.6.5. --- .../scripts/build-python-3.6.sh | 12 ++++++------ tests/no-virtualenv/no-virtualenv.yaml | 2 +- tests/virtualenv/virtualenv_python36.yaml | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/python-interpreter-builder/scripts/build-python-3.6.sh b/python-interpreter-builder/scripts/build-python-3.6.sh index fc19fc5e..63a09d2f 100755 --- a/python-interpreter-builder/scripts/build-python-3.6.sh +++ b/python-interpreter-builder/scripts/build-python-3.6.sh @@ -6,14 +6,14 @@ set -x # Get the source mkdir -p /opt/sources cd /opt/sources -wget --no-verbose https://www.python.org/ftp/python/3.6.4/Python-3.6.4.tgz +wget --no-verbose https://www.python.org/ftp/python/3.6.5/Python-3.6.5.tgz # SHA-256 generated via `shasum -a 256 [file]` shasum --check < Date: Wed, 4 Apr 2018 23:39:32 -0700 Subject: [PATCH 272/362] Remove duplicate pip executables to avoid confusion. (#191) --- runtime-image/Dockerfile.in | 3 +++ 1 file changed, 3 insertions(+) diff --git a/runtime-image/Dockerfile.in b/runtime-image/Dockerfile.in index f0040cff..b6c76d71 100644 --- a/runtime-image/Dockerfile.in +++ b/runtime-image/Dockerfile.in @@ -30,8 +30,11 @@ RUN update-alternatives --install /usr/local/bin/pip3 pip3 /opt/python3.6/bin/pi # install virtualenv system-wide. RUN /usr/bin/pip install --upgrade -r /resources/requirements.txt && \ /opt/python3.4/bin/pip3.4 install --upgrade -r /resources/requirements.txt && \ + rm -f /opt/python3.4/bin/pip /opt/python3.4/bin/pip3 && \ /opt/python3.5/bin/pip3.5 install --upgrade -r /resources/requirements.txt && \ + rm -f /opt/python3.5/bin/pip /opt/python3.5/bin/pip3 && \ /opt/python3.6/bin/pip3.6 install --upgrade -r /resources/requirements.txt && \ + rm -f /opt/python3.6/bin/pip /opt/python3.6/bin/pip3 && \ /usr/bin/pip install --upgrade -r /resources/requirements-virtualenv.txt # Setup the app working directory From a8a3e8b2d3239c184843db818e34a06f12dc1190 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Thu, 5 Apr 2018 14:09:34 -0700 Subject: [PATCH 273/362] Auto-update dependencies. (#188) --- .../resources/requirements-virtualenv.txt | 2 +- runtime-image/resources/requirements.txt | 6 +- scripts/requirements-test.txt | 2 +- tests/python2-libraries/requirements.txt | 72 +++++++++---------- tests/python3-libraries/requirements.txt | 72 +++++++++---------- 5 files changed, 77 insertions(+), 77 deletions(-) diff --git a/runtime-image/resources/requirements-virtualenv.txt b/runtime-image/resources/requirements-virtualenv.txt index a4ce32e5..fa784bd2 100644 --- a/runtime-image/resources/requirements-virtualenv.txt +++ b/runtime-image/resources/requirements-virtualenv.txt @@ -1 +1 @@ -virtualenv==15.1.0 +virtualenv==15.2.0 diff --git a/runtime-image/resources/requirements.txt b/runtime-image/resources/requirements.txt index ef675285..c3f99d24 100644 --- a/runtime-image/resources/requirements.txt +++ b/runtime-image/resources/requirements.txt @@ -1,3 +1,3 @@ -pip==9.0.1 -setuptools==38.5.2 -wheel==0.30.0 +pip==9.0.3 +setuptools==39.0.1 +wheel==0.31.0 diff --git a/scripts/requirements-test.txt b/scripts/requirements-test.txt index 1ae0abd8..8be25b0f 100644 --- a/scripts/requirements-test.txt +++ b/scripts/requirements-test.txt @@ -1,4 +1,4 @@ flask==0.12.2 -pytest==3.4.2 +pytest==3.5.0 pytest-cov==2.5.1 pyyaml==3.12 diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index f261209e..c9c61dec 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -1,12 +1,12 @@ -alembic==0.9.8 +alembic==0.9.9 amqp==2.2.2 amqplib==1.0.2 -ansible==2.4.3.0 +ansible==2.5.0 anyjson==0.3.3 apache-libcloud==2.3.0 argparse==1.4.0 -astroid==1.6.1 -awscli==1.14.53 +astroid==1.6.2 +awscli==1.14.68 babel==2.5.3 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 @@ -16,7 +16,7 @@ billiard==3.5.0.3 blessings==1.6.1 blinker==1.4 boto==2.48.0 -botocore==1.9.6 +botocore==1.9.21 bottle==0.12.13 carbon<1.1.1 celery==4.1.0 @@ -25,20 +25,20 @@ cffi==1.11.5 chardet==3.0.4 click==6.7 cliff==2.11.0 -cmd2==0.8.1 +cmd2==0.8.2 colorama==0.3.9 configobj==5.0.6 cov-core==1.15.0 coverage==4.5.1 coveralls==1.3.0 crcmod==1.7 -cryptography==2.1.4 +cryptography==2.2.2 cssselect==1.0.3 -cython==0.27.3 +cython==0.28.1 decorator==4.2.1 django-celery==3.2.2 django-debug-toolbar==1.9.1 -django-extensions==2.0.5 +django-extensions==2.0.6 django<2.0 django_compress==1.0.1 djangorestframework==3.7.7 @@ -46,7 +46,7 @@ docker-py==1.10.6 docopt==0.6.2 docutils==0.14 ecdsa==0.13 -elasticsearch==6.1.1 +elasticsearch==6.2.0 enum34==1.1.6 eventlet==0.22.1 extras==1.0.0 @@ -58,14 +58,14 @@ funcsigs==1.0.2 functools32==3.2.3.post2 futures==3.2.0 gevent==1.2.2 -google-api-python-client==1.6.5 +google-api-python-client==1.6.6 graphite-web==1.1.2 greenlet==0.4.13 gunicorn==19.7.1 hiredis==0.2.0 honcho==1.0.1 html5lib==1.0.1 -httplib2==0.10.3 +httplib2==0.11.3 idna==2.6 ipaddress==1.0.19 iso8601==0.1.12 @@ -77,13 +77,13 @@ jsonschema==2.6.0 kombu==4.1.0 linecache2==1.0.0 logilab-common==1.4.1 -lxml==4.1.1 +lxml==4.2.1 m2crypto==0.29.0 mako==1.0.7 manifestparser==1.1 markdown==2.6.11 markupsafe==1.0 -matplotlib==2.2.0 +matplotlib==2.2.2 mccabe==0.6.1 meld3==1.0.2 mock==2.0.0 @@ -101,31 +101,31 @@ mysql-python==1.2.5 ndg-httpsclient==0.4.4 netaddr==0.7.19 netifaces==0.10.6 -newrelic==2.106.1.88 +newrelic==3.0.0.89 nose==1.3.7 -numpy==1.14.1 +numpy==1.14.2 oauth2==1.9.0.post1 oauth2client==4.1.2 -oauthlib==2.0.6 +oauthlib==2.0.7 ordereddict==1.1 -oslo.config==5.2.0 +oslo.config==6.0.0 pandas==0.22.0 -paramiko==2.4.0 +paramiko==2.4.1 passlib==1.7.1 paste==2.0.3 pastedeploy==1.5.2 pastescript==2.0.2 -pbr==3.1.1 +pbr==4.0.0 pep8==1.7.1 pexpect==4.4.0 pika==0.11.2 pillow==5.0.0 -pip==9.0.1 +pip==9.0.3 prettytable -protobuf==3.5.2 +protobuf==3.5.2.post1 psutil==5.4.3 psycopg2==2.7.4 -py==1.5.2 +py==1.5.3 pyasn1-modules==0.2.1 pyasn1==0.4.2 pycparser==2.18 @@ -133,9 +133,9 @@ pycrypto==2.6.1 pycurl==7.43.0.1 pyflakes==1.6.0 pygments==2.2.0 -pyjwt==1.6.0 +pyjwt==1.6.1 pylibmc==1.5.2 -pylint==1.8.2 +pylint==1.8.3 pymongo==3.6.1 pymysql==0.8.0 pyopenssl==17.5.0 @@ -143,10 +143,10 @@ pyparsing==2.2.0 pyramid==1.9.1 pystache==0.5.4 pytest-cov==2.5.1 -pytest==3.4.2 +pytest==3.5.0 python-cjson==1.2.1 python-daemon==2.1.2 -python-dateutil==2.7.0 +python-dateutil==2.7.2 python-gflags==3.1.2 python-keystoneclient==3.15.0 python-memcached==1.59 @@ -164,18 +164,18 @@ requests-oauthlib==0.8.0 requests==2.18.4 retrying==1.3.3 rsa==3.4.2 -scipy==1.0.0 -selenium==3.10.0 +scipy==1.0.1 +selenium==3.11.0 setuptools-git==1.2 -setuptools==38.5.2 +setuptools==39.0.1 sh==1.12.14 simplejson==3.13.2 six==1.11.0 snowballstemmer==1.2.1 south==1.0.2 -sphinx==1.7.1 +sphinx==1.7.2 sqlalchemy-migrate==0.11.0 -sqlalchemy==1.2.5 +sqlalchemy==1.2.6 sqlparse==0.2.4 statsd==3.2.2 stevedore==1.28.0 @@ -184,7 +184,7 @@ supervisor==3.3.4 testrepository==0.0.20 testtools==2.3.0 thrift==0.11.0 -tornado==5.0 +tornado==5.0.1 tox==2.9.1 twisted==17.9.0 ujson==1.35 @@ -194,14 +194,14 @@ uritemplate==3.0.0 urllib3==1.22 uwsgi==2.0.17 versiontools==1.9.1 -virtualenv==15.1.0 +virtualenv==15.2.0 waitress==1.1.0 warlock==1.3.0 webob==1.7.4 websocket-client==0.47.0 webtest==2.0.29 werkzeug==0.14.1 -wheel==0.30.0 +wheel==0.31.0 xlrd==1.1.0 -zc.buildout==2.11.1 +zc.buildout==2.11.2 zope.interface==4.4.3 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 30fb236e..855b0906 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -1,12 +1,12 @@ -alembic==0.9.8 +alembic==0.9.9 amqp==2.2.2 amqplib==1.0.2 -ansible==2.4.3.0 +ansible==2.5.0 anyjson==0.3.3 apache-libcloud==2.3.0 argparse==1.4.0 -astroid==1.6.1 -awscli==1.14.53 +astroid==1.6.2 +awscli==1.14.68 babel==2.5.3 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 @@ -15,7 +15,7 @@ billiard==3.5.0.3 blessings==1.6.1 blinker==1.4 boto==2.48.0 -botocore==1.9.6 +botocore==1.9.21 bottle==0.12.13 celery==4.1.0 certifi==2018.1.18 @@ -23,20 +23,20 @@ cffi==1.11.5 chardet==3.0.4 click==6.7 cliff==2.11.0 -cmd2==0.8.1 +cmd2==0.8.2 colorama==0.3.9 configobj==5.0.6 cov-core==1.15.0 coverage==4.5.1 coveralls==1.3.0 crcmod==1.7 -cryptography==2.1.4 +cryptography==2.2.2 cssselect==1.0.3 -cython==0.27.3 +cython==0.28.1 decorator==4.2.1 django-celery==3.2.2 django-debug-toolbar==1.9.1 -django-extensions==2.0.5 +django-extensions==2.0.6 django==2.0.3 django_compress==1.0.1 djangorestframework==3.7.7 @@ -44,7 +44,7 @@ docker-py==1.10.6 docopt==0.6.2 docutils==0.14 ecdsa==0.13 -elasticsearch==6.1.1 +elasticsearch==6.2.0 enum34==1.1.6 eventlet==0.22.1 extras==1.0.0 @@ -54,13 +54,13 @@ flake8==3.5.0 flask==0.12.2 funcsigs==1.0.2 gevent==1.2.2 -google-api-python-client==1.6.5 +google-api-python-client==1.6.6 greenlet==0.4.13 gunicorn==19.7.1 hiredis==0.2.0 honcho==1.0.1 html5lib==1.0.1 -httplib2==0.10.3 +httplib2==0.11.3 idna==2.6 ipaddress==1.0.19 ipython==6.2.1 @@ -73,12 +73,12 @@ jsonschema==2.6.0 kombu==4.1.0 linecache2==1.0.0 logilab-common==1.4.1 -lxml==4.1.1 +lxml==4.2.1 mako==1.0.7 manifestparser==1.1 markdown==2.6.11 markupsafe==1.0 -matplotlib==2.2.0 +matplotlib==2.2.2 mccabe==0.6.1 meld3==1.0.2 mock==2.0.0 @@ -93,40 +93,40 @@ msgpack-python==0.5.6 ndg-httpsclient==0.4.4 netaddr==0.7.19 netifaces==0.10.6 -newrelic==2.106.1.88 +newrelic==3.0.0.89 nose==1.3.7 -numpy==1.14.1 +numpy==1.14.2 oauth2==1.9.0.post1 oauth2client==4.1.2 -oauthlib==2.0.6 +oauthlib==2.0.7 ordereddict==1.1 -oslo.config==5.2.0 +oslo.config==6.0.0 pandas==0.22.0 -paramiko==2.4.0 +paramiko==2.4.1 passlib==1.7.1 paste==2.0.3 pastedeploy==1.5.2 pastescript==2.0.2 -pbr==3.1.1 +pbr==4.0.0 pep8==1.7.1 pexpect==4.4.0 pika==0.11.2 pillow==5.0.0 -pip==9.0.1 +pip==9.0.3 prettytable -protobuf==3.5.2 +protobuf==3.5.2.post1 psutil==5.4.3 psycopg2==2.7.4 -py==1.5.2 +py==1.5.3 pyasn1-modules==0.2.1 pyasn1==0.4.2 pycparser==2.18 pycrypto==2.6.1 pyflakes==1.6.0 pygments==2.2.0 -pyjwt==1.6.0 +pyjwt==1.6.1 pylibmc==1.5.2 -pylint==1.8.2 +pylint==1.8.3 pymongo==3.6.1 pymysql==0.8.0 pyopenssl==17.5.0 @@ -134,9 +134,9 @@ pyparsing==2.2.0 pyramid==1.9.1 pystache==0.5.4 pytest-cov==2.5.1 -pytest==3.4.2 +pytest==3.5.0 python-daemon==2.1.2 -python-dateutil==2.7.0 +python-dateutil==2.7.2 python-gflags==3.1.2 python-keystoneclient==3.15.0 python-memcached==1.59 @@ -154,25 +154,25 @@ requests-oauthlib==0.8.0 requests==2.18.4 retrying==1.3.3 rsa==3.4.2 -scipy==1.0.0 -selenium==3.10.0 +scipy==1.0.1 +selenium==3.11.0 setuptools-git==1.2 -setuptools==38.5.2 +setuptools==39.0.1 sh==1.12.14 simplejson==3.13.2 six==1.11.0 snowballstemmer==1.2.1 south==1.0.2 -sphinx==1.7.1 +sphinx==1.7.2 sqlalchemy-migrate==0.11.0 -sqlalchemy==1.2.5 +sqlalchemy==1.2.6 sqlparse==0.2.4 statsd==3.2.2 stevedore==1.28.0 testrepository==0.0.20 testtools==2.3.0 thrift==0.11.0 -tornado==5.0 +tornado==5.0.1 tox==2.9.1 twisted==17.9.0 ujson==1.35 @@ -182,14 +182,14 @@ uritemplate==3.0.0 urllib3==1.22 uwsgi==2.0.17 versiontools==1.9.1 -virtualenv==15.1.0 +virtualenv==15.2.0 waitress==1.1.0 warlock==1.3.0 webob==1.7.4 websocket-client==0.47.0 webtest==2.0.29 werkzeug==0.14.1 -wheel==0.30.0 +wheel==0.31.0 xlrd==1.1.0 -zc.buildout==2.11.1 +zc.buildout==2.11.2 zope.interface==4.4.3 From 11cc1eba0c3a8d44d5c90fe77a2546358aa3794a Mon Sep 17 00:00:00 2001 From: dlorenc Date: Thu, 17 May 2018 11:07:44 -0700 Subject: [PATCH 274/362] Split the runtime release into an interpreter and a runtime build. --- RELEASING.md | 11 +++++++++++ build.sh | 13 +++++++++++++ cloudbuild.yaml | 20 -------------------- cloudbuild_interpreters.yaml | 29 +++++++++++++++++++++++++++++ runtime-image/Dockerfile.in | 11 +++++++---- 5 files changed, 60 insertions(+), 24 deletions(-) create mode 100644 cloudbuild_interpreters.yaml diff --git a/RELEASING.md b/RELEASING.md index 00b0fe16..9f61130a 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -39,6 +39,17 @@ follows: 4. `build.sh` invokes Google Container Builder with the `cloudbuild-*.yaml` config files. +## Building interpreters + +The interpreters used are now built in a separate step, and stored on GCS. +This allows the runtime images to be build more rapidly. + +To build the interpreters, run: + +```shell +gcloud container builds submit . --config=cloudbuild_interpreters.yaml +``` + ## Building outside Jenkins To build this repository outside Jenkins, authenticate and authorize yourself diff --git a/build.sh b/build.sh index 9b3a258d..badda7b5 100755 --- a/build.sh +++ b/build.sh @@ -25,6 +25,7 @@ test=0 # Should run standard test suite? local=0 # Should run using local Docker daemon instead of GCR? os_base=debian8 # Which operating system base to use +interpreter=0 # Should build interpreters instead of images # Note that $gcloud_cmd has spaces in it gcloud_cmd="gcloud container builds submit" @@ -126,6 +127,10 @@ while [ $# -gt 0 ]; do test=0 shift ;; + --interpreter) + interpreter=1 + shift + ;; *) usage ;; @@ -184,6 +189,14 @@ done # Make a file available to the eventlet test. cp -a scripts/testdata/hello_world/main.py tests/eventlet/main.py +# Build interpreters and push to GCS +if [ "${interpreter}" -eq 1 ]; then + echo "Building interpreters" + ${gcloud_cmd} \ + --config=cloudbuild_interpreters.yaml \ + . +fi + # Build images and push to GCR if [ "${build}" -eq 1 ]; then echo "Building images" diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 51a77480..e240780e 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -1,29 +1,10 @@ timeout: 10800s steps: -- # Compile Python interpreters from source. This step happens first, then - # the next three in parallel. - name: gcr.io/cloud-builders/docker:latest - args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/interpreter-builder:${_TAG}', - '--no-cache', '/workspace/python-interpreter-builder/'] - id: interpreter-builder -- name: ${_DOCKER_NAMESPACE}/python/interpreter-builder:${_TAG} - args: ['/scripts/build-python-3.4.sh'] - id: build-3.4 - waitFor: ['interpreter-builder'] -- name: ${_DOCKER_NAMESPACE}/python/interpreter-builder:${_TAG} - args: ['/scripts/build-python-3.5.sh'] - id: build-3.5 - waitFor: ['interpreter-builder'] -- name: ${_DOCKER_NAMESPACE}/python/interpreter-builder:${_TAG} - args: ['/scripts/build-python-3.6.sh'] - id: build-3.6 - waitFor: ['interpreter-builder'] - # Build base runtime image name: gcr.io/cloud-builders/docker:latest args: ['build', '--tag=${_DOCKER_NAMESPACE}/python:${_TAG}', '--no-cache', '/workspace/runtime-image/'] id: runtime - waitFor: ['build-3.4', 'build-3.5', 'build-3.6'] - # Build runtime builder image name: gcr.io/cloud-builders/docker:latest args: ['build', '--tag=${_BUILDER_DOCKER_NAMESPACE}/python/gen-dockerfile:${_TAG}', @@ -31,7 +12,6 @@ steps: id: gen-dockerfile waitFor: ['runtime'] images: [ - '${_DOCKER_NAMESPACE}/python/interpreter-builder:${_TAG}', '${_DOCKER_NAMESPACE}/python:${_TAG}', '${_BUILDER_DOCKER_NAMESPACE}/python/gen-dockerfile:${_TAG}', ] diff --git a/cloudbuild_interpreters.yaml b/cloudbuild_interpreters.yaml new file mode 100644 index 00000000..7daecc57 --- /dev/null +++ b/cloudbuild_interpreters.yaml @@ -0,0 +1,29 @@ +timeout: 10800s +steps: +- # Compile Python interpreters from source. This step happens first, then + # the next three in parallel. + name: gcr.io/cloud-builders/docker:latest + args: ['build', '--tag=interpreter-builder', + '--no-cache', '/workspace/python-interpreter-builder/'] + id: interpreter-builder +- name: interpreter-builder + args: ['/scripts/build-python-3.4.sh'] + id: build-3.4 + waitFor: ['interpreter-builder'] +- name: interpreter-builder + args: ['/scripts/build-python-3.5.sh'] + id: build-3.5 + waitFor: ['interpreter-builder'] +- name: interpreter-builder + args: ['/scripts/build-python-3.6.sh'] + id: build-3.6 + waitFor: ['interpreter-builder'] + +# Upload them to tbe build-id location +- name: gcr.io/cloud-builders/gsutil:latest + args: ['cp', '/workspace/runtime-image/*.tar.gz', 'gs://python-interpreters/$BUILD_ID/'] + waitFor: ['build-3.4', 'build-3.5', 'build-3.6'] + +# "Tag" this as latest +- name: gcr.io/cloud-builders/gsutil:latest + args: ['cp', '-r', 'gs://python-interpreters/$BUILD_ID/*', 'gs://python-interpreters/latest/'] diff --git a/runtime-image/Dockerfile.in b/runtime-image/Dockerfile.in index b6c76d71..f291ae7f 100644 --- a/runtime-image/Dockerfile.in +++ b/runtime-image/Dockerfile.in @@ -16,10 +16,13 @@ ENV LANG C.UTF-8 # logging collection. ENV PYTHONUNBUFFERED 1 -# Install the Google-built interpreters -ADD interpreter-3.4.tar.gz / -ADD interpreter-3.5.tar.gz / -ADD interpreter-3.6.tar.gz / +RUN wget https://storage.googleapis.com/python-interpreters/latest/interpreter-3.4.tar.gz && \ + wget https://storage.googleapis.com/python-interpreters/latest/interpreter-3.5.tar.gz && \ + wget https://storage.googleapis.com/python-interpreters/latest/interpreter-3.6.tar.gz && \ + tar -xzf interpreter-3.4.tar.gz && \ + tar -xzf interpreter-3.5.tar.gz && \ + tar -xzf interpreter-3.6.tar.gz && \ + rm interpreter-*.tar.gz # Add Google-built interpreters to the path ENV PATH /opt/python3.6/bin:/opt/python3.5/bin:/opt/python3.4/bin:$PATH From 7a1444fead32ad3d9ae9e1d34782f034cd551ce6 Mon Sep 17 00:00:00 2001 From: David Bendory Date: Tue, 17 Jul 2018 18:32:44 -0400 Subject: [PATCH 275/362] container-builder-local has been renamed cloud-build-local. --- build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sh b/build.sh index badda7b5..476917c4 100755 --- a/build.sh +++ b/build.sh @@ -29,8 +29,8 @@ interpreter=0 # Should build interpreters instead of images # Note that $gcloud_cmd has spaces in it gcloud_cmd="gcloud container builds submit" -# May need to install via "gcloud components install container-builder-local" -local_gcloud_cmd="container-builder-local --push=false --dryrun=false" +# May need to install via "gcloud components install cloud-build-local" +local_gcloud_cmd="cloud-build-local --push=false --dryrun=false" # Helper functions function fatal() { From ddfbf2215ce2d315119a27944f8cf41c62836908 Mon Sep 17 00:00:00 2001 From: David Bendory Date: Tue, 17 Jul 2018 18:37:20 -0400 Subject: [PATCH 276/362] `gcloud builds` replaces `gcloud container builds`. --- RELEASING.md | 2 +- build.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/RELEASING.md b/RELEASING.md index 9f61130a..2dcd806d 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -47,7 +47,7 @@ This allows the runtime images to be build more rapidly. To build the interpreters, run: ```shell -gcloud container builds submit . --config=cloudbuild_interpreters.yaml +gcloud builds submit . --config=cloudbuild_interpreters.yaml ``` ## Building outside Jenkins diff --git a/build.sh b/build.sh index 476917c4..bdc4f658 100755 --- a/build.sh +++ b/build.sh @@ -28,7 +28,7 @@ os_base=debian8 # Which operating system base to use interpreter=0 # Should build interpreters instead of images # Note that $gcloud_cmd has spaces in it -gcloud_cmd="gcloud container builds submit" +gcloud_cmd="gcloud builds submit" # May need to install via "gcloud components install cloud-build-local" local_gcloud_cmd="cloud-build-local --push=false --dryrun=false" From 20e146f027aa137fd061c192721be28902a5900a Mon Sep 17 00:00:00 2001 From: sharifelgamal Date: Tue, 31 Jul 2018 16:28:41 -0700 Subject: [PATCH 277/362] bumping cryptography to 2.3 for CVE --- tests/python2-libraries/requirements.txt | 2 +- tests/python3-libraries/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index c9c61dec..1655846a 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -32,7 +32,7 @@ cov-core==1.15.0 coverage==4.5.1 coveralls==1.3.0 crcmod==1.7 -cryptography==2.2.2 +cryptography==2.3 cssselect==1.0.3 cython==0.28.1 decorator==4.2.1 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 855b0906..e8295d80 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -30,7 +30,7 @@ cov-core==1.15.0 coverage==4.5.1 coveralls==1.3.0 crcmod==1.7 -cryptography==2.2.2 +cryptography==2.3 cssselect==1.0.3 cython==0.28.1 decorator==4.2.1 From e0b754bf62e2087c5bbad077b02a9860eb132cb2 Mon Sep 17 00:00:00 2001 From: dlorenc Date: Fri, 10 Aug 2018 09:28:34 -0700 Subject: [PATCH 278/362] Bump Python 3.6. --- .../scripts/build-python-3.6.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/python-interpreter-builder/scripts/build-python-3.6.sh b/python-interpreter-builder/scripts/build-python-3.6.sh index 63a09d2f..ef192e1e 100755 --- a/python-interpreter-builder/scripts/build-python-3.6.sh +++ b/python-interpreter-builder/scripts/build-python-3.6.sh @@ -6,21 +6,21 @@ set -x # Get the source mkdir -p /opt/sources cd /opt/sources -wget --no-verbose https://www.python.org/ftp/python/3.6.5/Python-3.6.5.tgz +wget --no-verbose https://www.python.org/ftp/python/3.6.6/Python-3.6.6.tgz # SHA-256 generated via `shasum -a 256 [file]` shasum --check < Date: Fri, 10 Aug 2018 09:51:01 -0700 Subject: [PATCH 279/362] Bump the test value to 3.6.6. --- tests/no-virtualenv/no-virtualenv.yaml | 2 +- tests/virtualenv/virtualenv_python36.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/no-virtualenv/no-virtualenv.yaml b/tests/no-virtualenv/no-virtualenv.yaml index 2d6e5052..869b93b2 100644 --- a/tests/no-virtualenv/no-virtualenv.yaml +++ b/tests/no-virtualenv/no-virtualenv.yaml @@ -43,7 +43,7 @@ commandTests: expectedOutput: ["/usr/local/bin/python3\n"] - name: "default python3 version" command: ["python3", "--version"] - expectedOutput: ["Python 3.6.5\n"] + expectedOutput: ["Python 3.6.6\n"] - name: "default pip3 installation" command: ["which", "pip3"] expectedOutput: ["/usr/local/bin/pip3\n"] diff --git a/tests/virtualenv/virtualenv_python36.yaml b/tests/virtualenv/virtualenv_python36.yaml index c8578cca..d4de875c 100644 --- a/tests/virtualenv/virtualenv_python36.yaml +++ b/tests/virtualenv/virtualenv_python36.yaml @@ -25,7 +25,7 @@ commandTests: - name: "virtualenv36 python version" setup: [["virtualenv", "-p", "python3.6", "/env"]] command: ["python", "--version"] - expectedOutput: ["Python 3.6.5\n"] + expectedOutput: ["Python 3.6.6\n"] - name: "virtualenv36 pip installation" setup: [["virtualenv", "-p", "python3.6", "/env"]] From b032725790a0109b17d50caaefd96e8953154416 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Mon, 13 Aug 2018 00:51:19 -0700 Subject: [PATCH 280/362] Auto-update dependencies. --- builder/gen-dockerfile/requirements.txt | 2 +- .../resources/requirements-virtualenv.txt | 2 +- runtime-image/resources/requirements.txt | 6 +- scripts/requirements-test.txt | 6 +- scripts/testdata/hello_world/requirements.txt | 4 +- tests/deploy_check/requirements.txt | 4 +- tests/eventlet/requirements.txt | 8 +- tests/integration/requirements.txt | 10 +- tests/python2-libraries/requirements.txt | 194 +++++++++--------- tests/python3-libraries/requirements.txt | 188 ++++++++--------- 10 files changed, 212 insertions(+), 212 deletions(-) diff --git a/builder/gen-dockerfile/requirements.txt b/builder/gen-dockerfile/requirements.txt index 59bc593a..4a285555 100644 --- a/builder/gen-dockerfile/requirements.txt +++ b/builder/gen-dockerfile/requirements.txt @@ -1 +1 @@ -PyYAML==3.12 +PyYAML==3.13 diff --git a/runtime-image/resources/requirements-virtualenv.txt b/runtime-image/resources/requirements-virtualenv.txt index fa784bd2..52bb7da1 100644 --- a/runtime-image/resources/requirements-virtualenv.txt +++ b/runtime-image/resources/requirements-virtualenv.txt @@ -1 +1 @@ -virtualenv==15.2.0 +virtualenv==16.0.0 diff --git a/runtime-image/resources/requirements.txt b/runtime-image/resources/requirements.txt index c3f99d24..44c624d8 100644 --- a/runtime-image/resources/requirements.txt +++ b/runtime-image/resources/requirements.txt @@ -1,3 +1,3 @@ -pip==9.0.3 -setuptools==39.0.1 -wheel==0.31.0 +pip==18.0 +setuptools==40.0.0 +wheel==0.31.1 diff --git a/scripts/requirements-test.txt b/scripts/requirements-test.txt index 8be25b0f..df909d47 100644 --- a/scripts/requirements-test.txt +++ b/scripts/requirements-test.txt @@ -1,4 +1,4 @@ -flask==0.12.2 -pytest==3.5.0 +flask==1.0.2 +pytest==3.7.1 pytest-cov==2.5.1 -pyyaml==3.12 +pyyaml==3.13 diff --git a/scripts/testdata/hello_world/requirements.txt b/scripts/testdata/hello_world/requirements.txt index bb84e50e..a34d076b 100644 --- a/scripts/testdata/hello_world/requirements.txt +++ b/scripts/testdata/hello_world/requirements.txt @@ -1,2 +1,2 @@ -Flask==0.12.2 -gunicorn==19.7.1 +Flask==1.0.2 +gunicorn==19.9.0 diff --git a/tests/deploy_check/requirements.txt b/tests/deploy_check/requirements.txt index bb84e50e..a34d076b 100644 --- a/tests/deploy_check/requirements.txt +++ b/tests/deploy_check/requirements.txt @@ -1,2 +1,2 @@ -Flask==0.12.2 -gunicorn==19.7.1 +Flask==1.0.2 +gunicorn==19.9.0 diff --git a/tests/eventlet/requirements.txt b/tests/eventlet/requirements.txt index 5d104d0a..7f8b2468 100644 --- a/tests/eventlet/requirements.txt +++ b/tests/eventlet/requirements.txt @@ -1,9 +1,9 @@ click==6.7 enum-compat==0.0.2 -eventlet==0.22.1 -Flask==0.12.2 -greenlet==0.4.13 -gunicorn==19.7.1 +eventlet==0.24.1 +Flask==1.0.2 +greenlet==0.4.14 +gunicorn==19.9.0 itsdangerous==0.24 Jinja2==2.10 MarkupSafe==1.0 diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index 64e39e0c..4a462433 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -1,8 +1,8 @@ -Flask==0.12.2 -google-cloud-error-reporting==0.29.1 +Flask==1.0.2 +google-cloud-error-reporting==0.30.0 google-cloud-logging==1.6.0 -google-cloud-monitoring==0.28.1 -gunicorn==19.7.1 -requests==2.18.4 +google-cloud-monitoring==0.30.1 +gunicorn==19.9.0 +requests==2.19.1 retrying==1.3.3 six==1.11.0 diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 1655846a..6daad0ab 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -1,31 +1,31 @@ -alembic==0.9.9 -amqp==2.2.2 +alembic==1.0.0 +amqp==2.3.2 amqplib==1.0.2 -ansible==2.5.0 +ansible==2.6.2 anyjson==0.3.3 apache-libcloud==2.3.0 argparse==1.4.0 -astroid==1.6.2 -awscli==1.14.68 -babel==2.5.3 +astroid==2.0.4 +awscli==1.15.76 +babel==2.6.0 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 -beautifulsoup4==4.6.0 +beautifulsoup4==4.6.3 beautifulsoup==3.2.1 -billiard==3.5.0.3 -blessings==1.6.1 +billiard==3.5.0.4 +blessings==1.7 blinker==1.4 -boto==2.48.0 -botocore==1.9.21 +boto==2.49.0 +botocore==1.10.75 bottle==0.12.13 carbon<1.1.1 -celery==4.1.0 -certifi==2018.1.18 +celery==4.2.1 +certifi==2018.8.13 cffi==1.11.5 chardet==3.0.4 click==6.7 -cliff==2.11.0 -cmd2==0.8.2 +cliff==2.13.0 +cmd2==0.9.3 colorama==0.3.9 configobj==5.0.6 cov-core==1.15.0 @@ -34,174 +34,174 @@ coveralls==1.3.0 crcmod==1.7 cryptography==2.3 cssselect==1.0.3 -cython==0.28.1 -decorator==4.2.1 +cython==0.28.5 +decorator==4.3.0 django-celery==3.2.2 django-debug-toolbar==1.9.1 -django-extensions==2.0.6 +django-extensions==2.1.0 django<2.0 django_compress==1.0.1 -djangorestframework==3.7.7 +djangorestframework==3.8.2 docker-py==1.10.6 docopt==0.6.2 docutils==0.14 ecdsa==0.13 -elasticsearch==6.2.0 +elasticsearch==6.3.1 enum34==1.1.6 -eventlet==0.22.1 +eventlet==0.24.1 extras==1.0.0 -fabric==1.14.0 +fabric==2.3.1 fixtures==3.0.0 flake8==3.5.0 -flask==0.12.2 +flask==1.0.2 funcsigs==1.0.2 functools32==3.2.3.post2 futures==3.2.0 -gevent==1.2.2 -google-api-python-client==1.6.6 -graphite-web==1.1.2 -greenlet==0.4.13 -gunicorn==19.7.1 +gevent==1.3.5 +google-api-python-client==1.7.4 +graphite-web==1.1.3 +greenlet==0.4.14 +gunicorn==19.9.0 hiredis==0.2.0 honcho==1.0.1 html5lib==1.0.1 httplib2==0.11.3 -idna==2.6 -ipaddress==1.0.19 +idna==2.7 +ipaddress==1.0.22 iso8601==0.1.12 isodate==0.6.0 itsdangerous==0.24 jinja2==2.10 jmespath==0.9.3 jsonschema==2.6.0 -kombu==4.1.0 +kombu==4.2.1 linecache2==1.0.0 -logilab-common==1.4.1 -lxml==4.2.1 -m2crypto==0.29.0 +logilab-common==1.4.2 +lxml==4.2.4 +m2crypto==0.30.1 mako==1.0.7 manifestparser==1.1 markdown==2.6.11 markupsafe==1.0 -matplotlib==2.2.2 +matplotlib==2.2.3 mccabe==0.6.1 meld3==1.0.2 mock==2.0.0 mozcrash==1.0 -mozdevice==0.52 +mozdevice==1.0.0 mozfile==1.2 mozinfo==0.10 -mozlog==3.7 +mozlog==3.8 moznetwork==0.27 mozprocess==0.26 -mozprofile==0.29 -mozrunner==6.14 +mozprofile==1.1.0 +mozrunner==7.0.1 msgpack-python==0.5.6 mysql-python==1.2.5 -ndg-httpsclient==0.4.4 +ndg-httpsclient==0.5.1 netaddr==0.7.19 -netifaces==0.10.6 -newrelic==3.0.0.89 +netifaces==0.10.7 +newrelic==4.2.0.100 nose==1.3.7 -numpy==1.14.2 +numpy==1.15.0 oauth2==1.9.0.post1 oauth2client==4.1.2 -oauthlib==2.0.7 +oauthlib==2.1.0 ordereddict==1.1 -oslo.config==6.0.0 -pandas==0.22.0 +oslo.config==6.4.0 +pandas==0.23.4 paramiko==2.4.1 passlib==1.7.1 paste==2.0.3 pastedeploy==1.5.2 pastescript==2.0.2 -pbr==4.0.0 +pbr==4.2.0 pep8==1.7.1 -pexpect==4.4.0 -pika==0.11.2 -pillow==5.0.0 -pip==9.0.3 -prettytable -protobuf==3.5.2.post1 -psutil==5.4.3 -psycopg2==2.7.4 -py==1.5.3 -pyasn1-modules==0.2.1 -pyasn1==0.4.2 +pexpect==4.6.0 +pika==0.12.0 +pillow==5.2.0 +pip==18.0 +prettytable==7 +protobuf==3.6.0 +psutil==5.4.6 +psycopg2==2.7.5 +py==1.5.4 +pyasn1-modules==0.2.2 +pyasn1==0.4.4 pycparser==2.18 pycrypto==2.6.1 -pycurl==7.43.0.1 -pyflakes==1.6.0 +pycurl==7.43.0.2 +pyflakes==2.0.0 pygments==2.2.0 -pyjwt==1.6.1 +pyjwt==1.6.4 pylibmc==1.5.2 -pylint==1.8.3 -pymongo==3.6.1 -pymysql==0.8.0 -pyopenssl==17.5.0 +pylint==2.1.1 +pymongo==3.7.1 +pymysql==0.9.2 +pyopenssl==18.0.0 pyparsing==2.2.0 -pyramid==1.9.1 +pyramid==1.9.2 pystache==0.5.4 pytest-cov==2.5.1 -pytest==3.5.0 +pytest==3.7.1 python-cjson==1.2.1 python-daemon==2.1.2 -python-dateutil==2.7.2 +python-dateutil==2.7.3 python-gflags==3.1.2 -python-keystoneclient==3.15.0 +python-keystoneclient==3.17.0 python-memcached==1.59 python-mimeparse==1.6.0 -python-novaclient==10.1.0 -python-subunit==1.2.0 -python-swiftclient==3.5.0 -pytz==2018.3 -pyyaml==3.12 -pyzmq==17.0.0 -raven==6.6.0 +python-novaclient==11.0.0 +python-subunit==1.3.0 +python-swiftclient==3.6.0 +pytz==2018.5 +pyyaml==3.13 +pyzmq==17.1.2 +raven==6.9.0 redis==2.10.6 repoze.lru==0.7 -requests-oauthlib==0.8.0 -requests==2.18.4 +requests-oauthlib==1.0.0 +requests==2.19.1 retrying==1.3.3 rsa==3.4.2 -scipy==1.0.1 -selenium==3.11.0 +scipy==1.1.0 +selenium==3.14.0 setuptools-git==1.2 -setuptools==39.0.1 +setuptools==40.0.0 sh==1.12.14 -simplejson==3.13.2 +simplejson==3.16.0 six==1.11.0 snowballstemmer==1.2.1 south==1.0.2 -sphinx==1.7.2 +sphinx==1.7.6 sqlalchemy-migrate==0.11.0 -sqlalchemy==1.2.6 +sqlalchemy==1.2.10 sqlparse==0.2.4 statsd==3.2.2 -stevedore==1.28.0 +stevedore==1.29.0 suds==0.4 supervisor==3.3.4 testrepository==0.0.20 testtools==2.3.0 thrift==0.11.0 -tornado==5.0.1 -tox==2.9.1 -twisted==17.9.0 +tornado==5.1 +tox==3.2.1 +twisted==18.7.0 ujson==1.35 unidecode==1.0.22 unittest2==1.1.0 uritemplate==3.0.0 -urllib3==1.22 -uwsgi==2.0.17 +urllib3==1.23 +uwsgi==2.0.17.1 versiontools==1.9.1 -virtualenv==15.2.0 +virtualenv==16.0.0 waitress==1.1.0 warlock==1.3.0 -webob==1.7.4 -websocket-client==0.47.0 -webtest==2.0.29 +webob==1.8.2 +websocket-client==0.48.0 +webtest==2.0.30 werkzeug==0.14.1 -wheel==0.31.0 +wheel==0.31.1 xlrd==1.1.0 -zc.buildout==2.11.2 -zope.interface==4.4.3 +zc.buildout==2.12.1 +zope.interface==4.5.0 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index e8295d80..7b839959 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -1,29 +1,29 @@ -alembic==0.9.9 -amqp==2.2.2 +alembic==1.0.0 +amqp==2.3.2 amqplib==1.0.2 -ansible==2.5.0 +ansible==2.6.2 anyjson==0.3.3 apache-libcloud==2.3.0 argparse==1.4.0 -astroid==1.6.2 -awscli==1.14.68 -babel==2.5.3 +astroid==2.0.4 +awscli==1.15.76 +babel==2.6.0 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 -beautifulsoup4==4.6.0 -billiard==3.5.0.3 -blessings==1.6.1 +beautifulsoup4==4.6.3 +billiard==3.5.0.4 +blessings==1.7 blinker==1.4 -boto==2.48.0 -botocore==1.9.21 +boto==2.49.0 +botocore==1.10.75 bottle==0.12.13 -celery==4.1.0 -certifi==2018.1.18 +celery==4.2.1 +certifi==2018.8.13 cffi==1.11.5 chardet==3.0.4 click==6.7 -cliff==2.11.0 -cmd2==0.8.2 +cliff==2.13.0 +cmd2==0.9.3 colorama==0.3.9 configobj==5.0.6 cov-core==1.15.0 @@ -32,164 +32,164 @@ coveralls==1.3.0 crcmod==1.7 cryptography==2.3 cssselect==1.0.3 -cython==0.28.1 -decorator==4.2.1 +cython==0.28.5 +decorator==4.3.0 django-celery==3.2.2 django-debug-toolbar==1.9.1 -django-extensions==2.0.6 -django==2.0.3 +django-extensions==2.1.0 +django==2.1 django_compress==1.0.1 -djangorestframework==3.7.7 +djangorestframework==3.8.2 docker-py==1.10.6 docopt==0.6.2 docutils==0.14 ecdsa==0.13 -elasticsearch==6.2.0 +elasticsearch==6.3.1 enum34==1.1.6 -eventlet==0.22.1 +eventlet==0.24.1 extras==1.0.0 -fabric==1.14.0 +fabric==2.3.1 fixtures==3.0.0 flake8==3.5.0 -flask==0.12.2 +flask==1.0.2 funcsigs==1.0.2 -gevent==1.2.2 -google-api-python-client==1.6.6 -greenlet==0.4.13 -gunicorn==19.7.1 +gevent==1.3.5 +google-api-python-client==1.7.4 +greenlet==0.4.14 +gunicorn==19.9.0 hiredis==0.2.0 honcho==1.0.1 html5lib==1.0.1 httplib2==0.11.3 -idna==2.6 -ipaddress==1.0.19 -ipython==6.2.1 +idna==2.7 +ipaddress==1.0.22 +ipython==6.5.0 iso8601==0.1.12 isodate==0.6.0 itsdangerous==0.24 jinja2==2.10 jmespath==0.9.3 jsonschema==2.6.0 -kombu==4.1.0 +kombu==4.2.1 linecache2==1.0.0 -logilab-common==1.4.1 -lxml==4.2.1 +logilab-common==1.4.2 +lxml==4.2.4 mako==1.0.7 manifestparser==1.1 markdown==2.6.11 markupsafe==1.0 -matplotlib==2.2.2 +matplotlib==2.2.3 mccabe==0.6.1 meld3==1.0.2 mock==2.0.0 mozcrash==1.0 -mozdevice==0.52 +mozdevice==1.0.0 mozfile==1.2 mozinfo==0.10 -mozlog==3.7 +mozlog==3.8 moznetwork==0.27 mozprocess==0.26 msgpack-python==0.5.6 -ndg-httpsclient==0.4.4 +ndg-httpsclient==0.5.1 netaddr==0.7.19 -netifaces==0.10.6 -newrelic==3.0.0.89 +netifaces==0.10.7 +newrelic==4.2.0.100 nose==1.3.7 -numpy==1.14.2 +numpy==1.15.0 oauth2==1.9.0.post1 oauth2client==4.1.2 -oauthlib==2.0.7 +oauthlib==2.1.0 ordereddict==1.1 -oslo.config==6.0.0 -pandas==0.22.0 +oslo.config==6.4.0 +pandas==0.23.4 paramiko==2.4.1 passlib==1.7.1 paste==2.0.3 pastedeploy==1.5.2 pastescript==2.0.2 -pbr==4.0.0 +pbr==4.2.0 pep8==1.7.1 -pexpect==4.4.0 -pika==0.11.2 -pillow==5.0.0 -pip==9.0.3 -prettytable -protobuf==3.5.2.post1 -psutil==5.4.3 -psycopg2==2.7.4 -py==1.5.3 -pyasn1-modules==0.2.1 -pyasn1==0.4.2 +pexpect==4.6.0 +pika==0.12.0 +pillow==5.2.0 +pip==18.0 +prettytable==7 +protobuf==3.6.0 +psutil==5.4.6 +psycopg2==2.7.5 +py==1.5.4 +pyasn1-modules==0.2.2 +pyasn1==0.4.4 pycparser==2.18 pycrypto==2.6.1 -pyflakes==1.6.0 +pyflakes==2.0.0 pygments==2.2.0 -pyjwt==1.6.1 +pyjwt==1.6.4 pylibmc==1.5.2 -pylint==1.8.3 -pymongo==3.6.1 -pymysql==0.8.0 -pyopenssl==17.5.0 +pylint==2.1.1 +pymongo==3.7.1 +pymysql==0.9.2 +pyopenssl==18.0.0 pyparsing==2.2.0 -pyramid==1.9.1 +pyramid==1.9.2 pystache==0.5.4 pytest-cov==2.5.1 -pytest==3.5.0 +pytest==3.7.1 python-daemon==2.1.2 -python-dateutil==2.7.2 +python-dateutil==2.7.3 python-gflags==3.1.2 -python-keystoneclient==3.15.0 +python-keystoneclient==3.17.0 python-memcached==1.59 python-mimeparse==1.6.0 -python-novaclient==10.1.0 -python-subunit==1.2.0 -python-swiftclient==3.5.0 -pytz==2018.3 -pyyaml==3.12 -pyzmq==17.0.0 -raven==6.6.0 +python-novaclient==11.0.0 +python-subunit==1.3.0 +python-swiftclient==3.6.0 +pytz==2018.5 +pyyaml==3.13 +pyzmq==17.1.2 +raven==6.9.0 redis==2.10.6 repoze.lru==0.7 -requests-oauthlib==0.8.0 -requests==2.18.4 +requests-oauthlib==1.0.0 +requests==2.19.1 retrying==1.3.3 rsa==3.4.2 -scipy==1.0.1 -selenium==3.11.0 +scipy==1.1.0 +selenium==3.14.0 setuptools-git==1.2 -setuptools==39.0.1 +setuptools==40.0.0 sh==1.12.14 -simplejson==3.13.2 +simplejson==3.16.0 six==1.11.0 snowballstemmer==1.2.1 south==1.0.2 -sphinx==1.7.2 +sphinx==1.7.6 sqlalchemy-migrate==0.11.0 -sqlalchemy==1.2.6 +sqlalchemy==1.2.10 sqlparse==0.2.4 statsd==3.2.2 -stevedore==1.28.0 +stevedore==1.29.0 testrepository==0.0.20 testtools==2.3.0 thrift==0.11.0 -tornado==5.0.1 -tox==2.9.1 -twisted==17.9.0 +tornado==5.1 +tox==3.2.1 +twisted==18.7.0 ujson==1.35 unidecode==1.0.22 unittest2==1.1.0 uritemplate==3.0.0 -urllib3==1.22 -uwsgi==2.0.17 +urllib3==1.23 +uwsgi==2.0.17.1 versiontools==1.9.1 -virtualenv==15.2.0 +virtualenv==16.0.0 waitress==1.1.0 warlock==1.3.0 -webob==1.7.4 -websocket-client==0.47.0 -webtest==2.0.29 +webob==1.8.2 +websocket-client==0.48.0 +webtest==2.0.30 werkzeug==0.14.1 -wheel==0.31.0 +wheel==0.31.1 xlrd==1.1.0 -zc.buildout==2.11.2 -zope.interface==4.4.3 +zc.buildout==2.12.1 +zope.interface==4.5.0 From 721201e707efbba87005c974e6b8cbabf1e17010 Mon Sep 17 00:00:00 2001 From: David Bendory Date: Sun, 19 Aug 2018 16:21:41 -0400 Subject: [PATCH 281/362] Container Builder is now Cloud Build. --- RELEASING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/RELEASING.md b/RELEASING.md index 2dcd806d..f886a542 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -36,7 +36,7 @@ follows: b. `gcloud auth activate-service-account` is performed c. `gcloud config set project` is performed 3. The script invokes `build.sh` in this repository -4. `build.sh` invokes Google Container Builder with the `cloudbuild-*.yaml` +4. `build.sh` invokes Google Cloud Build with the `cloudbuild-*.yaml` config files. ## Building interpreters @@ -65,7 +65,7 @@ Debian or Ubuntu-like). ## Building locally To build this repository using local Docker commands instead of the Google -Container Builder service, add the `--local` flag as shown: +Cloud Build service, add the `--local` flag as shown: ``` shell ./build.sh --local From 56c9654a17f0662acffe4fb06a329cbce16e1829 Mon Sep 17 00:00:00 2001 From: David Bendory Date: Sun, 19 Aug 2018 16:25:27 -0400 Subject: [PATCH 282/362] Container Builder is now Cloud Build. --- scripts/local_cloudbuild.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/local_cloudbuild.py b/scripts/local_cloudbuild.py index f80e0f4b..5c23a1bc 100755 --- a/scripts/local_cloudbuild.py +++ b/scripts/local_cloudbuild.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Emulate the Google Container Builder locally. +"""Emulate the Google Cloud Build locally. The input is a local cloudbuild.yaml file. This is translated into a series of commands for the locally installed Docker daemon. These @@ -48,8 +48,8 @@ # Exclude non-printable control characters (including newlines) PRINTABLE_REGEX = re.compile(r"""^[^\x00-\x1f]*$""") -# Container Builder substitutions -# https://cloud.google.com/container-builder/docs/api/build-requests#substitutions +# Cloud Build substitutions +# https://cloud.google.com/cloud-build/docs/api/build-requests#substitutions SUBSTITUTION_REGEX = re.compile(r"""(?x) [$] # Dollar sign ( From 8369dc90a267a7fa933056393322b976f4d44419 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Mon, 27 Aug 2018 00:50:51 -0700 Subject: [PATCH 283/362] Auto-update dependencies. --- runtime-image/resources/requirements.txt | 2 +- scripts/requirements-test.txt | 2 +- tests/python2-libraries/requirements.txt | 40 ++++++++++++------------ tests/python3-libraries/requirements.txt | 40 ++++++++++++------------ 4 files changed, 42 insertions(+), 42 deletions(-) diff --git a/runtime-image/resources/requirements.txt b/runtime-image/resources/requirements.txt index 44c624d8..bc783013 100644 --- a/runtime-image/resources/requirements.txt +++ b/runtime-image/resources/requirements.txt @@ -1,3 +1,3 @@ pip==18.0 -setuptools==40.0.0 +setuptools==40.2.0 wheel==0.31.1 diff --git a/scripts/requirements-test.txt b/scripts/requirements-test.txt index df909d47..c8e698da 100644 --- a/scripts/requirements-test.txt +++ b/scripts/requirements-test.txt @@ -1,4 +1,4 @@ flask==1.0.2 -pytest==3.7.1 +pytest==3.7.3 pytest-cov==2.5.1 pyyaml==3.13 diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 6daad0ab..a319137a 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -1,12 +1,12 @@ alembic==1.0.0 amqp==2.3.2 amqplib==1.0.2 -ansible==2.6.2 +ansible==2.6.3 anyjson==0.3.3 apache-libcloud==2.3.0 argparse==1.4.0 astroid==2.0.4 -awscli==1.15.76 +awscli==1.16.1 babel==2.6.0 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 @@ -16,29 +16,29 @@ billiard==3.5.0.4 blessings==1.7 blinker==1.4 boto==2.49.0 -botocore==1.10.75 +botocore==1.11.1 bottle==0.12.13 carbon<1.1.1 celery==4.2.1 -certifi==2018.8.13 +certifi==2018.8.24 cffi==1.11.5 chardet==3.0.4 click==6.7 cliff==2.13.0 -cmd2==0.9.3 +cmd2==0.9.4 colorama==0.3.9 configobj==5.0.6 cov-core==1.15.0 coverage==4.5.1 -coveralls==1.3.0 +coveralls==1.4.0 crcmod==1.7 -cryptography==2.3 +cryptography==2.3.1 cssselect==1.0.3 cython==0.28.5 decorator==4.3.0 django-celery==3.2.2 django-debug-toolbar==1.9.1 -django-extensions==2.1.0 +django-extensions==2.1.1 django<2.0 django_compress==1.0.1 djangorestframework==3.8.2 @@ -57,7 +57,7 @@ flask==1.0.2 funcsigs==1.0.2 functools32==3.2.3.post2 futures==3.2.0 -gevent==1.3.5 +gevent==1.3.6 google-api-python-client==1.7.4 graphite-web==1.1.3 greenlet==0.4.14 @@ -88,7 +88,7 @@ mccabe==0.6.1 meld3==1.0.2 mock==2.0.0 mozcrash==1.0 -mozdevice==1.0.0 +mozdevice==1.0.1 mozfile==1.2 mozinfo==0.10 mozlog==3.8 @@ -103,7 +103,7 @@ netaddr==0.7.19 netifaces==0.10.7 newrelic==4.2.0.100 nose==1.3.7 -numpy==1.15.0 +numpy==1.15.1 oauth2==1.9.0.post1 oauth2client==4.1.2 oauthlib==2.1.0 @@ -122,8 +122,8 @@ pika==0.12.0 pillow==5.2.0 pip==18.0 prettytable==7 -protobuf==3.6.0 -psutil==5.4.6 +protobuf==3.6.1 +psutil==5.4.7 psycopg2==2.7.5 py==1.5.4 pyasn1-modules==0.2.2 @@ -143,9 +143,9 @@ pyparsing==2.2.0 pyramid==1.9.2 pystache==0.5.4 pytest-cov==2.5.1 -pytest==3.7.1 +pytest==3.7.3 python-cjson==1.2.1 -python-daemon==2.1.2 +python-daemon==2.2.0 python-dateutil==2.7.3 python-gflags==3.1.2 python-keystoneclient==3.17.0 @@ -167,17 +167,17 @@ rsa==3.4.2 scipy==1.1.0 selenium==3.14.0 setuptools-git==1.2 -setuptools==40.0.0 +setuptools==40.2.0 sh==1.12.14 simplejson==3.16.0 six==1.11.0 snowballstemmer==1.2.1 south==1.0.2 -sphinx==1.7.6 +sphinx==1.7.7 sqlalchemy-migrate==0.11.0 -sqlalchemy==1.2.10 +sqlalchemy==1.2.11 sqlparse==0.2.4 -statsd==3.2.2 +statsd==3.3.0 stevedore==1.29.0 suds==0.4 supervisor==3.3.4 @@ -198,7 +198,7 @@ virtualenv==16.0.0 waitress==1.1.0 warlock==1.3.0 webob==1.8.2 -websocket-client==0.48.0 +websocket-client==0.51.0 webtest==2.0.30 werkzeug==0.14.1 wheel==0.31.1 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 7b839959..f9f1b8f7 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -1,12 +1,12 @@ alembic==1.0.0 amqp==2.3.2 amqplib==1.0.2 -ansible==2.6.2 +ansible==2.6.3 anyjson==0.3.3 apache-libcloud==2.3.0 argparse==1.4.0 astroid==2.0.4 -awscli==1.15.76 +awscli==1.16.1 babel==2.6.0 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 @@ -15,28 +15,28 @@ billiard==3.5.0.4 blessings==1.7 blinker==1.4 boto==2.49.0 -botocore==1.10.75 +botocore==1.11.1 bottle==0.12.13 celery==4.2.1 -certifi==2018.8.13 +certifi==2018.8.24 cffi==1.11.5 chardet==3.0.4 click==6.7 cliff==2.13.0 -cmd2==0.9.3 +cmd2==0.9.4 colorama==0.3.9 configobj==5.0.6 cov-core==1.15.0 coverage==4.5.1 -coveralls==1.3.0 +coveralls==1.4.0 crcmod==1.7 -cryptography==2.3 +cryptography==2.3.1 cssselect==1.0.3 cython==0.28.5 decorator==4.3.0 django-celery==3.2.2 django-debug-toolbar==1.9.1 -django-extensions==2.1.0 +django-extensions==2.1.1 django==2.1 django_compress==1.0.1 djangorestframework==3.8.2 @@ -53,7 +53,7 @@ fixtures==3.0.0 flake8==3.5.0 flask==1.0.2 funcsigs==1.0.2 -gevent==1.3.5 +gevent==1.3.6 google-api-python-client==1.7.4 greenlet==0.4.14 gunicorn==19.9.0 @@ -83,7 +83,7 @@ mccabe==0.6.1 meld3==1.0.2 mock==2.0.0 mozcrash==1.0 -mozdevice==1.0.0 +mozdevice==1.0.1 mozfile==1.2 mozinfo==0.10 mozlog==3.8 @@ -95,7 +95,7 @@ netaddr==0.7.19 netifaces==0.10.7 newrelic==4.2.0.100 nose==1.3.7 -numpy==1.15.0 +numpy==1.15.1 oauth2==1.9.0.post1 oauth2client==4.1.2 oauthlib==2.1.0 @@ -114,8 +114,8 @@ pika==0.12.0 pillow==5.2.0 pip==18.0 prettytable==7 -protobuf==3.6.0 -psutil==5.4.6 +protobuf==3.6.1 +psutil==5.4.7 psycopg2==2.7.5 py==1.5.4 pyasn1-modules==0.2.2 @@ -134,8 +134,8 @@ pyparsing==2.2.0 pyramid==1.9.2 pystache==0.5.4 pytest-cov==2.5.1 -pytest==3.7.1 -python-daemon==2.1.2 +pytest==3.7.3 +python-daemon==2.2.0 python-dateutil==2.7.3 python-gflags==3.1.2 python-keystoneclient==3.17.0 @@ -157,17 +157,17 @@ rsa==3.4.2 scipy==1.1.0 selenium==3.14.0 setuptools-git==1.2 -setuptools==40.0.0 +setuptools==40.2.0 sh==1.12.14 simplejson==3.16.0 six==1.11.0 snowballstemmer==1.2.1 south==1.0.2 -sphinx==1.7.6 +sphinx==1.7.7 sqlalchemy-migrate==0.11.0 -sqlalchemy==1.2.10 +sqlalchemy==1.2.11 sqlparse==0.2.4 -statsd==3.2.2 +statsd==3.3.0 stevedore==1.29.0 testrepository==0.0.20 testtools==2.3.0 @@ -186,7 +186,7 @@ virtualenv==16.0.0 waitress==1.1.0 warlock==1.3.0 webob==1.8.2 -websocket-client==0.48.0 +websocket-client==0.51.0 webtest==2.0.30 werkzeug==0.14.1 wheel==0.31.1 From a4df99416aa22b571fb6f797e64d68c821bb6131 Mon Sep 17 00:00:00 2001 From: dlorenc Date: Mon, 1 Oct 2018 08:08:58 -0700 Subject: [PATCH 284/362] Pin pip to 9.0.3. Newer versions conflict with the OS installed package. --- runtime-image/resources/requirements.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/runtime-image/resources/requirements.txt b/runtime-image/resources/requirements.txt index bc783013..298e5f41 100644 --- a/runtime-image/resources/requirements.txt +++ b/runtime-image/resources/requirements.txt @@ -1,3 +1,5 @@ -pip==18.0 +# Do not update pip, it conflicts with the OS system installed version. +# https://github.com/pypa/pip/issues/5240 +pip==9.0.3 setuptools==40.2.0 wheel==0.31.1 From 7eb071f7f7526815fc4b90a5ed0225f1247eef84 Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Mon, 1 Oct 2018 14:36:10 -0700 Subject: [PATCH 285/362] fix library versions for tests (#202) --- tests/python2-libraries/requirements.txt | 8 ++++---- tests/python3-libraries/requirements.txt | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index a319137a..865cd1aa 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -5,7 +5,7 @@ ansible==2.6.3 anyjson==0.3.3 apache-libcloud==2.3.0 argparse==1.4.0 -astroid==2.0.4 +astroid==1.6.5 awscli==1.16.1 babel==2.6.0 backports.ssl_match_hostname==3.5.0.1 @@ -25,7 +25,7 @@ cffi==1.11.5 chardet==3.0.4 click==6.7 cliff==2.13.0 -cmd2==0.9.4 +cmd2==0.8.9 colorama==0.3.9 configobj==5.0.6 cov-core==1.15.0 @@ -121,7 +121,7 @@ pexpect==4.6.0 pika==0.12.0 pillow==5.2.0 pip==18.0 -prettytable==7 +prettytable==0.7.2 protobuf==3.6.1 psutil==5.4.7 psycopg2==2.7.5 @@ -135,7 +135,7 @@ pyflakes==2.0.0 pygments==2.2.0 pyjwt==1.6.4 pylibmc==1.5.2 -pylint==2.1.1 +pylint==1.9.3 pymongo==3.7.1 pymysql==0.9.2 pyopenssl==18.0.0 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index f9f1b8f7..3b283d1d 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -113,7 +113,7 @@ pexpect==4.6.0 pika==0.12.0 pillow==5.2.0 pip==18.0 -prettytable==7 +prettytable==0.7.2 protobuf==3.6.1 psutil==5.4.7 psycopg2==2.7.5 From 5bc7a6e49e591cb4771dbd9acdeddb7c3bc41a5d Mon Sep 17 00:00:00 2001 From: dlorenc Date: Fri, 2 Nov 2018 14:43:11 -0700 Subject: [PATCH 286/362] Bump Python 3.6.6 to 3.6.7. --- .../scripts/build-python-3.6.sh | 12 ++++++------ tests/no-virtualenv/no-virtualenv.yaml | 2 +- tests/virtualenv/virtualenv_python36.yaml | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/python-interpreter-builder/scripts/build-python-3.6.sh b/python-interpreter-builder/scripts/build-python-3.6.sh index ef192e1e..b3896e89 100755 --- a/python-interpreter-builder/scripts/build-python-3.6.sh +++ b/python-interpreter-builder/scripts/build-python-3.6.sh @@ -6,14 +6,14 @@ set -x # Get the source mkdir -p /opt/sources cd /opt/sources -wget --no-verbose https://www.python.org/ftp/python/3.6.6/Python-3.6.6.tgz +wget --no-verbose https://www.python.org/ftp/python/3.6.7/Python-3.6.7.tgz # SHA-256 generated via `shasum -a 256 [file]` shasum --check < Date: Fri, 2 Nov 2018 14:36:55 -0700 Subject: [PATCH 287/362] Add support for Python 3.7. --- cloudbuild_interpreters.yaml | 6 +- nox.py | 2 +- .../scripts/build-python-3.7.sh | 153 ++++++++++++++++++ runtime-image/Dockerfile.in | 4 + scripts/gen_dockerfile.py | 1 + 5 files changed, 164 insertions(+), 2 deletions(-) create mode 100755 python-interpreter-builder/scripts/build-python-3.7.sh diff --git a/cloudbuild_interpreters.yaml b/cloudbuild_interpreters.yaml index 7daecc57..c75e59ad 100644 --- a/cloudbuild_interpreters.yaml +++ b/cloudbuild_interpreters.yaml @@ -18,11 +18,15 @@ steps: args: ['/scripts/build-python-3.6.sh'] id: build-3.6 waitFor: ['interpreter-builder'] +- name: interpreter-builder + args: ['/scripts/build-python-3.7.sh'] + id: build-3.7 + waitFor: ['interpreter-builder'] # Upload them to tbe build-id location - name: gcr.io/cloud-builders/gsutil:latest args: ['cp', '/workspace/runtime-image/*.tar.gz', 'gs://python-interpreters/$BUILD_ID/'] - waitFor: ['build-3.4', 'build-3.5', 'build-3.6'] + waitFor: ['build-3.4', 'build-3.5', 'build-3.6', 'build-3.7'] # "Tag" this as latest - name: gcr.io/cloud-builders/gsutil:latest diff --git a/nox.py b/nox.py index 872eb88c..0ee6f443 100644 --- a/nox.py +++ b/nox.py @@ -57,7 +57,7 @@ def lint(session): @nox.session -@nox.parametrize('version', ['3.4', '3.5', '3.6']) +@nox.parametrize('version', ['3.4', '3.5', '3.6', '3.7']) def tests(session, version): session.interpreter = 'python' + version session.install('-r', 'scripts/requirements-test.txt') diff --git a/python-interpreter-builder/scripts/build-python-3.7.sh b/python-interpreter-builder/scripts/build-python-3.7.sh new file mode 100755 index 00000000..2a77e279 --- /dev/null +++ b/python-interpreter-builder/scripts/build-python-3.7.sh @@ -0,0 +1,153 @@ +#!/bin/bash + +set -euo pipefail +set -x + +# Get the source +mkdir -p /opt/sources +cd /opt/sources +wget --no-verbose https://www.python.org/ftp/python/3.7.1/Python-3.7.1.tgz +# SHA-256 generated via `shasum -a 256 [file]` +shasum --check < Date: Wed, 14 Nov 2018 14:44:56 -0800 Subject: [PATCH 288/362] Fixing up CODEOWNERS (#207) --- CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CODEOWNERS b/CODEOWNERS index c08c2d11..f8b549f9 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1,4 +1,4 @@ # Code owners file. # This file controls who is tagged for review for any given pull request. -* @duggelz @liyanhui1228 @dlorenc +* @dlorenc @sharifelgamal @nkubala @tstromberg From 608b42af72537ce5614f1066db8b992a33af1199 Mon Sep 17 00:00:00 2001 From: dlorenc Date: Tue, 27 Nov 2018 11:40:09 -0800 Subject: [PATCH 289/362] Fix the Python 3.7 runtime release script. --- runtime-image/Dockerfile.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime-image/Dockerfile.in b/runtime-image/Dockerfile.in index b872a501..87d1aff2 100644 --- a/runtime-image/Dockerfile.in +++ b/runtime-image/Dockerfile.in @@ -40,7 +40,7 @@ RUN /usr/bin/pip install --upgrade -r /resources/requirements.txt && \ rm -f /opt/python3.5/bin/pip /opt/python3.5/bin/pip3 && \ /opt/python3.6/bin/pip3.6 install --upgrade -r /resources/requirements.txt && \ rm -f /opt/python3.6/bin/pip /opt/python3.6/bin/pip3 && \ - /opt/python3.6/bin/pip3.7 install --upgrade -r /resources/requirements.txt && \ + /opt/python3.7/bin/pip3.7 install --upgrade -r /resources/requirements.txt && \ rm -f /opt/python3.7/bin/pip /opt/python3.7/bin/pip3 && \ /usr/bin/pip install --upgrade -r /resources/requirements-virtualenv.txt From 7be6af7551dfc489535404ca5c09ca2ccc6d0641 Mon Sep 17 00:00:00 2001 From: dlorenc Date: Tue, 4 Dec 2018 09:10:55 -0800 Subject: [PATCH 290/362] Add Python 3.7 to the path. --- runtime-image/Dockerfile.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime-image/Dockerfile.in b/runtime-image/Dockerfile.in index 87d1aff2..64201966 100644 --- a/runtime-image/Dockerfile.in +++ b/runtime-image/Dockerfile.in @@ -27,7 +27,7 @@ RUN wget https://storage.googleapis.com/python-interpreters/latest/interpreter-3 rm interpreter-*.tar.gz # Add Google-built interpreters to the path -ENV PATH /opt/python3.6/bin:/opt/python3.5/bin:/opt/python3.4/bin:$PATH +ENV PATH /opt/python3.7/bin:/opt/python3.6/bin:/opt/python3.5/bin:/opt/python3.4/bin:$PATH RUN update-alternatives --install /usr/local/bin/python3 python3 /opt/python3.6/bin/python3.6 50 RUN update-alternatives --install /usr/local/bin/pip3 pip3 /opt/python3.6/bin/pip3.6 50 From 0e07aba8b47e3fa64d82e0a2a3ea943bb60027f4 Mon Sep 17 00:00:00 2001 From: Evan Anderson Date: Tue, 15 Jan 2019 11:44:16 -0800 Subject: [PATCH 291/362] Document how to use Dockerfile with python3 --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 2cc7d173..51413044 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,7 @@ command or entrypoint. For example: # Create a virtualenv for dependencies. This isolates these packages from # system-level packages. + # Use -p python3 or -p python3.7 to select python version. Default is version 2. RUN virtualenv /env # Setting these environment variables are the same as running From db4ae4216b1783ab6ddc96ce6500d7257eb37ef8 Mon Sep 17 00:00:00 2001 From: sharifelgamal Date: Wed, 2 Jan 2019 17:08:02 -0800 Subject: [PATCH 292/362] Updating python to 3.6.8 and 3.7.2 and defaulting to 3.7 --- .../scripts/build-python-3.6.sh | 12 ++--- .../scripts/build-python-3.7.sh | 12 ++--- scripts/gen_dockerfile.py | 2 +- scripts/gen_dockerfile_test.py | 5 +- .../testdata/hello_world_golden/Dockerfile | 4 +- tests/virtualenv/virtualenv_python36.yaml | 2 +- tests/virtualenv/virtualenv_python37.yaml | 54 +++++++++++++++++++ 7 files changed, 74 insertions(+), 17 deletions(-) create mode 100644 tests/virtualenv/virtualenv_python37.yaml diff --git a/python-interpreter-builder/scripts/build-python-3.6.sh b/python-interpreter-builder/scripts/build-python-3.6.sh index b3896e89..73912401 100755 --- a/python-interpreter-builder/scripts/build-python-3.6.sh +++ b/python-interpreter-builder/scripts/build-python-3.6.sh @@ -6,14 +6,14 @@ set -x # Get the source mkdir -p /opt/sources cd /opt/sources -wget --no-verbose https://www.python.org/ftp/python/3.6.7/Python-3.6.7.tgz +wget --no-verbose https://www.python.org/ftp/python/3.6.8/Python-3.6.8.tgz # SHA-256 generated via `shasum -a 256 [file]` shasum --check < Date: Wed, 2 Jan 2019 17:09:16 -0800 Subject: [PATCH 293/362] fixing 3.7 virtualenv tests --- tests/virtualenv/virtualenv_python37.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/virtualenv/virtualenv_python37.yaml b/tests/virtualenv/virtualenv_python37.yaml index f75ffdcd..457933f9 100644 --- a/tests/virtualenv/virtualenv_python37.yaml +++ b/tests/virtualenv/virtualenv_python37.yaml @@ -7,48 +7,48 @@ globalEnvVars: value: "/env/bin:$PATH" commandTests: - - name: "virtualenv36 python installation" + - name: "virtualenv37 python installation" setup: [["virtualenv", "-p", "python3.7", "/env"]] command: ["which", "python"] expectedOutput: ["/env/bin/python\n"] - - name: "virtualenv36 python3 installation" + - name: "virtualenv37 python3 installation" setup: [["virtualenv", "-p", "python3.7", "/env"]] command: ["which", "python3"] expectedOutput: ["/env/bin/python3\n"] - - name: "virtualenv36 python3.7 installation" + - name: "virtualenv37 python3.7 installation" setup: [["virtualenv", "-p", "python3.7", "/env"]] command: ["which", "python3.7"] expectedOutput: ["/env/bin/python3.7\n"] - - name: "virtualenv36 python version" + - name: "virtualenv37 python version" setup: [["virtualenv", "-p", "python3.7", "/env"]] command: ["python", "--version"] expectedOutput: ["Python 3.7.2\n"] - - name: "virtualenv36 pip installation" + - name: "virtualenv37 pip installation" setup: [["virtualenv", "-p", "python3.7", "/env"]] command: ["which", "pip"] expectedOutput: ["/env/bin/pip\n"] - - name: "virtualenv36 pip3 installation" + - name: "virtualenv37 pip3 installation" setup: [["virtualenv", "-p", "python3.7", "/env"]] command: ["which", "pip3"] expectedOutput: ["/env/bin/pip3\n"] - - name: "virtualenv36 gunicorn installation" + - name: "virtualenv37 gunicorn installation" setup: [["virtualenv", "-p", "python3.7", "/env"], ["pip", "install", "gunicorn"]] command: ["which", "gunicorn"] expectedOutput: ["/env/bin/gunicorn"] - - name: "virtualenv36 flask installation" + - name: "virtualenv37 flask installation" setup: [["virtualenv", "-p", "python3.7", "/env"], ["pip", "install", "flask"]] command: ["python", "-c", "import flask; print(flask.__file__)"] expectedOutput: ["/env/lib/python3.7/site-packages/flask"] - - name: "virtualenv36 test.support availability" + - name: "virtualenv37 test.support availability" setup: [["virtualenv", "-p", "python3.7", "/env"]] command: ["python", "-c", "\"from test import pystone, regrtest, support\""] From a90395dfeebc6b095eaa59f2c38eb392c471487d Mon Sep 17 00:00:00 2001 From: sharifelgamal Date: Fri, 4 Jan 2019 16:35:38 -0800 Subject: [PATCH 294/362] fixing up defaulting to 3.7 --- builder/gen-dockerfile/Dockerfile.in | 4 ++-- cloudbuild_test.yaml | 7 +++++++ runtime-image/Dockerfile.in | 4 ++-- tests/no-virtualenv/no-virtualenv.yaml | 2 +- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/builder/gen-dockerfile/Dockerfile.in b/builder/gen-dockerfile/Dockerfile.in index 4f6447eb..627151bf 100644 --- a/builder/gen-dockerfile/Dockerfile.in +++ b/builder/gen-dockerfile/Dockerfile.in @@ -1,6 +1,6 @@ FROM ${STAGING_IMAGE} -LABEL python_version=python3.6 -RUN virtualenv --no-download /env -p python3.6 +LABEL python_version=python3.7 +RUN virtualenv --no-download /env -p python3.7 # Set virtualenv environment variables. This is equivalent to running # source /env/bin/activate diff --git a/cloudbuild_test.yaml b/cloudbuild_test.yaml index 416c5204..ad674026 100644 --- a/cloudbuild_test.yaml +++ b/cloudbuild_test.yaml @@ -43,6 +43,13 @@ steps: '/workspace/tests/virtualenv/virtualenv_python36.yaml', ] waitFor: ['runtime'] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 + args: [ + '-test.v', + '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', + '/workspace/tests/virtualenv/virtualenv_python37.yaml', + ] + waitFor: ['runtime'] - name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 args: [ '-test.v', diff --git a/runtime-image/Dockerfile.in b/runtime-image/Dockerfile.in index 64201966..705a915b 100644 --- a/runtime-image/Dockerfile.in +++ b/runtime-image/Dockerfile.in @@ -19,7 +19,7 @@ ENV PYTHONUNBUFFERED 1 RUN wget https://storage.googleapis.com/python-interpreters/latest/interpreter-3.4.tar.gz && \ wget https://storage.googleapis.com/python-interpreters/latest/interpreter-3.5.tar.gz && \ wget https://storage.googleapis.com/python-interpreters/latest/interpreter-3.6.tar.gz && \ - wget https://storage.googleapis.com/python-interpreters/latest/interpreter-3.7.tar.gz && \ + wget https://storage.googleapis.com/python-interpreters/latest/interpreter-3.7.tar.gz && \ tar -xzf interpreter-3.4.tar.gz && \ tar -xzf interpreter-3.5.tar.gz && \ tar -xzf interpreter-3.6.tar.gz && \ @@ -41,7 +41,7 @@ RUN /usr/bin/pip install --upgrade -r /resources/requirements.txt && \ /opt/python3.6/bin/pip3.6 install --upgrade -r /resources/requirements.txt && \ rm -f /opt/python3.6/bin/pip /opt/python3.6/bin/pip3 && \ /opt/python3.7/bin/pip3.7 install --upgrade -r /resources/requirements.txt && \ - rm -f /opt/python3.7/bin/pip /opt/python3.7/bin/pip3 && \ + rm -f /opt/python3.7/bin/pip /opt/python3.7/bin/pip3 && \ /usr/bin/pip install --upgrade -r /resources/requirements-virtualenv.txt # Setup the app working directory diff --git a/tests/no-virtualenv/no-virtualenv.yaml b/tests/no-virtualenv/no-virtualenv.yaml index 20c09501..c8350e4e 100644 --- a/tests/no-virtualenv/no-virtualenv.yaml +++ b/tests/no-virtualenv/no-virtualenv.yaml @@ -43,7 +43,7 @@ commandTests: expectedOutput: ["/usr/local/bin/python3\n"] - name: "default python3 version" command: ["python3", "--version"] - expectedOutput: ["Python 3.6.7\n"] + expectedOutput: ["Python 3.7.2\n"] - name: "default pip3 installation" command: ["which", "pip3"] expectedOutput: ["/usr/local/bin/pip3\n"] From 407695209e7397cf11b12f898e60a29a9529b328 Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Tue, 22 Jan 2019 09:30:15 -0800 Subject: [PATCH 295/362] Updating build's base image from debian8 to ubuntu_16_0_4 to address SSL issues with 3.6.8 and 3.7.2 build Updating Dockerfile to update_alternatives to 3.7.2 Updating Dockerfile to separate installs so python install errors are easier to differentiate --- build.sh | 8 ++++---- runtime-image/Dockerfile.in | 24 ++++++++++++------------ 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/build.sh b/build.sh index bdc4f658..63afa1ab 100755 --- a/build.sh +++ b/build.sh @@ -24,7 +24,7 @@ test=0 # Should run standard test suite? local=0 # Should run using local Docker daemon instead of GCR? -os_base=debian8 # Which operating system base to use +os_base=ubuntu16 # Which operating system base to use interpreter=0 # Should build interpreters instead of images # Note that $gcloud_cmd has spaces in it @@ -154,10 +154,10 @@ if [ "${local}" -eq 1 ]; then fi # Pick OS image to use as base -if [ "${os_base}" == "ubuntu16" ]; then - export OS_BASE_IMAGE="gcr.io/gcp-runtimes/ubuntu_16_0_4:latest" -else +if [ "${os_base}" == "debian8" ]; then export OS_BASE_IMAGE="gcr.io/google-appengine/debian8:latest" +else + export OS_BASE_IMAGE="gcr.io/gcp-runtimes/ubuntu_16_0_4:latest" fi export STAGING_IMAGE="${DOCKER_NAMESPACE}/python:${TAG}" echo "Using base image name ${STAGING_IMAGE}" diff --git a/runtime-image/Dockerfile.in b/runtime-image/Dockerfile.in index 705a915b..84cb5267 100644 --- a/runtime-image/Dockerfile.in +++ b/runtime-image/Dockerfile.in @@ -28,21 +28,21 @@ RUN wget https://storage.googleapis.com/python-interpreters/latest/interpreter-3 # Add Google-built interpreters to the path ENV PATH /opt/python3.7/bin:/opt/python3.6/bin:/opt/python3.5/bin:/opt/python3.4/bin:$PATH -RUN update-alternatives --install /usr/local/bin/python3 python3 /opt/python3.6/bin/python3.6 50 -RUN update-alternatives --install /usr/local/bin/pip3 pip3 /opt/python3.6/bin/pip3.6 50 +RUN update-alternatives --install /usr/local/bin/python3 python3 /opt/python3.7/bin/python3.7 50 +RUN update-alternatives --install /usr/local/bin/pip3 pip3 /opt/python3.7/bin/pip3.7 50 # Upgrade pip (debian package version tends to run a few version behind) and # install virtualenv system-wide. -RUN /usr/bin/pip install --upgrade -r /resources/requirements.txt && \ - /opt/python3.4/bin/pip3.4 install --upgrade -r /resources/requirements.txt && \ - rm -f /opt/python3.4/bin/pip /opt/python3.4/bin/pip3 && \ - /opt/python3.5/bin/pip3.5 install --upgrade -r /resources/requirements.txt && \ - rm -f /opt/python3.5/bin/pip /opt/python3.5/bin/pip3 && \ - /opt/python3.6/bin/pip3.6 install --upgrade -r /resources/requirements.txt && \ - rm -f /opt/python3.6/bin/pip /opt/python3.6/bin/pip3 && \ - /opt/python3.7/bin/pip3.7 install --upgrade -r /resources/requirements.txt && \ - rm -f /opt/python3.7/bin/pip /opt/python3.7/bin/pip3 && \ - /usr/bin/pip install --upgrade -r /resources/requirements-virtualenv.txt +RUN /usr/bin/pip install --upgrade -r /resources/requirements.txt +RUN /opt/python3.4/bin/pip3.4 install --upgrade -r /resources/requirements.txt +RUN rm -f /opt/python3.4/bin/pip /opt/python3.4/bin/pip3 +RUN /opt/python3.5/bin/pip3.5 install --upgrade -r /resources/requirements.txt +RUN rm -f /opt/python3.5/bin/pip /opt/python3.5/bin/pip3 +#RUN /opt/python3.6/bin/pip3.6 install --upgrade -r /resources/requirements.txt +#RUN rm -f /opt/python3.6/bin/pip /opt/python3.6/bin/pip3 +RUN /opt/python3.7/bin/pip3.7 install --upgrade -r /resources/requirements.txt +RUN rm -f /opt/python3.7/bin/pip /opt/python3.7/bin/pip3 +RUN /usr/bin/pip install --upgrade -r /resources/requirements-virtualenv.txt # Setup the app working directory RUN ln -s /home/vmagent/app /app From 52a469c2fe4f6bfb60fb36bad72e57fd184b8e76 Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Tue, 22 Jan 2019 10:28:05 -0800 Subject: [PATCH 296/362] Updating dockerfile to reduce RUN commands --- runtime-image/Dockerfile.in | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/runtime-image/Dockerfile.in b/runtime-image/Dockerfile.in index 84cb5267..dc842d9b 100644 --- a/runtime-image/Dockerfile.in +++ b/runtime-image/Dockerfile.in @@ -28,21 +28,21 @@ RUN wget https://storage.googleapis.com/python-interpreters/latest/interpreter-3 # Add Google-built interpreters to the path ENV PATH /opt/python3.7/bin:/opt/python3.6/bin:/opt/python3.5/bin:/opt/python3.4/bin:$PATH -RUN update-alternatives --install /usr/local/bin/python3 python3 /opt/python3.7/bin/python3.7 50 -RUN update-alternatives --install /usr/local/bin/pip3 pip3 /opt/python3.7/bin/pip3.7 50 +RUN update-alternatives --install /usr/local/bin/python3 python3 /opt/python3.7/bin/python3.7 50 && \ + update-alternatives --install /usr/local/bin/pip3 pip3 /opt/python3.7/bin/pip3.7 50 # Upgrade pip (debian package version tends to run a few version behind) and # install virtualenv system-wide. -RUN /usr/bin/pip install --upgrade -r /resources/requirements.txt -RUN /opt/python3.4/bin/pip3.4 install --upgrade -r /resources/requirements.txt -RUN rm -f /opt/python3.4/bin/pip /opt/python3.4/bin/pip3 -RUN /opt/python3.5/bin/pip3.5 install --upgrade -r /resources/requirements.txt -RUN rm -f /opt/python3.5/bin/pip /opt/python3.5/bin/pip3 -#RUN /opt/python3.6/bin/pip3.6 install --upgrade -r /resources/requirements.txt -#RUN rm -f /opt/python3.6/bin/pip /opt/python3.6/bin/pip3 -RUN /opt/python3.7/bin/pip3.7 install --upgrade -r /resources/requirements.txt -RUN rm -f /opt/python3.7/bin/pip /opt/python3.7/bin/pip3 -RUN /usr/bin/pip install --upgrade -r /resources/requirements-virtualenv.txt +RUN /usr/bin/pip install --upgrade -r /resources/requirements.txt && \ + /opt/python3.4/bin/pip3.4 install --upgrade -r /resources/requirements.txt && \ + rm -f /opt/python3.4/bin/pip /opt/python3.4/bin/pip3 && \ + /opt/python3.5/bin/pip3.5 install --upgrade -r /resources/requirements.txt && \ + rm -f /opt/python3.5/bin/pip /opt/python3.5/bin/pip3 && \ + /opt/python3.6/bin/pip3.6 install --upgrade -r /resources/requirements.txt && \ + rm -f /opt/python3.6/bin/pip /opt/python3.6/bin/pip3 && \ + /opt/python3.7/bin/pip3.7 install --upgrade -r /resources/requirements.txt && \ + rm -f /opt/python3.7/bin/pip /opt/python3.7/bin/pip3 && \ + /usr/bin/pip install --upgrade -r /resources/requirements-virtualenv.txt # Setup the app working directory RUN ln -s /home/vmagent/app /app From 9701e57eb1deb40e98caa9ca1dc62d55154d69ba Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Tue, 22 Jan 2019 11:16:28 -0800 Subject: [PATCH 297/362] Update python to 3.6.8 and 3.7.2 defaulting to 3.7 - taketwo (#212) * Updating python to 3.6.8 and 3.7.2 and defaulting to 3.7 * fixing 3.7 virtualenv tests * fixing up defaulting to 3.7 * Updating build's base image from debian8 to ubuntu_16_0_4 to address SSL issues with 3.6.8 and 3.7.2 build Updating Dockerfile to update_alternatives to 3.7.2 Updating Dockerfile to separate installs so python install errors are easier to differentiate * Updating dockerfile to reduce RUN commands --- build.sh | 8 +-- builder/gen-dockerfile/Dockerfile.in | 4 +- cloudbuild_test.yaml | 7 +++ .../scripts/build-python-3.6.sh | 12 ++--- .../scripts/build-python-3.7.sh | 12 ++--- runtime-image/Dockerfile.in | 8 +-- scripts/gen_dockerfile.py | 2 +- scripts/gen_dockerfile_test.py | 5 +- .../testdata/hello_world_golden/Dockerfile | 4 +- tests/no-virtualenv/no-virtualenv.yaml | 2 +- tests/virtualenv/virtualenv_python36.yaml | 2 +- tests/virtualenv/virtualenv_python37.yaml | 54 +++++++++++++++++++ 12 files changed, 92 insertions(+), 28 deletions(-) create mode 100644 tests/virtualenv/virtualenv_python37.yaml diff --git a/build.sh b/build.sh index bdc4f658..63afa1ab 100755 --- a/build.sh +++ b/build.sh @@ -24,7 +24,7 @@ test=0 # Should run standard test suite? local=0 # Should run using local Docker daemon instead of GCR? -os_base=debian8 # Which operating system base to use +os_base=ubuntu16 # Which operating system base to use interpreter=0 # Should build interpreters instead of images # Note that $gcloud_cmd has spaces in it @@ -154,10 +154,10 @@ if [ "${local}" -eq 1 ]; then fi # Pick OS image to use as base -if [ "${os_base}" == "ubuntu16" ]; then - export OS_BASE_IMAGE="gcr.io/gcp-runtimes/ubuntu_16_0_4:latest" -else +if [ "${os_base}" == "debian8" ]; then export OS_BASE_IMAGE="gcr.io/google-appengine/debian8:latest" +else + export OS_BASE_IMAGE="gcr.io/gcp-runtimes/ubuntu_16_0_4:latest" fi export STAGING_IMAGE="${DOCKER_NAMESPACE}/python:${TAG}" echo "Using base image name ${STAGING_IMAGE}" diff --git a/builder/gen-dockerfile/Dockerfile.in b/builder/gen-dockerfile/Dockerfile.in index 4f6447eb..627151bf 100644 --- a/builder/gen-dockerfile/Dockerfile.in +++ b/builder/gen-dockerfile/Dockerfile.in @@ -1,6 +1,6 @@ FROM ${STAGING_IMAGE} -LABEL python_version=python3.6 -RUN virtualenv --no-download /env -p python3.6 +LABEL python_version=python3.7 +RUN virtualenv --no-download /env -p python3.7 # Set virtualenv environment variables. This is equivalent to running # source /env/bin/activate diff --git a/cloudbuild_test.yaml b/cloudbuild_test.yaml index 416c5204..ad674026 100644 --- a/cloudbuild_test.yaml +++ b/cloudbuild_test.yaml @@ -43,6 +43,13 @@ steps: '/workspace/tests/virtualenv/virtualenv_python36.yaml', ] waitFor: ['runtime'] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 + args: [ + '-test.v', + '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', + '/workspace/tests/virtualenv/virtualenv_python37.yaml', + ] + waitFor: ['runtime'] - name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 args: [ '-test.v', diff --git a/python-interpreter-builder/scripts/build-python-3.6.sh b/python-interpreter-builder/scripts/build-python-3.6.sh index b3896e89..73912401 100755 --- a/python-interpreter-builder/scripts/build-python-3.6.sh +++ b/python-interpreter-builder/scripts/build-python-3.6.sh @@ -6,14 +6,14 @@ set -x # Get the source mkdir -p /opt/sources cd /opt/sources -wget --no-verbose https://www.python.org/ftp/python/3.6.7/Python-3.6.7.tgz +wget --no-verbose https://www.python.org/ftp/python/3.6.8/Python-3.6.8.tgz # SHA-256 generated via `shasum -a 256 [file]` shasum --check < Date: Wed, 30 Jan 2019 11:40:26 -0800 Subject: [PATCH 298/362] Fixing doc to reflect correct base image --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 51413044..51e70cd8 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ for running applications on [Google Kubernetes Engine](https://cloud.google.com/kubernetes-engine), or any other Docker host. -This image is based on Debian Jessie and contains packages required to build +This image is based on Ubuntu Xenial and contains packages required to build most of the popular Python libraries. For more information about this runtime, see the [documentation](https://cloud.google.com/appengine/docs/flexible/python/runtime). From cbc6240d561e3727c42a252bf09caa3d89c5a0b0 Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Fri, 1 Feb 2019 16:08:12 -0800 Subject: [PATCH 299/362] Revert default python3 version to 3.6 (#215) --- builder/gen-dockerfile/Dockerfile.in | 4 ++-- scripts/gen_dockerfile.py | 2 +- scripts/gen_dockerfile_test.py | 2 +- scripts/testdata/hello_world_golden/Dockerfile | 4 ++-- tests/no-virtualenv/no-virtualenv.yaml | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/builder/gen-dockerfile/Dockerfile.in b/builder/gen-dockerfile/Dockerfile.in index 627151bf..4f6447eb 100644 --- a/builder/gen-dockerfile/Dockerfile.in +++ b/builder/gen-dockerfile/Dockerfile.in @@ -1,6 +1,6 @@ FROM ${STAGING_IMAGE} -LABEL python_version=python3.7 -RUN virtualenv --no-download /env -p python3.7 +LABEL python_version=python3.6 +RUN virtualenv --no-download /env -p python3.6 # Set virtualenv environment variables. This is equivalent to running # source /env/bin/activate diff --git a/scripts/gen_dockerfile.py b/scripts/gen_dockerfile.py index 852405d8..97da60f6 100755 --- a/scripts/gen_dockerfile.py +++ b/scripts/gen_dockerfile.py @@ -50,7 +50,7 @@ PYTHON_INTERPRETER_VERSION_MAP = { '': '', # == 2.7 '2': '', # == 2.7 - '3': '3.7', + '3': '3.6', '3.4': '3.4', '3.5': '3.5', '3.6': '3.6', diff --git a/scripts/gen_dockerfile_test.py b/scripts/gen_dockerfile_test.py index c78c6ad8..03eaa079 100755 --- a/scripts/gen_dockerfile_test.py +++ b/scripts/gen_dockerfile_test.py @@ -71,7 +71,7 @@ def compare_file(filename, dir1, dir2): 'dockerfile_python_version': '', }), ('runtime_config:\n python_version: 3', { - 'dockerfile_python_version': '3.7', + 'dockerfile_python_version': '3.6', }), ('runtime_config:\n python_version: 3.4', { 'dockerfile_python_version': '3.4', diff --git a/scripts/testdata/hello_world_golden/Dockerfile b/scripts/testdata/hello_world_golden/Dockerfile index 55eb8cec..10396399 100644 --- a/scripts/testdata/hello_world_golden/Dockerfile +++ b/scripts/testdata/hello_world_golden/Dockerfile @@ -1,6 +1,6 @@ FROM gcr.io/google-appengine/python -LABEL python_version=python3.7 -RUN virtualenv --no-download /env -p python3.7 +LABEL python_version=python3.6 +RUN virtualenv --no-download /env -p python3.6 # Set virtualenv environment variables. This is equivalent to running # source /env/bin/activate diff --git a/tests/no-virtualenv/no-virtualenv.yaml b/tests/no-virtualenv/no-virtualenv.yaml index c8350e4e..6a3b1e7d 100644 --- a/tests/no-virtualenv/no-virtualenv.yaml +++ b/tests/no-virtualenv/no-virtualenv.yaml @@ -43,7 +43,7 @@ commandTests: expectedOutput: ["/usr/local/bin/python3\n"] - name: "default python3 version" command: ["python3", "--version"] - expectedOutput: ["Python 3.7.2\n"] + expectedOutput: ["Python 3.6.8\n"] - name: "default pip3 installation" command: ["which", "pip3"] expectedOutput: ["/usr/local/bin/pip3\n"] From d628b583eb2f5684d02a441b99d08f8b7310f79a Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Mon, 4 Feb 2019 14:31:15 -0800 Subject: [PATCH 300/362] Make python 3.6 the default python 3 version (#216) --- runtime-image/Dockerfile.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runtime-image/Dockerfile.in b/runtime-image/Dockerfile.in index dc842d9b..79862620 100644 --- a/runtime-image/Dockerfile.in +++ b/runtime-image/Dockerfile.in @@ -28,8 +28,8 @@ RUN wget https://storage.googleapis.com/python-interpreters/latest/interpreter-3 # Add Google-built interpreters to the path ENV PATH /opt/python3.7/bin:/opt/python3.6/bin:/opt/python3.5/bin:/opt/python3.4/bin:$PATH -RUN update-alternatives --install /usr/local/bin/python3 python3 /opt/python3.7/bin/python3.7 50 && \ - update-alternatives --install /usr/local/bin/pip3 pip3 /opt/python3.7/bin/pip3.7 50 +RUN update-alternatives --install /usr/local/bin/python3 python3 /opt/python3.6/bin/python3.6 50 && \ + update-alternatives --install /usr/local/bin/pip3 pip3 /opt/python3.6/bin/pip3.6 50 # Upgrade pip (debian package version tends to run a few version behind) and # install virtualenv system-wide. From adacb0fd5342590935ce76a9c480cd9bbf89a8bf Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Wed, 6 Feb 2019 13:59:05 -0800 Subject: [PATCH 301/362] explicitly run apt-get upgrade to get latest versions of packages (#217) --- runtime-image/scripts/install-apt-packages.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/runtime-image/scripts/install-apt-packages.sh b/runtime-image/scripts/install-apt-packages.sh index 9d23cab7..bafba2d6 100755 --- a/runtime-image/scripts/install-apt-packages.sh +++ b/runtime-image/scripts/install-apt-packages.sh @@ -7,6 +7,8 @@ apt-get -q update xargs -a <(awk '/^\s*[^#]/' '/resources/apt-packages.txt') -r -- \ apt-get install --no-install-recommends -yq +apt-get upgrade -yq + # Remove unneeded files. apt-get clean rm /var/lib/apt/lists/*_* From 272f8d9ebd311b9c4466a252cc6a71223317c7f2 Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Wed, 6 Feb 2019 13:59:16 -0800 Subject: [PATCH 302/362] add donmccasland to CODEOWNERS (#218) --- CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CODEOWNERS b/CODEOWNERS index f8b549f9..ea26d5dc 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1,4 +1,4 @@ # Code owners file. # This file controls who is tagged for review for any given pull request. -* @dlorenc @sharifelgamal @nkubala @tstromberg +* @dlorenc @sharifelgamal @nkubala @tstromberg @donmccasland From 953a810d823e99c45c43b19a9c960c426c114051 Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Thu, 4 Apr 2019 13:54:19 -0700 Subject: [PATCH 303/362] Adding build scripts --- scripts/deploy_check.sh | 34 +++++++++++++++++++++ scripts/integration-test.sh | 61 +++++++++++++++++++++++++++++++++++++ scripts/release.sh | 18 +++++++++++ 3 files changed, 113 insertions(+) create mode 100644 scripts/deploy_check.sh create mode 100644 scripts/integration-test.sh create mode 100644 scripts/release.sh diff --git a/scripts/deploy_check.sh b/scripts/deploy_check.sh new file mode 100644 index 00000000..77fccb54 --- /dev/null +++ b/scripts/deploy_check.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +set -ex + +export KOKORO_GITHUB_DIR=${KOKORO_ROOT}/src/github +source ${KOKORO_GFILE_DIR}/kokoro/common.sh + +cd ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY} +if [ -n "${RUNTIME_SPEC}" -a -f app.yaml.in ]; then + sed "s|\${RUNTIME_SPEC}|${RUNTIME_SPEC}|" app.yaml.in > app.yaml +fi + +cd ${KOKORO_GFILE_DIR}/appengine/integration_tests + +sudo /usr/local/bin/pip install --upgrade -r requirements.txt + +if [ -f ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY}/requirements.txt ] +then + sudo /usr/local/bin/pip install --upgrade -r ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY}/requirements.txt +fi + +export DEPLOY_LATENCY_PROJECT='cloud-deploy-latency' + +skip_flag="" + +if [ "${SKIP_CUSTOM_LOGGING_TESTS}" = "true" -o "${SKIP_BUILDERS}" = "true" ]; then + skip_flag="$skip_flag --skip-builders" +fi + +if [ "${SKIP_XRT}" = "true" ]; then + skip_flag="$skip_flag --skip-xrt" +fi + +python deploy_check.py -d ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY} -l ${LANGUAGE} ${skip_flag} diff --git a/scripts/integration-test.sh b/scripts/integration-test.sh new file mode 100644 index 00000000..63137b5f --- /dev/null +++ b/scripts/integration-test.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +set -ex + +export KOKORO_GITHUB_DIR=${KOKORO_ROOT}/src/github +source ${KOKORO_GFILE_DIR}/kokoro/common.sh + +export GOOGLE_CLOUD_PROJECT=gcp-runtimes + +sudo /usr/local/bin/pip install --upgrade -r ${KOKORO_GFILE_DIR}/appengine/integration_tests/requirements.txt + +if [ -f ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY}/requirements.txt ] +then + sudo /usr/local/bin/pip install --upgrade -r ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY}/requirements.txt +fi + +export GOPATH=${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY} + +flags="" + +if [ -n "${STAGING_IMAGE}" ]; then + flags="$flags -i ${STAGING_IMAGE}" +fi + +if [ "${SKIP_STANDARD_LOGGING_TESTS}" = "true" ]; then + flags="$flags --skip-standard-logging-tests" +fi + +if [ "${SKIP_CUSTOM_LOGGING_TESTS}" = "true" ]; then + flags="$flags --skip-custom-logging-tests" +fi + +if [ "${SKIP_MONITORING_TESTS}" = "true" ]; then + flags="$flags --skip-monitoring-tests" +fi + +if [ "${SKIP_EXCEPTION_TESTS}" = "true" ]; then + flags="$flags --skip-exception-tests" +fi + +if [ "${SKIP_CUSTOM_TESTS}" = "true" ]; then + flags="$flags --skip-custom-tests" +fi + +if [ -n "${URL}" ]; then + flags="$flags --url ${URL}" +fi + +if [ -n "${BUILDER}" ]; then + flags="$flags --builder ${BUILDER}" + gcloud config set app/use_runtime_builders True + gcloud config set app/runtime_builders_root file://${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY} +fi + +if [ -n "${YAML}" ]; then + flags="$flags --yaml ${KOKORO_GITHUB_DIR}/${YAML}" +fi + + +chmod a+x ${KOKORO_GFILE_DIR}/appengine/integration_tests/testsuite/driver.py +${KOKORO_GFILE_DIR}/appengine/integration_tests/testsuite/driver.py -d ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY} ${flags} diff --git a/scripts/release.sh b/scripts/release.sh new file mode 100644 index 00000000..1857deaf --- /dev/null +++ b/scripts/release.sh @@ -0,0 +1,18 @@ +#!/bin/bash +set -euo pipefail +export KOKORO_GITHUB_DIR=${KOKORO_ROOT}/src/github +source ${KOKORO_GFILE_DIR}/kokoro/common.sh + +source "${KOKORO_PIPER_DIR}/google3/third_party/runtimes_common/kokoro/common.sh" + +cd ${KOKORO_GITHUB_DIR}/python-runtime + +if [ -z "${TAG:+set}" ]; then + export TAG=$(date +%Y-%m-%d-%H%M%S) +fi + +./build.sh $BUILD_FLAGS + +METADATA=$(pwd)/METADATA +cd ${KOKORO_GFILE_DIR}/kokoro +python note.py python -m ${METADATA} -t ${TAG} From a1339d625a87da959d7aa7ddfa35ccce421a1ef8 Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Mon, 8 Apr 2019 08:59:33 -0700 Subject: [PATCH 304/362] Fixing release script --- scripts/release.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/release.sh b/scripts/release.sh index 1857deaf..cd499823 100644 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -3,8 +3,6 @@ set -euo pipefail export KOKORO_GITHUB_DIR=${KOKORO_ROOT}/src/github source ${KOKORO_GFILE_DIR}/kokoro/common.sh -source "${KOKORO_PIPER_DIR}/google3/third_party/runtimes_common/kokoro/common.sh" - cd ${KOKORO_GITHUB_DIR}/python-runtime if [ -z "${TAG:+set}" ]; then From c6864c03b87842d891f362559a73e0181c2e0c69 Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Tue, 17 Sep 2019 15:38:46 -0700 Subject: [PATCH 305/362] Updating integration test pip requirement: gcloud-logging version 1.12.1 --- tests/integration/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index 4a462433..ac718c60 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -1,6 +1,6 @@ Flask==1.0.2 google-cloud-error-reporting==0.30.0 -google-cloud-logging==1.6.0 +google-cloud-logging==1.12.1 google-cloud-monitoring==0.30.1 gunicorn==19.9.0 requests==2.19.1 From 7fc2e28c8ce121cbfa899f7852a54e5795818571 Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Tue, 17 Sep 2019 15:52:28 -0700 Subject: [PATCH 306/362] Updating integration test pip requirements --- tests/integration/requirements.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index ac718c60..73a48913 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -1,8 +1,8 @@ -Flask==1.0.2 -google-cloud-error-reporting==0.30.0 +Flask==1.1.0 +google-cloud-error-reporting==0.32.1 google-cloud-logging==1.12.1 -google-cloud-monitoring==0.30.1 +google-cloud-monitoring==0.33.0 gunicorn==19.9.0 -requests==2.19.1 +requests==2.22.0 retrying==1.3.3 -six==1.11.0 +six==1.12.0 From 17a4a6c3cec29fb772e8280ac13d23a53d30d8a8 Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Wed, 18 Sep 2019 10:59:59 -0700 Subject: [PATCH 307/362] Adding protobuf requirement so kokoro will upgrade it --- tests/integration/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index 73a48913..ea714c20 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -6,3 +6,4 @@ gunicorn==19.9.0 requests==2.22.0 retrying==1.3.3 six==1.12.0 +protobuf>=3.6.0 From 87a309ee632ade17162d85211e4fa0164ab613af Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Thu, 5 Mar 2020 09:49:48 -0800 Subject: [PATCH 308/362] update python 3 to use python 3.7 --- runtime-image/Dockerfile.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runtime-image/Dockerfile.in b/runtime-image/Dockerfile.in index 79862620..dc842d9b 100644 --- a/runtime-image/Dockerfile.in +++ b/runtime-image/Dockerfile.in @@ -28,8 +28,8 @@ RUN wget https://storage.googleapis.com/python-interpreters/latest/interpreter-3 # Add Google-built interpreters to the path ENV PATH /opt/python3.7/bin:/opt/python3.6/bin:/opt/python3.5/bin:/opt/python3.4/bin:$PATH -RUN update-alternatives --install /usr/local/bin/python3 python3 /opt/python3.6/bin/python3.6 50 && \ - update-alternatives --install /usr/local/bin/pip3 pip3 /opt/python3.6/bin/pip3.6 50 +RUN update-alternatives --install /usr/local/bin/python3 python3 /opt/python3.7/bin/python3.7 50 && \ + update-alternatives --install /usr/local/bin/pip3 pip3 /opt/python3.7/bin/pip3.7 50 # Upgrade pip (debian package version tends to run a few version behind) and # install virtualenv system-wide. From fbc2908de36bb94bb00dc4a25c80a6113c1bf231 Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Fri, 27 Mar 2020 10:01:17 -0700 Subject: [PATCH 309/362] Updating python 3,7 to 3.7.7 --- .../scripts/build-python-3.7.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/python-interpreter-builder/scripts/build-python-3.7.sh b/python-interpreter-builder/scripts/build-python-3.7.sh index 69febc88..3b3bdae5 100755 --- a/python-interpreter-builder/scripts/build-python-3.7.sh +++ b/python-interpreter-builder/scripts/build-python-3.7.sh @@ -6,14 +6,14 @@ set -x # Get the source mkdir -p /opt/sources cd /opt/sources -wget --no-verbose https://www.python.org/ftp/python/3.7.2/Python-3.7.2.tgz +wget --no-verbose https://www.python.org/ftp/python/3.7.7/Python-3.7.7.tgz # SHA-256 generated via `shasum -a 256 [file]` shasum --check < Date: Fri, 27 Mar 2020 10:43:39 -0700 Subject: [PATCH 310/362] Remove nkubala from owners (#233) --- CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CODEOWNERS b/CODEOWNERS index ea26d5dc..5deacc6c 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1,4 +1,4 @@ # Code owners file. # This file controls who is tagged for review for any given pull request. -* @dlorenc @sharifelgamal @nkubala @tstromberg @donmccasland +* @dlorenc @sharifelgamal @tstromberg @donmccasland From f6c18ed7aa36675eed95757ff8627871256e4be6 Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Fri, 27 Mar 2020 10:47:56 -0700 Subject: [PATCH 311/362] Fixing test --- tests/no-virtualenv/no-virtualenv.yaml | 2 +- tests/virtualenv/virtualenv_python37.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/no-virtualenv/no-virtualenv.yaml b/tests/no-virtualenv/no-virtualenv.yaml index 6a3b1e7d..19e48df9 100644 --- a/tests/no-virtualenv/no-virtualenv.yaml +++ b/tests/no-virtualenv/no-virtualenv.yaml @@ -43,7 +43,7 @@ commandTests: expectedOutput: ["/usr/local/bin/python3\n"] - name: "default python3 version" command: ["python3", "--version"] - expectedOutput: ["Python 3.6.8\n"] + expectedOutput: ["Python 3.7.7\n"] - name: "default pip3 installation" command: ["which", "pip3"] expectedOutput: ["/usr/local/bin/pip3\n"] diff --git a/tests/virtualenv/virtualenv_python37.yaml b/tests/virtualenv/virtualenv_python37.yaml index 457933f9..7de6d10c 100644 --- a/tests/virtualenv/virtualenv_python37.yaml +++ b/tests/virtualenv/virtualenv_python37.yaml @@ -25,7 +25,7 @@ commandTests: - name: "virtualenv37 python version" setup: [["virtualenv", "-p", "python3.7", "/env"]] command: ["python", "--version"] - expectedOutput: ["Python 3.7.2\n"] + expectedOutput: ["Python 3.7.7\n"] - name: "virtualenv37 pip installation" setup: [["virtualenv", "-p", "python3.7", "/env"]] From a0d1f9def6b296f789023b7973570d6aba857e0d Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Sat, 28 Mar 2020 20:18:17 -0700 Subject: [PATCH 312/362] Update interpreter builders with new python versions --- .../scripts/build-python-3.5.sh | 12 ++++++------ .../scripts/build-python-3.6.sh | 12 ++++++------ .../scripts/build-python-3.7.sh | 2 +- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/python-interpreter-builder/scripts/build-python-3.5.sh b/python-interpreter-builder/scripts/build-python-3.5.sh index 86a564c3..c2fd748b 100755 --- a/python-interpreter-builder/scripts/build-python-3.5.sh +++ b/python-interpreter-builder/scripts/build-python-3.5.sh @@ -6,14 +6,14 @@ set -x # Get the source mkdir -p /opt/sources cd /opt/sources -wget --no-verbose https://www.python.org/ftp/python/3.5.5/Python-3.5.5.tgz +wget --no-verbose https://www.python.org/ftp/python/3.5.9/Python-3.5.9.tgz # SHA-256 generated via `shasum -a 256 [file]` shasum --check < Date: Sat, 28 Mar 2020 20:20:28 -0700 Subject: [PATCH 313/362] Update tests --- tests/virtualenv/virtualenv_python35.yaml | 2 +- tests/virtualenv/virtualenv_python36.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/virtualenv/virtualenv_python35.yaml b/tests/virtualenv/virtualenv_python35.yaml index 5f37ee54..3bb3d814 100644 --- a/tests/virtualenv/virtualenv_python35.yaml +++ b/tests/virtualenv/virtualenv_python35.yaml @@ -25,7 +25,7 @@ commandTests: - name: "virtualenv35 python version" setup: [["virtualenv", "-p", "python3.5", "/env"]] command: ["python", "--version"] - expectedOutput: ["Python 3.5.5\n"] + expectedOutput: ["Python 3.5.9\n"] - name: "virtualenv35 pip installation" setup: [["virtualenv", "-p", "python3.5", "/env"]] diff --git a/tests/virtualenv/virtualenv_python36.yaml b/tests/virtualenv/virtualenv_python36.yaml index 64ba2596..f0949c6f 100644 --- a/tests/virtualenv/virtualenv_python36.yaml +++ b/tests/virtualenv/virtualenv_python36.yaml @@ -25,7 +25,7 @@ commandTests: - name: "virtualenv36 python version" setup: [["virtualenv", "-p", "python3.6", "/env"]] command: ["python", "--version"] - expectedOutput: ["Python 3.6.8\n"] + expectedOutput: ["Python 3.6.10\n"] - name: "virtualenv36 pip installation" setup: [["virtualenv", "-p", "python3.6", "/env"]] From 641e22349c6937e224ca8c7634de4399d1693bdf Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Sun, 29 Mar 2020 16:19:46 -0700 Subject: [PATCH 314/362] Fix 3.5 test failure --- tests/python3-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 3b283d1d..e3fd6424 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -77,7 +77,7 @@ lxml==4.2.4 mako==1.0.7 manifestparser==1.1 markdown==2.6.11 -markupsafe==1.0 +markupsafe==1.1.1 matplotlib==2.2.3 mccabe==0.6.1 meld3==1.0.2 From 3d5d38460ca4beac31a5b4cb35adaa25c31670d1 Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Mon, 30 Mar 2020 10:12:28 -0700 Subject: [PATCH 315/362] Attempting upgrade of pip version --- runtime-image/resources/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime-image/resources/requirements.txt b/runtime-image/resources/requirements.txt index 298e5f41..7ec375ff 100644 --- a/runtime-image/resources/requirements.txt +++ b/runtime-image/resources/requirements.txt @@ -1,5 +1,5 @@ # Do not update pip, it conflicts with the OS system installed version. # https://github.com/pypa/pip/issues/5240 -pip==9.0.3 +pip==10.0.1 setuptools==40.2.0 wheel==0.31.1 From bf65b6947c2e9f63a260f95d121ca07926361315 Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Mon, 30 Mar 2020 10:27:04 -0700 Subject: [PATCH 316/362] Latest pip is actually 20.0.2 --- runtime-image/resources/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime-image/resources/requirements.txt b/runtime-image/resources/requirements.txt index 7ec375ff..b374c0e8 100644 --- a/runtime-image/resources/requirements.txt +++ b/runtime-image/resources/requirements.txt @@ -1,5 +1,5 @@ # Do not update pip, it conflicts with the OS system installed version. # https://github.com/pypa/pip/issues/5240 -pip==10.0.1 +pip==20.0.2 setuptools==40.2.0 wheel==0.31.1 From cc6436428ad613f8e311045a26b7b54c6b5ad442 Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Mon, 30 Mar 2020 11:08:46 -0700 Subject: [PATCH 317/362] Looks like the latest available is 19.1.1 --- runtime-image/resources/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime-image/resources/requirements.txt b/runtime-image/resources/requirements.txt index b374c0e8..0032013d 100644 --- a/runtime-image/resources/requirements.txt +++ b/runtime-image/resources/requirements.txt @@ -1,5 +1,5 @@ # Do not update pip, it conflicts with the OS system installed version. # https://github.com/pypa/pip/issues/5240 -pip==20.0.2 +pip==19.1.1 setuptools==40.2.0 wheel==0.31.1 From 1f2f63dcad36e63ee0ced458cda7d53922290350 Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Tue, 31 Mar 2020 13:27:33 -0700 Subject: [PATCH 318/362] Letting pip go to latest to address python2 pip incompatibility issues. --- runtime-image/resources/requirements.txt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/runtime-image/resources/requirements.txt b/runtime-image/resources/requirements.txt index 0032013d..2d010eef 100644 --- a/runtime-image/resources/requirements.txt +++ b/runtime-image/resources/requirements.txt @@ -1,5 +1,3 @@ -# Do not update pip, it conflicts with the OS system installed version. -# https://github.com/pypa/pip/issues/5240 -pip==19.1.1 +pip setuptools==40.2.0 wheel==0.31.1 From e212b243482b17196725eaaf59c25ced9fd7960b Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Tue, 18 Aug 2020 09:15:39 -0700 Subject: [PATCH 319/362] Updating to 3.7.9 --- .../scripts/build-python-3.7.sh | 12 ++++++------ tests/virtualenv/virtualenv_python37.yaml | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/python-interpreter-builder/scripts/build-python-3.7.sh b/python-interpreter-builder/scripts/build-python-3.7.sh index bee33cef..b1b06806 100755 --- a/python-interpreter-builder/scripts/build-python-3.7.sh +++ b/python-interpreter-builder/scripts/build-python-3.7.sh @@ -6,14 +6,14 @@ set -x # Get the source mkdir -p /opt/sources cd /opt/sources -wget --no-verbose https://www.python.org/ftp/python/3.7.7/Python-3.7.7.tgz +wget --no-verbose https://www.python.org/ftp/python/3.7.9/Python-3.7.9.tgz # SHA-256 generated via `shasum -a 256 [file]` shasum --check < Date: Thu, 3 Sep 2020 12:56:15 -0700 Subject: [PATCH 320/362] Remove virtualenv pip before installing requirements --- builder/gen-dockerfile/Dockerfile.in | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/builder/gen-dockerfile/Dockerfile.in b/builder/gen-dockerfile/Dockerfile.in index 4f6447eb..98501c77 100644 --- a/builder/gen-dockerfile/Dockerfile.in +++ b/builder/gen-dockerfile/Dockerfile.in @@ -7,7 +7,9 @@ RUN virtualenv --no-download /env -p python3.6 ENV VIRTUAL_ENV /env ENV PATH /env/bin:$PATH ADD requirements.txt /builder/ -RUN pip install -r /builder/requirements.txt +#virtualenv's pip is pegged at version 10.0, removing so +#newer versions get picked up +RUN rm -f /env/bin/pip* && pip install -r /builder/requirements.txt ADD . /builder/ WORKDIR /workspace ENTRYPOINT [ "python", "/builder/gen_dockerfile.py" ] From c9f59421f0064de12c45eb258635fe591d182907 Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Thu, 3 Sep 2020 13:53:37 -0700 Subject: [PATCH 321/362] Fix 3.7.9 test --- tests/no-virtualenv/no-virtualenv.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/no-virtualenv/no-virtualenv.yaml b/tests/no-virtualenv/no-virtualenv.yaml index 19e48df9..4f6c3f48 100644 --- a/tests/no-virtualenv/no-virtualenv.yaml +++ b/tests/no-virtualenv/no-virtualenv.yaml @@ -43,7 +43,7 @@ commandTests: expectedOutput: ["/usr/local/bin/python3\n"] - name: "default python3 version" command: ["python3", "--version"] - expectedOutput: ["Python 3.7.7\n"] + expectedOutput: ["Python 3.7.9\n"] - name: "default pip3 installation" command: ["which", "pip3"] expectedOutput: ["/usr/local/bin/pip3\n"] From 71223557e38a88abc8b365d1bb870c5d4132e864 Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Thu, 10 Sep 2020 13:53:38 -0700 Subject: [PATCH 322/362] Update virtualenv version --- runtime-image/resources/requirements-virtualenv.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime-image/resources/requirements-virtualenv.txt b/runtime-image/resources/requirements-virtualenv.txt index 52bb7da1..25f09c4a 100644 --- a/runtime-image/resources/requirements-virtualenv.txt +++ b/runtime-image/resources/requirements-virtualenv.txt @@ -1 +1 @@ -virtualenv==16.0.0 +virtualenv==20.0.31 From 92a7aba5365a784aaf66192f86bfa5912f56d405 Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Thu, 10 Sep 2020 17:19:11 -0700 Subject: [PATCH 323/362] Updating virtualenv, test, and python builder version --- builder/gen-dockerfile/Dockerfile.in | 4 ++-- tests/eventlet/requirements.txt | 2 +- tests/virtualenv/virtualenv_default.yaml | 2 +- tests/virtualenv/virtualenv_python27.yaml | 2 +- tests/virtualenv/virtualenv_python34.yaml | 2 +- tests/virtualenv/virtualenv_python35.yaml | 2 +- tests/virtualenv/virtualenv_python36.yaml | 2 +- tests/virtualenv/virtualenv_python37.yaml | 2 +- 8 files changed, 9 insertions(+), 9 deletions(-) diff --git a/builder/gen-dockerfile/Dockerfile.in b/builder/gen-dockerfile/Dockerfile.in index 98501c77..4ddce7c2 100644 --- a/builder/gen-dockerfile/Dockerfile.in +++ b/builder/gen-dockerfile/Dockerfile.in @@ -1,6 +1,6 @@ FROM ${STAGING_IMAGE} -LABEL python_version=python3.6 -RUN virtualenv --no-download /env -p python3.6 +LABEL python_version=python3.7 +RUN virtualenv --no-download /env -p python3.7 # Set virtualenv environment variables. This is equivalent to running # source /env/bin/activate diff --git a/tests/eventlet/requirements.txt b/tests/eventlet/requirements.txt index 7f8b2468..65eabc74 100644 --- a/tests/eventlet/requirements.txt +++ b/tests/eventlet/requirements.txt @@ -6,5 +6,5 @@ greenlet==0.4.14 gunicorn==19.9.0 itsdangerous==0.24 Jinja2==2.10 -MarkupSafe==1.0 +MarkupSafe==1.1.1 Werkzeug==0.14.1 diff --git a/tests/virtualenv/virtualenv_default.yaml b/tests/virtualenv/virtualenv_default.yaml index 1659c746..6b6ad282 100644 --- a/tests/virtualenv/virtualenv_default.yaml +++ b/tests/virtualenv/virtualenv_default.yaml @@ -34,4 +34,4 @@ commandTests: setup: [["virtualenv", "/env"], ["pip", "install", "flask"]] command: ["python", "-c", "import flask; print(flask.__file__)"] - expectedOutput: ["/env/local/lib/python2.7/site-packages/flask"] + expectedOutput: ["/env/lib/python2.7/site-packages/flask/__init__.pyc"] diff --git a/tests/virtualenv/virtualenv_python27.yaml b/tests/virtualenv/virtualenv_python27.yaml index 8bd85289..09b78480 100644 --- a/tests/virtualenv/virtualenv_python27.yaml +++ b/tests/virtualenv/virtualenv_python27.yaml @@ -44,4 +44,4 @@ commandTests: setup: [["virtualenv", "-p", "python", "/env"], ["pip", "install", "flask"]] command: ["python", "-c", "import flask; print(flask.__file__)"] - expectedOutput: ["/env/local/lib/python2.7/site-packages/flask"] + expectedOutput: ["/env/lib/python2.7/site-packages/flask/__init__.pyc"] diff --git a/tests/virtualenv/virtualenv_python34.yaml b/tests/virtualenv/virtualenv_python34.yaml index 077606fa..9b5b77d0 100644 --- a/tests/virtualenv/virtualenv_python34.yaml +++ b/tests/virtualenv/virtualenv_python34.yaml @@ -47,7 +47,7 @@ commandTests: setup: [["virtualenv", "-p", "python3.4", "/env"], ["pip", "install", "flask"]] command: ["python", "-c", "import flask; print(flask.__file__)"] - expectedOutput: ["/env/lib/python3.4/site-packages/flask"] + expectedOutput: ["/env/lib/python3.4/site-packages/flask/__init__.py"] - name: "virtualenv34 test.support availability" setup: [["virtualenv", "-p", "python3.4", "/env"]] diff --git a/tests/virtualenv/virtualenv_python35.yaml b/tests/virtualenv/virtualenv_python35.yaml index 3bb3d814..5e4b394a 100644 --- a/tests/virtualenv/virtualenv_python35.yaml +++ b/tests/virtualenv/virtualenv_python35.yaml @@ -47,7 +47,7 @@ commandTests: setup: [["virtualenv", "-p", "python3.5", "/env"], ["pip", "install", "flask"]] command: ["python", "-c", "import flask; print(flask.__file__)"] - expectedOutput: ["/env/lib/python3.5/site-packages/flask"] + expectedOutput: ["/env/lib/python3.5/site-packages/flask/__init__.py"] - name: "virtualenv35 test.support availability" setup: [["virtualenv", "-p", "python3.5", "/env"]] diff --git a/tests/virtualenv/virtualenv_python36.yaml b/tests/virtualenv/virtualenv_python36.yaml index f0949c6f..b3a9e68e 100644 --- a/tests/virtualenv/virtualenv_python36.yaml +++ b/tests/virtualenv/virtualenv_python36.yaml @@ -47,7 +47,7 @@ commandTests: setup: [["virtualenv", "-p", "python3.6", "/env"], ["pip", "install", "flask"]] command: ["python", "-c", "import flask; print(flask.__file__)"] - expectedOutput: ["/env/lib/python3.6/site-packages/flask"] + expectedOutput: ["/env/lib/python3.6/site-packages/flask/__init__.py"] - name: "virtualenv36 test.support availability" setup: [["virtualenv", "-p", "python3.6", "/env"]] diff --git a/tests/virtualenv/virtualenv_python37.yaml b/tests/virtualenv/virtualenv_python37.yaml index 7f3520ad..9810c78e 100644 --- a/tests/virtualenv/virtualenv_python37.yaml +++ b/tests/virtualenv/virtualenv_python37.yaml @@ -47,7 +47,7 @@ commandTests: setup: [["virtualenv", "-p", "python3.7", "/env"], ["pip", "install", "flask"]] command: ["python", "-c", "import flask; print(flask.__file__)"] - expectedOutput: ["/env/lib/python3.7/site-packages/flask"] + expectedOutput: ["/env/lib/python3.7/site-packages/flask/__init__.py"] - name: "virtualenv37 test.support availability" setup: [["virtualenv", "-p", "python3.7", "/env"]] From df22badacb89bbd6cb21f0fc9f50189053ac5019 Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Fri, 11 Sep 2020 10:26:30 -0700 Subject: [PATCH 324/362] Use virtualenv pip --- builder/gen-dockerfile/Dockerfile.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/gen-dockerfile/Dockerfile.in b/builder/gen-dockerfile/Dockerfile.in index 4ddce7c2..92ceaecf 100644 --- a/builder/gen-dockerfile/Dockerfile.in +++ b/builder/gen-dockerfile/Dockerfile.in @@ -9,7 +9,7 @@ ENV PATH /env/bin:$PATH ADD requirements.txt /builder/ #virtualenv's pip is pegged at version 10.0, removing so #newer versions get picked up -RUN rm -f /env/bin/pip* && pip install -r /builder/requirements.txt +RUN pip install -r /builder/requirements.txt ADD . /builder/ WORKDIR /workspace ENTRYPOINT [ "python", "/builder/gen_dockerfile.py" ] From 764a5f0bc0dd5a5ea1a2404e813ed9a8569da218 Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Wed, 7 Jul 2021 12:23:41 -0700 Subject: [PATCH 325/362] Fixing build error for python2.7 pip --- build.sh | 15 ++++++++++++--- runtime-image/Dockerfile.in | 1 + runtime-image/resources/apt-packages.txt | 1 - 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/build.sh b/build.sh index 63afa1ab..2083237d 100755 --- a/build.sh +++ b/build.sh @@ -24,7 +24,7 @@ test=0 # Should run standard test suite? local=0 # Should run using local Docker daemon instead of GCR? -os_base=ubuntu16 # Which operating system base to use +os_base=ubuntu18 # Which operating system base to use interpreter=0 # Should build interpreters instead of images # Note that $gcloud_cmd has spaces in it @@ -48,7 +48,7 @@ Options: --[no]test: Run basic tests (default true if no options set) --[no]client_test: Run Google Cloud Client Library tests (default false) --[no]local: Build images using local Docker daemon (default false) - --os_base: Which OS image to build on top of [debian8, ubuntu16] + --os_base: Which OS image to build on top of [debian8, ubuntu16, ubuntu18] " } @@ -119,6 +119,10 @@ while [ $# -gt 0 ]; do os_base=ubuntu16 shift ;; + --os_base=ubuntu18) + os_base=ubuntu18 + shift + ;; --test) test=1 shift @@ -156,8 +160,13 @@ fi # Pick OS image to use as base if [ "${os_base}" == "debian8" ]; then export OS_BASE_IMAGE="gcr.io/google-appengine/debian8:latest" -else +elif [ "${os_base}" == "ubuntu16" ]; then export OS_BASE_IMAGE="gcr.io/gcp-runtimes/ubuntu_16_0_4:latest" +elif [ "${os_base}" == "ubuntu18" ]; then + export OS_BASE_IMAGE="gcr.io/gcp-runtimes/ubuntu_18_0_4:latest" +else + echo "Unsupported OS base image: $OS_BASE_IMAGE" + exit 1 fi export STAGING_IMAGE="${DOCKER_NAMESPACE}/python:${TAG}" echo "Using base image name ${STAGING_IMAGE}" diff --git a/runtime-image/Dockerfile.in b/runtime-image/Dockerfile.in index dc842d9b..46387705 100644 --- a/runtime-image/Dockerfile.in +++ b/runtime-image/Dockerfile.in @@ -9,6 +9,7 @@ ADD scripts /scripts # Install Python, pip, and C dev libraries necessary to compile the most popular # Python libraries. RUN /scripts/install-apt-packages.sh +RUN curl "https://bootstrap.pypa.io/pip/2.7/get-pip.py" -o "get-pip.py" && python ./get-pip.py && ln -s /usr/local/bin/pip /usr/bin/pip # Setup locale. This prevents Python 3 IO encoding issues. ENV LANG C.UTF-8 diff --git a/runtime-image/resources/apt-packages.txt b/runtime-image/resources/apt-packages.txt index 287f7abc..7b88f777 100644 --- a/runtime-image/resources/apt-packages.txt +++ b/runtime-image/resources/apt-packages.txt @@ -4,7 +4,6 @@ mercurial pkg-config wget # debian-provided interpreters -python-pip python2.7 python2.7-dev # Dependenies for third-party Python packages From b4754dd62596fa97499adde2085bc855571d5e57 Mon Sep 17 00:00:00 2001 From: Matthew Suozzo Date: Mon, 15 Aug 2022 17:21:19 -0400 Subject: [PATCH 326/362] Propagate env vars to sudo invocations. --- scripts/deploy_check.sh | 4 ++-- scripts/integration-test.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/deploy_check.sh b/scripts/deploy_check.sh index 77fccb54..1e2f02c2 100644 --- a/scripts/deploy_check.sh +++ b/scripts/deploy_check.sh @@ -12,11 +12,11 @@ fi cd ${KOKORO_GFILE_DIR}/appengine/integration_tests -sudo /usr/local/bin/pip install --upgrade -r requirements.txt +sudo -E /usr/local/bin/pip install --upgrade -r requirements.txt if [ -f ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY}/requirements.txt ] then - sudo /usr/local/bin/pip install --upgrade -r ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY}/requirements.txt + sudo -E /usr/local/bin/pip install --upgrade -r ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY}/requirements.txt fi export DEPLOY_LATENCY_PROJECT='cloud-deploy-latency' diff --git a/scripts/integration-test.sh b/scripts/integration-test.sh index 63137b5f..6210e0a1 100644 --- a/scripts/integration-test.sh +++ b/scripts/integration-test.sh @@ -7,11 +7,11 @@ source ${KOKORO_GFILE_DIR}/kokoro/common.sh export GOOGLE_CLOUD_PROJECT=gcp-runtimes -sudo /usr/local/bin/pip install --upgrade -r ${KOKORO_GFILE_DIR}/appengine/integration_tests/requirements.txt +sudo -E /usr/local/bin/pip install --upgrade -r ${KOKORO_GFILE_DIR}/appengine/integration_tests/requirements.txt if [ -f ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY}/requirements.txt ] then - sudo /usr/local/bin/pip install --upgrade -r ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY}/requirements.txt + sudo -E /usr/local/bin/pip install --upgrade -r ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY}/requirements.txt fi export GOPATH=${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY} From cc0da57364aec352e15dfa203b939a41b6cb7faa Mon Sep 17 00:00:00 2001 From: jinglundong <1683035+jinglundong@users.noreply.github.com> Date: Tue, 16 Aug 2022 16:21:39 -0700 Subject: [PATCH 327/362] Update CODEOWNERS --- CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CODEOWNERS b/CODEOWNERS index 5deacc6c..2b618c5f 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1,4 +1,4 @@ # Code owners file. # This file controls who is tagged for review for any given pull request. -* @dlorenc @sharifelgamal @tstromberg @donmccasland +* @jinglundong @donmccasland From 647d1a1344ef9bfc3ce560c4190ba3f2fb2b9903 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Aug 2023 21:52:18 +0000 Subject: [PATCH 328/362] Bump tornado from 5.1 to 6.3.3 in /tests/python2-libraries Bumps [tornado](https://github.com/tornadoweb/tornado) from 5.1 to 6.3.3. - [Changelog](https://github.com/tornadoweb/tornado/blob/master/docs/releases.rst) - [Commits](https://github.com/tornadoweb/tornado/compare/v5.1.0...v6.3.3) --- updated-dependencies: - dependency-name: tornado dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python2-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 865cd1aa..964eaf9e 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -184,7 +184,7 @@ supervisor==3.3.4 testrepository==0.0.20 testtools==2.3.0 thrift==0.11.0 -tornado==5.1 +tornado==6.3.3 tox==3.2.1 twisted==18.7.0 ujson==1.35 From 07b6f1ab2faee152ab2d4bef1dfc41863cfe752f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Aug 2023 21:53:13 +0000 Subject: [PATCH 329/362] Bump tornado from 5.1 to 6.3.3 in /tests/python3-libraries Bumps [tornado](https://github.com/tornadoweb/tornado) from 5.1 to 6.3.3. - [Changelog](https://github.com/tornadoweb/tornado/blob/master/docs/releases.rst) - [Commits](https://github.com/tornadoweb/tornado/compare/v5.1.0...v6.3.3) --- updated-dependencies: - dependency-name: tornado dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python3-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index e3fd6424..c0dc6a66 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -172,7 +172,7 @@ stevedore==1.29.0 testrepository==0.0.20 testtools==2.3.0 thrift==0.11.0 -tornado==5.1 +tornado==6.3.3 tox==3.2.1 twisted==18.7.0 ujson==1.35 From 8f6ff95c279eb8313b6f559c7e2b39d1f71bb2a5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 00:44:38 +0000 Subject: [PATCH 330/362] Bump pygments from 2.2.0 to 2.15.0 in /tests/python3-libraries Bumps [pygments](https://github.com/pygments/pygments) from 2.2.0 to 2.15.0. - [Release notes](https://github.com/pygments/pygments/releases) - [Changelog](https://github.com/pygments/pygments/blob/master/CHANGES) - [Commits](https://github.com/pygments/pygments/compare/2.2.0...2.15.0) --- updated-dependencies: - dependency-name: pygments dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python3-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index c0dc6a66..ccd826d4 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -123,7 +123,7 @@ pyasn1==0.4.4 pycparser==2.18 pycrypto==2.6.1 pyflakes==2.0.0 -pygments==2.2.0 +pygments==2.15.0 pyjwt==1.6.4 pylibmc==1.5.2 pylint==2.1.1 From 2f6f1a4fcd14bc52377692a66013bf85f7e518b2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 00:44:39 +0000 Subject: [PATCH 331/362] Bump cryptography from 2.3.1 to 41.0.3 in /tests/python3-libraries Bumps [cryptography](https://github.com/pyca/cryptography) from 2.3.1 to 41.0.3. - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/2.3.1...41.0.3) --- updated-dependencies: - dependency-name: cryptography dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python3-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index c0dc6a66..f761e959 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -30,7 +30,7 @@ cov-core==1.15.0 coverage==4.5.1 coveralls==1.4.0 crcmod==1.7 -cryptography==2.3.1 +cryptography==41.0.3 cssselect==1.0.3 cython==0.28.5 decorator==4.3.0 From f6682ba419bf583c5a34b258cb455159c67640a9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 00:44:43 +0000 Subject: [PATCH 332/362] Bump scipy from 1.1.0 to 1.10.0 in /tests/python3-libraries Bumps [scipy](https://github.com/scipy/scipy) from 1.1.0 to 1.10.0. - [Release notes](https://github.com/scipy/scipy/releases) - [Commits](https://github.com/scipy/scipy/compare/v1.1.0...v1.10.0) --- updated-dependencies: - dependency-name: scipy dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python3-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index c0dc6a66..58839946 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -154,7 +154,7 @@ requests-oauthlib==1.0.0 requests==2.19.1 retrying==1.3.3 rsa==3.4.2 -scipy==1.1.0 +scipy==1.10.0 selenium==3.14.0 setuptools-git==1.2 setuptools==40.2.0 From 0a395be4f042ac9517d35a1cac13d99ca65caa84 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 00:44:45 +0000 Subject: [PATCH 333/362] Bump certifi from 2018.8.24 to 2023.7.22 in /tests/python2-libraries Bumps [certifi](https://github.com/certifi/python-certifi) from 2018.8.24 to 2023.7.22. - [Commits](https://github.com/certifi/python-certifi/compare/2018.08.24...2023.07.22) --- updated-dependencies: - dependency-name: certifi dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python2-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 865cd1aa..bb083cde 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -20,7 +20,7 @@ botocore==1.11.1 bottle==0.12.13 carbon<1.1.1 celery==4.2.1 -certifi==2018.8.24 +certifi==2023.7.22 cffi==1.11.5 chardet==3.0.4 click==6.7 From b383ecd9913c3550e5a55602bc682a61f84c7781 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 00:44:45 +0000 Subject: [PATCH 334/362] Bump certifi from 2018.8.24 to 2023.7.22 in /tests/python3-libraries Bumps [certifi](https://github.com/certifi/python-certifi) from 2018.8.24 to 2023.7.22. - [Commits](https://github.com/certifi/python-certifi/compare/2018.08.24...2023.07.22) --- updated-dependencies: - dependency-name: certifi dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python3-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index c0dc6a66..8497d1d9 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -18,7 +18,7 @@ boto==2.49.0 botocore==1.11.1 bottle==0.12.13 celery==4.2.1 -certifi==2018.8.24 +certifi==2023.7.22 cffi==1.11.5 chardet==3.0.4 click==6.7 From 19ccc8a8be75537635ca00e280f71ee4fed4fbcd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 06:18:22 +0000 Subject: [PATCH 335/362] Bump requests from 2.19.1 to 2.31.0 in /tests/python3-libraries Bumps [requests](https://github.com/psf/requests) from 2.19.1 to 2.31.0. - [Release notes](https://github.com/psf/requests/releases) - [Changelog](https://github.com/psf/requests/blob/main/HISTORY.md) - [Commits](https://github.com/psf/requests/compare/v2.19.1...v2.31.0) --- updated-dependencies: - dependency-name: requests dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python3-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 71dab860..3568a7f3 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -151,7 +151,7 @@ raven==6.9.0 redis==2.10.6 repoze.lru==0.7 requests-oauthlib==1.0.0 -requests==2.19.1 +requests==2.31.0 retrying==1.3.3 rsa==3.4.2 scipy==1.1.0 From e1854082720277c75cdb5ef63ab386b314c2d2a6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 06:18:41 +0000 Subject: [PATCH 336/362] Bump scipy from 1.1.0 to 1.10.0 in /tests/python2-libraries Bumps [scipy](https://github.com/scipy/scipy) from 1.1.0 to 1.10.0. - [Release notes](https://github.com/scipy/scipy/releases) - [Commits](https://github.com/scipy/scipy/compare/v1.1.0...v1.10.0) --- updated-dependencies: - dependency-name: scipy dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python2-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 63d371ec..dace11da 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -164,7 +164,7 @@ requests-oauthlib==1.0.0 requests==2.19.1 retrying==1.3.3 rsa==3.4.2 -scipy==1.1.0 +scipy==1.10.0 selenium==3.14.0 setuptools-git==1.2 setuptools==40.2.0 From a3673f8691058c3c068dc2f42e9b2cb9804b441e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 06:18:58 +0000 Subject: [PATCH 337/362] Bump flask from 1.1.0 to 2.2.5 in /tests/integration Bumps [flask](https://github.com/pallets/flask) from 1.1.0 to 2.2.5. - [Release notes](https://github.com/pallets/flask/releases) - [Changelog](https://github.com/pallets/flask/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/flask/compare/1.1.0...2.2.5) --- updated-dependencies: - dependency-name: flask dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/integration/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index ea714c20..0e31f608 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -1,4 +1,4 @@ -Flask==1.1.0 +Flask==2.2.5 google-cloud-error-reporting==0.32.1 google-cloud-logging==1.12.1 google-cloud-monitoring==0.33.0 From 4cd3ee423bb00a17f27255f8f579ba110cf933e1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 06:19:23 +0000 Subject: [PATCH 338/362] Bump flask from 1.0.2 to 2.2.5 in /tests/python3-libraries Bumps [flask](https://github.com/pallets/flask) from 1.0.2 to 2.2.5. - [Release notes](https://github.com/pallets/flask/releases) - [Changelog](https://github.com/pallets/flask/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/flask/compare/1.0.2...2.2.5) --- updated-dependencies: - dependency-name: flask dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python3-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index c48b246e..24727967 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -51,7 +51,7 @@ extras==1.0.0 fabric==2.3.1 fixtures==3.0.0 flake8==3.5.0 -flask==1.0.2 +flask==2.2.5 funcsigs==1.0.2 gevent==1.3.6 google-api-python-client==1.7.4 From f3e034b17cd7d2558f0a4ee12a6f5599c6c645b5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 06:19:44 +0000 Subject: [PATCH 339/362] Bump requests from 2.19.1 to 2.31.0 in /tests/python2-libraries Bumps [requests](https://github.com/psf/requests) from 2.19.1 to 2.31.0. - [Release notes](https://github.com/psf/requests/releases) - [Changelog](https://github.com/psf/requests/blob/main/HISTORY.md) - [Commits](https://github.com/psf/requests/compare/v2.19.1...v2.31.0) --- updated-dependencies: - dependency-name: requests dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python2-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index dace11da..5db7a568 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -161,7 +161,7 @@ raven==6.9.0 redis==2.10.6 repoze.lru==0.7 requests-oauthlib==1.0.0 -requests==2.19.1 +requests==2.31.0 retrying==1.3.3 rsa==3.4.2 scipy==1.10.0 From a1dbe236ef109d8c88902a5fdb68d15990370c05 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 06:20:01 +0000 Subject: [PATCH 340/362] Bump requests from 2.22.0 to 2.31.0 in /tests/integration Bumps [requests](https://github.com/psf/requests) from 2.22.0 to 2.31.0. - [Release notes](https://github.com/psf/requests/releases) - [Changelog](https://github.com/psf/requests/blob/main/HISTORY.md) - [Commits](https://github.com/psf/requests/compare/v2.22.0...v2.31.0) --- updated-dependencies: - dependency-name: requests dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/integration/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index ea714c20..45782f06 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -3,7 +3,7 @@ google-cloud-error-reporting==0.32.1 google-cloud-logging==1.12.1 google-cloud-monitoring==0.33.0 gunicorn==19.9.0 -requests==2.22.0 +requests==2.31.0 retrying==1.3.3 six==1.12.0 protobuf>=3.6.0 From 257c0f560f110a5d555f7df657d38e09df4887a2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 06:21:34 +0000 Subject: [PATCH 341/362] Bump flask from 1.0.2 to 2.2.5 in /tests/python2-libraries Bumps [flask](https://github.com/pallets/flask) from 1.0.2 to 2.2.5. - [Release notes](https://github.com/pallets/flask/releases) - [Changelog](https://github.com/pallets/flask/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/flask/compare/1.0.2...2.2.5) --- updated-dependencies: - dependency-name: flask dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python2-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 5db7a568..92da6052 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -53,7 +53,7 @@ extras==1.0.0 fabric==2.3.1 fixtures==3.0.0 flake8==3.5.0 -flask==1.0.2 +flask==2.2.5 funcsigs==1.0.2 functools32==3.2.3.post2 futures==3.2.0 From c5e53836a50440973b14727de05399e35d20cd08 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 06:21:38 +0000 Subject: [PATCH 342/362] Bump sqlparse from 0.2.4 to 0.4.4 in /tests/python3-libraries Bumps [sqlparse](https://github.com/andialbrecht/sqlparse) from 0.2.4 to 0.4.4. - [Changelog](https://github.com/andialbrecht/sqlparse/blob/master/CHANGELOG) - [Commits](https://github.com/andialbrecht/sqlparse/compare/0.2.4...0.4.4) --- updated-dependencies: - dependency-name: sqlparse dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python3-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 82263711..1d6b37da 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -166,7 +166,7 @@ south==1.0.2 sphinx==1.7.7 sqlalchemy-migrate==0.11.0 sqlalchemy==1.2.11 -sqlparse==0.2.4 +sqlparse==0.4.4 statsd==3.3.0 stevedore==1.29.0 testrepository==0.0.20 From fd2cdb556238254b9b3201a48e7b1fafb12ce5a5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 06:21:51 +0000 Subject: [PATCH 343/362] Bump flask from 1.0.2 to 2.2.5 in /tests/eventlet Bumps [flask](https://github.com/pallets/flask) from 1.0.2 to 2.2.5. - [Release notes](https://github.com/pallets/flask/releases) - [Changelog](https://github.com/pallets/flask/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/flask/compare/1.0.2...2.2.5) --- updated-dependencies: - dependency-name: flask dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/eventlet/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/eventlet/requirements.txt b/tests/eventlet/requirements.txt index 65eabc74..75ee7c7d 100644 --- a/tests/eventlet/requirements.txt +++ b/tests/eventlet/requirements.txt @@ -1,7 +1,7 @@ click==6.7 enum-compat==0.0.2 eventlet==0.24.1 -Flask==1.0.2 +Flask==2.2.5 greenlet==0.4.14 gunicorn==19.9.0 itsdangerous==0.24 From abd0aa21189209da39fe79b4559fddaafe950a14 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 06:22:32 +0000 Subject: [PATCH 344/362] Bump urllib3 from 1.23 to 1.26.5 in /tests/python3-libraries Bumps [urllib3](https://github.com/urllib3/urllib3) from 1.23 to 1.26.5. - [Release notes](https://github.com/urllib3/urllib3/releases) - [Changelog](https://github.com/urllib3/urllib3/blob/main/CHANGES.rst) - [Commits](https://github.com/urllib3/urllib3/compare/1.23...1.26.5) --- updated-dependencies: - dependency-name: urllib3 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python3-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 82263711..06341ddd 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -179,7 +179,7 @@ ujson==1.35 unidecode==1.0.22 unittest2==1.1.0 uritemplate==3.0.0 -urllib3==1.23 +urllib3==1.26.5 uwsgi==2.0.17.1 versiontools==1.9.1 virtualenv==16.0.0 From 38e4e9d397da6d3e36f39dbcdec1da9501bc6f50 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 06:23:08 +0000 Subject: [PATCH 345/362] Bump ansible from 2.6.3 to 7.0.0 in /tests/python3-libraries Bumps [ansible](https://github.com/ansible/ansible) from 2.6.3 to 7.0.0. - [Release notes](https://github.com/ansible/ansible/releases) - [Commits](https://github.com/ansible/ansible/commits) --- updated-dependencies: - dependency-name: ansible dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python3-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 82263711..89b1a196 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -1,7 +1,7 @@ alembic==1.0.0 amqp==2.3.2 amqplib==1.0.2 -ansible==2.6.3 +ansible==7.0.0 anyjson==0.3.3 apache-libcloud==2.3.0 argparse==1.4.0 From 33570888d9e3359a02f70500c4fcf97a520f3dc9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 06:24:15 +0000 Subject: [PATCH 346/362] Bump sqlparse from 0.2.4 to 0.4.4 in /tests/python2-libraries Bumps [sqlparse](https://github.com/andialbrecht/sqlparse) from 0.2.4 to 0.4.4. - [Changelog](https://github.com/andialbrecht/sqlparse/blob/master/CHANGELOG) - [Commits](https://github.com/andialbrecht/sqlparse/compare/0.2.4...0.4.4) --- updated-dependencies: - dependency-name: sqlparse dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python2-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 92da6052..f3b9daec 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -176,7 +176,7 @@ south==1.0.2 sphinx==1.7.7 sqlalchemy-migrate==0.11.0 sqlalchemy==1.2.11 -sqlparse==0.2.4 +sqlparse==0.4.4 statsd==3.3.0 stevedore==1.29.0 suds==0.4 From 06f1cfa49fcb057c0440afb2d32868683061f494 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 06:24:41 +0000 Subject: [PATCH 347/362] Bump waitress from 1.1.0 to 2.1.2 in /tests/python3-libraries Bumps [waitress](https://github.com/Pylons/waitress) from 1.1.0 to 2.1.2. - [Release notes](https://github.com/Pylons/waitress/releases) - [Changelog](https://github.com/Pylons/waitress/blob/v2.1.2/CHANGES.txt) - [Commits](https://github.com/Pylons/waitress/compare/v1.1.0...v2.1.2) --- updated-dependencies: - dependency-name: waitress dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python3-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 4929dcc6..e8adc88f 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -183,7 +183,7 @@ urllib3==1.26.5 uwsgi==2.0.17.1 versiontools==1.9.1 virtualenv==16.0.0 -waitress==1.1.0 +waitress==2.1.2 warlock==1.3.0 webob==1.8.2 websocket-client==0.51.0 From b901f8f3ccbda6fcfec23082b4218bd2834d0a33 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 06:25:52 +0000 Subject: [PATCH 348/362] Bump pillow from 5.2.0 to 9.3.0 in /tests/python3-libraries Bumps [pillow](https://github.com/python-pillow/Pillow) from 5.2.0 to 9.3.0. - [Release notes](https://github.com/python-pillow/Pillow/releases) - [Changelog](https://github.com/python-pillow/Pillow/blob/main/CHANGES.rst) - [Commits](https://github.com/python-pillow/Pillow/compare/5.2.0...9.3.0) --- updated-dependencies: - dependency-name: pillow dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python3-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 0989646e..66e9b98e 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -111,7 +111,7 @@ pbr==4.2.0 pep8==1.7.1 pexpect==4.6.0 pika==0.12.0 -pillow==5.2.0 +pillow==9.3.0 pip==18.0 prettytable==0.7.2 protobuf==3.6.1 From f12acd1bc138fdeeedfcf042aab38661a88a57da Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 06:25:54 +0000 Subject: [PATCH 349/362] Bump pyyaml from 3.13 to 5.4 in /tests/python3-libraries Bumps [pyyaml](https://github.com/yaml/pyyaml) from 3.13 to 5.4. - [Changelog](https://github.com/yaml/pyyaml/blob/master/CHANGES) - [Commits](https://github.com/yaml/pyyaml/compare/3.13...5.4) --- updated-dependencies: - dependency-name: pyyaml dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python3-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 0989646e..f4d40d0e 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -145,7 +145,7 @@ python-novaclient==11.0.0 python-subunit==1.3.0 python-swiftclient==3.6.0 pytz==2018.5 -pyyaml==3.13 +pyyaml==5.4 pyzmq==17.1.2 raven==6.9.0 redis==2.10.6 From ce681862182eb79ab31d0f8f4162246b98c70dff Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 06:25:58 +0000 Subject: [PATCH 350/362] Bump numpy from 1.15.1 to 1.22.0 in /tests/python3-libraries Bumps [numpy](https://github.com/numpy/numpy) from 1.15.1 to 1.22.0. - [Release notes](https://github.com/numpy/numpy/releases) - [Changelog](https://github.com/numpy/numpy/blob/main/doc/RELEASE_WALKTHROUGH.rst) - [Commits](https://github.com/numpy/numpy/compare/v1.15.1...v1.22.0) --- updated-dependencies: - dependency-name: numpy dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python3-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 0989646e..9e70346b 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -95,7 +95,7 @@ netaddr==0.7.19 netifaces==0.10.7 newrelic==4.2.0.100 nose==1.3.7 -numpy==1.15.1 +numpy==1.22.0 oauth2==1.9.0.post1 oauth2client==4.1.2 oauthlib==2.1.0 From cf7bce61d466e477b8a0963a38a6d4199867c088 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 06:26:35 +0000 Subject: [PATCH 351/362] Bump django from 2.1 to 2.2.28 in /tests/python3-libraries Bumps [django](https://github.com/django/django) from 2.1 to 2.2.28. - [Commits](https://github.com/django/django/compare/2.1...2.2.28) --- updated-dependencies: - dependency-name: django dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python3-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 0989646e..31938505 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -37,7 +37,7 @@ decorator==4.3.0 django-celery==3.2.2 django-debug-toolbar==1.9.1 django-extensions==2.1.1 -django==2.1 +django==2.2.28 django_compress==1.0.1 djangorestframework==3.8.2 docker-py==1.10.6 From 78b72a16af2a8b4270505443061b850a3d749a79 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 18:22:59 +0000 Subject: [PATCH 352/362] Bump urllib3 from 1.23 to 1.26.5 in /tests/python2-libraries Bumps [urllib3](https://github.com/urllib3/urllib3) from 1.23 to 1.26.5. - [Release notes](https://github.com/urllib3/urllib3/releases) - [Changelog](https://github.com/urllib3/urllib3/blob/main/CHANGES.rst) - [Commits](https://github.com/urllib3/urllib3/compare/1.23...1.26.5) --- updated-dependencies: - dependency-name: urllib3 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python2-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index f3b9daec..2eaf5ba0 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -191,7 +191,7 @@ ujson==1.35 unidecode==1.0.22 unittest2==1.1.0 uritemplate==3.0.0 -urllib3==1.23 +urllib3==1.26.5 uwsgi==2.0.17.1 versiontools==1.9.1 virtualenv==16.0.0 From 2758277ba791b3629cfa54b6dd0553de8d729a79 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 18:23:18 +0000 Subject: [PATCH 353/362] Bump ansible from 2.6.3 to 7.0.0 in /tests/python2-libraries Bumps [ansible](https://github.com/ansible/ansible) from 2.6.3 to 7.0.0. - [Release notes](https://github.com/ansible/ansible/releases) - [Commits](https://github.com/ansible/ansible/commits) --- updated-dependencies: - dependency-name: ansible dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python2-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index f3b9daec..2795af2e 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -1,7 +1,7 @@ alembic==1.0.0 amqp==2.3.2 amqplib==1.0.2 -ansible==2.6.3 +ansible==7.0.0 anyjson==0.3.3 apache-libcloud==2.3.0 argparse==1.4.0 From c36d1079fc62e162b45020bc3abcdf70469c7cd8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 18:26:45 +0000 Subject: [PATCH 354/362] Bump waitress from 1.1.0 to 2.1.2 in /tests/python2-libraries Bumps [waitress](https://github.com/Pylons/waitress) from 1.1.0 to 2.1.2. - [Release notes](https://github.com/Pylons/waitress/releases) - [Changelog](https://github.com/Pylons/waitress/blob/v2.1.2/CHANGES.txt) - [Commits](https://github.com/Pylons/waitress/compare/v1.1.0...v2.1.2) --- updated-dependencies: - dependency-name: waitress dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python2-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index f3b9daec..09feaa44 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -195,7 +195,7 @@ urllib3==1.23 uwsgi==2.0.17.1 versiontools==1.9.1 virtualenv==16.0.0 -waitress==1.1.0 +waitress==2.1.2 warlock==1.3.0 webob==1.8.2 websocket-client==0.51.0 From 49e19e4b726a1950da60fabaa40cbce2a838a09b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 20:11:30 +0000 Subject: [PATCH 355/362] Bump numpy from 1.15.1 to 1.22.0 in /tests/python2-libraries Bumps [numpy](https://github.com/numpy/numpy) from 1.15.1 to 1.22.0. - [Release notes](https://github.com/numpy/numpy/releases) - [Changelog](https://github.com/numpy/numpy/blob/main/doc/RELEASE_WALKTHROUGH.rst) - [Commits](https://github.com/numpy/numpy/compare/v1.15.1...v1.22.0) --- updated-dependencies: - dependency-name: numpy dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python2-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 6bb9fc86..b2ecb1dd 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -103,7 +103,7 @@ netaddr==0.7.19 netifaces==0.10.7 newrelic==4.2.0.100 nose==1.3.7 -numpy==1.15.1 +numpy==1.22.0 oauth2==1.9.0.post1 oauth2client==4.1.2 oauthlib==2.1.0 From d12a82f5651abbc5c7f3a653aa8f909633e26f4e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 20:11:36 +0000 Subject: [PATCH 356/362] Bump pyyaml from 3.13 to 5.4 in /tests/python2-libraries Bumps [pyyaml](https://github.com/yaml/pyyaml) from 3.13 to 5.4. - [Changelog](https://github.com/yaml/pyyaml/blob/master/CHANGES) - [Commits](https://github.com/yaml/pyyaml/compare/3.13...5.4) --- updated-dependencies: - dependency-name: pyyaml dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python2-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 6bb9fc86..67b81897 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -155,7 +155,7 @@ python-novaclient==11.0.0 python-subunit==1.3.0 python-swiftclient==3.6.0 pytz==2018.5 -pyyaml==3.13 +pyyaml==5.4 pyzmq==17.1.2 raven==6.9.0 redis==2.10.6 From 4dbda4cfd7e58de057d291078e5143d421460eff Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 20:12:22 +0000 Subject: [PATCH 357/362] Bump pillow from 5.2.0 to 9.3.0 in /tests/python2-libraries Bumps [pillow](https://github.com/python-pillow/Pillow) from 5.2.0 to 9.3.0. - [Release notes](https://github.com/python-pillow/Pillow/releases) - [Changelog](https://github.com/python-pillow/Pillow/blob/main/CHANGES.rst) - [Commits](https://github.com/python-pillow/Pillow/compare/5.2.0...9.3.0) --- updated-dependencies: - dependency-name: pillow dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python2-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 6bb9fc86..cfbaab8b 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -119,7 +119,7 @@ pbr==4.2.0 pep8==1.7.1 pexpect==4.6.0 pika==0.12.0 -pillow==5.2.0 +pillow==9.3.0 pip==18.0 prettytable==0.7.2 protobuf==3.6.1 From 6c6077f604056fcf7f2383db23a82699e7c8450d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 16 Aug 2023 00:38:49 +0000 Subject: [PATCH 358/362] Bump werkzeug from 0.14.1 to 2.2.3 in /tests/python3-libraries Bumps [werkzeug](https://github.com/pallets/werkzeug) from 0.14.1 to 2.2.3. - [Release notes](https://github.com/pallets/werkzeug/releases) - [Changelog](https://github.com/pallets/werkzeug/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/werkzeug/compare/0.14.1...2.2.3) --- updated-dependencies: - dependency-name: werkzeug dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python3-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index d21238fd..d9117d4f 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -188,7 +188,7 @@ warlock==1.3.0 webob==1.8.2 websocket-client==0.51.0 webtest==2.0.30 -werkzeug==0.14.1 +werkzeug==2.2.3 wheel==0.31.1 xlrd==1.1.0 zc.buildout==2.12.1 From 17d4165fa40213904382fa9cdda5a067e2b86d5a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 16 Aug 2023 00:39:06 +0000 Subject: [PATCH 359/362] Bump werkzeug from 0.14.1 to 2.2.3 in /tests/python2-libraries Bumps [werkzeug](https://github.com/pallets/werkzeug) from 0.14.1 to 2.2.3. - [Release notes](https://github.com/pallets/werkzeug/releases) - [Changelog](https://github.com/pallets/werkzeug/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/werkzeug/compare/0.14.1...2.2.3) --- updated-dependencies: - dependency-name: werkzeug dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python2-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 29bdca44..67a0c0b3 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -200,7 +200,7 @@ warlock==1.3.0 webob==1.8.2 websocket-client==0.51.0 webtest==2.0.30 -werkzeug==0.14.1 +werkzeug==2.2.3 wheel==0.31.1 xlrd==1.1.0 zc.buildout==2.12.1 From 56de3a3edea2f3653ec018d874cb0f8cda9aa67d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 16 Aug 2023 00:40:22 +0000 Subject: [PATCH 360/362] Bump werkzeug from 0.14.1 to 2.2.3 in /tests/eventlet Bumps [werkzeug](https://github.com/pallets/werkzeug) from 0.14.1 to 2.2.3. - [Release notes](https://github.com/pallets/werkzeug/releases) - [Changelog](https://github.com/pallets/werkzeug/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/werkzeug/compare/0.14.1...2.2.3) --- updated-dependencies: - dependency-name: werkzeug dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/eventlet/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/eventlet/requirements.txt b/tests/eventlet/requirements.txt index 75ee7c7d..1e2ea66b 100644 --- a/tests/eventlet/requirements.txt +++ b/tests/eventlet/requirements.txt @@ -7,4 +7,4 @@ gunicorn==19.9.0 itsdangerous==0.24 Jinja2==2.10 MarkupSafe==1.1.1 -Werkzeug==0.14.1 +Werkzeug==2.2.3 From 461f8883bbd586319ffa8e1442858607c1473640 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 24 Aug 2023 20:11:21 +0000 Subject: [PATCH 361/362] Bump uwsgi from 2.0.17.1 to 2.0.22 in /tests/python2-libraries Bumps [uwsgi](https://github.com/unbit/uwsgi-docs) from 2.0.17.1 to 2.0.22. - [Commits](https://github.com/unbit/uwsgi-docs/commits) --- updated-dependencies: - dependency-name: uwsgi dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python2-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 67a0c0b3..f0596d67 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -192,7 +192,7 @@ unidecode==1.0.22 unittest2==1.1.0 uritemplate==3.0.0 urllib3==1.26.5 -uwsgi==2.0.17.1 +uwsgi==2.0.22 versiontools==1.9.1 virtualenv==16.0.0 waitress==2.1.2 From 5b017c9ecc9422ba6e959bc28d712ab1a4d4353d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 24 Aug 2023 20:11:33 +0000 Subject: [PATCH 362/362] Bump uwsgi from 2.0.17.1 to 2.0.22 in /tests/python3-libraries Bumps [uwsgi](https://github.com/unbit/uwsgi-docs) from 2.0.17.1 to 2.0.22. - [Commits](https://github.com/unbit/uwsgi-docs/commits) --- updated-dependencies: - dependency-name: uwsgi dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python3-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index d9117d4f..23175a06 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -180,7 +180,7 @@ unidecode==1.0.22 unittest2==1.1.0 uritemplate==3.0.0 urllib3==1.26.5 -uwsgi==2.0.17.1 +uwsgi==2.0.22 versiontools==1.9.1 virtualenv==16.0.0 waitress==2.1.2