diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 00000000..b08b2574 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,6 @@ +[run] +branch = True + +[report] +exclude_lines = + pragma: no cover diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..73ec28f0 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +*.pyc +.cache +.coverage +.nox +/*_local.sh +__pycache__ diff --git a/.travis.yml b/.travis.yml index 41768984..5f7debaf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,8 +2,6 @@ sudo: required services: - docker script: -- docker build -t google/python . -- docker build tests/virtualenv -- docker build tests/no-virtualenv -# Disabled temporarily. -#- docker build -t google/python-libraries tests/python2-libraries +- make build +- make tests/virtualenv +- make tests/no-virtualenv diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 00000000..2b618c5f --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1,4 @@ +# Code owners file. +# This file controls who is tagged for review for any given pull request. + +* @jinglundong @donmccasland diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 0e4e7f01..00000000 --- a/Dockerfile +++ /dev/null @@ -1,36 +0,0 @@ -# The Google App Engine base image is debian (jessie) with ca-certificates -# installed. -FROM gcr.io/google_appengine/base - -# Install Python, pip, and C dev libraries necessary to compile the most popular -# Python libraries. -RUN apt-get -q update && \ - apt-get install --no-install-recommends -y -q \ - python2.7 python3.4 python2.7-dev python3.4-dev python-pip build-essential git mercurial \ - libffi-dev libssl-dev libxml2-dev \ - libxslt1-dev libpq-dev libmysqlclient-dev libcurl4-openssl-dev \ - libjpeg-dev zlib1g-dev libpng12-dev \ - gfortran libblas-dev liblapack-dev libatlas-dev libquadmath0 \ - libfreetype6-dev pkg-config swig \ - && \ - apt-get clean && rm /var/lib/apt/lists/*_* - -# Setup locale. This prevents Python 3 IO encoding issues. -ENV LANG C.UTF-8 -# Make stdout/stderr unbuffered. This prevents delay between output and cloud -# logging collection. -ENV PYTHONUNBUFFERED 1 - -# Upgrade pip (debian package version tends to run a few version behind) and -# install virtualenv system-wide. -RUN pip install --upgrade pip virtualenv - -RUN ln -s /home/vmagent/app /app -WORKDIR /app - -# Port 8080 is the port used by Google App Engine for serving HTTP traffic. -EXPOSE 8080 -ENV PORT 8080 - -# The user's Dockerfile must specify an entrypoint with ENTRYPOINT or CMD. -CMD [] diff --git a/README.md b/README.md index 4562c7b1..51e70cd8 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,26 @@ # Google Cloud Platform - Python Runtime Docker Image -This repository contains the source for the `gcr.io/google_appengine/python` [docker](https://docker.io) base image. This image can be used as the base image for running applications on [Google App Engine Flexible](https://cloud.google.com/appengine/docs/flexible/), [Google Container Engine](https://cloud.google.com/container-engine), or any other Docker host. +This repository contains the source for the +[`gcr.io/google-appengine/python`](https://gcr.io/google-appengine/python) +[docker](https://docker.io) base image. This image can be used as the base image +for running applications on +[Google App Engine Flexible](https://cloud.google.com/appengine/docs/flexible/), +[Google Kubernetes Engine](https://cloud.google.com/kubernetes-engine), or any +other Docker host. -This image is based on Debian Jessie and contains packages required to build most of the popular Python libraries. For more information about this runtime, see the [documentation](https://cloud.google.com/appengine/docs/flexible/python/runtime). +This image is based on Ubuntu Xenial and contains packages required to build +most of the popular Python libraries. For more information about this runtime, +see the +[documentation](https://cloud.google.com/appengine/docs/flexible/python/runtime). ## App Engine -When using App Engine Flexible, you can use the runtime without worrying about docker by specifying `runtime: python` in your `app.yaml`: +When using App Engine Flexible, you can use the runtime without worrying about +docker by specifying `runtime: python` in your `app.yaml`: ```yaml runtime: python -vm: true +env: flex entrypoint: gunicorn -b :$PORT main:app runtime_config: @@ -18,20 +28,27 @@ runtime_config: python_version: 3 ``` -If you want to use this runtime as a base and customize it, you can generate a Dockerfile using the [`Cloud SDK`](https://cloud.google.com/sdk/gcloud/reference/preview/app/gen-config): +If you have an existing App Engine application using this runtime and want to +customize it, you can use the +[`Cloud SDK`](https://cloud.google.com/sdk/gcloud/reference/preview/app/gen-config) +to create a custom runtime: gcloud beta app gen-config --custom -You can then modify the `Dockerfile` and `.dockerignore` as needed for you application. +You can then modify the `Dockerfile` and `.dockerignore` as needed for you +application. -## Container Engine & other Docker hosts. +## Kubernetes Engine & other Docker hosts. -For other docker hosts, you'll need to create a `Dockerfile` based on this image that copies your application code, installs dependencies, and declares an command or entrypoint. For example: +For other docker hosts, you'll need to create a `Dockerfile` based on this image +that copies your application code, installs dependencies, and declares an +command or entrypoint. For example: - FROM gcr.io/google_appengine/python + FROM gcr.io/google-appengine/python # Create a virtualenv for dependencies. This isolates these packages from # system-level packages. + # Use -p python3 or -p python3.7 to select python version. Default is version 2. RUN virtualenv /env # Setting these environment variables are the same as running @@ -51,6 +68,13 @@ For other docker hosts, you'll need to create a `Dockerfile` based on this image # a dependency in requirements.txt. CMD gunicorn -b :$PORT main:app +## Building the image + +Google regularly builds and releases this image at +[`gcr.io/google-appengine/python`](https://gcr.io/google-appengine/python). + +See [RELEASING.md](RELEASING.md) for more information. + ## Contributing changes * See [CONTRIBUTING.md](CONTRIBUTING.md) diff --git a/RELEASING.md b/RELEASING.md new file mode 100644 index 00000000..f886a542 --- /dev/null +++ b/RELEASING.md @@ -0,0 +1,158 @@ +# Google Cloud Platform - Python Runtime Docker Image + +## `build.sh` + +There is a shell script called `build.sh` that builds everything in this +repository. + +### Environment variables for `build.sh` + +DOCKER_NAMESPACE +: The prefix applied to all images names created. To push images to Google +Container Registry (GCR), this should be `gcr.io/YOUR-PROJECT-NAME`. + +TAG +: The suffix applied to all images created. This should be unique. If not +specified, the current time will be used (timestamp format `YYYY-mm-dd-HHMMSS`). + +GOOGLE_APPLICATION_CREDENTIALS_FOR_TESTS +: (System test only) Path to service account credentials in JSON format. + +GOOGLE_CLOUD_PROJECT_FOR_TESTS +: (System test only) Name of the Google Cloud Platform project to run the system +tests under. + +## Building and Releasing + +A custom Jenkins job builds and releases this repository using scripts and job +configurations that are not yet available publicly. The control flow is as +follows: + +1. Jenkins job `python/release` is invoked by + a. Manually running the script `build_and_release.py` with arguments + b. Manually invoking the job from the GUI +2. The job runs the script `release.sh` + a. Service account credentials are read + b. `gcloud auth activate-service-account` is performed + c. `gcloud config set project` is performed +3. The script invokes `build.sh` in this repository +4. `build.sh` invokes Google Cloud Build with the `cloudbuild-*.yaml` + config files. + +## Building interpreters + +The interpreters used are now built in a separate step, and stored on GCS. +This allows the runtime images to be build more rapidly. + +To build the interpreters, run: + +```shell +gcloud builds submit . --config=cloudbuild_interpreters.yaml +``` + +## Building outside Jenkins + +To build this repository outside Jenkins, authenticate and authorize yourself +with `gcloud auth`, set the variables listed above, and run: + +``` shell +./build.sh +``` + +This assumes an environment similar to the internal Jenkins environment (Linux, +Debian or Ubuntu-like). + +## Building locally + +To build this repository using local Docker commands instead of the Google +Cloud Build service, add the `--local` flag as shown: + +``` shell +./build.sh --local +``` + +To open an interactive shell session to this image after building it, do the +following: + +``` shell +docker run -it --entrypoint /bin/bash YOUR-IMAGE-NAME +``` + +## Running tests against a released image + +To run compatibility tests against an existing image, such as +`gcr.io/google-appengine/python:latest`, run: + +```shell +DOCKER_NAMESPACE=gcr.io/google-appengine TAG=latest ./build.sh --nobuild --test +``` + +## Running benchmarks + +There is a benchmark suite which compares the performance of interpreters +against each other. + +**Benchmark different versions of interpreter in the same release + +``` shell +DOCKER_NAMESPACE=DOCKER_NAMESPACE_EXAMPLE TAG=TAG_EXAMPLE ./build.sh --nobuild --benchmark +``` + +**Benchmark same versions of interpreter from release to release + +``` shell +DOCKER_NAMESPACE=DOCKER_NAMESPACE_EXAMPLE TAG1=TAG1_EXAMPLE TAG2=TAG2_EXAMPLE ./benchmark_between_releases.sh +``` + +Since these benchmarks are run on cloud instances, the timings may vary from run +to run. + +## Running system tests + +**TAKE NOTE: You will incur charges for use of Google Cloud Platform services!** + +System tests perform mutating operations against the real Google Cloud services. +Since these system tests may fail or be flaky for outside reasons such as +netorking issues, configuration errors, or services outages, they are run +separately from building the images, and should be run in their own project. + +To run the system tests, you need a Google Cloud Project with a service account. +From the [Google Cloud Console](https://console.cloud.google.com/), either +create a new project or switch to an existing one. Next, +[create a service account]( +https://cloud.google.com/iam/docs/creating-managing-service-accounts) that will +be used to run the system tests. Once you have a service account, +[create and download a service account key](https://cloud.google.com/iam/docs/managing-service-account-keys). + +In the +[IAM & Admin](https://console.cloud.google.com/permissions/projectpermissions) +section, grant the `Owner` role to the service account you created above. Also +grant the `Editor` role to the `cloud-logs@google.com` service account. + +Then, follow the +[system test setup instructions](https://github.com/GoogleCloudPlatform/google-cloud-python/blob/master/CONTRIBUTING.rst#running-system-tests). It +describes various steps, including running some scripts to populate and/or +delete datastore example data and indexes (populate_datastore.py, +clear_datastore.py, and `gcloud preview datastore create-indexes +system_tests/data/index.yaml`). + +From the cloud console, you will need to enable at least the following APIs for +your project: + +- Bigquery API +- Cloud Bigtable Admin API +- Cloud Spanner API +- Google Cloud Natural Language API +- Google Cloud Pub/Sub API +- Google Cloud Speech API +- Google Cloud Storage JSON API +- Google Cloud Translation API +- Google Cloud Vision API +- Stackdriver Logging API +- Stackdriver Monitoring API + +Once all the setup has been done, run the following: + +``` shell +./build.sh --nobuild --system_tests +``` diff --git a/build.sh b/build.sh new file mode 100755 index 00000000..2083237d --- /dev/null +++ b/build.sh @@ -0,0 +1,243 @@ +#!/bin/bash + +# Copyright 2016 Google Inc. All rights reserved. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +# Actions +benchmark=0 # Should run benchmarks? +build=0 # Should build images? +client_test=0 # Should run Google Cloud Client Library tests +test=0 # Should run standard test suite? + +local=0 # Should run using local Docker daemon instead of GCR? + +os_base=ubuntu18 # Which operating system base to use +interpreter=0 # Should build interpreters instead of images + +# Note that $gcloud_cmd has spaces in it +gcloud_cmd="gcloud builds submit" +# May need to install via "gcloud components install cloud-build-local" +local_gcloud_cmd="cloud-build-local --push=false --dryrun=false" + +# Helper functions +function fatal() { + echo "$1" >&2 + exit 1 +} + +function usage { + fatal "Usage: $0 [OPTION]... +Build and test artifacts in this repository + +Options: + --[no]benchmark: Run benchmarking suite (default false) + --[no]build: Build all images (default true if no options set) + --[no]test: Run basic tests (default true if no options set) + --[no]client_test: Run Google Cloud Client Library tests (default false) + --[no]local: Build images using local Docker daemon (default false) + --os_base: Which OS image to build on top of [debian8, ubuntu16, ubuntu18] +" +} + +# Read environment variables +if [ -z "${DOCKER_NAMESPACE:+set}" ] ; then + fatal 'Error: $DOCKER_NAMESPACE is not set; invoke with something like DOCKER_NAMESPACE=gcr.io/YOUR-PROJECT-NAME' +fi + +if [ -z "${BUILDER_DOCKER_NAMESPACE:+set}" ] ; then + export BUILDER_DOCKER_NAMESPACE="${DOCKER_NAMESPACE}" +fi + +if [ -z "${TAG:+set}" ] ; then + export TAG=`date +%Y-%m-%d-%H%M%S` +fi + +build_substitutions="\ +_BUILDER_DOCKER_NAMESPACE=${BUILDER_DOCKER_NAMESPACE},\ +_DOCKER_NAMESPACE=${DOCKER_NAMESPACE},\ +_TAG=${TAG}\ +" + +substitutions="\ +_DOCKER_NAMESPACE=${DOCKER_NAMESPACE},\ +_TAG=${TAG}\ +" + +# Read command line arguments +while [ $# -gt 0 ]; do + case "$1" in + --benchmark) + benchmark=1 + shift + ;; + --nobenchmark) + benchmark=0 + shift + ;; + --build) + build=1 + shift + ;; + --nobuild) + build=0 + shift + ;; + --client_test) + client_test=1 + shift + ;; + --noclient_test) + client_test=0 + shift + ;; + --local) + local=1 + shift + ;; + --nolocal) + local=0 + shift + ;; + --os_base=debian8) + os_base=debian8 + shift + ;; + --os_base=ubuntu16) + os_base=ubuntu16 + shift + ;; + --os_base=ubuntu18) + os_base=ubuntu18 + shift + ;; + --test) + test=1 + shift + ;; + --notest) + test=0 + shift + ;; + --interpreter) + interpreter=1 + shift + ;; + *) + usage + ;; + esac +done + +# If no actions chosen, then tell the user +if [ "${benchmark}" -eq 0 -a \ + "${build}" -eq 0 -a \ + "${client_test}" -eq 0 -a \ + "${test}" -eq 0 \ +]; then + echo 'No actions specified, defaulting to --build --test' + build=1 + test=1 +fi + +# Running build local or remote? +if [ "${local}" -eq 1 ]; then + gcloud_cmd="${local_gcloud_cmd}" +fi + +# Pick OS image to use as base +if [ "${os_base}" == "debian8" ]; then + export OS_BASE_IMAGE="gcr.io/google-appengine/debian8:latest" +elif [ "${os_base}" == "ubuntu16" ]; then + export OS_BASE_IMAGE="gcr.io/gcp-runtimes/ubuntu_16_0_4:latest" +elif [ "${os_base}" == "ubuntu18" ]; then + export OS_BASE_IMAGE="gcr.io/gcp-runtimes/ubuntu_18_0_4:latest" +else + echo "Unsupported OS base image: $OS_BASE_IMAGE" + exit 1 +fi +export STAGING_IMAGE="${DOCKER_NAMESPACE}/python:${TAG}" +echo "Using base image name ${STAGING_IMAGE}" + +# Generate Dockerfiles +for outfile in \ + builder/gen-dockerfile/Dockerfile \ + python-interpreter-builder/Dockerfile \ + runtime-image/Dockerfile \ + tests/benchmark/Dockerfile \ + tests/eventlet/Dockerfile \ + tests/google-cloud-python/Dockerfile \ + tests/integration/Dockerfile \ + ; do + envsubst <"${outfile}".in >"${outfile}" \ + '$OS_BASE_IMAGE $STAGING_IMAGE $GOOGLE_CLOUD_PROJECT_FOR_TESTS $TAG' +done + +# Make some files available to the runtime builder Docker context +mkdir -p builder/gen-dockerfile/data +for file in \ + scripts/gen_dockerfile.py \ + scripts/validation_utils.py \ + scripts/data/* \ + ; do + cp -a "${file}" "builder/gen-dockerfile/${file##scripts/}" +done + +# Make a file available to the eventlet test. +cp -a scripts/testdata/hello_world/main.py tests/eventlet/main.py + +# Build interpreters and push to GCS +if [ "${interpreter}" -eq 1 ]; then + echo "Building interpreters" + ${gcloud_cmd} \ + --config=cloudbuild_interpreters.yaml \ + . +fi + +# Build images and push to GCR +if [ "${build}" -eq 1 ]; then + echo "Building images" + ${gcloud_cmd} \ + --config=cloudbuild.yaml \ + --substitutions="${build_substitutions}" \ + . +fi + +# Run the tests that don't require (too many) external services +if [ "${test}" -eq 1 ]; then + echo "Testing compatibility with popular Python libraries" + ${gcloud_cmd} \ + --config=cloudbuild_test.yaml \ + --substitutions="${substitutions}" \ + . +fi + +# Run client library tests +if [ "${client_test}" -eq 1 ]; then + echo "Testing compatibility with Google Cloud Client Libraries" + ${gcloud_cmd} \ + --config=cloudbuild_client_test.yaml \ + --substitutions="${substitutions}" \ + . +fi + +# Run benchmarks +if [ "${benchmark}" -eq 1 ] ; then + echo "Running benchmark" + ${gcloud_cmd} \ + --config=cloudbuild_benchmark.yaml \ + --substitutions="${substitutions}" \ + . +fi diff --git a/builder/gen-dockerfile/.gitignore b/builder/gen-dockerfile/.gitignore new file mode 100644 index 00000000..f45549bd --- /dev/null +++ b/builder/gen-dockerfile/.gitignore @@ -0,0 +1,3 @@ +Dockerfile +*.py +data/ diff --git a/builder/gen-dockerfile/Dockerfile.in b/builder/gen-dockerfile/Dockerfile.in new file mode 100644 index 00000000..92ceaecf --- /dev/null +++ b/builder/gen-dockerfile/Dockerfile.in @@ -0,0 +1,15 @@ +FROM ${STAGING_IMAGE} +LABEL python_version=python3.7 +RUN virtualenv --no-download /env -p python3.7 + +# Set virtualenv environment variables. This is equivalent to running +# source /env/bin/activate +ENV VIRTUAL_ENV /env +ENV PATH /env/bin:$PATH +ADD requirements.txt /builder/ +#virtualenv's pip is pegged at version 10.0, removing so +#newer versions get picked up +RUN pip install -r /builder/requirements.txt +ADD . /builder/ +WORKDIR /workspace +ENTRYPOINT [ "python", "/builder/gen_dockerfile.py" ] diff --git a/builder/gen-dockerfile/requirements.txt b/builder/gen-dockerfile/requirements.txt new file mode 100644 index 00000000..4a285555 --- /dev/null +++ b/builder/gen-dockerfile/requirements.txt @@ -0,0 +1 @@ +PyYAML==3.13 diff --git a/builder/python-latest.yaml b/builder/python-latest.yaml new file mode 100644 index 00000000..3e75acad --- /dev/null +++ b/builder/python-latest.yaml @@ -0,0 +1,12 @@ +# This is a cloudbuild.yaml template for the runtime builder +steps: +- # Generate application Dockerfile + name: 'gcr.io/gcp-runtimes/python/gen-dockerfile:latest' + args: [ + '--base-image=gcr.io/google-appengine/python:latest' + ] +- # Use that Dockerfile to create final application image + name: 'gcr.io/cloud-builders/docker:latest' + args: ['build', '-t', '$_OUTPUT_IMAGE', '.'] +images: + - '$_OUTPUT_IMAGE' diff --git a/builder/python-staging.yaml b/builder/python-staging.yaml new file mode 100644 index 00000000..1e977a89 --- /dev/null +++ b/builder/python-staging.yaml @@ -0,0 +1,12 @@ +# This is a cloudbuild.yaml template for the runtime builder +steps: +- # Generate application Dockerfile + name: 'gcr.io/gcp-runtimes/python/gen-dockerfile:staging' + args: [ + '--base-image=gcr.io/google-appengine/python:staging' + ] +- # Use that Dockerfile to create final application image + name: 'gcr.io/cloud-builders/docker:latest' + args: ['build', '-t', '$_OUTPUT_IMAGE', '.'] +images: + - '$_OUTPUT_IMAGE' diff --git a/cloudbuild.yaml b/cloudbuild.yaml new file mode 100644 index 00000000..e240780e --- /dev/null +++ b/cloudbuild.yaml @@ -0,0 +1,17 @@ +timeout: 10800s +steps: +- # Build base runtime image + name: gcr.io/cloud-builders/docker:latest + args: ['build', '--tag=${_DOCKER_NAMESPACE}/python:${_TAG}', + '--no-cache', '/workspace/runtime-image/'] + id: runtime +- # Build runtime builder image + name: gcr.io/cloud-builders/docker:latest + args: ['build', '--tag=${_BUILDER_DOCKER_NAMESPACE}/python/gen-dockerfile:${_TAG}', + '--no-cache', '/workspace/builder/gen-dockerfile/'] + id: gen-dockerfile + waitFor: ['runtime'] +images: [ + '${_DOCKER_NAMESPACE}/python:${_TAG}', + '${_BUILDER_DOCKER_NAMESPACE}/python/gen-dockerfile:${_TAG}', +] diff --git a/cloudbuild_benchmark.yaml b/cloudbuild_benchmark.yaml new file mode 100644 index 00000000..a960bc9b --- /dev/null +++ b/cloudbuild_benchmark.yaml @@ -0,0 +1,8 @@ +timeout: 3600s +steps: +- name: gcr.io/cloud-builders/docker:latest + args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/benchmark:${_TAG}', + '--no-cache', '/workspace/tests/benchmark/'] +images: [ + # Intentionally empty +] diff --git a/cloudbuild_client_test.yaml b/cloudbuild_client_test.yaml new file mode 100644 index 00000000..010e1ffd --- /dev/null +++ b/cloudbuild_client_test.yaml @@ -0,0 +1,9 @@ +timeout: 3600s +steps: +- # Build image to run google client library unit tests + name: gcr.io/cloud-builders/docker:latest + args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/google-cloud-python:${_TAG}', + '--no-cache', '/workspace/tests/google-cloud-python/'] +- # Run google client library unit tests + name: ${_DOCKER_NAMESPACE}/python/tests/google-cloud-python:${_TAG} +images: [] diff --git a/cloudbuild_interpreters.yaml b/cloudbuild_interpreters.yaml new file mode 100644 index 00000000..c75e59ad --- /dev/null +++ b/cloudbuild_interpreters.yaml @@ -0,0 +1,33 @@ +timeout: 10800s +steps: +- # Compile Python interpreters from source. This step happens first, then + # the next three in parallel. + name: gcr.io/cloud-builders/docker:latest + args: ['build', '--tag=interpreter-builder', + '--no-cache', '/workspace/python-interpreter-builder/'] + id: interpreter-builder +- name: interpreter-builder + args: ['/scripts/build-python-3.4.sh'] + id: build-3.4 + waitFor: ['interpreter-builder'] +- name: interpreter-builder + args: ['/scripts/build-python-3.5.sh'] + id: build-3.5 + waitFor: ['interpreter-builder'] +- name: interpreter-builder + args: ['/scripts/build-python-3.6.sh'] + id: build-3.6 + waitFor: ['interpreter-builder'] +- name: interpreter-builder + args: ['/scripts/build-python-3.7.sh'] + id: build-3.7 + waitFor: ['interpreter-builder'] + +# Upload them to tbe build-id location +- name: gcr.io/cloud-builders/gsutil:latest + args: ['cp', '/workspace/runtime-image/*.tar.gz', 'gs://python-interpreters/$BUILD_ID/'] + waitFor: ['build-3.4', 'build-3.5', 'build-3.6', 'build-3.7'] + +# "Tag" this as latest +- name: gcr.io/cloud-builders/gsutil:latest + args: ['cp', '-r', 'gs://python-interpreters/$BUILD_ID/*', 'gs://python-interpreters/latest/'] diff --git a/cloudbuild_test.yaml b/cloudbuild_test.yaml new file mode 100644 index 00000000..ad674026 --- /dev/null +++ b/cloudbuild_test.yaml @@ -0,0 +1,114 @@ +timeout: 3600s +steps: +- # Explicitly pull image into GCB so that later steps work + name: '${_DOCKER_NAMESPACE}/python:${_TAG}' + args: [ + '/bin/true', + ] + id: runtime + +- # Validate structure of base runtime image + name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 + args: [ + '-test.v', + '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', + '/workspace/tests/virtualenv/virtualenv_default.yaml', + ] + waitFor: ['runtime'] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 + args: [ + '-test.v', + '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', + '/workspace/tests/virtualenv/virtualenv_python27.yaml', + ] + waitFor: ['runtime'] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 + args: [ + '-test.v', + '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', + '/workspace/tests/virtualenv/virtualenv_python34.yaml', + ] + waitFor: ['runtime'] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 + args: [ + '-test.v', + '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', + '/workspace/tests/virtualenv/virtualenv_python35.yaml', + ] + waitFor: ['runtime'] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 + args: [ + '-test.v', + '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', + '/workspace/tests/virtualenv/virtualenv_python36.yaml', + ] + waitFor: ['runtime'] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 + args: [ + '-test.v', + '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', + '/workspace/tests/virtualenv/virtualenv_python37.yaml', + ] + waitFor: ['runtime'] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 + args: [ + '-test.v', + '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', + '/workspace/tests/no-virtualenv/no-virtualenv.yaml', + ] + waitFor: ['runtime'] + +# Temporarily disabled because it fails on symbolic links in Ubuntu: +# https://github.com/GoogleCloudPlatform/container-structure-test/issues/77 +#- # Check license compliance +# name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 +# args: [ +# '-test.v', +# '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', +# '/workspace/tests/license-test/license-test.yaml' +# ] +# waitFor: ['runtime'] + +- # Do third-party library compatibility tests for Python 2 + name: gcr.io/cloud-builders/docker:latest + args: [ + 'build', '-t', 'python2-libraries-intermediate', '--build-arg', + 'intermediate_image=${_DOCKER_NAMESPACE}/python:${_TAG}', + '/workspace/tests/python2-libraries' + ] + id: python2-libraries-intermediate + waitFor: ['runtime'] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 + args: [ + '-test.v', + '-image', 'python2-libraries-intermediate', + '/workspace/tests/python2-libraries/python2-libraries.yaml' + ] + waitFor: ['python2-libraries-intermediate'] + +- # Do third-party library compatibility tests for Python 3 + name: gcr.io/cloud-builders/docker:latest + args: [ + 'build', '-t', 'python3-libraries-intermediate', '--build-arg', + 'intermediate_image=${_DOCKER_NAMESPACE}/python:${_TAG}', + '/workspace/tests/python3-libraries' + ] + id: python3-libraries-intermediate + waitFor: ['runtime'] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 + args: [ + '-test.v', + '-image', 'python3-libraries-intermediate', + '/workspace/tests/python3-libraries/python3-libraries.yaml' + ] + waitFor: ['python3-libraries-intermediate'] + +- # Run other compatibility tests + name: gcr.io/cloud-builders/docker:latest + args: [ + 'build', '--tag=${_DOCKER_NAMESPACE}/python/tests/eventlet:${_TAG}', + '--no-cache', '/workspace/tests/eventlet/' + ] + waitFor: ['runtime'] + +images: [] diff --git a/nox.py b/nox.py new file mode 100644 index 00000000..0ee6f443 --- /dev/null +++ b/nox.py @@ -0,0 +1,86 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import fnmatch +import os + +import nox + + +def _list_files(folder, pattern): + """Lists all files below the given folder that match the pattern.""" + for root, folders, files in os.walk(folder): + for filename in files: + if fnmatch.fnmatch(filename, pattern): + yield os.path.join(root, filename) + + +@nox.session +def check_requirements(session): + """Checks for out of date requirements and optionally updates them.""" + session.install('gcp-devrel-py-tools') + + if 'update' in session.posargs: + command = 'update-requirements' + else: + command = 'check-requirements' + + reqfiles = list(_list_files('.', 'requirements*.txt')) + + for reqfile in reqfiles: + session.run('gcp-devrel-py-tools', command, reqfile) + + +@nox.session +def lint(session): + session.interpreter = 'python3' # So it understands Python3 syntax + session.install('flake8', 'flake8-import-order') + session.run( + 'flake8', + '--import-order-style', 'google', + '--application-import-names', + 'gen_dockerfile,local_cloudbuild,validation_utils', + 'scripts', + 'nox.py', + ) + + +@nox.session +@nox.parametrize('version', ['3.4', '3.5', '3.6', '3.7']) +def tests(session, version): + session.interpreter = 'python' + version + session.install('-r', 'scripts/requirements-test.txt') + session.run( + 'py.test', + '--ignore=scripts/testdata', + '--cov=scripts', + '--cov-append', + '--cov-config=.coveragerc', + '--cov-report=', # Report generated below + 'scripts', + env={'PYTHONPATH': ''} + ) + + +@nox.session +def cover(session): + """Run the final coverage report. + + This outputs the coverage report aggregating coverage from the unit + test runs (not system test runs), and then erases coverage data. + """ + session.interpreter = 'python3.6' + session.install('coverage', 'pytest-cov') + session.run('coverage', 'report', '--show-missing', '--fail-under=97') + session.run('coverage', 'erase') diff --git a/perf_dashboard/__init__.py b/perf_dashboard/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/perf_dashboard/bq_utils.py b/perf_dashboard/bq_utils.py new file mode 100644 index 00000000..cbad65b9 --- /dev/null +++ b/perf_dashboard/bq_utils.py @@ -0,0 +1,36 @@ +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Common util methods for processing data in BigQuery.""" + +import uuid + +from google.cloud import bigquery + + +def insert_rows(project, dataset_name, table_name, rows): + """Insert rows to bigquery table.""" + client = bigquery.Client(project=project) + dataset_ref = client.dataset(dataset_name) + table_ref = dataset_ref.table(table_name) + table = client.get_table(table_ref) + client.create_rows(table, rows) + +def execute_query(query): + """Execute query and return the query results.""" + client = bigquery.Client() + query_job = client.query((query)) + + # Start the query job and wait it to complete + return [row.values() for row in query_job.result()] diff --git a/perf_dashboard/posts_stats.py b/perf_dashboard/posts_stats.py new file mode 100644 index 00000000..efb9c6fc --- /dev/null +++ b/perf_dashboard/posts_stats.py @@ -0,0 +1,105 @@ +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A script to collect the number of StackOverflow posts related to +Python and Google Cloud Platform.""" + +import datetime +import os +import sys +import time +import uuid + +from collections import Counter + +from google.cloud import bigquery + +import bq_utils + +GCLOUD_PROJECT_ENV = 'GCLOUD_PROJECT' +DATASET_NAME = 'stackoverflow' +TAG_COUNT_TABLE_NAME = 'tag_count_timestamp' +UNANSWERED_POSTS_TABLE_NAME = 'unanswered_posts' + + +def get_stackoverflow_tags_count(): + """Get all the tags contains python and cloud key words""" + query = """ + SELECT + SPLIT(tags, '|') tags + FROM + `bigquery-public-data.stackoverflow.posts_questions` + WHERE + tags LIKE '%python%' + AND (tags LIKE '%google-cloud-platform%' OR tags LIKE '%gcp%') + """ + + results = bq_utils.execute_query(query) + + rows = [row[0] for row in results] + + return rows + + +def get_posts_list_unanswered(): + # Get the list of posts that are unanswered + query = """ + SELECT + id, title, tags + FROM + `bigquery-public-data.stackoverflow.posts_questions` + WHERE + tags LIKE '%python%' + AND (tags LIKE '%google-cloud-platform%' OR tags LIKE '%gcp%') + AND accepted_answer_id is NULL + AND answer_count = 0; + """ + + results = bq_utils.execute_query(query) + + # Add current timestamp to the rows + date_time = datetime.datetime.now() + rows = [(date_time,) + row for row in results] + + return rows + + +def count_unique_tags(data): + flattened_tag_list = [tag for tag_list in data for tag in tag_list] + tag_count = Counter(flattened_tag_list) + + # Add current timestamp to the rows + date_time = datetime.datetime.now() + time_tag_count = [(date_time,) + item for item in tag_count.items()] + + return time_tag_count + + +def main(): + project = os.environ.get(GCLOUD_PROJECT_ENV) + + # Get the posts count for each tag + rows = get_stackoverflow_tags_count() + tag_count = count_unique_tags(rows) + bq_utils.insert_rows( + project, DATASET_NAME, TAG_COUNT_TABLE_NAME, tag_count) + + # Get the list of unanswered posts + unanswered_posts = get_posts_list_unanswered() + bq_utils.insert_rows( + project, DATASET_NAME, UNANSWERED_POSTS_TABLE_NAME, unanswered_posts) + + +if __name__ == '__main__': + main() diff --git a/perf_dashboard/python_clientlibs_download.py b/perf_dashboard/python_clientlibs_download.py new file mode 100644 index 00000000..cae3d2c4 --- /dev/null +++ b/perf_dashboard/python_clientlibs_download.py @@ -0,0 +1,135 @@ +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import os +import sys +import time +import uuid + +from google.cloud import bigquery + +import bq_utils + +GCLOUD_PROJECT_ENV = 'GCLOUD_PROJECT' + +DATETIME_FORMAT = '%Y%m%d' + +DATASET_NAME = 'python_clientlibs_download_by_week' + +VENEER_TABLE_NAME = 'veneer_client_libs' +STACKDRIVER_TABLE_NAME = 'stackdriver_client_libs' +GRPC_TABLE_NAME = 'grpc_lib' +THIRD_PARTY_TABLE_NAME = 'third_party_client_libs' + +TABLES = [ + VENEER_TABLE_NAME, + GRPC_TABLE_NAME, + STACKDRIVER_TABLE_NAME, + THIRD_PARTY_TABLE_NAME, +] + +CLIENTLIBS = { + VENEER_TABLE_NAME: [ + 'google-cloud-core', + 'google-cloud-speech', + 'google-cloud-language', + 'google-cloud-pubsub', + 'google-cloud-bigquery', + 'google-cloud-bigtable', + 'google-cloud-datastore', + 'google-cloud-spanner', + 'google-cloud-storage', + 'google-cloud-vision', + 'google-cloud-translate', + 'google-cloud-dns', + 'google-cloud-videointelligence', + ], + STACKDRIVER_TABLE_NAME: [ + 'google-cloud-logging', + 'google-cloud-monitoring', + 'google-cloud-error_reporting', + 'google-cloud-trace', + ], + GRPC_TABLE_NAME: [ + 'grpcio', + ], + THIRD_PARTY_TABLE_NAME: [ + 'pandas-gbq', + ] +} + + +def get_weekly_clientlibs_downloads(clientlibs_table_name, date_str): + """Use a SQL query to collect the weekly download data of the client + libraries. + + Args: + clientlibs_table_name (str): Table name, which is the key in the + CLIENTLIBS dict. + date_str (str): A date string in "YYYYMMDD" format. + + Returns: + list: rows of the query result. + """ + client_libs = CLIENTLIBS[clientlibs_table_name] + date_time = datetime.datetime.strptime(date_str, DATETIME_FORMAT) + week_dates = [(date_time + datetime.timedelta(days=-i)) + .strftime(DATETIME_FORMAT) + for i in range(7)] + query = """ + SELECT + file.project as client_library_name, + COUNT(*) as download_count + FROM + `the-psf.pypi.downloads*` + WHERE + file.project IN UNNEST(@client_libs) + AND + _TABLE_SUFFIX IN UNNEST(@week_dates) + GROUP BY client_library_name + """ + client = bigquery.Client() + query_parameters=[ + bigquery.ArrayQueryParameter( + 'client_libs', 'STRING', client_libs), + bigquery.ArrayQueryParameter( + 'week_dates', 'STRING', week_dates) + ] + job_config = bigquery.QueryJobConfig() + job_config.query_parameters = query_parameters + query_job = client.query(query, job_config=job_config) + + # Wait for the job to complete and get the results + results = [row.values() for row in query_job.result()] + + rows = [(date_time,) + row for row in results] + + return rows + + +def main(): + for table_name in CLIENTLIBS.keys(): + rows = get_weekly_clientlibs_downloads( + clientlibs_table_name=table_name, + date_str=datetime.datetime.now().strftime("%Y%m%d")) + bq_utils.insert_rows( + project=os.environ.get(GCLOUD_PROJECT_ENV), + dataset_name=DATASET_NAME, + table_name=table_name, + rows=rows) + + +if __name__ == '__main__': + main() diff --git a/python-interpreter-builder/.dockerignore b/python-interpreter-builder/.dockerignore new file mode 100644 index 00000000..e69de29b diff --git a/python-interpreter-builder/.gitignore b/python-interpreter-builder/.gitignore new file mode 100644 index 00000000..94143827 --- /dev/null +++ b/python-interpreter-builder/.gitignore @@ -0,0 +1 @@ +Dockerfile diff --git a/python-interpreter-builder/DEBIAN/control.in b/python-interpreter-builder/DEBIAN/control.in new file mode 100644 index 00000000..49980654 --- /dev/null +++ b/python-interpreter-builder/DEBIAN/control.in @@ -0,0 +1,25 @@ +Package: ${DEB_PACKAGE_NAME} +Version: ${DEB_PACKAGE_VERSION} +Section: python +Priority: optional +Architecture: amd64 +Maintainer: Douglas Greiman +Description: Interactive high-level object-oriented language (version ${SHORT_VERSION}) + Python is a high-level, interactive, object-oriented language. Its ${SHORT_VERSION} version + includes an extensive class library with lots of goodies for + network programming, system administration, sounds and graphics. +Depends: libbz2-1.0, + libc6, + libdb5.3, + libexpat1, + libffi6, + liblzma5, + libmpdec2, + libncursesw5, + libreadline6, + libsqlite3-0, + libssl1.0.0, + libtinfo5, + mime-support, + zlib1g +Homepage: https://www.python.org diff --git a/python-interpreter-builder/Dockerfile.in b/python-interpreter-builder/Dockerfile.in new file mode 100644 index 00000000..9039fbf3 --- /dev/null +++ b/python-interpreter-builder/Dockerfile.in @@ -0,0 +1,48 @@ +# The Google App Engine base image is debian (jessie) with ca-certificates +# installed. +FROM ${OS_BASE_IMAGE} + +# Install Python build dependencies (based on Debian Build-Depends) +RUN apt-get update && apt-get install -yq \ + autoconf \ + blt-dev \ + bzip2 \ + debhelper \ + dpkg-dev \ + gcc \ + gettext-base \ + libbluetooth-dev \ + libbz2-dev \ + libdb-dev \ + libexpat1-dev \ + libffi-dev \ + libgdbm-dev \ + libgpm2 \ + liblzma-dev \ + libmpdec-dev \ + libncursesw5-dev \ + libreadline-dev \ + libsqlite3-dev \ + libssl-dev \ + locales \ + lsb-release \ + mime-support \ + net-tools \ + netbase \ + python \ + python3 \ + sharutils \ + time \ + tk-dev \ + wget \ + xauth \ + xvfb \ + zlib1g-dev \ + && rm -rf /var/lib/apt/lists/* + +# Setup locale. This prevents Python 3 IO encoding issues. +ENV LANG C.UTF-8 + +# Add build scripts +ADD scripts /scripts +ADD DEBIAN /DEBIAN diff --git a/python-interpreter-builder/README.md b/python-interpreter-builder/README.md new file mode 100644 index 00000000..7b718581 --- /dev/null +++ b/python-interpreter-builder/README.md @@ -0,0 +1,21 @@ +# Python Interpreter Builder + +This is a Docker-based Python interpreter builder. It builds Python interpreters +using a Debian-based Docker image. These interpreters are suitable to be moved +to another Debian-based Docker image. This avoids needing to install build +dependencies in the final container. + + +## Building + +Use: + + docker build --tag=google/python/interpreter-builder . + +The interpreters will be stored in the image at `/interpreters.tar.gz`. This is +suitable to be extracted from this image and added directly to another Docker +image via: + + ADD interpreters.tar.gz / + +Docker will automatically un-tar the interpreters into `/opt`. diff --git a/python-interpreter-builder/scripts/build-python-3.4.sh b/python-interpreter-builder/scripts/build-python-3.4.sh new file mode 100755 index 00000000..5c0bfb8e --- /dev/null +++ b/python-interpreter-builder/scripts/build-python-3.4.sh @@ -0,0 +1,141 @@ +#!/bin/bash + +set -euo pipefail +set -x + +# Get the source +mkdir -p /opt/sources +cd /opt/sources +wget --no-verbose https://www.python.org/ftp/python/3.4.8/Python-3.4.8.tgz +# SHA-256 generated via `shasum -a 256 [file]` +shasum --check <&2 + exit 1 +} + # Process command line +if [ -z "${1:+set}" -o -z "${2:+set}" ]; then + usage +fi +LONG_VERSION=$1 +BUILD_TAG=$2 +SHORT_VERSION=${1%.*} + +# Compute version specs +DEB_PACKAGE_NAME=gcp-python${SHORT_VERSION} +# Can't have - (hyphen) in debian revision as per +# https://www.debian.org/doc/debian-policy/ch-controlfields.html#s-f-Version +DEBIAN_REVISION=${BUILD_TAG//-/.} +DEB_PACKAGE_VERSION=${LONG_VERSION}-${DEBIAN_REVISION} + +PACKAGE_DIR=/opt/packages +# E.g. gcp-python3.6_3.6.2-1gcp~2017.07.25.110644_amd64.deb +DEB_FILENAME=${DEB_PACKAGE_NAME}_${DEB_PACKAGE_VERSION}_amd64.deb + +# Create directory for intermediate files +SCRATCH_DIR=$(mktemp --directory) +cd "${SCRATCH_DIR}" + +# Synthesize Debian control file. Note that the "Depends:" is +# currently Debian8-specific, and lacks version specifiers present in +# the standard Debian Python packages. +export DEB_PACKAGE_NAME DEB_PACKAGE_VERSION SHORT_VERSION +envsubst control \ + '${DEB_PACKAGE_NAME} ${DEB_PACKAGE_VERSION} ${SHORT_VERSION}' + +# Generate components of .deb archive +tar czf control.tar.gz control +tar czf data.tar.gz "/opt/python${SHORT_VERSION}" +echo "2.0" >debian-binary + +# Generate final .deb. +mkdir -p "${PACKAGE_DIR}" +ar rcD "${PACKAGE_DIR}/${DEB_FILENAME}" \ + debian-binary control.tar.gz data.tar.gz +rm debian-binary control.tar.gz data.tar.gz + +# Validate .deb +dpkg --install --dry-run "${PACKAGE_DIR}/${DEB_FILENAME}" + +# Add to list +echo "${DEB_FILENAME}" >> "${PACKAGE_DIR}/packages.txt" diff --git a/runtime-image/.gitignore b/runtime-image/.gitignore new file mode 100644 index 00000000..94143827 --- /dev/null +++ b/runtime-image/.gitignore @@ -0,0 +1 @@ +Dockerfile diff --git a/runtime-image/Dockerfile.in b/runtime-image/Dockerfile.in new file mode 100644 index 00000000..46387705 --- /dev/null +++ b/runtime-image/Dockerfile.in @@ -0,0 +1,57 @@ +# The Google App Engine base image is debian (jessie) with ca-certificates +# installed. +# Source: https://github.com/GoogleCloudPlatform/debian-docker +FROM ${OS_BASE_IMAGE} + +ADD resources /resources +ADD scripts /scripts + +# Install Python, pip, and C dev libraries necessary to compile the most popular +# Python libraries. +RUN /scripts/install-apt-packages.sh +RUN curl "https://bootstrap.pypa.io/pip/2.7/get-pip.py" -o "get-pip.py" && python ./get-pip.py && ln -s /usr/local/bin/pip /usr/bin/pip + +# Setup locale. This prevents Python 3 IO encoding issues. +ENV LANG C.UTF-8 +# Make stdout/stderr unbuffered. This prevents delay between output and cloud +# logging collection. +ENV PYTHONUNBUFFERED 1 + +RUN wget https://storage.googleapis.com/python-interpreters/latest/interpreter-3.4.tar.gz && \ + wget https://storage.googleapis.com/python-interpreters/latest/interpreter-3.5.tar.gz && \ + wget https://storage.googleapis.com/python-interpreters/latest/interpreter-3.6.tar.gz && \ + wget https://storage.googleapis.com/python-interpreters/latest/interpreter-3.7.tar.gz && \ + tar -xzf interpreter-3.4.tar.gz && \ + tar -xzf interpreter-3.5.tar.gz && \ + tar -xzf interpreter-3.6.tar.gz && \ + tar -xzf interpreter-3.7.tar.gz && \ + rm interpreter-*.tar.gz + +# Add Google-built interpreters to the path +ENV PATH /opt/python3.7/bin:/opt/python3.6/bin:/opt/python3.5/bin:/opt/python3.4/bin:$PATH +RUN update-alternatives --install /usr/local/bin/python3 python3 /opt/python3.7/bin/python3.7 50 && \ + update-alternatives --install /usr/local/bin/pip3 pip3 /opt/python3.7/bin/pip3.7 50 + +# Upgrade pip (debian package version tends to run a few version behind) and +# install virtualenv system-wide. +RUN /usr/bin/pip install --upgrade -r /resources/requirements.txt && \ + /opt/python3.4/bin/pip3.4 install --upgrade -r /resources/requirements.txt && \ + rm -f /opt/python3.4/bin/pip /opt/python3.4/bin/pip3 && \ + /opt/python3.5/bin/pip3.5 install --upgrade -r /resources/requirements.txt && \ + rm -f /opt/python3.5/bin/pip /opt/python3.5/bin/pip3 && \ + /opt/python3.6/bin/pip3.6 install --upgrade -r /resources/requirements.txt && \ + rm -f /opt/python3.6/bin/pip /opt/python3.6/bin/pip3 && \ + /opt/python3.7/bin/pip3.7 install --upgrade -r /resources/requirements.txt && \ + rm -f /opt/python3.7/bin/pip /opt/python3.7/bin/pip3 && \ + /usr/bin/pip install --upgrade -r /resources/requirements-virtualenv.txt + +# Setup the app working directory +RUN ln -s /home/vmagent/app /app +WORKDIR /app + +# Port 8080 is the port used by Google App Engine for serving HTTP traffic. +EXPOSE 8080 +ENV PORT 8080 + +# The user's Dockerfile must specify an entrypoint with ENTRYPOINT or CMD. +CMD [] diff --git a/runtime-image/resources/apt-packages.txt b/runtime-image/resources/apt-packages.txt new file mode 100644 index 00000000..7b88f777 --- /dev/null +++ b/runtime-image/resources/apt-packages.txt @@ -0,0 +1,37 @@ +# utilities +git +mercurial +pkg-config +wget +# debian-provided interpreters +python2.7 +python2.7-dev +# Dependenies for third-party Python packages +# with C-extensions +build-essential +libcurl4-openssl-dev +libffi-dev +libjpeg-dev +libmysqlclient-dev +libpng12-dev +libpq-dev +libssl-dev +libxml2-dev +libxslt1-dev +swig +zlib1g-dev +# Needed by scipy/numpy +gfortran +libatlas-dev +libblas-dev +libfreetype6-dev +liblapack-dev +libquadmath0 +# Needed by pylibmc +libmemcached-dev +libsasl2-2 +libsasl2-dev +libsasl2-modules +sasl2-bin +# Needed by eventlet +netbase diff --git a/runtime-image/resources/requirements-virtualenv.txt b/runtime-image/resources/requirements-virtualenv.txt new file mode 100644 index 00000000..25f09c4a --- /dev/null +++ b/runtime-image/resources/requirements-virtualenv.txt @@ -0,0 +1 @@ +virtualenv==20.0.31 diff --git a/runtime-image/resources/requirements.txt b/runtime-image/resources/requirements.txt new file mode 100644 index 00000000..2d010eef --- /dev/null +++ b/runtime-image/resources/requirements.txt @@ -0,0 +1,3 @@ +pip +setuptools==40.2.0 +wheel==0.31.1 diff --git a/runtime-image/scripts/install-apt-packages.sh b/runtime-image/scripts/install-apt-packages.sh new file mode 100755 index 00000000..bafba2d6 --- /dev/null +++ b/runtime-image/scripts/install-apt-packages.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -e + +apt-get -q update + +xargs -a <(awk '/^\s*[^#]/' '/resources/apt-packages.txt') -r -- \ + apt-get install --no-install-recommends -yq + +apt-get upgrade -yq + +# Remove unneeded files. +apt-get clean +rm /var/lib/apt/lists/*_* diff --git a/scripts/data/Dockerfile.entrypoint.template b/scripts/data/Dockerfile.entrypoint.template new file mode 100644 index 00000000..f6b52bf6 --- /dev/null +++ b/scripts/data/Dockerfile.entrypoint.template @@ -0,0 +1 @@ +CMD {entrypoint} diff --git a/scripts/data/Dockerfile.install_app b/scripts/data/Dockerfile.install_app new file mode 100644 index 00000000..54c3d6cc --- /dev/null +++ b/scripts/data/Dockerfile.install_app @@ -0,0 +1 @@ +ADD . /app/ diff --git a/scripts/data/Dockerfile.preamble.template b/scripts/data/Dockerfile.preamble.template new file mode 100644 index 00000000..e4d005bd --- /dev/null +++ b/scripts/data/Dockerfile.preamble.template @@ -0,0 +1 @@ +FROM {base_image} diff --git a/scripts/data/Dockerfile.python_compat b/scripts/data/Dockerfile.python_compat new file mode 100644 index 00000000..1e4d6352 --- /dev/null +++ b/scripts/data/Dockerfile.python_compat @@ -0,0 +1,3 @@ +FROM gcr.io/google_appengine/python-compat-multicore +ADD . /app/ +RUN if [ -s requirements.txt ]; then pip install -r requirements.txt; fi diff --git a/scripts/data/Dockerfile.requirements_txt b/scripts/data/Dockerfile.requirements_txt new file mode 100644 index 00000000..f684c45c --- /dev/null +++ b/scripts/data/Dockerfile.requirements_txt @@ -0,0 +1,2 @@ +ADD requirements.txt /app/ +RUN pip install -r requirements.txt diff --git a/scripts/data/Dockerfile.virtualenv.template b/scripts/data/Dockerfile.virtualenv.template new file mode 100644 index 00000000..557b1992 --- /dev/null +++ b/scripts/data/Dockerfile.virtualenv.template @@ -0,0 +1,7 @@ +LABEL python_version=python{python_version} +RUN virtualenv --no-download /env -p python{python_version} + +# Set virtualenv environment variables. This is equivalent to running +# source /env/bin/activate +ENV VIRTUAL_ENV /env +ENV PATH /env/bin:$PATH diff --git a/scripts/data/dockerignore b/scripts/data/dockerignore new file mode 100644 index 00000000..8b927bb7 --- /dev/null +++ b/scripts/data/dockerignore @@ -0,0 +1,19 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +.dockerignore +Dockerfile +.git +.hg +.svn diff --git a/scripts/data/dockerignore.python_compat b/scripts/data/dockerignore.python_compat new file mode 100644 index 00000000..5ce5abfa --- /dev/null +++ b/scripts/data/dockerignore.python_compat @@ -0,0 +1,5 @@ +.dockerignore +Dockerfile +.git +.hg +.svn diff --git a/scripts/deploy_check.sh b/scripts/deploy_check.sh new file mode 100644 index 00000000..1e2f02c2 --- /dev/null +++ b/scripts/deploy_check.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +set -ex + +export KOKORO_GITHUB_DIR=${KOKORO_ROOT}/src/github +source ${KOKORO_GFILE_DIR}/kokoro/common.sh + +cd ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY} +if [ -n "${RUNTIME_SPEC}" -a -f app.yaml.in ]; then + sed "s|\${RUNTIME_SPEC}|${RUNTIME_SPEC}|" app.yaml.in > app.yaml +fi + +cd ${KOKORO_GFILE_DIR}/appengine/integration_tests + +sudo -E /usr/local/bin/pip install --upgrade -r requirements.txt + +if [ -f ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY}/requirements.txt ] +then + sudo -E /usr/local/bin/pip install --upgrade -r ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY}/requirements.txt +fi + +export DEPLOY_LATENCY_PROJECT='cloud-deploy-latency' + +skip_flag="" + +if [ "${SKIP_CUSTOM_LOGGING_TESTS}" = "true" -o "${SKIP_BUILDERS}" = "true" ]; then + skip_flag="$skip_flag --skip-builders" +fi + +if [ "${SKIP_XRT}" = "true" ]; then + skip_flag="$skip_flag --skip-xrt" +fi + +python deploy_check.py -d ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY} -l ${LANGUAGE} ${skip_flag} diff --git a/scripts/gen_dockerfile.py b/scripts/gen_dockerfile.py new file mode 100755 index 00000000..97da60f6 --- /dev/null +++ b/scripts/gen_dockerfile.py @@ -0,0 +1,277 @@ +#!/usr/bin/env python3 + +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Generate a Dockerfile and helper files for a Python application.""" + +import argparse +import collections +import collections.abc +import functools +import io +import os +import re +import sys + +import yaml + +import validation_utils + +# Validate characters for dockerfile image names. +# +# This roots out obvious mistakes, the full gory details are here: +# https://github.com/docker/distribution/blob/master/reference/regexp.go +IMAGE_REGEX = re.compile(r"""(?x) + ^ + [a-zA-Z0-9] # First char must be alphanumeric + [a-zA-Z0-9-_./:@+]* # Punctuation allowed after that + $ +""") + +# `entrypoint` is specified as free-form text parsed as a unix shell +# command line, which limits the sanity checking possible. We +# disallow newlines and control characters which would break the +# Dockerfile format. +PRINTABLE_REGEX = re.compile(r"""^[^\x00-\x1f]*$""") + +# Map from app.yaml "python_version" to {python_version} in Dockerfile +PYTHON_INTERPRETER_VERSION_MAP = { + '': '', # == 2.7 + '2': '', # == 2.7 + '3': '3.6', + '3.4': '3.4', + '3.5': '3.5', + '3.6': '3.6', + '3.7': '3.7', +} + +# Name of environment variable potentially set by gcloud +GAE_APPLICATION_YAML_PATH = 'GAE_APPLICATION_YAML_PATH' + +# Validated application configuration +AppConfig = collections.namedtuple( + 'AppConfig', + 'base_image dockerfile_python_version entrypoint has_requirements_txt is_python_compat' +) + + +def get_app_config(raw_config, base_image, config_file, source_dir): + """Read and validate the application runtime configuration. + + We validate the user input for security and better error messages. + + Consider parsing a yaml file which has a string value where we + expected a list. Python will happily use the string as a sequence + of individual characters, at least for a while, leading to + confusing results when it finally fails. + + We also try to prevent Dockerfile and Bash injection attacks. For + example, specifying entrypoint as "true\\nADD /etc/passwd /pwned" + would allow the user to inject arbitrary directives into the + Dockerfile, which is a support problem if nothing else. + + Args: + raw_config (dict): deserialized app.yaml + base_image (str): Docker image name to build on top of + config_file (str): Path to user's app.yaml (might be .yaml) + source_dir (str): Directory containing user's source code + + Returns: + AppConfig: valid configuration + """ + # Examine app.yaml + if not isinstance(raw_config, collections.abc.Mapping): + raise ValueError( + 'Expected {} contents to be a Mapping type, but found type "{}"'. + format(config_file, type(raw_config))) + + # Short circuit for python compat. + if validation_utils.get_field_value( + raw_config, 'runtime', str) == 'python-compat': + return AppConfig( + base_image=None, + dockerfile_python_version=None, + entrypoint=None, + has_requirements_txt=None, + is_python_compat=True) + + entrypoint = validation_utils.get_field_value( + raw_config, 'entrypoint', str) + if not PRINTABLE_REGEX.match(entrypoint): + raise ValueError( + 'Invalid "entrypoint" value in app.yaml: {!r}'.format(entrypoint)) + + # Mangle entrypoint in the same way as the Cloud SDK + # (googlecloudsdk/third_party/appengine/api/validation.py) + # + # We could handle both string ("shell form") and list ("exec + # form") but it appears that gcloud only handles string form. + if entrypoint and not entrypoint.startswith('exec '): + entrypoint = 'exec ' + entrypoint + + raw_runtime_config = validation_utils.get_field_value( + raw_config, 'runtime_config', dict) + python_version = validation_utils.get_field_value( + raw_runtime_config, 'python_version', str) + + dockerfile_python_version = PYTHON_INTERPRETER_VERSION_MAP.get( + python_version) + if dockerfile_python_version is None: + valid_versions = str(sorted(PYTHON_INTERPRETER_VERSION_MAP.keys())) + raise ValueError( + 'Invalid "python_version" field in "runtime_config" section ' + 'of app.yaml: {!r}. Valid options are: {}'. + format(python_version, valid_versions)) + + # Examine user's files + has_requirements_txt = os.path.isfile( + os.path.join(source_dir, 'requirements.txt')) + + return AppConfig( + base_image=base_image, + dockerfile_python_version=dockerfile_python_version, + entrypoint=entrypoint, + has_requirements_txt=has_requirements_txt, + is_python_compat=False) + + +def get_data(name): + """Return the contents of the named data resource + + These templates are copied from the Google Cloud SDK at + google-cloud-sdk/platform/ext-runtime/python/data + and the two should be kept in sync. + + Args: + name (str): Name of file, without directory + + Returns: + str: Contents of data file + """ + filename = os.path.join(os.path.dirname(__file__), 'data', name) + with io.open(filename, 'r', encoding='utf8') as template_file: + return template_file.read() + + +def generate_files(app_config): + """Generate a Dockerfile and helper files for an application. + + Args: + app_config (AppConfig): Validated configuration + + Returns: + dict: Map of filename to desired file contents + """ + if app_config.has_requirements_txt: + optional_requirements_txt = get_data('Dockerfile.requirements_txt') + else: + optional_requirements_txt = '' + + if app_config.entrypoint: + optional_entrypoint = get_data( + 'Dockerfile.entrypoint.template').format( + entrypoint=app_config.entrypoint) + else: + optional_entrypoint = '' + + if app_config.is_python_compat: + dockerfile = get_data('Dockerfile.python_compat') + dockerignore = get_data('dockerignore.python_compat') + else: + dockerfile = ''.join([ + get_data('Dockerfile.preamble.template').format( + base_image=app_config.base_image), + get_data('Dockerfile.virtualenv.template').format( + python_version=app_config.dockerfile_python_version), + optional_requirements_txt, + get_data('Dockerfile.install_app'), + optional_entrypoint, + ]) + dockerignore = get_data('dockerignore') + + return { + 'Dockerfile': dockerfile, + '.dockerignore': dockerignore, + } + + +def generate_dockerfile_command(base_image, config_file, source_dir): + """Write a Dockerfile and helper files for an application. + + Args: + base_image (str): Docker image name to build on top of + config_file (str): Path to user's app.yaml (might be .yaml) + source_dir (str): Directory container user's source code + """ + # Read yaml file. Does not currently support multiple services + # with configuration filenames besides app.yaml + with io.open(config_file, 'r', encoding='utf8') as yaml_config_file: + raw_config = yaml.safe_load(yaml_config_file) + + # Determine complete configuration + app_config = get_app_config(raw_config, base_image, config_file, + source_dir) + + # Generate list of filenames and their textual contents + files = generate_files(app_config) + + # Write files + for filename, contents in files.items(): + full_filename = os.path.join(source_dir, filename) + with io.open(full_filename, 'w', encoding='utf8') as outfile: + outfile.write(contents) + + +def parse_args(argv): + """Parse and validate command line flags""" + parser = argparse.ArgumentParser() + parser.add_argument( + '--base-image', + type=functools.partial( + validation_utils.validate_arg_regex, flag_regex=IMAGE_REGEX), + default='gcr.io/google-appengine/python:latest', + help='Name of Docker image to use as base') + # In some cases, gcloud sets an environment variable to indicate + # the location of the application configuration file, rather than + # using the --config flag. The order of precedence from highest + # to lowest is: + # + # 1) --config flag + # 2) $GAE_APPLICATION_YAML_PATH environment variable + # 3) a file named "app.yaml" in the current working directory + parser.add_argument( + '--config', + type=functools.partial( + validation_utils.validate_arg_regex, flag_regex=PRINTABLE_REGEX), + default=(os.environ.get(GAE_APPLICATION_YAML_PATH) or 'app.yaml'), + help='Path to application configuration file' + ) + parser.add_argument( + '--source-dir', + type=functools.partial( + validation_utils.validate_arg_regex, flag_regex=PRINTABLE_REGEX), + default='.', + help=('Application source and output directory')) + args = parser.parse_args(argv[1:]) + return args + + +def main(): + args = parse_args(sys.argv) + generate_dockerfile_command(args.base_image, args.config, args.source_dir) + + +if __name__ == '__main__': + main() diff --git a/scripts/gen_dockerfile_test.py b/scripts/gen_dockerfile_test.py new file mode 100755 index 00000000..03eaa079 --- /dev/null +++ b/scripts/gen_dockerfile_test.py @@ -0,0 +1,274 @@ +#!/usr/bin/env python3 + +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit test for gen_dockerfile.py""" + +import argparse +import filecmp +import os +import shutil +import subprocess +import unittest.mock + +import pytest +import yaml + +import gen_dockerfile + + +# Expected list of files generated +EXPECTED_OUTPUT_FILES = set(('Dockerfile', '.dockerignore')) + + +@pytest.fixture +def testdata_dir(): + testdata_dir = os.path.join(os.path.dirname(__file__), 'testdata') + assert os.path.isdir(testdata_dir), ( + 'Could not run test: testdata directory not found') + return testdata_dir + + +def compare_file(filename, dir1, dir2): + """Compare identically named files in two different directories""" + assert filecmp.cmp( + os.path.join(dir1, filename), os.path.join(dir2, filename)) + + +@pytest.mark.parametrize('app_yaml, expected', [ + # Basic app.yaml + ('env: flex', { + 'base_image': 'some_image_name', + 'dockerfile_python_version': '', + 'has_requirements_txt': False, + 'entrypoint': '', + 'is_python_compat': False, + }), + ('env: flex\nruntime: python-compat', { + 'base_image': None, + 'dockerfile_python_version': None, + 'has_requirements_txt': None, + 'entrypoint': None, + 'is_python_compat': True, + }), + # All supported python versions + ('runtime_config:\n python_version:', { + 'dockerfile_python_version': '', + }), + ('runtime_config:\n python_version: 2', { + 'dockerfile_python_version': '', + }), + ('runtime_config:\n python_version: 3', { + 'dockerfile_python_version': '3.6', + }), + ('runtime_config:\n python_version: 3.4', { + 'dockerfile_python_version': '3.4', + }), + ('runtime_config:\n python_version: 3.5', { + 'dockerfile_python_version': '3.5', + }), + ('runtime_config:\n python_version: 3.6', { + 'dockerfile_python_version': '3.6', + }), + ('runtime_config:\n python_version: 3.7', { + 'dockerfile_python_version': '3.7', + }), + # entrypoint present + ('entrypoint: my entrypoint', { + 'entrypoint': 'exec my entrypoint', + }), +]) +def test_get_app_config_valid(app_yaml, expected): + config_file = 'some_config_file' + base_image = 'some_image_name' + source_dir = 'some_source_dir' + raw_app_config = yaml.safe_load(app_yaml) + actual = gen_dockerfile.get_app_config( + raw_app_config, base_image, config_file, + source_dir) + for key, value in expected.items(): + assert getattr(actual, key) == value + + +def test_get_app_config_requirements_txt(): + """requirements.txt file present""" + app_yaml = 'env: flex' + expected = { + 'has_requirements_txt': True, + } + with unittest.mock.patch.object(os.path, 'isfile', return_value=True): + test_get_app_config_valid(app_yaml, expected) + + +@pytest.mark.parametrize('app_yaml', [ + # Empty app.yaml + '', + # Invalid entrypoint + 'entrypoint: "bad \\n entrypoint"', + # Invalid python version + 'runtime_config:\n python_version: 1', + 'runtime_config:\n python_version: python2', +]) +def test_get_app_config_invalid(app_yaml): + config_file = 'some_config_file' + base_image = 'some_image_name' + source_dir = 'some_source_dir' + raw_app_config = yaml.safe_load(app_yaml) + with pytest.raises(ValueError): + gen_dockerfile.get_app_config( + raw_app_config, base_image, config_file, source_dir) + + +# Basic AppConfig used below +_BASE_APP_CONFIG = gen_dockerfile.AppConfig( + base_image='', + dockerfile_python_version='', + entrypoint='', + has_requirements_txt=False, + is_python_compat=False, +) + + +@pytest.mark.parametrize('app_config, should_find, test_string', [ + # Requirements.txt + (_BASE_APP_CONFIG, False, 'ADD requirements.txt'), + (_BASE_APP_CONFIG._replace(has_requirements_txt=True), True, + 'ADD requirements.txt'), + # Entrypoint + (_BASE_APP_CONFIG, False, 'CMD'), + (_BASE_APP_CONFIG._replace(entrypoint='my entrypoint'), True, + 'CMD my entrypoint'), + (_BASE_APP_CONFIG._replace(entrypoint='exec my entrypoint'), True, + 'CMD exec my entrypoint'), + # Base runtime image + (_BASE_APP_CONFIG._replace(base_image='my_base_runtime_image'), True, + 'FROM my_base_runtime_image'), + # Python version + (_BASE_APP_CONFIG._replace(dockerfile_python_version='_my_version'), True, + 'python_version=python_my_version'), + # python-compat runtime + (_BASE_APP_CONFIG._replace(is_python_compat=True), True, + 'FROM gcr.io/google_appengine/python-compat-multicore'), +]) +def test_generate_files(app_config, should_find, test_string): + result = gen_dockerfile.generate_files(app_config) + assert set(result.keys()) == EXPECTED_OUTPUT_FILES + dockerfile = result['Dockerfile'] + if should_find: + assert test_string in dockerfile + else: + assert test_string not in dockerfile + + +def compare_against_golden_files(app, config_dir, testdata_dir): + golden_dir = os.path.join(testdata_dir, app + '_golden') + for filename in EXPECTED_OUTPUT_FILES: + compare_file(filename, config_dir, golden_dir) + + +@pytest.mark.parametrize('app', [ + # Sampled from https://github.com/GoogleCloudPlatform/python-docs-samples + 'hello_world', + # From an internal source. + 'hello_world_compat']) +def test_generate_dockerfile_command(tmpdir, testdata_dir, app): + """Generates output and compares against a set of golden files.""" + app_dir = os.path.join(testdata_dir, app) + + # Copy sample app to writable temp dir, and generate Dockerfile. + config_dir = os.path.join(str(tmpdir), 'config') + shutil.copytree(app_dir, config_dir) + gen_dockerfile.generate_dockerfile_command( + base_image='gcr.io/google-appengine/python', + config_file=os.path.join(config_dir, 'app.yaml'), + source_dir=config_dir) + compare_against_golden_files(app, config_dir, testdata_dir) + + +@pytest.mark.parametrize('app', [ + # Sampled from https://github.com/GoogleCloudPlatform/python-docs-samples + 'hello_world', + # From an internal source. + 'hello_world_compat']) +@pytest.mark.xfail(not shutil.which('gcloud'), + reason='Google Cloud SDK is not installed') +def test_generate_dockerfile_golden(tmpdir, testdata_dir, app): + """Validate our golden files against gcloud app gen-config""" + app_dir = os.path.join(testdata_dir, app) + + # Copy sample app to writable temp dir, and generate Dockerfile. + gen_config_dir = os.path.join(str(tmpdir), 'gen_config') + shutil.copytree(app_dir, gen_config_dir) + app_yaml = os.path.join(gen_config_dir, 'app.yaml') + gcloud_args = [ + 'gcloud', '--quiet', 'beta', 'app', 'gen-config', + gen_config_dir, '--custom', '--config={}'.format(app_yaml) + ] + print('Invoking gcloud as {}'.format(gcloud_args)) + subprocess.check_call(gcloud_args) + compare_against_golden_files(app, gen_config_dir, testdata_dir) + + +@pytest.mark.parametrize('argv', [ + [], + ['argv0', '--base-image=nocolon'], + ['argv0', '--base-image=name:andcolon'], + ['argv0', '--base-image=name@sha256:digest'], +]) +def test_parse_args_valid(argv): + args = gen_dockerfile.parse_args(argv) + assert args is not None + + +@pytest.mark.parametrize('argv', [ + ['argv0', '--base-image='], + ['argv0', '--base-image=:'], + ['argv0', '--base-image=:noname'], +]) +def test_parse_args_invalid(argv): + def mock_error(*args): + """Prevent argparse from calling sys.exit()""" + raise AssertionError(*args) + + error_patch = unittest.mock.patch.object( + argparse.ArgumentParser, 'error', mock_error) + with error_patch: + with pytest.raises(AssertionError): + gen_dockerfile.parse_args(argv) + + +@pytest.mark.parametrize('argv, env, expected', [ + # Explicit flag wins + (['argv0', '--config=flag/path'], 'env/path', 'flag/path'), + (['argv0', '--config=flag/path'], '', 'flag/path'), + (['argv0', '--config=flag/path'], None, 'flag/path'), + # Otherwise env var wins + (['argv0'], 'env/path', 'env/path'), + # Otherwise use default name + (['argv0'], '', 'app.yaml'), + (['argv0'], None, 'app.yaml'), +]) +def test_parse_args_config(argv, env, expected): + if env is None: + mock_environ = {} + else: + mock_environ = {gen_dockerfile.GAE_APPLICATION_YAML_PATH: env} + with unittest.mock.patch.dict('os.environ', mock_environ, clear=True): + args = gen_dockerfile.parse_args(argv) + assert args.config == expected + + +if __name__ == '__main__': + pytest.main([__file__]) diff --git a/scripts/integration-test.sh b/scripts/integration-test.sh new file mode 100644 index 00000000..6210e0a1 --- /dev/null +++ b/scripts/integration-test.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +set -ex + +export KOKORO_GITHUB_DIR=${KOKORO_ROOT}/src/github +source ${KOKORO_GFILE_DIR}/kokoro/common.sh + +export GOOGLE_CLOUD_PROJECT=gcp-runtimes + +sudo -E /usr/local/bin/pip install --upgrade -r ${KOKORO_GFILE_DIR}/appengine/integration_tests/requirements.txt + +if [ -f ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY}/requirements.txt ] +then + sudo -E /usr/local/bin/pip install --upgrade -r ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY}/requirements.txt +fi + +export GOPATH=${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY} + +flags="" + +if [ -n "${STAGING_IMAGE}" ]; then + flags="$flags -i ${STAGING_IMAGE}" +fi + +if [ "${SKIP_STANDARD_LOGGING_TESTS}" = "true" ]; then + flags="$flags --skip-standard-logging-tests" +fi + +if [ "${SKIP_CUSTOM_LOGGING_TESTS}" = "true" ]; then + flags="$flags --skip-custom-logging-tests" +fi + +if [ "${SKIP_MONITORING_TESTS}" = "true" ]; then + flags="$flags --skip-monitoring-tests" +fi + +if [ "${SKIP_EXCEPTION_TESTS}" = "true" ]; then + flags="$flags --skip-exception-tests" +fi + +if [ "${SKIP_CUSTOM_TESTS}" = "true" ]; then + flags="$flags --skip-custom-tests" +fi + +if [ -n "${URL}" ]; then + flags="$flags --url ${URL}" +fi + +if [ -n "${BUILDER}" ]; then + flags="$flags --builder ${BUILDER}" + gcloud config set app/use_runtime_builders True + gcloud config set app/runtime_builders_root file://${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY} +fi + +if [ -n "${YAML}" ]; then + flags="$flags --yaml ${KOKORO_GITHUB_DIR}/${YAML}" +fi + + +chmod a+x ${KOKORO_GFILE_DIR}/appengine/integration_tests/testsuite/driver.py +${KOKORO_GFILE_DIR}/appengine/integration_tests/testsuite/driver.py -d ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY} ${flags} diff --git a/scripts/local_cloudbuild.py b/scripts/local_cloudbuild.py new file mode 100755 index 00000000..5c23a1bc --- /dev/null +++ b/scripts/local_cloudbuild.py @@ -0,0 +1,372 @@ +#!/usr/bin/env python3 + +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Emulate the Google Cloud Build locally. + +The input is a local cloudbuild.yaml file. This is translated into a +series of commands for the locally installed Docker daemon. These +commands are output as a shell script and optionally executed. + +The output images are not pushed to the Google Container Registry. +Not all cloudbuild.yaml functionality is supported. In particular, +substitutions are a simplified subset that doesn't include all the +corner cases and error conditions. + +See https://cloud.google.com/container-builder/docs/api/build-steps +for more information. +""" + +import argparse +import collections +import collections.abc +import functools +import io +import os +import re +import shlex +import subprocess +import sys + +import yaml + +import validation_utils + + +# Exclude non-printable control characters (including newlines) +PRINTABLE_REGEX = re.compile(r"""^[^\x00-\x1f]*$""") + +# Cloud Build substitutions +# https://cloud.google.com/cloud-build/docs/api/build-requests#substitutions +SUBSTITUTION_REGEX = re.compile(r"""(?x) + [$] # Dollar sign + ( + [A-Z_][A-Z0-9_]* # Variable name, no curly brackets + | + {[A-Z_][A-Z0-9_]*} # Variable name, with curly brackets + | + [$] # $$, translated to a single literal $ + ) +""") + +# Default builtin substitutions +DEFAULT_SUBSTITUTIONS = { + 'BRANCH_NAME': '', + 'BUILD_ID': 'abcdef12-3456-7890-abcd-ef0123456789', + 'COMMIT_SHA': '', + 'PROJECT_ID': 'dummy-project-id', + 'REPO_NAME': '', + 'REVISION_ID': '', + 'TAG_NAME': '', +} + +# Use this image for cleanup actions +DEBIAN_IMAGE = 'gcr.io/google-appengine/debian8' + +# File template +BUILD_SCRIPT_TEMPLATE = """\ +#!/bin/bash +# This is a generated file. Do not edit. + +set -euo pipefail + +SOURCE_DIR=. + +# Setup staging directory +HOST_WORKSPACE=$(mktemp -d -t local_cloudbuild_XXXXXXXXXX) +function cleanup {{ + if [ "${{HOST_WORKSPACE}}" != '/' -a -d "${{HOST_WORKSPACE}}" ]; then + # Expect a single error message about /workspace busy + {cleanup_str} 2>/dev/null || true + # Do not expect error messages here. Display but ignore. + rmdir "${{HOST_WORKSPACE}}" || true + fi +}} +trap cleanup EXIT + +# Copy source to staging directory +echo "Copying source to staging directory ${{HOST_WORKSPACE}}" +rsync -avzq --exclude=.git "${{SOURCE_DIR}}" "${{HOST_WORKSPACE}}" + +# Build commands +{docker_str} +# End of build commands +echo "Build completed successfully" +""" + + +# Validated cloudbuild recipe + flags +CloudBuild = collections.namedtuple('CloudBuild', + 'output_script run steps substitutions') + +# Single validated step in a cloudbuild recipe +Step = collections.namedtuple('Step', 'args dir_ env name') + + +def sub_and_quote(s, substitutions, substitutions_used): + """Return a shell-escaped, variable substituted, version of the string s. + + Args: + s (str): Any string + subs (dict): Substitution map to apply + subs_used (set): Updated with names from `subs.keys()` when those + substitutions are encountered in `s` + """ + + def sub(match): + """Perform a single substitution.""" + variable_name = match.group(1) + if variable_name[0] == '{': + # Strip curly brackets + variable_name = variable_name[1:-1] + if variable_name == '$': + value = '$' + elif variable_name not in substitutions: + # Variables must be set + raise ValueError( + 'Variable "{}" used without being defined. Try adding ' + 'it to the --substitutions flag'.format(variable_name)) + else: + value = substitutions.get(variable_name) + substitutions_used.add(variable_name) + return value + + substituted_s = re.sub(SUBSTITUTION_REGEX, sub, s) + quoted_s = shlex.quote(substituted_s) + return quoted_s + + +def get_cloudbuild(raw_config, args): + """Read and validate a cloudbuild recipe + + Args: + raw_config (dict): deserialized cloudbuild.yaml + args (argparse.Namespace): command line flags + + Returns: + CloudBuild: valid configuration + """ + if not isinstance(raw_config, dict): + raise ValueError( + 'Expected {} contents to be of type "dict", but found type "{}"'. + format(args.config, type(raw_config))) + + raw_steps = validation_utils.get_field_value(raw_config, 'steps', list) + if not raw_steps: + raise ValueError('No steps defined in {}'.format(args.config)) + + steps = [get_step(raw_step) for raw_step in raw_steps] + return CloudBuild( + output_script=args.output_script, + run=args.run, + steps=steps, + substitutions=args.substitutions, + ) + + +def get_step(raw_step): + """Read and validate a single cloudbuild step + + Args: + raw_step (dict): deserialized step + + Returns: + Step: valid build step + """ + if not isinstance(raw_step, dict): + raise ValueError( + 'Expected step to be of type "dict", but found type "{}"'. + format(type(raw_step))) + raw_args = validation_utils.get_field_value(raw_step, 'args', list) + args = [validation_utils.get_field_value(raw_args, index, str) + for index in range(len(raw_args))] + dir_ = validation_utils.get_field_value(raw_step, 'dir', str) + raw_env = validation_utils.get_field_value(raw_step, 'env', list) + env = [validation_utils.get_field_value(raw_env, index, str) + for index in range(len(raw_env))] + name = validation_utils.get_field_value(raw_step, 'name', str) + return Step( + args=args, + dir_=dir_, + env=env, + name=name, + ) + + +def generate_command(step, substitutions, substitutions_used): + """Generate a single shell command to run for a single cloudbuild step + + Args: + step (Step): Valid build step + subs (dict): Substitution map to apply + subs_used (set): Updated with names from `subs.keys()` when those + substitutions are encountered in an element of `step` + + Returns: + [str]: A single shell command, expressed as a list of quoted tokens. + """ + quoted_args = [sub_and_quote(arg, substitutions, substitutions_used) + for arg in step.args] + quoted_env = [] + for env in step.env: + quoted_env.extend(['--env', sub_and_quote(env, substitutions, + substitutions_used)]) + quoted_name = sub_and_quote(step.name, substitutions, substitutions_used) + workdir = '/workspace' + if step.dir_: + workdir = os.path.join(workdir, sub_and_quote(step.dir_, substitutions, + substitutions_used)) + process_args = [ + 'docker', + 'run', + '--volume', + '/var/run/docker.sock:/var/run/docker.sock', + '--volume', + '/root/.docker:/root/.docker', + '--volume', + '${HOST_WORKSPACE}:/workspace', + '--workdir', + workdir, + ] + quoted_env + [quoted_name] + quoted_args + return process_args + + +def generate_script(cloudbuild): + """Generate the contents of a shell script + + Args: + cloudbuild (CloudBuild): Valid cloudbuild configuration + + Returns: + (str): Contents of shell script + """ + # This deletes everything in /workspace including hidden files, + # but not /workspace itself + cleanup_step = Step( + args=['rm', '-rf', '/workspace'], + dir_='', + env=[], + name=DEBIAN_IMAGE, + ) + cleanup_command = generate_command(cleanup_step, {}, set()) + subs_used = set() + docker_commands = [ + generate_command(step, cloudbuild.substitutions, subs_used) + for step in cloudbuild.steps] + + # Check that all user variables were referenced at least once + user_subs_unused = [name for name in cloudbuild.substitutions.keys() + if name not in subs_used and name[0] == '_'] + if user_subs_unused: + nice_list = '"' + '", "'.join(sorted(user_subs_unused)) + '"' + raise ValueError( + 'User substitution variables {} were defined in the ' + '--substitution flag but never used in the cloudbuild file.'. + format(nice_list)) + + cleanup_str = ' '.join(cleanup_command) + docker_lines = [] + for docker_command in docker_commands: + line = ' '.join(docker_command) + '\n\n' + docker_lines.append(line) + docker_str = ''.join(docker_lines) + + s = BUILD_SCRIPT_TEMPLATE.format(cleanup_str=cleanup_str, + docker_str=docker_str) + return s + + +def make_executable(path): + """Set executable bit(s) on file""" + # http://stackoverflow.com/questions/12791997 + mode = os.stat(path).st_mode + mode |= (mode & 0o444) >> 2 # copy R bits to X + os.chmod(path, mode) + + +def write_script(cloudbuild, contents): + """Write a shell script to a file.""" + print('Writing build script to {}'.format(cloudbuild.output_script)) + with io.open(cloudbuild.output_script, 'w', encoding='utf8') as outfile: + outfile.write(contents) + make_executable(cloudbuild.output_script) + + +def local_cloudbuild(args): + """Execute the steps of a cloudbuild.yaml locally + + Args: + args: command line flags as per parse_args + """ + # Load and parse cloudbuild.yaml + with io.open(args.config, 'r', encoding='utf8') as cloudbuild_file: + raw_config = yaml.safe_load(cloudbuild_file) + + # Determine configuration + cloudbuild = get_cloudbuild(raw_config, args) + + # Create shell script + contents = generate_script(cloudbuild) + write_script(cloudbuild, contents) + + # Run shell script + if cloudbuild.run: + print('Running {}'.format(cloudbuild.output_script)) + args = [os.path.abspath(cloudbuild.output_script)] + subprocess.check_call(args) + + +def parse_args(argv): + """Parse and validate command line flags""" + parser = argparse.ArgumentParser( + description='Process cloudbuild.yaml locally to build Docker images') + parser.add_argument( + '--config', + type=functools.partial( + validation_utils.validate_arg_regex, flag_regex=PRINTABLE_REGEX), + default='cloudbuild.yaml', + help='Path to cloudbuild.yaml file' + ) + parser.add_argument( + '--output_script', + type=functools.partial( + validation_utils.validate_arg_regex, flag_regex=PRINTABLE_REGEX), + help='Filename to write shell script to', + ) + parser.add_argument( + '--no-run', + action='store_false', + help='Create shell script but don\'t execute it', + dest='run', + ) + parser.add_argument( + '--substitutions', + type=validation_utils.validate_arg_dict, + default={}, + help='Parameters to be substituted in the build specification', + ) + args = parser.parse_args(argv[1:]) + if not args.output_script: + args.output_script = args.config + "_local.sh" + return args + + +def main(): + args = parse_args(sys.argv) + local_cloudbuild(args) + + +if __name__ == '__main__': + main() diff --git a/scripts/local_cloudbuild_test.py b/scripts/local_cloudbuild_test.py new file mode 100755 index 00000000..b22d585b --- /dev/null +++ b/scripts/local_cloudbuild_test.py @@ -0,0 +1,410 @@ +#!/usr/bin/env python3 + +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit test for local_cloudbuild.py""" + +import argparse +import contextlib +import os +import re +import shutil +import subprocess +import unittest.mock + +import pytest +import yaml + +import local_cloudbuild + + +# Matches script boilerplate +STAGING_DIR_REGEX = re.compile( + b'(?m)Copying source to staging directory (.+)$') + + +@pytest.fixture +def testdata_dir(): + testdata_dir = os.path.join(os.path.dirname(__file__), 'testdata') + assert os.path.isdir(testdata_dir), ( + 'Could not run test: testdata directory not found') + return testdata_dir + + +@pytest.mark.parametrize('s, subs, expected, expected_used', [ + # Empty string + ('', {}, "''", []), + # No substitutions + ('a', {}, 'a', []), + # Unused substitition (ok here but error in generate_script) + ('a', {'FOO': 'foo'}, 'a', []), + ('a', {'_FOO': '_foo'}, 'a', []), + # Defined builtin substitution + ('a$FOOb', {'FOO': 'foo'}, 'afoob', ['FOO']), + ('a${FOO}b', {'FOO': 'foo'}, 'afoob', ['FOO']), + # Defined user substitution + ('a$_FOOb', {'_FOO': '_foo'}, 'a_foob', ['_FOO']), + ('a${_FOO}b', {'_FOO': '_foo'}, 'a_foob', ['_FOO']), + # Multiple substitutions + ('$FOO${FOO}${BAR}$FOO', + {'FOO': 'foo', 'BAR': 'bar'}, + 'foofoobarfoo', + ['FOO', 'BAR']), + # Invalid names + ('a $ b', {}, "'a $ b'", []), + ('a$foo b', {}, "'a$foo b'", []), + ('a$0FOO b', {}, "'a$0FOO b'", []), +]) +def test_sub_and_quote_valid(s, subs, expected, expected_used): + used = set() + actual = local_cloudbuild.sub_and_quote(s, subs, used) + assert actual == expected + assert used == set(expected_used) + + +@pytest.mark.parametrize('s, subs', [ + # Undefined builtin substitution + ('a$FOOb', {}), + ('a${FOO}b', {}), + # Undefined user substitution + ('a$_FOOb', {}), + ('a${_FOO}b', {}), +]) +def test_sub_and_quote_invalid(s, subs): + with pytest.raises(ValueError): + used = set() + local_cloudbuild.sub_and_quote(s, subs, used) + + +def have_docker(): + """Determine if the Docker daemon is present and usable""" + if ((shutil.which('docker') is not None) and + (subprocess.call(['docker', 'info'], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) == 0)): + return True + return False + + +_args = argparse.Namespace( + config='some_config_file', + output_script='some_output_script', + run=False, + substitutions={}, +) + + +def test_get_cloudbuild_valid(): + raw_yaml = 'steps:\n- name: step1\n- name: step2\n' + raw_config = yaml.safe_load(raw_yaml) + actual = local_cloudbuild.get_cloudbuild(raw_config, _args) + assert len(actual.steps) == 2 + + +@pytest.mark.parametrize('raw_yaml', [ + # Empty cloud build + '', + # No steps + 'foo: bar\n', + # Steps not a list + 'steps: astring\n', + ]) +def test_get_cloudbuild_invalid(raw_yaml): + raw_config = yaml.safe_load(raw_yaml) + with pytest.raises(ValueError): + local_cloudbuild.get_cloudbuild(raw_config, _args) + + +@pytest.mark.parametrize('raw_step, expected', [ + # Empty step + ({}, local_cloudbuild.Step( + args=[], + dir_='', + env=[], + name='', + )), + # Full step + ({'name': 'aname', + 'args': ['arg1', 2, 'arg3 with \n newline'], + 'env': ['ENV1=value1', 'ENV2=space in value2'], + 'dir': 'adir', + }, local_cloudbuild.Step( + args=['arg1', '2', 'arg3 with \n newline'], + env=['ENV1=value1', 'ENV2=space in value2'], + dir_='adir', + name='aname', + )), +]) +def test_get_step_valid(raw_step, expected): + actual = local_cloudbuild.get_step(raw_step) + assert actual == expected + + +@pytest.mark.parametrize('raw_step', [ + # Wrong type + [], + # More wrong types + {'args': 'not_a_list'}, + {'args': [[]]}, + {'env': 'not_a_list'}, + {'env': [{}]}, + {'dir': {}}, + {'name': []}, +]) +def test_get_step_invalid(raw_step): + with pytest.raises(ValueError): + local_cloudbuild.get_step(raw_step) + + +# Basic valid case +_base_step = local_cloudbuild.Step( + args=['arg1', 'arg2'], + dir_='', + env=['ENV1=value1', 'ENV2=value2'], + name='aname', +) +_subs = {'BUILTIN': 'builtin', '_USER': '_user'} + + +def test_generate_command_basic(): + command = local_cloudbuild.generate_command(_base_step, _subs, set()) + assert command == [ + 'docker', + 'run', + '--volume', + '/var/run/docker.sock:/var/run/docker.sock', + '--volume', + '/root/.docker:/root/.docker', + '--volume', + '${HOST_WORKSPACE}:/workspace', + '--workdir', + '/workspace', + '--env', + 'ENV1=value1', + '--env', + 'ENV2=value2', + 'aname', + 'arg1', + 'arg2', + ] + + +@pytest.mark.parametrize('step, args', [ + # dir specified + (_base_step._replace(dir_='adir'), + ['--workdir', '/workspace/adir']), + # Shell quoting + (_base_step._replace(args=['arg with \n newline']), + ["'arg with \n newline'"]), + (_base_step._replace(dir_='dir/ with space/'), + ["/workspace/'dir/ with space/'"]), + (_base_step._replace(env=['env with space']), + ["'env with space'"]), + (_base_step._replace(name='a name'), + ["'a name'"]), + # Variable substitution + (_base_step._replace(name='a $BUILTIN substitution'), + ["'a builtin substitution'"]), + (_base_step._replace(name='a $_USER substitution'), + ["'a _user substitution'"]), + (_base_step._replace(name='a curly brace ${BUILTIN} substitution'), + ["'a curly brace builtin substitution'"]), + (_base_step._replace( + name='an escaped $$ or $$$$ or $$FOO or $${_FOO} is unescaped'), + ["'an escaped $ or $$ or $FOO or ${_FOO} is unescaped'"]), +]) +def test_generate_command_valid(step, args): + command = local_cloudbuild.generate_command(step, _subs, set()) + for arg in args: + assert arg in command + + +@pytest.mark.parametrize('step', [ + _base_step._replace(name='a $UNSET_BUILTIN substitution'), + _base_step._replace(name='a $_UNSET_USER substitution'), +]) +def test_generate_command_invalid(step): + with pytest.raises(ValueError): + local_cloudbuild.generate_command(step, _subs, set()) + + +def test_generate_script_golden(testdata_dir): + config_name = 'cloudbuild_ok.yaml' + expected_output_script = os.path.join( + testdata_dir, config_name + '_golden.sh') + cloudbuild = local_cloudbuild.CloudBuild( + output_script='test_generate_script', + run=False, + steps=[ + local_cloudbuild.Step( + args=['/bin/sh', '-c', 'printenv MESSAGE'], + dir_='', + env=['MESSAGE=Hello World!'], + name='debian', + ), + local_cloudbuild.Step( + args=['/bin/sh', '-c', 'printenv MESSAGE'], + dir_='', + env=['MESSAGE=Goodbye\\n And Farewell!', 'UNUSED=unused'], + name='debian', + ) + ], + substitutions=local_cloudbuild.DEFAULT_SUBSTITUTIONS, + ) + actual = local_cloudbuild.generate_script(cloudbuild) + # Compare output against golden + with open(expected_output_script, 'r', encoding='utf8') as expected_file: + expected = expected_file.read() + assert actual == expected + + +def test_generate_script_unused_user_substitution(): + cloudbuild = local_cloudbuild.CloudBuild( + output_script='', + run=False, + steps=[], + substitutions={'_FOO': '_foo'}, + ) + with pytest.raises(ValueError, match='User substitution variables'): + local_cloudbuild.generate_script(cloudbuild) + + +def test_make_executable(tmpdir): + test_script_filename = tmpdir.join('test_make_executable.sh') + with test_script_filename.open('w', encoding='utf8') as test_script: + test_script.write('#!/bin/sh\necho "Output from test_make_executable"') + local_cloudbuild.make_executable(str(test_script_filename)) + output = subprocess.check_output([str(test_script_filename)]) + assert output.decode('utf8') == "Output from test_make_executable\n" + + +def test_write_script(tmpdir): + contents = 'The contents\n' + output_script_filename = tmpdir.join('test_write_script') + cloudbuild = local_cloudbuild.CloudBuild( + output_script=str(output_script_filename), + run=False, + steps=[], + substitutions={}, + ) + local_cloudbuild.write_script(cloudbuild, contents) + with output_script_filename.open('r', encoding='utf8') as output_script: + actual = output_script.read() + assert actual == contents + + +@contextlib.contextmanager +def chdir(new_dir): + """Not threadsafe""" + old_dir = os.getcwd() + os.chdir(new_dir) + yield + os.chdir(old_dir) + + +@pytest.mark.parametrize('config_name, substitutions, exception, cleanup', [ + # Everything is ok + ('cloudbuild_ok.yaml', None, None, True), + # Builtin substitutions like $PROJECT_ID work + ('cloudbuild_builtin_substitutions.yaml', None, None, True), + # User substitutions like $_FOO work + ('cloudbuild_user_substitutions.yaml', + {'_FOO': 'this is foo value'}, + None, True + ), + # User substitutions like $_FOO fails when undefined + ('cloudbuild_user_substitutions.yaml', None, ValueError, False), + # Exit code 1 (failure) + ('cloudbuild_err_rc1.yaml', None, subprocess.CalledProcessError, True), + # Command not found + ('cloudbuild_err_not_found.yaml', None, subprocess.CalledProcessError, + True), + # Cleaning up files owned by root + ('cloudbuild_difficult_cleanup.yaml', None, None, True), +]) +def test_local_cloudbuild(testdata_dir, tmpdir, config_name, substitutions, + exception, cleanup): + if not have_docker(): + pytest.fail('This test requires a working Docker daemon') + + check_call_output = None + + def check_call(*args, **kw_args): + """Act like subprocess.check_call but store stdout""" + nonlocal check_call_output + try: + check_call_output = subprocess.check_output(*args, **kw_args) + print(check_call_output) + except subprocess.CalledProcessError as e: + check_call_output = e.output + print(check_call_output) + raise + + # Read cloudbuild.yaml from testdata file, write output to + # tempdir, and maybe try to run it + with unittest.mock.patch('subprocess.check_call', check_call): + if substitutions is None: + substitutions = local_cloudbuild.DEFAULT_SUBSTITUTIONS + should_succeed = (exception is None) + config = os.path.join(testdata_dir, config_name) + actual_output_script = tmpdir.join(config_name + '_local.sh') + args = argparse.Namespace( + config=config, + output_script=str(actual_output_script), + run=True, + substitutions=substitutions, + ) + + # The source directory of the build is currently hardcoded as + # '.', so we must chdir there. + with chdir(testdata_dir): + if should_succeed: + local_cloudbuild.local_cloudbuild(args) + else: + with pytest.raises(exception): + local_cloudbuild.local_cloudbuild(args) + + # Check that staging dir was cleaned up + if cleanup: + assert check_call_output is not None + match = re.search(STAGING_DIR_REGEX, check_call_output) + assert match + staging_dir = match.group(1) + assert not os.path.isdir(staging_dir) + + +@pytest.mark.parametrize('argv, expected', [ + # Test explicit output_script + (['argv0', '--output_script=my_output'], 'my_output'), + # Test implicit output_script + (['argv0', '--config=my_config'], 'my_config_local.sh'), +]) +def test_parse_args_output_script(argv, expected): + args = local_cloudbuild.parse_args(argv) + assert args.output_script == expected + + +@pytest.mark.parametrize('argv, expected', [ + # Test run flag (default) + (['argv0'], True), + (['argv0', '--no-run'], False), +]) +def test_parse_args_run_flag(argv, expected): + args = local_cloudbuild.parse_args(argv) + assert args.run == expected + + +if __name__ == '__main__': + pytest.main([__file__]) diff --git a/scripts/release.sh b/scripts/release.sh new file mode 100644 index 00000000..cd499823 --- /dev/null +++ b/scripts/release.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set -euo pipefail +export KOKORO_GITHUB_DIR=${KOKORO_ROOT}/src/github +source ${KOKORO_GFILE_DIR}/kokoro/common.sh + +cd ${KOKORO_GITHUB_DIR}/python-runtime + +if [ -z "${TAG:+set}" ]; then + export TAG=$(date +%Y-%m-%d-%H%M%S) +fi + +./build.sh $BUILD_FLAGS + +METADATA=$(pwd)/METADATA +cd ${KOKORO_GFILE_DIR}/kokoro +python note.py python -m ${METADATA} -t ${TAG} diff --git a/scripts/requirements-test.txt b/scripts/requirements-test.txt new file mode 100644 index 00000000..c8e698da --- /dev/null +++ b/scripts/requirements-test.txt @@ -0,0 +1,4 @@ +flask==1.0.2 +pytest==3.7.3 +pytest-cov==2.5.1 +pyyaml==3.13 diff --git a/scripts/testdata/cloudbuild_builtin_substitutions.yaml b/scripts/testdata/cloudbuild_builtin_substitutions.yaml new file mode 100644 index 00000000..0e39ba68 --- /dev/null +++ b/scripts/testdata/cloudbuild_builtin_substitutions.yaml @@ -0,0 +1,11 @@ +steps: +- name: debian + args: ['/bin/sh', '-c', 'echo "$PROJECT_ID"'] +- name: debian + args: ['/usr/bin/test', 'dummy-project-id', '=', '$PROJECT_ID'] +- name: debian + args: ['/usr/bin/test', 'dummy-project-id', '=', '${PROJECT_ID}'] +- name: debian + args: ['/bin/sh', '-c', 'test dummy-project-id = "$PROJECT_ID"'] +- name: debian + args: ['/bin/sh', '-c', 'test dummy-project-id = "${PROJECT_ID}"'] diff --git a/scripts/testdata/cloudbuild_difficult_cleanup.yaml b/scripts/testdata/cloudbuild_difficult_cleanup.yaml new file mode 100644 index 00000000..e76846d5 --- /dev/null +++ b/scripts/testdata/cloudbuild_difficult_cleanup.yaml @@ -0,0 +1,3 @@ +steps: +- name: debian + args: ['/bin/sh', '-c', 'mkdir root; umask 0000; touch root/deny_all.txt'] diff --git a/scripts/testdata/cloudbuild_err_not_found.yaml b/scripts/testdata/cloudbuild_err_not_found.yaml new file mode 100644 index 00000000..c7eb070d --- /dev/null +++ b/scripts/testdata/cloudbuild_err_not_found.yaml @@ -0,0 +1,3 @@ +steps: +- name: debian + args: ['/expected file not found'] diff --git a/scripts/testdata/cloudbuild_err_rc1.yaml b/scripts/testdata/cloudbuild_err_rc1.yaml new file mode 100644 index 00000000..3953a586 --- /dev/null +++ b/scripts/testdata/cloudbuild_err_rc1.yaml @@ -0,0 +1,3 @@ +steps: +- name: debian + args: ['/bin/sh', '-c', 'exit 1'] diff --git a/scripts/testdata/cloudbuild_ok.yaml b/scripts/testdata/cloudbuild_ok.yaml new file mode 100644 index 00000000..d58ca3f9 --- /dev/null +++ b/scripts/testdata/cloudbuild_ok.yaml @@ -0,0 +1,7 @@ +steps: +- name: debian + args: ['/bin/sh', '-c', 'printenv MESSAGE'] + env: ['MESSAGE=Hello World!'] +- name: debian + args: ['/bin/sh', '-c', 'printenv MESSAGE'] + env: ['MESSAGE=Goodbye\n And Farewell!', 'UNUSED=unused'] diff --git a/scripts/testdata/cloudbuild_ok.yaml_golden.sh b/scripts/testdata/cloudbuild_ok.yaml_golden.sh new file mode 100755 index 00000000..bda52bd2 --- /dev/null +++ b/scripts/testdata/cloudbuild_ok.yaml_golden.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# This is a generated file. Do not edit. + +set -euo pipefail + +SOURCE_DIR=. + +# Setup staging directory +HOST_WORKSPACE=$(mktemp -d -t local_cloudbuild_XXXXXXXXXX) +function cleanup { + if [ "${HOST_WORKSPACE}" != '/' -a -d "${HOST_WORKSPACE}" ]; then + # Expect a single error message about /workspace busy + docker run --volume /var/run/docker.sock:/var/run/docker.sock --volume /root/.docker:/root/.docker --volume ${HOST_WORKSPACE}:/workspace --workdir /workspace gcr.io/google-appengine/debian8 rm -rf /workspace 2>/dev/null || true + # Do not expect error messages here. Display but ignore. + rmdir "${HOST_WORKSPACE}" || true + fi +} +trap cleanup EXIT + +# Copy source to staging directory +echo "Copying source to staging directory ${HOST_WORKSPACE}" +rsync -avzq --exclude=.git "${SOURCE_DIR}" "${HOST_WORKSPACE}" + +# Build commands +docker run --volume /var/run/docker.sock:/var/run/docker.sock --volume /root/.docker:/root/.docker --volume ${HOST_WORKSPACE}:/workspace --workdir /workspace --env 'MESSAGE=Hello World!' debian /bin/sh -c 'printenv MESSAGE' + +docker run --volume /var/run/docker.sock:/var/run/docker.sock --volume /root/.docker:/root/.docker --volume ${HOST_WORKSPACE}:/workspace --workdir /workspace --env 'MESSAGE=Goodbye\n And Farewell!' --env UNUSED=unused debian /bin/sh -c 'printenv MESSAGE' + + +# End of build commands +echo "Build completed successfully" diff --git a/scripts/testdata/cloudbuild_user_substitutions.yaml b/scripts/testdata/cloudbuild_user_substitutions.yaml new file mode 100644 index 00000000..13cb6602 --- /dev/null +++ b/scripts/testdata/cloudbuild_user_substitutions.yaml @@ -0,0 +1,11 @@ +steps: +- name: debian + args: ['/bin/sh', '-c', 'echo "$_FOO"'] +- name: debian + args: ['/usr/bin/test', 'this is foo value', '=', '$_FOO'] +- name: debian + args: ['/usr/bin/test', 'this is foo value', '=', '${_FOO}'] +- name: debian + args: ['/bin/sh', '-c', 'test "this is foo value" = "$_FOO"'] +- name: debian + args: ['/bin/sh', '-c', 'test "this is foo value" = "${_FOO}"'] diff --git a/scripts/testdata/hello_world/app.yaml b/scripts/testdata/hello_world/app.yaml new file mode 100644 index 00000000..e5ac514e --- /dev/null +++ b/scripts/testdata/hello_world/app.yaml @@ -0,0 +1,6 @@ +runtime: python +env: flex +entrypoint: gunicorn -b :$PORT main:app + +runtime_config: + python_version: 3 diff --git a/scripts/testdata/hello_world/main.py b/scripts/testdata/hello_world/main.py new file mode 100644 index 00000000..97eb37d8 --- /dev/null +++ b/scripts/testdata/hello_world/main.py @@ -0,0 +1,43 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START app] +import logging + +from flask import Flask + + +app = Flask(__name__) + + +@app.route('/') +def hello(): + """Return a friendly HTTP greeting.""" + return 'Hello World!' + + +@app.errorhandler(500) +def server_error(e): + logging.exception('An error occurred during a request.') + return """ + An internal error occurred: 
{}
+ See logs for full stacktrace. + """.format(e), 500 + + +if __name__ == '__main__': + # This is used when running locally. Gunicorn is used to run the + # application on Google App Engine. See entrypoint in app.yaml. + app.run(host='127.0.0.1', port=8080, debug=True) +# [END app] diff --git a/scripts/testdata/hello_world/main_test.py b/scripts/testdata/hello_world/main_test.py new file mode 100644 index 00000000..4e230185 --- /dev/null +++ b/scripts/testdata/hello_world/main_test.py @@ -0,0 +1,24 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import main + + +def test_index(): + main.app.testing = True + client = main.app.test_client() + + r = client.get('/') + assert r.status_code == 200 + assert 'Hello World' in r.data.decode('utf-8') diff --git a/scripts/testdata/hello_world/requirements.txt b/scripts/testdata/hello_world/requirements.txt new file mode 100644 index 00000000..a34d076b --- /dev/null +++ b/scripts/testdata/hello_world/requirements.txt @@ -0,0 +1,2 @@ +Flask==1.0.2 +gunicorn==19.9.0 diff --git a/scripts/testdata/hello_world_compat/app.yaml b/scripts/testdata/hello_world_compat/app.yaml new file mode 100644 index 00000000..e514d42c --- /dev/null +++ b/scripts/testdata/hello_world_compat/app.yaml @@ -0,0 +1,13 @@ +service: default +runtime: python-compat +env: flex + +api_version: 1 +threadsafe: true + +beta_settings: + enable_app_engine_apis: true # Needed for compat apps. + +handlers: +- url: .* + script: main.app diff --git a/scripts/testdata/hello_world_compat/main.py b/scripts/testdata/hello_world_compat/main.py new file mode 100644 index 00000000..40302722 --- /dev/null +++ b/scripts/testdata/hello_world_compat/main.py @@ -0,0 +1,14 @@ +"""The hello world flex app!""" + +import webapp2 + + +class HelloHandler(webapp2.RequestHandler): + + def get(self): + msg = 'Hello GAE Flex (env: flex) Compat-Runtime App\n' + self.response.headers['Content-Type'] = 'text/plain' + self.response.out.write(msg) + +app = webapp2.WSGIApplication([('/', HelloHandler)], + debug=True) diff --git a/scripts/testdata/hello_world_compat_golden/.dockerignore b/scripts/testdata/hello_world_compat_golden/.dockerignore new file mode 100644 index 00000000..5ce5abfa --- /dev/null +++ b/scripts/testdata/hello_world_compat_golden/.dockerignore @@ -0,0 +1,5 @@ +.dockerignore +Dockerfile +.git +.hg +.svn diff --git a/scripts/testdata/hello_world_compat_golden/Dockerfile b/scripts/testdata/hello_world_compat_golden/Dockerfile new file mode 100644 index 00000000..1e4d6352 --- /dev/null +++ b/scripts/testdata/hello_world_compat_golden/Dockerfile @@ -0,0 +1,3 @@ +FROM gcr.io/google_appengine/python-compat-multicore +ADD . /app/ +RUN if [ -s requirements.txt ]; then pip install -r requirements.txt; fi diff --git a/scripts/testdata/hello_world_golden/.dockerignore b/scripts/testdata/hello_world_golden/.dockerignore new file mode 100644 index 00000000..8b927bb7 --- /dev/null +++ b/scripts/testdata/hello_world_golden/.dockerignore @@ -0,0 +1,19 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +.dockerignore +Dockerfile +.git +.hg +.svn diff --git a/scripts/testdata/hello_world_golden/Dockerfile b/scripts/testdata/hello_world_golden/Dockerfile new file mode 100644 index 00000000..10396399 --- /dev/null +++ b/scripts/testdata/hello_world_golden/Dockerfile @@ -0,0 +1,12 @@ +FROM gcr.io/google-appengine/python +LABEL python_version=python3.6 +RUN virtualenv --no-download /env -p python3.6 + +# Set virtualenv environment variables. This is equivalent to running +# source /env/bin/activate +ENV VIRTUAL_ENV /env +ENV PATH /env/bin:$PATH +ADD requirements.txt /app/ +RUN pip install -r requirements.txt +ADD . /app/ +CMD exec gunicorn -b :$PORT main:app diff --git a/scripts/validation_utils.py b/scripts/validation_utils.py new file mode 100644 index 00000000..28de2d52 --- /dev/null +++ b/scripts/validation_utils.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 + +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Utilities for schema and command line validation""" + +import argparse +import re + + +# For easier development, we allow redefining builtins like +# --substitutions=PROJECT_ID=foo even though gcloud doesn't. +KEY_VALUE_REGEX = re.compile(r'^([A-Z_][A-Z0-9_]*)=(.*)$') + + +def get_field_value(container, field_name, field_type): + """Fetch a field from a container with typechecking and default values. + + The field value is coerced to the desired type. If the field is + not present, an instance of `field_type` is constructed with no + arguments and used as the default value. + + This function exists because yaml parsing can lead to surprising + outputs, and the resulting errors are confusing. For example: + entrypoint1: a string, but I can accidentally treat as an sequence + entrypoint2: [a, list, but, I, might, think, its, a, string] + version1: 3 # Parsed to int + version2: 3.1 # Parsed to float + version3: 3.1.1 # Parsed to str + feature: off # Parsed to the boolean False + + Args: + container (dict): Object decoded from yaml + field_name (str): Field that should be present in `container` + field_type (type): Expected type for field value + + Returns: + Any: Fetched or default value of field + + Raises: + ValueError: if field value cannot be converted to the desired type + """ + try: + value = container[field_name] + if value is None: + return field_type() + except (IndexError, KeyError): + return field_type() + + msg = 'Expected "{}" field to be of type "{}", but found type "{}"' + if not isinstance(value, field_type): + # list('some string') is a successful type cast as far as Python + # is concerned, but doesn't exactly produce the results we want. + # We have a whitelist of conversions we will attempt. + whitelist = ( + (float, str), + (int, str), + (str, float), + (str, int), + (int, float), + ) + if (type(value), field_type) not in whitelist: + raise ValueError(msg.format(field_name, field_type, type(value))) + + try: + value = field_type(value) + except ValueError as e: + e.message = msg.format(field_name, field_type, type(value)) + raise + return value + + +def validate_arg_regex(flag_value, flag_regex): + """Check a named command line flag against a regular expression""" + if not re.match(flag_regex, flag_value): + raise argparse.ArgumentTypeError( + 'Value "{}" does not match pattern "{}"'.format( + flag_value, flag_regex.pattern)) + return flag_value + + +def validate_arg_dict(flag_value): + """Parse a command line flag as a key=val,... dict""" + if not flag_value: + return {} + entries = flag_value.split(',') + pairs = [] + for entry in entries: + match = re.match(KEY_VALUE_REGEX, entry) + if not match: + raise argparse.ArgumentTypeError( + 'Value "{}" should be a list like _KEY1=value1,_KEY2=value2"'. + format(flag_value)) + pairs.append((match.group(1), match.group(2))) + return dict(pairs) diff --git a/scripts/validation_utils_test.py b/scripts/validation_utils_test.py new file mode 100755 index 00000000..d759b276 --- /dev/null +++ b/scripts/validation_utils_test.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 + +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit test for validation_utils.py""" + +import argparse +import re + +import pytest + +import validation_utils + + +@pytest.mark.parametrize('container, field_name, field_type, expected', [ + # Normal case, field present and correct type + ({'present': 1}, 'present', int, 1), + ({'present': '1'}, 'present', str, '1'), + ({'present': [1]}, 'present', list, [1]), + ({'present': {1: 2}}, 'present', dict, {1: 2}), + # Missing field replaced by default + ({}, 'missing', str, ''), + # Valid conversions + ({'str_to_int': '1'}, 'str_to_int', int, 1), + ({'int_to_str': 1}, 'int_to_str', str, '1'), + # None + ({'None_to_int': None}, 'None_to_int', int, 0), + ({'None_to_str': None}, 'None_to_str', str, ''), +]) +def test_get_field_value_valid(container, field_name, field_type, expected): + assert validation_utils.get_field_value( + container, field_name, field_type) == expected + + +@pytest.mark.parametrize('container, field_name, field_type', [ + # Type conversion failures + ({'bad_list_to_dict': [1]}, 'bad_list_to_dict', dict), + ({'bad_list_to_str': [1]}, 'bad_list_to_str', str), + ({'bad_dict_to_list': {1: 2}}, 'bad_dict_to_list', list), + ({'bad_str_to_int': 'not_an_int'}, 'bad_str_to_int', int), + ({'bad_str_to_list': 'abc'}, 'bad_str_to_list', list), +]) +def test_get_field_value_invalid(container, field_name, field_type): + with pytest.raises(ValueError): + validation_utils.get_field_value(container, field_name, field_type) + + +def test_validate_arg_regex(): + assert validation_utils.validate_arg_regex( + 'abc', re.compile('a[b]c')) == 'abc' + with pytest.raises(argparse.ArgumentTypeError): + validation_utils.validate_arg_regex('abc', re.compile('a[d]c')) + + +@pytest.mark.parametrize('arg, expected', [ + # Normal case, field present and correct type + ('', {}), + ('_A=1', {'_A': '1'}), + ('_A=1,_B=2', {'_A': '1', '_B': '2'}), + # Repeated key is ok + ('_A=1,_A=2', {'_A': '2'}), + # Extra = is ok + ('_A=x=y=z,_B=2', {'_A': 'x=y=z', '_B': '2'}), + # No value is ok + ('_A=', {'_A': ''}), +]) +def test_validate_arg_dicts_valid(arg, expected): + assert validation_utils.validate_arg_dict(arg) == expected + + +@pytest.mark.parametrize('arg', [ + # No key + ',_A', + '_A,', + # Invalid variable name + '_Aa=1', + '_aA=1', + '0A=1', +]) +def test_validate_arg_dicts_invalid(arg): + with pytest.raises(argparse.ArgumentTypeError): + validation_utils.validate_arg_dict(arg) + + +if __name__ == '__main__': + pytest.main([__file__]) diff --git a/tests/benchmark/.gitignore b/tests/benchmark/.gitignore new file mode 100644 index 00000000..94143827 --- /dev/null +++ b/tests/benchmark/.gitignore @@ -0,0 +1 @@ +Dockerfile diff --git a/tests/benchmark/Dockerfile.in b/tests/benchmark/Dockerfile.in new file mode 100644 index 00000000..ca0edc24 --- /dev/null +++ b/tests/benchmark/Dockerfile.in @@ -0,0 +1,24 @@ +FROM ${STAGING_IMAGE} + +# Install performance +RUN pip install performance + +# Create virtual environment +RUN pip install --upgrade virtualenv + +# Required for Python 3.4, see +# https://bugs.launchpad.net/ubuntu/+source/python3.4/+bug/1290847 +RUN apt-get update && apt-get install -y --force-yes python3-pip python3-venv + +RUN mkdir /result + +# Run the benchmark and compare the performance, add the +# --debug-single-value flag to let the benchmark run in fastest mode +RUN pyperformance run --debug-single-value --python=python2.7 -o /result/py2.7.json +RUN pyperformance run --debug-single-value --python=python3.4 -o /result/py3.4.json +RUN pyperformance run --debug-single-value --python=python3.5 -o /result/py3.5.json +RUN if [ -e "/opt/python3.6/bin/python3.6" ]; then pyperformance run --debug-single-value --python=python3.6 -o /result/py3.6.json; fi + +RUN pyperformance compare /result/py2.7.json /result/py3.4.json --output_style table +RUN pyperformance compare /result/py3.4.json /result/py3.5.json --output_style table +RUN if [ -e "/result/py3.6.json" ]; then pyperformance compare /result/py3.5.json /result/py3.6.json --output_style table; fi \ No newline at end of file diff --git a/tests/benchmark/benchmark_between_releases.sh b/tests/benchmark/benchmark_between_releases.sh new file mode 100755 index 00000000..0e105cf3 --- /dev/null +++ b/tests/benchmark/benchmark_between_releases.sh @@ -0,0 +1,72 @@ +#!/bin/bash + +# Build the benchmark image for release 1 from Dockerfile +echo "Building image for release 1" +export STAGING_IMAGE="${DOCKER_NAMESPACE}/python:${TAG1}" +envsubst <"Dockerfile".in >"Dockerfile" '$STAGING_IMAGE' +docker build --no-cache -t benchmark_1 . +rm Dockerfile + +# Build the benchmark image for release 2 from Dockerfile +echo "Building image for release 2" +export STAGING_IMAGE="${DOCKER_NAMESPACE}/python:${TAG2}" +envsubst <"Dockerfile".in >"Dockerfile" '$STAGING_IMAGE' +docker build --no-cache -t benchmark_2 . +rm Dockerfile + +echo "Successfully built images" + +# Create folders to hold the files +mkdir "$TAG1" +mkdir "$TAG2" + +# Start running the containers and copy the benchmark result for python versions from container to host +docker run -it --name benchmark_1 -h CONTAINER1 -v "${PWD}"/"$TAG1":/export benchmark_1 /bin/bash -c "cp /result/py*.json /export/" +docker run -it --name benchmark_2 -h CONTAINER2 -v "${PWD}"/"$TAG2":/export benchmark_2 /bin/bash -c "cp /result/py*.json /export/" + +echo "Start benchmarking the python interpreter performance between the two releases" + +# Compare the performance between the interpreter in different release +pyperformance compare "$TAG1"/py2.7.json "$TAG2"/py2.7.json --output_style table > py2.7_res +pyperformance compare "$TAG1"/py3.4.json "$TAG2"/py3.4.json --output_style table > py3.4_res +pyperformance compare "$TAG1"/py3.5.json "$TAG2"/py3.5.json --output_style table > py3.5_res + +# Check if the python3.6 benchmark result exists +if [[ ( -e '"$TAG1"/py3.6.json' ) && ( -e '"$TAG2"/py3.6.json' ) ]]; then + pyperformance compare "$TAG1"/py3.6.json "$TAG2"/py3.6.json --output_style table > py3.6_res; +fi + +echo "Start extracting data and generating CSV file, then upload to Cloud Storage and insert to Big Query table" + +# Extracting memory usage and running time data from the performace result json, generating CSV files +for path_to_file in $TAG1/*.json; do + python generate_csv.py --filename $path_to_file --tag $TAG1 +done + +for path_to_file in $TAG2/*.json; do + python generate_csv.py --filename $path_to_file --tag $TAG2 +done + +# Set the project that hold the cloud storage bucket and big query tables +gcloud config set project cloud-python-runtime-qa + +# Get the list of existing release data on Cloud Storage and skip if the current TAG1 or TAG2 existing in the list +gsutil ls gs://python-runtime-benchmark > existing_releases + +for container_tag in $TAG1 $TAG2; do + if grep --fixed-strings --quiet "$container_tag" existing_releases; then + echo "Performance data of $container_tag existed, so skip processing it." + else + # Upload the CSV files to Cloud Storage + gsutil cp -r $container_tag gs://python-runtime-benchmark + # Load the CSV files from Cloud Storage to Big Query table + # Load the performance data of each function + for path_to_file in $container_tag/py2.7.csv $container_tag/py3.4.csv $container_tag/py3.5.csv; do + bq load benchmark.benchmark_functions gs://python-runtime-benchmark/"$path_to_file" container_tag:string,runtime_version:string,function_name:string,time_used:float,mem_usage:float + done + # Load the average performance data of each runtime version in a release + bq load benchmark.benchmark_statistics gs://python-runtime-benchmark/"$container_tag"/averages.csv container_tag:string,runtime_version:string,ave_time_used:float,ave_mem_usage:float + fi +done + +echo "Completed" diff --git a/tests/benchmark/generate_csv.py b/tests/benchmark/generate_csv.py new file mode 100644 index 00000000..e9882303 --- /dev/null +++ b/tests/benchmark/generate_csv.py @@ -0,0 +1,106 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import argparse +import csv +import json +import os +import sys + + +def generate_csv(filename, tag): + """Extract function name, time used and memory usage from the metadata and write to the output CSV file. + + Args: + filename (str): Filename of the performance json file to read + tag (str): Tag of the docker container + """ + with open(filename) as input: + data = json.load(input) + benchmarks = data["benchmarks"] + runtime_version = os.path.basename(filename).split(".json")[0] + + # Write data to CSV file + with open("{}.csv".format(os.path.splitext(filename)[0]), "wb") as output: + csv_writer = csv.writer(output, delimiter=',') + for benchmark in benchmarks: + try: + # Get the function name + func_name = benchmark["metadata"]["name"] + # Get the time used for this function, convert to millisecond + time_used = float(benchmark["runs"][0]["values"][0]) * 1000 + # Get the memory usage, convert to MB + mem_usage = float(benchmark["metadata"]["mem_max_rss"]) / float(1<<20) + line = [tag, runtime_version, func_name, time_used, mem_usage] + # Write to CSV file + csv_writer.writerow(line) + except KeyError: + # Skip the benchmark result if it does not contain the fields we want + pass + + +def get_averages(filename, tag): + """Calculate the averages of time_used and memory_usage and append to CSV file. + + Args: + filename (str): Filename of the performance json file to read + tag (str): Tag of the docker container + """ + with open("{}.csv".format(os.path.splitext(filename)[0]), "rb") as input: + lines = input.readlines() + # Get the two columns of times_used and mem_usage + rows_of_data = [map(float, line.split(',')[-2:]) for line in lines] + # Calculate the sum of the two columns + col_sums = map(sum, zip(*rows_of_data)) + # Calculate the average of the two columns by using the sum divided by the total number of lines + averages = [col_sum / len(lines) for col_sum in col_sums] + + # Get the runtime version from filename + runtime_version = os.path.basename(filename).split(".json")[0] + + # Write the averages to CSV file in appending mode + with open("{}/averages.csv".format(tag), "a+") as output: + try: + csv_writer = csv.writer(output, delimiter=',') + csv_writer.writerow([tag, runtime_version] + averages) + except IOError: + print "Could not write averages to file." + + +def parse_args(argv): + """Parse and validate command line flags""" + parser = argparse.ArgumentParser( + description='Read the python performance json file and extract data to genarate CSV file.' + ) + parser.add_argument( + '--filename', + help='Filename of the performance json file to read' + ) + parser.add_argument( + '--tag', + help='Tag of the docker container' + ) + args = parser.parse_args(argv[1:]) + return args + + +def main(): + args = parse_args(sys.argv) + generate_csv(args.filename, args.tag) + get_averages(args.filename, args.tag) + + +if __name__ == '__main__': + main() diff --git a/tests/deploy_check/app.yaml b/tests/deploy_check/app.yaml new file mode 100644 index 00000000..c4a838e6 --- /dev/null +++ b/tests/deploy_check/app.yaml @@ -0,0 +1,3 @@ +runtime: python +env: flex +entrypoint: gunicorn -b :$PORT main:app diff --git a/tests/deploy_check/main.py b/tests/deploy_check/main.py new file mode 100644 index 00000000..97eb37d8 --- /dev/null +++ b/tests/deploy_check/main.py @@ -0,0 +1,43 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START app] +import logging + +from flask import Flask + + +app = Flask(__name__) + + +@app.route('/') +def hello(): + """Return a friendly HTTP greeting.""" + return 'Hello World!' + + +@app.errorhandler(500) +def server_error(e): + logging.exception('An error occurred during a request.') + return """ + An internal error occurred: 
{}
+ See logs for full stacktrace. + """.format(e), 500 + + +if __name__ == '__main__': + # This is used when running locally. Gunicorn is used to run the + # application on Google App Engine. See entrypoint in app.yaml. + app.run(host='127.0.0.1', port=8080, debug=True) +# [END app] diff --git a/tests/deploy_check/requirements.txt b/tests/deploy_check/requirements.txt new file mode 100644 index 00000000..a34d076b --- /dev/null +++ b/tests/deploy_check/requirements.txt @@ -0,0 +1,2 @@ +Flask==1.0.2 +gunicorn==19.9.0 diff --git a/tests/eventlet/.gitignore b/tests/eventlet/.gitignore new file mode 100644 index 00000000..14be8077 --- /dev/null +++ b/tests/eventlet/.gitignore @@ -0,0 +1,2 @@ +Dockerfile +main.py diff --git a/tests/eventlet/Dockerfile.in b/tests/eventlet/Dockerfile.in new file mode 100644 index 00000000..fa65a236 --- /dev/null +++ b/tests/eventlet/Dockerfile.in @@ -0,0 +1,13 @@ +FROM ${STAGING_IMAGE} +LABEL python_version=python3.6 +RUN virtualenv --no-download /env -p python3.6 + +# Set virtualenv environment variables. This is equivalent to running +# source /env/bin/activate +ENV VIRTUAL_ENV /env +ENV PATH /env/bin:$PATH +ADD requirements.txt /app/ +RUN pip install -r requirements.txt +ADD . /app/ +RUN gunicorn -k eventlet -b :$PORT --daemon main:app ; \ + wget --retry-connrefused --tries=5 http://localhost:$PORT/ diff --git a/tests/eventlet/README.md b/tests/eventlet/README.md new file mode 100644 index 00000000..0c2a969c --- /dev/null +++ b/tests/eventlet/README.md @@ -0,0 +1 @@ +# Test the Python base image against the 'eventlet' library diff --git a/tests/eventlet/requirements.txt b/tests/eventlet/requirements.txt new file mode 100644 index 00000000..1e2ea66b --- /dev/null +++ b/tests/eventlet/requirements.txt @@ -0,0 +1,10 @@ +click==6.7 +enum-compat==0.0.2 +eventlet==0.24.1 +Flask==2.2.5 +greenlet==0.4.14 +gunicorn==19.9.0 +itsdangerous==0.24 +Jinja2==2.10 +MarkupSafe==1.1.1 +Werkzeug==2.2.3 diff --git a/tests/google-cloud-python/.gitignore b/tests/google-cloud-python/.gitignore new file mode 100644 index 00000000..94143827 --- /dev/null +++ b/tests/google-cloud-python/.gitignore @@ -0,0 +1 @@ +Dockerfile diff --git a/tests/google-cloud-python/Dockerfile.in b/tests/google-cloud-python/Dockerfile.in new file mode 100644 index 00000000..b82c18d1 --- /dev/null +++ b/tests/google-cloud-python/Dockerfile.in @@ -0,0 +1,15 @@ +FROM ${STAGING_IMAGE} + +# Get the source. +RUN git clone --depth 1 https://github.com/GoogleCloudPlatform/google-cloud-python.git +WORKDIR google-cloud-python + +# Upgrade setuptools +RUN pip install --upgrade setuptools + +# Install nox +RUN pip install --upgrade nox-automation + +# Run unit tests for all supported Python versions +ADD run_unit_tests.sh /run_unit_tests.sh +ENTRYPOINT ["/run_unit_tests.sh"] diff --git a/tests/google-cloud-python/run_unit_tests.sh b/tests/google-cloud-python/run_unit_tests.sh new file mode 100755 index 00000000..c386f1c0 --- /dev/null +++ b/tests/google-cloud-python/run_unit_tests.sh @@ -0,0 +1,34 @@ +#!/bin/sh +set -eu + +cd /app/google-cloud-python + +exit_code=0 +failed_files= +for noxfile in */nox.py; do + if [ "${noxfile}" = "dlp/nox.py" ]; then + echo "**** Skipping ${noxfile} ****" + continue + fi + echo "**** Starting tests in ${noxfile} ****" + nox \ + -f "${noxfile}" \ + -e \ + "unit(py='2.7')" \ + "unit(py='3.4')" \ + "unit(py='3.5')" \ + "unit(py='3.6')" \ + || { + echo "**** FAILED tests in ${noxfile} ****" + exit_code=1 + failed_files="${failed_files} ${noxfile}" + } + echo "**** Finished tests in ${noxfile} ****" +done + +if [ "${exit_code}" -eq 0 ]; then + echo "**** All tests passed ****" +else + echo "**** There were test failures:${failed_files} ****" +fi +exit "${exit_code}" diff --git a/tests/integration/.gitignore b/tests/integration/.gitignore new file mode 100644 index 00000000..94143827 --- /dev/null +++ b/tests/integration/.gitignore @@ -0,0 +1 @@ +Dockerfile diff --git a/tests/integration/Dockerfile.in b/tests/integration/Dockerfile.in new file mode 100644 index 00000000..e18d60ef --- /dev/null +++ b/tests/integration/Dockerfile.in @@ -0,0 +1,22 @@ +# Copyright 2016 Google Inc. All rights reserved. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM ${STAGING_IMAGE} + +COPY . /app +WORKDIR /app + +RUN pip install -r requirements.txt + +ENTRYPOINT ["gunicorn", "-b", ":8080", "server:app"] diff --git a/tests/integration/app.yaml b/tests/integration/app.yaml new file mode 100644 index 00000000..ce2a1243 --- /dev/null +++ b/tests/integration/app.yaml @@ -0,0 +1,2 @@ +runtime: custom +env: flex diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt new file mode 100644 index 00000000..a0634bee --- /dev/null +++ b/tests/integration/requirements.txt @@ -0,0 +1,9 @@ +Flask==2.2.5 +google-cloud-error-reporting==0.32.1 +google-cloud-logging==1.12.1 +google-cloud-monitoring==0.33.0 +gunicorn==19.9.0 +requests==2.31.0 +retrying==1.3.3 +six==1.12.0 +protobuf>=3.6.0 diff --git a/tests/integration/server.py b/tests/integration/server.py new file mode 100755 index 00000000..0b4382c6 --- /dev/null +++ b/tests/integration/server.py @@ -0,0 +1,283 @@ +#!/usr/bin/python + +# Copyright 2017 Google Inc. All rights reserved. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from functools import wraps +import json +import logging +import os + +import google.cloud.logging +import google.cloud.monitoring +import google.cloud.error_reporting +import google.cloud.exceptions + +from flask import Flask, request, jsonify + +log_funcs = { + 'DEBUG': (logging.debug, 'stderr'), + 'INFO': (logging.info, 'stderr'), + 'WARNING': (logging.warn, 'stderr'), + 'ERROR': (logging.error, 'stderr'), + 'CRITICAL': (logging.critical, 'stderr') +} + +_APPENGINE_FLEXIBLE_ENV_VM = 'GAE_APPENGINE_HOSTNAME' +"""Environment variable set in App Engine when vm:true is set.""" + +_APPENGINE_FLEXIBLE_ENV_FLEX = 'GAE_INSTANCE' +"""Environment variable set in App Engine when env:flex is set.""" + +app = Flask(__name__) + + +def verify_request(f): + @wraps(f) + def verified_func(*args, **kwargs): + request_data = request.get_json() + if request_data is None: + raise ErrorResponse('Unable to parse request JSON: ' + 'did you set the Content-type header?') + token = request_data.get('token') + if not token: + raise ErrorResponse('Please provide token name') + return f(*args, request_data=request_data, token=token, **kwargs) + return verified_func + + +@app.route('/') +def hello_world(): + return 'Hello World!' + + +@app.route('/logging_custom', methods=['POST']) +@verify_request +def _logging_custom(request_data, token): + log_name = request_data.get('log_name') + if not log_name: + raise ErrorResponse('Please provide log name') + level = request_data.get('level') + if not level: + raise ErrorResponse('Please provide log level') + + log_source = _log_custom(token, log_name, level) + + return log_source, 200 + + +@app.route('/logging_standard', methods=['POST']) +@verify_request +def _logging_standard(request_data, token): + level = request_data.get('level') + if not level: + raise ErrorResponse('Please provide log level') + + log_source = _log_default(token, level) + + return log_source, 200 + + +def _log_custom(token, log_name, level): + """ + Write a custom log entry to Stackdriver using a client library. + + Keyword arguments: + token -- 16-character (8-byte) hexadecimal token, to be written + as a log entry. + log_name -- The name of the logging group to be written to. + level -- enum(LogSeverity), level of the log to write + + Once the entry is written to Stackdriver, the test driver will retrieve + all entries with the name 'log_name' at level 'level', and verify there + is an entry with the same value as 'token', indicating the entry + was written successfully. + """ + try: + client = google.cloud.logging.Client() + gcloud_logger = client.logger(log_name) + gcloud_logger.log_text(token, severity=str(level)) + except google.cloud.exceptions.GoogleCloudError as e: + logging.error('Error while writing logs: {0}'.format(e)) + raise ErrorResponse('Error while writing logs: {0}'.format(e)) + + return log_name + + +def _log_default(token, level): + """ + Write a log entry to Stackdriver through the default logging module. + + Keyword arguments: + token -- 16-character (8-byte) hexadecimal token, to be written + as a log entry. + level -- enum(LogSeverity), level of the log to write + + Once the entry is written to Stackdriver, the test driver will retrieve + all entries from the default log stream (sent back to the driver) at level + 'level', and verify there is an entry with the same value as 'token', + indicating the entry was written successfully. + """ + + try: + func_pair = log_funcs[level] + f = func_pair[0] + source = func_pair[1] + f(token) + except google.cloud.exceptions.GoogleCloudError as e: + logging.error('Error while writing logs: {0}'.format(e)) + raise ErrorResponse('Error while writing logs: {0}'.format(e)) + + # this is fine regardless of environment, it's only used in GAE logs + return 'appengine.googleapis.com%2F{0}'.format(source) + + +@app.route('/monitoring', methods=['POST']) +@verify_request +def _monitoring(request_data, token): + name = request_data.get('name') + if not name: + raise ErrorResponse('Please provide metric name') + + try: + client = google.cloud.monitoring.Client() + + try: + descriptor = client.fetch_metric_descriptor(name) + if descriptor is None: + _create_descriptor(name, client) + except (google.cloud.exceptions.Forbidden, + google.cloud.exceptions.NotFound) as ignored: # noqa: F841 + _create_descriptor(name, client) + + _write_metric(name, client, token) + + except google.cloud.exceptions.GoogleCloudError as e: + logging.error('Error while writing custom metric: {0}'.format(e)) + raise ErrorResponse('Error while writing custom metric: {0}'.format(e)) + + return 'OK', 200 + + +def _write_metric(name, client, token): + """ + Write a metric to Stackdriver Monitoring. + + Keyword arguments: + name -- The name of the metric to write. Takes the form + 'custom.googleapis.com/{metric_name}' + client -- the authenticated instance of a Google Cloud Client + token -- a random 64-bit integer token. The metric value to be written. + + Once the metric is written, the test driver will retrieve all metrics + written with the provided name, and verify there is an entry with the + same value as the provided token. + """ + metric = client.metric(name, {}) + resource = client.resource('global', labels={}) + client.write_point(metric, resource, token) + + +def _create_descriptor(name, client): + """ + Create a new metric descriptor. + This descriptor is implicitly used to write a point-value metric to + Stackdriver. + """ + logging.info('No descriptor found with name {0}: Creating...'.format(name)) + descriptor = client.metric_descriptor( + name, + metric_kind=google.cloud.monitoring.MetricKind.GAUGE, + value_type=google.cloud.monitoring.ValueType.INT64, + description='Test Metric') + descriptor.create() + + +@app.route('/exception', methods=['POST']) +@verify_request +def _exception(request_data, token): + try: + client = google.cloud.error_reporting.Client() + try: + raise NameError + except Exception: + client.report_exception() + + client.report(token) + except google.cloud.exceptions.GoogleCloudError as e: + logging.error('Error while reporting exception: {0}'.format(e)) + raise ErrorResponse('Error while reporting exception: {0}'.format(e)) + + return 'OK', 200 + + +@app.route('/trace', methods=['POST']) +def _trace(): + return 'OK', 204 + + +@app.route('/custom', methods=['GET']) +def _custom(): + tests = [ + { + 'name': 'foo', + 'path': '/', + 'timeout': 500 + } + ] + return json.dumps(tests), 200 + + +@app.route('/environment', methods=['GET']) +def _check_environment(): + # determine what cloud env we're running in; essentially, GAE vs GKE + # for GAE, we'll check the existence env vars set on + # vm:true or env:flex + # if neither exist, assume we're in GKE + environment = "GKE" + if (_APPENGINE_FLEXIBLE_ENV_VM in os.environ or + _APPENGINE_FLEXIBLE_ENV_FLEX in os.environ): + environment = "GAE" + return environment, 200 + + +class ErrorResponse(Exception): + status_code = 400 + + def __init__(self, message, status_code=None, payload=None): + Exception.__init__(self) + self.message = message + if status_code is not None: + self.status_code = status_code + self.payload = payload + + def to_dict(self): + rv = dict(self.payload or ()) + rv['message'] = self.message + return rv + + +@app.errorhandler(ErrorResponse) +def handle_invalid_usage(error): + response = jsonify(error.to_dict()) + response.status_code = error.status_code + return response + + +if __name__ == '__main__': + # set up logging module to write to Stackdriver + client = google.cloud.logging.Client() + client.setup_logging(log_level=logging.DEBUG) + logging.getLogger().setLevel(logging.DEBUG) + app.run(debug=True, port=8080) diff --git a/tests/license-test/license-test.yaml b/tests/license-test/license-test.yaml new file mode 100644 index 00000000..348fbc9b --- /dev/null +++ b/tests/license-test/license-test.yaml @@ -0,0 +1,6 @@ +schemaVersion: "1.0.0" + +# See https://github.com/GoogleCloudPlatform/container-structure-test/blob/master/README.md +licenseTests: + - debian: true + files: [] diff --git a/tests/no-virtualenv/Dockerfile b/tests/no-virtualenv/Dockerfile deleted file mode 100644 index 68089c81..00000000 --- a/tests/no-virtualenv/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM google/python - -RUN if [ "$(which python)" != "/usr/bin/python" ]; then exit 1; fi; -RUN pip install gunicorn -RUN if [ "$(which gunicorn)" != "/usr/local/bin/gunicorn" ]; then exit 1; fi; diff --git a/tests/no-virtualenv/no-virtualenv.yaml b/tests/no-virtualenv/no-virtualenv.yaml new file mode 100644 index 00000000..4f6c3f48 --- /dev/null +++ b/tests/no-virtualenv/no-virtualenv.yaml @@ -0,0 +1,55 @@ +schemaVersion: "1.0.0" +commandTests: + - name: "default python installation" + command: ["which", "python"] + expectedOutput: ["/usr/bin/python\n"] + + - name: "default pip installation" + command: ["which", "pip"] + expectedOutput: ["/usr/local/bin/pip\n"] + + - name: "default pip python version" + command: ["pip", "-V"] + expectedOutput: ["pip .* from .*python 2[.]7"] + + - name: "default virtualenv installation" + command: ["which", "virtualenv"] + expectedOutput: ["/usr/local/bin/virtualenv\n"] + + - name: "default python2.7 installation" + command: ["which", "python2.7"] + expectedOutput: ["/usr/bin/python2.7\n"] + + - name: "default python3.4 installation" + command: ["which", "python3.4"] + expectedOutput: ["/opt/python3.4/bin/python3.4\n"] + + - name: "default python3.5 installation" + command: ["which", "python3.5"] + expectedOutput: ["/opt/python3.5/bin/python3.5\n"] + + - name: "default python3.6 installation" + command: ["which", "python3.6"] + expectedOutput: ["/opt/python3.6/bin/python3.6\n"] + + - name: "default gunicorn installation" + setup: [["pip", "install", "gunicorn"]] + command: ["which", "gunicorn"] + expectedOutput: ["/usr/local/bin/gunicorn\n"] + + - # Regression test for issue187 + name: "default python3 installation" + command: ["which", "python3"] + expectedOutput: ["/usr/local/bin/python3\n"] + - name: "default python3 version" + command: ["python3", "--version"] + expectedOutput: ["Python 3.7.9\n"] + - name: "default pip3 installation" + command: ["which", "pip3"] + expectedOutput: ["/usr/local/bin/pip3\n"] + + - name: "default flask installation" + # Checks that 'pip' and 'python' are using the same Python version + setup: [["pip", "install", "flask"]] + command: ["python", "-c", "import flask; print(flask.__file__)"] + expectedOutput: ["/usr/local/lib/python2.7/dist-packages/flask"] diff --git a/tests/python2-libraries/Dockerfile b/tests/python2-libraries/Dockerfile index 11a7dc25..c2647292 100644 --- a/tests/python2-libraries/Dockerfile +++ b/tests/python2-libraries/Dockerfile @@ -1,7 +1,3 @@ -FROM google/python - -RUN virtualenv /env -ENV VIRTUAL_ENV /env -ENV PATH /env/bin:$PATH -ADD requirements.txt /app/requirements.txt -RUN pip install -r /app/requirements.txt +ARG intermediate_image +FROM $intermediate_image +COPY requirements.txt /requirements.txt diff --git a/tests/python2-libraries/python2-libraries.yaml b/tests/python2-libraries/python2-libraries.yaml new file mode 100644 index 00000000..f9fef5a0 --- /dev/null +++ b/tests/python2-libraries/python2-libraries.yaml @@ -0,0 +1,13 @@ +schemaVersion: "1.0.0" + +globalEnvVars: + - key: "VIRTUAL_ENV" + value: "/env" + - key: "PATH" + value: "/env/bin:$PATH" + +commandTests: + - name: "requirements" + setup: [["virtualenv", "-p", "python", "/env"]] + command: ["pip", "install", "-r", "/requirements.txt"] + exitCode: 0 diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index f2f261e4..f0596d67 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -1,211 +1,207 @@ -simplejson -setuptools -six -requests -virtualenv -pip -distribute -python-dateutil -certifi -boto -pbr -#wincertstore -docutils -pyasn1 -pyyaml -jinja2 -markupsafe -pytz -nose -lxml -pycrypto -rsa -colorama -botocore -cffi -awscli -coverage -jmespath -pycparser -pika -django -psycopg2 -paramiko -ecdsa -sqlalchemy -mock -redis -werkzeug -selenium -bcdoc -supervisor -pep8 -httplib2 -flask -pymongo -zc.buildout -psutil -mysql-python -argparse -carbon -pygments -babel -paste -anyjson -meld3 -# ssl (already included in standard library) -cryptography -py -tornado -pyopenssl -greenlet -kombu -graphite-web -docopt -mako -itsdangerous -pillow -wheel -beautifulsoup4 -enum34 -pyflakes -zope.interface -decorator -futures -pastedeploy -ordereddict -setuptools-git -fabric -backports.ssl_match_hostname -amqp -numpy -sphinx -iso8601 -flake8 -celery -pyparsing -mccabe -stevedore -pytest -webob -gunicorn -urllib3 -billiard -jsonschema -msgpack-python -gevent -logilab-common -unittest2 -prettytable -pylint -blessings -south -mozrunner -netaddr -oslo.config -twisted -ipaddress -ujson -moznetwork -mozdevice -mozprofile -mozprocess -mozfile -mozinfo -html5lib -mozlog -mozcrash -oauthlib -idna -ipython -tox -astroid -google-api-python-client -pycurl -isodate -python-keystoneclient -websocket-client -markdown -python-mimeparse -python-daemon -raven -suds -oauth2client -cython -eventlet -netifaces -repoze.lru -thrift -sqlparse -ndg-httpsclient -djangorestframework -python-novaclient -testtools -alembic -uritemplate -statsd -python-memcached -coveralls -funcsigs -configobj -linecache2 -extras -beautifulsoup -# scikit-learn -cliff -oauth2 -# pycups -cmd2 -unidecode -newrelic -python-gflags -cov-core -pytest-cov -fixtures -pyasn1-modules -python-swiftclient -django-debug-toolbar -elasticsearch -webtest -docker-py -python-subunit -retrying -django-extensions -pystache -waitress -pexpect -blinker -scipy -requests-oauthlib -protobuf -manifestparser -passlib -ansible -click -versiontools -django_compress -pyzmq -chardet -xlrd -snowballstemmer -testrepository -pandas -functools32 -python-cjson -pastescript -warlock -sqlalchemy-migrate -django-celery -uwsgi -cssselect -# Hand selected -matplotlib -pymysql -amqplib -sh -m2crypto -apache-libcloud -hiredis -bottle -pyramid -pyjwt +alembic==1.0.0 +amqp==2.3.2 +amqplib==1.0.2 +ansible==7.0.0 +anyjson==0.3.3 +apache-libcloud==2.3.0 +argparse==1.4.0 +astroid==1.6.5 +awscli==1.16.1 +babel==2.6.0 +backports.ssl_match_hostname==3.5.0.1 +bcdoc==0.16.0 +beautifulsoup4==4.6.3 +beautifulsoup==3.2.1 +billiard==3.5.0.4 +blessings==1.7 +blinker==1.4 +boto==2.49.0 +botocore==1.11.1 +bottle==0.12.13 +carbon<1.1.1 +celery==4.2.1 +certifi==2023.7.22 +cffi==1.11.5 +chardet==3.0.4 +click==6.7 +cliff==2.13.0 +cmd2==0.8.9 +colorama==0.3.9 +configobj==5.0.6 +cov-core==1.15.0 +coverage==4.5.1 +coveralls==1.4.0 +crcmod==1.7 +cryptography==2.3.1 +cssselect==1.0.3 +cython==0.28.5 +decorator==4.3.0 +django-celery==3.2.2 +django-debug-toolbar==1.9.1 +django-extensions==2.1.1 +django<2.0 +django_compress==1.0.1 +djangorestframework==3.8.2 +docker-py==1.10.6 +docopt==0.6.2 +docutils==0.14 +ecdsa==0.13 +elasticsearch==6.3.1 +enum34==1.1.6 +eventlet==0.24.1 +extras==1.0.0 +fabric==2.3.1 +fixtures==3.0.0 +flake8==3.5.0 +flask==2.2.5 +funcsigs==1.0.2 +functools32==3.2.3.post2 +futures==3.2.0 +gevent==1.3.6 +google-api-python-client==1.7.4 +graphite-web==1.1.3 +greenlet==0.4.14 +gunicorn==19.9.0 +hiredis==0.2.0 +honcho==1.0.1 +html5lib==1.0.1 +httplib2==0.11.3 +idna==2.7 +ipaddress==1.0.22 +iso8601==0.1.12 +isodate==0.6.0 +itsdangerous==0.24 +jinja2==2.10 +jmespath==0.9.3 +jsonschema==2.6.0 +kombu==4.2.1 +linecache2==1.0.0 +logilab-common==1.4.2 +lxml==4.2.4 +m2crypto==0.30.1 +mako==1.0.7 +manifestparser==1.1 +markdown==2.6.11 +markupsafe==1.0 +matplotlib==2.2.3 +mccabe==0.6.1 +meld3==1.0.2 +mock==2.0.0 +mozcrash==1.0 +mozdevice==1.0.1 +mozfile==1.2 +mozinfo==0.10 +mozlog==3.8 +moznetwork==0.27 +mozprocess==0.26 +mozprofile==1.1.0 +mozrunner==7.0.1 +msgpack-python==0.5.6 +mysql-python==1.2.5 +ndg-httpsclient==0.5.1 +netaddr==0.7.19 +netifaces==0.10.7 +newrelic==4.2.0.100 +nose==1.3.7 +numpy==1.22.0 +oauth2==1.9.0.post1 +oauth2client==4.1.2 +oauthlib==2.1.0 +ordereddict==1.1 +oslo.config==6.4.0 +pandas==0.23.4 +paramiko==2.4.1 +passlib==1.7.1 +paste==2.0.3 +pastedeploy==1.5.2 +pastescript==2.0.2 +pbr==4.2.0 +pep8==1.7.1 +pexpect==4.6.0 +pika==0.12.0 +pillow==9.3.0 +pip==18.0 +prettytable==0.7.2 +protobuf==3.6.1 +psutil==5.4.7 +psycopg2==2.7.5 +py==1.5.4 +pyasn1-modules==0.2.2 +pyasn1==0.4.4 +pycparser==2.18 +pycrypto==2.6.1 +pycurl==7.43.0.2 +pyflakes==2.0.0 +pygments==2.2.0 +pyjwt==1.6.4 +pylibmc==1.5.2 +pylint==1.9.3 +pymongo==3.7.1 +pymysql==0.9.2 +pyopenssl==18.0.0 +pyparsing==2.2.0 +pyramid==1.9.2 +pystache==0.5.4 +pytest-cov==2.5.1 +pytest==3.7.3 +python-cjson==1.2.1 +python-daemon==2.2.0 +python-dateutil==2.7.3 +python-gflags==3.1.2 +python-keystoneclient==3.17.0 +python-memcached==1.59 +python-mimeparse==1.6.0 +python-novaclient==11.0.0 +python-subunit==1.3.0 +python-swiftclient==3.6.0 +pytz==2018.5 +pyyaml==5.4 +pyzmq==17.1.2 +raven==6.9.0 +redis==2.10.6 +repoze.lru==0.7 +requests-oauthlib==1.0.0 +requests==2.31.0 +retrying==1.3.3 +rsa==3.4.2 +scipy==1.10.0 +selenium==3.14.0 +setuptools-git==1.2 +setuptools==40.2.0 +sh==1.12.14 +simplejson==3.16.0 +six==1.11.0 +snowballstemmer==1.2.1 +south==1.0.2 +sphinx==1.7.7 +sqlalchemy-migrate==0.11.0 +sqlalchemy==1.2.11 +sqlparse==0.4.4 +statsd==3.3.0 +stevedore==1.29.0 +suds==0.4 +supervisor==3.3.4 +testrepository==0.0.20 +testtools==2.3.0 +thrift==0.11.0 +tornado==6.3.3 +tox==3.2.1 +twisted==18.7.0 +ujson==1.35 +unidecode==1.0.22 +unittest2==1.1.0 +uritemplate==3.0.0 +urllib3==1.26.5 +uwsgi==2.0.22 +versiontools==1.9.1 +virtualenv==16.0.0 +waitress==2.1.2 +warlock==1.3.0 +webob==1.8.2 +websocket-client==0.51.0 +webtest==2.0.30 +werkzeug==2.2.3 +wheel==0.31.1 +xlrd==1.1.0 +zc.buildout==2.12.1 +zope.interface==4.5.0 diff --git a/tests/python3-libraries/Dockerfile b/tests/python3-libraries/Dockerfile new file mode 100644 index 00000000..c2647292 --- /dev/null +++ b/tests/python3-libraries/Dockerfile @@ -0,0 +1,3 @@ +ARG intermediate_image +FROM $intermediate_image +COPY requirements.txt /requirements.txt diff --git a/tests/python3-libraries/python3-libraries.yaml b/tests/python3-libraries/python3-libraries.yaml new file mode 100644 index 00000000..ace58132 --- /dev/null +++ b/tests/python3-libraries/python3-libraries.yaml @@ -0,0 +1,18 @@ +schemaVersion: "1.0.0" + +globalEnvVars: + - key: "VIRTUAL_ENV" + value: "/env" + - key: "PATH" + value: "/env/bin:$PATH" + +commandTests: + - name: "requirements 3.5" + setup: [["virtualenv", "-p", "/opt/python3.5/bin/python3.5", "/env"]] + command: ["pip", "install", "-r", "/requirements.txt"] + exitCode: 0 + + - name: "requirements 3.6" + setup: [["virtualenv", "-p", "/opt/python3.6/bin/python3.6", "/env"]] + command: ["pip", "install", "-r", "/requirements.txt"] + exitCode: 0 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt new file mode 100644 index 00000000..23175a06 --- /dev/null +++ b/tests/python3-libraries/requirements.txt @@ -0,0 +1,195 @@ +alembic==1.0.0 +amqp==2.3.2 +amqplib==1.0.2 +ansible==7.0.0 +anyjson==0.3.3 +apache-libcloud==2.3.0 +argparse==1.4.0 +astroid==2.0.4 +awscli==1.16.1 +babel==2.6.0 +backports.ssl_match_hostname==3.5.0.1 +bcdoc==0.16.0 +beautifulsoup4==4.6.3 +billiard==3.5.0.4 +blessings==1.7 +blinker==1.4 +boto==2.49.0 +botocore==1.11.1 +bottle==0.12.13 +celery==4.2.1 +certifi==2023.7.22 +cffi==1.11.5 +chardet==3.0.4 +click==6.7 +cliff==2.13.0 +cmd2==0.9.4 +colorama==0.3.9 +configobj==5.0.6 +cov-core==1.15.0 +coverage==4.5.1 +coveralls==1.4.0 +crcmod==1.7 +cryptography==41.0.3 +cssselect==1.0.3 +cython==0.28.5 +decorator==4.3.0 +django-celery==3.2.2 +django-debug-toolbar==1.9.1 +django-extensions==2.1.1 +django==2.2.28 +django_compress==1.0.1 +djangorestframework==3.8.2 +docker-py==1.10.6 +docopt==0.6.2 +docutils==0.14 +ecdsa==0.13 +elasticsearch==6.3.1 +enum34==1.1.6 +eventlet==0.24.1 +extras==1.0.0 +fabric==2.3.1 +fixtures==3.0.0 +flake8==3.5.0 +flask==2.2.5 +funcsigs==1.0.2 +gevent==1.3.6 +google-api-python-client==1.7.4 +greenlet==0.4.14 +gunicorn==19.9.0 +hiredis==0.2.0 +honcho==1.0.1 +html5lib==1.0.1 +httplib2==0.11.3 +idna==2.7 +ipaddress==1.0.22 +ipython==6.5.0 +iso8601==0.1.12 +isodate==0.6.0 +itsdangerous==0.24 +jinja2==2.10 +jmespath==0.9.3 +jsonschema==2.6.0 +kombu==4.2.1 +linecache2==1.0.0 +logilab-common==1.4.2 +lxml==4.2.4 +mako==1.0.7 +manifestparser==1.1 +markdown==2.6.11 +markupsafe==1.1.1 +matplotlib==2.2.3 +mccabe==0.6.1 +meld3==1.0.2 +mock==2.0.0 +mozcrash==1.0 +mozdevice==1.0.1 +mozfile==1.2 +mozinfo==0.10 +mozlog==3.8 +moznetwork==0.27 +mozprocess==0.26 +msgpack-python==0.5.6 +ndg-httpsclient==0.5.1 +netaddr==0.7.19 +netifaces==0.10.7 +newrelic==4.2.0.100 +nose==1.3.7 +numpy==1.22.0 +oauth2==1.9.0.post1 +oauth2client==4.1.2 +oauthlib==2.1.0 +ordereddict==1.1 +oslo.config==6.4.0 +pandas==0.23.4 +paramiko==2.4.1 +passlib==1.7.1 +paste==2.0.3 +pastedeploy==1.5.2 +pastescript==2.0.2 +pbr==4.2.0 +pep8==1.7.1 +pexpect==4.6.0 +pika==0.12.0 +pillow==9.3.0 +pip==18.0 +prettytable==0.7.2 +protobuf==3.6.1 +psutil==5.4.7 +psycopg2==2.7.5 +py==1.5.4 +pyasn1-modules==0.2.2 +pyasn1==0.4.4 +pycparser==2.18 +pycrypto==2.6.1 +pyflakes==2.0.0 +pygments==2.15.0 +pyjwt==1.6.4 +pylibmc==1.5.2 +pylint==2.1.1 +pymongo==3.7.1 +pymysql==0.9.2 +pyopenssl==18.0.0 +pyparsing==2.2.0 +pyramid==1.9.2 +pystache==0.5.4 +pytest-cov==2.5.1 +pytest==3.7.3 +python-daemon==2.2.0 +python-dateutil==2.7.3 +python-gflags==3.1.2 +python-keystoneclient==3.17.0 +python-memcached==1.59 +python-mimeparse==1.6.0 +python-novaclient==11.0.0 +python-subunit==1.3.0 +python-swiftclient==3.6.0 +pytz==2018.5 +pyyaml==5.4 +pyzmq==17.1.2 +raven==6.9.0 +redis==2.10.6 +repoze.lru==0.7 +requests-oauthlib==1.0.0 +requests==2.31.0 +retrying==1.3.3 +rsa==3.4.2 +scipy==1.10.0 +selenium==3.14.0 +setuptools-git==1.2 +setuptools==40.2.0 +sh==1.12.14 +simplejson==3.16.0 +six==1.11.0 +snowballstemmer==1.2.1 +south==1.0.2 +sphinx==1.7.7 +sqlalchemy-migrate==0.11.0 +sqlalchemy==1.2.11 +sqlparse==0.4.4 +statsd==3.3.0 +stevedore==1.29.0 +testrepository==0.0.20 +testtools==2.3.0 +thrift==0.11.0 +tornado==6.3.3 +tox==3.2.1 +twisted==18.7.0 +ujson==1.35 +unidecode==1.0.22 +unittest2==1.1.0 +uritemplate==3.0.0 +urllib3==1.26.5 +uwsgi==2.0.22 +versiontools==1.9.1 +virtualenv==16.0.0 +waitress==2.1.2 +warlock==1.3.0 +webob==1.8.2 +websocket-client==0.51.0 +webtest==2.0.30 +werkzeug==2.2.3 +wheel==0.31.1 +xlrd==1.1.0 +zc.buildout==2.12.1 +zope.interface==4.5.0 diff --git a/tests/virtualenv/Dockerfile b/tests/virtualenv/Dockerfile deleted file mode 100644 index d4effb54..00000000 --- a/tests/virtualenv/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -FROM google/python - -ENV VIRTUAL_ENV /env -ENV PATH /env/bin:$PATH - -RUN if [ "$(which python)" != "/usr/bin/python" ]; then exit 1; fi; -RUN virtualenv /env -# All commands from this point on should use the virtualenv -RUN if [ "$(which python)" != "/env/bin/python" ]; then exit 1; fi; -RUN if [ "$(python --version 2>&1)" != "Python 2.7.9" ]; then exit 1; fi; -RUN if [ "$(which pip)" != "/env/bin/pip" ]; then exit 1; fi; -RUN pip install gunicorn flask -RUN if [ "$(which gunicorn)" != "/env/bin/gunicorn" ]; then exit 1; fi; -RUN python -c "import sys; import flask; sys.exit(0 if flask.__file__.startswith('/env') else 1)" - -# Python 3 -RUN rm -rf /env -RUN if [ "$(which python3.4)" != "/usr/bin/python3.4" ]; then exit 1; fi; -RUN virtualenv -p python3.4 /env -# All commands from this point on should use the virtualenv -RUN if [ "$(which python)" != "/env/bin/python" ]; then exit 1; fi; -RUN if [ "$(which python3)" != "/env/bin/python3" ]; then exit 1; fi; -RUN if [ "$(python --version 2>&1)" != "Python 3.4.2" ]; then exit 1; fi; -RUN if [ "$(which pip)" != "/env/bin/pip" ]; then exit 1; fi; -RUN if [ "$(which pip3)" != "/env/bin/pip3" ]; then exit 1; fi; -RUN pip install gunicorn flask -RUN if [ "$(which gunicorn)" != "/env/bin/gunicorn" ]; then exit 1; fi; -RUN python -c "import sys; import flask; sys.exit(0 if flask.__file__.startswith('/env') else 1)" diff --git a/tests/virtualenv/virtualenv_default.yaml b/tests/virtualenv/virtualenv_default.yaml new file mode 100644 index 00000000..6b6ad282 --- /dev/null +++ b/tests/virtualenv/virtualenv_default.yaml @@ -0,0 +1,37 @@ +schemaVersion: "1.0.0" + +globalEnvVars: + - key: "VIRTUAL_ENV" + value: "/env" + - key: "PATH" + value: "/env/bin:$PATH" + +commandTests: + - name: "virtualenv python installation" + setup: [["virtualenv", "/env"]] + command: ["which", "python"] + expectedOutput: ["/env/bin/python\n"] + + - name: "virtualenv python version" + setup: [["virtualenv", "/env"]] + command: ["python", "--version"] + # we check stderr instead of stdout for Python versions < 3.4 + # https://bugs.python.org/issue18338 + expectedError: ["Python 2.7.(9|12)\n"] + + - name: "virtualenv pip installation" + setup: [["virtualenv", "/env"]] + command: ["which", "pip"] + expectedOutput: ["/env/bin/pip\n"] + + - name: "virtualenv gunicorn installation" + setup: [["virtualenv", "/env"], + ["pip", "install", "gunicorn"]] + command: ["which", "gunicorn"] + expectedOutput: ["/env/bin/gunicorn"] + + - name: "virtualenv flask installation" + setup: [["virtualenv", "/env"], + ["pip", "install", "flask"]] + command: ["python", "-c", "import flask; print(flask.__file__)"] + expectedOutput: ["/env/lib/python2.7/site-packages/flask/__init__.pyc"] diff --git a/tests/virtualenv/virtualenv_python27.yaml b/tests/virtualenv/virtualenv_python27.yaml new file mode 100644 index 00000000..09b78480 --- /dev/null +++ b/tests/virtualenv/virtualenv_python27.yaml @@ -0,0 +1,47 @@ +schemaVersion: "1.0.0" + +globalEnvVars: + - key: "VIRTUAL_ENV" + value: "/env" + - key: "PATH" + value: "/env/bin:$PATH" + +commandTests: + - name: "virtualenv27 python installation" + setup: [["virtualenv", "-p", "python", "/env"]] + command: ["which", "python"] + expectedOutput: ["/env/bin/python\n"] + + - name: "virtualenv27 python2 installation" + setup: [["virtualenv", "-p", "python", "/env"]] + command: ["which", "python2"] + expectedOutput: ["/env/bin/python2\n"] + + - name: "virtualenv27python2.7 installation" + setup: [["virtualenv", "-p", "python", "/env"]] + command: ["which", "python2.7"] + expectedOutput: ["/env/bin/python2.7\n"] + + - name: "virtualenv27 python version" + setup: [["virtualenv", "-p", "python", "/env"]] + command: ["python", "--version"] + # we check stderr instead of stdout for Python versions < 3.4 + # https://bugs.python.org/issue18338 + expectedError: ["Python 2.7.(9|12)\n"] + + - name: "virtualenv27 pip installation" + setup: [["virtualenv", "-p", "python", "/env"]] + command: ["which", "pip"] + expectedOutput: ["/env/bin/pip\n"] + + - name: "virtualenv27 gunicorn installation" + setup: [["virtualenv", "-p", "python", "/env"], + ["pip", "install", "gunicorn"]] + command: ["which", "gunicorn"] + expectedOutput: ["/env/bin/gunicorn"] + + - name: "virtualenv27 flask installation" + setup: [["virtualenv", "-p", "python", "/env"], + ["pip", "install", "flask"]] + command: ["python", "-c", "import flask; print(flask.__file__)"] + expectedOutput: ["/env/lib/python2.7/site-packages/flask/__init__.pyc"] diff --git a/tests/virtualenv/virtualenv_python34.yaml b/tests/virtualenv/virtualenv_python34.yaml new file mode 100644 index 00000000..9b5b77d0 --- /dev/null +++ b/tests/virtualenv/virtualenv_python34.yaml @@ -0,0 +1,54 @@ +schemaVersion: "1.0.0" + +globalEnvVars: + - key: "VIRTUAL_ENV" + value: "/env" + - key: "PATH" + value: "/env/bin:$PATH" + +commandTests: + - name: "virtualenv34 python installation" + setup: [["virtualenv", "-p", "python3.4", "/env"]] + command: ["which", "python"] + expectedOutput: ["/env/bin/python\n"] + + - name: "virtualenv34 python3 installation" + setup: [["virtualenv", "-p", "python3.4", "/env"]] + command: ["which", "python3"] + expectedOutput: ["/env/bin/python3\n"] + + - name: "virtualenv34 python3.4 installation" + setup: [["virtualenv", "-p", "python3.4", "/env"]] + command: ["which", "python3.4"] + expectedOutput: ["/env/bin/python3.4\n"] + + - name: "virtualenv34 python version" + setup: [["virtualenv", "-p", "python3.4", "/env"]] + command: ["python", "--version"] + expectedOutput: ["Python 3.4.8\n"] + + - name: "virtualenv34 pip installation" + setup: [["virtualenv", "-p", "python3.4", "/env"]] + command: ["which", "pip"] + expectedOutput: ["/env/bin/pip\n"] + + - name: "virtualenv34 pip3 installation" + setup: [["virtualenv", "-p", "python3.4", "/env"]] + command: ["which", "pip3"] + expectedOutput: ["/env/bin/pip3\n"] + + - name: "virtualenv34 gunicorn installation" + setup: [["virtualenv", "-p", "python3.4", "/env"], + ["pip", "install", "gunicorn"]] + command: ["which", "gunicorn"] + expectedOutput: ["/env/bin/gunicorn"] + + - name: "virtualenv34 flask installation" + setup: [["virtualenv", "-p", "python3.4", "/env"], + ["pip", "install", "flask"]] + command: ["python", "-c", "import flask; print(flask.__file__)"] + expectedOutput: ["/env/lib/python3.4/site-packages/flask/__init__.py"] + + - name: "virtualenv34 test.support availability" + setup: [["virtualenv", "-p", "python3.4", "/env"]] + command: ["python", "-c", "\"from test import pystone, regrtest, support\""] diff --git a/tests/virtualenv/virtualenv_python35.yaml b/tests/virtualenv/virtualenv_python35.yaml new file mode 100644 index 00000000..5e4b394a --- /dev/null +++ b/tests/virtualenv/virtualenv_python35.yaml @@ -0,0 +1,54 @@ +schemaVersion: "1.0.0" + +globalEnvVars: + - key: "VIRTUAL_ENV" + value: "/env" + - key: "PATH" + value: "/env/bin:$PATH" + +commandTests: + - name: "virtualenv35 python installation" + setup: [["virtualenv", "-p", "python3.5", "/env"]] + command: ["which", "python"] + expectedOutput: ["/env/bin/python\n"] + + - name: "virtualenv35 python3 installation" + setup: [["virtualenv", "-p", "python3.5", "/env"]] + command: ["which", "python3"] + expectedOutput: ["/env/bin/python3\n"] + + - name: "virtualenv35 python3.5 installation" + setup: [["virtualenv", "-p", "python3.5", "/env"]] + command: ["which", "python3.5"] + expectedOutput: ["/env/bin/python3.5\n"] + + - name: "virtualenv35 python version" + setup: [["virtualenv", "-p", "python3.5", "/env"]] + command: ["python", "--version"] + expectedOutput: ["Python 3.5.9\n"] + + - name: "virtualenv35 pip installation" + setup: [["virtualenv", "-p", "python3.5", "/env"]] + command: ["which", "pip"] + expectedOutput: ["/env/bin/pip\n"] + + - name: "virtualenv35 pip3 installation" + setup: [["virtualenv", "-p", "python3.5", "/env"]] + command: ["which", "pip3"] + expectedOutput: ["/env/bin/pip3\n"] + + - name: "virtualenv35 gunicorn installation" + setup: [["virtualenv", "-p", "python3.5", "/env"], + ["pip", "install", "gunicorn"]] + command: ["which", "gunicorn"] + expectedOutput: ["/env/bin/gunicorn"] + + - name: "virtualenv35 flask installation" + setup: [["virtualenv", "-p", "python3.5", "/env"], + ["pip", "install", "flask"]] + command: ["python", "-c", "import flask; print(flask.__file__)"] + expectedOutput: ["/env/lib/python3.5/site-packages/flask/__init__.py"] + + - name: "virtualenv35 test.support availability" + setup: [["virtualenv", "-p", "python3.5", "/env"]] + command: ["python", "-c", "\"from test import pystone, regrtest, support\""] diff --git a/tests/virtualenv/virtualenv_python36.yaml b/tests/virtualenv/virtualenv_python36.yaml new file mode 100644 index 00000000..b3a9e68e --- /dev/null +++ b/tests/virtualenv/virtualenv_python36.yaml @@ -0,0 +1,54 @@ +schemaVersion: "1.0.0" + +globalEnvVars: + - key: "VIRTUAL_ENV" + value: "/env" + - key: "PATH" + value: "/env/bin:$PATH" + +commandTests: + - name: "virtualenv36 python installation" + setup: [["virtualenv", "-p", "python3.6", "/env"]] + command: ["which", "python"] + expectedOutput: ["/env/bin/python\n"] + + - name: "virtualenv36 python3 installation" + setup: [["virtualenv", "-p", "python3.6", "/env"]] + command: ["which", "python3"] + expectedOutput: ["/env/bin/python3\n"] + + - name: "virtualenv36 python3.6 installation" + setup: [["virtualenv", "-p", "python3.6", "/env"]] + command: ["which", "python3.6"] + expectedOutput: ["/env/bin/python3.6\n"] + + - name: "virtualenv36 python version" + setup: [["virtualenv", "-p", "python3.6", "/env"]] + command: ["python", "--version"] + expectedOutput: ["Python 3.6.10\n"] + + - name: "virtualenv36 pip installation" + setup: [["virtualenv", "-p", "python3.6", "/env"]] + command: ["which", "pip"] + expectedOutput: ["/env/bin/pip\n"] + + - name: "virtualenv36 pip3 installation" + setup: [["virtualenv", "-p", "python3.6", "/env"]] + command: ["which", "pip3"] + expectedOutput: ["/env/bin/pip3\n"] + + - name: "virtualenv36 gunicorn installation" + setup: [["virtualenv", "-p", "python3.6", "/env"], + ["pip", "install", "gunicorn"]] + command: ["which", "gunicorn"] + expectedOutput: ["/env/bin/gunicorn"] + + - name: "virtualenv36 flask installation" + setup: [["virtualenv", "-p", "python3.6", "/env"], + ["pip", "install", "flask"]] + command: ["python", "-c", "import flask; print(flask.__file__)"] + expectedOutput: ["/env/lib/python3.6/site-packages/flask/__init__.py"] + + - name: "virtualenv36 test.support availability" + setup: [["virtualenv", "-p", "python3.6", "/env"]] + command: ["python", "-c", "\"from test import pystone, regrtest, support\""] diff --git a/tests/virtualenv/virtualenv_python37.yaml b/tests/virtualenv/virtualenv_python37.yaml new file mode 100644 index 00000000..9810c78e --- /dev/null +++ b/tests/virtualenv/virtualenv_python37.yaml @@ -0,0 +1,54 @@ +schemaVersion: "1.0.0" + +globalEnvVars: + - key: "VIRTUAL_ENV" + value: "/env" + - key: "PATH" + value: "/env/bin:$PATH" + +commandTests: + - name: "virtualenv37 python installation" + setup: [["virtualenv", "-p", "python3.7", "/env"]] + command: ["which", "python"] + expectedOutput: ["/env/bin/python\n"] + + - name: "virtualenv37 python3 installation" + setup: [["virtualenv", "-p", "python3.7", "/env"]] + command: ["which", "python3"] + expectedOutput: ["/env/bin/python3\n"] + + - name: "virtualenv37 python3.7 installation" + setup: [["virtualenv", "-p", "python3.7", "/env"]] + command: ["which", "python3.7"] + expectedOutput: ["/env/bin/python3.7\n"] + + - name: "virtualenv37 python version" + setup: [["virtualenv", "-p", "python3.7", "/env"]] + command: ["python", "--version"] + expectedOutput: ["Python 3.7.9\n"] + + - name: "virtualenv37 pip installation" + setup: [["virtualenv", "-p", "python3.7", "/env"]] + command: ["which", "pip"] + expectedOutput: ["/env/bin/pip\n"] + + - name: "virtualenv37 pip3 installation" + setup: [["virtualenv", "-p", "python3.7", "/env"]] + command: ["which", "pip3"] + expectedOutput: ["/env/bin/pip3\n"] + + - name: "virtualenv37 gunicorn installation" + setup: [["virtualenv", "-p", "python3.7", "/env"], + ["pip", "install", "gunicorn"]] + command: ["which", "gunicorn"] + expectedOutput: ["/env/bin/gunicorn"] + + - name: "virtualenv37 flask installation" + setup: [["virtualenv", "-p", "python3.7", "/env"], + ["pip", "install", "flask"]] + command: ["python", "-c", "import flask; print(flask.__file__)"] + expectedOutput: ["/env/lib/python3.7/site-packages/flask/__init__.py"] + + - name: "virtualenv37 test.support availability" + setup: [["virtualenv", "-p", "python3.7", "/env"]] + command: ["python", "-c", "\"from test import pystone, regrtest, support\""]