diff --git a/.gitignore b/.gitignore index 62f3090b..73ec28f0 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,5 @@ .cache .coverage .nox -/cloudbuild.yaml_local.sh -/cloudbuild_benchmark.yaml_local.sh -/cloudbuild_system_tests.yaml_local.sh +/*_local.sh __pycache__ diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 00000000..2b618c5f --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1,4 @@ +# Code owners file. +# This file controls who is tagged for review for any given pull request. + +* @jinglundong @donmccasland diff --git a/README.md b/README.md index 78a02c5c..51e70cd8 100644 --- a/README.md +++ b/README.md @@ -5,10 +5,10 @@ This repository contains the source for the [docker](https://docker.io) base image. This image can be used as the base image for running applications on [Google App Engine Flexible](https://cloud.google.com/appengine/docs/flexible/), -[Google Container Engine](https://cloud.google.com/container-engine), or any +[Google Kubernetes Engine](https://cloud.google.com/kubernetes-engine), or any other Docker host. -This image is based on Debian Jessie and contains packages required to build +This image is based on Ubuntu Xenial and contains packages required to build most of the popular Python libraries. For more information about this runtime, see the [documentation](https://cloud.google.com/appengine/docs/flexible/python/runtime). @@ -38,7 +38,7 @@ to create a custom runtime: You can then modify the `Dockerfile` and `.dockerignore` as needed for you application. -## Container Engine & other Docker hosts. +## Kubernetes Engine & other Docker hosts. For other docker hosts, you'll need to create a `Dockerfile` based on this image that copies your application code, installs dependencies, and declares an @@ -48,6 +48,7 @@ command or entrypoint. For example: # Create a virtualenv for dependencies. This isolates these packages from # system-level packages. + # Use -p python3 or -p python3.7 to select python version. Default is version 2. RUN virtualenv /env # Setting these environment variables are the same as running diff --git a/RELEASING.md b/RELEASING.md index f7d5a459..f886a542 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -36,9 +36,20 @@ follows: b. `gcloud auth activate-service-account` is performed c. `gcloud config set project` is performed 3. The script invokes `build.sh` in this repository -4. `build.sh` invokes Google Container Builder with the `cloudbuild-*.yaml` +4. `build.sh` invokes Google Cloud Build with the `cloudbuild-*.yaml` config files. +## Building interpreters + +The interpreters used are now built in a separate step, and stored on GCS. +This allows the runtime images to be build more rapidly. + +To build the interpreters, run: + +```shell +gcloud builds submit . --config=cloudbuild_interpreters.yaml +``` + ## Building outside Jenkins To build this repository outside Jenkins, authenticate and authorize yourself @@ -54,7 +65,7 @@ Debian or Ubuntu-like). ## Building locally To build this repository using local Docker commands instead of the Google -Container Builder service, add the `--local` flag as shown: +Cloud Build service, add the `--local` flag as shown: ``` shell ./build.sh --local @@ -67,6 +78,15 @@ following: docker run -it --entrypoint /bin/bash YOUR-IMAGE-NAME ``` +## Running tests against a released image + +To run compatibility tests against an existing image, such as +`gcr.io/google-appengine/python:latest`, run: + +```shell +DOCKER_NAMESPACE=gcr.io/google-appengine TAG=latest ./build.sh --nobuild --test +``` + ## Running benchmarks There is a benchmark suite which compares the performance of interpreters diff --git a/build.sh b/build.sh index 7ee1416b..2083237d 100755 --- a/build.sh +++ b/build.sh @@ -18,14 +18,19 @@ set -euo pipefail # Actions benchmark=0 # Should run benchmarks? -build=1 # Should build images? -system_tests=0 # Should run system tests? +build=0 # Should build images? +client_test=0 # Should run Google Cloud Client Library tests +test=0 # Should run standard test suite? local=0 # Should run using local Docker daemon instead of GCR? +os_base=ubuntu18 # Which operating system base to use +interpreter=0 # Should build interpreters instead of images + # Note that $gcloud_cmd has spaces in it -gcloud_cmd="gcloud beta container builds submit ." -local_gcloud_cmd="scripts/local_cloudbuild.py" +gcloud_cmd="gcloud builds submit" +# May need to install via "gcloud components install cloud-build-local" +local_gcloud_cmd="cloud-build-local --push=false --dryrun=false" # Helper functions function fatal() { @@ -38,32 +43,39 @@ function usage { Build and test artifacts in this repository Options: - --benchmark: Run benchmarking suite - --[no]build: Build all images (default) - --local: Build images using local Docker daemon - --system_tests: Run system tests + --[no]benchmark: Run benchmarking suite (default false) + --[no]build: Build all images (default true if no options set) + --[no]test: Run basic tests (default true if no options set) + --[no]client_test: Run Google Cloud Client Library tests (default false) + --[no]local: Build images using local Docker daemon (default false) + --os_base: Which OS image to build on top of [debian8, ubuntu16, ubuntu18] " } - + # Read environment variables -if [ -z "${DOCKER_NAMESPACE+set}" ] ; then +if [ -z "${DOCKER_NAMESPACE:+set}" ] ; then fatal 'Error: $DOCKER_NAMESPACE is not set; invoke with something like DOCKER_NAMESPACE=gcr.io/YOUR-PROJECT-NAME' fi -if [ -z "${BUILDER_DOCKER_NAMESPACE+set}" ] ; then +if [ -z "${BUILDER_DOCKER_NAMESPACE:+set}" ] ; then export BUILDER_DOCKER_NAMESPACE="${DOCKER_NAMESPACE}" fi -if [ -z "${TAG+set}" ] ; then +if [ -z "${TAG:+set}" ] ; then export TAG=`date +%Y-%m-%d-%H%M%S` fi -substitutions="\ +build_substitutions="\ _BUILDER_DOCKER_NAMESPACE=${BUILDER_DOCKER_NAMESPACE},\ _DOCKER_NAMESPACE=${DOCKER_NAMESPACE},\ _TAG=${TAG}\ " +substitutions="\ +_DOCKER_NAMESPACE=${DOCKER_NAMESPACE},\ +_TAG=${TAG}\ +" + # Read command line arguments while [ $# -gt 0 ]; do case "$1" in @@ -71,6 +83,10 @@ while [ $# -gt 0 ]; do benchmark=1 shift ;; + --nobenchmark) + benchmark=0 + shift + ;; --build) build=1 shift @@ -79,12 +95,44 @@ while [ $# -gt 0 ]; do build=0 shift ;; + --client_test) + client_test=1 + shift + ;; + --noclient_test) + client_test=0 + shift + ;; --local) local=1 shift ;; - --system_tests) - system_tests=1 + --nolocal) + local=0 + shift + ;; + --os_base=debian8) + os_base=debian8 + shift + ;; + --os_base=ubuntu16) + os_base=ubuntu16 + shift + ;; + --os_base=ubuntu18) + os_base=ubuntu18 + shift + ;; + --test) + test=1 + shift + ;; + --notest) + test=0 + shift + ;; + --interpreter) + interpreter=1 shift ;; *) @@ -94,8 +142,14 @@ while [ $# -gt 0 ]; do done # If no actions chosen, then tell the user -if [ "${benchmark}" -eq 0 -a "${build}" -eq 0 -a "${system_tests}" -eq 0 ]; then - fatal 'Error: No actions specified (for example, --build), exiting' +if [ "${benchmark}" -eq 0 -a \ + "${build}" -eq 0 -a \ + "${client_test}" -eq 0 -a \ + "${test}" -eq 0 \ +]; then + echo 'No actions specified, defaulting to --build --test' + build=1 + test=1 fi # Running build local or remote? @@ -103,19 +157,17 @@ if [ "${local}" -eq 1 ]; then gcloud_cmd="${local_gcloud_cmd}" fi -# Read action-specific environment variables -if [ "${system_tests}" -eq 1 ]; then - if [ -z "${GOOGLE_APPLICATION_CREDENTIALS_FOR_TESTS+set}" ] ; then - fatal 'Error: $GOOGLE_APPLICATION_CREDENTIALS_FOR_TESTS is not set; invoke with something like GOOGLE_APPLICATION_CREDENTIALS_FOR_TESTS=/path/to/service/account/creds.json' - fi - - if [ -z "${GOOGLE_CLOUD_PROJECT_FOR_TESTS+set}" ] ; then - fatal 'Error: $GOOGLE_CLOUD_PROJECT_FOR_TESTS is not set; invoke with something like GOOGLE_CLOUD_PROJECT_FOR_TESTS=YOUR-PROJECT-NAME' - fi +# Pick OS image to use as base +if [ "${os_base}" == "debian8" ]; then + export OS_BASE_IMAGE="gcr.io/google-appengine/debian8:latest" +elif [ "${os_base}" == "ubuntu16" ]; then + export OS_BASE_IMAGE="gcr.io/gcp-runtimes/ubuntu_16_0_4:latest" +elif [ "${os_base}" == "ubuntu18" ]; then + export OS_BASE_IMAGE="gcr.io/gcp-runtimes/ubuntu_18_0_4:latest" +else + echo "Unsupported OS base image: $OS_BASE_IMAGE" + exit 1 fi - -# Use latest released Debian as our base image -export DEBIAN_BASE_IMAGE="gcr.io/google-appengine/debian8:latest" export STAGING_IMAGE="${DOCKER_NAMESPACE}/python:${TAG}" echo "Using base image name ${STAGING_IMAGE}" @@ -125,11 +177,12 @@ for outfile in \ python-interpreter-builder/Dockerfile \ runtime-image/Dockerfile \ tests/benchmark/Dockerfile \ + tests/eventlet/Dockerfile \ tests/google-cloud-python/Dockerfile \ - tests/google-cloud-python-system/Dockerfile \ tests/integration/Dockerfile \ ; do - envsubst <"${outfile}".in >"${outfile}" '$DEBIAN_BASE_IMAGE $STAGING_IMAGE $GOOGLE_CLOUD_PROJECT_FOR_TESTS' + envsubst <"${outfile}".in >"${outfile}" \ + '$OS_BASE_IMAGE $STAGING_IMAGE $GOOGLE_CLOUD_PROJECT_FOR_TESTS $TAG' done # Make some files available to the runtime builder Docker context @@ -142,33 +195,49 @@ for file in \ cp -a "${file}" "builder/gen-dockerfile/${file##scripts/}" done +# Make a file available to the eventlet test. +cp -a scripts/testdata/hello_world/main.py tests/eventlet/main.py + +# Build interpreters and push to GCS +if [ "${interpreter}" -eq 1 ]; then + echo "Building interpreters" + ${gcloud_cmd} \ + --config=cloudbuild_interpreters.yaml \ + . +fi + # Build images and push to GCR if [ "${build}" -eq 1 ]; then echo "Building images" - ${gcloud_cmd} --config cloudbuild.yaml --substitutions "${substitutions}" + ${gcloud_cmd} \ + --config=cloudbuild.yaml \ + --substitutions="${build_substitutions}" \ + . fi -# If both system tests and benchmarks are requested, run them both -# even if one or the other has errors. If the build step had errors, -# this script will have already exited. -exit_code=0 - -# Run system tests -if [ "${system_tests}" -eq 1 ]; then - echo "Running system tests using project ${GOOGLE_CLOUD_PROJECT_FOR_TESTS}" +# Run the tests that don't require (too many) external services +if [ "${test}" -eq 1 ]; then + echo "Testing compatibility with popular Python libraries" + ${gcloud_cmd} \ + --config=cloudbuild_test.yaml \ + --substitutions="${substitutions}" \ + . +fi - trap "rm -f tests/google-cloud-python-system/credentials.json" EXIT - cp "${GOOGLE_APPLICATION_CREDENTIALS_FOR_TESTS}" tests/google-cloud-python-system/credentials.json - ${gcloud_cmd} --config cloudbuild_system_tests.yaml --substitutions "${substitutions}" || \ - exit_code=1 - rm -f tests/google-cloud-python-system/credentials.json +# Run client library tests +if [ "${client_test}" -eq 1 ]; then + echo "Testing compatibility with Google Cloud Client Libraries" + ${gcloud_cmd} \ + --config=cloudbuild_client_test.yaml \ + --substitutions="${substitutions}" \ + . fi # Run benchmarks if [ "${benchmark}" -eq 1 ] ; then echo "Running benchmark" - ${gcloud_cmd} --config cloudbuild_benchmark.yaml --substitutions "${substitutions}" || \ - exit_code=1 + ${gcloud_cmd} \ + --config=cloudbuild_benchmark.yaml \ + --substitutions="${substitutions}" \ + . fi - -exit ${exit_code} diff --git a/builder/gen-dockerfile/Dockerfile.in b/builder/gen-dockerfile/Dockerfile.in index e2fe4463..92ceaecf 100644 --- a/builder/gen-dockerfile/Dockerfile.in +++ b/builder/gen-dockerfile/Dockerfile.in @@ -1,12 +1,14 @@ FROM ${STAGING_IMAGE} -LABEL python_version=python3.5 -RUN virtualenv --no-download /env -p python3.5 +LABEL python_version=python3.7 +RUN virtualenv --no-download /env -p python3.7 # Set virtualenv environment variables. This is equivalent to running # source /env/bin/activate ENV VIRTUAL_ENV /env ENV PATH /env/bin:$PATH ADD requirements.txt /builder/ +#virtualenv's pip is pegged at version 10.0, removing so +#newer versions get picked up RUN pip install -r /builder/requirements.txt ADD . /builder/ WORKDIR /workspace diff --git a/builder/gen-dockerfile/requirements.txt b/builder/gen-dockerfile/requirements.txt index 59bc593a..4a285555 100644 --- a/builder/gen-dockerfile/requirements.txt +++ b/builder/gen-dockerfile/requirements.txt @@ -1 +1 @@ -PyYAML==3.12 +PyYAML==3.13 diff --git a/builder/python-staging.yaml b/builder/python-staging.yaml new file mode 100644 index 00000000..1e977a89 --- /dev/null +++ b/builder/python-staging.yaml @@ -0,0 +1,12 @@ +# This is a cloudbuild.yaml template for the runtime builder +steps: +- # Generate application Dockerfile + name: 'gcr.io/gcp-runtimes/python/gen-dockerfile:staging' + args: [ + '--base-image=gcr.io/google-appengine/python:staging' + ] +- # Use that Dockerfile to create final application image + name: 'gcr.io/cloud-builders/docker:latest' + args: ['build', '-t', '$_OUTPUT_IMAGE', '.'] +images: + - '$_OUTPUT_IMAGE' diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 187884dc..e240780e 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -1,40 +1,16 @@ -timeout: 7200s +timeout: 10800s steps: -- # Compile Python interpreters from source - name: gcr.io/cloud-builders/docker:latest - args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/interpreter-builder:${_TAG}', - '--no-cache', '/workspace/python-interpreter-builder/'] -- # Copy interpreters back to workspace - name: ${_DOCKER_NAMESPACE}/python/interpreter-builder:${_TAG} - args: ['cp', '/interpreters.tar.gz', '/workspace/runtime-image/interpreters.tar.gz'] - # Build base runtime image name: gcr.io/cloud-builders/docker:latest args: ['build', '--tag=${_DOCKER_NAMESPACE}/python:${_TAG}', '--no-cache', '/workspace/runtime-image/'] -- # Validate structure of base runtime image - name: gcr.io/gcp-runtimes/structure_test:latest - args: [ - '-i', '${_DOCKER_NAMESPACE}/python:${_TAG}', - '--config', '/workspace/tests/virtualenv/virtualenv_default.yaml', - '--config', '/workspace/tests/virtualenv/virtualenv_python34.yaml', - '--config', '/workspace/tests/virtualenv/virtualenv_python35.yaml', - '--config', '/workspace/tests/virtualenv/virtualenv_python36.yaml', - '--config', '/workspace/tests/no-virtualenv/no-virtualenv.yaml', - '--config', '/workspace/tests/python2-libraries/python2-libraries.yaml', - '--config', '/workspace/tests/python3-libraries/python3-libraries.yaml', - '--config', '/workspace/tests/license-test/license-test.yaml', - '-v' - ] -- # Build image to run google client library unit tests - name: gcr.io/cloud-builders/docker:latest - args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/google-cloud-python:${_TAG}', - '--no-cache', '/workspace/tests/google-cloud-python/'] -- # Run google client library unit tests - name: ${_DOCKER_NAMESPACE}/python/tests/google-cloud-python:${_TAG} + id: runtime - # Build runtime builder image name: gcr.io/cloud-builders/docker:latest args: ['build', '--tag=${_BUILDER_DOCKER_NAMESPACE}/python/gen-dockerfile:${_TAG}', '--no-cache', '/workspace/builder/gen-dockerfile/'] + id: gen-dockerfile + waitFor: ['runtime'] images: [ '${_DOCKER_NAMESPACE}/python:${_TAG}', '${_BUILDER_DOCKER_NAMESPACE}/python/gen-dockerfile:${_TAG}', diff --git a/cloudbuild_client_test.yaml b/cloudbuild_client_test.yaml new file mode 100644 index 00000000..010e1ffd --- /dev/null +++ b/cloudbuild_client_test.yaml @@ -0,0 +1,9 @@ +timeout: 3600s +steps: +- # Build image to run google client library unit tests + name: gcr.io/cloud-builders/docker:latest + args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/google-cloud-python:${_TAG}', + '--no-cache', '/workspace/tests/google-cloud-python/'] +- # Run google client library unit tests + name: ${_DOCKER_NAMESPACE}/python/tests/google-cloud-python:${_TAG} +images: [] diff --git a/cloudbuild_interpreters.yaml b/cloudbuild_interpreters.yaml new file mode 100644 index 00000000..c75e59ad --- /dev/null +++ b/cloudbuild_interpreters.yaml @@ -0,0 +1,33 @@ +timeout: 10800s +steps: +- # Compile Python interpreters from source. This step happens first, then + # the next three in parallel. + name: gcr.io/cloud-builders/docker:latest + args: ['build', '--tag=interpreter-builder', + '--no-cache', '/workspace/python-interpreter-builder/'] + id: interpreter-builder +- name: interpreter-builder + args: ['/scripts/build-python-3.4.sh'] + id: build-3.4 + waitFor: ['interpreter-builder'] +- name: interpreter-builder + args: ['/scripts/build-python-3.5.sh'] + id: build-3.5 + waitFor: ['interpreter-builder'] +- name: interpreter-builder + args: ['/scripts/build-python-3.6.sh'] + id: build-3.6 + waitFor: ['interpreter-builder'] +- name: interpreter-builder + args: ['/scripts/build-python-3.7.sh'] + id: build-3.7 + waitFor: ['interpreter-builder'] + +# Upload them to tbe build-id location +- name: gcr.io/cloud-builders/gsutil:latest + args: ['cp', '/workspace/runtime-image/*.tar.gz', 'gs://python-interpreters/$BUILD_ID/'] + waitFor: ['build-3.4', 'build-3.5', 'build-3.6', 'build-3.7'] + +# "Tag" this as latest +- name: gcr.io/cloud-builders/gsutil:latest + args: ['cp', '-r', 'gs://python-interpreters/$BUILD_ID/*', 'gs://python-interpreters/latest/'] diff --git a/cloudbuild_system_tests.yaml b/cloudbuild_system_tests.yaml deleted file mode 100644 index 3cac03e2..00000000 --- a/cloudbuild_system_tests.yaml +++ /dev/null @@ -1,9 +0,0 @@ -timeout: 3600s -steps: -- name: gcr.io/cloud-builders/docker:latest - args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/google-cloud-python-system:${_TAG}', - '--no-cache', '/workspace/tests/google-cloud-python-system/'] -- name: ${_DOCKER_NAMESPACE}/python/tests/google-cloud-python-system:${_TAG} -images: [ - # Intentionally empty -] diff --git a/cloudbuild_test.yaml b/cloudbuild_test.yaml new file mode 100644 index 00000000..ad674026 --- /dev/null +++ b/cloudbuild_test.yaml @@ -0,0 +1,114 @@ +timeout: 3600s +steps: +- # Explicitly pull image into GCB so that later steps work + name: '${_DOCKER_NAMESPACE}/python:${_TAG}' + args: [ + '/bin/true', + ] + id: runtime + +- # Validate structure of base runtime image + name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 + args: [ + '-test.v', + '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', + '/workspace/tests/virtualenv/virtualenv_default.yaml', + ] + waitFor: ['runtime'] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 + args: [ + '-test.v', + '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', + '/workspace/tests/virtualenv/virtualenv_python27.yaml', + ] + waitFor: ['runtime'] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 + args: [ + '-test.v', + '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', + '/workspace/tests/virtualenv/virtualenv_python34.yaml', + ] + waitFor: ['runtime'] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 + args: [ + '-test.v', + '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', + '/workspace/tests/virtualenv/virtualenv_python35.yaml', + ] + waitFor: ['runtime'] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 + args: [ + '-test.v', + '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', + '/workspace/tests/virtualenv/virtualenv_python36.yaml', + ] + waitFor: ['runtime'] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 + args: [ + '-test.v', + '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', + '/workspace/tests/virtualenv/virtualenv_python37.yaml', + ] + waitFor: ['runtime'] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 + args: [ + '-test.v', + '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', + '/workspace/tests/no-virtualenv/no-virtualenv.yaml', + ] + waitFor: ['runtime'] + +# Temporarily disabled because it fails on symbolic links in Ubuntu: +# https://github.com/GoogleCloudPlatform/container-structure-test/issues/77 +#- # Check license compliance +# name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 +# args: [ +# '-test.v', +# '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', +# '/workspace/tests/license-test/license-test.yaml' +# ] +# waitFor: ['runtime'] + +- # Do third-party library compatibility tests for Python 2 + name: gcr.io/cloud-builders/docker:latest + args: [ + 'build', '-t', 'python2-libraries-intermediate', '--build-arg', + 'intermediate_image=${_DOCKER_NAMESPACE}/python:${_TAG}', + '/workspace/tests/python2-libraries' + ] + id: python2-libraries-intermediate + waitFor: ['runtime'] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 + args: [ + '-test.v', + '-image', 'python2-libraries-intermediate', + '/workspace/tests/python2-libraries/python2-libraries.yaml' + ] + waitFor: ['python2-libraries-intermediate'] + +- # Do third-party library compatibility tests for Python 3 + name: gcr.io/cloud-builders/docker:latest + args: [ + 'build', '-t', 'python3-libraries-intermediate', '--build-arg', + 'intermediate_image=${_DOCKER_NAMESPACE}/python:${_TAG}', + '/workspace/tests/python3-libraries' + ] + id: python3-libraries-intermediate + waitFor: ['runtime'] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 + args: [ + '-test.v', + '-image', 'python3-libraries-intermediate', + '/workspace/tests/python3-libraries/python3-libraries.yaml' + ] + waitFor: ['python3-libraries-intermediate'] + +- # Run other compatibility tests + name: gcr.io/cloud-builders/docker:latest + args: [ + 'build', '--tag=${_DOCKER_NAMESPACE}/python/tests/eventlet:${_TAG}', + '--no-cache', '/workspace/tests/eventlet/' + ] + waitFor: ['runtime'] + +images: [] diff --git a/jenkins_build.sh b/jenkins_build.sh deleted file mode 100755 index f88007f5..00000000 --- a/jenkins_build.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -./build.sh "$@" diff --git a/nox.py b/nox.py index 872eb88c..0ee6f443 100644 --- a/nox.py +++ b/nox.py @@ -57,7 +57,7 @@ def lint(session): @nox.session -@nox.parametrize('version', ['3.4', '3.5', '3.6']) +@nox.parametrize('version', ['3.4', '3.5', '3.6', '3.7']) def tests(session, version): session.interpreter = 'python' + version session.install('-r', 'scripts/requirements-test.txt') diff --git a/perf_dashboard/__init__.py b/perf_dashboard/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/perf_dashboard/bq_utils.py b/perf_dashboard/bq_utils.py new file mode 100644 index 00000000..cbad65b9 --- /dev/null +++ b/perf_dashboard/bq_utils.py @@ -0,0 +1,36 @@ +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Common util methods for processing data in BigQuery.""" + +import uuid + +from google.cloud import bigquery + + +def insert_rows(project, dataset_name, table_name, rows): + """Insert rows to bigquery table.""" + client = bigquery.Client(project=project) + dataset_ref = client.dataset(dataset_name) + table_ref = dataset_ref.table(table_name) + table = client.get_table(table_ref) + client.create_rows(table, rows) + +def execute_query(query): + """Execute query and return the query results.""" + client = bigquery.Client() + query_job = client.query((query)) + + # Start the query job and wait it to complete + return [row.values() for row in query_job.result()] diff --git a/perf_dashboard/posts_stats.py b/perf_dashboard/posts_stats.py new file mode 100644 index 00000000..efb9c6fc --- /dev/null +++ b/perf_dashboard/posts_stats.py @@ -0,0 +1,105 @@ +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A script to collect the number of StackOverflow posts related to +Python and Google Cloud Platform.""" + +import datetime +import os +import sys +import time +import uuid + +from collections import Counter + +from google.cloud import bigquery + +import bq_utils + +GCLOUD_PROJECT_ENV = 'GCLOUD_PROJECT' +DATASET_NAME = 'stackoverflow' +TAG_COUNT_TABLE_NAME = 'tag_count_timestamp' +UNANSWERED_POSTS_TABLE_NAME = 'unanswered_posts' + + +def get_stackoverflow_tags_count(): + """Get all the tags contains python and cloud key words""" + query = """ + SELECT + SPLIT(tags, '|') tags + FROM + `bigquery-public-data.stackoverflow.posts_questions` + WHERE + tags LIKE '%python%' + AND (tags LIKE '%google-cloud-platform%' OR tags LIKE '%gcp%') + """ + + results = bq_utils.execute_query(query) + + rows = [row[0] for row in results] + + return rows + + +def get_posts_list_unanswered(): + # Get the list of posts that are unanswered + query = """ + SELECT + id, title, tags + FROM + `bigquery-public-data.stackoverflow.posts_questions` + WHERE + tags LIKE '%python%' + AND (tags LIKE '%google-cloud-platform%' OR tags LIKE '%gcp%') + AND accepted_answer_id is NULL + AND answer_count = 0; + """ + + results = bq_utils.execute_query(query) + + # Add current timestamp to the rows + date_time = datetime.datetime.now() + rows = [(date_time,) + row for row in results] + + return rows + + +def count_unique_tags(data): + flattened_tag_list = [tag for tag_list in data for tag in tag_list] + tag_count = Counter(flattened_tag_list) + + # Add current timestamp to the rows + date_time = datetime.datetime.now() + time_tag_count = [(date_time,) + item for item in tag_count.items()] + + return time_tag_count + + +def main(): + project = os.environ.get(GCLOUD_PROJECT_ENV) + + # Get the posts count for each tag + rows = get_stackoverflow_tags_count() + tag_count = count_unique_tags(rows) + bq_utils.insert_rows( + project, DATASET_NAME, TAG_COUNT_TABLE_NAME, tag_count) + + # Get the list of unanswered posts + unanswered_posts = get_posts_list_unanswered() + bq_utils.insert_rows( + project, DATASET_NAME, UNANSWERED_POSTS_TABLE_NAME, unanswered_posts) + + +if __name__ == '__main__': + main() diff --git a/perf_dashboard/python_clientlibs_download.py b/perf_dashboard/python_clientlibs_download.py new file mode 100644 index 00000000..cae3d2c4 --- /dev/null +++ b/perf_dashboard/python_clientlibs_download.py @@ -0,0 +1,135 @@ +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import os +import sys +import time +import uuid + +from google.cloud import bigquery + +import bq_utils + +GCLOUD_PROJECT_ENV = 'GCLOUD_PROJECT' + +DATETIME_FORMAT = '%Y%m%d' + +DATASET_NAME = 'python_clientlibs_download_by_week' + +VENEER_TABLE_NAME = 'veneer_client_libs' +STACKDRIVER_TABLE_NAME = 'stackdriver_client_libs' +GRPC_TABLE_NAME = 'grpc_lib' +THIRD_PARTY_TABLE_NAME = 'third_party_client_libs' + +TABLES = [ + VENEER_TABLE_NAME, + GRPC_TABLE_NAME, + STACKDRIVER_TABLE_NAME, + THIRD_PARTY_TABLE_NAME, +] + +CLIENTLIBS = { + VENEER_TABLE_NAME: [ + 'google-cloud-core', + 'google-cloud-speech', + 'google-cloud-language', + 'google-cloud-pubsub', + 'google-cloud-bigquery', + 'google-cloud-bigtable', + 'google-cloud-datastore', + 'google-cloud-spanner', + 'google-cloud-storage', + 'google-cloud-vision', + 'google-cloud-translate', + 'google-cloud-dns', + 'google-cloud-videointelligence', + ], + STACKDRIVER_TABLE_NAME: [ + 'google-cloud-logging', + 'google-cloud-monitoring', + 'google-cloud-error_reporting', + 'google-cloud-trace', + ], + GRPC_TABLE_NAME: [ + 'grpcio', + ], + THIRD_PARTY_TABLE_NAME: [ + 'pandas-gbq', + ] +} + + +def get_weekly_clientlibs_downloads(clientlibs_table_name, date_str): + """Use a SQL query to collect the weekly download data of the client + libraries. + + Args: + clientlibs_table_name (str): Table name, which is the key in the + CLIENTLIBS dict. + date_str (str): A date string in "YYYYMMDD" format. + + Returns: + list: rows of the query result. + """ + client_libs = CLIENTLIBS[clientlibs_table_name] + date_time = datetime.datetime.strptime(date_str, DATETIME_FORMAT) + week_dates = [(date_time + datetime.timedelta(days=-i)) + .strftime(DATETIME_FORMAT) + for i in range(7)] + query = """ + SELECT + file.project as client_library_name, + COUNT(*) as download_count + FROM + `the-psf.pypi.downloads*` + WHERE + file.project IN UNNEST(@client_libs) + AND + _TABLE_SUFFIX IN UNNEST(@week_dates) + GROUP BY client_library_name + """ + client = bigquery.Client() + query_parameters=[ + bigquery.ArrayQueryParameter( + 'client_libs', 'STRING', client_libs), + bigquery.ArrayQueryParameter( + 'week_dates', 'STRING', week_dates) + ] + job_config = bigquery.QueryJobConfig() + job_config.query_parameters = query_parameters + query_job = client.query(query, job_config=job_config) + + # Wait for the job to complete and get the results + results = [row.values() for row in query_job.result()] + + rows = [(date_time,) + row for row in results] + + return rows + + +def main(): + for table_name in CLIENTLIBS.keys(): + rows = get_weekly_clientlibs_downloads( + clientlibs_table_name=table_name, + date_str=datetime.datetime.now().strftime("%Y%m%d")) + bq_utils.insert_rows( + project=os.environ.get(GCLOUD_PROJECT_ENV), + dataset_name=DATASET_NAME, + table_name=table_name, + rows=rows) + + +if __name__ == '__main__': + main() diff --git a/python-interpreter-builder/DEBIAN/control.in b/python-interpreter-builder/DEBIAN/control.in new file mode 100644 index 00000000..49980654 --- /dev/null +++ b/python-interpreter-builder/DEBIAN/control.in @@ -0,0 +1,25 @@ +Package: ${DEB_PACKAGE_NAME} +Version: ${DEB_PACKAGE_VERSION} +Section: python +Priority: optional +Architecture: amd64 +Maintainer: Douglas Greiman +Description: Interactive high-level object-oriented language (version ${SHORT_VERSION}) + Python is a high-level, interactive, object-oriented language. Its ${SHORT_VERSION} version + includes an extensive class library with lots of goodies for + network programming, system administration, sounds and graphics. +Depends: libbz2-1.0, + libc6, + libdb5.3, + libexpat1, + libffi6, + liblzma5, + libmpdec2, + libncursesw5, + libreadline6, + libsqlite3-0, + libssl1.0.0, + libtinfo5, + mime-support, + zlib1g +Homepage: https://www.python.org diff --git a/python-interpreter-builder/Dockerfile.in b/python-interpreter-builder/Dockerfile.in index ee9fecf3..9039fbf3 100644 --- a/python-interpreter-builder/Dockerfile.in +++ b/python-interpreter-builder/Dockerfile.in @@ -1,6 +1,6 @@ # The Google App Engine base image is debian (jessie) with ca-certificates # installed. -FROM ${DEBIAN_BASE_IMAGE} +FROM ${OS_BASE_IMAGE} # Install Python build dependencies (based on Debian Build-Depends) RUN apt-get update && apt-get install -yq \ @@ -10,6 +10,7 @@ RUN apt-get update && apt-get install -yq \ debhelper \ dpkg-dev \ gcc \ + gettext-base \ libbluetooth-dev \ libbz2-dev \ libdb-dev \ @@ -28,8 +29,8 @@ RUN apt-get update && apt-get install -yq \ mime-support \ net-tools \ netbase \ + python \ python3 \ - quilt \ sharutils \ time \ tk-dev \ @@ -44,12 +45,4 @@ ENV LANG C.UTF-8 # Add build scripts ADD scripts /scripts -ADD patches /patches - -# Build the Python interpreters -RUN /scripts/build-python-3.5.sh -RUN /scripts/build-python-3.6.sh - -# Tar the interpreters. Tarring is needed because docker cp doesn't handle -# links correctly. -RUN tar czf /interpreters.tar.gz /opt/python?.? +ADD DEBIAN /DEBIAN diff --git a/python-interpreter-builder/patches/3.5/double-build.diff b/python-interpreter-builder/patches/3.5/double-build.diff deleted file mode 100644 index cb52c64b..00000000 --- a/python-interpreter-builder/patches/3.5/double-build.diff +++ /dev/null @@ -1,96 +0,0 @@ -# Source is https://github.com/python/cpython/pull/1478 - -Index: Python-3.5.3/Makefile.pre.in -=================================================================== ---- Python-3.5.3.orig/Makefile.pre.in -+++ Python-3.5.3/Makefile.pre.in -@@ -982,7 +982,7 @@ TESTTIMEOUT= 3600 - - # Run a basic set of regression tests. - # This excludes some tests that are particularly resource-intensive. --test: all platform -+test: @DEF_MAKE_RULE@ platform - $(TESTRUNNER) $(TESTOPTS) - - # Run the full test suite twice - once without .pyc files, and once with. -@@ -992,7 +992,7 @@ test: all platform - # the bytecode read from a .pyc file had the bug, sometimes the directly - # generated bytecode. This is sometimes a very shy bug needing a lot of - # sample data. --testall: all platform -+testall: @DEF_MAKE_RULE@ platform - -find $(srcdir)/Lib -name '*.py[co]' -print | xargs rm -f - $(TESTPYTHON) -E $(srcdir)/Lib/compileall.py - -find $(srcdir)/Lib -name '*.py[co]' -print | xargs rm -f -@@ -1001,7 +1001,7 @@ testall: all platform - - # Run the test suite for both architectures in a Universal build on OSX. - # Must be run on an Intel box. --testuniversal: all platform -+testuniversal: @DEF_MAKE_RULE@ platform - if [ `arch` != 'i386' ];then \ - echo "This can only be used on OSX/i386" ;\ - exit 1 ;\ -@@ -1024,7 +1024,7 @@ QUICKTESTOPTS= $(TESTOPTS) -x test_subpr - test_multiprocessing_forkserver \ - test_mailbox test_socket test_poll \ - test_select test_zipfile test_concurrent_futures --quicktest: all platform -+quicktest: @DEF_MAKE_RULE@ platform - $(TESTRUNNER) $(QUICKTESTOPTS) - - -@@ -1376,7 +1376,7 @@ LIBPL= @LIBPL@ - # pkgconfig directory - LIBPC= $(LIBDIR)/pkgconfig - --libainstall: all python-config -+libainstall: @DEF_MAKE_RULE@ python-config - @for i in $(LIBDIR) $(LIBPL) $(LIBPC); \ - do \ - if test ! -d $(DESTDIR)$$i; then \ -@@ -1635,7 +1635,7 @@ distclean: clobber - -exec rm -f {} ';' - - # Check for smelly exported symbols (not starting with Py/_Py) --smelly: all -+smelly: @DEF_MAKE_RULE@ - nm -p $(LIBRARY) | \ - sed -n "/ [TDB] /s/.* //p" | grep -v "^_*Py" | sort -u; \ - -@@ -1673,7 +1673,7 @@ funny: - -o -print - - # Perform some verification checks on any modified files. --patchcheck: all -+patchcheck: @DEF_MAKE_RULE@ - $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/scripts/patchcheck.py - - # Dependencies -Index: Python-3.5.3/Misc/ACKS -=================================================================== ---- Python-3.5.3.orig/Misc/ACKS -+++ Python-3.5.3/Misc/ACKS -@@ -1092,6 +1092,7 @@ Jason Orendorff - Douglas Orr - William Orr - Michele Orrù -+Tomáš Orsava - Oleg Oshmyan - Denis S. Otkidach - Peter Otten -Index: Python-3.5.3/Misc/NEWS -=================================================================== ---- Python-3.5.3.orig/Misc/NEWS -+++ Python-3.5.3/Misc/NEWS -@@ -634,6 +634,10 @@ Windows - Build - ----- - -+- bpo-29243: Prevent unnecessary rebuilding of Python during ``make test``, -+ ``make install`` and some other make targets when configured with -+ ``--enable-optimizations``. -+ - - Issue #29080: Removes hard dependency on hg.exe from PCBuild/build.bat - - - Issue #23903: Added missed names to PC/python3.def. diff --git a/python-interpreter-builder/patches/3.5/series b/python-interpreter-builder/patches/3.5/series deleted file mode 100644 index f6c2876b..00000000 --- a/python-interpreter-builder/patches/3.5/series +++ /dev/null @@ -1 +0,0 @@ -double-build.diff diff --git a/python-interpreter-builder/patches/3.6/double-build.diff b/python-interpreter-builder/patches/3.6/double-build.diff deleted file mode 100644 index a89a20f6..00000000 --- a/python-interpreter-builder/patches/3.6/double-build.diff +++ /dev/null @@ -1,96 +0,0 @@ -# Source is https://github.com/python/cpython/pull/1478 - -Index: Python-3.6.1/Makefile.pre.in -=================================================================== ---- Python-3.6.1.orig/Makefile.pre.in -+++ Python-3.6.1/Makefile.pre.in -@@ -1000,7 +1000,7 @@ TESTTIMEOUT= 1200 - - # Run a basic set of regression tests. - # This excludes some tests that are particularly resource-intensive. --test: all platform -+test: @DEF_MAKE_RULE@ platform - $(TESTRUNNER) $(TESTOPTS) - - # Run the full test suite twice - once without .pyc files, and once with. -@@ -1010,7 +1010,7 @@ test: all platform - # the bytecode read from a .pyc file had the bug, sometimes the directly - # generated bytecode. This is sometimes a very shy bug needing a lot of - # sample data. --testall: all platform -+testall: @DEF_MAKE_RULE@ platform - -find $(srcdir)/Lib -name '*.py[co]' -print | xargs rm -f - $(TESTPYTHON) -E $(srcdir)/Lib/compileall.py - -find $(srcdir)/Lib -name '*.py[co]' -print | xargs rm -f -@@ -1019,7 +1019,7 @@ testall: all platform - - # Run the test suite for both architectures in a Universal build on OSX. - # Must be run on an Intel box. --testuniversal: all platform -+testuniversal: @DEF_MAKE_RULE@ platform - if [ `arch` != 'i386' ];then \ - echo "This can only be used on OSX/i386" ;\ - exit 1 ;\ -@@ -1042,7 +1042,7 @@ QUICKTESTOPTS= $(TESTOPTS) -x test_subpr - test_multiprocessing_forkserver \ - test_mailbox test_socket test_poll \ - test_select test_zipfile test_concurrent_futures --quicktest: all platform -+quicktest: @DEF_MAKE_RULE@ platform - $(TESTRUNNER) $(QUICKTESTOPTS) - - -@@ -1379,7 +1379,7 @@ LIBPL= @LIBPL@ - # pkgconfig directory - LIBPC= $(LIBDIR)/pkgconfig - --libainstall: all python-config -+libainstall: @DEF_MAKE_RULE@ python-config - @for i in $(LIBDIR) $(LIBPL) $(LIBPC); \ - do \ - if test ! -d $(DESTDIR)$$i; then \ -@@ -1639,7 +1639,7 @@ distclean: clobber - -exec rm -f {} ';' - - # Check for smelly exported symbols (not starting with Py/_Py) --smelly: all -+smelly: @DEF_MAKE_RULE@ - nm -p $(LIBRARY) | \ - sed -n "/ [TDB] /s/.* //p" | grep -v "^_*Py" | sort -u; \ - -@@ -1676,7 +1676,7 @@ funny: - -o -print - - # Perform some verification checks on any modified files. --patchcheck: all -+patchcheck: @DEF_MAKE_RULE@ - $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/scripts/patchcheck.py - - # Dependencies -Index: Python-3.6.1/Misc/ACKS -=================================================================== ---- Python-3.6.1.orig/Misc/ACKS -+++ Python-3.6.1/Misc/ACKS -@@ -1111,6 +1111,7 @@ Jason Orendorff - Douglas Orr - William Orr - Michele Orrù -+Tomáš Orsava - Oleg Oshmyan - Denis S. Otkidach - Peter Otten -Index: Python-3.6.1/Misc/NEWS -=================================================================== ---- Python-3.6.1.orig/Misc/NEWS -+++ Python-3.6.1/Misc/NEWS -@@ -306,6 +306,10 @@ Tests - Build - ----- - -+- bpo-29243: Prevent unnecessary rebuilding of Python during ``make test``, -+ ``make install`` and some other make targets when configured with -+ ``--enable-optimizations``. -+ - - bpo-27593: sys.version and the platform module python_build(), - python_branch(), and python_revision() functions now use - git information rather than hg when building from a repo. diff --git a/python-interpreter-builder/patches/3.6/series b/python-interpreter-builder/patches/3.6/series deleted file mode 100644 index f6c2876b..00000000 --- a/python-interpreter-builder/patches/3.6/series +++ /dev/null @@ -1 +0,0 @@ -double-build.diff diff --git a/python-interpreter-builder/scripts/build-python-3.4.sh b/python-interpreter-builder/scripts/build-python-3.4.sh new file mode 100755 index 00000000..5c0bfb8e --- /dev/null +++ b/python-interpreter-builder/scripts/build-python-3.4.sh @@ -0,0 +1,141 @@ +#!/bin/bash + +set -euo pipefail +set -x + +# Get the source +mkdir -p /opt/sources +cd /opt/sources +wget --no-verbose https://www.python.org/ftp/python/3.4.8/Python-3.4.8.tgz +# SHA-256 generated via `shasum -a 256 [file]` +shasum --check <&2 + exit 1 +} + # Process command line +if [ -z "${1:+set}" -o -z "${2:+set}" ]; then + usage +fi +LONG_VERSION=$1 +BUILD_TAG=$2 +SHORT_VERSION=${1%.*} + +# Compute version specs +DEB_PACKAGE_NAME=gcp-python${SHORT_VERSION} +# Can't have - (hyphen) in debian revision as per +# https://www.debian.org/doc/debian-policy/ch-controlfields.html#s-f-Version +DEBIAN_REVISION=${BUILD_TAG//-/.} +DEB_PACKAGE_VERSION=${LONG_VERSION}-${DEBIAN_REVISION} + +PACKAGE_DIR=/opt/packages +# E.g. gcp-python3.6_3.6.2-1gcp~2017.07.25.110644_amd64.deb +DEB_FILENAME=${DEB_PACKAGE_NAME}_${DEB_PACKAGE_VERSION}_amd64.deb + +# Create directory for intermediate files +SCRATCH_DIR=$(mktemp --directory) +cd "${SCRATCH_DIR}" + +# Synthesize Debian control file. Note that the "Depends:" is +# currently Debian8-specific, and lacks version specifiers present in +# the standard Debian Python packages. +export DEB_PACKAGE_NAME DEB_PACKAGE_VERSION SHORT_VERSION +envsubst control \ + '${DEB_PACKAGE_NAME} ${DEB_PACKAGE_VERSION} ${SHORT_VERSION}' + +# Generate components of .deb archive +tar czf control.tar.gz control +tar czf data.tar.gz "/opt/python${SHORT_VERSION}" +echo "2.0" >debian-binary + +# Generate final .deb. +mkdir -p "${PACKAGE_DIR}" +ar rcD "${PACKAGE_DIR}/${DEB_FILENAME}" \ + debian-binary control.tar.gz data.tar.gz +rm debian-binary control.tar.gz data.tar.gz + +# Validate .deb +dpkg --install --dry-run "${PACKAGE_DIR}/${DEB_FILENAME}" + +# Add to list +echo "${DEB_FILENAME}" >> "${PACKAGE_DIR}/packages.txt" diff --git a/runtime-image/Dockerfile.in b/runtime-image/Dockerfile.in index ed194257..46387705 100644 --- a/runtime-image/Dockerfile.in +++ b/runtime-image/Dockerfile.in @@ -1,7 +1,7 @@ # The Google App Engine base image is debian (jessie) with ca-certificates # installed. # Source: https://github.com/GoogleCloudPlatform/debian-docker -FROM ${DEBIAN_BASE_IMAGE} +FROM ${OS_BASE_IMAGE} ADD resources /resources ADD scripts /scripts @@ -9,6 +9,7 @@ ADD scripts /scripts # Install Python, pip, and C dev libraries necessary to compile the most popular # Python libraries. RUN /scripts/install-apt-packages.sh +RUN curl "https://bootstrap.pypa.io/pip/2.7/get-pip.py" -o "get-pip.py" && python ./get-pip.py && ln -s /usr/local/bin/pip /usr/bin/pip # Setup locale. This prevents Python 3 IO encoding issues. ENV LANG C.UTF-8 @@ -16,15 +17,33 @@ ENV LANG C.UTF-8 # logging collection. ENV PYTHONUNBUFFERED 1 -# Upgrade pip (debian package version tends to run a few version behind) and -# install virtualenv system-wide. -RUN pip install --upgrade pip virtualenv - -# Install the Google-built interpreters -ADD interpreters.tar.gz / +RUN wget https://storage.googleapis.com/python-interpreters/latest/interpreter-3.4.tar.gz && \ + wget https://storage.googleapis.com/python-interpreters/latest/interpreter-3.5.tar.gz && \ + wget https://storage.googleapis.com/python-interpreters/latest/interpreter-3.6.tar.gz && \ + wget https://storage.googleapis.com/python-interpreters/latest/interpreter-3.7.tar.gz && \ + tar -xzf interpreter-3.4.tar.gz && \ + tar -xzf interpreter-3.5.tar.gz && \ + tar -xzf interpreter-3.6.tar.gz && \ + tar -xzf interpreter-3.7.tar.gz && \ + rm interpreter-*.tar.gz # Add Google-built interpreters to the path -ENV PATH /opt/python3.5/bin:/opt/python3.6/bin:$PATH +ENV PATH /opt/python3.7/bin:/opt/python3.6/bin:/opt/python3.5/bin:/opt/python3.4/bin:$PATH +RUN update-alternatives --install /usr/local/bin/python3 python3 /opt/python3.7/bin/python3.7 50 && \ + update-alternatives --install /usr/local/bin/pip3 pip3 /opt/python3.7/bin/pip3.7 50 + +# Upgrade pip (debian package version tends to run a few version behind) and +# install virtualenv system-wide. +RUN /usr/bin/pip install --upgrade -r /resources/requirements.txt && \ + /opt/python3.4/bin/pip3.4 install --upgrade -r /resources/requirements.txt && \ + rm -f /opt/python3.4/bin/pip /opt/python3.4/bin/pip3 && \ + /opt/python3.5/bin/pip3.5 install --upgrade -r /resources/requirements.txt && \ + rm -f /opt/python3.5/bin/pip /opt/python3.5/bin/pip3 && \ + /opt/python3.6/bin/pip3.6 install --upgrade -r /resources/requirements.txt && \ + rm -f /opt/python3.6/bin/pip /opt/python3.6/bin/pip3 && \ + /opt/python3.7/bin/pip3.7 install --upgrade -r /resources/requirements.txt && \ + rm -f /opt/python3.7/bin/pip /opt/python3.7/bin/pip3 && \ + /usr/bin/pip install --upgrade -r /resources/requirements-virtualenv.txt # Setup the app working directory RUN ln -s /home/vmagent/app /app diff --git a/runtime-image/resources/apt-packages.txt b/runtime-image/resources/apt-packages.txt index 4bb6d6fc..7b88f777 100644 --- a/runtime-image/resources/apt-packages.txt +++ b/runtime-image/resources/apt-packages.txt @@ -4,11 +4,8 @@ mercurial pkg-config wget # debian-provided interpreters -python-pip python2.7 python2.7-dev -python3.4 -python3.4-dev # Dependenies for third-party Python packages # with C-extensions build-essential @@ -36,3 +33,5 @@ libsasl2-2 libsasl2-dev libsasl2-modules sasl2-bin +# Needed by eventlet +netbase diff --git a/runtime-image/resources/requirements-virtualenv.txt b/runtime-image/resources/requirements-virtualenv.txt new file mode 100644 index 00000000..25f09c4a --- /dev/null +++ b/runtime-image/resources/requirements-virtualenv.txt @@ -0,0 +1 @@ +virtualenv==20.0.31 diff --git a/runtime-image/resources/requirements.txt b/runtime-image/resources/requirements.txt new file mode 100644 index 00000000..2d010eef --- /dev/null +++ b/runtime-image/resources/requirements.txt @@ -0,0 +1,3 @@ +pip +setuptools==40.2.0 +wheel==0.31.1 diff --git a/runtime-image/scripts/install-apt-packages.sh b/runtime-image/scripts/install-apt-packages.sh index 9d23cab7..bafba2d6 100755 --- a/runtime-image/scripts/install-apt-packages.sh +++ b/runtime-image/scripts/install-apt-packages.sh @@ -7,6 +7,8 @@ apt-get -q update xargs -a <(awk '/^\s*[^#]/' '/resources/apt-packages.txt') -r -- \ apt-get install --no-install-recommends -yq +apt-get upgrade -yq + # Remove unneeded files. apt-get clean rm /var/lib/apt/lists/*_* diff --git a/scripts/data/Dockerfile.python_compat b/scripts/data/Dockerfile.python_compat new file mode 100644 index 00000000..1e4d6352 --- /dev/null +++ b/scripts/data/Dockerfile.python_compat @@ -0,0 +1,3 @@ +FROM gcr.io/google_appengine/python-compat-multicore +ADD . /app/ +RUN if [ -s requirements.txt ]; then pip install -r requirements.txt; fi diff --git a/scripts/data/dockerignore.python_compat b/scripts/data/dockerignore.python_compat new file mode 100644 index 00000000..5ce5abfa --- /dev/null +++ b/scripts/data/dockerignore.python_compat @@ -0,0 +1,5 @@ +.dockerignore +Dockerfile +.git +.hg +.svn diff --git a/scripts/deploy_check.sh b/scripts/deploy_check.sh new file mode 100644 index 00000000..1e2f02c2 --- /dev/null +++ b/scripts/deploy_check.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +set -ex + +export KOKORO_GITHUB_DIR=${KOKORO_ROOT}/src/github +source ${KOKORO_GFILE_DIR}/kokoro/common.sh + +cd ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY} +if [ -n "${RUNTIME_SPEC}" -a -f app.yaml.in ]; then + sed "s|\${RUNTIME_SPEC}|${RUNTIME_SPEC}|" app.yaml.in > app.yaml +fi + +cd ${KOKORO_GFILE_DIR}/appengine/integration_tests + +sudo -E /usr/local/bin/pip install --upgrade -r requirements.txt + +if [ -f ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY}/requirements.txt ] +then + sudo -E /usr/local/bin/pip install --upgrade -r ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY}/requirements.txt +fi + +export DEPLOY_LATENCY_PROJECT='cloud-deploy-latency' + +skip_flag="" + +if [ "${SKIP_CUSTOM_LOGGING_TESTS}" = "true" -o "${SKIP_BUILDERS}" = "true" ]; then + skip_flag="$skip_flag --skip-builders" +fi + +if [ "${SKIP_XRT}" = "true" ]; then + skip_flag="$skip_flag --skip-xrt" +fi + +python deploy_check.py -d ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY} -l ${LANGUAGE} ${skip_flag} diff --git a/scripts/gen_dockerfile.py b/scripts/gen_dockerfile.py index 9bfbca2a..97da60f6 100755 --- a/scripts/gen_dockerfile.py +++ b/scripts/gen_dockerfile.py @@ -29,7 +29,6 @@ import validation_utils - # Validate characters for dockerfile image names. # # This roots out obvious mistakes, the full gory details are here: @@ -51,17 +50,20 @@ PYTHON_INTERPRETER_VERSION_MAP = { '': '', # == 2.7 '2': '', # == 2.7 - '3': '3.5', + '3': '3.6', '3.4': '3.4', '3.5': '3.5', '3.6': '3.6', + '3.7': '3.7', } +# Name of environment variable potentially set by gcloud +GAE_APPLICATION_YAML_PATH = 'GAE_APPLICATION_YAML_PATH' # Validated application configuration AppConfig = collections.namedtuple( 'AppConfig', - 'base_image dockerfile_python_version entrypoint has_requirements_txt' + 'base_image dockerfile_python_version entrypoint has_requirements_txt is_python_compat' ) @@ -95,6 +97,16 @@ def get_app_config(raw_config, base_image, config_file, source_dir): 'Expected {} contents to be a Mapping type, but found type "{}"'. format(config_file, type(raw_config))) + # Short circuit for python compat. + if validation_utils.get_field_value( + raw_config, 'runtime', str) == 'python-compat': + return AppConfig( + base_image=None, + dockerfile_python_version=None, + entrypoint=None, + has_requirements_txt=None, + is_python_compat=True) + entrypoint = validation_utils.get_field_value( raw_config, 'entrypoint', str) if not PRINTABLE_REGEX.match(entrypoint): @@ -131,7 +143,8 @@ def get_app_config(raw_config, base_image, config_file, source_dir): base_image=base_image, dockerfile_python_version=dockerfile_python_version, entrypoint=entrypoint, - has_requirements_txt=has_requirements_txt) + has_requirements_txt=has_requirements_txt, + is_python_compat=False) def get_data(name): @@ -173,19 +186,24 @@ def generate_files(app_config): else: optional_entrypoint = '' - dockerfile = ''.join([ - get_data('Dockerfile.preamble.template').format( - base_image=app_config.base_image), - get_data('Dockerfile.virtualenv.template').format( - python_version=app_config.dockerfile_python_version), - optional_requirements_txt, - get_data('Dockerfile.install_app'), - optional_entrypoint, - ]) + if app_config.is_python_compat: + dockerfile = get_data('Dockerfile.python_compat') + dockerignore = get_data('dockerignore.python_compat') + else: + dockerfile = ''.join([ + get_data('Dockerfile.preamble.template').format( + base_image=app_config.base_image), + get_data('Dockerfile.virtualenv.template').format( + python_version=app_config.dockerfile_python_version), + optional_requirements_txt, + get_data('Dockerfile.install_app'), + optional_entrypoint, + ]) + dockerignore = get_data('dockerignore') return { 'Dockerfile': dockerfile, - '.dockerignore': get_data('dockerignore'), + '.dockerignore': dockerignore, } @@ -204,7 +222,7 @@ def generate_dockerfile_command(base_image, config_file, source_dir): # Determine complete configuration app_config = get_app_config(raw_config, base_image, config_file, - source_dir) + source_dir) # Generate list of filenames and their textual contents files = generate_files(app_config) @@ -225,11 +243,19 @@ def parse_args(argv): validation_utils.validate_arg_regex, flag_regex=IMAGE_REGEX), default='gcr.io/google-appengine/python:latest', help='Name of Docker image to use as base') + # In some cases, gcloud sets an environment variable to indicate + # the location of the application configuration file, rather than + # using the --config flag. The order of precedence from highest + # to lowest is: + # + # 1) --config flag + # 2) $GAE_APPLICATION_YAML_PATH environment variable + # 3) a file named "app.yaml" in the current working directory parser.add_argument( '--config', type=functools.partial( validation_utils.validate_arg_regex, flag_regex=PRINTABLE_REGEX), - default='app.yaml', + default=(os.environ.get(GAE_APPLICATION_YAML_PATH) or 'app.yaml'), help='Path to application configuration file' ) parser.add_argument( diff --git a/scripts/gen_dockerfile_test.py b/scripts/gen_dockerfile_test.py index 6a7bd33a..03eaa079 100755 --- a/scripts/gen_dockerfile_test.py +++ b/scripts/gen_dockerfile_test.py @@ -54,6 +54,14 @@ def compare_file(filename, dir1, dir2): 'dockerfile_python_version': '', 'has_requirements_txt': False, 'entrypoint': '', + 'is_python_compat': False, + }), + ('env: flex\nruntime: python-compat', { + 'base_image': None, + 'dockerfile_python_version': None, + 'has_requirements_txt': None, + 'entrypoint': None, + 'is_python_compat': True, }), # All supported python versions ('runtime_config:\n python_version:', { @@ -63,7 +71,7 @@ def compare_file(filename, dir1, dir2): 'dockerfile_python_version': '', }), ('runtime_config:\n python_version: 3', { - 'dockerfile_python_version': '3.5', + 'dockerfile_python_version': '3.6', }), ('runtime_config:\n python_version: 3.4', { 'dockerfile_python_version': '3.4', @@ -71,6 +79,12 @@ def compare_file(filename, dir1, dir2): ('runtime_config:\n python_version: 3.5', { 'dockerfile_python_version': '3.5', }), + ('runtime_config:\n python_version: 3.6', { + 'dockerfile_python_version': '3.6', + }), + ('runtime_config:\n python_version: 3.7', { + 'dockerfile_python_version': '3.7', + }), # entrypoint present ('entrypoint: my entrypoint', { 'entrypoint': 'exec my entrypoint', @@ -122,7 +136,8 @@ def test_get_app_config_invalid(app_yaml): base_image='', dockerfile_python_version='', entrypoint='', - has_requirements_txt=False + has_requirements_txt=False, + is_python_compat=False, ) @@ -143,6 +158,9 @@ def test_get_app_config_invalid(app_yaml): # Python version (_BASE_APP_CONFIG._replace(dockerfile_python_version='_my_version'), True, 'python_version=python_my_version'), + # python-compat runtime + (_BASE_APP_CONFIG._replace(is_python_compat=True), True, + 'FROM gcr.io/google_appengine/python-compat-multicore'), ]) def test_generate_files(app_config, should_find, test_string): result = gen_dockerfile.generate_files(app_config) @@ -160,10 +178,13 @@ def compare_against_golden_files(app, config_dir, testdata_dir): compare_file(filename, config_dir, golden_dir) -def test_generate_dockerfile_command(tmpdir, testdata_dir): +@pytest.mark.parametrize('app', [ + # Sampled from https://github.com/GoogleCloudPlatform/python-docs-samples + 'hello_world', + # From an internal source. + 'hello_world_compat']) +def test_generate_dockerfile_command(tmpdir, testdata_dir, app): """Generates output and compares against a set of golden files.""" - # Sample from https://github.com/GoogleCloudPlatform/python-docs-samples - app = 'hello_world' app_dir = os.path.join(testdata_dir, app) # Copy sample app to writable temp dir, and generate Dockerfile. @@ -176,12 +197,15 @@ def test_generate_dockerfile_command(tmpdir, testdata_dir): compare_against_golden_files(app, config_dir, testdata_dir) +@pytest.mark.parametrize('app', [ + # Sampled from https://github.com/GoogleCloudPlatform/python-docs-samples + 'hello_world', + # From an internal source. + 'hello_world_compat']) @pytest.mark.xfail(not shutil.which('gcloud'), reason='Google Cloud SDK is not installed') -def test_generate_dockerfile_golden(tmpdir, testdata_dir): +def test_generate_dockerfile_golden(tmpdir, testdata_dir, app): """Validate our golden files against gcloud app gen-config""" - # Sample from https://github.com/GoogleCloudPlatform/python-docs-samples - app = 'hello_world' app_dir = os.path.join(testdata_dir, app) # Copy sample app to writable temp dir, and generate Dockerfile. @@ -225,5 +249,26 @@ def mock_error(*args): gen_dockerfile.parse_args(argv) +@pytest.mark.parametrize('argv, env, expected', [ + # Explicit flag wins + (['argv0', '--config=flag/path'], 'env/path', 'flag/path'), + (['argv0', '--config=flag/path'], '', 'flag/path'), + (['argv0', '--config=flag/path'], None, 'flag/path'), + # Otherwise env var wins + (['argv0'], 'env/path', 'env/path'), + # Otherwise use default name + (['argv0'], '', 'app.yaml'), + (['argv0'], None, 'app.yaml'), +]) +def test_parse_args_config(argv, env, expected): + if env is None: + mock_environ = {} + else: + mock_environ = {gen_dockerfile.GAE_APPLICATION_YAML_PATH: env} + with unittest.mock.patch.dict('os.environ', mock_environ, clear=True): + args = gen_dockerfile.parse_args(argv) + assert args.config == expected + + if __name__ == '__main__': pytest.main([__file__]) diff --git a/scripts/integration-test.sh b/scripts/integration-test.sh new file mode 100644 index 00000000..6210e0a1 --- /dev/null +++ b/scripts/integration-test.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +set -ex + +export KOKORO_GITHUB_DIR=${KOKORO_ROOT}/src/github +source ${KOKORO_GFILE_DIR}/kokoro/common.sh + +export GOOGLE_CLOUD_PROJECT=gcp-runtimes + +sudo -E /usr/local/bin/pip install --upgrade -r ${KOKORO_GFILE_DIR}/appengine/integration_tests/requirements.txt + +if [ -f ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY}/requirements.txt ] +then + sudo -E /usr/local/bin/pip install --upgrade -r ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY}/requirements.txt +fi + +export GOPATH=${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY} + +flags="" + +if [ -n "${STAGING_IMAGE}" ]; then + flags="$flags -i ${STAGING_IMAGE}" +fi + +if [ "${SKIP_STANDARD_LOGGING_TESTS}" = "true" ]; then + flags="$flags --skip-standard-logging-tests" +fi + +if [ "${SKIP_CUSTOM_LOGGING_TESTS}" = "true" ]; then + flags="$flags --skip-custom-logging-tests" +fi + +if [ "${SKIP_MONITORING_TESTS}" = "true" ]; then + flags="$flags --skip-monitoring-tests" +fi + +if [ "${SKIP_EXCEPTION_TESTS}" = "true" ]; then + flags="$flags --skip-exception-tests" +fi + +if [ "${SKIP_CUSTOM_TESTS}" = "true" ]; then + flags="$flags --skip-custom-tests" +fi + +if [ -n "${URL}" ]; then + flags="$flags --url ${URL}" +fi + +if [ -n "${BUILDER}" ]; then + flags="$flags --builder ${BUILDER}" + gcloud config set app/use_runtime_builders True + gcloud config set app/runtime_builders_root file://${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY} +fi + +if [ -n "${YAML}" ]; then + flags="$flags --yaml ${KOKORO_GITHUB_DIR}/${YAML}" +fi + + +chmod a+x ${KOKORO_GFILE_DIR}/appengine/integration_tests/testsuite/driver.py +${KOKORO_GFILE_DIR}/appengine/integration_tests/testsuite/driver.py -d ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY} ${flags} diff --git a/scripts/local_cloudbuild.py b/scripts/local_cloudbuild.py index f80e0f4b..5c23a1bc 100755 --- a/scripts/local_cloudbuild.py +++ b/scripts/local_cloudbuild.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Emulate the Google Container Builder locally. +"""Emulate the Google Cloud Build locally. The input is a local cloudbuild.yaml file. This is translated into a series of commands for the locally installed Docker daemon. These @@ -48,8 +48,8 @@ # Exclude non-printable control characters (including newlines) PRINTABLE_REGEX = re.compile(r"""^[^\x00-\x1f]*$""") -# Container Builder substitutions -# https://cloud.google.com/container-builder/docs/api/build-requests#substitutions +# Cloud Build substitutions +# https://cloud.google.com/cloud-build/docs/api/build-requests#substitutions SUBSTITUTION_REGEX = re.compile(r"""(?x) [$] # Dollar sign ( diff --git a/scripts/release.sh b/scripts/release.sh new file mode 100644 index 00000000..cd499823 --- /dev/null +++ b/scripts/release.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set -euo pipefail +export KOKORO_GITHUB_DIR=${KOKORO_ROOT}/src/github +source ${KOKORO_GFILE_DIR}/kokoro/common.sh + +cd ${KOKORO_GITHUB_DIR}/python-runtime + +if [ -z "${TAG:+set}" ]; then + export TAG=$(date +%Y-%m-%d-%H%M%S) +fi + +./build.sh $BUILD_FLAGS + +METADATA=$(pwd)/METADATA +cd ${KOKORO_GFILE_DIR}/kokoro +python note.py python -m ${METADATA} -t ${TAG} diff --git a/scripts/requirements-test.txt b/scripts/requirements-test.txt index 77b8c636..c8e698da 100644 --- a/scripts/requirements-test.txt +++ b/scripts/requirements-test.txt @@ -1,4 +1,4 @@ -flask -pytest -pytest-cov -pyyaml +flask==1.0.2 +pytest==3.7.3 +pytest-cov==2.5.1 +pyyaml==3.13 diff --git a/scripts/testdata/hello_world/requirements.txt b/scripts/testdata/hello_world/requirements.txt index 7861fb49..a34d076b 100644 --- a/scripts/testdata/hello_world/requirements.txt +++ b/scripts/testdata/hello_world/requirements.txt @@ -1,2 +1,2 @@ -Flask==0.12 -gunicorn==19.6.0 +Flask==1.0.2 +gunicorn==19.9.0 diff --git a/scripts/testdata/hello_world_compat/app.yaml b/scripts/testdata/hello_world_compat/app.yaml new file mode 100644 index 00000000..e514d42c --- /dev/null +++ b/scripts/testdata/hello_world_compat/app.yaml @@ -0,0 +1,13 @@ +service: default +runtime: python-compat +env: flex + +api_version: 1 +threadsafe: true + +beta_settings: + enable_app_engine_apis: true # Needed for compat apps. + +handlers: +- url: .* + script: main.app diff --git a/scripts/testdata/hello_world_compat/main.py b/scripts/testdata/hello_world_compat/main.py new file mode 100644 index 00000000..40302722 --- /dev/null +++ b/scripts/testdata/hello_world_compat/main.py @@ -0,0 +1,14 @@ +"""The hello world flex app!""" + +import webapp2 + + +class HelloHandler(webapp2.RequestHandler): + + def get(self): + msg = 'Hello GAE Flex (env: flex) Compat-Runtime App\n' + self.response.headers['Content-Type'] = 'text/plain' + self.response.out.write(msg) + +app = webapp2.WSGIApplication([('/', HelloHandler)], + debug=True) diff --git a/scripts/testdata/hello_world_compat_golden/.dockerignore b/scripts/testdata/hello_world_compat_golden/.dockerignore new file mode 100644 index 00000000..5ce5abfa --- /dev/null +++ b/scripts/testdata/hello_world_compat_golden/.dockerignore @@ -0,0 +1,5 @@ +.dockerignore +Dockerfile +.git +.hg +.svn diff --git a/scripts/testdata/hello_world_compat_golden/Dockerfile b/scripts/testdata/hello_world_compat_golden/Dockerfile new file mode 100644 index 00000000..1e4d6352 --- /dev/null +++ b/scripts/testdata/hello_world_compat_golden/Dockerfile @@ -0,0 +1,3 @@ +FROM gcr.io/google_appengine/python-compat-multicore +ADD . /app/ +RUN if [ -s requirements.txt ]; then pip install -r requirements.txt; fi diff --git a/scripts/testdata/hello_world_golden/Dockerfile b/scripts/testdata/hello_world_golden/Dockerfile index e3e0563c..10396399 100644 --- a/scripts/testdata/hello_world_golden/Dockerfile +++ b/scripts/testdata/hello_world_golden/Dockerfile @@ -1,6 +1,6 @@ FROM gcr.io/google-appengine/python -LABEL python_version=python3.5 -RUN virtualenv --no-download /env -p python3.5 +LABEL python_version=python3.6 +RUN virtualenv --no-download /env -p python3.6 # Set virtualenv environment variables. This is equivalent to running # source /env/bin/activate diff --git a/tests/deploy_check/requirements.txt b/tests/deploy_check/requirements.txt index e7676bba..a34d076b 100644 --- a/tests/deploy_check/requirements.txt +++ b/tests/deploy_check/requirements.txt @@ -1,2 +1,2 @@ -Flask==0.12.1 -gunicorn==19.7.1 +Flask==1.0.2 +gunicorn==19.9.0 diff --git a/tests/eventlet/.gitignore b/tests/eventlet/.gitignore new file mode 100644 index 00000000..14be8077 --- /dev/null +++ b/tests/eventlet/.gitignore @@ -0,0 +1,2 @@ +Dockerfile +main.py diff --git a/tests/eventlet/Dockerfile.in b/tests/eventlet/Dockerfile.in new file mode 100644 index 00000000..fa65a236 --- /dev/null +++ b/tests/eventlet/Dockerfile.in @@ -0,0 +1,13 @@ +FROM ${STAGING_IMAGE} +LABEL python_version=python3.6 +RUN virtualenv --no-download /env -p python3.6 + +# Set virtualenv environment variables. This is equivalent to running +# source /env/bin/activate +ENV VIRTUAL_ENV /env +ENV PATH /env/bin:$PATH +ADD requirements.txt /app/ +RUN pip install -r requirements.txt +ADD . /app/ +RUN gunicorn -k eventlet -b :$PORT --daemon main:app ; \ + wget --retry-connrefused --tries=5 http://localhost:$PORT/ diff --git a/tests/eventlet/README.md b/tests/eventlet/README.md new file mode 100644 index 00000000..0c2a969c --- /dev/null +++ b/tests/eventlet/README.md @@ -0,0 +1 @@ +# Test the Python base image against the 'eventlet' library diff --git a/tests/eventlet/requirements.txt b/tests/eventlet/requirements.txt new file mode 100644 index 00000000..1e2ea66b --- /dev/null +++ b/tests/eventlet/requirements.txt @@ -0,0 +1,10 @@ +click==6.7 +enum-compat==0.0.2 +eventlet==0.24.1 +Flask==2.2.5 +greenlet==0.4.14 +gunicorn==19.9.0 +itsdangerous==0.24 +Jinja2==2.10 +MarkupSafe==1.1.1 +Werkzeug==2.2.3 diff --git a/tests/google-cloud-python-system/.gitignore b/tests/google-cloud-python-system/.gitignore deleted file mode 100644 index 470715ec..00000000 --- a/tests/google-cloud-python-system/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -credentials.json -Dockerfile diff --git a/tests/google-cloud-python-system/Dockerfile.in b/tests/google-cloud-python-system/Dockerfile.in deleted file mode 100644 index 04f28bd5..00000000 --- a/tests/google-cloud-python-system/Dockerfile.in +++ /dev/null @@ -1,16 +0,0 @@ -FROM ${STAGING_IMAGE} - -# Get the source. -RUN git clone --depth 1 https://github.com/GoogleCloudPlatform/google-cloud-python.git -WORKDIR google-cloud-python - -# Install nox -RUN pip install --upgrade nox-automation - -# Secrets injected at runtime -ENV GOOGLE_APPLICATION_CREDENTIALS=/workspace/tests/google-cloud-python-system/credentials.json -ENV GOOGLE_CLOUD_PROJECT=${GOOGLE_CLOUD_PROJECT_FOR_TESTS} - -# Run system tests for all supported Python versions -ADD run_system_tests.sh /run_system_tests.sh -ENTRYPOINT ["/run_system_tests.sh"] diff --git a/tests/google-cloud-python-system/run_system_tests.sh b/tests/google-cloud-python-system/run_system_tests.sh deleted file mode 100755 index 17d3aac3..00000000 --- a/tests/google-cloud-python-system/run_system_tests.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/sh -set -eu - -cd /app/google-cloud-python - -# Not all packages have system tests -packages=" -bigquery -bigtable -datastore -language -logging -monitoring -pubsub -spanner -speech -storage -vision -" - -# translate has system test but it gives error message: -# BadRequest: 400 Invalid JSON payload received. Unknown name "model": Cannot bind 'nmt'. Field 'model' could not be found in request message. (GET https://translation.googleapis.com/language/translate/v2?target=de&q=hvala+ti&q=dankon&q=Me+llamo+Jeff&q=My+name+is+Jeff&model=nmt) -disabled_packages="translate" - -# Spanner system test needs this -export GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE=1 - -exit_code=0 -for package in ${packages}; do - noxfile="${package}/nox.py" - nox \ - -f "${noxfile}" \ - -e \ - "system_tests(python_version='2.7')" \ - "system_tests(python_version='3.6')" \ - || exit_code=1 -done - -exit "${exit_code}" diff --git a/tests/google-cloud-python/Dockerfile.in b/tests/google-cloud-python/Dockerfile.in index a2873863..b82c18d1 100644 --- a/tests/google-cloud-python/Dockerfile.in +++ b/tests/google-cloud-python/Dockerfile.in @@ -4,6 +4,9 @@ FROM ${STAGING_IMAGE} RUN git clone --depth 1 https://github.com/GoogleCloudPlatform/google-cloud-python.git WORKDIR google-cloud-python +# Upgrade setuptools +RUN pip install --upgrade setuptools + # Install nox RUN pip install --upgrade nox-automation diff --git a/tests/google-cloud-python/run_unit_tests.sh b/tests/google-cloud-python/run_unit_tests.sh index 030919d8..c386f1c0 100755 --- a/tests/google-cloud-python/run_unit_tests.sh +++ b/tests/google-cloud-python/run_unit_tests.sh @@ -4,15 +4,31 @@ set -eu cd /app/google-cloud-python exit_code=0 +failed_files= for noxfile in */nox.py; do + if [ "${noxfile}" = "dlp/nox.py" ]; then + echo "**** Skipping ${noxfile} ****" + continue + fi + echo "**** Starting tests in ${noxfile} ****" nox \ -f "${noxfile}" \ -e \ - "unit_tests(python_version='2.7')" \ - "unit_tests(python_version='3.4')" \ - "unit_tests(python_version='3.5')" \ - "unit_tests(python_version='3.6')" \ - || exit_code=1 + "unit(py='2.7')" \ + "unit(py='3.4')" \ + "unit(py='3.5')" \ + "unit(py='3.6')" \ + || { + echo "**** FAILED tests in ${noxfile} ****" + exit_code=1 + failed_files="${failed_files} ${noxfile}" + } + echo "**** Finished tests in ${noxfile} ****" done +if [ "${exit_code}" -eq 0 ]; then + echo "**** All tests passed ****" +else + echo "**** There were test failures:${failed_files} ****" +fi exit "${exit_code}" diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index 71d785ae..a0634bee 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -1,7 +1,9 @@ -Flask==0.12.1 -google-cloud-error-reporting==0.24.2 -google-cloud-logging==1.0.0 -google-cloud-monitoring==0.24.0 -gunicorn==19.7.1 -requests==2.14.2 +Flask==2.2.5 +google-cloud-error-reporting==0.32.1 +google-cloud-logging==1.12.1 +google-cloud-monitoring==0.33.0 +gunicorn==19.9.0 +requests==2.31.0 retrying==1.3.3 +six==1.12.0 +protobuf>=3.6.0 diff --git a/tests/integration/server.py b/tests/integration/server.py index cc82fb93..0b4382c6 100755 --- a/tests/integration/server.py +++ b/tests/integration/server.py @@ -17,6 +17,7 @@ from functools import wraps import json import logging +import os import google.cloud.logging import google.cloud.monitoring @@ -33,6 +34,12 @@ 'CRITICAL': (logging.critical, 'stderr') } +_APPENGINE_FLEXIBLE_ENV_VM = 'GAE_APPENGINE_HOSTNAME' +"""Environment variable set in App Engine when vm:true is set.""" + +_APPENGINE_FLEXIBLE_ENV_FLEX = 'GAE_INSTANCE' +"""Environment variable set in App Engine when env:flex is set.""" + app = Flask(__name__) @@ -132,6 +139,7 @@ def _log_default(token, level): logging.error('Error while writing logs: {0}'.format(e)) raise ErrorResponse('Error while writing logs: {0}'.format(e)) + # this is fine regardless of environment, it's only used in GAE logs return 'appengine.googleapis.com%2F{0}'.format(source) @@ -231,6 +239,19 @@ def _custom(): return json.dumps(tests), 200 +@app.route('/environment', methods=['GET']) +def _check_environment(): + # determine what cloud env we're running in; essentially, GAE vs GKE + # for GAE, we'll check the existence env vars set on + # vm:true or env:flex + # if neither exist, assume we're in GKE + environment = "GKE" + if (_APPENGINE_FLEXIBLE_ENV_VM in os.environ or + _APPENGINE_FLEXIBLE_ENV_FLEX in os.environ): + environment = "GAE" + return environment, 200 + + class ErrorResponse(Exception): status_code = 400 diff --git a/tests/license-test/license-test.yaml b/tests/license-test/license-test.yaml index cb2d67ce..348fbc9b 100644 --- a/tests/license-test/license-test.yaml +++ b/tests/license-test/license-test.yaml @@ -1,6 +1,6 @@ schemaVersion: "1.0.0" -# See https://github.com/GoogleCloudPlatform/runtimes-common/blob/master/structure_tests/README.md#license-tests +# See https://github.com/GoogleCloudPlatform/container-structure-test/blob/master/README.md licenseTests: - debian: true files: [] diff --git a/tests/no-virtualenv/no-virtualenv.yaml b/tests/no-virtualenv/no-virtualenv.yaml index de932617..4f6c3f48 100644 --- a/tests/no-virtualenv/no-virtualenv.yaml +++ b/tests/no-virtualenv/no-virtualenv.yaml @@ -4,7 +4,52 @@ commandTests: command: ["which", "python"] expectedOutput: ["/usr/bin/python\n"] + - name: "default pip installation" + command: ["which", "pip"] + expectedOutput: ["/usr/local/bin/pip\n"] + + - name: "default pip python version" + command: ["pip", "-V"] + expectedOutput: ["pip .* from .*python 2[.]7"] + + - name: "default virtualenv installation" + command: ["which", "virtualenv"] + expectedOutput: ["/usr/local/bin/virtualenv\n"] + + - name: "default python2.7 installation" + command: ["which", "python2.7"] + expectedOutput: ["/usr/bin/python2.7\n"] + + - name: "default python3.4 installation" + command: ["which", "python3.4"] + expectedOutput: ["/opt/python3.4/bin/python3.4\n"] + + - name: "default python3.5 installation" + command: ["which", "python3.5"] + expectedOutput: ["/opt/python3.5/bin/python3.5\n"] + + - name: "default python3.6 installation" + command: ["which", "python3.6"] + expectedOutput: ["/opt/python3.6/bin/python3.6\n"] + - name: "default gunicorn installation" setup: [["pip", "install", "gunicorn"]] command: ["which", "gunicorn"] expectedOutput: ["/usr/local/bin/gunicorn\n"] + + - # Regression test for issue187 + name: "default python3 installation" + command: ["which", "python3"] + expectedOutput: ["/usr/local/bin/python3\n"] + - name: "default python3 version" + command: ["python3", "--version"] + expectedOutput: ["Python 3.7.9\n"] + - name: "default pip3 installation" + command: ["which", "pip3"] + expectedOutput: ["/usr/local/bin/pip3\n"] + + - name: "default flask installation" + # Checks that 'pip' and 'python' are using the same Python version + setup: [["pip", "install", "flask"]] + command: ["python", "-c", "import flask; print(flask.__file__)"] + expectedOutput: ["/usr/local/lib/python2.7/dist-packages/flask"] diff --git a/tests/python2-libraries/Dockerfile b/tests/python2-libraries/Dockerfile new file mode 100644 index 00000000..c2647292 --- /dev/null +++ b/tests/python2-libraries/Dockerfile @@ -0,0 +1,3 @@ +ARG intermediate_image +FROM $intermediate_image +COPY requirements.txt /requirements.txt diff --git a/tests/python2-libraries/python2-libraries.yaml b/tests/python2-libraries/python2-libraries.yaml index 5d298b58..f9fef5a0 100644 --- a/tests/python2-libraries/python2-libraries.yaml +++ b/tests/python2-libraries/python2-libraries.yaml @@ -7,10 +7,7 @@ globalEnvVars: value: "/env/bin:$PATH" commandTests: - - name: "virtual env teardown" - command: ["rm", "-rf", "/env"] - - name: "requirements" - setup: [["virtualenv", "/env"]] - command: ["pip", "install", "-r", "/workspace/tests/python2-libraries/requirements.txt"] + setup: [["virtualenv", "-p", "python", "/env"]] + command: ["pip", "install", "-r", "/requirements.txt"] exitCode: 0 diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 61e36d58..f0596d67 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -1,206 +1,207 @@ -alembic==0.9.1 -amqp==2.1.4 +alembic==1.0.0 +amqp==2.3.2 amqplib==1.0.2 -ansible==2.3.0.0 +ansible==7.0.0 anyjson==0.3.3 -apache-libcloud==2.0.0 +apache-libcloud==2.3.0 argparse==1.4.0 -astroid==1.5.2 -awscli==1.11.85 -babel==2.4.0 +astroid==1.6.5 +awscli==1.16.1 +babel==2.6.0 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 -beautifulsoup4==4.6.0 +beautifulsoup4==4.6.3 beautifulsoup==3.2.1 -billiard==3.5.0.2 -blessings==1.6 +billiard==3.5.0.4 +blessings==1.7 blinker==1.4 -boto==2.46.1 -botocore==1.5.48 +boto==2.49.0 +botocore==1.11.1 bottle==0.12.13 -carbon==1.0.1 -celery==4.0.2 -certifi==2017.4.17 -cffi==1.10.0 -chardet==3.0.2 +carbon<1.1.1 +celery==4.2.1 +certifi==2023.7.22 +cffi==1.11.5 +chardet==3.0.4 click==6.7 -cliff==2.7.0 -cmd2==0.7.0 +cliff==2.13.0 +cmd2==0.8.9 colorama==0.3.9 configobj==5.0.6 cov-core==1.15.0 -coverage==4.4 -coveralls==1.1 -cryptography==1.8.1 -cssselect==1.0.1 -cython==0.25.2 -decorator==4.0.11 -django-celery==3.2.1 -django-debug-toolbar==1.8 -django-extensions==1.7.9 -django==1.11.1 +coverage==4.5.1 +coveralls==1.4.0 +crcmod==1.7 +cryptography==2.3.1 +cssselect==1.0.3 +cython==0.28.5 +decorator==4.3.0 +django-celery==3.2.2 +django-debug-toolbar==1.9.1 +django-extensions==2.1.1 +django<2.0 django_compress==1.0.1 -djangorestframework==3.6.3 +djangorestframework==3.8.2 docker-py==1.10.6 docopt==0.6.2 -docutils==0.13.1 +docutils==0.14 ecdsa==0.13 -elasticsearch==5.3.0 +elasticsearch==6.3.1 enum34==1.1.6 -eventlet==0.21.0 +eventlet==0.24.1 extras==1.0.0 -fabric==1.13.2 +fabric==2.3.1 fixtures==3.0.0 -flake8==3.3.0 -flask==0.12.1 +flake8==3.5.0 +flask==2.2.5 funcsigs==1.0.2 functools32==3.2.3.post2 -futures==3.1.1 -gevent==1.2.1 -google-api-python-client==1.6.2 -graphite-web==1.0.1 -greenlet==0.4.12 -gunicorn==19.7.1 +futures==3.2.0 +gevent==1.3.6 +google-api-python-client==1.7.4 +graphite-web==1.1.3 +greenlet==0.4.14 +gunicorn==19.9.0 hiredis==0.2.0 -html5lib -httplib2==0.10.3 -idna==2.5 -ipaddress==1.0.18 -#ipython # No longer supports Python 2 as of version 6.0 -iso8601==0.1.11 -isodate==0.5.4 +honcho==1.0.1 +html5lib==1.0.1 +httplib2==0.11.3 +idna==2.7 +ipaddress==1.0.22 +iso8601==0.1.12 +isodate==0.6.0 itsdangerous==0.24 -jinja2==2.9.6 -jmespath==0.9.2 +jinja2==2.10 +jmespath==0.9.3 jsonschema==2.6.0 -kombu==4.0.2 +kombu==4.2.1 linecache2==1.0.0 -logilab-common==1.4.0 -lxml==3.7.3 -m2crypto==0.26.0 -mako==1.0.6 +logilab-common==1.4.2 +lxml==4.2.4 +m2crypto==0.30.1 +mako==1.0.7 manifestparser==1.1 -markdown==2.6.8 +markdown==2.6.11 markupsafe==1.0 -matplotlib==2.0.2 +matplotlib==2.2.3 mccabe==0.6.1 meld3==1.0.2 mock==2.0.0 mozcrash==1.0 -mozdevice==0.50 +mozdevice==1.0.1 mozfile==1.2 -mozinfo==0.9 -mozlog==3.4 +mozinfo==0.10 +mozlog==3.8 moznetwork==0.27 -mozprocess==0.25 -mozprofile==0.28 -mozrunner==6.13 -msgpack-python==0.4.8 +mozprocess==0.26 +mozprofile==1.1.0 +mozrunner==7.0.1 +msgpack-python==0.5.6 mysql-python==1.2.5 -ndg-httpsclient==0.4.2 +ndg-httpsclient==0.5.1 netaddr==0.7.19 -netifaces==0.10.5 -newrelic==2.86.0.65 +netifaces==0.10.7 +newrelic==4.2.0.100 nose==1.3.7 -numpy==1.12.1 +numpy==1.22.0 oauth2==1.9.0.post1 -oauth2client==4.1.0 -oauthlib==2.0.2 +oauth2client==4.1.2 +oauthlib==2.1.0 ordereddict==1.1 -oslo.config==4.1.0 -pandas==0.20.1 -paramiko==2.1.2 +oslo.config==6.4.0 +pandas==0.23.4 +paramiko==2.4.1 passlib==1.7.1 paste==2.0.3 pastedeploy==1.5.2 pastescript==2.0.2 -pbr==3.0.0 -pep8==1.7.0 -pexpect==4.2.1 -pika==0.10.0 -pillow==4.1.1 -pip==9.0.1 -prettytable -protobuf==3.3.0 -psutil==5.2.2 -psycopg2==2.7.1 -py==1.4.33 -pyasn1-modules==0.0.8 -pyasn1==0.2.3 -pycparser==2.17 +pbr==4.2.0 +pep8==1.7.1 +pexpect==4.6.0 +pika==0.12.0 +pillow==9.3.0 +pip==18.0 +prettytable==0.7.2 +protobuf==3.6.1 +psutil==5.4.7 +psycopg2==2.7.5 +py==1.5.4 +pyasn1-modules==0.2.2 +pyasn1==0.4.4 +pycparser==2.18 pycrypto==2.6.1 -pycurl==7.43.0 -pyflakes==1.5.0 +pycurl==7.43.0.2 +pyflakes==2.0.0 pygments==2.2.0 -pyjwt==1.5.0 +pyjwt==1.6.4 pylibmc==1.5.2 -pylint==1.7.1 -pymongo==3.4.0 -pymysql==0.7.11 -pyopenssl==17.0.0 +pylint==1.9.3 +pymongo==3.7.1 +pymysql==0.9.2 +pyopenssl==18.0.0 pyparsing==2.2.0 -pyramid==1.8.3 +pyramid==1.9.2 pystache==0.5.4 pytest-cov==2.5.1 -pytest==3.0.7 +pytest==3.7.3 python-cjson==1.2.1 -python-daemon==2.1.2 -python-dateutil==2.6.0 -python-gflags==3.1.1 -python-keystoneclient==3.10.0 -python-memcached==1.58 +python-daemon==2.2.0 +python-dateutil==2.7.3 +python-gflags==3.1.2 +python-keystoneclient==3.17.0 +python-memcached==1.59 python-mimeparse==1.6.0 -python-novaclient==8.0.0 -python-subunit==1.2.0 -python-swiftclient==3.3.0 -pytz==2017.2 -pyyaml==3.12 -pyzmq==16.0.2 -raven==6.0.0 -redis==2.10.5 -repoze.lru==0.6 -requests-oauthlib==0.8.0 -requests==2.14.2 +python-novaclient==11.0.0 +python-subunit==1.3.0 +python-swiftclient==3.6.0 +pytz==2018.5 +pyyaml==5.4 +pyzmq==17.1.2 +raven==6.9.0 +redis==2.10.6 +repoze.lru==0.7 +requests-oauthlib==1.0.0 +requests==2.31.0 retrying==1.3.3 rsa==3.4.2 -scipy==0.19.0 -selenium==3.4.1 +scipy==1.10.0 +selenium==3.14.0 setuptools-git==1.2 -setuptools==35.0.2 -sh==1.12.13 -simplejson==3.10.0 -six==1.10.0 +setuptools==40.2.0 +sh==1.12.14 +simplejson==3.16.0 +six==1.11.0 snowballstemmer==1.2.1 south==1.0.2 -sphinx==1.5.5 +sphinx==1.7.7 sqlalchemy-migrate==0.11.0 -sqlalchemy==1.1.9 -sqlparse==0.2.3 -statsd==3.2.1 -stevedore==1.21.0 +sqlalchemy==1.2.11 +sqlparse==0.4.4 +statsd==3.3.0 +stevedore==1.29.0 suds==0.4 -supervisor==3.3.1 +supervisor==3.3.4 testrepository==0.0.20 testtools==2.3.0 -thrift==0.10.0 -tornado==4.5.1 -tox==2.7.0 -twisted==17.1.0 +thrift==0.11.0 +tornado==6.3.3 +tox==3.2.1 +twisted==18.7.0 ujson==1.35 -unidecode==0.4.20 +unidecode==1.0.22 unittest2==1.1.0 uritemplate==3.0.0 -urllib3==1.21.1 -uwsgi==2.0.15 +urllib3==1.26.5 +uwsgi==2.0.22 versiontools==1.9.1 -virtualenv==15.1.0 -waitress==1.0.2 +virtualenv==16.0.0 +waitress==2.1.2 warlock==1.3.0 -webob==1.7.2 -websocket-client==0.40.0 -webtest==2.0.27 -werkzeug==0.12.1 -wheel==0.29.0 -xlrd==1.0.0 -zc.buildout==2.9.3 -zope.interface==4.4.1 +webob==1.8.2 +websocket-client==0.51.0 +webtest==2.0.30 +werkzeug==2.2.3 +wheel==0.31.1 +xlrd==1.1.0 +zc.buildout==2.12.1 +zope.interface==4.5.0 diff --git a/tests/python3-libraries/Dockerfile b/tests/python3-libraries/Dockerfile new file mode 100644 index 00000000..c2647292 --- /dev/null +++ b/tests/python3-libraries/Dockerfile @@ -0,0 +1,3 @@ +ARG intermediate_image +FROM $intermediate_image +COPY requirements.txt /requirements.txt diff --git a/tests/python3-libraries/python3-libraries.yaml b/tests/python3-libraries/python3-libraries.yaml index ef15c0a1..ace58132 100644 --- a/tests/python3-libraries/python3-libraries.yaml +++ b/tests/python3-libraries/python3-libraries.yaml @@ -8,15 +8,11 @@ globalEnvVars: commandTests: - name: "requirements 3.5" - setup: - - ["rm", "-rf", "/env"] - - ["virtualenv", "-p", "/opt/python3.5/bin/python3.5", "/env"] - command: ["pip", "install", "-r", "/workspace/tests/python3-libraries/requirements.txt"] + setup: [["virtualenv", "-p", "/opt/python3.5/bin/python3.5", "/env"]] + command: ["pip", "install", "-r", "/requirements.txt"] exitCode: 0 - name: "requirements 3.6" - setup: - - ["rm", "-rf", "/env"] - - ["virtualenv", "-p", "/opt/python3.6/bin/python3.6", "/env"] - command: ["pip", "install", "-r", "/workspace/tests/python3-libraries/requirements.txt"] + setup: [["virtualenv", "-p", "/opt/python3.6/bin/python3.6", "/env"]] + command: ["pip", "install", "-r", "/requirements.txt"] exitCode: 0 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index e99f3629..23175a06 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -1,195 +1,195 @@ -alembic==0.9.1 -amqp==2.1.4 +alembic==1.0.0 +amqp==2.3.2 amqplib==1.0.2 -ansible==2.3.0.0 +ansible==7.0.0 anyjson==0.3.3 -apache-libcloud==2.0.0 +apache-libcloud==2.3.0 argparse==1.4.0 -astroid==1.5.2 -awscli==1.11.85 -babel==2.4.0 +astroid==2.0.4 +awscli==1.16.1 +babel==2.6.0 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 -beautifulsoup4==4.6.0 -billiard==3.5.0.2 -blessings==1.6 +beautifulsoup4==4.6.3 +billiard==3.5.0.4 +blessings==1.7 blinker==1.4 -boto==2.46.1 -botocore==1.5.48 +boto==2.49.0 +botocore==1.11.1 bottle==0.12.13 -celery==4.0.2 -certifi==2017.4.17 -cffi==1.10.0 -chardet==3.0.2 +celery==4.2.1 +certifi==2023.7.22 +cffi==1.11.5 +chardet==3.0.4 click==6.7 -cliff==2.7.0 -cmd2==0.7.0 +cliff==2.13.0 +cmd2==0.9.4 colorama==0.3.9 configobj==5.0.6 cov-core==1.15.0 -coverage==4.4 -coveralls==1.1 -cryptography==1.8.1 -cssselect==1.0.1 -cython==0.25.2 -decorator==4.0.11 -django-celery==3.2.1 -django-debug-toolbar==1.8 -django-extensions==1.7.9 -django==1.11.1 +coverage==4.5.1 +coveralls==1.4.0 +crcmod==1.7 +cryptography==41.0.3 +cssselect==1.0.3 +cython==0.28.5 +decorator==4.3.0 +django-celery==3.2.2 +django-debug-toolbar==1.9.1 +django-extensions==2.1.1 +django==2.2.28 django_compress==1.0.1 -djangorestframework==3.6.3 +djangorestframework==3.8.2 docker-py==1.10.6 docopt==0.6.2 -docutils==0.13.1 +docutils==0.14 ecdsa==0.13 -elasticsearch==5.3.0 +elasticsearch==6.3.1 enum34==1.1.6 -eventlet==0.21.0 +eventlet==0.24.1 extras==1.0.0 -fabric==1.13.2 +fabric==2.3.1 fixtures==3.0.0 -flake8==3.3.0 -flask==0.12.1 +flake8==3.5.0 +flask==2.2.5 funcsigs==1.0.2 -futures==3.1.1 -gevent==1.2.1 -google-api-python-client==1.6.2 -greenlet==0.4.12 -gunicorn==19.7.1 +gevent==1.3.6 +google-api-python-client==1.7.4 +greenlet==0.4.14 +gunicorn==19.9.0 hiredis==0.2.0 -html5lib -httplib2==0.10.3 -idna==2.5 -ipaddress==1.0.18 -ipython==6.0.0 -iso8601==0.1.11 -isodate==0.5.4 +honcho==1.0.1 +html5lib==1.0.1 +httplib2==0.11.3 +idna==2.7 +ipaddress==1.0.22 +ipython==6.5.0 +iso8601==0.1.12 +isodate==0.6.0 itsdangerous==0.24 -jinja2==2.9.6 -jmespath==0.9.2 +jinja2==2.10 +jmespath==0.9.3 jsonschema==2.6.0 -kombu==4.0.2 +kombu==4.2.1 linecache2==1.0.0 -logilab-common==1.4.0 -lxml==3.7.3 -m2crypto==0.26.0 -mako==1.0.6 +logilab-common==1.4.2 +lxml==4.2.4 +mako==1.0.7 manifestparser==1.1 -markdown==2.6.8 -markupsafe==1.0 -matplotlib==2.0.2 +markdown==2.6.11 +markupsafe==1.1.1 +matplotlib==2.2.3 mccabe==0.6.1 meld3==1.0.2 mock==2.0.0 mozcrash==1.0 -mozdevice==0.50 +mozdevice==1.0.1 mozfile==1.2 -mozinfo==0.9 -mozlog==3.4 +mozinfo==0.10 +mozlog==3.8 moznetwork==0.27 -mozprocess==0.25 -msgpack-python==0.4.8 -ndg-httpsclient==0.4.2 +mozprocess==0.26 +msgpack-python==0.5.6 +ndg-httpsclient==0.5.1 netaddr==0.7.19 -netifaces==0.10.5 -newrelic==2.86.0.65 +netifaces==0.10.7 +newrelic==4.2.0.100 nose==1.3.7 -numpy==1.12.1 +numpy==1.22.0 oauth2==1.9.0.post1 -oauth2client==4.1.0 -oauthlib==2.0.2 +oauth2client==4.1.2 +oauthlib==2.1.0 ordereddict==1.1 -oslo.config==4.1.0 -pandas==0.20.1 -paramiko==2.1.2 +oslo.config==6.4.0 +pandas==0.23.4 +paramiko==2.4.1 passlib==1.7.1 paste==2.0.3 pastedeploy==1.5.2 pastescript==2.0.2 -pbr==3.0.0 -pep8==1.7.0 -pexpect==4.2.1 -pika==0.10.0 -pillow==4.1.1 -pip==9.0.1 -prettytable -protobuf==3.3.0 -psutil==5.2.2 -psycopg2==2.7.1 -py==1.4.33 -pyasn1-modules==0.0.8 -pyasn1==0.2.3 -pycparser==2.17 +pbr==4.2.0 +pep8==1.7.1 +pexpect==4.6.0 +pika==0.12.0 +pillow==9.3.0 +pip==18.0 +prettytable==0.7.2 +protobuf==3.6.1 +psutil==5.4.7 +psycopg2==2.7.5 +py==1.5.4 +pyasn1-modules==0.2.2 +pyasn1==0.4.4 +pycparser==2.18 pycrypto==2.6.1 -pyflakes==1.5.0 -pygments==2.2.0 -pyjwt==1.5.0 +pyflakes==2.0.0 +pygments==2.15.0 +pyjwt==1.6.4 pylibmc==1.5.2 -pylint==1.7.1 -pymongo==3.4.0 -pymysql==0.7.11 -pyopenssl==17.0.0 +pylint==2.1.1 +pymongo==3.7.1 +pymysql==0.9.2 +pyopenssl==18.0.0 pyparsing==2.2.0 -pyramid==1.8.3 +pyramid==1.9.2 pystache==0.5.4 pytest-cov==2.5.1 -pytest==3.0.7 -python-daemon==2.1.2 -python-dateutil==2.6.0 -python-gflags==3.1.1 -python-keystoneclient==3.10.0 -python-memcached==1.58 +pytest==3.7.3 +python-daemon==2.2.0 +python-dateutil==2.7.3 +python-gflags==3.1.2 +python-keystoneclient==3.17.0 +python-memcached==1.59 python-mimeparse==1.6.0 -python-novaclient==8.0.0 -python-subunit==1.2.0 -python-swiftclient==3.3.0 -pytz==2017.2 -pyyaml==3.12 -pyzmq==16.0.2 -raven==6.0.0 -redis==2.10.5 -repoze.lru==0.6 -requests-oauthlib==0.8.0 -requests==2.14.2 +python-novaclient==11.0.0 +python-subunit==1.3.0 +python-swiftclient==3.6.0 +pytz==2018.5 +pyyaml==5.4 +pyzmq==17.1.2 +raven==6.9.0 +redis==2.10.6 +repoze.lru==0.7 +requests-oauthlib==1.0.0 +requests==2.31.0 retrying==1.3.3 rsa==3.4.2 -scipy==0.19.0 -selenium==3.4.1 +scipy==1.10.0 +selenium==3.14.0 setuptools-git==1.2 -setuptools==35.0.2 -sh==1.12.13 -simplejson==3.10.0 -six==1.10.0 +setuptools==40.2.0 +sh==1.12.14 +simplejson==3.16.0 +six==1.11.0 snowballstemmer==1.2.1 south==1.0.2 -sphinx==1.5.5 +sphinx==1.7.7 sqlalchemy-migrate==0.11.0 -sqlalchemy==1.1.9 -sqlparse==0.2.3 -statsd==3.2.1 -stevedore==1.21.0 +sqlalchemy==1.2.11 +sqlparse==0.4.4 +statsd==3.3.0 +stevedore==1.29.0 testrepository==0.0.20 testtools==2.3.0 -thrift==0.10.0 -tornado==4.5.1 -tox==2.7.0 -twisted==17.1.0 +thrift==0.11.0 +tornado==6.3.3 +tox==3.2.1 +twisted==18.7.0 ujson==1.35 -unidecode==0.4.20 +unidecode==1.0.22 unittest2==1.1.0 uritemplate==3.0.0 -urllib3==1.21.1 -uwsgi==2.0.15 +urllib3==1.26.5 +uwsgi==2.0.22 versiontools==1.9.1 -virtualenv==15.1.0 -waitress==1.0.2 +virtualenv==16.0.0 +waitress==2.1.2 warlock==1.3.0 -webob==1.7.2 -websocket-client==0.40.0 -webtest==2.0.27 -werkzeug==0.12.1 -wheel==0.29.0 -xlrd==1.0.0 -zc.buildout==2.9.3 -zope.interface==4.4.1 +webob==1.8.2 +websocket-client==0.51.0 +webtest==2.0.30 +werkzeug==2.2.3 +wheel==0.31.1 +xlrd==1.1.0 +zc.buildout==2.12.1 +zope.interface==4.5.0 diff --git a/tests/virtualenv/virtualenv_default.yaml b/tests/virtualenv/virtualenv_default.yaml index 314a3d0c..6b6ad282 100644 --- a/tests/virtualenv/virtualenv_default.yaml +++ b/tests/virtualenv/virtualenv_default.yaml @@ -7,32 +7,31 @@ globalEnvVars: value: "/env/bin:$PATH" commandTests: - - name: "virtual env teardown" - command: ["rm", "-rf", "/env"] - - - name: "python installation" - command: ["which", "python"] - expectedOutput: ["/usr/bin/python\n"] - - - name: "virtualenv installation" + - name: "virtualenv python installation" setup: [["virtualenv", "/env"]] command: ["which", "python"] expectedOutput: ["/env/bin/python\n"] - - name: "python version" + - name: "virtualenv python version" + setup: [["virtualenv", "/env"]] command: ["python", "--version"] # we check stderr instead of stdout for Python versions < 3.4 # https://bugs.python.org/issue18338 - expectedError: ["Python 2.7.9\n"] + expectedError: ["Python 2.7.(9|12)\n"] - - name: "pip installation" + - name: "virtualenv pip installation" + setup: [["virtualenv", "/env"]] command: ["which", "pip"] expectedOutput: ["/env/bin/pip\n"] - - name: "gunicorn flask" - setup: [["pip", "install", "gunicorn", "flask"]] + - name: "virtualenv gunicorn installation" + setup: [["virtualenv", "/env"], + ["pip", "install", "gunicorn"]] command: ["which", "gunicorn"] expectedOutput: ["/env/bin/gunicorn"] - - name: "flask integration" - command: ["python", "-c", "\"import sys; import flask; sys.exit(0 if flask.__file__.startswith('/env') else 1)\""] + - name: "virtualenv flask installation" + setup: [["virtualenv", "/env"], + ["pip", "install", "flask"]] + command: ["python", "-c", "import flask; print(flask.__file__)"] + expectedOutput: ["/env/lib/python2.7/site-packages/flask/__init__.pyc"] diff --git a/tests/virtualenv/virtualenv_python27.yaml b/tests/virtualenv/virtualenv_python27.yaml new file mode 100644 index 00000000..09b78480 --- /dev/null +++ b/tests/virtualenv/virtualenv_python27.yaml @@ -0,0 +1,47 @@ +schemaVersion: "1.0.0" + +globalEnvVars: + - key: "VIRTUAL_ENV" + value: "/env" + - key: "PATH" + value: "/env/bin:$PATH" + +commandTests: + - name: "virtualenv27 python installation" + setup: [["virtualenv", "-p", "python", "/env"]] + command: ["which", "python"] + expectedOutput: ["/env/bin/python\n"] + + - name: "virtualenv27 python2 installation" + setup: [["virtualenv", "-p", "python", "/env"]] + command: ["which", "python2"] + expectedOutput: ["/env/bin/python2\n"] + + - name: "virtualenv27python2.7 installation" + setup: [["virtualenv", "-p", "python", "/env"]] + command: ["which", "python2.7"] + expectedOutput: ["/env/bin/python2.7\n"] + + - name: "virtualenv27 python version" + setup: [["virtualenv", "-p", "python", "/env"]] + command: ["python", "--version"] + # we check stderr instead of stdout for Python versions < 3.4 + # https://bugs.python.org/issue18338 + expectedError: ["Python 2.7.(9|12)\n"] + + - name: "virtualenv27 pip installation" + setup: [["virtualenv", "-p", "python", "/env"]] + command: ["which", "pip"] + expectedOutput: ["/env/bin/pip\n"] + + - name: "virtualenv27 gunicorn installation" + setup: [["virtualenv", "-p", "python", "/env"], + ["pip", "install", "gunicorn"]] + command: ["which", "gunicorn"] + expectedOutput: ["/env/bin/gunicorn"] + + - name: "virtualenv27 flask installation" + setup: [["virtualenv", "-p", "python", "/env"], + ["pip", "install", "flask"]] + command: ["python", "-c", "import flask; print(flask.__file__)"] + expectedOutput: ["/env/lib/python2.7/site-packages/flask/__init__.pyc"] diff --git a/tests/virtualenv/virtualenv_python34.yaml b/tests/virtualenv/virtualenv_python34.yaml index 082b3b6e..9b5b77d0 100644 --- a/tests/virtualenv/virtualenv_python34.yaml +++ b/tests/virtualenv/virtualenv_python34.yaml @@ -7,41 +7,48 @@ globalEnvVars: value: "/env/bin:$PATH" commandTests: - - name: "virtual env teardown" - command: ["rm", "-rf", "/env"] - - - name: "python installation" - command: ["which", "python3.4"] - expectedOutput: ["/usr/bin/python3.4\n"] - - - name: "virtualenv python installation" + - name: "virtualenv34 python installation" setup: [["virtualenv", "-p", "python3.4", "/env"]] command: ["which", "python"] expectedOutput: ["/env/bin/python\n"] - - name: "virtualenv python3 installation" + - name: "virtualenv34 python3 installation" + setup: [["virtualenv", "-p", "python3.4", "/env"]] command: ["which", "python3"] expectedOutput: ["/env/bin/python3\n"] - - name: "python version" + - name: "virtualenv34 python3.4 installation" + setup: [["virtualenv", "-p", "python3.4", "/env"]] + command: ["which", "python3.4"] + expectedOutput: ["/env/bin/python3.4\n"] + + - name: "virtualenv34 python version" + setup: [["virtualenv", "-p", "python3.4", "/env"]] command: ["python", "--version"] - expectedOutput: ["Python 3.4.2\n"] + expectedOutput: ["Python 3.4.8\n"] - - name: "pip installation" + - name: "virtualenv34 pip installation" + setup: [["virtualenv", "-p", "python3.4", "/env"]] command: ["which", "pip"] expectedOutput: ["/env/bin/pip\n"] - - name: "pip3 installation" + - name: "virtualenv34 pip3 installation" + setup: [["virtualenv", "-p", "python3.4", "/env"]] command: ["which", "pip3"] expectedOutput: ["/env/bin/pip3\n"] - - name: "gunicorn flask" - setup: [["pip", "install", "gunicorn", "flask"]] + - name: "virtualenv34 gunicorn installation" + setup: [["virtualenv", "-p", "python3.4", "/env"], + ["pip", "install", "gunicorn"]] command: ["which", "gunicorn"] expectedOutput: ["/env/bin/gunicorn"] - - name: "flask integration" - command: ["python", "-c", "\"import sys; import flask; sys.exit(0 if flask.__file__.startswith('/env') else 1)\""] + - name: "virtualenv34 flask installation" + setup: [["virtualenv", "-p", "python3.4", "/env"], + ["pip", "install", "flask"]] + command: ["python", "-c", "import flask; print(flask.__file__)"] + expectedOutput: ["/env/lib/python3.4/site-packages/flask/__init__.py"] - - name: "test.support" + - name: "virtualenv34 test.support availability" + setup: [["virtualenv", "-p", "python3.4", "/env"]] command: ["python", "-c", "\"from test import pystone, regrtest, support\""] diff --git a/tests/virtualenv/virtualenv_python35.yaml b/tests/virtualenv/virtualenv_python35.yaml index a9248f88..5e4b394a 100644 --- a/tests/virtualenv/virtualenv_python35.yaml +++ b/tests/virtualenv/virtualenv_python35.yaml @@ -7,45 +7,48 @@ globalEnvVars: value: "/env/bin:$PATH" commandTests: - - name: "virtual env teardown" - command: ["rm", "-rf", "/env"] - - - name: "python installation" - command: ["which", "python3.5"] - expectedOutput: ["/opt/python3.5/bin/python3.5\n"] - - - name: "virtualenv python installation" + - name: "virtualenv35 python installation" setup: [["virtualenv", "-p", "python3.5", "/env"]] command: ["which", "python"] expectedOutput: ["/env/bin/python\n"] - - name: "virtualenv python3 installation" + - name: "virtualenv35 python3 installation" + setup: [["virtualenv", "-p", "python3.5", "/env"]] command: ["which", "python3"] expectedOutput: ["/env/bin/python3\n"] - - name: "virtualenv python3.5 installation" + - name: "virtualenv35 python3.5 installation" + setup: [["virtualenv", "-p", "python3.5", "/env"]] command: ["which", "python3.5"] expectedOutput: ["/env/bin/python3.5\n"] - - name: "python version" + - name: "virtualenv35 python version" + setup: [["virtualenv", "-p", "python3.5", "/env"]] command: ["python", "--version"] - expectedOutput: ["Python 3.5.3\n"] + expectedOutput: ["Python 3.5.9\n"] - - name: "pip installation" + - name: "virtualenv35 pip installation" + setup: [["virtualenv", "-p", "python3.5", "/env"]] command: ["which", "pip"] expectedOutput: ["/env/bin/pip\n"] - - name: "pip3 installation" + - name: "virtualenv35 pip3 installation" + setup: [["virtualenv", "-p", "python3.5", "/env"]] command: ["which", "pip3"] expectedOutput: ["/env/bin/pip3\n"] - - name: "gunicorn flask" - setup: [["pip", "install", "gunicorn", "flask"]] + - name: "virtualenv35 gunicorn installation" + setup: [["virtualenv", "-p", "python3.5", "/env"], + ["pip", "install", "gunicorn"]] command: ["which", "gunicorn"] expectedOutput: ["/env/bin/gunicorn"] - - name: "flask integration" - command: ["python", "-c", "\"import sys; import flask; sys.exit(0 if flask.__file__.startswith('/env') else 1)\""] + - name: "virtualenv35 flask installation" + setup: [["virtualenv", "-p", "python3.5", "/env"], + ["pip", "install", "flask"]] + command: ["python", "-c", "import flask; print(flask.__file__)"] + expectedOutput: ["/env/lib/python3.5/site-packages/flask/__init__.py"] - - name: "test.support" + - name: "virtualenv35 test.support availability" + setup: [["virtualenv", "-p", "python3.5", "/env"]] command: ["python", "-c", "\"from test import pystone, regrtest, support\""] diff --git a/tests/virtualenv/virtualenv_python36.yaml b/tests/virtualenv/virtualenv_python36.yaml index 83938fc9..b3a9e68e 100644 --- a/tests/virtualenv/virtualenv_python36.yaml +++ b/tests/virtualenv/virtualenv_python36.yaml @@ -7,45 +7,48 @@ globalEnvVars: value: "/env/bin:$PATH" commandTests: - - name: "virtual env teardown" - command: ["rm", "-rf", "/env"] - - - name: "python installation" - command: ["which", "python3.6"] - expectedOutput: ["/opt/python3.6/bin/python3.6\n"] - - - name: "virtualenv python installation" + - name: "virtualenv36 python installation" setup: [["virtualenv", "-p", "python3.6", "/env"]] command: ["which", "python"] expectedOutput: ["/env/bin/python\n"] - - name: "virtualenv python3 installation" + - name: "virtualenv36 python3 installation" + setup: [["virtualenv", "-p", "python3.6", "/env"]] command: ["which", "python3"] expectedOutput: ["/env/bin/python3\n"] - - name: "virtualenv python3.6 installation" + - name: "virtualenv36 python3.6 installation" + setup: [["virtualenv", "-p", "python3.6", "/env"]] command: ["which", "python3.6"] expectedOutput: ["/env/bin/python3.6\n"] - - name: "python version" + - name: "virtualenv36 python version" + setup: [["virtualenv", "-p", "python3.6", "/env"]] command: ["python", "--version"] - expectedOutput: ["Python 3.6.1\n"] + expectedOutput: ["Python 3.6.10\n"] - - name: "pip installation" + - name: "virtualenv36 pip installation" + setup: [["virtualenv", "-p", "python3.6", "/env"]] command: ["which", "pip"] expectedOutput: ["/env/bin/pip\n"] - - name: "pip3 installation" + - name: "virtualenv36 pip3 installation" + setup: [["virtualenv", "-p", "python3.6", "/env"]] command: ["which", "pip3"] expectedOutput: ["/env/bin/pip3\n"] - - name: "gunicorn flask" - setup: [["pip", "install", "gunicorn", "flask"]] + - name: "virtualenv36 gunicorn installation" + setup: [["virtualenv", "-p", "python3.6", "/env"], + ["pip", "install", "gunicorn"]] command: ["which", "gunicorn"] expectedOutput: ["/env/bin/gunicorn"] - - name: "flask integration" - command: ["python", "-c", "\"import sys; import flask; sys.exit(0 if flask.__file__.startswith('/env') else 1)\""] + - name: "virtualenv36 flask installation" + setup: [["virtualenv", "-p", "python3.6", "/env"], + ["pip", "install", "flask"]] + command: ["python", "-c", "import flask; print(flask.__file__)"] + expectedOutput: ["/env/lib/python3.6/site-packages/flask/__init__.py"] - - name: "test.support" + - name: "virtualenv36 test.support availability" + setup: [["virtualenv", "-p", "python3.6", "/env"]] command: ["python", "-c", "\"from test import pystone, regrtest, support\""] diff --git a/tests/virtualenv/virtualenv_python37.yaml b/tests/virtualenv/virtualenv_python37.yaml new file mode 100644 index 00000000..9810c78e --- /dev/null +++ b/tests/virtualenv/virtualenv_python37.yaml @@ -0,0 +1,54 @@ +schemaVersion: "1.0.0" + +globalEnvVars: + - key: "VIRTUAL_ENV" + value: "/env" + - key: "PATH" + value: "/env/bin:$PATH" + +commandTests: + - name: "virtualenv37 python installation" + setup: [["virtualenv", "-p", "python3.7", "/env"]] + command: ["which", "python"] + expectedOutput: ["/env/bin/python\n"] + + - name: "virtualenv37 python3 installation" + setup: [["virtualenv", "-p", "python3.7", "/env"]] + command: ["which", "python3"] + expectedOutput: ["/env/bin/python3\n"] + + - name: "virtualenv37 python3.7 installation" + setup: [["virtualenv", "-p", "python3.7", "/env"]] + command: ["which", "python3.7"] + expectedOutput: ["/env/bin/python3.7\n"] + + - name: "virtualenv37 python version" + setup: [["virtualenv", "-p", "python3.7", "/env"]] + command: ["python", "--version"] + expectedOutput: ["Python 3.7.9\n"] + + - name: "virtualenv37 pip installation" + setup: [["virtualenv", "-p", "python3.7", "/env"]] + command: ["which", "pip"] + expectedOutput: ["/env/bin/pip\n"] + + - name: "virtualenv37 pip3 installation" + setup: [["virtualenv", "-p", "python3.7", "/env"]] + command: ["which", "pip3"] + expectedOutput: ["/env/bin/pip3\n"] + + - name: "virtualenv37 gunicorn installation" + setup: [["virtualenv", "-p", "python3.7", "/env"], + ["pip", "install", "gunicorn"]] + command: ["which", "gunicorn"] + expectedOutput: ["/env/bin/gunicorn"] + + - name: "virtualenv37 flask installation" + setup: [["virtualenv", "-p", "python3.7", "/env"], + ["pip", "install", "flask"]] + command: ["python", "-c", "import flask; print(flask.__file__)"] + expectedOutput: ["/env/lib/python3.7/site-packages/flask/__init__.py"] + + - name: "virtualenv37 test.support availability" + setup: [["virtualenv", "-p", "python3.7", "/env"]] + command: ["python", "-c", "\"from test import pystone, regrtest, support\""]