diff --git a/CODEOWNERS b/CODEOWNERS index a2d4a67f..2b618c5f 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1,4 +1,4 @@ # Code owners file. # This file controls who is tagged for review for any given pull request. -* @duggelz @liyanhui1228 @jonparrott +* @jinglundong @donmccasland diff --git a/README.md b/README.md index 78a02c5c..51e70cd8 100644 --- a/README.md +++ b/README.md @@ -5,10 +5,10 @@ This repository contains the source for the [docker](https://docker.io) base image. This image can be used as the base image for running applications on [Google App Engine Flexible](https://cloud.google.com/appengine/docs/flexible/), -[Google Container Engine](https://cloud.google.com/container-engine), or any +[Google Kubernetes Engine](https://cloud.google.com/kubernetes-engine), or any other Docker host. -This image is based on Debian Jessie and contains packages required to build +This image is based on Ubuntu Xenial and contains packages required to build most of the popular Python libraries. For more information about this runtime, see the [documentation](https://cloud.google.com/appengine/docs/flexible/python/runtime). @@ -38,7 +38,7 @@ to create a custom runtime: You can then modify the `Dockerfile` and `.dockerignore` as needed for you application. -## Container Engine & other Docker hosts. +## Kubernetes Engine & other Docker hosts. For other docker hosts, you'll need to create a `Dockerfile` based on this image that copies your application code, installs dependencies, and declares an @@ -48,6 +48,7 @@ command or entrypoint. For example: # Create a virtualenv for dependencies. This isolates these packages from # system-level packages. + # Use -p python3 or -p python3.7 to select python version. Default is version 2. RUN virtualenv /env # Setting these environment variables are the same as running diff --git a/RELEASING.md b/RELEASING.md index 00b0fe16..f886a542 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -36,9 +36,20 @@ follows: b. `gcloud auth activate-service-account` is performed c. `gcloud config set project` is performed 3. The script invokes `build.sh` in this repository -4. `build.sh` invokes Google Container Builder with the `cloudbuild-*.yaml` +4. `build.sh` invokes Google Cloud Build with the `cloudbuild-*.yaml` config files. +## Building interpreters + +The interpreters used are now built in a separate step, and stored on GCS. +This allows the runtime images to be build more rapidly. + +To build the interpreters, run: + +```shell +gcloud builds submit . --config=cloudbuild_interpreters.yaml +``` + ## Building outside Jenkins To build this repository outside Jenkins, authenticate and authorize yourself @@ -54,7 +65,7 @@ Debian or Ubuntu-like). ## Building locally To build this repository using local Docker commands instead of the Google -Container Builder service, add the `--local` flag as shown: +Cloud Build service, add the `--local` flag as shown: ``` shell ./build.sh --local diff --git a/build.sh b/build.sh index 7e2370d4..2083237d 100755 --- a/build.sh +++ b/build.sh @@ -19,14 +19,18 @@ set -euo pipefail # Actions benchmark=0 # Should run benchmarks? build=0 # Should build images? -system_test=0 # Should run system tests? +client_test=0 # Should run Google Cloud Client Library tests test=0 # Should run standard test suite? local=0 # Should run using local Docker daemon instead of GCR? +os_base=ubuntu18 # Which operating system base to use +interpreter=0 # Should build interpreters instead of images + # Note that $gcloud_cmd has spaces in it -gcloud_cmd="gcloud beta container builds submit ." -local_gcloud_cmd="scripts/local_cloudbuild.py" +gcloud_cmd="gcloud builds submit" +# May need to install via "gcloud components install cloud-build-local" +local_gcloud_cmd="cloud-build-local --push=false --dryrun=false" # Helper functions function fatal() { @@ -42,8 +46,9 @@ Options: --[no]benchmark: Run benchmarking suite (default false) --[no]build: Build all images (default true if no options set) --[no]test: Run basic tests (default true if no options set) + --[no]client_test: Run Google Cloud Client Library tests (default false) --[no]local: Build images using local Docker daemon (default false) - --[no]system_test: Run system tests (default false) + --os_base: Which OS image to build on top of [debian8, ubuntu16, ubuntu18] " } @@ -90,6 +95,14 @@ while [ $# -gt 0 ]; do build=0 shift ;; + --client_test) + client_test=1 + shift + ;; + --noclient_test) + client_test=0 + shift + ;; --local) local=1 shift @@ -98,12 +111,16 @@ while [ $# -gt 0 ]; do local=0 shift ;; - --system_test) - system_test=1 + --os_base=debian8) + os_base=debian8 + shift + ;; + --os_base=ubuntu16) + os_base=ubuntu16 shift ;; - --nosystem_test) - system_test=0 + --os_base=ubuntu18) + os_base=ubuntu18 shift ;; --test) @@ -114,6 +131,10 @@ while [ $# -gt 0 ]; do test=0 shift ;; + --interpreter) + interpreter=1 + shift + ;; *) usage ;; @@ -123,7 +144,7 @@ done # If no actions chosen, then tell the user if [ "${benchmark}" -eq 0 -a \ "${build}" -eq 0 -a \ - "${system_test}" -eq 0 -a \ + "${client_test}" -eq 0 -a \ "${test}" -eq 0 \ ]; then echo 'No actions specified, defaulting to --build --test' @@ -136,19 +157,17 @@ if [ "${local}" -eq 1 ]; then gcloud_cmd="${local_gcloud_cmd}" fi -# Read action-specific environment variables -if [ "${system_test}" -eq 1 ]; then - if [ -z "${GOOGLE_APPLICATION_CREDENTIALS_FOR_TESTS+set}" ] ; then - fatal 'Error: $GOOGLE_APPLICATION_CREDENTIALS_FOR_TESTS is not set; invoke with something like GOOGLE_APPLICATION_CREDENTIALS_FOR_TESTS=/path/to/service/account/creds.json' - fi - - if [ -z "${GOOGLE_CLOUD_PROJECT_FOR_TESTS+set}" ] ; then - fatal 'Error: $GOOGLE_CLOUD_PROJECT_FOR_TESTS is not set; invoke with something like GOOGLE_CLOUD_PROJECT_FOR_TESTS=YOUR-PROJECT-NAME' - fi +# Pick OS image to use as base +if [ "${os_base}" == "debian8" ]; then + export OS_BASE_IMAGE="gcr.io/google-appengine/debian8:latest" +elif [ "${os_base}" == "ubuntu16" ]; then + export OS_BASE_IMAGE="gcr.io/gcp-runtimes/ubuntu_16_0_4:latest" +elif [ "${os_base}" == "ubuntu18" ]; then + export OS_BASE_IMAGE="gcr.io/gcp-runtimes/ubuntu_18_0_4:latest" +else + echo "Unsupported OS base image: $OS_BASE_IMAGE" + exit 1 fi - -# Use latest released Debian as our base image -export DEBIAN_BASE_IMAGE="gcr.io/google-appengine/debian8:latest" export STAGING_IMAGE="${DOCKER_NAMESPACE}/python:${TAG}" echo "Using base image name ${STAGING_IMAGE}" @@ -160,11 +179,10 @@ for outfile in \ tests/benchmark/Dockerfile \ tests/eventlet/Dockerfile \ tests/google-cloud-python/Dockerfile \ - tests/google-cloud-python-system/Dockerfile \ tests/integration/Dockerfile \ ; do envsubst <"${outfile}".in >"${outfile}" \ - '$DEBIAN_BASE_IMAGE $STAGING_IMAGE $GOOGLE_CLOUD_PROJECT_FOR_TESTS $TAG' + '$OS_BASE_IMAGE $STAGING_IMAGE $GOOGLE_CLOUD_PROJECT_FOR_TESTS $TAG' done # Make some files available to the runtime builder Docker context @@ -180,30 +198,46 @@ done # Make a file available to the eventlet test. cp -a scripts/testdata/hello_world/main.py tests/eventlet/main.py +# Build interpreters and push to GCS +if [ "${interpreter}" -eq 1 ]; then + echo "Building interpreters" + ${gcloud_cmd} \ + --config=cloudbuild_interpreters.yaml \ + . +fi + # Build images and push to GCR if [ "${build}" -eq 1 ]; then echo "Building images" - ${gcloud_cmd} --config cloudbuild.yaml --substitutions "${build_substitutions}" + ${gcloud_cmd} \ + --config=cloudbuild.yaml \ + --substitutions="${build_substitutions}" \ + . fi # Run the tests that don't require (too many) external services if [ "${test}" -eq 1 ]; then echo "Testing compatibility with popular Python libraries" - ${gcloud_cmd} --config cloudbuild_test.yaml --substitutions "${substitutions}" + ${gcloud_cmd} \ + --config=cloudbuild_test.yaml \ + --substitutions="${substitutions}" \ + . fi -# Run system tests -if [ "${system_test}" -eq 1 ]; then - echo "Running system tests using project ${GOOGLE_CLOUD_PROJECT_FOR_TESTS}" - - trap "rm -f tests/google-cloud-python-system/credentials.json" EXIT - cp "${GOOGLE_APPLICATION_CREDENTIALS_FOR_TESTS}" tests/google-cloud-python-system/credentials.json - ${gcloud_cmd} --config cloudbuild_system_test.yaml --substitutions "${substitutions}" - rm -f tests/google-cloud-python-system/credentials.json +# Run client library tests +if [ "${client_test}" -eq 1 ]; then + echo "Testing compatibility with Google Cloud Client Libraries" + ${gcloud_cmd} \ + --config=cloudbuild_client_test.yaml \ + --substitutions="${substitutions}" \ + . fi # Run benchmarks if [ "${benchmark}" -eq 1 ] ; then echo "Running benchmark" - ${gcloud_cmd} --config cloudbuild_benchmark.yaml --substitutions "${substitutions}" + ${gcloud_cmd} \ + --config=cloudbuild_benchmark.yaml \ + --substitutions="${substitutions}" \ + . fi diff --git a/builder/gen-dockerfile/Dockerfile.in b/builder/gen-dockerfile/Dockerfile.in index 4f6447eb..92ceaecf 100644 --- a/builder/gen-dockerfile/Dockerfile.in +++ b/builder/gen-dockerfile/Dockerfile.in @@ -1,12 +1,14 @@ FROM ${STAGING_IMAGE} -LABEL python_version=python3.6 -RUN virtualenv --no-download /env -p python3.6 +LABEL python_version=python3.7 +RUN virtualenv --no-download /env -p python3.7 # Set virtualenv environment variables. This is equivalent to running # source /env/bin/activate ENV VIRTUAL_ENV /env ENV PATH /env/bin:$PATH ADD requirements.txt /builder/ +#virtualenv's pip is pegged at version 10.0, removing so +#newer versions get picked up RUN pip install -r /builder/requirements.txt ADD . /builder/ WORKDIR /workspace diff --git a/builder/gen-dockerfile/requirements.txt b/builder/gen-dockerfile/requirements.txt index 59bc593a..4a285555 100644 --- a/builder/gen-dockerfile/requirements.txt +++ b/builder/gen-dockerfile/requirements.txt @@ -1 +1 @@ -PyYAML==3.12 +PyYAML==3.13 diff --git a/builder/python-staging.yaml b/builder/python-staging.yaml new file mode 100644 index 00000000..1e977a89 --- /dev/null +++ b/builder/python-staging.yaml @@ -0,0 +1,12 @@ +# This is a cloudbuild.yaml template for the runtime builder +steps: +- # Generate application Dockerfile + name: 'gcr.io/gcp-runtimes/python/gen-dockerfile:staging' + args: [ + '--base-image=gcr.io/google-appengine/python:staging' + ] +- # Use that Dockerfile to create final application image + name: 'gcr.io/cloud-builders/docker:latest' + args: ['build', '-t', '$_OUTPUT_IMAGE', '.'] +images: + - '$_OUTPUT_IMAGE' diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 1eabb997..e240780e 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -1,22 +1,17 @@ -timeout: 7200s +timeout: 10800s steps: -- # Compile Python interpreters from source - name: gcr.io/cloud-builders/docker:latest - args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/interpreter-builder:${_TAG}', - '--no-cache', '/workspace/python-interpreter-builder/'] -- # Copy interpreters back to workspace - name: ${_DOCKER_NAMESPACE}/python/interpreter-builder:${_TAG} - args: ['cp', '/interpreters.tar.gz', '/workspace/runtime-image/interpreters.tar.gz'] - # Build base runtime image name: gcr.io/cloud-builders/docker:latest args: ['build', '--tag=${_DOCKER_NAMESPACE}/python:${_TAG}', '--no-cache', '/workspace/runtime-image/'] + id: runtime - # Build runtime builder image name: gcr.io/cloud-builders/docker:latest args: ['build', '--tag=${_BUILDER_DOCKER_NAMESPACE}/python/gen-dockerfile:${_TAG}', '--no-cache', '/workspace/builder/gen-dockerfile/'] + id: gen-dockerfile + waitFor: ['runtime'] images: [ - '${_DOCKER_NAMESPACE}/python/interpreter-builder:${_TAG}', '${_DOCKER_NAMESPACE}/python:${_TAG}', '${_BUILDER_DOCKER_NAMESPACE}/python/gen-dockerfile:${_TAG}', ] diff --git a/cloudbuild_client_test.yaml b/cloudbuild_client_test.yaml new file mode 100644 index 00000000..010e1ffd --- /dev/null +++ b/cloudbuild_client_test.yaml @@ -0,0 +1,9 @@ +timeout: 3600s +steps: +- # Build image to run google client library unit tests + name: gcr.io/cloud-builders/docker:latest + args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/google-cloud-python:${_TAG}', + '--no-cache', '/workspace/tests/google-cloud-python/'] +- # Run google client library unit tests + name: ${_DOCKER_NAMESPACE}/python/tests/google-cloud-python:${_TAG} +images: [] diff --git a/cloudbuild_interpreters.yaml b/cloudbuild_interpreters.yaml new file mode 100644 index 00000000..c75e59ad --- /dev/null +++ b/cloudbuild_interpreters.yaml @@ -0,0 +1,33 @@ +timeout: 10800s +steps: +- # Compile Python interpreters from source. This step happens first, then + # the next three in parallel. + name: gcr.io/cloud-builders/docker:latest + args: ['build', '--tag=interpreter-builder', + '--no-cache', '/workspace/python-interpreter-builder/'] + id: interpreter-builder +- name: interpreter-builder + args: ['/scripts/build-python-3.4.sh'] + id: build-3.4 + waitFor: ['interpreter-builder'] +- name: interpreter-builder + args: ['/scripts/build-python-3.5.sh'] + id: build-3.5 + waitFor: ['interpreter-builder'] +- name: interpreter-builder + args: ['/scripts/build-python-3.6.sh'] + id: build-3.6 + waitFor: ['interpreter-builder'] +- name: interpreter-builder + args: ['/scripts/build-python-3.7.sh'] + id: build-3.7 + waitFor: ['interpreter-builder'] + +# Upload them to tbe build-id location +- name: gcr.io/cloud-builders/gsutil:latest + args: ['cp', '/workspace/runtime-image/*.tar.gz', 'gs://python-interpreters/$BUILD_ID/'] + waitFor: ['build-3.4', 'build-3.5', 'build-3.6', 'build-3.7'] + +# "Tag" this as latest +- name: gcr.io/cloud-builders/gsutil:latest + args: ['cp', '-r', 'gs://python-interpreters/$BUILD_ID/*', 'gs://python-interpreters/latest/'] diff --git a/cloudbuild_system_test.yaml b/cloudbuild_system_test.yaml deleted file mode 100644 index 3cac03e2..00000000 --- a/cloudbuild_system_test.yaml +++ /dev/null @@ -1,9 +0,0 @@ -timeout: 3600s -steps: -- name: gcr.io/cloud-builders/docker:latest - args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/google-cloud-python-system:${_TAG}', - '--no-cache', '/workspace/tests/google-cloud-python-system/'] -- name: ${_DOCKER_NAMESPACE}/python/tests/google-cloud-python-system:${_TAG} -images: [ - # Intentionally empty -] diff --git a/cloudbuild_test.yaml b/cloudbuild_test.yaml index acf70bc7..ad674026 100644 --- a/cloudbuild_test.yaml +++ b/cloudbuild_test.yaml @@ -1,46 +1,114 @@ timeout: 3600s steps: - # Validate structure of base runtime image -- name: gcr.io/cloud-builders/docker:latest - args: ['build', '-t', 'python2-libraries-intermediate', '--build-arg', - 'intermediate_image=${_DOCKER_NAMESPACE}/python:${_TAG}', - '/workspace/tests/python2-libraries'] -- name: gcr.io/gcp-runtimes/container-structure-test:v0.1.1 +- # Explicitly pull image into GCB so that later steps work + name: '${_DOCKER_NAMESPACE}/python:${_TAG}' args: [ - '-test.v', - '-image', 'python2-libraries-intermediate', - '/workspace/tests/python2-libraries/python2-libraries.yaml' - ] -- name: gcr.io/cloud-builders/docker:latest - args: ['build', '-t', 'python3-libraries-intermediate', '--build-arg', - 'intermediate_image=${_DOCKER_NAMESPACE}/python:${_TAG}', - '/workspace/tests/python3-libraries'] -- name: gcr.io/gcp-runtimes/container-structure-test:v0.1.1 + '/bin/true', + ] + id: runtime + +- # Validate structure of base runtime image + name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 args: [ '-test.v', - '-image', 'python3-libraries-intermediate', - '/workspace/tests/python3-libraries/python3-libraries.yaml' - ] -- name: gcr.io/gcp-runtimes/container-structure-test:v0.1.1 + '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', + '/workspace/tests/virtualenv/virtualenv_default.yaml', + ] + waitFor: ['runtime'] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 args: [ '-test.v', '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', - '/workspace/tests/virtualenv/virtualenv_default.yaml', '/workspace/tests/virtualenv/virtualenv_python27.yaml', + ] + waitFor: ['runtime'] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 + args: [ + '-test.v', + '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', '/workspace/tests/virtualenv/virtualenv_python34.yaml', + ] + waitFor: ['runtime'] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 + args: [ + '-test.v', + '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', '/workspace/tests/virtualenv/virtualenv_python35.yaml', + ] + waitFor: ['runtime'] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 + args: [ + '-test.v', + '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', '/workspace/tests/virtualenv/virtualenv_python36.yaml', + ] + waitFor: ['runtime'] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 + args: [ + '-test.v', + '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', + '/workspace/tests/virtualenv/virtualenv_python37.yaml', + ] + waitFor: ['runtime'] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 + args: [ + '-test.v', + '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', '/workspace/tests/no-virtualenv/no-virtualenv.yaml', - '/workspace/tests/license-test/license-test.yaml' ] -- # Run compatibility tests + waitFor: ['runtime'] + +# Temporarily disabled because it fails on symbolic links in Ubuntu: +# https://github.com/GoogleCloudPlatform/container-structure-test/issues/77 +#- # Check license compliance +# name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 +# args: [ +# '-test.v', +# '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', +# '/workspace/tests/license-test/license-test.yaml' +# ] +# waitFor: ['runtime'] + +- # Do third-party library compatibility tests for Python 2 name: gcr.io/cloud-builders/docker:latest - args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/eventlet:${_TAG}', - '--no-cache', '/workspace/tests/eventlet/'] -- # Build image to run google client library unit tests + args: [ + 'build', '-t', 'python2-libraries-intermediate', '--build-arg', + 'intermediate_image=${_DOCKER_NAMESPACE}/python:${_TAG}', + '/workspace/tests/python2-libraries' + ] + id: python2-libraries-intermediate + waitFor: ['runtime'] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 + args: [ + '-test.v', + '-image', 'python2-libraries-intermediate', + '/workspace/tests/python2-libraries/python2-libraries.yaml' + ] + waitFor: ['python2-libraries-intermediate'] + +- # Do third-party library compatibility tests for Python 3 name: gcr.io/cloud-builders/docker:latest - args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/google-cloud-python:${_TAG}', - '--no-cache', '/workspace/tests/google-cloud-python/'] -- # Run google client library unit tests - name: ${_DOCKER_NAMESPACE}/python/tests/google-cloud-python:${_TAG} + args: [ + 'build', '-t', 'python3-libraries-intermediate', '--build-arg', + 'intermediate_image=${_DOCKER_NAMESPACE}/python:${_TAG}', + '/workspace/tests/python3-libraries' + ] + id: python3-libraries-intermediate + waitFor: ['runtime'] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 + args: [ + '-test.v', + '-image', 'python3-libraries-intermediate', + '/workspace/tests/python3-libraries/python3-libraries.yaml' + ] + waitFor: ['python3-libraries-intermediate'] + +- # Run other compatibility tests + name: gcr.io/cloud-builders/docker:latest + args: [ + 'build', '--tag=${_DOCKER_NAMESPACE}/python/tests/eventlet:${_TAG}', + '--no-cache', '/workspace/tests/eventlet/' + ] + waitFor: ['runtime'] + images: [] diff --git a/jenkins_build.sh b/jenkins_build.sh deleted file mode 100755 index f88007f5..00000000 --- a/jenkins_build.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -./build.sh "$@" diff --git a/nox.py b/nox.py index 872eb88c..0ee6f443 100644 --- a/nox.py +++ b/nox.py @@ -57,7 +57,7 @@ def lint(session): @nox.session -@nox.parametrize('version', ['3.4', '3.5', '3.6']) +@nox.parametrize('version', ['3.4', '3.5', '3.6', '3.7']) def tests(session, version): session.interpreter = 'python' + version session.install('-r', 'scripts/requirements-test.txt') diff --git a/python-interpreter-builder/Dockerfile.in b/python-interpreter-builder/Dockerfile.in index 90e606eb..9039fbf3 100644 --- a/python-interpreter-builder/Dockerfile.in +++ b/python-interpreter-builder/Dockerfile.in @@ -1,6 +1,6 @@ # The Google App Engine base image is debian (jessie) with ca-certificates # installed. -FROM ${DEBIAN_BASE_IMAGE} +FROM ${OS_BASE_IMAGE} # Install Python build dependencies (based on Debian Build-Depends) RUN apt-get update && apt-get install -yq \ @@ -29,6 +29,7 @@ RUN apt-get update && apt-get install -yq \ mime-support \ net-tools \ netbase \ + python \ python3 \ sharutils \ time \ @@ -45,17 +46,3 @@ ENV LANG C.UTF-8 # Add build scripts ADD scripts /scripts ADD DEBIAN /DEBIAN - -# Build the Python interpreters -RUN mkdir -p /opt/packages && \ - echo -n "" > /opt/packages/packages.txt - -RUN /scripts/build-python-3.5.sh && \ - /scripts/package-python.sh 3.5.4 "1gcp~${TAG}" - -RUN /scripts/build-python-3.6.sh && \ - /scripts/package-python.sh 3.6.2 "1gcp~${TAG}" - -# Tar the interpreters. Tarring is needed because docker cp doesn't handle -# links correctly. -RUN tar czf /interpreters.tar.gz /opt/python?.? diff --git a/python-interpreter-builder/scripts/build-python-3.4.sh b/python-interpreter-builder/scripts/build-python-3.4.sh new file mode 100755 index 00000000..5c0bfb8e --- /dev/null +++ b/python-interpreter-builder/scripts/build-python-3.4.sh @@ -0,0 +1,141 @@ +#!/bin/bash + +set -euo pipefail +set -x + +# Get the source +mkdir -p /opt/sources +cd /opt/sources +wget --no-verbose https://www.python.org/ftp/python/3.4.8/Python-3.4.8.tgz +# SHA-256 generated via `shasum -a 256 [file]` +shasum --check < app.yaml +fi + +cd ${KOKORO_GFILE_DIR}/appengine/integration_tests + +sudo -E /usr/local/bin/pip install --upgrade -r requirements.txt + +if [ -f ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY}/requirements.txt ] +then + sudo -E /usr/local/bin/pip install --upgrade -r ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY}/requirements.txt +fi + +export DEPLOY_LATENCY_PROJECT='cloud-deploy-latency' + +skip_flag="" + +if [ "${SKIP_CUSTOM_LOGGING_TESTS}" = "true" -o "${SKIP_BUILDERS}" = "true" ]; then + skip_flag="$skip_flag --skip-builders" +fi + +if [ "${SKIP_XRT}" = "true" ]; then + skip_flag="$skip_flag --skip-xrt" +fi + +python deploy_check.py -d ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY} -l ${LANGUAGE} ${skip_flag} diff --git a/scripts/gen_dockerfile.py b/scripts/gen_dockerfile.py index eaff183f..97da60f6 100755 --- a/scripts/gen_dockerfile.py +++ b/scripts/gen_dockerfile.py @@ -29,7 +29,6 @@ import validation_utils - # Validate characters for dockerfile image names. # # This roots out obvious mistakes, the full gory details are here: @@ -55,6 +54,7 @@ '3.4': '3.4', '3.5': '3.5', '3.6': '3.6', + '3.7': '3.7', } # Name of environment variable potentially set by gcloud @@ -63,7 +63,7 @@ # Validated application configuration AppConfig = collections.namedtuple( 'AppConfig', - 'base_image dockerfile_python_version entrypoint has_requirements_txt' + 'base_image dockerfile_python_version entrypoint has_requirements_txt is_python_compat' ) @@ -97,6 +97,16 @@ def get_app_config(raw_config, base_image, config_file, source_dir): 'Expected {} contents to be a Mapping type, but found type "{}"'. format(config_file, type(raw_config))) + # Short circuit for python compat. + if validation_utils.get_field_value( + raw_config, 'runtime', str) == 'python-compat': + return AppConfig( + base_image=None, + dockerfile_python_version=None, + entrypoint=None, + has_requirements_txt=None, + is_python_compat=True) + entrypoint = validation_utils.get_field_value( raw_config, 'entrypoint', str) if not PRINTABLE_REGEX.match(entrypoint): @@ -133,7 +143,8 @@ def get_app_config(raw_config, base_image, config_file, source_dir): base_image=base_image, dockerfile_python_version=dockerfile_python_version, entrypoint=entrypoint, - has_requirements_txt=has_requirements_txt) + has_requirements_txt=has_requirements_txt, + is_python_compat=False) def get_data(name): @@ -175,19 +186,24 @@ def generate_files(app_config): else: optional_entrypoint = '' - dockerfile = ''.join([ - get_data('Dockerfile.preamble.template').format( - base_image=app_config.base_image), - get_data('Dockerfile.virtualenv.template').format( - python_version=app_config.dockerfile_python_version), - optional_requirements_txt, - get_data('Dockerfile.install_app'), - optional_entrypoint, - ]) + if app_config.is_python_compat: + dockerfile = get_data('Dockerfile.python_compat') + dockerignore = get_data('dockerignore.python_compat') + else: + dockerfile = ''.join([ + get_data('Dockerfile.preamble.template').format( + base_image=app_config.base_image), + get_data('Dockerfile.virtualenv.template').format( + python_version=app_config.dockerfile_python_version), + optional_requirements_txt, + get_data('Dockerfile.install_app'), + optional_entrypoint, + ]) + dockerignore = get_data('dockerignore') return { 'Dockerfile': dockerfile, - '.dockerignore': get_data('dockerignore'), + '.dockerignore': dockerignore, } @@ -206,7 +222,7 @@ def generate_dockerfile_command(base_image, config_file, source_dir): # Determine complete configuration app_config = get_app_config(raw_config, base_image, config_file, - source_dir) + source_dir) # Generate list of filenames and their textual contents files = generate_files(app_config) diff --git a/scripts/gen_dockerfile_test.py b/scripts/gen_dockerfile_test.py index b52e15e8..03eaa079 100755 --- a/scripts/gen_dockerfile_test.py +++ b/scripts/gen_dockerfile_test.py @@ -54,6 +54,14 @@ def compare_file(filename, dir1, dir2): 'dockerfile_python_version': '', 'has_requirements_txt': False, 'entrypoint': '', + 'is_python_compat': False, + }), + ('env: flex\nruntime: python-compat', { + 'base_image': None, + 'dockerfile_python_version': None, + 'has_requirements_txt': None, + 'entrypoint': None, + 'is_python_compat': True, }), # All supported python versions ('runtime_config:\n python_version:', { @@ -74,6 +82,9 @@ def compare_file(filename, dir1, dir2): ('runtime_config:\n python_version: 3.6', { 'dockerfile_python_version': '3.6', }), + ('runtime_config:\n python_version: 3.7', { + 'dockerfile_python_version': '3.7', + }), # entrypoint present ('entrypoint: my entrypoint', { 'entrypoint': 'exec my entrypoint', @@ -125,7 +136,8 @@ def test_get_app_config_invalid(app_yaml): base_image='', dockerfile_python_version='', entrypoint='', - has_requirements_txt=False + has_requirements_txt=False, + is_python_compat=False, ) @@ -146,6 +158,9 @@ def test_get_app_config_invalid(app_yaml): # Python version (_BASE_APP_CONFIG._replace(dockerfile_python_version='_my_version'), True, 'python_version=python_my_version'), + # python-compat runtime + (_BASE_APP_CONFIG._replace(is_python_compat=True), True, + 'FROM gcr.io/google_appengine/python-compat-multicore'), ]) def test_generate_files(app_config, should_find, test_string): result = gen_dockerfile.generate_files(app_config) @@ -163,10 +178,13 @@ def compare_against_golden_files(app, config_dir, testdata_dir): compare_file(filename, config_dir, golden_dir) -def test_generate_dockerfile_command(tmpdir, testdata_dir): +@pytest.mark.parametrize('app', [ + # Sampled from https://github.com/GoogleCloudPlatform/python-docs-samples + 'hello_world', + # From an internal source. + 'hello_world_compat']) +def test_generate_dockerfile_command(tmpdir, testdata_dir, app): """Generates output and compares against a set of golden files.""" - # Sample from https://github.com/GoogleCloudPlatform/python-docs-samples - app = 'hello_world' app_dir = os.path.join(testdata_dir, app) # Copy sample app to writable temp dir, and generate Dockerfile. @@ -179,12 +197,15 @@ def test_generate_dockerfile_command(tmpdir, testdata_dir): compare_against_golden_files(app, config_dir, testdata_dir) +@pytest.mark.parametrize('app', [ + # Sampled from https://github.com/GoogleCloudPlatform/python-docs-samples + 'hello_world', + # From an internal source. + 'hello_world_compat']) @pytest.mark.xfail(not shutil.which('gcloud'), reason='Google Cloud SDK is not installed') -def test_generate_dockerfile_golden(tmpdir, testdata_dir): +def test_generate_dockerfile_golden(tmpdir, testdata_dir, app): """Validate our golden files against gcloud app gen-config""" - # Sample from https://github.com/GoogleCloudPlatform/python-docs-samples - app = 'hello_world' app_dir = os.path.join(testdata_dir, app) # Copy sample app to writable temp dir, and generate Dockerfile. diff --git a/scripts/integration-test.sh b/scripts/integration-test.sh new file mode 100644 index 00000000..6210e0a1 --- /dev/null +++ b/scripts/integration-test.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +set -ex + +export KOKORO_GITHUB_DIR=${KOKORO_ROOT}/src/github +source ${KOKORO_GFILE_DIR}/kokoro/common.sh + +export GOOGLE_CLOUD_PROJECT=gcp-runtimes + +sudo -E /usr/local/bin/pip install --upgrade -r ${KOKORO_GFILE_DIR}/appengine/integration_tests/requirements.txt + +if [ -f ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY}/requirements.txt ] +then + sudo -E /usr/local/bin/pip install --upgrade -r ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY}/requirements.txt +fi + +export GOPATH=${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY} + +flags="" + +if [ -n "${STAGING_IMAGE}" ]; then + flags="$flags -i ${STAGING_IMAGE}" +fi + +if [ "${SKIP_STANDARD_LOGGING_TESTS}" = "true" ]; then + flags="$flags --skip-standard-logging-tests" +fi + +if [ "${SKIP_CUSTOM_LOGGING_TESTS}" = "true" ]; then + flags="$flags --skip-custom-logging-tests" +fi + +if [ "${SKIP_MONITORING_TESTS}" = "true" ]; then + flags="$flags --skip-monitoring-tests" +fi + +if [ "${SKIP_EXCEPTION_TESTS}" = "true" ]; then + flags="$flags --skip-exception-tests" +fi + +if [ "${SKIP_CUSTOM_TESTS}" = "true" ]; then + flags="$flags --skip-custom-tests" +fi + +if [ -n "${URL}" ]; then + flags="$flags --url ${URL}" +fi + +if [ -n "${BUILDER}" ]; then + flags="$flags --builder ${BUILDER}" + gcloud config set app/use_runtime_builders True + gcloud config set app/runtime_builders_root file://${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY} +fi + +if [ -n "${YAML}" ]; then + flags="$flags --yaml ${KOKORO_GITHUB_DIR}/${YAML}" +fi + + +chmod a+x ${KOKORO_GFILE_DIR}/appengine/integration_tests/testsuite/driver.py +${KOKORO_GFILE_DIR}/appengine/integration_tests/testsuite/driver.py -d ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY} ${flags} diff --git a/scripts/local_cloudbuild.py b/scripts/local_cloudbuild.py index f80e0f4b..5c23a1bc 100755 --- a/scripts/local_cloudbuild.py +++ b/scripts/local_cloudbuild.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Emulate the Google Container Builder locally. +"""Emulate the Google Cloud Build locally. The input is a local cloudbuild.yaml file. This is translated into a series of commands for the locally installed Docker daemon. These @@ -48,8 +48,8 @@ # Exclude non-printable control characters (including newlines) PRINTABLE_REGEX = re.compile(r"""^[^\x00-\x1f]*$""") -# Container Builder substitutions -# https://cloud.google.com/container-builder/docs/api/build-requests#substitutions +# Cloud Build substitutions +# https://cloud.google.com/cloud-build/docs/api/build-requests#substitutions SUBSTITUTION_REGEX = re.compile(r"""(?x) [$] # Dollar sign ( diff --git a/scripts/release.sh b/scripts/release.sh new file mode 100644 index 00000000..cd499823 --- /dev/null +++ b/scripts/release.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set -euo pipefail +export KOKORO_GITHUB_DIR=${KOKORO_ROOT}/src/github +source ${KOKORO_GFILE_DIR}/kokoro/common.sh + +cd ${KOKORO_GITHUB_DIR}/python-runtime + +if [ -z "${TAG:+set}" ]; then + export TAG=$(date +%Y-%m-%d-%H%M%S) +fi + +./build.sh $BUILD_FLAGS + +METADATA=$(pwd)/METADATA +cd ${KOKORO_GFILE_DIR}/kokoro +python note.py python -m ${METADATA} -t ${TAG} diff --git a/scripts/requirements-test.txt b/scripts/requirements-test.txt index 5afb5315..c8e698da 100644 --- a/scripts/requirements-test.txt +++ b/scripts/requirements-test.txt @@ -1,4 +1,4 @@ -flask==0.12.2 -pytest==3.2.3 +flask==1.0.2 +pytest==3.7.3 pytest-cov==2.5.1 -pyyaml==3.12 +pyyaml==3.13 diff --git a/scripts/testdata/hello_world/requirements.txt b/scripts/testdata/hello_world/requirements.txt index bb84e50e..a34d076b 100644 --- a/scripts/testdata/hello_world/requirements.txt +++ b/scripts/testdata/hello_world/requirements.txt @@ -1,2 +1,2 @@ -Flask==0.12.2 -gunicorn==19.7.1 +Flask==1.0.2 +gunicorn==19.9.0 diff --git a/scripts/testdata/hello_world_compat/app.yaml b/scripts/testdata/hello_world_compat/app.yaml new file mode 100644 index 00000000..e514d42c --- /dev/null +++ b/scripts/testdata/hello_world_compat/app.yaml @@ -0,0 +1,13 @@ +service: default +runtime: python-compat +env: flex + +api_version: 1 +threadsafe: true + +beta_settings: + enable_app_engine_apis: true # Needed for compat apps. + +handlers: +- url: .* + script: main.app diff --git a/scripts/testdata/hello_world_compat/main.py b/scripts/testdata/hello_world_compat/main.py new file mode 100644 index 00000000..40302722 --- /dev/null +++ b/scripts/testdata/hello_world_compat/main.py @@ -0,0 +1,14 @@ +"""The hello world flex app!""" + +import webapp2 + + +class HelloHandler(webapp2.RequestHandler): + + def get(self): + msg = 'Hello GAE Flex (env: flex) Compat-Runtime App\n' + self.response.headers['Content-Type'] = 'text/plain' + self.response.out.write(msg) + +app = webapp2.WSGIApplication([('/', HelloHandler)], + debug=True) diff --git a/scripts/testdata/hello_world_compat_golden/.dockerignore b/scripts/testdata/hello_world_compat_golden/.dockerignore new file mode 100644 index 00000000..5ce5abfa --- /dev/null +++ b/scripts/testdata/hello_world_compat_golden/.dockerignore @@ -0,0 +1,5 @@ +.dockerignore +Dockerfile +.git +.hg +.svn diff --git a/scripts/testdata/hello_world_compat_golden/Dockerfile b/scripts/testdata/hello_world_compat_golden/Dockerfile new file mode 100644 index 00000000..1e4d6352 --- /dev/null +++ b/scripts/testdata/hello_world_compat_golden/Dockerfile @@ -0,0 +1,3 @@ +FROM gcr.io/google_appengine/python-compat-multicore +ADD . /app/ +RUN if [ -s requirements.txt ]; then pip install -r requirements.txt; fi diff --git a/tests/deploy_check/requirements.txt b/tests/deploy_check/requirements.txt index bb84e50e..a34d076b 100644 --- a/tests/deploy_check/requirements.txt +++ b/tests/deploy_check/requirements.txt @@ -1,2 +1,2 @@ -Flask==0.12.2 -gunicorn==19.7.1 +Flask==1.0.2 +gunicorn==19.9.0 diff --git a/tests/eventlet/requirements.txt b/tests/eventlet/requirements.txt index c33e53f2..1e2ea66b 100644 --- a/tests/eventlet/requirements.txt +++ b/tests/eventlet/requirements.txt @@ -1,10 +1,10 @@ click==6.7 enum-compat==0.0.2 -eventlet==0.21.0 -Flask==0.12.2 -greenlet==0.4.12 -gunicorn==19.7.1 +eventlet==0.24.1 +Flask==2.2.5 +greenlet==0.4.14 +gunicorn==19.9.0 itsdangerous==0.24 -Jinja2==2.9.6 -MarkupSafe==1.0 -Werkzeug==0.12.2 +Jinja2==2.10 +MarkupSafe==1.1.1 +Werkzeug==2.2.3 diff --git a/tests/google-cloud-python-system/.gitignore b/tests/google-cloud-python-system/.gitignore deleted file mode 100644 index 470715ec..00000000 --- a/tests/google-cloud-python-system/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -credentials.json -Dockerfile diff --git a/tests/google-cloud-python-system/Dockerfile.in b/tests/google-cloud-python-system/Dockerfile.in deleted file mode 100644 index 04f28bd5..00000000 --- a/tests/google-cloud-python-system/Dockerfile.in +++ /dev/null @@ -1,16 +0,0 @@ -FROM ${STAGING_IMAGE} - -# Get the source. -RUN git clone --depth 1 https://github.com/GoogleCloudPlatform/google-cloud-python.git -WORKDIR google-cloud-python - -# Install nox -RUN pip install --upgrade nox-automation - -# Secrets injected at runtime -ENV GOOGLE_APPLICATION_CREDENTIALS=/workspace/tests/google-cloud-python-system/credentials.json -ENV GOOGLE_CLOUD_PROJECT=${GOOGLE_CLOUD_PROJECT_FOR_TESTS} - -# Run system tests for all supported Python versions -ADD run_system_tests.sh /run_system_tests.sh -ENTRYPOINT ["/run_system_tests.sh"] diff --git a/tests/google-cloud-python-system/run_system_tests.sh b/tests/google-cloud-python-system/run_system_tests.sh deleted file mode 100755 index 17d3aac3..00000000 --- a/tests/google-cloud-python-system/run_system_tests.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/sh -set -eu - -cd /app/google-cloud-python - -# Not all packages have system tests -packages=" -bigquery -bigtable -datastore -language -logging -monitoring -pubsub -spanner -speech -storage -vision -" - -# translate has system test but it gives error message: -# BadRequest: 400 Invalid JSON payload received. Unknown name "model": Cannot bind 'nmt'. Field 'model' could not be found in request message. (GET https://translation.googleapis.com/language/translate/v2?target=de&q=hvala+ti&q=dankon&q=Me+llamo+Jeff&q=My+name+is+Jeff&model=nmt) -disabled_packages="translate" - -# Spanner system test needs this -export GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE=1 - -exit_code=0 -for package in ${packages}; do - noxfile="${package}/nox.py" - nox \ - -f "${noxfile}" \ - -e \ - "system_tests(python_version='2.7')" \ - "system_tests(python_version='3.6')" \ - || exit_code=1 -done - -exit "${exit_code}" diff --git a/tests/google-cloud-python/run_unit_tests.sh b/tests/google-cloud-python/run_unit_tests.sh index 81a6a389..c386f1c0 100755 --- a/tests/google-cloud-python/run_unit_tests.sh +++ b/tests/google-cloud-python/run_unit_tests.sh @@ -4,7 +4,13 @@ set -eu cd /app/google-cloud-python exit_code=0 +failed_files= for noxfile in */nox.py; do + if [ "${noxfile}" = "dlp/nox.py" ]; then + echo "**** Skipping ${noxfile} ****" + continue + fi + echo "**** Starting tests in ${noxfile} ****" nox \ -f "${noxfile}" \ -e \ @@ -12,7 +18,17 @@ for noxfile in */nox.py; do "unit(py='3.4')" \ "unit(py='3.5')" \ "unit(py='3.6')" \ - || exit_code=1 + || { + echo "**** FAILED tests in ${noxfile} ****" + exit_code=1 + failed_files="${failed_files} ${noxfile}" + } + echo "**** Finished tests in ${noxfile} ****" done +if [ "${exit_code}" -eq 0 ]; then + echo "**** All tests passed ****" +else + echo "**** There were test failures:${failed_files} ****" +fi exit "${exit_code}" diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index 3e51c81d..a0634bee 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -1,8 +1,9 @@ -Flask==0.12.2 -google-cloud-error-reporting==0.28.0 -google-cloud-logging==1.4.0 -google-cloud-monitoring==0.28.0 -gunicorn==19.7.1 -requests==2.18.4 +Flask==2.2.5 +google-cloud-error-reporting==0.32.1 +google-cloud-logging==1.12.1 +google-cloud-monitoring==0.33.0 +gunicorn==19.9.0 +requests==2.31.0 retrying==1.3.3 -six==1.11.0 +six==1.12.0 +protobuf>=3.6.0 diff --git a/tests/license-test/license-test.yaml b/tests/license-test/license-test.yaml index cb2d67ce..348fbc9b 100644 --- a/tests/license-test/license-test.yaml +++ b/tests/license-test/license-test.yaml @@ -1,6 +1,6 @@ schemaVersion: "1.0.0" -# See https://github.com/GoogleCloudPlatform/runtimes-common/blob/master/structure_tests/README.md#license-tests +# See https://github.com/GoogleCloudPlatform/container-structure-test/blob/master/README.md licenseTests: - debian: true files: [] diff --git a/tests/no-virtualenv/no-virtualenv.yaml b/tests/no-virtualenv/no-virtualenv.yaml index f43d08a5..4f6c3f48 100644 --- a/tests/no-virtualenv/no-virtualenv.yaml +++ b/tests/no-virtualenv/no-virtualenv.yaml @@ -22,7 +22,7 @@ commandTests: - name: "default python3.4 installation" command: ["which", "python3.4"] - expectedOutput: ["/usr/bin/python3.4\n"] + expectedOutput: ["/opt/python3.4/bin/python3.4\n"] - name: "default python3.5 installation" command: ["which", "python3.5"] @@ -37,6 +37,17 @@ commandTests: command: ["which", "gunicorn"] expectedOutput: ["/usr/local/bin/gunicorn\n"] + - # Regression test for issue187 + name: "default python3 installation" + command: ["which", "python3"] + expectedOutput: ["/usr/local/bin/python3\n"] + - name: "default python3 version" + command: ["python3", "--version"] + expectedOutput: ["Python 3.7.9\n"] + - name: "default pip3 installation" + command: ["which", "pip3"] + expectedOutput: ["/usr/local/bin/pip3\n"] + - name: "default flask installation" # Checks that 'pip' and 'python' are using the same Python version setup: [["pip", "install", "flask"]] diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 8c134e48..f0596d67 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -1,207 +1,207 @@ -alembic==0.9.6 -amqp==2.2.2 +alembic==1.0.0 +amqp==2.3.2 amqplib==1.0.2 -ansible==2.4.1.0 +ansible==7.0.0 anyjson==0.3.3 -apache-libcloud==2.2.1 +apache-libcloud==2.3.0 argparse==1.4.0 -astroid==1.5.3 -awscli==1.11.180 -babel==2.5.1 +astroid==1.6.5 +awscli==1.16.1 +babel==2.6.0 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 -beautifulsoup4==4.6.0 +beautifulsoup4==4.6.3 beautifulsoup==3.2.1 -billiard==3.5.0.3 -blessings==1.6 +billiard==3.5.0.4 +blessings==1.7 blinker==1.4 -boto==2.48.0 -botocore==1.7.38 +boto==2.49.0 +botocore==1.11.1 bottle==0.12.13 -carbon==1.0.2 -celery==4.1.0 -certifi==2017.7.27.1 -cffi==1.11.2 +carbon<1.1.1 +celery==4.2.1 +certifi==2023.7.22 +cffi==1.11.5 chardet==3.0.4 click==6.7 -cliff==2.9.1 -cmd2==0.7.7 +cliff==2.13.0 +cmd2==0.8.9 colorama==0.3.9 configobj==5.0.6 cov-core==1.15.0 -coverage==4.4.1 -coveralls==1.2.0 +coverage==4.5.1 +coveralls==1.4.0 crcmod==1.7 -cryptography==2.1.3 -cssselect==1.0.1 -cython==0.27.2 -decorator==4.1.2 -django-celery==3.2.1 -django-debug-toolbar==1.8 -django-extensions==1.9.7 -django==1.11.7 +cryptography==2.3.1 +cssselect==1.0.3 +cython==0.28.5 +decorator==4.3.0 +django-celery==3.2.2 +django-debug-toolbar==1.9.1 +django-extensions==2.1.1 +django<2.0 django_compress==1.0.1 -djangorestframework==3.7.1 +djangorestframework==3.8.2 docker-py==1.10.6 docopt==0.6.2 docutils==0.14 ecdsa==0.13 -elasticsearch==5.4.0 +elasticsearch==6.3.1 enum34==1.1.6 -eventlet==0.21.0 +eventlet==0.24.1 extras==1.0.0 -fabric==1.14.0 +fabric==2.3.1 fixtures==3.0.0 flake8==3.5.0 -flask==0.12.2 +flask==2.2.5 funcsigs==1.0.2 functools32==3.2.3.post2 -futures==3.1.1 -gevent==1.2.2 -google-api-python-client==1.6.4 -graphite-web==1.0.2 -greenlet==0.4.12 -gunicorn==19.7.1 +futures==3.2.0 +gevent==1.3.6 +google-api-python-client==1.7.4 +graphite-web==1.1.3 +greenlet==0.4.14 +gunicorn==19.9.0 hiredis==0.2.0 honcho==1.0.1 -html5lib==0.999999999 -httplib2==0.10.3 -idna==2.6 -ipaddress==1.0.18 +html5lib==1.0.1 +httplib2==0.11.3 +idna==2.7 +ipaddress==1.0.22 iso8601==0.1.12 isodate==0.6.0 itsdangerous==0.24 -jinja2==2.9.6 +jinja2==2.10 jmespath==0.9.3 jsonschema==2.6.0 -kombu==4.1.0 +kombu==4.2.1 linecache2==1.0.0 -logilab-common==1.4.1 -lxml==4.1.0 -m2crypto==0.27.0 +logilab-common==1.4.2 +lxml==4.2.4 +m2crypto==0.30.1 mako==1.0.7 manifestparser==1.1 -markdown==2.6.9 +markdown==2.6.11 markupsafe==1.0 -matplotlib==2.1.0 +matplotlib==2.2.3 mccabe==0.6.1 meld3==1.0.2 mock==2.0.0 mozcrash==1.0 -mozdevice==0.50 +mozdevice==1.0.1 mozfile==1.2 mozinfo==0.10 -mozlog==3.5 +mozlog==3.8 moznetwork==0.27 -mozprocess==0.25 -mozprofile==0.28 -mozrunner==6.13 -msgpack-python==0.4.8 +mozprocess==0.26 +mozprofile==1.1.0 +mozrunner==7.0.1 +msgpack-python==0.5.6 mysql-python==1.2.5 -ndg-httpsclient==0.4.3 +ndg-httpsclient==0.5.1 netaddr==0.7.19 -netifaces==0.10.6 -newrelic==2.96.0.80 +netifaces==0.10.7 +newrelic==4.2.0.100 nose==1.3.7 -numpy==1.13.3 +numpy==1.22.0 oauth2==1.9.0.post1 oauth2client==4.1.2 -oauthlib==2.0.6 +oauthlib==2.1.0 ordereddict==1.1 -oslo.config==5.0.0 -pandas==0.21.0 -paramiko==2.3.1 +oslo.config==6.4.0 +pandas==0.23.4 +paramiko==2.4.1 passlib==1.7.1 paste==2.0.3 pastedeploy==1.5.2 pastescript==2.0.2 -pbr==3.1.1 +pbr==4.2.0 pep8==1.7.1 -pexpect==4.2.1 -pika==0.11.0 -pillow==4.3.0 -pip==9.0.1 -prettytable -protobuf==3.4.0 -psutil==5.4.0 -psycopg2==2.7.3.2 -py==1.4.34 -pyasn1-modules==0.1.5 -pyasn1==0.3.7 +pexpect==4.6.0 +pika==0.12.0 +pillow==9.3.0 +pip==18.0 +prettytable==0.7.2 +protobuf==3.6.1 +psutil==5.4.7 +psycopg2==2.7.5 +py==1.5.4 +pyasn1-modules==0.2.2 +pyasn1==0.4.4 pycparser==2.18 pycrypto==2.6.1 -pycurl==7.43.0 -pyflakes==1.6.0 +pycurl==7.43.0.2 +pyflakes==2.0.0 pygments==2.2.0 -pyjwt==1.5.3 +pyjwt==1.6.4 pylibmc==1.5.2 -pylint==1.7.4 -pymongo==3.5.1 -pymysql==0.7.11 -pyopenssl==17.3.0 +pylint==1.9.3 +pymongo==3.7.1 +pymysql==0.9.2 +pyopenssl==18.0.0 pyparsing==2.2.0 -pyramid==1.9.1 +pyramid==1.9.2 pystache==0.5.4 pytest-cov==2.5.1 -pytest==3.2.3 +pytest==3.7.3 python-cjson==1.2.1 -python-daemon==2.1.2 -python-dateutil==2.6.1 +python-daemon==2.2.0 +python-dateutil==2.7.3 python-gflags==3.1.2 -python-keystoneclient==3.13.0 -python-memcached==1.58 +python-keystoneclient==3.17.0 +python-memcached==1.59 python-mimeparse==1.6.0 -python-novaclient==9.1.0 -python-subunit==1.2.0 -python-swiftclient==3.4.0 -pytz==2017.3 -pyyaml==3.12 -pyzmq==16.0.3 -raven==6.3.0 +python-novaclient==11.0.0 +python-subunit==1.3.0 +python-swiftclient==3.6.0 +pytz==2018.5 +pyyaml==5.4 +pyzmq==17.1.2 +raven==6.9.0 redis==2.10.6 repoze.lru==0.7 -requests-oauthlib==0.8.0 -requests==2.18.4 +requests-oauthlib==1.0.0 +requests==2.31.0 retrying==1.3.3 rsa==3.4.2 -scipy==1.0.0 -selenium==3.7.0 +scipy==1.10.0 +selenium==3.14.0 setuptools-git==1.2 -setuptools==36.6.0 +setuptools==40.2.0 sh==1.12.14 -simplejson==3.11.1 +simplejson==3.16.0 six==1.11.0 snowballstemmer==1.2.1 south==1.0.2 -sphinx==1.6.5 +sphinx==1.7.7 sqlalchemy-migrate==0.11.0 -sqlalchemy==1.1.14 -sqlparse==0.2.4 -statsd==3.2.1 -stevedore==1.27.1 +sqlalchemy==1.2.11 +sqlparse==0.4.4 +statsd==3.3.0 +stevedore==1.29.0 suds==0.4 -supervisor==3.3.3 +supervisor==3.3.4 testrepository==0.0.20 testtools==2.3.0 -thrift==0.10.0 -tornado==4.5.2 -tox==2.9.1 -twisted==17.9.0 +thrift==0.11.0 +tornado==6.3.3 +tox==3.2.1 +twisted==18.7.0 ujson==1.35 -unidecode==0.4.21 +unidecode==1.0.22 unittest2==1.1.0 uritemplate==3.0.0 -urllib3==1.22 -uwsgi==2.0.15 +urllib3==1.26.5 +uwsgi==2.0.22 versiontools==1.9.1 -virtualenv==15.1.0 -waitress==1.1.0 +virtualenv==16.0.0 +waitress==2.1.2 warlock==1.3.0 -webob==1.7.3 -websocket-client==0.44.0 -webtest==2.0.29 -werkzeug==0.12.2 -wheel==0.30.0 +webob==1.8.2 +websocket-client==0.51.0 +webtest==2.0.30 +werkzeug==2.2.3 +wheel==0.31.1 xlrd==1.1.0 -zc.buildout==2.9.5 -zope.interface==4.4.3 +zc.buildout==2.12.1 +zope.interface==4.5.0 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 2bdcc565..23175a06 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -1,196 +1,195 @@ -alembic==0.9.6 -amqp==2.2.2 +alembic==1.0.0 +amqp==2.3.2 amqplib==1.0.2 -ansible==2.4.1.0 +ansible==7.0.0 anyjson==0.3.3 -apache-libcloud==2.2.1 +apache-libcloud==2.3.0 argparse==1.4.0 -astroid==1.5.3 -awscli==1.11.180 -babel==2.5.1 +astroid==2.0.4 +awscli==1.16.1 +babel==2.6.0 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 -beautifulsoup4==4.6.0 -billiard==3.5.0.3 -blessings==1.6 +beautifulsoup4==4.6.3 +billiard==3.5.0.4 +blessings==1.7 blinker==1.4 -boto==2.48.0 -botocore==1.7.38 +boto==2.49.0 +botocore==1.11.1 bottle==0.12.13 -celery==4.1.0 -certifi==2017.7.27.1 -cffi==1.11.2 +celery==4.2.1 +certifi==2023.7.22 +cffi==1.11.5 chardet==3.0.4 click==6.7 -cliff==2.9.1 -cmd2==0.7.7 +cliff==2.13.0 +cmd2==0.9.4 colorama==0.3.9 configobj==5.0.6 cov-core==1.15.0 -coverage==4.4.1 -coveralls==1.2.0 +coverage==4.5.1 +coveralls==1.4.0 crcmod==1.7 -cryptography==2.1.3 -cssselect==1.0.1 -cython==0.27.2 -decorator==4.1.2 -django-celery==3.2.1 -django-debug-toolbar==1.8 -django-extensions==1.9.7 -django==1.11.7 +cryptography==41.0.3 +cssselect==1.0.3 +cython==0.28.5 +decorator==4.3.0 +django-celery==3.2.2 +django-debug-toolbar==1.9.1 +django-extensions==2.1.1 +django==2.2.28 django_compress==1.0.1 -djangorestframework==3.7.1 +djangorestframework==3.8.2 docker-py==1.10.6 docopt==0.6.2 docutils==0.14 ecdsa==0.13 -elasticsearch==5.4.0 +elasticsearch==6.3.1 enum34==1.1.6 -eventlet==0.21.0 +eventlet==0.24.1 extras==1.0.0 -fabric==1.14.0 +fabric==2.3.1 fixtures==3.0.0 flake8==3.5.0 -flask==0.12.2 +flask==2.2.5 funcsigs==1.0.2 -futures==3.1.1 -gevent==1.2.2 -google-api-python-client==1.6.4 -greenlet==0.4.12 -gunicorn==19.7.1 +gevent==1.3.6 +google-api-python-client==1.7.4 +greenlet==0.4.14 +gunicorn==19.9.0 hiredis==0.2.0 honcho==1.0.1 -html5lib==0.999999999 -httplib2==0.10.3 -idna==2.6 -ipaddress==1.0.18 -ipython==6.2.1 +html5lib==1.0.1 +httplib2==0.11.3 +idna==2.7 +ipaddress==1.0.22 +ipython==6.5.0 iso8601==0.1.12 isodate==0.6.0 itsdangerous==0.24 -jinja2==2.9.6 +jinja2==2.10 jmespath==0.9.3 jsonschema==2.6.0 -kombu==4.1.0 +kombu==4.2.1 linecache2==1.0.0 -logilab-common==1.4.1 -lxml==4.1.0 +logilab-common==1.4.2 +lxml==4.2.4 mako==1.0.7 manifestparser==1.1 -markdown==2.6.9 -markupsafe==1.0 -matplotlib==2.1.0 +markdown==2.6.11 +markupsafe==1.1.1 +matplotlib==2.2.3 mccabe==0.6.1 meld3==1.0.2 mock==2.0.0 mozcrash==1.0 -mozdevice==0.50 +mozdevice==1.0.1 mozfile==1.2 mozinfo==0.10 -mozlog==3.5 +mozlog==3.8 moznetwork==0.27 -mozprocess==0.25 -msgpack-python==0.4.8 -ndg-httpsclient==0.4.3 +mozprocess==0.26 +msgpack-python==0.5.6 +ndg-httpsclient==0.5.1 netaddr==0.7.19 -netifaces==0.10.6 -newrelic==2.96.0.80 +netifaces==0.10.7 +newrelic==4.2.0.100 nose==1.3.7 -numpy==1.13.3 +numpy==1.22.0 oauth2==1.9.0.post1 oauth2client==4.1.2 -oauthlib==2.0.6 +oauthlib==2.1.0 ordereddict==1.1 -oslo.config==5.0.0 -pandas==0.21.0 -paramiko==2.3.1 +oslo.config==6.4.0 +pandas==0.23.4 +paramiko==2.4.1 passlib==1.7.1 paste==2.0.3 pastedeploy==1.5.2 pastescript==2.0.2 -pbr==3.1.1 +pbr==4.2.0 pep8==1.7.1 -pexpect==4.2.1 -pika==0.11.0 -pillow==4.3.0 -pip==9.0.1 -prettytable -protobuf==3.4.0 -psutil==5.4.0 -psycopg2==2.7.3.2 -py==1.4.34 -pyasn1-modules==0.1.5 -pyasn1==0.3.7 +pexpect==4.6.0 +pika==0.12.0 +pillow==9.3.0 +pip==18.0 +prettytable==0.7.2 +protobuf==3.6.1 +psutil==5.4.7 +psycopg2==2.7.5 +py==1.5.4 +pyasn1-modules==0.2.2 +pyasn1==0.4.4 pycparser==2.18 pycrypto==2.6.1 -pyflakes==1.6.0 -pygments==2.2.0 -pyjwt==1.5.3 +pyflakes==2.0.0 +pygments==2.15.0 +pyjwt==1.6.4 pylibmc==1.5.2 -pylint==1.7.4 -pymongo==3.5.1 -pymysql==0.7.11 -pyopenssl==17.3.0 +pylint==2.1.1 +pymongo==3.7.1 +pymysql==0.9.2 +pyopenssl==18.0.0 pyparsing==2.2.0 -pyramid==1.9.1 +pyramid==1.9.2 pystache==0.5.4 pytest-cov==2.5.1 -pytest==3.2.3 -python-daemon==2.1.2 -python-dateutil==2.6.1 +pytest==3.7.3 +python-daemon==2.2.0 +python-dateutil==2.7.3 python-gflags==3.1.2 -python-keystoneclient==3.13.0 -python-memcached==1.58 +python-keystoneclient==3.17.0 +python-memcached==1.59 python-mimeparse==1.6.0 -python-novaclient==9.1.0 -python-subunit==1.2.0 -python-swiftclient==3.4.0 -pytz==2017.3 -pyyaml==3.12 -pyzmq==16.0.3 -raven==6.3.0 +python-novaclient==11.0.0 +python-subunit==1.3.0 +python-swiftclient==3.6.0 +pytz==2018.5 +pyyaml==5.4 +pyzmq==17.1.2 +raven==6.9.0 redis==2.10.6 repoze.lru==0.7 -requests-oauthlib==0.8.0 -requests==2.18.4 +requests-oauthlib==1.0.0 +requests==2.31.0 retrying==1.3.3 rsa==3.4.2 -scipy==1.0.0 -selenium==3.7.0 +scipy==1.10.0 +selenium==3.14.0 setuptools-git==1.2 -setuptools==36.6.0 +setuptools==40.2.0 sh==1.12.14 -simplejson==3.11.1 +simplejson==3.16.0 six==1.11.0 snowballstemmer==1.2.1 south==1.0.2 -sphinx==1.6.5 +sphinx==1.7.7 sqlalchemy-migrate==0.11.0 -sqlalchemy==1.1.14 -sqlparse==0.2.4 -statsd==3.2.1 -stevedore==1.27.1 +sqlalchemy==1.2.11 +sqlparse==0.4.4 +statsd==3.3.0 +stevedore==1.29.0 testrepository==0.0.20 testtools==2.3.0 -thrift==0.10.0 -tornado==4.5.2 -tox==2.9.1 -twisted==17.9.0 +thrift==0.11.0 +tornado==6.3.3 +tox==3.2.1 +twisted==18.7.0 ujson==1.35 -unidecode==0.4.21 +unidecode==1.0.22 unittest2==1.1.0 uritemplate==3.0.0 -urllib3==1.22 -uwsgi==2.0.15 +urllib3==1.26.5 +uwsgi==2.0.22 versiontools==1.9.1 -virtualenv==15.1.0 -waitress==1.1.0 +virtualenv==16.0.0 +waitress==2.1.2 warlock==1.3.0 -webob==1.7.3 -websocket-client==0.44.0 -webtest==2.0.29 -werkzeug==0.12.2 -wheel==0.30.0 +webob==1.8.2 +websocket-client==0.51.0 +webtest==2.0.30 +werkzeug==2.2.3 +wheel==0.31.1 xlrd==1.1.0 -zc.buildout==2.9.5 -zope.interface==4.4.3 +zc.buildout==2.12.1 +zope.interface==4.5.0 diff --git a/tests/virtualenv/virtualenv_default.yaml b/tests/virtualenv/virtualenv_default.yaml index 60714de6..6b6ad282 100644 --- a/tests/virtualenv/virtualenv_default.yaml +++ b/tests/virtualenv/virtualenv_default.yaml @@ -17,7 +17,7 @@ commandTests: command: ["python", "--version"] # we check stderr instead of stdout for Python versions < 3.4 # https://bugs.python.org/issue18338 - expectedError: ["Python 2.7.9\n"] + expectedError: ["Python 2.7.(9|12)\n"] - name: "virtualenv pip installation" setup: [["virtualenv", "/env"]] @@ -34,4 +34,4 @@ commandTests: setup: [["virtualenv", "/env"], ["pip", "install", "flask"]] command: ["python", "-c", "import flask; print(flask.__file__)"] - expectedOutput: ["/env/local/lib/python2.7/site-packages/flask"] + expectedOutput: ["/env/lib/python2.7/site-packages/flask/__init__.pyc"] diff --git a/tests/virtualenv/virtualenv_python27.yaml b/tests/virtualenv/virtualenv_python27.yaml index d28ea541..09b78480 100644 --- a/tests/virtualenv/virtualenv_python27.yaml +++ b/tests/virtualenv/virtualenv_python27.yaml @@ -27,7 +27,7 @@ commandTests: command: ["python", "--version"] # we check stderr instead of stdout for Python versions < 3.4 # https://bugs.python.org/issue18338 - expectedError: ["Python 2.7.9\n"] + expectedError: ["Python 2.7.(9|12)\n"] - name: "virtualenv27 pip installation" setup: [["virtualenv", "-p", "python", "/env"]] @@ -44,4 +44,4 @@ commandTests: setup: [["virtualenv", "-p", "python", "/env"], ["pip", "install", "flask"]] command: ["python", "-c", "import flask; print(flask.__file__)"] - expectedOutput: ["/env/local/lib/python2.7/site-packages/flask"] + expectedOutput: ["/env/lib/python2.7/site-packages/flask/__init__.pyc"] diff --git a/tests/virtualenv/virtualenv_python34.yaml b/tests/virtualenv/virtualenv_python34.yaml index 779f4d45..9b5b77d0 100644 --- a/tests/virtualenv/virtualenv_python34.yaml +++ b/tests/virtualenv/virtualenv_python34.yaml @@ -25,7 +25,7 @@ commandTests: - name: "virtualenv34 python version" setup: [["virtualenv", "-p", "python3.4", "/env"]] command: ["python", "--version"] - expectedOutput: ["Python 3.4.2\n"] + expectedOutput: ["Python 3.4.8\n"] - name: "virtualenv34 pip installation" setup: [["virtualenv", "-p", "python3.4", "/env"]] @@ -47,7 +47,7 @@ commandTests: setup: [["virtualenv", "-p", "python3.4", "/env"], ["pip", "install", "flask"]] command: ["python", "-c", "import flask; print(flask.__file__)"] - expectedOutput: ["/env/lib/python3.4/site-packages/flask"] + expectedOutput: ["/env/lib/python3.4/site-packages/flask/__init__.py"] - name: "virtualenv34 test.support availability" setup: [["virtualenv", "-p", "python3.4", "/env"]] diff --git a/tests/virtualenv/virtualenv_python35.yaml b/tests/virtualenv/virtualenv_python35.yaml index 4150ba19..5e4b394a 100644 --- a/tests/virtualenv/virtualenv_python35.yaml +++ b/tests/virtualenv/virtualenv_python35.yaml @@ -25,7 +25,7 @@ commandTests: - name: "virtualenv35 python version" setup: [["virtualenv", "-p", "python3.5", "/env"]] command: ["python", "--version"] - expectedOutput: ["Python 3.5.4\n"] + expectedOutput: ["Python 3.5.9\n"] - name: "virtualenv35 pip installation" setup: [["virtualenv", "-p", "python3.5", "/env"]] @@ -47,7 +47,7 @@ commandTests: setup: [["virtualenv", "-p", "python3.5", "/env"], ["pip", "install", "flask"]] command: ["python", "-c", "import flask; print(flask.__file__)"] - expectedOutput: ["/env/lib/python3.5/site-packages/flask"] + expectedOutput: ["/env/lib/python3.5/site-packages/flask/__init__.py"] - name: "virtualenv35 test.support availability" setup: [["virtualenv", "-p", "python3.5", "/env"]] diff --git a/tests/virtualenv/virtualenv_python36.yaml b/tests/virtualenv/virtualenv_python36.yaml index 71c91a9f..b3a9e68e 100644 --- a/tests/virtualenv/virtualenv_python36.yaml +++ b/tests/virtualenv/virtualenv_python36.yaml @@ -25,7 +25,7 @@ commandTests: - name: "virtualenv36 python version" setup: [["virtualenv", "-p", "python3.6", "/env"]] command: ["python", "--version"] - expectedOutput: ["Python 3.6.2\n"] + expectedOutput: ["Python 3.6.10\n"] - name: "virtualenv36 pip installation" setup: [["virtualenv", "-p", "python3.6", "/env"]] @@ -47,7 +47,7 @@ commandTests: setup: [["virtualenv", "-p", "python3.6", "/env"], ["pip", "install", "flask"]] command: ["python", "-c", "import flask; print(flask.__file__)"] - expectedOutput: ["/env/lib/python3.6/site-packages/flask"] + expectedOutput: ["/env/lib/python3.6/site-packages/flask/__init__.py"] - name: "virtualenv36 test.support availability" setup: [["virtualenv", "-p", "python3.6", "/env"]] diff --git a/tests/virtualenv/virtualenv_python37.yaml b/tests/virtualenv/virtualenv_python37.yaml new file mode 100644 index 00000000..9810c78e --- /dev/null +++ b/tests/virtualenv/virtualenv_python37.yaml @@ -0,0 +1,54 @@ +schemaVersion: "1.0.0" + +globalEnvVars: + - key: "VIRTUAL_ENV" + value: "/env" + - key: "PATH" + value: "/env/bin:$PATH" + +commandTests: + - name: "virtualenv37 python installation" + setup: [["virtualenv", "-p", "python3.7", "/env"]] + command: ["which", "python"] + expectedOutput: ["/env/bin/python\n"] + + - name: "virtualenv37 python3 installation" + setup: [["virtualenv", "-p", "python3.7", "/env"]] + command: ["which", "python3"] + expectedOutput: ["/env/bin/python3\n"] + + - name: "virtualenv37 python3.7 installation" + setup: [["virtualenv", "-p", "python3.7", "/env"]] + command: ["which", "python3.7"] + expectedOutput: ["/env/bin/python3.7\n"] + + - name: "virtualenv37 python version" + setup: [["virtualenv", "-p", "python3.7", "/env"]] + command: ["python", "--version"] + expectedOutput: ["Python 3.7.9\n"] + + - name: "virtualenv37 pip installation" + setup: [["virtualenv", "-p", "python3.7", "/env"]] + command: ["which", "pip"] + expectedOutput: ["/env/bin/pip\n"] + + - name: "virtualenv37 pip3 installation" + setup: [["virtualenv", "-p", "python3.7", "/env"]] + command: ["which", "pip3"] + expectedOutput: ["/env/bin/pip3\n"] + + - name: "virtualenv37 gunicorn installation" + setup: [["virtualenv", "-p", "python3.7", "/env"], + ["pip", "install", "gunicorn"]] + command: ["which", "gunicorn"] + expectedOutput: ["/env/bin/gunicorn"] + + - name: "virtualenv37 flask installation" + setup: [["virtualenv", "-p", "python3.7", "/env"], + ["pip", "install", "flask"]] + command: ["python", "-c", "import flask; print(flask.__file__)"] + expectedOutput: ["/env/lib/python3.7/site-packages/flask/__init__.py"] + + - name: "virtualenv37 test.support availability" + setup: [["virtualenv", "-p", "python3.7", "/env"]] + command: ["python", "-c", "\"from test import pystone, regrtest, support\""]