From 8e35a5afd7cc9a6406873742129df12ab5e88909 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Tue, 7 Feb 2017 16:22:53 -0800 Subject: [PATCH 001/256] Fix typo in structure test --- tests/virtualenv/virtualenv_default.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/virtualenv/virtualenv_default.yaml b/tests/virtualenv/virtualenv_default.yaml index e9187f13..314a3d0c 100644 --- a/tests/virtualenv/virtualenv_default.yaml +++ b/tests/virtualenv/virtualenv_default.yaml @@ -17,7 +17,7 @@ commandTests: - name: "virtualenv installation" setup: [["virtualenv", "/env"]] command: ["which", "python"] - expectedOutput": ["/env/bin/python\n"] + expectedOutput: ["/env/bin/python\n"] - name: "python version" command: ["python", "--version"] From fccdcd65ff5791d340f830dc3e5ada06a6aa7eb1 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Tue, 7 Feb 2017 16:28:01 -0800 Subject: [PATCH 002/256] Switch from 'gcloud alpha container' to 'gcloud beta container' --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 9db5dc93..080b9d8d 100644 --- a/Makefile +++ b/Makefile @@ -29,7 +29,7 @@ ext_run.sh: .PHONY: cloud-build cloud-build: cloudbuild.yaml tests/google-cloud-python/Dockerfile - gcloud alpha container builds create . --config=cloudbuild.yaml + gcloud beta container builds submit . --config=cloudbuild.yaml .PHONY: cloud-test # structure-tests and google-cloud-python-tests are implicit in cloud-build From bc562810d198a44d764e3f054ac3fcb5261517ef Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Tue, 7 Feb 2017 12:37:26 -0800 Subject: [PATCH 003/256] Move main Dockerfile and supporting files to a subdirectory. Partially addresses #57 --- .gitignore | 1 - Makefile | 2 +- cloudbuild.yaml.in | 6 +++--- python-interpreter-builder/Makefile | 2 +- runtime-image/.gitignore | 1 + Dockerfile => runtime-image/Dockerfile | 0 {resources => runtime-image/resources}/apt-packages.txt | 0 {scripts => runtime-image/scripts}/install-apt-packages.sh | 0 8 files changed, 6 insertions(+), 6 deletions(-) create mode 100644 runtime-image/.gitignore rename Dockerfile => runtime-image/Dockerfile (100%) rename {resources => runtime-image/resources}/apt-packages.txt (100%) rename {scripts => runtime-image/scripts}/install-apt-packages.sh (100%) diff --git a/.gitignore b/.gitignore index 2c260f13..05626c51 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,2 @@ cloudbuild.yaml ext_run.sh -interpreters.tar.gz diff --git a/Makefile b/Makefile index 080b9d8d..395d43b6 100644 --- a/Makefile +++ b/Makefile @@ -39,7 +39,7 @@ cloud-test: cloud-build integration-tests .PHONY: local-build local-build: local-build-interpreters - docker build $(DOCKER_FLAGS) -t "$(IMAGE_NAME)" . + docker build $(DOCKER_FLAGS) -t "$(IMAGE_NAME)" runtime-image .PHONY: local-build-interpreters local-build-interpreters: diff --git a/cloudbuild.yaml.in b/cloudbuild.yaml.in index b8ed6baa..4c505105 100644 --- a/cloudbuild.yaml.in +++ b/cloudbuild.yaml.in @@ -1,11 +1,11 @@ timeout: 7200s steps: - name: gcr.io/cloud-builders/docker - args: ['build', '--tag=interpreter', '--no-cache', 'python-interpreter-builder'] + args: ['build', '--tag=interpreter', '--no-cache', '/workspace/python-interpreter-builder/'] - name: interpreter - args: ['cp', '/interpreters.tar.gz', '/workspace/'] + args: ['cp', '/interpreters.tar.gz', '/workspace/runtime-image/interpreters.tar.gz'] - name: gcr.io/cloud-builders/docker - args: ['build', '--tag=${IMAGE_NAME}', '--no-cache', '.'] + args: ['build', '--tag=${IMAGE_NAME}', '--no-cache', '/workspace/runtime-image/'] - name: gcr.io/gcp-runtimes/structure_test args: [ '-i', '${IMAGE_NAME}', diff --git a/python-interpreter-builder/Makefile b/python-interpreter-builder/Makefile index 6acfe6bb..d125b1c8 100644 --- a/python-interpreter-builder/Makefile +++ b/python-interpreter-builder/Makefile @@ -6,5 +6,5 @@ build: -docker rm python-interpreter-builder docker run --name python-interpreter-builder google/python-interpreter-builder /bin/bash mkdir -p output - docker cp python-interpreter-builder:/interpreters.tar.gz ../interpreters.tar.gz + docker cp python-interpreter-builder:/interpreters.tar.gz ../runtime-image/interpreters.tar.gz docker rm python-interpreter-builder diff --git a/runtime-image/.gitignore b/runtime-image/.gitignore new file mode 100644 index 00000000..cb1afebc --- /dev/null +++ b/runtime-image/.gitignore @@ -0,0 +1 @@ +interpreters.tar.gz diff --git a/Dockerfile b/runtime-image/Dockerfile similarity index 100% rename from Dockerfile rename to runtime-image/Dockerfile diff --git a/resources/apt-packages.txt b/runtime-image/resources/apt-packages.txt similarity index 100% rename from resources/apt-packages.txt rename to runtime-image/resources/apt-packages.txt diff --git a/scripts/install-apt-packages.sh b/runtime-image/scripts/install-apt-packages.sh similarity index 100% rename from scripts/install-apt-packages.sh rename to runtime-image/scripts/install-apt-packages.sh From 7e5075618b578161eb15c37eb5c1afbcaf4bf431 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Tue, 7 Feb 2017 16:32:26 -0800 Subject: [PATCH 004/256] Add tool to run cloudbuild.yaml steps locally. Supports just enough Container Builder functionality to build this image. --- local-cloudbuild.py | 148 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 148 insertions(+) create mode 100755 local-cloudbuild.py diff --git a/local-cloudbuild.py b/local-cloudbuild.py new file mode 100755 index 00000000..31a2b903 --- /dev/null +++ b/local-cloudbuild.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python +# +# Run the steps of cloudbuild.yaml locally, emulating Google Container +# Builder functionality. Does not actually push images to the +# Container Registry. Not all functionality is supported. +# +# Based on https://cloud.google.com/container-builder/docs/api/build-steps + +# System packages +import argparse +import getpass +import os +import shutil +import subprocess +import sys +import tempfile + +# Third party packages +import yaml + + +# Types +class CloudBuildError(Exception): + pass + + +def main(argv): + """Main entrypoint for cli""" + parser = argparse.ArgumentParser( + description='Process cloudbuild.yaml locally to build Docker images') + parser.add_argument( + 'cloudbuild', type=str, help='Path to cloudbuild.yaml input file') + parser.add_argument( + '--keep_workspace', + type=bool, + default=False, + help='Retain workspace directory after building') + args = parser.parse_args(argv[1:]) + + # Load and parse cloudbuild.yaml + cloudbuild = None + with open(args.cloudbuild, 'rb') as infile: + cloudbuild = yaml.safe_load(infile) + + host_workspace_parent = tempfile.mkdtemp(prefix='local-cloudbuild_%s_' % + getpass.getuser()) + host_workspace = os.path.join(host_workspace_parent, 'workspace') + try: + # Prepare workspace + shutil.copytree('.', host_workspace, symlinks=True) + + # Execute a series of 'docker run' commands locally + print('Running cloudbuild locally. Host workspace directory is %s' % + host_workspace) + run_steps(cloudbuild, host_workspace) + finally: + if not args.keep_workspace: + shutil.rmtree(host_workspace_parent, ignore_errors=True) + + +def run_steps(cloudbuild, host_workspace): + """Run the steps listed in a cloudbuild.yaml file. + + Args: + cloudbuild (dict): The decoded contents of a cloudbuild.yaml + host_workspace (str): Scratch directory + + Raises: + CloudBuildError if the yaml contents are invalid + """ + steps = cloudbuild.get('steps', {}) + if not steps: + raise CloudBuildError('No steps defined in cloudbuild.yaml') + + for step in steps: + run_one_step(step, host_workspace) + + +def run_one_step(step, host_workspace): + """Run a single step listed in a cloudbuild.yaml file. + + Args: + step (dict): A single step to perform + host_workspace (str): Scratch directory + """ + name = get(step, 'name', str) + dir_ = get(step, 'dir', list) + env = get(step, 'env', list) + args = get(step, 'args', list) + run_docker(name, dir_, env, args, host_workspace) + + +def get(container, field_name, field_type): + """Fetch a field from a container with typechecking and default values. + + If the field is not present, a instance of `field_type` is + constructed with no arguments and used as the default value. + + Args: + container (dict): Object decoded from yaml + field_name (str): Field that should be present in `container` + field_type (type): Expected type for field value + + Returns: + fetched or default value of field + + Raises: + CloudBuildError if field value is present but is the wrong type. + """ + value = container.get(field_name) + if value is None: + return field_type() + if not isinstance(value, field_type): + raise CloudBuildError( + 'Syntax error: Expected "%d" to be of type "%d", but found "%d"', + field_name, field_type, type(value)) + return value + + +def run_docker(name, dir_, env_args, args, host_workspace): + """Construct and execute a single 'docker run' command""" + workdir = '/workspace' + if dir_: + workdir += '/' + dir_ + + env_pairs = [] + for env_arg in env_args: + env_pairs.append('--env', env_arg) + + process_args = [ + 'docker', + 'run', + '--volume', + '/var/run/docker.sock:/var/run/docker.sock', + '--volume', + '/root/.docker:/root/.docker', + '--volume', + '%s:/workspace' % host_workspace, + '--workdir', + workdir, + ] + env_args + [name] + args + + print('Executing ' + ' '.join(process_args)) + subprocess.check_call(process_args) + + +if __name__ == '__main__': + sys.exit(main(sys.argv)) From e84e14e4712b0f3fac41b9b07110877b6287d88c Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Thu, 9 Feb 2017 14:19:22 -0800 Subject: [PATCH 005/256] Minor review changes --- local-cloudbuild.py | 56 ++++++++++++++++++++++++++++----------------- 1 file changed, 35 insertions(+), 21 deletions(-) diff --git a/local-cloudbuild.py b/local-cloudbuild.py index 31a2b903..51bd2d79 100755 --- a/local-cloudbuild.py +++ b/local-cloudbuild.py @@ -1,12 +1,29 @@ #!/usr/bin/env python + +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# Run the steps of cloudbuild.yaml locally, emulating Google Container -# Builder functionality. Does not actually push images to the -# Container Registry. Not all functionality is supported. +# http://www.apache.org/licenses/LICENSE-2.0 # -# Based on https://cloud.google.com/container-builder/docs/api/build-steps +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Emulate the Google Container Builder locally. + +The input is a cloudbuild.yaml file locally, which is processed using +a locally installed Docker daemon. The output images are not pushed +to the Google Container Registry. Not all functionality is supported. + +See https://cloud.google.com/container-builder/docs/api/build-steps +for more information. +""" -# System packages import argparse import getpass import os @@ -15,12 +32,11 @@ import sys import tempfile -# Third party packages import yaml -# Types class CloudBuildError(Exception): + """Syntax error in cloudbuild.yaml or other user error""" pass @@ -38,20 +54,18 @@ def main(argv): args = parser.parse_args(argv[1:]) # Load and parse cloudbuild.yaml - cloudbuild = None with open(args.cloudbuild, 'rb') as infile: cloudbuild = yaml.safe_load(infile) - host_workspace_parent = tempfile.mkdtemp(prefix='local-cloudbuild_%s_' % - getpass.getuser()) + host_workspace_parent = tempfile.mkdtemp(prefix='local-cloudbuild_') host_workspace = os.path.join(host_workspace_parent, 'workspace') try: # Prepare workspace + print('Running cloudbuild locally. Host workspace directory is %s' % + host_workspace) shutil.copytree('.', host_workspace, symlinks=True) # Execute a series of 'docker run' commands locally - print('Running cloudbuild locally. Host workspace directory is %s' % - host_workspace) run_steps(cloudbuild, host_workspace) finally: if not args.keep_workspace: @@ -68,7 +82,7 @@ def run_steps(cloudbuild, host_workspace): Raises: CloudBuildError if the yaml contents are invalid """ - steps = cloudbuild.get('steps', {}) + steps = cloudbuild.get_field_value('steps', {}) if not steps: raise CloudBuildError('No steps defined in cloudbuild.yaml') @@ -83,14 +97,14 @@ def run_one_step(step, host_workspace): step (dict): A single step to perform host_workspace (str): Scratch directory """ - name = get(step, 'name', str) - dir_ = get(step, 'dir', list) - env = get(step, 'env', list) - args = get(step, 'args', list) + name = get_field_value(step, 'name', str) + dir_ = get_field_value(step, 'dir', list) + env = get_field_value(step, 'env', list) + args = get_field_value(step, 'args', list) run_docker(name, dir_, env, args, host_workspace) -def get(container, field_name, field_type): +def get_field_value(container, field_name, field_type): """Fetch a field from a container with typechecking and default values. If the field is not present, a instance of `field_type` is @@ -102,7 +116,7 @@ def get(container, field_name, field_type): field_type (type): Expected type for field value Returns: - fetched or default value of field + (any) fetched or default value of field Raises: CloudBuildError if field value is present but is the wrong type. @@ -112,7 +126,7 @@ def get(container, field_name, field_type): return field_type() if not isinstance(value, field_type): raise CloudBuildError( - 'Syntax error: Expected "%d" to be of type "%d", but found "%d"', + 'Expected "%d" to be of type "%d", but found "%d"', field_name, field_type, type(value)) return value @@ -121,7 +135,7 @@ def run_docker(name, dir_, env_args, args, host_workspace): """Construct and execute a single 'docker run' command""" workdir = '/workspace' if dir_: - workdir += '/' + dir_ + workdir = os.path.join(workdir, dir_) env_pairs = [] for env_arg in env_args: From c2aabc49fea8567a88ee50fba92c98ecd40b582b Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Thu, 9 Feb 2017 15:30:07 -0800 Subject: [PATCH 006/256] Move main() to bottom of file --- local-cloudbuild.py | 64 ++++++++++++++++++++++----------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/local-cloudbuild.py b/local-cloudbuild.py index 51bd2d79..b44446ff 100755 --- a/local-cloudbuild.py +++ b/local-cloudbuild.py @@ -40,38 +40,6 @@ class CloudBuildError(Exception): pass -def main(argv): - """Main entrypoint for cli""" - parser = argparse.ArgumentParser( - description='Process cloudbuild.yaml locally to build Docker images') - parser.add_argument( - 'cloudbuild', type=str, help='Path to cloudbuild.yaml input file') - parser.add_argument( - '--keep_workspace', - type=bool, - default=False, - help='Retain workspace directory after building') - args = parser.parse_args(argv[1:]) - - # Load and parse cloudbuild.yaml - with open(args.cloudbuild, 'rb') as infile: - cloudbuild = yaml.safe_load(infile) - - host_workspace_parent = tempfile.mkdtemp(prefix='local-cloudbuild_') - host_workspace = os.path.join(host_workspace_parent, 'workspace') - try: - # Prepare workspace - print('Running cloudbuild locally. Host workspace directory is %s' % - host_workspace) - shutil.copytree('.', host_workspace, symlinks=True) - - # Execute a series of 'docker run' commands locally - run_steps(cloudbuild, host_workspace) - finally: - if not args.keep_workspace: - shutil.rmtree(host_workspace_parent, ignore_errors=True) - - def run_steps(cloudbuild, host_workspace): """Run the steps listed in a cloudbuild.yaml file. @@ -158,5 +126,37 @@ def run_docker(name, dir_, env_args, args, host_workspace): subprocess.check_call(process_args) +def main(argv): + """Main entrypoint for cli""" + parser = argparse.ArgumentParser( + description='Process cloudbuild.yaml locally to build Docker images') + parser.add_argument( + 'cloudbuild', type=str, help='Path to cloudbuild.yaml input file') + parser.add_argument( + '--keep_workspace', + type=bool, + default=False, + help='Retain workspace directory after building') + args = parser.parse_args(argv[1:]) + + # Load and parse cloudbuild.yaml + with open(args.cloudbuild, 'rb') as infile: + cloudbuild = yaml.safe_load(infile) + + host_workspace_parent = tempfile.mkdtemp(prefix='local-cloudbuild_') + host_workspace = os.path.join(host_workspace_parent, 'workspace') + try: + # Prepare workspace + print('Running cloudbuild locally. Host workspace directory is %s' % + host_workspace) + shutil.copytree('.', host_workspace, symlinks=True) + + # Execute a series of 'docker run' commands locally + run_steps(cloudbuild, host_workspace) + finally: + if not args.keep_workspace: + shutil.rmtree(host_workspace_parent, ignore_errors=True) + + if __name__ == '__main__': sys.exit(main(sys.argv)) From 6dc37bdee5f75d909eafdca1df1e1bfb39494e17 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Thu, 16 Feb 2017 17:25:38 -0800 Subject: [PATCH 007/256] Improve tool to run cloudbuild.yaml steps locally. The tool was moved to scripts/ and renamed. It now produces a shell script of Docker commands that can be stored and examined without necessarily running them (by passing --no-run). Command line flags changed to match 'gcloud container builds submit'. Added tests. Various other bugfixes and improvements. --- .gitignore | 6 +- local-cloudbuild.py | 162 --------- scripts/local_cloudbuild.py | 322 ++++++++++++++++++ scripts/local_cloudbuild_test.py | 322 ++++++++++++++++++ .../testdata/cloudbuild_err_not_found.yaml | 3 + scripts/testdata/cloudbuild_err_rc1.yaml | 3 + scripts/testdata/cloudbuild_ok.yaml | 7 + scripts/testdata/cloudbuild_ok.yaml_golden.sh | 28 ++ 8 files changed, 689 insertions(+), 164 deletions(-) delete mode 100755 local-cloudbuild.py create mode 100755 scripts/local_cloudbuild.py create mode 100755 scripts/local_cloudbuild_test.py create mode 100644 scripts/testdata/cloudbuild_err_not_found.yaml create mode 100644 scripts/testdata/cloudbuild_err_rc1.yaml create mode 100644 scripts/testdata/cloudbuild_ok.yaml create mode 100755 scripts/testdata/cloudbuild_ok.yaml_golden.sh diff --git a/.gitignore b/.gitignore index 05626c51..0c354cc9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ -cloudbuild.yaml -ext_run.sh +/cloudbuild.yaml +/cloudbuild.yaml_local.sh +/ext_run.sh +__pycache__ diff --git a/local-cloudbuild.py b/local-cloudbuild.py deleted file mode 100755 index b44446ff..00000000 --- a/local-cloudbuild.py +++ /dev/null @@ -1,162 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Emulate the Google Container Builder locally. - -The input is a cloudbuild.yaml file locally, which is processed using -a locally installed Docker daemon. The output images are not pushed -to the Google Container Registry. Not all functionality is supported. - -See https://cloud.google.com/container-builder/docs/api/build-steps -for more information. -""" - -import argparse -import getpass -import os -import shutil -import subprocess -import sys -import tempfile - -import yaml - - -class CloudBuildError(Exception): - """Syntax error in cloudbuild.yaml or other user error""" - pass - - -def run_steps(cloudbuild, host_workspace): - """Run the steps listed in a cloudbuild.yaml file. - - Args: - cloudbuild (dict): The decoded contents of a cloudbuild.yaml - host_workspace (str): Scratch directory - - Raises: - CloudBuildError if the yaml contents are invalid - """ - steps = cloudbuild.get_field_value('steps', {}) - if not steps: - raise CloudBuildError('No steps defined in cloudbuild.yaml') - - for step in steps: - run_one_step(step, host_workspace) - - -def run_one_step(step, host_workspace): - """Run a single step listed in a cloudbuild.yaml file. - - Args: - step (dict): A single step to perform - host_workspace (str): Scratch directory - """ - name = get_field_value(step, 'name', str) - dir_ = get_field_value(step, 'dir', list) - env = get_field_value(step, 'env', list) - args = get_field_value(step, 'args', list) - run_docker(name, dir_, env, args, host_workspace) - - -def get_field_value(container, field_name, field_type): - """Fetch a field from a container with typechecking and default values. - - If the field is not present, a instance of `field_type` is - constructed with no arguments and used as the default value. - - Args: - container (dict): Object decoded from yaml - field_name (str): Field that should be present in `container` - field_type (type): Expected type for field value - - Returns: - (any) fetched or default value of field - - Raises: - CloudBuildError if field value is present but is the wrong type. - """ - value = container.get(field_name) - if value is None: - return field_type() - if not isinstance(value, field_type): - raise CloudBuildError( - 'Expected "%d" to be of type "%d", but found "%d"', - field_name, field_type, type(value)) - return value - - -def run_docker(name, dir_, env_args, args, host_workspace): - """Construct and execute a single 'docker run' command""" - workdir = '/workspace' - if dir_: - workdir = os.path.join(workdir, dir_) - - env_pairs = [] - for env_arg in env_args: - env_pairs.append('--env', env_arg) - - process_args = [ - 'docker', - 'run', - '--volume', - '/var/run/docker.sock:/var/run/docker.sock', - '--volume', - '/root/.docker:/root/.docker', - '--volume', - '%s:/workspace' % host_workspace, - '--workdir', - workdir, - ] + env_args + [name] + args - - print('Executing ' + ' '.join(process_args)) - subprocess.check_call(process_args) - - -def main(argv): - """Main entrypoint for cli""" - parser = argparse.ArgumentParser( - description='Process cloudbuild.yaml locally to build Docker images') - parser.add_argument( - 'cloudbuild', type=str, help='Path to cloudbuild.yaml input file') - parser.add_argument( - '--keep_workspace', - type=bool, - default=False, - help='Retain workspace directory after building') - args = parser.parse_args(argv[1:]) - - # Load and parse cloudbuild.yaml - with open(args.cloudbuild, 'rb') as infile: - cloudbuild = yaml.safe_load(infile) - - host_workspace_parent = tempfile.mkdtemp(prefix='local-cloudbuild_') - host_workspace = os.path.join(host_workspace_parent, 'workspace') - try: - # Prepare workspace - print('Running cloudbuild locally. Host workspace directory is %s' % - host_workspace) - shutil.copytree('.', host_workspace, symlinks=True) - - # Execute a series of 'docker run' commands locally - run_steps(cloudbuild, host_workspace) - finally: - if not args.keep_workspace: - shutil.rmtree(host_workspace_parent, ignore_errors=True) - - -if __name__ == '__main__': - sys.exit(main(sys.argv)) diff --git a/scripts/local_cloudbuild.py b/scripts/local_cloudbuild.py new file mode 100755 index 00000000..53ae86e0 --- /dev/null +++ b/scripts/local_cloudbuild.py @@ -0,0 +1,322 @@ +#!/usr/bin/env python3 + +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Emulate the Google Container Builder locally. + +The input is a local cloudbuild.yaml file. This is translated into a +series of commands for the locally installed Docker daemon. These +commands are output as a shell script and optionally executed. + +The output images are not pushed to the Google Container Registry. +Not all cloudbuild.yaml functionality is supported. + +See https://cloud.google.com/container-builder/docs/api/build-steps +for more information. +""" + +import argparse +import collections +import collections.abc +import functools +import io +import os +import re +import shlex +import subprocess +import sys + +import yaml + + +# Exclude non-printable control characters (including newlines) +PRINTABLE_REGEX = re.compile(r"""^[^\x00-\x1f]*$""") + +# File template +BUILD_SCRIPT_HEADER = """\ +#!/bin/bash +# This is a generated file. Do not edit. + +set -euo pipefail + +SOURCE_DIR=. + +# Setup staging directory +HOST_WORKSPACE=$(mktemp -d) +function cleanup { + if [ "${HOST_WORKSPACE}" != '/' -a -d "${HOST_WORKSPACE}" ]; then + rm -rf "${HOST_WORKSPACE}" + fi +} +trap cleanup EXIT + +# Copy source to staging directory +echo "Copying source to staging directory ${HOST_WORKSPACE}" +rsync -avzq --exclude=.git "${SOURCE_DIR}" "${HOST_WORKSPACE}" + +# Build commands +""" + +BUILD_SCRIPT_FOOTER = """\ +# End of build commands + +echo "Build completed successfully" +""" + + +# Validated cloudbuild recipe + flags +CloudBuild = collections.namedtuple('CloudBuild', 'output_script run steps') + +# Single validated step in a cloudbuild recipe +Step = collections.namedtuple('Step', 'args dir_ env name') + + +def get_field_value(container, field_name, field_type): + """Fetch a field from a container with typechecking and default values. + + The field value is coerced to the desired type. If the field is + not present, a instance of `field_type` is constructed with no + arguments and used as the default value. + + Args: + container (dict): Object decoded from yaml + field_name (str): Field that should be present in `container` + field_type (type): Expected type for field value + + Returns: + any: Fetched or default value of field + + Raises: + ValueError: if field value cannot be converted to the desired type + """ + try: + value = container[field_name] + except (IndexError, KeyError): + return field_type() + + msg = 'Expected "{}" field to be of type "{}", but found type "{}"' + if not isinstance(value, field_type): + # list('some string') is a successful type cast as far as Python + # is concerned, but doesn't exactly produce the results we want. + # We have a whitelist of conversions we will attempt. + whitelist = ( + (float, str), + (int, str), + (str, float), + (str, int), + (int, float), + ) + if (type(value), field_type) not in whitelist: + raise ValueError(msg.format(field_name, field_type, type(value))) + + try: + value = field_type(value) + except ValueError as e: + e.message = msg.format(field_name, field_type, type(value)) + raise + return value + + +def get_cloudbuild(raw_config, args): + """Read and validate a cloudbuild recipe + + Args: + raw_config (dict): deserialized cloudbuild.yaml + args (argparse.Namespace): ccommand line flags + + Returns: + CloudBuild: valid configuration + """ + if not isinstance(raw_config, dict): + raise ValueError( + 'Expected {} contents to be of type "dict", but found type "{}"'. + format(args.config, type(raw_config))) + raw_steps = get_field_value(raw_config, 'steps', list) + if not raw_steps: + raise ValueError('No steps defined in {}'.format(args.config)) + steps = [get_step(raw_step) for raw_step in raw_steps] + return CloudBuild( + output_script=args.output_script, + run=args.run, + steps=steps, + ) + + +def get_step(raw_step): + """Read and validate a single cloudbuild step + + Args: + raw_step (dict): deserialized step + + Returns: + Step: valid build step + """ + if not isinstance(raw_step, dict): + raise ValueError( + 'Expected step to be of type "dict", but found type "{}"'. + format(type(raw_step))) + raw_args = get_field_value(raw_step, 'args', list) + args = [get_field_value(raw_args, i, str) + for i in range(len(raw_args))] + dir_ = get_field_value(raw_step, 'dir', str) + raw_env = get_field_value(raw_step, 'env', list) + env = [get_field_value(raw_env, i, str) + for i in range(len(raw_env))] + name = get_field_value(raw_step, 'name', str) + return Step( + args=args, + dir_=dir_, + env=env, + name=name, + ) + + +def generate_command(step): + """Generate a single shell command to run for a single cloudbuild step + + Args: + step (Step): Valid build step + + Returns: + [str]: A single shell command, expressed as a list of quoted tokens. + """ + quoted_args = [shlex.quote(arg) for arg in step.args] + quoted_env = [] + for env in step.env: + quoted_env.extend(['--env', shlex.quote(env)]) + quoted_name = shlex.quote(step.name) + workdir = '/workspace' + if step.dir_: + workdir = os.path.join(workdir, shlex.quote(step.dir_)) + process_args = [ + 'docker', + 'run', + '--volume', + '/var/run/docker.sock:/var/run/docker.sock', + '--volume', + '/root/.docker:/root/.docker', + '--volume', + '${HOST_WORKSPACE}:/workspace', + '--workdir', + workdir, + ] + quoted_env + [quoted_name] + quoted_args + return process_args + + +def generate_script(cloudbuild): + """Generate the contents of a shell script + + Args: + cloudbuild (CloudBuild): Valid cloudbuild configuration + + Returns: + (str): Contents of shell script + """ + outfile = io.StringIO() + outfile.write(BUILD_SCRIPT_HEADER) + docker_commands = [generate_command(step) for step in cloudbuild.steps] + for docker_command in docker_commands: + line = ' '.join(docker_command) + '\n\n' + outfile.write(line) + outfile.write(BUILD_SCRIPT_FOOTER) + s = outfile.getvalue() + outfile.close() + return s + + +def make_executable(path): + """Set executable bit(s) on file""" + # http://stackoverflow.com/questions/12791997 + mode = os.stat(path).st_mode + mode |= (mode & 0o444) >> 2 # copy R bits to X + os.chmod(path, mode) + + +def write_script(cloudbuild, contents): + """Write a shell script to a file.""" + print('Writing build script to {}'.format(cloudbuild.output_script)) + with open(cloudbuild.output_script, 'w', encoding='utf8') as outfile: + outfile.write(contents) + make_executable(cloudbuild.output_script) + + +def local_cloudbuild(args): + """Execute the steps of a cloudbuild.yaml locally + + Args: + args: command line flags as per parse_args + """ + # Load and parse cloudbuild.yaml + with open(args.config, 'r', encoding='utf8') as cloudbuild_file: + raw_config = yaml.safe_load(cloudbuild_file) + + # Determine configuration + cloudbuild = get_cloudbuild(raw_config, args) + + # Create shell script + contents = generate_script(cloudbuild) + write_script(cloudbuild, contents) + + # Run shell script + if cloudbuild.run: + args = [os.path.abspath(cloudbuild.output_script)] + subprocess.check_call(args) + + +def validate_arg_regex(flag_value, flag_regex): + """Check a named command line flag against a regular expression""" + if not re.match(flag_regex, flag_value): + raise argparse.ArgumentTypeError( + 'Value "{}" does not match pattern "{}"'.format( + flag_value, flag_regex.pattern)) + return flag_value + + +def parse_args(argv): + """Parse and validate command line flags""" + parser = argparse.ArgumentParser( + description='Process cloudbuild.yaml locally to build Docker images') + parser.add_argument( + '--config', + type=functools.partial( + validate_arg_regex, flag_regex=PRINTABLE_REGEX), + default='cloudbuild.yaml', + help='Path to cloudbuild.yaml file' + ) + parser.add_argument( + '--output_script', + type=functools.partial( + validate_arg_regex, flag_regex=PRINTABLE_REGEX), + help='Filename to write shell script to', + ) + parser.add_argument( + '--no-run', + action='store_false', + help='Create shell script but don\'t execute it', + dest='run', + ) + args = parser.parse_args(argv[1:]) + if not args.output_script: + args.output_script = args.config + "_local.sh" + return args + + +def main(): + args = parse_args(sys.argv) + local_cloudbuild(args) + + +if __name__ == '__main__': + main() diff --git a/scripts/local_cloudbuild_test.py b/scripts/local_cloudbuild_test.py new file mode 100755 index 00000000..481e5ae9 --- /dev/null +++ b/scripts/local_cloudbuild_test.py @@ -0,0 +1,322 @@ +#!/usr/bin/env python3 + +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit test for local_cloudbuild.py""" + +import argparse +import os +import re +import shutil +import subprocess +import tempfile +import unittest +import unittest.mock + +import yaml + +import local_cloudbuild + + +class ValidationUtilsTest(unittest.TestCase): + + def test_get_field_value(self): + valid_cases = ( + # Normal case, field present and correct type + ({ 'present': 1 }, 'present', int, 1), + ({ 'present': '1' }, 'present', str, '1'), + ({ 'present': [1] }, 'present', list, [1]), + ({ 'present': {1: 2} }, 'present', dict, {1: 2}), + # Missing field replaced by default + ({}, 'missing', str, ''), + # Valid conversions + ({ 'str_to_int': '1' }, 'str_to_int', int, 1), + ({ 'int_to_str': 1 }, 'int_to_str', str, '1'), + ) + for valid_case in valid_cases: + with self.subTest(valid_case=valid_case): + container, field_name, field_type, expected = valid_case + self.assertEqual( + local_cloudbuild.get_field_value( + container, field_name, field_type), + expected) + + invalid_cases = ( + # Type conversion failures + ({ 'bad_list_to_dict': [1] }, 'bad_list_to_dict', dict), + ({ 'bad_list_to_str': [1] }, 'bad_list_to_str', str), + ({ 'bad_dict_to_list': {1: 2} }, 'bad_dict_to_list', list), + ({ 'bad_str_to_int': 'not_an_int' }, 'bad_str_to_int', int), + ({ 'bad_str_to_list': 'abc' }, 'bad_str_to_list', list), + ) + for invalid_case in invalid_cases: + with self.subTest(invalid_case=invalid_case): + container, field_name, field_type = invalid_case + with self.assertRaises(ValueError): + local_cloudbuild.get_field_value( + container, field_name, field_type) + + def test_validate_arg_regex(self): + self.assertEqual( + local_cloudbuild.validate_arg_regex('abc', re.compile('a[b]c')), + 'abc') + with self.assertRaises(argparse.ArgumentTypeError): + local_cloudbuild.validate_arg_regex('abc', re.compile('a[d]c')) + + +class LocalCloudbuildTest(unittest.TestCase): + + def setUp(self): + self.testdata_dir = 'testdata' + assert os.path.isdir(self.testdata_dir), 'Could not run test: testdata directory not found' + + def test_get_cloudbuild(self): + args = argparse.Namespace( + config='some_config_file', + output_script='some_output_script', + run=False, + ) + # Basic valid case + valid_case = 'steps:\n- name: step1\n- name: step2\n' + raw_config = yaml.safe_load(valid_case) + actual = local_cloudbuild.get_cloudbuild(raw_config, args) + self.assertEqual(len(actual.steps), 2) + + invalid_cases = ( + # Empty cloud build + '', + # No steps + 'foo: bar\n', + # Steps not a list + 'steps: astring\n', + ) + for invalid_case in invalid_cases: + with self.subTest(invalid_case=invalid_case): + raw_config = yaml.safe_load(invalid_case) + with self.assertRaises(ValueError): + local_cloudbuild.get_cloudbuild(raw_config, args) + + def test_get_step(self): + valid_cases = ( + # Empty step + ({}, local_cloudbuild.Step( + args=[], + dir_='', + env=[], + name='', + )), + # Full step + ({'name' : 'aname', + 'args' : [ 'arg1', 2, 'arg3 with \n newline', ], + 'env' : [ 'ENV1=value1', 'ENV2=space in value2' ], + 'dir' : 'adir', + }, local_cloudbuild.Step( + args = [ 'arg1', '2', 'arg3 with \n newline', ], + env = [ 'ENV1=value1', 'ENV2=space in value2' ], + dir_ = 'adir', + name = 'aname', + )), + ) + for valid_case in valid_cases: + with self.subTest(valid_case=valid_case): + raw_step, expected = valid_case + actual = local_cloudbuild.get_step(raw_step) + self.assertEqual(actual, expected) + + invalid_cases = ( + # Wrong type + [], + # More wrong types + {'args': 'not_a_list'}, + {'args': [ [] ]}, + {'env': 'not_a_list'}, + {'env': [ {} ]}, + {'dir': {}}, + {'name': []}, + ) + for invalid_case in invalid_cases: + with self.subTest(invalid_case=invalid_case): + with self.assertRaises(ValueError): + local_cloudbuild.get_step(invalid_case) + + def test_generate_command(self): + # Basic valid case + base_step = local_cloudbuild.Step( + args = ['arg1','arg2'], + dir_ = '', + env = ['ENV1=value1', 'ENV2=value2'], + name = 'aname', + ) + command = local_cloudbuild.generate_command(base_step) + self.assertEqual(command, [ + 'docker', + 'run', + '--volume', + '/var/run/docker.sock:/var/run/docker.sock', + '--volume', + '/root/.docker:/root/.docker', + '--volume', + '${HOST_WORKSPACE}:/workspace', + '--workdir', + '/workspace', + '--env', + 'ENV1=value1', + '--env', + 'ENV2=value2', + 'aname', + 'arg1', + 'arg2', + ]) + + # dir specified + step = base_step._replace(dir_='adir') + command = local_cloudbuild.generate_command(step) + self.assertIn('--workdir', command) + self.assertIn('/workspace/adir', command) + + # Shell quoting + step = base_step._replace(args=['arg with \n newline']) + command = local_cloudbuild.generate_command(step) + self.assertIn("'arg with \n newline'", command) + + step = base_step._replace(dir_='dir/ with space/') + command = local_cloudbuild.generate_command(step) + self.assertIn("/workspace/'dir/ with space/'", command) + + step = base_step._replace(env=['env with space']) + command = local_cloudbuild.generate_command(step) + self.assertIn("'env with space'", command) + + step = base_step._replace(name='a name') + command = local_cloudbuild.generate_command(step) + self.assertIn("'a name'", command) + + def test_generate_script(self): + config_name = 'cloudbuild_ok.yaml' + config = os.path.join(self.testdata_dir, config_name) + expected_output_script = os.path.join(self.testdata_dir, config_name + '_golden.sh') + cloudbuild = local_cloudbuild.CloudBuild( + output_script='test_generate_script', + run=False, + steps=[ + local_cloudbuild.Step( + args=['/bin/sh', '-c', 'echo "${MESSAGE}"'], + dir_='', + env=['MESSAGE=Hello World!'], + name='debian', + ), + local_cloudbuild.Step( + args=['/bin/sh', '-c', 'echo "${MESSAGE}"'], + dir_='', + env=['MESSAGE=Goodbye\\n And Farewell!', 'UNUSED=unused'], + name='debian', + ) + ] + ) + actual = local_cloudbuild.generate_script(cloudbuild) + self.maxDiff = 2**16 + # Compare output against golden + with open(expected_output_script, 'r', encoding='utf8') as expected: + self.assertEqual(actual, expected.read()) + + def test_make_executable(self): + with tempfile.TemporaryDirectory( + prefix='local_cloudbuild_test_') as tempdir: + test_script_filename = os.path.join(tempdir, 'test_make_executable.sh') + with open(test_script_filename, 'w', encoding='utf8') as test_script: + test_script.write('#!/bin/sh\necho "Output from test_make_executable"') + local_cloudbuild.make_executable(test_script_filename) + output = subprocess.check_output([test_script_filename]) + self.assertEqual(output.decode('utf8'), "Output from test_make_executable\n") + + def test_write_script(self): + with tempfile.TemporaryDirectory( + prefix='local_cloudbuild_test_') as tempdir: + contents = 'The contents\n' + output_script_filename = os.path.join(tempdir, 'test_write_script') + cloudbuild = local_cloudbuild.CloudBuild( + output_script=output_script_filename, + run=False, + steps=[], + ) + local_cloudbuild.write_script(cloudbuild, contents) + with open(output_script_filename, 'r', encoding='utf8') as output_script: + actual = output_script.read() + self.assertEqual(actual, contents) + + def test_local_cloudbuild(self): + # Actually run it if we can find a docker command. + should_run = False + if ((shutil.which('docker') is not None) and + (subprocess.call(['docker', 'info'], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) == 0)): + should_run = True + + # Read cloudbuild.yaml from testdata file, write output to + # tempdir, and maybe try to run it + with tempfile.TemporaryDirectory( + prefix='local_cloudbuild_test_') as tempdir: + cases = ( + # Everything is ok + ('cloudbuild_ok.yaml', True), + # Exit code 1 (failure) + ('cloudbuild_err_rc1.yaml', False), + # Command not found + ('cloudbuild_err_not_found.yaml', False), + ) + for case in cases: + with self.subTest(case=cases): + config_name, should_succeed = case + config = os.path.join(self.testdata_dir, config_name) + actual_output_script = os.path.join( + tempdir, config_name + '_local.sh') + args = argparse.Namespace( + config=config, + output_script=actual_output_script, + run=should_run) + if should_run: + print("Executing docker commands in {}".format(actual_output_script)) + if should_succeed: + local_cloudbuild.local_cloudbuild(args) + else: + with self.assertRaises(subprocess.CalledProcessError): + local_cloudbuild.local_cloudbuild(args) + else: + # Generate but don't execute script + local_cloudbuild.local_cloudbuild(args) + + + def test_parse_args(self): + # Test explicit output_script + argv = ['argv0', '--output_script=my_output'] + args = local_cloudbuild.parse_args(argv) + self.assertEqual(args.output_script, 'my_output') + # Test implicit output_script + argv = ['argv0', '--config=my_config'] + args = local_cloudbuild.parse_args(argv) + self.assertEqual(args.output_script, 'my_config_local.sh') + + # Test run flag (default and --no-run) + argv = ['argv0'] + args = local_cloudbuild.parse_args(argv) + self.assertEqual(args.run, True) + argv = ['argv0', '--no-run'] + args = local_cloudbuild.parse_args(argv) + self.assertEqual(args.run, False) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/testdata/cloudbuild_err_not_found.yaml b/scripts/testdata/cloudbuild_err_not_found.yaml new file mode 100644 index 00000000..c7eb070d --- /dev/null +++ b/scripts/testdata/cloudbuild_err_not_found.yaml @@ -0,0 +1,3 @@ +steps: +- name: debian + args: ['/expected file not found'] diff --git a/scripts/testdata/cloudbuild_err_rc1.yaml b/scripts/testdata/cloudbuild_err_rc1.yaml new file mode 100644 index 00000000..3953a586 --- /dev/null +++ b/scripts/testdata/cloudbuild_err_rc1.yaml @@ -0,0 +1,3 @@ +steps: +- name: debian + args: ['/bin/sh', '-c', 'exit 1'] diff --git a/scripts/testdata/cloudbuild_ok.yaml b/scripts/testdata/cloudbuild_ok.yaml new file mode 100644 index 00000000..8478269b --- /dev/null +++ b/scripts/testdata/cloudbuild_ok.yaml @@ -0,0 +1,7 @@ +steps: +- name: debian + args: ['/bin/sh', '-c', 'echo "${MESSAGE}"'] + env: ['MESSAGE=Hello World!'] +- name: debian + args: ['/bin/sh', '-c', 'echo "${MESSAGE}"'] + env: ['MESSAGE=Goodbye\n And Farewell!', 'UNUSED=unused'] diff --git a/scripts/testdata/cloudbuild_ok.yaml_golden.sh b/scripts/testdata/cloudbuild_ok.yaml_golden.sh new file mode 100755 index 00000000..996c436e --- /dev/null +++ b/scripts/testdata/cloudbuild_ok.yaml_golden.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# This is a generated file. Do not edit. + +set -euo pipefail + +SOURCE_DIR=. + +# Setup staging directory +HOST_WORKSPACE=$(mktemp -d) +function cleanup { + if [ "${HOST_WORKSPACE}" != '/' -a -d "${HOST_WORKSPACE}" ]; then + rm -rf "${HOST_WORKSPACE}" + fi +} +trap cleanup EXIT + +# Copy source to staging directory +echo "Copying source to staging directory ${HOST_WORKSPACE}" +rsync -avzq --exclude=.git "${SOURCE_DIR}" "${HOST_WORKSPACE}" + +# Build commands +docker run --volume /var/run/docker.sock:/var/run/docker.sock --volume /root/.docker:/root/.docker --volume ${HOST_WORKSPACE}:/workspace --workdir /workspace --env 'MESSAGE=Hello World!' debian /bin/sh -c 'echo "${MESSAGE}"' + +docker run --volume /var/run/docker.sock:/var/run/docker.sock --volume /root/.docker:/root/.docker --volume ${HOST_WORKSPACE}:/workspace --workdir /workspace --env 'MESSAGE=Goodbye\n And Farewell!' --env UNUSED=unused debian /bin/sh -c 'echo "${MESSAGE}"' + +# End of build commands + +echo "Build completed successfully" From 2d128fcea89362652ab25b12aa37123dc54aa47a Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Thu, 16 Feb 2017 17:33:04 -0800 Subject: [PATCH 008/256] Revert "Improve tool to run cloudbuild.yaml steps locally." This reverts commit 6dc37bdee5f75d909eafdca1df1e1bfb39494e17. --- .gitignore | 6 +- local-cloudbuild.py | 162 +++++++++ scripts/local_cloudbuild.py | 322 ------------------ scripts/local_cloudbuild_test.py | 322 ------------------ .../testdata/cloudbuild_err_not_found.yaml | 3 - scripts/testdata/cloudbuild_err_rc1.yaml | 3 - scripts/testdata/cloudbuild_ok.yaml | 7 - scripts/testdata/cloudbuild_ok.yaml_golden.sh | 28 -- 8 files changed, 164 insertions(+), 689 deletions(-) create mode 100755 local-cloudbuild.py delete mode 100755 scripts/local_cloudbuild.py delete mode 100755 scripts/local_cloudbuild_test.py delete mode 100644 scripts/testdata/cloudbuild_err_not_found.yaml delete mode 100644 scripts/testdata/cloudbuild_err_rc1.yaml delete mode 100644 scripts/testdata/cloudbuild_ok.yaml delete mode 100755 scripts/testdata/cloudbuild_ok.yaml_golden.sh diff --git a/.gitignore b/.gitignore index 0c354cc9..05626c51 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,2 @@ -/cloudbuild.yaml -/cloudbuild.yaml_local.sh -/ext_run.sh -__pycache__ +cloudbuild.yaml +ext_run.sh diff --git a/local-cloudbuild.py b/local-cloudbuild.py new file mode 100755 index 00000000..b44446ff --- /dev/null +++ b/local-cloudbuild.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Emulate the Google Container Builder locally. + +The input is a cloudbuild.yaml file locally, which is processed using +a locally installed Docker daemon. The output images are not pushed +to the Google Container Registry. Not all functionality is supported. + +See https://cloud.google.com/container-builder/docs/api/build-steps +for more information. +""" + +import argparse +import getpass +import os +import shutil +import subprocess +import sys +import tempfile + +import yaml + + +class CloudBuildError(Exception): + """Syntax error in cloudbuild.yaml or other user error""" + pass + + +def run_steps(cloudbuild, host_workspace): + """Run the steps listed in a cloudbuild.yaml file. + + Args: + cloudbuild (dict): The decoded contents of a cloudbuild.yaml + host_workspace (str): Scratch directory + + Raises: + CloudBuildError if the yaml contents are invalid + """ + steps = cloudbuild.get_field_value('steps', {}) + if not steps: + raise CloudBuildError('No steps defined in cloudbuild.yaml') + + for step in steps: + run_one_step(step, host_workspace) + + +def run_one_step(step, host_workspace): + """Run a single step listed in a cloudbuild.yaml file. + + Args: + step (dict): A single step to perform + host_workspace (str): Scratch directory + """ + name = get_field_value(step, 'name', str) + dir_ = get_field_value(step, 'dir', list) + env = get_field_value(step, 'env', list) + args = get_field_value(step, 'args', list) + run_docker(name, dir_, env, args, host_workspace) + + +def get_field_value(container, field_name, field_type): + """Fetch a field from a container with typechecking and default values. + + If the field is not present, a instance of `field_type` is + constructed with no arguments and used as the default value. + + Args: + container (dict): Object decoded from yaml + field_name (str): Field that should be present in `container` + field_type (type): Expected type for field value + + Returns: + (any) fetched or default value of field + + Raises: + CloudBuildError if field value is present but is the wrong type. + """ + value = container.get(field_name) + if value is None: + return field_type() + if not isinstance(value, field_type): + raise CloudBuildError( + 'Expected "%d" to be of type "%d", but found "%d"', + field_name, field_type, type(value)) + return value + + +def run_docker(name, dir_, env_args, args, host_workspace): + """Construct and execute a single 'docker run' command""" + workdir = '/workspace' + if dir_: + workdir = os.path.join(workdir, dir_) + + env_pairs = [] + for env_arg in env_args: + env_pairs.append('--env', env_arg) + + process_args = [ + 'docker', + 'run', + '--volume', + '/var/run/docker.sock:/var/run/docker.sock', + '--volume', + '/root/.docker:/root/.docker', + '--volume', + '%s:/workspace' % host_workspace, + '--workdir', + workdir, + ] + env_args + [name] + args + + print('Executing ' + ' '.join(process_args)) + subprocess.check_call(process_args) + + +def main(argv): + """Main entrypoint for cli""" + parser = argparse.ArgumentParser( + description='Process cloudbuild.yaml locally to build Docker images') + parser.add_argument( + 'cloudbuild', type=str, help='Path to cloudbuild.yaml input file') + parser.add_argument( + '--keep_workspace', + type=bool, + default=False, + help='Retain workspace directory after building') + args = parser.parse_args(argv[1:]) + + # Load and parse cloudbuild.yaml + with open(args.cloudbuild, 'rb') as infile: + cloudbuild = yaml.safe_load(infile) + + host_workspace_parent = tempfile.mkdtemp(prefix='local-cloudbuild_') + host_workspace = os.path.join(host_workspace_parent, 'workspace') + try: + # Prepare workspace + print('Running cloudbuild locally. Host workspace directory is %s' % + host_workspace) + shutil.copytree('.', host_workspace, symlinks=True) + + # Execute a series of 'docker run' commands locally + run_steps(cloudbuild, host_workspace) + finally: + if not args.keep_workspace: + shutil.rmtree(host_workspace_parent, ignore_errors=True) + + +if __name__ == '__main__': + sys.exit(main(sys.argv)) diff --git a/scripts/local_cloudbuild.py b/scripts/local_cloudbuild.py deleted file mode 100755 index 53ae86e0..00000000 --- a/scripts/local_cloudbuild.py +++ /dev/null @@ -1,322 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2017 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Emulate the Google Container Builder locally. - -The input is a local cloudbuild.yaml file. This is translated into a -series of commands for the locally installed Docker daemon. These -commands are output as a shell script and optionally executed. - -The output images are not pushed to the Google Container Registry. -Not all cloudbuild.yaml functionality is supported. - -See https://cloud.google.com/container-builder/docs/api/build-steps -for more information. -""" - -import argparse -import collections -import collections.abc -import functools -import io -import os -import re -import shlex -import subprocess -import sys - -import yaml - - -# Exclude non-printable control characters (including newlines) -PRINTABLE_REGEX = re.compile(r"""^[^\x00-\x1f]*$""") - -# File template -BUILD_SCRIPT_HEADER = """\ -#!/bin/bash -# This is a generated file. Do not edit. - -set -euo pipefail - -SOURCE_DIR=. - -# Setup staging directory -HOST_WORKSPACE=$(mktemp -d) -function cleanup { - if [ "${HOST_WORKSPACE}" != '/' -a -d "${HOST_WORKSPACE}" ]; then - rm -rf "${HOST_WORKSPACE}" - fi -} -trap cleanup EXIT - -# Copy source to staging directory -echo "Copying source to staging directory ${HOST_WORKSPACE}" -rsync -avzq --exclude=.git "${SOURCE_DIR}" "${HOST_WORKSPACE}" - -# Build commands -""" - -BUILD_SCRIPT_FOOTER = """\ -# End of build commands - -echo "Build completed successfully" -""" - - -# Validated cloudbuild recipe + flags -CloudBuild = collections.namedtuple('CloudBuild', 'output_script run steps') - -# Single validated step in a cloudbuild recipe -Step = collections.namedtuple('Step', 'args dir_ env name') - - -def get_field_value(container, field_name, field_type): - """Fetch a field from a container with typechecking and default values. - - The field value is coerced to the desired type. If the field is - not present, a instance of `field_type` is constructed with no - arguments and used as the default value. - - Args: - container (dict): Object decoded from yaml - field_name (str): Field that should be present in `container` - field_type (type): Expected type for field value - - Returns: - any: Fetched or default value of field - - Raises: - ValueError: if field value cannot be converted to the desired type - """ - try: - value = container[field_name] - except (IndexError, KeyError): - return field_type() - - msg = 'Expected "{}" field to be of type "{}", but found type "{}"' - if not isinstance(value, field_type): - # list('some string') is a successful type cast as far as Python - # is concerned, but doesn't exactly produce the results we want. - # We have a whitelist of conversions we will attempt. - whitelist = ( - (float, str), - (int, str), - (str, float), - (str, int), - (int, float), - ) - if (type(value), field_type) not in whitelist: - raise ValueError(msg.format(field_name, field_type, type(value))) - - try: - value = field_type(value) - except ValueError as e: - e.message = msg.format(field_name, field_type, type(value)) - raise - return value - - -def get_cloudbuild(raw_config, args): - """Read and validate a cloudbuild recipe - - Args: - raw_config (dict): deserialized cloudbuild.yaml - args (argparse.Namespace): ccommand line flags - - Returns: - CloudBuild: valid configuration - """ - if not isinstance(raw_config, dict): - raise ValueError( - 'Expected {} contents to be of type "dict", but found type "{}"'. - format(args.config, type(raw_config))) - raw_steps = get_field_value(raw_config, 'steps', list) - if not raw_steps: - raise ValueError('No steps defined in {}'.format(args.config)) - steps = [get_step(raw_step) for raw_step in raw_steps] - return CloudBuild( - output_script=args.output_script, - run=args.run, - steps=steps, - ) - - -def get_step(raw_step): - """Read and validate a single cloudbuild step - - Args: - raw_step (dict): deserialized step - - Returns: - Step: valid build step - """ - if not isinstance(raw_step, dict): - raise ValueError( - 'Expected step to be of type "dict", but found type "{}"'. - format(type(raw_step))) - raw_args = get_field_value(raw_step, 'args', list) - args = [get_field_value(raw_args, i, str) - for i in range(len(raw_args))] - dir_ = get_field_value(raw_step, 'dir', str) - raw_env = get_field_value(raw_step, 'env', list) - env = [get_field_value(raw_env, i, str) - for i in range(len(raw_env))] - name = get_field_value(raw_step, 'name', str) - return Step( - args=args, - dir_=dir_, - env=env, - name=name, - ) - - -def generate_command(step): - """Generate a single shell command to run for a single cloudbuild step - - Args: - step (Step): Valid build step - - Returns: - [str]: A single shell command, expressed as a list of quoted tokens. - """ - quoted_args = [shlex.quote(arg) for arg in step.args] - quoted_env = [] - for env in step.env: - quoted_env.extend(['--env', shlex.quote(env)]) - quoted_name = shlex.quote(step.name) - workdir = '/workspace' - if step.dir_: - workdir = os.path.join(workdir, shlex.quote(step.dir_)) - process_args = [ - 'docker', - 'run', - '--volume', - '/var/run/docker.sock:/var/run/docker.sock', - '--volume', - '/root/.docker:/root/.docker', - '--volume', - '${HOST_WORKSPACE}:/workspace', - '--workdir', - workdir, - ] + quoted_env + [quoted_name] + quoted_args - return process_args - - -def generate_script(cloudbuild): - """Generate the contents of a shell script - - Args: - cloudbuild (CloudBuild): Valid cloudbuild configuration - - Returns: - (str): Contents of shell script - """ - outfile = io.StringIO() - outfile.write(BUILD_SCRIPT_HEADER) - docker_commands = [generate_command(step) for step in cloudbuild.steps] - for docker_command in docker_commands: - line = ' '.join(docker_command) + '\n\n' - outfile.write(line) - outfile.write(BUILD_SCRIPT_FOOTER) - s = outfile.getvalue() - outfile.close() - return s - - -def make_executable(path): - """Set executable bit(s) on file""" - # http://stackoverflow.com/questions/12791997 - mode = os.stat(path).st_mode - mode |= (mode & 0o444) >> 2 # copy R bits to X - os.chmod(path, mode) - - -def write_script(cloudbuild, contents): - """Write a shell script to a file.""" - print('Writing build script to {}'.format(cloudbuild.output_script)) - with open(cloudbuild.output_script, 'w', encoding='utf8') as outfile: - outfile.write(contents) - make_executable(cloudbuild.output_script) - - -def local_cloudbuild(args): - """Execute the steps of a cloudbuild.yaml locally - - Args: - args: command line flags as per parse_args - """ - # Load and parse cloudbuild.yaml - with open(args.config, 'r', encoding='utf8') as cloudbuild_file: - raw_config = yaml.safe_load(cloudbuild_file) - - # Determine configuration - cloudbuild = get_cloudbuild(raw_config, args) - - # Create shell script - contents = generate_script(cloudbuild) - write_script(cloudbuild, contents) - - # Run shell script - if cloudbuild.run: - args = [os.path.abspath(cloudbuild.output_script)] - subprocess.check_call(args) - - -def validate_arg_regex(flag_value, flag_regex): - """Check a named command line flag against a regular expression""" - if not re.match(flag_regex, flag_value): - raise argparse.ArgumentTypeError( - 'Value "{}" does not match pattern "{}"'.format( - flag_value, flag_regex.pattern)) - return flag_value - - -def parse_args(argv): - """Parse and validate command line flags""" - parser = argparse.ArgumentParser( - description='Process cloudbuild.yaml locally to build Docker images') - parser.add_argument( - '--config', - type=functools.partial( - validate_arg_regex, flag_regex=PRINTABLE_REGEX), - default='cloudbuild.yaml', - help='Path to cloudbuild.yaml file' - ) - parser.add_argument( - '--output_script', - type=functools.partial( - validate_arg_regex, flag_regex=PRINTABLE_REGEX), - help='Filename to write shell script to', - ) - parser.add_argument( - '--no-run', - action='store_false', - help='Create shell script but don\'t execute it', - dest='run', - ) - args = parser.parse_args(argv[1:]) - if not args.output_script: - args.output_script = args.config + "_local.sh" - return args - - -def main(): - args = parse_args(sys.argv) - local_cloudbuild(args) - - -if __name__ == '__main__': - main() diff --git a/scripts/local_cloudbuild_test.py b/scripts/local_cloudbuild_test.py deleted file mode 100755 index 481e5ae9..00000000 --- a/scripts/local_cloudbuild_test.py +++ /dev/null @@ -1,322 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2017 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Unit test for local_cloudbuild.py""" - -import argparse -import os -import re -import shutil -import subprocess -import tempfile -import unittest -import unittest.mock - -import yaml - -import local_cloudbuild - - -class ValidationUtilsTest(unittest.TestCase): - - def test_get_field_value(self): - valid_cases = ( - # Normal case, field present and correct type - ({ 'present': 1 }, 'present', int, 1), - ({ 'present': '1' }, 'present', str, '1'), - ({ 'present': [1] }, 'present', list, [1]), - ({ 'present': {1: 2} }, 'present', dict, {1: 2}), - # Missing field replaced by default - ({}, 'missing', str, ''), - # Valid conversions - ({ 'str_to_int': '1' }, 'str_to_int', int, 1), - ({ 'int_to_str': 1 }, 'int_to_str', str, '1'), - ) - for valid_case in valid_cases: - with self.subTest(valid_case=valid_case): - container, field_name, field_type, expected = valid_case - self.assertEqual( - local_cloudbuild.get_field_value( - container, field_name, field_type), - expected) - - invalid_cases = ( - # Type conversion failures - ({ 'bad_list_to_dict': [1] }, 'bad_list_to_dict', dict), - ({ 'bad_list_to_str': [1] }, 'bad_list_to_str', str), - ({ 'bad_dict_to_list': {1: 2} }, 'bad_dict_to_list', list), - ({ 'bad_str_to_int': 'not_an_int' }, 'bad_str_to_int', int), - ({ 'bad_str_to_list': 'abc' }, 'bad_str_to_list', list), - ) - for invalid_case in invalid_cases: - with self.subTest(invalid_case=invalid_case): - container, field_name, field_type = invalid_case - with self.assertRaises(ValueError): - local_cloudbuild.get_field_value( - container, field_name, field_type) - - def test_validate_arg_regex(self): - self.assertEqual( - local_cloudbuild.validate_arg_regex('abc', re.compile('a[b]c')), - 'abc') - with self.assertRaises(argparse.ArgumentTypeError): - local_cloudbuild.validate_arg_regex('abc', re.compile('a[d]c')) - - -class LocalCloudbuildTest(unittest.TestCase): - - def setUp(self): - self.testdata_dir = 'testdata' - assert os.path.isdir(self.testdata_dir), 'Could not run test: testdata directory not found' - - def test_get_cloudbuild(self): - args = argparse.Namespace( - config='some_config_file', - output_script='some_output_script', - run=False, - ) - # Basic valid case - valid_case = 'steps:\n- name: step1\n- name: step2\n' - raw_config = yaml.safe_load(valid_case) - actual = local_cloudbuild.get_cloudbuild(raw_config, args) - self.assertEqual(len(actual.steps), 2) - - invalid_cases = ( - # Empty cloud build - '', - # No steps - 'foo: bar\n', - # Steps not a list - 'steps: astring\n', - ) - for invalid_case in invalid_cases: - with self.subTest(invalid_case=invalid_case): - raw_config = yaml.safe_load(invalid_case) - with self.assertRaises(ValueError): - local_cloudbuild.get_cloudbuild(raw_config, args) - - def test_get_step(self): - valid_cases = ( - # Empty step - ({}, local_cloudbuild.Step( - args=[], - dir_='', - env=[], - name='', - )), - # Full step - ({'name' : 'aname', - 'args' : [ 'arg1', 2, 'arg3 with \n newline', ], - 'env' : [ 'ENV1=value1', 'ENV2=space in value2' ], - 'dir' : 'adir', - }, local_cloudbuild.Step( - args = [ 'arg1', '2', 'arg3 with \n newline', ], - env = [ 'ENV1=value1', 'ENV2=space in value2' ], - dir_ = 'adir', - name = 'aname', - )), - ) - for valid_case in valid_cases: - with self.subTest(valid_case=valid_case): - raw_step, expected = valid_case - actual = local_cloudbuild.get_step(raw_step) - self.assertEqual(actual, expected) - - invalid_cases = ( - # Wrong type - [], - # More wrong types - {'args': 'not_a_list'}, - {'args': [ [] ]}, - {'env': 'not_a_list'}, - {'env': [ {} ]}, - {'dir': {}}, - {'name': []}, - ) - for invalid_case in invalid_cases: - with self.subTest(invalid_case=invalid_case): - with self.assertRaises(ValueError): - local_cloudbuild.get_step(invalid_case) - - def test_generate_command(self): - # Basic valid case - base_step = local_cloudbuild.Step( - args = ['arg1','arg2'], - dir_ = '', - env = ['ENV1=value1', 'ENV2=value2'], - name = 'aname', - ) - command = local_cloudbuild.generate_command(base_step) - self.assertEqual(command, [ - 'docker', - 'run', - '--volume', - '/var/run/docker.sock:/var/run/docker.sock', - '--volume', - '/root/.docker:/root/.docker', - '--volume', - '${HOST_WORKSPACE}:/workspace', - '--workdir', - '/workspace', - '--env', - 'ENV1=value1', - '--env', - 'ENV2=value2', - 'aname', - 'arg1', - 'arg2', - ]) - - # dir specified - step = base_step._replace(dir_='adir') - command = local_cloudbuild.generate_command(step) - self.assertIn('--workdir', command) - self.assertIn('/workspace/adir', command) - - # Shell quoting - step = base_step._replace(args=['arg with \n newline']) - command = local_cloudbuild.generate_command(step) - self.assertIn("'arg with \n newline'", command) - - step = base_step._replace(dir_='dir/ with space/') - command = local_cloudbuild.generate_command(step) - self.assertIn("/workspace/'dir/ with space/'", command) - - step = base_step._replace(env=['env with space']) - command = local_cloudbuild.generate_command(step) - self.assertIn("'env with space'", command) - - step = base_step._replace(name='a name') - command = local_cloudbuild.generate_command(step) - self.assertIn("'a name'", command) - - def test_generate_script(self): - config_name = 'cloudbuild_ok.yaml' - config = os.path.join(self.testdata_dir, config_name) - expected_output_script = os.path.join(self.testdata_dir, config_name + '_golden.sh') - cloudbuild = local_cloudbuild.CloudBuild( - output_script='test_generate_script', - run=False, - steps=[ - local_cloudbuild.Step( - args=['/bin/sh', '-c', 'echo "${MESSAGE}"'], - dir_='', - env=['MESSAGE=Hello World!'], - name='debian', - ), - local_cloudbuild.Step( - args=['/bin/sh', '-c', 'echo "${MESSAGE}"'], - dir_='', - env=['MESSAGE=Goodbye\\n And Farewell!', 'UNUSED=unused'], - name='debian', - ) - ] - ) - actual = local_cloudbuild.generate_script(cloudbuild) - self.maxDiff = 2**16 - # Compare output against golden - with open(expected_output_script, 'r', encoding='utf8') as expected: - self.assertEqual(actual, expected.read()) - - def test_make_executable(self): - with tempfile.TemporaryDirectory( - prefix='local_cloudbuild_test_') as tempdir: - test_script_filename = os.path.join(tempdir, 'test_make_executable.sh') - with open(test_script_filename, 'w', encoding='utf8') as test_script: - test_script.write('#!/bin/sh\necho "Output from test_make_executable"') - local_cloudbuild.make_executable(test_script_filename) - output = subprocess.check_output([test_script_filename]) - self.assertEqual(output.decode('utf8'), "Output from test_make_executable\n") - - def test_write_script(self): - with tempfile.TemporaryDirectory( - prefix='local_cloudbuild_test_') as tempdir: - contents = 'The contents\n' - output_script_filename = os.path.join(tempdir, 'test_write_script') - cloudbuild = local_cloudbuild.CloudBuild( - output_script=output_script_filename, - run=False, - steps=[], - ) - local_cloudbuild.write_script(cloudbuild, contents) - with open(output_script_filename, 'r', encoding='utf8') as output_script: - actual = output_script.read() - self.assertEqual(actual, contents) - - def test_local_cloudbuild(self): - # Actually run it if we can find a docker command. - should_run = False - if ((shutil.which('docker') is not None) and - (subprocess.call(['docker', 'info'], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL) == 0)): - should_run = True - - # Read cloudbuild.yaml from testdata file, write output to - # tempdir, and maybe try to run it - with tempfile.TemporaryDirectory( - prefix='local_cloudbuild_test_') as tempdir: - cases = ( - # Everything is ok - ('cloudbuild_ok.yaml', True), - # Exit code 1 (failure) - ('cloudbuild_err_rc1.yaml', False), - # Command not found - ('cloudbuild_err_not_found.yaml', False), - ) - for case in cases: - with self.subTest(case=cases): - config_name, should_succeed = case - config = os.path.join(self.testdata_dir, config_name) - actual_output_script = os.path.join( - tempdir, config_name + '_local.sh') - args = argparse.Namespace( - config=config, - output_script=actual_output_script, - run=should_run) - if should_run: - print("Executing docker commands in {}".format(actual_output_script)) - if should_succeed: - local_cloudbuild.local_cloudbuild(args) - else: - with self.assertRaises(subprocess.CalledProcessError): - local_cloudbuild.local_cloudbuild(args) - else: - # Generate but don't execute script - local_cloudbuild.local_cloudbuild(args) - - - def test_parse_args(self): - # Test explicit output_script - argv = ['argv0', '--output_script=my_output'] - args = local_cloudbuild.parse_args(argv) - self.assertEqual(args.output_script, 'my_output') - # Test implicit output_script - argv = ['argv0', '--config=my_config'] - args = local_cloudbuild.parse_args(argv) - self.assertEqual(args.output_script, 'my_config_local.sh') - - # Test run flag (default and --no-run) - argv = ['argv0'] - args = local_cloudbuild.parse_args(argv) - self.assertEqual(args.run, True) - argv = ['argv0', '--no-run'] - args = local_cloudbuild.parse_args(argv) - self.assertEqual(args.run, False) - - -if __name__ == '__main__': - unittest.main() diff --git a/scripts/testdata/cloudbuild_err_not_found.yaml b/scripts/testdata/cloudbuild_err_not_found.yaml deleted file mode 100644 index c7eb070d..00000000 --- a/scripts/testdata/cloudbuild_err_not_found.yaml +++ /dev/null @@ -1,3 +0,0 @@ -steps: -- name: debian - args: ['/expected file not found'] diff --git a/scripts/testdata/cloudbuild_err_rc1.yaml b/scripts/testdata/cloudbuild_err_rc1.yaml deleted file mode 100644 index 3953a586..00000000 --- a/scripts/testdata/cloudbuild_err_rc1.yaml +++ /dev/null @@ -1,3 +0,0 @@ -steps: -- name: debian - args: ['/bin/sh', '-c', 'exit 1'] diff --git a/scripts/testdata/cloudbuild_ok.yaml b/scripts/testdata/cloudbuild_ok.yaml deleted file mode 100644 index 8478269b..00000000 --- a/scripts/testdata/cloudbuild_ok.yaml +++ /dev/null @@ -1,7 +0,0 @@ -steps: -- name: debian - args: ['/bin/sh', '-c', 'echo "${MESSAGE}"'] - env: ['MESSAGE=Hello World!'] -- name: debian - args: ['/bin/sh', '-c', 'echo "${MESSAGE}"'] - env: ['MESSAGE=Goodbye\n And Farewell!', 'UNUSED=unused'] diff --git a/scripts/testdata/cloudbuild_ok.yaml_golden.sh b/scripts/testdata/cloudbuild_ok.yaml_golden.sh deleted file mode 100755 index 996c436e..00000000 --- a/scripts/testdata/cloudbuild_ok.yaml_golden.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -# This is a generated file. Do not edit. - -set -euo pipefail - -SOURCE_DIR=. - -# Setup staging directory -HOST_WORKSPACE=$(mktemp -d) -function cleanup { - if [ "${HOST_WORKSPACE}" != '/' -a -d "${HOST_WORKSPACE}" ]; then - rm -rf "${HOST_WORKSPACE}" - fi -} -trap cleanup EXIT - -# Copy source to staging directory -echo "Copying source to staging directory ${HOST_WORKSPACE}" -rsync -avzq --exclude=.git "${SOURCE_DIR}" "${HOST_WORKSPACE}" - -# Build commands -docker run --volume /var/run/docker.sock:/var/run/docker.sock --volume /root/.docker:/root/.docker --volume ${HOST_WORKSPACE}:/workspace --workdir /workspace --env 'MESSAGE=Hello World!' debian /bin/sh -c 'echo "${MESSAGE}"' - -docker run --volume /var/run/docker.sock:/var/run/docker.sock --volume /root/.docker:/root/.docker --volume ${HOST_WORKSPACE}:/workspace --workdir /workspace --env 'MESSAGE=Goodbye\n And Farewell!' --env UNUSED=unused debian /bin/sh -c 'echo "${MESSAGE}"' - -# End of build commands - -echo "Build completed successfully" From c4b43870ac19cddb0d4dec9a63ace2914c51633c Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Thu, 16 Feb 2017 17:25:38 -0800 Subject: [PATCH 009/256] Improve tool to run cloudbuild.yaml steps locally. The tool was moved to scripts/ and renamed. It now produces a shell script of Docker commands that can be stored and examined without necessarily running them (by passing --no-run). Command line flags changed to match 'gcloud container builds submit'. Added tests. Various other bugfixes and improvements. --- .gitignore | 6 +- local-cloudbuild.py | 162 --------- scripts/local_cloudbuild.py | 322 ++++++++++++++++++ scripts/local_cloudbuild_test.py | 322 ++++++++++++++++++ .../testdata/cloudbuild_err_not_found.yaml | 3 + scripts/testdata/cloudbuild_err_rc1.yaml | 3 + scripts/testdata/cloudbuild_ok.yaml | 7 + scripts/testdata/cloudbuild_ok.yaml_golden.sh | 28 ++ 8 files changed, 689 insertions(+), 164 deletions(-) delete mode 100755 local-cloudbuild.py create mode 100755 scripts/local_cloudbuild.py create mode 100755 scripts/local_cloudbuild_test.py create mode 100644 scripts/testdata/cloudbuild_err_not_found.yaml create mode 100644 scripts/testdata/cloudbuild_err_rc1.yaml create mode 100644 scripts/testdata/cloudbuild_ok.yaml create mode 100755 scripts/testdata/cloudbuild_ok.yaml_golden.sh diff --git a/.gitignore b/.gitignore index 05626c51..0c354cc9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ -cloudbuild.yaml -ext_run.sh +/cloudbuild.yaml +/cloudbuild.yaml_local.sh +/ext_run.sh +__pycache__ diff --git a/local-cloudbuild.py b/local-cloudbuild.py deleted file mode 100755 index b44446ff..00000000 --- a/local-cloudbuild.py +++ /dev/null @@ -1,162 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Emulate the Google Container Builder locally. - -The input is a cloudbuild.yaml file locally, which is processed using -a locally installed Docker daemon. The output images are not pushed -to the Google Container Registry. Not all functionality is supported. - -See https://cloud.google.com/container-builder/docs/api/build-steps -for more information. -""" - -import argparse -import getpass -import os -import shutil -import subprocess -import sys -import tempfile - -import yaml - - -class CloudBuildError(Exception): - """Syntax error in cloudbuild.yaml or other user error""" - pass - - -def run_steps(cloudbuild, host_workspace): - """Run the steps listed in a cloudbuild.yaml file. - - Args: - cloudbuild (dict): The decoded contents of a cloudbuild.yaml - host_workspace (str): Scratch directory - - Raises: - CloudBuildError if the yaml contents are invalid - """ - steps = cloudbuild.get_field_value('steps', {}) - if not steps: - raise CloudBuildError('No steps defined in cloudbuild.yaml') - - for step in steps: - run_one_step(step, host_workspace) - - -def run_one_step(step, host_workspace): - """Run a single step listed in a cloudbuild.yaml file. - - Args: - step (dict): A single step to perform - host_workspace (str): Scratch directory - """ - name = get_field_value(step, 'name', str) - dir_ = get_field_value(step, 'dir', list) - env = get_field_value(step, 'env', list) - args = get_field_value(step, 'args', list) - run_docker(name, dir_, env, args, host_workspace) - - -def get_field_value(container, field_name, field_type): - """Fetch a field from a container with typechecking and default values. - - If the field is not present, a instance of `field_type` is - constructed with no arguments and used as the default value. - - Args: - container (dict): Object decoded from yaml - field_name (str): Field that should be present in `container` - field_type (type): Expected type for field value - - Returns: - (any) fetched or default value of field - - Raises: - CloudBuildError if field value is present but is the wrong type. - """ - value = container.get(field_name) - if value is None: - return field_type() - if not isinstance(value, field_type): - raise CloudBuildError( - 'Expected "%d" to be of type "%d", but found "%d"', - field_name, field_type, type(value)) - return value - - -def run_docker(name, dir_, env_args, args, host_workspace): - """Construct and execute a single 'docker run' command""" - workdir = '/workspace' - if dir_: - workdir = os.path.join(workdir, dir_) - - env_pairs = [] - for env_arg in env_args: - env_pairs.append('--env', env_arg) - - process_args = [ - 'docker', - 'run', - '--volume', - '/var/run/docker.sock:/var/run/docker.sock', - '--volume', - '/root/.docker:/root/.docker', - '--volume', - '%s:/workspace' % host_workspace, - '--workdir', - workdir, - ] + env_args + [name] + args - - print('Executing ' + ' '.join(process_args)) - subprocess.check_call(process_args) - - -def main(argv): - """Main entrypoint for cli""" - parser = argparse.ArgumentParser( - description='Process cloudbuild.yaml locally to build Docker images') - parser.add_argument( - 'cloudbuild', type=str, help='Path to cloudbuild.yaml input file') - parser.add_argument( - '--keep_workspace', - type=bool, - default=False, - help='Retain workspace directory after building') - args = parser.parse_args(argv[1:]) - - # Load and parse cloudbuild.yaml - with open(args.cloudbuild, 'rb') as infile: - cloudbuild = yaml.safe_load(infile) - - host_workspace_parent = tempfile.mkdtemp(prefix='local-cloudbuild_') - host_workspace = os.path.join(host_workspace_parent, 'workspace') - try: - # Prepare workspace - print('Running cloudbuild locally. Host workspace directory is %s' % - host_workspace) - shutil.copytree('.', host_workspace, symlinks=True) - - # Execute a series of 'docker run' commands locally - run_steps(cloudbuild, host_workspace) - finally: - if not args.keep_workspace: - shutil.rmtree(host_workspace_parent, ignore_errors=True) - - -if __name__ == '__main__': - sys.exit(main(sys.argv)) diff --git a/scripts/local_cloudbuild.py b/scripts/local_cloudbuild.py new file mode 100755 index 00000000..53ae86e0 --- /dev/null +++ b/scripts/local_cloudbuild.py @@ -0,0 +1,322 @@ +#!/usr/bin/env python3 + +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Emulate the Google Container Builder locally. + +The input is a local cloudbuild.yaml file. This is translated into a +series of commands for the locally installed Docker daemon. These +commands are output as a shell script and optionally executed. + +The output images are not pushed to the Google Container Registry. +Not all cloudbuild.yaml functionality is supported. + +See https://cloud.google.com/container-builder/docs/api/build-steps +for more information. +""" + +import argparse +import collections +import collections.abc +import functools +import io +import os +import re +import shlex +import subprocess +import sys + +import yaml + + +# Exclude non-printable control characters (including newlines) +PRINTABLE_REGEX = re.compile(r"""^[^\x00-\x1f]*$""") + +# File template +BUILD_SCRIPT_HEADER = """\ +#!/bin/bash +# This is a generated file. Do not edit. + +set -euo pipefail + +SOURCE_DIR=. + +# Setup staging directory +HOST_WORKSPACE=$(mktemp -d) +function cleanup { + if [ "${HOST_WORKSPACE}" != '/' -a -d "${HOST_WORKSPACE}" ]; then + rm -rf "${HOST_WORKSPACE}" + fi +} +trap cleanup EXIT + +# Copy source to staging directory +echo "Copying source to staging directory ${HOST_WORKSPACE}" +rsync -avzq --exclude=.git "${SOURCE_DIR}" "${HOST_WORKSPACE}" + +# Build commands +""" + +BUILD_SCRIPT_FOOTER = """\ +# End of build commands + +echo "Build completed successfully" +""" + + +# Validated cloudbuild recipe + flags +CloudBuild = collections.namedtuple('CloudBuild', 'output_script run steps') + +# Single validated step in a cloudbuild recipe +Step = collections.namedtuple('Step', 'args dir_ env name') + + +def get_field_value(container, field_name, field_type): + """Fetch a field from a container with typechecking and default values. + + The field value is coerced to the desired type. If the field is + not present, a instance of `field_type` is constructed with no + arguments and used as the default value. + + Args: + container (dict): Object decoded from yaml + field_name (str): Field that should be present in `container` + field_type (type): Expected type for field value + + Returns: + any: Fetched or default value of field + + Raises: + ValueError: if field value cannot be converted to the desired type + """ + try: + value = container[field_name] + except (IndexError, KeyError): + return field_type() + + msg = 'Expected "{}" field to be of type "{}", but found type "{}"' + if not isinstance(value, field_type): + # list('some string') is a successful type cast as far as Python + # is concerned, but doesn't exactly produce the results we want. + # We have a whitelist of conversions we will attempt. + whitelist = ( + (float, str), + (int, str), + (str, float), + (str, int), + (int, float), + ) + if (type(value), field_type) not in whitelist: + raise ValueError(msg.format(field_name, field_type, type(value))) + + try: + value = field_type(value) + except ValueError as e: + e.message = msg.format(field_name, field_type, type(value)) + raise + return value + + +def get_cloudbuild(raw_config, args): + """Read and validate a cloudbuild recipe + + Args: + raw_config (dict): deserialized cloudbuild.yaml + args (argparse.Namespace): ccommand line flags + + Returns: + CloudBuild: valid configuration + """ + if not isinstance(raw_config, dict): + raise ValueError( + 'Expected {} contents to be of type "dict", but found type "{}"'. + format(args.config, type(raw_config))) + raw_steps = get_field_value(raw_config, 'steps', list) + if not raw_steps: + raise ValueError('No steps defined in {}'.format(args.config)) + steps = [get_step(raw_step) for raw_step in raw_steps] + return CloudBuild( + output_script=args.output_script, + run=args.run, + steps=steps, + ) + + +def get_step(raw_step): + """Read and validate a single cloudbuild step + + Args: + raw_step (dict): deserialized step + + Returns: + Step: valid build step + """ + if not isinstance(raw_step, dict): + raise ValueError( + 'Expected step to be of type "dict", but found type "{}"'. + format(type(raw_step))) + raw_args = get_field_value(raw_step, 'args', list) + args = [get_field_value(raw_args, i, str) + for i in range(len(raw_args))] + dir_ = get_field_value(raw_step, 'dir', str) + raw_env = get_field_value(raw_step, 'env', list) + env = [get_field_value(raw_env, i, str) + for i in range(len(raw_env))] + name = get_field_value(raw_step, 'name', str) + return Step( + args=args, + dir_=dir_, + env=env, + name=name, + ) + + +def generate_command(step): + """Generate a single shell command to run for a single cloudbuild step + + Args: + step (Step): Valid build step + + Returns: + [str]: A single shell command, expressed as a list of quoted tokens. + """ + quoted_args = [shlex.quote(arg) for arg in step.args] + quoted_env = [] + for env in step.env: + quoted_env.extend(['--env', shlex.quote(env)]) + quoted_name = shlex.quote(step.name) + workdir = '/workspace' + if step.dir_: + workdir = os.path.join(workdir, shlex.quote(step.dir_)) + process_args = [ + 'docker', + 'run', + '--volume', + '/var/run/docker.sock:/var/run/docker.sock', + '--volume', + '/root/.docker:/root/.docker', + '--volume', + '${HOST_WORKSPACE}:/workspace', + '--workdir', + workdir, + ] + quoted_env + [quoted_name] + quoted_args + return process_args + + +def generate_script(cloudbuild): + """Generate the contents of a shell script + + Args: + cloudbuild (CloudBuild): Valid cloudbuild configuration + + Returns: + (str): Contents of shell script + """ + outfile = io.StringIO() + outfile.write(BUILD_SCRIPT_HEADER) + docker_commands = [generate_command(step) for step in cloudbuild.steps] + for docker_command in docker_commands: + line = ' '.join(docker_command) + '\n\n' + outfile.write(line) + outfile.write(BUILD_SCRIPT_FOOTER) + s = outfile.getvalue() + outfile.close() + return s + + +def make_executable(path): + """Set executable bit(s) on file""" + # http://stackoverflow.com/questions/12791997 + mode = os.stat(path).st_mode + mode |= (mode & 0o444) >> 2 # copy R bits to X + os.chmod(path, mode) + + +def write_script(cloudbuild, contents): + """Write a shell script to a file.""" + print('Writing build script to {}'.format(cloudbuild.output_script)) + with open(cloudbuild.output_script, 'w', encoding='utf8') as outfile: + outfile.write(contents) + make_executable(cloudbuild.output_script) + + +def local_cloudbuild(args): + """Execute the steps of a cloudbuild.yaml locally + + Args: + args: command line flags as per parse_args + """ + # Load and parse cloudbuild.yaml + with open(args.config, 'r', encoding='utf8') as cloudbuild_file: + raw_config = yaml.safe_load(cloudbuild_file) + + # Determine configuration + cloudbuild = get_cloudbuild(raw_config, args) + + # Create shell script + contents = generate_script(cloudbuild) + write_script(cloudbuild, contents) + + # Run shell script + if cloudbuild.run: + args = [os.path.abspath(cloudbuild.output_script)] + subprocess.check_call(args) + + +def validate_arg_regex(flag_value, flag_regex): + """Check a named command line flag against a regular expression""" + if not re.match(flag_regex, flag_value): + raise argparse.ArgumentTypeError( + 'Value "{}" does not match pattern "{}"'.format( + flag_value, flag_regex.pattern)) + return flag_value + + +def parse_args(argv): + """Parse and validate command line flags""" + parser = argparse.ArgumentParser( + description='Process cloudbuild.yaml locally to build Docker images') + parser.add_argument( + '--config', + type=functools.partial( + validate_arg_regex, flag_regex=PRINTABLE_REGEX), + default='cloudbuild.yaml', + help='Path to cloudbuild.yaml file' + ) + parser.add_argument( + '--output_script', + type=functools.partial( + validate_arg_regex, flag_regex=PRINTABLE_REGEX), + help='Filename to write shell script to', + ) + parser.add_argument( + '--no-run', + action='store_false', + help='Create shell script but don\'t execute it', + dest='run', + ) + args = parser.parse_args(argv[1:]) + if not args.output_script: + args.output_script = args.config + "_local.sh" + return args + + +def main(): + args = parse_args(sys.argv) + local_cloudbuild(args) + + +if __name__ == '__main__': + main() diff --git a/scripts/local_cloudbuild_test.py b/scripts/local_cloudbuild_test.py new file mode 100755 index 00000000..481e5ae9 --- /dev/null +++ b/scripts/local_cloudbuild_test.py @@ -0,0 +1,322 @@ +#!/usr/bin/env python3 + +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit test for local_cloudbuild.py""" + +import argparse +import os +import re +import shutil +import subprocess +import tempfile +import unittest +import unittest.mock + +import yaml + +import local_cloudbuild + + +class ValidationUtilsTest(unittest.TestCase): + + def test_get_field_value(self): + valid_cases = ( + # Normal case, field present and correct type + ({ 'present': 1 }, 'present', int, 1), + ({ 'present': '1' }, 'present', str, '1'), + ({ 'present': [1] }, 'present', list, [1]), + ({ 'present': {1: 2} }, 'present', dict, {1: 2}), + # Missing field replaced by default + ({}, 'missing', str, ''), + # Valid conversions + ({ 'str_to_int': '1' }, 'str_to_int', int, 1), + ({ 'int_to_str': 1 }, 'int_to_str', str, '1'), + ) + for valid_case in valid_cases: + with self.subTest(valid_case=valid_case): + container, field_name, field_type, expected = valid_case + self.assertEqual( + local_cloudbuild.get_field_value( + container, field_name, field_type), + expected) + + invalid_cases = ( + # Type conversion failures + ({ 'bad_list_to_dict': [1] }, 'bad_list_to_dict', dict), + ({ 'bad_list_to_str': [1] }, 'bad_list_to_str', str), + ({ 'bad_dict_to_list': {1: 2} }, 'bad_dict_to_list', list), + ({ 'bad_str_to_int': 'not_an_int' }, 'bad_str_to_int', int), + ({ 'bad_str_to_list': 'abc' }, 'bad_str_to_list', list), + ) + for invalid_case in invalid_cases: + with self.subTest(invalid_case=invalid_case): + container, field_name, field_type = invalid_case + with self.assertRaises(ValueError): + local_cloudbuild.get_field_value( + container, field_name, field_type) + + def test_validate_arg_regex(self): + self.assertEqual( + local_cloudbuild.validate_arg_regex('abc', re.compile('a[b]c')), + 'abc') + with self.assertRaises(argparse.ArgumentTypeError): + local_cloudbuild.validate_arg_regex('abc', re.compile('a[d]c')) + + +class LocalCloudbuildTest(unittest.TestCase): + + def setUp(self): + self.testdata_dir = 'testdata' + assert os.path.isdir(self.testdata_dir), 'Could not run test: testdata directory not found' + + def test_get_cloudbuild(self): + args = argparse.Namespace( + config='some_config_file', + output_script='some_output_script', + run=False, + ) + # Basic valid case + valid_case = 'steps:\n- name: step1\n- name: step2\n' + raw_config = yaml.safe_load(valid_case) + actual = local_cloudbuild.get_cloudbuild(raw_config, args) + self.assertEqual(len(actual.steps), 2) + + invalid_cases = ( + # Empty cloud build + '', + # No steps + 'foo: bar\n', + # Steps not a list + 'steps: astring\n', + ) + for invalid_case in invalid_cases: + with self.subTest(invalid_case=invalid_case): + raw_config = yaml.safe_load(invalid_case) + with self.assertRaises(ValueError): + local_cloudbuild.get_cloudbuild(raw_config, args) + + def test_get_step(self): + valid_cases = ( + # Empty step + ({}, local_cloudbuild.Step( + args=[], + dir_='', + env=[], + name='', + )), + # Full step + ({'name' : 'aname', + 'args' : [ 'arg1', 2, 'arg3 with \n newline', ], + 'env' : [ 'ENV1=value1', 'ENV2=space in value2' ], + 'dir' : 'adir', + }, local_cloudbuild.Step( + args = [ 'arg1', '2', 'arg3 with \n newline', ], + env = [ 'ENV1=value1', 'ENV2=space in value2' ], + dir_ = 'adir', + name = 'aname', + )), + ) + for valid_case in valid_cases: + with self.subTest(valid_case=valid_case): + raw_step, expected = valid_case + actual = local_cloudbuild.get_step(raw_step) + self.assertEqual(actual, expected) + + invalid_cases = ( + # Wrong type + [], + # More wrong types + {'args': 'not_a_list'}, + {'args': [ [] ]}, + {'env': 'not_a_list'}, + {'env': [ {} ]}, + {'dir': {}}, + {'name': []}, + ) + for invalid_case in invalid_cases: + with self.subTest(invalid_case=invalid_case): + with self.assertRaises(ValueError): + local_cloudbuild.get_step(invalid_case) + + def test_generate_command(self): + # Basic valid case + base_step = local_cloudbuild.Step( + args = ['arg1','arg2'], + dir_ = '', + env = ['ENV1=value1', 'ENV2=value2'], + name = 'aname', + ) + command = local_cloudbuild.generate_command(base_step) + self.assertEqual(command, [ + 'docker', + 'run', + '--volume', + '/var/run/docker.sock:/var/run/docker.sock', + '--volume', + '/root/.docker:/root/.docker', + '--volume', + '${HOST_WORKSPACE}:/workspace', + '--workdir', + '/workspace', + '--env', + 'ENV1=value1', + '--env', + 'ENV2=value2', + 'aname', + 'arg1', + 'arg2', + ]) + + # dir specified + step = base_step._replace(dir_='adir') + command = local_cloudbuild.generate_command(step) + self.assertIn('--workdir', command) + self.assertIn('/workspace/adir', command) + + # Shell quoting + step = base_step._replace(args=['arg with \n newline']) + command = local_cloudbuild.generate_command(step) + self.assertIn("'arg with \n newline'", command) + + step = base_step._replace(dir_='dir/ with space/') + command = local_cloudbuild.generate_command(step) + self.assertIn("/workspace/'dir/ with space/'", command) + + step = base_step._replace(env=['env with space']) + command = local_cloudbuild.generate_command(step) + self.assertIn("'env with space'", command) + + step = base_step._replace(name='a name') + command = local_cloudbuild.generate_command(step) + self.assertIn("'a name'", command) + + def test_generate_script(self): + config_name = 'cloudbuild_ok.yaml' + config = os.path.join(self.testdata_dir, config_name) + expected_output_script = os.path.join(self.testdata_dir, config_name + '_golden.sh') + cloudbuild = local_cloudbuild.CloudBuild( + output_script='test_generate_script', + run=False, + steps=[ + local_cloudbuild.Step( + args=['/bin/sh', '-c', 'echo "${MESSAGE}"'], + dir_='', + env=['MESSAGE=Hello World!'], + name='debian', + ), + local_cloudbuild.Step( + args=['/bin/sh', '-c', 'echo "${MESSAGE}"'], + dir_='', + env=['MESSAGE=Goodbye\\n And Farewell!', 'UNUSED=unused'], + name='debian', + ) + ] + ) + actual = local_cloudbuild.generate_script(cloudbuild) + self.maxDiff = 2**16 + # Compare output against golden + with open(expected_output_script, 'r', encoding='utf8') as expected: + self.assertEqual(actual, expected.read()) + + def test_make_executable(self): + with tempfile.TemporaryDirectory( + prefix='local_cloudbuild_test_') as tempdir: + test_script_filename = os.path.join(tempdir, 'test_make_executable.sh') + with open(test_script_filename, 'w', encoding='utf8') as test_script: + test_script.write('#!/bin/sh\necho "Output from test_make_executable"') + local_cloudbuild.make_executable(test_script_filename) + output = subprocess.check_output([test_script_filename]) + self.assertEqual(output.decode('utf8'), "Output from test_make_executable\n") + + def test_write_script(self): + with tempfile.TemporaryDirectory( + prefix='local_cloudbuild_test_') as tempdir: + contents = 'The contents\n' + output_script_filename = os.path.join(tempdir, 'test_write_script') + cloudbuild = local_cloudbuild.CloudBuild( + output_script=output_script_filename, + run=False, + steps=[], + ) + local_cloudbuild.write_script(cloudbuild, contents) + with open(output_script_filename, 'r', encoding='utf8') as output_script: + actual = output_script.read() + self.assertEqual(actual, contents) + + def test_local_cloudbuild(self): + # Actually run it if we can find a docker command. + should_run = False + if ((shutil.which('docker') is not None) and + (subprocess.call(['docker', 'info'], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) == 0)): + should_run = True + + # Read cloudbuild.yaml from testdata file, write output to + # tempdir, and maybe try to run it + with tempfile.TemporaryDirectory( + prefix='local_cloudbuild_test_') as tempdir: + cases = ( + # Everything is ok + ('cloudbuild_ok.yaml', True), + # Exit code 1 (failure) + ('cloudbuild_err_rc1.yaml', False), + # Command not found + ('cloudbuild_err_not_found.yaml', False), + ) + for case in cases: + with self.subTest(case=cases): + config_name, should_succeed = case + config = os.path.join(self.testdata_dir, config_name) + actual_output_script = os.path.join( + tempdir, config_name + '_local.sh') + args = argparse.Namespace( + config=config, + output_script=actual_output_script, + run=should_run) + if should_run: + print("Executing docker commands in {}".format(actual_output_script)) + if should_succeed: + local_cloudbuild.local_cloudbuild(args) + else: + with self.assertRaises(subprocess.CalledProcessError): + local_cloudbuild.local_cloudbuild(args) + else: + # Generate but don't execute script + local_cloudbuild.local_cloudbuild(args) + + + def test_parse_args(self): + # Test explicit output_script + argv = ['argv0', '--output_script=my_output'] + args = local_cloudbuild.parse_args(argv) + self.assertEqual(args.output_script, 'my_output') + # Test implicit output_script + argv = ['argv0', '--config=my_config'] + args = local_cloudbuild.parse_args(argv) + self.assertEqual(args.output_script, 'my_config_local.sh') + + # Test run flag (default and --no-run) + argv = ['argv0'] + args = local_cloudbuild.parse_args(argv) + self.assertEqual(args.run, True) + argv = ['argv0', '--no-run'] + args = local_cloudbuild.parse_args(argv) + self.assertEqual(args.run, False) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/testdata/cloudbuild_err_not_found.yaml b/scripts/testdata/cloudbuild_err_not_found.yaml new file mode 100644 index 00000000..c7eb070d --- /dev/null +++ b/scripts/testdata/cloudbuild_err_not_found.yaml @@ -0,0 +1,3 @@ +steps: +- name: debian + args: ['/expected file not found'] diff --git a/scripts/testdata/cloudbuild_err_rc1.yaml b/scripts/testdata/cloudbuild_err_rc1.yaml new file mode 100644 index 00000000..3953a586 --- /dev/null +++ b/scripts/testdata/cloudbuild_err_rc1.yaml @@ -0,0 +1,3 @@ +steps: +- name: debian + args: ['/bin/sh', '-c', 'exit 1'] diff --git a/scripts/testdata/cloudbuild_ok.yaml b/scripts/testdata/cloudbuild_ok.yaml new file mode 100644 index 00000000..8478269b --- /dev/null +++ b/scripts/testdata/cloudbuild_ok.yaml @@ -0,0 +1,7 @@ +steps: +- name: debian + args: ['/bin/sh', '-c', 'echo "${MESSAGE}"'] + env: ['MESSAGE=Hello World!'] +- name: debian + args: ['/bin/sh', '-c', 'echo "${MESSAGE}"'] + env: ['MESSAGE=Goodbye\n And Farewell!', 'UNUSED=unused'] diff --git a/scripts/testdata/cloudbuild_ok.yaml_golden.sh b/scripts/testdata/cloudbuild_ok.yaml_golden.sh new file mode 100755 index 00000000..996c436e --- /dev/null +++ b/scripts/testdata/cloudbuild_ok.yaml_golden.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# This is a generated file. Do not edit. + +set -euo pipefail + +SOURCE_DIR=. + +# Setup staging directory +HOST_WORKSPACE=$(mktemp -d) +function cleanup { + if [ "${HOST_WORKSPACE}" != '/' -a -d "${HOST_WORKSPACE}" ]; then + rm -rf "${HOST_WORKSPACE}" + fi +} +trap cleanup EXIT + +# Copy source to staging directory +echo "Copying source to staging directory ${HOST_WORKSPACE}" +rsync -avzq --exclude=.git "${SOURCE_DIR}" "${HOST_WORKSPACE}" + +# Build commands +docker run --volume /var/run/docker.sock:/var/run/docker.sock --volume /root/.docker:/root/.docker --volume ${HOST_WORKSPACE}:/workspace --workdir /workspace --env 'MESSAGE=Hello World!' debian /bin/sh -c 'echo "${MESSAGE}"' + +docker run --volume /var/run/docker.sock:/var/run/docker.sock --volume /root/.docker:/root/.docker --volume ${HOST_WORKSPACE}:/workspace --workdir /workspace --env 'MESSAGE=Goodbye\n And Farewell!' --env UNUSED=unused debian /bin/sh -c 'echo "${MESSAGE}"' + +# End of build commands + +echo "Build completed successfully" From 5b68d6b903a41057d8b8a2ccaeab1abd54933416 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 17 Feb 2017 11:01:12 -0800 Subject: [PATCH 010/256] Style fixes --- scripts/local_cloudbuild.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/local_cloudbuild.py b/scripts/local_cloudbuild.py index 53ae86e0..d67d5727 100755 --- a/scripts/local_cloudbuild.py +++ b/scripts/local_cloudbuild.py @@ -96,7 +96,7 @@ def get_field_value(container, field_name, field_type): field_type (type): Expected type for field value Returns: - any: Fetched or default value of field + Any: Fetched or default value of field Raises: ValueError: if field value cannot be converted to the desired type @@ -143,9 +143,11 @@ def get_cloudbuild(raw_config, args): raise ValueError( 'Expected {} contents to be of type "dict", but found type "{}"'. format(args.config, type(raw_config))) + raw_steps = get_field_value(raw_config, 'steps', list) if not raw_steps: raise ValueError('No steps defined in {}'.format(args.config)) + steps = [get_step(raw_step) for raw_step in raw_steps] return CloudBuild( output_script=args.output_script, @@ -180,7 +182,7 @@ def get_step(raw_step): dir_=dir_, env=env, name=name, - ) + ) def generate_command(step): From 3437eac89095d3280a3fcde2c87b6dfc85b5d0d3 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 17 Feb 2017 11:19:07 -0800 Subject: [PATCH 011/256] Fix style nits --- scripts/local_cloudbuild.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/local_cloudbuild.py b/scripts/local_cloudbuild.py index d67d5727..f7904570 100755 --- a/scripts/local_cloudbuild.py +++ b/scripts/local_cloudbuild.py @@ -170,12 +170,12 @@ def get_step(raw_step): 'Expected step to be of type "dict", but found type "{}"'. format(type(raw_step))) raw_args = get_field_value(raw_step, 'args', list) - args = [get_field_value(raw_args, i, str) - for i in range(len(raw_args))] + args = [get_field_value(raw_args, index, str) + for index in range(len(raw_args))] dir_ = get_field_value(raw_step, 'dir', str) raw_env = get_field_value(raw_step, 'env', list) - env = [get_field_value(raw_env, i, str) - for i in range(len(raw_env))] + env = [get_field_value(raw_env, index, str) + for index in range(len(raw_env))] name = get_field_value(raw_step, 'name', str) return Step( args=args, From e44b310c2f2b20c2137d454f458868c18a8b51a5 Mon Sep 17 00:00:00 2001 From: Nick Kubala Date: Thu, 2 Mar 2017 13:58:20 -0800 Subject: [PATCH 012/256] add support for logging at an explicit level passed from the client. update requirement versions --- tests/integration/requirements.txt | 8 ++++---- tests/integration/server.py | 21 ++++++++++----------- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index 369de14a..dbd26d30 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -13,9 +13,9 @@ # limitations under the License. Flask==0.11.1 -google-cloud-logging==0.21.0 -google-cloud-monitoring==0.21.0 -google-cloud-error-reporting==0.21.0 +google-cloud-logging==0.22.0 +google-cloud-monitoring==0.22.0 +google-cloud-error-reporting==0.22.0 gunicorn==19.6.0 retrying==1.3.3 -requests==2.2.1 +requests==2.13.0 diff --git a/tests/integration/server.py b/tests/integration/server.py index 528ebefa..8083cbf7 100755 --- a/tests/integration/server.py +++ b/tests/integration/server.py @@ -52,13 +52,16 @@ def _logging(request_data, token): log_name = request_data.get('log_name') if not log_name: raise ErrorResponse('Please provide log name') + level = request_data.get('level') + if not level: + raise ErrorResponse('Please provide log level') - _log(token, log_name) + _log(token, log_name, level) return 'OK', 200 -def _log(token, log_name='stdout'): +def _log(token, log_name, level): """ Write a log entry to Stackdriver. @@ -66,22 +69,18 @@ def _log(token, log_name='stdout'): token -- 16-character (8-byte) hexadecimal token, to be written as a log entry. log_name -- The name of the logging group to be written to. + level -- enum(LogSeverity), level of the log to write Once the entry is written to Stackdriver, the test driver will retrieve - all entries with the name 'log_name', and verify there is an entry with - the same value as 'token', indicating the entry was written successfully. + all entries with the name 'log_name' at level 'level', and verify there + is an entry with the same value as 'token', indicating the entry + was written successfully. """ - # TODO (nkubala): just as a note, currently the client logging API is - # broken - - # TODO (nkubala): write token to 'log_name' log, instead of stdout - # is this possible in non-standard (flex)??? - try: client = google.cloud.logging.Client() gcloud_logger = client.logger(log_name) - gcloud_logger.log_text(token) + gcloud_logger.log_text(token, severity=str(level)) except google.cloud.exceptions.GoogleCloudError as e: logging.error('Error while writing logs: {0}'.format(e)) raise ErrorResponse('Error while writing logs: {0}'.format(e)) From 87a55c3824eda4460efaf84b47eda90f335e3807 Mon Sep 17 00:00:00 2001 From: slhawkins Date: Tue, 7 Mar 2017 03:11:17 -0600 Subject: [PATCH 013/256] Update README.md Correction to the docker command that opens an interactive shell session. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4a35d047..3cef3315 100644 --- a/README.md +++ b/README.md @@ -102,7 +102,7 @@ $ make local-test To open an interactive shell session to this image after building it, do the following: ``` shell -docker run --it --entrypoint /bin/bash ${IMAGE_NAME} +docker run -it --entrypoint /bin/bash ${IMAGE_NAME} ``` ## Running the system tests From 03dd7e65df4babc4ffea32d6ebca61e764b7e710 Mon Sep 17 00:00:00 2001 From: Nick Kubala Date: Thu, 9 Mar 2017 14:51:09 -0800 Subject: [PATCH 014/256] add support for testing logging through standard module and custom client --- tests/integration/server.py | 74 +++++++++++++++++++++++++++++++------ 1 file changed, 62 insertions(+), 12 deletions(-) diff --git a/tests/integration/server.py b/tests/integration/server.py index 8083cbf7..eaeb992c 100755 --- a/tests/integration/server.py +++ b/tests/integration/server.py @@ -1,6 +1,6 @@ #!/usr/bin/python -# Copyright 2016 Google Inc. All rights reserved. +# Copyright 2017 Google Inc. All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -24,6 +24,19 @@ from flask import Flask, request, jsonify +# set up logging module to write to Stackdriver +client = google.cloud.logging.Client() +client.setup_logging(log_level=logging.DEBUG) +logging.getLogger().setLevel(logging.DEBUG) + +log_funcs = { + 'DEBUG': (logging.debug, 'stderr'), + 'INFO': (logging.info, 'stderr'), + 'WARNING': (logging.warn, 'stderr'), + 'ERROR': (logging.error, 'stderr'), + 'CRITICAL': (logging.critical, 'stderr') +} + app = Flask(__name__) @@ -46,9 +59,9 @@ def hello_world(): return 'Hello World!' -@app.route('/logging', methods=['POST']) +@app.route('/logging_custom', methods=['POST']) @verify_request -def _logging(request_data, token): +def _logging_custom(request_data, token): log_name = request_data.get('log_name') if not log_name: raise ErrorResponse('Please provide log name') @@ -56,14 +69,26 @@ def _logging(request_data, token): if not level: raise ErrorResponse('Please provide log level') - _log(token, log_name, level) + log_source = _log_custom(token, log_name, level) + + return log_source, 200 - return 'OK', 200 +@app.route('/logging_standard', methods=['POST']) +@verify_request +def _logging_standard(request_data, token): + level = request_data.get('level') + if not level: + raise ErrorResponse('Please provide log level') -def _log(token, log_name, level): + log_source = _log_default(token, level) + + return log_source, 200 + + +def _log_custom(token, log_name, level): """ - Write a log entry to Stackdriver. + Write a custom log entry to Stackdriver using a client library. Keyword arguments: token -- 16-character (8-byte) hexadecimal token, to be written @@ -72,11 +97,10 @@ def _log(token, log_name, level): level -- enum(LogSeverity), level of the log to write Once the entry is written to Stackdriver, the test driver will retrieve - all entries with the name 'log_name' at level 'level', and verify there - is an entry with the same value as 'token', indicating the entry + all entries with the name 'log_name' at level 'level', and verify there + is an entry with the same value as 'token', indicating the entry was written successfully. """ - try: client = google.cloud.logging.Client() gcloud_logger = client.logger(log_name) @@ -85,8 +109,34 @@ def _log(token, log_name, level): logging.error('Error while writing logs: {0}'.format(e)) raise ErrorResponse('Error while writing logs: {0}'.format(e)) - logging.debug(token) - print(token) + return log_name + + +def _log_default(token, level): + """ + Write a log entry to Stackdriver through the default logging module. + + Keyword arguments: + token -- 16-character (8-byte) hexadecimal token, to be written + as a log entry. + level -- enum(LogSeverity), level of the log to write + + Once the entry is written to Stackdriver, the test driver will retrieve + all entries from the default log stream (sent back to the driver) at level + 'level', and verify there is an entry with the same value as 'token', + indicating the entry was written successfully. + """ + + try: + func_pair = log_funcs[level] + f = func_pair[0] + source = func_pair[1] + f(token) + except google.cloud.exceptions.GoogleCloudError as e: + logging.error('Error while writing logs: {0}'.format(e)) + raise ErrorResponse('Error while writing logs: {0}'.format(e)) + + return 'appengine.googleapis.com%2F{0}'.format(source) @app.route('/monitoring', methods=['POST']) From 4dae3bb3a92a7f0c3b9397dad9b7b53458f24d27 Mon Sep 17 00:00:00 2001 From: Nick Kubala Date: Tue, 14 Mar 2017 14:19:44 -0700 Subject: [PATCH 015/256] move client setup code to __name__ check block --- tests/integration/server.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/integration/server.py b/tests/integration/server.py index eaeb992c..ff675046 100755 --- a/tests/integration/server.py +++ b/tests/integration/server.py @@ -24,11 +24,6 @@ from flask import Flask, request, jsonify -# set up logging module to write to Stackdriver -client = google.cloud.logging.Client() -client.setup_logging(log_level=logging.DEBUG) -logging.getLogger().setLevel(logging.DEBUG) - log_funcs = { 'DEBUG': (logging.debug, 'stderr'), 'INFO': (logging.info, 'stderr'), @@ -247,4 +242,8 @@ def handle_invalid_usage(error): if __name__ == '__main__': + # set up logging module to write to Stackdriver + client = google.cloud.logging.Client() + client.setup_logging(log_level=logging.DEBUG) + logging.getLogger().setLevel(logging.DEBUG) app.run(debug=True, port=8080) From 895dc4540b248a78fc8af6f087f14dc6369753d7 Mon Sep 17 00:00:00 2001 From: Nick Kubala Date: Tue, 14 Mar 2017 14:02:44 -0700 Subject: [PATCH 016/256] add sample custom integration test --- tests/integration/server.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/integration/server.py b/tests/integration/server.py index ff675046..cc82fb93 100755 --- a/tests/integration/server.py +++ b/tests/integration/server.py @@ -15,6 +15,7 @@ # limitations under the License. from functools import wraps +import json import logging import google.cloud.logging @@ -218,6 +219,18 @@ def _trace(): return 'OK', 204 +@app.route('/custom', methods=['GET']) +def _custom(): + tests = [ + { + 'name': 'foo', + 'path': '/', + 'timeout': 500 + } + ] + return json.dumps(tests), 200 + + class ErrorResponse(Exception): status_code = 400 From 2ed27358d02f78175cdb258dda61fdf16ce388c1 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Mon, 6 Mar 2017 18:22:31 -0800 Subject: [PATCH 017/256] Support the --substitutions flag as per 'gcloud container builds submit' --- scripts/local_cloudbuild.py | 92 ++++++++++- scripts/local_cloudbuild_test.py | 151 +++++++++++++++--- .../cloudbuild_builtin_substitutions.yaml | 11 ++ scripts/testdata/cloudbuild_ok.yaml | 4 +- scripts/testdata/cloudbuild_ok.yaml_golden.sh | 4 +- .../cloudbuild_user_substitutions.yaml | 11 ++ 6 files changed, 239 insertions(+), 34 deletions(-) create mode 100644 scripts/testdata/cloudbuild_builtin_substitutions.yaml create mode 100644 scripts/testdata/cloudbuild_user_substitutions.yaml diff --git a/scripts/local_cloudbuild.py b/scripts/local_cloudbuild.py index f7904570..78759ffb 100755 --- a/scripts/local_cloudbuild.py +++ b/scripts/local_cloudbuild.py @@ -21,7 +21,9 @@ commands are output as a shell script and optionally executed. The output images are not pushed to the Google Container Registry. -Not all cloudbuild.yaml functionality is supported. +Not all cloudbuild.yaml functionality is supported. In particular, +substitutions are a simplified subset that doesn't include all the +corner cases and error conditions. See https://cloud.google.com/container-builder/docs/api/build-steps for more information. @@ -44,6 +46,28 @@ # Exclude non-printable control characters (including newlines) PRINTABLE_REGEX = re.compile(r"""^[^\x00-\x1f]*$""") +# Container Builder substitutions +# https://cloud.google.com/container-builder/docs/api/build-requests#substitutions +SUBSTITUTION_REGEX = re.compile(r"""(?x) + (? Date: Tue, 7 Mar 2017 17:10:48 -0800 Subject: [PATCH 018/256] Add BUILD_ID and COMMIT_SHA builtin substitutions. --- scripts/local_cloudbuild.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/local_cloudbuild.py b/scripts/local_cloudbuild.py index 78759ffb..7bdd4fda 100755 --- a/scripts/local_cloudbuild.py +++ b/scripts/local_cloudbuild.py @@ -61,11 +61,13 @@ # Default builtin substitutions DEFAULT_SUBSTITUTIONS = { + 'BRANCH_NAME': '', + 'BUILD_ID': 'abcdef12-3456-7890-abcd-ef0123456789', + 'COMMIT_SHA': '', 'PROJECT_ID': 'dummy-project-id', 'REPO_NAME': '', - 'BRANCH_NAME': '', - 'TAG_NAME': '', 'REVISION_ID': '', + 'TAG_NAME': '', } # File template From e6dfe0cd53edfbf679546f5d15de7c5ce27c423f Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Tue, 7 Mar 2017 17:21:40 -0800 Subject: [PATCH 019/256] Substitition names must begin with a letter or underscore. --- scripts/local_cloudbuild.py | 13 ++++++++----- scripts/local_cloudbuild_test.py | 5 +++++ 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/scripts/local_cloudbuild.py b/scripts/local_cloudbuild.py index 7bdd4fda..08b9558e 100755 --- a/scripts/local_cloudbuild.py +++ b/scripts/local_cloudbuild.py @@ -49,15 +49,18 @@ # Container Builder substitutions # https://cloud.google.com/container-builder/docs/api/build-requests#substitutions SUBSTITUTION_REGEX = re.compile(r"""(?x) - (? Date: Tue, 7 Mar 2017 18:30:10 -0800 Subject: [PATCH 020/256] Validate that all user substitutions are used at least once --- scripts/local_cloudbuild.py | 58 +++++++++++++++++-------- scripts/local_cloudbuild_test.py | 74 +++++++++++++++++++------------- 2 files changed, 86 insertions(+), 46 deletions(-) diff --git a/scripts/local_cloudbuild.py b/scripts/local_cloudbuild.py index 08b9558e..21908543 100755 --- a/scripts/local_cloudbuild.py +++ b/scripts/local_cloudbuild.py @@ -112,8 +112,15 @@ Step = collections.namedtuple('Step', 'args dir_ env name') -def sub_and_quote(s, substitutions): - """Return a shell-escaped, variable substituted, version of the string s.""" +def sub_and_quote(s, substitutions, substitutions_used): + """Return a shell-escaped, variable substituted, version of the string s. + + Args: + s (str): Any string + subs (dict): Substitution map to apply + subs_used (set): Updated with names from `subs.keys()` when those + substitutions are encountered in `s` + """ def sub(match): """Perform a single substitution.""" @@ -133,6 +140,7 @@ def sub(match): value = '' else: value = substitutions.get(variable_name) + substitutions_used.add(variable_name) return value substituted_s = re.sub(SUBSTITUTION_REGEX, sub, s) @@ -242,24 +250,29 @@ def get_step(raw_step): ) -def generate_command(step, subs): +def generate_command(step, substitutions, substitutions_used): """Generate a single shell command to run for a single cloudbuild step Args: step (Step): Valid build step subs (dict): Substitution map to apply + subs_used (set): Updated with names from `subs.keys()` when those + substitutions are encountered in an element of `step` Returns: [str]: A single shell command, expressed as a list of quoted tokens. """ - quoted_args = [sub_and_quote(arg, subs) for arg in step.args] + quoted_args = [sub_and_quote(arg, substitutions, substitutions_used) + for arg in step.args] quoted_env = [] for env in step.env: - quoted_env.extend(['--env', sub_and_quote(env, subs)]) - quoted_name = sub_and_quote(step.name, subs) + quoted_env.extend(['--env', sub_and_quote(env, substitutions, + substitutions_used)]) + quoted_name = sub_and_quote(step.name, substitutions, substitutions_used) workdir = '/workspace' if step.dir_: - workdir = os.path.join(workdir, sub_and_quote(step.dir_, subs)) + workdir = os.path.join(workdir, sub_and_quote(step.dir_, substitutions, + substitutions_used)) process_args = [ 'docker', 'run', @@ -284,16 +297,27 @@ def generate_script(cloudbuild): Returns: (str): Contents of shell script """ - outfile = io.StringIO() - outfile.write(BUILD_SCRIPT_HEADER) - docker_commands = [generate_command(step, cloudbuild.substitutions) - for step in cloudbuild.steps] - for docker_command in docker_commands: - line = ' '.join(docker_command) + '\n\n' - outfile.write(line) - outfile.write(BUILD_SCRIPT_FOOTER) - s = outfile.getvalue() - outfile.close() + with io.StringIO() as outfile: + outfile.write(BUILD_SCRIPT_HEADER) + subs_used = set() + docker_commands = [ + generate_command(step, cloudbuild.substitutions, subs_used) + for step in cloudbuild.steps] + for docker_command in docker_commands: + line = ' '.join(docker_command) + '\n\n' + outfile.write(line) + outfile.write(BUILD_SCRIPT_FOOTER) + s = outfile.getvalue() + + # Check that all user variables were referenced at least once + user_subs_unused = [name for name in cloudbuild.substitutions.keys() + if name not in subs_used and name[0] == '_'] + if user_subs_unused: + nice_list = '"' + '", "'.join(sorted(user_subs_unused)) + '"' + raise ValueError( + 'User substitution variables {} were defined in the --substitution ' + 'flag but never used in the cloudbuild file.'.format(nice_list)) + return s diff --git a/scripts/local_cloudbuild_test.py b/scripts/local_cloudbuild_test.py index abb47da9..f6138fbe 100755 --- a/scripts/local_cloudbuild_test.py +++ b/scripts/local_cloudbuild_test.py @@ -120,32 +120,37 @@ def setUp(self): def test_sub_and_quote(self): valid_cases = ( # Empty string - ('', {}, "''"), + ('', {}, "''", []), # No substitutions - ('a', {}, 'a'), - # Unused substitutions - ('a', {'FOO':'foo'}, 'a'), + ('a', {}, 'a', []), + # Unused builtin substitutions are fine + ('a', {'FOO':'foo'}, 'a', []), + # Unused user substitition (ok here but error in generate_script) + ('a', {'_FOO':'_foo'}, 'a', []), # Defined builtin substitution - ('a$FOOb', {'FOO':'foo'}, 'afoob'), - ('a${FOO}b', {'FOO':'foo'}, 'afoob'), + ('a$FOOb', {'FOO':'foo'}, 'afoob', ['FOO']), + ('a${FOO}b', {'FOO':'foo'}, 'afoob', ['FOO']), # Undefined builtin substitution - ('a$FOOb', {}, 'ab'), - ('a${FOO}b', {}, 'ab'), + ('a$FOOb', {}, 'ab', ['FOO']), + ('a${FOO}b', {}, 'ab', ['FOO']), # Defined user substitution - ('a$_FOOb', {'_FOO':'_foo'}, 'a_foob'), - ('a${_FOO}b', {'_FOO':'_foo'}, 'a_foob'), + ('a$_FOOb', {'_FOO':'_foo'}, 'a_foob', ['_FOO']), + ('a${_FOO}b', {'_FOO':'_foo'}, 'a_foob', ['_FOO']), # Multiple substitutions - ('$FOO${FOO}${BAR}$FOO', {'FOO':'foo', 'BAR':'bar'}, 'foofoobarfoo'), + ('$FOO${FOO}${BAR}$FOO', {'FOO':'foo', 'BAR':'bar'}, + 'foofoobarfoo', ['FOO', 'BAR']), # Invalid names - ('a $ b', {}, "'a $ b'"), - ('a$foo b', {}, "'a$foo b'"), - ('a$0FOO b', {}, "'a$0FOO b'"), + ('a $ b', {}, "'a $ b'", []), + ('a$foo b', {}, "'a$foo b'", []), + ('a$0FOO b', {}, "'a$0FOO b'", []), ) for valid_case in valid_cases: with self.subTest(valid_case=valid_case): - s, subs, expected = valid_case - actual = local_cloudbuild.sub_and_quote(s, subs) + s, subs, expected, expected_used = valid_case + used = set() + actual = local_cloudbuild.sub_and_quote(s, subs, used) self.assertEqual(actual, expected) + self.assertEqual(used, set(expected_used)) invalid_cases = ( # Undefined user substitution @@ -156,7 +161,8 @@ def test_sub_and_quote(self): with self.subTest(invalid_case=invalid_case): s, subs = invalid_case with self.assertRaises(ValueError): - local_cloudbuild.sub_and_quote(s, subs) + used = set() + local_cloudbuild.sub_and_quote(s, subs, used) def test_get_cloudbuild(self): args = argparse.Namespace( @@ -237,7 +243,7 @@ def test_generate_command(self): name = 'aname', ) subs = {'BUILTIN':'builtin', '_USER':'_user'} - command = local_cloudbuild.generate_command(base_step, subs) + command = local_cloudbuild.generate_command(base_step, subs, set()) self.assertEqual(command, [ 'docker', 'run', @@ -260,49 +266,49 @@ def test_generate_command(self): # dir specified step = base_step._replace(dir_='adir') - command = local_cloudbuild.generate_command(step, subs) + command = local_cloudbuild.generate_command(step, subs, set()) self.assertIn('--workdir', command) self.assertIn('/workspace/adir', command) # Shell quoting step = base_step._replace(args=['arg with \n newline']) - command = local_cloudbuild.generate_command(step, subs) + command = local_cloudbuild.generate_command(step, subs, set()) self.assertIn("'arg with \n newline'", command) step = base_step._replace(dir_='dir/ with space/') - command = local_cloudbuild.generate_command(step, subs) + command = local_cloudbuild.generate_command(step, subs, set()) self.assertIn("/workspace/'dir/ with space/'", command) step = base_step._replace(env=['env with space']) - command = local_cloudbuild.generate_command(step, subs) + command = local_cloudbuild.generate_command(step, subs, set()) self.assertIn("'env with space'", command) step = base_step._replace(name='a name') - command = local_cloudbuild.generate_command(step, subs) + command = local_cloudbuild.generate_command(step, subs, set()) self.assertIn("'a name'", command) # Variable substitution step = base_step._replace(name='a $BUILTIN substitution') - command = local_cloudbuild.generate_command(step, subs) + command = local_cloudbuild.generate_command(step, subs, set()) self.assertIn("'a builtin substitution'", command) step = base_step._replace(name='a $UNSET_BUILTIN substitution') - command = local_cloudbuild.generate_command(step, subs) + command = local_cloudbuild.generate_command(step, subs, set()) self.assertIn("'a substitution'", command) step = base_step._replace(name='a $_USER substitution') - command = local_cloudbuild.generate_command(step, subs) + command = local_cloudbuild.generate_command(step, subs, set()) self.assertIn("'a _user substitution'", command) step = base_step._replace(name='a $_UNSET_USER substitution') with self.assertRaises(ValueError): - local_cloudbuild.generate_command(step, subs) + local_cloudbuild.generate_command(step, subs, set()) step = base_step._replace(name='a curly brace ${BUILTIN} substitution') - command = local_cloudbuild.generate_command(step, subs) + command = local_cloudbuild.generate_command(step, subs, set()) self.assertIn("'a curly brace builtin substitution'", command) - def test_generate_script(self): + def test_generate_script_golden(self): config_name = 'cloudbuild_ok.yaml' config = os.path.join(self.testdata_dir, config_name) expected_output_script = os.path.join(self.testdata_dir, config_name + '_golden.sh') @@ -331,6 +337,16 @@ def test_generate_script(self): with open(expected_output_script, 'r', encoding='utf8') as expected: self.assertEqual(actual, expected.read()) + def test_generate_script_unused_user_substitution(self): + cloudbuild = local_cloudbuild.CloudBuild( + output_script='', + run=False, + steps=[], + substitutions={'_FOO':'_foo'}, + ) + with self.assertRaisesRegexp(ValueError, 'User substitution variables'): + actual = local_cloudbuild.generate_script(cloudbuild) + def test_make_executable(self): with tempfile.TemporaryDirectory( prefix='local_cloudbuild_test_') as tempdir: From 4ed577487b3444681e50a94548f6317bd98d8dc8 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Wed, 15 Mar 2017 16:52:02 -0700 Subject: [PATCH 021/256] Update cloudbuild substitution rules * Escaping $ is done with $$ not \$ * Undefined builtin substitutions are now an error, instead of being replaced by the empty string --- scripts/local_cloudbuild.py | 23 +++++++++++------------ scripts/local_cloudbuild_test.py | 17 ++++++++++------- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/scripts/local_cloudbuild.py b/scripts/local_cloudbuild.py index 21908543..894be5ac 100755 --- a/scripts/local_cloudbuild.py +++ b/scripts/local_cloudbuild.py @@ -49,12 +49,13 @@ # Container Builder substitutions # https://cloud.google.com/container-builder/docs/api/build-requests#substitutions SUBSTITUTION_REGEX = re.compile(r"""(?x) - (? Date: Wed, 15 Mar 2017 17:37:35 -0700 Subject: [PATCH 022/256] Minor test cleanup --- scripts/local_cloudbuild_test.py | 86 +++++++++++++++----------------- 1 file changed, 39 insertions(+), 47 deletions(-) diff --git a/scripts/local_cloudbuild_test.py b/scripts/local_cloudbuild_test.py index f22001c8..b33065db 100755 --- a/scripts/local_cloudbuild_test.py +++ b/scripts/local_cloudbuild_test.py @@ -263,53 +263,45 @@ def test_generate_command(self): 'arg2', ]) - # dir specified - step = base_step._replace(dir_='adir') - command = local_cloudbuild.generate_command(step, subs, set()) - self.assertIn('--workdir', command) - self.assertIn('/workspace/adir', command) - - # Shell quoting - step = base_step._replace(args=['arg with \n newline']) - command = local_cloudbuild.generate_command(step, subs, set()) - self.assertIn("'arg with \n newline'", command) - - step = base_step._replace(dir_='dir/ with space/') - command = local_cloudbuild.generate_command(step, subs, set()) - self.assertIn("/workspace/'dir/ with space/'", command) - - step = base_step._replace(env=['env with space']) - command = local_cloudbuild.generate_command(step, subs, set()) - self.assertIn("'env with space'", command) - - step = base_step._replace(name='a name') - command = local_cloudbuild.generate_command(step, subs, set()) - self.assertIn("'a name'", command) - - # Variable substitution - step = base_step._replace(name='a $BUILTIN substitution') - command = local_cloudbuild.generate_command(step, subs, set()) - self.assertIn("'a builtin substitution'", command) - - step = base_step._replace(name='a $UNSET_BUILTIN substitution') - with self.assertRaises(ValueError): - command = local_cloudbuild.generate_command(step, subs, set()) - - step = base_step._replace(name='a $_USER substitution') - command = local_cloudbuild.generate_command(step, subs, set()) - self.assertIn("'a _user substitution'", command) - - step = base_step._replace(name='a $_UNSET_USER substitution') - with self.assertRaises(ValueError): - local_cloudbuild.generate_command(step, subs, set()) - - step = base_step._replace(name='a curly brace ${BUILTIN} substitution') - command = local_cloudbuild.generate_command(step, subs, set()) - self.assertIn("'a curly brace builtin substitution'", command) - - step = base_step._replace(name='an escaped $$ or $$$$ or $$BUILTIN or $${BUILTIN} is unescaped') - command = local_cloudbuild.generate_command(step, subs, set()) - self.assertIn("'an escaped $ or $$ or $BUILTIN or ${BUILTIN} is unescaped'", command) + valid_cases = ( + # dir specified + (base_step._replace(dir_='adir'), + ['--workdir', '/workspace/adir']), + # Shell quoting + (base_step._replace(args=['arg with \n newline']), + ["'arg with \n newline'"]), + (base_step._replace(dir_='dir/ with space/'), + ["/workspace/'dir/ with space/'"]), + (base_step._replace(env=['env with space']), + ["'env with space'"]), + (base_step._replace(name='a name'), + ["'a name'"]), + # Variable substitution + (base_step._replace(name='a $BUILTIN substitution'), + ["'a builtin substitution'"]), + (base_step._replace(name='a $_USER substitution'), + ["'a _user substitution'"]), + (base_step._replace(name='a curly brace ${BUILTIN} substitution'), + ["'a curly brace builtin substitution'"]), + (base_step._replace(name='an escaped $$ or $$$$ or $$FOO or $${_FOO} is unescaped'), + ["'an escaped $ or $$ or $FOO or ${_FOO} is unescaped'"]), + ) + for valid_case in valid_cases: + with self.subTest(valid_case=valid_case): + step, args = valid_case + command = local_cloudbuild.generate_command(step, subs, set()) + for arg in args: + self.assertIn(arg, command) + + invalid_cases = ( + base_step._replace(name='a $UNSET_BUILTIN substitution'), + base_step._replace(name='a $_UNSET_USER substitution'), + ) + for invalid_case in invalid_cases: + with self.subTest(invalid_case=invalid_case): + step = invalid_case + with self.assertRaises(ValueError): + local_cloudbuild.generate_command(step, subs, set()) def test_generate_script_golden(self): config_name = 'cloudbuild_ok.yaml' From 428e39f6ddb2017d56f0ea0e712ce9b4a3fdbc48 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Wed, 15 Mar 2017 19:13:47 -0700 Subject: [PATCH 023/256] Fix staging directory cleanup --- scripts/local_cloudbuild.py | 54 ++++++++++++------- scripts/local_cloudbuild_test.py | 31 +++++++---- .../cloudbuild_difficult_cleanup.yaml | 3 ++ scripts/testdata/cloudbuild_ok.yaml_golden.sh | 7 +-- 4 files changed, 64 insertions(+), 31 deletions(-) create mode 100644 scripts/testdata/cloudbuild_difficult_cleanup.yaml diff --git a/scripts/local_cloudbuild.py b/scripts/local_cloudbuild.py index f7904570..12edd125 100755 --- a/scripts/local_cloudbuild.py +++ b/scripts/local_cloudbuild.py @@ -44,8 +44,11 @@ # Exclude non-printable control characters (including newlines) PRINTABLE_REGEX = re.compile(r"""^[^\x00-\x1f]*$""") +# Use this image for cleanup actions +DEBIAN_IMAGE='gcr.io/google-appengine/debian8' + # File template -BUILD_SCRIPT_HEADER = """\ +BUILD_SCRIPT_TEMPLATE = """\ #!/bin/bash # This is a generated file. Do not edit. @@ -54,24 +57,22 @@ SOURCE_DIR=. # Setup staging directory -HOST_WORKSPACE=$(mktemp -d) -function cleanup { - if [ "${HOST_WORKSPACE}" != '/' -a -d "${HOST_WORKSPACE}" ]; then - rm -rf "${HOST_WORKSPACE}" +HOST_WORKSPACE=$(mktemp -d -t local_cloudbuild_XXXXXXXXXX) +function cleanup {{ + if [ "${{HOST_WORKSPACE}}" != '/' -a -d "${{HOST_WORKSPACE}}" ]; then + {cleanup_str} 2>/dev/null || true + rmdir "${{HOST_WORKSPACE}}" fi -} +}} trap cleanup EXIT # Copy source to staging directory -echo "Copying source to staging directory ${HOST_WORKSPACE}" -rsync -avzq --exclude=.git "${SOURCE_DIR}" "${HOST_WORKSPACE}" +echo "Copying source to staging directory ${{HOST_WORKSPACE}}" +rsync -avzq --exclude=.git "${{SOURCE_DIR}}" "${{HOST_WORKSPACE}}" # Build commands -""" - -BUILD_SCRIPT_FOOTER = """\ +{docker_str} # End of build commands - echo "Build completed successfully" """ @@ -226,15 +227,25 @@ def generate_script(cloudbuild): Returns: (str): Contents of shell script """ - outfile = io.StringIO() - outfile.write(BUILD_SCRIPT_HEADER) + # This deletes everything in /workspace including hidden files, + # but not /workspace itself + cleanup_step = Step( + args=['rm', '-rf', '/workspace'], + dir_='', + env=[], + name=DEBIAN_IMAGE, + ) + cleanup_command = generate_command(cleanup_step) + cleanup_str = ' '.join(cleanup_command) docker_commands = [generate_command(step) for step in cloudbuild.steps] + docker_lines = [] for docker_command in docker_commands: line = ' '.join(docker_command) + '\n\n' - outfile.write(line) - outfile.write(BUILD_SCRIPT_FOOTER) - s = outfile.getvalue() - outfile.close() + docker_lines.append(line) + docker_str = ''.join(docker_lines) + + s = BUILD_SCRIPT_TEMPLATE.format(cleanup_str=cleanup_str, + docker_str=docker_str) return s @@ -259,6 +270,9 @@ def local_cloudbuild(args): Args: args: command line flags as per parse_args + + Returns: + str: Output of build, or None if build not run """ # Load and parse cloudbuild.yaml with open(args.config, 'r', encoding='utf8') as cloudbuild_file: @@ -274,7 +288,9 @@ def local_cloudbuild(args): # Run shell script if cloudbuild.run: args = [os.path.abspath(cloudbuild.output_script)] - subprocess.check_call(args) + return subprocess.check_output(args) + else: + return None def validate_arg_regex(flag_value, flag_regex): diff --git a/scripts/local_cloudbuild_test.py b/scripts/local_cloudbuild_test.py index 481e5ae9..0cbb0ecc 100755 --- a/scripts/local_cloudbuild_test.py +++ b/scripts/local_cloudbuild_test.py @@ -82,6 +82,15 @@ def setUp(self): self.testdata_dir = 'testdata' assert os.path.isdir(self.testdata_dir), 'Could not run test: testdata directory not found' + def have_docker(self): + """Determine if the Docker daemon is present and usable""" + if ((shutil.which('docker') is not None) and + (subprocess.call(['docker', 'info'], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) == 0)): + return True + return False + def test_get_cloudbuild(self): args = argparse.Namespace( config='some_config_file', @@ -258,12 +267,7 @@ def test_write_script(self): def test_local_cloudbuild(self): # Actually run it if we can find a docker command. - should_run = False - if ((shutil.which('docker') is not None) and - (subprocess.call(['docker', 'info'], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL) == 0)): - should_run = True + should_run = self.have_docker() # Read cloudbuild.yaml from testdata file, write output to # tempdir, and maybe try to run it @@ -276,9 +280,11 @@ def test_local_cloudbuild(self): ('cloudbuild_err_rc1.yaml', False), # Command not found ('cloudbuild_err_not_found.yaml', False), + # Cleaning up files owned by root + ('cloudbuild_difficult_cleanup.yaml', True), ) for case in cases: - with self.subTest(case=cases): + with self.subTest(case=case): config_name, should_succeed = case config = os.path.join(self.testdata_dir, config_name) actual_output_script = os.path.join( @@ -288,9 +294,16 @@ def test_local_cloudbuild(self): output_script=actual_output_script, run=should_run) if should_run: - print("Executing docker commands in {}".format(actual_output_script)) + print('Executing docker commands in {}'.format(actual_output_script)) if should_succeed: - local_cloudbuild.local_cloudbuild(args) + output = local_cloudbuild.local_cloudbuild(args) + # Check that staging dir was cleaned up + staging_regex = re.compile( + b'(?m)Copying source to staging directory (.+)$') + match = re.search(staging_regex, output) + self.assertTrue(match, output) + staging_dir = match.group(1) + self.assertFalse(os.path.isdir(staging_dir), staging_dir) else: with self.assertRaises(subprocess.CalledProcessError): local_cloudbuild.local_cloudbuild(args) diff --git a/scripts/testdata/cloudbuild_difficult_cleanup.yaml b/scripts/testdata/cloudbuild_difficult_cleanup.yaml new file mode 100644 index 00000000..e76846d5 --- /dev/null +++ b/scripts/testdata/cloudbuild_difficult_cleanup.yaml @@ -0,0 +1,3 @@ +steps: +- name: debian + args: ['/bin/sh', '-c', 'mkdir root; umask 0000; touch root/deny_all.txt'] diff --git a/scripts/testdata/cloudbuild_ok.yaml_golden.sh b/scripts/testdata/cloudbuild_ok.yaml_golden.sh index 996c436e..a6d5d12e 100755 --- a/scripts/testdata/cloudbuild_ok.yaml_golden.sh +++ b/scripts/testdata/cloudbuild_ok.yaml_golden.sh @@ -6,10 +6,11 @@ set -euo pipefail SOURCE_DIR=. # Setup staging directory -HOST_WORKSPACE=$(mktemp -d) +HOST_WORKSPACE=$(mktemp -d -t local_cloudbuild_XXXXXXXXXX) function cleanup { if [ "${HOST_WORKSPACE}" != '/' -a -d "${HOST_WORKSPACE}" ]; then - rm -rf "${HOST_WORKSPACE}" + docker run --volume /var/run/docker.sock:/var/run/docker.sock --volume /root/.docker:/root/.docker --volume ${HOST_WORKSPACE}:/workspace --workdir /workspace gcr.io/google-appengine/debian8 rm -rf /workspace 2>/dev/null || true + rmdir "${HOST_WORKSPACE}" fi } trap cleanup EXIT @@ -23,6 +24,6 @@ docker run --volume /var/run/docker.sock:/var/run/docker.sock --volume /root/.do docker run --volume /var/run/docker.sock:/var/run/docker.sock --volume /root/.docker:/root/.docker --volume ${HOST_WORKSPACE}:/workspace --workdir /workspace --env 'MESSAGE=Goodbye\n And Farewell!' --env UNUSED=unused debian /bin/sh -c 'echo "${MESSAGE}"' -# End of build commands +# End of build commands echo "Build completed successfully" From 42e58f1d4514f65247822c9bba3d1bbccaaf3712 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Thu, 16 Mar 2017 15:18:27 -0700 Subject: [PATCH 024/256] Restore previous behavior of printing docker output. --- scripts/local_cloudbuild.py | 11 +-- scripts/local_cloudbuild_test.py | 91 +++++++++++-------- scripts/testdata/cloudbuild_ok.yaml_golden.sh | 4 +- 3 files changed, 59 insertions(+), 47 deletions(-) diff --git a/scripts/local_cloudbuild.py b/scripts/local_cloudbuild.py index 12edd125..b577aa9d 100755 --- a/scripts/local_cloudbuild.py +++ b/scripts/local_cloudbuild.py @@ -60,8 +60,10 @@ HOST_WORKSPACE=$(mktemp -d -t local_cloudbuild_XXXXXXXXXX) function cleanup {{ if [ "${{HOST_WORKSPACE}}" != '/' -a -d "${{HOST_WORKSPACE}}" ]; then + # Expect a single error message about /workspace busy {cleanup_str} 2>/dev/null || true - rmdir "${{HOST_WORKSPACE}}" + # Do not expect error messages here. Display but ignore any that happen. + rmdir "${{HOST_WORKSPACE}}" || true fi }} trap cleanup EXIT @@ -270,9 +272,6 @@ def local_cloudbuild(args): Args: args: command line flags as per parse_args - - Returns: - str: Output of build, or None if build not run """ # Load and parse cloudbuild.yaml with open(args.config, 'r', encoding='utf8') as cloudbuild_file: @@ -288,9 +287,7 @@ def local_cloudbuild(args): # Run shell script if cloudbuild.run: args = [os.path.abspath(cloudbuild.output_script)] - return subprocess.check_output(args) - else: - return None + subprocess.check_call(args) def validate_arg_regex(flag_value, flag_regex): diff --git a/scripts/local_cloudbuild_test.py b/scripts/local_cloudbuild_test.py index 0cbb0ecc..26ac4c0b 100755 --- a/scripts/local_cloudbuild_test.py +++ b/scripts/local_cloudbuild_test.py @@ -30,6 +30,10 @@ import local_cloudbuild +# Matches script boilerplate +STAGING_DIR_REGEX = re.compile( + b'(?m)Copying source to staging directory (.+)$') + class ValidationUtilsTest(unittest.TestCase): def test_get_field_value(self): @@ -82,6 +86,16 @@ def setUp(self): self.testdata_dir = 'testdata' assert os.path.isdir(self.testdata_dir), 'Could not run test: testdata directory not found' + def check_call_with_capture(self, *args, **kw_args): + """Act like subprocess.check_call but capture stdout""" + try: + self.check_call_output = subprocess.check_output(*args, **kw_args) + print(self.check_call_output) + except subprocess.CalledProcessError as e: + self.check_call_output = e.output + print(self.check_call_output) + raise + def have_docker(self): """Determine if the Docker daemon is present and usable""" if ((shutil.which('docker') is not None) and @@ -271,46 +285,45 @@ def test_local_cloudbuild(self): # Read cloudbuild.yaml from testdata file, write output to # tempdir, and maybe try to run it - with tempfile.TemporaryDirectory( - prefix='local_cloudbuild_test_') as tempdir: - cases = ( - # Everything is ok - ('cloudbuild_ok.yaml', True), - # Exit code 1 (failure) - ('cloudbuild_err_rc1.yaml', False), - # Command not found - ('cloudbuild_err_not_found.yaml', False), - # Cleaning up files owned by root - ('cloudbuild_difficult_cleanup.yaml', True), - ) - for case in cases: - with self.subTest(case=case): - config_name, should_succeed = case - config = os.path.join(self.testdata_dir, config_name) - actual_output_script = os.path.join( - tempdir, config_name + '_local.sh') - args = argparse.Namespace( - config=config, - output_script=actual_output_script, - run=should_run) - if should_run: - print('Executing docker commands in {}'.format(actual_output_script)) - if should_succeed: - output = local_cloudbuild.local_cloudbuild(args) - # Check that staging dir was cleaned up - staging_regex = re.compile( - b'(?m)Copying source to staging directory (.+)$') - match = re.search(staging_regex, output) - self.assertTrue(match, output) - staging_dir = match.group(1) - self.assertFalse(os.path.isdir(staging_dir), staging_dir) - else: - with self.assertRaises(subprocess.CalledProcessError): - local_cloudbuild.local_cloudbuild(args) - else: - # Generate but don't execute script + cases = ( + # Everything is ok + ('cloudbuild_ok.yaml', True), + # Exit code 1 (failure) + ('cloudbuild_err_rc1.yaml', False), + # Command not found + ('cloudbuild_err_not_found.yaml', False), + # Cleaning up files owned by root + ('cloudbuild_difficult_cleanup.yaml', True), + ) + for case in cases: + with self.subTest(case=case), \ + tempfile.TemporaryDirectory(prefix='local_cloudbuild_test_') as tempdir, \ + unittest.mock.patch('subprocess.check_call', self.check_call_with_capture): + config_name, should_succeed = case + config = os.path.join(self.testdata_dir, config_name) + actual_output_script = os.path.join( + tempdir, config_name + '_local.sh') + args = argparse.Namespace( + config=config, + output_script=actual_output_script, + run=should_run) + + if should_run: + print('Executing docker commands in {}'.format(actual_output_script)) + if should_succeed: local_cloudbuild.local_cloudbuild(args) - + else: + with self.assertRaises(subprocess.CalledProcessError): + local_cloudbuild.local_cloudbuild(args) + + # Check that staging dir was cleaned up + match = re.search(STAGING_DIR_REGEX, self.check_call_output) + self.assertTrue(match) + staging_dir = match.group(1) + self.assertFalse(os.path.isdir(staging_dir), staging_dir) + else: + # Generate but don't execute script + local_cloudbuild.local_cloudbuild(args) def test_parse_args(self): # Test explicit output_script diff --git a/scripts/testdata/cloudbuild_ok.yaml_golden.sh b/scripts/testdata/cloudbuild_ok.yaml_golden.sh index a6d5d12e..0b945fa9 100755 --- a/scripts/testdata/cloudbuild_ok.yaml_golden.sh +++ b/scripts/testdata/cloudbuild_ok.yaml_golden.sh @@ -9,8 +9,10 @@ SOURCE_DIR=. HOST_WORKSPACE=$(mktemp -d -t local_cloudbuild_XXXXXXXXXX) function cleanup { if [ "${HOST_WORKSPACE}" != '/' -a -d "${HOST_WORKSPACE}" ]; then + # Expect a single error message about /workspace busy docker run --volume /var/run/docker.sock:/var/run/docker.sock --volume /root/.docker:/root/.docker --volume ${HOST_WORKSPACE}:/workspace --workdir /workspace gcr.io/google-appengine/debian8 rm -rf /workspace 2>/dev/null || true - rmdir "${HOST_WORKSPACE}" + # Do not expect error messages here. Display but ignore any that happen. + rmdir "${HOST_WORKSPACE}" || true fi } trap cleanup EXIT From 7480ba51653ef4580b1c8a77423c27ed12c8d19a Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Tue, 21 Mar 2017 13:58:11 -0700 Subject: [PATCH 025/256] adding support for TAG environment variable (#82) --- jenkins_build.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/jenkins_build.sh b/jenkins_build.sh index f28196a7..253d5ca0 100755 --- a/jenkins_build.sh +++ b/jenkins_build.sh @@ -4,7 +4,11 @@ set -eu RUNTIME_NAME="python" -CANDIDATE_NAME=`date +%Y-%m-%d_%H_%M` +if [ -z "${TAG}" ] ; then + TAG=`date +%Y-%m-%d_%H_%M` +fi + +CANDIDATE_NAME="${TAG}" echo "CANDIDATE_NAME:${CANDIDATE_NAME}" if [ -z "${DOCKER_NAMESPACE+set}" ] ; then From 4f16a4714442a2c971e83ad4fcc66d1f275c7993 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Tue, 21 Mar 2017 14:55:33 -0700 Subject: [PATCH 026/256] Add nox.py to update requirements (#83) --- .gitignore | 2 ++ nox.py | 43 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 nox.py diff --git a/.gitignore b/.gitignore index 0c354cc9..c9aa6307 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ /cloudbuild.yaml_local.sh /ext_run.sh __pycache__ +.nox +*.pyc diff --git a/nox.py b/nox.py new file mode 100644 index 00000000..6d2e6723 --- /dev/null +++ b/nox.py @@ -0,0 +1,43 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import fnmatch +import os + +# Location of our common testing utilities. This isn't published to PyPI. +GCP_REPO_TOOLS_REQ =\ + 'git+https://github.com/GoogleCloudPlatform/python-repo-tools.git' + + +def _list_files(folder, pattern): + """Lists all files below the given folder that match the pattern.""" + for root, folders, files in os.walk(folder): + for filename in files: + if fnmatch.fnmatch(filename, pattern): + yield os.path.join(root, filename) + + +def session_check_requirements(session): + """Checks for out of date requirements and optionally updates them.""" + session.install(GCP_REPO_TOOLS_REQ) + + if 'update' in session.posargs: + command = 'update-requirements' + else: + command = 'check-requirements' + + reqfiles = list(_list_files('.', 'requirements*.txt')) + + for reqfile in reqfiles: + session.run('gcprepotools', command, reqfile) From 8613c33f32617da799df75b858cc418292812d19 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Tue, 21 Mar 2017 22:17:53 +0000 Subject: [PATCH 027/256] Auto-update dependencies. --- tests/integration/requirements.txt | 24 +- tests/python2-libraries/requirements.txt | 415 +++++++++++------------ tests/python3-libraries/requirements.txt | 339 +++++++++--------- 3 files changed, 379 insertions(+), 399 deletions(-) diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index dbd26d30..acc54ff8 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -1,21 +1,7 @@ -# Copyright 2016 Google Inc. All rights reserved. - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -Flask==0.11.1 -google-cloud-logging==0.22.0 -google-cloud-monitoring==0.22.0 -google-cloud-error-reporting==0.22.0 -gunicorn==19.6.0 +Flask==0.12 +google-cloud-logging==0.23.1 +google-cloud-monitoring==0.23.0 +google-cloud-error-reporting==0.23.1 +gunicorn==19.7.1 retrying==1.3.3 requests==2.13.0 diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 36dcbc16..7acbab47 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -1,212 +1,207 @@ -simplejson -setuptools -six -requests -virtualenv -pip -distribute -python-dateutil -certifi -boto -pbr -#wincertstore -docutils -pyasn1 -pyyaml -jinja2 -markupsafe -pytz -nose -lxml -pycrypto -rsa -colorama -botocore -cffi -awscli -coverage -jmespath -pycparser -pika -django -psycopg2 -paramiko -ecdsa -sqlalchemy -mock -redis -werkzeug -selenium -bcdoc -supervisor -pep8 -httplib2 -flask -pymongo -zc.buildout -psutil -mysql-python -argparse -carbon -pygments -babel -paste -anyjson -meld3 -# ssl (already included in standard library) -cryptography -py -tornado -pyopenssl -greenlet -kombu -graphite-web -docopt -mako -itsdangerous -pillow -wheel -beautifulsoup4 -enum34 -pyflakes -zope.interface -decorator -futures -pastedeploy -ordereddict -setuptools-git -fabric -backports.ssl_match_hostname -amqp -numpy -sphinx -iso8601 -flake8 -celery -pyparsing -mccabe -stevedore -pytest -webob -gunicorn -urllib3 -billiard -jsonschema -msgpack-python -gevent -logilab-common -unittest2 +simplejson==3.10.0 +setuptools==34.3.2 +six==1.10.0 +requests==2.13.0 +virtualenv==15.1.0 +pip==9.0.1 +distribute==0.7.3 +python-dateutil==2.6.0 +certifi==2017.1.23 +boto==2.46.1 +pbr==2.0.0 +docutils==0.13.1 +pyasn1==0.2.3 +pyyaml==3.12 +jinja2==2.9.5 +markupsafe==1.0 +pytz==2016.10 +nose==1.3.7 +lxml==3.7.3 +pycrypto==2.6.1 +rsa==3.4.2 +colorama==0.3.7 +botocore==1.5.27 +cffi==1.10.0 +awscli==1.11.64 +coverage==4.3.4 +jmespath==0.9.2 +pycparser==2.17 +pika==0.10.0 +django==1.10.6 +psycopg2==2.7.1 +paramiko==2.1.2 +ecdsa==0.13 +sqlalchemy==1.1.6 +mock==2.0.0 +redis==2.10.5 +werkzeug==0.12.1 +selenium==3.3.1 +bcdoc==0.16.0 +supervisor==3.3.1 +pep8==1.7.0 +httplib2==0.10.3 +flask==0.12 +pymongo==3.4.0 +zc.buildout==2.9.2 +psutil==5.2.0 +mysql-python==1.2.5 +argparse==1.4.0 +carbon==0.9.15 +pygments==2.2.0 +babel==2.3.4 +paste==2.0.3 +anyjson==0.3.3 +meld3==1.0.2 +cryptography==1.8.1 +py==1.4.33 +tornado==4.4.2 +pyopenssl==16.2.0 +greenlet==0.4.12 +kombu==4.0.2 +graphite-web==0.9.15 +docopt==0.6.2 +mako==1.0.6 +itsdangerous==0.24 +pillow==4.0.0 +wheel==0.29.0 +beautifulsoup4==4.5.3 +enum34==1.1.6 +pyflakes==1.5.0 +zope.interface==4.3.3 +decorator==4.0.11 +futures==3.0.5 +pastedeploy==1.5.2 +ordereddict==1.1 +setuptools-git==1.2 +fabric==1.13.1 +backports.ssl_match_hostname==3.5.0.1 +amqp==2.1.4 +numpy==1.12.1 +sphinx==1.5.3 +iso8601==0.1.11 +flake8==3.3.0 +celery==4.0.2 +pyparsing==2.2.0 +mccabe==0.6.1 +stevedore==1.21.0 +pytest==3.0.7 +webob==1.7.2 +gunicorn==19.7.1 +urllib3==1.20 +billiard==3.5.0.2 +jsonschema==2.6.0 +msgpack-python==0.4.8 +gevent==1.2.1 +logilab-common==1.4.0 +unittest2==1.1.0 prettytable -pylint -blessings -south -mozrunner -netaddr -oslo.config -twisted -ipaddress -ujson -moznetwork -mozdevice -mozprofile -mozprocess -mozfile -mozinfo +pylint==1.6.5 +blessings==1.6 +south==1.0.2 +mozrunner==6.13 +netaddr==0.7.19 +oslo.config==3.23.0 +twisted==17.1.0 +ipaddress==1.0.18 +ujson==1.35 +moznetwork==0.27 +mozdevice==0.48 +mozprofile==0.28 +mozprocess==0.25 +mozfile==1.2 +mozinfo==0.9 html5lib -mozlog -mozcrash -oauthlib -idna -ipython -tox -astroid -google-api-python-client -pycurl -isodate -python-keystoneclient -websocket-client -markdown -python-mimeparse -python-daemon -raven -suds -oauth2client -cython -eventlet -netifaces -repoze.lru -thrift -sqlparse -ndg-httpsclient -djangorestframework -python-novaclient -testtools -alembic -uritemplate -statsd -python-memcached -coveralls -funcsigs -configobj -linecache2 -extras -beautifulsoup -# scikit-learn -cliff -oauth2 -# pycups -cmd2 -unidecode -newrelic -python-gflags -cov-core -pytest-cov -fixtures -pyasn1-modules -python-swiftclient -django-debug-toolbar -elasticsearch -webtest -docker-py -python-subunit -retrying -django-extensions -pystache -waitress -pexpect -blinker -scipy -requests-oauthlib -protobuf -manifestparser -passlib -ansible -click -versiontools -django_compress -pyzmq -chardet -xlrd -snowballstemmer -testrepository -pandas -functools32 -python-cjson -pastescript -warlock -sqlalchemy-migrate -django-celery -uwsgi -cssselect -# Hand selected -matplotlib -pymysql -amqplib -sh -m2crypto -apache-libcloud -hiredis -bottle -pyramid -pyjwt -pylibmc +mozlog==3.4 +mozcrash==1.0 +oauthlib==2.0.2 +idna==2.5 +ipython==5.3.0 +tox==2.6.0 +astroid==1.4.9 +google-api-python-client==1.6.2 +pycurl==7.43.0 +isodate==0.5.4 +python-keystoneclient==3.10.0 +websocket-client==0.40.0 +markdown==2.6.8 +python-mimeparse==1.6.0 +python-daemon==2.1.2 +raven==6.0.0 +suds==0.4 +oauth2client==4.0.0 +cython==0.25.2 +eventlet==0.20.1 +netifaces==0.10.5 +repoze.lru==0.6 +thrift==0.10.0 +sqlparse==0.2.3 +ndg-httpsclient==0.4.2 +djangorestframework==3.6.2 +python-novaclient==7.1.0 +testtools==2.2.0 +alembic==0.9.1 +uritemplate==3.0.0 +statsd==3.2.1 +python-memcached==1.58 +coveralls==1.1 +funcsigs==1.0.2 +configobj==5.0.6 +linecache2==1.0.0 +extras==1.0.0 +beautifulsoup==3.2.1 +cliff==2.4.0 +oauth2==1.9.0.post1 +cmd2==0.7.0 +unidecode==0.4.20 +newrelic==2.82.0.62 +python-gflags==3.1.1 +cov-core==1.15.0 +pytest-cov==2.4.0 +fixtures==3.0.0 +pyasn1-modules==0.0.8 +python-swiftclient==3.3.0 +django-debug-toolbar==1.7 +elasticsearch==5.2.0 +webtest==2.0.27 +docker-py==1.10.6 +python-subunit==1.2.0 +retrying==1.3.3 +django-extensions==1.7.7 +pystache==0.5.4 +waitress==1.0.2 +pexpect==4.2.1 +blinker==1.4 +scipy==0.19.0 +requests-oauthlib==0.8.0 +protobuf==3.2.0 +manifestparser==1.1 +passlib==1.7.1 +ansible==2.2.1.0 +click==6.7 +versiontools==1.9.1 +django_compress==1.0.1 +pyzmq==16.0.2 +chardet==2.3.0 +xlrd==1.0.0 +snowballstemmer==1.2.1 +testrepository==0.0.20 +pandas==0.19.2 +functools32==3.2.3.post2 +python-cjson==1.2.1 +pastescript==2.0.2 +warlock==1.3.0 +sqlalchemy-migrate==0.11.0 +django-celery==3.2.1 +uwsgi==2.0.14 +cssselect==1.0.1 +matplotlib==2.0.0 +pymysql==0.7.10 +amqplib==1.0.2 +sh==1.12.11 +m2crypto==0.25.1 +apache-libcloud==1.5.0 +hiredis==0.2.0 +bottle==0.12.13 +pyramid==1.8.3 +pyjwt==1.4.2 +pylibmc==1.5.2 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index aeef827e..7f0b4ae0 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -1,172 +1,171 @@ -simplejson -six -requests -virtualenv -pip -certifi -docutils -pyasn1 -pyyaml -jinja2 -markupsafe -pytz -nose -lxml -pycrypto -rsa -colorama -cffi -pycparser -django -psycopg2 -ecdsa -sqlalchemy -mock -redis -werkzeug -pep8 -httplib2 -flask -pymongo -zc.buildout -psutil -argparse -carbon -pygments -babel -paste -anyjson -cryptography -py -tornado -pyopenssl -greenlet -kombu -docopt -mako -itsdangerous -pillow -wheel -beautifulsoup4 -pyflakes -zope.interface -decorator -futures -pastedeploy -setuptools-git -amqp -numpy -sphinx -iso8601 -flake8 -celery -pyparsing -mccabe -stevedore -pytest -webob -gunicorn -urllib3 -billiard -jsonschema -msgpack-python -gevent -logilab-common +simplejson==3.10.0 +six==1.10.0 +requests==2.13.0 +virtualenv==15.1.0 +pip==9.0.1 +certifi==2017.1.23 +docutils==0.13.1 +pyasn1==0.2.3 +pyyaml==3.12 +jinja2==2.9.5 +markupsafe==1.0 +pytz==2016.10 +nose==1.3.7 +lxml==3.7.3 +pycrypto==2.6.1 +rsa==3.4.2 +colorama==0.3.7 +cffi==1.10.0 +pycparser==2.17 +django==1.10.6 +psycopg2==2.7.1 +ecdsa==0.13 +sqlalchemy==1.1.6 +mock==2.0.0 +redis==2.10.5 +werkzeug==0.12.1 +pep8==1.7.0 +httplib2==0.10.3 +flask==0.12 +pymongo==3.4.0 +zc.buildout==2.9.2 +psutil==5.2.0 +argparse==1.4.0 +carbon==0.9.15 +pygments==2.2.0 +babel==2.3.4 +paste==2.0.3 +anyjson==0.3.3 +cryptography==1.8.1 +py==1.4.33 +tornado==4.4.2 +pyopenssl==16.2.0 +greenlet==0.4.12 +kombu==4.0.2 +docopt==0.6.2 +mako==1.0.6 +itsdangerous==0.24 +pillow==4.0.0 +wheel==0.29.0 +beautifulsoup4==4.5.3 +pyflakes==1.5.0 +zope.interface==4.3.3 +decorator==4.0.11 +futures==3.0.5 +pastedeploy==1.5.2 +setuptools-git==1.2 +amqp==2.1.4 +numpy==1.12.1 +sphinx==1.5.3 +iso8601==0.1.11 +flake8==3.3.0 +celery==4.0.2 +pyparsing==2.2.0 +mccabe==0.6.1 +stevedore==1.21.0 +pytest==3.0.7 +webob==1.7.2 +gunicorn==19.7.1 +urllib3==1.20 +billiard==3.5.0.2 +jsonschema==2.6.0 +msgpack-python==0.4.8 +gevent==1.2.1 +logilab-common==1.4.0 prettytable -pylint -blessings -south -netaddr -oslo.config -twisted -ipaddress -ujson +pylint==1.6.5 +blessings==1.6 +south==1.0.2 +netaddr==0.7.19 +oslo.config==3.23.0 +twisted==17.1.0 +ipaddress==1.0.18 +ujson==1.35 html5lib -oauthlib -idna -ipython -tox -astroid -google-api-python-client -pycurl -isodate -python-keystoneclient -websocket-client -markdown -python-mimeparse -python-daemon -raven -oauth2client -cython -eventlet -netifaces -repoze.lru -thrift -sqlparse -ndg-httpsclient -djangorestframework -python-novaclient -testtools -alembic -uritemplate -statsd -python-memcached -coveralls -funcsigs -configobj -linecache2 -extras -cliff -oauth2 -cmd2 -unidecode -newrelic -python-gflags -cov-core -pytest-cov -fixtures -pyasn1-modules -python-swiftclient -django-debug-toolbar -elasticsearch -webtest -docker-py -python-subunit -retrying -django-extensions -pystache -waitress -pexpect -blinker -scipy -requests-oauthlib -protobuf -manifestparser -passlib -ansible -click -versiontools -django_compress -pyzmq -chardet -xlrd -snowballstemmer -testrepository -pandas -pastescript -warlock -sqlalchemy-migrate -django-celery -cssselect -# Hand selected -matplotlib -pymysql -amqplib -sh -m2crypto -apache-libcloud -hiredis -bottle -pyramid -pyjwt -pylibmc +oauthlib==2.0.2 +idna==2.5 +ipython==5.3.0 +tox==2.6.0 +astroid==1.4.9 +google-api-python-client==1.6.2 +pycurl==7.43.0 +isodate==0.5.4 +python-keystoneclient==3.10.0 +websocket-client==0.40.0 +markdown==2.6.8 +python-mimeparse==1.6.0 +python-daemon==2.1.2 +raven==6.0.0 +oauth2client==4.0.0 +cython==0.25.2 +eventlet==0.20.1 +netifaces==0.10.5 +repoze.lru==0.6 +thrift==0.10.0 +sqlparse==0.2.3 +ndg-httpsclient==0.4.2 +djangorestframework==3.6.2 +python-novaclient==7.1.0 +testtools==2.2.0 +alembic==0.9.1 +uritemplate==3.0.0 +statsd==3.2.1 +python-memcached==1.58 +coveralls==1.1 +funcsigs==1.0.2 +configobj==5.0.6 +linecache2==1.0.0 +extras==1.0.0 +cliff==2.4.0 +oauth2==1.9.0.post1 +cmd2==0.7.0 +unidecode==0.4.20 +newrelic==2.82.0.62 +python-gflags==3.1.1 +cov-core==1.15.0 +pytest-cov==2.4.0 +fixtures==3.0.0 +pyasn1-modules==0.0.8 +python-swiftclient==3.3.0 +django-debug-toolbar==1.7 +elasticsearch==5.2.0 +webtest==2.0.27 +docker-py==1.10.6 +python-subunit==1.2.0 +retrying==1.3.3 +django-extensions==1.7.7 +pystache==0.5.4 +waitress==1.0.2 +pexpect==4.2.1 +blinker==1.4 +scipy==0.19.0 +requests-oauthlib==0.8.0 +protobuf==3.2.0 +manifestparser==1.1 +passlib==1.7.1 +ansible==2.2.1.0 +click==6.7 +versiontools==1.9.1 +django_compress==1.0.1 +pyzmq==16.0.2 +chardet==2.3.0 +xlrd==1.0.0 +snowballstemmer==1.2.1 +testrepository==0.0.20 +pandas==0.19.2 +pastescript==2.0.2 +warlock==1.3.0 +sqlalchemy-migrate==0.11.0 +django-celery==3.2.1 +cssselect==1.0.1 +matplotlib==2.0.0 +pymysql==0.7.10 +amqplib==1.0.2 +sh==1.12.11 +m2crypto==0.25.1 +apache-libcloud==1.5.0 +hiredis==0.2.0 +bottle==0.12.13 +pyramid==1.8.3 +pyjwt==1.4.2 +pylibmc==1.5.2 From 53926b59fa1eaba937526eb3b20104cd980d202f Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Tue, 21 Mar 2017 15:41:33 -0700 Subject: [PATCH 028/256] Sort requirements.txt files --- tests/integration/requirements.txt | 4 +- tests/python2-libraries/requirements.txt | 360 +++++++++++------------ tests/python3-libraries/requirements.txt | 298 +++++++++---------- 3 files changed, 331 insertions(+), 331 deletions(-) diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index acc54ff8..1a8489db 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -1,7 +1,7 @@ Flask==0.12 +google-cloud-error-reporting==0.23.1 google-cloud-logging==0.23.1 google-cloud-monitoring==0.23.0 -google-cloud-error-reporting==0.23.1 gunicorn==19.7.1 -retrying==1.3.3 requests==2.13.0 +retrying==1.3.3 diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 7acbab47..d64b5d2c 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -1,207 +1,207 @@ -simplejson==3.10.0 -setuptools==34.3.2 -six==1.10.0 -requests==2.13.0 -virtualenv==15.1.0 -pip==9.0.1 -distribute==0.7.3 -python-dateutil==2.6.0 -certifi==2017.1.23 +alembic==0.9.1 +amqp==2.1.4 +amqplib==1.0.2 +ansible==2.2.1.0 +anyjson==0.3.3 +apache-libcloud==1.5.0 +argparse==1.4.0 +astroid==1.4.9 +awscli==1.11.64 +babel==2.3.4 +backports.ssl_match_hostname==3.5.0.1 +bcdoc==0.16.0 +beautifulsoup4==4.5.3 +beautifulsoup==3.2.1 +billiard==3.5.0.2 +blessings==1.6 +blinker==1.4 boto==2.46.1 -pbr==2.0.0 -docutils==0.13.1 -pyasn1==0.2.3 -pyyaml==3.12 -jinja2==2.9.5 -markupsafe==1.0 -pytz==2016.10 -nose==1.3.7 -lxml==3.7.3 -pycrypto==2.6.1 -rsa==3.4.2 -colorama==0.3.7 botocore==1.5.27 +bottle==0.12.13 +carbon==0.9.15 +celery==4.0.2 +certifi==2017.1.23 cffi==1.10.0 -awscli==1.11.64 +chardet==2.3.0 +click==6.7 +cliff==2.4.0 +cmd2==0.7.0 +colorama==0.3.7 +configobj==5.0.6 +cov-core==1.15.0 coverage==4.3.4 -jmespath==0.9.2 -pycparser==2.17 -pika==0.10.0 -django==1.10.6 -psycopg2==2.7.1 -paramiko==2.1.2 -ecdsa==0.13 -sqlalchemy==1.1.6 -mock==2.0.0 -redis==2.10.5 -werkzeug==0.12.1 -selenium==3.3.1 -bcdoc==0.16.0 -supervisor==3.3.1 -pep8==1.7.0 -httplib2==0.10.3 -flask==0.12 -pymongo==3.4.0 -zc.buildout==2.9.2 -psutil==5.2.0 -mysql-python==1.2.5 -argparse==1.4.0 -carbon==0.9.15 -pygments==2.2.0 -babel==2.3.4 -paste==2.0.3 -anyjson==0.3.3 -meld3==1.0.2 +coveralls==1.1 cryptography==1.8.1 -py==1.4.33 -tornado==4.4.2 -pyopenssl==16.2.0 -greenlet==0.4.12 -kombu==4.0.2 -graphite-web==0.9.15 +cssselect==1.0.1 +cython==0.25.2 +decorator==4.0.11 +distribute==0.7.3 +django-celery==3.2.1 +django-debug-toolbar==1.7 +django-extensions==1.7.7 +django==1.10.6 +django_compress==1.0.1 +djangorestframework==3.6.2 +docker-py==1.10.6 docopt==0.6.2 -mako==1.0.6 -itsdangerous==0.24 -pillow==4.0.0 -wheel==0.29.0 -beautifulsoup4==4.5.3 +docutils==0.13.1 +ecdsa==0.13 +elasticsearch==5.2.0 enum34==1.1.6 -pyflakes==1.5.0 -zope.interface==4.3.3 -decorator==4.0.11 -futures==3.0.5 -pastedeploy==1.5.2 -ordereddict==1.1 -setuptools-git==1.2 +eventlet==0.20.1 +extras==1.0.0 fabric==1.13.1 -backports.ssl_match_hostname==3.5.0.1 -amqp==2.1.4 -numpy==1.12.1 -sphinx==1.5.3 -iso8601==0.1.11 +fixtures==3.0.0 flake8==3.3.0 -celery==4.0.2 -pyparsing==2.2.0 -mccabe==0.6.1 -stevedore==1.21.0 -pytest==3.0.7 -webob==1.7.2 +flask==0.12 +funcsigs==1.0.2 +functools32==3.2.3.post2 +futures==3.0.5 +gevent==1.2.1 +google-api-python-client==1.6.2 +graphite-web==0.9.15 +greenlet==0.4.12 gunicorn==19.7.1 -urllib3==1.20 -billiard==3.5.0.2 +hiredis==0.2.0 +html5lib +httplib2==0.10.3 +idna==2.5 +ipaddress==1.0.18 +ipython==5.3.0 +iso8601==0.1.11 +isodate==0.5.4 +itsdangerous==0.24 +jinja2==2.9.5 +jmespath==0.9.2 jsonschema==2.6.0 -msgpack-python==0.4.8 -gevent==1.2.1 +kombu==4.0.2 +linecache2==1.0.0 logilab-common==1.4.0 -unittest2==1.1.0 -prettytable -pylint==1.6.5 -blessings==1.6 -south==1.0.2 -mozrunner==6.13 -netaddr==0.7.19 -oslo.config==3.23.0 -twisted==17.1.0 -ipaddress==1.0.18 -ujson==1.35 -moznetwork==0.27 +lxml==3.7.3 +m2crypto==0.25.1 +mako==1.0.6 +manifestparser==1.1 +markdown==2.6.8 +markupsafe==1.0 +matplotlib==2.0.0 +mccabe==0.6.1 +meld3==1.0.2 +mock==2.0.0 +mozcrash==1.0 mozdevice==0.48 -mozprofile==0.28 -mozprocess==0.25 mozfile==1.2 mozinfo==0.9 -html5lib mozlog==3.4 -mozcrash==1.0 +moznetwork==0.27 +mozprocess==0.25 +mozprofile==0.28 +mozrunner==6.13 +msgpack-python==0.4.8 +mysql-python==1.2.5 +ndg-httpsclient==0.4.2 +netaddr==0.7.19 +netifaces==0.10.5 +newrelic==2.82.0.62 +nose==1.3.7 +numpy==1.12.1 +oauth2==1.9.0.post1 +oauth2client==4.0.0 oauthlib==2.0.2 -idna==2.5 -ipython==5.3.0 -tox==2.6.0 -astroid==1.4.9 -google-api-python-client==1.6.2 +ordereddict==1.1 +oslo.config==3.23.0 +pandas==0.19.2 +paramiko==2.1.2 +passlib==1.7.1 +paste==2.0.3 +pastedeploy==1.5.2 +pastescript==2.0.2 +pbr==2.0.0 +pep8==1.7.0 +pexpect==4.2.1 +pika==0.10.0 +pillow==4.0.0 +pip==9.0.1 +prettytable +protobuf==3.2.0 +psutil==5.2.0 +psycopg2==2.7.1 +py==1.4.33 +pyasn1-modules==0.0.8 +pyasn1==0.2.3 +pycparser==2.17 +pycrypto==2.6.1 pycurl==7.43.0 -isodate==0.5.4 +pyflakes==1.5.0 +pygments==2.2.0 +pyjwt==1.4.2 +pylibmc==1.5.2 +pylint==1.6.5 +pymongo==3.4.0 +pymysql==0.7.10 +pyopenssl==16.2.0 +pyparsing==2.2.0 +pyramid==1.8.3 +pystache==0.5.4 +pytest-cov==2.4.0 +pytest==3.0.7 +python-cjson==1.2.1 +python-daemon==2.1.2 +python-dateutil==2.6.0 +python-gflags==3.1.1 python-keystoneclient==3.10.0 -websocket-client==0.40.0 -markdown==2.6.8 +python-memcached==1.58 python-mimeparse==1.6.0 -python-daemon==2.1.2 -raven==6.0.0 -suds==0.4 -oauth2client==4.0.0 -cython==0.25.2 -eventlet==0.20.1 -netifaces==0.10.5 -repoze.lru==0.6 -thrift==0.10.0 -sqlparse==0.2.3 -ndg-httpsclient==0.4.2 -djangorestframework==3.6.2 python-novaclient==7.1.0 -testtools==2.2.0 -alembic==0.9.1 -uritemplate==3.0.0 -statsd==3.2.1 -python-memcached==1.58 -coveralls==1.1 -funcsigs==1.0.2 -configobj==5.0.6 -linecache2==1.0.0 -extras==1.0.0 -beautifulsoup==3.2.1 -cliff==2.4.0 -oauth2==1.9.0.post1 -cmd2==0.7.0 -unidecode==0.4.20 -newrelic==2.82.0.62 -python-gflags==3.1.1 -cov-core==1.15.0 -pytest-cov==2.4.0 -fixtures==3.0.0 -pyasn1-modules==0.0.8 -python-swiftclient==3.3.0 -django-debug-toolbar==1.7 -elasticsearch==5.2.0 -webtest==2.0.27 -docker-py==1.10.6 python-subunit==1.2.0 +python-swiftclient==3.3.0 +pytz==2016.10 +pyyaml==3.12 +pyzmq==16.0.2 +raven==6.0.0 +redis==2.10.5 +repoze.lru==0.6 +requests-oauthlib==0.8.0 +requests==2.13.0 retrying==1.3.3 -django-extensions==1.7.7 -pystache==0.5.4 -waitress==1.0.2 -pexpect==4.2.1 -blinker==1.4 +rsa==3.4.2 scipy==0.19.0 -requests-oauthlib==0.8.0 -protobuf==3.2.0 -manifestparser==1.1 -passlib==1.7.1 -ansible==2.2.1.0 -click==6.7 -versiontools==1.9.1 -django_compress==1.0.1 -pyzmq==16.0.2 -chardet==2.3.0 -xlrd==1.0.0 +selenium==3.3.1 +setuptools-git==1.2 +setuptools==34.3.2 +sh==1.12.11 +simplejson==3.10.0 +six==1.10.0 snowballstemmer==1.2.1 -testrepository==0.0.20 -pandas==0.19.2 -functools32==3.2.3.post2 -python-cjson==1.2.1 -pastescript==2.0.2 -warlock==1.3.0 +south==1.0.2 +sphinx==1.5.3 sqlalchemy-migrate==0.11.0 -django-celery==3.2.1 +sqlalchemy==1.1.6 +sqlparse==0.2.3 +statsd==3.2.1 +stevedore==1.21.0 +suds==0.4 +supervisor==3.3.1 +testrepository==0.0.20 +testtools==2.2.0 +thrift==0.10.0 +tornado==4.4.2 +tox==2.6.0 +twisted==17.1.0 +ujson==1.35 +unidecode==0.4.20 +unittest2==1.1.0 +uritemplate==3.0.0 +urllib3==1.20 uwsgi==2.0.14 -cssselect==1.0.1 -matplotlib==2.0.0 -pymysql==0.7.10 -amqplib==1.0.2 -sh==1.12.11 -m2crypto==0.25.1 -apache-libcloud==1.5.0 -hiredis==0.2.0 -bottle==0.12.13 -pyramid==1.8.3 -pyjwt==1.4.2 -pylibmc==1.5.2 +versiontools==1.9.1 +virtualenv==15.1.0 +waitress==1.0.2 +warlock==1.3.0 +webob==1.7.2 +websocket-client==0.40.0 +webtest==2.0.27 +werkzeug==0.12.1 +wheel==0.29.0 +xlrd==1.0.0 +zc.buildout==2.9.2 +zope.interface==4.3.3 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 7f0b4ae0..c87fc842 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -1,171 +1,171 @@ -simplejson==3.10.0 -six==1.10.0 -requests==2.13.0 -virtualenv==15.1.0 -pip==9.0.1 +alembic==0.9.1 +amqp==2.1.4 +amqplib==1.0.2 +ansible==2.2.1.0 +anyjson==0.3.3 +apache-libcloud==1.5.0 +argparse==1.4.0 +astroid==1.4.9 +babel==2.3.4 +beautifulsoup4==4.5.3 +billiard==3.5.0.2 +blessings==1.6 +blinker==1.4 +bottle==0.12.13 +carbon==0.9.15 +celery==4.0.2 certifi==2017.1.23 -docutils==0.13.1 -pyasn1==0.2.3 -pyyaml==3.12 -jinja2==2.9.5 -markupsafe==1.0 -pytz==2016.10 -nose==1.3.7 -lxml==3.7.3 -pycrypto==2.6.1 -rsa==3.4.2 -colorama==0.3.7 cffi==1.10.0 -pycparser==2.17 +chardet==2.3.0 +click==6.7 +cliff==2.4.0 +cmd2==0.7.0 +colorama==0.3.7 +configobj==5.0.6 +cov-core==1.15.0 +coveralls==1.1 +cryptography==1.8.1 +cssselect==1.0.1 +cython==0.25.2 +decorator==4.0.11 +django-celery==3.2.1 +django-debug-toolbar==1.7 +django-extensions==1.7.7 django==1.10.6 -psycopg2==2.7.1 +django_compress==1.0.1 +djangorestframework==3.6.2 +docker-py==1.10.6 +docopt==0.6.2 +docutils==0.13.1 ecdsa==0.13 -sqlalchemy==1.1.6 -mock==2.0.0 -redis==2.10.5 -werkzeug==0.12.1 -pep8==1.7.0 -httplib2==0.10.3 +elasticsearch==5.2.0 +eventlet==0.20.1 +extras==1.0.0 +fixtures==3.0.0 +flake8==3.3.0 flask==0.12 -pymongo==3.4.0 -zc.buildout==2.9.2 -psutil==5.2.0 -argparse==1.4.0 -carbon==0.9.15 -pygments==2.2.0 -babel==2.3.4 -paste==2.0.3 -anyjson==0.3.3 -cryptography==1.8.1 -py==1.4.33 -tornado==4.4.2 -pyopenssl==16.2.0 +funcsigs==1.0.2 +futures==3.0.5 +gevent==1.2.1 +google-api-python-client==1.6.2 greenlet==0.4.12 +gunicorn==19.7.1 +hiredis==0.2.0 +html5lib +httplib2==0.10.3 +idna==2.5 +ipaddress==1.0.18 +ipython==5.3.0 +iso8601==0.1.11 +isodate==0.5.4 +itsdangerous==0.24 +jinja2==2.9.5 +jsonschema==2.6.0 kombu==4.0.2 -docopt==0.6.2 +linecache2==1.0.0 +logilab-common==1.4.0 +lxml==3.7.3 +m2crypto==0.25.1 mako==1.0.6 -itsdangerous==0.24 -pillow==4.0.0 -wheel==0.29.0 -beautifulsoup4==4.5.3 -pyflakes==1.5.0 -zope.interface==4.3.3 -decorator==4.0.11 -futures==3.0.5 -pastedeploy==1.5.2 -setuptools-git==1.2 -amqp==2.1.4 -numpy==1.12.1 -sphinx==1.5.3 -iso8601==0.1.11 -flake8==3.3.0 -celery==4.0.2 -pyparsing==2.2.0 +manifestparser==1.1 +markdown==2.6.8 +markupsafe==1.0 +matplotlib==2.0.0 mccabe==0.6.1 -stevedore==1.21.0 -pytest==3.0.7 -webob==1.7.2 -gunicorn==19.7.1 -urllib3==1.20 -billiard==3.5.0.2 -jsonschema==2.6.0 +mock==2.0.0 msgpack-python==0.4.8 -gevent==1.2.1 -logilab-common==1.4.0 -prettytable -pylint==1.6.5 -blessings==1.6 -south==1.0.2 +ndg-httpsclient==0.4.2 netaddr==0.7.19 -oslo.config==3.23.0 -twisted==17.1.0 -ipaddress==1.0.18 -ujson==1.35 -html5lib +netifaces==0.10.5 +newrelic==2.82.0.62 +nose==1.3.7 +numpy==1.12.1 +oauth2==1.9.0.post1 +oauth2client==4.0.0 oauthlib==2.0.2 -idna==2.5 -ipython==5.3.0 -tox==2.6.0 -astroid==1.4.9 -google-api-python-client==1.6.2 +oslo.config==3.23.0 +pandas==0.19.2 +passlib==1.7.1 +paste==2.0.3 +pastedeploy==1.5.2 +pastescript==2.0.2 +pep8==1.7.0 +pexpect==4.2.1 +pillow==4.0.0 +pip==9.0.1 +prettytable +protobuf==3.2.0 +psutil==5.2.0 +psycopg2==2.7.1 +py==1.4.33 +pyasn1-modules==0.0.8 +pyasn1==0.2.3 +pycparser==2.17 +pycrypto==2.6.1 pycurl==7.43.0 -isodate==0.5.4 +pyflakes==1.5.0 +pygments==2.2.0 +pyjwt==1.4.2 +pylibmc==1.5.2 +pylint==1.6.5 +pymongo==3.4.0 +pymysql==0.7.10 +pyopenssl==16.2.0 +pyparsing==2.2.0 +pyramid==1.8.3 +pystache==0.5.4 +pytest-cov==2.4.0 +pytest==3.0.7 +python-daemon==2.1.2 +python-gflags==3.1.1 python-keystoneclient==3.10.0 -websocket-client==0.40.0 -markdown==2.6.8 +python-memcached==1.58 python-mimeparse==1.6.0 -python-daemon==2.1.2 -raven==6.0.0 -oauth2client==4.0.0 -cython==0.25.2 -eventlet==0.20.1 -netifaces==0.10.5 -repoze.lru==0.6 -thrift==0.10.0 -sqlparse==0.2.3 -ndg-httpsclient==0.4.2 -djangorestframework==3.6.2 python-novaclient==7.1.0 -testtools==2.2.0 -alembic==0.9.1 -uritemplate==3.0.0 -statsd==3.2.1 -python-memcached==1.58 -coveralls==1.1 -funcsigs==1.0.2 -configobj==5.0.6 -linecache2==1.0.0 -extras==1.0.0 -cliff==2.4.0 -oauth2==1.9.0.post1 -cmd2==0.7.0 -unidecode==0.4.20 -newrelic==2.82.0.62 -python-gflags==3.1.1 -cov-core==1.15.0 -pytest-cov==2.4.0 -fixtures==3.0.0 -pyasn1-modules==0.0.8 -python-swiftclient==3.3.0 -django-debug-toolbar==1.7 -elasticsearch==5.2.0 -webtest==2.0.27 -docker-py==1.10.6 python-subunit==1.2.0 +python-swiftclient==3.3.0 +pytz==2016.10 +pyyaml==3.12 +pyzmq==16.0.2 +raven==6.0.0 +redis==2.10.5 +repoze.lru==0.6 +requests-oauthlib==0.8.0 +requests==2.13.0 retrying==1.3.3 -django-extensions==1.7.7 -pystache==0.5.4 -waitress==1.0.2 -pexpect==4.2.1 -blinker==1.4 +rsa==3.4.2 scipy==0.19.0 -requests-oauthlib==0.8.0 -protobuf==3.2.0 -manifestparser==1.1 -passlib==1.7.1 -ansible==2.2.1.0 -click==6.7 -versiontools==1.9.1 -django_compress==1.0.1 -pyzmq==16.0.2 -chardet==2.3.0 -xlrd==1.0.0 +setuptools-git==1.2 +sh==1.12.11 +simplejson==3.10.0 +six==1.10.0 snowballstemmer==1.2.1 +south==1.0.2 +sphinx==1.5.3 +sqlalchemy-migrate==0.11.0 +sqlalchemy==1.1.6 +sqlparse==0.2.3 +statsd==3.2.1 +stevedore==1.21.0 testrepository==0.0.20 -pandas==0.19.2 -pastescript==2.0.2 +testtools==2.2.0 +thrift==0.10.0 +tornado==4.4.2 +tox==2.6.0 +twisted==17.1.0 +ujson==1.35 +unidecode==0.4.20 +uritemplate==3.0.0 +urllib3==1.20 +versiontools==1.9.1 +virtualenv==15.1.0 +waitress==1.0.2 warlock==1.3.0 -sqlalchemy-migrate==0.11.0 -django-celery==3.2.1 -cssselect==1.0.1 -matplotlib==2.0.0 -pymysql==0.7.10 -amqplib==1.0.2 -sh==1.12.11 -m2crypto==0.25.1 -apache-libcloud==1.5.0 -hiredis==0.2.0 -bottle==0.12.13 -pyramid==1.8.3 -pyjwt==1.4.2 -pylibmc==1.5.2 +webob==1.7.2 +websocket-client==0.40.0 +webtest==2.0.27 +werkzeug==0.12.1 +wheel==0.29.0 +xlrd==1.0.0 +zc.buildout==2.9.2 +zope.interface==4.3.3 From f85f392fd5000628e8eb6cb58134672ff29b95e8 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Fri, 24 Mar 2017 10:17:44 -0700 Subject: [PATCH 029/256] Auto-update dependencies. (#86) --- tests/python2-libraries/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index d64b5d2c..9a1fc8a2 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -6,7 +6,7 @@ anyjson==0.3.3 apache-libcloud==1.5.0 argparse==1.4.0 astroid==1.4.9 -awscli==1.11.64 +awscli==1.11.66 babel==2.3.4 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 @@ -16,7 +16,7 @@ billiard==3.5.0.2 blessings==1.6 blinker==1.4 boto==2.46.1 -botocore==1.5.27 +botocore==1.5.29 bottle==0.12.13 carbon==0.9.15 celery==4.0.2 From d63ab6d0c7640ef9c6c54e86dc348a1c31cad495 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Sun, 26 Mar 2017 09:07:48 +0000 Subject: [PATCH 030/256] Auto-update dependencies. --- tests/python2-libraries/requirements.txt | 6 +++--- tests/python3-libraries/requirements.txt | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 9a1fc8a2..0a018109 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -7,7 +7,7 @@ apache-libcloud==1.5.0 argparse==1.4.0 astroid==1.4.9 awscli==1.11.66 -babel==2.3.4 +babel==2.4.0 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 beautifulsoup4==4.5.3 @@ -78,7 +78,7 @@ kombu==4.0.2 linecache2==1.0.0 logilab-common==1.4.0 lxml==3.7.3 -m2crypto==0.25.1 +m2crypto==0.26.0 mako==1.0.6 manifestparser==1.1 markdown==2.6.8 @@ -123,7 +123,7 @@ pillow==4.0.0 pip==9.0.1 prettytable protobuf==3.2.0 -psutil==5.2.0 +psutil==5.2.1 psycopg2==2.7.1 py==1.4.33 pyasn1-modules==0.0.8 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index c87fc842..ed1a7ffd 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -6,7 +6,7 @@ anyjson==0.3.3 apache-libcloud==1.5.0 argparse==1.4.0 astroid==1.4.9 -babel==2.3.4 +babel==2.4.0 beautifulsoup4==4.5.3 billiard==3.5.0.2 blessings==1.6 @@ -65,7 +65,7 @@ kombu==4.0.2 linecache2==1.0.0 logilab-common==1.4.0 lxml==3.7.3 -m2crypto==0.25.1 +m2crypto==0.26.0 mako==1.0.6 manifestparser==1.1 markdown==2.6.8 @@ -95,7 +95,7 @@ pillow==4.0.0 pip==9.0.1 prettytable protobuf==3.2.0 -psutil==5.2.0 +psutil==5.2.1 psycopg2==2.7.1 py==1.4.33 pyasn1-modules==0.0.8 From 9416574b7ddc61d93c5ab17873bd15deeabcf6fa Mon Sep 17 00:00:00 2001 From: DPE bot Date: Mon, 27 Mar 2017 09:07:54 +0000 Subject: [PATCH 031/256] Auto-update dependencies. --- tests/python2-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 0a018109..7d168be6 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -167,7 +167,7 @@ rsa==3.4.2 scipy==0.19.0 selenium==3.3.1 setuptools-git==1.2 -setuptools==34.3.2 +setuptools==34.3.3 sh==1.12.11 simplejson==3.10.0 six==1.10.0 From 22f420c841b4beaa1e6946e06d6e03484d309bc6 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Tue, 28 Mar 2017 09:07:49 +0000 Subject: [PATCH 032/256] Auto-update dependencies. --- tests/python2-libraries/requirements.txt | 8 ++++---- tests/python3-libraries/requirements.txt | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 7d168be6..99a95bad 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -1,12 +1,12 @@ alembic==0.9.1 amqp==2.1.4 amqplib==1.0.2 -ansible==2.2.1.0 +ansible==2.2.2.0 anyjson==0.3.3 apache-libcloud==1.5.0 argparse==1.4.0 astroid==1.4.9 -awscli==1.11.66 +awscli==1.11.67 babel==2.4.0 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 @@ -16,7 +16,7 @@ billiard==3.5.0.2 blessings==1.6 blinker==1.4 boto==2.46.1 -botocore==1.5.29 +botocore==1.5.30 bottle==0.12.13 carbon==0.9.15 celery==4.0.2 @@ -175,7 +175,7 @@ snowballstemmer==1.2.1 south==1.0.2 sphinx==1.5.3 sqlalchemy-migrate==0.11.0 -sqlalchemy==1.1.6 +sqlalchemy==1.1.7 sqlparse==0.2.3 statsd==3.2.1 stevedore==1.21.0 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index ed1a7ffd..3ea2903e 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -1,7 +1,7 @@ alembic==0.9.1 amqp==2.1.4 amqplib==1.0.2 -ansible==2.2.1.0 +ansible==2.2.2.0 anyjson==0.3.3 apache-libcloud==1.5.0 argparse==1.4.0 @@ -143,7 +143,7 @@ snowballstemmer==1.2.1 south==1.0.2 sphinx==1.5.3 sqlalchemy-migrate==0.11.0 -sqlalchemy==1.1.6 +sqlalchemy==1.1.7 sqlparse==0.2.3 statsd==3.2.1 stevedore==1.21.0 From 8a467434353fb6d8d81213ebd5b716b9f9620106 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Wed, 29 Mar 2017 09:07:49 +0000 Subject: [PATCH 033/256] Auto-update dependencies. --- tests/integration/requirements.txt | 2 +- tests/python2-libraries/requirements.txt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index 1a8489db..c5004b32 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -1,5 +1,5 @@ Flask==0.12 -google-cloud-error-reporting==0.23.1 +google-cloud-error-reporting==0.23.2 google-cloud-logging==0.23.1 google-cloud-monitoring==0.23.0 gunicorn==19.7.1 diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 99a95bad..11a2cb63 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -6,7 +6,7 @@ anyjson==0.3.3 apache-libcloud==1.5.0 argparse==1.4.0 astroid==1.4.9 -awscli==1.11.67 +awscli==1.11.68 babel==2.4.0 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 @@ -16,7 +16,7 @@ billiard==3.5.0.2 blessings==1.6 blinker==1.4 boto==2.46.1 -botocore==1.5.30 +botocore==1.5.31 bottle==0.12.13 carbon==0.9.15 celery==4.0.2 From 54a6085dab5128f8ce74eb9863a05a649b4a7016 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Thu, 30 Mar 2017 09:07:51 +0000 Subject: [PATCH 034/256] Auto-update dependencies. --- tests/python2-libraries/requirements.txt | 4 ++-- tests/python3-libraries/requirements.txt | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 11a2cb63..409342f4 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -38,7 +38,7 @@ decorator==4.0.11 distribute==0.7.3 django-celery==3.2.1 django-debug-toolbar==1.7 -django-extensions==1.7.7 +django-extensions==1.7.8 django==1.10.6 django_compress==1.0.1 djangorestframework==3.6.2 @@ -154,7 +154,7 @@ python-mimeparse==1.6.0 python-novaclient==7.1.0 python-subunit==1.2.0 python-swiftclient==3.3.0 -pytz==2016.10 +pytz==2017.2 pyyaml==3.12 pyzmq==16.0.2 raven==6.0.0 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 3ea2903e..a7269999 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -30,7 +30,7 @@ cython==0.25.2 decorator==4.0.11 django-celery==3.2.1 django-debug-toolbar==1.7 -django-extensions==1.7.7 +django-extensions==1.7.8 django==1.10.6 django_compress==1.0.1 djangorestframework==3.6.2 @@ -124,7 +124,7 @@ python-mimeparse==1.6.0 python-novaclient==7.1.0 python-subunit==1.2.0 python-swiftclient==3.3.0 -pytz==2016.10 +pytz==2017.2 pyyaml==3.12 pyzmq==16.0.2 raven==6.0.0 From 41d3d61cb8ddda83af006e7887925cead90d567d Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Thu, 30 Mar 2017 15:23:25 -0700 Subject: [PATCH 035/256] Remove 'distribute' from library test. 'pip install distribute' unpredictably fails when other packages are being updated at the same time. See https://github.com/pypa/setuptools/issues/535 for related discussion. Error message looks like: Running setup.py install for distribute: started Running setup.py install for distribute: finished with status 'error' Complete output from command /env/bin/python -u -c "import setuptools, tokenize;__file__='/tmp/pip-build-9DiKZ8/distribute/setup.py';f=getattr(tokenize, 'open', open)(__file__);code=f.read().replace('\r\n', '\n');f.close();exec(compile(code, __file__, 'exec'))" install --record /tmp/pip-zFa039-record/install-record.txt --single-version-externally-managed --compile --install-headers /env/include/site/python2.7/distribute: running install running build running install_egg_info running egg_info writing requirements to distribute.egg-info/requires.txt writing distribute.egg-info/PKG-INFO writing top-level names to distribute.egg-info/top_level.txt writing dependency_links to distribute.egg-info/dependency_links.txt Traceback (most recent call last): File "", line 1, in File "/tmp/pip-build-9DiKZ8/distribute/setup.py", line 58, in setuptools.setup(**setup_params) File "/usr/lib/python2.7/distutils/core.py", line 151, in setup dist.run_commands() File "/usr/lib/python2.7/distutils/dist.py", line 953, in run_commands self.run_command(cmd) File "/usr/lib/python2.7/distutils/dist.py", line 972, in run_command cmd_obj.run() File "setuptools/command/install.py", line 53, in run return _install.run(self) File "/usr/lib/python2.7/distutils/command/install.py", line 613, in run self.run_command(cmd_name) File "/usr/lib/python2.7/distutils/cmd.py", line 326, in run_command self.distribution.run_command(command) File "/usr/lib/python2.7/distutils/dist.py", line 972, in run_command cmd_obj.run() File "setuptools/command/install_egg_info.py", line 29, in run self.run_command('egg_info') File "/usr/lib/python2.7/distutils/cmd.py", line 326, in run_command self.distribution.run_command(command) File "/usr/lib/python2.7/distutils/dist.py", line 972, in run_command cmd_obj.run() File "setuptools/command/egg_info.py", line 177, in run writer = ep.load(installer=installer) File "pkg_resources.py", line 2241, in load if require: self.require(env, installer) File "pkg_resources.py", line 2254, in require working_set.resolve(self.dist.requires(self.extras),env,installer))) File "pkg_resources.py", line 2471, in requires dm = self._dep_map File "pkg_resources.py", line 2682, in _dep_map self.__dep_map = self._compute_dependencies() File "pkg_resources.py", line 2699, in _compute_dependencies from _markerlib import compile as compile_marker ImportError: No module named _markerlib --- tests/python2-libraries/requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 409342f4..d92af43f 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -35,7 +35,6 @@ cryptography==1.8.1 cssselect==1.0.1 cython==0.25.2 decorator==4.0.11 -distribute==0.7.3 django-celery==3.2.1 django-debug-toolbar==1.7 django-extensions==1.7.8 From e9ecfac40107413048414c7e91f5834fdc9c5fb3 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 31 Mar 2017 17:02:20 -0700 Subject: [PATCH 036/256] Update build process. build.sh is the entrypoint for all builds. cloudbuild*.yaml files are now the single source of truth for the build steps. Makefiles have been removed. Local builds are done via local_cloudbuild.py. jenkins_build.sh will be removed once the Jenkins config is updated. The benchmarking and system tests now work in Container Builder. Build instructions have been moved into their own file. gcloud substitutions are used for cloudbuild.yaml substitions. See 'gcloud beta container builds submit --help' for more info. 'envsubst' is still used for Dockerifle substitutions. Updated the google-cloud-python test logic. --- .gitignore | 8 +- Makefile | 73 --------- README.md | 68 +-------- RELEASING.md | 130 ++++++++++++++++ build.sh | 139 +++++++++++++++++- cloudbuild.yaml | 34 +++++ cloudbuild.yaml.in | 24 --- cloudbuild_benchmark.yaml | 8 + cloudbuild_system_tests.yaml | 9 ++ jenkins_build.sh | 44 +----- python-interpreter-builder/.gitignore | 2 +- .../{Dockerfile => Dockerfile.in} | 2 +- python-interpreter-builder/Makefile | 10 -- runtime-image/.gitignore | 2 +- runtime-image/{Dockerfile => Dockerfile.in} | 2 +- system_tests/.gitignore | 3 - system_tests/Dockerfile.in | 15 -- system_tests/Makefile | 22 --- tests/Makefile | 18 --- tests/benchmark/Dockerfile.2vs3 | 1 - tests/benchmark/Dockerfile.34vs35 | 1 - tests/benchmark/Dockerfile.in | 6 +- tests/benchmark/Makefile | 14 -- tests/google-cloud-python-system/.gitignore | 2 + .../google-cloud-python-system/Dockerfile.in | 16 ++ .../run_system_tests.sh | 34 +++++ tests/google-cloud-python/Dockerfile.in | 19 +-- tests/google-cloud-python/Makefile | 5 - tests/google-cloud-python/run_unit_tests.sh | 11 ++ tests/integration/.gitignore | 1 + tests/integration/Dockerfile.in | 2 +- tests/license-test/Makefile | 3 - tests/no-virtualenv/Makefile | 3 - tests/python2-libraries/Makefile | 3 - tests/python3-libraries/Makefile | 3 - tests/virtualenv/Makefile | 5 - 36 files changed, 400 insertions(+), 342 deletions(-) delete mode 100644 Makefile create mode 100644 RELEASING.md create mode 100644 cloudbuild.yaml delete mode 100644 cloudbuild.yaml.in create mode 100644 cloudbuild_benchmark.yaml create mode 100644 cloudbuild_system_tests.yaml rename python-interpreter-builder/{Dockerfile => Dockerfile.in} (94%) delete mode 100644 python-interpreter-builder/Makefile rename runtime-image/{Dockerfile => Dockerfile.in} (96%) delete mode 100644 system_tests/.gitignore delete mode 100644 system_tests/Dockerfile.in delete mode 100644 system_tests/Makefile delete mode 100644 tests/Makefile delete mode 100644 tests/benchmark/Dockerfile.2vs3 delete mode 100644 tests/benchmark/Dockerfile.34vs35 delete mode 100644 tests/benchmark/Makefile create mode 100644 tests/google-cloud-python-system/.gitignore create mode 100644 tests/google-cloud-python-system/Dockerfile.in create mode 100755 tests/google-cloud-python-system/run_system_tests.sh delete mode 100644 tests/google-cloud-python/Makefile create mode 100755 tests/google-cloud-python/run_unit_tests.sh create mode 100644 tests/integration/.gitignore delete mode 100644 tests/license-test/Makefile delete mode 100644 tests/no-virtualenv/Makefile delete mode 100644 tests/python2-libraries/Makefile delete mode 100644 tests/python3-libraries/Makefile delete mode 100644 tests/virtualenv/Makefile diff --git a/.gitignore b/.gitignore index c9aa6307..2a84c77d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ -/cloudbuild.yaml +*.pyc +.nox /cloudbuild.yaml_local.sh -/ext_run.sh +/cloudbuild_benchmark.yaml_local.sh +/cloudbuild_system_tests.yaml_local.sh __pycache__ -.nox -*.pyc diff --git a/Makefile b/Makefile deleted file mode 100644 index 395d43b6..00000000 --- a/Makefile +++ /dev/null @@ -1,73 +0,0 @@ -ifneq ($(FORCE_REBUILD),0) - export DOCKER_FLAGS=--no-cache --pull -endif - -ifndef IMAGE_NAME -$(error IMAGE_NAME is not set; invoke make with something like IMAGE_NAME=google/python:2017-01-02_03_45) -endif - -.PHONY: all -all: cloud-test - -## Files that must be refreshed every build - -.PHONY: cloudbuild.yaml # Force reevaluation of env vars every time -cloudbuild.yaml: cloudbuild.yaml.in - envsubst < $< > $@ - - -.PHONY: tests/google-cloud-python/Dockerfile # Force reevaluation of env vars every time -tests/google-cloud-python/Dockerfile: tests/google-cloud-python/Dockerfile.in - envsubst < $< > $@ - -.PHONY: ext_run.sh # Force refetch every time -ext_run.sh: - curl https://raw.githubusercontent.com/GoogleCloudPlatform/runtimes-common/master/structure_tests/ext_run.sh > ext_run.sh - chmod +x ext_run.sh - -## Build using Google Container Builder service - -.PHONY: cloud-build -cloud-build: cloudbuild.yaml tests/google-cloud-python/Dockerfile - gcloud beta container builds submit . --config=cloudbuild.yaml - -.PHONY: cloud-test -# structure-tests and google-cloud-python-tests are implicit in cloud-build -cloud-test: cloud-build integration-tests - -## Build using local Docker daemon - -.PHONY: local-build -local-build: local-build-interpreters - docker build $(DOCKER_FLAGS) -t "$(IMAGE_NAME)" runtime-image - -.PHONY: local-build-interpreters -local-build-interpreters: - make -C python-interpreter-builder build - -.PHONY: local-test -local-test: local-build local-structure-tests local-google-cloud-python-tests integration-tests - -.PHONY: local-structure-tests -local-structure-tests: local-build ext_run.sh - make -C tests structure-tests - -# Unit tests for Google Cloud Client Library for Python -.PHONY: local-google-cloud-python-tests -local-google-cloud-python-tests: tests/google-cloud-python/Dockerfile - make -C tests google-cloud-python - -## Always local - -.PHONY: integration-tests -integration-tests: google-cloud-python-system-tests benchmarks - -# System tests for Google Cloud Client Library for Python. -# They require gcloud auth and network access. -.PHONY: google-cloud-python-system-tests -google-cloud-python-system-tests: - make -C system_tests - -.PHONY: benchmarks -benchmarks: - make -C tests benchmarks diff --git a/README.md b/README.md index 3cef3315..7c14904e 100644 --- a/README.md +++ b/README.md @@ -72,73 +72,7 @@ command or entrypoint. For example: Google regularly builds and releases this image at [`gcr.io/google-appengine/python`](https://gcr.io/google-appengine/python). -To rebuild the image yourself, first set the following variables in your -shell. You need to be authenticated to a Google Cloud Project to invoke the -Google Container Builder service, and also to run the system tests. - -```shell -$ export GOOGLE_CLOUD_PROJECT=YOUR-PROJECT-NAME -$ DOCKER_NAMESPACE=gcr.io/${GCLOUD_PROJECT} -$ CANDIDATE_NAME=`date +%Y-%m-%d_%H_%M` -$ export IMAGE_NAME=${DOCKER_NAMESPACE}/python:${CANDIDATE_NAME} -$ gcloud config set project ${GOOGLE_CLOUD_PROJECT} -``` - -To rebuild the image using the Google Container Builder service, do the -following: - -```shell -$ make cloud-build -$ make cloud-test -``` - -To rebuild the image using your local Docker daemon, do the following: - -``` shell -$ make local-build -$ make local-test -``` - -To open an interactive shell session to this image after building it, do the following: - -``` shell -docker run -it --entrypoint /bin/bash ${IMAGE_NAME} -``` - -## Running the system tests - -To run the system tests, you need a Google Cloud Project with a service account. -From the [Google Cloud Console](https://console.cloud.google.com/), either -create a new project or switch to an existing one. Next, -[create a service account]( -https://cloud.google.com/iam/docs/creating-managing-service-accounts) that will -be used to run the system tests. Once you have a service account, -[create and download a service account key](https://cloud.google.com/iam/docs/managing-service-account-keys). - -In the -[IAM & Admin](https://console.cloud.google.com/permissions/projectpermissions) -section, grant the `Owner` role to the service account you created above. Also -grant the `Editor` role to the `cloud-logs@google.com` service account. - - -Then, follow the -[system test setup instructions](https://github.com/GoogleCloudPlatform/google-cloud-python/blob/master/CONTRIBUTING.rst#running-system-tests). It -describes various steps, including running some scripts to populate and/or -delete datastore example data and indexes (populate_datastore.py, -clear_datastore.py, and `gcloud preview datastore create-indexes -system_tests/data/index.yaml`). - -From the cloud console, you will need to enable the following APIs for your project: - -- Bigquery API -- Cloud Bigtable Admin API -- Google Cloud Natural Language API -- Google Cloud Pub/Sub API -- Google Cloud Storage JSON API -- Google Cloud Vision API -- Google Translate API -- Stackdriver Logging API -- Stackdriver Monitoring API +See [RELEASING.md](RELEASING.md) for more information. ## Contributing changes diff --git a/RELEASING.md b/RELEASING.md new file mode 100644 index 00000000..887ac7ce --- /dev/null +++ b/RELEASING.md @@ -0,0 +1,130 @@ +# Google Cloud Platform - Python Runtime Docker Image + +## `build.sh` + +There is a shell script called `build.sh` that builds everything in this +repository. + +### Environment variables for `build.sh` + +DOCKER_NAMESPACE +: The prefix applied to all images names created. To push images to Google +Container Registry (GCR), this should be `gcr.io/YOUR-PROJECT-NAME`. + +TAG +: The suffix applied to all images created. This should be unique. If not +specified, the current time will be used (timestamp format `YYYY-mm-dd-HHMMSS`). + +GOOGLE_APPLICATION_CREDENTIALS +: (System test only) Path to service account credentials in JSON format. + +GOOGLE_CLOUD_PROJECT +: (System test only) Name of the Google Cloud Platform project to run the system +tests under. + +## Building and Releasing + +A custom Jenkins job builds and releases this repository using scripts and job +configurations that are not yet available publicly. The control flow is as +follows: + +1. Jenkins job `python/release` is invoked by + a. Manually running the script `build_and_release.py` with arguments + b. Manually invoking the job from the GUI +2. The job runs the script `release.sh` + a. Service account credentials are read + b. `gcloud auth activate-service-account` is performed + c. `gcloud config set project` is performed +3. The script invokes `build.sh` in this repository +4. `build.sh` invokes Google Container Builder with the `cloudbuild-*.yaml` + config files. + +## Building outside Jenkins + +To build this repository outside Jenkins, authenticate and authorize yourself +with `gcloud auth`, set the variables listed above, and run: + +``` shell +./build.sh +``` + +This assumes an environment similar to the internal Jenkins environment (Linux, +Debian or Ubuntu-like). + +## Building locally + +To build this repository using local Docker commands instead of the Google +Container Builder service, add the `--local` flag as shown: + +``` shell +./build.sh --local +``` + +To open an interactive shell session to this image after building it, do the +following: + +``` shell +docker run -it --entrypoint /bin/bash YOUR-IMAGE-NAME +``` + +## Running benchmarks + +There is a benchmark suite which compares the performance of interpreters +against each other. + +``` shell +./build.sh --nobuild --benchmark +``` + +Since these benchmarks are run on cloud instances, the timings may vary from run +to run. + +## Running system tests + +**TAKE NOTE: You will incur charges for use of Google Cloud Platform services!** + +System tests perform mutating operations against the real Google Cloud services. +Since these system tests may fail or be flaky for outside reasons such as +netorking issues, configuration errors, or services outages, they are run +separately from building the images, and should be run in their own project. + +To run the system tests, you need a Google Cloud Project with a service account. +From the [Google Cloud Console](https://console.cloud.google.com/), either +create a new project or switch to an existing one. Next, +[create a service account]( +https://cloud.google.com/iam/docs/creating-managing-service-accounts) that will +be used to run the system tests. Once you have a service account, +[create and download a service account key](https://cloud.google.com/iam/docs/managing-service-account-keys). + +In the +[IAM & Admin](https://console.cloud.google.com/permissions/projectpermissions) +section, grant the `Owner` role to the service account you created above. Also +grant the `Editor` role to the `cloud-logs@google.com` service account. + +Then, follow the +[system test setup instructions](https://github.com/GoogleCloudPlatform/google-cloud-python/blob/master/CONTRIBUTING.rst#running-system-tests). It +describes various steps, including running some scripts to populate and/or +delete datastore example data and indexes (populate_datastore.py, +clear_datastore.py, and `gcloud preview datastore create-indexes +system_tests/data/index.yaml`). + +From the cloud console, you will need to enable at least the following APIs for +your project: + +- Bigquery API +- Cloud Bigtable Admin API +- Cloud Spanner API +- Google Cloud Natural Language API +- Google Cloud Pub/Sub API +- Google Cloud Speech API +- Google Cloud Storage JSON API +- Google Cloud Translation API +- Google Cloud Vision API +- Stackdriver Logging API +- Stackdriver Monitoring API + +Once all the setup has been done, run the following: + +``` shell +./build.sh --nobuild --system_tests +``` diff --git a/build.sh b/build.sh index 2eae6ce3..5e4302ba 100755 --- a/build.sh +++ b/build.sh @@ -16,13 +16,140 @@ set -euo pipefail -export IMAGE_NAME=$1 +# Actions +benchmark=0 # Should run benchmarks? +build=1 # Should build images? +system_tests=0 # Should run system tests? -if [ -z "$IMAGE_NAME" ]; then - echo "Usage: ./build.sh [image_path]" - echo "Please provide fully qualified path to target image." +local=0 # Should run using local Docker daemon instead of GCR? + +# Note that $gcloud_cmd has spaces in it +gcloud_cmd="gcloud beta container builds submit ." +local_gcloud_cmd="scripts/local_cloudbuild.py" + +# Helper functions +function fatal() { + echo "$1" >&2 exit 1 +} + +function usage { + fatal "Usage: $0 [OPTION]... +Build and test artifacts in this repository + +Options: + --benchmark: Run benchmarking suite + --[no]build: Build all images (default) + --local: Build images using local Docker daemon + --system_tests: Run system tests +" +} + +# Read environment variables +if [ -z "${DOCKER_NAMESPACE+set}" ] ; then + fatal 'Error: $DOCKER_NAMESPACE is not set; invoke with something like DOCKER_NAMESPACE=gcr.io/YOUR-PROJECT-NAME' +fi + +if [ -z "${TAG+set}" ] ; then + export TAG=`date +%Y-%m-%d-%H%M%S` +fi + +substitutions="_DOCKER_NAMESPACE=${DOCKER_NAMESPACE},_TAG=${TAG}" + +# Read command line arguments +while [ $# -gt 0 ]; do + case "$1" in + --benchmark) + benchmark=1 + shift + ;; + --build) + build=1 + shift + ;; + --nobuild) + build=0 + shift + ;; + --local) + local=1 + shift + ;; + --system_tests) + system_tests=1 + shift + ;; + *) + usage + ;; + esac +done + +# If no actions chosen, then tell the user +if [ "${benchmark}" -eq 0 -a "${build}" -eq 0 -a "${system_tests}" -eq 0 ]; then + fatal 'Error: No actions specified (for example, --build), exiting' +fi + +# Running build local or remote? +if [ "${local}" -eq 1 ]; then + gcloud_cmd="${local_gcloud_cmd}" +fi + +# Read action-specific environment variables +if [ "${system_tests}" -eq 1 ]; then + if [ -z "${GOOGLE_APPLICATION_CREDENTIALS+set}" ] ; then + fatal 'Error: $GOOGLE_APPLICATION_CREDENTIALS is not set; invoke with something like GOOGLE_APPLICATION_CREDENTIALS=/path/to/service/account/creds.json' + fi + + if [ -z "${GOOGLE_CLOUD_PROJECT+set}" ] ; then + fatal 'Error: $GOOGLE_CLOUD_PROJECT is not set; invoke with something like GOOGLE_CLOUD_PROJECT=YOUR-PROJECT-NAME' + fi +fi + +# Use latest released Debian as our base image +export DEBIAN_BASE_IMAGE="gcr.io/google-appengine/debian8:latest" +export FULL_BASE_IMAGE="${DOCKER_NAMESPACE}/python:${TAG}" +echo "Using base image name ${FULL_BASE_IMAGE}" + +# Generate Dockerfiles +for outfile in \ + python-interpreter-builder/Dockerfile \ + runtime-image/Dockerfile \ + tests/benchmark/Dockerfile \ + tests/google-cloud-python/Dockerfile \ + tests/google-cloud-python-system/Dockerfile \ + tests/integration/Dockerfile \ + ; do + envsubst <"${outfile}".in >"${outfile}" '$DEBIAN_BASE_IMAGE $FULL_BASE_IMAGE $GOOGLE_CLOUD_PROJECT' +done + +# Build images and push to GCR +if [ "${build}" -eq 1 ]; then + echo "Building images" + ${gcloud_cmd} --config cloudbuild.yaml --substitutions "${substitutions}" +fi + +# If both system tests and benchmarks are requested, run them both +# even if one or the other has errors. If the build step had errors, +# this script will have already exited. +exit_code=0 + +# Run system tests +if [ "${system_tests}" -eq 1 ]; then + echo "Running system tests using project ${GOOGLE_CLOUD_PROJECT}" + + trap "rm -f tests/google-cloud-python-system/credentials.json" EXIT + cp "${GOOGLE_APPLICATION_CREDENTIALS}" tests/google-cloud-python-system/credentials.json + ${gcloud_cmd} --config cloudbuild_system_tests.yaml --substitutions "${substitutions}" || \ + exit_code=1 + rm -f tests/google-cloud-python-system/credentials.json +fi + +# Run benchmarks +if [ "${benchmark}" -eq 1 ] ; then + echo "Running benchmark" + ${gcloud_cmd} --config cloudbuild_benchmark.yaml --substitutions "${substitutions}" || \ + exit_code=1 fi -envsubst < cloudbuild.yaml.in > cloudbuild.yaml -gcloud beta container builds submit . --config=cloudbuild.yaml +exit ${exit_code} diff --git a/cloudbuild.yaml b/cloudbuild.yaml new file mode 100644 index 00000000..32c501b2 --- /dev/null +++ b/cloudbuild.yaml @@ -0,0 +1,34 @@ +timeout: 7200s +steps: +- name: gcr.io/cloud-builders/docker:latest + # Compile Python interpreters from source + args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/interpreter-builder:${_TAG}', + '--no-cache', '/workspace/python-interpreter-builder/'] +- name: ${_DOCKER_NAMESPACE}/python/interpreter-builder:${_TAG} + # Copy interpreters back to workspace + args: ['cp', '/interpreters.tar.gz', '/workspace/runtime-image/interpreters.tar.gz'] +- name: gcr.io/cloud-builders/docker:latest + # Build base runtime image + args: ['build', '--tag=${_DOCKER_NAMESPACE}/python:${_TAG}', + '--no-cache', '/workspace/runtime-image/'] +- name: gcr.io/gcp-runtimes/structure_test:latest + # Validate structure of base runtime image + args: [ + '-i', '${_DOCKER_NAMESPACE}/python:${_TAG}', + '--config', '/workspace/tests/virtualenv/virtualenv_default.yaml', + '--config', '/workspace/tests/virtualenv/virtualenv_python34.yaml', + '--config', '/workspace/tests/virtualenv/virtualenv_python35.yaml', + '--config', '/workspace/tests/no-virtualenv/no-virtualenv.yaml', + '--config', '/workspace/tests/python2-libraries/python2-libraries.yaml', + '--config', '/workspace/tests/python3-libraries/python3-libraries.yaml', + '--config', '/workspace/tests/license-test/license-test.yaml', + '-v' + ] +- name: gcr.io/cloud-builders/docker:latest + # Run google client library unit tests against base runtime image + args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/google-cloud-python:${_TAG}', + '--no-cache', '/workspace/tests/google-cloud-python/'] +- name: ${_DOCKER_NAMESPACE}/python/tests/google-cloud-python:${_TAG} +images: [ + '${_DOCKER_NAMESPACE}/python:${_TAG}' +] diff --git a/cloudbuild.yaml.in b/cloudbuild.yaml.in deleted file mode 100644 index 4c505105..00000000 --- a/cloudbuild.yaml.in +++ /dev/null @@ -1,24 +0,0 @@ -timeout: 7200s -steps: -- name: gcr.io/cloud-builders/docker - args: ['build', '--tag=interpreter', '--no-cache', '/workspace/python-interpreter-builder/'] -- name: interpreter - args: ['cp', '/interpreters.tar.gz', '/workspace/runtime-image/interpreters.tar.gz'] -- name: gcr.io/cloud-builders/docker - args: ['build', '--tag=${IMAGE_NAME}', '--no-cache', '/workspace/runtime-image/'] -- name: gcr.io/gcp-runtimes/structure_test - args: [ - '-i', '${IMAGE_NAME}', - '--config', '/workspace/tests/virtualenv/virtualenv_default.yaml', - '--config', '/workspace/tests/virtualenv/virtualenv_python34.yaml', - '--config', '/workspace/tests/virtualenv/virtualenv_python35.yaml', - '--config', '/workspace/tests/no-virtualenv/no-virtualenv.yaml', - '--config', '/workspace/tests/python2-libraries/python2-libraries.yaml', - '--config', '/workspace/tests/python3-libraries/python3-libraries.yaml', - '--config', '/workspace/tests/license-test/license-test.yaml', - '-v' - ] -- name: gcr.io/cloud-builders/docker - args: ['build', '--tag=google-cloud-python-tests', '--no-cache', '/workspace/tests/google-cloud-python/'] -images: - ['${IMAGE_NAME}'] diff --git a/cloudbuild_benchmark.yaml b/cloudbuild_benchmark.yaml new file mode 100644 index 00000000..a960bc9b --- /dev/null +++ b/cloudbuild_benchmark.yaml @@ -0,0 +1,8 @@ +timeout: 3600s +steps: +- name: gcr.io/cloud-builders/docker:latest + args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/benchmark:${_TAG}', + '--no-cache', '/workspace/tests/benchmark/'] +images: [ + # Intentionally empty +] diff --git a/cloudbuild_system_tests.yaml b/cloudbuild_system_tests.yaml new file mode 100644 index 00000000..3cac03e2 --- /dev/null +++ b/cloudbuild_system_tests.yaml @@ -0,0 +1,9 @@ +timeout: 3600s +steps: +- name: gcr.io/cloud-builders/docker:latest + args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/google-cloud-python-system:${_TAG}', + '--no-cache', '/workspace/tests/google-cloud-python-system/'] +- name: ${_DOCKER_NAMESPACE}/python/tests/google-cloud-python-system:${_TAG} +images: [ + # Intentionally empty +] diff --git a/jenkins_build.sh b/jenkins_build.sh index 253d5ca0..f88007f5 100755 --- a/jenkins_build.sh +++ b/jenkins_build.sh @@ -1,45 +1,3 @@ #!/bin/sh -set -eu - -RUNTIME_NAME="python" - -if [ -z "${TAG}" ] ; then - TAG=`date +%Y-%m-%d_%H_%M` -fi - -CANDIDATE_NAME="${TAG}" -echo "CANDIDATE_NAME:${CANDIDATE_NAME}" - -if [ -z "${DOCKER_NAMESPACE+set}" ] ; then - echo "Error: DOCKER_NAMESPACE is not set; invoke with something like DOCKER_NAMESPACE=gcr.io/YOUR-PROJECT-NAME" >&2 - exit 1 -fi -export IMAGE_NAME="${DOCKER_NAMESPACE}/${RUNTIME_NAME}:${CANDIDATE_NAME}" - -if [ -z "${GOOGLE_CLOUD_PROJECT+set}" ] ; then - echo "Error: GOOGLE_CLOUD_PROJECT is not set; invoke with something like GOOGLE_CLOUD_PROJECT=YOUR-PROJECT-NAME" >&2 - exit 1 -fi - -export FORCE_REBUILD - -echo "==================================================================" -echo "Building image ${IMAGE_NAME} and pushing to ${DOCKER_NAMESPACE} using ${GOOGLE_CLOUD_PROJECT}" -echo "==================================================================" - -make cloud-build - -# We explicitly pull the image using 'gcloud', instead of letting -# Docker do it, so that we have the right credentials. -echo "==================================================================" -gcloud info -echo gcloud docker -- pull "${IMAGE_NAME}" -gcloud docker -- pull "${IMAGE_NAME}" -echo "==================================================================" - -# Note that system test failures might be caused environment factors -# outside our control. Also, the images will be pushed to GCR by the -# previous build step regardless of system test failures. -make integration-tests || \ - echo "ERROR: System test failure, please examine the logs" +./build.sh "$@" diff --git a/python-interpreter-builder/.gitignore b/python-interpreter-builder/.gitignore index 53752db2..94143827 100644 --- a/python-interpreter-builder/.gitignore +++ b/python-interpreter-builder/.gitignore @@ -1 +1 @@ -output +Dockerfile diff --git a/python-interpreter-builder/Dockerfile b/python-interpreter-builder/Dockerfile.in similarity index 94% rename from python-interpreter-builder/Dockerfile rename to python-interpreter-builder/Dockerfile.in index e3dffd4c..54181518 100644 --- a/python-interpreter-builder/Dockerfile +++ b/python-interpreter-builder/Dockerfile.in @@ -1,6 +1,6 @@ # The Google App Engine base image is debian (jessie) with ca-certificates # installed. -FROM gcr.io/google-appengine/debian8 +FROM ${DEBIAN_BASE_IMAGE} # Install Python build dependencies RUN apt-get update && apt-get install -yq \ diff --git a/python-interpreter-builder/Makefile b/python-interpreter-builder/Makefile deleted file mode 100644 index d125b1c8..00000000 --- a/python-interpreter-builder/Makefile +++ /dev/null @@ -1,10 +0,0 @@ -.PHONY: build -build: - docker build $(DOCKER_FLAGS) -t google/python-interpreter-builder . - # Extract the built interpreters - # This is needed because `docker cp` doesn't work on images, just containers. - -docker rm python-interpreter-builder - docker run --name python-interpreter-builder google/python-interpreter-builder /bin/bash - mkdir -p output - docker cp python-interpreter-builder:/interpreters.tar.gz ../runtime-image/interpreters.tar.gz - docker rm python-interpreter-builder diff --git a/runtime-image/.gitignore b/runtime-image/.gitignore index cb1afebc..94143827 100644 --- a/runtime-image/.gitignore +++ b/runtime-image/.gitignore @@ -1 +1 @@ -interpreters.tar.gz +Dockerfile diff --git a/runtime-image/Dockerfile b/runtime-image/Dockerfile.in similarity index 96% rename from runtime-image/Dockerfile rename to runtime-image/Dockerfile.in index 03cd19fc..15cf3ee4 100644 --- a/runtime-image/Dockerfile +++ b/runtime-image/Dockerfile.in @@ -1,7 +1,7 @@ # The Google App Engine base image is debian (jessie) with ca-certificates # installed. # Source: https://github.com/GoogleCloudPlatform/debian-docker -FROM gcr.io/google-appengine/debian8 +FROM ${DEBIAN_BASE_IMAGE} ADD resources /resources ADD scripts /scripts diff --git a/system_tests/.gitignore b/system_tests/.gitignore deleted file mode 100644 index 3a8c6aa8..00000000 --- a/system_tests/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -data/ -secrets.tar -Dockerfile diff --git a/system_tests/Dockerfile.in b/system_tests/Dockerfile.in deleted file mode 100644 index 97f8e004..00000000 --- a/system_tests/Dockerfile.in +++ /dev/null @@ -1,15 +0,0 @@ -FROM ${IMAGE_NAME} - -# Secrets injected at runtime -ENV GOOGLE_APPLICATION_CREDENTIALS=/app/credentials/credentials.json -ENV GOOGLE_CLOUD_PROJECT=${GOOGLE_CLOUD_PROJECT} - -# Get the source. -RUN git clone --depth 1 https://github.com/GoogleCloudPlatform/google-cloud-python.git -WORKDIR google-cloud-python - -# Install tox for running the system tests -RUN pip install --upgrade tox - -# Run Python 2.7, 3.5 system tests -ENTRYPOINT ["tox", "-e", "system-tests,system-tests3"] diff --git a/system_tests/Makefile b/system_tests/Makefile deleted file mode 100644 index cec219ca..00000000 --- a/system_tests/Makefile +++ /dev/null @@ -1,22 +0,0 @@ -# Use no-cache to prevent layer caching because there is a layer that does -# a `git clone` which can not be cached. -DOCKER_FLAGS ?= --no-cache - -ifndef GOOGLE_APPLICATION_CREDENTIALS -$(error GOOGLE_APPLICATION_CREDENTIALS is not set; download service account credentials in JSON format from the Google Cloud Console and invoke make with something like GOOGLE_APPLICATION_CREDENTIALS=/path/to/credentials.json) -endif - -ifndef GOOGLE_CLOUD_PROJECT -$(error GOOGLE_CLOUD_PROJECT is not set; invoke make with something like GOOGLE_CLOUD_PROJECT=my-project-name) -endif - - -.PHONY: all -all: Dockerfile - @echo "Running system tests in project ${GOOGLE_CLOUD_PROJECT} using service account credentials from ${GOOGLE_APPLICATION_CREDENTIALS}" - docker build --tag google-cloud-python-system-tests $(DOCKER_FLAGS) . - docker run --rm -v $(GOOGLE_APPLICATION_CREDENTIALS):/app/credentials/credentials.json google-cloud-python-system-tests - -.PHONY: Dockerfile -Dockerfile: Dockerfile.in - envsubst < $< > $@ diff --git a/tests/Makefile b/tests/Makefile deleted file mode 100644 index 0535885b..00000000 --- a/tests/Makefile +++ /dev/null @@ -1,18 +0,0 @@ -.PHONY: all -all: structure-tests benchmarks google-cloud-python - -.PHONY: structure-tests -structure-tests: - make -C no-virtualenv - make -C virtualenv - make -C python2-libraries - make -C python3-libraries - make -C license-test - -.PHONY: benchmarks -benchmarks: - make -C benchmark all - -.PHONY: google-cloud-python -google-cloud-python: - make -C google-cloud-python all diff --git a/tests/benchmark/Dockerfile.2vs3 b/tests/benchmark/Dockerfile.2vs3 deleted file mode 100644 index 507cd4bb..00000000 --- a/tests/benchmark/Dockerfile.2vs3 +++ /dev/null @@ -1 +0,0 @@ -RUN python perf.py -r -b default /usr/bin/python2.7 /usr/bin/python3.4 diff --git a/tests/benchmark/Dockerfile.34vs35 b/tests/benchmark/Dockerfile.34vs35 deleted file mode 100644 index d319d14d..00000000 --- a/tests/benchmark/Dockerfile.34vs35 +++ /dev/null @@ -1 +0,0 @@ -RUN python perf.py -r -b default /usr/bin/python3.4 /opt/python3.5/bin/python3.5 diff --git a/tests/benchmark/Dockerfile.in b/tests/benchmark/Dockerfile.in index c202cb2f..e39eac1c 100644 --- a/tests/benchmark/Dockerfile.in +++ b/tests/benchmark/Dockerfile.in @@ -1,4 +1,6 @@ -FROM ${IMAGE_NAME} +FROM ${FULL_BASE_IMAGE} -RUN hg clone https://hg.python.org/benchmarks /app/benchmarks +RUN hg clone -r 9923b81a1d34 https://hg.python.org/benchmarks /app/benchmarks WORKDIR /app/benchmarks +RUN python perf.py -r -b default /usr/bin/python2.7 /usr/bin/python3.4 +RUN python perf.py -r -b default /usr/bin/python3.4 /opt/python3.5/bin/python3.5 diff --git a/tests/benchmark/Makefile b/tests/benchmark/Makefile deleted file mode 100644 index dd56208a..00000000 --- a/tests/benchmark/Makefile +++ /dev/null @@ -1,14 +0,0 @@ -.PHONY: all -all: 2vs3 34vs35 - -# Tests Python 3.4 against Python 2.7. -.PHONY: 2vs3 -2vs3: - envsubst < Dockerfile.in > Dockerfile - cat Dockerfile Dockerfile.2vs3 | docker build - - -# Tests Python 3.5 against Python 3.4. -.PHONY: 2vs3 -34vs35: - envsubst < Dockerfile.in > Dockerfile - cat Dockerfile Dockerfile.34vs35 | docker build - diff --git a/tests/google-cloud-python-system/.gitignore b/tests/google-cloud-python-system/.gitignore new file mode 100644 index 00000000..470715ec --- /dev/null +++ b/tests/google-cloud-python-system/.gitignore @@ -0,0 +1,2 @@ +credentials.json +Dockerfile diff --git a/tests/google-cloud-python-system/Dockerfile.in b/tests/google-cloud-python-system/Dockerfile.in new file mode 100644 index 00000000..86a2d97d --- /dev/null +++ b/tests/google-cloud-python-system/Dockerfile.in @@ -0,0 +1,16 @@ +FROM ${FULL_BASE_IMAGE} + +# Get the source. +RUN git clone --depth 1 https://github.com/GoogleCloudPlatform/google-cloud-python.git +WORKDIR google-cloud-python + +# Install nox +RUN pip install --upgrade nox-automation + +# Secrets injected at runtime +ENV GOOGLE_APPLICATION_CREDENTIALS=/workspace/tests/google-cloud-python-system/credentials.json +ENV GOOGLE_CLOUD_PROJECT=${GOOGLE_CLOUD_PROJECT} + +# Run system tests for all supported Python versions +ADD run_system_tests.sh /run_system_tests.sh +ENTRYPOINT ["/run_system_tests.sh"] diff --git a/tests/google-cloud-python-system/run_system_tests.sh b/tests/google-cloud-python-system/run_system_tests.sh new file mode 100755 index 00000000..aab10c88 --- /dev/null +++ b/tests/google-cloud-python-system/run_system_tests.sh @@ -0,0 +1,34 @@ +#!/bin/sh +set -eu + +cd /app/google-cloud-python + +# Not all packages have system tests +packages=" +bigquery +bigtable +datastore +language +logging +monitoring +pubsub +spanner +speech +storage +vision +" + +# translate has system test but it gives error message: +# BadRequest: 400 Invalid JSON payload received. Unknown name "model": Cannot bind 'nmt'. Field 'model' could not be found in request message. (GET https://translation.googleapis.com/language/translate/v2?target=de&q=hvala+ti&q=dankon&q=Me+llamo+Jeff&q=My+name+is+Jeff&model=nmt) +disabled_packages="translate" + +# Spanner system test needs this +export GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE=1 + +exit_code=0 +for package in ${packages}; do + noxfile="${package}/nox.py" + nox -f "${noxfile}" -e "system_tests(python_version='2.7')" || exit_code=1 +done + +exit "${exit_code}" diff --git a/tests/google-cloud-python/Dockerfile.in b/tests/google-cloud-python/Dockerfile.in index 94bf4648..ada429fb 100644 --- a/tests/google-cloud-python/Dockerfile.in +++ b/tests/google-cloud-python/Dockerfile.in @@ -1,17 +1,12 @@ -FROM ${IMAGE_NAME} - -# Install tox -RUN pip install --upgrade tox +FROM ${FULL_BASE_IMAGE} # Get the source. -RUN git clone https://github.com/GoogleCloudPlatform/google-cloud-python.git +RUN git clone --depth 1 https://github.com/GoogleCloudPlatform/google-cloud-python.git WORKDIR google-cloud-python -# Run Python 2.7 unit tests -RUN python2.7 scripts/run_unit_tests.py - -# Run Python 3.4 unit tests -RUN python3.4 scripts/run_unit_tests.py +# Install nox +RUN pip install --upgrade nox-automation -# Run Python 3.5 unit tests -RUN python3.5 scripts/run_unit_tests.py +# Run unit tests for all supported Python versions +ADD run_unit_tests.sh /run_unit_tests.sh +ENTRYPOINT ["/run_unit_tests.sh"] diff --git a/tests/google-cloud-python/Makefile b/tests/google-cloud-python/Makefile deleted file mode 100644 index 6f9c5522..00000000 --- a/tests/google-cloud-python/Makefile +++ /dev/null @@ -1,5 +0,0 @@ -.PHONY: all -all: - # Use no-cache to prevent layer caching because there is a layer that does - # a `git clone` which can not be cached. - docker build --no-cache . diff --git a/tests/google-cloud-python/run_unit_tests.sh b/tests/google-cloud-python/run_unit_tests.sh new file mode 100755 index 00000000..8563606e --- /dev/null +++ b/tests/google-cloud-python/run_unit_tests.sh @@ -0,0 +1,11 @@ +#!/bin/sh +set -eu + +cd /app/google-cloud-python + +exit_code=0 +for noxfile in */nox.py; do + nox -f "${noxfile}" -e "unit_tests(python_version='2.7')" "unit_tests(python_version='3.4')" "unit_tests(python_version='3.5')" || exit_code=1 +done + +exit "${exit_code}" diff --git a/tests/integration/.gitignore b/tests/integration/.gitignore new file mode 100644 index 00000000..94143827 --- /dev/null +++ b/tests/integration/.gitignore @@ -0,0 +1 @@ +Dockerfile diff --git a/tests/integration/Dockerfile.in b/tests/integration/Dockerfile.in index e18d60ef..52f848f6 100644 --- a/tests/integration/Dockerfile.in +++ b/tests/integration/Dockerfile.in @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM ${STAGING_IMAGE} +FROM ${FULL_BASE_IMAGE} COPY . /app WORKDIR /app diff --git a/tests/license-test/Makefile b/tests/license-test/Makefile deleted file mode 100644 index b56b60ea..00000000 --- a/tests/license-test/Makefile +++ /dev/null @@ -1,3 +0,0 @@ -.PHONY: all -all: - ../../ext_run.sh -i "${IMAGE_NAME}" -c license-test.yaml -v diff --git a/tests/no-virtualenv/Makefile b/tests/no-virtualenv/Makefile deleted file mode 100644 index 2d9ec8e3..00000000 --- a/tests/no-virtualenv/Makefile +++ /dev/null @@ -1,3 +0,0 @@ -.PHONY: all -all: - ../../ext_run.sh -i "${IMAGE_NAME}" -c no-virtualenv.yaml -v diff --git a/tests/python2-libraries/Makefile b/tests/python2-libraries/Makefile deleted file mode 100644 index 13ab239e..00000000 --- a/tests/python2-libraries/Makefile +++ /dev/null @@ -1,3 +0,0 @@ -.PHONY: all -all: - ../../ext_run.sh -i "${IMAGE_NAME}" -c python2-libraries.yaml -w ../.. -v diff --git a/tests/python3-libraries/Makefile b/tests/python3-libraries/Makefile deleted file mode 100644 index 92fb683b..00000000 --- a/tests/python3-libraries/Makefile +++ /dev/null @@ -1,3 +0,0 @@ -.PHONY: all -all: - ../../ext_run.sh -i "${IMAGE_NAME}" -c python3-libraries.yaml -w ../.. -v diff --git a/tests/virtualenv/Makefile b/tests/virtualenv/Makefile deleted file mode 100644 index 66a19642..00000000 --- a/tests/virtualenv/Makefile +++ /dev/null @@ -1,5 +0,0 @@ -.PHONY: all -all: - ../../ext_run.sh -i "${IMAGE_NAME}" -c virtualenv_default.yaml -v - ../../ext_run.sh -i "${IMAGE_NAME}" -c virtualenv_python34.yaml -v - ../../ext_run.sh -i "${IMAGE_NAME}" -c virtualenv_python35.yaml -v From 7bee8a75c65c3eab1e23e87faa86492a08dbf93f Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 7 Apr 2017 14:43:43 -0700 Subject: [PATCH 037/256] Change handling of GOOGLE_CLOUD_PROJECT in build.sh. The previous behavior could possibly, conceivably be causing problems. Now, the user now specifies GOOGLE_CLOUD_PROJECT_FOR_TESTS, which should not collide with anything, and the GOOGLE_CLOUD_PROJECT environment variable is only set for the single thing that should need it, which is run_system_tests.sh, and only in the single, ephemeral Docker container where we run those system tests. --- RELEASING.md | 2 +- build.sh | 8 ++++---- tests/google-cloud-python-system/Dockerfile.in | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/RELEASING.md b/RELEASING.md index 887ac7ce..01f7077d 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -18,7 +18,7 @@ specified, the current time will be used (timestamp format `YYYY-mm-dd-HHMMSS`). GOOGLE_APPLICATION_CREDENTIALS : (System test only) Path to service account credentials in JSON format. -GOOGLE_CLOUD_PROJECT +GOOGLE_CLOUD_PROJECT_FOR_TESTS : (System test only) Name of the Google Cloud Platform project to run the system tests under. diff --git a/build.sh b/build.sh index 5e4302ba..bc80277c 100755 --- a/build.sh +++ b/build.sh @@ -101,8 +101,8 @@ if [ "${system_tests}" -eq 1 ]; then fatal 'Error: $GOOGLE_APPLICATION_CREDENTIALS is not set; invoke with something like GOOGLE_APPLICATION_CREDENTIALS=/path/to/service/account/creds.json' fi - if [ -z "${GOOGLE_CLOUD_PROJECT+set}" ] ; then - fatal 'Error: $GOOGLE_CLOUD_PROJECT is not set; invoke with something like GOOGLE_CLOUD_PROJECT=YOUR-PROJECT-NAME' + if [ -z "${GOOGLE_CLOUD_PROJECT_FOR_TESTS+set}" ] ; then + fatal 'Error: $GOOGLE_CLOUD_PROJECT_FOR_TESTS is not set; invoke with something like GOOGLE_CLOUD_PROJECT_FOR_TESTS=YOUR-PROJECT-NAME' fi fi @@ -120,7 +120,7 @@ for outfile in \ tests/google-cloud-python-system/Dockerfile \ tests/integration/Dockerfile \ ; do - envsubst <"${outfile}".in >"${outfile}" '$DEBIAN_BASE_IMAGE $FULL_BASE_IMAGE $GOOGLE_CLOUD_PROJECT' + envsubst <"${outfile}".in >"${outfile}" '$DEBIAN_BASE_IMAGE $FULL_BASE_IMAGE $GOOGLE_CLOUD_PROJECT_FOR_TESTS' done # Build images and push to GCR @@ -136,7 +136,7 @@ exit_code=0 # Run system tests if [ "${system_tests}" -eq 1 ]; then - echo "Running system tests using project ${GOOGLE_CLOUD_PROJECT}" + echo "Running system tests using project ${GOOGLE_CLOUD_PROJECT_FOR_TESTS}" trap "rm -f tests/google-cloud-python-system/credentials.json" EXIT cp "${GOOGLE_APPLICATION_CREDENTIALS}" tests/google-cloud-python-system/credentials.json diff --git a/tests/google-cloud-python-system/Dockerfile.in b/tests/google-cloud-python-system/Dockerfile.in index 86a2d97d..18506681 100644 --- a/tests/google-cloud-python-system/Dockerfile.in +++ b/tests/google-cloud-python-system/Dockerfile.in @@ -9,7 +9,7 @@ RUN pip install --upgrade nox-automation # Secrets injected at runtime ENV GOOGLE_APPLICATION_CREDENTIALS=/workspace/tests/google-cloud-python-system/credentials.json -ENV GOOGLE_CLOUD_PROJECT=${GOOGLE_CLOUD_PROJECT} +ENV GOOGLE_CLOUD_PROJECT=${GOOGLE_CLOUD_PROJECT_FOR_TESTS} # Run system tests for all supported Python versions ADD run_system_tests.sh /run_system_tests.sh From 9f6233a2be614ad136270bee44169de1d7c0c329 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 7 Apr 2017 15:55:51 -0700 Subject: [PATCH 038/256] Rename GOOGLE_APPLICATIONS_CREDENTIALS. Changed to GOOGLE_APPLICATIONS_CREDENTIALS_FOR_TESTS to match the handling of GOOGLE_CLOUD_PROJECT. --- RELEASING.md | 2 +- build.sh | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/RELEASING.md b/RELEASING.md index 01f7077d..1f92084a 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -15,7 +15,7 @@ TAG : The suffix applied to all images created. This should be unique. If not specified, the current time will be used (timestamp format `YYYY-mm-dd-HHMMSS`). -GOOGLE_APPLICATION_CREDENTIALS +GOOGLE_APPLICATION_CREDENTIALS_FOR_TESTS : (System test only) Path to service account credentials in JSON format. GOOGLE_CLOUD_PROJECT_FOR_TESTS diff --git a/build.sh b/build.sh index bc80277c..5929fbb2 100755 --- a/build.sh +++ b/build.sh @@ -97,8 +97,8 @@ fi # Read action-specific environment variables if [ "${system_tests}" -eq 1 ]; then - if [ -z "${GOOGLE_APPLICATION_CREDENTIALS+set}" ] ; then - fatal 'Error: $GOOGLE_APPLICATION_CREDENTIALS is not set; invoke with something like GOOGLE_APPLICATION_CREDENTIALS=/path/to/service/account/creds.json' + if [ -z "${GOOGLE_APPLICATION_CREDENTIALS_FOR_TESTS+set}" ] ; then + fatal 'Error: $GOOGLE_APPLICATION_CREDENTIALS_FOR_TESTS is not set; invoke with something like GOOGLE_APPLICATION_CREDENTIALS_FOR_TESTS=/path/to/service/account/creds.json' fi if [ -z "${GOOGLE_CLOUD_PROJECT_FOR_TESTS+set}" ] ; then @@ -139,7 +139,7 @@ if [ "${system_tests}" -eq 1 ]; then echo "Running system tests using project ${GOOGLE_CLOUD_PROJECT_FOR_TESTS}" trap "rm -f tests/google-cloud-python-system/credentials.json" EXIT - cp "${GOOGLE_APPLICATION_CREDENTIALS}" tests/google-cloud-python-system/credentials.json + cp "${GOOGLE_APPLICATION_CREDENTIALS_FOR_TESTS}" tests/google-cloud-python-system/credentials.json ${gcloud_cmd} --config cloudbuild_system_tests.yaml --substitutions "${substitutions}" || \ exit_code=1 rm -f tests/google-cloud-python-system/credentials.json From 092807920eeddfc37b403ff5210d8431b6fd74c6 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Mon, 10 Apr 2017 09:08:20 +0000 Subject: [PATCH 039/256] Auto-update dependencies. --- tests/integration/requirements.txt | 8 ++--- tests/python2-libraries/requirements.txt | 44 ++++++++++++------------ tests/python3-libraries/requirements.txt | 32 ++++++++--------- 3 files changed, 42 insertions(+), 42 deletions(-) diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index c5004b32..c5c38c56 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -1,7 +1,7 @@ -Flask==0.12 -google-cloud-error-reporting==0.23.2 -google-cloud-logging==0.23.1 -google-cloud-monitoring==0.23.0 +Flask==0.12.1 +google-cloud-error-reporting==0.24.0 +google-cloud-logging==1.0.0 +google-cloud-monitoring==0.24.0 gunicorn==19.7.1 requests==2.13.0 retrying==1.3.3 diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index d92af43f..48d0bf43 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -6,7 +6,7 @@ anyjson==0.3.3 apache-libcloud==1.5.0 argparse==1.4.0 astroid==1.4.9 -awscli==1.11.68 +awscli==1.11.75 babel==2.4.0 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 @@ -16,7 +16,7 @@ billiard==3.5.0.2 blessings==1.6 blinker==1.4 boto==2.46.1 -botocore==1.5.31 +botocore==1.5.38 bottle==0.12.13 carbon==0.9.15 celery==4.0.2 @@ -24,7 +24,7 @@ certifi==2017.1.23 cffi==1.10.0 chardet==2.3.0 click==6.7 -cliff==2.4.0 +cliff==2.5.0 cmd2==0.7.0 colorama==0.3.7 configobj==5.0.6 @@ -38,21 +38,21 @@ decorator==4.0.11 django-celery==3.2.1 django-debug-toolbar==1.7 django-extensions==1.7.8 -django==1.10.6 +django==1.11 django_compress==1.0.1 djangorestframework==3.6.2 docker-py==1.10.6 docopt==0.6.2 docutils==0.13.1 ecdsa==0.13 -elasticsearch==5.2.0 +elasticsearch==5.3.0 enum34==1.1.6 -eventlet==0.20.1 +eventlet==0.21.0 extras==1.0.0 fabric==1.13.1 fixtures==3.0.0 flake8==3.3.0 -flask==0.12 +flask==0.12.1 funcsigs==1.0.2 functools32==3.2.3.post2 futures==3.0.5 @@ -70,7 +70,7 @@ ipython==5.3.0 iso8601==0.1.11 isodate==0.5.4 itsdangerous==0.24 -jinja2==2.9.5 +jinja2==2.9.6 jmespath==0.9.2 jsonschema==2.6.0 kombu==4.0.2 @@ -87,7 +87,7 @@ mccabe==0.6.1 meld3==1.0.2 mock==2.0.0 mozcrash==1.0 -mozdevice==0.48 +mozdevice==0.50 mozfile==1.2 mozinfo==0.9 mozlog==3.4 @@ -107,7 +107,7 @@ oauth2==1.9.0.post1 oauth2client==4.0.0 oauthlib==2.0.2 ordereddict==1.1 -oslo.config==3.23.0 +oslo.config==3.24.0 pandas==0.19.2 paramiko==2.1.2 passlib==1.7.1 @@ -118,7 +118,7 @@ pbr==2.0.0 pep8==1.7.0 pexpect==4.2.1 pika==0.10.0 -pillow==4.0.0 +pillow==4.1.0 pip==9.0.1 prettytable protobuf==3.2.0 @@ -136,7 +136,7 @@ pyjwt==1.4.2 pylibmc==1.5.2 pylint==1.6.5 pymongo==3.4.0 -pymysql==0.7.10 +pymysql==0.7.11 pyopenssl==16.2.0 pyparsing==2.2.0 pyramid==1.8.3 @@ -150,7 +150,7 @@ python-gflags==3.1.1 python-keystoneclient==3.10.0 python-memcached==1.58 python-mimeparse==1.6.0 -python-novaclient==7.1.0 +python-novaclient==8.0.0 python-subunit==1.2.0 python-swiftclient==3.3.0 pytz==2017.2 @@ -164,17 +164,17 @@ requests==2.13.0 retrying==1.3.3 rsa==3.4.2 scipy==0.19.0 -selenium==3.3.1 +selenium==3.3.3 setuptools-git==1.2 -setuptools==34.3.3 -sh==1.12.11 +setuptools==34.4.0 +sh==1.12.13 simplejson==3.10.0 six==1.10.0 snowballstemmer==1.2.1 south==1.0.2 -sphinx==1.5.3 +sphinx==1.5.5 sqlalchemy-migrate==0.11.0 -sqlalchemy==1.1.7 +sqlalchemy==1.1.9 sqlparse==0.2.3 statsd==3.2.1 stevedore==1.21.0 @@ -183,15 +183,15 @@ supervisor==3.3.1 testrepository==0.0.20 testtools==2.2.0 thrift==0.10.0 -tornado==4.4.2 -tox==2.6.0 +tornado==4.4.3 +tox==2.7.0 twisted==17.1.0 ujson==1.35 unidecode==0.4.20 unittest2==1.1.0 uritemplate==3.0.0 urllib3==1.20 -uwsgi==2.0.14 +uwsgi==2.0.15 versiontools==1.9.1 virtualenv==15.1.0 waitress==1.0.2 @@ -202,5 +202,5 @@ webtest==2.0.27 werkzeug==0.12.1 wheel==0.29.0 xlrd==1.0.0 -zc.buildout==2.9.2 +zc.buildout==2.9.3 zope.interface==4.3.3 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index a7269999..41b23682 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -18,7 +18,7 @@ certifi==2017.1.23 cffi==1.10.0 chardet==2.3.0 click==6.7 -cliff==2.4.0 +cliff==2.5.0 cmd2==0.7.0 colorama==0.3.7 configobj==5.0.6 @@ -31,19 +31,19 @@ decorator==4.0.11 django-celery==3.2.1 django-debug-toolbar==1.7 django-extensions==1.7.8 -django==1.10.6 +django==1.11 django_compress==1.0.1 djangorestframework==3.6.2 docker-py==1.10.6 docopt==0.6.2 docutils==0.13.1 ecdsa==0.13 -elasticsearch==5.2.0 -eventlet==0.20.1 +elasticsearch==5.3.0 +eventlet==0.21.0 extras==1.0.0 fixtures==3.0.0 flake8==3.3.0 -flask==0.12 +flask==0.12.1 funcsigs==1.0.2 futures==3.0.5 gevent==1.2.1 @@ -59,7 +59,7 @@ ipython==5.3.0 iso8601==0.1.11 isodate==0.5.4 itsdangerous==0.24 -jinja2==2.9.5 +jinja2==2.9.6 jsonschema==2.6.0 kombu==4.0.2 linecache2==1.0.0 @@ -83,7 +83,7 @@ numpy==1.12.1 oauth2==1.9.0.post1 oauth2client==4.0.0 oauthlib==2.0.2 -oslo.config==3.23.0 +oslo.config==3.24.0 pandas==0.19.2 passlib==1.7.1 paste==2.0.3 @@ -91,7 +91,7 @@ pastedeploy==1.5.2 pastescript==2.0.2 pep8==1.7.0 pexpect==4.2.1 -pillow==4.0.0 +pillow==4.1.0 pip==9.0.1 prettytable protobuf==3.2.0 @@ -109,7 +109,7 @@ pyjwt==1.4.2 pylibmc==1.5.2 pylint==1.6.5 pymongo==3.4.0 -pymysql==0.7.10 +pymysql==0.7.11 pyopenssl==16.2.0 pyparsing==2.2.0 pyramid==1.8.3 @@ -121,7 +121,7 @@ python-gflags==3.1.1 python-keystoneclient==3.10.0 python-memcached==1.58 python-mimeparse==1.6.0 -python-novaclient==7.1.0 +python-novaclient==8.0.0 python-subunit==1.2.0 python-swiftclient==3.3.0 pytz==2017.2 @@ -136,22 +136,22 @@ retrying==1.3.3 rsa==3.4.2 scipy==0.19.0 setuptools-git==1.2 -sh==1.12.11 +sh==1.12.13 simplejson==3.10.0 six==1.10.0 snowballstemmer==1.2.1 south==1.0.2 -sphinx==1.5.3 +sphinx==1.5.5 sqlalchemy-migrate==0.11.0 -sqlalchemy==1.1.7 +sqlalchemy==1.1.9 sqlparse==0.2.3 statsd==3.2.1 stevedore==1.21.0 testrepository==0.0.20 testtools==2.2.0 thrift==0.10.0 -tornado==4.4.2 -tox==2.6.0 +tornado==4.4.3 +tox==2.7.0 twisted==17.1.0 ujson==1.35 unidecode==0.4.20 @@ -167,5 +167,5 @@ webtest==2.0.27 werkzeug==0.12.1 wheel==0.29.0 xlrd==1.0.0 -zc.buildout==2.9.2 +zc.buildout==2.9.3 zope.interface==4.3.3 From 25de7309a6c959c2894411a8fff535a08c19e993 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 7 Apr 2017 15:15:47 -0700 Subject: [PATCH 040/256] Harmonize Python build flags with Debian. Also check sha256 of Python tarball when downloading. --- python-interpreter-builder/.dockerignore | 1 - python-interpreter-builder/Dockerfile.in | 33 ++++- python-interpreter-builder/README.md | 9 +- .../scripts/build-python-3.5.sh | 124 +++++++++++++++++- tests/python3-libraries/requirements.txt | 37 +++++- 5 files changed, 188 insertions(+), 16 deletions(-) diff --git a/python-interpreter-builder/.dockerignore b/python-interpreter-builder/.dockerignore index 53752db2..e69de29b 100644 --- a/python-interpreter-builder/.dockerignore +++ b/python-interpreter-builder/.dockerignore @@ -1 +0,0 @@ -output diff --git a/python-interpreter-builder/Dockerfile.in b/python-interpreter-builder/Dockerfile.in index 54181518..c3b7dada 100644 --- a/python-interpreter-builder/Dockerfile.in +++ b/python-interpreter-builder/Dockerfile.in @@ -2,20 +2,43 @@ # installed. FROM ${DEBIAN_BASE_IMAGE} -# Install Python build dependencies +# Install Python build dependencies (based on Debian Build-Depends) RUN apt-get update && apt-get install -yq \ - build-essential \ - wget \ - pkg-config \ + autoconf \ + blt-dev \ + bzip2 \ + debhelper \ + dpkg-dev \ + gcc \ + libbluetooth-dev \ libbz2-dev \ + libdb-dev \ + libexpat1-dev \ + libffi-dev \ libgdbm-dev \ + libgpm2 \ liblzma-dev \ - libncurses5-dev \ + libmpdec-dev \ + libncursesw5-dev \ libreadline-dev \ libsqlite3-dev \ libssl-dev \ + locales \ + lsb-release \ + mime-support \ + net-tools \ + netbase \ + python3 \ + quilt \ + sharutils \ + time \ + tk-dev \ + wget \ + xauth \ + xvfb \ zlib1g-dev + # Setup locale. This prevents Python 3 IO encoding issues. ENV LANG C.UTF-8 diff --git a/python-interpreter-builder/README.md b/python-interpreter-builder/README.md index 3c1fc588..7b718581 100644 --- a/python-interpreter-builder/README.md +++ b/python-interpreter-builder/README.md @@ -8,12 +8,13 @@ dependencies in the final container. ## Building -Use make: +Use: - make build + docker build --tag=google/python/interpreter-builder . -The interpreters will be outputted to `output/interpreters.tar.gz`, this is -suitable to be added directly to a Docker container: +The interpreters will be stored in the image at `/interpreters.tar.gz`. This is +suitable to be extracted from this image and added directly to another Docker +image via: ADD interpreters.tar.gz / diff --git a/python-interpreter-builder/scripts/build-python-3.5.sh b/python-interpreter-builder/scripts/build-python-3.5.sh index 263312af..b5e49ac8 100755 --- a/python-interpreter-builder/scripts/build-python-3.5.sh +++ b/python-interpreter-builder/scripts/build-python-3.5.sh @@ -1,21 +1,135 @@ #!/bin/bash -set -e +set -euo pipefail # Get the source mkdir -p /opt/sources cd /opt/sources -wget -nv https://www.python.org/ftp/python/3.5.2/Python-3.5.2.tgz +wget --no-verbose https://www.python.org/ftp/python/3.5.2/Python-3.5.2.tgz +shasum --check < Date: Wed, 12 Apr 2017 09:08:14 +0000 Subject: [PATCH 041/256] Auto-update dependencies. --- tests/python2-libraries/requirements.txt | 16 ++++++++-------- tests/python3-libraries/requirements.txt | 6 +++--- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 48d0bf43..1f635b65 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -6,7 +6,7 @@ anyjson==0.3.3 apache-libcloud==1.5.0 argparse==1.4.0 astroid==1.4.9 -awscli==1.11.75 +awscli==1.11.76 babel==2.4.0 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 @@ -16,13 +16,13 @@ billiard==3.5.0.2 blessings==1.6 blinker==1.4 boto==2.46.1 -botocore==1.5.38 +botocore==1.5.39 bottle==0.12.13 -carbon==0.9.15 +carbon==1.0.0 celery==4.0.2 certifi==2017.1.23 cffi==1.10.0 -chardet==2.3.0 +chardet==3.0.1 click==6.7 cliff==2.5.0 cmd2==0.7.0 @@ -58,7 +58,7 @@ functools32==3.2.3.post2 futures==3.0.5 gevent==1.2.1 google-api-python-client==1.6.2 -graphite-web==0.9.15 +graphite-web==1.0.0 greenlet==0.4.12 gunicorn==19.7.1 hiredis==0.2.0 @@ -114,7 +114,7 @@ passlib==1.7.1 paste==2.0.3 pastedeploy==1.5.2 pastescript==2.0.2 -pbr==2.0.0 +pbr==2.1.0 pep8==1.7.0 pexpect==4.2.1 pika==0.10.0 @@ -122,7 +122,7 @@ pillow==4.1.0 pip==9.0.1 prettytable protobuf==3.2.0 -psutil==5.2.1 +psutil==5.2.2 psycopg2==2.7.1 py==1.4.33 pyasn1-modules==0.0.8 @@ -166,7 +166,7 @@ rsa==3.4.2 scipy==0.19.0 selenium==3.3.3 setuptools-git==1.2 -setuptools==34.4.0 +setuptools==34.4.1 sh==1.12.13 simplejson==3.10.0 six==1.10.0 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 41b23682..0cdacb82 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -12,11 +12,11 @@ billiard==3.5.0.2 blessings==1.6 blinker==1.4 bottle==0.12.13 -carbon==0.9.15 +carbon==1.0.0 celery==4.0.2 certifi==2017.1.23 cffi==1.10.0 -chardet==2.3.0 +chardet==3.0.1 click==6.7 cliff==2.5.0 cmd2==0.7.0 @@ -95,7 +95,7 @@ pillow==4.1.0 pip==9.0.1 prettytable protobuf==3.2.0 -psutil==5.2.1 +psutil==5.2.2 psycopg2==2.7.1 py==1.4.33 pyasn1-modules==0.0.8 From a07d390cf68597768398e338c81426c63ac5f6c5 Mon Sep 17 00:00:00 2001 From: liyanhui1228 Date: Thu, 13 Apr 2017 14:56:46 -0700 Subject: [PATCH 042/256] Update the benchmark code to use the new Python benchmark suite (#105) --- tests/benchmark/Dockerfile.in | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/tests/benchmark/Dockerfile.in b/tests/benchmark/Dockerfile.in index e39eac1c..fcdaa5f9 100644 --- a/tests/benchmark/Dockerfile.in +++ b/tests/benchmark/Dockerfile.in @@ -1,6 +1,19 @@ FROM ${FULL_BASE_IMAGE} -RUN hg clone -r 9923b81a1d34 https://hg.python.org/benchmarks /app/benchmarks -WORKDIR /app/benchmarks -RUN python perf.py -r -b default /usr/bin/python2.7 /usr/bin/python3.4 -RUN python perf.py -r -b default /usr/bin/python3.4 /opt/python3.5/bin/python3.5 +# Install performance +RUN pip install performance + +# Create virtual environment +RUN pip install --upgrade virtualenv + +# Download ensurepip module which prevents the failure when benchmarking python3, can be removed once we drop debian's 3.4 +RUN wget https://www.python.org/ftp/python/3.4.2/Python-3.4.2.tgz +RUN tar xzf Python-3.4.2.tgz +RUN cp -R Python-3.4.2/Lib/ensurepip /usr/lib/python3.4 + +# Run the benchmark and compare the performance, add the --debug-single-value flag to let the benchmark run in fastest mode +RUN pyperformance run --debug-single-value --python=python2.7 -o py2.7.json +RUN pyperformance run --debug-single-value --python=python3.4 -o py3.4.json +RUN pyperformance run --debug-single-value --python=python3.5 -o py3.5.json +RUN pyperformance compare py2.7.json py3.4.json --output_style table +RUN pyperformance compare py3.4.json py3.5.json --output_style table From 94999df2ea7008b9a90490720ab3e379c5fabb60 Mon Sep 17 00:00:00 2001 From: Angela Li Date: Tue, 18 Apr 2017 15:24:07 -0700 Subject: [PATCH 043/256] Benchmark the same interpreter from release to release. --- .../benchmark_between_releases/Dockerfile.in | 24 +++++++++++ .../benchmark_between_releases.sh | 40 +++++++++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 tests/benchmark/benchmark_between_releases/Dockerfile.in create mode 100755 tests/benchmark/benchmark_between_releases/benchmark_between_releases.sh diff --git a/tests/benchmark/benchmark_between_releases/Dockerfile.in b/tests/benchmark/benchmark_between_releases/Dockerfile.in new file mode 100644 index 00000000..0bf1f65a --- /dev/null +++ b/tests/benchmark/benchmark_between_releases/Dockerfile.in @@ -0,0 +1,24 @@ +FROM ${FULL_BASE_IMAGE} + +# Install performance +RUN pip install performance + +# Create virtual environment +RUN pip install --upgrade virtualenv + +# Download ensurepip module which prevents the failure when benchmarking python3, can be removed once we drop debian's 3.4 +RUN wget https://www.python.org/ftp/python/3.4.2/Python-3.4.2.tgz +RUN tar xzf Python-3.4.2.tgz +RUN cp -R Python-3.4.2/Lib/ensurepip /usr/lib/python3.4 + +RUN mkdir /${TAG} + +# Run the benchmark and compare the performance, add the --debug-single-value flag to let the benchmark run in fastest mode +RUN pyperformance run --debug-single-value --python=python2.7 -o /${TAG}/py2.7.json +RUN pyperformance run --debug-single-value --python=python3.4 -o /${TAG}/py3.4.json +RUN pyperformance run --debug-single-value --python=python3.5 -o /${TAG}/py3.5.json +RUN pyperformance compare /${TAG}/py2.7.json /${TAG}/py3.4.json --output_style table +RUN pyperformance compare /${TAG}/py3.4.json /${TAG}/py3.5.json --output_style table + +# Initialize mount volume +VOLUME /${TAG} diff --git a/tests/benchmark/benchmark_between_releases/benchmark_between_releases.sh b/tests/benchmark/benchmark_between_releases/benchmark_between_releases.sh new file mode 100755 index 00000000..1b4ac7fc --- /dev/null +++ b/tests/benchmark/benchmark_between_releases/benchmark_between_releases.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +# Build the benchmark image for release 1 from Dockerfile +echo "Building image for release 1" +export FULL_BASE_IMAGE="${DOCKER_NAMESPACE}/python:${TAG1}" +export TAG="${TAG1}" +envsubst <"Dockerfile".in >"Dockerfile" '$FULL_BASE_IMAGE $TAG' +docker build -t benchmark_1 . +rm Dockerfile + +# Build the benchmark image for release 2 from Dockerfile +echo "Building image for release 2" +export FULL_BASE_IMAGE="${DOCKER_NAMESPACE}/python:${TAG2}" +export TAG="${TAG2}" +envsubst <"Dockerfile".in >"Dockerfile" '$FULL_BASE_IMAGE $TAG' +docker build -t benchmark_2 . +rm Dockerfile + +echo "Successfully built images" + +# Start running the containers +docker run -it --name benchmark_1 -h CONTAINER1 -v /"${TAG1}" benchmark_1 ls +docker run -it --name benchmark_2 -h CONTAINER2 -v /"${TAG2}" benchmark_2 ls + +# Create folders to hold the files +mkdir release1 +mkdir release2 + +# Copy the benchmark result for python versions from container to host +docker cp benchmark_1:/"${TAG1}"/ release1/ +docker cp benchmark_2:/"${TAG2}"/ release2/ + +echo "Start benchmarking the python interpreter performance between the two releases" + +# Compare the performance between the interpreter in different release +pyperformance compare release1/"${TAG1}"/py2.7.json release2/"${TAG2}"/py2.7.json --output_style table > py2.7_res +pyperformance compare release1/"${TAG1}"/py3.4.json release2/"${TAG2}"/py3.4.json --output_style table > py3.4_res +pyperformance compare release1/"${TAG1}"/py3.5.json release2/"${TAG2}"/py3.5.json --output_style table > py3.5_res + +echo "Completed" \ No newline at end of file From 009fb07c4dcf449293cf502ecae0dd3a14809776 Mon Sep 17 00:00:00 2001 From: Angela Li Date: Tue, 18 Apr 2017 15:24:07 -0700 Subject: [PATCH 044/256] Benchmark the same interpreter from release to release. --- RELEASING.md | 10 +++++- tests/benchmark/Dockerfile.in | 12 ++++--- tests/benchmark/benchmark_between_releases.sh | 34 +++++++++++++++++++ 3 files changed, 50 insertions(+), 6 deletions(-) create mode 100755 tests/benchmark/benchmark_between_releases.sh diff --git a/RELEASING.md b/RELEASING.md index 1f92084a..f7d5a459 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -72,8 +72,16 @@ docker run -it --entrypoint /bin/bash YOUR-IMAGE-NAME There is a benchmark suite which compares the performance of interpreters against each other. +**Benchmark different versions of interpreter in the same release + +``` shell +DOCKER_NAMESPACE=DOCKER_NAMESPACE_EXAMPLE TAG=TAG_EXAMPLE ./build.sh --nobuild --benchmark +``` + +**Benchmark same versions of interpreter from release to release + ``` shell -./build.sh --nobuild --benchmark +DOCKER_NAMESPACE=DOCKER_NAMESPACE_EXAMPLE TAG1=TAG1_EXAMPLE TAG2=TAG2_EXAMPLE ./benchmark_between_releases.sh ``` Since these benchmarks are run on cloud instances, the timings may vary from run diff --git a/tests/benchmark/Dockerfile.in b/tests/benchmark/Dockerfile.in index fcdaa5f9..9a379a99 100644 --- a/tests/benchmark/Dockerfile.in +++ b/tests/benchmark/Dockerfile.in @@ -11,9 +11,11 @@ RUN wget https://www.python.org/ftp/python/3.4.2/Python-3.4.2.tgz RUN tar xzf Python-3.4.2.tgz RUN cp -R Python-3.4.2/Lib/ensurepip /usr/lib/python3.4 +RUN mkdir /result + # Run the benchmark and compare the performance, add the --debug-single-value flag to let the benchmark run in fastest mode -RUN pyperformance run --debug-single-value --python=python2.7 -o py2.7.json -RUN pyperformance run --debug-single-value --python=python3.4 -o py3.4.json -RUN pyperformance run --debug-single-value --python=python3.5 -o py3.5.json -RUN pyperformance compare py2.7.json py3.4.json --output_style table -RUN pyperformance compare py3.4.json py3.5.json --output_style table +RUN pyperformance run --debug-single-value --python=python2.7 -o /result/py2.7.json +RUN pyperformance run --debug-single-value --python=python3.4 -o /result/py3.4.json +RUN pyperformance run --debug-single-value --python=python3.5 -o /result/py3.5.json +RUN pyperformance compare /result/py2.7.json /result/py3.4.json --output_style table +RUN pyperformance compare /result/py3.4.json /result/py3.5.json --output_style table diff --git a/tests/benchmark/benchmark_between_releases.sh b/tests/benchmark/benchmark_between_releases.sh new file mode 100755 index 00000000..ea3d5ffe --- /dev/null +++ b/tests/benchmark/benchmark_between_releases.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# Build the benchmark image for release 1 from Dockerfile +echo "Building image for release 1" +export FULL_BASE_IMAGE="${DOCKER_NAMESPACE}/python:${TAG1}" +envsubst <"Dockerfile".in >"Dockerfile" '$FULL_BASE_IMAGE' +docker build --no-cache -t benchmark_1 . +rm Dockerfile + +# Build the benchmark image for release 2 from Dockerfile +echo "Building image for release 2" +export FULL_BASE_IMAGE="${DOCKER_NAMESPACE}/python:${TAG2}" +envsubst <"Dockerfile".in >"Dockerfile" '$FULL_BASE_IMAGE' +docker build --no-cache -t benchmark_2 . +rm Dockerfile + +echo "Successfully built images" + +# Create folders to hold the files +mkdir release1 +mkdir release2 + +# Start running the containers and copy the benchmark result for python versions from container to host +docker run -it --name benchmark_1 -h CONTAINER1 -v "${PWD}"/release1:/export benchmark_1 /bin/bash -c "cp /result/py*.json /export/" +docker run -it --name benchmark_2 -h CONTAINER2 -v "${PWD}"/release2:/export benchmark_2 /bin/bash -c "cp /result/py*.json /export/" + +echo "Start benchmarking the python interpreter performance between the two releases" + +# Compare the performance between the interpreter in different release +pyperformance compare release1/py2.7.json release2/py2.7.json --output_style table > py2.7_res +pyperformance compare release1/py3.4.json release2/py3.4.json --output_style table > py3.4_res +pyperformance compare release1/py3.5.json release2/py3.5.json --output_style table > py3.5_res + +echo "Completed" From 89bc0b95ff570e500288dc2c5dccc919c69ca5ba Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Thu, 20 Apr 2017 14:46:34 -0700 Subject: [PATCH 045/256] Update from Python 3.5.2 to 3.5.3 --- .../scripts/build-python-3.5.sh | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/python-interpreter-builder/scripts/build-python-3.5.sh b/python-interpreter-builder/scripts/build-python-3.5.sh index b5e49ac8..fa54900b 100755 --- a/python-interpreter-builder/scripts/build-python-3.5.sh +++ b/python-interpreter-builder/scripts/build-python-3.5.sh @@ -5,14 +5,15 @@ set -euo pipefail # Get the source mkdir -p /opt/sources cd /opt/sources -wget --no-verbose https://www.python.org/ftp/python/3.5.2/Python-3.5.2.tgz +wget --no-verbose https://www.python.org/ftp/python/3.5.3/Python-3.5.3.tgz +# SHA-256 generated via `shasum -a 256 [file]` shasum --check < Date: Mon, 24 Apr 2017 20:17:44 -0700 Subject: [PATCH 046/256] Tweak Debian flags based on manually compiling .deb package --- .../scripts/build-python-3.5.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/python-interpreter-builder/scripts/build-python-3.5.sh b/python-interpreter-builder/scripts/build-python-3.5.sh index fa54900b..f05bf58b 100755 --- a/python-interpreter-builder/scripts/build-python-3.5.sh +++ b/python-interpreter-builder/scripts/build-python-3.5.sh @@ -67,8 +67,6 @@ cd Python-3.5.3 # (Debian) Security hardening # CFLAGS=-g # (Debian) More debug info -# CFLAGS=-O2 -# (Debian) the default is -O3, don't know why the difference # CFLAGS=-Wformat -Werror=format-security # (Debian) Security hardening # CPPFLAGS=-D_FORTIFY_SOURCE=2 @@ -113,7 +111,6 @@ cd build-static CFLAGS="\ -fstack-protector-strong \ -g \ - -O2 \ -Wformat -Werror=format-security \ " \ CPPFLAGS="\ @@ -125,13 +122,16 @@ cd build-static RANLIB="x86_64-linux-gnu-gcc-ranlib" \ # Explicitly build the profile-guided-optimized interpreter -EXTRA_OPT_CFLAGS="-g -flto -fuse-linker-plugin -ffat-lto-objects" -NUM_JOBS=$(nproc) -make -j"$NUM_JOBS" EXTRA_CFLAGS="${EXTRA_OPT_CFLAGS}" profile-opt -make -j"$NUM_JOBS" EXTRA_CFLAGS="${EXTRA_OPT_CFLAGS}" test +NUM_JOBS="$(nproc)" +make \ + -j"${NUM_JOBS}" \ + EXTRA_CFLAGS="-g -flto -fuse-linker-plugin -ffat-lto-objects" \ + PROFILE_TASK='../Lib/test/regrtest.py -s -j 1 -unone,decimal -x test_cmd_line_script test_compiler test_concurrent_futures test_ctypes test_dbm_dumb test_dbm_ndbm test_distutils test_ensurepip test_gdb test_ioctl test_linuxaudiodev test_multiprocessing test_multiprocessing_fork test_multiprocessing_forkserver test_multiprocessing_main_handling test_multiprocessing_spawn test_ossaudiodev test_pydoc test_signal test_socket test_socketserver test_subprocess test_sundry test_thread test_threaded_import test_threadedtempfile test_threading test_threading_local test_threadsignals test_venv test_zipimport_support' \ + profile-opt + make altinstall # Clean-up sources -cd / +cd /opt rm /opt/sources/Python-3.5.3.tgz rm -r /opt/sources/Python-3.5.3 From d0c20d1c3ba3f7c97013a036f99a1354529b874d Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Mon, 24 Apr 2017 20:30:55 -0700 Subject: [PATCH 047/256] Build Python 3.6.1 interpreter --- .../scripts/build-python-3.6.sh | 145 ++++++++++++++++++ 1 file changed, 145 insertions(+) create mode 100644 python-interpreter-builder/scripts/build-python-3.6.sh diff --git a/python-interpreter-builder/scripts/build-python-3.6.sh b/python-interpreter-builder/scripts/build-python-3.6.sh new file mode 100644 index 00000000..c15aa090 --- /dev/null +++ b/python-interpreter-builder/scripts/build-python-3.6.sh @@ -0,0 +1,145 @@ +#!/bin/bash + +set -euo pipefail + +# Get the source +mkdir -p /opt/sources +cd /opt/sources +wget --no-verbose https://www.python.org/ftp/python/3.6.1/Python-3.6.1.tgz +# SHA-256 generated via `shasum -a 256 [file]` +shasum --check < Date: Mon, 1 May 2017 13:20:39 -0700 Subject: [PATCH 048/256] Further compile flag tweaks. Turn off LTO for now. --- python-interpreter-builder/Dockerfile.in | 5 +- .../scripts/build-python-3.5.sh | 43 +++++++-------- .../scripts/build-python-3.6.sh | 53 +++++++------------ tests/python3-libraries/requirements.txt | 4 +- 4 files changed, 44 insertions(+), 61 deletions(-) mode change 100644 => 100755 python-interpreter-builder/scripts/build-python-3.6.sh diff --git a/python-interpreter-builder/Dockerfile.in b/python-interpreter-builder/Dockerfile.in index c3b7dada..9323f750 100644 --- a/python-interpreter-builder/Dockerfile.in +++ b/python-interpreter-builder/Dockerfile.in @@ -45,9 +45,10 @@ ENV LANG C.UTF-8 # Add build scripts ADD scripts /scripts -# Build the Python 3.5 interpreter +# Build the Python interpreters RUN /scripts/build-python-3.5.sh +RUN /scripts/build-python-3.6.sh # Tar the interpreters. Tarring is needed because docker cp doesn't handle # links correctly. -RUN tar czf /interpreters.tar.gz /opt +RUN tar czf /interpreters.tar.gz /opt/python?.? diff --git a/python-interpreter-builder/scripts/build-python-3.5.sh b/python-interpreter-builder/scripts/build-python-3.5.sh index f05bf58b..873b7257 100755 --- a/python-interpreter-builder/scripts/build-python-3.5.sh +++ b/python-interpreter-builder/scripts/build-python-3.5.sh @@ -1,6 +1,7 @@ #!/bin/bash set -euo pipefail +set -x # Get the source mkdir -p /opt/sources @@ -26,12 +27,6 @@ cd Python-3.5.3 # (https://bugs.python.org/issue27685) # --without-ensurepip # Debian unbundles pip for their own reasons -# --with-system-expat -# (Debian) for compatibility with other Debian packages -# --with-system-ffi -# (Debian) for compatibility with other Debian packages -# --with-system-libmpdec -# (Debian) for compatibility with other Debian packages # CFLAGS=-fdebug-prefix-map # Unnecessary in our build environment # @@ -59,6 +54,12 @@ cd Python-3.5.3 # would prefer one over the other. # --with-fpectl # (Debian) Floating point exception control +# --with-system-expat +# (Debian) for compatibility with other Debian packages +# --with-system-ffi +# (Debian) for compatibility with other Debian packages +# --with-system-libmpdec +# (Debian) for compatibility with other Debian packages # AR= # (Debian) No-op # CC= @@ -83,29 +84,26 @@ cd Python-3.5.3 # # LTO (Link time optimization) # -# There is a --with-lto flag, but Debian doesn't use it. We used to -# use it, but it caused trouble with the uWGSI module. Instead, we -# pass lto related flags in EXTRA_CFLAGS (to make, rather than -# configure), as Debian does. -# -# -# Debugging: It is very helpful to view and diff sysconfig data from two -# python interpreters. For example: -# docker run -it --entrypoint=/opt/python3.5/bin/python3.5 google/python/interpreter-builder -c 'import sysconfig;print("\n".join("%s:%s"%(key,value) for key,value in sorted(sysconfig.get_config_vars().items())))' +# Currently disabled, due to unresolved compile problems. There is a +# --with-lto flag, but Debian doesn't use it. Instead, they pass lto +# related flags in EXTRA_CFLAGS (to make, rather than configure). +# Specifically EXTRA_CFLAGS="-g -flto -fuse-linker-plugin +# -ffat-lto-objects" mkdir build-static cd build-static ../configure \ - --build=x86_64-pc-linux-gnu \ --enable-ipv6 \ --enable-loadable-sqlite-extensions \ --enable-optimizations \ - --host=x86_64-pc-linux-gnu \ --prefix=/opt/python3.5 \ --with-dbmliborder=bdb:gdbm \ --with-computed-gotos \ --with-fpectl \ + --with-system-expat \ + --with-system-ffi \ + --with-system-libmpdec \ AR="x86_64-linux-gnu-gcc-ar" \ CC="x86_64-linux-gnu-gcc" \ CFLAGS="\ @@ -121,14 +119,11 @@ cd build-static LDFLAGS="-Wl,-z,relro" \ RANLIB="x86_64-linux-gnu-gcc-ranlib" \ -# Explicitly build the profile-guided-optimized interpreter -NUM_JOBS="$(nproc)" -make \ - -j"${NUM_JOBS}" \ - EXTRA_CFLAGS="-g -flto -fuse-linker-plugin -ffat-lto-objects" \ - PROFILE_TASK='../Lib/test/regrtest.py -s -j 1 -unone,decimal -x test_cmd_line_script test_compiler test_concurrent_futures test_ctypes test_dbm_dumb test_dbm_ndbm test_distutils test_ensurepip test_gdb test_ioctl test_linuxaudiodev test_multiprocessing test_multiprocessing_fork test_multiprocessing_forkserver test_multiprocessing_main_handling test_multiprocessing_spawn test_ossaudiodev test_pydoc test_signal test_socket test_socketserver test_subprocess test_sundry test_thread test_threaded_import test_threadedtempfile test_threading test_threading_local test_threadsignals test_venv test_zipimport_support' \ - profile-opt +# Due to https://bugs.python.org/issue29243, "make altinstall" +# rebuilds everything from scratch, twice. This is a workaround. +sed -i 's/^all:.*$/all: build_all/' Makefile +make profile-opt make altinstall # Clean-up sources diff --git a/python-interpreter-builder/scripts/build-python-3.6.sh b/python-interpreter-builder/scripts/build-python-3.6.sh old mode 100644 new mode 100755 index c15aa090..af4fc0b4 --- a/python-interpreter-builder/scripts/build-python-3.6.sh +++ b/python-interpreter-builder/scripts/build-python-3.6.sh @@ -1,6 +1,7 @@ #!/bin/bash set -euo pipefail +set -x # Get the source mkdir -p /opt/sources @@ -26,12 +27,6 @@ cd Python-3.6.1 # (https://bugs.python.org/issue27685) # --without-ensurepip # Debian unbundles pip for their own reasons -# --with-system-expat -# (Debian) for compatibility with other Debian packages -# --with-system-ffi -# (Debian) for compatibility with other Debian packages -# --with-system-libmpdec -# (Debian) for compatibility with other Debian packages # CFLAGS=-fdebug-prefix-map # Unnecessary in our build environment # @@ -59,6 +54,12 @@ cd Python-3.6.1 # would prefer one over the other. # --with-fpectl # (Debian) Floating point exception control +# --with-system-expat +# (Debian) for compatibility with other Debian packages +# --with-system-ffi +# (Debian) for compatibility with other Debian packages +# --with-system-libmpdec +# (Debian) for compatibility with other Debian packages # AR= # (Debian) No-op # CC= @@ -67,8 +68,6 @@ cd Python-3.6.1 # (Debian) Security hardening # CFLAGS=-g # (Debian) More debug info -# CFLAGS=-specs=/usr/share/dpkg/no-pie-link.specs -# (Debian) Temporarily disable security hardening # CFLAGS=-Wformat -Werror=format-security # (Debian) Security hardening # CPPFLAGS=-D_FORTIFY_SOURCE=2 @@ -77,8 +76,6 @@ cd Python-3.6.1 # (Debian) Warnings about non-reproducible builds # CXX= # (Debian) No-op -# LDFLAGS=-specs=/usr/share/dpkg/no-pie-link.specs -# (Debian) Temporarily disable security hardening # LDFLAGS=-Wl,-z,relro: # (Debian) Security hardening # RANLIB= @@ -87,35 +84,31 @@ cd Python-3.6.1 # # LTO (Link time optimization) # -# There is a --with-lto flag, but Debian doesn't use it. We used to -# use it, but it caused trouble with the uWGSI module. Instead, we -# pass lto related flags in EXTRA_CFLAGS (to make, rather than -# configure), as Debian does. -# -# -# Debugging: It is very helpful to view and diff sysconfig data from two -# python interpreters. For example: -# docker run -it --entrypoint=/opt/python3.6/bin/python3.6 google/python/interpreter-builder -c 'import sysconfig;print("\n".join("%s:%s"%(key,value) for key,value in sorted(sysconfig.get_config_vars().items())))' +# Currently disabled, due to unresolved compile problems. There is a +# --with-lto flag, but Debian doesn't use it. Instead, they pass lto +# related flags in EXTRA_CFLAGS (to make, rather than configure). +# Specifically EXTRA_CFLAGS="-g -flto -fuse-linker-plugin +# -ffat-lto-objects" mkdir build-static cd build-static ../configure \ - --build=x86_64-pc-linux-gnu \ --enable-ipv6 \ --enable-loadable-sqlite-extensions \ --enable-optimizations \ - --host=x86_64-pc-linux-gnu \ --prefix=/opt/python3.6 \ --with-dbmliborder=bdb:gdbm \ --with-computed-gotos \ --with-fpectl \ + --with-system-expat \ + --with-system-ffi \ + --with-system-libmpdec \ AR="x86_64-linux-gnu-gcc-ar" \ CC="x86_64-linux-gnu-gcc" \ CFLAGS="\ -fstack-protector-strong \ -g \ - -specs=/usr/share/dpkg/no-pie-compile.specs \ -Wformat -Werror=format-security \ " \ CPPFLAGS="\ @@ -123,20 +116,14 @@ cd build-static -Wdate-time \ " \ CXX="x86_64-linux-gnu-g++" \ - LDFLAGS="\ - -specs=/usr/share/dpkg/no-pie-link.specs \ - -Wl,-z,relro \ - " \ + LDFLAGS="-Wl,-z,relro" \ RANLIB="x86_64-linux-gnu-gcc-ranlib" \ -# Explicitly build the profile-guided-optimized interpreter -NUM_JOBS="$(nproc)" -make \ - -j"${NUM_JOBS}" \ - EXTRA_CFLAGS="" \ - PROFILE_TASK="../Lib/test/regrtest.py -s -j 1 -unone,decimal -x test_cmd_line_script test_compiler test_concurrent_futures test_ctypes test_dbm_dumb test_dbm_ndbm test_distutils test_ensurepip test_gdb test_ioctl test_linuxaudiodev test_multiprocessing test_ossaudiodev test_pydoc test_signal test_socket test_socketserver test_subprocess test_sundry test_thread test_threaded_import test_threadedtempfile test_threading test_threading_local test_threadsignals test_venv test_zipimport_support" \ - profile-opt +# Due to https://bugs.python.org/issue29243, "make altinstall" +# rebuilds everything from scratch, twice. This is a workaround. +sed -i 's/^all:.*$/all: build_all/' Makefile +make profile-opt make altinstall # Clean-up sources diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 784f5751..74546872 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -143,7 +143,7 @@ pyramid==1.8.3 pystache==0.5.4 pytest-cov==2.4.0 pytest==3.0.7 -python-cjson==1.2.1 +#python-cjson python-daemon==2.1.2 python-dateutil==2.6.0 python-gflags==3.1.1 @@ -191,7 +191,7 @@ unidecode==0.4.20 unittest2==1.1.0 uritemplate==3.0.0 urllib3==1.20 -uwsgi==2.0.15 +#uwsgi versiontools==1.9.1 virtualenv==15.1.0 waitress==1.0.2 From b6c4dc4dc9f5e1ef287a9d6ea21dadb03aa8f3c6 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Mon, 1 May 2017 14:44:12 -0700 Subject: [PATCH 049/256] Update expected version in structure test. --- tests/virtualenv/virtualenv_python35.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/virtualenv/virtualenv_python35.yaml b/tests/virtualenv/virtualenv_python35.yaml index 23dc1367..b37d1461 100644 --- a/tests/virtualenv/virtualenv_python35.yaml +++ b/tests/virtualenv/virtualenv_python35.yaml @@ -29,7 +29,7 @@ commandTests: - name: "python version" command: ["python", "--version"] - expectedOutput: ["Python 3.5.2\n"] + expectedOutput: ["Python 3.5.3\n"] - name: "pip installation" command: ["which", "pip"] From 3499ecd55bf699ca5389abcd0335950dd3d8466f Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Mon, 1 May 2017 16:46:23 -0700 Subject: [PATCH 050/256] Disable 'carbon' module which fails under Python 3 --- tests/python3-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 7279102f..6f971d8d 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -18,7 +18,7 @@ blinker==1.4 boto==2.46.1 botocore==1.5.31 bottle==0.12.13 -carbon==1.0.0 +#carbon celery==4.0.2 certifi==2017.1.23 cffi==1.10.0 From efd52e351eecbee5fc07d5433809c90e9031b8bc Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Thu, 4 May 2017 14:38:16 -0700 Subject: [PATCH 051/256] Add explanation of --enable-ipv6 flag --- python-interpreter-builder/scripts/build-python-3.5.sh | 6 +----- python-interpreter-builder/scripts/build-python-3.6.sh | 6 +----- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/python-interpreter-builder/scripts/build-python-3.5.sh b/python-interpreter-builder/scripts/build-python-3.5.sh index 873b7257..a80473d8 100755 --- a/python-interpreter-builder/scripts/build-python-3.5.sh +++ b/python-interpreter-builder/scripts/build-python-3.5.sh @@ -34,17 +34,13 @@ cd Python-3.5.3 # Flags that we _do_ use: # (Debian) means it was taken from Debian build rules. # -# --build -# (Debian) # --enable-ipv6 -# (Debian) +# (Debian) Ensure support is compiled in instead of relying on autodetection # --enable-loadable-sqlite-extensions # (Debian) # --enable-optimizations # Performance optimization (Enables PGO and may or may not enable # LTO based on complex logic and bugs) -# --host -# (Debian) # --prefix # Avoid possible collisions with Debian or others # --with-computed-gotos diff --git a/python-interpreter-builder/scripts/build-python-3.6.sh b/python-interpreter-builder/scripts/build-python-3.6.sh index af4fc0b4..cb4fa360 100755 --- a/python-interpreter-builder/scripts/build-python-3.6.sh +++ b/python-interpreter-builder/scripts/build-python-3.6.sh @@ -34,17 +34,13 @@ cd Python-3.6.1 # Flags that we _do_ use: # (Debian) means it was taken from Debian build rules. # -# --build -# (Debian) # --enable-ipv6 -# (Debian) +# (Debian) Ensure support is compiled in instead of relying on autodetection # --enable-loadable-sqlite-extensions # (Debian) # --enable-optimizations # Performance optimization (Enables PGO and may or may not enable # LTO based on complex logic and bugs) -# --host -# (Debian) # --prefix # Avoid possible collisions with Debian or others # --with-computed-gotos From 069c598f4763dc6b691019c955c7b96b945ee527 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Thu, 4 May 2017 15:20:50 -0700 Subject: [PATCH 052/256] Enable Python 3.6 in the base runtime image. --- runtime-image/Dockerfile.in | 2 +- tests/benchmark/Dockerfile.in | 13 +++-- tests/benchmark/benchmark_between_releases.sh | 1 + .../run_system_tests.sh | 7 ++- tests/google-cloud-python/run_unit_tests.sh | 9 +++- .../python3-libraries/python3-libraries.yaml | 14 ++++-- tests/virtualenv/virtualenv_python36.yaml | 48 +++++++++++++++++++ 7 files changed, 82 insertions(+), 12 deletions(-) create mode 100644 tests/virtualenv/virtualenv_python36.yaml diff --git a/runtime-image/Dockerfile.in b/runtime-image/Dockerfile.in index 15cf3ee4..ed194257 100644 --- a/runtime-image/Dockerfile.in +++ b/runtime-image/Dockerfile.in @@ -24,7 +24,7 @@ RUN pip install --upgrade pip virtualenv ADD interpreters.tar.gz / # Add Google-built interpreters to the path -ENV PATH /opt/python3.5/bin:$PATH +ENV PATH /opt/python3.5/bin:/opt/python3.6/bin:$PATH # Setup the app working directory RUN ln -s /home/vmagent/app /app diff --git a/tests/benchmark/Dockerfile.in b/tests/benchmark/Dockerfile.in index 9a379a99..69f41862 100644 --- a/tests/benchmark/Dockerfile.in +++ b/tests/benchmark/Dockerfile.in @@ -6,16 +6,19 @@ RUN pip install performance # Create virtual environment RUN pip install --upgrade virtualenv -# Download ensurepip module which prevents the failure when benchmarking python3, can be removed once we drop debian's 3.4 -RUN wget https://www.python.org/ftp/python/3.4.2/Python-3.4.2.tgz -RUN tar xzf Python-3.4.2.tgz -RUN cp -R Python-3.4.2/Lib/ensurepip /usr/lib/python3.4 +# Required for Python 3.4, see +# https://bugs.launchpad.net/ubuntu/+source/python3.4/+bug/1290847 +RUN apt-get update && apt-get install -y --force-yes python3-pip python3-venv RUN mkdir /result -# Run the benchmark and compare the performance, add the --debug-single-value flag to let the benchmark run in fastest mode +# Run the benchmark and compare the performance, add the +# --debug-single-value flag to let the benchmark run in fastest mode RUN pyperformance run --debug-single-value --python=python2.7 -o /result/py2.7.json RUN pyperformance run --debug-single-value --python=python3.4 -o /result/py3.4.json RUN pyperformance run --debug-single-value --python=python3.5 -o /result/py3.5.json +RUN pyperformance run --debug-single-value --python=python3.6 -o /result/py3.6.json + RUN pyperformance compare /result/py2.7.json /result/py3.4.json --output_style table RUN pyperformance compare /result/py3.4.json /result/py3.5.json --output_style table +RUN pyperformance compare /result/py3.5.json /result/py3.6.json --output_style table diff --git a/tests/benchmark/benchmark_between_releases.sh b/tests/benchmark/benchmark_between_releases.sh index ea3d5ffe..c540ae7f 100755 --- a/tests/benchmark/benchmark_between_releases.sh +++ b/tests/benchmark/benchmark_between_releases.sh @@ -30,5 +30,6 @@ echo "Start benchmarking the python interpreter performance between the two rele pyperformance compare release1/py2.7.json release2/py2.7.json --output_style table > py2.7_res pyperformance compare release1/py3.4.json release2/py3.4.json --output_style table > py3.4_res pyperformance compare release1/py3.5.json release2/py3.5.json --output_style table > py3.5_res +pyperformance compare release1/py3.6.json release2/py3.6.json --output_style table > py3.6_res echo "Completed" diff --git a/tests/google-cloud-python-system/run_system_tests.sh b/tests/google-cloud-python-system/run_system_tests.sh index aab10c88..17d3aac3 100755 --- a/tests/google-cloud-python-system/run_system_tests.sh +++ b/tests/google-cloud-python-system/run_system_tests.sh @@ -28,7 +28,12 @@ export GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE=1 exit_code=0 for package in ${packages}; do noxfile="${package}/nox.py" - nox -f "${noxfile}" -e "system_tests(python_version='2.7')" || exit_code=1 + nox \ + -f "${noxfile}" \ + -e \ + "system_tests(python_version='2.7')" \ + "system_tests(python_version='3.6')" \ + || exit_code=1 done exit "${exit_code}" diff --git a/tests/google-cloud-python/run_unit_tests.sh b/tests/google-cloud-python/run_unit_tests.sh index 8563606e..030919d8 100755 --- a/tests/google-cloud-python/run_unit_tests.sh +++ b/tests/google-cloud-python/run_unit_tests.sh @@ -5,7 +5,14 @@ cd /app/google-cloud-python exit_code=0 for noxfile in */nox.py; do - nox -f "${noxfile}" -e "unit_tests(python_version='2.7')" "unit_tests(python_version='3.4')" "unit_tests(python_version='3.5')" || exit_code=1 + nox \ + -f "${noxfile}" \ + -e \ + "unit_tests(python_version='2.7')" \ + "unit_tests(python_version='3.4')" \ + "unit_tests(python_version='3.5')" \ + "unit_tests(python_version='3.6')" \ + || exit_code=1 done exit "${exit_code}" diff --git a/tests/python3-libraries/python3-libraries.yaml b/tests/python3-libraries/python3-libraries.yaml index 28613555..ef15c0a1 100644 --- a/tests/python3-libraries/python3-libraries.yaml +++ b/tests/python3-libraries/python3-libraries.yaml @@ -7,10 +7,16 @@ globalEnvVars: value: "/env/bin:$PATH" commandTests: - - name: "virtual env teardown" - command: ["rm", "-rf", "/env"] + - name: "requirements 3.5" + setup: + - ["rm", "-rf", "/env"] + - ["virtualenv", "-p", "/opt/python3.5/bin/python3.5", "/env"] + command: ["pip", "install", "-r", "/workspace/tests/python3-libraries/requirements.txt"] + exitCode: 0 - - name: "requirements" - setup: [["virtualenv", "-p", "/opt/python3.5/bin/python3.5", "/env"]] + - name: "requirements 3.6" + setup: + - ["rm", "-rf", "/env"] + - ["virtualenv", "-p", "/opt/python3.6/bin/python3.6", "/env"] command: ["pip", "install", "-r", "/workspace/tests/python3-libraries/requirements.txt"] exitCode: 0 diff --git a/tests/virtualenv/virtualenv_python36.yaml b/tests/virtualenv/virtualenv_python36.yaml new file mode 100644 index 00000000..241099b6 --- /dev/null +++ b/tests/virtualenv/virtualenv_python36.yaml @@ -0,0 +1,48 @@ +schemaVersion: "1.0.0" + +globalEnvVars: + - key: "VIRTUAL_ENV" + value: "/env" + - key: "PATH" + value: "/env/bin:$PATH" + +commandTests: + - name: "virtual env teardown" + command: ["rm", "-rf", "/env"] + + - name: "python installation" + command: ["which", "python3.6"] + expectedOutput: ["/opt/python3.6/bin/python3.6\n"] + + - name: "virtualenv python installation" + setup: [["virtualenv", "-p", "python3.6", "/env"]] + command: ["which", "python"] + expectedOutput: ["/env/bin/python\n"] + + - name: "virtualenv python3 installation" + command: ["which", "python3"] + expectedOutput: ["/env/bin/python3\n"] + + - name: "virtualenv python3.6 installation" + command: ["which", "python3.6"] + expectedOutput: ["/env/bin/python3.6\n"] + + - name: "python version" + command: ["python", "--version"] + expectedOutput: ["Python 3.6.3\n"] + + - name: "pip installation" + command: ["which", "pip"] + expectedOutput: ["/env/bin/pip\n"] + + - name: "pip3 installation" + command: ["which", "pip3"] + expectedOutput: ["/env/bin/pip3\n"] + + - name: "gunicorn flask" + setup: [["pip", "install", "gunicorn", "flask"]] + command: ["which", "gunicorn"] + expectedOutput: ["/env/bin/gunicorn"] + + - name: "flask integration" + command: ["python", "-c", "\"import sys; import flask; sys.exit(0 if flask.__file__.startswith('/env') else 1)\""] From cb0fad1f7ef358ba0f49e27507597c8284762d6e Mon Sep 17 00:00:00 2001 From: Angela Li Date: Mon, 8 May 2017 10:46:34 -0700 Subject: [PATCH 053/256] Implemented the benchmark dashboard --- tests/benchmark/Dockerfile.in | 1 + tests/benchmark/benchmark_between_releases.sh | 46 ++++++-- tests/benchmark/generate_csv.py | 107 ++++++++++++++++++ 3 files changed, 146 insertions(+), 8 deletions(-) create mode 100644 tests/benchmark/generate_csv.py diff --git a/tests/benchmark/Dockerfile.in b/tests/benchmark/Dockerfile.in index 69f41862..5ca308f4 100644 --- a/tests/benchmark/Dockerfile.in +++ b/tests/benchmark/Dockerfile.in @@ -22,3 +22,4 @@ RUN pyperformance run --debug-single-value --python=python3.6 -o /result/py3.6.j RUN pyperformance compare /result/py2.7.json /result/py3.4.json --output_style table RUN pyperformance compare /result/py3.4.json /result/py3.5.json --output_style table RUN pyperformance compare /result/py3.5.json /result/py3.6.json --output_style table + diff --git a/tests/benchmark/benchmark_between_releases.sh b/tests/benchmark/benchmark_between_releases.sh index c540ae7f..11fda0eb 100755 --- a/tests/benchmark/benchmark_between_releases.sh +++ b/tests/benchmark/benchmark_between_releases.sh @@ -17,19 +17,49 @@ rm Dockerfile echo "Successfully built images" # Create folders to hold the files -mkdir release1 -mkdir release2 +mkdir $TAG1 +mkdir $TAG2 # Start running the containers and copy the benchmark result for python versions from container to host -docker run -it --name benchmark_1 -h CONTAINER1 -v "${PWD}"/release1:/export benchmark_1 /bin/bash -c "cp /result/py*.json /export/" -docker run -it --name benchmark_2 -h CONTAINER2 -v "${PWD}"/release2:/export benchmark_2 /bin/bash -c "cp /result/py*.json /export/" +docker run -it --name benchmark_1 -h CONTAINER1 -v "${PWD}"/"$TAG1":/export benchmark_1 /bin/bash -c "cp /result/py*.json /export/" +docker run -it --name benchmark_2 -h CONTAINER2 -v "${PWD}"/"$TAG2":/export benchmark_2 /bin/bash -c "cp /result/py*.json /export/" echo "Start benchmarking the python interpreter performance between the two releases" # Compare the performance between the interpreter in different release -pyperformance compare release1/py2.7.json release2/py2.7.json --output_style table > py2.7_res -pyperformance compare release1/py3.4.json release2/py3.4.json --output_style table > py3.4_res -pyperformance compare release1/py3.5.json release2/py3.5.json --output_style table > py3.5_res -pyperformance compare release1/py3.6.json release2/py3.6.json --output_style table > py3.6_res +pyperformance compare "$TAG1"/py2.7.json "$TAG2"/py2.7.json --output_style table > py2.7_res +pyperformance compare "$TAG1"/py3.4.json "$TAG2"/py3.4.json --output_style table > py3.4_res +pyperformance compare "$TAG1"/py3.5.json "$TAG2"/py3.5.json --output_style table > py3.5_res +pyperformance compare "$TAG1"/py3.6.json "$TAG2"/py3.6.json --output_style table > py3.6_res + +echo "Start extracting data and generating CSV file, then upload to Cloud Storage and insert to Big Query table" + +# Extracting memory usage and running time data from the performace result json, generating CSV files +for path_to_file in $TAG1/*.json; do + python generate_csv.py --filename $path_to_file --tag $TAG1 +done + +for path_to_file in $TAG2/*.json; do + python generate_csv.py --filename $path_to_file --tag $TAG2 +done + +# Get the list of existed release data on Cloud Storage and skip if the current TAG1 or TAG2 existed in the list +gsutil ls gs://python_runtime_benchmark > existed_releases + +for container_tag in $TAG1 $TAG2; do + if grep --fixed-strings --quiet "$container_tag" existed_releases; then + echo "Performace data of $container_tag existed, so skip processing it." + else + # Upload the CSV files to Cloud Storage + gsutil cp -r $container_tag gs://python_runtime_benchmark + # Load the CSV files from Cloud Storage to Big Query table + # Load the performance data of each function + for path_to_file in $container_tag/py2.7.csv $container_tag/py3.4.csv $container_tag/py3.5.csv; do + bq load benchmark_test.benchmark_functions gs://python_runtime_benchmark/"$path_to_file" container_tag:string,runtime_version:string,function_name:string,time_used:float,mem_usage:float + done + # Load the average performance data of each runtime version in a release + bq load benchmark_test.benchmark_statistics gs://python_runtime_benchmark/"$container_tag"/averages.csv container_tag:string,runtime_version:string,ave_time_used:float,ave_mem_usage:float + fi +done echo "Completed" diff --git a/tests/benchmark/generate_csv.py b/tests/benchmark/generate_csv.py new file mode 100644 index 00000000..9409dfa3 --- /dev/null +++ b/tests/benchmark/generate_csv.py @@ -0,0 +1,107 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import argparse +import sys +import json +import csv + + +def generate_csv(args): + """Extract function name, time used and memory usage from the metadata and write to the output CSV file. + + :param args: the command line parameters, including filename and tag + :type: str + """ + with open(args.filename) as input: + data = json.load(input) + benchmarks = data["benchmarks"] + runtime_version = os.path.basename(args.filename).split(".json")[0] + + # Write data to CSV file + with open("{}.csv".format(os.path.splitext(args.filename)[0]), "wb") as output: + csv_writer = csv.writer(output, delimiter=',') + for benchmark in benchmarks: + try: + # Get the function name + func_name = benchmark["metadata"]["name"] + # Get the time used for this function, convert to millisecond + time_used = benchmark["runs"][0]["values"][0] * 1000 + # Get the memory usage, convert to MB + mem_usage = benchmark["metadata"]["mem_max_rss"] / float(1<<20) + line = [args.tag, runtime_version, func_name, time_used, mem_usage] + # Write to CSV file + csv_writer.writerow(line) + except KeyError: + # Skip the benchmark result if it does not contain the fields we want + pass + output.close() + + +def get_averages(args): + """Calculate the averages of time_used and memory_usage and append to CSV file. + + :param args: the command line parameters, including filename and tag + :type: str + """ + with open("{}.csv".format(os.path.splitext(args.filename)[0]), "rb") as input: + lines = input.readlines() + # Get the two columns of times_used and mem_usage + rows_of_data = [map(float, line.split(',')[-2:]) for line in lines] + # Calculate the sum of the two columns + col_sums = map(sum, zip(*rows_of_data)) + # Calculate the average of the two columns by using the sum divided by the total number of lines + averages = [col_sum / len(lines) for col_sum in col_sums] + input.close() + + # Get the runtime version from filename + runtime_version = os.path.basename(args.filename).split(".json")[0] + + # Write the averages to CSV file in appending mode + with open("{}/averages.csv".format(args.tag), "a+") as output: + try: + csv_writer = csv.writer(output, delimiter=',') + csv_writer.writerow([args.tag, runtime_version] + averages) + except IOError: + print "Could not write averages to file." + output.close() + + +def parse_args(argv): + """Parse and validate command line flags""" + parser = argparse.ArgumentParser( + description='Read the python performance json file and extract data to genarate CSV file.' + ) + parser.add_argument( + '--filename', + help='Filename of the performance json file to read' + ) + parser.add_argument( + '--tag', + help='Tag of the docker container' + ) + args = parser.parse_args(argv[1:]) + return args + + +def main(): + args = parse_args(sys.argv) + generate_csv(args) + get_averages(args) + + +if __name__ == '__main__': + main() From 4a2e096f27cb7eda046fa5a5ea560852bce742c6 Mon Sep 17 00:00:00 2001 From: Angela Li Date: Mon, 8 May 2017 10:46:34 -0700 Subject: [PATCH 054/256] Implemented the benchmark dashboard --- tests/benchmark/Dockerfile.in | 1 + tests/benchmark/benchmark_between_releases.sh | 46 ++++++-- tests/benchmark/generate_csv.py | 109 ++++++++++++++++++ 3 files changed, 148 insertions(+), 8 deletions(-) create mode 100644 tests/benchmark/generate_csv.py diff --git a/tests/benchmark/Dockerfile.in b/tests/benchmark/Dockerfile.in index 69f41862..5ca308f4 100644 --- a/tests/benchmark/Dockerfile.in +++ b/tests/benchmark/Dockerfile.in @@ -22,3 +22,4 @@ RUN pyperformance run --debug-single-value --python=python3.6 -o /result/py3.6.j RUN pyperformance compare /result/py2.7.json /result/py3.4.json --output_style table RUN pyperformance compare /result/py3.4.json /result/py3.5.json --output_style table RUN pyperformance compare /result/py3.5.json /result/py3.6.json --output_style table + diff --git a/tests/benchmark/benchmark_between_releases.sh b/tests/benchmark/benchmark_between_releases.sh index c540ae7f..11fda0eb 100755 --- a/tests/benchmark/benchmark_between_releases.sh +++ b/tests/benchmark/benchmark_between_releases.sh @@ -17,19 +17,49 @@ rm Dockerfile echo "Successfully built images" # Create folders to hold the files -mkdir release1 -mkdir release2 +mkdir $TAG1 +mkdir $TAG2 # Start running the containers and copy the benchmark result for python versions from container to host -docker run -it --name benchmark_1 -h CONTAINER1 -v "${PWD}"/release1:/export benchmark_1 /bin/bash -c "cp /result/py*.json /export/" -docker run -it --name benchmark_2 -h CONTAINER2 -v "${PWD}"/release2:/export benchmark_2 /bin/bash -c "cp /result/py*.json /export/" +docker run -it --name benchmark_1 -h CONTAINER1 -v "${PWD}"/"$TAG1":/export benchmark_1 /bin/bash -c "cp /result/py*.json /export/" +docker run -it --name benchmark_2 -h CONTAINER2 -v "${PWD}"/"$TAG2":/export benchmark_2 /bin/bash -c "cp /result/py*.json /export/" echo "Start benchmarking the python interpreter performance between the two releases" # Compare the performance between the interpreter in different release -pyperformance compare release1/py2.7.json release2/py2.7.json --output_style table > py2.7_res -pyperformance compare release1/py3.4.json release2/py3.4.json --output_style table > py3.4_res -pyperformance compare release1/py3.5.json release2/py3.5.json --output_style table > py3.5_res -pyperformance compare release1/py3.6.json release2/py3.6.json --output_style table > py3.6_res +pyperformance compare "$TAG1"/py2.7.json "$TAG2"/py2.7.json --output_style table > py2.7_res +pyperformance compare "$TAG1"/py3.4.json "$TAG2"/py3.4.json --output_style table > py3.4_res +pyperformance compare "$TAG1"/py3.5.json "$TAG2"/py3.5.json --output_style table > py3.5_res +pyperformance compare "$TAG1"/py3.6.json "$TAG2"/py3.6.json --output_style table > py3.6_res + +echo "Start extracting data and generating CSV file, then upload to Cloud Storage and insert to Big Query table" + +# Extracting memory usage and running time data from the performace result json, generating CSV files +for path_to_file in $TAG1/*.json; do + python generate_csv.py --filename $path_to_file --tag $TAG1 +done + +for path_to_file in $TAG2/*.json; do + python generate_csv.py --filename $path_to_file --tag $TAG2 +done + +# Get the list of existed release data on Cloud Storage and skip if the current TAG1 or TAG2 existed in the list +gsutil ls gs://python_runtime_benchmark > existed_releases + +for container_tag in $TAG1 $TAG2; do + if grep --fixed-strings --quiet "$container_tag" existed_releases; then + echo "Performace data of $container_tag existed, so skip processing it." + else + # Upload the CSV files to Cloud Storage + gsutil cp -r $container_tag gs://python_runtime_benchmark + # Load the CSV files from Cloud Storage to Big Query table + # Load the performance data of each function + for path_to_file in $container_tag/py2.7.csv $container_tag/py3.4.csv $container_tag/py3.5.csv; do + bq load benchmark_test.benchmark_functions gs://python_runtime_benchmark/"$path_to_file" container_tag:string,runtime_version:string,function_name:string,time_used:float,mem_usage:float + done + # Load the average performance data of each runtime version in a release + bq load benchmark_test.benchmark_statistics gs://python_runtime_benchmark/"$container_tag"/averages.csv container_tag:string,runtime_version:string,ave_time_used:float,ave_mem_usage:float + fi +done echo "Completed" diff --git a/tests/benchmark/generate_csv.py b/tests/benchmark/generate_csv.py new file mode 100644 index 00000000..d42ca745 --- /dev/null +++ b/tests/benchmark/generate_csv.py @@ -0,0 +1,109 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import argparse +import csv +import json +import os +import sys + + +def generate_csv(filename, tag): + """Extract function name, time used and memory usage from the metadata and write to the output CSV file. + + Args: + filename (str): Filename of the performance json file to read + tag (str): Tag of the docker container + """ + with open(filename) as input: + data = json.load(input) + benchmarks = data["benchmarks"] + runtime_version = os.path.basename(filename).split(".json")[0] + + # Write data to CSV file + with open("{}.csv".format(os.path.splitext(filename)[0]), "wb") as output: + csv_writer = csv.writer(output, delimiter=',') + for benchmark in benchmarks: + try: + # Get the function name + func_name = benchmark["metadata"]["name"] + # Get the time used for this function, convert to millisecond + time_used = benchmark["runs"][0]["values"][0] * 1000 + # Get the memory usage, convert to MB + mem_usage = benchmark["metadata"]["mem_max_rss"] / float(1<<20) + line = [tag, runtime_version, func_name, time_used, mem_usage] + # Write to CSV file + csv_writer.writerow(line) + except KeyError: + # Skip the benchmark result if it does not contain the fields we want + pass + output.close() + + +def get_averages(filename, tag): + """Calculate the averages of time_used and memory_usage and append to CSV file. + + Args: + filename (str): Filename of the performance json file to read + tag (str): Tag of the docker container + """ + with open("{}.csv".format(os.path.splitext(filename)[0]), "rb") as input: + lines = input.readlines() + # Get the two columns of times_used and mem_usage + rows_of_data = [map(float, line.split(',')[-2:]) for line in lines] + # Calculate the sum of the two columns + col_sums = map(sum, zip(*rows_of_data)) + # Calculate the average of the two columns by using the sum divided by the total number of lines + averages = [col_sum / len(lines) for col_sum in col_sums] + input.close() + + # Get the runtime version from filename + runtime_version = os.path.basename(filename).split(".json")[0] + + # Write the averages to CSV file in appending mode + with open("{}/averages.csv".format(tag), "a+") as output: + try: + csv_writer = csv.writer(output, delimiter=',') + csv_writer.writerow([tag, runtime_version] + averages) + except IOError: + print "Could not write averages to file." + output.close() + + +def parse_args(argv): + """Parse and validate command line flags""" + parser = argparse.ArgumentParser( + description='Read the python performance json file and extract data to genarate CSV file.' + ) + parser.add_argument( + '--filename', + help='Filename of the performance json file to read' + ) + parser.add_argument( + '--tag', + help='Tag of the docker container' + ) + args = parser.parse_args(argv[1:]) + return args + + +def main(): + args = parse_args(sys.argv) + generate_csv(args.filename, args.tag) + get_averages(args.filename, args.tag) + + +if __name__ == '__main__': + main() From 5ee1e6c31c1a605d18679086986cfc2f58204a94 Mon Sep 17 00:00:00 2001 From: Angela Li Date: Mon, 8 May 2017 16:44:40 -0700 Subject: [PATCH 055/256] Removed the merging conflicts. --- tests/benchmark/generate_csv.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/benchmark/generate_csv.py b/tests/benchmark/generate_csv.py index e60fb439..e9882303 100644 --- a/tests/benchmark/generate_csv.py +++ b/tests/benchmark/generate_csv.py @@ -42,11 +42,7 @@ def generate_csv(filename, tag): # Get the time used for this function, convert to millisecond time_used = float(benchmark["runs"][0]["values"][0]) * 1000 # Get the memory usage, convert to MB -<<<<<<< HEAD mem_usage = float(benchmark["metadata"]["mem_max_rss"]) / float(1<<20) -======= - mem_usage = benchmark["metadata"]["mem_max_rss"] / float(1<<20) ->>>>>>> 38dcf3b17c2eb6509002f92ac99f43b665ac5005 line = [tag, runtime_version, func_name, time_used, mem_usage] # Write to CSV file csv_writer.writerow(line) From a8d033a5be4cb0ceadf06c0bcd6c1362d2046e29 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Tue, 9 May 2017 10:45:20 -0700 Subject: [PATCH 056/256] Fix the requirements updating script (#115) --- nox.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/nox.py b/nox.py index 6d2e6723..83be1ffa 100644 --- a/nox.py +++ b/nox.py @@ -15,10 +15,6 @@ import fnmatch import os -# Location of our common testing utilities. This isn't published to PyPI. -GCP_REPO_TOOLS_REQ =\ - 'git+https://github.com/GoogleCloudPlatform/python-repo-tools.git' - def _list_files(folder, pattern): """Lists all files below the given folder that match the pattern.""" @@ -30,7 +26,7 @@ def _list_files(folder, pattern): def session_check_requirements(session): """Checks for out of date requirements and optionally updates them.""" - session.install(GCP_REPO_TOOLS_REQ) + session.install('gcp-devrel-py-tools') if 'update' in session.posargs: command = 'update-requirements' @@ -40,4 +36,4 @@ def session_check_requirements(session): reqfiles = list(_list_files('.', 'requirements*.txt')) for reqfile in reqfiles: - session.run('gcprepotools', command, reqfile) + session.run('gcp-devrel-py-tools', command, reqfile) From 0ef2f0b76e0c7012a3268c8ac0242144d37ce44b Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Mon, 8 May 2017 18:06:36 -0700 Subject: [PATCH 057/256] Apply patch to fix double-build issue. Source is https://github.com/python/cpython/pull/1478, backported to 3.5 and 3.6. --- python-interpreter-builder/Dockerfile.in | 1 + .../patches/3.5/double-build.diff | 96 +++++++++++++++++++ python-interpreter-builder/patches/3.5/series | 1 + .../patches/3.6/double-build.diff | 96 +++++++++++++++++++ python-interpreter-builder/patches/3.6/series | 1 + .../scripts/build-python-3.5.sh | 7 +- .../scripts/build-python-3.6.sh | 7 +- 7 files changed, 199 insertions(+), 10 deletions(-) create mode 100644 python-interpreter-builder/patches/3.5/double-build.diff create mode 100644 python-interpreter-builder/patches/3.5/series create mode 100644 python-interpreter-builder/patches/3.6/double-build.diff create mode 100644 python-interpreter-builder/patches/3.6/series diff --git a/python-interpreter-builder/Dockerfile.in b/python-interpreter-builder/Dockerfile.in index 9323f750..4ce3bd02 100644 --- a/python-interpreter-builder/Dockerfile.in +++ b/python-interpreter-builder/Dockerfile.in @@ -44,6 +44,7 @@ ENV LANG C.UTF-8 # Add build scripts ADD scripts /scripts +ADD patches /patches # Build the Python interpreters RUN /scripts/build-python-3.5.sh diff --git a/python-interpreter-builder/patches/3.5/double-build.diff b/python-interpreter-builder/patches/3.5/double-build.diff new file mode 100644 index 00000000..cb52c64b --- /dev/null +++ b/python-interpreter-builder/patches/3.5/double-build.diff @@ -0,0 +1,96 @@ +# Source is https://github.com/python/cpython/pull/1478 + +Index: Python-3.5.3/Makefile.pre.in +=================================================================== +--- Python-3.5.3.orig/Makefile.pre.in ++++ Python-3.5.3/Makefile.pre.in +@@ -982,7 +982,7 @@ TESTTIMEOUT= 3600 + + # Run a basic set of regression tests. + # This excludes some tests that are particularly resource-intensive. +-test: all platform ++test: @DEF_MAKE_RULE@ platform + $(TESTRUNNER) $(TESTOPTS) + + # Run the full test suite twice - once without .pyc files, and once with. +@@ -992,7 +992,7 @@ test: all platform + # the bytecode read from a .pyc file had the bug, sometimes the directly + # generated bytecode. This is sometimes a very shy bug needing a lot of + # sample data. +-testall: all platform ++testall: @DEF_MAKE_RULE@ platform + -find $(srcdir)/Lib -name '*.py[co]' -print | xargs rm -f + $(TESTPYTHON) -E $(srcdir)/Lib/compileall.py + -find $(srcdir)/Lib -name '*.py[co]' -print | xargs rm -f +@@ -1001,7 +1001,7 @@ testall: all platform + + # Run the test suite for both architectures in a Universal build on OSX. + # Must be run on an Intel box. +-testuniversal: all platform ++testuniversal: @DEF_MAKE_RULE@ platform + if [ `arch` != 'i386' ];then \ + echo "This can only be used on OSX/i386" ;\ + exit 1 ;\ +@@ -1024,7 +1024,7 @@ QUICKTESTOPTS= $(TESTOPTS) -x test_subpr + test_multiprocessing_forkserver \ + test_mailbox test_socket test_poll \ + test_select test_zipfile test_concurrent_futures +-quicktest: all platform ++quicktest: @DEF_MAKE_RULE@ platform + $(TESTRUNNER) $(QUICKTESTOPTS) + + +@@ -1376,7 +1376,7 @@ LIBPL= @LIBPL@ + # pkgconfig directory + LIBPC= $(LIBDIR)/pkgconfig + +-libainstall: all python-config ++libainstall: @DEF_MAKE_RULE@ python-config + @for i in $(LIBDIR) $(LIBPL) $(LIBPC); \ + do \ + if test ! -d $(DESTDIR)$$i; then \ +@@ -1635,7 +1635,7 @@ distclean: clobber + -exec rm -f {} ';' + + # Check for smelly exported symbols (not starting with Py/_Py) +-smelly: all ++smelly: @DEF_MAKE_RULE@ + nm -p $(LIBRARY) | \ + sed -n "/ [TDB] /s/.* //p" | grep -v "^_*Py" | sort -u; \ + +@@ -1673,7 +1673,7 @@ funny: + -o -print + + # Perform some verification checks on any modified files. +-patchcheck: all ++patchcheck: @DEF_MAKE_RULE@ + $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/scripts/patchcheck.py + + # Dependencies +Index: Python-3.5.3/Misc/ACKS +=================================================================== +--- Python-3.5.3.orig/Misc/ACKS ++++ Python-3.5.3/Misc/ACKS +@@ -1092,6 +1092,7 @@ Jason Orendorff + Douglas Orr + William Orr + Michele Orrù ++Tomáš Orsava + Oleg Oshmyan + Denis S. Otkidach + Peter Otten +Index: Python-3.5.3/Misc/NEWS +=================================================================== +--- Python-3.5.3.orig/Misc/NEWS ++++ Python-3.5.3/Misc/NEWS +@@ -634,6 +634,10 @@ Windows + Build + ----- + ++- bpo-29243: Prevent unnecessary rebuilding of Python during ``make test``, ++ ``make install`` and some other make targets when configured with ++ ``--enable-optimizations``. ++ + - Issue #29080: Removes hard dependency on hg.exe from PCBuild/build.bat + + - Issue #23903: Added missed names to PC/python3.def. diff --git a/python-interpreter-builder/patches/3.5/series b/python-interpreter-builder/patches/3.5/series new file mode 100644 index 00000000..f6c2876b --- /dev/null +++ b/python-interpreter-builder/patches/3.5/series @@ -0,0 +1 @@ +double-build.diff diff --git a/python-interpreter-builder/patches/3.6/double-build.diff b/python-interpreter-builder/patches/3.6/double-build.diff new file mode 100644 index 00000000..a89a20f6 --- /dev/null +++ b/python-interpreter-builder/patches/3.6/double-build.diff @@ -0,0 +1,96 @@ +# Source is https://github.com/python/cpython/pull/1478 + +Index: Python-3.6.1/Makefile.pre.in +=================================================================== +--- Python-3.6.1.orig/Makefile.pre.in ++++ Python-3.6.1/Makefile.pre.in +@@ -1000,7 +1000,7 @@ TESTTIMEOUT= 1200 + + # Run a basic set of regression tests. + # This excludes some tests that are particularly resource-intensive. +-test: all platform ++test: @DEF_MAKE_RULE@ platform + $(TESTRUNNER) $(TESTOPTS) + + # Run the full test suite twice - once without .pyc files, and once with. +@@ -1010,7 +1010,7 @@ test: all platform + # the bytecode read from a .pyc file had the bug, sometimes the directly + # generated bytecode. This is sometimes a very shy bug needing a lot of + # sample data. +-testall: all platform ++testall: @DEF_MAKE_RULE@ platform + -find $(srcdir)/Lib -name '*.py[co]' -print | xargs rm -f + $(TESTPYTHON) -E $(srcdir)/Lib/compileall.py + -find $(srcdir)/Lib -name '*.py[co]' -print | xargs rm -f +@@ -1019,7 +1019,7 @@ testall: all platform + + # Run the test suite for both architectures in a Universal build on OSX. + # Must be run on an Intel box. +-testuniversal: all platform ++testuniversal: @DEF_MAKE_RULE@ platform + if [ `arch` != 'i386' ];then \ + echo "This can only be used on OSX/i386" ;\ + exit 1 ;\ +@@ -1042,7 +1042,7 @@ QUICKTESTOPTS= $(TESTOPTS) -x test_subpr + test_multiprocessing_forkserver \ + test_mailbox test_socket test_poll \ + test_select test_zipfile test_concurrent_futures +-quicktest: all platform ++quicktest: @DEF_MAKE_RULE@ platform + $(TESTRUNNER) $(QUICKTESTOPTS) + + +@@ -1379,7 +1379,7 @@ LIBPL= @LIBPL@ + # pkgconfig directory + LIBPC= $(LIBDIR)/pkgconfig + +-libainstall: all python-config ++libainstall: @DEF_MAKE_RULE@ python-config + @for i in $(LIBDIR) $(LIBPL) $(LIBPC); \ + do \ + if test ! -d $(DESTDIR)$$i; then \ +@@ -1639,7 +1639,7 @@ distclean: clobber + -exec rm -f {} ';' + + # Check for smelly exported symbols (not starting with Py/_Py) +-smelly: all ++smelly: @DEF_MAKE_RULE@ + nm -p $(LIBRARY) | \ + sed -n "/ [TDB] /s/.* //p" | grep -v "^_*Py" | sort -u; \ + +@@ -1676,7 +1676,7 @@ funny: + -o -print + + # Perform some verification checks on any modified files. +-patchcheck: all ++patchcheck: @DEF_MAKE_RULE@ + $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/scripts/patchcheck.py + + # Dependencies +Index: Python-3.6.1/Misc/ACKS +=================================================================== +--- Python-3.6.1.orig/Misc/ACKS ++++ Python-3.6.1/Misc/ACKS +@@ -1111,6 +1111,7 @@ Jason Orendorff + Douglas Orr + William Orr + Michele Orrù ++Tomáš Orsava + Oleg Oshmyan + Denis S. Otkidach + Peter Otten +Index: Python-3.6.1/Misc/NEWS +=================================================================== +--- Python-3.6.1.orig/Misc/NEWS ++++ Python-3.6.1/Misc/NEWS +@@ -306,6 +306,10 @@ Tests + Build + ----- + ++- bpo-29243: Prevent unnecessary rebuilding of Python during ``make test``, ++ ``make install`` and some other make targets when configured with ++ ``--enable-optimizations``. ++ + - bpo-27593: sys.version and the platform module python_build(), + python_branch(), and python_revision() functions now use + git information rather than hg when building from a repo. diff --git a/python-interpreter-builder/patches/3.6/series b/python-interpreter-builder/patches/3.6/series new file mode 100644 index 00000000..f6c2876b --- /dev/null +++ b/python-interpreter-builder/patches/3.6/series @@ -0,0 +1 @@ +double-build.diff diff --git a/python-interpreter-builder/scripts/build-python-3.5.sh b/python-interpreter-builder/scripts/build-python-3.5.sh index a80473d8..8f536144 100755 --- a/python-interpreter-builder/scripts/build-python-3.5.sh +++ b/python-interpreter-builder/scripts/build-python-3.5.sh @@ -13,8 +13,9 @@ d8890b84d773cd7059e597dbefa510340de8336ec9b9e9032bf030f19291565a Python-3.5.3.t EOF tar xzf Python-3.5.3.tgz -# Build +# Apply patches cd Python-3.5.3 +QUILT_PATCHES=/patches/3.5 quilt push -a # Explanation of flags: # @@ -115,10 +116,6 @@ cd build-static LDFLAGS="-Wl,-z,relro" \ RANLIB="x86_64-linux-gnu-gcc-ranlib" \ -# Due to https://bugs.python.org/issue29243, "make altinstall" -# rebuilds everything from scratch, twice. This is a workaround. -sed -i 's/^all:.*$/all: build_all/' Makefile - make profile-opt make altinstall diff --git a/python-interpreter-builder/scripts/build-python-3.6.sh b/python-interpreter-builder/scripts/build-python-3.6.sh index cb4fa360..05a4e00e 100755 --- a/python-interpreter-builder/scripts/build-python-3.6.sh +++ b/python-interpreter-builder/scripts/build-python-3.6.sh @@ -13,8 +13,9 @@ aa50b0143df7c89ce91be020fe41382613a817354b33acdc6641b44f8ced3828 Python-3.6.1.t EOF tar xzf Python-3.6.1.tgz -# Build +# Apply patches cd Python-3.6.1 +QUILT_PATCHES=/patches/3.6 quilt push -a # Explanation of flags: # @@ -115,10 +116,6 @@ cd build-static LDFLAGS="-Wl,-z,relro" \ RANLIB="x86_64-linux-gnu-gcc-ranlib" \ -# Due to https://bugs.python.org/issue29243, "make altinstall" -# rebuilds everything from scratch, twice. This is a workaround. -sed -i 's/^all:.*$/all: build_all/' Makefile - make profile-opt make altinstall From 3e4599c416654ad00323134707bc7d2b850c372d Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Mon, 8 May 2017 19:17:58 -0700 Subject: [PATCH 058/256] Run Python self-tests when building interpreter Fixes #107 --- python-interpreter-builder/scripts/build-python-3.5.sh | 6 ++++++ python-interpreter-builder/scripts/build-python-3.6.sh | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/python-interpreter-builder/scripts/build-python-3.5.sh b/python-interpreter-builder/scripts/build-python-3.5.sh index 8f536144..32b21ead 100755 --- a/python-interpreter-builder/scripts/build-python-3.5.sh +++ b/python-interpreter-builder/scripts/build-python-3.5.sh @@ -119,6 +119,12 @@ cd build-static make profile-opt make altinstall +# Run tests +# test___all__: Depends on Debian-specific locale changes +# test_imap: https://bugs.python.org/issue30175 +# test_shutil: https://bugs.python.org/issue29317 +make test TESTOPTS="--exclude test___all__ test_imaplib test_shutil" + # Clean-up sources cd /opt rm /opt/sources/Python-3.5.3.tgz diff --git a/python-interpreter-builder/scripts/build-python-3.6.sh b/python-interpreter-builder/scripts/build-python-3.6.sh index 05a4e00e..1d697756 100755 --- a/python-interpreter-builder/scripts/build-python-3.6.sh +++ b/python-interpreter-builder/scripts/build-python-3.6.sh @@ -119,6 +119,12 @@ cd build-static make profile-opt make altinstall +# Run tests +# test___all__: Depends on Debian-specific locale changes +# test_imap: https://bugs.python.org/issue30175 +# test_shutil: https://bugs.python.org/issue29317 +make test TESTOPTS="--exclude test___all__ test_imaplib test_shutil" + # Clean-up sources cd /opt rm /opt/sources/Python-3.6.1.tgz From 26255b491d55390f93fb225e0f3ebc93405e9ffb Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Tue, 9 May 2017 13:15:20 -0700 Subject: [PATCH 059/256] Disable test_dbm for Python 3.6 --- python-interpreter-builder/scripts/build-python-3.6.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python-interpreter-builder/scripts/build-python-3.6.sh b/python-interpreter-builder/scripts/build-python-3.6.sh index 1d697756..9ad508e0 100755 --- a/python-interpreter-builder/scripts/build-python-3.6.sh +++ b/python-interpreter-builder/scripts/build-python-3.6.sh @@ -121,9 +121,10 @@ make altinstall # Run tests # test___all__: Depends on Debian-specific locale changes +# test_dbm: https://bugs.python.org/issue28700 # test_imap: https://bugs.python.org/issue30175 # test_shutil: https://bugs.python.org/issue29317 -make test TESTOPTS="--exclude test___all__ test_imaplib test_shutil" +make test TESTOPTS="--exclude test___all__ test_dbm test_imaplib test_shutil" # Clean-up sources cd /opt From 97dec06d02c608305917036ecccea59b0dd13163 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Tue, 9 May 2017 18:18:17 -0700 Subject: [PATCH 060/256] Actually run the structure tests for Python 3.6 --- cloudbuild.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 32c501b2..f242ac8e 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -18,6 +18,7 @@ steps: '--config', '/workspace/tests/virtualenv/virtualenv_default.yaml', '--config', '/workspace/tests/virtualenv/virtualenv_python34.yaml', '--config', '/workspace/tests/virtualenv/virtualenv_python35.yaml', + '--config', '/workspace/tests/virtualenv/virtualenv_python36.yaml', '--config', '/workspace/tests/no-virtualenv/no-virtualenv.yaml', '--config', '/workspace/tests/python2-libraries/python2-libraries.yaml', '--config', '/workspace/tests/python3-libraries/python3-libraries.yaml', From 37d384d1b83d66e77db689740d5d017f65c4167a Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Tue, 9 May 2017 18:18:56 -0700 Subject: [PATCH 061/256] Perform standard Docker cleanup after running apt-get --- python-interpreter-builder/Dockerfile.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python-interpreter-builder/Dockerfile.in b/python-interpreter-builder/Dockerfile.in index 4ce3bd02..ee9fecf3 100644 --- a/python-interpreter-builder/Dockerfile.in +++ b/python-interpreter-builder/Dockerfile.in @@ -36,8 +36,8 @@ RUN apt-get update && apt-get install -yq \ wget \ xauth \ xvfb \ - zlib1g-dev - + zlib1g-dev \ + && rm -rf /var/lib/apt/lists/* # Setup locale. This prevents Python 3 IO encoding issues. ENV LANG C.UTF-8 From 8fcf1d94aee91b1dc9bf3728e87f5b08245d28de Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Tue, 9 May 2017 18:19:48 -0700 Subject: [PATCH 062/256] Remove unused files to slim down the image Add tests that the 'test' standard library module is functional. --- .../scripts/build-python-3.5.sh | 27 ++++++++++++++++--- .../scripts/build-python-3.6.sh | 24 +++++++++++++++-- tests/virtualenv/virtualenv_python34.yaml | 3 +++ tests/virtualenv/virtualenv_python35.yaml | 3 +++ tests/virtualenv/virtualenv_python36.yaml | 5 +++- 5 files changed, 56 insertions(+), 6 deletions(-) diff --git a/python-interpreter-builder/scripts/build-python-3.5.sh b/python-interpreter-builder/scripts/build-python-3.5.sh index 32b21ead..d43b4bf6 100755 --- a/python-interpreter-builder/scripts/build-python-3.5.sh +++ b/python-interpreter-builder/scripts/build-python-3.5.sh @@ -87,6 +87,8 @@ QUILT_PATCHES=/patches/3.5 quilt push -a # Specifically EXTRA_CFLAGS="-g -flto -fuse-linker-plugin # -ffat-lto-objects" +PREFIX=/opt/python3.5 + mkdir build-static cd build-static @@ -94,7 +96,7 @@ cd build-static --enable-ipv6 \ --enable-loadable-sqlite-extensions \ --enable-optimizations \ - --prefix=/opt/python3.5 \ + --prefix="$PREFIX" \ --with-dbmliborder=bdb:gdbm \ --with-computed-gotos \ --with-fpectl \ @@ -117,13 +119,32 @@ cd build-static RANLIB="x86_64-linux-gnu-gcc-ranlib" \ make profile-opt -make altinstall # Run tests # test___all__: Depends on Debian-specific locale changes +# test_dbm: https://bugs.python.org/issue28700 # test_imap: https://bugs.python.org/issue30175 # test_shutil: https://bugs.python.org/issue29317 -make test TESTOPTS="--exclude test___all__ test_imaplib test_shutil" +make test TESTOPTS="--exclude test___all__ test_dbm test_imaplib test_shutil" + +# Install +make altinstall +# We don't expect users to statically link Python into a C/C++ program +rm "$PREFIX"/lib/libpython3.5m.a \ + "$PREFIX"/lib/python3.5/config-*/libpython3.5m.a +# Remove opt-mode bytecode +find "$PREFIX"/lib/python3.5/ \ + -name \*.opt-\?.pyc \ + -exec rm {} \; +# Remove all but a few files in the 'test' subdirectory +find "$PREFIX"/lib/python3.5/test \ + -mindepth 1 -maxdepth 1 \ + \! -name support \ + -a \! -name __init__.py \ + -a \! -name pystone.\* \ + -a \! -name regrtest.\* \ + -a \! -name test_support.py \ + -exec rm -rf {} \; # Clean-up sources cd /opt diff --git a/python-interpreter-builder/scripts/build-python-3.6.sh b/python-interpreter-builder/scripts/build-python-3.6.sh index 9ad508e0..d8cdf023 100755 --- a/python-interpreter-builder/scripts/build-python-3.6.sh +++ b/python-interpreter-builder/scripts/build-python-3.6.sh @@ -87,6 +87,8 @@ QUILT_PATCHES=/patches/3.6 quilt push -a # Specifically EXTRA_CFLAGS="-g -flto -fuse-linker-plugin # -ffat-lto-objects" +PREFIX=/opt/python3.6 + mkdir build-static cd build-static @@ -94,7 +96,7 @@ cd build-static --enable-ipv6 \ --enable-loadable-sqlite-extensions \ --enable-optimizations \ - --prefix=/opt/python3.6 \ + --prefix="$PREFIX" \ --with-dbmliborder=bdb:gdbm \ --with-computed-gotos \ --with-fpectl \ @@ -117,7 +119,6 @@ cd build-static RANLIB="x86_64-linux-gnu-gcc-ranlib" \ make profile-opt -make altinstall # Run tests # test___all__: Depends on Debian-specific locale changes @@ -126,6 +127,25 @@ make altinstall # test_shutil: https://bugs.python.org/issue29317 make test TESTOPTS="--exclude test___all__ test_dbm test_imaplib test_shutil" +# Install +make altinstall +# We don't expect users to statically link Python into a C/C++ program +rm "$PREFIX"/lib/libpython3.6m.a \ + "$PREFIX"/lib/python3.6/config-*/libpython3.6m.a +# Remove opt-mode bytecode +find "$PREFIX"/lib/python3.6/ \ + -name \*.opt-\?.pyc \ + -exec rm {} \; +# Remove all but a few files in the 'test' subdirectory +find "$PREFIX"/lib/python3.6/test \ + -mindepth 1 -maxdepth 1 \ + \! -name support \ + -a \! -name __init__.py \ + -a \! -name pystone.\* \ + -a \! -name regrtest.\* \ + -a \! -name test_support.py \ + -exec rm -rf {} \; + # Clean-up sources cd /opt rm /opt/sources/Python-3.6.1.tgz diff --git a/tests/virtualenv/virtualenv_python34.yaml b/tests/virtualenv/virtualenv_python34.yaml index 0a498186..082b3b6e 100644 --- a/tests/virtualenv/virtualenv_python34.yaml +++ b/tests/virtualenv/virtualenv_python34.yaml @@ -42,3 +42,6 @@ commandTests: - name: "flask integration" command: ["python", "-c", "\"import sys; import flask; sys.exit(0 if flask.__file__.startswith('/env') else 1)\""] + + - name: "test.support" + command: ["python", "-c", "\"from test import pystone, regrtest, support\""] diff --git a/tests/virtualenv/virtualenv_python35.yaml b/tests/virtualenv/virtualenv_python35.yaml index b37d1461..a9248f88 100644 --- a/tests/virtualenv/virtualenv_python35.yaml +++ b/tests/virtualenv/virtualenv_python35.yaml @@ -46,3 +46,6 @@ commandTests: - name: "flask integration" command: ["python", "-c", "\"import sys; import flask; sys.exit(0 if flask.__file__.startswith('/env') else 1)\""] + + - name: "test.support" + command: ["python", "-c", "\"from test import pystone, regrtest, support\""] diff --git a/tests/virtualenv/virtualenv_python36.yaml b/tests/virtualenv/virtualenv_python36.yaml index 241099b6..83938fc9 100644 --- a/tests/virtualenv/virtualenv_python36.yaml +++ b/tests/virtualenv/virtualenv_python36.yaml @@ -29,7 +29,7 @@ commandTests: - name: "python version" command: ["python", "--version"] - expectedOutput: ["Python 3.6.3\n"] + expectedOutput: ["Python 3.6.1\n"] - name: "pip installation" command: ["which", "pip"] @@ -46,3 +46,6 @@ commandTests: - name: "flask integration" command: ["python", "-c", "\"import sys; import flask; sys.exit(0 if flask.__file__.startswith('/env') else 1)\""] + + - name: "test.support" + command: ["python", "-c", "\"from test import pystone, regrtest, support\""] From 1d7490665baa6e4d3031802adeab4bcf8eaf292f Mon Sep 17 00:00:00 2001 From: DPE bot Date: Sun, 14 May 2017 09:07:52 +0000 Subject: [PATCH 063/256] Auto-update dependencies. --- tests/integration/requirements.txt | 4 +- tests/python2-libraries/requirements.txt | 78 ++++++++++----------- tests/python3-libraries/requirements.txt | 88 ++++++++++-------------- 3 files changed, 79 insertions(+), 91 deletions(-) diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index c5c38c56..71d785ae 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -1,7 +1,7 @@ Flask==0.12.1 -google-cloud-error-reporting==0.24.0 +google-cloud-error-reporting==0.24.2 google-cloud-logging==1.0.0 google-cloud-monitoring==0.24.0 gunicorn==19.7.1 -requests==2.13.0 +requests==2.14.2 retrying==1.3.3 diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 1f635b65..a2f9be77 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -1,46 +1,46 @@ alembic==0.9.1 amqp==2.1.4 amqplib==1.0.2 -ansible==2.2.2.0 +ansible==2.3.0.0 anyjson==0.3.3 -apache-libcloud==1.5.0 +apache-libcloud==2.0.0 argparse==1.4.0 -astroid==1.4.9 -awscli==1.11.76 +astroid==1.5.2 +awscli==1.11.85 babel==2.4.0 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 -beautifulsoup4==4.5.3 +beautifulsoup4==4.6.0 beautifulsoup==3.2.1 billiard==3.5.0.2 blessings==1.6 blinker==1.4 boto==2.46.1 -botocore==1.5.39 +botocore==1.5.48 bottle==0.12.13 -carbon==1.0.0 +carbon==1.0.1 celery==4.0.2 -certifi==2017.1.23 +certifi==2017.4.17 cffi==1.10.0 -chardet==3.0.1 +chardet==3.0.2 click==6.7 -cliff==2.5.0 +cliff==2.7.0 cmd2==0.7.0 -colorama==0.3.7 +colorama==0.3.9 configobj==5.0.6 cov-core==1.15.0 -coverage==4.3.4 +coverage==4.4 coveralls==1.1 cryptography==1.8.1 cssselect==1.0.1 cython==0.25.2 decorator==4.0.11 django-celery==3.2.1 -django-debug-toolbar==1.7 -django-extensions==1.7.8 -django==1.11 +django-debug-toolbar==1.8 +django-extensions==1.7.9 +django==1.11.1 django_compress==1.0.1 -djangorestframework==3.6.2 +djangorestframework==3.6.3 docker-py==1.10.6 docopt==0.6.2 docutils==0.13.1 @@ -49,16 +49,16 @@ elasticsearch==5.3.0 enum34==1.1.6 eventlet==0.21.0 extras==1.0.0 -fabric==1.13.1 +fabric==1.13.2 fixtures==3.0.0 flake8==3.3.0 flask==0.12.1 funcsigs==1.0.2 functools32==3.2.3.post2 -futures==3.0.5 +futures==3.1.1 gevent==1.2.1 google-api-python-client==1.6.2 -graphite-web==1.0.0 +graphite-web==1.0.1 greenlet==0.4.12 gunicorn==19.7.1 hiredis==0.2.0 @@ -66,7 +66,7 @@ html5lib httplib2==0.10.3 idna==2.5 ipaddress==1.0.18 -ipython==5.3.0 +ipython==6.0.0 iso8601==0.1.11 isodate==0.5.4 itsdangerous==0.24 @@ -82,7 +82,7 @@ mako==1.0.6 manifestparser==1.1 markdown==2.6.8 markupsafe==1.0 -matplotlib==2.0.0 +matplotlib==2.0.2 mccabe==0.6.1 meld3==1.0.2 mock==2.0.0 @@ -100,28 +100,28 @@ mysql-python==1.2.5 ndg-httpsclient==0.4.2 netaddr==0.7.19 netifaces==0.10.5 -newrelic==2.82.0.62 +newrelic==2.86.0.65 nose==1.3.7 numpy==1.12.1 oauth2==1.9.0.post1 -oauth2client==4.0.0 +oauth2client==4.1.0 oauthlib==2.0.2 ordereddict==1.1 -oslo.config==3.24.0 -pandas==0.19.2 +oslo.config==4.1.0 +pandas==0.20.1 paramiko==2.1.2 passlib==1.7.1 paste==2.0.3 pastedeploy==1.5.2 pastescript==2.0.2 -pbr==2.1.0 +pbr==3.0.0 pep8==1.7.0 pexpect==4.2.1 pika==0.10.0 -pillow==4.1.0 +pillow==4.1.1 pip==9.0.1 prettytable -protobuf==3.2.0 +protobuf==3.3.0 psutil==5.2.2 psycopg2==2.7.1 py==1.4.33 @@ -132,16 +132,16 @@ pycrypto==2.6.1 pycurl==7.43.0 pyflakes==1.5.0 pygments==2.2.0 -pyjwt==1.4.2 +pyjwt==1.5.0 pylibmc==1.5.2 -pylint==1.6.5 +pylint==1.7.1 pymongo==3.4.0 pymysql==0.7.11 -pyopenssl==16.2.0 +pyopenssl==17.0.0 pyparsing==2.2.0 pyramid==1.8.3 pystache==0.5.4 -pytest-cov==2.4.0 +pytest-cov==2.5.1 pytest==3.0.7 python-cjson==1.2.1 python-daemon==2.1.2 @@ -160,13 +160,13 @@ raven==6.0.0 redis==2.10.5 repoze.lru==0.6 requests-oauthlib==0.8.0 -requests==2.13.0 +requests==2.14.2 retrying==1.3.3 rsa==3.4.2 scipy==0.19.0 -selenium==3.3.3 +selenium==3.4.1 setuptools-git==1.2 -setuptools==34.4.1 +setuptools==35.0.2 sh==1.12.13 simplejson==3.10.0 six==1.10.0 @@ -181,16 +181,16 @@ stevedore==1.21.0 suds==0.4 supervisor==3.3.1 testrepository==0.0.20 -testtools==2.2.0 +testtools==2.3.0 thrift==0.10.0 -tornado==4.4.3 +tornado==4.5.1 tox==2.7.0 twisted==17.1.0 ujson==1.35 unidecode==0.4.20 unittest2==1.1.0 uritemplate==3.0.0 -urllib3==1.20 +urllib3==1.21.1 uwsgi==2.0.15 versiontools==1.9.1 virtualenv==15.1.0 @@ -203,4 +203,4 @@ werkzeug==0.12.1 wheel==0.29.0 xlrd==1.0.0 zc.buildout==2.9.3 -zope.interface==4.3.3 +zope.interface==4.4.1 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 6f971d8d..2364fbed 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -1,46 +1,44 @@ alembic==0.9.1 amqp==2.1.4 amqplib==1.0.2 -ansible==2.2.2.0 +ansible==2.3.0.0 anyjson==0.3.3 -apache-libcloud==1.5.0 +apache-libcloud==2.0.0 argparse==1.4.0 -astroid==1.4.9 -awscli==1.11.68 +astroid==1.5.2 +awscli==1.11.85 babel==2.4.0 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 -#beautifulsoup -beautifulsoup4==4.5.3 +beautifulsoup4==4.6.0 billiard==3.5.0.2 blessings==1.6 blinker==1.4 boto==2.46.1 -botocore==1.5.31 +botocore==1.5.48 bottle==0.12.13 -#carbon celery==4.0.2 -certifi==2017.1.23 +certifi==2017.4.17 cffi==1.10.0 -chardet==3.0.1 +chardet==3.0.2 click==6.7 -cliff==2.5.0 +cliff==2.7.0 cmd2==0.7.0 -colorama==0.3.7 +colorama==0.3.9 configobj==5.0.6 cov-core==1.15.0 -coverage==4.3.4 +coverage==4.4 coveralls==1.1 cryptography==1.8.1 cssselect==1.0.1 cython==0.25.2 decorator==4.0.11 django-celery==3.2.1 -django-debug-toolbar==1.7 -django-extensions==1.7.8 -django==1.11 +django-debug-toolbar==1.8 +django-extensions==1.7.9 +django==1.11.1 django_compress==1.0.1 -djangorestframework==3.6.2 +djangorestframework==3.6.3 docker-py==1.10.6 docopt==0.6.2 docutils==0.13.1 @@ -49,16 +47,14 @@ elasticsearch==5.3.0 enum34==1.1.6 eventlet==0.21.0 extras==1.0.0 -fabric==1.13.1 +fabric==1.13.2 fixtures==3.0.0 flake8==3.3.0 flask==0.12.1 funcsigs==1.0.2 -#functools32 -futures==3.0.5 +futures==3.1.1 gevent==1.2.1 google-api-python-client==1.6.2 -#graphite-web greenlet==0.4.12 gunicorn==19.7.1 hiredis==0.2.0 @@ -66,7 +62,7 @@ html5lib httplib2==0.10.3 idna==2.5 ipaddress==1.0.18 -ipython==5.3.0 +ipython==6.0.0 iso8601==0.1.11 isodate==0.5.4 itsdangerous==0.24 @@ -82,46 +78,43 @@ mako==1.0.6 manifestparser==1.1 markdown==2.6.8 markupsafe==1.0 -matplotlib==2.0.0 +matplotlib==2.0.2 mccabe==0.6.1 meld3==1.0.2 mock==2.0.0 mozcrash==1.0 -mozdevice==0.48 +mozdevice==0.50 mozfile==1.2 mozinfo==0.9 mozlog==3.4 moznetwork==0.27 mozprocess==0.25 -#mozprofile -#mozrunner msgpack-python==0.4.8 -#mysql-python ndg-httpsclient==0.4.2 netaddr==0.7.19 netifaces==0.10.5 -newrelic==2.82.0.62 +newrelic==2.86.0.65 nose==1.3.7 numpy==1.12.1 oauth2==1.9.0.post1 -oauth2client==4.0.0 +oauth2client==4.1.0 oauthlib==2.0.2 ordereddict==1.1 -oslo.config==3.24.0 -pandas==0.19.2 +oslo.config==4.1.0 +pandas==0.20.1 paramiko==2.1.2 passlib==1.7.1 paste==2.0.3 pastedeploy==1.5.2 pastescript==2.0.2 -pbr==2.0.0 +pbr==3.0.0 pep8==1.7.0 pexpect==4.2.1 pika==0.10.0 -pillow==4.1.0 +pillow==4.1.1 pip==9.0.1 prettytable -protobuf==3.2.0 +protobuf==3.3.0 psutil==5.2.2 psycopg2==2.7.1 py==1.4.33 @@ -129,21 +122,19 @@ pyasn1-modules==0.0.8 pyasn1==0.2.3 pycparser==2.17 pycrypto==2.6.1 -#pycurl pyflakes==1.5.0 pygments==2.2.0 -pyjwt==1.4.2 +pyjwt==1.5.0 pylibmc==1.5.2 -pylint==1.6.5 +pylint==1.7.1 pymongo==3.4.0 pymysql==0.7.11 -pyopenssl==16.2.0 +pyopenssl==17.0.0 pyparsing==2.2.0 pyramid==1.8.3 pystache==0.5.4 -pytest-cov==2.4.0 +pytest-cov==2.5.1 pytest==3.0.7 -#python-cjson python-daemon==2.1.2 python-dateutil==2.6.0 python-gflags==3.1.1 @@ -160,13 +151,13 @@ raven==6.0.0 redis==2.10.5 repoze.lru==0.6 requests-oauthlib==0.8.0 -requests==2.13.0 +requests==2.14.2 retrying==1.3.3 rsa==3.4.2 scipy==0.19.0 -selenium==3.3.1 +selenium==3.4.1 setuptools-git==1.2 -setuptools==34.3.3 +setuptools==35.0.2 sh==1.12.13 simplejson==3.10.0 six==1.10.0 @@ -178,20 +169,17 @@ sqlalchemy==1.1.9 sqlparse==0.2.3 statsd==3.2.1 stevedore==1.21.0 -#suds -#supervisor testrepository==0.0.20 -testtools==2.2.0 +testtools==2.3.0 thrift==0.10.0 -tornado==4.4.3 +tornado==4.5.1 tox==2.7.0 twisted==17.1.0 ujson==1.35 unidecode==0.4.20 unittest2==1.1.0 uritemplate==3.0.0 -urllib3==1.20 -#uwsgi +urllib3==1.21.1 versiontools==1.9.1 virtualenv==15.1.0 waitress==1.0.2 @@ -203,4 +191,4 @@ werkzeug==0.12.1 wheel==0.29.0 xlrd==1.0.0 zc.buildout==2.9.3 -zope.interface==4.3.3 +zope.interface==4.4.1 From 5d9d6de32cc1b1071e889534bf116dd1230a4c81 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Mon, 22 May 2017 17:59:32 -0700 Subject: [PATCH 064/256] Add libpython.a back into runtime image. Needed for things that statically link the Python interpreter into a program, such as the uWSGI web server. --- python-interpreter-builder/scripts/build-python-3.5.sh | 5 ++--- python-interpreter-builder/scripts/build-python-3.6.sh | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/python-interpreter-builder/scripts/build-python-3.5.sh b/python-interpreter-builder/scripts/build-python-3.5.sh index d43b4bf6..09653669 100755 --- a/python-interpreter-builder/scripts/build-python-3.5.sh +++ b/python-interpreter-builder/scripts/build-python-3.5.sh @@ -129,9 +129,8 @@ make test TESTOPTS="--exclude test___all__ test_dbm test_imaplib test_shutil" # Install make altinstall -# We don't expect users to statically link Python into a C/C++ program -rm "$PREFIX"/lib/libpython3.5m.a \ - "$PREFIX"/lib/python3.5/config-*/libpython3.5m.a +# Remove redundant copy of libpython +rm "$PREFIX"/lib/libpython3.5m.a # Remove opt-mode bytecode find "$PREFIX"/lib/python3.5/ \ -name \*.opt-\?.pyc \ diff --git a/python-interpreter-builder/scripts/build-python-3.6.sh b/python-interpreter-builder/scripts/build-python-3.6.sh index d8cdf023..d3eb0c33 100755 --- a/python-interpreter-builder/scripts/build-python-3.6.sh +++ b/python-interpreter-builder/scripts/build-python-3.6.sh @@ -129,9 +129,8 @@ make test TESTOPTS="--exclude test___all__ test_dbm test_imaplib test_shutil" # Install make altinstall -# We don't expect users to statically link Python into a C/C++ program -rm "$PREFIX"/lib/libpython3.6m.a \ - "$PREFIX"/lib/python3.6/config-*/libpython3.6m.a +# Remove redundant copy of libpython +rm "$PREFIX"/lib/libpython3.6m.a # Remove opt-mode bytecode find "$PREFIX"/lib/python3.6/ \ -name \*.opt-\?.pyc \ From e7529122cd5862dea808fddb930cc3a6ef632590 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Mon, 22 May 2017 18:00:31 -0700 Subject: [PATCH 065/256] Test that uWSGI installs properly in Python 3.x --- tests/python3-libraries/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 2364fbed..e99f3629 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -180,6 +180,7 @@ unidecode==0.4.20 unittest2==1.1.0 uritemplate==3.0.0 urllib3==1.21.1 +uwsgi==2.0.15 versiontools==1.9.1 virtualenv==15.1.0 waitress==1.0.2 From b3ef908ef3a1f7ed3c4420394f45b0c0ff972651 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Mon, 22 May 2017 18:00:52 -0700 Subject: [PATCH 066/256] Workaround the fact that ipython 6.x does not support Python 2 --- tests/python2-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index a2f9be77..61e36d58 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -66,7 +66,7 @@ html5lib httplib2==0.10.3 idna==2.5 ipaddress==1.0.18 -ipython==6.0.0 +#ipython # No longer supports Python 2 as of version 6.0 iso8601==0.1.11 isodate==0.5.4 itsdangerous==0.24 From 6f2275b44ff4a4b083f013f014fc37849952c79e Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Thu, 15 Jun 2017 18:20:56 -0700 Subject: [PATCH 067/256] Move some utility functions into their own file --- scripts/local_cloudbuild.py | 96 ++++----------------------- scripts/local_cloudbuild_test.py | 83 +---------------------- scripts/validation_utils.py | 98 +++++++++++++++++++++++++++ scripts/validation_utils_test.py | 110 +++++++++++++++++++++++++++++++ 4 files changed, 221 insertions(+), 166 deletions(-) create mode 100644 scripts/validation_utils.py create mode 100755 scripts/validation_utils_test.py diff --git a/scripts/local_cloudbuild.py b/scripts/local_cloudbuild.py index f134278d..f0c6659d 100755 --- a/scripts/local_cloudbuild.py +++ b/scripts/local_cloudbuild.py @@ -42,6 +42,8 @@ import yaml +import validation_utils + # Exclude non-printable control characters (including newlines) PRINTABLE_REGEX = re.compile(r"""^[^\x00-\x1f]*$""") @@ -59,10 +61,6 @@ ) """) -# For easier development, we allow redefining builtins like -# --substitutions=PROJECT_ID=foo even though gcloud doesn't. -KEY_VALUE_REGEX = re.compile(r'^([A-Z_][A-Z0-9_]*)=(.*)$') - # Default builtin substitutions DEFAULT_SUBSTITUTIONS = { 'BRANCH_NAME': '', @@ -149,51 +147,6 @@ def sub(match): quoted_s = shlex.quote(substituted_s) return quoted_s -def get_field_value(container, field_name, field_type): - """Fetch a field from a container with typechecking and default values. - - The field value is coerced to the desired type. If the field is - not present, a instance of `field_type` is constructed with no - arguments and used as the default value. - - Args: - container (dict): Object decoded from yaml - field_name (str): Field that should be present in `container` - field_type (type): Expected type for field value - - Returns: - Any: Fetched or default value of field - - Raises: - ValueError: if field value cannot be converted to the desired type - """ - try: - value = container[field_name] - except (IndexError, KeyError): - return field_type() - - msg = 'Expected "{}" field to be of type "{}", but found type "{}"' - if not isinstance(value, field_type): - # list('some string') is a successful type cast as far as Python - # is concerned, but doesn't exactly produce the results we want. - # We have a whitelist of conversions we will attempt. - whitelist = ( - (float, str), - (int, str), - (str, float), - (str, int), - (int, float), - ) - if (type(value), field_type) not in whitelist: - raise ValueError(msg.format(field_name, field_type, type(value))) - - try: - value = field_type(value) - except ValueError as e: - e.message = msg.format(field_name, field_type, type(value)) - raise - return value - def get_cloudbuild(raw_config, args): """Read and validate a cloudbuild recipe @@ -210,7 +163,7 @@ def get_cloudbuild(raw_config, args): 'Expected {} contents to be of type "dict", but found type "{}"'. format(args.config, type(raw_config))) - raw_steps = get_field_value(raw_config, 'steps', list) + raw_steps = validation_utils.get_field_value(raw_config, 'steps', list) if not raw_steps: raise ValueError('No steps defined in {}'.format(args.config)) @@ -236,14 +189,14 @@ def get_step(raw_step): raise ValueError( 'Expected step to be of type "dict", but found type "{}"'. format(type(raw_step))) - raw_args = get_field_value(raw_step, 'args', list) - args = [get_field_value(raw_args, index, str) + raw_args = validation_utils.get_field_value(raw_step, 'args', list) + args = [validation_utils.get_field_value(raw_args, index, str) for index in range(len(raw_args))] - dir_ = get_field_value(raw_step, 'dir', str) - raw_env = get_field_value(raw_step, 'env', list) - env = [get_field_value(raw_env, index, str) + dir_ = validation_utils.get_field_value(raw_step, 'dir', str) + raw_env = validation_utils.get_field_value(raw_step, 'env', list) + env = [validation_utils.get_field_value(raw_env, index, str) for index in range(len(raw_env))] - name = get_field_value(raw_step, 'name', str) + name = validation_utils.get_field_value(raw_step, 'name', str) return Step( args=args, dir_=dir_, @@ -373,31 +326,6 @@ def local_cloudbuild(args): subprocess.check_call(args) -def validate_arg_regex(flag_value, flag_regex): - """Check a named command line flag against a regular expression""" - if not re.match(flag_regex, flag_value): - raise argparse.ArgumentTypeError( - 'Value "{}" does not match pattern "{}"'.format( - flag_value, flag_regex.pattern)) - return flag_value - - -def validate_arg_dict(flag_value): - """Parse a command line flag as a key=val,... dict""" - if not flag_value: - return {} - entries = flag_value.split(',') - pairs = [] - for entry in entries: - match = re.match(KEY_VALUE_REGEX, entry) - if not match: - raise argparse.ArgumentTypeError( - 'Value "{}" should be a list like _KEY1=value1,_KEY2=value2"'.format( - flag_value)) - pairs.append((match.group(1), match.group(2))) - return dict(pairs) - - def parse_args(argv): """Parse and validate command line flags""" parser = argparse.ArgumentParser( @@ -405,14 +333,14 @@ def parse_args(argv): parser.add_argument( '--config', type=functools.partial( - validate_arg_regex, flag_regex=PRINTABLE_REGEX), + validation_utils.validate_arg_regex, flag_regex=PRINTABLE_REGEX), default='cloudbuild.yaml', help='Path to cloudbuild.yaml file' ) parser.add_argument( '--output_script', type=functools.partial( - validate_arg_regex, flag_regex=PRINTABLE_REGEX), + validation_utils.validate_arg_regex, flag_regex=PRINTABLE_REGEX), help='Filename to write shell script to', ) parser.add_argument( @@ -423,7 +351,7 @@ def parse_args(argv): ) parser.add_argument( '--substitutions', - type=validate_arg_dict, + type=validation_utils.validate_arg_dict, default={}, help='Parameters to be substituted in the build specification', ) diff --git a/scripts/local_cloudbuild_test.py b/scripts/local_cloudbuild_test.py index 0409c98f..68a7ff27 100755 --- a/scripts/local_cloudbuild_test.py +++ b/scripts/local_cloudbuild_test.py @@ -34,91 +34,10 @@ STAGING_DIR_REGEX = re.compile( b'(?m)Copying source to staging directory (.+)$') -class ValidationUtilsTest(unittest.TestCase): - - def test_get_field_value(self): - valid_cases = ( - # Normal case, field present and correct type - ({ 'present': 1 }, 'present', int, 1), - ({ 'present': '1' }, 'present', str, '1'), - ({ 'present': [1] }, 'present', list, [1]), - ({ 'present': {1: 2} }, 'present', dict, {1: 2}), - # Missing field replaced by default - ({}, 'missing', str, ''), - # Valid conversions - ({ 'str_to_int': '1' }, 'str_to_int', int, 1), - ({ 'int_to_str': 1 }, 'int_to_str', str, '1'), - ) - for valid_case in valid_cases: - with self.subTest(valid_case=valid_case): - container, field_name, field_type, expected = valid_case - self.assertEqual( - local_cloudbuild.get_field_value( - container, field_name, field_type), - expected) - - invalid_cases = ( - # Type conversion failures - ({ 'bad_list_to_dict': [1] }, 'bad_list_to_dict', dict), - ({ 'bad_list_to_str': [1] }, 'bad_list_to_str', str), - ({ 'bad_dict_to_list': {1: 2} }, 'bad_dict_to_list', list), - ({ 'bad_str_to_int': 'not_an_int' }, 'bad_str_to_int', int), - ({ 'bad_str_to_list': 'abc' }, 'bad_str_to_list', list), - ) - for invalid_case in invalid_cases: - with self.subTest(invalid_case=invalid_case): - container, field_name, field_type = invalid_case - with self.assertRaises(ValueError): - local_cloudbuild.get_field_value( - container, field_name, field_type) - - def test_validate_arg_regex(self): - self.assertEqual( - local_cloudbuild.validate_arg_regex('abc', re.compile('a[b]c')), - 'abc') - with self.assertRaises(argparse.ArgumentTypeError): - local_cloudbuild.validate_arg_regex('abc', re.compile('a[d]c')) - - - def test_validate_arg_dict(self): - valid_cases = ( - # Normal case, field present and correct type - ('', {}), - ('_A=1', {'_A':'1'}), - ('_A=1,_B=2', {'_A':'1', '_B':'2'}), - # Repeated key is ok - ('_A=1,_A=2', {'_A':'2'}), - # Extra = is ok - ('_A=x=y=z,_B=2', {'_A':'x=y=z', '_B':'2'}), - # No value is ok - ('_A=', {'_A':''}), - ) - for valid_case in valid_cases: - with self.subTest(valid_case=valid_case): - s, expected = valid_case - self.assertEqual( - local_cloudbuild.validate_arg_dict(s), - expected) - - invalid_cases = ( - # No key - ',_A', - '_A,', - # Invalid variable name - '_Aa=1', - '_aA=1', - '0A=1', - ) - for invalid_case in invalid_cases: - with self.subTest(invalid_case=invalid_case): - with self.assertRaises(argparse.ArgumentTypeError): - local_cloudbuild.validate_arg_dict(invalid_case) - - class LocalCloudbuildTest(unittest.TestCase): def setUp(self): - self.testdata_dir = 'testdata' + self.testdata_dir = os.path.join(os.path.dirname(__file__), 'testdata') # Sigh assert os.path.isdir(self.testdata_dir), 'Could not run test: testdata directory not found' def test_sub_and_quote(self): diff --git a/scripts/validation_utils.py b/scripts/validation_utils.py new file mode 100644 index 00000000..7a04847e --- /dev/null +++ b/scripts/validation_utils.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 + +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Utilities for schema and command line validation""" + +import argparse +import re + + +# For easier development, we allow redefining builtins like +# --substitutions=PROJECT_ID=foo even though gcloud doesn't. +KEY_VALUE_REGEX = re.compile(r'^([A-Z_][A-Z0-9_]*)=(.*)$') + + +def get_field_value(container, field_name, field_type): + """Fetch a field from a container with typechecking and default values. + + The field value is coerced to the desired type. If the field is + not present, a instance of `field_type` is constructed with no + arguments and used as the default value. + + Args: + container (dict): Object decoded from yaml + field_name (str): Field that should be present in `container` + field_type (type): Expected type for field value + + Returns: + Any: Fetched or default value of field + + Raises: + ValueError: if field value cannot be converted to the desired type + """ + try: + value = container[field_name] + if value is None: + return field_type() + except (IndexError, KeyError): + return field_type() + + msg = 'Expected "{}" field to be of type "{}", but found type "{}"' + if not isinstance(value, field_type): + # list('some string') is a successful type cast as far as Python + # is concerned, but doesn't exactly produce the results we want. + # We have a whitelist of conversions we will attempt. + whitelist = ( + (float, str), + (int, str), + (str, float), + (str, int), + (int, float), + ) + if (type(value), field_type) not in whitelist: + raise ValueError(msg.format(field_name, field_type, type(value))) + + try: + value = field_type(value) + except ValueError as e: + e.message = msg.format(field_name, field_type, type(value)) + raise + return value + + +def validate_arg_regex(flag_value, flag_regex): + """Check a named command line flag against a regular expression""" + if not re.match(flag_regex, flag_value): + raise argparse.ArgumentTypeError( + 'Value "{}" does not match pattern "{}"'.format( + flag_value, flag_regex.pattern)) + return flag_value + + +def validate_arg_dict(flag_value): + """Parse a command line flag as a key=val,... dict""" + if not flag_value: + return {} + entries = flag_value.split(',') + pairs = [] + for entry in entries: + match = re.match(KEY_VALUE_REGEX, entry) + if not match: + raise argparse.ArgumentTypeError( + 'Value "{}" should be a list like _KEY1=value1,_KEY2=value2"'.format( + flag_value)) + pairs.append((match.group(1), match.group(2))) + return dict(pairs) diff --git a/scripts/validation_utils_test.py b/scripts/validation_utils_test.py new file mode 100755 index 00000000..9ef87096 --- /dev/null +++ b/scripts/validation_utils_test.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 + +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit test for validation_utils.py""" + +import argparse +import re +import unittest + +import validation_utils + + +class ValidationUtilsTest(unittest.TestCase): + + def test_get_field_value(self): + valid_cases = ( + # Normal case, field present and correct type + ({ 'present': 1 }, 'present', int, 1), + ({ 'present': '1' }, 'present', str, '1'), + ({ 'present': [1] }, 'present', list, [1]), + ({ 'present': {1: 2} }, 'present', dict, {1: 2}), + # Missing field replaced by default + ({}, 'missing', str, ''), + # Valid conversions + ({ 'str_to_int': '1' }, 'str_to_int', int, 1), + ({ 'int_to_str': 1 }, 'int_to_str', str, '1'), + # None + ({ 'None_to_int': None }, 'None_to_int', int, 0), + ({ 'None_to_str': None }, 'None_to_str', str, ''), + ) + for valid_case in valid_cases: + with self.subTest(valid_case=valid_case): + container, field_name, field_type, expected = valid_case + self.assertEqual( + validation_utils.get_field_value( + container, field_name, field_type), + expected) + + invalid_cases = ( + # Type conversion failures + ({ 'bad_list_to_dict': [1] }, 'bad_list_to_dict', dict), + ({ 'bad_list_to_str': [1] }, 'bad_list_to_str', str), + ({ 'bad_dict_to_list': {1: 2} }, 'bad_dict_to_list', list), + ({ 'bad_str_to_int': 'not_an_int' }, 'bad_str_to_int', int), + ({ 'bad_str_to_list': 'abc' }, 'bad_str_to_list', list), + ) + for invalid_case in invalid_cases: + with self.subTest(invalid_case=invalid_case): + container, field_name, field_type = invalid_case + with self.assertRaises(ValueError): + validation_utils.get_field_value( + container, field_name, field_type) + + def test_validate_arg_regex(self): + self.assertEqual( + validation_utils.validate_arg_regex('abc', re.compile('a[b]c')), + 'abc') + with self.assertRaises(argparse.ArgumentTypeError): + validation_utils.validate_arg_regex('abc', re.compile('a[d]c')) + + def test_validate_arg_dict(self): + valid_cases = ( + # Normal case, field present and correct type + ('', {}), + ('_A=1', {'_A':'1'}), + ('_A=1,_B=2', {'_A':'1', '_B':'2'}), + # Repeated key is ok + ('_A=1,_A=2', {'_A':'2'}), + # Extra = is ok + ('_A=x=y=z,_B=2', {'_A':'x=y=z', '_B':'2'}), + # No value is ok + ('_A=', {'_A':''}), + ) + for valid_case in valid_cases: + with self.subTest(valid_case=valid_case): + s, expected = valid_case + self.assertEqual( + validation_utils.validate_arg_dict(s), + expected) + + invalid_cases = ( + # No key + ',_A', + '_A,', + # Invalid variable name + '_Aa=1', + '_aA=1', + '0A=1', + ) + for invalid_case in invalid_cases: + with self.subTest(invalid_case=invalid_case): + with self.assertRaises(argparse.ArgumentTypeError): + validation_utils.validate_arg_dict(invalid_case) + + +if __name__ == '__main__': + unittest.main() From 58e8abb82578964a8c3997ac0e6dd590f9ab52d3 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Thu, 15 Jun 2017 18:22:06 -0700 Subject: [PATCH 068/256] Add script to generate Dockerfiles like Google Cloud SDK does. --- scripts/gen_dockerfile.py | 252 ++++++++++++++++++ scripts/gen_dockerfile_test.py | 227 ++++++++++++++++ scripts/testdata/hello_world/app.yaml | 6 + scripts/testdata/hello_world/main.py | 43 +++ scripts/testdata/hello_world/main_test.py | 24 ++ scripts/testdata/hello_world/requirements.txt | 2 + .../testdata/hello_world_golden/.dockerignore | 19 ++ .../testdata/hello_world_golden/Dockerfile | 12 + 8 files changed, 585 insertions(+) create mode 100755 scripts/gen_dockerfile.py create mode 100755 scripts/gen_dockerfile_test.py create mode 100644 scripts/testdata/hello_world/app.yaml create mode 100644 scripts/testdata/hello_world/main.py create mode 100644 scripts/testdata/hello_world/main_test.py create mode 100644 scripts/testdata/hello_world/requirements.txt create mode 100644 scripts/testdata/hello_world_golden/.dockerignore create mode 100644 scripts/testdata/hello_world_golden/Dockerfile diff --git a/scripts/gen_dockerfile.py b/scripts/gen_dockerfile.py new file mode 100755 index 00000000..07d2f8c5 --- /dev/null +++ b/scripts/gen_dockerfile.py @@ -0,0 +1,252 @@ +#!/usr/bin/env python2 + +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Generate a Dockerfile and helper files for a Python application.""" + +import argparse +import collections +import functools +import io +import os +import re +import sys + +import yaml + +import validation_utils + + +# Validate characters for dockerfile image names +# https://github.com/docker/docker/blob/master/api/names.go +IMAGE_REGEX = re.compile(r"""(?x) + ^ + [a-zA-Z0-9] # First char must be alphanumeric + [-a-zA-Z0-9_./]* # Punctuation allowed after that + ( + : # At most one colon allowed + [-a-zA-Z0-9_./]+ # Colon must be followed by other chars + )? + $ +""") + +# `entrypoint` is specified as free-form text parsed as a unix shell +# command line, which limits the sanity checking possible. We +# disallow newlines and control characters which would break the +# Dockerfile format. +PRINTABLE_REGEX = re.compile(r"""^[^\x00-\x1f]*$""") + +# Map from app.yaml "python_version" to {python_version} in Dockerfile +PYTHON_INTERPRETER_VERSION_MAP = { + '': '', # == 2.7 + '2': '', # == 2.7 + '3': '3.5', + '3.4': '3.4', + '3.5': '3.5', + '3.6': '3.6', +} + +# File templates. +# Designed to exactly match the current output of 'gcloud app gen-config' +DOCKERFILE_TEMPLATE = """\ +FROM {base_image} +LABEL python_version=python{dockerfile_python_version} +RUN virtualenv --no-download /env -p python{dockerfile_python_version} + +# Set virtualenv environment variables. This is equivalent to running +# source /env/bin/activate +ENV VIRTUAL_ENV /env +ENV PATH /env/bin:$PATH +{optional_requirements_txt}ADD . /app/ +{optional_entrypoint}""" + +DOCKERFILE_REQUIREMENTS_TXT = """\ +ADD requirements.txt /app/ +RUN pip install -r requirements.txt +""" + +DOCKERFILE_ENTRYPOINT_TEMPLATE = """\ +CMD {entrypoint} +""" + +DOCKERIGNORE = """\ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +.dockerignore +Dockerfile +.git +.hg +.svn +""" + +# Validated application configuration +AppConfig = collections.namedtuple( + 'AppConfig', + 'base_image dockerfile_python_version entrypoint has_requirements_txt' +) + + +def get_app_config(raw_config, args): + """Read and validate the application runtime configuration. + + Args: + raw_config (dict): deserialized app.yaml + args (argparse.Namespace): ccommand line flags + + Returns: + AppConfig: valid configuration + """ + # Examine app.yaml + if not isinstance(raw_config, dict): + raise ValueError( + 'Expected {} contents to be of type "dict", but found type "{}"'. + format(args.config, type(raw_config))) + + entrypoint = validation_utils.get_field_value(raw_config, 'entrypoint', str) + if not PRINTABLE_REGEX.match(entrypoint): + raise ValueError('Invalid character in "entrypoint" field of app.yaml') + raw_runtime_config = validation_utils.get_field_value(raw_config, 'runtime_config', dict) + python_version = validation_utils.get_field_value(raw_runtime_config, 'python_version', str) + dockerfile_python_version = PYTHON_INTERPRETER_VERSION_MAP.get( + python_version) + if dockerfile_python_version is None: + valid_versions = str(sorted(PYTHON_INTERPRETER_VERSION_MAP.keys())) + msg = ('Invalid "python_version" field in "runtime_config" section ' + 'of app.yaml. Valid options are:\n{}').format(valid_versions) + raise ValueError(msg) + + # Examine user's files + has_requirements_txt = os.path.isfile( + os.path.join(args.source_dir, 'requirements.txt')) + + # Compute base image name and Dockerfile contents + base_image = args.base_image + + return AppConfig( + base_image=base_image, + dockerfile_python_version=dockerfile_python_version, + entrypoint=entrypoint, + has_requirements_txt=has_requirements_txt) + + +def generate_files(app_config): + """Generate a Dockerfile and helper files for an application. + + Args: + app_config (AppConfig): Validated configuration + + Returns: + dict: Map of filename to desired file contents + """ + if app_config.has_requirements_txt: + optional_requirements_txt = DOCKERFILE_REQUIREMENTS_TXT + else: + optional_requirements_txt = '' + + if app_config.entrypoint: + # Mangle entrypoint in the same way as the Cloud SDK + # (googlecloudsdk/third_party/appengine/api/validation.py) + # + # We could handle both string ("shell form") and list ("exec + # form") but it appears that gcloud only handles string form. + entrypoint = app_config.entrypoint + if entrypoint and not entrypoint.startswith('exec '): + entrypoint = 'exec ' + entrypoint + optional_entrypoint = DOCKERFILE_ENTRYPOINT_TEMPLATE.format( + entrypoint=entrypoint) + else: + optional_entrypoint = '' + + dockerfile = DOCKERFILE_TEMPLATE.format( + base_image=app_config.base_image, + dockerfile_python_version=app_config.dockerfile_python_version, + optional_requirements_txt=optional_requirements_txt, + optional_entrypoint=optional_entrypoint) + + return { + 'Dockerfile': dockerfile, + '.dockerignore': DOCKERIGNORE, + } + + +def gen_dockerfile(args): + """Write a Dockerfile and helper files for an application. + + Args: + args (argparse.Namespace): ccommand line flags + """ + # Read yaml file. Does not currently support multiple services + # with configuration filenames besides app.yaml + with io.open(args.config, 'r', encoding='utf8') as yaml_config_file: + raw_config = yaml.load(yaml_config_file) + + # Determine configuration + app_config = get_app_config(raw_config, args) + + # Generate list of filenames and their textual contents + files = generate_files(app_config) + + # Write files + for filename, contents in files.items(): + full_filename = os.path.join(args.source_dir, filename) + with io.open(full_filename, 'w', encoding='utf8') as outfile: + outfile.write(contents) + + +def parse_args(argv): + """Parse and validate command line flags""" + parser = argparse.ArgumentParser() + parser.add_argument( + '--config', + type=functools.partial( + validation_utils.validate_arg_regex, flag_regex=PRINTABLE_REGEX), + default='app.yaml', + help='Path to application configuration file' + ) + parser.add_argument( + '--base-image', + type=functools.partial( + validation_utils.validate_arg_regex, flag_regex=IMAGE_REGEX), + default='gcr.io/google-appengine/python:latest', + help='Name of Docker image to use as base') + parser.add_argument( + '--source-dir', + type=functools.partial( + validation_utils.validate_arg_regex, flag_regex=PRINTABLE_REGEX), + default='.', + help=('Application source and output directory')) + args = parser.parse_args(argv[1:]) + return args + + +def main(): + args = parse_args(sys.argv) + gen_dockerfile(args) + + +if __name__ == '__main__': + main() diff --git a/scripts/gen_dockerfile_test.py b/scripts/gen_dockerfile_test.py new file mode 100755 index 00000000..faf44fc7 --- /dev/null +++ b/scripts/gen_dockerfile_test.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python2 + +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit test for gen_dockerfile.py""" + +import argparse +import os +import re +import shutil +import subprocess +import tempfile +import unittest +import unittest.mock + +import yaml + +import gen_dockerfile + +# Expected list of files generated +EXPECTED_OUTPUT_FILES = ['Dockerfile', '.dockerignore'] + + +class GenDockerfileTest(unittest.TestCase): + def setUp(self): + self.testdata_dir = os.path.join(os.path.dirname(__file__), 'testdata') # Sigh + assert os.path.isdir(self.testdata_dir), 'Could not run test: testdata directory not found' + + def compare_file(self, filename, dir1, dir2): + """Compare identically named files in two different directories""" + with open(os.path.join(dir1, filename), 'r', encoding='utf8') as file1: + with open(os.path.join(dir2, filename), 'r', encoding='utf8') as file2: + contents1 = file1.read() + contents2 = file2.read() + msg = 'Contents of "{}" differ between "{}" and "{}"'.format( + filename, dir1, dir2) + self.assertMultiLineEqual(contents1, contents2, msg) + + def test_get_app_config(self): + args = argparse.Namespace( + config='some_config_file', + base_image='some_image_name', + source_dir='some_source_dir') + + valid_cases = ( + # Basic app.yaml + ('env: flex', False, { + 'base_image': 'some_image_name', + 'dockerfile_python_version': '', + 'has_requirements_txt': False, + 'entrypoint': '', + }), + # All supported python versions + ('runtime_config:\n python_version:', False, { + 'dockerfile_python_version': '', + }), + ('runtime_config:\n python_version: 2', False, { + 'dockerfile_python_version': '', + }), + ('runtime_config:\n python_version: 3', False, { + 'dockerfile_python_version': '3.5', + }), + ('runtime_config:\n python_version: 3.4', False, { + 'dockerfile_python_version': '3.4', + }), + ('runtime_config:\n python_version: 3.5', False, { + 'dockerfile_python_version': '3.5', + }), + # requirements.txt present + ('env: flex', True, { + 'has_requirements_txt': True, + }), + # entrypoint present + ('entrypoint: my entrypoint', False, { + 'entrypoint': 'my entrypoint', + }), + ) + for valid_case in valid_cases: + with self.subTest(valid_case=valid_case): + app_yaml, isfile, expected = valid_case + raw_app_config = yaml.safe_load(app_yaml) + with unittest.mock.patch.object( + os.path, 'isfile', return_value=isfile): + actual = gen_dockerfile.get_app_config(raw_app_config, args) + for key, value in expected.items(): + self.assertEqual(getattr(actual, key), value) + + invalid_cases = ( + # Empty app.yaml + '', + # Invalid entrypoint + 'entrypoint: "bad \\n entrypoint"', + # Invalid python version + 'runtime_config:\n python_version: 1', + 'runtime_config:\n python_version: python2', + ) + for invalid_case in invalid_cases: + with self.subTest(invalid_case=invalid_case): + raw_app_config = yaml.safe_load(invalid_case) + with self.assertRaises(ValueError): + gen_dockerfile.get_app_config(raw_app_config, args) + + def test_generate_files(self): + base = gen_dockerfile.AppConfig( + base_image='', + dockerfile_python_version='', + entrypoint='', + has_requirements_txt=False + ) + cases = ( + # Requirements.txt + (base, False, 'ADD requirements.txt'), + (base._replace(has_requirements_txt=True), True, + 'ADD requirements.txt'), + # Entrypoint + (base, False, 'CMD'), + (base._replace(entrypoint='my entrypoint'), True, + 'CMD exec my entrypoint'), + (base._replace(entrypoint='exec my entrypoint'), True, + 'CMD exec my entrypoint'), + # Base runtime image + (base._replace(base_image='my_base_runtime_image'), True, + 'FROM my_base_runtime_image'), + # Python version + (base._replace(dockerfile_python_version='_my_version'), True, + 'python_version=python_my_version'), + ) + for case in cases: + with self.subTest(case=case): + app_config, should_find, test_string = case + result = gen_dockerfile.generate_files(app_config) + self.assertEqual( + sorted(result.keys()), sorted(EXPECTED_OUTPUT_FILES)) + dockerfile = result['Dockerfile'] + if should_find: + self.assertIn(test_string, dockerfile) + else: + self.assertNotIn(test_string, dockerfile) + + def test_gen_dockerfile(self): + """Generates output and compares against a set of golden files. + + Optionally runs 'gcloud app gen-config' and compares against that. + """ + # Sample app from https://github.com/GoogleCloudPlatform/python-docs-samples + with tempfile.TemporaryDirectory( + prefix='gen_dockerfile_test_') as parent_tempdir: + for app in ['hello_world']: + app_dir = os.path.join(self.testdata_dir, app) + temp_dir = os.path.join(parent_tempdir, app) + os.mkdir(temp_dir) + + # Copy sample app to writable temp dir, and generate Dockerfile. + config_dir = os.path.join(temp_dir, 'config') + shutil.copytree(app_dir, config_dir) + args = argparse.Namespace( + config=os.path.join(config_dir, 'app.yaml'), + base_image='gcr.io/google-appengine/python', + source_dir=config_dir, + ) + gen_dockerfile.gen_dockerfile(args) + + # Compare against golden files + golden_dir = os.path.join(self.testdata_dir, app + '_golden') + for filename in EXPECTED_OUTPUT_FILES: + with self.subTest(source='golden', filename=filename): + self.compare_file(filename, config_dir, golden_dir) + + # Copy sample app to different writable temp dir, and + # generate Dockerfile using gcloud. + if not shutil.which('gcloud'): + self.skipTest( + '"gcloud" tool not found in $PATH, skipping test') + gen_config_dir = os.path.join(temp_dir, 'gen_config') + shutil.copytree(app_dir, gen_config_dir) + app_yaml = os.path.join(gen_config_dir, 'app.yaml') + gcloud_args = [ + 'gcloud', '--quiet', 'beta', 'app', 'gen-config', + gen_config_dir, '--custom', '--config={}'.format(app_yaml) + ] + print('Invoking gcloud as {}'.format(gcloud_args)) + subprocess.check_call(gcloud_args) + for filename in EXPECTED_OUTPUT_FILES: + with self.subTest(source='gcloud', filename=filename): + self.compare_file(filename, config_dir, gen_config_dir) + + def test_parse_args(self): + valid_cases = ( + [], + ['argv0', '--base-image=nocolon'], + ['argv0', '--base-image=name:andcolon'], + ) + for argv in valid_cases: + with self.subTest(valid_argv=argv): + args = gen_dockerfile.parse_args(argv) + + def mock_error(*args): + """Prevent argparse from calling sys.exit()""" + raise AssertionError(*args) + + invalid_cases = ( + ['argv0', '--base-image=:noname'], + ['argv0', '--base-image=notag:'], + ['argv0', '--base-image=bad_:_:_multiple_colons'], + ) + for argv in invalid_cases: + with self.subTest(invalid_argv=argv): + with unittest.mock.patch.object( + argparse.ArgumentParser, 'error', mock_error): + with self.assertRaises(AssertionError): + gen_dockerfile.parse_args(argv) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/testdata/hello_world/app.yaml b/scripts/testdata/hello_world/app.yaml new file mode 100644 index 00000000..e5ac514e --- /dev/null +++ b/scripts/testdata/hello_world/app.yaml @@ -0,0 +1,6 @@ +runtime: python +env: flex +entrypoint: gunicorn -b :$PORT main:app + +runtime_config: + python_version: 3 diff --git a/scripts/testdata/hello_world/main.py b/scripts/testdata/hello_world/main.py new file mode 100644 index 00000000..97eb37d8 --- /dev/null +++ b/scripts/testdata/hello_world/main.py @@ -0,0 +1,43 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START app] +import logging + +from flask import Flask + + +app = Flask(__name__) + + +@app.route('/') +def hello(): + """Return a friendly HTTP greeting.""" + return 'Hello World!' + + +@app.errorhandler(500) +def server_error(e): + logging.exception('An error occurred during a request.') + return """ + An internal error occurred: 
{}
+ See logs for full stacktrace. + """.format(e), 500 + + +if __name__ == '__main__': + # This is used when running locally. Gunicorn is used to run the + # application on Google App Engine. See entrypoint in app.yaml. + app.run(host='127.0.0.1', port=8080, debug=True) +# [END app] diff --git a/scripts/testdata/hello_world/main_test.py b/scripts/testdata/hello_world/main_test.py new file mode 100644 index 00000000..4e230185 --- /dev/null +++ b/scripts/testdata/hello_world/main_test.py @@ -0,0 +1,24 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import main + + +def test_index(): + main.app.testing = True + client = main.app.test_client() + + r = client.get('/') + assert r.status_code == 200 + assert 'Hello World' in r.data.decode('utf-8') diff --git a/scripts/testdata/hello_world/requirements.txt b/scripts/testdata/hello_world/requirements.txt new file mode 100644 index 00000000..7861fb49 --- /dev/null +++ b/scripts/testdata/hello_world/requirements.txt @@ -0,0 +1,2 @@ +Flask==0.12 +gunicorn==19.6.0 diff --git a/scripts/testdata/hello_world_golden/.dockerignore b/scripts/testdata/hello_world_golden/.dockerignore new file mode 100644 index 00000000..8b927bb7 --- /dev/null +++ b/scripts/testdata/hello_world_golden/.dockerignore @@ -0,0 +1,19 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +.dockerignore +Dockerfile +.git +.hg +.svn diff --git a/scripts/testdata/hello_world_golden/Dockerfile b/scripts/testdata/hello_world_golden/Dockerfile new file mode 100644 index 00000000..e3e0563c --- /dev/null +++ b/scripts/testdata/hello_world_golden/Dockerfile @@ -0,0 +1,12 @@ +FROM gcr.io/google-appengine/python +LABEL python_version=python3.5 +RUN virtualenv --no-download /env -p python3.5 + +# Set virtualenv environment variables. This is equivalent to running +# source /env/bin/activate +ENV VIRTUAL_ENV /env +ENV PATH /env/bin:$PATH +ADD requirements.txt /app/ +RUN pip install -r requirements.txt +ADD . /app/ +CMD exec gunicorn -b :$PORT main:app From e4855c2c50f02704f67ee81b1b7e3db03af8acce Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Thu, 15 Jun 2017 18:23:31 -0700 Subject: [PATCH 069/256] Create a "runtime builder" image that builds other Docker images This builder will be called by the Google Cloud SDK during "gcloud app deploy" to create Docker images for AppEngine Flex applications. --- build.sh | 9 ++++++++ builder/gen-dockerfile/.gitignore | 2 ++ builder/gen-dockerfile/Dockerfile.in | 13 +++++++++++ builder/gen-dockerfile/requirements.txt | 1 + builder/python.yaml | 12 ++++++++++ cloudbuild.yaml | 30 +++++++++++++++---------- 6 files changed, 55 insertions(+), 12 deletions(-) create mode 100644 builder/gen-dockerfile/.gitignore create mode 100644 builder/gen-dockerfile/Dockerfile.in create mode 100644 builder/gen-dockerfile/requirements.txt create mode 100644 builder/python.yaml diff --git a/build.sh b/build.sh index 5929fbb2..0e01abcd 100755 --- a/build.sh +++ b/build.sh @@ -113,6 +113,7 @@ echo "Using base image name ${FULL_BASE_IMAGE}" # Generate Dockerfiles for outfile in \ + builder/gen-dockerfile/Dockerfile \ python-interpreter-builder/Dockerfile \ runtime-image/Dockerfile \ tests/benchmark/Dockerfile \ @@ -123,6 +124,14 @@ for outfile in \ envsubst <"${outfile}".in >"${outfile}" '$DEBIAN_BASE_IMAGE $FULL_BASE_IMAGE $GOOGLE_CLOUD_PROJECT_FOR_TESTS' done +# Make some files available to the runtime builder Docker context +for file in \ + gen_dockerfile.py \ + validation_utils.py \ + ; do + cp -a "scripts/${file}" "builder/gen-dockerfile/${file}" +done + # Build images and push to GCR if [ "${build}" -eq 1 ]; then echo "Building images" diff --git a/builder/gen-dockerfile/.gitignore b/builder/gen-dockerfile/.gitignore new file mode 100644 index 00000000..d8ffddc3 --- /dev/null +++ b/builder/gen-dockerfile/.gitignore @@ -0,0 +1,2 @@ +Dockerfile +*.py diff --git a/builder/gen-dockerfile/Dockerfile.in b/builder/gen-dockerfile/Dockerfile.in new file mode 100644 index 00000000..4a8de5ec --- /dev/null +++ b/builder/gen-dockerfile/Dockerfile.in @@ -0,0 +1,13 @@ +FROM ${FULL_BASE_IMAGE} +LABEL python_version=python3.5 +RUN virtualenv --no-download /env -p python3.5 + +# Set virtualenv environment variables. This is equivalent to running +# source /env/bin/activate +ENV VIRTUAL_ENV /env +ENV PATH /env/bin:$PATH +ADD requirements.txt /builder/ +RUN pip install -r /builder/requirements.txt +ADD . /builder/ +WORKDIR /workspace +ENTRYPOINT [ "python", "/builder/gen_dockerfile.py" ] diff --git a/builder/gen-dockerfile/requirements.txt b/builder/gen-dockerfile/requirements.txt new file mode 100644 index 00000000..c3726e8b --- /dev/null +++ b/builder/gen-dockerfile/requirements.txt @@ -0,0 +1 @@ +pyyaml diff --git a/builder/python.yaml b/builder/python.yaml new file mode 100644 index 00000000..3e75acad --- /dev/null +++ b/builder/python.yaml @@ -0,0 +1,12 @@ +# This is a cloudbuild.yaml template for the runtime builder +steps: +- # Generate application Dockerfile + name: 'gcr.io/gcp-runtimes/python/gen-dockerfile:latest' + args: [ + '--base-image=gcr.io/google-appengine/python:latest' + ] +- # Use that Dockerfile to create final application image + name: 'gcr.io/cloud-builders/docker:latest' + args: ['build', '-t', '$_OUTPUT_IMAGE', '.'] +images: + - '$_OUTPUT_IMAGE' diff --git a/cloudbuild.yaml b/cloudbuild.yaml index f242ac8e..3f52e41d 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -1,18 +1,18 @@ timeout: 7200s steps: -- name: gcr.io/cloud-builders/docker:latest - # Compile Python interpreters from source +- # Compile Python interpreters from source + name: gcr.io/cloud-builders/docker:latest args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/interpreter-builder:${_TAG}', '--no-cache', '/workspace/python-interpreter-builder/'] -- name: ${_DOCKER_NAMESPACE}/python/interpreter-builder:${_TAG} - # Copy interpreters back to workspace +- # Copy interpreters back to workspace + name: ${_DOCKER_NAMESPACE}/python/interpreter-builder:${_TAG} args: ['cp', '/interpreters.tar.gz', '/workspace/runtime-image/interpreters.tar.gz'] -- name: gcr.io/cloud-builders/docker:latest - # Build base runtime image +- # Build base runtime image + name: gcr.io/cloud-builders/docker:latest args: ['build', '--tag=${_DOCKER_NAMESPACE}/python:${_TAG}', '--no-cache', '/workspace/runtime-image/'] -- name: gcr.io/gcp-runtimes/structure_test:latest - # Validate structure of base runtime image +- # Validate structure of base runtime image + name: gcr.io/gcp-runtimes/structure_test:latest args: [ '-i', '${_DOCKER_NAMESPACE}/python:${_TAG}', '--config', '/workspace/tests/virtualenv/virtualenv_default.yaml', @@ -25,11 +25,17 @@ steps: '--config', '/workspace/tests/license-test/license-test.yaml', '-v' ] -- name: gcr.io/cloud-builders/docker:latest - # Run google client library unit tests against base runtime image +- # Build image to run google client library unit tests + name: gcr.io/cloud-builders/docker:latest args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/google-cloud-python:${_TAG}', '--no-cache', '/workspace/tests/google-cloud-python/'] -- name: ${_DOCKER_NAMESPACE}/python/tests/google-cloud-python:${_TAG} +- # Run google client library unit tests + name: ${_DOCKER_NAMESPACE}/python/tests/google-cloud-python:${_TAG} +- # Build runtime builder image + name: gcr.io/cloud-builders/docker:latest + args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/builder/gen-dockerfile:${_TAG}', + '--no-cache', '/workspace/builder/gen-dockerfile/'] images: [ - '${_DOCKER_NAMESPACE}/python:${_TAG}' + '${_DOCKER_NAMESPACE}/python:${_TAG}', + '${_DOCKER_NAMESPACE}/python/builder/gen-dockerfile:${_TAG}', ] From b67eff102c2e3c4dddcf5bb2e7d757691d5628d6 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 16 Jun 2017 14:18:36 -0700 Subject: [PATCH 070/256] Rename cloudbuild.yaml template file. --- builder/{python.yaml => python-latest.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename builder/{python.yaml => python-latest.yaml} (100%) diff --git a/builder/python.yaml b/builder/python-latest.yaml similarity index 100% rename from builder/python.yaml rename to builder/python-latest.yaml From 1091d19ff841e095f366a85d5ba321d04aa62999 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 16 Jun 2017 14:24:37 -0700 Subject: [PATCH 071/256] Fix shebang lines --- scripts/gen_dockerfile.py | 2 +- scripts/gen_dockerfile_test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/gen_dockerfile.py b/scripts/gen_dockerfile.py index 07d2f8c5..77c91e5b 100755 --- a/scripts/gen_dockerfile.py +++ b/scripts/gen_dockerfile.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # Copyright 2017 Google Inc. All Rights Reserved. # diff --git a/scripts/gen_dockerfile_test.py b/scripts/gen_dockerfile_test.py index faf44fc7..56655159 100755 --- a/scripts/gen_dockerfile_test.py +++ b/scripts/gen_dockerfile_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # Copyright 2017 Google Inc. All Rights Reserved. # From a42b40e67e5a85914e0a97d3de8770f2a79cd840 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 16 Jun 2017 14:46:14 -0700 Subject: [PATCH 072/256] Support Docker images names with sha256 digests --- scripts/gen_dockerfile.py | 14 ++++++-------- scripts/gen_dockerfile_test.py | 5 +++-- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/scripts/gen_dockerfile.py b/scripts/gen_dockerfile.py index 77c91e5b..2637a5f5 100755 --- a/scripts/gen_dockerfile.py +++ b/scripts/gen_dockerfile.py @@ -29,16 +29,14 @@ import validation_utils -# Validate characters for dockerfile image names -# https://github.com/docker/docker/blob/master/api/names.go +# Validate characters for dockerfile image names. +# +# This roots out obvious mistakes, the full gory details are here: +# https://github.com/docker/distribution/blob/master/reference/regexp.go IMAGE_REGEX = re.compile(r"""(?x) ^ - [a-zA-Z0-9] # First char must be alphanumeric - [-a-zA-Z0-9_./]* # Punctuation allowed after that - ( - : # At most one colon allowed - [-a-zA-Z0-9_./]+ # Colon must be followed by other chars - )? + [a-zA-Z0-9] # First char must be alphanumeric + [a-zA-Z0-9-_./:@+]* # Punctuation allowed after that $ """) diff --git a/scripts/gen_dockerfile_test.py b/scripts/gen_dockerfile_test.py index 56655159..4a99c51d 100755 --- a/scripts/gen_dockerfile_test.py +++ b/scripts/gen_dockerfile_test.py @@ -201,6 +201,7 @@ def test_parse_args(self): [], ['argv0', '--base-image=nocolon'], ['argv0', '--base-image=name:andcolon'], + ['argv0', '--base-image=name@sha256:digest'], ) for argv in valid_cases: with self.subTest(valid_argv=argv): @@ -211,9 +212,9 @@ def mock_error(*args): raise AssertionError(*args) invalid_cases = ( + ['argv0', '--base-image='], + ['argv0', '--base-image=:'], ['argv0', '--base-image=:noname'], - ['argv0', '--base-image=notag:'], - ['argv0', '--base-image=bad_:_:_multiple_colons'], ) for argv in invalid_cases: with self.subTest(invalid_argv=argv): From e5dab4cde5144edaa4e2a27e6ba9bb441c4fa2bf Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 16 Jun 2017 14:56:38 -0700 Subject: [PATCH 073/256] Fix some comments --- scripts/gen_dockerfile.py | 4 ++-- scripts/gen_dockerfile_test.py | 2 +- scripts/local_cloudbuild.py | 2 +- scripts/local_cloudbuild_test.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/gen_dockerfile.py b/scripts/gen_dockerfile.py index 2637a5f5..b02684fb 100755 --- a/scripts/gen_dockerfile.py +++ b/scripts/gen_dockerfile.py @@ -113,7 +113,7 @@ def get_app_config(raw_config, args): Args: raw_config (dict): deserialized app.yaml - args (argparse.Namespace): ccommand line flags + args (argparse.Namespace): command line flags Returns: AppConfig: valid configuration @@ -195,7 +195,7 @@ def gen_dockerfile(args): """Write a Dockerfile and helper files for an application. Args: - args (argparse.Namespace): ccommand line flags + args (argparse.Namespace): command line flags """ # Read yaml file. Does not currently support multiple services # with configuration filenames besides app.yaml diff --git a/scripts/gen_dockerfile_test.py b/scripts/gen_dockerfile_test.py index 4a99c51d..e8ad371d 100755 --- a/scripts/gen_dockerfile_test.py +++ b/scripts/gen_dockerfile_test.py @@ -35,7 +35,7 @@ class GenDockerfileTest(unittest.TestCase): def setUp(self): - self.testdata_dir = os.path.join(os.path.dirname(__file__), 'testdata') # Sigh + self.testdata_dir = os.path.join(os.path.dirname(__file__), 'testdata') assert os.path.isdir(self.testdata_dir), 'Could not run test: testdata directory not found' def compare_file(self, filename, dir1, dir2): diff --git a/scripts/local_cloudbuild.py b/scripts/local_cloudbuild.py index f0c6659d..a7c5eda9 100755 --- a/scripts/local_cloudbuild.py +++ b/scripts/local_cloudbuild.py @@ -153,7 +153,7 @@ def get_cloudbuild(raw_config, args): Args: raw_config (dict): deserialized cloudbuild.yaml - args (argparse.Namespace): ccommand line flags + args (argparse.Namespace): command line flags Returns: CloudBuild: valid configuration diff --git a/scripts/local_cloudbuild_test.py b/scripts/local_cloudbuild_test.py index 68a7ff27..fff089dc 100755 --- a/scripts/local_cloudbuild_test.py +++ b/scripts/local_cloudbuild_test.py @@ -37,7 +37,7 @@ class LocalCloudbuildTest(unittest.TestCase): def setUp(self): - self.testdata_dir = os.path.join(os.path.dirname(__file__), 'testdata') # Sigh + self.testdata_dir = os.path.join(os.path.dirname(__file__), 'testdata') assert os.path.isdir(self.testdata_dir), 'Could not run test: testdata directory not found' def test_sub_and_quote(self): From af2f40ff58a8b0c8994dae7bf13ffd4b881b2d98 Mon Sep 17 00:00:00 2001 From: Nick Kubala Date: Fri, 16 Jun 2017 16:09:27 -0700 Subject: [PATCH 074/256] add simple application for running deploy health check --- tests/health_check/app.yaml | 3 ++ tests/health_check/main.py | 43 +++++++++++++++++++++++++++++ tests/health_check/requirements.txt | 2 ++ 3 files changed, 48 insertions(+) create mode 100644 tests/health_check/app.yaml create mode 100644 tests/health_check/main.py create mode 100644 tests/health_check/requirements.txt diff --git a/tests/health_check/app.yaml b/tests/health_check/app.yaml new file mode 100644 index 00000000..c4a838e6 --- /dev/null +++ b/tests/health_check/app.yaml @@ -0,0 +1,3 @@ +runtime: python +env: flex +entrypoint: gunicorn -b :$PORT main:app diff --git a/tests/health_check/main.py b/tests/health_check/main.py new file mode 100644 index 00000000..97eb37d8 --- /dev/null +++ b/tests/health_check/main.py @@ -0,0 +1,43 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START app] +import logging + +from flask import Flask + + +app = Flask(__name__) + + +@app.route('/') +def hello(): + """Return a friendly HTTP greeting.""" + return 'Hello World!' + + +@app.errorhandler(500) +def server_error(e): + logging.exception('An error occurred during a request.') + return """ + An internal error occurred: 
{}
+ See logs for full stacktrace. + """.format(e), 500 + + +if __name__ == '__main__': + # This is used when running locally. Gunicorn is used to run the + # application on Google App Engine. See entrypoint in app.yaml. + app.run(host='127.0.0.1', port=8080, debug=True) +# [END app] diff --git a/tests/health_check/requirements.txt b/tests/health_check/requirements.txt new file mode 100644 index 00000000..e7676bba --- /dev/null +++ b/tests/health_check/requirements.txt @@ -0,0 +1,2 @@ +Flask==0.12.1 +gunicorn==19.7.1 From 0feceedcdfefc972d7c31bc9ba2a38e71f5bfc9b Mon Sep 17 00:00:00 2001 From: Nick Kubala Date: Fri, 16 Jun 2017 16:12:23 -0700 Subject: [PATCH 075/256] health_check -> deploy_check --- tests/{health_check => deploy_check}/app.yaml | 0 tests/{health_check => deploy_check}/main.py | 0 tests/{health_check => deploy_check}/requirements.txt | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename tests/{health_check => deploy_check}/app.yaml (100%) rename tests/{health_check => deploy_check}/main.py (100%) rename tests/{health_check => deploy_check}/requirements.txt (100%) diff --git a/tests/health_check/app.yaml b/tests/deploy_check/app.yaml similarity index 100% rename from tests/health_check/app.yaml rename to tests/deploy_check/app.yaml diff --git a/tests/health_check/main.py b/tests/deploy_check/main.py similarity index 100% rename from tests/health_check/main.py rename to tests/deploy_check/main.py diff --git a/tests/health_check/requirements.txt b/tests/deploy_check/requirements.txt similarity index 100% rename from tests/health_check/requirements.txt rename to tests/deploy_check/requirements.txt From 2d172e042218c926f367344b4a6064984717401b Mon Sep 17 00:00:00 2001 From: Nick Kubala Date: Mon, 19 Jun 2017 13:29:20 -0700 Subject: [PATCH 076/256] rename FULL_BASE_IMAGE to STAGING_IMAGE to match integration test framework --- build.sh | 6 +++--- tests/benchmark/Dockerfile.in | 2 +- tests/benchmark/benchmark_between_releases.sh | 8 ++++---- tests/google-cloud-python-system/Dockerfile.in | 2 +- tests/google-cloud-python/Dockerfile.in | 2 +- tests/integration/Dockerfile.in | 2 +- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/build.sh b/build.sh index 5929fbb2..b7796dd6 100755 --- a/build.sh +++ b/build.sh @@ -108,8 +108,8 @@ fi # Use latest released Debian as our base image export DEBIAN_BASE_IMAGE="gcr.io/google-appengine/debian8:latest" -export FULL_BASE_IMAGE="${DOCKER_NAMESPACE}/python:${TAG}" -echo "Using base image name ${FULL_BASE_IMAGE}" +export STAGING_IMAGE="${DOCKER_NAMESPACE}/python:${TAG}" +echo "Using base image name ${STAGING_IMAGE}" # Generate Dockerfiles for outfile in \ @@ -120,7 +120,7 @@ for outfile in \ tests/google-cloud-python-system/Dockerfile \ tests/integration/Dockerfile \ ; do - envsubst <"${outfile}".in >"${outfile}" '$DEBIAN_BASE_IMAGE $FULL_BASE_IMAGE $GOOGLE_CLOUD_PROJECT_FOR_TESTS' + envsubst <"${outfile}".in >"${outfile}" '$DEBIAN_BASE_IMAGE $STAGING_IMAGE $GOOGLE_CLOUD_PROJECT_FOR_TESTS' done # Build images and push to GCR diff --git a/tests/benchmark/Dockerfile.in b/tests/benchmark/Dockerfile.in index 6166cdd9..ca0edc24 100644 --- a/tests/benchmark/Dockerfile.in +++ b/tests/benchmark/Dockerfile.in @@ -1,4 +1,4 @@ -FROM ${FULL_BASE_IMAGE} +FROM ${STAGING_IMAGE} # Install performance RUN pip install performance diff --git a/tests/benchmark/benchmark_between_releases.sh b/tests/benchmark/benchmark_between_releases.sh index f63a2877..0e105cf3 100755 --- a/tests/benchmark/benchmark_between_releases.sh +++ b/tests/benchmark/benchmark_between_releases.sh @@ -2,15 +2,15 @@ # Build the benchmark image for release 1 from Dockerfile echo "Building image for release 1" -export FULL_BASE_IMAGE="${DOCKER_NAMESPACE}/python:${TAG1}" -envsubst <"Dockerfile".in >"Dockerfile" '$FULL_BASE_IMAGE' +export STAGING_IMAGE="${DOCKER_NAMESPACE}/python:${TAG1}" +envsubst <"Dockerfile".in >"Dockerfile" '$STAGING_IMAGE' docker build --no-cache -t benchmark_1 . rm Dockerfile # Build the benchmark image for release 2 from Dockerfile echo "Building image for release 2" -export FULL_BASE_IMAGE="${DOCKER_NAMESPACE}/python:${TAG2}" -envsubst <"Dockerfile".in >"Dockerfile" '$FULL_BASE_IMAGE' +export STAGING_IMAGE="${DOCKER_NAMESPACE}/python:${TAG2}" +envsubst <"Dockerfile".in >"Dockerfile" '$STAGING_IMAGE' docker build --no-cache -t benchmark_2 . rm Dockerfile diff --git a/tests/google-cloud-python-system/Dockerfile.in b/tests/google-cloud-python-system/Dockerfile.in index 18506681..04f28bd5 100644 --- a/tests/google-cloud-python-system/Dockerfile.in +++ b/tests/google-cloud-python-system/Dockerfile.in @@ -1,4 +1,4 @@ -FROM ${FULL_BASE_IMAGE} +FROM ${STAGING_IMAGE} # Get the source. RUN git clone --depth 1 https://github.com/GoogleCloudPlatform/google-cloud-python.git diff --git a/tests/google-cloud-python/Dockerfile.in b/tests/google-cloud-python/Dockerfile.in index ada429fb..a2873863 100644 --- a/tests/google-cloud-python/Dockerfile.in +++ b/tests/google-cloud-python/Dockerfile.in @@ -1,4 +1,4 @@ -FROM ${FULL_BASE_IMAGE} +FROM ${STAGING_IMAGE} # Get the source. RUN git clone --depth 1 https://github.com/GoogleCloudPlatform/google-cloud-python.git diff --git a/tests/integration/Dockerfile.in b/tests/integration/Dockerfile.in index 52f848f6..e18d60ef 100644 --- a/tests/integration/Dockerfile.in +++ b/tests/integration/Dockerfile.in @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM ${FULL_BASE_IMAGE} +FROM ${STAGING_IMAGE} COPY . /app WORKDIR /app From cb1b7a58503c6b90c14cca877fbdd51a0d16ad7c Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Mon, 19 Jun 2017 15:03:35 -0700 Subject: [PATCH 077/256] Explicitly pass individual arg values instead of argparse object. --- scripts/gen_dockerfile.py | 42 ++++++++++++++++++---------------- scripts/gen_dockerfile_test.py | 23 ++++++++++--------- 2 files changed, 34 insertions(+), 31 deletions(-) diff --git a/scripts/gen_dockerfile.py b/scripts/gen_dockerfile.py index b02684fb..071b1cd1 100755 --- a/scripts/gen_dockerfile.py +++ b/scripts/gen_dockerfile.py @@ -108,12 +108,14 @@ ) -def get_app_config(raw_config, args): +def get_app_config(raw_config, base_image, config_file, source_dir): """Read and validate the application runtime configuration. Args: raw_config (dict): deserialized app.yaml - args (argparse.Namespace): command line flags + base_image (str): Docker image name to build on top of + config_file (str): Path to user's app.yaml (might be .yaml) + source_dir (str): Directory container user's source code Returns: AppConfig: valid configuration @@ -122,7 +124,7 @@ def get_app_config(raw_config, args): if not isinstance(raw_config, dict): raise ValueError( 'Expected {} contents to be of type "dict", but found type "{}"'. - format(args.config, type(raw_config))) + format(config_file, type(raw_config))) entrypoint = validation_utils.get_field_value(raw_config, 'entrypoint', str) if not PRINTABLE_REGEX.match(entrypoint): @@ -139,10 +141,7 @@ def get_app_config(raw_config, args): # Examine user's files has_requirements_txt = os.path.isfile( - os.path.join(args.source_dir, 'requirements.txt')) - - # Compute base image name and Dockerfile contents - base_image = args.base_image + os.path.join(source_dir, 'requirements.txt')) return AppConfig( base_image=base_image, @@ -191,26 +190,29 @@ def generate_files(app_config): } -def gen_dockerfile(args): +def gen_dockerfile(base_image, config_file, source_dir): """Write a Dockerfile and helper files for an application. Args: - args (argparse.Namespace): command line flags + base_image (str): Docker image name to build on top of + config_file (str): Path to user's app.yaml (might be .yaml) + source_dir (str): Directory container user's source code """ # Read yaml file. Does not currently support multiple services # with configuration filenames besides app.yaml - with io.open(args.config, 'r', encoding='utf8') as yaml_config_file: + with io.open(config_file, 'r', encoding='utf8') as yaml_config_file: raw_config = yaml.load(yaml_config_file) - # Determine configuration - app_config = get_app_config(raw_config, args) + # Determine complete configuration + app_config = get_app_config(raw_config, base_image, config_file, + source_dir) # Generate list of filenames and their textual contents files = generate_files(app_config) # Write files for filename, contents in files.items(): - full_filename = os.path.join(args.source_dir, filename) + full_filename = os.path.join(source_dir, filename) with io.open(full_filename, 'w', encoding='utf8') as outfile: outfile.write(contents) @@ -218,6 +220,12 @@ def gen_dockerfile(args): def parse_args(argv): """Parse and validate command line flags""" parser = argparse.ArgumentParser() + parser.add_argument( + '--base-image', + type=functools.partial( + validation_utils.validate_arg_regex, flag_regex=IMAGE_REGEX), + default='gcr.io/google-appengine/python:latest', + help='Name of Docker image to use as base') parser.add_argument( '--config', type=functools.partial( @@ -225,12 +233,6 @@ def parse_args(argv): default='app.yaml', help='Path to application configuration file' ) - parser.add_argument( - '--base-image', - type=functools.partial( - validation_utils.validate_arg_regex, flag_regex=IMAGE_REGEX), - default='gcr.io/google-appengine/python:latest', - help='Name of Docker image to use as base') parser.add_argument( '--source-dir', type=functools.partial( @@ -243,7 +245,7 @@ def parse_args(argv): def main(): args = parse_args(sys.argv) - gen_dockerfile(args) + gen_dockerfile(args.base_image, args.config, args.source_dir) if __name__ == '__main__': diff --git a/scripts/gen_dockerfile_test.py b/scripts/gen_dockerfile_test.py index e8ad371d..8cbbd1d6 100755 --- a/scripts/gen_dockerfile_test.py +++ b/scripts/gen_dockerfile_test.py @@ -49,10 +49,9 @@ def compare_file(self, filename, dir1, dir2): self.assertMultiLineEqual(contents1, contents2, msg) def test_get_app_config(self): - args = argparse.Namespace( - config='some_config_file', - base_image='some_image_name', - source_dir='some_source_dir') + config_file = 'some_config_file' + base_image = 'some_image_name' + source_dir = 'some_source_dir' valid_cases = ( # Basic app.yaml @@ -93,7 +92,9 @@ def test_get_app_config(self): raw_app_config = yaml.safe_load(app_yaml) with unittest.mock.patch.object( os.path, 'isfile', return_value=isfile): - actual = gen_dockerfile.get_app_config(raw_app_config, args) + actual = gen_dockerfile.get_app_config( + raw_app_config, base_image, config_file, + source_dir) for key, value in expected.items(): self.assertEqual(getattr(actual, key), value) @@ -110,7 +111,9 @@ def test_get_app_config(self): with self.subTest(invalid_case=invalid_case): raw_app_config = yaml.safe_load(invalid_case) with self.assertRaises(ValueError): - gen_dockerfile.get_app_config(raw_app_config, args) + gen_dockerfile.get_app_config( + raw_app_config, base_image, config_file, + source_dir) def test_generate_files(self): base = gen_dockerfile.AppConfig( @@ -165,12 +168,10 @@ def test_gen_dockerfile(self): # Copy sample app to writable temp dir, and generate Dockerfile. config_dir = os.path.join(temp_dir, 'config') shutil.copytree(app_dir, config_dir) - args = argparse.Namespace( - config=os.path.join(config_dir, 'app.yaml'), + gen_dockerfile.gen_dockerfile( base_image='gcr.io/google-appengine/python', - source_dir=config_dir, - ) - gen_dockerfile.gen_dockerfile(args) + config_file=os.path.join(config_dir, 'app.yaml'), + source_dir=config_dir) # Compare against golden files golden_dir = os.path.join(self.testdata_dir, app + '_golden') From 6e4a63d36de781ad8a0d1e7384ccf84a593128eb Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Mon, 19 Jun 2017 15:23:51 -0700 Subject: [PATCH 078/256] Pin package version --- builder/gen-dockerfile/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/gen-dockerfile/requirements.txt b/builder/gen-dockerfile/requirements.txt index c3726e8b..59bc593a 100644 --- a/builder/gen-dockerfile/requirements.txt +++ b/builder/gen-dockerfile/requirements.txt @@ -1 +1 @@ -pyyaml +PyYAML==3.12 From 4ca86d15d6887e8ff5678b7d5d05400ff9d188f2 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Mon, 19 Jun 2017 15:24:24 -0700 Subject: [PATCH 079/256] Use filecmp module instead of handrolled comparison --- scripts/gen_dockerfile.py | 2 ++ scripts/gen_dockerfile_test.py | 13 ++++++------- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/scripts/gen_dockerfile.py b/scripts/gen_dockerfile.py index 071b1cd1..e9debe83 100755 --- a/scripts/gen_dockerfile.py +++ b/scripts/gen_dockerfile.py @@ -129,8 +129,10 @@ def get_app_config(raw_config, base_image, config_file, source_dir): entrypoint = validation_utils.get_field_value(raw_config, 'entrypoint', str) if not PRINTABLE_REGEX.match(entrypoint): raise ValueError('Invalid character in "entrypoint" field of app.yaml') + raw_runtime_config = validation_utils.get_field_value(raw_config, 'runtime_config', dict) python_version = validation_utils.get_field_value(raw_runtime_config, 'python_version', str) + dockerfile_python_version = PYTHON_INTERPRETER_VERSION_MAP.get( python_version) if dockerfile_python_version is None: diff --git a/scripts/gen_dockerfile_test.py b/scripts/gen_dockerfile_test.py index 8cbbd1d6..9ba439aa 100755 --- a/scripts/gen_dockerfile_test.py +++ b/scripts/gen_dockerfile_test.py @@ -17,6 +17,7 @@ """Unit test for gen_dockerfile.py""" import argparse +import filecmp import os import re import shutil @@ -40,13 +41,11 @@ def setUp(self): def compare_file(self, filename, dir1, dir2): """Compare identically named files in two different directories""" - with open(os.path.join(dir1, filename), 'r', encoding='utf8') as file1: - with open(os.path.join(dir2, filename), 'r', encoding='utf8') as file2: - contents1 = file1.read() - contents2 = file2.read() - msg = 'Contents of "{}" differ between "{}" and "{}"'.format( - filename, dir1, dir2) - self.assertMultiLineEqual(contents1, contents2, msg) + if not filecmp.cmp(os.path.join(dir1, filename), + os.path.join(dir2, filename)): + msg = 'Contents of "{}" differ between "{}" and "{}"'.format( + filename, dir1, dir2) + self.assertMultiLineEqual(contents1, contents2, msg) def test_get_app_config(self): config_file = 'some_config_file' From ad9a940bdf41bc433d2dbc262f77f659fd9a3cf6 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Mon, 19 Jun 2017 15:49:24 -0700 Subject: [PATCH 080/256] Fix style nit --- scripts/gen_dockerfile.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/gen_dockerfile.py b/scripts/gen_dockerfile.py index e9debe83..abb891bc 100755 --- a/scripts/gen_dockerfile.py +++ b/scripts/gen_dockerfile.py @@ -137,9 +137,9 @@ def get_app_config(raw_config, base_image, config_file, source_dir): python_version) if dockerfile_python_version is None: valid_versions = str(sorted(PYTHON_INTERPRETER_VERSION_MAP.keys())) - msg = ('Invalid "python_version" field in "runtime_config" section ' - 'of app.yaml. Valid options are:\n{}').format(valid_versions) - raise ValueError(msg) + raise ValueError( + 'Invalid "python_version" field in "runtime_config" section ' + 'of app.yaml. Valid options are:\n{}'.format(valid_versions)) # Examine user's files has_requirements_txt = os.path.isfile( From 336e42a16fd9cb29c4eff3cd69d85686edadad6c Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Mon, 19 Jun 2017 16:07:13 -0700 Subject: [PATCH 081/256] Rename function so it doesn't shadow module name. --- scripts/gen_dockerfile.py | 4 ++-- scripts/gen_dockerfile_test.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/gen_dockerfile.py b/scripts/gen_dockerfile.py index abb891bc..bdbf1802 100755 --- a/scripts/gen_dockerfile.py +++ b/scripts/gen_dockerfile.py @@ -192,7 +192,7 @@ def generate_files(app_config): } -def gen_dockerfile(base_image, config_file, source_dir): +def generate_dockerfile_command(base_image, config_file, source_dir): """Write a Dockerfile and helper files for an application. Args: @@ -247,7 +247,7 @@ def parse_args(argv): def main(): args = parse_args(sys.argv) - gen_dockerfile(args.base_image, args.config, args.source_dir) + generate_dockerfile_command(args.base_image, args.config, args.source_dir) if __name__ == '__main__': diff --git a/scripts/gen_dockerfile_test.py b/scripts/gen_dockerfile_test.py index 9ba439aa..92f70061 100755 --- a/scripts/gen_dockerfile_test.py +++ b/scripts/gen_dockerfile_test.py @@ -151,7 +151,7 @@ def test_generate_files(self): else: self.assertNotIn(test_string, dockerfile) - def test_gen_dockerfile(self): + def test_generate_dockerfile_command(self): """Generates output and compares against a set of golden files. Optionally runs 'gcloud app gen-config' and compares against that. @@ -167,7 +167,7 @@ def test_gen_dockerfile(self): # Copy sample app to writable temp dir, and generate Dockerfile. config_dir = os.path.join(temp_dir, 'config') shutil.copytree(app_dir, config_dir) - gen_dockerfile.gen_dockerfile( + gen_dockerfile.generate_dockerfile_command( base_image='gcr.io/google-appengine/python', config_file=os.path.join(config_dir, 'app.yaml'), source_dir=config_dir) From 881196ac012829edccc5e9544d8dd88f4c3e8a76 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Wed, 21 Jun 2017 13:43:46 -0700 Subject: [PATCH 082/256] Move file templates into separate files. The templates were copied from the gae-xrt-python template files in the gcloud SDK, with the same names and contents, except for Dockerfile.preamble which no longer hardcodes the base image, and Dockerfile.entrypoint.template which is new. --- build.sh | 1 + scripts/data/Dockerfile.entrypoint.template | 1 + scripts/data/Dockerfile.install_app | 1 + scripts/data/Dockerfile.preamble.template | 1 + scripts/data/Dockerfile.requirements_txt | 2 + scripts/data/Dockerfile.virtualenv.template | 7 ++ scripts/data/dockerignore | 19 +++++ scripts/gen_dockerfile.py | 88 +++++++-------------- scripts/gen_dockerfile_test.py | 4 +- 9 files changed, 61 insertions(+), 63 deletions(-) create mode 100644 scripts/data/Dockerfile.entrypoint.template create mode 100644 scripts/data/Dockerfile.install_app create mode 100644 scripts/data/Dockerfile.preamble.template create mode 100644 scripts/data/Dockerfile.requirements_txt create mode 100644 scripts/data/Dockerfile.virtualenv.template create mode 100644 scripts/data/dockerignore diff --git a/build.sh b/build.sh index 0e01abcd..7d20ef03 100755 --- a/build.sh +++ b/build.sh @@ -128,6 +128,7 @@ done for file in \ gen_dockerfile.py \ validation_utils.py \ + 'data/*' \ ; do cp -a "scripts/${file}" "builder/gen-dockerfile/${file}" done diff --git a/scripts/data/Dockerfile.entrypoint.template b/scripts/data/Dockerfile.entrypoint.template new file mode 100644 index 00000000..f6b52bf6 --- /dev/null +++ b/scripts/data/Dockerfile.entrypoint.template @@ -0,0 +1 @@ +CMD {entrypoint} diff --git a/scripts/data/Dockerfile.install_app b/scripts/data/Dockerfile.install_app new file mode 100644 index 00000000..54c3d6cc --- /dev/null +++ b/scripts/data/Dockerfile.install_app @@ -0,0 +1 @@ +ADD . /app/ diff --git a/scripts/data/Dockerfile.preamble.template b/scripts/data/Dockerfile.preamble.template new file mode 100644 index 00000000..e4d005bd --- /dev/null +++ b/scripts/data/Dockerfile.preamble.template @@ -0,0 +1 @@ +FROM {base_image} diff --git a/scripts/data/Dockerfile.requirements_txt b/scripts/data/Dockerfile.requirements_txt new file mode 100644 index 00000000..f684c45c --- /dev/null +++ b/scripts/data/Dockerfile.requirements_txt @@ -0,0 +1,2 @@ +ADD requirements.txt /app/ +RUN pip install -r requirements.txt diff --git a/scripts/data/Dockerfile.virtualenv.template b/scripts/data/Dockerfile.virtualenv.template new file mode 100644 index 00000000..557b1992 --- /dev/null +++ b/scripts/data/Dockerfile.virtualenv.template @@ -0,0 +1,7 @@ +LABEL python_version=python{python_version} +RUN virtualenv --no-download /env -p python{python_version} + +# Set virtualenv environment variables. This is equivalent to running +# source /env/bin/activate +ENV VIRTUAL_ENV /env +ENV PATH /env/bin:$PATH diff --git a/scripts/data/dockerignore b/scripts/data/dockerignore new file mode 100644 index 00000000..8b927bb7 --- /dev/null +++ b/scripts/data/dockerignore @@ -0,0 +1,19 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +.dockerignore +Dockerfile +.git +.hg +.svn diff --git a/scripts/gen_dockerfile.py b/scripts/gen_dockerfile.py index bdbf1802..30004ead 100755 --- a/scripts/gen_dockerfile.py +++ b/scripts/gen_dockerfile.py @@ -56,50 +56,6 @@ '3.6': '3.6', } -# File templates. -# Designed to exactly match the current output of 'gcloud app gen-config' -DOCKERFILE_TEMPLATE = """\ -FROM {base_image} -LABEL python_version=python{dockerfile_python_version} -RUN virtualenv --no-download /env -p python{dockerfile_python_version} - -# Set virtualenv environment variables. This is equivalent to running -# source /env/bin/activate -ENV VIRTUAL_ENV /env -ENV PATH /env/bin:$PATH -{optional_requirements_txt}ADD . /app/ -{optional_entrypoint}""" - -DOCKERFILE_REQUIREMENTS_TXT = """\ -ADD requirements.txt /app/ -RUN pip install -r requirements.txt -""" - -DOCKERFILE_ENTRYPOINT_TEMPLATE = """\ -CMD {entrypoint} -""" - -DOCKERIGNORE = """\ -# Copyright 2015 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -.dockerignore -Dockerfile -.git -.hg -.svn -""" # Validated application configuration AppConfig = collections.namedtuple( @@ -130,6 +86,14 @@ def get_app_config(raw_config, base_image, config_file, source_dir): if not PRINTABLE_REGEX.match(entrypoint): raise ValueError('Invalid character in "entrypoint" field of app.yaml') + # Mangle entrypoint in the same way as the Cloud SDK + # (googlecloudsdk/third_party/appengine/api/validation.py) + # + # We could handle both string ("shell form") and list ("exec + # form") but it appears that gcloud only handles string form. + if entrypoint and not entrypoint.startswith('exec '): + entrypoint = 'exec ' + entrypoint + raw_runtime_config = validation_utils.get_field_value(raw_config, 'runtime_config', dict) python_version = validation_utils.get_field_value(raw_runtime_config, 'python_version', str) @@ -152,6 +116,13 @@ def get_app_config(raw_config, base_image, config_file, source_dir): has_requirements_txt=has_requirements_txt) +def get_data(name): + """Return the contents of the named data resource""" + filename = os.path.join(os.path.dirname(__file__), 'data', name) + with io.open(filename, 'r', encoding='utf8') as template_file: + return template_file.read() + + def generate_files(app_config): """Generate a Dockerfile and helper files for an application. @@ -162,33 +133,28 @@ def generate_files(app_config): dict: Map of filename to desired file contents """ if app_config.has_requirements_txt: - optional_requirements_txt = DOCKERFILE_REQUIREMENTS_TXT + optional_requirements_txt = get_data('Dockerfile.requirements_txt') else: optional_requirements_txt = '' if app_config.entrypoint: - # Mangle entrypoint in the same way as the Cloud SDK - # (googlecloudsdk/third_party/appengine/api/validation.py) - # - # We could handle both string ("shell form") and list ("exec - # form") but it appears that gcloud only handles string form. - entrypoint = app_config.entrypoint - if entrypoint and not entrypoint.startswith('exec '): - entrypoint = 'exec ' + entrypoint - optional_entrypoint = DOCKERFILE_ENTRYPOINT_TEMPLATE.format( - entrypoint=entrypoint) + optional_entrypoint = get_data('Dockerfile.entrypoint.template').format( + entrypoint=app_config.entrypoint) else: optional_entrypoint = '' - dockerfile = DOCKERFILE_TEMPLATE.format( - base_image=app_config.base_image, - dockerfile_python_version=app_config.dockerfile_python_version, - optional_requirements_txt=optional_requirements_txt, - optional_entrypoint=optional_entrypoint) + dockerfile = ( + get_data('Dockerfile.preamble.template').format( + base_image=app_config.base_image) + + get_data('Dockerfile.virtualenv.template').format( + python_version=app_config.dockerfile_python_version) + + optional_requirements_txt + + get_data('Dockerfile.install_app') + + optional_entrypoint) return { 'Dockerfile': dockerfile, - '.dockerignore': DOCKERIGNORE, + '.dockerignore': get_data('dockerignore'), } diff --git a/scripts/gen_dockerfile_test.py b/scripts/gen_dockerfile_test.py index 92f70061..8169b297 100755 --- a/scripts/gen_dockerfile_test.py +++ b/scripts/gen_dockerfile_test.py @@ -82,7 +82,7 @@ def test_get_app_config(self): }), # entrypoint present ('entrypoint: my entrypoint', False, { - 'entrypoint': 'my entrypoint', + 'entrypoint': 'exec my entrypoint', }), ) for valid_case in valid_cases: @@ -129,7 +129,7 @@ def test_generate_files(self): # Entrypoint (base, False, 'CMD'), (base._replace(entrypoint='my entrypoint'), True, - 'CMD exec my entrypoint'), + 'CMD my entrypoint'), (base._replace(entrypoint='exec my entrypoint'), True, 'CMD exec my entrypoint'), # Base runtime image From c24977b57fff8be3ee228d91405670f707aa400b Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Wed, 21 Jun 2017 14:11:21 -0700 Subject: [PATCH 083/256] Relax type check and add comments about validation --- scripts/gen_dockerfile.py | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/scripts/gen_dockerfile.py b/scripts/gen_dockerfile.py index 30004ead..eb5dabbf 100755 --- a/scripts/gen_dockerfile.py +++ b/scripts/gen_dockerfile.py @@ -18,6 +18,7 @@ import argparse import collections +import collections.abc import functools import io import os @@ -73,13 +74,25 @@ def get_app_config(raw_config, base_image, config_file, source_dir): config_file (str): Path to user's app.yaml (might be .yaml) source_dir (str): Directory container user's source code + We validate the user input for security and better error messages. + + Yaml parsing rules can lead to extremely unhelpful error messages. + For example, parsing a string value where we expected a list. + Python will happily use the string as a sequence of individual + characters, leading to confusing results. + + We also try to prevent Dockerfile and Bash injection attacks. For + example, specifying entrypoint as "true\\nADD /etc/passwd /pwned" + would allow the user to inject arbitrary directives into the + Dockerfile, which is a support problem if nothing else. + Returns: AppConfig: valid configuration """ # Examine app.yaml - if not isinstance(raw_config, dict): + if not isinstance(raw_config, collections.abc.Mapping): raise ValueError( - 'Expected {} contents to be of type "dict", but found type "{}"'. + 'Expected {} contents to be a Mapping type, but found type "{}"'. format(config_file, type(raw_config))) entrypoint = validation_utils.get_field_value(raw_config, 'entrypoint', str) @@ -117,7 +130,18 @@ def get_app_config(raw_config, base_image, config_file, source_dir): def get_data(name): - """Return the contents of the named data resource""" + """Return the contents of the named data resource + + Args: + name (str): Name of file, without directory + + These templates are copied from the Google Cloud SDK at + google-cloud-sdk/platform/ext-runtime/python/data + and the two should be kept in sync. + + Returns: + str: Contents of data file + """ filename = os.path.join(os.path.dirname(__file__), 'data', name) with io.open(filename, 'r', encoding='utf8') as template_file: return template_file.read() From c34d4f04afbee13cd90d44430ac154162a34fd0c Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Wed, 21 Jun 2017 14:40:00 -0700 Subject: [PATCH 084/256] Document rationale for yaml validation --- scripts/gen_dockerfile.py | 26 +++++++++++++------------- scripts/validation_utils.py | 11 ++++++++++- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/scripts/gen_dockerfile.py b/scripts/gen_dockerfile.py index eb5dabbf..c75bf52f 100755 --- a/scripts/gen_dockerfile.py +++ b/scripts/gen_dockerfile.py @@ -68,24 +68,24 @@ def get_app_config(raw_config, base_image, config_file, source_dir): """Read and validate the application runtime configuration. - Args: - raw_config (dict): deserialized app.yaml - base_image (str): Docker image name to build on top of - config_file (str): Path to user's app.yaml (might be .yaml) - source_dir (str): Directory container user's source code - We validate the user input for security and better error messages. - Yaml parsing rules can lead to extremely unhelpful error messages. - For example, parsing a string value where we expected a list. - Python will happily use the string as a sequence of individual - characters, leading to confusing results. + Consider, parsing a yaml file which has a string value where we + expected a list. Python will happily use the string as a sequence + of individual characters, at least for a while, leading to + confusing results when it finally fails. We also try to prevent Dockerfile and Bash injection attacks. For example, specifying entrypoint as "true\\nADD /etc/passwd /pwned" would allow the user to inject arbitrary directives into the Dockerfile, which is a support problem if nothing else. + Args: + raw_config (dict): deserialized app.yaml + base_image (str): Docker image name to build on top of + config_file (str): Path to user's app.yaml (might be .yaml) + source_dir (str): Directory container user's source code + Returns: AppConfig: valid configuration """ @@ -132,13 +132,13 @@ def get_app_config(raw_config, base_image, config_file, source_dir): def get_data(name): """Return the contents of the named data resource - Args: - name (str): Name of file, without directory - These templates are copied from the Google Cloud SDK at google-cloud-sdk/platform/ext-runtime/python/data and the two should be kept in sync. + Args: + name (str): Name of file, without directory + Returns: str: Contents of data file """ diff --git a/scripts/validation_utils.py b/scripts/validation_utils.py index 7a04847e..2792dcb1 100644 --- a/scripts/validation_utils.py +++ b/scripts/validation_utils.py @@ -29,9 +29,18 @@ def get_field_value(container, field_name, field_type): """Fetch a field from a container with typechecking and default values. The field value is coerced to the desired type. If the field is - not present, a instance of `field_type` is constructed with no + not present, an instance of `field_type` is constructed with no arguments and used as the default value. + This function exists because yaml parsing can lead to surprising + outputs, and the resulting errors are confusing. For example: + entrypoint1: a string, but I can accidentally treat as an sequence + entrypoint2: [a, list, but, I, might, think, its, a, string] + version1: 3 # Parsed to int + version2: 3.1 # Parsed to float + version3: 3.1.1 # Parsed to str + feature: off # Parsed to the boolean False + Args: container (dict): Object decoded from yaml field_name (str): Field that should be present in `container` From 2fd8330f30ba7f8554cdb41c233711d22f610fbc Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Thu, 22 Jun 2017 14:40:15 -0700 Subject: [PATCH 085/256] Add more information to error messages Also use yaml "safe" mode, because "unsafe" is the default for some reason. --- scripts/gen_dockerfile.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/scripts/gen_dockerfile.py b/scripts/gen_dockerfile.py index c75bf52f..441fb153 100755 --- a/scripts/gen_dockerfile.py +++ b/scripts/gen_dockerfile.py @@ -97,7 +97,8 @@ def get_app_config(raw_config, base_image, config_file, source_dir): entrypoint = validation_utils.get_field_value(raw_config, 'entrypoint', str) if not PRINTABLE_REGEX.match(entrypoint): - raise ValueError('Invalid character in "entrypoint" field of app.yaml') + raise ValueError( + 'Invalid "entrypoint" value in app.yaml: {!r}'.format(entrypoint)) # Mangle entrypoint in the same way as the Cloud SDK # (googlecloudsdk/third_party/appengine/api/validation.py) @@ -116,7 +117,8 @@ def get_app_config(raw_config, base_image, config_file, source_dir): valid_versions = str(sorted(PYTHON_INTERPRETER_VERSION_MAP.keys())) raise ValueError( 'Invalid "python_version" field in "runtime_config" section ' - 'of app.yaml. Valid options are:\n{}'.format(valid_versions)) + 'of app.yaml: {!r}. Valid options are: {}'. + format(python_version, valid_versions)) # Examine user's files has_requirements_txt = os.path.isfile( @@ -167,14 +169,15 @@ def generate_files(app_config): else: optional_entrypoint = '' - dockerfile = ( + dockerfile = ''.join([ get_data('Dockerfile.preamble.template').format( - base_image=app_config.base_image) + + base_image=app_config.base_image), get_data('Dockerfile.virtualenv.template').format( - python_version=app_config.dockerfile_python_version) + - optional_requirements_txt + - get_data('Dockerfile.install_app') + - optional_entrypoint) + python_version=app_config.dockerfile_python_version), + optional_requirements_txt, + get_data('Dockerfile.install_app'), + optional_entrypoint , + ]) return { 'Dockerfile': dockerfile, @@ -193,7 +196,7 @@ def generate_dockerfile_command(base_image, config_file, source_dir): # Read yaml file. Does not currently support multiple services # with configuration filenames besides app.yaml with io.open(config_file, 'r', encoding='utf8') as yaml_config_file: - raw_config = yaml.load(yaml_config_file) + raw_config = yaml.safe_load(yaml_config_file) # Determine complete configuration app_config = get_app_config(raw_config, base_image, config_file, From 118871687d3eed8cbe6696ccccfd1fb29dbf268e Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Thu, 22 Jun 2017 18:40:33 -0700 Subject: [PATCH 086/256] Add nox support for running tests, lint, coverage, and requirements checking --- .coveragerc | 6 +++++ .gitignore | 2 ++ nox.py | 46 ++++++++++++++++++++++++++++++++++- scripts/requirements-test.txt | 4 +++ 4 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 .coveragerc create mode 100644 scripts/requirements-test.txt diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 00000000..b08b2574 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,6 @@ +[run] +branch = True + +[report] +exclude_lines = + pragma: no cover diff --git a/.gitignore b/.gitignore index 2a84c77d..62f3090b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ *.pyc +.cache +.coverage .nox /cloudbuild.yaml_local.sh /cloudbuild_benchmark.yaml_local.sh diff --git a/nox.py b/nox.py index 83be1ffa..fd79945e 100644 --- a/nox.py +++ b/nox.py @@ -15,6 +15,8 @@ import fnmatch import os +import nox + def _list_files(folder, pattern): """Lists all files below the given folder that match the pattern.""" @@ -24,7 +26,8 @@ def _list_files(folder, pattern): yield os.path.join(root, filename) -def session_check_requirements(session): +@nox.session +def check_requirements(session): """Checks for out of date requirements and optionally updates them.""" session.install('gcp-devrel-py-tools') @@ -37,3 +40,44 @@ def session_check_requirements(session): for reqfile in reqfiles: session.run('gcp-devrel-py-tools', command, reqfile) + + +@nox.session +def lint(session): + session.install('flake8', 'flake8-import-order') + session.run( + 'flake8', + '--import-order-style=google', + 'scripts', + 'nox.py', + ) + + +@nox.session +@nox.parametrize('version', ['3.4', '3.5', '3.6']) +def tests(session, version): + session.interpreter = 'python' + version + session.install('-r', 'scripts/requirements-test.txt') + session.run( + 'py.test', + '--ignore=scripts/testdata', + '--cov=scripts', + '--cov-append', + '--cov-config=.coveragerc', + '--cov-report=', # Report generated below + 'scripts', + env={'PYTHONPATH': ''} + ) + + +@nox.session +def cover(session): + """Run the final coverage report. + + This outputs the coverage report aggregating coverage from the unit + test runs (not system test runs), and then erases coverage data. + """ + session.interpreter = 'python3.6' + session.install('coverage', 'pytest-cov') + session.run('coverage', 'report', '--show-missing', '--fail-under=97') + session.run('coverage', 'erase') diff --git a/scripts/requirements-test.txt b/scripts/requirements-test.txt new file mode 100644 index 00000000..77b8c636 --- /dev/null +++ b/scripts/requirements-test.txt @@ -0,0 +1,4 @@ +flask +pytest +pytest-cov +pyyaml From 17359a7515e880cfa6c494d5e71d08fa8b000477 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 23 Jun 2017 18:11:33 -0700 Subject: [PATCH 087/256] Be more verbose when running a build --- scripts/local_cloudbuild.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/local_cloudbuild.py b/scripts/local_cloudbuild.py index a7c5eda9..ab02d83a 100755 --- a/scripts/local_cloudbuild.py +++ b/scripts/local_cloudbuild.py @@ -322,6 +322,7 @@ def local_cloudbuild(args): # Run shell script if cloudbuild.run: + print('Running {}'.format(cloudbuild.output_script)) args = [os.path.abspath(cloudbuild.output_script)] subprocess.check_call(args) From bb138e6f65ba4941ed1d1d3bafe681e31b3770f6 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 23 Jun 2017 18:58:25 -0700 Subject: [PATCH 088/256] Switch local_cloudbuild.py to py.test --- scripts/local_cloudbuild_test.py | 712 ++++++++++++++++--------------- 1 file changed, 365 insertions(+), 347 deletions(-) diff --git a/scripts/local_cloudbuild_test.py b/scripts/local_cloudbuild_test.py index fff089dc..00609f01 100755 --- a/scripts/local_cloudbuild_test.py +++ b/scripts/local_cloudbuild_test.py @@ -17,14 +17,14 @@ """Unit test for local_cloudbuild.py""" import argparse +import contextlib import os import re import shutil import subprocess -import tempfile -import unittest import unittest.mock +import pytest import yaml import local_cloudbuild @@ -34,355 +34,373 @@ STAGING_DIR_REGEX = re.compile( b'(?m)Copying source to staging directory (.+)$') -class LocalCloudbuildTest(unittest.TestCase): - - def setUp(self): - self.testdata_dir = os.path.join(os.path.dirname(__file__), 'testdata') - assert os.path.isdir(self.testdata_dir), 'Could not run test: testdata directory not found' - - def test_sub_and_quote(self): - valid_cases = ( - # Empty string - ('', {}, "''", []), - # No substitutions - ('a', {}, 'a', []), - # Unused substitition (ok here but error in generate_script) - ('a', {'FOO':'foo'}, 'a', []), - ('a', {'_FOO':'_foo'}, 'a', []), - # Defined builtin substitution - ('a$FOOb', {'FOO':'foo'}, 'afoob', ['FOO']), - ('a${FOO}b', {'FOO':'foo'}, 'afoob', ['FOO']), - # Defined user substitution - ('a$_FOOb', {'_FOO':'_foo'}, 'a_foob', ['_FOO']), - ('a${_FOO}b', {'_FOO':'_foo'}, 'a_foob', ['_FOO']), - # Multiple substitutions - ('$FOO${FOO}${BAR}$FOO', {'FOO':'foo', 'BAR':'bar'}, - 'foofoobarfoo', ['FOO', 'BAR']), - # Invalid names - ('a $ b', {}, "'a $ b'", []), - ('a$foo b', {}, "'a$foo b'", []), - ('a$0FOO b', {}, "'a$0FOO b'", []), - ) - for valid_case in valid_cases: - with self.subTest(valid_case=valid_case): - s, subs, expected, expected_used = valid_case - used = set() - actual = local_cloudbuild.sub_and_quote(s, subs, used) - self.assertEqual(actual, expected) - self.assertEqual(used, set(expected_used)) - - invalid_cases = ( - # Undefined builtin substitution - ('a$FOOb', {}), - ('a${FOO}b', {}), - # Undefined user substitution - ('a$_FOOb', {}), - ('a${_FOO}b', {}), - ) - for invalid_case in invalid_cases: - with self.subTest(invalid_case=invalid_case): - s, subs = invalid_case - with self.assertRaises(ValueError): - used = set() - local_cloudbuild.sub_and_quote(s, subs, used) - - def check_call_with_capture(self, *args, **kw_args): - """Act like subprocess.check_call but capture stdout""" + +@pytest.fixture +def testdata_dir(): + testdata_dir = os.path.join(os.path.dirname(__file__), 'testdata') + assert os.path.isdir(testdata_dir), ( + 'Could not run test: testdata directory not found') + return testdata_dir + + +@pytest.mark.parametrize('s, subs, expected, expected_used', [ + # Empty string + ('', {}, "''", []), + # No substitutions + ('a', {}, 'a', []), + # Unused substitition (ok here but error in generate_script) + ('a', {'FOO': 'foo'}, 'a', []), + ('a', {'_FOO': '_foo'}, 'a', []), + # Defined builtin substitution + ('a$FOOb', {'FOO': 'foo'}, 'afoob', ['FOO']), + ('a${FOO}b', {'FOO': 'foo'}, 'afoob', ['FOO']), + # Defined user substitution + ('a$_FOOb', {'_FOO': '_foo'}, 'a_foob', ['_FOO']), + ('a${_FOO}b', {'_FOO': '_foo'}, 'a_foob', ['_FOO']), + # Multiple substitutions + ('$FOO${FOO}${BAR}$FOO', + {'FOO': 'foo', 'BAR': 'bar'}, + 'foofoobarfoo', + ['FOO', 'BAR']), + # Invalid names + ('a $ b', {}, "'a $ b'", []), + ('a$foo b', {}, "'a$foo b'", []), + ('a$0FOO b', {}, "'a$0FOO b'", []), +]) +def test_sub_and_quote_valid(s, subs, expected, expected_used): + used = set() + actual = local_cloudbuild.sub_and_quote(s, subs, used) + assert actual == expected + assert used == set(expected_used) + + +@pytest.mark.parametrize('s, subs', [ + # Undefined builtin substitution + ('a$FOOb', {}), + ('a${FOO}b', {}), + # Undefined user substitution + ('a$_FOOb', {}), + ('a${_FOO}b', {}), +]) +def test_sub_and_quote_invalid(s, subs): + with pytest.raises(ValueError): + used = set() + local_cloudbuild.sub_and_quote(s, subs, used) + + +def have_docker(): + """Determine if the Docker daemon is present and usable""" + if ((shutil.which('docker') is not None) and + (subprocess.call(['docker', 'info'], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) == 0)): + return True + return False + + +_args = argparse.Namespace( + config='some_config_file', + output_script='some_output_script', + run=False, + substitutions={}, +) + + +def test_get_cloudbuild_valid(): + raw_yaml = 'steps:\n- name: step1\n- name: step2\n' + raw_config = yaml.safe_load(raw_yaml) + actual = local_cloudbuild.get_cloudbuild(raw_config, _args) + assert len(actual.steps) == 2 + + +@pytest.mark.parametrize('raw_yaml', [ + # Empty cloud build + '', + # No steps + 'foo: bar\n', + # Steps not a list + 'steps: astring\n', + ]) +def test_get_cloudbuild_invalid(raw_yaml): + raw_config = yaml.safe_load(raw_yaml) + with pytest.raises(ValueError): + local_cloudbuild.get_cloudbuild(raw_config, _args) + + +@pytest.mark.parametrize('raw_step, expected', [ + # Empty step + ({}, local_cloudbuild.Step( + args=[], + dir_='', + env=[], + name='', + )), + # Full step + ({'name': 'aname', + 'args': ['arg1', 2, 'arg3 with \n newline'], + 'env': ['ENV1=value1', 'ENV2=space in value2'], + 'dir': 'adir', + }, local_cloudbuild.Step( + args=['arg1', '2', 'arg3 with \n newline'], + env=['ENV1=value1', 'ENV2=space in value2'], + dir_='adir', + name='aname', + )), +]) +def test_get_step_valid(raw_step, expected): + actual = local_cloudbuild.get_step(raw_step) + assert actual == expected + + +@pytest.mark.parametrize('raw_step', [ + # Wrong type + [], + # More wrong types + {'args': 'not_a_list'}, + {'args': [[]]}, + {'env': 'not_a_list'}, + {'env': [{}]}, + {'dir': {}}, + {'name': []}, +]) +def test_get_step_invalid(raw_step): + with pytest.raises(ValueError): + local_cloudbuild.get_step(raw_step) + + +# Basic valid case +_base_step = local_cloudbuild.Step( + args=['arg1', 'arg2'], + dir_='', + env=['ENV1=value1', 'ENV2=value2'], + name='aname', +) +_subs = {'BUILTIN': 'builtin', '_USER': '_user'} + + +def test_generate_command_basic(): + command = local_cloudbuild.generate_command(_base_step, _subs, set()) + assert command == [ + 'docker', + 'run', + '--volume', + '/var/run/docker.sock:/var/run/docker.sock', + '--volume', + '/root/.docker:/root/.docker', + '--volume', + '${HOST_WORKSPACE}:/workspace', + '--workdir', + '/workspace', + '--env', + 'ENV1=value1', + '--env', + 'ENV2=value2', + 'aname', + 'arg1', + 'arg2', + ] + + +@pytest.mark.parametrize('step, args', [ + # dir specified + (_base_step._replace(dir_='adir'), + ['--workdir', '/workspace/adir']), + # Shell quoting + (_base_step._replace(args=['arg with \n newline']), + ["'arg with \n newline'"]), + (_base_step._replace(dir_='dir/ with space/'), + ["/workspace/'dir/ with space/'"]), + (_base_step._replace(env=['env with space']), + ["'env with space'"]), + (_base_step._replace(name='a name'), + ["'a name'"]), + # Variable substitution + (_base_step._replace(name='a $BUILTIN substitution'), + ["'a builtin substitution'"]), + (_base_step._replace(name='a $_USER substitution'), + ["'a _user substitution'"]), + (_base_step._replace(name='a curly brace ${BUILTIN} substitution'), + ["'a curly brace builtin substitution'"]), + (_base_step._replace( + name='an escaped $$ or $$$$ or $$FOO or $${_FOO} is unescaped'), + ["'an escaped $ or $$ or $FOO or ${_FOO} is unescaped'"]), +]) +def test_generate_command_valid(step, args): + command = local_cloudbuild.generate_command(step, _subs, set()) + for arg in args: + assert arg in command + + +@pytest.mark.parametrize('step', [ + _base_step._replace(name='a $UNSET_BUILTIN substitution'), + _base_step._replace(name='a $_UNSET_USER substitution'), +]) +def test_generate_command_invalid(step): + with pytest.raises(ValueError): + local_cloudbuild.generate_command(step, _subs, set()) + + +def test_generate_script_golden(testdata_dir): + config_name = 'cloudbuild_ok.yaml' + expected_output_script = os.path.join( + testdata_dir, config_name + '_golden.sh') + cloudbuild = local_cloudbuild.CloudBuild( + output_script='test_generate_script', + run=False, + steps=[ + local_cloudbuild.Step( + args=['/bin/sh', '-c', 'printenv MESSAGE'], + dir_='', + env=['MESSAGE=Hello World!'], + name='debian', + ), + local_cloudbuild.Step( + args=['/bin/sh', '-c', 'printenv MESSAGE'], + dir_='', + env=['MESSAGE=Goodbye\\n And Farewell!', 'UNUSED=unused'], + name='debian', + ) + ], + substitutions=local_cloudbuild.DEFAULT_SUBSTITUTIONS, + ) + actual = local_cloudbuild.generate_script(cloudbuild) + # Compare output against golden + with open(expected_output_script, 'r', encoding='utf8') as expected_file: + expected = expected_file.read() + assert actual == expected + + +def test_generate_script_unused_user_substitution(): + cloudbuild = local_cloudbuild.CloudBuild( + output_script='', + run=False, + steps=[], + substitutions={'_FOO': '_foo'}, + ) + with pytest.raises(ValueError, match='User substitution variables'): + local_cloudbuild.generate_script(cloudbuild) + + +def test_make_executable(tmpdir): + test_script_filename = tmpdir.join('test_make_executable.sh') + with test_script_filename.open('w', encoding='utf8') as test_script: + test_script.write('#!/bin/sh\necho "Output from test_make_executable"') + local_cloudbuild.make_executable(str(test_script_filename)) + output = subprocess.check_output([str(test_script_filename)]) + assert output.decode('utf8') == "Output from test_make_executable\n" + + +def test_write_script(tmpdir): + contents = 'The contents\n' + output_script_filename = tmpdir.join('test_write_script') + cloudbuild = local_cloudbuild.CloudBuild( + output_script=str(output_script_filename), + run=False, + steps=[], + substitutions={}, + ) + local_cloudbuild.write_script(cloudbuild, contents) + with output_script_filename.open('r', encoding='utf8') as output_script: + actual = output_script.read() + assert actual == contents + + +@contextlib.contextmanager +def chdir(new_dir): + """Not threadsafe""" + old_dir = os.getcwd() + os.chdir(new_dir) + yield + os.chdir(old_dir) + + +@pytest.mark.parametrize('config_name, substitutions, exception, cleanup', [ + # Everything is ok + ('cloudbuild_ok.yaml', None, None, True), + # Builtin substitutions like $PROJECT_ID work + ('cloudbuild_builtin_substitutions.yaml', None, None, True), + # User substitutions like $_FOO work + ('cloudbuild_user_substitutions.yaml', + {'_FOO': 'this is foo value'}, + None, True + ), + # User substitutions like $_FOO fails when undefined + ('cloudbuild_user_substitutions.yaml', None, ValueError, False), + # Exit code 1 (failure) + ('cloudbuild_err_rc1.yaml', None, subprocess.CalledProcessError, True), + # Command not found + ('cloudbuild_err_not_found.yaml', None, subprocess.CalledProcessError, + True), + # Cleaning up files owned by root + ('cloudbuild_difficult_cleanup.yaml', None, None, True), +]) +def test_local_cloudbuild(testdata_dir, tmpdir, config_name, substitutions, + exception, cleanup): + if not have_docker(): + pytest.fail('This test requires a working Docker daemon') + + check_call_output = None + + def check_call(*args, **kw_args): + """Act like subprocess.check_call but store stdout""" + nonlocal check_call_output try: - self.check_call_output = subprocess.check_output(*args, **kw_args) - print(self.check_call_output) + check_call_output = subprocess.check_output(*args, **kw_args) + print(check_call_output) except subprocess.CalledProcessError as e: - self.check_call_output = e.output - print(self.check_call_output) + check_call_output = e.output + print(check_call_output) raise - def have_docker(self): - """Determine if the Docker daemon is present and usable""" - if ((shutil.which('docker') is not None) and - (subprocess.call(['docker', 'info'], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL) == 0)): - return True - return False - - def test_get_cloudbuild(self): + # Read cloudbuild.yaml from testdata file, write output to + # tempdir, and maybe try to run it + with unittest.mock.patch('subprocess.check_call', check_call): + if substitutions is None: + substitutions = local_cloudbuild.DEFAULT_SUBSTITUTIONS + should_succeed = (exception is None) + config = os.path.join(testdata_dir, config_name) + actual_output_script = tmpdir.join(config_name + '_local.sh') args = argparse.Namespace( - config='some_config_file', - output_script='some_output_script', - run=False, - substitutions={}, - ) - # Basic valid case - valid_case = 'steps:\n- name: step1\n- name: step2\n' - raw_config = yaml.safe_load(valid_case) - actual = local_cloudbuild.get_cloudbuild(raw_config, args) - self.assertEqual(len(actual.steps), 2) - - invalid_cases = ( - # Empty cloud build - '', - # No steps - 'foo: bar\n', - # Steps not a list - 'steps: astring\n', - ) - for invalid_case in invalid_cases: - with self.subTest(invalid_case=invalid_case): - raw_config = yaml.safe_load(invalid_case) - with self.assertRaises(ValueError): - local_cloudbuild.get_cloudbuild(raw_config, args) - - def test_get_step(self): - valid_cases = ( - # Empty step - ({}, local_cloudbuild.Step( - args=[], - dir_='', - env=[], - name='', - )), - # Full step - ({'name' : 'aname', - 'args' : [ 'arg1', 2, 'arg3 with \n newline', ], - 'env' : [ 'ENV1=value1', 'ENV2=space in value2' ], - 'dir' : 'adir', - }, local_cloudbuild.Step( - args = [ 'arg1', '2', 'arg3 with \n newline', ], - env = [ 'ENV1=value1', 'ENV2=space in value2' ], - dir_ = 'adir', - name = 'aname', - )), + config=config, + output_script=str(actual_output_script), + run=True, + substitutions=substitutions, ) - for valid_case in valid_cases: - with self.subTest(valid_case=valid_case): - raw_step, expected = valid_case - actual = local_cloudbuild.get_step(raw_step) - self.assertEqual(actual, expected) - - invalid_cases = ( - # Wrong type - [], - # More wrong types - {'args': 'not_a_list'}, - {'args': [ [] ]}, - {'env': 'not_a_list'}, - {'env': [ {} ]}, - {'dir': {}}, - {'name': []}, - ) - for invalid_case in invalid_cases: - with self.subTest(invalid_case=invalid_case): - with self.assertRaises(ValueError): - local_cloudbuild.get_step(invalid_case) - - def test_generate_command(self): - # Basic valid case - base_step = local_cloudbuild.Step( - args = ['arg1','arg2'], - dir_ = '', - env = ['ENV1=value1', 'ENV2=value2'], - name = 'aname', - ) - subs = {'BUILTIN':'builtin', '_USER':'_user'} - command = local_cloudbuild.generate_command(base_step, subs, set()) - self.assertEqual(command, [ - 'docker', - 'run', - '--volume', - '/var/run/docker.sock:/var/run/docker.sock', - '--volume', - '/root/.docker:/root/.docker', - '--volume', - '${HOST_WORKSPACE}:/workspace', - '--workdir', - '/workspace', - '--env', - 'ENV1=value1', - '--env', - 'ENV2=value2', - 'aname', - 'arg1', - 'arg2', - ]) - - valid_cases = ( - # dir specified - (base_step._replace(dir_='adir'), - ['--workdir', '/workspace/adir']), - # Shell quoting - (base_step._replace(args=['arg with \n newline']), - ["'arg with \n newline'"]), - (base_step._replace(dir_='dir/ with space/'), - ["/workspace/'dir/ with space/'"]), - (base_step._replace(env=['env with space']), - ["'env with space'"]), - (base_step._replace(name='a name'), - ["'a name'"]), - # Variable substitution - (base_step._replace(name='a $BUILTIN substitution'), - ["'a builtin substitution'"]), - (base_step._replace(name='a $_USER substitution'), - ["'a _user substitution'"]), - (base_step._replace(name='a curly brace ${BUILTIN} substitution'), - ["'a curly brace builtin substitution'"]), - (base_step._replace(name='an escaped $$ or $$$$ or $$FOO or $${_FOO} is unescaped'), - ["'an escaped $ or $$ or $FOO or ${_FOO} is unescaped'"]), - ) - for valid_case in valid_cases: - with self.subTest(valid_case=valid_case): - step, args = valid_case - command = local_cloudbuild.generate_command(step, subs, set()) - for arg in args: - self.assertIn(arg, command) - - invalid_cases = ( - base_step._replace(name='a $UNSET_BUILTIN substitution'), - base_step._replace(name='a $_UNSET_USER substitution'), - ) - for invalid_case in invalid_cases: - with self.subTest(invalid_case=invalid_case): - step = invalid_case - with self.assertRaises(ValueError): - local_cloudbuild.generate_command(step, subs, set()) - - def test_generate_script_golden(self): - config_name = 'cloudbuild_ok.yaml' - config = os.path.join(self.testdata_dir, config_name) - expected_output_script = os.path.join(self.testdata_dir, config_name + '_golden.sh') - cloudbuild = local_cloudbuild.CloudBuild( - output_script='test_generate_script', - run=False, - steps=[ - local_cloudbuild.Step( - args=['/bin/sh', '-c', 'printenv MESSAGE'], - dir_='', - env=['MESSAGE=Hello World!'], - name='debian', - ), - local_cloudbuild.Step( - args=['/bin/sh', '-c', 'printenv MESSAGE'], - dir_='', - env=['MESSAGE=Goodbye\\n And Farewell!', 'UNUSED=unused'], - name='debian', - ) - ], - substitutions=local_cloudbuild.DEFAULT_SUBSTITUTIONS, - ) - actual = local_cloudbuild.generate_script(cloudbuild) - self.maxDiff = 2**16 - # Compare output against golden - with open(expected_output_script, 'r', encoding='utf8') as expected: - self.assertEqual(actual, expected.read()) - - def test_generate_script_unused_user_substitution(self): - cloudbuild = local_cloudbuild.CloudBuild( - output_script='', - run=False, - steps=[], - substitutions={'_FOO':'_foo'}, - ) - with self.assertRaisesRegex(ValueError, 'User substitution variables'): - actual = local_cloudbuild.generate_script(cloudbuild) - - def test_make_executable(self): - with tempfile.TemporaryDirectory( - prefix='local_cloudbuild_test_') as tempdir: - test_script_filename = os.path.join(tempdir, 'test_make_executable.sh') - with open(test_script_filename, 'w', encoding='utf8') as test_script: - test_script.write('#!/bin/sh\necho "Output from test_make_executable"') - local_cloudbuild.make_executable(test_script_filename) - output = subprocess.check_output([test_script_filename]) - self.assertEqual(output.decode('utf8'), "Output from test_make_executable\n") - - def test_write_script(self): - with tempfile.TemporaryDirectory( - prefix='local_cloudbuild_test_') as tempdir: - contents = 'The contents\n' - output_script_filename = os.path.join(tempdir, 'test_write_script') - cloudbuild = local_cloudbuild.CloudBuild( - output_script=output_script_filename, - run=False, - steps=[], - substitutions={}, - ) - local_cloudbuild.write_script(cloudbuild, contents) - with open(output_script_filename, 'r', encoding='utf8') as output_script: - actual = output_script.read() - self.assertEqual(actual, contents) - - def test_local_cloudbuild(self): - if not self.have_docker(): - self.fail('This test requires a working Docker daemon') - - # Read cloudbuild.yaml from testdata file, write output to - # tempdir, and maybe try to run it - cases = ( - # Everything is ok - ('cloudbuild_ok.yaml', None, None), - # Builtin substitutions like $PROJECT_ID work - ('cloudbuild_builtin_substitutions.yaml', None, None), - # User substitutions like $_FOO work - ('cloudbuild_user_substitutions.yaml', - {'_FOO':'this is foo value'}, - None - ), - # User substitutions like $_FOO fails when undefined - ('cloudbuild_user_substitutions.yaml', None, ValueError), - # Exit code 1 (failure) - ('cloudbuild_err_rc1.yaml', None, subprocess.CalledProcessError), - # Command not found - ('cloudbuild_err_not_found.yaml', None, subprocess.CalledProcessError), - # Cleaning up files owned by root - ('cloudbuild_difficult_cleanup.yaml', None, None), - ) - for case in cases: - with self.subTest(case=case), \ - tempfile.TemporaryDirectory(prefix='local_cloudbuild_test_') as tempdir, \ - unittest.mock.patch('subprocess.check_call', self.check_call_with_capture): - config_name, substitutions, exception = case - if substitutions is None: - substitutions = local_cloudbuild.DEFAULT_SUBSTITUTIONS - should_succeed = (exception is None) - config = os.path.join(self.testdata_dir, config_name) - actual_output_script = os.path.join( - tempdir, config_name + '_local.sh') - args = argparse.Namespace( - config=config, - output_script=actual_output_script, - run=True, - substitutions=substitutions, - ) - - if should_succeed: + + # The source directory of the build is currently hardcoded as + # '.', so we must chdir there. + with chdir(testdata_dir): + if should_succeed: + local_cloudbuild.local_cloudbuild(args) + else: + with pytest.raises(exception): local_cloudbuild.local_cloudbuild(args) - else: - with self.assertRaises(exception): - local_cloudbuild.local_cloudbuild(args) - - # Check that staging dir was cleaned up - match = re.search(STAGING_DIR_REGEX, self.check_call_output) - self.assertTrue(match) - staging_dir = match.group(1) - self.assertFalse(os.path.isdir(staging_dir), staging_dir) - - def test_parse_args(self): - # Test explicit output_script - argv = ['argv0', '--output_script=my_output'] - args = local_cloudbuild.parse_args(argv) - self.assertEqual(args.output_script, 'my_output') - # Test implicit output_script - argv = ['argv0', '--config=my_config'] - args = local_cloudbuild.parse_args(argv) - self.assertEqual(args.output_script, 'my_config_local.sh') - - # Test run flag (default and --no-run) - argv = ['argv0'] - args = local_cloudbuild.parse_args(argv) - self.assertEqual(args.run, True) - argv = ['argv0', '--no-run'] - args = local_cloudbuild.parse_args(argv) - self.assertEqual(args.run, False) - - -if __name__ == '__main__': - unittest.main() + + # Check that staging dir was cleaned up + if cleanup: + assert check_call_output is not None + match = re.search(STAGING_DIR_REGEX, check_call_output) + assert match + staging_dir = match.group(1) + assert not os.path.isdir(staging_dir) + + +@pytest.mark.parametrize('argv, expected', [ + # Test explicit output_script + (['argv0', '--output_script=my_output'], 'my_output'), + # Test implicit output_script + (['argv0', '--config=my_config'], 'my_config_local.sh'), +]) +def test_parse_args_output_script(argv, expected): + args = local_cloudbuild.parse_args(argv) + assert args.output_script == expected + + +@pytest.mark.parametrize('argv, expected', [ + # Test run flag (default) + (['argv0'], True), + (['argv0', '--no-run'], False), +]) +def test_parse_args_run_flag(argv, expected): + args = local_cloudbuild.parse_args(argv) + assert args.run == expected From 83380501fcab435638cccc2446207f5f1505e916 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 23 Jun 2017 21:18:25 -0700 Subject: [PATCH 089/256] Fix style violations --- scripts/gen_dockerfile.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/scripts/gen_dockerfile.py b/scripts/gen_dockerfile.py index 441fb153..c9632b04 100755 --- a/scripts/gen_dockerfile.py +++ b/scripts/gen_dockerfile.py @@ -83,7 +83,7 @@ def get_app_config(raw_config, base_image, config_file, source_dir): Args: raw_config (dict): deserialized app.yaml base_image (str): Docker image name to build on top of - config_file (str): Path to user's app.yaml (might be .yaml) + config_file (str): Path to user's app.yaml (might be .yaml) source_dir (str): Directory container user's source code Returns: @@ -95,7 +95,8 @@ def get_app_config(raw_config, base_image, config_file, source_dir): 'Expected {} contents to be a Mapping type, but found type "{}"'. format(config_file, type(raw_config))) - entrypoint = validation_utils.get_field_value(raw_config, 'entrypoint', str) + entrypoint = validation_utils.get_field_value( + raw_config, 'entrypoint', str) if not PRINTABLE_REGEX.match(entrypoint): raise ValueError( 'Invalid "entrypoint" value in app.yaml: {!r}'.format(entrypoint)) @@ -108,8 +109,10 @@ def get_app_config(raw_config, base_image, config_file, source_dir): if entrypoint and not entrypoint.startswith('exec '): entrypoint = 'exec ' + entrypoint - raw_runtime_config = validation_utils.get_field_value(raw_config, 'runtime_config', dict) - python_version = validation_utils.get_field_value(raw_runtime_config, 'python_version', str) + raw_runtime_config = validation_utils.get_field_value( + raw_config, 'runtime_config', dict) + python_version = validation_utils.get_field_value( + raw_runtime_config, 'python_version', str) dockerfile_python_version = PYTHON_INTERPRETER_VERSION_MAP.get( python_version) @@ -164,8 +167,9 @@ def generate_files(app_config): optional_requirements_txt = '' if app_config.entrypoint: - optional_entrypoint = get_data('Dockerfile.entrypoint.template').format( - entrypoint=app_config.entrypoint) + optional_entrypoint = get_data( + 'Dockerfile.entrypoint.template').format( + entrypoint=app_config.entrypoint) else: optional_entrypoint = '' @@ -176,8 +180,8 @@ def generate_files(app_config): python_version=app_config.dockerfile_python_version), optional_requirements_txt, get_data('Dockerfile.install_app'), - optional_entrypoint , - ]) + optional_entrypoint, + ]) return { 'Dockerfile': dockerfile, @@ -190,7 +194,7 @@ def generate_dockerfile_command(base_image, config_file, source_dir): Args: base_image (str): Docker image name to build on top of - config_file (str): Path to user's app.yaml (might be .yaml) + config_file (str): Path to user's app.yaml (might be .yaml) source_dir (str): Directory container user's source code """ # Read yaml file. Does not currently support multiple services From 7675e2e2a11c330ab3afd4850dcbe43cb6f4cc9d Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 23 Jun 2017 21:22:56 -0700 Subject: [PATCH 090/256] Fix style violations --- scripts/local_cloudbuild.py | 19 ++++++++++--------- scripts/testdata/cloudbuild_ok.yaml_golden.sh | 2 +- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/scripts/local_cloudbuild.py b/scripts/local_cloudbuild.py index ab02d83a..f80e0f4b 100755 --- a/scripts/local_cloudbuild.py +++ b/scripts/local_cloudbuild.py @@ -73,7 +73,7 @@ } # Use this image for cleanup actions -DEBIAN_IMAGE='gcr.io/google-appengine/debian8' +DEBIAN_IMAGE = 'gcr.io/google-appengine/debian8' # File template BUILD_SCRIPT_TEMPLATE = """\ @@ -90,7 +90,7 @@ if [ "${{HOST_WORKSPACE}}" != '/' -a -d "${{HOST_WORKSPACE}}" ]; then # Expect a single error message about /workspace busy {cleanup_str} 2>/dev/null || true - # Do not expect error messages here. Display but ignore any that happen. + # Do not expect error messages here. Display but ignore. rmdir "${{HOST_WORKSPACE}}" || true fi }} @@ -108,7 +108,8 @@ # Validated cloudbuild recipe + flags -CloudBuild = collections.namedtuple('CloudBuild', 'output_script run steps substitutions') +CloudBuild = collections.namedtuple('CloudBuild', + 'output_script run steps substitutions') # Single validated step in a cloudbuild recipe Step = collections.namedtuple('Step', 'args dir_ env name') @@ -136,8 +137,7 @@ def sub(match): # Variables must be set raise ValueError( 'Variable "{}" used without being defined. Try adding ' - 'it to the --substitutions flag'.format( - variable_name)) + 'it to the --substitutions flag'.format(variable_name)) else: value = substitutions.get(variable_name) substitutions_used.add(variable_name) @@ -272,8 +272,9 @@ def generate_script(cloudbuild): if user_subs_unused: nice_list = '"' + '", "'.join(sorted(user_subs_unused)) + '"' raise ValueError( - 'User substitution variables {} were defined in the --substitution ' - 'flag but never used in the cloudbuild file.'.format(nice_list)) + 'User substitution variables {} were defined in the ' + '--substitution flag but never used in the cloudbuild file.'. + format(nice_list)) cleanup_str = ' '.join(cleanup_command) docker_lines = [] @@ -298,7 +299,7 @@ def make_executable(path): def write_script(cloudbuild, contents): """Write a shell script to a file.""" print('Writing build script to {}'.format(cloudbuild.output_script)) - with open(cloudbuild.output_script, 'w', encoding='utf8') as outfile: + with io.open(cloudbuild.output_script, 'w', encoding='utf8') as outfile: outfile.write(contents) make_executable(cloudbuild.output_script) @@ -310,7 +311,7 @@ def local_cloudbuild(args): args: command line flags as per parse_args """ # Load and parse cloudbuild.yaml - with open(args.config, 'r', encoding='utf8') as cloudbuild_file: + with io.open(args.config, 'r', encoding='utf8') as cloudbuild_file: raw_config = yaml.safe_load(cloudbuild_file) # Determine configuration diff --git a/scripts/testdata/cloudbuild_ok.yaml_golden.sh b/scripts/testdata/cloudbuild_ok.yaml_golden.sh index ba4a52e1..bda52bd2 100755 --- a/scripts/testdata/cloudbuild_ok.yaml_golden.sh +++ b/scripts/testdata/cloudbuild_ok.yaml_golden.sh @@ -11,7 +11,7 @@ function cleanup { if [ "${HOST_WORKSPACE}" != '/' -a -d "${HOST_WORKSPACE}" ]; then # Expect a single error message about /workspace busy docker run --volume /var/run/docker.sock:/var/run/docker.sock --volume /root/.docker:/root/.docker --volume ${HOST_WORKSPACE}:/workspace --workdir /workspace gcr.io/google-appengine/debian8 rm -rf /workspace 2>/dev/null || true - # Do not expect error messages here. Display but ignore any that happen. + # Do not expect error messages here. Display but ignore. rmdir "${HOST_WORKSPACE}" || true fi } From 66a1a2b812c3637c99e26ca7b9fd1c0b7347e69f Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 23 Jun 2017 21:25:00 -0700 Subject: [PATCH 091/256] Convert tests to pytest and fix style violations --- scripts/gen_dockerfile_test.py | 382 +++++++++++++++---------------- scripts/validation_utils.py | 4 +- scripts/validation_utils_test.py | 142 +++++------- 3 files changed, 252 insertions(+), 276 deletions(-) diff --git a/scripts/gen_dockerfile_test.py b/scripts/gen_dockerfile_test.py index 8169b297..09194eb8 100755 --- a/scripts/gen_dockerfile_test.py +++ b/scripts/gen_dockerfile_test.py @@ -19,210 +19,202 @@ import argparse import filecmp import os -import re import shutil import subprocess -import tempfile -import unittest import unittest.mock +import pytest import yaml import gen_dockerfile + # Expected list of files generated EXPECTED_OUTPUT_FILES = ['Dockerfile', '.dockerignore'] -class GenDockerfileTest(unittest.TestCase): - def setUp(self): - self.testdata_dir = os.path.join(os.path.dirname(__file__), 'testdata') - assert os.path.isdir(self.testdata_dir), 'Could not run test: testdata directory not found' - - def compare_file(self, filename, dir1, dir2): - """Compare identically named files in two different directories""" - if not filecmp.cmp(os.path.join(dir1, filename), - os.path.join(dir2, filename)): - msg = 'Contents of "{}" differ between "{}" and "{}"'.format( - filename, dir1, dir2) - self.assertMultiLineEqual(contents1, contents2, msg) - - def test_get_app_config(self): - config_file = 'some_config_file' - base_image = 'some_image_name' - source_dir = 'some_source_dir' - - valid_cases = ( - # Basic app.yaml - ('env: flex', False, { - 'base_image': 'some_image_name', - 'dockerfile_python_version': '', - 'has_requirements_txt': False, - 'entrypoint': '', - }), - # All supported python versions - ('runtime_config:\n python_version:', False, { - 'dockerfile_python_version': '', - }), - ('runtime_config:\n python_version: 2', False, { - 'dockerfile_python_version': '', - }), - ('runtime_config:\n python_version: 3', False, { - 'dockerfile_python_version': '3.5', - }), - ('runtime_config:\n python_version: 3.4', False, { - 'dockerfile_python_version': '3.4', - }), - ('runtime_config:\n python_version: 3.5', False, { - 'dockerfile_python_version': '3.5', - }), - # requirements.txt present - ('env: flex', True, { - 'has_requirements_txt': True, - }), - # entrypoint present - ('entrypoint: my entrypoint', False, { - 'entrypoint': 'exec my entrypoint', - }), - ) - for valid_case in valid_cases: - with self.subTest(valid_case=valid_case): - app_yaml, isfile, expected = valid_case - raw_app_config = yaml.safe_load(app_yaml) - with unittest.mock.patch.object( - os.path, 'isfile', return_value=isfile): - actual = gen_dockerfile.get_app_config( - raw_app_config, base_image, config_file, - source_dir) - for key, value in expected.items(): - self.assertEqual(getattr(actual, key), value) - - invalid_cases = ( - # Empty app.yaml - '', - # Invalid entrypoint - 'entrypoint: "bad \\n entrypoint"', - # Invalid python version - 'runtime_config:\n python_version: 1', - 'runtime_config:\n python_version: python2', - ) - for invalid_case in invalid_cases: - with self.subTest(invalid_case=invalid_case): - raw_app_config = yaml.safe_load(invalid_case) - with self.assertRaises(ValueError): - gen_dockerfile.get_app_config( - raw_app_config, base_image, config_file, - source_dir) - - def test_generate_files(self): - base = gen_dockerfile.AppConfig( - base_image='', - dockerfile_python_version='', - entrypoint='', - has_requirements_txt=False - ) - cases = ( - # Requirements.txt - (base, False, 'ADD requirements.txt'), - (base._replace(has_requirements_txt=True), True, - 'ADD requirements.txt'), - # Entrypoint - (base, False, 'CMD'), - (base._replace(entrypoint='my entrypoint'), True, - 'CMD my entrypoint'), - (base._replace(entrypoint='exec my entrypoint'), True, - 'CMD exec my entrypoint'), - # Base runtime image - (base._replace(base_image='my_base_runtime_image'), True, - 'FROM my_base_runtime_image'), - # Python version - (base._replace(dockerfile_python_version='_my_version'), True, - 'python_version=python_my_version'), - ) - for case in cases: - with self.subTest(case=case): - app_config, should_find, test_string = case - result = gen_dockerfile.generate_files(app_config) - self.assertEqual( - sorted(result.keys()), sorted(EXPECTED_OUTPUT_FILES)) - dockerfile = result['Dockerfile'] - if should_find: - self.assertIn(test_string, dockerfile) - else: - self.assertNotIn(test_string, dockerfile) - - def test_generate_dockerfile_command(self): - """Generates output and compares against a set of golden files. - - Optionally runs 'gcloud app gen-config' and compares against that. - """ - # Sample app from https://github.com/GoogleCloudPlatform/python-docs-samples - with tempfile.TemporaryDirectory( - prefix='gen_dockerfile_test_') as parent_tempdir: - for app in ['hello_world']: - app_dir = os.path.join(self.testdata_dir, app) - temp_dir = os.path.join(parent_tempdir, app) - os.mkdir(temp_dir) - - # Copy sample app to writable temp dir, and generate Dockerfile. - config_dir = os.path.join(temp_dir, 'config') - shutil.copytree(app_dir, config_dir) - gen_dockerfile.generate_dockerfile_command( - base_image='gcr.io/google-appengine/python', - config_file=os.path.join(config_dir, 'app.yaml'), - source_dir=config_dir) - - # Compare against golden files - golden_dir = os.path.join(self.testdata_dir, app + '_golden') - for filename in EXPECTED_OUTPUT_FILES: - with self.subTest(source='golden', filename=filename): - self.compare_file(filename, config_dir, golden_dir) - - # Copy sample app to different writable temp dir, and - # generate Dockerfile using gcloud. - if not shutil.which('gcloud'): - self.skipTest( - '"gcloud" tool not found in $PATH, skipping test') - gen_config_dir = os.path.join(temp_dir, 'gen_config') - shutil.copytree(app_dir, gen_config_dir) - app_yaml = os.path.join(gen_config_dir, 'app.yaml') - gcloud_args = [ - 'gcloud', '--quiet', 'beta', 'app', 'gen-config', - gen_config_dir, '--custom', '--config={}'.format(app_yaml) - ] - print('Invoking gcloud as {}'.format(gcloud_args)) - subprocess.check_call(gcloud_args) - for filename in EXPECTED_OUTPUT_FILES: - with self.subTest(source='gcloud', filename=filename): - self.compare_file(filename, config_dir, gen_config_dir) - - def test_parse_args(self): - valid_cases = ( - [], - ['argv0', '--base-image=nocolon'], - ['argv0', '--base-image=name:andcolon'], - ['argv0', '--base-image=name@sha256:digest'], - ) - for argv in valid_cases: - with self.subTest(valid_argv=argv): - args = gen_dockerfile.parse_args(argv) - - def mock_error(*args): - """Prevent argparse from calling sys.exit()""" - raise AssertionError(*args) - - invalid_cases = ( - ['argv0', '--base-image='], - ['argv0', '--base-image=:'], - ['argv0', '--base-image=:noname'], - ) - for argv in invalid_cases: - with self.subTest(invalid_argv=argv): - with unittest.mock.patch.object( - argparse.ArgumentParser, 'error', mock_error): - with self.assertRaises(AssertionError): - gen_dockerfile.parse_args(argv) - - -if __name__ == '__main__': - unittest.main() +@pytest.fixture +def testdata_dir(): + testdata_dir = os.path.join(os.path.dirname(__file__), 'testdata') + assert os.path.isdir(testdata_dir), ( + 'Could not run test: testdata directory not found') + return testdata_dir + + +def compare_file(filename, dir1, dir2): + """Compare identically named files in two different directories""" + assert filecmp.cmp( + os.path.join(dir1, filename), os.path.join(dir2, filename)) + + +@pytest.mark.parametrize('app_yaml, isfile, expected', [ + # Basic app.yaml + ('env: flex', False, { + 'base_image': 'some_image_name', + 'dockerfile_python_version': '', + 'has_requirements_txt': False, + 'entrypoint': '', + }), + # All supported python versions + ('runtime_config:\n python_version:', False, { + 'dockerfile_python_version': '', + }), + ('runtime_config:\n python_version: 2', False, { + 'dockerfile_python_version': '', + }), + ('runtime_config:\n python_version: 3', False, { + 'dockerfile_python_version': '3.5', + }), + ('runtime_config:\n python_version: 3.4', False, { + 'dockerfile_python_version': '3.4', + }), + ('runtime_config:\n python_version: 3.5', False, { + 'dockerfile_python_version': '3.5', + }), + # requirements.txt present + ('env: flex', True, { + 'has_requirements_txt': True, + }), + # entrypoint present + ('entrypoint: my entrypoint', False, { + 'entrypoint': 'exec my entrypoint', + }), +]) +def test_get_app_config_valid(app_yaml, isfile, expected): + config_file = 'some_config_file' + base_image = 'some_image_name' + source_dir = 'some_source_dir' + raw_app_config = yaml.safe_load(app_yaml) + with unittest.mock.patch.object(os.path, 'isfile', return_value=isfile): + actual = gen_dockerfile.get_app_config( + raw_app_config, base_image, config_file, + source_dir) + for key, value in expected.items(): + assert getattr(actual, key) == value + + +@pytest.mark.parametrize('app_yaml', [ + # Empty app.yaml + '', + # Invalid entrypoint + 'entrypoint: "bad \\n entrypoint"', + # Invalid python version + 'runtime_config:\n python_version: 1', + 'runtime_config:\n python_version: python2', +]) +def test_get_app_config_invalid(app_yaml): + config_file = 'some_config_file' + base_image = 'some_image_name' + source_dir = 'some_source_dir' + raw_app_config = yaml.safe_load(app_yaml) + with pytest.raises(ValueError): + gen_dockerfile.get_app_config( + raw_app_config, base_image, config_file, source_dir) + + +# Basic AppConfig used below +_base = gen_dockerfile.AppConfig( + base_image='', + dockerfile_python_version='', + entrypoint='', + has_requirements_txt=False +) + + +@pytest.mark.parametrize('app_config, should_find, test_string', [ + # Requirements.txt + (_base, False, 'ADD requirements.txt'), + (_base._replace(has_requirements_txt=True), True, + 'ADD requirements.txt'), + # Entrypoint + (_base, False, 'CMD'), + (_base._replace(entrypoint='my entrypoint'), True, + 'CMD my entrypoint'), + (_base._replace(entrypoint='exec my entrypoint'), True, + 'CMD exec my entrypoint'), + # Base runtime image + (_base._replace(base_image='my_base_runtime_image'), True, + 'FROM my_base_runtime_image'), + # Python version + (_base._replace(dockerfile_python_version='_my_version'), True, + 'python_version=python_my_version'), +]) +def test_generate_files(app_config, should_find, test_string): + result = gen_dockerfile.generate_files(app_config) + assert sorted(result.keys()) == sorted(EXPECTED_OUTPUT_FILES) + dockerfile = result['Dockerfile'] + if should_find: + assert test_string in dockerfile + else: + assert test_string not in dockerfile + + +@pytest.mark.parametrize('app', [ + # Sample from https://github.com/GoogleCloudPlatform/python-docs-samples + 'hello_world', +]) +def test_generate_dockerfile_command(tmpdir, testdata_dir, app): + """Generates output and compares against a set of golden files. + + Optionally runs 'gcloud app gen-config' and compares against that. + """ + app_dir = os.path.join(testdata_dir, app) + temp_dir = os.path.join(str(tmpdir), app) + os.mkdir(temp_dir) + + # Copy sample app to writable temp dir, and generate Dockerfile. + config_dir = os.path.join(temp_dir, 'config') + shutil.copytree(app_dir, config_dir) + gen_dockerfile.generate_dockerfile_command( + base_image='gcr.io/google-appengine/python', + config_file=os.path.join(config_dir, 'app.yaml'), + source_dir=config_dir) + + # Compare against golden files + golden_dir = os.path.join(testdata_dir, app + '_golden') + for filename in EXPECTED_OUTPUT_FILES: + compare_file(filename, config_dir, golden_dir) + + # Copy sample app to different writable temp dir, and + # generate Dockerfile using gcloud. + if not shutil.which('gcloud'): + print('"gcloud" tool not found in $PATH, skipping rest of test') + return + gen_config_dir = os.path.join(temp_dir, 'gen_config') + shutil.copytree(app_dir, gen_config_dir) + app_yaml = os.path.join(gen_config_dir, 'app.yaml') + gcloud_args = [ + 'gcloud', '--quiet', 'beta', 'app', 'gen-config', + gen_config_dir, '--custom', '--config={}'.format(app_yaml) + ] + print('Invoking gcloud as {}'.format(gcloud_args)) + subprocess.check_call(gcloud_args) + for filename in EXPECTED_OUTPUT_FILES: + compare_file(filename, config_dir, gen_config_dir) + + +@pytest.mark.parametrize('argv', [ + [], + ['argv0', '--base-image=nocolon'], + ['argv0', '--base-image=name:andcolon'], + ['argv0', '--base-image=name@sha256:digest'], +]) +def test_parse_args_valid(argv): + args = gen_dockerfile.parse_args(argv) + assert args is not None + + +@pytest.mark.parametrize('argv', [ + ['argv0', '--base-image='], + ['argv0', '--base-image=:'], + ['argv0', '--base-image=:noname'], +]) +def test_parse_args_invalid(argv): + def mock_error(*args): + """Prevent argparse from calling sys.exit()""" + raise AssertionError(*args) + + with unittest.mock.patch.object(argparse.ArgumentParser, 'error', + mock_error): + with pytest.raises(AssertionError): + gen_dockerfile.parse_args(argv) diff --git a/scripts/validation_utils.py b/scripts/validation_utils.py index 2792dcb1..28de2d52 100644 --- a/scripts/validation_utils.py +++ b/scripts/validation_utils.py @@ -101,7 +101,7 @@ def validate_arg_dict(flag_value): match = re.match(KEY_VALUE_REGEX, entry) if not match: raise argparse.ArgumentTypeError( - 'Value "{}" should be a list like _KEY1=value1,_KEY2=value2"'.format( - flag_value)) + 'Value "{}" should be a list like _KEY1=value1,_KEY2=value2"'. + format(flag_value)) pairs.append((match.group(1), match.group(2))) return dict(pairs) diff --git a/scripts/validation_utils_test.py b/scripts/validation_utils_test.py index 9ef87096..ff853e4e 100755 --- a/scripts/validation_utils_test.py +++ b/scripts/validation_utils_test.py @@ -18,93 +18,77 @@ import argparse import re -import unittest + +import pytest import validation_utils -class ValidationUtilsTest(unittest.TestCase): +@pytest.mark.parametrize('container, field_name, field_type, expected', [ + # Normal case, field present and correct type + ({'present': 1}, 'present', int, 1), + ({'present': '1'}, 'present', str, '1'), + ({'present': [1]}, 'present', list, [1]), + ({'present': {1: 2}}, 'present', dict, {1: 2}), + # Missing field replaced by default + ({}, 'missing', str, ''), + # Valid conversions + ({'str_to_int': '1'}, 'str_to_int', int, 1), + ({'int_to_str': 1}, 'int_to_str', str, '1'), + # None + ({'None_to_int': None}, 'None_to_int', int, 0), + ({'None_to_str': None}, 'None_to_str', str, ''), +]) +def test_get_field_value_valid(container, field_name, field_type, expected): + assert validation_utils.get_field_value( + container, field_name, field_type) == expected + - def test_get_field_value(self): - valid_cases = ( - # Normal case, field present and correct type - ({ 'present': 1 }, 'present', int, 1), - ({ 'present': '1' }, 'present', str, '1'), - ({ 'present': [1] }, 'present', list, [1]), - ({ 'present': {1: 2} }, 'present', dict, {1: 2}), - # Missing field replaced by default - ({}, 'missing', str, ''), - # Valid conversions - ({ 'str_to_int': '1' }, 'str_to_int', int, 1), - ({ 'int_to_str': 1 }, 'int_to_str', str, '1'), - # None - ({ 'None_to_int': None }, 'None_to_int', int, 0), - ({ 'None_to_str': None }, 'None_to_str', str, ''), - ) - for valid_case in valid_cases: - with self.subTest(valid_case=valid_case): - container, field_name, field_type, expected = valid_case - self.assertEqual( - validation_utils.get_field_value( - container, field_name, field_type), - expected) +@pytest.mark.parametrize('container, field_name, field_type', [ + # Type conversion failures + ({'bad_list_to_dict': [1]}, 'bad_list_to_dict', dict), + ({'bad_list_to_str': [1]}, 'bad_list_to_str', str), + ({'bad_dict_to_list': {1: 2}}, 'bad_dict_to_list', list), + ({'bad_str_to_int': 'not_an_int'}, 'bad_str_to_int', int), + ({'bad_str_to_list': 'abc'}, 'bad_str_to_list', list), +]) +def test_get_field_value_invalid(container, field_name, field_type): + with pytest.raises(ValueError): + validation_utils.get_field_value(container, field_name, field_type) - invalid_cases = ( - # Type conversion failures - ({ 'bad_list_to_dict': [1] }, 'bad_list_to_dict', dict), - ({ 'bad_list_to_str': [1] }, 'bad_list_to_str', str), - ({ 'bad_dict_to_list': {1: 2} }, 'bad_dict_to_list', list), - ({ 'bad_str_to_int': 'not_an_int' }, 'bad_str_to_int', int), - ({ 'bad_str_to_list': 'abc' }, 'bad_str_to_list', list), - ) - for invalid_case in invalid_cases: - with self.subTest(invalid_case=invalid_case): - container, field_name, field_type = invalid_case - with self.assertRaises(ValueError): - validation_utils.get_field_value( - container, field_name, field_type) - def test_validate_arg_regex(self): - self.assertEqual( - validation_utils.validate_arg_regex('abc', re.compile('a[b]c')), - 'abc') - with self.assertRaises(argparse.ArgumentTypeError): - validation_utils.validate_arg_regex('abc', re.compile('a[d]c')) +def test_validate_arg_regex(): + assert validation_utils.validate_arg_regex( + 'abc', re.compile('a[b]c')) == 'abc' + with pytest.raises(argparse.ArgumentTypeError): + validation_utils.validate_arg_regex('abc', re.compile('a[d]c')) - def test_validate_arg_dict(self): - valid_cases = ( - # Normal case, field present and correct type - ('', {}), - ('_A=1', {'_A':'1'}), - ('_A=1,_B=2', {'_A':'1', '_B':'2'}), - # Repeated key is ok - ('_A=1,_A=2', {'_A':'2'}), - # Extra = is ok - ('_A=x=y=z,_B=2', {'_A':'x=y=z', '_B':'2'}), - # No value is ok - ('_A=', {'_A':''}), - ) - for valid_case in valid_cases: - with self.subTest(valid_case=valid_case): - s, expected = valid_case - self.assertEqual( - validation_utils.validate_arg_dict(s), - expected) - invalid_cases = ( - # No key - ',_A', - '_A,', - # Invalid variable name - '_Aa=1', - '_aA=1', - '0A=1', - ) - for invalid_case in invalid_cases: - with self.subTest(invalid_case=invalid_case): - with self.assertRaises(argparse.ArgumentTypeError): - validation_utils.validate_arg_dict(invalid_case) +@pytest.mark.parametrize('arg, expected', [ + # Normal case, field present and correct type + ('', {}), + ('_A=1', {'_A': '1'}), + ('_A=1,_B=2', {'_A': '1', '_B': '2'}), + # Repeated key is ok + ('_A=1,_A=2', {'_A': '2'}), + # Extra = is ok + ('_A=x=y=z,_B=2', {'_A': 'x=y=z', '_B': '2'}), + # No value is ok + ('_A=', {'_A': ''}), +]) +def test_validate_arg_dicts_valid(arg, expected): + assert validation_utils.validate_arg_dict(arg) == expected -if __name__ == '__main__': - unittest.main() +@pytest.mark.parametrize('arg', [ + # No key + ',_A', + '_A,', + # Invalid variable name + '_Aa=1', + '_aA=1', + '0A=1', +]) +def test_validate_arg_dicts_invalid(arg): + with pytest.raises(argparse.ArgumentTypeError): + validation_utils.validate_arg_dict(arg) From d9ff1ae9b452edbf179e19e0ab9afec6857b53f2 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 23 Jun 2017 21:28:28 -0700 Subject: [PATCH 092/256] Use Python3 for nox lint, and make nox.py pass lint --- nox.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/nox.py b/nox.py index fd79945e..08ffd872 100644 --- a/nox.py +++ b/nox.py @@ -17,7 +17,6 @@ import nox - def _list_files(folder, pattern): """Lists all files below the given folder that match the pattern.""" for root, folders, files in os.walk(folder): @@ -44,10 +43,13 @@ def check_requirements(session): @nox.session def lint(session): + session.interpreter = 'python3' session.install('flake8', 'flake8-import-order') session.run( 'flake8', '--import-order-style=google', + ('--application-import-names=' + 'gen_dockerfile,local_cloudbuild,validation_utils'), 'scripts', 'nox.py', ) @@ -64,9 +66,9 @@ def tests(session, version): '--cov=scripts', '--cov-append', '--cov-config=.coveragerc', - '--cov-report=', # Report generated below + '--cov-report=', # Report generated below 'scripts', - env={'PYTHONPATH': ''} + env={'PYTHONPATH':''} ) From d4b6fb1a1ac43faee5d5b51ae259dcc2fbc4255d Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 23 Jun 2017 21:52:47 -0700 Subject: [PATCH 093/256] Make nox.py actually pass lint --- nox.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nox.py b/nox.py index 08ffd872..ab2f0917 100644 --- a/nox.py +++ b/nox.py @@ -17,6 +17,7 @@ import nox + def _list_files(folder, pattern): """Lists all files below the given folder that match the pattern.""" for root, folders, files in os.walk(folder): @@ -66,9 +67,9 @@ def tests(session, version): '--cov=scripts', '--cov-append', '--cov-config=.coveragerc', - '--cov-report=', # Report generated below + '--cov-report=', # Report generated below 'scripts', - env={'PYTHONPATH':''} + env={'PYTHONPATH': ''} ) From 24e4d218f2a21d970f8a178d71ddd720bfe8f4db Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 23 Jun 2017 21:58:07 -0700 Subject: [PATCH 094/256] FULL_BASE_IMAGE -> STAGING_IMAGE to match other Dockerfiles --- builder/gen-dockerfile/Dockerfile.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/gen-dockerfile/Dockerfile.in b/builder/gen-dockerfile/Dockerfile.in index 4a8de5ec..e2fe4463 100644 --- a/builder/gen-dockerfile/Dockerfile.in +++ b/builder/gen-dockerfile/Dockerfile.in @@ -1,4 +1,4 @@ -FROM ${FULL_BASE_IMAGE} +FROM ${STAGING_IMAGE} LABEL python_version=python3.5 RUN virtualenv --no-download /env -p python3.5 From 3d14f06219e61a33a5b1b0957f7dc16ed8e21fbb Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 23 Jun 2017 22:04:36 -0700 Subject: [PATCH 095/256] Fix build.sh file copy --- build.sh | 9 +++++---- builder/gen-dockerfile/.gitignore | 1 + 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/build.sh b/build.sh index ee1d0ddf..943c5b92 100755 --- a/build.sh +++ b/build.sh @@ -125,12 +125,13 @@ for outfile in \ done # Make some files available to the runtime builder Docker context +mkdir -p builder/gen-dockerfile/data for file in \ - gen_dockerfile.py \ - validation_utils.py \ - 'data/*' \ + scripts/gen_dockerfile.py \ + scripts/validation_utils.py \ + scripts/data/* \ ; do - cp -a "scripts/${file}" "builder/gen-dockerfile/${file}" + cp -a "${file}" "builder/gen-dockerfile/${file##scripts/}" done # Build images and push to GCR diff --git a/builder/gen-dockerfile/.gitignore b/builder/gen-dockerfile/.gitignore index d8ffddc3..f45549bd 100644 --- a/builder/gen-dockerfile/.gitignore +++ b/builder/gen-dockerfile/.gitignore @@ -1,2 +1,3 @@ Dockerfile *.py +data/ From 67fbea3b4ed83c95b9b51e6d65f96813bf44f7bd Mon Sep 17 00:00:00 2001 From: Chris Heng Date: Mon, 26 Jun 2017 23:51:20 +0800 Subject: [PATCH 096/256] Replace deprecated "vm: true" in app.yaml example with "env: flex" (#126) This brings it up to date with official documentation: https://cloud.google.com/appengine/docs/flexible/python/runtime --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7c14904e..78a02c5c 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ docker by specifying `runtime: python` in your `app.yaml`: ```yaml runtime: python -vm: true +env: flex entrypoint: gunicorn -b :$PORT main:app runtime_config: From eef60bf009ec89d89e32d47c83d0ad17deb18ec6 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Mon, 26 Jun 2017 10:58:44 -0700 Subject: [PATCH 097/256] Slight tweak to nox.py format --- nox.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nox.py b/nox.py index ab2f0917..8bbe47fa 100644 --- a/nox.py +++ b/nox.py @@ -44,13 +44,13 @@ def check_requirements(session): @nox.session def lint(session): - session.interpreter = 'python3' + session.interpreter = 'python3' # So it understands Python3 syntax session.install('flake8', 'flake8-import-order') session.run( 'flake8', '--import-order-style=google', - ('--application-import-names=' - 'gen_dockerfile,local_cloudbuild,validation_utils'), + '--application-import-names', + 'gen_dockerfile,local_cloudbuild,validation_utils', 'scripts', 'nox.py', ) From 46b18dfb8fa5a4320f1d8ba3a35b9a6df877a5d5 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Mon, 26 Jun 2017 11:10:32 -0700 Subject: [PATCH 098/256] Split one test into two --- scripts/gen_dockerfile_test.py | 43 +++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/scripts/gen_dockerfile_test.py b/scripts/gen_dockerfile_test.py index 09194eb8..78520205 100755 --- a/scripts/gen_dockerfile_test.py +++ b/scripts/gen_dockerfile_test.py @@ -47,50 +47,55 @@ def compare_file(filename, dir1, dir2): os.path.join(dir1, filename), os.path.join(dir2, filename)) -@pytest.mark.parametrize('app_yaml, isfile, expected', [ +@pytest.mark.parametrize('app_yaml, expected', [ # Basic app.yaml - ('env: flex', False, { + ('env: flex', { 'base_image': 'some_image_name', 'dockerfile_python_version': '', 'has_requirements_txt': False, 'entrypoint': '', }), # All supported python versions - ('runtime_config:\n python_version:', False, { + ('runtime_config:\n python_version:', { 'dockerfile_python_version': '', }), - ('runtime_config:\n python_version: 2', False, { + ('runtime_config:\n python_version: 2', { 'dockerfile_python_version': '', }), - ('runtime_config:\n python_version: 3', False, { + ('runtime_config:\n python_version: 3', { 'dockerfile_python_version': '3.5', }), - ('runtime_config:\n python_version: 3.4', False, { + ('runtime_config:\n python_version: 3.4', { 'dockerfile_python_version': '3.4', }), - ('runtime_config:\n python_version: 3.5', False, { + ('runtime_config:\n python_version: 3.5', { 'dockerfile_python_version': '3.5', }), - # requirements.txt present - ('env: flex', True, { - 'has_requirements_txt': True, - }), # entrypoint present - ('entrypoint: my entrypoint', False, { + ('entrypoint: my entrypoint', { 'entrypoint': 'exec my entrypoint', }), ]) -def test_get_app_config_valid(app_yaml, isfile, expected): +def test_get_app_config_valid(app_yaml, expected): config_file = 'some_config_file' base_image = 'some_image_name' source_dir = 'some_source_dir' raw_app_config = yaml.safe_load(app_yaml) - with unittest.mock.patch.object(os.path, 'isfile', return_value=isfile): - actual = gen_dockerfile.get_app_config( - raw_app_config, base_image, config_file, - source_dir) - for key, value in expected.items(): - assert getattr(actual, key) == value + actual = gen_dockerfile.get_app_config( + raw_app_config, base_image, config_file, + source_dir) + for key, value in expected.items(): + assert getattr(actual, key) == value + + +def test_get_app_config_requirements_txt(): + """requirements.txt file present""" + app_yaml = 'env: flex' + expected = { + 'has_requirements_txt': True, + } + with unittest.mock.patch.object(os.path, 'isfile', return_value=True): + test_get_app_config_valid(app_yaml, expected) @pytest.mark.parametrize('app_yaml', [ From d53ac16caa8c583dee84ec2d1c058d89c8c9335d Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Mon, 26 Jun 2017 11:50:31 -0700 Subject: [PATCH 099/256] Minor restructure and tidying of test code --- scripts/gen_dockerfile_test.py | 69 +++++++++++++++++----------------- 1 file changed, 34 insertions(+), 35 deletions(-) diff --git a/scripts/gen_dockerfile_test.py b/scripts/gen_dockerfile_test.py index 78520205..705cd524 100755 --- a/scripts/gen_dockerfile_test.py +++ b/scripts/gen_dockerfile_test.py @@ -30,7 +30,7 @@ # Expected list of files generated -EXPECTED_OUTPUT_FILES = ['Dockerfile', '.dockerignore'] +EXPECTED_OUTPUT_FILES = set(('Dockerfile', '.dockerignore')) @pytest.fixture @@ -118,7 +118,7 @@ def test_get_app_config_invalid(app_yaml): # Basic AppConfig used below -_base = gen_dockerfile.AppConfig( +_BASE_APP_CONFIG = gen_dockerfile.AppConfig( base_image='', dockerfile_python_version='', entrypoint='', @@ -128,25 +128,25 @@ def test_get_app_config_invalid(app_yaml): @pytest.mark.parametrize('app_config, should_find, test_string', [ # Requirements.txt - (_base, False, 'ADD requirements.txt'), - (_base._replace(has_requirements_txt=True), True, + (_BASE_APP_CONFIG, False, 'ADD requirements.txt'), + (_BASE_APP_CONFIG._replace(has_requirements_txt=True), True, 'ADD requirements.txt'), # Entrypoint - (_base, False, 'CMD'), - (_base._replace(entrypoint='my entrypoint'), True, + (_BASE_APP_CONFIG, False, 'CMD'), + (_BASE_APP_CONFIG._replace(entrypoint='my entrypoint'), True, 'CMD my entrypoint'), - (_base._replace(entrypoint='exec my entrypoint'), True, + (_BASE_APP_CONFIG._replace(entrypoint='exec my entrypoint'), True, 'CMD exec my entrypoint'), # Base runtime image - (_base._replace(base_image='my_base_runtime_image'), True, + (_BASE_APP_CONFIG._replace(base_image='my_base_runtime_image'), True, 'FROM my_base_runtime_image'), # Python version - (_base._replace(dockerfile_python_version='_my_version'), True, + (_BASE_APP_CONFIG._replace(dockerfile_python_version='_my_version'), True, 'python_version=python_my_version'), ]) def test_generate_files(app_config, should_find, test_string): result = gen_dockerfile.generate_files(app_config) - assert sorted(result.keys()) == sorted(EXPECTED_OUTPUT_FILES) + assert set(result.keys()) == EXPECTED_OUTPUT_FILES dockerfile = result['Dockerfile'] if should_find: assert test_string in dockerfile @@ -154,38 +154,38 @@ def test_generate_files(app_config, should_find, test_string): assert test_string not in dockerfile -@pytest.mark.parametrize('app', [ - # Sample from https://github.com/GoogleCloudPlatform/python-docs-samples - 'hello_world', -]) -def test_generate_dockerfile_command(tmpdir, testdata_dir, app): - """Generates output and compares against a set of golden files. +def compare_against_golden_files(app, config_dir, testdata_dir): + golden_dir = os.path.join(testdata_dir, app + '_golden') + for filename in EXPECTED_OUTPUT_FILES: + compare_file(filename, config_dir, golden_dir) + - Optionally runs 'gcloud app gen-config' and compares against that. - """ +def test_generate_dockerfile_command(tmpdir, testdata_dir): + """Generates output and compares against a set of golden files.""" + # Sample from https://github.com/GoogleCloudPlatform/python-docs-samples + app = 'hello_world' app_dir = os.path.join(testdata_dir, app) - temp_dir = os.path.join(str(tmpdir), app) - os.mkdir(temp_dir) # Copy sample app to writable temp dir, and generate Dockerfile. - config_dir = os.path.join(temp_dir, 'config') + config_dir = os.path.join(str(tmpdir), 'config') shutil.copytree(app_dir, config_dir) gen_dockerfile.generate_dockerfile_command( base_image='gcr.io/google-appengine/python', config_file=os.path.join(config_dir, 'app.yaml'), source_dir=config_dir) + compare_against_golden_files(app, config_dir, testdata_dir) - # Compare against golden files - golden_dir = os.path.join(testdata_dir, app + '_golden') - for filename in EXPECTED_OUTPUT_FILES: - compare_file(filename, config_dir, golden_dir) - # Copy sample app to different writable temp dir, and - # generate Dockerfile using gcloud. - if not shutil.which('gcloud'): - print('"gcloud" tool not found in $PATH, skipping rest of test') - return - gen_config_dir = os.path.join(temp_dir, 'gen_config') +@pytest.mark.xfail(not shutil.which('gcloud'), + reason='Google Cloud SDK is not installed') +def test_generate_dockerfile_golden(tmpdir, testdata_dir): + """Validate our golden files against gcloud app gen-config""" + # Sample from https://github.com/GoogleCloudPlatform/python-docs-samples + app = 'hello_world' + app_dir = os.path.join(testdata_dir, app) + + # Copy sample app to writable temp dir, and generate Dockerfile. + gen_config_dir = os.path.join(str(tmpdir), 'gen_config') shutil.copytree(app_dir, gen_config_dir) app_yaml = os.path.join(gen_config_dir, 'app.yaml') gcloud_args = [ @@ -194,8 +194,7 @@ def test_generate_dockerfile_command(tmpdir, testdata_dir, app): ] print('Invoking gcloud as {}'.format(gcloud_args)) subprocess.check_call(gcloud_args) - for filename in EXPECTED_OUTPUT_FILES: - compare_file(filename, config_dir, gen_config_dir) + compare_against_golden_files(app, gen_config_dir, testdata_dir) @pytest.mark.parametrize('argv', [ @@ -219,7 +218,7 @@ def mock_error(*args): """Prevent argparse from calling sys.exit()""" raise AssertionError(*args) - with unittest.mock.patch.object(argparse.ArgumentParser, 'error', - mock_error): + with unittest.mock.patch.object( + argparse.ArgumentParser, 'error', mock_error): with pytest.raises(AssertionError): gen_dockerfile.parse_args(argv) From 17191cc674de5983fa91474c32b711a63943a794 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Mon, 26 Jun 2017 11:53:57 -0700 Subject: [PATCH 100/256] Fix nit in nox.py --- nox.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nox.py b/nox.py index 8bbe47fa..872eb88c 100644 --- a/nox.py +++ b/nox.py @@ -48,7 +48,7 @@ def lint(session): session.install('flake8', 'flake8-import-order') session.run( 'flake8', - '--import-order-style=google', + '--import-order-style', 'google', '--application-import-names', 'gen_dockerfile,local_cloudbuild,validation_utils', 'scripts', From d722f0053e59b9436eccd16196477c2289b560a9 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Mon, 26 Jun 2017 11:58:19 -0700 Subject: [PATCH 101/256] Fix lint warning --- scripts/gen_dockerfile_test.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/gen_dockerfile_test.py b/scripts/gen_dockerfile_test.py index 705cd524..046580dd 100755 --- a/scripts/gen_dockerfile_test.py +++ b/scripts/gen_dockerfile_test.py @@ -218,7 +218,8 @@ def mock_error(*args): """Prevent argparse from calling sys.exit()""" raise AssertionError(*args) - with unittest.mock.patch.object( - argparse.ArgumentParser, 'error', mock_error): + error_patch = unittest.mock.patch.object( + argparse.ArgumentParser, 'error', mock_error) + with error_patch: with pytest.raises(AssertionError): gen_dockerfile.parse_args(argv) From 65183249c9e05566b7f05636d8cfb94dafcdc9f7 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Mon, 26 Jun 2017 15:15:38 -0700 Subject: [PATCH 102/256] Fix minor typos in comments --- scripts/gen_dockerfile.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/gen_dockerfile.py b/scripts/gen_dockerfile.py index c9632b04..9bfbca2a 100755 --- a/scripts/gen_dockerfile.py +++ b/scripts/gen_dockerfile.py @@ -70,7 +70,7 @@ def get_app_config(raw_config, base_image, config_file, source_dir): We validate the user input for security and better error messages. - Consider, parsing a yaml file which has a string value where we + Consider parsing a yaml file which has a string value where we expected a list. Python will happily use the string as a sequence of individual characters, at least for a while, leading to confusing results when it finally fails. @@ -84,7 +84,7 @@ def get_app_config(raw_config, base_image, config_file, source_dir): raw_config (dict): deserialized app.yaml base_image (str): Docker image name to build on top of config_file (str): Path to user's app.yaml (might be .yaml) - source_dir (str): Directory container user's source code + source_dir (str): Directory containing user's source code Returns: AppConfig: valid configuration From 7b8c4de65a606d93045073a8ea6ef8befedb7475 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Mon, 26 Jun 2017 15:17:07 -0700 Subject: [PATCH 103/256] Add main so these tests can be run standalone --- scripts/gen_dockerfile_test.py | 4 ++++ scripts/local_cloudbuild_test.py | 4 ++++ scripts/validation_utils_test.py | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/scripts/gen_dockerfile_test.py b/scripts/gen_dockerfile_test.py index 046580dd..6a7bd33a 100755 --- a/scripts/gen_dockerfile_test.py +++ b/scripts/gen_dockerfile_test.py @@ -223,3 +223,7 @@ def mock_error(*args): with error_patch: with pytest.raises(AssertionError): gen_dockerfile.parse_args(argv) + + +if __name__ == '__main__': + pytest.main([__file__]) diff --git a/scripts/local_cloudbuild_test.py b/scripts/local_cloudbuild_test.py index 00609f01..b22d585b 100755 --- a/scripts/local_cloudbuild_test.py +++ b/scripts/local_cloudbuild_test.py @@ -404,3 +404,7 @@ def test_parse_args_output_script(argv, expected): def test_parse_args_run_flag(argv, expected): args = local_cloudbuild.parse_args(argv) assert args.run == expected + + +if __name__ == '__main__': + pytest.main([__file__]) diff --git a/scripts/validation_utils_test.py b/scripts/validation_utils_test.py index ff853e4e..d759b276 100755 --- a/scripts/validation_utils_test.py +++ b/scripts/validation_utils_test.py @@ -92,3 +92,7 @@ def test_validate_arg_dicts_valid(arg, expected): def test_validate_arg_dicts_invalid(arg): with pytest.raises(argparse.ArgumentTypeError): validation_utils.validate_arg_dict(arg) + + +if __name__ == '__main__': + pytest.main([__file__]) From 251cd92369dd750a493f8a091a924d675a957a82 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Wed, 28 Jun 2017 19:19:49 -0700 Subject: [PATCH 104/256] Rename builder image to parallel the other languages' builders. --- build.sh | 10 +++++++++- cloudbuild.yaml | 4 ++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/build.sh b/build.sh index 943c5b92..7ee1416b 100755 --- a/build.sh +++ b/build.sh @@ -50,11 +50,19 @@ if [ -z "${DOCKER_NAMESPACE+set}" ] ; then fatal 'Error: $DOCKER_NAMESPACE is not set; invoke with something like DOCKER_NAMESPACE=gcr.io/YOUR-PROJECT-NAME' fi +if [ -z "${BUILDER_DOCKER_NAMESPACE+set}" ] ; then + export BUILDER_DOCKER_NAMESPACE="${DOCKER_NAMESPACE}" +fi + if [ -z "${TAG+set}" ] ; then export TAG=`date +%Y-%m-%d-%H%M%S` fi -substitutions="_DOCKER_NAMESPACE=${DOCKER_NAMESPACE},_TAG=${TAG}" +substitutions="\ +_BUILDER_DOCKER_NAMESPACE=${BUILDER_DOCKER_NAMESPACE},\ +_DOCKER_NAMESPACE=${DOCKER_NAMESPACE},\ +_TAG=${TAG}\ +" # Read command line arguments while [ $# -gt 0 ]; do diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 3f52e41d..187884dc 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -33,9 +33,9 @@ steps: name: ${_DOCKER_NAMESPACE}/python/tests/google-cloud-python:${_TAG} - # Build runtime builder image name: gcr.io/cloud-builders/docker:latest - args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/builder/gen-dockerfile:${_TAG}', + args: ['build', '--tag=${_BUILDER_DOCKER_NAMESPACE}/python/gen-dockerfile:${_TAG}', '--no-cache', '/workspace/builder/gen-dockerfile/'] images: [ '${_DOCKER_NAMESPACE}/python:${_TAG}', - '${_DOCKER_NAMESPACE}/python/builder/gen-dockerfile:${_TAG}', + '${_BUILDER_DOCKER_NAMESPACE}/python/gen-dockerfile:${_TAG}', ] From 35c666ff7a9717fc53da17e2bc20006f706b8e66 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Wed, 12 Jul 2017 01:41:50 -0700 Subject: [PATCH 105/256] Auto-update dependencies. --- scripts/requirements-test.txt | 8 +- scripts/testdata/hello_world/requirements.txt | 4 +- tests/deploy_check/requirements.txt | 2 +- tests/integration/requirements.txt | 10 +- tests/python2-libraries/requirements.txt | 123 +++++++++--------- tests/python3-libraries/requirements.txt | 118 ++++++++--------- 6 files changed, 132 insertions(+), 133 deletions(-) diff --git a/scripts/requirements-test.txt b/scripts/requirements-test.txt index 77b8c636..d1db042b 100644 --- a/scripts/requirements-test.txt +++ b/scripts/requirements-test.txt @@ -1,4 +1,4 @@ -flask -pytest -pytest-cov -pyyaml +flask==0.12.2 +pytest==3.1.3 +pytest-cov==2.5.1 +pyyaml==3.12 diff --git a/scripts/testdata/hello_world/requirements.txt b/scripts/testdata/hello_world/requirements.txt index 7861fb49..bb84e50e 100644 --- a/scripts/testdata/hello_world/requirements.txt +++ b/scripts/testdata/hello_world/requirements.txt @@ -1,2 +1,2 @@ -Flask==0.12 -gunicorn==19.6.0 +Flask==0.12.2 +gunicorn==19.7.1 diff --git a/tests/deploy_check/requirements.txt b/tests/deploy_check/requirements.txt index e7676bba..bb84e50e 100644 --- a/tests/deploy_check/requirements.txt +++ b/tests/deploy_check/requirements.txt @@ -1,2 +1,2 @@ -Flask==0.12.1 +Flask==0.12.2 gunicorn==19.7.1 diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index 71d785ae..ee8d85c3 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -1,7 +1,7 @@ -Flask==0.12.1 -google-cloud-error-reporting==0.24.2 -google-cloud-logging==1.0.0 -google-cloud-monitoring==0.24.0 +Flask==0.12.2 +google-cloud-error-reporting==0.25.1 +google-cloud-logging==1.1.0 +google-cloud-monitoring==0.25.0 gunicorn==19.7.1 -requests==2.14.2 +requests==2.18.1 retrying==1.3.3 diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 61e36d58..2e51b07f 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -1,12 +1,12 @@ -alembic==0.9.1 +alembic==0.9.3 amqp==2.1.4 amqplib==1.0.2 -ansible==2.3.0.0 +ansible==2.3.1.0 anyjson==0.3.3 apache-libcloud==2.0.0 argparse==1.4.0 -astroid==1.5.2 -awscli==1.11.85 +astroid==1.5.3 +awscli==1.11.117 babel==2.4.0 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 @@ -15,50 +15,50 @@ beautifulsoup==3.2.1 billiard==3.5.0.2 blessings==1.6 blinker==1.4 -boto==2.46.1 -botocore==1.5.48 +boto==2.48.0 +botocore==1.5.80 bottle==0.12.13 -carbon==1.0.1 +carbon==1.0.2 celery==4.0.2 certifi==2017.4.17 cffi==1.10.0 -chardet==3.0.2 +chardet==3.0.4 click==6.7 -cliff==2.7.0 -cmd2==0.7.0 +cliff==2.8.0 +cmd2==0.7.5 colorama==0.3.9 configobj==5.0.6 cov-core==1.15.0 -coverage==4.4 +coverage==4.4.1 coveralls==1.1 -cryptography==1.8.1 +cryptography==1.9 cssselect==1.0.1 cython==0.25.2 decorator==4.0.11 django-celery==3.2.1 django-debug-toolbar==1.8 -django-extensions==1.7.9 -django==1.11.1 +django-extensions==1.8.1 +django==1.11.3 django_compress==1.0.1 djangorestframework==3.6.3 docker-py==1.10.6 docopt==0.6.2 docutils==0.13.1 ecdsa==0.13 -elasticsearch==5.3.0 +elasticsearch==5.4.0 enum34==1.1.6 eventlet==0.21.0 extras==1.0.0 fabric==1.13.2 fixtures==3.0.0 flake8==3.3.0 -flask==0.12.1 +flask==0.12.2 funcsigs==1.0.2 functools32==3.2.3.post2 futures==3.1.1 -gevent==1.2.1 +gevent==1.2.2 google-api-python-client==1.6.2 -graphite-web==1.0.1 +graphite-web==1.0.2 greenlet==0.4.12 gunicorn==19.7.1 hiredis==0.2.0 @@ -66,17 +66,16 @@ html5lib httplib2==0.10.3 idna==2.5 ipaddress==1.0.18 -#ipython # No longer supports Python 2 as of version 6.0 iso8601==0.1.11 isodate==0.5.4 itsdangerous==0.24 jinja2==2.9.6 -jmespath==0.9.2 +jmespath==0.9.3 jsonschema==2.6.0 kombu==4.0.2 linecache2==1.0.0 -logilab-common==1.4.0 -lxml==3.7.3 +logilab-common==1.4.1 +lxml==3.8.0 m2crypto==0.26.0 mako==1.0.6 manifestparser==1.1 @@ -89,8 +88,8 @@ mock==2.0.0 mozcrash==1.0 mozdevice==0.50 mozfile==1.2 -mozinfo==0.9 -mozlog==3.4 +mozinfo==0.10 +mozlog==3.5 moznetwork==0.27 mozprocess==0.25 mozprofile==0.28 @@ -99,95 +98,95 @@ msgpack-python==0.4.8 mysql-python==1.2.5 ndg-httpsclient==0.4.2 netaddr==0.7.19 -netifaces==0.10.5 -newrelic==2.86.0.65 +netifaces==0.10.6 +newrelic==2.88.0.72 nose==1.3.7 -numpy==1.12.1 +numpy==1.13.1 oauth2==1.9.0.post1 -oauth2client==4.1.0 +oauth2client==4.1.2 oauthlib==2.0.2 ordereddict==1.1 -oslo.config==4.1.0 -pandas==0.20.1 -paramiko==2.1.2 +oslo.config==4.8.0 +pandas==0.20.3 +paramiko==2.2.1 passlib==1.7.1 paste==2.0.3 pastedeploy==1.5.2 pastescript==2.0.2 -pbr==3.0.0 +pbr==3.1.1 pep8==1.7.0 pexpect==4.2.1 pika==0.10.0 -pillow==4.1.1 +pillow==4.2.1 pip==9.0.1 prettytable protobuf==3.3.0 psutil==5.2.2 psycopg2==2.7.1 -py==1.4.33 -pyasn1-modules==0.0.8 +py==1.4.34 +pyasn1-modules==0.0.9 pyasn1==0.2.3 -pycparser==2.17 +pycparser==2.18 pycrypto==2.6.1 pycurl==7.43.0 pyflakes==1.5.0 pygments==2.2.0 -pyjwt==1.5.0 +pyjwt==1.5.2 pylibmc==1.5.2 -pylint==1.7.1 +pylint==1.7.2 pymongo==3.4.0 pymysql==0.7.11 -pyopenssl==17.0.0 +pyopenssl==17.1.0 pyparsing==2.2.0 -pyramid==1.8.3 +pyramid==1.9 pystache==0.5.4 pytest-cov==2.5.1 -pytest==3.0.7 +pytest==3.1.3 python-cjson==1.2.1 python-daemon==2.1.2 -python-dateutil==2.6.0 +python-dateutil==2.6.1 python-gflags==3.1.1 -python-keystoneclient==3.10.0 +python-keystoneclient==3.12.0 python-memcached==1.58 python-mimeparse==1.6.0 -python-novaclient==8.0.0 +python-novaclient==9.0.1 python-subunit==1.2.0 python-swiftclient==3.3.0 pytz==2017.2 pyyaml==3.12 pyzmq==16.0.2 -raven==6.0.0 +raven==6.1.0 redis==2.10.5 repoze.lru==0.6 requests-oauthlib==0.8.0 -requests==2.14.2 +requests==2.18.1 retrying==1.3.3 rsa==3.4.2 -scipy==0.19.0 -selenium==3.4.1 +scipy==0.19.1 +selenium==3.4.3 setuptools-git==1.2 -setuptools==35.0.2 -sh==1.12.13 -simplejson==3.10.0 +setuptools==36.0.1 +sh==1.12.14 +simplejson==3.11.1 six==1.10.0 snowballstemmer==1.2.1 south==1.0.2 -sphinx==1.5.5 +sphinx==1.6.3 sqlalchemy-migrate==0.11.0 -sqlalchemy==1.1.9 +sqlalchemy==1.1.11 sqlparse==0.2.3 statsd==3.2.1 -stevedore==1.21.0 +stevedore==1.24.0 suds==0.4 -supervisor==3.3.1 +supervisor==3.3.2 testrepository==0.0.20 testtools==2.3.0 thrift==0.10.0 tornado==4.5.1 tox==2.7.0 -twisted==17.1.0 +twisted==17.5.0 ujson==1.35 -unidecode==0.4.20 +unidecode==0.4.21 unittest2==1.1.0 uritemplate==3.0.0 urllib3==1.21.1 @@ -196,11 +195,11 @@ versiontools==1.9.1 virtualenv==15.1.0 waitress==1.0.2 warlock==1.3.0 -webob==1.7.2 -websocket-client==0.40.0 +webob==1.7.3 +websocket-client==0.44.0 webtest==2.0.27 -werkzeug==0.12.1 +werkzeug==0.12.2 wheel==0.29.0 xlrd==1.0.0 -zc.buildout==2.9.3 -zope.interface==4.4.1 +zc.buildout==2.9.4 +zope.interface==4.4.2 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index e99f3629..92261832 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -1,12 +1,12 @@ -alembic==0.9.1 +alembic==0.9.3 amqp==2.1.4 amqplib==1.0.2 -ansible==2.3.0.0 +ansible==2.3.1.0 anyjson==0.3.3 apache-libcloud==2.0.0 argparse==1.4.0 -astroid==1.5.2 -awscli==1.11.85 +astroid==1.5.3 +awscli==1.11.117 babel==2.4.0 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 @@ -14,46 +14,46 @@ beautifulsoup4==4.6.0 billiard==3.5.0.2 blessings==1.6 blinker==1.4 -boto==2.46.1 -botocore==1.5.48 +boto==2.48.0 +botocore==1.5.80 bottle==0.12.13 celery==4.0.2 certifi==2017.4.17 cffi==1.10.0 -chardet==3.0.2 +chardet==3.0.4 click==6.7 -cliff==2.7.0 -cmd2==0.7.0 +cliff==2.8.0 +cmd2==0.7.5 colorama==0.3.9 configobj==5.0.6 cov-core==1.15.0 -coverage==4.4 +coverage==4.4.1 coveralls==1.1 -cryptography==1.8.1 +cryptography==1.9 cssselect==1.0.1 cython==0.25.2 decorator==4.0.11 django-celery==3.2.1 django-debug-toolbar==1.8 -django-extensions==1.7.9 -django==1.11.1 +django-extensions==1.8.1 +django==1.11.3 django_compress==1.0.1 djangorestframework==3.6.3 docker-py==1.10.6 docopt==0.6.2 docutils==0.13.1 ecdsa==0.13 -elasticsearch==5.3.0 +elasticsearch==5.4.0 enum34==1.1.6 eventlet==0.21.0 extras==1.0.0 fabric==1.13.2 fixtures==3.0.0 flake8==3.3.0 -flask==0.12.1 +flask==0.12.2 funcsigs==1.0.2 futures==3.1.1 -gevent==1.2.1 +gevent==1.2.2 google-api-python-client==1.6.2 greenlet==0.4.12 gunicorn==19.7.1 @@ -62,17 +62,17 @@ html5lib httplib2==0.10.3 idna==2.5 ipaddress==1.0.18 -ipython==6.0.0 +ipython==6.1.0 iso8601==0.1.11 isodate==0.5.4 itsdangerous==0.24 jinja2==2.9.6 -jmespath==0.9.2 +jmespath==0.9.3 jsonschema==2.6.0 kombu==4.0.2 linecache2==1.0.0 -logilab-common==1.4.0 -lxml==3.7.3 +logilab-common==1.4.1 +lxml==3.8.0 m2crypto==0.26.0 mako==1.0.6 manifestparser==1.1 @@ -85,98 +85,98 @@ mock==2.0.0 mozcrash==1.0 mozdevice==0.50 mozfile==1.2 -mozinfo==0.9 -mozlog==3.4 +mozinfo==0.10 +mozlog==3.5 moznetwork==0.27 mozprocess==0.25 msgpack-python==0.4.8 ndg-httpsclient==0.4.2 netaddr==0.7.19 -netifaces==0.10.5 -newrelic==2.86.0.65 +netifaces==0.10.6 +newrelic==2.88.0.72 nose==1.3.7 -numpy==1.12.1 +numpy==1.13.1 oauth2==1.9.0.post1 -oauth2client==4.1.0 +oauth2client==4.1.2 oauthlib==2.0.2 ordereddict==1.1 -oslo.config==4.1.0 -pandas==0.20.1 -paramiko==2.1.2 +oslo.config==4.8.0 +pandas==0.20.3 +paramiko==2.2.1 passlib==1.7.1 paste==2.0.3 pastedeploy==1.5.2 pastescript==2.0.2 -pbr==3.0.0 +pbr==3.1.1 pep8==1.7.0 pexpect==4.2.1 pika==0.10.0 -pillow==4.1.1 +pillow==4.2.1 pip==9.0.1 prettytable protobuf==3.3.0 psutil==5.2.2 psycopg2==2.7.1 -py==1.4.33 -pyasn1-modules==0.0.8 +py==1.4.34 +pyasn1-modules==0.0.9 pyasn1==0.2.3 -pycparser==2.17 +pycparser==2.18 pycrypto==2.6.1 pyflakes==1.5.0 pygments==2.2.0 -pyjwt==1.5.0 +pyjwt==1.5.2 pylibmc==1.5.2 -pylint==1.7.1 +pylint==1.7.2 pymongo==3.4.0 pymysql==0.7.11 -pyopenssl==17.0.0 +pyopenssl==17.1.0 pyparsing==2.2.0 -pyramid==1.8.3 +pyramid==1.9 pystache==0.5.4 pytest-cov==2.5.1 -pytest==3.0.7 +pytest==3.1.3 python-daemon==2.1.2 -python-dateutil==2.6.0 +python-dateutil==2.6.1 python-gflags==3.1.1 -python-keystoneclient==3.10.0 +python-keystoneclient==3.12.0 python-memcached==1.58 python-mimeparse==1.6.0 -python-novaclient==8.0.0 +python-novaclient==9.0.1 python-subunit==1.2.0 python-swiftclient==3.3.0 pytz==2017.2 pyyaml==3.12 pyzmq==16.0.2 -raven==6.0.0 +raven==6.1.0 redis==2.10.5 repoze.lru==0.6 requests-oauthlib==0.8.0 -requests==2.14.2 +requests==2.18.1 retrying==1.3.3 rsa==3.4.2 -scipy==0.19.0 -selenium==3.4.1 +scipy==0.19.1 +selenium==3.4.3 setuptools-git==1.2 -setuptools==35.0.2 -sh==1.12.13 -simplejson==3.10.0 +setuptools==36.0.1 +sh==1.12.14 +simplejson==3.11.1 six==1.10.0 snowballstemmer==1.2.1 south==1.0.2 -sphinx==1.5.5 +sphinx==1.6.3 sqlalchemy-migrate==0.11.0 -sqlalchemy==1.1.9 +sqlalchemy==1.1.11 sqlparse==0.2.3 statsd==3.2.1 -stevedore==1.21.0 +stevedore==1.24.0 testrepository==0.0.20 testtools==2.3.0 thrift==0.10.0 tornado==4.5.1 tox==2.7.0 -twisted==17.1.0 +twisted==17.5.0 ujson==1.35 -unidecode==0.4.20 +unidecode==0.4.21 unittest2==1.1.0 uritemplate==3.0.0 urllib3==1.21.1 @@ -185,11 +185,11 @@ versiontools==1.9.1 virtualenv==15.1.0 waitress==1.0.2 warlock==1.3.0 -webob==1.7.2 -websocket-client==0.40.0 +webob==1.7.3 +websocket-client==0.44.0 webtest==2.0.27 -werkzeug==0.12.1 +werkzeug==0.12.2 wheel==0.29.0 xlrd==1.0.0 -zc.buildout==2.9.3 -zope.interface==4.4.1 +zc.buildout==2.9.4 +zope.interface==4.4.2 From 6dd33a4378d6252addefe6793f9992fbfc98720a Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Wed, 12 Jul 2017 13:26:41 -0700 Subject: [PATCH 106/256] Add a way to run just the library install tests for the DPE Gardener bot. --- .gitignore | 4 +--- build.sh | 41 ++++++++++++++++++++++++++++++----- cloudbuild_benchmark.yaml | 4 ++++ cloudbuild_library_tests.yaml | 15 +++++++++++++ cloudbuild_system_tests.yaml | 4 ++++ 5 files changed, 60 insertions(+), 8 deletions(-) create mode 100644 cloudbuild_library_tests.yaml diff --git a/.gitignore b/.gitignore index 62f3090b..73ec28f0 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,5 @@ .cache .coverage .nox -/cloudbuild.yaml_local.sh -/cloudbuild_benchmark.yaml_local.sh -/cloudbuild_system_tests.yaml_local.sh +/*_local.sh __pycache__ diff --git a/build.sh b/build.sh index 7ee1416b..82d8c05e 100755 --- a/build.sh +++ b/build.sh @@ -19,6 +19,7 @@ set -euo pipefail # Actions benchmark=0 # Should run benchmarks? build=1 # Should build images? +library_tests=0 # Should try to install top N Python libraries system_tests=0 # Should run system tests? local=0 # Should run using local Docker daemon instead of GCR? @@ -38,10 +39,11 @@ function usage { Build and test artifacts in this repository Options: - --benchmark: Run benchmarking suite - --[no]build: Build all images (default) - --local: Build images using local Docker daemon - --system_tests: Run system tests + --[no]benchmark: Run benchmarking suite (default false) + --[no]build: Build all images (default true) + --[no]library_tests: Run library compatiblity tests (default false) + --[no]local: Build images using local Docker daemon (default false) + --[no]system_tests: Run system tests (default false) " } @@ -71,6 +73,10 @@ while [ $# -gt 0 ]; do benchmark=1 shift ;; + --nobenchmark) + benchmark=0 + shift + ;; --build) build=1 shift @@ -79,14 +85,30 @@ while [ $# -gt 0 ]; do build=0 shift ;; + --library_tests) + library_tests=1 + shift + ;; + --nolibrary_tests) + library_tests=0 + shift + ;; --local) local=1 shift ;; + --nolocal) + local=0 + shift + ;; --system_tests) system_tests=1 shift ;; + --nosystem_tests) + system_tests=0 + shift + ;; *) usage ;; @@ -94,7 +116,10 @@ while [ $# -gt 0 ]; do done # If no actions chosen, then tell the user -if [ "${benchmark}" -eq 0 -a "${build}" -eq 0 -a "${system_tests}" -eq 0 ]; then +if [ "${benchmark}" -eq 0 -a \ + "${build}" -eq 0 -a \ + "${library_tests}" -eq 0 -a \ + "${system_tests}" -eq 0 ]; then fatal 'Error: No actions specified (for example, --build), exiting' fi @@ -148,6 +173,12 @@ if [ "${build}" -eq 1 ]; then ${gcloud_cmd} --config cloudbuild.yaml --substitutions "${substitutions}" fi +# Run just the library compatibility tests (for DPE Gardener bot usually) +if [ "${library_tests}" -eq 1 ]; then + echo "Testing compatibility with popular Python libraries" + ${gcloud_cmd} --config cloudbuild_library_tests.yaml --substitutions "${substitutions}" +fi + # If both system tests and benchmarks are requested, run them both # even if one or the other has errors. If the build step had errors, # this script will have already exited. diff --git a/cloudbuild_benchmark.yaml b/cloudbuild_benchmark.yaml index a960bc9b..616991ef 100644 --- a/cloudbuild_benchmark.yaml +++ b/cloudbuild_benchmark.yaml @@ -3,6 +3,10 @@ steps: - name: gcr.io/cloud-builders/docker:latest args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/benchmark:${_TAG}', '--no-cache', '/workspace/tests/benchmark/'] + env: [ + # Avoid warning about unused substitutions + 'UNUSED1=${_BUILDER_DOCKER_NAMESPACE}', + ] images: [ # Intentionally empty ] diff --git a/cloudbuild_library_tests.yaml b/cloudbuild_library_tests.yaml new file mode 100644 index 00000000..01fe7f40 --- /dev/null +++ b/cloudbuild_library_tests.yaml @@ -0,0 +1,15 @@ +steps: +- # Check that we can install important libraries without error + name: gcr.io/gcp-runtimes/structure_test:latest + args: [ + '-i', '${_DOCKER_NAMESPACE}/python:${_TAG}', + '--config', '/workspace/tests/python2-libraries/python2-libraries.yaml', + '--config', '/workspace/tests/python3-libraries/python3-libraries.yaml', + '-v' + ] + env: [ + # Avoid warning about unused substitutions + 'UNUSED1=${_BUILDER_DOCKER_NAMESPACE}', + ] +images: [ +] diff --git a/cloudbuild_system_tests.yaml b/cloudbuild_system_tests.yaml index 3cac03e2..f01a48e7 100644 --- a/cloudbuild_system_tests.yaml +++ b/cloudbuild_system_tests.yaml @@ -4,6 +4,10 @@ steps: args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/google-cloud-python-system:${_TAG}', '--no-cache', '/workspace/tests/google-cloud-python-system/'] - name: ${_DOCKER_NAMESPACE}/python/tests/google-cloud-python-system:${_TAG} + env: [ + # Avoid warning about unused substitutions + 'UNUSED1=${_BUILDER_DOCKER_NAMESPACE}', + ] images: [ # Intentionally empty ] From e0049040e1367088bec7d04d9132ea933348df18 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Wed, 12 Jul 2017 15:15:07 -0700 Subject: [PATCH 107/256] Allow $GAE_APPLICATION_YAML_PATH as an alternative to --config In some cases, gcloud sets an environment variable to indicate the location of the application configuration file, rather than using the --config flag. --- scripts/gen_dockerfile.py | 12 +++++++++++- scripts/gen_dockerfile_test.py | 21 +++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/scripts/gen_dockerfile.py b/scripts/gen_dockerfile.py index 9bfbca2a..5514b092 100755 --- a/scripts/gen_dockerfile.py +++ b/scripts/gen_dockerfile.py @@ -57,6 +57,8 @@ '3.6': '3.6', } +# Name of environment variable potentially set by gcloud +GAE_APPLICATION_YAML_PATH = 'GAE_APPLICATION_YAML_PATH' # Validated application configuration AppConfig = collections.namedtuple( @@ -225,11 +227,19 @@ def parse_args(argv): validation_utils.validate_arg_regex, flag_regex=IMAGE_REGEX), default='gcr.io/google-appengine/python:latest', help='Name of Docker image to use as base') + # In some cases, gcloud sets an environment variable to indicate + # the location of the application configuration file, rather than + # using the --config flag. The order of precedence from highest + # to lowest is: + # + # 1) --config flag + # 2) $GAE_APPLICATION_YAML_PATH environment variable + # 3) a file named "app.yaml" in the current working directory parser.add_argument( '--config', type=functools.partial( validation_utils.validate_arg_regex, flag_regex=PRINTABLE_REGEX), - default='app.yaml', + default=(os.environ.get(GAE_APPLICATION_YAML_PATH) or 'app.yaml'), help='Path to application configuration file' ) parser.add_argument( diff --git a/scripts/gen_dockerfile_test.py b/scripts/gen_dockerfile_test.py index 6a7bd33a..31ef7d42 100755 --- a/scripts/gen_dockerfile_test.py +++ b/scripts/gen_dockerfile_test.py @@ -225,5 +225,26 @@ def mock_error(*args): gen_dockerfile.parse_args(argv) +@pytest.mark.parametrize('argv, env, expected', [ + # Explicit flag wins + (['argv0', '--config=flag/path'], 'env/path', 'flag/path'), + (['argv0', '--config=flag/path'], '', 'flag/path'), + (['argv0', '--config=flag/path'], None, 'flag/path'), + # Otherwise env var wins + (['argv0'], 'env/path', 'env/path'), + # Otherwise use default name + (['argv0'], '', 'app.yaml'), + (['argv0'], None, 'app.yaml'), +]) +def test_parse_args_config(argv, env, expected): + if env is None: + mock_environ = {} + else: + mock_environ = {gen_dockerfile.GAE_APPLICATION_YAML_PATH: env} + with unittest.mock.patch.dict('os.environ', mock_environ, clear=True): + args = gen_dockerfile.parse_args(argv) + assert args.config == expected + + if __name__ == '__main__': pytest.main([__file__]) From 7e19176d3ae08f10b601b9a24bb8943c5b997171 Mon Sep 17 00:00:00 2001 From: Angela Li Date: Thu, 13 Jul 2017 16:36:08 -0700 Subject: [PATCH 108/256] Added a script for collecting the python client libraries download data (#129) Added a script for collecting the python client libraries download data --- .../python_clientlibs_download.py | 162 ++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100644 perf-dashboard/clientlibs-download/python_clientlibs_download.py diff --git a/perf-dashboard/clientlibs-download/python_clientlibs_download.py b/perf-dashboard/clientlibs-download/python_clientlibs_download.py new file mode 100644 index 00000000..888e6070 --- /dev/null +++ b/perf-dashboard/clientlibs-download/python_clientlibs_download.py @@ -0,0 +1,162 @@ +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import time +import uuid + +from google.cloud import bigquery + +DATETIME_FORMAT = '%Y%m%d' + +DATASET_NAME = 'python_clientlibs_download_by_week' + +VENEER_TABLE_NAME = 'veneer_client_libs' +STACKDRIVER_TABLE_NAME = 'stackdriver_client_libs' +GRPC_TABLE_NAME = 'grpc_lib' +TABLES = [VENEER_TABLE_NAME, GRPC_TABLE_NAME, STACKDRIVER_TABLE_NAME] + +CLIENTLIBS = { + VENEER_TABLE_NAME: [ + 'google-cloud-core', + 'google-cloud-speech', + 'google-cloud-language', + 'google-cloud-pubsub', + 'google-cloud-bigquery', + 'google-cloud-bigtable', + 'google-cloud-datastore', + 'google-cloud-spanner', + 'google-cloud-storage', + 'google-cloud-vision', + 'google-cloud-translate', + 'google-cloud-dns', + 'google-cloud-videointelligence', + ], + STACKDRIVER_TABLE_NAME: [ + 'google-cloud-logging', + 'google-cloud-monitoring', + 'google-cloud-error_reporting', + ], + GRPC_TABLE_NAME: [ + 'grpcio', + ], +} + + +def wait_for_job(job): + """Wait for the query job to complete.""" + while True: + job.reload() # Refreshes the state via a GET request. + if job.state == 'DONE': + if job.error_result: + raise RuntimeError(job.errors) + return + time.sleep(1) + + +def get_weekly_clientlibs_downloads(clientlibs_table_name, date_str): + """Use a SQL query to collect the weekly download data of the client + libraries. + + Args: + clientlibs_table_name (str): Table name, which is the key in the + CLIENTLIBS dict. + date_str (str): A date string in "YYYYMMDD" format. + + Returns: + list: rows of the query result. + """ + client_libs = CLIENTLIBS[clientlibs_table_name] + date_time = datetime.datetime.strptime(date_str, DATETIME_FORMAT) + week_dates = [(date_time + datetime.timedelta(days=-i)) + .strftime(DATETIME_FORMAT) + for i in range(7)] + query = """ + SELECT + file.project as client_library_name, + COUNT(*) as download_count + FROM + `the-psf.pypi.downloads*` + WHERE + file.project IN UNNEST(@client_libs) + AND + _TABLE_SUFFIX IN UNNEST(@week_dates) + GROUP BY client_library_name + """ + client = bigquery.Client() + query_job = client.run_async_query( + str(uuid.uuid4()), + query, + query_parameters=( + bigquery.ArrayQueryParameter( + 'client_libs', 'STRING', + client_libs), + bigquery.ArrayQueryParameter( + 'week_dates', 'STRING', + week_dates) + )) + query_job.use_legacy_sql = False + + # Start the query job and wait it to complete + query_job.begin() + wait_for_job(query_job) + + # Fetch the results + result = query_job.results().fetch_data() + result_list = [item for item in result] + + # In case the result_list contains the metadata like total_rows, the + # actual rows will be the first element of the result_list. + if len(result_list) > 0 and isinstance(result_list[0], list): + result_list = result_list[0] + + rows = [(date_time,) + row for row in result_list] + print rows + + return rows + + +def insert_rows(dataset_name, table_name, rows): + """Insert rows to a bigquery table. + + Args: + dataset_name (str): Name of the dataset that holds the tables. + table_name (str): Name of the bigquery table. + rows (list): The rows that going to be inserted into the table. + + Returns: + list: Empty if inserted successfully, else the errors when inserting + each row. + """ + client = bigquery.Client() + dataset = client.dataset(dataset_name) + table = bigquery.Table(name=table_name, dataset=dataset) + table.reload() + error = table.insert_data(rows) + return error + + +def main(): + for table_name in CLIENTLIBS.keys(): + rows = get_weekly_clientlibs_downloads( + clientlibs_table_name=table_name, + date_str=datetime.datetime.now().strftime("%Y%m%d")) + insert_rows( + dataset_name=DATASET_NAME, + table_name=table_name, + rows=rows) + + +if __name__ == '__main__': + main() From fa68b87ddccebdd5727329093a27e26284a355f3 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Mon, 17 Jul 2017 01:41:31 -0700 Subject: [PATCH 109/256] Auto-update dependencies. --- tests/python2-libraries/requirements.txt | 18 +++++++++--------- tests/python3-libraries/requirements.txt | 18 +++++++++--------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 2e51b07f..5b879a08 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -1,22 +1,22 @@ alembic==0.9.3 -amqp==2.1.4 +amqp==2.2.1 amqplib==1.0.2 ansible==2.3.1.0 anyjson==0.3.3 apache-libcloud==2.0.0 argparse==1.4.0 astroid==1.5.3 -awscli==1.11.117 +awscli==1.11.120 babel==2.4.0 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 beautifulsoup4==4.6.0 beautifulsoup==3.2.1 -billiard==3.5.0.2 +billiard==3.5.0.3 blessings==1.6 blinker==1.4 boto==2.48.0 -botocore==1.5.80 +botocore==1.5.83 bottle==0.12.13 carbon==1.0.2 celery==4.0.2 @@ -34,7 +34,7 @@ coveralls==1.1 cryptography==1.9 cssselect==1.0.1 cython==0.25.2 -decorator==4.0.11 +decorator==4.1.1 django-celery==3.2.1 django-debug-toolbar==1.8 django-extensions==1.8.1 @@ -77,7 +77,7 @@ linecache2==1.0.0 logilab-common==1.4.1 lxml==3.8.0 m2crypto==0.26.0 -mako==1.0.6 +mako==1.0.7 manifestparser==1.1 markdown==2.6.8 markupsafe==1.0 @@ -99,7 +99,7 @@ mysql-python==1.2.5 ndg-httpsclient==0.4.2 netaddr==0.7.19 netifaces==0.10.6 -newrelic==2.88.0.72 +newrelic==2.88.1.73 nose==1.3.7 numpy==1.13.1 oauth2==1.9.0.post1 @@ -138,7 +138,7 @@ pymongo==3.4.0 pymysql==0.7.11 pyopenssl==17.1.0 pyparsing==2.2.0 -pyramid==1.9 +pyramid==1.9.1 pystache==0.5.4 pytest-cov==2.5.1 pytest==3.1.3 @@ -165,7 +165,7 @@ rsa==3.4.2 scipy==0.19.1 selenium==3.4.3 setuptools-git==1.2 -setuptools==36.0.1 +setuptools==36.2.0 sh==1.12.14 simplejson==3.11.1 six==1.10.0 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 92261832..f4559cb0 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -1,21 +1,21 @@ alembic==0.9.3 -amqp==2.1.4 +amqp==2.2.1 amqplib==1.0.2 ansible==2.3.1.0 anyjson==0.3.3 apache-libcloud==2.0.0 argparse==1.4.0 astroid==1.5.3 -awscli==1.11.117 +awscli==1.11.120 babel==2.4.0 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 beautifulsoup4==4.6.0 -billiard==3.5.0.2 +billiard==3.5.0.3 blessings==1.6 blinker==1.4 boto==2.48.0 -botocore==1.5.80 +botocore==1.5.83 bottle==0.12.13 celery==4.0.2 certifi==2017.4.17 @@ -32,7 +32,7 @@ coveralls==1.1 cryptography==1.9 cssselect==1.0.1 cython==0.25.2 -decorator==4.0.11 +decorator==4.1.1 django-celery==3.2.1 django-debug-toolbar==1.8 django-extensions==1.8.1 @@ -74,7 +74,7 @@ linecache2==1.0.0 logilab-common==1.4.1 lxml==3.8.0 m2crypto==0.26.0 -mako==1.0.6 +mako==1.0.7 manifestparser==1.1 markdown==2.6.8 markupsafe==1.0 @@ -93,7 +93,7 @@ msgpack-python==0.4.8 ndg-httpsclient==0.4.2 netaddr==0.7.19 netifaces==0.10.6 -newrelic==2.88.0.72 +newrelic==2.88.1.73 nose==1.3.7 numpy==1.13.1 oauth2==1.9.0.post1 @@ -131,7 +131,7 @@ pymongo==3.4.0 pymysql==0.7.11 pyopenssl==17.1.0 pyparsing==2.2.0 -pyramid==1.9 +pyramid==1.9.1 pystache==0.5.4 pytest-cov==2.5.1 pytest==3.1.3 @@ -157,7 +157,7 @@ rsa==3.4.2 scipy==0.19.1 selenium==3.4.3 setuptools-git==1.2 -setuptools==36.0.1 +setuptools==36.2.0 sh==1.12.14 simplejson==3.11.1 six==1.10.0 From 62e80bf702e3568df0d03646334bbe1799722d21 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Mon, 17 Jul 2017 13:08:48 -0700 Subject: [PATCH 110/256] Increase timeout for library compatibility test --- cloudbuild_library_tests.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/cloudbuild_library_tests.yaml b/cloudbuild_library_tests.yaml index 01fe7f40..846348fe 100644 --- a/cloudbuild_library_tests.yaml +++ b/cloudbuild_library_tests.yaml @@ -1,3 +1,4 @@ +timeout: 1800s steps: - # Check that we can install important libraries without error name: gcr.io/gcp-runtimes/structure_test:latest From 66e5264c2d71c8b2f8acc80fe06cbbf74ac850fa Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Mon, 17 Jul 2017 17:57:00 -0700 Subject: [PATCH 111/256] Switch image from Python 3.6.1 to 3.6.2 --- .../patches/3.6/double-build.diff | 96 ------------------- python-interpreter-builder/patches/3.6/series | 1 - .../scripts/build-python-3.6.sh | 14 ++- tests/virtualenv/virtualenv_python36.yaml | 2 +- 4 files changed, 7 insertions(+), 106 deletions(-) delete mode 100644 python-interpreter-builder/patches/3.6/double-build.diff delete mode 100644 python-interpreter-builder/patches/3.6/series diff --git a/python-interpreter-builder/patches/3.6/double-build.diff b/python-interpreter-builder/patches/3.6/double-build.diff deleted file mode 100644 index a89a20f6..00000000 --- a/python-interpreter-builder/patches/3.6/double-build.diff +++ /dev/null @@ -1,96 +0,0 @@ -# Source is https://github.com/python/cpython/pull/1478 - -Index: Python-3.6.1/Makefile.pre.in -=================================================================== ---- Python-3.6.1.orig/Makefile.pre.in -+++ Python-3.6.1/Makefile.pre.in -@@ -1000,7 +1000,7 @@ TESTTIMEOUT= 1200 - - # Run a basic set of regression tests. - # This excludes some tests that are particularly resource-intensive. --test: all platform -+test: @DEF_MAKE_RULE@ platform - $(TESTRUNNER) $(TESTOPTS) - - # Run the full test suite twice - once without .pyc files, and once with. -@@ -1010,7 +1010,7 @@ test: all platform - # the bytecode read from a .pyc file had the bug, sometimes the directly - # generated bytecode. This is sometimes a very shy bug needing a lot of - # sample data. --testall: all platform -+testall: @DEF_MAKE_RULE@ platform - -find $(srcdir)/Lib -name '*.py[co]' -print | xargs rm -f - $(TESTPYTHON) -E $(srcdir)/Lib/compileall.py - -find $(srcdir)/Lib -name '*.py[co]' -print | xargs rm -f -@@ -1019,7 +1019,7 @@ testall: all platform - - # Run the test suite for both architectures in a Universal build on OSX. - # Must be run on an Intel box. --testuniversal: all platform -+testuniversal: @DEF_MAKE_RULE@ platform - if [ `arch` != 'i386' ];then \ - echo "This can only be used on OSX/i386" ;\ - exit 1 ;\ -@@ -1042,7 +1042,7 @@ QUICKTESTOPTS= $(TESTOPTS) -x test_subpr - test_multiprocessing_forkserver \ - test_mailbox test_socket test_poll \ - test_select test_zipfile test_concurrent_futures --quicktest: all platform -+quicktest: @DEF_MAKE_RULE@ platform - $(TESTRUNNER) $(QUICKTESTOPTS) - - -@@ -1379,7 +1379,7 @@ LIBPL= @LIBPL@ - # pkgconfig directory - LIBPC= $(LIBDIR)/pkgconfig - --libainstall: all python-config -+libainstall: @DEF_MAKE_RULE@ python-config - @for i in $(LIBDIR) $(LIBPL) $(LIBPC); \ - do \ - if test ! -d $(DESTDIR)$$i; then \ -@@ -1639,7 +1639,7 @@ distclean: clobber - -exec rm -f {} ';' - - # Check for smelly exported symbols (not starting with Py/_Py) --smelly: all -+smelly: @DEF_MAKE_RULE@ - nm -p $(LIBRARY) | \ - sed -n "/ [TDB] /s/.* //p" | grep -v "^_*Py" | sort -u; \ - -@@ -1676,7 +1676,7 @@ funny: - -o -print - - # Perform some verification checks on any modified files. --patchcheck: all -+patchcheck: @DEF_MAKE_RULE@ - $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/scripts/patchcheck.py - - # Dependencies -Index: Python-3.6.1/Misc/ACKS -=================================================================== ---- Python-3.6.1.orig/Misc/ACKS -+++ Python-3.6.1/Misc/ACKS -@@ -1111,6 +1111,7 @@ Jason Orendorff - Douglas Orr - William Orr - Michele Orrù -+Tomáš Orsava - Oleg Oshmyan - Denis S. Otkidach - Peter Otten -Index: Python-3.6.1/Misc/NEWS -=================================================================== ---- Python-3.6.1.orig/Misc/NEWS -+++ Python-3.6.1/Misc/NEWS -@@ -306,6 +306,10 @@ Tests - Build - ----- - -+- bpo-29243: Prevent unnecessary rebuilding of Python during ``make test``, -+ ``make install`` and some other make targets when configured with -+ ``--enable-optimizations``. -+ - - bpo-27593: sys.version and the platform module python_build(), - python_branch(), and python_revision() functions now use - git information rather than hg when building from a repo. diff --git a/python-interpreter-builder/patches/3.6/series b/python-interpreter-builder/patches/3.6/series deleted file mode 100644 index f6c2876b..00000000 --- a/python-interpreter-builder/patches/3.6/series +++ /dev/null @@ -1 +0,0 @@ -double-build.diff diff --git a/python-interpreter-builder/scripts/build-python-3.6.sh b/python-interpreter-builder/scripts/build-python-3.6.sh index d3eb0c33..431c305d 100755 --- a/python-interpreter-builder/scripts/build-python-3.6.sh +++ b/python-interpreter-builder/scripts/build-python-3.6.sh @@ -6,16 +6,14 @@ set -x # Get the source mkdir -p /opt/sources cd /opt/sources -wget --no-verbose https://www.python.org/ftp/python/3.6.1/Python-3.6.1.tgz +wget --no-verbose https://www.python.org/ftp/python/3.6.2/Python-3.6.2.tgz # SHA-256 generated via `shasum -a 256 [file]` shasum --check < Date: Wed, 19 Jul 2017 09:37:39 -0700 Subject: [PATCH 112/256] Explicitly set the gcloud project when creating bigquery client (#134) Explicitly set the gcloud project when creating bigquery client --- .../clientlibs-download/python_clientlibs_download.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/perf-dashboard/clientlibs-download/python_clientlibs_download.py b/perf-dashboard/clientlibs-download/python_clientlibs_download.py index 888e6070..e93a6f93 100644 --- a/perf-dashboard/clientlibs-download/python_clientlibs_download.py +++ b/perf-dashboard/clientlibs-download/python_clientlibs_download.py @@ -13,11 +13,14 @@ # limitations under the License. import datetime +import os import time import uuid from google.cloud import bigquery +GCLOUD_PROJECT_ENV = 'GCLOUD_PROJECT' + DATETIME_FORMAT = '%Y%m%d' DATASET_NAME = 'python_clientlibs_download_by_week' @@ -70,7 +73,7 @@ def get_weekly_clientlibs_downloads(clientlibs_table_name, date_str): libraries. Args: - clientlibs_table_name (str): Table name, which is the key in the + clientlibs_table_name (str): Table name, which is the key in the CLIENTLIBS dict. date_str (str): A date string in "YYYYMMDD" format. @@ -139,7 +142,8 @@ def insert_rows(dataset_name, table_name, rows): list: Empty if inserted successfully, else the errors when inserting each row. """ - client = bigquery.Client() + project = os.environ.get(GCLOUD_PROJECT_ENV) + client = bigquery.Client(project=project) dataset = client.dataset(dataset_name) table = bigquery.Table(name=table_name, dataset=dataset) table.reload() From 0bdbf2b422b7627fea36c13875d575d3647790de Mon Sep 17 00:00:00 2001 From: Angela Li Date: Wed, 9 Aug 2017 13:38:57 -0700 Subject: [PATCH 113/256] Update the dashboard script to match the bigquery library changes (#137) --- .../clientlibs-download/python_clientlibs_download.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/perf-dashboard/clientlibs-download/python_clientlibs_download.py b/perf-dashboard/clientlibs-download/python_clientlibs_download.py index e93a6f93..fcfb048e 100644 --- a/perf-dashboard/clientlibs-download/python_clientlibs_download.py +++ b/perf-dashboard/clientlibs-download/python_clientlibs_download.py @@ -116,7 +116,7 @@ def get_weekly_clientlibs_downloads(clientlibs_table_name, date_str): wait_for_job(query_job) # Fetch the results - result = query_job.results().fetch_data() + result = query_job.result().fetch_data() result_list = [item for item in result] # In case the result_list contains the metadata like total_rows, the @@ -125,7 +125,7 @@ def get_weekly_clientlibs_downloads(clientlibs_table_name, date_str): result_list = result_list[0] rows = [(date_time,) + row for row in result_list] - print rows + print(rows) return rows From 7002a73e7ce80b7b594f2f22cafe420e6910f217 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Thu, 10 Aug 2017 01:41:54 -0700 Subject: [PATCH 114/256] Auto-update dependencies. --- scripts/requirements-test.txt | 2 +- tests/integration/requirements.txt | 8 +-- tests/python2-libraries/requirements.txt | 68 ++++++++++++------------ tests/python3-libraries/requirements.txt | 66 +++++++++++------------ 4 files changed, 72 insertions(+), 72 deletions(-) diff --git a/scripts/requirements-test.txt b/scripts/requirements-test.txt index d1db042b..6e01a83d 100644 --- a/scripts/requirements-test.txt +++ b/scripts/requirements-test.txt @@ -1,4 +1,4 @@ flask==0.12.2 -pytest==3.1.3 +pytest==3.2.1 pytest-cov==2.5.1 pyyaml==3.12 diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index ee8d85c3..05779ae6 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -1,7 +1,7 @@ Flask==0.12.2 -google-cloud-error-reporting==0.25.1 -google-cloud-logging==1.1.0 -google-cloud-monitoring==0.25.0 +google-cloud-error-reporting==0.26.0 +google-cloud-logging==1.2.0 +google-cloud-monitoring==0.26.0 gunicorn==19.7.1 -requests==2.18.1 +requests==2.18.3 retrying==1.3.3 diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 5b879a08..05a6dbe3 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -1,12 +1,12 @@ -alembic==0.9.3 +alembic==0.9.5 amqp==2.2.1 amqplib==1.0.2 -ansible==2.3.1.0 +ansible==2.3.2.0 anyjson==0.3.3 -apache-libcloud==2.0.0 +apache-libcloud==2.1.0 argparse==1.4.0 astroid==1.5.3 -awscli==1.11.120 +awscli==1.11.131 babel==2.4.0 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 @@ -16,11 +16,11 @@ billiard==3.5.0.3 blessings==1.6 blinker==1.4 boto==2.48.0 -botocore==1.5.83 +botocore==1.5.94 bottle==0.12.13 carbon==1.0.2 -celery==4.0.2 -certifi==2017.4.17 +celery==4.1.0 +certifi==2017.7.27.1 cffi==1.10.0 chardet==3.0.4 click==6.7 @@ -31,19 +31,19 @@ configobj==5.0.6 cov-core==1.15.0 coverage==4.4.1 coveralls==1.1 -cryptography==1.9 +cryptography==2.0.3 cssselect==1.0.1 -cython==0.25.2 -decorator==4.1.1 +cython==0.26 +decorator==4.1.2 django-celery==3.2.1 django-debug-toolbar==1.8 django-extensions==1.8.1 -django==1.11.3 +django==1.11.4 django_compress==1.0.1 djangorestframework==3.6.3 docker-py==1.10.6 docopt==0.6.2 -docutils==0.13.1 +docutils==0.14 ecdsa==0.13 elasticsearch==5.4.0 enum34==1.1.6 @@ -51,7 +51,7 @@ eventlet==0.21.0 extras==1.0.0 fabric==1.13.2 fixtures==3.0.0 -flake8==3.3.0 +flake8==3.4.1 flask==0.12.2 funcsigs==1.0.2 functools32==3.2.3.post2 @@ -64,15 +64,15 @@ gunicorn==19.7.1 hiredis==0.2.0 html5lib httplib2==0.10.3 -idna==2.5 +idna==2.6 ipaddress==1.0.18 -iso8601==0.1.11 +iso8601==0.1.12 isodate==0.5.4 itsdangerous==0.24 jinja2==2.9.6 jmespath==0.9.3 jsonschema==2.6.0 -kombu==4.0.2 +kombu==4.1.0 linecache2==1.0.0 logilab-common==1.4.1 lxml==3.8.0 @@ -99,14 +99,14 @@ mysql-python==1.2.5 ndg-httpsclient==0.4.2 netaddr==0.7.19 netifaces==0.10.6 -newrelic==2.88.1.73 +newrelic==2.90.0.75 nose==1.3.7 numpy==1.13.1 oauth2==1.9.0.post1 oauth2client==4.1.2 oauthlib==2.0.2 ordereddict==1.1 -oslo.config==4.8.0 +oslo.config==4.11.0 pandas==0.20.3 paramiko==2.2.1 passlib==1.7.1 @@ -122,36 +122,36 @@ pip==9.0.1 prettytable protobuf==3.3.0 psutil==5.2.2 -psycopg2==2.7.1 +psycopg2==2.7.3 py==1.4.34 -pyasn1-modules==0.0.9 -pyasn1==0.2.3 +pyasn1-modules==0.0.11 +pyasn1==0.3.2 pycparser==2.18 pycrypto==2.6.1 pycurl==7.43.0 -pyflakes==1.5.0 +pyflakes==1.6.0 pygments==2.2.0 pyjwt==1.5.2 pylibmc==1.5.2 pylint==1.7.2 -pymongo==3.4.0 +pymongo==3.5.0 pymysql==0.7.11 -pyopenssl==17.1.0 +pyopenssl==17.2.0 pyparsing==2.2.0 pyramid==1.9.1 pystache==0.5.4 pytest-cov==2.5.1 -pytest==3.1.3 +pytest==3.2.1 python-cjson==1.2.1 python-daemon==2.1.2 python-dateutil==2.6.1 python-gflags==3.1.1 -python-keystoneclient==3.12.0 +python-keystoneclient==3.13.0 python-memcached==1.58 python-mimeparse==1.6.0 -python-novaclient==9.0.1 +python-novaclient==9.1.0 python-subunit==1.2.0 -python-swiftclient==3.3.0 +python-swiftclient==3.4.0 pytz==2017.2 pyyaml==3.12 pyzmq==16.0.2 @@ -159,13 +159,13 @@ raven==6.1.0 redis==2.10.5 repoze.lru==0.6 requests-oauthlib==0.8.0 -requests==2.18.1 +requests==2.18.3 retrying==1.3.3 rsa==3.4.2 scipy==0.19.1 selenium==3.4.3 setuptools-git==1.2 -setuptools==36.2.0 +setuptools==36.2.7 sh==1.12.14 simplejson==3.11.1 six==1.10.0 @@ -176,9 +176,9 @@ sqlalchemy-migrate==0.11.0 sqlalchemy==1.1.11 sqlparse==0.2.3 statsd==3.2.1 -stevedore==1.24.0 +stevedore==1.25.0 suds==0.4 -supervisor==3.3.2 +supervisor==3.3.3 testrepository==0.0.20 testtools==2.3.0 thrift==0.10.0 @@ -189,7 +189,7 @@ ujson==1.35 unidecode==0.4.21 unittest2==1.1.0 uritemplate==3.0.0 -urllib3==1.21.1 +urllib3==1.22 uwsgi==2.0.15 versiontools==1.9.1 virtualenv==15.1.0 @@ -197,7 +197,7 @@ waitress==1.0.2 warlock==1.3.0 webob==1.7.3 websocket-client==0.44.0 -webtest==2.0.27 +webtest==2.0.28 werkzeug==0.12.2 wheel==0.29.0 xlrd==1.0.0 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index f4559cb0..f1e3539b 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -1,12 +1,12 @@ -alembic==0.9.3 +alembic==0.9.5 amqp==2.2.1 amqplib==1.0.2 -ansible==2.3.1.0 +ansible==2.3.2.0 anyjson==0.3.3 -apache-libcloud==2.0.0 +apache-libcloud==2.1.0 argparse==1.4.0 astroid==1.5.3 -awscli==1.11.120 +awscli==1.11.131 babel==2.4.0 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 @@ -15,10 +15,10 @@ billiard==3.5.0.3 blessings==1.6 blinker==1.4 boto==2.48.0 -botocore==1.5.83 +botocore==1.5.94 bottle==0.12.13 -celery==4.0.2 -certifi==2017.4.17 +celery==4.1.0 +certifi==2017.7.27.1 cffi==1.10.0 chardet==3.0.4 click==6.7 @@ -29,19 +29,19 @@ configobj==5.0.6 cov-core==1.15.0 coverage==4.4.1 coveralls==1.1 -cryptography==1.9 +cryptography==2.0.3 cssselect==1.0.1 -cython==0.25.2 -decorator==4.1.1 +cython==0.26 +decorator==4.1.2 django-celery==3.2.1 django-debug-toolbar==1.8 django-extensions==1.8.1 -django==1.11.3 +django==1.11.4 django_compress==1.0.1 djangorestframework==3.6.3 docker-py==1.10.6 docopt==0.6.2 -docutils==0.13.1 +docutils==0.14 ecdsa==0.13 elasticsearch==5.4.0 enum34==1.1.6 @@ -49,7 +49,7 @@ eventlet==0.21.0 extras==1.0.0 fabric==1.13.2 fixtures==3.0.0 -flake8==3.3.0 +flake8==3.4.1 flask==0.12.2 funcsigs==1.0.2 futures==3.1.1 @@ -60,16 +60,16 @@ gunicorn==19.7.1 hiredis==0.2.0 html5lib httplib2==0.10.3 -idna==2.5 +idna==2.6 ipaddress==1.0.18 ipython==6.1.0 -iso8601==0.1.11 +iso8601==0.1.12 isodate==0.5.4 itsdangerous==0.24 jinja2==2.9.6 jmespath==0.9.3 jsonschema==2.6.0 -kombu==4.0.2 +kombu==4.1.0 linecache2==1.0.0 logilab-common==1.4.1 lxml==3.8.0 @@ -93,14 +93,14 @@ msgpack-python==0.4.8 ndg-httpsclient==0.4.2 netaddr==0.7.19 netifaces==0.10.6 -newrelic==2.88.1.73 +newrelic==2.90.0.75 nose==1.3.7 numpy==1.13.1 oauth2==1.9.0.post1 oauth2client==4.1.2 oauthlib==2.0.2 ordereddict==1.1 -oslo.config==4.8.0 +oslo.config==4.11.0 pandas==0.20.3 paramiko==2.2.1 passlib==1.7.1 @@ -116,34 +116,34 @@ pip==9.0.1 prettytable protobuf==3.3.0 psutil==5.2.2 -psycopg2==2.7.1 +psycopg2==2.7.3 py==1.4.34 -pyasn1-modules==0.0.9 -pyasn1==0.2.3 +pyasn1-modules==0.0.11 +pyasn1==0.3.2 pycparser==2.18 pycrypto==2.6.1 -pyflakes==1.5.0 +pyflakes==1.6.0 pygments==2.2.0 pyjwt==1.5.2 pylibmc==1.5.2 pylint==1.7.2 -pymongo==3.4.0 +pymongo==3.5.0 pymysql==0.7.11 -pyopenssl==17.1.0 +pyopenssl==17.2.0 pyparsing==2.2.0 pyramid==1.9.1 pystache==0.5.4 pytest-cov==2.5.1 -pytest==3.1.3 +pytest==3.2.1 python-daemon==2.1.2 python-dateutil==2.6.1 python-gflags==3.1.1 -python-keystoneclient==3.12.0 +python-keystoneclient==3.13.0 python-memcached==1.58 python-mimeparse==1.6.0 -python-novaclient==9.0.1 +python-novaclient==9.1.0 python-subunit==1.2.0 -python-swiftclient==3.3.0 +python-swiftclient==3.4.0 pytz==2017.2 pyyaml==3.12 pyzmq==16.0.2 @@ -151,13 +151,13 @@ raven==6.1.0 redis==2.10.5 repoze.lru==0.6 requests-oauthlib==0.8.0 -requests==2.18.1 +requests==2.18.3 retrying==1.3.3 rsa==3.4.2 scipy==0.19.1 selenium==3.4.3 setuptools-git==1.2 -setuptools==36.2.0 +setuptools==36.2.7 sh==1.12.14 simplejson==3.11.1 six==1.10.0 @@ -168,7 +168,7 @@ sqlalchemy-migrate==0.11.0 sqlalchemy==1.1.11 sqlparse==0.2.3 statsd==3.2.1 -stevedore==1.24.0 +stevedore==1.25.0 testrepository==0.0.20 testtools==2.3.0 thrift==0.10.0 @@ -179,7 +179,7 @@ ujson==1.35 unidecode==0.4.21 unittest2==1.1.0 uritemplate==3.0.0 -urllib3==1.21.1 +urllib3==1.22 uwsgi==2.0.15 versiontools==1.9.1 virtualenv==15.1.0 @@ -187,7 +187,7 @@ waitress==1.0.2 warlock==1.3.0 webob==1.7.3 websocket-client==0.44.0 -webtest==2.0.27 +webtest==2.0.28 werkzeug==0.12.2 wheel==0.29.0 xlrd==1.0.0 From bc20cb49aeb5d018698a9ca4c870116b772e2722 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Thu, 10 Aug 2017 16:56:40 -0700 Subject: [PATCH 115/256] Add 'netbase' package needed by eventlet package --- runtime-image/resources/apt-packages.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/runtime-image/resources/apt-packages.txt b/runtime-image/resources/apt-packages.txt index 4bb6d6fc..1fa11bcb 100644 --- a/runtime-image/resources/apt-packages.txt +++ b/runtime-image/resources/apt-packages.txt @@ -36,3 +36,5 @@ libsasl2-2 libsasl2-dev libsasl2-modules sasl2-bin +# Needed by eventlet +netbase From 50e51f682cd21d874af9361efe8cb4423cb87a6a Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Thu, 10 Aug 2017 17:04:12 -0700 Subject: [PATCH 116/256] Add compatibility test for the 'eventlet' Python package. The eventlet worker for gunicorn dies if /etc/protocols isn't present, which was previously the case. We now install the Debian 'netbase' package which provides this file and others. --- build.sh | 4 ++++ cloudbuild.yaml | 4 ++++ cloudbuild_library_tests.yaml | 4 ++++ tests/eventlet/.gitignore | 2 ++ tests/eventlet/Dockerfile.in | 13 +++++++++++++ tests/eventlet/README.md | 1 + tests/eventlet/requirements.txt | 10 ++++++++++ 7 files changed, 38 insertions(+) create mode 100644 tests/eventlet/.gitignore create mode 100644 tests/eventlet/Dockerfile.in create mode 100644 tests/eventlet/README.md create mode 100644 tests/eventlet/requirements.txt diff --git a/build.sh b/build.sh index 82d8c05e..2f0595e9 100755 --- a/build.sh +++ b/build.sh @@ -150,6 +150,7 @@ for outfile in \ python-interpreter-builder/Dockerfile \ runtime-image/Dockerfile \ tests/benchmark/Dockerfile \ + tests/eventlet/Dockerfile \ tests/google-cloud-python/Dockerfile \ tests/google-cloud-python-system/Dockerfile \ tests/integration/Dockerfile \ @@ -167,6 +168,9 @@ for file in \ cp -a "${file}" "builder/gen-dockerfile/${file##scripts/}" done +# Make a file available to the eventlet test. +cp -a scripts/testdata/hello_world/main.py tests/eventlet/main.py + # Build images and push to GCR if [ "${build}" -eq 1 ]; then echo "Building images" diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 187884dc..68b5c5bb 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -25,6 +25,10 @@ steps: '--config', '/workspace/tests/license-test/license-test.yaml', '-v' ] +- # Run compatibility tests + name: gcr.io/cloud-builders/docker:latest + args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/eventlet:${_TAG}', + '--no-cache', '/workspace/tests/eventlet/'] - # Build image to run google client library unit tests name: gcr.io/cloud-builders/docker:latest args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/google-cloud-python:${_TAG}', diff --git a/cloudbuild_library_tests.yaml b/cloudbuild_library_tests.yaml index 846348fe..f547d73f 100644 --- a/cloudbuild_library_tests.yaml +++ b/cloudbuild_library_tests.yaml @@ -12,5 +12,9 @@ steps: # Avoid warning about unused substitutions 'UNUSED1=${_BUILDER_DOCKER_NAMESPACE}', ] +- # Run compatibility tests + name: gcr.io/cloud-builders/docker:latest + args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/eventlet:${_TAG}', + '--no-cache', '/workspace/tests/eventlet/'] images: [ ] diff --git a/tests/eventlet/.gitignore b/tests/eventlet/.gitignore new file mode 100644 index 00000000..14be8077 --- /dev/null +++ b/tests/eventlet/.gitignore @@ -0,0 +1,2 @@ +Dockerfile +main.py diff --git a/tests/eventlet/Dockerfile.in b/tests/eventlet/Dockerfile.in new file mode 100644 index 00000000..fa65a236 --- /dev/null +++ b/tests/eventlet/Dockerfile.in @@ -0,0 +1,13 @@ +FROM ${STAGING_IMAGE} +LABEL python_version=python3.6 +RUN virtualenv --no-download /env -p python3.6 + +# Set virtualenv environment variables. This is equivalent to running +# source /env/bin/activate +ENV VIRTUAL_ENV /env +ENV PATH /env/bin:$PATH +ADD requirements.txt /app/ +RUN pip install -r requirements.txt +ADD . /app/ +RUN gunicorn -k eventlet -b :$PORT --daemon main:app ; \ + wget --retry-connrefused --tries=5 http://localhost:$PORT/ diff --git a/tests/eventlet/README.md b/tests/eventlet/README.md new file mode 100644 index 00000000..0c2a969c --- /dev/null +++ b/tests/eventlet/README.md @@ -0,0 +1 @@ +# Test the Python base image against the 'eventlet' library diff --git a/tests/eventlet/requirements.txt b/tests/eventlet/requirements.txt new file mode 100644 index 00000000..c33e53f2 --- /dev/null +++ b/tests/eventlet/requirements.txt @@ -0,0 +1,10 @@ +click==6.7 +enum-compat==0.0.2 +eventlet==0.21.0 +Flask==0.12.2 +greenlet==0.4.12 +gunicorn==19.7.1 +itsdangerous==0.24 +Jinja2==2.9.6 +MarkupSafe==1.0 +Werkzeug==0.12.2 From d287c50143d05dee1fe3ba9b3d93fd1f61948848 Mon Sep 17 00:00:00 2001 From: Angela Li Date: Mon, 21 Aug 2017 16:57:44 -0700 Subject: [PATCH 117/256] Add pandas-gbq to clientlibs download metrics (#141) * Add pandas-gbq to clientlibs download metrics * Create table for third party client libs --- .../python_clientlibs_download.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/perf-dashboard/clientlibs-download/python_clientlibs_download.py b/perf-dashboard/clientlibs-download/python_clientlibs_download.py index fcfb048e..350eb7ed 100644 --- a/perf-dashboard/clientlibs-download/python_clientlibs_download.py +++ b/perf-dashboard/clientlibs-download/python_clientlibs_download.py @@ -28,7 +28,14 @@ VENEER_TABLE_NAME = 'veneer_client_libs' STACKDRIVER_TABLE_NAME = 'stackdriver_client_libs' GRPC_TABLE_NAME = 'grpc_lib' -TABLES = [VENEER_TABLE_NAME, GRPC_TABLE_NAME, STACKDRIVER_TABLE_NAME] +THIRD_PARTY_TABLE_NAME = 'third_party_client_libs' + +TABLES = [ + VENEER_TABLE_NAME, + GRPC_TABLE_NAME, + STACKDRIVER_TABLE_NAME, + THIRD_PARTY_TABLE_NAME, +] CLIENTLIBS = { VENEER_TABLE_NAME: [ @@ -54,6 +61,9 @@ GRPC_TABLE_NAME: [ 'grpcio', ], + THIRD_PARTY_TABLE_NAME: [ + 'pandas-gbq', + ] } From f89776254ebcf82c4770ba30e79e7d3458492d75 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Tue, 22 Aug 2017 18:02:52 -0700 Subject: [PATCH 118/256] Update Python interpreter from 3.5.3 to 3.5.4 --- python-interpreter-builder/Dockerfile.in | 1 - .../patches/3.5/double-build.diff | 96 ------------------- python-interpreter-builder/patches/3.5/series | 1 - .../scripts/build-python-3.5.sh | 14 ++- tests/virtualenv/virtualenv_python35.yaml | 2 +- 5 files changed, 7 insertions(+), 107 deletions(-) delete mode 100644 python-interpreter-builder/patches/3.5/double-build.diff delete mode 100644 python-interpreter-builder/patches/3.5/series diff --git a/python-interpreter-builder/Dockerfile.in b/python-interpreter-builder/Dockerfile.in index ee9fecf3..83640de3 100644 --- a/python-interpreter-builder/Dockerfile.in +++ b/python-interpreter-builder/Dockerfile.in @@ -29,7 +29,6 @@ RUN apt-get update && apt-get install -yq \ net-tools \ netbase \ python3 \ - quilt \ sharutils \ time \ tk-dev \ diff --git a/python-interpreter-builder/patches/3.5/double-build.diff b/python-interpreter-builder/patches/3.5/double-build.diff deleted file mode 100644 index cb52c64b..00000000 --- a/python-interpreter-builder/patches/3.5/double-build.diff +++ /dev/null @@ -1,96 +0,0 @@ -# Source is https://github.com/python/cpython/pull/1478 - -Index: Python-3.5.3/Makefile.pre.in -=================================================================== ---- Python-3.5.3.orig/Makefile.pre.in -+++ Python-3.5.3/Makefile.pre.in -@@ -982,7 +982,7 @@ TESTTIMEOUT= 3600 - - # Run a basic set of regression tests. - # This excludes some tests that are particularly resource-intensive. --test: all platform -+test: @DEF_MAKE_RULE@ platform - $(TESTRUNNER) $(TESTOPTS) - - # Run the full test suite twice - once without .pyc files, and once with. -@@ -992,7 +992,7 @@ test: all platform - # the bytecode read from a .pyc file had the bug, sometimes the directly - # generated bytecode. This is sometimes a very shy bug needing a lot of - # sample data. --testall: all platform -+testall: @DEF_MAKE_RULE@ platform - -find $(srcdir)/Lib -name '*.py[co]' -print | xargs rm -f - $(TESTPYTHON) -E $(srcdir)/Lib/compileall.py - -find $(srcdir)/Lib -name '*.py[co]' -print | xargs rm -f -@@ -1001,7 +1001,7 @@ testall: all platform - - # Run the test suite for both architectures in a Universal build on OSX. - # Must be run on an Intel box. --testuniversal: all platform -+testuniversal: @DEF_MAKE_RULE@ platform - if [ `arch` != 'i386' ];then \ - echo "This can only be used on OSX/i386" ;\ - exit 1 ;\ -@@ -1024,7 +1024,7 @@ QUICKTESTOPTS= $(TESTOPTS) -x test_subpr - test_multiprocessing_forkserver \ - test_mailbox test_socket test_poll \ - test_select test_zipfile test_concurrent_futures --quicktest: all platform -+quicktest: @DEF_MAKE_RULE@ platform - $(TESTRUNNER) $(QUICKTESTOPTS) - - -@@ -1376,7 +1376,7 @@ LIBPL= @LIBPL@ - # pkgconfig directory - LIBPC= $(LIBDIR)/pkgconfig - --libainstall: all python-config -+libainstall: @DEF_MAKE_RULE@ python-config - @for i in $(LIBDIR) $(LIBPL) $(LIBPC); \ - do \ - if test ! -d $(DESTDIR)$$i; then \ -@@ -1635,7 +1635,7 @@ distclean: clobber - -exec rm -f {} ';' - - # Check for smelly exported symbols (not starting with Py/_Py) --smelly: all -+smelly: @DEF_MAKE_RULE@ - nm -p $(LIBRARY) | \ - sed -n "/ [TDB] /s/.* //p" | grep -v "^_*Py" | sort -u; \ - -@@ -1673,7 +1673,7 @@ funny: - -o -print - - # Perform some verification checks on any modified files. --patchcheck: all -+patchcheck: @DEF_MAKE_RULE@ - $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/scripts/patchcheck.py - - # Dependencies -Index: Python-3.5.3/Misc/ACKS -=================================================================== ---- Python-3.5.3.orig/Misc/ACKS -+++ Python-3.5.3/Misc/ACKS -@@ -1092,6 +1092,7 @@ Jason Orendorff - Douglas Orr - William Orr - Michele Orrù -+Tomáš Orsava - Oleg Oshmyan - Denis S. Otkidach - Peter Otten -Index: Python-3.5.3/Misc/NEWS -=================================================================== ---- Python-3.5.3.orig/Misc/NEWS -+++ Python-3.5.3/Misc/NEWS -@@ -634,6 +634,10 @@ Windows - Build - ----- - -+- bpo-29243: Prevent unnecessary rebuilding of Python during ``make test``, -+ ``make install`` and some other make targets when configured with -+ ``--enable-optimizations``. -+ - - Issue #29080: Removes hard dependency on hg.exe from PCBuild/build.bat - - - Issue #23903: Added missed names to PC/python3.def. diff --git a/python-interpreter-builder/patches/3.5/series b/python-interpreter-builder/patches/3.5/series deleted file mode 100644 index f6c2876b..00000000 --- a/python-interpreter-builder/patches/3.5/series +++ /dev/null @@ -1 +0,0 @@ -double-build.diff diff --git a/python-interpreter-builder/scripts/build-python-3.5.sh b/python-interpreter-builder/scripts/build-python-3.5.sh index 09653669..f9113879 100755 --- a/python-interpreter-builder/scripts/build-python-3.5.sh +++ b/python-interpreter-builder/scripts/build-python-3.5.sh @@ -6,16 +6,14 @@ set -x # Get the source mkdir -p /opt/sources cd /opt/sources -wget --no-verbose https://www.python.org/ftp/python/3.5.3/Python-3.5.3.tgz +wget --no-verbose https://www.python.org/ftp/python/3.5.4/Python-3.5.4.tgz # SHA-256 generated via `shasum -a 256 [file]` shasum --check < Date: Thu, 24 Aug 2017 15:58:50 -0700 Subject: [PATCH 119/256] Fix Dockerfile reference to deleted directory. The 'patches' directory was implicitly removed in https://github.com/GoogleCloudPlatform/python-runtime/pull/142 when we deleted the last file in it. However, this wasn't found in testing because my local client still had a patches directory. --- python-interpreter-builder/Dockerfile.in | 1 - 1 file changed, 1 deletion(-) diff --git a/python-interpreter-builder/Dockerfile.in b/python-interpreter-builder/Dockerfile.in index 83640de3..eba153e9 100644 --- a/python-interpreter-builder/Dockerfile.in +++ b/python-interpreter-builder/Dockerfile.in @@ -43,7 +43,6 @@ ENV LANG C.UTF-8 # Add build scripts ADD scripts /scripts -ADD patches /patches # Build the Python interpreters RUN /scripts/build-python-3.5.sh From d4bc05472792ca5f419fdcce9eccbe85df23cd46 Mon Sep 17 00:00:00 2001 From: Angela Li Date: Mon, 28 Aug 2017 17:39:54 -0700 Subject: [PATCH 120/256] Add google-cloud-trace to clientlib metrics (#144) --- perf-dashboard/clientlibs-download/python_clientlibs_download.py | 1 + 1 file changed, 1 insertion(+) diff --git a/perf-dashboard/clientlibs-download/python_clientlibs_download.py b/perf-dashboard/clientlibs-download/python_clientlibs_download.py index 350eb7ed..53873e70 100644 --- a/perf-dashboard/clientlibs-download/python_clientlibs_download.py +++ b/perf-dashboard/clientlibs-download/python_clientlibs_download.py @@ -57,6 +57,7 @@ 'google-cloud-logging', 'google-cloud-monitoring', 'google-cloud-error_reporting', + 'google-cloud-trace', ], GRPC_TABLE_NAME: [ 'grpcio', From 3c145063fbbc0f8fd5c5a2219ce8ab529f7ba420 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Tue, 29 Aug 2017 15:20:49 -0700 Subject: [PATCH 121/256] Switch default Python 3 from 3.5 to 3.6 The gen_dockerfile_test.test_generate_dockerfile_golden fails, as expected, until the corresponding XRT change is released in gcloud. --- builder/gen-dockerfile/Dockerfile.in | 4 ++-- scripts/gen_dockerfile.py | 2 +- scripts/gen_dockerfile_test.py | 5 ++++- scripts/testdata/hello_world_golden/Dockerfile | 4 ++-- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/builder/gen-dockerfile/Dockerfile.in b/builder/gen-dockerfile/Dockerfile.in index e2fe4463..4f6447eb 100644 --- a/builder/gen-dockerfile/Dockerfile.in +++ b/builder/gen-dockerfile/Dockerfile.in @@ -1,6 +1,6 @@ FROM ${STAGING_IMAGE} -LABEL python_version=python3.5 -RUN virtualenv --no-download /env -p python3.5 +LABEL python_version=python3.6 +RUN virtualenv --no-download /env -p python3.6 # Set virtualenv environment variables. This is equivalent to running # source /env/bin/activate diff --git a/scripts/gen_dockerfile.py b/scripts/gen_dockerfile.py index 5514b092..eaff183f 100755 --- a/scripts/gen_dockerfile.py +++ b/scripts/gen_dockerfile.py @@ -51,7 +51,7 @@ PYTHON_INTERPRETER_VERSION_MAP = { '': '', # == 2.7 '2': '', # == 2.7 - '3': '3.5', + '3': '3.6', '3.4': '3.4', '3.5': '3.5', '3.6': '3.6', diff --git a/scripts/gen_dockerfile_test.py b/scripts/gen_dockerfile_test.py index 31ef7d42..b52e15e8 100755 --- a/scripts/gen_dockerfile_test.py +++ b/scripts/gen_dockerfile_test.py @@ -63,7 +63,7 @@ def compare_file(filename, dir1, dir2): 'dockerfile_python_version': '', }), ('runtime_config:\n python_version: 3', { - 'dockerfile_python_version': '3.5', + 'dockerfile_python_version': '3.6', }), ('runtime_config:\n python_version: 3.4', { 'dockerfile_python_version': '3.4', @@ -71,6 +71,9 @@ def compare_file(filename, dir1, dir2): ('runtime_config:\n python_version: 3.5', { 'dockerfile_python_version': '3.5', }), + ('runtime_config:\n python_version: 3.6', { + 'dockerfile_python_version': '3.6', + }), # entrypoint present ('entrypoint: my entrypoint', { 'entrypoint': 'exec my entrypoint', diff --git a/scripts/testdata/hello_world_golden/Dockerfile b/scripts/testdata/hello_world_golden/Dockerfile index e3e0563c..10396399 100644 --- a/scripts/testdata/hello_world_golden/Dockerfile +++ b/scripts/testdata/hello_world_golden/Dockerfile @@ -1,6 +1,6 @@ FROM gcr.io/google-appengine/python -LABEL python_version=python3.5 -RUN virtualenv --no-download /env -p python3.5 +LABEL python_version=python3.6 +RUN virtualenv --no-download /env -p python3.6 # Set virtualenv environment variables. This is equivalent to running # source /env/bin/activate From db3c95817da35d45a06f2e7d66419476873048c2 Mon Sep 17 00:00:00 2001 From: Douglas Greiman Date: Thu, 31 Aug 2017 14:20:30 -0700 Subject: [PATCH 122/256] Revert "Switch default Python 3 from 3.5 to 3.6" --- builder/gen-dockerfile/Dockerfile.in | 4 ++-- scripts/gen_dockerfile.py | 2 +- scripts/gen_dockerfile_test.py | 5 +---- scripts/testdata/hello_world_golden/Dockerfile | 4 ++-- 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/builder/gen-dockerfile/Dockerfile.in b/builder/gen-dockerfile/Dockerfile.in index 4f6447eb..e2fe4463 100644 --- a/builder/gen-dockerfile/Dockerfile.in +++ b/builder/gen-dockerfile/Dockerfile.in @@ -1,6 +1,6 @@ FROM ${STAGING_IMAGE} -LABEL python_version=python3.6 -RUN virtualenv --no-download /env -p python3.6 +LABEL python_version=python3.5 +RUN virtualenv --no-download /env -p python3.5 # Set virtualenv environment variables. This is equivalent to running # source /env/bin/activate diff --git a/scripts/gen_dockerfile.py b/scripts/gen_dockerfile.py index eaff183f..5514b092 100755 --- a/scripts/gen_dockerfile.py +++ b/scripts/gen_dockerfile.py @@ -51,7 +51,7 @@ PYTHON_INTERPRETER_VERSION_MAP = { '': '', # == 2.7 '2': '', # == 2.7 - '3': '3.6', + '3': '3.5', '3.4': '3.4', '3.5': '3.5', '3.6': '3.6', diff --git a/scripts/gen_dockerfile_test.py b/scripts/gen_dockerfile_test.py index b52e15e8..31ef7d42 100755 --- a/scripts/gen_dockerfile_test.py +++ b/scripts/gen_dockerfile_test.py @@ -63,7 +63,7 @@ def compare_file(filename, dir1, dir2): 'dockerfile_python_version': '', }), ('runtime_config:\n python_version: 3', { - 'dockerfile_python_version': '3.6', + 'dockerfile_python_version': '3.5', }), ('runtime_config:\n python_version: 3.4', { 'dockerfile_python_version': '3.4', @@ -71,9 +71,6 @@ def compare_file(filename, dir1, dir2): ('runtime_config:\n python_version: 3.5', { 'dockerfile_python_version': '3.5', }), - ('runtime_config:\n python_version: 3.6', { - 'dockerfile_python_version': '3.6', - }), # entrypoint present ('entrypoint: my entrypoint', { 'entrypoint': 'exec my entrypoint', diff --git a/scripts/testdata/hello_world_golden/Dockerfile b/scripts/testdata/hello_world_golden/Dockerfile index 10396399..e3e0563c 100644 --- a/scripts/testdata/hello_world_golden/Dockerfile +++ b/scripts/testdata/hello_world_golden/Dockerfile @@ -1,6 +1,6 @@ FROM gcr.io/google-appengine/python -LABEL python_version=python3.6 -RUN virtualenv --no-download /env -p python3.6 +LABEL python_version=python3.5 +RUN virtualenv --no-download /env -p python3.5 # Set virtualenv environment variables. This is equivalent to running # source /env/bin/activate From 34c4d0f67a18b62fc286343370da48340926a887 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Wed, 6 Sep 2017 11:18:47 -0700 Subject: [PATCH 123/256] Auto-update dependencies. (#139) --- tests/integration/requirements.txt | 8 +-- tests/python2-libraries/requirements.txt | 62 ++++++++++++------------ tests/python3-libraries/requirements.txt | 62 ++++++++++++------------ 3 files changed, 66 insertions(+), 66 deletions(-) diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index 05779ae6..6e6be47e 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -1,7 +1,7 @@ Flask==0.12.2 -google-cloud-error-reporting==0.26.0 -google-cloud-logging==1.2.0 -google-cloud-monitoring==0.26.0 +google-cloud-error-reporting==0.27.0 +google-cloud-logging==1.3.0 +google-cloud-monitoring==0.27.0 gunicorn==19.7.1 -requests==2.18.3 +requests==2.18.4 retrying==1.3.3 diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 05a6dbe3..91807712 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -3,11 +3,11 @@ amqp==2.2.1 amqplib==1.0.2 ansible==2.3.2.0 anyjson==0.3.3 -apache-libcloud==2.1.0 +apache-libcloud==2.2.0 argparse==1.4.0 astroid==1.5.3 -awscli==1.11.131 -babel==2.4.0 +awscli==1.11.146 +babel==2.5.0 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 beautifulsoup4==4.6.0 @@ -16,7 +16,7 @@ billiard==3.5.0.3 blessings==1.6 blinker==1.4 boto==2.48.0 -botocore==1.5.94 +botocore==1.7.4 bottle==0.12.13 carbon==1.0.2 celery==4.1.0 @@ -25,22 +25,22 @@ cffi==1.10.0 chardet==3.0.4 click==6.7 cliff==2.8.0 -cmd2==0.7.5 +cmd2==0.7.7 colorama==0.3.9 configobj==5.0.6 cov-core==1.15.0 coverage==4.4.1 -coveralls==1.1 +coveralls==1.2.0 cryptography==2.0.3 cssselect==1.0.1 -cython==0.26 +cython==0.26.1 decorator==4.1.2 django-celery==3.2.1 django-debug-toolbar==1.8 -django-extensions==1.8.1 -django==1.11.4 +django-extensions==1.9.0 +django==1.11.5 django_compress==1.0.1 -djangorestframework==3.6.3 +djangorestframework==3.6.4 docker-py==1.10.6 docopt==0.6.2 docutils==0.14 @@ -49,7 +49,7 @@ elasticsearch==5.4.0 enum34==1.1.6 eventlet==0.21.0 extras==1.0.0 -fabric==1.13.2 +fabric==1.14.0 fixtures==3.0.0 flake8==3.4.1 flask==0.12.2 @@ -57,7 +57,7 @@ funcsigs==1.0.2 functools32==3.2.3.post2 futures==3.1.1 gevent==1.2.2 -google-api-python-client==1.6.2 +google-api-python-client==1.6.3 graphite-web==1.0.2 greenlet==0.4.12 gunicorn==19.7.1 @@ -79,7 +79,7 @@ lxml==3.8.0 m2crypto==0.26.0 mako==1.0.7 manifestparser==1.1 -markdown==2.6.8 +markdown==2.6.9 markupsafe==1.0 matplotlib==2.0.2 mccabe==0.6.1 @@ -96,7 +96,7 @@ mozprofile==0.28 mozrunner==6.13 msgpack-python==0.4.8 mysql-python==1.2.5 -ndg-httpsclient==0.4.2 +ndg-httpsclient==0.4.3 netaddr==0.7.19 netifaces==0.10.6 newrelic==2.90.0.75 @@ -106,7 +106,7 @@ oauth2==1.9.0.post1 oauth2client==4.1.2 oauthlib==2.0.2 ordereddict==1.1 -oslo.config==4.11.0 +oslo.config==4.12.0 pandas==0.20.3 paramiko==2.2.1 passlib==1.7.1 @@ -116,25 +116,25 @@ pastescript==2.0.2 pbr==3.1.1 pep8==1.7.0 pexpect==4.2.1 -pika==0.10.0 +pika==0.11.0 pillow==4.2.1 pip==9.0.1 prettytable -protobuf==3.3.0 -psutil==5.2.2 -psycopg2==2.7.3 +protobuf==3.4.0 +psutil==5.3.0 +psycopg2==2.7.3.1 py==1.4.34 -pyasn1-modules==0.0.11 -pyasn1==0.3.2 +pyasn1-modules==0.1.1 +pyasn1==0.3.3 pycparser==2.18 pycrypto==2.6.1 pycurl==7.43.0 pyflakes==1.6.0 pygments==2.2.0 -pyjwt==1.5.2 +pyjwt==1.5.3 pylibmc==1.5.2 pylint==1.7.2 -pymongo==3.5.0 +pymongo==3.5.1 pymysql==0.7.11 pyopenssl==17.2.0 pyparsing==2.2.0 @@ -156,16 +156,16 @@ pytz==2017.2 pyyaml==3.12 pyzmq==16.0.2 raven==6.1.0 -redis==2.10.5 +redis==2.10.6 repoze.lru==0.6 requests-oauthlib==0.8.0 -requests==2.18.3 +requests==2.18.4 retrying==1.3.3 rsa==3.4.2 scipy==0.19.1 -selenium==3.4.3 +selenium==3.5.0 setuptools-git==1.2 -setuptools==36.2.7 +setuptools==36.4.0 sh==1.12.14 simplejson==3.11.1 six==1.10.0 @@ -176,14 +176,14 @@ sqlalchemy-migrate==0.11.0 sqlalchemy==1.1.11 sqlparse==0.2.3 statsd==3.2.1 -stevedore==1.25.0 +stevedore==1.26.0 suds==0.4 supervisor==3.3.3 testrepository==0.0.20 testtools==2.3.0 thrift==0.10.0 -tornado==4.5.1 -tox==2.7.0 +tornado==4.5.2 +tox==2.8.1 twisted==17.5.0 ujson==1.35 unidecode==0.4.21 @@ -200,6 +200,6 @@ websocket-client==0.44.0 webtest==2.0.28 werkzeug==0.12.2 wheel==0.29.0 -xlrd==1.0.0 +xlrd==1.1.0 zc.buildout==2.9.4 zope.interface==4.4.2 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index f1e3539b..f2d6973b 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -3,11 +3,11 @@ amqp==2.2.1 amqplib==1.0.2 ansible==2.3.2.0 anyjson==0.3.3 -apache-libcloud==2.1.0 +apache-libcloud==2.2.0 argparse==1.4.0 astroid==1.5.3 -awscli==1.11.131 -babel==2.4.0 +awscli==1.11.146 +babel==2.5.0 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 beautifulsoup4==4.6.0 @@ -15,7 +15,7 @@ billiard==3.5.0.3 blessings==1.6 blinker==1.4 boto==2.48.0 -botocore==1.5.94 +botocore==1.7.4 bottle==0.12.13 celery==4.1.0 certifi==2017.7.27.1 @@ -23,22 +23,22 @@ cffi==1.10.0 chardet==3.0.4 click==6.7 cliff==2.8.0 -cmd2==0.7.5 +cmd2==0.7.7 colorama==0.3.9 configobj==5.0.6 cov-core==1.15.0 coverage==4.4.1 -coveralls==1.1 +coveralls==1.2.0 cryptography==2.0.3 cssselect==1.0.1 -cython==0.26 +cython==0.26.1 decorator==4.1.2 django-celery==3.2.1 django-debug-toolbar==1.8 -django-extensions==1.8.1 -django==1.11.4 +django-extensions==1.9.0 +django==1.11.5 django_compress==1.0.1 -djangorestframework==3.6.3 +djangorestframework==3.6.4 docker-py==1.10.6 docopt==0.6.2 docutils==0.14 @@ -47,14 +47,14 @@ elasticsearch==5.4.0 enum34==1.1.6 eventlet==0.21.0 extras==1.0.0 -fabric==1.13.2 +fabric==1.14.0 fixtures==3.0.0 flake8==3.4.1 flask==0.12.2 funcsigs==1.0.2 futures==3.1.1 gevent==1.2.2 -google-api-python-client==1.6.2 +google-api-python-client==1.6.3 greenlet==0.4.12 gunicorn==19.7.1 hiredis==0.2.0 @@ -76,7 +76,7 @@ lxml==3.8.0 m2crypto==0.26.0 mako==1.0.7 manifestparser==1.1 -markdown==2.6.8 +markdown==2.6.9 markupsafe==1.0 matplotlib==2.0.2 mccabe==0.6.1 @@ -90,7 +90,7 @@ mozlog==3.5 moznetwork==0.27 mozprocess==0.25 msgpack-python==0.4.8 -ndg-httpsclient==0.4.2 +ndg-httpsclient==0.4.3 netaddr==0.7.19 netifaces==0.10.6 newrelic==2.90.0.75 @@ -100,7 +100,7 @@ oauth2==1.9.0.post1 oauth2client==4.1.2 oauthlib==2.0.2 ordereddict==1.1 -oslo.config==4.11.0 +oslo.config==4.12.0 pandas==0.20.3 paramiko==2.2.1 passlib==1.7.1 @@ -110,24 +110,24 @@ pastescript==2.0.2 pbr==3.1.1 pep8==1.7.0 pexpect==4.2.1 -pika==0.10.0 +pika==0.11.0 pillow==4.2.1 pip==9.0.1 prettytable -protobuf==3.3.0 -psutil==5.2.2 -psycopg2==2.7.3 +protobuf==3.4.0 +psutil==5.3.0 +psycopg2==2.7.3.1 py==1.4.34 -pyasn1-modules==0.0.11 -pyasn1==0.3.2 +pyasn1-modules==0.1.1 +pyasn1==0.3.3 pycparser==2.18 pycrypto==2.6.1 pyflakes==1.6.0 pygments==2.2.0 -pyjwt==1.5.2 +pyjwt==1.5.3 pylibmc==1.5.2 pylint==1.7.2 -pymongo==3.5.0 +pymongo==3.5.1 pymysql==0.7.11 pyopenssl==17.2.0 pyparsing==2.2.0 @@ -148,16 +148,16 @@ pytz==2017.2 pyyaml==3.12 pyzmq==16.0.2 raven==6.1.0 -redis==2.10.5 +redis==2.10.6 repoze.lru==0.6 requests-oauthlib==0.8.0 -requests==2.18.3 +requests==2.18.4 retrying==1.3.3 rsa==3.4.2 scipy==0.19.1 -selenium==3.4.3 +selenium==3.5.0 setuptools-git==1.2 -setuptools==36.2.7 +setuptools==36.4.0 sh==1.12.14 simplejson==3.11.1 six==1.10.0 @@ -168,12 +168,12 @@ sqlalchemy-migrate==0.11.0 sqlalchemy==1.1.11 sqlparse==0.2.3 statsd==3.2.1 -stevedore==1.25.0 +stevedore==1.26.0 testrepository==0.0.20 testtools==2.3.0 thrift==0.10.0 -tornado==4.5.1 -tox==2.7.0 +tornado==4.5.2 +tox==2.8.1 twisted==17.5.0 ujson==1.35 unidecode==0.4.21 @@ -190,6 +190,6 @@ websocket-client==0.44.0 webtest==2.0.28 werkzeug==0.12.2 wheel==0.29.0 -xlrd==1.0.0 +xlrd==1.1.0 zc.buildout==2.9.4 zope.interface==4.4.2 From 2e3a11e93f53f1f59a3d735410cbdbd0fbb9969e Mon Sep 17 00:00:00 2001 From: Angela Li Date: Wed, 6 Sep 2017 16:22:07 -0700 Subject: [PATCH 124/256] Update bigquery method to match the latest version (#147) --- .../clientlibs-download/python_clientlibs_download.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-dashboard/clientlibs-download/python_clientlibs_download.py b/perf-dashboard/clientlibs-download/python_clientlibs_download.py index 53873e70..1d307ac2 100644 --- a/perf-dashboard/clientlibs-download/python_clientlibs_download.py +++ b/perf-dashboard/clientlibs-download/python_clientlibs_download.py @@ -127,7 +127,7 @@ def get_weekly_clientlibs_downloads(clientlibs_table_name, date_str): wait_for_job(query_job) # Fetch the results - result = query_job.result().fetch_data() + result = query_job.query_results().fetch_data() result_list = [item for item in result] # In case the result_list contains the metadata like total_rows, the From 76d04c4c8251cf9f2b90b098365999c2606fba9e Mon Sep 17 00:00:00 2001 From: Douglas Greiman Date: Wed, 27 Sep 2017 15:54:42 -0700 Subject: [PATCH 125/256] Revert "Revert "Switch default Python 3 from 3.5 to 3.6"" --- builder/gen-dockerfile/Dockerfile.in | 4 ++-- scripts/gen_dockerfile.py | 2 +- scripts/gen_dockerfile_test.py | 5 ++++- scripts/testdata/hello_world_golden/Dockerfile | 4 ++-- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/builder/gen-dockerfile/Dockerfile.in b/builder/gen-dockerfile/Dockerfile.in index e2fe4463..4f6447eb 100644 --- a/builder/gen-dockerfile/Dockerfile.in +++ b/builder/gen-dockerfile/Dockerfile.in @@ -1,6 +1,6 @@ FROM ${STAGING_IMAGE} -LABEL python_version=python3.5 -RUN virtualenv --no-download /env -p python3.5 +LABEL python_version=python3.6 +RUN virtualenv --no-download /env -p python3.6 # Set virtualenv environment variables. This is equivalent to running # source /env/bin/activate diff --git a/scripts/gen_dockerfile.py b/scripts/gen_dockerfile.py index 5514b092..eaff183f 100755 --- a/scripts/gen_dockerfile.py +++ b/scripts/gen_dockerfile.py @@ -51,7 +51,7 @@ PYTHON_INTERPRETER_VERSION_MAP = { '': '', # == 2.7 '2': '', # == 2.7 - '3': '3.5', + '3': '3.6', '3.4': '3.4', '3.5': '3.5', '3.6': '3.6', diff --git a/scripts/gen_dockerfile_test.py b/scripts/gen_dockerfile_test.py index 31ef7d42..b52e15e8 100755 --- a/scripts/gen_dockerfile_test.py +++ b/scripts/gen_dockerfile_test.py @@ -63,7 +63,7 @@ def compare_file(filename, dir1, dir2): 'dockerfile_python_version': '', }), ('runtime_config:\n python_version: 3', { - 'dockerfile_python_version': '3.5', + 'dockerfile_python_version': '3.6', }), ('runtime_config:\n python_version: 3.4', { 'dockerfile_python_version': '3.4', @@ -71,6 +71,9 @@ def compare_file(filename, dir1, dir2): ('runtime_config:\n python_version: 3.5', { 'dockerfile_python_version': '3.5', }), + ('runtime_config:\n python_version: 3.6', { + 'dockerfile_python_version': '3.6', + }), # entrypoint present ('entrypoint: my entrypoint', { 'entrypoint': 'exec my entrypoint', diff --git a/scripts/testdata/hello_world_golden/Dockerfile b/scripts/testdata/hello_world_golden/Dockerfile index e3e0563c..10396399 100644 --- a/scripts/testdata/hello_world_golden/Dockerfile +++ b/scripts/testdata/hello_world_golden/Dockerfile @@ -1,6 +1,6 @@ FROM gcr.io/google-appengine/python -LABEL python_version=python3.5 -RUN virtualenv --no-download /env -p python3.5 +LABEL python_version=python3.6 +RUN virtualenv --no-download /env -p python3.6 # Set virtualenv environment variables. This is equivalent to running # source /env/bin/activate From 1d4e6df0f8780ea2be67c9a2fe0a01b9c7927e3c Mon Sep 17 00:00:00 2001 From: Douglas Greiman Date: Mon, 2 Oct 2017 09:37:27 -0700 Subject: [PATCH 126/256] Add missing dependency (was using a too-old version of six in some cases). (#151) --- tests/integration/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index 6e6be47e..eae15fe9 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -5,3 +5,4 @@ google-cloud-monitoring==0.27.0 gunicorn==19.7.1 requests==2.18.4 retrying==1.3.3 +six==1.11.0 From 18397afdf4f8fb7e81310ff55951063830f80546 Mon Sep 17 00:00:00 2001 From: Angela Li Date: Fri, 13 Oct 2017 10:42:29 -0700 Subject: [PATCH 127/256] Add Stackoverflow metrics (#149) --- perf_dashboard/bq_utils.py | 46 ++++++++ .../python_clientlibs_download.py | 57 +++------- perf_dashboard/stackoverflow/posts_stats.py | 107 ++++++++++++++++++ 3 files changed, 166 insertions(+), 44 deletions(-) create mode 100644 perf_dashboard/bq_utils.py rename {perf-dashboard/clientlibs-download => perf_dashboard/clientlibs_download}/python_clientlibs_download.py (71%) create mode 100644 perf_dashboard/stackoverflow/posts_stats.py diff --git a/perf_dashboard/bq_utils.py b/perf_dashboard/bq_utils.py new file mode 100644 index 00000000..d9441f07 --- /dev/null +++ b/perf_dashboard/bq_utils.py @@ -0,0 +1,46 @@ +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Common util methods for processing data in BigQuery.""" + +import uuid + +from google.cloud import bigquery + + +def insert_rows(project, dataset_name, table_name, rows): + """Insert rows to bigquery table.""" + client = bigquery.Client(project=project) + dataset = client.dataset(dataset_name) + table = bigquery.Table(table_name, dataset) + table.reload() + table.insert_data(rows) + + +def execute_query(query): + """Execute query and return the query results.""" + client = bigquery.Client() + query_job = client.run_async_query(str(uuid.uuid4()), query) + query_job.use_legacy_sql = False + + # Start the query job and wait it to complete + query_job.begin() + query_job.result() + + # Get the results + destination_table = query_job.destination + destination_table.reload() + results = destination_table.fetch_data() + + return results diff --git a/perf-dashboard/clientlibs-download/python_clientlibs_download.py b/perf_dashboard/clientlibs_download/python_clientlibs_download.py similarity index 71% rename from perf-dashboard/clientlibs-download/python_clientlibs_download.py rename to perf_dashboard/clientlibs_download/python_clientlibs_download.py index 1d307ac2..3c8f79fe 100644 --- a/perf-dashboard/clientlibs-download/python_clientlibs_download.py +++ b/perf_dashboard/clientlibs_download/python_clientlibs_download.py @@ -14,11 +14,16 @@ import datetime import os +import sys import time import uuid from google.cloud import bigquery +sys.path.insert(0, os.path.abspath(__file__+"/../../..")) +from perf_dashboard import bq_utils + + GCLOUD_PROJECT_ENV = 'GCLOUD_PROJECT' DATETIME_FORMAT = '%Y%m%d' @@ -68,17 +73,6 @@ } -def wait_for_job(job): - """Wait for the query job to complete.""" - while True: - job.reload() # Refreshes the state via a GET request. - if job.state == 'DONE': - if job.error_result: - raise RuntimeError(job.errors) - return - time.sleep(1) - - def get_weekly_clientlibs_downloads(clientlibs_table_name, date_str): """Use a SQL query to collect the weekly download data of the client libraries. @@ -124,50 +118,25 @@ def get_weekly_clientlibs_downloads(clientlibs_table_name, date_str): # Start the query job and wait it to complete query_job.begin() - wait_for_job(query_job) - - # Fetch the results - result = query_job.query_results().fetch_data() - result_list = [item for item in result] + query_job.result() - # In case the result_list contains the metadata like total_rows, the - # actual rows will be the first element of the result_list. - if len(result_list) > 0 and isinstance(result_list[0], list): - result_list = result_list[0] + # Get the results + destination_table = query_job.destination + destination_table.reload() + results = destination_table.fetch_data() - rows = [(date_time,) + row for row in result_list] - print(rows) + rows = [(date_time,) + row for row in results] return rows -def insert_rows(dataset_name, table_name, rows): - """Insert rows to a bigquery table. - - Args: - dataset_name (str): Name of the dataset that holds the tables. - table_name (str): Name of the bigquery table. - rows (list): The rows that going to be inserted into the table. - - Returns: - list: Empty if inserted successfully, else the errors when inserting - each row. - """ - project = os.environ.get(GCLOUD_PROJECT_ENV) - client = bigquery.Client(project=project) - dataset = client.dataset(dataset_name) - table = bigquery.Table(name=table_name, dataset=dataset) - table.reload() - error = table.insert_data(rows) - return error - - def main(): for table_name in CLIENTLIBS.keys(): rows = get_weekly_clientlibs_downloads( clientlibs_table_name=table_name, date_str=datetime.datetime.now().strftime("%Y%m%d")) - insert_rows( + bq_utils.insert_rows( + project=project, dataset_name=DATASET_NAME, table_name=table_name, rows=rows) diff --git a/perf_dashboard/stackoverflow/posts_stats.py b/perf_dashboard/stackoverflow/posts_stats.py new file mode 100644 index 00000000..280fd42c --- /dev/null +++ b/perf_dashboard/stackoverflow/posts_stats.py @@ -0,0 +1,107 @@ +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A script to collect the number of StackOverflow posts related to +Python and Google Cloud Platform.""" + +import datetime +import os +import sys +import time +import uuid + +from collections import Counter + +from google.cloud import bigquery + +# Need this to import the local helper function +sys.path.insert(0, os.path.abspath(__file__+"/../../..")) +from perf_dashboard import bq_utils + +GCLOUD_PROJECT_ENV = 'GCLOUD_PROJECT' +DATASET_NAME = 'stackoverflow' +TAG_COUNT_TABLE_NAME = 'tag_count_timestamp' +UNANSWERED_POSTS_TABLE_NAME = 'unanswered_posts' + + +def get_stackoverflow_tags_count(): + """Get all the tags contains python and cloud key words""" + query = """ + SELECT + SPLIT(tags, '|') tags + FROM + `bigquery-public-data.stackoverflow.posts_questions` + WHERE + tags LIKE '%python%' + AND (tags LIKE '%google-cloud-platform%' OR tags LIKE '%gcp%') + """ + + results = bq_utils.execute_query(query) + + rows = [row[0] for row in results] + + return rows + + +def get_posts_list_unanswered(): + # Get the list of posts that are unanswered + query = """ + SELECT + id, title, tags + FROM + `bigquery-public-data.stackoverflow.posts_questions` + WHERE + tags LIKE '%python%' + AND (tags LIKE '%google-cloud-platform%' OR tags LIKE '%gcp%') + AND accepted_answer_id is NULL + AND answer_count = 0; + """ + + results = bq_utils.execute_query(query) + + # Add current timestamp to the rows + date_time = datetime.datetime.now() + rows = [(date_time,) + row for row in results] + + return rows + + +def count_unique_tags(data): + flattened_tag_list = [tag for tag_list in data for tag in tag_list] + tag_count = Counter(flattened_tag_list) + + # Add current timestamp to the rows + date_time = datetime.datetime.now() + time_tag_count = [(date_time,) + item for item in tag_count.items()] + + return time_tag_count + + +def main(): + project = os.environ.get(GCLOUD_PROJECT_ENV) + + # Get the posts count for each tag + rows = get_stackoverflow_tags_count() + tag_count = count_unique_tags(rows) + bq_utils.insert_rows( + project, DATASET_NAME, TAG_COUNT_TABLE_NAME, tag_count) + + # Get the list of unanswered posts + unanswered_posts = get_posts_list_unanswered() + bq_utils.insert_rows( + project, DATASET_NAME, UNANSWERED_POSTS_TABLE_NAME, unanswered_posts) + + +if __name__ == '__main__': + main() From fed4c56915fe360cc98f6e9f144098d3ed6c4f78 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Mon, 16 Oct 2017 16:40:07 -0700 Subject: [PATCH 128/256] Auto-update dependencies. (#148) --- scripts/requirements-test.txt | 2 +- tests/python2-libraries/requirements.txt | 92 +++++++++++------------ tests/python3-libraries/requirements.txt | 94 ++++++++++++------------ 3 files changed, 94 insertions(+), 94 deletions(-) diff --git a/scripts/requirements-test.txt b/scripts/requirements-test.txt index 6e01a83d..5afb5315 100644 --- a/scripts/requirements-test.txt +++ b/scripts/requirements-test.txt @@ -1,4 +1,4 @@ flask==0.12.2 -pytest==3.2.1 +pytest==3.2.3 pytest-cov==2.5.1 pyyaml==3.12 diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 91807712..87e9e53a 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -1,13 +1,13 @@ -alembic==0.9.5 -amqp==2.2.1 +alembic==0.9.6 +amqp==2.2.2 amqplib==1.0.2 -ansible==2.3.2.0 +ansible==2.4.0.0 anyjson==0.3.3 -apache-libcloud==2.2.0 +apache-libcloud==2.2.1 argparse==1.4.0 astroid==1.5.3 -awscli==1.11.146 -babel==2.5.0 +awscli==1.11.170 +babel==2.5.1 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 beautifulsoup4==4.6.0 @@ -16,31 +16,31 @@ billiard==3.5.0.3 blessings==1.6 blinker==1.4 boto==2.48.0 -botocore==1.7.4 +botocore==1.7.28 bottle==0.12.13 carbon==1.0.2 celery==4.1.0 certifi==2017.7.27.1 -cffi==1.10.0 +cffi==1.11.2 chardet==3.0.4 click==6.7 -cliff==2.8.0 +cliff==2.9.1 cmd2==0.7.7 colorama==0.3.9 configobj==5.0.6 cov-core==1.15.0 coverage==4.4.1 coveralls==1.2.0 -cryptography==2.0.3 +cryptography==2.1.1 cssselect==1.0.1 -cython==0.26.1 +cython==0.27.1 decorator==4.1.2 django-celery==3.2.1 django-debug-toolbar==1.8 -django-extensions==1.9.0 -django==1.11.5 +django-extensions==1.9.1 +django==1.11.6 django_compress==1.0.1 -djangorestframework==3.6.4 +djangorestframework==3.7.0 docker-py==1.10.6 docopt==0.6.2 docutils==0.14 @@ -57,7 +57,7 @@ funcsigs==1.0.2 functools32==3.2.3.post2 futures==3.1.1 gevent==1.2.2 -google-api-python-client==1.6.3 +google-api-python-client==1.6.4 graphite-web==1.0.2 greenlet==0.4.12 gunicorn==19.7.1 @@ -67,7 +67,7 @@ httplib2==0.10.3 idna==2.6 ipaddress==1.0.18 iso8601==0.1.12 -isodate==0.5.4 +isodate==0.6.0 itsdangerous==0.24 jinja2==2.9.6 jmespath==0.9.3 @@ -75,13 +75,13 @@ jsonschema==2.6.0 kombu==4.1.0 linecache2==1.0.0 logilab-common==1.4.1 -lxml==3.8.0 -m2crypto==0.26.0 +lxml==4.1.0 +m2crypto==0.27.0 mako==1.0.7 manifestparser==1.1 markdown==2.6.9 markupsafe==1.0 -matplotlib==2.0.2 +matplotlib==2.1.0 mccabe==0.6.1 meld3==1.0.2 mock==2.0.0 @@ -99,16 +99,16 @@ mysql-python==1.2.5 ndg-httpsclient==0.4.3 netaddr==0.7.19 netifaces==0.10.6 -newrelic==2.90.0.75 +newrelic==2.94.0.79 nose==1.3.7 -numpy==1.13.1 +numpy==1.13.3 oauth2==1.9.0.post1 oauth2client==4.1.2 -oauthlib==2.0.2 +oauthlib==2.0.4 ordereddict==1.1 -oslo.config==4.12.0 +oslo.config==4.13.1 pandas==0.20.3 -paramiko==2.2.1 +paramiko==2.3.1 passlib==1.7.1 paste==2.0.3 pastedeploy==1.5.2 @@ -117,15 +117,15 @@ pbr==3.1.1 pep8==1.7.0 pexpect==4.2.1 pika==0.11.0 -pillow==4.2.1 +pillow==4.3.0 pip==9.0.1 prettytable protobuf==3.4.0 -psutil==5.3.0 +psutil==5.4.0 psycopg2==2.7.3.1 py==1.4.34 -pyasn1-modules==0.1.1 -pyasn1==0.3.3 +pyasn1-modules==0.1.5 +pyasn1==0.3.7 pycparser==2.18 pycrypto==2.6.1 pycurl==7.43.0 @@ -133,15 +133,15 @@ pyflakes==1.6.0 pygments==2.2.0 pyjwt==1.5.3 pylibmc==1.5.2 -pylint==1.7.2 +pylint==1.7.4 pymongo==3.5.1 pymysql==0.7.11 -pyopenssl==17.2.0 +pyopenssl==17.3.0 pyparsing==2.2.0 pyramid==1.9.1 pystache==0.5.4 pytest-cov==2.5.1 -pytest==3.2.1 +pytest==3.2.3 python-cjson==1.2.1 python-daemon==2.1.2 python-dateutil==2.6.1 @@ -155,36 +155,36 @@ python-swiftclient==3.4.0 pytz==2017.2 pyyaml==3.12 pyzmq==16.0.2 -raven==6.1.0 +raven==6.2.1 redis==2.10.6 -repoze.lru==0.6 +repoze.lru==0.7 requests-oauthlib==0.8.0 requests==2.18.4 retrying==1.3.3 rsa==3.4.2 scipy==0.19.1 -selenium==3.5.0 +selenium==3.6.0 setuptools-git==1.2 -setuptools==36.4.0 +setuptools==36.6.0 sh==1.12.14 simplejson==3.11.1 -six==1.10.0 +six==1.11.0 snowballstemmer==1.2.1 south==1.0.2 -sphinx==1.6.3 +sphinx==1.6.4 sqlalchemy-migrate==0.11.0 -sqlalchemy==1.1.11 -sqlparse==0.2.3 +sqlalchemy==1.1.14 +sqlparse==0.2.4 statsd==3.2.1 -stevedore==1.26.0 +stevedore==1.27.1 suds==0.4 supervisor==3.3.3 testrepository==0.0.20 testtools==2.3.0 thrift==0.10.0 tornado==4.5.2 -tox==2.8.1 -twisted==17.5.0 +tox==2.9.1 +twisted==17.9.0 ujson==1.35 unidecode==0.4.21 unittest2==1.1.0 @@ -193,13 +193,13 @@ urllib3==1.22 uwsgi==2.0.15 versiontools==1.9.1 virtualenv==15.1.0 -waitress==1.0.2 +waitress==1.1.0 warlock==1.3.0 webob==1.7.3 websocket-client==0.44.0 webtest==2.0.28 werkzeug==0.12.2 -wheel==0.29.0 +wheel==0.30.0 xlrd==1.1.0 -zc.buildout==2.9.4 -zope.interface==4.4.2 +zc.buildout==2.9.5 +zope.interface==4.4.3 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index f2d6973b..614f2f37 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -1,13 +1,13 @@ -alembic==0.9.5 -amqp==2.2.1 +alembic==0.9.6 +amqp==2.2.2 amqplib==1.0.2 -ansible==2.3.2.0 +ansible==2.4.0.0 anyjson==0.3.3 -apache-libcloud==2.2.0 +apache-libcloud==2.2.1 argparse==1.4.0 astroid==1.5.3 -awscli==1.11.146 -babel==2.5.0 +awscli==1.11.170 +babel==2.5.1 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 beautifulsoup4==4.6.0 @@ -15,30 +15,30 @@ billiard==3.5.0.3 blessings==1.6 blinker==1.4 boto==2.48.0 -botocore==1.7.4 +botocore==1.7.28 bottle==0.12.13 celery==4.1.0 certifi==2017.7.27.1 -cffi==1.10.0 +cffi==1.11.2 chardet==3.0.4 click==6.7 -cliff==2.8.0 +cliff==2.9.1 cmd2==0.7.7 colorama==0.3.9 configobj==5.0.6 cov-core==1.15.0 coverage==4.4.1 coveralls==1.2.0 -cryptography==2.0.3 +cryptography==2.1.1 cssselect==1.0.1 -cython==0.26.1 +cython==0.27.1 decorator==4.1.2 django-celery==3.2.1 django-debug-toolbar==1.8 -django-extensions==1.9.0 -django==1.11.5 +django-extensions==1.9.1 +django==1.11.6 django_compress==1.0.1 -djangorestframework==3.6.4 +djangorestframework==3.7.0 docker-py==1.10.6 docopt==0.6.2 docutils==0.14 @@ -54,7 +54,7 @@ flask==0.12.2 funcsigs==1.0.2 futures==3.1.1 gevent==1.2.2 -google-api-python-client==1.6.3 +google-api-python-client==1.6.4 greenlet==0.4.12 gunicorn==19.7.1 hiredis==0.2.0 @@ -62,9 +62,9 @@ html5lib httplib2==0.10.3 idna==2.6 ipaddress==1.0.18 -ipython==6.1.0 +ipython==6.2.1 iso8601==0.1.12 -isodate==0.5.4 +isodate==0.6.0 itsdangerous==0.24 jinja2==2.9.6 jmespath==0.9.3 @@ -72,13 +72,13 @@ jsonschema==2.6.0 kombu==4.1.0 linecache2==1.0.0 logilab-common==1.4.1 -lxml==3.8.0 -m2crypto==0.26.0 +lxml==4.1.0 +m2crypto==0.27.0 mako==1.0.7 manifestparser==1.1 markdown==2.6.9 markupsafe==1.0 -matplotlib==2.0.2 +matplotlib==2.1.0 mccabe==0.6.1 meld3==1.0.2 mock==2.0.0 @@ -93,16 +93,16 @@ msgpack-python==0.4.8 ndg-httpsclient==0.4.3 netaddr==0.7.19 netifaces==0.10.6 -newrelic==2.90.0.75 +newrelic==2.94.0.79 nose==1.3.7 -numpy==1.13.1 +numpy==1.13.3 oauth2==1.9.0.post1 oauth2client==4.1.2 -oauthlib==2.0.2 +oauthlib==2.0.4 ordereddict==1.1 -oslo.config==4.12.0 +oslo.config==4.13.1 pandas==0.20.3 -paramiko==2.2.1 +paramiko==2.3.1 passlib==1.7.1 paste==2.0.3 pastedeploy==1.5.2 @@ -111,30 +111,30 @@ pbr==3.1.1 pep8==1.7.0 pexpect==4.2.1 pika==0.11.0 -pillow==4.2.1 +pillow==4.3.0 pip==9.0.1 prettytable protobuf==3.4.0 -psutil==5.3.0 +psutil==5.4.0 psycopg2==2.7.3.1 py==1.4.34 -pyasn1-modules==0.1.1 -pyasn1==0.3.3 +pyasn1-modules==0.1.5 +pyasn1==0.3.7 pycparser==2.18 pycrypto==2.6.1 pyflakes==1.6.0 pygments==2.2.0 pyjwt==1.5.3 pylibmc==1.5.2 -pylint==1.7.2 +pylint==1.7.4 pymongo==3.5.1 pymysql==0.7.11 -pyopenssl==17.2.0 +pyopenssl==17.3.0 pyparsing==2.2.0 pyramid==1.9.1 pystache==0.5.4 pytest-cov==2.5.1 -pytest==3.2.1 +pytest==3.2.3 python-daemon==2.1.2 python-dateutil==2.6.1 python-gflags==3.1.1 @@ -147,34 +147,34 @@ python-swiftclient==3.4.0 pytz==2017.2 pyyaml==3.12 pyzmq==16.0.2 -raven==6.1.0 +raven==6.2.1 redis==2.10.6 -repoze.lru==0.6 +repoze.lru==0.7 requests-oauthlib==0.8.0 requests==2.18.4 retrying==1.3.3 rsa==3.4.2 scipy==0.19.1 -selenium==3.5.0 +selenium==3.6.0 setuptools-git==1.2 -setuptools==36.4.0 +setuptools==36.6.0 sh==1.12.14 simplejson==3.11.1 -six==1.10.0 +six==1.11.0 snowballstemmer==1.2.1 south==1.0.2 -sphinx==1.6.3 +sphinx==1.6.4 sqlalchemy-migrate==0.11.0 -sqlalchemy==1.1.11 -sqlparse==0.2.3 +sqlalchemy==1.1.14 +sqlparse==0.2.4 statsd==3.2.1 -stevedore==1.26.0 +stevedore==1.27.1 testrepository==0.0.20 testtools==2.3.0 thrift==0.10.0 tornado==4.5.2 -tox==2.8.1 -twisted==17.5.0 +tox==2.9.1 +twisted==17.9.0 ujson==1.35 unidecode==0.4.21 unittest2==1.1.0 @@ -183,13 +183,13 @@ urllib3==1.22 uwsgi==2.0.15 versiontools==1.9.1 virtualenv==15.1.0 -waitress==1.0.2 +waitress==1.1.0 warlock==1.3.0 webob==1.7.3 websocket-client==0.44.0 webtest==2.0.28 werkzeug==0.12.2 -wheel==0.29.0 +wheel==0.30.0 xlrd==1.1.0 -zc.buildout==2.9.4 -zope.interface==4.4.2 +zc.buildout==2.9.5 +zope.interface==4.4.3 From 8df5ad6b9fa2c99ced22302d4867fdf73cf4d60b Mon Sep 17 00:00:00 2001 From: Angela Li Date: Fri, 20 Oct 2017 11:13:09 -0700 Subject: [PATCH 129/256] Upgrade setuptools in test (#154) --- tests/google-cloud-python/Dockerfile.in | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/google-cloud-python/Dockerfile.in b/tests/google-cloud-python/Dockerfile.in index a2873863..b82c18d1 100644 --- a/tests/google-cloud-python/Dockerfile.in +++ b/tests/google-cloud-python/Dockerfile.in @@ -4,6 +4,9 @@ FROM ${STAGING_IMAGE} RUN git clone --depth 1 https://github.com/GoogleCloudPlatform/google-cloud-python.git WORKDIR google-cloud-python +# Upgrade setuptools +RUN pip install --upgrade setuptools + # Install nox RUN pip install --upgrade nox-automation From 44af51158a5ca6eb5e7ec5ee4981613980d7c629 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 20 Oct 2017 15:36:14 -0700 Subject: [PATCH 130/256] Separate building and testing phases via --build and --test flags. Sometimes the tests fails for reasons not related to the runtime. This gives us a way to build and push the image anyway. --- build.sh | 72 +++++++++++++++++------------------ cloudbuild.yaml | 25 +----------- cloudbuild_benchmark.yaml | 4 -- cloudbuild_library_tests.yaml | 20 ---------- cloudbuild_system_tests.yaml | 4 -- cloudbuild_tests.yaml | 28 ++++++++++++++ 6 files changed, 65 insertions(+), 88 deletions(-) delete mode 100644 cloudbuild_library_tests.yaml create mode 100644 cloudbuild_tests.yaml diff --git a/build.sh b/build.sh index 2f0595e9..ca651fb7 100755 --- a/build.sh +++ b/build.sh @@ -18,9 +18,9 @@ set -euo pipefail # Actions benchmark=0 # Should run benchmarks? -build=1 # Should build images? -library_tests=0 # Should try to install top N Python libraries +build=0 # Should build images? system_tests=0 # Should run system tests? +tests=0 # Should run standard test suite? local=0 # Should run using local Docker daemon instead of GCR? @@ -40,32 +40,37 @@ Build and test artifacts in this repository Options: --[no]benchmark: Run benchmarking suite (default false) - --[no]build: Build all images (default true) - --[no]library_tests: Run library compatiblity tests (default false) + --[no]build: Build all images (default true if no options set) + --[no]tests: Run basic tests (default true if no options set) --[no]local: Build images using local Docker daemon (default false) --[no]system_tests: Run system tests (default false) " } - + # Read environment variables -if [ -z "${DOCKER_NAMESPACE+set}" ] ; then +if [ -z "${DOCKER_NAMESPACE:+set}" ] ; then fatal 'Error: $DOCKER_NAMESPACE is not set; invoke with something like DOCKER_NAMESPACE=gcr.io/YOUR-PROJECT-NAME' fi -if [ -z "${BUILDER_DOCKER_NAMESPACE+set}" ] ; then +if [ -z "${BUILDER_DOCKER_NAMESPACE:+set}" ] ; then export BUILDER_DOCKER_NAMESPACE="${DOCKER_NAMESPACE}" fi -if [ -z "${TAG+set}" ] ; then +if [ -z "${TAG:+set}" ] ; then export TAG=`date +%Y-%m-%d-%H%M%S` fi -substitutions="\ +build_substitutions="\ _BUILDER_DOCKER_NAMESPACE=${BUILDER_DOCKER_NAMESPACE},\ _DOCKER_NAMESPACE=${DOCKER_NAMESPACE},\ _TAG=${TAG}\ " +substitutions="\ +_DOCKER_NAMESPACE=${DOCKER_NAMESPACE},\ +_TAG=${TAG}\ +" + # Read command line arguments while [ $# -gt 0 ]; do case "$1" in @@ -85,14 +90,6 @@ while [ $# -gt 0 ]; do build=0 shift ;; - --library_tests) - library_tests=1 - shift - ;; - --nolibrary_tests) - library_tests=0 - shift - ;; --local) local=1 shift @@ -109,6 +106,14 @@ while [ $# -gt 0 ]; do system_tests=0 shift ;; + --tests) + tests=1 + shift + ;; + --notests) + tests=0 + shift + ;; *) usage ;; @@ -118,9 +123,12 @@ done # If no actions chosen, then tell the user if [ "${benchmark}" -eq 0 -a \ "${build}" -eq 0 -a \ - "${library_tests}" -eq 0 -a \ - "${system_tests}" -eq 0 ]; then - fatal 'Error: No actions specified (for example, --build), exiting' + "${system_tests}" -eq 0 -a \ + "${tests}" -eq 0 \ +]; then + echo 'No actions specified, defaulting to --build --tests' + build=1 + tests=1 fi # Running build local or remote? @@ -155,7 +163,8 @@ for outfile in \ tests/google-cloud-python-system/Dockerfile \ tests/integration/Dockerfile \ ; do - envsubst <"${outfile}".in >"${outfile}" '$DEBIAN_BASE_IMAGE $STAGING_IMAGE $GOOGLE_CLOUD_PROJECT_FOR_TESTS' + envsubst <"${outfile}".in >"${outfile}" \ + '$DEBIAN_BASE_IMAGE $STAGING_IMAGE $GOOGLE_CLOUD_PROJECT_FOR_TESTS $TAG' done # Make some files available to the runtime builder Docker context @@ -174,36 +183,27 @@ cp -a scripts/testdata/hello_world/main.py tests/eventlet/main.py # Build images and push to GCR if [ "${build}" -eq 1 ]; then echo "Building images" - ${gcloud_cmd} --config cloudbuild.yaml --substitutions "${substitutions}" + ${gcloud_cmd} --config cloudbuild.yaml --substitutions "${build_substitutions}" fi -# Run just the library compatibility tests (for DPE Gardener bot usually) -if [ "${library_tests}" -eq 1 ]; then +# Run the tests that don't require (too many) external services +if [ "${tests}" -eq 1 ]; then echo "Testing compatibility with popular Python libraries" - ${gcloud_cmd} --config cloudbuild_library_tests.yaml --substitutions "${substitutions}" + ${gcloud_cmd} --config cloudbuild_tests.yaml --substitutions "${substitutions}" fi -# If both system tests and benchmarks are requested, run them both -# even if one or the other has errors. If the build step had errors, -# this script will have already exited. -exit_code=0 - # Run system tests if [ "${system_tests}" -eq 1 ]; then echo "Running system tests using project ${GOOGLE_CLOUD_PROJECT_FOR_TESTS}" trap "rm -f tests/google-cloud-python-system/credentials.json" EXIT cp "${GOOGLE_APPLICATION_CREDENTIALS_FOR_TESTS}" tests/google-cloud-python-system/credentials.json - ${gcloud_cmd} --config cloudbuild_system_tests.yaml --substitutions "${substitutions}" || \ - exit_code=1 + ${gcloud_cmd} --config cloudbuild_system_tests.yaml --substitutions "${substitutions}" rm -f tests/google-cloud-python-system/credentials.json fi # Run benchmarks if [ "${benchmark}" -eq 1 ] ; then echo "Running benchmark" - ${gcloud_cmd} --config cloudbuild_benchmark.yaml --substitutions "${substitutions}" || \ - exit_code=1 + ${gcloud_cmd} --config cloudbuild_benchmark.yaml --substitutions "${substitutions}" fi - -exit ${exit_code} diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 68b5c5bb..1eabb997 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -11,35 +11,12 @@ steps: name: gcr.io/cloud-builders/docker:latest args: ['build', '--tag=${_DOCKER_NAMESPACE}/python:${_TAG}', '--no-cache', '/workspace/runtime-image/'] -- # Validate structure of base runtime image - name: gcr.io/gcp-runtimes/structure_test:latest - args: [ - '-i', '${_DOCKER_NAMESPACE}/python:${_TAG}', - '--config', '/workspace/tests/virtualenv/virtualenv_default.yaml', - '--config', '/workspace/tests/virtualenv/virtualenv_python34.yaml', - '--config', '/workspace/tests/virtualenv/virtualenv_python35.yaml', - '--config', '/workspace/tests/virtualenv/virtualenv_python36.yaml', - '--config', '/workspace/tests/no-virtualenv/no-virtualenv.yaml', - '--config', '/workspace/tests/python2-libraries/python2-libraries.yaml', - '--config', '/workspace/tests/python3-libraries/python3-libraries.yaml', - '--config', '/workspace/tests/license-test/license-test.yaml', - '-v' - ] -- # Run compatibility tests - name: gcr.io/cloud-builders/docker:latest - args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/eventlet:${_TAG}', - '--no-cache', '/workspace/tests/eventlet/'] -- # Build image to run google client library unit tests - name: gcr.io/cloud-builders/docker:latest - args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/google-cloud-python:${_TAG}', - '--no-cache', '/workspace/tests/google-cloud-python/'] -- # Run google client library unit tests - name: ${_DOCKER_NAMESPACE}/python/tests/google-cloud-python:${_TAG} - # Build runtime builder image name: gcr.io/cloud-builders/docker:latest args: ['build', '--tag=${_BUILDER_DOCKER_NAMESPACE}/python/gen-dockerfile:${_TAG}', '--no-cache', '/workspace/builder/gen-dockerfile/'] images: [ + '${_DOCKER_NAMESPACE}/python/interpreter-builder:${_TAG}', '${_DOCKER_NAMESPACE}/python:${_TAG}', '${_BUILDER_DOCKER_NAMESPACE}/python/gen-dockerfile:${_TAG}', ] diff --git a/cloudbuild_benchmark.yaml b/cloudbuild_benchmark.yaml index 616991ef..a960bc9b 100644 --- a/cloudbuild_benchmark.yaml +++ b/cloudbuild_benchmark.yaml @@ -3,10 +3,6 @@ steps: - name: gcr.io/cloud-builders/docker:latest args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/benchmark:${_TAG}', '--no-cache', '/workspace/tests/benchmark/'] - env: [ - # Avoid warning about unused substitutions - 'UNUSED1=${_BUILDER_DOCKER_NAMESPACE}', - ] images: [ # Intentionally empty ] diff --git a/cloudbuild_library_tests.yaml b/cloudbuild_library_tests.yaml deleted file mode 100644 index f547d73f..00000000 --- a/cloudbuild_library_tests.yaml +++ /dev/null @@ -1,20 +0,0 @@ -timeout: 1800s -steps: -- # Check that we can install important libraries without error - name: gcr.io/gcp-runtimes/structure_test:latest - args: [ - '-i', '${_DOCKER_NAMESPACE}/python:${_TAG}', - '--config', '/workspace/tests/python2-libraries/python2-libraries.yaml', - '--config', '/workspace/tests/python3-libraries/python3-libraries.yaml', - '-v' - ] - env: [ - # Avoid warning about unused substitutions - 'UNUSED1=${_BUILDER_DOCKER_NAMESPACE}', - ] -- # Run compatibility tests - name: gcr.io/cloud-builders/docker:latest - args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/eventlet:${_TAG}', - '--no-cache', '/workspace/tests/eventlet/'] -images: [ -] diff --git a/cloudbuild_system_tests.yaml b/cloudbuild_system_tests.yaml index f01a48e7..3cac03e2 100644 --- a/cloudbuild_system_tests.yaml +++ b/cloudbuild_system_tests.yaml @@ -4,10 +4,6 @@ steps: args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/google-cloud-python-system:${_TAG}', '--no-cache', '/workspace/tests/google-cloud-python-system/'] - name: ${_DOCKER_NAMESPACE}/python/tests/google-cloud-python-system:${_TAG} - env: [ - # Avoid warning about unused substitutions - 'UNUSED1=${_BUILDER_DOCKER_NAMESPACE}', - ] images: [ # Intentionally empty ] diff --git a/cloudbuild_tests.yaml b/cloudbuild_tests.yaml new file mode 100644 index 00000000..806e8b12 --- /dev/null +++ b/cloudbuild_tests.yaml @@ -0,0 +1,28 @@ +timeout: 3600s +steps: +- # Validate structure of base runtime image + name: gcr.io/gcp-runtimes/structure_test:latest + args: [ + '-i', '${_DOCKER_NAMESPACE}/python:${_TAG}', + '--config', '/workspace/tests/virtualenv/virtualenv_default.yaml', + '--config', '/workspace/tests/virtualenv/virtualenv_python34.yaml', + '--config', '/workspace/tests/virtualenv/virtualenv_python35.yaml', + '--config', '/workspace/tests/virtualenv/virtualenv_python36.yaml', + '--config', '/workspace/tests/no-virtualenv/no-virtualenv.yaml', + '--config', '/workspace/tests/python2-libraries/python2-libraries.yaml', + '--config', '/workspace/tests/python3-libraries/python3-libraries.yaml', + '--config', '/workspace/tests/license-test/license-test.yaml', + '-v' + ] +- # Run compatibility tests + name: gcr.io/cloud-builders/docker:latest + args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/eventlet:${_TAG}', + '--no-cache', '/workspace/tests/eventlet/'] +- # Build image to run google client library unit tests + name: gcr.io/cloud-builders/docker:latest + args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/google-cloud-python:${_TAG}', + '--no-cache', '/workspace/tests/google-cloud-python/'] +- # Run google client library unit tests + name: ${_DOCKER_NAMESPACE}/python/tests/google-cloud-python:${_TAG} +images: [ +] From f6bc889f1bb99dee8277b43d6a8dea1d00c91f4e Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 20 Oct 2017 15:53:28 -0700 Subject: [PATCH 131/256] Remove --tests and --systems_tests to --test and --system_test --- build.sh | 38 +++++++++---------- ..._tests.yaml => cloudbuild_system_test.yaml | 0 cloudbuild_tests.yaml => cloudbuild_test.yaml | 0 3 files changed, 19 insertions(+), 19 deletions(-) rename cloudbuild_system_tests.yaml => cloudbuild_system_test.yaml (100%) rename cloudbuild_tests.yaml => cloudbuild_test.yaml (100%) diff --git a/build.sh b/build.sh index ca651fb7..ba20b8c8 100755 --- a/build.sh +++ b/build.sh @@ -19,8 +19,8 @@ set -euo pipefail # Actions benchmark=0 # Should run benchmarks? build=0 # Should build images? -system_tests=0 # Should run system tests? -tests=0 # Should run standard test suite? +system_test=0 # Should run system tests? +test=0 # Should run standard test suite? local=0 # Should run using local Docker daemon instead of GCR? @@ -41,9 +41,9 @@ Build and test artifacts in this repository Options: --[no]benchmark: Run benchmarking suite (default false) --[no]build: Build all images (default true if no options set) - --[no]tests: Run basic tests (default true if no options set) + --[no]test: Run basic tests (default true if no options set) --[no]local: Build images using local Docker daemon (default false) - --[no]system_tests: Run system tests (default false) + --[no]system_test: Run system tests (default false) " } @@ -98,20 +98,20 @@ while [ $# -gt 0 ]; do local=0 shift ;; - --system_tests) - system_tests=1 + --system_test) + system_test=1 shift ;; - --nosystem_tests) - system_tests=0 + --nosystem_test) + system_test=0 shift ;; - --tests) - tests=1 + --test) + test=1 shift ;; - --notests) - tests=0 + --notest) + test=0 shift ;; *) @@ -123,12 +123,12 @@ done # If no actions chosen, then tell the user if [ "${benchmark}" -eq 0 -a \ "${build}" -eq 0 -a \ - "${system_tests}" -eq 0 -a \ - "${tests}" -eq 0 \ + "${system_test}" -eq 0 -a \ + "${test}" -eq 0 \ ]; then - echo 'No actions specified, defaulting to --build --tests' + echo 'No actions specified, defaulting to --build --test' build=1 - tests=1 + test=1 fi # Running build local or remote? @@ -137,7 +137,7 @@ if [ "${local}" -eq 1 ]; then fi # Read action-specific environment variables -if [ "${system_tests}" -eq 1 ]; then +if [ "${system_test}" -eq 1 ]; then if [ -z "${GOOGLE_APPLICATION_CREDENTIALS_FOR_TESTS+set}" ] ; then fatal 'Error: $GOOGLE_APPLICATION_CREDENTIALS_FOR_TESTS is not set; invoke with something like GOOGLE_APPLICATION_CREDENTIALS_FOR_TESTS=/path/to/service/account/creds.json' fi @@ -187,13 +187,13 @@ if [ "${build}" -eq 1 ]; then fi # Run the tests that don't require (too many) external services -if [ "${tests}" -eq 1 ]; then +if [ "${test}" -eq 1 ]; then echo "Testing compatibility with popular Python libraries" ${gcloud_cmd} --config cloudbuild_tests.yaml --substitutions "${substitutions}" fi # Run system tests -if [ "${system_tests}" -eq 1 ]; then +if [ "${system_test}" -eq 1 ]; then echo "Running system tests using project ${GOOGLE_CLOUD_PROJECT_FOR_TESTS}" trap "rm -f tests/google-cloud-python-system/credentials.json" EXIT diff --git a/cloudbuild_system_tests.yaml b/cloudbuild_system_test.yaml similarity index 100% rename from cloudbuild_system_tests.yaml rename to cloudbuild_system_test.yaml diff --git a/cloudbuild_tests.yaml b/cloudbuild_test.yaml similarity index 100% rename from cloudbuild_tests.yaml rename to cloudbuild_test.yaml From b25687d8fe0348421901fb0b255417649691e2ed Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 20 Oct 2017 16:52:19 -0700 Subject: [PATCH 132/256] Fix typo in filename --- build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sh b/build.sh index ba20b8c8..7e2370d4 100755 --- a/build.sh +++ b/build.sh @@ -189,7 +189,7 @@ fi # Run the tests that don't require (too many) external services if [ "${test}" -eq 1 ]; then echo "Testing compatibility with popular Python libraries" - ${gcloud_cmd} --config cloudbuild_tests.yaml --substitutions "${substitutions}" + ${gcloud_cmd} --config cloudbuild_test.yaml --substitutions "${substitutions}" fi # Run system tests @@ -198,7 +198,7 @@ if [ "${system_test}" -eq 1 ]; then trap "rm -f tests/google-cloud-python-system/credentials.json" EXIT cp "${GOOGLE_APPLICATION_CREDENTIALS_FOR_TESTS}" tests/google-cloud-python-system/credentials.json - ${gcloud_cmd} --config cloudbuild_system_tests.yaml --substitutions "${substitutions}" + ${gcloud_cmd} --config cloudbuild_system_test.yaml --substitutions "${substitutions}" rm -f tests/google-cloud-python-system/credentials.json fi From ca918fda5a777bc7a127fa52c8d207d4f038c7dd Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 20 Oct 2017 17:07:23 -0700 Subject: [PATCH 133/256] Remove m2crypto package which doesn't work under Python 3 --- tests/python3-libraries/requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 614f2f37..edbe81f6 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -73,7 +73,6 @@ kombu==4.1.0 linecache2==1.0.0 logilab-common==1.4.1 lxml==4.1.0 -m2crypto==0.27.0 mako==1.0.7 manifestparser==1.1 markdown==2.6.9 From d4afefe05e8b5ae72faba986d9429bfbc2f6c919 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 20 Oct 2017 18:44:30 -0700 Subject: [PATCH 134/256] Add crcmod to compatilibity test suite, it's used by gsutil. --- tests/python2-libraries/requirements.txt | 1 + tests/python3-libraries/requirements.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 87e9e53a..8711bad0 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -31,6 +31,7 @@ configobj==5.0.6 cov-core==1.15.0 coverage==4.4.1 coveralls==1.2.0 +crcmod==1.7 cryptography==2.1.1 cssselect==1.0.1 cython==0.27.1 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index edbe81f6..dba60564 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -29,6 +29,7 @@ configobj==5.0.6 cov-core==1.15.0 coverage==4.4.1 coveralls==1.2.0 +crcmod==1.7 cryptography==2.1.1 cssselect==1.0.1 cython==0.27.1 From 509cb5ce2768733948f3da42d25bac3ff735bf75 Mon Sep 17 00:00:00 2001 From: Angela Li Date: Tue, 24 Oct 2017 16:32:44 -0700 Subject: [PATCH 135/256] Remove sub directories of metrics scripts (#159) --- perf_dashboard/__init__.py | 0 perf_dashboard/{stackoverflow => }/posts_stats.py | 4 +--- .../{clientlibs_download => }/python_clientlibs_download.py | 6 ++---- 3 files changed, 3 insertions(+), 7 deletions(-) create mode 100644 perf_dashboard/__init__.py rename perf_dashboard/{stackoverflow => }/posts_stats.py (95%) rename perf_dashboard/{clientlibs_download => }/python_clientlibs_download.py (97%) diff --git a/perf_dashboard/__init__.py b/perf_dashboard/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/perf_dashboard/stackoverflow/posts_stats.py b/perf_dashboard/posts_stats.py similarity index 95% rename from perf_dashboard/stackoverflow/posts_stats.py rename to perf_dashboard/posts_stats.py index 280fd42c..efb9c6fc 100644 --- a/perf_dashboard/stackoverflow/posts_stats.py +++ b/perf_dashboard/posts_stats.py @@ -25,9 +25,7 @@ from google.cloud import bigquery -# Need this to import the local helper function -sys.path.insert(0, os.path.abspath(__file__+"/../../..")) -from perf_dashboard import bq_utils +import bq_utils GCLOUD_PROJECT_ENV = 'GCLOUD_PROJECT' DATASET_NAME = 'stackoverflow' diff --git a/perf_dashboard/clientlibs_download/python_clientlibs_download.py b/perf_dashboard/python_clientlibs_download.py similarity index 97% rename from perf_dashboard/clientlibs_download/python_clientlibs_download.py rename to perf_dashboard/python_clientlibs_download.py index 3c8f79fe..3866fd92 100644 --- a/perf_dashboard/clientlibs_download/python_clientlibs_download.py +++ b/perf_dashboard/python_clientlibs_download.py @@ -20,9 +20,7 @@ from google.cloud import bigquery -sys.path.insert(0, os.path.abspath(__file__+"/../../..")) -from perf_dashboard import bq_utils - +import bq_utils GCLOUD_PROJECT_ENV = 'GCLOUD_PROJECT' @@ -136,7 +134,7 @@ def main(): clientlibs_table_name=table_name, date_str=datetime.datetime.now().strftime("%Y%m%d")) bq_utils.insert_rows( - project=project, + project=os.environ.get(GCLOUD_PROJECT_ENV), dataset_name=DATASET_NAME, table_name=table_name, rows=rows) From 2e621ba203ed43d7e562cbc35850cdc6ac97dd7a Mon Sep 17 00:00:00 2001 From: Douglas Greiman Date: Tue, 31 Oct 2017 11:05:07 -0700 Subject: [PATCH 136/256] Add CODEOWNERS file (#162) --- CODEOWNERS | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 CODEOWNERS diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 00000000..a2d4a67f --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1,4 @@ +# Code owners file. +# This file controls who is tagged for review for any given pull request. + +* @duggelz @liyanhui1228 @jonparrott From 29fe28bf4fbb8e3b02122081d5908ee7c71450e7 Mon Sep 17 00:00:00 2001 From: Douglas Greiman Date: Tue, 31 Oct 2017 11:05:45 -0700 Subject: [PATCH 137/256] Add honcho package to library test (#163) --- tests/python2-libraries/requirements.txt | 3 ++- tests/python3-libraries/requirements.txt | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 8711bad0..dee9abbf 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -63,7 +63,8 @@ graphite-web==1.0.2 greenlet==0.4.12 gunicorn==19.7.1 hiredis==0.2.0 -html5lib +honcho==1.0.1 +html5lib==0.999999999 httplib2==0.10.3 idna==2.6 ipaddress==1.0.18 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index dba60564..fe9d4156 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -59,7 +59,8 @@ google-api-python-client==1.6.4 greenlet==0.4.12 gunicorn==19.7.1 hiredis==0.2.0 -html5lib +honcho=1.0.1 +html5lib==0.999999999 httplib2==0.10.3 idna==2.6 ipaddress==1.0.18 From 0965d52c5f1b557b17ad480cd9c298cb25ba5c84 Mon Sep 17 00:00:00 2001 From: Douglas Greiman Date: Tue, 31 Oct 2017 11:06:10 -0700 Subject: [PATCH 138/256] Update pip, setuptools, virtualenv when we build the base runtime image. (#161) --- runtime-image/Dockerfile.in | 13 ++++++++----- runtime-image/resources/apt-packages.txt | 1 + runtime-image/resources/requirements.txt | 3 +++ 3 files changed, 12 insertions(+), 5 deletions(-) create mode 100644 runtime-image/resources/requirements.txt diff --git a/runtime-image/Dockerfile.in b/runtime-image/Dockerfile.in index ed194257..60a6fa6f 100644 --- a/runtime-image/Dockerfile.in +++ b/runtime-image/Dockerfile.in @@ -16,15 +16,18 @@ ENV LANG C.UTF-8 # logging collection. ENV PYTHONUNBUFFERED 1 -# Upgrade pip (debian package version tends to run a few version behind) and -# install virtualenv system-wide. -RUN pip install --upgrade pip virtualenv - # Install the Google-built interpreters ADD interpreters.tar.gz / # Add Google-built interpreters to the path -ENV PATH /opt/python3.5/bin:/opt/python3.6/bin:$PATH +ENV PATH /opt/python3.6/bin:/opt/python3.5/bin:$PATH + +# Upgrade pip (debian package version tends to run a few version behind) and +# install virtualenv system-wide. +RUN /usr/bin/pip install --upgrade -r /resources/requirements.txt && \ + /usr/bin/pip3 install --upgrade -r /resources/requirements.txt && \ + /opt/python3.5/bin/pip3.5 install --upgrade -r /resources/requirements.txt && \ + /opt/python3.6/bin/pip3.6 install --upgrade -r /resources/requirements.txt # Setup the app working directory RUN ln -s /home/vmagent/app /app diff --git a/runtime-image/resources/apt-packages.txt b/runtime-image/resources/apt-packages.txt index 1fa11bcb..a62a3195 100644 --- a/runtime-image/resources/apt-packages.txt +++ b/runtime-image/resources/apt-packages.txt @@ -7,6 +7,7 @@ wget python-pip python2.7 python2.7-dev +python3-pip python3.4 python3.4-dev # Dependenies for third-party Python packages diff --git a/runtime-image/resources/requirements.txt b/runtime-image/resources/requirements.txt new file mode 100644 index 00000000..40155447 --- /dev/null +++ b/runtime-image/resources/requirements.txt @@ -0,0 +1,3 @@ +pip==9.0.1 +setuptools==36.6.0 +virtualenv==15.1.0 From ba4e5184b1715f4ee89c68d53bd912c44f216215 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Tue, 31 Oct 2017 11:06:59 -0700 Subject: [PATCH 139/256] Auto-update dependencies. (#153) --- tests/python2-libraries/requirements.txt | 42 ++++++++++++------------ tests/python3-libraries/requirements.txt | 42 ++++++++++++------------ 2 files changed, 42 insertions(+), 42 deletions(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index dee9abbf..6db2c17c 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -1,12 +1,12 @@ alembic==0.9.6 amqp==2.2.2 amqplib==1.0.2 -ansible==2.4.0.0 +ansible==2.4.1.0 anyjson==0.3.3 apache-libcloud==2.2.1 argparse==1.4.0 astroid==1.5.3 -awscli==1.11.170 +awscli==1.11.178 babel==2.5.1 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 @@ -16,7 +16,7 @@ billiard==3.5.0.3 blessings==1.6 blinker==1.4 boto==2.48.0 -botocore==1.7.28 +botocore==1.7.36 bottle==0.12.13 carbon==1.0.2 celery==4.1.0 @@ -32,16 +32,16 @@ cov-core==1.15.0 coverage==4.4.1 coveralls==1.2.0 crcmod==1.7 -cryptography==2.1.1 +cryptography==2.1.2 cssselect==1.0.1 -cython==0.27.1 +cython==0.27.2 decorator==4.1.2 django-celery==3.2.1 django-debug-toolbar==1.8 -django-extensions==1.9.1 +django-extensions==1.9.6 django==1.11.6 django_compress==1.0.1 -djangorestframework==3.7.0 +djangorestframework==3.7.1 docker-py==1.10.6 docopt==0.6.2 docutils==0.14 @@ -52,7 +52,7 @@ eventlet==0.21.0 extras==1.0.0 fabric==1.14.0 fixtures==3.0.0 -flake8==3.4.1 +flake8==3.5.0 flask==0.12.2 funcsigs==1.0.2 functools32==3.2.3.post2 @@ -101,22 +101,22 @@ mysql-python==1.2.5 ndg-httpsclient==0.4.3 netaddr==0.7.19 netifaces==0.10.6 -newrelic==2.94.0.79 +newrelic==2.96.0.80 nose==1.3.7 numpy==1.13.3 oauth2==1.9.0.post1 oauth2client==4.1.2 -oauthlib==2.0.4 +oauthlib==2.0.6 ordereddict==1.1 -oslo.config==4.13.1 -pandas==0.20.3 +oslo.config==5.0.0 +pandas==0.21.0 paramiko==2.3.1 passlib==1.7.1 paste==2.0.3 pastedeploy==1.5.2 pastescript==2.0.2 pbr==3.1.1 -pep8==1.7.0 +pep8==1.7.1 pexpect==4.2.1 pika==0.11.0 pillow==4.3.0 @@ -124,7 +124,7 @@ pip==9.0.1 prettytable protobuf==3.4.0 psutil==5.4.0 -psycopg2==2.7.3.1 +psycopg2==2.7.3.2 py==1.4.34 pyasn1-modules==0.1.5 pyasn1==0.3.7 @@ -147,24 +147,24 @@ pytest==3.2.3 python-cjson==1.2.1 python-daemon==2.1.2 python-dateutil==2.6.1 -python-gflags==3.1.1 +python-gflags==3.1.2 python-keystoneclient==3.13.0 python-memcached==1.58 python-mimeparse==1.6.0 python-novaclient==9.1.0 python-subunit==1.2.0 python-swiftclient==3.4.0 -pytz==2017.2 +pytz==2017.3 pyyaml==3.12 -pyzmq==16.0.2 -raven==6.2.1 +pyzmq==16.0.3 +raven==6.3.0 redis==2.10.6 repoze.lru==0.7 requests-oauthlib==0.8.0 requests==2.18.4 retrying==1.3.3 rsa==3.4.2 -scipy==0.19.1 +scipy==1.0.0 selenium==3.6.0 setuptools-git==1.2 setuptools==36.6.0 @@ -173,7 +173,7 @@ simplejson==3.11.1 six==1.11.0 snowballstemmer==1.2.1 south==1.0.2 -sphinx==1.6.4 +sphinx==1.6.5 sqlalchemy-migrate==0.11.0 sqlalchemy==1.1.14 sqlparse==0.2.4 @@ -199,7 +199,7 @@ waitress==1.1.0 warlock==1.3.0 webob==1.7.3 websocket-client==0.44.0 -webtest==2.0.28 +webtest==2.0.29 werkzeug==0.12.2 wheel==0.30.0 xlrd==1.1.0 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index fe9d4156..cca4bef2 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -1,12 +1,12 @@ alembic==0.9.6 amqp==2.2.2 amqplib==1.0.2 -ansible==2.4.0.0 +ansible==2.4.1.0 anyjson==0.3.3 apache-libcloud==2.2.1 argparse==1.4.0 astroid==1.5.3 -awscli==1.11.170 +awscli==1.11.178 babel==2.5.1 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 @@ -15,7 +15,7 @@ billiard==3.5.0.3 blessings==1.6 blinker==1.4 boto==2.48.0 -botocore==1.7.28 +botocore==1.7.36 bottle==0.12.13 celery==4.1.0 certifi==2017.7.27.1 @@ -30,16 +30,16 @@ cov-core==1.15.0 coverage==4.4.1 coveralls==1.2.0 crcmod==1.7 -cryptography==2.1.1 +cryptography==2.1.2 cssselect==1.0.1 -cython==0.27.1 +cython==0.27.2 decorator==4.1.2 django-celery==3.2.1 django-debug-toolbar==1.8 -django-extensions==1.9.1 +django-extensions==1.9.6 django==1.11.6 django_compress==1.0.1 -djangorestframework==3.7.0 +djangorestframework==3.7.1 docker-py==1.10.6 docopt==0.6.2 docutils==0.14 @@ -50,7 +50,7 @@ eventlet==0.21.0 extras==1.0.0 fabric==1.14.0 fixtures==3.0.0 -flake8==3.4.1 +flake8==3.5.0 flask==0.12.2 funcsigs==1.0.2 futures==3.1.1 @@ -94,22 +94,22 @@ msgpack-python==0.4.8 ndg-httpsclient==0.4.3 netaddr==0.7.19 netifaces==0.10.6 -newrelic==2.94.0.79 +newrelic==2.96.0.80 nose==1.3.7 numpy==1.13.3 oauth2==1.9.0.post1 oauth2client==4.1.2 -oauthlib==2.0.4 +oauthlib==2.0.6 ordereddict==1.1 -oslo.config==4.13.1 -pandas==0.20.3 +oslo.config==5.0.0 +pandas==0.21.0 paramiko==2.3.1 passlib==1.7.1 paste==2.0.3 pastedeploy==1.5.2 pastescript==2.0.2 pbr==3.1.1 -pep8==1.7.0 +pep8==1.7.1 pexpect==4.2.1 pika==0.11.0 pillow==4.3.0 @@ -117,7 +117,7 @@ pip==9.0.1 prettytable protobuf==3.4.0 psutil==5.4.0 -psycopg2==2.7.3.1 +psycopg2==2.7.3.2 py==1.4.34 pyasn1-modules==0.1.5 pyasn1==0.3.7 @@ -138,24 +138,24 @@ pytest-cov==2.5.1 pytest==3.2.3 python-daemon==2.1.2 python-dateutil==2.6.1 -python-gflags==3.1.1 +python-gflags==3.1.2 python-keystoneclient==3.13.0 python-memcached==1.58 python-mimeparse==1.6.0 python-novaclient==9.1.0 python-subunit==1.2.0 python-swiftclient==3.4.0 -pytz==2017.2 +pytz==2017.3 pyyaml==3.12 -pyzmq==16.0.2 -raven==6.2.1 +pyzmq==16.0.3 +raven==6.3.0 redis==2.10.6 repoze.lru==0.7 requests-oauthlib==0.8.0 requests==2.18.4 retrying==1.3.3 rsa==3.4.2 -scipy==0.19.1 +scipy==1.0.0 selenium==3.6.0 setuptools-git==1.2 setuptools==36.6.0 @@ -164,7 +164,7 @@ simplejson==3.11.1 six==1.11.0 snowballstemmer==1.2.1 south==1.0.2 -sphinx==1.6.4 +sphinx==1.6.5 sqlalchemy-migrate==0.11.0 sqlalchemy==1.1.14 sqlparse==0.2.4 @@ -188,7 +188,7 @@ waitress==1.1.0 warlock==1.3.0 webob==1.7.3 websocket-client==0.44.0 -webtest==2.0.28 +webtest==2.0.29 werkzeug==0.12.2 wheel==0.30.0 xlrd==1.1.0 From babc14b911dc1c49d8bdc97130510f89a0bcabf7 Mon Sep 17 00:00:00 2001 From: Nick Kubala Date: Thu, 26 Oct 2017 15:53:51 -0700 Subject: [PATCH 140/256] migrate structure tests to new version --- cloudbuild_test.yaml | 29 +++++++++++++++---- tests/python2-libraries/Dockerfile | 3 ++ .../python2-libraries/python2-libraries.yaml | 5 +--- tests/python3-libraries/Dockerfile | 3 ++ .../python3-libraries/python3-libraries.yaml | 12 +++----- tests/virtualenv/virtualenv_default.yaml | 14 +++++---- tests/virtualenv/virtualenv_python34.yaml | 9 +++--- tests/virtualenv/virtualenv_python35.yaml | 7 ++++- tests/virtualenv/virtualenv_python36.yaml | 7 ++++- 9 files changed, 59 insertions(+), 30 deletions(-) create mode 100644 tests/python2-libraries/Dockerfile create mode 100644 tests/python3-libraries/Dockerfile diff --git a/cloudbuild_test.yaml b/cloudbuild_test.yaml index 806e8b12..fbad487a 100644 --- a/cloudbuild_test.yaml +++ b/cloudbuild_test.yaml @@ -1,7 +1,27 @@ timeout: 3600s steps: -- # Validate structure of base runtime image - name: gcr.io/gcp-runtimes/structure_test:latest + # Validate structure of base runtime image +- name: gcr.io/cloud-builders/docker:latest + args: ['build', '-t', 'python2-libraries-intermediate', '--build-arg', + 'intermediate_image=${_DOCKER_NAMESPACE}/python:${_TAG}', + '/workspace/tests/python2-libraries'] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.1.1 + args: [ + '-test.v', + '-image', 'python2-libraries-intermediate', + '/workspace/tests/python2-libraries/python2-libraries.yaml' + ] +- name: gcr.io/cloud-builders/docker:latest + args: ['build', '-t', 'python3-libraries-intermediate', '--build-arg', + 'intermediate_image=${_DOCKER_NAMESPACE}/python:${_TAG}', + '/workspace/tests/python3-libraries'] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.1.1 + args: [ + '-test.v', + '-image', 'python3-libraries-intermediate', + '/workspace/tests/python3-libraries/python3-libraries.yaml' + ] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.1.1 args: [ '-i', '${_DOCKER_NAMESPACE}/python:${_TAG}', '--config', '/workspace/tests/virtualenv/virtualenv_default.yaml', @@ -9,8 +29,6 @@ steps: '--config', '/workspace/tests/virtualenv/virtualenv_python35.yaml', '--config', '/workspace/tests/virtualenv/virtualenv_python36.yaml', '--config', '/workspace/tests/no-virtualenv/no-virtualenv.yaml', - '--config', '/workspace/tests/python2-libraries/python2-libraries.yaml', - '--config', '/workspace/tests/python3-libraries/python3-libraries.yaml', '--config', '/workspace/tests/license-test/license-test.yaml', '-v' ] @@ -24,5 +42,4 @@ steps: '--no-cache', '/workspace/tests/google-cloud-python/'] - # Run google client library unit tests name: ${_DOCKER_NAMESPACE}/python/tests/google-cloud-python:${_TAG} -images: [ -] +images: [] diff --git a/tests/python2-libraries/Dockerfile b/tests/python2-libraries/Dockerfile new file mode 100644 index 00000000..c2647292 --- /dev/null +++ b/tests/python2-libraries/Dockerfile @@ -0,0 +1,3 @@ +ARG intermediate_image +FROM $intermediate_image +COPY requirements.txt /requirements.txt diff --git a/tests/python2-libraries/python2-libraries.yaml b/tests/python2-libraries/python2-libraries.yaml index 5d298b58..3a2771ac 100644 --- a/tests/python2-libraries/python2-libraries.yaml +++ b/tests/python2-libraries/python2-libraries.yaml @@ -7,10 +7,7 @@ globalEnvVars: value: "/env/bin:$PATH" commandTests: - - name: "virtual env teardown" - command: ["rm", "-rf", "/env"] - - name: "requirements" setup: [["virtualenv", "/env"]] - command: ["pip", "install", "-r", "/workspace/tests/python2-libraries/requirements.txt"] + command: ["pip", "install", "-r", "/requirements.txt"] exitCode: 0 diff --git a/tests/python3-libraries/Dockerfile b/tests/python3-libraries/Dockerfile new file mode 100644 index 00000000..c2647292 --- /dev/null +++ b/tests/python3-libraries/Dockerfile @@ -0,0 +1,3 @@ +ARG intermediate_image +FROM $intermediate_image +COPY requirements.txt /requirements.txt diff --git a/tests/python3-libraries/python3-libraries.yaml b/tests/python3-libraries/python3-libraries.yaml index ef15c0a1..ace58132 100644 --- a/tests/python3-libraries/python3-libraries.yaml +++ b/tests/python3-libraries/python3-libraries.yaml @@ -8,15 +8,11 @@ globalEnvVars: commandTests: - name: "requirements 3.5" - setup: - - ["rm", "-rf", "/env"] - - ["virtualenv", "-p", "/opt/python3.5/bin/python3.5", "/env"] - command: ["pip", "install", "-r", "/workspace/tests/python3-libraries/requirements.txt"] + setup: [["virtualenv", "-p", "/opt/python3.5/bin/python3.5", "/env"]] + command: ["pip", "install", "-r", "/requirements.txt"] exitCode: 0 - name: "requirements 3.6" - setup: - - ["rm", "-rf", "/env"] - - ["virtualenv", "-p", "/opt/python3.6/bin/python3.6", "/env"] - command: ["pip", "install", "-r", "/workspace/tests/python3-libraries/requirements.txt"] + setup: [["virtualenv", "-p", "/opt/python3.6/bin/python3.6", "/env"]] + command: ["pip", "install", "-r", "/requirements.txt"] exitCode: 0 diff --git a/tests/virtualenv/virtualenv_default.yaml b/tests/virtualenv/virtualenv_default.yaml index 314a3d0c..8695e319 100644 --- a/tests/virtualenv/virtualenv_default.yaml +++ b/tests/virtualenv/virtualenv_default.yaml @@ -7,13 +7,14 @@ globalEnvVars: value: "/env/bin:$PATH" commandTests: - - name: "virtual env teardown" - command: ["rm", "-rf", "/env"] - - name: "python installation" command: ["which", "python"] expectedOutput: ["/usr/bin/python\n"] + - name: "pip installation" + command: ["which", "pip"] + expectedOutput: ["/usr/local/bin/pip\n"] + - name: "virtualenv installation" setup: [["virtualenv", "/env"]] command: ["which", "python"] @@ -25,12 +26,13 @@ commandTests: # https://bugs.python.org/issue18338 expectedError: ["Python 2.7.9\n"] - - name: "pip installation" + - name: "virtualenv pip installation" + setup: [["virtualenv", "/env"]] command: ["which", "pip"] expectedOutput: ["/env/bin/pip\n"] - - name: "gunicorn flask" - setup: [["pip", "install", "gunicorn", "flask"]] + - name: "virtualenv gunicorn flask" + setup: [["virtualenv", "/env"], ["pip", "install", "gunicorn", "flask"]] command: ["which", "gunicorn"] expectedOutput: ["/env/bin/gunicorn"] diff --git a/tests/virtualenv/virtualenv_python34.yaml b/tests/virtualenv/virtualenv_python34.yaml index 082b3b6e..e35e9693 100644 --- a/tests/virtualenv/virtualenv_python34.yaml +++ b/tests/virtualenv/virtualenv_python34.yaml @@ -7,9 +7,6 @@ globalEnvVars: value: "/env/bin:$PATH" commandTests: - - name: "virtual env teardown" - command: ["rm", "-rf", "/env"] - - name: "python installation" command: ["which", "python3.4"] expectedOutput: ["/usr/bin/python3.4\n"] @@ -20,23 +17,27 @@ commandTests: expectedOutput: ["/env/bin/python\n"] - name: "virtualenv python3 installation" + setup: [["virtualenv", "-p", "python3.4", "/env"]] command: ["which", "python3"] expectedOutput: ["/env/bin/python3\n"] - name: "python version" + setup: [["virtualenv", "-p", "python3.4", "/env"]] command: ["python", "--version"] expectedOutput: ["Python 3.4.2\n"] - name: "pip installation" + setup: [["virtualenv", "-p", "python3.4", "/env"]] command: ["which", "pip"] expectedOutput: ["/env/bin/pip\n"] - name: "pip3 installation" + setup: [["virtualenv", "-p", "python3.4", "/env"]] command: ["which", "pip3"] expectedOutput: ["/env/bin/pip3\n"] - name: "gunicorn flask" - setup: [["pip", "install", "gunicorn", "flask"]] + setup: [["virtualenv", "-p", "python3.4", "/env"], ["pip", "install", "gunicorn", "flask"]] command: ["which", "gunicorn"] expectedOutput: ["/env/bin/gunicorn"] diff --git a/tests/virtualenv/virtualenv_python35.yaml b/tests/virtualenv/virtualenv_python35.yaml index c0b461e1..4b019967 100644 --- a/tests/virtualenv/virtualenv_python35.yaml +++ b/tests/virtualenv/virtualenv_python35.yaml @@ -20,27 +20,32 @@ commandTests: expectedOutput: ["/env/bin/python\n"] - name: "virtualenv python3 installation" + setup: [["virtualenv", "-p", "python3.5", "/env"]] command: ["which", "python3"] expectedOutput: ["/env/bin/python3\n"] - name: "virtualenv python3.5 installation" + setup: [["virtualenv", "-p", "python3.5", "/env"]] command: ["which", "python3.5"] expectedOutput: ["/env/bin/python3.5\n"] - name: "python version" + setup: [["virtualenv", "-p", "python3.5", "/env"]] command: ["python", "--version"] expectedOutput: ["Python 3.5.4\n"] - name: "pip installation" + setup: [["virtualenv", "-p", "python3.5", "/env"]] command: ["which", "pip"] expectedOutput: ["/env/bin/pip\n"] - name: "pip3 installation" + setup: [["virtualenv", "-p", "python3.5", "/env"]] command: ["which", "pip3"] expectedOutput: ["/env/bin/pip3\n"] - name: "gunicorn flask" - setup: [["pip", "install", "gunicorn", "flask"]] + setup: [["virtualenv", "-p", "python3.5", "/env"], ["pip", "install", "gunicorn", "flask"]] command: ["which", "gunicorn"] expectedOutput: ["/env/bin/gunicorn"] diff --git a/tests/virtualenv/virtualenv_python36.yaml b/tests/virtualenv/virtualenv_python36.yaml index ab25c28b..deea9b5d 100644 --- a/tests/virtualenv/virtualenv_python36.yaml +++ b/tests/virtualenv/virtualenv_python36.yaml @@ -20,27 +20,32 @@ commandTests: expectedOutput: ["/env/bin/python\n"] - name: "virtualenv python3 installation" + setup: [["virtualenv", "-p", "python3.6", "/env"]] command: ["which", "python3"] expectedOutput: ["/env/bin/python3\n"] - name: "virtualenv python3.6 installation" + setup: [["virtualenv", "-p", "python3.6", "/env"]] command: ["which", "python3.6"] expectedOutput: ["/env/bin/python3.6\n"] - name: "python version" + setup: [["virtualenv", "-p", "python3.6", "/env"]] command: ["python", "--version"] expectedOutput: ["Python 3.6.2\n"] - name: "pip installation" + setup: [["virtualenv", "-p", "python3.6", "/env"]] command: ["which", "pip"] expectedOutput: ["/env/bin/pip\n"] - name: "pip3 installation" + setup: [["virtualenv", "-p", "python3.6", "/env"]] command: ["which", "pip3"] expectedOutput: ["/env/bin/pip3\n"] - name: "gunicorn flask" - setup: [["pip", "install", "gunicorn", "flask"]] + setup: [["virtualenv", "-p", "python3.6", "/env"], ["pip", "install", "gunicorn", "flask"]] command: ["which", "gunicorn"] expectedOutput: ["/env/bin/gunicorn"] From e95dfdd7e4546231f69b13a6c8d182f1b31583bb Mon Sep 17 00:00:00 2001 From: Nick Kubala Date: Mon, 30 Oct 2017 12:45:24 -0700 Subject: [PATCH 141/256] fix args --- cloudbuild_test.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cloudbuild_test.yaml b/cloudbuild_test.yaml index fbad487a..c6496dec 100644 --- a/cloudbuild_test.yaml +++ b/cloudbuild_test.yaml @@ -23,14 +23,14 @@ steps: ] - name: gcr.io/gcp-runtimes/container-structure-test:v0.1.1 args: [ - '-i', '${_DOCKER_NAMESPACE}/python:${_TAG}', - '--config', '/workspace/tests/virtualenv/virtualenv_default.yaml', - '--config', '/workspace/tests/virtualenv/virtualenv_python34.yaml', - '--config', '/workspace/tests/virtualenv/virtualenv_python35.yaml', - '--config', '/workspace/tests/virtualenv/virtualenv_python36.yaml', - '--config', '/workspace/tests/no-virtualenv/no-virtualenv.yaml', - '--config', '/workspace/tests/license-test/license-test.yaml', - '-v' + '-test.v', + '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', + '/workspace/tests/virtualenv/virtualenv_default.yaml', + '/workspace/tests/virtualenv/virtualenv_python34.yaml', + '/workspace/tests/virtualenv/virtualenv_python35.yaml', + '/workspace/tests/virtualenv/virtualenv_python36.yaml', + '/workspace/tests/no-virtualenv/no-virtualenv.yaml', + '/workspace/tests/license-test/license-test.yaml' ] - # Run compatibility tests name: gcr.io/cloud-builders/docker:latest From 8995620601d3db38a9edd9e6ddc48abf2d5739ae Mon Sep 17 00:00:00 2001 From: Nick Kubala Date: Thu, 2 Nov 2017 14:19:23 -0700 Subject: [PATCH 142/256] expose /environment endpoint for gke tests (#164) * expose /environment endpoint for gke tests * mimic logic in python logging client library for determining cloud env --- tests/integration/server.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/integration/server.py b/tests/integration/server.py index cc82fb93..8e2958cd 100755 --- a/tests/integration/server.py +++ b/tests/integration/server.py @@ -17,6 +17,7 @@ from functools import wraps import json import logging +import os import google.cloud.logging import google.cloud.monitoring @@ -33,6 +34,12 @@ 'CRITICAL': (logging.critical, 'stderr') } +_APPENGINE_FLEXIBLE_ENV_VM = 'GAE_APPENGINE_HOSTNAME' +"""Environment variable set in App Engine when vm:true is set.""" + +_APPENGINE_FLEXIBLE_ENV_FLEX = 'GAE_INSTANCE' +"""Environment variable set in App Engine when env:flex is set.""" + app = Flask(__name__) @@ -132,6 +139,7 @@ def _log_default(token, level): logging.error('Error while writing logs: {0}'.format(e)) raise ErrorResponse('Error while writing logs: {0}'.format(e)) + # this is fine regardless of environment, it's only used in GAE logs return 'appengine.googleapis.com%2F{0}'.format(source) @@ -231,6 +239,16 @@ def _custom(): return json.dumps(tests), 200 +@app.route('/environment', methods=['GET']) +def _check_environment(): + # determine what cloud env we're running in; essentially, GAE vs GKE + # for GAE, we'll check the existence env vars set on + # vm:true or env:flex + # if neither exist, assume we're in GKE + return (_APPENGINE_FLEXIBLE_ENV_VM in os.environ or + _APPENGINE_FLEXIBLE_ENV_FLEX in os.environ), 200 + + class ErrorResponse(Exception): status_code = 400 From 5aa45062d97abf6419418069bbd29e1c13b942c8 Mon Sep 17 00:00:00 2001 From: Douglas Greiman Date: Thu, 2 Nov 2017 14:20:09 -0700 Subject: [PATCH 143/256] Create Debian packages for GCP Python interpreters (#140) We build binary packages directly, rather than building from a source package as the standard Debian Python packages are. --- python-interpreter-builder/DEBIAN/control.in | 25 ++++++++ python-interpreter-builder/Dockerfile.in | 12 +++- .../scripts/package-python.sh | 59 +++++++++++++++++++ 3 files changed, 94 insertions(+), 2 deletions(-) create mode 100644 python-interpreter-builder/DEBIAN/control.in create mode 100755 python-interpreter-builder/scripts/package-python.sh diff --git a/python-interpreter-builder/DEBIAN/control.in b/python-interpreter-builder/DEBIAN/control.in new file mode 100644 index 00000000..49980654 --- /dev/null +++ b/python-interpreter-builder/DEBIAN/control.in @@ -0,0 +1,25 @@ +Package: ${DEB_PACKAGE_NAME} +Version: ${DEB_PACKAGE_VERSION} +Section: python +Priority: optional +Architecture: amd64 +Maintainer: Douglas Greiman +Description: Interactive high-level object-oriented language (version ${SHORT_VERSION}) + Python is a high-level, interactive, object-oriented language. Its ${SHORT_VERSION} version + includes an extensive class library with lots of goodies for + network programming, system administration, sounds and graphics. +Depends: libbz2-1.0, + libc6, + libdb5.3, + libexpat1, + libffi6, + liblzma5, + libmpdec2, + libncursesw5, + libreadline6, + libsqlite3-0, + libssl1.0.0, + libtinfo5, + mime-support, + zlib1g +Homepage: https://www.python.org diff --git a/python-interpreter-builder/Dockerfile.in b/python-interpreter-builder/Dockerfile.in index eba153e9..90e606eb 100644 --- a/python-interpreter-builder/Dockerfile.in +++ b/python-interpreter-builder/Dockerfile.in @@ -10,6 +10,7 @@ RUN apt-get update && apt-get install -yq \ debhelper \ dpkg-dev \ gcc \ + gettext-base \ libbluetooth-dev \ libbz2-dev \ libdb-dev \ @@ -43,10 +44,17 @@ ENV LANG C.UTF-8 # Add build scripts ADD scripts /scripts +ADD DEBIAN /DEBIAN # Build the Python interpreters -RUN /scripts/build-python-3.5.sh -RUN /scripts/build-python-3.6.sh +RUN mkdir -p /opt/packages && \ + echo -n "" > /opt/packages/packages.txt + +RUN /scripts/build-python-3.5.sh && \ + /scripts/package-python.sh 3.5.4 "1gcp~${TAG}" + +RUN /scripts/build-python-3.6.sh && \ + /scripts/package-python.sh 3.6.2 "1gcp~${TAG}" # Tar the interpreters. Tarring is needed because docker cp doesn't handle # links correctly. diff --git a/python-interpreter-builder/scripts/package-python.sh b/python-interpreter-builder/scripts/package-python.sh new file mode 100755 index 00000000..58cd02e6 --- /dev/null +++ b/python-interpreter-builder/scripts/package-python.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +set -euo pipefail +set -x + +function usage { + echo "Usage: $0 long_version tag +Create .deb package file for a Python interpreter with + long_version: (x.y.z) Interpreter version + tag: version suffix unique to this build +" >&2 + exit 1 +} + # Process command line +if [ -z "${1:+set}" -o -z "${2:+set}" ]; then + usage +fi +LONG_VERSION=$1 +BUILD_TAG=$2 +SHORT_VERSION=${1%.*} + +# Compute version specs +DEB_PACKAGE_NAME=gcp-python${SHORT_VERSION} +# Can't have - (hyphen) in debian revision as per +# https://www.debian.org/doc/debian-policy/ch-controlfields.html#s-f-Version +DEBIAN_REVISION=${BUILD_TAG//-/.} +DEB_PACKAGE_VERSION=${LONG_VERSION}-${DEBIAN_REVISION} + +PACKAGE_DIR=/opt/packages +# E.g. gcp-python3.6_3.6.2-1gcp~2017.07.25.110644_amd64.deb +DEB_FILENAME=${DEB_PACKAGE_NAME}_${DEB_PACKAGE_VERSION}_amd64.deb + +# Create directory for intermediate files +SCRATCH_DIR=$(mktemp --directory) +cd "${SCRATCH_DIR}" + +# Synthesize Debian control file. Note that the "Depends:" is +# currently Debian8-specific, and lacks version specifiers present in +# the standard Debian Python packages. +export DEB_PACKAGE_NAME DEB_PACKAGE_VERSION SHORT_VERSION +envsubst control \ + '${DEB_PACKAGE_NAME} ${DEB_PACKAGE_VERSION} ${SHORT_VERSION}' + +# Generate components of .deb archive +tar czf control.tar.gz control +tar czf data.tar.gz "/opt/python${SHORT_VERSION}" +echo "2.0" >debian-binary + +# Generate final .deb. +mkdir -p "${PACKAGE_DIR}" +ar rcD "${PACKAGE_DIR}/${DEB_FILENAME}" \ + debian-binary control.tar.gz data.tar.gz +rm debian-binary control.tar.gz data.tar.gz + +# Validate .deb +dpkg --install --dry-run "${PACKAGE_DIR}/${DEB_FILENAME}" + +# Add to list +echo "${DEB_FILENAME}" >> "${PACKAGE_DIR}/packages.txt" From 3aa916b1ddc1e116d62cea5f776379e1dcb7e6fb Mon Sep 17 00:00:00 2001 From: Douglas Greiman Date: Thu, 2 Nov 2017 14:20:59 -0700 Subject: [PATCH 144/256] Fix unit test failures (#165) * Update unit test script with new nox session names. * Fix typo --- RELEASING.md | 9 +++++++++ tests/google-cloud-python/run_unit_tests.sh | 8 ++++---- tests/python3-libraries/requirements.txt | 2 +- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/RELEASING.md b/RELEASING.md index f7d5a459..00b0fe16 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -67,6 +67,15 @@ following: docker run -it --entrypoint /bin/bash YOUR-IMAGE-NAME ``` +## Running tests against a released image + +To run compatibility tests against an existing image, such as +`gcr.io/google-appengine/python:latest`, run: + +```shell +DOCKER_NAMESPACE=gcr.io/google-appengine TAG=latest ./build.sh --nobuild --test +``` + ## Running benchmarks There is a benchmark suite which compares the performance of interpreters diff --git a/tests/google-cloud-python/run_unit_tests.sh b/tests/google-cloud-python/run_unit_tests.sh index 030919d8..81a6a389 100755 --- a/tests/google-cloud-python/run_unit_tests.sh +++ b/tests/google-cloud-python/run_unit_tests.sh @@ -8,10 +8,10 @@ for noxfile in */nox.py; do nox \ -f "${noxfile}" \ -e \ - "unit_tests(python_version='2.7')" \ - "unit_tests(python_version='3.4')" \ - "unit_tests(python_version='3.5')" \ - "unit_tests(python_version='3.6')" \ + "unit(py='2.7')" \ + "unit(py='3.4')" \ + "unit(py='3.5')" \ + "unit(py='3.6')" \ || exit_code=1 done diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index cca4bef2..b5bb7cae 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -59,7 +59,7 @@ google-api-python-client==1.6.4 greenlet==0.4.12 gunicorn==19.7.1 hiredis==0.2.0 -honcho=1.0.1 +honcho==1.0.1 html5lib==0.999999999 httplib2==0.10.3 idna==2.6 From 3a3b04c4a002f6a6d87f377dae8bbea475359ee5 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Fri, 3 Nov 2017 11:05:31 -0700 Subject: [PATCH 145/256] Auto-update dependencies. (#166) --- tests/integration/requirements.txt | 6 +++--- tests/python2-libraries/requirements.txt | 12 ++++++------ tests/python3-libraries/requirements.txt | 12 ++++++------ 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index eae15fe9..3e51c81d 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -1,7 +1,7 @@ Flask==0.12.2 -google-cloud-error-reporting==0.27.0 -google-cloud-logging==1.3.0 -google-cloud-monitoring==0.27.0 +google-cloud-error-reporting==0.28.0 +google-cloud-logging==1.4.0 +google-cloud-monitoring==0.28.0 gunicorn==19.7.1 requests==2.18.4 retrying==1.3.3 diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 6db2c17c..8c134e48 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -6,7 +6,7 @@ anyjson==0.3.3 apache-libcloud==2.2.1 argparse==1.4.0 astroid==1.5.3 -awscli==1.11.178 +awscli==1.11.180 babel==2.5.1 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 @@ -16,7 +16,7 @@ billiard==3.5.0.3 blessings==1.6 blinker==1.4 boto==2.48.0 -botocore==1.7.36 +botocore==1.7.38 bottle==0.12.13 carbon==1.0.2 celery==4.1.0 @@ -32,14 +32,14 @@ cov-core==1.15.0 coverage==4.4.1 coveralls==1.2.0 crcmod==1.7 -cryptography==2.1.2 +cryptography==2.1.3 cssselect==1.0.1 cython==0.27.2 decorator==4.1.2 django-celery==3.2.1 django-debug-toolbar==1.8 -django-extensions==1.9.6 -django==1.11.6 +django-extensions==1.9.7 +django==1.11.7 django_compress==1.0.1 djangorestframework==3.7.1 docker-py==1.10.6 @@ -165,7 +165,7 @@ requests==2.18.4 retrying==1.3.3 rsa==3.4.2 scipy==1.0.0 -selenium==3.6.0 +selenium==3.7.0 setuptools-git==1.2 setuptools==36.6.0 sh==1.12.14 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index b5bb7cae..2bdcc565 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -6,7 +6,7 @@ anyjson==0.3.3 apache-libcloud==2.2.1 argparse==1.4.0 astroid==1.5.3 -awscli==1.11.178 +awscli==1.11.180 babel==2.5.1 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 @@ -15,7 +15,7 @@ billiard==3.5.0.3 blessings==1.6 blinker==1.4 boto==2.48.0 -botocore==1.7.36 +botocore==1.7.38 bottle==0.12.13 celery==4.1.0 certifi==2017.7.27.1 @@ -30,14 +30,14 @@ cov-core==1.15.0 coverage==4.4.1 coveralls==1.2.0 crcmod==1.7 -cryptography==2.1.2 +cryptography==2.1.3 cssselect==1.0.1 cython==0.27.2 decorator==4.1.2 django-celery==3.2.1 django-debug-toolbar==1.8 -django-extensions==1.9.6 -django==1.11.6 +django-extensions==1.9.7 +django==1.11.7 django_compress==1.0.1 djangorestframework==3.7.1 docker-py==1.10.6 @@ -156,7 +156,7 @@ requests==2.18.4 retrying==1.3.3 rsa==3.4.2 scipy==1.0.0 -selenium==3.6.0 +selenium==3.7.0 setuptools-git==1.2 setuptools==36.6.0 sh==1.12.14 From f811fb8083a3fcd3e00c9c211af5c50d8b0e8076 Mon Sep 17 00:00:00 2001 From: Angela Li Date: Tue, 14 Nov 2017 10:35:37 -0800 Subject: [PATCH 146/256] Update metrics script to use the latest bigquery client library (#169) --- perf_dashboard/bq_utils.py | 22 ++++--------- perf_dashboard/python_clientlibs_download.py | 33 +++++++------------- 2 files changed, 18 insertions(+), 37 deletions(-) diff --git a/perf_dashboard/bq_utils.py b/perf_dashboard/bq_utils.py index d9441f07..cbad65b9 100644 --- a/perf_dashboard/bq_utils.py +++ b/perf_dashboard/bq_utils.py @@ -22,25 +22,15 @@ def insert_rows(project, dataset_name, table_name, rows): """Insert rows to bigquery table.""" client = bigquery.Client(project=project) - dataset = client.dataset(dataset_name) - table = bigquery.Table(table_name, dataset) - table.reload() - table.insert_data(rows) - + dataset_ref = client.dataset(dataset_name) + table_ref = dataset_ref.table(table_name) + table = client.get_table(table_ref) + client.create_rows(table, rows) def execute_query(query): """Execute query and return the query results.""" client = bigquery.Client() - query_job = client.run_async_query(str(uuid.uuid4()), query) - query_job.use_legacy_sql = False + query_job = client.query((query)) # Start the query job and wait it to complete - query_job.begin() - query_job.result() - - # Get the results - destination_table = query_job.destination - destination_table.reload() - results = destination_table.fetch_data() - - return results + return [row.values() for row in query_job.result()] diff --git a/perf_dashboard/python_clientlibs_download.py b/perf_dashboard/python_clientlibs_download.py index 3866fd92..cae3d2c4 100644 --- a/perf_dashboard/python_clientlibs_download.py +++ b/perf_dashboard/python_clientlibs_download.py @@ -101,27 +101,18 @@ def get_weekly_clientlibs_downloads(clientlibs_table_name, date_str): GROUP BY client_library_name """ client = bigquery.Client() - query_job = client.run_async_query( - str(uuid.uuid4()), - query, - query_parameters=( - bigquery.ArrayQueryParameter( - 'client_libs', 'STRING', - client_libs), - bigquery.ArrayQueryParameter( - 'week_dates', 'STRING', - week_dates) - )) - query_job.use_legacy_sql = False - - # Start the query job and wait it to complete - query_job.begin() - query_job.result() - - # Get the results - destination_table = query_job.destination - destination_table.reload() - results = destination_table.fetch_data() + query_parameters=[ + bigquery.ArrayQueryParameter( + 'client_libs', 'STRING', client_libs), + bigquery.ArrayQueryParameter( + 'week_dates', 'STRING', week_dates) + ] + job_config = bigquery.QueryJobConfig() + job_config.query_parameters = query_parameters + query_job = client.query(query, job_config=job_config) + + # Wait for the job to complete and get the results + results = [row.values() for row in query_job.result()] rows = [(date_time,) + row for row in results] From 4a5026620adc4916b17ce099f5ca642ede5a2bda Mon Sep 17 00:00:00 2001 From: Douglas Greiman Date: Wed, 15 Nov 2017 10:45:41 -0800 Subject: [PATCH 147/256] Revert to previous virtualenv behavior. (#168) * Revert to previous virtualenv behavior. A previous change caused the 'virtualenv' command change to switch its default Python version from 2.7 to 3.6 (when no -p flag is passed). Although not a bad change, it was accidental, and this reverts to the previous behavior. This also add tests for the behavior, and fixes various tests that had been subtly broken by a previous refactoring. The cause of this behavior change was that the 'virtualenv' package was installed under each Python interpreter, instead of the 'wheel' package. * Pin the virtualenv version. * Fix typos * Fix another typo --- cloudbuild_test.yaml | 1 + runtime-image/Dockerfile.in | 3 +- .../resources/requirements-virtualenv.txt | 1 + runtime-image/resources/requirements.txt | 2 +- tests/no-virtualenv/no-virtualenv.yaml | 34 ++++++++++++++ .../python2-libraries/python2-libraries.yaml | 2 +- tests/virtualenv/virtualenv_default.yaml | 25 +++++----- tests/virtualenv/virtualenv_python27.yaml | 47 +++++++++++++++++++ tests/virtualenv/virtualenv_python34.yaml | 34 ++++++++------ tests/virtualenv/virtualenv_python35.yaml | 34 +++++++------- tests/virtualenv/virtualenv_python36.yaml | 34 +++++++------- 11 files changed, 150 insertions(+), 67 deletions(-) create mode 100644 runtime-image/resources/requirements-virtualenv.txt create mode 100644 tests/virtualenv/virtualenv_python27.yaml diff --git a/cloudbuild_test.yaml b/cloudbuild_test.yaml index c6496dec..acf70bc7 100644 --- a/cloudbuild_test.yaml +++ b/cloudbuild_test.yaml @@ -26,6 +26,7 @@ steps: '-test.v', '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', '/workspace/tests/virtualenv/virtualenv_default.yaml', + '/workspace/tests/virtualenv/virtualenv_python27.yaml', '/workspace/tests/virtualenv/virtualenv_python34.yaml', '/workspace/tests/virtualenv/virtualenv_python35.yaml', '/workspace/tests/virtualenv/virtualenv_python36.yaml', diff --git a/runtime-image/Dockerfile.in b/runtime-image/Dockerfile.in index 60a6fa6f..7fafb2c4 100644 --- a/runtime-image/Dockerfile.in +++ b/runtime-image/Dockerfile.in @@ -27,7 +27,8 @@ ENV PATH /opt/python3.6/bin:/opt/python3.5/bin:$PATH RUN /usr/bin/pip install --upgrade -r /resources/requirements.txt && \ /usr/bin/pip3 install --upgrade -r /resources/requirements.txt && \ /opt/python3.5/bin/pip3.5 install --upgrade -r /resources/requirements.txt && \ - /opt/python3.6/bin/pip3.6 install --upgrade -r /resources/requirements.txt + /opt/python3.6/bin/pip3.6 install --upgrade -r /resources/requirements.txt && \ + /usr/bin/pip install --upgrade -r /resources/requirements-virtualenv.txt # Setup the app working directory RUN ln -s /home/vmagent/app /app diff --git a/runtime-image/resources/requirements-virtualenv.txt b/runtime-image/resources/requirements-virtualenv.txt new file mode 100644 index 00000000..a4ce32e5 --- /dev/null +++ b/runtime-image/resources/requirements-virtualenv.txt @@ -0,0 +1 @@ +virtualenv==15.1.0 diff --git a/runtime-image/resources/requirements.txt b/runtime-image/resources/requirements.txt index 40155447..c9f190ec 100644 --- a/runtime-image/resources/requirements.txt +++ b/runtime-image/resources/requirements.txt @@ -1,3 +1,3 @@ pip==9.0.1 setuptools==36.6.0 -virtualenv==15.1.0 +wheel==0.30.0 diff --git a/tests/no-virtualenv/no-virtualenv.yaml b/tests/no-virtualenv/no-virtualenv.yaml index de932617..f43d08a5 100644 --- a/tests/no-virtualenv/no-virtualenv.yaml +++ b/tests/no-virtualenv/no-virtualenv.yaml @@ -4,7 +4,41 @@ commandTests: command: ["which", "python"] expectedOutput: ["/usr/bin/python\n"] + - name: "default pip installation" + command: ["which", "pip"] + expectedOutput: ["/usr/local/bin/pip\n"] + + - name: "default pip python version" + command: ["pip", "-V"] + expectedOutput: ["pip .* from .*python 2[.]7"] + + - name: "default virtualenv installation" + command: ["which", "virtualenv"] + expectedOutput: ["/usr/local/bin/virtualenv\n"] + + - name: "default python2.7 installation" + command: ["which", "python2.7"] + expectedOutput: ["/usr/bin/python2.7\n"] + + - name: "default python3.4 installation" + command: ["which", "python3.4"] + expectedOutput: ["/usr/bin/python3.4\n"] + + - name: "default python3.5 installation" + command: ["which", "python3.5"] + expectedOutput: ["/opt/python3.5/bin/python3.5\n"] + + - name: "default python3.6 installation" + command: ["which", "python3.6"] + expectedOutput: ["/opt/python3.6/bin/python3.6\n"] + - name: "default gunicorn installation" setup: [["pip", "install", "gunicorn"]] command: ["which", "gunicorn"] expectedOutput: ["/usr/local/bin/gunicorn\n"] + + - name: "default flask installation" + # Checks that 'pip' and 'python' are using the same Python version + setup: [["pip", "install", "flask"]] + command: ["python", "-c", "import flask; print(flask.__file__)"] + expectedOutput: ["/usr/local/lib/python2.7/dist-packages/flask"] diff --git a/tests/python2-libraries/python2-libraries.yaml b/tests/python2-libraries/python2-libraries.yaml index 3a2771ac..f9fef5a0 100644 --- a/tests/python2-libraries/python2-libraries.yaml +++ b/tests/python2-libraries/python2-libraries.yaml @@ -8,6 +8,6 @@ globalEnvVars: commandTests: - name: "requirements" - setup: [["virtualenv", "/env"]] + setup: [["virtualenv", "-p", "python", "/env"]] command: ["pip", "install", "-r", "/requirements.txt"] exitCode: 0 diff --git a/tests/virtualenv/virtualenv_default.yaml b/tests/virtualenv/virtualenv_default.yaml index 8695e319..60714de6 100644 --- a/tests/virtualenv/virtualenv_default.yaml +++ b/tests/virtualenv/virtualenv_default.yaml @@ -7,20 +7,13 @@ globalEnvVars: value: "/env/bin:$PATH" commandTests: - - name: "python installation" - command: ["which", "python"] - expectedOutput: ["/usr/bin/python\n"] - - - name: "pip installation" - command: ["which", "pip"] - expectedOutput: ["/usr/local/bin/pip\n"] - - - name: "virtualenv installation" + - name: "virtualenv python installation" setup: [["virtualenv", "/env"]] command: ["which", "python"] expectedOutput: ["/env/bin/python\n"] - - name: "python version" + - name: "virtualenv python version" + setup: [["virtualenv", "/env"]] command: ["python", "--version"] # we check stderr instead of stdout for Python versions < 3.4 # https://bugs.python.org/issue18338 @@ -31,10 +24,14 @@ commandTests: command: ["which", "pip"] expectedOutput: ["/env/bin/pip\n"] - - name: "virtualenv gunicorn flask" - setup: [["virtualenv", "/env"], ["pip", "install", "gunicorn", "flask"]] + - name: "virtualenv gunicorn installation" + setup: [["virtualenv", "/env"], + ["pip", "install", "gunicorn"]] command: ["which", "gunicorn"] expectedOutput: ["/env/bin/gunicorn"] - - name: "flask integration" - command: ["python", "-c", "\"import sys; import flask; sys.exit(0 if flask.__file__.startswith('/env') else 1)\""] + - name: "virtualenv flask installation" + setup: [["virtualenv", "/env"], + ["pip", "install", "flask"]] + command: ["python", "-c", "import flask; print(flask.__file__)"] + expectedOutput: ["/env/local/lib/python2.7/site-packages/flask"] diff --git a/tests/virtualenv/virtualenv_python27.yaml b/tests/virtualenv/virtualenv_python27.yaml new file mode 100644 index 00000000..d28ea541 --- /dev/null +++ b/tests/virtualenv/virtualenv_python27.yaml @@ -0,0 +1,47 @@ +schemaVersion: "1.0.0" + +globalEnvVars: + - key: "VIRTUAL_ENV" + value: "/env" + - key: "PATH" + value: "/env/bin:$PATH" + +commandTests: + - name: "virtualenv27 python installation" + setup: [["virtualenv", "-p", "python", "/env"]] + command: ["which", "python"] + expectedOutput: ["/env/bin/python\n"] + + - name: "virtualenv27 python2 installation" + setup: [["virtualenv", "-p", "python", "/env"]] + command: ["which", "python2"] + expectedOutput: ["/env/bin/python2\n"] + + - name: "virtualenv27python2.7 installation" + setup: [["virtualenv", "-p", "python", "/env"]] + command: ["which", "python2.7"] + expectedOutput: ["/env/bin/python2.7\n"] + + - name: "virtualenv27 python version" + setup: [["virtualenv", "-p", "python", "/env"]] + command: ["python", "--version"] + # we check stderr instead of stdout for Python versions < 3.4 + # https://bugs.python.org/issue18338 + expectedError: ["Python 2.7.9\n"] + + - name: "virtualenv27 pip installation" + setup: [["virtualenv", "-p", "python", "/env"]] + command: ["which", "pip"] + expectedOutput: ["/env/bin/pip\n"] + + - name: "virtualenv27 gunicorn installation" + setup: [["virtualenv", "-p", "python", "/env"], + ["pip", "install", "gunicorn"]] + command: ["which", "gunicorn"] + expectedOutput: ["/env/bin/gunicorn"] + + - name: "virtualenv27 flask installation" + setup: [["virtualenv", "-p", "python", "/env"], + ["pip", "install", "flask"]] + command: ["python", "-c", "import flask; print(flask.__file__)"] + expectedOutput: ["/env/local/lib/python2.7/site-packages/flask"] diff --git a/tests/virtualenv/virtualenv_python34.yaml b/tests/virtualenv/virtualenv_python34.yaml index e35e9693..779f4d45 100644 --- a/tests/virtualenv/virtualenv_python34.yaml +++ b/tests/virtualenv/virtualenv_python34.yaml @@ -7,42 +7,48 @@ globalEnvVars: value: "/env/bin:$PATH" commandTests: - - name: "python installation" - command: ["which", "python3.4"] - expectedOutput: ["/usr/bin/python3.4\n"] - - - name: "virtualenv python installation" + - name: "virtualenv34 python installation" setup: [["virtualenv", "-p", "python3.4", "/env"]] command: ["which", "python"] expectedOutput: ["/env/bin/python\n"] - - name: "virtualenv python3 installation" + - name: "virtualenv34 python3 installation" setup: [["virtualenv", "-p", "python3.4", "/env"]] command: ["which", "python3"] expectedOutput: ["/env/bin/python3\n"] - - name: "python version" + - name: "virtualenv34 python3.4 installation" + setup: [["virtualenv", "-p", "python3.4", "/env"]] + command: ["which", "python3.4"] + expectedOutput: ["/env/bin/python3.4\n"] + + - name: "virtualenv34 python version" setup: [["virtualenv", "-p", "python3.4", "/env"]] command: ["python", "--version"] expectedOutput: ["Python 3.4.2\n"] - - name: "pip installation" + - name: "virtualenv34 pip installation" setup: [["virtualenv", "-p", "python3.4", "/env"]] command: ["which", "pip"] expectedOutput: ["/env/bin/pip\n"] - - name: "pip3 installation" + - name: "virtualenv34 pip3 installation" setup: [["virtualenv", "-p", "python3.4", "/env"]] command: ["which", "pip3"] expectedOutput: ["/env/bin/pip3\n"] - - name: "gunicorn flask" - setup: [["virtualenv", "-p", "python3.4", "/env"], ["pip", "install", "gunicorn", "flask"]] + - name: "virtualenv34 gunicorn installation" + setup: [["virtualenv", "-p", "python3.4", "/env"], + ["pip", "install", "gunicorn"]] command: ["which", "gunicorn"] expectedOutput: ["/env/bin/gunicorn"] - - name: "flask integration" - command: ["python", "-c", "\"import sys; import flask; sys.exit(0 if flask.__file__.startswith('/env') else 1)\""] + - name: "virtualenv34 flask installation" + setup: [["virtualenv", "-p", "python3.4", "/env"], + ["pip", "install", "flask"]] + command: ["python", "-c", "import flask; print(flask.__file__)"] + expectedOutput: ["/env/lib/python3.4/site-packages/flask"] - - name: "test.support" + - name: "virtualenv34 test.support availability" + setup: [["virtualenv", "-p", "python3.4", "/env"]] command: ["python", "-c", "\"from test import pystone, regrtest, support\""] diff --git a/tests/virtualenv/virtualenv_python35.yaml b/tests/virtualenv/virtualenv_python35.yaml index 4b019967..4150ba19 100644 --- a/tests/virtualenv/virtualenv_python35.yaml +++ b/tests/virtualenv/virtualenv_python35.yaml @@ -7,50 +7,48 @@ globalEnvVars: value: "/env/bin:$PATH" commandTests: - - name: "virtual env teardown" - command: ["rm", "-rf", "/env"] - - - name: "python installation" - command: ["which", "python3.5"] - expectedOutput: ["/opt/python3.5/bin/python3.5\n"] - - - name: "virtualenv python installation" + - name: "virtualenv35 python installation" setup: [["virtualenv", "-p", "python3.5", "/env"]] command: ["which", "python"] expectedOutput: ["/env/bin/python\n"] - - name: "virtualenv python3 installation" + - name: "virtualenv35 python3 installation" setup: [["virtualenv", "-p", "python3.5", "/env"]] command: ["which", "python3"] expectedOutput: ["/env/bin/python3\n"] - - name: "virtualenv python3.5 installation" + - name: "virtualenv35 python3.5 installation" setup: [["virtualenv", "-p", "python3.5", "/env"]] command: ["which", "python3.5"] expectedOutput: ["/env/bin/python3.5\n"] - - name: "python version" + - name: "virtualenv35 python version" setup: [["virtualenv", "-p", "python3.5", "/env"]] command: ["python", "--version"] expectedOutput: ["Python 3.5.4\n"] - - name: "pip installation" + - name: "virtualenv35 pip installation" setup: [["virtualenv", "-p", "python3.5", "/env"]] command: ["which", "pip"] expectedOutput: ["/env/bin/pip\n"] - - name: "pip3 installation" + - name: "virtualenv35 pip3 installation" setup: [["virtualenv", "-p", "python3.5", "/env"]] command: ["which", "pip3"] expectedOutput: ["/env/bin/pip3\n"] - - name: "gunicorn flask" - setup: [["virtualenv", "-p", "python3.5", "/env"], ["pip", "install", "gunicorn", "flask"]] + - name: "virtualenv35 gunicorn installation" + setup: [["virtualenv", "-p", "python3.5", "/env"], + ["pip", "install", "gunicorn"]] command: ["which", "gunicorn"] expectedOutput: ["/env/bin/gunicorn"] - - name: "flask integration" - command: ["python", "-c", "\"import sys; import flask; sys.exit(0 if flask.__file__.startswith('/env') else 1)\""] + - name: "virtualenv35 flask installation" + setup: [["virtualenv", "-p", "python3.5", "/env"], + ["pip", "install", "flask"]] + command: ["python", "-c", "import flask; print(flask.__file__)"] + expectedOutput: ["/env/lib/python3.5/site-packages/flask"] - - name: "test.support" + - name: "virtualenv35 test.support availability" + setup: [["virtualenv", "-p", "python3.5", "/env"]] command: ["python", "-c", "\"from test import pystone, regrtest, support\""] diff --git a/tests/virtualenv/virtualenv_python36.yaml b/tests/virtualenv/virtualenv_python36.yaml index deea9b5d..71c91a9f 100644 --- a/tests/virtualenv/virtualenv_python36.yaml +++ b/tests/virtualenv/virtualenv_python36.yaml @@ -7,50 +7,48 @@ globalEnvVars: value: "/env/bin:$PATH" commandTests: - - name: "virtual env teardown" - command: ["rm", "-rf", "/env"] - - - name: "python installation" - command: ["which", "python3.6"] - expectedOutput: ["/opt/python3.6/bin/python3.6\n"] - - - name: "virtualenv python installation" + - name: "virtualenv36 python installation" setup: [["virtualenv", "-p", "python3.6", "/env"]] command: ["which", "python"] expectedOutput: ["/env/bin/python\n"] - - name: "virtualenv python3 installation" + - name: "virtualenv36 python3 installation" setup: [["virtualenv", "-p", "python3.6", "/env"]] command: ["which", "python3"] expectedOutput: ["/env/bin/python3\n"] - - name: "virtualenv python3.6 installation" + - name: "virtualenv36 python3.6 installation" setup: [["virtualenv", "-p", "python3.6", "/env"]] command: ["which", "python3.6"] expectedOutput: ["/env/bin/python3.6\n"] - - name: "python version" + - name: "virtualenv36 python version" setup: [["virtualenv", "-p", "python3.6", "/env"]] command: ["python", "--version"] expectedOutput: ["Python 3.6.2\n"] - - name: "pip installation" + - name: "virtualenv36 pip installation" setup: [["virtualenv", "-p", "python3.6", "/env"]] command: ["which", "pip"] expectedOutput: ["/env/bin/pip\n"] - - name: "pip3 installation" + - name: "virtualenv36 pip3 installation" setup: [["virtualenv", "-p", "python3.6", "/env"]] command: ["which", "pip3"] expectedOutput: ["/env/bin/pip3\n"] - - name: "gunicorn flask" - setup: [["virtualenv", "-p", "python3.6", "/env"], ["pip", "install", "gunicorn", "flask"]] + - name: "virtualenv36 gunicorn installation" + setup: [["virtualenv", "-p", "python3.6", "/env"], + ["pip", "install", "gunicorn"]] command: ["which", "gunicorn"] expectedOutput: ["/env/bin/gunicorn"] - - name: "flask integration" - command: ["python", "-c", "\"import sys; import flask; sys.exit(0 if flask.__file__.startswith('/env') else 1)\""] + - name: "virtualenv36 flask installation" + setup: [["virtualenv", "-p", "python3.6", "/env"], + ["pip", "install", "flask"]] + command: ["python", "-c", "import flask; print(flask.__file__)"] + expectedOutput: ["/env/lib/python3.6/site-packages/flask"] - - name: "test.support" + - name: "virtualenv36 test.support availability" + setup: [["virtualenv", "-p", "python3.6", "/env"]] command: ["python", "-c", "\"from test import pystone, regrtest, support\""] From 714dcb7f83aba9e1c80b9a739294a4864109a607 Mon Sep 17 00:00:00 2001 From: Nick Kubala Date: Wed, 15 Nov 2017 14:18:04 -0800 Subject: [PATCH 148/256] return string instead of bool in integration test sample app env handler --- tests/integration/server.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/integration/server.py b/tests/integration/server.py index 8e2958cd..0b4382c6 100755 --- a/tests/integration/server.py +++ b/tests/integration/server.py @@ -245,8 +245,11 @@ def _check_environment(): # for GAE, we'll check the existence env vars set on # vm:true or env:flex # if neither exist, assume we're in GKE - return (_APPENGINE_FLEXIBLE_ENV_VM in os.environ or - _APPENGINE_FLEXIBLE_ENV_FLEX in os.environ), 200 + environment = "GKE" + if (_APPENGINE_FLEXIBLE_ENV_VM in os.environ or + _APPENGINE_FLEXIBLE_ENV_FLEX in os.environ): + environment = "GAE" + return environment, 200 class ErrorResponse(Exception): From 7b8bf15059075f06ce2c5c6e61d1445ba3c1094e Mon Sep 17 00:00:00 2001 From: Angela Li Date: Mon, 4 Dec 2017 13:34:58 -0800 Subject: [PATCH 149/256] Update system tests script (#171) --- tests/google-cloud-python-system/run_system_tests.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/google-cloud-python-system/run_system_tests.sh b/tests/google-cloud-python-system/run_system_tests.sh index 17d3aac3..f9f81a47 100755 --- a/tests/google-cloud-python-system/run_system_tests.sh +++ b/tests/google-cloud-python-system/run_system_tests.sh @@ -31,8 +31,8 @@ for package in ${packages}; do nox \ -f "${noxfile}" \ -e \ - "system_tests(python_version='2.7')" \ - "system_tests(python_version='3.6')" \ + "system(py='2.7')" \ + "system(py='3.6')" \ || exit_code=1 done From 597a87474472699e2611f0d170becd69c2b244b2 Mon Sep 17 00:00:00 2001 From: Angela Li Date: Wed, 3 Jan 2018 18:07:37 -0800 Subject: [PATCH 150/256] Skip test_xmlrpc_net when building python interpreters (#175) --- python-interpreter-builder/scripts/build-python-3.5.sh | 3 ++- python-interpreter-builder/scripts/build-python-3.6.sh | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/python-interpreter-builder/scripts/build-python-3.5.sh b/python-interpreter-builder/scripts/build-python-3.5.sh index f9113879..eeea1f76 100755 --- a/python-interpreter-builder/scripts/build-python-3.5.sh +++ b/python-interpreter-builder/scripts/build-python-3.5.sh @@ -123,7 +123,8 @@ make profile-opt # test_dbm: https://bugs.python.org/issue28700 # test_imap: https://bugs.python.org/issue30175 # test_shutil: https://bugs.python.org/issue29317 -make test TESTOPTS="--exclude test___all__ test_dbm test_imaplib test_shutil" +# test_xmlrpc_net: https://bugs.python.org/issue31724 +make test TESTOPTS="--exclude test___all__ test_dbm test_imaplib test_shutil test_xmlrpc_net" # Install make altinstall diff --git a/python-interpreter-builder/scripts/build-python-3.6.sh b/python-interpreter-builder/scripts/build-python-3.6.sh index 431c305d..ca089d59 100755 --- a/python-interpreter-builder/scripts/build-python-3.6.sh +++ b/python-interpreter-builder/scripts/build-python-3.6.sh @@ -123,7 +123,8 @@ make profile-opt # test_dbm: https://bugs.python.org/issue28700 # test_imap: https://bugs.python.org/issue30175 # test_shutil: https://bugs.python.org/issue29317 -make test TESTOPTS="--exclude test___all__ test_dbm test_imaplib test_shutil" +# test_xmlrpc_net: https://bugs.python.org/issue31724 +make test TESTOPTS="--exclude test___all__ test_dbm test_imaplib test_shutil test_xmlrpc_net" # Install make altinstall From 01a4da0bc4a429b24550912f2f2ba1e732b4ac36 Mon Sep 17 00:00:00 2001 From: Jon Donovan Date: Fri, 12 Jan 2018 12:43:31 -0800 Subject: [PATCH 151/256] Python compat (#174) * Add support for python-compat * Update app.yaml and tests * Add missing .dockerignore file * Updating per review comments * Adding missing files * Update gen_dockerfile.py * Update gen_dockerfile_test.py --- scripts/data/Dockerfile.python_compat | 3 ++ scripts/data/dockerignore.python_compat | 5 +++ scripts/gen_dockerfile.py | 43 +++++++++++++------ scripts/gen_dockerfile_test.py | 32 +++++++++++--- scripts/testdata/hello_world_compat/app.yaml | 13 ++++++ scripts/testdata/hello_world_compat/main.py | 14 ++++++ .../hello_world_compat_golden/.dockerignore | 5 +++ .../hello_world_compat_golden/Dockerfile | 3 ++ 8 files changed, 97 insertions(+), 21 deletions(-) create mode 100644 scripts/data/Dockerfile.python_compat create mode 100644 scripts/data/dockerignore.python_compat create mode 100644 scripts/testdata/hello_world_compat/app.yaml create mode 100644 scripts/testdata/hello_world_compat/main.py create mode 100644 scripts/testdata/hello_world_compat_golden/.dockerignore create mode 100644 scripts/testdata/hello_world_compat_golden/Dockerfile diff --git a/scripts/data/Dockerfile.python_compat b/scripts/data/Dockerfile.python_compat new file mode 100644 index 00000000..1e4d6352 --- /dev/null +++ b/scripts/data/Dockerfile.python_compat @@ -0,0 +1,3 @@ +FROM gcr.io/google_appengine/python-compat-multicore +ADD . /app/ +RUN if [ -s requirements.txt ]; then pip install -r requirements.txt; fi diff --git a/scripts/data/dockerignore.python_compat b/scripts/data/dockerignore.python_compat new file mode 100644 index 00000000..5ce5abfa --- /dev/null +++ b/scripts/data/dockerignore.python_compat @@ -0,0 +1,5 @@ +.dockerignore +Dockerfile +.git +.hg +.svn diff --git a/scripts/gen_dockerfile.py b/scripts/gen_dockerfile.py index eaff183f..5a4ceae2 100755 --- a/scripts/gen_dockerfile.py +++ b/scripts/gen_dockerfile.py @@ -29,7 +29,6 @@ import validation_utils - # Validate characters for dockerfile image names. # # This roots out obvious mistakes, the full gory details are here: @@ -63,7 +62,7 @@ # Validated application configuration AppConfig = collections.namedtuple( 'AppConfig', - 'base_image dockerfile_python_version entrypoint has_requirements_txt' + 'base_image dockerfile_python_version entrypoint has_requirements_txt is_python_compat' ) @@ -97,6 +96,16 @@ def get_app_config(raw_config, base_image, config_file, source_dir): 'Expected {} contents to be a Mapping type, but found type "{}"'. format(config_file, type(raw_config))) + # Short circuit for python compat. + if validation_utils.get_field_value( + raw_config, 'runtime', str) == 'python-compat': + return AppConfig( + base_image=None, + dockerfile_python_version=None, + entrypoint=None, + has_requirements_txt=None, + is_python_compat=True) + entrypoint = validation_utils.get_field_value( raw_config, 'entrypoint', str) if not PRINTABLE_REGEX.match(entrypoint): @@ -133,7 +142,8 @@ def get_app_config(raw_config, base_image, config_file, source_dir): base_image=base_image, dockerfile_python_version=dockerfile_python_version, entrypoint=entrypoint, - has_requirements_txt=has_requirements_txt) + has_requirements_txt=has_requirements_txt, + is_python_compat=False) def get_data(name): @@ -175,19 +185,24 @@ def generate_files(app_config): else: optional_entrypoint = '' - dockerfile = ''.join([ - get_data('Dockerfile.preamble.template').format( - base_image=app_config.base_image), - get_data('Dockerfile.virtualenv.template').format( - python_version=app_config.dockerfile_python_version), - optional_requirements_txt, - get_data('Dockerfile.install_app'), - optional_entrypoint, - ]) + if app_config.is_python_compat: + dockerfile = get_data('Dockerfile.python_compat') + dockerignore = get_data('dockerignore.python_compat') + else: + dockerfile = ''.join([ + get_data('Dockerfile.preamble.template').format( + base_image=app_config.base_image), + get_data('Dockerfile.virtualenv.template').format( + python_version=app_config.dockerfile_python_version), + optional_requirements_txt, + get_data('Dockerfile.install_app'), + optional_entrypoint, + ]) + dockerignore = get_data('dockerignore') return { 'Dockerfile': dockerfile, - '.dockerignore': get_data('dockerignore'), + '.dockerignore': dockerignore, } @@ -206,7 +221,7 @@ def generate_dockerfile_command(base_image, config_file, source_dir): # Determine complete configuration app_config = get_app_config(raw_config, base_image, config_file, - source_dir) + source_dir) # Generate list of filenames and their textual contents files = generate_files(app_config) diff --git a/scripts/gen_dockerfile_test.py b/scripts/gen_dockerfile_test.py index b52e15e8..8d1d0798 100755 --- a/scripts/gen_dockerfile_test.py +++ b/scripts/gen_dockerfile_test.py @@ -54,6 +54,14 @@ def compare_file(filename, dir1, dir2): 'dockerfile_python_version': '', 'has_requirements_txt': False, 'entrypoint': '', + 'is_python_compat': False, + }), + ('env: flex\nruntime: python-compat', { + 'base_image': None, + 'dockerfile_python_version': None, + 'has_requirements_txt': None, + 'entrypoint': None, + 'is_python_compat': True, }), # All supported python versions ('runtime_config:\n python_version:', { @@ -125,7 +133,8 @@ def test_get_app_config_invalid(app_yaml): base_image='', dockerfile_python_version='', entrypoint='', - has_requirements_txt=False + has_requirements_txt=False, + is_python_compat=False, ) @@ -146,6 +155,9 @@ def test_get_app_config_invalid(app_yaml): # Python version (_BASE_APP_CONFIG._replace(dockerfile_python_version='_my_version'), True, 'python_version=python_my_version'), + # python-compat runtime + (_BASE_APP_CONFIG._replace(is_python_compat=True), True, + 'FROM gcr.io/google_appengine/python-compat-multicore'), ]) def test_generate_files(app_config, should_find, test_string): result = gen_dockerfile.generate_files(app_config) @@ -163,10 +175,13 @@ def compare_against_golden_files(app, config_dir, testdata_dir): compare_file(filename, config_dir, golden_dir) -def test_generate_dockerfile_command(tmpdir, testdata_dir): +@pytest.mark.parametrize('app', [ + # Sampled from https://github.com/GoogleCloudPlatform/python-docs-samples + 'hello_world', + # From an internal source. + 'hello_world_compat']) +def test_generate_dockerfile_command(tmpdir, testdata_dir, app): """Generates output and compares against a set of golden files.""" - # Sample from https://github.com/GoogleCloudPlatform/python-docs-samples - app = 'hello_world' app_dir = os.path.join(testdata_dir, app) # Copy sample app to writable temp dir, and generate Dockerfile. @@ -179,12 +194,15 @@ def test_generate_dockerfile_command(tmpdir, testdata_dir): compare_against_golden_files(app, config_dir, testdata_dir) +@pytest.mark.parametrize('app', [ + # Sampled from https://github.com/GoogleCloudPlatform/python-docs-samples + 'hello_world', + # From an internal source. + 'hello_world_compat']) @pytest.mark.xfail(not shutil.which('gcloud'), reason='Google Cloud SDK is not installed') -def test_generate_dockerfile_golden(tmpdir, testdata_dir): +def test_generate_dockerfile_golden(tmpdir, testdata_dir, app): """Validate our golden files against gcloud app gen-config""" - # Sample from https://github.com/GoogleCloudPlatform/python-docs-samples - app = 'hello_world' app_dir = os.path.join(testdata_dir, app) # Copy sample app to writable temp dir, and generate Dockerfile. diff --git a/scripts/testdata/hello_world_compat/app.yaml b/scripts/testdata/hello_world_compat/app.yaml new file mode 100644 index 00000000..e514d42c --- /dev/null +++ b/scripts/testdata/hello_world_compat/app.yaml @@ -0,0 +1,13 @@ +service: default +runtime: python-compat +env: flex + +api_version: 1 +threadsafe: true + +beta_settings: + enable_app_engine_apis: true # Needed for compat apps. + +handlers: +- url: .* + script: main.app diff --git a/scripts/testdata/hello_world_compat/main.py b/scripts/testdata/hello_world_compat/main.py new file mode 100644 index 00000000..40302722 --- /dev/null +++ b/scripts/testdata/hello_world_compat/main.py @@ -0,0 +1,14 @@ +"""The hello world flex app!""" + +import webapp2 + + +class HelloHandler(webapp2.RequestHandler): + + def get(self): + msg = 'Hello GAE Flex (env: flex) Compat-Runtime App\n' + self.response.headers['Content-Type'] = 'text/plain' + self.response.out.write(msg) + +app = webapp2.WSGIApplication([('/', HelloHandler)], + debug=True) diff --git a/scripts/testdata/hello_world_compat_golden/.dockerignore b/scripts/testdata/hello_world_compat_golden/.dockerignore new file mode 100644 index 00000000..5ce5abfa --- /dev/null +++ b/scripts/testdata/hello_world_compat_golden/.dockerignore @@ -0,0 +1,5 @@ +.dockerignore +Dockerfile +.git +.hg +.svn diff --git a/scripts/testdata/hello_world_compat_golden/Dockerfile b/scripts/testdata/hello_world_compat_golden/Dockerfile new file mode 100644 index 00000000..1e4d6352 --- /dev/null +++ b/scripts/testdata/hello_world_compat_golden/Dockerfile @@ -0,0 +1,3 @@ +FROM gcr.io/google_appengine/python-compat-multicore +ADD . /app/ +RUN if [ -s requirements.txt ]; then pip install -r requirements.txt; fi From dfd43223fd8b40b8e90fd3b6cd8e088ca8f04e86 Mon Sep 17 00:00:00 2001 From: Julius Adorf Date: Wed, 24 Jan 2018 17:14:01 +0000 Subject: [PATCH 152/256] Replace "Container Engine" by "Kubernetes Engine". (#176) --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 78a02c5c..2cc7d173 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ This repository contains the source for the [docker](https://docker.io) base image. This image can be used as the base image for running applications on [Google App Engine Flexible](https://cloud.google.com/appengine/docs/flexible/), -[Google Container Engine](https://cloud.google.com/container-engine), or any +[Google Kubernetes Engine](https://cloud.google.com/kubernetes-engine), or any other Docker host. This image is based on Debian Jessie and contains packages required to build @@ -38,7 +38,7 @@ to create a custom runtime: You can then modify the `Dockerfile` and `.dockerignore` as needed for you application. -## Container Engine & other Docker hosts. +## Kubernetes Engine & other Docker hosts. For other docker hosts, you'll need to create a `Dockerfile` based on this image that copies your application code, installs dependencies, and declares an From 389436f9d0d62bf1144ca7cb96f55388c5603f70 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Wed, 7 Feb 2018 17:59:17 -0800 Subject: [PATCH 153/256] Auto-update dependencies. (#167) * Auto-update dependencies. * Pin some library versions to deal with Python 2 and 3 incompatibilties. --- runtime-image/resources/requirements.txt | 2 +- scripts/requirements-test.txt | 2 +- tests/eventlet/requirements.txt | 8 +- tests/integration/requirements.txt | 2 +- tests/python2-libraries/requirements.txt | 158 +++++++++++------------ tests/python3-libraries/requirements.txt | 147 +++++++++++---------- 6 files changed, 159 insertions(+), 160 deletions(-) diff --git a/runtime-image/resources/requirements.txt b/runtime-image/resources/requirements.txt index c9f190ec..6a1b2cc3 100644 --- a/runtime-image/resources/requirements.txt +++ b/runtime-image/resources/requirements.txt @@ -1,3 +1,3 @@ pip==9.0.1 -setuptools==36.6.0 +setuptools==38.5.0 wheel==0.30.0 diff --git a/scripts/requirements-test.txt b/scripts/requirements-test.txt index 5afb5315..12a6ca4a 100644 --- a/scripts/requirements-test.txt +++ b/scripts/requirements-test.txt @@ -1,4 +1,4 @@ flask==0.12.2 -pytest==3.2.3 +pytest==3.4.0 pytest-cov==2.5.1 pyyaml==3.12 diff --git a/tests/eventlet/requirements.txt b/tests/eventlet/requirements.txt index c33e53f2..a8f7b0a9 100644 --- a/tests/eventlet/requirements.txt +++ b/tests/eventlet/requirements.txt @@ -1,10 +1,10 @@ click==6.7 enum-compat==0.0.2 -eventlet==0.21.0 +eventlet==0.22.0 Flask==0.12.2 -greenlet==0.4.12 +greenlet==0.4.13 gunicorn==19.7.1 itsdangerous==0.24 -Jinja2==2.9.6 +Jinja2==2.10 MarkupSafe==1.0 -Werkzeug==0.12.2 +Werkzeug==0.14.1 diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index 3e51c81d..99a06d3b 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -1,5 +1,5 @@ Flask==0.12.2 -google-cloud-error-reporting==0.28.0 +google-cloud-error-reporting==0.29.0 google-cloud-logging==1.4.0 google-cloud-monitoring==0.28.0 gunicorn==19.7.1 diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 8c134e48..f512ca18 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -1,54 +1,54 @@ -alembic==0.9.6 +alembic==0.9.7 amqp==2.2.2 amqplib==1.0.2 -ansible==2.4.1.0 +ansible==2.4.3.0 anyjson==0.3.3 apache-libcloud==2.2.1 argparse==1.4.0 -astroid==1.5.3 -awscli==1.11.180 -babel==2.5.1 +astroid==1.6.1 +awscli==1.14.32 +babel==2.5.3 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 beautifulsoup4==4.6.0 beautifulsoup==3.2.1 billiard==3.5.0.3 -blessings==1.6 +blessings==1.6.1 blinker==1.4 boto==2.48.0 -botocore==1.7.38 +botocore==1.8.36 bottle==0.12.13 -carbon==1.0.2 +carbon<1.1.1 celery==4.1.0 -certifi==2017.7.27.1 -cffi==1.11.2 +certifi==2018.1.18 +cffi==1.11.4 chardet==3.0.4 click==6.7 -cliff==2.9.1 -cmd2==0.7.7 +cliff==2.11.0 +cmd2==0.8.0 colorama==0.3.9 configobj==5.0.6 cov-core==1.15.0 -coverage==4.4.1 +coverage==4.5 coveralls==1.2.0 crcmod==1.7 -cryptography==2.1.3 -cssselect==1.0.1 -cython==0.27.2 -decorator==4.1.2 -django-celery==3.2.1 -django-debug-toolbar==1.8 -django-extensions==1.9.7 -django==1.11.7 +cryptography==2.1.4 +cssselect==1.0.3 +cython==0.27.3 +decorator==4.2.1 +django-celery==3.2.2 +django-debug-toolbar==1.9.1 +django-extensions==1.9.9 +django<2.0 django_compress==1.0.1 -djangorestframework==3.7.1 +djangorestframework==3.7.7 docker-py==1.10.6 docopt==0.6.2 docutils==0.14 ecdsa==0.13 -elasticsearch==5.4.0 +elasticsearch==6.1.1 enum34==1.1.6 -eventlet==0.21.0 +eventlet==0.22.0 extras==1.0.0 fabric==1.14.0 fixtures==3.0.0 @@ -56,108 +56,108 @@ flake8==3.5.0 flask==0.12.2 funcsigs==1.0.2 functools32==3.2.3.post2 -futures==3.1.1 +futures==3.2.0 gevent==1.2.2 -google-api-python-client==1.6.4 -graphite-web==1.0.2 -greenlet==0.4.12 +google-api-python-client==1.6.5 +graphite-web==1.1.1 +greenlet==0.4.13 gunicorn==19.7.1 hiredis==0.2.0 honcho==1.0.1 -html5lib==0.999999999 +html5lib==1.0.1 httplib2==0.10.3 idna==2.6 -ipaddress==1.0.18 +ipaddress==1.0.19 iso8601==0.1.12 isodate==0.6.0 itsdangerous==0.24 -jinja2==2.9.6 +jinja2==2.10 jmespath==0.9.3 jsonschema==2.6.0 kombu==4.1.0 linecache2==1.0.0 logilab-common==1.4.1 -lxml==4.1.0 +lxml==4.1.1 m2crypto==0.27.0 mako==1.0.7 manifestparser==1.1 -markdown==2.6.9 +markdown==2.6.11 markupsafe==1.0 -matplotlib==2.1.0 +matplotlib==2.1.2 mccabe==0.6.1 meld3==1.0.2 mock==2.0.0 mozcrash==1.0 -mozdevice==0.50 +mozdevice==0.51 mozfile==1.2 mozinfo==0.10 -mozlog==3.5 +mozlog==3.7 moznetwork==0.27 -mozprocess==0.25 -mozprofile==0.28 -mozrunner==6.13 -msgpack-python==0.4.8 +mozprocess==0.26 +mozprofile==0.29 +mozrunner==6.14 +msgpack-python==0.5.2 mysql-python==1.2.5 -ndg-httpsclient==0.4.3 +ndg-httpsclient==0.4.4 netaddr==0.7.19 netifaces==0.10.6 -newrelic==2.96.0.80 +newrelic==2.100.0.84 nose==1.3.7 -numpy==1.13.3 +numpy==1.14.0 oauth2==1.9.0.post1 oauth2client==4.1.2 oauthlib==2.0.6 ordereddict==1.1 -oslo.config==5.0.0 -pandas==0.21.0 -paramiko==2.3.1 +oslo.config==5.2.0 +pandas==0.22.0 +paramiko==2.4.0 passlib==1.7.1 paste==2.0.3 pastedeploy==1.5.2 pastescript==2.0.2 pbr==3.1.1 pep8==1.7.1 -pexpect==4.2.1 -pika==0.11.0 -pillow==4.3.0 +pexpect==4.3.1 +pika==0.11.2 +pillow==5.0.0 pip==9.0.1 prettytable -protobuf==3.4.0 -psutil==5.4.0 +protobuf==3.5.1 +psutil==5.4.3 psycopg2==2.7.3.2 -py==1.4.34 -pyasn1-modules==0.1.5 -pyasn1==0.3.7 +py==1.5.2 +pyasn1-modules==0.2.1 +pyasn1==0.4.2 pycparser==2.18 pycrypto==2.6.1 -pycurl==7.43.0 +pycurl==7.43.0.1 pyflakes==1.6.0 pygments==2.2.0 pyjwt==1.5.3 pylibmc==1.5.2 -pylint==1.7.4 -pymongo==3.5.1 -pymysql==0.7.11 -pyopenssl==17.3.0 +pylint==1.8.2 +pymongo==3.6.0 +pymysql==0.8.0 +pyopenssl==17.5.0 pyparsing==2.2.0 pyramid==1.9.1 pystache==0.5.4 pytest-cov==2.5.1 -pytest==3.2.3 +pytest==3.4.0 python-cjson==1.2.1 python-daemon==2.1.2 python-dateutil==2.6.1 python-gflags==3.1.2 -python-keystoneclient==3.13.0 -python-memcached==1.58 +python-keystoneclient==3.15.0 +python-memcached==1.59 python-mimeparse==1.6.0 -python-novaclient==9.1.0 +python-novaclient==10.1.0 python-subunit==1.2.0 -python-swiftclient==3.4.0 +python-swiftclient==3.5.0 pytz==2017.3 pyyaml==3.12 pyzmq==16.0.3 -raven==6.3.0 +raven==6.5.0 redis==2.10.6 repoze.lru==0.7 requests-oauthlib==0.8.0 @@ -165,30 +165,30 @@ requests==2.18.4 retrying==1.3.3 rsa==3.4.2 scipy==1.0.0 -selenium==3.7.0 +selenium==3.8.1 setuptools-git==1.2 -setuptools==36.6.0 +setuptools==38.5.0 sh==1.12.14 -simplejson==3.11.1 +simplejson==3.13.2 six==1.11.0 snowballstemmer==1.2.1 south==1.0.2 -sphinx==1.6.5 +sphinx==1.6.7 sqlalchemy-migrate==0.11.0 -sqlalchemy==1.1.14 +sqlalchemy==1.2.2 sqlparse==0.2.4 -statsd==3.2.1 -stevedore==1.27.1 +statsd==3.2.2 +stevedore==1.28.0 suds==0.4 supervisor==3.3.3 testrepository==0.0.20 testtools==2.3.0 -thrift==0.10.0 -tornado==4.5.2 +thrift==0.11.0 +tornado==4.5.3 tox==2.9.1 twisted==17.9.0 ujson==1.35 -unidecode==0.4.21 +unidecode==1.0.22 unittest2==1.1.0 uritemplate==3.0.0 urllib3==1.22 @@ -197,11 +197,11 @@ versiontools==1.9.1 virtualenv==15.1.0 waitress==1.1.0 warlock==1.3.0 -webob==1.7.3 -websocket-client==0.44.0 +webob==1.7.4 +websocket-client==0.46.0 webtest==2.0.29 -werkzeug==0.12.2 +werkzeug==0.14.1 wheel==0.30.0 xlrd==1.1.0 -zc.buildout==2.9.5 +zc.buildout==2.11.0 zope.interface==4.4.3 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 2bdcc565..a8664b03 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -1,154 +1,153 @@ -alembic==0.9.6 +alembic==0.9.7 amqp==2.2.2 amqplib==1.0.2 -ansible==2.4.1.0 +ansible==2.4.3.0 anyjson==0.3.3 apache-libcloud==2.2.1 argparse==1.4.0 -astroid==1.5.3 -awscli==1.11.180 -babel==2.5.1 +astroid==1.6.1 +awscli==1.14.32 +babel==2.5.3 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 beautifulsoup4==4.6.0 billiard==3.5.0.3 -blessings==1.6 +blessings==1.6.1 blinker==1.4 boto==2.48.0 -botocore==1.7.38 +botocore==1.8.36 bottle==0.12.13 celery==4.1.0 -certifi==2017.7.27.1 -cffi==1.11.2 +certifi==2018.1.18 +cffi==1.11.4 chardet==3.0.4 click==6.7 -cliff==2.9.1 -cmd2==0.7.7 +cliff==2.11.0 +cmd2==0.8.0 colorama==0.3.9 configobj==5.0.6 cov-core==1.15.0 -coverage==4.4.1 +coverage==4.5 coveralls==1.2.0 crcmod==1.7 -cryptography==2.1.3 -cssselect==1.0.1 -cython==0.27.2 -decorator==4.1.2 -django-celery==3.2.1 -django-debug-toolbar==1.8 -django-extensions==1.9.7 -django==1.11.7 +cryptography==2.1.4 +cssselect==1.0.3 +cython==0.27.3 +decorator==4.2.1 +django-celery==3.2.2 +django-debug-toolbar==1.9.1 +django-extensions==1.9.9 +django==2.0.2 django_compress==1.0.1 -djangorestframework==3.7.1 +djangorestframework==3.7.7 docker-py==1.10.6 docopt==0.6.2 docutils==0.14 ecdsa==0.13 -elasticsearch==5.4.0 +elasticsearch==6.1.1 enum34==1.1.6 -eventlet==0.21.0 +eventlet==0.22.0 extras==1.0.0 fabric==1.14.0 fixtures==3.0.0 flake8==3.5.0 flask==0.12.2 funcsigs==1.0.2 -futures==3.1.1 gevent==1.2.2 -google-api-python-client==1.6.4 -greenlet==0.4.12 +google-api-python-client==1.6.5 +greenlet==0.4.13 gunicorn==19.7.1 hiredis==0.2.0 honcho==1.0.1 -html5lib==0.999999999 +html5lib==1.0.1 httplib2==0.10.3 idna==2.6 -ipaddress==1.0.18 +ipaddress==1.0.19 ipython==6.2.1 iso8601==0.1.12 isodate==0.6.0 itsdangerous==0.24 -jinja2==2.9.6 +jinja2==2.10 jmespath==0.9.3 jsonschema==2.6.0 kombu==4.1.0 linecache2==1.0.0 logilab-common==1.4.1 -lxml==4.1.0 +lxml==4.1.1 mako==1.0.7 manifestparser==1.1 -markdown==2.6.9 +markdown==2.6.11 markupsafe==1.0 -matplotlib==2.1.0 +matplotlib==2.1.2 mccabe==0.6.1 meld3==1.0.2 mock==2.0.0 mozcrash==1.0 -mozdevice==0.50 +mozdevice==0.51 mozfile==1.2 mozinfo==0.10 -mozlog==3.5 +mozlog==3.7 moznetwork==0.27 -mozprocess==0.25 -msgpack-python==0.4.8 -ndg-httpsclient==0.4.3 +mozprocess==0.26 +msgpack-python==0.5.2 +ndg-httpsclient==0.4.4 netaddr==0.7.19 netifaces==0.10.6 -newrelic==2.96.0.80 +newrelic==2.100.0.84 nose==1.3.7 -numpy==1.13.3 +numpy==1.14.0 oauth2==1.9.0.post1 oauth2client==4.1.2 oauthlib==2.0.6 ordereddict==1.1 -oslo.config==5.0.0 -pandas==0.21.0 -paramiko==2.3.1 +oslo.config==5.2.0 +pandas==0.22.0 +paramiko==2.4.0 passlib==1.7.1 paste==2.0.3 pastedeploy==1.5.2 pastescript==2.0.2 pbr==3.1.1 pep8==1.7.1 -pexpect==4.2.1 -pika==0.11.0 -pillow==4.3.0 +pexpect==4.3.1 +pika==0.11.2 +pillow==5.0.0 pip==9.0.1 prettytable -protobuf==3.4.0 -psutil==5.4.0 +protobuf==3.5.1 +psutil==5.4.3 psycopg2==2.7.3.2 -py==1.4.34 -pyasn1-modules==0.1.5 -pyasn1==0.3.7 +py==1.5.2 +pyasn1-modules==0.2.1 +pyasn1==0.4.2 pycparser==2.18 pycrypto==2.6.1 pyflakes==1.6.0 pygments==2.2.0 pyjwt==1.5.3 pylibmc==1.5.2 -pylint==1.7.4 -pymongo==3.5.1 -pymysql==0.7.11 -pyopenssl==17.3.0 +pylint==1.8.2 +pymongo==3.6.0 +pymysql==0.8.0 +pyopenssl==17.5.0 pyparsing==2.2.0 pyramid==1.9.1 pystache==0.5.4 pytest-cov==2.5.1 -pytest==3.2.3 +pytest==3.4.0 python-daemon==2.1.2 python-dateutil==2.6.1 python-gflags==3.1.2 -python-keystoneclient==3.13.0 -python-memcached==1.58 +python-keystoneclient==3.15.0 +python-memcached==1.59 python-mimeparse==1.6.0 -python-novaclient==9.1.0 +python-novaclient==10.1.0 python-subunit==1.2.0 -python-swiftclient==3.4.0 +python-swiftclient==3.5.0 pytz==2017.3 pyyaml==3.12 pyzmq==16.0.3 -raven==6.3.0 +raven==6.5.0 redis==2.10.6 repoze.lru==0.7 requests-oauthlib==0.8.0 @@ -156,28 +155,28 @@ requests==2.18.4 retrying==1.3.3 rsa==3.4.2 scipy==1.0.0 -selenium==3.7.0 +selenium==3.8.1 setuptools-git==1.2 -setuptools==36.6.0 +setuptools==38.5.0 sh==1.12.14 -simplejson==3.11.1 +simplejson==3.13.2 six==1.11.0 snowballstemmer==1.2.1 south==1.0.2 -sphinx==1.6.5 +sphinx==1.6.7 sqlalchemy-migrate==0.11.0 -sqlalchemy==1.1.14 +sqlalchemy==1.2.2 sqlparse==0.2.4 -statsd==3.2.1 -stevedore==1.27.1 +statsd==3.2.2 +stevedore==1.28.0 testrepository==0.0.20 testtools==2.3.0 -thrift==0.10.0 -tornado==4.5.2 +thrift==0.11.0 +tornado==4.5.3 tox==2.9.1 twisted==17.9.0 ujson==1.35 -unidecode==0.4.21 +unidecode==1.0.22 unittest2==1.1.0 uritemplate==3.0.0 urllib3==1.22 @@ -186,11 +185,11 @@ versiontools==1.9.1 virtualenv==15.1.0 waitress==1.1.0 warlock==1.3.0 -webob==1.7.3 -websocket-client==0.44.0 +webob==1.7.4 +websocket-client==0.46.0 webtest==2.0.29 -werkzeug==0.12.2 +werkzeug==0.14.1 wheel==0.30.0 xlrd==1.1.0 -zc.buildout==2.9.5 +zc.buildout==2.11.0 zope.interface==4.4.3 From 1a69141db0b08cc73eccc40a2d3ad03d71a09e73 Mon Sep 17 00:00:00 2001 From: Douglas Greiman Date: Mon, 12 Feb 2018 14:20:24 -0800 Subject: [PATCH 154/256] Update Python interpreters to 3.5.5 and 3.6.4 (#177) --- python-interpreter-builder/Dockerfile.in | 4 ++-- .../scripts/build-python-3.5.sh | 12 ++++++------ .../scripts/build-python-3.6.sh | 12 ++++++------ tests/virtualenv/virtualenv_python35.yaml | 2 +- tests/virtualenv/virtualenv_python36.yaml | 2 +- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/python-interpreter-builder/Dockerfile.in b/python-interpreter-builder/Dockerfile.in index 90e606eb..3c5fc31c 100644 --- a/python-interpreter-builder/Dockerfile.in +++ b/python-interpreter-builder/Dockerfile.in @@ -51,10 +51,10 @@ RUN mkdir -p /opt/packages && \ echo -n "" > /opt/packages/packages.txt RUN /scripts/build-python-3.5.sh && \ - /scripts/package-python.sh 3.5.4 "1gcp~${TAG}" + /scripts/package-python.sh 3.5.5 "1gcp~${TAG}" RUN /scripts/build-python-3.6.sh && \ - /scripts/package-python.sh 3.6.2 "1gcp~${TAG}" + /scripts/package-python.sh 3.6.4 "1gcp~${TAG}" # Tar the interpreters. Tarring is needed because docker cp doesn't handle # links correctly. diff --git a/python-interpreter-builder/scripts/build-python-3.5.sh b/python-interpreter-builder/scripts/build-python-3.5.sh index eeea1f76..c2a3337e 100755 --- a/python-interpreter-builder/scripts/build-python-3.5.sh +++ b/python-interpreter-builder/scripts/build-python-3.5.sh @@ -6,14 +6,14 @@ set -x # Get the source mkdir -p /opt/sources cd /opt/sources -wget --no-verbose https://www.python.org/ftp/python/3.5.4/Python-3.5.4.tgz +wget --no-verbose https://www.python.org/ftp/python/3.5.5/Python-3.5.5.tgz # SHA-256 generated via `shasum -a 256 [file]` shasum --check < Date: Mon, 26 Feb 2018 13:08:16 -0800 Subject: [PATCH 155/256] Test cleanups and fixes (#179) * Separate Google Cloud Client Library tests from other tests. Use the --client_test flag to build.sh to run them. * Fix test failure with google-cloud-dlp. See https://github.com/GoogleCloudPlatform/google-cloud-python/issues/4924 Also add better logging of exactly which files experience failures. * Remove system test which is redundant with the integration test. * Fix punctuation. --- build.sh | 46 ++++++------------- cloudbuild_client_test.yaml | 9 ++++ cloudbuild_system_test.yaml | 9 ---- cloudbuild_test.yaml | 6 --- tests/google-cloud-python-system/.gitignore | 2 - .../google-cloud-python-system/Dockerfile.in | 16 ------- .../run_system_tests.sh | 39 ---------------- tests/google-cloud-python/run_unit_tests.sh | 18 +++++++- 8 files changed, 41 insertions(+), 104 deletions(-) create mode 100644 cloudbuild_client_test.yaml delete mode 100644 cloudbuild_system_test.yaml delete mode 100644 tests/google-cloud-python-system/.gitignore delete mode 100644 tests/google-cloud-python-system/Dockerfile.in delete mode 100755 tests/google-cloud-python-system/run_system_tests.sh diff --git a/build.sh b/build.sh index 7e2370d4..62269d36 100755 --- a/build.sh +++ b/build.sh @@ -19,7 +19,7 @@ set -euo pipefail # Actions benchmark=0 # Should run benchmarks? build=0 # Should build images? -system_test=0 # Should run system tests? +client_test=0 # Should run Google Cloud Client Library tests test=0 # Should run standard test suite? local=0 # Should run using local Docker daemon instead of GCR? @@ -42,8 +42,8 @@ Options: --[no]benchmark: Run benchmarking suite (default false) --[no]build: Build all images (default true if no options set) --[no]test: Run basic tests (default true if no options set) + --[no]client_test: Run Google Cloud Client Library tests (default false) --[no]local: Build images using local Docker daemon (default false) - --[no]system_test: Run system tests (default false) " } @@ -90,20 +90,20 @@ while [ $# -gt 0 ]; do build=0 shift ;; - --local) - local=1 + --client_test) + client_test=1 shift ;; - --nolocal) - local=0 + --noclient_test) + client_test=0 shift ;; - --system_test) - system_test=1 + --local) + local=1 shift ;; - --nosystem_test) - system_test=0 + --nolocal) + local=0 shift ;; --test) @@ -123,7 +123,7 @@ done # If no actions chosen, then tell the user if [ "${benchmark}" -eq 0 -a \ "${build}" -eq 0 -a \ - "${system_test}" -eq 0 -a \ + "${client_test}" -eq 0 -a \ "${test}" -eq 0 \ ]; then echo 'No actions specified, defaulting to --build --test' @@ -136,17 +136,6 @@ if [ "${local}" -eq 1 ]; then gcloud_cmd="${local_gcloud_cmd}" fi -# Read action-specific environment variables -if [ "${system_test}" -eq 1 ]; then - if [ -z "${GOOGLE_APPLICATION_CREDENTIALS_FOR_TESTS+set}" ] ; then - fatal 'Error: $GOOGLE_APPLICATION_CREDENTIALS_FOR_TESTS is not set; invoke with something like GOOGLE_APPLICATION_CREDENTIALS_FOR_TESTS=/path/to/service/account/creds.json' - fi - - if [ -z "${GOOGLE_CLOUD_PROJECT_FOR_TESTS+set}" ] ; then - fatal 'Error: $GOOGLE_CLOUD_PROJECT_FOR_TESTS is not set; invoke with something like GOOGLE_CLOUD_PROJECT_FOR_TESTS=YOUR-PROJECT-NAME' - fi -fi - # Use latest released Debian as our base image export DEBIAN_BASE_IMAGE="gcr.io/google-appengine/debian8:latest" export STAGING_IMAGE="${DOCKER_NAMESPACE}/python:${TAG}" @@ -160,7 +149,6 @@ for outfile in \ tests/benchmark/Dockerfile \ tests/eventlet/Dockerfile \ tests/google-cloud-python/Dockerfile \ - tests/google-cloud-python-system/Dockerfile \ tests/integration/Dockerfile \ ; do envsubst <"${outfile}".in >"${outfile}" \ @@ -192,14 +180,10 @@ if [ "${test}" -eq 1 ]; then ${gcloud_cmd} --config cloudbuild_test.yaml --substitutions "${substitutions}" fi -# Run system tests -if [ "${system_test}" -eq 1 ]; then - echo "Running system tests using project ${GOOGLE_CLOUD_PROJECT_FOR_TESTS}" - - trap "rm -f tests/google-cloud-python-system/credentials.json" EXIT - cp "${GOOGLE_APPLICATION_CREDENTIALS_FOR_TESTS}" tests/google-cloud-python-system/credentials.json - ${gcloud_cmd} --config cloudbuild_system_test.yaml --substitutions "${substitutions}" - rm -f tests/google-cloud-python-system/credentials.json +# Run client library tests +if [ "${client_test}" -eq 1 ]; then + echo "Testing compatibility with Google Cloud Client Libraries" + ${gcloud_cmd} --config cloudbuild_client_test.yaml --substitutions "${substitutions}" fi # Run benchmarks diff --git a/cloudbuild_client_test.yaml b/cloudbuild_client_test.yaml new file mode 100644 index 00000000..010e1ffd --- /dev/null +++ b/cloudbuild_client_test.yaml @@ -0,0 +1,9 @@ +timeout: 3600s +steps: +- # Build image to run google client library unit tests + name: gcr.io/cloud-builders/docker:latest + args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/google-cloud-python:${_TAG}', + '--no-cache', '/workspace/tests/google-cloud-python/'] +- # Run google client library unit tests + name: ${_DOCKER_NAMESPACE}/python/tests/google-cloud-python:${_TAG} +images: [] diff --git a/cloudbuild_system_test.yaml b/cloudbuild_system_test.yaml deleted file mode 100644 index 3cac03e2..00000000 --- a/cloudbuild_system_test.yaml +++ /dev/null @@ -1,9 +0,0 @@ -timeout: 3600s -steps: -- name: gcr.io/cloud-builders/docker:latest - args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/google-cloud-python-system:${_TAG}', - '--no-cache', '/workspace/tests/google-cloud-python-system/'] -- name: ${_DOCKER_NAMESPACE}/python/tests/google-cloud-python-system:${_TAG} -images: [ - # Intentionally empty -] diff --git a/cloudbuild_test.yaml b/cloudbuild_test.yaml index acf70bc7..6efe5a45 100644 --- a/cloudbuild_test.yaml +++ b/cloudbuild_test.yaml @@ -37,10 +37,4 @@ steps: name: gcr.io/cloud-builders/docker:latest args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/eventlet:${_TAG}', '--no-cache', '/workspace/tests/eventlet/'] -- # Build image to run google client library unit tests - name: gcr.io/cloud-builders/docker:latest - args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/google-cloud-python:${_TAG}', - '--no-cache', '/workspace/tests/google-cloud-python/'] -- # Run google client library unit tests - name: ${_DOCKER_NAMESPACE}/python/tests/google-cloud-python:${_TAG} images: [] diff --git a/tests/google-cloud-python-system/.gitignore b/tests/google-cloud-python-system/.gitignore deleted file mode 100644 index 470715ec..00000000 --- a/tests/google-cloud-python-system/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -credentials.json -Dockerfile diff --git a/tests/google-cloud-python-system/Dockerfile.in b/tests/google-cloud-python-system/Dockerfile.in deleted file mode 100644 index 04f28bd5..00000000 --- a/tests/google-cloud-python-system/Dockerfile.in +++ /dev/null @@ -1,16 +0,0 @@ -FROM ${STAGING_IMAGE} - -# Get the source. -RUN git clone --depth 1 https://github.com/GoogleCloudPlatform/google-cloud-python.git -WORKDIR google-cloud-python - -# Install nox -RUN pip install --upgrade nox-automation - -# Secrets injected at runtime -ENV GOOGLE_APPLICATION_CREDENTIALS=/workspace/tests/google-cloud-python-system/credentials.json -ENV GOOGLE_CLOUD_PROJECT=${GOOGLE_CLOUD_PROJECT_FOR_TESTS} - -# Run system tests for all supported Python versions -ADD run_system_tests.sh /run_system_tests.sh -ENTRYPOINT ["/run_system_tests.sh"] diff --git a/tests/google-cloud-python-system/run_system_tests.sh b/tests/google-cloud-python-system/run_system_tests.sh deleted file mode 100755 index f9f81a47..00000000 --- a/tests/google-cloud-python-system/run_system_tests.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/sh -set -eu - -cd /app/google-cloud-python - -# Not all packages have system tests -packages=" -bigquery -bigtable -datastore -language -logging -monitoring -pubsub -spanner -speech -storage -vision -" - -# translate has system test but it gives error message: -# BadRequest: 400 Invalid JSON payload received. Unknown name "model": Cannot bind 'nmt'. Field 'model' could not be found in request message. (GET https://translation.googleapis.com/language/translate/v2?target=de&q=hvala+ti&q=dankon&q=Me+llamo+Jeff&q=My+name+is+Jeff&model=nmt) -disabled_packages="translate" - -# Spanner system test needs this -export GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE=1 - -exit_code=0 -for package in ${packages}; do - noxfile="${package}/nox.py" - nox \ - -f "${noxfile}" \ - -e \ - "system(py='2.7')" \ - "system(py='3.6')" \ - || exit_code=1 -done - -exit "${exit_code}" diff --git a/tests/google-cloud-python/run_unit_tests.sh b/tests/google-cloud-python/run_unit_tests.sh index 81a6a389..c386f1c0 100755 --- a/tests/google-cloud-python/run_unit_tests.sh +++ b/tests/google-cloud-python/run_unit_tests.sh @@ -4,7 +4,13 @@ set -eu cd /app/google-cloud-python exit_code=0 +failed_files= for noxfile in */nox.py; do + if [ "${noxfile}" = "dlp/nox.py" ]; then + echo "**** Skipping ${noxfile} ****" + continue + fi + echo "**** Starting tests in ${noxfile} ****" nox \ -f "${noxfile}" \ -e \ @@ -12,7 +18,17 @@ for noxfile in */nox.py; do "unit(py='3.4')" \ "unit(py='3.5')" \ "unit(py='3.6')" \ - || exit_code=1 + || { + echo "**** FAILED tests in ${noxfile} ****" + exit_code=1 + failed_files="${failed_files} ${noxfile}" + } + echo "**** Finished tests in ${noxfile} ****" done +if [ "${exit_code}" -eq 0 ]; then + echo "**** All tests passed ****" +else + echo "**** There were test failures:${failed_files} ****" +fi exit "${exit_code}" From a1a0c2d7efa00dacc0f197fa0a97426f783da8e3 Mon Sep 17 00:00:00 2001 From: Douglas Greiman Date: Mon, 26 Feb 2018 18:50:31 -0800 Subject: [PATCH 156/256] Remove unused file (#181) --- jenkins_build.sh | 3 --- 1 file changed, 3 deletions(-) delete mode 100755 jenkins_build.sh diff --git a/jenkins_build.sh b/jenkins_build.sh deleted file mode 100755 index f88007f5..00000000 --- a/jenkins_build.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -./build.sh "$@" From fe98d8b5b58d8dee486797d55398e38f76bdfd4d Mon Sep 17 00:00:00 2001 From: Douglas Greiman Date: Wed, 28 Feb 2018 11:32:41 -0800 Subject: [PATCH 157/256] Stop building Debian packages. (#180) We never started using these packages, and have a different internal plan for packaging in the future. --- python-interpreter-builder/Dockerfile.in | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/python-interpreter-builder/Dockerfile.in b/python-interpreter-builder/Dockerfile.in index 3c5fc31c..834047b9 100644 --- a/python-interpreter-builder/Dockerfile.in +++ b/python-interpreter-builder/Dockerfile.in @@ -50,11 +50,9 @@ ADD DEBIAN /DEBIAN RUN mkdir -p /opt/packages && \ echo -n "" > /opt/packages/packages.txt -RUN /scripts/build-python-3.5.sh && \ - /scripts/package-python.sh 3.5.5 "1gcp~${TAG}" +RUN /scripts/build-python-3.5.sh -RUN /scripts/build-python-3.6.sh && \ - /scripts/package-python.sh 3.6.4 "1gcp~${TAG}" +RUN /scripts/build-python-3.6.sh # Tar the interpreters. Tarring is needed because docker cp doesn't handle # links correctly. From 58a537c00e55a75cdfd5fe6e637d33829139b9e8 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Mon, 12 Mar 2018 17:53:31 -0700 Subject: [PATCH 158/256] Auto-update dependencies. (#178) --- runtime-image/resources/requirements.txt | 2 +- scripts/requirements-test.txt | 2 +- tests/eventlet/requirements.txt | 2 +- tests/integration/requirements.txt | 6 +- tests/python2-libraries/requirements.txt | 72 ++++++++++++------------ tests/python3-libraries/requirements.txt | 68 +++++++++++----------- 6 files changed, 76 insertions(+), 76 deletions(-) diff --git a/runtime-image/resources/requirements.txt b/runtime-image/resources/requirements.txt index 6a1b2cc3..ef675285 100644 --- a/runtime-image/resources/requirements.txt +++ b/runtime-image/resources/requirements.txt @@ -1,3 +1,3 @@ pip==9.0.1 -setuptools==38.5.0 +setuptools==38.5.2 wheel==0.30.0 diff --git a/scripts/requirements-test.txt b/scripts/requirements-test.txt index 12a6ca4a..1ae0abd8 100644 --- a/scripts/requirements-test.txt +++ b/scripts/requirements-test.txt @@ -1,4 +1,4 @@ flask==0.12.2 -pytest==3.4.0 +pytest==3.4.2 pytest-cov==2.5.1 pyyaml==3.12 diff --git a/tests/eventlet/requirements.txt b/tests/eventlet/requirements.txt index a8f7b0a9..5d104d0a 100644 --- a/tests/eventlet/requirements.txt +++ b/tests/eventlet/requirements.txt @@ -1,6 +1,6 @@ click==6.7 enum-compat==0.0.2 -eventlet==0.22.0 +eventlet==0.22.1 Flask==0.12.2 greenlet==0.4.13 gunicorn==19.7.1 diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index 99a06d3b..64e39e0c 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -1,7 +1,7 @@ Flask==0.12.2 -google-cloud-error-reporting==0.29.0 -google-cloud-logging==1.4.0 -google-cloud-monitoring==0.28.0 +google-cloud-error-reporting==0.29.1 +google-cloud-logging==1.6.0 +google-cloud-monitoring==0.28.1 gunicorn==19.7.1 requests==2.18.4 retrying==1.3.3 diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index f512ca18..f261209e 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -1,12 +1,12 @@ -alembic==0.9.7 +alembic==0.9.8 amqp==2.2.2 amqplib==1.0.2 ansible==2.4.3.0 anyjson==0.3.3 -apache-libcloud==2.2.1 +apache-libcloud==2.3.0 argparse==1.4.0 astroid==1.6.1 -awscli==1.14.32 +awscli==1.14.53 babel==2.5.3 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 @@ -16,21 +16,21 @@ billiard==3.5.0.3 blessings==1.6.1 blinker==1.4 boto==2.48.0 -botocore==1.8.36 +botocore==1.9.6 bottle==0.12.13 carbon<1.1.1 celery==4.1.0 certifi==2018.1.18 -cffi==1.11.4 +cffi==1.11.5 chardet==3.0.4 click==6.7 cliff==2.11.0 -cmd2==0.8.0 +cmd2==0.8.1 colorama==0.3.9 configobj==5.0.6 cov-core==1.15.0 -coverage==4.5 -coveralls==1.2.0 +coverage==4.5.1 +coveralls==1.3.0 crcmod==1.7 cryptography==2.1.4 cssselect==1.0.3 @@ -38,7 +38,7 @@ cython==0.27.3 decorator==4.2.1 django-celery==3.2.2 django-debug-toolbar==1.9.1 -django-extensions==1.9.9 +django-extensions==2.0.5 django<2.0 django_compress==1.0.1 djangorestframework==3.7.7 @@ -48,7 +48,7 @@ docutils==0.14 ecdsa==0.13 elasticsearch==6.1.1 enum34==1.1.6 -eventlet==0.22.0 +eventlet==0.22.1 extras==1.0.0 fabric==1.14.0 fixtures==3.0.0 @@ -59,7 +59,7 @@ functools32==3.2.3.post2 futures==3.2.0 gevent==1.2.2 google-api-python-client==1.6.5 -graphite-web==1.1.1 +graphite-web==1.1.2 greenlet==0.4.13 gunicorn==19.7.1 hiredis==0.2.0 @@ -78,17 +78,17 @@ kombu==4.1.0 linecache2==1.0.0 logilab-common==1.4.1 lxml==4.1.1 -m2crypto==0.27.0 +m2crypto==0.29.0 mako==1.0.7 manifestparser==1.1 markdown==2.6.11 markupsafe==1.0 -matplotlib==2.1.2 +matplotlib==2.2.0 mccabe==0.6.1 meld3==1.0.2 mock==2.0.0 mozcrash==1.0 -mozdevice==0.51 +mozdevice==0.52 mozfile==1.2 mozinfo==0.10 mozlog==3.7 @@ -96,14 +96,14 @@ moznetwork==0.27 mozprocess==0.26 mozprofile==0.29 mozrunner==6.14 -msgpack-python==0.5.2 +msgpack-python==0.5.6 mysql-python==1.2.5 ndg-httpsclient==0.4.4 netaddr==0.7.19 netifaces==0.10.6 -newrelic==2.100.0.84 +newrelic==2.106.1.88 nose==1.3.7 -numpy==1.14.0 +numpy==1.14.1 oauth2==1.9.0.post1 oauth2client==4.1.2 oauthlib==2.0.6 @@ -117,14 +117,14 @@ pastedeploy==1.5.2 pastescript==2.0.2 pbr==3.1.1 pep8==1.7.1 -pexpect==4.3.1 +pexpect==4.4.0 pika==0.11.2 pillow==5.0.0 pip==9.0.1 prettytable -protobuf==3.5.1 +protobuf==3.5.2 psutil==5.4.3 -psycopg2==2.7.3.2 +psycopg2==2.7.4 py==1.5.2 pyasn1-modules==0.2.1 pyasn1==0.4.2 @@ -133,20 +133,20 @@ pycrypto==2.6.1 pycurl==7.43.0.1 pyflakes==1.6.0 pygments==2.2.0 -pyjwt==1.5.3 +pyjwt==1.6.0 pylibmc==1.5.2 pylint==1.8.2 -pymongo==3.6.0 +pymongo==3.6.1 pymysql==0.8.0 pyopenssl==17.5.0 pyparsing==2.2.0 pyramid==1.9.1 pystache==0.5.4 pytest-cov==2.5.1 -pytest==3.4.0 +pytest==3.4.2 python-cjson==1.2.1 python-daemon==2.1.2 -python-dateutil==2.6.1 +python-dateutil==2.7.0 python-gflags==3.1.2 python-keystoneclient==3.15.0 python-memcached==1.59 @@ -154,10 +154,10 @@ python-mimeparse==1.6.0 python-novaclient==10.1.0 python-subunit==1.2.0 python-swiftclient==3.5.0 -pytz==2017.3 +pytz==2018.3 pyyaml==3.12 -pyzmq==16.0.3 -raven==6.5.0 +pyzmq==17.0.0 +raven==6.6.0 redis==2.10.6 repoze.lru==0.7 requests-oauthlib==0.8.0 @@ -165,26 +165,26 @@ requests==2.18.4 retrying==1.3.3 rsa==3.4.2 scipy==1.0.0 -selenium==3.8.1 +selenium==3.10.0 setuptools-git==1.2 -setuptools==38.5.0 +setuptools==38.5.2 sh==1.12.14 simplejson==3.13.2 six==1.11.0 snowballstemmer==1.2.1 south==1.0.2 -sphinx==1.6.7 +sphinx==1.7.1 sqlalchemy-migrate==0.11.0 -sqlalchemy==1.2.2 +sqlalchemy==1.2.5 sqlparse==0.2.4 statsd==3.2.2 stevedore==1.28.0 suds==0.4 -supervisor==3.3.3 +supervisor==3.3.4 testrepository==0.0.20 testtools==2.3.0 thrift==0.11.0 -tornado==4.5.3 +tornado==5.0 tox==2.9.1 twisted==17.9.0 ujson==1.35 @@ -192,16 +192,16 @@ unidecode==1.0.22 unittest2==1.1.0 uritemplate==3.0.0 urllib3==1.22 -uwsgi==2.0.15 +uwsgi==2.0.17 versiontools==1.9.1 virtualenv==15.1.0 waitress==1.1.0 warlock==1.3.0 webob==1.7.4 -websocket-client==0.46.0 +websocket-client==0.47.0 webtest==2.0.29 werkzeug==0.14.1 wheel==0.30.0 xlrd==1.1.0 -zc.buildout==2.11.0 +zc.buildout==2.11.1 zope.interface==4.4.3 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index a8664b03..30fb236e 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -1,12 +1,12 @@ -alembic==0.9.7 +alembic==0.9.8 amqp==2.2.2 amqplib==1.0.2 ansible==2.4.3.0 anyjson==0.3.3 -apache-libcloud==2.2.1 +apache-libcloud==2.3.0 argparse==1.4.0 astroid==1.6.1 -awscli==1.14.32 +awscli==1.14.53 babel==2.5.3 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 @@ -15,20 +15,20 @@ billiard==3.5.0.3 blessings==1.6.1 blinker==1.4 boto==2.48.0 -botocore==1.8.36 +botocore==1.9.6 bottle==0.12.13 celery==4.1.0 certifi==2018.1.18 -cffi==1.11.4 +cffi==1.11.5 chardet==3.0.4 click==6.7 cliff==2.11.0 -cmd2==0.8.0 +cmd2==0.8.1 colorama==0.3.9 configobj==5.0.6 cov-core==1.15.0 -coverage==4.5 -coveralls==1.2.0 +coverage==4.5.1 +coveralls==1.3.0 crcmod==1.7 cryptography==2.1.4 cssselect==1.0.3 @@ -36,8 +36,8 @@ cython==0.27.3 decorator==4.2.1 django-celery==3.2.2 django-debug-toolbar==1.9.1 -django-extensions==1.9.9 -django==2.0.2 +django-extensions==2.0.5 +django==2.0.3 django_compress==1.0.1 djangorestframework==3.7.7 docker-py==1.10.6 @@ -46,7 +46,7 @@ docutils==0.14 ecdsa==0.13 elasticsearch==6.1.1 enum34==1.1.6 -eventlet==0.22.0 +eventlet==0.22.1 extras==1.0.0 fabric==1.14.0 fixtures==3.0.0 @@ -78,24 +78,24 @@ mako==1.0.7 manifestparser==1.1 markdown==2.6.11 markupsafe==1.0 -matplotlib==2.1.2 +matplotlib==2.2.0 mccabe==0.6.1 meld3==1.0.2 mock==2.0.0 mozcrash==1.0 -mozdevice==0.51 +mozdevice==0.52 mozfile==1.2 mozinfo==0.10 mozlog==3.7 moznetwork==0.27 mozprocess==0.26 -msgpack-python==0.5.2 +msgpack-python==0.5.6 ndg-httpsclient==0.4.4 netaddr==0.7.19 netifaces==0.10.6 -newrelic==2.100.0.84 +newrelic==2.106.1.88 nose==1.3.7 -numpy==1.14.0 +numpy==1.14.1 oauth2==1.9.0.post1 oauth2client==4.1.2 oauthlib==2.0.6 @@ -109,14 +109,14 @@ pastedeploy==1.5.2 pastescript==2.0.2 pbr==3.1.1 pep8==1.7.1 -pexpect==4.3.1 +pexpect==4.4.0 pika==0.11.2 pillow==5.0.0 pip==9.0.1 prettytable -protobuf==3.5.1 +protobuf==3.5.2 psutil==5.4.3 -psycopg2==2.7.3.2 +psycopg2==2.7.4 py==1.5.2 pyasn1-modules==0.2.1 pyasn1==0.4.2 @@ -124,19 +124,19 @@ pycparser==2.18 pycrypto==2.6.1 pyflakes==1.6.0 pygments==2.2.0 -pyjwt==1.5.3 +pyjwt==1.6.0 pylibmc==1.5.2 pylint==1.8.2 -pymongo==3.6.0 +pymongo==3.6.1 pymysql==0.8.0 pyopenssl==17.5.0 pyparsing==2.2.0 pyramid==1.9.1 pystache==0.5.4 pytest-cov==2.5.1 -pytest==3.4.0 +pytest==3.4.2 python-daemon==2.1.2 -python-dateutil==2.6.1 +python-dateutil==2.7.0 python-gflags==3.1.2 python-keystoneclient==3.15.0 python-memcached==1.59 @@ -144,10 +144,10 @@ python-mimeparse==1.6.0 python-novaclient==10.1.0 python-subunit==1.2.0 python-swiftclient==3.5.0 -pytz==2017.3 +pytz==2018.3 pyyaml==3.12 -pyzmq==16.0.3 -raven==6.5.0 +pyzmq==17.0.0 +raven==6.6.0 redis==2.10.6 repoze.lru==0.7 requests-oauthlib==0.8.0 @@ -155,24 +155,24 @@ requests==2.18.4 retrying==1.3.3 rsa==3.4.2 scipy==1.0.0 -selenium==3.8.1 +selenium==3.10.0 setuptools-git==1.2 -setuptools==38.5.0 +setuptools==38.5.2 sh==1.12.14 simplejson==3.13.2 six==1.11.0 snowballstemmer==1.2.1 south==1.0.2 -sphinx==1.6.7 +sphinx==1.7.1 sqlalchemy-migrate==0.11.0 -sqlalchemy==1.2.2 +sqlalchemy==1.2.5 sqlparse==0.2.4 statsd==3.2.2 stevedore==1.28.0 testrepository==0.0.20 testtools==2.3.0 thrift==0.11.0 -tornado==4.5.3 +tornado==5.0 tox==2.9.1 twisted==17.9.0 ujson==1.35 @@ -180,16 +180,16 @@ unidecode==1.0.22 unittest2==1.1.0 uritemplate==3.0.0 urllib3==1.22 -uwsgi==2.0.15 +uwsgi==2.0.17 versiontools==1.9.1 virtualenv==15.1.0 waitress==1.1.0 warlock==1.3.0 webob==1.7.4 -websocket-client==0.46.0 +websocket-client==0.47.0 webtest==2.0.29 werkzeug==0.14.1 wheel==0.30.0 xlrd==1.1.0 -zc.buildout==2.11.0 +zc.buildout==2.11.1 zope.interface==4.4.3 From 8106a265e0ad5f44d9db43bbf44d742282a96a74 Mon Sep 17 00:00:00 2001 From: Douglas Greiman Date: Tue, 13 Mar 2018 14:05:07 -0700 Subject: [PATCH 159/256] Add support for using Ubuntu 16 as a base instead of Debian 8 (#182) * Add --os_base flag to build against different OS base images. Also, change --local to use the official gcloud tool instead of our own. * Update from Python 3.4.2 to Python 3.4.8 Contains various bugfixes and security patches. We switch from using the Debian-provided "python3.4" packages to building it from source, in preparation for moving to newer OS base images that don't have a prebuilt Python 3.4 interpreter available. * Update comments about configure flags * Temporarily disable license check * Add some necessary packages for Ubuntu * Ubuntu ships with Python 2.7.12 * Remove errant whitespace * Rename DEBIAN_BASE_IMAGE to OS_BASE_IMAGE --- build.sh | 46 ++++-- cloudbuild_test.yaml | 70 +++++---- python-interpreter-builder/Dockerfile.in | 5 +- .../scripts/build-python-3.4.sh | 138 ++++++++++++++++++ runtime-image/Dockerfile.in | 6 +- runtime-image/resources/apt-packages.txt | 3 - tests/license-test/license-test.yaml | 2 +- tests/no-virtualenv/no-virtualenv.yaml | 2 +- tests/virtualenv/virtualenv_default.yaml | 2 +- tests/virtualenv/virtualenv_python27.yaml | 2 +- tests/virtualenv/virtualenv_python34.yaml | 2 +- 11 files changed, 232 insertions(+), 46 deletions(-) create mode 100755 python-interpreter-builder/scripts/build-python-3.4.sh diff --git a/build.sh b/build.sh index 62269d36..9b3a258d 100755 --- a/build.sh +++ b/build.sh @@ -24,9 +24,12 @@ test=0 # Should run standard test suite? local=0 # Should run using local Docker daemon instead of GCR? +os_base=debian8 # Which operating system base to use + # Note that $gcloud_cmd has spaces in it -gcloud_cmd="gcloud beta container builds submit ." -local_gcloud_cmd="scripts/local_cloudbuild.py" +gcloud_cmd="gcloud container builds submit" +# May need to install via "gcloud components install container-builder-local" +local_gcloud_cmd="container-builder-local --push=false --dryrun=false" # Helper functions function fatal() { @@ -44,6 +47,7 @@ Options: --[no]test: Run basic tests (default true if no options set) --[no]client_test: Run Google Cloud Client Library tests (default false) --[no]local: Build images using local Docker daemon (default false) + --os_base: Which OS image to build on top of [debian8, ubuntu16] " } @@ -106,6 +110,14 @@ while [ $# -gt 0 ]; do local=0 shift ;; + --os_base=debian8) + os_base=debian8 + shift + ;; + --os_base=ubuntu16) + os_base=ubuntu16 + shift + ;; --test) test=1 shift @@ -136,8 +148,12 @@ if [ "${local}" -eq 1 ]; then gcloud_cmd="${local_gcloud_cmd}" fi -# Use latest released Debian as our base image -export DEBIAN_BASE_IMAGE="gcr.io/google-appengine/debian8:latest" +# Pick OS image to use as base +if [ "${os_base}" == "ubuntu16" ]; then + export OS_BASE_IMAGE="gcr.io/gcp-runtimes/ubuntu_16_0_4:latest" +else + export OS_BASE_IMAGE="gcr.io/google-appengine/debian8:latest" +fi export STAGING_IMAGE="${DOCKER_NAMESPACE}/python:${TAG}" echo "Using base image name ${STAGING_IMAGE}" @@ -152,7 +168,7 @@ for outfile in \ tests/integration/Dockerfile \ ; do envsubst <"${outfile}".in >"${outfile}" \ - '$DEBIAN_BASE_IMAGE $STAGING_IMAGE $GOOGLE_CLOUD_PROJECT_FOR_TESTS $TAG' + '$OS_BASE_IMAGE $STAGING_IMAGE $GOOGLE_CLOUD_PROJECT_FOR_TESTS $TAG' done # Make some files available to the runtime builder Docker context @@ -171,23 +187,35 @@ cp -a scripts/testdata/hello_world/main.py tests/eventlet/main.py # Build images and push to GCR if [ "${build}" -eq 1 ]; then echo "Building images" - ${gcloud_cmd} --config cloudbuild.yaml --substitutions "${build_substitutions}" + ${gcloud_cmd} \ + --config=cloudbuild.yaml \ + --substitutions="${build_substitutions}" \ + . fi # Run the tests that don't require (too many) external services if [ "${test}" -eq 1 ]; then echo "Testing compatibility with popular Python libraries" - ${gcloud_cmd} --config cloudbuild_test.yaml --substitutions "${substitutions}" + ${gcloud_cmd} \ + --config=cloudbuild_test.yaml \ + --substitutions="${substitutions}" \ + . fi # Run client library tests if [ "${client_test}" -eq 1 ]; then echo "Testing compatibility with Google Cloud Client Libraries" - ${gcloud_cmd} --config cloudbuild_client_test.yaml --substitutions "${substitutions}" + ${gcloud_cmd} \ + --config=cloudbuild_client_test.yaml \ + --substitutions="${substitutions}" \ + . fi # Run benchmarks if [ "${benchmark}" -eq 1 ] ; then echo "Running benchmark" - ${gcloud_cmd} --config cloudbuild_benchmark.yaml --substitutions "${substitutions}" + ${gcloud_cmd} \ + --config=cloudbuild_benchmark.yaml \ + --substitutions="${substitutions}" \ + . fi diff --git a/cloudbuild_test.yaml b/cloudbuild_test.yaml index 6efe5a45..ef0b48b3 100644 --- a/cloudbuild_test.yaml +++ b/cloudbuild_test.yaml @@ -1,27 +1,12 @@ timeout: 3600s steps: - # Validate structure of base runtime image -- name: gcr.io/cloud-builders/docker:latest - args: ['build', '-t', 'python2-libraries-intermediate', '--build-arg', - 'intermediate_image=${_DOCKER_NAMESPACE}/python:${_TAG}', - '/workspace/tests/python2-libraries'] -- name: gcr.io/gcp-runtimes/container-structure-test:v0.1.1 +- # Explicitly pull image into GCB so that later steps work + name: '${_DOCKER_NAMESPACE}/python:${_TAG}' args: [ - '-test.v', - '-image', 'python2-libraries-intermediate', - '/workspace/tests/python2-libraries/python2-libraries.yaml' - ] -- name: gcr.io/cloud-builders/docker:latest - args: ['build', '-t', 'python3-libraries-intermediate', '--build-arg', - 'intermediate_image=${_DOCKER_NAMESPACE}/python:${_TAG}', - '/workspace/tests/python3-libraries'] -- name: gcr.io/gcp-runtimes/container-structure-test:v0.1.1 - args: [ - '-test.v', - '-image', 'python3-libraries-intermediate', - '/workspace/tests/python3-libraries/python3-libraries.yaml' - ] -- name: gcr.io/gcp-runtimes/container-structure-test:v0.1.1 + '/bin/true', + ] +- # Validate structure of base runtime image + name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 args: [ '-test.v', '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', @@ -31,10 +16,45 @@ steps: '/workspace/tests/virtualenv/virtualenv_python35.yaml', '/workspace/tests/virtualenv/virtualenv_python36.yaml', '/workspace/tests/no-virtualenv/no-virtualenv.yaml', - '/workspace/tests/license-test/license-test.yaml' ] -- # Run compatibility tests +# Temporarily disabled because it fails on symbolic links in Ubuntu: +# https://github.com/GoogleCloudPlatform/container-structure-test/issues/77 +#- # Check license compliance +# name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 +# args: [ +# '-test.v', +# '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', +# '/workspace/tests/license-test/license-test.yaml' +# ] +- # Do third-party library compatibility tests + name: gcr.io/cloud-builders/docker:latest + args: [ + 'build', '-t', 'python2-libraries-intermediate', '--build-arg', + 'intermediate_image=${_DOCKER_NAMESPACE}/python:${_TAG}', + '/workspace/tests/python2-libraries' + ] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 + args: [ + '-test.v', + '-image', 'python2-libraries-intermediate', + '/workspace/tests/python2-libraries/python2-libraries.yaml' + ] +- name: gcr.io/cloud-builders/docker:latest + args: [ + 'build', '-t', 'python3-libraries-intermediate', '--build-arg', + 'intermediate_image=${_DOCKER_NAMESPACE}/python:${_TAG}', + '/workspace/tests/python3-libraries' + ] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 + args: [ + '-test.v', + '-image', 'python3-libraries-intermediate', + '/workspace/tests/python3-libraries/python3-libraries.yaml' + ] +- # Run other compatibility tests name: gcr.io/cloud-builders/docker:latest - args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/eventlet:${_TAG}', - '--no-cache', '/workspace/tests/eventlet/'] + args: [ + 'build', '--tag=${_DOCKER_NAMESPACE}/python/tests/eventlet:${_TAG}', + '--no-cache', '/workspace/tests/eventlet/' + ] images: [] diff --git a/python-interpreter-builder/Dockerfile.in b/python-interpreter-builder/Dockerfile.in index 834047b9..46bc092a 100644 --- a/python-interpreter-builder/Dockerfile.in +++ b/python-interpreter-builder/Dockerfile.in @@ -1,6 +1,6 @@ # The Google App Engine base image is debian (jessie) with ca-certificates # installed. -FROM ${DEBIAN_BASE_IMAGE} +FROM ${OS_BASE_IMAGE} # Install Python build dependencies (based on Debian Build-Depends) RUN apt-get update && apt-get install -yq \ @@ -29,6 +29,7 @@ RUN apt-get update && apt-get install -yq \ mime-support \ net-tools \ netbase \ + python \ python3 \ sharutils \ time \ @@ -50,6 +51,8 @@ ADD DEBIAN /DEBIAN RUN mkdir -p /opt/packages && \ echo -n "" > /opt/packages/packages.txt +RUN /scripts/build-python-3.4.sh + RUN /scripts/build-python-3.5.sh RUN /scripts/build-python-3.6.sh diff --git a/python-interpreter-builder/scripts/build-python-3.4.sh b/python-interpreter-builder/scripts/build-python-3.4.sh new file mode 100755 index 00000000..95089f75 --- /dev/null +++ b/python-interpreter-builder/scripts/build-python-3.4.sh @@ -0,0 +1,138 @@ +#!/bin/bash + +set -euo pipefail +set -x + +# Get the source +mkdir -p /opt/sources +cd /opt/sources +wget --no-verbose https://www.python.org/ftp/python/3.4.8/Python-3.4.8.tgz +# SHA-256 generated via `shasum -a 256 [file]` +shasum --check < Date: Tue, 20 Mar 2018 16:55:49 -0700 Subject: [PATCH 160/256] Increase cloudbuild timeout now that we're building Python 3.4 from source. (#183) --- cloudbuild.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 1eabb997..5554015f 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -1,4 +1,4 @@ -timeout: 7200s +timeout: 10800s steps: - # Compile Python interpreters from source name: gcr.io/cloud-builders/docker:latest From f5364134cd9115ebf47f835e81140107f67c0465 Mon Sep 17 00:00:00 2001 From: Douglas Greiman Date: Thu, 22 Mar 2018 15:02:13 -0700 Subject: [PATCH 161/256] Remove jonwayne, add dlorenc to CODEOWNERS (#185) --- CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CODEOWNERS b/CODEOWNERS index a2d4a67f..c08c2d11 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1,4 +1,4 @@ # Code owners file. # This file controls who is tagged for review for any given pull request. -* @duggelz @liyanhui1228 @jonparrott +* @duggelz @liyanhui1228 @dlorenc From dd6fe8f470466cb52490d6f011e9f232744b7975 Mon Sep 17 00:00:00 2001 From: Douglas Greiman Date: Thu, 22 Mar 2018 15:02:30 -0700 Subject: [PATCH 162/256] Add Runtime Builder template file for "staging" tag. (#184) --- builder/python-staging.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 builder/python-staging.yaml diff --git a/builder/python-staging.yaml b/builder/python-staging.yaml new file mode 100644 index 00000000..1e977a89 --- /dev/null +++ b/builder/python-staging.yaml @@ -0,0 +1,12 @@ +# This is a cloudbuild.yaml template for the runtime builder +steps: +- # Generate application Dockerfile + name: 'gcr.io/gcp-runtimes/python/gen-dockerfile:staging' + args: [ + '--base-image=gcr.io/google-appengine/python:staging' + ] +- # Use that Dockerfile to create final application image + name: 'gcr.io/cloud-builders/docker:latest' + args: ['build', '-t', '$_OUTPUT_IMAGE', '.'] +images: + - '$_OUTPUT_IMAGE' From 9b43e372a1495eceea50240dadda565f5744e225 Mon Sep 17 00:00:00 2001 From: Douglas Greiman Date: Tue, 27 Mar 2018 16:32:38 -0700 Subject: [PATCH 163/256] Build and test python interpreters in parallel instead of serially. (#186) --- cloudbuild.yaml | 23 +++++++-- cloudbuild_test.yaml | 51 ++++++++++++++++++- python-interpreter-builder/Dockerfile.in | 14 ----- .../scripts/build-python-3.4.sh | 3 ++ .../scripts/build-python-3.5.sh | 3 ++ .../scripts/build-python-3.6.sh | 3 ++ runtime-image/Dockerfile.in | 4 +- 7 files changed, 80 insertions(+), 21 deletions(-) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 5554015f..51a77480 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -1,20 +1,35 @@ timeout: 10800s steps: -- # Compile Python interpreters from source +- # Compile Python interpreters from source. This step happens first, then + # the next three in parallel. name: gcr.io/cloud-builders/docker:latest args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/interpreter-builder:${_TAG}', '--no-cache', '/workspace/python-interpreter-builder/'] -- # Copy interpreters back to workspace - name: ${_DOCKER_NAMESPACE}/python/interpreter-builder:${_TAG} - args: ['cp', '/interpreters.tar.gz', '/workspace/runtime-image/interpreters.tar.gz'] + id: interpreter-builder +- name: ${_DOCKER_NAMESPACE}/python/interpreter-builder:${_TAG} + args: ['/scripts/build-python-3.4.sh'] + id: build-3.4 + waitFor: ['interpreter-builder'] +- name: ${_DOCKER_NAMESPACE}/python/interpreter-builder:${_TAG} + args: ['/scripts/build-python-3.5.sh'] + id: build-3.5 + waitFor: ['interpreter-builder'] +- name: ${_DOCKER_NAMESPACE}/python/interpreter-builder:${_TAG} + args: ['/scripts/build-python-3.6.sh'] + id: build-3.6 + waitFor: ['interpreter-builder'] - # Build base runtime image name: gcr.io/cloud-builders/docker:latest args: ['build', '--tag=${_DOCKER_NAMESPACE}/python:${_TAG}', '--no-cache', '/workspace/runtime-image/'] + id: runtime + waitFor: ['build-3.4', 'build-3.5', 'build-3.6'] - # Build runtime builder image name: gcr.io/cloud-builders/docker:latest args: ['build', '--tag=${_BUILDER_DOCKER_NAMESPACE}/python/gen-dockerfile:${_TAG}', '--no-cache', '/workspace/builder/gen-dockerfile/'] + id: gen-dockerfile + waitFor: ['runtime'] images: [ '${_DOCKER_NAMESPACE}/python/interpreter-builder:${_TAG}', '${_DOCKER_NAMESPACE}/python:${_TAG}', diff --git a/cloudbuild_test.yaml b/cloudbuild_test.yaml index ef0b48b3..416c5204 100644 --- a/cloudbuild_test.yaml +++ b/cloudbuild_test.yaml @@ -5,18 +5,52 @@ steps: args: [ '/bin/true', ] + id: runtime + - # Validate structure of base runtime image name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 args: [ '-test.v', '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', '/workspace/tests/virtualenv/virtualenv_default.yaml', + ] + waitFor: ['runtime'] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 + args: [ + '-test.v', + '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', '/workspace/tests/virtualenv/virtualenv_python27.yaml', + ] + waitFor: ['runtime'] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 + args: [ + '-test.v', + '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', '/workspace/tests/virtualenv/virtualenv_python34.yaml', + ] + waitFor: ['runtime'] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 + args: [ + '-test.v', + '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', '/workspace/tests/virtualenv/virtualenv_python35.yaml', + ] + waitFor: ['runtime'] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 + args: [ + '-test.v', + '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', '/workspace/tests/virtualenv/virtualenv_python36.yaml', + ] + waitFor: ['runtime'] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 + args: [ + '-test.v', + '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', '/workspace/tests/no-virtualenv/no-virtualenv.yaml', ] + waitFor: ['runtime'] + # Temporarily disabled because it fails on symbolic links in Ubuntu: # https://github.com/GoogleCloudPlatform/container-structure-test/issues/77 #- # Check license compliance @@ -26,35 +60,48 @@ steps: # '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', # '/workspace/tests/license-test/license-test.yaml' # ] -- # Do third-party library compatibility tests +# waitFor: ['runtime'] + +- # Do third-party library compatibility tests for Python 2 name: gcr.io/cloud-builders/docker:latest args: [ 'build', '-t', 'python2-libraries-intermediate', '--build-arg', 'intermediate_image=${_DOCKER_NAMESPACE}/python:${_TAG}', '/workspace/tests/python2-libraries' ] + id: python2-libraries-intermediate + waitFor: ['runtime'] - name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 args: [ '-test.v', '-image', 'python2-libraries-intermediate', '/workspace/tests/python2-libraries/python2-libraries.yaml' ] -- name: gcr.io/cloud-builders/docker:latest + waitFor: ['python2-libraries-intermediate'] + +- # Do third-party library compatibility tests for Python 3 + name: gcr.io/cloud-builders/docker:latest args: [ 'build', '-t', 'python3-libraries-intermediate', '--build-arg', 'intermediate_image=${_DOCKER_NAMESPACE}/python:${_TAG}', '/workspace/tests/python3-libraries' ] + id: python3-libraries-intermediate + waitFor: ['runtime'] - name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 args: [ '-test.v', '-image', 'python3-libraries-intermediate', '/workspace/tests/python3-libraries/python3-libraries.yaml' ] + waitFor: ['python3-libraries-intermediate'] + - # Run other compatibility tests name: gcr.io/cloud-builders/docker:latest args: [ 'build', '--tag=${_DOCKER_NAMESPACE}/python/tests/eventlet:${_TAG}', '--no-cache', '/workspace/tests/eventlet/' ] + waitFor: ['runtime'] + images: [] diff --git a/python-interpreter-builder/Dockerfile.in b/python-interpreter-builder/Dockerfile.in index 46bc092a..9039fbf3 100644 --- a/python-interpreter-builder/Dockerfile.in +++ b/python-interpreter-builder/Dockerfile.in @@ -46,17 +46,3 @@ ENV LANG C.UTF-8 # Add build scripts ADD scripts /scripts ADD DEBIAN /DEBIAN - -# Build the Python interpreters -RUN mkdir -p /opt/packages && \ - echo -n "" > /opt/packages/packages.txt - -RUN /scripts/build-python-3.4.sh - -RUN /scripts/build-python-3.5.sh - -RUN /scripts/build-python-3.6.sh - -# Tar the interpreters. Tarring is needed because docker cp doesn't handle -# links correctly. -RUN tar czf /interpreters.tar.gz /opt/python?.? diff --git a/python-interpreter-builder/scripts/build-python-3.4.sh b/python-interpreter-builder/scripts/build-python-3.4.sh index 95089f75..5c0bfb8e 100755 --- a/python-interpreter-builder/scripts/build-python-3.4.sh +++ b/python-interpreter-builder/scripts/build-python-3.4.sh @@ -136,3 +136,6 @@ find "$PREFIX"/lib/python3.4/test \ cd /opt rm /opt/sources/Python-3.4.8.tgz rm -r /opt/sources/Python-3.4.8 + +# Archive and copy to persistent external volume +tar czf /workspace/runtime-image/interpreter-3.4.tar.gz /opt/python3.4 diff --git a/python-interpreter-builder/scripts/build-python-3.5.sh b/python-interpreter-builder/scripts/build-python-3.5.sh index c2a3337e..86a564c3 100755 --- a/python-interpreter-builder/scripts/build-python-3.5.sh +++ b/python-interpreter-builder/scripts/build-python-3.5.sh @@ -148,3 +148,6 @@ find "$PREFIX"/lib/python3.5/test \ cd /opt rm /opt/sources/Python-3.5.5.tgz rm -r /opt/sources/Python-3.5.5 + +# Archive and copy to persistent external volume +tar czf /workspace/runtime-image/interpreter-3.5.tar.gz /opt/python3.5 diff --git a/python-interpreter-builder/scripts/build-python-3.6.sh b/python-interpreter-builder/scripts/build-python-3.6.sh index a61b2a8b..fc19fc5e 100755 --- a/python-interpreter-builder/scripts/build-python-3.6.sh +++ b/python-interpreter-builder/scripts/build-python-3.6.sh @@ -148,3 +148,6 @@ find "$PREFIX"/lib/python3.6/test \ cd /opt rm /opt/sources/Python-3.6.4.tgz rm -r /opt/sources/Python-3.6.4 + +# Archive and copy to persistent external volume +tar czf /workspace/runtime-image/interpreter-3.6.tar.gz /opt/python3.6 diff --git a/runtime-image/Dockerfile.in b/runtime-image/Dockerfile.in index d0a2e060..1332279c 100644 --- a/runtime-image/Dockerfile.in +++ b/runtime-image/Dockerfile.in @@ -17,7 +17,9 @@ ENV LANG C.UTF-8 ENV PYTHONUNBUFFERED 1 # Install the Google-built interpreters -ADD interpreters.tar.gz / +ADD interpreter-3.4.tar.gz / +ADD interpreter-3.5.tar.gz / +ADD interpreter-3.6.tar.gz / # Add Google-built interpreters to the path ENV PATH /opt/python3.6/bin:/opt/python3.5/bin:/opt/python3.4/bin:$PATH From 8d34b501d5b08d3995e1c45c9e212dfa82387043 Mon Sep 17 00:00:00 2001 From: Douglas Greiman Date: Thu, 29 Mar 2018 14:04:02 -0700 Subject: [PATCH 164/256] Restore python3 and pip3 $PATH entries. (#189) They now invoke Python 3.6. Previously they invoked Python 3.4. --- runtime-image/Dockerfile.in | 2 ++ tests/no-virtualenv/no-virtualenv.yaml | 11 +++++++++++ 2 files changed, 13 insertions(+) diff --git a/runtime-image/Dockerfile.in b/runtime-image/Dockerfile.in index 1332279c..f0040cff 100644 --- a/runtime-image/Dockerfile.in +++ b/runtime-image/Dockerfile.in @@ -23,6 +23,8 @@ ADD interpreter-3.6.tar.gz / # Add Google-built interpreters to the path ENV PATH /opt/python3.6/bin:/opt/python3.5/bin:/opt/python3.4/bin:$PATH +RUN update-alternatives --install /usr/local/bin/python3 python3 /opt/python3.6/bin/python3.6 50 +RUN update-alternatives --install /usr/local/bin/pip3 pip3 /opt/python3.6/bin/pip3.6 50 # Upgrade pip (debian package version tends to run a few version behind) and # install virtualenv system-wide. diff --git a/tests/no-virtualenv/no-virtualenv.yaml b/tests/no-virtualenv/no-virtualenv.yaml index cf997e29..1015b0da 100644 --- a/tests/no-virtualenv/no-virtualenv.yaml +++ b/tests/no-virtualenv/no-virtualenv.yaml @@ -37,6 +37,17 @@ commandTests: command: ["which", "gunicorn"] expectedOutput: ["/usr/local/bin/gunicorn\n"] + - # Regression test for issue187 + name: "default python3 installation" + command: ["which", "python3"] + expectedOutput: ["/usr/local/bin/python3\n"] + - name: "default python3 version" + command: ["python3", "--version"] + expectedOutput: ["Python 3.6.4\n"] + - name: "default pip3 installation" + command: ["which", "pip3"] + expectedOutput: ["/usr/local/bin/pip3\n"] + - name: "default flask installation" # Checks that 'pip' and 'python' are using the same Python version setup: [["pip", "install", "flask"]] From bae048399184724052ee344ec6de9553e13fbb6b Mon Sep 17 00:00:00 2001 From: dlorenc Date: Tue, 3 Apr 2018 08:38:37 -0700 Subject: [PATCH 165/256] Update to Python 3.6.5. --- .../scripts/build-python-3.6.sh | 12 ++++++------ tests/no-virtualenv/no-virtualenv.yaml | 2 +- tests/virtualenv/virtualenv_python36.yaml | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/python-interpreter-builder/scripts/build-python-3.6.sh b/python-interpreter-builder/scripts/build-python-3.6.sh index fc19fc5e..63a09d2f 100755 --- a/python-interpreter-builder/scripts/build-python-3.6.sh +++ b/python-interpreter-builder/scripts/build-python-3.6.sh @@ -6,14 +6,14 @@ set -x # Get the source mkdir -p /opt/sources cd /opt/sources -wget --no-verbose https://www.python.org/ftp/python/3.6.4/Python-3.6.4.tgz +wget --no-verbose https://www.python.org/ftp/python/3.6.5/Python-3.6.5.tgz # SHA-256 generated via `shasum -a 256 [file]` shasum --check < Date: Wed, 4 Apr 2018 23:39:32 -0700 Subject: [PATCH 166/256] Remove duplicate pip executables to avoid confusion. (#191) --- runtime-image/Dockerfile.in | 3 +++ 1 file changed, 3 insertions(+) diff --git a/runtime-image/Dockerfile.in b/runtime-image/Dockerfile.in index f0040cff..b6c76d71 100644 --- a/runtime-image/Dockerfile.in +++ b/runtime-image/Dockerfile.in @@ -30,8 +30,11 @@ RUN update-alternatives --install /usr/local/bin/pip3 pip3 /opt/python3.6/bin/pi # install virtualenv system-wide. RUN /usr/bin/pip install --upgrade -r /resources/requirements.txt && \ /opt/python3.4/bin/pip3.4 install --upgrade -r /resources/requirements.txt && \ + rm -f /opt/python3.4/bin/pip /opt/python3.4/bin/pip3 && \ /opt/python3.5/bin/pip3.5 install --upgrade -r /resources/requirements.txt && \ + rm -f /opt/python3.5/bin/pip /opt/python3.5/bin/pip3 && \ /opt/python3.6/bin/pip3.6 install --upgrade -r /resources/requirements.txt && \ + rm -f /opt/python3.6/bin/pip /opt/python3.6/bin/pip3 && \ /usr/bin/pip install --upgrade -r /resources/requirements-virtualenv.txt # Setup the app working directory From a8a3e8b2d3239c184843db818e34a06f12dc1190 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Thu, 5 Apr 2018 14:09:34 -0700 Subject: [PATCH 167/256] Auto-update dependencies. (#188) --- .../resources/requirements-virtualenv.txt | 2 +- runtime-image/resources/requirements.txt | 6 +- scripts/requirements-test.txt | 2 +- tests/python2-libraries/requirements.txt | 72 +++++++++---------- tests/python3-libraries/requirements.txt | 72 +++++++++---------- 5 files changed, 77 insertions(+), 77 deletions(-) diff --git a/runtime-image/resources/requirements-virtualenv.txt b/runtime-image/resources/requirements-virtualenv.txt index a4ce32e5..fa784bd2 100644 --- a/runtime-image/resources/requirements-virtualenv.txt +++ b/runtime-image/resources/requirements-virtualenv.txt @@ -1 +1 @@ -virtualenv==15.1.0 +virtualenv==15.2.0 diff --git a/runtime-image/resources/requirements.txt b/runtime-image/resources/requirements.txt index ef675285..c3f99d24 100644 --- a/runtime-image/resources/requirements.txt +++ b/runtime-image/resources/requirements.txt @@ -1,3 +1,3 @@ -pip==9.0.1 -setuptools==38.5.2 -wheel==0.30.0 +pip==9.0.3 +setuptools==39.0.1 +wheel==0.31.0 diff --git a/scripts/requirements-test.txt b/scripts/requirements-test.txt index 1ae0abd8..8be25b0f 100644 --- a/scripts/requirements-test.txt +++ b/scripts/requirements-test.txt @@ -1,4 +1,4 @@ flask==0.12.2 -pytest==3.4.2 +pytest==3.5.0 pytest-cov==2.5.1 pyyaml==3.12 diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index f261209e..c9c61dec 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -1,12 +1,12 @@ -alembic==0.9.8 +alembic==0.9.9 amqp==2.2.2 amqplib==1.0.2 -ansible==2.4.3.0 +ansible==2.5.0 anyjson==0.3.3 apache-libcloud==2.3.0 argparse==1.4.0 -astroid==1.6.1 -awscli==1.14.53 +astroid==1.6.2 +awscli==1.14.68 babel==2.5.3 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 @@ -16,7 +16,7 @@ billiard==3.5.0.3 blessings==1.6.1 blinker==1.4 boto==2.48.0 -botocore==1.9.6 +botocore==1.9.21 bottle==0.12.13 carbon<1.1.1 celery==4.1.0 @@ -25,20 +25,20 @@ cffi==1.11.5 chardet==3.0.4 click==6.7 cliff==2.11.0 -cmd2==0.8.1 +cmd2==0.8.2 colorama==0.3.9 configobj==5.0.6 cov-core==1.15.0 coverage==4.5.1 coveralls==1.3.0 crcmod==1.7 -cryptography==2.1.4 +cryptography==2.2.2 cssselect==1.0.3 -cython==0.27.3 +cython==0.28.1 decorator==4.2.1 django-celery==3.2.2 django-debug-toolbar==1.9.1 -django-extensions==2.0.5 +django-extensions==2.0.6 django<2.0 django_compress==1.0.1 djangorestframework==3.7.7 @@ -46,7 +46,7 @@ docker-py==1.10.6 docopt==0.6.2 docutils==0.14 ecdsa==0.13 -elasticsearch==6.1.1 +elasticsearch==6.2.0 enum34==1.1.6 eventlet==0.22.1 extras==1.0.0 @@ -58,14 +58,14 @@ funcsigs==1.0.2 functools32==3.2.3.post2 futures==3.2.0 gevent==1.2.2 -google-api-python-client==1.6.5 +google-api-python-client==1.6.6 graphite-web==1.1.2 greenlet==0.4.13 gunicorn==19.7.1 hiredis==0.2.0 honcho==1.0.1 html5lib==1.0.1 -httplib2==0.10.3 +httplib2==0.11.3 idna==2.6 ipaddress==1.0.19 iso8601==0.1.12 @@ -77,13 +77,13 @@ jsonschema==2.6.0 kombu==4.1.0 linecache2==1.0.0 logilab-common==1.4.1 -lxml==4.1.1 +lxml==4.2.1 m2crypto==0.29.0 mako==1.0.7 manifestparser==1.1 markdown==2.6.11 markupsafe==1.0 -matplotlib==2.2.0 +matplotlib==2.2.2 mccabe==0.6.1 meld3==1.0.2 mock==2.0.0 @@ -101,31 +101,31 @@ mysql-python==1.2.5 ndg-httpsclient==0.4.4 netaddr==0.7.19 netifaces==0.10.6 -newrelic==2.106.1.88 +newrelic==3.0.0.89 nose==1.3.7 -numpy==1.14.1 +numpy==1.14.2 oauth2==1.9.0.post1 oauth2client==4.1.2 -oauthlib==2.0.6 +oauthlib==2.0.7 ordereddict==1.1 -oslo.config==5.2.0 +oslo.config==6.0.0 pandas==0.22.0 -paramiko==2.4.0 +paramiko==2.4.1 passlib==1.7.1 paste==2.0.3 pastedeploy==1.5.2 pastescript==2.0.2 -pbr==3.1.1 +pbr==4.0.0 pep8==1.7.1 pexpect==4.4.0 pika==0.11.2 pillow==5.0.0 -pip==9.0.1 +pip==9.0.3 prettytable -protobuf==3.5.2 +protobuf==3.5.2.post1 psutil==5.4.3 psycopg2==2.7.4 -py==1.5.2 +py==1.5.3 pyasn1-modules==0.2.1 pyasn1==0.4.2 pycparser==2.18 @@ -133,9 +133,9 @@ pycrypto==2.6.1 pycurl==7.43.0.1 pyflakes==1.6.0 pygments==2.2.0 -pyjwt==1.6.0 +pyjwt==1.6.1 pylibmc==1.5.2 -pylint==1.8.2 +pylint==1.8.3 pymongo==3.6.1 pymysql==0.8.0 pyopenssl==17.5.0 @@ -143,10 +143,10 @@ pyparsing==2.2.0 pyramid==1.9.1 pystache==0.5.4 pytest-cov==2.5.1 -pytest==3.4.2 +pytest==3.5.0 python-cjson==1.2.1 python-daemon==2.1.2 -python-dateutil==2.7.0 +python-dateutil==2.7.2 python-gflags==3.1.2 python-keystoneclient==3.15.0 python-memcached==1.59 @@ -164,18 +164,18 @@ requests-oauthlib==0.8.0 requests==2.18.4 retrying==1.3.3 rsa==3.4.2 -scipy==1.0.0 -selenium==3.10.0 +scipy==1.0.1 +selenium==3.11.0 setuptools-git==1.2 -setuptools==38.5.2 +setuptools==39.0.1 sh==1.12.14 simplejson==3.13.2 six==1.11.0 snowballstemmer==1.2.1 south==1.0.2 -sphinx==1.7.1 +sphinx==1.7.2 sqlalchemy-migrate==0.11.0 -sqlalchemy==1.2.5 +sqlalchemy==1.2.6 sqlparse==0.2.4 statsd==3.2.2 stevedore==1.28.0 @@ -184,7 +184,7 @@ supervisor==3.3.4 testrepository==0.0.20 testtools==2.3.0 thrift==0.11.0 -tornado==5.0 +tornado==5.0.1 tox==2.9.1 twisted==17.9.0 ujson==1.35 @@ -194,14 +194,14 @@ uritemplate==3.0.0 urllib3==1.22 uwsgi==2.0.17 versiontools==1.9.1 -virtualenv==15.1.0 +virtualenv==15.2.0 waitress==1.1.0 warlock==1.3.0 webob==1.7.4 websocket-client==0.47.0 webtest==2.0.29 werkzeug==0.14.1 -wheel==0.30.0 +wheel==0.31.0 xlrd==1.1.0 -zc.buildout==2.11.1 +zc.buildout==2.11.2 zope.interface==4.4.3 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 30fb236e..855b0906 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -1,12 +1,12 @@ -alembic==0.9.8 +alembic==0.9.9 amqp==2.2.2 amqplib==1.0.2 -ansible==2.4.3.0 +ansible==2.5.0 anyjson==0.3.3 apache-libcloud==2.3.0 argparse==1.4.0 -astroid==1.6.1 -awscli==1.14.53 +astroid==1.6.2 +awscli==1.14.68 babel==2.5.3 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 @@ -15,7 +15,7 @@ billiard==3.5.0.3 blessings==1.6.1 blinker==1.4 boto==2.48.0 -botocore==1.9.6 +botocore==1.9.21 bottle==0.12.13 celery==4.1.0 certifi==2018.1.18 @@ -23,20 +23,20 @@ cffi==1.11.5 chardet==3.0.4 click==6.7 cliff==2.11.0 -cmd2==0.8.1 +cmd2==0.8.2 colorama==0.3.9 configobj==5.0.6 cov-core==1.15.0 coverage==4.5.1 coveralls==1.3.0 crcmod==1.7 -cryptography==2.1.4 +cryptography==2.2.2 cssselect==1.0.3 -cython==0.27.3 +cython==0.28.1 decorator==4.2.1 django-celery==3.2.2 django-debug-toolbar==1.9.1 -django-extensions==2.0.5 +django-extensions==2.0.6 django==2.0.3 django_compress==1.0.1 djangorestframework==3.7.7 @@ -44,7 +44,7 @@ docker-py==1.10.6 docopt==0.6.2 docutils==0.14 ecdsa==0.13 -elasticsearch==6.1.1 +elasticsearch==6.2.0 enum34==1.1.6 eventlet==0.22.1 extras==1.0.0 @@ -54,13 +54,13 @@ flake8==3.5.0 flask==0.12.2 funcsigs==1.0.2 gevent==1.2.2 -google-api-python-client==1.6.5 +google-api-python-client==1.6.6 greenlet==0.4.13 gunicorn==19.7.1 hiredis==0.2.0 honcho==1.0.1 html5lib==1.0.1 -httplib2==0.10.3 +httplib2==0.11.3 idna==2.6 ipaddress==1.0.19 ipython==6.2.1 @@ -73,12 +73,12 @@ jsonschema==2.6.0 kombu==4.1.0 linecache2==1.0.0 logilab-common==1.4.1 -lxml==4.1.1 +lxml==4.2.1 mako==1.0.7 manifestparser==1.1 markdown==2.6.11 markupsafe==1.0 -matplotlib==2.2.0 +matplotlib==2.2.2 mccabe==0.6.1 meld3==1.0.2 mock==2.0.0 @@ -93,40 +93,40 @@ msgpack-python==0.5.6 ndg-httpsclient==0.4.4 netaddr==0.7.19 netifaces==0.10.6 -newrelic==2.106.1.88 +newrelic==3.0.0.89 nose==1.3.7 -numpy==1.14.1 +numpy==1.14.2 oauth2==1.9.0.post1 oauth2client==4.1.2 -oauthlib==2.0.6 +oauthlib==2.0.7 ordereddict==1.1 -oslo.config==5.2.0 +oslo.config==6.0.0 pandas==0.22.0 -paramiko==2.4.0 +paramiko==2.4.1 passlib==1.7.1 paste==2.0.3 pastedeploy==1.5.2 pastescript==2.0.2 -pbr==3.1.1 +pbr==4.0.0 pep8==1.7.1 pexpect==4.4.0 pika==0.11.2 pillow==5.0.0 -pip==9.0.1 +pip==9.0.3 prettytable -protobuf==3.5.2 +protobuf==3.5.2.post1 psutil==5.4.3 psycopg2==2.7.4 -py==1.5.2 +py==1.5.3 pyasn1-modules==0.2.1 pyasn1==0.4.2 pycparser==2.18 pycrypto==2.6.1 pyflakes==1.6.0 pygments==2.2.0 -pyjwt==1.6.0 +pyjwt==1.6.1 pylibmc==1.5.2 -pylint==1.8.2 +pylint==1.8.3 pymongo==3.6.1 pymysql==0.8.0 pyopenssl==17.5.0 @@ -134,9 +134,9 @@ pyparsing==2.2.0 pyramid==1.9.1 pystache==0.5.4 pytest-cov==2.5.1 -pytest==3.4.2 +pytest==3.5.0 python-daemon==2.1.2 -python-dateutil==2.7.0 +python-dateutil==2.7.2 python-gflags==3.1.2 python-keystoneclient==3.15.0 python-memcached==1.59 @@ -154,25 +154,25 @@ requests-oauthlib==0.8.0 requests==2.18.4 retrying==1.3.3 rsa==3.4.2 -scipy==1.0.0 -selenium==3.10.0 +scipy==1.0.1 +selenium==3.11.0 setuptools-git==1.2 -setuptools==38.5.2 +setuptools==39.0.1 sh==1.12.14 simplejson==3.13.2 six==1.11.0 snowballstemmer==1.2.1 south==1.0.2 -sphinx==1.7.1 +sphinx==1.7.2 sqlalchemy-migrate==0.11.0 -sqlalchemy==1.2.5 +sqlalchemy==1.2.6 sqlparse==0.2.4 statsd==3.2.2 stevedore==1.28.0 testrepository==0.0.20 testtools==2.3.0 thrift==0.11.0 -tornado==5.0 +tornado==5.0.1 tox==2.9.1 twisted==17.9.0 ujson==1.35 @@ -182,14 +182,14 @@ uritemplate==3.0.0 urllib3==1.22 uwsgi==2.0.17 versiontools==1.9.1 -virtualenv==15.1.0 +virtualenv==15.2.0 waitress==1.1.0 warlock==1.3.0 webob==1.7.4 websocket-client==0.47.0 webtest==2.0.29 werkzeug==0.14.1 -wheel==0.30.0 +wheel==0.31.0 xlrd==1.1.0 -zc.buildout==2.11.1 +zc.buildout==2.11.2 zope.interface==4.4.3 From 11cc1eba0c3a8d44d5c90fe77a2546358aa3794a Mon Sep 17 00:00:00 2001 From: dlorenc Date: Thu, 17 May 2018 11:07:44 -0700 Subject: [PATCH 168/256] Split the runtime release into an interpreter and a runtime build. --- RELEASING.md | 11 +++++++++++ build.sh | 13 +++++++++++++ cloudbuild.yaml | 20 -------------------- cloudbuild_interpreters.yaml | 29 +++++++++++++++++++++++++++++ runtime-image/Dockerfile.in | 11 +++++++---- 5 files changed, 60 insertions(+), 24 deletions(-) create mode 100644 cloudbuild_interpreters.yaml diff --git a/RELEASING.md b/RELEASING.md index 00b0fe16..9f61130a 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -39,6 +39,17 @@ follows: 4. `build.sh` invokes Google Container Builder with the `cloudbuild-*.yaml` config files. +## Building interpreters + +The interpreters used are now built in a separate step, and stored on GCS. +This allows the runtime images to be build more rapidly. + +To build the interpreters, run: + +```shell +gcloud container builds submit . --config=cloudbuild_interpreters.yaml +``` + ## Building outside Jenkins To build this repository outside Jenkins, authenticate and authorize yourself diff --git a/build.sh b/build.sh index 9b3a258d..badda7b5 100755 --- a/build.sh +++ b/build.sh @@ -25,6 +25,7 @@ test=0 # Should run standard test suite? local=0 # Should run using local Docker daemon instead of GCR? os_base=debian8 # Which operating system base to use +interpreter=0 # Should build interpreters instead of images # Note that $gcloud_cmd has spaces in it gcloud_cmd="gcloud container builds submit" @@ -126,6 +127,10 @@ while [ $# -gt 0 ]; do test=0 shift ;; + --interpreter) + interpreter=1 + shift + ;; *) usage ;; @@ -184,6 +189,14 @@ done # Make a file available to the eventlet test. cp -a scripts/testdata/hello_world/main.py tests/eventlet/main.py +# Build interpreters and push to GCS +if [ "${interpreter}" -eq 1 ]; then + echo "Building interpreters" + ${gcloud_cmd} \ + --config=cloudbuild_interpreters.yaml \ + . +fi + # Build images and push to GCR if [ "${build}" -eq 1 ]; then echo "Building images" diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 51a77480..e240780e 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -1,29 +1,10 @@ timeout: 10800s steps: -- # Compile Python interpreters from source. This step happens first, then - # the next three in parallel. - name: gcr.io/cloud-builders/docker:latest - args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/interpreter-builder:${_TAG}', - '--no-cache', '/workspace/python-interpreter-builder/'] - id: interpreter-builder -- name: ${_DOCKER_NAMESPACE}/python/interpreter-builder:${_TAG} - args: ['/scripts/build-python-3.4.sh'] - id: build-3.4 - waitFor: ['interpreter-builder'] -- name: ${_DOCKER_NAMESPACE}/python/interpreter-builder:${_TAG} - args: ['/scripts/build-python-3.5.sh'] - id: build-3.5 - waitFor: ['interpreter-builder'] -- name: ${_DOCKER_NAMESPACE}/python/interpreter-builder:${_TAG} - args: ['/scripts/build-python-3.6.sh'] - id: build-3.6 - waitFor: ['interpreter-builder'] - # Build base runtime image name: gcr.io/cloud-builders/docker:latest args: ['build', '--tag=${_DOCKER_NAMESPACE}/python:${_TAG}', '--no-cache', '/workspace/runtime-image/'] id: runtime - waitFor: ['build-3.4', 'build-3.5', 'build-3.6'] - # Build runtime builder image name: gcr.io/cloud-builders/docker:latest args: ['build', '--tag=${_BUILDER_DOCKER_NAMESPACE}/python/gen-dockerfile:${_TAG}', @@ -31,7 +12,6 @@ steps: id: gen-dockerfile waitFor: ['runtime'] images: [ - '${_DOCKER_NAMESPACE}/python/interpreter-builder:${_TAG}', '${_DOCKER_NAMESPACE}/python:${_TAG}', '${_BUILDER_DOCKER_NAMESPACE}/python/gen-dockerfile:${_TAG}', ] diff --git a/cloudbuild_interpreters.yaml b/cloudbuild_interpreters.yaml new file mode 100644 index 00000000..7daecc57 --- /dev/null +++ b/cloudbuild_interpreters.yaml @@ -0,0 +1,29 @@ +timeout: 10800s +steps: +- # Compile Python interpreters from source. This step happens first, then + # the next three in parallel. + name: gcr.io/cloud-builders/docker:latest + args: ['build', '--tag=interpreter-builder', + '--no-cache', '/workspace/python-interpreter-builder/'] + id: interpreter-builder +- name: interpreter-builder + args: ['/scripts/build-python-3.4.sh'] + id: build-3.4 + waitFor: ['interpreter-builder'] +- name: interpreter-builder + args: ['/scripts/build-python-3.5.sh'] + id: build-3.5 + waitFor: ['interpreter-builder'] +- name: interpreter-builder + args: ['/scripts/build-python-3.6.sh'] + id: build-3.6 + waitFor: ['interpreter-builder'] + +# Upload them to tbe build-id location +- name: gcr.io/cloud-builders/gsutil:latest + args: ['cp', '/workspace/runtime-image/*.tar.gz', 'gs://python-interpreters/$BUILD_ID/'] + waitFor: ['build-3.4', 'build-3.5', 'build-3.6'] + +# "Tag" this as latest +- name: gcr.io/cloud-builders/gsutil:latest + args: ['cp', '-r', 'gs://python-interpreters/$BUILD_ID/*', 'gs://python-interpreters/latest/'] diff --git a/runtime-image/Dockerfile.in b/runtime-image/Dockerfile.in index b6c76d71..f291ae7f 100644 --- a/runtime-image/Dockerfile.in +++ b/runtime-image/Dockerfile.in @@ -16,10 +16,13 @@ ENV LANG C.UTF-8 # logging collection. ENV PYTHONUNBUFFERED 1 -# Install the Google-built interpreters -ADD interpreter-3.4.tar.gz / -ADD interpreter-3.5.tar.gz / -ADD interpreter-3.6.tar.gz / +RUN wget https://storage.googleapis.com/python-interpreters/latest/interpreter-3.4.tar.gz && \ + wget https://storage.googleapis.com/python-interpreters/latest/interpreter-3.5.tar.gz && \ + wget https://storage.googleapis.com/python-interpreters/latest/interpreter-3.6.tar.gz && \ + tar -xzf interpreter-3.4.tar.gz && \ + tar -xzf interpreter-3.5.tar.gz && \ + tar -xzf interpreter-3.6.tar.gz && \ + rm interpreter-*.tar.gz # Add Google-built interpreters to the path ENV PATH /opt/python3.6/bin:/opt/python3.5/bin:/opt/python3.4/bin:$PATH From 7a1444fead32ad3d9ae9e1d34782f034cd551ce6 Mon Sep 17 00:00:00 2001 From: David Bendory Date: Tue, 17 Jul 2018 18:32:44 -0400 Subject: [PATCH 169/256] container-builder-local has been renamed cloud-build-local. --- build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sh b/build.sh index badda7b5..476917c4 100755 --- a/build.sh +++ b/build.sh @@ -29,8 +29,8 @@ interpreter=0 # Should build interpreters instead of images # Note that $gcloud_cmd has spaces in it gcloud_cmd="gcloud container builds submit" -# May need to install via "gcloud components install container-builder-local" -local_gcloud_cmd="container-builder-local --push=false --dryrun=false" +# May need to install via "gcloud components install cloud-build-local" +local_gcloud_cmd="cloud-build-local --push=false --dryrun=false" # Helper functions function fatal() { From ddfbf2215ce2d315119a27944f8cf41c62836908 Mon Sep 17 00:00:00 2001 From: David Bendory Date: Tue, 17 Jul 2018 18:37:20 -0400 Subject: [PATCH 170/256] `gcloud builds` replaces `gcloud container builds`. --- RELEASING.md | 2 +- build.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/RELEASING.md b/RELEASING.md index 9f61130a..2dcd806d 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -47,7 +47,7 @@ This allows the runtime images to be build more rapidly. To build the interpreters, run: ```shell -gcloud container builds submit . --config=cloudbuild_interpreters.yaml +gcloud builds submit . --config=cloudbuild_interpreters.yaml ``` ## Building outside Jenkins diff --git a/build.sh b/build.sh index 476917c4..bdc4f658 100755 --- a/build.sh +++ b/build.sh @@ -28,7 +28,7 @@ os_base=debian8 # Which operating system base to use interpreter=0 # Should build interpreters instead of images # Note that $gcloud_cmd has spaces in it -gcloud_cmd="gcloud container builds submit" +gcloud_cmd="gcloud builds submit" # May need to install via "gcloud components install cloud-build-local" local_gcloud_cmd="cloud-build-local --push=false --dryrun=false" From 20e146f027aa137fd061c192721be28902a5900a Mon Sep 17 00:00:00 2001 From: sharifelgamal Date: Tue, 31 Jul 2018 16:28:41 -0700 Subject: [PATCH 171/256] bumping cryptography to 2.3 for CVE --- tests/python2-libraries/requirements.txt | 2 +- tests/python3-libraries/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index c9c61dec..1655846a 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -32,7 +32,7 @@ cov-core==1.15.0 coverage==4.5.1 coveralls==1.3.0 crcmod==1.7 -cryptography==2.2.2 +cryptography==2.3 cssselect==1.0.3 cython==0.28.1 decorator==4.2.1 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 855b0906..e8295d80 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -30,7 +30,7 @@ cov-core==1.15.0 coverage==4.5.1 coveralls==1.3.0 crcmod==1.7 -cryptography==2.2.2 +cryptography==2.3 cssselect==1.0.3 cython==0.28.1 decorator==4.2.1 From e0b754bf62e2087c5bbad077b02a9860eb132cb2 Mon Sep 17 00:00:00 2001 From: dlorenc Date: Fri, 10 Aug 2018 09:28:34 -0700 Subject: [PATCH 172/256] Bump Python 3.6. --- .../scripts/build-python-3.6.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/python-interpreter-builder/scripts/build-python-3.6.sh b/python-interpreter-builder/scripts/build-python-3.6.sh index 63a09d2f..ef192e1e 100755 --- a/python-interpreter-builder/scripts/build-python-3.6.sh +++ b/python-interpreter-builder/scripts/build-python-3.6.sh @@ -6,21 +6,21 @@ set -x # Get the source mkdir -p /opt/sources cd /opt/sources -wget --no-verbose https://www.python.org/ftp/python/3.6.5/Python-3.6.5.tgz +wget --no-verbose https://www.python.org/ftp/python/3.6.6/Python-3.6.6.tgz # SHA-256 generated via `shasum -a 256 [file]` shasum --check < Date: Fri, 10 Aug 2018 09:51:01 -0700 Subject: [PATCH 173/256] Bump the test value to 3.6.6. --- tests/no-virtualenv/no-virtualenv.yaml | 2 +- tests/virtualenv/virtualenv_python36.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/no-virtualenv/no-virtualenv.yaml b/tests/no-virtualenv/no-virtualenv.yaml index 2d6e5052..869b93b2 100644 --- a/tests/no-virtualenv/no-virtualenv.yaml +++ b/tests/no-virtualenv/no-virtualenv.yaml @@ -43,7 +43,7 @@ commandTests: expectedOutput: ["/usr/local/bin/python3\n"] - name: "default python3 version" command: ["python3", "--version"] - expectedOutput: ["Python 3.6.5\n"] + expectedOutput: ["Python 3.6.6\n"] - name: "default pip3 installation" command: ["which", "pip3"] expectedOutput: ["/usr/local/bin/pip3\n"] diff --git a/tests/virtualenv/virtualenv_python36.yaml b/tests/virtualenv/virtualenv_python36.yaml index c8578cca..d4de875c 100644 --- a/tests/virtualenv/virtualenv_python36.yaml +++ b/tests/virtualenv/virtualenv_python36.yaml @@ -25,7 +25,7 @@ commandTests: - name: "virtualenv36 python version" setup: [["virtualenv", "-p", "python3.6", "/env"]] command: ["python", "--version"] - expectedOutput: ["Python 3.6.5\n"] + expectedOutput: ["Python 3.6.6\n"] - name: "virtualenv36 pip installation" setup: [["virtualenv", "-p", "python3.6", "/env"]] From b032725790a0109b17d50caaefd96e8953154416 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Mon, 13 Aug 2018 00:51:19 -0700 Subject: [PATCH 174/256] Auto-update dependencies. --- builder/gen-dockerfile/requirements.txt | 2 +- .../resources/requirements-virtualenv.txt | 2 +- runtime-image/resources/requirements.txt | 6 +- scripts/requirements-test.txt | 6 +- scripts/testdata/hello_world/requirements.txt | 4 +- tests/deploy_check/requirements.txt | 4 +- tests/eventlet/requirements.txt | 8 +- tests/integration/requirements.txt | 10 +- tests/python2-libraries/requirements.txt | 194 +++++++++--------- tests/python3-libraries/requirements.txt | 188 ++++++++--------- 10 files changed, 212 insertions(+), 212 deletions(-) diff --git a/builder/gen-dockerfile/requirements.txt b/builder/gen-dockerfile/requirements.txt index 59bc593a..4a285555 100644 --- a/builder/gen-dockerfile/requirements.txt +++ b/builder/gen-dockerfile/requirements.txt @@ -1 +1 @@ -PyYAML==3.12 +PyYAML==3.13 diff --git a/runtime-image/resources/requirements-virtualenv.txt b/runtime-image/resources/requirements-virtualenv.txt index fa784bd2..52bb7da1 100644 --- a/runtime-image/resources/requirements-virtualenv.txt +++ b/runtime-image/resources/requirements-virtualenv.txt @@ -1 +1 @@ -virtualenv==15.2.0 +virtualenv==16.0.0 diff --git a/runtime-image/resources/requirements.txt b/runtime-image/resources/requirements.txt index c3f99d24..44c624d8 100644 --- a/runtime-image/resources/requirements.txt +++ b/runtime-image/resources/requirements.txt @@ -1,3 +1,3 @@ -pip==9.0.3 -setuptools==39.0.1 -wheel==0.31.0 +pip==18.0 +setuptools==40.0.0 +wheel==0.31.1 diff --git a/scripts/requirements-test.txt b/scripts/requirements-test.txt index 8be25b0f..df909d47 100644 --- a/scripts/requirements-test.txt +++ b/scripts/requirements-test.txt @@ -1,4 +1,4 @@ -flask==0.12.2 -pytest==3.5.0 +flask==1.0.2 +pytest==3.7.1 pytest-cov==2.5.1 -pyyaml==3.12 +pyyaml==3.13 diff --git a/scripts/testdata/hello_world/requirements.txt b/scripts/testdata/hello_world/requirements.txt index bb84e50e..a34d076b 100644 --- a/scripts/testdata/hello_world/requirements.txt +++ b/scripts/testdata/hello_world/requirements.txt @@ -1,2 +1,2 @@ -Flask==0.12.2 -gunicorn==19.7.1 +Flask==1.0.2 +gunicorn==19.9.0 diff --git a/tests/deploy_check/requirements.txt b/tests/deploy_check/requirements.txt index bb84e50e..a34d076b 100644 --- a/tests/deploy_check/requirements.txt +++ b/tests/deploy_check/requirements.txt @@ -1,2 +1,2 @@ -Flask==0.12.2 -gunicorn==19.7.1 +Flask==1.0.2 +gunicorn==19.9.0 diff --git a/tests/eventlet/requirements.txt b/tests/eventlet/requirements.txt index 5d104d0a..7f8b2468 100644 --- a/tests/eventlet/requirements.txt +++ b/tests/eventlet/requirements.txt @@ -1,9 +1,9 @@ click==6.7 enum-compat==0.0.2 -eventlet==0.22.1 -Flask==0.12.2 -greenlet==0.4.13 -gunicorn==19.7.1 +eventlet==0.24.1 +Flask==1.0.2 +greenlet==0.4.14 +gunicorn==19.9.0 itsdangerous==0.24 Jinja2==2.10 MarkupSafe==1.0 diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index 64e39e0c..4a462433 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -1,8 +1,8 @@ -Flask==0.12.2 -google-cloud-error-reporting==0.29.1 +Flask==1.0.2 +google-cloud-error-reporting==0.30.0 google-cloud-logging==1.6.0 -google-cloud-monitoring==0.28.1 -gunicorn==19.7.1 -requests==2.18.4 +google-cloud-monitoring==0.30.1 +gunicorn==19.9.0 +requests==2.19.1 retrying==1.3.3 six==1.11.0 diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 1655846a..6daad0ab 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -1,31 +1,31 @@ -alembic==0.9.9 -amqp==2.2.2 +alembic==1.0.0 +amqp==2.3.2 amqplib==1.0.2 -ansible==2.5.0 +ansible==2.6.2 anyjson==0.3.3 apache-libcloud==2.3.0 argparse==1.4.0 -astroid==1.6.2 -awscli==1.14.68 -babel==2.5.3 +astroid==2.0.4 +awscli==1.15.76 +babel==2.6.0 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 -beautifulsoup4==4.6.0 +beautifulsoup4==4.6.3 beautifulsoup==3.2.1 -billiard==3.5.0.3 -blessings==1.6.1 +billiard==3.5.0.4 +blessings==1.7 blinker==1.4 -boto==2.48.0 -botocore==1.9.21 +boto==2.49.0 +botocore==1.10.75 bottle==0.12.13 carbon<1.1.1 -celery==4.1.0 -certifi==2018.1.18 +celery==4.2.1 +certifi==2018.8.13 cffi==1.11.5 chardet==3.0.4 click==6.7 -cliff==2.11.0 -cmd2==0.8.2 +cliff==2.13.0 +cmd2==0.9.3 colorama==0.3.9 configobj==5.0.6 cov-core==1.15.0 @@ -34,174 +34,174 @@ coveralls==1.3.0 crcmod==1.7 cryptography==2.3 cssselect==1.0.3 -cython==0.28.1 -decorator==4.2.1 +cython==0.28.5 +decorator==4.3.0 django-celery==3.2.2 django-debug-toolbar==1.9.1 -django-extensions==2.0.6 +django-extensions==2.1.0 django<2.0 django_compress==1.0.1 -djangorestframework==3.7.7 +djangorestframework==3.8.2 docker-py==1.10.6 docopt==0.6.2 docutils==0.14 ecdsa==0.13 -elasticsearch==6.2.0 +elasticsearch==6.3.1 enum34==1.1.6 -eventlet==0.22.1 +eventlet==0.24.1 extras==1.0.0 -fabric==1.14.0 +fabric==2.3.1 fixtures==3.0.0 flake8==3.5.0 -flask==0.12.2 +flask==1.0.2 funcsigs==1.0.2 functools32==3.2.3.post2 futures==3.2.0 -gevent==1.2.2 -google-api-python-client==1.6.6 -graphite-web==1.1.2 -greenlet==0.4.13 -gunicorn==19.7.1 +gevent==1.3.5 +google-api-python-client==1.7.4 +graphite-web==1.1.3 +greenlet==0.4.14 +gunicorn==19.9.0 hiredis==0.2.0 honcho==1.0.1 html5lib==1.0.1 httplib2==0.11.3 -idna==2.6 -ipaddress==1.0.19 +idna==2.7 +ipaddress==1.0.22 iso8601==0.1.12 isodate==0.6.0 itsdangerous==0.24 jinja2==2.10 jmespath==0.9.3 jsonschema==2.6.0 -kombu==4.1.0 +kombu==4.2.1 linecache2==1.0.0 -logilab-common==1.4.1 -lxml==4.2.1 -m2crypto==0.29.0 +logilab-common==1.4.2 +lxml==4.2.4 +m2crypto==0.30.1 mako==1.0.7 manifestparser==1.1 markdown==2.6.11 markupsafe==1.0 -matplotlib==2.2.2 +matplotlib==2.2.3 mccabe==0.6.1 meld3==1.0.2 mock==2.0.0 mozcrash==1.0 -mozdevice==0.52 +mozdevice==1.0.0 mozfile==1.2 mozinfo==0.10 -mozlog==3.7 +mozlog==3.8 moznetwork==0.27 mozprocess==0.26 -mozprofile==0.29 -mozrunner==6.14 +mozprofile==1.1.0 +mozrunner==7.0.1 msgpack-python==0.5.6 mysql-python==1.2.5 -ndg-httpsclient==0.4.4 +ndg-httpsclient==0.5.1 netaddr==0.7.19 -netifaces==0.10.6 -newrelic==3.0.0.89 +netifaces==0.10.7 +newrelic==4.2.0.100 nose==1.3.7 -numpy==1.14.2 +numpy==1.15.0 oauth2==1.9.0.post1 oauth2client==4.1.2 -oauthlib==2.0.7 +oauthlib==2.1.0 ordereddict==1.1 -oslo.config==6.0.0 -pandas==0.22.0 +oslo.config==6.4.0 +pandas==0.23.4 paramiko==2.4.1 passlib==1.7.1 paste==2.0.3 pastedeploy==1.5.2 pastescript==2.0.2 -pbr==4.0.0 +pbr==4.2.0 pep8==1.7.1 -pexpect==4.4.0 -pika==0.11.2 -pillow==5.0.0 -pip==9.0.3 -prettytable -protobuf==3.5.2.post1 -psutil==5.4.3 -psycopg2==2.7.4 -py==1.5.3 -pyasn1-modules==0.2.1 -pyasn1==0.4.2 +pexpect==4.6.0 +pika==0.12.0 +pillow==5.2.0 +pip==18.0 +prettytable==7 +protobuf==3.6.0 +psutil==5.4.6 +psycopg2==2.7.5 +py==1.5.4 +pyasn1-modules==0.2.2 +pyasn1==0.4.4 pycparser==2.18 pycrypto==2.6.1 -pycurl==7.43.0.1 -pyflakes==1.6.0 +pycurl==7.43.0.2 +pyflakes==2.0.0 pygments==2.2.0 -pyjwt==1.6.1 +pyjwt==1.6.4 pylibmc==1.5.2 -pylint==1.8.3 -pymongo==3.6.1 -pymysql==0.8.0 -pyopenssl==17.5.0 +pylint==2.1.1 +pymongo==3.7.1 +pymysql==0.9.2 +pyopenssl==18.0.0 pyparsing==2.2.0 -pyramid==1.9.1 +pyramid==1.9.2 pystache==0.5.4 pytest-cov==2.5.1 -pytest==3.5.0 +pytest==3.7.1 python-cjson==1.2.1 python-daemon==2.1.2 -python-dateutil==2.7.2 +python-dateutil==2.7.3 python-gflags==3.1.2 -python-keystoneclient==3.15.0 +python-keystoneclient==3.17.0 python-memcached==1.59 python-mimeparse==1.6.0 -python-novaclient==10.1.0 -python-subunit==1.2.0 -python-swiftclient==3.5.0 -pytz==2018.3 -pyyaml==3.12 -pyzmq==17.0.0 -raven==6.6.0 +python-novaclient==11.0.0 +python-subunit==1.3.0 +python-swiftclient==3.6.0 +pytz==2018.5 +pyyaml==3.13 +pyzmq==17.1.2 +raven==6.9.0 redis==2.10.6 repoze.lru==0.7 -requests-oauthlib==0.8.0 -requests==2.18.4 +requests-oauthlib==1.0.0 +requests==2.19.1 retrying==1.3.3 rsa==3.4.2 -scipy==1.0.1 -selenium==3.11.0 +scipy==1.1.0 +selenium==3.14.0 setuptools-git==1.2 -setuptools==39.0.1 +setuptools==40.0.0 sh==1.12.14 -simplejson==3.13.2 +simplejson==3.16.0 six==1.11.0 snowballstemmer==1.2.1 south==1.0.2 -sphinx==1.7.2 +sphinx==1.7.6 sqlalchemy-migrate==0.11.0 -sqlalchemy==1.2.6 +sqlalchemy==1.2.10 sqlparse==0.2.4 statsd==3.2.2 -stevedore==1.28.0 +stevedore==1.29.0 suds==0.4 supervisor==3.3.4 testrepository==0.0.20 testtools==2.3.0 thrift==0.11.0 -tornado==5.0.1 -tox==2.9.1 -twisted==17.9.0 +tornado==5.1 +tox==3.2.1 +twisted==18.7.0 ujson==1.35 unidecode==1.0.22 unittest2==1.1.0 uritemplate==3.0.0 -urllib3==1.22 -uwsgi==2.0.17 +urllib3==1.23 +uwsgi==2.0.17.1 versiontools==1.9.1 -virtualenv==15.2.0 +virtualenv==16.0.0 waitress==1.1.0 warlock==1.3.0 -webob==1.7.4 -websocket-client==0.47.0 -webtest==2.0.29 +webob==1.8.2 +websocket-client==0.48.0 +webtest==2.0.30 werkzeug==0.14.1 -wheel==0.31.0 +wheel==0.31.1 xlrd==1.1.0 -zc.buildout==2.11.2 -zope.interface==4.4.3 +zc.buildout==2.12.1 +zope.interface==4.5.0 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index e8295d80..7b839959 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -1,29 +1,29 @@ -alembic==0.9.9 -amqp==2.2.2 +alembic==1.0.0 +amqp==2.3.2 amqplib==1.0.2 -ansible==2.5.0 +ansible==2.6.2 anyjson==0.3.3 apache-libcloud==2.3.0 argparse==1.4.0 -astroid==1.6.2 -awscli==1.14.68 -babel==2.5.3 +astroid==2.0.4 +awscli==1.15.76 +babel==2.6.0 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 -beautifulsoup4==4.6.0 -billiard==3.5.0.3 -blessings==1.6.1 +beautifulsoup4==4.6.3 +billiard==3.5.0.4 +blessings==1.7 blinker==1.4 -boto==2.48.0 -botocore==1.9.21 +boto==2.49.0 +botocore==1.10.75 bottle==0.12.13 -celery==4.1.0 -certifi==2018.1.18 +celery==4.2.1 +certifi==2018.8.13 cffi==1.11.5 chardet==3.0.4 click==6.7 -cliff==2.11.0 -cmd2==0.8.2 +cliff==2.13.0 +cmd2==0.9.3 colorama==0.3.9 configobj==5.0.6 cov-core==1.15.0 @@ -32,164 +32,164 @@ coveralls==1.3.0 crcmod==1.7 cryptography==2.3 cssselect==1.0.3 -cython==0.28.1 -decorator==4.2.1 +cython==0.28.5 +decorator==4.3.0 django-celery==3.2.2 django-debug-toolbar==1.9.1 -django-extensions==2.0.6 -django==2.0.3 +django-extensions==2.1.0 +django==2.1 django_compress==1.0.1 -djangorestframework==3.7.7 +djangorestframework==3.8.2 docker-py==1.10.6 docopt==0.6.2 docutils==0.14 ecdsa==0.13 -elasticsearch==6.2.0 +elasticsearch==6.3.1 enum34==1.1.6 -eventlet==0.22.1 +eventlet==0.24.1 extras==1.0.0 -fabric==1.14.0 +fabric==2.3.1 fixtures==3.0.0 flake8==3.5.0 -flask==0.12.2 +flask==1.0.2 funcsigs==1.0.2 -gevent==1.2.2 -google-api-python-client==1.6.6 -greenlet==0.4.13 -gunicorn==19.7.1 +gevent==1.3.5 +google-api-python-client==1.7.4 +greenlet==0.4.14 +gunicorn==19.9.0 hiredis==0.2.0 honcho==1.0.1 html5lib==1.0.1 httplib2==0.11.3 -idna==2.6 -ipaddress==1.0.19 -ipython==6.2.1 +idna==2.7 +ipaddress==1.0.22 +ipython==6.5.0 iso8601==0.1.12 isodate==0.6.0 itsdangerous==0.24 jinja2==2.10 jmespath==0.9.3 jsonschema==2.6.0 -kombu==4.1.0 +kombu==4.2.1 linecache2==1.0.0 -logilab-common==1.4.1 -lxml==4.2.1 +logilab-common==1.4.2 +lxml==4.2.4 mako==1.0.7 manifestparser==1.1 markdown==2.6.11 markupsafe==1.0 -matplotlib==2.2.2 +matplotlib==2.2.3 mccabe==0.6.1 meld3==1.0.2 mock==2.0.0 mozcrash==1.0 -mozdevice==0.52 +mozdevice==1.0.0 mozfile==1.2 mozinfo==0.10 -mozlog==3.7 +mozlog==3.8 moznetwork==0.27 mozprocess==0.26 msgpack-python==0.5.6 -ndg-httpsclient==0.4.4 +ndg-httpsclient==0.5.1 netaddr==0.7.19 -netifaces==0.10.6 -newrelic==3.0.0.89 +netifaces==0.10.7 +newrelic==4.2.0.100 nose==1.3.7 -numpy==1.14.2 +numpy==1.15.0 oauth2==1.9.0.post1 oauth2client==4.1.2 -oauthlib==2.0.7 +oauthlib==2.1.0 ordereddict==1.1 -oslo.config==6.0.0 -pandas==0.22.0 +oslo.config==6.4.0 +pandas==0.23.4 paramiko==2.4.1 passlib==1.7.1 paste==2.0.3 pastedeploy==1.5.2 pastescript==2.0.2 -pbr==4.0.0 +pbr==4.2.0 pep8==1.7.1 -pexpect==4.4.0 -pika==0.11.2 -pillow==5.0.0 -pip==9.0.3 -prettytable -protobuf==3.5.2.post1 -psutil==5.4.3 -psycopg2==2.7.4 -py==1.5.3 -pyasn1-modules==0.2.1 -pyasn1==0.4.2 +pexpect==4.6.0 +pika==0.12.0 +pillow==5.2.0 +pip==18.0 +prettytable==7 +protobuf==3.6.0 +psutil==5.4.6 +psycopg2==2.7.5 +py==1.5.4 +pyasn1-modules==0.2.2 +pyasn1==0.4.4 pycparser==2.18 pycrypto==2.6.1 -pyflakes==1.6.0 +pyflakes==2.0.0 pygments==2.2.0 -pyjwt==1.6.1 +pyjwt==1.6.4 pylibmc==1.5.2 -pylint==1.8.3 -pymongo==3.6.1 -pymysql==0.8.0 -pyopenssl==17.5.0 +pylint==2.1.1 +pymongo==3.7.1 +pymysql==0.9.2 +pyopenssl==18.0.0 pyparsing==2.2.0 -pyramid==1.9.1 +pyramid==1.9.2 pystache==0.5.4 pytest-cov==2.5.1 -pytest==3.5.0 +pytest==3.7.1 python-daemon==2.1.2 -python-dateutil==2.7.2 +python-dateutil==2.7.3 python-gflags==3.1.2 -python-keystoneclient==3.15.0 +python-keystoneclient==3.17.0 python-memcached==1.59 python-mimeparse==1.6.0 -python-novaclient==10.1.0 -python-subunit==1.2.0 -python-swiftclient==3.5.0 -pytz==2018.3 -pyyaml==3.12 -pyzmq==17.0.0 -raven==6.6.0 +python-novaclient==11.0.0 +python-subunit==1.3.0 +python-swiftclient==3.6.0 +pytz==2018.5 +pyyaml==3.13 +pyzmq==17.1.2 +raven==6.9.0 redis==2.10.6 repoze.lru==0.7 -requests-oauthlib==0.8.0 -requests==2.18.4 +requests-oauthlib==1.0.0 +requests==2.19.1 retrying==1.3.3 rsa==3.4.2 -scipy==1.0.1 -selenium==3.11.0 +scipy==1.1.0 +selenium==3.14.0 setuptools-git==1.2 -setuptools==39.0.1 +setuptools==40.0.0 sh==1.12.14 -simplejson==3.13.2 +simplejson==3.16.0 six==1.11.0 snowballstemmer==1.2.1 south==1.0.2 -sphinx==1.7.2 +sphinx==1.7.6 sqlalchemy-migrate==0.11.0 -sqlalchemy==1.2.6 +sqlalchemy==1.2.10 sqlparse==0.2.4 statsd==3.2.2 -stevedore==1.28.0 +stevedore==1.29.0 testrepository==0.0.20 testtools==2.3.0 thrift==0.11.0 -tornado==5.0.1 -tox==2.9.1 -twisted==17.9.0 +tornado==5.1 +tox==3.2.1 +twisted==18.7.0 ujson==1.35 unidecode==1.0.22 unittest2==1.1.0 uritemplate==3.0.0 -urllib3==1.22 -uwsgi==2.0.17 +urllib3==1.23 +uwsgi==2.0.17.1 versiontools==1.9.1 -virtualenv==15.2.0 +virtualenv==16.0.0 waitress==1.1.0 warlock==1.3.0 -webob==1.7.4 -websocket-client==0.47.0 -webtest==2.0.29 +webob==1.8.2 +websocket-client==0.48.0 +webtest==2.0.30 werkzeug==0.14.1 -wheel==0.31.0 +wheel==0.31.1 xlrd==1.1.0 -zc.buildout==2.11.2 -zope.interface==4.4.3 +zc.buildout==2.12.1 +zope.interface==4.5.0 From 721201e707efbba87005c974e6b8cbabf1e17010 Mon Sep 17 00:00:00 2001 From: David Bendory Date: Sun, 19 Aug 2018 16:21:41 -0400 Subject: [PATCH 175/256] Container Builder is now Cloud Build. --- RELEASING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/RELEASING.md b/RELEASING.md index 2dcd806d..f886a542 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -36,7 +36,7 @@ follows: b. `gcloud auth activate-service-account` is performed c. `gcloud config set project` is performed 3. The script invokes `build.sh` in this repository -4. `build.sh` invokes Google Container Builder with the `cloudbuild-*.yaml` +4. `build.sh` invokes Google Cloud Build with the `cloudbuild-*.yaml` config files. ## Building interpreters @@ -65,7 +65,7 @@ Debian or Ubuntu-like). ## Building locally To build this repository using local Docker commands instead of the Google -Container Builder service, add the `--local` flag as shown: +Cloud Build service, add the `--local` flag as shown: ``` shell ./build.sh --local From 56c9654a17f0662acffe4fb06a329cbce16e1829 Mon Sep 17 00:00:00 2001 From: David Bendory Date: Sun, 19 Aug 2018 16:25:27 -0400 Subject: [PATCH 176/256] Container Builder is now Cloud Build. --- scripts/local_cloudbuild.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/local_cloudbuild.py b/scripts/local_cloudbuild.py index f80e0f4b..5c23a1bc 100755 --- a/scripts/local_cloudbuild.py +++ b/scripts/local_cloudbuild.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Emulate the Google Container Builder locally. +"""Emulate the Google Cloud Build locally. The input is a local cloudbuild.yaml file. This is translated into a series of commands for the locally installed Docker daemon. These @@ -48,8 +48,8 @@ # Exclude non-printable control characters (including newlines) PRINTABLE_REGEX = re.compile(r"""^[^\x00-\x1f]*$""") -# Container Builder substitutions -# https://cloud.google.com/container-builder/docs/api/build-requests#substitutions +# Cloud Build substitutions +# https://cloud.google.com/cloud-build/docs/api/build-requests#substitutions SUBSTITUTION_REGEX = re.compile(r"""(?x) [$] # Dollar sign ( From 8369dc90a267a7fa933056393322b976f4d44419 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Mon, 27 Aug 2018 00:50:51 -0700 Subject: [PATCH 177/256] Auto-update dependencies. --- runtime-image/resources/requirements.txt | 2 +- scripts/requirements-test.txt | 2 +- tests/python2-libraries/requirements.txt | 40 ++++++++++++------------ tests/python3-libraries/requirements.txt | 40 ++++++++++++------------ 4 files changed, 42 insertions(+), 42 deletions(-) diff --git a/runtime-image/resources/requirements.txt b/runtime-image/resources/requirements.txt index 44c624d8..bc783013 100644 --- a/runtime-image/resources/requirements.txt +++ b/runtime-image/resources/requirements.txt @@ -1,3 +1,3 @@ pip==18.0 -setuptools==40.0.0 +setuptools==40.2.0 wheel==0.31.1 diff --git a/scripts/requirements-test.txt b/scripts/requirements-test.txt index df909d47..c8e698da 100644 --- a/scripts/requirements-test.txt +++ b/scripts/requirements-test.txt @@ -1,4 +1,4 @@ flask==1.0.2 -pytest==3.7.1 +pytest==3.7.3 pytest-cov==2.5.1 pyyaml==3.13 diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 6daad0ab..a319137a 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -1,12 +1,12 @@ alembic==1.0.0 amqp==2.3.2 amqplib==1.0.2 -ansible==2.6.2 +ansible==2.6.3 anyjson==0.3.3 apache-libcloud==2.3.0 argparse==1.4.0 astroid==2.0.4 -awscli==1.15.76 +awscli==1.16.1 babel==2.6.0 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 @@ -16,29 +16,29 @@ billiard==3.5.0.4 blessings==1.7 blinker==1.4 boto==2.49.0 -botocore==1.10.75 +botocore==1.11.1 bottle==0.12.13 carbon<1.1.1 celery==4.2.1 -certifi==2018.8.13 +certifi==2018.8.24 cffi==1.11.5 chardet==3.0.4 click==6.7 cliff==2.13.0 -cmd2==0.9.3 +cmd2==0.9.4 colorama==0.3.9 configobj==5.0.6 cov-core==1.15.0 coverage==4.5.1 -coveralls==1.3.0 +coveralls==1.4.0 crcmod==1.7 -cryptography==2.3 +cryptography==2.3.1 cssselect==1.0.3 cython==0.28.5 decorator==4.3.0 django-celery==3.2.2 django-debug-toolbar==1.9.1 -django-extensions==2.1.0 +django-extensions==2.1.1 django<2.0 django_compress==1.0.1 djangorestframework==3.8.2 @@ -57,7 +57,7 @@ flask==1.0.2 funcsigs==1.0.2 functools32==3.2.3.post2 futures==3.2.0 -gevent==1.3.5 +gevent==1.3.6 google-api-python-client==1.7.4 graphite-web==1.1.3 greenlet==0.4.14 @@ -88,7 +88,7 @@ mccabe==0.6.1 meld3==1.0.2 mock==2.0.0 mozcrash==1.0 -mozdevice==1.0.0 +mozdevice==1.0.1 mozfile==1.2 mozinfo==0.10 mozlog==3.8 @@ -103,7 +103,7 @@ netaddr==0.7.19 netifaces==0.10.7 newrelic==4.2.0.100 nose==1.3.7 -numpy==1.15.0 +numpy==1.15.1 oauth2==1.9.0.post1 oauth2client==4.1.2 oauthlib==2.1.0 @@ -122,8 +122,8 @@ pika==0.12.0 pillow==5.2.0 pip==18.0 prettytable==7 -protobuf==3.6.0 -psutil==5.4.6 +protobuf==3.6.1 +psutil==5.4.7 psycopg2==2.7.5 py==1.5.4 pyasn1-modules==0.2.2 @@ -143,9 +143,9 @@ pyparsing==2.2.0 pyramid==1.9.2 pystache==0.5.4 pytest-cov==2.5.1 -pytest==3.7.1 +pytest==3.7.3 python-cjson==1.2.1 -python-daemon==2.1.2 +python-daemon==2.2.0 python-dateutil==2.7.3 python-gflags==3.1.2 python-keystoneclient==3.17.0 @@ -167,17 +167,17 @@ rsa==3.4.2 scipy==1.1.0 selenium==3.14.0 setuptools-git==1.2 -setuptools==40.0.0 +setuptools==40.2.0 sh==1.12.14 simplejson==3.16.0 six==1.11.0 snowballstemmer==1.2.1 south==1.0.2 -sphinx==1.7.6 +sphinx==1.7.7 sqlalchemy-migrate==0.11.0 -sqlalchemy==1.2.10 +sqlalchemy==1.2.11 sqlparse==0.2.4 -statsd==3.2.2 +statsd==3.3.0 stevedore==1.29.0 suds==0.4 supervisor==3.3.4 @@ -198,7 +198,7 @@ virtualenv==16.0.0 waitress==1.1.0 warlock==1.3.0 webob==1.8.2 -websocket-client==0.48.0 +websocket-client==0.51.0 webtest==2.0.30 werkzeug==0.14.1 wheel==0.31.1 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 7b839959..f9f1b8f7 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -1,12 +1,12 @@ alembic==1.0.0 amqp==2.3.2 amqplib==1.0.2 -ansible==2.6.2 +ansible==2.6.3 anyjson==0.3.3 apache-libcloud==2.3.0 argparse==1.4.0 astroid==2.0.4 -awscli==1.15.76 +awscli==1.16.1 babel==2.6.0 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 @@ -15,28 +15,28 @@ billiard==3.5.0.4 blessings==1.7 blinker==1.4 boto==2.49.0 -botocore==1.10.75 +botocore==1.11.1 bottle==0.12.13 celery==4.2.1 -certifi==2018.8.13 +certifi==2018.8.24 cffi==1.11.5 chardet==3.0.4 click==6.7 cliff==2.13.0 -cmd2==0.9.3 +cmd2==0.9.4 colorama==0.3.9 configobj==5.0.6 cov-core==1.15.0 coverage==4.5.1 -coveralls==1.3.0 +coveralls==1.4.0 crcmod==1.7 -cryptography==2.3 +cryptography==2.3.1 cssselect==1.0.3 cython==0.28.5 decorator==4.3.0 django-celery==3.2.2 django-debug-toolbar==1.9.1 -django-extensions==2.1.0 +django-extensions==2.1.1 django==2.1 django_compress==1.0.1 djangorestframework==3.8.2 @@ -53,7 +53,7 @@ fixtures==3.0.0 flake8==3.5.0 flask==1.0.2 funcsigs==1.0.2 -gevent==1.3.5 +gevent==1.3.6 google-api-python-client==1.7.4 greenlet==0.4.14 gunicorn==19.9.0 @@ -83,7 +83,7 @@ mccabe==0.6.1 meld3==1.0.2 mock==2.0.0 mozcrash==1.0 -mozdevice==1.0.0 +mozdevice==1.0.1 mozfile==1.2 mozinfo==0.10 mozlog==3.8 @@ -95,7 +95,7 @@ netaddr==0.7.19 netifaces==0.10.7 newrelic==4.2.0.100 nose==1.3.7 -numpy==1.15.0 +numpy==1.15.1 oauth2==1.9.0.post1 oauth2client==4.1.2 oauthlib==2.1.0 @@ -114,8 +114,8 @@ pika==0.12.0 pillow==5.2.0 pip==18.0 prettytable==7 -protobuf==3.6.0 -psutil==5.4.6 +protobuf==3.6.1 +psutil==5.4.7 psycopg2==2.7.5 py==1.5.4 pyasn1-modules==0.2.2 @@ -134,8 +134,8 @@ pyparsing==2.2.0 pyramid==1.9.2 pystache==0.5.4 pytest-cov==2.5.1 -pytest==3.7.1 -python-daemon==2.1.2 +pytest==3.7.3 +python-daemon==2.2.0 python-dateutil==2.7.3 python-gflags==3.1.2 python-keystoneclient==3.17.0 @@ -157,17 +157,17 @@ rsa==3.4.2 scipy==1.1.0 selenium==3.14.0 setuptools-git==1.2 -setuptools==40.0.0 +setuptools==40.2.0 sh==1.12.14 simplejson==3.16.0 six==1.11.0 snowballstemmer==1.2.1 south==1.0.2 -sphinx==1.7.6 +sphinx==1.7.7 sqlalchemy-migrate==0.11.0 -sqlalchemy==1.2.10 +sqlalchemy==1.2.11 sqlparse==0.2.4 -statsd==3.2.2 +statsd==3.3.0 stevedore==1.29.0 testrepository==0.0.20 testtools==2.3.0 @@ -186,7 +186,7 @@ virtualenv==16.0.0 waitress==1.1.0 warlock==1.3.0 webob==1.8.2 -websocket-client==0.48.0 +websocket-client==0.51.0 webtest==2.0.30 werkzeug==0.14.1 wheel==0.31.1 From a4df99416aa22b571fb6f797e64d68c821bb6131 Mon Sep 17 00:00:00 2001 From: dlorenc Date: Mon, 1 Oct 2018 08:08:58 -0700 Subject: [PATCH 178/256] Pin pip to 9.0.3. Newer versions conflict with the OS installed package. --- runtime-image/resources/requirements.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/runtime-image/resources/requirements.txt b/runtime-image/resources/requirements.txt index bc783013..298e5f41 100644 --- a/runtime-image/resources/requirements.txt +++ b/runtime-image/resources/requirements.txt @@ -1,3 +1,5 @@ -pip==18.0 +# Do not update pip, it conflicts with the OS system installed version. +# https://github.com/pypa/pip/issues/5240 +pip==9.0.3 setuptools==40.2.0 wheel==0.31.1 From 7eb071f7f7526815fc4b90a5ed0225f1247eef84 Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Mon, 1 Oct 2018 14:36:10 -0700 Subject: [PATCH 179/256] fix library versions for tests (#202) --- tests/python2-libraries/requirements.txt | 8 ++++---- tests/python3-libraries/requirements.txt | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index a319137a..865cd1aa 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -5,7 +5,7 @@ ansible==2.6.3 anyjson==0.3.3 apache-libcloud==2.3.0 argparse==1.4.0 -astroid==2.0.4 +astroid==1.6.5 awscli==1.16.1 babel==2.6.0 backports.ssl_match_hostname==3.5.0.1 @@ -25,7 +25,7 @@ cffi==1.11.5 chardet==3.0.4 click==6.7 cliff==2.13.0 -cmd2==0.9.4 +cmd2==0.8.9 colorama==0.3.9 configobj==5.0.6 cov-core==1.15.0 @@ -121,7 +121,7 @@ pexpect==4.6.0 pika==0.12.0 pillow==5.2.0 pip==18.0 -prettytable==7 +prettytable==0.7.2 protobuf==3.6.1 psutil==5.4.7 psycopg2==2.7.5 @@ -135,7 +135,7 @@ pyflakes==2.0.0 pygments==2.2.0 pyjwt==1.6.4 pylibmc==1.5.2 -pylint==2.1.1 +pylint==1.9.3 pymongo==3.7.1 pymysql==0.9.2 pyopenssl==18.0.0 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index f9f1b8f7..3b283d1d 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -113,7 +113,7 @@ pexpect==4.6.0 pika==0.12.0 pillow==5.2.0 pip==18.0 -prettytable==7 +prettytable==0.7.2 protobuf==3.6.1 psutil==5.4.7 psycopg2==2.7.5 From 5bc7a6e49e591cb4771dbd9acdeddb7c3bc41a5d Mon Sep 17 00:00:00 2001 From: dlorenc Date: Fri, 2 Nov 2018 14:43:11 -0700 Subject: [PATCH 180/256] Bump Python 3.6.6 to 3.6.7. --- .../scripts/build-python-3.6.sh | 12 ++++++------ tests/no-virtualenv/no-virtualenv.yaml | 2 +- tests/virtualenv/virtualenv_python36.yaml | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/python-interpreter-builder/scripts/build-python-3.6.sh b/python-interpreter-builder/scripts/build-python-3.6.sh index ef192e1e..b3896e89 100755 --- a/python-interpreter-builder/scripts/build-python-3.6.sh +++ b/python-interpreter-builder/scripts/build-python-3.6.sh @@ -6,14 +6,14 @@ set -x # Get the source mkdir -p /opt/sources cd /opt/sources -wget --no-verbose https://www.python.org/ftp/python/3.6.6/Python-3.6.6.tgz +wget --no-verbose https://www.python.org/ftp/python/3.6.7/Python-3.6.7.tgz # SHA-256 generated via `shasum -a 256 [file]` shasum --check < Date: Fri, 2 Nov 2018 14:36:55 -0700 Subject: [PATCH 181/256] Add support for Python 3.7. --- cloudbuild_interpreters.yaml | 6 +- nox.py | 2 +- .../scripts/build-python-3.7.sh | 153 ++++++++++++++++++ runtime-image/Dockerfile.in | 4 + scripts/gen_dockerfile.py | 1 + 5 files changed, 164 insertions(+), 2 deletions(-) create mode 100755 python-interpreter-builder/scripts/build-python-3.7.sh diff --git a/cloudbuild_interpreters.yaml b/cloudbuild_interpreters.yaml index 7daecc57..c75e59ad 100644 --- a/cloudbuild_interpreters.yaml +++ b/cloudbuild_interpreters.yaml @@ -18,11 +18,15 @@ steps: args: ['/scripts/build-python-3.6.sh'] id: build-3.6 waitFor: ['interpreter-builder'] +- name: interpreter-builder + args: ['/scripts/build-python-3.7.sh'] + id: build-3.7 + waitFor: ['interpreter-builder'] # Upload them to tbe build-id location - name: gcr.io/cloud-builders/gsutil:latest args: ['cp', '/workspace/runtime-image/*.tar.gz', 'gs://python-interpreters/$BUILD_ID/'] - waitFor: ['build-3.4', 'build-3.5', 'build-3.6'] + waitFor: ['build-3.4', 'build-3.5', 'build-3.6', 'build-3.7'] # "Tag" this as latest - name: gcr.io/cloud-builders/gsutil:latest diff --git a/nox.py b/nox.py index 872eb88c..0ee6f443 100644 --- a/nox.py +++ b/nox.py @@ -57,7 +57,7 @@ def lint(session): @nox.session -@nox.parametrize('version', ['3.4', '3.5', '3.6']) +@nox.parametrize('version', ['3.4', '3.5', '3.6', '3.7']) def tests(session, version): session.interpreter = 'python' + version session.install('-r', 'scripts/requirements-test.txt') diff --git a/python-interpreter-builder/scripts/build-python-3.7.sh b/python-interpreter-builder/scripts/build-python-3.7.sh new file mode 100755 index 00000000..2a77e279 --- /dev/null +++ b/python-interpreter-builder/scripts/build-python-3.7.sh @@ -0,0 +1,153 @@ +#!/bin/bash + +set -euo pipefail +set -x + +# Get the source +mkdir -p /opt/sources +cd /opt/sources +wget --no-verbose https://www.python.org/ftp/python/3.7.1/Python-3.7.1.tgz +# SHA-256 generated via `shasum -a 256 [file]` +shasum --check < Date: Wed, 14 Nov 2018 14:44:56 -0800 Subject: [PATCH 182/256] Fixing up CODEOWNERS (#207) --- CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CODEOWNERS b/CODEOWNERS index c08c2d11..f8b549f9 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1,4 +1,4 @@ # Code owners file. # This file controls who is tagged for review for any given pull request. -* @duggelz @liyanhui1228 @dlorenc +* @dlorenc @sharifelgamal @nkubala @tstromberg From 608b42af72537ce5614f1066db8b992a33af1199 Mon Sep 17 00:00:00 2001 From: dlorenc Date: Tue, 27 Nov 2018 11:40:09 -0800 Subject: [PATCH 183/256] Fix the Python 3.7 runtime release script. --- runtime-image/Dockerfile.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime-image/Dockerfile.in b/runtime-image/Dockerfile.in index b872a501..87d1aff2 100644 --- a/runtime-image/Dockerfile.in +++ b/runtime-image/Dockerfile.in @@ -40,7 +40,7 @@ RUN /usr/bin/pip install --upgrade -r /resources/requirements.txt && \ rm -f /opt/python3.5/bin/pip /opt/python3.5/bin/pip3 && \ /opt/python3.6/bin/pip3.6 install --upgrade -r /resources/requirements.txt && \ rm -f /opt/python3.6/bin/pip /opt/python3.6/bin/pip3 && \ - /opt/python3.6/bin/pip3.7 install --upgrade -r /resources/requirements.txt && \ + /opt/python3.7/bin/pip3.7 install --upgrade -r /resources/requirements.txt && \ rm -f /opt/python3.7/bin/pip /opt/python3.7/bin/pip3 && \ /usr/bin/pip install --upgrade -r /resources/requirements-virtualenv.txt From 7be6af7551dfc489535404ca5c09ca2ccc6d0641 Mon Sep 17 00:00:00 2001 From: dlorenc Date: Tue, 4 Dec 2018 09:10:55 -0800 Subject: [PATCH 184/256] Add Python 3.7 to the path. --- runtime-image/Dockerfile.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime-image/Dockerfile.in b/runtime-image/Dockerfile.in index 87d1aff2..64201966 100644 --- a/runtime-image/Dockerfile.in +++ b/runtime-image/Dockerfile.in @@ -27,7 +27,7 @@ RUN wget https://storage.googleapis.com/python-interpreters/latest/interpreter-3 rm interpreter-*.tar.gz # Add Google-built interpreters to the path -ENV PATH /opt/python3.6/bin:/opt/python3.5/bin:/opt/python3.4/bin:$PATH +ENV PATH /opt/python3.7/bin:/opt/python3.6/bin:/opt/python3.5/bin:/opt/python3.4/bin:$PATH RUN update-alternatives --install /usr/local/bin/python3 python3 /opt/python3.6/bin/python3.6 50 RUN update-alternatives --install /usr/local/bin/pip3 pip3 /opt/python3.6/bin/pip3.6 50 From 0e07aba8b47e3fa64d82e0a2a3ea943bb60027f4 Mon Sep 17 00:00:00 2001 From: Evan Anderson Date: Tue, 15 Jan 2019 11:44:16 -0800 Subject: [PATCH 185/256] Document how to use Dockerfile with python3 --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 2cc7d173..51413044 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,7 @@ command or entrypoint. For example: # Create a virtualenv for dependencies. This isolates these packages from # system-level packages. + # Use -p python3 or -p python3.7 to select python version. Default is version 2. RUN virtualenv /env # Setting these environment variables are the same as running From db4ae4216b1783ab6ddc96ce6500d7257eb37ef8 Mon Sep 17 00:00:00 2001 From: sharifelgamal Date: Wed, 2 Jan 2019 17:08:02 -0800 Subject: [PATCH 186/256] Updating python to 3.6.8 and 3.7.2 and defaulting to 3.7 --- .../scripts/build-python-3.6.sh | 12 ++--- .../scripts/build-python-3.7.sh | 12 ++--- scripts/gen_dockerfile.py | 2 +- scripts/gen_dockerfile_test.py | 5 +- .../testdata/hello_world_golden/Dockerfile | 4 +- tests/virtualenv/virtualenv_python36.yaml | 2 +- tests/virtualenv/virtualenv_python37.yaml | 54 +++++++++++++++++++ 7 files changed, 74 insertions(+), 17 deletions(-) create mode 100644 tests/virtualenv/virtualenv_python37.yaml diff --git a/python-interpreter-builder/scripts/build-python-3.6.sh b/python-interpreter-builder/scripts/build-python-3.6.sh index b3896e89..73912401 100755 --- a/python-interpreter-builder/scripts/build-python-3.6.sh +++ b/python-interpreter-builder/scripts/build-python-3.6.sh @@ -6,14 +6,14 @@ set -x # Get the source mkdir -p /opt/sources cd /opt/sources -wget --no-verbose https://www.python.org/ftp/python/3.6.7/Python-3.6.7.tgz +wget --no-verbose https://www.python.org/ftp/python/3.6.8/Python-3.6.8.tgz # SHA-256 generated via `shasum -a 256 [file]` shasum --check < Date: Wed, 2 Jan 2019 17:09:16 -0800 Subject: [PATCH 187/256] fixing 3.7 virtualenv tests --- tests/virtualenv/virtualenv_python37.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/virtualenv/virtualenv_python37.yaml b/tests/virtualenv/virtualenv_python37.yaml index f75ffdcd..457933f9 100644 --- a/tests/virtualenv/virtualenv_python37.yaml +++ b/tests/virtualenv/virtualenv_python37.yaml @@ -7,48 +7,48 @@ globalEnvVars: value: "/env/bin:$PATH" commandTests: - - name: "virtualenv36 python installation" + - name: "virtualenv37 python installation" setup: [["virtualenv", "-p", "python3.7", "/env"]] command: ["which", "python"] expectedOutput: ["/env/bin/python\n"] - - name: "virtualenv36 python3 installation" + - name: "virtualenv37 python3 installation" setup: [["virtualenv", "-p", "python3.7", "/env"]] command: ["which", "python3"] expectedOutput: ["/env/bin/python3\n"] - - name: "virtualenv36 python3.7 installation" + - name: "virtualenv37 python3.7 installation" setup: [["virtualenv", "-p", "python3.7", "/env"]] command: ["which", "python3.7"] expectedOutput: ["/env/bin/python3.7\n"] - - name: "virtualenv36 python version" + - name: "virtualenv37 python version" setup: [["virtualenv", "-p", "python3.7", "/env"]] command: ["python", "--version"] expectedOutput: ["Python 3.7.2\n"] - - name: "virtualenv36 pip installation" + - name: "virtualenv37 pip installation" setup: [["virtualenv", "-p", "python3.7", "/env"]] command: ["which", "pip"] expectedOutput: ["/env/bin/pip\n"] - - name: "virtualenv36 pip3 installation" + - name: "virtualenv37 pip3 installation" setup: [["virtualenv", "-p", "python3.7", "/env"]] command: ["which", "pip3"] expectedOutput: ["/env/bin/pip3\n"] - - name: "virtualenv36 gunicorn installation" + - name: "virtualenv37 gunicorn installation" setup: [["virtualenv", "-p", "python3.7", "/env"], ["pip", "install", "gunicorn"]] command: ["which", "gunicorn"] expectedOutput: ["/env/bin/gunicorn"] - - name: "virtualenv36 flask installation" + - name: "virtualenv37 flask installation" setup: [["virtualenv", "-p", "python3.7", "/env"], ["pip", "install", "flask"]] command: ["python", "-c", "import flask; print(flask.__file__)"] expectedOutput: ["/env/lib/python3.7/site-packages/flask"] - - name: "virtualenv36 test.support availability" + - name: "virtualenv37 test.support availability" setup: [["virtualenv", "-p", "python3.7", "/env"]] command: ["python", "-c", "\"from test import pystone, regrtest, support\""] From a90395dfeebc6b095eaa59f2c38eb392c471487d Mon Sep 17 00:00:00 2001 From: sharifelgamal Date: Fri, 4 Jan 2019 16:35:38 -0800 Subject: [PATCH 188/256] fixing up defaulting to 3.7 --- builder/gen-dockerfile/Dockerfile.in | 4 ++-- cloudbuild_test.yaml | 7 +++++++ runtime-image/Dockerfile.in | 4 ++-- tests/no-virtualenv/no-virtualenv.yaml | 2 +- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/builder/gen-dockerfile/Dockerfile.in b/builder/gen-dockerfile/Dockerfile.in index 4f6447eb..627151bf 100644 --- a/builder/gen-dockerfile/Dockerfile.in +++ b/builder/gen-dockerfile/Dockerfile.in @@ -1,6 +1,6 @@ FROM ${STAGING_IMAGE} -LABEL python_version=python3.6 -RUN virtualenv --no-download /env -p python3.6 +LABEL python_version=python3.7 +RUN virtualenv --no-download /env -p python3.7 # Set virtualenv environment variables. This is equivalent to running # source /env/bin/activate diff --git a/cloudbuild_test.yaml b/cloudbuild_test.yaml index 416c5204..ad674026 100644 --- a/cloudbuild_test.yaml +++ b/cloudbuild_test.yaml @@ -43,6 +43,13 @@ steps: '/workspace/tests/virtualenv/virtualenv_python36.yaml', ] waitFor: ['runtime'] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 + args: [ + '-test.v', + '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', + '/workspace/tests/virtualenv/virtualenv_python37.yaml', + ] + waitFor: ['runtime'] - name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 args: [ '-test.v', diff --git a/runtime-image/Dockerfile.in b/runtime-image/Dockerfile.in index 64201966..705a915b 100644 --- a/runtime-image/Dockerfile.in +++ b/runtime-image/Dockerfile.in @@ -19,7 +19,7 @@ ENV PYTHONUNBUFFERED 1 RUN wget https://storage.googleapis.com/python-interpreters/latest/interpreter-3.4.tar.gz && \ wget https://storage.googleapis.com/python-interpreters/latest/interpreter-3.5.tar.gz && \ wget https://storage.googleapis.com/python-interpreters/latest/interpreter-3.6.tar.gz && \ - wget https://storage.googleapis.com/python-interpreters/latest/interpreter-3.7.tar.gz && \ + wget https://storage.googleapis.com/python-interpreters/latest/interpreter-3.7.tar.gz && \ tar -xzf interpreter-3.4.tar.gz && \ tar -xzf interpreter-3.5.tar.gz && \ tar -xzf interpreter-3.6.tar.gz && \ @@ -41,7 +41,7 @@ RUN /usr/bin/pip install --upgrade -r /resources/requirements.txt && \ /opt/python3.6/bin/pip3.6 install --upgrade -r /resources/requirements.txt && \ rm -f /opt/python3.6/bin/pip /opt/python3.6/bin/pip3 && \ /opt/python3.7/bin/pip3.7 install --upgrade -r /resources/requirements.txt && \ - rm -f /opt/python3.7/bin/pip /opt/python3.7/bin/pip3 && \ + rm -f /opt/python3.7/bin/pip /opt/python3.7/bin/pip3 && \ /usr/bin/pip install --upgrade -r /resources/requirements-virtualenv.txt # Setup the app working directory diff --git a/tests/no-virtualenv/no-virtualenv.yaml b/tests/no-virtualenv/no-virtualenv.yaml index 20c09501..c8350e4e 100644 --- a/tests/no-virtualenv/no-virtualenv.yaml +++ b/tests/no-virtualenv/no-virtualenv.yaml @@ -43,7 +43,7 @@ commandTests: expectedOutput: ["/usr/local/bin/python3\n"] - name: "default python3 version" command: ["python3", "--version"] - expectedOutput: ["Python 3.6.7\n"] + expectedOutput: ["Python 3.7.2\n"] - name: "default pip3 installation" command: ["which", "pip3"] expectedOutput: ["/usr/local/bin/pip3\n"] From 407695209e7397cf11b12f898e60a29a9529b328 Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Tue, 22 Jan 2019 09:30:15 -0800 Subject: [PATCH 189/256] Updating build's base image from debian8 to ubuntu_16_0_4 to address SSL issues with 3.6.8 and 3.7.2 build Updating Dockerfile to update_alternatives to 3.7.2 Updating Dockerfile to separate installs so python install errors are easier to differentiate --- build.sh | 8 ++++---- runtime-image/Dockerfile.in | 24 ++++++++++++------------ 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/build.sh b/build.sh index bdc4f658..63afa1ab 100755 --- a/build.sh +++ b/build.sh @@ -24,7 +24,7 @@ test=0 # Should run standard test suite? local=0 # Should run using local Docker daemon instead of GCR? -os_base=debian8 # Which operating system base to use +os_base=ubuntu16 # Which operating system base to use interpreter=0 # Should build interpreters instead of images # Note that $gcloud_cmd has spaces in it @@ -154,10 +154,10 @@ if [ "${local}" -eq 1 ]; then fi # Pick OS image to use as base -if [ "${os_base}" == "ubuntu16" ]; then - export OS_BASE_IMAGE="gcr.io/gcp-runtimes/ubuntu_16_0_4:latest" -else +if [ "${os_base}" == "debian8" ]; then export OS_BASE_IMAGE="gcr.io/google-appengine/debian8:latest" +else + export OS_BASE_IMAGE="gcr.io/gcp-runtimes/ubuntu_16_0_4:latest" fi export STAGING_IMAGE="${DOCKER_NAMESPACE}/python:${TAG}" echo "Using base image name ${STAGING_IMAGE}" diff --git a/runtime-image/Dockerfile.in b/runtime-image/Dockerfile.in index 705a915b..84cb5267 100644 --- a/runtime-image/Dockerfile.in +++ b/runtime-image/Dockerfile.in @@ -28,21 +28,21 @@ RUN wget https://storage.googleapis.com/python-interpreters/latest/interpreter-3 # Add Google-built interpreters to the path ENV PATH /opt/python3.7/bin:/opt/python3.6/bin:/opt/python3.5/bin:/opt/python3.4/bin:$PATH -RUN update-alternatives --install /usr/local/bin/python3 python3 /opt/python3.6/bin/python3.6 50 -RUN update-alternatives --install /usr/local/bin/pip3 pip3 /opt/python3.6/bin/pip3.6 50 +RUN update-alternatives --install /usr/local/bin/python3 python3 /opt/python3.7/bin/python3.7 50 +RUN update-alternatives --install /usr/local/bin/pip3 pip3 /opt/python3.7/bin/pip3.7 50 # Upgrade pip (debian package version tends to run a few version behind) and # install virtualenv system-wide. -RUN /usr/bin/pip install --upgrade -r /resources/requirements.txt && \ - /opt/python3.4/bin/pip3.4 install --upgrade -r /resources/requirements.txt && \ - rm -f /opt/python3.4/bin/pip /opt/python3.4/bin/pip3 && \ - /opt/python3.5/bin/pip3.5 install --upgrade -r /resources/requirements.txt && \ - rm -f /opt/python3.5/bin/pip /opt/python3.5/bin/pip3 && \ - /opt/python3.6/bin/pip3.6 install --upgrade -r /resources/requirements.txt && \ - rm -f /opt/python3.6/bin/pip /opt/python3.6/bin/pip3 && \ - /opt/python3.7/bin/pip3.7 install --upgrade -r /resources/requirements.txt && \ - rm -f /opt/python3.7/bin/pip /opt/python3.7/bin/pip3 && \ - /usr/bin/pip install --upgrade -r /resources/requirements-virtualenv.txt +RUN /usr/bin/pip install --upgrade -r /resources/requirements.txt +RUN /opt/python3.4/bin/pip3.4 install --upgrade -r /resources/requirements.txt +RUN rm -f /opt/python3.4/bin/pip /opt/python3.4/bin/pip3 +RUN /opt/python3.5/bin/pip3.5 install --upgrade -r /resources/requirements.txt +RUN rm -f /opt/python3.5/bin/pip /opt/python3.5/bin/pip3 +#RUN /opt/python3.6/bin/pip3.6 install --upgrade -r /resources/requirements.txt +#RUN rm -f /opt/python3.6/bin/pip /opt/python3.6/bin/pip3 +RUN /opt/python3.7/bin/pip3.7 install --upgrade -r /resources/requirements.txt +RUN rm -f /opt/python3.7/bin/pip /opt/python3.7/bin/pip3 +RUN /usr/bin/pip install --upgrade -r /resources/requirements-virtualenv.txt # Setup the app working directory RUN ln -s /home/vmagent/app /app From 52a469c2fe4f6bfb60fb36bad72e57fd184b8e76 Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Tue, 22 Jan 2019 10:28:05 -0800 Subject: [PATCH 190/256] Updating dockerfile to reduce RUN commands --- runtime-image/Dockerfile.in | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/runtime-image/Dockerfile.in b/runtime-image/Dockerfile.in index 84cb5267..dc842d9b 100644 --- a/runtime-image/Dockerfile.in +++ b/runtime-image/Dockerfile.in @@ -28,21 +28,21 @@ RUN wget https://storage.googleapis.com/python-interpreters/latest/interpreter-3 # Add Google-built interpreters to the path ENV PATH /opt/python3.7/bin:/opt/python3.6/bin:/opt/python3.5/bin:/opt/python3.4/bin:$PATH -RUN update-alternatives --install /usr/local/bin/python3 python3 /opt/python3.7/bin/python3.7 50 -RUN update-alternatives --install /usr/local/bin/pip3 pip3 /opt/python3.7/bin/pip3.7 50 +RUN update-alternatives --install /usr/local/bin/python3 python3 /opt/python3.7/bin/python3.7 50 && \ + update-alternatives --install /usr/local/bin/pip3 pip3 /opt/python3.7/bin/pip3.7 50 # Upgrade pip (debian package version tends to run a few version behind) and # install virtualenv system-wide. -RUN /usr/bin/pip install --upgrade -r /resources/requirements.txt -RUN /opt/python3.4/bin/pip3.4 install --upgrade -r /resources/requirements.txt -RUN rm -f /opt/python3.4/bin/pip /opt/python3.4/bin/pip3 -RUN /opt/python3.5/bin/pip3.5 install --upgrade -r /resources/requirements.txt -RUN rm -f /opt/python3.5/bin/pip /opt/python3.5/bin/pip3 -#RUN /opt/python3.6/bin/pip3.6 install --upgrade -r /resources/requirements.txt -#RUN rm -f /opt/python3.6/bin/pip /opt/python3.6/bin/pip3 -RUN /opt/python3.7/bin/pip3.7 install --upgrade -r /resources/requirements.txt -RUN rm -f /opt/python3.7/bin/pip /opt/python3.7/bin/pip3 -RUN /usr/bin/pip install --upgrade -r /resources/requirements-virtualenv.txt +RUN /usr/bin/pip install --upgrade -r /resources/requirements.txt && \ + /opt/python3.4/bin/pip3.4 install --upgrade -r /resources/requirements.txt && \ + rm -f /opt/python3.4/bin/pip /opt/python3.4/bin/pip3 && \ + /opt/python3.5/bin/pip3.5 install --upgrade -r /resources/requirements.txt && \ + rm -f /opt/python3.5/bin/pip /opt/python3.5/bin/pip3 && \ + /opt/python3.6/bin/pip3.6 install --upgrade -r /resources/requirements.txt && \ + rm -f /opt/python3.6/bin/pip /opt/python3.6/bin/pip3 && \ + /opt/python3.7/bin/pip3.7 install --upgrade -r /resources/requirements.txt && \ + rm -f /opt/python3.7/bin/pip /opt/python3.7/bin/pip3 && \ + /usr/bin/pip install --upgrade -r /resources/requirements-virtualenv.txt # Setup the app working directory RUN ln -s /home/vmagent/app /app From 9701e57eb1deb40e98caa9ca1dc62d55154d69ba Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Tue, 22 Jan 2019 11:16:28 -0800 Subject: [PATCH 191/256] Update python to 3.6.8 and 3.7.2 defaulting to 3.7 - taketwo (#212) * Updating python to 3.6.8 and 3.7.2 and defaulting to 3.7 * fixing 3.7 virtualenv tests * fixing up defaulting to 3.7 * Updating build's base image from debian8 to ubuntu_16_0_4 to address SSL issues with 3.6.8 and 3.7.2 build Updating Dockerfile to update_alternatives to 3.7.2 Updating Dockerfile to separate installs so python install errors are easier to differentiate * Updating dockerfile to reduce RUN commands --- build.sh | 8 +-- builder/gen-dockerfile/Dockerfile.in | 4 +- cloudbuild_test.yaml | 7 +++ .../scripts/build-python-3.6.sh | 12 ++--- .../scripts/build-python-3.7.sh | 12 ++--- runtime-image/Dockerfile.in | 8 +-- scripts/gen_dockerfile.py | 2 +- scripts/gen_dockerfile_test.py | 5 +- .../testdata/hello_world_golden/Dockerfile | 4 +- tests/no-virtualenv/no-virtualenv.yaml | 2 +- tests/virtualenv/virtualenv_python36.yaml | 2 +- tests/virtualenv/virtualenv_python37.yaml | 54 +++++++++++++++++++ 12 files changed, 92 insertions(+), 28 deletions(-) create mode 100644 tests/virtualenv/virtualenv_python37.yaml diff --git a/build.sh b/build.sh index bdc4f658..63afa1ab 100755 --- a/build.sh +++ b/build.sh @@ -24,7 +24,7 @@ test=0 # Should run standard test suite? local=0 # Should run using local Docker daemon instead of GCR? -os_base=debian8 # Which operating system base to use +os_base=ubuntu16 # Which operating system base to use interpreter=0 # Should build interpreters instead of images # Note that $gcloud_cmd has spaces in it @@ -154,10 +154,10 @@ if [ "${local}" -eq 1 ]; then fi # Pick OS image to use as base -if [ "${os_base}" == "ubuntu16" ]; then - export OS_BASE_IMAGE="gcr.io/gcp-runtimes/ubuntu_16_0_4:latest" -else +if [ "${os_base}" == "debian8" ]; then export OS_BASE_IMAGE="gcr.io/google-appengine/debian8:latest" +else + export OS_BASE_IMAGE="gcr.io/gcp-runtimes/ubuntu_16_0_4:latest" fi export STAGING_IMAGE="${DOCKER_NAMESPACE}/python:${TAG}" echo "Using base image name ${STAGING_IMAGE}" diff --git a/builder/gen-dockerfile/Dockerfile.in b/builder/gen-dockerfile/Dockerfile.in index 4f6447eb..627151bf 100644 --- a/builder/gen-dockerfile/Dockerfile.in +++ b/builder/gen-dockerfile/Dockerfile.in @@ -1,6 +1,6 @@ FROM ${STAGING_IMAGE} -LABEL python_version=python3.6 -RUN virtualenv --no-download /env -p python3.6 +LABEL python_version=python3.7 +RUN virtualenv --no-download /env -p python3.7 # Set virtualenv environment variables. This is equivalent to running # source /env/bin/activate diff --git a/cloudbuild_test.yaml b/cloudbuild_test.yaml index 416c5204..ad674026 100644 --- a/cloudbuild_test.yaml +++ b/cloudbuild_test.yaml @@ -43,6 +43,13 @@ steps: '/workspace/tests/virtualenv/virtualenv_python36.yaml', ] waitFor: ['runtime'] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 + args: [ + '-test.v', + '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', + '/workspace/tests/virtualenv/virtualenv_python37.yaml', + ] + waitFor: ['runtime'] - name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 args: [ '-test.v', diff --git a/python-interpreter-builder/scripts/build-python-3.6.sh b/python-interpreter-builder/scripts/build-python-3.6.sh index b3896e89..73912401 100755 --- a/python-interpreter-builder/scripts/build-python-3.6.sh +++ b/python-interpreter-builder/scripts/build-python-3.6.sh @@ -6,14 +6,14 @@ set -x # Get the source mkdir -p /opt/sources cd /opt/sources -wget --no-verbose https://www.python.org/ftp/python/3.6.7/Python-3.6.7.tgz +wget --no-verbose https://www.python.org/ftp/python/3.6.8/Python-3.6.8.tgz # SHA-256 generated via `shasum -a 256 [file]` shasum --check < Date: Wed, 30 Jan 2019 11:40:26 -0800 Subject: [PATCH 192/256] Fixing doc to reflect correct base image --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 51413044..51e70cd8 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ for running applications on [Google Kubernetes Engine](https://cloud.google.com/kubernetes-engine), or any other Docker host. -This image is based on Debian Jessie and contains packages required to build +This image is based on Ubuntu Xenial and contains packages required to build most of the popular Python libraries. For more information about this runtime, see the [documentation](https://cloud.google.com/appengine/docs/flexible/python/runtime). From cbc6240d561e3727c42a252bf09caa3d89c5a0b0 Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Fri, 1 Feb 2019 16:08:12 -0800 Subject: [PATCH 193/256] Revert default python3 version to 3.6 (#215) --- builder/gen-dockerfile/Dockerfile.in | 4 ++-- scripts/gen_dockerfile.py | 2 +- scripts/gen_dockerfile_test.py | 2 +- scripts/testdata/hello_world_golden/Dockerfile | 4 ++-- tests/no-virtualenv/no-virtualenv.yaml | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/builder/gen-dockerfile/Dockerfile.in b/builder/gen-dockerfile/Dockerfile.in index 627151bf..4f6447eb 100644 --- a/builder/gen-dockerfile/Dockerfile.in +++ b/builder/gen-dockerfile/Dockerfile.in @@ -1,6 +1,6 @@ FROM ${STAGING_IMAGE} -LABEL python_version=python3.7 -RUN virtualenv --no-download /env -p python3.7 +LABEL python_version=python3.6 +RUN virtualenv --no-download /env -p python3.6 # Set virtualenv environment variables. This is equivalent to running # source /env/bin/activate diff --git a/scripts/gen_dockerfile.py b/scripts/gen_dockerfile.py index 852405d8..97da60f6 100755 --- a/scripts/gen_dockerfile.py +++ b/scripts/gen_dockerfile.py @@ -50,7 +50,7 @@ PYTHON_INTERPRETER_VERSION_MAP = { '': '', # == 2.7 '2': '', # == 2.7 - '3': '3.7', + '3': '3.6', '3.4': '3.4', '3.5': '3.5', '3.6': '3.6', diff --git a/scripts/gen_dockerfile_test.py b/scripts/gen_dockerfile_test.py index c78c6ad8..03eaa079 100755 --- a/scripts/gen_dockerfile_test.py +++ b/scripts/gen_dockerfile_test.py @@ -71,7 +71,7 @@ def compare_file(filename, dir1, dir2): 'dockerfile_python_version': '', }), ('runtime_config:\n python_version: 3', { - 'dockerfile_python_version': '3.7', + 'dockerfile_python_version': '3.6', }), ('runtime_config:\n python_version: 3.4', { 'dockerfile_python_version': '3.4', diff --git a/scripts/testdata/hello_world_golden/Dockerfile b/scripts/testdata/hello_world_golden/Dockerfile index 55eb8cec..10396399 100644 --- a/scripts/testdata/hello_world_golden/Dockerfile +++ b/scripts/testdata/hello_world_golden/Dockerfile @@ -1,6 +1,6 @@ FROM gcr.io/google-appengine/python -LABEL python_version=python3.7 -RUN virtualenv --no-download /env -p python3.7 +LABEL python_version=python3.6 +RUN virtualenv --no-download /env -p python3.6 # Set virtualenv environment variables. This is equivalent to running # source /env/bin/activate diff --git a/tests/no-virtualenv/no-virtualenv.yaml b/tests/no-virtualenv/no-virtualenv.yaml index c8350e4e..6a3b1e7d 100644 --- a/tests/no-virtualenv/no-virtualenv.yaml +++ b/tests/no-virtualenv/no-virtualenv.yaml @@ -43,7 +43,7 @@ commandTests: expectedOutput: ["/usr/local/bin/python3\n"] - name: "default python3 version" command: ["python3", "--version"] - expectedOutput: ["Python 3.7.2\n"] + expectedOutput: ["Python 3.6.8\n"] - name: "default pip3 installation" command: ["which", "pip3"] expectedOutput: ["/usr/local/bin/pip3\n"] From d628b583eb2f5684d02a441b99d08f8b7310f79a Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Mon, 4 Feb 2019 14:31:15 -0800 Subject: [PATCH 194/256] Make python 3.6 the default python 3 version (#216) --- runtime-image/Dockerfile.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runtime-image/Dockerfile.in b/runtime-image/Dockerfile.in index dc842d9b..79862620 100644 --- a/runtime-image/Dockerfile.in +++ b/runtime-image/Dockerfile.in @@ -28,8 +28,8 @@ RUN wget https://storage.googleapis.com/python-interpreters/latest/interpreter-3 # Add Google-built interpreters to the path ENV PATH /opt/python3.7/bin:/opt/python3.6/bin:/opt/python3.5/bin:/opt/python3.4/bin:$PATH -RUN update-alternatives --install /usr/local/bin/python3 python3 /opt/python3.7/bin/python3.7 50 && \ - update-alternatives --install /usr/local/bin/pip3 pip3 /opt/python3.7/bin/pip3.7 50 +RUN update-alternatives --install /usr/local/bin/python3 python3 /opt/python3.6/bin/python3.6 50 && \ + update-alternatives --install /usr/local/bin/pip3 pip3 /opt/python3.6/bin/pip3.6 50 # Upgrade pip (debian package version tends to run a few version behind) and # install virtualenv system-wide. From adacb0fd5342590935ce76a9c480cd9bbf89a8bf Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Wed, 6 Feb 2019 13:59:05 -0800 Subject: [PATCH 195/256] explicitly run apt-get upgrade to get latest versions of packages (#217) --- runtime-image/scripts/install-apt-packages.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/runtime-image/scripts/install-apt-packages.sh b/runtime-image/scripts/install-apt-packages.sh index 9d23cab7..bafba2d6 100755 --- a/runtime-image/scripts/install-apt-packages.sh +++ b/runtime-image/scripts/install-apt-packages.sh @@ -7,6 +7,8 @@ apt-get -q update xargs -a <(awk '/^\s*[^#]/' '/resources/apt-packages.txt') -r -- \ apt-get install --no-install-recommends -yq +apt-get upgrade -yq + # Remove unneeded files. apt-get clean rm /var/lib/apt/lists/*_* From 272f8d9ebd311b9c4466a252cc6a71223317c7f2 Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Wed, 6 Feb 2019 13:59:16 -0800 Subject: [PATCH 196/256] add donmccasland to CODEOWNERS (#218) --- CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CODEOWNERS b/CODEOWNERS index f8b549f9..ea26d5dc 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1,4 +1,4 @@ # Code owners file. # This file controls who is tagged for review for any given pull request. -* @dlorenc @sharifelgamal @nkubala @tstromberg +* @dlorenc @sharifelgamal @nkubala @tstromberg @donmccasland From 953a810d823e99c45c43b19a9c960c426c114051 Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Thu, 4 Apr 2019 13:54:19 -0700 Subject: [PATCH 197/256] Adding build scripts --- scripts/deploy_check.sh | 34 +++++++++++++++++++++ scripts/integration-test.sh | 61 +++++++++++++++++++++++++++++++++++++ scripts/release.sh | 18 +++++++++++ 3 files changed, 113 insertions(+) create mode 100644 scripts/deploy_check.sh create mode 100644 scripts/integration-test.sh create mode 100644 scripts/release.sh diff --git a/scripts/deploy_check.sh b/scripts/deploy_check.sh new file mode 100644 index 00000000..77fccb54 --- /dev/null +++ b/scripts/deploy_check.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +set -ex + +export KOKORO_GITHUB_DIR=${KOKORO_ROOT}/src/github +source ${KOKORO_GFILE_DIR}/kokoro/common.sh + +cd ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY} +if [ -n "${RUNTIME_SPEC}" -a -f app.yaml.in ]; then + sed "s|\${RUNTIME_SPEC}|${RUNTIME_SPEC}|" app.yaml.in > app.yaml +fi + +cd ${KOKORO_GFILE_DIR}/appengine/integration_tests + +sudo /usr/local/bin/pip install --upgrade -r requirements.txt + +if [ -f ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY}/requirements.txt ] +then + sudo /usr/local/bin/pip install --upgrade -r ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY}/requirements.txt +fi + +export DEPLOY_LATENCY_PROJECT='cloud-deploy-latency' + +skip_flag="" + +if [ "${SKIP_CUSTOM_LOGGING_TESTS}" = "true" -o "${SKIP_BUILDERS}" = "true" ]; then + skip_flag="$skip_flag --skip-builders" +fi + +if [ "${SKIP_XRT}" = "true" ]; then + skip_flag="$skip_flag --skip-xrt" +fi + +python deploy_check.py -d ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY} -l ${LANGUAGE} ${skip_flag} diff --git a/scripts/integration-test.sh b/scripts/integration-test.sh new file mode 100644 index 00000000..63137b5f --- /dev/null +++ b/scripts/integration-test.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +set -ex + +export KOKORO_GITHUB_DIR=${KOKORO_ROOT}/src/github +source ${KOKORO_GFILE_DIR}/kokoro/common.sh + +export GOOGLE_CLOUD_PROJECT=gcp-runtimes + +sudo /usr/local/bin/pip install --upgrade -r ${KOKORO_GFILE_DIR}/appengine/integration_tests/requirements.txt + +if [ -f ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY}/requirements.txt ] +then + sudo /usr/local/bin/pip install --upgrade -r ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY}/requirements.txt +fi + +export GOPATH=${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY} + +flags="" + +if [ -n "${STAGING_IMAGE}" ]; then + flags="$flags -i ${STAGING_IMAGE}" +fi + +if [ "${SKIP_STANDARD_LOGGING_TESTS}" = "true" ]; then + flags="$flags --skip-standard-logging-tests" +fi + +if [ "${SKIP_CUSTOM_LOGGING_TESTS}" = "true" ]; then + flags="$flags --skip-custom-logging-tests" +fi + +if [ "${SKIP_MONITORING_TESTS}" = "true" ]; then + flags="$flags --skip-monitoring-tests" +fi + +if [ "${SKIP_EXCEPTION_TESTS}" = "true" ]; then + flags="$flags --skip-exception-tests" +fi + +if [ "${SKIP_CUSTOM_TESTS}" = "true" ]; then + flags="$flags --skip-custom-tests" +fi + +if [ -n "${URL}" ]; then + flags="$flags --url ${URL}" +fi + +if [ -n "${BUILDER}" ]; then + flags="$flags --builder ${BUILDER}" + gcloud config set app/use_runtime_builders True + gcloud config set app/runtime_builders_root file://${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY} +fi + +if [ -n "${YAML}" ]; then + flags="$flags --yaml ${KOKORO_GITHUB_DIR}/${YAML}" +fi + + +chmod a+x ${KOKORO_GFILE_DIR}/appengine/integration_tests/testsuite/driver.py +${KOKORO_GFILE_DIR}/appengine/integration_tests/testsuite/driver.py -d ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY} ${flags} diff --git a/scripts/release.sh b/scripts/release.sh new file mode 100644 index 00000000..1857deaf --- /dev/null +++ b/scripts/release.sh @@ -0,0 +1,18 @@ +#!/bin/bash +set -euo pipefail +export KOKORO_GITHUB_DIR=${KOKORO_ROOT}/src/github +source ${KOKORO_GFILE_DIR}/kokoro/common.sh + +source "${KOKORO_PIPER_DIR}/google3/third_party/runtimes_common/kokoro/common.sh" + +cd ${KOKORO_GITHUB_DIR}/python-runtime + +if [ -z "${TAG:+set}" ]; then + export TAG=$(date +%Y-%m-%d-%H%M%S) +fi + +./build.sh $BUILD_FLAGS + +METADATA=$(pwd)/METADATA +cd ${KOKORO_GFILE_DIR}/kokoro +python note.py python -m ${METADATA} -t ${TAG} From a1339d625a87da959d7aa7ddfa35ccce421a1ef8 Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Mon, 8 Apr 2019 08:59:33 -0700 Subject: [PATCH 198/256] Fixing release script --- scripts/release.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/release.sh b/scripts/release.sh index 1857deaf..cd499823 100644 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -3,8 +3,6 @@ set -euo pipefail export KOKORO_GITHUB_DIR=${KOKORO_ROOT}/src/github source ${KOKORO_GFILE_DIR}/kokoro/common.sh -source "${KOKORO_PIPER_DIR}/google3/third_party/runtimes_common/kokoro/common.sh" - cd ${KOKORO_GITHUB_DIR}/python-runtime if [ -z "${TAG:+set}" ]; then From c6864c03b87842d891f362559a73e0181c2e0c69 Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Tue, 17 Sep 2019 15:38:46 -0700 Subject: [PATCH 199/256] Updating integration test pip requirement: gcloud-logging version 1.12.1 --- tests/integration/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index 4a462433..ac718c60 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -1,6 +1,6 @@ Flask==1.0.2 google-cloud-error-reporting==0.30.0 -google-cloud-logging==1.6.0 +google-cloud-logging==1.12.1 google-cloud-monitoring==0.30.1 gunicorn==19.9.0 requests==2.19.1 From 7fc2e28c8ce121cbfa899f7852a54e5795818571 Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Tue, 17 Sep 2019 15:52:28 -0700 Subject: [PATCH 200/256] Updating integration test pip requirements --- tests/integration/requirements.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index ac718c60..73a48913 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -1,8 +1,8 @@ -Flask==1.0.2 -google-cloud-error-reporting==0.30.0 +Flask==1.1.0 +google-cloud-error-reporting==0.32.1 google-cloud-logging==1.12.1 -google-cloud-monitoring==0.30.1 +google-cloud-monitoring==0.33.0 gunicorn==19.9.0 -requests==2.19.1 +requests==2.22.0 retrying==1.3.3 -six==1.11.0 +six==1.12.0 From 17a4a6c3cec29fb772e8280ac13d23a53d30d8a8 Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Wed, 18 Sep 2019 10:59:59 -0700 Subject: [PATCH 201/256] Adding protobuf requirement so kokoro will upgrade it --- tests/integration/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index 73a48913..ea714c20 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -6,3 +6,4 @@ gunicorn==19.9.0 requests==2.22.0 retrying==1.3.3 six==1.12.0 +protobuf>=3.6.0 From 87a309ee632ade17162d85211e4fa0164ab613af Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Thu, 5 Mar 2020 09:49:48 -0800 Subject: [PATCH 202/256] update python 3 to use python 3.7 --- runtime-image/Dockerfile.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runtime-image/Dockerfile.in b/runtime-image/Dockerfile.in index 79862620..dc842d9b 100644 --- a/runtime-image/Dockerfile.in +++ b/runtime-image/Dockerfile.in @@ -28,8 +28,8 @@ RUN wget https://storage.googleapis.com/python-interpreters/latest/interpreter-3 # Add Google-built interpreters to the path ENV PATH /opt/python3.7/bin:/opt/python3.6/bin:/opt/python3.5/bin:/opt/python3.4/bin:$PATH -RUN update-alternatives --install /usr/local/bin/python3 python3 /opt/python3.6/bin/python3.6 50 && \ - update-alternatives --install /usr/local/bin/pip3 pip3 /opt/python3.6/bin/pip3.6 50 +RUN update-alternatives --install /usr/local/bin/python3 python3 /opt/python3.7/bin/python3.7 50 && \ + update-alternatives --install /usr/local/bin/pip3 pip3 /opt/python3.7/bin/pip3.7 50 # Upgrade pip (debian package version tends to run a few version behind) and # install virtualenv system-wide. From fbc2908de36bb94bb00dc4a25c80a6113c1bf231 Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Fri, 27 Mar 2020 10:01:17 -0700 Subject: [PATCH 203/256] Updating python 3,7 to 3.7.7 --- .../scripts/build-python-3.7.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/python-interpreter-builder/scripts/build-python-3.7.sh b/python-interpreter-builder/scripts/build-python-3.7.sh index 69febc88..3b3bdae5 100755 --- a/python-interpreter-builder/scripts/build-python-3.7.sh +++ b/python-interpreter-builder/scripts/build-python-3.7.sh @@ -6,14 +6,14 @@ set -x # Get the source mkdir -p /opt/sources cd /opt/sources -wget --no-verbose https://www.python.org/ftp/python/3.7.2/Python-3.7.2.tgz +wget --no-verbose https://www.python.org/ftp/python/3.7.7/Python-3.7.7.tgz # SHA-256 generated via `shasum -a 256 [file]` shasum --check < Date: Fri, 27 Mar 2020 10:43:39 -0700 Subject: [PATCH 204/256] Remove nkubala from owners (#233) --- CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CODEOWNERS b/CODEOWNERS index ea26d5dc..5deacc6c 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1,4 +1,4 @@ # Code owners file. # This file controls who is tagged for review for any given pull request. -* @dlorenc @sharifelgamal @nkubala @tstromberg @donmccasland +* @dlorenc @sharifelgamal @tstromberg @donmccasland From f6c18ed7aa36675eed95757ff8627871256e4be6 Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Fri, 27 Mar 2020 10:47:56 -0700 Subject: [PATCH 205/256] Fixing test --- tests/no-virtualenv/no-virtualenv.yaml | 2 +- tests/virtualenv/virtualenv_python37.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/no-virtualenv/no-virtualenv.yaml b/tests/no-virtualenv/no-virtualenv.yaml index 6a3b1e7d..19e48df9 100644 --- a/tests/no-virtualenv/no-virtualenv.yaml +++ b/tests/no-virtualenv/no-virtualenv.yaml @@ -43,7 +43,7 @@ commandTests: expectedOutput: ["/usr/local/bin/python3\n"] - name: "default python3 version" command: ["python3", "--version"] - expectedOutput: ["Python 3.6.8\n"] + expectedOutput: ["Python 3.7.7\n"] - name: "default pip3 installation" command: ["which", "pip3"] expectedOutput: ["/usr/local/bin/pip3\n"] diff --git a/tests/virtualenv/virtualenv_python37.yaml b/tests/virtualenv/virtualenv_python37.yaml index 457933f9..7de6d10c 100644 --- a/tests/virtualenv/virtualenv_python37.yaml +++ b/tests/virtualenv/virtualenv_python37.yaml @@ -25,7 +25,7 @@ commandTests: - name: "virtualenv37 python version" setup: [["virtualenv", "-p", "python3.7", "/env"]] command: ["python", "--version"] - expectedOutput: ["Python 3.7.2\n"] + expectedOutput: ["Python 3.7.7\n"] - name: "virtualenv37 pip installation" setup: [["virtualenv", "-p", "python3.7", "/env"]] From a0d1f9def6b296f789023b7973570d6aba857e0d Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Sat, 28 Mar 2020 20:18:17 -0700 Subject: [PATCH 206/256] Update interpreter builders with new python versions --- .../scripts/build-python-3.5.sh | 12 ++++++------ .../scripts/build-python-3.6.sh | 12 ++++++------ .../scripts/build-python-3.7.sh | 2 +- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/python-interpreter-builder/scripts/build-python-3.5.sh b/python-interpreter-builder/scripts/build-python-3.5.sh index 86a564c3..c2fd748b 100755 --- a/python-interpreter-builder/scripts/build-python-3.5.sh +++ b/python-interpreter-builder/scripts/build-python-3.5.sh @@ -6,14 +6,14 @@ set -x # Get the source mkdir -p /opt/sources cd /opt/sources -wget --no-verbose https://www.python.org/ftp/python/3.5.5/Python-3.5.5.tgz +wget --no-verbose https://www.python.org/ftp/python/3.5.9/Python-3.5.9.tgz # SHA-256 generated via `shasum -a 256 [file]` shasum --check < Date: Sat, 28 Mar 2020 20:20:28 -0700 Subject: [PATCH 207/256] Update tests --- tests/virtualenv/virtualenv_python35.yaml | 2 +- tests/virtualenv/virtualenv_python36.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/virtualenv/virtualenv_python35.yaml b/tests/virtualenv/virtualenv_python35.yaml index 5f37ee54..3bb3d814 100644 --- a/tests/virtualenv/virtualenv_python35.yaml +++ b/tests/virtualenv/virtualenv_python35.yaml @@ -25,7 +25,7 @@ commandTests: - name: "virtualenv35 python version" setup: [["virtualenv", "-p", "python3.5", "/env"]] command: ["python", "--version"] - expectedOutput: ["Python 3.5.5\n"] + expectedOutput: ["Python 3.5.9\n"] - name: "virtualenv35 pip installation" setup: [["virtualenv", "-p", "python3.5", "/env"]] diff --git a/tests/virtualenv/virtualenv_python36.yaml b/tests/virtualenv/virtualenv_python36.yaml index 64ba2596..f0949c6f 100644 --- a/tests/virtualenv/virtualenv_python36.yaml +++ b/tests/virtualenv/virtualenv_python36.yaml @@ -25,7 +25,7 @@ commandTests: - name: "virtualenv36 python version" setup: [["virtualenv", "-p", "python3.6", "/env"]] command: ["python", "--version"] - expectedOutput: ["Python 3.6.8\n"] + expectedOutput: ["Python 3.6.10\n"] - name: "virtualenv36 pip installation" setup: [["virtualenv", "-p", "python3.6", "/env"]] From 641e22349c6937e224ca8c7634de4399d1693bdf Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Sun, 29 Mar 2020 16:19:46 -0700 Subject: [PATCH 208/256] Fix 3.5 test failure --- tests/python3-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 3b283d1d..e3fd6424 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -77,7 +77,7 @@ lxml==4.2.4 mako==1.0.7 manifestparser==1.1 markdown==2.6.11 -markupsafe==1.0 +markupsafe==1.1.1 matplotlib==2.2.3 mccabe==0.6.1 meld3==1.0.2 From 3d5d38460ca4beac31a5b4cb35adaa25c31670d1 Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Mon, 30 Mar 2020 10:12:28 -0700 Subject: [PATCH 209/256] Attempting upgrade of pip version --- runtime-image/resources/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime-image/resources/requirements.txt b/runtime-image/resources/requirements.txt index 298e5f41..7ec375ff 100644 --- a/runtime-image/resources/requirements.txt +++ b/runtime-image/resources/requirements.txt @@ -1,5 +1,5 @@ # Do not update pip, it conflicts with the OS system installed version. # https://github.com/pypa/pip/issues/5240 -pip==9.0.3 +pip==10.0.1 setuptools==40.2.0 wheel==0.31.1 From bf65b6947c2e9f63a260f95d121ca07926361315 Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Mon, 30 Mar 2020 10:27:04 -0700 Subject: [PATCH 210/256] Latest pip is actually 20.0.2 --- runtime-image/resources/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime-image/resources/requirements.txt b/runtime-image/resources/requirements.txt index 7ec375ff..b374c0e8 100644 --- a/runtime-image/resources/requirements.txt +++ b/runtime-image/resources/requirements.txt @@ -1,5 +1,5 @@ # Do not update pip, it conflicts with the OS system installed version. # https://github.com/pypa/pip/issues/5240 -pip==10.0.1 +pip==20.0.2 setuptools==40.2.0 wheel==0.31.1 From cc6436428ad613f8e311045a26b7b54c6b5ad442 Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Mon, 30 Mar 2020 11:08:46 -0700 Subject: [PATCH 211/256] Looks like the latest available is 19.1.1 --- runtime-image/resources/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime-image/resources/requirements.txt b/runtime-image/resources/requirements.txt index b374c0e8..0032013d 100644 --- a/runtime-image/resources/requirements.txt +++ b/runtime-image/resources/requirements.txt @@ -1,5 +1,5 @@ # Do not update pip, it conflicts with the OS system installed version. # https://github.com/pypa/pip/issues/5240 -pip==20.0.2 +pip==19.1.1 setuptools==40.2.0 wheel==0.31.1 From 1f2f63dcad36e63ee0ced458cda7d53922290350 Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Tue, 31 Mar 2020 13:27:33 -0700 Subject: [PATCH 212/256] Letting pip go to latest to address python2 pip incompatibility issues. --- runtime-image/resources/requirements.txt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/runtime-image/resources/requirements.txt b/runtime-image/resources/requirements.txt index 0032013d..2d010eef 100644 --- a/runtime-image/resources/requirements.txt +++ b/runtime-image/resources/requirements.txt @@ -1,5 +1,3 @@ -# Do not update pip, it conflicts with the OS system installed version. -# https://github.com/pypa/pip/issues/5240 -pip==19.1.1 +pip setuptools==40.2.0 wheel==0.31.1 From e212b243482b17196725eaaf59c25ced9fd7960b Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Tue, 18 Aug 2020 09:15:39 -0700 Subject: [PATCH 213/256] Updating to 3.7.9 --- .../scripts/build-python-3.7.sh | 12 ++++++------ tests/virtualenv/virtualenv_python37.yaml | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/python-interpreter-builder/scripts/build-python-3.7.sh b/python-interpreter-builder/scripts/build-python-3.7.sh index bee33cef..b1b06806 100755 --- a/python-interpreter-builder/scripts/build-python-3.7.sh +++ b/python-interpreter-builder/scripts/build-python-3.7.sh @@ -6,14 +6,14 @@ set -x # Get the source mkdir -p /opt/sources cd /opt/sources -wget --no-verbose https://www.python.org/ftp/python/3.7.7/Python-3.7.7.tgz +wget --no-verbose https://www.python.org/ftp/python/3.7.9/Python-3.7.9.tgz # SHA-256 generated via `shasum -a 256 [file]` shasum --check < Date: Thu, 3 Sep 2020 12:56:15 -0700 Subject: [PATCH 214/256] Remove virtualenv pip before installing requirements --- builder/gen-dockerfile/Dockerfile.in | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/builder/gen-dockerfile/Dockerfile.in b/builder/gen-dockerfile/Dockerfile.in index 4f6447eb..98501c77 100644 --- a/builder/gen-dockerfile/Dockerfile.in +++ b/builder/gen-dockerfile/Dockerfile.in @@ -7,7 +7,9 @@ RUN virtualenv --no-download /env -p python3.6 ENV VIRTUAL_ENV /env ENV PATH /env/bin:$PATH ADD requirements.txt /builder/ -RUN pip install -r /builder/requirements.txt +#virtualenv's pip is pegged at version 10.0, removing so +#newer versions get picked up +RUN rm -f /env/bin/pip* && pip install -r /builder/requirements.txt ADD . /builder/ WORKDIR /workspace ENTRYPOINT [ "python", "/builder/gen_dockerfile.py" ] From c9f59421f0064de12c45eb258635fe591d182907 Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Thu, 3 Sep 2020 13:53:37 -0700 Subject: [PATCH 215/256] Fix 3.7.9 test --- tests/no-virtualenv/no-virtualenv.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/no-virtualenv/no-virtualenv.yaml b/tests/no-virtualenv/no-virtualenv.yaml index 19e48df9..4f6c3f48 100644 --- a/tests/no-virtualenv/no-virtualenv.yaml +++ b/tests/no-virtualenv/no-virtualenv.yaml @@ -43,7 +43,7 @@ commandTests: expectedOutput: ["/usr/local/bin/python3\n"] - name: "default python3 version" command: ["python3", "--version"] - expectedOutput: ["Python 3.7.7\n"] + expectedOutput: ["Python 3.7.9\n"] - name: "default pip3 installation" command: ["which", "pip3"] expectedOutput: ["/usr/local/bin/pip3\n"] From 71223557e38a88abc8b365d1bb870c5d4132e864 Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Thu, 10 Sep 2020 13:53:38 -0700 Subject: [PATCH 216/256] Update virtualenv version --- runtime-image/resources/requirements-virtualenv.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime-image/resources/requirements-virtualenv.txt b/runtime-image/resources/requirements-virtualenv.txt index 52bb7da1..25f09c4a 100644 --- a/runtime-image/resources/requirements-virtualenv.txt +++ b/runtime-image/resources/requirements-virtualenv.txt @@ -1 +1 @@ -virtualenv==16.0.0 +virtualenv==20.0.31 From 92a7aba5365a784aaf66192f86bfa5912f56d405 Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Thu, 10 Sep 2020 17:19:11 -0700 Subject: [PATCH 217/256] Updating virtualenv, test, and python builder version --- builder/gen-dockerfile/Dockerfile.in | 4 ++-- tests/eventlet/requirements.txt | 2 +- tests/virtualenv/virtualenv_default.yaml | 2 +- tests/virtualenv/virtualenv_python27.yaml | 2 +- tests/virtualenv/virtualenv_python34.yaml | 2 +- tests/virtualenv/virtualenv_python35.yaml | 2 +- tests/virtualenv/virtualenv_python36.yaml | 2 +- tests/virtualenv/virtualenv_python37.yaml | 2 +- 8 files changed, 9 insertions(+), 9 deletions(-) diff --git a/builder/gen-dockerfile/Dockerfile.in b/builder/gen-dockerfile/Dockerfile.in index 98501c77..4ddce7c2 100644 --- a/builder/gen-dockerfile/Dockerfile.in +++ b/builder/gen-dockerfile/Dockerfile.in @@ -1,6 +1,6 @@ FROM ${STAGING_IMAGE} -LABEL python_version=python3.6 -RUN virtualenv --no-download /env -p python3.6 +LABEL python_version=python3.7 +RUN virtualenv --no-download /env -p python3.7 # Set virtualenv environment variables. This is equivalent to running # source /env/bin/activate diff --git a/tests/eventlet/requirements.txt b/tests/eventlet/requirements.txt index 7f8b2468..65eabc74 100644 --- a/tests/eventlet/requirements.txt +++ b/tests/eventlet/requirements.txt @@ -6,5 +6,5 @@ greenlet==0.4.14 gunicorn==19.9.0 itsdangerous==0.24 Jinja2==2.10 -MarkupSafe==1.0 +MarkupSafe==1.1.1 Werkzeug==0.14.1 diff --git a/tests/virtualenv/virtualenv_default.yaml b/tests/virtualenv/virtualenv_default.yaml index 1659c746..6b6ad282 100644 --- a/tests/virtualenv/virtualenv_default.yaml +++ b/tests/virtualenv/virtualenv_default.yaml @@ -34,4 +34,4 @@ commandTests: setup: [["virtualenv", "/env"], ["pip", "install", "flask"]] command: ["python", "-c", "import flask; print(flask.__file__)"] - expectedOutput: ["/env/local/lib/python2.7/site-packages/flask"] + expectedOutput: ["/env/lib/python2.7/site-packages/flask/__init__.pyc"] diff --git a/tests/virtualenv/virtualenv_python27.yaml b/tests/virtualenv/virtualenv_python27.yaml index 8bd85289..09b78480 100644 --- a/tests/virtualenv/virtualenv_python27.yaml +++ b/tests/virtualenv/virtualenv_python27.yaml @@ -44,4 +44,4 @@ commandTests: setup: [["virtualenv", "-p", "python", "/env"], ["pip", "install", "flask"]] command: ["python", "-c", "import flask; print(flask.__file__)"] - expectedOutput: ["/env/local/lib/python2.7/site-packages/flask"] + expectedOutput: ["/env/lib/python2.7/site-packages/flask/__init__.pyc"] diff --git a/tests/virtualenv/virtualenv_python34.yaml b/tests/virtualenv/virtualenv_python34.yaml index 077606fa..9b5b77d0 100644 --- a/tests/virtualenv/virtualenv_python34.yaml +++ b/tests/virtualenv/virtualenv_python34.yaml @@ -47,7 +47,7 @@ commandTests: setup: [["virtualenv", "-p", "python3.4", "/env"], ["pip", "install", "flask"]] command: ["python", "-c", "import flask; print(flask.__file__)"] - expectedOutput: ["/env/lib/python3.4/site-packages/flask"] + expectedOutput: ["/env/lib/python3.4/site-packages/flask/__init__.py"] - name: "virtualenv34 test.support availability" setup: [["virtualenv", "-p", "python3.4", "/env"]] diff --git a/tests/virtualenv/virtualenv_python35.yaml b/tests/virtualenv/virtualenv_python35.yaml index 3bb3d814..5e4b394a 100644 --- a/tests/virtualenv/virtualenv_python35.yaml +++ b/tests/virtualenv/virtualenv_python35.yaml @@ -47,7 +47,7 @@ commandTests: setup: [["virtualenv", "-p", "python3.5", "/env"], ["pip", "install", "flask"]] command: ["python", "-c", "import flask; print(flask.__file__)"] - expectedOutput: ["/env/lib/python3.5/site-packages/flask"] + expectedOutput: ["/env/lib/python3.5/site-packages/flask/__init__.py"] - name: "virtualenv35 test.support availability" setup: [["virtualenv", "-p", "python3.5", "/env"]] diff --git a/tests/virtualenv/virtualenv_python36.yaml b/tests/virtualenv/virtualenv_python36.yaml index f0949c6f..b3a9e68e 100644 --- a/tests/virtualenv/virtualenv_python36.yaml +++ b/tests/virtualenv/virtualenv_python36.yaml @@ -47,7 +47,7 @@ commandTests: setup: [["virtualenv", "-p", "python3.6", "/env"], ["pip", "install", "flask"]] command: ["python", "-c", "import flask; print(flask.__file__)"] - expectedOutput: ["/env/lib/python3.6/site-packages/flask"] + expectedOutput: ["/env/lib/python3.6/site-packages/flask/__init__.py"] - name: "virtualenv36 test.support availability" setup: [["virtualenv", "-p", "python3.6", "/env"]] diff --git a/tests/virtualenv/virtualenv_python37.yaml b/tests/virtualenv/virtualenv_python37.yaml index 7f3520ad..9810c78e 100644 --- a/tests/virtualenv/virtualenv_python37.yaml +++ b/tests/virtualenv/virtualenv_python37.yaml @@ -47,7 +47,7 @@ commandTests: setup: [["virtualenv", "-p", "python3.7", "/env"], ["pip", "install", "flask"]] command: ["python", "-c", "import flask; print(flask.__file__)"] - expectedOutput: ["/env/lib/python3.7/site-packages/flask"] + expectedOutput: ["/env/lib/python3.7/site-packages/flask/__init__.py"] - name: "virtualenv37 test.support availability" setup: [["virtualenv", "-p", "python3.7", "/env"]] From df22badacb89bbd6cb21f0fc9f50189053ac5019 Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Fri, 11 Sep 2020 10:26:30 -0700 Subject: [PATCH 218/256] Use virtualenv pip --- builder/gen-dockerfile/Dockerfile.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/gen-dockerfile/Dockerfile.in b/builder/gen-dockerfile/Dockerfile.in index 4ddce7c2..92ceaecf 100644 --- a/builder/gen-dockerfile/Dockerfile.in +++ b/builder/gen-dockerfile/Dockerfile.in @@ -9,7 +9,7 @@ ENV PATH /env/bin:$PATH ADD requirements.txt /builder/ #virtualenv's pip is pegged at version 10.0, removing so #newer versions get picked up -RUN rm -f /env/bin/pip* && pip install -r /builder/requirements.txt +RUN pip install -r /builder/requirements.txt ADD . /builder/ WORKDIR /workspace ENTRYPOINT [ "python", "/builder/gen_dockerfile.py" ] From 764a5f0bc0dd5a5ea1a2404e813ed9a8569da218 Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Wed, 7 Jul 2021 12:23:41 -0700 Subject: [PATCH 219/256] Fixing build error for python2.7 pip --- build.sh | 15 ++++++++++++--- runtime-image/Dockerfile.in | 1 + runtime-image/resources/apt-packages.txt | 1 - 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/build.sh b/build.sh index 63afa1ab..2083237d 100755 --- a/build.sh +++ b/build.sh @@ -24,7 +24,7 @@ test=0 # Should run standard test suite? local=0 # Should run using local Docker daemon instead of GCR? -os_base=ubuntu16 # Which operating system base to use +os_base=ubuntu18 # Which operating system base to use interpreter=0 # Should build interpreters instead of images # Note that $gcloud_cmd has spaces in it @@ -48,7 +48,7 @@ Options: --[no]test: Run basic tests (default true if no options set) --[no]client_test: Run Google Cloud Client Library tests (default false) --[no]local: Build images using local Docker daemon (default false) - --os_base: Which OS image to build on top of [debian8, ubuntu16] + --os_base: Which OS image to build on top of [debian8, ubuntu16, ubuntu18] " } @@ -119,6 +119,10 @@ while [ $# -gt 0 ]; do os_base=ubuntu16 shift ;; + --os_base=ubuntu18) + os_base=ubuntu18 + shift + ;; --test) test=1 shift @@ -156,8 +160,13 @@ fi # Pick OS image to use as base if [ "${os_base}" == "debian8" ]; then export OS_BASE_IMAGE="gcr.io/google-appengine/debian8:latest" -else +elif [ "${os_base}" == "ubuntu16" ]; then export OS_BASE_IMAGE="gcr.io/gcp-runtimes/ubuntu_16_0_4:latest" +elif [ "${os_base}" == "ubuntu18" ]; then + export OS_BASE_IMAGE="gcr.io/gcp-runtimes/ubuntu_18_0_4:latest" +else + echo "Unsupported OS base image: $OS_BASE_IMAGE" + exit 1 fi export STAGING_IMAGE="${DOCKER_NAMESPACE}/python:${TAG}" echo "Using base image name ${STAGING_IMAGE}" diff --git a/runtime-image/Dockerfile.in b/runtime-image/Dockerfile.in index dc842d9b..46387705 100644 --- a/runtime-image/Dockerfile.in +++ b/runtime-image/Dockerfile.in @@ -9,6 +9,7 @@ ADD scripts /scripts # Install Python, pip, and C dev libraries necessary to compile the most popular # Python libraries. RUN /scripts/install-apt-packages.sh +RUN curl "https://bootstrap.pypa.io/pip/2.7/get-pip.py" -o "get-pip.py" && python ./get-pip.py && ln -s /usr/local/bin/pip /usr/bin/pip # Setup locale. This prevents Python 3 IO encoding issues. ENV LANG C.UTF-8 diff --git a/runtime-image/resources/apt-packages.txt b/runtime-image/resources/apt-packages.txt index 287f7abc..7b88f777 100644 --- a/runtime-image/resources/apt-packages.txt +++ b/runtime-image/resources/apt-packages.txt @@ -4,7 +4,6 @@ mercurial pkg-config wget # debian-provided interpreters -python-pip python2.7 python2.7-dev # Dependenies for third-party Python packages From b4754dd62596fa97499adde2085bc855571d5e57 Mon Sep 17 00:00:00 2001 From: Matthew Suozzo Date: Mon, 15 Aug 2022 17:21:19 -0400 Subject: [PATCH 220/256] Propagate env vars to sudo invocations. --- scripts/deploy_check.sh | 4 ++-- scripts/integration-test.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/deploy_check.sh b/scripts/deploy_check.sh index 77fccb54..1e2f02c2 100644 --- a/scripts/deploy_check.sh +++ b/scripts/deploy_check.sh @@ -12,11 +12,11 @@ fi cd ${KOKORO_GFILE_DIR}/appengine/integration_tests -sudo /usr/local/bin/pip install --upgrade -r requirements.txt +sudo -E /usr/local/bin/pip install --upgrade -r requirements.txt if [ -f ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY}/requirements.txt ] then - sudo /usr/local/bin/pip install --upgrade -r ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY}/requirements.txt + sudo -E /usr/local/bin/pip install --upgrade -r ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY}/requirements.txt fi export DEPLOY_LATENCY_PROJECT='cloud-deploy-latency' diff --git a/scripts/integration-test.sh b/scripts/integration-test.sh index 63137b5f..6210e0a1 100644 --- a/scripts/integration-test.sh +++ b/scripts/integration-test.sh @@ -7,11 +7,11 @@ source ${KOKORO_GFILE_DIR}/kokoro/common.sh export GOOGLE_CLOUD_PROJECT=gcp-runtimes -sudo /usr/local/bin/pip install --upgrade -r ${KOKORO_GFILE_DIR}/appengine/integration_tests/requirements.txt +sudo -E /usr/local/bin/pip install --upgrade -r ${KOKORO_GFILE_DIR}/appengine/integration_tests/requirements.txt if [ -f ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY}/requirements.txt ] then - sudo /usr/local/bin/pip install --upgrade -r ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY}/requirements.txt + sudo -E /usr/local/bin/pip install --upgrade -r ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY}/requirements.txt fi export GOPATH=${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY} From cc0da57364aec352e15dfa203b939a41b6cb7faa Mon Sep 17 00:00:00 2001 From: jinglundong <1683035+jinglundong@users.noreply.github.com> Date: Tue, 16 Aug 2022 16:21:39 -0700 Subject: [PATCH 221/256] Update CODEOWNERS --- CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CODEOWNERS b/CODEOWNERS index 5deacc6c..2b618c5f 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1,4 +1,4 @@ # Code owners file. # This file controls who is tagged for review for any given pull request. -* @dlorenc @sharifelgamal @tstromberg @donmccasland +* @jinglundong @donmccasland From 647d1a1344ef9bfc3ce560c4190ba3f2fb2b9903 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Aug 2023 21:52:18 +0000 Subject: [PATCH 222/256] Bump tornado from 5.1 to 6.3.3 in /tests/python2-libraries Bumps [tornado](https://github.com/tornadoweb/tornado) from 5.1 to 6.3.3. - [Changelog](https://github.com/tornadoweb/tornado/blob/master/docs/releases.rst) - [Commits](https://github.com/tornadoweb/tornado/compare/v5.1.0...v6.3.3) --- updated-dependencies: - dependency-name: tornado dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python2-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 865cd1aa..964eaf9e 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -184,7 +184,7 @@ supervisor==3.3.4 testrepository==0.0.20 testtools==2.3.0 thrift==0.11.0 -tornado==5.1 +tornado==6.3.3 tox==3.2.1 twisted==18.7.0 ujson==1.35 From 07b6f1ab2faee152ab2d4bef1dfc41863cfe752f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Aug 2023 21:53:13 +0000 Subject: [PATCH 223/256] Bump tornado from 5.1 to 6.3.3 in /tests/python3-libraries Bumps [tornado](https://github.com/tornadoweb/tornado) from 5.1 to 6.3.3. - [Changelog](https://github.com/tornadoweb/tornado/blob/master/docs/releases.rst) - [Commits](https://github.com/tornadoweb/tornado/compare/v5.1.0...v6.3.3) --- updated-dependencies: - dependency-name: tornado dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python3-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index e3fd6424..c0dc6a66 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -172,7 +172,7 @@ stevedore==1.29.0 testrepository==0.0.20 testtools==2.3.0 thrift==0.11.0 -tornado==5.1 +tornado==6.3.3 tox==3.2.1 twisted==18.7.0 ujson==1.35 From 8f6ff95c279eb8313b6f559c7e2b39d1f71bb2a5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 00:44:38 +0000 Subject: [PATCH 224/256] Bump pygments from 2.2.0 to 2.15.0 in /tests/python3-libraries Bumps [pygments](https://github.com/pygments/pygments) from 2.2.0 to 2.15.0. - [Release notes](https://github.com/pygments/pygments/releases) - [Changelog](https://github.com/pygments/pygments/blob/master/CHANGES) - [Commits](https://github.com/pygments/pygments/compare/2.2.0...2.15.0) --- updated-dependencies: - dependency-name: pygments dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python3-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index c0dc6a66..ccd826d4 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -123,7 +123,7 @@ pyasn1==0.4.4 pycparser==2.18 pycrypto==2.6.1 pyflakes==2.0.0 -pygments==2.2.0 +pygments==2.15.0 pyjwt==1.6.4 pylibmc==1.5.2 pylint==2.1.1 From 2f6f1a4fcd14bc52377692a66013bf85f7e518b2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 00:44:39 +0000 Subject: [PATCH 225/256] Bump cryptography from 2.3.1 to 41.0.3 in /tests/python3-libraries Bumps [cryptography](https://github.com/pyca/cryptography) from 2.3.1 to 41.0.3. - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/2.3.1...41.0.3) --- updated-dependencies: - dependency-name: cryptography dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python3-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index c0dc6a66..f761e959 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -30,7 +30,7 @@ cov-core==1.15.0 coverage==4.5.1 coveralls==1.4.0 crcmod==1.7 -cryptography==2.3.1 +cryptography==41.0.3 cssselect==1.0.3 cython==0.28.5 decorator==4.3.0 From f6682ba419bf583c5a34b258cb455159c67640a9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 00:44:43 +0000 Subject: [PATCH 226/256] Bump scipy from 1.1.0 to 1.10.0 in /tests/python3-libraries Bumps [scipy](https://github.com/scipy/scipy) from 1.1.0 to 1.10.0. - [Release notes](https://github.com/scipy/scipy/releases) - [Commits](https://github.com/scipy/scipy/compare/v1.1.0...v1.10.0) --- updated-dependencies: - dependency-name: scipy dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python3-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index c0dc6a66..58839946 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -154,7 +154,7 @@ requests-oauthlib==1.0.0 requests==2.19.1 retrying==1.3.3 rsa==3.4.2 -scipy==1.1.0 +scipy==1.10.0 selenium==3.14.0 setuptools-git==1.2 setuptools==40.2.0 From 0a395be4f042ac9517d35a1cac13d99ca65caa84 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 00:44:45 +0000 Subject: [PATCH 227/256] Bump certifi from 2018.8.24 to 2023.7.22 in /tests/python2-libraries Bumps [certifi](https://github.com/certifi/python-certifi) from 2018.8.24 to 2023.7.22. - [Commits](https://github.com/certifi/python-certifi/compare/2018.08.24...2023.07.22) --- updated-dependencies: - dependency-name: certifi dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python2-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 865cd1aa..bb083cde 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -20,7 +20,7 @@ botocore==1.11.1 bottle==0.12.13 carbon<1.1.1 celery==4.2.1 -certifi==2018.8.24 +certifi==2023.7.22 cffi==1.11.5 chardet==3.0.4 click==6.7 From b383ecd9913c3550e5a55602bc682a61f84c7781 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 00:44:45 +0000 Subject: [PATCH 228/256] Bump certifi from 2018.8.24 to 2023.7.22 in /tests/python3-libraries Bumps [certifi](https://github.com/certifi/python-certifi) from 2018.8.24 to 2023.7.22. - [Commits](https://github.com/certifi/python-certifi/compare/2018.08.24...2023.07.22) --- updated-dependencies: - dependency-name: certifi dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python3-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index c0dc6a66..8497d1d9 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -18,7 +18,7 @@ boto==2.49.0 botocore==1.11.1 bottle==0.12.13 celery==4.2.1 -certifi==2018.8.24 +certifi==2023.7.22 cffi==1.11.5 chardet==3.0.4 click==6.7 From 19ccc8a8be75537635ca00e280f71ee4fed4fbcd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 06:18:22 +0000 Subject: [PATCH 229/256] Bump requests from 2.19.1 to 2.31.0 in /tests/python3-libraries Bumps [requests](https://github.com/psf/requests) from 2.19.1 to 2.31.0. - [Release notes](https://github.com/psf/requests/releases) - [Changelog](https://github.com/psf/requests/blob/main/HISTORY.md) - [Commits](https://github.com/psf/requests/compare/v2.19.1...v2.31.0) --- updated-dependencies: - dependency-name: requests dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python3-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 71dab860..3568a7f3 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -151,7 +151,7 @@ raven==6.9.0 redis==2.10.6 repoze.lru==0.7 requests-oauthlib==1.0.0 -requests==2.19.1 +requests==2.31.0 retrying==1.3.3 rsa==3.4.2 scipy==1.1.0 From e1854082720277c75cdb5ef63ab386b314c2d2a6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 06:18:41 +0000 Subject: [PATCH 230/256] Bump scipy from 1.1.0 to 1.10.0 in /tests/python2-libraries Bumps [scipy](https://github.com/scipy/scipy) from 1.1.0 to 1.10.0. - [Release notes](https://github.com/scipy/scipy/releases) - [Commits](https://github.com/scipy/scipy/compare/v1.1.0...v1.10.0) --- updated-dependencies: - dependency-name: scipy dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python2-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 63d371ec..dace11da 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -164,7 +164,7 @@ requests-oauthlib==1.0.0 requests==2.19.1 retrying==1.3.3 rsa==3.4.2 -scipy==1.1.0 +scipy==1.10.0 selenium==3.14.0 setuptools-git==1.2 setuptools==40.2.0 From a3673f8691058c3c068dc2f42e9b2cb9804b441e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 06:18:58 +0000 Subject: [PATCH 231/256] Bump flask from 1.1.0 to 2.2.5 in /tests/integration Bumps [flask](https://github.com/pallets/flask) from 1.1.0 to 2.2.5. - [Release notes](https://github.com/pallets/flask/releases) - [Changelog](https://github.com/pallets/flask/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/flask/compare/1.1.0...2.2.5) --- updated-dependencies: - dependency-name: flask dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/integration/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index ea714c20..0e31f608 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -1,4 +1,4 @@ -Flask==1.1.0 +Flask==2.2.5 google-cloud-error-reporting==0.32.1 google-cloud-logging==1.12.1 google-cloud-monitoring==0.33.0 From 4cd3ee423bb00a17f27255f8f579ba110cf933e1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 06:19:23 +0000 Subject: [PATCH 232/256] Bump flask from 1.0.2 to 2.2.5 in /tests/python3-libraries Bumps [flask](https://github.com/pallets/flask) from 1.0.2 to 2.2.5. - [Release notes](https://github.com/pallets/flask/releases) - [Changelog](https://github.com/pallets/flask/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/flask/compare/1.0.2...2.2.5) --- updated-dependencies: - dependency-name: flask dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python3-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index c48b246e..24727967 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -51,7 +51,7 @@ extras==1.0.0 fabric==2.3.1 fixtures==3.0.0 flake8==3.5.0 -flask==1.0.2 +flask==2.2.5 funcsigs==1.0.2 gevent==1.3.6 google-api-python-client==1.7.4 From f3e034b17cd7d2558f0a4ee12a6f5599c6c645b5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 06:19:44 +0000 Subject: [PATCH 233/256] Bump requests from 2.19.1 to 2.31.0 in /tests/python2-libraries Bumps [requests](https://github.com/psf/requests) from 2.19.1 to 2.31.0. - [Release notes](https://github.com/psf/requests/releases) - [Changelog](https://github.com/psf/requests/blob/main/HISTORY.md) - [Commits](https://github.com/psf/requests/compare/v2.19.1...v2.31.0) --- updated-dependencies: - dependency-name: requests dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python2-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index dace11da..5db7a568 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -161,7 +161,7 @@ raven==6.9.0 redis==2.10.6 repoze.lru==0.7 requests-oauthlib==1.0.0 -requests==2.19.1 +requests==2.31.0 retrying==1.3.3 rsa==3.4.2 scipy==1.10.0 From a1dbe236ef109d8c88902a5fdb68d15990370c05 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 06:20:01 +0000 Subject: [PATCH 234/256] Bump requests from 2.22.0 to 2.31.0 in /tests/integration Bumps [requests](https://github.com/psf/requests) from 2.22.0 to 2.31.0. - [Release notes](https://github.com/psf/requests/releases) - [Changelog](https://github.com/psf/requests/blob/main/HISTORY.md) - [Commits](https://github.com/psf/requests/compare/v2.22.0...v2.31.0) --- updated-dependencies: - dependency-name: requests dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/integration/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index ea714c20..45782f06 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -3,7 +3,7 @@ google-cloud-error-reporting==0.32.1 google-cloud-logging==1.12.1 google-cloud-monitoring==0.33.0 gunicorn==19.9.0 -requests==2.22.0 +requests==2.31.0 retrying==1.3.3 six==1.12.0 protobuf>=3.6.0 From 257c0f560f110a5d555f7df657d38e09df4887a2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 06:21:34 +0000 Subject: [PATCH 235/256] Bump flask from 1.0.2 to 2.2.5 in /tests/python2-libraries Bumps [flask](https://github.com/pallets/flask) from 1.0.2 to 2.2.5. - [Release notes](https://github.com/pallets/flask/releases) - [Changelog](https://github.com/pallets/flask/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/flask/compare/1.0.2...2.2.5) --- updated-dependencies: - dependency-name: flask dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python2-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 5db7a568..92da6052 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -53,7 +53,7 @@ extras==1.0.0 fabric==2.3.1 fixtures==3.0.0 flake8==3.5.0 -flask==1.0.2 +flask==2.2.5 funcsigs==1.0.2 functools32==3.2.3.post2 futures==3.2.0 From c5e53836a50440973b14727de05399e35d20cd08 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 06:21:38 +0000 Subject: [PATCH 236/256] Bump sqlparse from 0.2.4 to 0.4.4 in /tests/python3-libraries Bumps [sqlparse](https://github.com/andialbrecht/sqlparse) from 0.2.4 to 0.4.4. - [Changelog](https://github.com/andialbrecht/sqlparse/blob/master/CHANGELOG) - [Commits](https://github.com/andialbrecht/sqlparse/compare/0.2.4...0.4.4) --- updated-dependencies: - dependency-name: sqlparse dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python3-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 82263711..1d6b37da 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -166,7 +166,7 @@ south==1.0.2 sphinx==1.7.7 sqlalchemy-migrate==0.11.0 sqlalchemy==1.2.11 -sqlparse==0.2.4 +sqlparse==0.4.4 statsd==3.3.0 stevedore==1.29.0 testrepository==0.0.20 From fd2cdb556238254b9b3201a48e7b1fafb12ce5a5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 06:21:51 +0000 Subject: [PATCH 237/256] Bump flask from 1.0.2 to 2.2.5 in /tests/eventlet Bumps [flask](https://github.com/pallets/flask) from 1.0.2 to 2.2.5. - [Release notes](https://github.com/pallets/flask/releases) - [Changelog](https://github.com/pallets/flask/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/flask/compare/1.0.2...2.2.5) --- updated-dependencies: - dependency-name: flask dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/eventlet/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/eventlet/requirements.txt b/tests/eventlet/requirements.txt index 65eabc74..75ee7c7d 100644 --- a/tests/eventlet/requirements.txt +++ b/tests/eventlet/requirements.txt @@ -1,7 +1,7 @@ click==6.7 enum-compat==0.0.2 eventlet==0.24.1 -Flask==1.0.2 +Flask==2.2.5 greenlet==0.4.14 gunicorn==19.9.0 itsdangerous==0.24 From abd0aa21189209da39fe79b4559fddaafe950a14 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 06:22:32 +0000 Subject: [PATCH 238/256] Bump urllib3 from 1.23 to 1.26.5 in /tests/python3-libraries Bumps [urllib3](https://github.com/urllib3/urllib3) from 1.23 to 1.26.5. - [Release notes](https://github.com/urllib3/urllib3/releases) - [Changelog](https://github.com/urllib3/urllib3/blob/main/CHANGES.rst) - [Commits](https://github.com/urllib3/urllib3/compare/1.23...1.26.5) --- updated-dependencies: - dependency-name: urllib3 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python3-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 82263711..06341ddd 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -179,7 +179,7 @@ ujson==1.35 unidecode==1.0.22 unittest2==1.1.0 uritemplate==3.0.0 -urllib3==1.23 +urllib3==1.26.5 uwsgi==2.0.17.1 versiontools==1.9.1 virtualenv==16.0.0 From 38e4e9d397da6d3e36f39dbcdec1da9501bc6f50 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 06:23:08 +0000 Subject: [PATCH 239/256] Bump ansible from 2.6.3 to 7.0.0 in /tests/python3-libraries Bumps [ansible](https://github.com/ansible/ansible) from 2.6.3 to 7.0.0. - [Release notes](https://github.com/ansible/ansible/releases) - [Commits](https://github.com/ansible/ansible/commits) --- updated-dependencies: - dependency-name: ansible dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python3-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 82263711..89b1a196 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -1,7 +1,7 @@ alembic==1.0.0 amqp==2.3.2 amqplib==1.0.2 -ansible==2.6.3 +ansible==7.0.0 anyjson==0.3.3 apache-libcloud==2.3.0 argparse==1.4.0 From 33570888d9e3359a02f70500c4fcf97a520f3dc9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 06:24:15 +0000 Subject: [PATCH 240/256] Bump sqlparse from 0.2.4 to 0.4.4 in /tests/python2-libraries Bumps [sqlparse](https://github.com/andialbrecht/sqlparse) from 0.2.4 to 0.4.4. - [Changelog](https://github.com/andialbrecht/sqlparse/blob/master/CHANGELOG) - [Commits](https://github.com/andialbrecht/sqlparse/compare/0.2.4...0.4.4) --- updated-dependencies: - dependency-name: sqlparse dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python2-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 92da6052..f3b9daec 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -176,7 +176,7 @@ south==1.0.2 sphinx==1.7.7 sqlalchemy-migrate==0.11.0 sqlalchemy==1.2.11 -sqlparse==0.2.4 +sqlparse==0.4.4 statsd==3.3.0 stevedore==1.29.0 suds==0.4 From 06f1cfa49fcb057c0440afb2d32868683061f494 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 06:24:41 +0000 Subject: [PATCH 241/256] Bump waitress from 1.1.0 to 2.1.2 in /tests/python3-libraries Bumps [waitress](https://github.com/Pylons/waitress) from 1.1.0 to 2.1.2. - [Release notes](https://github.com/Pylons/waitress/releases) - [Changelog](https://github.com/Pylons/waitress/blob/v2.1.2/CHANGES.txt) - [Commits](https://github.com/Pylons/waitress/compare/v1.1.0...v2.1.2) --- updated-dependencies: - dependency-name: waitress dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python3-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 4929dcc6..e8adc88f 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -183,7 +183,7 @@ urllib3==1.26.5 uwsgi==2.0.17.1 versiontools==1.9.1 virtualenv==16.0.0 -waitress==1.1.0 +waitress==2.1.2 warlock==1.3.0 webob==1.8.2 websocket-client==0.51.0 From b901f8f3ccbda6fcfec23082b4218bd2834d0a33 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 06:25:52 +0000 Subject: [PATCH 242/256] Bump pillow from 5.2.0 to 9.3.0 in /tests/python3-libraries Bumps [pillow](https://github.com/python-pillow/Pillow) from 5.2.0 to 9.3.0. - [Release notes](https://github.com/python-pillow/Pillow/releases) - [Changelog](https://github.com/python-pillow/Pillow/blob/main/CHANGES.rst) - [Commits](https://github.com/python-pillow/Pillow/compare/5.2.0...9.3.0) --- updated-dependencies: - dependency-name: pillow dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python3-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 0989646e..66e9b98e 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -111,7 +111,7 @@ pbr==4.2.0 pep8==1.7.1 pexpect==4.6.0 pika==0.12.0 -pillow==5.2.0 +pillow==9.3.0 pip==18.0 prettytable==0.7.2 protobuf==3.6.1 From f12acd1bc138fdeeedfcf042aab38661a88a57da Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 06:25:54 +0000 Subject: [PATCH 243/256] Bump pyyaml from 3.13 to 5.4 in /tests/python3-libraries Bumps [pyyaml](https://github.com/yaml/pyyaml) from 3.13 to 5.4. - [Changelog](https://github.com/yaml/pyyaml/blob/master/CHANGES) - [Commits](https://github.com/yaml/pyyaml/compare/3.13...5.4) --- updated-dependencies: - dependency-name: pyyaml dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python3-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 0989646e..f4d40d0e 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -145,7 +145,7 @@ python-novaclient==11.0.0 python-subunit==1.3.0 python-swiftclient==3.6.0 pytz==2018.5 -pyyaml==3.13 +pyyaml==5.4 pyzmq==17.1.2 raven==6.9.0 redis==2.10.6 From ce681862182eb79ab31d0f8f4162246b98c70dff Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 06:25:58 +0000 Subject: [PATCH 244/256] Bump numpy from 1.15.1 to 1.22.0 in /tests/python3-libraries Bumps [numpy](https://github.com/numpy/numpy) from 1.15.1 to 1.22.0. - [Release notes](https://github.com/numpy/numpy/releases) - [Changelog](https://github.com/numpy/numpy/blob/main/doc/RELEASE_WALKTHROUGH.rst) - [Commits](https://github.com/numpy/numpy/compare/v1.15.1...v1.22.0) --- updated-dependencies: - dependency-name: numpy dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python3-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 0989646e..9e70346b 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -95,7 +95,7 @@ netaddr==0.7.19 netifaces==0.10.7 newrelic==4.2.0.100 nose==1.3.7 -numpy==1.15.1 +numpy==1.22.0 oauth2==1.9.0.post1 oauth2client==4.1.2 oauthlib==2.1.0 From cf7bce61d466e477b8a0963a38a6d4199867c088 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 06:26:35 +0000 Subject: [PATCH 245/256] Bump django from 2.1 to 2.2.28 in /tests/python3-libraries Bumps [django](https://github.com/django/django) from 2.1 to 2.2.28. - [Commits](https://github.com/django/django/compare/2.1...2.2.28) --- updated-dependencies: - dependency-name: django dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python3-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 0989646e..31938505 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -37,7 +37,7 @@ decorator==4.3.0 django-celery==3.2.2 django-debug-toolbar==1.9.1 django-extensions==2.1.1 -django==2.1 +django==2.2.28 django_compress==1.0.1 djangorestframework==3.8.2 docker-py==1.10.6 From 78b72a16af2a8b4270505443061b850a3d749a79 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 18:22:59 +0000 Subject: [PATCH 246/256] Bump urllib3 from 1.23 to 1.26.5 in /tests/python2-libraries Bumps [urllib3](https://github.com/urllib3/urllib3) from 1.23 to 1.26.5. - [Release notes](https://github.com/urllib3/urllib3/releases) - [Changelog](https://github.com/urllib3/urllib3/blob/main/CHANGES.rst) - [Commits](https://github.com/urllib3/urllib3/compare/1.23...1.26.5) --- updated-dependencies: - dependency-name: urllib3 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python2-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index f3b9daec..2eaf5ba0 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -191,7 +191,7 @@ ujson==1.35 unidecode==1.0.22 unittest2==1.1.0 uritemplate==3.0.0 -urllib3==1.23 +urllib3==1.26.5 uwsgi==2.0.17.1 versiontools==1.9.1 virtualenv==16.0.0 From 2758277ba791b3629cfa54b6dd0553de8d729a79 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 18:23:18 +0000 Subject: [PATCH 247/256] Bump ansible from 2.6.3 to 7.0.0 in /tests/python2-libraries Bumps [ansible](https://github.com/ansible/ansible) from 2.6.3 to 7.0.0. - [Release notes](https://github.com/ansible/ansible/releases) - [Commits](https://github.com/ansible/ansible/commits) --- updated-dependencies: - dependency-name: ansible dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python2-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index f3b9daec..2795af2e 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -1,7 +1,7 @@ alembic==1.0.0 amqp==2.3.2 amqplib==1.0.2 -ansible==2.6.3 +ansible==7.0.0 anyjson==0.3.3 apache-libcloud==2.3.0 argparse==1.4.0 From c36d1079fc62e162b45020bc3abcdf70469c7cd8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 18:26:45 +0000 Subject: [PATCH 248/256] Bump waitress from 1.1.0 to 2.1.2 in /tests/python2-libraries Bumps [waitress](https://github.com/Pylons/waitress) from 1.1.0 to 2.1.2. - [Release notes](https://github.com/Pylons/waitress/releases) - [Changelog](https://github.com/Pylons/waitress/blob/v2.1.2/CHANGES.txt) - [Commits](https://github.com/Pylons/waitress/compare/v1.1.0...v2.1.2) --- updated-dependencies: - dependency-name: waitress dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python2-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index f3b9daec..09feaa44 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -195,7 +195,7 @@ urllib3==1.23 uwsgi==2.0.17.1 versiontools==1.9.1 virtualenv==16.0.0 -waitress==1.1.0 +waitress==2.1.2 warlock==1.3.0 webob==1.8.2 websocket-client==0.51.0 From 49e19e4b726a1950da60fabaa40cbce2a838a09b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 20:11:30 +0000 Subject: [PATCH 249/256] Bump numpy from 1.15.1 to 1.22.0 in /tests/python2-libraries Bumps [numpy](https://github.com/numpy/numpy) from 1.15.1 to 1.22.0. - [Release notes](https://github.com/numpy/numpy/releases) - [Changelog](https://github.com/numpy/numpy/blob/main/doc/RELEASE_WALKTHROUGH.rst) - [Commits](https://github.com/numpy/numpy/compare/v1.15.1...v1.22.0) --- updated-dependencies: - dependency-name: numpy dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python2-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 6bb9fc86..b2ecb1dd 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -103,7 +103,7 @@ netaddr==0.7.19 netifaces==0.10.7 newrelic==4.2.0.100 nose==1.3.7 -numpy==1.15.1 +numpy==1.22.0 oauth2==1.9.0.post1 oauth2client==4.1.2 oauthlib==2.1.0 From d12a82f5651abbc5c7f3a653aa8f909633e26f4e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 20:11:36 +0000 Subject: [PATCH 250/256] Bump pyyaml from 3.13 to 5.4 in /tests/python2-libraries Bumps [pyyaml](https://github.com/yaml/pyyaml) from 3.13 to 5.4. - [Changelog](https://github.com/yaml/pyyaml/blob/master/CHANGES) - [Commits](https://github.com/yaml/pyyaml/compare/3.13...5.4) --- updated-dependencies: - dependency-name: pyyaml dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python2-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 6bb9fc86..67b81897 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -155,7 +155,7 @@ python-novaclient==11.0.0 python-subunit==1.3.0 python-swiftclient==3.6.0 pytz==2018.5 -pyyaml==3.13 +pyyaml==5.4 pyzmq==17.1.2 raven==6.9.0 redis==2.10.6 From 4dbda4cfd7e58de057d291078e5143d421460eff Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 20:12:22 +0000 Subject: [PATCH 251/256] Bump pillow from 5.2.0 to 9.3.0 in /tests/python2-libraries Bumps [pillow](https://github.com/python-pillow/Pillow) from 5.2.0 to 9.3.0. - [Release notes](https://github.com/python-pillow/Pillow/releases) - [Changelog](https://github.com/python-pillow/Pillow/blob/main/CHANGES.rst) - [Commits](https://github.com/python-pillow/Pillow/compare/5.2.0...9.3.0) --- updated-dependencies: - dependency-name: pillow dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python2-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 6bb9fc86..cfbaab8b 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -119,7 +119,7 @@ pbr==4.2.0 pep8==1.7.1 pexpect==4.6.0 pika==0.12.0 -pillow==5.2.0 +pillow==9.3.0 pip==18.0 prettytable==0.7.2 protobuf==3.6.1 From 6c6077f604056fcf7f2383db23a82699e7c8450d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 16 Aug 2023 00:38:49 +0000 Subject: [PATCH 252/256] Bump werkzeug from 0.14.1 to 2.2.3 in /tests/python3-libraries Bumps [werkzeug](https://github.com/pallets/werkzeug) from 0.14.1 to 2.2.3. - [Release notes](https://github.com/pallets/werkzeug/releases) - [Changelog](https://github.com/pallets/werkzeug/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/werkzeug/compare/0.14.1...2.2.3) --- updated-dependencies: - dependency-name: werkzeug dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python3-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index d21238fd..d9117d4f 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -188,7 +188,7 @@ warlock==1.3.0 webob==1.8.2 websocket-client==0.51.0 webtest==2.0.30 -werkzeug==0.14.1 +werkzeug==2.2.3 wheel==0.31.1 xlrd==1.1.0 zc.buildout==2.12.1 From 17d4165fa40213904382fa9cdda5a067e2b86d5a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 16 Aug 2023 00:39:06 +0000 Subject: [PATCH 253/256] Bump werkzeug from 0.14.1 to 2.2.3 in /tests/python2-libraries Bumps [werkzeug](https://github.com/pallets/werkzeug) from 0.14.1 to 2.2.3. - [Release notes](https://github.com/pallets/werkzeug/releases) - [Changelog](https://github.com/pallets/werkzeug/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/werkzeug/compare/0.14.1...2.2.3) --- updated-dependencies: - dependency-name: werkzeug dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python2-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 29bdca44..67a0c0b3 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -200,7 +200,7 @@ warlock==1.3.0 webob==1.8.2 websocket-client==0.51.0 webtest==2.0.30 -werkzeug==0.14.1 +werkzeug==2.2.3 wheel==0.31.1 xlrd==1.1.0 zc.buildout==2.12.1 From 56de3a3edea2f3653ec018d874cb0f8cda9aa67d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 16 Aug 2023 00:40:22 +0000 Subject: [PATCH 254/256] Bump werkzeug from 0.14.1 to 2.2.3 in /tests/eventlet Bumps [werkzeug](https://github.com/pallets/werkzeug) from 0.14.1 to 2.2.3. - [Release notes](https://github.com/pallets/werkzeug/releases) - [Changelog](https://github.com/pallets/werkzeug/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/werkzeug/compare/0.14.1...2.2.3) --- updated-dependencies: - dependency-name: werkzeug dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/eventlet/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/eventlet/requirements.txt b/tests/eventlet/requirements.txt index 75ee7c7d..1e2ea66b 100644 --- a/tests/eventlet/requirements.txt +++ b/tests/eventlet/requirements.txt @@ -7,4 +7,4 @@ gunicorn==19.9.0 itsdangerous==0.24 Jinja2==2.10 MarkupSafe==1.1.1 -Werkzeug==0.14.1 +Werkzeug==2.2.3 From 461f8883bbd586319ffa8e1442858607c1473640 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 24 Aug 2023 20:11:21 +0000 Subject: [PATCH 255/256] Bump uwsgi from 2.0.17.1 to 2.0.22 in /tests/python2-libraries Bumps [uwsgi](https://github.com/unbit/uwsgi-docs) from 2.0.17.1 to 2.0.22. - [Commits](https://github.com/unbit/uwsgi-docs/commits) --- updated-dependencies: - dependency-name: uwsgi dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python2-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 67a0c0b3..f0596d67 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -192,7 +192,7 @@ unidecode==1.0.22 unittest2==1.1.0 uritemplate==3.0.0 urllib3==1.26.5 -uwsgi==2.0.17.1 +uwsgi==2.0.22 versiontools==1.9.1 virtualenv==16.0.0 waitress==2.1.2 From 5b017c9ecc9422ba6e959bc28d712ab1a4d4353d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 24 Aug 2023 20:11:33 +0000 Subject: [PATCH 256/256] Bump uwsgi from 2.0.17.1 to 2.0.22 in /tests/python3-libraries Bumps [uwsgi](https://github.com/unbit/uwsgi-docs) from 2.0.17.1 to 2.0.22. - [Commits](https://github.com/unbit/uwsgi-docs/commits) --- updated-dependencies: - dependency-name: uwsgi dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python3-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index d9117d4f..23175a06 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -180,7 +180,7 @@ unidecode==1.0.22 unittest2==1.1.0 uritemplate==3.0.0 urllib3==1.26.5 -uwsgi==2.0.17.1 +uwsgi==2.0.22 versiontools==1.9.1 virtualenv==16.0.0 waitress==2.1.2