From 76d04c4c8251cf9f2b90b098365999c2606fba9e Mon Sep 17 00:00:00 2001 From: Douglas Greiman Date: Wed, 27 Sep 2017 15:54:42 -0700 Subject: [PATCH 001/132] Revert "Revert "Switch default Python 3 from 3.5 to 3.6"" --- builder/gen-dockerfile/Dockerfile.in | 4 ++-- scripts/gen_dockerfile.py | 2 +- scripts/gen_dockerfile_test.py | 5 ++++- scripts/testdata/hello_world_golden/Dockerfile | 4 ++-- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/builder/gen-dockerfile/Dockerfile.in b/builder/gen-dockerfile/Dockerfile.in index e2fe4463..4f6447eb 100644 --- a/builder/gen-dockerfile/Dockerfile.in +++ b/builder/gen-dockerfile/Dockerfile.in @@ -1,6 +1,6 @@ FROM ${STAGING_IMAGE} -LABEL python_version=python3.5 -RUN virtualenv --no-download /env -p python3.5 +LABEL python_version=python3.6 +RUN virtualenv --no-download /env -p python3.6 # Set virtualenv environment variables. This is equivalent to running # source /env/bin/activate diff --git a/scripts/gen_dockerfile.py b/scripts/gen_dockerfile.py index 5514b092..eaff183f 100755 --- a/scripts/gen_dockerfile.py +++ b/scripts/gen_dockerfile.py @@ -51,7 +51,7 @@ PYTHON_INTERPRETER_VERSION_MAP = { '': '', # == 2.7 '2': '', # == 2.7 - '3': '3.5', + '3': '3.6', '3.4': '3.4', '3.5': '3.5', '3.6': '3.6', diff --git a/scripts/gen_dockerfile_test.py b/scripts/gen_dockerfile_test.py index 31ef7d42..b52e15e8 100755 --- a/scripts/gen_dockerfile_test.py +++ b/scripts/gen_dockerfile_test.py @@ -63,7 +63,7 @@ def compare_file(filename, dir1, dir2): 'dockerfile_python_version': '', }), ('runtime_config:\n python_version: 3', { - 'dockerfile_python_version': '3.5', + 'dockerfile_python_version': '3.6', }), ('runtime_config:\n python_version: 3.4', { 'dockerfile_python_version': '3.4', @@ -71,6 +71,9 @@ def compare_file(filename, dir1, dir2): ('runtime_config:\n python_version: 3.5', { 'dockerfile_python_version': '3.5', }), + ('runtime_config:\n python_version: 3.6', { + 'dockerfile_python_version': '3.6', + }), # entrypoint present ('entrypoint: my entrypoint', { 'entrypoint': 'exec my entrypoint', diff --git a/scripts/testdata/hello_world_golden/Dockerfile b/scripts/testdata/hello_world_golden/Dockerfile index e3e0563c..10396399 100644 --- a/scripts/testdata/hello_world_golden/Dockerfile +++ b/scripts/testdata/hello_world_golden/Dockerfile @@ -1,6 +1,6 @@ FROM gcr.io/google-appengine/python -LABEL python_version=python3.5 -RUN virtualenv --no-download /env -p python3.5 +LABEL python_version=python3.6 +RUN virtualenv --no-download /env -p python3.6 # Set virtualenv environment variables. This is equivalent to running # source /env/bin/activate From 1d4e6df0f8780ea2be67c9a2fe0a01b9c7927e3c Mon Sep 17 00:00:00 2001 From: Douglas Greiman Date: Mon, 2 Oct 2017 09:37:27 -0700 Subject: [PATCH 002/132] Add missing dependency (was using a too-old version of six in some cases). (#151) --- tests/integration/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index 6e6be47e..eae15fe9 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -5,3 +5,4 @@ google-cloud-monitoring==0.27.0 gunicorn==19.7.1 requests==2.18.4 retrying==1.3.3 +six==1.11.0 From 18397afdf4f8fb7e81310ff55951063830f80546 Mon Sep 17 00:00:00 2001 From: Angela Li Date: Fri, 13 Oct 2017 10:42:29 -0700 Subject: [PATCH 003/132] Add Stackoverflow metrics (#149) --- perf_dashboard/bq_utils.py | 46 ++++++++ .../python_clientlibs_download.py | 57 +++------- perf_dashboard/stackoverflow/posts_stats.py | 107 ++++++++++++++++++ 3 files changed, 166 insertions(+), 44 deletions(-) create mode 100644 perf_dashboard/bq_utils.py rename {perf-dashboard/clientlibs-download => perf_dashboard/clientlibs_download}/python_clientlibs_download.py (71%) create mode 100644 perf_dashboard/stackoverflow/posts_stats.py diff --git a/perf_dashboard/bq_utils.py b/perf_dashboard/bq_utils.py new file mode 100644 index 00000000..d9441f07 --- /dev/null +++ b/perf_dashboard/bq_utils.py @@ -0,0 +1,46 @@ +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Common util methods for processing data in BigQuery.""" + +import uuid + +from google.cloud import bigquery + + +def insert_rows(project, dataset_name, table_name, rows): + """Insert rows to bigquery table.""" + client = bigquery.Client(project=project) + dataset = client.dataset(dataset_name) + table = bigquery.Table(table_name, dataset) + table.reload() + table.insert_data(rows) + + +def execute_query(query): + """Execute query and return the query results.""" + client = bigquery.Client() + query_job = client.run_async_query(str(uuid.uuid4()), query) + query_job.use_legacy_sql = False + + # Start the query job and wait it to complete + query_job.begin() + query_job.result() + + # Get the results + destination_table = query_job.destination + destination_table.reload() + results = destination_table.fetch_data() + + return results diff --git a/perf-dashboard/clientlibs-download/python_clientlibs_download.py b/perf_dashboard/clientlibs_download/python_clientlibs_download.py similarity index 71% rename from perf-dashboard/clientlibs-download/python_clientlibs_download.py rename to perf_dashboard/clientlibs_download/python_clientlibs_download.py index 1d307ac2..3c8f79fe 100644 --- a/perf-dashboard/clientlibs-download/python_clientlibs_download.py +++ b/perf_dashboard/clientlibs_download/python_clientlibs_download.py @@ -14,11 +14,16 @@ import datetime import os +import sys import time import uuid from google.cloud import bigquery +sys.path.insert(0, os.path.abspath(__file__+"/../../..")) +from perf_dashboard import bq_utils + + GCLOUD_PROJECT_ENV = 'GCLOUD_PROJECT' DATETIME_FORMAT = '%Y%m%d' @@ -68,17 +73,6 @@ } -def wait_for_job(job): - """Wait for the query job to complete.""" - while True: - job.reload() # Refreshes the state via a GET request. - if job.state == 'DONE': - if job.error_result: - raise RuntimeError(job.errors) - return - time.sleep(1) - - def get_weekly_clientlibs_downloads(clientlibs_table_name, date_str): """Use a SQL query to collect the weekly download data of the client libraries. @@ -124,50 +118,25 @@ def get_weekly_clientlibs_downloads(clientlibs_table_name, date_str): # Start the query job and wait it to complete query_job.begin() - wait_for_job(query_job) - - # Fetch the results - result = query_job.query_results().fetch_data() - result_list = [item for item in result] + query_job.result() - # In case the result_list contains the metadata like total_rows, the - # actual rows will be the first element of the result_list. - if len(result_list) > 0 and isinstance(result_list[0], list): - result_list = result_list[0] + # Get the results + destination_table = query_job.destination + destination_table.reload() + results = destination_table.fetch_data() - rows = [(date_time,) + row for row in result_list] - print(rows) + rows = [(date_time,) + row for row in results] return rows -def insert_rows(dataset_name, table_name, rows): - """Insert rows to a bigquery table. - - Args: - dataset_name (str): Name of the dataset that holds the tables. - table_name (str): Name of the bigquery table. - rows (list): The rows that going to be inserted into the table. - - Returns: - list: Empty if inserted successfully, else the errors when inserting - each row. - """ - project = os.environ.get(GCLOUD_PROJECT_ENV) - client = bigquery.Client(project=project) - dataset = client.dataset(dataset_name) - table = bigquery.Table(name=table_name, dataset=dataset) - table.reload() - error = table.insert_data(rows) - return error - - def main(): for table_name in CLIENTLIBS.keys(): rows = get_weekly_clientlibs_downloads( clientlibs_table_name=table_name, date_str=datetime.datetime.now().strftime("%Y%m%d")) - insert_rows( + bq_utils.insert_rows( + project=project, dataset_name=DATASET_NAME, table_name=table_name, rows=rows) diff --git a/perf_dashboard/stackoverflow/posts_stats.py b/perf_dashboard/stackoverflow/posts_stats.py new file mode 100644 index 00000000..280fd42c --- /dev/null +++ b/perf_dashboard/stackoverflow/posts_stats.py @@ -0,0 +1,107 @@ +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A script to collect the number of StackOverflow posts related to +Python and Google Cloud Platform.""" + +import datetime +import os +import sys +import time +import uuid + +from collections import Counter + +from google.cloud import bigquery + +# Need this to import the local helper function +sys.path.insert(0, os.path.abspath(__file__+"/../../..")) +from perf_dashboard import bq_utils + +GCLOUD_PROJECT_ENV = 'GCLOUD_PROJECT' +DATASET_NAME = 'stackoverflow' +TAG_COUNT_TABLE_NAME = 'tag_count_timestamp' +UNANSWERED_POSTS_TABLE_NAME = 'unanswered_posts' + + +def get_stackoverflow_tags_count(): + """Get all the tags contains python and cloud key words""" + query = """ + SELECT + SPLIT(tags, '|') tags + FROM + `bigquery-public-data.stackoverflow.posts_questions` + WHERE + tags LIKE '%python%' + AND (tags LIKE '%google-cloud-platform%' OR tags LIKE '%gcp%') + """ + + results = bq_utils.execute_query(query) + + rows = [row[0] for row in results] + + return rows + + +def get_posts_list_unanswered(): + # Get the list of posts that are unanswered + query = """ + SELECT + id, title, tags + FROM + `bigquery-public-data.stackoverflow.posts_questions` + WHERE + tags LIKE '%python%' + AND (tags LIKE '%google-cloud-platform%' OR tags LIKE '%gcp%') + AND accepted_answer_id is NULL + AND answer_count = 0; + """ + + results = bq_utils.execute_query(query) + + # Add current timestamp to the rows + date_time = datetime.datetime.now() + rows = [(date_time,) + row for row in results] + + return rows + + +def count_unique_tags(data): + flattened_tag_list = [tag for tag_list in data for tag in tag_list] + tag_count = Counter(flattened_tag_list) + + # Add current timestamp to the rows + date_time = datetime.datetime.now() + time_tag_count = [(date_time,) + item for item in tag_count.items()] + + return time_tag_count + + +def main(): + project = os.environ.get(GCLOUD_PROJECT_ENV) + + # Get the posts count for each tag + rows = get_stackoverflow_tags_count() + tag_count = count_unique_tags(rows) + bq_utils.insert_rows( + project, DATASET_NAME, TAG_COUNT_TABLE_NAME, tag_count) + + # Get the list of unanswered posts + unanswered_posts = get_posts_list_unanswered() + bq_utils.insert_rows( + project, DATASET_NAME, UNANSWERED_POSTS_TABLE_NAME, unanswered_posts) + + +if __name__ == '__main__': + main() From fed4c56915fe360cc98f6e9f144098d3ed6c4f78 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Mon, 16 Oct 2017 16:40:07 -0700 Subject: [PATCH 004/132] Auto-update dependencies. (#148) --- scripts/requirements-test.txt | 2 +- tests/python2-libraries/requirements.txt | 92 +++++++++++------------ tests/python3-libraries/requirements.txt | 94 ++++++++++++------------ 3 files changed, 94 insertions(+), 94 deletions(-) diff --git a/scripts/requirements-test.txt b/scripts/requirements-test.txt index 6e01a83d..5afb5315 100644 --- a/scripts/requirements-test.txt +++ b/scripts/requirements-test.txt @@ -1,4 +1,4 @@ flask==0.12.2 -pytest==3.2.1 +pytest==3.2.3 pytest-cov==2.5.1 pyyaml==3.12 diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 91807712..87e9e53a 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -1,13 +1,13 @@ -alembic==0.9.5 -amqp==2.2.1 +alembic==0.9.6 +amqp==2.2.2 amqplib==1.0.2 -ansible==2.3.2.0 +ansible==2.4.0.0 anyjson==0.3.3 -apache-libcloud==2.2.0 +apache-libcloud==2.2.1 argparse==1.4.0 astroid==1.5.3 -awscli==1.11.146 -babel==2.5.0 +awscli==1.11.170 +babel==2.5.1 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 beautifulsoup4==4.6.0 @@ -16,31 +16,31 @@ billiard==3.5.0.3 blessings==1.6 blinker==1.4 boto==2.48.0 -botocore==1.7.4 +botocore==1.7.28 bottle==0.12.13 carbon==1.0.2 celery==4.1.0 certifi==2017.7.27.1 -cffi==1.10.0 +cffi==1.11.2 chardet==3.0.4 click==6.7 -cliff==2.8.0 +cliff==2.9.1 cmd2==0.7.7 colorama==0.3.9 configobj==5.0.6 cov-core==1.15.0 coverage==4.4.1 coveralls==1.2.0 -cryptography==2.0.3 +cryptography==2.1.1 cssselect==1.0.1 -cython==0.26.1 +cython==0.27.1 decorator==4.1.2 django-celery==3.2.1 django-debug-toolbar==1.8 -django-extensions==1.9.0 -django==1.11.5 +django-extensions==1.9.1 +django==1.11.6 django_compress==1.0.1 -djangorestframework==3.6.4 +djangorestframework==3.7.0 docker-py==1.10.6 docopt==0.6.2 docutils==0.14 @@ -57,7 +57,7 @@ funcsigs==1.0.2 functools32==3.2.3.post2 futures==3.1.1 gevent==1.2.2 -google-api-python-client==1.6.3 +google-api-python-client==1.6.4 graphite-web==1.0.2 greenlet==0.4.12 gunicorn==19.7.1 @@ -67,7 +67,7 @@ httplib2==0.10.3 idna==2.6 ipaddress==1.0.18 iso8601==0.1.12 -isodate==0.5.4 +isodate==0.6.0 itsdangerous==0.24 jinja2==2.9.6 jmespath==0.9.3 @@ -75,13 +75,13 @@ jsonschema==2.6.0 kombu==4.1.0 linecache2==1.0.0 logilab-common==1.4.1 -lxml==3.8.0 -m2crypto==0.26.0 +lxml==4.1.0 +m2crypto==0.27.0 mako==1.0.7 manifestparser==1.1 markdown==2.6.9 markupsafe==1.0 -matplotlib==2.0.2 +matplotlib==2.1.0 mccabe==0.6.1 meld3==1.0.2 mock==2.0.0 @@ -99,16 +99,16 @@ mysql-python==1.2.5 ndg-httpsclient==0.4.3 netaddr==0.7.19 netifaces==0.10.6 -newrelic==2.90.0.75 +newrelic==2.94.0.79 nose==1.3.7 -numpy==1.13.1 +numpy==1.13.3 oauth2==1.9.0.post1 oauth2client==4.1.2 -oauthlib==2.0.2 +oauthlib==2.0.4 ordereddict==1.1 -oslo.config==4.12.0 +oslo.config==4.13.1 pandas==0.20.3 -paramiko==2.2.1 +paramiko==2.3.1 passlib==1.7.1 paste==2.0.3 pastedeploy==1.5.2 @@ -117,15 +117,15 @@ pbr==3.1.1 pep8==1.7.0 pexpect==4.2.1 pika==0.11.0 -pillow==4.2.1 +pillow==4.3.0 pip==9.0.1 prettytable protobuf==3.4.0 -psutil==5.3.0 +psutil==5.4.0 psycopg2==2.7.3.1 py==1.4.34 -pyasn1-modules==0.1.1 -pyasn1==0.3.3 +pyasn1-modules==0.1.5 +pyasn1==0.3.7 pycparser==2.18 pycrypto==2.6.1 pycurl==7.43.0 @@ -133,15 +133,15 @@ pyflakes==1.6.0 pygments==2.2.0 pyjwt==1.5.3 pylibmc==1.5.2 -pylint==1.7.2 +pylint==1.7.4 pymongo==3.5.1 pymysql==0.7.11 -pyopenssl==17.2.0 +pyopenssl==17.3.0 pyparsing==2.2.0 pyramid==1.9.1 pystache==0.5.4 pytest-cov==2.5.1 -pytest==3.2.1 +pytest==3.2.3 python-cjson==1.2.1 python-daemon==2.1.2 python-dateutil==2.6.1 @@ -155,36 +155,36 @@ python-swiftclient==3.4.0 pytz==2017.2 pyyaml==3.12 pyzmq==16.0.2 -raven==6.1.0 +raven==6.2.1 redis==2.10.6 -repoze.lru==0.6 +repoze.lru==0.7 requests-oauthlib==0.8.0 requests==2.18.4 retrying==1.3.3 rsa==3.4.2 scipy==0.19.1 -selenium==3.5.0 +selenium==3.6.0 setuptools-git==1.2 -setuptools==36.4.0 +setuptools==36.6.0 sh==1.12.14 simplejson==3.11.1 -six==1.10.0 +six==1.11.0 snowballstemmer==1.2.1 south==1.0.2 -sphinx==1.6.3 +sphinx==1.6.4 sqlalchemy-migrate==0.11.0 -sqlalchemy==1.1.11 -sqlparse==0.2.3 +sqlalchemy==1.1.14 +sqlparse==0.2.4 statsd==3.2.1 -stevedore==1.26.0 +stevedore==1.27.1 suds==0.4 supervisor==3.3.3 testrepository==0.0.20 testtools==2.3.0 thrift==0.10.0 tornado==4.5.2 -tox==2.8.1 -twisted==17.5.0 +tox==2.9.1 +twisted==17.9.0 ujson==1.35 unidecode==0.4.21 unittest2==1.1.0 @@ -193,13 +193,13 @@ urllib3==1.22 uwsgi==2.0.15 versiontools==1.9.1 virtualenv==15.1.0 -waitress==1.0.2 +waitress==1.1.0 warlock==1.3.0 webob==1.7.3 websocket-client==0.44.0 webtest==2.0.28 werkzeug==0.12.2 -wheel==0.29.0 +wheel==0.30.0 xlrd==1.1.0 -zc.buildout==2.9.4 -zope.interface==4.4.2 +zc.buildout==2.9.5 +zope.interface==4.4.3 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index f2d6973b..614f2f37 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -1,13 +1,13 @@ -alembic==0.9.5 -amqp==2.2.1 +alembic==0.9.6 +amqp==2.2.2 amqplib==1.0.2 -ansible==2.3.2.0 +ansible==2.4.0.0 anyjson==0.3.3 -apache-libcloud==2.2.0 +apache-libcloud==2.2.1 argparse==1.4.0 astroid==1.5.3 -awscli==1.11.146 -babel==2.5.0 +awscli==1.11.170 +babel==2.5.1 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 beautifulsoup4==4.6.0 @@ -15,30 +15,30 @@ billiard==3.5.0.3 blessings==1.6 blinker==1.4 boto==2.48.0 -botocore==1.7.4 +botocore==1.7.28 bottle==0.12.13 celery==4.1.0 certifi==2017.7.27.1 -cffi==1.10.0 +cffi==1.11.2 chardet==3.0.4 click==6.7 -cliff==2.8.0 +cliff==2.9.1 cmd2==0.7.7 colorama==0.3.9 configobj==5.0.6 cov-core==1.15.0 coverage==4.4.1 coveralls==1.2.0 -cryptography==2.0.3 +cryptography==2.1.1 cssselect==1.0.1 -cython==0.26.1 +cython==0.27.1 decorator==4.1.2 django-celery==3.2.1 django-debug-toolbar==1.8 -django-extensions==1.9.0 -django==1.11.5 +django-extensions==1.9.1 +django==1.11.6 django_compress==1.0.1 -djangorestframework==3.6.4 +djangorestframework==3.7.0 docker-py==1.10.6 docopt==0.6.2 docutils==0.14 @@ -54,7 +54,7 @@ flask==0.12.2 funcsigs==1.0.2 futures==3.1.1 gevent==1.2.2 -google-api-python-client==1.6.3 +google-api-python-client==1.6.4 greenlet==0.4.12 gunicorn==19.7.1 hiredis==0.2.0 @@ -62,9 +62,9 @@ html5lib httplib2==0.10.3 idna==2.6 ipaddress==1.0.18 -ipython==6.1.0 +ipython==6.2.1 iso8601==0.1.12 -isodate==0.5.4 +isodate==0.6.0 itsdangerous==0.24 jinja2==2.9.6 jmespath==0.9.3 @@ -72,13 +72,13 @@ jsonschema==2.6.0 kombu==4.1.0 linecache2==1.0.0 logilab-common==1.4.1 -lxml==3.8.0 -m2crypto==0.26.0 +lxml==4.1.0 +m2crypto==0.27.0 mako==1.0.7 manifestparser==1.1 markdown==2.6.9 markupsafe==1.0 -matplotlib==2.0.2 +matplotlib==2.1.0 mccabe==0.6.1 meld3==1.0.2 mock==2.0.0 @@ -93,16 +93,16 @@ msgpack-python==0.4.8 ndg-httpsclient==0.4.3 netaddr==0.7.19 netifaces==0.10.6 -newrelic==2.90.0.75 +newrelic==2.94.0.79 nose==1.3.7 -numpy==1.13.1 +numpy==1.13.3 oauth2==1.9.0.post1 oauth2client==4.1.2 -oauthlib==2.0.2 +oauthlib==2.0.4 ordereddict==1.1 -oslo.config==4.12.0 +oslo.config==4.13.1 pandas==0.20.3 -paramiko==2.2.1 +paramiko==2.3.1 passlib==1.7.1 paste==2.0.3 pastedeploy==1.5.2 @@ -111,30 +111,30 @@ pbr==3.1.1 pep8==1.7.0 pexpect==4.2.1 pika==0.11.0 -pillow==4.2.1 +pillow==4.3.0 pip==9.0.1 prettytable protobuf==3.4.0 -psutil==5.3.0 +psutil==5.4.0 psycopg2==2.7.3.1 py==1.4.34 -pyasn1-modules==0.1.1 -pyasn1==0.3.3 +pyasn1-modules==0.1.5 +pyasn1==0.3.7 pycparser==2.18 pycrypto==2.6.1 pyflakes==1.6.0 pygments==2.2.0 pyjwt==1.5.3 pylibmc==1.5.2 -pylint==1.7.2 +pylint==1.7.4 pymongo==3.5.1 pymysql==0.7.11 -pyopenssl==17.2.0 +pyopenssl==17.3.0 pyparsing==2.2.0 pyramid==1.9.1 pystache==0.5.4 pytest-cov==2.5.1 -pytest==3.2.1 +pytest==3.2.3 python-daemon==2.1.2 python-dateutil==2.6.1 python-gflags==3.1.1 @@ -147,34 +147,34 @@ python-swiftclient==3.4.0 pytz==2017.2 pyyaml==3.12 pyzmq==16.0.2 -raven==6.1.0 +raven==6.2.1 redis==2.10.6 -repoze.lru==0.6 +repoze.lru==0.7 requests-oauthlib==0.8.0 requests==2.18.4 retrying==1.3.3 rsa==3.4.2 scipy==0.19.1 -selenium==3.5.0 +selenium==3.6.0 setuptools-git==1.2 -setuptools==36.4.0 +setuptools==36.6.0 sh==1.12.14 simplejson==3.11.1 -six==1.10.0 +six==1.11.0 snowballstemmer==1.2.1 south==1.0.2 -sphinx==1.6.3 +sphinx==1.6.4 sqlalchemy-migrate==0.11.0 -sqlalchemy==1.1.11 -sqlparse==0.2.3 +sqlalchemy==1.1.14 +sqlparse==0.2.4 statsd==3.2.1 -stevedore==1.26.0 +stevedore==1.27.1 testrepository==0.0.20 testtools==2.3.0 thrift==0.10.0 tornado==4.5.2 -tox==2.8.1 -twisted==17.5.0 +tox==2.9.1 +twisted==17.9.0 ujson==1.35 unidecode==0.4.21 unittest2==1.1.0 @@ -183,13 +183,13 @@ urllib3==1.22 uwsgi==2.0.15 versiontools==1.9.1 virtualenv==15.1.0 -waitress==1.0.2 +waitress==1.1.0 warlock==1.3.0 webob==1.7.3 websocket-client==0.44.0 webtest==2.0.28 werkzeug==0.12.2 -wheel==0.29.0 +wheel==0.30.0 xlrd==1.1.0 -zc.buildout==2.9.4 -zope.interface==4.4.2 +zc.buildout==2.9.5 +zope.interface==4.4.3 From 8df5ad6b9fa2c99ced22302d4867fdf73cf4d60b Mon Sep 17 00:00:00 2001 From: Angela Li Date: Fri, 20 Oct 2017 11:13:09 -0700 Subject: [PATCH 005/132] Upgrade setuptools in test (#154) --- tests/google-cloud-python/Dockerfile.in | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/google-cloud-python/Dockerfile.in b/tests/google-cloud-python/Dockerfile.in index a2873863..b82c18d1 100644 --- a/tests/google-cloud-python/Dockerfile.in +++ b/tests/google-cloud-python/Dockerfile.in @@ -4,6 +4,9 @@ FROM ${STAGING_IMAGE} RUN git clone --depth 1 https://github.com/GoogleCloudPlatform/google-cloud-python.git WORKDIR google-cloud-python +# Upgrade setuptools +RUN pip install --upgrade setuptools + # Install nox RUN pip install --upgrade nox-automation From 44af51158a5ca6eb5e7ec5ee4981613980d7c629 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 20 Oct 2017 15:36:14 -0700 Subject: [PATCH 006/132] Separate building and testing phases via --build and --test flags. Sometimes the tests fails for reasons not related to the runtime. This gives us a way to build and push the image anyway. --- build.sh | 72 +++++++++++++++++------------------ cloudbuild.yaml | 25 +----------- cloudbuild_benchmark.yaml | 4 -- cloudbuild_library_tests.yaml | 20 ---------- cloudbuild_system_tests.yaml | 4 -- cloudbuild_tests.yaml | 28 ++++++++++++++ 6 files changed, 65 insertions(+), 88 deletions(-) delete mode 100644 cloudbuild_library_tests.yaml create mode 100644 cloudbuild_tests.yaml diff --git a/build.sh b/build.sh index 2f0595e9..ca651fb7 100755 --- a/build.sh +++ b/build.sh @@ -18,9 +18,9 @@ set -euo pipefail # Actions benchmark=0 # Should run benchmarks? -build=1 # Should build images? -library_tests=0 # Should try to install top N Python libraries +build=0 # Should build images? system_tests=0 # Should run system tests? +tests=0 # Should run standard test suite? local=0 # Should run using local Docker daemon instead of GCR? @@ -40,32 +40,37 @@ Build and test artifacts in this repository Options: --[no]benchmark: Run benchmarking suite (default false) - --[no]build: Build all images (default true) - --[no]library_tests: Run library compatiblity tests (default false) + --[no]build: Build all images (default true if no options set) + --[no]tests: Run basic tests (default true if no options set) --[no]local: Build images using local Docker daemon (default false) --[no]system_tests: Run system tests (default false) " } - + # Read environment variables -if [ -z "${DOCKER_NAMESPACE+set}" ] ; then +if [ -z "${DOCKER_NAMESPACE:+set}" ] ; then fatal 'Error: $DOCKER_NAMESPACE is not set; invoke with something like DOCKER_NAMESPACE=gcr.io/YOUR-PROJECT-NAME' fi -if [ -z "${BUILDER_DOCKER_NAMESPACE+set}" ] ; then +if [ -z "${BUILDER_DOCKER_NAMESPACE:+set}" ] ; then export BUILDER_DOCKER_NAMESPACE="${DOCKER_NAMESPACE}" fi -if [ -z "${TAG+set}" ] ; then +if [ -z "${TAG:+set}" ] ; then export TAG=`date +%Y-%m-%d-%H%M%S` fi -substitutions="\ +build_substitutions="\ _BUILDER_DOCKER_NAMESPACE=${BUILDER_DOCKER_NAMESPACE},\ _DOCKER_NAMESPACE=${DOCKER_NAMESPACE},\ _TAG=${TAG}\ " +substitutions="\ +_DOCKER_NAMESPACE=${DOCKER_NAMESPACE},\ +_TAG=${TAG}\ +" + # Read command line arguments while [ $# -gt 0 ]; do case "$1" in @@ -85,14 +90,6 @@ while [ $# -gt 0 ]; do build=0 shift ;; - --library_tests) - library_tests=1 - shift - ;; - --nolibrary_tests) - library_tests=0 - shift - ;; --local) local=1 shift @@ -109,6 +106,14 @@ while [ $# -gt 0 ]; do system_tests=0 shift ;; + --tests) + tests=1 + shift + ;; + --notests) + tests=0 + shift + ;; *) usage ;; @@ -118,9 +123,12 @@ done # If no actions chosen, then tell the user if [ "${benchmark}" -eq 0 -a \ "${build}" -eq 0 -a \ - "${library_tests}" -eq 0 -a \ - "${system_tests}" -eq 0 ]; then - fatal 'Error: No actions specified (for example, --build), exiting' + "${system_tests}" -eq 0 -a \ + "${tests}" -eq 0 \ +]; then + echo 'No actions specified, defaulting to --build --tests' + build=1 + tests=1 fi # Running build local or remote? @@ -155,7 +163,8 @@ for outfile in \ tests/google-cloud-python-system/Dockerfile \ tests/integration/Dockerfile \ ; do - envsubst <"${outfile}".in >"${outfile}" '$DEBIAN_BASE_IMAGE $STAGING_IMAGE $GOOGLE_CLOUD_PROJECT_FOR_TESTS' + envsubst <"${outfile}".in >"${outfile}" \ + '$DEBIAN_BASE_IMAGE $STAGING_IMAGE $GOOGLE_CLOUD_PROJECT_FOR_TESTS $TAG' done # Make some files available to the runtime builder Docker context @@ -174,36 +183,27 @@ cp -a scripts/testdata/hello_world/main.py tests/eventlet/main.py # Build images and push to GCR if [ "${build}" -eq 1 ]; then echo "Building images" - ${gcloud_cmd} --config cloudbuild.yaml --substitutions "${substitutions}" + ${gcloud_cmd} --config cloudbuild.yaml --substitutions "${build_substitutions}" fi -# Run just the library compatibility tests (for DPE Gardener bot usually) -if [ "${library_tests}" -eq 1 ]; then +# Run the tests that don't require (too many) external services +if [ "${tests}" -eq 1 ]; then echo "Testing compatibility with popular Python libraries" - ${gcloud_cmd} --config cloudbuild_library_tests.yaml --substitutions "${substitutions}" + ${gcloud_cmd} --config cloudbuild_tests.yaml --substitutions "${substitutions}" fi -# If both system tests and benchmarks are requested, run them both -# even if one or the other has errors. If the build step had errors, -# this script will have already exited. -exit_code=0 - # Run system tests if [ "${system_tests}" -eq 1 ]; then echo "Running system tests using project ${GOOGLE_CLOUD_PROJECT_FOR_TESTS}" trap "rm -f tests/google-cloud-python-system/credentials.json" EXIT cp "${GOOGLE_APPLICATION_CREDENTIALS_FOR_TESTS}" tests/google-cloud-python-system/credentials.json - ${gcloud_cmd} --config cloudbuild_system_tests.yaml --substitutions "${substitutions}" || \ - exit_code=1 + ${gcloud_cmd} --config cloudbuild_system_tests.yaml --substitutions "${substitutions}" rm -f tests/google-cloud-python-system/credentials.json fi # Run benchmarks if [ "${benchmark}" -eq 1 ] ; then echo "Running benchmark" - ${gcloud_cmd} --config cloudbuild_benchmark.yaml --substitutions "${substitutions}" || \ - exit_code=1 + ${gcloud_cmd} --config cloudbuild_benchmark.yaml --substitutions "${substitutions}" fi - -exit ${exit_code} diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 68b5c5bb..1eabb997 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -11,35 +11,12 @@ steps: name: gcr.io/cloud-builders/docker:latest args: ['build', '--tag=${_DOCKER_NAMESPACE}/python:${_TAG}', '--no-cache', '/workspace/runtime-image/'] -- # Validate structure of base runtime image - name: gcr.io/gcp-runtimes/structure_test:latest - args: [ - '-i', '${_DOCKER_NAMESPACE}/python:${_TAG}', - '--config', '/workspace/tests/virtualenv/virtualenv_default.yaml', - '--config', '/workspace/tests/virtualenv/virtualenv_python34.yaml', - '--config', '/workspace/tests/virtualenv/virtualenv_python35.yaml', - '--config', '/workspace/tests/virtualenv/virtualenv_python36.yaml', - '--config', '/workspace/tests/no-virtualenv/no-virtualenv.yaml', - '--config', '/workspace/tests/python2-libraries/python2-libraries.yaml', - '--config', '/workspace/tests/python3-libraries/python3-libraries.yaml', - '--config', '/workspace/tests/license-test/license-test.yaml', - '-v' - ] -- # Run compatibility tests - name: gcr.io/cloud-builders/docker:latest - args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/eventlet:${_TAG}', - '--no-cache', '/workspace/tests/eventlet/'] -- # Build image to run google client library unit tests - name: gcr.io/cloud-builders/docker:latest - args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/google-cloud-python:${_TAG}', - '--no-cache', '/workspace/tests/google-cloud-python/'] -- # Run google client library unit tests - name: ${_DOCKER_NAMESPACE}/python/tests/google-cloud-python:${_TAG} - # Build runtime builder image name: gcr.io/cloud-builders/docker:latest args: ['build', '--tag=${_BUILDER_DOCKER_NAMESPACE}/python/gen-dockerfile:${_TAG}', '--no-cache', '/workspace/builder/gen-dockerfile/'] images: [ + '${_DOCKER_NAMESPACE}/python/interpreter-builder:${_TAG}', '${_DOCKER_NAMESPACE}/python:${_TAG}', '${_BUILDER_DOCKER_NAMESPACE}/python/gen-dockerfile:${_TAG}', ] diff --git a/cloudbuild_benchmark.yaml b/cloudbuild_benchmark.yaml index 616991ef..a960bc9b 100644 --- a/cloudbuild_benchmark.yaml +++ b/cloudbuild_benchmark.yaml @@ -3,10 +3,6 @@ steps: - name: gcr.io/cloud-builders/docker:latest args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/benchmark:${_TAG}', '--no-cache', '/workspace/tests/benchmark/'] - env: [ - # Avoid warning about unused substitutions - 'UNUSED1=${_BUILDER_DOCKER_NAMESPACE}', - ] images: [ # Intentionally empty ] diff --git a/cloudbuild_library_tests.yaml b/cloudbuild_library_tests.yaml deleted file mode 100644 index f547d73f..00000000 --- a/cloudbuild_library_tests.yaml +++ /dev/null @@ -1,20 +0,0 @@ -timeout: 1800s -steps: -- # Check that we can install important libraries without error - name: gcr.io/gcp-runtimes/structure_test:latest - args: [ - '-i', '${_DOCKER_NAMESPACE}/python:${_TAG}', - '--config', '/workspace/tests/python2-libraries/python2-libraries.yaml', - '--config', '/workspace/tests/python3-libraries/python3-libraries.yaml', - '-v' - ] - env: [ - # Avoid warning about unused substitutions - 'UNUSED1=${_BUILDER_DOCKER_NAMESPACE}', - ] -- # Run compatibility tests - name: gcr.io/cloud-builders/docker:latest - args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/eventlet:${_TAG}', - '--no-cache', '/workspace/tests/eventlet/'] -images: [ -] diff --git a/cloudbuild_system_tests.yaml b/cloudbuild_system_tests.yaml index f01a48e7..3cac03e2 100644 --- a/cloudbuild_system_tests.yaml +++ b/cloudbuild_system_tests.yaml @@ -4,10 +4,6 @@ steps: args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/google-cloud-python-system:${_TAG}', '--no-cache', '/workspace/tests/google-cloud-python-system/'] - name: ${_DOCKER_NAMESPACE}/python/tests/google-cloud-python-system:${_TAG} - env: [ - # Avoid warning about unused substitutions - 'UNUSED1=${_BUILDER_DOCKER_NAMESPACE}', - ] images: [ # Intentionally empty ] diff --git a/cloudbuild_tests.yaml b/cloudbuild_tests.yaml new file mode 100644 index 00000000..806e8b12 --- /dev/null +++ b/cloudbuild_tests.yaml @@ -0,0 +1,28 @@ +timeout: 3600s +steps: +- # Validate structure of base runtime image + name: gcr.io/gcp-runtimes/structure_test:latest + args: [ + '-i', '${_DOCKER_NAMESPACE}/python:${_TAG}', + '--config', '/workspace/tests/virtualenv/virtualenv_default.yaml', + '--config', '/workspace/tests/virtualenv/virtualenv_python34.yaml', + '--config', '/workspace/tests/virtualenv/virtualenv_python35.yaml', + '--config', '/workspace/tests/virtualenv/virtualenv_python36.yaml', + '--config', '/workspace/tests/no-virtualenv/no-virtualenv.yaml', + '--config', '/workspace/tests/python2-libraries/python2-libraries.yaml', + '--config', '/workspace/tests/python3-libraries/python3-libraries.yaml', + '--config', '/workspace/tests/license-test/license-test.yaml', + '-v' + ] +- # Run compatibility tests + name: gcr.io/cloud-builders/docker:latest + args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/eventlet:${_TAG}', + '--no-cache', '/workspace/tests/eventlet/'] +- # Build image to run google client library unit tests + name: gcr.io/cloud-builders/docker:latest + args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/google-cloud-python:${_TAG}', + '--no-cache', '/workspace/tests/google-cloud-python/'] +- # Run google client library unit tests + name: ${_DOCKER_NAMESPACE}/python/tests/google-cloud-python:${_TAG} +images: [ +] From f6bc889f1bb99dee8277b43d6a8dea1d00c91f4e Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 20 Oct 2017 15:53:28 -0700 Subject: [PATCH 007/132] Remove --tests and --systems_tests to --test and --system_test --- build.sh | 38 +++++++++---------- ..._tests.yaml => cloudbuild_system_test.yaml | 0 cloudbuild_tests.yaml => cloudbuild_test.yaml | 0 3 files changed, 19 insertions(+), 19 deletions(-) rename cloudbuild_system_tests.yaml => cloudbuild_system_test.yaml (100%) rename cloudbuild_tests.yaml => cloudbuild_test.yaml (100%) diff --git a/build.sh b/build.sh index ca651fb7..ba20b8c8 100755 --- a/build.sh +++ b/build.sh @@ -19,8 +19,8 @@ set -euo pipefail # Actions benchmark=0 # Should run benchmarks? build=0 # Should build images? -system_tests=0 # Should run system tests? -tests=0 # Should run standard test suite? +system_test=0 # Should run system tests? +test=0 # Should run standard test suite? local=0 # Should run using local Docker daemon instead of GCR? @@ -41,9 +41,9 @@ Build and test artifacts in this repository Options: --[no]benchmark: Run benchmarking suite (default false) --[no]build: Build all images (default true if no options set) - --[no]tests: Run basic tests (default true if no options set) + --[no]test: Run basic tests (default true if no options set) --[no]local: Build images using local Docker daemon (default false) - --[no]system_tests: Run system tests (default false) + --[no]system_test: Run system tests (default false) " } @@ -98,20 +98,20 @@ while [ $# -gt 0 ]; do local=0 shift ;; - --system_tests) - system_tests=1 + --system_test) + system_test=1 shift ;; - --nosystem_tests) - system_tests=0 + --nosystem_test) + system_test=0 shift ;; - --tests) - tests=1 + --test) + test=1 shift ;; - --notests) - tests=0 + --notest) + test=0 shift ;; *) @@ -123,12 +123,12 @@ done # If no actions chosen, then tell the user if [ "${benchmark}" -eq 0 -a \ "${build}" -eq 0 -a \ - "${system_tests}" -eq 0 -a \ - "${tests}" -eq 0 \ + "${system_test}" -eq 0 -a \ + "${test}" -eq 0 \ ]; then - echo 'No actions specified, defaulting to --build --tests' + echo 'No actions specified, defaulting to --build --test' build=1 - tests=1 + test=1 fi # Running build local or remote? @@ -137,7 +137,7 @@ if [ "${local}" -eq 1 ]; then fi # Read action-specific environment variables -if [ "${system_tests}" -eq 1 ]; then +if [ "${system_test}" -eq 1 ]; then if [ -z "${GOOGLE_APPLICATION_CREDENTIALS_FOR_TESTS+set}" ] ; then fatal 'Error: $GOOGLE_APPLICATION_CREDENTIALS_FOR_TESTS is not set; invoke with something like GOOGLE_APPLICATION_CREDENTIALS_FOR_TESTS=/path/to/service/account/creds.json' fi @@ -187,13 +187,13 @@ if [ "${build}" -eq 1 ]; then fi # Run the tests that don't require (too many) external services -if [ "${tests}" -eq 1 ]; then +if [ "${test}" -eq 1 ]; then echo "Testing compatibility with popular Python libraries" ${gcloud_cmd} --config cloudbuild_tests.yaml --substitutions "${substitutions}" fi # Run system tests -if [ "${system_tests}" -eq 1 ]; then +if [ "${system_test}" -eq 1 ]; then echo "Running system tests using project ${GOOGLE_CLOUD_PROJECT_FOR_TESTS}" trap "rm -f tests/google-cloud-python-system/credentials.json" EXIT diff --git a/cloudbuild_system_tests.yaml b/cloudbuild_system_test.yaml similarity index 100% rename from cloudbuild_system_tests.yaml rename to cloudbuild_system_test.yaml diff --git a/cloudbuild_tests.yaml b/cloudbuild_test.yaml similarity index 100% rename from cloudbuild_tests.yaml rename to cloudbuild_test.yaml From b25687d8fe0348421901fb0b255417649691e2ed Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 20 Oct 2017 16:52:19 -0700 Subject: [PATCH 008/132] Fix typo in filename --- build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sh b/build.sh index ba20b8c8..7e2370d4 100755 --- a/build.sh +++ b/build.sh @@ -189,7 +189,7 @@ fi # Run the tests that don't require (too many) external services if [ "${test}" -eq 1 ]; then echo "Testing compatibility with popular Python libraries" - ${gcloud_cmd} --config cloudbuild_tests.yaml --substitutions "${substitutions}" + ${gcloud_cmd} --config cloudbuild_test.yaml --substitutions "${substitutions}" fi # Run system tests @@ -198,7 +198,7 @@ if [ "${system_test}" -eq 1 ]; then trap "rm -f tests/google-cloud-python-system/credentials.json" EXIT cp "${GOOGLE_APPLICATION_CREDENTIALS_FOR_TESTS}" tests/google-cloud-python-system/credentials.json - ${gcloud_cmd} --config cloudbuild_system_tests.yaml --substitutions "${substitutions}" + ${gcloud_cmd} --config cloudbuild_system_test.yaml --substitutions "${substitutions}" rm -f tests/google-cloud-python-system/credentials.json fi From ca918fda5a777bc7a127fa52c8d207d4f038c7dd Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 20 Oct 2017 17:07:23 -0700 Subject: [PATCH 009/132] Remove m2crypto package which doesn't work under Python 3 --- tests/python3-libraries/requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 614f2f37..edbe81f6 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -73,7 +73,6 @@ kombu==4.1.0 linecache2==1.0.0 logilab-common==1.4.1 lxml==4.1.0 -m2crypto==0.27.0 mako==1.0.7 manifestparser==1.1 markdown==2.6.9 From d4afefe05e8b5ae72faba986d9429bfbc2f6c919 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 20 Oct 2017 18:44:30 -0700 Subject: [PATCH 010/132] Add crcmod to compatilibity test suite, it's used by gsutil. --- tests/python2-libraries/requirements.txt | 1 + tests/python3-libraries/requirements.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 87e9e53a..8711bad0 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -31,6 +31,7 @@ configobj==5.0.6 cov-core==1.15.0 coverage==4.4.1 coveralls==1.2.0 +crcmod==1.7 cryptography==2.1.1 cssselect==1.0.1 cython==0.27.1 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index edbe81f6..dba60564 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -29,6 +29,7 @@ configobj==5.0.6 cov-core==1.15.0 coverage==4.4.1 coveralls==1.2.0 +crcmod==1.7 cryptography==2.1.1 cssselect==1.0.1 cython==0.27.1 From 509cb5ce2768733948f3da42d25bac3ff735bf75 Mon Sep 17 00:00:00 2001 From: Angela Li Date: Tue, 24 Oct 2017 16:32:44 -0700 Subject: [PATCH 011/132] Remove sub directories of metrics scripts (#159) --- perf_dashboard/__init__.py | 0 perf_dashboard/{stackoverflow => }/posts_stats.py | 4 +--- .../{clientlibs_download => }/python_clientlibs_download.py | 6 ++---- 3 files changed, 3 insertions(+), 7 deletions(-) create mode 100644 perf_dashboard/__init__.py rename perf_dashboard/{stackoverflow => }/posts_stats.py (95%) rename perf_dashboard/{clientlibs_download => }/python_clientlibs_download.py (97%) diff --git a/perf_dashboard/__init__.py b/perf_dashboard/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/perf_dashboard/stackoverflow/posts_stats.py b/perf_dashboard/posts_stats.py similarity index 95% rename from perf_dashboard/stackoverflow/posts_stats.py rename to perf_dashboard/posts_stats.py index 280fd42c..efb9c6fc 100644 --- a/perf_dashboard/stackoverflow/posts_stats.py +++ b/perf_dashboard/posts_stats.py @@ -25,9 +25,7 @@ from google.cloud import bigquery -# Need this to import the local helper function -sys.path.insert(0, os.path.abspath(__file__+"/../../..")) -from perf_dashboard import bq_utils +import bq_utils GCLOUD_PROJECT_ENV = 'GCLOUD_PROJECT' DATASET_NAME = 'stackoverflow' diff --git a/perf_dashboard/clientlibs_download/python_clientlibs_download.py b/perf_dashboard/python_clientlibs_download.py similarity index 97% rename from perf_dashboard/clientlibs_download/python_clientlibs_download.py rename to perf_dashboard/python_clientlibs_download.py index 3c8f79fe..3866fd92 100644 --- a/perf_dashboard/clientlibs_download/python_clientlibs_download.py +++ b/perf_dashboard/python_clientlibs_download.py @@ -20,9 +20,7 @@ from google.cloud import bigquery -sys.path.insert(0, os.path.abspath(__file__+"/../../..")) -from perf_dashboard import bq_utils - +import bq_utils GCLOUD_PROJECT_ENV = 'GCLOUD_PROJECT' @@ -136,7 +134,7 @@ def main(): clientlibs_table_name=table_name, date_str=datetime.datetime.now().strftime("%Y%m%d")) bq_utils.insert_rows( - project=project, + project=os.environ.get(GCLOUD_PROJECT_ENV), dataset_name=DATASET_NAME, table_name=table_name, rows=rows) From 2e621ba203ed43d7e562cbc35850cdc6ac97dd7a Mon Sep 17 00:00:00 2001 From: Douglas Greiman Date: Tue, 31 Oct 2017 11:05:07 -0700 Subject: [PATCH 012/132] Add CODEOWNERS file (#162) --- CODEOWNERS | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 CODEOWNERS diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 00000000..a2d4a67f --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1,4 @@ +# Code owners file. +# This file controls who is tagged for review for any given pull request. + +* @duggelz @liyanhui1228 @jonparrott From 29fe28bf4fbb8e3b02122081d5908ee7c71450e7 Mon Sep 17 00:00:00 2001 From: Douglas Greiman Date: Tue, 31 Oct 2017 11:05:45 -0700 Subject: [PATCH 013/132] Add honcho package to library test (#163) --- tests/python2-libraries/requirements.txt | 3 ++- tests/python3-libraries/requirements.txt | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 8711bad0..dee9abbf 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -63,7 +63,8 @@ graphite-web==1.0.2 greenlet==0.4.12 gunicorn==19.7.1 hiredis==0.2.0 -html5lib +honcho==1.0.1 +html5lib==0.999999999 httplib2==0.10.3 idna==2.6 ipaddress==1.0.18 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index dba60564..fe9d4156 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -59,7 +59,8 @@ google-api-python-client==1.6.4 greenlet==0.4.12 gunicorn==19.7.1 hiredis==0.2.0 -html5lib +honcho=1.0.1 +html5lib==0.999999999 httplib2==0.10.3 idna==2.6 ipaddress==1.0.18 From 0965d52c5f1b557b17ad480cd9c298cb25ba5c84 Mon Sep 17 00:00:00 2001 From: Douglas Greiman Date: Tue, 31 Oct 2017 11:06:10 -0700 Subject: [PATCH 014/132] Update pip, setuptools, virtualenv when we build the base runtime image. (#161) --- runtime-image/Dockerfile.in | 13 ++++++++----- runtime-image/resources/apt-packages.txt | 1 + runtime-image/resources/requirements.txt | 3 +++ 3 files changed, 12 insertions(+), 5 deletions(-) create mode 100644 runtime-image/resources/requirements.txt diff --git a/runtime-image/Dockerfile.in b/runtime-image/Dockerfile.in index ed194257..60a6fa6f 100644 --- a/runtime-image/Dockerfile.in +++ b/runtime-image/Dockerfile.in @@ -16,15 +16,18 @@ ENV LANG C.UTF-8 # logging collection. ENV PYTHONUNBUFFERED 1 -# Upgrade pip (debian package version tends to run a few version behind) and -# install virtualenv system-wide. -RUN pip install --upgrade pip virtualenv - # Install the Google-built interpreters ADD interpreters.tar.gz / # Add Google-built interpreters to the path -ENV PATH /opt/python3.5/bin:/opt/python3.6/bin:$PATH +ENV PATH /opt/python3.6/bin:/opt/python3.5/bin:$PATH + +# Upgrade pip (debian package version tends to run a few version behind) and +# install virtualenv system-wide. +RUN /usr/bin/pip install --upgrade -r /resources/requirements.txt && \ + /usr/bin/pip3 install --upgrade -r /resources/requirements.txt && \ + /opt/python3.5/bin/pip3.5 install --upgrade -r /resources/requirements.txt && \ + /opt/python3.6/bin/pip3.6 install --upgrade -r /resources/requirements.txt # Setup the app working directory RUN ln -s /home/vmagent/app /app diff --git a/runtime-image/resources/apt-packages.txt b/runtime-image/resources/apt-packages.txt index 1fa11bcb..a62a3195 100644 --- a/runtime-image/resources/apt-packages.txt +++ b/runtime-image/resources/apt-packages.txt @@ -7,6 +7,7 @@ wget python-pip python2.7 python2.7-dev +python3-pip python3.4 python3.4-dev # Dependenies for third-party Python packages diff --git a/runtime-image/resources/requirements.txt b/runtime-image/resources/requirements.txt new file mode 100644 index 00000000..40155447 --- /dev/null +++ b/runtime-image/resources/requirements.txt @@ -0,0 +1,3 @@ +pip==9.0.1 +setuptools==36.6.0 +virtualenv==15.1.0 From ba4e5184b1715f4ee89c68d53bd912c44f216215 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Tue, 31 Oct 2017 11:06:59 -0700 Subject: [PATCH 015/132] Auto-update dependencies. (#153) --- tests/python2-libraries/requirements.txt | 42 ++++++++++++------------ tests/python3-libraries/requirements.txt | 42 ++++++++++++------------ 2 files changed, 42 insertions(+), 42 deletions(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index dee9abbf..6db2c17c 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -1,12 +1,12 @@ alembic==0.9.6 amqp==2.2.2 amqplib==1.0.2 -ansible==2.4.0.0 +ansible==2.4.1.0 anyjson==0.3.3 apache-libcloud==2.2.1 argparse==1.4.0 astroid==1.5.3 -awscli==1.11.170 +awscli==1.11.178 babel==2.5.1 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 @@ -16,7 +16,7 @@ billiard==3.5.0.3 blessings==1.6 blinker==1.4 boto==2.48.0 -botocore==1.7.28 +botocore==1.7.36 bottle==0.12.13 carbon==1.0.2 celery==4.1.0 @@ -32,16 +32,16 @@ cov-core==1.15.0 coverage==4.4.1 coveralls==1.2.0 crcmod==1.7 -cryptography==2.1.1 +cryptography==2.1.2 cssselect==1.0.1 -cython==0.27.1 +cython==0.27.2 decorator==4.1.2 django-celery==3.2.1 django-debug-toolbar==1.8 -django-extensions==1.9.1 +django-extensions==1.9.6 django==1.11.6 django_compress==1.0.1 -djangorestframework==3.7.0 +djangorestframework==3.7.1 docker-py==1.10.6 docopt==0.6.2 docutils==0.14 @@ -52,7 +52,7 @@ eventlet==0.21.0 extras==1.0.0 fabric==1.14.0 fixtures==3.0.0 -flake8==3.4.1 +flake8==3.5.0 flask==0.12.2 funcsigs==1.0.2 functools32==3.2.3.post2 @@ -101,22 +101,22 @@ mysql-python==1.2.5 ndg-httpsclient==0.4.3 netaddr==0.7.19 netifaces==0.10.6 -newrelic==2.94.0.79 +newrelic==2.96.0.80 nose==1.3.7 numpy==1.13.3 oauth2==1.9.0.post1 oauth2client==4.1.2 -oauthlib==2.0.4 +oauthlib==2.0.6 ordereddict==1.1 -oslo.config==4.13.1 -pandas==0.20.3 +oslo.config==5.0.0 +pandas==0.21.0 paramiko==2.3.1 passlib==1.7.1 paste==2.0.3 pastedeploy==1.5.2 pastescript==2.0.2 pbr==3.1.1 -pep8==1.7.0 +pep8==1.7.1 pexpect==4.2.1 pika==0.11.0 pillow==4.3.0 @@ -124,7 +124,7 @@ pip==9.0.1 prettytable protobuf==3.4.0 psutil==5.4.0 -psycopg2==2.7.3.1 +psycopg2==2.7.3.2 py==1.4.34 pyasn1-modules==0.1.5 pyasn1==0.3.7 @@ -147,24 +147,24 @@ pytest==3.2.3 python-cjson==1.2.1 python-daemon==2.1.2 python-dateutil==2.6.1 -python-gflags==3.1.1 +python-gflags==3.1.2 python-keystoneclient==3.13.0 python-memcached==1.58 python-mimeparse==1.6.0 python-novaclient==9.1.0 python-subunit==1.2.0 python-swiftclient==3.4.0 -pytz==2017.2 +pytz==2017.3 pyyaml==3.12 -pyzmq==16.0.2 -raven==6.2.1 +pyzmq==16.0.3 +raven==6.3.0 redis==2.10.6 repoze.lru==0.7 requests-oauthlib==0.8.0 requests==2.18.4 retrying==1.3.3 rsa==3.4.2 -scipy==0.19.1 +scipy==1.0.0 selenium==3.6.0 setuptools-git==1.2 setuptools==36.6.0 @@ -173,7 +173,7 @@ simplejson==3.11.1 six==1.11.0 snowballstemmer==1.2.1 south==1.0.2 -sphinx==1.6.4 +sphinx==1.6.5 sqlalchemy-migrate==0.11.0 sqlalchemy==1.1.14 sqlparse==0.2.4 @@ -199,7 +199,7 @@ waitress==1.1.0 warlock==1.3.0 webob==1.7.3 websocket-client==0.44.0 -webtest==2.0.28 +webtest==2.0.29 werkzeug==0.12.2 wheel==0.30.0 xlrd==1.1.0 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index fe9d4156..cca4bef2 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -1,12 +1,12 @@ alembic==0.9.6 amqp==2.2.2 amqplib==1.0.2 -ansible==2.4.0.0 +ansible==2.4.1.0 anyjson==0.3.3 apache-libcloud==2.2.1 argparse==1.4.0 astroid==1.5.3 -awscli==1.11.170 +awscli==1.11.178 babel==2.5.1 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 @@ -15,7 +15,7 @@ billiard==3.5.0.3 blessings==1.6 blinker==1.4 boto==2.48.0 -botocore==1.7.28 +botocore==1.7.36 bottle==0.12.13 celery==4.1.0 certifi==2017.7.27.1 @@ -30,16 +30,16 @@ cov-core==1.15.0 coverage==4.4.1 coveralls==1.2.0 crcmod==1.7 -cryptography==2.1.1 +cryptography==2.1.2 cssselect==1.0.1 -cython==0.27.1 +cython==0.27.2 decorator==4.1.2 django-celery==3.2.1 django-debug-toolbar==1.8 -django-extensions==1.9.1 +django-extensions==1.9.6 django==1.11.6 django_compress==1.0.1 -djangorestframework==3.7.0 +djangorestframework==3.7.1 docker-py==1.10.6 docopt==0.6.2 docutils==0.14 @@ -50,7 +50,7 @@ eventlet==0.21.0 extras==1.0.0 fabric==1.14.0 fixtures==3.0.0 -flake8==3.4.1 +flake8==3.5.0 flask==0.12.2 funcsigs==1.0.2 futures==3.1.1 @@ -94,22 +94,22 @@ msgpack-python==0.4.8 ndg-httpsclient==0.4.3 netaddr==0.7.19 netifaces==0.10.6 -newrelic==2.94.0.79 +newrelic==2.96.0.80 nose==1.3.7 numpy==1.13.3 oauth2==1.9.0.post1 oauth2client==4.1.2 -oauthlib==2.0.4 +oauthlib==2.0.6 ordereddict==1.1 -oslo.config==4.13.1 -pandas==0.20.3 +oslo.config==5.0.0 +pandas==0.21.0 paramiko==2.3.1 passlib==1.7.1 paste==2.0.3 pastedeploy==1.5.2 pastescript==2.0.2 pbr==3.1.1 -pep8==1.7.0 +pep8==1.7.1 pexpect==4.2.1 pika==0.11.0 pillow==4.3.0 @@ -117,7 +117,7 @@ pip==9.0.1 prettytable protobuf==3.4.0 psutil==5.4.0 -psycopg2==2.7.3.1 +psycopg2==2.7.3.2 py==1.4.34 pyasn1-modules==0.1.5 pyasn1==0.3.7 @@ -138,24 +138,24 @@ pytest-cov==2.5.1 pytest==3.2.3 python-daemon==2.1.2 python-dateutil==2.6.1 -python-gflags==3.1.1 +python-gflags==3.1.2 python-keystoneclient==3.13.0 python-memcached==1.58 python-mimeparse==1.6.0 python-novaclient==9.1.0 python-subunit==1.2.0 python-swiftclient==3.4.0 -pytz==2017.2 +pytz==2017.3 pyyaml==3.12 -pyzmq==16.0.2 -raven==6.2.1 +pyzmq==16.0.3 +raven==6.3.0 redis==2.10.6 repoze.lru==0.7 requests-oauthlib==0.8.0 requests==2.18.4 retrying==1.3.3 rsa==3.4.2 -scipy==0.19.1 +scipy==1.0.0 selenium==3.6.0 setuptools-git==1.2 setuptools==36.6.0 @@ -164,7 +164,7 @@ simplejson==3.11.1 six==1.11.0 snowballstemmer==1.2.1 south==1.0.2 -sphinx==1.6.4 +sphinx==1.6.5 sqlalchemy-migrate==0.11.0 sqlalchemy==1.1.14 sqlparse==0.2.4 @@ -188,7 +188,7 @@ waitress==1.1.0 warlock==1.3.0 webob==1.7.3 websocket-client==0.44.0 -webtest==2.0.28 +webtest==2.0.29 werkzeug==0.12.2 wheel==0.30.0 xlrd==1.1.0 From babc14b911dc1c49d8bdc97130510f89a0bcabf7 Mon Sep 17 00:00:00 2001 From: Nick Kubala Date: Thu, 26 Oct 2017 15:53:51 -0700 Subject: [PATCH 016/132] migrate structure tests to new version --- cloudbuild_test.yaml | 29 +++++++++++++++---- tests/python2-libraries/Dockerfile | 3 ++ .../python2-libraries/python2-libraries.yaml | 5 +--- tests/python3-libraries/Dockerfile | 3 ++ .../python3-libraries/python3-libraries.yaml | 12 +++----- tests/virtualenv/virtualenv_default.yaml | 14 +++++---- tests/virtualenv/virtualenv_python34.yaml | 9 +++--- tests/virtualenv/virtualenv_python35.yaml | 7 ++++- tests/virtualenv/virtualenv_python36.yaml | 7 ++++- 9 files changed, 59 insertions(+), 30 deletions(-) create mode 100644 tests/python2-libraries/Dockerfile create mode 100644 tests/python3-libraries/Dockerfile diff --git a/cloudbuild_test.yaml b/cloudbuild_test.yaml index 806e8b12..fbad487a 100644 --- a/cloudbuild_test.yaml +++ b/cloudbuild_test.yaml @@ -1,7 +1,27 @@ timeout: 3600s steps: -- # Validate structure of base runtime image - name: gcr.io/gcp-runtimes/structure_test:latest + # Validate structure of base runtime image +- name: gcr.io/cloud-builders/docker:latest + args: ['build', '-t', 'python2-libraries-intermediate', '--build-arg', + 'intermediate_image=${_DOCKER_NAMESPACE}/python:${_TAG}', + '/workspace/tests/python2-libraries'] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.1.1 + args: [ + '-test.v', + '-image', 'python2-libraries-intermediate', + '/workspace/tests/python2-libraries/python2-libraries.yaml' + ] +- name: gcr.io/cloud-builders/docker:latest + args: ['build', '-t', 'python3-libraries-intermediate', '--build-arg', + 'intermediate_image=${_DOCKER_NAMESPACE}/python:${_TAG}', + '/workspace/tests/python3-libraries'] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.1.1 + args: [ + '-test.v', + '-image', 'python3-libraries-intermediate', + '/workspace/tests/python3-libraries/python3-libraries.yaml' + ] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.1.1 args: [ '-i', '${_DOCKER_NAMESPACE}/python:${_TAG}', '--config', '/workspace/tests/virtualenv/virtualenv_default.yaml', @@ -9,8 +29,6 @@ steps: '--config', '/workspace/tests/virtualenv/virtualenv_python35.yaml', '--config', '/workspace/tests/virtualenv/virtualenv_python36.yaml', '--config', '/workspace/tests/no-virtualenv/no-virtualenv.yaml', - '--config', '/workspace/tests/python2-libraries/python2-libraries.yaml', - '--config', '/workspace/tests/python3-libraries/python3-libraries.yaml', '--config', '/workspace/tests/license-test/license-test.yaml', '-v' ] @@ -24,5 +42,4 @@ steps: '--no-cache', '/workspace/tests/google-cloud-python/'] - # Run google client library unit tests name: ${_DOCKER_NAMESPACE}/python/tests/google-cloud-python:${_TAG} -images: [ -] +images: [] diff --git a/tests/python2-libraries/Dockerfile b/tests/python2-libraries/Dockerfile new file mode 100644 index 00000000..c2647292 --- /dev/null +++ b/tests/python2-libraries/Dockerfile @@ -0,0 +1,3 @@ +ARG intermediate_image +FROM $intermediate_image +COPY requirements.txt /requirements.txt diff --git a/tests/python2-libraries/python2-libraries.yaml b/tests/python2-libraries/python2-libraries.yaml index 5d298b58..3a2771ac 100644 --- a/tests/python2-libraries/python2-libraries.yaml +++ b/tests/python2-libraries/python2-libraries.yaml @@ -7,10 +7,7 @@ globalEnvVars: value: "/env/bin:$PATH" commandTests: - - name: "virtual env teardown" - command: ["rm", "-rf", "/env"] - - name: "requirements" setup: [["virtualenv", "/env"]] - command: ["pip", "install", "-r", "/workspace/tests/python2-libraries/requirements.txt"] + command: ["pip", "install", "-r", "/requirements.txt"] exitCode: 0 diff --git a/tests/python3-libraries/Dockerfile b/tests/python3-libraries/Dockerfile new file mode 100644 index 00000000..c2647292 --- /dev/null +++ b/tests/python3-libraries/Dockerfile @@ -0,0 +1,3 @@ +ARG intermediate_image +FROM $intermediate_image +COPY requirements.txt /requirements.txt diff --git a/tests/python3-libraries/python3-libraries.yaml b/tests/python3-libraries/python3-libraries.yaml index ef15c0a1..ace58132 100644 --- a/tests/python3-libraries/python3-libraries.yaml +++ b/tests/python3-libraries/python3-libraries.yaml @@ -8,15 +8,11 @@ globalEnvVars: commandTests: - name: "requirements 3.5" - setup: - - ["rm", "-rf", "/env"] - - ["virtualenv", "-p", "/opt/python3.5/bin/python3.5", "/env"] - command: ["pip", "install", "-r", "/workspace/tests/python3-libraries/requirements.txt"] + setup: [["virtualenv", "-p", "/opt/python3.5/bin/python3.5", "/env"]] + command: ["pip", "install", "-r", "/requirements.txt"] exitCode: 0 - name: "requirements 3.6" - setup: - - ["rm", "-rf", "/env"] - - ["virtualenv", "-p", "/opt/python3.6/bin/python3.6", "/env"] - command: ["pip", "install", "-r", "/workspace/tests/python3-libraries/requirements.txt"] + setup: [["virtualenv", "-p", "/opt/python3.6/bin/python3.6", "/env"]] + command: ["pip", "install", "-r", "/requirements.txt"] exitCode: 0 diff --git a/tests/virtualenv/virtualenv_default.yaml b/tests/virtualenv/virtualenv_default.yaml index 314a3d0c..8695e319 100644 --- a/tests/virtualenv/virtualenv_default.yaml +++ b/tests/virtualenv/virtualenv_default.yaml @@ -7,13 +7,14 @@ globalEnvVars: value: "/env/bin:$PATH" commandTests: - - name: "virtual env teardown" - command: ["rm", "-rf", "/env"] - - name: "python installation" command: ["which", "python"] expectedOutput: ["/usr/bin/python\n"] + - name: "pip installation" + command: ["which", "pip"] + expectedOutput: ["/usr/local/bin/pip\n"] + - name: "virtualenv installation" setup: [["virtualenv", "/env"]] command: ["which", "python"] @@ -25,12 +26,13 @@ commandTests: # https://bugs.python.org/issue18338 expectedError: ["Python 2.7.9\n"] - - name: "pip installation" + - name: "virtualenv pip installation" + setup: [["virtualenv", "/env"]] command: ["which", "pip"] expectedOutput: ["/env/bin/pip\n"] - - name: "gunicorn flask" - setup: [["pip", "install", "gunicorn", "flask"]] + - name: "virtualenv gunicorn flask" + setup: [["virtualenv", "/env"], ["pip", "install", "gunicorn", "flask"]] command: ["which", "gunicorn"] expectedOutput: ["/env/bin/gunicorn"] diff --git a/tests/virtualenv/virtualenv_python34.yaml b/tests/virtualenv/virtualenv_python34.yaml index 082b3b6e..e35e9693 100644 --- a/tests/virtualenv/virtualenv_python34.yaml +++ b/tests/virtualenv/virtualenv_python34.yaml @@ -7,9 +7,6 @@ globalEnvVars: value: "/env/bin:$PATH" commandTests: - - name: "virtual env teardown" - command: ["rm", "-rf", "/env"] - - name: "python installation" command: ["which", "python3.4"] expectedOutput: ["/usr/bin/python3.4\n"] @@ -20,23 +17,27 @@ commandTests: expectedOutput: ["/env/bin/python\n"] - name: "virtualenv python3 installation" + setup: [["virtualenv", "-p", "python3.4", "/env"]] command: ["which", "python3"] expectedOutput: ["/env/bin/python3\n"] - name: "python version" + setup: [["virtualenv", "-p", "python3.4", "/env"]] command: ["python", "--version"] expectedOutput: ["Python 3.4.2\n"] - name: "pip installation" + setup: [["virtualenv", "-p", "python3.4", "/env"]] command: ["which", "pip"] expectedOutput: ["/env/bin/pip\n"] - name: "pip3 installation" + setup: [["virtualenv", "-p", "python3.4", "/env"]] command: ["which", "pip3"] expectedOutput: ["/env/bin/pip3\n"] - name: "gunicorn flask" - setup: [["pip", "install", "gunicorn", "flask"]] + setup: [["virtualenv", "-p", "python3.4", "/env"], ["pip", "install", "gunicorn", "flask"]] command: ["which", "gunicorn"] expectedOutput: ["/env/bin/gunicorn"] diff --git a/tests/virtualenv/virtualenv_python35.yaml b/tests/virtualenv/virtualenv_python35.yaml index c0b461e1..4b019967 100644 --- a/tests/virtualenv/virtualenv_python35.yaml +++ b/tests/virtualenv/virtualenv_python35.yaml @@ -20,27 +20,32 @@ commandTests: expectedOutput: ["/env/bin/python\n"] - name: "virtualenv python3 installation" + setup: [["virtualenv", "-p", "python3.5", "/env"]] command: ["which", "python3"] expectedOutput: ["/env/bin/python3\n"] - name: "virtualenv python3.5 installation" + setup: [["virtualenv", "-p", "python3.5", "/env"]] command: ["which", "python3.5"] expectedOutput: ["/env/bin/python3.5\n"] - name: "python version" + setup: [["virtualenv", "-p", "python3.5", "/env"]] command: ["python", "--version"] expectedOutput: ["Python 3.5.4\n"] - name: "pip installation" + setup: [["virtualenv", "-p", "python3.5", "/env"]] command: ["which", "pip"] expectedOutput: ["/env/bin/pip\n"] - name: "pip3 installation" + setup: [["virtualenv", "-p", "python3.5", "/env"]] command: ["which", "pip3"] expectedOutput: ["/env/bin/pip3\n"] - name: "gunicorn flask" - setup: [["pip", "install", "gunicorn", "flask"]] + setup: [["virtualenv", "-p", "python3.5", "/env"], ["pip", "install", "gunicorn", "flask"]] command: ["which", "gunicorn"] expectedOutput: ["/env/bin/gunicorn"] diff --git a/tests/virtualenv/virtualenv_python36.yaml b/tests/virtualenv/virtualenv_python36.yaml index ab25c28b..deea9b5d 100644 --- a/tests/virtualenv/virtualenv_python36.yaml +++ b/tests/virtualenv/virtualenv_python36.yaml @@ -20,27 +20,32 @@ commandTests: expectedOutput: ["/env/bin/python\n"] - name: "virtualenv python3 installation" + setup: [["virtualenv", "-p", "python3.6", "/env"]] command: ["which", "python3"] expectedOutput: ["/env/bin/python3\n"] - name: "virtualenv python3.6 installation" + setup: [["virtualenv", "-p", "python3.6", "/env"]] command: ["which", "python3.6"] expectedOutput: ["/env/bin/python3.6\n"] - name: "python version" + setup: [["virtualenv", "-p", "python3.6", "/env"]] command: ["python", "--version"] expectedOutput: ["Python 3.6.2\n"] - name: "pip installation" + setup: [["virtualenv", "-p", "python3.6", "/env"]] command: ["which", "pip"] expectedOutput: ["/env/bin/pip\n"] - name: "pip3 installation" + setup: [["virtualenv", "-p", "python3.6", "/env"]] command: ["which", "pip3"] expectedOutput: ["/env/bin/pip3\n"] - name: "gunicorn flask" - setup: [["pip", "install", "gunicorn", "flask"]] + setup: [["virtualenv", "-p", "python3.6", "/env"], ["pip", "install", "gunicorn", "flask"]] command: ["which", "gunicorn"] expectedOutput: ["/env/bin/gunicorn"] From e95dfdd7e4546231f69b13a6c8d182f1b31583bb Mon Sep 17 00:00:00 2001 From: Nick Kubala Date: Mon, 30 Oct 2017 12:45:24 -0700 Subject: [PATCH 017/132] fix args --- cloudbuild_test.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cloudbuild_test.yaml b/cloudbuild_test.yaml index fbad487a..c6496dec 100644 --- a/cloudbuild_test.yaml +++ b/cloudbuild_test.yaml @@ -23,14 +23,14 @@ steps: ] - name: gcr.io/gcp-runtimes/container-structure-test:v0.1.1 args: [ - '-i', '${_DOCKER_NAMESPACE}/python:${_TAG}', - '--config', '/workspace/tests/virtualenv/virtualenv_default.yaml', - '--config', '/workspace/tests/virtualenv/virtualenv_python34.yaml', - '--config', '/workspace/tests/virtualenv/virtualenv_python35.yaml', - '--config', '/workspace/tests/virtualenv/virtualenv_python36.yaml', - '--config', '/workspace/tests/no-virtualenv/no-virtualenv.yaml', - '--config', '/workspace/tests/license-test/license-test.yaml', - '-v' + '-test.v', + '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', + '/workspace/tests/virtualenv/virtualenv_default.yaml', + '/workspace/tests/virtualenv/virtualenv_python34.yaml', + '/workspace/tests/virtualenv/virtualenv_python35.yaml', + '/workspace/tests/virtualenv/virtualenv_python36.yaml', + '/workspace/tests/no-virtualenv/no-virtualenv.yaml', + '/workspace/tests/license-test/license-test.yaml' ] - # Run compatibility tests name: gcr.io/cloud-builders/docker:latest From 8995620601d3db38a9edd9e6ddc48abf2d5739ae Mon Sep 17 00:00:00 2001 From: Nick Kubala Date: Thu, 2 Nov 2017 14:19:23 -0700 Subject: [PATCH 018/132] expose /environment endpoint for gke tests (#164) * expose /environment endpoint for gke tests * mimic logic in python logging client library for determining cloud env --- tests/integration/server.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/integration/server.py b/tests/integration/server.py index cc82fb93..8e2958cd 100755 --- a/tests/integration/server.py +++ b/tests/integration/server.py @@ -17,6 +17,7 @@ from functools import wraps import json import logging +import os import google.cloud.logging import google.cloud.monitoring @@ -33,6 +34,12 @@ 'CRITICAL': (logging.critical, 'stderr') } +_APPENGINE_FLEXIBLE_ENV_VM = 'GAE_APPENGINE_HOSTNAME' +"""Environment variable set in App Engine when vm:true is set.""" + +_APPENGINE_FLEXIBLE_ENV_FLEX = 'GAE_INSTANCE' +"""Environment variable set in App Engine when env:flex is set.""" + app = Flask(__name__) @@ -132,6 +139,7 @@ def _log_default(token, level): logging.error('Error while writing logs: {0}'.format(e)) raise ErrorResponse('Error while writing logs: {0}'.format(e)) + # this is fine regardless of environment, it's only used in GAE logs return 'appengine.googleapis.com%2F{0}'.format(source) @@ -231,6 +239,16 @@ def _custom(): return json.dumps(tests), 200 +@app.route('/environment', methods=['GET']) +def _check_environment(): + # determine what cloud env we're running in; essentially, GAE vs GKE + # for GAE, we'll check the existence env vars set on + # vm:true or env:flex + # if neither exist, assume we're in GKE + return (_APPENGINE_FLEXIBLE_ENV_VM in os.environ or + _APPENGINE_FLEXIBLE_ENV_FLEX in os.environ), 200 + + class ErrorResponse(Exception): status_code = 400 From 5aa45062d97abf6419418069bbd29e1c13b942c8 Mon Sep 17 00:00:00 2001 From: Douglas Greiman Date: Thu, 2 Nov 2017 14:20:09 -0700 Subject: [PATCH 019/132] Create Debian packages for GCP Python interpreters (#140) We build binary packages directly, rather than building from a source package as the standard Debian Python packages are. --- python-interpreter-builder/DEBIAN/control.in | 25 ++++++++ python-interpreter-builder/Dockerfile.in | 12 +++- .../scripts/package-python.sh | 59 +++++++++++++++++++ 3 files changed, 94 insertions(+), 2 deletions(-) create mode 100644 python-interpreter-builder/DEBIAN/control.in create mode 100755 python-interpreter-builder/scripts/package-python.sh diff --git a/python-interpreter-builder/DEBIAN/control.in b/python-interpreter-builder/DEBIAN/control.in new file mode 100644 index 00000000..49980654 --- /dev/null +++ b/python-interpreter-builder/DEBIAN/control.in @@ -0,0 +1,25 @@ +Package: ${DEB_PACKAGE_NAME} +Version: ${DEB_PACKAGE_VERSION} +Section: python +Priority: optional +Architecture: amd64 +Maintainer: Douglas Greiman +Description: Interactive high-level object-oriented language (version ${SHORT_VERSION}) + Python is a high-level, interactive, object-oriented language. Its ${SHORT_VERSION} version + includes an extensive class library with lots of goodies for + network programming, system administration, sounds and graphics. +Depends: libbz2-1.0, + libc6, + libdb5.3, + libexpat1, + libffi6, + liblzma5, + libmpdec2, + libncursesw5, + libreadline6, + libsqlite3-0, + libssl1.0.0, + libtinfo5, + mime-support, + zlib1g +Homepage: https://www.python.org diff --git a/python-interpreter-builder/Dockerfile.in b/python-interpreter-builder/Dockerfile.in index eba153e9..90e606eb 100644 --- a/python-interpreter-builder/Dockerfile.in +++ b/python-interpreter-builder/Dockerfile.in @@ -10,6 +10,7 @@ RUN apt-get update && apt-get install -yq \ debhelper \ dpkg-dev \ gcc \ + gettext-base \ libbluetooth-dev \ libbz2-dev \ libdb-dev \ @@ -43,10 +44,17 @@ ENV LANG C.UTF-8 # Add build scripts ADD scripts /scripts +ADD DEBIAN /DEBIAN # Build the Python interpreters -RUN /scripts/build-python-3.5.sh -RUN /scripts/build-python-3.6.sh +RUN mkdir -p /opt/packages && \ + echo -n "" > /opt/packages/packages.txt + +RUN /scripts/build-python-3.5.sh && \ + /scripts/package-python.sh 3.5.4 "1gcp~${TAG}" + +RUN /scripts/build-python-3.6.sh && \ + /scripts/package-python.sh 3.6.2 "1gcp~${TAG}" # Tar the interpreters. Tarring is needed because docker cp doesn't handle # links correctly. diff --git a/python-interpreter-builder/scripts/package-python.sh b/python-interpreter-builder/scripts/package-python.sh new file mode 100755 index 00000000..58cd02e6 --- /dev/null +++ b/python-interpreter-builder/scripts/package-python.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +set -euo pipefail +set -x + +function usage { + echo "Usage: $0 long_version tag +Create .deb package file for a Python interpreter with + long_version: (x.y.z) Interpreter version + tag: version suffix unique to this build +" >&2 + exit 1 +} + # Process command line +if [ -z "${1:+set}" -o -z "${2:+set}" ]; then + usage +fi +LONG_VERSION=$1 +BUILD_TAG=$2 +SHORT_VERSION=${1%.*} + +# Compute version specs +DEB_PACKAGE_NAME=gcp-python${SHORT_VERSION} +# Can't have - (hyphen) in debian revision as per +# https://www.debian.org/doc/debian-policy/ch-controlfields.html#s-f-Version +DEBIAN_REVISION=${BUILD_TAG//-/.} +DEB_PACKAGE_VERSION=${LONG_VERSION}-${DEBIAN_REVISION} + +PACKAGE_DIR=/opt/packages +# E.g. gcp-python3.6_3.6.2-1gcp~2017.07.25.110644_amd64.deb +DEB_FILENAME=${DEB_PACKAGE_NAME}_${DEB_PACKAGE_VERSION}_amd64.deb + +# Create directory for intermediate files +SCRATCH_DIR=$(mktemp --directory) +cd "${SCRATCH_DIR}" + +# Synthesize Debian control file. Note that the "Depends:" is +# currently Debian8-specific, and lacks version specifiers present in +# the standard Debian Python packages. +export DEB_PACKAGE_NAME DEB_PACKAGE_VERSION SHORT_VERSION +envsubst control \ + '${DEB_PACKAGE_NAME} ${DEB_PACKAGE_VERSION} ${SHORT_VERSION}' + +# Generate components of .deb archive +tar czf control.tar.gz control +tar czf data.tar.gz "/opt/python${SHORT_VERSION}" +echo "2.0" >debian-binary + +# Generate final .deb. +mkdir -p "${PACKAGE_DIR}" +ar rcD "${PACKAGE_DIR}/${DEB_FILENAME}" \ + debian-binary control.tar.gz data.tar.gz +rm debian-binary control.tar.gz data.tar.gz + +# Validate .deb +dpkg --install --dry-run "${PACKAGE_DIR}/${DEB_FILENAME}" + +# Add to list +echo "${DEB_FILENAME}" >> "${PACKAGE_DIR}/packages.txt" From 3aa916b1ddc1e116d62cea5f776379e1dcb7e6fb Mon Sep 17 00:00:00 2001 From: Douglas Greiman Date: Thu, 2 Nov 2017 14:20:59 -0700 Subject: [PATCH 020/132] Fix unit test failures (#165) * Update unit test script with new nox session names. * Fix typo --- RELEASING.md | 9 +++++++++ tests/google-cloud-python/run_unit_tests.sh | 8 ++++---- tests/python3-libraries/requirements.txt | 2 +- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/RELEASING.md b/RELEASING.md index f7d5a459..00b0fe16 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -67,6 +67,15 @@ following: docker run -it --entrypoint /bin/bash YOUR-IMAGE-NAME ``` +## Running tests against a released image + +To run compatibility tests against an existing image, such as +`gcr.io/google-appengine/python:latest`, run: + +```shell +DOCKER_NAMESPACE=gcr.io/google-appengine TAG=latest ./build.sh --nobuild --test +``` + ## Running benchmarks There is a benchmark suite which compares the performance of interpreters diff --git a/tests/google-cloud-python/run_unit_tests.sh b/tests/google-cloud-python/run_unit_tests.sh index 030919d8..81a6a389 100755 --- a/tests/google-cloud-python/run_unit_tests.sh +++ b/tests/google-cloud-python/run_unit_tests.sh @@ -8,10 +8,10 @@ for noxfile in */nox.py; do nox \ -f "${noxfile}" \ -e \ - "unit_tests(python_version='2.7')" \ - "unit_tests(python_version='3.4')" \ - "unit_tests(python_version='3.5')" \ - "unit_tests(python_version='3.6')" \ + "unit(py='2.7')" \ + "unit(py='3.4')" \ + "unit(py='3.5')" \ + "unit(py='3.6')" \ || exit_code=1 done diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index cca4bef2..b5bb7cae 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -59,7 +59,7 @@ google-api-python-client==1.6.4 greenlet==0.4.12 gunicorn==19.7.1 hiredis==0.2.0 -honcho=1.0.1 +honcho==1.0.1 html5lib==0.999999999 httplib2==0.10.3 idna==2.6 From 3a3b04c4a002f6a6d87f377dae8bbea475359ee5 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Fri, 3 Nov 2017 11:05:31 -0700 Subject: [PATCH 021/132] Auto-update dependencies. (#166) --- tests/integration/requirements.txt | 6 +++--- tests/python2-libraries/requirements.txt | 12 ++++++------ tests/python3-libraries/requirements.txt | 12 ++++++------ 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index eae15fe9..3e51c81d 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -1,7 +1,7 @@ Flask==0.12.2 -google-cloud-error-reporting==0.27.0 -google-cloud-logging==1.3.0 -google-cloud-monitoring==0.27.0 +google-cloud-error-reporting==0.28.0 +google-cloud-logging==1.4.0 +google-cloud-monitoring==0.28.0 gunicorn==19.7.1 requests==2.18.4 retrying==1.3.3 diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 6db2c17c..8c134e48 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -6,7 +6,7 @@ anyjson==0.3.3 apache-libcloud==2.2.1 argparse==1.4.0 astroid==1.5.3 -awscli==1.11.178 +awscli==1.11.180 babel==2.5.1 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 @@ -16,7 +16,7 @@ billiard==3.5.0.3 blessings==1.6 blinker==1.4 boto==2.48.0 -botocore==1.7.36 +botocore==1.7.38 bottle==0.12.13 carbon==1.0.2 celery==4.1.0 @@ -32,14 +32,14 @@ cov-core==1.15.0 coverage==4.4.1 coveralls==1.2.0 crcmod==1.7 -cryptography==2.1.2 +cryptography==2.1.3 cssselect==1.0.1 cython==0.27.2 decorator==4.1.2 django-celery==3.2.1 django-debug-toolbar==1.8 -django-extensions==1.9.6 -django==1.11.6 +django-extensions==1.9.7 +django==1.11.7 django_compress==1.0.1 djangorestframework==3.7.1 docker-py==1.10.6 @@ -165,7 +165,7 @@ requests==2.18.4 retrying==1.3.3 rsa==3.4.2 scipy==1.0.0 -selenium==3.6.0 +selenium==3.7.0 setuptools-git==1.2 setuptools==36.6.0 sh==1.12.14 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index b5bb7cae..2bdcc565 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -6,7 +6,7 @@ anyjson==0.3.3 apache-libcloud==2.2.1 argparse==1.4.0 astroid==1.5.3 -awscli==1.11.178 +awscli==1.11.180 babel==2.5.1 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 @@ -15,7 +15,7 @@ billiard==3.5.0.3 blessings==1.6 blinker==1.4 boto==2.48.0 -botocore==1.7.36 +botocore==1.7.38 bottle==0.12.13 celery==4.1.0 certifi==2017.7.27.1 @@ -30,14 +30,14 @@ cov-core==1.15.0 coverage==4.4.1 coveralls==1.2.0 crcmod==1.7 -cryptography==2.1.2 +cryptography==2.1.3 cssselect==1.0.1 cython==0.27.2 decorator==4.1.2 django-celery==3.2.1 django-debug-toolbar==1.8 -django-extensions==1.9.6 -django==1.11.6 +django-extensions==1.9.7 +django==1.11.7 django_compress==1.0.1 djangorestframework==3.7.1 docker-py==1.10.6 @@ -156,7 +156,7 @@ requests==2.18.4 retrying==1.3.3 rsa==3.4.2 scipy==1.0.0 -selenium==3.6.0 +selenium==3.7.0 setuptools-git==1.2 setuptools==36.6.0 sh==1.12.14 From f811fb8083a3fcd3e00c9c211af5c50d8b0e8076 Mon Sep 17 00:00:00 2001 From: Angela Li Date: Tue, 14 Nov 2017 10:35:37 -0800 Subject: [PATCH 022/132] Update metrics script to use the latest bigquery client library (#169) --- perf_dashboard/bq_utils.py | 22 ++++--------- perf_dashboard/python_clientlibs_download.py | 33 +++++++------------- 2 files changed, 18 insertions(+), 37 deletions(-) diff --git a/perf_dashboard/bq_utils.py b/perf_dashboard/bq_utils.py index d9441f07..cbad65b9 100644 --- a/perf_dashboard/bq_utils.py +++ b/perf_dashboard/bq_utils.py @@ -22,25 +22,15 @@ def insert_rows(project, dataset_name, table_name, rows): """Insert rows to bigquery table.""" client = bigquery.Client(project=project) - dataset = client.dataset(dataset_name) - table = bigquery.Table(table_name, dataset) - table.reload() - table.insert_data(rows) - + dataset_ref = client.dataset(dataset_name) + table_ref = dataset_ref.table(table_name) + table = client.get_table(table_ref) + client.create_rows(table, rows) def execute_query(query): """Execute query and return the query results.""" client = bigquery.Client() - query_job = client.run_async_query(str(uuid.uuid4()), query) - query_job.use_legacy_sql = False + query_job = client.query((query)) # Start the query job and wait it to complete - query_job.begin() - query_job.result() - - # Get the results - destination_table = query_job.destination - destination_table.reload() - results = destination_table.fetch_data() - - return results + return [row.values() for row in query_job.result()] diff --git a/perf_dashboard/python_clientlibs_download.py b/perf_dashboard/python_clientlibs_download.py index 3866fd92..cae3d2c4 100644 --- a/perf_dashboard/python_clientlibs_download.py +++ b/perf_dashboard/python_clientlibs_download.py @@ -101,27 +101,18 @@ def get_weekly_clientlibs_downloads(clientlibs_table_name, date_str): GROUP BY client_library_name """ client = bigquery.Client() - query_job = client.run_async_query( - str(uuid.uuid4()), - query, - query_parameters=( - bigquery.ArrayQueryParameter( - 'client_libs', 'STRING', - client_libs), - bigquery.ArrayQueryParameter( - 'week_dates', 'STRING', - week_dates) - )) - query_job.use_legacy_sql = False - - # Start the query job and wait it to complete - query_job.begin() - query_job.result() - - # Get the results - destination_table = query_job.destination - destination_table.reload() - results = destination_table.fetch_data() + query_parameters=[ + bigquery.ArrayQueryParameter( + 'client_libs', 'STRING', client_libs), + bigquery.ArrayQueryParameter( + 'week_dates', 'STRING', week_dates) + ] + job_config = bigquery.QueryJobConfig() + job_config.query_parameters = query_parameters + query_job = client.query(query, job_config=job_config) + + # Wait for the job to complete and get the results + results = [row.values() for row in query_job.result()] rows = [(date_time,) + row for row in results] From 4a5026620adc4916b17ce099f5ca642ede5a2bda Mon Sep 17 00:00:00 2001 From: Douglas Greiman Date: Wed, 15 Nov 2017 10:45:41 -0800 Subject: [PATCH 023/132] Revert to previous virtualenv behavior. (#168) * Revert to previous virtualenv behavior. A previous change caused the 'virtualenv' command change to switch its default Python version from 2.7 to 3.6 (when no -p flag is passed). Although not a bad change, it was accidental, and this reverts to the previous behavior. This also add tests for the behavior, and fixes various tests that had been subtly broken by a previous refactoring. The cause of this behavior change was that the 'virtualenv' package was installed under each Python interpreter, instead of the 'wheel' package. * Pin the virtualenv version. * Fix typos * Fix another typo --- cloudbuild_test.yaml | 1 + runtime-image/Dockerfile.in | 3 +- .../resources/requirements-virtualenv.txt | 1 + runtime-image/resources/requirements.txt | 2 +- tests/no-virtualenv/no-virtualenv.yaml | 34 ++++++++++++++ .../python2-libraries/python2-libraries.yaml | 2 +- tests/virtualenv/virtualenv_default.yaml | 25 +++++----- tests/virtualenv/virtualenv_python27.yaml | 47 +++++++++++++++++++ tests/virtualenv/virtualenv_python34.yaml | 34 ++++++++------ tests/virtualenv/virtualenv_python35.yaml | 34 +++++++------- tests/virtualenv/virtualenv_python36.yaml | 34 +++++++------- 11 files changed, 150 insertions(+), 67 deletions(-) create mode 100644 runtime-image/resources/requirements-virtualenv.txt create mode 100644 tests/virtualenv/virtualenv_python27.yaml diff --git a/cloudbuild_test.yaml b/cloudbuild_test.yaml index c6496dec..acf70bc7 100644 --- a/cloudbuild_test.yaml +++ b/cloudbuild_test.yaml @@ -26,6 +26,7 @@ steps: '-test.v', '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', '/workspace/tests/virtualenv/virtualenv_default.yaml', + '/workspace/tests/virtualenv/virtualenv_python27.yaml', '/workspace/tests/virtualenv/virtualenv_python34.yaml', '/workspace/tests/virtualenv/virtualenv_python35.yaml', '/workspace/tests/virtualenv/virtualenv_python36.yaml', diff --git a/runtime-image/Dockerfile.in b/runtime-image/Dockerfile.in index 60a6fa6f..7fafb2c4 100644 --- a/runtime-image/Dockerfile.in +++ b/runtime-image/Dockerfile.in @@ -27,7 +27,8 @@ ENV PATH /opt/python3.6/bin:/opt/python3.5/bin:$PATH RUN /usr/bin/pip install --upgrade -r /resources/requirements.txt && \ /usr/bin/pip3 install --upgrade -r /resources/requirements.txt && \ /opt/python3.5/bin/pip3.5 install --upgrade -r /resources/requirements.txt && \ - /opt/python3.6/bin/pip3.6 install --upgrade -r /resources/requirements.txt + /opt/python3.6/bin/pip3.6 install --upgrade -r /resources/requirements.txt && \ + /usr/bin/pip install --upgrade -r /resources/requirements-virtualenv.txt # Setup the app working directory RUN ln -s /home/vmagent/app /app diff --git a/runtime-image/resources/requirements-virtualenv.txt b/runtime-image/resources/requirements-virtualenv.txt new file mode 100644 index 00000000..a4ce32e5 --- /dev/null +++ b/runtime-image/resources/requirements-virtualenv.txt @@ -0,0 +1 @@ +virtualenv==15.1.0 diff --git a/runtime-image/resources/requirements.txt b/runtime-image/resources/requirements.txt index 40155447..c9f190ec 100644 --- a/runtime-image/resources/requirements.txt +++ b/runtime-image/resources/requirements.txt @@ -1,3 +1,3 @@ pip==9.0.1 setuptools==36.6.0 -virtualenv==15.1.0 +wheel==0.30.0 diff --git a/tests/no-virtualenv/no-virtualenv.yaml b/tests/no-virtualenv/no-virtualenv.yaml index de932617..f43d08a5 100644 --- a/tests/no-virtualenv/no-virtualenv.yaml +++ b/tests/no-virtualenv/no-virtualenv.yaml @@ -4,7 +4,41 @@ commandTests: command: ["which", "python"] expectedOutput: ["/usr/bin/python\n"] + - name: "default pip installation" + command: ["which", "pip"] + expectedOutput: ["/usr/local/bin/pip\n"] + + - name: "default pip python version" + command: ["pip", "-V"] + expectedOutput: ["pip .* from .*python 2[.]7"] + + - name: "default virtualenv installation" + command: ["which", "virtualenv"] + expectedOutput: ["/usr/local/bin/virtualenv\n"] + + - name: "default python2.7 installation" + command: ["which", "python2.7"] + expectedOutput: ["/usr/bin/python2.7\n"] + + - name: "default python3.4 installation" + command: ["which", "python3.4"] + expectedOutput: ["/usr/bin/python3.4\n"] + + - name: "default python3.5 installation" + command: ["which", "python3.5"] + expectedOutput: ["/opt/python3.5/bin/python3.5\n"] + + - name: "default python3.6 installation" + command: ["which", "python3.6"] + expectedOutput: ["/opt/python3.6/bin/python3.6\n"] + - name: "default gunicorn installation" setup: [["pip", "install", "gunicorn"]] command: ["which", "gunicorn"] expectedOutput: ["/usr/local/bin/gunicorn\n"] + + - name: "default flask installation" + # Checks that 'pip' and 'python' are using the same Python version + setup: [["pip", "install", "flask"]] + command: ["python", "-c", "import flask; print(flask.__file__)"] + expectedOutput: ["/usr/local/lib/python2.7/dist-packages/flask"] diff --git a/tests/python2-libraries/python2-libraries.yaml b/tests/python2-libraries/python2-libraries.yaml index 3a2771ac..f9fef5a0 100644 --- a/tests/python2-libraries/python2-libraries.yaml +++ b/tests/python2-libraries/python2-libraries.yaml @@ -8,6 +8,6 @@ globalEnvVars: commandTests: - name: "requirements" - setup: [["virtualenv", "/env"]] + setup: [["virtualenv", "-p", "python", "/env"]] command: ["pip", "install", "-r", "/requirements.txt"] exitCode: 0 diff --git a/tests/virtualenv/virtualenv_default.yaml b/tests/virtualenv/virtualenv_default.yaml index 8695e319..60714de6 100644 --- a/tests/virtualenv/virtualenv_default.yaml +++ b/tests/virtualenv/virtualenv_default.yaml @@ -7,20 +7,13 @@ globalEnvVars: value: "/env/bin:$PATH" commandTests: - - name: "python installation" - command: ["which", "python"] - expectedOutput: ["/usr/bin/python\n"] - - - name: "pip installation" - command: ["which", "pip"] - expectedOutput: ["/usr/local/bin/pip\n"] - - - name: "virtualenv installation" + - name: "virtualenv python installation" setup: [["virtualenv", "/env"]] command: ["which", "python"] expectedOutput: ["/env/bin/python\n"] - - name: "python version" + - name: "virtualenv python version" + setup: [["virtualenv", "/env"]] command: ["python", "--version"] # we check stderr instead of stdout for Python versions < 3.4 # https://bugs.python.org/issue18338 @@ -31,10 +24,14 @@ commandTests: command: ["which", "pip"] expectedOutput: ["/env/bin/pip\n"] - - name: "virtualenv gunicorn flask" - setup: [["virtualenv", "/env"], ["pip", "install", "gunicorn", "flask"]] + - name: "virtualenv gunicorn installation" + setup: [["virtualenv", "/env"], + ["pip", "install", "gunicorn"]] command: ["which", "gunicorn"] expectedOutput: ["/env/bin/gunicorn"] - - name: "flask integration" - command: ["python", "-c", "\"import sys; import flask; sys.exit(0 if flask.__file__.startswith('/env') else 1)\""] + - name: "virtualenv flask installation" + setup: [["virtualenv", "/env"], + ["pip", "install", "flask"]] + command: ["python", "-c", "import flask; print(flask.__file__)"] + expectedOutput: ["/env/local/lib/python2.7/site-packages/flask"] diff --git a/tests/virtualenv/virtualenv_python27.yaml b/tests/virtualenv/virtualenv_python27.yaml new file mode 100644 index 00000000..d28ea541 --- /dev/null +++ b/tests/virtualenv/virtualenv_python27.yaml @@ -0,0 +1,47 @@ +schemaVersion: "1.0.0" + +globalEnvVars: + - key: "VIRTUAL_ENV" + value: "/env" + - key: "PATH" + value: "/env/bin:$PATH" + +commandTests: + - name: "virtualenv27 python installation" + setup: [["virtualenv", "-p", "python", "/env"]] + command: ["which", "python"] + expectedOutput: ["/env/bin/python\n"] + + - name: "virtualenv27 python2 installation" + setup: [["virtualenv", "-p", "python", "/env"]] + command: ["which", "python2"] + expectedOutput: ["/env/bin/python2\n"] + + - name: "virtualenv27python2.7 installation" + setup: [["virtualenv", "-p", "python", "/env"]] + command: ["which", "python2.7"] + expectedOutput: ["/env/bin/python2.7\n"] + + - name: "virtualenv27 python version" + setup: [["virtualenv", "-p", "python", "/env"]] + command: ["python", "--version"] + # we check stderr instead of stdout for Python versions < 3.4 + # https://bugs.python.org/issue18338 + expectedError: ["Python 2.7.9\n"] + + - name: "virtualenv27 pip installation" + setup: [["virtualenv", "-p", "python", "/env"]] + command: ["which", "pip"] + expectedOutput: ["/env/bin/pip\n"] + + - name: "virtualenv27 gunicorn installation" + setup: [["virtualenv", "-p", "python", "/env"], + ["pip", "install", "gunicorn"]] + command: ["which", "gunicorn"] + expectedOutput: ["/env/bin/gunicorn"] + + - name: "virtualenv27 flask installation" + setup: [["virtualenv", "-p", "python", "/env"], + ["pip", "install", "flask"]] + command: ["python", "-c", "import flask; print(flask.__file__)"] + expectedOutput: ["/env/local/lib/python2.7/site-packages/flask"] diff --git a/tests/virtualenv/virtualenv_python34.yaml b/tests/virtualenv/virtualenv_python34.yaml index e35e9693..779f4d45 100644 --- a/tests/virtualenv/virtualenv_python34.yaml +++ b/tests/virtualenv/virtualenv_python34.yaml @@ -7,42 +7,48 @@ globalEnvVars: value: "/env/bin:$PATH" commandTests: - - name: "python installation" - command: ["which", "python3.4"] - expectedOutput: ["/usr/bin/python3.4\n"] - - - name: "virtualenv python installation" + - name: "virtualenv34 python installation" setup: [["virtualenv", "-p", "python3.4", "/env"]] command: ["which", "python"] expectedOutput: ["/env/bin/python\n"] - - name: "virtualenv python3 installation" + - name: "virtualenv34 python3 installation" setup: [["virtualenv", "-p", "python3.4", "/env"]] command: ["which", "python3"] expectedOutput: ["/env/bin/python3\n"] - - name: "python version" + - name: "virtualenv34 python3.4 installation" + setup: [["virtualenv", "-p", "python3.4", "/env"]] + command: ["which", "python3.4"] + expectedOutput: ["/env/bin/python3.4\n"] + + - name: "virtualenv34 python version" setup: [["virtualenv", "-p", "python3.4", "/env"]] command: ["python", "--version"] expectedOutput: ["Python 3.4.2\n"] - - name: "pip installation" + - name: "virtualenv34 pip installation" setup: [["virtualenv", "-p", "python3.4", "/env"]] command: ["which", "pip"] expectedOutput: ["/env/bin/pip\n"] - - name: "pip3 installation" + - name: "virtualenv34 pip3 installation" setup: [["virtualenv", "-p", "python3.4", "/env"]] command: ["which", "pip3"] expectedOutput: ["/env/bin/pip3\n"] - - name: "gunicorn flask" - setup: [["virtualenv", "-p", "python3.4", "/env"], ["pip", "install", "gunicorn", "flask"]] + - name: "virtualenv34 gunicorn installation" + setup: [["virtualenv", "-p", "python3.4", "/env"], + ["pip", "install", "gunicorn"]] command: ["which", "gunicorn"] expectedOutput: ["/env/bin/gunicorn"] - - name: "flask integration" - command: ["python", "-c", "\"import sys; import flask; sys.exit(0 if flask.__file__.startswith('/env') else 1)\""] + - name: "virtualenv34 flask installation" + setup: [["virtualenv", "-p", "python3.4", "/env"], + ["pip", "install", "flask"]] + command: ["python", "-c", "import flask; print(flask.__file__)"] + expectedOutput: ["/env/lib/python3.4/site-packages/flask"] - - name: "test.support" + - name: "virtualenv34 test.support availability" + setup: [["virtualenv", "-p", "python3.4", "/env"]] command: ["python", "-c", "\"from test import pystone, regrtest, support\""] diff --git a/tests/virtualenv/virtualenv_python35.yaml b/tests/virtualenv/virtualenv_python35.yaml index 4b019967..4150ba19 100644 --- a/tests/virtualenv/virtualenv_python35.yaml +++ b/tests/virtualenv/virtualenv_python35.yaml @@ -7,50 +7,48 @@ globalEnvVars: value: "/env/bin:$PATH" commandTests: - - name: "virtual env teardown" - command: ["rm", "-rf", "/env"] - - - name: "python installation" - command: ["which", "python3.5"] - expectedOutput: ["/opt/python3.5/bin/python3.5\n"] - - - name: "virtualenv python installation" + - name: "virtualenv35 python installation" setup: [["virtualenv", "-p", "python3.5", "/env"]] command: ["which", "python"] expectedOutput: ["/env/bin/python\n"] - - name: "virtualenv python3 installation" + - name: "virtualenv35 python3 installation" setup: [["virtualenv", "-p", "python3.5", "/env"]] command: ["which", "python3"] expectedOutput: ["/env/bin/python3\n"] - - name: "virtualenv python3.5 installation" + - name: "virtualenv35 python3.5 installation" setup: [["virtualenv", "-p", "python3.5", "/env"]] command: ["which", "python3.5"] expectedOutput: ["/env/bin/python3.5\n"] - - name: "python version" + - name: "virtualenv35 python version" setup: [["virtualenv", "-p", "python3.5", "/env"]] command: ["python", "--version"] expectedOutput: ["Python 3.5.4\n"] - - name: "pip installation" + - name: "virtualenv35 pip installation" setup: [["virtualenv", "-p", "python3.5", "/env"]] command: ["which", "pip"] expectedOutput: ["/env/bin/pip\n"] - - name: "pip3 installation" + - name: "virtualenv35 pip3 installation" setup: [["virtualenv", "-p", "python3.5", "/env"]] command: ["which", "pip3"] expectedOutput: ["/env/bin/pip3\n"] - - name: "gunicorn flask" - setup: [["virtualenv", "-p", "python3.5", "/env"], ["pip", "install", "gunicorn", "flask"]] + - name: "virtualenv35 gunicorn installation" + setup: [["virtualenv", "-p", "python3.5", "/env"], + ["pip", "install", "gunicorn"]] command: ["which", "gunicorn"] expectedOutput: ["/env/bin/gunicorn"] - - name: "flask integration" - command: ["python", "-c", "\"import sys; import flask; sys.exit(0 if flask.__file__.startswith('/env') else 1)\""] + - name: "virtualenv35 flask installation" + setup: [["virtualenv", "-p", "python3.5", "/env"], + ["pip", "install", "flask"]] + command: ["python", "-c", "import flask; print(flask.__file__)"] + expectedOutput: ["/env/lib/python3.5/site-packages/flask"] - - name: "test.support" + - name: "virtualenv35 test.support availability" + setup: [["virtualenv", "-p", "python3.5", "/env"]] command: ["python", "-c", "\"from test import pystone, regrtest, support\""] diff --git a/tests/virtualenv/virtualenv_python36.yaml b/tests/virtualenv/virtualenv_python36.yaml index deea9b5d..71c91a9f 100644 --- a/tests/virtualenv/virtualenv_python36.yaml +++ b/tests/virtualenv/virtualenv_python36.yaml @@ -7,50 +7,48 @@ globalEnvVars: value: "/env/bin:$PATH" commandTests: - - name: "virtual env teardown" - command: ["rm", "-rf", "/env"] - - - name: "python installation" - command: ["which", "python3.6"] - expectedOutput: ["/opt/python3.6/bin/python3.6\n"] - - - name: "virtualenv python installation" + - name: "virtualenv36 python installation" setup: [["virtualenv", "-p", "python3.6", "/env"]] command: ["which", "python"] expectedOutput: ["/env/bin/python\n"] - - name: "virtualenv python3 installation" + - name: "virtualenv36 python3 installation" setup: [["virtualenv", "-p", "python3.6", "/env"]] command: ["which", "python3"] expectedOutput: ["/env/bin/python3\n"] - - name: "virtualenv python3.6 installation" + - name: "virtualenv36 python3.6 installation" setup: [["virtualenv", "-p", "python3.6", "/env"]] command: ["which", "python3.6"] expectedOutput: ["/env/bin/python3.6\n"] - - name: "python version" + - name: "virtualenv36 python version" setup: [["virtualenv", "-p", "python3.6", "/env"]] command: ["python", "--version"] expectedOutput: ["Python 3.6.2\n"] - - name: "pip installation" + - name: "virtualenv36 pip installation" setup: [["virtualenv", "-p", "python3.6", "/env"]] command: ["which", "pip"] expectedOutput: ["/env/bin/pip\n"] - - name: "pip3 installation" + - name: "virtualenv36 pip3 installation" setup: [["virtualenv", "-p", "python3.6", "/env"]] command: ["which", "pip3"] expectedOutput: ["/env/bin/pip3\n"] - - name: "gunicorn flask" - setup: [["virtualenv", "-p", "python3.6", "/env"], ["pip", "install", "gunicorn", "flask"]] + - name: "virtualenv36 gunicorn installation" + setup: [["virtualenv", "-p", "python3.6", "/env"], + ["pip", "install", "gunicorn"]] command: ["which", "gunicorn"] expectedOutput: ["/env/bin/gunicorn"] - - name: "flask integration" - command: ["python", "-c", "\"import sys; import flask; sys.exit(0 if flask.__file__.startswith('/env') else 1)\""] + - name: "virtualenv36 flask installation" + setup: [["virtualenv", "-p", "python3.6", "/env"], + ["pip", "install", "flask"]] + command: ["python", "-c", "import flask; print(flask.__file__)"] + expectedOutput: ["/env/lib/python3.6/site-packages/flask"] - - name: "test.support" + - name: "virtualenv36 test.support availability" + setup: [["virtualenv", "-p", "python3.6", "/env"]] command: ["python", "-c", "\"from test import pystone, regrtest, support\""] From 714dcb7f83aba9e1c80b9a739294a4864109a607 Mon Sep 17 00:00:00 2001 From: Nick Kubala Date: Wed, 15 Nov 2017 14:18:04 -0800 Subject: [PATCH 024/132] return string instead of bool in integration test sample app env handler --- tests/integration/server.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/integration/server.py b/tests/integration/server.py index 8e2958cd..0b4382c6 100755 --- a/tests/integration/server.py +++ b/tests/integration/server.py @@ -245,8 +245,11 @@ def _check_environment(): # for GAE, we'll check the existence env vars set on # vm:true or env:flex # if neither exist, assume we're in GKE - return (_APPENGINE_FLEXIBLE_ENV_VM in os.environ or - _APPENGINE_FLEXIBLE_ENV_FLEX in os.environ), 200 + environment = "GKE" + if (_APPENGINE_FLEXIBLE_ENV_VM in os.environ or + _APPENGINE_FLEXIBLE_ENV_FLEX in os.environ): + environment = "GAE" + return environment, 200 class ErrorResponse(Exception): From 7b8bf15059075f06ce2c5c6e61d1445ba3c1094e Mon Sep 17 00:00:00 2001 From: Angela Li Date: Mon, 4 Dec 2017 13:34:58 -0800 Subject: [PATCH 025/132] Update system tests script (#171) --- tests/google-cloud-python-system/run_system_tests.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/google-cloud-python-system/run_system_tests.sh b/tests/google-cloud-python-system/run_system_tests.sh index 17d3aac3..f9f81a47 100755 --- a/tests/google-cloud-python-system/run_system_tests.sh +++ b/tests/google-cloud-python-system/run_system_tests.sh @@ -31,8 +31,8 @@ for package in ${packages}; do nox \ -f "${noxfile}" \ -e \ - "system_tests(python_version='2.7')" \ - "system_tests(python_version='3.6')" \ + "system(py='2.7')" \ + "system(py='3.6')" \ || exit_code=1 done From 597a87474472699e2611f0d170becd69c2b244b2 Mon Sep 17 00:00:00 2001 From: Angela Li Date: Wed, 3 Jan 2018 18:07:37 -0800 Subject: [PATCH 026/132] Skip test_xmlrpc_net when building python interpreters (#175) --- python-interpreter-builder/scripts/build-python-3.5.sh | 3 ++- python-interpreter-builder/scripts/build-python-3.6.sh | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/python-interpreter-builder/scripts/build-python-3.5.sh b/python-interpreter-builder/scripts/build-python-3.5.sh index f9113879..eeea1f76 100755 --- a/python-interpreter-builder/scripts/build-python-3.5.sh +++ b/python-interpreter-builder/scripts/build-python-3.5.sh @@ -123,7 +123,8 @@ make profile-opt # test_dbm: https://bugs.python.org/issue28700 # test_imap: https://bugs.python.org/issue30175 # test_shutil: https://bugs.python.org/issue29317 -make test TESTOPTS="--exclude test___all__ test_dbm test_imaplib test_shutil" +# test_xmlrpc_net: https://bugs.python.org/issue31724 +make test TESTOPTS="--exclude test___all__ test_dbm test_imaplib test_shutil test_xmlrpc_net" # Install make altinstall diff --git a/python-interpreter-builder/scripts/build-python-3.6.sh b/python-interpreter-builder/scripts/build-python-3.6.sh index 431c305d..ca089d59 100755 --- a/python-interpreter-builder/scripts/build-python-3.6.sh +++ b/python-interpreter-builder/scripts/build-python-3.6.sh @@ -123,7 +123,8 @@ make profile-opt # test_dbm: https://bugs.python.org/issue28700 # test_imap: https://bugs.python.org/issue30175 # test_shutil: https://bugs.python.org/issue29317 -make test TESTOPTS="--exclude test___all__ test_dbm test_imaplib test_shutil" +# test_xmlrpc_net: https://bugs.python.org/issue31724 +make test TESTOPTS="--exclude test___all__ test_dbm test_imaplib test_shutil test_xmlrpc_net" # Install make altinstall From 01a4da0bc4a429b24550912f2f2ba1e732b4ac36 Mon Sep 17 00:00:00 2001 From: Jon Donovan Date: Fri, 12 Jan 2018 12:43:31 -0800 Subject: [PATCH 027/132] Python compat (#174) * Add support for python-compat * Update app.yaml and tests * Add missing .dockerignore file * Updating per review comments * Adding missing files * Update gen_dockerfile.py * Update gen_dockerfile_test.py --- scripts/data/Dockerfile.python_compat | 3 ++ scripts/data/dockerignore.python_compat | 5 +++ scripts/gen_dockerfile.py | 43 +++++++++++++------ scripts/gen_dockerfile_test.py | 32 +++++++++++--- scripts/testdata/hello_world_compat/app.yaml | 13 ++++++ scripts/testdata/hello_world_compat/main.py | 14 ++++++ .../hello_world_compat_golden/.dockerignore | 5 +++ .../hello_world_compat_golden/Dockerfile | 3 ++ 8 files changed, 97 insertions(+), 21 deletions(-) create mode 100644 scripts/data/Dockerfile.python_compat create mode 100644 scripts/data/dockerignore.python_compat create mode 100644 scripts/testdata/hello_world_compat/app.yaml create mode 100644 scripts/testdata/hello_world_compat/main.py create mode 100644 scripts/testdata/hello_world_compat_golden/.dockerignore create mode 100644 scripts/testdata/hello_world_compat_golden/Dockerfile diff --git a/scripts/data/Dockerfile.python_compat b/scripts/data/Dockerfile.python_compat new file mode 100644 index 00000000..1e4d6352 --- /dev/null +++ b/scripts/data/Dockerfile.python_compat @@ -0,0 +1,3 @@ +FROM gcr.io/google_appengine/python-compat-multicore +ADD . /app/ +RUN if [ -s requirements.txt ]; then pip install -r requirements.txt; fi diff --git a/scripts/data/dockerignore.python_compat b/scripts/data/dockerignore.python_compat new file mode 100644 index 00000000..5ce5abfa --- /dev/null +++ b/scripts/data/dockerignore.python_compat @@ -0,0 +1,5 @@ +.dockerignore +Dockerfile +.git +.hg +.svn diff --git a/scripts/gen_dockerfile.py b/scripts/gen_dockerfile.py index eaff183f..5a4ceae2 100755 --- a/scripts/gen_dockerfile.py +++ b/scripts/gen_dockerfile.py @@ -29,7 +29,6 @@ import validation_utils - # Validate characters for dockerfile image names. # # This roots out obvious mistakes, the full gory details are here: @@ -63,7 +62,7 @@ # Validated application configuration AppConfig = collections.namedtuple( 'AppConfig', - 'base_image dockerfile_python_version entrypoint has_requirements_txt' + 'base_image dockerfile_python_version entrypoint has_requirements_txt is_python_compat' ) @@ -97,6 +96,16 @@ def get_app_config(raw_config, base_image, config_file, source_dir): 'Expected {} contents to be a Mapping type, but found type "{}"'. format(config_file, type(raw_config))) + # Short circuit for python compat. + if validation_utils.get_field_value( + raw_config, 'runtime', str) == 'python-compat': + return AppConfig( + base_image=None, + dockerfile_python_version=None, + entrypoint=None, + has_requirements_txt=None, + is_python_compat=True) + entrypoint = validation_utils.get_field_value( raw_config, 'entrypoint', str) if not PRINTABLE_REGEX.match(entrypoint): @@ -133,7 +142,8 @@ def get_app_config(raw_config, base_image, config_file, source_dir): base_image=base_image, dockerfile_python_version=dockerfile_python_version, entrypoint=entrypoint, - has_requirements_txt=has_requirements_txt) + has_requirements_txt=has_requirements_txt, + is_python_compat=False) def get_data(name): @@ -175,19 +185,24 @@ def generate_files(app_config): else: optional_entrypoint = '' - dockerfile = ''.join([ - get_data('Dockerfile.preamble.template').format( - base_image=app_config.base_image), - get_data('Dockerfile.virtualenv.template').format( - python_version=app_config.dockerfile_python_version), - optional_requirements_txt, - get_data('Dockerfile.install_app'), - optional_entrypoint, - ]) + if app_config.is_python_compat: + dockerfile = get_data('Dockerfile.python_compat') + dockerignore = get_data('dockerignore.python_compat') + else: + dockerfile = ''.join([ + get_data('Dockerfile.preamble.template').format( + base_image=app_config.base_image), + get_data('Dockerfile.virtualenv.template').format( + python_version=app_config.dockerfile_python_version), + optional_requirements_txt, + get_data('Dockerfile.install_app'), + optional_entrypoint, + ]) + dockerignore = get_data('dockerignore') return { 'Dockerfile': dockerfile, - '.dockerignore': get_data('dockerignore'), + '.dockerignore': dockerignore, } @@ -206,7 +221,7 @@ def generate_dockerfile_command(base_image, config_file, source_dir): # Determine complete configuration app_config = get_app_config(raw_config, base_image, config_file, - source_dir) + source_dir) # Generate list of filenames and their textual contents files = generate_files(app_config) diff --git a/scripts/gen_dockerfile_test.py b/scripts/gen_dockerfile_test.py index b52e15e8..8d1d0798 100755 --- a/scripts/gen_dockerfile_test.py +++ b/scripts/gen_dockerfile_test.py @@ -54,6 +54,14 @@ def compare_file(filename, dir1, dir2): 'dockerfile_python_version': '', 'has_requirements_txt': False, 'entrypoint': '', + 'is_python_compat': False, + }), + ('env: flex\nruntime: python-compat', { + 'base_image': None, + 'dockerfile_python_version': None, + 'has_requirements_txt': None, + 'entrypoint': None, + 'is_python_compat': True, }), # All supported python versions ('runtime_config:\n python_version:', { @@ -125,7 +133,8 @@ def test_get_app_config_invalid(app_yaml): base_image='', dockerfile_python_version='', entrypoint='', - has_requirements_txt=False + has_requirements_txt=False, + is_python_compat=False, ) @@ -146,6 +155,9 @@ def test_get_app_config_invalid(app_yaml): # Python version (_BASE_APP_CONFIG._replace(dockerfile_python_version='_my_version'), True, 'python_version=python_my_version'), + # python-compat runtime + (_BASE_APP_CONFIG._replace(is_python_compat=True), True, + 'FROM gcr.io/google_appengine/python-compat-multicore'), ]) def test_generate_files(app_config, should_find, test_string): result = gen_dockerfile.generate_files(app_config) @@ -163,10 +175,13 @@ def compare_against_golden_files(app, config_dir, testdata_dir): compare_file(filename, config_dir, golden_dir) -def test_generate_dockerfile_command(tmpdir, testdata_dir): +@pytest.mark.parametrize('app', [ + # Sampled from https://github.com/GoogleCloudPlatform/python-docs-samples + 'hello_world', + # From an internal source. + 'hello_world_compat']) +def test_generate_dockerfile_command(tmpdir, testdata_dir, app): """Generates output and compares against a set of golden files.""" - # Sample from https://github.com/GoogleCloudPlatform/python-docs-samples - app = 'hello_world' app_dir = os.path.join(testdata_dir, app) # Copy sample app to writable temp dir, and generate Dockerfile. @@ -179,12 +194,15 @@ def test_generate_dockerfile_command(tmpdir, testdata_dir): compare_against_golden_files(app, config_dir, testdata_dir) +@pytest.mark.parametrize('app', [ + # Sampled from https://github.com/GoogleCloudPlatform/python-docs-samples + 'hello_world', + # From an internal source. + 'hello_world_compat']) @pytest.mark.xfail(not shutil.which('gcloud'), reason='Google Cloud SDK is not installed') -def test_generate_dockerfile_golden(tmpdir, testdata_dir): +def test_generate_dockerfile_golden(tmpdir, testdata_dir, app): """Validate our golden files against gcloud app gen-config""" - # Sample from https://github.com/GoogleCloudPlatform/python-docs-samples - app = 'hello_world' app_dir = os.path.join(testdata_dir, app) # Copy sample app to writable temp dir, and generate Dockerfile. diff --git a/scripts/testdata/hello_world_compat/app.yaml b/scripts/testdata/hello_world_compat/app.yaml new file mode 100644 index 00000000..e514d42c --- /dev/null +++ b/scripts/testdata/hello_world_compat/app.yaml @@ -0,0 +1,13 @@ +service: default +runtime: python-compat +env: flex + +api_version: 1 +threadsafe: true + +beta_settings: + enable_app_engine_apis: true # Needed for compat apps. + +handlers: +- url: .* + script: main.app diff --git a/scripts/testdata/hello_world_compat/main.py b/scripts/testdata/hello_world_compat/main.py new file mode 100644 index 00000000..40302722 --- /dev/null +++ b/scripts/testdata/hello_world_compat/main.py @@ -0,0 +1,14 @@ +"""The hello world flex app!""" + +import webapp2 + + +class HelloHandler(webapp2.RequestHandler): + + def get(self): + msg = 'Hello GAE Flex (env: flex) Compat-Runtime App\n' + self.response.headers['Content-Type'] = 'text/plain' + self.response.out.write(msg) + +app = webapp2.WSGIApplication([('/', HelloHandler)], + debug=True) diff --git a/scripts/testdata/hello_world_compat_golden/.dockerignore b/scripts/testdata/hello_world_compat_golden/.dockerignore new file mode 100644 index 00000000..5ce5abfa --- /dev/null +++ b/scripts/testdata/hello_world_compat_golden/.dockerignore @@ -0,0 +1,5 @@ +.dockerignore +Dockerfile +.git +.hg +.svn diff --git a/scripts/testdata/hello_world_compat_golden/Dockerfile b/scripts/testdata/hello_world_compat_golden/Dockerfile new file mode 100644 index 00000000..1e4d6352 --- /dev/null +++ b/scripts/testdata/hello_world_compat_golden/Dockerfile @@ -0,0 +1,3 @@ +FROM gcr.io/google_appengine/python-compat-multicore +ADD . /app/ +RUN if [ -s requirements.txt ]; then pip install -r requirements.txt; fi From dfd43223fd8b40b8e90fd3b6cd8e088ca8f04e86 Mon Sep 17 00:00:00 2001 From: Julius Adorf Date: Wed, 24 Jan 2018 17:14:01 +0000 Subject: [PATCH 028/132] Replace "Container Engine" by "Kubernetes Engine". (#176) --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 78a02c5c..2cc7d173 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ This repository contains the source for the [docker](https://docker.io) base image. This image can be used as the base image for running applications on [Google App Engine Flexible](https://cloud.google.com/appengine/docs/flexible/), -[Google Container Engine](https://cloud.google.com/container-engine), or any +[Google Kubernetes Engine](https://cloud.google.com/kubernetes-engine), or any other Docker host. This image is based on Debian Jessie and contains packages required to build @@ -38,7 +38,7 @@ to create a custom runtime: You can then modify the `Dockerfile` and `.dockerignore` as needed for you application. -## Container Engine & other Docker hosts. +## Kubernetes Engine & other Docker hosts. For other docker hosts, you'll need to create a `Dockerfile` based on this image that copies your application code, installs dependencies, and declares an From 389436f9d0d62bf1144ca7cb96f55388c5603f70 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Wed, 7 Feb 2018 17:59:17 -0800 Subject: [PATCH 029/132] Auto-update dependencies. (#167) * Auto-update dependencies. * Pin some library versions to deal with Python 2 and 3 incompatibilties. --- runtime-image/resources/requirements.txt | 2 +- scripts/requirements-test.txt | 2 +- tests/eventlet/requirements.txt | 8 +- tests/integration/requirements.txt | 2 +- tests/python2-libraries/requirements.txt | 158 +++++++++++------------ tests/python3-libraries/requirements.txt | 147 +++++++++++---------- 6 files changed, 159 insertions(+), 160 deletions(-) diff --git a/runtime-image/resources/requirements.txt b/runtime-image/resources/requirements.txt index c9f190ec..6a1b2cc3 100644 --- a/runtime-image/resources/requirements.txt +++ b/runtime-image/resources/requirements.txt @@ -1,3 +1,3 @@ pip==9.0.1 -setuptools==36.6.0 +setuptools==38.5.0 wheel==0.30.0 diff --git a/scripts/requirements-test.txt b/scripts/requirements-test.txt index 5afb5315..12a6ca4a 100644 --- a/scripts/requirements-test.txt +++ b/scripts/requirements-test.txt @@ -1,4 +1,4 @@ flask==0.12.2 -pytest==3.2.3 +pytest==3.4.0 pytest-cov==2.5.1 pyyaml==3.12 diff --git a/tests/eventlet/requirements.txt b/tests/eventlet/requirements.txt index c33e53f2..a8f7b0a9 100644 --- a/tests/eventlet/requirements.txt +++ b/tests/eventlet/requirements.txt @@ -1,10 +1,10 @@ click==6.7 enum-compat==0.0.2 -eventlet==0.21.0 +eventlet==0.22.0 Flask==0.12.2 -greenlet==0.4.12 +greenlet==0.4.13 gunicorn==19.7.1 itsdangerous==0.24 -Jinja2==2.9.6 +Jinja2==2.10 MarkupSafe==1.0 -Werkzeug==0.12.2 +Werkzeug==0.14.1 diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index 3e51c81d..99a06d3b 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -1,5 +1,5 @@ Flask==0.12.2 -google-cloud-error-reporting==0.28.0 +google-cloud-error-reporting==0.29.0 google-cloud-logging==1.4.0 google-cloud-monitoring==0.28.0 gunicorn==19.7.1 diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 8c134e48..f512ca18 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -1,54 +1,54 @@ -alembic==0.9.6 +alembic==0.9.7 amqp==2.2.2 amqplib==1.0.2 -ansible==2.4.1.0 +ansible==2.4.3.0 anyjson==0.3.3 apache-libcloud==2.2.1 argparse==1.4.0 -astroid==1.5.3 -awscli==1.11.180 -babel==2.5.1 +astroid==1.6.1 +awscli==1.14.32 +babel==2.5.3 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 beautifulsoup4==4.6.0 beautifulsoup==3.2.1 billiard==3.5.0.3 -blessings==1.6 +blessings==1.6.1 blinker==1.4 boto==2.48.0 -botocore==1.7.38 +botocore==1.8.36 bottle==0.12.13 -carbon==1.0.2 +carbon<1.1.1 celery==4.1.0 -certifi==2017.7.27.1 -cffi==1.11.2 +certifi==2018.1.18 +cffi==1.11.4 chardet==3.0.4 click==6.7 -cliff==2.9.1 -cmd2==0.7.7 +cliff==2.11.0 +cmd2==0.8.0 colorama==0.3.9 configobj==5.0.6 cov-core==1.15.0 -coverage==4.4.1 +coverage==4.5 coveralls==1.2.0 crcmod==1.7 -cryptography==2.1.3 -cssselect==1.0.1 -cython==0.27.2 -decorator==4.1.2 -django-celery==3.2.1 -django-debug-toolbar==1.8 -django-extensions==1.9.7 -django==1.11.7 +cryptography==2.1.4 +cssselect==1.0.3 +cython==0.27.3 +decorator==4.2.1 +django-celery==3.2.2 +django-debug-toolbar==1.9.1 +django-extensions==1.9.9 +django<2.0 django_compress==1.0.1 -djangorestframework==3.7.1 +djangorestframework==3.7.7 docker-py==1.10.6 docopt==0.6.2 docutils==0.14 ecdsa==0.13 -elasticsearch==5.4.0 +elasticsearch==6.1.1 enum34==1.1.6 -eventlet==0.21.0 +eventlet==0.22.0 extras==1.0.0 fabric==1.14.0 fixtures==3.0.0 @@ -56,108 +56,108 @@ flake8==3.5.0 flask==0.12.2 funcsigs==1.0.2 functools32==3.2.3.post2 -futures==3.1.1 +futures==3.2.0 gevent==1.2.2 -google-api-python-client==1.6.4 -graphite-web==1.0.2 -greenlet==0.4.12 +google-api-python-client==1.6.5 +graphite-web==1.1.1 +greenlet==0.4.13 gunicorn==19.7.1 hiredis==0.2.0 honcho==1.0.1 -html5lib==0.999999999 +html5lib==1.0.1 httplib2==0.10.3 idna==2.6 -ipaddress==1.0.18 +ipaddress==1.0.19 iso8601==0.1.12 isodate==0.6.0 itsdangerous==0.24 -jinja2==2.9.6 +jinja2==2.10 jmespath==0.9.3 jsonschema==2.6.0 kombu==4.1.0 linecache2==1.0.0 logilab-common==1.4.1 -lxml==4.1.0 +lxml==4.1.1 m2crypto==0.27.0 mako==1.0.7 manifestparser==1.1 -markdown==2.6.9 +markdown==2.6.11 markupsafe==1.0 -matplotlib==2.1.0 +matplotlib==2.1.2 mccabe==0.6.1 meld3==1.0.2 mock==2.0.0 mozcrash==1.0 -mozdevice==0.50 +mozdevice==0.51 mozfile==1.2 mozinfo==0.10 -mozlog==3.5 +mozlog==3.7 moznetwork==0.27 -mozprocess==0.25 -mozprofile==0.28 -mozrunner==6.13 -msgpack-python==0.4.8 +mozprocess==0.26 +mozprofile==0.29 +mozrunner==6.14 +msgpack-python==0.5.2 mysql-python==1.2.5 -ndg-httpsclient==0.4.3 +ndg-httpsclient==0.4.4 netaddr==0.7.19 netifaces==0.10.6 -newrelic==2.96.0.80 +newrelic==2.100.0.84 nose==1.3.7 -numpy==1.13.3 +numpy==1.14.0 oauth2==1.9.0.post1 oauth2client==4.1.2 oauthlib==2.0.6 ordereddict==1.1 -oslo.config==5.0.0 -pandas==0.21.0 -paramiko==2.3.1 +oslo.config==5.2.0 +pandas==0.22.0 +paramiko==2.4.0 passlib==1.7.1 paste==2.0.3 pastedeploy==1.5.2 pastescript==2.0.2 pbr==3.1.1 pep8==1.7.1 -pexpect==4.2.1 -pika==0.11.0 -pillow==4.3.0 +pexpect==4.3.1 +pika==0.11.2 +pillow==5.0.0 pip==9.0.1 prettytable -protobuf==3.4.0 -psutil==5.4.0 +protobuf==3.5.1 +psutil==5.4.3 psycopg2==2.7.3.2 -py==1.4.34 -pyasn1-modules==0.1.5 -pyasn1==0.3.7 +py==1.5.2 +pyasn1-modules==0.2.1 +pyasn1==0.4.2 pycparser==2.18 pycrypto==2.6.1 -pycurl==7.43.0 +pycurl==7.43.0.1 pyflakes==1.6.0 pygments==2.2.0 pyjwt==1.5.3 pylibmc==1.5.2 -pylint==1.7.4 -pymongo==3.5.1 -pymysql==0.7.11 -pyopenssl==17.3.0 +pylint==1.8.2 +pymongo==3.6.0 +pymysql==0.8.0 +pyopenssl==17.5.0 pyparsing==2.2.0 pyramid==1.9.1 pystache==0.5.4 pytest-cov==2.5.1 -pytest==3.2.3 +pytest==3.4.0 python-cjson==1.2.1 python-daemon==2.1.2 python-dateutil==2.6.1 python-gflags==3.1.2 -python-keystoneclient==3.13.0 -python-memcached==1.58 +python-keystoneclient==3.15.0 +python-memcached==1.59 python-mimeparse==1.6.0 -python-novaclient==9.1.0 +python-novaclient==10.1.0 python-subunit==1.2.0 -python-swiftclient==3.4.0 +python-swiftclient==3.5.0 pytz==2017.3 pyyaml==3.12 pyzmq==16.0.3 -raven==6.3.0 +raven==6.5.0 redis==2.10.6 repoze.lru==0.7 requests-oauthlib==0.8.0 @@ -165,30 +165,30 @@ requests==2.18.4 retrying==1.3.3 rsa==3.4.2 scipy==1.0.0 -selenium==3.7.0 +selenium==3.8.1 setuptools-git==1.2 -setuptools==36.6.0 +setuptools==38.5.0 sh==1.12.14 -simplejson==3.11.1 +simplejson==3.13.2 six==1.11.0 snowballstemmer==1.2.1 south==1.0.2 -sphinx==1.6.5 +sphinx==1.6.7 sqlalchemy-migrate==0.11.0 -sqlalchemy==1.1.14 +sqlalchemy==1.2.2 sqlparse==0.2.4 -statsd==3.2.1 -stevedore==1.27.1 +statsd==3.2.2 +stevedore==1.28.0 suds==0.4 supervisor==3.3.3 testrepository==0.0.20 testtools==2.3.0 -thrift==0.10.0 -tornado==4.5.2 +thrift==0.11.0 +tornado==4.5.3 tox==2.9.1 twisted==17.9.0 ujson==1.35 -unidecode==0.4.21 +unidecode==1.0.22 unittest2==1.1.0 uritemplate==3.0.0 urllib3==1.22 @@ -197,11 +197,11 @@ versiontools==1.9.1 virtualenv==15.1.0 waitress==1.1.0 warlock==1.3.0 -webob==1.7.3 -websocket-client==0.44.0 +webob==1.7.4 +websocket-client==0.46.0 webtest==2.0.29 -werkzeug==0.12.2 +werkzeug==0.14.1 wheel==0.30.0 xlrd==1.1.0 -zc.buildout==2.9.5 +zc.buildout==2.11.0 zope.interface==4.4.3 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 2bdcc565..a8664b03 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -1,154 +1,153 @@ -alembic==0.9.6 +alembic==0.9.7 amqp==2.2.2 amqplib==1.0.2 -ansible==2.4.1.0 +ansible==2.4.3.0 anyjson==0.3.3 apache-libcloud==2.2.1 argparse==1.4.0 -astroid==1.5.3 -awscli==1.11.180 -babel==2.5.1 +astroid==1.6.1 +awscli==1.14.32 +babel==2.5.3 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 beautifulsoup4==4.6.0 billiard==3.5.0.3 -blessings==1.6 +blessings==1.6.1 blinker==1.4 boto==2.48.0 -botocore==1.7.38 +botocore==1.8.36 bottle==0.12.13 celery==4.1.0 -certifi==2017.7.27.1 -cffi==1.11.2 +certifi==2018.1.18 +cffi==1.11.4 chardet==3.0.4 click==6.7 -cliff==2.9.1 -cmd2==0.7.7 +cliff==2.11.0 +cmd2==0.8.0 colorama==0.3.9 configobj==5.0.6 cov-core==1.15.0 -coverage==4.4.1 +coverage==4.5 coveralls==1.2.0 crcmod==1.7 -cryptography==2.1.3 -cssselect==1.0.1 -cython==0.27.2 -decorator==4.1.2 -django-celery==3.2.1 -django-debug-toolbar==1.8 -django-extensions==1.9.7 -django==1.11.7 +cryptography==2.1.4 +cssselect==1.0.3 +cython==0.27.3 +decorator==4.2.1 +django-celery==3.2.2 +django-debug-toolbar==1.9.1 +django-extensions==1.9.9 +django==2.0.2 django_compress==1.0.1 -djangorestframework==3.7.1 +djangorestframework==3.7.7 docker-py==1.10.6 docopt==0.6.2 docutils==0.14 ecdsa==0.13 -elasticsearch==5.4.0 +elasticsearch==6.1.1 enum34==1.1.6 -eventlet==0.21.0 +eventlet==0.22.0 extras==1.0.0 fabric==1.14.0 fixtures==3.0.0 flake8==3.5.0 flask==0.12.2 funcsigs==1.0.2 -futures==3.1.1 gevent==1.2.2 -google-api-python-client==1.6.4 -greenlet==0.4.12 +google-api-python-client==1.6.5 +greenlet==0.4.13 gunicorn==19.7.1 hiredis==0.2.0 honcho==1.0.1 -html5lib==0.999999999 +html5lib==1.0.1 httplib2==0.10.3 idna==2.6 -ipaddress==1.0.18 +ipaddress==1.0.19 ipython==6.2.1 iso8601==0.1.12 isodate==0.6.0 itsdangerous==0.24 -jinja2==2.9.6 +jinja2==2.10 jmespath==0.9.3 jsonschema==2.6.0 kombu==4.1.0 linecache2==1.0.0 logilab-common==1.4.1 -lxml==4.1.0 +lxml==4.1.1 mako==1.0.7 manifestparser==1.1 -markdown==2.6.9 +markdown==2.6.11 markupsafe==1.0 -matplotlib==2.1.0 +matplotlib==2.1.2 mccabe==0.6.1 meld3==1.0.2 mock==2.0.0 mozcrash==1.0 -mozdevice==0.50 +mozdevice==0.51 mozfile==1.2 mozinfo==0.10 -mozlog==3.5 +mozlog==3.7 moznetwork==0.27 -mozprocess==0.25 -msgpack-python==0.4.8 -ndg-httpsclient==0.4.3 +mozprocess==0.26 +msgpack-python==0.5.2 +ndg-httpsclient==0.4.4 netaddr==0.7.19 netifaces==0.10.6 -newrelic==2.96.0.80 +newrelic==2.100.0.84 nose==1.3.7 -numpy==1.13.3 +numpy==1.14.0 oauth2==1.9.0.post1 oauth2client==4.1.2 oauthlib==2.0.6 ordereddict==1.1 -oslo.config==5.0.0 -pandas==0.21.0 -paramiko==2.3.1 +oslo.config==5.2.0 +pandas==0.22.0 +paramiko==2.4.0 passlib==1.7.1 paste==2.0.3 pastedeploy==1.5.2 pastescript==2.0.2 pbr==3.1.1 pep8==1.7.1 -pexpect==4.2.1 -pika==0.11.0 -pillow==4.3.0 +pexpect==4.3.1 +pika==0.11.2 +pillow==5.0.0 pip==9.0.1 prettytable -protobuf==3.4.0 -psutil==5.4.0 +protobuf==3.5.1 +psutil==5.4.3 psycopg2==2.7.3.2 -py==1.4.34 -pyasn1-modules==0.1.5 -pyasn1==0.3.7 +py==1.5.2 +pyasn1-modules==0.2.1 +pyasn1==0.4.2 pycparser==2.18 pycrypto==2.6.1 pyflakes==1.6.0 pygments==2.2.0 pyjwt==1.5.3 pylibmc==1.5.2 -pylint==1.7.4 -pymongo==3.5.1 -pymysql==0.7.11 -pyopenssl==17.3.0 +pylint==1.8.2 +pymongo==3.6.0 +pymysql==0.8.0 +pyopenssl==17.5.0 pyparsing==2.2.0 pyramid==1.9.1 pystache==0.5.4 pytest-cov==2.5.1 -pytest==3.2.3 +pytest==3.4.0 python-daemon==2.1.2 python-dateutil==2.6.1 python-gflags==3.1.2 -python-keystoneclient==3.13.0 -python-memcached==1.58 +python-keystoneclient==3.15.0 +python-memcached==1.59 python-mimeparse==1.6.0 -python-novaclient==9.1.0 +python-novaclient==10.1.0 python-subunit==1.2.0 -python-swiftclient==3.4.0 +python-swiftclient==3.5.0 pytz==2017.3 pyyaml==3.12 pyzmq==16.0.3 -raven==6.3.0 +raven==6.5.0 redis==2.10.6 repoze.lru==0.7 requests-oauthlib==0.8.0 @@ -156,28 +155,28 @@ requests==2.18.4 retrying==1.3.3 rsa==3.4.2 scipy==1.0.0 -selenium==3.7.0 +selenium==3.8.1 setuptools-git==1.2 -setuptools==36.6.0 +setuptools==38.5.0 sh==1.12.14 -simplejson==3.11.1 +simplejson==3.13.2 six==1.11.0 snowballstemmer==1.2.1 south==1.0.2 -sphinx==1.6.5 +sphinx==1.6.7 sqlalchemy-migrate==0.11.0 -sqlalchemy==1.1.14 +sqlalchemy==1.2.2 sqlparse==0.2.4 -statsd==3.2.1 -stevedore==1.27.1 +statsd==3.2.2 +stevedore==1.28.0 testrepository==0.0.20 testtools==2.3.0 -thrift==0.10.0 -tornado==4.5.2 +thrift==0.11.0 +tornado==4.5.3 tox==2.9.1 twisted==17.9.0 ujson==1.35 -unidecode==0.4.21 +unidecode==1.0.22 unittest2==1.1.0 uritemplate==3.0.0 urllib3==1.22 @@ -186,11 +185,11 @@ versiontools==1.9.1 virtualenv==15.1.0 waitress==1.1.0 warlock==1.3.0 -webob==1.7.3 -websocket-client==0.44.0 +webob==1.7.4 +websocket-client==0.46.0 webtest==2.0.29 -werkzeug==0.12.2 +werkzeug==0.14.1 wheel==0.30.0 xlrd==1.1.0 -zc.buildout==2.9.5 +zc.buildout==2.11.0 zope.interface==4.4.3 From 1a69141db0b08cc73eccc40a2d3ad03d71a09e73 Mon Sep 17 00:00:00 2001 From: Douglas Greiman Date: Mon, 12 Feb 2018 14:20:24 -0800 Subject: [PATCH 030/132] Update Python interpreters to 3.5.5 and 3.6.4 (#177) --- python-interpreter-builder/Dockerfile.in | 4 ++-- .../scripts/build-python-3.5.sh | 12 ++++++------ .../scripts/build-python-3.6.sh | 12 ++++++------ tests/virtualenv/virtualenv_python35.yaml | 2 +- tests/virtualenv/virtualenv_python36.yaml | 2 +- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/python-interpreter-builder/Dockerfile.in b/python-interpreter-builder/Dockerfile.in index 90e606eb..3c5fc31c 100644 --- a/python-interpreter-builder/Dockerfile.in +++ b/python-interpreter-builder/Dockerfile.in @@ -51,10 +51,10 @@ RUN mkdir -p /opt/packages && \ echo -n "" > /opt/packages/packages.txt RUN /scripts/build-python-3.5.sh && \ - /scripts/package-python.sh 3.5.4 "1gcp~${TAG}" + /scripts/package-python.sh 3.5.5 "1gcp~${TAG}" RUN /scripts/build-python-3.6.sh && \ - /scripts/package-python.sh 3.6.2 "1gcp~${TAG}" + /scripts/package-python.sh 3.6.4 "1gcp~${TAG}" # Tar the interpreters. Tarring is needed because docker cp doesn't handle # links correctly. diff --git a/python-interpreter-builder/scripts/build-python-3.5.sh b/python-interpreter-builder/scripts/build-python-3.5.sh index eeea1f76..c2a3337e 100755 --- a/python-interpreter-builder/scripts/build-python-3.5.sh +++ b/python-interpreter-builder/scripts/build-python-3.5.sh @@ -6,14 +6,14 @@ set -x # Get the source mkdir -p /opt/sources cd /opt/sources -wget --no-verbose https://www.python.org/ftp/python/3.5.4/Python-3.5.4.tgz +wget --no-verbose https://www.python.org/ftp/python/3.5.5/Python-3.5.5.tgz # SHA-256 generated via `shasum -a 256 [file]` shasum --check < Date: Mon, 26 Feb 2018 13:08:16 -0800 Subject: [PATCH 031/132] Test cleanups and fixes (#179) * Separate Google Cloud Client Library tests from other tests. Use the --client_test flag to build.sh to run them. * Fix test failure with google-cloud-dlp. See https://github.com/GoogleCloudPlatform/google-cloud-python/issues/4924 Also add better logging of exactly which files experience failures. * Remove system test which is redundant with the integration test. * Fix punctuation. --- build.sh | 46 ++++++------------- cloudbuild_client_test.yaml | 9 ++++ cloudbuild_system_test.yaml | 9 ---- cloudbuild_test.yaml | 6 --- tests/google-cloud-python-system/.gitignore | 2 - .../google-cloud-python-system/Dockerfile.in | 16 ------- .../run_system_tests.sh | 39 ---------------- tests/google-cloud-python/run_unit_tests.sh | 18 +++++++- 8 files changed, 41 insertions(+), 104 deletions(-) create mode 100644 cloudbuild_client_test.yaml delete mode 100644 cloudbuild_system_test.yaml delete mode 100644 tests/google-cloud-python-system/.gitignore delete mode 100644 tests/google-cloud-python-system/Dockerfile.in delete mode 100755 tests/google-cloud-python-system/run_system_tests.sh diff --git a/build.sh b/build.sh index 7e2370d4..62269d36 100755 --- a/build.sh +++ b/build.sh @@ -19,7 +19,7 @@ set -euo pipefail # Actions benchmark=0 # Should run benchmarks? build=0 # Should build images? -system_test=0 # Should run system tests? +client_test=0 # Should run Google Cloud Client Library tests test=0 # Should run standard test suite? local=0 # Should run using local Docker daemon instead of GCR? @@ -42,8 +42,8 @@ Options: --[no]benchmark: Run benchmarking suite (default false) --[no]build: Build all images (default true if no options set) --[no]test: Run basic tests (default true if no options set) + --[no]client_test: Run Google Cloud Client Library tests (default false) --[no]local: Build images using local Docker daemon (default false) - --[no]system_test: Run system tests (default false) " } @@ -90,20 +90,20 @@ while [ $# -gt 0 ]; do build=0 shift ;; - --local) - local=1 + --client_test) + client_test=1 shift ;; - --nolocal) - local=0 + --noclient_test) + client_test=0 shift ;; - --system_test) - system_test=1 + --local) + local=1 shift ;; - --nosystem_test) - system_test=0 + --nolocal) + local=0 shift ;; --test) @@ -123,7 +123,7 @@ done # If no actions chosen, then tell the user if [ "${benchmark}" -eq 0 -a \ "${build}" -eq 0 -a \ - "${system_test}" -eq 0 -a \ + "${client_test}" -eq 0 -a \ "${test}" -eq 0 \ ]; then echo 'No actions specified, defaulting to --build --test' @@ -136,17 +136,6 @@ if [ "${local}" -eq 1 ]; then gcloud_cmd="${local_gcloud_cmd}" fi -# Read action-specific environment variables -if [ "${system_test}" -eq 1 ]; then - if [ -z "${GOOGLE_APPLICATION_CREDENTIALS_FOR_TESTS+set}" ] ; then - fatal 'Error: $GOOGLE_APPLICATION_CREDENTIALS_FOR_TESTS is not set; invoke with something like GOOGLE_APPLICATION_CREDENTIALS_FOR_TESTS=/path/to/service/account/creds.json' - fi - - if [ -z "${GOOGLE_CLOUD_PROJECT_FOR_TESTS+set}" ] ; then - fatal 'Error: $GOOGLE_CLOUD_PROJECT_FOR_TESTS is not set; invoke with something like GOOGLE_CLOUD_PROJECT_FOR_TESTS=YOUR-PROJECT-NAME' - fi -fi - # Use latest released Debian as our base image export DEBIAN_BASE_IMAGE="gcr.io/google-appengine/debian8:latest" export STAGING_IMAGE="${DOCKER_NAMESPACE}/python:${TAG}" @@ -160,7 +149,6 @@ for outfile in \ tests/benchmark/Dockerfile \ tests/eventlet/Dockerfile \ tests/google-cloud-python/Dockerfile \ - tests/google-cloud-python-system/Dockerfile \ tests/integration/Dockerfile \ ; do envsubst <"${outfile}".in >"${outfile}" \ @@ -192,14 +180,10 @@ if [ "${test}" -eq 1 ]; then ${gcloud_cmd} --config cloudbuild_test.yaml --substitutions "${substitutions}" fi -# Run system tests -if [ "${system_test}" -eq 1 ]; then - echo "Running system tests using project ${GOOGLE_CLOUD_PROJECT_FOR_TESTS}" - - trap "rm -f tests/google-cloud-python-system/credentials.json" EXIT - cp "${GOOGLE_APPLICATION_CREDENTIALS_FOR_TESTS}" tests/google-cloud-python-system/credentials.json - ${gcloud_cmd} --config cloudbuild_system_test.yaml --substitutions "${substitutions}" - rm -f tests/google-cloud-python-system/credentials.json +# Run client library tests +if [ "${client_test}" -eq 1 ]; then + echo "Testing compatibility with Google Cloud Client Libraries" + ${gcloud_cmd} --config cloudbuild_client_test.yaml --substitutions "${substitutions}" fi # Run benchmarks diff --git a/cloudbuild_client_test.yaml b/cloudbuild_client_test.yaml new file mode 100644 index 00000000..010e1ffd --- /dev/null +++ b/cloudbuild_client_test.yaml @@ -0,0 +1,9 @@ +timeout: 3600s +steps: +- # Build image to run google client library unit tests + name: gcr.io/cloud-builders/docker:latest + args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/google-cloud-python:${_TAG}', + '--no-cache', '/workspace/tests/google-cloud-python/'] +- # Run google client library unit tests + name: ${_DOCKER_NAMESPACE}/python/tests/google-cloud-python:${_TAG} +images: [] diff --git a/cloudbuild_system_test.yaml b/cloudbuild_system_test.yaml deleted file mode 100644 index 3cac03e2..00000000 --- a/cloudbuild_system_test.yaml +++ /dev/null @@ -1,9 +0,0 @@ -timeout: 3600s -steps: -- name: gcr.io/cloud-builders/docker:latest - args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/google-cloud-python-system:${_TAG}', - '--no-cache', '/workspace/tests/google-cloud-python-system/'] -- name: ${_DOCKER_NAMESPACE}/python/tests/google-cloud-python-system:${_TAG} -images: [ - # Intentionally empty -] diff --git a/cloudbuild_test.yaml b/cloudbuild_test.yaml index acf70bc7..6efe5a45 100644 --- a/cloudbuild_test.yaml +++ b/cloudbuild_test.yaml @@ -37,10 +37,4 @@ steps: name: gcr.io/cloud-builders/docker:latest args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/eventlet:${_TAG}', '--no-cache', '/workspace/tests/eventlet/'] -- # Build image to run google client library unit tests - name: gcr.io/cloud-builders/docker:latest - args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/google-cloud-python:${_TAG}', - '--no-cache', '/workspace/tests/google-cloud-python/'] -- # Run google client library unit tests - name: ${_DOCKER_NAMESPACE}/python/tests/google-cloud-python:${_TAG} images: [] diff --git a/tests/google-cloud-python-system/.gitignore b/tests/google-cloud-python-system/.gitignore deleted file mode 100644 index 470715ec..00000000 --- a/tests/google-cloud-python-system/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -credentials.json -Dockerfile diff --git a/tests/google-cloud-python-system/Dockerfile.in b/tests/google-cloud-python-system/Dockerfile.in deleted file mode 100644 index 04f28bd5..00000000 --- a/tests/google-cloud-python-system/Dockerfile.in +++ /dev/null @@ -1,16 +0,0 @@ -FROM ${STAGING_IMAGE} - -# Get the source. -RUN git clone --depth 1 https://github.com/GoogleCloudPlatform/google-cloud-python.git -WORKDIR google-cloud-python - -# Install nox -RUN pip install --upgrade nox-automation - -# Secrets injected at runtime -ENV GOOGLE_APPLICATION_CREDENTIALS=/workspace/tests/google-cloud-python-system/credentials.json -ENV GOOGLE_CLOUD_PROJECT=${GOOGLE_CLOUD_PROJECT_FOR_TESTS} - -# Run system tests for all supported Python versions -ADD run_system_tests.sh /run_system_tests.sh -ENTRYPOINT ["/run_system_tests.sh"] diff --git a/tests/google-cloud-python-system/run_system_tests.sh b/tests/google-cloud-python-system/run_system_tests.sh deleted file mode 100755 index f9f81a47..00000000 --- a/tests/google-cloud-python-system/run_system_tests.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/sh -set -eu - -cd /app/google-cloud-python - -# Not all packages have system tests -packages=" -bigquery -bigtable -datastore -language -logging -monitoring -pubsub -spanner -speech -storage -vision -" - -# translate has system test but it gives error message: -# BadRequest: 400 Invalid JSON payload received. Unknown name "model": Cannot bind 'nmt'. Field 'model' could not be found in request message. (GET https://translation.googleapis.com/language/translate/v2?target=de&q=hvala+ti&q=dankon&q=Me+llamo+Jeff&q=My+name+is+Jeff&model=nmt) -disabled_packages="translate" - -# Spanner system test needs this -export GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE=1 - -exit_code=0 -for package in ${packages}; do - noxfile="${package}/nox.py" - nox \ - -f "${noxfile}" \ - -e \ - "system(py='2.7')" \ - "system(py='3.6')" \ - || exit_code=1 -done - -exit "${exit_code}" diff --git a/tests/google-cloud-python/run_unit_tests.sh b/tests/google-cloud-python/run_unit_tests.sh index 81a6a389..c386f1c0 100755 --- a/tests/google-cloud-python/run_unit_tests.sh +++ b/tests/google-cloud-python/run_unit_tests.sh @@ -4,7 +4,13 @@ set -eu cd /app/google-cloud-python exit_code=0 +failed_files= for noxfile in */nox.py; do + if [ "${noxfile}" = "dlp/nox.py" ]; then + echo "**** Skipping ${noxfile} ****" + continue + fi + echo "**** Starting tests in ${noxfile} ****" nox \ -f "${noxfile}" \ -e \ @@ -12,7 +18,17 @@ for noxfile in */nox.py; do "unit(py='3.4')" \ "unit(py='3.5')" \ "unit(py='3.6')" \ - || exit_code=1 + || { + echo "**** FAILED tests in ${noxfile} ****" + exit_code=1 + failed_files="${failed_files} ${noxfile}" + } + echo "**** Finished tests in ${noxfile} ****" done +if [ "${exit_code}" -eq 0 ]; then + echo "**** All tests passed ****" +else + echo "**** There were test failures:${failed_files} ****" +fi exit "${exit_code}" From a1a0c2d7efa00dacc0f197fa0a97426f783da8e3 Mon Sep 17 00:00:00 2001 From: Douglas Greiman Date: Mon, 26 Feb 2018 18:50:31 -0800 Subject: [PATCH 032/132] Remove unused file (#181) --- jenkins_build.sh | 3 --- 1 file changed, 3 deletions(-) delete mode 100755 jenkins_build.sh diff --git a/jenkins_build.sh b/jenkins_build.sh deleted file mode 100755 index f88007f5..00000000 --- a/jenkins_build.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -./build.sh "$@" From fe98d8b5b58d8dee486797d55398e38f76bdfd4d Mon Sep 17 00:00:00 2001 From: Douglas Greiman Date: Wed, 28 Feb 2018 11:32:41 -0800 Subject: [PATCH 033/132] Stop building Debian packages. (#180) We never started using these packages, and have a different internal plan for packaging in the future. --- python-interpreter-builder/Dockerfile.in | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/python-interpreter-builder/Dockerfile.in b/python-interpreter-builder/Dockerfile.in index 3c5fc31c..834047b9 100644 --- a/python-interpreter-builder/Dockerfile.in +++ b/python-interpreter-builder/Dockerfile.in @@ -50,11 +50,9 @@ ADD DEBIAN /DEBIAN RUN mkdir -p /opt/packages && \ echo -n "" > /opt/packages/packages.txt -RUN /scripts/build-python-3.5.sh && \ - /scripts/package-python.sh 3.5.5 "1gcp~${TAG}" +RUN /scripts/build-python-3.5.sh -RUN /scripts/build-python-3.6.sh && \ - /scripts/package-python.sh 3.6.4 "1gcp~${TAG}" +RUN /scripts/build-python-3.6.sh # Tar the interpreters. Tarring is needed because docker cp doesn't handle # links correctly. From 58a537c00e55a75cdfd5fe6e637d33829139b9e8 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Mon, 12 Mar 2018 17:53:31 -0700 Subject: [PATCH 034/132] Auto-update dependencies. (#178) --- runtime-image/resources/requirements.txt | 2 +- scripts/requirements-test.txt | 2 +- tests/eventlet/requirements.txt | 2 +- tests/integration/requirements.txt | 6 +- tests/python2-libraries/requirements.txt | 72 ++++++++++++------------ tests/python3-libraries/requirements.txt | 68 +++++++++++----------- 6 files changed, 76 insertions(+), 76 deletions(-) diff --git a/runtime-image/resources/requirements.txt b/runtime-image/resources/requirements.txt index 6a1b2cc3..ef675285 100644 --- a/runtime-image/resources/requirements.txt +++ b/runtime-image/resources/requirements.txt @@ -1,3 +1,3 @@ pip==9.0.1 -setuptools==38.5.0 +setuptools==38.5.2 wheel==0.30.0 diff --git a/scripts/requirements-test.txt b/scripts/requirements-test.txt index 12a6ca4a..1ae0abd8 100644 --- a/scripts/requirements-test.txt +++ b/scripts/requirements-test.txt @@ -1,4 +1,4 @@ flask==0.12.2 -pytest==3.4.0 +pytest==3.4.2 pytest-cov==2.5.1 pyyaml==3.12 diff --git a/tests/eventlet/requirements.txt b/tests/eventlet/requirements.txt index a8f7b0a9..5d104d0a 100644 --- a/tests/eventlet/requirements.txt +++ b/tests/eventlet/requirements.txt @@ -1,6 +1,6 @@ click==6.7 enum-compat==0.0.2 -eventlet==0.22.0 +eventlet==0.22.1 Flask==0.12.2 greenlet==0.4.13 gunicorn==19.7.1 diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index 99a06d3b..64e39e0c 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -1,7 +1,7 @@ Flask==0.12.2 -google-cloud-error-reporting==0.29.0 -google-cloud-logging==1.4.0 -google-cloud-monitoring==0.28.0 +google-cloud-error-reporting==0.29.1 +google-cloud-logging==1.6.0 +google-cloud-monitoring==0.28.1 gunicorn==19.7.1 requests==2.18.4 retrying==1.3.3 diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index f512ca18..f261209e 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -1,12 +1,12 @@ -alembic==0.9.7 +alembic==0.9.8 amqp==2.2.2 amqplib==1.0.2 ansible==2.4.3.0 anyjson==0.3.3 -apache-libcloud==2.2.1 +apache-libcloud==2.3.0 argparse==1.4.0 astroid==1.6.1 -awscli==1.14.32 +awscli==1.14.53 babel==2.5.3 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 @@ -16,21 +16,21 @@ billiard==3.5.0.3 blessings==1.6.1 blinker==1.4 boto==2.48.0 -botocore==1.8.36 +botocore==1.9.6 bottle==0.12.13 carbon<1.1.1 celery==4.1.0 certifi==2018.1.18 -cffi==1.11.4 +cffi==1.11.5 chardet==3.0.4 click==6.7 cliff==2.11.0 -cmd2==0.8.0 +cmd2==0.8.1 colorama==0.3.9 configobj==5.0.6 cov-core==1.15.0 -coverage==4.5 -coveralls==1.2.0 +coverage==4.5.1 +coveralls==1.3.0 crcmod==1.7 cryptography==2.1.4 cssselect==1.0.3 @@ -38,7 +38,7 @@ cython==0.27.3 decorator==4.2.1 django-celery==3.2.2 django-debug-toolbar==1.9.1 -django-extensions==1.9.9 +django-extensions==2.0.5 django<2.0 django_compress==1.0.1 djangorestframework==3.7.7 @@ -48,7 +48,7 @@ docutils==0.14 ecdsa==0.13 elasticsearch==6.1.1 enum34==1.1.6 -eventlet==0.22.0 +eventlet==0.22.1 extras==1.0.0 fabric==1.14.0 fixtures==3.0.0 @@ -59,7 +59,7 @@ functools32==3.2.3.post2 futures==3.2.0 gevent==1.2.2 google-api-python-client==1.6.5 -graphite-web==1.1.1 +graphite-web==1.1.2 greenlet==0.4.13 gunicorn==19.7.1 hiredis==0.2.0 @@ -78,17 +78,17 @@ kombu==4.1.0 linecache2==1.0.0 logilab-common==1.4.1 lxml==4.1.1 -m2crypto==0.27.0 +m2crypto==0.29.0 mako==1.0.7 manifestparser==1.1 markdown==2.6.11 markupsafe==1.0 -matplotlib==2.1.2 +matplotlib==2.2.0 mccabe==0.6.1 meld3==1.0.2 mock==2.0.0 mozcrash==1.0 -mozdevice==0.51 +mozdevice==0.52 mozfile==1.2 mozinfo==0.10 mozlog==3.7 @@ -96,14 +96,14 @@ moznetwork==0.27 mozprocess==0.26 mozprofile==0.29 mozrunner==6.14 -msgpack-python==0.5.2 +msgpack-python==0.5.6 mysql-python==1.2.5 ndg-httpsclient==0.4.4 netaddr==0.7.19 netifaces==0.10.6 -newrelic==2.100.0.84 +newrelic==2.106.1.88 nose==1.3.7 -numpy==1.14.0 +numpy==1.14.1 oauth2==1.9.0.post1 oauth2client==4.1.2 oauthlib==2.0.6 @@ -117,14 +117,14 @@ pastedeploy==1.5.2 pastescript==2.0.2 pbr==3.1.1 pep8==1.7.1 -pexpect==4.3.1 +pexpect==4.4.0 pika==0.11.2 pillow==5.0.0 pip==9.0.1 prettytable -protobuf==3.5.1 +protobuf==3.5.2 psutil==5.4.3 -psycopg2==2.7.3.2 +psycopg2==2.7.4 py==1.5.2 pyasn1-modules==0.2.1 pyasn1==0.4.2 @@ -133,20 +133,20 @@ pycrypto==2.6.1 pycurl==7.43.0.1 pyflakes==1.6.0 pygments==2.2.0 -pyjwt==1.5.3 +pyjwt==1.6.0 pylibmc==1.5.2 pylint==1.8.2 -pymongo==3.6.0 +pymongo==3.6.1 pymysql==0.8.0 pyopenssl==17.5.0 pyparsing==2.2.0 pyramid==1.9.1 pystache==0.5.4 pytest-cov==2.5.1 -pytest==3.4.0 +pytest==3.4.2 python-cjson==1.2.1 python-daemon==2.1.2 -python-dateutil==2.6.1 +python-dateutil==2.7.0 python-gflags==3.1.2 python-keystoneclient==3.15.0 python-memcached==1.59 @@ -154,10 +154,10 @@ python-mimeparse==1.6.0 python-novaclient==10.1.0 python-subunit==1.2.0 python-swiftclient==3.5.0 -pytz==2017.3 +pytz==2018.3 pyyaml==3.12 -pyzmq==16.0.3 -raven==6.5.0 +pyzmq==17.0.0 +raven==6.6.0 redis==2.10.6 repoze.lru==0.7 requests-oauthlib==0.8.0 @@ -165,26 +165,26 @@ requests==2.18.4 retrying==1.3.3 rsa==3.4.2 scipy==1.0.0 -selenium==3.8.1 +selenium==3.10.0 setuptools-git==1.2 -setuptools==38.5.0 +setuptools==38.5.2 sh==1.12.14 simplejson==3.13.2 six==1.11.0 snowballstemmer==1.2.1 south==1.0.2 -sphinx==1.6.7 +sphinx==1.7.1 sqlalchemy-migrate==0.11.0 -sqlalchemy==1.2.2 +sqlalchemy==1.2.5 sqlparse==0.2.4 statsd==3.2.2 stevedore==1.28.0 suds==0.4 -supervisor==3.3.3 +supervisor==3.3.4 testrepository==0.0.20 testtools==2.3.0 thrift==0.11.0 -tornado==4.5.3 +tornado==5.0 tox==2.9.1 twisted==17.9.0 ujson==1.35 @@ -192,16 +192,16 @@ unidecode==1.0.22 unittest2==1.1.0 uritemplate==3.0.0 urllib3==1.22 -uwsgi==2.0.15 +uwsgi==2.0.17 versiontools==1.9.1 virtualenv==15.1.0 waitress==1.1.0 warlock==1.3.0 webob==1.7.4 -websocket-client==0.46.0 +websocket-client==0.47.0 webtest==2.0.29 werkzeug==0.14.1 wheel==0.30.0 xlrd==1.1.0 -zc.buildout==2.11.0 +zc.buildout==2.11.1 zope.interface==4.4.3 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index a8664b03..30fb236e 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -1,12 +1,12 @@ -alembic==0.9.7 +alembic==0.9.8 amqp==2.2.2 amqplib==1.0.2 ansible==2.4.3.0 anyjson==0.3.3 -apache-libcloud==2.2.1 +apache-libcloud==2.3.0 argparse==1.4.0 astroid==1.6.1 -awscli==1.14.32 +awscli==1.14.53 babel==2.5.3 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 @@ -15,20 +15,20 @@ billiard==3.5.0.3 blessings==1.6.1 blinker==1.4 boto==2.48.0 -botocore==1.8.36 +botocore==1.9.6 bottle==0.12.13 celery==4.1.0 certifi==2018.1.18 -cffi==1.11.4 +cffi==1.11.5 chardet==3.0.4 click==6.7 cliff==2.11.0 -cmd2==0.8.0 +cmd2==0.8.1 colorama==0.3.9 configobj==5.0.6 cov-core==1.15.0 -coverage==4.5 -coveralls==1.2.0 +coverage==4.5.1 +coveralls==1.3.0 crcmod==1.7 cryptography==2.1.4 cssselect==1.0.3 @@ -36,8 +36,8 @@ cython==0.27.3 decorator==4.2.1 django-celery==3.2.2 django-debug-toolbar==1.9.1 -django-extensions==1.9.9 -django==2.0.2 +django-extensions==2.0.5 +django==2.0.3 django_compress==1.0.1 djangorestframework==3.7.7 docker-py==1.10.6 @@ -46,7 +46,7 @@ docutils==0.14 ecdsa==0.13 elasticsearch==6.1.1 enum34==1.1.6 -eventlet==0.22.0 +eventlet==0.22.1 extras==1.0.0 fabric==1.14.0 fixtures==3.0.0 @@ -78,24 +78,24 @@ mako==1.0.7 manifestparser==1.1 markdown==2.6.11 markupsafe==1.0 -matplotlib==2.1.2 +matplotlib==2.2.0 mccabe==0.6.1 meld3==1.0.2 mock==2.0.0 mozcrash==1.0 -mozdevice==0.51 +mozdevice==0.52 mozfile==1.2 mozinfo==0.10 mozlog==3.7 moznetwork==0.27 mozprocess==0.26 -msgpack-python==0.5.2 +msgpack-python==0.5.6 ndg-httpsclient==0.4.4 netaddr==0.7.19 netifaces==0.10.6 -newrelic==2.100.0.84 +newrelic==2.106.1.88 nose==1.3.7 -numpy==1.14.0 +numpy==1.14.1 oauth2==1.9.0.post1 oauth2client==4.1.2 oauthlib==2.0.6 @@ -109,14 +109,14 @@ pastedeploy==1.5.2 pastescript==2.0.2 pbr==3.1.1 pep8==1.7.1 -pexpect==4.3.1 +pexpect==4.4.0 pika==0.11.2 pillow==5.0.0 pip==9.0.1 prettytable -protobuf==3.5.1 +protobuf==3.5.2 psutil==5.4.3 -psycopg2==2.7.3.2 +psycopg2==2.7.4 py==1.5.2 pyasn1-modules==0.2.1 pyasn1==0.4.2 @@ -124,19 +124,19 @@ pycparser==2.18 pycrypto==2.6.1 pyflakes==1.6.0 pygments==2.2.0 -pyjwt==1.5.3 +pyjwt==1.6.0 pylibmc==1.5.2 pylint==1.8.2 -pymongo==3.6.0 +pymongo==3.6.1 pymysql==0.8.0 pyopenssl==17.5.0 pyparsing==2.2.0 pyramid==1.9.1 pystache==0.5.4 pytest-cov==2.5.1 -pytest==3.4.0 +pytest==3.4.2 python-daemon==2.1.2 -python-dateutil==2.6.1 +python-dateutil==2.7.0 python-gflags==3.1.2 python-keystoneclient==3.15.0 python-memcached==1.59 @@ -144,10 +144,10 @@ python-mimeparse==1.6.0 python-novaclient==10.1.0 python-subunit==1.2.0 python-swiftclient==3.5.0 -pytz==2017.3 +pytz==2018.3 pyyaml==3.12 -pyzmq==16.0.3 -raven==6.5.0 +pyzmq==17.0.0 +raven==6.6.0 redis==2.10.6 repoze.lru==0.7 requests-oauthlib==0.8.0 @@ -155,24 +155,24 @@ requests==2.18.4 retrying==1.3.3 rsa==3.4.2 scipy==1.0.0 -selenium==3.8.1 +selenium==3.10.0 setuptools-git==1.2 -setuptools==38.5.0 +setuptools==38.5.2 sh==1.12.14 simplejson==3.13.2 six==1.11.0 snowballstemmer==1.2.1 south==1.0.2 -sphinx==1.6.7 +sphinx==1.7.1 sqlalchemy-migrate==0.11.0 -sqlalchemy==1.2.2 +sqlalchemy==1.2.5 sqlparse==0.2.4 statsd==3.2.2 stevedore==1.28.0 testrepository==0.0.20 testtools==2.3.0 thrift==0.11.0 -tornado==4.5.3 +tornado==5.0 tox==2.9.1 twisted==17.9.0 ujson==1.35 @@ -180,16 +180,16 @@ unidecode==1.0.22 unittest2==1.1.0 uritemplate==3.0.0 urllib3==1.22 -uwsgi==2.0.15 +uwsgi==2.0.17 versiontools==1.9.1 virtualenv==15.1.0 waitress==1.1.0 warlock==1.3.0 webob==1.7.4 -websocket-client==0.46.0 +websocket-client==0.47.0 webtest==2.0.29 werkzeug==0.14.1 wheel==0.30.0 xlrd==1.1.0 -zc.buildout==2.11.0 +zc.buildout==2.11.1 zope.interface==4.4.3 From 8106a265e0ad5f44d9db43bbf44d742282a96a74 Mon Sep 17 00:00:00 2001 From: Douglas Greiman Date: Tue, 13 Mar 2018 14:05:07 -0700 Subject: [PATCH 035/132] Add support for using Ubuntu 16 as a base instead of Debian 8 (#182) * Add --os_base flag to build against different OS base images. Also, change --local to use the official gcloud tool instead of our own. * Update from Python 3.4.2 to Python 3.4.8 Contains various bugfixes and security patches. We switch from using the Debian-provided "python3.4" packages to building it from source, in preparation for moving to newer OS base images that don't have a prebuilt Python 3.4 interpreter available. * Update comments about configure flags * Temporarily disable license check * Add some necessary packages for Ubuntu * Ubuntu ships with Python 2.7.12 * Remove errant whitespace * Rename DEBIAN_BASE_IMAGE to OS_BASE_IMAGE --- build.sh | 46 ++++-- cloudbuild_test.yaml | 70 +++++---- python-interpreter-builder/Dockerfile.in | 5 +- .../scripts/build-python-3.4.sh | 138 ++++++++++++++++++ runtime-image/Dockerfile.in | 6 +- runtime-image/resources/apt-packages.txt | 3 - tests/license-test/license-test.yaml | 2 +- tests/no-virtualenv/no-virtualenv.yaml | 2 +- tests/virtualenv/virtualenv_default.yaml | 2 +- tests/virtualenv/virtualenv_python27.yaml | 2 +- tests/virtualenv/virtualenv_python34.yaml | 2 +- 11 files changed, 232 insertions(+), 46 deletions(-) create mode 100755 python-interpreter-builder/scripts/build-python-3.4.sh diff --git a/build.sh b/build.sh index 62269d36..9b3a258d 100755 --- a/build.sh +++ b/build.sh @@ -24,9 +24,12 @@ test=0 # Should run standard test suite? local=0 # Should run using local Docker daemon instead of GCR? +os_base=debian8 # Which operating system base to use + # Note that $gcloud_cmd has spaces in it -gcloud_cmd="gcloud beta container builds submit ." -local_gcloud_cmd="scripts/local_cloudbuild.py" +gcloud_cmd="gcloud container builds submit" +# May need to install via "gcloud components install container-builder-local" +local_gcloud_cmd="container-builder-local --push=false --dryrun=false" # Helper functions function fatal() { @@ -44,6 +47,7 @@ Options: --[no]test: Run basic tests (default true if no options set) --[no]client_test: Run Google Cloud Client Library tests (default false) --[no]local: Build images using local Docker daemon (default false) + --os_base: Which OS image to build on top of [debian8, ubuntu16] " } @@ -106,6 +110,14 @@ while [ $# -gt 0 ]; do local=0 shift ;; + --os_base=debian8) + os_base=debian8 + shift + ;; + --os_base=ubuntu16) + os_base=ubuntu16 + shift + ;; --test) test=1 shift @@ -136,8 +148,12 @@ if [ "${local}" -eq 1 ]; then gcloud_cmd="${local_gcloud_cmd}" fi -# Use latest released Debian as our base image -export DEBIAN_BASE_IMAGE="gcr.io/google-appengine/debian8:latest" +# Pick OS image to use as base +if [ "${os_base}" == "ubuntu16" ]; then + export OS_BASE_IMAGE="gcr.io/gcp-runtimes/ubuntu_16_0_4:latest" +else + export OS_BASE_IMAGE="gcr.io/google-appengine/debian8:latest" +fi export STAGING_IMAGE="${DOCKER_NAMESPACE}/python:${TAG}" echo "Using base image name ${STAGING_IMAGE}" @@ -152,7 +168,7 @@ for outfile in \ tests/integration/Dockerfile \ ; do envsubst <"${outfile}".in >"${outfile}" \ - '$DEBIAN_BASE_IMAGE $STAGING_IMAGE $GOOGLE_CLOUD_PROJECT_FOR_TESTS $TAG' + '$OS_BASE_IMAGE $STAGING_IMAGE $GOOGLE_CLOUD_PROJECT_FOR_TESTS $TAG' done # Make some files available to the runtime builder Docker context @@ -171,23 +187,35 @@ cp -a scripts/testdata/hello_world/main.py tests/eventlet/main.py # Build images and push to GCR if [ "${build}" -eq 1 ]; then echo "Building images" - ${gcloud_cmd} --config cloudbuild.yaml --substitutions "${build_substitutions}" + ${gcloud_cmd} \ + --config=cloudbuild.yaml \ + --substitutions="${build_substitutions}" \ + . fi # Run the tests that don't require (too many) external services if [ "${test}" -eq 1 ]; then echo "Testing compatibility with popular Python libraries" - ${gcloud_cmd} --config cloudbuild_test.yaml --substitutions "${substitutions}" + ${gcloud_cmd} \ + --config=cloudbuild_test.yaml \ + --substitutions="${substitutions}" \ + . fi # Run client library tests if [ "${client_test}" -eq 1 ]; then echo "Testing compatibility with Google Cloud Client Libraries" - ${gcloud_cmd} --config cloudbuild_client_test.yaml --substitutions "${substitutions}" + ${gcloud_cmd} \ + --config=cloudbuild_client_test.yaml \ + --substitutions="${substitutions}" \ + . fi # Run benchmarks if [ "${benchmark}" -eq 1 ] ; then echo "Running benchmark" - ${gcloud_cmd} --config cloudbuild_benchmark.yaml --substitutions "${substitutions}" + ${gcloud_cmd} \ + --config=cloudbuild_benchmark.yaml \ + --substitutions="${substitutions}" \ + . fi diff --git a/cloudbuild_test.yaml b/cloudbuild_test.yaml index 6efe5a45..ef0b48b3 100644 --- a/cloudbuild_test.yaml +++ b/cloudbuild_test.yaml @@ -1,27 +1,12 @@ timeout: 3600s steps: - # Validate structure of base runtime image -- name: gcr.io/cloud-builders/docker:latest - args: ['build', '-t', 'python2-libraries-intermediate', '--build-arg', - 'intermediate_image=${_DOCKER_NAMESPACE}/python:${_TAG}', - '/workspace/tests/python2-libraries'] -- name: gcr.io/gcp-runtimes/container-structure-test:v0.1.1 +- # Explicitly pull image into GCB so that later steps work + name: '${_DOCKER_NAMESPACE}/python:${_TAG}' args: [ - '-test.v', - '-image', 'python2-libraries-intermediate', - '/workspace/tests/python2-libraries/python2-libraries.yaml' - ] -- name: gcr.io/cloud-builders/docker:latest - args: ['build', '-t', 'python3-libraries-intermediate', '--build-arg', - 'intermediate_image=${_DOCKER_NAMESPACE}/python:${_TAG}', - '/workspace/tests/python3-libraries'] -- name: gcr.io/gcp-runtimes/container-structure-test:v0.1.1 - args: [ - '-test.v', - '-image', 'python3-libraries-intermediate', - '/workspace/tests/python3-libraries/python3-libraries.yaml' - ] -- name: gcr.io/gcp-runtimes/container-structure-test:v0.1.1 + '/bin/true', + ] +- # Validate structure of base runtime image + name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 args: [ '-test.v', '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', @@ -31,10 +16,45 @@ steps: '/workspace/tests/virtualenv/virtualenv_python35.yaml', '/workspace/tests/virtualenv/virtualenv_python36.yaml', '/workspace/tests/no-virtualenv/no-virtualenv.yaml', - '/workspace/tests/license-test/license-test.yaml' ] -- # Run compatibility tests +# Temporarily disabled because it fails on symbolic links in Ubuntu: +# https://github.com/GoogleCloudPlatform/container-structure-test/issues/77 +#- # Check license compliance +# name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 +# args: [ +# '-test.v', +# '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', +# '/workspace/tests/license-test/license-test.yaml' +# ] +- # Do third-party library compatibility tests + name: gcr.io/cloud-builders/docker:latest + args: [ + 'build', '-t', 'python2-libraries-intermediate', '--build-arg', + 'intermediate_image=${_DOCKER_NAMESPACE}/python:${_TAG}', + '/workspace/tests/python2-libraries' + ] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 + args: [ + '-test.v', + '-image', 'python2-libraries-intermediate', + '/workspace/tests/python2-libraries/python2-libraries.yaml' + ] +- name: gcr.io/cloud-builders/docker:latest + args: [ + 'build', '-t', 'python3-libraries-intermediate', '--build-arg', + 'intermediate_image=${_DOCKER_NAMESPACE}/python:${_TAG}', + '/workspace/tests/python3-libraries' + ] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 + args: [ + '-test.v', + '-image', 'python3-libraries-intermediate', + '/workspace/tests/python3-libraries/python3-libraries.yaml' + ] +- # Run other compatibility tests name: gcr.io/cloud-builders/docker:latest - args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/tests/eventlet:${_TAG}', - '--no-cache', '/workspace/tests/eventlet/'] + args: [ + 'build', '--tag=${_DOCKER_NAMESPACE}/python/tests/eventlet:${_TAG}', + '--no-cache', '/workspace/tests/eventlet/' + ] images: [] diff --git a/python-interpreter-builder/Dockerfile.in b/python-interpreter-builder/Dockerfile.in index 834047b9..46bc092a 100644 --- a/python-interpreter-builder/Dockerfile.in +++ b/python-interpreter-builder/Dockerfile.in @@ -1,6 +1,6 @@ # The Google App Engine base image is debian (jessie) with ca-certificates # installed. -FROM ${DEBIAN_BASE_IMAGE} +FROM ${OS_BASE_IMAGE} # Install Python build dependencies (based on Debian Build-Depends) RUN apt-get update && apt-get install -yq \ @@ -29,6 +29,7 @@ RUN apt-get update && apt-get install -yq \ mime-support \ net-tools \ netbase \ + python \ python3 \ sharutils \ time \ @@ -50,6 +51,8 @@ ADD DEBIAN /DEBIAN RUN mkdir -p /opt/packages && \ echo -n "" > /opt/packages/packages.txt +RUN /scripts/build-python-3.4.sh + RUN /scripts/build-python-3.5.sh RUN /scripts/build-python-3.6.sh diff --git a/python-interpreter-builder/scripts/build-python-3.4.sh b/python-interpreter-builder/scripts/build-python-3.4.sh new file mode 100755 index 00000000..95089f75 --- /dev/null +++ b/python-interpreter-builder/scripts/build-python-3.4.sh @@ -0,0 +1,138 @@ +#!/bin/bash + +set -euo pipefail +set -x + +# Get the source +mkdir -p /opt/sources +cd /opt/sources +wget --no-verbose https://www.python.org/ftp/python/3.4.8/Python-3.4.8.tgz +# SHA-256 generated via `shasum -a 256 [file]` +shasum --check < Date: Tue, 20 Mar 2018 16:55:49 -0700 Subject: [PATCH 036/132] Increase cloudbuild timeout now that we're building Python 3.4 from source. (#183) --- cloudbuild.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 1eabb997..5554015f 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -1,4 +1,4 @@ -timeout: 7200s +timeout: 10800s steps: - # Compile Python interpreters from source name: gcr.io/cloud-builders/docker:latest From f5364134cd9115ebf47f835e81140107f67c0465 Mon Sep 17 00:00:00 2001 From: Douglas Greiman Date: Thu, 22 Mar 2018 15:02:13 -0700 Subject: [PATCH 037/132] Remove jonwayne, add dlorenc to CODEOWNERS (#185) --- CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CODEOWNERS b/CODEOWNERS index a2d4a67f..c08c2d11 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1,4 +1,4 @@ # Code owners file. # This file controls who is tagged for review for any given pull request. -* @duggelz @liyanhui1228 @jonparrott +* @duggelz @liyanhui1228 @dlorenc From dd6fe8f470466cb52490d6f011e9f232744b7975 Mon Sep 17 00:00:00 2001 From: Douglas Greiman Date: Thu, 22 Mar 2018 15:02:30 -0700 Subject: [PATCH 038/132] Add Runtime Builder template file for "staging" tag. (#184) --- builder/python-staging.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 builder/python-staging.yaml diff --git a/builder/python-staging.yaml b/builder/python-staging.yaml new file mode 100644 index 00000000..1e977a89 --- /dev/null +++ b/builder/python-staging.yaml @@ -0,0 +1,12 @@ +# This is a cloudbuild.yaml template for the runtime builder +steps: +- # Generate application Dockerfile + name: 'gcr.io/gcp-runtimes/python/gen-dockerfile:staging' + args: [ + '--base-image=gcr.io/google-appengine/python:staging' + ] +- # Use that Dockerfile to create final application image + name: 'gcr.io/cloud-builders/docker:latest' + args: ['build', '-t', '$_OUTPUT_IMAGE', '.'] +images: + - '$_OUTPUT_IMAGE' From 9b43e372a1495eceea50240dadda565f5744e225 Mon Sep 17 00:00:00 2001 From: Douglas Greiman Date: Tue, 27 Mar 2018 16:32:38 -0700 Subject: [PATCH 039/132] Build and test python interpreters in parallel instead of serially. (#186) --- cloudbuild.yaml | 23 +++++++-- cloudbuild_test.yaml | 51 ++++++++++++++++++- python-interpreter-builder/Dockerfile.in | 14 ----- .../scripts/build-python-3.4.sh | 3 ++ .../scripts/build-python-3.5.sh | 3 ++ .../scripts/build-python-3.6.sh | 3 ++ runtime-image/Dockerfile.in | 4 +- 7 files changed, 80 insertions(+), 21 deletions(-) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 5554015f..51a77480 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -1,20 +1,35 @@ timeout: 10800s steps: -- # Compile Python interpreters from source +- # Compile Python interpreters from source. This step happens first, then + # the next three in parallel. name: gcr.io/cloud-builders/docker:latest args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/interpreter-builder:${_TAG}', '--no-cache', '/workspace/python-interpreter-builder/'] -- # Copy interpreters back to workspace - name: ${_DOCKER_NAMESPACE}/python/interpreter-builder:${_TAG} - args: ['cp', '/interpreters.tar.gz', '/workspace/runtime-image/interpreters.tar.gz'] + id: interpreter-builder +- name: ${_DOCKER_NAMESPACE}/python/interpreter-builder:${_TAG} + args: ['/scripts/build-python-3.4.sh'] + id: build-3.4 + waitFor: ['interpreter-builder'] +- name: ${_DOCKER_NAMESPACE}/python/interpreter-builder:${_TAG} + args: ['/scripts/build-python-3.5.sh'] + id: build-3.5 + waitFor: ['interpreter-builder'] +- name: ${_DOCKER_NAMESPACE}/python/interpreter-builder:${_TAG} + args: ['/scripts/build-python-3.6.sh'] + id: build-3.6 + waitFor: ['interpreter-builder'] - # Build base runtime image name: gcr.io/cloud-builders/docker:latest args: ['build', '--tag=${_DOCKER_NAMESPACE}/python:${_TAG}', '--no-cache', '/workspace/runtime-image/'] + id: runtime + waitFor: ['build-3.4', 'build-3.5', 'build-3.6'] - # Build runtime builder image name: gcr.io/cloud-builders/docker:latest args: ['build', '--tag=${_BUILDER_DOCKER_NAMESPACE}/python/gen-dockerfile:${_TAG}', '--no-cache', '/workspace/builder/gen-dockerfile/'] + id: gen-dockerfile + waitFor: ['runtime'] images: [ '${_DOCKER_NAMESPACE}/python/interpreter-builder:${_TAG}', '${_DOCKER_NAMESPACE}/python:${_TAG}', diff --git a/cloudbuild_test.yaml b/cloudbuild_test.yaml index ef0b48b3..416c5204 100644 --- a/cloudbuild_test.yaml +++ b/cloudbuild_test.yaml @@ -5,18 +5,52 @@ steps: args: [ '/bin/true', ] + id: runtime + - # Validate structure of base runtime image name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 args: [ '-test.v', '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', '/workspace/tests/virtualenv/virtualenv_default.yaml', + ] + waitFor: ['runtime'] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 + args: [ + '-test.v', + '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', '/workspace/tests/virtualenv/virtualenv_python27.yaml', + ] + waitFor: ['runtime'] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 + args: [ + '-test.v', + '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', '/workspace/tests/virtualenv/virtualenv_python34.yaml', + ] + waitFor: ['runtime'] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 + args: [ + '-test.v', + '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', '/workspace/tests/virtualenv/virtualenv_python35.yaml', + ] + waitFor: ['runtime'] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 + args: [ + '-test.v', + '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', '/workspace/tests/virtualenv/virtualenv_python36.yaml', + ] + waitFor: ['runtime'] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 + args: [ + '-test.v', + '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', '/workspace/tests/no-virtualenv/no-virtualenv.yaml', ] + waitFor: ['runtime'] + # Temporarily disabled because it fails on symbolic links in Ubuntu: # https://github.com/GoogleCloudPlatform/container-structure-test/issues/77 #- # Check license compliance @@ -26,35 +60,48 @@ steps: # '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', # '/workspace/tests/license-test/license-test.yaml' # ] -- # Do third-party library compatibility tests +# waitFor: ['runtime'] + +- # Do third-party library compatibility tests for Python 2 name: gcr.io/cloud-builders/docker:latest args: [ 'build', '-t', 'python2-libraries-intermediate', '--build-arg', 'intermediate_image=${_DOCKER_NAMESPACE}/python:${_TAG}', '/workspace/tests/python2-libraries' ] + id: python2-libraries-intermediate + waitFor: ['runtime'] - name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 args: [ '-test.v', '-image', 'python2-libraries-intermediate', '/workspace/tests/python2-libraries/python2-libraries.yaml' ] -- name: gcr.io/cloud-builders/docker:latest + waitFor: ['python2-libraries-intermediate'] + +- # Do third-party library compatibility tests for Python 3 + name: gcr.io/cloud-builders/docker:latest args: [ 'build', '-t', 'python3-libraries-intermediate', '--build-arg', 'intermediate_image=${_DOCKER_NAMESPACE}/python:${_TAG}', '/workspace/tests/python3-libraries' ] + id: python3-libraries-intermediate + waitFor: ['runtime'] - name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 args: [ '-test.v', '-image', 'python3-libraries-intermediate', '/workspace/tests/python3-libraries/python3-libraries.yaml' ] + waitFor: ['python3-libraries-intermediate'] + - # Run other compatibility tests name: gcr.io/cloud-builders/docker:latest args: [ 'build', '--tag=${_DOCKER_NAMESPACE}/python/tests/eventlet:${_TAG}', '--no-cache', '/workspace/tests/eventlet/' ] + waitFor: ['runtime'] + images: [] diff --git a/python-interpreter-builder/Dockerfile.in b/python-interpreter-builder/Dockerfile.in index 46bc092a..9039fbf3 100644 --- a/python-interpreter-builder/Dockerfile.in +++ b/python-interpreter-builder/Dockerfile.in @@ -46,17 +46,3 @@ ENV LANG C.UTF-8 # Add build scripts ADD scripts /scripts ADD DEBIAN /DEBIAN - -# Build the Python interpreters -RUN mkdir -p /opt/packages && \ - echo -n "" > /opt/packages/packages.txt - -RUN /scripts/build-python-3.4.sh - -RUN /scripts/build-python-3.5.sh - -RUN /scripts/build-python-3.6.sh - -# Tar the interpreters. Tarring is needed because docker cp doesn't handle -# links correctly. -RUN tar czf /interpreters.tar.gz /opt/python?.? diff --git a/python-interpreter-builder/scripts/build-python-3.4.sh b/python-interpreter-builder/scripts/build-python-3.4.sh index 95089f75..5c0bfb8e 100755 --- a/python-interpreter-builder/scripts/build-python-3.4.sh +++ b/python-interpreter-builder/scripts/build-python-3.4.sh @@ -136,3 +136,6 @@ find "$PREFIX"/lib/python3.4/test \ cd /opt rm /opt/sources/Python-3.4.8.tgz rm -r /opt/sources/Python-3.4.8 + +# Archive and copy to persistent external volume +tar czf /workspace/runtime-image/interpreter-3.4.tar.gz /opt/python3.4 diff --git a/python-interpreter-builder/scripts/build-python-3.5.sh b/python-interpreter-builder/scripts/build-python-3.5.sh index c2a3337e..86a564c3 100755 --- a/python-interpreter-builder/scripts/build-python-3.5.sh +++ b/python-interpreter-builder/scripts/build-python-3.5.sh @@ -148,3 +148,6 @@ find "$PREFIX"/lib/python3.5/test \ cd /opt rm /opt/sources/Python-3.5.5.tgz rm -r /opt/sources/Python-3.5.5 + +# Archive and copy to persistent external volume +tar czf /workspace/runtime-image/interpreter-3.5.tar.gz /opt/python3.5 diff --git a/python-interpreter-builder/scripts/build-python-3.6.sh b/python-interpreter-builder/scripts/build-python-3.6.sh index a61b2a8b..fc19fc5e 100755 --- a/python-interpreter-builder/scripts/build-python-3.6.sh +++ b/python-interpreter-builder/scripts/build-python-3.6.sh @@ -148,3 +148,6 @@ find "$PREFIX"/lib/python3.6/test \ cd /opt rm /opt/sources/Python-3.6.4.tgz rm -r /opt/sources/Python-3.6.4 + +# Archive and copy to persistent external volume +tar czf /workspace/runtime-image/interpreter-3.6.tar.gz /opt/python3.6 diff --git a/runtime-image/Dockerfile.in b/runtime-image/Dockerfile.in index d0a2e060..1332279c 100644 --- a/runtime-image/Dockerfile.in +++ b/runtime-image/Dockerfile.in @@ -17,7 +17,9 @@ ENV LANG C.UTF-8 ENV PYTHONUNBUFFERED 1 # Install the Google-built interpreters -ADD interpreters.tar.gz / +ADD interpreter-3.4.tar.gz / +ADD interpreter-3.5.tar.gz / +ADD interpreter-3.6.tar.gz / # Add Google-built interpreters to the path ENV PATH /opt/python3.6/bin:/opt/python3.5/bin:/opt/python3.4/bin:$PATH From 8d34b501d5b08d3995e1c45c9e212dfa82387043 Mon Sep 17 00:00:00 2001 From: Douglas Greiman Date: Thu, 29 Mar 2018 14:04:02 -0700 Subject: [PATCH 040/132] Restore python3 and pip3 $PATH entries. (#189) They now invoke Python 3.6. Previously they invoked Python 3.4. --- runtime-image/Dockerfile.in | 2 ++ tests/no-virtualenv/no-virtualenv.yaml | 11 +++++++++++ 2 files changed, 13 insertions(+) diff --git a/runtime-image/Dockerfile.in b/runtime-image/Dockerfile.in index 1332279c..f0040cff 100644 --- a/runtime-image/Dockerfile.in +++ b/runtime-image/Dockerfile.in @@ -23,6 +23,8 @@ ADD interpreter-3.6.tar.gz / # Add Google-built interpreters to the path ENV PATH /opt/python3.6/bin:/opt/python3.5/bin:/opt/python3.4/bin:$PATH +RUN update-alternatives --install /usr/local/bin/python3 python3 /opt/python3.6/bin/python3.6 50 +RUN update-alternatives --install /usr/local/bin/pip3 pip3 /opt/python3.6/bin/pip3.6 50 # Upgrade pip (debian package version tends to run a few version behind) and # install virtualenv system-wide. diff --git a/tests/no-virtualenv/no-virtualenv.yaml b/tests/no-virtualenv/no-virtualenv.yaml index cf997e29..1015b0da 100644 --- a/tests/no-virtualenv/no-virtualenv.yaml +++ b/tests/no-virtualenv/no-virtualenv.yaml @@ -37,6 +37,17 @@ commandTests: command: ["which", "gunicorn"] expectedOutput: ["/usr/local/bin/gunicorn\n"] + - # Regression test for issue187 + name: "default python3 installation" + command: ["which", "python3"] + expectedOutput: ["/usr/local/bin/python3\n"] + - name: "default python3 version" + command: ["python3", "--version"] + expectedOutput: ["Python 3.6.4\n"] + - name: "default pip3 installation" + command: ["which", "pip3"] + expectedOutput: ["/usr/local/bin/pip3\n"] + - name: "default flask installation" # Checks that 'pip' and 'python' are using the same Python version setup: [["pip", "install", "flask"]] From bae048399184724052ee344ec6de9553e13fbb6b Mon Sep 17 00:00:00 2001 From: dlorenc Date: Tue, 3 Apr 2018 08:38:37 -0700 Subject: [PATCH 041/132] Update to Python 3.6.5. --- .../scripts/build-python-3.6.sh | 12 ++++++------ tests/no-virtualenv/no-virtualenv.yaml | 2 +- tests/virtualenv/virtualenv_python36.yaml | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/python-interpreter-builder/scripts/build-python-3.6.sh b/python-interpreter-builder/scripts/build-python-3.6.sh index fc19fc5e..63a09d2f 100755 --- a/python-interpreter-builder/scripts/build-python-3.6.sh +++ b/python-interpreter-builder/scripts/build-python-3.6.sh @@ -6,14 +6,14 @@ set -x # Get the source mkdir -p /opt/sources cd /opt/sources -wget --no-verbose https://www.python.org/ftp/python/3.6.4/Python-3.6.4.tgz +wget --no-verbose https://www.python.org/ftp/python/3.6.5/Python-3.6.5.tgz # SHA-256 generated via `shasum -a 256 [file]` shasum --check < Date: Wed, 4 Apr 2018 23:39:32 -0700 Subject: [PATCH 042/132] Remove duplicate pip executables to avoid confusion. (#191) --- runtime-image/Dockerfile.in | 3 +++ 1 file changed, 3 insertions(+) diff --git a/runtime-image/Dockerfile.in b/runtime-image/Dockerfile.in index f0040cff..b6c76d71 100644 --- a/runtime-image/Dockerfile.in +++ b/runtime-image/Dockerfile.in @@ -30,8 +30,11 @@ RUN update-alternatives --install /usr/local/bin/pip3 pip3 /opt/python3.6/bin/pi # install virtualenv system-wide. RUN /usr/bin/pip install --upgrade -r /resources/requirements.txt && \ /opt/python3.4/bin/pip3.4 install --upgrade -r /resources/requirements.txt && \ + rm -f /opt/python3.4/bin/pip /opt/python3.4/bin/pip3 && \ /opt/python3.5/bin/pip3.5 install --upgrade -r /resources/requirements.txt && \ + rm -f /opt/python3.5/bin/pip /opt/python3.5/bin/pip3 && \ /opt/python3.6/bin/pip3.6 install --upgrade -r /resources/requirements.txt && \ + rm -f /opt/python3.6/bin/pip /opt/python3.6/bin/pip3 && \ /usr/bin/pip install --upgrade -r /resources/requirements-virtualenv.txt # Setup the app working directory From a8a3e8b2d3239c184843db818e34a06f12dc1190 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Thu, 5 Apr 2018 14:09:34 -0700 Subject: [PATCH 043/132] Auto-update dependencies. (#188) --- .../resources/requirements-virtualenv.txt | 2 +- runtime-image/resources/requirements.txt | 6 +- scripts/requirements-test.txt | 2 +- tests/python2-libraries/requirements.txt | 72 +++++++++---------- tests/python3-libraries/requirements.txt | 72 +++++++++---------- 5 files changed, 77 insertions(+), 77 deletions(-) diff --git a/runtime-image/resources/requirements-virtualenv.txt b/runtime-image/resources/requirements-virtualenv.txt index a4ce32e5..fa784bd2 100644 --- a/runtime-image/resources/requirements-virtualenv.txt +++ b/runtime-image/resources/requirements-virtualenv.txt @@ -1 +1 @@ -virtualenv==15.1.0 +virtualenv==15.2.0 diff --git a/runtime-image/resources/requirements.txt b/runtime-image/resources/requirements.txt index ef675285..c3f99d24 100644 --- a/runtime-image/resources/requirements.txt +++ b/runtime-image/resources/requirements.txt @@ -1,3 +1,3 @@ -pip==9.0.1 -setuptools==38.5.2 -wheel==0.30.0 +pip==9.0.3 +setuptools==39.0.1 +wheel==0.31.0 diff --git a/scripts/requirements-test.txt b/scripts/requirements-test.txt index 1ae0abd8..8be25b0f 100644 --- a/scripts/requirements-test.txt +++ b/scripts/requirements-test.txt @@ -1,4 +1,4 @@ flask==0.12.2 -pytest==3.4.2 +pytest==3.5.0 pytest-cov==2.5.1 pyyaml==3.12 diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index f261209e..c9c61dec 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -1,12 +1,12 @@ -alembic==0.9.8 +alembic==0.9.9 amqp==2.2.2 amqplib==1.0.2 -ansible==2.4.3.0 +ansible==2.5.0 anyjson==0.3.3 apache-libcloud==2.3.0 argparse==1.4.0 -astroid==1.6.1 -awscli==1.14.53 +astroid==1.6.2 +awscli==1.14.68 babel==2.5.3 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 @@ -16,7 +16,7 @@ billiard==3.5.0.3 blessings==1.6.1 blinker==1.4 boto==2.48.0 -botocore==1.9.6 +botocore==1.9.21 bottle==0.12.13 carbon<1.1.1 celery==4.1.0 @@ -25,20 +25,20 @@ cffi==1.11.5 chardet==3.0.4 click==6.7 cliff==2.11.0 -cmd2==0.8.1 +cmd2==0.8.2 colorama==0.3.9 configobj==5.0.6 cov-core==1.15.0 coverage==4.5.1 coveralls==1.3.0 crcmod==1.7 -cryptography==2.1.4 +cryptography==2.2.2 cssselect==1.0.3 -cython==0.27.3 +cython==0.28.1 decorator==4.2.1 django-celery==3.2.2 django-debug-toolbar==1.9.1 -django-extensions==2.0.5 +django-extensions==2.0.6 django<2.0 django_compress==1.0.1 djangorestframework==3.7.7 @@ -46,7 +46,7 @@ docker-py==1.10.6 docopt==0.6.2 docutils==0.14 ecdsa==0.13 -elasticsearch==6.1.1 +elasticsearch==6.2.0 enum34==1.1.6 eventlet==0.22.1 extras==1.0.0 @@ -58,14 +58,14 @@ funcsigs==1.0.2 functools32==3.2.3.post2 futures==3.2.0 gevent==1.2.2 -google-api-python-client==1.6.5 +google-api-python-client==1.6.6 graphite-web==1.1.2 greenlet==0.4.13 gunicorn==19.7.1 hiredis==0.2.0 honcho==1.0.1 html5lib==1.0.1 -httplib2==0.10.3 +httplib2==0.11.3 idna==2.6 ipaddress==1.0.19 iso8601==0.1.12 @@ -77,13 +77,13 @@ jsonschema==2.6.0 kombu==4.1.0 linecache2==1.0.0 logilab-common==1.4.1 -lxml==4.1.1 +lxml==4.2.1 m2crypto==0.29.0 mako==1.0.7 manifestparser==1.1 markdown==2.6.11 markupsafe==1.0 -matplotlib==2.2.0 +matplotlib==2.2.2 mccabe==0.6.1 meld3==1.0.2 mock==2.0.0 @@ -101,31 +101,31 @@ mysql-python==1.2.5 ndg-httpsclient==0.4.4 netaddr==0.7.19 netifaces==0.10.6 -newrelic==2.106.1.88 +newrelic==3.0.0.89 nose==1.3.7 -numpy==1.14.1 +numpy==1.14.2 oauth2==1.9.0.post1 oauth2client==4.1.2 -oauthlib==2.0.6 +oauthlib==2.0.7 ordereddict==1.1 -oslo.config==5.2.0 +oslo.config==6.0.0 pandas==0.22.0 -paramiko==2.4.0 +paramiko==2.4.1 passlib==1.7.1 paste==2.0.3 pastedeploy==1.5.2 pastescript==2.0.2 -pbr==3.1.1 +pbr==4.0.0 pep8==1.7.1 pexpect==4.4.0 pika==0.11.2 pillow==5.0.0 -pip==9.0.1 +pip==9.0.3 prettytable -protobuf==3.5.2 +protobuf==3.5.2.post1 psutil==5.4.3 psycopg2==2.7.4 -py==1.5.2 +py==1.5.3 pyasn1-modules==0.2.1 pyasn1==0.4.2 pycparser==2.18 @@ -133,9 +133,9 @@ pycrypto==2.6.1 pycurl==7.43.0.1 pyflakes==1.6.0 pygments==2.2.0 -pyjwt==1.6.0 +pyjwt==1.6.1 pylibmc==1.5.2 -pylint==1.8.2 +pylint==1.8.3 pymongo==3.6.1 pymysql==0.8.0 pyopenssl==17.5.0 @@ -143,10 +143,10 @@ pyparsing==2.2.0 pyramid==1.9.1 pystache==0.5.4 pytest-cov==2.5.1 -pytest==3.4.2 +pytest==3.5.0 python-cjson==1.2.1 python-daemon==2.1.2 -python-dateutil==2.7.0 +python-dateutil==2.7.2 python-gflags==3.1.2 python-keystoneclient==3.15.0 python-memcached==1.59 @@ -164,18 +164,18 @@ requests-oauthlib==0.8.0 requests==2.18.4 retrying==1.3.3 rsa==3.4.2 -scipy==1.0.0 -selenium==3.10.0 +scipy==1.0.1 +selenium==3.11.0 setuptools-git==1.2 -setuptools==38.5.2 +setuptools==39.0.1 sh==1.12.14 simplejson==3.13.2 six==1.11.0 snowballstemmer==1.2.1 south==1.0.2 -sphinx==1.7.1 +sphinx==1.7.2 sqlalchemy-migrate==0.11.0 -sqlalchemy==1.2.5 +sqlalchemy==1.2.6 sqlparse==0.2.4 statsd==3.2.2 stevedore==1.28.0 @@ -184,7 +184,7 @@ supervisor==3.3.4 testrepository==0.0.20 testtools==2.3.0 thrift==0.11.0 -tornado==5.0 +tornado==5.0.1 tox==2.9.1 twisted==17.9.0 ujson==1.35 @@ -194,14 +194,14 @@ uritemplate==3.0.0 urllib3==1.22 uwsgi==2.0.17 versiontools==1.9.1 -virtualenv==15.1.0 +virtualenv==15.2.0 waitress==1.1.0 warlock==1.3.0 webob==1.7.4 websocket-client==0.47.0 webtest==2.0.29 werkzeug==0.14.1 -wheel==0.30.0 +wheel==0.31.0 xlrd==1.1.0 -zc.buildout==2.11.1 +zc.buildout==2.11.2 zope.interface==4.4.3 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 30fb236e..855b0906 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -1,12 +1,12 @@ -alembic==0.9.8 +alembic==0.9.9 amqp==2.2.2 amqplib==1.0.2 -ansible==2.4.3.0 +ansible==2.5.0 anyjson==0.3.3 apache-libcloud==2.3.0 argparse==1.4.0 -astroid==1.6.1 -awscli==1.14.53 +astroid==1.6.2 +awscli==1.14.68 babel==2.5.3 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 @@ -15,7 +15,7 @@ billiard==3.5.0.3 blessings==1.6.1 blinker==1.4 boto==2.48.0 -botocore==1.9.6 +botocore==1.9.21 bottle==0.12.13 celery==4.1.0 certifi==2018.1.18 @@ -23,20 +23,20 @@ cffi==1.11.5 chardet==3.0.4 click==6.7 cliff==2.11.0 -cmd2==0.8.1 +cmd2==0.8.2 colorama==0.3.9 configobj==5.0.6 cov-core==1.15.0 coverage==4.5.1 coveralls==1.3.0 crcmod==1.7 -cryptography==2.1.4 +cryptography==2.2.2 cssselect==1.0.3 -cython==0.27.3 +cython==0.28.1 decorator==4.2.1 django-celery==3.2.2 django-debug-toolbar==1.9.1 -django-extensions==2.0.5 +django-extensions==2.0.6 django==2.0.3 django_compress==1.0.1 djangorestframework==3.7.7 @@ -44,7 +44,7 @@ docker-py==1.10.6 docopt==0.6.2 docutils==0.14 ecdsa==0.13 -elasticsearch==6.1.1 +elasticsearch==6.2.0 enum34==1.1.6 eventlet==0.22.1 extras==1.0.0 @@ -54,13 +54,13 @@ flake8==3.5.0 flask==0.12.2 funcsigs==1.0.2 gevent==1.2.2 -google-api-python-client==1.6.5 +google-api-python-client==1.6.6 greenlet==0.4.13 gunicorn==19.7.1 hiredis==0.2.0 honcho==1.0.1 html5lib==1.0.1 -httplib2==0.10.3 +httplib2==0.11.3 idna==2.6 ipaddress==1.0.19 ipython==6.2.1 @@ -73,12 +73,12 @@ jsonschema==2.6.0 kombu==4.1.0 linecache2==1.0.0 logilab-common==1.4.1 -lxml==4.1.1 +lxml==4.2.1 mako==1.0.7 manifestparser==1.1 markdown==2.6.11 markupsafe==1.0 -matplotlib==2.2.0 +matplotlib==2.2.2 mccabe==0.6.1 meld3==1.0.2 mock==2.0.0 @@ -93,40 +93,40 @@ msgpack-python==0.5.6 ndg-httpsclient==0.4.4 netaddr==0.7.19 netifaces==0.10.6 -newrelic==2.106.1.88 +newrelic==3.0.0.89 nose==1.3.7 -numpy==1.14.1 +numpy==1.14.2 oauth2==1.9.0.post1 oauth2client==4.1.2 -oauthlib==2.0.6 +oauthlib==2.0.7 ordereddict==1.1 -oslo.config==5.2.0 +oslo.config==6.0.0 pandas==0.22.0 -paramiko==2.4.0 +paramiko==2.4.1 passlib==1.7.1 paste==2.0.3 pastedeploy==1.5.2 pastescript==2.0.2 -pbr==3.1.1 +pbr==4.0.0 pep8==1.7.1 pexpect==4.4.0 pika==0.11.2 pillow==5.0.0 -pip==9.0.1 +pip==9.0.3 prettytable -protobuf==3.5.2 +protobuf==3.5.2.post1 psutil==5.4.3 psycopg2==2.7.4 -py==1.5.2 +py==1.5.3 pyasn1-modules==0.2.1 pyasn1==0.4.2 pycparser==2.18 pycrypto==2.6.1 pyflakes==1.6.0 pygments==2.2.0 -pyjwt==1.6.0 +pyjwt==1.6.1 pylibmc==1.5.2 -pylint==1.8.2 +pylint==1.8.3 pymongo==3.6.1 pymysql==0.8.0 pyopenssl==17.5.0 @@ -134,9 +134,9 @@ pyparsing==2.2.0 pyramid==1.9.1 pystache==0.5.4 pytest-cov==2.5.1 -pytest==3.4.2 +pytest==3.5.0 python-daemon==2.1.2 -python-dateutil==2.7.0 +python-dateutil==2.7.2 python-gflags==3.1.2 python-keystoneclient==3.15.0 python-memcached==1.59 @@ -154,25 +154,25 @@ requests-oauthlib==0.8.0 requests==2.18.4 retrying==1.3.3 rsa==3.4.2 -scipy==1.0.0 -selenium==3.10.0 +scipy==1.0.1 +selenium==3.11.0 setuptools-git==1.2 -setuptools==38.5.2 +setuptools==39.0.1 sh==1.12.14 simplejson==3.13.2 six==1.11.0 snowballstemmer==1.2.1 south==1.0.2 -sphinx==1.7.1 +sphinx==1.7.2 sqlalchemy-migrate==0.11.0 -sqlalchemy==1.2.5 +sqlalchemy==1.2.6 sqlparse==0.2.4 statsd==3.2.2 stevedore==1.28.0 testrepository==0.0.20 testtools==2.3.0 thrift==0.11.0 -tornado==5.0 +tornado==5.0.1 tox==2.9.1 twisted==17.9.0 ujson==1.35 @@ -182,14 +182,14 @@ uritemplate==3.0.0 urllib3==1.22 uwsgi==2.0.17 versiontools==1.9.1 -virtualenv==15.1.0 +virtualenv==15.2.0 waitress==1.1.0 warlock==1.3.0 webob==1.7.4 websocket-client==0.47.0 webtest==2.0.29 werkzeug==0.14.1 -wheel==0.30.0 +wheel==0.31.0 xlrd==1.1.0 -zc.buildout==2.11.1 +zc.buildout==2.11.2 zope.interface==4.4.3 From 11cc1eba0c3a8d44d5c90fe77a2546358aa3794a Mon Sep 17 00:00:00 2001 From: dlorenc Date: Thu, 17 May 2018 11:07:44 -0700 Subject: [PATCH 044/132] Split the runtime release into an interpreter and a runtime build. --- RELEASING.md | 11 +++++++++++ build.sh | 13 +++++++++++++ cloudbuild.yaml | 20 -------------------- cloudbuild_interpreters.yaml | 29 +++++++++++++++++++++++++++++ runtime-image/Dockerfile.in | 11 +++++++---- 5 files changed, 60 insertions(+), 24 deletions(-) create mode 100644 cloudbuild_interpreters.yaml diff --git a/RELEASING.md b/RELEASING.md index 00b0fe16..9f61130a 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -39,6 +39,17 @@ follows: 4. `build.sh` invokes Google Container Builder with the `cloudbuild-*.yaml` config files. +## Building interpreters + +The interpreters used are now built in a separate step, and stored on GCS. +This allows the runtime images to be build more rapidly. + +To build the interpreters, run: + +```shell +gcloud container builds submit . --config=cloudbuild_interpreters.yaml +``` + ## Building outside Jenkins To build this repository outside Jenkins, authenticate and authorize yourself diff --git a/build.sh b/build.sh index 9b3a258d..badda7b5 100755 --- a/build.sh +++ b/build.sh @@ -25,6 +25,7 @@ test=0 # Should run standard test suite? local=0 # Should run using local Docker daemon instead of GCR? os_base=debian8 # Which operating system base to use +interpreter=0 # Should build interpreters instead of images # Note that $gcloud_cmd has spaces in it gcloud_cmd="gcloud container builds submit" @@ -126,6 +127,10 @@ while [ $# -gt 0 ]; do test=0 shift ;; + --interpreter) + interpreter=1 + shift + ;; *) usage ;; @@ -184,6 +189,14 @@ done # Make a file available to the eventlet test. cp -a scripts/testdata/hello_world/main.py tests/eventlet/main.py +# Build interpreters and push to GCS +if [ "${interpreter}" -eq 1 ]; then + echo "Building interpreters" + ${gcloud_cmd} \ + --config=cloudbuild_interpreters.yaml \ + . +fi + # Build images and push to GCR if [ "${build}" -eq 1 ]; then echo "Building images" diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 51a77480..e240780e 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -1,29 +1,10 @@ timeout: 10800s steps: -- # Compile Python interpreters from source. This step happens first, then - # the next three in parallel. - name: gcr.io/cloud-builders/docker:latest - args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/interpreter-builder:${_TAG}', - '--no-cache', '/workspace/python-interpreter-builder/'] - id: interpreter-builder -- name: ${_DOCKER_NAMESPACE}/python/interpreter-builder:${_TAG} - args: ['/scripts/build-python-3.4.sh'] - id: build-3.4 - waitFor: ['interpreter-builder'] -- name: ${_DOCKER_NAMESPACE}/python/interpreter-builder:${_TAG} - args: ['/scripts/build-python-3.5.sh'] - id: build-3.5 - waitFor: ['interpreter-builder'] -- name: ${_DOCKER_NAMESPACE}/python/interpreter-builder:${_TAG} - args: ['/scripts/build-python-3.6.sh'] - id: build-3.6 - waitFor: ['interpreter-builder'] - # Build base runtime image name: gcr.io/cloud-builders/docker:latest args: ['build', '--tag=${_DOCKER_NAMESPACE}/python:${_TAG}', '--no-cache', '/workspace/runtime-image/'] id: runtime - waitFor: ['build-3.4', 'build-3.5', 'build-3.6'] - # Build runtime builder image name: gcr.io/cloud-builders/docker:latest args: ['build', '--tag=${_BUILDER_DOCKER_NAMESPACE}/python/gen-dockerfile:${_TAG}', @@ -31,7 +12,6 @@ steps: id: gen-dockerfile waitFor: ['runtime'] images: [ - '${_DOCKER_NAMESPACE}/python/interpreter-builder:${_TAG}', '${_DOCKER_NAMESPACE}/python:${_TAG}', '${_BUILDER_DOCKER_NAMESPACE}/python/gen-dockerfile:${_TAG}', ] diff --git a/cloudbuild_interpreters.yaml b/cloudbuild_interpreters.yaml new file mode 100644 index 00000000..7daecc57 --- /dev/null +++ b/cloudbuild_interpreters.yaml @@ -0,0 +1,29 @@ +timeout: 10800s +steps: +- # Compile Python interpreters from source. This step happens first, then + # the next three in parallel. + name: gcr.io/cloud-builders/docker:latest + args: ['build', '--tag=interpreter-builder', + '--no-cache', '/workspace/python-interpreter-builder/'] + id: interpreter-builder +- name: interpreter-builder + args: ['/scripts/build-python-3.4.sh'] + id: build-3.4 + waitFor: ['interpreter-builder'] +- name: interpreter-builder + args: ['/scripts/build-python-3.5.sh'] + id: build-3.5 + waitFor: ['interpreter-builder'] +- name: interpreter-builder + args: ['/scripts/build-python-3.6.sh'] + id: build-3.6 + waitFor: ['interpreter-builder'] + +# Upload them to tbe build-id location +- name: gcr.io/cloud-builders/gsutil:latest + args: ['cp', '/workspace/runtime-image/*.tar.gz', 'gs://python-interpreters/$BUILD_ID/'] + waitFor: ['build-3.4', 'build-3.5', 'build-3.6'] + +# "Tag" this as latest +- name: gcr.io/cloud-builders/gsutil:latest + args: ['cp', '-r', 'gs://python-interpreters/$BUILD_ID/*', 'gs://python-interpreters/latest/'] diff --git a/runtime-image/Dockerfile.in b/runtime-image/Dockerfile.in index b6c76d71..f291ae7f 100644 --- a/runtime-image/Dockerfile.in +++ b/runtime-image/Dockerfile.in @@ -16,10 +16,13 @@ ENV LANG C.UTF-8 # logging collection. ENV PYTHONUNBUFFERED 1 -# Install the Google-built interpreters -ADD interpreter-3.4.tar.gz / -ADD interpreter-3.5.tar.gz / -ADD interpreter-3.6.tar.gz / +RUN wget https://storage.googleapis.com/python-interpreters/latest/interpreter-3.4.tar.gz && \ + wget https://storage.googleapis.com/python-interpreters/latest/interpreter-3.5.tar.gz && \ + wget https://storage.googleapis.com/python-interpreters/latest/interpreter-3.6.tar.gz && \ + tar -xzf interpreter-3.4.tar.gz && \ + tar -xzf interpreter-3.5.tar.gz && \ + tar -xzf interpreter-3.6.tar.gz && \ + rm interpreter-*.tar.gz # Add Google-built interpreters to the path ENV PATH /opt/python3.6/bin:/opt/python3.5/bin:/opt/python3.4/bin:$PATH From 7a1444fead32ad3d9ae9e1d34782f034cd551ce6 Mon Sep 17 00:00:00 2001 From: David Bendory Date: Tue, 17 Jul 2018 18:32:44 -0400 Subject: [PATCH 045/132] container-builder-local has been renamed cloud-build-local. --- build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sh b/build.sh index badda7b5..476917c4 100755 --- a/build.sh +++ b/build.sh @@ -29,8 +29,8 @@ interpreter=0 # Should build interpreters instead of images # Note that $gcloud_cmd has spaces in it gcloud_cmd="gcloud container builds submit" -# May need to install via "gcloud components install container-builder-local" -local_gcloud_cmd="container-builder-local --push=false --dryrun=false" +# May need to install via "gcloud components install cloud-build-local" +local_gcloud_cmd="cloud-build-local --push=false --dryrun=false" # Helper functions function fatal() { From ddfbf2215ce2d315119a27944f8cf41c62836908 Mon Sep 17 00:00:00 2001 From: David Bendory Date: Tue, 17 Jul 2018 18:37:20 -0400 Subject: [PATCH 046/132] `gcloud builds` replaces `gcloud container builds`. --- RELEASING.md | 2 +- build.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/RELEASING.md b/RELEASING.md index 9f61130a..2dcd806d 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -47,7 +47,7 @@ This allows the runtime images to be build more rapidly. To build the interpreters, run: ```shell -gcloud container builds submit . --config=cloudbuild_interpreters.yaml +gcloud builds submit . --config=cloudbuild_interpreters.yaml ``` ## Building outside Jenkins diff --git a/build.sh b/build.sh index 476917c4..bdc4f658 100755 --- a/build.sh +++ b/build.sh @@ -28,7 +28,7 @@ os_base=debian8 # Which operating system base to use interpreter=0 # Should build interpreters instead of images # Note that $gcloud_cmd has spaces in it -gcloud_cmd="gcloud container builds submit" +gcloud_cmd="gcloud builds submit" # May need to install via "gcloud components install cloud-build-local" local_gcloud_cmd="cloud-build-local --push=false --dryrun=false" From 20e146f027aa137fd061c192721be28902a5900a Mon Sep 17 00:00:00 2001 From: sharifelgamal Date: Tue, 31 Jul 2018 16:28:41 -0700 Subject: [PATCH 047/132] bumping cryptography to 2.3 for CVE --- tests/python2-libraries/requirements.txt | 2 +- tests/python3-libraries/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index c9c61dec..1655846a 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -32,7 +32,7 @@ cov-core==1.15.0 coverage==4.5.1 coveralls==1.3.0 crcmod==1.7 -cryptography==2.2.2 +cryptography==2.3 cssselect==1.0.3 cython==0.28.1 decorator==4.2.1 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 855b0906..e8295d80 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -30,7 +30,7 @@ cov-core==1.15.0 coverage==4.5.1 coveralls==1.3.0 crcmod==1.7 -cryptography==2.2.2 +cryptography==2.3 cssselect==1.0.3 cython==0.28.1 decorator==4.2.1 From e0b754bf62e2087c5bbad077b02a9860eb132cb2 Mon Sep 17 00:00:00 2001 From: dlorenc Date: Fri, 10 Aug 2018 09:28:34 -0700 Subject: [PATCH 048/132] Bump Python 3.6. --- .../scripts/build-python-3.6.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/python-interpreter-builder/scripts/build-python-3.6.sh b/python-interpreter-builder/scripts/build-python-3.6.sh index 63a09d2f..ef192e1e 100755 --- a/python-interpreter-builder/scripts/build-python-3.6.sh +++ b/python-interpreter-builder/scripts/build-python-3.6.sh @@ -6,21 +6,21 @@ set -x # Get the source mkdir -p /opt/sources cd /opt/sources -wget --no-verbose https://www.python.org/ftp/python/3.6.5/Python-3.6.5.tgz +wget --no-verbose https://www.python.org/ftp/python/3.6.6/Python-3.6.6.tgz # SHA-256 generated via `shasum -a 256 [file]` shasum --check < Date: Fri, 10 Aug 2018 09:51:01 -0700 Subject: [PATCH 049/132] Bump the test value to 3.6.6. --- tests/no-virtualenv/no-virtualenv.yaml | 2 +- tests/virtualenv/virtualenv_python36.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/no-virtualenv/no-virtualenv.yaml b/tests/no-virtualenv/no-virtualenv.yaml index 2d6e5052..869b93b2 100644 --- a/tests/no-virtualenv/no-virtualenv.yaml +++ b/tests/no-virtualenv/no-virtualenv.yaml @@ -43,7 +43,7 @@ commandTests: expectedOutput: ["/usr/local/bin/python3\n"] - name: "default python3 version" command: ["python3", "--version"] - expectedOutput: ["Python 3.6.5\n"] + expectedOutput: ["Python 3.6.6\n"] - name: "default pip3 installation" command: ["which", "pip3"] expectedOutput: ["/usr/local/bin/pip3\n"] diff --git a/tests/virtualenv/virtualenv_python36.yaml b/tests/virtualenv/virtualenv_python36.yaml index c8578cca..d4de875c 100644 --- a/tests/virtualenv/virtualenv_python36.yaml +++ b/tests/virtualenv/virtualenv_python36.yaml @@ -25,7 +25,7 @@ commandTests: - name: "virtualenv36 python version" setup: [["virtualenv", "-p", "python3.6", "/env"]] command: ["python", "--version"] - expectedOutput: ["Python 3.6.5\n"] + expectedOutput: ["Python 3.6.6\n"] - name: "virtualenv36 pip installation" setup: [["virtualenv", "-p", "python3.6", "/env"]] From b032725790a0109b17d50caaefd96e8953154416 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Mon, 13 Aug 2018 00:51:19 -0700 Subject: [PATCH 050/132] Auto-update dependencies. --- builder/gen-dockerfile/requirements.txt | 2 +- .../resources/requirements-virtualenv.txt | 2 +- runtime-image/resources/requirements.txt | 6 +- scripts/requirements-test.txt | 6 +- scripts/testdata/hello_world/requirements.txt | 4 +- tests/deploy_check/requirements.txt | 4 +- tests/eventlet/requirements.txt | 8 +- tests/integration/requirements.txt | 10 +- tests/python2-libraries/requirements.txt | 194 +++++++++--------- tests/python3-libraries/requirements.txt | 188 ++++++++--------- 10 files changed, 212 insertions(+), 212 deletions(-) diff --git a/builder/gen-dockerfile/requirements.txt b/builder/gen-dockerfile/requirements.txt index 59bc593a..4a285555 100644 --- a/builder/gen-dockerfile/requirements.txt +++ b/builder/gen-dockerfile/requirements.txt @@ -1 +1 @@ -PyYAML==3.12 +PyYAML==3.13 diff --git a/runtime-image/resources/requirements-virtualenv.txt b/runtime-image/resources/requirements-virtualenv.txt index fa784bd2..52bb7da1 100644 --- a/runtime-image/resources/requirements-virtualenv.txt +++ b/runtime-image/resources/requirements-virtualenv.txt @@ -1 +1 @@ -virtualenv==15.2.0 +virtualenv==16.0.0 diff --git a/runtime-image/resources/requirements.txt b/runtime-image/resources/requirements.txt index c3f99d24..44c624d8 100644 --- a/runtime-image/resources/requirements.txt +++ b/runtime-image/resources/requirements.txt @@ -1,3 +1,3 @@ -pip==9.0.3 -setuptools==39.0.1 -wheel==0.31.0 +pip==18.0 +setuptools==40.0.0 +wheel==0.31.1 diff --git a/scripts/requirements-test.txt b/scripts/requirements-test.txt index 8be25b0f..df909d47 100644 --- a/scripts/requirements-test.txt +++ b/scripts/requirements-test.txt @@ -1,4 +1,4 @@ -flask==0.12.2 -pytest==3.5.0 +flask==1.0.2 +pytest==3.7.1 pytest-cov==2.5.1 -pyyaml==3.12 +pyyaml==3.13 diff --git a/scripts/testdata/hello_world/requirements.txt b/scripts/testdata/hello_world/requirements.txt index bb84e50e..a34d076b 100644 --- a/scripts/testdata/hello_world/requirements.txt +++ b/scripts/testdata/hello_world/requirements.txt @@ -1,2 +1,2 @@ -Flask==0.12.2 -gunicorn==19.7.1 +Flask==1.0.2 +gunicorn==19.9.0 diff --git a/tests/deploy_check/requirements.txt b/tests/deploy_check/requirements.txt index bb84e50e..a34d076b 100644 --- a/tests/deploy_check/requirements.txt +++ b/tests/deploy_check/requirements.txt @@ -1,2 +1,2 @@ -Flask==0.12.2 -gunicorn==19.7.1 +Flask==1.0.2 +gunicorn==19.9.0 diff --git a/tests/eventlet/requirements.txt b/tests/eventlet/requirements.txt index 5d104d0a..7f8b2468 100644 --- a/tests/eventlet/requirements.txt +++ b/tests/eventlet/requirements.txt @@ -1,9 +1,9 @@ click==6.7 enum-compat==0.0.2 -eventlet==0.22.1 -Flask==0.12.2 -greenlet==0.4.13 -gunicorn==19.7.1 +eventlet==0.24.1 +Flask==1.0.2 +greenlet==0.4.14 +gunicorn==19.9.0 itsdangerous==0.24 Jinja2==2.10 MarkupSafe==1.0 diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index 64e39e0c..4a462433 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -1,8 +1,8 @@ -Flask==0.12.2 -google-cloud-error-reporting==0.29.1 +Flask==1.0.2 +google-cloud-error-reporting==0.30.0 google-cloud-logging==1.6.0 -google-cloud-monitoring==0.28.1 -gunicorn==19.7.1 -requests==2.18.4 +google-cloud-monitoring==0.30.1 +gunicorn==19.9.0 +requests==2.19.1 retrying==1.3.3 six==1.11.0 diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 1655846a..6daad0ab 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -1,31 +1,31 @@ -alembic==0.9.9 -amqp==2.2.2 +alembic==1.0.0 +amqp==2.3.2 amqplib==1.0.2 -ansible==2.5.0 +ansible==2.6.2 anyjson==0.3.3 apache-libcloud==2.3.0 argparse==1.4.0 -astroid==1.6.2 -awscli==1.14.68 -babel==2.5.3 +astroid==2.0.4 +awscli==1.15.76 +babel==2.6.0 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 -beautifulsoup4==4.6.0 +beautifulsoup4==4.6.3 beautifulsoup==3.2.1 -billiard==3.5.0.3 -blessings==1.6.1 +billiard==3.5.0.4 +blessings==1.7 blinker==1.4 -boto==2.48.0 -botocore==1.9.21 +boto==2.49.0 +botocore==1.10.75 bottle==0.12.13 carbon<1.1.1 -celery==4.1.0 -certifi==2018.1.18 +celery==4.2.1 +certifi==2018.8.13 cffi==1.11.5 chardet==3.0.4 click==6.7 -cliff==2.11.0 -cmd2==0.8.2 +cliff==2.13.0 +cmd2==0.9.3 colorama==0.3.9 configobj==5.0.6 cov-core==1.15.0 @@ -34,174 +34,174 @@ coveralls==1.3.0 crcmod==1.7 cryptography==2.3 cssselect==1.0.3 -cython==0.28.1 -decorator==4.2.1 +cython==0.28.5 +decorator==4.3.0 django-celery==3.2.2 django-debug-toolbar==1.9.1 -django-extensions==2.0.6 +django-extensions==2.1.0 django<2.0 django_compress==1.0.1 -djangorestframework==3.7.7 +djangorestframework==3.8.2 docker-py==1.10.6 docopt==0.6.2 docutils==0.14 ecdsa==0.13 -elasticsearch==6.2.0 +elasticsearch==6.3.1 enum34==1.1.6 -eventlet==0.22.1 +eventlet==0.24.1 extras==1.0.0 -fabric==1.14.0 +fabric==2.3.1 fixtures==3.0.0 flake8==3.5.0 -flask==0.12.2 +flask==1.0.2 funcsigs==1.0.2 functools32==3.2.3.post2 futures==3.2.0 -gevent==1.2.2 -google-api-python-client==1.6.6 -graphite-web==1.1.2 -greenlet==0.4.13 -gunicorn==19.7.1 +gevent==1.3.5 +google-api-python-client==1.7.4 +graphite-web==1.1.3 +greenlet==0.4.14 +gunicorn==19.9.0 hiredis==0.2.0 honcho==1.0.1 html5lib==1.0.1 httplib2==0.11.3 -idna==2.6 -ipaddress==1.0.19 +idna==2.7 +ipaddress==1.0.22 iso8601==0.1.12 isodate==0.6.0 itsdangerous==0.24 jinja2==2.10 jmespath==0.9.3 jsonschema==2.6.0 -kombu==4.1.0 +kombu==4.2.1 linecache2==1.0.0 -logilab-common==1.4.1 -lxml==4.2.1 -m2crypto==0.29.0 +logilab-common==1.4.2 +lxml==4.2.4 +m2crypto==0.30.1 mako==1.0.7 manifestparser==1.1 markdown==2.6.11 markupsafe==1.0 -matplotlib==2.2.2 +matplotlib==2.2.3 mccabe==0.6.1 meld3==1.0.2 mock==2.0.0 mozcrash==1.0 -mozdevice==0.52 +mozdevice==1.0.0 mozfile==1.2 mozinfo==0.10 -mozlog==3.7 +mozlog==3.8 moznetwork==0.27 mozprocess==0.26 -mozprofile==0.29 -mozrunner==6.14 +mozprofile==1.1.0 +mozrunner==7.0.1 msgpack-python==0.5.6 mysql-python==1.2.5 -ndg-httpsclient==0.4.4 +ndg-httpsclient==0.5.1 netaddr==0.7.19 -netifaces==0.10.6 -newrelic==3.0.0.89 +netifaces==0.10.7 +newrelic==4.2.0.100 nose==1.3.7 -numpy==1.14.2 +numpy==1.15.0 oauth2==1.9.0.post1 oauth2client==4.1.2 -oauthlib==2.0.7 +oauthlib==2.1.0 ordereddict==1.1 -oslo.config==6.0.0 -pandas==0.22.0 +oslo.config==6.4.0 +pandas==0.23.4 paramiko==2.4.1 passlib==1.7.1 paste==2.0.3 pastedeploy==1.5.2 pastescript==2.0.2 -pbr==4.0.0 +pbr==4.2.0 pep8==1.7.1 -pexpect==4.4.0 -pika==0.11.2 -pillow==5.0.0 -pip==9.0.3 -prettytable -protobuf==3.5.2.post1 -psutil==5.4.3 -psycopg2==2.7.4 -py==1.5.3 -pyasn1-modules==0.2.1 -pyasn1==0.4.2 +pexpect==4.6.0 +pika==0.12.0 +pillow==5.2.0 +pip==18.0 +prettytable==7 +protobuf==3.6.0 +psutil==5.4.6 +psycopg2==2.7.5 +py==1.5.4 +pyasn1-modules==0.2.2 +pyasn1==0.4.4 pycparser==2.18 pycrypto==2.6.1 -pycurl==7.43.0.1 -pyflakes==1.6.0 +pycurl==7.43.0.2 +pyflakes==2.0.0 pygments==2.2.0 -pyjwt==1.6.1 +pyjwt==1.6.4 pylibmc==1.5.2 -pylint==1.8.3 -pymongo==3.6.1 -pymysql==0.8.0 -pyopenssl==17.5.0 +pylint==2.1.1 +pymongo==3.7.1 +pymysql==0.9.2 +pyopenssl==18.0.0 pyparsing==2.2.0 -pyramid==1.9.1 +pyramid==1.9.2 pystache==0.5.4 pytest-cov==2.5.1 -pytest==3.5.0 +pytest==3.7.1 python-cjson==1.2.1 python-daemon==2.1.2 -python-dateutil==2.7.2 +python-dateutil==2.7.3 python-gflags==3.1.2 -python-keystoneclient==3.15.0 +python-keystoneclient==3.17.0 python-memcached==1.59 python-mimeparse==1.6.0 -python-novaclient==10.1.0 -python-subunit==1.2.0 -python-swiftclient==3.5.0 -pytz==2018.3 -pyyaml==3.12 -pyzmq==17.0.0 -raven==6.6.0 +python-novaclient==11.0.0 +python-subunit==1.3.0 +python-swiftclient==3.6.0 +pytz==2018.5 +pyyaml==3.13 +pyzmq==17.1.2 +raven==6.9.0 redis==2.10.6 repoze.lru==0.7 -requests-oauthlib==0.8.0 -requests==2.18.4 +requests-oauthlib==1.0.0 +requests==2.19.1 retrying==1.3.3 rsa==3.4.2 -scipy==1.0.1 -selenium==3.11.0 +scipy==1.1.0 +selenium==3.14.0 setuptools-git==1.2 -setuptools==39.0.1 +setuptools==40.0.0 sh==1.12.14 -simplejson==3.13.2 +simplejson==3.16.0 six==1.11.0 snowballstemmer==1.2.1 south==1.0.2 -sphinx==1.7.2 +sphinx==1.7.6 sqlalchemy-migrate==0.11.0 -sqlalchemy==1.2.6 +sqlalchemy==1.2.10 sqlparse==0.2.4 statsd==3.2.2 -stevedore==1.28.0 +stevedore==1.29.0 suds==0.4 supervisor==3.3.4 testrepository==0.0.20 testtools==2.3.0 thrift==0.11.0 -tornado==5.0.1 -tox==2.9.1 -twisted==17.9.0 +tornado==5.1 +tox==3.2.1 +twisted==18.7.0 ujson==1.35 unidecode==1.0.22 unittest2==1.1.0 uritemplate==3.0.0 -urllib3==1.22 -uwsgi==2.0.17 +urllib3==1.23 +uwsgi==2.0.17.1 versiontools==1.9.1 -virtualenv==15.2.0 +virtualenv==16.0.0 waitress==1.1.0 warlock==1.3.0 -webob==1.7.4 -websocket-client==0.47.0 -webtest==2.0.29 +webob==1.8.2 +websocket-client==0.48.0 +webtest==2.0.30 werkzeug==0.14.1 -wheel==0.31.0 +wheel==0.31.1 xlrd==1.1.0 -zc.buildout==2.11.2 -zope.interface==4.4.3 +zc.buildout==2.12.1 +zope.interface==4.5.0 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index e8295d80..7b839959 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -1,29 +1,29 @@ -alembic==0.9.9 -amqp==2.2.2 +alembic==1.0.0 +amqp==2.3.2 amqplib==1.0.2 -ansible==2.5.0 +ansible==2.6.2 anyjson==0.3.3 apache-libcloud==2.3.0 argparse==1.4.0 -astroid==1.6.2 -awscli==1.14.68 -babel==2.5.3 +astroid==2.0.4 +awscli==1.15.76 +babel==2.6.0 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 -beautifulsoup4==4.6.0 -billiard==3.5.0.3 -blessings==1.6.1 +beautifulsoup4==4.6.3 +billiard==3.5.0.4 +blessings==1.7 blinker==1.4 -boto==2.48.0 -botocore==1.9.21 +boto==2.49.0 +botocore==1.10.75 bottle==0.12.13 -celery==4.1.0 -certifi==2018.1.18 +celery==4.2.1 +certifi==2018.8.13 cffi==1.11.5 chardet==3.0.4 click==6.7 -cliff==2.11.0 -cmd2==0.8.2 +cliff==2.13.0 +cmd2==0.9.3 colorama==0.3.9 configobj==5.0.6 cov-core==1.15.0 @@ -32,164 +32,164 @@ coveralls==1.3.0 crcmod==1.7 cryptography==2.3 cssselect==1.0.3 -cython==0.28.1 -decorator==4.2.1 +cython==0.28.5 +decorator==4.3.0 django-celery==3.2.2 django-debug-toolbar==1.9.1 -django-extensions==2.0.6 -django==2.0.3 +django-extensions==2.1.0 +django==2.1 django_compress==1.0.1 -djangorestframework==3.7.7 +djangorestframework==3.8.2 docker-py==1.10.6 docopt==0.6.2 docutils==0.14 ecdsa==0.13 -elasticsearch==6.2.0 +elasticsearch==6.3.1 enum34==1.1.6 -eventlet==0.22.1 +eventlet==0.24.1 extras==1.0.0 -fabric==1.14.0 +fabric==2.3.1 fixtures==3.0.0 flake8==3.5.0 -flask==0.12.2 +flask==1.0.2 funcsigs==1.0.2 -gevent==1.2.2 -google-api-python-client==1.6.6 -greenlet==0.4.13 -gunicorn==19.7.1 +gevent==1.3.5 +google-api-python-client==1.7.4 +greenlet==0.4.14 +gunicorn==19.9.0 hiredis==0.2.0 honcho==1.0.1 html5lib==1.0.1 httplib2==0.11.3 -idna==2.6 -ipaddress==1.0.19 -ipython==6.2.1 +idna==2.7 +ipaddress==1.0.22 +ipython==6.5.0 iso8601==0.1.12 isodate==0.6.0 itsdangerous==0.24 jinja2==2.10 jmespath==0.9.3 jsonschema==2.6.0 -kombu==4.1.0 +kombu==4.2.1 linecache2==1.0.0 -logilab-common==1.4.1 -lxml==4.2.1 +logilab-common==1.4.2 +lxml==4.2.4 mako==1.0.7 manifestparser==1.1 markdown==2.6.11 markupsafe==1.0 -matplotlib==2.2.2 +matplotlib==2.2.3 mccabe==0.6.1 meld3==1.0.2 mock==2.0.0 mozcrash==1.0 -mozdevice==0.52 +mozdevice==1.0.0 mozfile==1.2 mozinfo==0.10 -mozlog==3.7 +mozlog==3.8 moznetwork==0.27 mozprocess==0.26 msgpack-python==0.5.6 -ndg-httpsclient==0.4.4 +ndg-httpsclient==0.5.1 netaddr==0.7.19 -netifaces==0.10.6 -newrelic==3.0.0.89 +netifaces==0.10.7 +newrelic==4.2.0.100 nose==1.3.7 -numpy==1.14.2 +numpy==1.15.0 oauth2==1.9.0.post1 oauth2client==4.1.2 -oauthlib==2.0.7 +oauthlib==2.1.0 ordereddict==1.1 -oslo.config==6.0.0 -pandas==0.22.0 +oslo.config==6.4.0 +pandas==0.23.4 paramiko==2.4.1 passlib==1.7.1 paste==2.0.3 pastedeploy==1.5.2 pastescript==2.0.2 -pbr==4.0.0 +pbr==4.2.0 pep8==1.7.1 -pexpect==4.4.0 -pika==0.11.2 -pillow==5.0.0 -pip==9.0.3 -prettytable -protobuf==3.5.2.post1 -psutil==5.4.3 -psycopg2==2.7.4 -py==1.5.3 -pyasn1-modules==0.2.1 -pyasn1==0.4.2 +pexpect==4.6.0 +pika==0.12.0 +pillow==5.2.0 +pip==18.0 +prettytable==7 +protobuf==3.6.0 +psutil==5.4.6 +psycopg2==2.7.5 +py==1.5.4 +pyasn1-modules==0.2.2 +pyasn1==0.4.4 pycparser==2.18 pycrypto==2.6.1 -pyflakes==1.6.0 +pyflakes==2.0.0 pygments==2.2.0 -pyjwt==1.6.1 +pyjwt==1.6.4 pylibmc==1.5.2 -pylint==1.8.3 -pymongo==3.6.1 -pymysql==0.8.0 -pyopenssl==17.5.0 +pylint==2.1.1 +pymongo==3.7.1 +pymysql==0.9.2 +pyopenssl==18.0.0 pyparsing==2.2.0 -pyramid==1.9.1 +pyramid==1.9.2 pystache==0.5.4 pytest-cov==2.5.1 -pytest==3.5.0 +pytest==3.7.1 python-daemon==2.1.2 -python-dateutil==2.7.2 +python-dateutil==2.7.3 python-gflags==3.1.2 -python-keystoneclient==3.15.0 +python-keystoneclient==3.17.0 python-memcached==1.59 python-mimeparse==1.6.0 -python-novaclient==10.1.0 -python-subunit==1.2.0 -python-swiftclient==3.5.0 -pytz==2018.3 -pyyaml==3.12 -pyzmq==17.0.0 -raven==6.6.0 +python-novaclient==11.0.0 +python-subunit==1.3.0 +python-swiftclient==3.6.0 +pytz==2018.5 +pyyaml==3.13 +pyzmq==17.1.2 +raven==6.9.0 redis==2.10.6 repoze.lru==0.7 -requests-oauthlib==0.8.0 -requests==2.18.4 +requests-oauthlib==1.0.0 +requests==2.19.1 retrying==1.3.3 rsa==3.4.2 -scipy==1.0.1 -selenium==3.11.0 +scipy==1.1.0 +selenium==3.14.0 setuptools-git==1.2 -setuptools==39.0.1 +setuptools==40.0.0 sh==1.12.14 -simplejson==3.13.2 +simplejson==3.16.0 six==1.11.0 snowballstemmer==1.2.1 south==1.0.2 -sphinx==1.7.2 +sphinx==1.7.6 sqlalchemy-migrate==0.11.0 -sqlalchemy==1.2.6 +sqlalchemy==1.2.10 sqlparse==0.2.4 statsd==3.2.2 -stevedore==1.28.0 +stevedore==1.29.0 testrepository==0.0.20 testtools==2.3.0 thrift==0.11.0 -tornado==5.0.1 -tox==2.9.1 -twisted==17.9.0 +tornado==5.1 +tox==3.2.1 +twisted==18.7.0 ujson==1.35 unidecode==1.0.22 unittest2==1.1.0 uritemplate==3.0.0 -urllib3==1.22 -uwsgi==2.0.17 +urllib3==1.23 +uwsgi==2.0.17.1 versiontools==1.9.1 -virtualenv==15.2.0 +virtualenv==16.0.0 waitress==1.1.0 warlock==1.3.0 -webob==1.7.4 -websocket-client==0.47.0 -webtest==2.0.29 +webob==1.8.2 +websocket-client==0.48.0 +webtest==2.0.30 werkzeug==0.14.1 -wheel==0.31.0 +wheel==0.31.1 xlrd==1.1.0 -zc.buildout==2.11.2 -zope.interface==4.4.3 +zc.buildout==2.12.1 +zope.interface==4.5.0 From 721201e707efbba87005c974e6b8cbabf1e17010 Mon Sep 17 00:00:00 2001 From: David Bendory Date: Sun, 19 Aug 2018 16:21:41 -0400 Subject: [PATCH 051/132] Container Builder is now Cloud Build. --- RELEASING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/RELEASING.md b/RELEASING.md index 2dcd806d..f886a542 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -36,7 +36,7 @@ follows: b. `gcloud auth activate-service-account` is performed c. `gcloud config set project` is performed 3. The script invokes `build.sh` in this repository -4. `build.sh` invokes Google Container Builder with the `cloudbuild-*.yaml` +4. `build.sh` invokes Google Cloud Build with the `cloudbuild-*.yaml` config files. ## Building interpreters @@ -65,7 +65,7 @@ Debian or Ubuntu-like). ## Building locally To build this repository using local Docker commands instead of the Google -Container Builder service, add the `--local` flag as shown: +Cloud Build service, add the `--local` flag as shown: ``` shell ./build.sh --local From 56c9654a17f0662acffe4fb06a329cbce16e1829 Mon Sep 17 00:00:00 2001 From: David Bendory Date: Sun, 19 Aug 2018 16:25:27 -0400 Subject: [PATCH 052/132] Container Builder is now Cloud Build. --- scripts/local_cloudbuild.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/local_cloudbuild.py b/scripts/local_cloudbuild.py index f80e0f4b..5c23a1bc 100755 --- a/scripts/local_cloudbuild.py +++ b/scripts/local_cloudbuild.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Emulate the Google Container Builder locally. +"""Emulate the Google Cloud Build locally. The input is a local cloudbuild.yaml file. This is translated into a series of commands for the locally installed Docker daemon. These @@ -48,8 +48,8 @@ # Exclude non-printable control characters (including newlines) PRINTABLE_REGEX = re.compile(r"""^[^\x00-\x1f]*$""") -# Container Builder substitutions -# https://cloud.google.com/container-builder/docs/api/build-requests#substitutions +# Cloud Build substitutions +# https://cloud.google.com/cloud-build/docs/api/build-requests#substitutions SUBSTITUTION_REGEX = re.compile(r"""(?x) [$] # Dollar sign ( From 8369dc90a267a7fa933056393322b976f4d44419 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Mon, 27 Aug 2018 00:50:51 -0700 Subject: [PATCH 053/132] Auto-update dependencies. --- runtime-image/resources/requirements.txt | 2 +- scripts/requirements-test.txt | 2 +- tests/python2-libraries/requirements.txt | 40 ++++++++++++------------ tests/python3-libraries/requirements.txt | 40 ++++++++++++------------ 4 files changed, 42 insertions(+), 42 deletions(-) diff --git a/runtime-image/resources/requirements.txt b/runtime-image/resources/requirements.txt index 44c624d8..bc783013 100644 --- a/runtime-image/resources/requirements.txt +++ b/runtime-image/resources/requirements.txt @@ -1,3 +1,3 @@ pip==18.0 -setuptools==40.0.0 +setuptools==40.2.0 wheel==0.31.1 diff --git a/scripts/requirements-test.txt b/scripts/requirements-test.txt index df909d47..c8e698da 100644 --- a/scripts/requirements-test.txt +++ b/scripts/requirements-test.txt @@ -1,4 +1,4 @@ flask==1.0.2 -pytest==3.7.1 +pytest==3.7.3 pytest-cov==2.5.1 pyyaml==3.13 diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 6daad0ab..a319137a 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -1,12 +1,12 @@ alembic==1.0.0 amqp==2.3.2 amqplib==1.0.2 -ansible==2.6.2 +ansible==2.6.3 anyjson==0.3.3 apache-libcloud==2.3.0 argparse==1.4.0 astroid==2.0.4 -awscli==1.15.76 +awscli==1.16.1 babel==2.6.0 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 @@ -16,29 +16,29 @@ billiard==3.5.0.4 blessings==1.7 blinker==1.4 boto==2.49.0 -botocore==1.10.75 +botocore==1.11.1 bottle==0.12.13 carbon<1.1.1 celery==4.2.1 -certifi==2018.8.13 +certifi==2018.8.24 cffi==1.11.5 chardet==3.0.4 click==6.7 cliff==2.13.0 -cmd2==0.9.3 +cmd2==0.9.4 colorama==0.3.9 configobj==5.0.6 cov-core==1.15.0 coverage==4.5.1 -coveralls==1.3.0 +coveralls==1.4.0 crcmod==1.7 -cryptography==2.3 +cryptography==2.3.1 cssselect==1.0.3 cython==0.28.5 decorator==4.3.0 django-celery==3.2.2 django-debug-toolbar==1.9.1 -django-extensions==2.1.0 +django-extensions==2.1.1 django<2.0 django_compress==1.0.1 djangorestframework==3.8.2 @@ -57,7 +57,7 @@ flask==1.0.2 funcsigs==1.0.2 functools32==3.2.3.post2 futures==3.2.0 -gevent==1.3.5 +gevent==1.3.6 google-api-python-client==1.7.4 graphite-web==1.1.3 greenlet==0.4.14 @@ -88,7 +88,7 @@ mccabe==0.6.1 meld3==1.0.2 mock==2.0.0 mozcrash==1.0 -mozdevice==1.0.0 +mozdevice==1.0.1 mozfile==1.2 mozinfo==0.10 mozlog==3.8 @@ -103,7 +103,7 @@ netaddr==0.7.19 netifaces==0.10.7 newrelic==4.2.0.100 nose==1.3.7 -numpy==1.15.0 +numpy==1.15.1 oauth2==1.9.0.post1 oauth2client==4.1.2 oauthlib==2.1.0 @@ -122,8 +122,8 @@ pika==0.12.0 pillow==5.2.0 pip==18.0 prettytable==7 -protobuf==3.6.0 -psutil==5.4.6 +protobuf==3.6.1 +psutil==5.4.7 psycopg2==2.7.5 py==1.5.4 pyasn1-modules==0.2.2 @@ -143,9 +143,9 @@ pyparsing==2.2.0 pyramid==1.9.2 pystache==0.5.4 pytest-cov==2.5.1 -pytest==3.7.1 +pytest==3.7.3 python-cjson==1.2.1 -python-daemon==2.1.2 +python-daemon==2.2.0 python-dateutil==2.7.3 python-gflags==3.1.2 python-keystoneclient==3.17.0 @@ -167,17 +167,17 @@ rsa==3.4.2 scipy==1.1.0 selenium==3.14.0 setuptools-git==1.2 -setuptools==40.0.0 +setuptools==40.2.0 sh==1.12.14 simplejson==3.16.0 six==1.11.0 snowballstemmer==1.2.1 south==1.0.2 -sphinx==1.7.6 +sphinx==1.7.7 sqlalchemy-migrate==0.11.0 -sqlalchemy==1.2.10 +sqlalchemy==1.2.11 sqlparse==0.2.4 -statsd==3.2.2 +statsd==3.3.0 stevedore==1.29.0 suds==0.4 supervisor==3.3.4 @@ -198,7 +198,7 @@ virtualenv==16.0.0 waitress==1.1.0 warlock==1.3.0 webob==1.8.2 -websocket-client==0.48.0 +websocket-client==0.51.0 webtest==2.0.30 werkzeug==0.14.1 wheel==0.31.1 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 7b839959..f9f1b8f7 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -1,12 +1,12 @@ alembic==1.0.0 amqp==2.3.2 amqplib==1.0.2 -ansible==2.6.2 +ansible==2.6.3 anyjson==0.3.3 apache-libcloud==2.3.0 argparse==1.4.0 astroid==2.0.4 -awscli==1.15.76 +awscli==1.16.1 babel==2.6.0 backports.ssl_match_hostname==3.5.0.1 bcdoc==0.16.0 @@ -15,28 +15,28 @@ billiard==3.5.0.4 blessings==1.7 blinker==1.4 boto==2.49.0 -botocore==1.10.75 +botocore==1.11.1 bottle==0.12.13 celery==4.2.1 -certifi==2018.8.13 +certifi==2018.8.24 cffi==1.11.5 chardet==3.0.4 click==6.7 cliff==2.13.0 -cmd2==0.9.3 +cmd2==0.9.4 colorama==0.3.9 configobj==5.0.6 cov-core==1.15.0 coverage==4.5.1 -coveralls==1.3.0 +coveralls==1.4.0 crcmod==1.7 -cryptography==2.3 +cryptography==2.3.1 cssselect==1.0.3 cython==0.28.5 decorator==4.3.0 django-celery==3.2.2 django-debug-toolbar==1.9.1 -django-extensions==2.1.0 +django-extensions==2.1.1 django==2.1 django_compress==1.0.1 djangorestframework==3.8.2 @@ -53,7 +53,7 @@ fixtures==3.0.0 flake8==3.5.0 flask==1.0.2 funcsigs==1.0.2 -gevent==1.3.5 +gevent==1.3.6 google-api-python-client==1.7.4 greenlet==0.4.14 gunicorn==19.9.0 @@ -83,7 +83,7 @@ mccabe==0.6.1 meld3==1.0.2 mock==2.0.0 mozcrash==1.0 -mozdevice==1.0.0 +mozdevice==1.0.1 mozfile==1.2 mozinfo==0.10 mozlog==3.8 @@ -95,7 +95,7 @@ netaddr==0.7.19 netifaces==0.10.7 newrelic==4.2.0.100 nose==1.3.7 -numpy==1.15.0 +numpy==1.15.1 oauth2==1.9.0.post1 oauth2client==4.1.2 oauthlib==2.1.0 @@ -114,8 +114,8 @@ pika==0.12.0 pillow==5.2.0 pip==18.0 prettytable==7 -protobuf==3.6.0 -psutil==5.4.6 +protobuf==3.6.1 +psutil==5.4.7 psycopg2==2.7.5 py==1.5.4 pyasn1-modules==0.2.2 @@ -134,8 +134,8 @@ pyparsing==2.2.0 pyramid==1.9.2 pystache==0.5.4 pytest-cov==2.5.1 -pytest==3.7.1 -python-daemon==2.1.2 +pytest==3.7.3 +python-daemon==2.2.0 python-dateutil==2.7.3 python-gflags==3.1.2 python-keystoneclient==3.17.0 @@ -157,17 +157,17 @@ rsa==3.4.2 scipy==1.1.0 selenium==3.14.0 setuptools-git==1.2 -setuptools==40.0.0 +setuptools==40.2.0 sh==1.12.14 simplejson==3.16.0 six==1.11.0 snowballstemmer==1.2.1 south==1.0.2 -sphinx==1.7.6 +sphinx==1.7.7 sqlalchemy-migrate==0.11.0 -sqlalchemy==1.2.10 +sqlalchemy==1.2.11 sqlparse==0.2.4 -statsd==3.2.2 +statsd==3.3.0 stevedore==1.29.0 testrepository==0.0.20 testtools==2.3.0 @@ -186,7 +186,7 @@ virtualenv==16.0.0 waitress==1.1.0 warlock==1.3.0 webob==1.8.2 -websocket-client==0.48.0 +websocket-client==0.51.0 webtest==2.0.30 werkzeug==0.14.1 wheel==0.31.1 From a4df99416aa22b571fb6f797e64d68c821bb6131 Mon Sep 17 00:00:00 2001 From: dlorenc Date: Mon, 1 Oct 2018 08:08:58 -0700 Subject: [PATCH 054/132] Pin pip to 9.0.3. Newer versions conflict with the OS installed package. --- runtime-image/resources/requirements.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/runtime-image/resources/requirements.txt b/runtime-image/resources/requirements.txt index bc783013..298e5f41 100644 --- a/runtime-image/resources/requirements.txt +++ b/runtime-image/resources/requirements.txt @@ -1,3 +1,5 @@ -pip==18.0 +# Do not update pip, it conflicts with the OS system installed version. +# https://github.com/pypa/pip/issues/5240 +pip==9.0.3 setuptools==40.2.0 wheel==0.31.1 From 7eb071f7f7526815fc4b90a5ed0225f1247eef84 Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Mon, 1 Oct 2018 14:36:10 -0700 Subject: [PATCH 055/132] fix library versions for tests (#202) --- tests/python2-libraries/requirements.txt | 8 ++++---- tests/python3-libraries/requirements.txt | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index a319137a..865cd1aa 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -5,7 +5,7 @@ ansible==2.6.3 anyjson==0.3.3 apache-libcloud==2.3.0 argparse==1.4.0 -astroid==2.0.4 +astroid==1.6.5 awscli==1.16.1 babel==2.6.0 backports.ssl_match_hostname==3.5.0.1 @@ -25,7 +25,7 @@ cffi==1.11.5 chardet==3.0.4 click==6.7 cliff==2.13.0 -cmd2==0.9.4 +cmd2==0.8.9 colorama==0.3.9 configobj==5.0.6 cov-core==1.15.0 @@ -121,7 +121,7 @@ pexpect==4.6.0 pika==0.12.0 pillow==5.2.0 pip==18.0 -prettytable==7 +prettytable==0.7.2 protobuf==3.6.1 psutil==5.4.7 psycopg2==2.7.5 @@ -135,7 +135,7 @@ pyflakes==2.0.0 pygments==2.2.0 pyjwt==1.6.4 pylibmc==1.5.2 -pylint==2.1.1 +pylint==1.9.3 pymongo==3.7.1 pymysql==0.9.2 pyopenssl==18.0.0 diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index f9f1b8f7..3b283d1d 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -113,7 +113,7 @@ pexpect==4.6.0 pika==0.12.0 pillow==5.2.0 pip==18.0 -prettytable==7 +prettytable==0.7.2 protobuf==3.6.1 psutil==5.4.7 psycopg2==2.7.5 From 5bc7a6e49e591cb4771dbd9acdeddb7c3bc41a5d Mon Sep 17 00:00:00 2001 From: dlorenc Date: Fri, 2 Nov 2018 14:43:11 -0700 Subject: [PATCH 056/132] Bump Python 3.6.6 to 3.6.7. --- .../scripts/build-python-3.6.sh | 12 ++++++------ tests/no-virtualenv/no-virtualenv.yaml | 2 +- tests/virtualenv/virtualenv_python36.yaml | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/python-interpreter-builder/scripts/build-python-3.6.sh b/python-interpreter-builder/scripts/build-python-3.6.sh index ef192e1e..b3896e89 100755 --- a/python-interpreter-builder/scripts/build-python-3.6.sh +++ b/python-interpreter-builder/scripts/build-python-3.6.sh @@ -6,14 +6,14 @@ set -x # Get the source mkdir -p /opt/sources cd /opt/sources -wget --no-verbose https://www.python.org/ftp/python/3.6.6/Python-3.6.6.tgz +wget --no-verbose https://www.python.org/ftp/python/3.6.7/Python-3.6.7.tgz # SHA-256 generated via `shasum -a 256 [file]` shasum --check < Date: Fri, 2 Nov 2018 14:36:55 -0700 Subject: [PATCH 057/132] Add support for Python 3.7. --- cloudbuild_interpreters.yaml | 6 +- nox.py | 2 +- .../scripts/build-python-3.7.sh | 153 ++++++++++++++++++ runtime-image/Dockerfile.in | 4 + scripts/gen_dockerfile.py | 1 + 5 files changed, 164 insertions(+), 2 deletions(-) create mode 100755 python-interpreter-builder/scripts/build-python-3.7.sh diff --git a/cloudbuild_interpreters.yaml b/cloudbuild_interpreters.yaml index 7daecc57..c75e59ad 100644 --- a/cloudbuild_interpreters.yaml +++ b/cloudbuild_interpreters.yaml @@ -18,11 +18,15 @@ steps: args: ['/scripts/build-python-3.6.sh'] id: build-3.6 waitFor: ['interpreter-builder'] +- name: interpreter-builder + args: ['/scripts/build-python-3.7.sh'] + id: build-3.7 + waitFor: ['interpreter-builder'] # Upload them to tbe build-id location - name: gcr.io/cloud-builders/gsutil:latest args: ['cp', '/workspace/runtime-image/*.tar.gz', 'gs://python-interpreters/$BUILD_ID/'] - waitFor: ['build-3.4', 'build-3.5', 'build-3.6'] + waitFor: ['build-3.4', 'build-3.5', 'build-3.6', 'build-3.7'] # "Tag" this as latest - name: gcr.io/cloud-builders/gsutil:latest diff --git a/nox.py b/nox.py index 872eb88c..0ee6f443 100644 --- a/nox.py +++ b/nox.py @@ -57,7 +57,7 @@ def lint(session): @nox.session -@nox.parametrize('version', ['3.4', '3.5', '3.6']) +@nox.parametrize('version', ['3.4', '3.5', '3.6', '3.7']) def tests(session, version): session.interpreter = 'python' + version session.install('-r', 'scripts/requirements-test.txt') diff --git a/python-interpreter-builder/scripts/build-python-3.7.sh b/python-interpreter-builder/scripts/build-python-3.7.sh new file mode 100755 index 00000000..2a77e279 --- /dev/null +++ b/python-interpreter-builder/scripts/build-python-3.7.sh @@ -0,0 +1,153 @@ +#!/bin/bash + +set -euo pipefail +set -x + +# Get the source +mkdir -p /opt/sources +cd /opt/sources +wget --no-verbose https://www.python.org/ftp/python/3.7.1/Python-3.7.1.tgz +# SHA-256 generated via `shasum -a 256 [file]` +shasum --check < Date: Wed, 14 Nov 2018 14:44:56 -0800 Subject: [PATCH 058/132] Fixing up CODEOWNERS (#207) --- CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CODEOWNERS b/CODEOWNERS index c08c2d11..f8b549f9 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1,4 +1,4 @@ # Code owners file. # This file controls who is tagged for review for any given pull request. -* @duggelz @liyanhui1228 @dlorenc +* @dlorenc @sharifelgamal @nkubala @tstromberg From 608b42af72537ce5614f1066db8b992a33af1199 Mon Sep 17 00:00:00 2001 From: dlorenc Date: Tue, 27 Nov 2018 11:40:09 -0800 Subject: [PATCH 059/132] Fix the Python 3.7 runtime release script. --- runtime-image/Dockerfile.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime-image/Dockerfile.in b/runtime-image/Dockerfile.in index b872a501..87d1aff2 100644 --- a/runtime-image/Dockerfile.in +++ b/runtime-image/Dockerfile.in @@ -40,7 +40,7 @@ RUN /usr/bin/pip install --upgrade -r /resources/requirements.txt && \ rm -f /opt/python3.5/bin/pip /opt/python3.5/bin/pip3 && \ /opt/python3.6/bin/pip3.6 install --upgrade -r /resources/requirements.txt && \ rm -f /opt/python3.6/bin/pip /opt/python3.6/bin/pip3 && \ - /opt/python3.6/bin/pip3.7 install --upgrade -r /resources/requirements.txt && \ + /opt/python3.7/bin/pip3.7 install --upgrade -r /resources/requirements.txt && \ rm -f /opt/python3.7/bin/pip /opt/python3.7/bin/pip3 && \ /usr/bin/pip install --upgrade -r /resources/requirements-virtualenv.txt From 7be6af7551dfc489535404ca5c09ca2ccc6d0641 Mon Sep 17 00:00:00 2001 From: dlorenc Date: Tue, 4 Dec 2018 09:10:55 -0800 Subject: [PATCH 060/132] Add Python 3.7 to the path. --- runtime-image/Dockerfile.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime-image/Dockerfile.in b/runtime-image/Dockerfile.in index 87d1aff2..64201966 100644 --- a/runtime-image/Dockerfile.in +++ b/runtime-image/Dockerfile.in @@ -27,7 +27,7 @@ RUN wget https://storage.googleapis.com/python-interpreters/latest/interpreter-3 rm interpreter-*.tar.gz # Add Google-built interpreters to the path -ENV PATH /opt/python3.6/bin:/opt/python3.5/bin:/opt/python3.4/bin:$PATH +ENV PATH /opt/python3.7/bin:/opt/python3.6/bin:/opt/python3.5/bin:/opt/python3.4/bin:$PATH RUN update-alternatives --install /usr/local/bin/python3 python3 /opt/python3.6/bin/python3.6 50 RUN update-alternatives --install /usr/local/bin/pip3 pip3 /opt/python3.6/bin/pip3.6 50 From 0e07aba8b47e3fa64d82e0a2a3ea943bb60027f4 Mon Sep 17 00:00:00 2001 From: Evan Anderson Date: Tue, 15 Jan 2019 11:44:16 -0800 Subject: [PATCH 061/132] Document how to use Dockerfile with python3 --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 2cc7d173..51413044 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,7 @@ command or entrypoint. For example: # Create a virtualenv for dependencies. This isolates these packages from # system-level packages. + # Use -p python3 or -p python3.7 to select python version. Default is version 2. RUN virtualenv /env # Setting these environment variables are the same as running From db4ae4216b1783ab6ddc96ce6500d7257eb37ef8 Mon Sep 17 00:00:00 2001 From: sharifelgamal Date: Wed, 2 Jan 2019 17:08:02 -0800 Subject: [PATCH 062/132] Updating python to 3.6.8 and 3.7.2 and defaulting to 3.7 --- .../scripts/build-python-3.6.sh | 12 ++--- .../scripts/build-python-3.7.sh | 12 ++--- scripts/gen_dockerfile.py | 2 +- scripts/gen_dockerfile_test.py | 5 +- .../testdata/hello_world_golden/Dockerfile | 4 +- tests/virtualenv/virtualenv_python36.yaml | 2 +- tests/virtualenv/virtualenv_python37.yaml | 54 +++++++++++++++++++ 7 files changed, 74 insertions(+), 17 deletions(-) create mode 100644 tests/virtualenv/virtualenv_python37.yaml diff --git a/python-interpreter-builder/scripts/build-python-3.6.sh b/python-interpreter-builder/scripts/build-python-3.6.sh index b3896e89..73912401 100755 --- a/python-interpreter-builder/scripts/build-python-3.6.sh +++ b/python-interpreter-builder/scripts/build-python-3.6.sh @@ -6,14 +6,14 @@ set -x # Get the source mkdir -p /opt/sources cd /opt/sources -wget --no-verbose https://www.python.org/ftp/python/3.6.7/Python-3.6.7.tgz +wget --no-verbose https://www.python.org/ftp/python/3.6.8/Python-3.6.8.tgz # SHA-256 generated via `shasum -a 256 [file]` shasum --check < Date: Wed, 2 Jan 2019 17:09:16 -0800 Subject: [PATCH 063/132] fixing 3.7 virtualenv tests --- tests/virtualenv/virtualenv_python37.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/virtualenv/virtualenv_python37.yaml b/tests/virtualenv/virtualenv_python37.yaml index f75ffdcd..457933f9 100644 --- a/tests/virtualenv/virtualenv_python37.yaml +++ b/tests/virtualenv/virtualenv_python37.yaml @@ -7,48 +7,48 @@ globalEnvVars: value: "/env/bin:$PATH" commandTests: - - name: "virtualenv36 python installation" + - name: "virtualenv37 python installation" setup: [["virtualenv", "-p", "python3.7", "/env"]] command: ["which", "python"] expectedOutput: ["/env/bin/python\n"] - - name: "virtualenv36 python3 installation" + - name: "virtualenv37 python3 installation" setup: [["virtualenv", "-p", "python3.7", "/env"]] command: ["which", "python3"] expectedOutput: ["/env/bin/python3\n"] - - name: "virtualenv36 python3.7 installation" + - name: "virtualenv37 python3.7 installation" setup: [["virtualenv", "-p", "python3.7", "/env"]] command: ["which", "python3.7"] expectedOutput: ["/env/bin/python3.7\n"] - - name: "virtualenv36 python version" + - name: "virtualenv37 python version" setup: [["virtualenv", "-p", "python3.7", "/env"]] command: ["python", "--version"] expectedOutput: ["Python 3.7.2\n"] - - name: "virtualenv36 pip installation" + - name: "virtualenv37 pip installation" setup: [["virtualenv", "-p", "python3.7", "/env"]] command: ["which", "pip"] expectedOutput: ["/env/bin/pip\n"] - - name: "virtualenv36 pip3 installation" + - name: "virtualenv37 pip3 installation" setup: [["virtualenv", "-p", "python3.7", "/env"]] command: ["which", "pip3"] expectedOutput: ["/env/bin/pip3\n"] - - name: "virtualenv36 gunicorn installation" + - name: "virtualenv37 gunicorn installation" setup: [["virtualenv", "-p", "python3.7", "/env"], ["pip", "install", "gunicorn"]] command: ["which", "gunicorn"] expectedOutput: ["/env/bin/gunicorn"] - - name: "virtualenv36 flask installation" + - name: "virtualenv37 flask installation" setup: [["virtualenv", "-p", "python3.7", "/env"], ["pip", "install", "flask"]] command: ["python", "-c", "import flask; print(flask.__file__)"] expectedOutput: ["/env/lib/python3.7/site-packages/flask"] - - name: "virtualenv36 test.support availability" + - name: "virtualenv37 test.support availability" setup: [["virtualenv", "-p", "python3.7", "/env"]] command: ["python", "-c", "\"from test import pystone, regrtest, support\""] From a90395dfeebc6b095eaa59f2c38eb392c471487d Mon Sep 17 00:00:00 2001 From: sharifelgamal Date: Fri, 4 Jan 2019 16:35:38 -0800 Subject: [PATCH 064/132] fixing up defaulting to 3.7 --- builder/gen-dockerfile/Dockerfile.in | 4 ++-- cloudbuild_test.yaml | 7 +++++++ runtime-image/Dockerfile.in | 4 ++-- tests/no-virtualenv/no-virtualenv.yaml | 2 +- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/builder/gen-dockerfile/Dockerfile.in b/builder/gen-dockerfile/Dockerfile.in index 4f6447eb..627151bf 100644 --- a/builder/gen-dockerfile/Dockerfile.in +++ b/builder/gen-dockerfile/Dockerfile.in @@ -1,6 +1,6 @@ FROM ${STAGING_IMAGE} -LABEL python_version=python3.6 -RUN virtualenv --no-download /env -p python3.6 +LABEL python_version=python3.7 +RUN virtualenv --no-download /env -p python3.7 # Set virtualenv environment variables. This is equivalent to running # source /env/bin/activate diff --git a/cloudbuild_test.yaml b/cloudbuild_test.yaml index 416c5204..ad674026 100644 --- a/cloudbuild_test.yaml +++ b/cloudbuild_test.yaml @@ -43,6 +43,13 @@ steps: '/workspace/tests/virtualenv/virtualenv_python36.yaml', ] waitFor: ['runtime'] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 + args: [ + '-test.v', + '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', + '/workspace/tests/virtualenv/virtualenv_python37.yaml', + ] + waitFor: ['runtime'] - name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 args: [ '-test.v', diff --git a/runtime-image/Dockerfile.in b/runtime-image/Dockerfile.in index 64201966..705a915b 100644 --- a/runtime-image/Dockerfile.in +++ b/runtime-image/Dockerfile.in @@ -19,7 +19,7 @@ ENV PYTHONUNBUFFERED 1 RUN wget https://storage.googleapis.com/python-interpreters/latest/interpreter-3.4.tar.gz && \ wget https://storage.googleapis.com/python-interpreters/latest/interpreter-3.5.tar.gz && \ wget https://storage.googleapis.com/python-interpreters/latest/interpreter-3.6.tar.gz && \ - wget https://storage.googleapis.com/python-interpreters/latest/interpreter-3.7.tar.gz && \ + wget https://storage.googleapis.com/python-interpreters/latest/interpreter-3.7.tar.gz && \ tar -xzf interpreter-3.4.tar.gz && \ tar -xzf interpreter-3.5.tar.gz && \ tar -xzf interpreter-3.6.tar.gz && \ @@ -41,7 +41,7 @@ RUN /usr/bin/pip install --upgrade -r /resources/requirements.txt && \ /opt/python3.6/bin/pip3.6 install --upgrade -r /resources/requirements.txt && \ rm -f /opt/python3.6/bin/pip /opt/python3.6/bin/pip3 && \ /opt/python3.7/bin/pip3.7 install --upgrade -r /resources/requirements.txt && \ - rm -f /opt/python3.7/bin/pip /opt/python3.7/bin/pip3 && \ + rm -f /opt/python3.7/bin/pip /opt/python3.7/bin/pip3 && \ /usr/bin/pip install --upgrade -r /resources/requirements-virtualenv.txt # Setup the app working directory diff --git a/tests/no-virtualenv/no-virtualenv.yaml b/tests/no-virtualenv/no-virtualenv.yaml index 20c09501..c8350e4e 100644 --- a/tests/no-virtualenv/no-virtualenv.yaml +++ b/tests/no-virtualenv/no-virtualenv.yaml @@ -43,7 +43,7 @@ commandTests: expectedOutput: ["/usr/local/bin/python3\n"] - name: "default python3 version" command: ["python3", "--version"] - expectedOutput: ["Python 3.6.7\n"] + expectedOutput: ["Python 3.7.2\n"] - name: "default pip3 installation" command: ["which", "pip3"] expectedOutput: ["/usr/local/bin/pip3\n"] From 407695209e7397cf11b12f898e60a29a9529b328 Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Tue, 22 Jan 2019 09:30:15 -0800 Subject: [PATCH 065/132] Updating build's base image from debian8 to ubuntu_16_0_4 to address SSL issues with 3.6.8 and 3.7.2 build Updating Dockerfile to update_alternatives to 3.7.2 Updating Dockerfile to separate installs so python install errors are easier to differentiate --- build.sh | 8 ++++---- runtime-image/Dockerfile.in | 24 ++++++++++++------------ 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/build.sh b/build.sh index bdc4f658..63afa1ab 100755 --- a/build.sh +++ b/build.sh @@ -24,7 +24,7 @@ test=0 # Should run standard test suite? local=0 # Should run using local Docker daemon instead of GCR? -os_base=debian8 # Which operating system base to use +os_base=ubuntu16 # Which operating system base to use interpreter=0 # Should build interpreters instead of images # Note that $gcloud_cmd has spaces in it @@ -154,10 +154,10 @@ if [ "${local}" -eq 1 ]; then fi # Pick OS image to use as base -if [ "${os_base}" == "ubuntu16" ]; then - export OS_BASE_IMAGE="gcr.io/gcp-runtimes/ubuntu_16_0_4:latest" -else +if [ "${os_base}" == "debian8" ]; then export OS_BASE_IMAGE="gcr.io/google-appengine/debian8:latest" +else + export OS_BASE_IMAGE="gcr.io/gcp-runtimes/ubuntu_16_0_4:latest" fi export STAGING_IMAGE="${DOCKER_NAMESPACE}/python:${TAG}" echo "Using base image name ${STAGING_IMAGE}" diff --git a/runtime-image/Dockerfile.in b/runtime-image/Dockerfile.in index 705a915b..84cb5267 100644 --- a/runtime-image/Dockerfile.in +++ b/runtime-image/Dockerfile.in @@ -28,21 +28,21 @@ RUN wget https://storage.googleapis.com/python-interpreters/latest/interpreter-3 # Add Google-built interpreters to the path ENV PATH /opt/python3.7/bin:/opt/python3.6/bin:/opt/python3.5/bin:/opt/python3.4/bin:$PATH -RUN update-alternatives --install /usr/local/bin/python3 python3 /opt/python3.6/bin/python3.6 50 -RUN update-alternatives --install /usr/local/bin/pip3 pip3 /opt/python3.6/bin/pip3.6 50 +RUN update-alternatives --install /usr/local/bin/python3 python3 /opt/python3.7/bin/python3.7 50 +RUN update-alternatives --install /usr/local/bin/pip3 pip3 /opt/python3.7/bin/pip3.7 50 # Upgrade pip (debian package version tends to run a few version behind) and # install virtualenv system-wide. -RUN /usr/bin/pip install --upgrade -r /resources/requirements.txt && \ - /opt/python3.4/bin/pip3.4 install --upgrade -r /resources/requirements.txt && \ - rm -f /opt/python3.4/bin/pip /opt/python3.4/bin/pip3 && \ - /opt/python3.5/bin/pip3.5 install --upgrade -r /resources/requirements.txt && \ - rm -f /opt/python3.5/bin/pip /opt/python3.5/bin/pip3 && \ - /opt/python3.6/bin/pip3.6 install --upgrade -r /resources/requirements.txt && \ - rm -f /opt/python3.6/bin/pip /opt/python3.6/bin/pip3 && \ - /opt/python3.7/bin/pip3.7 install --upgrade -r /resources/requirements.txt && \ - rm -f /opt/python3.7/bin/pip /opt/python3.7/bin/pip3 && \ - /usr/bin/pip install --upgrade -r /resources/requirements-virtualenv.txt +RUN /usr/bin/pip install --upgrade -r /resources/requirements.txt +RUN /opt/python3.4/bin/pip3.4 install --upgrade -r /resources/requirements.txt +RUN rm -f /opt/python3.4/bin/pip /opt/python3.4/bin/pip3 +RUN /opt/python3.5/bin/pip3.5 install --upgrade -r /resources/requirements.txt +RUN rm -f /opt/python3.5/bin/pip /opt/python3.5/bin/pip3 +#RUN /opt/python3.6/bin/pip3.6 install --upgrade -r /resources/requirements.txt +#RUN rm -f /opt/python3.6/bin/pip /opt/python3.6/bin/pip3 +RUN /opt/python3.7/bin/pip3.7 install --upgrade -r /resources/requirements.txt +RUN rm -f /opt/python3.7/bin/pip /opt/python3.7/bin/pip3 +RUN /usr/bin/pip install --upgrade -r /resources/requirements-virtualenv.txt # Setup the app working directory RUN ln -s /home/vmagent/app /app From 52a469c2fe4f6bfb60fb36bad72e57fd184b8e76 Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Tue, 22 Jan 2019 10:28:05 -0800 Subject: [PATCH 066/132] Updating dockerfile to reduce RUN commands --- runtime-image/Dockerfile.in | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/runtime-image/Dockerfile.in b/runtime-image/Dockerfile.in index 84cb5267..dc842d9b 100644 --- a/runtime-image/Dockerfile.in +++ b/runtime-image/Dockerfile.in @@ -28,21 +28,21 @@ RUN wget https://storage.googleapis.com/python-interpreters/latest/interpreter-3 # Add Google-built interpreters to the path ENV PATH /opt/python3.7/bin:/opt/python3.6/bin:/opt/python3.5/bin:/opt/python3.4/bin:$PATH -RUN update-alternatives --install /usr/local/bin/python3 python3 /opt/python3.7/bin/python3.7 50 -RUN update-alternatives --install /usr/local/bin/pip3 pip3 /opt/python3.7/bin/pip3.7 50 +RUN update-alternatives --install /usr/local/bin/python3 python3 /opt/python3.7/bin/python3.7 50 && \ + update-alternatives --install /usr/local/bin/pip3 pip3 /opt/python3.7/bin/pip3.7 50 # Upgrade pip (debian package version tends to run a few version behind) and # install virtualenv system-wide. -RUN /usr/bin/pip install --upgrade -r /resources/requirements.txt -RUN /opt/python3.4/bin/pip3.4 install --upgrade -r /resources/requirements.txt -RUN rm -f /opt/python3.4/bin/pip /opt/python3.4/bin/pip3 -RUN /opt/python3.5/bin/pip3.5 install --upgrade -r /resources/requirements.txt -RUN rm -f /opt/python3.5/bin/pip /opt/python3.5/bin/pip3 -#RUN /opt/python3.6/bin/pip3.6 install --upgrade -r /resources/requirements.txt -#RUN rm -f /opt/python3.6/bin/pip /opt/python3.6/bin/pip3 -RUN /opt/python3.7/bin/pip3.7 install --upgrade -r /resources/requirements.txt -RUN rm -f /opt/python3.7/bin/pip /opt/python3.7/bin/pip3 -RUN /usr/bin/pip install --upgrade -r /resources/requirements-virtualenv.txt +RUN /usr/bin/pip install --upgrade -r /resources/requirements.txt && \ + /opt/python3.4/bin/pip3.4 install --upgrade -r /resources/requirements.txt && \ + rm -f /opt/python3.4/bin/pip /opt/python3.4/bin/pip3 && \ + /opt/python3.5/bin/pip3.5 install --upgrade -r /resources/requirements.txt && \ + rm -f /opt/python3.5/bin/pip /opt/python3.5/bin/pip3 && \ + /opt/python3.6/bin/pip3.6 install --upgrade -r /resources/requirements.txt && \ + rm -f /opt/python3.6/bin/pip /opt/python3.6/bin/pip3 && \ + /opt/python3.7/bin/pip3.7 install --upgrade -r /resources/requirements.txt && \ + rm -f /opt/python3.7/bin/pip /opt/python3.7/bin/pip3 && \ + /usr/bin/pip install --upgrade -r /resources/requirements-virtualenv.txt # Setup the app working directory RUN ln -s /home/vmagent/app /app From 9701e57eb1deb40e98caa9ca1dc62d55154d69ba Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Tue, 22 Jan 2019 11:16:28 -0800 Subject: [PATCH 067/132] Update python to 3.6.8 and 3.7.2 defaulting to 3.7 - taketwo (#212) * Updating python to 3.6.8 and 3.7.2 and defaulting to 3.7 * fixing 3.7 virtualenv tests * fixing up defaulting to 3.7 * Updating build's base image from debian8 to ubuntu_16_0_4 to address SSL issues with 3.6.8 and 3.7.2 build Updating Dockerfile to update_alternatives to 3.7.2 Updating Dockerfile to separate installs so python install errors are easier to differentiate * Updating dockerfile to reduce RUN commands --- build.sh | 8 +-- builder/gen-dockerfile/Dockerfile.in | 4 +- cloudbuild_test.yaml | 7 +++ .../scripts/build-python-3.6.sh | 12 ++--- .../scripts/build-python-3.7.sh | 12 ++--- runtime-image/Dockerfile.in | 8 +-- scripts/gen_dockerfile.py | 2 +- scripts/gen_dockerfile_test.py | 5 +- .../testdata/hello_world_golden/Dockerfile | 4 +- tests/no-virtualenv/no-virtualenv.yaml | 2 +- tests/virtualenv/virtualenv_python36.yaml | 2 +- tests/virtualenv/virtualenv_python37.yaml | 54 +++++++++++++++++++ 12 files changed, 92 insertions(+), 28 deletions(-) create mode 100644 tests/virtualenv/virtualenv_python37.yaml diff --git a/build.sh b/build.sh index bdc4f658..63afa1ab 100755 --- a/build.sh +++ b/build.sh @@ -24,7 +24,7 @@ test=0 # Should run standard test suite? local=0 # Should run using local Docker daemon instead of GCR? -os_base=debian8 # Which operating system base to use +os_base=ubuntu16 # Which operating system base to use interpreter=0 # Should build interpreters instead of images # Note that $gcloud_cmd has spaces in it @@ -154,10 +154,10 @@ if [ "${local}" -eq 1 ]; then fi # Pick OS image to use as base -if [ "${os_base}" == "ubuntu16" ]; then - export OS_BASE_IMAGE="gcr.io/gcp-runtimes/ubuntu_16_0_4:latest" -else +if [ "${os_base}" == "debian8" ]; then export OS_BASE_IMAGE="gcr.io/google-appengine/debian8:latest" +else + export OS_BASE_IMAGE="gcr.io/gcp-runtimes/ubuntu_16_0_4:latest" fi export STAGING_IMAGE="${DOCKER_NAMESPACE}/python:${TAG}" echo "Using base image name ${STAGING_IMAGE}" diff --git a/builder/gen-dockerfile/Dockerfile.in b/builder/gen-dockerfile/Dockerfile.in index 4f6447eb..627151bf 100644 --- a/builder/gen-dockerfile/Dockerfile.in +++ b/builder/gen-dockerfile/Dockerfile.in @@ -1,6 +1,6 @@ FROM ${STAGING_IMAGE} -LABEL python_version=python3.6 -RUN virtualenv --no-download /env -p python3.6 +LABEL python_version=python3.7 +RUN virtualenv --no-download /env -p python3.7 # Set virtualenv environment variables. This is equivalent to running # source /env/bin/activate diff --git a/cloudbuild_test.yaml b/cloudbuild_test.yaml index 416c5204..ad674026 100644 --- a/cloudbuild_test.yaml +++ b/cloudbuild_test.yaml @@ -43,6 +43,13 @@ steps: '/workspace/tests/virtualenv/virtualenv_python36.yaml', ] waitFor: ['runtime'] +- name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 + args: [ + '-test.v', + '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}', + '/workspace/tests/virtualenv/virtualenv_python37.yaml', + ] + waitFor: ['runtime'] - name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1 args: [ '-test.v', diff --git a/python-interpreter-builder/scripts/build-python-3.6.sh b/python-interpreter-builder/scripts/build-python-3.6.sh index b3896e89..73912401 100755 --- a/python-interpreter-builder/scripts/build-python-3.6.sh +++ b/python-interpreter-builder/scripts/build-python-3.6.sh @@ -6,14 +6,14 @@ set -x # Get the source mkdir -p /opt/sources cd /opt/sources -wget --no-verbose https://www.python.org/ftp/python/3.6.7/Python-3.6.7.tgz +wget --no-verbose https://www.python.org/ftp/python/3.6.8/Python-3.6.8.tgz # SHA-256 generated via `shasum -a 256 [file]` shasum --check < Date: Wed, 30 Jan 2019 11:40:26 -0800 Subject: [PATCH 068/132] Fixing doc to reflect correct base image --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 51413044..51e70cd8 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ for running applications on [Google Kubernetes Engine](https://cloud.google.com/kubernetes-engine), or any other Docker host. -This image is based on Debian Jessie and contains packages required to build +This image is based on Ubuntu Xenial and contains packages required to build most of the popular Python libraries. For more information about this runtime, see the [documentation](https://cloud.google.com/appengine/docs/flexible/python/runtime). From cbc6240d561e3727c42a252bf09caa3d89c5a0b0 Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Fri, 1 Feb 2019 16:08:12 -0800 Subject: [PATCH 069/132] Revert default python3 version to 3.6 (#215) --- builder/gen-dockerfile/Dockerfile.in | 4 ++-- scripts/gen_dockerfile.py | 2 +- scripts/gen_dockerfile_test.py | 2 +- scripts/testdata/hello_world_golden/Dockerfile | 4 ++-- tests/no-virtualenv/no-virtualenv.yaml | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/builder/gen-dockerfile/Dockerfile.in b/builder/gen-dockerfile/Dockerfile.in index 627151bf..4f6447eb 100644 --- a/builder/gen-dockerfile/Dockerfile.in +++ b/builder/gen-dockerfile/Dockerfile.in @@ -1,6 +1,6 @@ FROM ${STAGING_IMAGE} -LABEL python_version=python3.7 -RUN virtualenv --no-download /env -p python3.7 +LABEL python_version=python3.6 +RUN virtualenv --no-download /env -p python3.6 # Set virtualenv environment variables. This is equivalent to running # source /env/bin/activate diff --git a/scripts/gen_dockerfile.py b/scripts/gen_dockerfile.py index 852405d8..97da60f6 100755 --- a/scripts/gen_dockerfile.py +++ b/scripts/gen_dockerfile.py @@ -50,7 +50,7 @@ PYTHON_INTERPRETER_VERSION_MAP = { '': '', # == 2.7 '2': '', # == 2.7 - '3': '3.7', + '3': '3.6', '3.4': '3.4', '3.5': '3.5', '3.6': '3.6', diff --git a/scripts/gen_dockerfile_test.py b/scripts/gen_dockerfile_test.py index c78c6ad8..03eaa079 100755 --- a/scripts/gen_dockerfile_test.py +++ b/scripts/gen_dockerfile_test.py @@ -71,7 +71,7 @@ def compare_file(filename, dir1, dir2): 'dockerfile_python_version': '', }), ('runtime_config:\n python_version: 3', { - 'dockerfile_python_version': '3.7', + 'dockerfile_python_version': '3.6', }), ('runtime_config:\n python_version: 3.4', { 'dockerfile_python_version': '3.4', diff --git a/scripts/testdata/hello_world_golden/Dockerfile b/scripts/testdata/hello_world_golden/Dockerfile index 55eb8cec..10396399 100644 --- a/scripts/testdata/hello_world_golden/Dockerfile +++ b/scripts/testdata/hello_world_golden/Dockerfile @@ -1,6 +1,6 @@ FROM gcr.io/google-appengine/python -LABEL python_version=python3.7 -RUN virtualenv --no-download /env -p python3.7 +LABEL python_version=python3.6 +RUN virtualenv --no-download /env -p python3.6 # Set virtualenv environment variables. This is equivalent to running # source /env/bin/activate diff --git a/tests/no-virtualenv/no-virtualenv.yaml b/tests/no-virtualenv/no-virtualenv.yaml index c8350e4e..6a3b1e7d 100644 --- a/tests/no-virtualenv/no-virtualenv.yaml +++ b/tests/no-virtualenv/no-virtualenv.yaml @@ -43,7 +43,7 @@ commandTests: expectedOutput: ["/usr/local/bin/python3\n"] - name: "default python3 version" command: ["python3", "--version"] - expectedOutput: ["Python 3.7.2\n"] + expectedOutput: ["Python 3.6.8\n"] - name: "default pip3 installation" command: ["which", "pip3"] expectedOutput: ["/usr/local/bin/pip3\n"] From d628b583eb2f5684d02a441b99d08f8b7310f79a Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Mon, 4 Feb 2019 14:31:15 -0800 Subject: [PATCH 070/132] Make python 3.6 the default python 3 version (#216) --- runtime-image/Dockerfile.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runtime-image/Dockerfile.in b/runtime-image/Dockerfile.in index dc842d9b..79862620 100644 --- a/runtime-image/Dockerfile.in +++ b/runtime-image/Dockerfile.in @@ -28,8 +28,8 @@ RUN wget https://storage.googleapis.com/python-interpreters/latest/interpreter-3 # Add Google-built interpreters to the path ENV PATH /opt/python3.7/bin:/opt/python3.6/bin:/opt/python3.5/bin:/opt/python3.4/bin:$PATH -RUN update-alternatives --install /usr/local/bin/python3 python3 /opt/python3.7/bin/python3.7 50 && \ - update-alternatives --install /usr/local/bin/pip3 pip3 /opt/python3.7/bin/pip3.7 50 +RUN update-alternatives --install /usr/local/bin/python3 python3 /opt/python3.6/bin/python3.6 50 && \ + update-alternatives --install /usr/local/bin/pip3 pip3 /opt/python3.6/bin/pip3.6 50 # Upgrade pip (debian package version tends to run a few version behind) and # install virtualenv system-wide. From adacb0fd5342590935ce76a9c480cd9bbf89a8bf Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Wed, 6 Feb 2019 13:59:05 -0800 Subject: [PATCH 071/132] explicitly run apt-get upgrade to get latest versions of packages (#217) --- runtime-image/scripts/install-apt-packages.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/runtime-image/scripts/install-apt-packages.sh b/runtime-image/scripts/install-apt-packages.sh index 9d23cab7..bafba2d6 100755 --- a/runtime-image/scripts/install-apt-packages.sh +++ b/runtime-image/scripts/install-apt-packages.sh @@ -7,6 +7,8 @@ apt-get -q update xargs -a <(awk '/^\s*[^#]/' '/resources/apt-packages.txt') -r -- \ apt-get install --no-install-recommends -yq +apt-get upgrade -yq + # Remove unneeded files. apt-get clean rm /var/lib/apt/lists/*_* From 272f8d9ebd311b9c4466a252cc6a71223317c7f2 Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Wed, 6 Feb 2019 13:59:16 -0800 Subject: [PATCH 072/132] add donmccasland to CODEOWNERS (#218) --- CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CODEOWNERS b/CODEOWNERS index f8b549f9..ea26d5dc 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1,4 +1,4 @@ # Code owners file. # This file controls who is tagged for review for any given pull request. -* @dlorenc @sharifelgamal @nkubala @tstromberg +* @dlorenc @sharifelgamal @nkubala @tstromberg @donmccasland From 953a810d823e99c45c43b19a9c960c426c114051 Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Thu, 4 Apr 2019 13:54:19 -0700 Subject: [PATCH 073/132] Adding build scripts --- scripts/deploy_check.sh | 34 +++++++++++++++++++++ scripts/integration-test.sh | 61 +++++++++++++++++++++++++++++++++++++ scripts/release.sh | 18 +++++++++++ 3 files changed, 113 insertions(+) create mode 100644 scripts/deploy_check.sh create mode 100644 scripts/integration-test.sh create mode 100644 scripts/release.sh diff --git a/scripts/deploy_check.sh b/scripts/deploy_check.sh new file mode 100644 index 00000000..77fccb54 --- /dev/null +++ b/scripts/deploy_check.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +set -ex + +export KOKORO_GITHUB_DIR=${KOKORO_ROOT}/src/github +source ${KOKORO_GFILE_DIR}/kokoro/common.sh + +cd ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY} +if [ -n "${RUNTIME_SPEC}" -a -f app.yaml.in ]; then + sed "s|\${RUNTIME_SPEC}|${RUNTIME_SPEC}|" app.yaml.in > app.yaml +fi + +cd ${KOKORO_GFILE_DIR}/appengine/integration_tests + +sudo /usr/local/bin/pip install --upgrade -r requirements.txt + +if [ -f ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY}/requirements.txt ] +then + sudo /usr/local/bin/pip install --upgrade -r ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY}/requirements.txt +fi + +export DEPLOY_LATENCY_PROJECT='cloud-deploy-latency' + +skip_flag="" + +if [ "${SKIP_CUSTOM_LOGGING_TESTS}" = "true" -o "${SKIP_BUILDERS}" = "true" ]; then + skip_flag="$skip_flag --skip-builders" +fi + +if [ "${SKIP_XRT}" = "true" ]; then + skip_flag="$skip_flag --skip-xrt" +fi + +python deploy_check.py -d ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY} -l ${LANGUAGE} ${skip_flag} diff --git a/scripts/integration-test.sh b/scripts/integration-test.sh new file mode 100644 index 00000000..63137b5f --- /dev/null +++ b/scripts/integration-test.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +set -ex + +export KOKORO_GITHUB_DIR=${KOKORO_ROOT}/src/github +source ${KOKORO_GFILE_DIR}/kokoro/common.sh + +export GOOGLE_CLOUD_PROJECT=gcp-runtimes + +sudo /usr/local/bin/pip install --upgrade -r ${KOKORO_GFILE_DIR}/appengine/integration_tests/requirements.txt + +if [ -f ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY}/requirements.txt ] +then + sudo /usr/local/bin/pip install --upgrade -r ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY}/requirements.txt +fi + +export GOPATH=${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY} + +flags="" + +if [ -n "${STAGING_IMAGE}" ]; then + flags="$flags -i ${STAGING_IMAGE}" +fi + +if [ "${SKIP_STANDARD_LOGGING_TESTS}" = "true" ]; then + flags="$flags --skip-standard-logging-tests" +fi + +if [ "${SKIP_CUSTOM_LOGGING_TESTS}" = "true" ]; then + flags="$flags --skip-custom-logging-tests" +fi + +if [ "${SKIP_MONITORING_TESTS}" = "true" ]; then + flags="$flags --skip-monitoring-tests" +fi + +if [ "${SKIP_EXCEPTION_TESTS}" = "true" ]; then + flags="$flags --skip-exception-tests" +fi + +if [ "${SKIP_CUSTOM_TESTS}" = "true" ]; then + flags="$flags --skip-custom-tests" +fi + +if [ -n "${URL}" ]; then + flags="$flags --url ${URL}" +fi + +if [ -n "${BUILDER}" ]; then + flags="$flags --builder ${BUILDER}" + gcloud config set app/use_runtime_builders True + gcloud config set app/runtime_builders_root file://${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY} +fi + +if [ -n "${YAML}" ]; then + flags="$flags --yaml ${KOKORO_GITHUB_DIR}/${YAML}" +fi + + +chmod a+x ${KOKORO_GFILE_DIR}/appengine/integration_tests/testsuite/driver.py +${KOKORO_GFILE_DIR}/appengine/integration_tests/testsuite/driver.py -d ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY} ${flags} diff --git a/scripts/release.sh b/scripts/release.sh new file mode 100644 index 00000000..1857deaf --- /dev/null +++ b/scripts/release.sh @@ -0,0 +1,18 @@ +#!/bin/bash +set -euo pipefail +export KOKORO_GITHUB_DIR=${KOKORO_ROOT}/src/github +source ${KOKORO_GFILE_DIR}/kokoro/common.sh + +source "${KOKORO_PIPER_DIR}/google3/third_party/runtimes_common/kokoro/common.sh" + +cd ${KOKORO_GITHUB_DIR}/python-runtime + +if [ -z "${TAG:+set}" ]; then + export TAG=$(date +%Y-%m-%d-%H%M%S) +fi + +./build.sh $BUILD_FLAGS + +METADATA=$(pwd)/METADATA +cd ${KOKORO_GFILE_DIR}/kokoro +python note.py python -m ${METADATA} -t ${TAG} From a1339d625a87da959d7aa7ddfa35ccce421a1ef8 Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Mon, 8 Apr 2019 08:59:33 -0700 Subject: [PATCH 074/132] Fixing release script --- scripts/release.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/release.sh b/scripts/release.sh index 1857deaf..cd499823 100644 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -3,8 +3,6 @@ set -euo pipefail export KOKORO_GITHUB_DIR=${KOKORO_ROOT}/src/github source ${KOKORO_GFILE_DIR}/kokoro/common.sh -source "${KOKORO_PIPER_DIR}/google3/third_party/runtimes_common/kokoro/common.sh" - cd ${KOKORO_GITHUB_DIR}/python-runtime if [ -z "${TAG:+set}" ]; then From c6864c03b87842d891f362559a73e0181c2e0c69 Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Tue, 17 Sep 2019 15:38:46 -0700 Subject: [PATCH 075/132] Updating integration test pip requirement: gcloud-logging version 1.12.1 --- tests/integration/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index 4a462433..ac718c60 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -1,6 +1,6 @@ Flask==1.0.2 google-cloud-error-reporting==0.30.0 -google-cloud-logging==1.6.0 +google-cloud-logging==1.12.1 google-cloud-monitoring==0.30.1 gunicorn==19.9.0 requests==2.19.1 From 7fc2e28c8ce121cbfa899f7852a54e5795818571 Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Tue, 17 Sep 2019 15:52:28 -0700 Subject: [PATCH 076/132] Updating integration test pip requirements --- tests/integration/requirements.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index ac718c60..73a48913 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -1,8 +1,8 @@ -Flask==1.0.2 -google-cloud-error-reporting==0.30.0 +Flask==1.1.0 +google-cloud-error-reporting==0.32.1 google-cloud-logging==1.12.1 -google-cloud-monitoring==0.30.1 +google-cloud-monitoring==0.33.0 gunicorn==19.9.0 -requests==2.19.1 +requests==2.22.0 retrying==1.3.3 -six==1.11.0 +six==1.12.0 From 17a4a6c3cec29fb772e8280ac13d23a53d30d8a8 Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Wed, 18 Sep 2019 10:59:59 -0700 Subject: [PATCH 077/132] Adding protobuf requirement so kokoro will upgrade it --- tests/integration/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index 73a48913..ea714c20 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -6,3 +6,4 @@ gunicorn==19.9.0 requests==2.22.0 retrying==1.3.3 six==1.12.0 +protobuf>=3.6.0 From 87a309ee632ade17162d85211e4fa0164ab613af Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Thu, 5 Mar 2020 09:49:48 -0800 Subject: [PATCH 078/132] update python 3 to use python 3.7 --- runtime-image/Dockerfile.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runtime-image/Dockerfile.in b/runtime-image/Dockerfile.in index 79862620..dc842d9b 100644 --- a/runtime-image/Dockerfile.in +++ b/runtime-image/Dockerfile.in @@ -28,8 +28,8 @@ RUN wget https://storage.googleapis.com/python-interpreters/latest/interpreter-3 # Add Google-built interpreters to the path ENV PATH /opt/python3.7/bin:/opt/python3.6/bin:/opt/python3.5/bin:/opt/python3.4/bin:$PATH -RUN update-alternatives --install /usr/local/bin/python3 python3 /opt/python3.6/bin/python3.6 50 && \ - update-alternatives --install /usr/local/bin/pip3 pip3 /opt/python3.6/bin/pip3.6 50 +RUN update-alternatives --install /usr/local/bin/python3 python3 /opt/python3.7/bin/python3.7 50 && \ + update-alternatives --install /usr/local/bin/pip3 pip3 /opt/python3.7/bin/pip3.7 50 # Upgrade pip (debian package version tends to run a few version behind) and # install virtualenv system-wide. From fbc2908de36bb94bb00dc4a25c80a6113c1bf231 Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Fri, 27 Mar 2020 10:01:17 -0700 Subject: [PATCH 079/132] Updating python 3,7 to 3.7.7 --- .../scripts/build-python-3.7.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/python-interpreter-builder/scripts/build-python-3.7.sh b/python-interpreter-builder/scripts/build-python-3.7.sh index 69febc88..3b3bdae5 100755 --- a/python-interpreter-builder/scripts/build-python-3.7.sh +++ b/python-interpreter-builder/scripts/build-python-3.7.sh @@ -6,14 +6,14 @@ set -x # Get the source mkdir -p /opt/sources cd /opt/sources -wget --no-verbose https://www.python.org/ftp/python/3.7.2/Python-3.7.2.tgz +wget --no-verbose https://www.python.org/ftp/python/3.7.7/Python-3.7.7.tgz # SHA-256 generated via `shasum -a 256 [file]` shasum --check < Date: Fri, 27 Mar 2020 10:43:39 -0700 Subject: [PATCH 080/132] Remove nkubala from owners (#233) --- CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CODEOWNERS b/CODEOWNERS index ea26d5dc..5deacc6c 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1,4 +1,4 @@ # Code owners file. # This file controls who is tagged for review for any given pull request. -* @dlorenc @sharifelgamal @nkubala @tstromberg @donmccasland +* @dlorenc @sharifelgamal @tstromberg @donmccasland From f6c18ed7aa36675eed95757ff8627871256e4be6 Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Fri, 27 Mar 2020 10:47:56 -0700 Subject: [PATCH 081/132] Fixing test --- tests/no-virtualenv/no-virtualenv.yaml | 2 +- tests/virtualenv/virtualenv_python37.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/no-virtualenv/no-virtualenv.yaml b/tests/no-virtualenv/no-virtualenv.yaml index 6a3b1e7d..19e48df9 100644 --- a/tests/no-virtualenv/no-virtualenv.yaml +++ b/tests/no-virtualenv/no-virtualenv.yaml @@ -43,7 +43,7 @@ commandTests: expectedOutput: ["/usr/local/bin/python3\n"] - name: "default python3 version" command: ["python3", "--version"] - expectedOutput: ["Python 3.6.8\n"] + expectedOutput: ["Python 3.7.7\n"] - name: "default pip3 installation" command: ["which", "pip3"] expectedOutput: ["/usr/local/bin/pip3\n"] diff --git a/tests/virtualenv/virtualenv_python37.yaml b/tests/virtualenv/virtualenv_python37.yaml index 457933f9..7de6d10c 100644 --- a/tests/virtualenv/virtualenv_python37.yaml +++ b/tests/virtualenv/virtualenv_python37.yaml @@ -25,7 +25,7 @@ commandTests: - name: "virtualenv37 python version" setup: [["virtualenv", "-p", "python3.7", "/env"]] command: ["python", "--version"] - expectedOutput: ["Python 3.7.2\n"] + expectedOutput: ["Python 3.7.7\n"] - name: "virtualenv37 pip installation" setup: [["virtualenv", "-p", "python3.7", "/env"]] From a0d1f9def6b296f789023b7973570d6aba857e0d Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Sat, 28 Mar 2020 20:18:17 -0700 Subject: [PATCH 082/132] Update interpreter builders with new python versions --- .../scripts/build-python-3.5.sh | 12 ++++++------ .../scripts/build-python-3.6.sh | 12 ++++++------ .../scripts/build-python-3.7.sh | 2 +- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/python-interpreter-builder/scripts/build-python-3.5.sh b/python-interpreter-builder/scripts/build-python-3.5.sh index 86a564c3..c2fd748b 100755 --- a/python-interpreter-builder/scripts/build-python-3.5.sh +++ b/python-interpreter-builder/scripts/build-python-3.5.sh @@ -6,14 +6,14 @@ set -x # Get the source mkdir -p /opt/sources cd /opt/sources -wget --no-verbose https://www.python.org/ftp/python/3.5.5/Python-3.5.5.tgz +wget --no-verbose https://www.python.org/ftp/python/3.5.9/Python-3.5.9.tgz # SHA-256 generated via `shasum -a 256 [file]` shasum --check < Date: Sat, 28 Mar 2020 20:20:28 -0700 Subject: [PATCH 083/132] Update tests --- tests/virtualenv/virtualenv_python35.yaml | 2 +- tests/virtualenv/virtualenv_python36.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/virtualenv/virtualenv_python35.yaml b/tests/virtualenv/virtualenv_python35.yaml index 5f37ee54..3bb3d814 100644 --- a/tests/virtualenv/virtualenv_python35.yaml +++ b/tests/virtualenv/virtualenv_python35.yaml @@ -25,7 +25,7 @@ commandTests: - name: "virtualenv35 python version" setup: [["virtualenv", "-p", "python3.5", "/env"]] command: ["python", "--version"] - expectedOutput: ["Python 3.5.5\n"] + expectedOutput: ["Python 3.5.9\n"] - name: "virtualenv35 pip installation" setup: [["virtualenv", "-p", "python3.5", "/env"]] diff --git a/tests/virtualenv/virtualenv_python36.yaml b/tests/virtualenv/virtualenv_python36.yaml index 64ba2596..f0949c6f 100644 --- a/tests/virtualenv/virtualenv_python36.yaml +++ b/tests/virtualenv/virtualenv_python36.yaml @@ -25,7 +25,7 @@ commandTests: - name: "virtualenv36 python version" setup: [["virtualenv", "-p", "python3.6", "/env"]] command: ["python", "--version"] - expectedOutput: ["Python 3.6.8\n"] + expectedOutput: ["Python 3.6.10\n"] - name: "virtualenv36 pip installation" setup: [["virtualenv", "-p", "python3.6", "/env"]] From 641e22349c6937e224ca8c7634de4399d1693bdf Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Sun, 29 Mar 2020 16:19:46 -0700 Subject: [PATCH 084/132] Fix 3.5 test failure --- tests/python3-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 3b283d1d..e3fd6424 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -77,7 +77,7 @@ lxml==4.2.4 mako==1.0.7 manifestparser==1.1 markdown==2.6.11 -markupsafe==1.0 +markupsafe==1.1.1 matplotlib==2.2.3 mccabe==0.6.1 meld3==1.0.2 From 3d5d38460ca4beac31a5b4cb35adaa25c31670d1 Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Mon, 30 Mar 2020 10:12:28 -0700 Subject: [PATCH 085/132] Attempting upgrade of pip version --- runtime-image/resources/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime-image/resources/requirements.txt b/runtime-image/resources/requirements.txt index 298e5f41..7ec375ff 100644 --- a/runtime-image/resources/requirements.txt +++ b/runtime-image/resources/requirements.txt @@ -1,5 +1,5 @@ # Do not update pip, it conflicts with the OS system installed version. # https://github.com/pypa/pip/issues/5240 -pip==9.0.3 +pip==10.0.1 setuptools==40.2.0 wheel==0.31.1 From bf65b6947c2e9f63a260f95d121ca07926361315 Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Mon, 30 Mar 2020 10:27:04 -0700 Subject: [PATCH 086/132] Latest pip is actually 20.0.2 --- runtime-image/resources/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime-image/resources/requirements.txt b/runtime-image/resources/requirements.txt index 7ec375ff..b374c0e8 100644 --- a/runtime-image/resources/requirements.txt +++ b/runtime-image/resources/requirements.txt @@ -1,5 +1,5 @@ # Do not update pip, it conflicts with the OS system installed version. # https://github.com/pypa/pip/issues/5240 -pip==10.0.1 +pip==20.0.2 setuptools==40.2.0 wheel==0.31.1 From cc6436428ad613f8e311045a26b7b54c6b5ad442 Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Mon, 30 Mar 2020 11:08:46 -0700 Subject: [PATCH 087/132] Looks like the latest available is 19.1.1 --- runtime-image/resources/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime-image/resources/requirements.txt b/runtime-image/resources/requirements.txt index b374c0e8..0032013d 100644 --- a/runtime-image/resources/requirements.txt +++ b/runtime-image/resources/requirements.txt @@ -1,5 +1,5 @@ # Do not update pip, it conflicts with the OS system installed version. # https://github.com/pypa/pip/issues/5240 -pip==20.0.2 +pip==19.1.1 setuptools==40.2.0 wheel==0.31.1 From 1f2f63dcad36e63ee0ced458cda7d53922290350 Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Tue, 31 Mar 2020 13:27:33 -0700 Subject: [PATCH 088/132] Letting pip go to latest to address python2 pip incompatibility issues. --- runtime-image/resources/requirements.txt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/runtime-image/resources/requirements.txt b/runtime-image/resources/requirements.txt index 0032013d..2d010eef 100644 --- a/runtime-image/resources/requirements.txt +++ b/runtime-image/resources/requirements.txt @@ -1,5 +1,3 @@ -# Do not update pip, it conflicts with the OS system installed version. -# https://github.com/pypa/pip/issues/5240 -pip==19.1.1 +pip setuptools==40.2.0 wheel==0.31.1 From e212b243482b17196725eaaf59c25ced9fd7960b Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Tue, 18 Aug 2020 09:15:39 -0700 Subject: [PATCH 089/132] Updating to 3.7.9 --- .../scripts/build-python-3.7.sh | 12 ++++++------ tests/virtualenv/virtualenv_python37.yaml | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/python-interpreter-builder/scripts/build-python-3.7.sh b/python-interpreter-builder/scripts/build-python-3.7.sh index bee33cef..b1b06806 100755 --- a/python-interpreter-builder/scripts/build-python-3.7.sh +++ b/python-interpreter-builder/scripts/build-python-3.7.sh @@ -6,14 +6,14 @@ set -x # Get the source mkdir -p /opt/sources cd /opt/sources -wget --no-verbose https://www.python.org/ftp/python/3.7.7/Python-3.7.7.tgz +wget --no-verbose https://www.python.org/ftp/python/3.7.9/Python-3.7.9.tgz # SHA-256 generated via `shasum -a 256 [file]` shasum --check < Date: Thu, 3 Sep 2020 12:56:15 -0700 Subject: [PATCH 090/132] Remove virtualenv pip before installing requirements --- builder/gen-dockerfile/Dockerfile.in | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/builder/gen-dockerfile/Dockerfile.in b/builder/gen-dockerfile/Dockerfile.in index 4f6447eb..98501c77 100644 --- a/builder/gen-dockerfile/Dockerfile.in +++ b/builder/gen-dockerfile/Dockerfile.in @@ -7,7 +7,9 @@ RUN virtualenv --no-download /env -p python3.6 ENV VIRTUAL_ENV /env ENV PATH /env/bin:$PATH ADD requirements.txt /builder/ -RUN pip install -r /builder/requirements.txt +#virtualenv's pip is pegged at version 10.0, removing so +#newer versions get picked up +RUN rm -f /env/bin/pip* && pip install -r /builder/requirements.txt ADD . /builder/ WORKDIR /workspace ENTRYPOINT [ "python", "/builder/gen_dockerfile.py" ] From c9f59421f0064de12c45eb258635fe591d182907 Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Thu, 3 Sep 2020 13:53:37 -0700 Subject: [PATCH 091/132] Fix 3.7.9 test --- tests/no-virtualenv/no-virtualenv.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/no-virtualenv/no-virtualenv.yaml b/tests/no-virtualenv/no-virtualenv.yaml index 19e48df9..4f6c3f48 100644 --- a/tests/no-virtualenv/no-virtualenv.yaml +++ b/tests/no-virtualenv/no-virtualenv.yaml @@ -43,7 +43,7 @@ commandTests: expectedOutput: ["/usr/local/bin/python3\n"] - name: "default python3 version" command: ["python3", "--version"] - expectedOutput: ["Python 3.7.7\n"] + expectedOutput: ["Python 3.7.9\n"] - name: "default pip3 installation" command: ["which", "pip3"] expectedOutput: ["/usr/local/bin/pip3\n"] From 71223557e38a88abc8b365d1bb870c5d4132e864 Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Thu, 10 Sep 2020 13:53:38 -0700 Subject: [PATCH 092/132] Update virtualenv version --- runtime-image/resources/requirements-virtualenv.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime-image/resources/requirements-virtualenv.txt b/runtime-image/resources/requirements-virtualenv.txt index 52bb7da1..25f09c4a 100644 --- a/runtime-image/resources/requirements-virtualenv.txt +++ b/runtime-image/resources/requirements-virtualenv.txt @@ -1 +1 @@ -virtualenv==16.0.0 +virtualenv==20.0.31 From 92a7aba5365a784aaf66192f86bfa5912f56d405 Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Thu, 10 Sep 2020 17:19:11 -0700 Subject: [PATCH 093/132] Updating virtualenv, test, and python builder version --- builder/gen-dockerfile/Dockerfile.in | 4 ++-- tests/eventlet/requirements.txt | 2 +- tests/virtualenv/virtualenv_default.yaml | 2 +- tests/virtualenv/virtualenv_python27.yaml | 2 +- tests/virtualenv/virtualenv_python34.yaml | 2 +- tests/virtualenv/virtualenv_python35.yaml | 2 +- tests/virtualenv/virtualenv_python36.yaml | 2 +- tests/virtualenv/virtualenv_python37.yaml | 2 +- 8 files changed, 9 insertions(+), 9 deletions(-) diff --git a/builder/gen-dockerfile/Dockerfile.in b/builder/gen-dockerfile/Dockerfile.in index 98501c77..4ddce7c2 100644 --- a/builder/gen-dockerfile/Dockerfile.in +++ b/builder/gen-dockerfile/Dockerfile.in @@ -1,6 +1,6 @@ FROM ${STAGING_IMAGE} -LABEL python_version=python3.6 -RUN virtualenv --no-download /env -p python3.6 +LABEL python_version=python3.7 +RUN virtualenv --no-download /env -p python3.7 # Set virtualenv environment variables. This is equivalent to running # source /env/bin/activate diff --git a/tests/eventlet/requirements.txt b/tests/eventlet/requirements.txt index 7f8b2468..65eabc74 100644 --- a/tests/eventlet/requirements.txt +++ b/tests/eventlet/requirements.txt @@ -6,5 +6,5 @@ greenlet==0.4.14 gunicorn==19.9.0 itsdangerous==0.24 Jinja2==2.10 -MarkupSafe==1.0 +MarkupSafe==1.1.1 Werkzeug==0.14.1 diff --git a/tests/virtualenv/virtualenv_default.yaml b/tests/virtualenv/virtualenv_default.yaml index 1659c746..6b6ad282 100644 --- a/tests/virtualenv/virtualenv_default.yaml +++ b/tests/virtualenv/virtualenv_default.yaml @@ -34,4 +34,4 @@ commandTests: setup: [["virtualenv", "/env"], ["pip", "install", "flask"]] command: ["python", "-c", "import flask; print(flask.__file__)"] - expectedOutput: ["/env/local/lib/python2.7/site-packages/flask"] + expectedOutput: ["/env/lib/python2.7/site-packages/flask/__init__.pyc"] diff --git a/tests/virtualenv/virtualenv_python27.yaml b/tests/virtualenv/virtualenv_python27.yaml index 8bd85289..09b78480 100644 --- a/tests/virtualenv/virtualenv_python27.yaml +++ b/tests/virtualenv/virtualenv_python27.yaml @@ -44,4 +44,4 @@ commandTests: setup: [["virtualenv", "-p", "python", "/env"], ["pip", "install", "flask"]] command: ["python", "-c", "import flask; print(flask.__file__)"] - expectedOutput: ["/env/local/lib/python2.7/site-packages/flask"] + expectedOutput: ["/env/lib/python2.7/site-packages/flask/__init__.pyc"] diff --git a/tests/virtualenv/virtualenv_python34.yaml b/tests/virtualenv/virtualenv_python34.yaml index 077606fa..9b5b77d0 100644 --- a/tests/virtualenv/virtualenv_python34.yaml +++ b/tests/virtualenv/virtualenv_python34.yaml @@ -47,7 +47,7 @@ commandTests: setup: [["virtualenv", "-p", "python3.4", "/env"], ["pip", "install", "flask"]] command: ["python", "-c", "import flask; print(flask.__file__)"] - expectedOutput: ["/env/lib/python3.4/site-packages/flask"] + expectedOutput: ["/env/lib/python3.4/site-packages/flask/__init__.py"] - name: "virtualenv34 test.support availability" setup: [["virtualenv", "-p", "python3.4", "/env"]] diff --git a/tests/virtualenv/virtualenv_python35.yaml b/tests/virtualenv/virtualenv_python35.yaml index 3bb3d814..5e4b394a 100644 --- a/tests/virtualenv/virtualenv_python35.yaml +++ b/tests/virtualenv/virtualenv_python35.yaml @@ -47,7 +47,7 @@ commandTests: setup: [["virtualenv", "-p", "python3.5", "/env"], ["pip", "install", "flask"]] command: ["python", "-c", "import flask; print(flask.__file__)"] - expectedOutput: ["/env/lib/python3.5/site-packages/flask"] + expectedOutput: ["/env/lib/python3.5/site-packages/flask/__init__.py"] - name: "virtualenv35 test.support availability" setup: [["virtualenv", "-p", "python3.5", "/env"]] diff --git a/tests/virtualenv/virtualenv_python36.yaml b/tests/virtualenv/virtualenv_python36.yaml index f0949c6f..b3a9e68e 100644 --- a/tests/virtualenv/virtualenv_python36.yaml +++ b/tests/virtualenv/virtualenv_python36.yaml @@ -47,7 +47,7 @@ commandTests: setup: [["virtualenv", "-p", "python3.6", "/env"], ["pip", "install", "flask"]] command: ["python", "-c", "import flask; print(flask.__file__)"] - expectedOutput: ["/env/lib/python3.6/site-packages/flask"] + expectedOutput: ["/env/lib/python3.6/site-packages/flask/__init__.py"] - name: "virtualenv36 test.support availability" setup: [["virtualenv", "-p", "python3.6", "/env"]] diff --git a/tests/virtualenv/virtualenv_python37.yaml b/tests/virtualenv/virtualenv_python37.yaml index 7f3520ad..9810c78e 100644 --- a/tests/virtualenv/virtualenv_python37.yaml +++ b/tests/virtualenv/virtualenv_python37.yaml @@ -47,7 +47,7 @@ commandTests: setup: [["virtualenv", "-p", "python3.7", "/env"], ["pip", "install", "flask"]] command: ["python", "-c", "import flask; print(flask.__file__)"] - expectedOutput: ["/env/lib/python3.7/site-packages/flask"] + expectedOutput: ["/env/lib/python3.7/site-packages/flask/__init__.py"] - name: "virtualenv37 test.support availability" setup: [["virtualenv", "-p", "python3.7", "/env"]] From df22badacb89bbd6cb21f0fc9f50189053ac5019 Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Fri, 11 Sep 2020 10:26:30 -0700 Subject: [PATCH 094/132] Use virtualenv pip --- builder/gen-dockerfile/Dockerfile.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/gen-dockerfile/Dockerfile.in b/builder/gen-dockerfile/Dockerfile.in index 4ddce7c2..92ceaecf 100644 --- a/builder/gen-dockerfile/Dockerfile.in +++ b/builder/gen-dockerfile/Dockerfile.in @@ -9,7 +9,7 @@ ENV PATH /env/bin:$PATH ADD requirements.txt /builder/ #virtualenv's pip is pegged at version 10.0, removing so #newer versions get picked up -RUN rm -f /env/bin/pip* && pip install -r /builder/requirements.txt +RUN pip install -r /builder/requirements.txt ADD . /builder/ WORKDIR /workspace ENTRYPOINT [ "python", "/builder/gen_dockerfile.py" ] From 764a5f0bc0dd5a5ea1a2404e813ed9a8569da218 Mon Sep 17 00:00:00 2001 From: Don McCasland Date: Wed, 7 Jul 2021 12:23:41 -0700 Subject: [PATCH 095/132] Fixing build error for python2.7 pip --- build.sh | 15 ++++++++++++--- runtime-image/Dockerfile.in | 1 + runtime-image/resources/apt-packages.txt | 1 - 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/build.sh b/build.sh index 63afa1ab..2083237d 100755 --- a/build.sh +++ b/build.sh @@ -24,7 +24,7 @@ test=0 # Should run standard test suite? local=0 # Should run using local Docker daemon instead of GCR? -os_base=ubuntu16 # Which operating system base to use +os_base=ubuntu18 # Which operating system base to use interpreter=0 # Should build interpreters instead of images # Note that $gcloud_cmd has spaces in it @@ -48,7 +48,7 @@ Options: --[no]test: Run basic tests (default true if no options set) --[no]client_test: Run Google Cloud Client Library tests (default false) --[no]local: Build images using local Docker daemon (default false) - --os_base: Which OS image to build on top of [debian8, ubuntu16] + --os_base: Which OS image to build on top of [debian8, ubuntu16, ubuntu18] " } @@ -119,6 +119,10 @@ while [ $# -gt 0 ]; do os_base=ubuntu16 shift ;; + --os_base=ubuntu18) + os_base=ubuntu18 + shift + ;; --test) test=1 shift @@ -156,8 +160,13 @@ fi # Pick OS image to use as base if [ "${os_base}" == "debian8" ]; then export OS_BASE_IMAGE="gcr.io/google-appengine/debian8:latest" -else +elif [ "${os_base}" == "ubuntu16" ]; then export OS_BASE_IMAGE="gcr.io/gcp-runtimes/ubuntu_16_0_4:latest" +elif [ "${os_base}" == "ubuntu18" ]; then + export OS_BASE_IMAGE="gcr.io/gcp-runtimes/ubuntu_18_0_4:latest" +else + echo "Unsupported OS base image: $OS_BASE_IMAGE" + exit 1 fi export STAGING_IMAGE="${DOCKER_NAMESPACE}/python:${TAG}" echo "Using base image name ${STAGING_IMAGE}" diff --git a/runtime-image/Dockerfile.in b/runtime-image/Dockerfile.in index dc842d9b..46387705 100644 --- a/runtime-image/Dockerfile.in +++ b/runtime-image/Dockerfile.in @@ -9,6 +9,7 @@ ADD scripts /scripts # Install Python, pip, and C dev libraries necessary to compile the most popular # Python libraries. RUN /scripts/install-apt-packages.sh +RUN curl "https://bootstrap.pypa.io/pip/2.7/get-pip.py" -o "get-pip.py" && python ./get-pip.py && ln -s /usr/local/bin/pip /usr/bin/pip # Setup locale. This prevents Python 3 IO encoding issues. ENV LANG C.UTF-8 diff --git a/runtime-image/resources/apt-packages.txt b/runtime-image/resources/apt-packages.txt index 287f7abc..7b88f777 100644 --- a/runtime-image/resources/apt-packages.txt +++ b/runtime-image/resources/apt-packages.txt @@ -4,7 +4,6 @@ mercurial pkg-config wget # debian-provided interpreters -python-pip python2.7 python2.7-dev # Dependenies for third-party Python packages From b4754dd62596fa97499adde2085bc855571d5e57 Mon Sep 17 00:00:00 2001 From: Matthew Suozzo Date: Mon, 15 Aug 2022 17:21:19 -0400 Subject: [PATCH 096/132] Propagate env vars to sudo invocations. --- scripts/deploy_check.sh | 4 ++-- scripts/integration-test.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/deploy_check.sh b/scripts/deploy_check.sh index 77fccb54..1e2f02c2 100644 --- a/scripts/deploy_check.sh +++ b/scripts/deploy_check.sh @@ -12,11 +12,11 @@ fi cd ${KOKORO_GFILE_DIR}/appengine/integration_tests -sudo /usr/local/bin/pip install --upgrade -r requirements.txt +sudo -E /usr/local/bin/pip install --upgrade -r requirements.txt if [ -f ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY}/requirements.txt ] then - sudo /usr/local/bin/pip install --upgrade -r ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY}/requirements.txt + sudo -E /usr/local/bin/pip install --upgrade -r ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY}/requirements.txt fi export DEPLOY_LATENCY_PROJECT='cloud-deploy-latency' diff --git a/scripts/integration-test.sh b/scripts/integration-test.sh index 63137b5f..6210e0a1 100644 --- a/scripts/integration-test.sh +++ b/scripts/integration-test.sh @@ -7,11 +7,11 @@ source ${KOKORO_GFILE_DIR}/kokoro/common.sh export GOOGLE_CLOUD_PROJECT=gcp-runtimes -sudo /usr/local/bin/pip install --upgrade -r ${KOKORO_GFILE_DIR}/appengine/integration_tests/requirements.txt +sudo -E /usr/local/bin/pip install --upgrade -r ${KOKORO_GFILE_DIR}/appengine/integration_tests/requirements.txt if [ -f ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY}/requirements.txt ] then - sudo /usr/local/bin/pip install --upgrade -r ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY}/requirements.txt + sudo -E /usr/local/bin/pip install --upgrade -r ${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY}/requirements.txt fi export GOPATH=${KOKORO_GITHUB_DIR}/${SAMPLE_APP_DIRECTORY} From cc0da57364aec352e15dfa203b939a41b6cb7faa Mon Sep 17 00:00:00 2001 From: jinglundong <1683035+jinglundong@users.noreply.github.com> Date: Tue, 16 Aug 2022 16:21:39 -0700 Subject: [PATCH 097/132] Update CODEOWNERS --- CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CODEOWNERS b/CODEOWNERS index 5deacc6c..2b618c5f 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1,4 +1,4 @@ # Code owners file. # This file controls who is tagged for review for any given pull request. -* @dlorenc @sharifelgamal @tstromberg @donmccasland +* @jinglundong @donmccasland From 647d1a1344ef9bfc3ce560c4190ba3f2fb2b9903 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Aug 2023 21:52:18 +0000 Subject: [PATCH 098/132] Bump tornado from 5.1 to 6.3.3 in /tests/python2-libraries Bumps [tornado](https://github.com/tornadoweb/tornado) from 5.1 to 6.3.3. - [Changelog](https://github.com/tornadoweb/tornado/blob/master/docs/releases.rst) - [Commits](https://github.com/tornadoweb/tornado/compare/v5.1.0...v6.3.3) --- updated-dependencies: - dependency-name: tornado dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python2-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 865cd1aa..964eaf9e 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -184,7 +184,7 @@ supervisor==3.3.4 testrepository==0.0.20 testtools==2.3.0 thrift==0.11.0 -tornado==5.1 +tornado==6.3.3 tox==3.2.1 twisted==18.7.0 ujson==1.35 From 07b6f1ab2faee152ab2d4bef1dfc41863cfe752f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Aug 2023 21:53:13 +0000 Subject: [PATCH 099/132] Bump tornado from 5.1 to 6.3.3 in /tests/python3-libraries Bumps [tornado](https://github.com/tornadoweb/tornado) from 5.1 to 6.3.3. - [Changelog](https://github.com/tornadoweb/tornado/blob/master/docs/releases.rst) - [Commits](https://github.com/tornadoweb/tornado/compare/v5.1.0...v6.3.3) --- updated-dependencies: - dependency-name: tornado dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python3-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index e3fd6424..c0dc6a66 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -172,7 +172,7 @@ stevedore==1.29.0 testrepository==0.0.20 testtools==2.3.0 thrift==0.11.0 -tornado==5.1 +tornado==6.3.3 tox==3.2.1 twisted==18.7.0 ujson==1.35 From 8f6ff95c279eb8313b6f559c7e2b39d1f71bb2a5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 00:44:38 +0000 Subject: [PATCH 100/132] Bump pygments from 2.2.0 to 2.15.0 in /tests/python3-libraries Bumps [pygments](https://github.com/pygments/pygments) from 2.2.0 to 2.15.0. - [Release notes](https://github.com/pygments/pygments/releases) - [Changelog](https://github.com/pygments/pygments/blob/master/CHANGES) - [Commits](https://github.com/pygments/pygments/compare/2.2.0...2.15.0) --- updated-dependencies: - dependency-name: pygments dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python3-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index c0dc6a66..ccd826d4 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -123,7 +123,7 @@ pyasn1==0.4.4 pycparser==2.18 pycrypto==2.6.1 pyflakes==2.0.0 -pygments==2.2.0 +pygments==2.15.0 pyjwt==1.6.4 pylibmc==1.5.2 pylint==2.1.1 From 2f6f1a4fcd14bc52377692a66013bf85f7e518b2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 00:44:39 +0000 Subject: [PATCH 101/132] Bump cryptography from 2.3.1 to 41.0.3 in /tests/python3-libraries Bumps [cryptography](https://github.com/pyca/cryptography) from 2.3.1 to 41.0.3. - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/2.3.1...41.0.3) --- updated-dependencies: - dependency-name: cryptography dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python3-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index c0dc6a66..f761e959 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -30,7 +30,7 @@ cov-core==1.15.0 coverage==4.5.1 coveralls==1.4.0 crcmod==1.7 -cryptography==2.3.1 +cryptography==41.0.3 cssselect==1.0.3 cython==0.28.5 decorator==4.3.0 From f6682ba419bf583c5a34b258cb455159c67640a9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 00:44:43 +0000 Subject: [PATCH 102/132] Bump scipy from 1.1.0 to 1.10.0 in /tests/python3-libraries Bumps [scipy](https://github.com/scipy/scipy) from 1.1.0 to 1.10.0. - [Release notes](https://github.com/scipy/scipy/releases) - [Commits](https://github.com/scipy/scipy/compare/v1.1.0...v1.10.0) --- updated-dependencies: - dependency-name: scipy dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python3-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index c0dc6a66..58839946 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -154,7 +154,7 @@ requests-oauthlib==1.0.0 requests==2.19.1 retrying==1.3.3 rsa==3.4.2 -scipy==1.1.0 +scipy==1.10.0 selenium==3.14.0 setuptools-git==1.2 setuptools==40.2.0 From 0a395be4f042ac9517d35a1cac13d99ca65caa84 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 00:44:45 +0000 Subject: [PATCH 103/132] Bump certifi from 2018.8.24 to 2023.7.22 in /tests/python2-libraries Bumps [certifi](https://github.com/certifi/python-certifi) from 2018.8.24 to 2023.7.22. - [Commits](https://github.com/certifi/python-certifi/compare/2018.08.24...2023.07.22) --- updated-dependencies: - dependency-name: certifi dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python2-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 865cd1aa..bb083cde 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -20,7 +20,7 @@ botocore==1.11.1 bottle==0.12.13 carbon<1.1.1 celery==4.2.1 -certifi==2018.8.24 +certifi==2023.7.22 cffi==1.11.5 chardet==3.0.4 click==6.7 From b383ecd9913c3550e5a55602bc682a61f84c7781 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 00:44:45 +0000 Subject: [PATCH 104/132] Bump certifi from 2018.8.24 to 2023.7.22 in /tests/python3-libraries Bumps [certifi](https://github.com/certifi/python-certifi) from 2018.8.24 to 2023.7.22. - [Commits](https://github.com/certifi/python-certifi/compare/2018.08.24...2023.07.22) --- updated-dependencies: - dependency-name: certifi dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python3-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index c0dc6a66..8497d1d9 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -18,7 +18,7 @@ boto==2.49.0 botocore==1.11.1 bottle==0.12.13 celery==4.2.1 -certifi==2018.8.24 +certifi==2023.7.22 cffi==1.11.5 chardet==3.0.4 click==6.7 From 19ccc8a8be75537635ca00e280f71ee4fed4fbcd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 06:18:22 +0000 Subject: [PATCH 105/132] Bump requests from 2.19.1 to 2.31.0 in /tests/python3-libraries Bumps [requests](https://github.com/psf/requests) from 2.19.1 to 2.31.0. - [Release notes](https://github.com/psf/requests/releases) - [Changelog](https://github.com/psf/requests/blob/main/HISTORY.md) - [Commits](https://github.com/psf/requests/compare/v2.19.1...v2.31.0) --- updated-dependencies: - dependency-name: requests dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python3-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 71dab860..3568a7f3 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -151,7 +151,7 @@ raven==6.9.0 redis==2.10.6 repoze.lru==0.7 requests-oauthlib==1.0.0 -requests==2.19.1 +requests==2.31.0 retrying==1.3.3 rsa==3.4.2 scipy==1.1.0 From e1854082720277c75cdb5ef63ab386b314c2d2a6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 06:18:41 +0000 Subject: [PATCH 106/132] Bump scipy from 1.1.0 to 1.10.0 in /tests/python2-libraries Bumps [scipy](https://github.com/scipy/scipy) from 1.1.0 to 1.10.0. - [Release notes](https://github.com/scipy/scipy/releases) - [Commits](https://github.com/scipy/scipy/compare/v1.1.0...v1.10.0) --- updated-dependencies: - dependency-name: scipy dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python2-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 63d371ec..dace11da 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -164,7 +164,7 @@ requests-oauthlib==1.0.0 requests==2.19.1 retrying==1.3.3 rsa==3.4.2 -scipy==1.1.0 +scipy==1.10.0 selenium==3.14.0 setuptools-git==1.2 setuptools==40.2.0 From a3673f8691058c3c068dc2f42e9b2cb9804b441e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 06:18:58 +0000 Subject: [PATCH 107/132] Bump flask from 1.1.0 to 2.2.5 in /tests/integration Bumps [flask](https://github.com/pallets/flask) from 1.1.0 to 2.2.5. - [Release notes](https://github.com/pallets/flask/releases) - [Changelog](https://github.com/pallets/flask/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/flask/compare/1.1.0...2.2.5) --- updated-dependencies: - dependency-name: flask dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/integration/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index ea714c20..0e31f608 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -1,4 +1,4 @@ -Flask==1.1.0 +Flask==2.2.5 google-cloud-error-reporting==0.32.1 google-cloud-logging==1.12.1 google-cloud-monitoring==0.33.0 From 4cd3ee423bb00a17f27255f8f579ba110cf933e1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 06:19:23 +0000 Subject: [PATCH 108/132] Bump flask from 1.0.2 to 2.2.5 in /tests/python3-libraries Bumps [flask](https://github.com/pallets/flask) from 1.0.2 to 2.2.5. - [Release notes](https://github.com/pallets/flask/releases) - [Changelog](https://github.com/pallets/flask/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/flask/compare/1.0.2...2.2.5) --- updated-dependencies: - dependency-name: flask dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python3-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index c48b246e..24727967 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -51,7 +51,7 @@ extras==1.0.0 fabric==2.3.1 fixtures==3.0.0 flake8==3.5.0 -flask==1.0.2 +flask==2.2.5 funcsigs==1.0.2 gevent==1.3.6 google-api-python-client==1.7.4 From f3e034b17cd7d2558f0a4ee12a6f5599c6c645b5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 06:19:44 +0000 Subject: [PATCH 109/132] Bump requests from 2.19.1 to 2.31.0 in /tests/python2-libraries Bumps [requests](https://github.com/psf/requests) from 2.19.1 to 2.31.0. - [Release notes](https://github.com/psf/requests/releases) - [Changelog](https://github.com/psf/requests/blob/main/HISTORY.md) - [Commits](https://github.com/psf/requests/compare/v2.19.1...v2.31.0) --- updated-dependencies: - dependency-name: requests dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python2-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index dace11da..5db7a568 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -161,7 +161,7 @@ raven==6.9.0 redis==2.10.6 repoze.lru==0.7 requests-oauthlib==1.0.0 -requests==2.19.1 +requests==2.31.0 retrying==1.3.3 rsa==3.4.2 scipy==1.10.0 From a1dbe236ef109d8c88902a5fdb68d15990370c05 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 06:20:01 +0000 Subject: [PATCH 110/132] Bump requests from 2.22.0 to 2.31.0 in /tests/integration Bumps [requests](https://github.com/psf/requests) from 2.22.0 to 2.31.0. - [Release notes](https://github.com/psf/requests/releases) - [Changelog](https://github.com/psf/requests/blob/main/HISTORY.md) - [Commits](https://github.com/psf/requests/compare/v2.22.0...v2.31.0) --- updated-dependencies: - dependency-name: requests dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/integration/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index ea714c20..45782f06 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -3,7 +3,7 @@ google-cloud-error-reporting==0.32.1 google-cloud-logging==1.12.1 google-cloud-monitoring==0.33.0 gunicorn==19.9.0 -requests==2.22.0 +requests==2.31.0 retrying==1.3.3 six==1.12.0 protobuf>=3.6.0 From 257c0f560f110a5d555f7df657d38e09df4887a2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 06:21:34 +0000 Subject: [PATCH 111/132] Bump flask from 1.0.2 to 2.2.5 in /tests/python2-libraries Bumps [flask](https://github.com/pallets/flask) from 1.0.2 to 2.2.5. - [Release notes](https://github.com/pallets/flask/releases) - [Changelog](https://github.com/pallets/flask/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/flask/compare/1.0.2...2.2.5) --- updated-dependencies: - dependency-name: flask dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python2-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 5db7a568..92da6052 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -53,7 +53,7 @@ extras==1.0.0 fabric==2.3.1 fixtures==3.0.0 flake8==3.5.0 -flask==1.0.2 +flask==2.2.5 funcsigs==1.0.2 functools32==3.2.3.post2 futures==3.2.0 From c5e53836a50440973b14727de05399e35d20cd08 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 06:21:38 +0000 Subject: [PATCH 112/132] Bump sqlparse from 0.2.4 to 0.4.4 in /tests/python3-libraries Bumps [sqlparse](https://github.com/andialbrecht/sqlparse) from 0.2.4 to 0.4.4. - [Changelog](https://github.com/andialbrecht/sqlparse/blob/master/CHANGELOG) - [Commits](https://github.com/andialbrecht/sqlparse/compare/0.2.4...0.4.4) --- updated-dependencies: - dependency-name: sqlparse dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python3-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 82263711..1d6b37da 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -166,7 +166,7 @@ south==1.0.2 sphinx==1.7.7 sqlalchemy-migrate==0.11.0 sqlalchemy==1.2.11 -sqlparse==0.2.4 +sqlparse==0.4.4 statsd==3.3.0 stevedore==1.29.0 testrepository==0.0.20 From fd2cdb556238254b9b3201a48e7b1fafb12ce5a5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 06:21:51 +0000 Subject: [PATCH 113/132] Bump flask from 1.0.2 to 2.2.5 in /tests/eventlet Bumps [flask](https://github.com/pallets/flask) from 1.0.2 to 2.2.5. - [Release notes](https://github.com/pallets/flask/releases) - [Changelog](https://github.com/pallets/flask/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/flask/compare/1.0.2...2.2.5) --- updated-dependencies: - dependency-name: flask dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/eventlet/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/eventlet/requirements.txt b/tests/eventlet/requirements.txt index 65eabc74..75ee7c7d 100644 --- a/tests/eventlet/requirements.txt +++ b/tests/eventlet/requirements.txt @@ -1,7 +1,7 @@ click==6.7 enum-compat==0.0.2 eventlet==0.24.1 -Flask==1.0.2 +Flask==2.2.5 greenlet==0.4.14 gunicorn==19.9.0 itsdangerous==0.24 From abd0aa21189209da39fe79b4559fddaafe950a14 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 06:22:32 +0000 Subject: [PATCH 114/132] Bump urllib3 from 1.23 to 1.26.5 in /tests/python3-libraries Bumps [urllib3](https://github.com/urllib3/urllib3) from 1.23 to 1.26.5. - [Release notes](https://github.com/urllib3/urllib3/releases) - [Changelog](https://github.com/urllib3/urllib3/blob/main/CHANGES.rst) - [Commits](https://github.com/urllib3/urllib3/compare/1.23...1.26.5) --- updated-dependencies: - dependency-name: urllib3 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python3-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 82263711..06341ddd 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -179,7 +179,7 @@ ujson==1.35 unidecode==1.0.22 unittest2==1.1.0 uritemplate==3.0.0 -urllib3==1.23 +urllib3==1.26.5 uwsgi==2.0.17.1 versiontools==1.9.1 virtualenv==16.0.0 From 38e4e9d397da6d3e36f39dbcdec1da9501bc6f50 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 06:23:08 +0000 Subject: [PATCH 115/132] Bump ansible from 2.6.3 to 7.0.0 in /tests/python3-libraries Bumps [ansible](https://github.com/ansible/ansible) from 2.6.3 to 7.0.0. - [Release notes](https://github.com/ansible/ansible/releases) - [Commits](https://github.com/ansible/ansible/commits) --- updated-dependencies: - dependency-name: ansible dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python3-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 82263711..89b1a196 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -1,7 +1,7 @@ alembic==1.0.0 amqp==2.3.2 amqplib==1.0.2 -ansible==2.6.3 +ansible==7.0.0 anyjson==0.3.3 apache-libcloud==2.3.0 argparse==1.4.0 From 33570888d9e3359a02f70500c4fcf97a520f3dc9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 06:24:15 +0000 Subject: [PATCH 116/132] Bump sqlparse from 0.2.4 to 0.4.4 in /tests/python2-libraries Bumps [sqlparse](https://github.com/andialbrecht/sqlparse) from 0.2.4 to 0.4.4. - [Changelog](https://github.com/andialbrecht/sqlparse/blob/master/CHANGELOG) - [Commits](https://github.com/andialbrecht/sqlparse/compare/0.2.4...0.4.4) --- updated-dependencies: - dependency-name: sqlparse dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python2-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 92da6052..f3b9daec 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -176,7 +176,7 @@ south==1.0.2 sphinx==1.7.7 sqlalchemy-migrate==0.11.0 sqlalchemy==1.2.11 -sqlparse==0.2.4 +sqlparse==0.4.4 statsd==3.3.0 stevedore==1.29.0 suds==0.4 From 06f1cfa49fcb057c0440afb2d32868683061f494 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 06:24:41 +0000 Subject: [PATCH 117/132] Bump waitress from 1.1.0 to 2.1.2 in /tests/python3-libraries Bumps [waitress](https://github.com/Pylons/waitress) from 1.1.0 to 2.1.2. - [Release notes](https://github.com/Pylons/waitress/releases) - [Changelog](https://github.com/Pylons/waitress/blob/v2.1.2/CHANGES.txt) - [Commits](https://github.com/Pylons/waitress/compare/v1.1.0...v2.1.2) --- updated-dependencies: - dependency-name: waitress dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python3-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 4929dcc6..e8adc88f 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -183,7 +183,7 @@ urllib3==1.26.5 uwsgi==2.0.17.1 versiontools==1.9.1 virtualenv==16.0.0 -waitress==1.1.0 +waitress==2.1.2 warlock==1.3.0 webob==1.8.2 websocket-client==0.51.0 From b901f8f3ccbda6fcfec23082b4218bd2834d0a33 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 06:25:52 +0000 Subject: [PATCH 118/132] Bump pillow from 5.2.0 to 9.3.0 in /tests/python3-libraries Bumps [pillow](https://github.com/python-pillow/Pillow) from 5.2.0 to 9.3.0. - [Release notes](https://github.com/python-pillow/Pillow/releases) - [Changelog](https://github.com/python-pillow/Pillow/blob/main/CHANGES.rst) - [Commits](https://github.com/python-pillow/Pillow/compare/5.2.0...9.3.0) --- updated-dependencies: - dependency-name: pillow dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python3-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 0989646e..66e9b98e 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -111,7 +111,7 @@ pbr==4.2.0 pep8==1.7.1 pexpect==4.6.0 pika==0.12.0 -pillow==5.2.0 +pillow==9.3.0 pip==18.0 prettytable==0.7.2 protobuf==3.6.1 From f12acd1bc138fdeeedfcf042aab38661a88a57da Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 06:25:54 +0000 Subject: [PATCH 119/132] Bump pyyaml from 3.13 to 5.4 in /tests/python3-libraries Bumps [pyyaml](https://github.com/yaml/pyyaml) from 3.13 to 5.4. - [Changelog](https://github.com/yaml/pyyaml/blob/master/CHANGES) - [Commits](https://github.com/yaml/pyyaml/compare/3.13...5.4) --- updated-dependencies: - dependency-name: pyyaml dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python3-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 0989646e..f4d40d0e 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -145,7 +145,7 @@ python-novaclient==11.0.0 python-subunit==1.3.0 python-swiftclient==3.6.0 pytz==2018.5 -pyyaml==3.13 +pyyaml==5.4 pyzmq==17.1.2 raven==6.9.0 redis==2.10.6 From ce681862182eb79ab31d0f8f4162246b98c70dff Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 06:25:58 +0000 Subject: [PATCH 120/132] Bump numpy from 1.15.1 to 1.22.0 in /tests/python3-libraries Bumps [numpy](https://github.com/numpy/numpy) from 1.15.1 to 1.22.0. - [Release notes](https://github.com/numpy/numpy/releases) - [Changelog](https://github.com/numpy/numpy/blob/main/doc/RELEASE_WALKTHROUGH.rst) - [Commits](https://github.com/numpy/numpy/compare/v1.15.1...v1.22.0) --- updated-dependencies: - dependency-name: numpy dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python3-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 0989646e..9e70346b 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -95,7 +95,7 @@ netaddr==0.7.19 netifaces==0.10.7 newrelic==4.2.0.100 nose==1.3.7 -numpy==1.15.1 +numpy==1.22.0 oauth2==1.9.0.post1 oauth2client==4.1.2 oauthlib==2.1.0 From cf7bce61d466e477b8a0963a38a6d4199867c088 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 06:26:35 +0000 Subject: [PATCH 121/132] Bump django from 2.1 to 2.2.28 in /tests/python3-libraries Bumps [django](https://github.com/django/django) from 2.1 to 2.2.28. - [Commits](https://github.com/django/django/compare/2.1...2.2.28) --- updated-dependencies: - dependency-name: django dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python3-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index 0989646e..31938505 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -37,7 +37,7 @@ decorator==4.3.0 django-celery==3.2.2 django-debug-toolbar==1.9.1 django-extensions==2.1.1 -django==2.1 +django==2.2.28 django_compress==1.0.1 djangorestframework==3.8.2 docker-py==1.10.6 From 78b72a16af2a8b4270505443061b850a3d749a79 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 18:22:59 +0000 Subject: [PATCH 122/132] Bump urllib3 from 1.23 to 1.26.5 in /tests/python2-libraries Bumps [urllib3](https://github.com/urllib3/urllib3) from 1.23 to 1.26.5. - [Release notes](https://github.com/urllib3/urllib3/releases) - [Changelog](https://github.com/urllib3/urllib3/blob/main/CHANGES.rst) - [Commits](https://github.com/urllib3/urllib3/compare/1.23...1.26.5) --- updated-dependencies: - dependency-name: urllib3 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python2-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index f3b9daec..2eaf5ba0 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -191,7 +191,7 @@ ujson==1.35 unidecode==1.0.22 unittest2==1.1.0 uritemplate==3.0.0 -urllib3==1.23 +urllib3==1.26.5 uwsgi==2.0.17.1 versiontools==1.9.1 virtualenv==16.0.0 From 2758277ba791b3629cfa54b6dd0553de8d729a79 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 18:23:18 +0000 Subject: [PATCH 123/132] Bump ansible from 2.6.3 to 7.0.0 in /tests/python2-libraries Bumps [ansible](https://github.com/ansible/ansible) from 2.6.3 to 7.0.0. - [Release notes](https://github.com/ansible/ansible/releases) - [Commits](https://github.com/ansible/ansible/commits) --- updated-dependencies: - dependency-name: ansible dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python2-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index f3b9daec..2795af2e 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -1,7 +1,7 @@ alembic==1.0.0 amqp==2.3.2 amqplib==1.0.2 -ansible==2.6.3 +ansible==7.0.0 anyjson==0.3.3 apache-libcloud==2.3.0 argparse==1.4.0 From c36d1079fc62e162b45020bc3abcdf70469c7cd8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 18:26:45 +0000 Subject: [PATCH 124/132] Bump waitress from 1.1.0 to 2.1.2 in /tests/python2-libraries Bumps [waitress](https://github.com/Pylons/waitress) from 1.1.0 to 2.1.2. - [Release notes](https://github.com/Pylons/waitress/releases) - [Changelog](https://github.com/Pylons/waitress/blob/v2.1.2/CHANGES.txt) - [Commits](https://github.com/Pylons/waitress/compare/v1.1.0...v2.1.2) --- updated-dependencies: - dependency-name: waitress dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python2-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index f3b9daec..09feaa44 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -195,7 +195,7 @@ urllib3==1.23 uwsgi==2.0.17.1 versiontools==1.9.1 virtualenv==16.0.0 -waitress==1.1.0 +waitress==2.1.2 warlock==1.3.0 webob==1.8.2 websocket-client==0.51.0 From 49e19e4b726a1950da60fabaa40cbce2a838a09b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 20:11:30 +0000 Subject: [PATCH 125/132] Bump numpy from 1.15.1 to 1.22.0 in /tests/python2-libraries Bumps [numpy](https://github.com/numpy/numpy) from 1.15.1 to 1.22.0. - [Release notes](https://github.com/numpy/numpy/releases) - [Changelog](https://github.com/numpy/numpy/blob/main/doc/RELEASE_WALKTHROUGH.rst) - [Commits](https://github.com/numpy/numpy/compare/v1.15.1...v1.22.0) --- updated-dependencies: - dependency-name: numpy dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python2-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 6bb9fc86..b2ecb1dd 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -103,7 +103,7 @@ netaddr==0.7.19 netifaces==0.10.7 newrelic==4.2.0.100 nose==1.3.7 -numpy==1.15.1 +numpy==1.22.0 oauth2==1.9.0.post1 oauth2client==4.1.2 oauthlib==2.1.0 From d12a82f5651abbc5c7f3a653aa8f909633e26f4e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 20:11:36 +0000 Subject: [PATCH 126/132] Bump pyyaml from 3.13 to 5.4 in /tests/python2-libraries Bumps [pyyaml](https://github.com/yaml/pyyaml) from 3.13 to 5.4. - [Changelog](https://github.com/yaml/pyyaml/blob/master/CHANGES) - [Commits](https://github.com/yaml/pyyaml/compare/3.13...5.4) --- updated-dependencies: - dependency-name: pyyaml dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python2-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 6bb9fc86..67b81897 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -155,7 +155,7 @@ python-novaclient==11.0.0 python-subunit==1.3.0 python-swiftclient==3.6.0 pytz==2018.5 -pyyaml==3.13 +pyyaml==5.4 pyzmq==17.1.2 raven==6.9.0 redis==2.10.6 From 4dbda4cfd7e58de057d291078e5143d421460eff Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 20:12:22 +0000 Subject: [PATCH 127/132] Bump pillow from 5.2.0 to 9.3.0 in /tests/python2-libraries Bumps [pillow](https://github.com/python-pillow/Pillow) from 5.2.0 to 9.3.0. - [Release notes](https://github.com/python-pillow/Pillow/releases) - [Changelog](https://github.com/python-pillow/Pillow/blob/main/CHANGES.rst) - [Commits](https://github.com/python-pillow/Pillow/compare/5.2.0...9.3.0) --- updated-dependencies: - dependency-name: pillow dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python2-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 6bb9fc86..cfbaab8b 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -119,7 +119,7 @@ pbr==4.2.0 pep8==1.7.1 pexpect==4.6.0 pika==0.12.0 -pillow==5.2.0 +pillow==9.3.0 pip==18.0 prettytable==0.7.2 protobuf==3.6.1 From 6c6077f604056fcf7f2383db23a82699e7c8450d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 16 Aug 2023 00:38:49 +0000 Subject: [PATCH 128/132] Bump werkzeug from 0.14.1 to 2.2.3 in /tests/python3-libraries Bumps [werkzeug](https://github.com/pallets/werkzeug) from 0.14.1 to 2.2.3. - [Release notes](https://github.com/pallets/werkzeug/releases) - [Changelog](https://github.com/pallets/werkzeug/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/werkzeug/compare/0.14.1...2.2.3) --- updated-dependencies: - dependency-name: werkzeug dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python3-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index d21238fd..d9117d4f 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -188,7 +188,7 @@ warlock==1.3.0 webob==1.8.2 websocket-client==0.51.0 webtest==2.0.30 -werkzeug==0.14.1 +werkzeug==2.2.3 wheel==0.31.1 xlrd==1.1.0 zc.buildout==2.12.1 From 17d4165fa40213904382fa9cdda5a067e2b86d5a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 16 Aug 2023 00:39:06 +0000 Subject: [PATCH 129/132] Bump werkzeug from 0.14.1 to 2.2.3 in /tests/python2-libraries Bumps [werkzeug](https://github.com/pallets/werkzeug) from 0.14.1 to 2.2.3. - [Release notes](https://github.com/pallets/werkzeug/releases) - [Changelog](https://github.com/pallets/werkzeug/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/werkzeug/compare/0.14.1...2.2.3) --- updated-dependencies: - dependency-name: werkzeug dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python2-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 29bdca44..67a0c0b3 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -200,7 +200,7 @@ warlock==1.3.0 webob==1.8.2 websocket-client==0.51.0 webtest==2.0.30 -werkzeug==0.14.1 +werkzeug==2.2.3 wheel==0.31.1 xlrd==1.1.0 zc.buildout==2.12.1 From 56de3a3edea2f3653ec018d874cb0f8cda9aa67d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 16 Aug 2023 00:40:22 +0000 Subject: [PATCH 130/132] Bump werkzeug from 0.14.1 to 2.2.3 in /tests/eventlet Bumps [werkzeug](https://github.com/pallets/werkzeug) from 0.14.1 to 2.2.3. - [Release notes](https://github.com/pallets/werkzeug/releases) - [Changelog](https://github.com/pallets/werkzeug/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/werkzeug/compare/0.14.1...2.2.3) --- updated-dependencies: - dependency-name: werkzeug dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/eventlet/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/eventlet/requirements.txt b/tests/eventlet/requirements.txt index 75ee7c7d..1e2ea66b 100644 --- a/tests/eventlet/requirements.txt +++ b/tests/eventlet/requirements.txt @@ -7,4 +7,4 @@ gunicorn==19.9.0 itsdangerous==0.24 Jinja2==2.10 MarkupSafe==1.1.1 -Werkzeug==0.14.1 +Werkzeug==2.2.3 From 461f8883bbd586319ffa8e1442858607c1473640 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 24 Aug 2023 20:11:21 +0000 Subject: [PATCH 131/132] Bump uwsgi from 2.0.17.1 to 2.0.22 in /tests/python2-libraries Bumps [uwsgi](https://github.com/unbit/uwsgi-docs) from 2.0.17.1 to 2.0.22. - [Commits](https://github.com/unbit/uwsgi-docs/commits) --- updated-dependencies: - dependency-name: uwsgi dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python2-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python2-libraries/requirements.txt b/tests/python2-libraries/requirements.txt index 67a0c0b3..f0596d67 100644 --- a/tests/python2-libraries/requirements.txt +++ b/tests/python2-libraries/requirements.txt @@ -192,7 +192,7 @@ unidecode==1.0.22 unittest2==1.1.0 uritemplate==3.0.0 urllib3==1.26.5 -uwsgi==2.0.17.1 +uwsgi==2.0.22 versiontools==1.9.1 virtualenv==16.0.0 waitress==2.1.2 From 5b017c9ecc9422ba6e959bc28d712ab1a4d4353d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 24 Aug 2023 20:11:33 +0000 Subject: [PATCH 132/132] Bump uwsgi from 2.0.17.1 to 2.0.22 in /tests/python3-libraries Bumps [uwsgi](https://github.com/unbit/uwsgi-docs) from 2.0.17.1 to 2.0.22. - [Commits](https://github.com/unbit/uwsgi-docs/commits) --- updated-dependencies: - dependency-name: uwsgi dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/python3-libraries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python3-libraries/requirements.txt b/tests/python3-libraries/requirements.txt index d9117d4f..23175a06 100644 --- a/tests/python3-libraries/requirements.txt +++ b/tests/python3-libraries/requirements.txt @@ -180,7 +180,7 @@ unidecode==1.0.22 unittest2==1.1.0 uritemplate==3.0.0 urllib3==1.26.5 -uwsgi==2.0.17.1 +uwsgi==2.0.22 versiontools==1.9.1 virtualenv==16.0.0 waitress==2.1.2