From acdca32addb61dd3e080a137b587759c99428c61 Mon Sep 17 00:00:00 2001 From: Stephen Hoover Date: Mon, 1 May 2017 13:32:49 -0500 Subject: [PATCH 01/63] [CIVP-10979] Update installed package versions (#28) - python 3.6.0 -> 3.6.1 - boto 2.45.0 -> 2.46.1 - boto3 1.4.3 -> 1.4.4 - numpy 1.12.0 -> 1.12.1 - pubnub 4.0.8 -> 4.0.10 - requests 2.12.4 -> 2.13.0 - scipy 0.18.1 -> 0.19.0 - muffnn 1.0.0 -> 1.1.1 - tensorflow 1.0.0 -> 1.1.0 * Make version of botocore consistent Each increment of `awscli` seems to require a different version of `botocore`. Make the versions consistent between the conda-installed `botocore` and what's required by `awscli`. --- CHANGELOG.md | 17 +++++++++++++++++ Dockerfile | 6 +++--- environment.yml | 21 +++++++++++---------- 3 files changed, 31 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 738fe07..2c1d2f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,9 +8,26 @@ Version number changes (major.minor.micro) in this package denote the following: - A major version will increase if there are any backwards-incompatible changes in any of the packages contained in this Docker image, or any other backwards-incompabile changes in the execution environment. ## Unreleased + + +## [2.2.0] - 2017-05-02 ### Removed - Remove pinned conda installs of `libgcc` and `libsodium`. This prevented use of the environment file in OS X, and they are dependencies automatically installed by conda in the Docker image build. +### Additions +- Explicitly added `botocore` v1.5.38. We had `botocore` installed before (it's a dependency of other AWS libraries), but we're now explicitly including the version number. + +### Package updates +- python 3.6.0 -> 3.6.1 +- awscli 1.11.60 -> 1.11.75 +- boto 2.45.0 -> 2.46.1 +- boto3 1.4.3 -> 1.4.4 +- numpy 1.12.0 -> 1.12.1 +- pubnub 4.0.8 -> 4.0.10 +- requests 2.12.4 -> 2.13.0 +- scipy 0.18.1 -> 0.19.0 +- muffnn 1.0.0 -> 1.1.1 +- tensorflow 1.0.0 -> 1.1.0 ## [2.1.0] - 2017-03-17 ### Changed diff --git a/Dockerfile b/Dockerfile index e892a38..6e08fd6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ ENV LANG=en_US.UTF-8 \ BASH_ENV=/etc/profile \ PATH=/opt/conda/bin:$PATH \ CIVIS_CONDA_VERSION=4.3.11 \ - CIVIS_PYTHON_VERSION=3.6.0 + CIVIS_PYTHON_VERSION=3.6.1 RUN DEBIAN_FRONTEND=noninteractive apt-get update -y --no-install-recommends && \ apt-get install -y --no-install-recommends software-properties-common && \ @@ -92,7 +92,7 @@ RUN mkdir -p ${HOME}/.config/matplotlib && \ echo "backend : Agg" > ${HOME}/.config/matplotlib/matplotlibrc && \ python -c "import matplotlib.pyplot" -ENV VERSION=2.1.0 \ +ENV VERSION=2.2.0 \ VERSION_MAJOR=2 \ - VERSION_MINOR=1 \ + VERSION_MINOR=2 \ VERSION_MICRO=0 diff --git a/environment.yml b/environment.yml index 0540b2a..1b3dc12 100644 --- a/environment.yml +++ b/environment.yml @@ -1,8 +1,9 @@ name: datascience dependencies: - beautifulsoup4=4.5.3 -- boto=2.45.0 -- boto3==1.4.3 +- botocore=1.5.38 +- boto=2.46.1 +- boto3==1.4.4 - cython=0.25.2 - ipython=5.1.0 - jinja2=2.8 @@ -16,32 +17,32 @@ dependencies: - nomkl=1.0 - nose=1.3.7 - numexpr=2.6.2 -- numpy=1.12.0 +- numpy=1.12.1 - openblas=0.2.19 - pandas=0.19.2 - patsy=0.4.1 - psycopg2=2.6.2 - pycrypto=2.6.1 - pytest=3.0.5 -- python=3.6.0 +- python=3.6.1 - pyyaml=3.12 -- requests=2.12.4 +- requests=2.13.0 - seaborn=0.7.1 -- scipy=0.18.1 +- scipy=0.19.0 - scikit-learn=0.18.1 - statsmodels=0.8.0 - pip: - - awscli==1.11.60 + - awscli==1.11.75 - civis==1.4.0 - dropbox==7.1.1 - ftputil==3.3.1 - glmnet==2.0.0 - joblib==0.11.0 - - muffnn==1.0.0 - - pubnub==4.0.8 + - muffnn==1.1.1 + - pubnub==4.0.10 - pysftp==0.2.9 - python-simple-hipchat==0.4.0 - requests-toolbelt==0.7.1 - - tensorflow==1.0.0 + - tensorflow==1.1.0 - urllib3==1.19 - xgboost==0.6a2 From 963592a92e968b31a6d0b798d65730119a51f128 Mon Sep 17 00:00:00 2001 From: Keith Ingersoll Date: Thu, 11 May 2017 09:47:48 -0500 Subject: [PATCH 02/63] Updating environment.yml to latest civis version --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 1b3dc12..28dc306 100644 --- a/environment.yml +++ b/environment.yml @@ -33,7 +33,7 @@ dependencies: - statsmodels=0.8.0 - pip: - awscli==1.11.75 - - civis==1.4.0 + - civis==1.5.0 - dropbox==7.1.1 - ftputil==3.3.1 - glmnet==2.0.0 From 404c9ab9525c1868b9097fa471b8a6531a27db6e Mon Sep 17 00:00:00 2001 From: Stephen Hoover Date: Mon, 15 May 2017 12:15:40 -0500 Subject: [PATCH 03/63] ENH Update packages with new versions (#30) - ipython 5.1.0 -> 6.0.0 - matplotlib 2.0.0 -> 2.0.2 - pandas 0.19.2 -> 0.20.1 - requests 2.13.0 -> 2.14.2 --- CHANGELOG.md | 6 ++++++ environment.yml | 8 ++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c1d2f3..5718198 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,12 @@ Version number changes (major.minor.micro) in this package denote the following: ## Unreleased +### Package updates +- civis 1.4.0 -> 1.5.0 +- ipython 5.1.0 -> 6.0.0 +- matplotlib 2.0.0 -> 2.0.2 +- pandas 0.19.2 -> 0.20.1 +- requests 2.13.0 -> 2.14.2 ## [2.2.0] - 2017-05-02 ### Removed diff --git a/environment.yml b/environment.yml index 28dc306..5f32426 100644 --- a/environment.yml +++ b/environment.yml @@ -5,7 +5,7 @@ dependencies: - boto=2.46.1 - boto3==1.4.4 - cython=0.25.2 -- ipython=5.1.0 +- ipython=6.0.0 - jinja2=2.8 - jsonschema=2.5.1 - jupyter=1.0.0 @@ -13,20 +13,20 @@ dependencies: - libgfortran=3.0.0 - libtiff=4.0.6 - libxml2=2.9.2 -- matplotlib=2.0.0 +- matplotlib=2.0.2 - nomkl=1.0 - nose=1.3.7 - numexpr=2.6.2 - numpy=1.12.1 - openblas=0.2.19 -- pandas=0.19.2 +- pandas=0.20.1 - patsy=0.4.1 - psycopg2=2.6.2 - pycrypto=2.6.1 - pytest=3.0.5 - python=3.6.1 - pyyaml=3.12 -- requests=2.13.0 +- requests=2.14.2 - seaborn=0.7.1 - scipy=0.19.0 - scikit-learn=0.18.1 From 91e74dcdd48016880f86634dad6ff5b47b4b5fce Mon Sep 17 00:00:00 2001 From: Keith Ingersoll Date: Mon, 15 May 2017 18:01:14 -0500 Subject: [PATCH 04/63] Updating civis to version 1.5.1 --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 5f32426..23e56b8 100644 --- a/environment.yml +++ b/environment.yml @@ -33,7 +33,7 @@ dependencies: - statsmodels=0.8.0 - pip: - awscli==1.11.75 - - civis==1.5.0 + - civis==1.5.1 - dropbox==7.1.1 - ftputil==3.3.1 - glmnet==2.0.0 From 5077972a2d715be584560af2c9e8fc114bf7218c Mon Sep 17 00:00:00 2001 From: Stephen Hoover Date: Tue, 16 May 2017 10:31:14 -0500 Subject: [PATCH 05/63] MAINT Update version number to 3.0.0 (#31) Prepare for next release --- CHANGELOG.md | 1 + Dockerfile | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5718198..0f3fdcb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ Version number changes (major.minor.micro) in this package denote the following: ## Unreleased +## [3.0.0] - 2017-05-16 ### Package updates - civis 1.4.0 -> 1.5.0 - ipython 5.1.0 -> 6.0.0 diff --git a/Dockerfile b/Dockerfile index 6e08fd6..7ef63d0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -92,7 +92,7 @@ RUN mkdir -p ${HOME}/.config/matplotlib && \ echo "backend : Agg" > ${HOME}/.config/matplotlib/matplotlibrc && \ python -c "import matplotlib.pyplot" -ENV VERSION=2.2.0 \ - VERSION_MAJOR=2 \ - VERSION_MINOR=2 \ +ENV VERSION=3.0.0 \ + VERSION_MAJOR=3 \ + VERSION_MINOR=0 \ VERSION_MICRO=0 From d21c424523e5131f9b60cd087f90d83e22bf11f9 Mon Sep 17 00:00:00 2001 From: Stephen Hoover Date: Tue, 16 May 2017 10:55:50 -0500 Subject: [PATCH 06/63] DOC Correct civis-python version number in changelog (#33) We're on v1.5.1 of `civis`, as of PR #32 . --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f3fdcb..04dc407 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ Version number changes (major.minor.micro) in this package denote the following: ## [3.0.0] - 2017-05-16 ### Package updates -- civis 1.4.0 -> 1.5.0 +- civis 1.4.0 -> 1.5.1 - ipython 5.1.0 -> 6.0.0 - matplotlib 2.0.0 -> 2.0.2 - pandas 0.19.2 -> 0.20.1 From bf8202d43cd35ccb28db74af82cce9ce6987cf7c Mon Sep 17 00:00:00 2001 From: Keith Ingersoll Date: Wed, 17 May 2017 17:10:12 -0500 Subject: [PATCH 07/63] Update civis-python to latest version (#34) * Update civis-python to latest version * Adding civis 1.5.2 change to v3.0.0 * Typo, adding in missing space --- CHANGELOG.md | 4 ++-- environment.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 04dc407..87def8f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,9 +9,9 @@ Version number changes (major.minor.micro) in this package denote the following: ## Unreleased -## [3.0.0] - 2017-05-16 +## [3.0.0] - 2017-05-17 ### Package updates -- civis 1.4.0 -> 1.5.1 +- civis 1.4.0 -> 1.5.2 - ipython 5.1.0 -> 6.0.0 - matplotlib 2.0.0 -> 2.0.2 - pandas 0.19.2 -> 0.20.1 diff --git a/environment.yml b/environment.yml index 23e56b8..997e9da 100644 --- a/environment.yml +++ b/environment.yml @@ -33,7 +33,7 @@ dependencies: - statsmodels=0.8.0 - pip: - awscli==1.11.75 - - civis==1.5.1 + - civis==1.5.2 - dropbox==7.1.1 - ftputil==3.3.1 - glmnet==2.0.0 From e8e81a3a1238c5a2d80ff801cc1c6d46c758df37 Mon Sep 17 00:00:00 2001 From: "Matthew R. Becker" Date: Thu, 25 May 2017 09:54:34 -0500 Subject: [PATCH 08/63] PROD updated muffnn to 1.1.2 (#36) --- CHANGELOG.md | 4 ++++ Dockerfile | 4 ++-- environment.yml | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 87def8f..4d3beb9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ Version number changes (major.minor.micro) in this package denote the following: ## Unreleased +## [3.0.1] - 2017-05-25 +### Package Updates +- muffnn 1.1.1 -> 1.1.2 + ## [3.0.0] - 2017-05-17 ### Package updates - civis 1.4.0 -> 1.5.2 diff --git a/Dockerfile b/Dockerfile index 7ef63d0..e01aa31 100644 --- a/Dockerfile +++ b/Dockerfile @@ -92,7 +92,7 @@ RUN mkdir -p ${HOME}/.config/matplotlib && \ echo "backend : Agg" > ${HOME}/.config/matplotlib/matplotlibrc && \ python -c "import matplotlib.pyplot" -ENV VERSION=3.0.0 \ +ENV VERSION=3.0.1 \ VERSION_MAJOR=3 \ VERSION_MINOR=0 \ - VERSION_MICRO=0 + VERSION_MICRO=1 diff --git a/environment.yml b/environment.yml index 997e9da..4a1e8da 100644 --- a/environment.yml +++ b/environment.yml @@ -38,7 +38,7 @@ dependencies: - ftputil==3.3.1 - glmnet==2.0.0 - joblib==0.11.0 - - muffnn==1.1.1 + - muffnn==1.1.2 - pubnub==4.0.10 - pysftp==0.2.9 - python-simple-hipchat==0.4.0 From 5289ae8fc4986f94afe06689564fb3e603c76c50 Mon Sep 17 00:00:00 2001 From: Stephen Hoover Date: Tue, 20 Jun 2017 09:40:58 -0500 Subject: [PATCH 09/63] BUG Use /tmp for joblib temp files instead of /shm (#37) Instruct joblib to use disk for temporary files. Joblib defaults to /shm when that directory is present. In the Docker container, /shm is present but defaults to 64 MB. That's not enough space to handle even moderate-sized computations. The temporary directory is a bit slower, but has much more space. https://github.com/joblib/joblib/blob/0.11/joblib/parallel.py#L328L342 --- CHANGELOG.md | 4 ++++ Dockerfile | 6 ++++++ README.md | 9 +++++++++ 3 files changed, 19 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d3beb9..15dff7c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ Version number changes (major.minor.micro) in this package denote the following: ## Unreleased +### Fixes +- Use /tmp for joblib temporary files instead of /shm + + ## [3.0.1] - 2017-05-25 ### Package Updates - muffnn 1.1.1 -> 1.1.2 diff --git a/Dockerfile b/Dockerfile index e01aa31..6ad9510 100644 --- a/Dockerfile +++ b/Dockerfile @@ -92,6 +92,12 @@ RUN mkdir -p ${HOME}/.config/matplotlib && \ echo "backend : Agg" > ${HOME}/.config/matplotlib/matplotlibrc && \ python -c "import matplotlib.pyplot" +# Instruct joblib to use disk for temporary files. Joblib defaults to +# /shm when that directory is present. In the Docker container, /shm is +# present but defaults to 64 MB. +# https://github.com/joblib/joblib/blob/0.11/joblib/parallel.py#L328L342 +ENV JOBLIB_TEMP_FOLDER=/tmp + ENV VERSION=3.0.1 \ VERSION_MAJOR=3 \ VERSION_MINOR=0 \ diff --git a/README.md b/README.md index a084091..5a7b7c7 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,15 @@ VERSION_MICRO VERSION contains the full version string, e.g. "1.0.3". VERSION_MAJOR, VERSION_MINOR, and VERSION_MICRO each contain a single integer. +## Joblib Temporary Files + +The [`joblib`](https://pythonhosted.org/joblib/) library enhances multiprocessing +capabilities for scientific Python computing. In particular, the `scikit-learn` +library uses `joblib` for parallelization. This Docker image sets `joblib`'s +default location for staging temporary files to the /tmp directory. +The normal default is /shm. /shm is a RAM disk which defaults to a 64 MB size +in Docker containers, too small for typical scientific computing. + # Creating Equivalent Local Environments The `environment.yml` file in this repo can be used to create a python environment that is From f7496d238478e24b437c169aafde36687661fd08 Mon Sep 17 00:00:00 2001 From: Stephen Hoover Date: Fri, 28 Jul 2017 08:22:38 -0500 Subject: [PATCH 10/63] DEP Update package versions; add pyarrow and cloudpickle (#38) - Python v3.6.1 -> v3.6.2 - cloudpickle 0.3.1 - pyarrow 0.5.0 (from conda-forge) - civis 1.5.2 -> 1.6.0 - cython 0.25.2 -> 0.26 - ipython 6.0.0 -> 6.1.0 - jinja2 2.8 -> 2.9.6 - numpy 1.12.1 -> 1.13.1 - pandas 0.20.1 -> 0.20.3 - pytest 3.0.5 -> 3.1.3 - seaborn 0.7.1 -> 0.8 - scipy 0.19.0 -> 0.19.1 - scikit-learn 0.18.1 -> 0.18.2 - pubnub 4.0.10 -> 4.0.12 - requests-toolbelt 0.7.1 -> 0.8.0 - tensorflow 1.1.0 -> 1.2.1 Also install `xgboost` from conda-forge instead of from PyPI, and add a test to verify that we aren't getting more than expected from conda-forge. --- CHANGELOG.md | 25 +++++++++++++++++++++++++ Dockerfile | 2 +- circle.yml | 1 + environment.yml | 35 ++++++++++++++++++++--------------- 4 files changed, 47 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 15dff7c..cb287cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,31 @@ Version number changes (major.minor.micro) in this package denote the following: ## Unreleased +### New packages +- cloudpickle 0.3.1 +- pyarrow 0.5.0 (from conda-forge) + +### Python +- Update from v3.6.1 to v3.6.2 + +### Package Updates +- civis 1.5.2 -> 1.6.0 +- cython 0.25.2 -> 0.26 +- ipython 6.0.0 -> 6.1.0 +- jinja2 2.8 -> 2.9.6 +- numpy 1.12.1 -> 1.13.1 +- pandas 0.20.1 -> 0.20.3 +- pytest 3.0.5 -> 3.1.3 +- seaborn 0.7.1 -> 0.8 +- scipy 0.19.0 -> 0.19.1 +- scikit-learn 0.18.1 -> 0.18.2 +- pubnub 4.0.10 -> 4.0.12 +- requests-toolbelt 0.7.1 -> 0.8.0 +- tensorflow 1.1.0 -> 1.2.1 + +### Changed +- Install xgboost from conda-forge instead of from PyPI + ### Fixes - Use /tmp for joblib temporary files instead of /shm diff --git a/Dockerfile b/Dockerfile index 6ad9510..ebc95a6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ ENV LANG=en_US.UTF-8 \ BASH_ENV=/etc/profile \ PATH=/opt/conda/bin:$PATH \ CIVIS_CONDA_VERSION=4.3.11 \ - CIVIS_PYTHON_VERSION=3.6.1 + CIVIS_PYTHON_VERSION=3.6.2 RUN DEBIAN_FRONTEND=noninteractive apt-get update -y --no-install-recommends && \ apt-get install -y --no-install-recommends software-properties-common && \ diff --git a/circle.yml b/circle.yml index 3ea7fa4..c9db539 100644 --- a/circle.yml +++ b/circle.yml @@ -15,3 +15,4 @@ test: - docker run -t civisanalytics/datascience-python /bin/bash -c "python -c 'import numpy'" - docker run civisanalytics/datascience-python python -c "from numpy.distutils import system_info; assert system_info.get_info('mkl') == {}" - docker run civisanalytics/datascience-python python -c "import numpy; numpy.test()" + - docker run civisanalytics/datascience-python /bin/bash -c "conda list|grep -c conda-forge|python -c \"import sys; assert int(sys.stdin.readlines()[0]) == 5, 'There should be 5 conda-forge packages'\"" diff --git a/environment.yml b/environment.yml index 4a1e8da..d7ae978 100644 --- a/environment.yml +++ b/environment.yml @@ -1,12 +1,15 @@ name: datascience +channels: +- defaults +- conda-forge dependencies: - beautifulsoup4=4.5.3 - botocore=1.5.38 - boto=2.46.1 - boto3==1.4.4 -- cython=0.25.2 -- ipython=6.0.0 -- jinja2=2.8 +- cython=0.26 +- ipython=6.1.0 +- jinja2=2.9.6 - jsonschema=2.5.1 - jupyter=1.0.0 - libffi=3.2.1 @@ -17,32 +20,34 @@ dependencies: - nomkl=1.0 - nose=1.3.7 - numexpr=2.6.2 -- numpy=1.12.1 +- numpy=1.13.1 - openblas=0.2.19 -- pandas=0.20.1 +- pandas=0.20.3 - patsy=0.4.1 - psycopg2=2.6.2 +- pyarrow=0.5.0 - pycrypto=2.6.1 -- pytest=3.0.5 -- python=3.6.1 +- pytest=3.1.3 +- python=3.6.2 - pyyaml=3.12 - requests=2.14.2 -- seaborn=0.7.1 -- scipy=0.19.0 -- scikit-learn=0.18.1 +- seaborn=0.8 +- scipy=0.19.1 +- scikit-learn=0.18.2 - statsmodels=0.8.0 +- xgboost=0.6a2 - pip: - awscli==1.11.75 - - civis==1.5.2 + - civis==1.6.0 + - cloudpickle==0.3.1 - dropbox==7.1.1 - ftputil==3.3.1 - glmnet==2.0.0 - joblib==0.11.0 - muffnn==1.1.2 - - pubnub==4.0.10 + - pubnub==4.0.12 - pysftp==0.2.9 - python-simple-hipchat==0.4.0 - - requests-toolbelt==0.7.1 - - tensorflow==1.1.0 + - requests-toolbelt==0.8.0 + - tensorflow==1.2.1 - urllib3==1.19 - - xgboost==0.6a2 From 8007f919e97de04f187e61bcd5c5de1497c2909b Mon Sep 17 00:00:00 2001 From: Stephen Hoover Date: Mon, 31 Jul 2017 08:48:03 -0500 Subject: [PATCH 11/63] MAINT Increase version number to 3.1.0 (#39) Prepare for v3.1.0 release. --- CHANGELOG.md | 1 + Dockerfile | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cb287cf..3c228b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ Version number changes (major.minor.micro) in this package denote the following: ## Unreleased +## [3.1.0] - 2017-07-31 ### New packages - cloudpickle 0.3.1 - pyarrow 0.5.0 (from conda-forge) diff --git a/Dockerfile b/Dockerfile index ebc95a6..a38574d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -98,7 +98,7 @@ RUN mkdir -p ${HOME}/.config/matplotlib && \ # https://github.com/joblib/joblib/blob/0.11/joblib/parallel.py#L328L342 ENV JOBLIB_TEMP_FOLDER=/tmp -ENV VERSION=3.0.1 \ +ENV VERSION=3.1.0 \ VERSION_MAJOR=3 \ - VERSION_MINOR=0 \ - VERSION_MICRO=1 + VERSION_MINOR=1 \ + VERSION_MICRO=0 From 2e90b7b341cde9a582ca6dbc8421a2607705c75a Mon Sep 17 00:00:00 2001 From: Elizabeth Sander Date: Tue, 15 Aug 2017 13:43:49 -0500 Subject: [PATCH 12/63] [CIVP-12352] DEP update sklearn version (#40) --- CHANGELOG.md | 2 ++ environment.yml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c228b0..c296b8f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ Version number changes (major.minor.micro) in this package denote the following: - A major version will increase if there are any backwards-incompatible changes in any of the packages contained in this Docker image, or any other backwards-incompabile changes in the execution environment. ## Unreleased +### Package Updates +- scikit-learn 0.18.2 -> 0.19.0 ## [3.1.0] - 2017-07-31 ### New packages diff --git a/environment.yml b/environment.yml index d7ae978..23b63dc 100644 --- a/environment.yml +++ b/environment.yml @@ -33,7 +33,7 @@ dependencies: - requests=2.14.2 - seaborn=0.8 - scipy=0.19.1 -- scikit-learn=0.18.2 +- scikit-learn=0.19.0 - statsmodels=0.8.0 - xgboost=0.6a2 - pip: From bbf110b6bd3102d26c023497edde50bff0bcc7a8 Mon Sep 17 00:00:00 2001 From: "Matthew R. Becker" Date: Tue, 22 Aug 2017 13:38:23 -0500 Subject: [PATCH 13/63] ENH update the civis version (#41) --- CHANGELOG.md | 1 + environment.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c296b8f..f2d216d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ Version number changes (major.minor.micro) in this package denote the following: ## Unreleased ### Package Updates - scikit-learn 0.18.2 -> 0.19.0 +- civis 1.6.0 -> 1.6.1 ## [3.1.0] - 2017-07-31 ### New packages diff --git a/environment.yml b/environment.yml index 23b63dc..da5eb41 100644 --- a/environment.yml +++ b/environment.yml @@ -38,7 +38,7 @@ dependencies: - xgboost=0.6a2 - pip: - awscli==1.11.75 - - civis==1.6.0 + - civis==1.6.1 - cloudpickle==0.3.1 - dropbox==7.1.1 - ftputil==3.3.1 From 0aecfdbf7760743f9151e11de38f73c182911d27 Mon Sep 17 00:00:00 2001 From: Stephen Hoover Date: Fri, 8 Sep 2017 09:04:01 -0500 Subject: [PATCH 14/63] MAINT Increase client version to v1.6.2 (#43) Incorporate latest bugfix release of civis-python. --- CHANGELOG.md | 2 +- environment.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f2d216d..795ca21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ Version number changes (major.minor.micro) in this package denote the following: ## Unreleased ### Package Updates - scikit-learn 0.18.2 -> 0.19.0 -- civis 1.6.0 -> 1.6.1 +- civis 1.6.0 -> 1.6.2 ## [3.1.0] - 2017-07-31 ### New packages diff --git a/environment.yml b/environment.yml index da5eb41..a30f57d 100644 --- a/environment.yml +++ b/environment.yml @@ -38,7 +38,7 @@ dependencies: - xgboost=0.6a2 - pip: - awscli==1.11.75 - - civis==1.6.1 + - civis==1.6.2 - cloudpickle==0.3.1 - dropbox==7.1.1 - ftputil==3.3.1 From 19a0e5a7012d1d7142d479ed392f7b963efebee3 Mon Sep 17 00:00:00 2001 From: "Jackson L. Lee" Date: Mon, 11 Sep 2017 09:05:25 -0500 Subject: [PATCH 15/63] [CIVP-12592] DEP requests -> 2.18.4 and urllib3 -> 1.22 (#42) * DEP requests -> 2.18.4, use pip install instead * MAINT Update changelog * DEP urllib3 -> 1.22 because of requests 2.18.4 warning * Revert "DEP requests -> 2.18.4, use pip install instead" This reverts commit d3b9760167c23c585db1418824774b8be7e6e55a. * MAINT Update changelog for urllib3 -> 1.22 * DEP requests -> 2.18.4 * MAINT Update build test for conda-forge package count --- CHANGELOG.md | 4 +++- circle.yml | 3 ++- environment.yml | 4 ++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 795ca21..d0baf60 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,12 +5,14 @@ All changes to this project will be documented in this file. Version number changes (major.minor.micro) in this package denote the following: - A micro version will increase if the only change in a release is incrementing micro versions (bugfix-only releases) on the packages contained in this image. - A minor version will increase if one or more packages contained in the Docker image add new, backwards-compatible features, or if a new package is added to the Docker image. -- A major version will increase if there are any backwards-incompatible changes in any of the packages contained in this Docker image, or any other backwards-incompabile changes in the execution environment. +- A major version will increase if there are any backwards-incompatible changes in any of the packages contained in this Docker image, or any other backwards-incompatible changes in the execution environment. ## Unreleased ### Package Updates - scikit-learn 0.18.2 -> 0.19.0 - civis 1.6.0 -> 1.6.2 +- requests 2.14.2 -> 2.18.4 +- urllib3 1.19 -> 1.22 ## [3.1.0] - 2017-07-31 ### New packages diff --git a/circle.yml b/circle.yml index c9db539..7c0e8b6 100644 --- a/circle.yml +++ b/circle.yml @@ -15,4 +15,5 @@ test: - docker run -t civisanalytics/datascience-python /bin/bash -c "python -c 'import numpy'" - docker run civisanalytics/datascience-python python -c "from numpy.distutils import system_info; assert system_info.get_info('mkl') == {}" - docker run civisanalytics/datascience-python python -c "import numpy; numpy.test()" - - docker run civisanalytics/datascience-python /bin/bash -c "conda list|grep -c conda-forge|python -c \"import sys; assert int(sys.stdin.readlines()[0]) == 5, 'There should be 5 conda-forge packages'\"" + - docker run civisanalytics/datascience-python /bin/bash -c "conda list|grep conda-forge" + - docker run civisanalytics/datascience-python /bin/bash -c "conda list|grep -c conda-forge|python -c \"import sys; actual_count = int(sys.stdin.readlines()[0]); expected_count = 7; assert actual_count == expected_count, 'There should be %d conda-forge packages, but there are actually %d.' % (expected_count, actual_count)\"" diff --git a/environment.yml b/environment.yml index a30f57d..72b0472 100644 --- a/environment.yml +++ b/environment.yml @@ -30,7 +30,7 @@ dependencies: - pytest=3.1.3 - python=3.6.2 - pyyaml=3.12 -- requests=2.14.2 +- requests=2.18.4 - seaborn=0.8 - scipy=0.19.1 - scikit-learn=0.19.0 @@ -50,4 +50,4 @@ dependencies: - python-simple-hipchat==0.4.0 - requests-toolbelt==0.8.0 - tensorflow==1.2.1 - - urllib3==1.19 + - urllib3==1.22 From 9fe6ea8aa77f7d61660fe14e9d22f322d0f28a66 Mon Sep 17 00:00:00 2001 From: Stephen Hoover Date: Mon, 11 Sep 2017 09:58:51 -0500 Subject: [PATCH 16/63] MAINT Update version to 3.2.0 (#44) --- CHANGELOG.md | 3 +++ Dockerfile | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d0baf60..a11b6d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,9 @@ Version number changes (major.minor.micro) in this package denote the following: - A major version will increase if there are any backwards-incompatible changes in any of the packages contained in this Docker image, or any other backwards-incompatible changes in the execution environment. ## Unreleased + + +## [3.2.0] - 2017-09-11 ### Package Updates - scikit-learn 0.18.2 -> 0.19.0 - civis 1.6.0 -> 1.6.2 diff --git a/Dockerfile b/Dockerfile index a38574d..e3c8830 100644 --- a/Dockerfile +++ b/Dockerfile @@ -98,7 +98,7 @@ RUN mkdir -p ${HOME}/.config/matplotlib && \ # https://github.com/joblib/joblib/blob/0.11/joblib/parallel.py#L328L342 ENV JOBLIB_TEMP_FOLDER=/tmp -ENV VERSION=3.1.0 \ +ENV VERSION=3.2.0 \ VERSION_MAJOR=3 \ - VERSION_MINOR=1 \ + VERSION_MINOR=2 \ VERSION_MICRO=0 From 6e11a629832f945659ba1b0b4714c1ca3aeec595 Mon Sep 17 00:00:00 2001 From: "Matthew R. Becker" Date: Thu, 14 Sep 2017 10:24:07 -0500 Subject: [PATCH 17/63] ENH added civisml-extensions (#45) --- CHANGELOG.md | 3 ++- environment.yml | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a11b6d0..d0904eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,8 @@ Version number changes (major.minor.micro) in this package denote the following: - A major version will increase if there are any backwards-incompatible changes in any of the packages contained in this Docker image, or any other backwards-incompatible changes in the execution environment. ## Unreleased - +### New packages +- civisml-extensions 0.1.1 ## [3.2.0] - 2017-09-11 ### Package Updates diff --git a/environment.yml b/environment.yml index 72b0472..3691f72 100644 --- a/environment.yml +++ b/environment.yml @@ -39,6 +39,7 @@ dependencies: - pip: - awscli==1.11.75 - civis==1.6.2 + - civisml-extensions==0.1.1 - cloudpickle==0.3.1 - dropbox==7.1.1 - ftputil==3.3.1 From 96dcc58c4f337b8925edc23f31437e4d3a0744f7 Mon Sep 17 00:00:00 2001 From: Stephen Hoover Date: Thu, 16 Nov 2017 13:51:56 -0600 Subject: [PATCH 18/63] DEP Update package versions (#49) - boto3 1.4.4 -> 1.4.5 - matplotlib 2.0.2 -> 2.1.0 - numpy 1.13.1 -> 1.13.3 - pandas 0.20.3 -> 0.21.0 - pyarrow 0.5.0 -> 0.7.1 - scikit-learn 0.19.0 -> 0.19.1 - cloudpickle 0.3.1 -> 0.5.1 - muffnn 1.1.2 -> 1.2.0 - pubnub 4.0.12 -> 4.0.13 - tensorflow 1.2.1 -> 1.4.0 Also upgrade conda to 4.3.30 (latest miniconda version) to avoid a conda bug which prevented package installation. Add new packages dask v0.15.4 and s3fs v0.1.2. --- CHANGELOG.md | 18 ++++++++++++++++++ Dockerfile | 2 +- circle.yml | 2 +- environment.yml | 22 ++++++++++++---------- 4 files changed, 32 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d0904eb..87a1930 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,8 +8,26 @@ Version number changes (major.minor.micro) in this package denote the following: - A major version will increase if there are any backwards-incompatible changes in any of the packages contained in this Docker image, or any other backwards-incompatible changes in the execution environment. ## Unreleased + ### New packages - civisml-extensions 0.1.1 +- dask 0.15.4 +- s3fs 0.1.2 + +### Changed +- Moved conda to version 4.3.30 + +### Package Updates +- boto3 1.4.4 -> 1.4.5 +- matplotlib 2.0.2 -> 2.1.0 +- numpy 1.13.1 -> 1.13.3 +- pandas 0.20.3 -> 0.21.0 +- pyarrow 0.5.0 -> 0.7.1 +- scikit-learn 0.19.0 -> 0.19.1 +- cloudpickle 0.3.1 -> 0.5.1 +- muffnn 1.1.2 -> 1.2.0 +- pubnub 4.0.12 -> 4.0.13 +- tensorflow 1.2.1 -> 1.4.0 ## [3.2.0] - 2017-09-11 ### Package Updates diff --git a/Dockerfile b/Dockerfile index e3c8830..4aeab5c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,7 @@ ENV LANG=en_US.UTF-8 \ CONDARC=/opt/conda/.condarc \ BASH_ENV=/etc/profile \ PATH=/opt/conda/bin:$PATH \ - CIVIS_CONDA_VERSION=4.3.11 \ + CIVIS_CONDA_VERSION=4.3.30 \ CIVIS_PYTHON_VERSION=3.6.2 RUN DEBIAN_FRONTEND=noninteractive apt-get update -y --no-install-recommends && \ diff --git a/circle.yml b/circle.yml index 7c0e8b6..af28e54 100644 --- a/circle.yml +++ b/circle.yml @@ -16,4 +16,4 @@ test: - docker run civisanalytics/datascience-python python -c "from numpy.distutils import system_info; assert system_info.get_info('mkl') == {}" - docker run civisanalytics/datascience-python python -c "import numpy; numpy.test()" - docker run civisanalytics/datascience-python /bin/bash -c "conda list|grep conda-forge" - - docker run civisanalytics/datascience-python /bin/bash -c "conda list|grep -c conda-forge|python -c \"import sys; actual_count = int(sys.stdin.readlines()[0]); expected_count = 7; assert actual_count == expected_count, 'There should be %d conda-forge packages, but there are actually %d.' % (expected_count, actual_count)\"" + - docker run civisanalytics/datascience-python /bin/bash -c "conda list|grep -c conda-forge|python -c \"import sys; actual_count = int(sys.stdin.readlines()[0]); expected_count = 9; assert actual_count == expected_count, 'There should be %d conda-forge packages, but there are actually %d.' % (expected_count, actual_count)\"" diff --git a/environment.yml b/environment.yml index 3691f72..32a5963 100644 --- a/environment.yml +++ b/environment.yml @@ -6,7 +6,7 @@ dependencies: - beautifulsoup4=4.5.3 - botocore=1.5.38 - boto=2.46.1 -- boto3==1.4.4 +- boto3==1.4.5 - cython=0.26 - ipython=6.1.0 - jinja2=2.9.6 @@ -16,39 +16,41 @@ dependencies: - libgfortran=3.0.0 - libtiff=4.0.6 - libxml2=2.9.2 -- matplotlib=2.0.2 +- matplotlib=2.1.0 - nomkl=1.0 - nose=1.3.7 - numexpr=2.6.2 -- numpy=1.13.1 +- numpy=1.13.3 - openblas=0.2.19 -- pandas=0.20.3 +- pandas=0.21.0 - patsy=0.4.1 - psycopg2=2.6.2 -- pyarrow=0.5.0 +- pyarrow=0.7.1 - pycrypto=2.6.1 - pytest=3.1.3 - python=3.6.2 - pyyaml=3.12 - requests=2.18.4 +- s3fs=0.1.2 - seaborn=0.8 - scipy=0.19.1 -- scikit-learn=0.19.0 +- scikit-learn=0.19.1 - statsmodels=0.8.0 - xgboost=0.6a2 - pip: - awscli==1.11.75 - civis==1.6.2 - civisml-extensions==0.1.1 - - cloudpickle==0.3.1 + - cloudpickle==0.5.1 + - dask==0.15.4 - dropbox==7.1.1 - ftputil==3.3.1 - glmnet==2.0.0 - joblib==0.11.0 - - muffnn==1.1.2 - - pubnub==4.0.12 + - muffnn==1.2.0 + - pubnub==4.0.13 - pysftp==0.2.9 - python-simple-hipchat==0.4.0 - requests-toolbelt==0.8.0 - - tensorflow==1.2.1 + - tensorflow==1.4.0 - urllib3==1.22 From 6746cf5f18a96b1a37c887c9616a248ff1d483c1 Mon Sep 17 00:00:00 2001 From: Elizabeth Sander Date: Fri, 17 Nov 2017 15:53:45 -0600 Subject: [PATCH 19/63] V3.3.0 (#48) --- CHANGELOG.md | 6 ++++-- Dockerfile | 4 ++-- environment.yml | 4 ++-- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 87a1930..a564231 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,10 +7,12 @@ Version number changes (major.minor.micro) in this package denote the following: - A minor version will increase if one or more packages contained in the Docker image add new, backwards-compatible features, or if a new package is added to the Docker image. - A major version will increase if there are any backwards-incompatible changes in any of the packages contained in this Docker image, or any other backwards-incompatible changes in the execution environment. -## Unreleased +## [3.3.0] - 2017-11-17 +### Package Updates +- civis 1.6.2 -> 1.7.1 ### New packages -- civisml-extensions 0.1.1 +- civisml-extensions 0.1.5 - dask 0.15.4 - s3fs 0.1.2 diff --git a/Dockerfile b/Dockerfile index 4aeab5c..82f86f5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -98,7 +98,7 @@ RUN mkdir -p ${HOME}/.config/matplotlib && \ # https://github.com/joblib/joblib/blob/0.11/joblib/parallel.py#L328L342 ENV JOBLIB_TEMP_FOLDER=/tmp -ENV VERSION=3.2.0 \ +ENV VERSION=3.3.0 \ VERSION_MAJOR=3 \ - VERSION_MINOR=2 \ + VERSION_MINOR=3 \ VERSION_MICRO=0 diff --git a/environment.yml b/environment.yml index 32a5963..112e3c8 100644 --- a/environment.yml +++ b/environment.yml @@ -39,8 +39,8 @@ dependencies: - xgboost=0.6a2 - pip: - awscli==1.11.75 - - civis==1.6.2 - - civisml-extensions==0.1.1 + - civis==1.7.1 + - civisml-extensions==0.1.5 - cloudpickle==0.5.1 - dask==0.15.4 - dropbox==7.1.1 From 1cf3cea465529529cd9f1c85922f822a321e773f Mon Sep 17 00:00:00 2001 From: Michael Heilman Date: Tue, 19 Dec 2017 12:01:20 -0600 Subject: [PATCH 20/63] this makes sure that jupyter widgets are appropriately enabled in the image (they did not work before). also adds bqplot --- Dockerfile | 5 +++++ environment.yml | 3 +++ 2 files changed, 8 insertions(+) diff --git a/Dockerfile b/Dockerfile index 82f86f5..7905b67 100644 --- a/Dockerfile +++ b/Dockerfile @@ -92,6 +92,11 @@ RUN mkdir -p ${HOME}/.config/matplotlib && \ echo "backend : Agg" > ${HOME}/.config/matplotlib/matplotlibrc && \ python -c "import matplotlib.pyplot" +# Enable widgetsnbextension for jupyter widgets. +# See https://ipywidgets.readthedocs.io/en/stable/user_install.html. +# This enables the extension in the conda environment. +RUN jupyter nbextension enable --py widgetsnbextension + # Instruct joblib to use disk for temporary files. Joblib defaults to # /shm when that directory is present. In the Docker container, /shm is # present but defaults to 64 MB. diff --git a/environment.yml b/environment.yml index 112e3c8..aa4094a 100644 --- a/environment.yml +++ b/environment.yml @@ -7,8 +7,10 @@ dependencies: - botocore=1.5.38 - boto=2.46.1 - boto3==1.4.5 +- bqplot=0.10.1 - cython=0.26 - ipython=6.1.0 +- ipywidgets=7.0.5 - jinja2=2.9.6 - jsonschema=2.5.1 - jupyter=1.0.0 @@ -18,6 +20,7 @@ dependencies: - libxml2=2.9.2 - matplotlib=2.1.0 - nomkl=1.0 +- notebook=5.2.2 - nose=1.3.7 - numexpr=2.6.2 - numpy=1.13.3 From 2c2a97b8439e1be0b26270eeb40e7316d9e19a55 Mon Sep 17 00:00:00 2001 From: Michael Heilman Date: Tue, 19 Dec 2017 12:21:39 -0600 Subject: [PATCH 21/63] change expected conda forge package count (default channel appears to have more packages now) Also updates the changelog. 3.3.0 output: > conda list|grep conda-forge arrow-cpp 0.7.1 py36_2 conda-forge backports.functools_lru_cache 1.4 py36_1 conda-forge fontconfig 2.12.1 5 conda-forge freetype 2.7 2 conda-forge matplotlib 2.1.0 py36_1 conda-forge parquet-cpp 1.3.0.post 2 conda-forge pyarrow 0.7.1 py36_1 conda-forge qt 5.6.2 6 conda-forge xgboost 0.6a2 py36_2 conda-forge current output: > conda list |grep conda-forge arrow-cpp 0.7.1 py36_2 conda-forge bqplot 0.10.1 py36_0 conda-forge fontconfig 2.12.6 0 conda-forge freetype 2.8.1 0 conda-forge parquet-cpp 1.3.0.post 2 conda-forge pyarrow 0.7.1 py36_1 conda-forge traittypes 0.0.6 py36_0 conda-forge xgboost 0.6a2 py36_2 conda-forge --- CHANGELOG.md | 11 +++++++++++ circle.yml | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a564231..d896b7b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,17 @@ Version number changes (major.minor.micro) in this package denote the following: - A minor version will increase if one or more packages contained in the Docker image add new, backwards-compatible features, or if a new package is added to the Docker image. - A major version will increase if there are any backwards-incompatible changes in any of the packages contained in this Docker image, or any other backwards-incompatible changes in the execution environment. +## Unreleased +### New packages +- bqplot 0.10.1 + +### Package Updates +- ipywidgets 7.0.0 -> 7.0.5 +- notebook 5.2.0 -> 5.2.1 + +### Fixed +- Enabled widgetsnbextension so that ipywidgets works. + ## [3.3.0] - 2017-11-17 ### Package Updates - civis 1.6.2 -> 1.7.1 diff --git a/circle.yml b/circle.yml index af28e54..8734a1f 100644 --- a/circle.yml +++ b/circle.yml @@ -16,4 +16,4 @@ test: - docker run civisanalytics/datascience-python python -c "from numpy.distutils import system_info; assert system_info.get_info('mkl') == {}" - docker run civisanalytics/datascience-python python -c "import numpy; numpy.test()" - docker run civisanalytics/datascience-python /bin/bash -c "conda list|grep conda-forge" - - docker run civisanalytics/datascience-python /bin/bash -c "conda list|grep -c conda-forge|python -c \"import sys; actual_count = int(sys.stdin.readlines()[0]); expected_count = 9; assert actual_count == expected_count, 'There should be %d conda-forge packages, but there are actually %d.' % (expected_count, actual_count)\"" + - docker run civisanalytics/datascience-python /bin/bash -c "conda list|grep -c conda-forge|python -c \"import sys; actual_count = int(sys.stdin.readlines()[0]); expected_count = 8; assert actual_count == expected_count, 'There should be %d conda-forge packages, but there are actually %d.' % (expected_count, actual_count)\"" From 3f480c29b14a8ffd615e2ed23f68accfb74f5978 Mon Sep 17 00:00:00 2001 From: Michael Heilman Date: Thu, 28 Dec 2017 14:32:09 -0600 Subject: [PATCH 22/63] update ipywidgets and bqplot versions, remove unneeded widgetsnbextension enable call Also update conda-forge count to 10: > conda list|grep conda-forge arrow-cpp 0.7.1 py36_2 conda-forge bqplot 0.10.2 py36_0 conda-forge fontconfig 2.12.6 0 conda-forge freetype 2.8.1 0 conda-forge ipywidgets 7.1.0 py36_0 conda-forge parquet-cpp 1.3.0.post 2 conda-forge pyarrow 0.7.1 py36_1 conda-forge traittypes 0.0.6 py36_0 conda-forge widgetsnbextension 3.1.0 py36_0 conda-forge xgboost 0.6a2 py36_2 conda-forge --- Dockerfile | 5 ----- circle.yml | 2 +- environment.yml | 4 ++-- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7905b67..82f86f5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -92,11 +92,6 @@ RUN mkdir -p ${HOME}/.config/matplotlib && \ echo "backend : Agg" > ${HOME}/.config/matplotlib/matplotlibrc && \ python -c "import matplotlib.pyplot" -# Enable widgetsnbextension for jupyter widgets. -# See https://ipywidgets.readthedocs.io/en/stable/user_install.html. -# This enables the extension in the conda environment. -RUN jupyter nbextension enable --py widgetsnbextension - # Instruct joblib to use disk for temporary files. Joblib defaults to # /shm when that directory is present. In the Docker container, /shm is # present but defaults to 64 MB. diff --git a/circle.yml b/circle.yml index 8734a1f..b098431 100644 --- a/circle.yml +++ b/circle.yml @@ -16,4 +16,4 @@ test: - docker run civisanalytics/datascience-python python -c "from numpy.distutils import system_info; assert system_info.get_info('mkl') == {}" - docker run civisanalytics/datascience-python python -c "import numpy; numpy.test()" - docker run civisanalytics/datascience-python /bin/bash -c "conda list|grep conda-forge" - - docker run civisanalytics/datascience-python /bin/bash -c "conda list|grep -c conda-forge|python -c \"import sys; actual_count = int(sys.stdin.readlines()[0]); expected_count = 8; assert actual_count == expected_count, 'There should be %d conda-forge packages, but there are actually %d.' % (expected_count, actual_count)\"" + - docker run civisanalytics/datascience-python /bin/bash -c "conda list|grep -c conda-forge|python -c \"import sys; actual_count = int(sys.stdin.readlines()[0]); expected_count = 10; assert actual_count == expected_count, 'There should be %d conda-forge packages, but there are actually %d.' % (expected_count, actual_count)\"" diff --git a/environment.yml b/environment.yml index aa4094a..d1a25ab 100644 --- a/environment.yml +++ b/environment.yml @@ -7,10 +7,10 @@ dependencies: - botocore=1.5.38 - boto=2.46.1 - boto3==1.4.5 -- bqplot=0.10.1 +- bqplot=0.10.2 - cython=0.26 - ipython=6.1.0 -- ipywidgets=7.0.5 +- ipywidgets=7.1.0 - jinja2=2.9.6 - jsonschema=2.5.1 - jupyter=1.0.0 From b5e2857a48b9fb365cd822ab0fa6a7a436dac611 Mon Sep 17 00:00:00 2001 From: Michael Heilman Date: Thu, 28 Dec 2017 14:46:57 -0600 Subject: [PATCH 23/63] updated versions in changelog --- CHANGELOG.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d896b7b..f855bd4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,11 +9,11 @@ Version number changes (major.minor.micro) in this package denote the following: ## Unreleased ### New packages -- bqplot 0.10.1 +- bqplot 0.10.2 ### Package Updates -- ipywidgets 7.0.0 -> 7.0.5 -- notebook 5.2.0 -> 5.2.1 +- ipywidgets 7.0.0 -> 7.1.0 +- notebook 5.2.0 -> 5.2.2 ### Fixed - Enabled widgetsnbextension so that ipywidgets works. From fca76b8751095c604373fca09b26238bd8f996dd Mon Sep 17 00:00:00 2001 From: Michael Heilman Date: Thu, 28 Dec 2017 15:41:09 -0600 Subject: [PATCH 24/63] add back nbextension enable command just in case --- Dockerfile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Dockerfile b/Dockerfile index 82f86f5..98ad963 100644 --- a/Dockerfile +++ b/Dockerfile @@ -92,6 +92,13 @@ RUN mkdir -p ${HOME}/.config/matplotlib && \ echo "backend : Agg" > ${HOME}/.config/matplotlib/matplotlibrc && \ python -c "import matplotlib.pyplot" +# Enable widgetsnbextension for jupyter widgets. +# See https://ipywidgets.readthedocs.io/en/stable/user_install.html. +# This enables the extension in the conda environment. The conda-forge version +# does this upon installation, but the default channel version doesn't seem to, +# so we'll run this (again) just in case. +RUN jupyter nbextension enable --py widgetsnbextension + # Instruct joblib to use disk for temporary files. Joblib defaults to # /shm when that directory is present. In the Docker container, /shm is # present but defaults to 64 MB. From d482d05b3de06b7be00f099b6674dd0b548f8311 Mon Sep 17 00:00:00 2001 From: Stephen Hoover Date: Tue, 23 Jan 2018 09:38:43 -0600 Subject: [PATCH 25/63] MAINT Increase version number This will be a major version increase because of major version increases in `scipy` and `muffnn`, and a single breaking change in `pandas`. --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 98ad963..3e1ecb6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -105,7 +105,7 @@ RUN jupyter nbextension enable --py widgetsnbextension # https://github.com/joblib/joblib/blob/0.11/joblib/parallel.py#L328L342 ENV JOBLIB_TEMP_FOLDER=/tmp -ENV VERSION=3.3.0 \ - VERSION_MAJOR=3 \ - VERSION_MINOR=3 \ +ENV VERSION=4.0.0 \ + VERSION_MAJOR=4 \ + VERSION_MINOR=0 \ VERSION_MICRO=0 From f00f3b70beae8f52949e9ec0f4f49320c446661f Mon Sep 17 00:00:00 2001 From: Stephen Hoover Date: Tue, 23 Jan 2018 09:36:59 -0600 Subject: [PATCH 26/63] ENH Update packages, add feather-format Update Python from 3.6.2 to 3.6.4. Addition: - feather-format 0.4.0 Updates: - civis 1.7.1 -> 1.8.0 - civisml-extensions 0.1.5 -> 0.1.6 - muffnn 1.2.0 -> 2.0.0 - cloudpickle 0.5.1 -> 0.5.2 - dask 0.15.4 -> 0.16.1 - boto3 1.4.5 -> 1.5.11 - cython 0.26 -> 0.27.3 - openblas 0.2.19 -> 0.2.20 - pandas 0.21.0 -> 0.22.0 - pyarrow 0.7.1 -> 0.8.0 - scipy 0.19.1 -> 1.0.0 --- CHANGELOG.md | 19 +++++++++++++++++++ Dockerfile | 2 +- environment.yml | 27 ++++++++++++++------------- 3 files changed, 34 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f855bd4..c86deb3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,10 +8,29 @@ Version number changes (major.minor.micro) in this package denote the following: - A major version will increase if there are any backwards-incompatible changes in any of the packages contained in this Docker image, or any other backwards-incompatible changes in the execution environment. ## Unreleased + +## [4.0.0] - 2018-01-23 ### New packages - bqplot 0.10.2 +- feather-format 0.4.0 + +### Changed +- Updated Python version from 3.6.2 to 3.6.4. ### Package Updates +- civis 1.7.1 -> 1.8.0 +- civisml-extensions 0.1.5 -> 0.1.6 +- muffnn 1.2.0 -> 2.0.0 +- cloudpickle 0.5.1 -> 0.5.2 +- dask 0.15.4 -> 0.16.1 +- ftputil 3.3.1 -> 3.4 +- tensorflow 1.4.0 -> 1.4.1 +- boto3 1.4.5 -> 1.5.11 +- cython 0.26 -> 0.27.3 +- openblas 0.2.19 -> 0.2.20 +- pandas 0.21.0 -> 0.22.0 +- pyarrow 0.7.1 -> 0.8.0 +- scipy 0.19.1 -> 1.0.0 - ipywidgets 7.0.0 -> 7.1.0 - notebook 5.2.0 -> 5.2.2 diff --git a/Dockerfile b/Dockerfile index 3e1ecb6..412ded5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ ENV LANG=en_US.UTF-8 \ BASH_ENV=/etc/profile \ PATH=/opt/conda/bin:$PATH \ CIVIS_CONDA_VERSION=4.3.30 \ - CIVIS_PYTHON_VERSION=3.6.2 + CIVIS_PYTHON_VERSION=3.6.4 RUN DEBIAN_FRONTEND=noninteractive apt-get update -y --no-install-recommends && \ apt-get install -y --no-install-recommends software-properties-common && \ diff --git a/environment.yml b/environment.yml index d1a25ab..d98830b 100644 --- a/environment.yml +++ b/environment.yml @@ -6,9 +6,10 @@ dependencies: - beautifulsoup4=4.5.3 - botocore=1.5.38 - boto=2.46.1 -- boto3==1.4.5 +- boto3==1.5.11 - bqplot=0.10.2 -- cython=0.26 +- cython=0.27.3 +- feather-format=0.4.0 - ipython=6.1.0 - ipywidgets=7.1.0 - jinja2=2.9.6 @@ -24,36 +25,36 @@ dependencies: - nose=1.3.7 - numexpr=2.6.2 - numpy=1.13.3 -- openblas=0.2.19 -- pandas=0.21.0 +- openblas=0.2.20 +- pandas=0.22.0 - patsy=0.4.1 - psycopg2=2.6.2 -- pyarrow=0.7.1 +- pyarrow=0.8.0 - pycrypto=2.6.1 - pytest=3.1.3 -- python=3.6.2 +- python=3.6.4 - pyyaml=3.12 - requests=2.18.4 - s3fs=0.1.2 - seaborn=0.8 -- scipy=0.19.1 +- scipy=1.0.0 - scikit-learn=0.19.1 - statsmodels=0.8.0 - xgboost=0.6a2 - pip: - awscli==1.11.75 - - civis==1.7.1 - - civisml-extensions==0.1.5 - - cloudpickle==0.5.1 + - civis==1.8.0 + - civisml-extensions==0.1.6 + - cloudpickle==0.5.2 - dask==0.15.4 - dropbox==7.1.1 - - ftputil==3.3.1 + - ftputil==3.4 - glmnet==2.0.0 - joblib==0.11.0 - - muffnn==1.2.0 + - muffnn==2.0.0 - pubnub==4.0.13 - pysftp==0.2.9 - python-simple-hipchat==0.4.0 - requests-toolbelt==0.8.0 - - tensorflow==1.4.0 + - tensorflow==1.4.1 - urllib3==1.22 From d14275ade77c1be66958cf9905caa52caf76bdee Mon Sep 17 00:00:00 2001 From: Stephen Hoover Date: Wed, 24 Jan 2018 09:06:19 -0600 Subject: [PATCH 27/63] BUG Ignore irrelevant warning from tensorflow v1.4 (#52) This warning happens at import time. It doesn't indicate a problem and should be ignored. --- CHANGELOG.md | 1 + Dockerfile | 7 +++++++ warningsfilter.py | 5 +++++ 3 files changed, 13 insertions(+) create mode 100644 warningsfilter.py diff --git a/CHANGELOG.md b/CHANGELOG.md index c86deb3..6a0d4da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,7 @@ Version number changes (major.minor.micro) in this package denote the following: ### Fixed - Enabled widgetsnbextension so that ipywidgets works. +- Suppress irrelevant warning from tensorflow v1.4 ## [3.3.0] - 2017-11-17 ### Package Updates diff --git a/Dockerfile b/Dockerfile index 412ded5..f58908d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -105,6 +105,13 @@ RUN jupyter nbextension enable --py widgetsnbextension # https://github.com/joblib/joblib/blob/0.11/joblib/parallel.py#L328L342 ENV JOBLIB_TEMP_FOLDER=/tmp +# Filter out irrelevant warnings. +# This section should be removed when it's no longer needed. +# As of container version 4.0.0, it filters a tensorflow warning +# which should go away with tensorflow v1.5. +RUN mkdir -p /root/.local/lib/python3.6/site-packages +COPY warningsfilter.py /root/.local/lib/python3.6/site-packages/usercustomize.py + ENV VERSION=4.0.0 \ VERSION_MAJOR=4 \ VERSION_MINOR=0 \ diff --git a/warningsfilter.py b/warningsfilter.py new file mode 100644 index 0000000..8ac2653 --- /dev/null +++ b/warningsfilter.py @@ -0,0 +1,5 @@ +import warnings + +# This warning is raised from tensorflow v1.4 at import time. +# It's irrelevant and should be ignored. +warnings.filterwarnings('ignore', "compiletime version 3.5 of module 'tensorflow.python.framework.fast_tensor_util' does not match runtime version 3.6", category=RuntimeWarning) From e20469fbb2d246e7ca46f5a44bb8fd67bd9c2807 Mon Sep 17 00:00:00 2001 From: Stephen Hoover Date: Thu, 1 Feb 2018 12:46:18 -0600 Subject: [PATCH 28/63] MAINT Include patch for civis-python v1.8 (#53) --- CHANGELOG.md | 4 ++++ Dockerfile | 4 ++-- environment.yml | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a0d4da..14a1e6a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ Version number changes (major.minor.micro) in this package denote the following: ## Unreleased +## [4.0.1] - 2018-02-01 +### Package Updates +- civis 1.8.0 -> 1.8.1 + ## [4.0.0] - 2018-01-23 ### New packages - bqplot 0.10.2 diff --git a/Dockerfile b/Dockerfile index f58908d..5b398d9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -112,7 +112,7 @@ ENV JOBLIB_TEMP_FOLDER=/tmp RUN mkdir -p /root/.local/lib/python3.6/site-packages COPY warningsfilter.py /root/.local/lib/python3.6/site-packages/usercustomize.py -ENV VERSION=4.0.0 \ +ENV VERSION=4.0.1 \ VERSION_MAJOR=4 \ VERSION_MINOR=0 \ - VERSION_MICRO=0 + VERSION_MICRO=1 diff --git a/environment.yml b/environment.yml index d98830b..ba8b9da 100644 --- a/environment.yml +++ b/environment.yml @@ -43,7 +43,7 @@ dependencies: - xgboost=0.6a2 - pip: - awscli==1.11.75 - - civis==1.8.0 + - civis==1.8.1 - civisml-extensions==0.1.6 - cloudpickle==0.5.2 - dask==0.15.4 From dab11a0b3ea7920db488126856dd93876e5e483e Mon Sep 17 00:00:00 2001 From: Stephen Hoover Date: Wed, 14 Feb 2018 13:06:16 -0600 Subject: [PATCH 29/63] DOC Direct users from DockerHub to GitHub (#56) The DockerHub autobuild includes the README from GitHub, which is good. However, the README includes relative links which GitHub parses correctly but DockerHub does not. Therefore, direct users to read the README on GitHub instead of DockerHub. --- CHANGELOG.md | 3 +++ README.md | 2 ++ 2 files changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 14a1e6a..96f67a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ Version number changes (major.minor.micro) in this package denote the following: ## Unreleased +### Added +- Added a link in the README directing users who may be reading documentation on DockerHub to instead go to GitHub (#56). + ## [4.0.1] - 2018-02-01 ### Package Updates - civis 1.8.0 -> 1.8.1 diff --git a/README.md b/README.md index 5a7b7c7..69a153e 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,8 @@ This image is created from the official Ubuntu 14.04 Docker image and contains popular Python packages for data science. +If you are reading this README on DockerHub, then the links to files in the GitHub respository will be broken. Please read this documentation from [GitHub](https://github.com/civisanalytics/datascience-python) instead. + # Introduction This repository defines the "[civisanalytics/datascience-python](https://hub.docker.com/r/civisanalytics/datascience-python/)" From 5c06d458447bace13a34d641a0e6fd7d354c0f8f Mon Sep 17 00:00:00 2001 From: "Jackson L. Lee" Date: Thu, 19 Apr 2018 18:01:15 -0500 Subject: [PATCH 30/63] DEP Update versions of boto3, botocore, and awscli (#58) * DEP Update boto3 and botocore versions * TST boto3 1.7.4 is not on conda-forget yet * Revert "TST boto3 1.7.4 is not on conda-forget yet" This reverts commit c08e6c0e557d2d03ab1919806efa139147975654. * DEP awscli 1.11.75 (from pip) -> 1.15.4 (from conda) * TST awscli and rsa are installed from conda-forge --- CHANGELOG.md | 5 +++++ circle.yml | 2 +- environment.yml | 6 +++--- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 96f67a5..5610633 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,11 @@ Version number changes (major.minor.micro) in this package denote the following: ### Added - Added a link in the README directing users who may be reading documentation on DockerHub to instead go to GitHub (#56). +### Package Updates +- awscli 1.11.75 (from pip) -> 1.15.4 (from conda) +- botocore 1.5.38 -> 1.10.4 +- boto3 1.5.11 -> 1.7.4 + ## [4.0.1] - 2018-02-01 ### Package Updates - civis 1.8.0 -> 1.8.1 diff --git a/circle.yml b/circle.yml index b098431..81b6814 100644 --- a/circle.yml +++ b/circle.yml @@ -16,4 +16,4 @@ test: - docker run civisanalytics/datascience-python python -c "from numpy.distutils import system_info; assert system_info.get_info('mkl') == {}" - docker run civisanalytics/datascience-python python -c "import numpy; numpy.test()" - docker run civisanalytics/datascience-python /bin/bash -c "conda list|grep conda-forge" - - docker run civisanalytics/datascience-python /bin/bash -c "conda list|grep -c conda-forge|python -c \"import sys; actual_count = int(sys.stdin.readlines()[0]); expected_count = 10; assert actual_count == expected_count, 'There should be %d conda-forge packages, but there are actually %d.' % (expected_count, actual_count)\"" + - docker run civisanalytics/datascience-python /bin/bash -c "conda list|grep -c conda-forge|python -c \"import sys; actual_count = int(sys.stdin.readlines()[0]); expected_count = 11; assert actual_count == expected_count, 'There should be %d conda-forge packages, but there are actually %d.' % (expected_count, actual_count)\"" diff --git a/environment.yml b/environment.yml index ba8b9da..5c44e0d 100644 --- a/environment.yml +++ b/environment.yml @@ -3,10 +3,11 @@ channels: - defaults - conda-forge dependencies: +- awscli=1.15.4 - beautifulsoup4=4.5.3 -- botocore=1.5.38 +- botocore=1.10.4 - boto=2.46.1 -- boto3==1.5.11 +- boto3=1.7.4 - bqplot=0.10.2 - cython=0.27.3 - feather-format=0.4.0 @@ -42,7 +43,6 @@ dependencies: - statsmodels=0.8.0 - xgboost=0.6a2 - pip: - - awscli==1.11.75 - civis==1.8.1 - civisml-extensions==0.1.6 - cloudpickle==0.5.2 From 1205d67a5ce97bb965aa07b1426f31f8000a3676 Mon Sep 17 00:00:00 2001 From: "Jackson L. Lee" Date: Thu, 19 Apr 2018 18:18:14 -0500 Subject: [PATCH 31/63] MAINT Update version to 4.1.0 (#59) --- CHANGELOG.md | 1 + Dockerfile | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5610633..086180f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ Version number changes (major.minor.micro) in this package denote the following: ## Unreleased +## [4.1.0] - 2018-04-19 ### Added - Added a link in the README directing users who may be reading documentation on DockerHub to instead go to GitHub (#56). diff --git a/Dockerfile b/Dockerfile index 5b398d9..522466a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -112,7 +112,7 @@ ENV JOBLIB_TEMP_FOLDER=/tmp RUN mkdir -p /root/.local/lib/python3.6/site-packages COPY warningsfilter.py /root/.local/lib/python3.6/site-packages/usercustomize.py -ENV VERSION=4.0.1 \ +ENV VERSION=4.1.0 \ VERSION_MAJOR=4 \ - VERSION_MINOR=0 \ - VERSION_MICRO=1 + VERSION_MINOR=1 \ + VERSION_MICRO=0 From 9d06735844383581f65cf9cfc46386c4a74d0d20 Mon Sep 17 00:00:00 2001 From: Stephen Hoover Date: Wed, 25 Apr 2018 17:33:04 -0500 Subject: [PATCH 32/63] DEP Update packages (#60) Also update container version number and remove the no-longer-needed warning suppression for tensorflow. Package Updates - civis 1.8.1 -> 1.9.0 - civisml-extensions 0.1.6 -> 0.1.8 - muffnn 2.0.0 -> 2.1.0 - dask 0.15.4 (pip) -> 0.17.2 (conda) - tensorflow 1.4.1 -> 1.7.0 - ipython 6.1.0 -> 6.3.1 - matplotlib 2.1.0 -> 2.2.2 - notebook 5.2.2 -> 5.4.1 - scipy 1.0.0 -> 1.0.1 - urllib3 1.22 (pip) -> 1.22 (conda) --- CHANGELOG.md | 15 +++++++++++++++ Dockerfile | 11 ++--------- environment.yml | 18 +++++++++--------- warningsfilter.py | 5 ----- 4 files changed, 26 insertions(+), 23 deletions(-) delete mode 100644 warningsfilter.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 086180f..d2a10d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,21 @@ Version number changes (major.minor.micro) in this package denote the following: ## Unreleased + +## [4.2.0] - 2018-04-26 +### Package Updates +- civis 1.8.1 -> 1.9.0 +- civisml-extensions 0.1.6 -> 0.1.8 +- muffnn 2.0.0 -> 2.1.0 + +- dask 0.15.4 (pip) -> 0.17.2 (conda) +- tensorflow 1.4.1 -> 1.7.0 +- ipython 6.1.0 -> 6.3.1 +- matplotlib 2.1.0 -> 2.2.2 +- notebook 5.2.2 -> 5.4.1 +- scipy 1.0.0 -> 1.0.1 +- urllib3 1.22 (pip) -> 1.22 (conda) + ## [4.1.0] - 2018-04-19 ### Added - Added a link in the README directing users who may be reading documentation on DockerHub to instead go to GitHub (#56). diff --git a/Dockerfile b/Dockerfile index 522466a..bc3c02f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -105,14 +105,7 @@ RUN jupyter nbextension enable --py widgetsnbextension # https://github.com/joblib/joblib/blob/0.11/joblib/parallel.py#L328L342 ENV JOBLIB_TEMP_FOLDER=/tmp -# Filter out irrelevant warnings. -# This section should be removed when it's no longer needed. -# As of container version 4.0.0, it filters a tensorflow warning -# which should go away with tensorflow v1.5. -RUN mkdir -p /root/.local/lib/python3.6/site-packages -COPY warningsfilter.py /root/.local/lib/python3.6/site-packages/usercustomize.py - -ENV VERSION=4.1.0 \ +ENV VERSION=4.2.0 \ VERSION_MAJOR=4 \ - VERSION_MINOR=1 \ + VERSION_MINOR=2 \ VERSION_MICRO=0 diff --git a/environment.yml b/environment.yml index 5c44e0d..98f5b75 100644 --- a/environment.yml +++ b/environment.yml @@ -10,6 +10,7 @@ dependencies: - boto3=1.7.4 - bqplot=0.10.2 - cython=0.27.3 +- dask=0.17.2 - feather-format=0.4.0 - ipython=6.1.0 - ipywidgets=7.1.0 @@ -20,9 +21,9 @@ dependencies: - libgfortran=3.0.0 - libtiff=4.0.6 - libxml2=2.9.2 -- matplotlib=2.1.0 +- matplotlib=2.2.2 - nomkl=1.0 -- notebook=5.2.2 +- notebook=5.4.1 - nose=1.3.7 - numexpr=2.6.2 - numpy=1.13.3 @@ -38,23 +39,22 @@ dependencies: - requests=2.18.4 - s3fs=0.1.2 - seaborn=0.8 -- scipy=1.0.0 +- scipy=1.0.1 - scikit-learn=0.19.1 - statsmodels=0.8.0 +- urllib3=1.22 - xgboost=0.6a2 - pip: - - civis==1.8.1 - - civisml-extensions==0.1.6 + - civis==1.9.0 + - civisml-extensions==0.1.8 - cloudpickle==0.5.2 - - dask==0.15.4 - dropbox==7.1.1 - ftputil==3.4 - glmnet==2.0.0 - joblib==0.11.0 - - muffnn==2.0.0 + - muffnn==2.1.0 - pubnub==4.0.13 - pysftp==0.2.9 - python-simple-hipchat==0.4.0 - requests-toolbelt==0.8.0 - - tensorflow==1.4.1 - - urllib3==1.22 + - tensorflow==1.7.0 diff --git a/warningsfilter.py b/warningsfilter.py deleted file mode 100644 index 8ac2653..0000000 --- a/warningsfilter.py +++ /dev/null @@ -1,5 +0,0 @@ -import warnings - -# This warning is raised from tensorflow v1.4 at import time. -# It's irrelevant and should be ignored. -warnings.filterwarnings('ignore', "compiletime version 3.5 of module 'tensorflow.python.framework.fast_tensor_util' does not match runtime version 3.6", category=RuntimeWarning) From baef23c3247f88f67e2a5ddd0021b16896dbeda5 Mon Sep 17 00:00:00 2001 From: Stephen Hoover Date: Wed, 12 Sep 2018 14:01:16 -0500 Subject: [PATCH 33/63] MAINT Update CircleCI config to v2 (#62) CircleCI v1 was discontinued at the end of August. Update to a v2 config file. --- .circleci/config.yml | 38 ++++++++++++++++++++++++++++++++++++++ CHANGELOG.md | 2 ++ circle.yml | 19 ------------------- 3 files changed, 40 insertions(+), 19 deletions(-) create mode 100644 .circleci/config.yml delete mode 100644 circle.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..a9e5a92 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,38 @@ +version: 2 +jobs: + build: + docker: + - image: circleci/python:3.7.0 + steps: + - checkout + - setup_remote_docker + - run: + name: Build container + command: docker build -t ds-python . + - run: + name: Verify build completed + command: docker run ds-python /bin/bash -c "echo BUILDS OK" + - run: + name: Check that scipy links to OpenBLAS + command: docker run ds-python python -c "from scipy.linalg import _fblas" + - run: + name: Verify that numpy and matplotlib function + command: docker run ds-python python -c "import numpy, os; import matplotlib.pyplot as plt; x = numpy.arange(100); y = numpy.sin(x); plt.plot(x, y);" + - run: + name: Check that seaborn is installed + command: docker run ds-python python -c "import seaborn" + - run: + name: Check that numpy imports from bash shell + command: docker run -t ds-python /bin/bash -c "python -c 'import numpy'" + - run: + name: Verify that numpy does not link to MKL + command: docker run ds-python python -c "from numpy.distutils import system_info; assert system_info.get_info('mkl') == {}" + - run: + name: Run numpy unit tests + command: docker run ds-python python -c "import numpy; numpy.test()" + - run: + name: List packages installed from conda-forge + command: docker run ds-python /bin/bash -c "conda list|grep conda-forge" + - run: + name: Check that the expected number of packages come from conda-forge + command: docker run ds-python /bin/bash -c "conda list|grep -c conda-forge|python -c \"import sys; actual_count = int(sys.stdin.readlines()[0]); expected_count = 11; assert actual_count == expected_count, 'There should be %d conda-forge packages, but there are actually %d.' % (expected_count, actual_count)\"" diff --git a/CHANGELOG.md b/CHANGELOG.md index d2a10d6..3732148 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ Version number changes (major.minor.micro) in this package denote the following: ## Unreleased +### Maintenance +- Update CircleCI config to v2 (#62). ## [4.2.0] - 2018-04-26 ### Package Updates diff --git a/circle.yml b/circle.yml deleted file mode 100644 index 81b6814..0000000 --- a/circle.yml +++ /dev/null @@ -1,19 +0,0 @@ -machine: - services: - - docker - -dependencies: - override: - - docker build -t civisanalytics/datascience-python . - -test: - override: - - docker run civisanalytics/datascience-python /bin/bash -c "echo BUILDS OK" - - docker run civisanalytics/datascience-python python -c "from scipy.linalg import _fblas" - - docker run civisanalytics/datascience-python python -c "import numpy, os; import matplotlib.pyplot as plt; x = numpy.arange(100); y = numpy.sin(x); plt.plot(x, y);" - - docker run civisanalytics/datascience-python python -c "import seaborn" - - docker run -t civisanalytics/datascience-python /bin/bash -c "python -c 'import numpy'" - - docker run civisanalytics/datascience-python python -c "from numpy.distutils import system_info; assert system_info.get_info('mkl') == {}" - - docker run civisanalytics/datascience-python python -c "import numpy; numpy.test()" - - docker run civisanalytics/datascience-python /bin/bash -c "conda list|grep conda-forge" - - docker run civisanalytics/datascience-python /bin/bash -c "conda list|grep -c conda-forge|python -c \"import sys; actual_count = int(sys.stdin.readlines()[0]); expected_count = 11; assert actual_count == expected_count, 'There should be %d conda-forge packages, but there are actually %d.' % (expected_count, actual_count)\"" From 6ba7ebc8a0e4b87fcecf2cef167f71a016378810 Mon Sep 17 00:00:00 2001 From: Elizabeth Sander Date: Tue, 12 Mar 2019 13:39:54 -0500 Subject: [PATCH 34/63] DEP version 5.0.0 package updates (#66) --- .circleci/config.yml | 4 +- CHANGELOG.md | 53 +++++++++++++++++++++++++++ Dockerfile | 16 ++++---- environment.yml | 87 ++++++++++++++++++++++---------------------- 4 files changed, 108 insertions(+), 52 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a9e5a92..ae09d82 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -34,5 +34,5 @@ jobs: name: List packages installed from conda-forge command: docker run ds-python /bin/bash -c "conda list|grep conda-forge" - run: - name: Check that the expected number of packages come from conda-forge - command: docker run ds-python /bin/bash -c "conda list|grep -c conda-forge|python -c \"import sys; actual_count = int(sys.stdin.readlines()[0]); expected_count = 11; assert actual_count == expected_count, 'There should be %d conda-forge packages, but there are actually %d.' % (expected_count, actual_count)\"" + name: Check for conda-pip conflicts + command: docker run ds-python /bin/bash -c "conda list|python -c \"import sys; from collections import Counter; duplicates = {k for k, v in Counter([l.split()[0] for l in sys.stdin.readlines() if not l.startswith('#')]).items() if v>1}; assert not duplicates, f'These packages are installed by both pip and conda:\n{duplicates}'\"" diff --git a/CHANGELOG.md b/CHANGELOG.md index 3732148..007b1ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,59 @@ Version number changes (major.minor.micro) in this package denote the following: ## Unreleased +## [5.0.0] - 2019-03-12 +### Changed +- python 3.6.4 -> 3.7.1 +- conda 4.3.30 -> 4.6.8 + +### New Packages +- explicitly installs click=6.7 + +### Package Updates +- awscli 1.15.4 -> 1.16.121 +- beautifulsoup4 4.5.3 -> 4.7.1 +- botocore 1.10.4 -> 1.12.111 +- boto 2.46.1 -> 2.49.0 +- boto3 1.7.4 -> 1.9.111 +- bqplot 0.10.2 -> 0.11.5 +- cloudpickle 0.5.2 -> 0.8.0 +- cython 0.27.3 -> 0.29.6 +- dask 0.17.2 -> 1.1.4 +- ipython 6.1.0 -> 7.3.0 +- ipywidgets 7.1.0 -> 7.4.2 +- jinja2 2.9.6 -> 2.10 +- jsonschema 2.5.1 -> 3.0.1 +- libtiff 4.0.6 -> 4.0.10 +- libxml2 2.9.2 -> 2.9.8 +- matplotlib 2.2.2 -> 3.0.3 +- notebook 5.4.1 -> 5.7.5 +- numexpr 2.6.2 -> 2.6.9 +- numpy 1.13.3 -> 1.16.2 +- openblas 0.2.20 -> 0.3.5 +- pandas 0.22.0 -> 0.24.1 +- patsy 0.4.1 -> 0.5.1 +- psycopg2 2.6.2 -> 2.7.7 +- pyarrow 0.8.0 -> 0.12.1 +- pytest 3.1.3 -> 4.3.0 +- pyyaml 3.12 -> 3.13 +- requests 2.18.4 -> 2.21.0 +- s3fs 0.1.2 -> 0.2.0 +- seaborn 0.8 -> 0.9.0 +- scipy 1.0.1 -> 1.2.0 +- scikit-learn 0.19.1 -> 0.19.2 +- statsmodels 0.8.0 -> 0.9.0 +- urllib3 1.22 -> 1.24.1 +- xgboost 0.6a2 -> 0.81 +- civis 1.9.0 -> 1.9.4 +- civisml-extensions 0.1.8 -> 0.1.10 +- dropbox 7.1.1 -> 9.3.0 +- glmnet 2.0.0 -> 2.1.1 +- muffnn 2.1.0 -> 2.2.0 +- pubnub 4.0.13 -> 4.1.2 +- requests-toolbelt 0.8.0 -> 0.9.1 +- tensorflow 1.7.0 -> 1.13.1 + + ### Maintenance - Update CircleCI config to v2 (#62). diff --git a/Dockerfile b/Dockerfile index bc3c02f..5036644 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,8 +11,9 @@ ENV LANG=en_US.UTF-8 \ CONDARC=/opt/conda/.condarc \ BASH_ENV=/etc/profile \ PATH=/opt/conda/bin:$PATH \ - CIVIS_CONDA_VERSION=4.3.30 \ - CIVIS_PYTHON_VERSION=3.6.4 + CIVIS_MINICONDA_VERSION=4.5.12 \ + CIVIS_CONDA_VERSION=4.6.8 \ + CIVIS_PYTHON_VERSION=3.7.1 RUN DEBIAN_FRONTEND=noninteractive apt-get update -y --no-install-recommends && \ apt-get install -y --no-install-recommends software-properties-common && \ @@ -52,7 +53,7 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get update -y --no-install-recommends && # # The ordering of these steps seems to matter. You seem to have to # install a specific python version by hand and then pin it. -# 1) install conda +# 1) install conda using CIVIS_MINICONDA_VERSION (may not be most up to date conda version) # 2) pin conda to the version given by CIVIS_CONDA_VERSION # 3) install the python version CIVIS_PYTHON_VERSION # 4) pin the python version @@ -61,10 +62,11 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get update -y --no-install-recommends && # Red Hat and Debian use different names for this file. git2R wants the latter. # See conda-recipes GH 423 RUN echo 'export PATH=/opt/conda/bin:$PATH' > /etc/profile.d/conda.sh && \ - wget --quiet https://repo.continuum.io/miniconda/Miniconda3-${CIVIS_CONDA_VERSION}-Linux-x86_64.sh && \ - /bin/bash /Miniconda3-${CIVIS_CONDA_VERSION}-Linux-x86_64.sh -b -p /opt/conda && \ - rm Miniconda3-${CIVIS_CONDA_VERSION}-Linux-x86_64.sh && \ - /opt/conda/bin/conda install --yes conda==${CIVIS_CONDA_VERSION} && \ + wget --quiet https://repo.continuum.io/miniconda/Miniconda3-${CIVIS_MINICONDA_VERSION}-Linux-x86_64.sh && \ + /bin/bash /Miniconda3-${CIVIS_MINICONDA_VERSION}-Linux-x86_64.sh -b -p /opt/conda && \ + rm Miniconda3-${CIVIS_MINICONDA_VERSION}-Linux-x86_64.sh && \ + /opt/conda/bin/conda install --yes conda==${CIVIS_MINICONDA_VERSION} && \ + conda install conda=${CIVIS_CONDA_VERSION} && \ echo "conda ==${CIVIS_CONDA_VERSION}" > /opt/conda/conda-meta/pinned && \ conda install --yes python==${CIVIS_PYTHON_VERSION} && \ echo "python ==${CIVIS_PYTHON_VERSION}" >> /opt/conda/conda-meta/pinned && \ diff --git a/environment.yml b/environment.yml index 98f5b75..c7ec43b 100644 --- a/environment.yml +++ b/environment.yml @@ -3,58 +3,59 @@ channels: - defaults - conda-forge dependencies: -- awscli=1.15.4 -- beautifulsoup4=4.5.3 -- botocore=1.10.4 -- boto=2.46.1 -- boto3=1.7.4 -- bqplot=0.10.2 -- cython=0.27.3 -- dask=0.17.2 +- awscli=1.16.121 +- beautifulsoup4=4.7.1 +- botocore=1.12.111 +- boto=2.49.0 +- boto3=1.9.111 +- bqplot=0.11.5 +- click=6.7 +- cloudpickle=0.8.0 +- cython=0.29.6 +- dask=1.1.4 - feather-format=0.4.0 -- ipython=6.1.0 -- ipywidgets=7.1.0 -- jinja2=2.9.6 -- jsonschema=2.5.1 +- ipython=7.3.0 +- ipywidgets=7.4.2 +- jinja2=2.10 +- jsonschema=3.0.1 - jupyter=1.0.0 - libffi=3.2.1 - libgfortran=3.0.0 -- libtiff=4.0.6 -- libxml2=2.9.2 -- matplotlib=2.2.2 +- libtiff=4.0.10 +- libxml2=2.9.8 +- matplotlib=3.0.3 - nomkl=1.0 -- notebook=5.4.1 +- notebook=5.7.5 - nose=1.3.7 -- numexpr=2.6.2 -- numpy=1.13.3 -- openblas=0.2.20 -- pandas=0.22.0 -- patsy=0.4.1 -- psycopg2=2.6.2 -- pyarrow=0.8.0 +- numexpr=2.6.9 +- numpy=1.16.2 +- openblas=0.3.5 +- pandas=0.24.1 +- patsy=0.5.1 +- psycopg2=2.7.7 +- pyarrow=0.12.1 - pycrypto=2.6.1 -- pytest=3.1.3 -- python=3.6.4 -- pyyaml=3.12 -- requests=2.18.4 -- s3fs=0.1.2 -- seaborn=0.8 -- scipy=1.0.1 -- scikit-learn=0.19.1 -- statsmodels=0.8.0 -- urllib3=1.22 -- xgboost=0.6a2 +- pytest=4.3.0 +- python=3.7.1 +- pyyaml=3.13 +- requests=2.21.0 +- s3fs=0.2.0 +- seaborn=0.9.0 +- scipy=1.2.0 +- scikit-learn=0.19.2 +- statsmodels=0.9.0 +- urllib3=1.24.1 +- xgboost=0.81 - pip: - - civis==1.9.0 - - civisml-extensions==0.1.8 - - cloudpickle==0.5.2 - - dropbox==7.1.1 + - civis==1.9.4 + - civisml-extensions==0.1.10 + - dropbox==9.3.0 - ftputil==3.4 - - glmnet==2.0.0 + - glmnet==2.1.1 - joblib==0.11.0 - - muffnn==2.1.0 - - pubnub==4.0.13 + - muffnn==2.2.0 + - pubnub==4.1.2 - pysftp==0.2.9 - python-simple-hipchat==0.4.0 - - requests-toolbelt==0.8.0 - - tensorflow==1.7.0 + - requests-toolbelt==0.9.1 + - tensorflow==1.13.1 From a0c33ad736122791e0feb4f6bf21a0370b9813ac Mon Sep 17 00:00:00 2001 From: Elizabeth Sander Date: Tue, 12 Mar 2019 18:18:39 -0500 Subject: [PATCH 35/63] MAINT update ubuntu version and version in ENV (#67) Update ubuntu version and version in ENV. Explicitly install locales package, and move local environment variable setting to after `locales` installation. --- .circleci/config.yml | 3 +++ CHANGELOG.md | 3 ++- Dockerfile | 35 +++++++++++++++++------------------ 3 files changed, 22 insertions(+), 19 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ae09d82..b58f22f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -24,6 +24,9 @@ jobs: - run: name: Check that numpy imports from bash shell command: docker run -t ds-python /bin/bash -c "python -c 'import numpy'" + - run: + name: Check that tensorflow imports from bash shell + command: docker run ds-python /bin/bash -c "python -c 'import tensorflow'" - run: name: Verify that numpy does not link to MKL command: docker run ds-python python -c "from numpy.distutils import system_info; assert system_info.get_info('mkl') == {}" diff --git a/CHANGELOG.md b/CHANGELOG.md index 007b1ee..0f79a3c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ Version number changes (major.minor.micro) in this package denote the following: ## [5.0.0] - 2019-03-12 ### Changed +- Ubuntu 14.04 -> 18.04 (#67) - python 3.6.4 -> 3.7.1 - conda 4.3.30 -> 4.6.8 @@ -61,9 +62,9 @@ Version number changes (major.minor.micro) in this package denote the following: - requests-toolbelt 0.8.0 -> 0.9.1 - tensorflow 1.7.0 -> 1.13.1 - ### Maintenance - Update CircleCI config to v2 (#62). +- Test that tensorflow imports successfully (#67). ## [4.2.0] - 2018-04-26 ### Package Updates diff --git a/Dockerfile b/Dockerfile index 5036644..00f513a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,21 +1,9 @@ -FROM ubuntu:14.04 +FROM ubuntu:18.04 MAINTAINER support@civisanalytics.com -# Ensure UTF-8 locale. -RUN locale-gen en_US.UTF-8 - -# Set environment variables for UTF-8, conda, and shell environments -ENV LANG=en_US.UTF-8 \ - LANGUAGE=en_US:en \ - LC_ALL=en_US.UTF-8 \ - CONDARC=/opt/conda/.condarc \ - BASH_ENV=/etc/profile \ - PATH=/opt/conda/bin:$PATH \ - CIVIS_MINICONDA_VERSION=4.5.12 \ - CIVIS_CONDA_VERSION=4.6.8 \ - CIVIS_PYTHON_VERSION=3.7.1 - RUN DEBIAN_FRONTEND=noninteractive apt-get update -y --no-install-recommends && \ + apt-get install -y --no-install-recommends locales && \ + locale-gen en_US.UTF-8 && \ apt-get install -y --no-install-recommends software-properties-common && \ apt-get install -y --no-install-recommends \ make \ @@ -34,6 +22,17 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get update -y --no-install-recommends && apt-get clean -y && \ rm -rf /var/lib/apt/lists/* +# Set environment variables for UTF-8, conda, and shell environments +ENV LANG=en_US.UTF-8 \ + LANGUAGE=en_US:en \ + LC_ALL=en_US.UTF-8 \ + CONDARC=/opt/conda/.condarc \ + BASH_ENV=/etc/profile \ + PATH=/opt/conda/bin:$PATH \ + CIVIS_MINICONDA_VERSION=4.5.12 \ + CIVIS_CONDA_VERSION=4.6.8 \ + CIVIS_PYTHON_VERSION=3.7.1 + # Conda install. # # Everything is installed in the root environment. This allows for @@ -107,7 +106,7 @@ RUN jupyter nbextension enable --py widgetsnbextension # https://github.com/joblib/joblib/blob/0.11/joblib/parallel.py#L328L342 ENV JOBLIB_TEMP_FOLDER=/tmp -ENV VERSION=4.2.0 \ - VERSION_MAJOR=4 \ - VERSION_MINOR=2 \ +ENV VERSION=5.0.0 \ + VERSION_MAJOR=5 \ + VERSION_MINOR=0 \ VERSION_MICRO=0 From c16ea2686cd741ddbf8e8dad23046ad8cc86a618 Mon Sep 17 00:00:00 2001 From: Shelby Ruettiger Date: Thu, 11 Jul 2019 13:10:52 -0500 Subject: [PATCH 36/63] add buildspecs for autobuilds --- buildspec/release.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 buildspec/release.yaml diff --git a/buildspec/release.yaml b/buildspec/release.yaml new file mode 100644 index 0000000..3e7651b --- /dev/null +++ b/buildspec/release.yaml @@ -0,0 +1,17 @@ + +version: 0.2 +phases: + pre_build: + commands: + - echo Logging in to Amazon ECR... + - $(aws ecr get-login --no-include-email --region $AWS_DEFAULT_REGION) + build: + commands: + - echo Building the Docker image... + - TAG_NAME=${TAG_NAME#"v"} # remove "v" from beginning of tag + - docker build --tag ${REPOSITORY_URI}:${TAG_NAME} . + - docker push ${REPOSITORY_URI}:${TAG_NAME} + post_build: + commands: + - echo Build completed! + - printf '{"tag":"%s"}' $TAG_NAME > build.json \ No newline at end of file From d18dd7bd16fb7d58f2df08578b321c6ce29506b4 Mon Sep 17 00:00:00 2001 From: Stephen Hoover Date: Thu, 11 Jul 2019 14:50:07 -0500 Subject: [PATCH 37/63] BUG Update openblas (#70) Openblas v0.3.5 was removed from conda-forge because it was incompatible with Intel Skylake X CPUs. Update to the most recent release, v0.3.6. --- CHANGELOG.md | 7 +++++++ environment.yml | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f79a3c..74e8bda 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,13 @@ Version number changes (major.minor.micro) in this package denote the following: ## Unreleased +### Added + + +### Package Updates +- openblas 0.3.5 -> 0.3.6 (#68, #70) + + ## [5.0.0] - 2019-03-12 ### Changed - Ubuntu 14.04 -> 18.04 (#67) diff --git a/environment.yml b/environment.yml index c7ec43b..0815101 100644 --- a/environment.yml +++ b/environment.yml @@ -29,7 +29,7 @@ dependencies: - nose=1.3.7 - numexpr=2.6.9 - numpy=1.16.2 -- openblas=0.3.5 +- openblas=0.3.6 - pandas=0.24.1 - patsy=0.5.1 - psycopg2=2.7.7 From 3d164936189be0355e0992eb987898334927a266 Mon Sep 17 00:00:00 2001 From: Shelby Ruettiger Date: Thu, 11 Jul 2019 14:56:37 -0500 Subject: [PATCH 38/63] add line to changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 74e8bda..f9e9e92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ Version number changes (major.minor.micro) in this package denote the following: ## Unreleased ### Added - +- added buildspecs for autobuilding and pushing Docker image to Amazon ECR ### Package Updates - openblas 0.3.5 -> 0.3.6 (#68, #70) From 638f07173f7676af6d3f78a0605660340b7e9f10 Mon Sep 17 00:00:00 2001 From: Shelby Ruettiger Date: Mon, 26 Aug 2019 11:50:08 -0500 Subject: [PATCH 39/63] cascading tags --- buildspec/release.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/buildspec/release.yaml b/buildspec/release.yaml index 3e7651b..3636248 100644 --- a/buildspec/release.yaml +++ b/buildspec/release.yaml @@ -8,9 +8,11 @@ phases: build: commands: - echo Building the Docker image... - - TAG_NAME=${TAG_NAME#"v"} # remove "v" from beginning of tag - - docker build --tag ${REPOSITORY_URI}:${TAG_NAME} . - - docker push ${REPOSITORY_URI}:${TAG_NAME} + - PATCH_TAG=${TAG_NAME#"v"} # major.minor.patch + - MINOR_TAG=${PATCH_TAG%.*} # major.minor + - MAJOR_TAG=${MINOR_TAG%.*} # major + - docker build -t ${REPOSITORY_URI}:${PATCH_TAG} -t ${REPOSITORY_URI}:${MINOR_TAG} -t ${REPOSITORY_URI}:${MAJOR_TAG} -t ${REPOSITORY_URI}:latest . + - docker push ${REPOSITORY_URI} post_build: commands: - echo Build completed! From ee693bd163674a67b60863556b632f6b27ef6afa Mon Sep 17 00:00:00 2001 From: Shelby Ruettiger Date: Wed, 28 Aug 2019 17:01:54 -0500 Subject: [PATCH 40/63] [CIVP-19203] add merge master for proper latest tagging --- buildspec/merge_master.yaml | 14 ++++++++++++++ buildspec/release.yaml | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 buildspec/merge_master.yaml diff --git a/buildspec/merge_master.yaml b/buildspec/merge_master.yaml new file mode 100644 index 0000000..51f2de9 --- /dev/null +++ b/buildspec/merge_master.yaml @@ -0,0 +1,14 @@ +version: 0.2 +phases: + pre_build: + commands: + - echo Logging in to Amazon ECR... + - $(aws ecr get-login --no-include-email --region $AWS_DEFAULT_REGION) + build: + commands: + - echo Building the Docker image... + - docker build -t ${REPOSITORY_URI}:latest . + - docker push ${REPOSITORY_URI} + post_build: + commands: + - echo Build completed! \ No newline at end of file diff --git a/buildspec/release.yaml b/buildspec/release.yaml index 3636248..0bc92c6 100644 --- a/buildspec/release.yaml +++ b/buildspec/release.yaml @@ -11,7 +11,7 @@ phases: - PATCH_TAG=${TAG_NAME#"v"} # major.minor.patch - MINOR_TAG=${PATCH_TAG%.*} # major.minor - MAJOR_TAG=${MINOR_TAG%.*} # major - - docker build -t ${REPOSITORY_URI}:${PATCH_TAG} -t ${REPOSITORY_URI}:${MINOR_TAG} -t ${REPOSITORY_URI}:${MAJOR_TAG} -t ${REPOSITORY_URI}:latest . + - docker build -t ${REPOSITORY_URI}:${PATCH_TAG} -t ${REPOSITORY_URI}:${MINOR_TAG} -t ${REPOSITORY_URI}:${MAJOR_TAG} . - docker push ${REPOSITORY_URI} post_build: commands: From 90d07fdee3fcb82c9700d67cbc83d0b65ca60a4a Mon Sep 17 00:00:00 2001 From: Shelby Ruettiger Date: Wed, 28 Aug 2019 18:00:07 -0500 Subject: [PATCH 41/63] add newlines to eof --- buildspec/merge_master.yaml | 2 +- buildspec/release.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/buildspec/merge_master.yaml b/buildspec/merge_master.yaml index 51f2de9..71c9e38 100644 --- a/buildspec/merge_master.yaml +++ b/buildspec/merge_master.yaml @@ -11,4 +11,4 @@ phases: - docker push ${REPOSITORY_URI} post_build: commands: - - echo Build completed! \ No newline at end of file + - echo Build completed! diff --git a/buildspec/release.yaml b/buildspec/release.yaml index 0bc92c6..a1fcc27 100644 --- a/buildspec/release.yaml +++ b/buildspec/release.yaml @@ -16,4 +16,4 @@ phases: post_build: commands: - echo Build completed! - - printf '{"tag":"%s"}' $TAG_NAME > build.json \ No newline at end of file + - printf '{"tag":"%s"}' $TAG_NAME > build.json From cc9ffd9e1f9185af277fafd6bd48d41c06eb4491 Mon Sep 17 00:00:00 2001 From: "Jackson L. Lee" Date: Fri, 17 Jan 2020 10:10:54 -0600 Subject: [PATCH 42/63] [PARO-734] Bump versions of modeling packages (#74) * DEP upgrade pydata stack dependencies * DEP python 3.7.1 -> 3.7.6 * MAINT update changelog * DEP conda 4.6.8 -> 4.8.1 * DEP nomkl 1.0 -> 3.0 --- CHANGELOG.md | 19 ++++++++++++++++--- Dockerfile | 4 ++-- environment.yml | 26 +++++++++++++------------- 3 files changed, 31 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f9e9e92..cd89b07 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,13 +8,26 @@ Version number changes (major.minor.micro) in this package denote the following: - A major version will increase if there are any backwards-incompatible changes in any of the packages contained in this Docker image, or any other backwards-incompatible changes in the execution environment. ## Unreleased - -### Added -- added buildspecs for autobuilding and pushing Docker image to Amazon ECR +### Changed +- Python 3.7.1 -> 3.7.6 +- Conda 4.6.8 -> 4.8.1 ### Package Updates - openblas 0.3.5 -> 0.3.6 (#68, #70) +- civis 1.9.4 -> 1.12.0 (#74, same for packages below) +- civisml-extensions 0.1.10 -> 0.2.1 +- cloudpickle 0.8.0 -> 1.2.2 +- joblib 0.11.0 -> 0.14.1 +- muffnn 2.2.0 -> 2.3.0 +- nomkl 1.0 -> 3.0 +- numpy 1.16.2 -> 1.17.3 +- pandas 0.24.1 -> 0.25.3 +- pyarrow 0.12.1 -> 0.15.1 +- scipy 1.2.0 -> 1.4.1 +- scikit-learn 0.19.2 -> 0.22.1 +### Added +- added buildspecs for autobuilding and pushing Docker image to Amazon ECR (#69) ## [5.0.0] - 2019-03-12 ### Changed diff --git a/Dockerfile b/Dockerfile index 00f513a..fe9dadb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,8 +30,8 @@ ENV LANG=en_US.UTF-8 \ BASH_ENV=/etc/profile \ PATH=/opt/conda/bin:$PATH \ CIVIS_MINICONDA_VERSION=4.5.12 \ - CIVIS_CONDA_VERSION=4.6.8 \ - CIVIS_PYTHON_VERSION=3.7.1 + CIVIS_CONDA_VERSION=4.8.1 \ + CIVIS_PYTHON_VERSION=3.7.6 # Conda install. # diff --git a/environment.yml b/environment.yml index 0815101..95a2a68 100644 --- a/environment.yml +++ b/environment.yml @@ -10,13 +10,15 @@ dependencies: - boto3=1.9.111 - bqplot=0.11.5 - click=6.7 -- cloudpickle=0.8.0 +- cloudpickle=1.2.2 - cython=0.29.6 - dask=1.1.4 - feather-format=0.4.0 +- glmnet=2.1.1 - ipython=7.3.0 - ipywidgets=7.4.2 - jinja2=2.10 +- joblib=0.14.1 - jsonschema=3.0.1 - jupyter=1.0.0 - libffi=3.2.1 @@ -24,36 +26,34 @@ dependencies: - libtiff=4.0.10 - libxml2=2.9.8 - matplotlib=3.0.3 -- nomkl=1.0 +- nomkl=3.0 - notebook=5.7.5 - nose=1.3.7 - numexpr=2.6.9 -- numpy=1.16.2 +- numpy=1.17.3 - openblas=0.3.6 -- pandas=0.24.1 +- pandas=0.25.3 - patsy=0.5.1 - psycopg2=2.7.7 -- pyarrow=0.12.1 +- pyarrow=0.15.1 - pycrypto=2.6.1 - pytest=4.3.0 -- python=3.7.1 +- python=3.7.6 - pyyaml=3.13 - requests=2.21.0 - s3fs=0.2.0 - seaborn=0.9.0 -- scipy=1.2.0 -- scikit-learn=0.19.2 +- scipy=1.4.1 +- scikit-learn=0.22.1 - statsmodels=0.9.0 - urllib3=1.24.1 - xgboost=0.81 - pip: - - civis==1.9.4 - - civisml-extensions==0.1.10 + - civis==1.12.0 + - civisml-extensions==0.2.1 - dropbox==9.3.0 - ftputil==3.4 - - glmnet==2.1.1 - - joblib==0.11.0 - - muffnn==2.2.0 + - muffnn==2.3.0 - pubnub==4.1.2 - pysftp==0.2.9 - python-simple-hipchat==0.4.0 From 0f55b33bf950b8c767be97f88ba51ba3025b2f0a Mon Sep 17 00:00:00 2001 From: Elizabeth Sander Date: Mon, 17 Feb 2020 13:59:02 -0600 Subject: [PATCH 43/63] DEP Release v6.0.0 (#75) * SERD-1717 MAINT dependency bump * MAINT update civis and changelog Co-authored-by: Nick Lee --- CHANGELOG.md | 59 ++++++++++++++++++++++++++++++++++++++----------- Dockerfile | 2 +- environment.yml | 56 +++++++++++++++++++++++----------------------- 3 files changed, 75 insertions(+), 42 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cd89b07..c311290 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,23 +8,56 @@ Version number changes (major.minor.micro) in this package denote the following: - A major version will increase if there are any backwards-incompatible changes in any of the packages contained in this Docker image, or any other backwards-incompatible changes in the execution environment. ## Unreleased + +## [6.0.0] ### Changed -- Python 3.7.1 -> 3.7.6 -- Conda 4.6.8 -> 4.8.1 +- Python 3.7.1 -> 3.7.6 (#74) +- Conda 4.6.8 -> 4.8.1 (#74) + +### Removed Packages +- removes python-simple-hipchat (#75) + +### New Packages +- explicitly installs pip=20.0.2 (#75) ### Package Updates +- awscli 1.16.121 -> 1.17.15 (#75) +- beautifulsoup4 4.7.1 -> 4.8.2 (#75) +- botocore 1.12.111 -> 1.14.15 (#75) +- boto3 1.9.111 -> 1.11.15 (#75) +- bqplot 0.11.5 -> 0.12.3 (#75) +- civis 1.9.4 -> 1.12.1 (#74) +- civisml-extensions 0.1.10 -> 0.2.1 (#74) +- cloudpickle 0.8.0 -> 1.2.2 (#74) +- cython 0.29.6 -> 0.29.15 (#75) +- dask 1.1.4 -> 2.10.1 (#75) +- dropbox 9.3.0 -> 9.4.0 (#75) +- ipython 7.3.0 -> 7.12.0 (#75) +- ipywidgets 7.4.2 -> 7.5.1 (#75) +- jinja2 2.10 -> 2.11.1 (#75) +- joblib 0.11.0 -> 0.14.1 (#74) +- jsonschema 3.0.1 -> 3.2.0 (#75) +- libtiff 4.0.10 -> 4.1.0 (#75) +- libxml2 2.9.8 -> 2.9.10 (#75) +- matplotlib 3.0.3 -> 3.1.3 (#75) +- muffnn 2.2.0 -> 2.3.0 (#74) +- nomkl 1.0 -> 3.0 (#74) +- notebook 5.7.5 -> 6.0.3 (#75) +- numexpr 2.6.9 -> 2.7.1 (#75) +- numpy 1.16.2 -> 1.17.3 (#74) - openblas 0.3.5 -> 0.3.6 (#68, #70) -- civis 1.9.4 -> 1.12.0 (#74, same for packages below) -- civisml-extensions 0.1.10 -> 0.2.1 -- cloudpickle 0.8.0 -> 1.2.2 -- joblib 0.11.0 -> 0.14.1 -- muffnn 2.2.0 -> 2.3.0 -- nomkl 1.0 -> 3.0 -- numpy 1.16.2 -> 1.17.3 -- pandas 0.24.1 -> 0.25.3 -- pyarrow 0.12.1 -> 0.15.1 -- scipy 1.2.0 -> 1.4.1 -- scikit-learn 0.19.2 -> 0.22.1 +- pandas 0.24.1 -> 0.25.3 (#74) +- psycopg2 2.7.7 -> 2.8.4 (#75) +- pubnub 4.1.2 -> 4.3.0 (#75) +- pyarrow 0.12.1 -> 0.16.0 (#75) +- pytest 4.3.0 -> 5.3.5 (#75) +- requests 2.21.0 -> 2.22.0 (#75) +- s3fs 0.2.0 -> 0.4.0 (#75) +- scipy 1.2.0 -> 1.4.1 (#74) +- scikit-learn 0.19.2 -> 0.22.1 (#74) +- seaborn 0.9.0 -> 0.10.0 (#75) +- statsmodels 0.9.0 -> 0.11.0 (#75) +- urllib3 1.24.1 -> 1.25.7 (#75) ### Added - added buildspecs for autobuilding and pushing Docker image to Amazon ECR (#69) diff --git a/Dockerfile b/Dockerfile index fe9dadb..c463735 100644 --- a/Dockerfile +++ b/Dockerfile @@ -43,7 +43,7 @@ ENV LANG=en_US.UTF-8 \ # Things are pinned to prevent upgrades from conda and force it to # resolve dependencies relative to a fixed conda & python version. # -# Note that the python version is also listed in the enviornment.yml +# Note that the python version is also listed in the environment.yml # file. The version in CIVIS_PYTHON_VERSION is the source of truth. # If you want to change the python version, you need to change it in # **both** places. The python version has been left in the `environment.yml` diff --git a/environment.yml b/environment.yml index 95a2a68..cee5ec2 100644 --- a/environment.yml +++ b/environment.yml @@ -3,59 +3,59 @@ channels: - defaults - conda-forge dependencies: -- awscli=1.16.121 -- beautifulsoup4=4.7.1 -- botocore=1.12.111 +- awscli=1.17.15 +- beautifulsoup4=4.8.2 +- botocore=1.14.15 - boto=2.49.0 -- boto3=1.9.111 -- bqplot=0.11.5 +- boto3=1.11.15 +- bqplot=0.12.3 - click=6.7 - cloudpickle=1.2.2 -- cython=0.29.6 -- dask=1.1.4 +- cython=0.29.15 +- dask=2.10.1 - feather-format=0.4.0 - glmnet=2.1.1 -- ipython=7.3.0 -- ipywidgets=7.4.2 -- jinja2=2.10 +- ipython=7.12.0 +- ipywidgets=7.5.1 +- jinja2=2.11.1 - joblib=0.14.1 -- jsonschema=3.0.1 +- jsonschema=3.2.0 - jupyter=1.0.0 - libffi=3.2.1 - libgfortran=3.0.0 -- libtiff=4.0.10 -- libxml2=2.9.8 -- matplotlib=3.0.3 +- libtiff=4.1.0 +- libxml2=2.9.10 +- matplotlib=3.1.3 - nomkl=3.0 -- notebook=5.7.5 +- notebook=6.0.3 - nose=1.3.7 -- numexpr=2.6.9 +- numexpr=2.7.1 - numpy=1.17.3 - openblas=0.3.6 - pandas=0.25.3 - patsy=0.5.1 -- psycopg2=2.7.7 -- pyarrow=0.15.1 +- pip=20.0.2 +- psycopg2=2.8.4 +- pyarrow=0.16.0 - pycrypto=2.6.1 -- pytest=4.3.0 +- pytest=5.3.5 - python=3.7.6 - pyyaml=3.13 -- requests=2.21.0 -- s3fs=0.2.0 -- seaborn=0.9.0 +- requests=2.22.0 +- s3fs=0.4.0 +- seaborn=0.10.0 - scipy=1.4.1 - scikit-learn=0.22.1 -- statsmodels=0.9.0 -- urllib3=1.24.1 +- statsmodels=0.11.0 +- urllib3=1.25.7 - xgboost=0.81 - pip: - - civis==1.12.0 + - civis==1.12.1 - civisml-extensions==0.2.1 - - dropbox==9.3.0 + - dropbox==9.4.0 - ftputil==3.4 - muffnn==2.3.0 - - pubnub==4.1.2 + - pubnub==4.3.0 - pysftp==0.2.9 - - python-simple-hipchat==0.4.0 - requests-toolbelt==0.9.1 - tensorflow==1.13.1 From d064bdcd029af822b68ce4cc744304f240ae9c75 Mon Sep 17 00:00:00 2001 From: Salil Gupta Date: Thu, 5 Mar 2020 14:39:51 -0600 Subject: [PATCH 44/63] upgrade civis-python (#76) * upgrade civis-python * update version and changelog --- CHANGELOG.md | 4 ++++ Dockerfile | 6 +++--- environment.yml | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c311290..af5f4f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ Version number changes (major.minor.micro) in this package denote the following: ## Unreleased +## [6.1.0] +### Package Updates +- civis 1.12.1 -> 1.13.0 (#76) + ## [6.0.0] ### Changed - Python 3.7.1 -> 3.7.6 (#74) diff --git a/Dockerfile b/Dockerfile index c463735..c04bf7a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -106,7 +106,7 @@ RUN jupyter nbextension enable --py widgetsnbextension # https://github.com/joblib/joblib/blob/0.11/joblib/parallel.py#L328L342 ENV JOBLIB_TEMP_FOLDER=/tmp -ENV VERSION=5.0.0 \ - VERSION_MAJOR=5 \ - VERSION_MINOR=0 \ +ENV VERSION=6.1.0 \ + VERSION_MAJOR=6 \ + VERSION_MINOR=1 \ VERSION_MICRO=0 diff --git a/environment.yml b/environment.yml index cee5ec2..1361529 100644 --- a/environment.yml +++ b/environment.yml @@ -50,7 +50,7 @@ dependencies: - urllib3=1.25.7 - xgboost=0.81 - pip: - - civis==1.12.1 + - civis==1.13.0 - civisml-extensions==0.2.1 - dropbox==9.4.0 - ftputil==3.4 From 0d95b0e4557bb85ab054438caefad59b17ed2c08 Mon Sep 17 00:00:00 2001 From: Michael Heilman Date: Wed, 22 Apr 2020 10:12:22 -0500 Subject: [PATCH 45/63] update civis to 1.14.0, muffnn to 2.3.1, and tensorflow to 1.15.2. bump version to 6.2.0 (#77) --- CHANGELOG.md | 6 ++++++ Dockerfile | 4 ++-- README.md | 2 +- environment.yml | 6 +++--- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index af5f4f8..8e33c17 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,12 @@ Version number changes (major.minor.micro) in this package denote the following: ## Unreleased +## [6.2.0] +### Package Updates +- civis 1.13.0 -> 1.14.0 (#77) +- muffnn 2.3.0 -> 2.3.1 (#77) +- tensorflow 1.13.1 -> 1.15.2 (#77) + ## [6.1.0] ### Package Updates - civis 1.12.1 -> 1.13.0 (#76) diff --git a/Dockerfile b/Dockerfile index c04bf7a..dd6a2e9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -106,7 +106,7 @@ RUN jupyter nbextension enable --py widgetsnbextension # https://github.com/joblib/joblib/blob/0.11/joblib/parallel.py#L328L342 ENV JOBLIB_TEMP_FOLDER=/tmp -ENV VERSION=6.1.0 \ +ENV VERSION=6.2.0 \ VERSION_MAJOR=6 \ - VERSION_MINOR=1 \ + VERSION_MINOR=2 \ VERSION_MICRO=0 diff --git a/README.md b/README.md index 69a153e..c280192 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![CircleCI](https://circleci.com/gh/civisanalytics/datascience-python/tree/master.svg?style=svg)](https://circleci.com/gh/civisanalytics/datascience-python/tree/master) -This image is created from the official Ubuntu 14.04 Docker image and contains popular Python packages for data science. +This image is created from the official Ubuntu 18.04 Docker image and contains popular Python packages for data science. If you are reading this README on DockerHub, then the links to files in the GitHub respository will be broken. Please read this documentation from [GitHub](https://github.com/civisanalytics/datascience-python) instead. diff --git a/environment.yml b/environment.yml index 1361529..e886a69 100644 --- a/environment.yml +++ b/environment.yml @@ -50,12 +50,12 @@ dependencies: - urllib3=1.25.7 - xgboost=0.81 - pip: - - civis==1.13.0 + - civis==1.14.0 - civisml-extensions==0.2.1 - dropbox==9.4.0 - ftputil==3.4 - - muffnn==2.3.0 + - muffnn==2.3.1 - pubnub==4.3.0 - pysftp==0.2.9 - requests-toolbelt==0.9.1 - - tensorflow==1.13.1 + - tensorflow==1.15.2 From a48266a94ea419884bce30774dadb2984a9c8765 Mon Sep 17 00:00:00 2001 From: Michael Heilman Date: Thu, 23 Apr 2020 09:29:30 -0500 Subject: [PATCH 46/63] bump civis-python to 1.14.1, bump version to 6.2.1 (#78) --- CHANGELOG.md | 4 ++++ Dockerfile | 4 ++-- environment.yml | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e33c17..54e240b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ Version number changes (major.minor.micro) in this package denote the following: ## Unreleased +## [6.2.1] +### Package Updates +- civis 1.14.0 -> 1.14.1 (#78) + ## [6.2.0] ### Package Updates - civis 1.13.0 -> 1.14.0 (#77) diff --git a/Dockerfile b/Dockerfile index dd6a2e9..1737de6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -106,7 +106,7 @@ RUN jupyter nbextension enable --py widgetsnbextension # https://github.com/joblib/joblib/blob/0.11/joblib/parallel.py#L328L342 ENV JOBLIB_TEMP_FOLDER=/tmp -ENV VERSION=6.2.0 \ +ENV VERSION=6.2.1 \ VERSION_MAJOR=6 \ VERSION_MINOR=2 \ - VERSION_MICRO=0 + VERSION_MICRO=1 diff --git a/environment.yml b/environment.yml index e886a69..b30a845 100644 --- a/environment.yml +++ b/environment.yml @@ -50,7 +50,7 @@ dependencies: - urllib3=1.25.7 - xgboost=0.81 - pip: - - civis==1.14.0 + - civis==1.14.1 - civisml-extensions==0.2.1 - dropbox==9.4.0 - ftputil==3.4 From 2e95ef83f50c536557be396053b36e47fda8672b Mon Sep 17 00:00:00 2001 From: Michael Heilman Date: Wed, 30 Sep 2020 11:47:15 -0500 Subject: [PATCH 47/63] update civis to 1.15.0 and tensorflow to 1.15.4 (cf. https://github.com/advisories/GHSA-4g9f-63rx-5cw4) (#79) --- CHANGELOG.md | 5 +++++ Dockerfile | 6 +++--- environment.yml | 4 ++-- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 54e240b..58c04bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ Version number changes (major.minor.micro) in this package denote the following: ## Unreleased +## [6.3.0] +### Package Updates +- civis 1.14.0 -> 1.15.0 (#78) +- tensorflow 1.15.2 -> 1.15.4 (#78) + ## [6.2.1] ### Package Updates - civis 1.14.0 -> 1.14.1 (#78) diff --git a/Dockerfile b/Dockerfile index 1737de6..5a3a6de 100644 --- a/Dockerfile +++ b/Dockerfile @@ -106,7 +106,7 @@ RUN jupyter nbextension enable --py widgetsnbextension # https://github.com/joblib/joblib/blob/0.11/joblib/parallel.py#L328L342 ENV JOBLIB_TEMP_FOLDER=/tmp -ENV VERSION=6.2.1 \ +ENV VERSION=6.3.0 \ VERSION_MAJOR=6 \ - VERSION_MINOR=2 \ - VERSION_MICRO=1 + VERSION_MINOR=3 \ + VERSION_MICRO=0 diff --git a/environment.yml b/environment.yml index b30a845..5102747 100644 --- a/environment.yml +++ b/environment.yml @@ -50,7 +50,7 @@ dependencies: - urllib3=1.25.7 - xgboost=0.81 - pip: - - civis==1.14.1 + - civis==1.15.0 - civisml-extensions==0.2.1 - dropbox==9.4.0 - ftputil==3.4 @@ -58,4 +58,4 @@ dependencies: - pubnub==4.3.0 - pysftp==0.2.9 - requests-toolbelt==0.9.1 - - tensorflow==1.15.2 + - tensorflow==1.15.4 From 696e25ef09dbb818dbda5156e1d64eec3705d663 Mon Sep 17 00:00:00 2001 From: Michael Heilman Date: Wed, 30 Sep 2020 12:08:49 -0500 Subject: [PATCH 48/63] fix changelog PR numbers (#80) --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 58c04bb..ba4511b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,8 +11,8 @@ Version number changes (major.minor.micro) in this package denote the following: ## [6.3.0] ### Package Updates -- civis 1.14.0 -> 1.15.0 (#78) -- tensorflow 1.15.2 -> 1.15.4 (#78) +- civis 1.14.0 -> 1.15.0 (#79) +- tensorflow 1.15.2 -> 1.15.4 (#79) ## [6.2.1] ### Package Updates From 07031bc92425e1fc65926488de86db54abe4c159 Mon Sep 17 00:00:00 2001 From: Chris Perry Date: Fri, 13 Nov 2020 12:19:13 -0600 Subject: [PATCH 49/63] [CIVP-22369] - bugfix python client release (#81) * [CIVP-22369] - bugfix python client release * updated changelog --- CHANGELOG.md | 4 ++++ Dockerfile | 4 ++-- environment.yml | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ba4511b..ae41844 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ Version number changes (major.minor.micro) in this package denote the following: ## Unreleased +## [6.3.1] +### Package Updates +- civis 1.15.0 -> 1.15.1 (#81) + ## [6.3.0] ### Package Updates - civis 1.14.0 -> 1.15.0 (#79) diff --git a/Dockerfile b/Dockerfile index 5a3a6de..01d3b16 100644 --- a/Dockerfile +++ b/Dockerfile @@ -106,7 +106,7 @@ RUN jupyter nbextension enable --py widgetsnbextension # https://github.com/joblib/joblib/blob/0.11/joblib/parallel.py#L328L342 ENV JOBLIB_TEMP_FOLDER=/tmp -ENV VERSION=6.3.0 \ +ENV VERSION=6.3.1 \ VERSION_MAJOR=6 \ VERSION_MINOR=3 \ - VERSION_MICRO=0 + VERSION_MICRO=1 diff --git a/environment.yml b/environment.yml index 5102747..6d7f076 100644 --- a/environment.yml +++ b/environment.yml @@ -50,7 +50,7 @@ dependencies: - urllib3=1.25.7 - xgboost=0.81 - pip: - - civis==1.15.0 + - civis==1.15.1 - civisml-extensions==0.2.1 - dropbox==9.4.0 - ftputil==3.4 From 89392be169e3e99670a0c29d0381889da2ffa2fd Mon Sep 17 00:00:00 2001 From: Katherine Date: Fri, 30 Jul 2021 10:51:35 -0400 Subject: [PATCH 50/63] [CIVP-24102] upgrade pyyaml to 5.2 (#82) * [CIVP-24102] upgrade pyyaml * fix changelog Co-authored-by: Virginia Fu --- CHANGELOG.md | 4 ++++ Dockerfile | 6 +++--- environment.yml | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ae41844..1266aaf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ Version number changes (major.minor.micro) in this package denote the following: ## Unreleased +## [6.4.0] +### Package Updates +- PyYAML 3.13 -> 5.2.0 + ## [6.3.1] ### Package Updates - civis 1.15.0 -> 1.15.1 (#81) diff --git a/Dockerfile b/Dockerfile index 01d3b16..4a10e7b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -106,7 +106,7 @@ RUN jupyter nbextension enable --py widgetsnbextension # https://github.com/joblib/joblib/blob/0.11/joblib/parallel.py#L328L342 ENV JOBLIB_TEMP_FOLDER=/tmp -ENV VERSION=6.3.1 \ +ENV VERSION=6.4.0 \ VERSION_MAJOR=6 \ - VERSION_MINOR=3 \ - VERSION_MICRO=1 + VERSION_MINOR=4 \ + VERSION_MICRO=0 diff --git a/environment.yml b/environment.yml index 6d7f076..9bbdcb9 100644 --- a/environment.yml +++ b/environment.yml @@ -40,7 +40,7 @@ dependencies: - pycrypto=2.6.1 - pytest=5.3.5 - python=3.7.6 -- pyyaml=3.13 +- pyyaml=5.2 - requests=2.22.0 - s3fs=0.4.0 - seaborn=0.10.0 From 64b245d897f15ec21e945b3e505b0b5f007d310a Mon Sep 17 00:00:00 2001 From: Jackson Lee <86482098+jacksonlee-civis@users.noreply.github.com> Date: Tue, 14 Dec 2021 11:02:41 -0500 Subject: [PATCH 51/63] [CIVIS-1836] ENH upgrade civis-python to v1.16.0 and prepare for v6.5.0 release (#84) * try civis-python v1.16.0 from PR branch * ENH use civis==1.16.0 * ENH update ds-py version in Dockerfile * MAINT update changelog --- CHANGELOG.md | 5 +++++ Dockerfile | 4 ++-- environment.yml | 3 +-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1266aaf..04b229b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ Version number changes (major.minor.micro) in this package denote the following: ## Unreleased +## [6.5.0] +### Package Updates +- civis 1.15.1 -> 1.16.0 (#84) +- Removed pubnub - not considered a breaking change, as it was a dependency for `civis` (#84) + ## [6.4.0] ### Package Updates - PyYAML 3.13 -> 5.2.0 diff --git a/Dockerfile b/Dockerfile index 4a10e7b..b4790b2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -106,7 +106,7 @@ RUN jupyter nbextension enable --py widgetsnbextension # https://github.com/joblib/joblib/blob/0.11/joblib/parallel.py#L328L342 ENV JOBLIB_TEMP_FOLDER=/tmp -ENV VERSION=6.4.0 \ +ENV VERSION=6.5.0 \ VERSION_MAJOR=6 \ - VERSION_MINOR=4 \ + VERSION_MINOR=5 \ VERSION_MICRO=0 diff --git a/environment.yml b/environment.yml index 9bbdcb9..e3c32fa 100644 --- a/environment.yml +++ b/environment.yml @@ -50,12 +50,11 @@ dependencies: - urllib3=1.25.7 - xgboost=0.81 - pip: - - civis==1.15.1 + - civis==1.16.0 - civisml-extensions==0.2.1 - dropbox==9.4.0 - ftputil==3.4 - muffnn==2.3.1 - - pubnub==4.3.0 - pysftp==0.2.9 - requests-toolbelt==0.9.1 - tensorflow==1.15.4 From 71df4e69d73e268aa57d9f48560806cb7e36a9ef Mon Sep 17 00:00:00 2001 From: Jackson Lee <86482098+jacksonlee-civis@users.noreply.github.com> Date: Wed, 15 Dec 2021 18:00:56 -0500 Subject: [PATCH 52/63] [CIVIS-1912] FIX pin cffi at 1.14.0; prepare for 6.5.1 bug fix release (#85) * FIX pin cffi to 1.14.0 * MAINT update changelog --- CHANGELOG.md | 4 ++++ environment.yml | 1 + 2 files changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 04b229b..31baf80 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ Version number changes (major.minor.micro) in this package denote the following: ## Unreleased +## [6.5.1] +### Package Updates +- Pin cffi at 1.14.0 (#85) + ## [6.5.0] ### Package Updates - civis 1.15.1 -> 1.16.0 (#84) diff --git a/environment.yml b/environment.yml index e3c32fa..88773f7 100644 --- a/environment.yml +++ b/environment.yml @@ -9,6 +9,7 @@ dependencies: - boto=2.49.0 - boto3=1.11.15 - bqplot=0.12.3 +- cffi=1.14.0 - click=6.7 - cloudpickle=1.2.2 - cython=0.29.15 From 6e2cb3c99c151192c3dc8430017bcf7e2dddf961 Mon Sep 17 00:00:00 2001 From: videogaber <6244640+videogaber@users.noreply.github.com> Date: Thu, 27 Jul 2023 13:14:59 -0500 Subject: [PATCH 53/63] [CIVIS-6103] Update to Python 3.11.4 + Refactor Dockerfile (#89) * switch to using debian based official python image. using the bookworm image * update readme * maintainer * remove matplot from circle ci * remove seaborn from circle ci * remove more unused things * numpy tests are running * PR feedback updates * gitignore * update readme * polish up * update changelog --- .circleci/config.yml | 20 ++---- .condarc | 7 -- .gitignore | 2 + CHANGELOG.md | 6 ++ Dockerfile | 119 +++++++--------------------------- README.md | 22 +++---- environment.yml | 61 ----------------- generate-requirements-full.sh | 3 + requirements-core.txt | 8 +++ requirements-full.txt | 105 ++++++++++++++++++++++++++++++ 10 files changed, 163 insertions(+), 190 deletions(-) delete mode 100644 .condarc create mode 100644 .gitignore delete mode 100644 environment.yml create mode 100755 generate-requirements-full.sh create mode 100644 requirements-core.txt create mode 100644 requirements-full.txt diff --git a/.circleci/config.yml b/.circleci/config.yml index b58f22f..b74c16f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,27 +15,15 @@ jobs: - run: name: Check that scipy links to OpenBLAS command: docker run ds-python python -c "from scipy.linalg import _fblas" - - run: - name: Verify that numpy and matplotlib function - command: docker run ds-python python -c "import numpy, os; import matplotlib.pyplot as plt; x = numpy.arange(100); y = numpy.sin(x); plt.plot(x, y);" - - run: - name: Check that seaborn is installed - command: docker run ds-python python -c "import seaborn" - run: name: Check that numpy imports from bash shell command: docker run -t ds-python /bin/bash -c "python -c 'import numpy'" - - run: - name: Check that tensorflow imports from bash shell - command: docker run ds-python /bin/bash -c "python -c 'import tensorflow'" - run: name: Verify that numpy does not link to MKL command: docker run ds-python python -c "from numpy.distutils import system_info; assert system_info.get_info('mkl') == {}" - run: - name: Run numpy unit tests - command: docker run ds-python python -c "import numpy; numpy.test()" - - run: - name: List packages installed from conda-forge - command: docker run ds-python /bin/bash -c "conda list|grep conda-forge" + name: Validate compatability of civis-python with other packages + command: docker run ds-python /bin/bash -c "python -c 'import civis'" - run: - name: Check for conda-pip conflicts - command: docker run ds-python /bin/bash -c "conda list|python -c \"import sys; from collections import Counter; duplicates = {k for k, v in Counter([l.split()[0] for l in sys.stdin.readlines() if not l.startswith('#')]).items() if v>1}; assert not duplicates, f'These packages are installed by both pip and conda:\n{duplicates}'\"" + name: Run numpy unit tests + command: docker run ds-python /bin/bash -c 'pip install pytest hypothesis; python -c "import numpy; numpy.test()"' diff --git a/.condarc b/.condarc deleted file mode 100644 index 364cb1a..0000000 --- a/.condarc +++ /dev/null @@ -1,7 +0,0 @@ -channels: - - defaults - -show_channel_urls: True - -create_default_packages: - - nomkl diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c1d6aa5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.venv +.vscode diff --git a/CHANGELOG.md b/CHANGELOG.md index 31baf80..400f61c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,12 @@ Version number changes (major.minor.micro) in this package denote the following: ## Unreleased +## [7.0.0] +- Python updated to v3.11.4 +- Refactors Dockerfile to use the official Python docker image +- Removes Conda Dependency +- Keeps only the core data science python packages + ## [6.5.1] ### Package Updates - Pin cffi at 1.14.0 (#85) diff --git a/Dockerfile b/Dockerfile index b4790b2..8289494 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,104 +1,33 @@ -FROM ubuntu:18.04 -MAINTAINER support@civisanalytics.com +FROM python:3.11.4-bookworm + +LABEL maintainer = support@civisanalytics.com RUN DEBIAN_FRONTEND=noninteractive apt-get update -y --no-install-recommends && \ apt-get install -y --no-install-recommends locales && \ locale-gen en_US.UTF-8 && \ apt-get install -y --no-install-recommends software-properties-common && \ apt-get install -y --no-install-recommends \ - make \ - automake \ - libpq-dev \ - libffi-dev \ - gfortran \ - g++ \ - git \ - libboost-program-options-dev \ - libtool \ - libxrender1 \ - wget \ - ca-certificates \ - curl && \ + make \ + automake \ + libpq-dev \ + libffi-dev \ + gfortran \ + g++ \ + git \ + libboost-program-options-dev \ + libtool \ + libxrender1 \ + wget \ + ca-certificates \ + curl && \ apt-get clean -y && \ rm -rf /var/lib/apt/lists/* -# Set environment variables for UTF-8, conda, and shell environments -ENV LANG=en_US.UTF-8 \ - LANGUAGE=en_US:en \ - LC_ALL=en_US.UTF-8 \ - CONDARC=/opt/conda/.condarc \ - BASH_ENV=/etc/profile \ - PATH=/opt/conda/bin:$PATH \ - CIVIS_MINICONDA_VERSION=4.5.12 \ - CIVIS_CONDA_VERSION=4.8.1 \ - CIVIS_PYTHON_VERSION=3.7.6 - -# Conda install. -# -# Everything is installed in the root environment. This allows for -# upgrades to the packages and eliminates the pain of trying to activate -# some other environment automatically for the many different ways -# people can use a docker image. -# -# Things are pinned to prevent upgrades from conda and force it to -# resolve dependencies relative to a fixed conda & python version. -# -# Note that the python version is also listed in the environment.yml -# file. The version in CIVIS_PYTHON_VERSION is the source of truth. -# If you want to change the python version, you need to change it in -# **both** places. The python version has been left in the `environment.yml` -# file so that people can create environments equivalent to this -# container. -# -# The ordering of these steps seems to matter. You seem to have to -# install a specific python version by hand and then pin it. -# 1) install conda using CIVIS_MINICONDA_VERSION (may not be most up to date conda version) -# 2) pin conda to the version given by CIVIS_CONDA_VERSION -# 3) install the python version CIVIS_PYTHON_VERSION -# 4) pin the python version -# -# Extra symlinks are added at the end because... -# Red Hat and Debian use different names for this file. git2R wants the latter. -# See conda-recipes GH 423 -RUN echo 'export PATH=/opt/conda/bin:$PATH' > /etc/profile.d/conda.sh && \ - wget --quiet https://repo.continuum.io/miniconda/Miniconda3-${CIVIS_MINICONDA_VERSION}-Linux-x86_64.sh && \ - /bin/bash /Miniconda3-${CIVIS_MINICONDA_VERSION}-Linux-x86_64.sh -b -p /opt/conda && \ - rm Miniconda3-${CIVIS_MINICONDA_VERSION}-Linux-x86_64.sh && \ - /opt/conda/bin/conda install --yes conda==${CIVIS_MINICONDA_VERSION} && \ - conda install conda=${CIVIS_CONDA_VERSION} && \ - echo "conda ==${CIVIS_CONDA_VERSION}" > /opt/conda/conda-meta/pinned && \ - conda install --yes python==${CIVIS_PYTHON_VERSION} && \ - echo "python ==${CIVIS_PYTHON_VERSION}" >> /opt/conda/conda-meta/pinned && \ - conda clean --all -y && \ - ln -s /opt/conda/lib/libopenblas.so /opt/conda/lib/libblas.so && \ - ln -s /opt/conda/lib/libopenblas.so /opt/conda/lib/liblapack.so && \ - ln -s /opt/conda/lib/libssl.so /opt/conda/lib/libssl.so.6 && \ - ln -s /opt/conda/lib/libcrypto.so /opt/conda/lib/libcrypto.so.6 - -# Install boto in the base environment for private s3 channel support. -# Install Python Packages -COPY .condarc /opt/conda/.condarc -COPY environment.yml environment.yml -RUN conda install -y boto && \ - conda install -y nomkl && \ - conda env update -f environment.yml -n root && \ - conda clean --all -y && \ - rm -rf ~/.cache/pip - -# We aren't running a GUI, so force matplotlib to use -# the non-interactive "Agg" backend for graphics. -# Run matplotlib once to build the font cache. -ENV MATPLOTLIBRC=${HOME}/.config/matplotlib/matplotlibrc -RUN mkdir -p ${HOME}/.config/matplotlib && \ - echo "backend : Agg" > ${HOME}/.config/matplotlib/matplotlibrc && \ - python -c "import matplotlib.pyplot" +COPY requirements-full.txt . -# Enable widgetsnbextension for jupyter widgets. -# See https://ipywidgets.readthedocs.io/en/stable/user_install.html. -# This enables the extension in the conda environment. The conda-forge version -# does this upon installation, but the default channel version doesn't seem to, -# so we'll run this (again) just in case. -RUN jupyter nbextension enable --py widgetsnbextension +RUN pip install -r requirements-full.txt && \ + pip cache purge && \ + rm requirements-full.txt # Instruct joblib to use disk for temporary files. Joblib defaults to # /shm when that directory is present. In the Docker container, /shm is @@ -106,7 +35,7 @@ RUN jupyter nbextension enable --py widgetsnbextension # https://github.com/joblib/joblib/blob/0.11/joblib/parallel.py#L328L342 ENV JOBLIB_TEMP_FOLDER=/tmp -ENV VERSION=6.5.0 \ - VERSION_MAJOR=6 \ - VERSION_MINOR=5 \ - VERSION_MICRO=0 +ENV VERSION=7.0.0 \ + VERSION_MAJOR=7 \ + VERSION_MINOR=0 \ + VERSION_MICRO=0 diff --git a/README.md b/README.md index c280192..232d95e 100644 --- a/README.md +++ b/README.md @@ -68,18 +68,18 @@ default location for staging temporary files to the /tmp directory. The normal default is /shm. /shm is a RAM disk which defaults to a 64 MB size in Docker containers, too small for typical scientific computing. -# Creating Equivalent Local Environments +# Install a new PyPi Package Dependency +1. Add new dependency to `requirements-core.txt` file. Include the version +2. Run script `generate-requirements-full.sh` + +# Updating existing PyPi Package Version +1. Update version of existing package in `requirements-core.txt` +2. Run script `generate-requirements-full.sh` -The `environment.yml` file in this repo can be used to create a python environment that is -equivalent to the one in the container. This environment will be named `datascience`. -The environment installs in Ubuntu Linux (this is the OS of the Dockerfile). -It will install in OS X, but the `xgboost` install requires either -the `gcc` v5 or the `clang-omp` compiler, neither of which are natively provided in OS X. -If you wish to set up this environment in OS X, you may either -- Remove `xgboost` from the `environment.yml` file before using it to create the environment -- Use [Homebrew](https://brew.sh/) to install `gcc-5`. You can do that via -`brew install gcc@5 --without-multilib`. Be warned that this installation will take -a long time. +# Creating Equivalent Local Environments +1. Create a new python environment `python -m venv .venv`. +2. Activate your new python environment `source .venv/bin/activate` +3. Install requirements.txt `pip install -r requirements-full.txt` # Contributing diff --git a/environment.yml b/environment.yml deleted file mode 100644 index 88773f7..0000000 --- a/environment.yml +++ /dev/null @@ -1,61 +0,0 @@ -name: datascience -channels: -- defaults -- conda-forge -dependencies: -- awscli=1.17.15 -- beautifulsoup4=4.8.2 -- botocore=1.14.15 -- boto=2.49.0 -- boto3=1.11.15 -- bqplot=0.12.3 -- cffi=1.14.0 -- click=6.7 -- cloudpickle=1.2.2 -- cython=0.29.15 -- dask=2.10.1 -- feather-format=0.4.0 -- glmnet=2.1.1 -- ipython=7.12.0 -- ipywidgets=7.5.1 -- jinja2=2.11.1 -- joblib=0.14.1 -- jsonschema=3.2.0 -- jupyter=1.0.0 -- libffi=3.2.1 -- libgfortran=3.0.0 -- libtiff=4.1.0 -- libxml2=2.9.10 -- matplotlib=3.1.3 -- nomkl=3.0 -- notebook=6.0.3 -- nose=1.3.7 -- numexpr=2.7.1 -- numpy=1.17.3 -- openblas=0.3.6 -- pandas=0.25.3 -- patsy=0.5.1 -- pip=20.0.2 -- psycopg2=2.8.4 -- pyarrow=0.16.0 -- pycrypto=2.6.1 -- pytest=5.3.5 -- python=3.7.6 -- pyyaml=5.2 -- requests=2.22.0 -- s3fs=0.4.0 -- seaborn=0.10.0 -- scipy=1.4.1 -- scikit-learn=0.22.1 -- statsmodels=0.11.0 -- urllib3=1.25.7 -- xgboost=0.81 -- pip: - - civis==1.16.0 - - civisml-extensions==0.2.1 - - dropbox==9.4.0 - - ftputil==3.4 - - muffnn==2.3.1 - - pysftp==0.2.9 - - requests-toolbelt==0.9.1 - - tensorflow==1.15.4 diff --git a/generate-requirements-full.sh b/generate-requirements-full.sh new file mode 100755 index 0000000..5a98a86 --- /dev/null +++ b/generate-requirements-full.sh @@ -0,0 +1,3 @@ +#!/bin/bash +pip install pip-tools +pip-compile --output-file=requirements-full.txt --pip-args='--prefer-binary' requirements-core.txt diff --git a/requirements-core.txt b/requirements-core.txt new file mode 100644 index 0000000..7cbbd8a --- /dev/null +++ b/requirements-core.txt @@ -0,0 +1,8 @@ +awscli==1.29.5 +boto3==1.28.5 +civis==1.16.1 +numpy==1.25.1 +pandas==2.0.3 +requests==2.31.0 +scikit-learn==1.3.0 +scipy==1.11.1 diff --git a/requirements-full.txt b/requirements-full.txt new file mode 100644 index 0000000..f80ac49 --- /dev/null +++ b/requirements-full.txt @@ -0,0 +1,105 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile --output-file=requirements-full.txt --pip-args='--prefer-binary' requirements-core.txt +# +attrs==23.1.0 + # via + # jsonschema + # referencing +awscli==1.29.5 + # via -r requirements-core.txt +boto3==1.28.5 + # via -r requirements-core.txt +botocore==1.31.5 + # via + # awscli + # boto3 + # s3transfer +certifi==2023.5.7 + # via requests +charset-normalizer==3.2.0 + # via requests +civis==1.16.1 + # via -r requirements-core.txt +click==8.1.6 + # via civis +cloudpickle==2.2.1 + # via civis +colorama==0.4.4 + # via awscli +docutils==0.16 + # via awscli +idna==3.4 + # via requests +jmespath==1.0.1 + # via + # boto3 + # botocore +joblib==1.2.0 + # via + # civis + # scikit-learn +jsonref==0.2 + # via civis +jsonschema==4.18.4 + # via civis +jsonschema-specifications==2023.7.1 + # via jsonschema +numpy==1.25.1 + # via + # -r requirements-core.txt + # pandas + # scikit-learn + # scipy +pandas==2.0.3 + # via -r requirements-core.txt +pyasn1==0.5.0 + # via rsa +python-dateutil==2.8.2 + # via + # botocore + # pandas +pytz==2023.3 + # via pandas +pyyaml==6.0.1 + # via + # awscli + # civis +referencing==0.30.0 + # via + # jsonschema + # jsonschema-specifications +requests==2.31.0 + # via + # -r requirements-core.txt + # civis +rpds-py==0.9.2 + # via + # jsonschema + # referencing +rsa==4.7.2 + # via awscli +s3transfer==0.6.1 + # via + # awscli + # boto3 +scikit-learn==1.3.0 + # via -r requirements-core.txt +scipy==1.11.1 + # via + # -r requirements-core.txt + # scikit-learn +six==1.16.0 + # via python-dateutil +tenacity==8.2.2 + # via civis +threadpoolctl==3.2.0 + # via scikit-learn +tzdata==2023.3 + # via pandas +urllib3==1.26.16 + # via + # botocore + # requests From 73a61702b1b3f5c3381d7ee95feb6194e43c2b9d Mon Sep 17 00:00:00 2001 From: videogaber <6244640+videogaber@users.noreply.github.com> Date: Mon, 31 Jul 2023 13:02:33 -0500 Subject: [PATCH 54/63] [CIVIS-6103] fix missing public keys (#91) * add public keys * add temporary push file * break up RUN command for debugging * seperate keys * remmove apt-key commands * curl * try again * update build spec command * add a space * stop push for now * copy pasta * collapse run commands * collapse commands * fix tag * cleanup * Update buildspec/release.yaml use --all-tags Co-authored-by: Michael Peterson --------- Co-authored-by: Michael Peterson --- buildspec/release.yaml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/buildspec/release.yaml b/buildspec/release.yaml index a1fcc27..d31ecd2 100644 --- a/buildspec/release.yaml +++ b/buildspec/release.yaml @@ -1,18 +1,16 @@ version: 0.2 phases: - pre_build: - commands: - - echo Logging in to Amazon ECR... - - $(aws ecr get-login --no-include-email --region $AWS_DEFAULT_REGION) build: commands: + - echo Logging in to Amazon ECR... + - aws ecr get-login-password --region ${AWS_DEFAULT_REGION} | docker login --username AWS --password-stdin ${REPOSITORY_URI} - echo Building the Docker image... - PATCH_TAG=${TAG_NAME#"v"} # major.minor.patch - MINOR_TAG=${PATCH_TAG%.*} # major.minor - MAJOR_TAG=${MINOR_TAG%.*} # major - docker build -t ${REPOSITORY_URI}:${PATCH_TAG} -t ${REPOSITORY_URI}:${MINOR_TAG} -t ${REPOSITORY_URI}:${MAJOR_TAG} . - - docker push ${REPOSITORY_URI} + - docker image push --all-tags ${REPOSITORY_URI} post_build: commands: - echo Build completed! From 3b879f40cb9647fa0368302e73f7731af87467b5 Mon Sep 17 00:00:00 2001 From: videogaber <6244640+videogaber@users.noreply.github.com> Date: Mon, 31 Jul 2023 15:52:42 -0500 Subject: [PATCH 55/63] update merge_master.yaml (#93) --- buildspec/merge_master.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/buildspec/merge_master.yaml b/buildspec/merge_master.yaml index 71c9e38..7cfb899 100644 --- a/buildspec/merge_master.yaml +++ b/buildspec/merge_master.yaml @@ -3,12 +3,12 @@ phases: pre_build: commands: - echo Logging in to Amazon ECR... - - $(aws ecr get-login --no-include-email --region $AWS_DEFAULT_REGION) + - aws ecr get-login-password --region ${AWS_DEFAULT_REGION} | docker login --username AWS --password-stdin ${REPOSITORY_URI} build: commands: - echo Building the Docker image... - docker build -t ${REPOSITORY_URI}:latest . - - docker push ${REPOSITORY_URI} + - docker image push --all-tags ${REPOSITORY_URI} post_build: commands: - echo Build completed! From 40f88cbf0cbfe0cbc936af23b28bed9937747ef5 Mon Sep 17 00:00:00 2001 From: Jackson Lee <86482098+jacksonlee-civis@users.noreply.github.com> Date: Wed, 22 May 2024 11:00:28 -0400 Subject: [PATCH 56/63] [CIVIS-2325] DEP upgrade to Python 3.12, update dependency versions (#94) * DEP use python 3.12, update dependency versions * TST drop legacy test for numpy not using mkl * MAINT update changelog * DEP refresh req files for requests * TST set up test suite * MAINT add github templates * TST close file object properly * TST run test suite in verbose mode --- .circleci/config.yml | 22 +++-------- .circleci/test_image.py | 37 ++++++++++++++++++ .github/ISSUE_TEMPLATE/general.md | 11 ++++++ .github/PULL_REQUEST_TEMPLATE.md | 9 +++++ CHANGELOG.md | 12 ++++++ Dockerfile | 13 +++++-- README.md | 14 ++----- generate-requirements-full.sh | 5 ++- requirements-core.txt | 16 ++++---- requirements-full.txt | 62 +++++++++++++++---------------- 10 files changed, 130 insertions(+), 71 deletions(-) create mode 100755 .circleci/test_image.py create mode 100644 .github/ISSUE_TEMPLATE/general.md create mode 100644 .github/PULL_REQUEST_TEMPLATE.md diff --git a/.circleci/config.yml b/.circleci/config.yml index b74c16f..e6c439a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,29 +1,17 @@ -version: 2 +version: 2.1 jobs: build: docker: - - image: circleci/python:3.7.0 + - image: cimg/python:3.12 steps: - checkout - setup_remote_docker - run: name: Build container - command: docker build -t ds-python . + command: docker build --target test -t ds-python . - run: name: Verify build completed command: docker run ds-python /bin/bash -c "echo BUILDS OK" - run: - name: Check that scipy links to OpenBLAS - command: docker run ds-python python -c "from scipy.linalg import _fblas" - - run: - name: Check that numpy imports from bash shell - command: docker run -t ds-python /bin/bash -c "python -c 'import numpy'" - - run: - name: Verify that numpy does not link to MKL - command: docker run ds-python python -c "from numpy.distutils import system_info; assert system_info.get_info('mkl') == {}" - - run: - name: Validate compatability of civis-python with other packages - command: docker run ds-python /bin/bash -c "python -c 'import civis'" - - run: - name: Run numpy unit tests - command: docker run ds-python /bin/bash -c 'pip install pytest hypothesis; python -c "import numpy; numpy.test()"' + name: Run tests + command: docker run ds-python /test_image.py -vv diff --git a/.circleci/test_image.py b/.circleci/test_image.py new file mode 100755 index 0000000..7d35606 --- /dev/null +++ b/.circleci/test_image.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python + +import os +import re +import unittest + + +class TestImage(unittest.TestCase): + + def test_version(self): + version_in_env_var = os.getenv("VERSION") + major = os.getenv("VERSION_MAJOR") + minor = os.getenv("VERSION_MINOR") + micro = os.getenv("VERSION_MICRO") + self.assertTrue(major.isdigit()) + self.assertTrue(minor.isdigit()) + self.assertTrue(micro.isdigit()) + self.assertEqual(version_in_env_var, f"{major}.{minor}.{micro}") + + with open("CHANGELOG.md") as changelog: + version_in_changelog = re.search( + r"##\s+\[(\d+\.\d+\.\d+)]", changelog.read() + ).groups()[0] + self.assertEqual(version_in_changelog, version_in_env_var) + + def test_scipy_links_to_openblas(self): + from scipy.linalg import _fblas # noqa: F401 + + def test_numpy_can_import(self): + import numpy as np # noqa: F401 + + def test_civis_can_import(self): + import civis # noqa: F401 + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/ISSUE_TEMPLATE/general.md b/.github/ISSUE_TEMPLATE/general.md new file mode 100644 index 0000000..5e6ede5 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/general.md @@ -0,0 +1,11 @@ +--- +name: General +about: Ask a question, report a potential issue, etc. +title: '' +labels: '' +assignees: '' + +--- + +**Note:** Civis employees should _not_ use the GitHub Issues feature at the public "civis-python" codebase +to file a ticket, and should instead use the internal ticketing system. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..83b1957 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,9 @@ + + +--- + +- [ ] (For Civis employees only) Reference to a relevant ticket in the pull request title +- [ ] Changelog entry added to `CHANGELOG.md` at the repo's root level +- [ ] Description of change in the pull request description +- [ ] If applicable, unit tests have been added and/or updated +- [ ] The CircleCI builds have all passed diff --git a/CHANGELOG.md b/CHANGELOG.md index 400f61c..04ad3ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,18 @@ Version number changes (major.minor.micro) in this package denote the following: ## Unreleased +## [7.1.0] +- Python updated to v3.12.3 +- Core dependencies updated to latest versions: + * awscli 1.29.5 -> 1.32.109 + * boto3 1.28.5 -> 1.34.109 + * civis 1.16.1 -> 2.0.0 + * numpy 1.25.1 -> 1.26.4 + * pandas 2.0.3 -> 2.2.2 + * requests 2.31.0 -> 2.32.2 + * scikit-learn 1.3.0 -> 1.5.0 + * scipy 1.11.1 -> 1.13.0 + ## [7.0.0] - Python updated to v3.11.4 - Refactors Dockerfile to use the official Python docker image diff --git a/Dockerfile b/Dockerfile index 8289494..49f1124 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11.4-bookworm +FROM python:3.12.3-slim AS production LABEL maintainer = support@civisanalytics.com @@ -35,7 +35,14 @@ RUN pip install -r requirements-full.txt && \ # https://github.com/joblib/joblib/blob/0.11/joblib/parallel.py#L328L342 ENV JOBLIB_TEMP_FOLDER=/tmp -ENV VERSION=7.0.0 \ +ENV VERSION=7.1.0 \ VERSION_MAJOR=7 \ - VERSION_MINOR=0 \ + VERSION_MINOR=1 \ VERSION_MICRO=0 + +FROM production AS test +COPY .circleci/test_image.py . +COPY CHANGELOG.md . + +# Defaults to production as the final stage +FROM production diff --git a/README.md b/README.md index 232d95e..ae170de 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,7 @@ [![CircleCI](https://circleci.com/gh/civisanalytics/datascience-python/tree/master.svg?style=svg)](https://circleci.com/gh/civisanalytics/datascience-python/tree/master) -This image is created from the official Ubuntu 18.04 Docker image and contains popular Python packages for data science. - -If you are reading this README on DockerHub, then the links to files in the GitHub respository will be broken. Please read this documentation from [GitHub](https://github.com/civisanalytics/datascience-python) instead. +If you are reading this README on DockerHub, then the links to files in the GitHub repository will be broken. Please read this documentation from [GitHub](https://github.com/civisanalytics/datascience-python) instead. # Introduction @@ -35,7 +33,7 @@ to retrieve a reproducible environment. Inside the datascience-python Docker image, Python packages are installed in the `root` environment. For a full list of included Python libraries, see the -[environment.yml](environment.yml) file. +[requirements-core.txt](requirements-core.txt) file. To start a Docker container from the datascience-python image and interact with it from a bash prompt, use @@ -56,7 +54,7 @@ VERSION_MAJOR VERSION_MINOR VERSION_MICRO ``` -VERSION contains the full version string, e.g. "1.0.3". VERSION_MAJOR, +VERSION contains the full version string, e.g., "1.0.3". VERSION_MAJOR, VERSION_MINOR, and VERSION_MICRO each contain a single integer. ## Joblib Temporary Files @@ -68,10 +66,6 @@ default location for staging temporary files to the /tmp directory. The normal default is /shm. /shm is a RAM disk which defaults to a 64 MB size in Docker containers, too small for typical scientific computing. -# Install a new PyPi Package Dependency -1. Add new dependency to `requirements-core.txt` file. Include the version -2. Run script `generate-requirements-full.sh` - # Updating existing PyPi Package Version 1. Update version of existing package in `requirements-core.txt` 2. Run script `generate-requirements-full.sh` @@ -96,7 +90,7 @@ and describe any changes in the [change log](CHANGELOG.md). This repo has autobuild enabled. Any PR that is merged to master will be built as the `latest` tag on Dockerhub. Once you are ready to create a new version, go to the "releases" tab of the repository and click -"Draft a new release". Github will prompt you to create a new tag, release title, and release +"Draft a new release". GitHub will prompt you to create a new tag, release title, and release description. The tag should use semantic versioning in the form "vX.X.X"; "major.minor.micro". The title of the release should be the same as the tag. Include a change log in the release description. Once the release is tagged, DockerHub will automatically build three identical containers, with labels diff --git a/generate-requirements-full.sh b/generate-requirements-full.sh index 5a98a86..84753d5 100755 --- a/generate-requirements-full.sh +++ b/generate-requirements-full.sh @@ -1,3 +1,4 @@ #!/bin/bash -pip install pip-tools -pip-compile --output-file=requirements-full.txt --pip-args='--prefer-binary' requirements-core.txt +python --version +pip install --upgrade pip-tools +pip-compile --output-file=requirements-full.txt --pip-args='--prefer-binary' --strip-extras --upgrade requirements-core.txt diff --git a/requirements-core.txt b/requirements-core.txt index 7cbbd8a..65b630e 100644 --- a/requirements-core.txt +++ b/requirements-core.txt @@ -1,8 +1,8 @@ -awscli==1.29.5 -boto3==1.28.5 -civis==1.16.1 -numpy==1.25.1 -pandas==2.0.3 -requests==2.31.0 -scikit-learn==1.3.0 -scipy==1.11.1 +awscli==1.32.109 +boto3==1.34.109 +civis==2.0.0 +numpy==1.26.4 +pandas==2.2.2 +requests==2.32.2 +scikit-learn==1.5.0 +scipy==1.13.0 diff --git a/requirements-full.txt b/requirements-full.txt index f80ac49..af54071 100644 --- a/requirements-full.txt +++ b/requirements-full.txt @@ -1,37 +1,37 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # -# pip-compile --output-file=requirements-full.txt --pip-args='--prefer-binary' requirements-core.txt +# pip-compile --output-file=requirements-full.txt --pip-args='--prefer-binary' --strip-extras requirements-core.txt # -attrs==23.1.0 +attrs==23.2.0 # via # jsonschema # referencing -awscli==1.29.5 +awscli==1.32.109 # via -r requirements-core.txt -boto3==1.28.5 +boto3==1.34.109 # via -r requirements-core.txt -botocore==1.31.5 +botocore==1.34.109 # via # awscli # boto3 # s3transfer -certifi==2023.5.7 +certifi==2024.2.2 # via requests -charset-normalizer==3.2.0 +charset-normalizer==3.3.2 # via requests -civis==1.16.1 +civis==2.0.0 # via -r requirements-core.txt -click==8.1.6 +click==8.1.7 # via civis -cloudpickle==2.2.1 +cloudpickle==3.0.0 # via civis -colorama==0.4.4 +colorama==0.4.6 # via awscli docutils==0.16 # via awscli -idna==3.4 +idna==3.7 # via requests jmespath==1.0.1 # via @@ -41,65 +41,65 @@ joblib==1.2.0 # via # civis # scikit-learn -jsonref==0.2 +jsonref==1.1.0 # via civis -jsonschema==4.18.4 +jsonschema==4.22.0 # via civis -jsonschema-specifications==2023.7.1 +jsonschema-specifications==2023.12.1 # via jsonschema -numpy==1.25.1 +numpy==1.26.4 # via # -r requirements-core.txt # pandas # scikit-learn # scipy -pandas==2.0.3 +pandas==2.2.2 # via -r requirements-core.txt -pyasn1==0.5.0 +pyasn1==0.6.0 # via rsa -python-dateutil==2.8.2 +python-dateutil==2.9.0.post0 # via # botocore # pandas -pytz==2023.3 +pytz==2024.1 # via pandas pyyaml==6.0.1 # via # awscli # civis -referencing==0.30.0 +referencing==0.35.1 # via # jsonschema # jsonschema-specifications -requests==2.31.0 +requests==2.32.2 # via # -r requirements-core.txt # civis -rpds-py==0.9.2 +rpds-py==0.18.1 # via # jsonschema # referencing rsa==4.7.2 # via awscli -s3transfer==0.6.1 +s3transfer==0.10.1 # via # awscli # boto3 -scikit-learn==1.3.0 +scikit-learn==1.5.0 # via -r requirements-core.txt -scipy==1.11.1 +scipy==1.13.0 # via # -r requirements-core.txt # scikit-learn six==1.16.0 # via python-dateutil -tenacity==8.2.2 +tenacity==8.3.0 # via civis -threadpoolctl==3.2.0 +threadpoolctl==3.5.0 # via scikit-learn -tzdata==2023.3 +tzdata==2024.1 # via pandas -urllib3==1.26.16 +urllib3==2.2.1 # via # botocore # requests From 9bd3581d16628fdfceb7d954f917861b0e5b02b8 Mon Sep 17 00:00:00 2001 From: Jackson Lee <86482098+jacksonlee-civis@users.noreply.github.com> Date: Thu, 23 May 2024 16:28:40 -0400 Subject: [PATCH 57/63] [CIVIS-8647] DEP update civis to 2.1.0 (#95) --- CHANGELOG.md | 7 +++++++ Dockerfile | 4 ++-- requirements-core.txt | 8 ++++---- requirements-full.txt | 10 +++++----- 4 files changed, 18 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 04ad3ae..3cf4ab0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,13 @@ Version number changes (major.minor.micro) in this package denote the following: ## Unreleased +## [7.2.0] +- Core dependencies updated to latest versions: + * awscli 1.32.109 -> 1.32.112 + * boto3 1.34.109 -> 1.34.112 + * civis 2.0.0 -> 2.1.0 + * scipy 1.13.0 -> 1.13.1 + ## [7.1.0] - Python updated to v3.12.3 - Core dependencies updated to latest versions: diff --git a/Dockerfile b/Dockerfile index 49f1124..16ed216 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,9 +35,9 @@ RUN pip install -r requirements-full.txt && \ # https://github.com/joblib/joblib/blob/0.11/joblib/parallel.py#L328L342 ENV JOBLIB_TEMP_FOLDER=/tmp -ENV VERSION=7.1.0 \ +ENV VERSION=7.2.0 \ VERSION_MAJOR=7 \ - VERSION_MINOR=1 \ + VERSION_MINOR=2 \ VERSION_MICRO=0 FROM production AS test diff --git a/requirements-core.txt b/requirements-core.txt index 65b630e..7a24a2e 100644 --- a/requirements-core.txt +++ b/requirements-core.txt @@ -1,8 +1,8 @@ -awscli==1.32.109 -boto3==1.34.109 -civis==2.0.0 +awscli==1.32.112 +boto3==1.34.112 +civis==2.1.0 numpy==1.26.4 pandas==2.2.2 requests==2.32.2 scikit-learn==1.5.0 -scipy==1.13.0 +scipy==1.13.1 diff --git a/requirements-full.txt b/requirements-full.txt index af54071..b4effcb 100644 --- a/requirements-full.txt +++ b/requirements-full.txt @@ -8,11 +8,11 @@ attrs==23.2.0 # via # jsonschema # referencing -awscli==1.32.109 +awscli==1.32.112 # via -r requirements-core.txt -boto3==1.34.109 +boto3==1.34.112 # via -r requirements-core.txt -botocore==1.34.109 +botocore==1.34.112 # via # awscli # boto3 @@ -21,7 +21,7 @@ certifi==2024.2.2 # via requests charset-normalizer==3.3.2 # via requests -civis==2.0.0 +civis==2.1.0 # via -r requirements-core.txt click==8.1.7 # via civis @@ -87,7 +87,7 @@ s3transfer==0.10.1 # boto3 scikit-learn==1.5.0 # via -r requirements-core.txt -scipy==1.13.0 +scipy==1.13.1 # via # -r requirements-core.txt # scikit-learn From 3271eb0cad84abdc14ec62229222ae057ab1d0ec Mon Sep 17 00:00:00 2001 From: Jackson Lee <86482098+jacksonlee-civis@users.noreply.github.com> Date: Tue, 18 Jun 2024 10:01:08 -0400 Subject: [PATCH 58/63] [CIVIS-8784] ENH update dependency versions: civis-python v2.3.0, python v3.12.4 (#96) * ENH bump civis-python to v2.3.0 * ENH bump python to v3.12.4 --- .circleci/test_image.py | 10 ++++++++++ CHANGELOG.md | 9 +++++++++ Dockerfile | 9 ++++----- requirements-core.txt | 10 +++++----- requirements-full.txt | 20 ++++++++++---------- 5 files changed, 38 insertions(+), 20 deletions(-) diff --git a/.circleci/test_image.py b/.circleci/test_image.py index 7d35606..99b6239 100755 --- a/.circleci/test_image.py +++ b/.circleci/test_image.py @@ -29,8 +29,18 @@ def test_scipy_links_to_openblas(self): def test_numpy_can_import(self): import numpy as np # noqa: F401 + def test_sklearn_can_import(self): + import sklearn # noqa: F401 + def test_civis_can_import(self): import civis # noqa: F401 + # civis-python uses lazy imports since v2.3.0, + # so try to import the top-level modules. + import civis.io # noqa: F401 + import civis.parallel # noqa: F401 + import civis.futures # noqa: F401 + import civis.ml # noqa: F401 + import civis.utils # noqa: F401 if __name__ == "__main__": diff --git a/CHANGELOG.md b/CHANGELOG.md index 3cf4ab0..4ee9065 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,15 @@ Version number changes (major.minor.micro) in this package denote the following: ## Unreleased +## [7.3.0] +- Core dependencies updated to latest versions: + * awscli 1.32.112 -> 1.33.9 + * boto3 1.34.112 -> 1.34.127 + * civis 2.1.0 -> 2.3.0 + * numpy 1.26.4 -> 2.0.0 + * requests 2.32.2 -> 2.32.3 +- Python version updated to v3.12.4 + ## [7.2.0] - Core dependencies updated to latest versions: * awscli 1.32.109 -> 1.32.112 diff --git a/Dockerfile b/Dockerfile index 16ed216..17c831c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.12.3-slim AS production +FROM python:3.12.4-slim AS production LABEL maintainer = support@civisanalytics.com @@ -25,8 +25,7 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get update -y --no-install-recommends && COPY requirements-full.txt . -RUN pip install -r requirements-full.txt && \ - pip cache purge && \ +RUN pip install --progress-bar off --no-cache-dir -r requirements-full.txt && \ rm requirements-full.txt # Instruct joblib to use disk for temporary files. Joblib defaults to @@ -35,9 +34,9 @@ RUN pip install -r requirements-full.txt && \ # https://github.com/joblib/joblib/blob/0.11/joblib/parallel.py#L328L342 ENV JOBLIB_TEMP_FOLDER=/tmp -ENV VERSION=7.2.0 \ +ENV VERSION=7.3.0 \ VERSION_MAJOR=7 \ - VERSION_MINOR=2 \ + VERSION_MINOR=3 \ VERSION_MICRO=0 FROM production AS test diff --git a/requirements-core.txt b/requirements-core.txt index 7a24a2e..2552408 100644 --- a/requirements-core.txt +++ b/requirements-core.txt @@ -1,8 +1,8 @@ -awscli==1.32.112 -boto3==1.34.112 -civis==2.1.0 -numpy==1.26.4 +awscli==1.33.9 +boto3==1.34.127 +civis==2.3.0 +numpy==2.0.0 pandas==2.2.2 -requests==2.32.2 +requests==2.32.3 scikit-learn==1.5.0 scipy==1.13.1 diff --git a/requirements-full.txt b/requirements-full.txt index b4effcb..fd068ad 100644 --- a/requirements-full.txt +++ b/requirements-full.txt @@ -8,20 +8,20 @@ attrs==23.2.0 # via # jsonschema # referencing -awscli==1.32.112 +awscli==1.33.9 # via -r requirements-core.txt -boto3==1.34.112 +boto3==1.34.127 # via -r requirements-core.txt -botocore==1.34.112 +botocore==1.34.127 # via # awscli # boto3 # s3transfer -certifi==2024.2.2 +certifi==2024.6.2 # via requests charset-normalizer==3.3.2 # via requests -civis==2.1.0 +civis==2.3.0 # via -r requirements-core.txt click==8.1.7 # via civis @@ -37,7 +37,7 @@ jmespath==1.0.1 # via # boto3 # botocore -joblib==1.2.0 +joblib==1.4.2 # via # civis # scikit-learn @@ -47,7 +47,7 @@ jsonschema==4.22.0 # via civis jsonschema-specifications==2023.12.1 # via jsonschema -numpy==1.26.4 +numpy==2.0.0 # via # -r requirements-core.txt # pandas @@ -71,7 +71,7 @@ referencing==0.35.1 # via # jsonschema # jsonschema-specifications -requests==2.32.2 +requests==2.32.3 # via # -r requirements-core.txt # civis @@ -93,13 +93,13 @@ scipy==1.13.1 # scikit-learn six==1.16.0 # via python-dateutil -tenacity==8.3.0 +tenacity==8.4.1 # via civis threadpoolctl==3.5.0 # via scikit-learn tzdata==2024.1 # via pandas -urllib3==2.2.1 +urllib3==2.2.2 # via # botocore # requests From c6a15dde3941200df83da75ba467b9f6467c3dc9 Mon Sep 17 00:00:00 2001 From: Michael Heilman Date: Mon, 26 Aug 2024 13:46:45 -0500 Subject: [PATCH 59/63] [CIVIS-9315] Fix apt-get and update AWS CLI (#97) This fixes `apt-get` by updating the base image. Also: * Updates the AWS CLI to v2. * Makes `generate-requirements-full.sh` run in a docker container to ensure the environment matches. * Updates the code build config to use a FIPS URL (see, e.g., https://github.com/civisanalytics/datascience-r/pull/52) and to run builds for branches. * Updates the major version to 8 since the AWS CLI major version changed. * Addresses https://github.com/civisanalytics/datascience-python/security/dependabot/10 --- .circleci/config.yml | 2 +- CHANGELOG.md | 6 ++++++ Dockerfile | 39 ++++++++++++++++++++++++++++------- buildspec/merge_master.yaml | 6 +++--- buildspec/push.yaml | 19 +++++++++++++++++ buildspec/release.yaml | 6 +++--- docker-compose.yml | 11 ++++++++++ generate-requirements-full.sh | 7 ++++--- requirements-core.txt | 1 - requirements-full.txt | 37 ++++++++++----------------------- 10 files changed, 89 insertions(+), 45 deletions(-) create mode 100644 buildspec/push.yaml create mode 100644 docker-compose.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index e6c439a..8b16708 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,7 +7,7 @@ jobs: - checkout - setup_remote_docker - run: - name: Build container + name: Build image command: docker build --target test -t ds-python . - run: name: Verify build completed diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ee9065..fd35c1e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,12 @@ Version number changes (major.minor.micro) in this package denote the following: ## Unreleased +## [8.0.0] +- Core dependencies updated to latest versions: + * awscli 1.33.9 -> 2.17.37 +- Python version updated to v3.12.5 +- Fixes apt-get for debian package installations + ## [7.3.0] - Core dependencies updated to latest versions: * awscli 1.32.112 -> 1.33.9 diff --git a/Dockerfile b/Dockerfile index 17c831c..be39bc5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,13 @@ -FROM python:3.12.4-slim AS production +ARG PLATFORM=linux/x86_64 +ARG BASE_IMAGE=python:3.12.5-slim -LABEL maintainer = support@civisanalytics.com +# This is the primary build target used for the production image +FROM --platform=$PLATFORM $BASE_IMAGE AS production + +# Disable pip warnings https://stackoverflow.com/a/72551258 +ENV PIP_ROOT_USER_ACTION=ignore + +LABEL maintainer=support@civisanalytics.com RUN DEBIAN_FRONTEND=noninteractive apt-get update -y --no-install-recommends && \ apt-get install -y --no-install-recommends locales && \ @@ -19,7 +26,9 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get update -y --no-install-recommends && libxrender1 \ wget \ ca-certificates \ - curl && \ + curl \ + mandoc \ + unzip && \ apt-get clean -y && \ rm -rf /var/lib/apt/lists/* @@ -34,14 +43,28 @@ RUN pip install --progress-bar off --no-cache-dir -r requirements-full.txt && \ # https://github.com/joblib/joblib/blob/0.11/joblib/parallel.py#L328L342 ENV JOBLIB_TEMP_FOLDER=/tmp -ENV VERSION=7.3.0 \ - VERSION_MAJOR=7 \ - VERSION_MINOR=3 \ +ENV VERSION=8.0.0 \ + VERSION_MAJOR=8 \ + VERSION_MINOR=0 \ VERSION_MICRO=0 -FROM production AS test +# Install the AWSCLI for moving match targets in the QC workflow. +# See https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html#cliv2-linux-install +RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \ + unzip awscliv2.zip && \ + ./aws/install && \ + rm -rf aws awscliv2.zip + +# This build target is for testing in Circle CI. +FROM --platform=$PLATFORM production AS test COPY .circleci/test_image.py . COPY CHANGELOG.md . -# Defaults to production as the final stage +# This build target is for updating dependencies. +# See generate-requirements.full.sh. +FROM --platform=$PLATFORM $BASE_IMAGE AS pip-tools +RUN pip install -U --no-cache-dir pip pip-tools --progress-bar off +CMD ["/bin/bash"] + +# Default to the production build target. FROM production diff --git a/buildspec/merge_master.yaml b/buildspec/merge_master.yaml index 7cfb899..27fcd9d 100644 --- a/buildspec/merge_master.yaml +++ b/buildspec/merge_master.yaml @@ -3,12 +3,12 @@ phases: pre_build: commands: - echo Logging in to Amazon ECR... - - aws ecr get-login-password --region ${AWS_DEFAULT_REGION} | docker login --username AWS --password-stdin ${REPOSITORY_URI} + - aws ecr get-login-password --region ${AWS_DEFAULT_REGION} | docker login --username AWS --password-stdin ${FIPS_REPOSITORY_URI} build: commands: - echo Building the Docker image... - - docker build -t ${REPOSITORY_URI}:latest . - - docker image push --all-tags ${REPOSITORY_URI} + - docker build -t ${FIPS_REPOSITORY_URI}:latest . + - docker image push --all-tags ${FIPS_REPOSITORY_URI} post_build: commands: - echo Build completed! diff --git a/buildspec/push.yaml b/buildspec/push.yaml new file mode 100644 index 0000000..671bfa1 --- /dev/null +++ b/buildspec/push.yaml @@ -0,0 +1,19 @@ +version: 0.2 +phases: + build: + commands: + - echo Logging in to Amazon ECR... + - aws ecr get-login-password --region ${AWS_DEFAULT_REGION} | docker login --username AWS --password-stdin ${FIPS_REPOSITORY_URI} + - export COMMIT_HASH_SHORT="$(echo $COMMIT_HASH | cut -c 1-7)" + - echo Building the Docker image... + - echo $FIPS_REPOSITORY_URI + - echo $COMMIT_HASH_SHORT + - echo $BRANCH_NAME + - docker build --tag ${FIPS_REPOSITORY_URI}:${COMMIT_HASH_SHORT} --tag ${FIPS_REPOSITORY_URI}:${BRANCH_NAME} . + # This config tests the codebuild login and the build but does not push dev images. + # The following lines can be temporarily uncommented to test a dev image. + # - docker image push --all-tags ${FIPS_REPOSITORY_URI} + post_build: + commands: + - echo Build completed! + - printf '{"tag":"%s"}' $COMMIT_HASH_SHORT > build.json diff --git a/buildspec/release.yaml b/buildspec/release.yaml index d31ecd2..add6384 100644 --- a/buildspec/release.yaml +++ b/buildspec/release.yaml @@ -4,13 +4,13 @@ phases: build: commands: - echo Logging in to Amazon ECR... - - aws ecr get-login-password --region ${AWS_DEFAULT_REGION} | docker login --username AWS --password-stdin ${REPOSITORY_URI} + - aws ecr get-login-password --region ${AWS_DEFAULT_REGION} | docker login --username AWS --password-stdin ${FIPS_REPOSITORY_URI} - echo Building the Docker image... - PATCH_TAG=${TAG_NAME#"v"} # major.minor.patch - MINOR_TAG=${PATCH_TAG%.*} # major.minor - MAJOR_TAG=${MINOR_TAG%.*} # major - - docker build -t ${REPOSITORY_URI}:${PATCH_TAG} -t ${REPOSITORY_URI}:${MINOR_TAG} -t ${REPOSITORY_URI}:${MAJOR_TAG} . - - docker image push --all-tags ${REPOSITORY_URI} + - docker build -t ${FIPS_REPOSITORY_URI}:${PATCH_TAG} -t ${FIPS_REPOSITORY_URI}:${MINOR_TAG} -t ${FIPS_REPOSITORY_URI}:${MAJOR_TAG} . + - docker image push --all-tags ${FIPS_REPOSITORY_URI} post_build: commands: - echo Build completed! diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..54e6f0a --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,11 @@ +services: + pip-tools: + build: + context: . + dockerfile: ./Dockerfile + target: pip-tools + volumes: + - .:/app + stdin_open: true + tty: true + working_dir: /app \ No newline at end of file diff --git a/generate-requirements-full.sh b/generate-requirements-full.sh index 84753d5..b77b33e 100755 --- a/generate-requirements-full.sh +++ b/generate-requirements-full.sh @@ -1,4 +1,5 @@ #!/bin/bash -python --version -pip install --upgrade pip-tools -pip-compile --output-file=requirements-full.txt --pip-args='--prefer-binary' --strip-extras --upgrade requirements-core.txt +# Run this script to update requirements-core.txt. +# It uses Docker to ensure that the environment matches what will be used in the production image. +set -e +docker compose run --rm pip-tools /bin/sh -c "pip install --upgrade pip-tools && pip-compile --output-file=requirements-full.txt --pip-args='--prefer-binary' --strip-extras --upgrade requirements-core.txt" diff --git a/requirements-core.txt b/requirements-core.txt index 2552408..2dad232 100644 --- a/requirements-core.txt +++ b/requirements-core.txt @@ -1,4 +1,3 @@ -awscli==1.33.9 boto3==1.34.127 civis==2.3.0 numpy==2.0.0 diff --git a/requirements-full.txt b/requirements-full.txt index fd068ad..d60c45e 100644 --- a/requirements-full.txt +++ b/requirements-full.txt @@ -4,20 +4,17 @@ # # pip-compile --output-file=requirements-full.txt --pip-args='--prefer-binary' --strip-extras requirements-core.txt # -attrs==23.2.0 +attrs==24.2.0 # via # jsonschema # referencing -awscli==1.33.9 - # via -r requirements-core.txt boto3==1.34.127 # via -r requirements-core.txt -botocore==1.34.127 +botocore==1.34.162 # via - # awscli # boto3 # s3transfer -certifi==2024.6.2 +certifi==2024.7.4 # via requests charset-normalizer==3.3.2 # via requests @@ -27,11 +24,7 @@ click==8.1.7 # via civis cloudpickle==3.0.0 # via civis -colorama==0.4.6 - # via awscli -docutils==0.16 - # via awscli -idna==3.7 +idna==3.8 # via requests jmespath==1.0.1 # via @@ -43,7 +36,7 @@ joblib==1.4.2 # scikit-learn jsonref==1.1.0 # via civis -jsonschema==4.22.0 +jsonschema==4.23.0 # via civis jsonschema-specifications==2023.12.1 # via jsonschema @@ -55,18 +48,14 @@ numpy==2.0.0 # scipy pandas==2.2.2 # via -r requirements-core.txt -pyasn1==0.6.0 - # via rsa python-dateutil==2.9.0.post0 # via # botocore # pandas pytz==2024.1 # via pandas -pyyaml==6.0.1 - # via - # awscli - # civis +pyyaml==6.0.2 + # via civis referencing==0.35.1 # via # jsonschema @@ -75,16 +64,12 @@ requests==2.32.3 # via # -r requirements-core.txt # civis -rpds-py==0.18.1 +rpds-py==0.20.0 # via # jsonschema # referencing -rsa==4.7.2 - # via awscli -s3transfer==0.10.1 - # via - # awscli - # boto3 +s3transfer==0.10.2 + # via boto3 scikit-learn==1.5.0 # via -r requirements-core.txt scipy==1.13.1 @@ -93,7 +78,7 @@ scipy==1.13.1 # scikit-learn six==1.16.0 # via python-dateutil -tenacity==8.4.1 +tenacity==9.0.0 # via civis threadpoolctl==3.5.0 # via scikit-learn From 739d8b64f34945163ecb0e643893872b4eb5c331 Mon Sep 17 00:00:00 2001 From: Michael Heilman Date: Wed, 25 Sep 2024 13:11:13 -0500 Subject: [PATCH 60/63] [CIVIS-9315] Update Python to 3.12.6, fix apt-get (#98) --- CHANGELOG.md | 4 ++++ Dockerfile | 6 +++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fd35c1e..91766d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ Version number changes (major.minor.micro) in this package denote the following: ## Unreleased +## [8.0.1] + +- Python version updated to v3.12.6 + ## [8.0.0] - Core dependencies updated to latest versions: * awscli 1.33.9 -> 2.17.37 diff --git a/Dockerfile b/Dockerfile index be39bc5..54deebe 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ ARG PLATFORM=linux/x86_64 -ARG BASE_IMAGE=python:3.12.5-slim +ARG BASE_IMAGE=python:3.12.6-slim # This is the primary build target used for the production image FROM --platform=$PLATFORM $BASE_IMAGE AS production @@ -43,10 +43,10 @@ RUN pip install --progress-bar off --no-cache-dir -r requirements-full.txt && \ # https://github.com/joblib/joblib/blob/0.11/joblib/parallel.py#L328L342 ENV JOBLIB_TEMP_FOLDER=/tmp -ENV VERSION=8.0.0 \ +ENV VERSION=8.0.1 \ VERSION_MAJOR=8 \ VERSION_MINOR=0 \ - VERSION_MICRO=0 + VERSION_MICRO=1 # Install the AWSCLI for moving match targets in the QC workflow. # See https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html#cliv2-linux-install From 15253e3c683e16b5580c5af49d8ff46e4741abff Mon Sep 17 00:00:00 2001 From: Jackson Lee <86482098+jacksonlee-civis@users.noreply.github.com> Date: Mon, 18 Nov 2024 12:16:00 -0500 Subject: [PATCH 61/63] [CIVIS-9806] ENH update civis-python to v2.4.0; add uv (#99) * ENH update base image to python 3.12.7 * ENH update core dependency versions * FIX include awscli v2 in requirements files * TST expected shell commands should be available * ENH add uv to image * MAINT update changelog * FIX version env vars * MAINT update readme * MAINT always build images for dev branches * TST add an apt-get test * FIX set env var for uv to use system python * STY typos * TST uv --- .circleci/test_image.py | 29 ++++++++++++++++ CHANGELOG.md | 13 ++++++++ Dockerfile | 23 ++++++------- README.md | 6 ++-- buildspec/push.yaml | 6 ++-- docker-compose.yml | 2 +- generate-requirements-full.sh | 2 +- requirements-core.txt | 16 +++++---- requirements-full.txt | 63 +++++++++++++++++++++++++---------- 9 files changed, 116 insertions(+), 44 deletions(-) diff --git a/.circleci/test_image.py b/.circleci/test_image.py index 99b6239..04e2df1 100755 --- a/.circleci/test_image.py +++ b/.circleci/test_image.py @@ -2,9 +2,12 @@ import os import re +import shutil +import subprocess import unittest +# Just use the stdlib's `unittest` rather than needing to install `pytest`. class TestImage(unittest.TestCase): def test_version(self): @@ -42,6 +45,32 @@ def test_civis_can_import(self): import civis.ml # noqa: F401 import civis.utils # noqa: F401 + def test_shell_commands_available(self): + """Ensure the main shell commands are available.""" + # A non-exhaustive list of commands -- we just test those we'd likely use. + expected_cmds = "aws civis curl git pip python unzip uv wget".split() + for cmd in expected_cmds: + self.assertIsNotNone(shutil.which(cmd), f"{cmd} not found in PATH") + + def _test_shell_command(self, cmd: str): + """Check if the shell command runs successfully in the image.""" + try: + subprocess.check_call(cmd, shell=True) + except subprocess.CalledProcessError as e: + self.fail( + f"apt-get test failed with return code {e.returncode}\n" + f"stdout: {e.stdout}\n" + f"stderr: {e.stderr}" + ) + + def test_apt_get(self): + """Ensure that apt-get works in the image.""" + self._test_shell_command("apt-get update -y && apt-get install -y htop") + + def test_uv(self): + """Ensure that uv works in the image.""" + self._test_shell_command("uv pip install python-iso639") + if __name__ == "__main__": unittest.main() diff --git a/CHANGELOG.md b/CHANGELOG.md index 91766d8..f792bec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,19 @@ Version number changes (major.minor.micro) in this package denote the following: ## Unreleased +## [8.1.0] + +- Python version updated to v3.12.7 +- Core dependencies updated to latest versions: + * awscli 2.17.37 -> 2.19.5 + * boto3 1.34.127 -> 1.35.58 + * civis 2.3.0 -> 2.4.0 + * numpy 2.0.0 -> 2.1.3 + * pandas 2.2.2 -> 2.2.3 + * scikit-learn 1.5.0 -> 1.5.2 + * scipy 1.13.1 -> 1.14.1 +- uv added to the image + ## [8.0.1] - Python version updated to v3.12.6 diff --git a/Dockerfile b/Dockerfile index 54deebe..040ee80 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ ARG PLATFORM=linux/x86_64 -ARG BASE_IMAGE=python:3.12.6-slim +ARG BASE_IMAGE=python:3.12.7-slim # This is the primary build target used for the production image FROM --platform=$PLATFORM $BASE_IMAGE AS production @@ -37,25 +37,24 @@ COPY requirements-full.txt . RUN pip install --progress-bar off --no-cache-dir -r requirements-full.txt && \ rm requirements-full.txt +# Install uv. +ADD https://astral.sh/uv/0.5.1/install.sh /uv-installer.sh +RUN sh /uv-installer.sh && rm /uv-installer.sh +ENV PATH="/root/.local/bin/:$PATH" \ + UV_SYSTEM_PYTHON=1 + # Instruct joblib to use disk for temporary files. Joblib defaults to # /shm when that directory is present. In the Docker container, /shm is # present but defaults to 64 MB. # https://github.com/joblib/joblib/blob/0.11/joblib/parallel.py#L328L342 ENV JOBLIB_TEMP_FOLDER=/tmp -ENV VERSION=8.0.1 \ +ENV VERSION=8.1.0 \ VERSION_MAJOR=8 \ - VERSION_MINOR=0 \ - VERSION_MICRO=1 - -# Install the AWSCLI for moving match targets in the QC workflow. -# See https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html#cliv2-linux-install -RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \ - unzip awscliv2.zip && \ - ./aws/install && \ - rm -rf aws awscliv2.zip + VERSION_MINOR=1 \ + VERSION_MICRO=0 -# This build target is for testing in Circle CI. +# This build target is for testing in CircleCI. FROM --platform=$PLATFORM production AS test COPY .circleci/test_image.py . COPY CHANGELOG.md . diff --git a/README.md b/README.md index ae170de..ea97734 100644 --- a/README.md +++ b/README.md @@ -66,8 +66,8 @@ default location for staging temporary files to the /tmp directory. The normal default is /shm. /shm is a RAM disk which defaults to a 64 MB size in Docker containers, too small for typical scientific computing. -# Updating existing PyPi Package Version -1. Update version of existing package in `requirements-core.txt` +# Updating Existing Package Versions +1. Update versions of existing packages in `requirements-core.txt` 2. Run script `generate-requirements-full.sh` # Creating Equivalent Local Environments @@ -88,7 +88,7 @@ and describe any changes in the [change log](CHANGELOG.md). ## For Maintainers This repo has autobuild enabled. Any PR that is merged to master will -be built as the `latest` tag on Dockerhub. +be built as the `latest` tag on DockerHub. Once you are ready to create a new version, go to the "releases" tab of the repository and click "Draft a new release". GitHub will prompt you to create a new tag, release title, and release description. The tag should use semantic versioning in the form "vX.X.X"; "major.minor.micro". diff --git a/buildspec/push.yaml b/buildspec/push.yaml index 671bfa1..0f10bc4 100644 --- a/buildspec/push.yaml +++ b/buildspec/push.yaml @@ -10,9 +10,9 @@ phases: - echo $COMMIT_HASH_SHORT - echo $BRANCH_NAME - docker build --tag ${FIPS_REPOSITORY_URI}:${COMMIT_HASH_SHORT} --tag ${FIPS_REPOSITORY_URI}:${BRANCH_NAME} . - # This config tests the codebuild login and the build but does not push dev images. - # The following lines can be temporarily uncommented to test a dev image. - # - docker image push --all-tags ${FIPS_REPOSITORY_URI} + # We have a life cycle policy in place to expire and delete images from dev branches, + # so there are no issues with pushing as many of these images as there may be. + - docker image push --all-tags ${FIPS_REPOSITORY_URI} post_build: commands: - echo Build completed! diff --git a/docker-compose.yml b/docker-compose.yml index 54e6f0a..dedb24b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,4 +8,4 @@ services: - .:/app stdin_open: true tty: true - working_dir: /app \ No newline at end of file + working_dir: /app diff --git a/generate-requirements-full.sh b/generate-requirements-full.sh index b77b33e..460b358 100755 --- a/generate-requirements-full.sh +++ b/generate-requirements-full.sh @@ -2,4 +2,4 @@ # Run this script to update requirements-core.txt. # It uses Docker to ensure that the environment matches what will be used in the production image. set -e -docker compose run --rm pip-tools /bin/sh -c "pip install --upgrade pip-tools && pip-compile --output-file=requirements-full.txt --pip-args='--prefer-binary' --strip-extras --upgrade requirements-core.txt" +docker compose run --rm pip-tools /bin/sh -c "apt-get update -y && apt-get install -y git && pip install --upgrade pip-tools && pip-compile --output-file=requirements-full.txt --pip-args='--prefer-binary' --strip-extras --upgrade requirements-core.txt" diff --git a/requirements-core.txt b/requirements-core.txt index 2dad232..7f4e1eb 100644 --- a/requirements-core.txt +++ b/requirements-core.txt @@ -1,7 +1,11 @@ -boto3==1.34.127 -civis==2.3.0 -numpy==2.0.0 -pandas==2.2.2 +# awscli v2 is not officially available on PyPI (https://github.com/aws/aws-cli/issues/4947). +# Specifying awscli in requirements-core.txt here ensures that it (and its transitive dependencies) +# are taken into account when generating the final requirements-full.txt file. +awscli @ git+https://github.com/aws/aws-cli@2.19.5 +boto3==1.35.58 +civis==2.4.0 +numpy==2.1.3 +pandas==2.2.3 requests==2.32.3 -scikit-learn==1.5.0 -scipy==1.13.1 +scikit-learn==1.5.2 +scipy==1.14.1 diff --git a/requirements-full.txt b/requirements-full.txt index d60c45e..0476d0a 100644 --- a/requirements-full.txt +++ b/requirements-full.txt @@ -8,26 +8,41 @@ attrs==24.2.0 # via # jsonschema # referencing -boto3==1.34.127 +awscli @ git+https://github.com/aws/aws-cli@2.19.5 # via -r requirements-core.txt -botocore==1.34.162 +awscrt==0.22.0 + # via awscli +boto3==1.35.58 + # via -r requirements-core.txt +botocore==1.35.58 # via # boto3 # s3transfer -certifi==2024.7.4 +certifi==2024.8.30 # via requests -charset-normalizer==3.3.2 +cffi==1.17.1 + # via cryptography +charset-normalizer==3.4.0 # via requests -civis==2.3.0 +civis==2.4.0 # via -r requirements-core.txt click==8.1.7 # via civis -cloudpickle==3.0.0 +cloudpickle==3.1.0 # via civis -idna==3.8 +colorama==0.4.6 + # via awscli +cryptography==43.0.1 + # via awscli +distro==1.8.0 + # via awscli +docutils==0.19 + # via awscli +idna==3.10 # via requests jmespath==1.0.1 # via + # awscli # boto3 # botocore joblib==1.4.2 @@ -38,21 +53,26 @@ jsonref==1.1.0 # via civis jsonschema==4.23.0 # via civis -jsonschema-specifications==2023.12.1 +jsonschema-specifications==2024.10.1 # via jsonschema -numpy==2.0.0 +numpy==2.1.3 # via # -r requirements-core.txt # pandas # scikit-learn # scipy -pandas==2.2.2 +pandas==2.2.3 # via -r requirements-core.txt -python-dateutil==2.9.0.post0 +prompt-toolkit==3.0.38 + # via awscli +pycparser==2.22 + # via cffi +python-dateutil==2.9.0 # via + # awscli # botocore # pandas -pytz==2024.1 +pytz==2024.2 # via pandas pyyaml==6.0.2 # via civis @@ -64,15 +84,19 @@ requests==2.32.3 # via # -r requirements-core.txt # civis -rpds-py==0.20.0 +rpds-py==0.21.0 # via # jsonschema # referencing -s3transfer==0.10.2 +ruamel-yaml==0.17.21 + # via awscli +ruamel-yaml-clib==0.2.8 + # via awscli +s3transfer==0.10.3 # via boto3 -scikit-learn==1.5.0 +scikit-learn==1.5.2 # via -r requirements-core.txt -scipy==1.13.1 +scipy==1.14.1 # via # -r requirements-core.txt # scikit-learn @@ -82,9 +106,12 @@ tenacity==9.0.0 # via civis threadpoolctl==3.5.0 # via scikit-learn -tzdata==2024.1 +tzdata==2024.2 # via pandas -urllib3==2.2.2 +urllib3==1.26.20 # via + # awscli # botocore # requests +wcwidth==0.2.13 + # via prompt-toolkit From adf1c0390b19adaf0c8093458fb3d69104c84c0d Mon Sep 17 00:00:00 2001 From: Jackson Lee <86482098+jacksonlee-civis@users.noreply.github.com> Date: Mon, 13 Jan 2025 15:12:04 -0500 Subject: [PATCH 62/63] [CIVIS-10059] ENH update civis-python to v2.4.3 (#100) * DEP update dependency versions * MAINT update changelog --- CHANGELOG.md | 12 ++++++++++++ Dockerfile | 8 ++++---- requirements-core.txt | 12 ++++++------ requirements-full.txt | 32 +++++++++++++++++--------------- 4 files changed, 39 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f792bec..79bfd5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,18 @@ Version number changes (major.minor.micro) in this package denote the following: ## Unreleased +## [8.2.0] + +- Python version updated: 3.12.7 -> 3.12.8 +- uv version updated: 0.5.1 -> 0.5.18 +- Core dependencies updated to latest versions: + * awscli 2.19.5 -> 2.22.33 + * boto3 1.35.58 -> 1.35.97 + * civis 2.4.0 -> 2.4.3 + * numpy 2.1.3 -> 2.2.1 + * scikit-learn 1.5.2 -> 1.6.1 + * scipy 1.14.1 -> 1.15.1 + ## [8.1.0] - Python version updated to v3.12.7 diff --git a/Dockerfile b/Dockerfile index 040ee80..f965e54 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ ARG PLATFORM=linux/x86_64 -ARG BASE_IMAGE=python:3.12.7-slim +ARG BASE_IMAGE=python:3.12.8-slim # This is the primary build target used for the production image FROM --platform=$PLATFORM $BASE_IMAGE AS production @@ -38,7 +38,7 @@ RUN pip install --progress-bar off --no-cache-dir -r requirements-full.txt && \ rm requirements-full.txt # Install uv. -ADD https://astral.sh/uv/0.5.1/install.sh /uv-installer.sh +ADD https://astral.sh/uv/0.5.18/install.sh /uv-installer.sh RUN sh /uv-installer.sh && rm /uv-installer.sh ENV PATH="/root/.local/bin/:$PATH" \ UV_SYSTEM_PYTHON=1 @@ -49,9 +49,9 @@ ENV PATH="/root/.local/bin/:$PATH" \ # https://github.com/joblib/joblib/blob/0.11/joblib/parallel.py#L328L342 ENV JOBLIB_TEMP_FOLDER=/tmp -ENV VERSION=8.1.0 \ +ENV VERSION=8.2.0 \ VERSION_MAJOR=8 \ - VERSION_MINOR=1 \ + VERSION_MINOR=2 \ VERSION_MICRO=0 # This build target is for testing in CircleCI. diff --git a/requirements-core.txt b/requirements-core.txt index 7f4e1eb..9334db0 100644 --- a/requirements-core.txt +++ b/requirements-core.txt @@ -1,11 +1,11 @@ # awscli v2 is not officially available on PyPI (https://github.com/aws/aws-cli/issues/4947). # Specifying awscli in requirements-core.txt here ensures that it (and its transitive dependencies) # are taken into account when generating the final requirements-full.txt file. -awscli @ git+https://github.com/aws/aws-cli@2.19.5 -boto3==1.35.58 -civis==2.4.0 -numpy==2.1.3 +awscli @ git+https://github.com/aws/aws-cli@2.22.33 +boto3==1.35.97 +civis==2.4.3 +numpy==2.2.1 pandas==2.2.3 requests==2.32.3 -scikit-learn==1.5.2 -scipy==1.14.1 +scikit-learn==1.6.1 +scipy==1.15.1 diff --git a/requirements-full.txt b/requirements-full.txt index 0476d0a..3dfe5ae 100644 --- a/requirements-full.txt +++ b/requirements-full.txt @@ -4,29 +4,29 @@ # # pip-compile --output-file=requirements-full.txt --pip-args='--prefer-binary' --strip-extras requirements-core.txt # -attrs==24.2.0 +attrs==24.3.0 # via # jsonschema # referencing -awscli @ git+https://github.com/aws/aws-cli@2.19.5 +awscli @ git+https://github.com/aws/aws-cli@2.22.33 # via -r requirements-core.txt -awscrt==0.22.0 +awscrt==0.23.4 # via awscli -boto3==1.35.58 +boto3==1.35.97 # via -r requirements-core.txt -botocore==1.35.58 +botocore==1.35.97 # via # boto3 # s3transfer -certifi==2024.8.30 +certifi==2024.12.14 # via requests cffi==1.17.1 # via cryptography -charset-normalizer==3.4.0 +charset-normalizer==3.4.1 # via requests -civis==2.4.0 +civis==2.4.3 # via -r requirements-core.txt -click==8.1.7 +click==8.1.8 # via civis cloudpickle==3.1.0 # via civis @@ -55,7 +55,7 @@ jsonschema==4.23.0 # via civis jsonschema-specifications==2024.10.1 # via jsonschema -numpy==2.1.3 +numpy==2.2.1 # via # -r requirements-core.txt # pandas @@ -84,7 +84,7 @@ requests==2.32.3 # via # -r requirements-core.txt # civis -rpds-py==0.21.0 +rpds-py==0.22.3 # via # jsonschema # referencing @@ -92,15 +92,15 @@ ruamel-yaml==0.17.21 # via awscli ruamel-yaml-clib==0.2.8 # via awscli -s3transfer==0.10.3 +s3transfer==0.10.4 # via boto3 -scikit-learn==1.5.2 +scikit-learn==1.6.1 # via -r requirements-core.txt -scipy==1.14.1 +scipy==1.15.1 # via # -r requirements-core.txt # scikit-learn -six==1.16.0 +six==1.17.0 # via python-dateutil tenacity==9.0.0 # via civis @@ -115,3 +115,5 @@ urllib3==1.26.20 # requests wcwidth==0.2.13 # via prompt-toolkit +zipp==3.20.2 + # via awscli From cce8b0d4f1136367e0ceb2ba99ade0debb32fca4 Mon Sep 17 00:00:00 2001 From: Jackson Lee <86482098+jacksonlee-civis@users.noreply.github.com> Date: Mon, 7 Jul 2025 14:19:47 -0400 Subject: [PATCH 63/63] [CIVIS-11075] ENH updates for Python 3.13 and civis-python v2.7.1 (#101) * DEP update python core dependencies * DEP update python and uv versions * ENH update core deps * MAINT update changelog * MAINT update changelog --- CHANGELOG.md | 16 ++++++++ Dockerfile | 27 +++++++------- docker-compose.yml | 4 +- generate-requirements-full.sh | 2 +- requirements-core.txt | 17 +++++---- requirements-full.txt | 70 +++++++++++++++-------------------- 6 files changed, 72 insertions(+), 64 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 79bfd5a..c4a22c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,22 @@ Version number changes (major.minor.micro) in this package denote the following: ## Unreleased +## [8.3.0] + +- Python version updated: 3.12.8 -> 3.13.5 +- uv version updated: 0.5.18 -> 0.7.19 +- New core dependencies added: + * polars 1.31.0 (supported by civis-python since v2.6.0) +- Core dependencies updated to latest versions: + * awscli 2.22.33 -> 2.27.48 + * boto3 1.35.97 -> 1.39.2 + * civis 2.4.3 -> 2.7.1 + * numpy 2.2.1 -> 2.3.1 + * pandas 2.2.3 -> 2.3.0 + * requests 2.32.3 -> 2.32.4 + * scikit-learn 1.6.1 -> 1.7.0 + * scipy 1.15.1 -> 1.16.0 + ## [8.2.0] - Python version updated: 3.12.7 -> 3.12.8 diff --git a/Dockerfile b/Dockerfile index f965e54..0e9ab9b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,7 @@ ARG PLATFORM=linux/x86_64 -ARG BASE_IMAGE=python:3.12.8-slim +ARG BASE_IMAGE=python:3.13.5-slim -# This is the primary build target used for the production image -FROM --platform=$PLATFORM $BASE_IMAGE AS production +FROM --platform=$PLATFORM $BASE_IMAGE AS uv-installed # Disable pip warnings https://stackoverflow.com/a/72551258 ENV PIP_ROOT_USER_ACTION=ignore @@ -32,26 +31,29 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get update -y --no-install-recommends && apt-get clean -y && \ rm -rf /var/lib/apt/lists/* -COPY requirements-full.txt . - -RUN pip install --progress-bar off --no-cache-dir -r requirements-full.txt && \ - rm requirements-full.txt - # Install uv. -ADD https://astral.sh/uv/0.5.18/install.sh /uv-installer.sh +ADD https://astral.sh/uv/0.7.19/install.sh /uv-installer.sh RUN sh /uv-installer.sh && rm /uv-installer.sh ENV PATH="/root/.local/bin/:$PATH" \ UV_SYSTEM_PYTHON=1 +# This is the primary build target used for the production image +FROM --platform=$PLATFORM uv-installed AS production + +COPY requirements-full.txt . + +RUN uv pip install --no-progress --no-cache -r requirements-full.txt && \ + rm requirements-full.txt + # Instruct joblib to use disk for temporary files. Joblib defaults to # /shm when that directory is present. In the Docker container, /shm is # present but defaults to 64 MB. # https://github.com/joblib/joblib/blob/0.11/joblib/parallel.py#L328L342 ENV JOBLIB_TEMP_FOLDER=/tmp -ENV VERSION=8.2.0 \ +ENV VERSION=8.3.0 \ VERSION_MAJOR=8 \ - VERSION_MINOR=2 \ + VERSION_MINOR=3 \ VERSION_MICRO=0 # This build target is for testing in CircleCI. @@ -61,8 +63,7 @@ COPY CHANGELOG.md . # This build target is for updating dependencies. # See generate-requirements.full.sh. -FROM --platform=$PLATFORM $BASE_IMAGE AS pip-tools -RUN pip install -U --no-cache-dir pip pip-tools --progress-bar off +FROM --platform=$PLATFORM uv-installed AS update-deps CMD ["/bin/bash"] # Default to the production build target. diff --git a/docker-compose.yml b/docker-compose.yml index dedb24b..3742956 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,9 +1,9 @@ services: - pip-tools: + update-deps: build: context: . dockerfile: ./Dockerfile - target: pip-tools + target: update-deps volumes: - .:/app stdin_open: true diff --git a/generate-requirements-full.sh b/generate-requirements-full.sh index 460b358..40c7889 100755 --- a/generate-requirements-full.sh +++ b/generate-requirements-full.sh @@ -2,4 +2,4 @@ # Run this script to update requirements-core.txt. # It uses Docker to ensure that the environment matches what will be used in the production image. set -e -docker compose run --rm pip-tools /bin/sh -c "apt-get update -y && apt-get install -y git && pip install --upgrade pip-tools && pip-compile --output-file=requirements-full.txt --pip-args='--prefer-binary' --strip-extras --upgrade requirements-core.txt" +docker compose run --rm update-deps /bin/sh -c "uv pip compile --output-file=requirements-full.txt --upgrade requirements-core.txt" diff --git a/requirements-core.txt b/requirements-core.txt index 9334db0..fb90216 100644 --- a/requirements-core.txt +++ b/requirements-core.txt @@ -1,11 +1,12 @@ # awscli v2 is not officially available on PyPI (https://github.com/aws/aws-cli/issues/4947). # Specifying awscli in requirements-core.txt here ensures that it (and its transitive dependencies) # are taken into account when generating the final requirements-full.txt file. -awscli @ git+https://github.com/aws/aws-cli@2.22.33 -boto3==1.35.97 -civis==2.4.3 -numpy==2.2.1 -pandas==2.2.3 -requests==2.32.3 -scikit-learn==1.6.1 -scipy==1.15.1 +awscli @ git+https://github.com/aws/aws-cli@2.27.48 +boto3==1.39.2 +civis==2.7.1 +numpy==2.3.1 +pandas==2.3.0 +polars==1.31.0 +requests==2.32.4 +scikit-learn==1.7.0 +scipy==1.16.0 diff --git a/requirements-full.txt b/requirements-full.txt index 3dfe5ae..1fd81f5 100644 --- a/requirements-full.txt +++ b/requirements-full.txt @@ -1,39 +1,31 @@ -# -# This file is autogenerated by pip-compile with Python 3.12 -# by the following command: -# -# pip-compile --output-file=requirements-full.txt --pip-args='--prefer-binary' --strip-extras requirements-core.txt -# -attrs==24.3.0 +# This file was autogenerated by uv via the following command: +# uv pip compile --output-file=requirements-full.txt requirements-core.txt +attrs==25.3.0 # via # jsonschema # referencing -awscli @ git+https://github.com/aws/aws-cli@2.22.33 +awscli @ git+https://github.com/aws/aws-cli@6804a17061546394f88e1fe6d1bf9b24cd8a09ec # via -r requirements-core.txt -awscrt==0.23.4 +awscrt==0.26.1 # via awscli -boto3==1.35.97 +boto3==1.39.2 # via -r requirements-core.txt -botocore==1.35.97 +botocore==1.39.2 # via # boto3 # s3transfer -certifi==2024.12.14 +certifi==2025.6.15 # via requests -cffi==1.17.1 - # via cryptography -charset-normalizer==3.4.1 +charset-normalizer==3.4.2 # via requests -civis==2.4.3 +civis==2.7.1 # via -r requirements-core.txt -click==8.1.8 +click==8.2.1 # via civis -cloudpickle==3.1.0 +cloudpickle==3.1.1 # via civis colorama==0.4.6 # via awscli -cryptography==43.0.1 - # via awscli distro==1.8.0 # via awscli docutils==0.19 @@ -45,68 +37,68 @@ jmespath==1.0.1 # awscli # boto3 # botocore -joblib==1.4.2 +joblib==1.5.1 # via # civis # scikit-learn jsonref==1.1.0 # via civis -jsonschema==4.23.0 +jsonschema==4.24.0 # via civis -jsonschema-specifications==2024.10.1 +jsonschema-specifications==2025.4.1 # via jsonschema -numpy==2.2.1 +numpy==2.3.1 # via # -r requirements-core.txt # pandas # scikit-learn # scipy -pandas==2.2.3 +pandas==2.3.0 + # via -r requirements-core.txt +polars==1.31.0 # via -r requirements-core.txt prompt-toolkit==3.0.38 # via awscli -pycparser==2.22 - # via cffi python-dateutil==2.9.0 # via # awscli # botocore # pandas -pytz==2024.2 +pytz==2025.2 # via pandas pyyaml==6.0.2 # via civis -referencing==0.35.1 +referencing==0.36.2 # via # jsonschema # jsonschema-specifications -requests==2.32.3 +requests==2.32.4 # via # -r requirements-core.txt # civis -rpds-py==0.22.3 +rpds-py==0.26.0 # via # jsonschema # referencing ruamel-yaml==0.17.21 # via awscli -ruamel-yaml-clib==0.2.8 +ruamel-yaml-clib==0.2.12 # via awscli -s3transfer==0.10.4 +s3transfer==0.13.0 # via boto3 -scikit-learn==1.6.1 +scikit-learn==1.7.0 # via -r requirements-core.txt -scipy==1.15.1 +scipy==1.16.0 # via # -r requirements-core.txt # scikit-learn six==1.17.0 # via python-dateutil -tenacity==9.0.0 +tenacity==9.1.2 # via civis -threadpoolctl==3.5.0 +threadpoolctl==3.6.0 # via scikit-learn -tzdata==2024.2 +tzdata==2025.2 # via pandas urllib3==1.26.20 # via @@ -115,5 +107,3 @@ urllib3==1.26.20 # requests wcwidth==0.2.13 # via prompt-toolkit -zipp==3.20.2 - # via awscli