diff --git a/.gitreview b/.gitreview index e1bf63ba7a..1637a4a66f 100644 --- a/.gitreview +++ b/.gitreview @@ -2,3 +2,4 @@ host=review.opendev.org port=29418 project=openstack/devstack.git +defaultbranch=stable/train diff --git a/.zuul.yaml b/.zuul.yaml index f7594d4ddc..4f15656c6d 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -48,16 +48,6 @@ nodes: - controller -- nodeset: - name: devstack-single-node-fedora-latest - nodes: - - name: controller - label: fedora-29 - groups: - - name: tempest - nodes: - - controller - - nodeset: name: openstack-two-node nodes: @@ -203,6 +193,7 @@ - zuul: opendev.org/openstack/devstack-gate - zuul: opendev.org/openstack/openstack-zuul-jobs vars: + configure_swap_size: 8192 devstack_localrc: DATABASE_PASSWORD: secretdatabase RABBIT_PASSWORD: secretrabbit @@ -534,18 +525,15 @@ nodeset: devstack-single-node-opensuse-15 voting: false -- job: - name: devstack-platform-fedora-latest - parent: tempest-full-py3 - description: Fedora latest platform test - nodeset: devstack-single-node-fedora-latest - voting: false - - job: name: devstack-platform-xenial parent: tempest-full-py3 description: Ubuntu Xenial platform test nodeset: openstack-single-node-xenial + vars: + devstack_localrc: + # use py3.5 compatible Tempest here. + TEMPEST_BRANCH: "23.0.0" voting: false - job: @@ -604,6 +592,7 @@ - job: name: devstack-unit-tests + nodeset: ubuntu-bionic description: | Runs unit tests on devstack project. @@ -623,7 +612,6 @@ - devstack-ipv6 - devstack-platform-centos-7 - devstack-platform-opensuse-15 - - devstack-platform-fedora-latest - devstack-platform-xenial - devstack-multinode - devstack-multinode-xenial @@ -640,7 +628,13 @@ irrelevant-files: - ^.*\.rst$ - ^doc/.*$ + - grenade: + voting: false + irrelevant-files: + - ^.*\.rst$ + - ^doc/.*$ - neutron-grenade-multinode: + voting: false irrelevant-files: - ^.*\.rst$ - ^doc/.*$ @@ -657,6 +651,7 @@ irrelevant-files: - ^.*\.rst$ - ^doc/.*$ + voting: false - tempest-ipv6-only: irrelevant-files: - ^.*\.rst$ @@ -670,10 +665,10 @@ - devstack-multinode-xenial - devstack-unit-tests - openstack-tox-bashate - - neutron-grenade-multinode: - irrelevant-files: - - ^.*\.rst$ - - ^doc/.*$ + # - neutron-grenade-multinode: + # irrelevant-files: + # - ^.*\.rst$ + # - ^doc/.*$ - neutron-tempest-linuxbridge: irrelevant-files: - ^.*\.rst$ @@ -682,10 +677,14 @@ irrelevant-files: - ^.*\.rst$ - ^doc/.*$ - - openstacksdk-functional-devstack: - irrelevant-files: - - ^.*\.rst$ - - ^doc/.*$ + # - grenade: + # irrelevant-files: + # - ^.*\.rst$ + # - ^doc/.*$ + # - openstacksdk-functional-devstack: + # irrelevant-files: + # - ^.*\.rst$ + # - ^doc/.*$ - tempest-ipv6-only: irrelevant-files: - ^.*\.rst$ @@ -727,15 +726,11 @@ irrelevant-files: - ^.*\.rst$ - ^doc/.*$ - - legacy-tempest-dsvm-neutron-dvr-multinode-full: - irrelevant-files: - - ^.*\.rst$ - - ^doc/.*$ - neutron-tempest-dvr-ha-multinode-full: irrelevant-files: - ^.*\.rst$ - ^doc/.*$ - - legacy-tempest-dsvm-lvm-multibackend: + - cinder-tempest-lvm-multibackend: irrelevant-files: - ^.*\.rst$ - ^doc/.*$ diff --git a/doc/source/guides.rst b/doc/source/guides.rst index 82e0dd6ac6..e7ec629962 100644 --- a/doc/source/guides.rst +++ b/doc/source/guides.rst @@ -10,6 +10,7 @@ Walk through various setups used by stackers .. toctree:: :glob: + :hidden: :maxdepth: 1 guides/single-vm @@ -68,6 +69,11 @@ Nova and devstack Guide to working with nova features :doc:`Nova and devstack `. +Configure Load-Balancer Version 2 +----------------------------------- + +Guide on :doc:`Configure Load-Balancer Version 2 `. + Deploying DevStack with LDAP ---------------------------- diff --git a/files/debs/neutron-common b/files/debs/neutron-common index e30f678f7a..b269f6330b 100644 --- a/files/debs/neutron-common +++ b/files/debs/neutron-common @@ -1,6 +1,6 @@ acl dnsmasq-base -dnsmasq-utils # for dhcp_release only available in dist:precise +dnsmasq-utils # for dhcp_release ebtables haproxy # to serve as metadata proxy inside router/dhcp namespaces iptables diff --git a/functions-common b/functions-common index a13d611415..dc448d306f 100644 --- a/functions-common +++ b/functions-common @@ -451,6 +451,7 @@ function is_fedora { [ "$os_VENDOR" = "Fedora" ] || [ "$os_VENDOR" = "Red Hat" ] || \ [ "$os_VENDOR" = "RedHatEnterpriseServer" ] || \ + [ "$os_VENDOR" = "RedHatEnterprise" ] || \ [ "$os_VENDOR" = "CentOS" ] || [ "$os_VENDOR" = "OracleServer" ] || \ [ "$os_VENDOR" = "Virtuozzo" ] } diff --git a/inc/python b/inc/python index ea8ff67e08..ed25fab23a 100644 --- a/inc/python +++ b/inc/python @@ -247,7 +247,13 @@ function pip_install { # and we will let pip sort out the install, regardless of # the package being local or remote. echo "Using $PYTHON3_VERSION version to install $package_dir based on default behavior" - sudo_pip="$sudo_pip LC_ALL=en_US.UTF-8" + # See + # https://github.com/pypa/setuptools/issues/2232 + # http://lists.openstack.org/pipermail/openstack-discuss/2020-August/016905.html + # this makes setuptools >=50 use the platform distutils. + # We only want to do this on global pip installs, not if + # installing in a virtualenv + sudo_pip="$sudo_pip LC_ALL=en_US.UTF-8 SETUPTOOLS_USE_DISTUTILS=stdlib " cmd_pip=$(get_pip_command $PYTHON3_VERSION) fi fi @@ -270,13 +276,6 @@ function pip_install { $xtrace - # Also install test requirements - local install_test_reqs="" - local test_req="${package_dir}/test-requirements.txt" - if [[ -e "$test_req" ]]; then - install_test_reqs="-r $test_req" - fi - # adding SETUPTOOLS_SYS_PATH_TECHNIQUE is a workaround to keep # the same behaviour of setuptools before version 25.0.0. # related issue: https://github.com/pypa/pip/issues/3874 @@ -286,7 +285,7 @@ function pip_install { no_proxy="${no_proxy:-}" \ PIP_FIND_LINKS=$PIP_FIND_LINKS \ SETUPTOOLS_SYS_PATH_TECHNIQUE=rewrite \ - $cmd_pip $upgrade $install_test_reqs \ + $cmd_pip $upgrade \ $@ result=$? @@ -380,16 +379,6 @@ function setup_dev_lib { fi local name=$1 local dir=${GITDIR[$name]} - if python3_enabled; then - # Turn off Python 3 mode and install the package again, - # forcing a Python 2 installation. This ensures that all libs - # being used for development are installed under both versions - # of Python. - echo "Installing $name again without Python 3 enabled" - USE_PYTHON3=False - setup_develop $bindep $dir - USE_PYTHON3=True - fi setup_develop $bindep $dir } diff --git a/lib/apache b/lib/apache index 84cec73234..41c2e3d8dd 100644 --- a/lib/apache +++ b/lib/apache @@ -82,26 +82,56 @@ function install_apache_uwsgi { apxs="apxs" fi - # Ubuntu xenial is back level on uwsgi so the proxy doesn't - # actually work. Hence we have to build from source for now. + # This varies based on packaged/installed. If we've + # pip_installed, then the pip setup will only build a "python" + # module that will be either python2 or python3 depending on what + # it was built with. # - # Centos 7 actually has the module in epel, but there was a big - # push to disable epel by default. As such, compile from source - # there as well. - - local dir - dir=$(mktemp -d) - pushd $dir - pip_install uwsgi - pip download uwsgi -c $REQUIREMENTS_DIR/upper-constraints.txt - local uwsgi - uwsgi=$(ls uwsgi*) - tar xvf $uwsgi - cd uwsgi*/apache2 - sudo $apxs -i -c mod_proxy_uwsgi.c - popd - # delete the temp directory - sudo rm -rf $dir + # For package installs, the distro ships both plugins and you need + # to select the right one ... it will not be autodetected. + if python3_enabled; then + UWSGI_PYTHON_PLUGIN=python3 + else + UWSGI_PYTHON_PLUGIN=python + fi + + if is_ubuntu; then + install_package uwsgi \ + uwsgi-plugin-python \ + uwsgi-plugin-python3 \ + libapache2-mod-proxy-uwsgi + elif [[ $os_VENDOR == "Fedora" ]]; then + # Note httpd comes with mod_proxy_uwsgi and it is loaded by + # default; the mod_proxy_uwsgi package actually conflicts now. + # See: + # https://bugzilla.redhat.com/show_bug.cgi?id=1574335 + # + # Thus there is nothing else to do after this install + install_package uwsgi \ + uwsgi-plugin-python3 + elif [[ $os_VENDOR =~ openSUSE ]]; then + install_package uwsgi \ + uwsgi-python3 \ + apache2-mod_uwsgi + else + # Centos actually has the module in epel, but there was a big + # push to disable epel by default. As such, compile from source + # there. + local dir + dir=$(mktemp -d) + pushd $dir + pip_install uwsgi + pip download uwsgi -c $REQUIREMENTS_DIR/upper-constraints.txt + local uwsgi + uwsgi=$(ls uwsgi*) + tar xvf $uwsgi + cd uwsgi*/apache2 + sudo $apxs -i -c mod_proxy_uwsgi.c + popd + # delete the temp directory + sudo rm -rf $dir + UWSGI_PYTHON_PLUGIN=python + fi if is_ubuntu || is_suse ; then # we've got to enable proxy and proxy_uwsgi for this to work @@ -265,7 +295,7 @@ function write_uwsgi_config { # configured after graceful shutdown iniset "$file" uwsgi worker-reload-mercy $WORKER_TIMEOUT iniset "$file" uwsgi enable-threads true - iniset "$file" uwsgi plugins python + iniset "$file" uwsgi plugins http,${UWSGI_PYTHON_PLUGIN} # uwsgi recommends this to prevent thundering herd on accept. iniset "$file" uwsgi thunder-lock true # Set hook to trigger graceful shutdown on SIGTERM @@ -318,7 +348,7 @@ function write_local_uwsgi_http_config { iniset "$file" uwsgi die-on-term true iniset "$file" uwsgi exit-on-reload false iniset "$file" uwsgi enable-threads true - iniset "$file" uwsgi plugins python + iniset "$file" uwsgi plugins http,${UWSGI_PYTHON_PLUGIN} # uwsgi recommends this to prevent thundering herd on accept. iniset "$file" uwsgi thunder-lock true # Set hook to trigger graceful shutdown on SIGTERM diff --git a/lib/cinder b/lib/cinder index fd960535d9..c2e55f9173 100644 --- a/lib/cinder +++ b/lib/cinder @@ -492,7 +492,7 @@ function start_cinder { start_tls_proxy cinder '*' $CINDER_SERVICE_PORT $CINDER_SERVICE_HOST $CINDER_SERVICE_PORT_INT fi else - run_process "c-api" "$CINDER_BIN_DIR/uwsgi --procname-prefix cinder-api --ini $CINDER_UWSGI_CONF" + run_process "c-api" "$(which uwsgi) --procname-prefix cinder-api --ini $CINDER_UWSGI_CONF" cinder_url=$service_protocol://$SERVICE_HOST/volume/v3 fi fi diff --git a/lib/glance b/lib/glance index 54d3276433..56166a0013 100644 --- a/lib/glance +++ b/lib/glance @@ -325,7 +325,7 @@ function start_glance { run_process g-reg "$GLANCE_BIN_DIR/glance-registry --config-file=$GLANCE_CONF_DIR/glance-registry.conf" if [[ "$WSGI_MODE" == "uwsgi" ]]; then - run_process g-api "$GLANCE_BIN_DIR/uwsgi --procname-prefix glance-api --ini $GLANCE_UWSGI_CONF" + run_process g-api "$(which uwsgi) --procname-prefix glance-api --ini $GLANCE_UWSGI_CONF" else run_process g-api "$GLANCE_BIN_DIR/glance-api --config-dir=$GLANCE_CONF_DIR" fi diff --git a/lib/infra b/lib/infra index cf003cce01..b983f2b739 100644 --- a/lib/infra +++ b/lib/infra @@ -29,7 +29,7 @@ GITDIR["pbr"]=$DEST/pbr # install_infra() - Collect source and prepare function install_infra { local PIP_VIRTUAL_ENV="$REQUIREMENTS_DIR/.venv" - [ ! -d $PIP_VIRTUAL_ENV ] && virtualenv $PIP_VIRTUAL_ENV + [ ! -d $PIP_VIRTUAL_ENV ] && ${VIRTUALENV_CMD} $PIP_VIRTUAL_ENV # We don't care about testing git pbr in the requirements venv. PIP_VIRTUAL_ENV=$PIP_VIRTUAL_ENV pip_install -U pbr PIP_VIRTUAL_ENV=$PIP_VIRTUAL_ENV pip_install $REQUIREMENTS_DIR diff --git a/lib/keystone b/lib/keystone index 9ceb829264..1fdd941a19 100644 --- a/lib/keystone +++ b/lib/keystone @@ -523,7 +523,7 @@ function start_keystone { enable_apache_site keystone restart_apache_server else # uwsgi - run_process keystone "$KEYSTONE_BIN_DIR/uwsgi --procname-prefix keystone --ini $KEYSTONE_PUBLIC_UWSGI_CONF" "" + run_process keystone "$(which uwsgi) --procname-prefix keystone --ini $KEYSTONE_PUBLIC_UWSGI_CONF" "" fi echo "Waiting for keystone to start..." diff --git a/lib/neutron b/lib/neutron index 888b5e864e..16c20563bd 100644 --- a/lib/neutron +++ b/lib/neutron @@ -466,7 +466,7 @@ function start_neutron_api { done if [ "$NEUTRON_DEPLOY_MOD_WSGI" == "True" ]; then - run_process neutron-api "$NEUTRON_BIN_DIR/uwsgi --procname-prefix neutron-api --ini $NEUTRON_UWSGI_CONF" + run_process neutron-api "$(which uwsgi) --procname-prefix neutron-api --ini $NEUTRON_UWSGI_CONF" neutron_url=$service_protocol://$NEUTRON_SERVICE_HOST/networking/ enable_service neutron-rpc-server run_process neutron-rpc-server "$NEUTRON_BIN_DIR/neutron-rpc-server $opts" diff --git a/lib/neutron-legacy b/lib/neutron-legacy index dbd6e2c06b..49014808d8 100644 --- a/lib/neutron-legacy +++ b/lib/neutron-legacy @@ -483,7 +483,7 @@ function start_neutron_service_and_check { # Start the Neutron service if [ "$NEUTRON_DEPLOY_MOD_WSGI" == "True" ]; then enable_service neutron-api - run_process neutron-api "$NEUTRON_BIN_DIR/uwsgi --procname-prefix neutron-api --ini $NEUTRON_UWSGI_CONF" + run_process neutron-api "$(which uwsgi) --procname-prefix neutron-api --ini $NEUTRON_UWSGI_CONF" neutron_url=$Q_PROTOCOL://$Q_HOST/networking/ enable_service neutron-rpc-server run_process neutron-rpc-server "$NEUTRON_BIN_DIR/neutron-rpc-server $cfg_file_options" diff --git a/lib/nova b/lib/nova index c41f881fa1..767a7d2a74 100644 --- a/lib/nova +++ b/lib/nova @@ -306,6 +306,9 @@ function configure_nova { sudo dnf update -y fi + # Ensure each compute host uses a unique iSCSI initiator + echo InitiatorName=$(iscsi-iname) | sudo tee /etc/iscsi/initiatorname.iscsi + if [[ ${ISCSID_DEBUG} == "True" ]]; then # Install an override that starts iscsid with debugging # enabled. @@ -850,7 +853,7 @@ function start_nova_api { start_tls_proxy nova '*' $NOVA_SERVICE_PORT $NOVA_SERVICE_HOST $NOVA_SERVICE_PORT_INT fi else - run_process "n-api" "$NOVA_BIN_DIR/uwsgi --procname-prefix nova-api --ini $NOVA_UWSGI_CONF" + run_process "n-api" "$(which uwsgi) --procname-prefix nova-api --ini $NOVA_UWSGI_CONF" nova_url=$service_protocol://$SERVICE_HOST/compute/v2.1/ fi @@ -941,7 +944,7 @@ function start_nova_rest { if [ "$NOVA_USE_MOD_WSGI" == "False" ]; then run_process n-api-meta "$NOVA_BIN_DIR/nova-api-metadata --config-file $compute_cell_conf" else - run_process n-api-meta "$NOVA_BIN_DIR/uwsgi --procname-prefix nova-api-meta --ini $NOVA_METADATA_UWSGI_CONF" + run_process n-api-meta "$(which uwsgi) --procname-prefix nova-api-meta --ini $NOVA_METADATA_UWSGI_CONF" fi export PATH=$old_path diff --git a/lib/nova_plugins/functions-libvirt b/lib/nova_plugins/functions-libvirt index 463986944f..914ee7bcf7 100644 --- a/lib/nova_plugins/functions-libvirt +++ b/lib/nova_plugins/functions-libvirt @@ -150,21 +150,19 @@ EOF fi if is_nova_console_proxy_compute_tls_enabled ; then - if is_service_enabled n-novnc ; then - echo "vnc_tls = 1" | sudo tee -a $QEMU_CONF - echo "vnc_tls_x509_verify = 1" | sudo tee -a $QEMU_CONF - - sudo mkdir -p /etc/pki/libvirt-vnc - deploy_int_CA /etc/pki/libvirt-vnc/ca-cert.pem - deploy_int_cert /etc/pki/libvirt-vnc/server-cert.pem /etc/pki/libvirt-vnc/server-key.pem - # OpenSSL 1.1.0 generates the key file with permissions: 600, by - # default and the deploy_int* methods use 'sudo cp' to copy the - # files, making them owned by root:root. - # Change ownership of everything under /etc/pki/libvirt-vnc to - # libvirt-qemu:libvirt-qemu so that libvirt-qemu can read the key - # file. - sudo chown -R libvirt-qemu:libvirt-qemu /etc/pki/libvirt-vnc - fi + echo "vnc_tls = 1" | sudo tee -a $QEMU_CONF + echo "vnc_tls_x509_verify = 1" | sudo tee -a $QEMU_CONF + + sudo mkdir -p /etc/pki/libvirt-vnc + deploy_int_CA /etc/pki/libvirt-vnc/ca-cert.pem + deploy_int_cert /etc/pki/libvirt-vnc/server-cert.pem /etc/pki/libvirt-vnc/server-key.pem + # OpenSSL 1.1.0 generates the key file with permissions: 600, by + # default and the deploy_int* methods use 'sudo cp' to copy the + # files, making them owned by root:root. + # Change ownership of everything under /etc/pki/libvirt-vnc to + # libvirt-qemu:libvirt-qemu so that libvirt-qemu can read the key + # file. + sudo chown -R libvirt-qemu:libvirt-qemu /etc/pki/libvirt-vnc fi # Service needs to be started on redhat/fedora -- do a restart for diff --git a/lib/placement b/lib/placement index 785b0ddfca..2a449bfa90 100644 --- a/lib/placement +++ b/lib/placement @@ -144,7 +144,7 @@ function install_placement { # start_placement_api() - Start the API processes ahead of other things function start_placement_api { if [[ "$WSGI_MODE" == "uwsgi" ]]; then - run_process "placement-api" "$PLACEMENT_BIN_DIR/uwsgi --procname-prefix placement --ini $PLACEMENT_UWSGI_CONF" + run_process "placement-api" "$(which uwsgi) --procname-prefix placement --ini $PLACEMENT_UWSGI_CONF" else enable_apache_site placement-api restart_apache_server diff --git a/lib/swift b/lib/swift index 5be9e3575e..6afe5cc7f8 100644 --- a/lib/swift +++ b/lib/swift @@ -738,7 +738,9 @@ function init_swift { function install_swift { git_clone $SWIFT_REPO $SWIFT_DIR $SWIFT_BRANCH - setup_develop $SWIFT_DIR + # keystonemiddleware needs to be installed via keystone extras as defined + # in setup.cfg, see bug #1909018 for more details. + setup_develop $SWIFT_DIR keystone if [ "$SWIFT_USE_MOD_WSGI" == "True" ]; then install_apache_wsgi fi diff --git a/lib/tempest b/lib/tempest index 96c9ced14a..cfefc197c1 100644 --- a/lib/tempest +++ b/lib/tempest @@ -57,7 +57,7 @@ BUILD_TIMEOUT=${BUILD_TIMEOUT:-196} # This must be False on stable branches, as master tempest # deps do not match stable branch deps. Set this to True to # have tempest installed in DevStack by default. -INSTALL_TEMPEST=${INSTALL_TEMPEST:-"True"} +INSTALL_TEMPEST=${INSTALL_TEMPEST:-"False"} # This variable is passed directly to pip install inside the common tox venv # that is created @@ -110,6 +110,21 @@ function image_size_in_gib { echo $size | python -c "import math; import six; print(int(math.ceil(float(int(six.moves.input()) / 1024.0 ** 3))))" } +function set_tempest_venv_constraints { + local tmp_c + tmp_c=$1 + if [[ $TEMPEST_VENV_UPPER_CONSTRAINTS == "master" ]]; then + (cd $REQUIREMENTS_DIR && git show origin/master:upper-constraints.txt) > $tmp_c + else + echo "Using $TEMPEST_VENV_UPPER_CONSTRAINTS constraints in Tempest virtual env." + cat $TEMPEST_VENV_UPPER_CONSTRAINTS > $tmp_c + # NOTE: setting both tox env var and once Tempest start using new var + # TOX_CONSTRAINTS_FILE then we can remove the old one. + export UPPER_CONSTRAINTS_FILE=$TEMPEST_VENV_UPPER_CONSTRAINTS + export TOX_CONSTRAINTS_FILE=$TEMPEST_VENV_UPPER_CONSTRAINTS + fi +} + # configure_tempest() - Set config files, create data dirs, etc function configure_tempest { if [[ "$INSTALL_TEMPEST" == "True" ]]; then @@ -203,13 +218,13 @@ function configure_tempest { if [[ ! ( $available_flavors =~ 'm1.nano' ) ]]; then # Determine the flavor disk size based on the image size. disk=$(image_size_in_gib $image_uuid) - openstack flavor create --id 42 --ram 64 --disk $disk --vcpus 1 m1.nano + openstack flavor create --id 42 --ram 128 --disk $disk --vcpus 1 m1.nano fi flavor_ref=42 if [[ ! ( $available_flavors =~ 'm1.micro' ) ]]; then # Determine the alt flavor disk size based on the alt image size. disk=$(image_size_in_gib $image_uuid_alt) - openstack flavor create --id 84 --ram 128 --disk $disk --vcpus 1 m1.micro + openstack flavor create --id 84 --ram 192 --disk $disk --vcpus 1 m1.micro fi flavor_ref_alt=84 else @@ -376,7 +391,7 @@ function configure_tempest { # NOTE- To avoid microversion tests failure on stable branch, we need to change "tempest_compute_max_microversion" # for stable branch on each release which should be changed from "latest" to max supported version of that release. local tempest_compute_min_microversion=${TEMPEST_COMPUTE_MIN_MICROVERSION:-None} - local tempest_compute_max_microversion=${TEMPEST_COMPUTE_MAX_MICROVERSION:-"latest"} + local tempest_compute_max_microversion=${TEMPEST_COMPUTE_MAX_MICROVERSION:-"2.79"} # Reset microversions to None where v2.0 is running which does not support microversion. # Both "None" means no microversion testing. if [[ "$TEMPEST_COMPUTE_TYPE" == "compute_legacy" ]]; then @@ -433,8 +448,7 @@ function configure_tempest { SCENARIO_IMAGE_DIR=${SCENARIO_IMAGE_DIR:-$FILES} SCENARIO_IMAGE_FILE=$DEFAULT_IMAGE_FILE_NAME fi - iniset $TEMPEST_CONFIG scenario img_dir $SCENARIO_IMAGE_DIR - iniset $TEMPEST_CONFIG scenario img_file $SCENARIO_IMAGE_FILE + iniset $TEMPEST_CONFIG scenario img_file $SCENARIO_IMAGE_DIR/$SCENARIO_IMAGE_FILE # If using provider networking, use the physical network for validation rather than private TEMPEST_SSH_NETWORK_NAME=$PRIVATE_NETWORK_NAME @@ -472,8 +486,13 @@ function configure_tempest { TEMPEST_EXTEND_ATTACHED_VOLUME=${TEMPEST_EXTEND_ATTACHED_VOLUME:-True} fi iniset $TEMPEST_CONFIG volume-feature-enabled extend_attached_volume $(trueorfalse False TEMPEST_EXTEND_ATTACHED_VOLUME) + # Only turn on TEMPEST_VOLUME_REVERT_TO_SNAPSHOT by default for "lvm" backends + if [[ "$CINDER_ENABLED_BACKENDS" == *"lvm"* ]]; then + TEMPEST_VOLUME_REVERT_TO_SNAPSHOT=${TEMPEST_VOLUME_REVERT_TO_SNAPSHOT:-True} + fi + iniset $TEMPEST_CONFIG volume-feature-enabled volume_revert $(trueorfalse False TEMPEST_VOLUME_REVERT_TO_SNAPSHOT) local tempest_volume_min_microversion=${TEMPEST_VOLUME_MIN_MICROVERSION:-None} - local tempest_volume_max_microversion=${TEMPEST_VOLUME_MAX_MICROVERSION:-"latest"} + local tempest_volume_max_microversion=${TEMPEST_VOLUME_MAX_MICROVERSION:-"3.59"} # Reset microversions to None where v2 is running which does not support microversion. # Both "None" means no microversion testing. if [[ "$TEMPEST_VOLUME_TYPE" == "volumev2" ]]; then @@ -534,7 +553,7 @@ function configure_tempest { # NOTE- To avoid microversion tests failure on stable branch, we need to change "tempest_placement_max_microversion" # for stable branch on each release which should be changed from "latest" to max supported version of that release. local tempest_placement_min_microversion=${TEMPEST_PLACEMENT_MIN_MICROVERSION:-None} - local tempest_placement_max_microversion=${TEMPEST_PLACEMENT_MAX_MICROVERSION:-"latest"} + local tempest_placement_max_microversion=${TEMPEST_PLACEMENT_MAX_MICROVERSION:-"1.36"} if [ "$tempest_placement_min_microversion" == "None" ]; then inicomment $TEMPEST_CONFIG placement min_microversion else @@ -606,17 +625,19 @@ function configure_tempest { tox -revenv-tempest --notest fi - # The requirements might be on a different branch, while tempest needs master requirements. - (cd $REQUIREMENTS_DIR && git show origin/master:upper-constraints.txt) > u-c-m.txt - tox -evenv-tempest -- pip install -c u-c-m.txt -r requirements.txt + local tmp_u_c_m + tmp_u_c_m=$(mktemp -t tempest_u_c_m.XXXXXXXXXX) + set_tempest_venv_constraints $tmp_u_c_m + tox -evenv-tempest -- pip install -c $tmp_u_c_m -r requirements.txt + rm -f $tmp_u_c_m # Auth: iniset $TEMPEST_CONFIG auth tempest_roles "member" if [[ $TEMPEST_USE_TEST_ACCOUNTS == "True" ]]; then if [[ $TEMPEST_HAS_ADMIN == "True" ]]; then - tox -evenv-tempest -- tempest-account-generator -c $TEMPEST_CONFIG --os-username $admin_username --os-password "$password" --os-tenant-name $admin_project_name -r $TEMPEST_CONCURRENCY --with-admin etc/accounts.yaml + tox -evenv-tempest -- tempest account-generator -c $TEMPEST_CONFIG --os-username $admin_username --os-password "$password" --os-project-name $admin_project_name -r $TEMPEST_CONCURRENCY --with-admin etc/accounts.yaml else - tox -evenv-tempest -- tempest-account-generator -c $TEMPEST_CONFIG --os-username $admin_username --os-password "$password" --os-tenant-name $admin_project_name -r $TEMPEST_CONCURRENCY etc/accounts.yaml + tox -evenv-tempest -- tempest account-generator -c $TEMPEST_CONFIG --os-username $admin_username --os-password "$password" --os-project-name $admin_project_name -r $TEMPEST_CONCURRENCY etc/accounts.yaml fi iniset $TEMPEST_CONFIG auth use_dynamic_credentials False iniset $TEMPEST_CONFIG auth test_accounts_file "etc/accounts.yaml" @@ -643,8 +664,20 @@ function configure_tempest { if ! is_service_enabled q-l3; then DISABLE_NETWORK_API_EXTENSIONS+=", l3_agent_scheduler" fi - - local network_api_extensions=${NETWORK_API_EXTENSIONS:-"all"} + DEFAULT_NET_EXT="address-scope,agent,allowed-address-pairs,auto-allocated-topology" + DEFAULT_NET_EXT+=",availability_zone,binding,default-subnetpools,dhcp_agent_scheduler" + DEFAULT_NET_EXT+=",dvr,ext-gw-mode,external-net,extra_dhcp_opt,extraroute,flavors" + DEFAULT_NET_EXT+=",l3-flavors,l3-ha,l3_agent_scheduler,multi-provider,net-mtu" + DEFAULT_NET_EXT+=",network-ip-availability,network_availability_zone,pagination" + DEFAULT_NET_EXT+=",port-security,project-id,provider,quotas,quota_details,rbac-policies" + DEFAULT_NET_EXT+=",revision-if-match,router,router_availability_zone,security-group,service-type,sorting" + DEFAULT_NET_EXT+=",standard-attr-description,standard-attr-revisions,standard-attr-tag,standard-attr-timestamp" + DEFAULT_NET_EXT+=",subnet-service-types,subnet_allocation,net-mtu-writable,ip-substring-filtering" + DEFAULT_NET_EXT+=",availability_zone_filter,filter-validation,empty-string-filtering,port-mac-address-regenerate" + DEFAULT_NET_EXT+=",port-security-groups-filtering,fip-port-details,binding-extended" + DEFAULT_NET_EXT+=",subnet_onboard,l3-port-ip-change-not-allowed,agent-resources-synced" + DEFAULT_NET_EXT+=",floatingip-pools,rbac-security-groups,subnetpool-prefix-ops,router-admin-state-down-before-update" + local network_api_extensions=${NETWORK_API_EXTENSIONS:-$DEFAULT_NET_EXT} if [[ ! -z "$DISABLE_NETWORK_API_EXTENSIONS" ]]; then # Enabled extensions are either the ones explicitly specified or those available on the API endpoint network_api_extensions=${NETWORK_API_EXTENSIONS:-$(iniget $tmp_cfg_file network-feature-enabled api_extensions | tr -d " ")} @@ -656,7 +689,10 @@ function configure_tempest { fi iniset $TEMPEST_CONFIG network-feature-enabled api_extensions $network_api_extensions # Swift API Extensions - local object_storage_api_extensions=${OBJECT_STORAGE_API_EXTENSIONS:-"all"} + DEFAULT_SWIFT_OPT="account_quotas,bulk_delete,bulk_upload,container_quotas" + DEFAULT_SWIFT_OPT+=",container_sync,crossdomain,formpost,ratelimit,slo" + DEFAULT_SWIFT_OPT+=",staticweb,tempauth,tempurl,versioned_writes" + local object_storage_api_extensions=${OBJECT_STORAGE_API_EXTENSIONS:-$DEFAULT_SWIFT_OPT} if [[ ! -z "$DISABLE_OBJECT_STORAGE_API_EXTENSIONS" ]]; then # Enabled extensions are either the ones explicitly specified or those available on the API endpoint object_storage_api_extensions=${OBJECT_STORAGE_API_EXTENSIONS:-$(iniget $tmp_cfg_file object-storage-feature-enabled discoverable_apis | tr -d " ")} @@ -664,8 +700,30 @@ function configure_tempest { object_storage_api_extensions=$(remove_disabled_extensions $object_storage_api_extensions $DISABLE_STORAGE_API_EXTENSIONS) fi iniset $TEMPEST_CONFIG object-storage-feature-enabled discoverable_apis $object_storage_api_extensions + + # This feature list for Neutron used from Victoria, the aim is to skip tests + # for features in Neutron that has no API extension, or can't be detected + # otherwise from the API. + # It is added here to avoid stable gating to run these tests (like for IPv6 metadata + # in neutron-tempest-plugin or updating min_bw QoS values for ports already attached + # to VMs). + # To check new features grep neutron-tempest-plugin and tempest for + # is_network_feature_enabled, and please check with Neutron team. + local network_available_features="" + iniset $TEMPEST_CONFIG network-feature-enabled available_features $network_available_features # Cinder API Extensions - local volume_api_extensions=${VOLUME_API_EXTENSIONS:-"all"} + DEFAULT_VOL_EXT="OS-SCH-HNT,backups,capabilities,cgsnapshots,consistencygroups" + DEFAULT_VOL_EXT+=",encryption,os-admin-actions,os-availability-zone" + DEFAULT_VOL_EXT+=",os-extended-services,os-extended-snapshot-attributes" + DEFAULT_VOL_EXT+=",os-hosts,os-quota-class-sets,os-quota-sets" + DEFAULT_VOL_EXT+=",os-services,os-snapshot-actions,os-snapshot-manage" + DEFAULT_VOL_EXT+=",os-snapshot-unmanage,os-types-extra-specs,os-types-manage" + DEFAULT_VOL_EXT+=",os-used-limits,os-vol-host-attr,os-vol-image-meta" + DEFAULT_VOL_EXT+=",os-vol-mig-status-attr,os-vol-tenant-attr,os-volume-actions" + DEFAULT_VOL_EXT+=",os-volume-encryption-metadata,os-volume-manage" + DEFAULT_VOL_EXT+=",os-volume-transfer,os-volume-type-access" + DEFAULT_VOL_EXT+=",os-volume-unmanage,qos-specs,scheduler-stats" + local volume_api_extensions=${VOLUME_API_EXTENSIONS:-$DEFAULT_VOL_EXT} if [[ ! -z "$DISABLE_VOLUME_API_EXTENSIONS" ]]; then # Enabled extensions are either the ones explicitly specified or those available on the API endpoint volume_api_extensions=${VOLUME_API_EXTENSIONS:-$(iniget $tmp_cfg_file volume-feature-enabled api_extensions | tr -d " ")} @@ -683,12 +741,22 @@ function install_tempest { git_clone $TEMPEST_REPO $TEMPEST_DIR $TEMPEST_BRANCH pip_install 'tox!=2.8.0' pushd $TEMPEST_DIR + # NOTE(gmann): checkout the TEMPEST_BRANCH in case TEMPEST_BRANCH + # is tag name not master. git_clone would not checkout tag because + # TEMPEST_DIR already exist until RECLONE is true. + git checkout $TEMPEST_BRANCH + + local tmp_u_c_m + tmp_u_c_m=$(mktemp -t tempest_u_c_m.XXXXXXXXXX) + set_tempest_venv_constraints $tmp_u_c_m + tox -r --notest -efull # NOTE(mtreinish) Respect constraints in the tempest full venv, things that # are using a tox job other than full will not be respecting constraints but # running pip install -U on tempest requirements - $TEMPEST_DIR/.tox/tempest/bin/pip install -c $REQUIREMENTS_DIR/upper-constraints.txt -r requirements.txt + $TEMPEST_DIR/.tox/tempest/bin/pip install -c $tmp_u_c_m -r requirements.txt PROJECT_VENV["tempest"]=${TEMPEST_DIR}/.tox/tempest + rm -f $tmp_u_c_m popd } @@ -696,9 +764,11 @@ function install_tempest { function install_tempest_plugins { pushd $TEMPEST_DIR if [[ $TEMPEST_PLUGINS != 0 ]] ; then - # The requirements might be on a different branch, while tempest & tempest plugins needs master requirements. - (cd $REQUIREMENTS_DIR && git show origin/master:upper-constraints.txt) > u-c-m.txt - tox -evenv-tempest -- pip install -c u-c-m.txt $TEMPEST_PLUGINS + local tmp_u_c_m + tmp_u_c_m=$(mktemp -t tempest_u_c_m.XXXXXXXXXX) + set_tempest_venv_constraints $tmp_u_c_m + tox -evenv-tempest -- pip install -c $tmp_u_c_m $TEMPEST_PLUGINS + rm -f $tmp_u_c_m echo "Checking installed Tempest plugins:" tox -evenv-tempest -- tempest list-plugins fi diff --git a/playbooks/pre.yaml b/playbooks/pre.yaml index 4689a6354f..175888cbdd 100644 --- a/playbooks/pre.yaml +++ b/playbooks/pre.yaml @@ -19,6 +19,7 @@ set_fact: external_bridge_mtu: "{{ local_mtu | int - 50 }}" roles: + - ensure-virtualenv - test-matrix - configure-swap - setup-stack-user diff --git a/roles/devstack-ipv6-only-deployments-verification/README.rst b/roles/devstack-ipv6-only-deployments-verification/README.rst new file mode 100644 index 0000000000..400a8da222 --- /dev/null +++ b/roles/devstack-ipv6-only-deployments-verification/README.rst @@ -0,0 +1,16 @@ +Verify the IPv6-only deployments + +This role needs to be invoked from a playbook that +run tests. This role verifies the IPv6 setting on +devstack side and devstack deploy services on IPv6. +This role is invoked before tests are run so that +if any missing IPv6 setting or deployments can fail +the job early. + + +**Role Variables** + +.. zuul:rolevar:: devstack_base_dir + :default: /opt/stack + + The devstack base directory. diff --git a/roles/devstack-ipv6-only-deployments-verification/defaults/main.yaml b/roles/devstack-ipv6-only-deployments-verification/defaults/main.yaml new file mode 100644 index 0000000000..fea05c8146 --- /dev/null +++ b/roles/devstack-ipv6-only-deployments-verification/defaults/main.yaml @@ -0,0 +1 @@ +devstack_base_dir: /opt/stack diff --git a/roles/devstack-ipv6-only-deployments-verification/tasks/main.yaml b/roles/devstack-ipv6-only-deployments-verification/tasks/main.yaml new file mode 100644 index 0000000000..59d3b79bc1 --- /dev/null +++ b/roles/devstack-ipv6-only-deployments-verification/tasks/main.yaml @@ -0,0 +1,4 @@ +- name: Verify the ipv6-only deployments + become: true + become_user: stack + shell: "{{ devstack_base_dir }}/devstack/tools/verify-ipv6-only-deployments.sh" diff --git a/roles/export-devstack-journal/tasks/main.yaml b/roles/export-devstack-journal/tasks/main.yaml index cbec4447b8..ef839edaaf 100644 --- a/roles/export-devstack-journal/tasks/main.yaml +++ b/roles/export-devstack-journal/tasks/main.yaml @@ -14,7 +14,7 @@ name="" for u in $(systemctl list-unit-files | grep devstack | awk '{print $1}'); do name=$(echo $u | sed 's/devstack@/screen-/' | sed 's/\.service//') - journalctl -o short-precise --unit $u | gzip - > {{ stage_dir }}/logs/$name.txt.gz + journalctl -o short-precise --unit $u > {{ stage_dir }}/logs/$name.txt done - name: Export legacy syslog.txt @@ -29,7 +29,7 @@ -t sudo \ --no-pager \ --since="$(cat {{ devstack_base_dir }}/log-start-timestamp.txt)" \ - | gzip - > {{ stage_dir }}/logs/syslog.txt.gz + > {{ stage_dir }}/logs/syslog.txt # TODO: convert this to ansible # - make a list of the above units diff --git a/roles/process-stackviz/README.rst b/roles/process-stackviz/README.rst new file mode 100644 index 0000000000..a8447d2355 --- /dev/null +++ b/roles/process-stackviz/README.rst @@ -0,0 +1,22 @@ +Generate stackviz report. + +Generate stackviz report using subunit and dstat data, using +the stackviz archive embedded in test images. + +**Role Variables** + +.. zuul:rolevar:: devstack_base_dir + :default: /opt/stack + + The devstack base directory. + +.. zuul:rolevar:: stage_dir + :default: "{{ ansible_user_dir }}" + + The stage directory where the input data can be found and + the output will be produced. + +.. zuul:rolevar:: zuul_work_dir + :default: {{ devstack_base_dir }}/tempest + + Directory to work in. It has to be a fully qualified path. diff --git a/roles/process-stackviz/defaults/main.yaml b/roles/process-stackviz/defaults/main.yaml new file mode 100644 index 0000000000..f3bc32b149 --- /dev/null +++ b/roles/process-stackviz/defaults/main.yaml @@ -0,0 +1,3 @@ +devstack_base_dir: /opt/stack +stage_dir: "{{ ansible_user_dir }}" +zuul_work_dir: "{{ devstack_base_dir }}/tempest" diff --git a/roles/process-stackviz/tasks/main.yaml b/roles/process-stackviz/tasks/main.yaml new file mode 100644 index 0000000000..3ba3d9c2e6 --- /dev/null +++ b/roles/process-stackviz/tasks/main.yaml @@ -0,0 +1,73 @@ +- name: Process Stackviz + block: + + - name: Devstack checks if stackviz archive exists + stat: + path: "/opt/cache/files/stackviz-latest.tar.gz" + register: stackviz_archive + + - debug: + msg: "Stackviz archive could not be found in /opt/cache/files/stackviz-latest.tar.gz" + when: not stackviz_archive.stat.exists + + - name: Check if subunit data exists + stat: + path: "{{ zuul_work_dir }}/testrepository.subunit" + register: subunit_input + + - debug: + msg: "Subunit file could not be found at {{ zuul_work_dir }}/testrepository.subunit" + when: not subunit_input.stat.exists + + - name: Install stackviz + when: + - stackviz_archive.stat.exists + - subunit_input.stat.exists + block: + - include_role: + name: ensure-pip + + - pip: + name: "file://{{ stackviz_archive.stat.path }}" + virtualenv: /tmp/stackviz + virtualenv_command: '{{ ensure_pip_virtualenv_command }}' + extra_args: -U + + - name: Deploy stackviz static html+js + command: cp -pR /tmp/stackviz/share/stackviz-html {{ stage_dir }}/stackviz + when: + - stackviz_archive.stat.exists + - subunit_input.stat.exists + + - name: Check if dstat data exists + stat: + path: "{{ devstack_base_dir }}/logs/dstat-csv.log" + register: dstat_input + when: + - stackviz_archive.stat.exists + - subunit_input.stat.exists + + - name: Run stackviz with dstat + shell: | + cat {{ subunit_input.stat.path }} | \ + /tmp/stackviz/bin/stackviz-export \ + --dstat "{{ devstack_base_dir }}/logs/dstat-csv.log" \ + --env --stdin \ + {{ stage_dir }}/stackviz/data + when: + - stackviz_archive.stat.exists + - subunit_input.stat.exists + - dstat_input.stat.exists + + - name: Run stackviz without dstat + shell: | + cat {{ subunit_input.stat.path }} | \ + /tmp/stackviz/bin/stackviz-export \ + --env --stdin \ + {{ stage_dir }}/stackviz/data + when: + - stackviz_archive.stat.exists + - subunit_input.stat.exists + - not dstat_input.stat.exists + + ignore_errors: yes diff --git a/stack.sh b/stack.sh index b7b37e2cf6..29a8aa31f7 100755 --- a/stack.sh +++ b/stack.sh @@ -69,6 +69,9 @@ umask 022 # Not all distros have sbin in PATH for regular users. PATH=$PATH:/usr/local/sbin:/usr/sbin:/sbin +# Add for openSUSE +PATH=$PATH:/usr/local/bin + # Keep track of the DevStack directory TOP_DIR=$(cd $(dirname "$0") && pwd) diff --git a/stackrc b/stackrc index 10117f2700..0d92df5043 100644 --- a/stackrc +++ b/stackrc @@ -15,7 +15,7 @@ source $RC_DIR/functions # Set the target branch. This is used so that stable branching # does not need to update each repo below. -TARGET_BRANCH=master +TARGET_BRANCH=stable/train # Cycle trailing projects need to branch later than the others. TRAILING_TARGET_BRANCH=master @@ -314,7 +314,10 @@ REQUIREMENTS_BRANCH=${REQUIREMENTS_BRANCH:-$TARGET_BRANCH} # Tempest test suite TEMPEST_REPO=${TEMPEST_REPO:-${GIT_BASE}/openstack/tempest.git} -TEMPEST_BRANCH=${TEMPEST_BRANCH:-$BRANCHLESS_TARGET_BRANCH} +# NOTE (gmann): using 26.1.0 as that is compatible with stable/train +# constraints. We are not using 28.0.0 or 27.0.0 due to bug#1955418 +TEMPEST_BRANCH=${TEMPEST_BRANCH:-26.1.0} +TEMPEST_VENV_UPPER_CONSTRAINTS=${TEMPEST_VENV_UPPER_CONSTRAINTS:-$REQUIREMENTS_DIR/upper-constraints.txt} ############## diff --git a/tests/test_libs_from_pypi.sh b/tests/test_libs_from_pypi.sh index c3b4457171..da2d970482 100755 --- a/tests/test_libs_from_pypi.sh +++ b/tests/test_libs_from_pypi.sh @@ -95,19 +95,7 @@ function test_libs_exist { echo "test_libs_exist PASSED" } -function test_branch_master { - for lib in $ALL_LIBS; do - if [[ ${GITBRANCH[$lib]} != "master" ]]; then - echo "GITBRANCH for $lib not master (${GITBRANCH[$lib]})" - exit 1 - fi - done - - echo "test_branch_master PASSED" -} - set -o errexit test_libs_exist -test_branch_master test_all_libs_upto_date diff --git a/tests/test_refs.sh b/tests/test_refs.sh index 0f9aa4a5ca..d9b328294a 100755 --- a/tests/test_refs.sh +++ b/tests/test_refs.sh @@ -15,7 +15,7 @@ echo "Ensuring we don't have crazy refs" -REFS=`grep BRANCH stackrc | grep -v 'TARGET_BRANCH' | grep -v 'NOVNC_BRANCH'` +REFS=`grep BRANCH stackrc | grep -v 'TARGET_BRANCH' | grep -v 'NOVNC_BRANCH' | grep -v 'TEMPEST_BRANCH'` rc=$? if [[ $rc -eq 0 ]]; then echo "Branch defaults must be one of the *TARGET_BRANCH values. Found:" diff --git a/tools/fixup_stuff.sh b/tools/fixup_stuff.sh index d2989379fe..c21fd2d9c5 100755 --- a/tools/fixup_stuff.sh +++ b/tools/fixup_stuff.sh @@ -257,6 +257,11 @@ function fixup_suse { # have been dragged in by some other system dependency sudo rm -rf /usr/lib/python3.6/site-packages/ply-*.egg-info sudo rm -rf /usr/lib/python3.6/site-packages/six-*.egg-info + + # Ensure trusted CA certificates are up to date + # See https://bugzilla.suse.com/show_bug.cgi?id=1154871 + # May be removed once a new opensuse-15 image is available in nodepool + sudo zypper up -y p11-kit ca-certificates-mozilla } # The version of pip(1.5.4) supported by python-virtualenv(1.11.4) has diff --git a/tools/install_pip.sh b/tools/install_pip.sh index 2b6aa4c2e8..ad4010a8d3 100755 --- a/tools/install_pip.sh +++ b/tools/install_pip.sh @@ -38,7 +38,9 @@ FILES=$TOP_DIR/files # [1] https://opendev.org/openstack/project-config/src/branch/master/nodepool/elements/cache-devstack/source-repository-pip PIP_GET_PIP_URL=${PIP_GET_PIP_URL:-"https://bootstrap.pypa.io/get-pip.py"} +PIP_GET_PIP_PY27_URL=${PIP_GET_PIP_PY27_URL:-"https://bootstrap.pypa.io/pip/2.7/get-pip.py"} LOCAL_PIP="$FILES/$(basename $PIP_GET_PIP_URL)" +LOCAL_PIP_PY27="${LOCAL_PIP}-py27" GetDistro echo "Distro: $DISTRO" @@ -56,13 +58,17 @@ function get_versions { } -function install_get_pip { +function _install_get_pip { + local interpreter=$1 + local pip_get_pip_url=$2 + local local_pip=$3 + # If get-pip.py isn't python, delete it. This was probably an # outage on the server. - if [[ -r $LOCAL_PIP ]]; then - if ! head -1 $LOCAL_PIP | grep -q '#!/usr/bin/env python'; then - echo "WARNING: Corrupt $LOCAL_PIP found removing" - rm $LOCAL_PIP + if [[ -r $local_pip ]]; then + if ! head -1 $local_pip | grep -q '#!/usr/bin/env python'; then + echo "WARNING: Corrupt $local_pip found removing" + rm $local_pip fi fi @@ -76,22 +82,34 @@ function install_get_pip { # Thus we use curl's "-z" feature to always check the modified # since and only download if a new version is out -- but only if # it seems we downloaded the file originally. - if [[ ! -r $LOCAL_PIP || -r $LOCAL_PIP.downloaded ]]; then + if [[ ! -r $local_pip || -r $local_pip.downloaded ]]; then # only test freshness if LOCAL_PIP is actually there, # otherwise we generate a scary warning. local timecond="" - if [[ -r $LOCAL_PIP ]]; then - timecond="-z $LOCAL_PIP" + if [[ -r $local_pip ]]; then + timecond="-z $local_pip" fi curl -f --retry 6 --retry-delay 5 \ - $timecond -o $LOCAL_PIP $PIP_GET_PIP_URL || \ + $timecond -o $local_pip $pip_get_pip_url || \ die $LINENO "Download of get-pip.py failed" - touch $LOCAL_PIP.downloaded + touch $local_pip.downloaded fi - sudo -H -E python $LOCAL_PIP -c $TOOLS_DIR/cap-pip.txt + sudo -H -E $interpreter $local_pip -c $TOOLS_DIR/cap-pip.txt +} + + +function install_get_pip { + _install_get_pip python $PIP_GET_PIP_PY27_URL $LOCAL_PIP_PY27 if python3_enabled; then - sudo -H -E python${PYTHON3_VERSION} $LOCAL_PIP -c $TOOLS_DIR/cap-pip.txt + if [[ "$PYTHON3_VERSION" == "3.5" ]]; then + PIP_GET_PIP_URL=$(dirname $PIP_GET_PIP_URL)/pip/3.5/$(basename $PIP_GET_PIP_URL) + LOCAL_PIP=${LOCAL_PIP}-py35 + elif [[ "$PYTHON3_VERSION" == "3.6" ]]; then + PIP_GET_PIP_URL=$(dirname $PIP_GET_PIP_URL)/pip/3.6/$(basename $PIP_GET_PIP_URL) + LOCAL_PIP=${LOCAL_PIP}-py36 + fi + _install_get_pip python${PYTHON3_VERSION} $PIP_GET_PIP_URL $LOCAL_PIP fi } diff --git a/tools/verify-ipv6-only-deployments.sh b/tools/verify-ipv6-only-deployments.sh new file mode 100755 index 0000000000..2596395165 --- /dev/null +++ b/tools/verify-ipv6-only-deployments.sh @@ -0,0 +1,92 @@ +#!/bin/bash +# +# +# NOTE(gmann): This script is used in 'devstack-tempest-ipv6' zuul job to verify that +# services are deployed on IPv6 properly or not. This will capture if any devstck or devstack +# plugins are missing the required setting to listen on IPv6 address. This is run as part of +# run phase of zuul job and before test run. Child job of 'devstack-tempest-ipv6' +# can expand the IPv6 verification specific to project by defining the new post-run script which +# will run along with this base script. +# If there are more common verification for IPv6 then we can always extent this script. + +# Keep track of the DevStack directory +TOP_DIR=$(cd $(dirname "$0")/../../devstack && pwd) +source $TOP_DIR/stackrc +source $TOP_DIR/openrc admin admin + +function verify_devstack_ipv6_setting { + local _service_host='' + _service_host=$(echo $SERVICE_HOST | tr -d []) + local _host_ipv6='' + _host_ipv6=$(echo $HOST_IPV6 | tr -d []) + local _service_listen_address='' + _service_listen_address=$(echo $SERVICE_LISTEN_ADDRESS | tr -d []) + local _service_local_host='' + _service_local_host=$(echo $SERVICE_LOCAL_HOST | tr -d []) + if [[ "$SERVICE_IP_VERSION" != 6 ]]; then + echo $SERVICE_IP_VERSION "SERVICE_IP_VERSION is not set to 6 which is must for devstack to deploy services with IPv6 address." + exit 1 + fi + is_service_host_ipv6=$(python3 -c 'import oslo_utils.netutils as nutils; print(nutils.is_valid_ipv6("'$_service_host'"))') + if [[ "$is_service_host_ipv6" != "True" ]]; then + echo $SERVICE_HOST "SERVICE_HOST is not ipv6 which means devstack cannot deploy services on IPv6 address." + exit 1 + fi + is_host_ipv6=$(python3 -c 'import oslo_utils.netutils as nutils; print(nutils.is_valid_ipv6("'$_host_ipv6'"))') + if [[ "$is_host_ipv6" != "True" ]]; then + echo $HOST_IPV6 "HOST_IPV6 is not ipv6 which means devstack cannot deploy services on IPv6 address." + exit 1 + fi + is_service_listen_address=$(python3 -c 'import oslo_utils.netutils as nutils; print(nutils.is_valid_ipv6("'$_service_listen_address'"))') + if [[ "$is_service_listen_address" != "True" ]]; then + echo $SERVICE_LISTEN_ADDRESS "SERVICE_LISTEN_ADDRESS is not ipv6 which means devstack cannot deploy services on IPv6 address." + exit 1 + fi + is_service_local_host=$(python3 -c 'import oslo_utils.netutils as nutils; print(nutils.is_valid_ipv6("'$_service_local_host'"))') + if [[ "$is_service_local_host" != "True" ]]; then + echo $SERVICE_LOCAL_HOST "SERVICE_LOCAL_HOST is not ipv6 which means devstack cannot deploy services on IPv6 address." + exit 1 + fi + echo "Devstack is properly configured with IPv6" + echo "SERVICE_IP_VERSION: " $SERVICE_IP_VERSION "HOST_IPV6: " $HOST_IPV6 "SERVICE_HOST: " $SERVICE_HOST "SERVICE_LISTEN_ADDRESS: " $SERVICE_LISTEN_ADDRESS "SERVICE_LOCAL_HOST: " $SERVICE_LOCAL_HOST +} + +function sanity_check_system_ipv6_enabled { + system_ipv6_enabled=$(python3 -c 'import oslo_utils.netutils as nutils; print(nutils.is_ipv6_enabled())') + if [[ $system_ipv6_enabled != "True" ]]; then + echo "IPv6 is disabled in system" + exit 1 + fi + echo "IPv6 is enabled in system" +} + +function verify_service_listen_address_is_ipv6 { + local endpoints_verified=False + local all_ipv6=True + endpoints=$(openstack endpoint list -f value -c URL) + for endpoint in ${endpoints}; do + local endpoint_address='' + endpoint_address=$(echo "$endpoint" | awk -F/ '{print $3}' | awk -F] '{print $1}') + endpoint_address=$(echo $endpoint_address | tr -d []) + local is_endpoint_ipv6='' + is_endpoint_ipv6=$(python3 -c 'import oslo_utils.netutils as nutils; print(nutils.is_valid_ipv6("'$endpoint_address'"))') + if [[ "$is_endpoint_ipv6" != "True" ]]; then + all_ipv6=False + echo $endpoint ": This is not ipv6 endpoint which means corresponding service is not listening on IPv6 address." + continue + fi + endpoints_verified=True + done + if [[ "$all_ipv6" == "False" ]] || [[ "$endpoints_verified" == "False" ]]; then + exit 1 + fi + echo "All services deployed by devstack is on IPv6 endpoints" + echo $endpoints +} + +#First thing to verify if system has IPv6 enabled or not +sanity_check_system_ipv6_enabled +#Verify whether devstack is configured properly with IPv6 setting +verify_devstack_ipv6_setting +#Get all registrfed endpoints by devstack in keystone and verify that each endpoints address is IPv6. +verify_service_listen_address_is_ipv6 diff --git a/tox.ini b/tox.ini index 26baa2a1c5..45808d1aee 100644 --- a/tox.ini +++ b/tox.ini @@ -36,7 +36,9 @@ commands = bash -c "find {toxinidir} \ [testenv:docs] basepython = python3 -deps = -r{toxinidir}/doc/requirements.txt +deps = + -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/train} + -r{toxinidir}/doc/requirements.txt whitelist_externals = bash setenv = TOP_DIR={toxinidir}