From 5102106c4d0c94bb56ff998d3f34d62645e21a34 Mon Sep 17 00:00:00 2001 From: OpenStack Release Bot Date: Mon, 7 Oct 2019 09:04:41 +0000 Subject: [PATCH 01/45] Update .gitreview for stable/train Change-Id: I06cc37e97b83fb9b84834af3a5b7f22940af1f54 --- .gitreview | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitreview b/.gitreview index e1bf63ba7a..1637a4a66f 100644 --- a/.gitreview +++ b/.gitreview @@ -2,3 +2,4 @@ host=review.opendev.org port=29418 project=openstack/devstack.git +defaultbranch=stable/train From cfd8d1d26596c01f754d15c4f33348d1594fcaf8 Mon Sep 17 00:00:00 2001 From: Thierry Carrez Date: Mon, 7 Oct 2019 16:51:25 +0200 Subject: [PATCH 02/45] Update branches for stable/train Change-Id: I17aadc17a0ccc1beb32579bdd79d9d6c71dde908 --- stackrc | 2 +- tests/test_libs_from_pypi.sh | 12 ------------ 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/stackrc b/stackrc index 10117f2700..8b2a90d552 100644 --- a/stackrc +++ b/stackrc @@ -15,7 +15,7 @@ source $RC_DIR/functions # Set the target branch. This is used so that stable branching # does not need to update each repo below. -TARGET_BRANCH=master +TARGET_BRANCH=stable/train # Cycle trailing projects need to branch later than the others. TRAILING_TARGET_BRANCH=master diff --git a/tests/test_libs_from_pypi.sh b/tests/test_libs_from_pypi.sh index c3b4457171..da2d970482 100755 --- a/tests/test_libs_from_pypi.sh +++ b/tests/test_libs_from_pypi.sh @@ -95,19 +95,7 @@ function test_libs_exist { echo "test_libs_exist PASSED" } -function test_branch_master { - for lib in $ALL_LIBS; do - if [[ ${GITBRANCH[$lib]} != "master" ]]; then - echo "GITBRANCH for $lib not master (${GITBRANCH[$lib]})" - exit 1 - fi - done - - echo "test_branch_master PASSED" -} - set -o errexit test_libs_exist -test_branch_master test_all_libs_upto_date From 2f1e501325b1d49928840806f16864087f8f15dd Mon Sep 17 00:00:00 2001 From: Ghanshyam Mann Date: Mon, 7 Oct 2019 17:29:28 +0000 Subject: [PATCH 03/45] Cap max microversions for stable/train This commit cap the max microversions for compute and volume API for stable/train. Compute- https://opendev.org/openstack/nova/src/commit/6b177d42c7a0918018645033b6325b710f8fb925/nova/api/openstack/compute/rest_api_version_history.rst#279-maximum-in-train Volume- https://opendev.org/openstack/cinder/src/commit/ad5342445060bc63cb46531c21f27202bea78905/cinder/api/openstack/rest_api_version_history.rst#359-maximum-in-stein-and-train Placement - https://opendev.org/openstack/placement/src/commit/dbd03867118ee059b39be7677b8881b0f1a0b9c8/placement/rest_api_version_history.rst#136---support-same_subtree-queryparam-on-get-allocation_candidates Change-Id: I5fe683256301bf1a00bc15733f8c7093853123f2 --- lib/tempest | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/tempest b/lib/tempest index 96c9ced14a..ab39eba982 100644 --- a/lib/tempest +++ b/lib/tempest @@ -376,7 +376,7 @@ function configure_tempest { # NOTE- To avoid microversion tests failure on stable branch, we need to change "tempest_compute_max_microversion" # for stable branch on each release which should be changed from "latest" to max supported version of that release. local tempest_compute_min_microversion=${TEMPEST_COMPUTE_MIN_MICROVERSION:-None} - local tempest_compute_max_microversion=${TEMPEST_COMPUTE_MAX_MICROVERSION:-"latest"} + local tempest_compute_max_microversion=${TEMPEST_COMPUTE_MAX_MICROVERSION:-"2.79"} # Reset microversions to None where v2.0 is running which does not support microversion. # Both "None" means no microversion testing. if [[ "$TEMPEST_COMPUTE_TYPE" == "compute_legacy" ]]; then @@ -473,7 +473,7 @@ function configure_tempest { fi iniset $TEMPEST_CONFIG volume-feature-enabled extend_attached_volume $(trueorfalse False TEMPEST_EXTEND_ATTACHED_VOLUME) local tempest_volume_min_microversion=${TEMPEST_VOLUME_MIN_MICROVERSION:-None} - local tempest_volume_max_microversion=${TEMPEST_VOLUME_MAX_MICROVERSION:-"latest"} + local tempest_volume_max_microversion=${TEMPEST_VOLUME_MAX_MICROVERSION:-"3.59"} # Reset microversions to None where v2 is running which does not support microversion. # Both "None" means no microversion testing. if [[ "$TEMPEST_VOLUME_TYPE" == "volumev2" ]]; then @@ -534,7 +534,7 @@ function configure_tempest { # NOTE- To avoid microversion tests failure on stable branch, we need to change "tempest_placement_max_microversion" # for stable branch on each release which should be changed from "latest" to max supported version of that release. local tempest_placement_min_microversion=${TEMPEST_PLACEMENT_MIN_MICROVERSION:-None} - local tempest_placement_max_microversion=${TEMPEST_PLACEMENT_MAX_MICROVERSION:-"latest"} + local tempest_placement_max_microversion=${TEMPEST_PLACEMENT_MAX_MICROVERSION:-"1.36"} if [ "$tempest_placement_min_microversion" == "None" ]; then inicomment $TEMPEST_CONFIG placement min_microversion else From dd42ee52d7b9d628a8e51d9a297c3253bdd936e2 Mon Sep 17 00:00:00 2001 From: Ghanshyam Mann Date: Mon, 7 Oct 2019 22:23:22 +0000 Subject: [PATCH 04/45] Cap stable/train network, swift, volume api_extensions for tempest This commit cap the network, volume and swift extensions on Tempest's config option api_extensions. Change-Id: I1595730e9bcf07f8bc4eac96ff17a837fa68e0b7 --- lib/tempest | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/lib/tempest b/lib/tempest index 96c9ced14a..66d5cdf459 100644 --- a/lib/tempest +++ b/lib/tempest @@ -643,8 +643,20 @@ function configure_tempest { if ! is_service_enabled q-l3; then DISABLE_NETWORK_API_EXTENSIONS+=", l3_agent_scheduler" fi - - local network_api_extensions=${NETWORK_API_EXTENSIONS:-"all"} + DEFAULT_NET_EXT="address-scope,agent,allowed-address-pairs,auto-allocated-topology" + DEFAULT_NET_EXT+=",availability_zone,binding,default-subnetpools,dhcp_agent_scheduler" + DEFAULT_NET_EXT+=",dvr,ext-gw-mode,external-net,extra_dhcp_opt,extraroute,flavors" + DEFAULT_NET_EXT+=",l3-flavors,l3-ha,l3_agent_scheduler,multi-provider,net-mtu" + DEFAULT_NET_EXT+=",network-ip-availability,network_availability_zone,pagination" + DEFAULT_NET_EXT+=",port-security,project-id,provider,quotas,quota_details,rbac-policies" + DEFAULT_NET_EXT+=",revision-if-match,router,router_availability_zone,security-group,service-type,sorting" + DEFAULT_NET_EXT+=",standard-attr-description,standard-attr-revisions,standard-attr-tag,standard-attr-timestamp" + DEFAULT_NET_EXT+=",subnet-service-types,subnet_allocation,net-mtu-writable,ip-substring-filtering" + DEFAULT_NET_EXT+=",availability_zone_filter,filter-validation,empty-string-filtering,port-mac-address-regenerate" + DEFAULT_NET_EXT+=",port-security-groups-filtering,fip-port-details,binding-extended" + DEFAULT_NET_EXT+=",subnet_onboard,l3-port-ip-change-not-allowed,agent-resources-synced" + DEFAULT_NET_EXT+=",floatingip-pools,rbac-security-groups,subnetpool-prefix-ops,router-admin-state-down-before-update" + local network_api_extensions=${NETWORK_API_EXTENSIONS:-$DEFAULT_NET_EXT} if [[ ! -z "$DISABLE_NETWORK_API_EXTENSIONS" ]]; then # Enabled extensions are either the ones explicitly specified or those available on the API endpoint network_api_extensions=${NETWORK_API_EXTENSIONS:-$(iniget $tmp_cfg_file network-feature-enabled api_extensions | tr -d " ")} @@ -656,7 +668,10 @@ function configure_tempest { fi iniset $TEMPEST_CONFIG network-feature-enabled api_extensions $network_api_extensions # Swift API Extensions - local object_storage_api_extensions=${OBJECT_STORAGE_API_EXTENSIONS:-"all"} + DEFAULT_SWIFT_OPT="account_quotas,bulk_delete,bulk_upload,container_quotas" + DEFAULT_SWIFT_OPT+=",container_sync,crossdomain,formpost,ratelimit,slo" + DEFAULT_SWIFT_OPT+=",staticweb,tempauth,tempurl,versioned_writes" + local object_storage_api_extensions=${OBJECT_STORAGE_API_EXTENSIONS:-$DEFAULT_SWIFT_OPT} if [[ ! -z "$DISABLE_OBJECT_STORAGE_API_EXTENSIONS" ]]; then # Enabled extensions are either the ones explicitly specified or those available on the API endpoint object_storage_api_extensions=${OBJECT_STORAGE_API_EXTENSIONS:-$(iniget $tmp_cfg_file object-storage-feature-enabled discoverable_apis | tr -d " ")} @@ -665,7 +680,18 @@ function configure_tempest { fi iniset $TEMPEST_CONFIG object-storage-feature-enabled discoverable_apis $object_storage_api_extensions # Cinder API Extensions - local volume_api_extensions=${VOLUME_API_EXTENSIONS:-"all"} + DEFAULT_VOL_EXT="OS-SCH-HNT,backups,capabilities,cgsnapshots,consistencygroups" + DEFAULT_VOL_EXT+=",encryption,os-admin-actions,os-availability-zone" + DEFAULT_VOL_EXT+=",os-extended-services,os-extended-snapshot-attributes" + DEFAULT_VOL_EXT+=",os-hosts,os-quota-class-sets,os-quota-sets" + DEFAULT_VOL_EXT+=",os-services,os-snapshot-actions,os-snapshot-manage" + DEFAULT_VOL_EXT+=",os-snapshot-unmanage,os-types-extra-specs,os-types-manage" + DEFAULT_VOL_EXT+=",os-used-limits,os-vol-host-attr,os-vol-image-meta" + DEFAULT_VOL_EXT+=",os-vol-mig-status-attr,os-vol-tenant-attr,os-volume-actions" + DEFAULT_VOL_EXT+=",os-volume-encryption-metadata,os-volume-manage" + DEFAULT_VOL_EXT+=",os-volume-transfer,os-volume-type-access" + DEFAULT_VOL_EXT+=",os-volume-unmanage,qos-specs,scheduler-stats" + local volume_api_extensions=${VOLUME_API_EXTENSIONS:-$DEFAULT_VOL_EXT} if [[ ! -z "$DISABLE_VOLUME_API_EXTENSIONS" ]]; then # Enabled extensions are either the ones explicitly specified or those available on the API endpoint volume_api_extensions=${VOLUME_API_EXTENSIONS:-$(iniget $tmp_cfg_file volume-feature-enabled api_extensions | tr -d " ")} From 5bdb94bfa09918be1c33f504289f253441326954 Mon Sep 17 00:00:00 2001 From: Jens Harbott Date: Fri, 11 Oct 2019 10:39:45 +0000 Subject: [PATCH 05/45] Hardcode fedora nodeset to F29 In master we want to update fedora-latest to F30, the definition in this branch is blocking this. We have two options: - Remove the nodeset and follow master - Rename the nodeset and keep it pinned for this branch For previous stable branches we have chosen the first option, but I think we should go for the second option now favoring stability. We can always drop the nodeset lateron if it becomes broken. Change-Id: I85ab8afa9a251a160d09b960a214ac579d3305df --- .zuul.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.zuul.yaml b/.zuul.yaml index f7594d4ddc..ff3f205d32 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -49,7 +49,7 @@ - controller - nodeset: - name: devstack-single-node-fedora-latest + name: devstack-single-node-fedora-29 nodes: - name: controller label: fedora-29 @@ -535,10 +535,10 @@ voting: false - job: - name: devstack-platform-fedora-latest + name: devstack-platform-fedora-29 parent: tempest-full-py3 description: Fedora latest platform test - nodeset: devstack-single-node-fedora-latest + nodeset: devstack-single-node-fedora-29 voting: false - job: @@ -623,7 +623,7 @@ - devstack-ipv6 - devstack-platform-centos-7 - devstack-platform-opensuse-15 - - devstack-platform-fedora-latest + - devstack-platform-fedora-29 - devstack-platform-xenial - devstack-multinode - devstack-multinode-xenial From 540d581df76f3579662b2f25e0c71164b2871b85 Mon Sep 17 00:00:00 2001 From: melanie witt Date: Wed, 23 Oct 2019 04:20:23 +0000 Subject: [PATCH 06/45] Remove n-novnc service requirement for TLS configuration When configuring TLS between the console proxy (where the n-novnc service runs) and the compute host, some configuration for QEMU needs to be done on the compute host. The existing code for this requires the n-novnc service to be running, which it is in a single node all-in-one deployment. However, when running in a multinode deployment, the n-novnc service runs only on the controller and not on the subnode. Yet, we need to configure QEMU on the subnode compute host as well. This removes the n-novnc service requirement to enable TLS QEMU configuration to occur on a compute subnode in a multinode deployment. Closes-Bug: #1849418 Change-Id: I8b6970e91ad7f52ff489cb9f776ca216d8f86aa4 (cherry picked from commit 1d378dcf6d3699d99838050cc804c64a1862ba8f) --- lib/nova_plugins/functions-libvirt | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/lib/nova_plugins/functions-libvirt b/lib/nova_plugins/functions-libvirt index 463986944f..914ee7bcf7 100644 --- a/lib/nova_plugins/functions-libvirt +++ b/lib/nova_plugins/functions-libvirt @@ -150,21 +150,19 @@ EOF fi if is_nova_console_proxy_compute_tls_enabled ; then - if is_service_enabled n-novnc ; then - echo "vnc_tls = 1" | sudo tee -a $QEMU_CONF - echo "vnc_tls_x509_verify = 1" | sudo tee -a $QEMU_CONF - - sudo mkdir -p /etc/pki/libvirt-vnc - deploy_int_CA /etc/pki/libvirt-vnc/ca-cert.pem - deploy_int_cert /etc/pki/libvirt-vnc/server-cert.pem /etc/pki/libvirt-vnc/server-key.pem - # OpenSSL 1.1.0 generates the key file with permissions: 600, by - # default and the deploy_int* methods use 'sudo cp' to copy the - # files, making them owned by root:root. - # Change ownership of everything under /etc/pki/libvirt-vnc to - # libvirt-qemu:libvirt-qemu so that libvirt-qemu can read the key - # file. - sudo chown -R libvirt-qemu:libvirt-qemu /etc/pki/libvirt-vnc - fi + echo "vnc_tls = 1" | sudo tee -a $QEMU_CONF + echo "vnc_tls_x509_verify = 1" | sudo tee -a $QEMU_CONF + + sudo mkdir -p /etc/pki/libvirt-vnc + deploy_int_CA /etc/pki/libvirt-vnc/ca-cert.pem + deploy_int_cert /etc/pki/libvirt-vnc/server-cert.pem /etc/pki/libvirt-vnc/server-key.pem + # OpenSSL 1.1.0 generates the key file with permissions: 600, by + # default and the deploy_int* methods use 'sudo cp' to copy the + # files, making them owned by root:root. + # Change ownership of everything under /etc/pki/libvirt-vnc to + # libvirt-qemu:libvirt-qemu so that libvirt-qemu can read the key + # file. + sudo chown -R libvirt-qemu:libvirt-qemu /etc/pki/libvirt-vnc fi # Service needs to be started on redhat/fedora -- do a restart for From 527aeb9bec9c9017cd7cedcfb307b83820951156 Mon Sep 17 00:00:00 2001 From: Slawek Kaplonski Date: Wed, 11 Dec 2019 09:51:58 +0100 Subject: [PATCH 07/45] Include dnsmasq-utils package on all Ubuntu versions This package provides dhcp_release tool but in files/debs/neutron-common it was listed to be installed only on Ubuntu Precise. The same file is also in Nova's packages but there is no restriction to Ubuntu Precise only there. So on all Neutron jobs it was fine but on Ironic's job where Nova wasn't enabled, this package was not installed and caused problems in Neutron DHCP agent. Change-Id: Idd0711cfe6d43f21754a2f0c230cd094ea33cb27 Closes-Bug: #1855910 (cherry picked from commit f7a7076facbe8ef7c77863abf977af5cdafdeb6e) --- files/debs/neutron-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/files/debs/neutron-common b/files/debs/neutron-common index e30f678f7a..b269f6330b 100644 --- a/files/debs/neutron-common +++ b/files/debs/neutron-common @@ -1,6 +1,6 @@ acl dnsmasq-base -dnsmasq-utils # for dhcp_release only available in dist:precise +dnsmasq-utils # for dhcp_release ebtables haproxy # to serve as metadata proxy inside router/dhcp namespaces iptables From dd367e26ccf43b00d3d17b24382c2e2579521526 Mon Sep 17 00:00:00 2001 From: Sean Mooney Date: Tue, 14 Jan 2020 15:47:51 +0000 Subject: [PATCH 08/45] do not gzip legacy service logs This change removes the .gz extension from the service and syslog logs exported via journalctl. This change nolonger gzip compresses the exported logs so that they can be rendered in the browser directly when served from swift. Change-Id: I4557a737cb13b9c2406056be08ab8a32ddd45162 (cherry picked from commit d02fa6f856ac5951b8a879c23b57d5a752f28918) --- roles/export-devstack-journal/tasks/main.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/export-devstack-journal/tasks/main.yaml b/roles/export-devstack-journal/tasks/main.yaml index cbec4447b8..ef839edaaf 100644 --- a/roles/export-devstack-journal/tasks/main.yaml +++ b/roles/export-devstack-journal/tasks/main.yaml @@ -14,7 +14,7 @@ name="" for u in $(systemctl list-unit-files | grep devstack | awk '{print $1}'); do name=$(echo $u | sed 's/devstack@/screen-/' | sed 's/\.service//') - journalctl -o short-precise --unit $u | gzip - > {{ stage_dir }}/logs/$name.txt.gz + journalctl -o short-precise --unit $u > {{ stage_dir }}/logs/$name.txt done - name: Export legacy syslog.txt @@ -29,7 +29,7 @@ -t sudo \ --no-pager \ --since="$(cat {{ devstack_base_dir }}/log-start-timestamp.txt)" \ - | gzip - > {{ stage_dir }}/logs/syslog.txt.gz + > {{ stage_dir }}/logs/syslog.txt # TODO: convert this to ansible # - make a list of the above units From 1ed5bde039e7a8b2a0bc8c2fec362f1382f840fb Mon Sep 17 00:00:00 2001 From: Peter Penchev Date: Mon, 9 Sep 2019 15:14:11 +0300 Subject: [PATCH 09/45] Revert "install LIBS_FROM_GIT using python 2 and 3 where appropriate" All the OpenStack projects should be able to run under Python 3 now so the fallback installation of the Python 2 libraries should not be needed any longer. This also avoids the problem of script files installed by the libraries sometimes being overwritten by the Python 2 version leading to incorrect execution later, as discussed in http://lists.openstack.org/pipermail/openstack-discuss/2019-September/009226.html This reverts commit a2eb89417fbb6d61526b1819cbe3d0a60537eedd. Change-Id: I1cdb7e4a209872f1620be556b7278879a4b86df5 (cherry picked from commit 16bccbcea410ce426f83b5086424080b5bfaf925) --- inc/python | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/inc/python b/inc/python index ea8ff67e08..81b6a960a4 100644 --- a/inc/python +++ b/inc/python @@ -380,16 +380,6 @@ function setup_dev_lib { fi local name=$1 local dir=${GITDIR[$name]} - if python3_enabled; then - # Turn off Python 3 mode and install the package again, - # forcing a Python 2 installation. This ensures that all libs - # being used for development are installed under both versions - # of Python. - echo "Installing $name again without Python 3 enabled" - USE_PYTHON3=False - setup_develop $bindep $dir - USE_PYTHON3=True - fi setup_develop $bindep $dir } From d5906687a38f1d0d4ff76e678f089750f60b4625 Mon Sep 17 00:00:00 2001 From: Ghanshyam Date: Wed, 29 Jan 2020 15:39:17 -0600 Subject: [PATCH 10/45] Support TEMPEST_BRANCH with tag name TEMPEST_BRANCH which is mostly set as master so that Tempest master is run to test the env. With stable branch going to EM state and Tempest master might not work due to incompatibility of code or requirements. In that case we pin the Tempest so that older Tempest can be used for their testing. Till now for ocata, pike and, queens we used the gerrit style ref to pin the Tempest which is not preferred way. We should be able to use the tag name on TEMPEST_BRANCH. This commit explicitly checkout the tag set in TEMPEST_BRANCH as git_clone does not checkout the tag directly until RECLONE is true or tempest dir does not exist. After this stable branch or job can set the tag directly with name. For exmaple: TEMPEST_BRANCH=23.0.0. Change-Id: Ic777e4b56c4932dde135ac909cb5c6f4a7d5cc78 --- lib/tempest | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/tempest b/lib/tempest index ca5d972917..93b5e7ff62 100644 --- a/lib/tempest +++ b/lib/tempest @@ -709,6 +709,11 @@ function install_tempest { git_clone $TEMPEST_REPO $TEMPEST_DIR $TEMPEST_BRANCH pip_install 'tox!=2.8.0' pushd $TEMPEST_DIR + # NOTE(gmann): checkout the TEMPEST_BRANCH in case TEMPEST_BRANCH + # is tag name not master. git_clone would not checkout tag because + # TEMPEST_DIR already exist until RECLONE is true. + git checkout $TEMPEST_BRANCH + tox -r --notest -efull # NOTE(mtreinish) Respect constraints in the tempest full venv, things that # are using a tox job other than full will not be respecting constraints but From ef44118eb8b9595d01df3e9bf50a8d7c2265604e Mon Sep 17 00:00:00 2001 From: Soniya Vyas Date: Thu, 26 Dec 2019 16:59:56 +0530 Subject: [PATCH 11/45] Removal of deprecated command and deprecated optional argument lib/tempest uses 'tempest-account-generator' which is deprecated 4 years back. In addition to above, lib/tempest also uses 'os-tenant-name' which is also deprecated. Use of 'tempest account-generator' and 'os-project-name' should be done now. Signed-off by: Soniya Vyas Change-Id: I624e1dc57a3d3533322fb298c01f70241d0400ed (cherry picked from commit 7634c780027ad05e416c2a0c5ac688751aba4be6) --- lib/tempest | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/tempest b/lib/tempest index 93b5e7ff62..ed0df6feca 100644 --- a/lib/tempest +++ b/lib/tempest @@ -614,9 +614,9 @@ function configure_tempest { iniset $TEMPEST_CONFIG auth tempest_roles "member" if [[ $TEMPEST_USE_TEST_ACCOUNTS == "True" ]]; then if [[ $TEMPEST_HAS_ADMIN == "True" ]]; then - tox -evenv-tempest -- tempest-account-generator -c $TEMPEST_CONFIG --os-username $admin_username --os-password "$password" --os-tenant-name $admin_project_name -r $TEMPEST_CONCURRENCY --with-admin etc/accounts.yaml + tox -evenv-tempest -- tempest account-generator -c $TEMPEST_CONFIG --os-username $admin_username --os-password "$password" --os-project-name $admin_project_name -r $TEMPEST_CONCURRENCY --with-admin etc/accounts.yaml else - tox -evenv-tempest -- tempest-account-generator -c $TEMPEST_CONFIG --os-username $admin_username --os-password "$password" --os-tenant-name $admin_project_name -r $TEMPEST_CONCURRENCY etc/accounts.yaml + tox -evenv-tempest -- tempest account-generator -c $TEMPEST_CONFIG --os-username $admin_username --os-password "$password" --os-project-name $admin_project_name -r $TEMPEST_CONCURRENCY etc/accounts.yaml fi iniset $TEMPEST_CONFIG auth use_dynamic_credentials False iniset $TEMPEST_CONFIG auth test_accounts_file "etc/accounts.yaml" From 90b9964384a667713e2840743c4add07e3060e38 Mon Sep 17 00:00:00 2001 From: shenjiatong Date: Mon, 3 Jun 2019 10:52:01 +0800 Subject: [PATCH 12/45] remove duplicate entries under title guides in table of contents right now, there are duplicate titles under contents --> guides as shown in https://docs.openstack.org/devstack/latest/, although they will forward to different pages, it is still a little confusing for users. This change will hide toctree in guides.rst and users could click links in page to jump to detail page to read more. Change-Id: I2f3abeca7f56a3aedeabb48630ed2c61635b93cd (cherry picked from commit 16395949385356c4ab3c82227cd6c6a92496d356) --- doc/source/guides.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/guides.rst b/doc/source/guides.rst index 82e0dd6ac6..ca134c4520 100644 --- a/doc/source/guides.rst +++ b/doc/source/guides.rst @@ -10,6 +10,7 @@ Walk through various setups used by stackers .. toctree:: :glob: + :hidden: :maxdepth: 1 guides/single-vm From eb64fe162528d35ee96fe83fc6b26aa0e0a4f445 Mon Sep 17 00:00:00 2001 From: ushen Date: Wed, 18 Mar 2020 20:08:16 +0800 Subject: [PATCH 13/45] Bring back load balance v2 guide Previous commit forgets to add an entry for load balancer version 2. This ps brings them back. Change-Id: Idac09e8259d58ed130b79b40528f733708000da7 (cherry picked from commit 90f0baf3cb84118e51bc5990ba6885f07b679090) --- doc/source/guides.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/source/guides.rst b/doc/source/guides.rst index 82e0dd6ac6..52e5febc16 100644 --- a/doc/source/guides.rst +++ b/doc/source/guides.rst @@ -68,6 +68,11 @@ Nova and devstack Guide to working with nova features :doc:`Nova and devstack `. +Configure Load-Balancer Version 2 +----------------------------------- + +Guide on :doc:`Configure Load-Balancer Version 2 `. + Deploying DevStack with LDAP ---------------------------- From 7921e4673f2d0488ca6dc1892354e200ef38cca9 Mon Sep 17 00:00:00 2001 From: Colleen Murphy Date: Fri, 3 Apr 2020 10:14:07 -0700 Subject: [PATCH 14/45] Fix opensuse trusted certificates There's a bug[1] with the combination of the p11-kit and ca-certificates-mozilla packages available on the latest built opensuse-15 node in nodepool (which has not been rebuilt for weeks due to a separate issue[2]) which causes the standard CA bundle to not be installed correctly and causes jobs that call to external HTTPS services to fail. Upgrading both packages in sync fixes the issue. [1] https://bugzilla.suse.com/show_bug.cgi?id=1154871 [2] http://bugzilla.suse.com/show_bug.cgi?id=1166139 Change-Id: Ia8fdfe12fd9089e178adcb2b5eec997eebada262 Needed-by: https://review.opendev.org/713566 (cherry picked from commit 497caf015729d451428d5b608853741689f153b3) --- tools/fixup_stuff.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tools/fixup_stuff.sh b/tools/fixup_stuff.sh index d2989379fe..c21fd2d9c5 100755 --- a/tools/fixup_stuff.sh +++ b/tools/fixup_stuff.sh @@ -257,6 +257,11 @@ function fixup_suse { # have been dragged in by some other system dependency sudo rm -rf /usr/lib/python3.6/site-packages/ply-*.egg-info sudo rm -rf /usr/lib/python3.6/site-packages/six-*.egg-info + + # Ensure trusted CA certificates are up to date + # See https://bugzilla.suse.com/show_bug.cgi?id=1154871 + # May be removed once a new opensuse-15 image is available in nodepool + sudo zypper up -y p11-kit ca-certificates-mozilla } # The version of pip(1.5.4) supported by python-virtualenv(1.11.4) has From 5e0c142e808c6fe00a3ae843b78ddeb7311f1cb7 Mon Sep 17 00:00:00 2001 From: Ghanshyam Mann Date: Tue, 7 Apr 2020 09:32:06 -0500 Subject: [PATCH 15/45] Fix Tempest installtion on system wide for stable branch INSTALL_TEMPEST flag enable to install Tempest by default on devstack env which is meant for using on master gate only and has to be false for stable branch. On stable branch master Tempest on system wide can fail to install for various reason like constraint etc. That is why we install Tempest on venv always. This started failing tempest-full on stein gate[1] where devstack try to install Tempest on system wide also which try to use py2 env because jobs is py2 and fail. - https://zuul.opendev.org/t/openstack/build/f05569c475f44327bff7b7ec58faef8c/log/controller/logs/devstacklog.txt#30654 We need to set INSTALL_TEMPEST flag to false on all stable branch to avoid such issue which we did not do since pike. (I will make note of this in Devstack branch cut TODO list) - https://github.com/openstack/devstack/blob/6067e41a5cc44164e34faab2fd9741dbb788f42f/lib/tempest#L60 [1] https://zuul.opendev.org/t/openstack/build/f05569c475f44327bff7b7ec58faef8c Change-Id: I60949fb735c82959fb2cfcb6aeef9e33fb0445b6 Closes-Bug: #1871327 --- lib/tempest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tempest b/lib/tempest index ed0df6feca..9a04fe5a6a 100644 --- a/lib/tempest +++ b/lib/tempest @@ -57,7 +57,7 @@ BUILD_TIMEOUT=${BUILD_TIMEOUT:-196} # This must be False on stable branches, as master tempest # deps do not match stable branch deps. Set this to True to # have tempest installed in DevStack by default. -INSTALL_TEMPEST=${INSTALL_TEMPEST:-"True"} +INSTALL_TEMPEST=${INSTALL_TEMPEST:-"False"} # This variable is passed directly to pip install inside the common tox venv # that is created From ebf7705a982702fe6f4b1149656f9bcd613aa160 Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Mon, 13 Jan 2020 16:05:11 +1100 Subject: [PATCH 16/45] Create virtualenv with abstracted VIRTUALENV_CMD Just calling "virtualenv" makes a Python 2 based environment; setuptools just dropped Python 2 support (as Python 2 reached EOL in Jan 2020) so this has now become a breakage. Although the Python 2 path won't work, use the abstracted command. This should stop us having to revisit this for any future cleanups (or switing to venv, etc). Change-Id: I531e971b78491a9276753c0d86b04c4adbd224aa (cherry picked from commit 4143ce6fc95ccd9bde88c22baaa3eeaba58183f8) --- lib/infra | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/infra b/lib/infra index cf003cce01..b983f2b739 100644 --- a/lib/infra +++ b/lib/infra @@ -29,7 +29,7 @@ GITDIR["pbr"]=$DEST/pbr # install_infra() - Collect source and prepare function install_infra { local PIP_VIRTUAL_ENV="$REQUIREMENTS_DIR/.venv" - [ ! -d $PIP_VIRTUAL_ENV ] && virtualenv $PIP_VIRTUAL_ENV + [ ! -d $PIP_VIRTUAL_ENV ] && ${VIRTUALENV_CMD} $PIP_VIRTUAL_ENV # We don't care about testing git pbr in the requirements venv. PIP_VIRTUAL_ENV=$PIP_VIRTUAL_ENV pip_install -U pbr PIP_VIRTUAL_ENV=$PIP_VIRTUAL_ENV pip_install $REQUIREMENTS_DIR From 23cfb9e6ebc63a4da4577c0ef9e3450b9c946fa7 Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Tue, 12 May 2020 12:05:45 +1000 Subject: [PATCH 17/45] Add ensure-virtualenv role On this branch some tests are still running under Python 2 and thus rely on virtualenv (instead of us standardising on "python3 -m venv" on the master branch where we are all Python 3). Use the zuul-role to ensure the system package that provides "/usr/bin/virtualenv" is installed. Change-Id: I2b092bd70a9dbc6ef3a673ba45ca6925f9220bce --- playbooks/pre.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/playbooks/pre.yaml b/playbooks/pre.yaml index 4689a6354f..175888cbdd 100644 --- a/playbooks/pre.yaml +++ b/playbooks/pre.yaml @@ -19,6 +19,7 @@ set_fact: external_bridge_mtu: "{{ local_mtu | int - 50 }}" roles: + - ensure-virtualenv - test-matrix - configure-swap - setup-stack-user From a6919e5065ef7fce5bccbf2e8be22c8d18193339 Mon Sep 17 00:00:00 2001 From: Luigi Toscano Date: Tue, 12 May 2020 12:00:20 +0200 Subject: [PATCH 18/45] New TEMPEST_VOLUME_REVERT_TO_SNAPSHOT setting Follow the pattern of the other configuration keys. The new variables allows tests to enable/disable volume revert tests provided by cinder-tempest-plugin. Revert-to-snapshot was introduced in pike, and so the tests. Change-Id: If137f201c2f646703f5a1ff96e71e48caed63b67 (cherry picked from commit 8fd45dec50ca8872bf07563dcb41cffca3683965) --- lib/tempest | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/tempest b/lib/tempest index 9a04fe5a6a..831d118fa2 100644 --- a/lib/tempest +++ b/lib/tempest @@ -472,6 +472,11 @@ function configure_tempest { TEMPEST_EXTEND_ATTACHED_VOLUME=${TEMPEST_EXTEND_ATTACHED_VOLUME:-True} fi iniset $TEMPEST_CONFIG volume-feature-enabled extend_attached_volume $(trueorfalse False TEMPEST_EXTEND_ATTACHED_VOLUME) + # Only turn on TEMPEST_VOLUME_REVERT_TO_SNAPSHOT by default for "lvm" backends + if [[ "$CINDER_ENABLED_BACKENDS" == *"lvm"* ]]; then + TEMPEST_VOLUME_REVERT_TO_SNAPSHOT=${TEMPEST_VOLUME_REVERT_TO_SNAPSHOT:-True} + fi + iniset $TEMPEST_CONFIG volume-feature-enabled volume_revert $(trueorfalse False TEMPEST_VOLUME_REVERT_TO_SNAPSHOT) local tempest_volume_min_microversion=${TEMPEST_VOLUME_MIN_MICROVERSION:-None} local tempest_volume_max_microversion=${TEMPEST_VOLUME_MAX_MICROVERSION:-"3.59"} # Reset microversions to None where v2 is running which does not support microversion. From 2e73509fbfc6f96b1f1ba690ba4ddf238bdc6266 Mon Sep 17 00:00:00 2001 From: Lee Yarwood Date: Fri, 22 May 2020 13:03:15 +0100 Subject: [PATCH 19/45] tempest: Increase m1.nano and m1.micro RAM by 64MB to avoid tmpfs exhaustion tmpfs exhaustion has long been suspected as the root issue behind failures to load ssh keys and other metadata from local config drives as documented in bug #1808010. This can also lead to failures fetching metadata from n-metadata-api leaving Tempest unable to SSH into instances. This change increases the RAM of the m1.nano and m1.micro flavors by 64MB to hopefully avoid these errors going forward. This is also ahead of our eventual upgrade to Cirros 0.5.0 where 128MB becomes a requirement. Related-Bug: #1808010 Change-Id: I4b597579cf89939955d3c110c0bd58ca05de61f0 (cherry picked from commit 54edc7aeef97df768477b5fa14f8fc45266a9c2e) (cherry picked from commit 9707dba33830b8a3ca97fa6abf091d9d1d0d7899) --- lib/tempest | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/tempest b/lib/tempest index 9a04fe5a6a..8d2df9fadc 100644 --- a/lib/tempest +++ b/lib/tempest @@ -203,13 +203,13 @@ function configure_tempest { if [[ ! ( $available_flavors =~ 'm1.nano' ) ]]; then # Determine the flavor disk size based on the image size. disk=$(image_size_in_gib $image_uuid) - openstack flavor create --id 42 --ram 64 --disk $disk --vcpus 1 m1.nano + openstack flavor create --id 42 --ram 128 --disk $disk --vcpus 1 m1.nano fi flavor_ref=42 if [[ ! ( $available_flavors =~ 'm1.micro' ) ]]; then # Determine the alt flavor disk size based on the alt image size. disk=$(image_size_in_gib $image_uuid_alt) - openstack flavor create --id 84 --ram 128 --disk $disk --vcpus 1 m1.micro + openstack flavor create --id 84 --ram 192 --disk $disk --vcpus 1 m1.micro fi flavor_ref_alt=84 else From 20e04092c8ce311017b2d490c4041cfb9b3a9395 Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Thu, 3 May 2018 10:51:30 +1000 Subject: [PATCH 20/45] Fix gate issues This is a combination of 3 commits. This is the 1st commit message: Work around uwsgi breakage uwsgi broke installation from source with their latest release [0]. Since we want to move away from source installation anyway, make grenade based jobs non-voting for the moment so that we can backport [1] properly. [0] https://bugs.launchpad.net/bugs/1883468 [1] https://review.opendev.org/577955 This is the commit message #2: Use packaged uwsgi on Fedora and Ubuntu Building uwsgi from source was a workaround that was introduced a long time ago, it doesn't seem like it is needed anymore and will actually fail for Ubuntu 20.04. Also it doesn't match what will happen for most real-world installations, so let's try to get back to using distro packages. We'll still use the source install for RHEL/Centos, it remains to be tested whether we can get back to using distro versions there, too. This is the commit message #3: Use uwsgi binary from path All these uwsgi invocations assume that the uwsgi binary is in the same directory as their project binaries are installed into (probably /usr/bin). That may not be correct -- for example if using a packaged uwsgi on Fedora the binary will live in /usr/sbin/uwsgi (not /usr/bin where the project files from pip are). Switch invocations to just find it in the path. While all this was happening, Zuul updated to Ansible 2.9 as well, so we require a fix for Ansible syntax in 736006 as well. Depends-On: https://review.opendev.org/735535 Depends-On: https://review.opendev.org/736006 Related-Bug: 1883468 Change-Id: I82f539bfa533349293dd5a8ce309c9cc0ffb0393 (cherry picked from commit 84737ebd96327310ec5f8f7c312aeae12cbeb234) (cherry picked from commit 2d903568ed4158aa668bbda6986307a8780c71a4) (cherry picked from commit 312517d5101206b33d3c395d27ec93d385b7fd24) --- .zuul.yaml | 28 +++++++++++++------ lib/apache | 68 ++++++++++++++++++++++++++++++++-------------- lib/cinder | 2 +- lib/glance | 2 +- lib/keystone | 2 +- lib/neutron | 2 +- lib/neutron-legacy | 2 +- lib/nova | 4 +-- lib/placement | 2 +- 9 files changed, 75 insertions(+), 37 deletions(-) diff --git a/.zuul.yaml b/.zuul.yaml index ff3f205d32..0112bdc203 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -619,14 +619,16 @@ check: jobs: - devstack - - devstack-xenial + - devstack-xenial: + voting: false - devstack-ipv6 - devstack-platform-centos-7 - devstack-platform-opensuse-15 - devstack-platform-fedora-29 - devstack-platform-xenial - devstack-multinode - - devstack-multinode-xenial + - devstack-multinode-xenial: + voting: false - devstack-unit-tests - openstack-tox-bashate - ironic-tempest-ipa-wholedisk-bios-agent_ipmitool-tinyipa: @@ -640,7 +642,13 @@ irrelevant-files: - ^.*\.rst$ - ^doc/.*$ + - grenade: + voting: false + irrelevant-files: + - ^.*\.rst$ + - ^doc/.*$ - neutron-grenade-multinode: + voting: false irrelevant-files: - ^.*\.rst$ - ^doc/.*$ @@ -664,16 +672,16 @@ gate: jobs: - devstack - - devstack-xenial + #- devstack-xenial - devstack-ipv6 - devstack-multinode - - devstack-multinode-xenial + #- devstack-multinode-xenial - devstack-unit-tests - openstack-tox-bashate - - neutron-grenade-multinode: - irrelevant-files: - - ^.*\.rst$ - - ^doc/.*$ + # - neutron-grenade-multinode: + # irrelevant-files: + # - ^.*\.rst$ + # - ^doc/.*$ - neutron-tempest-linuxbridge: irrelevant-files: - ^.*\.rst$ @@ -682,6 +690,10 @@ irrelevant-files: - ^.*\.rst$ - ^doc/.*$ + # - grenade: + # irrelevant-files: + # - ^.*\.rst$ + # - ^doc/.*$ - openstacksdk-functional-devstack: irrelevant-files: - ^.*\.rst$ diff --git a/lib/apache b/lib/apache index 84cec73234..a31188bebb 100644 --- a/lib/apache +++ b/lib/apache @@ -82,26 +82,52 @@ function install_apache_uwsgi { apxs="apxs" fi - # Ubuntu xenial is back level on uwsgi so the proxy doesn't - # actually work. Hence we have to build from source for now. + # This varies based on packaged/installed. If we've + # pip_installed, then the pip setup will only build a "python" + # module that will be either python2 or python3 depending on what + # it was built with. # - # Centos 7 actually has the module in epel, but there was a big - # push to disable epel by default. As such, compile from source - # there as well. - - local dir - dir=$(mktemp -d) - pushd $dir - pip_install uwsgi - pip download uwsgi -c $REQUIREMENTS_DIR/upper-constraints.txt - local uwsgi - uwsgi=$(ls uwsgi*) - tar xvf $uwsgi - cd uwsgi*/apache2 - sudo $apxs -i -c mod_proxy_uwsgi.c - popd - # delete the temp directory - sudo rm -rf $dir + # For package installs, the distro ships both plugins and you need + # to select the right one ... it will not be autodetected. + if python3_enabled; then + UWSGI_PYTHON_PLUGIN=python3 + else + UWSGI_PYTHON_PLUGIN=python + fi + + if is_ubuntu; then + install_package uwsgi \ + uwsgi-plugin-python \ + uwsgi-plugin-python3 \ + libapache2-mod-proxy-uwsgi + elif [[ $os_VENDOR == "Fedora" ]]; then + # Note httpd comes with mod_proxy_uwsgi and it is loaded by + # default; the mod_proxy_uwsgi package actually conflicts now. + # See: + # https://bugzilla.redhat.com/show_bug.cgi?id=1574335 + # + # Thus there is nothing else to do after this install + install_package uwsgi \ + uwsgi-plugin-python3 + else + # Centos actually has the module in epel, but there was a big + # push to disable epel by default. As such, compile from source + # there. + local dir + dir=$(mktemp -d) + pushd $dir + pip_install uwsgi + pip download uwsgi -c $REQUIREMENTS_DIR/upper-constraints.txt + local uwsgi + uwsgi=$(ls uwsgi*) + tar xvf $uwsgi + cd uwsgi*/apache2 + sudo $apxs -i -c mod_proxy_uwsgi.c + popd + # delete the temp directory + sudo rm -rf $dir + UWSGI_PYTHON_PLUGIN=python + fi if is_ubuntu || is_suse ; then # we've got to enable proxy and proxy_uwsgi for this to work @@ -265,7 +291,7 @@ function write_uwsgi_config { # configured after graceful shutdown iniset "$file" uwsgi worker-reload-mercy $WORKER_TIMEOUT iniset "$file" uwsgi enable-threads true - iniset "$file" uwsgi plugins python + iniset "$file" uwsgi plugins http,${UWSGI_PYTHON_PLUGIN} # uwsgi recommends this to prevent thundering herd on accept. iniset "$file" uwsgi thunder-lock true # Set hook to trigger graceful shutdown on SIGTERM @@ -318,7 +344,7 @@ function write_local_uwsgi_http_config { iniset "$file" uwsgi die-on-term true iniset "$file" uwsgi exit-on-reload false iniset "$file" uwsgi enable-threads true - iniset "$file" uwsgi plugins python + iniset "$file" uwsgi plugins http,${UWSGI_PYTHON_PLUGIN} # uwsgi recommends this to prevent thundering herd on accept. iniset "$file" uwsgi thunder-lock true # Set hook to trigger graceful shutdown on SIGTERM diff --git a/lib/cinder b/lib/cinder index fd960535d9..c2e55f9173 100644 --- a/lib/cinder +++ b/lib/cinder @@ -492,7 +492,7 @@ function start_cinder { start_tls_proxy cinder '*' $CINDER_SERVICE_PORT $CINDER_SERVICE_HOST $CINDER_SERVICE_PORT_INT fi else - run_process "c-api" "$CINDER_BIN_DIR/uwsgi --procname-prefix cinder-api --ini $CINDER_UWSGI_CONF" + run_process "c-api" "$(which uwsgi) --procname-prefix cinder-api --ini $CINDER_UWSGI_CONF" cinder_url=$service_protocol://$SERVICE_HOST/volume/v3 fi fi diff --git a/lib/glance b/lib/glance index 54d3276433..56166a0013 100644 --- a/lib/glance +++ b/lib/glance @@ -325,7 +325,7 @@ function start_glance { run_process g-reg "$GLANCE_BIN_DIR/glance-registry --config-file=$GLANCE_CONF_DIR/glance-registry.conf" if [[ "$WSGI_MODE" == "uwsgi" ]]; then - run_process g-api "$GLANCE_BIN_DIR/uwsgi --procname-prefix glance-api --ini $GLANCE_UWSGI_CONF" + run_process g-api "$(which uwsgi) --procname-prefix glance-api --ini $GLANCE_UWSGI_CONF" else run_process g-api "$GLANCE_BIN_DIR/glance-api --config-dir=$GLANCE_CONF_DIR" fi diff --git a/lib/keystone b/lib/keystone index 9ceb829264..1fdd941a19 100644 --- a/lib/keystone +++ b/lib/keystone @@ -523,7 +523,7 @@ function start_keystone { enable_apache_site keystone restart_apache_server else # uwsgi - run_process keystone "$KEYSTONE_BIN_DIR/uwsgi --procname-prefix keystone --ini $KEYSTONE_PUBLIC_UWSGI_CONF" "" + run_process keystone "$(which uwsgi) --procname-prefix keystone --ini $KEYSTONE_PUBLIC_UWSGI_CONF" "" fi echo "Waiting for keystone to start..." diff --git a/lib/neutron b/lib/neutron index 888b5e864e..16c20563bd 100644 --- a/lib/neutron +++ b/lib/neutron @@ -466,7 +466,7 @@ function start_neutron_api { done if [ "$NEUTRON_DEPLOY_MOD_WSGI" == "True" ]; then - run_process neutron-api "$NEUTRON_BIN_DIR/uwsgi --procname-prefix neutron-api --ini $NEUTRON_UWSGI_CONF" + run_process neutron-api "$(which uwsgi) --procname-prefix neutron-api --ini $NEUTRON_UWSGI_CONF" neutron_url=$service_protocol://$NEUTRON_SERVICE_HOST/networking/ enable_service neutron-rpc-server run_process neutron-rpc-server "$NEUTRON_BIN_DIR/neutron-rpc-server $opts" diff --git a/lib/neutron-legacy b/lib/neutron-legacy index dbd6e2c06b..49014808d8 100644 --- a/lib/neutron-legacy +++ b/lib/neutron-legacy @@ -483,7 +483,7 @@ function start_neutron_service_and_check { # Start the Neutron service if [ "$NEUTRON_DEPLOY_MOD_WSGI" == "True" ]; then enable_service neutron-api - run_process neutron-api "$NEUTRON_BIN_DIR/uwsgi --procname-prefix neutron-api --ini $NEUTRON_UWSGI_CONF" + run_process neutron-api "$(which uwsgi) --procname-prefix neutron-api --ini $NEUTRON_UWSGI_CONF" neutron_url=$Q_PROTOCOL://$Q_HOST/networking/ enable_service neutron-rpc-server run_process neutron-rpc-server "$NEUTRON_BIN_DIR/neutron-rpc-server $cfg_file_options" diff --git a/lib/nova b/lib/nova index c41f881fa1..5b3246a2c2 100644 --- a/lib/nova +++ b/lib/nova @@ -850,7 +850,7 @@ function start_nova_api { start_tls_proxy nova '*' $NOVA_SERVICE_PORT $NOVA_SERVICE_HOST $NOVA_SERVICE_PORT_INT fi else - run_process "n-api" "$NOVA_BIN_DIR/uwsgi --procname-prefix nova-api --ini $NOVA_UWSGI_CONF" + run_process "n-api" "$(which uwsgi) --procname-prefix nova-api --ini $NOVA_UWSGI_CONF" nova_url=$service_protocol://$SERVICE_HOST/compute/v2.1/ fi @@ -941,7 +941,7 @@ function start_nova_rest { if [ "$NOVA_USE_MOD_WSGI" == "False" ]; then run_process n-api-meta "$NOVA_BIN_DIR/nova-api-metadata --config-file $compute_cell_conf" else - run_process n-api-meta "$NOVA_BIN_DIR/uwsgi --procname-prefix nova-api-meta --ini $NOVA_METADATA_UWSGI_CONF" + run_process n-api-meta "$(which uwsgi) --procname-prefix nova-api-meta --ini $NOVA_METADATA_UWSGI_CONF" fi export PATH=$old_path diff --git a/lib/placement b/lib/placement index 785b0ddfca..2a449bfa90 100644 --- a/lib/placement +++ b/lib/placement @@ -144,7 +144,7 @@ function install_placement { # start_placement_api() - Start the API processes ahead of other things function start_placement_api { if [[ "$WSGI_MODE" == "uwsgi" ]]; then - run_process "placement-api" "$PLACEMENT_BIN_DIR/uwsgi --procname-prefix placement --ini $PLACEMENT_UWSGI_CONF" + run_process "placement-api" "$(which uwsgi) --procname-prefix placement --ini $PLACEMENT_UWSGI_CONF" else enable_apache_site placement-api restart_apache_server From bce3f12aa412f76e2e2b3fa095f329b37f06a09b Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Tue, 16 Jun 2020 17:17:35 -0700 Subject: [PATCH 21/45] Make xenial jobs voting again Xenial jobs were failing because running virtualenv with system pip was failing. [0] installs a working pip on xenial that can accomodate mirrors. [0] https://review.opendev.org/736067 Change-Id: I6c75b1ccc33122f4f82427d743e1368be9bd2069 --- .zuul.yaml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.zuul.yaml b/.zuul.yaml index 0112bdc203..a4b70f1fa7 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -619,16 +619,14 @@ check: jobs: - devstack - - devstack-xenial: - voting: false + - devstack-xenial - devstack-ipv6 - devstack-platform-centos-7 - devstack-platform-opensuse-15 - devstack-platform-fedora-29 - devstack-platform-xenial - devstack-multinode - - devstack-multinode-xenial: - voting: false + - devstack-multinode-xenial - devstack-unit-tests - openstack-tox-bashate - ironic-tempest-ipa-wholedisk-bios-agent_ipmitool-tinyipa: @@ -672,10 +670,10 @@ gate: jobs: - devstack - #- devstack-xenial + - devstack-xenial - devstack-ipv6 - devstack-multinode - #- devstack-multinode-xenial + - devstack-multinode-xenial - devstack-unit-tests - openstack-tox-bashate # - neutron-grenade-multinode: From 9764fadc021eb4d69b8f07d31b64ef5c4ae05f43 Mon Sep 17 00:00:00 2001 From: Jens Harbott Date: Wed, 17 Jun 2020 08:48:14 +0200 Subject: [PATCH 22/45] Drop fedora29 nodeset and platform job Fedora 29 is EOL and we no longer have nodes available for it. Change-Id: If19b7b9858fef7422b2046cbc6c0fcd97832ebcd --- .zuul.yaml | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/.zuul.yaml b/.zuul.yaml index a4b70f1fa7..b7f6dcc354 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -48,16 +48,6 @@ nodes: - controller -- nodeset: - name: devstack-single-node-fedora-29 - nodes: - - name: controller - label: fedora-29 - groups: - - name: tempest - nodes: - - controller - - nodeset: name: openstack-two-node nodes: @@ -534,13 +524,6 @@ nodeset: devstack-single-node-opensuse-15 voting: false -- job: - name: devstack-platform-fedora-29 - parent: tempest-full-py3 - description: Fedora latest platform test - nodeset: devstack-single-node-fedora-29 - voting: false - - job: name: devstack-platform-xenial parent: tempest-full-py3 @@ -623,7 +606,6 @@ - devstack-ipv6 - devstack-platform-centos-7 - devstack-platform-opensuse-15 - - devstack-platform-fedora-29 - devstack-platform-xenial - devstack-multinode - devstack-multinode-xenial From 07a17314041d7589547a568dec2608e92a26dc94 Mon Sep 17 00:00:00 2001 From: Andreas Jaeger Date: Mon, 15 Jun 2020 10:03:42 +0200 Subject: [PATCH 23/45] Handle uwsgi install for openSUSE Add packages for openSUSE distribution for installation of uwsgi. For train add in addition: Add /usr/local/bin to PATH, this is not set on openSUSE and needed to fix the setup after the pip/virtualenv changes. Co-Authored-By: Jens Harbott Change-Id: I2ce8959460a79b6472bd9dd59edd7d94eccbacf5 (cherry-picked from commit 10c3ffd26557f1921841e456b003a77f6fb11948) --- lib/apache | 4 ++++ stack.sh | 3 +++ 2 files changed, 7 insertions(+) diff --git a/lib/apache b/lib/apache index a31188bebb..41c2e3d8dd 100644 --- a/lib/apache +++ b/lib/apache @@ -109,6 +109,10 @@ function install_apache_uwsgi { # Thus there is nothing else to do after this install install_package uwsgi \ uwsgi-plugin-python3 + elif [[ $os_VENDOR =~ openSUSE ]]; then + install_package uwsgi \ + uwsgi-python3 \ + apache2-mod_uwsgi else # Centos actually has the module in epel, but there was a big # push to disable epel by default. As such, compile from source diff --git a/stack.sh b/stack.sh index b7b37e2cf6..29a8aa31f7 100755 --- a/stack.sh +++ b/stack.sh @@ -69,6 +69,9 @@ umask 022 # Not all distros have sbin in PATH for regular users. PATH=$PATH:/usr/local/sbin:/usr/sbin:/sbin +# Add for openSUSE +PATH=$PATH:/usr/local/bin + # Keep track of the DevStack directory TOP_DIR=$(cd $(dirname "$0") && pwd) From d377574486f21b1ca2a67e31bcf359afab7c6263 Mon Sep 17 00:00:00 2001 From: Ghanshyam Mann Date: Fri, 19 Jun 2020 10:31:38 -0500 Subject: [PATCH 24/45] Move process-stackviz role from Tempest to Devstack process-stackviz role currently defined in Tempest repo[1] and used in tempest jobs. Issue with having it in branchless Tempest is that any fix in that role cannot be backported to older Tempest. Also stackviz is not Tempest things it is only used by Tempest jobs. Stackviz can be considered as a service similar to Tempest so inatlling it in Devstack side make sense. Also that will give us advantage to handle the branch specific fixes or backpoting the fixes. This can solve the current issues we are facing on rocky branch - https://zuul.opendev.org/t/openstack/build/c1e2da80537448bfb24331fb4198a931/console#4/0/29/controller [1] https://opendev.org/openstack/tempest/src/branch/master/roles/process-stackviz Change-Id: I0ba1fd05c6391cd8bb978119ecfbb891def2d497 --- roles/process-stackviz/README.rst | 22 +++++++ roles/process-stackviz/defaults/main.yaml | 3 + roles/process-stackviz/tasks/main.yaml | 70 +++++++++++++++++++++++ 3 files changed, 95 insertions(+) create mode 100644 roles/process-stackviz/README.rst create mode 100644 roles/process-stackviz/defaults/main.yaml create mode 100644 roles/process-stackviz/tasks/main.yaml diff --git a/roles/process-stackviz/README.rst b/roles/process-stackviz/README.rst new file mode 100644 index 0000000000..a8447d2355 --- /dev/null +++ b/roles/process-stackviz/README.rst @@ -0,0 +1,22 @@ +Generate stackviz report. + +Generate stackviz report using subunit and dstat data, using +the stackviz archive embedded in test images. + +**Role Variables** + +.. zuul:rolevar:: devstack_base_dir + :default: /opt/stack + + The devstack base directory. + +.. zuul:rolevar:: stage_dir + :default: "{{ ansible_user_dir }}" + + The stage directory where the input data can be found and + the output will be produced. + +.. zuul:rolevar:: zuul_work_dir + :default: {{ devstack_base_dir }}/tempest + + Directory to work in. It has to be a fully qualified path. diff --git a/roles/process-stackviz/defaults/main.yaml b/roles/process-stackviz/defaults/main.yaml new file mode 100644 index 0000000000..f3bc32b149 --- /dev/null +++ b/roles/process-stackviz/defaults/main.yaml @@ -0,0 +1,3 @@ +devstack_base_dir: /opt/stack +stage_dir: "{{ ansible_user_dir }}" +zuul_work_dir: "{{ devstack_base_dir }}/tempest" diff --git a/roles/process-stackviz/tasks/main.yaml b/roles/process-stackviz/tasks/main.yaml new file mode 100644 index 0000000000..c51c66cdb3 --- /dev/null +++ b/roles/process-stackviz/tasks/main.yaml @@ -0,0 +1,70 @@ +- name: Devstack checks if stackviz archive exists + stat: + path: "/opt/cache/files/stackviz-latest.tar.gz" + register: stackviz_archive + +- debug: + msg: "Stackviz archive could not be found in /opt/cache/files/stackviz-latest.tar.gz" + when: not stackviz_archive.stat.exists + +- name: Check if subunit data exists + stat: + path: "{{ zuul_work_dir }}/testrepository.subunit" + register: subunit_input + +- debug: + msg: "Subunit file could not be found at {{ zuul_work_dir }}/testrepository.subunit" + when: not subunit_input.stat.exists + +- name: Install stackviz + when: + - stackviz_archive.stat.exists + - subunit_input.stat.exists + block: + - include_role: + name: ensure-pip + + - pip: + name: "file://{{ stackviz_archive.stat.path }}" + virtualenv: /tmp/stackviz + virtualenv_command: '{{ ensure_pip_virtualenv_command }}' + extra_args: -U + +- name: Deploy stackviz static html+js + command: cp -pR /tmp/stackviz/share/stackviz-html {{ stage_dir }}/stackviz + when: + - stackviz_archive.stat.exists + - subunit_input.stat.exists + +- name: Check if dstat data exists + stat: + path: "{{ devstack_base_dir }}/logs/dstat-csv.log" + register: dstat_input + when: + - stackviz_archive.stat.exists + - subunit_input.stat.exists + +- name: Run stackviz with dstat + shell: | + cat {{ subunit_input.stat.path }} | \ + /tmp/stackviz/bin/stackviz-export \ + --dstat "{{ devstack_base_dir }}/logs/dstat-csv.log" \ + --env --stdin \ + {{ stage_dir }}/stackviz/data + when: + - stackviz_archive.stat.exists + - subunit_input.stat.exists + - dstat_input.stat.exists + failed_when: False + +- name: Run stackviz without dstat + shell: | + cat {{ subunit_input.stat.path }} | \ + /tmp/stackviz/bin/stackviz-export \ + --env --stdin \ + {{ stage_dir }}/stackviz/data + when: + - stackviz_archive.stat.exists + - subunit_input.stat.exists + - not dstat_input.stat.exists + failed_when: False From 1b35581bb096883ceafbfeea286153eaec184c17 Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Mon, 31 Aug 2020 16:22:57 +1000 Subject: [PATCH 25/45] Use SETUPTOOLS_USE_DISTUTILS=stdlib for global pip installs A new setuptools release has changed the way pip installs are done, see [0]. With this change we switch back to using the distro method for global pip installs. Temporarily make grenade jobs non-voting in order to allow this patch to be backported. [0] http://lists.openstack.org/pipermail/openstack-discuss/2020-August/016905.html Change-Id: I5d8aa0e58e0409c54451b51de5eb70ba9a68d849 (cherry picked from commit 18b4251bf4f689923a19bf7fbc50d5c2ea422b21) (cherry picked from commit 87d1da30a86a6ae3525eca9a5842ab0403e6ea56) --- inc/python | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/inc/python b/inc/python index 81b6a960a4..286e5d4f80 100644 --- a/inc/python +++ b/inc/python @@ -247,7 +247,13 @@ function pip_install { # and we will let pip sort out the install, regardless of # the package being local or remote. echo "Using $PYTHON3_VERSION version to install $package_dir based on default behavior" - sudo_pip="$sudo_pip LC_ALL=en_US.UTF-8" + # See + # https://github.com/pypa/setuptools/issues/2232 + # http://lists.openstack.org/pipermail/openstack-discuss/2020-August/016905.html + # this makes setuptools >=50 use the platform distutils. + # We only want to do this on global pip installs, not if + # installing in a virtualenv + sudo_pip="$sudo_pip LC_ALL=en_US.UTF-8 SETUPTOOLS_USE_DISTUTILS=stdlib " cmd_pip=$(get_pip_command $PYTHON3_VERSION) fi fi From 4bff24a204f9a976b3ac51ed5f64bf699246a2f7 Mon Sep 17 00:00:00 2001 From: Luigi Toscano Date: Fri, 10 Jul 2020 13:21:45 +0200 Subject: [PATCH 26/45] zuul: clean up some legacy jobs Replace legacy-tempest-dsvm-lvm-multibackend with its native version, cinder-tempest-lvm-multibackend. Remove legacy-tempest-dsvm-neutron-dvr-multinode-full, which was defined as an experimental job in neutron and removed during the ussuri lifecycle. See https://review.opendev.org/#/c/693630/ Change-Id: I76d1efaa3a6c1fe9825e8457438f514114b2ecad (cherry picked from commit 0cbbefc0cb6f58b09a722d1230e7ae86113b2989) (cherry picked from commit c69caa4a8f3e95ba64f8df88eebb9a69777396fa) --- .zuul.yaml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.zuul.yaml b/.zuul.yaml index b7f6dcc354..5cd6da98fd 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -719,15 +719,11 @@ irrelevant-files: - ^.*\.rst$ - ^doc/.*$ - - legacy-tempest-dsvm-neutron-dvr-multinode-full: - irrelevant-files: - - ^.*\.rst$ - - ^doc/.*$ - neutron-tempest-dvr-ha-multinode-full: irrelevant-files: - ^.*\.rst$ - ^doc/.*$ - - legacy-tempest-dsvm-lvm-multibackend: + - cinder-tempest-lvm-multibackend: irrelevant-files: - ^.*\.rst$ - ^doc/.*$ From fc02fb441578179e7cb4eba00563554da3e435ac Mon Sep 17 00:00:00 2001 From: Jens Harbott Date: Mon, 12 Oct 2020 11:13:25 +0200 Subject: [PATCH 27/45] Increase swap space size again Infra changed the default swap size from 8G to 1G, this breaks some stable jobs. Revert to 8G for our jobs. Change-Id: I4fb2372be7d861f09f365e96478b4663847a66a6 --- .zuul.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.zuul.yaml b/.zuul.yaml index b7f6dcc354..74330b582d 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -193,6 +193,7 @@ - zuul: opendev.org/openstack/devstack-gate - zuul: opendev.org/openstack/openstack-zuul-jobs vars: + configure_swap_size: 8192 devstack_localrc: DATABASE_PASSWORD: secretdatabase RABBIT_PASSWORD: secretrabbit From ecf59118d73d49eee130e810ea742c0191ae1775 Mon Sep 17 00:00:00 2001 From: Lee Yarwood Date: Mon, 21 Dec 2020 19:05:55 +0000 Subject: [PATCH 28/45] Stop installing test-requirements with projects This is a test of installing openstack and then seeing if it works. OpenStack components do not need test-requirements to operate, that's why they are test-requirements. Additionally, as we look forward to depsolver pip, this is going to screw us because we don't apply constraints to linters, which are expressed in - you guessed it, test-requirements. NOTE(lyarwood): openstacksdk-functional is currently failing due to I1e151454f39d51db52cdc7a23dbdcbf9b28c9381 not being present in stable/train. Unfortunatley the backport of this fix is held up by multiple other failures in other projects caused by the new pip dependency resolver behaviour. To fix these we first need to land I8f24b839bf42e2fb9803dc7df3a30ae20cf264eb in devstack on stable/train, allowing fixes to land in these projects that should eventually fix openstacksdk. To allow this we need to temporarily mark the openstacksdk-functional job as non-voting. Change-Id: I8f24b839bf42e2fb9803dc7df3a30ae20cf264eb (cherry picked from commit 09b5b05c471985ad53c4b321e993101f007ee915) --- .zuul.yaml | 9 +++++---- inc/python | 9 +-------- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/.zuul.yaml b/.zuul.yaml index b7f6dcc354..fcdfd0442a 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -645,6 +645,7 @@ irrelevant-files: - ^.*\.rst$ - ^doc/.*$ + voting: false - tempest-ipv6-only: irrelevant-files: - ^.*\.rst$ @@ -674,10 +675,10 @@ # irrelevant-files: # - ^.*\.rst$ # - ^doc/.*$ - - openstacksdk-functional-devstack: - irrelevant-files: - - ^.*\.rst$ - - ^doc/.*$ + # - openstacksdk-functional-devstack: + # irrelevant-files: + # - ^.*\.rst$ + # - ^doc/.*$ - tempest-ipv6-only: irrelevant-files: - ^.*\.rst$ diff --git a/inc/python b/inc/python index 286e5d4f80..ed25fab23a 100644 --- a/inc/python +++ b/inc/python @@ -276,13 +276,6 @@ function pip_install { $xtrace - # Also install test requirements - local install_test_reqs="" - local test_req="${package_dir}/test-requirements.txt" - if [[ -e "$test_req" ]]; then - install_test_reqs="-r $test_req" - fi - # adding SETUPTOOLS_SYS_PATH_TECHNIQUE is a workaround to keep # the same behaviour of setuptools before version 25.0.0. # related issue: https://github.com/pypa/pip/issues/3874 @@ -292,7 +285,7 @@ function pip_install { no_proxy="${no_proxy:-}" \ PIP_FIND_LINKS=$PIP_FIND_LINKS \ SETUPTOOLS_SYS_PATH_TECHNIQUE=rewrite \ - $cmd_pip $upgrade $install_test_reqs \ + $cmd_pip $upgrade \ $@ result=$? From b708f6ea0ac78d9dcc8b468854d1f3890c6c5d23 Mon Sep 17 00:00:00 2001 From: Carlos Goncalves Date: Sat, 1 Aug 2020 21:47:55 +0200 Subject: [PATCH 29/45] Fix is_fedora RHEL 8 detection While RHEL 7 is detected as RedHatEnterpriseServer, RHEL 8 is RedHatEnterprise. $ lsb_release -i -s RedHatEnterprise Change-Id: I3d750d808c6ebea9c230f0508cdbc016415b9922 (cherry picked from commit 587e0a3510ea3b40abbdffcd1fe3204e416c4bc9) --- functions-common | 1 + 1 file changed, 1 insertion(+) diff --git a/functions-common b/functions-common index a13d611415..dc448d306f 100644 --- a/functions-common +++ b/functions-common @@ -451,6 +451,7 @@ function is_fedora { [ "$os_VENDOR" = "Fedora" ] || [ "$os_VENDOR" = "Red Hat" ] || \ [ "$os_VENDOR" = "RedHatEnterpriseServer" ] || \ + [ "$os_VENDOR" = "RedHatEnterprise" ] || \ [ "$os_VENDOR" = "CentOS" ] || [ "$os_VENDOR" = "OracleServer" ] || \ [ "$os_VENDOR" = "Virtuozzo" ] } From a7c4377c17b6ca4a73a786af9cc6b1c502cd8061 Mon Sep 17 00:00:00 2001 From: Ghanshyam Mann Date: Tue, 22 Dec 2020 12:23:56 -0600 Subject: [PATCH 30/45] Install swift keystone extras requirements Since the introduction of I8f24b839bf42e2fb9803dc7df3a30ae20cf264 s-proxy is no longer able to launch as keystonemiddleware (listed under test-requirements.txt) has not been installed. keystonemiddleware is listed as extras requirements in swift - https://github.com/openstack/swift/blob/e0d46d77fa740768f1dd5b989a63be85ff1fec20/setup.cfg#L79 Let's install swift keystone extra requirements also. Closes-Bug: #1909018 Change-Id: I02c692e95d70017eea03d82d75ae6c5e87bde8b1 (cherry picked from commit 04b0b61557f7dad6c32b566255c21a36e4b0aefa) (cherry picked from commit 90651cb1a914999afd1dd1e4929c04e62fefb4c8) (cherry picked from commit 10fdf258bacf4702917f1bbdffef933e7a89907a) --- lib/swift | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/swift b/lib/swift index 5be9e3575e..6afe5cc7f8 100644 --- a/lib/swift +++ b/lib/swift @@ -738,7 +738,9 @@ function init_swift { function install_swift { git_clone $SWIFT_REPO $SWIFT_DIR $SWIFT_BRANCH - setup_develop $SWIFT_DIR + # keystonemiddleware needs to be installed via keystone extras as defined + # in setup.cfg, see bug #1909018 for more details. + setup_develop $SWIFT_DIR keystone if [ "$SWIFT_USE_MOD_WSGI" == "True" ]; then install_apache_wsgi fi From d82889d477f690c9811fba2c7d0b0f768e954c5f Mon Sep 17 00:00:00 2001 From: Martin Kopec Date: Thu, 12 Mar 2020 10:03:38 +0000 Subject: [PATCH 31/45] Remove tempest deprecated img_dir option Tempest option scenario.img_dir has been deprecated more than 4 years, it's time to remove it from devstack. img_file option should contain the full path to the image. This patch removes setting of img_dir option and makes img_file one contain the full path of an image. Change-Id: I71102095f3603915f0bc7d21f2e18c4eac4e95ec Related-Bug: #1393881 --- lib/tempest | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/tempest b/lib/tempest index b05753e8a0..427a7a1fe7 100644 --- a/lib/tempest +++ b/lib/tempest @@ -433,8 +433,7 @@ function configure_tempest { SCENARIO_IMAGE_DIR=${SCENARIO_IMAGE_DIR:-$FILES} SCENARIO_IMAGE_FILE=$DEFAULT_IMAGE_FILE_NAME fi - iniset $TEMPEST_CONFIG scenario img_dir $SCENARIO_IMAGE_DIR - iniset $TEMPEST_CONFIG scenario img_file $SCENARIO_IMAGE_FILE + iniset $TEMPEST_CONFIG scenario img_file $SCENARIO_IMAGE_DIR/$SCENARIO_IMAGE_FILE # If using provider networking, use the physical network for validation rather than private TEMPEST_SSH_NETWORK_NAME=$PRIVATE_NETWORK_NAME From 2f573b79bee3cc581ffc6b510924b24ddebe9f7b Mon Sep 17 00:00:00 2001 From: elajkat Date: Tue, 12 Jan 2021 13:14:25 +0100 Subject: [PATCH 32/45] Setting available_features for stable branch available_features for Neutron tempest tests is default to "all" and used from Victoria to enable/disable tests that has no API extension. It is added here to make sure that stable gating will not execute these tests (like for IPv6 metadata in neutron-tempest-plugin or updating min_bw QoS values for port already attached to VMs) Cherry-picked from stable/victoria patch (https://review.opendev.org/c/openstack/devstack/+/769885), but the list for available_features is empty on Ussuri and Train. Change-Id: Ic6056218e51e6b0da554bcdf139214d2229ba3c1 Related-Bug: #1882804 (cherry picked from commit e34a8720bfb263a0b4d585929ef9f035c3ba0f36) (cherry picked from commit ef784e195f85fe4ab28eb0c115e49a63cb3db76a) --- lib/tempest | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/lib/tempest b/lib/tempest index b05753e8a0..bccd9a9ec8 100644 --- a/lib/tempest +++ b/lib/tempest @@ -684,6 +684,17 @@ function configure_tempest { object_storage_api_extensions=$(remove_disabled_extensions $object_storage_api_extensions $DISABLE_STORAGE_API_EXTENSIONS) fi iniset $TEMPEST_CONFIG object-storage-feature-enabled discoverable_apis $object_storage_api_extensions + + # This feature list for Neutron used from Victoria, the aim is to skip tests + # for features in Neutron that has no API extension, or can't be detected + # otherwise from the API. + # It is added here to avoid stable gating to run these tests (like for IPv6 metadata + # in neutron-tempest-plugin or updating min_bw QoS values for ports already attached + # to VMs). + # To check new features grep neutron-tempest-plugin and tempest for + # is_network_feature_enabled, and please check with Neutron team. + local network_available_features="" + iniset $TEMPEST_CONFIG network-feature-enabled available_features $network_available_features # Cinder API Extensions DEFAULT_VOL_EXT="OS-SCH-HNT,backups,capabilities,cgsnapshots,consistencygroups" DEFAULT_VOL_EXT+=",encryption,os-admin-actions,os-availability-zone" From 469b26f74288b00f3b6128f3a260046ea51c8a43 Mon Sep 17 00:00:00 2001 From: Akihiro Motoki Date: Tue, 26 Jan 2021 09:47:13 +0900 Subject: [PATCH 33/45] Use a dedicated version of get-pip.py for python 2.7 get-pip.py dropped python 2.7 support, so we cannot use the default get-pip.py for python 2.7 now. get-pip.py for python 2.7 is available at https://bootstrap.pypa.io/2.7/get-pip.py. This commit tries to use python 2.7 version of get-pip.py for python 2.7 environment. Depends-On: https://review.opendev.org/772486 Change-Id: I88ba370de8887a05c473fe7c19a61a319e8812d2 (cherry picked from commit 1382e933c5b2c16b276e8813fbc7cd9a1d6d4499) --- tools/install_pip.sh | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/tools/install_pip.sh b/tools/install_pip.sh index 2b6aa4c2e8..04cb0f6133 100755 --- a/tools/install_pip.sh +++ b/tools/install_pip.sh @@ -38,7 +38,9 @@ FILES=$TOP_DIR/files # [1] https://opendev.org/openstack/project-config/src/branch/master/nodepool/elements/cache-devstack/source-repository-pip PIP_GET_PIP_URL=${PIP_GET_PIP_URL:-"https://bootstrap.pypa.io/get-pip.py"} +PIP_GET_PIP_PY27_URL=${PIP_GET_PIP_PY27_URL:-"https://bootstrap.pypa.io/2.7/get-pip.py"} LOCAL_PIP="$FILES/$(basename $PIP_GET_PIP_URL)" +LOCAL_PIP_PY27="${LOCAL_PIP}-py27" GetDistro echo "Distro: $DISTRO" @@ -56,13 +58,17 @@ function get_versions { } -function install_get_pip { +function _install_get_pip { + local interpreter=$1 + local pip_get_pip_url=$2 + local local_pip=$3 + # If get-pip.py isn't python, delete it. This was probably an # outage on the server. - if [[ -r $LOCAL_PIP ]]; then - if ! head -1 $LOCAL_PIP | grep -q '#!/usr/bin/env python'; then - echo "WARNING: Corrupt $LOCAL_PIP found removing" - rm $LOCAL_PIP + if [[ -r $local_pip ]]; then + if ! head -1 $local_pip | grep -q '#!/usr/bin/env python'; then + echo "WARNING: Corrupt $local_pip found removing" + rm $local_pip fi fi @@ -76,22 +82,27 @@ function install_get_pip { # Thus we use curl's "-z" feature to always check the modified # since and only download if a new version is out -- but only if # it seems we downloaded the file originally. - if [[ ! -r $LOCAL_PIP || -r $LOCAL_PIP.downloaded ]]; then + if [[ ! -r $local_pip || -r $local_pip.downloaded ]]; then # only test freshness if LOCAL_PIP is actually there, # otherwise we generate a scary warning. local timecond="" - if [[ -r $LOCAL_PIP ]]; then - timecond="-z $LOCAL_PIP" + if [[ -r $local_pip ]]; then + timecond="-z $local_pip" fi curl -f --retry 6 --retry-delay 5 \ - $timecond -o $LOCAL_PIP $PIP_GET_PIP_URL || \ + $timecond -o $local_pip $pip_get_pip_url || \ die $LINENO "Download of get-pip.py failed" - touch $LOCAL_PIP.downloaded + touch $local_pip.downloaded fi - sudo -H -E python $LOCAL_PIP -c $TOOLS_DIR/cap-pip.txt + sudo -H -E $interpreter $local_pip -c $TOOLS_DIR/cap-pip.txt +} + + +function install_get_pip { + _install_get_pip python $PIP_GET_PIP_PY27_URL $LOCAL_PIP_PY27 if python3_enabled; then - sudo -H -E python${PYTHON3_VERSION} $LOCAL_PIP -c $TOOLS_DIR/cap-pip.txt + _install_get_pip python${PYTHON3_VERSION} $PIP_GET_PIP_URL $LOCAL_PIP fi } From 812efb3ca7faea6e477c0b4c2dbe16081f7880a9 Mon Sep 17 00:00:00 2001 From: Ghanshyam Mann Date: Tue, 26 Jan 2021 15:04:53 -0600 Subject: [PATCH 34/45] Fix get-pip for python 3.5 get-pip.py is fixed for py2.7 in https://review.opendev.org/c/openstack/devstack/+/772462 This commit support it for py3.5 as devstack stable/train and stein still support ubuntu xenial which has py3.5 Also using Temepst 23.0.0 for xenial platform job which is compatible version on py3.5. Closes-Bug: #1913283 Change-Id: Icd40e41b5f3ef9a621ca1b9739875fcdd31abd74 --- .zuul.yaml | 4 ++++ tools/install_pip.sh | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/.zuul.yaml b/.zuul.yaml index 1b396951b7..bec9d7b0d1 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -530,6 +530,10 @@ parent: tempest-full-py3 description: Ubuntu Xenial platform test nodeset: openstack-single-node-xenial + vars: + devstack_localrc: + # use py3.5 compatible Tempest here. + TEMPEST_BRANCH: "23.0.0" voting: false - job: diff --git a/tools/install_pip.sh b/tools/install_pip.sh index 04cb0f6133..2901ba8507 100755 --- a/tools/install_pip.sh +++ b/tools/install_pip.sh @@ -102,6 +102,10 @@ function _install_get_pip { function install_get_pip { _install_get_pip python $PIP_GET_PIP_PY27_URL $LOCAL_PIP_PY27 if python3_enabled; then + if [[ "$PYTHON3_VERSION" == "3.5" ]]; then + PIP_GET_PIP_URL=$(dirname $PIP_GET_PIP_URL)/3.5/$(basename $PIP_GET_PIP_URL) + LOCAL_PIP=${LOCAL_PIP}-py35 + fi _install_get_pip python${PYTHON3_VERSION} $PIP_GET_PIP_URL $LOCAL_PIP fi } From 3c92daf73491cfeb77c978eb73c3a78ce0ce7bae Mon Sep 17 00:00:00 2001 From: Ghanshyam Mann Date: Tue, 9 Feb 2021 12:56:34 -0600 Subject: [PATCH 35/45] Add a variable to configure the Tempest venv upper constraints We use Tempest master for testing the supported stable branches so using master upper constraints works fine but when we need to use old Tempest in the below cases then master upper constraints do not work and devstack will not be able to install Tempest in vnenv: - Testing Extended Maintenance branch - Testing py2.7 jobs until stable/train with in-tree tempest plugins This commit adds a variable to set the compatible upper constraint to use for Tempest's old version. Few of the current failure which can be fixed by this new configurable var: - networking-generic-switch-tempest-dlm-python2 - https://zuul.opendev.org/t/openstack/build/ebcf3d68d62c4af3a43a222aa9ce5556 - devstack-platform-xenial on stable/steinand stable/train - https://zuul.opendev.org/t/openstack/build/37ffc1af6f3f4b44b5ca8cbfa27068ac Change-Id: I5b2217d85e6871ca3f7a3f6f859fdce9a50d3946 (cherry picked from commit 3bdc8f66ad243f7487ba494e6a71f63c4965413a) (cherry picked from commit 2e4b708e255135fe72537b5d37520555248dec6e) (cherry picked from commit 74a136ac9820a9282f9d75178b4dea40256565c0) --- lib/tempest | 38 +++++++++++++++++++++++++++++++------- stackrc | 1 + 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/lib/tempest b/lib/tempest index 78de2d378e..cfefc197c1 100644 --- a/lib/tempest +++ b/lib/tempest @@ -110,6 +110,21 @@ function image_size_in_gib { echo $size | python -c "import math; import six; print(int(math.ceil(float(int(six.moves.input()) / 1024.0 ** 3))))" } +function set_tempest_venv_constraints { + local tmp_c + tmp_c=$1 + if [[ $TEMPEST_VENV_UPPER_CONSTRAINTS == "master" ]]; then + (cd $REQUIREMENTS_DIR && git show origin/master:upper-constraints.txt) > $tmp_c + else + echo "Using $TEMPEST_VENV_UPPER_CONSTRAINTS constraints in Tempest virtual env." + cat $TEMPEST_VENV_UPPER_CONSTRAINTS > $tmp_c + # NOTE: setting both tox env var and once Tempest start using new var + # TOX_CONSTRAINTS_FILE then we can remove the old one. + export UPPER_CONSTRAINTS_FILE=$TEMPEST_VENV_UPPER_CONSTRAINTS + export TOX_CONSTRAINTS_FILE=$TEMPEST_VENV_UPPER_CONSTRAINTS + fi +} + # configure_tempest() - Set config files, create data dirs, etc function configure_tempest { if [[ "$INSTALL_TEMPEST" == "True" ]]; then @@ -610,9 +625,11 @@ function configure_tempest { tox -revenv-tempest --notest fi - # The requirements might be on a different branch, while tempest needs master requirements. - (cd $REQUIREMENTS_DIR && git show origin/master:upper-constraints.txt) > u-c-m.txt - tox -evenv-tempest -- pip install -c u-c-m.txt -r requirements.txt + local tmp_u_c_m + tmp_u_c_m=$(mktemp -t tempest_u_c_m.XXXXXXXXXX) + set_tempest_venv_constraints $tmp_u_c_m + tox -evenv-tempest -- pip install -c $tmp_u_c_m -r requirements.txt + rm -f $tmp_u_c_m # Auth: iniset $TEMPEST_CONFIG auth tempest_roles "member" @@ -729,12 +746,17 @@ function install_tempest { # TEMPEST_DIR already exist until RECLONE is true. git checkout $TEMPEST_BRANCH + local tmp_u_c_m + tmp_u_c_m=$(mktemp -t tempest_u_c_m.XXXXXXXXXX) + set_tempest_venv_constraints $tmp_u_c_m + tox -r --notest -efull # NOTE(mtreinish) Respect constraints in the tempest full venv, things that # are using a tox job other than full will not be respecting constraints but # running pip install -U on tempest requirements - $TEMPEST_DIR/.tox/tempest/bin/pip install -c $REQUIREMENTS_DIR/upper-constraints.txt -r requirements.txt + $TEMPEST_DIR/.tox/tempest/bin/pip install -c $tmp_u_c_m -r requirements.txt PROJECT_VENV["tempest"]=${TEMPEST_DIR}/.tox/tempest + rm -f $tmp_u_c_m popd } @@ -742,9 +764,11 @@ function install_tempest { function install_tempest_plugins { pushd $TEMPEST_DIR if [[ $TEMPEST_PLUGINS != 0 ]] ; then - # The requirements might be on a different branch, while tempest & tempest plugins needs master requirements. - (cd $REQUIREMENTS_DIR && git show origin/master:upper-constraints.txt) > u-c-m.txt - tox -evenv-tempest -- pip install -c u-c-m.txt $TEMPEST_PLUGINS + local tmp_u_c_m + tmp_u_c_m=$(mktemp -t tempest_u_c_m.XXXXXXXXXX) + set_tempest_venv_constraints $tmp_u_c_m + tox -evenv-tempest -- pip install -c $tmp_u_c_m $TEMPEST_PLUGINS + rm -f $tmp_u_c_m echo "Checking installed Tempest plugins:" tox -evenv-tempest -- tempest list-plugins fi diff --git a/stackrc b/stackrc index 8b2a90d552..a4d2600b09 100644 --- a/stackrc +++ b/stackrc @@ -315,6 +315,7 @@ REQUIREMENTS_BRANCH=${REQUIREMENTS_BRANCH:-$TARGET_BRANCH} # Tempest test suite TEMPEST_REPO=${TEMPEST_REPO:-${GIT_BASE}/openstack/tempest.git} TEMPEST_BRANCH=${TEMPEST_BRANCH:-$BRANCHLESS_TARGET_BRANCH} +TEMPEST_VENV_UPPER_CONSTRAINTS=${TEMPEST_VENV_UPPER_CONSTRAINTS:-master} ############## From fc7f929597816555490836d21c25a8adb725d2af Mon Sep 17 00:00:00 2001 From: Dan Smith Date: Sun, 7 Mar 2021 20:42:10 -0800 Subject: [PATCH 36/45] Fix get-pip URL for py27 This has been failing to install with this message, which I think means we need to change our URL for getting pip: The URL you are using to fetch this script has changed, and this one will no longer work. Please use get-pip.py from the following URL instead: https://bootstrap.pypa.io/pip/2.7/get-pip.py Sorry if this change causes any inconvenience for you! Change-Id: Id62e91b1609db4b1d2fa425010bac1ce77e9fc51 --- tools/install_pip.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/install_pip.sh b/tools/install_pip.sh index 2901ba8507..1f09141e20 100755 --- a/tools/install_pip.sh +++ b/tools/install_pip.sh @@ -38,7 +38,7 @@ FILES=$TOP_DIR/files # [1] https://opendev.org/openstack/project-config/src/branch/master/nodepool/elements/cache-devstack/source-repository-pip PIP_GET_PIP_URL=${PIP_GET_PIP_URL:-"https://bootstrap.pypa.io/get-pip.py"} -PIP_GET_PIP_PY27_URL=${PIP_GET_PIP_PY27_URL:-"https://bootstrap.pypa.io/2.7/get-pip.py"} +PIP_GET_PIP_PY27_URL=${PIP_GET_PIP_PY27_URL:-"https://bootstrap.pypa.io/pip/2.7/get-pip.py"} LOCAL_PIP="$FILES/$(basename $PIP_GET_PIP_URL)" LOCAL_PIP_PY27="${LOCAL_PIP}-py27" From 16da65c30272b45bf6e138adb68d3cfcf80813e6 Mon Sep 17 00:00:00 2001 From: Ghanshyam Mann Date: Thu, 8 Apr 2021 11:03:37 -0500 Subject: [PATCH 37/45] Make stackviz tasks not to fail jobs Due to issue on stckviz side, job start failing with POST_FAILURE. If we fix the issue still we need to wait for periodic job periodic-package-stackviz-element to publish the latest tarball on https://tarballs.openstack.org/stackviz/dist/. Let's not fail the job for any issue occur during stackviz processing. Closes-Bug: 1863161 Depends-On: https://review.opendev.org/c/openstack/grenade/+/785767 Change-Id: Ifee04f28ecee52e74803f1623aba5cfe5ee5ec90 (cherry picked from commit 580fec54c3a970de80ab66b3decca69704ff1179) (cherry picked from commit 800eb4dd446f93734d420005f93d0dcb88564dd4) (cherry picked from commit 3b2feba2ef6a08f639b36bfe995705ec3a785ff8) (cherry picked from commit 4f9c1e084c8d762c873549e8fc9524d6ee24f1c1) --- roles/process-stackviz/tasks/main.yaml | 125 +++++++++++++------------ 1 file changed, 64 insertions(+), 61 deletions(-) diff --git a/roles/process-stackviz/tasks/main.yaml b/roles/process-stackviz/tasks/main.yaml index c51c66cdb3..3ba3d9c2e6 100644 --- a/roles/process-stackviz/tasks/main.yaml +++ b/roles/process-stackviz/tasks/main.yaml @@ -1,70 +1,73 @@ -- name: Devstack checks if stackviz archive exists - stat: - path: "/opt/cache/files/stackviz-latest.tar.gz" - register: stackviz_archive +- name: Process Stackviz + block: -- debug: - msg: "Stackviz archive could not be found in /opt/cache/files/stackviz-latest.tar.gz" - when: not stackviz_archive.stat.exists + - name: Devstack checks if stackviz archive exists + stat: + path: "/opt/cache/files/stackviz-latest.tar.gz" + register: stackviz_archive -- name: Check if subunit data exists - stat: - path: "{{ zuul_work_dir }}/testrepository.subunit" - register: subunit_input + - debug: + msg: "Stackviz archive could not be found in /opt/cache/files/stackviz-latest.tar.gz" + when: not stackviz_archive.stat.exists -- debug: - msg: "Subunit file could not be found at {{ zuul_work_dir }}/testrepository.subunit" - when: not subunit_input.stat.exists + - name: Check if subunit data exists + stat: + path: "{{ zuul_work_dir }}/testrepository.subunit" + register: subunit_input -- name: Install stackviz - when: - - stackviz_archive.stat.exists - - subunit_input.stat.exists - block: - - include_role: - name: ensure-pip + - debug: + msg: "Subunit file could not be found at {{ zuul_work_dir }}/testrepository.subunit" + when: not subunit_input.stat.exists + + - name: Install stackviz + when: + - stackviz_archive.stat.exists + - subunit_input.stat.exists + block: + - include_role: + name: ensure-pip + + - pip: + name: "file://{{ stackviz_archive.stat.path }}" + virtualenv: /tmp/stackviz + virtualenv_command: '{{ ensure_pip_virtualenv_command }}' + extra_args: -U - - pip: - name: "file://{{ stackviz_archive.stat.path }}" - virtualenv: /tmp/stackviz - virtualenv_command: '{{ ensure_pip_virtualenv_command }}' - extra_args: -U + - name: Deploy stackviz static html+js + command: cp -pR /tmp/stackviz/share/stackviz-html {{ stage_dir }}/stackviz + when: + - stackviz_archive.stat.exists + - subunit_input.stat.exists -- name: Deploy stackviz static html+js - command: cp -pR /tmp/stackviz/share/stackviz-html {{ stage_dir }}/stackviz - when: - - stackviz_archive.stat.exists - - subunit_input.stat.exists + - name: Check if dstat data exists + stat: + path: "{{ devstack_base_dir }}/logs/dstat-csv.log" + register: dstat_input + when: + - stackviz_archive.stat.exists + - subunit_input.stat.exists -- name: Check if dstat data exists - stat: - path: "{{ devstack_base_dir }}/logs/dstat-csv.log" - register: dstat_input - when: - - stackviz_archive.stat.exists - - subunit_input.stat.exists + - name: Run stackviz with dstat + shell: | + cat {{ subunit_input.stat.path }} | \ + /tmp/stackviz/bin/stackviz-export \ + --dstat "{{ devstack_base_dir }}/logs/dstat-csv.log" \ + --env --stdin \ + {{ stage_dir }}/stackviz/data + when: + - stackviz_archive.stat.exists + - subunit_input.stat.exists + - dstat_input.stat.exists -- name: Run stackviz with dstat - shell: | - cat {{ subunit_input.stat.path }} | \ - /tmp/stackviz/bin/stackviz-export \ - --dstat "{{ devstack_base_dir }}/logs/dstat-csv.log" \ - --env --stdin \ - {{ stage_dir }}/stackviz/data - when: - - stackviz_archive.stat.exists - - subunit_input.stat.exists - - dstat_input.stat.exists - failed_when: False + - name: Run stackviz without dstat + shell: | + cat {{ subunit_input.stat.path }} | \ + /tmp/stackviz/bin/stackviz-export \ + --env --stdin \ + {{ stage_dir }}/stackviz/data + when: + - stackviz_archive.stat.exists + - subunit_input.stat.exists + - not dstat_input.stat.exists -- name: Run stackviz without dstat - shell: | - cat {{ subunit_input.stat.path }} | \ - /tmp/stackviz/bin/stackviz-export \ - --env --stdin \ - {{ stage_dir }}/stackviz/data - when: - - stackviz_archive.stat.exists - - subunit_input.stat.exists - - not dstat_input.stat.exists - failed_when: False + ignore_errors: yes From 4cf48714b6e8f89c2530c88bcb0d7608ec5da537 Mon Sep 17 00:00:00 2001 From: Lucas Alvares Gomes Date: Tue, 11 May 2021 11:04:32 +0100 Subject: [PATCH 38/45] Fix docs job Sphinx 4.0.0 added a new dependency [0] which is causing the job to fail at the moment. This patch fix the problem by adding UC to the docs jobs. [0] https://www.sphinx-doc.org/en/master/changes.html (LaTeX: add tex-gyre font dependency) Change-Id: I28019331017405c06577ada88f8e9f6d9a2afc23 Signed-off-by: Lucas Alvares Gomes (cherry picked from commit 69a66fb62bcb77145b6eec21fc2d56d40a861d0d) --- tox.ini | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 26baa2a1c5..45808d1aee 100644 --- a/tox.ini +++ b/tox.ini @@ -36,7 +36,9 @@ commands = bash -c "find {toxinidir} \ [testenv:docs] basepython = python3 -deps = -r{toxinidir}/doc/requirements.txt +deps = + -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/train} + -r{toxinidir}/doc/requirements.txt whitelist_externals = bash setenv = TOP_DIR={toxinidir} From ae035e739c6f05ce29c8c59ba6de843fe1fa37bd Mon Sep 17 00:00:00 2001 From: Ghanshyam Mann Date: Tue, 25 May 2021 13:06:14 -0500 Subject: [PATCH 39/45] Pin nodeset for unit test job devstack unit test job does not set any nodeset and so does use default nodeset defined in base jobs in opendev. When opendev switches the default nodeset to the latest distro version, devstack unit test job can start failing. Example: - https://review.opendev.org/q/I01408f2f2959b0788fe712ac268a526502226ee9 - https://review.opendev.org/q/Ib1ea47bc7384e1f579cb08c779a32151fccd6845 To avoid such a situation in future, let's set the working nodeset for this job also so that when we cut the stable branch we can run it on the working distro version. Change-Id: I302140778fedf08bc8ba72c453837fa7b8f8f9ae --- .zuul.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.zuul.yaml b/.zuul.yaml index 05680a3104..4f15656c6d 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -592,6 +592,7 @@ - job: name: devstack-unit-tests + nodeset: ubuntu-bionic description: | Runs unit tests on devstack project. From 224fee25a351afed8a0eb605db58bc6bdbf37609 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Piliszek?= Date: Mon, 7 Jun 2021 17:44:54 +0000 Subject: [PATCH 40/45] Move verify-ipv6-only-deployments from Tempest to DevStack as it tests DevStack side of things and is useful for projects not using Tempest. Verbatim copy except for the devstack- prefix and the /devstack/ path. Change-Id: Ie166730843f874b9c99e37244e460d7ad33b7eeb (cherry picked from commit 2fb8c7a5eea6f9321eef36fb9b8fd8e55465c91e) --- .../README.rst | 16 ++++ .../defaults/main.yaml | 1 + .../tasks/main.yaml | 4 + tools/verify-ipv6-only-deployments.sh | 92 +++++++++++++++++++ 4 files changed, 113 insertions(+) create mode 100644 roles/devstack-ipv6-only-deployments-verification/README.rst create mode 100644 roles/devstack-ipv6-only-deployments-verification/defaults/main.yaml create mode 100644 roles/devstack-ipv6-only-deployments-verification/tasks/main.yaml create mode 100755 tools/verify-ipv6-only-deployments.sh diff --git a/roles/devstack-ipv6-only-deployments-verification/README.rst b/roles/devstack-ipv6-only-deployments-verification/README.rst new file mode 100644 index 0000000000..400a8da222 --- /dev/null +++ b/roles/devstack-ipv6-only-deployments-verification/README.rst @@ -0,0 +1,16 @@ +Verify the IPv6-only deployments + +This role needs to be invoked from a playbook that +run tests. This role verifies the IPv6 setting on +devstack side and devstack deploy services on IPv6. +This role is invoked before tests are run so that +if any missing IPv6 setting or deployments can fail +the job early. + + +**Role Variables** + +.. zuul:rolevar:: devstack_base_dir + :default: /opt/stack + + The devstack base directory. diff --git a/roles/devstack-ipv6-only-deployments-verification/defaults/main.yaml b/roles/devstack-ipv6-only-deployments-verification/defaults/main.yaml new file mode 100644 index 0000000000..fea05c8146 --- /dev/null +++ b/roles/devstack-ipv6-only-deployments-verification/defaults/main.yaml @@ -0,0 +1 @@ +devstack_base_dir: /opt/stack diff --git a/roles/devstack-ipv6-only-deployments-verification/tasks/main.yaml b/roles/devstack-ipv6-only-deployments-verification/tasks/main.yaml new file mode 100644 index 0000000000..59d3b79bc1 --- /dev/null +++ b/roles/devstack-ipv6-only-deployments-verification/tasks/main.yaml @@ -0,0 +1,4 @@ +- name: Verify the ipv6-only deployments + become: true + become_user: stack + shell: "{{ devstack_base_dir }}/devstack/tools/verify-ipv6-only-deployments.sh" diff --git a/tools/verify-ipv6-only-deployments.sh b/tools/verify-ipv6-only-deployments.sh new file mode 100755 index 0000000000..2596395165 --- /dev/null +++ b/tools/verify-ipv6-only-deployments.sh @@ -0,0 +1,92 @@ +#!/bin/bash +# +# +# NOTE(gmann): This script is used in 'devstack-tempest-ipv6' zuul job to verify that +# services are deployed on IPv6 properly or not. This will capture if any devstck or devstack +# plugins are missing the required setting to listen on IPv6 address. This is run as part of +# run phase of zuul job and before test run. Child job of 'devstack-tempest-ipv6' +# can expand the IPv6 verification specific to project by defining the new post-run script which +# will run along with this base script. +# If there are more common verification for IPv6 then we can always extent this script. + +# Keep track of the DevStack directory +TOP_DIR=$(cd $(dirname "$0")/../../devstack && pwd) +source $TOP_DIR/stackrc +source $TOP_DIR/openrc admin admin + +function verify_devstack_ipv6_setting { + local _service_host='' + _service_host=$(echo $SERVICE_HOST | tr -d []) + local _host_ipv6='' + _host_ipv6=$(echo $HOST_IPV6 | tr -d []) + local _service_listen_address='' + _service_listen_address=$(echo $SERVICE_LISTEN_ADDRESS | tr -d []) + local _service_local_host='' + _service_local_host=$(echo $SERVICE_LOCAL_HOST | tr -d []) + if [[ "$SERVICE_IP_VERSION" != 6 ]]; then + echo $SERVICE_IP_VERSION "SERVICE_IP_VERSION is not set to 6 which is must for devstack to deploy services with IPv6 address." + exit 1 + fi + is_service_host_ipv6=$(python3 -c 'import oslo_utils.netutils as nutils; print(nutils.is_valid_ipv6("'$_service_host'"))') + if [[ "$is_service_host_ipv6" != "True" ]]; then + echo $SERVICE_HOST "SERVICE_HOST is not ipv6 which means devstack cannot deploy services on IPv6 address." + exit 1 + fi + is_host_ipv6=$(python3 -c 'import oslo_utils.netutils as nutils; print(nutils.is_valid_ipv6("'$_host_ipv6'"))') + if [[ "$is_host_ipv6" != "True" ]]; then + echo $HOST_IPV6 "HOST_IPV6 is not ipv6 which means devstack cannot deploy services on IPv6 address." + exit 1 + fi + is_service_listen_address=$(python3 -c 'import oslo_utils.netutils as nutils; print(nutils.is_valid_ipv6("'$_service_listen_address'"))') + if [[ "$is_service_listen_address" != "True" ]]; then + echo $SERVICE_LISTEN_ADDRESS "SERVICE_LISTEN_ADDRESS is not ipv6 which means devstack cannot deploy services on IPv6 address." + exit 1 + fi + is_service_local_host=$(python3 -c 'import oslo_utils.netutils as nutils; print(nutils.is_valid_ipv6("'$_service_local_host'"))') + if [[ "$is_service_local_host" != "True" ]]; then + echo $SERVICE_LOCAL_HOST "SERVICE_LOCAL_HOST is not ipv6 which means devstack cannot deploy services on IPv6 address." + exit 1 + fi + echo "Devstack is properly configured with IPv6" + echo "SERVICE_IP_VERSION: " $SERVICE_IP_VERSION "HOST_IPV6: " $HOST_IPV6 "SERVICE_HOST: " $SERVICE_HOST "SERVICE_LISTEN_ADDRESS: " $SERVICE_LISTEN_ADDRESS "SERVICE_LOCAL_HOST: " $SERVICE_LOCAL_HOST +} + +function sanity_check_system_ipv6_enabled { + system_ipv6_enabled=$(python3 -c 'import oslo_utils.netutils as nutils; print(nutils.is_ipv6_enabled())') + if [[ $system_ipv6_enabled != "True" ]]; then + echo "IPv6 is disabled in system" + exit 1 + fi + echo "IPv6 is enabled in system" +} + +function verify_service_listen_address_is_ipv6 { + local endpoints_verified=False + local all_ipv6=True + endpoints=$(openstack endpoint list -f value -c URL) + for endpoint in ${endpoints}; do + local endpoint_address='' + endpoint_address=$(echo "$endpoint" | awk -F/ '{print $3}' | awk -F] '{print $1}') + endpoint_address=$(echo $endpoint_address | tr -d []) + local is_endpoint_ipv6='' + is_endpoint_ipv6=$(python3 -c 'import oslo_utils.netutils as nutils; print(nutils.is_valid_ipv6("'$endpoint_address'"))') + if [[ "$is_endpoint_ipv6" != "True" ]]; then + all_ipv6=False + echo $endpoint ": This is not ipv6 endpoint which means corresponding service is not listening on IPv6 address." + continue + fi + endpoints_verified=True + done + if [[ "$all_ipv6" == "False" ]] || [[ "$endpoints_verified" == "False" ]]; then + exit 1 + fi + echo "All services deployed by devstack is on IPv6 endpoints" + echo $endpoints +} + +#First thing to verify if system has IPv6 enabled or not +sanity_check_system_ipv6_enabled +#Verify whether devstack is configured properly with IPv6 setting +verify_devstack_ipv6_setting +#Get all registrfed endpoints by devstack in keystone and verify that each endpoints address is IPv6. +verify_service_listen_address_is_ipv6 From c5f7cbfabe249498b45668c6838f284729fedc75 Mon Sep 17 00:00:00 2001 From: Lee Yarwood Date: Mon, 4 Oct 2021 18:07:17 +0100 Subject: [PATCH 41/45] nova: Ensure each compute uses a unique iSCSI initiator The current initiator name embedded in our CI images is not unique at present and can often cause failures during live migrations with attached volumes. This change ensures the name is unique by running iscsi-iname again and overwriting the existing name. We could potentially do this during the image build process itself but given that devstack systems are not supposed to be multi-purpose this should be safe to do during the devstack run. NOTE(lyarwood): Conflict due to If2f74f146a166b9721540aaf3f1f9fce3030525c not being present on stable/wallaby. Conflicts: lib/nova Closes-Bug: #1945983 Change-Id: I9ed26a17858df96c04be9ae52bf2e33e023869a5 (cherry picked from commit 714826d1a27085ba2384ca495c876588d77f0d27) (cherry picked from commit ee629cc77554b18cfca77506d1531f99523d2a58) (cherry picked from commit a41fff99b35a2723348c933dc6c37ccf218e280f) (cherry picked from commit 43364b7198a90b16a53f6e89d79238bd40a76953) --- lib/nova | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/nova b/lib/nova index 5b3246a2c2..767a7d2a74 100644 --- a/lib/nova +++ b/lib/nova @@ -306,6 +306,9 @@ function configure_nova { sudo dnf update -y fi + # Ensure each compute host uses a unique iSCSI initiator + echo InitiatorName=$(iscsi-iname) | sudo tee /etc/iscsi/initiatorname.iscsi + if [[ ${ISCSID_DEBUG} == "True" ]]; then # Install an override that starts iscsid with debugging # enabled. From 410d11ebc66e28d096646c178949639c285f3593 Mon Sep 17 00:00:00 2001 From: Ghanshyam Mann Date: Wed, 3 Nov 2021 14:28:17 -0500 Subject: [PATCH 42/45] Pin Tempest to train-last(28.0.0) tag for stable/train testing Stable train is in Extended maintenance state[1] and As per Tempest stable branmch testing policy, it stopped the support for stable/train in master version. Tempest train-last (28.0.0) tag is last supported tag for stable/train. Use that tag to test stable/train branch before any incompatible change merge in Tempest master. [1] https://releases.openstack.org/ Change-Id: I45d685725ec3204a7c0467085f87214f02ecbe1c --- stackrc | 4 ++-- tests/test_refs.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/stackrc b/stackrc index a4d2600b09..1a0a7abd45 100644 --- a/stackrc +++ b/stackrc @@ -314,8 +314,8 @@ REQUIREMENTS_BRANCH=${REQUIREMENTS_BRANCH:-$TARGET_BRANCH} # Tempest test suite TEMPEST_REPO=${TEMPEST_REPO:-${GIT_BASE}/openstack/tempest.git} -TEMPEST_BRANCH=${TEMPEST_BRANCH:-$BRANCHLESS_TARGET_BRANCH} -TEMPEST_VENV_UPPER_CONSTRAINTS=${TEMPEST_VENV_UPPER_CONSTRAINTS:-master} +TEMPEST_BRANCH=${TEMPEST_BRANCH:-train-last} +TEMPEST_VENV_UPPER_CONSTRAINTS=${TEMPEST_VENV_UPPER_CONSTRAINTS:-$REQUIREMENTS_DIR/upper-constraints.txt} ############## diff --git a/tests/test_refs.sh b/tests/test_refs.sh index 0f9aa4a5ca..d9b328294a 100755 --- a/tests/test_refs.sh +++ b/tests/test_refs.sh @@ -15,7 +15,7 @@ echo "Ensuring we don't have crazy refs" -REFS=`grep BRANCH stackrc | grep -v 'TARGET_BRANCH' | grep -v 'NOVNC_BRANCH'` +REFS=`grep BRANCH stackrc | grep -v 'TARGET_BRANCH' | grep -v 'NOVNC_BRANCH' | grep -v 'TEMPEST_BRANCH'` rc=$? if [[ $rc -eq 0 ]]; then echo "Branch defaults must be one of the *TARGET_BRANCH values. Found:" From 817bb7d50b06b03b7c56c3bb70db69e1c98cea46 Mon Sep 17 00:00:00 2001 From: Ghanshyam Date: Wed, 22 Dec 2021 17:26:50 +0000 Subject: [PATCH 43/45] Revert "Pin Tempest to train-last(28.0.0) tag for stable/train testing" This reverts commit 410d11ebc66e28d096646c178949639c285f3593. Reason for revert: it caused the bug for tempest plugin, let's figure out that first and then cap the appropriate tempest version- https://bugs.launchpad.net/tempest/+bug/1955418 Change-Id: Ie4305e7b6030d9edaa97a207a588b734cf7707dc --- stackrc | 4 ++-- tests/test_refs.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/stackrc b/stackrc index 1a0a7abd45..a4d2600b09 100644 --- a/stackrc +++ b/stackrc @@ -314,8 +314,8 @@ REQUIREMENTS_BRANCH=${REQUIREMENTS_BRANCH:-$TARGET_BRANCH} # Tempest test suite TEMPEST_REPO=${TEMPEST_REPO:-${GIT_BASE}/openstack/tempest.git} -TEMPEST_BRANCH=${TEMPEST_BRANCH:-train-last} -TEMPEST_VENV_UPPER_CONSTRAINTS=${TEMPEST_VENV_UPPER_CONSTRAINTS:-$REQUIREMENTS_DIR/upper-constraints.txt} +TEMPEST_BRANCH=${TEMPEST_BRANCH:-$BRANCHLESS_TARGET_BRANCH} +TEMPEST_VENV_UPPER_CONSTRAINTS=${TEMPEST_VENV_UPPER_CONSTRAINTS:-master} ############## diff --git a/tests/test_refs.sh b/tests/test_refs.sh index d9b328294a..0f9aa4a5ca 100755 --- a/tests/test_refs.sh +++ b/tests/test_refs.sh @@ -15,7 +15,7 @@ echo "Ensuring we don't have crazy refs" -REFS=`grep BRANCH stackrc | grep -v 'TARGET_BRANCH' | grep -v 'NOVNC_BRANCH' | grep -v 'TEMPEST_BRANCH'` +REFS=`grep BRANCH stackrc | grep -v 'TARGET_BRANCH' | grep -v 'NOVNC_BRANCH'` rc=$? if [[ $rc -eq 0 ]]; then echo "Branch defaults must be one of the *TARGET_BRANCH values. Found:" From 8a22f7380c7029e931fe9103320f24a223b619d1 Mon Sep 17 00:00:00 2001 From: Ghanshyam Mann Date: Tue, 18 Jan 2022 21:29:03 -0600 Subject: [PATCH 44/45] Pin Tempest to 26.1.0 tag for stable/train testing Stable train is in Extended maintenance state[1] and we need to cap stable/train with older Tempest. But because of oslo.utils version bump in Tempest 27.0.0 and 28.0.0 which is not compatible with stable/train upper constraints so these versions do not work in stable/train - https://zuul.opendev.org/t/openstack/build/1146c9ce9f474d0d82b99b48b61c531e Let's use the Tempest 26.1.0 which is also compatible with stable/train testing. Related-Bug: #1955418 Change-Id: I376be5f5a4d693aeaaf74ca5285269f796103c15 --- stackrc | 6 ++++-- tests/test_refs.sh | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/stackrc b/stackrc index a4d2600b09..0d92df5043 100644 --- a/stackrc +++ b/stackrc @@ -314,8 +314,10 @@ REQUIREMENTS_BRANCH=${REQUIREMENTS_BRANCH:-$TARGET_BRANCH} # Tempest test suite TEMPEST_REPO=${TEMPEST_REPO:-${GIT_BASE}/openstack/tempest.git} -TEMPEST_BRANCH=${TEMPEST_BRANCH:-$BRANCHLESS_TARGET_BRANCH} -TEMPEST_VENV_UPPER_CONSTRAINTS=${TEMPEST_VENV_UPPER_CONSTRAINTS:-master} +# NOTE (gmann): using 26.1.0 as that is compatible with stable/train +# constraints. We are not using 28.0.0 or 27.0.0 due to bug#1955418 +TEMPEST_BRANCH=${TEMPEST_BRANCH:-26.1.0} +TEMPEST_VENV_UPPER_CONSTRAINTS=${TEMPEST_VENV_UPPER_CONSTRAINTS:-$REQUIREMENTS_DIR/upper-constraints.txt} ############## diff --git a/tests/test_refs.sh b/tests/test_refs.sh index 0f9aa4a5ca..d9b328294a 100755 --- a/tests/test_refs.sh +++ b/tests/test_refs.sh @@ -15,7 +15,7 @@ echo "Ensuring we don't have crazy refs" -REFS=`grep BRANCH stackrc | grep -v 'TARGET_BRANCH' | grep -v 'NOVNC_BRANCH'` +REFS=`grep BRANCH stackrc | grep -v 'TARGET_BRANCH' | grep -v 'NOVNC_BRANCH' | grep -v 'TEMPEST_BRANCH'` rc=$? if [[ $rc -eq 0 ]]; then echo "Branch defaults must be one of the *TARGET_BRANCH values. Found:" From 05a2cebeb4e93377fa88633eef4b2cc0ea584450 Mon Sep 17 00:00:00 2001 From: Rodolfo Alonso Hernandez Date: Sat, 29 Jan 2022 11:29:07 +0000 Subject: [PATCH 45/45] Add python3.6 pip support Since pip v22, python3.6 is not supported (the minimum version is python3.7). This patch adds the reference for the pip3.6 URL to be used instead of the default one. NOTE: stable/train had its own method to provide support for py3x. This patch uses it instead of pushing the master patch approach. This patch is also fixing the py35 pip URL string that should be https://bootstrap.pypa.io/pip/3.5/get-pip.py. Conflicts: tools/install_pip.sh Closes-Bug: #1959600 Change-Id: Iab2c391d5388461fe9e9037cee81884ce8032e72 --- tools/install_pip.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/install_pip.sh b/tools/install_pip.sh index 1f09141e20..ad4010a8d3 100755 --- a/tools/install_pip.sh +++ b/tools/install_pip.sh @@ -103,8 +103,11 @@ function install_get_pip { _install_get_pip python $PIP_GET_PIP_PY27_URL $LOCAL_PIP_PY27 if python3_enabled; then if [[ "$PYTHON3_VERSION" == "3.5" ]]; then - PIP_GET_PIP_URL=$(dirname $PIP_GET_PIP_URL)/3.5/$(basename $PIP_GET_PIP_URL) + PIP_GET_PIP_URL=$(dirname $PIP_GET_PIP_URL)/pip/3.5/$(basename $PIP_GET_PIP_URL) LOCAL_PIP=${LOCAL_PIP}-py35 + elif [[ "$PYTHON3_VERSION" == "3.6" ]]; then + PIP_GET_PIP_URL=$(dirname $PIP_GET_PIP_URL)/pip/3.6/$(basename $PIP_GET_PIP_URL) + LOCAL_PIP=${LOCAL_PIP}-py36 fi _install_get_pip python${PYTHON3_VERSION} $PIP_GET_PIP_URL $LOCAL_PIP fi