From 65653a53c5740e64db6bc0ef8213104cfe86700d Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Wed, 25 Oct 2023 19:38:02 +0000 Subject: [PATCH] Enable upgrade tests for charms and openstack Add charm and openstack upgrade tests for focal-ussuri->focal-yoga and jammy-yoga->jammy-caracal. Also some general cleanup to the README and tests.yaml to clean up model aliases. --- README.rst | 8 +- osci.yaml | 12 ++ .../focal-ussuri-to-yoga-upgrades.yaml | 1 + .../tests/bundles/focal-xena.yaml | 2 +- .../jammy-yoga-to-caracal-upgrades.yaml | 1 + tests/distro-regression/tests/tests.yaml | 178 ++++++++++++++---- 6 files changed, 158 insertions(+), 44 deletions(-) create mode 120000 tests/distro-regression/tests/bundles/focal-ussuri-to-yoga-upgrades.yaml create mode 120000 tests/distro-regression/tests/bundles/jammy-yoga-to-caracal-upgrades.yaml diff --git a/README.rst b/README.rst index 8661ac5a..b6516892 100644 --- a/README.rst +++ b/README.rst @@ -40,7 +40,7 @@ Deploy and test a specific bundle: .. code-block:: bash - tox -e func-target keystone_v3_smoke_focal:jammy-yoga + tox -e func-target jammy-yoga Deploy and test all smoke bundles: @@ -67,9 +67,9 @@ Specifying which tests to run can be done with the following keys: smoke, whitel tests_options: tempest: - keystone_v3_smoke: + model_alias_smoke: smoke: True - keystone_v3_full: + model_alias_targeted: whitelist: - "tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_list_servers" - "tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_verify_server_details" @@ -88,7 +88,7 @@ re-running tests, set keep-workspace to True in tests.yaml: tests_options: tempest: - keystone_v3_smoke: + model_alias: smoke: True keep-workspace: True diff --git a/osci.yaml b/osci.yaml index 49fe1809..96830309 100644 --- a/osci.yaml +++ b/osci.yaml @@ -22,6 +22,8 @@ - cot_distro-regression_focal-ussuri-security - cot_distro-regression_jammy-yoga-security - cot_distro-regression_lunar-antelope-security + - cot_distro-regression_focal-ussuri-to-yoga-upgrades + - cot_distro-regression_jammy-yoga-to-caracal-upgrades - job: name: cot-func-target parent: func-target @@ -117,3 +119,13 @@ parent: cot-func-target vars: tox_extra_args: '-- lunar-antelope-security' +- job: + name: cot_distro-regression_focal-ussuri-to-yoga-upgrades + parent: cot-func-target + vars: + tox_extra_args: '-- focal-ussuri-to-yoga-upgrades' +- job: + name: cot_distro-regression_jammy-yoga-to-caracal-upgrades + parent: cot-func-target + vars: + tox_extra_args: '-- jammy-yoga-to-caracal-upgrades' diff --git a/tests/distro-regression/tests/bundles/focal-ussuri-to-yoga-upgrades.yaml b/tests/distro-regression/tests/bundles/focal-ussuri-to-yoga-upgrades.yaml new file mode 120000 index 00000000..731ed946 --- /dev/null +++ b/tests/distro-regression/tests/bundles/focal-ussuri-to-yoga-upgrades.yaml @@ -0,0 +1 @@ +focal-ussuri.yaml \ No newline at end of file diff --git a/tests/distro-regression/tests/bundles/focal-xena.yaml b/tests/distro-regression/tests/bundles/focal-xena.yaml index 590c4afb..d304fff6 100644 --- a/tests/distro-regression/tests/bundles/focal-xena.yaml +++ b/tests/distro-regression/tests/bundles/focal-xena.yaml @@ -2,7 +2,7 @@ variables: source: &source cloud:focal-xena/proposed openstack-origin: &openstack-origin cloud:focal-xena/proposed retrofit-uca-pocket: &retrofit-uca-pocket xena - openstack-channel: &openstack-channel wallaby/edge + openstack-channel: &openstack-channel xena/edge ceph-channel: &ceph-channel pacific/edge ovn-channel: &ovn-channel 21.09/edge mysql-channel: &mysql-channel 8.0/edge diff --git a/tests/distro-regression/tests/bundles/jammy-yoga-to-caracal-upgrades.yaml b/tests/distro-regression/tests/bundles/jammy-yoga-to-caracal-upgrades.yaml new file mode 120000 index 00000000..6dcaab56 --- /dev/null +++ b/tests/distro-regression/tests/bundles/jammy-yoga-to-caracal-upgrades.yaml @@ -0,0 +1 @@ +jammy-yoga.yaml \ No newline at end of file diff --git a/tests/distro-regression/tests/tests.yaml b/tests/distro-regression/tests/tests.yaml index 8666321a..832ad8cf 100644 --- a/tests/distro-regression/tests/tests.yaml +++ b/tests/distro-regression/tests/tests.yaml @@ -1,25 +1,27 @@ smoke_bundles: - - keystone_bionic_queens: bionic-queens - - keystone_bionic_ussuri: bionic-ussuri - - keystone_focal_ussuri: focal-ussuri - - keystone_focal_ussuri: focal-ussuri-ovn-22.03 - - keystone_focal_ussuri: focal-victoria - - keystone_focal_wallaby: focal-wallaby - - keystone_focal_wallaby: focal-xena - - keystone_focal_wallaby: focal-yoga - - keystone_focal_wallaby: jammy-yoga - - keystone_focal_wallaby: jammy-zed - - keystone_focal_wallaby: jammy-antelope - - keystone_focal_wallaby: jammy-bobcat - - keystone_focal_wallaby: lunar-antelope - - keystone_focal_wallaby: mantic-bobcat - - keystone_bionic_queens_security: bionic-queens-security - - keystone_focal_ussuri_security: focal-ussuri-security - - keystone_focal_ussuri_security: jammy-yoga-security - - keystone_focal_ussuri_security: lunar-antelope-security - - keystone_focal_ussuri_security: mantic-bobcat-security + - bionic_queens: bionic-queens + - bionic_ussuri: bionic-ussuri + - focal_ussuri: focal-ussuri + - focal_ussuri: focal-ussuri-ovn-22.03 + - focal_ussuri: focal-victoria + - focal_wallaby: focal-wallaby + - focal_wallaby: focal-xena + - focal_wallaby: focal-yoga + - focal_wallaby: jammy-yoga + - focal_wallaby: jammy-zed + - focal_wallaby: jammy-antelope + - focal_wallaby: jammy-bobcat + - focal_wallaby: lunar-antelope + - focal_wallaby: mantic-bobcat + - bionic_queens_security: bionic-queens-security + - focal_ussuri_security: focal-ussuri-security + - focal_ussuri_security: jammy-yoga-security + - focal_ussuri_security: lunar-antelope-security + - focal_ussuri_security: mantic-bobcat-security + - focal_upgrades: focal-ussuri-to-yoga-upgrades + - jammy_upgrades: jammy-yoga-to-caracal-upgrades configure: - - keystone_bionic_queens: &keystone_bionic_queens + - bionic_queens: &bionic_queens - zaza.openstack.charm_tests.ceilometer.setup.basic_setup - zaza.openstack.charm_tests.glance.setup.add_lts_image - zaza.openstack.charm_tests.neutron.setup.basic_overcloud_network @@ -29,8 +31,8 @@ configure: - zaza.openstack.charm_tests.keystone.setup.add_tempest_roles - zaza.openstack.charm_tests.glance.setup.add_cirros_image - zaza.openstack.charm_tests.glance.setup.add_cirros_alt_image - - keystone_bionic_queens_security: *keystone_bionic_queens - - keystone_bionic_ussuri: + - bionic_queens_security: *bionic_queens + - bionic_ussuri: - zaza.openstack.charm_tests.ceilometer.setup.basic_setup - zaza.openstack.charm_tests.glance_simplestreams_sync.setup.sync_images - zaza.openstack.charm_tests.glance.setup.add_lts_image @@ -45,7 +47,7 @@ configure: - zaza.openstack.charm_tests.glance.setup.add_cirros_image - zaza.openstack.charm_tests.glance.setup.add_cirros_alt_image - zaza.openstack.charm_tests.octavia.setup.centralized_fip_network - - keystone_focal_ussuri: &keystone_focal_ussuri + - focal_ussuri: &focal_ussuri - zaza.openstack.charm_tests.vault.setup.auto_initialize - zaza.openstack.charm_tests.ceilometer.setup.basic_setup - zaza.openstack.charm_tests.glance_simplestreams_sync.setup.sync_images @@ -63,30 +65,61 @@ configure: - zaza.openstack.charm_tests.octavia.setup.centralized_fip_network - zaza.openstack.charm_tests.magnum.setup.domain_setup - zaza.openstack.charm_tests.magnum.setup.add_image - - keystone_focal_ussuri_security: *keystone_focal_ussuri - - keystone_focal_wallaby: *keystone_focal_ussuri + - focal_ussuri_security: *focal_ussuri + - focal_upgrades: *focal_ussuri + - focal_wallaby: *focal_ussuri + - jammy_upgrades: *focal_ussuri tests: - - keystone_bionic_queens: + - bionic_queens: - zaza.openstack.charm_tests.tempest.tests.TempestTestWithKeystoneV3 - - keystone_bionic_queens_security: + - bionic_queens_security: - zaza.openstack.charm_tests.tempest.tests.TempestTestWithKeystoneV3 - - keystone_bionic_ussuri: + - bionic_ussuri: - zaza.openstack.charm_tests.tempest.tests.TempestTestWithKeystoneV3 - - keystone_focal_ussuri: + - focal_ussuri: - zaza.openstack.charm_tests.tempest.tests.TempestTestWithKeystoneV3 - - keystone_focal_ussuri_security: + - focal_ussuri_security: - zaza.openstack.charm_tests.tempest.tests.TempestTestWithKeystoneV3 - - keystone_focal_wallaby: + - focal_wallaby: + - zaza.openstack.charm_tests.tempest.tests.TempestTestWithKeystoneV3 + - focal_upgrades: + # ussuri->victoria + - zaza.openstack.charm_tests.charm_upgrade.tests.FullCloudCharmUpgradeTest + - zaza.openstack.charm_tests.openstack_upgrade.tests.OpenStackUpgradeTestsByOption + # victoria->wallaby + - zaza.openstack.charm_tests.charm_upgrade.tests.FullCloudCharmUpgradeTest + - zaza.openstack.charm_tests.openstack_upgrade.tests.OpenStackUpgradeTestsByOption + # wallaby->xena + - zaza.openstack.charm_tests.charm_upgrade.tests.FullCloudCharmUpgradeTest + - zaza.openstack.charm_tests.openstack_upgrade.tests.OpenStackUpgradeTestsByOption + # xena->yoga + - zaza.openstack.charm_tests.charm_upgrade.tests.FullCloudCharmUpgradeTest + - zaza.openstack.charm_tests.openstack_upgrade.tests.OpenStackUpgradeTestsByOption + - zaza.openstack.charm_tests.tempest.tests.TempestTestWithKeystoneV3 + - jammy_upgrades: + # yoga->zed + - zaza.openstack.charm_tests.charm_upgrade.tests.FullCloudCharmUpgradeTest + - zaza.openstack.charm_tests.openstack_upgrade.tests.OpenStackUpgradeTestsByOption + # zed->antelope + - zaza.openstack.charm_tests.charm_upgrade.tests.FullCloudCharmUpgradeTest + - zaza.openstack.charm_tests.openstack_upgrade.tests.OpenStackUpgradeTestsByOption + # antelope->bobcat + - zaza.openstack.charm_tests.charm_upgrade.tests.FullCloudCharmUpgradeTest + - zaza.openstack.charm_tests.openstack_upgrade.tests.OpenStackUpgradeTestsByOption + # bobcat->caracal + # Note(coreycb): Consider skip level upgrade from antelope->caracal. + #- zaza.openstack.charm_tests.charm_upgrade.tests.FullCloudCharmUpgradeTest + #- zaza.openstack.charm_tests.openstack_upgrade.tests.OpenStackUpgradeTestsByOption - zaza.openstack.charm_tests.tempest.tests.TempestTestWithKeystoneV3 tests_options: - keystone_bionic_queens_security: + bionic_queens_security: overlay_ppas: - ppa:ubuntu-security-proposed/ppa - keystone_focal_ussuri_security: + focal_ussuri_security: overlay_ppas: - ppa:ubuntu-security-proposed/ppa tempest: - keystone_bionic_queens: + bionic_queens: smoke: True serial: True exclude-list: @@ -96,7 +129,7 @@ tests_options: - "designate_tempest_plugin.tests.api.v2.test_zones_imports.ZonesImportTest" # octavia test fails with self.creds_client.assign_user_role 'No "load-balancer_admin" role found' - "octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest" - keystone_bionic_queens_security: + bionic_queens_security: smoke: True serial: True exclude-list: @@ -106,10 +139,10 @@ tests_options: - "designate_tempest_plugin.tests.api.v2.test_zones_imports.ZonesImportTest" # octavia test fails with self.creds_client.assign_user_role 'No "load-balancer_admin" role found' - "octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest" - keystone_bionic_ussuri: + bionic_ussuri: smoke: True serial: True - keystone_focal_ussuri: + focal_ussuri: smoke: True serial: True include-list: @@ -144,7 +177,7 @@ tests_options: - "octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest" # Note(coreycb): Disable watcher tests until all the failures can be debugged. - "watcher_tempest_plugin.*" - keystone_focal_ussuri_security: + focal_ussuri_security: smoke: True serial: True include-list: @@ -179,7 +212,74 @@ tests_options: - "octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest" # Note(coreycb): Disable watcher tests until all the failures can be debugged. - "watcher_tempest_plugin.*" - keystone_focal_wallaby: + focal_upgrades: + smoke: True + serial: True + include-list: + - "manila_tempest_tests.tests.api.admin.test_admin_actions.AdminActionsTest.*" + - "manila_tempest_tests.tests.api.admin.test_share_instances.ShareInstancesTest.*" + - "manila_tempest_tests.tests.api.admin.test_share_snapshot_instances.ShareSnapshotInstancesTest.*" + - "manila_tempest_tests.tests.api.admin.test_share_types.ShareTypesAdminTest.*" + - "manila_tempest_tests.tests.api.admin.test_shares_actions.SharesActionsAdminTest.*" + - "magnum_tempest_plugin.tests.api.v1.test_cluster" + - "magnum_tempest_plugin.tests.api.v1.test_cluster_template" + - "magnum_tempest_plugin.tests.api.v1.test_cluster_template_admin" + - "magnum_tempest_plugin.tests.api.v1.test_magnum_service" + # Note(coreycb): Disable watcher tests until all the failures can be debugged. + # - "watcher_tempest_plugin.tests.api" + # - "watcher_tempest_plugin.tests.scenario.test_execute_host_maintenance" + # - "watcher_tempest_plugin.tests.scenario.test_execute_vm_workload_consolidation" + exclude-list: + # designate failures due to check_list_show_RBAC_enforcement returning "Unauthorized" + - "designate_tempest_plugin.tests.api.v2.test_zones_exports.ZonesExportTest.test_show_zone_export" + - "designate_tempest_plugin.tests.api.v2.test_zones_imports.ZonesImportTest.test_show_zone_import" + # Exclude the known failures due to issues with octavia/manila policy + - "manila_tempest_tests.tests.api.admin.test_share_networks.ShareNetworkAdminTest" + - "manila_tempest_tests.tests.api.test_share_networks.ShareNetworksTest" + # Implemented on container-infra 1.10 which is available in >=Xena + # https://opendev.org/openstack/magnum/commit/0e6d17893 + # https://opendev.org/openstack/magnum-tempest-plugin/commit/b68a678f37de0a769e7ee8dbefa9bdfe6cf445cc + - "magnum_tempest_plugin.tests.api.v1.test_cluster.ClusterTest.test_create_list_sign_delete_clusters" + - "magnum_tempest_plugin.tests.api.v1.test_cluster.ClusterTest.test_create_cluster_with_zero_nodes" + # The test expects a 400 error while the server returns a 401 error due to glance + # See logs at https://pastebin.ubuntu.com/p/V3DMcVmtyF/ + - "magnum_tempest_plugin.tests.api.v1.test_cluster.ClusterTest.test_create_cluster_with_nonexisting_flavor" + - "octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest" + # Note(coreycb): Disable watcher tests until all the failures can be debugged. + - "watcher_tempest_plugin.*" + focal_wallaby: + smoke: True + serial: True + include-list: + - "manila_tempest_tests.tests.api.admin.test_admin_actions.AdminActionsTest.*" + - "manila_tempest_tests.tests.api.admin.test_share_instances.ShareInstancesTest.*" + - "manila_tempest_tests.tests.api.admin.test_share_snapshot_instances.ShareSnapshotInstancesTest.*" + - "manila_tempest_tests.tests.api.admin.test_share_types.ShareTypesAdminTest.*" + - "manila_tempest_tests.tests.api.admin.test_shares_actions.SharesActionsAdminTest.*" + - "magnum_tempest_plugin.tests.api.v1.test_cluster" + - "magnum_tempest_plugin.tests.api.v1.test_cluster_template" + - "magnum_tempest_plugin.tests.api.v1.test_cluster_template_admin" + - "magnum_tempest_plugin.tests.api.v1.test_magnum_service" + # Note(coreycb): Disable watcher tests until all the failures can be debugged. + # - "watcher_tempest_plugin.tests.api" + # - "watcher_tempest_plugin.tests.scenario.test_execute_host_maintenance" + # - "watcher_tempest_plugin.tests.scenario.test_execute_vm_workload_consolidation" + exclude-list: + # Exclude the known failures due to issues with octavia/manila policy + - "manila_tempest_tests.tests.api.admin.test_share_networks.ShareNetworkAdminTest" + - "manila_tempest_tests.tests.api.test_share_networks.ShareNetworksTest" + # Implemented on container-infra 1.10 which is available in >=Xena + # https://opendev.org/openstack/magnum/commit/0e6d17893 + # https://opendev.org/openstack/magnum-tempest-plugin/commit/b68a678f37de0a769e7ee8dbefa9bdfe6cf445cc + - "magnum_tempest_plugin.tests.api.v1.test_cluster.ClusterTest.test_create_list_sign_delete_clusters" + - "magnum_tempest_plugin.tests.api.v1.test_cluster.ClusterTest.test_create_cluster_with_zero_nodes" + # The test expects a 400 error while the server returns a 401 error due to glance + # See logs at https://pastebin.ubuntu.com/p/V3DMcVmtyF/ + - "magnum_tempest_plugin.tests.api.v1.test_cluster.ClusterTest.test_create_cluster_with_nonexisting_flavor" + - "octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest" + # Note(coreycb): Disable watcher tests until all the failures can be debugged. + - "watcher_tempest_plugin.*" + jammy_upgrades: smoke: True serial: True include-list: