diff --git a/.github/workflows/tox.yaml b/.github/workflows/tox.yaml index 92da89ed6..023c2f3b7 100644 --- a/.github/workflows/tox.yaml +++ b/.github/workflows/tox.yaml @@ -10,7 +10,6 @@ jobs: strategy: matrix: python-version: ['3.8', '3.10', '3.11'] - steps: - uses: actions/checkout@v1 - name: Set up Python ${{ matrix.python-version }} @@ -36,10 +35,30 @@ jobs: matrix: juju_channel: - 2.9/stable + - 3.1/stable + # - 3.2/stable + # - 3.3/stable bundle: - first - second - third + include: + - juju_channel: 2.9/stable + snap_install_flags: "--classic" + pip_constraints: constraints-juju29.txt + - juju_channel: 3.1/stable + snap_install_flags: "" + pip_constraints: constraints-juju31.txt + # NOTE(freyes): disabled until "RuntimeError: set_wakeup_fd only works + # in main thread of the main interpreter" gets fixed. + # https://pastebin.ubuntu.com/p/GfYKgpx3SP/ + # + # - juju_channel: 3.2/stable + # snap_install_flags: "" + # pip_constraints: constraints-juju32.txt + # - juju_channel: 3.3/stable + # snap_install_flags: "" + # pip_constraints: constraints-juju33.txt env: TEST_ZAZA_BUG_LP1987332: "on" # http://pad.lv/1987332 needs: build @@ -48,9 +67,13 @@ jobs: - name: Install dependencies run: | set -euxo pipefail + sudo apt-get update + sudo apt-get install -yq snapd + sudo systemctl enable snapd + sudo systemctl restart snapd python -m pip install --upgrade pip pip install tox tox-gh-actions - sudo snap install --channel ${{ matrix.juju_channel }} --classic juju + sudo snap install ${{ matrix.snap_install_flags }} --channel ${{ matrix.juju_channel }} juju sudo snap install --classic juju-crashdump sudo lxd init --auto # This is a throw-away CI environment, do not do this at home @@ -64,11 +87,12 @@ jobs: lxc image copy --alias juju/focal/amd64 --copy-aliases ubuntu-daily:focal local: lxc image copy --alias juju/jammy/amd64 --copy-aliases ubuntu-daily:jammy local: lxc image list - juju bootstrap --no-gui localhost + juju bootstrap localhost - name: Functional test run: | set -euxo pipefail mkdir logs + export PIP_CONSTRAINTS=$(pwd)/${{ matrix.pip_constraints }} tox -e func-target -- ${{ matrix.bundle }} | tee logs/tox-output.txt - name: crashdump on failure if: failure() @@ -77,7 +101,7 @@ jobs: juju models model=$(juju models --format yaml|grep "^- name:.*zaza"|cut -f2 -d/) juju status -m $model | tee logs/juju-status.txt - juju crashdump -m $model -o logs/ + juju-crashdump -m $model -o logs/ - name: upload logs on failure if: failure() uses: actions/upload-artifact@v2 diff --git a/constraints-juju-default.txt b/constraints-juju-default.txt new file mode 100644 index 000000000..f85e9e1b3 --- /dev/null +++ b/constraints-juju-default.txt @@ -0,0 +1,9 @@ +# NOTE: this constraints file can be (and will be) consumed by downstream users. +# +# Known consumers: +# * zosci-config: job definitions that declare what juju version (snap channel) +# is used in tandem with this constraints file to lockdown python-libjuju +# version. +# * zaza-openstack-tests +# +juju>=3.1.0,<3.2.0 diff --git a/constraints-juju29.txt b/constraints-juju29.txt new file mode 100644 index 000000000..ba318b495 --- /dev/null +++ b/constraints-juju29.txt @@ -0,0 +1,11 @@ +# NOTE: this constraints file can be (and will be) consumed by downstream users. +# +# Known consumers: +# * zosci-config: job definitions that declare what juju version (snap channel) +# is used in tandem with this constraints file to lockdown python-libjuju +# version. +# * zaza-openstack-tests +# +# Reasons to block the use of a release: +# * 2.9.45, 2.9.46 - https://github.com/juju/python-libjuju/pull/975 +juju>=2.9.0,<3.0.0,!=2.9.45,!=2.9.46 diff --git a/constraints-juju31.txt b/constraints-juju31.txt new file mode 100644 index 000000000..f85e9e1b3 --- /dev/null +++ b/constraints-juju31.txt @@ -0,0 +1,9 @@ +# NOTE: this constraints file can be (and will be) consumed by downstream users. +# +# Known consumers: +# * zosci-config: job definitions that declare what juju version (snap channel) +# is used in tandem with this constraints file to lockdown python-libjuju +# version. +# * zaza-openstack-tests +# +juju>=3.1.0,<3.2.0 diff --git a/constraints-juju32.txt b/constraints-juju32.txt new file mode 100644 index 000000000..76eaa9034 --- /dev/null +++ b/constraints-juju32.txt @@ -0,0 +1,9 @@ +# NOTE: this constraints file can be (and will be) consumed by downstream users. +# +# Known consumers: +# * zosci-config: job definitions that declare what juju version (snap channel) +# is used in tandem with this constraints file to lockdown python-libjuju +# version. +# * zaza-openstack-tests +# +juju>=3.2.0,<3.3.0 diff --git a/constraints-juju33.txt b/constraints-juju33.txt new file mode 100644 index 000000000..f5263fa0f --- /dev/null +++ b/constraints-juju33.txt @@ -0,0 +1,9 @@ +# NOTE: this constraints file can be (and will be) consumed by downstream users. +# +# Known consumers: +# * zosci-config: job definitions that declare what juju version (snap channel) +# is used in tandem with this constraints file to lockdown python-libjuju +# version. +# * zaza-openstack-tests +# +juju>=3.3.0,<3.4.0 diff --git a/requirements.txt b/requirements.txt index d4288cbe6..9668b5355 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,6 @@ pyparsing<3.0.0 # pin for aodhclient which is held for py35 async_generator kubernetes<18.0.0; python_version < '3.6' # pined, as juju uses kubernetes -# pinned until 3.0 regressions are handled: https://github.com/openstack-charmers/zaza/issues/545 -juju<3.0 juju_wait PyYAML>=3.0 pbr==5.6.0 diff --git a/setup.py b/setup.py index 168fb9e54..7bdd8e157 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,6 @@ 'hvac<0.7.0', 'jinja2', - 'juju<3.0', 'juju-wait', 'PyYAML', 'tenacity>8.2.0', @@ -44,12 +43,17 @@ # https://github.com/go-macaroon-bakery/py-macaroon-bakery/issues/94 'macaroonbakery != 1.3.3', + 'juju', ] tests_require = [ 'tox >= 2.3.1', ] +extras_require = { + 'testing': tests_require, +} + class Tox(TestCommand): """Tox class.""" @@ -113,8 +117,6 @@ def run_tests(self): zip_safe=False, cmdclass={'test': Tox}, install_requires=install_require, - extras_require={ - 'testing': tests_require, - }, tests_require=tests_require, + extras_require=extras_require, ) diff --git a/test-requirements.txt b/test-requirements.txt index 371b1326b..3c66eb3da 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -16,7 +16,8 @@ keystoneauth1 oslo.config python-novaclient tenacity>8.2.0 -# pinned until 3.0 regressions are handled: https://github.com/openstack-charmers/zaza/issues/545 -juju<3.0 +# To force the installation of an specific version of libjuju use a constraints +# file, e.g.: `env PIP_CONSTRAINTS=./constraints-juju31.txt tox -e func-target` +juju # https://github.com/go-macaroon-bakery/py-macaroon-bakery/issues/94 macaroonbakery!=1.3.3 diff --git a/tox.ini b/tox.ini index 3830b9a1d..323a9cd0f 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] envlist = pep8,py3 -skipsdist = True +skipsdist = False # NOTE: https://wiki.canonical.com/engineering/OpenStack/InstallLatestToxOnOsci minversion = 3.2.0 @@ -17,59 +17,53 @@ passenv = CS_* OS_* TEST_* + PIP_* +deps = + -c{env:PIP_CONSTRAINTS:{toxinidir}/constraints-juju29.txt} + -r{toxinidir}/test-requirements.txt commands = pytest --cov=./zaza/ {posargs} {toxinidir}/unit_tests [testenv:py3] basepython = python3 -deps = -r{toxinidir}/test-requirements.txt [testenv:pep8] basepython = python3 -deps = -r{toxinidir}/test-requirements.txt commands = flake8 {posargs} zaza unit_tests [testenv:venv] basepython = python3 -deps = -r{toxinidir}/test-requirements.txt commands = /bin/true [flake8] ignore = E402,E226,W504 -deps = -r{toxinidir}/test-requirements.txt per-file-ignores = unit_tests/**: D [testenv:docs] basepython = python3 changedir = doc/source -deps = - -r{toxinidir}/doc-requirements.txt commands = sphinx-build -W -b html -d {toxinidir}/doc/build/doctrees . {toxinidir}/doc/build/html [testenv:func] basepython = python3 -deps = -r{toxinidir}/test-requirements.txt commands = {envdir}/bin/python3 setup.py install functest-run-suite --keep-faulty-model [testenv:func-target] basepython = python3 -deps = -r{toxinidir}/test-requirements.txt commands = {envdir}/bin/python3 setup.py install functest-run-suite --keep-model --bundle {posargs} [testenv:func-target-extended] basepython = python3 -deps = -r{toxinidir}/test-requirements.txt commands = {envdir}/bin/python3 setup.py install functest-run-suite --keep-model --test-directory {toxinidir}/tests-extended --log INFO --bundle {posargs} [testenv:remove-placement] basepython = python3 -deps = -r{toxinidir}/test-requirements.txt commands = {envdir}/bin/python3 setup.py install remove-placement {posargs} diff --git a/unit_tests/test_zaza_model.py b/unit_tests/test_zaza_model.py index b000fd153..66fd9b1bd 100644 --- a/unit_tests/test_zaza_model.py +++ b/unit_tests/test_zaza_model.py @@ -149,6 +149,7 @@ async def _inner_is_leader(): self.run_action = mock.MagicMock() self.run_action.wait.side_effect = _wait self.action = mock.MagicMock() + self.action.wait.side_effect = _wait self.action.data = { 'model-uuid': '1a035018-71ff-473e-8aab-d1a8d6b6cda7', 'id': 'e26ffb69-6626-4e93-8840-07f7e041e99d', @@ -162,6 +163,9 @@ async def _inner_is_leader(): 'enqueued': '2018-04-11T23:13:42Z', 'started': '2018-04-11T23:13:42Z', 'completed': '2018-04-11T23:13:43Z'} + self.action.results = { + 'return-code': '0', 'stderr': '', 'stdout': 'RESULT' + } self.machine3 = mock.MagicMock(status='active') self.machine7 = mock.MagicMock(status='active') @@ -787,7 +791,47 @@ def test_run_on_unit(self): expected) self.unit1.run.assert_called_once_with(cmd, timeout=None) + def test_run_on_unit_juju2_x(self): + del self.action.results + self.patch_object(model, 'get_juju_model', return_value='mname') + expected = { + 'Code': '0', + 'Stderr': '', + 'Stdout': 'RESULT', + 'stderr': '', + 'stdout': 'RESULT'} + self.cmd = cmd = 'somecommand someargument' + self.patch_object(model, 'Model') + self.patch_object(model, 'get_unit_from_name') + self.get_unit_from_name.return_value = self.unit1 + self.Model.return_value = self.Model_mock + self.assertEqual(model.run_on_unit('app/2', cmd), + expected) + self.unit1.run.assert_called_once_with(cmd, timeout=None) + def test_run_on_unit_lc_keys(self): + self.patch_object(model, 'get_juju_model', return_value='mname') + self.action.results = { + 'return-code': '0', + 'stdout': 'RESULT', + 'stderr': 'some error'} + expected = { + 'Code': '0', + 'Stderr': 'some error', + 'Stdout': 'RESULT', + 'stderr': 'some error', + 'stdout': 'RESULT'} + self.cmd = cmd = 'somecommand someargument' + self.patch_object(model, 'Model') + self.patch_object(model, 'get_unit_from_name') + self.get_unit_from_name.return_value = self.unit1 + self.Model.return_value = self.Model_mock + self.assertEqual(model.run_on_unit('app/2', cmd), + expected) + self.unit1.run.assert_called_once_with(cmd, timeout=None) + + def test_run_on_unit_lc_keys_juju2_x(self): + del self.action.results self.patch_object(model, 'get_juju_model', return_value='mname') self.action.data['results'] = { 'Code': '0', @@ -809,6 +853,25 @@ def test_run_on_unit_lc_keys(self): self.unit1.run.assert_called_once_with(cmd, timeout=None) def test_run_on_unit_missing_stderr(self): + self.patch_object(model, 'get_juju_model', return_value='mname') + expected = { + 'Code': '0', + 'Stderr': '', + 'Stdout': 'RESULT', + 'stderr': '', + 'stdout': 'RESULT'} + self.action.results = {'return-code': '0', 'stdout': 'RESULT'} + self.cmd = cmd = 'somecommand someargument' + self.patch_object(model, 'Model') + self.patch_object(model, 'get_unit_from_name') + self.get_unit_from_name.return_value = self.unit1 + self.Model.return_value = self.Model_mock + self.assertEqual(model.run_on_unit('app/2', cmd), + expected) + self.unit1.run.assert_called_once_with(cmd, timeout=None) + + def test_run_on_unit_missing_stderr_juju2_x(self): + del self.action.results self.patch_object(model, 'get_juju_model', return_value='mname') expected = { 'Code': '0', @@ -841,6 +904,22 @@ def test_run_on_leader(self): expected) self.unit2.run.assert_called_once_with(cmd, timeout=None) + def test_run_on_leader_juju2_x(self): + del self.action.results + self.patch_object(model, 'get_juju_model', return_value='mname') + expected = { + 'Code': '0', + 'Stderr': '', + 'Stdout': 'RESULT', + 'stderr': '', + 'stdout': 'RESULT'} + self.cmd = cmd = 'somecommand someargument' + self.patch_object(model, 'Model') + self.Model.return_value = self.Model_mock + self.assertEqual(model.run_on_leader('app', cmd), + expected) + self.unit2.run.assert_called_once_with(cmd, timeout=None) + def test_get_relation_id(self): self.patch_object(model, 'get_juju_model', return_value='mname') self.patch_object(model, 'Model') @@ -1047,7 +1126,7 @@ async def _async_get_unit_from_name(x, *args): return units[x] self.async_get_unit_from_name.side_effect = _async_get_unit_from_name - self.run_action.status = 'completed' + self.run_action.data = {'status': 'completed'} model.run_action_on_units( ['app/1', 'app/2'], 'backup', @@ -1065,7 +1144,7 @@ def test_run_action_on_units_timeout(self): self.Model.return_value = self.Model_mock self.patch_object(model, 'get_unit_from_name') self.get_unit_from_name.return_value = self.unit1 - self.run_action.status = 'running' + self.run_action.data = {'status': 'running'} with self.assertRaises(AsyncTimeoutError): model.run_action_on_units( ['app/2'], @@ -1083,7 +1162,7 @@ async def _fake_get_action_output(_): self.Model.return_value = self.Model_mock self.patch_object(model, 'get_unit_from_name') self.get_unit_from_name.return_value = self.unit1 - self.run_action.status = 'failed' + self.run_action.data = {'status': 'failed'} with self.assertRaises(model.ActionFailed): model.run_action_on_units( ['app/2'], @@ -1540,6 +1619,25 @@ def test_get_current_model(self): self.assertEqual(model.get_current_model(), self.model_name) def test_file_contents_success(self): + self.action.results = { + 'return-code': '0', + 'stderr': '', + 'stdout': 'somestring' + } + + self.patch_object(model, 'Model') + self.Model.return_value = self.Model_mock + self.patch_object(model, 'get_juju_model', return_value='mname') + contents = model.file_contents( + 'app/2', + '/tmp/src/myfile.txt', + timeout=0.1) + self.unit1.run.assert_called_once_with( + 'cat /tmp/src/myfile.txt', timeout=0.1) + self.assertEqual('somestring', contents) + + def test_file_contents_success_juju2_x(self): + del self.action.results self.action.data = { 'results': {'Code': '0', 'Stderr': '', 'Stdout': 'somestring'} } @@ -1556,6 +1654,23 @@ def test_file_contents_success(self): self.assertEqual('somestring', contents) def test_file_contents_fault(self): + self.action.results = { + 'return-code': '0', + 'stderr': 'fault', + 'stdout': '' + } + + self.patch_object(model, 'Model') + self.Model.return_value = self.Model_mock + self.patch_object(model, 'get_juju_model', return_value='mname') + with self.assertRaises(model.RemoteFileError) as ctxtmgr: + model.file_contents('app/2', '/tmp/src/myfile.txt', timeout=0.1) + self.unit1.run.assert_called_once_with( + 'cat /tmp/src/myfile.txt', timeout=0.1) + self.assertEqual(ctxtmgr.exception.args, ('fault',)) + + def test_file_contents_fault_juju2_x(self): + del self.action.results self.action.data = { 'results': {'Code': '0', 'Stderr': 'fault', 'Stdout': ''} } @@ -1570,6 +1685,33 @@ def test_file_contents_fault(self): self.assertEqual(ctxtmgr.exception.args, ('fault',)) def test_block_until_file_has_contents(self): + self.action.results = { + 'return-code': '0', + 'stderr': '', + 'stdout': 'somestring' + } + + self.patch_object(model, 'Model') + self.Model.return_value = self.Model_mock + self.patch_object(model, 'get_juju_model', return_value='mname') + self.patch("builtins.open", + new_callable=mock.mock_open(), + name="_open") + _fileobj = mock.MagicMock() + _fileobj.__enter__().read.return_value = "somestring" + self._open.return_value = _fileobj + model.block_until_file_has_contents( + 'app', + '/tmp/src/myfile.txt', + 'somestring', + timeout=0.1) + self.unit1.run.assert_called_once_with( + 'cat /tmp/src/myfile.txt') + self.unit2.run.assert_called_once_with( + 'cat /tmp/src/myfile.txt') + + def test_block_until_file_has_contents_juju2_x(self): + del self.action.results self.action.data = { 'results': {'Code': '0', 'Stderr': '', 'Stdout': 'somestring'} } @@ -1594,6 +1736,29 @@ def test_block_until_file_has_contents(self): 'cat /tmp/src/myfile.txt') def test_block_until_file_has_no_contents(self): + self.action.results = {'return-code': '0', 'stderr': ''} + + self.patch_object(model, 'Model') + self.Model.return_value = self.Model_mock + self.patch_object(model, 'get_juju_model', return_value='mname') + self.patch("builtins.open", + new_callable=mock.mock_open(), + name="_open") + _fileobj = mock.MagicMock() + _fileobj.__enter__().read.return_value = "" + self._open.return_value = _fileobj + model.block_until_file_has_contents( + 'app', + '/tmp/src/myfile.txt', + '', + timeout=0.1) + self.unit1.run.assert_called_once_with( + 'cat /tmp/src/myfile.txt') + self.unit2.run.assert_called_once_with( + 'cat /tmp/src/myfile.txt') + + def test_block_until_file_has_no_contents_juju2_x(self): + del self.action.results self.action.data = { 'results': {'Code': '0', 'Stderr': ''} } @@ -1639,6 +1804,19 @@ def test_block_until_file_missing(self): self.patch_object(model, 'Model') self.Model.return_value = self.Model_mock self.patch_object(model, 'get_juju_model', return_value='mname') + self.action.results = {'stdout': "1"} + model.block_until_file_missing( + 'app', + '/tmp/src/myfile.txt', + timeout=0.1) + self.unit1.run.assert_called_once_with( + 'test -e "/tmp/src/myfile.txt"; echo $?') + + def test_block_until_file_missing_juju2_x(self): + self.patch_object(model, 'Model') + self.Model.return_value = self.Model_mock + self.patch_object(model, 'get_juju_model', return_value='mname') + del self.action.results self.action.data['results']['Stdout'] = "1" model.block_until_file_missing( 'app', @@ -1659,6 +1837,27 @@ def test_block_until_file_missing_isnt_missing(self): timeout=0.1) def test_block_until_file_matches_re(self): + self.action.results = { + 'return-code': '0', + 'stderr': '', + 'stdout': 'somestring' + } + + self.patch_object(model, 'Model') + self.Model.return_value = self.Model_mock + self.patch_object(model, 'get_juju_model', return_value='mname') + model.block_until_file_matches_re( + 'app', + '/tmp/src/myfile.txt', + 's.*string', + timeout=0.1) + self.unit1.run.assert_called_once_with( + 'cat /tmp/src/myfile.txt') + self.unit2.run.assert_called_once_with( + 'cat /tmp/src/myfile.txt') + + def test_block_until_file_matches_re_juju2_x(self): + del self.action.results self.action.data = { 'results': {'Code': '0', 'Stderr': '', 'Stdout': 'somestring'} } @@ -1965,6 +2164,24 @@ async def _run_on_unit( def block_until_oslo_config_entries_match_base(self, file_contents, expected_contents): + self.action.results = { + 'return-code': '0', + 'stderr': '', + 'stdout': file_contents + } + self.patch_object(model, 'Model') + self.patch_object(model, 'get_juju_model', return_value='mname') + self.Model.return_value = self.Model_mock + model.block_until_oslo_config_entries_match( + 'app', + '/tmp/src/myfile.txt', + expected_contents, + timeout=0.1) + + def block_until_oslo_config_entries_match_base_juju2_x(self, + file_contents, + expected_contents): + del self.action.results self.action.data = { 'results': {'Code': '0', 'Stderr': '', 'Stdout': file_contents} } diff --git a/unit_tests/utilities/test_zaza_utilities_juju.py b/unit_tests/utilities/test_zaza_utilities_juju.py index 05ea9cf5b..0ef6b863e 100644 --- a/unit_tests/utilities/test_zaza_utilities_juju.py +++ b/unit_tests/utilities/test_zaza_utilities_juju.py @@ -370,16 +370,86 @@ def test_get_machine_series(self): new_callable=mock.MagicMock(), name='_get_machine_status' ) - self._get_machine_status.return_value = 'xenial' + self._get_machine_status.return_value = {'series': 'xenial'} expected = 'xenial' actual = juju_utils.get_machine_series('6') self._get_machine_status.assert_called_with( machine='6', - key='series', model_name=None ) self.assertEqual(expected, actual) + def test_get_machine_series_juju3x_exceptions(self): + self.patch( + 'zaza.utilities.juju.get_machine_status', + new_callable=mock.MagicMock(), + name='_get_machine_status' + ) + self.patch( + 'zaza.utilities.juju.launchpad.get_ubuntu_series_by_version', + new_callable=mock.MagicMock(), + name='_get_ubuntu_series_by_version' + ) + self._get_ubuntu_series_by_version.return_value = { + '22.04': {'name': 'jammy'}} + + status = mock.MagicMock() + status.__getitem__.side_effect = KeyError + + base = mock.MagicMock() + base.name = 'ubuntu' + base.channel = '22.04/stable' + status.get.return_value = base + self._get_machine_status.return_value = status + + try: + juju_utils.get_machine_series('6') + except KeyError: + self.fail('Did not expect `get_machine_series` ' + 'to raise a KeyError') + self._get_machine_status.reset_mock() + + self._get_machine_status.return_value = {} + self.assertRaises( + ValueError, + juju_utils.get_machine_series, + '6') + + base = mock.MagicMock() + base.name = 'someOtherDistro' + base.channel = '22.04/stable' + self._get_machine_status.return_value = {'base': base} + self.assertRaises( + NotImplementedError, + juju_utils.get_machine_series, + '6') + + def test_get_machine_series_juju3x(self): + self.patch( + 'zaza.utilities.juju.get_machine_status', + new_callable=mock.MagicMock(), + name='_get_machine_status' + ) + self.patch( + 'zaza.utilities.juju.launchpad.get_ubuntu_series_by_version', + new_callable=mock.MagicMock(), + name='_get_ubuntu_series_by_version' + ) + base = mock.MagicMock() + base.name = 'ubuntu' + base.channel = '22.04/stable' + self._get_machine_status.return_value = {'base': base} + self._get_ubuntu_series_by_version.return_value = { + '22.04': {'name': 'jammy'}} + expected = 'jammy' + actual = juju_utils.get_machine_series('6') + self._get_machine_status.assert_called_with( + machine='6', + model_name=None + ) + self._get_ubuntu_series_by_version.assert_called_once_with() + self.assertEqual(expected, actual) + def test_get_subordinate_units(self): juju_status = mock.MagicMock() juju_status.applications = { diff --git a/unit_tests/utilities/test_zaza_utilities_launchpad.py b/unit_tests/utilities/test_zaza_utilities_launchpad.py new file mode 100644 index 000000000..b25e130e9 --- /dev/null +++ b/unit_tests/utilities/test_zaza_utilities_launchpad.py @@ -0,0 +1,55 @@ +# Copyright 2023 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import unittest + +import unit_tests.utils as ut_utils +import zaza.utilities.launchpad as launchpad + + +class TestUtilitiesLaunchpad(ut_utils.BaseTestCase): + + def test_get_ubuntu_series(self): + self.patch_object(launchpad.requests, 'get') + expect = {'entries': {}} + r = unittest.mock.MagicMock() + r.text = json.dumps(expect) + self.get.return_value = r + self.assertEquals( + launchpad.get_ubuntu_series(), + expect, + ) + self.get.assert_called_once_with( + 'https://api.launchpad.net/devel/ubuntu/series') + + def test_get_ubuntu_series_by_version(self): + self.patch_object(launchpad, 'get_ubuntu_series') + + self.get_ubuntu_series.return_value = { + 'entries': [{'version': 'fakeVersion'}]} + + self.assertEquals( + launchpad.get_ubuntu_series_by_version(), + {'fakeVersion': {'version': 'fakeVersion'}}) + + def test_get_ubuntu_series_by_name(self): + self.patch_object(launchpad, 'get_ubuntu_series') + + self.get_ubuntu_series.return_value = { + 'entries': [{'name': 'fakeName'}]} + + self.assertEquals( + launchpad.get_ubuntu_series_by_name(), + {'fakeName': {'name': 'fakeName'}}) diff --git a/zaza/charm_lifecycle/deploy.py b/zaza/charm_lifecycle/deploy.py index dc2d7b301..5ea880be7 100755 --- a/zaza/charm_lifecycle/deploy.py +++ b/zaza/charm_lifecycle/deploy.py @@ -339,7 +339,7 @@ def deploy_bundle(bundle, model, model_ctxt=None, force=False, trust=False): if trust: cmd.append('--trust') bundle_out = None - with tempfile.TemporaryDirectory() as tmpdirname: + with tempfile.TemporaryDirectory(dir=os.environ['HOME']) as tmpdirname: # Bundle templates should only exist in the bundle directory so # explicitly set the Jinja2 load path. bundle_template = get_template( diff --git a/zaza/model.py b/zaza/model.py index ccec8150f..04166394d 100644 --- a/zaza/model.py +++ b/zaza/model.py @@ -545,6 +545,9 @@ def _normalise_action_results(results): results[old_key] = results[key] elif results.get(old_key) and not results.get(key): results[key] = results[old_key] + if 'return-code' in results: + results['Code'] = str(results.get('return-code')) + del results['return-code'] return results else: return {} @@ -567,6 +570,8 @@ async def async_run_on_unit(unit_name, command, model_name=None, timeout=None): model = await get_model(model_name) unit = await async_get_unit_from_name(unit_name, model) action = await unit.run(command, timeout=timeout) + await action.wait() + action = _normalise_action_object(action) results = action.data.get('results') return _normalise_action_results(results) @@ -593,6 +598,8 @@ async def async_run_on_leader(application_name, command, model_name=None, is_leader = await unit.is_leader_from_status() if is_leader: action = await unit.run(command, timeout=timeout) + await action.wait() + action = _normalise_action_object(action) results = action.data.get('results') return _normalise_action_results(results) @@ -1071,6 +1078,32 @@ def __init__(self, action, output=None): super(ActionFailed, self).__init__(message) +def _normalise_action_object(action_obj): + """Put run action results in a consistent format. + + Prior to libjuju 3.x, action status and results are + in obj.data['status'] and obj.data['results']. + From libjuju 3.x, status and results are modified + to obj._status and obj.results. + This functiona normalises the status to + obj.data['status'] and results to obj.data['results'] + + :param action_obj: action object + :type results: juju.action.Action + :returns: Updated action object + :rtype: juju.action.Action + """ + try: + # libjuju 3.x + action_obj.data['status'] = action_obj._status + action_obj.data['results'] = action_obj.results + except (AttributeError, KeyError): + # libjuju 2.x format, no changes needed + pass + + return action_obj + + async def async_run_action(unit_name, action_name, model_name=None, action_params=None, raise_on_failure=False): """Run action on given unit. @@ -1096,7 +1129,8 @@ async def async_run_action(unit_name, action_name, model_name=None, unit = await async_get_unit_from_name(unit_name, model) action_obj = await unit.run_action(action_name, **action_params) await action_obj.wait() - if raise_on_failure and action_obj.status != 'completed': + action_obj = _normalise_action_object(action_obj) + if raise_on_failure and action_obj.data['status'] != 'completed': try: output = await model.get_action_output(action_obj.id) except KeyError: @@ -1136,7 +1170,8 @@ async def async_run_action_on_leader(application_name, action_name, action_obj = await unit.run_action(action_name, **action_params) await action_obj.wait() - if raise_on_failure and action_obj.status != 'completed': + action_obj = _normalise_action_object(action_obj) + if raise_on_failure and action_obj.data['status'] != 'completed': try: output = await model.get_action_output(action_obj.id) except KeyError: @@ -1183,14 +1218,14 @@ async def async_run_action_on_units(units, action_name, action_params=None, async def _check_actions(): for action_obj in actions: - if action_obj.status in ['running', 'pending']: + if action_obj.data['status'] in ['running', 'pending']: return False return True await async_block_until(_check_actions, timeout=timeout) for action_obj in actions: - if raise_on_failure and action_obj.status != 'completed': + if raise_on_failure and action_obj.data['status'] != 'completed': try: output = await model.get_action_output(action_obj.id) except KeyError: @@ -2081,7 +2116,13 @@ async def _check_file(): for unit in units: try: output = await unit.run('cat {}'.format(remote_file)) - contents = output.data.get('results').get('Stdout', '') + await output.wait() + results = {} + try: + results = output.results + except (AttributeError, KeyError): + results = output.data.get('results', {}) + contents = results.get('Stdout', results.get('stdout', '')) if inspect.iscoroutinefunction(check_function): if not await check_function(contents): return False @@ -2216,7 +2257,11 @@ async def _check_for_file(model): for unit in units: try: output = await unit.run('test -e "{}"; echo $?'.format(path)) - contents = output.data.get('results')['Stdout'] + await output.wait() + output = _normalise_action_object(output) + output_result = _normalise_action_results( + output.data.get('results', {})) + contents = output_result.get('Stdout', '') results.append("1" in contents) # libjuju throws a generic error for connection failure. So we # cannot differentiate between a connectivity issue and a diff --git a/zaza/utilities/deployment_env.py b/zaza/utilities/deployment_env.py index 81799e99f..d225e596c 100644 --- a/zaza/utilities/deployment_env.py +++ b/zaza/utilities/deployment_env.py @@ -47,7 +47,7 @@ MODEL_DEFAULTS = { # Model defaults from charm-test-infra # https://jujucharms.com/docs/2.1/models-config - 'default-series': 'xenial', + 'default-series': 'focal', 'image-stream': 'daily', 'test-mode': 'true', 'transmit-vendor-metrics': 'false', diff --git a/zaza/utilities/juju.py b/zaza/utilities/juju.py index fea84c980..208130c02 100644 --- a/zaza/utilities/juju.py +++ b/zaza/utilities/juju.py @@ -24,8 +24,9 @@ model, controller, ) -from zaza.utilities import generic as generic_utils from zaza.utilities import exceptions as zaza_exceptions +from zaza.utilities import generic as generic_utils +from zaza.utilities import launchpad KUBERNETES_PROVIDER_NAME = 'kubernetes' @@ -279,11 +280,28 @@ def get_machine_series(machine, model_name=None): :returns: Juju series :rtype: string """ - return get_machine_status( + status = get_machine_status( machine=machine, - key='series', model_name=model_name ) + try: + if 'series' in status: + return status.get('series') + except KeyError: + # libjuju will raise make the above check return KeyError when not + # present... + pass + + base = status.get('base') + if not base: + raise ValueError("Unable to determine distro from status: '{}'" + .format(status)) + if base.name != 'ubuntu': + raise NotImplementedError("Series resolution not implemented for " + "distro: '{}'".format(base.name)) + + version, risk = base.channel.split('/') + return launchpad.get_ubuntu_series_by_version()[version]['name'] def get_machine_uuids_for_application(application, model_name=None): diff --git a/zaza/utilities/launchpad.py b/zaza/utilities/launchpad.py new file mode 100644 index 000000000..4ba7ee429 --- /dev/null +++ b/zaza/utilities/launchpad.py @@ -0,0 +1,52 @@ +# Copyright 2023 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Module for interacting with Launchpad API.""" + +import json +import requests +import typing + + +def get_ubuntu_series( +) -> typing.Dict[str, typing.List[typing.Dict[str, any]]]: + """Contact Launchpad API and retrieve a list of all Ubuntu releases. + + Launchpad documentation for the returned data structure can be found here: + https://launchpad.net/+apidoc/devel.html#distribution + https://launchpad.net/+apidoc/devel.html#distro_series + """ + r = requests.get('https://api.launchpad.net/devel/ubuntu/series') + return json.loads(r.text) + + +def get_ubuntu_series_by_version() -> typing.Dict[str, typing.Dict[str, any]]: + """Get a Dict of distro series information indexed by version number. + + Please refer to the `get_ubuntu_series()` function docstring for docs. + """ + return { + entry['version']: entry + for entry in get_ubuntu_series().get('entries', {}) + } + + +def get_ubuntu_series_by_name() -> typing.Dict[str, typing.Dict[str, any]]: + """Get a Dict of distro series information indexed by version name. + + Please refer to the `get_ubuntu_series()` function docstring for docs. + """ + return { + entry['name']: entry + for entry in get_ubuntu_series().get('entries', {}) + }