Merge pull request #605 from coreycb/zed-overlay-ppas

Zed overlay ppas

This also includes several cherry-picks to get the gate passing for this branch.

Cherry-picks included are:

Add overlay_ppa to tests_options
(cherry picked from commit 0c07920)
Updates to overlay_ppa
(cherry picked from commit 9cc938a)
Fix NoneType error in enumerate(overlay_ppas)
(cherry picked from commit 986474b)
Misc fixes for failing github workflows
(cherry picked from commit 9277a94)
(cherry picked from commit ee92b9b)
Drop py36, py37, and py39 from github runners
(cherry picked from commit 6122daa)
Unpin pip and virtualenv
(cherry picked from commit 55c3f3f)
Bump up flake8
(cherry picked from commit a9d6812)
Migrate from nosetest to pytest
(cherry picked from commit 4b0d23e)

.github/workflows/tox.yaml

@@ -9,7 +9,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ['3.6', '3.7', '3.8', '3.9', '3.10']
+        python-version: ['3.8', '3.10']
 
     steps:
     - uses: actions/checkout@v1
@@ -35,21 +35,11 @@ jobs:
       fail-fast: false
       matrix:
         juju_channel:
-          - latest/stable
           - 2.9/stable
-          - 2.8/stable
         bundle:
           - first
           - second
           - third
-        exclude:
-          # disable 'first' and 'second' bundles for juju 2.8 since 'magpie'
-          # is not a promulgated charm in the charmstore, only on charmhub
-          # which 2.8 can't talk to.
-          - juju_channel: 2.8/stable
-            bundle: first
-          - juju_channel: 2.8/stable
-            bundle: second
     env:
       TEST_ZAZA_BUG_LP1987332: "on"  # http://pad.lv/1987332
     needs: build
@@ -67,6 +57,8 @@ jobs:
         sudo chmod 666 /var/snap/lxd/common/lxd/unix.socket
         # until Juju provides stable IPv6-support we unfortunately need this
         lxc network set lxdbr0 ipv6.address none
+        sudo iptables -F FORWARD
+        sudo iptables -P FORWARD ACCEPT
         # pull images
         lxc image copy --alias juju/bionic/amd64 --copy-aliases ubuntu-daily:bionic local:
         lxc image copy --alias juju/focal/amd64 --copy-aliases ubuntu-daily:focal local:

test-requirements.txt

@@ -1,14 +1,13 @@
 aiounittest
-flake8>=2.2.4
+flake8>=5  # Python 3.8 compatibility in pyflakes 2.1.0+
 flake8-docstrings
 flake8-per-file-ignores
 pydocstyle<4.0.0
 coverage
 mock>=1.2
-# For some reason the PyPi distributed wheel of nose differ from the one on
-# GitHub, and it does not work with Python 3.10.
-nose>=1.3.7;python_version<'3.10'
-git+https://github.com/nose-devs/nose.git@release_1.3.7#egg=nose;python_version=='3.10'
+pytest
+pytest-cov
+pytest-asyncio
 
 # TODO: these requirements should be mocked out in unit_tests/__init__.py
 async_generator

tox.ini

@@ -2,28 +2,24 @@
 envlist = pep8,py3
 skipsdist = True
 
-# NOTES:
-# * We avoid the new dependency resolver by pinning pip < 20.3, see
-#   https://github.com/pypa/pip/issues/9187
-# * Pinning dependencies requires tox >= 3.2.0, see
-#   https://tox.readthedocs.io/en/latest/config.html#conf-requires
-# * It is also necessary to pin virtualenv as a newer virtualenv would still
-#   lead to fetching the latest pip in the func* tox targets, see
-#   https://stackoverflow.com/a/38133283
-requires = pip < 20.3
-           virtualenv < 20.0
 # NOTE: https://wiki.canonical.com/engineering/OpenStack/InstallLatestToxOnOsci
 minversion = 3.2.0
 
 [testenv]
-setenv = VIRTUAL_ENV={envdir}
-         PYTHONHASHSEED=0
-whitelist_external = juju
-passenv = HOME TERM CS_* OS_* TEST_*
-deps = -r{toxinidir}/test-requirements.txt
-install_command =
-  {toxinidir}/pip.sh install {opts} {packages}
-commands = nosetests --with-coverage --processes=0 --cover-package=zaza {posargs} {toxinidir}/unit_tests
+setenv =
+  VIRTUAL_ENV={envdir}
+  PYTHONHASHSEED=0
+allowlist_external =
+  juju
+passenv =
+  HOME
+  TERM
+  CS_*
+  OS_*
+  TEST_*
+deps =
+  -r{toxinidir}/test-requirements.txt
+commands = pytest --cov=./zaza/ {posargs} {toxinidir}/unit_tests
 
 [testenv:py3]
 basepython = python3

unit_tests/test_zaza_model.py

@@ -32,6 +32,7 @@
 import concurrent
 import datetime
 import mock
+import pytest
 import yaml
 
 import unit_tests.utils as ut_utils
@@ -100,6 +101,7 @@ def tearDownModule():
                                 }}}}}}
 
 
+@pytest.mark.asyncio
 class TestModel(ut_utils.BaseTestCase):
 
     def setUp(self):
@@ -493,7 +495,9 @@ def test_block_until_auto_reconnect_model_disconnected_sync(self):
         with mock.patch.object(zaza, 'RUN_LIBJUJU_IN_THREAD', new=False):
             model.sync_wrapper(self._wrapper)()
         self.Model_mock.disconnect.assert_has_calls([mock.call()])
-        self.Model_mock.connect_model.has_calls([mock.call('modelname')])
+        self.Model_mock.connect_model.assert_has_calls(
+            [mock.call('testmodel')]
+        )
 
     def test_block_until_auto_reconnect_model_disconnected_async(self):
         self._mocks_for_block_until_auto_reconnect_model(
@@ -506,7 +510,9 @@ async def _async_true():
         with mock.patch.object(zaza, 'RUN_LIBJUJU_IN_THREAD', new=False):
             model.sync_wrapper(self._wrapper)()
         self.Model_mock.disconnect.assert_has_calls([mock.call()])
-        self.Model_mock.connect_model.has_calls([mock.call('modelname')])
+        self.Model_mock.connect_model.assert_has_calls(
+            [mock.call('testmodel')]
+        )
 
     def test_block_until_auto_reconnect_model_blocks_till_true(self):
         self._mocks_for_block_until_auto_reconnect_model(True, True)

unit_tests/utilities/test_deployment_env.py

@@ -12,11 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import collections
 import copy
 import mock
 import yaml
 
 import zaza.utilities.deployment_env as deployment_env
+import zaza.utilities.ro_types as ro_types
 import unit_tests.utils as ut_utils
 
 
@@ -52,40 +54,98 @@ def test_parse_option_list_string_whitespace(self):
                 'test-mode': 'false',
                 'image-stream': 'released'})
 
+    def test_get_overlay_ppas(self):
+        with mock.patch('zaza.global_options.get_options') as get_options_mock:
+            config = collections.OrderedDict({'overlay_ppas':
+                                              ['ppa:ppa1', 'ppa:ppa2']})
+            get_options_mock.return_value = ro_types.resolve_immutable(config)
+            self.assertEqual(deployment_env.get_overlay_ppas(),
+                             ro_types.ReadOnlyList(['ppa:ppa1', 'ppa:ppa2']))
+
+            config = collections.OrderedDict({'force_deploy': 'x-y'})
+            get_options_mock.return_value = ro_types.resolve_immutable(config)
+            self.assertEqual(deployment_env.get_overlay_ppas(), None)
+
+    def test_get_cloudinit_userdata(self):
+        with mock.patch.object(deployment_env, 'get_overlay_ppas',
+                               return_value=['ppa:ppa0', 'ppa:ppa1']):
+            cloud_config = {
+                'apt': {
+                    'sources': {
+                        'overlay-ppa-0': {
+                            'source': 'ppa:ppa0'
+                        },
+                        'overlay-ppa-1': {
+                            'source': 'ppa:ppa1'
+                        }
+                    }
+                }
+            }
+            cloudinit_userdata = "#cloud-config\n{}".format(
+                yaml.safe_dump(cloud_config))
+            self.assertEqual(
+                deployment_env.get_cloudinit_userdata(),
+                cloudinit_userdata)
+
     def base_get_model_settings(self, env, expect):
         with mock.patch.dict(deployment_env.os.environ, env):
             self.assertEqual(deployment_env.get_model_settings(), expect)
 
     def test_get_model_settings_no_config(self):
-        self.base_get_model_settings({}, self.MODEL_CONFIG_DEFAULTS)
+        with mock.patch.object(deployment_env, 'get_cloudinit_userdata',
+                               return_value=None):
+            self.base_get_model_settings({}, self.MODEL_CONFIG_DEFAULTS)
 
     def test_get_model_settings_multiple_values_override(self):
-        expect_config = copy.deepcopy(self.MODEL_CONFIG_DEFAULTS)
-        expect_config.update({'test-mode': 'false'})
-        self.base_get_model_settings(
-            {'TEST_MODEL_SETTINGS': 'test-mode=false'},
-            expect_config)
+        with mock.patch.object(deployment_env, 'get_cloudinit_userdata',
+                               return_value=None):
+            expect_config = copy.deepcopy(self.MODEL_CONFIG_DEFAULTS)
+            expect_config.update({'test-mode': 'false'})
+            self.base_get_model_settings(
+                {'TEST_MODEL_SETTINGS': 'test-mode=false'},
+                expect_config)
 
     def test_get_model_settings_file_override(self):
-        expect_config = copy.deepcopy(self.MODEL_CONFIG_DEFAULTS)
-        expect_config.update({'default-series': 'file-setting'})
-        self.patch_object(
-            deployment_env,
-            'get_setup_file_section',
-            return_value={'default-series': 'file-setting'})
-        self.base_get_model_settings({}, expect_config)
+        with mock.patch.object(deployment_env, 'get_cloudinit_userdata',
+                               return_value=None):
+            expect_config = copy.deepcopy(self.MODEL_CONFIG_DEFAULTS)
+            expect_config.update({'default-series': 'file-setting'})
+            self.patch_object(
+                deployment_env,
+                'get_setup_file_section',
+                return_value={'default-series': 'file-setting'})
+            self.base_get_model_settings({}, expect_config)
 
     def test_get_model_settings_file_override_env_override(self):
-        # Check that env variables override defaults and file
-        expect_config = copy.deepcopy(self.MODEL_CONFIG_DEFAULTS)
-        expect_config.update({'default-series': 'env-setting'})
-        self.patch_object(
-            deployment_env,
-            'get_setup_file_section',
-            return_value={'default-series': 'file-setting'})
-        self.base_get_model_settings(
-            {'TEST_MODEL_SETTINGS': 'default-series=env-setting'},
-            expect_config)
+        with mock.patch.object(deployment_env, 'get_cloudinit_userdata',
+                               return_value=None):
+            # Check that env variables override defaults and file
+            expect_config = copy.deepcopy(self.MODEL_CONFIG_DEFAULTS)
+            expect_config.update({'default-series': 'env-setting'})
+            self.patch_object(
+                deployment_env,
+                'get_setup_file_section',
+                return_value={'default-series': 'file-setting'})
+            self.base_get_model_settings(
+                {'TEST_MODEL_SETTINGS': 'default-series=env-setting'},
+                expect_config)
+
+    def test_get_model_settings_cloudinit_userdata(self):
+        with mock.patch.object(deployment_env, 'get_cloudinit_userdata',
+                               return_value='x'):
+            expect_config = copy.deepcopy(self.MODEL_CONFIG_DEFAULTS)
+            expect_config.update({'cloudinit-userdata': 'x'})
+            self.base_get_model_settings({}, expect_config)
+
+            with mock.patch.object(deployment_env.logging, 'warn') as warn:
+                with mock.patch.dict(deployment_env.os.environ,
+                                     {'TEST_MODEL_SETTINGS':
+                                      "cloudinit-userdata=y"}):
+                    expect_config.update({'cloudinit-userdata': 'y'})
+                    self.base_get_model_settings({}, expect_config)
+                    warn.assert_called_once_with(
+                        'TEST_MODEL_SETTINGS contains cloudinit-userdata '
+                        'which overrides tests_options overlay_ppas')
 
     def base_get_model_constraints(self, env, expect):
         with mock.patch.dict(deployment_env.os.environ, env):

unit_tests/utilities/test_zaza_utilities_generic.py

@@ -254,7 +254,7 @@ def test_series_upgrade(self):
             _unit, _machine_num, origin=_origin,
             to_series=_to_series, from_series=_from_series,
             workaround_script=_workaround_script, files=_files)
-        self.block_until_all_units_idle.called_with()
+        self.block_until_all_units_idle.assert_called_with()
         self.prepare_series_upgrade.assert_called_once_with(
             _machine_num, to_series=_to_series)
         self.wrap_do_release_upgrade.assert_called_once_with(

zaza/utilities/deployment_env.py

@@ -19,6 +19,7 @@
 import functools
 import yaml
 
+import zaza.global_options
 import zaza.model
 
 ZAZA_SETUP_FILE_LOCATIONS = [
@@ -84,6 +85,54 @@ def parse_option_list_string(option_list, delimiter=None):
     return settings
 
 
+def get_overlay_ppas():
+    """Get overlay_ppas from global_config.
+
+    In the config file for the tests, the tests_options.overlay_ppa option
+    may be used to specify one or more PPAs that will be enabled for all
+    units in the model.
+
+    The tests_options section needs to look like:
+
+        tests_options:
+          overlay_ppas:
+            - ppa:ubuntu-security-proposed/ppa
+
+    :returns: List of overlay PPAs
+    :rtype: list[str]
+    """
+    config = zaza.global_options.get_options()
+    try:
+        return config.overlay_ppas
+    except KeyError:
+        pass
+    return None
+
+
+def get_cloudinit_userdata():
+    """Return cloudinit_userdata based on tests_options config.
+
+    :returns: YAML-formatted string of cloudinit_userdata
+    :rtype: str
+    """
+    cloudinit_userdata = None
+    cloud_config = {
+        'apt': {
+            'sources': {
+            }
+        }
+    }
+    overlay_ppas = get_overlay_ppas()
+    if overlay_ppas:
+        for index, overlay_ppa in enumerate(overlay_ppas):
+            cloud_config['apt']['sources']["overlay-ppa-{}".format(index)] = {
+                'source': overlay_ppa
+            }
+        cloudinit_userdata = "#cloud-config\n{}".format(
+            yaml.safe_dump(cloud_config))
+    return cloudinit_userdata
+
+
 def get_model_settings():
     """Return model settings from defaults, config file and env variables.
 
@@ -94,6 +143,12 @@ def get_model_settings():
     model_settings.update(get_setup_file_section(MODEL_SETTINGS_SECTION))
     env_settings = os.environ.get('MODEL_SETTINGS', '')
     test_env_settings = os.environ.get('TEST_MODEL_SETTINGS', '')
+    cloudinit_userdata = get_cloudinit_userdata()
+    if cloudinit_userdata:
+        if 'cloudinit-userdata' in test_env_settings:
+            logging.warn('TEST_MODEL_SETTINGS contains cloudinit-userdata '
+                         'which overrides tests_options overlay_ppas')
+        model_settings.update({'cloudinit-userdata': cloudinit_userdata})
     model_settings.update(
         parse_option_list_string(test_env_settings or env_settings))
     if env_settings: