Test template sync using proxy

pytest_fixtures/component/http_proxy.py

@@ -31,3 +31,29 @@ def setup_http_proxy(request, module_manifest_org, target_sat):
     target_sat.update_setting('http_proxy', general_proxy_value)
     if http_proxy:
         http_proxy.delete()
+
+
+@pytest.fixture
+def setup_http_proxy_without_global_settings(request, module_manifest_org, target_sat):
+    """Create a new HTTP proxy but don't set it as global or content proxy"""
+    http_proxy = target_sat.api_factory.make_http_proxy(module_manifest_org, request.param)
+    yield http_proxy, request.param
+    if http_proxy:
+        http_proxy.delete()
+
+
+@pytest.fixture
+def setup_http_proxy_global(request, target_sat):
+    """Create a new HTTP proxy and set related settings based on proxy"""
+    if request.param:
+        hostname = settings.http_proxy.auth_proxy_url[7:]
+        general_proxy = (
+            f'http://{settings.http_proxy.username}:' f'{settings.http_proxy.password}@{hostname}'
+        )
+    else:
+        general_proxy = settings.http_proxy.un_auth_proxy_url
+    general_proxy_value = target_sat.update_setting(
+        'http_proxy', general_proxy if request.param is not None else ''
+    )
+    yield general_proxy, request.param
+    target_sat.update_setting('http_proxy', general_proxy_value)

robottelo/host_helpers/capsule_mixins.py

@@ -4,6 +4,8 @@
 from box import Box
 from dateutil.parser import parse
 
+from robottelo import ssh
+from robottelo.config import settings
 from robottelo.constants import (
     PULP_ARTIFACT_DIR,
     PUPPET_CAPSULE_INSTALLER,
@@ -189,3 +191,35 @@ def get_artifact_info(self, checksum=None, path=None):
         info = self.execute(f'file {path}').stdout.strip().split(': ')[1]
 
         return Box(path=path, size=size, sum=real_sum, info=info)
+
+    def cutoff_host_setup_log(self, proxy_hostname, hostname):
+        """For testing of HTTP Proxy, disable direct connection to some host using firewall. On the Proxy, setup logs for later comparison that the Proxy was used."""
+        log_path = '/var/log/squid/access.log'
+        old_log = ssh.command('echo /tmp/$RANDOM').stdout.strip()
+        ssh.command(
+            f'sshpass -p "{settings.server.ssh_password}" scp -o StrictHostKeyChecking=no root@{proxy_hostname}:{log_path} {old_log}'
+        )
+        # make sure the system can't communicate with the git directly, without proxy
+        assert (
+            self.execute(
+                f'firewall-cmd --permanent --direct --add-rule ipv4 filter OUTPUT 1 -d $(dig +short A {hostname}) -j REJECT && firewall-cmd --reload'
+            ).status
+            == 0
+        )
+        assert self.execute(f'ping -c 2 {hostname}').status != 0
+        return old_log
+
+    def restore_host_check_log(self, proxy_hostname, hostname, old_log):
+        """For testing of HTTP Proxy, call after running the thing that should use Proxy."""
+        log_path = '/var/log/squid/access.log'
+        self.execute(
+            f'firewall-cmd --permanent --direct --remove-rule ipv4 filter OUTPUT 1 -d $(dig +short A {hostname}) -j REJECT && firewall-cmd --reload'
+        )
+        new_log = ssh.command('echo /tmp/$RANDOM').stdout.strip()
+        ssh.command(
+            f'sshpass -p "{settings.server.ssh_password}" scp -o StrictHostKeyChecking=no root@{proxy_hostname}:{log_path} {new_log}'
+        )
+        diff = ssh.command(f'diff {old_log} {new_log}').stdout
+        satellite_ip = ssh.command('dig A +short $(hostname)').stdout.strip()
+        # assert that proxy has been used
+        assert satellite_ip in diff

tests/foreman/api/test_templatesync.py

@@ -28,6 +28,7 @@
     FOREMAN_TEMPLATES_NOT_IMPORTED_COUNT,
 )
 from robottelo.logging import logger
+from robottelo.utils.issue_handlers import is_open
 
 git = settings.git
 
@@ -1078,13 +1079,34 @@ def test_positive_export_log_to_production(
         indirect=True,
         ids=['non_empty_repo', 'empty_repo'],
     )
+    @pytest.mark.parametrize(
+        'use_proxy',
+        [True, False],
+        ids=['use_proxy', 'do_not_use_proxy'],
+    )
+    @pytest.mark.parametrize(
+        'setup_http_proxy_without_global_settings',
+        [True, False],
+        indirect=True,
+        ids=['auth_http_proxy', 'unauth_http_proxy'],
+    )
     def test_positive_export_all_templates_to_repo(
-        self, module_org, git_repository, git_branch, url, module_target_sat
+        self,
+        module_org,
+        git_repository,
+        git_branch,
+        url,
+        module_target_sat,
+        use_proxy,
+        setup_http_proxy_without_global_settings,
     ):
         """Assure all templates are exported if no filter is specified.
 
         :id: 0bf6fe77-01a3-4843-86d6-22db5b8adf3b
 
+        :setup:
+            1. If using proxy, disable direct connection to the git instance
+
         :steps:
             1. Using nailgun export all templates to repository (ensure filters are empty)
 
@@ -1097,21 +1119,43 @@ def test_positive_export_all_templates_to_repo(
 
         :CaseImportance: Low
         """
-        output = module_target_sat.api.Template().exports(
-            data={
+        # TODO remove this
+        if is_open('SAT-28933') and 'ssh' in url:
+            pytest.skip("Temporary skip of SSH tests")
+        proxy, param = setup_http_proxy_without_global_settings
+        if not use_proxy and not param:
+            # only do-not-use one kind of proxy
+            pytest.skip(
+                "Invalid parameter combination. DO NOT USE PROXY scenario should only be tested once."
+            )
+        try:
+            data = {
                 'repo': f'{url}/{git.username}/{git_repository["name"]}',
                 'branch': git_branch,
                 'organization_ids': [module_org.id],
             }
-        )
-        auth = (git.username, git.password)
-        api_url = f'http://{git.hostname}:{git.http_port}/api/v1/repos/{git.username}'
-        res = requests.get(
-            url=f'{api_url}/{git_repository["name"]}/git/trees/{git_branch}',
-            auth=auth,
-            params={'recursive': True},
-        )
-        res.raise_for_status()
+            if use_proxy:
+                proxy_hostname = proxy.url.split('/')[2].split(':')[0]
+                old_log = module_target_sat.cutoff_host_setup_log(
+                    proxy_hostname, settings.git.hostname
+                )
+                data['http_proxy_policy'] = 'selected'
+                data['http_proxy_id'] = proxy.id
+            output = module_target_sat.api.Template().exports(data=data)
+            auth = (git.username, git.password)
+            api_url = f'http://{git.hostname}:{git.http_port}/api/v1/repos/{git.username}'
+            res = requests.get(
+                url=f'{api_url}/{git_repository["name"]}/git/trees/{git_branch}',
+                auth=auth,
+                params={'recursive': True},
+            )
+            res.raise_for_status()
+        finally:
+            if use_proxy:
+                module_target_sat.restore_host_check_log(
+                    proxy_hostname, settings.git.hostname, old_log
+                )
+
         try:
             tree = json.loads(res.text)['tree']
         except json.decoder.JSONDecodeError:

tests/foreman/cli/test_templatesync.py

@@ -21,6 +21,7 @@
     FOREMAN_TEMPLATE_IMPORT_URL,
     FOREMAN_TEMPLATE_TEST_TEMPLATE,
 )
+from robottelo.utils.issue_handlers import is_open
 
 git = settings.git
 
@@ -104,6 +105,89 @@ def test_positive_import_force_locked_template(
         else:
             pytest.fail('The template is not imported for force test')
 
+    @pytest.mark.skip_if_not_set('git')
+    @pytest.mark.parametrize(
+        'url',
+        [
+            'https://github.com/theforeman/community-templates.git',
+            'ssh://[email protected]/theforeman/community-templates.git',
+        ],
+        ids=['http', 'ssh'],
+    )
+    @pytest.mark.parametrize(
+        'setup_http_proxy_global',
+        [True, False],
+        indirect=True,
+        ids=['auth_http_proxy_global', 'unauth_http_proxy_global'],
+    )
+    @pytest.mark.parametrize(
+        'use_proxy_global',
+        [True, False],
+        ids=['use_proxy_global', 'do_not_use_proxy_global'],
+    )
+    @pytest.mark.tier2
+    def test_positive_import_dir_filtered(
+        self,
+        module_org,
+        create_import_export_local_dir,
+        target_sat,
+        use_proxy_global,
+        setup_http_proxy_global,
+        url,
+    ):
+        """Import a template from git, specifying directory and filter
+
+        :id: 17bfb25a-e215-4f57-b861-294cd018bcf1
+
+        :setup:
+            1. Unlock and remove a template to be imported
+
+        :steps:
+            1. Import a template, specifying its dir and filter
+
+        :expectedresults:
+            1. The template is present
+
+        :CaseImportance: Medium
+        """
+        # TODO remove this
+        if is_open('SAT-28933') and 'ssh' in url:
+            pytest.skip("Temporary skip of SSH tests")
+        proxy, param = setup_http_proxy_global
+        if not use_proxy_global and not param:
+            # only do-not-use one kind of proxy
+            pytest.skip(
+                "Invalid parameter combination. DO NOT USE PROXY scenario should only be tested once."
+            )
+        pt_name = 'FreeBSD default fake'
+        if target_sat.cli.PartitionTable.list({'search': f'name=\\"{pt_name}\\"'}):
+            target_sat.cli.PartitionTable.update({'name': pt_name, 'locked': 0})
+            target_sat.cli.PartitionTable.delete({'name': pt_name})
+        try:
+            data = {
+                'repo': url,
+                'organization-ids': module_org.id,
+                'branch': 'develop',
+                'dirname': '/partition_tables_templates/',
+                'filter': pt_name,
+            }
+            if use_proxy_global:
+                proxy_hostname = (
+                    proxy.split('/')[2].split(':')[0]
+                    if '@' not in proxy
+                    else proxy.split('@')[1].split(':')[0]
+                )
+                old_log = target_sat.cutoff_host_setup_log(proxy_hostname, settings.git.hostname)
+                data['http-proxy-policy'] = 'global'
+            target_sat.cli.TemplateSync.imports(data)
+        finally:
+            if use_proxy_global:
+                target_sat.restore_host_check_log(proxy_hostname, settings.git.hostname, old_log)
+        # assert that template has been synced -> is present on the Satellite
+        pt = target_sat.cli.PartitionTable.list({'search': f'name=\\"{pt_name}\\"'})
+        assert len(pt) == 1
+        assert pt_name == pt[0]['name']
+
     @pytest.mark.e2e
     @pytest.mark.tier2
     @pytest.mark.skip_if_not_set('git')
@@ -154,7 +238,6 @@ def test_positive_update_templates_in_git(
             f'{api_url}/{path}', auth=auth, json={'branch': git_branch, 'content': content}
         )
         assert res.status_code == 201
-        # export template to git
         url = f'{url}/{git.username}/{git_repository["name"]}'
         output = module_target_sat.cli.TemplateSync.exports(
             {

tests/foreman/ui/test_templatesync.py

@@ -31,9 +31,28 @@ def templates_loc(templates_org, module_target_sat):
 git = settings.git
 
 
+@pytest.mark.skip_if_not_set('git')
+@pytest.mark.parametrize(
+    'setup_http_proxy_without_global_settings',
+    [True, False],
+    indirect=True,
+    ids=['auth_http_proxy', 'unauth_http_proxy'],
+)
+@pytest.mark.parametrize(
+    'use_proxy',
+    [True, False],
+    ids=['use_proxy', 'do_not_use_proxy'],
+)
 @pytest.mark.tier2
 @pytest.mark.upgrade
-def test_positive_import_templates(session, templates_org, templates_loc):
+def test_positive_import_templates(
+    session,
+    templates_org,
+    templates_loc,
+    use_proxy,
+    setup_http_proxy_without_global_settings,
+    target_sat,
+):
     """Import template(s) from external source to satellite
 
     :id: 524bf384-703f-48a5-95ff-7c1cf97db694
@@ -58,34 +77,52 @@ def test_positive_import_templates(session, templates_org, templates_loc):
 
     :CaseImportance: Critical
     """
+    proxy, param = setup_http_proxy_without_global_settings
+    if not use_proxy and not param:
+        # only do-not-use one kind of proxy
+        pytest.skip(
+            "Invalid parameter combination. DO NOT USE PROXY scenario should only be tested once."
+        )
     import_template = 'Alterator default PXELinux'
     branch = 'automation'
     prefix_name = gen_string('alpha', 8)
-    with session:
-        session.organization.select(org_name=templates_org.name)
-        session.location.select(loc_name=templates_loc.name)
-        import_title = session.sync_template.sync(
-            {
-                'sync_type': 'Import',
-                'template.associate': 'Always',
-                'template.branch': branch,
-                'template.dirname': 'provisioning_templates',
-                'template.filter': import_template,
-                'template.lock': 'Lock',
-                'template.prefix': f'{prefix_name} ',
-                'template.repo': FOREMAN_TEMPLATE_IMPORT_URL,
-            }
-        )
-        assert import_title == f'Import from {FOREMAN_TEMPLATE_IMPORT_URL} and branch {branch}'
-        imported_template = f'{prefix_name} {import_template}'
-        assert session.provisioningtemplate.is_locked(imported_template)
-        pt = session.provisioningtemplate.read(imported_template)
-        assert pt['template']['name'] == imported_template
-        assert pt['template']['default'] is False
-        assert pt['type']['snippet'] is False
-        assert pt['locations']['resources']['assigned'][0] == templates_loc.name
-        assert pt['organizations']['resources']['assigned'][0] == templates_org.name
-        assert f'name: {import_template}' in pt['template']['template_editor']['editor']
+    # put the http proxy in the correct org and loc
+    target_sat.cli.HttpProxy.update(
+        {'id': proxy.id, 'organization-ids': [templates_org.id], 'location-ids': [templates_loc.id]}
+    )
+    try:
+        data = {
+            'sync_type': 'Import',
+            'template.associate': 'Always',
+            'template.branch': branch,
+            'template.dirname': 'provisioning_templates',
+            'template.filter': import_template,
+            'template.lock': 'Lock',
+            'template.prefix': f'{prefix_name} ',
+            'template.repo': FOREMAN_TEMPLATE_IMPORT_URL,
+        }
+        if use_proxy:
+            proxy_hostname = proxy.url.split('/')[2].split(':')[0]
+            old_log = target_sat.cutoff_host_setup_log(proxy_hostname, settings.git.hostname)
+            data['template.http_proxy_policy'] = 'Use selected HTTP proxy'
+            data['template.http_proxy_id'] = proxy.name
+        with session:
+            session.organization.select(org_name=templates_org.name)
+            session.location.select(loc_name=templates_loc.name)
+            import_title = session.sync_template.sync(data)
+            assert import_title == f'Import from {FOREMAN_TEMPLATE_IMPORT_URL} and branch {branch}'
+            imported_template = f'{prefix_name} {import_template}'
+            assert session.provisioningtemplate.is_locked(imported_template)
+            pt = session.provisioningtemplate.read(imported_template)
+    finally:
+        if use_proxy:
+            target_sat.restore_host_check_log(proxy_hostname, settings.git.hostname, old_log)
+    assert pt['template']['name'] == imported_template
+    assert pt['template']['default'] is False
+    assert pt['type']['snippet'] is False
+    assert pt['locations']['resources']['assigned'][0] == templates_loc.name
+    assert pt['organizations']['resources']['assigned'][0] == templates_org.name
+    assert f'name: {import_template}' in pt['template']['template_editor']['editor']
 
 
 @pytest.mark.tier2