From 7afd66b306681d8ad307486f497336dbd4183f71 Mon Sep 17 00:00:00 2001 From: Marius Leustean Date: Fri, 3 Feb 2023 15:39:36 +0200 Subject: [PATCH] WIP: Shard affinity for k8s workloads --- nova/scheduler/filters/shard_filter.py | 105 +++++++++++- .../functional/db/test_console_auth_token.py | 4 +- .../scheduler/filters/test_shard_filter.py | 161 ++++++++++++++---- .../weights/test_weights_resize_same_host.py | 2 +- 4 files changed, 236 insertions(+), 36 deletions(-) diff --git a/nova/scheduler/filters/shard_filter.py b/nova/scheduler/filters/shard_filter.py index 9393b069be2..1b73f35d29c 100644 --- a/nova/scheduler/filters/shard_filter.py +++ b/nova/scheduler/filters/shard_filter.py @@ -19,6 +19,9 @@ from oslo_log import log as logging import nova.conf +from nova import context as nova_context +from nova.objects.build_request import BuildRequest +from nova.objects.instance import InstanceList from nova.scheduler import filters from nova.scheduler import utils from nova import utils as nova_utils @@ -28,6 +31,9 @@ CONF = nova.conf.CONF _SERVICE_AUTH = None +GARDENER_PREFIX = "kubernetes.io-cluster-shoot--garden--" +KKS_PREFIX = "kubernikus:kluster" +HANA_PREFIX = "hana_" class ShardFilter(filters.BaseHostFilter): @@ -114,11 +120,62 @@ def _get_shards(self, project_id): return self._PROJECT_SHARD_CACHE.get(project_id) - def host_passes(self, host_state, spec_obj): + def _get_k8s_cluster_instances(self, spec_obj): + """If the instance will be part of a K8S cluster, it returns + the list of all other instances that are already part of it, + if any. + """ + k8s_filter = self._k8s_instance_query_filter(spec_obj) + + if not k8s_filter: + return [] + + k8s_filter['project_id'] = spec_obj.project_id + + return InstanceList.get_by_filters( + nova_context.get_admin_context(), filters=k8s_filter, + expected_attrs=['flavor', 'metadata', 'tags']) + + def _k8s_instance_query_filter(self, spec_obj): + elevated = nova_context.get_admin_context() + build_request = BuildRequest.get_by_instance_uuid( + elevated, spec_obj.instance_uuid) + + # Kubernikus + kks_tag = next((t.tag for t in build_request.tags + if t.tag.startswith(KKS_PREFIX)), None) + if kks_tag: + return {'tags': [kks_tag]} + + # Gardener + gardener_meta = \ + {k: v for k, v in build_request.instance.metadata.items() + if k.startswith(GARDENER_PREFIX)} + if gardener_meta: + return {'metadata': gardener_meta} + + return None + + def filter_all(self, filter_obj_list, spec_obj): + """Yield objects that pass the filter. + + Can be overridden in a subclass, if you need to base filtering + decisions on all objects. Otherwise, one can just override + _filter_one() to filter a single object. + """ # Only VMware if utils.is_non_vmware_spec(spec_obj): - return True + for obj in filter_obj_list: + yield obj + return + + k8s_instances = self._get_k8s_cluster_instances(spec_obj) + + for obj in filter_obj_list: + if self._host_passes(obj, spec_obj, k8s_instances): + yield obj + def _host_passes(self, host_state, spec_obj, k8s_instances): host_shard_aggrs = [aggr for aggr in host_state.aggregates if aggr.name.startswith(self._SHARD_PREFIX)] @@ -148,14 +205,12 @@ def host_passes(self, host_state, spec_obj): if self._ALL_SHARDS in shards: LOG.debug('project enabled for all shards %(project_shards)s.', {'project_shards': shards}) - return True elif host_shard_names & set(shards): LOG.debug('%(host_state)s shard %(host_shard)s found in project ' 'shards %(project_shards)s.', {'host_state': host_state, 'host_shard': host_shard_names, 'project_shards': shards}) - return True else: LOG.debug('%(host_state)s shard %(host_shard)s not found in ' 'project shards %(project_shards)s.', @@ -163,3 +218,45 @@ def host_passes(self, host_state, spec_obj): 'host_shard': host_shard_names, 'project_shards': shards}) return False + + if not utils.request_is_resize(spec_obj): + # K8S orchestrators are only creating or deleting nodes, + # therefore we shouldn't infer with resize/migrate requests. + return self._host_passes_k8s(host_state, host_shard_names, + spec_obj, k8s_instances) + + return True + + def _host_passes_k8s(self, host_state, host_shard_names, spec_obj, + k8s_instances): + """Instances of a K8S cluster must end up on the same shard. + The K8S cluster is identified by the metadata or tags set + by the orchestrator (Gardener or Kubernikus). + """ + if not k8s_instances: + # There are no instances in the cluster, yet. + # We allow any shard for the first instance. + return True + + def _is_hana(flavor): + return flavor.name.startswith(HANA_PREFIX) + + def _is_same_category(instance, flavor): + """Check whether instance is from the flavor's family.""" + if _is_hana(flavor): + return _is_hana(instance.flavor) + return True + + def _instance_matches(instance): + if spec_obj.availability_zone: + if (instance.availability_zone != + spec_obj.availability_zone): + return False + return _is_same_category(instance, spec_obj.flavor) + + k8s_hosts = set([i.host for i in k8s_instances + if _instance_matches(i)]) + + return any(agg.name in host_shard_names and + set(agg.hosts) & k8s_hosts + for agg in host_state.aggregates) diff --git a/nova/tests/functional/db/test_console_auth_token.py b/nova/tests/functional/db/test_console_auth_token.py index 35efcc64d5b..9b3b89ee718 100644 --- a/nova/tests/functional/db/test_console_auth_token.py +++ b/nova/tests/functional/db/test_console_auth_token.py @@ -28,11 +28,11 @@ def setUp(self): instance = objects.Instance( context=self.context, project_id=self.context.project_id, - uuid=uuidsentinel.fake_instance) + uuid=uuidsentinel.fake_build_req) instance.create() self.console = objects.ConsoleAuthToken( context=self.context, - instance_uuid=uuidsentinel.fake_instance, + instance_uuid=uuidsentinel.fake_build_req, console_type='fake-type', host='fake-host', port=1000, diff --git a/nova/tests/unit/scheduler/filters/test_shard_filter.py b/nova/tests/unit/scheduler/filters/test_shard_filter.py index 7acdcdf69e3..3d3457af241 100644 --- a/nova/tests/unit/scheduler/filters/test_shard_filter.py +++ b/nova/tests/unit/scheduler/filters/test_shard_filter.py @@ -19,6 +19,7 @@ from nova import objects from nova.scheduler.filters import shard_filter from nova import test +from nova.tests.unit import fake_instance from nova.tests.unit.scheduler import fakes @@ -31,6 +32,13 @@ def setUp(self): 'foo': ['vc-a-0', 'vc-b-0'], 'last_modified': time.time() } + instance = fake_instance.fake_instance_obj( + mock.sentinel.ctx, expected_attrs=['metadata', 'tags']) + build_req = objects.BuildRequest() + build_req.instance_uuid = instance.uuid + build_req.tags = objects.TagList(objects=[]) + build_req.instance = instance + self.fake_build_req = build_req @mock.patch('nova.scheduler.filters.shard_filter.' 'ShardFilter._update_cache') @@ -63,93 +71,121 @@ def set_cache(): ['vc-a-1', 'vc-b-0']) mock_update_cache.assert_called_once() + @mock.patch('nova.objects.BuildRequest.get_by_instance_uuid') @mock.patch('nova.scheduler.filters.utils.aggregate_metadata_get_by_host') - def test_shard_baremetal_passes(self, agg_mock): + def test_shard_baremetal_passes(self, agg_mock, get_by_uuid): + get_by_uuid.return_value = self.fake_build_req aggs = [objects.Aggregate(id=1, name='some-az-a', hosts=['host1']), objects.Aggregate(id=1, name='vc-a-0', hosts=['host1'])] host = fakes.FakeHostState('host1', 'compute', {'aggregates': aggs}) extra_specs = {'capabilities:cpu_arch': 'x86_64'} spec_obj = objects.RequestSpec( context=mock.sentinel.ctx, project_id='foo', + instance_uuid=self.fake_build_req.instance_uuid, flavor=objects.Flavor(extra_specs=extra_specs)) - self.assertTrue(self.filt_cls.host_passes(host, spec_obj)) + self._assert_passes(host, spec_obj, True) + @mock.patch('nova.objects.BuildRequest.get_by_instance_uuid') @mock.patch('nova.scheduler.filters.shard_filter.' 'ShardFilter._update_cache') @mock.patch('nova.scheduler.filters.utils.aggregate_metadata_get_by_host') - def test_shard_project_not_found(self, agg_mock, mock_update_cache): + def test_shard_project_not_found(self, agg_mock, mock_update_cache, + get_by_uuid): + get_by_uuid.return_value = self.fake_build_req aggs = [objects.Aggregate(id=1, name='some-az-a', hosts=['host1']), objects.Aggregate(id=1, name='vc-a-0', hosts=['host1'])] host = fakes.FakeHostState('host1', 'compute', {'aggregates': aggs}) spec_obj = objects.RequestSpec( context=mock.sentinel.ctx, project_id='bar', + instance_uuid=self.fake_build_req.instance_uuid, flavor=objects.Flavor(extra_specs={})) - self.assertFalse(self.filt_cls.host_passes(host, spec_obj)) + self._assert_passes(host, spec_obj, False) + @mock.patch('nova.objects.BuildRequest.get_by_instance_uuid') @mock.patch('nova.scheduler.filters.utils.aggregate_metadata_get_by_host') - def test_shard_project_no_shards(self, agg_mock): + def test_shard_project_no_shards(self, agg_mock, get_by_uuid): + get_by_uuid.return_value = self.fake_build_req aggs = [objects.Aggregate(id=1, name='some-az-a', hosts=['host1']), objects.Aggregate(id=1, name='vc-a-0', hosts=['host1'])] host = fakes.FakeHostState('host1', 'compute', {'aggregates': aggs}) spec_obj = objects.RequestSpec( context=mock.sentinel.ctx, project_id='foo', + instance_uuid=self.fake_build_req.instance_uuid, flavor=objects.Flavor(extra_specs={})) self.filt_cls._PROJECT_SHARD_CACHE['foo'] = [] - self.assertFalse(self.filt_cls.host_passes(host, spec_obj)) + self._assert_passes(host, spec_obj, False) + @mock.patch('nova.objects.BuildRequest.get_by_instance_uuid') @mock.patch('nova.scheduler.filters.utils.aggregate_metadata_get_by_host') - def test_shard_host_no_shard_aggregate(self, agg_mock): + def test_shard_host_no_shard_aggregate(self, agg_mock, get_by_uuid): + get_by_uuid.return_value = self.fake_build_req host = fakes.FakeHostState('host1', 'compute', {}) spec_obj = objects.RequestSpec( context=mock.sentinel.ctx, project_id='foo', + instance_uuid=self.fake_build_req.instance_uuid, flavor=objects.Flavor(extra_specs={})) agg_mock.return_value = {} - self.assertFalse(self.filt_cls.host_passes(host, spec_obj)) + self._assert_passes(host, spec_obj, False) - def test_shard_host_no_shards_in_aggregate(self): + @mock.patch('nova.objects.BuildRequest.get_by_instance_uuid') + def test_shard_host_no_shards_in_aggregate(self, get_by_uuid): + get_by_uuid.return_value = self.fake_build_req aggs = [objects.Aggregate(id=1, name='some-az-a', hosts=['host1'])] host = fakes.FakeHostState('host1', 'compute', {'aggregates': aggs}) spec_obj = objects.RequestSpec( context=mock.sentinel.ctx, project_id='foo', + instance_uuid=self.fake_build_req.instance_uuid, flavor=objects.Flavor(extra_specs={})) - self.assertFalse(self.filt_cls.host_passes(host, spec_obj)) + self._assert_passes(host, spec_obj, False) - def test_shard_project_shard_match_host_shard(self): + @mock.patch('nova.objects.BuildRequest.get_by_instance_uuid') + def test_shard_project_shard_match_host_shard(self, get_by_uuid): + get_by_uuid.return_value = self.fake_build_req aggs = [objects.Aggregate(id=1, name='some-az-a', hosts=['host1']), objects.Aggregate(id=1, name='vc-a-0', hosts=['host1'])] host = fakes.FakeHostState('host1', 'compute', {'aggregates': aggs}) spec_obj = objects.RequestSpec( context=mock.sentinel.ctx, project_id='foo', + instance_uuid=self.fake_build_req.instance_uuid, flavor=objects.Flavor(extra_specs={})) - self.assertTrue(self.filt_cls.host_passes(host, spec_obj)) + self._assert_passes(host, spec_obj, True) - def test_shard_project_shard_do_not_match_host_shard(self): + @mock.patch('nova.objects.BuildRequest.get_by_instance_uuid') + def test_shard_project_shard_do_not_match_host_shard(self, get_by_uuid): + get_by_uuid.return_value = self.fake_build_req aggs = [objects.Aggregate(id=1, name='some-az-a', hosts=['host1']), objects.Aggregate(id=1, name='vc-a-1', hosts=['host1'])] host = fakes.FakeHostState('host1', 'compute', {'aggregates': aggs}) spec_obj = objects.RequestSpec( context=mock.sentinel.ctx, project_id='foo', + instance_uuid=self.fake_build_req.instance_uuid, flavor=objects.Flavor(extra_specs={})) - self.assertFalse(self.filt_cls.host_passes(host, spec_obj)) + self._assert_passes(host, spec_obj, False) - def test_shard_project_has_multiple_shards_per_az(self): + @mock.patch('nova.objects.BuildRequest.get_by_instance_uuid') + def test_shard_project_has_multiple_shards_per_az(self, get_by_uuid): + get_by_uuid.return_value = self.fake_build_req aggs = [objects.Aggregate(id=1, name='some-az-a', hosts=['host1']), objects.Aggregate(id=1, name='vc-a-1', hosts=['host1'])] host = fakes.FakeHostState('host1', 'compute', {'aggregates': aggs}) spec_obj = objects.RequestSpec( context=mock.sentinel.ctx, project_id='foo', + instance_uuid=self.fake_build_req.instance_uuid, flavor=objects.Flavor(extra_specs={})) self.filt_cls._PROJECT_SHARD_CACHE['foo'] = ['vc-a-0', 'vc-a-1', 'vc-b-0'] - self.assertTrue(self.filt_cls.host_passes(host, spec_obj)) + self._assert_passes(host, spec_obj, True) - def test_shard_project_has_multiple_shards_per_az_resize_same_shard(self): + @mock.patch('nova.objects.BuildRequest.get_by_instance_uuid') + def test_shard_project_has_multiple_shards_per_az_resize_same_shard( + self, get_by_uuid): + get_by_uuid.return_value = self.fake_build_req aggs = [objects.Aggregate(id=1, name='some-az-a', hosts=['host1', 'host2']), objects.Aggregate(id=1, name='vc-a-1', hosts=['host1', @@ -157,15 +193,19 @@ def test_shard_project_has_multiple_shards_per_az_resize_same_shard(self): host = fakes.FakeHostState('host1', 'compute', {'aggregates': aggs}) spec_obj = objects.RequestSpec( context=mock.sentinel.ctx, project_id='foo', + instance_uuid=self.fake_build_req.instance_uuid, flavor=objects.Flavor(extra_specs={}), scheduler_hints=dict(_nova_check_type=['resize'], source_host=['host2'])) self.filt_cls._PROJECT_SHARD_CACHE['foo'] = ['vc-a-0', 'vc-a-1', 'vc-b-0'] - self.assertTrue(self.filt_cls.host_passes(host, spec_obj)) + self._assert_passes(host, spec_obj, True) - def test_shard_project_has_multiple_shards_per_az_resize_other_shard(self): + @mock.patch('nova.objects.BuildRequest.get_by_instance_uuid') + def test_shard_project_has_multiple_shards_per_az_resize_other_shard( + self, get_by_uuid): + get_by_uuid.return_value = self.fake_build_req aggs = [objects.Aggregate(id=1, name='some-az-a', hosts=['host1', 'host2']), objects.Aggregate(id=1, name='vc-a-1', hosts=['host1'])] @@ -173,24 +213,32 @@ def test_shard_project_has_multiple_shards_per_az_resize_other_shard(self): spec_obj = objects.RequestSpec( context=mock.sentinel.ctx, project_id='foo', flavor=objects.Flavor(extra_specs={}), + instance_uuid=self.fake_build_req.instance_uuid, scheduler_hints=dict(_nova_check_type=['resize'], source_host=['host2'])) self.filt_cls._PROJECT_SHARD_CACHE['foo'] = ['vc-a-0', 'vc-a-1', 'vc-b-0'] - self.assertTrue(self.filt_cls.host_passes(host, spec_obj)) + self._assert_passes(host, spec_obj, True) - def test_shard_project_has_sharding_enabled_any_host_passes(self): + @mock.patch('nova.objects.BuildRequest.get_by_instance_uuid') + def test_shard_project_has_sharding_enabled_any_host_passes( + self, get_by_uuid): + get_by_uuid.return_value = self.fake_build_req self.filt_cls._PROJECT_SHARD_CACHE['baz'] = ['sharding_enabled'] aggs = [objects.Aggregate(id=1, name='some-az-a', hosts=['host1']), objects.Aggregate(id=1, name='vc-a-0', hosts=['host1'])] host = fakes.FakeHostState('host1', 'compute', {'aggregates': aggs}) spec_obj = objects.RequestSpec( context=mock.sentinel.ctx, project_id='baz', + instance_uuid=self.fake_build_req.instance_uuid, flavor=objects.Flavor(extra_specs={})) - self.assertTrue(self.filt_cls.host_passes(host, spec_obj)) + self._assert_passes(host, spec_obj, True) - def test_shard_project_has_sharding_enabled_and_single_shards(self): + @mock.patch('nova.objects.BuildRequest.get_by_instance_uuid') + def test_shard_project_has_sharding_enabled_and_single_shards( + self, get_by_uuid): + get_by_uuid.return_value = self.fake_build_req self.filt_cls._PROJECT_SHARD_CACHE['baz'] = ['sharding_enabled', 'vc-a-1'] aggs = [objects.Aggregate(id=1, name='some-az-a', hosts=['host1']), @@ -198,15 +246,63 @@ def test_shard_project_has_sharding_enabled_and_single_shards(self): host = fakes.FakeHostState('host1', 'compute', {'aggregates': aggs}) spec_obj = objects.RequestSpec( context=mock.sentinel.ctx, project_id='baz', + instance_uuid=self.fake_build_req.instance_uuid, flavor=objects.Flavor(extra_specs={})) - self.assertTrue(self.filt_cls.host_passes(host, spec_obj)) + self._assert_passes(host, spec_obj, True) + + @mock.patch('nova.objects.InstanceList.get_by_filters') + @mock.patch('nova.objects.BuildRequest.get_by_instance_uuid') + @mock.patch('nova.context.get_admin_context') + def test_same_shard_for_kubernikus_cluster(self, get_context, + get_by_uuid, + get_by_filters): + kks_cluster = 'kubernikus:kluster-example' + build_req = objects.BuildRequest() + build_req.tags = objects.TagList(objects=[ + objects.Tag(tag=kks_cluster) + ]) + + sibling1 = fake_instance.fake_instance_obj(get_context.return_value, + host='host2') + + get_by_uuid.return_value = build_req + get_by_filters.return_value = objects.InstanceList(objects=[sibling1]) + self.filt_cls._PROJECT_SHARD_CACHE['baz'] = ['sharding_enabled', + 'vc-a-1'] + agg1 = objects.Aggregate(id=1, name='vc-a-0', hosts=['host1']) + agg2 = objects.Aggregate(id=1, name='vc-b-0', hosts=['host2']) + + host1 = fakes.FakeHostState('host1', 'compute', + {'aggregates': [agg1]}) + host2 = fakes.FakeHostState('host2', 'compute', + {'aggregates': [agg2]}) + + spec_obj = objects.RequestSpec( + context=get_context.return_value, project_id='foo', + availability_zone=None, + instance_uuid=self.fake_build_req.instance_uuid, + flavor=objects.Flavor(extra_specs={}, name='m1')) + + result = list(self.filt_cls.filter_all([host1, host2], spec_obj)) + get_by_filters.assert_called_once_with( + get_context.return_value, + filters={'tags': ['kubernikus:kluster-example'], + 'project_id': 'foo'}, + expected_attrs=['flavor', 'metadata', 'tags']) + self.assertEqual(1, len(result)) + self.assertEqual(result[0], host2) + + @mock.patch('nova.objects.BuildRequest.get_by_instance_uuid') @mock.patch('nova.scheduler.filters.shard_filter.LOG') @mock.patch('nova.scheduler.filters.utils.aggregate_metadata_get_by_host') - def test_log_level_for_missing_vc_aggregate(self, agg_mock, log_mock): + def test_log_level_for_missing_vc_aggregate(self, agg_mock, log_mock, + get_by_uuid): + get_by_uuid.return_value = self.fake_build_req host = fakes.FakeHostState('host1', 'compute', {}) spec_obj = objects.RequestSpec( context=mock.sentinel.ctx, project_id='foo', + instance_uuid=self.fake_build_req.instance_uuid, flavor=objects.Flavor(extra_specs={})) agg_mock.return_value = {} @@ -215,7 +311,7 @@ def test_log_level_for_missing_vc_aggregate(self, agg_mock, log_mock): log_mock.debug = mock.Mock() log_mock.error = mock.Mock() host.hypervisor_type = 'ironic' - self.assertFalse(self.filt_cls.host_passes(host, spec_obj)) + self._assert_passes(host, spec_obj, False) log_mock.debug.assert_called_once_with(mock.ANY, mock.ANY) log_mock.error.assert_not_called() @@ -223,14 +319,21 @@ def test_log_level_for_missing_vc_aggregate(self, agg_mock, log_mock): log_mock.debug = mock.Mock() log_mock.error = mock.Mock() host.hypervisor_type = 'Some HV' - self.assertFalse(self.filt_cls.host_passes(host, spec_obj)) + self._assert_passes(host, spec_obj, False) log_mock.error.assert_called_once_with(mock.ANY, mock.ANY) log_mock.debug.assert_not_called() @mock.patch('nova.scheduler.utils.is_non_vmware_spec', return_value=True) def test_non_vmware_spec(self, mock_is_non_vmware_spec): - host = mock.sentinel.host + host1 = mock.sentinel.host1 + host2 = mock.sentinel.host2 spec_obj = mock.sentinel.spec_obj - self.assertTrue(self.filt_cls.host_passes(host, spec_obj)) + result = list(self.filt_cls.filter_all([host1, host2], spec_obj)) + + self.assertEqual([host1, host2], result) mock_is_non_vmware_spec.assert_called_once_with(spec_obj) + + def _assert_passes(self, host, spec_obj, passes): + result = bool(list(self.filt_cls.filter_all([host], spec_obj))) + self.assertEqual(passes, result) diff --git a/nova/tests/unit/scheduler/weights/test_weights_resize_same_host.py b/nova/tests/unit/scheduler/weights/test_weights_resize_same_host.py index bff4a208364..35e701a0489 100644 --- a/nova/tests/unit/scheduler/weights/test_weights_resize_same_host.py +++ b/nova/tests/unit/scheduler/weights/test_weights_resize_same_host.py @@ -44,7 +44,7 @@ def setUp(self): memory_mb=1024 * 256, extra_specs=baremetal_extra_specs) old_instance = objects.Instance(host='same_host', - uuid=uuidsentinel.fake_instance, + uuid=uuidsentinel.fake_build_req, flavor=flavor_small) new_instance = objects.Instance(uuid=uuidsentinel.fake_new_instance, flavor=flavor_small)