From f5e5f1b1d26b68817fae0a072a14dec1194f1afd Mon Sep 17 00:00:00 2001 From: Tushar Date: Fri, 10 May 2024 05:32:18 +0000 Subject: [PATCH 1/6] Add vcenter as inventory source Signed-off-by: Tushar --- poetry.lock | 35 +++ pyproject.toml | 4 + pytest.ini | 1 + suzieq/poller/controller/source/vcenter.py | 220 ++++++++++++++++++ .../controller/sources/vcenter/__init__.py | 0 .../sources/vcenter/test_vcenter.py | 80 +++++++ tests/unit/poller/shared/utils.py | 13 ++ 7 files changed, 353 insertions(+) create mode 100644 suzieq/poller/controller/source/vcenter.py create mode 100644 tests/unit/poller/controller/sources/vcenter/__init__.py create mode 100644 tests/unit/poller/controller/sources/vcenter/test_vcenter.py diff --git a/poetry.lock b/poetry.lock index 441750c9b5..2c77f0cce1 100644 --- a/poetry.lock +++ b/poetry.lock @@ -798,6 +798,7 @@ files = [ {file = "contourpy-1.1.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:18a64814ae7bce73925131381603fff0116e2df25230dfc80d6d690aa6e20b37"}, {file = "contourpy-1.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90c81f22b4f572f8a2110b0b741bb64e5a6427e0a198b2cdc1fbaf85f352a3aa"}, {file = "contourpy-1.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:53cc3a40635abedbec7f1bde60f8c189c49e84ac180c665f2cd7c162cc454baa"}, + {file = "contourpy-1.1.0-cp310-cp310-win32.whl", hash = "sha256:9b2dd2ca3ac561aceef4c7c13ba654aaa404cf885b187427760d7f7d4c57cff8"}, {file = "contourpy-1.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:1f795597073b09d631782e7245016a4323cf1cf0b4e06eef7ea6627e06a37ff2"}, {file = "contourpy-1.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0b7b04ed0961647691cfe5d82115dd072af7ce8846d31a5fac6c142dcce8b882"}, {file = "contourpy-1.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:27bc79200c742f9746d7dd51a734ee326a292d77e7d94c8af6e08d1e6c15d545"}, @@ -806,6 +807,7 @@ files = [ {file = "contourpy-1.1.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e5cec36c5090e75a9ac9dbd0ff4a8cf7cecd60f1b6dc23a374c7d980a1cd710e"}, {file = "contourpy-1.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1f0cbd657e9bde94cd0e33aa7df94fb73c1ab7799378d3b3f902eb8eb2e04a3a"}, {file = "contourpy-1.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:181cbace49874f4358e2929aaf7ba84006acb76694102e88dd15af861996c16e"}, + {file = "contourpy-1.1.0-cp311-cp311-win32.whl", hash = "sha256:edb989d31065b1acef3828a3688f88b2abb799a7db891c9e282df5ec7e46221b"}, {file = "contourpy-1.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:fb3b7d9e6243bfa1efb93ccfe64ec610d85cfe5aec2c25f97fbbd2e58b531256"}, {file = "contourpy-1.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:bcb41692aa09aeb19c7c213411854402f29f6613845ad2453d30bf421fe68fed"}, {file = "contourpy-1.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:5d123a5bc63cd34c27ff9c7ac1cd978909e9c71da12e05be0231c608048bb2ae"}, @@ -814,6 +816,7 @@ files = [ {file = "contourpy-1.1.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:317267d915490d1e84577924bd61ba71bf8681a30e0d6c545f577363157e5e94"}, {file = "contourpy-1.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d551f3a442655f3dcc1285723f9acd646ca5858834efeab4598d706206b09c9f"}, {file = "contourpy-1.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:e7a117ce7df5a938fe035cad481b0189049e8d92433b4b33aa7fc609344aafa1"}, + {file = "contourpy-1.1.0-cp38-cp38-win32.whl", hash = "sha256:108dfb5b3e731046a96c60bdc46a1a0ebee0760418951abecbe0fc07b5b93b27"}, {file = "contourpy-1.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:d4f26b25b4f86087e7d75e63212756c38546e70f2a92d2be44f80114826e1cd4"}, {file = "contourpy-1.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bc00bb4225d57bff7ebb634646c0ee2a1298402ec10a5fe7af79df9a51c1bfd9"}, {file = "contourpy-1.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:189ceb1525eb0655ab8487a9a9c41f42a73ba52d6789754788d1883fb06b2d8a"}, @@ -822,6 +825,7 @@ files = [ {file = "contourpy-1.1.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:143dde50520a9f90e4a2703f367cf8ec96a73042b72e68fcd184e1279962eb6f"}, {file = "contourpy-1.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e94bef2580e25b5fdb183bf98a2faa2adc5b638736b2c0a4da98691da641316a"}, {file = "contourpy-1.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ed614aea8462735e7d70141374bd7650afd1c3f3cb0c2dbbcbe44e14331bf002"}, + {file = "contourpy-1.1.0-cp39-cp39-win32.whl", hash = "sha256:71551f9520f008b2950bef5f16b0e3587506ef4f23c734b71ffb7b89f8721999"}, {file = "contourpy-1.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:438ba416d02f82b692e371858143970ed2eb6337d9cdbbede0d8ad9f3d7dd17d"}, {file = "contourpy-1.1.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:a698c6a7a432789e587168573a864a7ea374c6be8d4f31f9d87c001d5a843493"}, {file = "contourpy-1.1.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:397b0ac8a12880412da3551a8cb5a187d3298a72802b45a3bd1805e204ad8439"}, @@ -2138,6 +2142,16 @@ files = [ {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-win32.whl", hash = "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"}, @@ -3525,6 +3539,23 @@ files = [ {file = "pytz-2023.3.tar.gz", hash = "sha256:1d8ce29db189191fb55338ee6d0387d82ab59f3d00eac103412d64e0ebd0c588"}, ] +[[package]] +name = "pyvmomi" +version = "8.0.2.0.1" +description = "VMware vSphere Python SDK" +category = "main" +optional = false +python-versions = ">=2.7.9, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pyvmomi-8.0.2.0.1.tar.gz", hash = "sha256:791c4d93252e3c0896fecae8d785f8342b123672e7cae9c5548a639b9bf668ad"}, +] + +[package.dependencies] +six = ">=1.7.3" + +[package.extras] +sso = ["lxml", "pyOpenSSL", "pywin32"] + [[package]] name = "pywin32" version = "306" @@ -5080,4 +5111,8 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">3.8.1, < 3.10" +<<<<<<< HEAD content-hash = "d5d9b1c1146a6684735d7e4c4016db001f8faad1eb0a55e00daac6dc4fc070fe" +======= +content-hash = "49314731cbc5a57e03fe03c0b3039aeec30db4ff2b37b6c3bc42a9cf71fc3090" +>>>>>>> 5f7cedbc37 (Add vcenter as inventory source) diff --git a/pyproject.toml b/pyproject.toml index 8e93812f3c..359c246f56 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,8 +52,12 @@ packaging = "~21.3" psutil = "~5.9.4" jellyfish = "~0.10" altair = '>3.2, <5.0' +<<<<<<< HEAD pydantic = '< 2.0' numpy = '~1.20' +======= +pyvmomi = "^8.0.2.0.1" +>>>>>>> 5f7cedbc37 (Add vcenter as inventory source) [tool.poetry.dev-dependencies] pylint = "*" diff --git a/pytest.ini b/pytest.ini index 3a29aa54a2..b8596d59ba 100644 --- a/pytest.ini +++ b/pytest.ini @@ -112,6 +112,7 @@ markers = controller_source_ansible controller_source_native controller_source_netbox + controller_source_vcenter controller_unit_tests # schema diff --git a/suzieq/poller/controller/source/vcenter.py b/suzieq/poller/controller/source/vcenter.py new file mode 100644 index 0000000000..1ae1e9291a --- /dev/null +++ b/suzieq/poller/controller/source/vcenter.py @@ -0,0 +1,220 @@ +"""Vcenter module + +This module contains the methods to connect to a Vcenter server to +retrieve the list of VMs. +""" +# pylint: disable=no-name-in-module +# pylint: disable=no-self-argument + +import asyncio +import logging +from requests.auth import HTTPBasicAuth +from requests.exceptions import RequestException +from typing import Any, Dict, List, Optional, Tuple, Union +from urllib.parse import urljoin, urlparse +from pyVim.connect import SmartConnect, Disconnect +from pyVmomi import vim, vmodl +import ssl + +from pydantic import BaseModel, validator, Field + +from suzieq.poller.controller.inventory_async_plugin import \ + InventoryAsyncPlugin +from suzieq.poller.controller.source.base_source import Source, SourceModel +from suzieq.shared.utils import get_sensitive_data +from suzieq.shared.exceptions import InventorySourceError, SensitiveLoadError + +_DEFAULT_PORTS = {'https': 443} + +logger = logging.getLogger(__name__) + + +class VcenterServerModel(BaseModel): + """Model containing data to connect with vcenter server.""" + host: str + port: str + + class Config: + """pydantic configuration + """ + extra = 'forbid' + + +class VcenterSourceModel(SourceModel): + """Vcenter source validation model.""" + username: str + password: str + attributes: Optional[List] = Field(default=['suzieq']) + period: Optional[int] = Field(default=3600) + ssl_verify: Optional[bool] = Field(alias='ssl-verify') + server: Union[str, VcenterServerModel] = Field(alias='url') + run_once: Optional[bool] = Field(default=False, alias='run_once') + + @validator('server', pre=True) + def validate_and_set(cls, url, values): + """Validate the field 'url' and set the correct parameters + """ + if isinstance(url, str): + url_data = urlparse(url) + host = url_data.hostname + if not host: + raise ValueError(f'Unable to parse hostname {url}') + port = url_data.port or _DEFAULT_PORTS.get("https") + if not port: + raise ValueError(f'Unable to parse port {url}') + server = VcenterServerModel(host=host, port=port) + ssl_verify = values['ssl_verify'] + if ssl_verify is None: + ssl_verify = True + values['ssl_verify'] = ssl_verify + return server + elif isinstance(url, VcenterServerModel): + return url + else: + raise ValueError('Unknown input type') + + @validator('password') + def validate_password(cls, password): + """checks if the password can be load as sensible data + """ + try: + if password == 'ask': + return password + return get_sensitive_data(password) + except SensitiveLoadError as e: + raise ValueError(e) + +class Vcenter(Source, InventoryAsyncPlugin): + def __init__(self, config_data: dict, validate: bool = True) -> None: + self._status = 'init' + self._server: VcenterServerModel = None + self._session = None + + super().__init__(config_data, validate) + + @classmethod + def get_data_model(cls): + return VcenterSourceModel + + def _load(self, input_data): + # load the server class from the dictionary + if not self._validate: + input_data['server'] = VcenterServerModel.construct( + **input_data.pop('url', {})) + input_data['ssl_verify'] = input_data.pop('ssl-verify', False) + super()._load(input_data) + if self._data.password == 'ask': + self._data.password = get_sensitive_data( + 'ask', f'{self.name} Insert vcenter password: ' + ) + self._server = self._data.server + if not self._auth: + raise InventorySourceError(f"{self.name} Vcenter must have an " + "'auth' set in the 'namespaces' section" + ) + + def _init_session(self): + """Initialize the session property""" + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + context.verify_mode = ssl.CERT_NONE if not self._data.ssl_verify else ssl.CERT_REQUIRED + try: + self._session = SmartConnect( + host=self._server.host, + port=self._server.port, + user=self._data.username, + pwd=self._data.password, + sslContext=context + ) + except Exception as e: + self._session = None + raise InventorySourceError(f"Failed to connect to VCenter: {str(e)}") + + def _get_custom_keys(self, content, attribute_names): + """Retrieve custom attribute keys based on their names.""" + all_custom_fields = {field.name: field.key for field in content.customFieldsManager.field} + return [all_custom_fields[name] for name in attribute_names if name in all_custom_fields] + + def _create_filter_spec(self, view): + """Create and return a FilterSpec based on provided view and attribute keys.""" + traversal_spec = vmodl.query.PropertyCollector.TraversalSpec( + name='traverseEntities', + path='view', + skip=False, + type=vim.view.ContainerView, + selectSet=[vmodl.query.PropertyCollector.SelectionSpec(name='traverseEntities')] + ) + prop_set = vmodl.query.PropertyCollector.PropertySpec(all=False, type=vim.VirtualMachine) + prop_set.pathSet = ['name', 'guest.ipAddress', 'customValue'] + obj_spec = vmodl.query.PropertyCollector.ObjectSpec(obj=view, selectSet=[traversal_spec]) + filter_spec = vmodl.query.PropertyCollector.FilterSpec() + filter_spec.objectSet = [obj_spec] + filter_spec.propSet = [prop_set] + return filter_spec + + async def get_inventory_list(self) -> List: + """ + Retrieve VMs that have any of a list of specified custom attribute names using the Property Collector. + + This method uses vSphere's Property Collector to fetch only properties that are required. + This is a lot faster than fetching the entire inventory and filtering on attributes. + """ + if not self._session: + self._init_session() + + content = self._session.RetrieveContent() + view = content.viewManager.CreateContainerView(content.rootFolder, [vim.VirtualMachine], True) + attribute_keys = self._get_custom_keys(content, self._data.attributes) + + filter_spec = self._create_filter_spec(view) + retrieve_options = vmodl.query.PropertyCollector.RetrieveOptions() + result = content.propertyCollector.RetrievePropertiesEx([filter_spec], retrieve_options) + vms_with_ip = {} + while result: + for obj in result.objects: + vm_name = None + vm_ip = None + has_custom_attr = False + for prop in obj.propSet: + if prop.name == 'name': + vm_name = prop.val + elif prop.name == 'guest.ipAddress' and prop.val: + vm_ip = prop.val + elif prop.name == 'customValue': + has_custom_attr = any(cv.key in attribute_keys for cv in prop.val) + if has_custom_attr and vm_ip: + vms_with_ip[vm_name] = vm_ip + + if hasattr(result, 'token') and result.token: + result = content.propertyCollector.ContinueRetrievePropertiesEx(token=result.token) + else: + break + + view.Destroy() + logger.info(f'Vcenter: Retrieved {len(vms_with_ip)} VMs with IPs that have any of the specified attribute names') + return vms_with_ip + + + def parse_inventory(self, inventory_list: list) -> Dict: + inventory = {} + for name, ip in inventory_list.items(): + namespace = self._namespace + inventory[f'{namespace}.{ip}'] = { + 'address': ip, + 'namespace': namespace, + 'hostname': name, + } + logger.info(f'Vcenter: Acting on inventory of {len(inventory)} devices') + return inventory + + async def _execute(self): + while True: + inventory_list = await self.get_inventory_list() + tmp_inventory = self.parse_inventory(inventory_list) + self.set_inventory(tmp_inventory) + if self._run_once: + break + await asyncio.sleep(self._data.period) + + async def _stop(self): + if self._session: + Disconnect(self._session) \ No newline at end of file diff --git a/tests/unit/poller/controller/sources/vcenter/__init__.py b/tests/unit/poller/controller/sources/vcenter/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/poller/controller/sources/vcenter/test_vcenter.py b/tests/unit/poller/controller/sources/vcenter/test_vcenter.py new file mode 100644 index 0000000000..496574a976 --- /dev/null +++ b/tests/unit/poller/controller/sources/vcenter/test_vcenter.py @@ -0,0 +1,80 @@ +import pytest +from unittest.mock import MagicMock, patch +from suzieq.poller.controller.source.vcenter import Vcenter +from tests.unit.poller.shared.utils import get_src_sample_config +from pyVmomi import vim, vmodl + + +@pytest.fixture(scope="function", autouse=True) +def service_instance(): + mock_si = MagicMock() + mock_content = MagicMock() + mock_si.RetrieveContent.return_value = mock_content + mock_content.viewManager.CreateContainerView.return_value = MagicMock() + + mock_content.propertyCollector.RetrievePropertiesEx = MagicMock() + mock_custom_field_def1 = vim.CustomFieldDef(name="suzieq", key=101) + mock_custom_field_def2 = vim.CustomFieldDef(name="monitoring", key=102) + mock_content.customFieldsManager.field = [mock_custom_field_def1, mock_custom_field_def2] + + with patch('suzieq.poller.controller.source.vcenter.SmartConnect', return_value=mock_si), \ + patch('suzieq.poller.controller.source.vcenter.vmodl.query.PropertyCollector.ObjectSpec', return_value=MagicMock()), \ + patch('suzieq.poller.controller.source.vcenter.vmodl.query.PropertyCollector.FilterSpec', return_value=MagicMock()): + yield mock_content + +@pytest.mark.controller_source +@pytest.mark.poller +@pytest.mark.controller +@pytest.mark.poller_unit_tests +@pytest.mark.controller_unit_tests +@pytest.mark.controller_source_vcenter +@pytest.mark.asyncio +async def test_get_inventory_list_successful(service_instance): + """Test successful retrieval of VM inventory.""" + + service_instance.propertyCollector.RetrievePropertiesEx.return_value = vim.PropertyCollector.RetrieveResult( + objects=[ + vim.ObjectContent( + obj=vim.VirtualMachine("vm-1234"), + propSet=[ + vmodl.DynamicProperty(name='name', val='multiple-attr-vm'), + vmodl.DynamicProperty(name='guest.ipAddress', val='192.168.1.1'), + vmodl.DynamicProperty(name='customValue', val=vim.ArrayOfCustomFieldValue([vim.CustomFieldStringValue(key=102, value='true'), vim.CustomFieldStringValue(key=101, value='true')])) + ] + ), + vim.ObjectContent( + obj=vim.VirtualMachine("vm-2345"), + propSet=[ + vmodl.DynamicProperty(name='name', val='single-attr-vm'), + vmodl.DynamicProperty(name='guest.ipAddress', val='192.168.1.2'), + vmodl.DynamicProperty(name='customValue', val=vim.ArrayOfCustomFieldValue([vim.CustomFieldStringValue(key=101, value='true')])) + ] + ), + vim.ObjectContent( + obj=vim.VirtualMachine("vm-3456"), + propSet=[ + vmodl.DynamicProperty(name='name', val='no-attr-vm'), + vmodl.DynamicProperty(name='guest.ipAddress', val='192.168.1.3'), + vmodl.DynamicProperty(name='customValue', val=[]) + ] + ), + vim.ObjectContent( + obj=vim.VirtualMachine("vm-4567"), + propSet=[ + vmodl.DynamicProperty(name='name', val='no-ip-vm'), + vmodl.DynamicProperty(name='guest.ipAddress', val=''), + vmodl.DynamicProperty(name='customValue', val=vim.ArrayOfCustomFieldValue([vim.CustomFieldStringValue(key=101, value='true')])) + ] + ) + ] + ) + + vc = Vcenter(get_src_sample_config('vcenter')) + inventory = await vc.get_inventory_list() + + # Asserts to verify the correct functionality + expected_inventory = { + 'multiple-attr-vm': '192.168.1.1', + 'single-attr-vm': '192.168.1.2', + } + assert inventory == expected_inventory, "Inventory should match expected output including multiple attributes and VM details" diff --git a/tests/unit/poller/shared/utils.py b/tests/unit/poller/shared/utils.py index e58f1143af..2a14e67e01 100644 --- a/tests/unit/poller/shared/utils.py +++ b/tests/unit/poller/shared/utils.py @@ -89,6 +89,19 @@ def get_src_sample_config(src_type: str) -> Dict: 'password': 'plain:password' }), }) + elif src_type == 'vcenter': + sample_config.update({ + 'url': 'http://fake-url:1234', + 'username': "test-user", + "password": "fake-password", + 'attributes': ['monitoring', 'suzieq'], + 'run_once': True, + 'auth': StaticLoader({ + 'name': 'static0', + 'username': 'username', + 'password': 'plain:password' + }), + }) return sample_config From 4f79191d982f137bfc1f59ba50600d15448937d4 Mon Sep 17 00:00:00 2001 From: Tushar Date: Sat, 11 May 2024 02:32:07 +0000 Subject: [PATCH 2/6] Fix lint Signed-off-by: Tushar --- suzieq/poller/controller/source/vcenter.py | 75 +++++++++++++--------- 1 file changed, 45 insertions(+), 30 deletions(-) diff --git a/suzieq/poller/controller/source/vcenter.py b/suzieq/poller/controller/source/vcenter.py index 1ae1e9291a..301e34fdd4 100644 --- a/suzieq/poller/controller/source/vcenter.py +++ b/suzieq/poller/controller/source/vcenter.py @@ -8,10 +8,8 @@ import asyncio import logging -from requests.auth import HTTPBasicAuth -from requests.exceptions import RequestException -from typing import Any, Dict, List, Optional, Tuple, Union -from urllib.parse import urljoin, urlparse +from typing import Dict, List, Optional, Union +from urllib.parse import urlparse from pyVim.connect import SmartConnect, Disconnect from pyVmomi import vim, vmodl import ssl @@ -84,6 +82,7 @@ def validate_password(cls, password): except SensitiveLoadError as e: raise ValueError(e) + class Vcenter(Source, InventoryAsyncPlugin): def __init__(self, config_data: dict, validate: bool = True) -> None: self._status = 'init' @@ -109,14 +108,17 @@ def _load(self, input_data): ) self._server = self._data.server if not self._auth: - raise InventorySourceError(f"{self.name} Vcenter must have an " - "'auth' set in the 'namespaces' section" - ) + raise InventorySourceError( + f"{self.name} Vcenter must have an " + "'auth' set in the 'namespaces' section") def _init_session(self): """Initialize the session property""" context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) - context.verify_mode = ssl.CERT_NONE if not self._data.ssl_verify else ssl.CERT_REQUIRED + context.verify_mode = ssl.CERT_REQUIRED + if not self._data.ssl_verify: + context.verify_mode = ssl.CERT_NONE + try: self._session = SmartConnect( host=self._server.host, @@ -127,25 +129,31 @@ def _init_session(self): ) except Exception as e: self._session = None - raise InventorySourceError(f"Failed to connect to VCenter: {str(e)}") + raise InventorySourceError( + f"Failed to connect to VCenter: {str(e)}") def _get_custom_keys(self, content, attribute_names): """Retrieve custom attribute keys based on their names.""" - all_custom_fields = {field.name: field.key for field in content.customFieldsManager.field} - return [all_custom_fields[name] for name in attribute_names if name in all_custom_fields] + all_custom_fields = {field.name: field.key + for field in content.customFieldsManager.field} + return [ + all_custom_fields[name] + for name in attribute_names + if name in all_custom_fields + ] def _create_filter_spec(self, view): - """Create and return a FilterSpec based on provided view and attribute keys.""" + """Return a FilterSpec based on provided view and attribute keys.""" traversal_spec = vmodl.query.PropertyCollector.TraversalSpec( - name='traverseEntities', - path='view', - skip=False, + name='traverseEntities', path='view', skip=False, type=vim.view.ContainerView, - selectSet=[vmodl.query.PropertyCollector.SelectionSpec(name='traverseEntities')] - ) - prop_set = vmodl.query.PropertyCollector.PropertySpec(all=False, type=vim.VirtualMachine) + selectSet=[vmodl.query.PropertyCollector.SelectionSpec( + name='traverseEntities')]) + prop_set = vmodl.query.PropertyCollector.PropertySpec( + all=False, type=vim.VirtualMachine) prop_set.pathSet = ['name', 'guest.ipAddress', 'customValue'] - obj_spec = vmodl.query.PropertyCollector.ObjectSpec(obj=view, selectSet=[traversal_spec]) + obj_spec = vmodl.query.PropertyCollector.ObjectSpec( + obj=view, selectSet=[traversal_spec]) filter_spec = vmodl.query.PropertyCollector.FilterSpec() filter_spec.objectSet = [obj_spec] filter_spec.propSet = [prop_set] @@ -153,21 +161,24 @@ def _create_filter_spec(self, view): async def get_inventory_list(self) -> List: """ - Retrieve VMs that have any of a list of specified custom attribute names using the Property Collector. + Retrieve VMs that have any specified custom attribute names. - This method uses vSphere's Property Collector to fetch only properties that are required. - This is a lot faster than fetching the entire inventory and filtering on attributes. + This method uses vSphere's Property Collector to fetch only + properties that are required. This is a lot faster than + fetching the entire inventory and filtering on attributes. """ if not self._session: self._init_session() content = self._session.RetrieveContent() - view = content.viewManager.CreateContainerView(content.rootFolder, [vim.VirtualMachine], True) + view = content.viewManager.CreateContainerView( + content.rootFolder, [vim.VirtualMachine], True) attribute_keys = self._get_custom_keys(content, self._data.attributes) filter_spec = self._create_filter_spec(view) retrieve_options = vmodl.query.PropertyCollector.RetrieveOptions() - result = content.propertyCollector.RetrievePropertiesEx([filter_spec], retrieve_options) + result = content.propertyCollector.RetrievePropertiesEx( + [filter_spec], retrieve_options) vms_with_ip = {} while result: for obj in result.objects: @@ -180,20 +191,23 @@ async def get_inventory_list(self) -> List: elif prop.name == 'guest.ipAddress' and prop.val: vm_ip = prop.val elif prop.name == 'customValue': - has_custom_attr = any(cv.key in attribute_keys for cv in prop.val) + has_custom_attr = any( + cv.key in attribute_keys for cv in prop.val) if has_custom_attr and vm_ip: vms_with_ip[vm_name] = vm_ip if hasattr(result, 'token') and result.token: - result = content.propertyCollector.ContinueRetrievePropertiesEx(token=result.token) + property_collector = content.propertyCollector + result = property_collector.ContinueRetrievePropertiesEx( + token=result.token) else: break view.Destroy() - logger.info(f'Vcenter: Retrieved {len(vms_with_ip)} VMs with IPs that have any of the specified attribute names') + logger.info( + f'Vcenter: Retrieved {len(vms_with_ip)} VMs with IPs') return vms_with_ip - def parse_inventory(self, inventory_list: list) -> Dict: inventory = {} for name, ip in inventory_list.items(): @@ -203,7 +217,8 @@ def parse_inventory(self, inventory_list: list) -> Dict: 'namespace': namespace, 'hostname': name, } - logger.info(f'Vcenter: Acting on inventory of {len(inventory)} devices') + logger.info( + f'Vcenter: Acting on inventory of {len(inventory)} devices') return inventory async def _execute(self): @@ -217,4 +232,4 @@ async def _execute(self): async def _stop(self): if self._session: - Disconnect(self._session) \ No newline at end of file + Disconnect(self._session) From 24e441009f984483adac3c148134f8b0969b50e2 Mon Sep 17 00:00:00 2001 From: Tushar Date: Sat, 11 May 2024 02:38:09 +0000 Subject: [PATCH 3/6] Fix pylint Signed-off-by: Tushar --- suzieq/poller/controller/source/vcenter.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/suzieq/poller/controller/source/vcenter.py b/suzieq/poller/controller/source/vcenter.py index 301e34fdd4..cc741a3bfe 100644 --- a/suzieq/poller/controller/source/vcenter.py +++ b/suzieq/poller/controller/source/vcenter.py @@ -10,9 +10,10 @@ import logging from typing import Dict, List, Optional, Union from urllib.parse import urlparse -from pyVim.connect import SmartConnect, Disconnect -from pyVmomi import vim, vmodl import ssl +from pyVim.connect import Disconnect, SmartConnect +from pyVmomi import vim, vmodl + from pydantic import BaseModel, validator, Field @@ -84,6 +85,9 @@ def validate_password(cls, password): class Vcenter(Source, InventoryAsyncPlugin): + """This class is used to dynamically retrieve the inventory + from Vcenter + """ def __init__(self, config_data: dict, validate: bool = True) -> None: self._status = 'init' self._server: VcenterServerModel = None @@ -208,7 +212,15 @@ async def get_inventory_list(self) -> List: f'Vcenter: Retrieved {len(vms_with_ip)} VMs with IPs') return vms_with_ip - def parse_inventory(self, inventory_list: list) -> Dict: + def parse_inventory(self, inventory_list: dict) -> Dict: + """parse the raw inventory collected from the server and generates + a new inventory with only the required information. + + Args: + raw_inventory: raw inventory received from vcenter. + + Returns: A dict containing the inventory. + """ inventory = {} for name, ip in inventory_list.items(): namespace = self._namespace From f21e0f12e647659bcde77b3fe5f4876456ec8351 Mon Sep 17 00:00:00 2001 From: Tushar Date: Mon, 3 Jun 2024 15:21:36 +0000 Subject: [PATCH 4/6] Update poetry lock Signed-off-by: Tushar --- poetry.lock | 6 +----- pyproject.toml | 3 --- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/poetry.lock b/poetry.lock index 2c77f0cce1..eb4f2a0c13 100644 --- a/poetry.lock +++ b/poetry.lock @@ -5111,8 +5111,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">3.8.1, < 3.10" -<<<<<<< HEAD -content-hash = "d5d9b1c1146a6684735d7e4c4016db001f8faad1eb0a55e00daac6dc4fc070fe" -======= -content-hash = "49314731cbc5a57e03fe03c0b3039aeec30db4ff2b37b6c3bc42a9cf71fc3090" ->>>>>>> 5f7cedbc37 (Add vcenter as inventory source) +content-hash = "ece1db60a0522861ddeff3f1274a91e08181ac0bafc0955199cbd0a934b6b2f8" diff --git a/pyproject.toml b/pyproject.toml index 359c246f56..a6bd7bad57 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,12 +52,9 @@ packaging = "~21.3" psutil = "~5.9.4" jellyfish = "~0.10" altair = '>3.2, <5.0' -<<<<<<< HEAD pydantic = '< 2.0' numpy = '~1.20' -======= pyvmomi = "^8.0.2.0.1" ->>>>>>> 5f7cedbc37 (Add vcenter as inventory source) [tool.poetry.dev-dependencies] pylint = "*" From 042bff38c438f96195a9577d53cfcb7cada9b8fb Mon Sep 17 00:00:00 2001 From: Michael Bear <38406045+mjbear@users.noreply.github.com> Date: Sat, 10 Aug 2024 18:17:52 -0400 Subject: [PATCH 5/6] fix a few markdown hyperlinks Signed-off-by: Michael Bear <38406045+mjbear@users.noreply.github.com> --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ea590f5b89..70c3aa34d3 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ If you answered yes to one or more of these questions, then SuzieQ is a tool tha SuzieQ does multiple things. It [collects](https://suzieq.readthedocs.io/en/latest/poller/) data from devices and systems across your network. It normalizes the data and then stores it in a vendor independent way. Then it allows analysis of that data. With the applications that we build on top of the framework we want to demonstrate a different and more systematic approach to thinking about networks. We want to show how useful it is to think of your network holistically. -**An enterprise version of SuzieQ is also available**. It has been deployed in production by multiple customers, and the company behind SuzieQ, (Stardust Systems)[https://stardustsystems.net] was named a "Cool Vendor" by Gartner for making network automation easy for enterprises. +**An enterprise version of SuzieQ is also available**. It has been deployed in production by multiple customers, and the company behind SuzieQ, [Stardust Systems](https://stardustsystems.net) was named a "Cool Vendor" by Gartner for making network automation easy for enterprises. ## Quick Start @@ -132,4 +132,4 @@ We've also been adding screencasts on [Youtube](https://www.youtube.com/results? # SuzieQ Enterprise -SuzieQ also has a commercial offering, SuzieQ Enterprise. To know more about this and contact us, please visit the Stardust Systems (website)[https://stardustsystems.net]. +SuzieQ also has a commercial offering, SuzieQ Enterprise. To know more about this and contact us, please visit the Stardust Systems [website](https://stardustsystems.net). From 6b5114d7d799adbca8a12bd688f936a38c9f0557 Mon Sep 17 00:00:00 2001 From: Vivek Date: Sun, 11 Aug 2024 14:34:34 +0930 Subject: [PATCH 6/6] Fix typos in README.md Signed-off-by: Vivek --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ea590f5b89..af8820a866 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ To start collecting data for your network, create an inventory file to gather th ### Using Python Packaging -If you don't want to use docker container or cannot use a docker container, an alternative approach is to install SuzieQ as a python package. It is **strongly** recommended to install suzieq inside a virtual environment. If you already use a tool to create and manage virtual environments, you can skip the step of creating a virtual envirobment below. +If you don't want to use docker container or cannot use a docker container, an alternative approach is to install SuzieQ as a python package. It is **strongly** recommended to install suzieq inside a virtual environment. If you already use a tool to create and manage virtual environments, you can skip the step of creating a virtual environment below. SuzieQ requires python version 3.7.1 at least, and has been tested with python versions 3.7 and 3.8. It has not been tested to work on Windows. Use Linux (recommended) or macOS. To create a virtual environment, in case you haven't got a tool to create one, type: @@ -86,7 +86,7 @@ The CLI supports the same kind of analysis as the explore page. ![CLI device](im ## Path -SuzieQ has the ability to show the path between two IP addresses, including the ability to show the path through EVPN overlay. You can use this to see each of the paths from a source to a destination and to see if you have anything asymetrical in your paths. ![GUI PATH](images/path-gui.png) +SuzieQ has the ability to show the path between two IP addresses, including the ability to show the path through EVPN overlay. You can use this to see each of the paths from a source to a destination and to see if you have anything asymmetrical in your paths. ![GUI PATH](images/path-gui.png) ## Asserts