From cbd0c24da3a7fb96483597d1bbae13087b0c6509 Mon Sep 17 00:00:00 2001 From: Rustam Date: Thu, 9 Jun 2022 08:59:02 +0300 Subject: [PATCH 1/2] Now collection elements with different schemas are stored in different files. --- sebastes/processor/processor.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/sebastes/processor/processor.py b/sebastes/processor/processor.py index 83504ff..84c1809 100644 --- a/sebastes/processor/processor.py +++ b/sebastes/processor/processor.py @@ -1,3 +1,4 @@ +import collections import dataclasses import os import typing @@ -25,6 +26,7 @@ def __init__(self, redfish_datas: typing.List[RedfishData], base_dir: str): self._redfish_datas = redfish_datas self._base_dir = base_dir self._problems: typing.List[Problem] = [] + self._collections = collections.defaultdict(int) @property def problems(self) -> typing.List[Problem]: @@ -94,7 +96,7 @@ def generate_lib(self) -> None: for data in self._redfish_datas: if data.category == RedfishCategory.ELEMENT: - if data not in processed_data and data.parent not in processed_data: + if data not in processed_data: processed_data.append(data) processed_data.append(data.parent) element = self._process_data(data) @@ -106,7 +108,12 @@ def generate_lib(self) -> None: for from_, targets in collection.result.imports.items(): for target in targets: imports.append(Import(from_=from_, import_=target, alias=None)) - file_name = collection.redfish_data.file_name + collection_file_name = collection.redfish_data.file_name + self._collections[collection_file_name] += 1 + if self._collections[collection_file_name] != 1: + file_name = f'{collection_file_name}{self._collections[collection_file_name]}' + else: + file_name = collection.redfish_data.file_name self._save_module([element.result.body, collection.result.body], imports, file_name) init_data.append((file_name, [collection.redfish_data.full_name, element.redfish_data.full_name])) From 1cae65d5945cb83b577653dcb00da4ec9bcb3a20 Mon Sep 17 00:00:00 2001 From: Rustam Date: Thu, 9 Jun 2022 10:44:44 +0300 Subject: [PATCH 2/2] Process collection elements with the same parents but with different schemas was added. --- sebastes/scanner/scanner.py | 70 ++++++++++++++++++++++++++++--------- 1 file changed, 54 insertions(+), 16 deletions(-) diff --git a/sebastes/scanner/scanner.py b/sebastes/scanner/scanner.py index 91a0508..dd84fe9 100644 --- a/sebastes/scanner/scanner.py +++ b/sebastes/scanner/scanner.py @@ -1,6 +1,9 @@ +import collections +import copy import dataclasses import enum import functools +import hashlib import json import random import re @@ -82,7 +85,7 @@ def category(self) -> RedfishCategory: return RedfishCategory.COLLECTION elif self.parent is not None and \ self.parent.category == RedfishCategory.COLLECTION and \ - self.name in self.parent.name: + self.name[:1] in self.parent.name: # if name without additional number. return RedfishCategory.ELEMENT else: return RedfishCategory.RESOURCE @@ -101,6 +104,13 @@ def name(self) -> str: """ return self._name + @name.setter + def name(self, name: str) -> None: + """ + Redfish model name + """ + self._name = name + @property def full_name(self) -> str: """ @@ -156,10 +166,15 @@ def __str__(self) -> str: def __eq__(self, other: typing.Any) -> bool: if isinstance(other, self.__class__): - return other.full_name == self.full_name and other.parent == self.parent + return other.full_name == self.full_name and other.parent == self.parent and other.__hash__() == self.__hash__() else: return False + def __hash__(self): + hash_str = ''.join(sorted([k for k in self._data.keys()])) + hash_str = hash_str + self._name + return hashlib.md5(hash_str.encode()).hexdigest() + # noinspection PyBroadException class Scanner: @@ -203,19 +218,22 @@ def _get_model_name(data: dict) -> typing.Optional[str]: def _get_uris(self, data: dict) -> typing.List[str]: result: typing.List[str] = [] - for key, value in data.items(): - if key == '@odata.id' and value not in result: - result.append(value) - if key == 'Members': - if len(data['Members']) > self._max_collection: - for _entry in random.sample(data['Members'], self._max_collection): - result += self._get_uris(_entry) - else: - for _entry in data['Members']: - result += self._get_uris(_entry) - if isinstance(value, dict): - result += self._get_uris(value) - return result + try: + for key, value in data.items(): + if key == '@odata.id' and value not in result: + result.append(value) + if key == 'Members': + if len(data['Members']) > self._max_collection: + for _entry in random.sample(data['Members'], self._max_collection): + result += self._get_uris(_entry) + else: + for _entry in data['Members']: + result += self._get_uris(_entry) + if isinstance(value, dict): + result += self._get_uris(value) + return result + except Exception as e: + print(e) def _get_json(self, url: str) -> dict: headers = {'content-type': 'application/json'} @@ -236,6 +254,10 @@ def _get_json(self, url: str) -> dict: raise Exception(response.content.decode()) def scan_models(self, entry_point: str = "/redfish/v1/", parent: typing.Optional[RedfishData] = None) -> None: + self._scan_models(entry_point, parent) + self._process_duplicated_names() + + def _scan_models(self, entry_point: str, parent: typing.Optional[RedfishData] = None) -> None: """ Scan target endpoint and all it's children. :param entry_point: scan start point @@ -261,8 +283,24 @@ def scan_models(self, entry_point: str = "/redfish/v1/", parent: typing.Optional for uri in self._get_uris(data): if uri not in self._scanned_uris: - self.scan_models(entry_point=uri, parent=model) + self._scan_models(entry_point=uri, parent=model) except Exception as error: self._problems.append(Problem(url=entry_point, description=str(error))) else: log.info(f'Models limit was reached - {self._max_models}') + + def _process_duplicated_names(self): + """ + Process collection elements with the same parents but with different schemas. + """ + scanned_models = collections.defaultdict(int) + scanned_parents = [] + for model in self._redfish: + scanned_models[model.name] += 1 + if scanned_models[model.name] > 1: # If there are models with the same parent and different schemas. + if model.parent in scanned_parents and model.category == RedfishCategory.ELEMENT: + new_parent = copy.deepcopy(model.parent) # Create new parent with new name. + new_parent.name = f'{new_parent.name}{scanned_models[model.name]}' + model._parent = new_parent + model.name = f'{model.name}{scanned_models[model.name]}' + scanned_parents.append(model.parent)