From eadd734918d1c7e6238ccb756fc470dd567c31c9 Mon Sep 17 00:00:00 2001 From: AnthonyLaw Date: Sun, 26 Mar 2023 19:09:12 +0800 Subject: [PATCH 01/13] [explorer/nodewatch] task: add new filed on NodeDescriptor and unit test --- .../nodewatch/nodewatch/NetworkRepository.py | 18 ++++ .../tests/resources/symbol_nodes.json | 22 +++++ .../nodewatch/tests/test_NetworkRepository.py | 98 ++++++++++++++++++- 3 files changed, 135 insertions(+), 3 deletions(-) diff --git a/explorer/nodewatch/nodewatch/NetworkRepository.py b/explorer/nodewatch/nodewatch/NetworkRepository.py index 5e14e1f0a..4f3825251 100644 --- a/explorer/nodewatch/nodewatch/NetworkRepository.py +++ b/explorer/nodewatch/nodewatch/NetworkRepository.py @@ -23,6 +23,10 @@ def __init__( height=0, finalized_height=0, balance=0, + is_healthy=None, + is_https_enable=None, + is_wss_enable=None, + rest_version=None, roles=0xFF): """Creates a descriptor.""" @@ -37,6 +41,10 @@ def __init__( self.height = height self.finalized_height = finalized_height self.balance = balance + self.is_healthy = is_healthy + self.is_https_enable = is_https_enable + self.is_wss_enable = is_wss_enable + self.rest_version = rest_version self.roles = roles @property @@ -57,6 +65,10 @@ def to_json(self): 'height': self.height, 'finalizedHeight': self.finalized_height, 'balance': self.balance, + 'isHealthy': self.is_healthy, + 'isHttpsEnable': self.is_https_enable, + 'isWssEnable': self.is_wss_enable, + 'restVersion': self.rest_version, 'roles': self.roles } @@ -192,6 +204,11 @@ def _create_descriptor_from_json(self, json_node): if self._network.generation_hash_seed != Hash256(json_node['networkGenerationHashSeed']): return None + api_node_info_data = (None, None, None, None) + if 'apiNodeInfo' in json_node: + json_api_node_info_data = json_node['apiNodeInfo'] + api_node_info_data = (json_api_node_info_data.get('isHealth', None), json_api_node_info_data.get('isHttpsEnable', None), json_api_node_info_data.get('isWssEnable', None), json_api_node_info_data.get('restVersion', None)) + main_public_key = PublicKey(json_node['publicKey']) node_public_key = PublicKey(json_node['nodePublicKey']) if 'nodePublicKey' in json_node else None return NodeDescriptor( @@ -202,6 +219,7 @@ def _create_descriptor_from_json(self, json_node): json_node['friendlyName'], self._format_symbol_version(json_node['version']), *extra_data, + *api_node_info_data, roles) @staticmethod diff --git a/explorer/nodewatch/tests/resources/symbol_nodes.json b/explorer/nodewatch/tests/resources/symbol_nodes.json index 5358e3e39..d388f1142 100644 --- a/explorer/nodewatch/tests/resources/symbol_nodes.json +++ b/explorer/nodewatch/tests/resources/symbol_nodes.json @@ -91,5 +91,27 @@ "host": "02.symbol-node.net", "friendlyName": "Apple", "nodePublicKey": "FBEAFCB15D2674ECB8DC1CD2C028C4AC0D463489069FDD415F30BB71EAE69864" + }, + { + "apiNodeInfo": { + "isHealth": true, + "isHttpsEnable": true, + "isWssEnable": true, + "restVersion": "2.4.2" + }, + "extraData": { + "balance": 101027.849383, + "finalizedHeight": 1486740, + "height": 1486762 + }, + "friendlyName": "xym pool", + "host": "xym.pool.me", + "networkGenerationHashSeed": "57F7DA205008026C776CB6AED843393F04CD458E0AA2D9F1D5F31A402072B2D6", + "networkIdentifier": 104, + "nodePublicKey": "FE7D3DBE8DDD219E1B20247DEBF150D9411EA5A312989103B037EFBD9D237DE0", + "port": 7900, + "publicKey": "A54CC798373F42B569AF21845CD0EBE755AB42EA04B3B8E2BE897166F89A971C", + "roles": 3, + "version": 16777987 } ] diff --git a/explorer/nodewatch/tests/test_NetworkRepository.py b/explorer/nodewatch/tests/test_NetworkRepository.py index f5d9c3799..cbda1cc80 100644 --- a/explorer/nodewatch/tests/test_NetworkRepository.py +++ b/explorer/nodewatch/tests/test_NetworkRepository.py @@ -15,7 +15,9 @@ class NetworkRepositoryTest(unittest.TestCase): def _assert_node_descriptor(self, descriptor, **kwargs): property_names = [ 'main_address', 'main_public_key', 'node_public_key', - 'endpoint', 'name', 'height', 'finalized_height', 'version', 'balance', 'roles', 'has_api' + 'endpoint', 'name', 'height', 'finalized_height', 'version', 'balance', + 'is_healthy', 'is_https_enable', 'is_wss_enable', 'rest_version', + 'roles', 'has_api' ] for name in property_names: self.assertEqual(kwargs[name], getattr(descriptor, name)) @@ -54,6 +56,10 @@ def test_can_load_nem_node_descriptors(self): version='0.6.100', balance=3355922.652725, roles=0xFF, + is_healthy=None, + is_https_enable=None, + is_wss_enable=None, + rest_version=None, has_api=True) self._assert_node_descriptor( repository.node_descriptors[1], @@ -67,6 +73,10 @@ def test_can_load_nem_node_descriptors(self): version='0.6.100', balance=20612359.516967, roles=0xFF, + is_healthy=None, + is_https_enable=None, + is_wss_enable=None, + rest_version=None, has_api=True) self._assert_node_descriptor( repository.node_descriptors[2], @@ -80,6 +90,10 @@ def test_can_load_nem_node_descriptors(self): version='0.6.99', balance=0, roles=0xFF, + is_healthy=None, + is_https_enable=None, + is_wss_enable=None, + rest_version=None, has_api=True) # simulates missing extraData self._assert_node_descriptor( repository.node_descriptors[3], @@ -93,6 +107,10 @@ def test_can_load_nem_node_descriptors(self): version='0.6.100', balance=0, roles=0xFF, + is_healthy=None, + is_https_enable=None, + is_wss_enable=None, + rest_version=None, has_api=True) # simulates incomplete extraData def test_can_load_zero_symbol_node_descriptors(self): @@ -114,9 +132,9 @@ def test_can_load_symbol_node_descriptors(self): # Assert: descriptors are sorted by name (desc) self.assertFalse(repository.is_nem) - self.assertEqual(6, len(repository.node_descriptors)) + self.assertEqual(7, len(repository.node_descriptors)) self.assertEqual(1486760, repository.estimate_height()) # median - self.assertEqual(1486740, repository.estimate_finalized_height()) # median (nonzero) + self.assertEqual(1486742, repository.estimate_finalized_height()) # median (nonzero) self._assert_node_descriptor( repository.node_descriptors[0], main_address=SymbolAddress('NDZOZPTDVCFFLDCNJL7NZGDQDNBB7TY3V6SZNGI'), @@ -129,6 +147,10 @@ def test_can_load_symbol_node_descriptors(self): version='1.0.3.4', balance=3155632.471994, roles=2, + is_healthy=None, + is_https_enable=None, + is_wss_enable=None, + rest_version=None, has_api=True) # simulates missing host self._assert_node_descriptor( repository.node_descriptors[1], @@ -142,6 +164,10 @@ def test_can_load_symbol_node_descriptors(self): version='1.0.3.3', balance=0, roles=7, + is_healthy=None, + is_https_enable=None, + is_wss_enable=None, + rest_version=None, has_api=True) # old version mapped to 'failure' self._assert_node_descriptor( repository.node_descriptors[2], @@ -155,6 +181,10 @@ def test_can_load_symbol_node_descriptors(self): version='1.0.3.5', balance=0, roles=3, + is_healthy=None, + is_https_enable=None, + is_wss_enable=None, + rest_version=None, has_api=True) # simulates incomplete extraData self._assert_node_descriptor( repository.node_descriptors[3], @@ -168,6 +198,10 @@ def test_can_load_symbol_node_descriptors(self): version='1.0.3.5', balance=82375.554976, roles=3, + is_healthy=None, + is_https_enable=None, + is_wss_enable=None, + rest_version=None, has_api=True) self._assert_node_descriptor( repository.node_descriptors[4], @@ -181,6 +215,10 @@ def test_can_load_symbol_node_descriptors(self): version='1.0.3.5', balance=28083310.571743, roles=5, + is_healthy=None, + is_https_enable=None, + is_wss_enable=None, + rest_version=None, has_api=False) self._assert_node_descriptor( repository.node_descriptors[5], @@ -194,7 +232,28 @@ def test_can_load_symbol_node_descriptors(self): version='1.0.3.4', balance=0, roles=3, + is_healthy=None, + is_https_enable=None, + is_wss_enable=None, + rest_version=None, has_api=True) # simulates missing extraData + self._assert_node_descriptor( + repository.node_descriptors[6], + main_address=SymbolAddress('NAU6BZUX5GHI7EDE6DMS6GVHXS4XZFCNVRPT2OQ'), + main_public_key=PublicKey('A54CC798373F42B569AF21845CD0EBE755AB42EA04B3B8E2BE897166F89A971C'), + node_public_key=PublicKey('FE7D3DBE8DDD219E1B20247DEBF150D9411EA5A312989103B037EFBD9D237DE0'), + endpoint='http://xym.pool.me:3000', + name='xym pool', + height=2094498, + finalized_height=2094464, + version='1.0.3.3', + balance=101027.849383, + roles=3, + is_healthy=True, + is_https_enable=True, + is_wss_enable=True, + rest_version='2.4.2', + has_api=True) def test_can_format_node_descriptor_as_json(self): # Arrange: @@ -214,6 +273,10 @@ def test_can_format_node_descriptor_as_json(self): 'finalizedHeight': 1486742, 'version': '1.0.3.5', 'balance': 28083310.571743, + 'isHealthy': None, + 'isHttpsEnable': None, + 'isWssEnable': None, + 'restVersion': None, 'roles': 5, }, json_object) @@ -235,6 +298,35 @@ def test_can_format_node_descriptor_with_node_public_key_as_json(self): 'finalizedHeight': 1486740, 'version': '1.0.3.5', 'balance': 82375.554976, + 'isHealthy': None, + 'isHttpsEnable': None, + 'isWssEnable': None, + 'restVersion': None, + 'roles': 3, + }, json_object) + + def test_can_format_node_descriptor_with_api_node_info_as_json(self): + # Arrange: + repository = NetworkRepository(SymbolNetwork.MAINNET, 'symbol') + repository.load_node_descriptors('tests/resources/symbol_nodes.json') + + # Act: + json_object = repository.node_descriptors[6].to_json() + + # Assert: + self.assertEqual({ + 'mainPublicKey': 'A54CC798373F42B569AF21845CD0EBE755AB42EA04B3B8E2BE897166F89A971C', + 'nodePublicKey': 'FE7D3DBE8DDD219E1B20247DEBF150D9411EA5A312989103B037EFBD9D237DE0', + 'endpoint': 'http://xym.pool.me:3000', + 'name': 'xym pool', + 'height': 2094498, + 'finalizedHeight': 2094464, + 'version': '1.0.3.3', + 'balance': 101027.849383, + 'isHealthy': True, + 'isHttpsEnable': True, + 'isWssEnable': True, + 'restVersion': '2.4.2', 'roles': 3, }, json_object) From 574bc4d9f657c94a56d7016d1c2c654fa0e3e442 Mon Sep 17 00:00:00 2001 From: AnthonyLaw Date: Sun, 26 Mar 2023 19:09:41 +0800 Subject: [PATCH 02/13] [explorer/nodewatch] task: update unit test --- explorer/nodewatch/tests/test_app.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/explorer/nodewatch/tests/test_app.py b/explorer/nodewatch/tests/test_app.py index 2e7749d1f..5f19064f6 100644 --- a/explorer/nodewatch/tests/test_app.py +++ b/explorer/nodewatch/tests/test_app.py @@ -184,9 +184,9 @@ def test_get_api_symbol_nodes_peer(client): # pylint: disable=redefined-outer-n # Assert: spot check names assert 200 == response.status_code assert 'application/json' == response.headers['Content-Type'] - assert 5 == len(response_json) + assert 6 == len(response_json) assert [ - 'Apple', 'Shin-Kuma-Node', 'ibone74', 'jaguar', 'symbol.ooo maxUnlockedAccounts:100' + 'Apple', 'Shin-Kuma-Node', 'ibone74', 'jaguar', 'symbol.ooo maxUnlockedAccounts:100', 'xym pool' ] == list(map(lambda descriptor: descriptor['name'], response_json)) @@ -211,7 +211,7 @@ def test_get_api_symbol_network_height_chart(client): # pylint: disable=redefin assert 200 == response.status_code assert 'application/json' == response.headers['Content-Type'] assert 2 == len(response_json) - assert 4 == len(chart_json['data']) + assert 6 == len(chart_json['data']) assert re.match(r'\d\d:\d\d', response_json['lastRefreshTime']) # endregion From 5052074414096fc18ca9938370e732268a81c1c8 Mon Sep 17 00:00:00 2001 From: AnthonyLaw Date: Tue, 28 Mar 2023 15:20:48 +0800 Subject: [PATCH 03/13] [explorer/nodewatch] task: add few new api symbol endpoint --- explorer/nodewatch/nodewatch/RoutesFacade.py | 39 ++++++-- explorer/nodewatch/nodewatch/__init__.py | 39 +++++++- .../tests/resources/symbol_nodes.json | 22 +++++ .../nodewatch/tests/test_NetworkRepository.py | 29 ++++-- explorer/nodewatch/tests/test_app.py | 90 ++++++++++++++++++- 5 files changed, 203 insertions(+), 16 deletions(-) diff --git a/explorer/nodewatch/nodewatch/RoutesFacade.py b/explorer/nodewatch/nodewatch/RoutesFacade.py index 78c2c6da0..83ef087dd 100644 --- a/explorer/nodewatch/nodewatch/RoutesFacade.py +++ b/explorer/nodewatch/nodewatch/RoutesFacade.py @@ -1,5 +1,7 @@ import asyncio import datetime +import random +from enum import Enum from zenlog import log @@ -65,15 +67,40 @@ def html_nodes(self): 'explorer_endpoint': self.explorer_endpoint }) - def json_nodes(self, role, exact_match=False): - """Returns all nodes with matching role.""" + def json_nodes(self, role=None, exact_match=False, limit=None, ssl=None, order=None): + """Returns all nodes with condition.""" - def role_filter(descriptor): - return role == descriptor.roles if exact_match else role == (role & descriptor.roles) + def filter_custom(descriptor): + role_condition = True - return list(map( + if role is not None: + role_condition = role == descriptor.roles if exact_match else role == (role & descriptor.roles) + + if ssl is not None: + ssl_condition = (descriptor.is_https_enable == ssl and descriptor.is_wss_enable == ssl) + return role_condition and ssl_condition + + return role_condition + + nodes = list(map( lambda descriptor: descriptor.to_json(), - filter(role_filter, self.repository.node_descriptors))) + filter(filter_custom, self.repository.node_descriptors))) + + if order is not None: + if order == 'random': + random.shuffle(nodes) + + if limit is not None: + nodes = nodes[:limit] + + return nodes + + def json_node(self, filter_field, public_key): + """Returns a node with matching public key.""" + + matching_items = [item.to_json() for item in self.repository.node_descriptors if item.to_json()[filter_field] == public_key] + + return next(iter(matching_items), None) def json_height_chart(self): """Builds a JSON height chart.""" diff --git a/explorer/nodewatch/nodewatch/__init__.py b/explorer/nodewatch/nodewatch/__init__.py index 80e425f61..0fbac8d8d 100644 --- a/explorer/nodewatch/nodewatch/__init__.py +++ b/explorer/nodewatch/nodewatch/__init__.py @@ -1,3 +1,4 @@ +from enum import Enum from pathlib import Path from apscheduler.schedulers.background import BackgroundScheduler @@ -11,6 +12,18 @@ from .RoutesFacade import MIN_HEIGHT_CLUSTER_SIZE, TIMESTAMP_FORMAT, NemRoutesFacade, SymbolRoutesFacade +class Field(Enum): + MAIN_PUBLIC_KEY = "mainPublicKey" + NODE_PUBLIC_KEY = "nodePublicKey" + +def str_to_bool(value): + if value.lower() == 'true': + return True + elif value.lower() == 'false': + return False + else: + return None + def create_app(): # pylint: disable=too-many-locals @@ -93,11 +106,33 @@ def symbol_summary(): # pylint: disable=unused-variable @app.route('/api/symbol/nodes/api') def api_symbol_nodes_api(): # pylint: disable=unused-variable - return jsonify(symbol_routes_facade.json_nodes(2, exact_match=True)) + return jsonify(symbol_routes_facade.json_nodes(role=2, exact_match=True)) @app.route('/api/symbol/nodes/peer') def api_symbol_nodes_peer(): # pylint: disable=unused-variable - return jsonify(symbol_routes_facade.json_nodes(1)) + return jsonify(symbol_routes_facade.json_nodes(role=1)) + + @app.route('/api/symbol/nodes') + def api_symbol_nodes(): # pylint: disable=unused-variable + ssl = request.args.get('ssl', None) + limit = request.args.get('limit', None) + order = request.args.get('order', None) + + if ssl is not None: + ssl = str_to_bool(ssl) + + if limit is not None: + limit = int(limit) + + return jsonify(symbol_routes_facade.json_nodes(ssl=ssl, limit=limit, order=order)) + + @app.route('/api/symbol/nodes/mainPublicKey/') + def api_symbol_nodes_get_main_public_key(main_public_key): # pylint: disable=unused-variable + return jsonify(symbol_routes_facade.json_node(filter_field=Field.MAIN_PUBLIC_KEY.value, public_key=main_public_key)) + + @app.route('/api/symbol/nodes/nodePublicKey/') + def api_symbol_nodes_get_node_public_key(node_public_key): # pylint: disable=unused-variable + return jsonify(symbol_routes_facade.json_node(filter_field=Field.NODE_PUBLIC_KEY.value, public_key=node_public_key)) @app.route('/api/symbol/chart/height') def api_symbol_chart_height(): # pylint: disable=unused-variable diff --git a/explorer/nodewatch/tests/resources/symbol_nodes.json b/explorer/nodewatch/tests/resources/symbol_nodes.json index d388f1142..19caeb7aa 100644 --- a/explorer/nodewatch/tests/resources/symbol_nodes.json +++ b/explorer/nodewatch/tests/resources/symbol_nodes.json @@ -113,5 +113,27 @@ "publicKey": "A54CC798373F42B569AF21845CD0EBE755AB42EA04B3B8E2BE897166F89A971C", "roles": 3, "version": 16777987 + }, + { + "apiNodeInfo": { + "isHealth": true, + "isHttpsEnable": false, + "isWssEnable": false, + "restVersion": "2.4.2" + }, + "extraData": { + "balance": 99.98108, + "finalizedHeight": 1486740, + "height": 1486760 + }, + "friendlyName": "yasmine farm", + "host": "yasmine.farm.tokyo", + "networkGenerationHashSeed": "57F7DA205008026C776CB6AED843393F04CD458E0AA2D9F1D5F31A402072B2D6", + "networkIdentifier": 104, + "nodePublicKey": "D05BE3101F2916AA34839DDC1199BE45092103A9B66172FA3D05911DC041AADA", + "port": 7900, + "publicKey": "5B20F8F228FF0E064DB0DE7951155F6F41EF449D0EC10960067C2BF2DCD61874", + "roles": 3, + "version": 16777988 } ] diff --git a/explorer/nodewatch/tests/test_NetworkRepository.py b/explorer/nodewatch/tests/test_NetworkRepository.py index cbda1cc80..4f4701187 100644 --- a/explorer/nodewatch/tests/test_NetworkRepository.py +++ b/explorer/nodewatch/tests/test_NetworkRepository.py @@ -132,9 +132,9 @@ def test_can_load_symbol_node_descriptors(self): # Assert: descriptors are sorted by name (desc) self.assertFalse(repository.is_nem) - self.assertEqual(7, len(repository.node_descriptors)) + self.assertEqual(8, len(repository.node_descriptors)) self.assertEqual(1486760, repository.estimate_height()) # median - self.assertEqual(1486742, repository.estimate_finalized_height()) # median (nonzero) + self.assertEqual(1486740, repository.estimate_finalized_height()) # median (nonzero) self._assert_node_descriptor( repository.node_descriptors[0], main_address=SymbolAddress('NDZOZPTDVCFFLDCNJL7NZGDQDNBB7TY3V6SZNGI'), @@ -244,8 +244,8 @@ def test_can_load_symbol_node_descriptors(self): node_public_key=PublicKey('FE7D3DBE8DDD219E1B20247DEBF150D9411EA5A312989103B037EFBD9D237DE0'), endpoint='http://xym.pool.me:3000', name='xym pool', - height=2094498, - finalized_height=2094464, + height=1486762, + finalized_height=1486740, version='1.0.3.3', balance=101027.849383, roles=3, @@ -254,6 +254,23 @@ def test_can_load_symbol_node_descriptors(self): is_wss_enable=True, rest_version='2.4.2', has_api=True) + self._assert_node_descriptor( + repository.node_descriptors[7], + main_address=SymbolAddress('NAOOI6NZA6TZMIKOGAQCQG7SXBPXVSDOTPLLDZY'), + main_public_key=PublicKey('5B20F8F228FF0E064DB0DE7951155F6F41EF449D0EC10960067C2BF2DCD61874'), + node_public_key=PublicKey('D05BE3101F2916AA34839DDC1199BE45092103A9B66172FA3D05911DC041AADA'), + endpoint='http://yasmine.farm.tokyo:3000', + name='yasmine farm', + height=1486760, + finalized_height=1486740, + version='1.0.3.4', + balance=99.98108, + roles=3, + is_healthy=True, + is_https_enable=False, + is_wss_enable=False, + rest_version='2.4.2', + has_api=True) # simulates ssl disabled def test_can_format_node_descriptor_as_json(self): # Arrange: @@ -319,8 +336,8 @@ def test_can_format_node_descriptor_with_api_node_info_as_json(self): 'nodePublicKey': 'FE7D3DBE8DDD219E1B20247DEBF150D9411EA5A312989103B037EFBD9D237DE0', 'endpoint': 'http://xym.pool.me:3000', 'name': 'xym pool', - 'height': 2094498, - 'finalizedHeight': 2094464, + 'height': 1486762, + 'finalizedHeight': 1486740, 'version': '1.0.3.3', 'balance': 101027.849383, 'isHealthy': True, diff --git a/explorer/nodewatch/tests/test_app.py b/explorer/nodewatch/tests/test_app.py index 5f19064f6..b6268e6e5 100644 --- a/explorer/nodewatch/tests/test_app.py +++ b/explorer/nodewatch/tests/test_app.py @@ -184,11 +184,97 @@ def test_get_api_symbol_nodes_peer(client): # pylint: disable=redefined-outer-n # Assert: spot check names assert 200 == response.status_code assert 'application/json' == response.headers['Content-Type'] - assert 6 == len(response_json) + assert 7 == len(response_json) assert [ - 'Apple', 'Shin-Kuma-Node', 'ibone74', 'jaguar', 'symbol.ooo maxUnlockedAccounts:100', 'xym pool' + 'Apple', 'Shin-Kuma-Node', 'ibone74', 'jaguar', 'symbol.ooo maxUnlockedAccounts:100', 'xym pool', 'yasmine farm' ] == list(map(lambda descriptor: descriptor['name'], response_json)) +def test_get_api_symbol_nodes(client): # pylint: disable=redefined-outer-name + # Act: + response = client.get('/api/symbol/nodes') + response_json = json.loads(response.data) + + # Assert: spot check names + assert 200 == response.status_code + assert 'application/json' == response.headers['Content-Type'] + assert 8 == len(response_json) + assert [ + 'Allnodes250', 'Apple', 'Shin-Kuma-Node', 'ibone74', 'jaguar', 'symbol.ooo maxUnlockedAccounts:100', 'xym pool', 'yasmine farm' + ] == list(map(lambda descriptor: descriptor['name'], response_json)) + +def test_get_api_symbol_nodes_order_random(client): # pylint: disable=redefined-outer-name + # Act: + response = client.get('/api/symbol/nodes?order=random') + response_json = json.loads(response.data) + + # Assert: spot check names + assert 200 == response.status_code + assert 'application/json' == response.headers['Content-Type'] + assert 8 == len(response_json) + expected_names = [ + 'Allnodes250', 'Apple', 'Shin-Kuma-Node', 'ibone74', 'jaguar', 'symbol.ooo maxUnlockedAccounts:100', 'xym pool', 'yasmine farm' + ] + actual_names = list(map(lambda descriptor: descriptor['name'], response_json)) + assert sorted(expected_names) == sorted(actual_names) + +def test_get_api_symbol_nodes_limit_2(client): # pylint: disable=redefined-outer-name + # Act: + response = client.get('/api/symbol/nodes?limit=2') + response_json = json.loads(response.data) + + # Assert: spot check names + assert 200 == response.status_code + assert 'application/json' == response.headers['Content-Type'] + assert 2 == len(response_json) + assert [ + 'Allnodes250', 'Apple' + ] == list(map(lambda descriptor: descriptor['name'], response_json)) + +def test_get_api_symbol_nodes_ssl_true(client): # pylint: disable=redefined-outer-name + # Act: + response = client.get('/api/symbol/nodes?ssl=true') + response_json = json.loads(response.data) + + # Assert: spot check names + assert 200 == response.status_code + assert 'application/json' == response.headers['Content-Type'] + assert 1 == len(response_json) + assert [ + 'xym pool' + ] == list(map(lambda descriptor: descriptor['name'], response_json)) + +def test_get_api_symbol_nodes_ssl_false(client): # pylint: disable=redefined-outer-name + # Act: + response = client.get('/api/symbol/nodes?ssl=false') + response_json = json.loads(response.data) + + # Assert: spot check names + assert 200 == response.status_code + assert 'application/json' == response.headers['Content-Type'] + assert 1 == len(response_json) + assert [ + 'yasmine farm' + ] == list(map(lambda descriptor: descriptor['name'], response_json)) + +def test_get_api_symbol_node_with_main_public_key(client): # pylint: disable=redefined-outer-name + # Act: + response = client.get('/api/symbol/nodes/mainPublicKey/A0AA48B6417BDB1845EB55FB0B1E13255EA8BD0D8FA29AD2D8A906E220571F21') + response_json = json.loads(response.data) + + # Assert: spot check names + assert 200 == response.status_code + assert 'application/json' == response.headers['Content-Type'] + assert 'Allnodes250' == response_json['name'] + +def test_get_api_symbol_node_with_node_public_key(client): # pylint: disable=redefined-outer-name + # Act: + response = client.get('/api/symbol/nodes/nodePublicKey/D05BE3101F2916AA34839DDC1199BE45092103A9B66172FA3D05911DC041AADA') + response_json = json.loads(response.data) + + # Assert: spot check names + assert 200 == response.status_code + assert 'application/json' == response.headers['Content-Type'] + assert 'yasmine farm' == response_json['name'] def test_get_api_symbol_network_height(client): # pylint: disable=redefined-outer-name # Act: From 3442a05e2ae39ccab4e9bfcb0034e452805410c8 Mon Sep 17 00:00:00 2001 From: AnthonyLaw Date: Tue, 28 Mar 2023 16:51:25 +0800 Subject: [PATCH 04/13] [explorer/nodewatch] task: fix lint issue --- .../nodewatch/nodewatch/NetworkRepository.py | 70 +++++++++++-------- explorer/nodewatch/nodewatch/RoutesFacade.py | 9 ++- explorer/nodewatch/nodewatch/__init__.py | 13 ++-- .../nodewatch/tests/test_NetworkRepository.py | 2 +- explorer/nodewatch/tests/test_RoutesFacade.py | 8 +-- explorer/nodewatch/tests/test_app.py | 26 ++++--- 6 files changed, 76 insertions(+), 52 deletions(-) diff --git a/explorer/nodewatch/nodewatch/NetworkRepository.py b/explorer/nodewatch/nodewatch/NetworkRepository.py index 4f3825251..1b0063973 100644 --- a/explorer/nodewatch/nodewatch/NetworkRepository.py +++ b/explorer/nodewatch/nodewatch/NetworkRepository.py @@ -162,37 +162,31 @@ def load_node_descriptors(self, nodes_data_filepath): # sort by name self.node_descriptors.sort(key=lambda descriptor: descriptor.name) - def _create_descriptor_from_json(self, json_node): - # network crawler extracts as much extra data as possible, but it might not always be available for all nodes - extra_data = (0, 0, 0) - if 'extraData' in json_node: - json_extra_data = json_node['extraData'] - extra_data = (json_extra_data.get('height', 0), json_extra_data.get('finalizedHeight', 0), json_extra_data.get('balance', 0)) + def _handle_nem_node(self, json_node, extra_data): + network_identifier = json_node['metaData']['networkId'] + if network_identifier < 0: + network_identifier += 0x100 - if self.is_nem: - network_identifier = json_node['metaData']['networkId'] - if network_identifier < 0: - network_identifier += 0x100 - - if self._network.identifier != network_identifier: - return None - - node_protocol = json_node['endpoint']['protocol'] - node_host = json_node['endpoint']['host'] - node_port = json_node['endpoint']['port'] - - json_identity = json_node['identity'] - main_public_key = PublicKey(json_identity['public-key']) - node_public_key = PublicKey(json_identity['node-public-key']) if 'node-public-key' in json_identity else None - return NodeDescriptor( - self._network.public_key_to_address(main_public_key), - main_public_key, - node_public_key, - f'{node_protocol}://{node_host}:{node_port}', - json_identity['name'], - json_node['metaData']['version'], - *extra_data) + if self._network.identifier != network_identifier: + return None + + node_protocol = json_node['endpoint']['protocol'] + node_host = json_node['endpoint']['host'] + node_port = json_node['endpoint']['port'] + + json_identity = json_node['identity'] + main_public_key = PublicKey(json_identity['public-key']) + node_public_key = PublicKey(json_identity['node-public-key']) if 'node-public-key' in json_identity else None + return NodeDescriptor( + self._network.public_key_to_address(main_public_key), + main_public_key, + node_public_key, + f'{node_protocol}://{node_host}:{node_port}', + json_identity['name'], + json_node['metaData']['version'], + *extra_data) + def _handle_symbol_node(self, json_node, extra_data): symbol_endpoint = '' roles = json_node['roles'] has_api = bool(roles & 2) @@ -207,7 +201,11 @@ def _create_descriptor_from_json(self, json_node): api_node_info_data = (None, None, None, None) if 'apiNodeInfo' in json_node: json_api_node_info_data = json_node['apiNodeInfo'] - api_node_info_data = (json_api_node_info_data.get('isHealth', None), json_api_node_info_data.get('isHttpsEnable', None), json_api_node_info_data.get('isWssEnable', None), json_api_node_info_data.get('restVersion', None)) + api_node_info_data = ( + json_api_node_info_data.get('isHealth', None), + json_api_node_info_data.get('isHttpsEnable', None), + json_api_node_info_data.get('isWssEnable', None), + json_api_node_info_data.get('restVersion', None)) main_public_key = PublicKey(json_node['publicKey']) node_public_key = PublicKey(json_node['nodePublicKey']) if 'nodePublicKey' in json_node else None @@ -222,6 +220,18 @@ def _create_descriptor_from_json(self, json_node): *api_node_info_data, roles) + def _create_descriptor_from_json(self, json_node): + # network crawler extracts as much extra data as possible, but it might not always be available for all nodes + extra_data = (0, 0, 0) + if 'extraData' in json_node: + json_extra_data = json_node['extraData'] + extra_data = (json_extra_data.get('height', 0), json_extra_data.get('finalizedHeight', 0), json_extra_data.get('balance', 0)) + + if self.is_nem: + return self._handle_nem_node(json_node, extra_data) + + return self._handle_symbol_node(json_node, extra_data) + @staticmethod def _format_symbol_version(version): version_parts = [(version >> 24) & 0xFF, (version >> 16) & 0xFF, (version >> 8) & 0xFF, version & 0xFF] diff --git a/explorer/nodewatch/nodewatch/RoutesFacade.py b/explorer/nodewatch/nodewatch/RoutesFacade.py index 83ef087dd..588c512e7 100644 --- a/explorer/nodewatch/nodewatch/RoutesFacade.py +++ b/explorer/nodewatch/nodewatch/RoutesFacade.py @@ -1,7 +1,6 @@ import asyncio import datetime import random -from enum import Enum from zenlog import log @@ -67,9 +66,15 @@ def html_nodes(self): 'explorer_endpoint': self.explorer_endpoint }) - def json_nodes(self, role=None, exact_match=False, limit=None, ssl=None, order=None): + def json_nodes(self, **kwargs): """Returns all nodes with condition.""" + role = kwargs.get('role', None) + exact_match = kwargs.get('exact_match', False) + limit = kwargs.get('limit', None) + ssl = kwargs.get('ssl', None) + order = kwargs.get('order', None) + def filter_custom(descriptor): role_condition = True diff --git a/explorer/nodewatch/nodewatch/__init__.py b/explorer/nodewatch/nodewatch/__init__.py index 0fbac8d8d..ea2bf021a 100644 --- a/explorer/nodewatch/nodewatch/__init__.py +++ b/explorer/nodewatch/nodewatch/__init__.py @@ -13,16 +13,17 @@ class Field(Enum): - MAIN_PUBLIC_KEY = "mainPublicKey" - NODE_PUBLIC_KEY = "nodePublicKey" + MAIN_PUBLIC_KEY = 'mainPublicKey' + NODE_PUBLIC_KEY = 'nodePublicKey' + def str_to_bool(value): if value.lower() == 'true': return True - elif value.lower() == 'false': + if value.lower() == 'false': return False - else: - return None + return None + def create_app(): # pylint: disable=too-many-locals @@ -74,7 +75,7 @@ def nem_summary(): # pylint: disable=unused-variable @app.route('/api/nem/nodes') def api_nem_nodes(): # pylint: disable=unused-variable - return jsonify(nem_routes_facade.json_nodes(1)) + return jsonify(nem_routes_facade.json_nodes(role=1)) @app.route('/api/nem/chart/height') def api_nem_chart_height(): # pylint: disable=unused-variable diff --git a/explorer/nodewatch/tests/test_NetworkRepository.py b/explorer/nodewatch/tests/test_NetworkRepository.py index 4f4701187..a4de6a68e 100644 --- a/explorer/nodewatch/tests/test_NetworkRepository.py +++ b/explorer/nodewatch/tests/test_NetworkRepository.py @@ -270,7 +270,7 @@ def test_can_load_symbol_node_descriptors(self): is_https_enable=False, is_wss_enable=False, rest_version='2.4.2', - has_api=True) # simulates ssl disabled + has_api=True) # simulates ssl disabled def test_can_format_node_descriptor_as_json(self): # Arrange: diff --git a/explorer/nodewatch/tests/test_RoutesFacade.py b/explorer/nodewatch/tests/test_RoutesFacade.py index 58bf2bbf9..13e64880a 100644 --- a/explorer/nodewatch/tests/test_RoutesFacade.py +++ b/explorer/nodewatch/tests/test_RoutesFacade.py @@ -150,7 +150,7 @@ def test_can_generate_nodes_json(self): facade.reload_all(Path('tests/resources'), True) # Act: - node_descriptors = facade.json_nodes(1) + node_descriptors = facade.json_nodes(role=1) # Assert: spot check names and roles self.assertEqual(4, len(node_descriptors)) @@ -361,7 +361,7 @@ def test_can_generate_nodes_json(self): facade.reload_all(Path('tests/resources'), True) # Act: - node_descriptors = facade.json_nodes(1) + node_descriptors = facade.json_nodes(role=1) # Assert: spot check names and roles self.assertEqual(5, len(node_descriptors)) @@ -378,7 +378,7 @@ def test_can_generate_nodes_json_filtered(self): facade.reload_all(Path('tests/resources'), True) # Act: select nodes with api role (role 2) - node_descriptors = facade.json_nodes(2) + node_descriptors = facade.json_nodes(role=2) # Assert: spot check names and roles self.assertEqual(5, len(node_descriptors)) @@ -395,7 +395,7 @@ def test_can_generate_nodes_json_filtered_exact_match(self): facade.reload_all(Path('tests/resources'), True) # Act: select nodes with only api role (role 2) - node_descriptors = facade.json_nodes(2, True) + node_descriptors = facade.json_nodes(role=2, exact_match=True) # Assert: spot check names and roles self.assertEqual(1, len(node_descriptors)) diff --git a/explorer/nodewatch/tests/test_app.py b/explorer/nodewatch/tests/test_app.py index b6268e6e5..8a14a441b 100644 --- a/explorer/nodewatch/tests/test_app.py +++ b/explorer/nodewatch/tests/test_app.py @@ -189,7 +189,8 @@ def test_get_api_symbol_nodes_peer(client): # pylint: disable=redefined-outer-n 'Apple', 'Shin-Kuma-Node', 'ibone74', 'jaguar', 'symbol.ooo maxUnlockedAccounts:100', 'xym pool', 'yasmine farm' ] == list(map(lambda descriptor: descriptor['name'], response_json)) -def test_get_api_symbol_nodes(client): # pylint: disable=redefined-outer-name + +def test_get_api_symbol_nodes(client): # pylint: disable=redefined-outer-name # Act: response = client.get('/api/symbol/nodes') response_json = json.loads(response.data) @@ -202,7 +203,8 @@ def test_get_api_symbol_nodes(client): # pylint: disable=redefined-outer-name 'Allnodes250', 'Apple', 'Shin-Kuma-Node', 'ibone74', 'jaguar', 'symbol.ooo maxUnlockedAccounts:100', 'xym pool', 'yasmine farm' ] == list(map(lambda descriptor: descriptor['name'], response_json)) -def test_get_api_symbol_nodes_order_random(client): # pylint: disable=redefined-outer-name + +def test_get_api_symbol_nodes_order_random(client): # pylint: disable=redefined-outer-name # Act: response = client.get('/api/symbol/nodes?order=random') response_json = json.loads(response.data) @@ -212,12 +214,13 @@ def test_get_api_symbol_nodes_order_random(client): # pylint: disable=redefined- assert 'application/json' == response.headers['Content-Type'] assert 8 == len(response_json) expected_names = [ - 'Allnodes250', 'Apple', 'Shin-Kuma-Node', 'ibone74', 'jaguar', 'symbol.ooo maxUnlockedAccounts:100', 'xym pool', 'yasmine farm' - ] + 'Allnodes250', 'Apple', 'Shin-Kuma-Node', 'ibone74', 'jaguar', 'symbol.ooo maxUnlockedAccounts:100', 'xym pool', 'yasmine farm' + ] actual_names = list(map(lambda descriptor: descriptor['name'], response_json)) assert sorted(expected_names) == sorted(actual_names) -def test_get_api_symbol_nodes_limit_2(client): # pylint: disable=redefined-outer-name + +def test_get_api_symbol_nodes_limit_2(client): # pylint: disable=redefined-outer-name # Act: response = client.get('/api/symbol/nodes?limit=2') response_json = json.loads(response.data) @@ -230,7 +233,8 @@ def test_get_api_symbol_nodes_limit_2(client): # pylint: disable=redefined-outer 'Allnodes250', 'Apple' ] == list(map(lambda descriptor: descriptor['name'], response_json)) -def test_get_api_symbol_nodes_ssl_true(client): # pylint: disable=redefined-outer-name + +def test_get_api_symbol_nodes_ssl_true(client): # pylint: disable=redefined-outer-name # Act: response = client.get('/api/symbol/nodes?ssl=true') response_json = json.loads(response.data) @@ -243,7 +247,8 @@ def test_get_api_symbol_nodes_ssl_true(client): # pylint: disable=redefined-oute 'xym pool' ] == list(map(lambda descriptor: descriptor['name'], response_json)) -def test_get_api_symbol_nodes_ssl_false(client): # pylint: disable=redefined-outer-name + +def test_get_api_symbol_nodes_ssl_false(client): # pylint: disable=redefined-outer-name # Act: response = client.get('/api/symbol/nodes?ssl=false') response_json = json.loads(response.data) @@ -256,7 +261,8 @@ def test_get_api_symbol_nodes_ssl_false(client): # pylint: disable=redefined-out 'yasmine farm' ] == list(map(lambda descriptor: descriptor['name'], response_json)) -def test_get_api_symbol_node_with_main_public_key(client): # pylint: disable=redefined-outer-name + +def test_get_api_symbol_node_with_main_public_key(client): # pylint: disable=redefined-outer-name # Act: response = client.get('/api/symbol/nodes/mainPublicKey/A0AA48B6417BDB1845EB55FB0B1E13255EA8BD0D8FA29AD2D8A906E220571F21') response_json = json.loads(response.data) @@ -266,7 +272,8 @@ def test_get_api_symbol_node_with_main_public_key(client): # pylint: disable=red assert 'application/json' == response.headers['Content-Type'] assert 'Allnodes250' == response_json['name'] -def test_get_api_symbol_node_with_node_public_key(client): # pylint: disable=redefined-outer-name + +def test_get_api_symbol_node_with_node_public_key(client): # pylint: disable=redefined-outer-name # Act: response = client.get('/api/symbol/nodes/nodePublicKey/D05BE3101F2916AA34839DDC1199BE45092103A9B66172FA3D05911DC041AADA') response_json = json.loads(response.data) @@ -276,6 +283,7 @@ def test_get_api_symbol_node_with_node_public_key(client): # pylint: disable=red assert 'application/json' == response.headers['Content-Type'] assert 'yasmine farm' == response_json['name'] + def test_get_api_symbol_network_height(client): # pylint: disable=redefined-outer-name # Act: response = client.get('/api/symbol/height') From 183255049233453d3cf3483928372edb1861c994 Mon Sep 17 00:00:00 2001 From: AnthonyLaw Date: Wed, 29 Mar 2023 00:35:03 +0800 Subject: [PATCH 05/13] [explorer/nodewatch] task: update unit test --- explorer/nodewatch/tests/test_RoutesFacade.py | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/explorer/nodewatch/tests/test_RoutesFacade.py b/explorer/nodewatch/tests/test_RoutesFacade.py index 13e64880a..db7b9b0d2 100644 --- a/explorer/nodewatch/tests/test_RoutesFacade.py +++ b/explorer/nodewatch/tests/test_RoutesFacade.py @@ -226,7 +226,7 @@ def test_can_reload_all(self): self.assertEqual(True, result) self.assertEqual(facade.last_reload_time, facade.last_refresh_time) - self.assertEqual(6, len(facade.repository.node_descriptors)) + self.assertEqual(8, len(facade.repository.node_descriptors)) self.assertEqual(4, len(facade.repository.harvester_descriptors)) self.assertEqual(4, len(facade.repository.voter_descriptors)) @@ -243,7 +243,7 @@ def test_can_skip_reload_when_noop(self): self.assertEqual([True, False, False], [result1, result2, result3]) self.assertEqual(facade.last_reload_time, facade.last_refresh_time) - self.assertEqual(6, len(facade.repository.node_descriptors)) + self.assertEqual(8, len(facade.repository.node_descriptors)) self.assertEqual(4, len(facade.repository.harvester_descriptors)) self.assertEqual(4, len(facade.repository.voter_descriptors)) @@ -293,9 +293,9 @@ def test_can_render_nodes_html(self): self.assertEqual(4, len(context)) self.assertEqual('Symbol Nodes', context['title']) self.assertEqual( - ['Allnodes250', 'Apple', 'Shin-Kuma-Node', 'ibone74', 'jaguar', 'symbol.ooo maxUnlockedAccounts:100'], + ['Allnodes250', 'Apple', 'Shin-Kuma-Node', 'ibone74', 'jaguar', 'symbol.ooo maxUnlockedAccounts:100', 'xym pool', 'yasmine farm'], _get_names(context['descriptors'])) - self.assertEqual([104] * 6, _get_network_bytes(context['descriptors'])) + self.assertEqual([104] * 8, _get_network_bytes(context['descriptors'])) self.assertIsNotNone(context['version_to_css_class']) self.assertEqual('', context['explorer_endpoint']) @@ -328,7 +328,7 @@ def test_can_render_summary_html(self): # Assert: self.assertEqual('symbol_summary.html', template_name) self.assertEqual(5, len(context)) - self.assertEqual(4, len(json.loads(context['height_chart_json'])['data'])) # { height, finalized_height } x { 1.0.3.5, 1.0.3.4 } + self.assertEqual(6, len(json.loads(context['height_chart_json'])['data'])) # { height, finalized_height } x { 1.0.3.5, 1.0.3.4 } self.assertEqual(4, len(json.loads(context['voting_power_chart_json'])['data'])) # 1.0.3.5, 1.0.3.4, 1.0.3.3, '' self.assertEqual(4, len(json.loads(context['harvesting_power_chart_json'])['data'])) # 1.0.3.5, 1.0.3.4, 1.0.3.3, '' self.assertEqual(4, len(json.loads(context['harvesting_count_chart_json'])['data'])) # 1.0.3.5, 1.0.3.4, 1.0.3.3, '' @@ -364,12 +364,12 @@ def test_can_generate_nodes_json(self): node_descriptors = facade.json_nodes(role=1) # Assert: spot check names and roles - self.assertEqual(5, len(node_descriptors)) + self.assertEqual(7, len(node_descriptors)) self.assertEqual( - ['Apple', 'Shin-Kuma-Node', 'ibone74', 'jaguar', 'symbol.ooo maxUnlockedAccounts:100'], + ['Apple', 'Shin-Kuma-Node', 'ibone74', 'jaguar', 'symbol.ooo maxUnlockedAccounts:100', 'xym pool', 'yasmine farm'], list(map(lambda descriptor: descriptor['name'], node_descriptors))) self.assertEqual( - [7, 3, 3, 5, 3], + [7, 3, 3, 5, 3, 3, 3], list(map(lambda descriptor: descriptor['roles'], node_descriptors))) def test_can_generate_nodes_json_filtered(self): @@ -381,12 +381,12 @@ def test_can_generate_nodes_json_filtered(self): node_descriptors = facade.json_nodes(role=2) # Assert: spot check names and roles - self.assertEqual(5, len(node_descriptors)) + self.assertEqual(7, len(node_descriptors)) self.assertEqual( - ['Allnodes250', 'Apple', 'Shin-Kuma-Node', 'ibone74', 'symbol.ooo maxUnlockedAccounts:100'], + ['Allnodes250', 'Apple', 'Shin-Kuma-Node', 'ibone74', 'symbol.ooo maxUnlockedAccounts:100', 'xym pool', 'yasmine farm'], list(map(lambda descriptor: descriptor['name'], node_descriptors))) self.assertEqual( - [2, 7, 3, 3, 3], + [2, 7, 3, 3, 3, 3, 3], list(map(lambda descriptor: descriptor['roles'], node_descriptors))) def test_can_generate_nodes_json_filtered_exact_match(self): @@ -415,7 +415,7 @@ def test_can_generate_height_chart_json(self): chart_json = json.loads(facade.json_height_chart()) # Assert: { height, finalized_height } x { 1.0.3.4, 1.0.3.5 } - self.assertEqual(4, len(chart_json['data'])) + self.assertEqual(6, len(chart_json['data'])) def test_can_generate_height_chart_with_metadata_json(self): # Arrange: @@ -428,7 +428,7 @@ def test_can_generate_height_chart_with_metadata_json(self): # Assert: { height, finalized_height } x { 1.0.3.4, 1.0.3.5 } self.assertEqual(2, len(chart_with_metadata_json)) - self.assertEqual(4, len(chart_json['data'])) + self.assertEqual(6, len(chart_json['data'])) self.assertTrue(re.match(r'\d\d:\d\d', chart_with_metadata_json['lastRefreshTime'])) def test_can_retrieve_estimated_network_height_json(self): From 6bd5ee5a0a38349aa4542b838bb6204e7a2f8616 Mon Sep 17 00:00:00 2001 From: AnthonyLaw Date: Fri, 31 Mar 2023 18:34:43 +0800 Subject: [PATCH 06/13] [explorer/nodewatch] task: rename enable to enabled --- .../nodewatch/nodewatch/NetworkRepository.py | 16 ++--- explorer/nodewatch/nodewatch/RoutesFacade.py | 2 +- .../tests/resources/symbol_nodes.json | 8 +-- .../nodewatch/tests/test_NetworkRepository.py | 62 +++++++++---------- 4 files changed, 44 insertions(+), 44 deletions(-) diff --git a/explorer/nodewatch/nodewatch/NetworkRepository.py b/explorer/nodewatch/nodewatch/NetworkRepository.py index 1b0063973..5a17755b9 100644 --- a/explorer/nodewatch/nodewatch/NetworkRepository.py +++ b/explorer/nodewatch/nodewatch/NetworkRepository.py @@ -24,8 +24,8 @@ def __init__( finalized_height=0, balance=0, is_healthy=None, - is_https_enable=None, - is_wss_enable=None, + is_https_enabled=None, + is_wss_enabled=None, rest_version=None, roles=0xFF): """Creates a descriptor.""" @@ -42,8 +42,8 @@ def __init__( self.finalized_height = finalized_height self.balance = balance self.is_healthy = is_healthy - self.is_https_enable = is_https_enable - self.is_wss_enable = is_wss_enable + self.is_https_enabled = is_https_enabled + self.is_wss_enabled = is_wss_enabled self.rest_version = rest_version self.roles = roles @@ -66,8 +66,8 @@ def to_json(self): 'finalizedHeight': self.finalized_height, 'balance': self.balance, 'isHealthy': self.is_healthy, - 'isHttpsEnable': self.is_https_enable, - 'isWssEnable': self.is_wss_enable, + 'isHttpsEnabled': self.is_https_enabled, + 'isWssEnabled': self.is_wss_enabled, 'restVersion': self.rest_version, 'roles': self.roles } @@ -203,8 +203,8 @@ def _handle_symbol_node(self, json_node, extra_data): json_api_node_info_data = json_node['apiNodeInfo'] api_node_info_data = ( json_api_node_info_data.get('isHealth', None), - json_api_node_info_data.get('isHttpsEnable', None), - json_api_node_info_data.get('isWssEnable', None), + json_api_node_info_data.get('isHttpsEnabled', None), + json_api_node_info_data.get('isWssEnabled', None), json_api_node_info_data.get('restVersion', None)) main_public_key = PublicKey(json_node['publicKey']) diff --git a/explorer/nodewatch/nodewatch/RoutesFacade.py b/explorer/nodewatch/nodewatch/RoutesFacade.py index 588c512e7..4d811ce91 100644 --- a/explorer/nodewatch/nodewatch/RoutesFacade.py +++ b/explorer/nodewatch/nodewatch/RoutesFacade.py @@ -82,7 +82,7 @@ def filter_custom(descriptor): role_condition = role == descriptor.roles if exact_match else role == (role & descriptor.roles) if ssl is not None: - ssl_condition = (descriptor.is_https_enable == ssl and descriptor.is_wss_enable == ssl) + ssl_condition = (descriptor.is_https_enabled == ssl and descriptor.is_wss_enabled == ssl) return role_condition and ssl_condition return role_condition diff --git a/explorer/nodewatch/tests/resources/symbol_nodes.json b/explorer/nodewatch/tests/resources/symbol_nodes.json index 19caeb7aa..29b2d3d2f 100644 --- a/explorer/nodewatch/tests/resources/symbol_nodes.json +++ b/explorer/nodewatch/tests/resources/symbol_nodes.json @@ -95,8 +95,8 @@ { "apiNodeInfo": { "isHealth": true, - "isHttpsEnable": true, - "isWssEnable": true, + "isHttpsEnabled": true, + "isWssEnabled": true, "restVersion": "2.4.2" }, "extraData": { @@ -117,8 +117,8 @@ { "apiNodeInfo": { "isHealth": true, - "isHttpsEnable": false, - "isWssEnable": false, + "isHttpsEnabled": false, + "isWssEnabled": false, "restVersion": "2.4.2" }, "extraData": { diff --git a/explorer/nodewatch/tests/test_NetworkRepository.py b/explorer/nodewatch/tests/test_NetworkRepository.py index a4de6a68e..6b46c8d37 100644 --- a/explorer/nodewatch/tests/test_NetworkRepository.py +++ b/explorer/nodewatch/tests/test_NetworkRepository.py @@ -16,7 +16,7 @@ def _assert_node_descriptor(self, descriptor, **kwargs): property_names = [ 'main_address', 'main_public_key', 'node_public_key', 'endpoint', 'name', 'height', 'finalized_height', 'version', 'balance', - 'is_healthy', 'is_https_enable', 'is_wss_enable', 'rest_version', + 'is_healthy', 'is_https_enabled', 'is_wss_enabled', 'rest_version', 'roles', 'has_api' ] for name in property_names: @@ -57,8 +57,8 @@ def test_can_load_nem_node_descriptors(self): balance=3355922.652725, roles=0xFF, is_healthy=None, - is_https_enable=None, - is_wss_enable=None, + is_https_enabled=None, + is_wss_enabled=None, rest_version=None, has_api=True) self._assert_node_descriptor( @@ -74,8 +74,8 @@ def test_can_load_nem_node_descriptors(self): balance=20612359.516967, roles=0xFF, is_healthy=None, - is_https_enable=None, - is_wss_enable=None, + is_https_enabled=None, + is_wss_enabled=None, rest_version=None, has_api=True) self._assert_node_descriptor( @@ -91,8 +91,8 @@ def test_can_load_nem_node_descriptors(self): balance=0, roles=0xFF, is_healthy=None, - is_https_enable=None, - is_wss_enable=None, + is_https_enabled=None, + is_wss_enabled=None, rest_version=None, has_api=True) # simulates missing extraData self._assert_node_descriptor( @@ -108,8 +108,8 @@ def test_can_load_nem_node_descriptors(self): balance=0, roles=0xFF, is_healthy=None, - is_https_enable=None, - is_wss_enable=None, + is_https_enabled=None, + is_wss_enabled=None, rest_version=None, has_api=True) # simulates incomplete extraData @@ -148,8 +148,8 @@ def test_can_load_symbol_node_descriptors(self): balance=3155632.471994, roles=2, is_healthy=None, - is_https_enable=None, - is_wss_enable=None, + is_https_enabled=None, + is_wss_enabled=None, rest_version=None, has_api=True) # simulates missing host self._assert_node_descriptor( @@ -165,8 +165,8 @@ def test_can_load_symbol_node_descriptors(self): balance=0, roles=7, is_healthy=None, - is_https_enable=None, - is_wss_enable=None, + is_https_enabled=None, + is_wss_enabled=None, rest_version=None, has_api=True) # old version mapped to 'failure' self._assert_node_descriptor( @@ -182,8 +182,8 @@ def test_can_load_symbol_node_descriptors(self): balance=0, roles=3, is_healthy=None, - is_https_enable=None, - is_wss_enable=None, + is_https_enabled=None, + is_wss_enabled=None, rest_version=None, has_api=True) # simulates incomplete extraData self._assert_node_descriptor( @@ -199,8 +199,8 @@ def test_can_load_symbol_node_descriptors(self): balance=82375.554976, roles=3, is_healthy=None, - is_https_enable=None, - is_wss_enable=None, + is_https_enabled=None, + is_wss_enabled=None, rest_version=None, has_api=True) self._assert_node_descriptor( @@ -216,8 +216,8 @@ def test_can_load_symbol_node_descriptors(self): balance=28083310.571743, roles=5, is_healthy=None, - is_https_enable=None, - is_wss_enable=None, + is_https_enabled=None, + is_wss_enabled=None, rest_version=None, has_api=False) self._assert_node_descriptor( @@ -233,8 +233,8 @@ def test_can_load_symbol_node_descriptors(self): balance=0, roles=3, is_healthy=None, - is_https_enable=None, - is_wss_enable=None, + is_https_enabled=None, + is_wss_enabled=None, rest_version=None, has_api=True) # simulates missing extraData self._assert_node_descriptor( @@ -250,8 +250,8 @@ def test_can_load_symbol_node_descriptors(self): balance=101027.849383, roles=3, is_healthy=True, - is_https_enable=True, - is_wss_enable=True, + is_https_enabled=True, + is_wss_enabled=True, rest_version='2.4.2', has_api=True) self._assert_node_descriptor( @@ -267,8 +267,8 @@ def test_can_load_symbol_node_descriptors(self): balance=99.98108, roles=3, is_healthy=True, - is_https_enable=False, - is_wss_enable=False, + is_https_enabled=False, + is_wss_enabled=False, rest_version='2.4.2', has_api=True) # simulates ssl disabled @@ -291,8 +291,8 @@ def test_can_format_node_descriptor_as_json(self): 'version': '1.0.3.5', 'balance': 28083310.571743, 'isHealthy': None, - 'isHttpsEnable': None, - 'isWssEnable': None, + 'isHttpsEnabled': None, + 'isWssEnabled': None, 'restVersion': None, 'roles': 5, }, json_object) @@ -316,8 +316,8 @@ def test_can_format_node_descriptor_with_node_public_key_as_json(self): 'version': '1.0.3.5', 'balance': 82375.554976, 'isHealthy': None, - 'isHttpsEnable': None, - 'isWssEnable': None, + 'isHttpsEnabled': None, + 'isWssEnabled': None, 'restVersion': None, 'roles': 3, }, json_object) @@ -341,8 +341,8 @@ def test_can_format_node_descriptor_with_api_node_info_as_json(self): 'version': '1.0.3.3', 'balance': 101027.849383, 'isHealthy': True, - 'isHttpsEnable': True, - 'isWssEnable': True, + 'isHttpsEnabled': True, + 'isWssEnabled': True, 'restVersion': '2.4.2', 'roles': 3, }, json_object) From 1e2628f28ce19bcef27d2261de3be2fd385a45e4 Mon Sep 17 00:00:00 2001 From: AnthonyLaw Date: Sat, 1 Apr 2023 15:18:28 +0800 Subject: [PATCH 07/13] [explorer/nodewatch] task: rename and update filter logic --- explorer/nodewatch/nodewatch/RoutesFacade.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/explorer/nodewatch/nodewatch/RoutesFacade.py b/explorer/nodewatch/nodewatch/RoutesFacade.py index 4d811ce91..1ead4dd82 100644 --- a/explorer/nodewatch/nodewatch/RoutesFacade.py +++ b/explorer/nodewatch/nodewatch/RoutesFacade.py @@ -75,7 +75,7 @@ def json_nodes(self, **kwargs): ssl = kwargs.get('ssl', None) order = kwargs.get('order', None) - def filter_custom(descriptor): + def custom_filter(descriptor): role_condition = True if role is not None: @@ -89,11 +89,10 @@ def filter_custom(descriptor): nodes = list(map( lambda descriptor: descriptor.to_json(), - filter(filter_custom, self.repository.node_descriptors))) + filter(custom_filter, self.repository.node_descriptors))) - if order is not None: - if order == 'random': - random.shuffle(nodes) + if order == 'random': + random.shuffle(nodes) if limit is not None: nodes = nodes[:limit] From 9688016c2f6c7d1b9f171237e361c016e22352c7 Mon Sep 17 00:00:00 2001 From: AnthonyLaw Date: Sat, 1 Apr 2023 15:18:51 +0800 Subject: [PATCH 08/13] [explorer/nodewatch] task: and more unit test and minor refactor --- explorer/nodewatch/tests/test_RoutesFacade.py | 114 +++++++++++++++++- explorer/nodewatch/tests/test_app.py | 44 +++++-- 2 files changed, 144 insertions(+), 14 deletions(-) diff --git a/explorer/nodewatch/tests/test_RoutesFacade.py b/explorer/nodewatch/tests/test_RoutesFacade.py index db7b9b0d2..ffbe563f4 100644 --- a/explorer/nodewatch/tests/test_RoutesFacade.py +++ b/explorer/nodewatch/tests/test_RoutesFacade.py @@ -328,7 +328,7 @@ def test_can_render_summary_html(self): # Assert: self.assertEqual('symbol_summary.html', template_name) self.assertEqual(5, len(context)) - self.assertEqual(6, len(json.loads(context['height_chart_json'])['data'])) # { height, finalized_height } x { 1.0.3.5, 1.0.3.4 } + self.assertEqual(6, len(json.loads(context['height_chart_json'])['data'])) # { height, finalized_height } x { 1.0.3.5, 1.0.3.4, 1.0.3.3 } self.assertEqual(4, len(json.loads(context['voting_power_chart_json'])['data'])) # 1.0.3.5, 1.0.3.4, 1.0.3.3, '' self.assertEqual(4, len(json.loads(context['harvesting_power_chart_json'])['data'])) # 1.0.3.5, 1.0.3.4, 1.0.3.3, '' self.assertEqual(4, len(json.loads(context['harvesting_count_chart_json'])['data'])) # 1.0.3.5, 1.0.3.4, 1.0.3.3, '' @@ -406,6 +406,114 @@ def test_can_generate_nodes_json_filtered_exact_match(self): [2], list(map(lambda descriptor: descriptor['roles'], node_descriptors))) + def test_can_generate_nodes_json_filtered_ssl(self): + # Arrange: + facade = SymbolRoutesFacade(SymbolNetwork.MAINNET, '') + facade.reload_all(Path('tests/resources'), True) + + # Act: select nodes with only ssl enabled + node_descriptors = facade.json_nodes(ssl=True) + + # Assert: spot check names and roles + self.assertEqual(1, len(node_descriptors)) + self.assertEqual( + ['xym pool'], + list(map(lambda descriptor: descriptor['name'], node_descriptors))) + self.assertEqual( + [3], + list(map(lambda descriptor: descriptor['roles'], node_descriptors))) + + def test_can_generate_nodes_json_filtered_order_random_limit_2(self): + # Arrange: + facade = SymbolRoutesFacade(SymbolNetwork.MAINNET, '') + facade.reload_all(Path('tests/resources'), True) + + # Act: select 2 nodes with order random + node_descriptors = facade.json_nodes(limit=2, order='random') + + # returns all nodes + all_node_descriptors = facade.json_nodes(role=1) + + full_node_names = list(map(lambda descriptor: descriptor['name'], all_node_descriptors)) + random_node_names = list(map(lambda descriptor: descriptor['name'], node_descriptors)) + + # Assert: spot check names + self.assertEqual(2, len(node_descriptors)) + for name in random_node_names: + self.assertIn(name, full_node_names) + + def test_can_generate_node_json_given_main_public_key(self): + # Arrange: + facade = SymbolRoutesFacade(SymbolNetwork.MAINNET, '') + facade.reload_all(Path('tests/resources'), True) + + # Act: select a node match with main public key + node_descriptors = facade.json_node( + filter_field='mainPublicKey', + public_key="A0AA48B6417BDB1845EB55FB0B1E13255EA8BD0D8FA29AD2D8A906E220571F21" + ) + expected_node = { + 'mainPublicKey': 'A0AA48B6417BDB1845EB55FB0B1E13255EA8BD0D8FA29AD2D8A906E220571F21', + 'nodePublicKey': '403D890915B68E290B3F519A602A13B17C58499A077D77DB7CCC6327761C84DC', + 'endpoint': '', + 'name': + 'Allnodes250', + 'version': '1.0.3.4', + 'height': 1486762, + 'finalizedHeight': 1486740, + 'balance': 3155632.471994, + 'isHealthy': None, + 'isHttpsEnabled': None, + 'isWssEnabled': None, + 'restVersion': None, + 'roles': 2 + } + + # Assert: + self.assertEqual(node_descriptors, expected_node) + + def test_can_generate_node_json_given_node_public_key(self): + # Arrange: + facade = SymbolRoutesFacade(SymbolNetwork.MAINNET, '') + facade.reload_all(Path('tests/resources'), True) + + # Act: select a node match with node public key + node_descriptors = facade.json_node( + filter_field='nodePublicKey', + public_key="D05BE3101F2916AA34839DDC1199BE45092103A9B66172FA3D05911DC041AADA" + ) + expected_node = { + 'mainPublicKey': '5B20F8F228FF0E064DB0DE7951155F6F41EF449D0EC10960067C2BF2DCD61874', + 'nodePublicKey': 'D05BE3101F2916AA34839DDC1199BE45092103A9B66172FA3D05911DC041AADA', + 'endpoint': 'http://yasmine.farm.tokyo:3000', + 'name': 'yasmine farm', + 'version': '1.0.3.4', + 'height': 1486760, + 'finalizedHeight': 1486740, + 'balance': 99.98108, + 'isHealthy': True, + 'isHttpsEnabled': False, + 'isWssEnabled': False, + 'restVersion': '2.4.2', + 'roles': 3 + } + + # Assert: + self.assertEqual(node_descriptors, expected_node) + + def test_can_generate_node_json_given_public_key_not_found(self): + # Arrange: + facade = SymbolRoutesFacade(SymbolNetwork.MAINNET, '') + facade.reload_all(Path('tests/resources'), True) + + # Act: + main_public_key_descriptors = facade.json_node(filter_field='mainPublicKey', public_key="invalidKey") + node_public_key_descriptors = facade.json_node(filter_field='nodePublicKey', public_key="invalidKey") + + # Assert: + self.assertIsNone(main_public_key_descriptors) + self.assertIsNone(node_public_key_descriptors) + def test_can_generate_height_chart_json(self): # Arrange: facade = SymbolRoutesFacade(SymbolNetwork.MAINNET, '', 1) @@ -414,7 +522,7 @@ def test_can_generate_height_chart_json(self): # Act: chart_json = json.loads(facade.json_height_chart()) - # Assert: { height, finalized_height } x { 1.0.3.4, 1.0.3.5 } + # Assert: { height, finalized_height } x { 1.0.3.3, 1.0.3.4, 1.0.3.5 } self.assertEqual(6, len(chart_json['data'])) def test_can_generate_height_chart_with_metadata_json(self): @@ -426,7 +534,7 @@ def test_can_generate_height_chart_with_metadata_json(self): chart_with_metadata_json = facade.json_height_chart_with_metadata() chart_json = json.loads(chart_with_metadata_json['chartJson']) - # Assert: { height, finalized_height } x { 1.0.3.4, 1.0.3.5 } + # Assert: { height, finalized_height } x { 1.0.3.3, 1.0.3.4, 1.0.3.5 } self.assertEqual(2, len(chart_with_metadata_json)) self.assertEqual(6, len(chart_json['data'])) self.assertTrue(re.match(r'\d\d:\d\d', chart_with_metadata_json['lastRefreshTime'])) diff --git a/explorer/nodewatch/tests/test_app.py b/explorer/nodewatch/tests/test_app.py index 8a14a441b..e29829332 100644 --- a/explorer/nodewatch/tests/test_app.py +++ b/explorer/nodewatch/tests/test_app.py @@ -204,20 +204,22 @@ def test_get_api_symbol_nodes(client): # pylint: disable=redefined-outer-name ] == list(map(lambda descriptor: descriptor['name'], response_json)) -def test_get_api_symbol_nodes_order_random(client): # pylint: disable=redefined-outer-name +def test_get_api_symbol_nodes_order_random_limit_2(client): # pylint: disable=redefined-outer-name # Act: - response = client.get('/api/symbol/nodes?order=random') + response = client.get('/api/symbol/nodes?order=random&limit=2') response_json = json.loads(response.data) # Assert: spot check names assert 200 == response.status_code assert 'application/json' == response.headers['Content-Type'] - assert 8 == len(response_json) - expected_names = [ + assert 2 == len(response_json) + full_node_names = [ 'Allnodes250', 'Apple', 'Shin-Kuma-Node', 'ibone74', 'jaguar', 'symbol.ooo maxUnlockedAccounts:100', 'xym pool', 'yasmine farm' ] actual_names = list(map(lambda descriptor: descriptor['name'], response_json)) - assert sorted(expected_names) == sorted(actual_names) + + for name in actual_names: + assert name in full_node_names def test_get_api_symbol_nodes_limit_2(client): # pylint: disable=redefined-outer-name @@ -262,26 +264,46 @@ def test_get_api_symbol_nodes_ssl_false(client): # pylint: disable=redefined-ou ] == list(map(lambda descriptor: descriptor['name'], response_json)) -def test_get_api_symbol_node_with_main_public_key(client): # pylint: disable=redefined-outer-name +def _assert_api_symbol_node_with_public_key_not_found(response): # Act: - response = client.get('/api/symbol/nodes/mainPublicKey/A0AA48B6417BDB1845EB55FB0B1E13255EA8BD0D8FA29AD2D8A906E220571F21') response_json = json.loads(response.data) # Assert: spot check names assert 200 == response.status_code assert 'application/json' == response.headers['Content-Type'] - assert 'Allnodes250' == response_json['name'] + assert response_json is None -def test_get_api_symbol_node_with_node_public_key(client): # pylint: disable=redefined-outer-name +def _assert_api_symbol_node_with_public_key_found(response, expected_name): # Act: - response = client.get('/api/symbol/nodes/nodePublicKey/D05BE3101F2916AA34839DDC1199BE45092103A9B66172FA3D05911DC041AADA') response_json = json.loads(response.data) # Assert: spot check names assert 200 == response.status_code assert 'application/json' == response.headers['Content-Type'] - assert 'yasmine farm' == response_json['name'] + assert expected_name == response_json['name'] + + +def test_get_api_symbol_node_with_invalid_main_public_key(client): # pylint: disable=redefined-outer-name + _assert_api_symbol_node_with_public_key_not_found(client.get('/api/symbol/nodes/mainPublicKey/invalid')) + + +def test_get_api_symbol_node_with_invalid_node_public_key(client): # pylint: disable=redefined-outer-name + _assert_api_symbol_node_with_public_key_not_found(client.get('/api/symbol/nodes/nodePublicKey/invalid')) + + +def test_get_api_symbol_node_with_main_public_key(client): # pylint: disable=redefined-outer-name + _assert_api_symbol_node_with_public_key_found( + client.get('/api/symbol/nodes/mainPublicKey/A0AA48B6417BDB1845EB55FB0B1E13255EA8BD0D8FA29AD2D8A906E220571F21'), + 'Allnodes250' + ) + + +def test_get_api_symbol_node_with_node_public_key(client): # pylint: disable=redefined-outer-name + _assert_api_symbol_node_with_public_key_found( + client.get('/api/symbol/nodes/nodePublicKey/D05BE3101F2916AA34839DDC1199BE45092103A9B66172FA3D05911DC041AADA'), + 'yasmine farm' + ) def test_get_api_symbol_network_height(client): # pylint: disable=redefined-outer-name From 214790a0048fad589bd7cd29e251026a7421dc0b Mon Sep 17 00:00:00 2001 From: AnthonyLaw Date: Tue, 11 Apr 2023 00:42:19 +0800 Subject: [PATCH 09/13] [explorer/nodewatch] task: refactor and unit test --- explorer/nodewatch/nodewatch/RoutesFacade.py | 2 +- explorer/nodewatch/nodewatch/__init__.py | 29 ++--- .../tests/resources/symbol_nodes.json | 22 ++++ .../nodewatch/tests/test_NetworkRepository.py | 37 ++++-- explorer/nodewatch/tests/test_RoutesFacade.py | 46 ++++--- explorer/nodewatch/tests/test_app.py | 112 +++++++----------- 6 files changed, 137 insertions(+), 111 deletions(-) diff --git a/explorer/nodewatch/nodewatch/RoutesFacade.py b/explorer/nodewatch/nodewatch/RoutesFacade.py index 1ead4dd82..e72869de1 100644 --- a/explorer/nodewatch/nodewatch/RoutesFacade.py +++ b/explorer/nodewatch/nodewatch/RoutesFacade.py @@ -82,7 +82,7 @@ def custom_filter(descriptor): role_condition = role == descriptor.roles if exact_match else role == (role & descriptor.roles) if ssl is not None: - ssl_condition = (descriptor.is_https_enabled == ssl and descriptor.is_wss_enabled == ssl) + ssl_condition = (descriptor.is_https_enabled and descriptor.is_wss_enabled) return role_condition and ssl_condition return role_condition diff --git a/explorer/nodewatch/nodewatch/__init__.py b/explorer/nodewatch/nodewatch/__init__.py index ea2bf021a..e602bd0b5 100644 --- a/explorer/nodewatch/nodewatch/__init__.py +++ b/explorer/nodewatch/nodewatch/__init__.py @@ -20,8 +20,6 @@ class Field(Enum): def str_to_bool(value): if value.lower() == 'true': return True - if value.lower() == 'false': - return False return None @@ -105,27 +103,30 @@ def symbol_summary(): # pylint: disable=unused-variable template_name, context = symbol_routes_facade.html_summary() return render_template(template_name, **context) + def _get_json_nodes(role, exact_match, ssl, limit, order): + if ssl is not None: + ssl = str_to_bool(ssl) + + if limit is not None: + limit = int(limit) + + return jsonify(symbol_routes_facade.json_nodes(role=role, exact_match=exact_match, ssl=ssl, limit=limit, order=order)) + @app.route('/api/symbol/nodes/api') def api_symbol_nodes_api(): # pylint: disable=unused-variable - return jsonify(symbol_routes_facade.json_nodes(role=2, exact_match=True)) + ssl = request.args.get('ssl', None) + limit = request.args.get('limit', None) + order = request.args.get('order', None) + + return _get_json_nodes(2, True, ssl, limit, order) @app.route('/api/symbol/nodes/peer') def api_symbol_nodes_peer(): # pylint: disable=unused-variable - return jsonify(symbol_routes_facade.json_nodes(role=1)) - - @app.route('/api/symbol/nodes') - def api_symbol_nodes(): # pylint: disable=unused-variable ssl = request.args.get('ssl', None) limit = request.args.get('limit', None) order = request.args.get('order', None) - if ssl is not None: - ssl = str_to_bool(ssl) - - if limit is not None: - limit = int(limit) - - return jsonify(symbol_routes_facade.json_nodes(ssl=ssl, limit=limit, order=order)) + return _get_json_nodes(1, False, ssl, limit, order) @app.route('/api/symbol/nodes/mainPublicKey/') def api_symbol_nodes_get_main_public_key(main_public_key): # pylint: disable=unused-variable diff --git a/explorer/nodewatch/tests/resources/symbol_nodes.json b/explorer/nodewatch/tests/resources/symbol_nodes.json index 29b2d3d2f..9af5f1e68 100644 --- a/explorer/nodewatch/tests/resources/symbol_nodes.json +++ b/explorer/nodewatch/tests/resources/symbol_nodes.json @@ -135,5 +135,27 @@ "publicKey": "5B20F8F228FF0E064DB0DE7951155F6F41EF449D0EC10960067C2BF2DCD61874", "roles": 3, "version": 16777988 + }, + { + "apiNodeInfo": { + "isHealth": true, + "isHttpsEnabled": true, + "isWssEnabled": true, + "restVersion": "2.4.2" + }, + "extraData": { + "balance": 3155632.471994, + "finalizedHeight": 1486740, + "height": 1486762 + }, + "friendlyName": "Allnodes251", + "host": "", + "networkGenerationHashSeed": "57F7DA205008026C776CB6AED843393F04CD458E0AA2D9F1D5F31A402072B2D6", + "networkIdentifier": 104, + "nodePublicKey": "F25FDB3CD1A97DDC71A993D124E9BDD6518699F9F4016C29D341F53208D150D8", + "port": 7900, + "publicKey": "C69B5BDE17EEF7449C6D92856C1D295FE02AA3472651F042FB3FC2F771DAAF7B", + "roles": 2, + "version": 16777988 } ] diff --git a/explorer/nodewatch/tests/test_NetworkRepository.py b/explorer/nodewatch/tests/test_NetworkRepository.py index 6b46c8d37..560366cac 100644 --- a/explorer/nodewatch/tests/test_NetworkRepository.py +++ b/explorer/nodewatch/tests/test_NetworkRepository.py @@ -132,7 +132,7 @@ def test_can_load_symbol_node_descriptors(self): # Assert: descriptors are sorted by name (desc) self.assertFalse(repository.is_nem) - self.assertEqual(8, len(repository.node_descriptors)) + self.assertEqual(9, len(repository.node_descriptors)) self.assertEqual(1486760, repository.estimate_height()) # median self.assertEqual(1486740, repository.estimate_finalized_height()) # median (nonzero) self._assert_node_descriptor( @@ -154,6 +154,23 @@ def test_can_load_symbol_node_descriptors(self): has_api=True) # simulates missing host self._assert_node_descriptor( repository.node_descriptors[1], + main_address=SymbolAddress('ND44QLRJQWF756VVP7XLTPDUBGXHT6ZFR7PSDQI'), + main_public_key=PublicKey('C69B5BDE17EEF7449C6D92856C1D295FE02AA3472651F042FB3FC2F771DAAF7B'), + node_public_key=PublicKey('F25FDB3CD1A97DDC71A993D124E9BDD6518699F9F4016C29D341F53208D150D8'), + endpoint='', + name='Allnodes251', + height=1486762, + finalized_height=1486740, + version='1.0.3.4', + balance=3155632.471994, + roles=2, + is_healthy=True, + is_https_enabled=True, + is_wss_enabled=True, + rest_version='2.4.2', + has_api=True) + self._assert_node_descriptor( + repository.node_descriptors[2], main_address=SymbolAddress('NCFJP3DM65U22JI5XZ2P2TBK5BV5MLKAR7334LQ'), main_public_key=PublicKey('A05329E4E5F068B323653F393CE0E3E6A1EB5056E122457354BA65158FFD33F4'), node_public_key=PublicKey('FBEAFCB15D2674ECB8DC1CD2C028C4AC0D463489069FDD415F30BB71EAE69864'), @@ -170,7 +187,7 @@ def test_can_load_symbol_node_descriptors(self): rest_version=None, has_api=True) # old version mapped to 'failure' self._assert_node_descriptor( - repository.node_descriptors[2], + repository.node_descriptors[3], main_address=SymbolAddress('NBPQMC4M2MMX2XOCOC3BCZ7N3ALUTRGLYPPQ56Q'), main_public_key=PublicKey('2784FBE82D8A46C4082519012970CBB42EC3EC83D5DB93963B71FD6C5DA3B072'), node_public_key=PublicKey('9CBE17EDFC8B333FE6BD3FF9B4D02914D55A9368F318D4CEF0AB4737BA5BB160'), @@ -187,7 +204,7 @@ def test_can_load_symbol_node_descriptors(self): rest_version=None, has_api=True) # simulates incomplete extraData self._assert_node_descriptor( - repository.node_descriptors[3], + repository.node_descriptors[4], main_address=SymbolAddress('NCPPDLXGYBHNPQAXQ6RTNS3T46A7FNTXDFBD43Y'), main_public_key=PublicKey('7DFB0D690BFFA4A4979C7466C7B669AE8FBAFD419DAA10DE948604CD9BE65F0B'), node_public_key=PublicKey('D561824BD4E3053C39A8D5A4AB00583A4D99302C541F046D3A1E6FF023006D7C'), @@ -204,7 +221,7 @@ def test_can_load_symbol_node_descriptors(self): rest_version=None, has_api=True) self._assert_node_descriptor( - repository.node_descriptors[4], + repository.node_descriptors[5], main_address=SymbolAddress('NAEONICSHRZATW7XGIVIDPTNHUMQA7N7XQ4EUPQ'), main_public_key=PublicKey('B26D01FC006EAC09B740A3C8F12C1055AE24AFD3268F0364C92D51800FC07361'), node_public_key=None, @@ -221,7 +238,7 @@ def test_can_load_symbol_node_descriptors(self): rest_version=None, has_api=False) self._assert_node_descriptor( - repository.node_descriptors[5], + repository.node_descriptors[6], main_address=SymbolAddress('NDLLVJIUHAAV6F5PG5KYSSQXCZDCPXCY4WFA6TQ'), main_public_key=PublicKey('71F953D3C3D0B7E70E29EC2DE761DD7339BA815C094B3BEE0917AEBD924B37EB'), node_public_key=PublicKey('C71C7D5E6981DE5ED27908C6749207E49001A0B0F0DD404D07451636A64BEBEB'), @@ -238,7 +255,7 @@ def test_can_load_symbol_node_descriptors(self): rest_version=None, has_api=True) # simulates missing extraData self._assert_node_descriptor( - repository.node_descriptors[6], + repository.node_descriptors[7], main_address=SymbolAddress('NAU6BZUX5GHI7EDE6DMS6GVHXS4XZFCNVRPT2OQ'), main_public_key=PublicKey('A54CC798373F42B569AF21845CD0EBE755AB42EA04B3B8E2BE897166F89A971C'), node_public_key=PublicKey('FE7D3DBE8DDD219E1B20247DEBF150D9411EA5A312989103B037EFBD9D237DE0'), @@ -255,7 +272,7 @@ def test_can_load_symbol_node_descriptors(self): rest_version='2.4.2', has_api=True) self._assert_node_descriptor( - repository.node_descriptors[7], + repository.node_descriptors[8], main_address=SymbolAddress('NAOOI6NZA6TZMIKOGAQCQG7SXBPXVSDOTPLLDZY'), main_public_key=PublicKey('5B20F8F228FF0E064DB0DE7951155F6F41EF449D0EC10960067C2BF2DCD61874'), node_public_key=PublicKey('D05BE3101F2916AA34839DDC1199BE45092103A9B66172FA3D05911DC041AADA'), @@ -278,7 +295,7 @@ def test_can_format_node_descriptor_as_json(self): repository.load_node_descriptors('tests/resources/symbol_nodes.json') # Act: - json_object = repository.node_descriptors[4].to_json() + json_object = repository.node_descriptors[5].to_json() # Assert: self.assertEqual({ @@ -303,7 +320,7 @@ def test_can_format_node_descriptor_with_node_public_key_as_json(self): repository.load_node_descriptors('tests/resources/symbol_nodes.json') # Act: - json_object = repository.node_descriptors[3].to_json() + json_object = repository.node_descriptors[4].to_json() # Assert: self.assertEqual({ @@ -328,7 +345,7 @@ def test_can_format_node_descriptor_with_api_node_info_as_json(self): repository.load_node_descriptors('tests/resources/symbol_nodes.json') # Act: - json_object = repository.node_descriptors[6].to_json() + json_object = repository.node_descriptors[7].to_json() # Assert: self.assertEqual({ diff --git a/explorer/nodewatch/tests/test_RoutesFacade.py b/explorer/nodewatch/tests/test_RoutesFacade.py index ffbe563f4..e892cca0f 100644 --- a/explorer/nodewatch/tests/test_RoutesFacade.py +++ b/explorer/nodewatch/tests/test_RoutesFacade.py @@ -226,7 +226,7 @@ def test_can_reload_all(self): self.assertEqual(True, result) self.assertEqual(facade.last_reload_time, facade.last_refresh_time) - self.assertEqual(8, len(facade.repository.node_descriptors)) + self.assertEqual(9, len(facade.repository.node_descriptors)) self.assertEqual(4, len(facade.repository.harvester_descriptors)) self.assertEqual(4, len(facade.repository.voter_descriptors)) @@ -243,7 +243,7 @@ def test_can_skip_reload_when_noop(self): self.assertEqual([True, False, False], [result1, result2, result3]) self.assertEqual(facade.last_reload_time, facade.last_refresh_time) - self.assertEqual(8, len(facade.repository.node_descriptors)) + self.assertEqual(9, len(facade.repository.node_descriptors)) self.assertEqual(4, len(facade.repository.harvester_descriptors)) self.assertEqual(4, len(facade.repository.voter_descriptors)) @@ -293,9 +293,19 @@ def test_can_render_nodes_html(self): self.assertEqual(4, len(context)) self.assertEqual('Symbol Nodes', context['title']) self.assertEqual( - ['Allnodes250', 'Apple', 'Shin-Kuma-Node', 'ibone74', 'jaguar', 'symbol.ooo maxUnlockedAccounts:100', 'xym pool', 'yasmine farm'], + [ + 'Allnodes250', + 'Allnodes251', + 'Apple', + 'Shin-Kuma-Node', + 'ibone74', + 'jaguar', + 'symbol.ooo maxUnlockedAccounts:100', + 'xym pool', + 'yasmine farm' + ], _get_names(context['descriptors'])) - self.assertEqual([104] * 8, _get_network_bytes(context['descriptors'])) + self.assertEqual([104] * 9, _get_network_bytes(context['descriptors'])) self.assertIsNotNone(context['version_to_css_class']) self.assertEqual('', context['explorer_endpoint']) @@ -381,12 +391,12 @@ def test_can_generate_nodes_json_filtered(self): node_descriptors = facade.json_nodes(role=2) # Assert: spot check names and roles - self.assertEqual(7, len(node_descriptors)) + self.assertEqual(8, len(node_descriptors)) self.assertEqual( - ['Allnodes250', 'Apple', 'Shin-Kuma-Node', 'ibone74', 'symbol.ooo maxUnlockedAccounts:100', 'xym pool', 'yasmine farm'], + ['Allnodes250', 'Allnodes251', 'Apple', 'Shin-Kuma-Node', 'ibone74', 'symbol.ooo maxUnlockedAccounts:100', 'xym pool', 'yasmine farm'], list(map(lambda descriptor: descriptor['name'], node_descriptors))) self.assertEqual( - [2, 7, 3, 3, 3, 3, 3], + [2, 2, 7, 3, 3, 3, 3, 3], list(map(lambda descriptor: descriptor['roles'], node_descriptors))) def test_can_generate_nodes_json_filtered_exact_match(self): @@ -398,12 +408,12 @@ def test_can_generate_nodes_json_filtered_exact_match(self): node_descriptors = facade.json_nodes(role=2, exact_match=True) # Assert: spot check names and roles - self.assertEqual(1, len(node_descriptors)) + self.assertEqual(2, len(node_descriptors)) self.assertEqual( - ['Allnodes250'], + ['Allnodes250', 'Allnodes251'], list(map(lambda descriptor: descriptor['name'], node_descriptors))) self.assertEqual( - [2], + [2, 2], list(map(lambda descriptor: descriptor['roles'], node_descriptors))) def test_can_generate_nodes_json_filtered_ssl(self): @@ -415,12 +425,12 @@ def test_can_generate_nodes_json_filtered_ssl(self): node_descriptors = facade.json_nodes(ssl=True) # Assert: spot check names and roles - self.assertEqual(1, len(node_descriptors)) + self.assertEqual(2, len(node_descriptors)) self.assertEqual( - ['xym pool'], + ['Allnodes251', 'xym pool'], list(map(lambda descriptor: descriptor['name'], node_descriptors))) self.assertEqual( - [3], + [2, 3], list(map(lambda descriptor: descriptor['roles'], node_descriptors))) def test_can_generate_nodes_json_filtered_order_random_limit_2(self): @@ -432,7 +442,7 @@ def test_can_generate_nodes_json_filtered_order_random_limit_2(self): node_descriptors = facade.json_nodes(limit=2, order='random') # returns all nodes - all_node_descriptors = facade.json_nodes(role=1) + all_node_descriptors = facade.json_nodes() full_node_names = list(map(lambda descriptor: descriptor['name'], all_node_descriptors)) random_node_names = list(map(lambda descriptor: descriptor['name'], node_descriptors)) @@ -450,7 +460,7 @@ def test_can_generate_node_json_given_main_public_key(self): # Act: select a node match with main public key node_descriptors = facade.json_node( filter_field='mainPublicKey', - public_key="A0AA48B6417BDB1845EB55FB0B1E13255EA8BD0D8FA29AD2D8A906E220571F21" + public_key='A0AA48B6417BDB1845EB55FB0B1E13255EA8BD0D8FA29AD2D8A906E220571F21' ) expected_node = { 'mainPublicKey': 'A0AA48B6417BDB1845EB55FB0B1E13255EA8BD0D8FA29AD2D8A906E220571F21', @@ -480,7 +490,7 @@ def test_can_generate_node_json_given_node_public_key(self): # Act: select a node match with node public key node_descriptors = facade.json_node( filter_field='nodePublicKey', - public_key="D05BE3101F2916AA34839DDC1199BE45092103A9B66172FA3D05911DC041AADA" + public_key='D05BE3101F2916AA34839DDC1199BE45092103A9B66172FA3D05911DC041AADA' ) expected_node = { 'mainPublicKey': '5B20F8F228FF0E064DB0DE7951155F6F41EF449D0EC10960067C2BF2DCD61874', @@ -507,8 +517,8 @@ def test_can_generate_node_json_given_public_key_not_found(self): facade.reload_all(Path('tests/resources'), True) # Act: - main_public_key_descriptors = facade.json_node(filter_field='mainPublicKey', public_key="invalidKey") - node_public_key_descriptors = facade.json_node(filter_field='nodePublicKey', public_key="invalidKey") + main_public_key_descriptors = facade.json_node(filter_field='mainPublicKey', public_key='invalidKey') + node_public_key_descriptors = facade.json_node(filter_field='nodePublicKey', public_key='invalidKey') # Assert: self.assertIsNone(main_public_key_descriptors) diff --git a/explorer/nodewatch/tests/test_app.py b/explorer/nodewatch/tests/test_app.py index e29829332..d8ca8eae5 100644 --- a/explorer/nodewatch/tests/test_app.py +++ b/explorer/nodewatch/tests/test_app.py @@ -162,106 +162,82 @@ def test_get_api_nem_network_height_chart(client): # pylint: disable=redefined- assert re.match(r'\d\d:\d\d', response_json['lastRefreshTime']) -def test_get_api_symbol_nodes_api(client): # pylint: disable=redefined-outer-name - # Act: - response = client.get('/api/symbol/nodes/api') +def _assert_symbol_node_response(response, expected_names): + # Arrange: response_json = json.loads(response.data) # Assert: spot check names assert 200 == response.status_code assert 'application/json' == response.headers['Content-Type'] - assert 1 == len(response_json) - assert [ - 'Allnodes250' - ] == list(map(lambda descriptor: descriptor['name'], response_json)) + assert len(expected_names) == len(response_json) + assert expected_names == list(map(lambda descriptor: descriptor['name'], response_json)) -def test_get_api_symbol_nodes_peer(client): # pylint: disable=redefined-outer-name - # Act: - response = client.get('/api/symbol/nodes/peer') - response_json = json.loads(response.data) - - # Assert: spot check names - assert 200 == response.status_code - assert 'application/json' == response.headers['Content-Type'] - assert 7 == len(response_json) - assert [ - 'Apple', 'Shin-Kuma-Node', 'ibone74', 'jaguar', 'symbol.ooo maxUnlockedAccounts:100', 'xym pool', 'yasmine farm' - ] == list(map(lambda descriptor: descriptor['name'], response_json)) - - -def test_get_api_symbol_nodes(client): # pylint: disable=redefined-outer-name +def test_get_api_symbol_nodes_api(client): # pylint: disable=redefined-outer-name # Act: - response = client.get('/api/symbol/nodes') - response_json = json.loads(response.data) + response = client.get('/api/symbol/nodes/api') - # Assert: spot check names - assert 200 == response.status_code - assert 'application/json' == response.headers['Content-Type'] - assert 8 == len(response_json) - assert [ - 'Allnodes250', 'Apple', 'Shin-Kuma-Node', 'ibone74', 'jaguar', 'symbol.ooo maxUnlockedAccounts:100', 'xym pool', 'yasmine farm' - ] == list(map(lambda descriptor: descriptor['name'], response_json)) + # Assert: + _assert_symbol_node_response(response, ['Allnodes250', 'Allnodes251']) -def test_get_api_symbol_nodes_order_random_limit_2(client): # pylint: disable=redefined-outer-name +def test_get_api_symbol_nodes_api_order_random_limit_2(client): # pylint: disable=redefined-outer-name # Act: - response = client.get('/api/symbol/nodes?order=random&limit=2') + response = client.get('/api/symbol/nodes/api?order=random&limit=2') response_json = json.loads(response.data) # Assert: spot check names - assert 200 == response.status_code - assert 'application/json' == response.headers['Content-Type'] - assert 2 == len(response_json) - full_node_names = [ - 'Allnodes250', 'Apple', 'Shin-Kuma-Node', 'ibone74', 'jaguar', 'symbol.ooo maxUnlockedAccounts:100', 'xym pool', 'yasmine farm' + full_api_node_names = [ + 'Allnodes250', 'Allnodes251' ] actual_names = list(map(lambda descriptor: descriptor['name'], response_json)) + _assert_symbol_node_response(response, actual_names) for name in actual_names: - assert name in full_node_names + assert name in full_api_node_names -def test_get_api_symbol_nodes_limit_2(client): # pylint: disable=redefined-outer-name +def test_get_api_symbol_nodes_api_ssl_true(client): # pylint: disable=redefined-outer-name # Act: - response = client.get('/api/symbol/nodes?limit=2') - response_json = json.loads(response.data) + response = client.get('/api/symbol/nodes/api?ssl=true') - # Assert: spot check names - assert 200 == response.status_code - assert 'application/json' == response.headers['Content-Type'] - assert 2 == len(response_json) - assert [ - 'Allnodes250', 'Apple' - ] == list(map(lambda descriptor: descriptor['name'], response_json)) + # Assert: + _assert_symbol_node_response(response, ['Allnodes251']) -def test_get_api_symbol_nodes_ssl_true(client): # pylint: disable=redefined-outer-name +def test_get_api_symbol_nodes_peer(client): # pylint: disable=redefined-outer-name # Act: - response = client.get('/api/symbol/nodes?ssl=true') - response_json = json.loads(response.data) + response = client.get('/api/symbol/nodes/peer') - # Assert: spot check names - assert 200 == response.status_code - assert 'application/json' == response.headers['Content-Type'] - assert 1 == len(response_json) - assert [ - 'xym pool' - ] == list(map(lambda descriptor: descriptor['name'], response_json)) + # Assert: + _assert_symbol_node_response( + response, + ['Apple', 'Shin-Kuma-Node', 'ibone74', 'jaguar', 'symbol.ooo maxUnlockedAccounts:100', 'xym pool', 'yasmine farm'] + ) -def test_get_api_symbol_nodes_ssl_false(client): # pylint: disable=redefined-outer-name +def test_get_api_symbol_nodes_peer_order_random_limit_2(client): # pylint: disable=redefined-outer-name # Act: - response = client.get('/api/symbol/nodes?ssl=false') + response = client.get('/api/symbol/nodes/peer?order=random&limit=2') response_json = json.loads(response.data) - # Assert: spot check names - assert 200 == response.status_code - assert 'application/json' == response.headers['Content-Type'] - assert 1 == len(response_json) - assert [ - 'yasmine farm' - ] == list(map(lambda descriptor: descriptor['name'], response_json)) + full_node_names = [ + 'Allnodes250', 'Apple', 'Shin-Kuma-Node', 'ibone74', 'jaguar', 'symbol.ooo maxUnlockedAccounts:100', 'xym pool', 'yasmine farm' + ] + actual_names = list(map(lambda descriptor: descriptor['name'], response_json)) + + # Assert: + _assert_symbol_node_response(response, actual_names) + for name in actual_names: + assert name in full_node_names + + +def test_get_api_symbol_nodes_peer_ssl_true(client): # pylint: disable=redefined-outer-name + # Act: + response = client.get('/api/symbol/nodes/peer?ssl=true') + + # Assert: + _assert_symbol_node_response(response, ['xym pool']) def _assert_api_symbol_node_with_public_key_not_found(response): From 9a18900865ffe7c692eb0e796baf73da89446d46 Mon Sep 17 00:00:00 2001 From: AnthonyLaw Date: Fri, 14 Apr 2023 04:13:29 +0800 Subject: [PATCH 10/13] [explorer/nodewatch] task: more refactor and unit test --- explorer/nodewatch/nodewatch/RoutesFacade.py | 21 +++--- explorer/nodewatch/nodewatch/__init__.py | 60 +++++++++-------- explorer/nodewatch/tests/test_RoutesFacade.py | 67 ++++++++++++++----- explorer/nodewatch/tests/test_app.py | 23 ++++--- 4 files changed, 103 insertions(+), 68 deletions(-) diff --git a/explorer/nodewatch/nodewatch/RoutesFacade.py b/explorer/nodewatch/nodewatch/RoutesFacade.py index e72869de1..907a0e1d4 100644 --- a/explorer/nodewatch/nodewatch/RoutesFacade.py +++ b/explorer/nodewatch/nodewatch/RoutesFacade.py @@ -69,11 +69,11 @@ def html_nodes(self): def json_nodes(self, **kwargs): """Returns all nodes with condition.""" - role = kwargs.get('role', None) - exact_match = kwargs.get('exact_match', False) - limit = kwargs.get('limit', None) - ssl = kwargs.get('ssl', None) - order = kwargs.get('order', None) + role = kwargs.get('role') + exact_match = kwargs.get('exact_match') + limit = kwargs.get('limit') + only_ssl = kwargs.get('only_ssl') + order = kwargs.get('order') def custom_filter(descriptor): role_condition = True @@ -81,7 +81,7 @@ def custom_filter(descriptor): if role is not None: role_condition = role == descriptor.roles if exact_match else role == (role & descriptor.roles) - if ssl is not None: + if only_ssl is True: ssl_condition = (descriptor.is_https_enabled and descriptor.is_wss_enabled) return role_condition and ssl_condition @@ -94,17 +94,14 @@ def custom_filter(descriptor): if order == 'random': random.shuffle(nodes) - if limit is not None: - nodes = nodes[:limit] - - return nodes + return nodes if limit == 0 else nodes[:limit] def json_node(self, filter_field, public_key): """Returns a node with matching public key.""" - matching_items = [item.to_json() for item in self.repository.node_descriptors if item.to_json()[filter_field] == public_key] + matching_items = [item for item in self.repository.node_descriptors if str(getattr(item, filter_field)) == public_key] - return next(iter(matching_items), None) + return next((item.to_json() for item in matching_items), None) def json_height_chart(self): """Builds a JSON height chart.""" diff --git a/explorer/nodewatch/nodewatch/__init__.py b/explorer/nodewatch/nodewatch/__init__.py index e602bd0b5..5b27c587a 100644 --- a/explorer/nodewatch/nodewatch/__init__.py +++ b/explorer/nodewatch/nodewatch/__init__.py @@ -2,7 +2,7 @@ from pathlib import Path from apscheduler.schedulers.background import BackgroundScheduler -from flask import Flask, jsonify, redirect, render_template, request, url_for +from flask import Flask, abort, jsonify, redirect, render_template, request, url_for from symbolchain.CryptoTypes import Hash256 from symbolchain.nem.Network import Network as NemNetwork from symbolchain.Network import NetworkLocator @@ -13,14 +13,8 @@ class Field(Enum): - MAIN_PUBLIC_KEY = 'mainPublicKey' - NODE_PUBLIC_KEY = 'nodePublicKey' - - -def str_to_bool(value): - if value.lower() == 'true': - return True - return None + MAIN_PUBLIC_KEY = 'main_public_key' + NODE_PUBLIC_KEY = 'node_public_key' def create_app(): @@ -103,38 +97,42 @@ def symbol_summary(): # pylint: disable=unused-variable template_name, context = symbol_routes_facade.html_summary() return render_template(template_name, **context) - def _get_json_nodes(role, exact_match, ssl, limit, order): - if ssl is not None: - ssl = str_to_bool(ssl) + def _get_json_nodes(role, exact_match, request_args): + only_ssl = None + if 'only_ssl' in request_args: + only_ssl = True + + order = request_args.get('order', None) - if limit is not None: - limit = int(limit) + limit = int(request_args.get('limit', 0)) - return jsonify(symbol_routes_facade.json_nodes(role=role, exact_match=exact_match, ssl=ssl, limit=limit, order=order)) + return jsonify(symbol_routes_facade.json_nodes(role=role, exact_match=exact_match, only_ssl=only_ssl, limit=limit, order=order)) + + def _get_json_node(result): + if not result: + abort(404) + + return jsonify(result) @app.route('/api/symbol/nodes/api') def api_symbol_nodes_api(): # pylint: disable=unused-variable - ssl = request.args.get('ssl', None) - limit = request.args.get('limit', None) - order = request.args.get('order', None) - - return _get_json_nodes(2, True, ssl, limit, order) + return _get_json_nodes(2, True, request.args) @app.route('/api/symbol/nodes/peer') def api_symbol_nodes_peer(): # pylint: disable=unused-variable - ssl = request.args.get('ssl', None) - limit = request.args.get('limit', None) - order = request.args.get('order', None) - - return _get_json_nodes(1, False, ssl, limit, order) + return _get_json_nodes(1, False, request.args) @app.route('/api/symbol/nodes/mainPublicKey/') def api_symbol_nodes_get_main_public_key(main_public_key): # pylint: disable=unused-variable - return jsonify(symbol_routes_facade.json_node(filter_field=Field.MAIN_PUBLIC_KEY.value, public_key=main_public_key)) + result = symbol_routes_facade.json_node(filter_field=Field.MAIN_PUBLIC_KEY.value, public_key=main_public_key) + + return _get_json_node(result) @app.route('/api/symbol/nodes/nodePublicKey/') def api_symbol_nodes_get_node_public_key(node_public_key): # pylint: disable=unused-variable - return jsonify(symbol_routes_facade.json_node(filter_field=Field.NODE_PUBLIC_KEY.value, public_key=node_public_key)) + result = symbol_routes_facade.json_node(filter_field=Field.NODE_PUBLIC_KEY.value, public_key=node_public_key) + + return _get_json_node(result) @app.route('/api/symbol/chart/height') def api_symbol_chart_height(): # pylint: disable=unused-variable @@ -152,6 +150,14 @@ def inject_timestamps(): # pylint: disable=unused-variable 'last_refresh_time': routes_facade.last_refresh_time.strftime(TIMESTAMP_FORMAT) } + @app.errorhandler(404) + def not_found(_): + response = { + 'status': 404, + 'message': 'Resource not found' + } + return jsonify(response), 404 + def reload_all(force=False): log.debug('reloading all data') nem_routes_facade.reload_all(resources_path, force) diff --git a/explorer/nodewatch/tests/test_RoutesFacade.py b/explorer/nodewatch/tests/test_RoutesFacade.py index e892cca0f..97833287d 100644 --- a/explorer/nodewatch/tests/test_RoutesFacade.py +++ b/explorer/nodewatch/tests/test_RoutesFacade.py @@ -212,7 +212,7 @@ def test_can_map_version_to_css_class(self): # endregion -class SymbolRoutesFacadeTest(unittest.TestCase): +class SymbolRoutesFacadeTest(unittest.TestCase): # pylint: disable=too-many-public-methods # region reload / refresh def test_can_reload_all(self): @@ -371,18 +371,28 @@ def test_can_generate_nodes_json(self): facade.reload_all(Path('tests/resources'), True) # Act: - node_descriptors = facade.json_nodes(role=1) + node_descriptors = facade.json_nodes() # Assert: spot check names and roles - self.assertEqual(7, len(node_descriptors)) + self.assertEqual(9, len(node_descriptors)) self.assertEqual( - ['Apple', 'Shin-Kuma-Node', 'ibone74', 'jaguar', 'symbol.ooo maxUnlockedAccounts:100', 'xym pool', 'yasmine farm'], + [ + 'Allnodes250', + 'Allnodes251', + 'Apple', + 'Shin-Kuma-Node', + 'ibone74', + 'jaguar', + 'symbol.ooo maxUnlockedAccounts:100', + 'xym pool', + 'yasmine farm' + ], list(map(lambda descriptor: descriptor['name'], node_descriptors))) self.assertEqual( - [7, 3, 3, 5, 3, 3, 3], + [2, 2, 7, 3, 3, 5, 3, 3, 3], list(map(lambda descriptor: descriptor['roles'], node_descriptors))) - def test_can_generate_nodes_json_filtered(self): + def test_can_generate_nodes_json_filtered_by_role(self): # Arrange: facade = SymbolRoutesFacade(SymbolNetwork.MAINNET, '') facade.reload_all(Path('tests/resources'), True) @@ -416,13 +426,13 @@ def test_can_generate_nodes_json_filtered_exact_match(self): [2, 2], list(map(lambda descriptor: descriptor['roles'], node_descriptors))) - def test_can_generate_nodes_json_filtered_ssl(self): + def test_can_generate_nodes_json_filtered_only_ssl(self): # Arrange: facade = SymbolRoutesFacade(SymbolNetwork.MAINNET, '') facade.reload_all(Path('tests/resources'), True) # Act: select nodes with only ssl enabled - node_descriptors = facade.json_nodes(ssl=True) + node_descriptors = facade.json_nodes(only_ssl=True) # Assert: spot check names and roles self.assertEqual(2, len(node_descriptors)) @@ -433,7 +443,7 @@ def test_can_generate_nodes_json_filtered_ssl(self): [2, 3], list(map(lambda descriptor: descriptor['roles'], node_descriptors))) - def test_can_generate_nodes_json_filtered_order_random_limit_2(self): + def test_can_generate_nodes_json_filtered_order_random_subset(self): # Arrange: facade = SymbolRoutesFacade(SymbolNetwork.MAINNET, '') facade.reload_all(Path('tests/resources'), True) @@ -441,18 +451,35 @@ def test_can_generate_nodes_json_filtered_order_random_limit_2(self): # Act: select 2 nodes with order random node_descriptors = facade.json_nodes(limit=2, order='random') - # returns all nodes + # Assert: all_node_descriptors = facade.json_nodes() full_node_names = list(map(lambda descriptor: descriptor['name'], all_node_descriptors)) random_node_names = list(map(lambda descriptor: descriptor['name'], node_descriptors)) - # Assert: spot check names self.assertEqual(2, len(node_descriptors)) + self.assertEqual(2, len(set(random_node_names))) for name in random_node_names: self.assertIn(name, full_node_names) - def test_can_generate_node_json_given_main_public_key(self): + def test_can_generate_nodes_json_filtered_limit_5(self): + # Arrange: + facade = SymbolRoutesFacade(SymbolNetwork.MAINNET, '') + facade.reload_all(Path('tests/resources'), True) + + # Act: + node_descriptors = facade.json_nodes(limit=5) + + # Assert: spot check names + self.assertEqual(5, len(node_descriptors)) + self.assertEqual( + ['Allnodes250', 'Allnodes251', 'Apple', 'Shin-Kuma-Node', 'ibone74'], + list(map(lambda descriptor: descriptor['name'], node_descriptors))) + self.assertEqual( + [2, 2, 7, 3, 3], + list(map(lambda descriptor: descriptor['roles'], node_descriptors))) + + def can_find_known_node_by_main_public_key(self): # pylint: disable=invalid-name # Arrange: facade = SymbolRoutesFacade(SymbolNetwork.MAINNET, '') facade.reload_all(Path('tests/resources'), True) @@ -482,7 +509,7 @@ def test_can_generate_node_json_given_main_public_key(self): # Assert: self.assertEqual(node_descriptors, expected_node) - def test_can_generate_node_json_given_node_public_key(self): + def can_find_known_node_by_node_public_key(self): # pylint: disable=invalid-name # Arrange: facade = SymbolRoutesFacade(SymbolNetwork.MAINNET, '') facade.reload_all(Path('tests/resources'), True) @@ -511,18 +538,22 @@ def test_can_generate_node_json_given_node_public_key(self): # Assert: self.assertEqual(node_descriptors, expected_node) - def test_can_generate_node_json_given_public_key_not_found(self): + def _assert_unknown_node_not_found(self, public_key): # Arrange: facade = SymbolRoutesFacade(SymbolNetwork.MAINNET, '') facade.reload_all(Path('tests/resources'), True) # Act: - main_public_key_descriptors = facade.json_node(filter_field='mainPublicKey', public_key='invalidKey') - node_public_key_descriptors = facade.json_node(filter_field='nodePublicKey', public_key='invalidKey') + node_descriptors = facade.json_node(filter_field=public_key, public_key='invalidKey') # Assert: - self.assertIsNone(main_public_key_descriptors) - self.assertIsNone(node_public_key_descriptors) + self.assertIsNone(node_descriptors) + + def cannot_find_unknown_node_by_main_public_key(self): # pylint: disable=invalid-name + self._assert_unknown_node_not_found('mainPublicKey') + + def cannot_find_unknown_node_by_node_public_key(self): # pylint: disable=invalid-name + self._assert_unknown_node_not_found('nodePublicKey') def test_can_generate_height_chart_json(self): # Arrange: diff --git a/explorer/nodewatch/tests/test_app.py b/explorer/nodewatch/tests/test_app.py index d8ca8eae5..4e28332e5 100644 --- a/explorer/nodewatch/tests/test_app.py +++ b/explorer/nodewatch/tests/test_app.py @@ -170,6 +170,7 @@ def _assert_symbol_node_response(response, expected_names): assert 200 == response.status_code assert 'application/json' == response.headers['Content-Type'] assert len(expected_names) == len(response_json) + assert len(expected_names) == len(set(expected_names)) assert expected_names == list(map(lambda descriptor: descriptor['name'], response_json)) @@ -181,25 +182,25 @@ def test_get_api_symbol_nodes_api(client): # pylint: disable=redefined-outer-na _assert_symbol_node_response(response, ['Allnodes250', 'Allnodes251']) -def test_get_api_symbol_nodes_api_order_random_limit_2(client): # pylint: disable=redefined-outer-name +def test_get_api_symbol_nodes_api_order_random_subset(client): # pylint: disable=redefined-outer-name # Act: - response = client.get('/api/symbol/nodes/api?order=random&limit=2') + response = client.get('/api/symbol/nodes/api?order=random&limit=1') response_json = json.loads(response.data) - # Assert: spot check names full_api_node_names = [ 'Allnodes250', 'Allnodes251' ] actual_names = list(map(lambda descriptor: descriptor['name'], response_json)) + # Assert: _assert_symbol_node_response(response, actual_names) for name in actual_names: assert name in full_api_node_names -def test_get_api_symbol_nodes_api_ssl_true(client): # pylint: disable=redefined-outer-name +def test_get_api_symbol_nodes_api_only_ssl(client): # pylint: disable=redefined-outer-name # Act: - response = client.get('/api/symbol/nodes/api?ssl=true') + response = client.get('/api/symbol/nodes/api?only_ssl') # Assert: _assert_symbol_node_response(response, ['Allnodes251']) @@ -216,7 +217,7 @@ def test_get_api_symbol_nodes_peer(client): # pylint: disable=redefined-outer-n ) -def test_get_api_symbol_nodes_peer_order_random_limit_2(client): # pylint: disable=redefined-outer-name +def test_get_api_symbol_nodes_peer_order_random_subset(client): # pylint: disable=redefined-outer-name # Act: response = client.get('/api/symbol/nodes/peer?order=random&limit=2') response_json = json.loads(response.data) @@ -232,9 +233,9 @@ def test_get_api_symbol_nodes_peer_order_random_limit_2(client): # pylint: disa assert name in full_node_names -def test_get_api_symbol_nodes_peer_ssl_true(client): # pylint: disable=redefined-outer-name +def test_get_api_symbol_nodes_peer_only_ssl(client): # pylint: disable=redefined-outer-name # Act: - response = client.get('/api/symbol/nodes/peer?ssl=true') + response = client.get('/api/symbol/nodes/peer?only_ssl') # Assert: _assert_symbol_node_response(response, ['xym pool']) @@ -244,10 +245,10 @@ def _assert_api_symbol_node_with_public_key_not_found(response): # Act: response_json = json.loads(response.data) - # Assert: spot check names - assert 200 == response.status_code + # Assert: + assert 404 == response.status_code assert 'application/json' == response.headers['Content-Type'] - assert response_json is None + assert response_json == {'message': 'Resource not found', 'status': 404} def _assert_api_symbol_node_with_public_key_found(response, expected_name): From 34721cb33cec2495f73f2abb473ec66c41911c21 Mon Sep 17 00:00:00 2001 From: AnthonyLaw Date: Fri, 26 May 2023 04:27:05 +0800 Subject: [PATCH 11/13] [explorer/nodewatch] task: improve unit test --- explorer/nodewatch/nodewatch/RoutesFacade.py | 2 +- explorer/nodewatch/tests/test_RoutesFacade.py | 1 + explorer/nodewatch/tests/test_app.py | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/explorer/nodewatch/nodewatch/RoutesFacade.py b/explorer/nodewatch/nodewatch/RoutesFacade.py index 907a0e1d4..a5193007d 100644 --- a/explorer/nodewatch/nodewatch/RoutesFacade.py +++ b/explorer/nodewatch/nodewatch/RoutesFacade.py @@ -81,7 +81,7 @@ def custom_filter(descriptor): if role is not None: role_condition = role == descriptor.roles if exact_match else role == (role & descriptor.roles) - if only_ssl is True: + if only_ssl: ssl_condition = (descriptor.is_https_enabled and descriptor.is_wss_enabled) return role_condition and ssl_condition diff --git a/explorer/nodewatch/tests/test_RoutesFacade.py b/explorer/nodewatch/tests/test_RoutesFacade.py index 97833287d..cda51dbb1 100644 --- a/explorer/nodewatch/tests/test_RoutesFacade.py +++ b/explorer/nodewatch/tests/test_RoutesFacade.py @@ -226,6 +226,7 @@ def test_can_reload_all(self): self.assertEqual(True, result) self.assertEqual(facade.last_reload_time, facade.last_refresh_time) + # loaded all mainnet (104) nodes, skipping testnet (152) nodes self.assertEqual(9, len(facade.repository.node_descriptors)) self.assertEqual(4, len(facade.repository.harvester_descriptors)) self.assertEqual(4, len(facade.repository.voter_descriptors)) diff --git a/explorer/nodewatch/tests/test_app.py b/explorer/nodewatch/tests/test_app.py index 4e28332e5..ad452af7a 100644 --- a/explorer/nodewatch/tests/test_app.py +++ b/explorer/nodewatch/tests/test_app.py @@ -194,6 +194,7 @@ def test_get_api_symbol_nodes_api_order_random_subset(client): # pylint: disabl # Assert: _assert_symbol_node_response(response, actual_names) + assert len(actual_names) == 1 for name in actual_names: assert name in full_api_node_names @@ -229,6 +230,7 @@ def test_get_api_symbol_nodes_peer_order_random_subset(client): # pylint: disab # Assert: _assert_symbol_node_response(response, actual_names) + assert len(actual_names) == 2 for name in actual_names: assert name in full_node_names From 3010a0cf0c08e2be3fb771f972c7549dec9c73ec Mon Sep 17 00:00:00 2001 From: AnthonyLaw Date: Fri, 26 May 2023 04:28:09 +0800 Subject: [PATCH 12/13] [explorer/nodewatch] task: fix undetected unit test --- explorer/nodewatch/tests/test_RoutesFacade.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/explorer/nodewatch/tests/test_RoutesFacade.py b/explorer/nodewatch/tests/test_RoutesFacade.py index cda51dbb1..209064b2a 100644 --- a/explorer/nodewatch/tests/test_RoutesFacade.py +++ b/explorer/nodewatch/tests/test_RoutesFacade.py @@ -480,14 +480,14 @@ def test_can_generate_nodes_json_filtered_limit_5(self): [2, 2, 7, 3, 3], list(map(lambda descriptor: descriptor['roles'], node_descriptors))) - def can_find_known_node_by_main_public_key(self): # pylint: disable=invalid-name + def test_can_find_known_node_by_main_public_key(self): # pylint: disable=invalid-name # Arrange: facade = SymbolRoutesFacade(SymbolNetwork.MAINNET, '') facade.reload_all(Path('tests/resources'), True) # Act: select a node match with main public key node_descriptors = facade.json_node( - filter_field='mainPublicKey', + filter_field='main_public_key', public_key='A0AA48B6417BDB1845EB55FB0B1E13255EA8BD0D8FA29AD2D8A906E220571F21' ) expected_node = { @@ -510,14 +510,14 @@ def can_find_known_node_by_main_public_key(self): # pylint: disable=invalid-nam # Assert: self.assertEqual(node_descriptors, expected_node) - def can_find_known_node_by_node_public_key(self): # pylint: disable=invalid-name + def test_can_find_known_node_by_node_public_key(self): # pylint: disable=invalid-name # Arrange: facade = SymbolRoutesFacade(SymbolNetwork.MAINNET, '') facade.reload_all(Path('tests/resources'), True) # Act: select a node match with node public key node_descriptors = facade.json_node( - filter_field='nodePublicKey', + filter_field='node_public_key', public_key='D05BE3101F2916AA34839DDC1199BE45092103A9B66172FA3D05911DC041AADA' ) expected_node = { @@ -550,11 +550,11 @@ def _assert_unknown_node_not_found(self, public_key): # Assert: self.assertIsNone(node_descriptors) - def cannot_find_unknown_node_by_main_public_key(self): # pylint: disable=invalid-name - self._assert_unknown_node_not_found('mainPublicKey') + def test_cannot_find_unknown_node_by_main_public_key(self): # pylint: disable=invalid-name + self._assert_unknown_node_not_found('main_public_key') - def cannot_find_unknown_node_by_node_public_key(self): # pylint: disable=invalid-name - self._assert_unknown_node_not_found('nodePublicKey') + def test_cannot_find_unknown_node_by_node_public_key(self): # pylint: disable=invalid-name + self._assert_unknown_node_not_found('node_public_key') def test_can_generate_height_chart_json(self): # Arrange: From ab1851b065a2e09bc289002c3fbb1ad78c595e7e Mon Sep 17 00:00:00 2001 From: AnthonyLaw Date: Fri, 26 May 2023 04:41:48 +0800 Subject: [PATCH 13/13] [explorer/nodewatch] task: add validate public key --- explorer/nodewatch/nodewatch/__init__.py | 20 +++++++++++++++++- explorer/nodewatch/tests/test_app.py | 26 ++++++++++++++++++++++-- 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/explorer/nodewatch/nodewatch/__init__.py b/explorer/nodewatch/nodewatch/__init__.py index 5b27c587a..d2bba994b 100644 --- a/explorer/nodewatch/nodewatch/__init__.py +++ b/explorer/nodewatch/nodewatch/__init__.py @@ -3,7 +3,7 @@ from apscheduler.schedulers.background import BackgroundScheduler from flask import Flask, abort, jsonify, redirect, render_template, request, url_for -from symbolchain.CryptoTypes import Hash256 +from symbolchain.CryptoTypes import Hash256, PublicKey from symbolchain.nem.Network import Network as NemNetwork from symbolchain.Network import NetworkLocator from symbolchain.symbol.Network import Network as SymbolNetwork @@ -114,6 +114,12 @@ def _get_json_node(result): return jsonify(result) + def _validate_public_key(public_key): + try: + PublicKey(public_key) + except ValueError: + abort(400) + @app.route('/api/symbol/nodes/api') def api_symbol_nodes_api(): # pylint: disable=unused-variable return _get_json_nodes(2, True, request.args) @@ -124,12 +130,16 @@ def api_symbol_nodes_peer(): # pylint: disable=unused-variable @app.route('/api/symbol/nodes/mainPublicKey/') def api_symbol_nodes_get_main_public_key(main_public_key): # pylint: disable=unused-variable + _validate_public_key(main_public_key) + result = symbol_routes_facade.json_node(filter_field=Field.MAIN_PUBLIC_KEY.value, public_key=main_public_key) return _get_json_node(result) @app.route('/api/symbol/nodes/nodePublicKey/') def api_symbol_nodes_get_node_public_key(node_public_key): # pylint: disable=unused-variable + _validate_public_key(node_public_key) + result = symbol_routes_facade.json_node(filter_field=Field.NODE_PUBLIC_KEY.value, public_key=node_public_key) return _get_json_node(result) @@ -150,6 +160,14 @@ def inject_timestamps(): # pylint: disable=unused-variable 'last_refresh_time': routes_facade.last_refresh_time.strftime(TIMESTAMP_FORMAT) } + @app.errorhandler(400) + def bad_request(_): + response = { + 'status': 400, + 'message': 'Bad request' + } + return jsonify(response), 400 + @app.errorhandler(404) def not_found(_): response = { diff --git a/explorer/nodewatch/tests/test_app.py b/explorer/nodewatch/tests/test_app.py index ad452af7a..9b277cd8e 100644 --- a/explorer/nodewatch/tests/test_app.py +++ b/explorer/nodewatch/tests/test_app.py @@ -253,6 +253,16 @@ def _assert_api_symbol_node_with_public_key_not_found(response): assert response_json == {'message': 'Resource not found', 'status': 404} +def _assert_api_symbol_node_with_invalid_public_key(response): + # Act: + response_json = json.loads(response.data) + + # Assert: + assert 400 == response.status_code + assert 'application/json' == response.headers['Content-Type'] + assert response_json == {'message': 'Bad request', 'status': 400} + + def _assert_api_symbol_node_with_public_key_found(response, expected_name): # Act: response_json = json.loads(response.data) @@ -264,11 +274,23 @@ def _assert_api_symbol_node_with_public_key_found(response, expected_name): def test_get_api_symbol_node_with_invalid_main_public_key(client): # pylint: disable=redefined-outer-name - _assert_api_symbol_node_with_public_key_not_found(client.get('/api/symbol/nodes/mainPublicKey/invalid')) + _assert_api_symbol_node_with_invalid_public_key(client.get('/api/symbol/nodes/mainPublicKey/invalid')) def test_get_api_symbol_node_with_invalid_node_public_key(client): # pylint: disable=redefined-outer-name - _assert_api_symbol_node_with_public_key_not_found(client.get('/api/symbol/nodes/nodePublicKey/invalid')) + _assert_api_symbol_node_with_invalid_public_key(client.get('/api/symbol/nodes/nodePublicKey/invalid')) + + +def test_get_api_symbol_node_with_main_public_key_not_found(client): # pylint: disable=redefined-outer-name + _assert_api_symbol_node_with_public_key_not_found( + client.get('/api/symbol/nodes/mainPublicKey/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa') + ) + + +def test_get_api_symbol_node_with_node_public_key_not_found(client): # pylint: disable=redefined-outer-name + _assert_api_symbol_node_with_public_key_not_found( + client.get('/api/symbol/nodes/nodePublicKey/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa') + ) def test_get_api_symbol_node_with_main_public_key(client): # pylint: disable=redefined-outer-name