[tools/shoestring]: light rest implementation

tools/shoestring/shoestring/commands/reset_data.py

@@ -3,7 +3,6 @@
 
 from zenlog import log
 
-from shoestring.internal.NodeFeatures import NodeFeatures
 from shoestring.internal.ShoestringConfiguration import parse_shoestring_configuration
 
 
@@ -77,7 +76,7 @@ async def run_main(args):
 	for name in ('data', 'logs'):
 		_purge_and_recreate(directory / name)
 
-	if NodeFeatures.API in config.node.features:
+	if config.node.full_api:
 		_purge_and_recreate(directory / 'dbdata')
 
 	stateful_data_processor.restore()

tools/shoestring/shoestring/commands/setup.py

@@ -92,7 +92,7 @@ async def run_main(args):
 		api_endpoints = await download_peers(
 			config.services.nodewatch,
 			preparer.directories.resources,
-			NodeFeatures.API in config.node.features)
+			config.node.full_api)
 		await download_and_extract_package(args.package, preparer.directories.temp)
 
 		# prepare nemesis data and resources
@@ -121,6 +121,7 @@ async def run_main(args):
 			'catapult_rest_image': config.images.rest,
 			'user': f'{config.node.user_id}:{config.node.group_id}',
 			'api_https': config.node.api_https,
+			'light_api': NodeFeatures.API in config.node.features and not config.node.full_api,
 			'domainname': hostname
 		})
 

tools/shoestring/shoestring/commands/upgrade.py

@@ -45,12 +45,12 @@ async def run_main(args):
 	_recreate_directory(directories.userconfig)
 	_recreate_directory(directories.resources)
 
-	if NodeFeatures.API in config.node.features:
+	if config.node.full_api:
 		_purge_directory(directories.startup)
 		_purge_directory(directories.mongo)
 
-		if config.node.api_https:
-			(directories.https_proxy / 'nginx.conf.erb').unlink()
+	if NodeFeatures.API in config.node.features and config.node.api_https:
+		(directories.https_proxy / 'nginx.conf.erb').unlink()
 
 	await run_setup_main(args)
 

tools/shoestring/shoestring/healthagents/websockets.py

@@ -5,13 +5,12 @@
 from zenlog import log
 
 from shoestring.internal.ConfigurationManager import parse_time_span
-from shoestring.internal.NodeFeatures import NodeFeatures
 
 NAME = 'REST websockets'
 
 
 def should_run(node_config):
-	return NodeFeatures.API in node_config.features
+	return node_config.full_api
 
 
 async def validate(context):

tools/shoestring/shoestring/internal/Preparer.py

@@ -164,12 +164,15 @@ def create_subdirectories(self):
 			self.directories.userconfig,
 			self.directories.resources
 		]
+
 		if NodeFeatures.API in self.config.node.features:
-			directories.append(self.directories.dbdata)
 			directories.append(self.directories.rest_cache)
 			if self.config.node.api_https:
 				directories.append(self.directories.https_proxy)
 
+		if self.config.node.full_api:
+			directories.append(self.directories.dbdata)
+
 		if NodeFeatures.VOTER in self.config.node.features:
 			directories.append(self.directories.voting_keys)
 
@@ -217,7 +220,7 @@ def prepare_resources(self):
 
 		self._copy_properties_files(PEER_EXTENSIONS)
 
-		if NodeFeatures.API in self.config.node.features:
+		if self.config.node.full_api:
 			self._copy_properties_files(API_EXTENSIONS)
 
 		if NodeFeatures.HARVESTER in self.config.node.features:
@@ -231,6 +234,14 @@ def configure_resources(self, user_patches=None):
 		"""Configures resources based on enabled features."""
 
 		if NodeFeatures.API in self.config.node.features:
+			self._patch_resources({
+				'node': [
+					('node', 'trustedHosts', '127.0.0.1,172.20.0.25'),
+					('node', 'localNetworks', '127.0.0.1,172.20'),
+				]
+			})
+
+		if self.config.node.full_api:
 			self._patch_resources({
 				'extensions-server': [
 					('extensions', 'extension.filespooling', 'true'),
@@ -243,8 +254,6 @@ def configure_resources(self, user_patches=None):
 				],
 				'node': [
 					('node', 'enableAutoSyncCleanup', 'false'),
-					('node', 'trustedHosts', '127.0.0.1,172.20.0.25'),
-					('node', 'localNetworks', '127.0.0.1,172.20'),
 					('localnode', 'roles', 'Peer,Api'),
 				]
 			})
@@ -273,10 +282,12 @@ def configure_rest(self, rest_overrides_filename=None):
 		if NodeFeatures.API not in self.config.node.features:
 			return
 
-		self._copy_tree_readonly(self.directories.temp / 'mongo', self.directories.mongo)
-		self._make_files_readonly(self.directories.mongo)
+		rest_file_name = 'rest' if self.config.node.full_api else 'rest-light'
+		self._copy_file(self.directories.temp / 'rest' / f'{rest_file_name}.json', self.directories.userconfig / 'rest.json')
 
-		self._copy_file(self.directories.temp / 'rest' / 'rest.json', self.directories.userconfig)
+		if self.config.node.full_api:
+			self._copy_tree_readonly(self.directories.temp / 'mongo', self.directories.mongo)
+			self._make_files_readonly(self.directories.mongo)
 
 		if rest_overrides_filename:
 			rest_json_filepath = self.directories.userconfig / 'rest.json'
@@ -340,10 +351,12 @@ def generate_certificates(self, ca_key_path, require_ca=True):
 	def configure_docker(self, template_mapping):
 		"""Prepares docker-compose file."""
 
-		if NodeFeatures.API in self.config.node.features:
+		if self.config.node.full_api:
 			self._copy_tree_readonly(_qualify_resource('startup'), self.directories.startup)
+			compose_template_filename_postfix = 'dual'
+		else:
+			compose_template_filename_postfix = 'peer'
 
-		compose_template_filename_postfix = 'dual' if NodeFeatures.API in self.config.node.features else 'peer'
 		compose_template_filename = _qualify_resource(f'templates/docker-compose-{compose_template_filename_postfix}.yaml')
 		compose_output_filepath = self.directory / 'docker-compose.yaml'
 		apply_template(compose_template_filename, template_mapping, compose_output_filepath)

tools/shoestring/shoestring/internal/ShoestringConfiguration.py

@@ -14,7 +14,7 @@
 ])
 ImportsConfiguration = namedtuple('ImportsConfiguration', ['harvester', 'voter', 'node_key'])
 NodeConfiguration = namedtuple('NodeConfiguration', [
-	'features', 'user_id', 'group_id', 'ca_password', 'api_https', 'ca_common_name', 'node_common_name'
+	'features', 'user_id', 'group_id', 'ca_password', 'api_https', 'full_api', 'ca_common_name', 'node_common_name'
 ])
 ShoestringConfiguration = namedtuple('ShoestringConfiguration', ['network', 'images', 'services', 'transaction', 'imports', 'node'])
 
@@ -74,10 +74,11 @@ def parse_node_configuration(config):
 	group_id = int(config['groupId'])
 	ca_password = config['caPassword']
 	api_https = config['apiHttps'].lower() == 'true'
+	full_api = NodeFeatures.API in features and not config['lightApi'].lower() == 'true'
 	ca_common_name = config['caCommonName']
 	node_common_name = config['nodeCommonName']
 
-	return NodeConfiguration(features, user_id, group_id, ca_password, api_https, ca_common_name, node_common_name)
+	return NodeConfiguration(features, user_id, group_id, ca_password, api_https, full_api, ca_common_name, node_common_name)
 
 
 def parse_shoestring_configuration(filename):

tools/shoestring/shoestring/templates/docker-compose-peer.yaml

@@ -25,3 +25,52 @@ services:
       - ./logs:/logs
       - ./keys/voting:/votingkeys
       - ./keys/cert:/certificates
+
+{%- if light_api %}
+  rest-api:
+    image: '{{ catapult_rest_image }}'
+    user: '{{ user }}'
+    command: npm run start-light --prefix /app /userconfig/rest.json
+    environment:
+      - HOME=/symbol-workdir
+      - NODE_ENV=production
+    working_dir: /symbol-workdir
+{%- if not api_https %}
+    ports:
+      - 3000:3000
+{%- endif %}
+    volumes:
+      - ./startup:/startup:ro
+      - ./userconfig:/userconfig:ro
+      - ./keys/cert:/certificates:ro
+      - ./data:/data
+      - ./logs:/logs
+      - ./rest-cache:/symbol-workdir
+    networks:
+      default:
+        ipv4_address: 172.20.0.25
+{%- if api_https %}
+  rest-api-https-proxy:
+    image: steveltn/https-portal:1
+    ports:
+      - 80:80
+      - 3000:80
+      - 3001:443
+    links:
+      - rest-api
+    environment:
+      - DOMAINS={{ domainname }} -> http://rest-api:3000
+      - WEBSOCKET=false
+      - STAGE=production
+      - FORCE_RENEW=false
+    volumes:
+      - ./https-proxy:/var/lib/https-portal
+      - ./https-proxy/nginx.conf.erb:/var/lib/nginx-conf/{{ domainname }}.conf.erb:ro
+{%- endif %}
+networks:
+  default:
+    name: catapult-node-network
+    ipam:
+      config:
+        - subnet: 172.20.0.0/24
+{%- endif %}

tools/shoestring/tests/commands/test_health.py

@@ -33,7 +33,7 @@ def _create_configuration(api_https):
 	return ShoestringConfiguration(
 		*(4 * [None]),
 		ImportsConfiguration(None, None, None),
-		NodeConfiguration(NodeFeatures.PEER, None, None, None, api_https, 'CA', 'NODE'))
+		NodeConfiguration(NodeFeatures.PEER, None, None, None, api_https, False, 'CA', 'NODE'))
 
 
 # pylint: disable=invalid-name

tools/shoestring/tests/commands/test_reset_data.py

@@ -21,11 +21,11 @@ def _create_directories_with_placeholders(directory, subdirectory_names):
 
 # region basic
 
-async def _assert_reset_data(node_features, expected_recreated_subdirectories):
+async def _assert_reset_data(node_features, expected_recreated_subdirectories, light_api=False):
 	# Arrange:
 	subdirectory_names = ('data', 'logs', 'dbdata', 'keys', 'unknown')
 	with tempfile.TemporaryDirectory() as output_directory:
-		config_filepath = prepare_shoestring_configuration(output_directory, node_features)
+		config_filepath = prepare_shoestring_configuration(output_directory, node_features, light_api=light_api)
 
 		# - create some directories each with a placeholder file
 		_create_directories_with_placeholders(output_directory, subdirectory_names)
@@ -59,6 +59,10 @@ async def test_can_reset_data_api_node():
 	await _assert_reset_data(NodeFeatures.API, ['data', 'logs', 'dbdata'])
 
 
+async def test_can_reset_data_light_api_node():
+	await _assert_reset_data(NodeFeatures.API, ['data', 'logs'], light_api=True)
+
+
 async def test_can_reset_data_voter_node_without_voter_state():
 	await _assert_reset_data(NodeFeatures.VOTER, ['data', 'logs'])
 

tools/shoestring/tests/commands/test_setup.py

@@ -100,6 +100,11 @@ def server(event_loop, aiohttp_client):
 	'userconfig/rest.json': 0o400
 }
 
+LIGHT_API_OUTPUT_FILES = {
+	'userconfig/rest.json': 0o400,
+	'rest-cache': 0o700
+}
+
 HARVESTER_OUTPUT_FILES = {
 	'keys/remote.pem': 0o400,
 	'keys/vrf.pem': 0o400,

tools/shoestring/tests/commands/test_upgrade.py

@@ -15,6 +15,7 @@
 	API_OUTPUT_FILES,
 	HARVESTER_OUTPUT_FILES,
 	HTTPS_OUTPUT_FILES,
+	LIGHT_API_OUTPUT_FILES,
 	PEER_OUTPUT_FILES,
 	STATE_CHANGE_OUTPUT_FILES,
 	VOTER_OUTPUT_FILES
@@ -82,6 +83,10 @@ def server(event_loop, aiohttp_client):
 	'userconfig/rest.json'
 ]
 
+LIGHT_API_CHANGED_FILES = [
+	'userconfig/rest.json'
+]
+
 HARVESTER_CHANGED_FILES = [
 	'userconfig/resources/config-harvesting.properties'
 ]
@@ -135,12 +140,13 @@ async def _assert_can_upgrade_node(
 	node_features,
 	expected_output_files,
 	expected_changed_files,
-	api_https=False
-):
+	api_https=False,
+	light_api=False
+):  # pylint: disable=too-many-arguments,too-many-positional-arguments
 	# Arrange:
 	with tempfile.TemporaryDirectory() as output_directory:
 		with tempfile.TemporaryDirectory() as package_directory:
-			prepare_shoestring_configuration(package_directory, node_features, server.make_url(''), api_https=api_https)
+			prepare_shoestring_configuration(package_directory, node_features, server.make_url(''), api_https=api_https, light_api=light_api)
 			_prepare_overrides(package_directory, 'name from setup')
 			prepare_testnet_package(package_directory, 'resources.zip')
 
@@ -210,6 +216,18 @@ async def test_can_upgrade_api_node_with_https(server):  # pylint: disable=redef
 	await _assert_can_upgrade_node(server, NodeFeatures.API, expected_output_files, expected_changed_files, api_https=True)
 
 
+async def test_can_upgrade_light_api_node(server):  # pylint: disable=redefined-outer-name
+	expected_output_files = {**PEER_OUTPUT_FILES, **LIGHT_API_OUTPUT_FILES}
+	expected_changed_files = sorted(PEER_CHANGED_FILES + LIGHT_API_CHANGED_FILES)
+	await _assert_can_upgrade_node(server, NodeFeatures.API, expected_output_files, expected_changed_files, light_api=True)
+
+
+async def test_can_upgrade_light_api_node_with_https(server):  # pylint: disable=redefined-outer-name
+	expected_output_files = {**PEER_OUTPUT_FILES, **LIGHT_API_OUTPUT_FILES, **HTTPS_OUTPUT_FILES}
+	expected_changed_files = sorted(PEER_CHANGED_FILES + LIGHT_API_CHANGED_FILES + HTTPS_CHANGED_FILES)
+	await _assert_can_upgrade_node(server, NodeFeatures.API, expected_output_files, expected_changed_files, api_https=True, light_api=True)
+
+
 async def test_can_upgrade_harvester_node(server):  # pylint: disable=redefined-outer-name
 	expected_output_files = {**PEER_OUTPUT_FILES, **HARVESTER_OUTPUT_FILES, **STATE_CHANGE_OUTPUT_FILES}
 	expected_changed_files = sorted(PEER_CHANGED_FILES + HARVESTER_CHANGED_FILES)

tools/shoestring/tests/healthagents/test_peer_api.py

@@ -19,7 +19,7 @@
 
 
 def _create_configuration():
-	node_config = NodeConfiguration(NodeFeatures.PEER, None, None, None, None, 'CA CN', 'NODE CN')
+	node_config = NodeConfiguration(NodeFeatures.PEER, None, None, None, None, None, 'CA CN', 'NODE CN')
 	return ShoestringConfiguration('testnet', *(3 * [None]), ImportsConfiguration(None, None, None), node_config)
 
 
@@ -83,7 +83,7 @@ async def handle_packet(reader, writer):
 def test_should_run_for_all_roles():
 	# Act + Assert:
 	for features in (NodeFeatures.PEER, NodeFeatures.API, NodeFeatures.HARVESTER, NodeFeatures.VOTER):
-		assert should_run(NodeConfiguration(features, *([None] * 6))), str(features)
+		assert should_run(NodeConfiguration(features, *([None] * 7))), str(features)
 
 # endregion
 

tools/shoestring/tests/healthagents/test_peer_certificate.py

@@ -22,7 +22,7 @@
 def test_should_run_for_all_roles():
 	# Act + Assert:
 	for features in (NodeFeatures.PEER, NodeFeatures.API, NodeFeatures.HARVESTER, NodeFeatures.VOTER):
-		assert should_run(NodeConfiguration(features, *([None] * 6))), str(features)
+		assert should_run(NodeConfiguration(features, *([None] * 7))), str(features)
 
 # endregion
 
@@ -34,7 +34,7 @@ def _create_executor():
 
 
 def _create_configuration():
-	node_config = NodeConfiguration(NodeFeatures.PEER, None, None, None, None, 'CA CN', 'NODE CN')
+	node_config = NodeConfiguration(NodeFeatures.PEER, None, None, None, None, None, 'CA CN', 'NODE CN')
 	return ShoestringConfiguration('testnet', *(3 * [None]), ImportsConfiguration(None, None, None), node_config)
 
 

tools/shoestring/tests/healthagents/test_rest_api.py

@@ -55,10 +55,16 @@ async def _process(_, response_body):
 
 def test_should_run_for_api_role():
 	# Act + Assert:
-	assert should_run(NodeConfiguration(NodeFeatures.API, *([None] * 6)))
+	assert should_run(NodeConfiguration(NodeFeatures.API, None, None, None, None, True, None, None))
 
 	for features in (NodeFeatures.PEER, NodeFeatures.HARVESTER, NodeFeatures.VOTER):
-		assert not should_run(NodeConfiguration(features, *([None] * 6))), str(features)
+		assert not should_run(NodeConfiguration(features, *([None] * 7))), str(features)
+
+
+def test_should_run_for_light_node():
+	# Act + Assert:
+	assert should_run(NodeConfiguration(NodeFeatures.API, None, None, None, None, False, None, None))
+
 
 # endregion
 

tools/shoestring/tests/healthagents/test_rest_https_certificate.py

@@ -34,7 +34,7 @@ async def server(aiohttp_server):
 # region should_run
 
 def _create_node_configuration(api_https):
-	return NodeConfiguration(None, None, None, None, api_https, None, None)
+	return NodeConfiguration(None, None, None, None, api_https, None, None, None)
 
 
 def test_should_run_for_https_role():

tools/shoestring/tests/healthagents/test_voting_keys.py

@@ -33,10 +33,10 @@ def server(event_loop, aiohttp_client):
 
 def test_should_run_for_voter_role():
 	# Act + Assert:
-	assert should_run(NodeConfiguration(NodeFeatures.VOTER, *([None] * 6)))
+	assert should_run(NodeConfiguration(NodeFeatures.VOTER, *([None] * 7)))
 
 	for features in (NodeFeatures.PEER, NodeFeatures.API, NodeFeatures.HARVESTER):
-		assert not should_run(NodeConfiguration(features, *([None] * 6))), str(features)
+		assert not should_run(NodeConfiguration(features, *([None] * 7))), str(features)
 
 # endregion