Implement new metadata structure and validation

app/__init__.py

@@ -26,7 +26,7 @@
 from flask_migrate import Migrate, upgrade
 from flask_caching import Cache
 from flask_redis import FlaskRedis
-from app.lib.ToscaInfo import ToscaInfo
+from app.lib.tosca_info import ToscaInfo
 from app.lib.Vault import Vault
 
 import logging
@@ -48,6 +48,7 @@
 app.secret_key = "30bb7cf2-1fef-4d26-83f0-8096b6dcc7a3"
 app.config.from_object('config.default')
 app.config.from_file('config.json', json.load)
+app.config.from_file('../config/schemas/metadata_schema.json', json.load)
 
 if app.config.get("FEATURE_VAULT_INTEGRATION") == "yes":
     app.config.from_file('vault-config.json', json.load)
@@ -107,7 +108,7 @@ def inject_settings():
 
 # initialize ToscaInfo
 tosca: ToscaInfo = ToscaInfo(redis_client, app.config.get("TOSCA_TEMPLATES_DIR"),
-                             app.config.get("SETTINGS_DIR"))
+                             app.config.get("SETTINGS_DIR"), app.config.get("METADATA_SCHEMA"))
 
 from app.errors.routes import errors_bp
 app.register_blueprint(errors_bp)

app/deployments/routes.py

@@ -13,27 +13,26 @@
 # limitations under the License.
 
 import copy
-
+import uuid as uuid_generator
+import io
+import os
+import re
+from urllib.parse import urlparse
+import yaml
 from flask import Blueprint, session, render_template, flash, redirect, url_for, json, request
+from packaging import version
+from werkzeug.exceptions import Forbidden
+from werkzeug.utils import secure_filename
 from app import app, iam_blueprint, tosca, vaultservice
 from app.lib import auth, utils, settings, dbhelpers, yourls
 from app.lib.ldap_user import LdapUserManager
 from app.models.Deployment import Deployment
 from app.providers import sla
-from app.lib import ToscaInfo as tosca_helpers
+from app.lib import tosca_info as tosca_helpers
 from app.lib import openstack as keystone
 from app.lib.orchestrator import Orchestrator
 from app.lib import s3 as s3
-from werkzeug.exceptions import Forbidden
-from werkzeug.utils import secure_filename
 from app.swift.swift import Swift
-from packaging import version
-from urllib.parse import urlparse
-import uuid as uuid_generator
-import yaml
-import io
-import os
-import re
 
 deployments_bp = Blueprint('deployments_bp', __name__,
                            template_folder='templates',

app/deployments/templates/deployments.html

@@ -88,7 +88,7 @@ <h4 class="font-weight-bold text-primary">My deployments</h4>
                         <span class="fas fa-edit mr-2 grey-text"></span>Edit</a>
                     <a class="dropdown-item" href="{{ url_for('deployments_bp.deptemplate', depid=deployment.uuid) }}"><span class="fas fa-search mr-2 grey-text"></span>Show template</a>
                   {% if enable_update_deployment == "yes" and deployment.updatable == 1 %}
-                    <a class="dropdown-item" href="{{ url_for('deployments_bp.depupdate', depid=deployment.uuid) }}"><span class="fas fa-plus mr-2 grey-text"></span>Update</a>
+                    <a class="dropdown-item" href="{{ url_for('deployments_bp.depupdate', depid=deployment.uuid) }}"><span class="fas fa-plus-minus mr-2 grey-text"></span>Add/Remove nodes</a>
                   {% endif %}
                   {% if deployment.deployment_type == "CLOUD" %}
                       <a class="dropdown-item {% if deployment.physicalId is not defined %}disabled{% endif %}" href="{{ url_for('deployments_bp.deplog', depid=deployment.uuid) }}">

app/home/routes.py

@@ -85,7 +85,12 @@ def submit_settings():
                 if repo_url: dashboard_configuration_info['dashboard_configuration_url'] = repo_url
                 if tag_or_branch: dashboard_configuration_info['dashboard_configuration_tag_or_branch'] = tag_or_branch
 
-        tosca.reload()
+        try:
+            tosca.reload()
+        except Exception as error:
+            app.logger.error(f"Error reloading configuration: {error}")
+            flash(f"Error reloading configuration: { type(error).__name__ }. Please check the logs.", "danger")
+
         app.logger.debug("Configuration reloaded")
 
         now = datetime.now()
@@ -124,12 +129,13 @@ def set_template_access(tosca, user_groups, active_group):
         visibility = v.get("metadata").get("visibility") if "visibility" in v.get("metadata") else {"type": "public"}
 
         if visibility.get("type") != "public":
-            allowed_groups = visibility.get("groups")
-            regex = False if "groups_regex" not in visibility else visibility.get("groups_regex")
+
+            regex = False if "groups_regex" not in visibility else True
 
             if regex:
-                access_locked = not re.match(allowed_groups, active_group)
+                access_locked = not re.match(visibility.get('groups_regex'), active_group)
             else:
+                allowed_groups = visibility.get("groups")
                 access_locked = True if active_group not in allowed_groups else False
 
             if (visibility.get("type") == "private" and not access_locked) or visibility.get("type") == "protected":

app/lib/ToscaInfo.py → app/lib/tosca_info.py

@@ -12,19 +12,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""
-    Class to load tosca templates at application start
-"""
+
 import json
 import os
 import io
 from fnmatch import fnmatch
+import uuid
 import yaml
+import jsonschema
 
 
-class ToscaInfo(object):
+class ToscaInfo:
+    """Class to load tosca templates and metadata at application start"""
 
-    def __init__(self, redis_client, tosca_dir=None, settings_dir=None):
+    def __init__(self, redis_client, tosca_dir, settings_dir, metadata_schema):
         """
         Initialize the flask extension
         :param tosca_dir: the dir of the tosca templates
@@ -34,17 +35,15 @@ def __init__(self, redis_client, tosca_dir=None, settings_dir=None):
         self.tosca_dir = tosca_dir + '/'
         self.tosca_params_dir = settings_dir + '/tosca-parameters'
         self.tosca_metadata_dir = settings_dir + '/tosca-metadata'
+        self.metadata_schema = metadata_schema
 
         tosca_info = {}
         tosca_gmetadata = {}
         tosca_templates = []
 
         tosca_templates = self._loadtoscatemplates()
         tosca_info = self._extractalltoscainfo(tosca_templates)
-
-        if os.path.isfile(self.tosca_metadata_dir + "/metadata.yml"):
-            with io.open(self.tosca_metadata_dir + "/metadata.yml") as stream:
-                tosca_gmetadata = yaml.full_load(stream)
+        tosca_gmetadata = self._loadmetadata()
 
         redis_client.set("tosca_templates", json.dumps(tosca_templates))
         redis_client.set("tosca_gmetadata", json.dumps(tosca_gmetadata))
@@ -54,16 +53,24 @@ def __init__(self, redis_client, tosca_dir=None, settings_dir=None):
     def reload(self):
         tosca_templates = self._loadtoscatemplates()
         tosca_info = self._extractalltoscainfo(tosca_templates)
-        tosca_gmetadata = {}
-
-        if os.path.isfile(self.tosca_metadata_dir + "/metadata.yml"):
-            with io.open(self.tosca_metadata_dir + "/metadata.yml") as stream:
-                tosca_gmetadata = yaml.full_load(stream)
+        tosca_gmetadata = self._loadmetadata()
 
         self.redis_client.set("tosca_templates", json.dumps(tosca_templates))
         self.redis_client.set("tosca_gmetadata", json.dumps(tosca_gmetadata))
         self.redis_client.set("tosca_info", json.dumps(tosca_info))
 
+    def _loadmetadata(self):
+        if os.path.isfile(self.tosca_metadata_dir + "/metadata.yml"):
+            with io.open(self.tosca_metadata_dir + "/metadata.yml") as stream:
+                metadata = yaml.full_load(stream)
+
+                # validate against schema
+                jsonschema.validate(metadata, self.metadata_schema, format_checker=jsonschema.Draft202012Validator.FORMAT_CHECKER)
+                #tosca_gmetadata = {service["id"]: {k: v for k, v in service.items() if k != "id"} for service in metadata['services']}
+                tosca_gmetadata = {str(uuid.uuid4()): service for service in metadata['services']}
+                return tosca_gmetadata
+
+
     def _loadtoscatemplates(self):
         toscatemplates = []
         for path, subdirs, files in os.walk(self.tosca_dir):
@@ -81,10 +88,12 @@ def _extractalltoscainfo(self, tosca_templates):
             with io.open(self.tosca_dir + tosca) as stream:
                 template = yaml.full_load(stream)
                 tosca_info[tosca] = self.extracttoscainfo(template, tosca)
+                #info = self.extracttoscainfo(template, tosca)
+                #tosca_info[info.get('id')] = info
         return tosca_info
 
     def extracttoscainfo(self, template, tosca):
-
+        
         tosca_info = {
             "valid": True,
             "description": "TOSCA Template",

config/schemas/metadata_schema.json

@@ -0,0 +1,113 @@
+{
+  "METADATA_SCHEMA": {
+    "$schema": "http://json-schema.org/draft/2020-12/schema#",
+    "type": "object",
+    "properties" : {
+        "services" : { 
+            "type": "array",
+            "items": {
+                "$ref": "#/$defs/service"
+            },
+            "minItems": 1,
+            "uniqueItems": true
+        }
+    },
+    "required": ["services"],
+    "additionalProperties": false,
+    "$defs": {
+        "service": { 
+            "type": "object",
+            "properties": {
+                "description": { "type": "string" },
+                "metadata": { "$ref": "#/$defs/metadata" },
+                "templates": {
+                    "type": "array",
+                    "items": { "$ref": "#/$defs/template" }
+                }
+            },
+            "required": ["description", "metadata", "templates"],
+            "additionalProperties": false
+        },
+        "metadata": {
+            "type": "object",
+            "properties": {
+                "display_name": { "type": "string" },
+                "icon": { "type": "string", "format": "uri" },
+                "visibility": {
+                    "$ref": "#/$defs/visibility"
+                },
+                "authorization_required" : {"$ref": "#/$defs/authorization_required" }
+            },
+            "required": ["display_name", "icon", "visibility"],
+            "additionalProperties": false
+        },
+        "visibility": {
+            "type": "object",
+            "oneOf": [
+              {
+                  "type": "object",
+                  "properties": {
+                    "type": { "type": "string" },
+                    "groups": { "type": "array", "items": { "type": "string" } }
+                  },
+                  "required": ["type", "groups"],
+                  "additionalProperties": false
+              },
+              {
+                "type": "object",
+                "properties": {
+                  "type": { "type": "string" },
+                  "groups_regex": { "type": "string", "format": "regex" }
+                },
+                "required": ["type", "groups_regex"],
+                "additionalProperties": false
+              },
+              {
+                "type": "object",
+                "properties": {
+                  "type": { "const": "public" }
+                },
+                "required": ["type"]
+              }
+            ]  
+        },
+        "template": {
+            "type": "object",
+            "properties": {
+                "name": { "type": "string" },
+                "option": { "type": "string" },
+                "description": { "type": "string" }
+            },
+            "additionalProperties": false,
+            "if":
+            {
+                "type": "array",
+                "minItems": 2
+            },
+            "then": {
+                "required": ["name", "option"]
+            },
+            "else": {
+                "required": ["name"]
+            }
+        },
+        "authorization_required": {
+          "type": "object",
+          "properties": {
+            "pre_tasks": { 
+              "type": "array", 
+              "items": { 
+                "type": "object",
+                "properties": {
+                  "action": { "type": "string" },
+                  "args": { "type": "object" }
+                },
+                "additionalProperties": false
+              } 
+            }
+          }
+        }
+    }
+  }
+}
+

requirements.txt

@@ -27,3 +27,4 @@ boto3==1.21.27
 python-ldap==3.4.0
 randomcolor
 sshpubkeys
+jsonschema[format]