Feature: Adds ImportDefinition and handling imports recursively

TODO.md

@@ -10,13 +10,9 @@
 ### ``parser.go``
 (line 84) (kenjones): Add hooks as method parameter
 
-(line 139) (kenjones): Does dropping the Imports list really have any impact?
-
 
 ### ``service_template.go``
-(line 21) (kenjones): Implement ImportDefinition as it is not always going to be a simple
-
-(line 187) (kenjones): assume the requirement has a node specified, otherwise need to use the
+(line 184) (kenjones): assume the requirement has a node specified, otherwise need to use the
 
 
 ### ``topology.go``
@@ -38,9 +34,9 @@
 
 
 ### ``tosca_reusable_modeling_definitions.go``
-(line 19) : Implement ArtifactDefinition struct
+(line 21) : Implement ArtifactDefinition struct
 
-(line 22) : Implement NodeFilter struct
+(line 26) : Implement ArtifactType struct
 
-(line 70) : Implement ArtifactType struct
+(line 29) : Implement NodeFilter struct
 

parser.go

@@ -84,6 +84,35 @@ func (t *ServiceTemplateDefinition) ParseCsar(zipfile string) error {
 	}, ParserHooks{ParsedSTD: noop}) // TODO(kenjones): Add hooks as method parameter
 }
 
+func parseImports(impDefs []ImportDefinition, resolver Resolver, hooks ParserHooks) (ServiceTemplateDefinition, error) {
+	var std ServiceTemplateDefinition
+
+	for _, im := range impDefs {
+		r, err := resolver(im.File)
+		if err != nil {
+			return std, err
+		}
+
+		var tt ServiceTemplateDefinition
+		err = yaml.Unmarshal(r, &tt)
+		if err != nil {
+			return std, err
+		}
+		err = hooks.ParsedSTD(im.File, &tt)
+		if err != nil {
+			return std, err
+		}
+
+		if len(tt.Imports) != 0 {
+			tt, err = parseImports(tt.Imports, resolver, hooks)
+		}
+
+		std = std.Merge(tt)
+	}
+
+	return std, nil
+}
+
 func (t *ServiceTemplateDefinition) parse(data []byte, resolver Resolver, hooks ParserHooks) error {
 	var std ServiceTemplateDefinition
 	// Unmarshal the data in an interface
@@ -117,27 +146,13 @@ func (t *ServiceTemplateDefinition) parse(data []byte, resolver Resolver, hooks
 		std = std.Merge(tt)
 	}
 
-	// Load all referenced Imports
-	for _, im := range std.Imports {
-		r, err := resolver(im)
-		if err != nil {
-			return err
-		}
-
-		var tt ServiceTemplateDefinition
-		err = yaml.Unmarshal(r, &tt)
-		if err != nil {
-			return err
-		}
-		err = hooks.ParsedSTD(im, &tt)
-		if err != nil {
-			return err
-		}
-		std = std.Merge(tt)
+	// Load all referenced Imports (recursively)
+	var tt ServiceTemplateDefinition
+	tt, err = parseImports(std.Imports, resolver, hooks)
+	if err != nil {
+		return err
 	}
-	// Free the imports
-	// TODO(kenjones): Does dropping the Imports list really have any impact?
-	std.Imports = []string{}
+	std = std.Merge(tt)
 
 	// update the initial context with the freshly loaded context
 	*t = std

service_template.go

@@ -18,9 +18,6 @@ package toscalib
 
 import "github.com/kenjones-cisco/mergo"
 
-// TODO(kenjones): Implement ImportDefinition as it is not always going to be a simple
-// list of strings.
-
 // ServiceTemplateDefinition is the meta structure containing an entire tosca document as described in
 // http://docs.oasis-open.org/tosca/TOSCA-Simple-Profile-YAML/v1.0/csd03/TOSCA-Simple-Profile-YAML-v1.0-csd03.html
 type ServiceTemplateDefinition struct {
@@ -29,7 +26,7 @@ type ServiceTemplateDefinition struct {
 	Description        string                          `yaml:"description,omitempty" json:"description,omitempty"`
 	DslDefinitions     interface{}                     `yaml:"dsl_definitions,omitempty" json:"dsl_definitions,omitempty"`       // Declares optional DSL-specific definitions and conventions.  For example, in YAML, this allows defining reusable YAML macros (i.e., YAML alias anchors) for use throughout the TOSCA Service Template.
 	Repositories       map[string]RepositoryDefinition `yaml:"repositories,omitempty" json:"repositories,omitempty"`             // Declares the list of external repositories which contain artifacts that are referenced in the service template along with their addresses and necessary credential information used to connect to them in order to retrieve the artifacts.
-	Imports            []string                        `yaml:"imports,omitempty" json:"imports,omitempty"`                       // Declares import statements external TOSCA Definitions documents. For example, these may be file location or URIs relative to the service template file within the same TOSCA CSAR file.
+	Imports            []ImportDefinition              `yaml:"imports,omitempty" json:"imports,omitempty"`                       // Declares import statements external TOSCA Definitions documents. For example, these may be file location or URIs relative to the service template file within the same TOSCA CSAR file.
 	ArtifactTypes      map[string]ArtifactType         `yaml:"artifact_types,omitempty" json:"artifact_types,omitempty"`         // This section contains an optional list of artifact type definitions for use in service templates
 	DataTypes          map[string]DataType             `yaml:"data_types,omitempty" json:"data_types,omitempty"`                 // Declares a list of optional TOSCA Data Type definitions.
 	CapabilityTypes    map[string]CapabilityType       `yaml:"capability_types,omitempty" json:"capability_types,omitempty"`     // This section contains an optional list of capability type definitions for use in service templates.

service_template_test.go

@@ -234,11 +234,38 @@ func TestParseVerifyPropertyExpression(t *testing.T) {
 	}
 }
 
+func TestParseBadImportsSimple(t *testing.T) {
+	fname := "./tests/invalids/test_bad_import_format.yaml"
+	var s ServiceTemplateDefinition
+	o, err := os.Open(fname)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = s.Parse(o)
+	if err == nil {
+		t.Log(fname, "has bad imports but it did not error out")
+		t.Fail()
+	}
+}
+
+func TestParseBadImportsComplex(t *testing.T) {
+	fname := "./tests/invalids/test_bad_import_format_defs.yaml"
+	var s ServiceTemplateDefinition
+	o, err := os.Open(fname)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = s.Parse(o)
+	if err == nil {
+		t.Log(fname, "has bad imports but it did not error out")
+		t.Fail()
+	}
+}
+
 func TestParseCsar(t *testing.T) {
 
 	testsko := []string{
 		"tests/csar_metadata_not_yaml.zip",
-		"tests/csar_wordpress_invalid_import_path.zip",
 		"tests/csar_wrong_metadata_file.zip",
 		"tests/csar_not_zip.zip",
 	}

tests/invalids/test_bad_import_format.yaml

@@ -0,0 +1,9 @@
+tosca_definitions_version: tosca_simple_yaml_1_0
+
+description: Imports another template that then imports other templates. (recursion)
+
+imports:
+  - other_import: tests/example1.yaml
+    myother_import: tests/example2.yaml
+
+# that is all

tests/invalids/test_bad_import_format_defs.yaml

@@ -0,0 +1,11 @@
+tosca_definitions_version: tosca_simple_yaml_1_0
+
+description: Imports another template that then imports other templates. (recursion)
+
+imports:
+  - complex_import:
+      file: tests/example2.yaml
+    mycomplex_import:
+      file: example3.yaml
+
+# that is all

tests/test_template_with_nested_imports.yaml

@@ -0,0 +1,13 @@
+tosca_definitions_version: tosca_simple_yaml_1_0
+
+description: Imports another template that then imports other templates. (recursion)
+
+imports:
+  - tests/test_host_assignment.yaml
+  - other_import: tests/example1.yaml
+  - complex_import:
+      file: tests/example2.yaml
+  - file: tests/example3.yaml
+    repository: example3
+
+# that is all

tests/tosca_container_nodes.yaml

@@ -5,7 +5,8 @@ description: >
 
 # Repositories to retrieve code artifacts from
 repositories:
-  docker_hub: https://registry.hub.docker.com/
+  docker_hub:
+    url: https://registry.hub.docker.com/
 
 topology_template:
 

tosca_reusable_modeling_definitions.go

@@ -16,9 +16,16 @@ limitations under the License.
 
 package toscalib
 
+import "fmt"
+
 // ArtifactDefinition Appendix 5.5 TODO: Implement ArtifactDefinition struct
 type ArtifactDefinition map[string]interface{}
 
+// ArtifactType as described in appendix 6.3
+// An Artifact Type is a reusable entity that defines the type of one or more files which Node Types or Node Templates can have dependent relationships and used during operations such as during installation or deployment.
+// TODO: Implement ArtifactType struct
+type ArtifactType interface{}
+
 // NodeFilter Appendix 5.4 TODO: Implement NodeFilter struct
 // A node filter definition defines criteria for selection of a TOSCA Node Template based upon the template’s property values, capabilities and capability properties.
 type NodeFilter interface{}
@@ -65,11 +72,74 @@ func (r *RepositoryDefinition) UnmarshalYAML(unmarshal func(interface{}) error)
 	return nil
 }
 
-// ArtifactType as described in appendix 6.3
-// An Artifact Type is a reusable entity that defines the type of one or more files which Node Types or Node Templates can have dependent relationships and used during operations such as during installation or deployment.
-// TODO: Implement ArtifactType struct
-type ArtifactType interface{}
-
 // Metadata is provides support for attaching provider specific attributes
 // to different structures.
 type Metadata map[string]string
+
+// ImportDefinition is used within a TOSCA Service Template to locate and uniquely name
+// another TOSCA Service Template file which has type and template definitions to be
+// imported (included) and referenced within another Service Template.
+type ImportDefinition struct {
+	File            string `yaml:"file" json:"file"`
+	Repository      string `yaml:"repository,omitempty" json:"repository,omitempty"`
+	NamespaceURI    string `yaml:"namespace_uri,omitempty" json:"namespace_uri,omitempty"`
+	NamespacePrefix string `yaml:"namespace_prefix,omitempty" json:"namespace_prefix,omitempty"`
+}
+
+// UnmarshalYAML is used to match both Simple Notation Example and Full Notation Example
+func (i *ImportDefinition) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	// First try the Short notation
+	var u string
+	err := unmarshal(&u)
+	if err == nil {
+		i.File = u
+		return nil
+	}
+
+	// if not a string then try full notation
+	var full struct {
+		File            string `yaml:"file" json:"file"`
+		Repository      string `yaml:"repository,omitempty" json:"repository,omitempty"`
+		NamespaceURI    string `yaml:"namespace_uri,omitempty" json:"namespace_uri,omitempty"`
+		NamespacePrefix string `yaml:"namespace_prefix,omitempty" json:"namespace_prefix,omitempty"`
+	}
+	err = unmarshal(&full)
+	if err == nil && full.File != "" {
+		i.File = full.File
+		i.Repository = full.Repository
+		i.NamespaceURI = full.NamespaceURI
+		i.NamespacePrefix = full.NamespacePrefix
+		return nil
+	}
+
+	// the spec indicates the import can be a simple list of strings or a list of maps with specific keywords
+	// but then in the examples there is a named file pattern;
+	var named map[string]string
+	err = unmarshal(&named)
+	if err == nil {
+		if len(named) != 1 {
+			return fmt.Errorf("Named imports file had multiple unrecognized keys: %v", named)
+		}
+		for _, v := range named {
+			i.File = v
+			return nil
+		}
+	}
+
+	var namedFull map[string]ImportDefinition
+	err = unmarshal(&namedFull)
+	if err == nil {
+		if len(namedFull) != 1 {
+			return fmt.Errorf("Named imports file had multiple unrecognized keys: %v", namedFull)
+		}
+		for _, v := range namedFull {
+			i.File = v.File
+			i.Repository = v.Repository
+			i.NamespaceURI = v.NamespaceURI
+			i.NamespacePrefix = v.NamespacePrefix
+			return nil
+		}
+	}
+
+	return err
+}