Merge pull request #24 from lightclient/better-speccheck-err

Better speccheck errors

cmd/speccheck/check.go

@@ -0,0 +1,80 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"regexp"
+	"strings"
+
+	openrpc "github.com/open-rpc/meta-schema"
+	"github.com/santhosh-tekuri/jsonschema/v5"
+)
+
+// checkSpec reads the schemas from the spec and test files, then validates
+// them against each other.
+func checkSpec(methods map[string]*methodSchema, rts []*roundTrip, re *regexp.Regexp) error {
+	for _, rt := range rts {
+		method, ok := methods[rt.method]
+		if !ok {
+			return fmt.Errorf("undefined method: %s", rt.method)
+		}
+		// skip validator of test if name includes "invalid" as the schema
+		// doesn't yet support it.
+		// TODO(matt): create error schemas.
+		if strings.Contains(rt.name, "invalid") {
+			continue
+		}
+		if len(method.params) < len(rt.params) {
+			return fmt.Errorf("%s: too many parameters", method.name)
+		}
+		// Validate each parameter value against their respective schema.
+		for i, cd := range method.params {
+			if len(rt.params) <= i {
+				if !cd.required {
+					// skip missing optional values
+					continue
+				}
+				return fmt.Errorf("missing required parameter %s.param[%d]", rt.method, i)
+			}
+			if err := validate(&method.params[i].schema, rt.params[i], fmt.Sprintf("%s.param[%d]", rt.method, i)); err != nil {
+				return fmt.Errorf("unable to validate parameter: %s", err)
+			}
+		}
+		if err := validate(&method.result.schema, rt.response, fmt.Sprintf("%s.result", rt.method)); err != nil {
+			// Print out the value and schema if there is an error to further debug.
+			buf, _ := json.Marshal(method.result.schema)
+			fmt.Println(string(buf))
+			fmt.Println(string(rt.response))
+			fmt.Println()
+			return fmt.Errorf("invalid result %s\n%#v", rt.name, err)
+		}
+	}
+
+	fmt.Println("all passing.")
+	return nil
+}
+
+// validateParam validates the provided value against schema using the url base.
+func validate(schema *openrpc.JSONSchemaObject, val []byte, url string) error {
+	// Set $schema explicitly to force jsonschema to use draft 2019-09.
+	draft := openrpc.Schema("https://json-schema.org/draft/2019-09/schema")
+	schema.Schema = &draft
+
+	// Compile schema.
+	b, err := json.Marshal(schema)
+	if err != nil {
+		return fmt.Errorf("unable to marshal schema to json")
+	}
+	s, err := jsonschema.CompileString(url, string(b))
+	if err != nil {
+		return err
+	}
+
+	// Validate value
+	var x interface{}
+	json.Unmarshal(val, &x)
+	if err := s.Validate(x); err != nil {
+		return err
+	}
+	return nil
+}

cmd/speccheck/main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"fmt"
 	"os"
+	"regexp"
 
 	"github.com/alexflint/go-arg"
 )
@@ -17,11 +18,32 @@ type Args struct {
 func main() {
 	var args Args
 	arg.MustParse(&args)
-	if err := checkSpec(&args); err != nil {
+	if err := run(&args); err != nil {
 		exit(err)
 	}
 }
 
+func run(args *Args) error {
+	re, err := regexp.Compile(args.TestsRegex)
+	if err != nil {
+		return err
+	}
+
+	// Read all method schemas (params+result) from the OpenRPC spec.
+	methods, err := parseSpec(args.SpecPath)
+	if err != nil {
+		return err
+	}
+
+	// Read all tests and parse out roundtrip HTTP exchanges so they can be validated.
+	rts, err := readRtts(args.TestsRoot, re)
+	if err != nil {
+		return err
+	}
+
+	return checkSpec(methods, rts, re)
+}
+
 func exit(err error) {
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "%s\n", err)

cmd/speccheck/parse.go → cmd/speccheck/roundtrips.go

@@ -7,8 +7,6 @@ import (
 	"path/filepath"
 	"regexp"
 	"strings"
-
-	openrpc "github.com/open-rpc/meta-schema"
 )
 
 type jsonrpcMessage struct {
@@ -26,9 +24,18 @@ type jsonError struct {
 	Data    interface{} `json:"data,omitempty"`
 }
 
-// parseRoundTrips walks a root directory and parses round trip HTTP exchanges
+// roundTrip is a single round trip interaction between a certain JSON-RPC
+// method.
+type roundTrip struct {
+	method   string
+	name     string
+	params   [][]byte
+	response []byte
+}
+
+// readRtts walks a root directory and parses round trip HTTP exchanges
 // from files that match the regular expression.
-func parseRoundTrips(root string, re *regexp.Regexp) ([]*roundTrip, error) {
+func readRtts(root string, re *regexp.Regexp) ([]*roundTrip, error) {
 	rts := make([]*roundTrip, 0)
 	err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
@@ -47,7 +54,7 @@ func parseRoundTrips(root string, re *regexp.Regexp) ([]*roundTrip, error) {
 			return nil // skip
 		}
 		// Found a good test, parse it and append to list.
-		test, err := parseTest(pathname, path)
+		test, err := readTest(pathname, path)
 		if err != nil {
 			return err
 		}
@@ -60,8 +67,8 @@ func parseRoundTrips(root string, re *regexp.Regexp) ([]*roundTrip, error) {
 	return rts, nil
 }
 
-// parseTest parses a single test into a slice of HTTP round trips.
-func parseTest(testname string, filename string) ([]*roundTrip, error) {
+// readTest reads a single test into a slice of HTTP round trips.
+func readTest(testname string, filename string) ([]*roundTrip, error) {
 	data, err := os.ReadFile(filename)
 	if err != nil {
 		return nil, err
@@ -103,63 +110,3 @@ func parseTest(testname string, filename string) ([]*roundTrip, error) {
 	}
 	return rts, nil
 }
-
-// parseParamValues parses each parameter out of the raw json value in its own byte
-// slice.
-func parseParamValues(raw json.RawMessage) ([][]byte, error) {
-	if len(raw) == 0 {
-		return [][]byte{}, nil
-	}
-	var params []interface{}
-	if err := json.Unmarshal(raw, &params); err != nil {
-		return nil, err
-	}
-	// Iterate over top-level parameter values and re-marshal them to get a
-	// list of json-encoded parameter values.
-	var out [][]byte
-	for _, param := range params {
-		buf, err := json.Marshal(param)
-		if err != nil {
-			return nil, err
-		}
-		out = append(out, buf)
-	}
-	return out, nil
-}
-
-// parseMethodSchemas reads an OpenRPC specification and parses out each
-// method's schemas.
-func parseMethodSchemas(filename string) (map[string]*methodSchema, error) {
-	spec, err := os.ReadFile(filename)
-	if err != nil {
-		return nil, err
-	}
-	var doc openrpc.OpenrpcDocument
-	if err := json.Unmarshal(spec, &doc); err != nil {
-		return nil, err
-	}
-	// Iterate over each method in the OpenRPC spec and pull out the parameter
-	// schema and result schema.
-	parsed := make(map[string]*methodSchema)
-	for _, method := range *doc.Methods {
-		var schema methodSchema
-
-		// Read parameter schemas.
-		for _, param := range *method.MethodObject.Params {
-			if param.ReferenceObject != nil {
-				return nil, fmt.Errorf("parameter references not supported")
-			}
-			schema.params = append(schema.params, *param.ContentDescriptorObject)
-		}
-
-		// Read result schema.
-		buf, err := json.Marshal(method.MethodObject.Result.ContentDescriptorObject.Schema)
-		if err != nil {
-			return nil, err
-		}
-		schema.result = buf
-		parsed[string(*method.MethodObject.Name)] = &schema
-	}
-
-	return parsed, nil
-}