diff --git a/cpanfile b/cpanfile index 0f906bebc..8a074ed4b 100644 --- a/cpanfile +++ b/cpanfile @@ -15,7 +15,7 @@ requires 'Mail::Sendmail'; requires 'Try::Tiny'; requires 'Time::HiRes'; requires 'Time::Moment', '>= 0.43'; # for PR#28, fixes use of stdbool.h (thanks Dale) -requires 'JSON::Validator', '2.14'; +requires 'JSON::Validator', '3.04'; requires 'Data::Validate::IP'; # for json schema validation of 'ipv4', 'ipv6' types requires 'HTTP::Tiny'; requires 'Safe::Isa'; @@ -59,7 +59,6 @@ requires 'Devel::Confess'; requires 'Pod::Usage'; requires 'Pod::Markdown::Github'; requires 'Getopt::Long'; -requires 'Data::Visitor::Tiny'; # database and rendering requires 'DBD::Pg'; diff --git a/cpanfile.snapshot b/cpanfile.snapshot index 11db386f6..de0b325fc 100644 --- a/cpanfile.snapshot +++ b/cpanfile.snapshot @@ -870,17 +870,6 @@ DISTRIBUTIONS perl 5.008 strict 0 warnings 0 - Data-Visitor-Tiny-0.001 - pathname: D/DA/DAGOLDEN/Data-Visitor-Tiny-0.001.tar.gz - provides: - Data::Visitor::Tiny 0.001 - requirements: - Carp 0 - Exporter 5.57 - ExtUtils::MakeMaker 6.17 - perl 5.010 - strict 0 - warnings 0 DateTime-1.50 pathname: D/DR/DROLSKY/DateTime-1.50.tar.gz provides: @@ -1798,19 +1787,18 @@ DISTRIBUTIONS JSON::PP 2.27300 Scalar::Util 0 perl 5.006 - JSON-Validator-2.18 - pathname: J/JH/JHTHORSEN/JSON-Validator-2.18.tar.gz + JSON-Validator-3.06 + pathname: J/JH/JHTHORSEN/JSON-Validator-3.06.tar.gz provides: - JSON::Validator 2.18 + JSON::Validator 3.06 JSON::Validator::Error undef + JSON::Validator::Formats undef JSON::Validator::Joi undef - JSON::Validator::OpenAPI undef - JSON::Validator::OpenAPI::Dancer2 undef - JSON::Validator::OpenAPI::Mojolicious undef JSON::Validator::Ref undef requirements: ExtUtils::MakeMaker 0 Mojolicious 7.28 + perl 5.010001 Lingua-EN-FindNumber-1.32 pathname: N/NE/NEILB/Lingua-EN-FindNumber-1.32.tar.gz provides: diff --git a/json-schema/common.yaml b/json-schema/common.yaml new file mode 100644 index 000000000..7bc06e4f8 --- /dev/null +++ b/json-schema/common.yaml @@ -0,0 +1,34 @@ +--- +$schema: 'http://json-schema.org/draft-07/schema#' +definitions: + uuid: + type: string + pattern: "^[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}$" + ipaddr: + oneOf: + - type: string + format: ipv4 + - type: string + format: ipv6 + macaddr: + type: string + pattern: "^[0-9A-Fa-f]{2}:[0-9A-Fa-f]{2}:[0-9A-Fa-f]{2}:[0-9A-Fa-f]{2}:[0-9A-Fa-f]{2}:[0-9A-Fa-f]{2}$" + relay_id: + type: string + pattern: ^\S+$ + device_id: + type: string + pattern: ^\S+$ + device_asset_tag: + type: string + pattern: ^\S+$ + int_or_stringy_int: + description: an integer that may be presented as a json string + # note that when JSON::Validator has 'coerce' mode on, both of these rules will match. + oneOf: + - type: integer + - type: string + pattern: "^[0-9]+$" + non_empty_string: + type: string + minLength: 1 diff --git a/json-schema/device_report.yaml b/json-schema/device_report.yaml new file mode 100644 index 000000000..b2af05109 --- /dev/null +++ b/json-schema/device_report.yaml @@ -0,0 +1,154 @@ +--- +$schema: 'http://json-schema.org/draft-07/schema#' +definitions: + DeviceReport_v2.24: + description: the contents of a posted device report from relays and reporters + type: object + required: + - bios_version + - product_name + - serial_number + - state + - system_uuid + properties: + bios_version: + type: string + cpus: + type: array + items: + type: object + dimms: + type: array + uniqueItems: true + items: + type: object + required: + - memory-locator + properties: + memory-locator: + type: string + memory-serial-number: + type: string + memory-size: + $ref: common.yaml#/definitions/int_or_stringy_int + disks: + type: object + patternProperties: + ^\S+$: + description: device_disk.serial_number + type: object + properties: + slot: + $ref: common.yaml#/definitions/int_or_stringy_int + size: + type: integer + vendor: + type: string + model: + type: string + firmware: + type: string + transport: + type: string + health: + type: string # TODO: enum? + drive_type: + type: string + temp: + $ref: common.yaml#/definitions/int_or_stringy_int + enclosure: + $ref: common.yaml#/definitions/int_or_stringy_int + hba: + $ref: common.yaml#/definitions/int_or_stringy_int + # any additional fields are not currently used. + device_type: + type: string + enum: + - server + - switch + interfaces: + # TODO: this is required for servers + type: object + patternProperties: + ^\S+$: + description: key = interface name + type: object + required: + - mac + - product + - vendor + properties: + mac: + $ref: common.yaml#/definitions/macaddr + product: + type: string + vendor: + type: string + state: + oneOf: + - type: string + - type: 'null' + # note: no speed yet? + ipaddr: + oneOf: + - $ref: common.yaml#/definitions/ipaddr + - type: 'null' + mtu: + oneOf: + - $ref: common.yaml#/definitions/int_or_stringy_int + - type: 'null' + peer_mac: + oneOf: + - $ref: common.yaml#/definitions/macaddr + - type: 'null' + # peer_text, peer_switch, peer_port, all optional with no constraints + # peer_vendor: # TODO! see Conch::Validation::SwitchPeers. + # type: string + # pattern: ^\S+$ + media: + # TODO: this is required for switches + type: object + patternProperties: + ^\S$: + description: port + # type: unknown and not used. + os: + type: object + required: + - hostname + properties: + hostname: + type: string + product_name: + # TODO: required for switches, and also for non-switches when 'sku' is not present. + type: string + relay: + type: object + required: + - serial + properties: + serial: + $ref: common.yaml#/definitions/non_empty_string + serial_number: + $ref: common.yaml#/definitions/device_id + state: + type: string + system_uuid: + $ref: common.yaml#/definitions/uuid + temp: + type: object + required: + - cpu0 + - cpu1 + properties: + cpu0: + $ref: common.yaml#/definitions/int_or_stringy_int + cpu1: + $ref: common.yaml#/definitions/int_or_stringy_int + exhaust: + $ref: common.yaml#/definitions/int_or_stringy_int + inlet: + $ref: common.yaml#/definitions/int_or_stringy_int + uptime_since: + type: string + diff --git a/json-schema/input.yaml b/json-schema/input.yaml index 4fe7b6e88..78e2c3077 100644 --- a/json-schema/input.yaml +++ b/json-schema/input.yaml @@ -1,36 +1,22 @@ --- +$schema: 'http://json-schema.org/draft-07/schema#' definitions: non_empty_string: - type: string - minLength: 1 + $ref: common.yaml#/definitions/non_empty_string uuid: - type: string - pattern: "^[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}$" + $ref: common.yaml#/definitions/uuid relay_id: - type: string - pattern: ^\S+$ + $ref: common.yaml#/definitions/relay_id device_id: - type: string - pattern: ^\S+$ + $ref: common.yaml#/definitions/device_id device_asset_tag: - type: string - pattern: ^\S+$ + $ref: common.yaml#/definitions/device_asset_tag ipaddr: - oneOf: - - type: string - format: ipv4 - - type: string - format: ipv6 + $ref: common.yaml#/definitions/ipaddr macaddr: - type: string - pattern: "^[0-9A-Fa-f]{2}:[0-9A-Fa-f]{2}:[0-9A-Fa-f]{2}:[0-9A-Fa-f]{2}:[0-9A-Fa-f]{2}:[0-9A-Fa-f]{2}$" + $ref: common.yaml#/definitions/macaddr int_or_stringy_int: - description: an integer that may be presented as a json string - # note that when JSON::Validator has 'coerce' mode on, both of these rules will match. - oneOf: - - type: integer - - type: string - pattern: "^[0-9]+$" + $ref: common.yaml#/definitions/int_or_stringy_int DatacenterCreate: type: object additionalProperties: false @@ -87,153 +73,8 @@ definitions: type: string vendor_name: type: string - DeviceReport: - type: object - required: - - bios_version - - product_name - - serial_number - - state - - system_uuid - properties: - bios_version: - type: string - cpus: - type: array - items: - type: object - dimms: - type: array - uniqueItems: true - items: - type: object - required: - - memory-locator - properties: - memory-locator: - type: string - memory-serial-number: - type: string - memory-size: - $ref: /definitions/int_or_stringy_int - disks: - type: object - patternProperties: - ^\S+$: - description: device_disk.serial_number - type: object - properties: - slot: - $ref: /definitions/int_or_stringy_int - size: - type: integer - vendor: - type: string - model: - type: string - firmware: - type: string - transport: - type: string - health: - type: string # TODO: enum? - drive_type: - type: string - temp: - $ref: /definitions/int_or_stringy_int - enclosure: - $ref: /definitions/int_or_stringy_int - hba: - $ref: /definitions/int_or_stringy_int - # any additional fields are not currently used. - device_type: - type: string - enum: - - server - - switch - interfaces: - # TODO: this is required for servers - type: object - patternProperties: - ^\S+$: - description: interface name - type: object - required: - - mac - - product - - vendor - properties: - mac: - $ref: /definitions/macaddr - product: - type: string - vendor: - type: string - state: - oneOf: - - type: string - - type: 'null' - # note: no speed yet? - ipaddr: - oneOf: - - $ref: /definitions/ipaddr - - type: 'null' - mtu: - oneOf: - - $ref: /definitions/int_or_stringy_int - - type: 'null' - peer_mac: - oneOf: - - $ref: /definitions/macaddr - - type: 'null' - # peer_text, peer_switch, peer_port, all optional with no constraints - media: - # TODO: this is required for switches - type: object - patternProperties: - ^\S$: - description: port - # type: unknown and not used. - os: - type: object - required: - - hostname - properties: - hostname: - type: string - product_name: - # TODO: required for switches, and also for non-switches when 'sku' is not present. - type: string - relay: - type: object - required: - - serial - properties: - serial: - $ref: /definitions/non_empty_string - serial_number: - $ref: /definitions/device_id - state: - type: string - system_uuid: - $ref: /definitions/uuid - temp: - type: object - required: - - cpu0 - - cpu1 - properties: - cpu0: - $ref: /definitions/int_or_stringy_int - cpu1: - $ref: /definitions/int_or_stringy_int - exhaust: - $ref: /definitions/int_or_stringy_int - inlet: - $ref: /definitions/int_or_stringy_int - uptime_since: - type: string + $ref: device_report.yaml#/definitions/DeviceReport_v2.24 RackCreate: type: object additionalProperties: false diff --git a/json-schema/response.yaml b/json-schema/response.yaml index 1fd4cecdb..a8cc91018 100644 --- a/json-schema/response.yaml +++ b/json-schema/response.yaml @@ -1,26 +1,18 @@ --- +$schema: 'http://json-schema.org/draft-07/schema#' definitions: uuid: - type: string - pattern: "^[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}$" + $ref: common.yaml#/definitions/uuid relay_id: - type: string - pattern: ^\S+$ + $ref: common.yaml#/definitions/relay_id device_id: - type: string - pattern: ^\S+$ + $ref: common.yaml#/definitions/device_id device_asset_tag: - type: string - pattern: ^\S+$ + $ref: common.yaml#/definitions/device_asset_tag ipaddr: - oneOf: - - type: string - format: ipv4 - - type: string - format: ipv6 + $ref: common.yaml#/definitions/ipaddr macaddr: - type: string - pattern: "^[0-9A-Fa-f]{2}:[0-9A-Fa-f]{2}:[0-9A-Fa-f]{2}:[0-9A-Fa-f]{2}:[0-9A-Fa-f]{2}:[0-9A-Fa-f]{2}$" + $ref: common.yaml#/definitions/macaddr validation_status: type: string enum: @@ -256,9 +248,11 @@ definitions: latest_report_is_invalid: type: boolean latest_report: - oneOf: - - $ref: /definitions/DeviceReport + description: the contents of the device report. Given its age we cannot provide a schema. + anyOf: - type: 'null' + - $ref: device_report.yaml#/definitions/DeviceReport_v2.24 + - type: object # TODO: remove this loose alternative when we purge historical db records invalid_report: description: this could be anything, from encoded json to random junk oneOf: @@ -550,39 +544,6 @@ definitions: properties: mac: $ref: /definitions/macaddr - DeviceReport: - type: object - required: - - product_name - - serial_number - - system_uuid - - state - - bios_version - - os - - processor - - memory - properties: - product_name: - type: string - serial_number: - $ref: /definitions/device_id - system_uuid: - $ref: /definitions/uuid - state: - type: string - bios_version: - type: string - os: - type: object - required: - - hostname - properties: - hostname: - type: string - processor: - type: object - memory: - type: object DeviceLocation: type: object additionalProperties: false diff --git a/lib/Conch/Controller/Schema.pm b/lib/Conch/Controller/Schema.pm index 16bf8275a..07bb9c9ba 100644 --- a/lib/Conch/Controller/Schema.pm +++ b/lib/Conch/Controller/Schema.pm @@ -2,7 +2,6 @@ package Conch::Controller::Schema; use Mojo::Base 'Mojolicious::Controller', -signatures; -use Data::Visitor::Tiny qw(visit); use Mojo::Util qw(camelize); =pod @@ -20,33 +19,88 @@ Get the json-schema in JSON format. =cut sub get ($c) { - my $type = lc $c->stash('request_or_response'); + my $type = $c->stash('request_or_response'); my $name = camelize $c->stash('name'); my $validator = $type eq 'response' ? $c->get_response_validator : $type eq 'request' ? $c->get_input_validator : undef; - return $c->status(404) if not $validator; + return $c->status(400, { error => 'Cannot find validator' }) if not $validator; - my $schema = $validator->get("/definitions/$name"); + my $schema = _extract_schema_definition($validator, $name); return $c->status(404) if not $schema; - my sub inline_ref ( $ref, $schema ) { - my ($other) = $ref =~ m|#?/definitions/(\w+)$|; - $schema->{definitions}{$other} = $validator->get($ref); - } + return $c->status(200, $schema); +} + +=head2 _extract_schema_definition + +Given a JSON::Validator object containing a schema definition, extract the requested portion +out of the "definitions" section, including any named references, and add some standard +headers. - visit $schema => sub ( $key, $ref, @ ) { - inline_ref( $_ => $schema ) if $key eq '$ref'; - if ( !defined $_ && $key eq "type" ) { - $$ref = "null"; +=cut + +sub _extract_schema_definition ($validator, $schema_name) { + my $top_schema = $validator->schema->get('/definitions/'.$schema_name); + return if not $top_schema; + + my %refs; + my %source; + my $definitions; + my @topics = ([ { schema => $top_schema }, my $target = {}]); + my $cloner = sub ($from) { + if (ref $from eq 'HASH' and my $tied = tied %$from) { + # this is a hashref which quacks like { '$ref' => $target } + my ($location, $path) = split /#/, $tied->fqn, 2; + (my $name = $path) =~ s!^/definitions/!!; + + if (not $refs{$tied->fqn}++) { + if ($name ne $schema_name and exists $source{$name}) { + die 'namespace collision: '.$tied->fqn.' but already have a /definitions/'.$name + .' from '.$source{$name}->fqn; + } + + $source{$name} = $tied; + push @topics, [$tied->schema, $definitions->{$name} = {}]; + } + + ++$refs{'/traversed_definitions/'.$name}; + tie my %ref, 'JSON::Validator::Ref', $tied->schema, '/definitions/'.$name; + return \%ref; } + + my $to = ref $from eq 'ARRAY' ? [] : ref $from eq 'HASH' ? {} : $from; + push @topics, [$from, $to] if ref $from; + return $to; }; - $schema->{title} //= $name; - $schema->{'$schema'} = 'http://json-schema.org/draft-07/schema#'; - $schema->{'$id'} = "urn:$name.schema.json"; - return $c->status( 200, $schema ); + while (@topics) { + my ($from, $to) = @{shift @topics}; + if (ref $from eq 'ARRAY') { + push @$to, $cloner->($_) foreach @$from; + } + elsif (ref $from eq 'HASH') { + $to->{$_} = $cloner->($from->{$_}) foreach keys %$from; + } + } + + $target = $target->{schema}; + + # cannot return a $ref at the top level (sibling keys disallowed) - inline the $ref. + while (my $tied = tied %$target) { + (my $name = $tied->fqn) =~ s!^/definitions/!!; + $target = $definitions->{$name}; + delete $definitions->{$name} if $refs{'/traversed_definitions/'.$name} == 1; + } + + return { + title => $schema_name, + '$schema' => $validator->get('/$schema') || 'http://json-schema.org/draft-07/schema#', + '$id' => 'urn:'.$schema_name.'.schema.json', + keys $definitions->%* ? ( definitions => $definitions ) : (), + $target->%*, + }; } 1; diff --git a/lib/Conch/Plugin/JsonValidator.pm b/lib/Conch/Plugin/JsonValidator.pm index 6db2df082..7398f7d48 100644 --- a/lib/Conch/Plugin/JsonValidator.pm +++ b/lib/Conch/Plugin/JsonValidator.pm @@ -61,9 +61,7 @@ response if validation failed; returns validated input on success. =cut $app->helper(validate_input => sub ($c, $schema_name, $input = $c->req->json) { - my $validator = JSON::Validator->new; - $validator->schema(INPUT_SCHEMA_FILE); - + my $validator = $c->get_input_validator; my $schema = $validator->get('/definitions/'.$schema_name); if (not $schema) { @@ -87,10 +85,13 @@ Returns a JSON::Validator object suitable for validating an endpoint input. =cut + my $_input_validator; $app->helper(get_input_validator => sub ($c) { - my $validator = JSON::Validator->new; - $validator->schema(INPUT_SCHEMA_FILE); - return $validator; + return $_input_validator if $_input_validator; + $_input_validator = JSON::Validator->new; + # FIXME: JSON::Validator should be picking this up out of the schema on its own. + $_input_validator->load_and_validate_schema(INPUT_SCHEMA_FILE, { schema => 'http://json-schema.org/draft-07/schema#' }); + return $_input_validator; }); @@ -100,10 +101,13 @@ Returns a JSON::Validator object suitable for validating an endpoint response. =cut + my $_response_validator; $app->helper(get_response_validator => sub ($c) { - my $validator = JSON::Validator->new; - $validator->schema(OUTPUT_SCHEMA_FILE); - return $validator; + return $_response_validator if $_response_validator; + my $_response_validator = JSON::Validator->new; + # FIXME: JSON::Validator should be picking this up out of the schema on its own. + $_response_validator->load_and_validate_schema(OUTPUT_SCHEMA_FILE, { schema => 'http://json-schema.org/draft-07/schema#' }); + return $_response_validator; }); } diff --git a/lib/Test/Conch.pm b/lib/Test/Conch.pm index d8c32019b..1ec6301e2 100644 --- a/lib/Test/Conch.pm +++ b/lib/Test/Conch.pm @@ -7,7 +7,6 @@ use Test::More (); use Test::PostgreSQL; use Conch::DB; use Test::Conch::Fixtures; -use JSON::Validator; use Path::Tiny; use Test::Deep (); use Mojo::Util 'trim'; @@ -53,15 +52,8 @@ has 'pg'; # Test::PostgreSQL object =cut -has 'validator' => sub { - my $spec_file = "json-schema/response.yaml"; - die("OpenAPI spec file '$spec_file' doesn't exist.") - unless -e $spec_file; - - my $validator = JSON::Validator->new; - $validator->schema($spec_file); - - $validator; +has validator => sub ($self) { + $self->app->get_response_validator; }; =head2 fixtures diff --git a/misc/extract-schema b/misc/extract-schema index 222c80553..c7f22e160 100755 --- a/misc/extract-schema +++ b/misc/extract-schema @@ -5,71 +5,74 @@ use experimental 'signatures'; use Getopt::Long; use Pod::Usage; use YAML::XS qw(LoadFile); -use Mojo::JSON qw(encode_json); +use JSON::MaybeXS (); use Mojo::File qw(path); -use Data::Visitor::Tiny qw(visit); +use JSON::Validator; + use Dir::Self; +use lib __DIR__ =~ s{/misc}{}r . '/lib'; + +use Conch::Controller::Schema; my $schema_file = __DIR__.'/../json-schema/input.yaml'; GetOptions( - 'file|f:s' => \$schema_file, - 'help|h' => \my $help, + 'file|f:s' => \$schema_file, + 'help|h' => \my $help, 'output|to:s' => \my $to, ); pod2usage(1) if $help; -my %schemas = LoadFile($schema_file)->{definitions}->%*; - -sub inline_ref ( $ref, $schema ) { - - # #/definitions/ValidationStateWithResults - my ($other) = $ref =~ m|#?/definitions/(\w+)$|; - $schema->{definitions}{$other} = $schemas{$other}; -} - -sub output_json_schema ( $name, $schema ) { +sub output_json_schema ($name, $schema) { + my $json = JSON::MaybeXS->new(pretty => 1, canonical => 1); if ($to) { - path("$to/$name.schema.json")->spurt( encode_json($schema) ); + path($to)->make_path; + path("$to/$name.schema.json")->spurt($json->encode($schema)); } else { say "$name.schema.json"; - say encode_json($schema); + say $json->encode($schema); say; } } -for my $name ( keys %schemas ) { - my $schema = $schemas{$name}; - visit $schema => sub ( $key, $ref, @ ) { - inline_ref( $_ => $schema ) if $key eq '$ref'; - if ( !defined $_ && $key eq "type" ) { - $$ref = "null"; - } - }; - $schema->{title} //= $name; - $schema->{'$schema'} = 'http://json-schema.org/draft-07/schema#'; - $schema->{'$id'} = "urn:$name.schema.json"; - - output_json_schema $name, $schema; +my $validator = JSON::Validator->new; +# TODO: do not pass 'schema' arg - just depend on JV +$validator->load_and_validate_schema($schema_file, { schema => 'http://json-schema.org/draft-07/schema#' }); + +for my $schema_name (sort keys $validator->schema->data->{definitions}->%*) { + my $schema = Conch::Controller::Schema::_extract_schema_definition($validator, $schema_name); + output_json_schema($schema_name, $schema); } __END__ =head1 NAME -extract-schema - extracts an embedded JSON schema from a combined (YAML) schema +extract-schema - extracts embedded JSON schemas from a combined (YAML) schema =head1 SYNOPSIS - extract-schema [-f FILE] [-h] + extract-schema [-f FILE] [-o DIR] [-h] + +=head1 DESCRIPTION + +Given a single YAML file containing a number of JSON schema definitions, creates a separate +C<$name.schema.json> file for each definition, conforming to the same JSON specification +as the original file. =head1 OPTIONS =over 4 -=item B<-f FILE> +=item B<--file|-f FILE> + +The base file for extracting from; defaulting to the C in this +repository. + +=item B<--output|--to DIR> -Provide a base file for extracting from, defaults to the C in this repository. +The directory in which to create the json files. If not provided, all content is emitted to +STDOUT, preceded by each definition's filename. =item B<-h> diff --git a/t/data/test-schema.yaml b/t/data/test-schema.yaml new file mode 100644 index 000000000..01060cdb9 --- /dev/null +++ b/t/data/test-schema.yaml @@ -0,0 +1,75 @@ +--- +$schema: http://json-schema.org/draft-07/schema# +definitions: + ref1: + type: array + items: + $ref: /definitions/ref2 + ref2: + type: string + minLength: 1 + ref3: + type: integer + dupe_name: + type: integer + i_have_nested_refs: + type: object + properties: + my_key1: + $ref: /definitions/ref1 + my_key2: + $ref: /definitions/ref1 + # actually a person, as in https://json-schema.org/understanding-json-schema/structuring.html + i_have_a_recursive_ref: + type: object + properties: + name: + type: string + children: + type: array + items: + $ref: /definitions/i_have_a_recursive_ref + default: [] + i_have_a_ref_to_another_file: + type: object + properties: + name: + $ref: test-schema2.yaml#/definitions/my_name + address: + $ref: test-schema2.yaml#/definitions/my_address + secrets: + $ref: /definitions/ref1 + i_am_a_ref: + $ref: /definitions/ref1 + i_am_a_ref_level_1: + $ref: /definitions/i_am_a_ref_level_2 + i_am_a_ref_level_2: + $ref: /definitions/ref3 + i_am_a_ref_to_another_file: + $ref: test-schema2.yaml#/definitions/i_have_a_ref_to_the_first_filename + i_am_a_ref_with_the_same_name: + $ref: test-schema2.yaml#/definitions/i_am_a_ref_with_the_same_name + i_have_refs_with_the_same_name: + type: object + properties: + me: + $ref: /definitions/i_am_a_ref_with_the_same_name + i_contain_refs_to_same_named_definitions: + type: object + properties: + foo: + $ref: /definitions/dupe_name + bar: + $ref: test-schema2.yaml#/definitions/dupe_name + i_have_a_ref_with_the_same_name: + type: object + properties: + name: + type: string + children: + type: array + items: + $ref: test-schema2.yaml#/definitions/i_have_a_ref_with_the_same_name + default: [] + +# vim: set sts=2 sw=2 et : diff --git a/t/data/test-schema2.yaml b/t/data/test-schema2.yaml new file mode 100644 index 000000000..5b32efada --- /dev/null +++ b/t/data/test-schema2.yaml @@ -0,0 +1,27 @@ +--- +$schema: http://json-schema.org/draft-07/schema# +definitions: + my_name: + type: string + minLength: 2 + my_address: + type: object + properties: + street: + type: string + city: + # this is a local ref in a secondary file - resolution is extra tricky + $ref: /definitions/my_name + dupe_name: + type: string + i_am_a_ref_with_the_same_name: + type: string + i_have_a_ref_to_the_first_filename: + type: object + properties: + gotcha: + $ref: test-schema.yaml#/definitions/ref3 + i_have_a_ref_with_the_same_name: + type: string + +# vim: set sts=2 sw=2 et : diff --git a/t/schema.t b/t/schema.t index 5e6954dd1..5ab9f5fb5 100644 --- a/t/schema.t +++ b/t/schema.t @@ -1,4 +1,5 @@ use strict; +use warnings; use Test::Conch; use Test::More; @@ -6,57 +7,357 @@ use Test::Warnings; use JSON::Validator; use Mojo::JSON qw(decode_json); use Mojo::File qw(path); +use Test::Deep; +use Test::Fatal; +use Conch::Controller::Schema; + +my $_validator = JSON::Validator->new; +$_validator->schema('http://json-schema.org/draft-07/schema#'); + +subtest 'extraction with $refs' => sub { + # these are tuples: expected result from extracting title name, and test name. + my @tests = ( + [ + { + title => 'i_have_nested_refs', + '$schema' => 'http://json-schema.org/draft-07/schema#', + '$id' => 'urn:i_have_nested_refs.schema.json', + # begin all referenced definitions + definitions => { + ref1 => { + type => 'array', + items => { + '$ref' => '/definitions/ref2', + }, + }, + ref2 => { + type => 'string', + minLength => 1, + }, + }, + # begin i_have_nested_refs definition + type => 'object', + properties => { + my_key1 => { + '$ref' => '/definitions/ref1', + }, + my_key2 => { + '$ref' => '/definitions/ref1', + }, + }, + }, + 'find and resolve nested $refs; main schema is at the top level', + ], + + [ + { + title => 'i_have_a_recursive_ref', + '$schema' => 'http://json-schema.org/draft-07/schema#', + '$id' => 'urn:i_have_a_recursive_ref.schema.json', + # begin all referenced definitions + definitions => { + i_have_a_recursive_ref => { + type => 'object', + properties => { + name => { type => 'string' }, + children => { + type => 'array', + items => { '$ref' => '/definitions/i_have_a_recursive_ref' }, + default => [], + }, + }, + }, + }, + # begin i_have_a_recursive_ref definition + # it is duplicated with the above, but there is no other way, + # because $ref cannot be combined with other sibling keys + type => 'object', + properties => { + name => { type => 'string' }, + children => { + type => 'array', + items => { '$ref' => '/definitions/i_have_a_recursive_ref' }, + default => [], + }, + }, + }, + 'find and resolve recursive $refs', + ], + + [ + { + title => 'i_have_a_ref_to_another_file', + '$schema' => 'http://json-schema.org/draft-07/schema#', + '$id' => 'urn:i_have_a_ref_to_another_file.schema.json', + # begin all referenced definitions + definitions => { + my_name => { + type => 'string', + minLength => 2, + }, + my_address => { + type => 'object', + properties => { + street => { + type => 'string', + }, + city => { + '$ref' => '/definitions/my_name', + }, + }, + }, + ref1 => { + type => 'array', + items => { + '$ref' => '/definitions/ref2', + }, + }, + ref2 => { + type => 'string', + minLength => 1, + }, + }, + # begin i_have_a_ref_to_another_file definition + type => 'object', + properties => { + # these ref targets are rewritten + name => { '$ref' => '/definitions/my_name' }, + address => { '$ref' => '/definitions/my_address' }, + secrets => { '$ref' => '/definitions/ref1' }, + }, + }, + 'find and resolve references to other local files', + ], + + [ + { + title => 'i_am_a_ref', + '$schema' => 'http://json-schema.org/draft-07/schema#', + '$id' => 'urn:i_am_a_ref.schema.json', + # begin all referenced definitions + definitions => { + ref2 => { + type => 'string', + minLength => 1, + }, + }, + # begin i_am_a_ref definition - which is actually ref1 + type => 'array', + items => { + '$ref' => '/definitions/ref2', + }, + }, + 'find and resolve references where the definition itself is a ref', + ], + + [ + { + title => 'i_am_a_ref_level_1', + '$schema' => 'http://json-schema.org/draft-07/schema#', + '$id' => 'urn:i_am_a_ref_level_1.schema.json', + # begin i_am_a_ref definition - which is actually (eventually) ref3 + type => 'integer', + }, + 'find and resolve references where the definition itself is a ref, multiple times over', + ], + + [ + { + title => 'i_have_refs_with_the_same_name', + '$schema' => 'http://json-schema.org/draft-07/schema#', + '$id' => 'urn:i_have_refs_with_the_same_name.schema.json', + # begin all referenced definitions + definitions => { + i_am_a_ref_with_the_same_name => { + type => 'string', + }, + }, + # begin i_have_a_ref_with_the_same_name definition + type => 'object', + properties => { + me => { + '$ref' => '/definitions/i_am_a_ref_with_the_same_name', + }, + }, + }, + '$refs which are simply $refs themselves are traversed automatically during resolution', + ], + + [ + { + title => 'i_am_a_ref_with_the_same_name', + '$schema' => 'http://json-schema.org/draft-07/schema#', + '$id' => 'urn:i_am_a_ref_with_the_same_name.schema.json', + # begin i_am_a_ref_with_the_same_name definition - pulled from secondary file + type => 'string', + }, + '$refs which are simply $refs themselves are traversed automatically during resolution, at the top level too', + ], + + [ + { + title => 'i_contain_refs_to_same_named_definitions', + exception => qr!namespace collision: .*t/data/test-schema2?\.yaml#/definitions/dupe_name but already have a /definitions/dupe_name from .*t/data/test-schema2?\.yaml#/definitions/dupe_name!, + }, + 'cannot handle pulling in references that have the same root name', + ], + + [ + { + title => 'i_have_a_ref_with_the_same_name', + '$schema' => 'http://json-schema.org/draft-07/schema#', + '$id' => 'urn:i_have_a_ref_with_the_same_name.schema.json', + # begin all referenced definitions + definitions => { + i_have_a_ref_with_the_same_name => { type => 'string' }, + }, + # begin i_have_a_ref_with_the_same_name definition + type => 'object', + properties => { + name => { type => 'string' }, + children => { + type => 'array', + items => { '$ref' => '/definitions/i_have_a_ref_with_the_same_name' }, + default => [], + }, + }, + }, + 'we can handle pulling in references that have the same root name as the top level name', + ], + + [ + { + title => 'i_am_a_ref_to_another_file', + '$schema' => 'http://json-schema.org/draft-07/schema#', + '$id' => 'urn:i_am_a_ref_to_another_file.schema.json', + # begin all referenced definitions + definitions => { + ref3 => { type => 'integer' }, + }, + # begin i_am_a_ref_to_another_file definition - which is actually i_have_a_ref_to_the_first_filename + type => 'object', + properties => { + gotcha => { '$ref' => '/definitions/ref3' }, + }, + }, + 'find and resolve a reference that immediately leaps to another file', + ], + + ); + + my $jv = JSON::Validator->new; + $jv->load_and_validate_schema('t/data/test-schema.yaml', { schema => 'http://json-schema.org/draft-07/schema#' }); + + subtest $_->[1] => sub { + my ($expected_output, $test_name) = $_->@*; + + my $title = $expected_output->{title}; + my $got; + my $exception = exception { + $got = Conch::Controller::Schema::_extract_schema_definition($jv, $title); + }; + + if (my $message = $expected_output->{exception}) { + like($exception, $message, 'died trying to extract schema for '.$title) + or note('lived, and got: ', explain($got)); + return; + } + + is($exception, undef, 'no exceptions extracting schema for '.$title) + or return; + cmp_deeply($got, $expected_output, 'extracted schema for '.$title); + + my @errors = $_validator->validate($got); + ok(!@errors, 'no validation errors in the generated schema'); + + my $_jv = JSON::Validator->new; + $_jv->load_and_validate_schema($got, { schema => 'http://json-schema.org/draft-07/schema#' }); + cmp_deeply( + $_jv->schema->data, + $expected_output, + 'our generated schema does not lose any data when parsed again by a validator', + ); + } + foreach @tests; +}; -my $json_schema = - JSON::Validator->new->schema('http://json-schema.org/draft-04/schema#') - ->schema->data; my $t = Test::Conch->new; +my $json_spec_schema = $_validator->schema->data; + +$t->get_ok('/schema/REQUEST/hello') + ->status_is(404) + ->json_is({ error => 'Not Found' }); $t->get_ok('/schema/request/hello') ->status_is(404); $t->get_ok('/schema/response/Login') ->status_is(200) - ->json_schema_is($json_schema) - ->json_is( '/type' => 'object' ) - ->json_is( '/properties/jwt_token/type' => 'string' ); + ->json_schema_is($json_spec_schema) + ->json_cmp_deeply(superhashof({ + '$schema' => 'http://json-schema.org/draft-07/schema#', + type => 'object', + properties => { jwt_token => { type => 'string' } }, + })); $t->get_ok('/schema/request/Login') ->status_is(200) - ->json_schema_is($json_schema) - ->json_is( '/type' => 'object' ) - ->json_is( '/properties/user/$ref' => '/definitions/non_empty_string' ) - ->json_is( '/properties/password/$ref' => '/definitions/non_empty_string' ) - ->json_cmp_deeply('/definitions/non_empty_string' => { type => 'string', minLength => 1 } ); + ->json_schema_is($json_spec_schema) + ->json_cmp_deeply(superhashof({ + '$schema' => 'http://json-schema.org/draft-07/schema#', + type => 'object', + properties => { + user => { '$ref' => '/definitions/non_empty_string' }, + password => { '$ref' => '/definitions/non_empty_string' }, + }, + definitions => { + non_empty_string => { type => 'string', minLength => 1 }, + }, + })); +$t->get_ok('/schema/request/HardwareProductCreate') + ->status_is(200) + ->json_schema_is($json_spec_schema) + ->json_cmp_deeply('', superhashof({ + definitions => { + uuid => superhashof({}), + HardwareProductProfileCreate => superhashof({}), + HardwareProductProfileUpdate => superhashof({}), + }, + }), 'nested definitions are found and included'); $t->get_ok('/schema/request/device_report') ->status_is(200) - ->json_schema_is($json_schema); + ->json_schema_is($json_spec_schema) + ->json_is('/$schema', 'http://json-schema.org/draft-07/schema#'); # ensure that one of the schemas can validate some data { - my $report = - decode_json path('t/integration/resource/passing-device-report.json') - ->slurp; + my $report = decode_json(path('t/integration/resource/passing-device-report.json')->slurp); my $schema = $t->get_ok('/schema/request/device_report')->tx->res->json; - my $jv = JSON::Validator->new()->load_and_validate_schema($schema); + + # FIXME: JSON::Validator should be picking this up out of the schema on its own. + my $jv = JSON::Validator->new; + $jv->load_and_validate_schema($schema, { schema => $schema->{'$schema'} }); + is($jv->version, 7, 'schema declares JSON Schema version 7'); my @errors = $jv->validate($report); - is scalar @errors, 0, 'no errors'; + is(scalar @errors, 0, 'no errors'); } $t->get_ok('/schema/request/device_report') ->status_is(200) - ->json_schema_is($json_schema); + ->json_schema_is($json_spec_schema) + ->json_is('/$schema', 'http://json-schema.org/draft-07/schema#'); # ensure that one of the schemas can validate some data { - my $report = decode_json path('t/integration/resource/passing-device-report.json')->slurp; + my $report = decode_json(path('t/integration/resource/passing-device-report.json')->slurp); my $schema = $t->get_ok('/schema/request/device_report')->tx->res->json; - my $jv = JSON::Validator->new()->load_and_validate_schema($schema); + my $jv = JSON::Validator->new; + $jv->load_and_validate_schema($schema); my @errors = $jv->validate($report); - is scalar @errors, 0, 'no errors'; + is(scalar @errors, 0, 'no errors'); } done_testing;