From e0aaebfafaddb8e58aa226477ad7ef5d6eaba170 Mon Sep 17 00:00:00 2001 From: Karen Etheridge Date: Wed, 2 Dec 2020 16:31:45 -0800 Subject: [PATCH 1/7] add a banner, e.g. for printing on console start --- docs/modules/Conch.md | 4 ++++ lib/Conch.pm | 23 +++++++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/docs/modules/Conch.md b/docs/modules/Conch.md index f05990307..d4a7e8871 100644 --- a/docs/modules/Conch.md +++ b/docs/modules/Conch.md @@ -36,6 +36,10 @@ Stores a [Conch::Time](../modules/Conch%3A%3ATime) instance representing the tim Retrieves the ["host" in Mojo::URL](https://metacpan.org/pod/Mojo%3A%3AURL#host) portion of the request URL, suitable for constructing base URLs in user-facing content. +### banner + +Banner text suitable for displaying on startup. + ## LICENSING Copyright Joyent, Inc. diff --git a/lib/Conch.pm b/lib/Conch.pm index 39237e86b..0ca55c5e7 100644 --- a/lib/Conch.pm +++ b/lib/Conch.pm @@ -234,6 +234,29 @@ in user-facing content. Conch::Route->all_routes($self->routes, $self); $self->log->info('Conch initialized at '.$self->version_tag); + +=head2 banner + +Banner text suitable for displaying on startup. + +=cut + + $self->helper(banner => sub ($c) { + my $banner = <<'BANNER'; + + /\ _ + {-.} ___ ___ _ __ ___| |__ + /'-._; / __/ _ \| '_ \ / __| '_ \ + _{._ } | (_| (_) | | | | (__| | | | + ,' \ `-./ \___\___/|_| |_|\___|_| |_| + \ | / + \, | / Version: %s + \_|/ Started: %s + +BANNER + + return sprintf($banner, $c->version_tag, $c->startup_time); + }); } 1; From e906c02aebe2076a3a080315127a645f6e7be02b Mon Sep 17 00:00:00 2001 From: Karen Etheridge Date: Thu, 3 Dec 2020 13:37:26 -0800 Subject: [PATCH 2/7] disallow go-style "null" datetimes in request payloads --- docs/json-schema/common.json | 7 +++++++ docs/json-schema/request.json | 6 +++--- json-schema/common.yaml | 5 +++++ json-schema/request.yaml | 6 +++--- 4 files changed, 18 insertions(+), 6 deletions(-) diff --git a/docs/json-schema/common.json b/docs/json-schema/common.json index 76bb0d7b7..668bed3c0 100644 --- a/docs/json-schema/common.json +++ b/docs/json-schema/common.json @@ -49,6 +49,13 @@ }, "type" : "object" }, + "date-time" : { + "format" : "date-time", + "not" : { + "pattern" : "^000[01]", + "type" : "string" + } + }, "device_asset_tag" : { "pattern" : "^\\S+$", "type" : "string" diff --git a/docs/json-schema/request.json b/docs/json-schema/request.json index 3afe40818..3a7db93c2 100644 --- a/docs/json-schema/request.json +++ b/docs/json-schema/request.json @@ -69,7 +69,7 @@ "$ref" : "common.json#/$defs/mojo_standard_placeholder" }, "started" : { - "format" : "date-time", + "$ref" : "common.json#/$defs/date-time", "type" : "string" } }, @@ -154,7 +154,7 @@ "minProperties" : 1, "properties" : { "completed" : { - "format" : "date-time", + "$ref" : "common.json#/$defs/date-time", "type" : [ "null", "string" @@ -177,7 +177,7 @@ "$ref" : "common.json#/$defs/mojo_standard_placeholder" }, "started" : { - "format" : "date-time", + "$ref" : "common.json#/$defs/date-time", "type" : [ "null", "string" diff --git a/json-schema/common.yaml b/json-schema/common.yaml index 19356062b..93882db25 100644 --- a/json-schema/common.yaml +++ b/json-schema/common.yaml @@ -13,6 +13,11 @@ $defs: oneOf: - format: ipv4 - format: ipv6 + date-time: + format: date-time + not: + type: string + pattern: '^000[01]' macaddr: type: string pattern: '^[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}$' diff --git a/json-schema/request.yaml b/json-schema/request.yaml index 9d73badec..73a65d141 100644 --- a/json-schema/request.yaml +++ b/json-schema/request.yaml @@ -602,7 +602,7 @@ $defs: $ref: common.yaml#/$defs/non_empty_string started: type: string - format: date-time + $ref: common.yaml#/$defs/date-time admins: type: array uniqueItems: true @@ -628,10 +628,10 @@ $defs: - $ref: common.yaml#/$defs/non_empty_string started: type: [ 'null', string ] - format: date-time + $ref: common.yaml#/$defs/date-time completed: type: [ 'null', string ] - format: date-time + $ref: common.yaml#/$defs/date-time links: $ref: common.yaml#/$defs/links BuildLinks: From 693765bd9b3f4376dff8ba72adcf28e019779569 Mon Sep 17 00:00:00 2001 From: Karen Etheridge Date: Thu, 3 Dec 2020 15:04:15 -0800 Subject: [PATCH 3/7] use the same YAML decoder for JSON translation as we use internally --- misc/pod2githubpages | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/misc/pod2githubpages b/misc/pod2githubpages index 9dc7a0617..5d9a67b97 100755 --- a/misc/pod2githubpages +++ b/misc/pod2githubpages @@ -15,7 +15,7 @@ use Pod::Markdown::Github; use Class::Method::Modifiers 'around'; use Path::Tiny; use JSON::MaybeXS; -use YAML::XS; +use YAML::PP; use List::Util 'any'; my %opts = ( @@ -144,9 +144,10 @@ foreach my $infile (@all_files) { path($outfile)->spew($markdown); } +my $yaml_decoder = YAML::PP->new(boolean => 'JSON::PP'); my $json_encoder = JSON()->new->pretty->indent_length(2)->canonical(1); foreach my $filename (grep m{^json-schema/[^.]+\.yaml$}, @all_files) { - my $data = YAML::XS::LoadFile($filename); + my $data = $yaml_decoder->load_file($filename); my $base = path($filename)->basename('.yaml'); my ($type) = $filename =~ m{^json-schema/([^.]+)\.yaml$}; if (any { $type eq $_ } qw(query_params request response common device_report)) { From 299e936e5d8d88ed9c310eff2dad4e85ebd793bc Mon Sep 17 00:00:00 2001 From: Karen Etheridge Date: Thu, 3 Dec 2020 15:05:37 -0800 Subject: [PATCH 4/7] blow up with graceful error if YAML is not parseable --- lib/Conch/Plugin/JSONValidator.pm | 10 +++++++++- t/02-json-schemas.t | 8 +++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/lib/Conch/Plugin/JSONValidator.pm b/lib/Conch/Plugin/JSONValidator.pm index ecab18a28..874817a1a 100644 --- a/lib/Conch/Plugin/JSONValidator.pm +++ b/lib/Conch/Plugin/JSONValidator.pm @@ -8,6 +8,7 @@ use YAML::PP; use Mojo::JSON 'to_json'; use Path::Tiny; use List::Util qw(any none first); +use Try::Tiny; =pod @@ -143,8 +144,15 @@ Returns a L object with all JSON Schemas pre-loaded. # TODO: blocked on https://github.com/ingydotnet/yaml-libyaml-pm/issues/68 # local $YAML::XS::Boolean = 'JSON::PP'; ... YAML::XS::LoadFile(...) my $yaml = YAML::PP->new(boolean => 'JSON::PP'); - $_validator->add_schema($_, $yaml->load_file('json-schema/'.$_)) + try { + $_validator->add_schema($_, $yaml->load_file('json-schema/'.$_)) foreach map path($_)->basename, glob('json-schema/*.yaml'); + } + catch { + require Data::Dumper; + die "problems adding schema (YAML is not parseable?) - ", + Data::Dumper->new([ [ map $_->TO_JSON, @$_ ] ])->Indent(0)->Terse(1)->Dump; + }; $_validator; }); diff --git a/t/02-json-schemas.t b/t/02-json-schemas.t index 1940afc24..dc0524609 100644 --- a/t/02-json-schemas.t +++ b/t/02-json-schemas.t @@ -5,6 +5,7 @@ use experimental 'signatures'; use Test::More; use YAML::PP; use Path::Tiny; +use Try::Tiny; use JSON::Schema::Draft201909; use JSON::Schema::Draft201909::Utilities 'canonical_schema_uri'; @@ -24,7 +25,12 @@ my @files; foreach my $filename (split /\n/, `git ls-files json-schema`) { diag('skipping '.$filename), next if $filename =~ /^\./ or $filename !~ /yaml$/; my $path = path($filename); - $js->add_schema($path->basename, $yaml->load_file($filename)); + try { + $js->add_schema($path->basename, $yaml->load_file($filename)); + } + catch { + die "$filename is not parseable: ", explain(map $_->TO_JSON, @$_); + }; push @files, $path->basename; } From 764be8e14ab18f1b71c966cb1191a0d73da09732 Mon Sep 17 00:00:00 2001 From: Karen Etheridge Date: Thu, 3 Dec 2020 15:23:00 -0800 Subject: [PATCH 5/7] doc fixes --- docs/modules/Conch::Route::Build.md | 1 + docs/modules/Conch::Route::Rack.md | 1 + lib/Conch/Route/Build.pm | 2 ++ lib/Conch/Route/Rack.pm | 2 ++ 4 files changed, 6 insertions(+) diff --git a/docs/modules/Conch::Route::Build.md b/docs/modules/Conch::Route::Build.md index 255d08603..6c2f34124 100644 --- a/docs/modules/Conch::Route::Build.md +++ b/docs/modules/Conch::Route::Build.md @@ -60,6 +60,7 @@ Supports the following optional query parameters: #### `DELETE /build/:build_id_or_name/links` - Requires system admin authorization or the admin role on the build +- Controller/Action: ["remove\_links" in Conch::Controller::Build](../modules/Conch%3A%3AController%3A%3ABuild#remove_links) - Request: [request.json#/$defs/BuildLinksOrNull](../json-schema/request.json#/$defs/BuildLinksOrNull) - Response: `204 No Content`, plus Location header diff --git a/docs/modules/Conch::Route::Rack.md b/docs/modules/Conch::Route::Rack.md index db544843f..dc5271245 100644 --- a/docs/modules/Conch::Route::Rack.md +++ b/docs/modules/Conch::Route::Rack.md @@ -103,6 +103,7 @@ only the rack's phase, or all the rack's devices' phases as well. #### `DELETE /rack/:rack_id_or_name/links` - User requires the read/write role on the rack +- Controller/Action: ["remove\_links" in Conch::Controller::Rack](../modules/Conch%3A%3AController%3A%3ARack#remove_links) - Request: [request.json#/$defs/RackLinksOrNull](../json-schema/request.json#/$defs/RackLinksOrNull) - Response: `204 No Content`, plus Location header diff --git a/lib/Conch/Route/Build.pm b/lib/Conch/Route/Build.pm index f6c4cc10a..129f3a71d 100644 --- a/lib/Conch/Route/Build.pm +++ b/lib/Conch/Route/Build.pm @@ -219,6 +219,8 @@ Supports the following optional query parameters: =item * Requires system admin authorization or the admin role on the build +=item * Controller/Action: L + =item * Request: F =item * Response: C<204 No Content>, plus Location header diff --git a/lib/Conch/Route/Rack.pm b/lib/Conch/Route/Rack.pm index 26b3a004f..cbaff95df 100644 --- a/lib/Conch/Route/Rack.pm +++ b/lib/Conch/Route/Rack.pm @@ -265,6 +265,8 @@ only the rack's phase, or all the rack's devices' phases as well. =item * User requires the read/write role on the rack +=item * Controller/Action: L + =item * Request: F =item * Response: C<204 No Content>, plus Location header From b23f65230fb11b23d9e1cdc1d984c2866d0dd4fe Mon Sep 17 00:00:00 2001 From: Karen Etheridge Date: Mon, 7 Dec 2020 11:30:46 -0800 Subject: [PATCH 6/7] update Mojolicious --- cpanfile.snapshot | 7 ++++--- t/conch-rollbar.t | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/cpanfile.snapshot b/cpanfile.snapshot index 3b538fa12..62bf36274 100644 --- a/cpanfile.snapshot +++ b/cpanfile.snapshot @@ -2805,8 +2805,8 @@ DISTRIBUTIONS Mojolicious 8.50 SQL::Abstract 1.86 perl 5.016 - Mojolicious-8.63 - pathname: S/SR/SRI/Mojolicious-8.63.tar.gz + Mojolicious-8.67 + pathname: S/SR/SRI/Mojolicious-8.67.tar.gz provides: Mojo undef Mojo::Asset undef @@ -2875,11 +2875,12 @@ DISTRIBUTIONS Mojo::UserAgent::Transactor undef Mojo::Util undef Mojo::WebSocket undef - Mojolicious 8.63 + Mojolicious 8.67 Mojolicious::Command undef Mojolicious::Command::Author::cpanify undef Mojolicious::Command::Author::generate undef Mojolicious::Command::Author::generate::app undef + Mojolicious::Command::Author::generate::dockerfile undef Mojolicious::Command::Author::generate::lite_app undef Mojolicious::Command::Author::generate::makefile undef Mojolicious::Command::Author::generate::plugin undef diff --git a/t/conch-rollbar.t b/t/conch-rollbar.t index 8ea9d06c1..64a7b4e40 100644 --- a/t/conch-rollbar.t +++ b/t/conch-rollbar.t @@ -93,6 +93,7 @@ package RollbarSimulator { }); $self->plugin('Conch::Plugin::Features', $self->config); $self->plugin('Conch::Plugin::JSONValidator', {}); + $self->log(Mojo::Log->new(level => 'fatal')); } } From 105035dc83b24220fd1114e9aba4b1a840fd09c0 Mon Sep 17 00:00:00 2001 From: Karen Etheridge Date: Mon, 7 Dec 2020 10:23:01 -0800 Subject: [PATCH 7/7] update JSD2 --- cpanfile | 2 +- cpanfile.snapshot | 32 +++++++++++++-------------- lib/Conch/Plugin/JSONValidator.pm | 7 +++--- misc/validate | 3 +-- t/02-json-schemas.t | 5 ++--- t/integration/crud/hardware-product.t | 12 +++++----- 6 files changed, 29 insertions(+), 32 deletions(-) diff --git a/cpanfile b/cpanfile index 78511867e..09d48a2a4 100644 --- a/cpanfile +++ b/cpanfile @@ -29,7 +29,7 @@ requires 'Email::Sender::Simple'; requires 'Email::Sender::Transport::SMTP'; requires 'Net::DNS'; # not used directly, but Email::Valid sometimes demands it requires 'experimental', '0.020'; -requires 'JSON::Schema::Draft201909', '0.017'; +requires 'JSON::Schema::Draft201909', '0.019'; requires 'Email::Address::XS', '1.01'; requires 'YAML::PP'; requires 'next::XS'; diff --git a/cpanfile.snapshot b/cpanfile.snapshot index 62bf36274..511605777 100644 --- a/cpanfile.snapshot +++ b/cpanfile.snapshot @@ -2283,22 +2283,22 @@ DISTRIBUTIONS JSON::PP 2.27300 Scalar::Util 0 perl 5.006 - JSON-Schema-Draft201909-0.017 - pathname: E/ET/ETHER/JSON-Schema-Draft201909-0.017.tar.gz - provides: - JSON::Schema::Draft201909 0.017 - JSON::Schema::Draft201909::Annotation 0.017 - JSON::Schema::Draft201909::Document 0.017 - JSON::Schema::Draft201909::Error 0.017 - JSON::Schema::Draft201909::Result 0.017 - JSON::Schema::Draft201909::Utilities 0.017 - JSON::Schema::Draft201909::Vocabulary 0.017 - JSON::Schema::Draft201909::Vocabulary::Applicator 0.017 - JSON::Schema::Draft201909::Vocabulary::Content 0.017 - JSON::Schema::Draft201909::Vocabulary::Core 0.017 - JSON::Schema::Draft201909::Vocabulary::Format 0.017 - JSON::Schema::Draft201909::Vocabulary::MetaData 0.017 - JSON::Schema::Draft201909::Vocabulary::Validation 0.017 + JSON-Schema-Draft201909-0.019 + pathname: E/ET/ETHER/JSON-Schema-Draft201909-0.019.tar.gz + provides: + JSON::Schema::Draft201909 0.019 + JSON::Schema::Draft201909::Annotation 0.019 + JSON::Schema::Draft201909::Document 0.019 + JSON::Schema::Draft201909::Error 0.019 + JSON::Schema::Draft201909::Result 0.019 + JSON::Schema::Draft201909::Utilities 0.019 + JSON::Schema::Draft201909::Vocabulary 0.019 + JSON::Schema::Draft201909::Vocabulary::Applicator 0.019 + JSON::Schema::Draft201909::Vocabulary::Content 0.019 + JSON::Schema::Draft201909::Vocabulary::Core 0.019 + JSON::Schema::Draft201909::Vocabulary::Format 0.019 + JSON::Schema::Draft201909::Vocabulary::MetaData 0.019 + JSON::Schema::Draft201909::Vocabulary::Validation 0.019 requirements: B 0 Carp 0 diff --git a/lib/Conch/Plugin/JSONValidator.pm b/lib/Conch/Plugin/JSONValidator.pm index 874817a1a..0a91bacfb 100644 --- a/lib/Conch/Plugin/JSONValidator.pm +++ b/lib/Conch/Plugin/JSONValidator.pm @@ -3,7 +3,7 @@ package Conch::Plugin::JSONValidator; use Mojo::Base 'Mojolicious::Plugin', -signatures; use feature 'unicode_strings'; -use JSON::Schema::Draft201909 '0.017'; +use JSON::Schema::Draft201909 '0.019'; use YAML::PP; use Mojo::JSON 'to_json'; use Path::Tiny; @@ -71,7 +71,7 @@ Returns a boolean. $app->helper(validate_query_params => sub ($c, $schema_name, $data = $c->req->query_params->to_hash) { my $validator = $c->json_schema_validator; - my $result = $validator->evaluate($data, 'query_params.yaml#/$defs/'.$schema_name); + my $result = $validator->evaluate($data, 'query_params.yaml#/$defs/'.$schema_name, { collect_annotations => 1 }); if (not $result) { my @errors = $c->normalize_evaluation_result($result); $c->log->warn("FAILED query_params validation for schema $schema_name: ".to_json(\@errors)); @@ -139,7 +139,6 @@ Returns a L object with all JSON Schemas pre-loaded. $_validator = JSON::Schema::Draft201909->new( output_format => 'terse', validate_formats => 1, - collect_annotations => 1, ); # TODO: blocked on https://github.com/ingydotnet/yaml-libyaml-pm/issues/68 # local $YAML::XS::Boolean = 'JSON::PP'; ... YAML::XS::LoadFile(...) @@ -151,7 +150,7 @@ Returns a L object with all JSON Schemas pre-loaded. catch { require Data::Dumper; die "problems adding schema (YAML is not parseable?) - ", - Data::Dumper->new([ [ map $_->TO_JSON, @$_ ] ])->Indent(0)->Terse(1)->Dump; + Data::Dumper->new([ $@->TO_JSON ])->Indent(0)->Terse(1)->Dump; }; $_validator; diff --git a/misc/validate b/misc/validate index 6adb79689..206298cba 100755 --- a/misc/validate +++ b/misc/validate @@ -4,7 +4,7 @@ use warnings; use open ':std', ':encoding(UTF-8)'; # force stdin, stdout, stderr into utf8 use Getopt::Long; -use JSON::Schema::Draft201909; +use JSON::Schema::Draft201909 0.019; use Mojo::File 'path'; use JSON::MaybeXS 'decode_json'; use Pod::Usage; @@ -25,7 +25,6 @@ pod2usage(1) if $help; my $js = JSON::Schema::Draft201909->new( output_format => 'basic', validate_formats => 1, - collect_annotations => 1, ); my $schema = $schema_file ? $schema_file : "$schema_url/$name"; diff --git a/t/02-json-schemas.t b/t/02-json-schemas.t index dc0524609..150e8ddef 100644 --- a/t/02-json-schemas.t +++ b/t/02-json-schemas.t @@ -6,7 +6,7 @@ use Test::More; use YAML::PP; use Path::Tiny; use Try::Tiny; -use JSON::Schema::Draft201909; +use JSON::Schema::Draft201909 0.019; use JSON::Schema::Draft201909::Utilities 'canonical_schema_uri'; diag 'using JSON::Schema::Draft201909 '.JSON::Schema::Draft201909->VERSION; @@ -15,7 +15,6 @@ my $yaml = YAML::PP->new(boolean => 'JSON::PP'); my $js = JSON::Schema::Draft201909->new( output_format => 'terse', validate_formats => 1, - collect_annotations => 1, max_traversal_depth => 67, # needed for other.yaml ); @@ -29,7 +28,7 @@ foreach my $filename (split /\n/, `git ls-files json-schema`) { $js->add_schema($path->basename, $yaml->load_file($filename)); } catch { - die "$filename is not parseable: ", explain(map $_->TO_JSON, @$_); + die "$filename is not parseable: ", explain($@->TO_JSON); }; push @files, $path->basename; } diff --git a/t/integration/crud/hardware-product.t b/t/integration/crud/hardware-product.t index 64327d1d6..4cb83b388 100644 --- a/t/integration/crud/hardware-product.t +++ b/t/integration/crud/hardware-product.t @@ -68,19 +68,19 @@ my %hw_fields = ( $t->post_ok('/hardware_product', json => { %hw_fields, specification => 'not json!' } ) ->status_is(400) ->json_schema_is('RequestValidationError') - ->json_cmp_deeply('/details', superbagof(superhashof({ data_location => '/specification', error => 'wrong type (expected object)' }))); + ->json_cmp_deeply('/details', [ superhashof({ data_location => '/specification', error => 'wrong type (expected object)' }) ]); $t->post_ok('/hardware_product', json => { %hw_fields, specification => '{"disk_size":"not an object"}' } ) ->status_is(400) ->json_schema_is('RequestValidationError') - ->json_cmp_deeply('/details', superbagof(superhashof({ data_location => '/specification', error => 'wrong type (expected object)' }))); + ->json_cmp_deeply('/details', [ superhashof({ data_location => '/specification', error => 'wrong type (expected object)' }) ]); $t->post_ok('/hardware_product', json => { %hw_fields, specification => { disk_size => 'not an object' } }) ->status_is(400) ->json_schema_is('RequestValidationError') ->json_cmp_deeply({ error => 'request did not match required format', - details => superbagof(superhashof({ data_location => '/specification/disk_size', error => 'wrong type (expected object)' })), + details => [ superhashof({ data_location => '/specification/disk_size', error => 'wrong type (expected object)' }) ], schema => $base_uri.'json_schema/request/HardwareProductCreate', }); @@ -181,19 +181,19 @@ $t->post_ok('/hardware_product', json => { $t->post_ok("/hardware_product/$new_hw_id", json => { specification => 'not json!' }) ->status_is(400) ->json_schema_is('RequestValidationError') - ->json_cmp_deeply('/details', superbagof(superhashof({ data_location => '/specification', error => 'wrong type (expected object)' }))); + ->json_cmp_deeply('/details', [ superhashof({ data_location => '/specification', error => 'wrong type (expected object)' }) ]); $t->post_ok("/hardware_product/$new_hw_id", json => { specification => '{"disk_size":"not an object"}' }) ->status_is(400) ->json_schema_is('RequestValidationError') - ->json_cmp_deeply('/details', superbagof(superhashof({ data_location => '/specification', error => 'wrong type (expected object)' }))); + ->json_cmp_deeply('/details', [ superhashof({ data_location => '/specification', error => 'wrong type (expected object)' }) ]); $t->post_ok("/hardware_product/$new_hw_id", json => { specification => { disk_size => 'not an object' } }) ->status_is(400) ->json_schema_is('RequestValidationError') ->json_cmp_deeply({ error => 'request did not match required format', - details => superbagof(superhashof({ data_location => '/specification/disk_size', error => 'wrong type (expected object)' })), + details => [ superhashof({ data_location => '/specification/disk_size', error => 'wrong type (expected object)' }) ], schema => $base_uri.'json_schema/request/HardwareProductUpdate', });