From 165d2b2d0af88310ee14da17ff398a435cd41014 Mon Sep 17 00:00:00 2001 From: Saketh Undurty Date: Fri, 19 Jun 2020 15:03:43 -0700 Subject: [PATCH 01/11] [Ruby SDK v3 upgrade] Remove deprecated underscore method call Prior to this change, the Agent was using aws sdk ruby v2. That version had a method called Seahorse::Util.underscore(), which was used in the Agent. We are in the process of upgrading the agent to use v3 of the SDK, in which that method does not exist. This change replaces that Seahorse::Util.underscore() method call with a newly created method that does the same thing. I also added validation to make sure the commands being passed in to the underscore method call were in the PascalForm, as that is what the Agent commands are always in. We need this update so that when we upgrade to v3, the code will be compatible. * Unit Tests : [Y] * Integration Tests : [N] Seahorse::Util.underscore method: https://www.rubydoc.info/gems/aws-sdk-core/2.9.44/Seahorse%2FUtil.underscore --- .../plugins/codedeploy/command_executor.rb | 5 +- lib/instance_agent/string_utils.rb | 16 +++++++ .../codedeploy/command_executor_test.rb | 4 +- test/instance_agent/string_utils_test.rb | 48 +++++++++++++++++++ test/test_helper.rb | 1 + 5 files changed, 70 insertions(+), 4 deletions(-) create mode 100644 lib/instance_agent/string_utils.rb create mode 100644 test/instance_agent/string_utils_test.rb diff --git a/lib/instance_agent/plugins/codedeploy/command_executor.rb b/lib/instance_agent/plugins/codedeploy/command_executor.rb index e8d2e12b..b27ebfda 100644 --- a/lib/instance_agent/plugins/codedeploy/command_executor.rb +++ b/lib/instance_agent/plugins/codedeploy/command_executor.rb @@ -12,6 +12,7 @@ require 'instance_agent/plugins/codedeploy/deployment_specification' require 'instance_agent/plugins/codedeploy/hook_executor' require 'instance_agent/plugins/codedeploy/installer' +require 'instance_agent/string_utils' module InstanceAgent module Plugins @@ -47,8 +48,8 @@ def initialize(options = {}) def self.command(name, &blk) @command_methods ||= Hash.new - - method = Seahorse::Util.underscore(name).to_sym + raise "Received command is not in PascalCase form: #{name.to_s}" unless StringUtils.is_pascal_case(name.to_s) + method = StringUtils.underscore(name.to_s) @command_methods[name] = method define_method(method, &blk) diff --git a/lib/instance_agent/string_utils.rb b/lib/instance_agent/string_utils.rb new file mode 100644 index 00000000..2746ff2d --- /dev/null +++ b/lib/instance_agent/string_utils.rb @@ -0,0 +1,16 @@ +module InstanceAgent + class StringUtils + + def self.underscore(string) + string. + gsub(/([A-Z0-9]+)([A-Z][a-z])/, '\1_\2'). + scan(/[a-z0-9]+|\d+|[A-Z0-9]+[a-z]*/). + join('_').downcase + end + + def self.is_pascal_case(string) + !!(string =~ /^([A-Z][a-z0-9]+)+/) + end + + end +end \ No newline at end of file diff --git a/test/instance_agent/plugins/codedeploy/command_executor_test.rb b/test/instance_agent/plugins/codedeploy/command_executor_test.rb index f6be14e1..ae5183bb 100644 --- a/test/instance_agent/plugins/codedeploy/command_executor_test.rb +++ b/test/instance_agent/plugins/codedeploy/command_executor_test.rb @@ -726,8 +726,8 @@ def generate_signed_message_for(map) #non 1:1 mapping tests context "one command hooks to multiple lifecycle events" do setup do - @command.command_name = "test_command" - @test_hook_mapping = { "test_command" => ["lifecycle_event_1","lifecycle_event_2"]} + @command.command_name = "TestCommand" + @test_hook_mapping = { "TestCommand" => ["lifecycle_event_1","lifecycle_event_2"]} @deploy_control_client = mock @command_executor = InstanceAgent::Plugins::CodeDeployPlugin::CommandExecutor.new({ :deploy_control_client => @deploy_control_client, diff --git a/test/instance_agent/string_utils_test.rb b/test/instance_agent/string_utils_test.rb new file mode 100644 index 00000000..38eadd64 --- /dev/null +++ b/test/instance_agent/string_utils_test.rb @@ -0,0 +1,48 @@ +require 'test_helper' + +class StringUtilsTest < InstanceAgentTestCase + + def test_underscore_two_words() + assert_equal("download_bundle", InstanceAgent::StringUtils.underscore("DownloadBundle")) + end + + def test_underscore_already_underscored() + assert_equal("download_bundle", InstanceAgent::StringUtils.underscore("download_bundle")) + end + + def test_underscore_two_words_lowercase_first() + assert_equal("download_bundle", InstanceAgent::StringUtils.underscore("downloadBundle")) + end + + def test_underscore_one_word() + assert_equal("install", InstanceAgent::StringUtils.underscore("Install")) + end + + def test_underscore_three_words() + assert_equal("after_allow_traffic", InstanceAgent::StringUtils.underscore("AfterAllowTraffic")) + end + + def test_underscore_four_words() + assert_equal("after_allow_test_traffic", InstanceAgent::StringUtils.underscore("AfterAllowTestTraffic")) + end + + def test_is_camel_case_all_uppercase() + assert_equal(false, InstanceAgent::StringUtils.is_pascal_case("DOWNLOADBUNDLE")) + end + + def test_is_camel_case_all_lowercase() + assert_equal(false, InstanceAgent::StringUtils.is_pascal_case("downloadbundle")) + end + + def test_is_camel_case_first_lowercase() + assert_equal(false, InstanceAgent::StringUtils.is_pascal_case("downloadBundle")) + end + + def test_is_camel_case_second_uppercase() + assert_equal(false, InstanceAgent::StringUtils.is_pascal_case("downloadbUndle")) + end + + def test_is_camel_case_happy_case() + assert_equal(true, InstanceAgent::StringUtils.is_pascal_case("DownloadBundle")) + end +end \ No newline at end of file diff --git a/test/test_helper.rb b/test/test_helper.rb index 160cdb26..6efc7623 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -19,3 +19,4 @@ # require local test helpers. If you need a helper write, # keep this pattern or you'll be punished hard require 'instance_agent_helper' +require 'instance_agent/string_utils' \ No newline at end of file From f42df34ac3657b0d79063d8f1938f9e1bbc5e572 Mon Sep 17 00:00:00 2001 From: Saketh Undurty Date: Mon, 29 Jun 2020 13:57:26 -0700 Subject: [PATCH 02/11] [Ruby SDK v3 Upgrade] Replace deprecated add_service method and upgrade to v3 Prior to this change, the ruby sdk v3 did not include the Aws.add_service method, which we used inside the agent. The ruby sdk v3 also split up the aws service clients into separate gems. This change replaces the Aws.add_service method with the implementation found in the AwsSdkRubyCodeGenWrapper It also includes the aws-sdk-s3 gem to replace all of the dependencies on 'aws-sdk-core/s3' with 'aws-sdk-s3'. * Unit Tests : [Y] * Integration Tests : [N] --- codedeploy_agent.gemspec | 4 +- spec/add_service_wrapper_spec.rb | 63 + spec/fixtures/sample_service.json | 2144 +++++++++++++++++ .../codedeploy/command_executor_test.rb | 2 +- .../codedeploy/version_tracking_test.rb | 2 +- .../lib/aws/add_service_wrapper.rb | 61 + .../lib/aws/codedeploy_commands.rb | 1 + 7 files changed, 2274 insertions(+), 3 deletions(-) create mode 100644 spec/add_service_wrapper_spec.rb create mode 100644 spec/fixtures/sample_service.json create mode 100644 vendor/gems/codedeploy-commands-1.0.0/lib/aws/add_service_wrapper.rb diff --git a/codedeploy_agent.gemspec b/codedeploy_agent.gemspec index 409be663..1ccc0dd5 100644 --- a/codedeploy_agent.gemspec +++ b/codedeploy_agent.gemspec @@ -16,7 +16,9 @@ Gem::Specification.new do |spec| spec.add_dependency('archive-tar-minitar', '~> 0.5.2') spec.add_dependency('rubyzip', '~> 1.1.0') spec.add_dependency('logging', '~> 1.8') - spec.add_dependency('aws-sdk-core', '~> 2.9') + spec.add_dependency('aws-sdk-core', '~> 3') + spec.add_dependency('aws-sdk-code-generator', '~> 0.2.1.pre') + spec.add_dependency('aws-sdk-s3', '~> 1.60.1') spec.add_dependency('simple_pid', '~> 0.2.1') spec.add_dependency('docopt', '~> 0.5.0') spec.add_dependency('concurrent-ruby', '~> 1.0.5') diff --git a/spec/add_service_wrapper_spec.rb b/spec/add_service_wrapper_spec.rb new file mode 100644 index 00000000..cddb7ce1 --- /dev/null +++ b/spec/add_service_wrapper_spec.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true +package_root = File.dirname(File.dirname(__FILE__)) + +require "#{package_root}/vendor/gems/codedeploy-commands-1.0.0/lib/aws/add_service_wrapper" + +RSpec.describe 'add_service_wrapper' do + + # This test is taken from the AwsSdkRubyCodeGenWrapper + # https://code.amazon.com/packages/AwsSdkRubyCodeGenWrapper/blobs/mainline/--/spec/add_service_wrapper_spec.rb + describe '#add_service' do + before(:all) do + @service_file = File.expand_path('../fixtures/sample_service.json', __FILE__) + @api = JSON.parse(File.read(@service_file)) + @svc_class = Aws.add_service('GeneratedService', api: @api) + end + + let(:client) {Aws::GeneratedService::Client.new(stub_responses: true) } + + it 'can create a valid client' do + expect(client).to be_instance_of(Aws::GeneratedService::Client) + end + + it 'can create a client from the returned namespace' do + expect(@svc_class::Client.new(stub_responses: true)) + .to be_instance_of(Aws::GeneratedService::Client) + end + + it 'can set constants on the returned namespace' do + @svc_class.const_set(:VERSION, '1.1.42') + expect(Aws::GeneratedService::VERSION).to eq('1.1.42') + end + + it 'can add plugins to the generated client' do + class MyPlugin; end + Aws::GeneratedService::Client.add_plugin(MyPlugin) + expect(Aws::GeneratedService::Client.plugins).to include(MyPlugin) + end + + it 'can generate a whitelabel (non-Aws) service' do + Aws.add_service('MyService', api: @api, whitelabel: true) + expect(MyService::Client.new(stub_responses: true)) + .to be_instance_of(MyService::Client) + end + + it 'loads the model from a string path' do + Aws.add_service('StringPathService', api: @service_file) + expect(Aws::StringPathService::Client.new(stub_responses: true)) + .to be_instance_of(Aws::StringPathService::Client) + end + + it 'loads the model from a PathName' do + Aws.add_service('PathService', api: Pathname.new(@service_file)) + expect(Aws::PathService::Client.new(stub_responses: true)) + .to be_instance_of(Aws::PathService::Client) + end + + it 'raises an ArgumentError if api is not provided' do + expect do + Aws.add_service('NoApiService') + end.to raise_exception(ArgumentError) + end + end +end \ No newline at end of file diff --git a/spec/fixtures/sample_service.json b/spec/fixtures/sample_service.json new file mode 100644 index 00000000..f9c7ea5d --- /dev/null +++ b/spec/fixtures/sample_service.json @@ -0,0 +1,2144 @@ +{ + "version":"2.0", + "metadata":{ + "apiVersion":"2012-06-01", + "checksumFormat":"sha256", + "endpointPrefix":"glacier", + "serviceFullName":"Amazon Glacier", + "signatureVersion":"v4", + "protocol":"rest-json" + }, + "operations":{ + "AbortMultipartUpload":{ + "name":"AbortMultipartUpload", + "http":{ + "method":"DELETE", + "requestUri":"/{accountId}/vaults/{vaultName}/multipart-uploads/{uploadId}", + "responseCode":204 + }, + "input":{"shape":"AbortMultipartUploadInput"}, + "errors":[ + { + "shape":"ResourceNotFoundException", + "error":{"httpStatusCode":404}, + "exception":true + }, + { + "shape":"InvalidParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"MissingParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"ServiceUnavailableException", + "error":{"httpStatusCode":500}, + "exception":true + } + ] + }, + "AbortVaultLock":{ + "name":"AbortVaultLock", + "http":{ + "method":"DELETE", + "requestUri":"/{accountId}/vaults/{vaultName}/lock-policy", + "responseCode":204 + }, + "input":{"shape":"AbortVaultLockInput"}, + "errors":[ + { + "shape":"ResourceNotFoundException", + "error":{"httpStatusCode":404}, + "exception":true + }, + { + "shape":"InvalidParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"MissingParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"ServiceUnavailableException", + "error":{"httpStatusCode":500}, + "exception":true + } + ] + }, + "AddTagsToVault":{ + "name":"AddTagsToVault", + "http":{ + "method":"POST", + "requestUri":"/{accountId}/vaults/{vaultName}/tags?operation=add", + "responseCode":204 + }, + "input":{"shape":"AddTagsToVaultInput"}, + "errors":[ + { + "shape":"InvalidParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"MissingParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"ResourceNotFoundException", + "error":{"httpStatusCode":404}, + "exception":true + }, + { + "shape":"LimitExceededException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"ServiceUnavailableException", + "error":{"httpStatusCode":500}, + "exception":true + } + ] + }, + "CompleteMultipartUpload":{ + "name":"CompleteMultipartUpload", + "http":{ + "method":"POST", + "requestUri":"/{accountId}/vaults/{vaultName}/multipart-uploads/{uploadId}", + "responseCode":201 + }, + "input":{"shape":"CompleteMultipartUploadInput"}, + "output":{"shape":"ArchiveCreationOutput"}, + "errors":[ + { + "shape":"ResourceNotFoundException", + "error":{"httpStatusCode":404}, + "exception":true + }, + { + "shape":"InvalidParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"MissingParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"ServiceUnavailableException", + "error":{"httpStatusCode":500}, + "exception":true + } + ] + }, + "CompleteVaultLock":{ + "name":"CompleteVaultLock", + "http":{ + "method":"POST", + "requestUri":"/{accountId}/vaults/{vaultName}/lock-policy/{lockId}", + "responseCode":204 + }, + "input":{"shape":"CompleteVaultLockInput"}, + "errors":[ + { + "shape":"ResourceNotFoundException", + "error":{"httpStatusCode":404}, + "exception":true + }, + { + "shape":"InvalidParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"MissingParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"ServiceUnavailableException", + "error":{"httpStatusCode":500}, + "exception":true + } + ] + }, + "CreateVault":{ + "name":"CreateVault", + "http":{ + "method":"PUT", + "requestUri":"/{accountId}/vaults/{vaultName}", + "responseCode":201 + }, + "input":{"shape":"CreateVaultInput"}, + "output":{"shape":"CreateVaultOutput"}, + "errors":[ + { + "shape":"InvalidParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"MissingParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"ServiceUnavailableException", + "error":{"httpStatusCode":500}, + "exception":true + }, + { + "shape":"LimitExceededException", + "error":{"httpStatusCode":400}, + "exception":true + } + ] + }, + "DeleteArchive":{ + "name":"DeleteArchive", + "http":{ + "method":"DELETE", + "requestUri":"/{accountId}/vaults/{vaultName}/archives/{archiveId}", + "responseCode":204 + }, + "input":{"shape":"DeleteArchiveInput"}, + "errors":[ + { + "shape":"ResourceNotFoundException", + "error":{"httpStatusCode":404}, + "exception":true + }, + { + "shape":"InvalidParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"MissingParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"ServiceUnavailableException", + "error":{"httpStatusCode":500}, + "exception":true + } + ] + }, + "DeleteVault":{ + "name":"DeleteVault", + "http":{ + "method":"DELETE", + "requestUri":"/{accountId}/vaults/{vaultName}", + "responseCode":204 + }, + "input":{"shape":"DeleteVaultInput"}, + "errors":[ + { + "shape":"ResourceNotFoundException", + "error":{"httpStatusCode":404}, + "exception":true + }, + { + "shape":"InvalidParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"MissingParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"ServiceUnavailableException", + "error":{"httpStatusCode":500}, + "exception":true + } + ] + }, + "DeleteVaultAccessPolicy":{ + "name":"DeleteVaultAccessPolicy", + "http":{ + "method":"DELETE", + "requestUri":"/{accountId}/vaults/{vaultName}/access-policy", + "responseCode":204 + }, + "input":{"shape":"DeleteVaultAccessPolicyInput"}, + "errors":[ + { + "shape":"ResourceNotFoundException", + "error":{"httpStatusCode":404}, + "exception":true + }, + { + "shape":"InvalidParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"MissingParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"ServiceUnavailableException", + "error":{"httpStatusCode":500}, + "exception":true + } + ] + }, + "DeleteVaultNotifications":{ + "name":"DeleteVaultNotifications", + "http":{ + "method":"DELETE", + "requestUri":"/{accountId}/vaults/{vaultName}/notification-configuration", + "responseCode":204 + }, + "input":{"shape":"DeleteVaultNotificationsInput"}, + "errors":[ + { + "shape":"ResourceNotFoundException", + "error":{"httpStatusCode":404}, + "exception":true + }, + { + "shape":"InvalidParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"MissingParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"ServiceUnavailableException", + "error":{"httpStatusCode":500}, + "exception":true + } + ] + }, + "DescribeJob":{ + "name":"DescribeJob", + "http":{ + "method":"GET", + "requestUri":"/{accountId}/vaults/{vaultName}/jobs/{jobId}" + }, + "input":{"shape":"DescribeJobInput"}, + "output":{"shape":"GlacierJobDescription"}, + "errors":[ + { + "shape":"ResourceNotFoundException", + "error":{"httpStatusCode":404}, + "exception":true + }, + { + "shape":"InvalidParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"MissingParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"ServiceUnavailableException", + "error":{"httpStatusCode":500}, + "exception":true + } + ] + }, + "DescribeVault":{ + "name":"DescribeVault", + "http":{ + "method":"GET", + "requestUri":"/{accountId}/vaults/{vaultName}" + }, + "input":{"shape":"DescribeVaultInput"}, + "output":{"shape":"DescribeVaultOutput"}, + "errors":[ + { + "shape":"ResourceNotFoundException", + "error":{"httpStatusCode":404}, + "exception":true + }, + { + "shape":"InvalidParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"MissingParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"ServiceUnavailableException", + "error":{"httpStatusCode":500}, + "exception":true + } + ] + }, + "GetDataRetrievalPolicy":{ + "name":"GetDataRetrievalPolicy", + "http":{ + "method":"GET", + "requestUri":"/{accountId}/policies/data-retrieval" + }, + "input":{"shape":"GetDataRetrievalPolicyInput"}, + "output":{"shape":"GetDataRetrievalPolicyOutput"}, + "errors":[ + { + "shape":"InvalidParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"MissingParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"ServiceUnavailableException", + "error":{"httpStatusCode":500}, + "exception":true + } + ] + }, + "GetJobOutput":{ + "name":"GetJobOutput", + "http":{ + "method":"GET", + "requestUri":"/{accountId}/vaults/{vaultName}/jobs/{jobId}/output" + }, + "input":{"shape":"GetJobOutputInput"}, + "output":{"shape":"GetJobOutputOutput"}, + "errors":[ + { + "shape":"ResourceNotFoundException", + "error":{"httpStatusCode":404}, + "exception":true + }, + { + "shape":"InvalidParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"MissingParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"ServiceUnavailableException", + "error":{"httpStatusCode":500}, + "exception":true + } + ] + }, + "GetVaultAccessPolicy":{ + "name":"GetVaultAccessPolicy", + "http":{ + "method":"GET", + "requestUri":"/{accountId}/vaults/{vaultName}/access-policy" + }, + "input":{"shape":"GetVaultAccessPolicyInput"}, + "output":{"shape":"GetVaultAccessPolicyOutput"}, + "errors":[ + { + "shape":"ResourceNotFoundException", + "error":{"httpStatusCode":404}, + "exception":true + }, + { + "shape":"InvalidParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"MissingParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"ServiceUnavailableException", + "error":{"httpStatusCode":500}, + "exception":true + } + ] + }, + "GetVaultLock":{ + "name":"GetVaultLock", + "http":{ + "method":"GET", + "requestUri":"/{accountId}/vaults/{vaultName}/lock-policy" + }, + "input":{"shape":"GetVaultLockInput"}, + "output":{"shape":"GetVaultLockOutput"}, + "errors":[ + { + "shape":"ResourceNotFoundException", + "error":{"httpStatusCode":404}, + "exception":true + }, + { + "shape":"InvalidParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"MissingParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"ServiceUnavailableException", + "error":{"httpStatusCode":500}, + "exception":true + } + ] + }, + "GetVaultNotifications":{ + "name":"GetVaultNotifications", + "http":{ + "method":"GET", + "requestUri":"/{accountId}/vaults/{vaultName}/notification-configuration" + }, + "input":{"shape":"GetVaultNotificationsInput"}, + "output":{"shape":"GetVaultNotificationsOutput"}, + "errors":[ + { + "shape":"ResourceNotFoundException", + "error":{"httpStatusCode":404}, + "exception":true + }, + { + "shape":"InvalidParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"MissingParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"ServiceUnavailableException", + "error":{"httpStatusCode":500}, + "exception":true + } + ] + }, + "InitiateJob":{ + "name":"InitiateJob", + "http":{ + "method":"POST", + "requestUri":"/{accountId}/vaults/{vaultName}/jobs", + "responseCode":202 + }, + "input":{"shape":"InitiateJobInput"}, + "output":{"shape":"InitiateJobOutput"}, + "errors":[ + { + "shape":"ResourceNotFoundException", + "error":{"httpStatusCode":404}, + "exception":true + }, + { + "shape":"PolicyEnforcedException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"InvalidParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"MissingParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"ServiceUnavailableException", + "error":{"httpStatusCode":500}, + "exception":true + } + ] + }, + "InitiateMultipartUpload":{ + "name":"InitiateMultipartUpload", + "http":{ + "method":"POST", + "requestUri":"/{accountId}/vaults/{vaultName}/multipart-uploads", + "responseCode":201 + }, + "input":{"shape":"InitiateMultipartUploadInput"}, + "output":{"shape":"InitiateMultipartUploadOutput"}, + "errors":[ + { + "shape":"ResourceNotFoundException", + "error":{"httpStatusCode":404}, + "exception":true + }, + { + "shape":"InvalidParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"MissingParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"ServiceUnavailableException", + "error":{"httpStatusCode":500}, + "exception":true + } + ] + }, + "InitiateVaultLock":{ + "name":"InitiateVaultLock", + "http":{ + "method":"POST", + "requestUri":"/{accountId}/vaults/{vaultName}/lock-policy", + "responseCode":201 + }, + "input":{"shape":"InitiateVaultLockInput"}, + "output":{"shape":"InitiateVaultLockOutput"}, + "errors":[ + { + "shape":"ResourceNotFoundException", + "error":{"httpStatusCode":404}, + "exception":true + }, + { + "shape":"InvalidParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"MissingParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"ServiceUnavailableException", + "error":{"httpStatusCode":500}, + "exception":true + } + ] + }, + "ListJobs":{ + "name":"ListJobs", + "http":{ + "method":"GET", + "requestUri":"/{accountId}/vaults/{vaultName}/jobs" + }, + "input":{"shape":"ListJobsInput"}, + "output":{"shape":"ListJobsOutput"}, + "errors":[ + { + "shape":"ResourceNotFoundException", + "error":{"httpStatusCode":404}, + "exception":true + }, + { + "shape":"InvalidParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"MissingParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"ServiceUnavailableException", + "error":{"httpStatusCode":500}, + "exception":true + } + ] + }, + "ListMultipartUploads":{ + "name":"ListMultipartUploads", + "http":{ + "method":"GET", + "requestUri":"/{accountId}/vaults/{vaultName}/multipart-uploads" + }, + "input":{"shape":"ListMultipartUploadsInput"}, + "output":{"shape":"ListMultipartUploadsOutput"}, + "errors":[ + { + "shape":"ResourceNotFoundException", + "error":{"httpStatusCode":404}, + "exception":true + }, + { + "shape":"InvalidParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"MissingParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"ServiceUnavailableException", + "error":{"httpStatusCode":500}, + "exception":true + } + ] + }, + "ListParts":{ + "name":"ListParts", + "http":{ + "method":"GET", + "requestUri":"/{accountId}/vaults/{vaultName}/multipart-uploads/{uploadId}" + }, + "input":{"shape":"ListPartsInput"}, + "output":{"shape":"ListPartsOutput"}, + "errors":[ + { + "shape":"ResourceNotFoundException", + "error":{"httpStatusCode":404}, + "exception":true + }, + { + "shape":"InvalidParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"MissingParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"ServiceUnavailableException", + "error":{"httpStatusCode":500}, + "exception":true + } + ] + }, + "ListTagsForVault":{ + "name":"ListTagsForVault", + "http":{ + "method":"GET", + "requestUri":"/{accountId}/vaults/{vaultName}/tags" + }, + "input":{"shape":"ListTagsForVaultInput"}, + "output":{"shape":"ListTagsForVaultOutput"}, + "errors":[ + { + "shape":"InvalidParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"MissingParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"ResourceNotFoundException", + "error":{"httpStatusCode":404}, + "exception":true + }, + { + "shape":"ServiceUnavailableException", + "error":{"httpStatusCode":500}, + "exception":true + } + ] + }, + "ListVaults":{ + "name":"ListVaults", + "http":{ + "method":"GET", + "requestUri":"/{accountId}/vaults" + }, + "input":{"shape":"ListVaultsInput"}, + "output":{"shape":"ListVaultsOutput"}, + "errors":[ + { + "shape":"ResourceNotFoundException", + "error":{"httpStatusCode":404}, + "exception":true + }, + { + "shape":"InvalidParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"MissingParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"ServiceUnavailableException", + "error":{"httpStatusCode":500}, + "exception":true + } + ] + }, + "RemoveTagsFromVault":{ + "name":"RemoveTagsFromVault", + "http":{ + "method":"POST", + "requestUri":"/{accountId}/vaults/{vaultName}/tags?operation=remove", + "responseCode":204 + }, + "input":{"shape":"RemoveTagsFromVaultInput"}, + "errors":[ + { + "shape":"InvalidParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"MissingParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"ResourceNotFoundException", + "error":{"httpStatusCode":404}, + "exception":true + }, + { + "shape":"ServiceUnavailableException", + "error":{"httpStatusCode":500}, + "exception":true + } + ] + }, + "SetDataRetrievalPolicy":{ + "name":"SetDataRetrievalPolicy", + "http":{ + "method":"PUT", + "requestUri":"/{accountId}/policies/data-retrieval", + "responseCode":204 + }, + "input":{"shape":"SetDataRetrievalPolicyInput"}, + "errors":[ + { + "shape":"InvalidParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"MissingParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"ServiceUnavailableException", + "error":{"httpStatusCode":500}, + "exception":true + } + ] + }, + "SetVaultAccessPolicy":{ + "name":"SetVaultAccessPolicy", + "http":{ + "method":"PUT", + "requestUri":"/{accountId}/vaults/{vaultName}/access-policy", + "responseCode":204 + }, + "input":{"shape":"SetVaultAccessPolicyInput"}, + "errors":[ + { + "shape":"ResourceNotFoundException", + "error":{"httpStatusCode":404}, + "exception":true + }, + { + "shape":"InvalidParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"MissingParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"ServiceUnavailableException", + "error":{"httpStatusCode":500}, + "exception":true + } + ] + }, + "SetVaultNotifications":{ + "name":"SetVaultNotifications", + "http":{ + "method":"PUT", + "requestUri":"/{accountId}/vaults/{vaultName}/notification-configuration", + "responseCode":204 + }, + "input":{"shape":"SetVaultNotificationsInput"}, + "errors":[ + { + "shape":"ResourceNotFoundException", + "error":{"httpStatusCode":404}, + "exception":true + }, + { + "shape":"InvalidParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"MissingParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"ServiceUnavailableException", + "error":{"httpStatusCode":500}, + "exception":true + } + ] + }, + "UploadArchive":{ + "name":"UploadArchive", + "http":{ + "method":"POST", + "requestUri":"/{accountId}/vaults/{vaultName}/archives", + "responseCode":201 + }, + "input":{"shape":"UploadArchiveInput"}, + "output":{"shape":"ArchiveCreationOutput"}, + "errors":[ + { + "shape":"ResourceNotFoundException", + "error":{"httpStatusCode":404}, + "exception":true + }, + { + "shape":"InvalidParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"MissingParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"RequestTimeoutException", + "error":{"httpStatusCode":408}, + "exception":true + }, + { + "shape":"ServiceUnavailableException", + "error":{"httpStatusCode":500}, + "exception":true + } + ] + }, + "UploadMultipartPart":{ + "name":"UploadMultipartPart", + "http":{ + "method":"PUT", + "requestUri":"/{accountId}/vaults/{vaultName}/multipart-uploads/{uploadId}", + "responseCode":204 + }, + "input":{"shape":"UploadMultipartPartInput"}, + "output":{"shape":"UploadMultipartPartOutput"}, + "errors":[ + { + "shape":"ResourceNotFoundException", + "error":{"httpStatusCode":404}, + "exception":true + }, + { + "shape":"InvalidParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"MissingParameterValueException", + "error":{"httpStatusCode":400}, + "exception":true + }, + { + "shape":"RequestTimeoutException", + "error":{"httpStatusCode":408}, + "exception":true + }, + { + "shape":"ServiceUnavailableException", + "error":{"httpStatusCode":500}, + "exception":true + } + ] + } + }, + "shapes":{ + "AbortMultipartUploadInput":{ + "type":"structure", + "members":{ + "accountId":{ + "shape":"string", + "location":"uri", + "locationName":"accountId" + }, + "vaultName":{ + "shape":"string", + "location":"uri", + "locationName":"vaultName" + }, + "uploadId":{ + "shape":"string", + "location":"uri", + "locationName":"uploadId" + } + }, + "required":[ + "accountId", + "vaultName", + "uploadId" + ] + }, + "AbortVaultLockInput":{ + "type":"structure", + "members":{ + "accountId":{ + "shape":"string", + "location":"uri", + "locationName":"accountId" + }, + "vaultName":{ + "shape":"string", + "location":"uri", + "locationName":"vaultName" + } + }, + "required":[ + "accountId", + "vaultName" + ] + }, + "ActionCode":{ + "type":"string", + "enum":[ + "ArchiveRetrieval", + "InventoryRetrieval" + ] + }, + "AddTagsToVaultInput":{ + "type":"structure", + "members":{ + "accountId":{ + "shape":"string", + "location":"uri", + "locationName":"accountId" + }, + "vaultName":{ + "shape":"string", + "location":"uri", + "locationName":"vaultName" + }, + "Tags":{"shape":"TagMap"} + }, + "required":[ + "accountId", + "vaultName" + ] + }, + "ArchiveCreationOutput":{ + "type":"structure", + "members":{ + "location":{ + "shape":"string", + "location":"header", + "locationName":"Location" + }, + "checksum":{ + "shape":"string", + "location":"header", + "locationName":"x-amz-sha256-tree-hash" + }, + "archiveId":{ + "shape":"string", + "location":"header", + "locationName":"x-amz-archive-id" + } + } + }, + "CompleteMultipartUploadInput":{ + "type":"structure", + "members":{ + "accountId":{ + "shape":"string", + "location":"uri", + "locationName":"accountId" + }, + "vaultName":{ + "shape":"string", + "location":"uri", + "locationName":"vaultName" + }, + "uploadId":{ + "shape":"string", + "location":"uri", + "locationName":"uploadId" + }, + "archiveSize":{ + "shape":"string", + "location":"header", + "locationName":"x-amz-archive-size" + }, + "checksum":{ + "shape":"string", + "location":"header", + "locationName":"x-amz-sha256-tree-hash" + } + }, + "required":[ + "accountId", + "vaultName", + "uploadId" + ] + }, + "CompleteVaultLockInput":{ + "type":"structure", + "members":{ + "accountId":{ + "shape":"string", + "location":"uri", + "locationName":"accountId" + }, + "vaultName":{ + "shape":"string", + "location":"uri", + "locationName":"vaultName" + }, + "lockId":{ + "shape":"string", + "location":"uri", + "locationName":"lockId" + } + }, + "required":[ + "accountId", + "vaultName", + "lockId" + ] + }, + "CreateVaultInput":{ + "type":"structure", + "members":{ + "accountId":{ + "shape":"string", + "location":"uri", + "locationName":"accountId" + }, + "vaultName":{ + "shape":"string", + "location":"uri", + "locationName":"vaultName" + } + }, + "required":[ + "accountId", + "vaultName" + ] + }, + "CreateVaultOutput":{ + "type":"structure", + "members":{ + "location":{ + "shape":"string", + "location":"header", + "locationName":"Location" + } + } + }, + "DataRetrievalPolicy":{ + "type":"structure", + "members":{ + "Rules":{"shape":"DataRetrievalRulesList"} + } + }, + "DataRetrievalRule":{ + "type":"structure", + "members":{ + "Strategy":{"shape":"string"}, + "BytesPerHour":{"shape":"NullableLong"} + } + }, + "DataRetrievalRulesList":{ + "type":"list", + "member":{"shape":"DataRetrievalRule"} + }, + "DateTime":{"type":"string"}, + "DeleteArchiveInput":{ + "type":"structure", + "members":{ + "accountId":{ + "shape":"string", + "location":"uri", + "locationName":"accountId" + }, + "vaultName":{ + "shape":"string", + "location":"uri", + "locationName":"vaultName" + }, + "archiveId":{ + "shape":"string", + "location":"uri", + "locationName":"archiveId" + } + }, + "required":[ + "accountId", + "vaultName", + "archiveId" + ] + }, + "DeleteVaultAccessPolicyInput":{ + "type":"structure", + "members":{ + "accountId":{ + "shape":"string", + "location":"uri", + "locationName":"accountId" + }, + "vaultName":{ + "shape":"string", + "location":"uri", + "locationName":"vaultName" + } + }, + "required":[ + "accountId", + "vaultName" + ] + }, + "DeleteVaultInput":{ + "type":"structure", + "members":{ + "accountId":{ + "shape":"string", + "location":"uri", + "locationName":"accountId" + }, + "vaultName":{ + "shape":"string", + "location":"uri", + "locationName":"vaultName" + } + }, + "required":[ + "accountId", + "vaultName" + ] + }, + "DeleteVaultNotificationsInput":{ + "type":"structure", + "members":{ + "accountId":{ + "shape":"string", + "location":"uri", + "locationName":"accountId" + }, + "vaultName":{ + "shape":"string", + "location":"uri", + "locationName":"vaultName" + } + }, + "required":[ + "accountId", + "vaultName" + ] + }, + "DescribeJobInput":{ + "type":"structure", + "members":{ + "accountId":{ + "shape":"string", + "location":"uri", + "locationName":"accountId" + }, + "vaultName":{ + "shape":"string", + "location":"uri", + "locationName":"vaultName" + }, + "jobId":{ + "shape":"string", + "location":"uri", + "locationName":"jobId" + } + }, + "required":[ + "accountId", + "vaultName", + "jobId" + ] + }, + "DescribeVaultInput":{ + "type":"structure", + "members":{ + "accountId":{ + "shape":"string", + "location":"uri", + "locationName":"accountId" + }, + "vaultName":{ + "shape":"string", + "location":"uri", + "locationName":"vaultName" + } + }, + "required":[ + "accountId", + "vaultName" + ] + }, + "DescribeVaultOutput":{ + "type":"structure", + "members":{ + "VaultARN":{"shape":"string"}, + "VaultName":{"shape":"string"}, + "CreationDate":{"shape":"string"}, + "LastInventoryDate":{"shape":"string"}, + "NumberOfArchives":{"shape":"long"}, + "SizeInBytes":{"shape":"long"} + } + }, + "GetDataRetrievalPolicyInput":{ + "type":"structure", + "members":{ + "accountId":{ + "shape":"string", + "location":"uri", + "locationName":"accountId" + } + }, + "required":["accountId"] + }, + "GetDataRetrievalPolicyOutput":{ + "type":"structure", + "members":{ + "Policy":{"shape":"DataRetrievalPolicy"} + } + }, + "GetJobOutputInput":{ + "type":"structure", + "members":{ + "accountId":{ + "shape":"string", + "location":"uri", + "locationName":"accountId" + }, + "vaultName":{ + "shape":"string", + "location":"uri", + "locationName":"vaultName" + }, + "jobId":{ + "shape":"string", + "location":"uri", + "locationName":"jobId" + }, + "range":{ + "shape":"string", + "location":"header", + "locationName":"Range" + } + }, + "required":[ + "accountId", + "vaultName", + "jobId" + ] + }, + "GetJobOutputOutput":{ + "type":"structure", + "members":{ + "body":{"shape":"Stream"}, + "checksum":{ + "shape":"string", + "location":"header", + "locationName":"x-amz-sha256-tree-hash" + }, + "status":{ + "shape":"httpstatus", + "location":"statusCode" + }, + "contentRange":{ + "shape":"string", + "location":"header", + "locationName":"Content-Range" + }, + "acceptRanges":{ + "shape":"string", + "location":"header", + "locationName":"Accept-Ranges" + }, + "contentType":{ + "shape":"string", + "location":"header", + "locationName":"Content-Type" + }, + "archiveDescription":{ + "shape":"string", + "location":"header", + "locationName":"x-amz-archive-description" + } + }, + "payload":"body" + }, + "GetVaultAccessPolicyInput":{ + "type":"structure", + "members":{ + "accountId":{ + "shape":"string", + "location":"uri", + "locationName":"accountId" + }, + "vaultName":{ + "shape":"string", + "location":"uri", + "locationName":"vaultName" + } + }, + "required":[ + "accountId", + "vaultName" + ] + }, + "GetVaultAccessPolicyOutput":{ + "type":"structure", + "members":{ + "policy":{"shape":"VaultAccessPolicy"} + }, + "payload":"policy" + }, + "GetVaultLockInput":{ + "type":"structure", + "members":{ + "accountId":{ + "shape":"string", + "location":"uri", + "locationName":"accountId" + }, + "vaultName":{ + "shape":"string", + "location":"uri", + "locationName":"vaultName" + } + }, + "required":[ + "accountId", + "vaultName" + ] + }, + "GetVaultLockOutput":{ + "type":"structure", + "members":{ + "Policy":{"shape":"string"}, + "State":{"shape":"string"}, + "ExpirationDate":{"shape":"string"}, + "CreationDate":{"shape":"string"} + } + }, + "GetVaultNotificationsInput":{ + "type":"structure", + "members":{ + "accountId":{ + "shape":"string", + "location":"uri", + "locationName":"accountId" + }, + "vaultName":{ + "shape":"string", + "location":"uri", + "locationName":"vaultName" + } + }, + "required":[ + "accountId", + "vaultName" + ] + }, + "GetVaultNotificationsOutput":{ + "type":"structure", + "members":{ + "vaultNotificationConfig":{"shape":"VaultNotificationConfig"} + }, + "payload":"vaultNotificationConfig" + }, + "GlacierJobDescription":{ + "type":"structure", + "members":{ + "JobId":{"shape":"string"}, + "JobDescription":{"shape":"string"}, + "Action":{"shape":"ActionCode"}, + "ArchiveId":{"shape":"string"}, + "VaultARN":{"shape":"string"}, + "CreationDate":{"shape":"string"}, + "Completed":{"shape":"boolean"}, + "StatusCode":{"shape":"StatusCode"}, + "StatusMessage":{"shape":"string"}, + "ArchiveSizeInBytes":{"shape":"Size"}, + "InventorySizeInBytes":{"shape":"Size"}, + "SNSTopic":{"shape":"string"}, + "CompletionDate":{"shape":"string"}, + "SHA256TreeHash":{"shape":"string"}, + "ArchiveSHA256TreeHash":{"shape":"string"}, + "RetrievalByteRange":{"shape":"string"}, + "InventoryRetrievalParameters":{"shape":"InventoryRetrievalJobDescription"} + } + }, + "InitiateJobInput":{ + "type":"structure", + "members":{ + "accountId":{ + "shape":"string", + "location":"uri", + "locationName":"accountId" + }, + "vaultName":{ + "shape":"string", + "location":"uri", + "locationName":"vaultName" + }, + "jobParameters":{"shape":"JobParameters"} + }, + "required":[ + "accountId", + "vaultName" + ], + "payload":"jobParameters" + }, + "InitiateJobOutput":{ + "type":"structure", + "members":{ + "location":{ + "shape":"string", + "location":"header", + "locationName":"Location" + }, + "jobId":{ + "shape":"string", + "location":"header", + "locationName":"x-amz-job-id" + } + } + }, + "InitiateMultipartUploadInput":{ + "type":"structure", + "members":{ + "accountId":{ + "shape":"string", + "location":"uri", + "locationName":"accountId" + }, + "vaultName":{ + "shape":"string", + "location":"uri", + "locationName":"vaultName" + }, + "archiveDescription":{ + "shape":"string", + "location":"header", + "locationName":"x-amz-archive-description" + }, + "partSize":{ + "shape":"string", + "location":"header", + "locationName":"x-amz-part-size" + } + }, + "required":[ + "accountId", + "vaultName" + ] + }, + "InitiateMultipartUploadOutput":{ + "type":"structure", + "members":{ + "location":{ + "shape":"string", + "location":"header", + "locationName":"Location" + }, + "uploadId":{ + "shape":"string", + "location":"header", + "locationName":"x-amz-multipart-upload-id" + } + } + }, + "InitiateVaultLockInput":{ + "type":"structure", + "members":{ + "accountId":{ + "shape":"string", + "location":"uri", + "locationName":"accountId" + }, + "vaultName":{ + "shape":"string", + "location":"uri", + "locationName":"vaultName" + }, + "policy":{"shape":"VaultLockPolicy"} + }, + "required":[ + "accountId", + "vaultName" + ], + "payload":"policy" + }, + "InitiateVaultLockOutput":{ + "type":"structure", + "members":{ + "lockId":{ + "shape":"string", + "location":"header", + "locationName":"x-amz-lock-id" + } + } + }, + "InvalidParameterValueException":{ + "type":"structure", + "members":{ + "type":{"shape":"string"}, + "code":{"shape":"string"}, + "message":{"shape":"string"} + }, + "error":{"httpStatusCode":400}, + "exception":true + }, + "InventoryRetrievalJobDescription":{ + "type":"structure", + "members":{ + "Format":{"shape":"string"}, + "StartDate":{"shape":"DateTime"}, + "EndDate":{"shape":"DateTime"}, + "Limit":{"shape":"string"}, + "Marker":{"shape":"string"} + } + }, + "InventoryRetrievalJobInput":{ + "type":"structure", + "members":{ + "StartDate":{"shape":"string"}, + "EndDate":{"shape":"string"}, + "Limit":{"shape":"string"}, + "Marker":{"shape":"string"} + } + }, + "JobList":{ + "type":"list", + "member":{"shape":"GlacierJobDescription"} + }, + "JobParameters":{ + "type":"structure", + "members":{ + "Format":{"shape":"string"}, + "Type":{"shape":"string"}, + "ArchiveId":{"shape":"string"}, + "Description":{"shape":"string"}, + "SNSTopic":{"shape":"string"}, + "RetrievalByteRange":{"shape":"string"}, + "InventoryRetrievalParameters":{"shape":"InventoryRetrievalJobInput"} + } + }, + "LimitExceededException":{ + "type":"structure", + "members":{ + "type":{"shape":"string"}, + "code":{"shape":"string"}, + "message":{"shape":"string"} + }, + "error":{"httpStatusCode":400}, + "exception":true + }, + "ListJobsInput":{ + "type":"structure", + "members":{ + "accountId":{ + "shape":"string", + "location":"uri", + "locationName":"accountId" + }, + "vaultName":{ + "shape":"string", + "location":"uri", + "locationName":"vaultName" + }, + "limit":{ + "shape":"string", + "location":"querystring", + "locationName":"limit" + }, + "marker":{ + "shape":"string", + "location":"querystring", + "locationName":"marker" + }, + "statuscode":{ + "shape":"string", + "location":"querystring", + "locationName":"statuscode" + }, + "completed":{ + "shape":"string", + "location":"querystring", + "locationName":"completed" + } + }, + "required":[ + "accountId", + "vaultName" + ] + }, + "ListJobsOutput":{ + "type":"structure", + "members":{ + "JobList":{"shape":"JobList"}, + "Marker":{"shape":"string"} + } + }, + "ListMultipartUploadsInput":{ + "type":"structure", + "members":{ + "accountId":{ + "shape":"string", + "location":"uri", + "locationName":"accountId" + }, + "vaultName":{ + "shape":"string", + "location":"uri", + "locationName":"vaultName" + }, + "marker":{ + "shape":"string", + "location":"querystring", + "locationName":"marker" + }, + "limit":{ + "shape":"string", + "location":"querystring", + "locationName":"limit" + } + }, + "required":[ + "accountId", + "vaultName" + ] + }, + "ListMultipartUploadsOutput":{ + "type":"structure", + "members":{ + "UploadsList":{"shape":"UploadsList"}, + "Marker":{"shape":"string"} + } + }, + "ListPartsInput":{ + "type":"structure", + "members":{ + "accountId":{ + "shape":"string", + "location":"uri", + "locationName":"accountId" + }, + "vaultName":{ + "shape":"string", + "location":"uri", + "locationName":"vaultName" + }, + "uploadId":{ + "shape":"string", + "location":"uri", + "locationName":"uploadId" + }, + "marker":{ + "shape":"string", + "location":"querystring", + "locationName":"marker" + }, + "limit":{ + "shape":"string", + "location":"querystring", + "locationName":"limit" + } + }, + "required":[ + "accountId", + "vaultName", + "uploadId" + ] + }, + "ListPartsOutput":{ + "type":"structure", + "members":{ + "MultipartUploadId":{"shape":"string"}, + "VaultARN":{"shape":"string"}, + "ArchiveDescription":{"shape":"string"}, + "PartSizeInBytes":{"shape":"long"}, + "CreationDate":{"shape":"string"}, + "Parts":{"shape":"PartList"}, + "Marker":{"shape":"string"} + } + }, + "ListTagsForVaultInput":{ + "type":"structure", + "members":{ + "accountId":{ + "shape":"string", + "location":"uri", + "locationName":"accountId" + }, + "vaultName":{ + "shape":"string", + "location":"uri", + "locationName":"vaultName" + } + }, + "required":[ + "accountId", + "vaultName" + ] + }, + "ListTagsForVaultOutput":{ + "type":"structure", + "members":{ + "Tags":{"shape":"TagMap"} + } + }, + "ListVaultsInput":{ + "type":"structure", + "members":{ + "accountId":{ + "shape":"string", + "location":"uri", + "locationName":"accountId" + }, + "marker":{ + "shape":"string", + "location":"querystring", + "locationName":"marker" + }, + "limit":{ + "shape":"string", + "location":"querystring", + "locationName":"limit" + } + }, + "required":["accountId"] + }, + "ListVaultsOutput":{ + "type":"structure", + "members":{ + "VaultList":{"shape":"VaultList"}, + "Marker":{"shape":"string"} + } + }, + "MissingParameterValueException":{ + "type":"structure", + "members":{ + "type":{"shape":"string"}, + "code":{"shape":"string"}, + "message":{"shape":"string"} + }, + "error":{"httpStatusCode":400}, + "exception":true + }, + "NotificationEventList":{ + "type":"list", + "member":{"shape":"string"} + }, + "NullableLong":{"type":"long"}, + "PartList":{ + "type":"list", + "member":{"shape":"PartListElement"} + }, + "PartListElement":{ + "type":"structure", + "members":{ + "RangeInBytes":{"shape":"string"}, + "SHA256TreeHash":{"shape":"string"} + } + }, + "PolicyEnforcedException":{ + "type":"structure", + "members":{ + "type":{"shape":"string"}, + "code":{"shape":"string"}, + "message":{"shape":"string"} + }, + "error":{"httpStatusCode":400}, + "exception":true + }, + "RemoveTagsFromVaultInput":{ + "type":"structure", + "members":{ + "accountId":{ + "shape":"string", + "location":"uri", + "locationName":"accountId" + }, + "vaultName":{ + "shape":"string", + "location":"uri", + "locationName":"vaultName" + }, + "TagKeys":{"shape":"TagKeyList"} + }, + "required":[ + "accountId", + "vaultName" + ] + }, + "RequestTimeoutException":{ + "type":"structure", + "members":{ + "type":{"shape":"string"}, + "code":{"shape":"string"}, + "message":{"shape":"string"} + }, + "error":{"httpStatusCode":408}, + "exception":true + }, + "ResourceNotFoundException":{ + "type":"structure", + "members":{ + "type":{"shape":"string"}, + "code":{"shape":"string"}, + "message":{"shape":"string"} + }, + "error":{"httpStatusCode":404}, + "exception":true + }, + "ServiceUnavailableException":{ + "type":"structure", + "members":{ + "type":{"shape":"string"}, + "code":{"shape":"string"}, + "message":{"shape":"string"} + }, + "error":{"httpStatusCode":500}, + "exception":true + }, + "SetDataRetrievalPolicyInput":{ + "type":"structure", + "members":{ + "accountId":{ + "shape":"string", + "location":"uri", + "locationName":"accountId" + }, + "Policy":{"shape":"DataRetrievalPolicy"} + }, + "required":["accountId"] + }, + "SetVaultAccessPolicyInput":{ + "type":"structure", + "members":{ + "accountId":{ + "shape":"string", + "location":"uri", + "locationName":"accountId" + }, + "vaultName":{ + "shape":"string", + "location":"uri", + "locationName":"vaultName" + }, + "policy":{"shape":"VaultAccessPolicy"} + }, + "required":[ + "accountId", + "vaultName" + ], + "payload":"policy" + }, + "SetVaultNotificationsInput":{ + "type":"structure", + "members":{ + "accountId":{ + "shape":"string", + "location":"uri", + "locationName":"accountId" + }, + "vaultName":{ + "shape":"string", + "location":"uri", + "locationName":"vaultName" + }, + "vaultNotificationConfig":{"shape":"VaultNotificationConfig"} + }, + "required":[ + "accountId", + "vaultName" + ], + "payload":"vaultNotificationConfig" + }, + "Size":{"type":"long"}, + "StatusCode":{ + "type":"string", + "enum":[ + "InProgress", + "Succeeded", + "Failed" + ] + }, + "Stream":{ + "type":"blob", + "streaming":true + }, + "TagKey":{"type":"string"}, + "TagKeyList":{ + "type":"list", + "member":{"shape":"string"} + }, + "TagMap":{ + "type":"map", + "key":{"shape":"TagKey"}, + "value":{"shape":"TagValue"} + }, + "TagValue":{"type":"string"}, + "UploadArchiveInput":{ + "type":"structure", + "members":{ + "vaultName":{ + "shape":"string", + "location":"uri", + "locationName":"vaultName" + }, + "accountId":{ + "shape":"string", + "location":"uri", + "locationName":"accountId" + }, + "archiveDescription":{ + "shape":"string", + "location":"header", + "locationName":"x-amz-archive-description" + }, + "checksum":{ + "shape":"string", + "location":"header", + "locationName":"x-amz-sha256-tree-hash" + }, + "body":{"shape":"Stream"} + }, + "required":[ + "vaultName", + "accountId" + ], + "payload":"body" + }, + "UploadListElement":{ + "type":"structure", + "members":{ + "MultipartUploadId":{"shape":"string"}, + "VaultARN":{"shape":"string"}, + "ArchiveDescription":{"shape":"string"}, + "PartSizeInBytes":{"shape":"long"}, + "CreationDate":{"shape":"string"} + } + }, + "UploadMultipartPartInput":{ + "type":"structure", + "members":{ + "accountId":{ + "shape":"string", + "location":"uri", + "locationName":"accountId" + }, + "vaultName":{ + "shape":"string", + "location":"uri", + "locationName":"vaultName" + }, + "uploadId":{ + "shape":"string", + "location":"uri", + "locationName":"uploadId" + }, + "checksum":{ + "shape":"string", + "location":"header", + "locationName":"x-amz-sha256-tree-hash" + }, + "range":{ + "shape":"string", + "location":"header", + "locationName":"Content-Range" + }, + "body":{"shape":"Stream"} + }, + "required":[ + "accountId", + "vaultName", + "uploadId" + ], + "payload":"body" + }, + "UploadMultipartPartOutput":{ + "type":"structure", + "members":{ + "checksum":{ + "shape":"string", + "location":"header", + "locationName":"x-amz-sha256-tree-hash" + } + } + }, + "UploadsList":{ + "type":"list", + "member":{"shape":"UploadListElement"} + }, + "VaultAccessPolicy":{ + "type":"structure", + "members":{ + "Policy":{"shape":"string"} + } + }, + "VaultList":{ + "type":"list", + "member":{"shape":"DescribeVaultOutput"} + }, + "VaultLockPolicy":{ + "type":"structure", + "members":{ + "Policy":{"shape":"string"} + } + }, + "VaultNotificationConfig":{ + "type":"structure", + "members":{ + "SNSTopic":{"shape":"string"}, + "Events":{"shape":"NotificationEventList"} + } + }, + "boolean":{"type":"boolean"}, + "httpstatus":{"type":"integer"}, + "long":{"type":"long"}, + "string":{"type":"string"} + } +} \ No newline at end of file diff --git a/test/instance_agent/plugins/codedeploy/command_executor_test.rb b/test/instance_agent/plugins/codedeploy/command_executor_test.rb index ae5183bb..46635e03 100644 --- a/test/instance_agent/plugins/codedeploy/command_executor_test.rb +++ b/test/instance_agent/plugins/codedeploy/command_executor_test.rb @@ -1,7 +1,7 @@ require 'test_helper' require 'certificate_helper' require 'stringio' -require 'aws-sdk-core/s3' +require 'aws-sdk-s3' require 'aws/codedeploy/local/deployer' diff --git a/test/instance_agent/plugins/codedeploy/version_tracking_test.rb b/test/instance_agent/plugins/codedeploy/version_tracking_test.rb index 84429de5..07983a1b 100644 --- a/test/instance_agent/plugins/codedeploy/version_tracking_test.rb +++ b/test/instance_agent/plugins/codedeploy/version_tracking_test.rb @@ -1,7 +1,7 @@ require 'test_helper' require 'certificate_helper' require 'stringio' -require 'aws-sdk-core/s3' +require 'aws-sdk-s3' class CodeDeployVersionTrackingTest < InstanceAgentTestCase context 'CodeDeploy version tracking' do diff --git a/vendor/gems/codedeploy-commands-1.0.0/lib/aws/add_service_wrapper.rb b/vendor/gems/codedeploy-commands-1.0.0/lib/aws/add_service_wrapper.rb new file mode 100644 index 00000000..f46b01fa --- /dev/null +++ b/vendor/gems/codedeploy-commands-1.0.0/lib/aws/add_service_wrapper.rb @@ -0,0 +1,61 @@ +require 'aws-sdk-code-generator' +require 'aws-sdk-core' + +module Aws + + # Registers a new service. + # + # Aws.add_service('SvcName', api: '/path/to/svc.api.json') + # + # Aws::SvcName::Client.new + # #=> # + # + # This implementation is taken from the AwsSdkRubyCodeGenWrapper: + # https://code.amazon.com/packages/AwsSdkRubyCodeGenWrapper/blobs/mainline/--/lib/add_service_wrapper.rb + # + # @param [String] svc_name The name of the service. This will also be + # the namespace under {Aws} unless options[:whitelabel] is true. + # This must be a valid constant name. + # @option options[Required, String,Pathname,Hash] :api A a path to a valid + # Coral2JSON model or a hash of a parsed model. + # @option options[Boolean, nil] :whitelabel If true do not prepend + # "Aws" to the generated module namespace. + # @option options[String, nil] :core_path The path to the aws-sdk-core libs + # if unset it will be inferred from the currently loaded aws-sdk-core. + # @option options[Hash,nil] :waiters + # @option options[Hash,nil] :resources + # @return [Module] Returns the new service module. + def self.add_service(name, options = {}) + api_hash = + case options[:api] + when String,Pathname then JSON.parse(File.read(options[:api])) + when Hash then options[:api] + else raise ArgumentError, 'Missing or invalid api: must be a path to a ' \ + 'valid Coral2JSON model or a hash of a parsed model.' + end + module_name = options[:whitelabel] ? name : "Aws::#{name}" + core_path = options[:core_path] || File.dirname($LOADED_FEATURES.find { |f| f.include? 'aws-sdk-core.rb' }) + + code = AwsSdkCodeGenerator::CodeBuilder.new( + aws_sdk_core_lib_path: core_path, + service: AwsSdkCodeGenerator::Service.new( + name: name, + module_name: module_name, + api: api_hash, + paginators: options[:paginators], + paginators: options[:paginators], + waiters: options[:waiters], + resources: options[:resources], + gem_dependencies: { 'aws-sdk-core' => '3' }, + gem_version: '1.0.0', + ) + ) + begin + Object.module_eval(code.source) + rescue => err + puts(code.source) + raise err + end + Object.const_get(module_name) + end +end diff --git a/vendor/gems/codedeploy-commands-1.0.0/lib/aws/codedeploy_commands.rb b/vendor/gems/codedeploy-commands-1.0.0/lib/aws/codedeploy_commands.rb index b132d641..600800b8 100644 --- a/vendor/gems/codedeploy-commands-1.0.0/lib/aws/codedeploy_commands.rb +++ b/vendor/gems/codedeploy-commands-1.0.0/lib/aws/codedeploy_commands.rb @@ -1,6 +1,7 @@ gem_root = File.dirname(File.dirname(File.dirname(__FILE__))) require 'aws-sdk-core' +require "#{gem_root}/lib/aws/add_service_wrapper" require "#{gem_root}/lib/aws/plugins/certificate_authority" require "#{gem_root}/lib/aws/plugins/deploy_control_endpoint" require "#{gem_root}/lib/aws/plugins/deploy_agent_version" From 4e432b0a463eb94b165aba3f8c1e46bc198c22e9 Mon Sep 17 00:00:00 2001 From: Tanuj Pankaj Date: Sat, 18 Jul 2020 00:48:13 +0000 Subject: [PATCH 03/11] [Simple Version Issue] Fix the gemspec versioning issue with AWS SDK. Prior to this change, agent could not run on an instance due to the conflict of version 2.9 in gemspec This change fixes the said issue by changing the version of AWS SDK in the gemspec. * Unit Tests : N * Integration Tests : N This was tested by installing the agent on an instance and ensuring it was able to run smoothly. --- vendor/specifications/codedeploy-commands-1.0.0.gemspec | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vendor/specifications/codedeploy-commands-1.0.0.gemspec b/vendor/specifications/codedeploy-commands-1.0.0.gemspec index 57ef455e..41fb522e 100644 --- a/vendor/specifications/codedeploy-commands-1.0.0.gemspec +++ b/vendor/specifications/codedeploy-commands-1.0.0.gemspec @@ -19,11 +19,11 @@ Gem::Specification.new do |s| s.specification_version = 3 if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then - s.add_runtime_dependency(%q, ["~> 2.9"]) + s.add_runtime_dependency(%q, ["~> 3"]) else - s.add_dependency(%q, ["~> 2.9"]) + s.add_dependency(%q, ["~> 3"]) end else - s.add_dependency(%q, ["~> 2.9"]) + s.add_dependency(%q, ["~> 3"]) end end From 49c1c450fab6652d0bb5f38eaedde88d4b70064a Mon Sep 17 00:00:00 2001 From: Saketh Undurty Date: Wed, 22 Jul 2020 15:51:49 -0700 Subject: [PATCH 04/11] [IMDSv2 support] Update agent code to use IMDSv2 Prior to this change, when retrieving instance metadata, the agent used IMDSv1. There is an escalation to remove all our dependencies on IMDSv1 and update them to use IMDSv2. This is a large ask because of the security vulnerabilities associated with IMDSv1. This change updates our agent code to use IMDSv2, which uses a session authentication method for retrieving instance metadata. * Unit Tests : [Y] * Integration Tests : [N] * Successfully tested install script and agent by spinning up an ec2 instance, building the rpm file for the agent using RBHAInstaller, and deploying a sample application via CodeDeploy console to that instance. --- Gemfile | 1 + bin/install | 46 ++++++++++-- bin/update | 46 ++++++++++-- codedeploy_agent.gemspec | 2 +- features/step_definitions/common_steps.rb | 1 + .../plugins/codedeploy/command_executor.rb | 1 + lib/instance_metadata.rb | 42 ++++++----- spec/spec_helper.rb | 1 + test/instance_metadata_test.rb | 70 +++++++++---------- 9 files changed, 145 insertions(+), 65 deletions(-) diff --git a/Gemfile b/Gemfile index cabed145..0ae69d18 100644 --- a/Gemfile +++ b/Gemfile @@ -16,6 +16,7 @@ group :test do gem 'fakefs', :require => 'fakefs/safe' gem 'mocha' gem 'rspec' + gem 'webmock', :require => 'webmock/rspec' gem 'shoulda' gem 'shoulda-matchers' gem 'shoulda-context' diff --git a/bin/install b/bin/install index 948e8a70..9a825e19 100755 --- a/bin/install +++ b/bin/install @@ -25,6 +25,43 @@ class Proxy end end +require 'net/http' +require 'json' + +class IMDSV2 + def self.region + doc['region'].strip + end + + private + def self.http_request(request) + Net::HTTP.start('169.254.169.254', 80, :read_timeout => 120, :open_timeout => 120) do |http| + response = http.request(request) + if response.code.to_i != 200 + raise "HTTP error from metadata service: #{response.message}, code #{response.code}" + end + return response.body + end + end + + def self.put_request(path) + request = Net::HTTP::Put.new(path) + request['X-aws-ec2-metadata-token-ttl-seconds'] = '21600' + http_request(request) + end + + def self.get_request(path, token) + request = Net::HTTP::Get.new(path) + request['X-aws-ec2-metadata-token'] = token + http_request(request) + end + + def self.doc + token = put_request('/latest/api/token') + JSON.parse(get_request('/latest/dynamic/instance-identity/document', token).strip) + end +end + log_file_path = "/tmp/codedeploy-agent.update.log" require 'logger' @@ -208,12 +245,9 @@ EOF def get_ec2_metadata_region begin - uri = URI.parse('http://169.254.169.254/latest/dynamic/instance-identity/document') - document_string = uri.read(:read_timeout => 120) - doc = JSON.parse(document_string.strip) - return doc['region'].strip - rescue - @log.warn("Could not get region from EC2 metadata service at '#{uri.to_s}'") + return IMDSV2.region + rescue => error + @log.warn("Could not get region from EC2 metadata service at '#{error.message}'") return nil end end diff --git a/bin/update b/bin/update index 33985bd2..83dcdf6e 100755 --- a/bin/update +++ b/bin/update @@ -25,6 +25,43 @@ class Proxy end end +require 'net/http' +require 'json' + +class IMDSV2 + def self.region + doc['region'].strip + end + + private + def self.http_request(request) + Net::HTTP.start('169.254.169.254', 80, :read_timeout => 120, :open_timeout => 120) do |http| + response = http.request(request) + if response.code.to_i != 200 + raise "HTTP error from metadata service: #{response.message}, code #{response.code}" + end + return response.body + end + end + + def self.put_request(path) + request = Net::HTTP::Put.new(path) + request['X-aws-ec2-metadata-token-ttl-seconds'] = '21600' + http_request(request) + end + + def self.get_request(path, token) + request = Net::HTTP::Get.new(path) + request['X-aws-ec2-metadata-token'] = token + http_request(request) + end + + def self.doc + token = put_request('/latest/api/token') + JSON.parse(get_request('/latest/dynamic/instance-identity/document', token).strip) + end +end + require 'tmpdir' require 'logger' @@ -275,12 +312,9 @@ EOF def get_ec2_metadata_region begin - uri = URI.parse('http://169.254.169.254/latest/dynamic/instance-identity/document') - document_string = uri.read(:read_timeout => 120) - doc = JSON.parse(document_string.strip) - return doc['region'].strip - rescue - @log.warn("Could not get region from EC2 metadata service at '#{uri.to_s}'") + return IMDSV2.region + rescue => error + @log.warn("Could not get region from EC2 metadata service at '#{error.message}'") return nil end end diff --git a/codedeploy_agent.gemspec b/codedeploy_agent.gemspec index 1ccc0dd5..23b50f08 100644 --- a/codedeploy_agent.gemspec +++ b/codedeploy_agent.gemspec @@ -18,7 +18,7 @@ Gem::Specification.new do |spec| spec.add_dependency('logging', '~> 1.8') spec.add_dependency('aws-sdk-core', '~> 3') spec.add_dependency('aws-sdk-code-generator', '~> 0.2.1.pre') - spec.add_dependency('aws-sdk-s3', '~> 1.60.1') + spec.add_dependency('aws-sdk-s3', '~> 1') spec.add_dependency('simple_pid', '~> 0.2.1') spec.add_dependency('docopt', '~> 0.5.0') spec.add_dependency('concurrent-ruby', '~> 1.0.5') diff --git a/features/step_definitions/common_steps.rb b/features/step_definitions/common_steps.rb index 296d5a9a..97508a78 100644 --- a/features/step_definitions/common_steps.rb +++ b/features/step_definitions/common_steps.rb @@ -1,4 +1,5 @@ require 'aws-sdk-core' +require 'aws-sdk-s3' $:.unshift File.join(File.dirname(File.expand_path('../..', __FILE__)), 'features') require 'step_definitions/step_constants' diff --git a/lib/instance_agent/plugins/codedeploy/command_executor.rb b/lib/instance_agent/plugins/codedeploy/command_executor.rb index b27ebfda..99463247 100644 --- a/lib/instance_agent/plugins/codedeploy/command_executor.rb +++ b/lib/instance_agent/plugins/codedeploy/command_executor.rb @@ -1,6 +1,7 @@ require 'openssl' require 'fileutils' require 'aws-sdk-core' +require 'aws-sdk-s3' require 'zlib' require 'zip' require 'instance_metadata' diff --git a/lib/instance_metadata.rb b/lib/instance_metadata.rb index 4559aca0..1defba2b 100644 --- a/lib/instance_metadata.rb +++ b/lib/instance_metadata.rb @@ -14,22 +14,16 @@ def self.host_identifier end def self.partition - http_get('/latest/meta-data/services/partition').strip + get_metadata_wrapper('/latest/meta-data/services/partition').strip end def self.region - doc['region'] + doc['region'].strip end def self.instance_id begin - Net::HTTP.start(IP_ADDRESS, PORT) do |http| - response = http.get('/latest/meta-data/instance-id') - if response.code.to_i != 200 - return nil - end - return response.body - end + get_metadata_wrapper('/latest/meta-data/instance-id') rescue return nil end @@ -39,19 +33,35 @@ class InstanceMetadataError < StandardError end private - def self.http_get(path) - Net::HTTP.start(IP_ADDRESS, PORT, :read_timeout => HTTP_TIMEOUT/2, :open_timeout => HTTP_TIMEOUT/2) do |http| - response = http.get(path) + def self.get_metadata_wrapper(path) + token = put_request('/latest/api/token') + get_request(path, token) + end + + def self.http_request(request) + Net::HTTP.start('169.254.169.254', 80, :read_timeout => 120, :open_timeout => 120) do |http| + response = http.request(request) if response.code.to_i != 200 - InstanceAgent::Log.send(:debug, "HTTP error from metadata service, code #{response.code}") - raise "HTTP error from metadata service, code #{response.code}" + raise "HTTP error from metadata service: #{response.message}, code #{response.code}" end return response.body end end - private + def self.put_request(path) + request = Net::HTTP::Put.new(path) + request['X-aws-ec2-metadata-token-ttl-seconds'] = '21600' + http_request(request) + end + + def self.get_request(path, token) + request = Net::HTTP::Get.new(path) + request['X-aws-ec2-metadata-token'] = token + http_request(request) + end + def self.doc - JSON.parse(http_get('/latest/dynamic/instance-identity/document').strip) + token = put_request('/latest/api/token') + JSON.parse(get_request('/latest/dynamic/instance-identity/document', token).strip) end end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 21e9156b..c779112c 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -1,6 +1,7 @@ # Encoding: UTF-8 # See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration require 'bundler/setup' +require 'webmock/rspec' Bundler.setup RSpec.configure do |config| diff --git a/test/instance_metadata_test.rb b/test/instance_metadata_test.rb index 0821f270..b30c80bd 100644 --- a/test/instance_metadata_test.rb +++ b/test/instance_metadata_test.rb @@ -1,51 +1,50 @@ require 'test_helper' require 'json' +require 'webmock/rspec' +require 'webmock/test_unit' class InstanceMetadataTest < InstanceAgentTestCase + include WebMock::API def self.should_check_status_code(&blk) should 'raise unless status code is 200' do - @instance_doc_response.stubs(:code).returns(503) + stub_request(:get, 'http://169.254.169.254/latest/dynamic/instance-identity/document'). + with(headers: {'X-aws-ec2-metadata-token' => @token}). + to_return(status: 503, body: @instance_document, headers: {}) assert_raise(&blk) end end context 'The instance metadata service' do setup do + WebMock.disable_net_connect!(allow_localhost: true) region = 'us-east-1' account_id = '123456789012' instance_id = 'i-deadbeef' @partition = 'aws' @host_identifier = "arn:#{@partition}:ec2:#{region}:#{account_id}:instance/#{instance_id}" @instance_document = JSON.dump({"accountId" => account_id, "region" => region, "instanceId" => instance_id}) - @http = mock() - @instance_doc_response = mock() - @partition_response = mock() - @partition_response.stubs(:code).returns("200") - @instance_doc_response.stubs(:code).returns("200") - - @http.stubs(:get).with('/latest/meta-data/services/partition').returns(@partition_response) - @http.stubs(:get).with('/latest/dynamic/instance-identity/document').returns(@instance_doc_response) - Net::HTTP.stubs(:start).yields(@http) + @instance_document_region_whitespace = JSON.dump({"accountId" => account_id, "region" => " us-east-1 \t", "instanceId" => instance_id}) + @token = "mock_token" + + stub_request(:put, 'http://169.254.169.254/latest/api/token'). + with(headers: {'X-aws-ec2-metadata-token-ttl-seconds' => '21600'}). + to_return(status: 200, body: @token, headers: {}) + stub_request(:get, 'http://169.254.169.254/latest/meta-data/services/partition'). + with(headers: {'X-aws-ec2-metadata-token' => @token}). + to_return(status: 200, body: @partition, headers: {}) + stub_request(:get, 'http://169.254.169.254/latest/dynamic/instance-identity/document'). + with(headers: {'X-aws-ec2-metadata-token' => @token}). + to_return(status: 200, body: @instance_document, headers: {}) end context 'getting the host identifier' do - setup do - @partition_response.stubs(:body).returns(@partition) - @instance_doc_response.stubs(:body).returns(@instance_document) - end - - should 'connect to the right host' do - Net::HTTP.expects(:start).with('169.254.169.254', 80, :read_timeout => InstanceMetadata::HTTP_TIMEOUT/2, :open_timeout => InstanceMetadata::HTTP_TIMEOUT/2).yields(@http) - InstanceMetadata.host_identifier - end - should 'call the correct URL' do - @http.expects(:get). - with("/latest/dynamic/instance-identity/document"). - returns(@instance_doc_response) InstanceMetadata.host_identifier + assert_requested(:put, 'http://169.254.169.254/latest/api/token', times: 4) + assert_requested(:get, 'http://169.254.169.254/latest/meta-data/services/partition', times: 1) + assert_requested(:get, 'http://169.254.169.254/latest/dynamic/instance-identity/document', times: 3) end should 'return the body' do @@ -53,7 +52,9 @@ def self.should_check_status_code(&blk) end should 'strip whitesace in the body' do - @instance_doc_response.stubs(:body).returns(" \t#{@instance_document} ") + stub_request(:get, 'http://169.254.169.254/latest/dynamic/instance-identity/document'). + with(headers: {'X-aws-ec2-metadata-token' => @token}). + to_return(status: 200, body: " \t#{@instance_document} ", headers: {}) assert_equal(@host_identifier, InstanceMetadata.host_identifier) end @@ -63,26 +64,23 @@ def self.should_check_status_code(&blk) context 'getting the region' do - setup do - @instance_doc_response.stubs(:body).returns(@instance_document) - end - - should 'connect to the right host' do - Net::HTTP.expects(:start).with('169.254.169.254', 80, :read_timeout => InstanceMetadata::HTTP_TIMEOUT/2, :open_timeout => InstanceMetadata::HTTP_TIMEOUT/2).yields(@http) - InstanceMetadata.region - end - should 'call the correct URL' do - @http.expects(:get). - with("/latest/dynamic/instance-identity/document"). - returns(@instance_doc_response) InstanceMetadata.region + assert_requested(:put, 'http://169.254.169.254/latest/api/token', times: 1) + assert_requested(:get, 'http://169.254.169.254/latest/dynamic/instance-identity/document', times: 1) end should 'return the region part of the AZ' do assert_equal("us-east-1", InstanceMetadata.region) end + should 'strip whitesace in the body' do + stub_request(:get, 'http://169.254.169.254/latest/dynamic/instance-identity/document'). + with(headers: {'X-aws-ec2-metadata-token' => @token}). + to_return(status: 200, body: @instance_document_region_whitespace , headers: {}) + assert_equal("us-east-1", InstanceMetadata.region) + end + should_check_status_code { InstanceMetadata.region } end From ce26d857ac68baacb10016f2047820468910b815 Mon Sep 17 00:00:00 2001 From: Tanuj Pankaj Date: Mon, 20 Jul 2020 22:16:14 +0000 Subject: [PATCH 05/11] [Security Dependency Issue] Update Rake and Rubyzip to the correct version due to security issues. Prior to this change, Rake and Rubyzip were at older versions that had issues that alerted the Dependabot on GitHub. This change upgrades those said packages to the newer versions. * Unit Tests : Y * Integration Tests : Y Unit and Integration tests passed with the upgraded versions.` --- codedeploy_agent.gemspec | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/codedeploy_agent.gemspec b/codedeploy_agent.gemspec index 23b50f08..a040063e 100644 --- a/codedeploy_agent.gemspec +++ b/codedeploy_agent.gemspec @@ -14,7 +14,7 @@ Gem::Specification.new do |spec| spec.add_dependency('gli', '~> 2.5') spec.add_dependency('json_pure', '~> 1.6') spec.add_dependency('archive-tar-minitar', '~> 0.5.2') - spec.add_dependency('rubyzip', '~> 1.1.0') + spec.add_dependency('rubyzip', '~> 1.3.0') spec.add_dependency('logging', '~> 1.8') spec.add_dependency('aws-sdk-core', '~> 3') spec.add_dependency('aws-sdk-code-generator', '~> 0.2.1.pre') @@ -23,6 +23,6 @@ Gem::Specification.new do |spec| spec.add_dependency('docopt', '~> 0.5.0') spec.add_dependency('concurrent-ruby', '~> 1.0.5') - spec.add_development_dependency('rake', '~> 10.0') + spec.add_development_dependency('rake', '~> 12.3.3') spec.add_development_dependency('rspec', '~> 3.2.0') end From f16c5a3195b5af300754449535a15b5f1b4d1494 Mon Sep 17 00:00:00 2001 From: Tanuj Pankaj Date: Fri, 17 Jul 2020 19:37:42 +0000 Subject: [PATCH 06/11] [SIM Fix] Fix PID file 0 issue. Prior to this change, an empty PID file would cause the status method to return that the agent was running at PID 0. This change ensures that an empty PID file will return a status of No CodeDeploy Agent Running and clean up the PID file. * Unit Tests : Y * Integration Tests : N --- spec/aws/codedeploy/local/master_spec.rb | 54 +++++++++++++++++++ .../lib/process_manager/master.rb | 2 +- 2 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 spec/aws/codedeploy/local/master_spec.rb diff --git a/spec/aws/codedeploy/local/master_spec.rb b/spec/aws/codedeploy/local/master_spec.rb new file mode 100644 index 00000000..5630de1b --- /dev/null +++ b/spec/aws/codedeploy/local/master_spec.rb @@ -0,0 +1,54 @@ +require 'spec_helper' +require_relative '../../../../vendor/gems/process_manager-0.0.13/lib/process_manager/master' +require 'fileutils' + +describe ProcessManager::Daemon::Master do + describe "check status" do + context "PID file is empty" do + it "status is nil and PID file is deleted" do + # Make directory + file_name = ProcessManager::Daemon::Master.pid_file + + dirname = File.dirname(file_name) + unless File.directory?(dirname) + FileUtils.mkdir_p(dirname) + end + + # Write empty file + out_file = File.new(file_name, "w") + out_file.close + + # Check that status is equal to nil and that the PID file is deleted + # Note: This used to give a status of 0 but we want it to be nil + expect(ProcessManager::Daemon::Master.status).to eq(nil) + expect(File.exist?(file_name)).to eq(false) + + # Clean up directory + FileUtils.remove_dir(dirname) if File.directory?(dirname) + end + end + + context "PID file has a process that is running" do + it "status is the PID number" do + # Make directory + file_name = ProcessManager::Daemon::Master.pid_file + + dirname = File.dirname(file_name) + unless File.directory?(dirname) + FileUtils.mkdir_p(dirname) + end + + # Write empty file + out_file = File.new(file_name, "w") + File.write(file_name, $$) # Using $$ to mock a running process + out_file.close + + expect(ProcessManager::Daemon::Master.status).to eq($$) + + # Clean up and delete the file and directory + File.delete(file_name) if File.exist?(file_name) + FileUtils.remove_dir(dirname) if File.directory?(dirname) + end + end + end +end \ No newline at end of file diff --git a/vendor/gems/process_manager-0.0.13/lib/process_manager/master.rb b/vendor/gems/process_manager-0.0.13/lib/process_manager/master.rb index 42366b58..489cf2c1 100644 --- a/vendor/gems/process_manager-0.0.13/lib/process_manager/master.rb +++ b/vendor/gems/process_manager-0.0.13/lib/process_manager/master.rb @@ -51,7 +51,7 @@ def self.restart def self.status if pid = find_pid - if ProcessManager::process_running?(pid) + if pid > 0 and ProcessManager::process_running?(pid) pid else clean_stale_pid From 44c805ed1b82bec49e1e0f740b8a273f32040fe5 Mon Sep 17 00:00:00 2001 From: Saketh Undurty Date: Wed, 29 Jul 2020 12:26:04 -0700 Subject: [PATCH 07/11] [Agent 1.2.1] Add fallback to IMDSv1 Prior to this change, there was no logic for falling back to IMDSv1 if IMDSv2 calls failed. This change adds fallbacks to IMDSv1 if calls fail. * Unit Tests : [Y] * Integration Tests : Successfully had a CodeDeploy deployment using this agent. --- bin/install | 52 +++++++++++++++++++------------- bin/update | 54 ++++++++++++++++++++-------------- lib/instance_metadata.rb | 36 +++++++++++++++++------ test/instance_metadata_test.rb | 41 +++++++++++++++++++++++++- 4 files changed, 130 insertions(+), 53 deletions(-) diff --git a/bin/install b/bin/install index 9a825e19..cc99d226 100755 --- a/bin/install +++ b/bin/install @@ -25,9 +25,29 @@ class Proxy end end +log_file_path = "/tmp/codedeploy-agent.update.log" + +require 'logger' + +if($stdout.isatty) + # if we are being run in a terminal, log to stdout and the log file. + @log = Logger.new(Proxy.new(File.open(log_file_path, 'a+'), $stdout)) +else + # keep at most 2MB of old logs rotating out 1MB at a time + @log = Logger.new(log_file_path, 2, 1048576) + # make sure anything coming out of ruby ends up in the log file + $stdout.reopen(log_file_path, 'a+') + $stderr.reopen(log_file_path, 'a+') +end + +@log.level = Logger::INFO + require 'net/http' require 'json' +TOKEN_PATH = '/latest/api/token' +DOCUMENT_PATH = '/latest/dynamic/instance-identity/document' + class IMDSV2 def self.region doc['region'].strip @@ -50,35 +70,25 @@ class IMDSV2 http_request(request) end - def self.get_request(path, token) + def self.get_request(path, token = nil) request = Net::HTTP::Get.new(path) - request['X-aws-ec2-metadata-token'] = token + unless token.nil? + request['X-aws-ec2-metadata-token'] = token + end http_request(request) end def self.doc - token = put_request('/latest/api/token') - JSON.parse(get_request('/latest/dynamic/instance-identity/document', token).strip) + begin + token = put_request(TOKEN_PATH) + JSON.parse(get_request(DOCUMENT_PATH, token).strip) + rescue + @log.info("IMDSv2 http request failed, falling back to IMDSv1.") + JSON.parse(get_request(DOCUMENT_PATH).strip) + end end end -log_file_path = "/tmp/codedeploy-agent.update.log" - -require 'logger' - -if($stdout.isatty) - # if we are being run in a terminal, log to stdout and the log file. - @log = Logger.new(Proxy.new(File.open(log_file_path, 'a+'), $stdout)) -else - # keep at most 2MB of old logs rotating out 1MB at a time - @log = Logger.new(log_file_path, 2, 1048576) - # make sure anything coming out of ruby ends up in the log file - $stdout.reopen(log_file_path, 'a+') - $stderr.reopen(log_file_path, 'a+') -end - -@log.level = Logger::INFO - begin require 'fileutils' require 'openssl' diff --git a/bin/update b/bin/update index 83dcdf6e..8a8159da 100755 --- a/bin/update +++ b/bin/update @@ -25,9 +25,30 @@ class Proxy end end +require 'tmpdir' +require 'logger' + +log_file_path = "#{Dir.tmpdir()}/codedeploy-agent.update.log" + +if($stdout.isatty) + # if we are being run in a terminal, log to stdout and the log file. + @log = Logger.new(Proxy.new(File.open(log_file_path, 'a+'), $stdout)) +else + # keep at most 2MB of old logs rotating out 1MB at a time + @log = Logger.new(log_file_path, 2, 1048576) + # make sure anything coming out of ruby ends up in the log file + $stdout.reopen(log_file_path, 'a+') + $stderr.reopen(log_file_path, 'a+') +end + +@log.level = Logger::INFO + require 'net/http' require 'json' +TOKEN_PATH = '/latest/api/token' +DOCUMENT_PATH = '/latest/dynamic/instance-identity/document' + class IMDSV2 def self.region doc['region'].strip @@ -50,36 +71,25 @@ class IMDSV2 http_request(request) end - def self.get_request(path, token) + def self.get_request(path, token = nil) request = Net::HTTP::Get.new(path) - request['X-aws-ec2-metadata-token'] = token + unless token.nil? + request['X-aws-ec2-metadata-token'] = token + end http_request(request) end def self.doc - token = put_request('/latest/api/token') - JSON.parse(get_request('/latest/dynamic/instance-identity/document', token).strip) + begin + token = put_request(TOKEN_PATH) + JSON.parse(get_request(DOCUMENT_PATH, token).strip) + rescue + @log.info("IMDSv2 http request failed, falling back to IMDSv1.") + JSON.parse(get_request(DOCUMENT_PATH).strip) + end end end -require 'tmpdir' -require 'logger' - -log_file_path = "#{Dir.tmpdir()}/codedeploy-agent.update.log" - -if($stdout.isatty) - # if we are being run in a terminal, log to stdout and the log file. - @log = Logger.new(Proxy.new(File.open(log_file_path, 'a+'), $stdout)) -else - # keep at most 2MB of old logs rotating out 1MB at a time - @log = Logger.new(log_file_path, 2, 1048576) - # make sure anything coming out of ruby ends up in the log file - $stdout.reopen(log_file_path, 'a+') - $stderr.reopen(log_file_path, 'a+') -end - -@log.level = Logger::INFO - require 'set' VALID_TYPES = Set.new ['rpm','zypper','deb','msi'] diff --git a/lib/instance_metadata.rb b/lib/instance_metadata.rb index 1defba2b..6c9238cc 100644 --- a/lib/instance_metadata.rb +++ b/lib/instance_metadata.rb @@ -9,12 +9,17 @@ class InstanceMetadata PORT = 80 HTTP_TIMEOUT = 30 + PARTITION_PATH = '/latest/meta-data/services/partition' + INSTANCE_ID_PATH = '/latest/meta-data/instance-id' + TOKEN_PATH = '/latest/api/token' + DOCUMENT_PATH = '/latest/dynamic/instance-identity/document' + def self.host_identifier "arn:#{partition}:ec2:#{doc['region']}:#{doc['accountId']}:instance/#{doc['instanceId']}" end def self.partition - get_metadata_wrapper('/latest/meta-data/services/partition').strip + get_metadata_wrapper(PARTITION_PATH).strip end def self.region @@ -23,7 +28,7 @@ def self.region def self.instance_id begin - get_metadata_wrapper('/latest/meta-data/instance-id') + get_metadata_wrapper(INSTANCE_ID_PATH) rescue return nil end @@ -34,12 +39,18 @@ class InstanceMetadataError < StandardError private def self.get_metadata_wrapper(path) - token = put_request('/latest/api/token') - get_request(path, token) + begin + token = put_request(TOKEN_PATH) + get_request(path, token) + rescue + InstanceAgent::Log.send(:info, "IMDSv2 http request failed, falling back to IMDSv1.") + get_request(path) + end + end def self.http_request(request) - Net::HTTP.start('169.254.169.254', 80, :read_timeout => 120, :open_timeout => 120) do |http| + Net::HTTP.start(IP_ADDRESS, PORT, :read_timeout => 120, :open_timeout => 120) do |http| response = http.request(request) if response.code.to_i != 200 raise "HTTP error from metadata service: #{response.message}, code #{response.code}" @@ -54,14 +65,21 @@ def self.put_request(path) http_request(request) end - def self.get_request(path, token) + def self.get_request(path, token = nil) request = Net::HTTP::Get.new(path) - request['X-aws-ec2-metadata-token'] = token + unless token.nil? + request['X-aws-ec2-metadata-token'] = token + end http_request(request) end def self.doc - token = put_request('/latest/api/token') - JSON.parse(get_request('/latest/dynamic/instance-identity/document', token).strip) + begin + token = put_request(TOKEN_PATH) + JSON.parse(get_request(DOCUMENT_PATH, token).strip) + rescue + InstanceAgent::Log.send(:info, "IMDSv2 http request failed, falling back to IMDSv1.") + JSON.parse(get_request(DOCUMENT_PATH).strip) + end end end diff --git a/test/instance_metadata_test.rb b/test/instance_metadata_test.rb index b30c80bd..09e58ce4 100644 --- a/test/instance_metadata_test.rb +++ b/test/instance_metadata_test.rb @@ -9,7 +9,6 @@ class InstanceMetadataTest < InstanceAgentTestCase def self.should_check_status_code(&blk) should 'raise unless status code is 200' do stub_request(:get, 'http://169.254.169.254/latest/dynamic/instance-identity/document'). - with(headers: {'X-aws-ec2-metadata-token' => @token}). to_return(status: 503, body: @instance_document, headers: {}) assert_raise(&blk) end @@ -51,6 +50,26 @@ def self.should_check_status_code(&blk) assert_equal(@host_identifier, InstanceMetadata.host_identifier) end + should 'return the body if IMDSv2 http request status code is not 200' do + stub_request(:get, 'http://169.254.169.254/latest/dynamic/instance-identity/document'). + with(headers: {'X-aws-ec2-metadata-token' => @token}). + to_return(status: 503, body: @instance_document, headers: {}) + stub_request(:get, 'http://169.254.169.254/latest/dynamic/instance-identity/document'). + with(headers: {'Accept'=>'*/*', 'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', 'User-Agent'=>'Ruby'}). + to_return(status: 200, body: @instance_document, headers: {}) + assert_equal(@host_identifier, InstanceMetadata.host_identifier) + end + + should 'return the body if IMDSv2 http request errors out' do + stub_request(:get, 'http://169.254.169.254/latest/dynamic/instance-identity/document'). + with(headers: {'X-aws-ec2-metadata-token' => @token}). + to_raise(StandardError) + stub_request(:get, 'http://169.254.169.254/latest/dynamic/instance-identity/document'). + with(headers: {'Accept'=>'*/*', 'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', 'User-Agent'=>'Ruby'}). + to_return(status: 200, body: @instance_document, headers: {}) + assert_equal(@host_identifier, InstanceMetadata.host_identifier) + end + should 'strip whitesace in the body' do stub_request(:get, 'http://169.254.169.254/latest/dynamic/instance-identity/document'). with(headers: {'X-aws-ec2-metadata-token' => @token}). @@ -74,6 +93,26 @@ def self.should_check_status_code(&blk) assert_equal("us-east-1", InstanceMetadata.region) end + should 'return the region if IMDSv2 http request status code is not 200' do + stub_request(:get, 'http://169.254.169.254/latest/dynamic/instance-identity/document'). + with(headers: {'X-aws-ec2-metadata-token' => @token}). + to_return(status: 503, body: @instance_document, headers: {}) + stub_request(:get, 'http://169.254.169.254/latest/dynamic/instance-identity/document'). + with(headers: {'Accept'=>'*/*', 'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', 'User-Agent'=>'Ruby'}). + to_return(status: 200, body: @instance_document, headers: {}) + assert_equal("us-east-1", InstanceMetadata.region) + end + + should 'return the region if IMDSv2 http request errors out' do + stub_request(:get, 'http://169.254.169.254/latest/dynamic/instance-identity/document'). + with(headers: {'X-aws-ec2-metadata-token' => @token}). + to_raise(StandardError) + stub_request(:get, 'http://169.254.169.254/latest/dynamic/instance-identity/document'). + with(headers: {'Accept'=>'*/*', 'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', 'User-Agent'=>'Ruby'}). + to_return(status: 200, body: @instance_document, headers: {}) + assert_equal("us-east-1", InstanceMetadata.region) + end + should 'strip whitesace in the body' do stub_request(:get, 'http://169.254.169.254/latest/dynamic/instance-identity/document'). with(headers: {'X-aws-ec2-metadata-token' => @token}). From 2a3718e8d32e4b3cd1be7a59b25f07f14177c0fe Mon Sep 17 00:00:00 2001 From: Saketh Undurty Date: Tue, 4 Aug 2020 10:54:16 -0700 Subject: [PATCH 08/11] [Bug Fix] Remove logging statements in install and update scripts inside IMDSv2 class Prior to this change, the logging statements inside the IMDSv2 class were being nil for some reason, which caused the install and update scripts to fall back to pulling the agent from the us-east-1 default region, which is not the expected behavior. This change removes those log statements so we don't hit that error. * Unit Tests : [Y] * Integration Tests : Manually spun up an instance, caused the IMDSv2 call to fail by changing the endpoint for the token, and the install script worked as expected. --- bin/install | 1 - bin/update | 1 - 2 files changed, 2 deletions(-) diff --git a/bin/install b/bin/install index cc99d226..76217349 100755 --- a/bin/install +++ b/bin/install @@ -83,7 +83,6 @@ class IMDSV2 token = put_request(TOKEN_PATH) JSON.parse(get_request(DOCUMENT_PATH, token).strip) rescue - @log.info("IMDSv2 http request failed, falling back to IMDSv1.") JSON.parse(get_request(DOCUMENT_PATH).strip) end end diff --git a/bin/update b/bin/update index 8a8159da..b4919fed 100755 --- a/bin/update +++ b/bin/update @@ -84,7 +84,6 @@ class IMDSV2 token = put_request(TOKEN_PATH) JSON.parse(get_request(DOCUMENT_PATH, token).strip) rescue - @log.info("IMDSv2 http request failed, falling back to IMDSv1.") JSON.parse(get_request(DOCUMENT_PATH).strip) end end From ec2a3a7c7d25d91f764352bc9f2fa26d0b31273f Mon Sep 17 00:00:00 2001 From: Saketh Undurty Date: Tue, 4 Aug 2020 11:22:18 -0700 Subject: [PATCH 09/11] [Bug Fix] Upgrade aws-sdk-code-generator dependency to 0.2.2.pre Prior to this change, there was a bug in the aws-sdk-code-generator ruby gem that caused it to not work on Windows. The fix for that issue was put up and then a new version of the gem was released that included that. This change updates our dependency on aws-sdk-code-generator to use the now fixed aws-sdk-code-generator gem. * Unit Tests : [Y] * Integration Tests : [N/A] --- codedeploy_agent.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codedeploy_agent.gemspec b/codedeploy_agent.gemspec index a040063e..70a9a386 100644 --- a/codedeploy_agent.gemspec +++ b/codedeploy_agent.gemspec @@ -17,7 +17,7 @@ Gem::Specification.new do |spec| spec.add_dependency('rubyzip', '~> 1.3.0') spec.add_dependency('logging', '~> 1.8') spec.add_dependency('aws-sdk-core', '~> 3') - spec.add_dependency('aws-sdk-code-generator', '~> 0.2.1.pre') + spec.add_dependency('aws-sdk-code-generator', '~> 0.2.2.pre') spec.add_dependency('aws-sdk-s3', '~> 1') spec.add_dependency('simple_pid', '~> 0.2.1') spec.add_dependency('docopt', '~> 0.5.0') From 89fbd4aabb8726e9aa8c1156870af6a6b17c262e Mon Sep 17 00:00:00 2001 From: Kaiwen Sun Date: Wed, 2 Sep 2020 22:31:18 -0700 Subject: [PATCH 10/11] load correct credentials in FileCredentials using v3 sdk --- .gitignore | 1 + lib/instance_agent/file_credentials.rb | 3 +- test/instance_agent/file_credentials_test.rb | 51 +++++++++++++++++-- .../codedeploy/onpremise_config_test.rb | 14 ++++- 4 files changed, 62 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index 1ba9a25b..abe22a85 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ codedeploy-local.*.log deployment/ .idea/ .DS_STORE +*.iml diff --git a/lib/instance_agent/file_credentials.rb b/lib/instance_agent/file_credentials.rb index 18944bf7..2d5b800b 100644 --- a/lib/instance_agent/file_credentials.rb +++ b/lib/instance_agent/file_credentials.rb @@ -14,7 +14,8 @@ def initialize(path) private def refresh - @credentials = Aws::SharedCredentials.new(path: @path) + @credentials = Aws::SharedCredentials.new(path: @path).credentials + raise "Failed to load credentials from path #{@path}" if @credentials.nil? @expiration = Time.new + 1800 end end diff --git a/test/instance_agent/file_credentials_test.rb b/test/instance_agent/file_credentials_test.rb index 634fd072..a25981bf 100644 --- a/test/instance_agent/file_credentials_test.rb +++ b/test/instance_agent/file_credentials_test.rb @@ -1,19 +1,60 @@ require 'test_helper' class FileCredentialsTest < InstanceAgentTestCase - context 'The file credentials' do - should 'pass the path to SharedCredentials' do - credentials = InstanceAgent::FileCredentials.new("/tmp/credentials_path") - Aws::SharedCredentials.expects(:new).with(path: "/tmp/credentials_path") + context 'With the file credentials' do + + access_key_id = "fake-aws-access-key-id" + secret_access_key = "fake-aws-secret-key" + credentials_path = "/tmp/credentials_path" + session_token_1 = "fake-aws-session-token-1" + session_token_2 = "fake-aws-session-token-2" + credential_file_pattern = <<-END +[default] +aws_access_key_id = #{access_key_id} +aws_secret_access_key = #{secret_access_key} +aws_session_token = %s +END + + setup do + File.stubs(:exist?).with(credentials_path).returns(true) + File.stubs(:exist?).with(Not(equals(credentials_path))).returns(false) + File.stubs(:readable?).with(credentials_path).returns(true) + File.expects(:read).with(credentials_path).returns(credential_file_pattern % session_token_2) + File.expects(:read).with(credentials_path).returns(credential_file_pattern % session_token_1) + end + + should 'load and refresh the credentials from the path to SharedCredentials' do + credentials = InstanceAgent::FileCredentials.new(credentials_path) + assert_equal access_key_id, credentials.credentials.access_key_id + assert_equal secret_access_key, credentials.credentials.secret_access_key + assert_equal session_token_1, credentials.credentials.session_token credentials.refresh! + assert_equal access_key_id, credentials.credentials.access_key_id + assert_equal secret_access_key, credentials.credentials.secret_access_key + assert_equal session_token_2, credentials.credentials.session_token end should 'set the refresh time to 30 minutes' do - credentials = InstanceAgent::FileCredentials.new("/tmp/credentials_path") + credentials = InstanceAgent::FileCredentials.new(credentials_path) credentials.refresh! # Around 30 minutes expected_time = Time.now + 1800 assert_in_delta(expected_time, credentials.expiration, 5, "Expiration time did not fall within 5 seconds of expected expiration") end end + + context 'Without the file credentials' do + + credentials_path = "/tmp/invalid_credentials_path" + + setup do + File.stubs(:exist?).with(credentials_path).returns(false) + end + + should 'raise error when credential file is missing' do + assert_raised_with_message("Failed to load credentials from path #{credentials_path}", RuntimeError) do + InstanceAgent::FileCredentials.new(credentials_path) + end + end + end end diff --git a/test/instance_agent/plugins/codedeploy/onpremise_config_test.rb b/test/instance_agent/plugins/codedeploy/onpremise_config_test.rb index e5f9ab1d..df138eb5 100644 --- a/test/instance_agent/plugins/codedeploy/onpremise_config_test.rb +++ b/test/instance_agent/plugins/codedeploy/onpremise_config_test.rb @@ -73,14 +73,25 @@ class OnPremiseConfigTest < InstanceAgentTestCase end context "config file with session configuration" do + credentials_path = "/etc/codedeploy-agent/conf/.aws_credentials" linux_file = <<-END region: us-east-test iam_session_arn: test:arn - aws_credentials_file: /etc/codedeploy-agent/conf/.aws_credentials + aws_credentials_file: #{credentials_path} END + access_key_id = "fake-access-key-id-#{rand 1000}" + credentials_file = <<-END +[default] +aws_access_key_id = #{access_key_id} +aws_secret_access_key = fake-secret-access-key +aws_session_token = fake-session-token +END setup do File.stubs(:read).with(linux_path).returns(linux_file) + File.stubs(:read).with(credentials_path).returns(credentials_file) + File.stubs(:exist?).with(credentials_path).returns(true) + File.stubs(:readable?).with(credentials_path).returns(true) end should "set the ENV variables correctly" do @@ -88,6 +99,7 @@ class OnPremiseConfigTest < InstanceAgentTestCase assert_equal 'us-east-test', ENV['AWS_REGION'] assert_equal 'test:arn', ENV['AWS_HOST_IDENTIFIER'] assert_equal '/etc/codedeploy-agent/conf/.aws_credentials', ENV['AWS_CREDENTIALS_FILE'] + assert_equal access_key_id, Aws.config[:credentials].credentials.access_key_id end end From 886e8957cfb46e6c24c6a447897f5a3d808bb13b Mon Sep 17 00:00:00 2001 From: Kaiwen Sun Date: Thu, 3 Sep 2020 14:10:08 -0700 Subject: [PATCH 11/11] [Release] bump up agent version to 1.2.1 --- codedeploy_agent.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codedeploy_agent.gemspec b/codedeploy_agent.gemspec index 70a9a386..f03ae183 100644 --- a/codedeploy_agent.gemspec +++ b/codedeploy_agent.gemspec @@ -1,6 +1,6 @@ Gem::Specification.new do |spec| spec.name = 'aws_codedeploy_agent' - spec.version = '1.1.2' + spec.version = '1.2.1' spec.summary = 'Packages AWS CodeDeploy agent libraries' spec.description = 'AWS CodeDeploy agent is responsible for doing the actual work of deploying software on an individual EC2 instance' spec.author = 'Amazon Web Services'