From ac916514d159acbb0466d826b39c7d9567f2cb17 Mon Sep 17 00:00:00 2001 From: Ian Ballou Date: Wed, 10 Apr 2024 12:05:05 -0400 Subject: [PATCH] Fixes #37302 - push containers through Katello (#10952) --- .../registry/registry_proxies_controller.rb | 175 +++++------------- app/lib/katello/http_resource.rb | 7 +- app/lib/katello/resources/registry.rb | 25 +++ config/routes/api/registry.rb | 12 +- .../permissions/registry_permissions.rb | 11 +- .../registry_proxies_controller_test.rb | 164 ++++++++-------- 6 files changed, 178 insertions(+), 216 deletions(-) diff --git a/app/controllers/katello/api/registry/registry_proxies_controller.rb b/app/controllers/katello/api/registry/registry_proxies_controller.rb index 17ff41ee990..28567775c3f 100644 --- a/app/controllers/katello/api/registry/registry_proxies_controller.rb +++ b/app/controllers/katello/api/registry/registry_proxies_controller.rb @@ -4,14 +4,16 @@ class Api::Registry::RegistryProxiesController < Api::V2::ApiController before_action :disable_strong_params before_action :confirm_settings before_action :confirm_push_settings, only: [:start_upload_blob, :upload_blob, :finish_upload_blob, - :chunk_upload_blob, :push_manifest] + :push_manifest] skip_before_action :authorize before_action :optional_authorize, only: [:token, :catalog] before_action :registry_authorize, except: [:token, :v1_search, :catalog] before_action :authorize_repository_read, only: [:pull_manifest, :tags_list] - before_action :authorize_repository_write, only: [:push_manifest] + # TODO: authorize_repository_write commented out until Katello indexes Pulp container push repositories. + # before_action :authorize_repository_write, only: [:start_upload_blob, :upload_blob, :finish_upload_blob, + # :push_manifest] skip_before_action :check_media_type, only: [:start_upload_blob, :upload_blob, :finish_upload_blob, - :chunk_upload_blob, :push_manifest] + :push_manifest] wrap_parameters false @@ -182,15 +184,8 @@ def pull_manifest end def check_blob - begin - r = Resources::Registry::Proxy.get(@_request.fullpath, 'Accept' => request.headers['Accept']) - response.header['Content-Length'] = "#{r.body.size}" - rescue RestClient::NotFound - digest_file = tmp_file("#{params[:digest][7..-1]}.tar") - raise unless File.exist? digest_file - response.header['Content-Length'] = "#{File.size digest_file}" - end - render json: {} + pulp_response = Resources::Registry::Proxy.get(@_request.fullpath, 'Accept' => request.headers['Accept']) + head pulp_response.code end def redirect_client @@ -210,94 +205,67 @@ def pull_blob redirect_client { Resources::Registry::Proxy.get(@_request.fullpath, headers, max_redirects: 0) } end - # FIXME: Reimplement for Pulp 3. def push_manifest - repository = params[:repository] - tag = params[:tag] - - manifest = create_manifest - return if manifest.nil? - - begin - files = get_manifest_files(repository, manifest) - return if files.nil? - - tar_file = create_tar_file(files, repository, tag) - return if tar_file.nil? - - digest = upload_manifest(tar_file) - return if digest.nil? - - tag = upload_tag(digest, tag) - return if tag.nil? - ensure - File.delete(tmp_file('manifest.json')) if File.exist? tmp_file('manifest.json') + headers = translated_headers_for_proxy + headers['Content-Type'] = request.headers['Content-Type'] if request.headers['Content-Type'] + body = @_request.body.read + pulp_response = Resources::Registry::Proxy.put(@_request.fullpath, body, headers) + pulp_response.headers.each do |key, value| + response.header[key.to_s] = value end - - render json: {} - end - - # FIXME: This is referring to a non-existent Pulp 2 server. - # Pulp 3 container push support is needed instead. - def pulp_content - Katello.pulp_server.resources.content + head pulp_response.code end def start_upload_blob - uuid = SecureRandom.hex(16) - response.header['Location'] = "#{request_url}/v2/#{params[:repository]}/blobs/uploads/#{uuid}" - response.header['Docker-Upload-UUID'] = uuid - response.header['Range'] = '0-0' - head 202 - end + headers = translated_headers_for_proxy + headers['Content-Type'] = request.headers['Content-Type'] if request.headers['Content-Type'] + headers['Content-Length'] = request.headers['Content-Length'] if request.headers['Content-Length'] + pulp_response = Resources::Registry::Proxy.post(@_request.fullpath, @_request.body, headers) - def status_upload_blob - response.header['Location'] = "#{request_url}/v2/#{params[:repository]}/blobs/uploads/#{params[:uuid]}" - response.header['Range'] = "123" - response.header['Docker-Upload-UUID'] = "123" - render plain: '', status: :no_content + pulp_response.headers.each do |key, value| + response.header[key.to_s] = value + end + head pulp_response.code end - def chunk_upload_blob - response.header['Location'] = "#{request_url}/v2/#{params[:repository]}/blobs/uploads/#{params[:uuid]}" - render plain: '', status: :accepted + def translated_headers_for_proxy + current_headers = {} + env = request.env.select do |key, _value| + key.match("^HTTP_.*") + end + env.each do |header| + current_headers[header[0].split('_')[1..-1].join('-')] = header[1] + end + current_headers end def upload_blob - File.open(tmp_file("#{params[:uuid]}.tar"), 'ab', 0600) do |file| - file.write request.body.read + headers = translated_headers_for_proxy + headers['Content-Type'] = request.headers['Content-Type'] if request.headers['Content-Type'] + headers['Content-Range'] = request.headers['Content-Range'] if request.headers['Content-Range'] + headers['Content-Length'] = request.headers['Content-Length'] if request.headers['Content-Length'] + body = @_request.body.read + pulp_response = Resources::Registry::Proxy.patch(@_request.fullpath, body, headers) + + pulp_response.headers.each do |key, value| + response.header[key.to_s] = value end - # ???? true chunked data? - if request.headers['Content-Range'] - render_error 'unprocessable_entity', :status => :unprocessable_entity - end - - response.header['Location'] = "#{request_url}/v2/#{params[:repository]}/blobs/uploads/#{params[:uuid]}" - response.header['Range'] = "1-#{request.body.size}" - response.header['Docker-Upload-UUID'] = params[:uuid] - head 204 + head pulp_response.code end def finish_upload_blob - # error by client if no params[:digest] - - uuid_file = tmp_file("#{params[:uuid]}.tar") - digest_file = tmp_file("#{params[:digest][7..-1]}.tar") - - File.delete(digest_file) if File.exist? digest_file - File.rename(uuid_file, digest_file) - - response.header['Location'] = "#{request_url}/v2/#{params[:repository]}/blobs/#{params[:digest]}" - response.header['Docker-Content-Digest'] = params[:digest] - response.header['Content-Range'] = "1-#{File.size(digest_file)}" - response.header['Content-Length'] = "0" - response.header['Docker-Upload-UUID'] = params[:uuid] - head 201 - end + headers = translated_headers_for_proxy + headers['Content-Type'] = request.headers['Content-Type'] if request.headers['Content-Type'] + headers['Content-Range'] = request.headers['Content-Range'] if request.headers['Content-Range'] + headers['Content-Length'] = request.headers['Content-Length'] if request.headers['Content-Length'] + pulp_response = Resources::Registry::Proxy.put(@_request.fullpath, @_request.body, headers) + + pulp_response.headers.each do |key, value| + response.header[key.to_s] = value + end - def cancel_upload_blob - render plain: '', status: :ok + head pulp_response.code end def ping @@ -411,47 +379,6 @@ def create_tar_file(files, repository, tag) tar_file end - # FIXME: Reimplement for Pulp 3. - def upload_manifest(tar_file) - upload_id = pulp_content.create_upload_request['upload_id'] - filename = tmp_file(tar_file) - uploads = [] - - File.open(filename, 'rb') do |file| - content = file.read - pulp_content.upload_bits(upload_id, 0, content) - - uploads << { - id: upload_id, - name: filename, - size: file.size, - checksum: Digest::SHA256.hexdigest(content) - } - end - - File.delete(filename) - task = sync_task(::Actions::Katello::Repository::ImportUpload, - @repository, uploads, generate_metadata: true, sync_capsule: true) - task.output['upload_results'][0]['digest'] - ensure - pulp_content.delete_upload_request(upload_id) if upload_id - end - - # FIXME: Reimplement for Pulp 3. - def upload_tag(digest, tag) - upload_id = pulp_content.create_upload_request['upload_id'] - uploads = [{ - id: upload_id, - name: tag, - digest: digest - }] - sync_task(::Actions::Katello::Repository::ImportUpload, @repository, uploads, - :generate_metadata => true, :sync_capsule => true) - tag - ensure - pulp_content.delete_upload_request(upload_id) if upload_id - end - def tmp_dir "#{Rails.root}/tmp" end diff --git a/app/lib/katello/http_resource.rb b/app/lib/katello/http_resource.rb index c63bb4327d3..3cd0e9d4bec 100644 --- a/app/lib/katello/http_resource.rb +++ b/app/lib/katello/http_resource.rb @@ -38,6 +38,7 @@ def []=(key, value) get: Net::HTTP::Get, post: Net::HTTP::Post, put: Net::HTTP::Put, + patch: Net::HTTP::Patch, delete: Net::HTTP::Delete }.freeze @@ -84,7 +85,11 @@ def process_response(resp) def issue_request(method:, path:, headers: {}, payload: nil) logger.debug("Resource #{method.upcase} request: #{path}") logger.debug "Headers: #{headers.to_json}" - logger.debug "Body: #{filter_sensitive_data(payload.to_json)}" + begin + logger.debug "Body: #{filter_sensitive_data(payload.to_json)}" + rescue JSON::GeneratorError + logger.debug "Body: Error: could not render payload as json" + end client = rest_client(REQUEST_MAP[method], method, path) args = [method, payload, headers].compact diff --git a/app/lib/katello/resources/registry.rb b/app/lib/katello/resources/registry.rb index 61e16d77163..ec85ba805c9 100644 --- a/app/lib/katello/resources/registry.rb +++ b/app/lib/katello/resources/registry.rb @@ -18,10 +18,35 @@ def self.get(path, headers = {:accept => :json}, options = {}) client.options.merge!(options) client.get(headers) end + + def self.put(path, body, headers) + logger.debug "Sending PUT request to Registry: #{path}" + resource = RegistryResource.load_class + joined_path = resource.prefix.chomp("/") + path + resource.issue_request(method: :put, path: joined_path, headers: headers, payload: body) + end + + def self.patch(path, body, headers) + logger.debug "Sending PATCH request to Registry: #{path}" + resource = RegistryResource.load_class + joined_path = resource.prefix.chomp("/") + path + resource.issue_request(method: :patch, path: joined_path, headers: headers, payload: body) + end + + def self.post(path, body, headers) + logger.debug "Sending PUT request to Registry: #{path}" + resource = RegistryResource.load_class + joined_path = resource.prefix.chomp("/") + path + resource.issue_request(method: :post, path: joined_path, headers: headers, payload: body) + end end class RegistryResource < HttpResource class << self + def logger + ::Foreman::Logging.logger('katello/registry_proxy') + end + def load_class pulp_primary = ::SmartProxy.pulp_primary content_app_url = pulp_primary.setting(SmartProxy::PULP3_FEATURE, 'content_app_url') diff --git a/config/routes/api/registry.rb b/config/routes/api/registry.rb index f0ab1309bc2..b68a7c09f16 100644 --- a/config/routes/api/registry.rb +++ b/config/routes/api/registry.rb @@ -10,16 +10,12 @@ class ActionDispatch::Routing::Mapper match '/v2/token' => 'registry_proxies#token', :via => :get match '/v2/token' => 'registry_proxies#token', :via => :post match '/v2/*repository/manifests/:tag' => 'registry_proxies#pull_manifest', :via => :get - # Push-related routes are disabled until there is support for pushing to Pulp 3. - # match '/v2/*repository/manifests/:tag' => 'registry_proxies#push_manifest', :via => :put + match '/v2/*repository/manifests/:tag' => 'registry_proxies#push_manifest', :via => :put match '/v2/*repository/blobs/:digest' => 'registry_proxies#pull_blob', :via => :get match '/v2/*repository/blobs/:digest' => 'registry_proxies#check_blob', :via => :head - # match '/v2/*repository/blobs/uploads' => 'registry_proxies#start_upload_blob', :via => :post - # match '/v2/*repository/blobs/uploads/:uuid' => 'registry_proxies#chunk_upload_blob', :via => :post - # match '/v2/*repository/blobs/uploads/:uuid' => 'registry_proxies#finish_upload_blob', :via => :put - # match '/v2/*repository/blobs/uploads/:uuid' => 'registry_proxies#upload_blob', :via => :patch - # match '/v2/*repository/blobs/uploads/:uuid' => 'registry_proxies#status_upload_blob', :via => :get - # match '/v2/*repository/blobs/uploads/:uuid' => 'registry_proxies#cancel_upload_blob', :via => :delete + match '/v2/*repository/blobs/uploads' => 'registry_proxies#start_upload_blob', :via => :post + match '/v2/*repository/blobs/uploads/:uuid' => 'registry_proxies#finish_upload_blob', :via => :put + match '/v2/*repository/blobs/uploads/:uuid' => 'registry_proxies#upload_blob', :via => :patch match '/v2/_catalog' => 'registry_proxies#catalog', :via => :get match '/v2/*repository/tags/list' => 'registry_proxies#tags_list', :via => :get match '/v2' => 'registry_proxies#ping', :via => :get diff --git a/lib/katello/permissions/registry_permissions.rb b/lib/katello/permissions/registry_permissions.rb index ff46496f4dc..0c61837d667 100644 --- a/lib/katello/permissions/registry_permissions.rb +++ b/lib/katello/permissions/registry_permissions.rb @@ -8,13 +8,10 @@ 'katello/api/registry/registry_proxies/catalog', 'katello/api/registry/registry_proxies/tags_list', 'katello/api/registry/registry_proxies/pull_manifest', - #'katello/api/registry/registry_proxies/push_manifest', + 'katello/api/registry/registry_proxies/push_manifest', 'katello/api/registry/registry_proxies/pull_blob', 'katello/api/registry/registry_proxies/check_blob', - #'katello/api/registry/registry_proxies/start_upload_blob', - #'katello/api/registry/registry_proxies/upload_blob', - #'katello/api/registry/registry_proxies/chunk_upload_blob', - #'katello/api/registry/registry_proxies/finish_upload_blob', - 'katello/api/registry/registry_proxies/status_upload_blob', - 'katello/api/registry/registry_proxies/cancel_upload_blob' + 'katello/api/registry/registry_proxies/start_upload_blob', + 'katello/api/registry/registry_proxies/upload_blob', + 'katello/api/registry/registry_proxies/finish_upload_blob' ] diff --git a/test/controllers/api/registry/registry_proxies_controller_test.rb b/test/controllers/api/registry/registry_proxies_controller_test.rb index b69be160b2d..78ec2b32de2 100644 --- a/test/controllers/api/registry/registry_proxies_controller_test.rb +++ b/test/controllers/api/registry/registry_proxies_controller_test.rb @@ -534,81 +534,93 @@ def setup_permissions end end - # Disabling docker push tests until it is implemented for Pulp 3. - # describe "docker push" do - # it "push manifest - error" do - # @controller.stubs(:authorize_repository_write).returns(true) - # put :push_manifest, params: { repository: 'repository', tag: 'tag' } - # assert_response 500 - # body = JSON.parse(response.body) - # assert_equal "Unsupported schema ", body['error']['message'] - # end - - # it "push manifest - manifest.json exists" do - # File.open("#{Rails.root}/tmp/manifest.json", 'wb', 0600) do |file| - # file.write "empty manifest" - # end - - # @controller.stubs(:authorize_repository_write).returns(true) - # put :push_manifest, params: { repository: 'repository', tag: 'tag' } - # assert_response 422 - # body = JSON.parse(response.body) - # assert_equal "Upload already in progress", body['error']['message'] - # end - - # it "push manifest - success" do - # @repository = katello_repositories(:busybox) - # mock_pulp_server([ - # { name: :create_upload_request, result: { 'upload_id' => 123 }, count: 2 }, - # { name: :delete_upload_request, result: true, count: 2 }, - # { name: :upload_bits, result: true, count: 1 } - # ]) - # @controller.expects(:sync_task) - # .times(2) - # .returns(stub('task', :output => {'upload_results' => [{ 'digest' => 'sha256:1234' }]}), true) - # .with do |action_class, repository, uploads, params| - # assert_equal ::Actions::Katello::Repository::ImportUpload, action_class - # assert_equal @repository, repository - # assert_equal [123], uploads.pluck(:id) - # assert params[:generate_metadata] - # assert params[:sync_capsule] - # end - - # manifest = { - # schemaVersion: 1 - # } - # @controller.stubs(:authorize).returns(true) - # @controller.stubs(:find_readable_repository).returns(@repository) - # @controller.stubs(:find_writable_repository).returns(@repository) - # put :push_manifest, params: { repository: 'repository', tag: 'tag' }, - # body: manifest.to_json - # assert_response 200 - # end - - # it "push manifest - disabled with false" do - # SETTINGS[:katello][:container_image_registry] = {crane_url: 'https://localhost:5000', crane_ca_cert_file: '/etc/pki/katello/certs/katello-default-ca.crt', allow_push: false} - # put :push_manifest, params: { repository: 'repository', tag: 'tag' } - # assert_response 404 - # body = JSON.parse(response.body) - # assert_equal "Registry push not supported", body['error']['message'] - # end - - # it "push manifest - disabled by omission" do - # SETTINGS[:katello][:container_image_registry] = {crane_url: 'https://localhost:5000', crane_ca_cert_file: '/etc/pki/katello/certs/katello-default-ca.crt'} - # put :push_manifest, params: { repository: 'repository', tag: 'tag' } - # assert_response 404 - # body = JSON.parse(response.body) - # assert_equal "Registry push not supported", body['error']['message'] - # end - # end - - # def mock_pulp_server(content_hash) - # content = mock - # content_hash.each do |method| - # content.stubs(method[:name]).times(method[:count]).returns(method[:result]) - # end - # @controller.stubs(:pulp_content).returns(content) - # end + describe 'container push' do + it 'starts a blob upload to Pulp' do + repo_name = 'a repo' + # The content type should be octet-stream, but action controller + # throws a non-existence error since Mime::Type.lookup('application/octet-stream').to_sym is nil. + Resources::Registry::Proxy.expects(:post).with( + "/v2/#{ERB::Util.url_encode(repo_name)}/blobs/uploads", + is_a(StringIO), + has_entries('MOCK-TEST' => 123, + 'Content-Type' => 'application/json', + 'Content-Length' => '2') + ).returns(mock_pulp_response(202, { 'Location' => 'Mars' })) + request.env['HTTP_MOCK_TEST'] = 123 + request.headers['Content-Type'] = 'application/json' + resp = post :start_upload_blob, params: { repository: repo_name } + assert_equal 'Mars', resp.headers['Location'] + assert_equal resp.code, '202' + end + + it 'uploads a blob chunk' do + repo_name = 'repo' + uuid = 'uuid' + # The content type should be octet-stream, but action controller + # throws a non-existence error since Mime::Type.lookup('application/octet-stream').to_sym is nil. + Resources::Registry::Proxy.expects(:patch).with( + "/v2/#{repo_name}/blobs/uploads/#{uuid}", + '{}', + has_entries('MOCK-TEST' => 123, + 'Content-Type' => 'application/json', + 'Content-Range' => 'bytes 500-1000/65989', + 'Content-Length' => '2') + ).returns(mock_pulp_response(202, { 'Location' => 'Mars' })) + request.env['HTTP_MOCK_TEST'] = 123 + request.headers['Content-Type'] = 'application/json' + request.headers['Content-Range'] = 'bytes 500-1000/65989' + request.env['HTTP_MOCK_TEST'] = 123 + resp = patch :upload_blob, params: { repository: repo_name, uuid: uuid } + assert_equal 'Mars', resp.headers['Location'] + assert_equal resp.code, '202' + end + + it 'finishes a blob upload to Pulp' do + repo_name = 'repo' + uuid = 'uuid' + # The content type should be octet-stream, but action controller + # throws a non-existence error since Mime::Type.lookup('application/octet-stream').to_sym is nil. + Resources::Registry::Proxy.expects(:put).with( + "/v2/#{repo_name}/blobs/uploads/#{uuid}", + is_a(StringIO), + has_entries('MOCK-TEST' => 123, + 'Content-Type' => 'application/json', + 'Content-Range' => 'bytes 500-1000/65989', + 'Content-Length' => '2') + ).returns(mock_pulp_response(201, { 'Location' => 'Mars' })) + request.env['HTTP_MOCK_TEST'] = 123 + request.headers['Content-Type'] = 'application/json' + request.headers['Content-Range'] = 'bytes 500-1000/65989' + resp = put :finish_upload_blob, params: { repository: repo_name, uuid: uuid } + assert_equal 'Mars', resp.headers['Location'] + assert_equal resp.code, '201' + end + + it 'pushes a manifest' do + repo_name = 'repo' + tag = 'latest' + # The content type should be application/vnd.oci.image.manifest.v1+json, but action controller + # throws a non-existence error since Mime::Type.lookup('application/octet-stream').to_sym is nil. + Resources::Registry::Proxy.expects(:put).with( + "/v2/#{repo_name}/manifests/#{tag}", + '{}', + has_entries('MOCK-TEST' => 123, + 'Content-Type' => 'application/json') + ).returns(mock_pulp_response(201, { 'Location' => 'Mars' })) + request.env['HTTP_MOCK_TEST'] = 123 + request.headers['Content-Type'] = 'application/json' + resp = put :push_manifest, params: { repository: repo_name, tag: tag } + assert_equal 'Mars', resp.headers['Location'] + assert_equal resp.code, '201' + end + + def mock_pulp_response(code, headers) + mock_response = mock + mock_response.stubs(:code).returns(code) + mock_response.stubs(:headers).returns(headers) + mock_response + end + end + #rubocop:enable Metrics/BlockLength end - #rubocop:enable Metrics/BlockLength end