Skip to content

Commit

Permalink
Fixes #37302 - push containers through Katello (Katello#10952)
Browse files Browse the repository at this point in the history
  • Loading branch information
@ianballou
ianballou authored Apr 10, 2024
1 parent 58d2cdb commit ac91651
Show file tree
Hide file tree
Showing 6 changed files with 178 additions and 216 deletions.
175 changes: 51 additions & 124 deletions app/controllers/katello/api/registry/registry_proxies_controller.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,16 @@ class Api::Registry::RegistryProxiesController < Api::V2::ApiController
before_action :disable_strong_params
before_action :confirm_settings
before_action :confirm_push_settings, only: [:start_upload_blob, :upload_blob, :finish_upload_blob,
:chunk_upload_blob, :push_manifest]
:push_manifest]
skip_before_action :authorize
before_action :optional_authorize, only: [:token, :catalog]
before_action :registry_authorize, except: [:token, :v1_search, :catalog]
before_action :authorize_repository_read, only: [:pull_manifest, :tags_list]
before_action :authorize_repository_write, only: [:push_manifest]
# TODO: authorize_repository_write commented out until Katello indexes Pulp container push repositories.
# before_action :authorize_repository_write, only: [:start_upload_blob, :upload_blob, :finish_upload_blob,
# :push_manifest]
skip_before_action :check_media_type, only: [:start_upload_blob, :upload_blob, :finish_upload_blob,
:chunk_upload_blob, :push_manifest]
:push_manifest]

wrap_parameters false

Expand Down Expand Up @@ -182,15 +184,8 @@ def pull_manifest
end

def check_blob
begin
r = Resources::Registry::Proxy.get(@_request.fullpath, 'Accept' => request.headers['Accept'])
response.header['Content-Length'] = "#{r.body.size}"
rescue RestClient::NotFound
digest_file = tmp_file("#{params[:digest][7..-1]}.tar")
raise unless File.exist? digest_file
response.header['Content-Length'] = "#{File.size digest_file}"
end
render json: {}
pulp_response = Resources::Registry::Proxy.get(@_request.fullpath, 'Accept' => request.headers['Accept'])
head pulp_response.code
end

def redirect_client
Expand All @@ -210,94 +205,67 @@ def pull_blob
redirect_client { Resources::Registry::Proxy.get(@_request.fullpath, headers, max_redirects: 0) }
end

# FIXME: Reimplement for Pulp 3.
def push_manifest
repository = params[:repository]
tag = params[:tag]

manifest = create_manifest
return if manifest.nil?

begin
files = get_manifest_files(repository, manifest)
return if files.nil?

tar_file = create_tar_file(files, repository, tag)
return if tar_file.nil?

digest = upload_manifest(tar_file)
return if digest.nil?

tag = upload_tag(digest, tag)
return if tag.nil?
ensure
File.delete(tmp_file('manifest.json')) if File.exist? tmp_file('manifest.json')
headers = translated_headers_for_proxy
headers['Content-Type'] = request.headers['Content-Type'] if request.headers['Content-Type']
body = @_request.body.read
pulp_response = Resources::Registry::Proxy.put(@_request.fullpath, body, headers)
pulp_response.headers.each do |key, value|
response.header[key.to_s] = value
end

render json: {}
end

# FIXME: This is referring to a non-existent Pulp 2 server.
# Pulp 3 container push support is needed instead.
def pulp_content
Katello.pulp_server.resources.content
head pulp_response.code
end

def start_upload_blob
uuid = SecureRandom.hex(16)
response.header['Location'] = "#{request_url}/v2/#{params[:repository]}/blobs/uploads/#{uuid}"
response.header['Docker-Upload-UUID'] = uuid
response.header['Range'] = '0-0'
head 202
end
headers = translated_headers_for_proxy
headers['Content-Type'] = request.headers['Content-Type'] if request.headers['Content-Type']
headers['Content-Length'] = request.headers['Content-Length'] if request.headers['Content-Length']
pulp_response = Resources::Registry::Proxy.post(@_request.fullpath, @_request.body, headers)

def status_upload_blob
response.header['Location'] = "#{request_url}/v2/#{params[:repository]}/blobs/uploads/#{params[:uuid]}"
response.header['Range'] = "123"
response.header['Docker-Upload-UUID'] = "123"
render plain: '', status: :no_content
pulp_response.headers.each do |key, value|
response.header[key.to_s] = value
end
head pulp_response.code
end

def chunk_upload_blob
response.header['Location'] = "#{request_url}/v2/#{params[:repository]}/blobs/uploads/#{params[:uuid]}"
render plain: '', status: :accepted
def translated_headers_for_proxy
current_headers = {}
env = request.env.select do |key, _value|
key.match("^HTTP_.*")
end
env.each do |header|
current_headers[header[0].split('_')[1..-1].join('-')] = header[1]
end
current_headers
end

def upload_blob
File.open(tmp_file("#{params[:uuid]}.tar"), 'ab', 0600) do |file|
file.write request.body.read
headers = translated_headers_for_proxy
headers['Content-Type'] = request.headers['Content-Type'] if request.headers['Content-Type']
headers['Content-Range'] = request.headers['Content-Range'] if request.headers['Content-Range']
headers['Content-Length'] = request.headers['Content-Length'] if request.headers['Content-Length']
body = @_request.body.read
pulp_response = Resources::Registry::Proxy.patch(@_request.fullpath, body, headers)

pulp_response.headers.each do |key, value|
response.header[key.to_s] = value
end

# ???? true chunked data?
if request.headers['Content-Range']
render_error 'unprocessable_entity', :status => :unprocessable_entity
end

response.header['Location'] = "#{request_url}/v2/#{params[:repository]}/blobs/uploads/#{params[:uuid]}"
response.header['Range'] = "1-#{request.body.size}"
response.header['Docker-Upload-UUID'] = params[:uuid]
head 204
head pulp_response.code
end

def finish_upload_blob
# error by client if no params[:digest]

uuid_file = tmp_file("#{params[:uuid]}.tar")
digest_file = tmp_file("#{params[:digest][7..-1]}.tar")

File.delete(digest_file) if File.exist? digest_file
File.rename(uuid_file, digest_file)

response.header['Location'] = "#{request_url}/v2/#{params[:repository]}/blobs/#{params[:digest]}"
response.header['Docker-Content-Digest'] = params[:digest]
response.header['Content-Range'] = "1-#{File.size(digest_file)}"
response.header['Content-Length'] = "0"
response.header['Docker-Upload-UUID'] = params[:uuid]
head 201
end
headers = translated_headers_for_proxy
headers['Content-Type'] = request.headers['Content-Type'] if request.headers['Content-Type']
headers['Content-Range'] = request.headers['Content-Range'] if request.headers['Content-Range']
headers['Content-Length'] = request.headers['Content-Length'] if request.headers['Content-Length']
pulp_response = Resources::Registry::Proxy.put(@_request.fullpath, @_request.body, headers)

pulp_response.headers.each do |key, value|
response.header[key.to_s] = value
end

def cancel_upload_blob
render plain: '', status: :ok
head pulp_response.code
end

def ping
Expand Down Expand Up @@ -411,47 +379,6 @@ def create_tar_file(files, repository, tag)
tar_file
end

# FIXME: Reimplement for Pulp 3.
def upload_manifest(tar_file)
upload_id = pulp_content.create_upload_request['upload_id']
filename = tmp_file(tar_file)
uploads = []

File.open(filename, 'rb') do |file|
content = file.read
pulp_content.upload_bits(upload_id, 0, content)

uploads << {
id: upload_id,
name: filename,
size: file.size,
checksum: Digest::SHA256.hexdigest(content)
}
end

File.delete(filename)
task = sync_task(::Actions::Katello::Repository::ImportUpload,
@repository, uploads, generate_metadata: true, sync_capsule: true)
task.output['upload_results'][0]['digest']
ensure
pulp_content.delete_upload_request(upload_id) if upload_id
end

# FIXME: Reimplement for Pulp 3.
def upload_tag(digest, tag)
upload_id = pulp_content.create_upload_request['upload_id']
uploads = [{
id: upload_id,
name: tag,
digest: digest
}]
sync_task(::Actions::Katello::Repository::ImportUpload, @repository, uploads,
:generate_metadata => true, :sync_capsule => true)
tag
ensure
pulp_content.delete_upload_request(upload_id) if upload_id
end

def tmp_dir
"#{Rails.root}/tmp"
end
Expand Down
7 changes: 6 additions & 1 deletion app/lib/katello/http_resource.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ def []=(key, value)
get: Net::HTTP::Get,
post: Net::HTTP::Post,
put: Net::HTTP::Put,
patch: Net::HTTP::Patch,
delete: Net::HTTP::Delete
}.freeze

Expand Down Expand Up @@ -84,7 +85,11 @@ def process_response(resp)
def issue_request(method:, path:, headers: {}, payload: nil)
logger.debug("Resource #{method.upcase} request: #{path}")
logger.debug "Headers: #{headers.to_json}"
logger.debug "Body: #{filter_sensitive_data(payload.to_json)}"
begin
logger.debug "Body: #{filter_sensitive_data(payload.to_json)}"
rescue JSON::GeneratorError
logger.debug "Body: Error: could not render payload as json"
end

client = rest_client(REQUEST_MAP[method], method, path)
args = [method, payload, headers].compact
Expand Down
25 changes: 25 additions & 0 deletions app/lib/katello/resources/registry.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,35 @@ def self.get(path, headers = {:accept => :json}, options = {})
client.options.merge!(options)
client.get(headers)
end

def self.put(path, body, headers)
logger.debug "Sending PUT request to Registry: #{path}"
resource = RegistryResource.load_class
joined_path = resource.prefix.chomp("/") + path
resource.issue_request(method: :put, path: joined_path, headers: headers, payload: body)
end

def self.patch(path, body, headers)
logger.debug "Sending PATCH request to Registry: #{path}"
resource = RegistryResource.load_class
joined_path = resource.prefix.chomp("/") + path
resource.issue_request(method: :patch, path: joined_path, headers: headers, payload: body)
end

def self.post(path, body, headers)
logger.debug "Sending PUT request to Registry: #{path}"
resource = RegistryResource.load_class
joined_path = resource.prefix.chomp("/") + path
resource.issue_request(method: :post, path: joined_path, headers: headers, payload: body)
end
end

class RegistryResource < HttpResource
class << self
def logger
::Foreman::Logging.logger('katello/registry_proxy')
end

def load_class
pulp_primary = ::SmartProxy.pulp_primary
content_app_url = pulp_primary.setting(SmartProxy::PULP3_FEATURE, 'content_app_url')
Expand Down
12 changes: 4 additions & 8 deletions config/routes/api/registry.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,16 +10,12 @@ class ActionDispatch::Routing::Mapper
match '/v2/token' => 'registry_proxies#token', :via => :get
match '/v2/token' => 'registry_proxies#token', :via => :post
match '/v2/*repository/manifests/:tag' => 'registry_proxies#pull_manifest', :via => :get
# Push-related routes are disabled until there is support for pushing to Pulp 3.
# match '/v2/*repository/manifests/:tag' => 'registry_proxies#push_manifest', :via => :put
match '/v2/*repository/manifests/:tag' => 'registry_proxies#push_manifest', :via => :put
match '/v2/*repository/blobs/:digest' => 'registry_proxies#pull_blob', :via => :get
match '/v2/*repository/blobs/:digest' => 'registry_proxies#check_blob', :via => :head
# match '/v2/*repository/blobs/uploads' => 'registry_proxies#start_upload_blob', :via => :post
# match '/v2/*repository/blobs/uploads/:uuid' => 'registry_proxies#chunk_upload_blob', :via => :post
# match '/v2/*repository/blobs/uploads/:uuid' => 'registry_proxies#finish_upload_blob', :via => :put
# match '/v2/*repository/blobs/uploads/:uuid' => 'registry_proxies#upload_blob', :via => :patch
# match '/v2/*repository/blobs/uploads/:uuid' => 'registry_proxies#status_upload_blob', :via => :get
# match '/v2/*repository/blobs/uploads/:uuid' => 'registry_proxies#cancel_upload_blob', :via => :delete
match '/v2/*repository/blobs/uploads' => 'registry_proxies#start_upload_blob', :via => :post
match '/v2/*repository/blobs/uploads/:uuid' => 'registry_proxies#finish_upload_blob', :via => :put
match '/v2/*repository/blobs/uploads/:uuid' => 'registry_proxies#upload_blob', :via => :patch
match '/v2/_catalog' => 'registry_proxies#catalog', :via => :get
match '/v2/*repository/tags/list' => 'registry_proxies#tags_list', :via => :get
match '/v2' => 'registry_proxies#ping', :via => :get
Expand Down
11 changes: 4 additions & 7 deletions lib/katello/permissions/registry_permissions.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,10 @@
'katello/api/registry/registry_proxies/catalog',
'katello/api/registry/registry_proxies/tags_list',
'katello/api/registry/registry_proxies/pull_manifest',
#'katello/api/registry/registry_proxies/push_manifest',
'katello/api/registry/registry_proxies/push_manifest',
'katello/api/registry/registry_proxies/pull_blob',
'katello/api/registry/registry_proxies/check_blob',
#'katello/api/registry/registry_proxies/start_upload_blob',
#'katello/api/registry/registry_proxies/upload_blob',
#'katello/api/registry/registry_proxies/chunk_upload_blob',
#'katello/api/registry/registry_proxies/finish_upload_blob',
'katello/api/registry/registry_proxies/status_upload_blob',
'katello/api/registry/registry_proxies/cancel_upload_blob'
'katello/api/registry/registry_proxies/start_upload_blob',
'katello/api/registry/registry_proxies/upload_blob',
'katello/api/registry/registry_proxies/finish_upload_blob'
]
Loading

0 comments on commit ac91651

Please sign in to comment.