Skip to content

Commit

Permalink
request CVMFS_EXTERNAL_METALINK hosts (if set) before CVMFS_EXTERNAL_…
Browse files Browse the repository at this point in the history 
…URL hosts
  • Loading branch information
@DrDaveD
DrDaveD committed Oct 23, 2024
1 parent a52d91b commit 59b43a6
Show file tree
Hide file tree
Showing 4 changed files with 119 additions and 46 deletions.
138 changes: 98 additions & 40 deletions cvmfs/network/download.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -931,6 +931,7 @@ void DownloadManager::InitializeRequest(JobInfo *info, CURL *handle) {
info->SetHttpCode(-1);
info->SetFollowRedirects(follow_redirects_);
info->SetNumUsedProxies(1);
info->SetNumUsedMetalinks(1);
info->SetNumUsedHosts(1);
info->SetNumRetries(0);
info->SetBackoffMs(0);
Expand Down Expand Up @@ -1010,19 +1011,20 @@ void DownloadManager::InitializeRequest(JobInfo *info, CURL *handle) {
void DownloadManager::CheckHostInfoReset(
std::string typ,
HostInfo &info,
JobInfo *jobinfo)
JobInfo *jobinfo,
time_t now)
{
if (info.timestamp_backup > 0) {
const time_t now = time(NULL);
if (now == 0)
now = time(NULL);
if (static_cast<int64_t>(now) >
static_cast<int64_t>(info.timestamp_backup +
info.reset_after))
static_cast<int64_t>(info.timestamp_backup + info.reset_after))
{
LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
"(manager %s - id %" PRId64 ") "
"switching %s from %s to %s (reset %s)", name_.c_str(),
jobinfo->id(), typ, (*info.chain)[info.current].c_str(),
(*info.chain)[0].c_str(), typ);
jobinfo->id(), typ.c_str(), (*info.chain)[info.current].c_str(),
(*info.chain)[0].c_str(), typ.c_str());
info.current = 0;
info.timestamp_backup = 0;
}
Expand All @@ -1037,6 +1039,7 @@ void DownloadManager::CheckHostInfoReset(
void DownloadManager::SetUrlOptions(JobInfo *info) {
CURL *curl_handle = info->curl_handle();
string url_prefix;
time_t now = 0;

MutexLockGuard m(lock_options_);

Expand All @@ -1053,7 +1056,7 @@ void DownloadManager::SetUrlOptions(JobInfo *info) {
} else { // no sharding policy
// Check if proxy group needs to be reset from backup to primary
if (opt_timestamp_backup_proxies_ > 0) {
const time_t now = time(NULL);
now = time(NULL);
if (static_cast<int64_t>(now) >
static_cast<int64_t>(opt_timestamp_backup_proxies_ +
opt_proxy_groups_reset_after_))
Expand All @@ -1065,7 +1068,8 @@ void DownloadManager::SetUrlOptions(JobInfo *info) {
}
// Check if load-balanced proxies within the group need to be reset
if (opt_timestamp_failover_proxies_ > 0) {
const time_t now = time(NULL);
if (now == 0)
now = time(NULL);
if (static_cast<int64_t>(now) >
static_cast<int64_t>(opt_timestamp_failover_proxies_ +
opt_proxy_groups_reset_after_))
Expand Down Expand Up @@ -1102,8 +1106,8 @@ void DownloadManager::SetUrlOptions(JobInfo *info) {
} // end !sharding

// Check if metalink and host chains need to be reset
CheckHostInfoReset("metalink", opt_metalink_, info);
CheckHostInfoReset("host", opt_metalink_, info);
CheckHostInfoReset("metalink", opt_metalink_, info, now);
CheckHostInfoReset("host", opt_metalink_, info, now);

curl_easy_setopt(curl_handle, CURLOPT_LOW_SPEED_LIMIT, opt_low_speed_limit_);
if (info->proxy() != "DIRECT") {
Expand All @@ -1116,9 +1120,24 @@ void DownloadManager::SetUrlOptions(JobInfo *info) {
if (!opt_dns_server_.empty())
curl_easy_setopt(curl_handle, CURLOPT_DNS_SERVERS, opt_dns_server_.c_str());

if (info->probe_hosts() && opt_host_.chain) {
url_prefix = (*opt_host_.chain)[opt_host_.current];
info->SetCurrentHostChainIndex(opt_host_.current);
if (info->probe_hosts()) {
if (opt_metalink_.chain &&
((opt_metalink_timestamp_link_ == 0) ||
(static_cast<int64_t>((now == 0) ? time(NULL) : now) >
static_cast<int64_t>(opt_metalink_timestamp_link_ +
opt_metalink_.reset_after)))) {
url_prefix = (*opt_metalink_.chain)[opt_metalink_.current];
info->SetCurrentMetalinkChainIndex(opt_metalink_.current);
LogCvmfs(kLogDownload, kLogDebug, "(manager %s - id %" PRId64 ") "
"reading from metalink %d",
name_.c_str(), info->id(), opt_metalink_.current);
} else if (opt_host_.chain) {
url_prefix = (*opt_host_.chain)[opt_host_.current];
info->SetCurrentHostChainIndex(opt_host_.current);
LogCvmfs(kLogDownload, kLogDebug, "(manager %s - id %" PRId64 ") "
"reading from host %d",
name_.c_str(), info->id(), opt_host_.current);
}
}

string url = url_prefix + *(info->url());
Expand Down Expand Up @@ -1493,7 +1512,21 @@ bool DownloadManager::VerifyAndFinalize(const int curl_error, JobInfo *info) {
break;
}

std::vector<std::string> *host_chain = opt_host_.chain;
bool was_metalink;
std::string typ;
std::vector<std::string> *host_chain;
unsigned char num_used_hosts;
if (info->current_metalink_chain_index() >= 0) {
was_metalink = true;
typ = "metalink";
host_chain = opt_metalink_.chain;
num_used_hosts = info->num_used_metalinks();
} else {
was_metalink = false;
typ = "host";
host_chain = opt_host_.chain;
num_used_hosts = info->num_used_hosts();
}

// Determination if download should be repeated
bool try_again = false;
Expand All @@ -1507,8 +1540,8 @@ bool DownloadManager::VerifyAndFinalize(const int curl_error, JobInfo *info) {
// Make it a host failure
LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
"(manager '%s' - id %" PRId64 ") "
"data corruption with no-cache header, try another host",
name_.c_str(), info->id());
"data corruption with no-cache header, try another %s",
name_.c_str(), info->id(), typ.c_str());

info->SetErrorCode(kFailHostHttp);
}
Expand All @@ -1518,7 +1551,7 @@ bool DownloadManager::VerifyAndFinalize(const int curl_error, JobInfo *info) {
IsHostTransferError(info->error_code()) ||
(info->error_code() == kFailHostHttp)) &&
info->probe_hosts() &&
host_chain && (info->num_used_hosts() < host_chain->size()))
host_chain && (num_used_hosts < host_chain->size()))
)
{
try_again = true;
Expand All @@ -1538,8 +1571,7 @@ bool DownloadManager::VerifyAndFinalize(const int curl_error, JobInfo *info) {
if (!same_url_retry && (info->num_used_proxies() >= opt_num_proxies_)) {
// Check if this can be made a host fail-over
if (info->probe_hosts() &&
host_chain &&
(info->num_used_hosts() < host_chain->size()))
host_chain && (num_used_hosts < host_chain->size()))
{
// reset proxy group if not already performed by other handle
if (opt_proxy_groups_) {
Expand All @@ -1548,14 +1580,15 @@ bool DownloadManager::VerifyAndFinalize(const int curl_error, JobInfo *info) {
{
opt_proxy_groups_current_ = 0;
opt_timestamp_backup_proxies_ = 0;
RebalanceProxiesUnlocked("reset proxies for host failover");
std::string msg = "reset proxies for " + typ + " failover";
RebalanceProxiesUnlocked(msg);
}
}

// Make it a host failure
LogCvmfs(kLogDownload, kLogDebug,
"(manager '%s' - id %" PRId64 ") make it a host failure",
name_.c_str(), info->id());
"(manager '%s' - id %" PRId64 ") make it a %s failure",
name_.c_str(), info->id(), typ.c_str());
info->SetNumUsedProxies(1);
info->SetErrorCode(kFailHostAfterProxy);
} else {
Expand All @@ -1566,12 +1599,13 @@ bool DownloadManager::VerifyAndFinalize(const int curl_error, JobInfo *info) {
"(manager '%s' - id %" PRId64 ") "
"VerifyAndFinalize() would fail the download here. "
"Instead switch proxy and retry download. "
"info->probe_hosts=%d host_chain=%p info->num_used_hosts=%d "
"typ=%s "
"info->probe_hosts=%d host_chain=%p num_used_hosts=%d "
"host_chain->size()=%lu same_url_retry=%d "
"info->num_used_proxies=%d opt_num_proxies_=%d",
name_.c_str(), info->id(),
name_.c_str(), info->id(), typ.c_str(),
static_cast<int>(info->probe_hosts()),
host_chain, info->num_used_hosts(),
host_chain, num_used_hosts,
host_chain ?
host_chain->size() : -1, static_cast<int>(same_url_retry),
info->num_used_proxies(), opt_num_proxies_);
Expand Down Expand Up @@ -1659,8 +1693,13 @@ bool DownloadManager::VerifyAndFinalize(const int curl_error, JobInfo *info) {
}
if (switch_host) {
ReleaseCredential(info);
SwitchHost(info);
info->SetNumUsedHosts(info->num_used_hosts() + 1);
if (was_metalink) {
SwitchMetalink(info);
info->SetNumUsedMetalinks(num_used_hosts + 1);
} else {
SwitchHost(info);
info->SetNumUsedHosts(num_used_hosts + 1);
}
SetUrlOptions(info);
}
} // end !sharding
Expand Down Expand Up @@ -2110,6 +2149,21 @@ void DownloadManager::SetMetalinkChain(
}


/**
* Retrieves the currently set chain of metalink hosts and the currently
* used metalink host.
*/
void DownloadManager::GetMetalinkInfo(vector<string> *metalink_chain,
unsigned *current_metalink)
{
MutexLockGuard m(lock_options_);
if (opt_metalink_.chain) {
if (current_metalink) {*current_metalink = opt_metalink_.current;}
if (metalink_chain) {*metalink_chain = *opt_metalink_.chain;}
}
}


/**
* Parses a list of ';'-separated hosts for the host chain. The empty string
* removes the host list.
Expand Down Expand Up @@ -2241,18 +2295,22 @@ void DownloadManager::SwitchHostInfo(const std::string typ,
return;
}

if (jobinfo && (typ == "host") &&
(jobinfo->current_host_chain_index() != info.current)) {
// The current_host_chain_index is only used by the probe_hosts
// feature which is not relevant to metalink
LogCvmfs(kLogDownload, kLogDebug,
"(manager '%s' - id %" PRId64 ")"
"don't switch host, "
"last used host: %s, current host: %s",
name_.c_str(), jobinfo->id(),
(*info.chain)[jobinfo->current_host_chain_index()].c_str(),
(*info.chain)[info.current].c_str());
return;
if (jobinfo) {
unsigned lastused;
if (typ == "host")
lastused = jobinfo->current_host_chain_index();
else
lastused = jobinfo->current_metalink_chain_index();
if (lastused != info.current) {
LogCvmfs(kLogDownload, kLogDebug,
"(manager '%s' - id %" PRId64 ")"
"don't switch %s, "
"last used %s: %s, current %s: %s",
name_.c_str(), jobinfo->id(), typ.c_str(),
typ.c_str(), (*info.chain)[lastused].c_str(),
typ.c_str(), (*info.chain)[info.current].c_str());
return;
}
}

string reason = "manually triggered";
Expand All @@ -2271,7 +2329,7 @@ void DownloadManager::SwitchHostInfo(const std::string typ,
perf::Inc(counters_->n_metalink_failover);
}
LogCvmfs(kLogDownload, kLogDebug | kLogSyslogWarn,
"%s switching %s from %s to %s (%s)", info_id.c_str(), typ,
"%s switching %s from %s to %s (%s)", info_id.c_str(), typ.c_str(),
old_host.c_str(), (*info.chain)[info.current].c_str(),
reason.c_str());

Expand Down
7 changes: 6 additions & 1 deletion cvmfs/network/download.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -251,6 +251,11 @@ class DownloadManager { // NOLINT(clang-analyzer-optin.performance.Padding)
return 0;
}

unsigned num_metalinks() {
if (opt_metalink_.chain) return opt_metalink_.chain->size();
return 0;
}

dns::IpPreference opt_ip_preference() const {
return opt_ip_preference_;
}
Expand Down Expand Up @@ -341,10 +346,10 @@ class DownloadManager { // NOLINT(clang-analyzer-optin.performance.Padding)

// Metalink
HostInfo opt_metalink_;
time_t opt_metalink_timestamp_link_;

// Host list
HostInfo opt_host_;

/**
* Created by SetHostChain(), filled by probe_hosts. Contains time to get
* .cvmfschecksum in ms. -1 is unprobed, -2 is error.
Expand Down
4 changes: 3 additions & 1 deletion cvmfs/network/jobinfo.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,10 +67,12 @@ void JobInfo::Init() {
error_code_ = kFailOther;
http_code_ = -1;
num_used_proxies_ = 0;
num_used_metalinks_ = 0;
num_used_hosts_ = 0;
num_retries_ = 0;
backoff_ms_ = 0;
current_host_chain_index_ = 0;
current_metalink_chain_index_ = -1;
current_host_chain_index_ = -1;

allow_failure_ = false;

Expand Down
16 changes: 12 additions & 4 deletions cvmfs/network/jobinfo.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,10 +104,12 @@ class JobInfo {
Failures error_code_;
int http_code_;
unsigned char num_used_proxies_;
unsigned char num_used_metalinks_;
unsigned char num_used_hosts_;
unsigned char num_retries_;
unsigned backoff_ms_;
unsigned int current_host_chain_index_;
int current_metalink_chain_index_;
int current_host_chain_index_;

// Don't fail-over proxies on download errors. default = false
bool allow_failure_;
Expand Down Expand Up @@ -202,11 +204,13 @@ class JobInfo {
Failures error_code() const { return error_code_; }
int http_code() const { return http_code_; }
unsigned char num_used_proxies() const { return num_used_proxies_; }
unsigned char num_used_metalinks() const { return num_used_metalinks_; }
unsigned char num_used_hosts() const { return num_used_hosts_; }
unsigned char num_retries() const { return num_retries_; }
unsigned backoff_ms() const { return backoff_ms_; }
unsigned int current_host_chain_index() const {
return current_host_chain_index_; }
int current_metalink_chain_index() const {
return current_metalink_chain_index_; }
int current_host_chain_index() const { return current_host_chain_index_; }

bool allow_failure() const { return allow_failure_; }
int64_t id() const { return id_; }
Expand Down Expand Up @@ -254,11 +258,15 @@ class JobInfo {
void SetHttpCode(int http_code) { http_code_ = http_code; }
void SetNumUsedProxies(unsigned char num_used_proxies)
{ num_used_proxies_ = num_used_proxies; }
void SetNumUsedMetalinks(unsigned char num_used_metalinks)
{ num_used_metalinks_ = num_used_metalinks; }
void SetNumUsedHosts(unsigned char num_used_hosts)
{ num_used_hosts_ = num_used_hosts; }
void SetNumRetries(unsigned char num_retries) { num_retries_ = num_retries; }
void SetBackoffMs(unsigned backoff_ms) { backoff_ms_ = backoff_ms; }
void SetCurrentHostChainIndex(unsigned int current_host_chain_index)
void SetCurrentMetalinkChainIndex(int current_metalink_chain_index)
{ current_metalink_chain_index_ = current_metalink_chain_index; }
void SetCurrentHostChainIndex(int current_host_chain_index)
{ current_host_chain_index_ = current_host_chain_index; }

void SetAllowFailure(bool allow_failure) { allow_failure_ = allow_failure; }
Expand Down

0 comments on commit 59b43a6

Please sign in to comment.