From 2c3bbd49f71c7f62126771c92d962031a7947c22 Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 10 Nov 2023 13:58:37 +0100 Subject: [PATCH] try to fix update_stats page by switching more to database based data and disabling twitter --- m1-docker-compose.yml | 8 +- public_html/stats.php | 329 ++++++++++++----------------------- update_stats.php | 389 +++++++++++++++++++++--------------------- 3 files changed, 311 insertions(+), 415 deletions(-) diff --git a/m1-docker-compose.yml b/m1-docker-compose.yml index 454e1b7013..043fe410ba 100644 --- a/m1-docker-compose.yml +++ b/m1-docker-compose.yml @@ -1,14 +1,14 @@ version: '3.8' services: db: - image: mariadb + image: mysql ports: - 3306:3306 environment: MYSQL_ROOT_PASSWORD: typist0tulip7console-PIMENTO4cosmic - MYSQL_DATABASE: nfcore - MYSQL_USER: nfcore_admin - MYSQL_PASSWORD: PEBBLY8exhibit_mead1cilium6despise + MYSQL_DATABASE: nfcore_db + MYSQL_USER: nfcore_db_user + MYSQL_PASSWORD: abaci!journey0sylvan*FLUB volumes: - ./.mysql-data/db:/var/lib/mysql web: diff --git a/public_html/stats.php b/public_html/stats.php index 5c711205d8..ecde43b126 100644 --- a/public_html/stats.php +++ b/public_html/stats.php @@ -39,161 +39,92 @@ $twitter_datekeys = array_keys(get_object_vars($stats_json->twitter->followers_count)); $twitter_users = $stats_json->twitter->followers_count->{max($twitter_datekeys)}; -// Get unique contributors - commits and issues -$gh_contributors = (array) $stats_json->gh_contributors; -$gh_contributor_commits = array_keys($gh_contributors); -$gh_contributor_issues = array_keys($issues_json['authors']); -foreach ($issues_json['authors'] as $author => $info) { - if (!isset($gh_contributors[$author])) { - $gh_contributors[$author] = $info['first_contribution']; +$config = parse_ini_file('../config.ini'); +$conn = mysqli_connect($config['host'], $config['username'], $config['password'], $config['dbname'], $config['port']); +// get stats for pipelines +$sql = 'SELECT timestamp, + SUM(views) AS sum_total_views, + SUM(views_uniques) AS sum_total_views_uniques, + SUM(clones) AS sum_total_clones, + SUM(clones_uniques) AS sum_total_clones_uniques, + MIN(timestamp) AS min_timestamp FROM github_traffic_stats + INNER JOIN nfcore_pipelines ON github_traffic_stats.pipeline_id = nfcore_pipelines.id GROUP BY timestamp ORDER BY timestamp ASC'; +$pipeline_metrics = []; +if ($result = mysqli_query($conn, $sql)) { + if (mysqli_num_rows($result) > 0) { + $pipeline_metrics = mysqli_fetch_all($result, MYSQLI_ASSOC); + // Free result set + mysqli_free_result($result); + } +} +$view_counts = []; +$clone_counts = []; +# get views per day +$sql = 'SELECT timestamp, SUM(views) AS sum_total_views, SUM(views_uniques) AS sum_total_views_unique FROM github_traffic_stats + INNER JOIN nfcore_pipelines ON github_traffic_stats.pipeline_id = nfcore_pipelines.id GROUP BY timestamp ORDER BY timestamp ASC'; +if ($result = mysqli_query($conn, $sql)) { + if (mysqli_num_rows($result) > 0) { + $view_counts = mysqli_fetch_all($result, MYSQLI_ASSOC); + // Free result set + mysqli_free_result($result); + } +} +$sql = 'SELECT timestamp, SUM(clones) AS sum_total_clones, SUM(clones_uniques) AS sum_total_clones_unique FROM github_traffic_stats + INNER JOIN nfcore_pipelines ON github_traffic_stats.pipeline_id = nfcore_pipelines.id GROUP BY timestamp ORDER BY timestamp ASC'; +if ($result = mysqli_query($conn, $sql)) { + if (mysqli_num_rows($result) > 0) { + $clone_counts = mysqli_fetch_all($result, MYSQLI_ASSOC); + // Free result set + mysqli_free_result($result); } - $gh_contributors[$author] = min($gh_contributors[$author], $info['first_contribution']); +} +// get all repos +$sql = 'SELECT * FROM nfcore_pipelines'; +$gh_repos = []; +if ($result = mysqli_query($conn, $sql)) { + if (mysqli_num_rows($result) > 0) { + $gh_repos = mysqli_fetch_all($result, MYSQLI_ASSOC); + // Free result set + mysqli_free_result($result); + } +} +// get unique contributor names from github_pipeline_contrib_stats table from the mysql database +$sql = 'SELECT DISTINCT author, avatar_url, SUM(week_commits) AS total_sum_commits +FROM github_pipeline_contrib_stats GROUP BY author,avatar_url ORDER BY total_sum_commits DESC'; +$gh_contributors_db = []; +if ($result = mysqli_query($conn, $sql)) { + if (mysqli_num_rows($result) > 0) { + $gh_contributors_db = mysqli_fetch_all($result, MYSQLI_ASSOC); + // Free result set + mysqli_free_result($result); + } +} else { + echo "ERROR: Could not able to execute $sql. " . mysqli_error($conn); } +// echo '
' . print_r($gh_contributors_db, true) . '
'; +// get maximum total_sum_commits from $gh_contributors_db +$max_total_sum_commits = max(array_column($gh_contributors_db, 'total_sum_commits')); +$total_sum_commits = array_sum(array_column($gh_contributors_db, 'total_sum_commits')); +$metrics_since = min(array_column($pipeline_metrics, 'min_timestamp')); +$total_sum_clones = array_sum(array_column($pipeline_metrics, 'sum_total_clones')); +$total_sum_views = array_sum(array_column($pipeline_metrics, 'sum_total_views')); +$total_sum_clones_uniques = array_sum(array_column($pipeline_metrics, 'sum_total_clones_uniques')); +$total_sum_views_uniques = array_sum(array_column($pipeline_metrics, 'sum_total_views_uniques')); # echo '
'.print_r($stats, true).'
'; - -// This is horrible, I know sorry. I'm in a rush :( -$stats_total_allrepos = [ - 'clones_count' => [], - 'clones_uniques' => [], - 'views_count' => [], - 'views_uniques' => [], -]; - -// Run everything twice - keep pipelines and core repos seperate -foreach (['pipelines', 'core_repos'] as $repo_type): - $stats = $stats_json->{$repo_type}; - $stats_total[$repo_type] = [ - 'releases' => 0, - 'stargazers' => 0, - 'watchers' => 0, - 'network_forks' => 0, - 'clones_count' => [], - 'clones_uniques' => [], - 'views_count' => [], - 'views_uniques' => [], - 'clones_count_since' => false, - 'clones_count_total' => 0, - 'clones_uniques_since' => false, - 'clones_uniques_total' => 0, - 'views_count_since' => false, - 'views_count_total' => 0, - 'views_uniques_since' => false, - 'views_uniques_total' => 0, - 'unique_committers' => [], - 'total_commits' => 0, - ]; - - $trows[$repo_type] = []; - foreach ($stats as $repo_name => $repo): - - // Exit quietly if something has gone wrong - if (!isset($repo->repo_metrics)) { - echo ''; - continue; - } - if (!isset($repo->repo_metrics->{$stats_json->updated})) { - echo ''; - continue; - } - // Ok, continue! - $metrics = $repo->repo_metrics->{$stats_json->updated}; - $stats_total[$repo_type]['releases'] += isset($repo->num_releases) ? $repo->num_releases : 0; - $stats_total[$repo_type]['stargazers'] += $metrics->stargazers_count; - $stats_total[$repo_type]['watchers'] += $metrics->subscribers_count; - $stats_total[$repo_type]['network_forks_count'] += $metrics->network_forks_count; - - foreach (['clones_count', 'clones_uniques', 'views_count', 'views_uniques'] as $key) { - if (isset($repo->{$key})) { - foreach ($repo->{$key} as $timestamp => $count) { - if (!isset($stats_total_allrepos[$key][$timestamp])) { - $stats_total_allrepos[$key][$timestamp] = 0; - } - $stats_total_allrepos[$key][$timestamp] += $count; - if (!isset($stats_total[$repo_type][$key][$timestamp])) { - $stats_total[$repo_type][$key][$timestamp] = 0; - } - $stats_total[$repo_type][$key][$timestamp] += $count; - if (!$stats_total[$repo_type][$key . '_since']) { - $stats_total[$repo_type][$key . '_since'] = strtotime($timestamp); - } - $stats_total[$repo_type][$key . '_since'] = min( - strtotime($timestamp), - $stats_total[$repo_type][$key . '_since'], - ); - } - } - $stats_total[$repo_type][$key . '_total'] += $repo->{$key . '_total'}; - } - - $total_commits = 0; - foreach ($repo->contributors as $contributor) { - $gh_username = $contributor->author->login; - $stats_total[$repo_type]['unique_committers'][$gh_username] = 0; - $stats_total['total']['unique_committers'][$gh_username] = 0; - $stats_total[$repo_type]['total_commits'] += $contributor->total; - $total_commits += $contributor->total; - } - ob_start(); - ?> - - archived) { - echo ''; - } elseif ($repo_type == 'pipelines') { - // Edge case where a new pipeline is added but stats hasn't rerun yet - if (!isset($repo->num_releases)) { - $repo->num_releases = 0; - } - if ($repo->num_releases) { - echo ''; - } else { - echo ''; - } - } - $alink = ''; - if ($repo_type == 'pipelines') { - $alink = ''; - } - ?> - nf-core/' . - $metrics->name . - ''; ?> - created_at, - false, -); ?> - num_releases; ?> - num_contributors; ?> - - stargazers_count; ?> - subscribers_count; ?> - network_forks_count; ?> - clones_count_total; ?> - clones_uniques_total; ?> - views_count_total; ?> - views_uniques_total; ?> - - $info) { + $gh_contributors[$author] = $info['first_contribution']; } -// -// -// TOP CARD DECK -// -// ?>

Introduction

@@ -365,8 +296,8 @@
-

pipelines)) + - count(get_object_vars($stats_json->core_repos)); ?>

+

Repositories

@@ -383,7 +314,7 @@

Commits

@@ -420,28 +351,17 @@
-
Clones since +
Clones since
-
Unique cloners since +
Unique cloners since
@@ -462,28 +382,15 @@
-
Views since +
Views since
-
Unique visitors since +
Unique visitors since
@@ -1452,14 +1359,8 @@ yAxisID: 'y-axis-count', data: [ $count) { - $timestamp = strtotime($timestamp); - // Skip zeros (anything before 2010) - if ($timestamp < 1262304000) { - continue; - } - echo '{ x: "' . date('Y-m-d', $timestamp) . '", y: ' . $count . ' },' . "\n\t\t\t"; + foreach( $clone_counts as $idx => $count ){ + echo '{ x: "' . date('Y-m-d', strtotime($count['timestamp'])) . '", y: ' . $count['sum_total_clones'] . ' },' . "\n\t\t\t"; } ?> ] @@ -1473,14 +1374,8 @@ yAxisID: 'y-axis-uniques', data: [ $count) { - $timestamp = strtotime($timestamp); - // Skip zeros (anything before 2010) - if ($timestamp < 1262304000) { - continue; - } - echo '{ x: "' . date('Y-m-d', $timestamp) . '", y: ' . $count . ' },' . "\n\t\t\t"; + foreach( $clone_counts as $idx => $count ){ + echo '{ x: "' . date('Y-m-d', strtotime($count['timestamp'])) . '", y: ' . $count['sum_total_clones_unique'] . ' },' . "\n\t\t\t"; } ?> ] @@ -1533,15 +1428,19 @@ yAxisID: 'y-axis-count', data: [ $count) { - $timestamp = strtotime($timestamp); - // Skip zeros (anything before 2010) - if ($timestamp < 1262304000) { - continue; - } - echo '{ x: "' . date('Y-m-d', $timestamp) . '", y: ' . $count . ' },' . "\n\t\t\t"; + // add data point for each timestamp in view_count + foreach( $view_counts as $idx => $count ){ + echo '{ x: "' . date('Y-m-d', strtotime($count['timestamp'])) . '", y: ' . $count['sum_total_views'] . ' },' . "\n\t\t\t"; } + // ksort($stats_total_allrepos['views_count']); + // foreach ($stats_total_allrepos['views_count'] as $timestamp => $count) { + // $timestamp = strtotime($timestamp); + // // Skip zeros (anything before 2010) + // if ($timestamp < 1262304000) { + // continue; + // } + // echo '{ x: "' . date('Y-m-d', $timestamp) . '", y: ' . $count . ' },' . "\n\t\t\t"; + // } ?> ] }, @@ -1554,14 +1453,8 @@ yAxisID: 'y-axis-uniques', data: [ $count) { - $timestamp = strtotime($timestamp); - // Skip zeros (anything before 2010) - if ($timestamp < 1262304000) { - continue; - } - echo '{ x: "' . date('Y-m-d', $timestamp) . '", y: ' . $count . ' },' . "\n\t\t\t"; + foreach( $view_counts as $idx => $count ){ + echo '{ x: "' . date('Y-m-d', strtotime($count['timestamp'])) . '", y: ' . $count['sum_total_views_unique'] . ' },' . "\n\t\t\t"; } ?> ] diff --git a/update_stats.php b/update_stats.php index 735dc0e2af..75a3637e47 100644 --- a/update_stats.php +++ b/update_stats.php @@ -60,7 +60,7 @@ function github_query($gh_query_url) { // Give the job a few moments to complete, and then submit the request again if (strpos($http_response_header[0], 'HTTP/1.1 202') !== false) { echo "\nWaiting for GitHub API to return results for $gh_query_url"; - sleep(10); + sleep(5); $first_page = true; continue; } @@ -125,7 +125,9 @@ function github_query($gh_query_url) { $week_deletions, $week_commits, ); + // add modules repo to pipelines foreach ($pipelines as $idx => $pipeline) { + echo "\nUpdating stats for " . $pipeline['name']; // get contributors $gh_contributors = github_query( 'https://api.github.com/repos/nf-core/' . $pipeline['name'] . '/stats/contributors', @@ -192,6 +194,7 @@ function github_query($gh_query_url) { mysqli_stmt_bind_param($stmt, 'iiiiis', $pipeline_id, $views, $views_uniques, $clones, $clones_uniques, $timestamp); foreach ($pipelines as $idx => $pipeline) { + echo "\nUpdating traffic stats for " . $pipeline['name']; $gh_views = github_query('https://api.github.com/repos/nf-core/' . $pipeline['name'] . '/traffic/views'); $gh_clones = github_query('https://api.github.com/repos/nf-core/' . $pipeline['name'] . '/traffic/clones'); foreach ($gh_views['views'] as $gh_view) { @@ -369,179 +372,179 @@ function github_query($gh_query_url) { } // Fetch all repositories at nf-core -$gh_repos_url = 'https://api.github.com/orgs/nf-core/repos?per_page=100'; -$gh_repos = json_decode(file_get_contents($gh_repos_url, false, $gh_api_opts)); -if (strpos($http_response_header[0], 'HTTP/1.1 200') === false) { - var_dump($http_response_header); - die("Could not fetch nf-core repositories! $gh_repos_url"); -} -foreach ($gh_repos as $repo) { - if (in_array($repo->name, $ignored_repos)) { - $repo_type = 'core_repos'; - } else { - $repo_type = 'pipelines'; - } - $results[$repo_type][$repo->name]['repo_metrics'][$updated] = [ - 'id' => $repo->id, - 'name' => $repo->name, - 'full_name' => $repo->full_name, - 'private' => $repo->private, - 'html_url' => $repo->html_url, - 'description' => $repo->description, - 'created_at' => $repo->created_at, - 'updated_at' => $repo->updated_at, - 'pushed_at' => $repo->pushed_at, - 'size' => $repo->size, - 'stargazers_count' => $repo->stargazers_count, - 'forks_count' => $repo->forks_count, - 'archived' => $repo->archived, - ]; - // Annoyingly, two values are only available if we query for just this repo - $gh_repo_url = 'https://api.github.com/repos/nf-core/' . $repo->name; - $gh_repo = json_decode(file_get_contents($gh_repo_url, false, $gh_api_opts)); - if (strpos($http_response_header[0], 'HTTP/1.1 200') === false) { - var_dump($http_response_header); - echo "Could not fetch nf-core repo! $gh_repo_url"; - continue; - } - $results[$repo_type][$repo->name]['repo_metrics'][$updated]['network_forks_count'] = $gh_repo->network_count; - $results[$repo_type][$repo->name]['repo_metrics'][$updated]['subscribers_count'] = $gh_repo->subscribers_count; -} - -// Fetch new statistics for each repo -foreach (['pipelines', 'core_repos'] as $repo_type) { - foreach ($results[$repo_type] as $repo_name => $repo_stats) { - // Views - $gh_views_url = 'https://api.github.com/repos/nf-core/' . $repo_name . '/traffic/views'; - $gh_views = json_decode(file_get_contents($gh_views_url, false, $gh_api_opts)); - if (strpos($http_response_header[0], 'HTTP/1.1 200') === false) { - // Pipelines are removed from the cache earlier as we know their names - if ($repo_type == 'core_repos' && strpos($http_response_header[0], 'HTTP/1.1 404') !== false) { - echo 'Removing ' . $repo_name . " from the cached results as it appears to have been deleted.\n"; - unset($results['core_repos'][$repo_name]); - } else { - echo "-------- Could not fetch nf-core repo views! $gh_views_url\n"; - var_dump($http_response_header); - echo "\n-------- End of header for $gh_views_url\n\n\n"; - } - continue; - } - foreach ($gh_views->views as $view) { - $results[$repo_type][$repo_name]['views_count'][$view->timestamp] = $view->count; - $results[$repo_type][$repo_name]['views_uniques'][$view->timestamp] = $view->uniques; - } - // Clones - $gh_clones_url = 'https://api.github.com/repos/nf-core/' . $repo_name . '/traffic/clones'; - $gh_clones = json_decode(file_get_contents($gh_clones_url, false, $gh_api_opts)); - if (strpos($http_response_header[0], 'HTTP/1.1 200') === false) { - var_dump($http_response_header); - echo "Could not fetch nf-core repo clones! $gh_clones_url"; - continue; - } - foreach ($gh_clones->clones as $clone) { - $results[$repo_type][$repo_name]['clones_count'][$clone->timestamp] = $clone->count; - $results[$repo_type][$repo_name]['clones_uniques'][$clone->timestamp] = $clone->uniques; - } - // Contributors - $gh_contributors_url = 'https://api.github.com/repos/nf-core/' . $repo_name . '/stats/contributors'; - $gh_contributors_raw = file_get_contents($gh_contributors_url, false, $gh_api_opts); - file_put_contents($contribs_fn_root . $repo_name . '.json', $gh_contributors_raw); - $gh_contributors = json_decode($gh_contributors_raw); - // If the data hasn't been cached when you query a repository's statistics, you'll receive a 202 response; - // a background job is also fired to start compiling these statistics. - // Give the job a few moments to complete, and then submit the request again - if (strpos($http_response_header[0], 'HTTP/1.1 202') !== false) { - $contribs_try_again[$repo_name] = [ - 'repo_type' => $repo_type, - 'gh_contributors_url' => $gh_contributors_url, - ]; - } elseif (strpos($http_response_header[0], 'HTTP/1.1 200') === false) { - var_dump($http_response_header); - echo "Could not fetch nf-core repo contributors! $gh_contributors_url"; - continue; - } - $results[$repo_type][$repo_name]['contributors'] = $gh_contributors; - $results[$repo_type][$repo_name]['num_contributors'] = count($gh_contributors); - - // Commits - $results[$repo_type][$repo_name]['commits'] = 0; - foreach ($gh_contributors as $contributor) { - $results[$repo_type][$repo_name]['commits'] += $contributor->total; - } - - // Recalculate totals - foreach (['views_count', 'views_uniques', 'clones_count', 'clones_uniques'] as $ctype) { - $results[$repo_type][$repo_name][$ctype . '_total'] = 0; - if ( - isset($results[$repo_type][$repo_name][$ctype]) && - count($results[$repo_type][$repo_name][$ctype]) > 0 - ) { - foreach ($results[$repo_type][$repo_name][$ctype] as $stat) { - $results[$repo_type][$repo_name][$ctype . '_total'] += $stat; - } - } - } - } -} - -// Try contribs again now that we've let it fire -if (count($contribs_try_again) > 0) { - sleep(10); - foreach ($contribs_try_again as $repo_name => $details) { - extract($details); // $repo_type, $gh_contributors_raw - $gh_contributors_raw = file_get_contents($gh_contributors_url, false, $gh_api_opts); - file_put_contents($contribs_fn_root . $repo_name . '.json', $gh_contributors_raw); - $gh_contributors = json_decode($gh_contributors_raw); - if (strpos($http_response_header[0], 'HTTP/1.1 202') !== false) { - echo "Tried getting contributors after delay for $repo_name, but took too long."; - continue; - } elseif (strpos($http_response_header[0], 'HTTP/1.1 200') === false) { - var_dump($http_response_header); - echo "Could not fetch nf-core repo contributors! $gh_contributors_url"; - continue; - } - $results[$repo_type][$repo_name]['contributors'] = $gh_contributors; - $results[$repo_type][$repo_name]['num_contributors'] = count($gh_contributors); - } -} - -foreach (['pipelines', 'core_repos'] as $repo_type) { - foreach ($results[$repo_type] as $repo_name => $repo_stats) { - foreach ($results[$repo_type][$repo_name]['contributors'] as $idx => $contributor) { - // Count how many total contributors and contributions we have per week - foreach ($contributor->weeks as $w) { - // Skip zeros (anything before 2010) - if ($w->w < 1262304000) { - continue; - } - // Find earliest contribution per author - if (!isset($results['gh_contributors'][$contributor->author->login])) { - $results['gh_contributors'][$contributor->author->login] = $w->w; - } - $results['gh_contributors'][$contributor->login] = min( - $w->w, - $results['gh_contributors'][$contributor->login], - ); - // Sum total contributions for everyone - if (!isset($results['gh_commits'][$w->w])) { - $results['gh_commits'][$w->w] = 0; - } - if (!isset($results['gh_additions'][$w->w])) { - $results['gh_additions'][$w->w] = 0; - } - if (!isset($results['gh_deletions'][$w->w])) { - $results['gh_deletions'][$w->w] = 0; - } - $results['gh_commits'][$w->w] += $w->c; - $results['gh_additions'][$w->w] += $w->a; - $results['gh_deletions'][$w->w] += $w->d; - } - - // The data for commits per week is massive - remove it - unset($results[$repo_type][$repo_name]['contributors'][$idx]->weeks); - } - } -} +// $gh_repos_url = 'https://api.github.com/orgs/nf-core/repos?per_page=100'; +// $gh_repos = json_decode(file_get_contents($gh_repos_url, false, $gh_api_opts)); +// if (strpos($http_response_header[0], 'HTTP/1.1 200') === false) { +// var_dump($http_response_header); +// die("Could not fetch nf-core repositories! $gh_repos_url"); +// } +// foreach ($gh_repos as $repo) { +// if (in_array($repo->name, $ignored_repos)) { +// $repo_type = 'core_repos'; +// } else { +// $repo_type = 'pipelines'; +// } +// $results[$repo_type][$repo->name]['repo_metrics'][$updated] = [ +// 'id' => $repo->id, +// 'name' => $repo->name, +// 'full_name' => $repo->full_name, +// 'private' => $repo->private, +// 'html_url' => $repo->html_url, +// 'description' => $repo->description, +// 'created_at' => $repo->created_at, +// 'updated_at' => $repo->updated_at, +// 'pushed_at' => $repo->pushed_at, +// 'size' => $repo->size, +// 'stargazers_count' => $repo->stargazers_count, +// 'forks_count' => $repo->forks_count, +// 'archived' => $repo->archived, +// ]; +// // Annoyingly, two values are only available if we query for just this repo +// $gh_repo_url = 'https://api.github.com/repos/nf-core/' . $repo->name; +// $gh_repo = json_decode(file_get_contents($gh_repo_url, false, $gh_api_opts)); +// if (strpos($http_response_header[0], 'HTTP/1.1 200') === false) { +// var_dump($http_response_header); +// echo "Could not fetch nf-core repo! $gh_repo_url"; +// continue; +// } +// $results[$repo_type][$repo->name]['repo_metrics'][$updated]['network_forks_count'] = $gh_repo->network_count; +// $results[$repo_type][$repo->name]['repo_metrics'][$updated]['subscribers_count'] = $gh_repo->subscribers_count; +// } + +// // Fetch new statistics for each repo +// foreach (['pipelines', 'core_repos'] as $repo_type) { +// foreach ($results[$repo_type] as $repo_name => $repo_stats) { +// // Views +// $gh_views_url = 'https://api.github.com/repos/nf-core/' . $repo_name . '/traffic/views'; +// $gh_views = json_decode(file_get_contents($gh_views_url, false, $gh_api_opts)); +// if (strpos($http_response_header[0], 'HTTP/1.1 200') === false) { +// // Pipelines are removed from the cache earlier as we know their names +// if ($repo_type == 'core_repos' && strpos($http_response_header[0], 'HTTP/1.1 404') !== false) { +// echo 'Removing ' . $repo_name . " from the cached results as it appears to have been deleted.\n"; +// unset($results['core_repos'][$repo_name]); +// } else { +// echo "-------- Could not fetch nf-core repo views! $gh_views_url\n"; +// var_dump($http_response_header); +// echo "\n-------- End of header for $gh_views_url\n\n\n"; +// } +// continue; +// } +// foreach ($gh_views->views as $view) { +// $results[$repo_type][$repo_name]['views_count'][$view->timestamp] = $view->count; +// $results[$repo_type][$repo_name]['views_uniques'][$view->timestamp] = $view->uniques; +// } +// // Clones +// $gh_clones_url = 'https://api.github.com/repos/nf-core/' . $repo_name . '/traffic/clones'; +// $gh_clones = json_decode(file_get_contents($gh_clones_url, false, $gh_api_opts)); +// if (strpos($http_response_header[0], 'HTTP/1.1 200') === false) { +// var_dump($http_response_header); +// echo "Could not fetch nf-core repo clones! $gh_clones_url"; +// continue; +// } +// foreach ($gh_clones->clones as $clone) { +// $results[$repo_type][$repo_name]['clones_count'][$clone->timestamp] = $clone->count; +// $results[$repo_type][$repo_name]['clones_uniques'][$clone->timestamp] = $clone->uniques; +// } +// // Contributors +// $gh_contributors_url = 'https://api.github.com/repos/nf-core/' . $repo_name . '/stats/contributors'; +// $gh_contributors_raw = file_get_contents($gh_contributors_url, false, $gh_api_opts); +// file_put_contents($contribs_fn_root . $repo_name . '.json', $gh_contributors_raw); +// $gh_contributors = json_decode($gh_contributors_raw); +// // If the data hasn't been cached when you query a repository's statistics, you'll receive a 202 response; +// // a background job is also fired to start compiling these statistics. +// // Give the job a few moments to complete, and then submit the request again +// if (strpos($http_response_header[0], 'HTTP/1.1 202') !== false) { +// $contribs_try_again[$repo_name] = [ +// 'repo_type' => $repo_type, +// 'gh_contributors_url' => $gh_contributors_url, +// ]; +// } elseif (strpos($http_response_header[0], 'HTTP/1.1 200') === false) { +// var_dump($http_response_header); +// echo "Could not fetch nf-core repo contributors! $gh_contributors_url"; +// continue; +// } +// $results[$repo_type][$repo_name]['contributors'] = $gh_contributors; +// $results[$repo_type][$repo_name]['num_contributors'] = count($gh_contributors); + +// // Commits +// $results[$repo_type][$repo_name]['commits'] = 0; +// foreach ($gh_contributors as $contributor) { +// $results[$repo_type][$repo_name]['commits'] += $contributor->total; +// } + +// // Recalculate totals +// foreach (['views_count', 'views_uniques', 'clones_count', 'clones_uniques'] as $ctype) { +// $results[$repo_type][$repo_name][$ctype . '_total'] = 0; +// if ( +// isset($results[$repo_type][$repo_name][$ctype]) && +// count($results[$repo_type][$repo_name][$ctype]) > 0 +// ) { +// foreach ($results[$repo_type][$repo_name][$ctype] as $stat) { +// $results[$repo_type][$repo_name][$ctype . '_total'] += $stat; +// } +// } +// } +// } +// } + +// // Try contribs again now that we've let it fire +// if (count($contribs_try_again) > 0) { +// sleep(5); +// foreach ($contribs_try_again as $repo_name => $details) { +// extract($details); // $repo_type, $gh_contributors_raw +// $gh_contributors_raw = file_get_contents($gh_contributors_url, false, $gh_api_opts); +// file_put_contents($contribs_fn_root . $repo_name . '.json', $gh_contributors_raw); +// $gh_contributors = json_decode($gh_contributors_raw); +// if (strpos($http_response_header[0], 'HTTP/1.1 202') !== false) { +// echo "Tried getting contributors after delay for $repo_name, but took too long."; +// continue; +// } elseif (strpos($http_response_header[0], 'HTTP/1.1 200') === false) { +// var_dump($http_response_header); +// echo "Could not fetch nf-core repo contributors! $gh_contributors_url"; +// continue; +// } +// $results[$repo_type][$repo_name]['contributors'] = $gh_contributors; +// $results[$repo_type][$repo_name]['num_contributors'] = count($gh_contributors); +// } +// } + +// foreach (['pipelines', 'core_repos'] as $repo_type) { +// foreach ($results[$repo_type] as $repo_name => $repo_stats) { +// foreach ($results[$repo_type][$repo_name]['contributors'] as $idx => $contributor) { +// // Count how many total contributors and contributions we have per week +// foreach ($contributor->weeks as $w) { +// // Skip zeros (anything before 2010) +// if ($w->w < 1262304000) { +// continue; +// } +// // Find earliest contribution per author +// if (!isset($results['gh_contributors'][$contributor->author->login])) { +// $results['gh_contributors'][$contributor->author->login] = $w->w; +// } +// $results['gh_contributors'][$contributor->login] = min( +// $w->w, +// $results['gh_contributors'][$contributor->login], +// ); +// // Sum total contributions for everyone +// if (!isset($results['gh_commits'][$w->w])) { +// $results['gh_commits'][$w->w] = 0; +// } +// if (!isset($results['gh_additions'][$w->w])) { +// $results['gh_additions'][$w->w] = 0; +// } +// if (!isset($results['gh_deletions'][$w->w])) { +// $results['gh_deletions'][$w->w] = 0; +// } +// $results['gh_commits'][$w->w] += $w->c; +// $results['gh_additions'][$w->w] += $w->a; +// $results['gh_deletions'][$w->w] += $w->d; +// } + +// // The data for commits per week is massive - remove it +// unset($results[$repo_type][$repo_name]['contributors'][$idx]->weeks); +// } +// } +// } // // @@ -577,25 +580,25 @@ function github_query($gh_query_url) { } } -// -// -// Twitter - get number of followers -// -// -echo 'update_stats - twitter updates - ' . date('Y-m-d h:i:s') . "\n"; -require 'vendor/autoload.php'; -use Abraham\TwitterOAuth\TwitterOAuth; -// Connect to twitter -$connection = new TwitterOAuth( - $config['twitter_key'], - $config['twitter_secret'], - $config['twitter_access_token'], - $config['twitter_access_token_secret'], -); -$twitter_stats = $connection->get('users/show', ['screen_name' => 'nf_core']); -if (isset($twitter_stats->followers_count)) { - $results['twitter']['followers_count'][$updated] = $twitter_stats->followers_count; -} +// // +// // +// // Twitter - get number of followers +// // +// // +// echo 'update_stats - twitter updates - ' . date('Y-m-d h:i:s') . "\n"; +// require 'vendor/autoload.php'; +// use Abraham\TwitterOAuth\TwitterOAuth; +// // Connect to twitter +// $connection = new TwitterOAuth( +// $config['twitter_key'], +// $config['twitter_secret'], +// $config['twitter_access_token'], +// $config['twitter_access_token_secret'], +// ); +// $twitter_stats = $connection->get('users/show', ['screen_name' => 'nf_core']); +// if (isset($twitter_stats->followers_count)) { +// $results['twitter']['followers_count'][$updated] = $twitter_stats->followers_count; +// } // //