metric: chaoss contributor (#1462)

Signed-off-by: frank-zsy <[email protected]>
X-lab2017 · Jan 4, 2024 · c796fc6 · c796fc6
1 parent 878fe8e
commit c796fc6
Show file tree

Hide file tree

Showing 11 changed files with 615 additions and 24 deletions.
diff --git a/notebook/handbook.ipynb b/notebook/handbook.ipynb
diff --git a/src/metrics/chaoss.ts b/src/metrics/chaoss.ts
@@ -827,6 +827,38 @@ export const chaossNewContributors = async (config: QueryConfig<NewContributorsO
   return processQueryResult(result, ['new_contributors', 'detail']);
 }
 
+export const chaossContributors = async (config: QueryConfig) => {
+  config = getMergedConfig(config);
+  const whereClauses: string[] = ["type = 'PullRequestEvent' AND action = 'closed' AND pull_merged = 1"];
+  const repoWhereClause = await getRepoWhereClause(config);
+  if (repoWhereClause) whereClauses.push(repoWhereClause);
+  whereClauses.push(getTimeRangeWhereClause(config));
+
+  const sql = `
+SELECT
+  id,
+  ${getTopLevelPlatform(config)},
+  argMax(name, time) AS name,
+  ${getGroupArrayInsertAtClause(config, { key: 'contributors_count', value: 'count' })}
+FROM
+(
+  SELECT
+    ${getGroupTimeClause(config)},
+    ${getGroupIdClause(config)},
+    COUNT(DISTINCT issue_author_id) AS count
+  FROM events
+  WHERE ${whereClauses.join(' AND ')}
+  GROUP BY id, platform, time
+  ${getInnerOrderAndLimit(config, 'count')}
+)
+GROUP BY id, platform
+${getOutterOrderAndLimit(config, 'contributors_count')}`;
+
+  const result: any = await clickhouse.query(sql);
+  const ret = processQueryResult(result, ['count']);
+  return ret;
+}
+
 interface InactiveContributorsOptions {
   // time interval to determine inactive contributor, default: 6
   timeInterval: number;

diff --git a/src/metrics/index.ts b/src/metrics/index.ts
@@ -2,7 +2,7 @@ import { getRepoOpenrank, getRepoActivity, getUserOpenrank, getUserActivity, get
 import {
   chaossCodeChangeCommits, chaossBusFactor, chaossIssuesNew, chaossIssuesClosed, chaossChangeRequestsAccepted,
   chaossChangeRequestsDeclined, chaossIssueResolutionDuration, chaossCodeChangeLines, chaossTechnicalFork,
-  chaossChangeRequests, chaossChangeRequestReviews, chaossNewContributors, chaossChangeRequestsDuration, chaossIssueResponseTime, chaossChangeRequestsAcceptanceRatio, chaossIssuesAndChangeRequestActive, chaossActiveDatesAndTimes, chaossChangeRequestResolutionDuration, chaossChangeRequestResponseTime, chaossIssueAge, chaossChangeRequestAge, chaossInactiveContributors,
+  chaossChangeRequests, chaossChangeRequestReviews, chaossNewContributors, chaossChangeRequestsDuration, chaossIssueResponseTime, chaossChangeRequestsAcceptanceRatio, chaossIssuesAndChangeRequestActive, chaossActiveDatesAndTimes, chaossChangeRequestResolutionDuration, chaossChangeRequestResponseTime, chaossIssueAge, chaossChangeRequestAge, chaossInactiveContributors, chaossContributors,
 } from './chaoss';
 import { repoStars, repoIssueComments, repoParticipants, userEquivalentTimeZone, contributorEmailSuffixes } from './metrics';
 
@@ -36,6 +36,7 @@ module.exports = {
   chaossChangeRequestsAcceptanceRatio: chaossChangeRequestsAcceptanceRatio,
   chaossRepoActiveDatesAndTimes: config => chaossActiveDatesAndTimes(config, 'repo'),
   chaossUserActiveDatesAndTimes: config => chaossActiveDatesAndTimes(config, 'user'),
+  chaossContributors: chaossContributors,
   chaossInactiveContributors: chaossInactiveContributors,
   // x-lab metrics
   repoStars: repoStars,

diff --git a/src/open_digger.js b/src/open_digger.js
@@ -51,6 +51,7 @@ const openDigger = {
       issueAge: func.chaossIssueAge,
       changeRequestAge: func.chaossChangeRequestAge,
       inactiveContributors: func.chaossInactiveContributors,
+      contributors: func.chaossContributors,
     },
     xlab: {
       repoStars: func.repoStars,

diff --git a/test/metrics.test.ts b/test/metrics.test.ts
@@ -251,6 +251,15 @@ describe("Metrics tests", () => {
                   inactive_contributors_file_name
                 );
               });
+              it("should test contributors interface", async () => {
+                const contributors_file_name =
+                  `contributors_${order}_${limit}_${limitOption}_${groupBy}_${groupTimeRange}`.toLowerCase();
+                await validateData(
+                  openDigger.chaossContributors,
+                  "contributors",
+                  contributors_file_name
+                );
+              });
               it("should test changeRequestsAcceptanceRatio interface", async () => {
                 const change_requests_acceptance_ratio_file_name =
                   `change_requests_acceptance_ratio_${order}_${limit}_${limitOption}_${groupBy}_${groupTimeRange}`.toLowerCase();

diff --git a/test/testdata/contributors/contributors_desc_3_all_null_month.json b/test/testdata/contributors/contributors_desc_3_all_null_month.json
@@ -0,0 +1,109 @@
+{
+  "sql": "\nSELECT\n  id,\n  argMax(name, time) AS name,\n  groupArrayInsertAt(\n    0,\n    toUInt32(dateDiff('month', toDate('2015-1-1'), toDate('2016-12-1'))) + 1)(ROUND(inactive_contributors, 2),\n    toUInt32(dateDiff('month', toDate('2015-1-1'), time)-1)) AS inactive_contributors,\n  groupArrayInsertAt(\n    [],\n    toUInt32(dateDiff('month', toDate('2015-1-1'), toDate('2016-12-1'))) + 1)(detail,\n    toUInt32(dateDiff('month', toDate('2015-1-1'), time)-1)) AS detail\nFROM\n(\n  SELECT\n    id,\n    argMax(name, time) AS name,\n    time,\n    countIf(first_time < time AND contributions <= 0) AS inactive_contributors,\n    groupArrayIf(author, first_time < time AND contributions <= 0) AS detail\n  FROM\n  (\n    SELECT\n      arrayJoin(arrayMap(x -> dateAdd(month, x + 1, toDate('2015-1-1')), range(toUInt64(dateDiff('month', toDate('2015-1-1'), toDate('2017-1-1')))))) AS time,\n      repo_id AS id, argMax(repo_name, time) AS name,\n      actor_id, argMax(author, created_at) AS author,\n      min(created_at) AS first_time,\n      countIf(created_at >= dateSub(month, 6, time) AND created_at <= time) AS contributions\n    FROM\n    (\n      SELECT \n        repo_id,\n        repo_name,\n        org_id,\n        org_login,\n        issue_author_id AS actor_id, issue_author_login AS author,\n        created_at\n      FROM gh_events\n      WHERE type = 'PullRequestEvent' AND action = 'closed' AND pull_merged = 1 AND (org_id IN [1342004]) AND created_at < toDate('2017-1-1')\n      HAVING author NOT LIKE '%[bot]'\n    )\n    GROUP BY id, actor_id, time\n  )\n  GROUP BY id, time\n  \n)\nGROUP BY id\nORDER BY inactive_contributors[-1] DESC\n    LIMIT 3",
+  "modifiedOption": {
+    "orgIds": [
+      1342004
+    ],
+    "startYear": 2015,
+    "endYear": 2016,
+    "startMonth": 1,
+    "endMonth": 12,
+    "order": "DESC",
+    "limit": 3,
+    "limitOption": "all",
+    "groupTimeRange": "month",
+    "groupBy": null
+  },
+  "contributors_desc_3_all_null_month": [
+    {
+      "id": "63809205",
+      "name": "google/oss-fuzz",
+      "count": [
+        "0",
+        "0",
+        "0",
+        "0",
+        "0",
+        "0",
+        "0",
+        "0",
+        "0",
+        "0",
+        "0",
+        "0",
+        "0",
+        "0",
+        "0",
+        "0",
+        "0",
+        "0",
+        "0",
+        "1",
+        "2",
+        "8",
+        "7",
+        "36"
+      ]
+    },
+    {
+      "id": "19249835",
+      "name": "google/WebFundamentals",
+      "count": [
+        "6",
+        "4",
+        "8",
+        "6",
+        "17",
+        "7",
+        "15",
+        "12",
+        "14",
+        "22",
+        "15",
+        "20",
+        "17",
+        "23",
+        "30",
+        "19",
+        "15",
+        "25",
+        "24",
+        "22",
+        "29",
+        "33",
+        "21",
+        "24"
+      ]
+    },
+    {
+      "id": "23357588",
+      "name": "google/protobuf",
+      "count": [
+        "3",
+        "10",
+        "10",
+        "9",
+        "19",
+        "17",
+        "12",
+        "18",
+        "11",
+        "15",
+        "14",
+        "15",
+        "14",
+        "21",
+        "18",
+        "18",
+        "22",
+        "14",
+        "20",
+        "18",
+        "25",
+        "16",
+        "23",
+        "22"
+      ]
+    }
+  ]
+}
diff --git a/test/testdata/contributors/contributors_desc_3_all_null_quarter.json b/test/testdata/contributors/contributors_desc_3_all_null_quarter.json
@@ -0,0 +1,61 @@
+{
+  "sql": "\nSELECT\n  id,\n  argMax(name, time) AS name,\n  groupArrayInsertAt(\n    0,\n    toUInt32(dateDiff('quarter', toDate('2015-1-1'), toDate('2016-12-1'))) + 1)(ROUND(inactive_contributors, 2),\n    toUInt32(dateDiff('quarter', toStartOfQuarter(toDate('2015-1-1')), time)-1)) AS inactive_contributors,\n  groupArrayInsertAt(\n    [],\n    toUInt32(dateDiff('quarter', toDate('2015-1-1'), toDate('2016-12-1'))) + 1)(detail,\n    toUInt32(dateDiff('quarter', toStartOfQuarter(toDate('2015-1-1')), time)-1)) AS detail\nFROM\n(\n  SELECT\n    id,\n    argMax(name, time) AS name,\n    time,\n    countIf(first_time < time AND contributions <= 0) AS inactive_contributors,\n    groupArrayIf(author, first_time < time AND contributions <= 0) AS detail\n  FROM\n  (\n    SELECT\n      arrayJoin(arrayMap(x -> dateAdd(quarter, x + 1, toDate('2015-1-1')), range(toUInt64(dateDiff('quarter', toDate('2015-1-1'), toDate('2017-1-1')))))) AS time,\n      repo_id AS id, argMax(repo_name, time) AS name,\n      actor_id, argMax(author, created_at) AS author,\n      min(created_at) AS first_time,\n      countIf(created_at >= dateSub(month, 6, time) AND created_at <= time) AS contributions\n    FROM\n    (\n      SELECT \n        repo_id,\n        repo_name,\n        org_id,\n        org_login,\n        issue_author_id AS actor_id, issue_author_login AS author,\n        created_at\n      FROM gh_events\n      WHERE type = 'PullRequestEvent' AND action = 'closed' AND pull_merged = 1 AND (org_id IN [1342004]) AND created_at < toDate('2017-1-1')\n      HAVING author NOT LIKE '%[bot]'\n    )\n    GROUP BY id, actor_id, time\n  )\n  GROUP BY id, time\n  \n)\nGROUP BY id\nORDER BY inactive_contributors[-1] DESC\n    LIMIT 3",
+  "modifiedOption": {
+    "orgIds": [
+      1342004
+    ],
+    "startYear": 2015,
+    "endYear": 2016,
+    "startMonth": 1,
+    "endMonth": 12,
+    "order": "DESC",
+    "limit": 3,
+    "limitOption": "all",
+    "groupTimeRange": "quarter",
+    "groupBy": null
+  },
+  "contributors_desc_3_all_null_quarter": [
+    {
+      "id": "19249835",
+      "name": "google/WebFundamentals",
+      "count": [
+        "15",
+        "24",
+        "27",
+        "37",
+        "50",
+        "41",
+        "53",
+        "51"
+      ]
+    },
+    {
+      "id": "23357588",
+      "name": "google/protobuf",
+      "count": [
+        "20",
+        "25",
+        "29",
+        "34",
+        "41",
+        "40",
+        "51",
+        "45"
+      ]
+    },
+    {
+      "id": "63809205",
+      "name": "google/oss-fuzz",
+      "count": [
+        "0",
+        "0",
+        "0",
+        "0",
+        "0",
+        "0",
+        "3",
+        "40"
+      ]
+    }
+  ]
+}
diff --git a/test/testdata/contributors/contributors_desc_3_all_null_year.json b/test/testdata/contributors/contributors_desc_3_all_null_year.json
@@ -0,0 +1,43 @@
+{
+  "sql": "\nSELECT\n  id,\n  argMax(name, time) AS name,\n  groupArrayInsertAt(\n    0,\n    toUInt32(dateDiff('year', toDate('2015-1-1'), toDate('2016-12-1'))) + 1)(ROUND(inactive_contributors, 2),\n    toUInt32(dateDiff('year', toStartOfYear(toDate('2015-1-1')), time)-1)) AS inactive_contributors,\n  groupArrayInsertAt(\n    [],\n    toUInt32(dateDiff('year', toDate('2015-1-1'), toDate('2016-12-1'))) + 1)(detail,\n    toUInt32(dateDiff('year', toStartOfYear(toDate('2015-1-1')), time)-1)) AS detail\nFROM\n(\n  SELECT\n    id,\n    argMax(name, time) AS name,\n    time,\n    countIf(first_time < time AND contributions <= 0) AS inactive_contributors,\n    groupArrayIf(author, first_time < time AND contributions <= 0) AS detail\n  FROM\n  (\n    SELECT\n      arrayJoin(arrayMap(x -> dateAdd(year, x + 1, toDate('2015-1-1')), range(toUInt64(dateDiff('year', toDate('2015-1-1'), toDate('2017-1-1')))))) AS time,\n      repo_id AS id, argMax(repo_name, time) AS name,\n      actor_id, argMax(author, created_at) AS author,\n      min(created_at) AS first_time,\n      countIf(created_at >= dateSub(month, 6, time) AND created_at <= time) AS contributions\n    FROM\n    (\n      SELECT \n        repo_id,\n        repo_name,\n        org_id,\n        org_login,\n        issue_author_id AS actor_id, issue_author_login AS author,\n        created_at\n      FROM gh_events\n      WHERE type = 'PullRequestEvent' AND action = 'closed' AND pull_merged = 1 AND (org_id IN [1342004]) AND created_at < toDate('2017-1-1')\n      HAVING author NOT LIKE '%[bot]'\n    )\n    GROUP BY id, actor_id, time\n  )\n  GROUP BY id, time\n  \n)\nGROUP BY id\nORDER BY inactive_contributors[-1] DESC\n    LIMIT 3",
+  "modifiedOption": {
+    "orgIds": [
+      1342004
+    ],
+    "startYear": 2015,
+    "endYear": 2016,
+    "startMonth": 1,
+    "endMonth": 12,
+    "order": "DESC",
+    "limit": 3,
+    "limitOption": "all",
+    "groupTimeRange": "year",
+    "groupBy": null
+  },
+  "contributors_desc_3_all_null_year": [
+    {
+      "id": "23357588",
+      "name": "google/protobuf",
+      "count": [
+        "79",
+        "138"
+      ]
+    },
+    {
+      "id": "19249835",
+      "name": "google/WebFundamentals",
+      "count": [
+        "77",
+        "131"
+      ]
+    },
+    {
+      "id": "19953044",
+      "name": "google/flatbuffers",
+      "count": [
+        "37",
+        "76"
+      ]
+    }
+  ]
+}
diff --git a/test/testdata/contributors/contributors_desc_3_all_org_month.json b/test/testdata/contributors/contributors_desc_3_all_org_month.json
@@ -0,0 +1,49 @@
+{
+  "sql": "\nSELECT\n  id,\n  argMax(name, time) AS name,\n  groupArrayInsertAt(\n    0,\n    toUInt32(dateDiff('month', toDate('2015-1-1'), toDate('2016-12-1'))) + 1)(ROUND(inactive_contributors, 2),\n    toUInt32(dateDiff('month', toDate('2015-1-1'), time)-1)) AS inactive_contributors,\n  groupArrayInsertAt(\n    [],\n    toUInt32(dateDiff('month', toDate('2015-1-1'), toDate('2016-12-1'))) + 1)(detail,\n    toUInt32(dateDiff('month', toDate('2015-1-1'), time)-1)) AS detail\nFROM\n(\n  SELECT\n    id,\n    argMax(name, time) AS name,\n    time,\n    countIf(first_time < time AND contributions <= 0) AS inactive_contributors,\n    groupArrayIf(author, first_time < time AND contributions <= 0) AS detail\n  FROM\n  (\n    SELECT\n      arrayJoin(arrayMap(x -> dateAdd(month, x + 1, toDate('2015-1-1')), range(toUInt64(dateDiff('month', toDate('2015-1-1'), toDate('2017-1-1')))))) AS time,\n      org_id AS id, argMax(org_login, time) AS name,\n      actor_id, argMax(author, created_at) AS author,\n      min(created_at) AS first_time,\n      countIf(created_at >= dateSub(month, 6, time) AND created_at <= time) AS contributions\n    FROM\n    (\n      SELECT \n        repo_id,\n        repo_name,\n        org_id,\n        org_login,\n        issue_author_id AS actor_id, issue_author_login AS author,\n        created_at\n      FROM gh_events\n      WHERE type = 'PullRequestEvent' AND action = 'closed' AND pull_merged = 1 AND (org_id IN [1342004]) AND created_at < toDate('2017-1-1')\n      HAVING author NOT LIKE '%[bot]'\n    )\n    GROUP BY id, actor_id, time\n  )\n  GROUP BY id, time\n  \n)\nGROUP BY id\nORDER BY inactive_contributors[-1] DESC\n    LIMIT 3",
+  "modifiedOption": {
+    "orgIds": [
+      1342004
+    ],
+    "startYear": 2015,
+    "endYear": 2016,
+    "startMonth": 1,
+    "endMonth": 12,
+    "order": "DESC",
+    "limit": 3,
+    "limitOption": "all",
+    "groupTimeRange": "month",
+    "groupBy": "org"
+  },
+  "contributors_desc_3_all_org_month": [
+    {
+      "id": "1342004",
+      "name": "google",
+      "count": [
+        "161",
+        "184",
+        "221",
+        "209",
+        "213",
+        "220",
+        "290",
+        "303",
+        "266",
+        "299",
+        "249",
+        "263",
+        "297",
+        "331",
+        "296",
+        "278",
+        "299",
+        "293",
+        "313",
+        "309",
+        "321",
+        "324",
+        "269",
+        "320"
+      ]
+    }
+  ]
+}
diff --git a/test/testdata/contributors/contributors_desc_3_all_org_quarter.json b/test/testdata/contributors/contributors_desc_3_all_org_quarter.json
@@ -0,0 +1,33 @@
+{
+  "sql": "\nSELECT\n  id,\n  argMax(name, time) AS name,\n  groupArrayInsertAt(\n    0,\n    toUInt32(dateDiff('quarter', toDate('2015-1-1'), toDate('2016-12-1'))) + 1)(ROUND(inactive_contributors, 2),\n    toUInt32(dateDiff('quarter', toStartOfQuarter(toDate('2015-1-1')), time)-1)) AS inactive_contributors,\n  groupArrayInsertAt(\n    [],\n    toUInt32(dateDiff('quarter', toDate('2015-1-1'), toDate('2016-12-1'))) + 1)(detail,\n    toUInt32(dateDiff('quarter', toStartOfQuarter(toDate('2015-1-1')), time)-1)) AS detail\nFROM\n(\n  SELECT\n    id,\n    argMax(name, time) AS name,\n    time,\n    countIf(first_time < time AND contributions <= 0) AS inactive_contributors,\n    groupArrayIf(author, first_time < time AND contributions <= 0) AS detail\n  FROM\n  (\n    SELECT\n      arrayJoin(arrayMap(x -> dateAdd(quarter, x + 1, toDate('2015-1-1')), range(toUInt64(dateDiff('quarter', toDate('2015-1-1'), toDate('2017-1-1')))))) AS time,\n      org_id AS id, argMax(org_login, time) AS name,\n      actor_id, argMax(author, created_at) AS author,\n      min(created_at) AS first_time,\n      countIf(created_at >= dateSub(month, 6, time) AND created_at <= time) AS contributions\n    FROM\n    (\n      SELECT \n        repo_id,\n        repo_name,\n        org_id,\n        org_login,\n        issue_author_id AS actor_id, issue_author_login AS author,\n        created_at\n      FROM gh_events\n      WHERE type = 'PullRequestEvent' AND action = 'closed' AND pull_merged = 1 AND (org_id IN [1342004]) AND created_at < toDate('2017-1-1')\n      HAVING author NOT LIKE '%[bot]'\n    )\n    GROUP BY id, actor_id, time\n  )\n  GROUP BY id, time\n  \n)\nGROUP BY id\nORDER BY inactive_contributors[-1] DESC\n    LIMIT 3",
+  "modifiedOption": {
+    "orgIds": [
+      1342004
+    ],
+    "startYear": 2015,
+    "endYear": 2016,
+    "startMonth": 1,
+    "endMonth": 12,
+    "order": "DESC",
+    "limit": 3,
+    "limitOption": "all",
+    "groupTimeRange": "quarter",
+    "groupBy": "org"
+  },
+  "contributors_desc_3_all_org_quarter": [
+    {
+      "id": "1342004",
+      "name": "google",
+      "count": [
+        "438",
+        "471",
+        "639",
+        "601",
+        "693",
+        "655",
+        "700",
+        "653"
+      ]
+    }
+  ]
+}