scrape.js: Use GraphQL API

Use GitHub GraphQL API instead of REST API Closes #111
coala · Apr 17, 2018 · 5f351f2 · 5f351f2
1 parent 5fe816c
commit 5f351f2
Show file tree

Hide file tree

Showing 5 changed files with 40 additions and 26 deletions.
diff --git a/lib/queries/github_search_org.graphql b/lib/queries/github_search_org.graphql
@@ -0,0 +1,9 @@
+query($query: String!) {
+    search(type:USER, query:$query, first:1) {
+        nodes {
+            ...on Organization {
+                login
+            }
+        }
+    }
+}
diff --git a/lib/queries/github_user_info.graphql b/lib/queries/github_user_info.graphql
@@ -0,0 +1,6 @@
+query ($user: String!) {
+    user(login: $user) {
+        login
+        updatedAt
+    }
+}
diff --git a/lib/queries/index.js b/lib/queries/index.js
@@ -1,3 +1,5 @@
 const { loadQuery } = require('../utils')
 
 module.exports.GITHUB_REPO_INFO_QUERY = loadQuery('github_repo_info')
+module.exports.GITHUB_SEARCH_ORG_QUERY = loadQuery('github_search_org')
+module.exports.GITHUB_USER_INFO_QUERY = loadQuery('github_user_info')
diff --git a/lib/rss.js b/lib/rss.js
@@ -25,8 +25,11 @@ module.exports = ({
   feed_items: current,
   data_updated: dataUpdated,
 }) => {
-  const oldOrgs = createOrgHash(oldData)
   const newOrgs = createOrgHash(newData)
+  let oldOrgs = newOrgs
+  if (oldData) {
+    oldOrgs = createOrgHash(oldData)
+  }
 
   if (oldOrgs && newOrgs) {
     const diffs = generateDiff(oldOrgs, newOrgs) || []

diff --git a/lib/scrape.js b/lib/scrape.js
@@ -8,18 +8,19 @@ const validUsername = require('valid-github-username')
 const wdk = require('wikidata-sdk')
 const cheerio = require('cheerio')
 
-const { GITHUB_REPO_INFO_QUERY } = require('./queries')
+const {
+  GITHUB_REPO_INFO_QUERY,
+  GITHUB_SEARCH_ORG_QUERY,
+  GITHUB_USER_INFO_QUERY,
+} = require('./queries')
 const { getLatestCommitMessage } = require('./utils')
 
 const GH_BASE = 'https://github.com'
 const GH_USER_BASE = `${GH_BASE}/users`
 const GH_ORG_BASE = `${GH_BASE}/orgs`
-const GH_API_BASE = 'https://api.github.com'
 const GH_GQL_BASE = 'https://api.github.com/graphql'
 const GCI_API_BASE = 'https://codein.withgoogle.com/api'
 
-const MIN_SEARCH_SCORE = 10
-
 // The time to cache GitHub usernames for in milliseconds
 const GITHUB_CACHE_TIME = 2 * 24 * 60 * 60 * 1000
 
@@ -53,12 +54,6 @@ const CHAT_IMAGES = {
   OTHER: 'images/chat.png',
 }
 
-const GH_API_OPTIONS = {
-  headers: process.env.GITHUB_TOKEN
-    ? { Authorization: `token ${process.env.GITHUB_TOKEN}` }
-    : {},
-}
-
 const GH_GQL_OPTIONS = {
   url: GH_GQL_BASE,
   headers: process.env.GITHUB_TOKEN
@@ -118,8 +113,7 @@ async function fetchRepositoryInfo(org) {
   if (repositoryInfo[org]) return repositoryInfo[org]
 
   const { data } = await client.query(GITHUB_REPO_INFO_QUERY, { org })
-
-  if (data) {
+  if (data && data.organization) {
     const info = data.organization.repositories.nodes.map(node => ({
       watchers: node.watchers.nodes,
       stargazers: node.stargazers.nodes,
@@ -184,12 +178,12 @@ async function checkGitHubUserExists(user) {
 }
 
 async function searchGitHubOrgs(query) {
-  const res = await fetch(
-    `${GH_API_BASE}/search/users?q=${query}%20type:org`,
-    GH_API_OPTIONS
-  )
-  const { items } = await res.json()
-  return items || []
+  const res = await client.query(GITHUB_SEARCH_ORG_QUERY, { query })
+  if (res && res.data) {
+    return res.data.search.nodes
+  } else {
+    return []
+  }
 }
 
 async function getGitHubUserHistory(user, from, to) {
@@ -238,12 +232,12 @@ function findMatches(input, pattern) {
 }
 
 async function getGitHubUser(user) {
-  const res = await fetch(`${GH_API_BASE}/users/${user}`, GH_API_OPTIONS)
-  let response = await res.json()
-  if (response && response.message) {
-    response = undefined
+  const res = await client.query(GITHUB_USER_INFO_QUERY, { user })
+  if (res && res.data.user) {
+    return res.data.user
+  } else {
+    return undefined
   }
-  return response
 }
 
 async function findOrganization({
@@ -283,10 +277,10 @@ async function findOrganization({
   )
 
   const removePattern = /the|project|\([a-zA-Z]+\)/gi
-  const searchQuery = name.replace(removePattern, '').trim()
+  const searchQuery = name.replace(removePattern, '').trim() + ' type:org'
   const searchResults = await searchGitHubOrgs(searchQuery)
 
-  if (searchResults.length > 0 && searchResults[0].score > MIN_SEARCH_SCORE) {
+  if (searchResults.length > 0) {
     return searchResults[0].login
   }