diff --git a/README.md b/README.md deleted file mode 100644 index 6466017..0000000 --- a/README.md +++ /dev/null @@ -1,8 +0,0 @@ -# How to run ---- -进入当前路径并运行： ->npm install -> ->node index.js - -访问：[http://localhost:3000/](http://localhost:3000/) diff --git a/Spider.js b/Spider.js index b20ad41..8eeacd0 100644 --- a/Spider.js +++ b/Spider.js @@ -1,7 +1,6 @@ var fetchFollwerOrFollwee = require('./fetchFollwerOrFollwee'); var getUser = require('./getUser'); var Promise = require('bluebird'); -var echartParser = require('./echartParser'); module.exports = Spider; @@ -10,6 +9,7 @@ function Spider(userPageUrl, socket) { return getUser(userPageUrl) .then(function(user) { socket.emit('notice', '抓取用户信息成功'); + socket.emit('get user', user); return getFriends(user, socket); }) .then(function(myFriends) { @@ -30,7 +30,7 @@ function Spider(userPageUrl, socket) { console.log(myFriends); return Promise.map(myFriends, function(myFriend) { return searchSameFriend(myFriend, myFriends, socket); - }, { concurrency: 2 }); + }, { concurrency: 3 }); }) .then(function(result) { var data = result; diff --git a/client/build/bundle.js b/client/build/bundle.js index 518e7c4..bc29f85 100644 --- a/client/build/bundle.js +++ b/client/build/bundle.js @@ -49,7 +49,7 @@ var $ = __webpack_require__(397); var myChart = echarts.init(document.getElementById('main')); var echartParser = __webpack_require__(398); - + var USER_NAME; $("#submit").click(function() { socket.emit('fetch start', { url: $("#url").val() @@ -61,6 +61,7 @@ console.log(data); dataStore = data; result = echartParser(dataStore); + result.data[0].name = USER_NAME; var option = { title: { text: 'Les Miserables', @@ -94,6 +95,7 @@ if (item.user.hash_id == data.hash_id) { item.sameFriends = data.sameFriends; result = echartParser(dataStore); + result.data[0].name = USER_NAME; var option = { title: { text: 'Les Miserables', @@ -123,6 +125,9 @@ } }) }) + socket.on('get user',function(user){ + USER_NAME = user.name; + }) socket.on('log', function(data) { console.log(data); }); @@ -131,39 +136,6 @@ socket.on('notice', function(data) { notice.html(data); }); - // 基于准备好的dom，初始化echarts实例 - - // 绘制图表 - - // $.get('/test', function(data) { - // console.log(data); - // option = { - // title: { - // text: 'Les Miserables', - // subtext: 'Default layout', - // top: 'bottom', - // left: 'right' - // }, - // tooltip: {}, - // animationDuration: 1500, - // animationEasingUpdate: 'quinticInOut', - // series: [{ - // name: 'Les Miserables', - // type: 'graph', - // layout: 'force', - // data: data.data, - // links: data.links, - // roam: true, - // label: { - // normal: { - // position: 'right', - // formatter: '{b}' - // } - // }, - // }] - // }; - // myChart.setOption(option); - // }) /***/ }, @@ -71489,9 +71461,13 @@ }] function echartParser(input) { + console.log(input); var hash_id_to_common_id = {} input.forEach(function(item, index) { - hash_id_to_common_id[item.user.hash_id] = index + 1; + if (item.user.hash_id) { + hash_id_to_common_id[item.user.hash_id] = index + 1; + console.log(item.user.hash_id, index + 1, item); + } }); var data = []; data.push({ @@ -71503,30 +71479,35 @@ } }) input.forEach(function(item, index) { - data.push({ - "id": hash_id_to_common_id[item.user.hash_id], - "name": item.user.name, - "symbolSize": 10, //item.user.followerAmount, - "label": { - "normal": { "show": true } - }, - draggable: true - }) + if (item.user.hash_id) { + data.push({ + "id": hash_id_to_common_id[item.user.hash_id], + "name": item.user.name, + "symbolSize": 10, //item.user.followerAmount, + "label": { + "normal": { "show": true } + }, + draggable: true + }) + } }); var links = []; input.forEach(function(item) { - links.push({ - source: 0, - target: hash_id_to_common_id[item.user.hash_id] - }) - item.sameFriends.forEach(function(item2) { + if (item.user.hash_id) { links.push({ - source: hash_id_to_common_id[item.user.hash_id], - target: hash_id_to_common_id[item2.hash_id] + source: 0, + target: hash_id_to_common_id[item.user.hash_id] }) - }) + item.sameFriends.forEach(function(item2) { + links.push({ + source: hash_id_to_common_id[item.user.hash_id], + target: hash_id_to_common_id[item2.hash_id] + }) + }) + } }); + console.log("data:" + data); return { data: data, links: links diff --git a/client/index.html b/client/index.html index e9eee6f..60e5034 100644 --- a/client/index.html +++ b/client/index.html @@ -7,7 +7,7 @@

- +
diff --git a/client/src/echartParser.js b/client/src/echartParser.js index c3b7c45..5169b4c 100644 --- a/client/src/echartParser.js +++ b/client/src/echartParser.js @@ -49,9 +49,13 @@ var input = [{ }] function echartParser(input) { + console.log(input); var hash_id_to_common_id = {} input.forEach(function(item, index) { - hash_id_to_common_id[item.user.hash_id] = index + 1; + if (item.user.hash_id) { + hash_id_to_common_id[item.user.hash_id] = index + 1; + console.log(item.user.hash_id, index + 1, item); + } }); var data = []; data.push({ @@ -63,30 +67,35 @@ function echartParser(input) { } }) input.forEach(function(item, index) { - data.push({ - "id": hash_id_to_common_id[item.user.hash_id], - "name": item.user.name, - "symbolSize": 10, //item.user.followerAmount, - "label": { - "normal": { "show": true } - }, - draggable: true - }) + if (item.user.hash_id) { + data.push({ + "id": hash_id_to_common_id[item.user.hash_id], + "name": item.user.name, + "symbolSize": 10, //item.user.followerAmount, + "label": { + "normal": { "show": true } + }, + draggable: true + }) + } }); var links = []; input.forEach(function(item) { - links.push({ - source: 0, - target: hash_id_to_common_id[item.user.hash_id] - }) - item.sameFriends.forEach(function(item2) { + if (item.user.hash_id) { links.push({ - source: hash_id_to_common_id[item.user.hash_id], - target: hash_id_to_common_id[item2.hash_id] + source: 0, + target: hash_id_to_common_id[item.user.hash_id] }) - }) + item.sameFriends.forEach(function(item2) { + links.push({ + source: hash_id_to_common_id[item.user.hash_id], + target: hash_id_to_common_id[item2.hash_id] + }) + }) + } }); + console.log("data:" + data); return { data: data, links: links diff --git a/client/src/index.js b/client/src/index.js index 32dc663..9b1d738 100644 --- a/client/src/index.js +++ b/client/src/index.js @@ -3,7 +3,7 @@ var socket = require('socket.io-client')('http://localhost:3001'); var $ = require('jquery'); var myChart = echarts.init(document.getElementById('main')); var echartParser = require('./echartParser'); - +var USER_NAME; $("#submit").click(function() { socket.emit('fetch start', { url: $("#url").val() @@ -15,6 +15,7 @@ socket.on('data', function(data) { console.log(data); dataStore = data; result = echartParser(dataStore); + result.data[0].name = USER_NAME; var option = { title: { text: 'Les Miserables', @@ -48,6 +49,7 @@ socket.on('same friend', function(data) { if (item.user.hash_id == data.hash_id) { item.sameFriends = data.sameFriends; result = echartParser(dataStore); + result.data[0].name = USER_NAME; var option = { title: { text: 'Les Miserables', @@ -77,6 +79,9 @@ socket.on('same friend', function(data) { } }) }) +socket.on('get user',function(user){ + USER_NAME = user.name; +}) socket.on('log', function(data) { console.log(data); }); @@ -85,36 +90,3 @@ var notice = $("#notice"); socket.on('notice', function(data) { notice.html(data); }); -// 基于准备好的dom，初始化echarts实例 - -// 绘制图表 - -// $.get('/test', function(data) { -// console.log(data); -// option = { -// title: { -// text: 'Les Miserables', -// subtext: 'Default layout', -// top: 'bottom', -// left: 'right' -// }, -// tooltip: {}, -// animationDuration: 1500, -// animationEasingUpdate: 'quinticInOut', -// series: [{ -// name: 'Les Miserables', -// type: 'graph', -// layout: 'force', -// data: data.data, -// links: data.links, -// roam: true, -// label: { -// normal: { -// position: 'right', -// formatter: '{b}' -// } -// }, -// }] -// }; -// myChart.setOption(option); -// }) diff --git a/config.js b/config.js index 2e38cbd..15165fe 100644 --- a/config.js +++ b/config.js @@ -1,3 +1,3 @@ module.exports = { - cookie:'_za=c392e6c0-5bdb-4b01-a06d-84ffb9836a61; _ga=GA1.2.378714859.1433690880; _xsrf=44f011b01f29816fc257fae1770a9ece; q_c1=fb660ee5c15b4c97ae2b4b075373e5b6|1457189679000|1433392648000; udid="AIAAQIMSlAmPTohRXib_bmZtsg_JbPW-tC8=|1457502059"; cap_id="YTFlMzQxMDY4OGY3NDA1NWIzYmU1MTEyYWQ0ZTMwODE=|1458058974|f565d65c26a2e091d9f042d19f5a16c73aa3c82d"; __utmt=1; z_c0="QUJETUhSWnAzUWdYQUFBQVlRSlZUU3JFRDFkRHJ2blhaazNJUUxiOFg3Uk1xTlhSa0hucml3PT0=|1458059050|e9ac863e03e6c396e1cf138b8c40eb6fb6003fff"; n_c=1; __utma=51854390.1008998230.1458040825.1458046212.1458055545.3; __utmb=51854390.75.9.1458055549055; __utmc=51854390; __utmz=51854390.1458010773.3.2.utmcsr=zhihu.com|utmccn=(referral)|utmcmd=referral|utmcct=/people/starkwei/followers; __utmv=51854390.100-2|2=registration_date=20151017=1^3=entry_date=20150604=1', + cookie:'', } \ No newline at end of file diff --git a/demo.png b/demo.png new file mode 100644 index 0000000..0aca34f Binary files /dev/null and b/demo.png differ diff --git a/echartParser.js b/echartParser.js deleted file mode 100644 index c3b7c45..0000000 --- a/echartParser.js +++ /dev/null @@ -1,96 +0,0 @@ -var input = [{ - "user": { - "hash_id": "d965f32a168564f9e58ad3a48a1585a4", - "followeeAmount": 91, - "followerAmount": 370, - "name": "Stark伟" - }, - "sameFriends": [] -}, { - "user": { - "hash_id": "4408b66aa5134567ad5591528d214c49", - "followeeAmount": 85, - "followerAmount": 50, - "name": "张凌祺" - }, - "sameFriends": [{ - "hash_id": "aed1cb0287a43c8aaa493cad5951b9e0", - "name": "符植煜", - "url": "https://www.zhihu.com/people/fu-zhi-yu-76" - }] -}, { - "user": { - "hash_id": "5d620c9af458e02062bdd4bf8ea11db2", - "followeeAmount": 15, - "followerAmount": 4, - "name": "谢箴" - }, - "sameFriends": [{ - "hash_id": "aed1cb0287a43c8aaa493cad5951b9e0", - "name": "符植煜", - "url": "https://www.zhihu.com/people/fu-zhi-yu-76" - }] -}, { - "user": { - "hash_id": "aed1cb0287a43c8aaa493cad5951b9e0", - "followeeAmount": 194, - "followerAmount": 158, - "name": "符植煜" - }, - "sameFriends": [{ - "hash_id": "4408b66aa5134567ad5591528d214c49", - "name": "张凌祺", - "url": "https://www.zhihu.com/people/zhang-ling-qi-64" - }, { - "hash_id": "5d620c9af458e02062bdd4bf8ea11db2", - "name": "谢箴", - "url": "https://www.zhihu.com/people/xie-zhen-94" - }] -}] - -function echartParser(input) { - var hash_id_to_common_id = {} - input.forEach(function(item, index) { - hash_id_to_common_id[item.user.hash_id] = index + 1; - }); - var data = []; - data.push({ - "id": 0, - "name": "我", - "symbolSize": 20, //item.user.followerAmount, - "label": { - "normal": { "show": true } - } - }) - input.forEach(function(item, index) { - data.push({ - "id": hash_id_to_common_id[item.user.hash_id], - "name": item.user.name, - "symbolSize": 10, //item.user.followerAmount, - "label": { - "normal": { "show": true } - }, - draggable: true - }) - }); - - var links = []; - input.forEach(function(item) { - links.push({ - source: 0, - target: hash_id_to_common_id[item.user.hash_id] - }) - item.sameFriends.forEach(function(item2) { - links.push({ - source: hash_id_to_common_id[item.user.hash_id], - target: hash_id_to_common_id[item2.hash_id] - }) - }) - }); - return { - data: data, - links: links - } -} - -module.exports = echartParser; diff --git a/fetchFollwerOrFollwee.js b/fetchFollwerOrFollwee.js index 5d81695..e7890ba 100644 --- a/fetchFollwerOrFollwee.js +++ b/fetchFollwerOrFollwee.js @@ -12,7 +12,7 @@ var fetchFollwerOrFollwee = function(options, socket) { } return Promise.map(offsets, function(offset) { return getFollwerOrFollwee(user, offset, isFollowees, socket); - }, { concurrency: 3 }).then(function(array) { + }, { concurrency: 2 }).then(function(array) { var result = []; array.forEach(function(item) { result = result.concat(item); @@ -41,7 +41,7 @@ function getFollwerOrFollwee(user, offset, isFollowees, socket) { 'cache-control': 'no-cache', 'x-requested-with': 'XMLHttpRequest' }, - timeout: 3000 + timeout: 1500 }, function(err, res, body) { var tmp = []; try { diff --git a/index.js b/index.js index 184c676..a57a9b5 100644 --- a/index.js +++ b/index.js @@ -1,6 +1,5 @@ var Spider = require('./Spider'); var express = require('express'); -var echartParser = require('./echartParser'); var bodyParser = require('body-parser'); var app = express(); @@ -22,13 +21,4 @@ app.get('/', function(req, res) { res.sendFile(__dirname + '/client/index.html'); }); -// app.get('/test', function(req, res) { -// Spider('https://www.zhihu.com/people/starkwei') -// .then(function(result) { -// var data = echartParser(result); -// console.log(data); -// res.send(data); -// }) -// }); - app.listen(3000);