Skip to content

Commit

Permalink
fetch
Browse files Browse the repository at this point in the history
  • Loading branch information
@dancerphil
dancerphil committed Mar 21, 2016
2 parents c0a0a3b + c91902e commit bc036e6
Show file tree
Hide file tree
Showing 11 changed files with 72 additions and 224 deletions.
8 changes: 0 additions & 8 deletions README.md
View file

This file was deleted.

4 changes: 2 additions & 2 deletions Spider.js
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
var fetchFollwerOrFollwee = require('./fetchFollwerOrFollwee');
var getUser = require('./getUser');
var Promise = require('bluebird');
var echartParser = require('./echartParser');

module.exports = Spider;

Expand All @@ -10,6 +9,7 @@ function Spider(userPageUrl, socket) {
return getUser(userPageUrl)
.then(function(user) {
socket.emit('notice', '抓取用户信息成功');
socket.emit('get user', user);
return getFriends(user, socket);
})
.then(function(myFriends) {
Expand All @@ -30,7 +30,7 @@ function Spider(userPageUrl, socket) {
console.log(myFriends);
return Promise.map(myFriends, function(myFriend) {
return searchSameFriend(myFriend, myFriends, socket);
}, { concurrency: 2 });
}, { concurrency: 3 });
})
.then(function(result) {
var data = result;
Expand Down
85 changes: 33 additions & 52 deletions client/build/bundle.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@
var $ = __webpack_require__(397);
var myChart = echarts.init(document.getElementById('main'));
var echartParser = __webpack_require__(398);

var USER_NAME;
$("#submit").click(function() {
socket.emit('fetch start', {
url: $("#url").val()
Expand All @@ -61,6 +61,7 @@
console.log(data);
dataStore = data;
result = echartParser(dataStore);
result.data[0].name = USER_NAME;
var option = {
title: {
text: 'Les Miserables',
Expand Down Expand Up @@ -94,6 +95,7 @@
if (item.user.hash_id == data.hash_id) {
item.sameFriends = data.sameFriends;
result = echartParser(dataStore);
result.data[0].name = USER_NAME;
var option = {
title: {
text: 'Les Miserables',
Expand Down Expand Up @@ -123,6 +125,9 @@
}
})
})
socket.on('get user',function(user){
USER_NAME = user.name;
})
socket.on('log', function(data) {
console.log(data);
});
Expand All @@ -131,39 +136,6 @@
socket.on('notice', function(data) {
notice.html(data);
});
// 基于准备好的dom，初始化echarts实例

// 绘制图表

// $.get('/test', function(data) {
// console.log(data);
// option = {
// title: {
// text: 'Les Miserables',
// subtext: 'Default layout',
// top: 'bottom',
// left: 'right'
// },
// tooltip: {},
// animationDuration: 1500,
// animationEasingUpdate: 'quinticInOut',
// series: [{
// name: 'Les Miserables',
// type: 'graph',
// layout: 'force',
// data: data.data,
// links: data.links,
// roam: true,
// label: {
// normal: {
// position: 'right',
// formatter: '{b}'
// }
// },
// }]
// };
// myChart.setOption(option);
// })


/***/ },
Expand Down Expand Up @@ -71489,9 +71461,13 @@
}]

function echartParser(input) {
console.log(input);
var hash_id_to_common_id = {}
input.forEach(function(item, index) {
hash_id_to_common_id[item.user.hash_id] = index + 1;
if (item.user.hash_id) {
hash_id_to_common_id[item.user.hash_id] = index + 1;
console.log(item.user.hash_id, index + 1, item);
}
});
var data = [];
data.push({
Expand All @@ -71503,30 +71479,35 @@
}
})
input.forEach(function(item, index) {
data.push({
"id": hash_id_to_common_id[item.user.hash_id],
"name": item.user.name,
"symbolSize": 10, //item.user.followerAmount,
"label": {
"normal": { "show": true }
},
draggable: true
})
if (item.user.hash_id) {
data.push({
"id": hash_id_to_common_id[item.user.hash_id],
"name": item.user.name,
"symbolSize": 10, //item.user.followerAmount,
"label": {
"normal": { "show": true }
},
draggable: true
})
}
});

var links = [];
input.forEach(function(item) {
links.push({
source: 0,
target: hash_id_to_common_id[item.user.hash_id]
})
item.sameFriends.forEach(function(item2) {
if (item.user.hash_id) {
links.push({
source: hash_id_to_common_id[item.user.hash_id],
target: hash_id_to_common_id[item2.hash_id]
source: 0,
target: hash_id_to_common_id[item.user.hash_id]
})
})
item.sameFriends.forEach(function(item2) {
links.push({
source: hash_id_to_common_id[item.user.hash_id],
target: hash_id_to_common_id[item2.hash_id]
})
})
}
});
console.log("data:" + data);
return {
data: data,
links: links
Expand Down
2 changes: 1 addition & 1 deletion client/index.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

<body>
<p id="notice"></p>
<input type="text" id="url">
<input type="text" id="url" placeholder="用户页的url，例如https://www.zhihu.com/people/starkwei" style="width:350px;">
<button id="submit">开始抓取</button>
<div id="main" style="width: 1000px;height:800px;"></div>
</body>
Expand Down
45 changes: 27 additions & 18 deletions client/src/echartParser.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,13 @@ var input = [{
}]

function echartParser(input) {
console.log(input);
var hash_id_to_common_id = {}
input.forEach(function(item, index) {
hash_id_to_common_id[item.user.hash_id] = index + 1;
if (item.user.hash_id) {
hash_id_to_common_id[item.user.hash_id] = index + 1;
console.log(item.user.hash_id, index + 1, item);
}
});
var data = [];
data.push({
Expand All @@ -63,30 +67,35 @@ function echartParser(input) {
}
})
input.forEach(function(item, index) {
data.push({
"id": hash_id_to_common_id[item.user.hash_id],
"name": item.user.name,
"symbolSize": 10, //item.user.followerAmount,
"label": {
"normal": { "show": true }
},
draggable: true
})
if (item.user.hash_id) {
data.push({
"id": hash_id_to_common_id[item.user.hash_id],
"name": item.user.name,
"symbolSize": 10, //item.user.followerAmount,
"label": {
"normal": { "show": true }
},
draggable: true
})
}
});

var links = [];
input.forEach(function(item) {
links.push({
source: 0,
target: hash_id_to_common_id[item.user.hash_id]
})
item.sameFriends.forEach(function(item2) {
if (item.user.hash_id) {
links.push({
source: hash_id_to_common_id[item.user.hash_id],
target: hash_id_to_common_id[item2.hash_id]
source: 0,
target: hash_id_to_common_id[item.user.hash_id]
})
})
item.sameFriends.forEach(function(item2) {
links.push({
source: hash_id_to_common_id[item.user.hash_id],
target: hash_id_to_common_id[item2.hash_id]
})
})
}
});
console.log("data:" + data);
return {
data: data,
links: links
Expand Down
40 changes: 6 additions & 34 deletions client/src/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ var socket = require('socket.io-client')('http://localhost:3001');
var $ = require('jquery');
var myChart = echarts.init(document.getElementById('main'));
var echartParser = require('./echartParser');

var USER_NAME;
$("#submit").click(function() {
socket.emit('fetch start', {
url: $("#url").val()
Expand All @@ -15,6 +15,7 @@ socket.on('data', function(data) {
console.log(data);
dataStore = data;
result = echartParser(dataStore);
result.data[0].name = USER_NAME;
var option = {
title: {
text: 'Les Miserables',
Expand Down Expand Up @@ -48,6 +49,7 @@ socket.on('same friend', function(data) {
if (item.user.hash_id == data.hash_id) {
item.sameFriends = data.sameFriends;
result = echartParser(dataStore);
result.data[0].name = USER_NAME;
var option = {
title: {
text: 'Les Miserables',
Expand Down Expand Up @@ -77,6 +79,9 @@ socket.on('same friend', function(data) {
}
})
})
socket.on('get user',function(user){
USER_NAME = user.name;
})
socket.on('log', function(data) {
console.log(data);
});
Expand All @@ -85,36 +90,3 @@ var notice = $("#notice");
socket.on('notice', function(data) {
notice.html(data);
});
// 基于准备好的dom，初始化echarts实例

// 绘制图表

// $.get('/test', function(data) {
// console.log(data);
// option = {
// title: {
// text: 'Les Miserables',
// subtext: 'Default layout',
// top: 'bottom',
// left: 'right'
// },
// tooltip: {},
// animationDuration: 1500,
// animationEasingUpdate: 'quinticInOut',
// series: [{
// name: 'Les Miserables',
// type: 'graph',
// layout: 'force',
// data: data.data,
// links: data.links,
// roam: true,
// label: {
// normal: {
// position: 'right',
// formatter: '{b}'
// }
// },
// }]
// };
// myChart.setOption(option);
// })
2 changes: 1 addition & 1 deletion config.js
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
module.exports = {
cookie:'_za=c392e6c0-5bdb-4b01-a06d-84ffb9836a61; _ga=GA1.2.378714859.1433690880; _xsrf=44f011b01f29816fc257fae1770a9ece; q_c1=fb660ee5c15b4c97ae2b4b075373e5b6|1457189679000|1433392648000; udid="AIAAQIMSlAmPTohRXib_bmZtsg_JbPW-tC8=|1457502059"; cap_id="YTFlMzQxMDY4OGY3NDA1NWIzYmU1MTEyYWQ0ZTMwODE=|1458058974|f565d65c26a2e091d9f042d19f5a16c73aa3c82d"; __utmt=1; z_c0="QUJETUhSWnAzUWdYQUFBQVlRSlZUU3JFRDFkRHJ2blhaazNJUUxiOFg3Uk1xTlhSa0hucml3PT0=|1458059050|e9ac863e03e6c396e1cf138b8c40eb6fb6003fff"; n_c=1; __utma=51854390.1008998230.1458040825.1458046212.1458055545.3; __utmb=51854390.75.9.1458055549055; __utmc=51854390; __utmz=51854390.1458010773.3.2.utmcsr=zhihu.com|utmccn=(referral)|utmcmd=referral|utmcct=/people/starkwei/followers; __utmv=51854390.100-2|2=registration_date=20151017=1^3=entry_date=20150604=1',
cookie:'',
}
Binary file added demo.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading

0 comments on commit bc036e6

Please sign in to comment.