diff --git a/Dockerfile b/Dockerfile index f416341..2fd3a26 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,21 +1,3 @@ -FROM tikazyq/crawlab:latest +FROM jelastic/nodejs:8.16.1-npm -ENV NVM_DIR /usr/local/nvm -ENV NODE_VERSION 8.12.0 - -RUN curl https://raw.githubusercontent.com/creationix/nvm/v0.24.0/install.sh | bash \ - && . $NVM_DIR/nvm.sh \ - && nvm install v$NODE_VERSION \ - && nvm use v$NODE_VERSION \ - && nvm alias default v$NODE_VERSION -ENV NODE_PATH $NVM_DIR/versions/node/v$NODE_VERSION/lib/node_modules -ENV PATH $NVM_DIR/versions/node/v$NODE_VERSION/bin:$PATH - -# frontend port -EXPOSE 8080 - -# backend port -EXPOSE 8000 - -# start backend -CMD ["/bin/sh", "/app/docker_init.sh"] +COPY . /app diff --git a/cli.js b/cli.js index ae1b081..09da67c 100644 --- a/cli.js +++ b/cli.js @@ -2,18 +2,43 @@ const os = require('os') const exec = require('child_process').exec const path = require('path') -const program = require('commander') +const program = require('caporal') program - .command('start') - .action(async () => { + .command('start', 'Start ArtiPub server') + .option('-H, --host', 'MongoDB host name', null) + .option('-P, --port', 'MongoDB port number', null, '27017') + .option('-d, --db', 'MongoDB database name', null, 'artipub') + .option('-u, --username', 'MongoDB username', null, '') + .option('-p, --password', 'MongoDB password', null, '') + .action((...arr) => { + const cmdObj = arr[arr.length - 2] + const umiCmd = path.join( __dirname, 'node_modules', '.bin', - os.platform().match(/^win/) ? 'umi' : 'umi.cmd' - ) - exec(umiCmd, ['dev']) + os.platform() + .match(/^win/) ? 'umi.cmd' : 'umi' + ) + ' dev' + + // 开启前端服务 + console.log(umiCmd) + exec(umiCmd, { shell: true }) + + const host = cmdObj.host || 'localhost' + const port = cmdObj.port || '27017' + const db = cmdObj.db || 'artipub' + const username = cmdObj.username || '' + const password = cmdObj.password || '' + + process.env.MONGO_HOST = host + process.env.MONGO_PORT = port + process.env.MONGO_DB = db + process.env.MONGO_USERNAME = username + process.env.MONGO_PASSWORD = password + + // 开启后段服务 require('./server') }) diff --git a/config.js b/config.js index 365365a..d4797ac 100644 --- a/config.js +++ b/config.js @@ -6,7 +6,4 @@ module.exports = { MONGO_DB: 'artipub', MONGO_USERNAME: '', MONGO_PASSWORD: '', - - // HEADLESS: true - HEADLESS: false } diff --git a/config/config.ts b/config/config.ts index 5209d58..d1ff8f7 100644 --- a/config/config.ts +++ b/config/config.ts @@ -75,8 +75,9 @@ export const apiEndpoint = 'http://localhost:3000'; export default { plugins, + history: 'hash', block: { - defaultGitUrl: 'https://github.com/ant-design/pro-blocks', + defaultGitUrl: 'https://github.com/crawlab-team/artipub', }, hash: true, targets: { @@ -145,6 +146,18 @@ export default { icon: 'read', component: './ArticleList/ArticleList', }, + { + path: '/helper', + name: 'helper', + icon: 'key', + component: './Helper/Helper', + }, + { + path: '/environments', + name: 'environments', + icon: 'setting', + component: './Environment/EnvironmentList', + }, { component: './404', }, diff --git a/constants.js b/constants.js index 60d9c5c..17b7a60 100644 --- a/constants.js +++ b/constants.js @@ -1,23 +1,34 @@ module.exports = { - platform: { - JUEJIN: 'juejin', - SEGMENTFAULT: 'segmentfault', - JIANSHU: 'jianshu', - CSDN: 'csdn', - ZHIHU: 'zhihu' - }, - status: { - NOT_STARTED: 'not-started', - PROCESSING: 'processing', - FINISHED: 'finished', - ERROR: 'error' - }, - authType: { - LOGIN: 'login', - COOKIES: 'cookie' - }, - editorType: { - MARKDOWN: 'markdown', - RICH_TEXT: 'rich-text' - } + platform: { + JUEJIN: 'juejin', + SEGMENTFAULT: 'segmentfault', + JIANSHU: 'jianshu', + CSDN: 'csdn', + ZHIHU: 'zhihu', + OSCHINA: 'oschina', + TOUTIAO: 'toutiao', + }, + status: { + NOT_STARTED: 'not-started', + PROCESSING: 'processing', + FINISHED: 'finished', + ERROR: 'error' + }, + authType: { + LOGIN: 'login', + COOKIES: 'cookie' + }, + editorType: { + MARKDOWN: 'markdown', + RICH_TEXT: 'rich-text' + }, + environment: { + UPDATE_STATS_CRON: 'update_stats_cron', + ENABLE_CHROME_DEBUG: 'enable_chrome_debug', + }, + cookieStatus: { + NO_COOKIE: 'no_cookie', + EXPIRED: 'expired', + EXISTS: 'exists', + }, } diff --git a/credentials.example.json b/credentials.example.json deleted file mode 100644 index c94b58e..0000000 --- a/credentials.example.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "juejin": { - "username": "", - "password": "" - }, - "segmentfault": { - "username": "", - "password": "" - }, - "csdn": { - "username": "", - "password": "" - }, - "jianshu": { - "username": "", - "password": "" - }, - "zhihu": { - "username": "", - "password": "" - } -} diff --git a/credentials.json b/credentials.json deleted file mode 100644 index c94b58e..0000000 --- a/credentials.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "juejin": { - "username": "", - "password": "" - }, - "segmentfault": { - "username": "", - "password": "" - }, - "csdn": { - "username": "", - "password": "" - }, - "jianshu": { - "username": "", - "password": "" - }, - "zhihu": { - "username": "", - "password": "" - } -} diff --git a/data.js b/data.js index 5755be2..5b7eea9 100644 --- a/data.js +++ b/data.js @@ -1,45 +1,83 @@ const constants = require('./constants') module.exports = { - platforms: [ - { - name: constants.platform.JUEJIN, - label: '掘金', - editorType: constants.editorType.MARKDOWN, - url: 'https://juejin.im', - description: '掘金是一个帮助开发者成长的社区，是给开发者用的 Hacker News，给设计师用的 Designer News，和给产品经理用的 Medium。掘金的技术文章由稀土上聚集的技术大牛和极客共同编辑为你筛选出最优质的干货，其中包括：Android、iOS、前端、后端等方面的内容。用户每天都可以在这里找到技术世界的头条内容。与此同时，掘金内还有沸点、掘金翻译计划、线下活动、专栏文章等内容。即使你是 GitHub、StackOverflow、开源中国的用户，我们相信你也可以在这里有所收获。', - enableImport: true, - }, - { - name: constants.platform.SEGMENTFAULT, - label: 'SegmentFault', - editorType: constants.editorType.MARKDOWN, - url: 'https://segmentfault.com', - description: 'SegmentFault 思否 为开发者提供问答、学习与交流编程知识的平台，创造属于开发者的时代！', - enableImport: false, - }, - { - name: constants.platform.CSDN, - label: 'CSDN', - editorType: constants.editorType.RICH_TEXT, - url: 'https://blog.csdn.net', - description: 'CSDN博客-专业IT技术发表平台', - enableImport: false, - }, - { - name: constants.platform.JIANSHU, - label: '简书', - editorType: constants.editorType.MARKDOWN, - url: 'https://jianshu.com', - description: '简书是一个优质的创作社区，在这里，你可以任性地创作，一篇短文、一张照片、一首诗、一幅画……我们相信，每个人都是生活中的艺术家，有着无穷的创造力。', - enableImport: false, - }, - { - name: constants.platform.ZHIHU, - label: '知乎', - editorType: constants.editorType.MARKDOWN, - url: 'https://zhihu.com', - description: '有问题，上知乎。知乎，可信赖的问答社区，以让每个人高效获得可信赖的解答为使命。知乎凭借认真、专业和友善的社区氛围，结构化、易获得的优质内容，基于问答的内容生产方式和独特的社区机制，吸引、聚集了各行各业中大量的亲历者、内行人、领域专家、领域爱好者，将高质量的内容透过人的节点来成规模地生产和分享。用户通过问答等交流方式建立信任和连接，打造和提升个人影响力，并发现、获得新机会。', - enableImport: false, - } - ] + // 平台 + platforms: [ + { + name: constants.platform.JUEJIN, + label: '掘金', + editorType: constants.editorType.MARKDOWN, + url: 'https://juejin.im', + description: '掘金是一个帮助开发者成长的社区，是给开发者用的 Hacker News，给设计师用的 Designer News，和给产品经理用的 Medium。掘金的技术文章由稀土上聚集的技术大牛和极客共同编辑为你筛选出最优质的干货，其中包括：Android、iOS、前端、后端等方面的内容。用户每天都可以在这里找到技术世界的头条内容。与此同时，掘金内还有沸点、掘金翻译计划、线下活动、专栏文章等内容。即使你是 GitHub、StackOverflow、开源中国的用户，我们相信你也可以在这里有所收获。', + enableImport: false, + enableLogin: true, + }, + { + name: constants.platform.SEGMENTFAULT, + label: 'SegmentFault', + editorType: constants.editorType.MARKDOWN, + url: 'https://segmentfault.com', + description: 'SegmentFault 思否 为开发者提供问答、学习与交流编程知识的平台，创造属于开发者的时代！', + enableImport: false, + enableLogin: true, + }, + { + name: constants.platform.CSDN, + label: 'CSDN', + editorType: constants.editorType.RICH_TEXT, + url: 'https://blog.csdn.net', + description: 'CSDN博客-专业IT技术发表平台', + enableImport: false, + enableLogin: false, + }, + { + name: constants.platform.JIANSHU, + label: '简书', + editorType: constants.editorType.MARKDOWN, + url: 'https://jianshu.com', + description: '简书是一个优质的创作社区，在这里，你可以任性地创作，一篇短文、一张照片、一首诗、一幅画……我们相信，每个人都是生活中的艺术家，有着无穷的创造力。', + enableImport: false, + enableLogin: false, + }, + { + name: constants.platform.ZHIHU, + label: '知乎', + editorType: constants.editorType.MARKDOWN, + url: 'https://zhihu.com', + description: '有问题，上知乎。知乎，可信赖的问答社区，以让每个人高效获得可信赖的解答为使命。知乎凭借认真、专业和友善的社区氛围，结构化、易获得的优质内容，基于问答的内容生产方式和独特的社区机制，吸引、聚集了各行各业中大量的亲历者、内行人、领域专家、领域爱好者，将高质量的内容透过人的节点来成规模地生产和分享。用户通过问答等交流方式建立信任和连接，打造和提升个人影响力，并发现、获得新机会。', + enableImport: false, + enableLogin: false, + }, + { + name: constants.platform.OSCHINA, + label: '开源中国', + editorType: constants.editorType.RICH_TEXT, + url: 'https://www.oschina.net', + description: 'OSCHINA.NET 是目前领先的中文开源技术社区。我们传播开源的理念，推广开源项目，为 IT 开发者提供了一个发现、使用、并交流开源技术的平台', + enableImport: false, + enableLogin: false, + }, + { + name: constants.platform.TOUTIAO, + label: '今日头条', + editorType: constants.editorType.MARKDOWN, + url: 'https://www.toutiao.com', + description: '《今日头条》(www.toutiao.com)是一款基于数据挖掘的推荐引擎产品，它为用户推荐有价值的、个性化的信息，提供连接人与信息的新型服务，是国内移动互联网领域成长最快的产品服务之一。', + enableImport: false, + enableLogin: false, + } + ], + + // 环境变量 + environments: [ + { + _id: constants.environment.UPDATE_STATS_CRON, + label: '更新文章统计数据频率', + value: '0 0/30 * * * *' + }, + { + _id: constants.environment.ENABLE_CHROME_DEBUG, + label: 'Chrome浏览器调试模式', + value: 'Y' + } + ] } diff --git a/exec.js b/exec.js index 4a186e6..1d1697e 100644 --- a/exec.js +++ b/exec.js @@ -20,7 +20,7 @@ class Runner { constructor() { } - run() { + async run() { // 任务执行器 const taskLock = new AsyncLock() const taskCronJob = new CronJob('* * * * * *', () => { @@ -42,15 +42,18 @@ class Runner { }) taskCronJob.start() + // 获取环境变量 + const updateStatsCron = await models.Environment.findOne({ _id: constants.environment.UPDATE_STATS_CRON }) + // 数据统计执行器 const statsLock = new AsyncLock() - const statsCronJob = new CronJob('0 0/30 * * * *', () => { + const statsCronJob = new CronJob(updateStatsCron.value, () => { if (!statsLock.isBusy()) { statsLock.acquire('key', async () => { const tasks = await models.Task.find({ url: { $ne: '', - $exists:true + $exists: true } }) for (let i = 0; i < tasks.length; i++) { diff --git a/extensions/src/popup/Popup.tsx b/extensions/src/popup/Popup.tsx index 9299d71..3686502 100644 --- a/extensions/src/popup/Popup.tsx +++ b/extensions/src/popup/Popup.tsx @@ -24,13 +24,7 @@ export default class Popup extends React.Component { chrome.runtime.sendMessage({popupMounted: true}); this.setState({ - allowedDomains: [ - 'juejin', - 'segmentfault', - 'jianshu', - 'csdn', - 'zhihu', - ], + allowedDomains: [], configVisible: false, url: 'http://localhost:3000', fetched: false, @@ -38,10 +32,17 @@ export default class Popup extends React.Component { }); } - onGetLoginInfo() { + async onGetLoginInfo() { this.setState({ loading: true }); + + const response = await axios.get(this.state.url + '/platforms'); + const platforms = response.data.data; + this.setState({ + allowedDomains: platforms.map((d: any) => d.name) + }); + chrome.cookies.getAllCookieStores(cookieStores => { // console.log(cookieStores); cookieStores.forEach(store => { diff --git a/init.js b/init.js index 2a77adb..00a0376 100644 --- a/init.js +++ b/init.js @@ -3,23 +3,44 @@ const data = require('./data') // 数据库初始化 const init = async () => { - for (let i = 0; i < data.platforms.length; i++) { - const platform = data.platforms[i] - let platformDb = await models.Platform.findOne({ name: platform.name }) - if (!platformDb) { - platformDb = new models.Platform(platform) - await platformDb.save() - } else { - for (let key in platform) { - if (platform.hasOwnProperty(key)) { - if (platform[key]) { - platformDb[key] = platform[key] - } - } - } - await platformDb.save() + // 初始化平台数据 + for (let i = 0; i < data.platforms.length; i++) { + const platform = data.platforms[i] + let platformDb = await models.Platform.findOne({ name: platform.name }) + if (!platformDb) { + platformDb = new models.Platform(platform) + await platformDb.save() + } else { + for (let key in platform) { + if (platform.hasOwnProperty(key)) { + if (platform[key] !== undefined) { + platformDb[key] = platform[key] + } } + } + await platformDb.save() } + } + + // 初始化环境变量数据 + for (let i = 0; i < data.environments.length; i++) { + const environment = data.environments[i] + let environmentDb = await models.Environment.findOne({ _id: environment._id }) + if (!environmentDb) { + environmentDb = new models.Environment(environment) + await environmentDb.save() + } else { + // do nothing + // for (let key in environment) { + // if (environment.hasOwnProperty(key)) { + // if (environment[key]) { + // environmentDb[key] = environment[key] + // } + // } + // } + // await environmentDb.save() + } + } } module.exports = init diff --git a/lib/ArticlePublisher.js b/lib/ArticlePublisher.js index 9e9cee7..7830a5b 100644 --- a/lib/ArticlePublisher.js +++ b/lib/ArticlePublisher.js @@ -4,49 +4,52 @@ const constants = require('../constants') const BaseExecutor = require('./BaseExecutor') class ArticlePublisher extends BaseExecutor { - async run() { - let task = this.task + async run() { + let task = this.task - // 判断任务状态 - if ( - task.status !== constants.status.NOT_STARTED && - task.status !== constants.status.ERROR - ) { - logger.info(`task (ID: ${task._id.toString()} has already been run. exit`) - return - } + // 判断任务状态 + if ( + task.status !== constants.status.NOT_STARTED && + task.status !== constants.status.ERROR + ) { + logger.info(`task (ID: ${task._id.toString()} has already been run. exit`) + return + } + + if (this.spider) { + try { + // 更新任务状态 + task.status = constants.status.PROCESSING + task.updateTs = new Date() + await task.save() - if (this.spider) { - try { - task.status = constants.status.PROCESSING - task.updateTs = new Date() - await task.save() - await this.spider.run() + // 运行爬虫 + await this.spider.run() - // 检查URL结果 - task = await models.Task.findOne({ _id: task._id }) - if (task.url) { - // URL保存成功 - task.status = constants.status.FINISHED - task.updateTs = new Date() - await task.save() - } else { - // URL保存失败 - task.status = constants.status.ERROR - task.error = '文章URL未保存成功' - task.updateTs = new Date() - await task.save() - } - } catch (e) { - task.status = constants.status.ERROR - task.error = e.toString() - task.updateTs = new Date() - await task.save() - console.error(e) - await this.spider.browser.close() - } + // 检查URL结果 + task = await models.Task.findOne({ _id: task._id }) + if (task.url) { + // URL保存成功 + task.status = constants.status.FINISHED + task.updateTs = new Date() + await task.save() + } else { + // URL保存失败 + task.status = constants.status.ERROR + task.error = '文章URL未保存成功' + task.updateTs = new Date() + await task.save() } + } catch (e) { + task.status = constants.status.ERROR + task.error = e.toString() + task.updateTs = new Date() + await task.save() + console.error(e) + await this.spider.browser.close() + } } + } } module.exports = ArticlePublisher diff --git a/lib/BaseExecutor.js b/lib/BaseExecutor.js index ece9d3a..c0f4517 100644 --- a/lib/BaseExecutor.js +++ b/lib/BaseExecutor.js @@ -27,6 +27,10 @@ class BaseExecutor { spider = new spiders.CsdnSpider(task._id) } else if (spiderName === constants.platform.ZHIHU) { spider = new spiders.ZhihuSpider(task._id) + } else if (spiderName === constants.platform.OSCHINA) { + spider = new spiders.OschinaSpider(task._id) + } else if (spiderName === constants.platform.TOUTIAO) { + spider = new spiders.ToutiaoSpider(task._id) } this.spider = spider } diff --git a/models/environment.js b/models/environment.js new file mode 100644 index 0000000..a1a1f27 --- /dev/null +++ b/models/environment.js @@ -0,0 +1,13 @@ +const mongoose = require('mongoose') + +const environmentSchema = new mongoose.Schema({ + _id: String, // key + label: String, // label + value: String, // value + updateTs: Date, + createTs: Date, +}) + +const Environment = mongoose.model('environments', environmentSchema) + +module.exports = Environment diff --git a/models/index.js b/models/index.js index 76b2c7a..e5b3604 100644 --- a/models/index.js +++ b/models/index.js @@ -1,6 +1,7 @@ module.exports = { - Article: require('./article'), - Task: require('./task'), - Platform: require('./platform'), - Cookie: require('./cookie'), + Article: require('./article'), + Task: require('./task'), + Platform: require('./platform'), + Cookie: require('./cookie'), + Environment: require('./environment'), } diff --git a/models/platform.js b/models/platform.js index 33df4c7..994f298 100644 --- a/models/platform.js +++ b/models/platform.js @@ -1,14 +1,20 @@ const mongoose = require('mongoose') const platformSchema = new mongoose.Schema({ - name: String, - label: String, - editorType: String, - description: String, - url: String, - enableImport: Boolean, - createTs: Date, - updateTs: Date, + name: String, + label: String, + editorType: String, + description: String, + url: String, + enableImport: Boolean, + enableLogin: Boolean, + username: String, + password: String, + createTs: Date, + updateTs: Date, + + // 前端字段 + cookieStatus: String, }) const Platform = mongoose.model('platforms', platformSchema) diff --git a/models/task.js b/models/task.js index aa88584..69de5a7 100644 --- a/models/task.js +++ b/models/task.js @@ -2,27 +2,27 @@ const mongoose = require('mongoose') const ObjectId = require('bson').ObjectId const taskSchema = new mongoose.Schema({ - articleId: ObjectId, - platformId: ObjectId, - status: String, - url: String, - createTs: Date, - updateTs: Date, - error: String, - checked: Boolean, - ready: Boolean, - authType: String, - readNum: Number, - likeNum: Number, - commentNum: Number, + articleId: ObjectId, + platformId: ObjectId, + status: String, + url: String, + createTs: Date, + updateTs: Date, + error: String, + checked: Boolean, + ready: Boolean, + authType: String, + readNum: Number, + likeNum: Number, + commentNum: Number, - // 配置信息 - category: String, // 类别: juejin - tag: String, // 标签: juejin (单选), segmentfault (逗号分割) - pubType: String, // 发布形式: csdn (单选) + // 配置信息 + category: String, // 类别: juejin + tag: String, // 标签: juejin (单选), segmentfault (逗号分割) + pubType: String, // 发布形式: csdn (单选) - // 前端数据（不用设置） - platform: Object, + // 前端数据（不用设置） + platform: Object, }) const Task = mongoose.model('tasks', taskSchema) diff --git a/package.json b/package.json index 427bae8..d293b8f 100644 --- a/package.json +++ b/package.json @@ -39,10 +39,10 @@ "antd": "^3.20.0", "async-lock": "^1.2.2", "body-parser": "^1.19.0", + "caporal": "^1.3.0", "classnames": "^2.2.6", "clipboardy": "^2.1.0", "codemirror": "^5.48.2", - "commander": "^3.0.1", "cors": "^2.8.5", "cron": "^1.7.1", "dva": "^2.4.1", @@ -66,6 +66,7 @@ "react-copy-to-clipboard": "^5.0.1", "react-document-title": "^2.0.3", "react-dom": "^16.8.6", + "react-markdown-editor-lite": "^0.4.3", "react-showdown": "^1.6.0", "redux": "^4.0.1", "showdown": "^1.9.0", diff --git a/public/artipub-helper.zip b/public/artipub-helper.zip new file mode 100644 index 0000000..9a0afc5 Binary files /dev/null and b/public/artipub-helper.zip differ diff --git a/routes/article.js b/routes/article.js index ede66ec..943bb29 100644 --- a/routes/article.js +++ b/routes/article.js @@ -3,131 +3,134 @@ const constants = require('../constants') const ObjectId = require('bson').ObjectId module.exports = { - getArticleList: async (req, res) => { - const articles = await models.Article.find().sort({ _id: -1 }) - for (let i = 0; i < articles.length; i++) { - const article = articles[i] - article.tasks = await models.Task.find({ articleId: article._id }) - const arr = ['readNum', 'likeNum', 'commentNum'] - arr.forEach(key => { - article[key] = 0 - article.tasks.forEach(task => { - article[key] += task[key] - }) - }) - } - await res.json({ - status: 'ok', - data: articles + getArticleList: async (req, res) => { + const articles = await models.Article.find().sort({ _id: -1 }) + for (let i = 0; i < articles.length; i++) { + const article = articles[i] + article.tasks = await models.Task.find({ articleId: article._id }) + const arr = ['readNum', 'likeNum', 'commentNum'] + arr.forEach(key => { + article[key] = 0 + article.tasks.forEach(task => { + article[key] += task[key] }) - }, - getArticle: async (req, res) => { - const article = await models.Article.findOne({ _id: ObjectId(req.params.id) }) - article.tasks = await models.Task.find({ articleId: article._id }) - await res.json({ - status: 'ok', - data: article - }) - }, - getArticleTaskList: async (req, res) => { - const article = await models.Article.findOne({ _id: ObjectId(req.params.id) }) - const tasks = await models.Task.find({ articleId: article._id }) - await res.json({ - status: 'ok', - data: tasks, - }) - }, - addArticle: async (req, res) => { - // 创建文章 - let article = new models.Article({ - title: req.body.title, - content: req.body.content, - contentHtml: req.body.contentHtml, - platformIds: [], - createTs: new Date(), - updateTs: new Date(), - }) - article = await article.save() - await res.json({ - status: 'ok', - data: article, - }) - }, - editArticle: async (req, res) => { - let article = await models.Article.findOne({ _id: ObjectId(req.params.id) }) - if (!article) { - return await res.json({ - status: 'ok', - error: 'not found' - }, 404) - } - article.title = req.body.title - article.content = req.body.content - article.contentHtml = req.body.contentHtml - article.updateTs = new Date() - article = await article.save() - await res.json({ - status: 'ok', - data: article, - }) - }, - deleteArticle: async (req, res) => { - let article = await models.Article.findOne({ _id: ObjectId(req.params.id) }) - if (!article) { - return await res.json({ - status: 'ok', - error: 'not found' - }, 404) - } - await models.Article.remove({ _id: ObjectId(req.params.id) }) - await res.json({ - status: 'ok', - data: req.body, - }) - }, - publishArticle: async (req, res) => { - let article = await models.Article.findOne({ _id: ObjectId(req.params.id) }) - if (!article) { - return await res.json({ - status: 'ok', - error: 'not found' - }, 404) - } - const tasks = await models.Task.find({ - articleId: article._id, - status: { - $in: [ - constants.status.NOT_STARTED, - constants.status.ERROR, - ] - }, - checked: true, - }) - for (let task of tasks) { - task.status = constants.status.NOT_STARTED - task.ready = true - task.updateTs = new Date() - await task.save() - } + }) + } + await res.json({ + status: 'ok', + data: articles + }) + }, + getArticle: async (req, res) => { + const article = await models.Article.findOne({ _id: ObjectId(req.params.id) }) + article.tasks = await models.Task.find({ articleId: article._id }) + await res.json({ + status: 'ok', + data: article + }) + }, + getArticleTaskList: async (req, res) => { + const article = await models.Article.findOne({ _id: ObjectId(req.params.id) }) + const tasks = await models.Task.find({ articleId: article._id }) + for (let i = 0; i < tasks.length; i++) { + tasks[i].platform = await models.Platform.findOne({ _id: tasks[i].platformId }) + } + await res.json({ + status: 'ok', + data: tasks, + }) + }, + addArticle: async (req, res) => { + // 创建文章 + let article = new models.Article({ + title: req.body.title, + content: req.body.content, + contentHtml: req.body.contentHtml, + platformIds: [], + createTs: new Date(), + updateTs: new Date(), + }) + article = await article.save() + await res.json({ + status: 'ok', + data: article, + }) + }, + editArticle: async (req, res) => { + let article = await models.Article.findOne({ _id: ObjectId(req.params.id) }) + if (!article) { + return await res.json({ + status: 'ok', + error: 'not found' + }, 404) + } + article.title = req.body.title + article.content = req.body.content + article.contentHtml = req.body.contentHtml + article.updateTs = new Date() + article = await article.save() + await res.json({ + status: 'ok', + data: article, + }) + }, + deleteArticle: async (req, res) => { + let article = await models.Article.findOne({ _id: ObjectId(req.params.id) }) + if (!article) { + return await res.json({ + status: 'ok', + error: 'not found' + }, 404) + } + await models.Article.remove({ _id: ObjectId(req.params.id) }) + await res.json({ + status: 'ok', + data: req.body, + }) + }, + publishArticle: async (req, res) => { + let article = await models.Article.findOne({ _id: ObjectId(req.params.id) }) + if (!article) { + return await res.json({ + status: 'ok', + error: 'not found' + }, 404) + } + const tasks = await models.Task.find({ + articleId: article._id, + status: { + $in: [ + constants.status.NOT_STARTED, + constants.status.ERROR, + ] + }, + checked: true, + }) + for (let task of tasks) { + task.status = constants.status.NOT_STARTED + task.ready = true + task.updateTs = new Date() + await task.save() + } - await res.json({ - status: 'ok', - }) - }, - addArticleTask: async (req, res) => { - let task = new models.Task({ - articleId: ObjectId(req.params.id), - platformId: ObjectId(req.body.platformId), - status: constants.status.NOT_STARTED, - createTs: new Date(), - updateTs: new Date(), - category: req.body.category, - tag: req.body.tag, - }) - task = await task.save() - await res.json({ - status: 'ok', - data: task, - }) - }, + await res.json({ + status: 'ok', + }) + }, + addArticleTask: async (req, res) => { + let task = new models.Task({ + articleId: ObjectId(req.params.id), + platformId: ObjectId(req.body.platformId), + status: constants.status.NOT_STARTED, + createTs: new Date(), + updateTs: new Date(), + category: req.body.category, + tag: req.body.tag, + }) + task = await task.save() + await res.json({ + status: 'ok', + data: task, + }) + }, } diff --git a/routes/environment.js b/routes/environment.js new file mode 100644 index 0000000..f557f3f --- /dev/null +++ b/routes/environment.js @@ -0,0 +1,56 @@ +const models = require('../models') + +module.exports = { + getEnvList: async (req, res) => { + const environments = await models.Environment.find() + await res.json({ + status: 'ok', + data: environments + }) + }, + addEnv: async (req, res) => { + let env = new models.Environment({ + _id: req.body._id, + value: req.body.value, + label: req.body.label, + updateTs: new Date(), + createTs: new Date(), + }) + env = await env.save() + await res.json({ + status: 'ok', + data: env, + }) + }, + editEnv: async (req, res) => { + let env = await models.Environment.findOne({ _id: req.body._id }) + if (!env) { + return res.json({ + status: 'ok', + error: 'not found' + }, 404) + } + env.value = req.body.value + env.label = req.body.label + env.updateTs = new Date() + env = await env.save() + res.json({ + status: 'ok', + data: env, + }) + }, + deleteEnv: async (req, res) => { + let env = await models.Environment.findOne({ _id: req.body._id }) + if (!env) { + return res.json({ + status: 'ok', + error: 'not found' + }, 404) + } + await models.Environment.remove({ _id: req.body._id }) + await res.json({ + status: 'ok', + data: req.body, + }) + }, +} diff --git a/routes/index.js b/routes/index.js index d3ee44e..4cbafde 100644 --- a/routes/index.js +++ b/routes/index.js @@ -1,6 +1,7 @@ module.exports = { - article: require('./article'), - task: require('./task'), - platform: require('./platform'), - cookie: require('./cookie'), + article: require('./article'), + task: require('./task'), + platform: require('./platform'), + cookie: require('./cookie'), + environment: require('./environment'), } diff --git a/routes/platform.js b/routes/platform.js index 2e16625..2c1ef55 100644 --- a/routes/platform.js +++ b/routes/platform.js @@ -1,147 +1,164 @@ -const models = require('../models') const ObjectId = require('bson').ObjectId +const constants = require('../constants') +const models = require('../models') + +const getCookieStatus = async (platform) => { + const cookies = await models.Cookie.find({ domain: { $regex: platform.name } }) + if (!cookies || !cookies.length) return constants.cookieStatus.NO_COOKIE + return constants.cookieStatus.EXISTS +} module.exports = { - getPlatformList: async (req, res) => { - const platforms = await models.Platform.find() - await res.json({ - status: 'ok', - data: platforms - }) - }, - getPlatform: async (req, res) => { - const platform = await models.Platform.findOne({ _id: ObjectId(req.params.id) }) - await res.json({ - status: 'ok', - data: platform - }) - }, - addPlatform: async (req, res) => { - let Platform = new models.Platform({ - name: req.body.name, - label: req.body.label, - editorType: req.body.editorType, - description: req.body.description, - enableImport: req.body.enableImport, - createTs: new Date(), - updateTs: new Date() - }) - Platform = await Platform.save() - await res.json({ - status: 'ok', - data: Platform - }) - }, - editPlatform: async (req, res) => { - let platform = await models.Platform.findOne({ _id: ObjectId(req.params.id) }) - if (!platform) { - return await res.json({ - status: 'ok', - error: 'not found' - }, 404) - } - platform.name = req.body.name - platform.label = req.body.label - platform.editorType = req.body.editorType - platform.description = req.body.description - platform.enableImport = req.body.enableImport - platform.updateTs = new Date() - platform.save() - await res.json({ - status: 'ok', - data: platform - }) - }, - deletePlatform: async (req, res) => { - let platform = await models.Platform.findOne({ _id: ObjectId(req.params.id) }) - if (!platform) { - return await res.json({ - status: 'ok', - error: 'not found' - }, 404) - } - await models.Platform.remove({ _id: ObjectId(req.params.id) }) - await res.json({ - status: 'ok', - data: platform - }) - }, - getPlatformArticles: async (req, res) => { - // 获取平台 - const platform = await models.Platform.findOne({ _id: ObjectId(req.params.id) }) - - // 如果平台不存在，返回404错误 - if (!platform) { - return await res.json({ - status: 'ok', - error: 'not found' - }, 404) - } - - // 获取导入爬虫类 - const ImportSpider = require('../spiders/import/' + platform.name) - - // 导入爬虫实例 - const spider = new ImportSpider(platform.name) - - // 获取网站文章列表 - const siteArticles = await spider.fetch() - - // 遍历网站文章列表 - for (let i = 0; i < siteArticles.length; i++) { - // 当前网站文章 - const siteArticle = siteArticles[i] - - // 根据title查找数据库中文章 - const article = await models.Article.findOne({ title: siteArticle.title }) - - // 网站文章是否存在 - siteArticles[i].exists = !!article - - // 尝试查找网站文章关联的任务 - let task - if (article) { - siteArticles[i].articleId = article._id - task = await models.Task.findOne({ platformId: platform._id, articleId: article._id }) - } - - // 网站文章是否已关联 - siteArticles[i].associated = !!(task && task.url && task.url === siteArticle.url) - } - - // 返回结果 - await res.json({ - status: 'ok', - data: siteArticles - }) - }, - importPlatformArticles: async (req, res) => { - // 获取平台 - const platform = await models.Platform.findOne({ _id: ObjectId(req.params.id) }) - - // 如果平台不存在，返回404错误 - if (!platform) { - return await res.json({ - status: 'ok', - error: 'not found' - }, 404) - } - - // 获取导入爬虫类 - const ImportSpider = require('../spiders/import/' + platform.name) - - // 导入爬虫实例 - const spider = new ImportSpider(platform.name) - - // 获取网站文章列表 - const siteArticles = req.body - - // 开始导入 - await spider.import(siteArticles) - - // 返回结果 - await res.json({ - status: 'ok' - }) + getPlatformList: async (req, res) => { + const platforms = await models.Platform.find() + for (let i = 0; i < platforms.length; i++) { + platforms[i].cookieStatus = await getCookieStatus(platforms[i]) + } + await res.json({ + status: 'ok', + data: platforms + }) + }, + getPlatform: async (req, res) => { + const platform = await models.Platform.findOne({ _id: ObjectId(req.params.id) }) + platform.cookieStatus = await getCookieStatus(d) + await res.json({ + status: 'ok', + data: platform + }) + }, + addPlatform: async (req, res) => { + let Platform = new models.Platform({ + name: req.body.name, + label: req.body.label, + editorType: req.body.editorType, + description: req.body.description, + enableImport: req.body.enableImport, + enableLogin: req.body.enableLogin, + username: req.body.username, + password: req.body.password, + createTs: new Date(), + updateTs: new Date() + }) + Platform = await Platform.save() + await res.json({ + status: 'ok', + data: Platform + }) + }, + editPlatform: async (req, res) => { + let platform = await models.Platform.findOne({ _id: ObjectId(req.params.id) }) + if (!platform) { + return await res.json({ + status: 'ok', + error: 'not found' + }, 404) + } + platform.name = req.body.name + platform.label = req.body.label + platform.editorType = req.body.editorType + platform.description = req.body.description + platform.enableImport = req.body.enableImport + platform.enableLogin = req.body.enableLogin + platform.username = req.body.username + platform.password = req.body.password + platform.updateTs = new Date() + platform.save() + await res.json({ + status: 'ok', + data: platform + }) + }, + deletePlatform: async (req, res) => { + let platform = await models.Platform.findOne({ _id: ObjectId(req.params.id) }) + if (!platform) { + return await res.json({ + status: 'ok', + error: 'not found' + }, 404) } + await models.Platform.remove({ _id: ObjectId(req.params.id) }) + await res.json({ + status: 'ok', + data: platform + }) + }, + getPlatformArticles: async (req, res) => { + // 获取平台 + const platform = await models.Platform.findOne({ _id: ObjectId(req.params.id) }) + + // 如果平台不存在，返回404错误 + if (!platform) { + return await res.json({ + status: 'ok', + error: 'not found' + }, 404) + } + + // 获取导入爬虫类 + const ImportSpider = require('../spiders/import/' + platform.name) + + // 导入爬虫实例 + const spider = new ImportSpider(platform.name) + + // 获取网站文章列表 + const siteArticles = await spider.fetch() + + // 遍历网站文章列表 + for (let i = 0; i < siteArticles.length; i++) { + // 当前网站文章 + const siteArticle = siteArticles[i] + + // 根据title查找数据库中文章 + const article = await models.Article.findOne({ title: siteArticle.title }) + + // 网站文章是否存在 + siteArticles[i].exists = !!article + + // 尝试查找网站文章关联的任务 + let task + if (article) { + siteArticles[i].articleId = article._id + task = await models.Task.findOne({ platformId: platform._id, articleId: article._id }) + } + + // 网站文章是否已关联 + siteArticles[i].associated = !!(task && task.url && task.url === siteArticle.url) + } + + // 返回结果 + await res.json({ + status: 'ok', + data: siteArticles + }) + }, + importPlatformArticles: async (req, res) => { + // 获取平台 + const platform = await models.Platform.findOne({ _id: ObjectId(req.params.id) }) + + // 如果平台不存在，返回404错误 + if (!platform) { + return await res.json({ + status: 'ok', + error: 'not found' + }, 404) + } + + // 获取导入爬虫类 + const ImportSpider = require('../spiders/import/' + platform.name) + + // 导入爬虫实例 + const spider = new ImportSpider(platform.name) + + // 获取网站文章列表 + const siteArticles = req.body + + // 开始导入 + await spider.import(siteArticles) + + // 返回结果 + await res.json({ + status: 'ok' + }) + } } diff --git a/routes/task.js b/routes/task.js index da66b8e..3a75828 100644 --- a/routes/task.js +++ b/routes/task.js @@ -5,174 +5,174 @@ const exec = require('child_process').exec const path = require('path') module.exports = { - getTaskList: async (req, res) => { - const tasks = await models.Task.find() - await res.json({ - status: 'ok', - data: tasks - }) - }, - getTask: async (req, res) => { - const task = await models.Task.findOne({ _id: ObjectId(req.params.id) }) - await res.json({ - status: 'ok', - data: task - }) - }, - addTasks: async (req, res) => { - for (let _task of req.body) { - let task - if (_task._id) { - task = await models.Task.findOne({ _id: ObjectId(_task._id) }) - task.category = _task.category - task.tag = _task.tag - task.pubType = _task.pubType - task.updateTs = new Date() - task.checked = _task.checked - task.authType = _task.authType - } else { - task = new models.Task({ - articleId: ObjectId(_task.articleId), - platformId: ObjectId(_task.platformId), - status: constants.status.NOT_STARTED, - createTs: new Date(), - updateTs: new Date(), - checked: _task.checked, - authType: _task.authType, - - // 配置信息 - category: _task.category, - tag: _task.tag, - pubType: _task.pubType, - }) - } - task = await task.save() - } - await res.json({ - status: 'ok', - }) - }, - addTask: async (req, res) => { - let task = new models.Task({ - articleId: ObjectId(req.body.articleId), - platformId: ObjectId(req.body.platformId), - status: constants.status.NOT_STARTED, - createTs: new Date(), - updateTs: new Date(), - - // 配置信息 - category: req.body.category, - tag: req.body.tag, - }) - task = await task.save() - await res.json({ - status: 'ok', - data: task, - }) - }, - editTask: async (req, res) => { - let task = await models.Task.findOne({ _id: ObjectId(req.params.id) }) - if (!task) { - return res.json({ - status: 'ok', - error: 'not found' - }, 404) - } - task.category = req.body.category - task.tag = req.body.tag + getTaskList: async (req, res) => { + const tasks = await models.Task.find() + await res.json({ + status: 'ok', + data: tasks + }) + }, + getTask: async (req, res) => { + const task = await models.Task.findOne({ _id: ObjectId(req.params.id) }) + await res.json({ + status: 'ok', + data: task + }) + }, + addTasks: async (req, res) => { + for (let _task of req.body) { + let task + if (_task._id) { + task = await models.Task.findOne({ _id: ObjectId(_task._id) }) + task.category = _task.category + task.tag = _task.tag + task.pubType = _task.pubType task.updateTs = new Date() - task = await task.save() - res.json({ - status: 'ok', - data: task, - }) - }, - deleteTask: async (req, res) => { - let task = await models.Task.findOne({ _id: ObjectId(req.params.id) }) - if (!task) { - return res.json({ - status: 'ok', - error: 'not found' - }, 404) - } - await models.Task.remove({ _id: ObjectId(req.params.id) }) - await res.json({ - status: 'ok', - data: req.body, + task.checked = _task.checked + task.authType = _task.authType + } else { + task = new models.Task({ + articleId: ObjectId(_task.articleId), + platformId: ObjectId(_task.platformId), + status: constants.status.NOT_STARTED, + createTs: new Date(), + updateTs: new Date(), + checked: _task.checked, + authType: _task.authType, + + // 配置信息 + category: _task.category, + tag: _task.tag, + pubType: _task.pubType, }) - }, - publishTask: async (req, res) => { - let Task = await models.Task.findOne({ _id: ObjectId(req.params.id) }) - if (!Task) { - return res.json({ - status: 'ok', - error: 'not found' - }, 404) - } - const platforms = req.body.platforms.split(',') - let isError = false - let errMsg = '' - for (let i = 0; i < platforms.length; i++) { - if (isError) break - const platform = platforms[i] + } + task = await task.save() + } + await res.json({ + status: 'ok', + }) + }, + addTask: async (req, res) => { + let task = new models.Task({ + articleId: ObjectId(req.body.articleId), + platformId: ObjectId(req.body.platformId), + status: constants.status.NOT_STARTED, + createTs: new Date(), + updateTs: new Date(), - // 获取执行路径 - let execPath - if (platform === 'juejin') { - execPath = 'juejin/juejin_spider.js' - } else if (platform === 'segmentfault') { - execPath = 'segmentfault/segmentfault_spider.js' - } else if (platform === 'jianshu') { - execPath = 'jianshu/jianshu_spider.js' - } else { - continue - } - const filePath = path.join(__dirname, '..', '..', 'spiders', execPath) + // 配置信息 + category: req.body.category, + tag: req.body.tag, + }) + task = await task.save() + await res.json({ + status: 'ok', + data: task, + }) + }, + editTask: async (req, res) => { + let task = await models.Task.findOne({ _id: ObjectId(req.params.id) }) + if (!task) { + return res.json({ + status: 'ok', + error: 'not found' + }, 404) + } + task.category = req.body.category + task.tag = req.body.tag + task.updateTs = new Date() + task = await task.save() + res.json({ + status: 'ok', + data: task, + }) + }, + deleteTask: async (req, res) => { + let task = await models.Task.findOne({ _id: ObjectId(req.params.id) }) + if (!task) { + return res.json({ + status: 'ok', + error: 'not found' + }, 404) + } + await models.Task.remove({ _id: ObjectId(req.params.id) }) + await res.json({ + status: 'ok', + data: req.body, + }) + }, + publishTask: async (req, res) => { + let Task = await models.Task.findOne({ _id: ObjectId(req.params.id) }) + if (!Task) { + return res.json({ + status: 'ok', + error: 'not found' + }, 404) + } + const platforms = req.body.platforms.split(',') + let isError = false + let errMsg = '' + for (let i = 0; i < platforms.length; i++) { + if (isError) break + const platform = platforms[i] - // 初始化平台 - if (!Task.platforms[platform]) { - Task.platforms[platform] = {} - } + // 获取执行路径 + let execPath + if (platform === 'juejin') { + execPath = 'juejin/juejin_spider.js' + } else if (platform === 'segmentfault') { + execPath = 'segmentfault/segmentfault_spider.js' + } else if (platform === 'jianshu') { + execPath = 'jianshu/jianshu_spider.js' + } else { + continue + } + const filePath = path.join(__dirname, '..', '..', 'spiders', execPath) - // 初始化执行结果 - if (Task.platforms[platform].url || Task.platforms[platform].status === 'processing') { - // 如果结果已经存在或状态为正在处理，跳过 - console.log(`skipped ${platform}`) - continue - } else { - Task.platforms[platform] = { - status: 'processing', - updateTs: new Date(), - } - await Task.updateOne(Task) - } + // 初始化平台 + if (!Task.platforms[platform]) { + Task.platforms[platform] = {} + } - console.log(`node ${filePath} ${Task._id.toString()}`) - await exec(`node ${filePath} ${Task._id.toString()}`, (err, stdout, stderr) => { - if (err) { - console.error(stderr) - isError = true - errMsg = stderr - Task.platforms[platform] = { - status: 'error', - updateTs: new Date(), - error: errMsg, - } - Task.updateOne(Task) - } - }) + // 初始化执行结果 + if (Task.platforms[platform].url || Task.platforms[platform].status === 'processing') { + // 如果结果已经存在或状态为正在处理，跳过 + console.log(`skipped ${platform}`) + continue + } else { + Task.platforms[platform] = { + status: 'processing', + updateTs: new Date(), } + await Task.updateOne(Task) + } - if (isError) { - await res.json({ - status: 'ok', - error: errMsg, - }, 500) - } else { - await res.json({ - status: 'ok', - data: Task, - }) + console.log(`node ${filePath} ${Task._id.toString()}`) + await exec(`node ${filePath} ${Task._id.toString()}`, (err, stdout, stderr) => { + if (err) { + console.error(stderr) + isError = true + errMsg = stderr + Task.platforms[platform] = { + status: 'error', + updateTs: new Date(), + error: errMsg, + } + Task.updateOne(Task) } + }) + } + + if (isError) { + await res.json({ + status: 'ok', + error: errMsg, + }, 500) + } else { + await res.json({ + status: 'ok', + data: Task, + }) } + } } diff --git a/server.js b/server.js index 0d53517..163f22e 100644 --- a/server.js +++ b/server.js @@ -12,12 +12,19 @@ const logger = require('./logger') // express实例 const app = express() +// 环境变量覆盖 +if (process.env.MONGO_HOST) config.MONGO_HOST = process.env.MONGO_HOST +if (process.env.MONGO_PORT) config.MONGO_PORT = process.env.MONGO_PORT +if (process.env.MONGO_DB) config.MONGO_DB = process.env.MONGO_DB +if (process.env.MONGO_USERNAME) config.MONGO_USERNAME = process.env.MONGO_USERNAME +if (process.env.MONGO_PASSWORD) config.MONGO_PASSWORD = process.env.MONGO_PASSWORD + // mongodb连接 mongoose.Promise = global.Promise if (config.MONGO_USERNAME) { - mongoose.connect(`mongodb://${ config.MONGO_USERNAME }:${ config.MONGO_PASSWORD }@${ config.MONGO_HOST }:${ config.MONGO_PORT }/${ config.MONGO_DB }`, { useNewUrlParser: true }) + mongoose.connect(`mongodb://${config.MONGO_USERNAME}:${config.MONGO_PASSWORD}@${config.MONGO_HOST}:${config.MONGO_PORT}/${config.MONGO_DB}`, { useNewUrlParser: true }) } else { - mongoose.connect(`mongodb://${ config.MONGO_HOST }:${ config.MONGO_PORT }/${ config.MONGO_DB }`, { useNewUrlParser: true }) + mongoose.connect(`mongodb://${config.MONGO_HOST}:${config.MONGO_PORT}/${config.MONGO_DB}`, { useNewUrlParser: true }) } // bodyParser中间件 @@ -28,15 +35,15 @@ app.use(morgan('dev')) // 跨域cors app.use('*', function (req, res, next) { - res.header('Access-Control-Allow-Origin', req.headers.origin) - res.header('Access-Control-Allow-Credentials', true) - res.header('Access-Control-Allow-Headers', 'Content-Type, Content-Length, Authorization, Accept, X-Requested-With') - res.header('Access-Control-Allow-Methods', 'PUT, POST, GET, DELETE, OPTIONS')//设置方法 - if (req.method === 'OPTIONS') { - res.send(200) // 意思是，在正常的请求之前，会发送一个验证，是否可以请求。 - } else { - next() - } + res.header('Access-Control-Allow-Origin', req.headers.origin) + res.header('Access-Control-Allow-Credentials', true) + res.header('Access-Control-Allow-Headers', 'Content-Type, Content-Length, Authorization, Accept, X-Requested-With') + res.header('Access-Control-Allow-Methods', 'PUT, POST, GET, DELETE, OPTIONS')//设置方法 + if (req.method === 'OPTIONS') { + res.send(200) // 意思是，在正常的请求之前，会发送一个验证，是否可以请求。 + } else { + next() + } }) // 路由 @@ -66,10 +73,13 @@ app.get('/platforms/:id/articles', routes.platform.getPlatformArticles) app.post('/platforms/:id/articles', routes.platform.importPlatformArticles) // Cookie app.post('/cookies', routes.cookie.addCookies) +// Environment +app.get('/environments', routes.environment.getEnvList) +app.post('/environments', routes.environment.editEnv) // 启动express server app.listen(config.PORT, () => { - logger.info('listening on port ' + config.PORT) + logger.info('listening on port ' + config.PORT) }) // 初始化 diff --git a/spiders/base.js b/spiders/base.js index 6d5391e..9216ac6 100644 --- a/spiders/base.js +++ b/spiders/base.js @@ -1,6 +1,5 @@ const PCR = require('puppeteer-chromium-resolver') const ObjectId = require('bson').ObjectId -const credentials = require('../credentials') const models = require('../models') const constants = require('../constants') const config = require('./config') @@ -8,282 +7,280 @@ const globalConfig = require('../config') const logger = require('../logger') class BaseSpider { - constructor(taskId) { - if (!taskId) { - throw new Error('taskId must not be empty') - } - - // 任务ID - this.taskId = taskId - } - - async init() { - // 任务 - this.task = await models.Task.findOne({ _id: ObjectId(this.taskId) }) - if (!this.task) { - throw new Error(`task (ID: ${this.taskId}) cannot be found`) - } - - // 文章 - this.article = await models.Article.findOne({ _id: this.task.articleId }) - if (!this.article) { - throw new Error(`article (ID: ${this.task.articleId.toString()}) cannot be found`) - } - - // 平台 - this.platform = await models.Platform.findOne({ _id: this.task.platformId }) - if (!this.platform) { - throw new Error(`platform (ID: ${this.task.platformId.toString()}) cannot be found`) - } - - // 用户名密码 - this.credential = credentials[this.platform.name] - if (!this.credential) { - throw new Error(`credential (platform: ${this.platform.name}) cannot be found`) - } - - // PCR - this.pcr = await PCR({ - revision: '', - detectionPath: '', - folderName: '.chromium-browser-snapshots', - hosts: ['https://storage.googleapis.com', 'https://npm.taobao.org/mirrors'], - retry: 3, - silent: false - }) - - // 浏览器 - this.browser = await this.pcr.puppeteer.launch({ - executablePath: this.pcr.executablePath, - //设置超时时间 - timeout: 120000, - //如果是访问https页面 此属性会忽略https错误 - ignoreHTTPSErrors: true, - // 打开开发者工具, 当此值为true时, headless总为false - devtools: false, - // 关闭headless模式, 不会打开浏览器 - headless: globalConfig.HEADLESS - }) - - // 页面 - this.page = await this.browser.newPage() - - // 状态 - this.status = { - loggedIn: false, - completedEditor: false, - published: false, - } - - // 配置 - this.config = config[this.platform.name] - if (!config) { - throw new Error(`config (platform: ${this.platform.name}) cannot be found`) - } - - // URL信息 - this.urls = this.config.urls - - // 登陆选择器 - this.loginSel = this.config.loginSel - - // 编辑器选择器 - this.editorSel = this.config.editorSel - - // 隐藏navigator - await this.page.evaluate(() => { - Object.defineProperty(navigator, 'webdriver', { - get: () => false - }) - }) + constructor(taskId) { + if (!taskId) { + throw new Error('taskId must not be empty') } - /** - * 登陆网站 - */ - async login() { - logger.info(`logging in... navigating to ${this.urls.login}`) - await this.page.goto(this.urls.login) - let errNum = 0 - while (errNum < 10) { - try { - await this.page.waitFor(1000) - const elUsername = await this.page.$(this.loginSel.username) - const elPassword = await this.page.$(this.loginSel.password) - const elSubmit = await this.page.$(this.loginSel.submit) - await elUsername.type(this.credential.username) - await elPassword.type(this.credential.password) - await elSubmit.click() - await this.page.waitFor(3000) - break - } catch (e) { - errNum++ - } - } - - // 查看是否登陆成功 - this.status.loggedIn = errNum !== 10 - - if (this.status.loggedIn) { - logger.info('Logged in') - } - } + // 任务ID + this.taskId = taskId + } - /** - * 设置Cookie - */ - async setCookies() { - const cookies = await models.Cookie.find({ domain: { $regex: this.platform.name } }) - for (let i = 0; i < cookies.length; i++) { - const c = cookies[i] - await this.page.setCookie({ - name: c.name, - value: c.value, - domain: c.domain, - }) - } + async init() { + // 任务 + this.task = await models.Task.findOne({ _id: ObjectId(this.taskId) }) + if (!this.task) { + throw new Error(`task (ID: ${this.taskId}) cannot be found`) } - /** - * 导航至写作页面 - */ - async goToEditor() { - logger.info(`navigating to ${this.urls.editor}`) - await this.page.goto(this.urls.editor) - await this.page.waitFor(5000) - await this.afterGoToEditor() + // 文章 + this.article = await models.Article.findOne({ _id: this.task.articleId }) + if (!this.article) { + throw new Error(`article (ID: ${this.task.articleId.toString()}) cannot be found`) } - async afterGoToEditor() { - // 导航至写作页面的后续处理 - // 掘金等网站会先弹出Markdown页面，需要特殊处理 + // 平台 + this.platform = await models.Platform.findOne({ _id: this.task.platformId }) + if (!this.platform) { + throw new Error(`platform (ID: ${this.task.platformId.toString()}) cannot be found`) } - async inputTitle(article, editorSel) { - const el = document.querySelector(editorSel.title) - el.focus() - el.select() - document.execCommand('delete', false) - document.execCommand('insertText', false, article.title) + // PCR + this.pcr = await PCR({ + revision: '', + detectionPath: '', + folderName: '.chromium-browser-snapshots', + hosts: ['https://storage.googleapis.com', 'https://npm.taobao.org/mirrors'], + retry: 3, + silent: false + }) + + // 是否开启chrome浏览器调试 + const enableChromeDebugEnv = await models.Environment.findOne({_id: constants.environment.ENABLE_CHROME_DEBUG}) + const enableChromeDebug = enableChromeDebugEnv.value + + // 浏览器 + this.browser = await this.pcr.puppeteer.launch({ + executablePath: this.pcr.executablePath, + //设置超时时间 + timeout: 120000, + //如果是访问https页面 此属性会忽略https错误 + ignoreHTTPSErrors: true, + // 打开开发者工具, 当此值为true时, headless总为false + devtools: false, + // 关闭headless模式, 不会打开浏览器 + headless: enableChromeDebug !== 'Y' + }) + + // 页面 + this.page = await this.browser.newPage() + + // 状态 + this.status = { + loggedIn: false, + completedEditor: false, + published: false, } - async inputContent(article, editorSel) { - const el = document.querySelector(editorSel.content) - el.focus() - el.select() - document.execCommand('delete', false) - document.execCommand('insertText', false, article.content) + // 配置 + this.config = config[this.platform.name] + if (!config) { + throw new Error(`config (platform: ${this.platform.name}) cannot be found`) } - async inputFooter(article, editorSel) { - const footerContent = `\n\n> 本篇文章由[ArtiPub](https://github.com/crawlab-team/artipub)自动发布, ArtiPub让您的文章随处可阅` - const el = document.querySelector(editorSel.content) - el.focus() - document.execCommand('insertText', false, footerContent) - } - - async inputEditor() { - logger.info(`input editor title and content`) - // 输入标题 - await this.page.evaluate(this.inputTitle, this.article, this.editorSel) - await this.page.waitFor(3000) - - // 输入内容 - await this.page.evaluate(this.inputContent, this.article, this.editorSel) - await this.page.waitFor(3000) - - // 输入脚注 - await this.page.evaluate(this.inputFooter, this.article, this.editorSel) + // URL信息 + this.urls = this.config.urls + + // 登陆选择器 + this.loginSel = this.config.loginSel + + // 编辑器选择器 + this.editorSel = this.config.editorSel + + // 隐藏navigator + await this.page.evaluate(() => { + Object.defineProperty(navigator, 'webdriver', { + get: () => false + }) + }) + } + + /** + * 登陆网站 + */ + async login() { + logger.info(`logging in... navigating to ${this.urls.login}`) + await this.page.goto(this.urls.login) + let errNum = 0 + while (errNum < 10) { + try { + await this.page.waitFor(1000) + const elUsername = await this.page.$(this.loginSel.username) + const elPassword = await this.page.$(this.loginSel.password) + const elSubmit = await this.page.$(this.loginSel.submit) + await elUsername.type(this.platform.username) + await elPassword.type(this.platform.password) + await elSubmit.click() await this.page.waitFor(3000) - - // 后续处理 - await this.afterInputEditor() + break + } catch (e) { + errNum++ + } } - async afterInputEditor() { - // 输入编辑器内容的后续处理 - // 标签、分类输入放在这里 - } - - async publish() { - logger.info(`publishing article`) - // 发布文章 - const elPub = await this.page.$(this.editorSel.publish) - await elPub.click() - await this.page.waitFor(10000) + // 查看是否登陆成功 + this.status.loggedIn = errNum !== 10 - // 后续处理 - await this.afterPublish() + if (this.status.loggedIn) { + logger.info('Logged in') } - - async afterPublish() { - // 提交文章的后续处理 - // 保存文章url等逻辑放在这里 + } + + /** + * 设置Cookie + */ + async setCookies() { + const cookies = await models.Cookie.find({ domain: { $regex: this.platform.name } }) + for (let i = 0; i < cookies.length; i++) { + const c = cookies[i] + await this.page.setCookie({ + name: c.name, + value: c.value, + domain: c.domain, + }) } - - async run() { - // 初始化 - await this.init() - - if (this.task.authType === constants.authType.LOGIN) { - // 登陆 - await this.login() - } else { - // 使用Cookie - await this.setCookies() - } - - // 导航至编辑器 - await this.goToEditor() - - // 输入编辑器内容 - await this.inputEditor() - - // 发布文章 - await this.publish() - - // 关闭浏览器 - await this.browser.close() + } + + /** + * 导航至写作页面 + */ + async goToEditor() { + logger.info(`navigating to ${this.urls.editor}`) + await this.page.goto(this.urls.editor) + await this.page.waitFor(5000) + await this.afterGoToEditor() + } + + async afterGoToEditor() { + // 导航至写作页面的后续处理 + // 掘金等网站会先弹出Markdown页面，需要特殊处理 + } + + async inputTitle(article, editorSel) { + const el = document.querySelector(editorSel.title) + el.focus() + el.select() + document.execCommand('delete', false) + document.execCommand('insertText', false, article.title) + } + + async inputContent(article, editorSel) { + const el = document.querySelector(editorSel.content) + el.focus() + el.select() + document.execCommand('delete', false) + document.execCommand('insertText', false, article.content) + } + + async inputFooter(article, editorSel) { + const footerContent = `\n\n> 本篇文章由[ArtiPub](https://github.com/crawlab-team/artipub)自动发布, ArtiPub让您的文章随处可阅` + const el = document.querySelector(editorSel.content) + el.focus() + document.execCommand('insertText', false, footerContent) + } + + async inputEditor() { + logger.info(`input editor title and content`) + // 输入标题 + await this.page.evaluate(this.inputTitle, this.article, this.editorSel) + await this.page.waitFor(3000) + + // 输入内容 + await this.page.evaluate(this.inputContent, this.article, this.editorSel) + await this.page.waitFor(3000) + + // 输入脚注 + await this.page.evaluate(this.inputFooter, this.article, this.editorSel) + await this.page.waitFor(3000) + + // 后续处理 + await this.afterInputEditor() + } + + async afterInputEditor() { + // 输入编辑器内容的后续处理 + // 标签、分类输入放在这里 + } + + async publish() { + logger.info(`publishing article`) + // 发布文章 + const elPub = await this.page.$(this.editorSel.publish) + await elPub.click() + await this.page.waitFor(10000) + + // 后续处理 + await this.afterPublish() + } + + async afterPublish() { + // 提交文章的后续处理 + // 保存文章url等逻辑放在这里 + } + + async run() { + // 初始化 + await this.init() + + if (this.task.authType === constants.authType.LOGIN) { + // 登陆 + await this.login() + } else { + // 使用Cookie + await this.setCookies() } - async fetchStats() { - // to be inherited - } - - async afterFetchStats() { - // 统计文章总阅读、点赞、评论数 - const tasks = await models.Task.find({articleId: this.article._id}) - let readNum = 0 - let likeNum = 0 - let commentNum = 0 - for (let i = 0; i < tasks.length; i++) { - const task = tasks[i] - readNum += task.readNum || 0 - likeNum += task.likeNum || 0 - commentNum += task.commentNum || 0 - } - this.article.readNum = readNum - this.article.likeNum = likeNum - this.article.commentNum = commentNum - await this.article.save() - } - - /** - * 获取文章数据 - */ - async runFetchStats() { - // 初始化 - await this.init() - - // 获取文章数据 - await this.fetchStats() - - // 后续操作 - await this.afterFetchStats() - - // 关闭浏览器 - await this.browser.close() + // 导航至编辑器 + await this.goToEditor() + + // 输入编辑器内容 + await this.inputEditor() + + // 发布文章 + await this.publish() + + // 关闭浏览器 + await this.browser.close() + } + + async fetchStats() { + // to be inherited + } + + async afterFetchStats() { + // 统计文章总阅读、点赞、评论数 + const tasks = await models.Task.find({ articleId: this.article._id }) + let readNum = 0 + let likeNum = 0 + let commentNum = 0 + for (let i = 0; i < tasks.length; i++) { + const task = tasks[i] + readNum += task.readNum || 0 + likeNum += task.likeNum || 0 + commentNum += task.commentNum || 0 } + this.article.readNum = readNum + this.article.likeNum = likeNum + this.article.commentNum = commentNum + await this.article.save() + } + + /** + * 获取文章数据 + */ + async runFetchStats() { + // 初始化 + await this.init() + + // 获取文章数据 + await this.fetchStats() + + // 后续操作 + await this.afterFetchStats() + + // 关闭浏览器 + await this.browser.close() + } } module.exports = BaseSpider diff --git a/spiders/config.js b/spiders/config.js index 418df04..3e1c36b 100644 --- a/spiders/config.js +++ b/spiders/config.js @@ -1,84 +1,116 @@ module.exports = { - juejin: { - urls: { - login: 'https://juejin.im/login', - editor: 'https://juejin.im/editor/drafts/new' - }, - loginSel: { - username: '.input[name="loginPhoneOrEmail"]', - password: '.input[name="loginPassword"]', - submit: '.btn:nth-child(3)' - }, - editorSel: { - title: '.title-input', - content: '.ace_text-input', - publish: '.publish-btn' - } + juejin: { + urls: { + login: 'https://juejin.im/login', + editor: 'https://juejin.im/editor/drafts/new' }, + loginSel: { + username: '.input[name="loginPhoneOrEmail"]', + password: '.input[name="loginPassword"]', + submit: '.btn:nth-child(3)' + }, + editorSel: { + title: '.title-input', + content: '.ace_text-input', + publish: '.publish-btn' + } + }, + + segmentfault: { + urls: { + login: 'https://segmentfault.com/user/login', + editor: 'https://segmentfault.com/write' + }, + loginSel: { + username: 'input[name="username"]', + password: 'input[name="password"]', + submit: 'button[type="submit"]' + }, + editorSel: { + title: '#myTitle', + content: '#myEditor', + publish: '#publishIt' + } + }, + + jianshu: { + urls: { + login: 'https://www.jianshu.com/sign_in', + editor: 'https://www.jianshu.com/writer' + }, + loginSel: { + username: '', + password: '', + submit: '' + }, + editorSel: { + title: 'input:not([name="name"])', + content: '#arthur-editor', + publish: 'a[data-action="publicize"]' + } + }, - segmentfault: { - urls: { - login: 'https://segmentfault.com/user/login', - editor: 'https://segmentfault.com/write' - }, - loginSel: { - username: 'input[name="username"]', - password: 'input[name="password"]', - submit: 'button[type="submit"]' - }, - editorSel: { - title: '#myTitle', - content: '#myEditor', - publish: '#publishIt' - } + csdn: { + urls: { + login: '', + editor: 'https://mp.csdn.net/postedit' + }, + loginSel: { + username: '', + password: '' }, + editorSel: { + title: '#txtTitle', + content: '.htmledit_views', + publish: '#btnPublish' + } + }, - jianshu: { - urls: { - login: 'https://www.jianshu.com/sign_in', - editor: 'https://www.jianshu.com/writer' - }, - loginSel: { - username: '', - password: '', - submit: '' - }, - editorSel: { - title: 'input:not([name="name"])', - content: '#arthur-editor', - publish: 'a[data-action="publicize"]' - } + zhihu: { + urls: { + login: '', + editor: 'https://zhuanlan.zhihu.com/write' }, + loginSel: { + username: '', + password: '' + }, + editorSel: { + title: '.WriteIndex-titleInput > .Input', + content: '.public-DraftEditor-content', + publish: '.PublishPanel-stepTwoButton' + } + }, - csdn: { - urls: { - login: '', - editor: 'https://mp.csdn.net/postedit' - }, - loginSel: { - username: '', - password: '' - }, - editorSel: { - title: '#txtTitle', - content: '.htmledit_views', - publish: '#btnPublish' - } + oschina: { + urls: { + login: '', + editor: '' }, + loginSel: { + username: '', + password: '' + }, + editorSel: { + title: 'input[name="title"]', + content: '.cke_editable', + publish: '.submit' + } + }, - zhihu: { - urls: { - login: '', - editor: 'https://zhuanlan.zhihu.com/write' - }, - loginSel: { - username: '', - password: '' - }, - editorSel: { - title: '.WriteIndex-titleInput > .Input', - content: '.public-DraftEditor-content', - publish: '.PublishPanel-stepTwoButton' - } + toutiao: { + urls: { + login: '', + editor: 'https://mp.toutiao.com/profile_v3/graphic/publish' + }, + loginSel: { + username: '', + password: '' + }, + editorSel: { + title: '#title', + content: '.ql-editor', + publish: '#publish' } + } } diff --git a/spiders/csdn.js b/spiders/csdn.js index dbd70e7..e996bf8 100644 --- a/spiders/csdn.js +++ b/spiders/csdn.js @@ -2,76 +2,76 @@ const constants = require('../constants') const BaseSpider = require('./base') class CsdnSpider extends BaseSpider { - async afterGoToEditor() { - await this.page.evaluate(() => { - const el = document.querySelector('#btnStart') - if (el) el.click() - }) - await this.page.waitFor(1000) - } + async afterGoToEditor() { + await this.page.evaluate(() => { + const el = document.querySelector('#btnStart') + if (el) el.click() + }) + await this.page.waitFor(1000) + } - async inputContent(article, editorSel) { - const iframeWindow = document.querySelector('.cke_wysiwyg_frame').contentWindow - const el = iframeWindow.document.querySelector(editorSel.content) - el.focus() - iframeWindow.document.execCommand('delete', false) - iframeWindow.document.execCommand('insertHTML', false, article.contentHtml) - } + async inputContent(article, editorSel) { + const iframeWindow = document.querySelector('.cke_wysiwyg_frame').contentWindow + const el = iframeWindow.document.querySelector(editorSel.content) + el.focus() + iframeWindow.document.execCommand('delete', false) + iframeWindow.document.execCommand('insertHTML', false, article.contentHtml) + } - async afterInputEditor() { - // 选择文章类型 - await this.page.evaluate(task => { - const el = document.querySelector('#selType') - el.value = task.category - }, this.task) + async afterInputEditor() { + // 选择文章类型 + await this.page.evaluate(task => { + const el = document.querySelector('#selType') + el.value = task.category + }, this.task) - // 选择发布形式 - await this.page.evaluate(task => { - const el = document.querySelector('#' + task.pubType) - el.click() - }, this.task) - } + // 选择发布形式 + await this.page.evaluate(task => { + const el = document.querySelector('#' + task.pubType) + el.click() + }, this.task) + } - async inputFooter(article, editorSel) { - const footerContent = `
本文由文章发布工具ArtiPub自动生成` - const iframeWindow = document.querySelector('.cke_wysiwyg_frame').contentWindow - const el = iframeWindow.document.querySelector(editorSel.content) - el.focus() - iframeWindow.document.execCommand('insertHTML', false, footerContent) - } + async inputFooter(article, editorSel) { + const footerContent = `
本篇文章由一文多发平台ArtiPub自动发布` + const iframeWindow = document.querySelector('.cke_wysiwyg_frame').contentWindow + const el = iframeWindow.document.querySelector(editorSel.content) + el.focus() + iframeWindow.document.execCommand('insertHTML', false, footerContent) + } - async afterPublish() { - this.task.url = await this.page.evaluate(() => { - const el = document.querySelector('.toarticle') - return el.getAttribute('href') - }) - this.task.updateTs = new Date() - await this.task.save() - } + async afterPublish() { + this.task.url = await this.page.evaluate(() => { + const el = document.querySelector('.toarticle') + return el.getAttribute('href') + }) + this.task.updateTs = new Date() + await this.task.save() + } - async fetchStats() { - if (!this.task.url) return - await this.page.goto(this.task.url, { timeout: 60000 }) - await this.page.waitFor(5000) + async fetchStats() { + if (!this.task.url) return + await this.page.goto(this.task.url, { timeout: 60000 }) + await this.page.waitFor(5000) - const stats = await this.page.evaluate(() => { - const text = document.querySelector('body').innerText - const mRead = text.match(/阅读数 (\d+)/) - const readNum = mRead ? Number(mRead[1]) : 0 - const likeNum = Number(document.querySelector('#supportCount').innerText) - const commentNum = 0 // 暂时获取不了评论数 - return { - readNum, - likeNum, - commentNum - } - }) - this.task.readNum = stats.readNum - this.task.likeNum = stats.likeNum - this.task.commentNum = stats.commentNum - await this.task.save() - await this.page.waitFor(3000) - } + const stats = await this.page.evaluate(() => { + const text = document.querySelector('body').innerText + const mRead = text.match(/阅读数 (\d+)/) + const readNum = mRead ? Number(mRead[1]) : 0 + const likeNum = Number(document.querySelector('#supportCount').innerText) + const commentNum = 0 // 暂时获取不了评论数 + return { + readNum, + likeNum, + commentNum + } + }) + this.task.readNum = stats.readNum + this.task.likeNum = stats.likeNum + this.task.commentNum = stats.commentNum + await this.task.save() + await this.page.waitFor(3000) + } } module.exports = CsdnSpider diff --git a/spiders/import/base.js b/spiders/import/base.js index 66680c8..b65e61d 100644 --- a/spiders/import/base.js +++ b/spiders/import/base.js @@ -11,110 +11,110 @@ showdown.setOption('tasklists', true) showdown.setFlavor('github') class BaseImportSpider extends BaseSpider { - constructor(platformName) { - super(BaseSpider) - if (!platformName) { - throw new Error('platformId must not be empty') - } - this.platformName = platformName + constructor(platformName) { + super(BaseSpider) + if (!platformName) { + throw new Error('platformId must not be empty') } - - async init() { - // 平台 - this.platform = await models.Platform.findOne({ name: this.platformName }) - - // PCR - this.pcr = await PCR({ - revision: '', - detectionPath: '', - folderName: '.chromium-browser-snapshots', - hosts: ['https://storage.googleapis.com', 'https://npm.taobao.org/mirrors'], - retry: 3, - silent: false - }) - - // 浏览器 - this.browser = await this.pcr.puppeteer.launch({ - executablePath: this.pcr.executablePath, - timeout: 60000, - //如果是访问https页面 此属性会忽略https错误 - ignoreHTTPSErrors: true, - devtools: false, - headless: globalConfig.HEADLESS, - args: [ - '--no-sandbox', - '--disable-setuid-sandbox' - ] - }) - - // 页面 - this.page = await this.browser.newPage() - - // 设置 浏览器视窗 - await this.page.setViewport({ - width: 1300, - height: 938 - }) - - // 配置 - this.config = config[this.platform.name] - if (!config) { - throw new Error(`config (platform: ${this.platform.name}) cannot be found`) - } - - // 编辑器选择器 - this.editorSel = this.config.editorSel - - // showdown配置 - showdown.setOption('tables', true) - showdown.setOption('tasklists', true) - showdown.setFlavor('github') - - // markdown to html转换器 - this.converter = new showdown.Converter() - } - - async fetchArticles() { - // to be overridden + this.platformName = platformName + } + + async init() { + // 平台 + this.platform = await models.Platform.findOne({ name: this.platformName }) + + // PCR + this.pcr = await PCR({ + revision: '', + detectionPath: '', + folderName: '.chromium-browser-snapshots', + hosts: ['https://storage.googleapis.com', 'https://npm.taobao.org/mirrors'], + retry: 3, + silent: false + }) + + // 是否开启chrome浏览器调试 + const enableChromeDebugEnv = await models.Environment.findOne({_id: constants.environment.ENABLE_CHROME_DEBUG}) + const enableChromeDebug = enableChromeDebugEnv.value + + // 浏览器 + this.browser = await this.pcr.puppeteer.launch({ + executablePath: this.pcr.executablePath, + timeout: 60000, + //如果是访问https页面 此属性会忽略https错误 + ignoreHTTPSErrors: true, + devtools: false, + headless: enableChromeDebug !== 'Y', + }) + + // 页面 + this.page = await this.browser.newPage() + + // 设置 浏览器视窗 + await this.page.setViewport({ + width: 1300, + height: 938 + }) + + // 配置 + this.config = config[this.platform.name] + if (!config) { + throw new Error(`config (platform: ${this.platform.name}) cannot be found`) } - async fetch() { - logger.info('fetching articles') - - await this.init() - await this.setCookies() - try { - await this.page.goto(this.platform.url, { timeout: 60000 }) - } catch (e) { - console.error(e) - await this.browser.close() - return [] - } - await this.page.waitFor(5000) - const articles = await this.fetchArticles() - await this.browser.close() - return articles + // 编辑器选择器 + this.editorSel = this.config.editorSel + + // showdown配置 + showdown.setOption('tables', true) + showdown.setOption('tasklists', true) + showdown.setFlavor('github') + + // markdown to html转换器 + this.converter = new showdown.Converter() + } + + async fetchArticles() { + // to be overridden + } + + async fetch() { + logger.info('fetching articles') + + await this.init() + await this.setCookies() + try { + await this.page.goto(this.platform.url, { timeout: 60000 }) + } catch (e) { + console.error(e) + await this.browser.close() + return [] } - - async importArticle(siteArticle) { - // to be overridden + await this.page.waitFor(5000) + const articles = await this.fetchArticles() + await this.browser.close() + return articles + } + + async importArticle(siteArticle) { + // to be overridden + } + + async import(siteArticles) { + logger.info('importing articles') + + await this.init() + await this.setCookies() + for (let i = 0; i < siteArticles.length; i++) { + const siteArticle = siteArticles[i] + if (siteArticle.exists && siteArticle.associated) continue + await this.importArticle(siteArticle) } - async import(siteArticles) { - logger.info('importing articles') + await this.browser.close() - await this.init() - await this.setCookies() - for (let i = 0; i < siteArticles.length; i++) { - const siteArticle = siteArticles[i] - if (siteArticle.exists && siteArticle.associated) continue - await this.importArticle(siteArticle) - } - - await this.browser.close() - - logger.info('imported articles') - } + logger.info('imported articles') + } } module.exports = BaseImportSpider diff --git a/spiders/index.js b/spiders/index.js index efe0241..686dfe5 100644 --- a/spiders/index.js +++ b/spiders/index.js @@ -1,7 +1,9 @@ module.exports = { - JuejinSpider: require('./juejin'), - SegmentfaultSpider: require('./segmentfault'), - JianshuSpider: require('./jianshu'), - CsdnSpider: require('./csdn'), - ZhihuSpider: require('./zhihu'), + JuejinSpider: require('./juejin'), + SegmentfaultSpider: require('./segmentfault'), + JianshuSpider: require('./jianshu'), + CsdnSpider: require('./csdn'), + ZhihuSpider: require('./zhihu'), + OschinaSpider: require('./oschina'), + ToutiaoSpider: require('./toutiao'), } diff --git a/spiders/oschina.js b/spiders/oschina.js new file mode 100644 index 0000000..df18144 --- /dev/null +++ b/spiders/oschina.js @@ -0,0 +1,130 @@ +const BaseSpider = require('./base') + +class OschinaSpider extends BaseSpider { + async goToEditor() { + // 导航至首页 + await this.page.goto('https://oschina.net') + await this.page.waitFor(5000) + + // 获取编辑器URL + const url = await this.page.evaluate(() => { + const aList = document.querySelectorAll('#userSidebar > a.item') + for (let i = 0; i < aList.length; i++) { + const a = aList[i] + if (a.innerHTML.match('写博客')) { + return a.getAttribute('href') + } + } + }) + + if (!url) throw new Error('editor url cannot be empty') + + await this.page.goto(url) + await this.page.waitFor(5000) + } + + async inputContent(article, editorSel) { + const footerContent = `
本篇文章由一文多发平台ArtiPub自动发布` + const content = article.contentHtml + footerContent; + const iframeWindow = document.querySelector('.cke_wysiwyg_frame').contentWindow + const el = iframeWindow.document.querySelector(editorSel.content) + el.focus() + iframeWindow.document.execCommand('delete', false) + iframeWindow.document.execCommand('insertHTML', false, content) + document.querySelector('textarea[name="body"]').value = content + } + + async inputFooter(article, editorSel) { + // do nothing + } + + async afterInputEditor() { + await this.page.click('.inline.fields > .field:nth-child(1) > .dropdown') + await this.page.waitFor(1000) + + await this.page.evaluate(task => { + const categories = [ + '移动开发', + '前端开发', + '人工智能', + '服务端开发/管理', + '游戏开发', + '编程语言', + '数据库', + '企业开发', + '图像/多媒体', + '系统运维', + '软件工程', + '大数据', + '云计算', + '开源硬件', + '区块链', + '其他类型', + '物联网', + ] + const index = categories.indexOf(task.category) + + console.log(index) + + const items = document.querySelectorAll('.inline.fields > .field:nth-child(1) > .dropdown .item') + for (let i = 0; i < items.length; i++) { + const item = items[i] + if (index === i) { + item.click() + return + } + } + }, this.task) + await this.page.waitFor(3000) + } + + async publish() { + // 发布文章 + await this.page.evaluate(editorSel => { + const el = document.querySelector(editorSel.publish) + el.click() + }, this.editorSel) + await this.page.waitFor(10000) + + // 后续处理 + await this.afterPublish() + } + + async afterPublish() { + const url = this.page.url() + if (!url.match(/\/blog\/\d+/)) { + return + } + this.task.url = url + this.task.updateTs = new Date() + await this.task.save() + } + + async fetchStats() { + if (!this.task.url) return + await this.page.goto(this.task.url, { timeout: 60000 }) + await this.page.waitFor(5000) + + const stats = await this.page.evaluate(() => { + const text = document.querySelector('body').innerText + const mRead = text.match(/阅读 (\d+)/) + const mLike = text.match(/点赞 (\d+)/) + const mComment = text.match(/评论 (\d+)/) + const readNum = mRead ? Number(mRead[1]) : 0 + const likeNum = mLike ? Number(mLike[1]) : 0 + const commentNum = mComment ? Number(mComment[1]) : 0 + return { + readNum, + likeNum, + commentNum + } + }) + this.task.readNum = stats.readNum + this.task.likeNum = stats.likeNum + this.task.commentNum = stats.commentNum + await this.task.save() + await this.page.waitFor(3000) + } +} + +module.exports = OschinaSpider diff --git a/spiders/toutiao.js b/spiders/toutiao.js new file mode 100644 index 0000000..6a19fe3 --- /dev/null +++ b/spiders/toutiao.js @@ -0,0 +1,44 @@ +const BaseSpider = require('./base') + +class ToutiaoSpider extends BaseSpider { + async inputContent(article, editorSel) { + const footerContent = `
本篇文章由一文多发平台ArtiPub自动发布. https://github.com/crawlab-team/artipub` + const content = article.contentHtml + footerContent + const el = document.querySelector(editorSel.content) + el.focus() + document.execCommand('insertHTML', false, content) + } + + async inputFooter(article, editorSel) { + // do nothing + } + + async publish() { + // 发布文章 + const elPub = await this.page.$(this.editorSel.publish) + await elPub.click() + await this.page.waitFor(10000) + + // 后续处理 + await this.afterPublish() + } + + async afterInputEditor() { + } + + async afterPublish() { + const id = await this.page.evaluate(() => { + const url = document.querySelector('.master-title > a').getAttribute('href') + return url.match(/pgc_id=(\d+)$/)[1] + }) + if (!id) return + this.task.url = `https://toutiao.com/i${id}` + this.task.updateTs = new Date() + await this.task.save() + } + + async fetchStats() { + } +} + +module.exports = ToutiaoSpider diff --git a/spiders/zhihu.js b/spiders/zhihu.js index 9b2dd20..db137ca 100644 --- a/spiders/zhihu.js +++ b/spiders/zhihu.js @@ -1,12 +1,42 @@ +const fs = require('fs') +const path = require('path') const BaseSpider = require('./base') const constants = require('../constants') class ZhihuSpider extends BaseSpider { + async afterGoToEditor() { + // 创建tmp临时文件夹 + const dirPath = path.resolve(path.join(__dirname, '..', 'tmp')) + if (!fs.existsSync(dirPath)) { + await fs.mkdirSync(dirPath) + } + + // 内容 + const content = this.article.content + `\n\n> 本篇文章由一文多发平台[ArtiPub](https://github.com/crawlab-team/artipub)自动发布` + + // 写入临时markdown文件 + const mdPath = path.join(dirPath, `${this.article._id.toString()}.md`) + await fs.writeFileSync(mdPath, content) + + // 点击更多 + await this.page.click('#Popover3-toggle') + await this.page.waitFor(1000) + + // 点击导入文档 + await this.page.click('.Editable-toolbarMenuItem:nth-child(1)') + await this.page.waitFor(1000) + + // 上传markdown文件 + const handle = await this.page.$('input[accept=".docx,.doc,.markdown,.mdown,.mkdn,.md"]') + await handle.uploadFile(mdPath) + await this.page.waitFor(1000) + + // 删除临时markdown文件 + await fs.unlinkSync(mdPath) + } + async inputContent(article, editorSel) { - const el = document.querySelector(editorSel.content) - el.focus() - document.execCommand('insertText', false, '') - document.execCommand('insertHTML', false, article.contentHtml) + // do nothing } async inputFooter(article, editorSel) { @@ -20,26 +50,41 @@ class ZhihuSpider extends BaseSpider { await this.page.waitFor(5000) // 选择标签 - const tags = this.task.tag.split(',') - for (const tag of tags) { - const elTagInput = await this.page.$('.PublishPanel-searchInput') - await elTagInput.type(tag) - await this.page.waitFor(5000) - await this.page.evaluate(() => { - document.querySelector('.PublishPanel-suggest > li:nth-child(1)').click() - }) - } + // const tags = this.task.tag.split(',') + // for (const tag of tags) { + // const elTagInput = await this.page.$('.PublishPanel-searchInput') + // await elTagInput.type(tag) + // await this.page.waitFor(5000) + // await this.page.evaluate(() => { + // document.querySelector('.PublishPanel-suggest > li:nth-child(1)').click() + // }) + // } + + // 点击下一步 + await this.page.evaluate(() => { + const el = document.querySelector('.PublishPanel-stepOneButton > button') + el.click() + }) + await this.page.waitFor(2000) + } + + async publish() { + // 发布文章 + await this.page.evaluate(() => { + const el = document.querySelector('.PublishPanel-stepTwoButton') + el.click() + }) await this.page.waitFor(5000) + + // 后续处理 + await this.afterPublish() } async afterPublish() { - // this.task.url = await this.page.evaluate(() => { - // const el = document.querySelector('a.title') - // return 'https://juejin.im' + el.getAttribute('href') - // }) - // this.task.updateTs = new Date() - // this.task.status = constants.status.FINISHED - // await this.task.save() + this.task.url = this.page.url() + this.task.updateTs = new Date() + this.task.status = constants.status.FINISHED + await this.task.save() } async fetchStats() { diff --git a/src/assets/img/oschina-logo.jpg b/src/assets/img/oschina-logo.jpg new file mode 100644 index 0000000..1efd103 Binary files /dev/null and b/src/assets/img/oschina-logo.jpg differ diff --git a/src/assets/img/toutiao-logo.png b/src/assets/img/toutiao-logo.png new file mode 100644 index 0000000..013f4a3 Binary files /dev/null and b/src/assets/img/toutiao-logo.png differ diff --git a/src/assets/img/zhihu-logo.jpg b/src/assets/img/zhihu-logo.jpg new file mode 100644 index 0000000..a5aeaff Binary files /dev/null and b/src/assets/img/zhihu-logo.jpg differ diff --git a/src/constants.ts b/src/constants.ts index 1591c39..d2be2c6 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -5,6 +5,8 @@ export default { JIANSHU: 'jianshu', CSDN: 'csdn', ZHIHU: 'zhihu', + OSCHINA: 'oschina', + TOUTIAO: 'toutiao', }, status: { NOT_STARTED: 'not-started', @@ -24,4 +26,13 @@ export default { MARKDOWN: 'markdown', RICH_TEXT: 'rich-text', }, + environment: { + UPDATE_STATS_CRON: 'update_stats_cron', + ENABLE_CHROME_DEBUG: 'enable_chrome_debug', + }, + cookieStatus: { + NO_COOKIE: 'no_cookie', + EXPIRED: 'expired', + EXISTS: 'exists', + } }; diff --git a/src/layouts/BasicLayout.scss b/src/layouts/BasicLayout.scss new file mode 100644 index 0000000..49927e3 --- /dev/null +++ b/src/layouts/BasicLayout.scss @@ -0,0 +1,22 @@ +.footer { + text-align: center; + margin: 20px 0; +} + +.info { + margin-bottom: 10px; +} + +.name { +} + +.copyright { +} + +.slogan { + margin-left: 10px; +} + +.github { + margin-left: 10px; +} diff --git a/src/layouts/BasicLayout.tsx b/src/layouts/BasicLayout.tsx index aa8f1b8..a9c6040 100644 --- a/src/layouts/BasicLayout.tsx +++ b/src/layouts/BasicLayout.tsx @@ -9,16 +9,18 @@ import ProLayout, { BasicLayoutProps as ProLayoutProps, Settings, } from '@ant-design/pro-layout'; -import React, { useEffect } from 'react'; +import React, {useEffect} from 'react'; import Link from 'umi/link'; -import { connect } from 'dva'; -import { setLocale, formatMessage } from 'umi-plugin-react/locale'; +import {connect} from 'dva'; +import {setLocale, formatMessage} from 'umi-plugin-react/locale'; import Authorized from '@/utils/Authorized'; import RightContent from '@/components/GlobalHeader/RightContent'; -import { ConnectState, Dispatch } from '@/models/connect'; -import { isAntDesignPro } from '@/utils/utils'; +import {ConnectState, Dispatch} from '@/models/connect'; +import {isAntDesignPro} from '@/utils/utils'; import logo from '../assets/logo.svg'; +import style from './BasicLayout.scss'; +import {Row} from "antd"; export interface BasicLayoutProps extends ProLayoutProps { breadcrumbNameMap: { @@ -27,6 +29,7 @@ export interface BasicLayoutProps extends ProLayoutProps { settings: Settings; dispatch: Dispatch; } + export type BasicLayoutContext = { [K in 'location']: BasicLayoutProps[K] } & { breadcrumbNameMap: { [path: string]: MenuDataItem; @@ -45,9 +48,24 @@ const menuDataRender = (menuList: MenuDataItem[]): MenuDataItem[] => return Authorized.check(item.authority, localItem, null) as MenuDataItem; }); +const footer = ( +
+ + ArtiPub + 让你的文章随处可阅 + + + + + + Copyright (c) 2019, Crawlab Team All rights reserved. + +
+); + const footerRender: BasicLayoutProps['footerRender'] = (_, defaultDom) => { if (!isAntDesignPro()) { - return defaultDom; + return footer; } return ( <> @@ -71,7 +89,7 @@ const footerRender: BasicLayoutProps['footerRender'] = (_, defaultDom) => { }; const BasicLayout: React.FC = props => { - const { dispatch, children, settings } = props; + const {dispatch, children, settings} = props; /** * constructor */ @@ -140,7 +158,7 @@ const BasicLayout: React.FC = props => { ); }; -export default connect(({ global, settings }: ConnectState) => ({ +export default connect(({global, settings}: ConnectState) => ({ collapsed: global.collapsed, settings, }))(BasicLayout); diff --git a/src/locales/zh-CN/menu.ts b/src/locales/zh-CN/menu.ts index ee5b102..71b701a 100644 --- a/src/locales/zh-CN/menu.ts +++ b/src/locales/zh-CN/menu.ts @@ -4,6 +4,8 @@ export default { 'menu.article-new': '写文章', 'menu.article-edit': '文章编辑', 'menu.platforms': '平台管理', + 'menu.environments': '系统设置', + 'menu.helper': '登陆助手', 'menu.more-blocks': '更多区块', 'menu.home': '首页', 'menu.login': '登录', diff --git a/src/manifest.json b/src/manifest.json index 839bc5b..0d3c4a8 100644 --- a/src/manifest.json +++ b/src/manifest.json @@ -1,6 +1,6 @@ { - "name": "Ant Design Pro", - "short_name": "Ant Design Pro", + "name": "ArtiPub", + "short_name": "ArtiPub", "display": "standalone", "start_url": "./?utm_source=homescreen", "theme_color": "#002140", diff --git a/src/models/connect.d.ts b/src/models/connect.d.ts index 695798a..bf287c1 100644 --- a/src/models/connect.d.ts +++ b/src/models/connect.d.ts @@ -8,6 +8,7 @@ import {UserModelState} from './user'; import {ArticleModelState} from "@/models/article"; import {PlatformModelState} from "@/models/platform"; import {TaskModelState} from "@/models/task"; +import {EnvironmentModelState} from "@/models/environment"; export {GlobalModelState, SettingModelState, UserModelState}; @@ -31,6 +32,7 @@ export interface ConnectState { article: ArticleModelState; platform: PlatformModelState; task: TaskModelState; + environment: EnvironmentModelState; } export type Effect = ( diff --git a/src/models/environment.ts b/src/models/environment.ts new file mode 100644 index 0000000..206107c --- /dev/null +++ b/src/models/environment.ts @@ -0,0 +1,63 @@ +import {Effect} from 'dva'; +import {Reducer} from 'redux'; +import {queryEnvironmentList, saveEnvironment} from "@/services/environment"; + +export interface Environment { + _id?: string; + value?: string; +} + +export interface EnvironmentModelState { + environments?: Environment[]; +} + +export interface EnvironmentModelType { + namespace: 'environment'; + state: EnvironmentModelState; + effects: { + fetchEnvironmentList: Effect; + saveEnvironment: Effect; + saveEnvironmentList: Effect; + }; + reducers: { + setEnvironmentList: Reducer; + }; +} + +const EnvironmentModel: EnvironmentModelType = { + namespace: 'environment', + + state: { + environments: [], + }, + + effects: { + * fetchEnvironmentList(_, {call, put}) { + const response = yield call(queryEnvironmentList); + yield put({ + type: 'setEnvironmentList', + payload: response.data, + }); + }, + * saveEnvironment(action, {call, put}) { + yield call(saveEnvironment, action.payload); + }, + * saveEnvironmentList(action, {put}) { + yield put({ + type: 'setEnvironmentList', + payload: action.payload, + }) + } + }, + + reducers: { + setEnvironmentList(state, action) { + return { + ...state, + environments: action.payload, + }; + }, + }, +}; + +export default EnvironmentModel; diff --git a/src/models/platform.ts b/src/models/platform.ts index 02b15c6..52e6663 100644 --- a/src/models/platform.ts +++ b/src/models/platform.ts @@ -17,6 +17,10 @@ export interface Platform { editorType: string; description: string; enableImport: boolean; + enableLogin: boolean; + username: string; + password: string; + cookieStatus: string; } export interface SiteArticle { @@ -36,6 +40,7 @@ export interface PlatformModelState { fetchModalVisible?: boolean; fetchLoading?: boolean; importLoading?: boolean; + accountModalVisible?: boolean; } export interface PlatformModelType { @@ -52,6 +57,7 @@ export interface PlatformModelType { fetchSiteArticles: Effect; saveSiteArticles: Effect; importArticles: Effect; + saveAccountModalVisible: Effect; }; reducers: { setPlatformList: Reducer; @@ -61,6 +67,7 @@ export interface PlatformModelType { setSiteArticles: Reducer; setFetchLoading: Reducer; setImportLoading: Reducer; + setAccountModalVisible: Reducer; }; } @@ -73,6 +80,7 @@ const PlatformModel: PlatformModelType = { fetchModalVisible: false, fetchLoading: false, importLoading: false, + accountModalVisible: false, }, effects: { @@ -110,6 +118,12 @@ const PlatformModel: PlatformModelType = { payload: action.payload, }); }, + *saveAccountModalVisible(action, { put }) { + yield put({ + type: 'setAccountModalVisible', + payload: action.payload, + }); + }, *fetchSiteArticles(action, { call, put }) { yield put({ type: 'setFetchLoading', @@ -179,6 +193,12 @@ const PlatformModel: PlatformModelType = { fetchModalVisible: action.payload, }; }, + setAccountModalVisible(state, action) { + return { + ...state, + accountModalVisible: action.payload, + }; + }, setCurrentPlatform(state, action) { return { ...state, diff --git a/src/models/task.ts b/src/models/task.ts index e63e0f8..aca6b87 100644 --- a/src/models/task.ts +++ b/src/models/task.ts @@ -1,6 +1,8 @@ import { Effect } from 'dva'; import { addTask, addTasks, queryTaskList, saveTask } from '@/services/task'; import { Reducer } from 'redux'; +import {Platform} from "@/models/platform"; +import {ConnectState} from "@/models/connect"; export interface Task { _id?: string; @@ -18,6 +20,8 @@ export interface Task { readNum?: number; likeNum?: number; commentNum?: number; + platform?: Platform; + platformName?: string; } export interface TaskModelState { @@ -50,13 +54,31 @@ const TaskModel: TaskModelType = { }, effects: { - *fetchTaskList(action, { call, put }) { - console.log('fetchTaskList'); - const response = yield call(queryTaskList, action.payload); - yield put({ - type: 'setTaskList', - payload: response.data, - }); + *fetchTaskList(action, { select, call, put }) { + const response = yield call(queryTaskList, { id: action.payload.id }); + const newTasks: Task[] = response.data; + const tasks: Task[] = yield select((state: ConnectState) => state.task.tasks); + if (action.payload.updateStatus) { + // 只更新状态 + for (let i = 0; i < tasks.length; i++) { + const task = tasks[i]; + const newTask = newTasks.filter((t: Task) => t._id === task._id)[0]; + if (!newTask) continue; + tasks[i].status = newTask.status; + tasks[i].error = newTask.error; + tasks[i].url = newTask.url; + } + yield put({ + type: 'setTaskList', + payload: tasks, + }); + } else { + // 更新全部 + yield put({ + type: 'setTaskList', + payload: response.data, + }); + } }, *addTasks(action, { call }) { yield call(addTasks, action.payload); @@ -93,7 +115,6 @@ const TaskModel: TaskModelType = { }; }, setTaskList(state, action) { - console.log('setTaskList'); return { ...state, tasks: action.payload, diff --git a/src/pages/.umi/LocaleWrapper.jsx b/src/pages/.umi/LocaleWrapper.jsx deleted file mode 100644 index e3bc378..0000000 --- a/src/pages/.umi/LocaleWrapper.jsx +++ /dev/null @@ -1,155 +0,0 @@ -import React from 'react'; -import { - _setIntlObject, - addLocaleData, - IntlProvider, - intlShape, - LangContext, - _setLocaleContext -} from 'umi-plugin-locale'; - -const InjectedWrapper = (() => { - let sfc = (props, context) => { - _setIntlObject(context.intl); - return props.children; - }; - sfc.contextTypes = { - intl: intlShape, - }; - return sfc; -})(); - -import 'moment/locale/pt-br'; -import 'moment/locale/zh-cn'; -import 'moment/locale/zh-tw'; - -const baseNavigator = true; -const baseSeparator = '-'; -const useLocalStorage = true; - -import { LocaleProvider, version } from 'antd'; -import moment from 'moment'; -import 'moment/locale/zh-cn'; -let defaultAntd = require('antd/lib/locale-provider/zh_CN'); -defaultAntd = defaultAntd.default || defaultAntd; - -const localeInfo = { - 'en-US': { - messages: { - ...((locale) => locale.__esModule ? locale.default : locale)(require('/Users/yeqing/projects/artipub/src/locales/en-US.ts')), - }, - locale: 'en-US', - antd: require('antd/lib/locale-provider/en_US'), - data: require('react-intl/locale-data/en'), - momentLocale: '', - }, - 'pt-BR': { - messages: { - ...((locale) => locale.__esModule ? locale.default : locale)(require('/Users/yeqing/projects/artipub/src/locales/pt-BR.ts')), - }, - locale: 'pt-BR', - antd: require('antd/lib/locale-provider/pt_BR'), - data: require('react-intl/locale-data/pt'), - momentLocale: 'pt-br', - }, - 'zh-CN': { - messages: { - ...((locale) => locale.__esModule ? locale.default : locale)(require('/Users/yeqing/projects/artipub/src/locales/zh-CN.ts')), - }, - locale: 'zh-CN', - antd: require('antd/lib/locale-provider/zh_CN'), - data: require('react-intl/locale-data/zh'), - momentLocale: 'zh-cn', - }, - 'zh-TW': { - messages: { - ...((locale) => locale.__esModule ? locale.default : locale)(require('/Users/yeqing/projects/artipub/src/locales/zh-TW.ts')), - }, - locale: 'zh-TW', - antd: require('antd/lib/locale-provider/zh_TW'), - data: require('react-intl/locale-data/zh'), - momentLocale: 'zh-tw', - }, -}; - -class LocaleWrapper extends React.Component{ - state = { - locale: 'zh-CN', - }; - getAppLocale(){ - let appLocale = { - locale: 'zh-CN', - messages: {}, - data: require('react-intl/locale-data/zh'), - momentLocale: 'zh-cn', - }; - - const runtimeLocale = require('umi/_runtimePlugin').mergeConfig('locale') || {}; - const runtimeLocaleDefault = typeof runtimeLocale.default === 'function' ? runtimeLocale.default() : runtimeLocale.default; - if ( - useLocalStorage - && typeof localStorage !== 'undefined' - && localStorage.getItem('umi_locale') - && localeInfo[localStorage.getItem('umi_locale')] - ) { - appLocale = localeInfo[localStorage.getItem('umi_locale')]; - } else if ( - typeof navigator !== 'undefined' - && localeInfo[navigator.language] - && baseNavigator - ) { - appLocale = localeInfo[navigator.language]; - } else if(localeInfo[runtimeLocaleDefault]){ - appLocale = localeInfo[runtimeLocaleDefault]; - } else { - appLocale = localeInfo['zh-CN'] || appLocale; - } - window.g_lang = appLocale.locale; - window.g_langSeparator = baseSeparator || '-'; - appLocale.data && addLocaleData(appLocale.data); - return appLocale; - } - reloadAppLocale = () => { - const appLocale = this.getAppLocale(); - this.setState({ - locale: appLocale.locale, - }); - }; - - render(){ - const appLocale = this.getAppLocale(); - // react-intl must use `-` separator - const reactIntlLocale = appLocale.locale.split(baseSeparator).join('-'); - const LangContextValue = { - locale: reactIntlLocale, - reloadAppLocale: this.reloadAppLocale, - }; - let ret = this.props.children; - ret = ( - - - {(value) => { - _setLocaleContext(value); - return this.props.children - }} - - - ) - // avoid antd ConfigProvider not found - let AntdProvider = LocaleProvider; - const [major, minor] = `${version || ''}`.split('.'); - // antd 3.21.0 use ConfigProvider not LocaleProvider - const isConfigProvider = Number(major) > 3 || (Number(major) >= 3 && Number(minor) >= 21); - if (isConfigProvider) { - try { - AntdProvider = require('antd/lib/config-provider').default; - } catch (e) {} - } - - return ( - {ret} - ); - return ret; - } -} -export default LocaleWrapper; diff --git a/src/pages/.umi/dva.js b/src/pages/.umi/dva.js deleted file mode 100644 index 3e65c1e..0000000 --- a/src/pages/.umi/dva.js +++ /dev/null @@ -1,43 +0,0 @@ -import dva from 'dva' -import { Component } from 'react' -import createLoading from 'dva-loading' -import history from '@tmp/history' - -let app = null - -export function _onCreate() { - const plugins = require('umi/_runtimePlugin') - const runtimeDva = plugins.mergeConfig('dva') - app = dva({ - history, - - ...(runtimeDva.config || {}), - ...(window.g_useSSR ? { initialState: window.g_initialData } : {}), - }) - - app.use(createLoading()); - (runtimeDva.plugins || []).forEach(plugin => { - app.use(plugin) - }) - - app.model({ namespace: 'article', ...(require('/Users/yeqing/projects/artipub/src/models/article.ts').default) }) - app.model({ namespace: 'global', ...(require('/Users/yeqing/projects/artipub/src/models/global.ts').default) }) - app.model({ namespace: 'login', ...(require('/Users/yeqing/projects/artipub/src/models/login.ts').default) }) - app.model({ namespace: 'platform', ...(require('/Users/yeqing/projects/artipub/src/models/platform.ts').default) }) - app.model({ namespace: 'setting', ...(require('/Users/yeqing/projects/artipub/src/models/setting.ts').default) }) - app.model({ namespace: 'task', ...(require('/Users/yeqing/projects/artipub/src/models/task.ts').default) }) - app.model({ namespace: 'user', ...(require('/Users/yeqing/projects/artipub/src/models/user.ts').default) }) - return app -} - -export function getApp() { - return app -} - -export class _DvaContainer extends Component { - render() { - const app = getApp() - app.router(() => this.props.children) - return app.start()() - } -} diff --git a/src/pages/.umi/history.js b/src/pages/.umi/history.js deleted file mode 100644 index 26941fa..0000000 --- a/src/pages/.umi/history.js +++ /dev/null @@ -1,6 +0,0 @@ -// create history -const history = require('umi/lib/createHistory').default({ - basename: window.routerBase, -}); -window.g_history = history; -export default history; diff --git a/src/pages/.umi/polyfills.js b/src/pages/.umi/polyfills.js deleted file mode 100644 index c3942c8..0000000 --- a/src/pages/.umi/polyfills.js +++ /dev/null @@ -1,6 +0,0 @@ -import 'core-js'; -import 'regenerator-runtime/runtime'; - -// Include this seperatly since it's not included in core-js -// ref: https://github.com/zloirock/core-js/issues/117 -import '../../../node_modules/url-polyfill/url-polyfill.js'; diff --git a/src/pages/.umi/router.js b/src/pages/.umi/router.js deleted file mode 100644 index 3b90aa3..0000000 --- a/src/pages/.umi/router.js +++ /dev/null @@ -1,200 +0,0 @@ -import React from 'react'; -import { Router as DefaultRouter, Route, Switch } from 'react-router-dom'; -import dynamic from 'umi/dynamic'; -import renderRoutes from 'umi/lib/renderRoutes'; -import history from '@tmp/history'; -import RendererWrapper0 from '/Users/yeqing/projects/artipub/src/pages/.umi/LocaleWrapper.jsx'; -import _dvaDynamic from 'dva/dynamic'; - -const Router = require('dva/router').routerRedux.ConnectedRouter; - -const routes = [ - { - path: '/articles/edit/:id', - name: 'article-edit', - authority: ['admin', 'user'], - icon: 'read', - hideInMenu: true, - component: __IS_BROWSER - ? _dvaDynamic({ - component: () => - import(/* webpackChunkName: "p__ArticleEdit__ArticleEdit" */ '../ArticleEdit/ArticleEdit'), - LoadingComponent: require('/Users/yeqing/projects/artipub/src/components/PageLoading/index') - .default, - }) - : require('../ArticleEdit/ArticleEdit').default, - exact: true, - }, - { - path: '/articles/new', - name: 'article-new', - authority: ['admin', 'user'], - icon: 'read', - hideInMenu: true, - component: __IS_BROWSER - ? _dvaDynamic({ - component: () => - import(/* webpackChunkName: "p__ArticleEdit__ArticleEdit" */ '../ArticleEdit/ArticleEdit'), - LoadingComponent: require('/Users/yeqing/projects/artipub/src/components/PageLoading/index') - .default, - }) - : require('../ArticleEdit/ArticleEdit').default, - exact: true, - }, - { - path: '/paste', - name: 'paste', - authority: ['admin', 'user'], - icon: 'read', - hideInMenu: true, - component: __IS_BROWSER - ? _dvaDynamic({ - component: () => - import(/* webpackChunkName: "p__Paste__Paste" */ '../Paste/Paste'), - LoadingComponent: require('/Users/yeqing/projects/artipub/src/components/PageLoading/index') - .default, - }) - : require('../Paste/Paste').default, - exact: true, - }, - { - path: '/demo', - name: 'demo', - authority: ['admin', 'user'], - icon: 'read', - hideInMenu: true, - component: __IS_BROWSER - ? _dvaDynamic({ - component: () => - import(/* webpackChunkName: "p__Demo__Demo" */ '../Demo/Demo'), - LoadingComponent: require('/Users/yeqing/projects/artipub/src/components/PageLoading/index') - .default, - }) - : require('../Demo/Demo').default, - exact: true, - }, - { - path: '/', - component: __IS_BROWSER - ? _dvaDynamic({ - component: () => - import(/* webpackChunkName: "layouts__BasicLayout" */ '../../layouts/BasicLayout'), - LoadingComponent: require('/Users/yeqing/projects/artipub/src/components/PageLoading/index') - .default, - }) - : require('../../layouts/BasicLayout').default, - Routes: [require('../Authorized').default], - authority: ['admin', 'user'], - routes: [ - { - path: '/', - redirect: '/platforms', - exact: true, - }, - { - path: '/platforms', - name: 'platforms', - icon: 'cloud', - component: __IS_BROWSER - ? _dvaDynamic({ - component: () => - import(/* webpackChunkName: "p__PlatformList__PlatformList" */ '../PlatformList/PlatformList'), - LoadingComponent: require('/Users/yeqing/projects/artipub/src/components/PageLoading/index') - .default, - }) - : require('../PlatformList/PlatformList').default, - exact: true, - }, - { - path: '/articles', - name: 'articles', - icon: 'read', - component: __IS_BROWSER - ? _dvaDynamic({ - component: () => - import(/* webpackChunkName: "p__ArticleList__ArticleList" */ '../ArticleList/ArticleList'), - LoadingComponent: require('/Users/yeqing/projects/artipub/src/components/PageLoading/index') - .default, - }) - : require('../ArticleList/ArticleList').default, - exact: true, - }, - { - component: __IS_BROWSER - ? _dvaDynamic({ - component: () => - import(/* webpackChunkName: "p__404" */ '../404'), - LoadingComponent: require('/Users/yeqing/projects/artipub/src/components/PageLoading/index') - .default, - }) - : require('../404').default, - exact: true, - }, - { - component: () => - React.createElement( - require('/Users/yeqing/projects/artipub/node_modules/umi-build-dev/lib/plugins/404/NotFound.js') - .default, - { pagesPath: 'src/pages', hasRoutesInConfig: true }, - ), - }, - ], - }, - { - component: __IS_BROWSER - ? _dvaDynamic({ - component: () => import(/* webpackChunkName: "p__404" */ '../404'), - LoadingComponent: require('/Users/yeqing/projects/artipub/src/components/PageLoading/index') - .default, - }) - : require('../404').default, - exact: true, - }, - { - component: () => - React.createElement( - require('/Users/yeqing/projects/artipub/node_modules/umi-build-dev/lib/plugins/404/NotFound.js') - .default, - { pagesPath: 'src/pages', hasRoutesInConfig: true }, - ), - }, -]; -window.g_routes = routes; -const plugins = require('umi/_runtimePlugin'); -plugins.applyForEach('patchRoutes', { initialValue: routes }); - -export { routes }; - -export default class RouterWrapper extends React.Component { - unListen = () => {}; - - constructor(props) { - super(props); - - // route change handler - function routeChangeHandler(location, action) { - plugins.applyForEach('onRouteChange', { - initialValue: { - routes, - location, - action, - }, - }); - } - this.unListen = history.listen(routeChangeHandler); - routeChangeHandler(history.location); - } - - componentWillUnmount() { - this.unListen(); - } - - render() { - const props = this.props || {}; - return ( - - {renderRoutes(routes, props)} - - ); - } -} diff --git a/src/pages/.umi/umi.js b/src/pages/.umi/umi.js deleted file mode 100644 index b5aa855..0000000 --- a/src/pages/.umi/umi.js +++ /dev/null @@ -1,159 +0,0 @@ -import './polyfills'; -import history from './history'; -import '../../global.tsx'; -import React from 'react'; -import ReactDOM from 'react-dom'; -import findRoute, { - getUrlQuery, -} from '/Users/yeqing/projects/artipub/node_modules/umi-build-dev/lib/findRoute.js'; - -// runtime plugins -const plugins = require('umi/_runtimePlugin'); -window.g_plugins = plugins; -plugins.init({ - validKeys: [ - 'patchRoutes', - 'render', - 'rootContainer', - 'modifyRouteProps', - 'onRouteChange', - 'modifyInitialProps', - 'initialProps', - 'dva', - 'locale', - ], -}); -plugins.use(require('../../../node_modules/umi-plugin-dva/lib/runtime')); - -const app = require('@tmp/dva')._onCreate(); -window.g_app = app; - -// render -let clientRender = async () => { - window.g_isBrowser = true; - let props = {}; - // Both support SSR and CSR - if (window.g_useSSR) { - // 如果开启服务端渲染则客户端组件初始化 props 使用服务端注入的数据 - props = window.g_initialData; - } else { - const pathname = location.pathname; - const activeRoute = findRoute(require('@tmp/router').routes, pathname); - // 在客户端渲染前，执行 getInitialProps 方法 - // 拿到初始数据 - if ( - activeRoute && - activeRoute.component && - activeRoute.component.getInitialProps - ) { - const initialProps = plugins.apply('modifyInitialProps', { - initialValue: {}, - }); - props = activeRoute.component.getInitialProps - ? await activeRoute.component.getInitialProps({ - route: activeRoute, - isServer: false, - location, - ...initialProps, - }) - : {}; - } - } - const rootContainer = plugins.apply('rootContainer', { - initialValue: React.createElement(require('./router').default, props), - }); - ReactDOM[window.g_useSSR ? 'hydrate' : 'render']( - rootContainer, - document.getElementById('root'), - ); -}; -const render = plugins.compose( - 'render', - { initialValue: clientRender }, -); - -const moduleBeforeRendererPromises = []; -// client render -if (__IS_BROWSER) { - Promise.all(moduleBeforeRendererPromises) - .then(() => { - render(); - }) - .catch(err => { - window.console && window.console.error(err); - }); -} - -// export server render -let serverRender, ReactDOMServer; -if (!__IS_BROWSER) { - serverRender = async (ctx = {}) => { - // ctx.req.url may be `/bar?locale=en-US` - const [pathname] = (ctx.req.url || '').split('?'); - const history = require('@tmp/history').default; - history.push(ctx.req.url); - let props = {}; - const activeRoute = - findRoute(require('./router').routes, pathname) || false; - if ( - activeRoute && - activeRoute.component && - activeRoute.component.getInitialProps - ) { - const initialProps = plugins.apply('modifyInitialProps', { - initialValue: {}, - }); - // patch query object - const location = history.location - ? { ...history.location, query: getUrlQuery(history.location.search) } - : {}; - props = await activeRoute.component.getInitialProps({ - route: activeRoute, - isServer: true, - location, - // only exist in server - req: ctx.req || {}, - res: ctx.res || {}, - ...initialProps, - }); - props = plugins.apply('initialProps', { - initialValue: props, - }); - } else { - // message activeRoute or getInitialProps not found - console.log( - !activeRoute - ? `${pathname} activeRoute not found` - : `${pathname} activeRoute's getInitialProps function not found`, - ); - } - const rootContainer = plugins.apply('rootContainer', { - initialValue: React.createElement(require('./router').default, props), - }); - const htmlTemplateMap = {}; - return { - htmlElement: - activeRoute && activeRoute.path - ? htmlTemplateMap[activeRoute.path] - : '', - rootContainer, - matchPath: activeRoute && activeRoute.path, - g_initialData: props, - }; - }; - // using project react-dom version - // https://github.com/facebook/react/issues/13991 - ReactDOMServer = require('react-dom/server'); -} - -export { ReactDOMServer }; -export default (__IS_BROWSER ? null : serverRender); - -require('../../global.less'); - -// hot module replacement -if (__IS_BROWSER && module.hot) { - module.hot.accept('./router', () => { - clientRender(); - }); -} diff --git a/src/pages/.umi/umiExports.js b/src/pages/.umi/umiExports.js deleted file mode 100644 index e69de29..0000000 diff --git a/src/pages/ArticleEdit/ArticleEdit.tsx b/src/pages/ArticleEdit/ArticleEdit.tsx index 3dcb233..0371da2 100644 --- a/src/pages/ArticleEdit/ArticleEdit.tsx +++ b/src/pages/ArticleEdit/ArticleEdit.tsx @@ -126,6 +126,20 @@ const ArticleEdit: React.FC = props => { // 点击保存 const onSave = async () => { + if (article.currentArticle) { + // 文章标题校验 + if (article.currentArticle.title.length < 5) { + message.error('标题字数不得小于5'); + return; + } + + // 文章内容校验 + if (article.currentArticle.content.length < 10) { + message.error('内容字数不得小于9'); + return; + } + } + if (isEdit()) { // 如果为编辑文章 await dispatch({ @@ -141,13 +155,19 @@ const ArticleEdit: React.FC = props => { }); message.success('文章保存成功'); } + + TDAPP.onEvent('文章编辑-保存文章'); }; // 点击返回 const onBack = () => { router.push('/articles'); + + TDAPP.onEvent('文章编辑-返回'); }; + TDAPP.onEvent('文章编辑-访问页面'); + return (
diff --git a/src/pages/ArticleList/ArticleList.tsx b/src/pages/ArticleList/ArticleList.tsx index 70d5d9a..77f4efe 100644 --- a/src/pages/ArticleList/ArticleList.tsx +++ b/src/pages/ArticleList/ArticleList.tsx @@ -1,6 +1,6 @@ import React, {useEffect} from 'react'; import {PageHeaderWrapper} from '@ant-design/pro-layout'; -import {Button, Form, Input, message, Modal, Popconfirm, Select, Table, Tag, Tooltip} from 'antd'; +import {Badge, Button, Card, Form, Input, message, Modal, Popconfirm, Select, Table, Tag, Tooltip} from 'antd'; import {Article, ArticleModelState} from '@/models/article'; import {ConnectProps, ConnectState, Dispatch} from '@/models/connect'; import {connect} from 'dva'; @@ -17,6 +17,9 @@ import imgJuejin from '@/assets/img/juejin-logo.svg'; import imgSegmentfault from '@/assets/img/segmentfault-logo.jpg'; import imgJianshu from '@/assets/img/jianshu-logo.png'; import imgCsdn from '@/assets/img/csdn-logo.jpg'; +import imgZhihu from '@/assets/img/zhihu-logo.jpg'; +import imgOschina from '@/assets/img/oschina-logo.jpg'; +import imgToutiao from '@/assets/img/toutiao-logo.png'; export interface ArticleListProps extends ConnectProps { task: TaskModelState; @@ -31,21 +34,21 @@ const ArticleList: React.FC = props => { const onArticleEdit: Function = (d: Article) => { return () => { router.push(`/articles/edit/${d._id}`); + + TDAPP.onEvent('文章管理-点击编辑'); }; }; const onArticleDelete: Function = (d: Article) => { - return () => { - if (dispatch) { - dispatch({ - type: 'article/deleteArticle', - payload: d, - }).then(() => { - dispatch({ - type: 'article/fetchArticleList', - }); - }); - } + return async () => { + await dispatch({ + type: 'article/deleteArticle', + payload: d, + }); + await dispatch({ + type: 'article/fetchArticleList', + }); + TDAPP.onEvent('文章管理-确认删除'); }; }; @@ -56,6 +59,8 @@ const ArticleList: React.FC = props => { }); } router.push('/articles/new'); + + TDAPP.onEvent('文章管理-创建文章'); }; const onArticleTasksModalOpen: Function = (a: Article) => { @@ -78,11 +83,12 @@ const ArticleList: React.FC = props => { }); // 持续请求更新状态 - const fetchHandle = setInterval(() => { + const fetchHandle = await setInterval(() => { dispatch({ type: 'task/fetchTaskList', payload: { - id: article.currentArticle ? article.currentArticle._id : '', + id: a._id, + updateStatus: true, }, }); }, 5000); @@ -90,6 +96,8 @@ const ArticleList: React.FC = props => { type: 'article/setFetchHandle', payload: fetchHandle, }); + + TDAPP.onEvent('文章管理-打开发布'); }; }; @@ -105,6 +113,8 @@ const ArticleList: React.FC = props => { // 取消更新任务状态handle await clearInterval(article.fetchHandle); + + TDAPP.onEvent('文章管理-关闭发布'); }; const onArticleTasksPublish: Function = () => { @@ -118,12 +128,16 @@ const ArticleList: React.FC = props => { }); message.success('已开始发布'); } + + TDAPP.onEvent('文章管理-确认发布'); }; }; const onTaskViewArticle: Function = (t: Task) => { return () => { window.open(t.url); + + TDAPP.onEvent('文章管理-查看文章原文'); }; }; @@ -139,6 +153,8 @@ const ArticleList: React.FC = props => { type: 'article/setPlatformModalVisible', payload: true, }); + + TDAPP.onEvent('文章管理-打开平台设置'); } }; }; @@ -148,6 +164,11 @@ const ArticleList: React.FC = props => { type: 'article/setPlatformModalVisible', payload: false, }); + dispatch({ + type: 'task/saveCurrentTask', + payload: undefined, + }); + TDAPP.onEvent('文章管理-关闭平台设置'); }; const onTaskModalConfirm = () => { @@ -159,6 +180,7 @@ const ArticleList: React.FC = props => { type: 'article/setPlatformModalVisible', payload: false, }); + TDAPP.onEvent('文章管理-确认平台设置'); }; const getDefaultCategory = (p: Platform) => { @@ -214,12 +236,16 @@ const ArticleList: React.FC = props => { ) => { if (article.currentArticle) { await saveTasks(selectedPlatforms, article.currentArticle); + + TDAPP.onEvent('文章管理-勾选平台'); } }; const onTaskSelectAll = async (selected: boolean, selectedPlatforms: Object[]) => { if (article.currentArticle) { await saveTasks(selectedPlatforms, article.currentArticle); + + TDAPP.onEvent('文章管理-勾选平台-全选'); } }; @@ -241,6 +267,19 @@ const ArticleList: React.FC = props => { }; }; + const getBadgeCount = (p: Platform) => { + const t = task.tasks.filter((d: Task) => d.platformId === p._id)[0]; + if (!t || !t.checked) return 0; + if (p.name === constants.platform.JUEJIN) { + return t.tag === "" ? 1 : 0; + } else if (p.name === constants.platform.SEGMENTFAULT) { + return t.tag === "" ? 1 : 0; + } else if (p.name === constants.platform.OSCHINA) { + return t.category === "" ? 1 : 0; + } + return 0 + }; + /** * 选择验证方式 * 假设已经获取task.tasks @@ -263,6 +302,8 @@ const ArticleList: React.FC = props => { type: 'task/addTasks', payload: tasks, }); + + TDAPP.onEvent('文章管理-选择登陆类型'); }; }; @@ -373,6 +414,12 @@ const ArticleList: React.FC = props => { return {d.label}; } else if (d.name === constants.platform.CSDN) { return {d.label}; + } else if (d.name === constants.platform.ZHIHU) { + return {d.label}; + } else if (d.name === constants.platform.OSCHINA) { + return {d.label}; + } else if (d.name === constants.platform.TOUTIAO) { + return {d.label}; } else { return
; } @@ -424,6 +471,7 @@ const ArticleList: React.FC = props => { type={t.authType === constants.authType.LOGIN ? 'primary' : 'default'} size="small" onClick={onSelectAuthType(t, constants.authType.LOGIN)} + disabled={t.platform ? !t.platform.enableLogin : false} > 登陆 @@ -470,14 +518,16 @@ const ArticleList: React.FC = props => { /> -
); @@ -516,7 +566,7 @@ const ArticleList: React.FC = props => { ]; platformContent = (
- + - + = props => { } else if (currentPlatform && currentPlatform.name === constants.platform.SEGMENTFAULT) { platformContent = ( - + = props => { ]; platformContent = ( - + - + + {categories.map(category => ( + {category} + ))} + + + + ); } + TDAPP.onEvent('文章管理-访问页面'); + return ( = props => { 创建文章
- -