Merge pull request #20 from tikazyq/develop

Develop

crawlab/db/manager.py

@@ -147,14 +147,24 @@ def get_latest_version(self, spider_id, node_id):
 
     def get_last_deploy(self, spider_id):
         """
-        @deprecated
+        Get latest deploy for a given spider_id
         """
         col = self.db['deploys']
         for item in col.find({'spider_id': ObjectId(spider_id)}) \
                 .sort('finish_ts', DESCENDING):
             return item
         return None
 
+    def get_last_task(self, spider_id):
+        """
+        Get latest deploy for a given spider_id
+        """
+        col = self.db['tasks']
+        for item in col.find({'spider_id': ObjectId(spider_id)}) \
+                .sort('create_ts', DESCENDING):
+            return item
+        return None
+
     def aggregate(self, col_name: str, pipelines, **kwargs):
         """
         Perform MongoDB col.aggregate action to aggregate stats given collection name and pipelines.

crawlab/routes/base.py

@@ -38,8 +38,8 @@ def get(self, id: str = None, action: str = None) -> (dict, tuple):
         :param action:
         :return:
         """
-        import pdb
-        pdb.set_trace()
+        # import pdb
+        # pdb.set_trace()
         args = self.parser.parse_args()
 
         # action by id
@@ -85,13 +85,13 @@ def get(self, id: str = None, action: str = None) -> (dict, tuple):
 
             # TODO: getting status for node
 
-            return jsonify({
+            return {
                 'status': 'ok',
                 'total_count': total_count,
                 'page_num': page,
                 'page_size': page_size,
-                'items': items
-            })
+                'items': jsonify(items)
+            }
 
         # get item by id
         else:
@@ -108,6 +108,9 @@ def put(self) -> (dict, tuple):
             if k not in DEFAULT_ARGS:
                 item[k] = args.get(k)
         item = db_manager.save(col_name=self.col_name, item=item)
+
+        self.after_update(item._id)
+
         return item
 
     def update(self, id: str = None) -> (dict, tuple):

crawlab/routes/schedules.py

@@ -13,6 +13,8 @@ class ScheduleApi(BaseApi):
     col_name = 'schedules'
 
     arguments = (
+        ('name', str),
+        ('description', str),
         ('cron', str),
         ('spider_id', str)
     )

crawlab/routes/spiders.py

@@ -81,7 +81,14 @@ def get(self, id=None, action=None):
 
         # get one node
         elif id is not None:
-            return jsonify(db_manager.get('spiders', id=id))
+            spider = db_manager.get('spiders', id=id)
+
+            # get deploy
+            last_deploy = db_manager.get_last_deploy(spider_id=spider['_id'])
+            if last_deploy is not None:
+                spider['deploy_ts'] = last_deploy['finish_ts']
+
+            return jsonify(spider)
 
         # get a list of items
         else:
@@ -108,8 +115,23 @@ def get(self, id=None, action=None):
 
                 # existing spider
                 else:
+                    # get last deploy
+                    last_deploy = db_manager.get_last_deploy(spider_id=spider['_id'])
+                    if last_deploy is not None:
+                        spider['deploy_ts'] = last_deploy['finish_ts']
+
+                    # get last task
+                    last_task = db_manager.get_last_task(spider_id=spider['_id'])
+                    if last_task is not None:
+                        spider['task_ts'] = last_task['create_ts']
+
+                    # file stats
                     stats = get_file_suffix_stats(dir_path)
+
+                    # language
                     lang = get_lang_by_stats(stats)
+
+                    # update spider data
                     db_manager.update_one('spiders', id=str(spider['_id']), values={
                         'lang': lang,
                         'suffix_stats': stats,

crawlab/routes/tasks.py

@@ -1,8 +1,9 @@
 import json
+from datetime import datetime
 
 import requests
 from bson import ObjectId
-from celery.worker.control import revoke
+from tasks.celery import celery_app
 
 from constants.task import TaskStatus
 from db.manager import db_manager
@@ -42,6 +43,8 @@ def get(self, id: str = None, action: str = None):
             task = db_manager.get(col_name=self.col_name, id=id)
             spider = db_manager.get(col_name='spiders', id=str(task['spider_id']))
             task['spider_name'] = spider['name']
+            if task.get('finish_ts') is not None:
+                task['duration'] = (task['finish_ts'] - task['create_ts']).total_seconds()
             try:
                 with open(task['log_file_path']) as f:
                     task['log'] = f.read()
@@ -61,7 +64,8 @@ def get(self, id: str = None, action: str = None):
             _spider = db_manager.get(col_name='spiders', id=str(task['spider_id']))
             if task.get('status') is None:
                 task['status'] = TaskStatus.UNAVAILABLE
-            task['spider_name'] = _spider['name']
+            if _spider:
+                task['spider_name'] = _spider['name']
             items.append(task)
         return {
             'status': 'ok',
@@ -146,11 +150,13 @@ def get_results(self, id: str) -> (dict, tuple):
     def stop(self, id):
         """
         Stop the task in progress.
-        TODO: work in progress
         :param id:
         :return:
         """
-        revoke(id, terminate=True)
+        celery_app.control.revoke(id, terminate=True)
+        db_manager.update_one('tasks', id=id, values={
+            'status': TaskStatus.REVOKED
+        })
         return {
             'id': id,
             'status': 'ok',

crawlab/tasks/scheduler.py

@@ -2,6 +2,7 @@
 from apscheduler.schedulers.background import BackgroundScheduler
 from apscheduler.jobstores.mongodb import MongoDBJobStore
 from pymongo import MongoClient
+from flask import current_app
 
 from config import MONGO_DB, MONGO_HOST, MONGO_PORT, FLASK_HOST, FLASK_PORT
 from constants.spider import CronEnabled
@@ -29,8 +30,11 @@ def execute_spider(self, id: str):
     def restart(self):
         self.scheduler.shutdown()
         self.scheduler.start()
+        current_app.logger.info('restarted')
 
     def update(self):
+        current_app.logger.info('updating...')
+
         # remove all existing periodic jobs
         self.scheduler.remove_all_jobs()
 
@@ -50,6 +54,8 @@ def update(self):
                                    day_of_week=day_of_week, month=month, day=day, hour=hour, minute=minute,
                                    second=second)
 
+        current_app.logger.info('updated')
+
     def run(self):
         self.update()
         self.scheduler.start()

crawlab/utils/node.py

@@ -33,9 +33,10 @@ def update_nodes_status(refresh=False):
 
         # new node
         if node is None:
-            node = {'_id': node_name, 'name': node_name, 'status': node_status}
+            node = {'_id': node_name, 'name': node_name, 'status': node_status, 'ip': 'localhost', 'port': '8000'}
             db_manager.save('nodes', node)
 
+        # existing node
         else:
             node['status'] = node_status
             db_manager.save('nodes', node)

frontend/src/components/InfoView/SpiderInfoView.vue

@@ -61,7 +61,7 @@
       </el-form>
     </el-row>
     <el-row class="button-container" v-if="!isView">
-      <el-button type="danger" @click="onRun">{{$t('Run')}}</el-button>
+      <el-button v-if="isShowRun" type="danger" @click="onRun">{{$t('Run')}}</el-button>
       <el-button type="primary" @click="onDeploy">{{$t('Deploy')}}</el-button>
       <el-button type="success" @click="onSave">{{$t('Save')}}</el-button>
     </el-row>
@@ -109,7 +109,16 @@ export default {
   computed: {
     ...mapState('spider', [
       'spiderForm'
-    ])
+    ]),
+    isShowRun () {
+      if (!this.spiderForm.deploy_ts) {
+        return false
+      }
+      if (!this.spiderForm.cmd) {
+        return false
+      }
+      return true
+    }
   },
   methods: {
     onRun () {
@@ -131,6 +140,11 @@ export default {
     },
     onDeploy () {
       const row = this.spiderForm
+
+      // save spider
+      this.$store.dispatch('spider/editSpider', row._id)
+
+      // validate fields
       this.$refs['spiderForm'].validate(res => {
         if (res) {
           this.$confirm(this.$t('Are you sure to deploy this spider?'), this.$t('Notification'), {

frontend/src/components/InfoView/TaskInfoView.vue

@@ -32,13 +32,13 @@
         </el-form-item>
         <el-form-item :label="$t('Error Message')" v-if="taskForm.status === 'FAILURE'">
           <div class="error-message">
-            {{taskForm.result}}
+            {{taskForm.log}}
           </div>
         </el-form-item>
       </el-form>
     </el-row>
     <el-row class="button-container">
-      <el-button v-if="isRunning" type="danger" @click="onStop">Stop</el-button>
+      <el-button v-if="isRunning" type="danger" @click="onStop">{{$t('Stop')}}</el-button>
       <!--<el-button type="danger" @click="onRestart">Restart</el-button>-->
     </el-row>
   </div>

frontend/src/i18n/zh.js

@@ -8,6 +8,7 @@ export default {
   'Task': '任务',
   'Tasks': '任务',
   'Task Detail': '任务详情',
+  'Schedules': '定时任务',
   'Deploys': '部署',
 
   // 标签
@@ -31,6 +32,7 @@ export default {
   SUCCESS: '成功',
   FAILURE: '错误',
   UNAVAILABLE: '未知',
+  REVOKED: '已取消',
 
   // 操作
   Run: '运行',
@@ -46,6 +48,7 @@ export default {
   Edit: '编辑',
   Remove: '删除',
   Confirm: '确认',
+  Stop: '停止',
 
   // 主页
   'Total Tasks': '总任务数',

frontend/src/router/index.js

@@ -140,7 +140,7 @@ export const constantRouterMap = [
       title: 'Schedules',
       icon: 'fa fa-calendar'
     },
-    hidden: true,
+    hidden: false,
     children: [
       {
         path: '',

frontend/src/store/index.js

@@ -9,6 +9,7 @@ import spider from './modules/spider'
 import deploy from './modules/deploy'
 import task from './modules/task'
 import file from './modules/file'
+import schedule from './modules/schedule'
 import lang from './modules/lang'
 import getters from './getters'
 
@@ -25,6 +26,7 @@ const store = new Vuex.Store({
     deploy,
     task,
     file,
+    schedule,
     lang
   },
   getters

frontend/src/store/modules/schedule.js

@@ -0,0 +1,43 @@
+import request from '../../api/request'
+
+const state = {
+  scheduleList: [],
+  scheduleForm: {}
+}
+
+const getters = {}
+
+const mutations = {
+  SET_SCHEDULE_LIST (state, value) {
+    state.scheduleList = value
+  },
+  SET_SCHEDULE_FORM (state, value) {
+    state.scheduleForm = value
+  }
+}
+
+const actions = {
+  getScheduleList ({ state, commit }) {
+    request.get('/schedules')
+      .then(response => {
+        commit('SET_SCHEDULE_LIST', response.data.items)
+      })
+  },
+  addSchedule ({ state }) {
+    request.put('/schedules', state.scheduleForm)
+  },
+  editSchedule ({ state }, id) {
+    request.post(`/schedules/${id}`, state.scheduleForm)
+  },
+  removeSchedule ({ state }, id) {
+    request.delete(`/schedules/${id}`)
+  }
+}
+
+export default {
+  namespaced: true,
+  state,
+  getters,
+  mutations,
+  actions
+}

frontend/src/store/modules/spider.js

@@ -95,6 +95,10 @@ const actions = {
       .then(response => {
         console.log(response.data)
       })
+      .then(response => {
+        dispatch('getSpiderData', id)
+        dispatch('getSpiderList')
+      })
   },
   crawlSpider ({ state, dispatch }, id) {
     return request.post(`/spiders/${id}/on_crawl`)

frontend/src/views/node/NodeList.vue

@@ -50,9 +50,9 @@
           <el-tooltip :content="$t('View')" placement="top">
             <el-button type="primary" icon="el-icon-search" size="mini" @click="onView(scope.row)"></el-button>
           </el-tooltip>
-          <el-tooltip :content="$t('Edit')" placement="top">
-            <el-button type="warning" icon="el-icon-edit" size="mini" @click="onView(scope.row)"></el-button>
-          </el-tooltip>
+          <!--<el-tooltip :content="$t('Edit')" placement="top">-->
+          <!--<el-button type="warning" icon="el-icon-edit" size="mini" @click="onView(scope.row)"></el-button>-->
+          <!--</el-tooltip>-->
           <el-tooltip :content="$t('Remove')" placement="top">
             <el-button type="danger" icon="el-icon-delete" size="mini" @click="onRemove(scope.row)"></el-button>
           </el-tooltip>