From de7c33416776b5635f39f99ec154a45e95ec6e0c Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Fri, 24 Dec 2021 15:07:04 +0800 Subject: [PATCH 1/6] updated version --- backend/cmd/root.go | 14 +++++++++++++ backend/go.mod | 2 +- backend/go.mod.local | 5 ++--- backend/test/config-worker-03.json | 10 +++++++++ frontend/package.json | 4 ++-- frontend/public/index.html | 6 +++--- workspace/docker-compose.yml | 33 ++++++++++++++++++++++++++---- 7 files changed, 61 insertions(+), 13 deletions(-) create mode 100644 backend/test/config-worker-03.json diff --git a/backend/cmd/root.go b/backend/cmd/root.go index 8c6c322f5..9d0ef3eb4 100644 --- a/backend/cmd/root.go +++ b/backend/cmd/root.go @@ -2,6 +2,7 @@ package cmd import ( "fmt" + "github.com/apex/log" "strings" "github.com/spf13/cobra" @@ -54,4 +55,17 @@ func initConfig() { if err := viper.ReadInConfig(); err == nil { fmt.Println("Using config file:", viper.ConfigFileUsed()) } + + // initialize log level + initLogLevel() +} + +func initLogLevel() { + // set log level + logLevel := viper.GetString("log.level") + l, err := log.ParseLevel(logLevel) + if err != nil { + l = log.InfoLevel + } + log.SetLevel(l) } diff --git a/backend/go.mod b/backend/go.mod index ef8367fc0..63f78dedb 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -4,7 +4,7 @@ go 1.16 require ( github.com/apex/log v1.9.0 - github.com/crawlab-team/crawlab-core v0.6.0-beta.20211219.2134 + github.com/crawlab-team/crawlab-core v0.6.0-beta.20211224.1500 github.com/crawlab-team/go-trace v0.1.1 github.com/gin-gonic/gin v1.7.1 github.com/spf13/cobra v1.1.3 diff --git a/backend/go.mod.local b/backend/go.mod.local index 129fe1943..41c139328 100644 --- a/backend/go.mod.local +++ b/backend/go.mod.local @@ -11,11 +11,10 @@ replace ( require ( github.com/apex/log v1.9.0 - github.com/crawlab-team/crawlab-core v0.6.0-beta.20211213.2006 - github.com/crawlab-team/go-trace v0.1.0 + github.com/crawlab-team/crawlab-core v0.6.0-beta.20211219.2134 + github.com/crawlab-team/go-trace v0.1.1 github.com/gin-gonic/gin v1.7.1 github.com/spf13/cobra v1.1.3 github.com/spf13/viper v1.7.1 - go.mongodb.org/mongo-driver v1.6.0 // indirect go.uber.org/dig v1.10.0 ) diff --git a/backend/test/config-worker-03.json b/backend/test/config-worker-03.json new file mode 100644 index 000000000..70da53f10 --- /dev/null +++ b/backend/test/config-worker-03.json @@ -0,0 +1,10 @@ +{ + "key": "worker-03", + "is_master": false, + "name": "Worker Node 03", + "ip": "", + "mac": "", + "hostname": "", + "description": "", + "auth_key": "Crawlab2021!" +} \ No newline at end of file diff --git a/frontend/package.json b/frontend/package.json index a41ba197d..1f460a36b 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "@crawlab/app", - "version": "0.6.0-beta.20211221.1200", + "version": "0.6.0-beta.20211224.1500", "description": "", "scripts": { "serve": "vue-cli-service serve", @@ -15,7 +15,7 @@ }, "license": "BSD-3-Clause", "dependencies": { - "crawlab-ui": "0.6.0-beta.20211219.1200", + "crawlab-ui": "0.6.0-beta.20211224.1500", "vue": "3.0.11", "vue-router": "^4.0.11" }, diff --git a/frontend/public/index.html b/frontend/public/index.html index e307c2bc2..113fdc38d 100644 --- a/frontend/public/index.html +++ b/frontend/public/index.html @@ -1,12 +1,12 @@ + <%= htmlWebpackPlugin.options.title %> + + - - - <%= htmlWebpackPlugin.options.title %> diff --git a/workspace/docker-compose.yml b/workspace/docker-compose.yml index b20f9cabe..52d35ce39 100644 --- a/workspace/docker-compose.yml +++ b/workspace/docker-compose.yml @@ -1,10 +1,24 @@ version: '3.3' services: + build: + build: + context: ./dockerfiles/golang + command: /bin/bash -c "rm -f /backend/dist/crawlab && go build -o ./dist/crawlab ./" + volumes: + - ./dist:/backend/dist + - ./.crawlab/go/pkg/mod:/go/pkg/mod + - ../backend:/backend + - ../backend/go.mod.local:/backend/go.mod + - ../..:/libs/crawlab-team + environment: + GOPROXY: https://goproxy.cn,direct + master: build: context: ./dockerfiles/golang - command: /bin/bash -c "/app/bin/docker-start-master.sh && air -c .air.master.conf" + command: /bin/bash -c "/app/bin/docker-start-master.sh && /dist/crawlab master" volumes: + - ./dist:/dist - ./.crawlab/master:/root/.crawlab - ./.crawlab/go/pkg/mod:/go/pkg/mod - ../backend:/backend @@ -17,22 +31,29 @@ services: CRAWLAB_NODE_MASTER: "Y" CRAWLAB_NODE_NAME: "Master Node" CRAWLAB_MONGO_HOST: "mongo" + CRAWLAB_LOG_LEVEL: debug + GOPROXY: https://goproxy.cn,direct ports: - "9080:8080" - "9000:8000" + - "9866:9666" + - "9888:8888" depends_on: - mongo worker01: build: context: ./dockerfiles/golang - command: "air -c .air.worker.conf" + command: /bin/bash -c "/dist/crawlab worker" environment: CRAWLAB_NODE_MASTER: "N" CRAWLAB_NODE_NAME: "Worker Node 01" CRAWLAB_GRPC_ADDRESS: "master" CRAWLAB_FS_FILER_URL: "http://master:8080/api/filer" + CRAWLAB_LOG_LEVEL: debug + GOPROXY: https://goproxy.cn,direct volumes: + - ./dist:/dist - ./.crawlab/worker01:/root/.crawlab - ./.crawlab/go/pkg/mod:/go/pkg/mod - ../backend:/backend @@ -44,13 +65,16 @@ services: worker02: build: context: ./dockerfiles/golang - command: "air -c .air.worker.conf" + command: /bin/bash -c "/dist/crawlab worker" environment: CRAWLAB_NODE_MASTER: "N" - CRAWLAB_NODE_NAME: "Worker Node 01" + CRAWLAB_NODE_NAME: "Worker Node 02" CRAWLAB_GRPC_ADDRESS: "master" CRAWLAB_FS_FILER_URL: "http://master:8080/api/filer" + CRAWLAB_LOG_LEVEL: debug + GOPROXY: https://goproxy.cn,direct volumes: + - ./dist:/dist - ./.crawlab/worker02:/root/.crawlab - ./.crawlab/go/pkg/mod:/go/pkg/mod - ../backend:/backend @@ -58,6 +82,7 @@ services: - ../..:/libs/crawlab-team depends_on: - master + - worker01 mongo: image: mongo:4 From 5c2e5a4ad0e5faa10478ae5c2c8a4f54ba55bea5 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Fri, 24 Dec 2021 15:07:13 +0800 Subject: [PATCH 2/6] updated go.mod --- backend/go.sum | 4 ++-- nginx/crawlab.conf | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/go.sum b/backend/go.sum index a47cfbe3e..43e165bb0 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -63,8 +63,8 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfc github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/crawlab-team/crawlab-core v0.0.1/go.mod h1:6dJHMvrmIJbfYHhYNeGZkGOLEBvur+yGiFzLCRXx92k= -github.com/crawlab-team/crawlab-core v0.6.0-beta.20211219.2134 h1:J2NKuIhkXZPlfx9hHT2L4VLS9FMHzEaDGgHP/8ZJKkI= -github.com/crawlab-team/crawlab-core v0.6.0-beta.20211219.2134/go.mod h1:Y/3Z1WPYvSas5pQtRNypgQoyS3JHvrk2wtdR8n94sHY= +github.com/crawlab-team/crawlab-core v0.6.0-beta.20211224.1500 h1:XEQNJ0Bf1OOj7n1gEFstVz8oilfLXZhZJCid0tiC+PA= +github.com/crawlab-team/crawlab-core v0.6.0-beta.20211224.1500/go.mod h1:Y/3Z1WPYvSas5pQtRNypgQoyS3JHvrk2wtdR8n94sHY= github.com/crawlab-team/crawlab-db v0.0.2/go.mod h1:o7o4rbcyAWlFGHg9VS7V7tM/GqRq+N2mnAXO71cZA78= github.com/crawlab-team/crawlab-db v0.1.3 h1:RqLoXGZEMUH1B8SQB5OcNmJeyY2xILvwyhv4X9faWl4= github.com/crawlab-team/crawlab-db v0.1.3/go.mod h1:kPkGZ1P802XdbFFb8byMpZfNG2lWTNoWNRy4beS0/QY= diff --git a/nginx/crawlab.conf b/nginx/crawlab.conf index 8d1cc3472..f7b03328c 100644 --- a/nginx/crawlab.conf +++ b/nginx/crawlab.conf @@ -3,7 +3,7 @@ server { gzip_min_length 1k; gzip_buffers 4 16k; gzip_comp_level 2; - gzip_types text/plain application/javascript application/x-javascript text/css application/xml text/javascript application/x-httpd-php image/jpeg image/gif image/png; + gzip_types text/plain application/javascript application/x-javascript text/css application/xml text/javascript application/x-httpd-php image/jpeg image/gif image/png image/x-icon; gzip_vary off; gzip_disable "MSIE [1-6]\."; client_max_body_size 200m; From d5fa092d63c3193e5ebc7411ff168a950291351b Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Fri, 24 Dec 2021 15:37:05 +0800 Subject: [PATCH 3/6] updated version --- frontend/package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/package.json b/frontend/package.json index 1f460a36b..c064153e7 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "@crawlab/app", - "version": "0.6.0-beta.20211224.1500", + "version": "0.6.0-beta.20211224.1536", "description": "", "scripts": { "serve": "vue-cli-service serve", @@ -15,7 +15,7 @@ }, "license": "BSD-3-Clause", "dependencies": { - "crawlab-ui": "0.6.0-beta.20211224.1500", + "crawlab-ui": "0.6.0-beta.20211224.1536", "vue": "3.0.11", "vue-router": "^4.0.11" }, From 8a616f795bff04138272dae4dd693f456f2f40bd Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Fri, 24 Dec 2021 17:24:48 +0800 Subject: [PATCH 4/6] updated go.mod --- backend/go.mod | 2 +- backend/go.sum | 4 ++-- frontend/package.json | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/backend/go.mod b/backend/go.mod index 63f78dedb..178b73305 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -4,7 +4,7 @@ go 1.16 require ( github.com/apex/log v1.9.0 - github.com/crawlab-team/crawlab-core v0.6.0-beta.20211224.1500 + github.com/crawlab-team/crawlab-core v0.6.0-beta.20211224.1718 github.com/crawlab-team/go-trace v0.1.1 github.com/gin-gonic/gin v1.7.1 github.com/spf13/cobra v1.1.3 diff --git a/backend/go.sum b/backend/go.sum index 43e165bb0..be239ac93 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -63,8 +63,8 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfc github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/crawlab-team/crawlab-core v0.0.1/go.mod h1:6dJHMvrmIJbfYHhYNeGZkGOLEBvur+yGiFzLCRXx92k= -github.com/crawlab-team/crawlab-core v0.6.0-beta.20211224.1500 h1:XEQNJ0Bf1OOj7n1gEFstVz8oilfLXZhZJCid0tiC+PA= -github.com/crawlab-team/crawlab-core v0.6.0-beta.20211224.1500/go.mod h1:Y/3Z1WPYvSas5pQtRNypgQoyS3JHvrk2wtdR8n94sHY= +github.com/crawlab-team/crawlab-core v0.6.0-beta.20211224.1718 h1:nNmFoR/o+xEZ6ywe9mwx0QroF+QSHrJR5DotOm/r/tg= +github.com/crawlab-team/crawlab-core v0.6.0-beta.20211224.1718/go.mod h1:Y/3Z1WPYvSas5pQtRNypgQoyS3JHvrk2wtdR8n94sHY= github.com/crawlab-team/crawlab-db v0.0.2/go.mod h1:o7o4rbcyAWlFGHg9VS7V7tM/GqRq+N2mnAXO71cZA78= github.com/crawlab-team/crawlab-db v0.1.3 h1:RqLoXGZEMUH1B8SQB5OcNmJeyY2xILvwyhv4X9faWl4= github.com/crawlab-team/crawlab-db v0.1.3/go.mod h1:kPkGZ1P802XdbFFb8byMpZfNG2lWTNoWNRy4beS0/QY= diff --git a/frontend/package.json b/frontend/package.json index c064153e7..00a9ddada 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "@crawlab/app", - "version": "0.6.0-beta.20211224.1536", + "version": "0.6.0-beta.20211224.1718", "description": "", "scripts": { "serve": "vue-cli-service serve", @@ -15,7 +15,7 @@ }, "license": "BSD-3-Clause", "dependencies": { - "crawlab-ui": "0.6.0-beta.20211224.1536", + "crawlab-ui": "0.6.0-beta.20211224.1718", "vue": "3.0.11", "vue-router": "^4.0.11" }, From b2631b9cc283dc7cccbadf2e6bfaca0d4a0a6356 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Fri, 24 Dec 2021 17:41:59 +0800 Subject: [PATCH 5/6] added changelogs --- changelog/v0.6.0-beta.20211224-zh.md | 32 ++++++++++++++++++++++++++++ changelog/v0.6.0-beta.20211224.md | 32 ++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 changelog/v0.6.0-beta.20211224-zh.md create mode 100644 changelog/v0.6.0-beta.20211224.md diff --git a/changelog/v0.6.0-beta.20211224-zh.md b/changelog/v0.6.0-beta.20211224-zh.md new file mode 100644 index 000000000..2de9e2f8d --- /dev/null +++ b/changelog/v0.6.0-beta.20211224-zh.md @@ -0,0 +1,32 @@ +# 更新日志 (v0.6.0-beta.20211224) + +## 概览 + +这是下个重大版本 v0.6.0 的第三次 beta 发布。伴随着更多功能和优化，正式版本 v0.6.0 将会很快发布了。 + +## 升级优化 + +- [x] **国际化**. 支持中文. +- [x] **命令行工具上传爬虫**. [#1020](https://github.com/crawlab-team/crawlab/issues/1020) +- [x] **官方插件**. 允许用户在 Crawlab 用户界面上安装 [官方插件](https://docs-next.crawlab.cn/zh/guide/use-crawlab/public-plugins/). +- [x] **更多文档**. 加入了关于插件和命令行工具的文档. + +## Bug 修复 + +- [x] **上传文件问题** [#990](https://github.com/crawlab-team/crawlab/issues/990) [#1014](https://github.com/crawlab-team/crawlab/issues/1014) [#1022](https://github.com/crawlab-team/crawlab/issues/1022) +- [x] **无法重命名文件** [#1008](https://github.com/crawlab-team/crawlab/issues/1008) + +## TODOs +- [ ] **关联任务**. 如果任务执行模式为 “所有节点” 或 “指定节点”，那么将会有主任务和子任务之分。 +- [ ] **Crontab 编辑器**. 可视化 Crontab 编辑的前端组件。 +- [ ] **结果去重**. +- [ ] **环境变量**. +- [ ] **前端易用性优化**. 更多高级功能，例如表格形式保存。 +- [ ] **日志自动清理**. +- [ ] **跟多文档**. +- [ ] **前端输出文件大小优化**. + +## 未来计划 + +下一个版本有可能是 v0.6.0 的正式版本，但是没有确定。本次发布版本将会进行更多的测试，以保证其健壮性以及生产可用。 + diff --git a/changelog/v0.6.0-beta.20211224.md b/changelog/v0.6.0-beta.20211224.md new file mode 100644 index 000000000..7c7e6daf0 --- /dev/null +++ b/changelog/v0.6.0-beta.20211224.md @@ -0,0 +1,32 @@ +# Change Log (v0.6.0-beta.20211224) + +## Overview + +This is the third beta release for the next major version v0.6.0. With more features and optimization coming in, the release of official version v0.6.0 is approaching soon. + +## Enhancement + +- [x] **Internationalization**. Support Chinese. +- [x] **CLI Upload Spider**. [#1020](https://github.com/crawlab-team/crawlab/issues/1020) +- [x] **Official Plugins**. Allow users to install [official plugins](https://docs-next.crawlab.cn/en/guide/use-crawlab/public-plugins/) on Crawlab web UI. +- [x] **More Documentation**. Added documentation for plugins and CLI. + +## Bug Fixes + +- [x] **Upload File Issue** [#990](https://github.com/crawlab-team/crawlab/issues/990) [#1014](https://github.com/crawlab-team/crawlab/issues/1014) [#1022](https://github.com/crawlab-team/crawlab/issues/1022) +- [x] **Unable to Rename File** [#1008](https://github.com/crawlab-team/crawlab/issues/1008) + +## TODOs +- [ ] **Associated Tasks**. There will be main tasks and their sub-tasks if task mode is "all nodes" or "selected nodes". +- [ ] **Crontab Editor**. Frontend component that visualize the crontab editing. +- [ ] **Results Deduplication**. +- [ ] **Environment Variables**. +- [ ] **Frontend Utility Enhancement**. Advanced features such as saved table customization. +- [ ] **Log Auto Cleanup**. +- [ ] **More Documentation**. +- [ ] **E2E Tests**. +- [ ] **Frontend Output File Size Optimization**. + +## What Next + +The next version could the official release of v0.6.0, but not determined yet. There will be more tests running against the current beta version to ensure robustness and production-ready deployment. From 6c1f78b114e8995d829feabe2efb5b0396915b23 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Fri, 24 Dec 2021 17:49:15 +0800 Subject: [PATCH 6/6] updated dockerfile --- Dockerfile | 2 +- Dockerfile.cn | 2 +- Dockerfile.local | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5d43b8396..b88ddf892 100644 --- a/Dockerfile +++ b/Dockerfile @@ -49,7 +49,7 @@ RUN wget https://github.com/crawlab-team/resources/raw/main/seaweedfs/2.79/linux # install backend RUN pip install scrapy pymongo bs4 requests -RUN pip install crawlab-sdk==0.6.b20211213-2123 +RUN pip install crawlab-sdk==0.6.b20211224-1500 # add files COPY ./backend/conf /app/backend/conf diff --git a/Dockerfile.cn b/Dockerfile.cn index 13dfecadf..225880bc3 100644 --- a/Dockerfile.cn +++ b/Dockerfile.cn @@ -49,7 +49,7 @@ RUN wget https://github.com/crawlab-team/resources/raw/main/seaweedfs/2.79/linux # install backend RUN pip install scrapy pymongo bs4 requests -i https://mirrors.aliyun.com/pypi/simple -RUN pip install crawlab-sdk==0.6.b20211213-2123 +RUN pip install crawlab-sdk==0.6.b20211224-1500 # add files COPY ./backend/conf /app/backend/conf diff --git a/Dockerfile.local b/Dockerfile.local index c101b77f6..4a1956b7e 100644 --- a/Dockerfile.local +++ b/Dockerfile.local @@ -49,7 +49,7 @@ RUN wget https://github.com/crawlab-team/resources/raw/main/seaweedfs/2.79/linux # install backend RUN pip install scrapy pymongo bs4 requests -i https://mirrors.aliyun.com/pypi/simple -RUN pip install crawlab-sdk==0.6.b20211213-2123 +RUN pip install crawlab-sdk==0.6.b20211224-1500 # add files COPY ./backend/conf /app/backend/conf