From 62f5c672ee40a1d8f55b156a2a36aadabf8e49bb Mon Sep 17 00:00:00 2001 From: zhang_lu Date: Tue, 12 Nov 2024 23:32:05 +0800 Subject: [PATCH] Add app backend deployment support --- README.md | 7 +- README_ZH.md | 3 +- docker/README.md | 38 ++++++ docker/conductor/docker-compose.yml | 2 - .../conductor/docker-compose.yaml | 118 ++++++++++++++++++ docker/conductor_with_app/conductor/init.sql | 85 +++++++++++++ 6 files changed, 247 insertions(+), 6 deletions(-) create mode 100644 docker/README.md create mode 100644 docker/conductor_with_app/conductor/docker-compose.yaml create mode 100644 docker/conductor_with_app/conductor/init.sql diff --git a/README.md b/README.md index e513e45..63692e9 100644 --- a/README.md +++ b/README.md @@ -48,10 +48,11 @@ In conclusion, key features of OmAgent include: OmAgent utilizes [Conductor](https://github.com/conductor-oss/conductor) as its workflow orchestration engine. Conductor is an open-source, distributed, and scalable workflow engine that supports a variety of programming languages and frameworks. By default, it uses Redis for persistence and Elasticsearch (7.x) as the indexing backend. It is recommended to deploy Conductor using Docker: ```bash -docker compose -f docker/conductor/docker-compose.yml up -d +docker-compose -f docker/conductor/docker-compose.yml up -d ``` - Once deployed, you can access the Conductor UI at `http://localhost:5001`. (Note: Mac system will occupy port 5000 by default, so we use 5001 here. You can specify other ports when deploying Conductor.) - The Conductor API can be accessed via `http://localhost:8080`. +- More details about the deployment can be found [here](docker/README.md). ### 2. Install OmAgent - **Python Version**: Ensure Python 3.10 or higher is installed. @@ -122,8 +123,8 @@ For more information about the container.yaml configuration, please refer to the cd examples/step2_outfit_with_switch python run_app.py ``` - - For the connection and usage of the OmAgent app, please refer to the [app usage documentation](./docs/concepts/app.md) + For app backend deployment, please refer to [here](docker/README.md) + For the connection and usage of the OmAgent app, please check [app usage documentation](./docs/concepts/app.md) ## 🏗 Architecture diff --git a/README_ZH.md b/README_ZH.md index 0c9ce1c..17b6a95 100644 --- a/README_ZH.md +++ b/README_ZH.md @@ -50,6 +50,7 @@ docker compose -f docker/conductor/docker-compose.yml up -d ``` - 部署完成后可以通过访问 `http://localhost:5001` 访问Conductor UI。（注：Mac系统默认会占用5000端口，因此我们使用5001端口，你可以在部署Conductor的时候指定其它端口。） - 通过 `http://localhost:8080` 调用Conductor API。 +- 更多部署细节请参考 [这里](docker/README.md) ### 2. 安装OmAgent - **Python 版本**: 确保已安装 Python 3.10 或更高版本。 @@ -118,7 +119,7 @@ docker compose -f docker/conductor/docker-compose.yml up -d cd examples/step2_outfit_with_switch python run_app.py ``` - + 部署app后端服务请参考 [这里](docker/README.md) OmAgent 的 app 的连接和使用方式请参考 [app使用文档](./docs/concepts/app.md) ## 🏗 架构 diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000..0ccff9f --- /dev/null +++ b/docker/README.md @@ -0,0 +1,38 @@ +## Deploy + +我们提供两种部署方式： +1. 仅包含Conductor的部署，适合基于CLI的开发和测试； +2. 包含Conductor以及App后端的部署，提供对于智能手机App功能的支持。 + +## Docker 和 Docker Compose 安装 +首先请确保您的环境已经安装docker以及docker-compose。如果没有，可以参考[docker官方文档](https://docs.docker.com/get-docker/) 和 [docker-compose官方文档](https://docs.docker.com/compose/install/) 进行安装。 + +## 仅包含Conductor的部署 + +从当前目录运行： +```bash +docker-compose -f conductor/docker-compose.yml up -d +``` +Conductor的基础部署包含一个Redis数据库，一个Elasticsearch数据库，以及Conductor服务共三个镜像。 +部署完成后可以通过访问 `http://localhost:5001` 访问Conductor UI。（注：Mac系统默认会占用5000端口，因此我们使用5001端口，你可以在部署Conductor的时候指定其它端口。） +通过 `http://localhost:8080` 调用Conductor API。 + +## 包含Conductor和App后端的部署 + +为了支持app的开发和调试我们需要额外部署App后端服务，以及MySQL数据库和minio对象存储服务这两个中间件。 + +1. 创建数据目录 + 从当前目录运行： + ```bash + mkdir -p ../pv-data/mysql/data ../pv-data/mysql/log ../pv-data/minio-data + ``` +这会为MySQL数据库和minio对象存储服务创建数据目录。（注：请注意保证目录具有足够的读写权限） + +2. 修改配置项 + 修改config/om-app-agent/resources/bootstrap-test.yaml中的linker.cos.minio.urlPrefix:后的ip为本机ip（如需公网访问，需要修改为对应公网ip） + +3. 启动服务 + 从当前目录运行： + ```bash + docker-compose -f conductor_with_app/conductor/docker-compose.yaml up -d + ``` \ No newline at end of file diff --git a/docker/conductor/docker-compose.yml b/docker/conductor/docker-compose.yml index b448d36..7cb4fc8 100644 --- a/docker/conductor/docker-compose.yml +++ b/docker/conductor/docker-compose.yml @@ -33,8 +33,6 @@ services: conductor-redis: image: redis:6.2.3-alpine - volumes: - - ../server/config/redis.conf:/usr/local/etc/redis/redis.conf networks: - internal ports: diff --git a/docker/conductor_with_app/conductor/docker-compose.yaml b/docker/conductor_with_app/conductor/docker-compose.yaml new file mode 100644 index 0000000..cf1a76f --- /dev/null +++ b/docker/conductor_with_app/conductor/docker-compose.yaml @@ -0,0 +1,118 @@ +services: + conductor-server: + environment: + - CONFIG_PROP=config-redis.properties + image: omailab/omagent-conductor:v3.21.6 + container_name: conductor-server + + networks: + - internal + ports: + - 8080:8080 + - 5000:5000 + healthcheck: + test: ["CMD", "curl","-I" ,"-XGET", "http://localhost:8080/health"] + interval: 60s + timeout: 30s + retries: 12 + links: + - conductor-elasticsearch:es + - conductor-redis:rs + depends_on: + conductor-elasticsearch: + condition: service_healthy + conductor-redis: + condition: service_healthy + logging: + driver: "json-file" + options: + max-size: "1k" + max-file: "3" + + conductor-redis: + image: redis:6.2.3-alpine + container_name: conductor-redis + networks: + - internal + ports: + - 6379:6379 + healthcheck: + test: [ "CMD", "redis-cli","ping" ] + + conductor-elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch:7.17.11 + container_name: conductor-elasticsearch + environment: + - "ES_JAVA_OPTS=-Xms512m -Xmx1024m" + - xpack.security.enabled=false + - discovery.type=single-node + volumes: + - esdata-conductor:/usr/share/elasticsearch/data + networks: + - internal + ports: + - 9200:9200 + healthcheck: + test: curl http://localhost:9200/_cluster/health -o /dev/null + interval: 5s + timeout: 5s + retries: 12 + logging: + driver: "json-file" + options: + max-size: "1k" + max-file: "3" + + om-app-agent: + #image: hbt.linker.cc/yting/omappagent:feature_om_app_1.0.0-202411110929 + #image: registry.linker.cc/yting/omappagent:feature_om_app_1.0.0-202411121609 + image: registry.linker.cc/yting/omappagent:feature_om_app_1.0.0-202411122032 + container_name: om-app-agent + restart: always + environment: + - SPRING_PROFILES=./bootstrap-test.yaml + ports: + - "8082:8082" + volumes: + - ../config/om-app-agent/resources/bootstrap-test.yaml:/usr/local/bootstrap-test.yaml:rw + networks: + - internal + + mysql: + image: mysql:5.7.44 + restart: always + ports: + - "3306:3306" + container_name: mysql + privileged: true + environment: + - MYSQL_ROOT_PASSWORD=Test@58455 + volumes: + - ../pv-data/mysql/data:/var/lib/mysql:rw + - ../pv-data/mysql/log:/var/log/mysql:rw + - ./init.sql:/docker-entrypoint-initdb.d/init.sql + networks: + - internal + + minio: + image: bitnami/minio:latest + container_name: minio + restart: always + environment: + MINIO_ROOT_USER: 8V51TQi0Haqgtv8C + MINIO_ROOT_PASSWORD: xfnGFZ7HakQlmurdG3LIHtB60luMJrD6 + MINIO_DEFAULT_BUCKETS: hzlh:public + volumes: + - ../pv-data/minio-data:/bitnami/minio/data:rw + ports: + - "9000:9000" + - "9001:9001" + networks: + - internal + +volumes: + esdata-conductor: + driver: local + +networks: + internal: diff --git a/docker/conductor_with_app/conductor/init.sql b/docker/conductor_with_app/conductor/init.sql new file mode 100644 index 0000000..c9afed4 --- /dev/null +++ b/docker/conductor_with_app/conductor/init.sql @@ -0,0 +1,85 @@ +CREATE DATABASE IF NOT EXISTS om_app_agent; +use om_app_agent; +create table tbl_chat_record +( + id bigint auto_increment comment '主键id' + primary key, + creator_id bigint not null comment '创建者id', + create_time datetime null comment '创建时间', + update_id bigint null comment '更新者id', + update_time datetime null comment '更新时间', + deleted int(2) default 0 not null comment '0.未删除 其他.已删除', + remark varchar(512) null comment '备注', + user_id varchar(64) null comment '预留，用户主键id', + session_id varchar(64) null comment 'sessionid 一次完整会话标识', + content json null comment '内容', + type int default 1 not null comment '消息类型，1：普通，2，预留，3：时序', + role varchar(16) null comment '角色user, assistant,system', + workflow_name varchar(512) null comment 'workflow名称', + content_status varchar(64) null comment '回答状态' + +) + comment '聊天记录表'; + +create table tbl_workflow_record +( + id bigint auto_increment comment '主键id' + primary key, + creator_id bigint not null comment '创建者id', + create_time datetime null comment '创建时间', + update_id bigint null comment '更新者id', + update_time datetime null comment '更新时间', + deleted int(2) default 0 not null comment '0.未删除 其他.已删除', + remark varchar(512) null comment '备注', + user_id varchar(64) null comment '预留，用户主键id', + session_id varchar(64) null comment 'sessionid 一次完整会话标识', + progress varchar(512) null comment '运行进度', + message varchar(512) null comment '运行进度说明', + workflow_name varchar(512) null comment 'workflow名称' + +) + comment 'workflow运行记录表'; + +create table tbl_resource +( + id bigint auto_increment comment '主键id' + primary key, + creator_id bigint not null comment '创建者id', + create_time datetime null comment '创建时间', + update_id bigint null comment '更新者id', + update_time datetime null comment '更新时间', + deleted int(2) default 0 not null comment '0.未删除 其他.已删除', + remark varchar(512) null comment '备注', + user_id varchar(64) null comment '预留，用户主键id', + resource_id varchar(64) null comment '资源唯一标识', + type int(4) default 1 null comment '资源类型，1：图片', + resource_url varchar(512) null comment '资源url', + resource_name varchar(512) null comment '资源名称', + preview_url varchar(512) null comment '预留，预览url', + resource_key varchar(512) null comment '对象存储key' +) comment '资源表'; + +create table tbl_dictionary +( + id int auto_increment comment '主键' + primary key, + type varchar(255) null comment '枚举值', + type_name varchar(255) null comment '类型描述', + name varchar(255) null comment '枚举名称', + value int null comment '类型', + pid int null comment '父节点', + sort int(16) null comment '排序', + create_time datetime default CURRENT_TIMESTAMP null on update CURRENT_TIMESTAMP, + update_time datetime default CURRENT_TIMESTAMP null on update CURRENT_TIMESTAMP, + deleted tinyint default 0 null comment '是否删除 1：已删除 0：未删除', + extension_value varchar(100) null comment '额外值' +) + comment '字典表'; + +INSERT INTO tbl_dictionary (id, type, type_name, name, value, pid, sort, create_time, update_time, deleted, + extension_value) +VALUES (1, 'chat_turns', 'turns of single chat conversation', '5', 1, null, 1, null, null, 0, null); + +CREATE USER 'hd'@'%' IDENTIFIED WITH mysql_native_password BY 'F098058455ec50b3a'; +GRANT ALL PRIVILEGES ON om_app_agent.* TO 'hd'@'%'; +FLUSH PRIVILEGES;