Merge pull request #403 from FederatedAI/dev-2.0-alpha

Dev 2.0 alpha

bin/init_env.sh

@@ -16,7 +16,7 @@
 #  limitations under the License.
 #
 
-fate_project_base=$(cd `dirname "$(realpath "${BASH_SOURCE[0]:-${(%):-%x}}")"`; cd ../cd ../;pwd)
+fate_project_base=$(cd `dirname "$(realpath "${BASH_SOURCE[0]:-${(%):-%x}}")"`; cd ../;cd ../;pwd)
 export FATE_PROJECT_BASE=$fate_project_base
 export FATE_DEPLOY_BASE=$fate_project_base
 export EGGROLL_HOME=

conf/service_conf.yaml

@@ -79,5 +79,5 @@ worker:
     # on host
     eggroll_conf_dir:
   k8s:
-    config:
-    image:
+    image: ccr.ccs.tencentyun.com/federatedai/fate_algorithm:2.0.0-alpha
+    namespace: fate-10000

doc/quick_start.md

@@ -2,8 +2,8 @@
 
 #### 1.1 源码获取
 ##### 1.1.1 从github拉取源码
-  - [FATE](https://github.com/FederatedAI/FATE/tree/dev-2.0-alpha)
-  - [FATE-Flow](https://github.com/FederatedAI/FATE-Flow/tree/dev-2.0-alpha)
+  - [FATE](https://github.com/FederatedAI/FATE/tree/release-2.0-alpha)
+  - [FATE-Flow](https://github.com/FederatedAI/FATE-Flow/tree/release-2.0-alpha)
 ##### 1.1.2 新建部署目录:
 ```shell
 mkdir -p /data/projects/fate2.0
@@ -15,11 +15,27 @@ mkdir -p /data/projects/fate2.0
   # fate算法包
   mv ./FATE/python /data/projects/fate2.0/python
 ```
-#### 1.2 依赖下载
+#### 1.2 依赖
+##### 1.2.1 miniconda安装
 ```shell
+wget https://webank-ai-1251170195.cos.ap-guangzhou.myqcloud.com/resources/Miniconda3-py38_4.12.0-Linux-x86_64.sh
+#创建python虚拟化安装目录
+mkdir -p /data/projects/fate2.0/common/python/venv
+
+#安装miniconda3
+bash Miniconda3-py38_4.12.0-Linux-x86_64.sh -b -p /data/projects/fate2.0/common/miniconda3
+#创建虚拟化环境
+/data/projects/fate2.0/common/miniconda3/bin/python3.8 -m venv /data/projects/fate2.0/common/python/venv
+```
+
+##### 1.2.2 依赖安装
+```shell
+source /data/projects/fate2.0/common/python/venv/bin/activate
+cd /data/projects/fate2.0/fate_flow/python
 pip install -r requirements.txt
 ```
 详细依赖参考： [requirements.txt](../python/requirements.txt)
+
 #### 1.3 修改配置
 #### 1.3.1 配置说明
 - 系统配置文件[service_conf.yaml](../conf/service_conf.yaml)说明：
@@ -42,11 +58,9 @@ federation: 通信服务详细地址
 - 根据实际部署情况修改系统配置service_conf.yaml
 - 修改fate_flow/bin/init_env.sh, 参考如下：
 ```yaml
-export FATE_PROJECT_BASE=/data/projects/fate2.0
-export FATE_DEPLOY_BASE=/data/projects/fate2.0
 export EGGROLL_HOME=/data/projects/fate/eggroll
 export PYTHONPATH=/data/projects/fate2.0/python:/data/projects/fate2.0/fate_flow/python:/data/projects/fate/eggroll/python
-venv=/data/projects/fate2.0/flow
+venv=/data/projects/fate2.0/common/python/venv
 export PATH=$PATH:$JAVA_HOME/bin
 source ${venv}/bin/activate
 ```
@@ -72,7 +86,7 @@ source ${venv}/bin/activate
     ```shell
     sh /data/projects/fate2.0/fate_flow/bin/service.sh status
     ```
-  
+
 ### 2. 使用指南
 #### 2.1 数据上传
 - 若计算引擎使用standalone，reader组件参数支持配置文件路径，数据无需上传，使用时配置如下：
@@ -195,20 +209,14 @@ worker:
     config:
       base_url: unix:///var/run/docker.sock
     image: ccr.ccs.tencentyun.com/federatedai/fate_algorithm:2.0.0-alpha
-    # on container
+    # 容器内路径，一般不需要更改
     fate_root_dir: /data/projects/fate
-    # on host
+    # 宿主机路径，根据实际情况填写
     eggroll_conf_dir:
   k8s:
-    config:
-    image:
+    image: ccr.ccs.tencentyun.com/federatedai/fate_algorithm:2.0.0-alpha
+    namespace: fate-10000
 ```
-- 在2.0.0-alpha版本中暂不支持算法容器注册功能，只支持固定模式的算法运行方案："local"、"docker"或"k8s", 由配置"type"决定运行模式。
-- type支持：docker、k8s，默认使用非容器模式
-
-
-
-
-
-
-
+- 在 2.0.0-alpha 版本中暂不支持算法容器注册功能，只支持固定模式的算法运行方案：`local`、`docker` 或 `k8s`, 由配置 `type` 决定运行模式。
+- `worker.type` 支持：`docker`、`k8s`，默认使用非容器模式，即 `native`。
+- 容器模式不支持通信组件使用 `standalone`，需更改 `default_engines.federation` 为其他组件。

python/fate_flow/manager/k8s_conf_template.yaml

@@ -0,0 +1,8 @@
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: ''
+  namespace: fate-9999
+data:
+  service_conf.yaml: |
+     

python/fate_flow/manager/k8s_manager.py

@@ -19,46 +19,69 @@
 from ruamel import yaml
 
 from fate_flow.settings import WORKER
+from fate_flow.utils.conf_utils import get_base_config
+from fate_flow.utils.log import getLogger
+
+LOGGER = getLogger("k8s-manager")
 
 
 class K8sManager:
     image = WORKER.get('k8s', {}).get('image', '')
-
+    namespace = WORKER.get('k8s', {}).get('namespace', '')
     def __init__(self):
-        config.load_incluster_config()
+        config.load_kube_config()
         self.job_template = yaml.safe_load(
             (Path(__file__).parent / 'k8s_template.yaml').read_text('utf-8')
         )
-
-    @property
-    def namespace(self):
-        # In below file, the pod can read its K8s namespace
-        with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace") as f:
-            namespace = f.readline()
-        return namespace
+        self.job_conf_template = yaml.safe_load(
+            (Path(__file__).parent / 'k8s_conf_template.yaml').read_text('utf-8')
+        )
 
     def populate_yaml_template(self, name, command, environment):
-        job_conf = copy.deepcopy(self.job_template)
-        metadata = job_conf['metadata']
-        container_spec = job_conf['spec']['template']['spec']['containers'][0]
-        metadata['name'] = name
+        job = copy.deepcopy(self.job_template)
+        metadata = job['metadata']
+        container_spec = job['spec']['template']['spec']['containers'][0]
+        metadata['name'] = self.convertname(name)
         metadata['namespace'] = self.namespace
-        container_spec['name'] = name
+        container_spec['name'] = self.convertname(name)
         container_spec['image'] = self.image
-        container_spec['command'] = command
+        container_spec['command'] = ["/data/projects/fate/env/python/venv/bin/python"]
+        container_spec['args'] = command
         container_spec['env'] = [{'name': k, 'value': v} for k, v in environment.items()]
+        volumes=job['spec']['template']['spec']['volumes'][0]
+        volumes['configMap']['name']=self.convertname(name + "job-conf")
+        return job
+    def populate_conf_yaml_template(self, name, service_conf):
+        job_conf = copy.deepcopy(self.job_conf_template)
+        metadata = job_conf['metadata']
+        metadata['name'] = self.convertname(name + "job-conf")
+        metadata['namespace'] = self.namespace
+        job_conf['data']['service_conf.yaml'] = service_conf
         return job_conf
 
     def start(self, name, command, environment):
-        job_conf = self.populate_yaml_template(name, command, environment)
-        client.BatchV1Api().create_namespaced_job(self.namespace, job_conf)
+        # LOGGER.debug(f"command: {type(command)}, {command}")
+        job = self.populate_yaml_template(self.convertname(name), command, environment)
+        service_conf=yaml.safe_dump(get_base_config(key=None), default_flow_style=False)
+        job_conf = self.populate_conf_yaml_template(self.convertname(name), service_conf)
+        LOGGER.debug(f"job: {job}")
+        LOGGER.debug(f"job_conf: {job}")
+        client.CoreV1Api().create_namespaced_config_map(self.namespace, job_conf)
+        client.BatchV1Api().create_namespaced_job(self.namespace, job)
 
     def stop(self, name):
+        LOGGER.debug(f"stop job {name}")
         body = client.V1DeleteOptions(propagation_policy='Background')
-        client.BatchV1Api().delete_namespaced_job(name, self.namespace, body=body)
+        client.BatchV1Api().delete_namespaced_job(self.convertname(name), self.namespace, body=body, async_req=True)
+        client.CoreV1Api().delete_namespaced_config_map(self.convertname(name + "job-conf"),self.namespace, body=body, async_req=True)
 
     def is_running(self, name):
-        res = client.BatchV1Api().read_namespaced_job_status(name, self.namespace)
+        res = client.BatchV1Api().read_namespaced_job_status(self.convertname(name), self.namespace)
+        # LOGGER.debug(f"res: {res}")
         if not res:
             return False
         return not (res.status.succeeded or res.status.failed)
+
+    # convertname: Ensure that name composes the RFC 1123 specification
+    def convertname(self, name):
+        return name.lower().replace('_','-')

python/fate_flow/manager/k8s_template.yaml

@@ -13,5 +13,21 @@ spec:
           image: ''
           imagePullPolicy: IfNotPresent
           command: []
+          args: []
           env: []
-      serviceAccountName: flow
+          volumeMounts:
+          - mountPath: /data/projects/fate/eggroll/conf/
+            name: service-conf
+          - mountPath: /data/projects/fate/fate_flow/logs/
+            name: local-path
+            subPath: logs
+          - mountPath: /data/projects/fate/fate_flow/data/
+            name: local-path
+            subPath: data
+      volumes:
+      - name: service-conf
+        configMap:
+          name: service-conf
+      - name: local-path
+        hostPath:
+          path: /data/projects/fate/fate_flow