Merge pull request #495 from FederatedAI/dev-2.0-beta

v2.0.0-beta

.github/workflows/doc.yml

@@ -4,7 +4,8 @@ on:
   push:
     branches:
       - 'main'
-      - 'develop-[0-9].[0-9].[0-9]'
+      - 'develop-[0-9]+.[0-9]+.[0-9]+'
+      - 'build-doc-[0-9]+.[0-9]+.[0-9]+-[a-zA-Z]+'
 
   schedule:
     - cron: '0 8 * * *'
@@ -41,6 +42,7 @@ jobs:
           VERSION='${{ github.ref_name }}'
           [ "$VERSION" == main ] && { VERSION=latest; ALIAS='main master'; }
           VERSION="${VERSION#develop-}"
+          VERSION="${VERSION#build-doc-}"
 
           mike deploy --push --update-aliases "$VERSION" $ALIAS
           mike set-default --push latest

.gitignore

@@ -17,12 +17,11 @@ venv
 /logs/
 /jobs/
 /audit/
+/localfs/
 .vscode/*
 /temp/
 /tmp
 /worker/
-/provider_registrar/
-/model_local_cache/
 *.db
 *.db-journal
 *.whl
@@ -32,3 +31,14 @@ venv
 
 # doc
 /site/
+
+/python/fate_flow/data
+/python/fate_flow/model
+/python/fate_flow/logs
+/python/fate_flow/jobs
+/python/fate_flow/localfs
+/python/fate_flow/*.env
+/python/fate_flow/conf
+/python/build
+/python/dist
+/python/*.egg-info

README.md

@@ -20,8 +20,6 @@ Providing production-level service capabilities:
 - High Availability
 - CLI, REST API, Python API
 
-For detailed introduction, please refer to [FATE Flow Overall Design](https://federatedai.github.io/FATE-Flow/latest/fate_flow/#overall-design)
-
 ## Deployment
 
 Please refer to [FATE](https://github.com/FederatedAI/FATE)

README.zh.md

@@ -4,7 +4,7 @@
 
 FATE Flow是一个联邦学习端到端全流程的多方联合任务安全调度平台, 基于:
 
-- [共享状态调度架构](https://storage.googleapis.com/pub-tools-public-publication-data/pdf/41684.pdf)
+- 共享状态调度架构
 - 跨数据中心的多方安全通信
 
 提供生产级服务能力:
@@ -20,7 +20,6 @@ FATE Flow是一个联邦学习端到端全流程的多方联合任务安全调
 - 系统高可用
 - CLI、REST API、Python API
 
-详细介绍请参考[FATE Flow整体设计](https://federatedai.github.io/FATE-Flow/latest/zh/fate_flow/)
 
 ## 部署
 

RELEASE.md

@@ -1,3 +1,14 @@
+## Release 2.0.0-beta
+### Major Features and Improvements
+* Migrated functions: data upload/download, process scheduling, component output data/model/metric management, multi-storage adaptation for models, authentication, authorization, feature anonymization, multi-computing/storage/communication engine adaptation, and system high availability
+* Optimized process scheduling, with scheduling separated and customizable, and added priority scheduling
+* Optimized algorithm component scheduling, dividing execution steps into preprocessing, running, and post-processing
+* Optimized multi-version algorithm component registration, supporting registration for mode of components
+* Optimized client authentication logic, supporting permission management for multiple clients
+* Optimized RESTful interface, making parameter fields and types, return fields, and status codes clearer
+* Decoupling the system layer from the algorithm layer, with system configuration moved from the FATE repository to the Flow repository
+* Published FATE Flow package to PyPI and added service-level CLI for service management
+
 ## Release 2.0.0-alpha
 ### Feature Highlights
 * Adapted to new scalable and standardized federated DSL IR

bin/init_env.sh

@@ -18,10 +18,11 @@
 
 fate_project_base=$(cd `dirname "$(realpath "${BASH_SOURCE[0]:-${(%):-%x}}")"`; cd ../;cd ../;pwd)
 export FATE_PROJECT_BASE=$fate_project_base
-export FATE_DEPLOY_BASE=$fate_project_base
+export FATE_PYTHONPATH=
 export EGGROLL_HOME=
 export PYTHONPATH=
 export FATE_ENV=
+export SPARK_HOME=
 
 export FATE_LOG_LEVEL=DEBUG
 

bin/service.sh

@@ -28,7 +28,7 @@ echo "PROJECT_BASE: "${PROJECT_BASE}
 INI_ENV_SCRIPT=${FATE_FLOW_BASE}/bin/init_env.sh
 echo $INI_ENV_SCRIPT
 if test -f "${INI_ENV_SCRIPT}"; then
-  source ${$INI_ENV_SCRIPT}/bin/init_env.sh
+  source $INI_ENV_SCRIPT
   echo "PYTHONPATH: "${PYTHONPATH}
 else
   echo "file not found: ${INI_ENV_SCRIPT}"

conf/casbin_model.conf

@@ -0,0 +1,14 @@
+[request_definition]
+r = sub, obj, act
+
+[policy_definition]
+p = sub, obj, act
+
+[role_definition]
+g = _, _
+
+[policy_effect]
+e = some(where (p.eft == allow))
+
+[matchers]
+m = g(r.sub, p.sub) && r.obj == p.obj && r.act == p.act

conf/job_default_config.yaml

@@ -1,45 +1,27 @@
 # resource
-total_cores_overweight_percent: 1  # 1 means no overweight
-total_memory_overweight_percent: 1  # 1 means no overweight
-task_parallelism: 1
-task_cores: 4
-task_memory: 0  # mb
-max_cores_percent_per_job: 1  # 1 means total
-
-# scheduling
+job_cores: 4
+computing_partitions: 8
+task_run:
+  spark:
+    num-executors: 2
+    executor-cores: 2
+  eggroll:
+    eggroll.session.processors.per.node: 4
+  standalone:
+    cores: 4
 job_timeout: 259200 # s
 remote_request_timeout: 30000  # ms
 federated_command_trys: 3
-end_status_job_scheduling_time_limit: 300000 # ms
-end_status_job_scheduling_updates: 1
-auto_retries: 1
-auto_retry_delay: 1  #seconds
-# It can also be specified in the job configuration using the federated_status_collect_type parameter
-federated_status_collect_type: PUSH
-detect_connect_max_retry_count: 3
-detect_connect_long_retry_count: 2
-
-# upload
-upload_block_max_bytes: 104857600 # bytes
-
-#component output
-output_data_summary_count_limit: 100
-
-task_default_conf:
-  logger:
-    type: flow
-    metadata:
-      level: DEBUG
-      debug_mode: true
-  device:
-    type: CPU
-  output:
-    data:
-      type: directory
-      format: dataframe
-    model:
-      type: directory
-      format: json
-    metric:
-      type: directory
-      format: json
+auto_retries: 0
+sync_type: callback # poll or callback
+task_logger:
+  type: flow
+  metadata:
+    level: DEBUG
+    debug_mode: true
+task_device:
+  type: CPU
+launcher:
+  deepspeed:
+    timeout: 21600 # s
+    world_size: 2

conf/permission_casbin_model.conf

@@ -0,0 +1,11 @@
+[request_definition]
+r = party_id, type, value
+
+[policy_definition]
+p = party_id, type, value
+
+[policy_effect]
+e = some(where (p.eft == allow))
+
+[matchers]
+m = r.party_id == p.party_id && r.type == p.type && r.value == p.value

conf/pulsar_route_table.yaml

@@ -1,14 +1,14 @@
 9999:
   # host can be a domain like 9999.fate.org
-  host: 172.16.153.37
+  host: 127.0.0.1
   port: 6650
   sslPort: 6651
   # set proxy address for this pulsar cluster
   proxy: ""
 
 10000:
   # host can be a domain like 10000.fate.org
-  host: 172.16.153.37
+  host: 127.0.0.1
   port: 6650
   sslPort: 6651
   proxy: ""

conf/service_conf.yaml

@@ -1,24 +1,44 @@
-force_use_sqlite: true
-party_id: "10000"
+party_id: "9999"
+use_registry: false
+encrypt:
+  key_0:
+    module: fate_flow.hub.encrypt.password_encrypt#pwdecrypt
+    # base on: fate_flow/conf/
+    private_path: private_key.pem
 fateflow:
-  # you must set real ip address, 127.0.0.1 and 0.0.0.0 is not supported
   host: 127.0.0.1
   http_port: 9380
   grpc_port: 9360
   proxy_name: rollsite
+  nginx:
+    host:
+    http_port:
+    grpc_port:
 database:
-  name: fate_flow
-  user: fate
-  passwd: fate
-  host: 127.0.0.1
-  port: 3306
-  max_connections: 100
-  stale_timeout: 30
-# engine services
+  engine: sqlite
+  # encrypt passwd key
+  decrypt_key:
+  mysql:
+    name: fate_flow
+    user: fate
+    passwd: fate
+    host: 127.0.0.1
+    port: 3306
+    max_connections: 100
+    stale_timeout: 30
+  sqlite:
+    # default fate_flow/runtime/system_settings: SQLITE_PATH
+    # /xxx/xxx.sqlite
+    path:
 default_engines:
   computing: standalone
   federation: standalone
   storage: standalone
+default_provider:
+  name: fate
+  # version default: fateflow.env
+  version:
+  device: local
 federation:
   pulsar:
     host: 192.168.0.5
@@ -57,27 +77,48 @@ federation:
     port: 9370
 computing:
   standalone:
-    cores_per_node: 20
-    nodes: 1
+    cores: 32
   eggroll:
-    cores_per_node: 16
-    nodes: 1
+    cores: 32
+    nodes: 2
   spark:
     # default use SPARK_HOME environment variable
     home:
-    cores_per_node: 20
-    nodes: 2
-worker:
-  type: native
-  docker:
-    config:
-      # https://docker-py.readthedocs.io/en/stable/client.html#docker.client.DockerClient
-      base_url: unix:///var/run/docker.sock
-    image: ccr.ccs.tencentyun.com/federatedai/fate_algorithm:2.0.0-alpha
-    # on container
-    fate_root_dir: /data/projects/fate
-    # on host
-    eggroll_conf_dir:
-  k8s:
-    image: ccr.ccs.tencentyun.com/federatedai/fate_algorithm:2.0.0-alpha
-    namespace: fate-10000
+    cores: 32
+storage:
+  hdfs:
+    name_node: hdfs://fate-cluster
+hook_module:
+  client_authentication: fate_flow.hook.flow.client_authentication
+  site_authentication: fate_flow.hook.flow.site_authentication
+  permission: fate_flow.hook.flow.permission
+authentication:
+  client: false
+  site: false
+  permission: false
+model_store:
+  engine: file
+  # encrypt passwd key
+  decrypt_key:
+  file:
+    # default fate_flow/runtime/system_settings: MODEL_STORE_PATH
+    path:
+  mysql:
+    name: fate_flow
+    user: fate
+    passwd: fate
+    host: 127.0.0.1
+    port: 3306
+    max_connections: 100
+    stale_timeout: 30
+  tencent_cos:
+    Region:
+    SecretId:
+    SecretKey:
+    Bucket:
+zookeeper:
+  hosts:
+    - 127.0.0.1:2181
+  use_acl: true
+  user: fate
+  password: fate